2024-08-17T16:35:29.154 INFO:root:teuthology version: 1.2.0 2024-08-17T16:35:29.181 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-08-17T16:35:29.296 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858317 branch: main description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - echo wait for mgr daemons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo wait for minority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for majority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for all mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for half of osds to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done first-half-tasks: - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' first_in_suite: false job_id: '7858317' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - mons down - mon down - MON_DOWN - out of quorum - PG_AVAILABILITY sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: main sha1: 91f88e4d132ac08dc53355fa3d9717a53236a3a9 owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - sleep 60 - echo wait for upgrade to complete - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 17 sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: reef clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 11/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 91f88e4d132ac08dc53355fa3d9717a53236a3a9 targets: smithi063.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFMQOcskodZ6lzcpMh2sCE2ajcqjc6FejcbJcdE/+h/F7CBb52+IPv9bckpvH4U3xUSjLzucMSMzvvOY5+jdk4Y= smithi162.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEJRb1Wr6gVlfDs55NiD4IMP+kEKTXh77087CKv9tskUPMVe97QJ5Y8yNqoiwfxuSjErvRH63SMGO5C8HOHFCg0= tasks: - pexec: all: - sudo dnf install runc nvmetcli nvme-cli -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: reef exclude_packages: - ceph-volume - cephadm: compiled_cephadm_branch: reef conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:reef - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 67db276914d29505c9707d04f3a3b8fb9d8aee4b timestamp: 2024-08-17_03:08:15 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-08-17T16:35:29.296 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa; will attempt to use it 2024-08-17T16:35:29.297 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks 2024-08-17T16:35:29.297 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-08-17T16:35:29.298 INFO:teuthology.task.internal:Checking packages... 2024-08-17T16:35:29.322 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453' 2024-08-17T16:35:29.322 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:35:29.322 INFO:teuthology.packaging:ref: None 2024-08-17T16:35:29.322 INFO:teuthology.packaging:tag: None 2024-08-17T16:35:29.322 INFO:teuthology.packaging:branch: main 2024-08-17T16:35:29.322 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:35:29.323 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-08-17T16:35:29.549 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-5842.g4887e20f 2024-08-17T16:35:29.551 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-08-17T16:35:29.557 INFO:teuthology.task.internal:no buildpackages task found 2024-08-17T16:35:29.558 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-08-17T16:35:29.573 INFO:teuthology.task.internal:Saving configuration 2024-08-17T16:35:29.596 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-08-17T16:35:29.604 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-08-17T16:35:29.636 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi063.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858317', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-17 16:30:19.839539', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFMQOcskodZ6lzcpMh2sCE2ajcqjc6FejcbJcdE/+h/F7CBb52+IPv9bckpvH4U3xUSjLzucMSMzvvOY5+jdk4Y='} 2024-08-17T16:35:29.669 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi162.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858317', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-08-17 16:30:19.841202', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEJRb1Wr6gVlfDs55NiD4IMP+kEKTXh77087CKv9tskUPMVe97QJ5Y8yNqoiwfxuSjErvRH63SMGO5C8HOHFCg0='} 2024-08-17T16:35:29.669 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-08-17T16:35:29.675 INFO:teuthology.task.internal:roles: ubuntu@smithi063.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-08-17T16:35:29.675 INFO:teuthology.task.internal:roles: ubuntu@smithi162.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-08-17T16:35:29.675 INFO:teuthology.run_tasks:Running task console_log... 2024-08-17T16:35:29.755 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f9d7ddabac0>, signals=[15]) 2024-08-17T16:35:29.755 INFO:teuthology.run_tasks:Running task internal.connect... 2024-08-17T16:35:29.762 INFO:teuthology.task.internal:Opening connections... 2024-08-17T16:35:29.762 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi063.front.sepia.ceph.com 2024-08-17T16:35:29.763 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi063.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:35:29.852 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi162.front.sepia.ceph.com 2024-08-17T16:35:29.853 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi162.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:35:29.937 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-08-17T16:35:29.944 DEBUG:teuthology.orchestra.run.smithi063:> uname -m 2024-08-17T16:35:29.963 INFO:teuthology.orchestra.run.smithi063.stdout:x86_64 2024-08-17T16:35:29.963 DEBUG:teuthology.orchestra.run.smithi063:> cat /etc/os-release 2024-08-17T16:35:30.021 INFO:teuthology.orchestra.run.smithi063.stdout:NAME="CentOS Stream" 2024-08-17T16:35:30.021 INFO:teuthology.orchestra.run.smithi063.stdout:VERSION="9" 2024-08-17T16:35:30.021 INFO:teuthology.orchestra.run.smithi063.stdout:ID="centos" 2024-08-17T16:35:30.021 INFO:teuthology.orchestra.run.smithi063.stdout:ID_LIKE="rhel fedora" 2024-08-17T16:35:30.021 INFO:teuthology.orchestra.run.smithi063.stdout:VERSION_ID="9" 2024-08-17T16:35:30.022 INFO:teuthology.orchestra.run.smithi063.stdout:PLATFORM_ID="platform:el9" 2024-08-17T16:35:30.022 INFO:teuthology.orchestra.run.smithi063.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-17T16:35:30.022 INFO:teuthology.orchestra.run.smithi063.stdout:ANSI_COLOR="0;31" 2024-08-17T16:35:30.022 INFO:teuthology.orchestra.run.smithi063.stdout:LOGO="fedora-logo-icon" 2024-08-17T16:35:30.022 INFO:teuthology.orchestra.run.smithi063.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-17T16:35:30.022 INFO:teuthology.orchestra.run.smithi063.stdout:HOME_URL="https://centos.org/" 2024-08-17T16:35:30.022 INFO:teuthology.orchestra.run.smithi063.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-08-17T16:35:30.022 INFO:teuthology.orchestra.run.smithi063.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-17T16:35:30.022 INFO:teuthology.orchestra.run.smithi063.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-17T16:35:30.023 INFO:teuthology.lock.ops:Updating smithi063.front.sepia.ceph.com on lock server 2024-08-17T16:35:30.055 DEBUG:teuthology.orchestra.run.smithi162:> uname -m 2024-08-17T16:35:30.073 INFO:teuthology.orchestra.run.smithi162.stdout:x86_64 2024-08-17T16:35:30.073 DEBUG:teuthology.orchestra.run.smithi162:> cat /etc/os-release 2024-08-17T16:35:30.130 INFO:teuthology.orchestra.run.smithi162.stdout:NAME="CentOS Stream" 2024-08-17T16:35:30.130 INFO:teuthology.orchestra.run.smithi162.stdout:VERSION="9" 2024-08-17T16:35:30.130 INFO:teuthology.orchestra.run.smithi162.stdout:ID="centos" 2024-08-17T16:35:30.130 INFO:teuthology.orchestra.run.smithi162.stdout:ID_LIKE="rhel fedora" 2024-08-17T16:35:30.130 INFO:teuthology.orchestra.run.smithi162.stdout:VERSION_ID="9" 2024-08-17T16:35:30.131 INFO:teuthology.orchestra.run.smithi162.stdout:PLATFORM_ID="platform:el9" 2024-08-17T16:35:30.131 INFO:teuthology.orchestra.run.smithi162.stdout:PRETTY_NAME="CentOS Stream 9" 2024-08-17T16:35:30.131 INFO:teuthology.orchestra.run.smithi162.stdout:ANSI_COLOR="0;31" 2024-08-17T16:35:30.131 INFO:teuthology.orchestra.run.smithi162.stdout:LOGO="fedora-logo-icon" 2024-08-17T16:35:30.131 INFO:teuthology.orchestra.run.smithi162.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-08-17T16:35:30.131 INFO:teuthology.orchestra.run.smithi162.stdout:HOME_URL="https://centos.org/" 2024-08-17T16:35:30.131 INFO:teuthology.orchestra.run.smithi162.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-08-17T16:35:30.131 INFO:teuthology.orchestra.run.smithi162.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-08-17T16:35:30.131 INFO:teuthology.orchestra.run.smithi162.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-08-17T16:35:30.131 INFO:teuthology.lock.ops:Updating smithi162.front.sepia.ceph.com on lock server 2024-08-17T16:35:30.163 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-08-17T16:35:30.173 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-08-17T16:35:30.178 INFO:teuthology.task.internal:Checking for old test directory... 2024-08-17T16:35:30.179 DEBUG:teuthology.orchestra.run.smithi063:> test '!' -e /home/ubuntu/cephtest 2024-08-17T16:35:30.181 DEBUG:teuthology.orchestra.run.smithi162:> test '!' -e /home/ubuntu/cephtest 2024-08-17T16:35:30.197 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-08-17T16:35:30.203 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-08-17T16:35:30.204 DEBUG:teuthology.orchestra.run.smithi063:> test -z $(ls -A /var/lib/ceph) 2024-08-17T16:35:30.239 DEBUG:teuthology.orchestra.run.smithi162:> test -z $(ls -A /var/lib/ceph) 2024-08-17T16:35:30.348 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-08-17T16:35:30.420 INFO:teuthology.run_tasks:Running task kernel... 2024-08-17T16:35:30.435 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-08-17T16:35:30.435 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-08-17T16:35:30.435 DEBUG:teuthology.orchestra.run.smithi063:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-17T16:35:30.436 DEBUG:teuthology.orchestra.run.smithi162:> test -f /run/.containerenv -o -f /.dockerenv 2024-08-17T16:35:30.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T16:35:30.452 DEBUG:teuthology.orchestra.run.smithi162:> uname -r 2024-08-17T16:35:30.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T16:35:30.454 DEBUG:teuthology.orchestra.run.smithi063:> uname -r 2024-08-17T16:35:30.510 INFO:teuthology.orchestra.run.smithi162.stdout:5.14.0-437.el9.x86_64 2024-08-17T16:35:30.510 INFO:teuthology.task.kernel:Running kernel on smithi162: 5.14.0-437.el9.x86_64 2024-08-17T16:35:30.511 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum install -y kernel 2024-08-17T16:35:30.512 INFO:teuthology.orchestra.run.smithi063.stdout:5.14.0-437.el9.x86_64 2024-08-17T16:35:30.512 INFO:teuthology.task.kernel:Running kernel on smithi063: 5.14.0-437.el9.x86_64 2024-08-17T16:35:30.512 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum install -y kernel 2024-08-17T16:35:32.771 INFO:teuthology.orchestra.run.smithi063.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.2 MB 00:00 2024-08-17T16:35:32.848 INFO:teuthology.orchestra.run.smithi162.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.2 MB 00:00 2024-08-17T16:35:34.747 INFO:teuthology.orchestra.run.smithi063.stdout:CentOS Stream 9 - AppStream 25 MB/s | 20 MB 00:00 2024-08-17T16:35:38.664 INFO:teuthology.orchestra.run.smithi162.stdout:CentOS Stream 9 - AppStream 4.2 MB/s | 20 MB 00:04 2024-08-17T16:35:39.298 INFO:teuthology.orchestra.run.smithi063.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.6 MB 00:00 2024-08-17T16:35:41.648 INFO:teuthology.orchestra.run.smithi063.stdout:CentOS Stream 9 - Extras packages 18 kB/s | 18 kB 00:01 2024-08-17T16:35:42.885 INFO:teuthology.orchestra.run.smithi063.stdout:Extra Packages for Enterprise Linux 20 MB/s | 22 MB 00:01 2024-08-17T16:35:43.098 INFO:teuthology.orchestra.run.smithi162.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.6 MB 00:00 2024-08-17T16:35:44.881 INFO:teuthology.orchestra.run.smithi162.stdout:CentOS Stream 9 - Extras packages 41 kB/s | 18 kB 00:00 2024-08-17T16:35:46.044 INFO:teuthology.orchestra.run.smithi162.stdout:Extra Packages for Enterprise Linux 22 MB/s | 22 MB 00:01 2024-08-17T16:35:48.558 INFO:teuthology.orchestra.run.smithi063.stdout:lab-extras 34 kB/s | 1.7 kB 00:00 2024-08-17T16:35:50.615 INFO:teuthology.orchestra.run.smithi063.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-17T16:35:50.615 INFO:teuthology.orchestra.run.smithi063.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-08-17T16:35:50.712 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:35:50.727 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:35:50.727 INFO:teuthology.orchestra.run.smithi063.stdout: Package Architecture Version Repository Size 2024-08-17T16:35:50.727 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:35:50.727 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:35:50.727 INFO:teuthology.orchestra.run.smithi063.stdout: kernel x86_64 5.14.0-480.el9 baseos 1.5 M 2024-08-17T16:35:50.727 INFO:teuthology.orchestra.run.smithi063.stdout:Installing dependencies: 2024-08-17T16:35:50.727 INFO:teuthology.orchestra.run.smithi063.stdout: kernel-core x86_64 5.14.0-480.el9 baseos 17 M 2024-08-17T16:35:50.727 INFO:teuthology.orchestra.run.smithi063.stdout: kernel-modules x86_64 5.14.0-480.el9 baseos 36 M 2024-08-17T16:35:50.728 INFO:teuthology.orchestra.run.smithi063.stdout: kernel-modules-core x86_64 5.14.0-480.el9 baseos 30 M 2024-08-17T16:35:50.728 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:35:50.728 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:35:50.728 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:35:50.728 INFO:teuthology.orchestra.run.smithi063.stdout:Install 4 Packages 2024-08-17T16:35:50.728 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:35:50.728 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 84 M 2024-08-17T16:35:50.728 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 126 M 2024-08-17T16:35:50.729 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:35:51.416 INFO:teuthology.orchestra.run.smithi162.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-08-17T16:35:51.476 INFO:teuthology.orchestra.run.smithi063.stdout:(1/4): kernel-5.14.0-480.el9.x86_64.rpm 2.4 MB/s | 1.5 MB 00:00 2024-08-17T16:35:53.397 INFO:teuthology.orchestra.run.smithi162.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-08-17T16:35:53.397 INFO:teuthology.orchestra.run.smithi162.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-08-17T16:35:53.477 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:35:53.484 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:35:53.484 INFO:teuthology.orchestra.run.smithi162.stdout: Package Architecture Version Repository Size 2024-08-17T16:35:53.484 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:35:53.485 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:35:53.485 INFO:teuthology.orchestra.run.smithi162.stdout: kernel x86_64 5.14.0-480.el9 baseos 1.5 M 2024-08-17T16:35:53.485 INFO:teuthology.orchestra.run.smithi162.stdout:Installing dependencies: 2024-08-17T16:35:53.485 INFO:teuthology.orchestra.run.smithi162.stdout: kernel-core x86_64 5.14.0-480.el9 baseos 17 M 2024-08-17T16:35:53.485 INFO:teuthology.orchestra.run.smithi162.stdout: kernel-modules x86_64 5.14.0-480.el9 baseos 36 M 2024-08-17T16:35:53.485 INFO:teuthology.orchestra.run.smithi162.stdout: kernel-modules-core x86_64 5.14.0-480.el9 baseos 30 M 2024-08-17T16:35:53.485 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:35:53.485 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:35:53.485 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:35:53.485 INFO:teuthology.orchestra.run.smithi162.stdout:Install 4 Packages 2024-08-17T16:35:53.485 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:35:53.486 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 84 M 2024-08-17T16:35:53.486 INFO:teuthology.orchestra.run.smithi162.stdout:Installed size: 126 M 2024-08-17T16:35:53.486 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:35:54.092 INFO:teuthology.orchestra.run.smithi063.stdout:(2/4): kernel-core-5.14.0-480.el9.x86_64.rpm 5.3 MB/s | 17 MB 00:03 2024-08-17T16:35:54.358 INFO:teuthology.orchestra.run.smithi162.stdout:(1/4): kernel-5.14.0-480.el9.x86_64.rpm 1.8 MB/s | 1.5 MB 00:00 2024-08-17T16:35:54.934 INFO:teuthology.orchestra.run.smithi162.stdout:(2/4): kernel-core-5.14.0-480.el9.x86_64.rpm 12 MB/s | 17 MB 00:01 2024-08-17T16:35:55.667 INFO:teuthology.orchestra.run.smithi162.stdout:(3/4): kernel-modules-5.14.0-480.el9.x86_64.rpm 17 MB/s | 36 MB 00:02 2024-08-17T16:35:56.026 INFO:teuthology.orchestra.run.smithi063.stdout:(3/4): kernel-modules-5.14.0-480.el9.x86_64.rpm 6.9 MB/s | 36 MB 00:05 2024-08-17T16:35:56.101 INFO:teuthology.orchestra.run.smithi162.stdout:(4/4): kernel-modules-core-5.14.0-480.el9.x86_6 17 MB/s | 30 MB 00:01 2024-08-17T16:35:56.101 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:35:56.101 INFO:teuthology.orchestra.run.smithi162.stdout:Total 32 MB/s | 84 MB 00:02 2024-08-17T16:35:56.835 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:35:56.943 INFO:teuthology.orchestra.run.smithi063.stdout:(4/4): kernel-modules-core-5.14.0-480.el9.x86_6 5.5 MB/s | 30 MB 00:05 2024-08-17T16:35:56.943 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:35:56.943 INFO:teuthology.orchestra.run.smithi063.stdout:Total 14 MB/s | 84 MB 00:06 2024-08-17T16:35:57.169 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:35:57.169 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:35:57.706 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:35:58.096 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:35:58.096 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:35:58.183 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:35:58.184 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:35:59.109 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:35:59.110 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:36:00.622 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:36:01.282 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : kernel-modules-core-5.14.0-480.el9.x86_64 1/4 2024-08-17T16:36:01.375 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:36:01.691 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:36:02.385 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : kernel-modules-core-5.14.0-480.el9.x86_64 1/4 2024-08-17T16:36:02.476 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:36:04.046 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:36:04.192 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:36:05.256 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:36:05.400 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:36:08.998 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:36:09.059 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:36:10.607 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:36:10.668 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:36:13.931 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:36:15.903 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:36:43.939 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:36:43.979 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:36:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:36:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : kernel-5.14.0-480.el9.x86_64 1/4 2024-08-17T16:36:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:36:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:36:45.169 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:36:45.170 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:36:45.170 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:36:45.170 INFO:teuthology.orchestra.run.smithi162.stdout: kernel-5.14.0-480.el9.x86_64 2024-08-17T16:36:45.170 INFO:teuthology.orchestra.run.smithi162.stdout: kernel-core-5.14.0-480.el9.x86_64 2024-08-17T16:36:45.170 INFO:teuthology.orchestra.run.smithi162.stdout: kernel-modules-5.14.0-480.el9.x86_64 2024-08-17T16:36:45.170 INFO:teuthology.orchestra.run.smithi162.stdout: kernel-modules-core-5.14.0-480.el9.x86_64 2024-08-17T16:36:45.170 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:36:45.170 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:36:45.577 DEBUG:teuthology.orchestra.run.smithi162:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T16:36:45.663 INFO:teuthology.orchestra.run.smithi162.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T16:36:45.665 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T16:36:45.665 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-480.el9.x86_64 2024-08-17T16:36:45.665 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-08-17T16:36:45.665 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-08-17T16:36:45.666 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-08-17T16:36:45.666 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi162.front.sepia.ceph.com, path=None, version=distro) 2024-08-17T16:36:45.666 DEBUG:teuthology.orchestra.run.smithi162:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T16:36:45.722 INFO:teuthology.orchestra.run.smithi162.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T16:36:45.722 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T16:36:45.723 DEBUG:teuthology.orchestra.run.smithi162:> sudo rpm -qi grub2-tools 2024-08-17T16:36:45.762 INFO:teuthology.orchestra.run.smithi162.stdout:Name : grub2-tools 2024-08-17T16:36:45.762 INFO:teuthology.orchestra.run.smithi162.stdout:Epoch : 1 2024-08-17T16:36:45.762 INFO:teuthology.orchestra.run.smithi162.stdout:Version : 2.06 2024-08-17T16:36:45.762 INFO:teuthology.orchestra.run.smithi162.stdout:Release : 68.el9 2024-08-17T16:36:45.762 INFO:teuthology.orchestra.run.smithi162.stdout:Architecture: x86_64 2024-08-17T16:36:45.762 INFO:teuthology.orchestra.run.smithi162.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-08-17T16:36:45.762 INFO:teuthology.orchestra.run.smithi162.stdout:Group : Unspecified 2024-08-17T16:36:45.762 INFO:teuthology.orchestra.run.smithi162.stdout:Size : 8264254 2024-08-17T16:36:45.762 INFO:teuthology.orchestra.run.smithi162.stdout:License : GPLv3+ 2024-08-17T16:36:45.762 INFO:teuthology.orchestra.run.smithi162.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-08-17T16:36:45.763 INFO:teuthology.orchestra.run.smithi162.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-08-17T16:36:45.763 INFO:teuthology.orchestra.run.smithi162.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-08-17T16:36:45.763 INFO:teuthology.orchestra.run.smithi162.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-08-17T16:36:45.763 INFO:teuthology.orchestra.run.smithi162.stdout:Packager : builder@centos.org 2024-08-17T16:36:45.763 INFO:teuthology.orchestra.run.smithi162.stdout:Vendor : CentOS 2024-08-17T16:36:45.763 INFO:teuthology.orchestra.run.smithi162.stdout:URL : http://www.gnu.org/software/grub/ 2024-08-17T16:36:45.763 INFO:teuthology.orchestra.run.smithi162.stdout:Summary : Support tools for GRUB. 2024-08-17T16:36:45.763 INFO:teuthology.orchestra.run.smithi162.stdout:Description : 2024-08-17T16:36:45.763 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:36:45.763 INFO:teuthology.orchestra.run.smithi162.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-08-17T16:36:45.763 INFO:teuthology.orchestra.run.smithi162.stdout:customizable bootloader with modular architecture. It supports a rich 2024-08-17T16:36:45.763 INFO:teuthology.orchestra.run.smithi162.stdout:variety of kernel formats, file systems, computer architectures and 2024-08-17T16:36:45.763 INFO:teuthology.orchestra.run.smithi162.stdout:hardware devices. 2024-08-17T16:36:45.764 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:36:45.764 INFO:teuthology.orchestra.run.smithi162.stdout:This subpackage provides tools for support of all platforms. 2024-08-17T16:36:45.764 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-08-17T16:36:45.764 INFO:teuthology.task.kernel:Updating grub on smithi162 to boot 5.14.0-480.el9.x86_64 2024-08-17T16:36:45.765 DEBUG:teuthology.orchestra.run.smithi162:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-08-17T16:36:46.376 INFO:teuthology.orchestra.run.smithi162.stderr:Generating grub configuration file ... 2024-08-17T16:36:47.274 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: kernel-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:36:47.316 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: kernel-modules-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:36:47.774 INFO:teuthology.orchestra.run.smithi162.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-08-17T16:36:47.805 INFO:teuthology.orchestra.run.smithi162.stderr:done 2024-08-17T16:36:47.805 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: kernel-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:36:47.806 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : kernel-5.14.0-480.el9.x86_64 1/4 2024-08-17T16:36:47.806 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : kernel-core-5.14.0-480.el9.x86_64 2/4 2024-08-17T16:36:47.806 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : kernel-modules-5.14.0-480.el9.x86_64 3/4 2024-08-17T16:36:47.807 DEBUG:teuthology.orchestra.run.smithi162:> mktemp 2024-08-17T16:36:47.822 INFO:teuthology.orchestra.run.smithi162.stdout:/tmp/tmp.thz2yoX0cx 2024-08-17T16:36:47.822 DEBUG:teuthology.orchestra.run.smithi162:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.thz2yoX0cx 2024-08-17T16:36:47.889 DEBUG:teuthology.orchestra.run.smithi162:> sudo chmod 0666 /tmp/tmp.thz2yoX0cx 2024-08-17T16:36:48.030 DEBUG:teuthology.orchestra.remote:smithi162:/tmp/tmp.thz2yoX0cx is 6KB 2024-08-17T16:36:48.050 DEBUG:teuthology.orchestra.run.smithi162:> rm -fr /tmp/tmp.thz2yoX0cx 2024-08-17T16:36:48.065 DEBUG:teuthology.orchestra.run.smithi162:> sudo /bin/ls /boot/loader/entries || true 2024-08-17T16:36:48.126 INFO:teuthology.orchestra.run.smithi162.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-08-17T16:36:48.126 INFO:teuthology.orchestra.run.smithi162.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-08-17T16:36:48.126 INFO:teuthology.orchestra.run.smithi162.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-08-17T16:36:48.126 INFO:teuthology.orchestra.run.smithi162.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64.conf 2024-08-17T16:36:48.128 DEBUG:teuthology.orchestra.run.smithi162:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64 2024-08-17T16:36:48.277 DEBUG:teuthology.orchestra.run.smithi162:> sudo shutdown -r now 2024-08-17T16:36:48.595 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : kernel-modules-core-5.14.0-480.el9.x86_64 4/4 2024-08-17T16:36:48.595 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:36:48.595 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:36:48.595 INFO:teuthology.orchestra.run.smithi063.stdout: kernel-5.14.0-480.el9.x86_64 2024-08-17T16:36:48.595 INFO:teuthology.orchestra.run.smithi063.stdout: kernel-core-5.14.0-480.el9.x86_64 2024-08-17T16:36:48.595 INFO:teuthology.orchestra.run.smithi063.stdout: kernel-modules-5.14.0-480.el9.x86_64 2024-08-17T16:36:48.596 INFO:teuthology.orchestra.run.smithi063.stdout: kernel-modules-core-5.14.0-480.el9.x86_64 2024-08-17T16:36:48.596 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:36:48.596 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:36:48.956 DEBUG:teuthology.orchestra.run.smithi063:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T16:36:49.035 INFO:teuthology.orchestra.run.smithi063.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T16:36:49.035 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T16:36:49.036 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-437.el9.x86_64 Expected: 5.14.0-480.el9.x86_64 2024-08-17T16:36:49.036 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-08-17T16:36:49.036 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-08-17T16:36:49.036 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-08-17T16:36:49.036 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi063.front.sepia.ceph.com, path=None, version=distro) 2024-08-17T16:36:49.036 DEBUG:teuthology.orchestra.run.smithi063:> rpm -q kernel | sort -rV | head -n 1 2024-08-17T16:36:49.096 INFO:teuthology.orchestra.run.smithi063.stdout:kernel-5.14.0-480.el9.x86_64 2024-08-17T16:36:49.096 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-480.el9.x86_64 2024-08-17T16:36:49.096 DEBUG:teuthology.orchestra.run.smithi063:> sudo rpm -qi grub2-tools 2024-08-17T16:36:49.128 INFO:teuthology.orchestra.run.smithi063.stdout:Name : grub2-tools 2024-08-17T16:36:49.128 INFO:teuthology.orchestra.run.smithi063.stdout:Epoch : 1 2024-08-17T16:36:49.129 INFO:teuthology.orchestra.run.smithi063.stdout:Version : 2.06 2024-08-17T16:36:49.129 INFO:teuthology.orchestra.run.smithi063.stdout:Release : 68.el9 2024-08-17T16:36:49.129 INFO:teuthology.orchestra.run.smithi063.stdout:Architecture: x86_64 2024-08-17T16:36:49.129 INFO:teuthology.orchestra.run.smithi063.stdout:Install Date: Fri 19 Apr 2024 09:32:39 PM UTC 2024-08-17T16:36:49.129 INFO:teuthology.orchestra.run.smithi063.stdout:Group : Unspecified 2024-08-17T16:36:49.129 INFO:teuthology.orchestra.run.smithi063.stdout:Size : 8264254 2024-08-17T16:36:49.130 INFO:teuthology.orchestra.run.smithi063.stdout:License : GPLv3+ 2024-08-17T16:36:49.130 INFO:teuthology.orchestra.run.smithi063.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-08-17T16:36:49.130 INFO:teuthology.orchestra.run.smithi063.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-08-17T16:36:49.130 INFO:teuthology.orchestra.run.smithi063.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-08-17T16:36:49.130 INFO:teuthology.orchestra.run.smithi063.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-08-17T16:36:49.130 INFO:teuthology.orchestra.run.smithi063.stdout:Packager : builder@centos.org 2024-08-17T16:36:49.130 INFO:teuthology.orchestra.run.smithi063.stdout:Vendor : CentOS 2024-08-17T16:36:49.130 INFO:teuthology.orchestra.run.smithi063.stdout:URL : http://www.gnu.org/software/grub/ 2024-08-17T16:36:49.130 INFO:teuthology.orchestra.run.smithi063.stdout:Summary : Support tools for GRUB. 2024-08-17T16:36:49.130 INFO:teuthology.orchestra.run.smithi063.stdout:Description : 2024-08-17T16:36:49.130 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:36:49.130 INFO:teuthology.orchestra.run.smithi063.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-08-17T16:36:49.131 INFO:teuthology.orchestra.run.smithi063.stdout:customizable bootloader with modular architecture. It supports a rich 2024-08-17T16:36:49.131 INFO:teuthology.orchestra.run.smithi063.stdout:variety of kernel formats, file systems, computer architectures and 2024-08-17T16:36:49.131 INFO:teuthology.orchestra.run.smithi063.stdout:hardware devices. 2024-08-17T16:36:49.131 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:36:49.131 INFO:teuthology.orchestra.run.smithi063.stdout:This subpackage provides tools for support of all platforms. 2024-08-17T16:36:49.132 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-08-17T16:36:49.132 INFO:teuthology.task.kernel:Updating grub on smithi063 to boot 5.14.0-480.el9.x86_64 2024-08-17T16:36:49.132 DEBUG:teuthology.orchestra.run.smithi063:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-08-17T16:36:49.792 INFO:teuthology.orchestra.run.smithi063.stderr:Generating grub configuration file ... 2024-08-17T16:36:51.318 INFO:teuthology.orchestra.run.smithi063.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-08-17T16:36:51.358 INFO:teuthology.orchestra.run.smithi063.stderr:done 2024-08-17T16:36:51.360 DEBUG:teuthology.orchestra.run.smithi063:> mktemp 2024-08-17T16:36:51.381 INFO:teuthology.orchestra.run.smithi063.stdout:/tmp/tmp.wzWDfLKiuJ 2024-08-17T16:36:51.381 DEBUG:teuthology.orchestra.run.smithi063:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.wzWDfLKiuJ 2024-08-17T16:36:51.448 DEBUG:teuthology.orchestra.run.smithi063:> sudo chmod 0666 /tmp/tmp.wzWDfLKiuJ 2024-08-17T16:36:51.582 DEBUG:teuthology.orchestra.remote:smithi063:/tmp/tmp.wzWDfLKiuJ is 6KB 2024-08-17T16:36:51.631 DEBUG:teuthology.orchestra.run.smithi063:> rm -fr /tmp/tmp.wzWDfLKiuJ 2024-08-17T16:36:51.647 DEBUG:teuthology.orchestra.run.smithi063:> sudo /bin/ls /boot/loader/entries || true 2024-08-17T16:36:51.709 INFO:teuthology.orchestra.run.smithi063.stdout:7fc8828e06d0482694bcf4a447d13cb6-0-rescue.conf 2024-08-17T16:36:51.709 INFO:teuthology.orchestra.run.smithi063.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-239.el9.x86_64.conf 2024-08-17T16:36:51.709 INFO:teuthology.orchestra.run.smithi063.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-437.el9.x86_64.conf 2024-08-17T16:36:51.709 INFO:teuthology.orchestra.run.smithi063.stdout:7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64.conf 2024-08-17T16:36:51.710 DEBUG:teuthology.orchestra.run.smithi063:> sudo grub2-set-default 7fc8828e06d0482694bcf4a447d13cb6-5.14.0-480.el9.x86_64 2024-08-17T16:36:51.856 DEBUG:teuthology.orchestra.run.smithi063:> sudo shutdown -r now 2024-08-17T16:37:18.306 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-08-17T16:37:18.307 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi162.front.sepia.ceph.com' 2024-08-17T16:37:18.307 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi162.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:37:21.862 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-08-17T16:37:21.862 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi063.front.sepia.ceph.com' 2024-08-17T16:37:21.863 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi063.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:37:36.866 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.162 2024-08-17T16:37:40.190 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.63 2024-08-17T16:37:45.871 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi162.front.sepia.ceph.com' 2024-08-17T16:37:45.871 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi162.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:37:48.962 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.162 2024-08-17T16:37:49.191 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi063.front.sepia.ceph.com' 2024-08-17T16:37:49.192 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi063.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:37:52.254 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.63 2024-08-17T16:38:00.970 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi162.front.sepia.ceph.com' 2024-08-17T16:38:00.971 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi162.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:38:04.256 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi063.front.sepia.ceph.com' 2024-08-17T16:38:04.256 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi063.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:38:22.686 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.63 2024-08-17T16:38:37.700 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi063.front.sepia.ceph.com' 2024-08-17T16:38:37.700 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi063.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:38:38.641 DEBUG:teuthology.orchestra.run.smithi063:> true 2024-08-17T16:38:39.104 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi063.front.sepia.ceph.com' 2024-08-17T16:38:39.105 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-480.el9.x86_64"... 2024-08-17T16:38:39.105 DEBUG:teuthology.orchestra.run.smithi063:> uname -r 2024-08-17T16:38:39.123 INFO:teuthology.orchestra.run.smithi063.stdout:5.14.0-480.el9.x86_64 2024-08-17T16:38:39.123 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-480.el9.x86_64 vs 5.14.0-480.el9.x86_64 2024-08-17T16:38:39.123 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-08-17T16:38:39.123 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-08-17T16:38:40.124 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-08-17T16:38:40.124 DEBUG:teuthology.orchestra.run.smithi063:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-08-17T16:38:40.238 INFO:teuthology.orchestra.run.smithi063.stdout:ttyS1 2024-08-17T16:38:40.268 DEBUG:teuthology.parallel:result is None 2024-08-17T16:39:00.982 DEBUG:teuthology.orchestra.remote:timed out 2024-08-17T16:39:15.983 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi162.front.sepia.ceph.com' 2024-08-17T16:39:15.984 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi162.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:39:16.615 DEBUG:teuthology.orchestra.run.smithi162:> true 2024-08-17T16:39:16.983 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi162.front.sepia.ceph.com' 2024-08-17T16:39:16.983 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-480.el9.x86_64"... 2024-08-17T16:39:16.983 DEBUG:teuthology.orchestra.run.smithi162:> uname -r 2024-08-17T16:39:16.999 INFO:teuthology.orchestra.run.smithi162.stdout:5.14.0-480.el9.x86_64 2024-08-17T16:39:16.999 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-480.el9.x86_64 vs 5.14.0-480.el9.x86_64 2024-08-17T16:39:16.999 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-08-17T16:39:16.999 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-08-17T16:39:18.000 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-08-17T16:39:18.001 DEBUG:teuthology.orchestra.run.smithi162:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-08-17T16:39:18.068 INFO:teuthology.orchestra.run.smithi162.stdout:ttyS1 2024-08-17T16:39:18.097 DEBUG:teuthology.parallel:result is None 2024-08-17T16:39:18.097 INFO:teuthology.run_tasks:Running task internal.base... 2024-08-17T16:39:18.104 INFO:teuthology.task.internal:Creating test directory... 2024-08-17T16:39:18.104 DEBUG:teuthology.orchestra.run.smithi063:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-17T16:39:18.107 DEBUG:teuthology.orchestra.run.smithi162:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-08-17T16:39:18.125 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-08-17T16:39:18.132 INFO:teuthology.run_tasks:Running task internal.archive... 2024-08-17T16:39:18.139 INFO:teuthology.task.internal:Creating archive directory... 2024-08-17T16:39:18.139 DEBUG:teuthology.orchestra.run.smithi063:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-17T16:39:18.168 DEBUG:teuthology.orchestra.run.smithi162:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-08-17T16:39:18.219 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-08-17T16:39:18.224 INFO:teuthology.task.internal:Enabling coredump saving... 2024-08-17T16:39:18.225 DEBUG:teuthology.orchestra.run.smithi063:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-17T16:39:18.227 DEBUG:teuthology.orchestra.run.smithi162:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-08-17T16:39:18.263 INFO:teuthology.orchestra.run.smithi063.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T16:39:18.275 INFO:teuthology.orchestra.run.smithi063.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T16:39:18.276 INFO:teuthology.orchestra.run.smithi162.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T16:39:18.286 INFO:teuthology.orchestra.run.smithi162.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-08-17T16:39:18.288 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-08-17T16:39:18.295 INFO:teuthology.task.internal:Configuring sudo... 2024-08-17T16:39:18.295 DEBUG:teuthology.orchestra.run.smithi063:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-17T16:39:18.320 DEBUG:teuthology.orchestra.run.smithi162:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-08-17T16:39:18.355 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-08-17T16:39:18.366 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-08-17T16:39:18.366 DEBUG:teuthology.orchestra.run.smithi063:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-17T16:39:18.389 DEBUG:teuthology.orchestra.run.smithi162:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-08-17T16:39:18.412 DEBUG:teuthology.orchestra.run.smithi063:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T16:39:18.497 DEBUG:teuthology.orchestra.run.smithi063:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T16:39:18.542 DEBUG:teuthology.orchestra.run.smithi063:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T16:39:18.634 DEBUG:teuthology.orchestra.run.smithi063:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T16:39:18.669 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T16:39:18.669 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-17T16:39:18.735 DEBUG:teuthology.orchestra.run.smithi162:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T16:39:18.786 DEBUG:teuthology.orchestra.run.smithi162:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-08-17T16:39:18.827 DEBUG:teuthology.orchestra.run.smithi162:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T16:39:18.918 DEBUG:teuthology.orchestra.run.smithi162:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-08-17T16:39:18.952 DEBUG:teuthology.orchestra.run.smithi162:> set -ex 2024-08-17T16:39:18.952 DEBUG:teuthology.orchestra.run.smithi162:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-08-17T16:39:19.017 DEBUG:teuthology.orchestra.run.smithi063:> sudo service rsyslog restart 2024-08-17T16:39:19.019 DEBUG:teuthology.orchestra.run.smithi162:> sudo service rsyslog restart 2024-08-17T16:39:19.074 INFO:teuthology.orchestra.run.smithi063.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T16:39:19.101 INFO:teuthology.orchestra.run.smithi162.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T16:39:19.570 INFO:teuthology.run_tasks:Running task internal.timer... 2024-08-17T16:39:19.582 INFO:teuthology.task.internal:Starting timer... 2024-08-17T16:39:19.582 INFO:teuthology.run_tasks:Running task pcp... 2024-08-17T16:39:19.592 INFO:teuthology.run_tasks:Running task selinux... 2024-08-17T16:39:19.601 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-08-17T16:39:19.602 DEBUG:teuthology.orchestra.run.smithi063:> sudo service auditd rotate 2024-08-17T16:39:19.665 INFO:teuthology.orchestra.run.smithi063.stdout:Rotating logs: 2024-08-17T16:39:19.667 DEBUG:teuthology.orchestra.run.smithi162:> sudo service auditd rotate 2024-08-17T16:39:19.715 INFO:teuthology.orchestra.run.smithi162.stdout:Rotating logs: 2024-08-17T16:39:19.717 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-08-17T16:39:19.718 DEBUG:teuthology.orchestra.run.smithi063:> /usr/sbin/getenforce 2024-08-17T16:39:19.789 INFO:teuthology.orchestra.run.smithi063.stdout:Permissive 2024-08-17T16:39:19.790 DEBUG:teuthology.orchestra.run.smithi162:> /usr/sbin/getenforce 2024-08-17T16:39:19.825 INFO:teuthology.orchestra.run.smithi162.stdout:Permissive 2024-08-17T16:39:19.825 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi063.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi162.front.sepia.ceph.com': 'permissive'} 2024-08-17T16:39:19.825 DEBUG:teuthology.orchestra.run.smithi063:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-08-17T16:39:19.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T16:39:19.858 DEBUG:teuthology.orchestra.run.smithi162:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-08-17T16:39:19.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T16:39:19.891 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-08-17T16:39:19.891 DEBUG:teuthology.orchestra.run.smithi063:> sudo /usr/sbin/setenforce permissive 2024-08-17T16:39:19.933 DEBUG:teuthology.orchestra.run.smithi162:> sudo /usr/sbin/setenforce permissive 2024-08-17T16:39:19.959 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-08-17T16:39:19.972 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-08-17T16:39:19.974 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-08-17T16:39:20.098 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-08-17T16:39:20.109 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-08-17T16:39:20.110 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi063.front.sepia.ceph.com,smithi162.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-08-17T16:43:53.291 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi063.front.sepia.ceph.com'), Remote(name='ubuntu@smithi162.front.sepia.ceph.com')] 2024-08-17T16:43:53.292 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi063.front.sepia.ceph.com' 2024-08-17T16:43:53.293 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi063.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:43:53.394 DEBUG:teuthology.orchestra.run.smithi063:> true 2024-08-17T16:43:53.434 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi063.front.sepia.ceph.com' 2024-08-17T16:43:53.434 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi162.front.sepia.ceph.com' 2024-08-17T16:43:53.435 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi162.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-08-17T16:43:53.519 DEBUG:teuthology.orchestra.run.smithi162:> true 2024-08-17T16:43:53.598 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi162.front.sepia.ceph.com' 2024-08-17T16:43:53.599 INFO:teuthology.run_tasks:Running task clock... 2024-08-17T16:43:53.608 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-08-17T16:43:53.609 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-17T16:43:53.609 DEBUG:teuthology.orchestra.run.smithi063:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T16:43:53.612 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-08-17T16:43:53.612 DEBUG:teuthology.orchestra.run.smithi162:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T16:43:53.642 INFO:teuthology.orchestra.run.smithi063.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-17T16:43:53.658 INFO:teuthology.orchestra.run.smithi063.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-17T16:43:53.669 INFO:teuthology.orchestra.run.smithi162.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-08-17T16:43:53.684 INFO:teuthology.orchestra.run.smithi162.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-08-17T16:43:53.684 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: ntpd: command not found 2024-08-17T16:43:53.696 INFO:teuthology.orchestra.run.smithi063.stdout:506 Cannot talk to daemon 2024-08-17T16:43:53.710 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: ntpd: command not found 2024-08-17T16:43:53.710 INFO:teuthology.orchestra.run.smithi063.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-17T16:43:53.719 INFO:teuthology.orchestra.run.smithi162.stdout:506 Cannot talk to daemon 2024-08-17T16:43:53.725 INFO:teuthology.orchestra.run.smithi063.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-17T16:43:53.733 INFO:teuthology.orchestra.run.smithi162.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-08-17T16:43:53.747 INFO:teuthology.orchestra.run.smithi162.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-08-17T16:43:53.788 INFO:teuthology.orchestra.run.smithi063.stderr:bash: line 1: ntpq: command not found 2024-08-17T16:43:53.792 INFO:teuthology.orchestra.run.smithi063.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T16:43:53.792 INFO:teuthology.orchestra.run.smithi063.stdout:=============================================================================== 2024-08-17T16:43:53.792 INFO:teuthology.orchestra.run.smithi063.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:43:53.792 INFO:teuthology.orchestra.run.smithi063.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:43:53.792 INFO:teuthology.orchestra.run.smithi063.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:43:53.792 INFO:teuthology.orchestra.run.smithi063.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:43:53.810 INFO:teuthology.orchestra.run.smithi162.stderr:bash: line 1: ntpq: command not found 2024-08-17T16:43:53.813 INFO:teuthology.orchestra.run.smithi162.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T16:43:53.814 INFO:teuthology.orchestra.run.smithi162.stdout:=============================================================================== 2024-08-17T16:43:53.814 INFO:teuthology.orchestra.run.smithi162.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:43:53.814 INFO:teuthology.orchestra.run.smithi162.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:43:53.814 INFO:teuthology.orchestra.run.smithi162.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:43:53.814 INFO:teuthology.orchestra.run.smithi162.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T16:43:53.815 INFO:teuthology.run_tasks:Running task pexec... 2024-08-17T16:43:53.824 INFO:teuthology.task.pexec:Executing custom commands... 2024-08-17T16:43:53.824 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi063.front.sepia.ceph.com 2024-08-17T16:43:53.824 DEBUG:teuthology.orchestra.run.smithi063:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-08-17T16:43:53.825 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi162.front.sepia.ceph.com 2024-08-17T16:43:53.825 DEBUG:teuthology.orchestra.run.smithi162:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-08-17T16:43:54.423 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:09 ago on Sat 17 Aug 2024 04:42:45 PM UTC. 2024-08-17T16:43:54.427 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:03 ago on Sat 17 Aug 2024 04:42:51 PM UTC. 2024-08-17T16:43:54.515 INFO:teuthology.orchestra.run.smithi162.stdout:Package nvme-cli-2.6-5.el9.x86_64 is already installed. 2024-08-17T16:43:54.516 INFO:teuthology.orchestra.run.smithi063.stdout:Package nvme-cli-2.6-5.el9.x86_64 is already installed. 2024-08-17T16:43:54.559 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:43:54.574 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:43:54.574 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:43:54.574 INFO:teuthology.orchestra.run.smithi063.stdout: Package Arch Version Repository Size 2024-08-17T16:43:54.574 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:43:54.574 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:43:54.575 INFO:teuthology.orchestra.run.smithi063.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-08-17T16:43:54.575 INFO:teuthology.orchestra.run.smithi063.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-08-17T16:43:54.575 INFO:teuthology.orchestra.run.smithi063.stdout:Upgrading: 2024-08-17T16:43:54.575 INFO:teuthology.orchestra.run.smithi063.stdout: libnvme x86_64 1.9-2.el9 baseos 106 k 2024-08-17T16:43:54.575 INFO:teuthology.orchestra.run.smithi063.stdout: nvme-cli x86_64 2.9.1-3.el9 baseos 903 k 2024-08-17T16:43:54.575 INFO:teuthology.orchestra.run.smithi063.stdout:Installing dependencies: 2024-08-17T16:43:54.575 INFO:teuthology.orchestra.run.smithi063.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-08-17T16:43:54.575 INFO:teuthology.orchestra.run.smithi063.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-08-17T16:43:54.575 INFO:teuthology.orchestra.run.smithi063.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-08-17T16:43:54.575 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:43:54.575 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:43:54.575 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:43:54.576 INFO:teuthology.orchestra.run.smithi063.stdout:Install 5 Packages 2024-08-17T16:43:54.576 INFO:teuthology.orchestra.run.smithi063.stdout:Upgrade 2 Packages 2024-08-17T16:43:54.576 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:43:54.581 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 5.2 M 2024-08-17T16:43:54.581 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:43:54.585 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:43:54.585 INFO:teuthology.orchestra.run.smithi162.stdout: Package Arch Version Repository Size 2024-08-17T16:43:54.585 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:43:54.585 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:43:54.585 INFO:teuthology.orchestra.run.smithi162.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-08-17T16:43:54.586 INFO:teuthology.orchestra.run.smithi162.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-08-17T16:43:54.586 INFO:teuthology.orchestra.run.smithi162.stdout:Upgrading: 2024-08-17T16:43:54.586 INFO:teuthology.orchestra.run.smithi162.stdout: libnvme x86_64 1.9-2.el9 baseos 106 k 2024-08-17T16:43:54.586 INFO:teuthology.orchestra.run.smithi162.stdout: nvme-cli x86_64 2.9.1-3.el9 baseos 903 k 2024-08-17T16:43:54.586 INFO:teuthology.orchestra.run.smithi162.stdout:Installing dependencies: 2024-08-17T16:43:54.586 INFO:teuthology.orchestra.run.smithi162.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-08-17T16:43:54.586 INFO:teuthology.orchestra.run.smithi162.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-08-17T16:43:54.586 INFO:teuthology.orchestra.run.smithi162.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-08-17T16:43:54.586 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:43:54.587 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:43:54.587 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:43:54.587 INFO:teuthology.orchestra.run.smithi162.stdout:Install 5 Packages 2024-08-17T16:43:54.587 INFO:teuthology.orchestra.run.smithi162.stdout:Upgrade 2 Packages 2024-08-17T16:43:54.587 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:43:54.588 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 5.2 M 2024-08-17T16:43:54.588 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:43:54.783 INFO:teuthology.orchestra.run.smithi162.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 486 kB/s | 44 kB 00:00 2024-08-17T16:43:54.808 INFO:teuthology.orchestra.run.smithi162.stdout:(2/7): python3-configshell-1.1.30-1.el9.noarch. 614 kB/s | 72 kB 00:00 2024-08-17T16:43:54.833 INFO:teuthology.orchestra.run.smithi162.stdout:(3/7): python3-kmod-0.9-32.el9.x86_64.rpm 593 kB/s | 84 kB 00:00 2024-08-17T16:43:54.892 INFO:teuthology.orchestra.run.smithi162.stdout:(4/7): libnvme-1.9-2.el9.x86_64.rpm 1.8 MB/s | 106 kB 00:00 2024-08-17T16:43:54.942 INFO:teuthology.orchestra.run.smithi162.stdout:(5/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 5.1 MB/s | 837 kB 00:00 2024-08-17T16:43:55.034 INFO:teuthology.orchestra.run.smithi162.stdout:(6/7): nvme-cli-2.9.1-3.el9.x86_64.rpm 6.2 MB/s | 903 kB 00:00 2024-08-17T16:43:55.272 INFO:teuthology.orchestra.run.smithi063.stdout:(1/7): python3-kmod-0.9-32.el9.x86_64.rpm 236 kB/s | 84 kB 00:00 2024-08-17T16:43:55.297 INFO:teuthology.orchestra.run.smithi063.stdout:(2/7): nvmetcli-0.7-3.el9.noarch.rpm 117 kB/s | 44 kB 00:00 2024-08-17T16:43:55.322 INFO:teuthology.orchestra.run.smithi063.stdout:(3/7): python3-configshell-1.1.30-1.el9.noarch. 176 kB/s | 72 kB 00:00 2024-08-17T16:43:55.648 INFO:teuthology.orchestra.run.smithi063.stdout:(4/7): runc-1.1.13-4.el9.x86_64.rpm 9.0 MB/s | 3.2 MB 00:00 2024-08-17T16:43:55.790 INFO:teuthology.orchestra.run.smithi063.stdout:(5/7): libnvme-1.9-2.el9.x86_64.rpm 227 kB/s | 106 kB 00:00 2024-08-17T16:43:55.860 INFO:teuthology.orchestra.run.smithi162.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 3.0 MB/s | 3.2 MB 00:01 2024-08-17T16:43:55.860 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:43:55.860 INFO:teuthology.orchestra.run.smithi162.stdout:Total 4.1 MB/s | 5.2 MB 00:01 2024-08-17T16:43:55.932 INFO:teuthology.orchestra.run.smithi063.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 1.2 MB/s | 837 kB 00:00 2024-08-17T16:43:55.977 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:43:55.994 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:43:55.994 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:43:56.254 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:43:56.254 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:43:56.324 INFO:teuthology.orchestra.run.smithi063.stdout:(7/7): nvme-cli-2.9.1-3.el9.x86_64.rpm 1.3 MB/s | 903 kB 00:00 2024-08-17T16:43:56.324 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:43:56.324 INFO:teuthology.orchestra.run.smithi063.stdout:Total 3.0 MB/s | 5.2 MB 00:01 2024-08-17T16:43:56.443 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:43:56.462 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:43:56.462 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:43:56.680 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:43:56.702 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:43:56.702 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:43:56.894 INFO:teuthology.orchestra.run.smithi162.stdout: Upgrading : libnvme-1.9-2.el9.x86_64 1/9 2024-08-17T16:43:56.955 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 2/9 2024-08-17T16:43:56.990 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 3/9 2024-08-17T16:43:57.024 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 4/9 2024-08-17T16:43:57.048 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : nvmetcli-0.7-3.el9.noarch 5/9 2024-08-17T16:43:57.133 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:43:57.305 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 5/9 2024-08-17T16:43:57.351 INFO:teuthology.orchestra.run.smithi063.stdout: Upgrading : libnvme-1.9-2.el9.x86_64 1/9 2024-08-17T16:43:57.376 INFO:teuthology.orchestra.run.smithi162.stdout: Upgrading : nvme-cli-2.9.1-3.el9.x86_64 6/9 2024-08-17T16:43:57.421 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 2/9 2024-08-17T16:43:57.455 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 3/9 2024-08-17T16:43:57.488 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 4/9 2024-08-17T16:43:57.512 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : nvmetcli-0.7-3.el9.noarch 5/9 2024-08-17T16:43:57.753 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: nvme-cli-2.9.1-3.el9.x86_64 6/9 2024-08-17T16:43:57.777 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 5/9 2024-08-17T16:43:57.788 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/9 2024-08-17T16:43:57.821 INFO:teuthology.orchestra.run.smithi162.stdout: Cleanup : nvme-cli-2.6-5.el9.x86_64 8/9 2024-08-17T16:43:57.865 INFO:teuthology.orchestra.run.smithi063.stdout: Upgrading : nvme-cli-2.9.1-3.el9.x86_64 6/9 2024-08-17T16:43:57.880 INFO:teuthology.orchestra.run.smithi162.stdout: Cleanup : libnvme-1.6-1.el9.x86_64 9/9 2024-08-17T16:43:58.285 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: nvme-cli-2.9.1-3.el9.x86_64 6/9 2024-08-17T16:43:58.320 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/9 2024-08-17T16:43:58.351 INFO:teuthology.orchestra.run.smithi063.stdout: Cleanup : nvme-cli-2.6-5.el9.x86_64 8/9 2024-08-17T16:43:58.414 INFO:teuthology.orchestra.run.smithi063.stdout: Cleanup : libnvme-1.6-1.el9.x86_64 9/9 2024-08-17T16:43:58.844 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: libnvme-1.6-1.el9.x86_64 9/9 2024-08-17T16:43:58.845 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 1/9 2024-08-17T16:43:58.845 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 2/9 2024-08-17T16:43:58.845 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 3/9 2024-08-17T16:43:58.845 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 4/9 2024-08-17T16:43:58.845 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 5/9 2024-08-17T16:43:58.845 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : libnvme-1.9-2.el9.x86_64 6/9 2024-08-17T16:43:58.845 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : libnvme-1.6-1.el9.x86_64 7/9 2024-08-17T16:43:58.845 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : nvme-cli-2.9.1-3.el9.x86_64 8/9 2024-08-17T16:43:59.212 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : nvme-cli-2.6-5.el9.x86_64 9/9 2024-08-17T16:43:59.213 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:43:59.213 INFO:teuthology.orchestra.run.smithi162.stdout:Upgraded: 2024-08-17T16:43:59.213 INFO:teuthology.orchestra.run.smithi162.stdout: libnvme-1.9-2.el9.x86_64 nvme-cli-2.9.1-3.el9.x86_64 2024-08-17T16:43:59.213 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:43:59.213 INFO:teuthology.orchestra.run.smithi162.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-08-17T16:43:59.213 INFO:teuthology.orchestra.run.smithi162.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-08-17T16:43:59.213 INFO:teuthology.orchestra.run.smithi162.stdout: runc-4:1.1.13-4.el9.x86_64 2024-08-17T16:43:59.213 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:43:59.213 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:43:59.347 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: libnvme-1.6-1.el9.x86_64 9/9 2024-08-17T16:43:59.347 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 1/9 2024-08-17T16:43:59.347 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 2/9 2024-08-17T16:43:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 3/9 2024-08-17T16:43:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 4/9 2024-08-17T16:43:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 5/9 2024-08-17T16:43:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : libnvme-1.9-2.el9.x86_64 6/9 2024-08-17T16:43:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : libnvme-1.6-1.el9.x86_64 7/9 2024-08-17T16:43:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : nvme-cli-2.9.1-3.el9.x86_64 8/9 2024-08-17T16:43:59.599 DEBUG:teuthology.parallel:result is None 2024-08-17T16:43:59.698 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : nvme-cli-2.6-5.el9.x86_64 9/9 2024-08-17T16:43:59.699 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:43:59.699 INFO:teuthology.orchestra.run.smithi063.stdout:Upgraded: 2024-08-17T16:43:59.699 INFO:teuthology.orchestra.run.smithi063.stdout: libnvme-1.9-2.el9.x86_64 nvme-cli-2.9.1-3.el9.x86_64 2024-08-17T16:43:59.699 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:43:59.699 INFO:teuthology.orchestra.run.smithi063.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-08-17T16:43:59.699 INFO:teuthology.orchestra.run.smithi063.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-08-17T16:43:59.699 INFO:teuthology.orchestra.run.smithi063.stdout: runc-4:1.1.13-4.el9.x86_64 2024-08-17T16:43:59.699 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:43:59.699 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:44:00.084 DEBUG:teuthology.parallel:result is None 2024-08-17T16:44:00.084 INFO:teuthology.run_tasks:Running task install... 2024-08-17T16:44:00.093 DEBUG:teuthology.task.install:project ceph 2024-08-17T16:44:00.093 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'}} 2024-08-17T16:44:00.093 DEBUG:teuthology.task.install:config {'branch': 'reef', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'} 2024-08-17T16:44:00.093 INFO:teuthology.task.install:Using flavor: default 2024-08-17T16:44:00.098 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-08-17T16:44:00.098 INFO:teuthology.task.install:extra packages: [] 2024-08-17T16:44:00.098 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453', 'tag': None, 'wait_for_package': False} 2024-08-17T16:44:00.099 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:44:00.099 INFO:teuthology.packaging:ref: None 2024-08-17T16:44:00.099 INFO:teuthology.packaging:tag: None 2024-08-17T16:44:00.099 INFO:teuthology.packaging:branch: reef 2024-08-17T16:44:00.099 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:44:00.099 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-08-17T16:44:00.101 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453', 'tag': None, 'wait_for_package': False} 2024-08-17T16:44:00.101 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:44:00.101 INFO:teuthology.packaging:ref: None 2024-08-17T16:44:00.101 INFO:teuthology.packaging:tag: None 2024-08-17T16:44:00.101 INFO:teuthology.packaging:branch: reef 2024-08-17T16:44:00.101 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:44:00.102 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-08-17T16:44:00.356 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/ 2024-08-17T16:44:00.356 INFO:teuthology.task.install.rpm:Package version is 18.2.4-719.gced649f8 2024-08-17T16:44:00.369 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/ 2024-08-17T16:44:00.369 INFO:teuthology.task.install.rpm:Package version is 18.2.4-719.gced649f8 2024-08-17T16:44:00.476 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-08-17T16:44:00.476 DEBUG:teuthology.orchestra.run.smithi162:> set -ex 2024-08-17T16:44:00.476 DEBUG:teuthology.orchestra.run.smithi162:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-08-17T16:44:00.489 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-08-17T16:44:00.489 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T16:44:00.489 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-08-17T16:44:00.503 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-08-17T16:44:00.503 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:44:00.503 INFO:teuthology.packaging:ref: None 2024-08-17T16:44:00.503 INFO:teuthology.packaging:tag: None 2024-08-17T16:44:00.504 INFO:teuthology.packaging:branch: reef 2024-08-17T16:44:00.504 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:44:00.504 DEBUG:teuthology.orchestra.run.smithi162:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-08-17T16:44:00.517 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-08-17T16:44:00.517 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:44:00.517 INFO:teuthology.packaging:ref: None 2024-08-17T16:44:00.517 INFO:teuthology.packaging:tag: None 2024-08-17T16:44:00.517 INFO:teuthology.packaging:branch: reef 2024-08-17T16:44:00.517 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:44:00.518 DEBUG:teuthology.orchestra.run.smithi063:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-08-17T16:44:00.570 DEBUG:teuthology.orchestra.run.smithi162:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-08-17T16:44:00.584 DEBUG:teuthology.orchestra.run.smithi063:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-08-17T16:44:00.646 DEBUG:teuthology.orchestra.run.smithi162:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-08-17T16:44:00.663 DEBUG:teuthology.orchestra.run.smithi063:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-08-17T16:44:00.710 INFO:teuthology.orchestra.run.smithi162.stdout:check_obsoletes = 1 2024-08-17T16:44:00.711 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum clean all 2024-08-17T16:44:00.728 INFO:teuthology.orchestra.run.smithi063.stdout:check_obsoletes = 1 2024-08-17T16:44:00.730 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum clean all 2024-08-17T16:44:00.938 INFO:teuthology.orchestra.run.smithi162.stdout:45 files removed 2024-08-17T16:44:00.964 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install ceph-radosgw 2024-08-17T16:44:00.968 INFO:teuthology.orchestra.run.smithi063.stdout:45 files removed 2024-08-17T16:44:00.996 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install ceph-radosgw 2024-08-17T16:44:01.552 INFO:teuthology.orchestra.run.smithi162.stdout:ceph packages for x86_64 252 kB/s | 77 kB 00:00 2024-08-17T16:44:01.616 INFO:teuthology.orchestra.run.smithi063.stdout:ceph packages for x86_64 235 kB/s | 77 kB 00:00 2024-08-17T16:44:01.886 INFO:teuthology.orchestra.run.smithi162.stdout:ceph noarch packages 40 kB/s | 11 kB 00:00 2024-08-17T16:44:01.959 INFO:teuthology.orchestra.run.smithi063.stdout:ceph noarch packages 40 kB/s | 11 kB 00:00 2024-08-17T16:44:02.219 INFO:teuthology.orchestra.run.smithi162.stdout:ceph source packages 6.5 kB/s | 1.9 kB 00:00 2024-08-17T16:44:02.284 INFO:teuthology.orchestra.run.smithi063.stdout:ceph source packages 6.8 kB/s | 1.9 kB 00:00 2024-08-17T16:44:03.786 INFO:teuthology.orchestra.run.smithi162.stdout:CentOS Stream 9 - BaseOS 5.4 MB/s | 8.2 MB 00:01 2024-08-17T16:44:05.559 INFO:teuthology.orchestra.run.smithi063.stdout:CentOS Stream 9 - BaseOS 2.5 MB/s | 8.2 MB 00:03 2024-08-17T16:44:07.003 INFO:teuthology.orchestra.run.smithi162.stdout:CentOS Stream 9 - AppStream 9.4 MB/s | 20 MB 00:02 2024-08-17T16:44:07.777 INFO:teuthology.orchestra.run.smithi063.stdout:CentOS Stream 9 - AppStream 19 MB/s | 20 MB 00:01 2024-08-17T16:44:11.329 INFO:teuthology.orchestra.run.smithi162.stdout:CentOS Stream 9 - CRB 12 MB/s | 6.6 MB 00:00 2024-08-17T16:44:12.821 INFO:teuthology.orchestra.run.smithi162.stdout:CentOS Stream 9 - Extras packages 73 kB/s | 18 kB 00:00 2024-08-17T16:44:13.644 INFO:teuthology.orchestra.run.smithi063.stdout:CentOS Stream 9 - CRB 3.3 MB/s | 6.6 MB 00:01 2024-08-17T16:44:14.209 INFO:teuthology.orchestra.run.smithi162.stdout:Extra Packages for Enterprise Linux 18 MB/s | 22 MB 00:01 2024-08-17T16:44:15.228 INFO:teuthology.orchestra.run.smithi063.stdout:CentOS Stream 9 - Extras packages 67 kB/s | 18 kB 00:00 2024-08-17T16:44:16.699 INFO:teuthology.orchestra.run.smithi063.stdout:Extra Packages for Enterprise Linux 17 MB/s | 22 MB 00:01 2024-08-17T16:44:19.556 INFO:teuthology.orchestra.run.smithi162.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-08-17T16:44:21.874 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:44:21.876 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:44:21.876 INFO:teuthology.orchestra.run.smithi162.stdout: Package Arch Version Repository Size 2024-08-17T16:44:21.876 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:44:21.877 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:44:21.877 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-radosgw x86_64 2:18.2.4-719.gced649f8.el9 ceph 7.7 M 2024-08-17T16:44:21.877 INFO:teuthology.orchestra.run.smithi162.stdout:Upgrading: 2024-08-17T16:44:21.877 INFO:teuthology.orchestra.run.smithi162.stdout: librados2 x86_64 2:18.2.4-719.gced649f8.el9 ceph 3.3 M 2024-08-17T16:44:21.877 INFO:teuthology.orchestra.run.smithi162.stdout: librbd1 x86_64 2:18.2.4-719.gced649f8.el9 ceph 3.0 M 2024-08-17T16:44:21.877 INFO:teuthology.orchestra.run.smithi162.stdout:Installing dependencies: 2024-08-17T16:44:21.877 INFO:teuthology.orchestra.run.smithi162.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-08-17T16:44:21.877 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-base x86_64 2:18.2.4-719.gced649f8.el9 ceph 5.1 M 2024-08-17T16:44:21.877 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-common x86_64 2:18.2.4-719.gced649f8.el9 ceph 18 M 2024-08-17T16:44:21.877 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-selinux x86_64 2:18.2.4-719.gced649f8.el9 ceph 25 k 2024-08-17T16:44:21.877 INFO:teuthology.orchestra.run.smithi162.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-08-17T16:44:21.877 INFO:teuthology.orchestra.run.smithi162.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-08-17T16:44:21.878 INFO:teuthology.orchestra.run.smithi162.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-08-17T16:44:21.878 INFO:teuthology.orchestra.run.smithi162.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-08-17T16:44:21.878 INFO:teuthology.orchestra.run.smithi162.stdout: libcephfs2 x86_64 2:18.2.4-719.gced649f8.el9 ceph 712 k 2024-08-17T16:44:21.878 INFO:teuthology.orchestra.run.smithi162.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-08-17T16:44:21.878 INFO:teuthology.orchestra.run.smithi162.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-08-17T16:44:21.878 INFO:teuthology.orchestra.run.smithi162.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-08-17T16:44:21.878 INFO:teuthology.orchestra.run.smithi162.stdout: libradosstriper1 x86_64 2:18.2.4-719.gced649f8.el9 ceph 477 k 2024-08-17T16:44:21.878 INFO:teuthology.orchestra.run.smithi162.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-08-17T16:44:21.878 INFO:teuthology.orchestra.run.smithi162.stdout: librgw2 x86_64 2:18.2.4-719.gced649f8.el9 ceph 4.5 M 2024-08-17T16:44:21.878 INFO:teuthology.orchestra.run.smithi162.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-08-17T16:44:21.878 INFO:teuthology.orchestra.run.smithi162.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-08-17T16:44:21.878 INFO:teuthology.orchestra.run.smithi162.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-08-17T16:44:21.879 INFO:teuthology.orchestra.run.smithi162.stdout: python3-ceph-argparse x86_64 2:18.2.4-719.gced649f8.el9 ceph 45 k 2024-08-17T16:44:21.879 INFO:teuthology.orchestra.run.smithi162.stdout: python3-ceph-common x86_64 2:18.2.4-719.gced649f8.el9 ceph 129 k 2024-08-17T16:44:21.879 INFO:teuthology.orchestra.run.smithi162.stdout: python3-cephfs x86_64 2:18.2.4-719.gced649f8.el9 ceph 188 k 2024-08-17T16:44:21.879 INFO:teuthology.orchestra.run.smithi162.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-08-17T16:44:21.879 INFO:teuthology.orchestra.run.smithi162.stdout: python3-rados x86_64 2:18.2.4-719.gced649f8.el9 ceph 372 k 2024-08-17T16:44:21.879 INFO:teuthology.orchestra.run.smithi162.stdout: python3-rbd x86_64 2:18.2.4-719.gced649f8.el9 ceph 347 k 2024-08-17T16:44:21.879 INFO:teuthology.orchestra.run.smithi162.stdout: python3-rgw x86_64 2:18.2.4-719.gced649f8.el9 ceph 116 k 2024-08-17T16:44:21.879 INFO:teuthology.orchestra.run.smithi162.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-08-17T16:44:21.879 INFO:teuthology.orchestra.run.smithi162.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-08-17T16:44:21.879 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:44:21.879 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:44:21.879 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:44:21.880 INFO:teuthology.orchestra.run.smithi162.stdout:Install 28 Packages 2024-08-17T16:44:21.880 INFO:teuthology.orchestra.run.smithi162.stdout:Upgrade 2 Packages 2024-08-17T16:44:21.880 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:44:21.880 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 53 M 2024-08-17T16:44:21.880 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:44:22.413 INFO:teuthology.orchestra.run.smithi063.stdout:lab-extras 41 kB/s | 1.7 kB 00:00 2024-08-17T16:44:22.507 INFO:teuthology.orchestra.run.smithi162.stdout:(1/30): ceph-base-18.2.4-719.gced649f8.el9.x86_ 10 MB/s | 5.1 MB 00:00 2024-08-17T16:44:22.565 INFO:teuthology.orchestra.run.smithi162.stdout:(2/30): ceph-selinux-18.2.4-719.gced649f8.el9.x 433 kB/s | 25 kB 00:00 2024-08-17T16:44:22.657 INFO:teuthology.orchestra.run.smithi162.stdout:(3/30): libcephfs2-18.2.4-719.gced649f8.el9.x86 7.6 MB/s | 712 kB 00:00 2024-08-17T16:44:22.758 INFO:teuthology.orchestra.run.smithi162.stdout:(4/30): ceph-radosgw-18.2.4-719.gced649f8.el9.x 10 MB/s | 7.7 MB 00:00 2024-08-17T16:44:22.783 INFO:teuthology.orchestra.run.smithi162.stdout:(5/30): libradosstriper1-18.2.4-719.gced649f8.e 3.7 MB/s | 477 kB 00:00 2024-08-17T16:44:22.841 INFO:teuthology.orchestra.run.smithi162.stdout:(6/30): python3-ceph-argparse-18.2.4-719.gced64 773 kB/s | 45 kB 00:00 2024-08-17T16:44:22.917 INFO:teuthology.orchestra.run.smithi162.stdout:(7/30): python3-ceph-common-18.2.4-719.gced649f 1.7 MB/s | 129 kB 00:00 2024-08-17T16:44:23.000 INFO:teuthology.orchestra.run.smithi162.stdout:(8/30): librgw2-18.2.4-719.gced649f8.el9.x86_64 18 MB/s | 4.5 MB 00:00 2024-08-17T16:44:23.026 INFO:teuthology.orchestra.run.smithi162.stdout:(9/30): python3-cephfs-18.2.4-719.gced649f8.el9 1.7 MB/s | 188 kB 00:00 2024-08-17T16:44:23.051 INFO:teuthology.orchestra.run.smithi162.stdout:(10/30): python3-rados-18.2.4-719.gced649f8.el9 7.2 MB/s | 372 kB 00:00 2024-08-17T16:44:23.076 INFO:teuthology.orchestra.run.smithi162.stdout:(11/30): python3-rbd-18.2.4-719.gced649f8.el9.x 6.8 MB/s | 347 kB 00:00 2024-08-17T16:44:23.102 INFO:teuthology.orchestra.run.smithi162.stdout:(12/30): python3-rgw-18.2.4-719.gced649f8.el9.x 2.3 MB/s | 116 kB 00:00 2024-08-17T16:44:23.277 INFO:teuthology.orchestra.run.smithi162.stdout:(13/30): ceph-common-18.2.4-719.gced649f8.el9.x 15 MB/s | 18 MB 00:01 2024-08-17T16:44:23.344 INFO:teuthology.orchestra.run.smithi162.stdout:(14/30): boost-program-options-1.75.0-8.el9.x86 400 kB/s | 107 kB 00:00 2024-08-17T16:44:23.369 INFO:teuthology.orchestra.run.smithi162.stdout:(15/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 600 kB/s | 160 kB 00:00 2024-08-17T16:44:23.394 INFO:teuthology.orchestra.run.smithi162.stdout:(16/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 387 kB/s | 45 kB 00:00 2024-08-17T16:44:23.453 INFO:teuthology.orchestra.run.smithi162.stdout:(17/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.4 MB/s | 292 kB 00:00 2024-08-17T16:44:23.478 INFO:teuthology.orchestra.run.smithi162.stdout:(18/30): python3-prettytable-0.7.2-27.el9.noarc 506 kB/s | 42 kB 00:00 2024-08-17T16:44:23.503 INFO:teuthology.orchestra.run.smithi162.stdout:(19/30): librdkafka-1.6.1-102.el9.x86_64.rpm 4.1 MB/s | 662 kB 00:00 2024-08-17T16:44:23.612 INFO:teuthology.orchestra.run.smithi162.stdout:(20/30): fmt-8.1.1-5.el9.x86_64.rpm 698 kB/s | 111 kB 00:00 2024-08-17T16:44:23.637 INFO:teuthology.orchestra.run.smithi162.stdout:(21/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 1.1 MB/s | 26 kB 00:00 2024-08-17T16:44:23.662 INFO:teuthology.orchestra.run.smithi162.stdout:(22/30): gperftools-libs-2.9.1-2.el9.x86_64.rpm 1.6 MB/s | 309 kB 00:00 2024-08-17T16:44:23.688 INFO:teuthology.orchestra.run.smithi162.stdout:(23/30): liboath-2.6.7-2.el9.x86_64.rpm 969 kB/s | 49 kB 00:00 2024-08-17T16:44:23.705 INFO:teuthology.orchestra.run.smithi162.stdout:(24/30): libunwind-1.6.2-1.el9.x86_64.rpm 1.6 MB/s | 67 kB 00:00 2024-08-17T16:44:23.747 INFO:teuthology.orchestra.run.smithi162.stdout:(25/30): re2-20211101-3.el9.x86_64.rpm 4.5 MB/s | 192 kB 00:00 2024-08-17T16:44:23.789 INFO:teuthology.orchestra.run.smithi162.stdout:(26/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 8.2 MB/s | 839 kB 00:00 2024-08-17T16:44:23.855 INFO:teuthology.orchestra.run.smithi162.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 15 MB/s | 1.6 MB 00:00 2024-08-17T16:44:23.973 INFO:teuthology.orchestra.run.smithi162.stdout:(28/30): librados2-18.2.4-719.gced649f8.el9.x86 18 MB/s | 3.3 MB 00:00 2024-08-17T16:44:24.039 INFO:teuthology.orchestra.run.smithi162.stdout:(29/30): libarrow-9.0.0-11.el9.x86_64.rpm 9.1 MB/s | 4.4 MB 00:00 2024-08-17T16:44:24.098 INFO:teuthology.orchestra.run.smithi162.stdout:(30/30): librbd1-18.2.4-719.gced649f8.el9.x86_6 12 MB/s | 3.0 MB 00:00 2024-08-17T16:44:24.102 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:44:24.103 INFO:teuthology.orchestra.run.smithi162.stdout:Total 24 MB/s | 53 MB 00:02 2024-08-17T16:44:24.171 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:44:24.206 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:44:24.206 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:44:24.656 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:44:24.656 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:44:24.841 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:44:24.843 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:44:24.843 INFO:teuthology.orchestra.run.smithi063.stdout: Package Arch Version Repository Size 2024-08-17T16:44:24.843 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:44:24.843 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:44:24.843 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-radosgw x86_64 2:18.2.4-719.gced649f8.el9 ceph 7.7 M 2024-08-17T16:44:24.843 INFO:teuthology.orchestra.run.smithi063.stdout:Upgrading: 2024-08-17T16:44:24.843 INFO:teuthology.orchestra.run.smithi063.stdout: librados2 x86_64 2:18.2.4-719.gced649f8.el9 ceph 3.3 M 2024-08-17T16:44:24.844 INFO:teuthology.orchestra.run.smithi063.stdout: librbd1 x86_64 2:18.2.4-719.gced649f8.el9 ceph 3.0 M 2024-08-17T16:44:24.844 INFO:teuthology.orchestra.run.smithi063.stdout:Installing dependencies: 2024-08-17T16:44:24.844 INFO:teuthology.orchestra.run.smithi063.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-08-17T16:44:24.844 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-base x86_64 2:18.2.4-719.gced649f8.el9 ceph 5.1 M 2024-08-17T16:44:24.844 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-common x86_64 2:18.2.4-719.gced649f8.el9 ceph 18 M 2024-08-17T16:44:24.844 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-selinux x86_64 2:18.2.4-719.gced649f8.el9 ceph 25 k 2024-08-17T16:44:24.844 INFO:teuthology.orchestra.run.smithi063.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-08-17T16:44:24.844 INFO:teuthology.orchestra.run.smithi063.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-08-17T16:44:24.844 INFO:teuthology.orchestra.run.smithi063.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-08-17T16:44:24.844 INFO:teuthology.orchestra.run.smithi063.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-08-17T16:44:24.844 INFO:teuthology.orchestra.run.smithi063.stdout: libcephfs2 x86_64 2:18.2.4-719.gced649f8.el9 ceph 712 k 2024-08-17T16:44:24.844 INFO:teuthology.orchestra.run.smithi063.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-08-17T16:44:24.844 INFO:teuthology.orchestra.run.smithi063.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-08-17T16:44:24.845 INFO:teuthology.orchestra.run.smithi063.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-08-17T16:44:24.845 INFO:teuthology.orchestra.run.smithi063.stdout: libradosstriper1 x86_64 2:18.2.4-719.gced649f8.el9 ceph 477 k 2024-08-17T16:44:24.845 INFO:teuthology.orchestra.run.smithi063.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-08-17T16:44:24.845 INFO:teuthology.orchestra.run.smithi063.stdout: librgw2 x86_64 2:18.2.4-719.gced649f8.el9 ceph 4.5 M 2024-08-17T16:44:24.845 INFO:teuthology.orchestra.run.smithi063.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-08-17T16:44:24.845 INFO:teuthology.orchestra.run.smithi063.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-08-17T16:44:24.845 INFO:teuthology.orchestra.run.smithi063.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-08-17T16:44:24.845 INFO:teuthology.orchestra.run.smithi063.stdout: python3-ceph-argparse x86_64 2:18.2.4-719.gced649f8.el9 ceph 45 k 2024-08-17T16:44:24.845 INFO:teuthology.orchestra.run.smithi063.stdout: python3-ceph-common x86_64 2:18.2.4-719.gced649f8.el9 ceph 129 k 2024-08-17T16:44:24.845 INFO:teuthology.orchestra.run.smithi063.stdout: python3-cephfs x86_64 2:18.2.4-719.gced649f8.el9 ceph 188 k 2024-08-17T16:44:24.845 INFO:teuthology.orchestra.run.smithi063.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-08-17T16:44:24.845 INFO:teuthology.orchestra.run.smithi063.stdout: python3-rados x86_64 2:18.2.4-719.gced649f8.el9 ceph 372 k 2024-08-17T16:44:24.846 INFO:teuthology.orchestra.run.smithi063.stdout: python3-rbd x86_64 2:18.2.4-719.gced649f8.el9 ceph 347 k 2024-08-17T16:44:24.846 INFO:teuthology.orchestra.run.smithi063.stdout: python3-rgw x86_64 2:18.2.4-719.gced649f8.el9 ceph 116 k 2024-08-17T16:44:24.846 INFO:teuthology.orchestra.run.smithi063.stdout: re2 x86_64 1:20211101-3.el9 epel 192 k 2024-08-17T16:44:24.846 INFO:teuthology.orchestra.run.smithi063.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-08-17T16:44:24.846 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:44:24.846 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:44:24.846 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:44:24.846 INFO:teuthology.orchestra.run.smithi063.stdout:Install 28 Packages 2024-08-17T16:44:24.846 INFO:teuthology.orchestra.run.smithi063.stdout:Upgrade 2 Packages 2024-08-17T16:44:24.846 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:44:24.850 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 53 M 2024-08-17T16:44:24.850 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:44:25.798 INFO:teuthology.orchestra.run.smithi063.stdout:(1/30): ceph-base-18.2.4-719.gced649f8.el9.x86_ 7.4 MB/s | 5.1 MB 00:00 2024-08-17T16:44:25.848 INFO:teuthology.orchestra.run.smithi063.stdout:(2/30): ceph-selinux-18.2.4-719.gced649f8.el9.x 504 kB/s | 25 kB 00:00 2024-08-17T16:44:25.973 INFO:teuthology.orchestra.run.smithi063.stdout:(3/30): ceph-radosgw-18.2.4-719.gced649f8.el9.x 9.0 MB/s | 7.7 MB 00:00 2024-08-17T16:44:25.999 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:44:26.091 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-08-17T16:44:26.128 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-08-17T16:44:26.149 INFO:teuthology.orchestra.run.smithi063.stdout:(4/30): ceph-common-18.2.4-719.gced649f8.el9.x8 18 MB/s | 18 MB 00:01 2024-08-17T16:44:26.174 INFO:teuthology.orchestra.run.smithi063.stdout:(5/30): libcephfs2-18.2.4-719.gced649f8.el9.x86 2.1 MB/s | 712 kB 00:00 2024-08-17T16:44:26.186 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-08-17T16:44:26.207 INFO:teuthology.orchestra.run.smithi063.stdout:(6/30): libradosstriper1-18.2.4-719.gced649f8.e 2.0 MB/s | 477 kB 00:00 2024-08-17T16:44:26.224 INFO:teuthology.orchestra.run.smithi063.stdout:(7/30): python3-ceph-argparse-18.2.4-719.gced64 896 kB/s | 45 kB 00:00 2024-08-17T16:44:26.258 INFO:teuthology.orchestra.run.smithi063.stdout:(8/30): python3-ceph-common-18.2.4-719.gced649f 2.5 MB/s | 129 kB 00:00 2024-08-17T16:44:26.283 INFO:teuthology.orchestra.run.smithi063.stdout:(9/30): python3-cephfs-18.2.4-719.gced649f8.el9 3.1 MB/s | 188 kB 00:00 2024-08-17T16:44:26.308 INFO:teuthology.orchestra.run.smithi063.stdout:(10/30): python3-rados-18.2.4-719.gced649f8.el9 7.3 MB/s | 372 kB 00:00 2024-08-17T16:44:26.333 INFO:teuthology.orchestra.run.smithi063.stdout:(11/30): python3-rbd-18.2.4-719.gced649f8.el9.x 6.8 MB/s | 347 kB 00:00 2024-08-17T16:44:26.359 INFO:teuthology.orchestra.run.smithi063.stdout:(12/30): python3-rgw-18.2.4-719.gced649f8.el9.x 2.3 MB/s | 116 kB 00:00 2024-08-17T16:44:26.426 INFO:teuthology.orchestra.run.smithi063.stdout:(13/30): librgw2-18.2.4-719.gced649f8.el9.x86_6 16 MB/s | 4.5 MB 00:00 2024-08-17T16:44:26.511 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-08-17T16:44:26.619 INFO:teuthology.orchestra.run.smithi162.stdout: Upgrading : librados2-2:18.2.4-719.gced649f8.el9.x86_64 5/32 2024-08-17T16:44:26.723 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: librados2-2:18.2.4-719.gced649f8.el9.x86_64 5/32 2024-08-17T16:44:26.803 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 6/32 2024-08-17T16:44:26.828 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 7/32 2024-08-17T16:44:26.874 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 7/32 2024-08-17T16:44:26.895 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-08-17T16:44:26.940 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-ceph-argparse-2:18.2.4-719.gced649f8.el9.x 9/32 2024-08-17T16:44:27.004 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 10/32 2024-08-17T16:44:27.030 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 11/32 2024-08-17T16:44:27.042 INFO:teuthology.orchestra.run.smithi063.stdout:(14/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 73 kB/s | 45 kB 00:00 2024-08-17T16:44:27.085 INFO:teuthology.orchestra.run.smithi063.stdout:(15/30): boost-program-options-1.75.0-8.el9.x86 142 kB/s | 107 kB 00:00 2024-08-17T16:44:27.096 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 11/32 2024-08-17T16:44:27.126 INFO:teuthology.orchestra.run.smithi063.stdout:(16/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 209 kB/s | 160 kB 00:00 2024-08-17T16:44:27.203 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : re2-1:20211101-3.el9.x86_64 12/32 2024-08-17T16:44:27.237 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-08-17T16:44:27.279 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-08-17T16:44:27.311 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/32 2024-08-17T16:44:27.338 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-08-17T16:44:27.374 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-08-17T16:44:27.406 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-08-17T16:44:27.706 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-08-17T16:44:27.752 INFO:teuthology.orchestra.run.smithi063.stdout:(17/30): python3-prettytable-0.7.2-27.el9.noarc 68 kB/s | 42 kB 00:00 2024-08-17T16:44:27.752 INFO:teuthology.orchestra.run.smithi162.stdout: Upgrading : librbd1-2:18.2.4-719.gced649f8.el9.x86_64 20/32 2024-08-17T16:44:27.819 INFO:teuthology.orchestra.run.smithi063.stdout:(18/30): fmt-8.1.1-5.el9.x86_64.rpm 1.6 MB/s | 111 kB 00:00 2024-08-17T16:44:27.846 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: librbd1-2:18.2.4-719.gced649f8.el9.x86_64 20/32 2024-08-17T16:44:27.852 INFO:teuthology.orchestra.run.smithi063.stdout:(19/30): gperftools-libs-2.9.1-2.el9.x86_64.rpm 9.1 MB/s | 309 kB 00:00 2024-08-17T16:44:27.874 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 21/32 2024-08-17T16:44:27.910 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-08-17T16:44:28.044 INFO:teuthology.orchestra.run.smithi063.stdout:(20/30): libarrow-9.0.0-11.el9.x86_64.rpm 23 MB/s | 4.4 MB 00:00 2024-08-17T16:44:28.069 INFO:teuthology.orchestra.run.smithi063.stdout:(21/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 297 kB/s | 292 kB 00:00 2024-08-17T16:44:28.095 INFO:teuthology.orchestra.run.smithi063.stdout:(22/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 519 kB/s | 26 kB 00:00 2024-08-17T16:44:28.153 INFO:teuthology.orchestra.run.smithi063.stdout:(23/30): libunwind-1.6.2-1.el9.x86_64.rpm 2.2 MB/s | 67 kB 00:00 2024-08-17T16:44:28.204 INFO:teuthology.orchestra.run.smithi063.stdout:(24/30): liboath-2.6.7-2.el9.x86_64.rpm 363 kB/s | 49 kB 00:00 2024-08-17T16:44:28.245 INFO:teuthology.orchestra.run.smithi063.stdout:(25/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 9.2 MB/s | 839 kB 00:00 2024-08-17T16:44:28.271 INFO:teuthology.orchestra.run.smithi063.stdout:(26/30): re2-20211101-3.el9.x86_64.rpm 2.8 MB/s | 192 kB 00:00 2024-08-17T16:44:28.313 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-08-17T16:44:28.329 INFO:teuthology.orchestra.run.smithi063.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 19 MB/s | 1.6 MB 00:00 2024-08-17T16:44:28.342 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : librgw2-2:18.2.4-719.gced649f8.el9.x86_64 24/32 2024-08-17T16:44:28.403 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: librgw2-2:18.2.4-719.gced649f8.el9.x86_64 24/32 2024-08-17T16:44:28.480 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 25/32 2024-08-17T16:44:28.508 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-ceph-common-2:18.2.4-719.gced649f8.el9.x86 26/32 2024-08-17T16:44:28.513 INFO:teuthology.orchestra.run.smithi063.stdout:(28/30): librdkafka-1.6.1-102.el9.x86_64.rpm 451 kB/s | 662 kB 00:01 2024-08-17T16:44:28.580 INFO:teuthology.orchestra.run.smithi063.stdout:(29/30): librados2-18.2.4-719.gced649f8.el9.x86 11 MB/s | 3.3 MB 00:00 2024-08-17T16:44:28.638 INFO:teuthology.orchestra.run.smithi063.stdout:(30/30): librbd1-18.2.4-719.gced649f8.el9.x86_6 9.8 MB/s | 3.0 MB 00:00 2024-08-17T16:44:28.643 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:44:28.643 INFO:teuthology.orchestra.run.smithi063.stdout:Total 14 MB/s | 53 MB 00:03 2024-08-17T16:44:28.720 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:44:28.762 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:44:28.762 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:44:29.230 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:44:29.230 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:44:30.583 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:44:30.611 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 27/32 2024-08-17T16:44:30.638 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 27/32 2024-08-17T16:44:30.683 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-08-17T16:44:30.721 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-08-17T16:44:30.782 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-08-17T16:44:31.149 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-08-17T16:44:31.168 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 27/32 2024-08-17T16:44:31.224 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 28/32 2024-08-17T16:44:31.245 INFO:teuthology.orchestra.run.smithi063.stdout: Upgrading : librados2-2:18.2.4-719.gced649f8.el9.x86_64 5/32 2024-08-17T16:44:31.321 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 28/32 2024-08-17T16:44:31.321 INFO:teuthology.orchestra.run.smithi162.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-08-17T16:44:31.321 INFO:teuthology.orchestra.run.smithi162.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-08-17T16:44:31.321 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:44:31.342 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: librados2-2:18.2.4-719.gced649f8.el9.x86_64 5/32 2024-08-17T16:44:31.342 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 29/32 2024-08-17T16:44:31.433 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 6/32 2024-08-17T16:44:31.471 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 7/32 2024-08-17T16:44:31.535 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 7/32 2024-08-17T16:44:31.571 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-08-17T16:44:31.618 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-ceph-argparse-2:18.2.4-719.gced649f8.el9.x 9/32 2024-08-17T16:44:31.683 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 10/32 2024-08-17T16:44:31.732 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 11/32 2024-08-17T16:44:31.782 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 11/32 2024-08-17T16:44:31.891 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : re2-1:20211101-3.el9.x86_64 12/32 2024-08-17T16:44:31.923 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-08-17T16:44:31.964 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-08-17T16:44:32.013 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/32 2024-08-17T16:44:32.040 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-08-17T16:44:32.076 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-08-17T16:44:32.108 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-08-17T16:44:32.430 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-08-17T16:44:32.471 INFO:teuthology.orchestra.run.smithi063.stdout: Upgrading : librbd1-2:18.2.4-719.gced649f8.el9.x86_64 20/32 2024-08-17T16:44:32.567 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: librbd1-2:18.2.4-719.gced649f8.el9.x86_64 20/32 2024-08-17T16:44:32.593 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 21/32 2024-08-17T16:44:32.629 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-08-17T16:44:33.049 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-08-17T16:44:33.082 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : librgw2-2:18.2.4-719.gced649f8.el9.x86_64 24/32 2024-08-17T16:44:33.156 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: librgw2-2:18.2.4-719.gced649f8.el9.x86_64 24/32 2024-08-17T16:44:33.233 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 25/32 2024-08-17T16:44:33.259 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-ceph-common-2:18.2.4-719.gced649f8.el9.x86 26/32 2024-08-17T16:44:35.428 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 27/32 2024-08-17T16:44:35.465 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 27/32 2024-08-17T16:44:36.030 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 27/32 2024-08-17T16:44:36.069 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 28/32 2024-08-17T16:44:36.164 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 28/32 2024-08-17T16:44:36.164 INFO:teuthology.orchestra.run.smithi063.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-08-17T16:44:36.164 INFO:teuthology.orchestra.run.smithi063.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-08-17T16:44:36.164 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:44:36.195 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 29/32 2024-08-17T16:44:41.342 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 29/32 2024-08-17T16:44:41.342 INFO:teuthology.orchestra.run.smithi162.stdout:skipping the directory /sys 2024-08-17T16:44:41.343 INFO:teuthology.orchestra.run.smithi162.stdout:skipping the directory /proc 2024-08-17T16:44:41.343 INFO:teuthology.orchestra.run.smithi162.stdout:skipping the directory /mnt 2024-08-17T16:44:41.343 INFO:teuthology.orchestra.run.smithi162.stdout:skipping the directory /var/tmp 2024-08-17T16:44:41.343 INFO:teuthology.orchestra.run.smithi162.stdout:skipping the directory /home 2024-08-17T16:44:41.343 INFO:teuthology.orchestra.run.smithi162.stdout:skipping the directory /root 2024-08-17T16:44:41.343 INFO:teuthology.orchestra.run.smithi162.stdout:skipping the directory /tmp 2024-08-17T16:44:41.343 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:44:42.087 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 30/32 2024-08-17T16:44:42.113 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 30/32 2024-08-17T16:44:42.113 INFO:teuthology.orchestra.run.smithi162.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:44:42.113 INFO:teuthology.orchestra.run.smithi162.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-08-17T16:44:42.113 INFO:teuthology.orchestra.run.smithi162.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T16:44:42.113 INFO:teuthology.orchestra.run.smithi162.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T16:44:42.113 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:44:42.115 INFO:teuthology.orchestra.run.smithi162.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-08-17T16:44:42.190 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-08-17T16:44:42.190 INFO:teuthology.orchestra.run.smithi162.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-08-17T16:44:43.312 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-08-17T16:44:43.312 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 1/32 2024-08-17T16:44:43.312 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 2/32 2024-08-17T16:44:43.312 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 3/32 2024-08-17T16:44:43.312 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 4/32 2024-08-17T16:44:43.312 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 5/32 2024-08-17T16:44:43.313 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 6/32 2024-08-17T16:44:43.313 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : librgw2-2:18.2.4-719.gced649f8.el9.x86_64 7/32 2024-08-17T16:44:43.313 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-ceph-argparse-2:18.2.4-719.gced649f8.el9.x 8/32 2024-08-17T16:44:43.313 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-ceph-common-2:18.2.4-719.gced649f8.el9.x86 9/32 2024-08-17T16:44:43.313 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 10/32 2024-08-17T16:44:43.313 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 11/32 2024-08-17T16:44:43.313 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 12/32 2024-08-17T16:44:43.313 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 13/32 2024-08-17T16:44:43.313 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-08-17T16:44:43.313 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-08-17T16:44:43.313 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-08-17T16:44:43.314 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-08-17T16:44:43.314 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-08-17T16:44:43.314 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-08-17T16:44:43.314 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-08-17T16:44:43.314 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 21/32 2024-08-17T16:44:43.314 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-08-17T16:44:43.314 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-08-17T16:44:43.314 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-08-17T16:44:43.314 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-08-17T16:44:43.314 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-08-17T16:44:43.314 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : re2-1:20211101-3.el9.x86_64 27/32 2024-08-17T16:44:43.314 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-08-17T16:44:43.314 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : librados2-2:18.2.4-719.gced649f8.el9.x86_64 29/32 2024-08-17T16:44:43.315 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-08-17T16:44:43.315 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : librbd1-2:18.2.4-719.gced649f8.el9.x86_64 31/32 2024-08-17T16:44:44.076 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-08-17T16:44:44.077 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:44:44.077 INFO:teuthology.orchestra.run.smithi162.stdout:Upgraded: 2024-08-17T16:44:44.077 INFO:teuthology.orchestra.run.smithi162.stdout: librados2-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:44.077 INFO:teuthology.orchestra.run.smithi162.stdout: librbd1-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:44.077 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:44:44.077 INFO:teuthology.orchestra.run.smithi162.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-08-17T16:44:44.077 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:44.077 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:44.077 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:44.077 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:44.078 INFO:teuthology.orchestra.run.smithi162.stdout: fmt-8.1.1-5.el9.x86_64 2024-08-17T16:44:44.078 INFO:teuthology.orchestra.run.smithi162.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-08-17T16:44:44.078 INFO:teuthology.orchestra.run.smithi162.stdout: libarrow-9.0.0-11.el9.x86_64 2024-08-17T16:44:44.078 INFO:teuthology.orchestra.run.smithi162.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-08-17T16:44:44.078 INFO:teuthology.orchestra.run.smithi162.stdout: libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:44.078 INFO:teuthology.orchestra.run.smithi162.stdout: liboath-2.6.7-2.el9.x86_64 2024-08-17T16:44:44.078 INFO:teuthology.orchestra.run.smithi162.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-08-17T16:44:44.078 INFO:teuthology.orchestra.run.smithi162.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-08-17T16:44:44.078 INFO:teuthology.orchestra.run.smithi162.stdout: libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:44.078 INFO:teuthology.orchestra.run.smithi162.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-08-17T16:44:44.078 INFO:teuthology.orchestra.run.smithi162.stdout: librgw2-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:44.078 INFO:teuthology.orchestra.run.smithi162.stdout: libunwind-1.6.2-1.el9.x86_64 2024-08-17T16:44:44.079 INFO:teuthology.orchestra.run.smithi162.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-08-17T16:44:44.079 INFO:teuthology.orchestra.run.smithi162.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-08-17T16:44:44.079 INFO:teuthology.orchestra.run.smithi162.stdout: python3-ceph-argparse-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:44.079 INFO:teuthology.orchestra.run.smithi162.stdout: python3-ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:44.079 INFO:teuthology.orchestra.run.smithi162.stdout: python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:44.079 INFO:teuthology.orchestra.run.smithi162.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-08-17T16:44:44.079 INFO:teuthology.orchestra.run.smithi162.stdout: python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:44.079 INFO:teuthology.orchestra.run.smithi162.stdout: python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:44.079 INFO:teuthology.orchestra.run.smithi162.stdout: python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:44.079 INFO:teuthology.orchestra.run.smithi162.stdout: re2-1:20211101-3.el9.x86_64 2024-08-17T16:44:44.079 INFO:teuthology.orchestra.run.smithi162.stdout: thrift-0.15.0-2.el9.x86_64 2024-08-17T16:44:44.079 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:44:44.080 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:44:44.432 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install ceph-test 2024-08-17T16:44:44.984 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:00:25 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:44:45.111 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:44:45.111 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:44:45.112 INFO:teuthology.orchestra.run.smithi162.stdout: Package Arch Version Repository Size 2024-08-17T16:44:45.112 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:44:45.112 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:44:45.112 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-test x86_64 2:18.2.4-719.gced649f8.el9 ceph 40 M 2024-08-17T16:44:45.112 INFO:teuthology.orchestra.run.smithi162.stdout:Installing dependencies: 2024-08-17T16:44:45.112 INFO:teuthology.orchestra.run.smithi162.stdout: libcephsqlite x86_64 2:18.2.4-719.gced649f8.el9 ceph 166 k 2024-08-17T16:44:45.112 INFO:teuthology.orchestra.run.smithi162.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-08-17T16:44:45.112 INFO:teuthology.orchestra.run.smithi162.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-08-17T16:44:45.112 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:44:45.112 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:44:45.112 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:44:45.112 INFO:teuthology.orchestra.run.smithi162.stdout:Install 4 Packages 2024-08-17T16:44:45.113 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:44:45.113 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 40 M 2024-08-17T16:44:45.113 INFO:teuthology.orchestra.run.smithi162.stdout:Installed size: 163 M 2024-08-17T16:44:45.113 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:44:45.427 INFO:teuthology.orchestra.run.smithi162.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.7 MB/s | 304 kB 00:00 2024-08-17T16:44:45.469 INFO:teuthology.orchestra.run.smithi162.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.5 MB/s | 64 kB 00:00 2024-08-17T16:44:45.510 INFO:teuthology.orchestra.run.smithi162.stdout:(3/4): libcephsqlite-18.2.4-719.gced649f8.el9.x 636 kB/s | 166 kB 00:00 2024-08-17T16:44:46.293 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 29/32 2024-08-17T16:44:46.293 INFO:teuthology.orchestra.run.smithi063.stdout:skipping the directory /sys 2024-08-17T16:44:46.293 INFO:teuthology.orchestra.run.smithi063.stdout:skipping the directory /proc 2024-08-17T16:44:46.293 INFO:teuthology.orchestra.run.smithi063.stdout:skipping the directory /mnt 2024-08-17T16:44:46.293 INFO:teuthology.orchestra.run.smithi063.stdout:skipping the directory /var/tmp 2024-08-17T16:44:46.293 INFO:teuthology.orchestra.run.smithi063.stdout:skipping the directory /home 2024-08-17T16:44:46.293 INFO:teuthology.orchestra.run.smithi063.stdout:skipping the directory /root 2024-08-17T16:44:46.294 INFO:teuthology.orchestra.run.smithi063.stdout:skipping the directory /tmp 2024-08-17T16:44:46.294 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:44:46.552 INFO:teuthology.orchestra.run.smithi162.stdout:(4/4): ceph-test-18.2.4-719.gced649f8.el9.x86_6 31 MB/s | 40 MB 00:01 2024-08-17T16:44:46.554 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:44:46.555 INFO:teuthology.orchestra.run.smithi162.stdout:Total 28 MB/s | 40 MB 00:01 2024-08-17T16:44:46.583 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:44:46.603 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:44:46.603 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:44:46.920 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:44:46.921 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:44:47.063 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 30/32 2024-08-17T16:44:47.090 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 30/32 2024-08-17T16:44:47.090 INFO:teuthology.orchestra.run.smithi063.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:44:47.090 INFO:teuthology.orchestra.run.smithi063.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-08-17T16:44:47.090 INFO:teuthology.orchestra.run.smithi063.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T16:44:47.090 INFO:teuthology.orchestra.run.smithi063.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-08-17T16:44:47.090 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:44:47.092 INFO:teuthology.orchestra.run.smithi063.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-08-17T16:44:47.166 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-08-17T16:44:47.166 INFO:teuthology.orchestra.run.smithi063.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-08-17T16:44:47.390 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:44:47.462 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-08-17T16:44:47.502 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-08-17T16:44:47.529 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 3/4 2024-08-17T16:44:48.224 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-08-17T16:44:48.225 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 1/32 2024-08-17T16:44:48.225 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 2/32 2024-08-17T16:44:48.225 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 3/32 2024-08-17T16:44:48.225 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 4/32 2024-08-17T16:44:48.225 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 5/32 2024-08-17T16:44:48.225 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 6/32 2024-08-17T16:44:48.225 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : librgw2-2:18.2.4-719.gced649f8.el9.x86_64 7/32 2024-08-17T16:44:48.225 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-ceph-argparse-2:18.2.4-719.gced649f8.el9.x 8/32 2024-08-17T16:44:48.225 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-ceph-common-2:18.2.4-719.gced649f8.el9.x86 9/32 2024-08-17T16:44:48.225 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 10/32 2024-08-17T16:44:48.225 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 11/32 2024-08-17T16:44:48.226 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 12/32 2024-08-17T16:44:48.226 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 13/32 2024-08-17T16:44:48.226 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-08-17T16:44:48.226 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-08-17T16:44:48.226 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-08-17T16:44:48.226 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-08-17T16:44:48.226 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-08-17T16:44:48.226 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-08-17T16:44:48.226 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-08-17T16:44:48.226 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 21/32 2024-08-17T16:44:48.226 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-08-17T16:44:48.226 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-08-17T16:44:48.227 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-08-17T16:44:48.227 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-08-17T16:44:48.227 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-08-17T16:44:48.227 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : re2-1:20211101-3.el9.x86_64 27/32 2024-08-17T16:44:48.227 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-08-17T16:44:48.227 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : librados2-2:18.2.4-719.gced649f8.el9.x86_64 29/32 2024-08-17T16:44:48.227 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-08-17T16:44:48.227 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : librbd1-2:18.2.4-719.gced649f8.el9.x86_64 31/32 2024-08-17T16:44:48.930 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-08-17T16:44:48.931 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:44:48.931 INFO:teuthology.orchestra.run.smithi063.stdout:Upgraded: 2024-08-17T16:44:48.931 INFO:teuthology.orchestra.run.smithi063.stdout: librados2-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:48.931 INFO:teuthology.orchestra.run.smithi063.stdout: librbd1-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:48.931 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:44:48.931 INFO:teuthology.orchestra.run.smithi063.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-08-17T16:44:48.931 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:48.931 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:48.931 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-radosgw-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:48.931 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-selinux-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:48.932 INFO:teuthology.orchestra.run.smithi063.stdout: fmt-8.1.1-5.el9.x86_64 2024-08-17T16:44:48.932 INFO:teuthology.orchestra.run.smithi063.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-08-17T16:44:48.932 INFO:teuthology.orchestra.run.smithi063.stdout: libarrow-9.0.0-11.el9.x86_64 2024-08-17T16:44:48.932 INFO:teuthology.orchestra.run.smithi063.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-08-17T16:44:48.932 INFO:teuthology.orchestra.run.smithi063.stdout: libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:48.932 INFO:teuthology.orchestra.run.smithi063.stdout: liboath-2.6.7-2.el9.x86_64 2024-08-17T16:44:48.932 INFO:teuthology.orchestra.run.smithi063.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-08-17T16:44:48.932 INFO:teuthology.orchestra.run.smithi063.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-08-17T16:44:48.932 INFO:teuthology.orchestra.run.smithi063.stdout: libradosstriper1-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:48.932 INFO:teuthology.orchestra.run.smithi063.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-08-17T16:44:48.933 INFO:teuthology.orchestra.run.smithi063.stdout: librgw2-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:48.933 INFO:teuthology.orchestra.run.smithi063.stdout: libunwind-1.6.2-1.el9.x86_64 2024-08-17T16:44:48.933 INFO:teuthology.orchestra.run.smithi063.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-08-17T16:44:48.933 INFO:teuthology.orchestra.run.smithi063.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-08-17T16:44:48.933 INFO:teuthology.orchestra.run.smithi063.stdout: python3-ceph-argparse-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:48.933 INFO:teuthology.orchestra.run.smithi063.stdout: python3-ceph-common-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:48.933 INFO:teuthology.orchestra.run.smithi063.stdout: python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:48.933 INFO:teuthology.orchestra.run.smithi063.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-08-17T16:44:48.933 INFO:teuthology.orchestra.run.smithi063.stdout: python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:48.933 INFO:teuthology.orchestra.run.smithi063.stdout: python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:48.934 INFO:teuthology.orchestra.run.smithi063.stdout: python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:48.934 INFO:teuthology.orchestra.run.smithi063.stdout: re2-1:20211101-3.el9.x86_64 2024-08-17T16:44:48.934 INFO:teuthology.orchestra.run.smithi063.stdout: thrift-0.15.0-2.el9.x86_64 2024-08-17T16:44:48.934 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:44:48.934 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:44:49.299 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install ceph-test 2024-08-17T16:44:49.868 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:00:27 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:44:50.000 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:44:50.000 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:44:50.001 INFO:teuthology.orchestra.run.smithi063.stdout: Package Arch Version Repository Size 2024-08-17T16:44:50.001 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:44:50.001 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:44:50.001 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-test x86_64 2:18.2.4-719.gced649f8.el9 ceph 40 M 2024-08-17T16:44:50.001 INFO:teuthology.orchestra.run.smithi063.stdout:Installing dependencies: 2024-08-17T16:44:50.001 INFO:teuthology.orchestra.run.smithi063.stdout: libcephsqlite x86_64 2:18.2.4-719.gced649f8.el9 ceph 166 k 2024-08-17T16:44:50.001 INFO:teuthology.orchestra.run.smithi063.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-08-17T16:44:50.001 INFO:teuthology.orchestra.run.smithi063.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-08-17T16:44:50.001 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:44:50.001 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:44:50.002 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:44:50.002 INFO:teuthology.orchestra.run.smithi063.stdout:Install 4 Packages 2024-08-17T16:44:50.002 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:44:50.002 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 40 M 2024-08-17T16:44:50.002 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 163 M 2024-08-17T16:44:50.002 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:44:50.405 INFO:teuthology.orchestra.run.smithi063.stdout:(1/4): libcephsqlite-18.2.4-719.gced649f8.el9.x 633 kB/s | 166 kB 00:00 2024-08-17T16:44:50.430 INFO:teuthology.orchestra.run.smithi063.stdout:(2/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.0 MB/s | 304 kB 00:00 2024-08-17T16:44:50.547 INFO:teuthology.orchestra.run.smithi063.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 449 kB/s | 64 kB 00:00 2024-08-17T16:44:51.222 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 3/4 2024-08-17T16:44:51.294 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 4/4 2024-08-17T16:44:51.472 INFO:teuthology.orchestra.run.smithi063.stdout:(4/4): ceph-test-18.2.4-719.gced649f8.el9.x86_6 30 MB/s | 40 MB 00:01 2024-08-17T16:44:51.474 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:44:51.474 INFO:teuthology.orchestra.run.smithi063.stdout:Total 27 MB/s | 40 MB 00:01 2024-08-17T16:44:51.503 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:44:51.525 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:44:51.525 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:44:51.863 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:44:51.863 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:44:52.356 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:44:52.437 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-08-17T16:44:52.488 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-08-17T16:44:52.516 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 3/4 2024-08-17T16:44:53.359 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 4/4 2024-08-17T16:44:53.359 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 1/4 2024-08-17T16:44:53.359 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 2/4 2024-08-17T16:44:53.359 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-08-17T16:44:53.664 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-08-17T16:44:53.664 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:44:53.664 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:44:53.664 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:53.664 INFO:teuthology.orchestra.run.smithi162.stdout: libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:53.664 INFO:teuthology.orchestra.run.smithi162.stdout: socat-1.7.4.1-6.el9.x86_64 2024-08-17T16:44:53.664 INFO:teuthology.orchestra.run.smithi162.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-08-17T16:44:53.664 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:44:53.664 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:44:53.939 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install ceph 2024-08-17T16:44:54.495 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:00:35 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:44:54.626 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:44:54.629 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:44:54.629 INFO:teuthology.orchestra.run.smithi162.stdout: Package Arch Version Repository Size 2024-08-17T16:44:54.629 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:44:54.629 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:44:54.629 INFO:teuthology.orchestra.run.smithi162.stdout: ceph x86_64 2:18.2.4-719.gced649f8.el9 ceph 6.5 k 2024-08-17T16:44:54.629 INFO:teuthology.orchestra.run.smithi162.stdout:Installing dependencies: 2024-08-17T16:44:54.629 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mds x86_64 2:18.2.4-719.gced649f8.el9 ceph 2.1 M 2024-08-17T16:44:54.629 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mgr x86_64 2:18.2.4-719.gced649f8.el9 ceph 1.5 M 2024-08-17T16:44:54.629 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mgr-modules-core noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 246 k 2024-08-17T16:44:54.629 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mon x86_64 2:18.2.4-719.gced649f8.el9 ceph 4.7 M 2024-08-17T16:44:54.630 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-osd x86_64 2:18.2.4-719.gced649f8.el9 ceph 17 M 2024-08-17T16:44:54.630 INFO:teuthology.orchestra.run.smithi162.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-08-17T16:44:54.630 INFO:teuthology.orchestra.run.smithi162.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-08-17T16:44:54.630 INFO:teuthology.orchestra.run.smithi162.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-08-17T16:44:54.630 INFO:teuthology.orchestra.run.smithi162.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-08-17T16:44:54.630 INFO:teuthology.orchestra.run.smithi162.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-08-17T16:44:54.630 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-08-17T16:44:54.630 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-08-17T16:44:54.630 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-08-17T16:44:54.630 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-08-17T16:44:54.630 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-08-17T16:44:54.630 INFO:teuthology.orchestra.run.smithi162.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-08-17T16:44:54.631 INFO:teuthology.orchestra.run.smithi162.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-08-17T16:44:54.631 INFO:teuthology.orchestra.run.smithi162.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-08-17T16:44:54.631 INFO:teuthology.orchestra.run.smithi162.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-08-17T16:44:54.631 INFO:teuthology.orchestra.run.smithi162.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-08-17T16:44:54.631 INFO:teuthology.orchestra.run.smithi162.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-08-17T16:44:54.631 INFO:teuthology.orchestra.run.smithi162.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-08-17T16:44:54.631 INFO:teuthology.orchestra.run.smithi162.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-08-17T16:44:54.631 INFO:teuthology.orchestra.run.smithi162.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-08-17T16:44:54.631 INFO:teuthology.orchestra.run.smithi162.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-08-17T16:44:54.631 INFO:teuthology.orchestra.run.smithi162.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-08-17T16:44:54.631 INFO:teuthology.orchestra.run.smithi162.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-08-17T16:44:54.631 INFO:teuthology.orchestra.run.smithi162.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-08-17T16:44:54.631 INFO:teuthology.orchestra.run.smithi162.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-08-17T16:44:54.631 INFO:teuthology.orchestra.run.smithi162.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-08-17T16:44:54.632 INFO:teuthology.orchestra.run.smithi162.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-08-17T16:44:54.632 INFO:teuthology.orchestra.run.smithi162.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-08-17T16:44:54.632 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:44:54.632 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:44:54.632 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:44:54.632 INFO:teuthology.orchestra.run.smithi162.stdout:Install 33 Packages 2024-08-17T16:44:54.632 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:44:54.633 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 29 M 2024-08-17T16:44:54.633 INFO:teuthology.orchestra.run.smithi162.stdout:Installed size: 104 M 2024-08-17T16:44:54.633 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:44:55.195 INFO:teuthology.orchestra.run.smithi162.stdout:(1/33): ceph-18.2.4-719.gced649f8.el9.x86_64.rp 34 kB/s | 6.5 kB 00:00 2024-08-17T16:44:55.362 INFO:teuthology.orchestra.run.smithi162.stdout:(2/33): ceph-mds-18.2.4-719.gced649f8.el9.x86_6 5.9 MB/s | 2.1 MB 00:00 2024-08-17T16:44:55.396 INFO:teuthology.orchestra.run.smithi162.stdout:(3/33): ceph-mgr-18.2.4-719.gced649f8.el9.x86_6 3.7 MB/s | 1.5 MB 00:00 2024-08-17T16:44:55.479 INFO:teuthology.orchestra.run.smithi162.stdout:(4/33): ceph-mgr-modules-core-18.2.4-719.gced64 2.9 MB/s | 246 kB 00:00 2024-08-17T16:44:55.779 INFO:teuthology.orchestra.run.smithi162.stdout:(5/33): ceph-mon-18.2.4-719.gced649f8.el9.x86_6 8.0 MB/s | 4.7 MB 00:00 2024-08-17T16:44:55.938 INFO:teuthology.orchestra.run.smithi162.stdout:(6/33): ceph-osd-18.2.4-719.gced649f8.el9.x86_6 29 MB/s | 17 MB 00:00 2024-08-17T16:44:56.063 INFO:teuthology.orchestra.run.smithi162.stdout:(7/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 434 kB/s | 253 kB 00:00 2024-08-17T16:44:56.163 INFO:teuthology.orchestra.run.smithi162.stdout:(8/33): python3-pycparser-2.20-6.el9.noarch.rpm 1.3 MB/s | 135 kB 00:00 2024-08-17T16:44:56.272 INFO:teuthology.orchestra.run.smithi162.stdout:(9/33): python3-requests-2.25.1-8.el9.noarch.rp 1.1 MB/s | 125 kB 00:00 2024-08-17T16:44:56.356 INFO:teuthology.orchestra.run.smithi162.stdout:(10/33): python3-ply-3.11-14.el9.noarch.rpm 255 kB/s | 106 kB 00:00 2024-08-17T16:44:56.390 INFO:teuthology.orchestra.run.smithi162.stdout:(11/33): python3-urllib3-1.26.5-6.el9.noarch.rp 1.8 MB/s | 215 kB 00:00 2024-08-17T16:44:56.449 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 3/4 2024-08-17T16:44:56.499 INFO:teuthology.orchestra.run.smithi162.stdout:(12/33): python3-markupsafe-1.1.1-12.el9.x86_64 321 kB/s | 35 kB 00:00 2024-08-17T16:44:56.526 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 4/4 2024-08-17T16:44:56.557 INFO:teuthology.orchestra.run.smithi162.stdout:(13/33): python3-mako-1.1.4-6.el9.noarch.rpm 858 kB/s | 172 kB 00:00 2024-08-17T16:44:56.590 INFO:teuthology.orchestra.run.smithi162.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 555 kB/s | 51 kB 00:00 2024-08-17T16:44:56.607 INFO:teuthology.orchestra.run.smithi162.stdout:(15/33): python3-toml-0.10.2-6.el9.noarch.rpm 829 kB/s | 42 kB 00:00 2024-08-17T16:44:56.691 INFO:teuthology.orchestra.run.smithi162.stdout:(16/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 433 kB/s | 43 kB 00:00 2024-08-17T16:44:56.725 INFO:teuthology.orchestra.run.smithi162.stdout:(17/33): python3-cryptography-36.0.1-4.el9.x86_ 1.3 MB/s | 1.2 MB 00:00 2024-08-17T16:44:56.775 INFO:teuthology.orchestra.run.smithi162.stdout:(18/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.0 MB/s | 172 kB 00:00 2024-08-17T16:44:56.800 INFO:teuthology.orchestra.run.smithi162.stdout:(19/33): python3-jaraco-classes-3.2.1-5.el9.noa 778 kB/s | 18 kB 00:00 2024-08-17T16:44:56.825 INFO:teuthology.orchestra.run.smithi162.stdout:(20/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 106 kB/s | 11 kB 00:00 2024-08-17T16:44:56.850 INFO:teuthology.orchestra.run.smithi162.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 461 kB/s | 23 kB 00:00 2024-08-17T16:44:56.877 INFO:teuthology.orchestra.run.smithi162.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 388 kB/s | 19 kB 00:00 2024-08-17T16:44:56.901 INFO:teuthology.orchestra.run.smithi162.stdout:(23/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.7 MB/s | 358 kB 00:00 2024-08-17T16:44:56.926 INFO:teuthology.orchestra.run.smithi162.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 261 kB/s | 20 kB 00:00 2024-08-17T16:44:56.951 INFO:teuthology.orchestra.run.smithi162.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 613 kB/s | 46 kB 00:00 2024-08-17T16:44:56.977 INFO:teuthology.orchestra.run.smithi162.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-08-17T16:44:57.002 INFO:teuthology.orchestra.run.smithi162.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 326 kB/s | 16 kB 00:00 2024-08-17T16:44:57.027 INFO:teuthology.orchestra.run.smithi162.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.7 MB/s | 90 kB 00:00 2024-08-17T16:44:57.053 INFO:teuthology.orchestra.run.smithi162.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 711 kB/s | 36 kB 00:00 2024-08-17T16:44:57.078 INFO:teuthology.orchestra.run.smithi162.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.8 MB/s | 272 kB 00:00 2024-08-17T16:44:57.103 INFO:teuthology.orchestra.run.smithi162.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 797 kB/s | 20 kB 00:00 2024-08-17T16:44:57.128 INFO:teuthology.orchestra.run.smithi162.stdout:(32/33): python3-webob-1.8.7-6.el9.noarch.rpm 2.2 MB/s | 230 kB 00:00 2024-08-17T16:44:57.154 INFO:teuthology.orchestra.run.smithi162.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 4.1 MB/s | 427 kB 00:00 2024-08-17T16:44:57.157 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:44:57.157 INFO:teuthology.orchestra.run.smithi162.stdout:Total 12 MB/s | 29 MB 00:02 2024-08-17T16:44:57.272 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:44:57.310 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:44:57.310 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:44:57.687 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:44:57.687 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:44:58.551 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 4/4 2024-08-17T16:44:58.551 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 1/4 2024-08-17T16:44:58.551 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 2/4 2024-08-17T16:44:58.551 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-08-17T16:44:58.851 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-08-17T16:44:58.851 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:44:58.851 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:44:58.851 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-test-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:58.852 INFO:teuthology.orchestra.run.smithi063.stdout: libcephsqlite-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:44:58.852 INFO:teuthology.orchestra.run.smithi063.stdout: socat-1.7.4.1-6.el9.x86_64 2024-08-17T16:44:58.852 INFO:teuthology.orchestra.run.smithi063.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-08-17T16:44:58.852 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:44:58.852 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:44:58.935 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:44:58.990 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-08-17T16:44:59.022 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-08-17T16:44:59.055 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-08-17T16:44:59.107 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install ceph 2024-08-17T16:44:59.147 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-08-17T16:44:59.225 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-08-17T16:44:59.275 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-08-17T16:44:59.310 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-08-17T16:44:59.348 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-08-17T16:44:59.382 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-08-17T16:44:59.416 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-08-17T16:44:59.456 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-08-17T16:44:59.491 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-08-17T16:44:59.526 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-08-17T16:44:59.570 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-08-17T16:44:59.645 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-08-17T16:44:59.672 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:00:37 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:44:59.793 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-08-17T16:44:59.807 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:44:59.810 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:44:59.810 INFO:teuthology.orchestra.run.smithi063.stdout: Package Arch Version Repository Size 2024-08-17T16:44:59.810 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:44:59.810 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:44:59.810 INFO:teuthology.orchestra.run.smithi063.stdout: ceph x86_64 2:18.2.4-719.gced649f8.el9 ceph 6.5 k 2024-08-17T16:44:59.810 INFO:teuthology.orchestra.run.smithi063.stdout:Installing dependencies: 2024-08-17T16:44:59.810 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mds x86_64 2:18.2.4-719.gced649f8.el9 ceph 2.1 M 2024-08-17T16:44:59.810 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mgr x86_64 2:18.2.4-719.gced649f8.el9 ceph 1.5 M 2024-08-17T16:44:59.811 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mgr-modules-core noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 246 k 2024-08-17T16:44:59.811 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mon x86_64 2:18.2.4-719.gced649f8.el9 ceph 4.7 M 2024-08-17T16:44:59.811 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-osd x86_64 2:18.2.4-719.gced649f8.el9 ceph 17 M 2024-08-17T16:44:59.811 INFO:teuthology.orchestra.run.smithi063.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-08-17T16:44:59.811 INFO:teuthology.orchestra.run.smithi063.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-08-17T16:44:59.811 INFO:teuthology.orchestra.run.smithi063.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-08-17T16:44:59.811 INFO:teuthology.orchestra.run.smithi063.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-08-17T16:44:59.811 INFO:teuthology.orchestra.run.smithi063.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-08-17T16:44:59.811 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-08-17T16:44:59.811 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-08-17T16:44:59.811 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-08-17T16:44:59.811 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-08-17T16:44:59.811 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-08-17T16:44:59.812 INFO:teuthology.orchestra.run.smithi063.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-08-17T16:44:59.812 INFO:teuthology.orchestra.run.smithi063.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-08-17T16:44:59.812 INFO:teuthology.orchestra.run.smithi063.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-08-17T16:44:59.812 INFO:teuthology.orchestra.run.smithi063.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-08-17T16:44:59.812 INFO:teuthology.orchestra.run.smithi063.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-08-17T16:44:59.812 INFO:teuthology.orchestra.run.smithi063.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-08-17T16:44:59.812 INFO:teuthology.orchestra.run.smithi063.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-08-17T16:44:59.812 INFO:teuthology.orchestra.run.smithi063.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-08-17T16:44:59.812 INFO:teuthology.orchestra.run.smithi063.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-08-17T16:44:59.812 INFO:teuthology.orchestra.run.smithi063.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-08-17T16:44:59.812 INFO:teuthology.orchestra.run.smithi063.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-08-17T16:44:59.812 INFO:teuthology.orchestra.run.smithi063.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-08-17T16:44:59.812 INFO:teuthology.orchestra.run.smithi063.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-08-17T16:44:59.813 INFO:teuthology.orchestra.run.smithi063.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-08-17T16:44:59.813 INFO:teuthology.orchestra.run.smithi063.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-08-17T16:44:59.813 INFO:teuthology.orchestra.run.smithi063.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-08-17T16:44:59.813 INFO:teuthology.orchestra.run.smithi063.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-08-17T16:44:59.813 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:44:59.813 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:44:59.813 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:44:59.813 INFO:teuthology.orchestra.run.smithi063.stdout:Install 33 Packages 2024-08-17T16:44:59.813 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:44:59.814 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 29 M 2024-08-17T16:44:59.814 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 104 M 2024-08-17T16:44:59.814 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:44:59.881 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-08-17T16:44:59.934 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-08-17T16:44:59.972 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-08-17T16:45:00.021 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-08-17T16:45:00.077 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-08-17T16:45:00.269 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-08-17T16:45:00.318 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-08-17T16:45:00.413 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-08-17T16:45:00.502 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-08-17T16:45:00.547 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-08-17T16:45:00.608 INFO:teuthology.orchestra.run.smithi063.stdout:(1/33): ceph-18.2.4-719.gced649f8.el9.x86_64.rp 33 kB/s | 6.5 kB 00:00 2024-08-17T16:45:00.667 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-08-17T16:45:00.767 INFO:teuthology.orchestra.run.smithi063.stdout:(2/33): ceph-mds-18.2.4-719.gced649f8.el9.x86_6 5.9 MB/s | 2.1 MB 00:00 2024-08-17T16:45:00.800 INFO:teuthology.orchestra.run.smithi063.stdout:(3/33): ceph-mgr-18.2.4-719.gced649f8.el9.x86_6 3.8 MB/s | 1.5 MB 00:00 2024-08-17T16:45:00.817 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-719.gced649f8.el9.n 28/33 2024-08-17T16:45:00.852 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 29/33 2024-08-17T16:45:00.878 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 29/33 2024-08-17T16:45:00.878 INFO:teuthology.orchestra.run.smithi162.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:45:00.878 INFO:teuthology.orchestra.run.smithi162.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-08-17T16:45:00.878 INFO:teuthology.orchestra.run.smithi162.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T16:45:00.878 INFO:teuthology.orchestra.run.smithi162.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T16:45:00.879 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:00.884 INFO:teuthology.orchestra.run.smithi063.stdout:(4/33): ceph-mgr-modules-core-18.2.4-719.gced64 2.9 MB/s | 246 kB 00:00 2024-08-17T16:45:01.010 INFO:teuthology.orchestra.run.smithi063.stdout:(5/33): ceph-mon-18.2.4-719.gced649f8.el9.x86_6 12 MB/s | 4.7 MB 00:00 2024-08-17T16:45:01.160 INFO:teuthology.orchestra.run.smithi063.stdout:(6/33): python3-cryptography-36.0.1-4.el9.x86_6 8.2 MB/s | 1.2 MB 00:00 2024-08-17T16:45:01.184 INFO:teuthology.orchestra.run.smithi063.stdout:(7/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 843 kB/s | 253 kB 00:00 2024-08-17T16:45:01.343 INFO:teuthology.orchestra.run.smithi063.stdout:(8/33): ceph-osd-18.2.4-719.gced649f8.el9.x86_6 29 MB/s | 17 MB 00:00 2024-08-17T16:45:01.369 INFO:teuthology.orchestra.run.smithi063.stdout:(9/33): python3-ply-3.11-14.el9.noarch.rpm 510 kB/s | 106 kB 00:00 2024-08-17T16:45:01.394 INFO:teuthology.orchestra.run.smithi063.stdout:(10/33): python3-pycparser-2.20-6.el9.noarch.rp 647 kB/s | 135 kB 00:00 2024-08-17T16:45:01.444 INFO:teuthology.orchestra.run.smithi063.stdout:(11/33): python3-urllib3-1.26.5-6.el9.noarch.rp 2.8 MB/s | 215 kB 00:00 2024-08-17T16:45:01.494 INFO:teuthology.orchestra.run.smithi063.stdout:(12/33): python3-requests-2.25.1-8.el9.noarch.r 829 kB/s | 125 kB 00:00 2024-08-17T16:45:01.545 INFO:teuthology.orchestra.run.smithi063.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 346 kB/s | 35 kB 00:00 2024-08-17T16:45:01.570 INFO:teuthology.orchestra.run.smithi063.stdout:(14/33): python3-mako-1.1.4-6.el9.noarch.rpm 979 kB/s | 172 kB 00:00 2024-08-17T16:45:01.620 INFO:teuthology.orchestra.run.smithi063.stdout:(15/33): python3-pytz-2021.1-5.el9.noarch.rpm 407 kB/s | 51 kB 00:00 2024-08-17T16:45:01.646 INFO:teuthology.orchestra.run.smithi063.stdout:(16/33): python3-toml-0.10.2-6.el9.noarch.rpm 414 kB/s | 42 kB 00:00 2024-08-17T16:45:01.696 INFO:teuthology.orchestra.run.smithi063.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 347 kB/s | 43 kB 00:00 2024-08-17T16:45:01.746 INFO:teuthology.orchestra.run.smithi063.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 496 kB/s | 11 kB 00:00 2024-08-17T16:45:01.763 INFO:teuthology.orchestra.run.smithi063.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.2 MB/s | 172 kB 00:00 2024-08-17T16:45:01.788 INFO:teuthology.orchestra.run.smithi063.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 448 kB/s | 18 kB 00:00 2024-08-17T16:45:01.813 INFO:teuthology.orchestra.run.smithi063.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 461 kB/s | 23 kB 00:00 2024-08-17T16:45:01.839 INFO:teuthology.orchestra.run.smithi063.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 387 kB/s | 19 kB 00:00 2024-08-17T16:45:01.864 INFO:teuthology.orchestra.run.smithi063.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 390 kB/s | 20 kB 00:00 2024-08-17T16:45:01.889 INFO:teuthology.orchestra.run.smithi063.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 925 kB/s | 46 kB 00:00 2024-08-17T16:45:01.915 INFO:teuthology.orchestra.run.smithi063.stdout:(25/33): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-08-17T16:45:01.940 INFO:teuthology.orchestra.run.smithi063.stdout:(26/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.3 MB/s | 358 kB 00:00 2024-08-17T16:45:01.965 INFO:teuthology.orchestra.run.smithi063.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 327 kB/s | 16 kB 00:00 2024-08-17T16:45:01.991 INFO:teuthology.orchestra.run.smithi063.stdout:(28/33): python3-tempora-5.0.0-2.el9.noarch.rpm 1.4 MB/s | 36 kB 00:00 2024-08-17T16:45:02.016 INFO:teuthology.orchestra.run.smithi063.stdout:(29/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.2 MB/s | 90 kB 00:00 2024-08-17T16:45:02.041 INFO:teuthology.orchestra.run.smithi063.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.8 MB/s | 272 kB 00:00 2024-08-17T16:45:02.066 INFO:teuthology.orchestra.run.smithi063.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 801 kB/s | 20 kB 00:00 2024-08-17T16:45:02.092 INFO:teuthology.orchestra.run.smithi063.stdout:(32/33): python3-webob-1.8.7-6.el9.noarch.rpm 2.2 MB/s | 230 kB 00:00 2024-08-17T16:45:02.125 INFO:teuthology.orchestra.run.smithi063.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.8 MB/s | 427 kB 00:00 2024-08-17T16:45:02.129 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:02.129 INFO:teuthology.orchestra.run.smithi063.stdout:Total 13 MB/s | 29 MB 00:02 2024-08-17T16:45:02.244 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:45:02.284 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:45:02.285 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:45:02.295 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 30/33 2024-08-17T16:45:02.323 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 30/33 2024-08-17T16:45:02.323 INFO:teuthology.orchestra.run.smithi162.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:45:02.324 INFO:teuthology.orchestra.run.smithi162.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-08-17T16:45:02.324 INFO:teuthology.orchestra.run.smithi162.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T16:45:02.324 INFO:teuthology.orchestra.run.smithi162.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T16:45:02.324 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:02.681 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:45:02.681 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:45:02.762 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 31/33 2024-08-17T16:45:02.786 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 31/33 2024-08-17T16:45:02.786 INFO:teuthology.orchestra.run.smithi162.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:45:02.786 INFO:teuthology.orchestra.run.smithi162.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-08-17T16:45:02.786 INFO:teuthology.orchestra.run.smithi162.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T16:45:02.786 INFO:teuthology.orchestra.run.smithi162.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T16:45:02.786 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:02.988 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 32/33 2024-08-17T16:45:03.012 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 32/33 2024-08-17T16:45:03.012 INFO:teuthology.orchestra.run.smithi162.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:45:03.012 INFO:teuthology.orchestra.run.smithi162.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-08-17T16:45:03.012 INFO:teuthology.orchestra.run.smithi162.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T16:45:03.012 INFO:teuthology.orchestra.run.smithi162.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T16:45:03.012 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:03.076 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-2:18.2.4-719.gced649f8.el9.x86_64 33/33 2024-08-17T16:45:03.948 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:45:04.020 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-08-17T16:45:04.053 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-08-17T16:45:04.086 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-08-17T16:45:04.182 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-08-17T16:45:04.268 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-08-17T16:45:04.315 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-08-17T16:45:04.349 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-08-17T16:45:04.386 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-08-17T16:45:04.421 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-08-17T16:45:04.454 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-08-17T16:45:04.496 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-08-17T16:45:04.539 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-08-17T16:45:04.573 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-08-17T16:45:04.636 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-08-17T16:45:04.716 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-2:18.2.4-719.gced649f8.el9.x86_64 33/33 2024-08-17T16:45:04.716 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-2:18.2.4-719.gced649f8.el9.x86_64 1/33 2024-08-17T16:45:04.717 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 2/33 2024-08-17T16:45:04.717 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 3/33 2024-08-17T16:45:04.717 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 4/33 2024-08-17T16:45:04.717 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 5/33 2024-08-17T16:45:04.717 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-719.gced649f8.el9.n 6/33 2024-08-17T16:45:04.717 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-08-17T16:45:04.717 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-08-17T16:45:04.717 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-08-17T16:45:04.717 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-08-17T16:45:04.717 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-08-17T16:45:04.717 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-08-17T16:45:04.717 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-08-17T16:45:04.718 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-08-17T16:45:04.718 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-08-17T16:45:04.718 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-08-17T16:45:04.718 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-08-17T16:45:04.718 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-08-17T16:45:04.718 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-08-17T16:45:04.718 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-08-17T16:45:04.718 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-08-17T16:45:04.718 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-08-17T16:45:04.718 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-08-17T16:45:04.718 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-08-17T16:45:04.718 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-08-17T16:45:04.719 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-08-17T16:45:04.720 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-08-17T16:45:04.720 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-08-17T16:45:04.720 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-08-17T16:45:04.720 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-08-17T16:45:04.720 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-08-17T16:45:04.720 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-08-17T16:45:04.720 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-08-17T16:45:04.867 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-08-17T16:45:04.947 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-08-17T16:45:04.999 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-08-17T16:45:05.045 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-08-17T16:45:05.104 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-08-17T16:45:05.173 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-08-17T16:45:05.382 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-08-17T16:45:05.424 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-08-17T16:45:05.507 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-08-17T16:45:05.604 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-08-17T16:45:05.647 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-08-17T16:45:05.673 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-08-17T16:45:05.673 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:05.673 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:45:05.674 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:05.674 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:05.674 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:05.674 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mgr-modules-core-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:05.674 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:05.674 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:05.674 INFO:teuthology.orchestra.run.smithi162.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-08-17T16:45:05.674 INFO:teuthology.orchestra.run.smithi162.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-08-17T16:45:05.674 INFO:teuthology.orchestra.run.smithi162.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-08-17T16:45:05.674 INFO:teuthology.orchestra.run.smithi162.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-08-17T16:45:05.674 INFO:teuthology.orchestra.run.smithi162.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-08-17T16:45:05.674 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-08-17T16:45:05.675 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-08-17T16:45:05.675 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-08-17T16:45:05.675 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-08-17T16:45:05.675 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-08-17T16:45:05.675 INFO:teuthology.orchestra.run.smithi162.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-08-17T16:45:05.675 INFO:teuthology.orchestra.run.smithi162.stdout: python3-mako-1.1.4-6.el9.noarch 2024-08-17T16:45:05.675 INFO:teuthology.orchestra.run.smithi162.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-08-17T16:45:05.675 INFO:teuthology.orchestra.run.smithi162.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-08-17T16:45:05.675 INFO:teuthology.orchestra.run.smithi162.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-08-17T16:45:05.675 INFO:teuthology.orchestra.run.smithi162.stdout: python3-ply-3.11-14.el9.noarch 2024-08-17T16:45:05.675 INFO:teuthology.orchestra.run.smithi162.stdout: python3-portend-3.1.0-2.el9.noarch 2024-08-17T16:45:05.675 INFO:teuthology.orchestra.run.smithi162.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-08-17T16:45:05.675 INFO:teuthology.orchestra.run.smithi162.stdout: python3-pycparser-2.20-6.el9.noarch 2024-08-17T16:45:05.676 INFO:teuthology.orchestra.run.smithi162.stdout: python3-pytz-2021.1-5.el9.noarch 2024-08-17T16:45:05.676 INFO:teuthology.orchestra.run.smithi162.stdout: python3-requests-2.25.1-8.el9.noarch 2024-08-17T16:45:05.676 INFO:teuthology.orchestra.run.smithi162.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-08-17T16:45:05.676 INFO:teuthology.orchestra.run.smithi162.stdout: python3-toml-0.10.2-6.el9.noarch 2024-08-17T16:45:05.676 INFO:teuthology.orchestra.run.smithi162.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-08-17T16:45:05.676 INFO:teuthology.orchestra.run.smithi162.stdout: python3-webob-1.8.7-6.el9.noarch 2024-08-17T16:45:05.676 INFO:teuthology.orchestra.run.smithi162.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-08-17T16:45:05.676 INFO:teuthology.orchestra.run.smithi162.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-08-17T16:45:05.676 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:05.676 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:05.770 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-08-17T16:45:05.929 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-719.gced649f8.el9.n 28/33 2024-08-17T16:45:05.958 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 29/33 2024-08-17T16:45:05.984 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 29/33 2024-08-17T16:45:05.985 INFO:teuthology.orchestra.run.smithi063.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:45:05.985 INFO:teuthology.orchestra.run.smithi063.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-08-17T16:45:05.985 INFO:teuthology.orchestra.run.smithi063.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T16:45:05.985 INFO:teuthology.orchestra.run.smithi063.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-08-17T16:45:05.985 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:06.052 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install ceph-base 2024-08-17T16:45:06.612 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:00:47 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:06.698 INFO:teuthology.orchestra.run.smithi162.stdout:Package ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:06.739 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:06.740 INFO:teuthology.orchestra.run.smithi162.stdout:Nothing to do. 2024-08-17T16:45:06.740 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:06.795 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install cephadm 2024-08-17T16:45:07.351 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:00:48 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:07.478 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:07.479 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:07.479 INFO:teuthology.orchestra.run.smithi162.stdout: Package Arch Version Repository Size 2024-08-17T16:45:07.479 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:07.479 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:45:07.479 INFO:teuthology.orchestra.run.smithi162.stdout: cephadm noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 224 k 2024-08-17T16:45:07.479 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:07.479 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:45:07.479 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:07.479 INFO:teuthology.orchestra.run.smithi162.stdout:Install 1 Package 2024-08-17T16:45:07.479 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:07.480 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 224 k 2024-08-17T16:45:07.480 INFO:teuthology.orchestra.run.smithi162.stdout:Installed size: 216 k 2024-08-17T16:45:07.480 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:45:07.493 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 30/33 2024-08-17T16:45:07.523 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 30/33 2024-08-17T16:45:07.523 INFO:teuthology.orchestra.run.smithi063.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:45:07.523 INFO:teuthology.orchestra.run.smithi063.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-08-17T16:45:07.523 INFO:teuthology.orchestra.run.smithi063.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T16:45:07.523 INFO:teuthology.orchestra.run.smithi063.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-08-17T16:45:07.524 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:07.712 INFO:teuthology.orchestra.run.smithi162.stdout:cephadm-18.2.4-719.gced649f8.el9.noarch.rpm 965 kB/s | 224 kB 00:00 2024-08-17T16:45:07.713 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:07.713 INFO:teuthology.orchestra.run.smithi162.stdout:Total 958 kB/s | 224 kB 00:00 2024-08-17T16:45:07.713 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:45:07.718 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:45:07.718 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:45:07.721 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:45:07.721 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:45:07.787 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:45:07.985 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 31/33 2024-08-17T16:45:08.010 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 31/33 2024-08-17T16:45:08.010 INFO:teuthology.orchestra.run.smithi063.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:45:08.010 INFO:teuthology.orchestra.run.smithi063.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-08-17T16:45:08.010 INFO:teuthology.orchestra.run.smithi063.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T16:45:08.010 INFO:teuthology.orchestra.run.smithi063.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-08-17T16:45:08.010 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:08.134 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:45:08.225 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:45:08.227 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 32/33 2024-08-17T16:45:08.252 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 32/33 2024-08-17T16:45:08.252 INFO:teuthology.orchestra.run.smithi063.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:45:08.252 INFO:teuthology.orchestra.run.smithi063.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-08-17T16:45:08.252 INFO:teuthology.orchestra.run.smithi063.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T16:45:08.252 INFO:teuthology.orchestra.run.smithi063.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-08-17T16:45:08.252 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:08.319 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-2:18.2.4-719.gced649f8.el9.x86_64 33/33 2024-08-17T16:45:08.644 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:45:08.877 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:45:08.877 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:08.877 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:45:08.878 INFO:teuthology.orchestra.run.smithi162.stdout: cephadm-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:08.878 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:08.878 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:09.085 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install ceph-immutable-object-cache 2024-08-17T16:45:09.651 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:00:50 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:09.779 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:09.780 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:09.780 INFO:teuthology.orchestra.run.smithi162.stdout: Package Arch Version Repo Size 2024-08-17T16:45:09.780 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:09.781 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:45:09.781 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-719.gced649f8.el9 ceph 143 k 2024-08-17T16:45:09.781 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:09.781 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:45:09.781 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:09.781 INFO:teuthology.orchestra.run.smithi162.stdout:Install 1 Package 2024-08-17T16:45:09.781 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:09.781 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 143 k 2024-08-17T16:45:09.781 INFO:teuthology.orchestra.run.smithi162.stdout:Installed size: 395 k 2024-08-17T16:45:09.781 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:45:09.949 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-2:18.2.4-719.gced649f8.el9.x86_64 33/33 2024-08-17T16:45:09.950 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-2:18.2.4-719.gced649f8.el9.x86_64 1/33 2024-08-17T16:45:09.950 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 2/33 2024-08-17T16:45:09.950 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 3/33 2024-08-17T16:45:09.950 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 4/33 2024-08-17T16:45:09.950 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 5/33 2024-08-17T16:45:09.950 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-719.gced649f8.el9.n 6/33 2024-08-17T16:45:09.950 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-08-17T16:45:09.950 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-08-17T16:45:09.950 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-08-17T16:45:09.950 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-08-17T16:45:09.950 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-08-17T16:45:09.951 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-08-17T16:45:09.951 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-08-17T16:45:09.951 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-08-17T16:45:09.951 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-08-17T16:45:09.951 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-08-17T16:45:09.951 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-08-17T16:45:09.951 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-08-17T16:45:09.951 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-08-17T16:45:09.951 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-08-17T16:45:09.951 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-08-17T16:45:09.951 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-08-17T16:45:09.951 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-08-17T16:45:09.951 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-08-17T16:45:09.951 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-08-17T16:45:09.952 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-08-17T16:45:09.952 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-08-17T16:45:09.952 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-08-17T16:45:09.952 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-08-17T16:45:09.953 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-08-17T16:45:09.953 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-08-17T16:45:09.953 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-08-17T16:45:09.979 INFO:teuthology.orchestra.run.smithi162.stdout:ceph-immutable-object-cache-18.2.4-719.gced649f 723 kB/s | 143 kB 00:00 2024-08-17T16:45:09.979 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:09.980 INFO:teuthology.orchestra.run.smithi162.stdout:Total 716 kB/s | 143 kB 00:00 2024-08-17T16:45:09.980 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:45:09.987 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:45:09.988 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:45:10.044 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:45:10.045 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:45:10.193 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:45:10.271 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-719.gced649f8.e 1/1 2024-08-17T16:45:10.296 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-719.gced649f8.e 1/1 2024-08-17T16:45:10.297 INFO:teuthology.orchestra.run.smithi162.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:45:10.297 INFO:teuthology.orchestra.run.smithi162.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-08-17T16:45:10.297 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:10.907 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-08-17T16:45:10.907 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:10.907 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:45:10.907 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:10.907 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mds-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:10.907 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:10.907 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mgr-modules-core-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:10.908 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mon-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:10.908 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-osd-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:10.908 INFO:teuthology.orchestra.run.smithi063.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-08-17T16:45:10.908 INFO:teuthology.orchestra.run.smithi063.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-08-17T16:45:10.908 INFO:teuthology.orchestra.run.smithi063.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-08-17T16:45:10.908 INFO:teuthology.orchestra.run.smithi063.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-08-17T16:45:10.908 INFO:teuthology.orchestra.run.smithi063.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-08-17T16:45:10.908 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-08-17T16:45:10.908 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-08-17T16:45:10.908 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-08-17T16:45:10.908 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-08-17T16:45:10.908 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-08-17T16:45:10.908 INFO:teuthology.orchestra.run.smithi063.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-08-17T16:45:10.909 INFO:teuthology.orchestra.run.smithi063.stdout: python3-mako-1.1.4-6.el9.noarch 2024-08-17T16:45:10.909 INFO:teuthology.orchestra.run.smithi063.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-08-17T16:45:10.909 INFO:teuthology.orchestra.run.smithi063.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-08-17T16:45:10.909 INFO:teuthology.orchestra.run.smithi063.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-08-17T16:45:10.909 INFO:teuthology.orchestra.run.smithi063.stdout: python3-ply-3.11-14.el9.noarch 2024-08-17T16:45:10.909 INFO:teuthology.orchestra.run.smithi063.stdout: python3-portend-3.1.0-2.el9.noarch 2024-08-17T16:45:10.909 INFO:teuthology.orchestra.run.smithi063.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-08-17T16:45:10.909 INFO:teuthology.orchestra.run.smithi063.stdout: python3-pycparser-2.20-6.el9.noarch 2024-08-17T16:45:10.909 INFO:teuthology.orchestra.run.smithi063.stdout: python3-pytz-2021.1-5.el9.noarch 2024-08-17T16:45:10.909 INFO:teuthology.orchestra.run.smithi063.stdout: python3-requests-2.25.1-8.el9.noarch 2024-08-17T16:45:10.909 INFO:teuthology.orchestra.run.smithi063.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-08-17T16:45:10.909 INFO:teuthology.orchestra.run.smithi063.stdout: python3-toml-0.10.2-6.el9.noarch 2024-08-17T16:45:10.909 INFO:teuthology.orchestra.run.smithi063.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-08-17T16:45:10.910 INFO:teuthology.orchestra.run.smithi063.stdout: python3-webob-1.8.7-6.el9.noarch 2024-08-17T16:45:10.910 INFO:teuthology.orchestra.run.smithi063.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-08-17T16:45:10.910 INFO:teuthology.orchestra.run.smithi063.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-08-17T16:45:10.910 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:10.910 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:11.217 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install ceph-base 2024-08-17T16:45:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-719.gced649f8.e 1/1 2024-08-17T16:45:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:11.229 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:45:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-immutable-object-cache-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:11.229 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:11.439 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install ceph-mgr 2024-08-17T16:45:11.799 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:00:49 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:11.887 INFO:teuthology.orchestra.run.smithi063.stdout:Package ceph-base-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:11.931 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:11.932 INFO:teuthology.orchestra.run.smithi063.stdout:Nothing to do. 2024-08-17T16:45:11.932 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:11.996 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install cephadm 2024-08-17T16:45:11.996 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:00:52 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:12.083 INFO:teuthology.orchestra.run.smithi162.stdout:Package ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:12.124 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:12.126 INFO:teuthology.orchestra.run.smithi162.stdout:Nothing to do. 2024-08-17T16:45:12.126 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:12.179 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install ceph-mgr-dashboard 2024-08-17T16:45:12.577 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:00:50 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:12.708 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:12.709 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:12.709 INFO:teuthology.orchestra.run.smithi063.stdout: Package Arch Version Repository Size 2024-08-17T16:45:12.709 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:12.709 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:45:12.709 INFO:teuthology.orchestra.run.smithi063.stdout: cephadm noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 224 k 2024-08-17T16:45:12.709 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:12.709 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:45:12.709 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:12.709 INFO:teuthology.orchestra.run.smithi063.stdout:Install 1 Package 2024-08-17T16:45:12.710 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:12.710 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 224 k 2024-08-17T16:45:12.710 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 216 k 2024-08-17T16:45:12.710 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:45:12.726 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:00:53 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:12.854 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:12.855 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:12.855 INFO:teuthology.orchestra.run.smithi162.stdout: Package Arch Version Repository Size 2024-08-17T16:45:12.855 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:12.855 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:45:12.855 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mgr-dashboard noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 3.5 M 2024-08-17T16:45:12.855 INFO:teuthology.orchestra.run.smithi162.stdout:Installing dependencies: 2024-08-17T16:45:12.855 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-grafana-dashboards noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 24 k 2024-08-17T16:45:12.855 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-prometheus-alerts noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 16 k 2024-08-17T16:45:12.855 INFO:teuthology.orchestra.run.smithi162.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-08-17T16:45:12.855 INFO:teuthology.orchestra.run.smithi162.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-08-17T16:45:12.855 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:12.855 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:45:12.856 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:12.856 INFO:teuthology.orchestra.run.smithi162.stdout:Install 5 Packages 2024-08-17T16:45:12.856 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:12.856 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 3.8 M 2024-08-17T16:45:12.856 INFO:teuthology.orchestra.run.smithi162.stdout:Installed size: 87 M 2024-08-17T16:45:12.856 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:45:12.961 INFO:teuthology.orchestra.run.smithi063.stdout:cephadm-18.2.4-719.gced649f8.el9.noarch.rpm 895 kB/s | 224 kB 00:00 2024-08-17T16:45:12.961 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:12.961 INFO:teuthology.orchestra.run.smithi063.stdout:Total 889 kB/s | 224 kB 00:00 2024-08-17T16:45:12.962 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:45:12.968 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:45:12.968 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:45:12.971 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:45:12.971 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:45:13.040 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:45:13.142 INFO:teuthology.orchestra.run.smithi162.stdout:(1/5): ceph-prometheus-alerts-18.2.4-719.gced64 79 kB/s | 16 kB 00:00 2024-08-17T16:45:13.167 INFO:teuthology.orchestra.run.smithi162.stdout:(2/5): ceph-grafana-dashboards-18.2.4-719.gced6 104 kB/s | 24 kB 00:00 2024-08-17T16:45:13.251 INFO:teuthology.orchestra.run.smithi162.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 283 kB/s | 31 kB 00:00 2024-08-17T16:45:13.284 INFO:teuthology.orchestra.run.smithi162.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 1.6 MB/s | 188 kB 00:00 2024-08-17T16:45:13.376 INFO:teuthology.orchestra.run.smithi162.stdout:(5/5): ceph-mgr-dashboard-18.2.4-719.gced649f8. 8.0 MB/s | 3.5 MB 00:00 2024-08-17T16:45:13.380 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:13.380 INFO:teuthology.orchestra.run.smithi162.stdout:Total 7.2 MB/s | 3.8 MB 00:00 2024-08-17T16:45:13.380 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:45:13.387 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:45:13.387 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:45:13.448 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:45:13.449 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:45:13.475 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:45:13.559 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:45:13.703 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:45:13.791 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-08-17T16:45:13.831 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-08-17T16:45:13.859 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-719.gced649f8.el9.no 3/5 2024-08-17T16:45:14.111 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:45:14.349 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/1 2024-08-17T16:45:14.349 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:14.349 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:45:14.349 INFO:teuthology.orchestra.run.smithi063.stdout: cephadm-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:14.349 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:14.349 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:14.550 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install ceph-immutable-object-cache 2024-08-17T16:45:14.911 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-719.gced649f8.el9.n 4/5 2024-08-17T16:45:14.955 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 5/5 2024-08-17T16:45:15.129 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:00:53 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:15.261 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:15.261 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:15.262 INFO:teuthology.orchestra.run.smithi063.stdout: Package Arch Version Repo Size 2024-08-17T16:45:15.262 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:15.262 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:45:15.262 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-719.gced649f8.el9 ceph 143 k 2024-08-17T16:45:15.262 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:15.262 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:45:15.262 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:15.262 INFO:teuthology.orchestra.run.smithi063.stdout:Install 1 Package 2024-08-17T16:45:15.262 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:15.262 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 143 k 2024-08-17T16:45:15.262 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 395 k 2024-08-17T16:45:15.262 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:45:15.475 INFO:teuthology.orchestra.run.smithi063.stdout:ceph-immutable-object-cache-18.2.4-719.gced649f 672 kB/s | 143 kB 00:00 2024-08-17T16:45:15.475 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:15.476 INFO:teuthology.orchestra.run.smithi063.stdout:Total 668 kB/s | 143 kB 00:00 2024-08-17T16:45:15.476 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:45:15.484 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:45:15.484 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:45:15.544 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:45:15.545 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:45:15.817 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:45:15.901 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-719.gced649f8.e 1/1 2024-08-17T16:45:15.928 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-719.gced649f8.e 1/1 2024-08-17T16:45:15.928 INFO:teuthology.orchestra.run.smithi063.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:45:15.928 INFO:teuthology.orchestra.run.smithi063.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-08-17T16:45:15.928 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:16.022 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 5/5 2024-08-17T16:45:16.022 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-719.gced649f8.el9.n 1/5 2024-08-17T16:45:16.022 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 2/5 2024-08-17T16:45:16.022 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-719.gced649f8.el9.no 3/5 2024-08-17T16:45:16.023 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-08-17T16:45:16.275 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-08-17T16:45:16.275 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:16.275 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:45:16.276 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-grafana-dashboards-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:16.276 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:16.276 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-prometheus-alerts-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:16.276 INFO:teuthology.orchestra.run.smithi162.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-08-17T16:45:16.276 INFO:teuthology.orchestra.run.smithi162.stdout: python3-routes-2.5.1-5.el9.noarch 2024-08-17T16:45:16.276 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:16.277 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:16.507 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install ceph-mgr-diskprediction-local 2024-08-17T16:45:16.863 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-719.gced649f8.e 1/1 2024-08-17T16:45:16.863 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:16.863 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:45:16.863 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-immutable-object-cache-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:16.863 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:16.863 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:17.053 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:00:58 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:17.078 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install ceph-mgr 2024-08-17T16:45:17.183 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:17.184 INFO:teuthology.orchestra.run.smithi162.stdout:====================================================================================== 2024-08-17T16:45:17.184 INFO:teuthology.orchestra.run.smithi162.stdout: Package Arch Version Repository Size 2024-08-17T16:45:17.184 INFO:teuthology.orchestra.run.smithi162.stdout:====================================================================================== 2024-08-17T16:45:17.184 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:45:17.184 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 7.4 M 2024-08-17T16:45:17.184 INFO:teuthology.orchestra.run.smithi162.stdout:Installing dependencies: 2024-08-17T16:45:17.185 INFO:teuthology.orchestra.run.smithi162.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-08-17T16:45:17.185 INFO:teuthology.orchestra.run.smithi162.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-08-17T16:45:17.185 INFO:teuthology.orchestra.run.smithi162.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-08-17T16:45:17.185 INFO:teuthology.orchestra.run.smithi162.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-08-17T16:45:17.185 INFO:teuthology.orchestra.run.smithi162.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-08-17T16:45:17.185 INFO:teuthology.orchestra.run.smithi162.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-08-17T16:45:17.185 INFO:teuthology.orchestra.run.smithi162.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-08-17T16:45:17.185 INFO:teuthology.orchestra.run.smithi162.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-08-17T16:45:17.185 INFO:teuthology.orchestra.run.smithi162.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-08-17T16:45:17.185 INFO:teuthology.orchestra.run.smithi162.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-08-17T16:45:17.185 INFO:teuthology.orchestra.run.smithi162.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-08-17T16:45:17.185 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:17.186 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:45:17.186 INFO:teuthology.orchestra.run.smithi162.stdout:====================================================================================== 2024-08-17T16:45:17.186 INFO:teuthology.orchestra.run.smithi162.stdout:Install 12 Packages 2024-08-17T16:45:17.186 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:17.186 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 38 M 2024-08-17T16:45:17.186 INFO:teuthology.orchestra.run.smithi162.stdout:Installed size: 209 M 2024-08-17T16:45:17.187 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:45:17.508 INFO:teuthology.orchestra.run.smithi162.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 1.3 MB/s | 195 kB 00:00 2024-08-17T16:45:17.550 INFO:teuthology.orchestra.run.smithi162.stdout:(2/12): libgfortran-11.4.1-3.el9.x86_64.rpm 4.2 MB/s | 803 kB 00:00 2024-08-17T16:45:17.634 INFO:teuthology.orchestra.run.smithi162.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 261 kB/s | 33 kB 00:00 2024-08-17T16:45:17.664 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:00:55 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:17.684 INFO:teuthology.orchestra.run.smithi162.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 363 kB/s | 18 kB 00:00 2024-08-17T16:45:17.735 INFO:teuthology.orchestra.run.smithi162.stdout:(5/12): openblas-0.3.21-2.el9.x86_64.rpm 706 kB/s | 35 kB 00:00 2024-08-17T16:45:17.754 INFO:teuthology.orchestra.run.smithi063.stdout:Package ceph-mgr-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:17.798 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:17.800 INFO:teuthology.orchestra.run.smithi063.stdout:Nothing to do. 2024-08-17T16:45:17.800 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:17.852 INFO:teuthology.orchestra.run.smithi162.stdout:(6/12): ceph-mgr-diskprediction-local-18.2.4-71 15 MB/s | 7.4 MB 00:00 2024-08-17T16:45:17.861 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install ceph-mgr-dashboard 2024-08-17T16:45:18.068 INFO:teuthology.orchestra.run.smithi162.stdout:(7/12): python3-devel-3.9.18-3.el9.x86_64.rpm 1.1 MB/s | 244 kB 00:00 2024-08-17T16:45:18.426 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:00:56 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:18.535 INFO:teuthology.orchestra.run.smithi162.stdout:(8/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 3.0 MB/s | 3.0 MB 00:00 2024-08-17T16:45:18.558 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:18.559 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:18.559 INFO:teuthology.orchestra.run.smithi063.stdout: Package Arch Version Repository Size 2024-08-17T16:45:18.559 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:18.559 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:45:18.559 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mgr-dashboard noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 3.5 M 2024-08-17T16:45:18.559 INFO:teuthology.orchestra.run.smithi063.stdout:Installing dependencies: 2024-08-17T16:45:18.559 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-grafana-dashboards noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 24 k 2024-08-17T16:45:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-prometheus-alerts noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 16 k 2024-08-17T16:45:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-08-17T16:45:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-08-17T16:45:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:18.560 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:45:18.560 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:18.560 INFO:teuthology.orchestra.run.smithi063.stdout:Install 5 Packages 2024-08-17T16:45:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:18.561 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 3.8 M 2024-08-17T16:45:18.561 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 87 M 2024-08-17T16:45:18.561 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:45:18.644 INFO:teuthology.orchestra.run.smithi162.stdout:(9/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 8.9 MB/s | 5.1 MB 00:00 2024-08-17T16:45:18.736 INFO:teuthology.orchestra.run.smithi162.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 1.2 MB/s | 250 kB 00:00 2024-08-17T16:45:18.845 INFO:teuthology.orchestra.run.smithi063.stdout:(1/5): ceph-prometheus-alerts-18.2.4-719.gced64 80 kB/s | 16 kB 00:00 2024-08-17T16:45:18.870 INFO:teuthology.orchestra.run.smithi063.stdout:(2/5): ceph-grafana-dashboards-18.2.4-719.gced6 106 kB/s | 24 kB 00:00 2024-08-17T16:45:18.954 INFO:teuthology.orchestra.run.smithi063.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 284 kB/s | 31 kB 00:00 2024-08-17T16:45:18.996 INFO:teuthology.orchestra.run.smithi063.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 1.5 MB/s | 188 kB 00:00 2024-08-17T16:45:19.019 INFO:teuthology.orchestra.run.smithi162.stdout:(11/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 43 MB/s | 16 MB 00:00 2024-08-17T16:45:19.071 INFO:teuthology.orchestra.run.smithi063.stdout:(5/5): ceph-mgr-dashboard-18.2.4-719.gced649f8. 8.2 MB/s | 3.5 MB 00:00 2024-08-17T16:45:19.074 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:19.075 INFO:teuthology.orchestra.run.smithi063.stdout:Total 7.4 MB/s | 3.8 MB 00:00 2024-08-17T16:45:19.075 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:45:19.082 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:45:19.083 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:45:19.103 INFO:teuthology.orchestra.run.smithi162.stdout:(12/12): openblas-openmp-0.3.21-2.el9.x86_64.rp 3.5 MB/s | 4.7 MB 00:01 2024-08-17T16:45:19.103 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:19.104 INFO:teuthology.orchestra.run.smithi162.stdout:Total 20 MB/s | 38 MB 00:01 2024-08-17T16:45:19.148 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:45:19.148 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:45:19.399 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:45:19.455 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:45:19.482 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:45:19.482 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:45:19.487 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-08-17T16:45:19.526 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-08-17T16:45:19.555 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-719.gced649f8.el9.no 3/5 2024-08-17T16:45:19.985 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:45:19.986 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:45:20.678 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-719.gced649f8.el9.n 4/5 2024-08-17T16:45:20.716 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 5/5 2024-08-17T16:45:20.746 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:45:20.815 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-08-17T16:45:20.864 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-08-17T16:45:20.981 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-08-17T16:45:21.019 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-08-17T16:45:21.197 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-08-17T16:45:21.226 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-08-17T16:45:21.304 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-08-17T16:45:21.789 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 5/5 2024-08-17T16:45:21.790 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-719.gced649f8.el9.n 1/5 2024-08-17T16:45:21.790 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 2/5 2024-08-17T16:45:21.790 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-719.gced649f8.el9.no 3/5 2024-08-17T16:45:21.790 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-08-17T16:45:21.963 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-08-17T16:45:22.046 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-08-17T16:45:22.046 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:22.046 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:45:22.047 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-grafana-dashboards-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:22.047 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mgr-dashboard-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:22.047 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-prometheus-alerts-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:22.047 INFO:teuthology.orchestra.run.smithi063.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-08-17T16:45:22.047 INFO:teuthology.orchestra.run.smithi063.stdout: python3-routes-2.5.1-5.el9.noarch 2024-08-17T16:45:22.047 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:22.047 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:22.091 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-08-17T16:45:22.255 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install ceph-mgr-diskprediction-local 2024-08-17T16:45:22.826 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:00 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:22.962 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:22.963 INFO:teuthology.orchestra.run.smithi063.stdout:====================================================================================== 2024-08-17T16:45:22.963 INFO:teuthology.orchestra.run.smithi063.stdout: Package Arch Version Repository Size 2024-08-17T16:45:22.963 INFO:teuthology.orchestra.run.smithi063.stdout:====================================================================================== 2024-08-17T16:45:22.964 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:45:22.964 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 7.4 M 2024-08-17T16:45:22.964 INFO:teuthology.orchestra.run.smithi063.stdout:Installing dependencies: 2024-08-17T16:45:22.964 INFO:teuthology.orchestra.run.smithi063.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-08-17T16:45:22.964 INFO:teuthology.orchestra.run.smithi063.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-08-17T16:45:22.964 INFO:teuthology.orchestra.run.smithi063.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-08-17T16:45:22.964 INFO:teuthology.orchestra.run.smithi063.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-08-17T16:45:22.964 INFO:teuthology.orchestra.run.smithi063.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-08-17T16:45:22.964 INFO:teuthology.orchestra.run.smithi063.stdout: openblas x86_64 0.3.21-2.el9 appstream 35 k 2024-08-17T16:45:22.964 INFO:teuthology.orchestra.run.smithi063.stdout: openblas-openmp x86_64 0.3.21-2.el9 appstream 4.7 M 2024-08-17T16:45:22.965 INFO:teuthology.orchestra.run.smithi063.stdout: python3-devel x86_64 3.9.18-3.el9 appstream 244 k 2024-08-17T16:45:22.965 INFO:teuthology.orchestra.run.smithi063.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-08-17T16:45:22.965 INFO:teuthology.orchestra.run.smithi063.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-08-17T16:45:22.965 INFO:teuthology.orchestra.run.smithi063.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-08-17T16:45:22.965 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:22.965 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:45:22.965 INFO:teuthology.orchestra.run.smithi063.stdout:====================================================================================== 2024-08-17T16:45:22.965 INFO:teuthology.orchestra.run.smithi063.stdout:Install 12 Packages 2024-08-17T16:45:22.965 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:22.966 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 38 M 2024-08-17T16:45:22.966 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 209 M 2024-08-17T16:45:22.966 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:45:23.319 INFO:teuthology.orchestra.run.smithi063.stdout:(1/12): libquadmath-11.4.1-3.el9.x86_64.rpm 868 kB/s | 195 kB 00:00 2024-08-17T16:45:23.342 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T16:45:23.411 INFO:teuthology.orchestra.run.smithi063.stdout:(2/12): libgfortran-11.4.1-3.el9.x86_64.rpm 2.5 MB/s | 803 kB 00:00 2024-08-17T16:45:23.553 INFO:teuthology.orchestra.run.smithi063.stdout:(3/12): ceph-mgr-diskprediction-local-18.2.4-71 16 MB/s | 7.4 MB 00:00 2024-08-17T16:45:24.045 INFO:teuthology.orchestra.run.smithi063.stdout:(4/12): flexiblas-3.0.4-8.el9.x86_64.rpm 45 kB/s | 33 kB 00:00 2024-08-17T16:45:24.103 INFO:teuthology.orchestra.run.smithi063.stdout:(5/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 33 kB/s | 18 kB 00:00 2024-08-17T16:45:24.381 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-08-17T16:45:24.402 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-719.gced649 12/12 2024-08-17T16:45:24.604 INFO:teuthology.orchestra.run.smithi063.stdout:(6/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 2.5 MB/s | 3.0 MB 00:01 2024-08-17T16:45:24.637 INFO:teuthology.orchestra.run.smithi063.stdout:(7/12): openblas-0.3.21-2.el9.x86_64.rpm 59 kB/s | 35 kB 00:00 2024-08-17T16:45:25.229 INFO:teuthology.orchestra.run.smithi063.stdout:(8/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 4.2 MB/s | 4.7 MB 00:01 2024-08-17T16:45:25.379 INFO:teuthology.orchestra.run.smithi063.stdout:(9/12): python3-devel-3.9.18-3.el9.x86_64.rpm 315 kB/s | 244 kB 00:00 2024-08-17T16:45:25.788 INFO:teuthology.orchestra.run.smithi063.stdout:(10/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 4.4 MB/s | 5.1 MB 00:01 2024-08-17T16:45:26.096 INFO:teuthology.orchestra.run.smithi063.stdout:(11/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 289 kB/s | 250 kB 00:00 2024-08-17T16:45:26.896 INFO:teuthology.orchestra.run.smithi063.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 11 MB/s | 16 MB 00:01 2024-08-17T16:45:26.898 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:26.898 INFO:teuthology.orchestra.run.smithi063.stdout:Total 9.7 MB/s | 38 MB 00:03 2024-08-17T16:45:26.917 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-719.gced649 12/12 2024-08-17T16:45:26.917 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-719.gced649 1/12 2024-08-17T16:45:26.918 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-08-17T16:45:26.918 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-08-17T16:45:26.918 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-08-17T16:45:26.918 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-08-17T16:45:26.918 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-08-17T16:45:26.919 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-08-17T16:45:26.919 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-08-17T16:45:26.919 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-08-17T16:45:26.919 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T16:45:26.919 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-08-17T16:45:27.267 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:45:27.296 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:45:27.297 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:45:27.312 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-08-17T16:45:27.312 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:27.313 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:45:27.313 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mgr-diskprediction-local-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:27.313 INFO:teuthology.orchestra.run.smithi162.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-08-17T16:45:27.313 INFO:teuthology.orchestra.run.smithi162.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-08-17T16:45:27.313 INFO:teuthology.orchestra.run.smithi162.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-08-17T16:45:27.313 INFO:teuthology.orchestra.run.smithi162.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-08-17T16:45:27.313 INFO:teuthology.orchestra.run.smithi162.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-08-17T16:45:27.313 INFO:teuthology.orchestra.run.smithi162.stdout: openblas-0.3.21-2.el9.x86_64 2024-08-17T16:45:27.313 INFO:teuthology.orchestra.run.smithi162.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-08-17T16:45:27.313 INFO:teuthology.orchestra.run.smithi162.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-08-17T16:45:27.313 INFO:teuthology.orchestra.run.smithi162.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-08-17T16:45:27.313 INFO:teuthology.orchestra.run.smithi162.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-08-17T16:45:27.314 INFO:teuthology.orchestra.run.smithi162.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-08-17T16:45:27.314 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:27.314 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:27.723 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install ceph-mgr-rook 2024-08-17T16:45:27.833 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:45:27.834 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:45:28.276 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:09 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:28.404 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:28.405 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:28.406 INFO:teuthology.orchestra.run.smithi162.stdout: Package Arch Version Repository Size 2024-08-17T16:45:28.406 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:28.406 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:45:28.406 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mgr-rook noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 49 k 2024-08-17T16:45:28.406 INFO:teuthology.orchestra.run.smithi162.stdout:Installing dependencies: 2024-08-17T16:45:28.406 INFO:teuthology.orchestra.run.smithi162.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-08-17T16:45:28.406 INFO:teuthology.orchestra.run.smithi162.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-08-17T16:45:28.406 INFO:teuthology.orchestra.run.smithi162.stdout: python3-google-auth noarch 1:2.33.0-1.el9 epel 220 k 2024-08-17T16:45:28.406 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-08-17T16:45:28.406 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-08-17T16:45:28.406 INFO:teuthology.orchestra.run.smithi162.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-08-17T16:45:28.407 INFO:teuthology.orchestra.run.smithi162.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-08-17T16:45:28.407 INFO:teuthology.orchestra.run.smithi162.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-08-17T16:45:28.407 INFO:teuthology.orchestra.run.smithi162.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-08-17T16:45:28.407 INFO:teuthology.orchestra.run.smithi162.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-08-17T16:45:28.407 INFO:teuthology.orchestra.run.smithi162.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-08-17T16:45:28.407 INFO:teuthology.orchestra.run.smithi162.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-08-17T16:45:28.407 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:28.407 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:45:28.407 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:28.407 INFO:teuthology.orchestra.run.smithi162.stdout:Install 13 Packages 2024-08-17T16:45:28.407 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:28.408 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 2.2 M 2024-08-17T16:45:28.408 INFO:teuthology.orchestra.run.smithi162.stdout:Installed size: 26 M 2024-08-17T16:45:28.408 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:45:28.619 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:45:28.710 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-08-17T16:45:28.757 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 2/12 2024-08-17T16:45:28.821 INFO:teuthology.orchestra.run.smithi162.stdout:(1/13): ceph-mgr-rook-18.2.4-719.gced649f8.el9. 275 kB/s | 49 kB 00:00 2024-08-17T16:45:28.860 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 3/12 2024-08-17T16:45:28.896 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-devel-3.9.18-3.el9.x86_64 4/12 2024-08-17T16:45:29.081 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-08-17T16:45:29.111 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-08-17T16:45:29.146 INFO:teuthology.orchestra.run.smithi162.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 38 kB/s | 19 kB 00:00 2024-08-17T16:45:29.190 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-08-17T16:45:29.238 INFO:teuthology.orchestra.run.smithi162.stdout:(3/13): python3-jsonpatch-1.21-16.el9.noarch.rp 43 kB/s | 26 kB 00:00 2024-08-17T16:45:29.597 INFO:teuthology.orchestra.run.smithi162.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 286 kB/s | 222 kB 00:00 2024-08-17T16:45:29.880 INFO:teuthology.orchestra.run.smithi162.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 217 kB/s | 159 kB 00:00 2024-08-17T16:45:29.909 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-08-17T16:45:29.939 INFO:teuthology.orchestra.run.smithi162.stdout:(6/13): python3-cachetools-4.2.4-1.el9.noarch.r 556 kB/s | 32 kB 00:00 2024-08-17T16:45:29.964 INFO:teuthology.orchestra.run.smithi162.stdout:(7/13): python3-certifi-2023.05.07-4.el9.noarch 566 kB/s | 14 kB 00:00 2024-08-17T16:45:29.989 INFO:teuthology.orchestra.run.smithi162.stdout:(8/13): python3-google-auth-2.33.0-1.el9.noarch 8.6 MB/s | 220 kB 00:00 2024-08-17T16:45:30.031 INFO:teuthology.orchestra.run.smithi162.stdout:(9/13): python3-kubernetes-26.1.0-2.el9.noarch. 25 MB/s | 1.0 MB 00:00 2024-08-17T16:45:30.038 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-08-17T16:45:30.057 INFO:teuthology.orchestra.run.smithi162.stdout:(10/13): python3-pyasn1-modules-0.4.8-6.el9.noa 341 kB/s | 279 kB 00:00 2024-08-17T16:45:30.082 INFO:teuthology.orchestra.run.smithi162.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 1.2 MB/s | 59 kB 00:00 2024-08-17T16:45:30.140 INFO:teuthology.orchestra.run.smithi162.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.1 MB/s | 90 kB 00:00 2024-08-17T16:45:30.224 INFO:teuthology.orchestra.run.smithi162.stdout:(13/13): python3-requests-oauthlib-1.3.0-12.el9 86 kB/s | 54 kB 00:00 2024-08-17T16:45:30.227 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:30.227 INFO:teuthology.orchestra.run.smithi162.stdout:Total 1.2 MB/s | 2.2 MB 00:01 2024-08-17T16:45:30.292 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:45:30.307 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:45:30.307 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:45:30.392 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:45:30.392 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:45:30.897 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:45:31.056 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-08-17T16:45:31.109 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-08-17T16:45:31.166 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-08-17T16:45:31.203 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-08-17T16:45:31.244 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-08-17T16:45:31.346 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-08-17T16:45:31.384 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T16:45:31.448 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-google-auth-1:2.33.0-1.el9.noarch 7/13 2024-08-17T16:45:31.510 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-08-17T16:45:32.237 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-08-17T16:45:32.291 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-08-17T16:45:32.324 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-08-17T16:45:32.373 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-08-17T16:45:32.403 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 13/13 2024-08-17T16:45:32.473 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-08-17T16:45:32.501 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-719.gced649 12/12 2024-08-17T16:45:33.232 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 13/13 2024-08-17T16:45:33.232 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 1/13 2024-08-17T16:45:33.233 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-08-17T16:45:33.233 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-08-17T16:45:33.233 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-08-17T16:45:33.233 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-08-17T16:45:33.233 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-08-17T16:45:33.233 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-08-17T16:45:33.233 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-08-17T16:45:33.233 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-08-17T16:45:33.233 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-google-auth-1:2.33.0-1.el9.noarch 10/13 2024-08-17T16:45:33.233 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-08-17T16:45:33.233 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-08-17T16:45:33.557 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-08-17T16:45:33.557 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:33.557 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:45:33.558 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:33.558 INFO:teuthology.orchestra.run.smithi162.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-08-17T16:45:33.558 INFO:teuthology.orchestra.run.smithi162.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-08-17T16:45:33.558 INFO:teuthology.orchestra.run.smithi162.stdout: python3-google-auth-1:2.33.0-1.el9.noarch 2024-08-17T16:45:33.558 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-08-17T16:45:33.558 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-08-17T16:45:33.558 INFO:teuthology.orchestra.run.smithi162.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-08-17T16:45:33.558 INFO:teuthology.orchestra.run.smithi162.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-08-17T16:45:33.558 INFO:teuthology.orchestra.run.smithi162.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-08-17T16:45:33.558 INFO:teuthology.orchestra.run.smithi162.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-08-17T16:45:33.558 INFO:teuthology.orchestra.run.smithi162.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-08-17T16:45:33.559 INFO:teuthology.orchestra.run.smithi162.stdout: python3-rsa-4.9-2.el9.noarch 2024-08-17T16:45:33.559 INFO:teuthology.orchestra.run.smithi162.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-08-17T16:45:33.559 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:33.559 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:33.828 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install ceph-mgr-cephadm 2024-08-17T16:45:34.390 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:15 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:34.517 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:34.518 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:34.518 INFO:teuthology.orchestra.run.smithi162.stdout: Package Arch Version Repository Size 2024-08-17T16:45:34.518 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:34.519 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:45:34.519 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mgr-cephadm noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 139 k 2024-08-17T16:45:34.519 INFO:teuthology.orchestra.run.smithi162.stdout:Installing dependencies: 2024-08-17T16:45:34.519 INFO:teuthology.orchestra.run.smithi162.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-08-17T16:45:34.519 INFO:teuthology.orchestra.run.smithi162.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-08-17T16:45:34.519 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-08-17T16:45:34.519 INFO:teuthology.orchestra.run.smithi162.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-08-17T16:45:34.519 INFO:teuthology.orchestra.run.smithi162.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-08-17T16:45:34.519 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:34.519 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:45:34.519 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:34.520 INFO:teuthology.orchestra.run.smithi162.stdout:Install 6 Packages 2024-08-17T16:45:34.520 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:34.520 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 7.0 M 2024-08-17T16:45:34.520 INFO:teuthology.orchestra.run.smithi162.stdout:Installed size: 33 M 2024-08-17T16:45:34.520 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:45:34.676 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-719.gced649 12/12 2024-08-17T16:45:34.677 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-719.gced649 1/12 2024-08-17T16:45:34.677 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-08-17T16:45:34.677 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 3/12 2024-08-17T16:45:34.677 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-08-17T16:45:34.677 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-08-17T16:45:34.677 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-08-17T16:45:34.677 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 7/12 2024-08-17T16:45:34.677 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 8/12 2024-08-17T16:45:34.677 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-devel-3.9.18-3.el9.x86_64 9/12 2024-08-17T16:45:34.677 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-08-17T16:45:34.678 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-08-17T16:45:35.066 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-08-17T16:45:35.066 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:35.066 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:45:35.066 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mgr-diskprediction-local-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:35.066 INFO:teuthology.orchestra.run.smithi063.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-08-17T16:45:35.066 INFO:teuthology.orchestra.run.smithi063.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-08-17T16:45:35.066 INFO:teuthology.orchestra.run.smithi063.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-08-17T16:45:35.066 INFO:teuthology.orchestra.run.smithi063.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-08-17T16:45:35.067 INFO:teuthology.orchestra.run.smithi063.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-08-17T16:45:35.067 INFO:teuthology.orchestra.run.smithi063.stdout: openblas-0.3.21-2.el9.x86_64 2024-08-17T16:45:35.067 INFO:teuthology.orchestra.run.smithi063.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-08-17T16:45:35.067 INFO:teuthology.orchestra.run.smithi063.stdout: python3-devel-3.9.18-3.el9.x86_64 2024-08-17T16:45:35.067 INFO:teuthology.orchestra.run.smithi063.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-08-17T16:45:35.067 INFO:teuthology.orchestra.run.smithi063.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-08-17T16:45:35.067 INFO:teuthology.orchestra.run.smithi063.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-08-17T16:45:35.067 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:35.067 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:35.087 INFO:teuthology.orchestra.run.smithi162.stdout:(1/6): ceph-mgr-cephadm-18.2.4-719.gced649f8.el 597 kB/s | 139 kB 00:00 2024-08-17T16:45:35.162 INFO:teuthology.orchestra.run.smithi162.stdout:(2/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 7.1 MB/s | 548 kB 00:00 2024-08-17T16:45:35.186 INFO:teuthology.orchestra.run.smithi162.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 2.4 MB/s | 58 kB 00:00 2024-08-17T16:45:35.203 INFO:teuthology.orchestra.run.smithi162.stdout:(4/6): python3-typing-extensions-4.12.2-2.el9.n 4.3 MB/s | 74 kB 00:00 2024-08-17T16:45:35.449 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install ceph-mgr-rook 2024-08-17T16:45:35.453 INFO:teuthology.orchestra.run.smithi162.stdout:(5/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 415 kB/s | 248 kB 00:00 2024-08-17T16:45:36.025 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:14 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:36.158 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:36.160 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:36.160 INFO:teuthology.orchestra.run.smithi063.stdout: Package Arch Version Repository Size 2024-08-17T16:45:36.160 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:36.160 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:45:36.160 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mgr-rook noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 49 k 2024-08-17T16:45:36.160 INFO:teuthology.orchestra.run.smithi063.stdout:Installing dependencies: 2024-08-17T16:45:36.160 INFO:teuthology.orchestra.run.smithi063.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-08-17T16:45:36.160 INFO:teuthology.orchestra.run.smithi063.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-08-17T16:45:36.161 INFO:teuthology.orchestra.run.smithi063.stdout: python3-google-auth noarch 1:2.33.0-1.el9 epel 220 k 2024-08-17T16:45:36.161 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-08-17T16:45:36.161 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-08-17T16:45:36.161 INFO:teuthology.orchestra.run.smithi063.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-08-17T16:45:36.161 INFO:teuthology.orchestra.run.smithi063.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-08-17T16:45:36.161 INFO:teuthology.orchestra.run.smithi063.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-08-17T16:45:36.161 INFO:teuthology.orchestra.run.smithi063.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-08-17T16:45:36.161 INFO:teuthology.orchestra.run.smithi063.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-08-17T16:45:36.161 INFO:teuthology.orchestra.run.smithi063.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-08-17T16:45:36.161 INFO:teuthology.orchestra.run.smithi063.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-08-17T16:45:36.161 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:36.161 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:45:36.161 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:36.162 INFO:teuthology.orchestra.run.smithi063.stdout:Install 13 Packages 2024-08-17T16:45:36.162 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:36.162 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 2.2 M 2024-08-17T16:45:36.162 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 26 M 2024-08-17T16:45:36.162 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:45:36.400 INFO:teuthology.orchestra.run.smithi063.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 206 kB/s | 19 kB 00:00 2024-08-17T16:45:36.425 INFO:teuthology.orchestra.run.smithi063.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 217 kB/s | 26 kB 00:00 2024-08-17T16:45:36.517 INFO:teuthology.orchestra.run.smithi063.stdout:(3/13): ceph-mgr-rook-18.2.4-719.gced649f8.el9. 235 kB/s | 49 kB 00:00 2024-08-17T16:45:36.543 INFO:teuthology.orchestra.run.smithi063.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.5 MB/s | 222 kB 00:00 2024-08-17T16:45:36.568 INFO:teuthology.orchestra.run.smithi063.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.1 MB/s | 159 kB 00:00 2024-08-17T16:45:36.593 INFO:teuthology.orchestra.run.smithi063.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 1.0 MB/s | 54 kB 00:00 2024-08-17T16:45:36.677 INFO:teuthology.orchestra.run.smithi063.stdout:(7/13): python3-cachetools-4.2.4-1.el9.noarch.r 296 kB/s | 32 kB 00:00 2024-08-17T16:45:36.700 INFO:teuthology.orchestra.run.smithi063.stdout:(8/13): python3-pyasn1-modules-0.4.8-6.el9.noar 1.5 MB/s | 279 kB 00:00 2024-08-17T16:45:36.726 INFO:teuthology.orchestra.run.smithi063.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 106 kB/s | 14 kB 00:00 2024-08-17T16:45:36.776 INFO:teuthology.orchestra.run.smithi063.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 2.4 MB/s | 59 kB 00:00 2024-08-17T16:45:36.810 INFO:teuthology.orchestra.run.smithi063.stdout:(11/13): python3-google-auth-2.33.0-1.el9.noarc 1.6 MB/s | 220 kB 00:00 2024-08-17T16:45:36.835 INFO:teuthology.orchestra.run.smithi063.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.6 MB/s | 90 kB 00:00 2024-08-17T16:45:36.885 INFO:teuthology.orchestra.run.smithi063.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 5.5 MB/s | 1.0 MB 00:00 2024-08-17T16:45:36.888 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:36.888 INFO:teuthology.orchestra.run.smithi063.stdout:Total 3.0 MB/s | 2.2 MB 00:00 2024-08-17T16:45:36.954 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:45:36.969 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:45:36.969 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:45:37.067 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:45:37.068 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:45:37.370 INFO:teuthology.orchestra.run.smithi162.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 2.4 MB/s | 6.0 MB 00:02 2024-08-17T16:45:37.372 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:37.372 INFO:teuthology.orchestra.run.smithi162.stdout:Total 2.5 MB/s | 7.0 MB 00:02 2024-08-17T16:45:37.446 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:45:37.458 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:45:37.458 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:45:37.556 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:45:37.556 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:45:37.601 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:45:37.775 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-08-17T16:45:37.842 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-08-17T16:45:37.858 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:45:37.899 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-08-17T16:45:37.935 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-08-17T16:45:37.977 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-08-17T16:45:38.044 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-08-17T16:45:38.088 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-08-17T16:45:38.091 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-08-17T16:45:38.201 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-google-auth-1:2.33.0-1.el9.noarch 7/13 2024-08-17T16:45:38.261 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-08-17T16:45:38.607 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-08-17T16:45:38.700 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-08-17T16:45:38.762 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-08-17T16:45:38.793 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 6/6 2024-08-17T16:45:39.021 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-08-17T16:45:39.068 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-08-17T16:45:39.101 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-08-17T16:45:39.150 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-08-17T16:45:39.179 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 13/13 2024-08-17T16:45:39.629 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 6/6 2024-08-17T16:45:39.629 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/6 2024-08-17T16:45:39.629 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-08-17T16:45:39.629 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-08-17T16:45:39.629 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-08-17T16:45:39.629 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-08-17T16:45:39.889 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-08-17T16:45:39.889 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:39.889 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:45:39.889 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:39.889 INFO:teuthology.orchestra.run.smithi162.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-08-17T16:45:39.889 INFO:teuthology.orchestra.run.smithi162.stdout: python3-babel-2.9.1-2.el9.noarch 2024-08-17T16:45:39.890 INFO:teuthology.orchestra.run.smithi162.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-08-17T16:45:39.890 INFO:teuthology.orchestra.run.smithi162.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-08-17T16:45:39.890 INFO:teuthology.orchestra.run.smithi162.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-08-17T16:45:39.890 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:39.890 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:39.958 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 13/13 2024-08-17T16:45:39.958 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 1/13 2024-08-17T16:45:39.958 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-08-17T16:45:39.958 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-08-17T16:45:39.958 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-08-17T16:45:39.958 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-08-17T16:45:39.958 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-08-17T16:45:39.958 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-08-17T16:45:39.958 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-08-17T16:45:39.958 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-08-17T16:45:39.959 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-google-auth-1:2.33.0-1.el9.noarch 10/13 2024-08-17T16:45:39.959 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-08-17T16:45:39.959 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-08-17T16:45:40.154 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install ceph-fuse 2024-08-17T16:45:40.288 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-08-17T16:45:40.289 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:40.289 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:45:40.289 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mgr-rook-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:40.289 INFO:teuthology.orchestra.run.smithi063.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-08-17T16:45:40.289 INFO:teuthology.orchestra.run.smithi063.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-08-17T16:45:40.289 INFO:teuthology.orchestra.run.smithi063.stdout: python3-google-auth-1:2.33.0-1.el9.noarch 2024-08-17T16:45:40.289 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-08-17T16:45:40.289 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-08-17T16:45:40.289 INFO:teuthology.orchestra.run.smithi063.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-08-17T16:45:40.289 INFO:teuthology.orchestra.run.smithi063.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-08-17T16:45:40.289 INFO:teuthology.orchestra.run.smithi063.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-08-17T16:45:40.289 INFO:teuthology.orchestra.run.smithi063.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-08-17T16:45:40.290 INFO:teuthology.orchestra.run.smithi063.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-08-17T16:45:40.290 INFO:teuthology.orchestra.run.smithi063.stdout: python3-rsa-4.9-2.el9.noarch 2024-08-17T16:45:40.290 INFO:teuthology.orchestra.run.smithi063.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-08-17T16:45:40.290 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:40.290 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:40.535 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install ceph-mgr-cephadm 2024-08-17T16:45:40.709 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:21 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:40.838 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:40.838 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:40.839 INFO:teuthology.orchestra.run.smithi162.stdout: Package Arch Version Repository Size 2024-08-17T16:45:40.839 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:40.839 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:45:40.839 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-fuse x86_64 2:18.2.4-719.gced649f8.el9 ceph 852 k 2024-08-17T16:45:40.839 INFO:teuthology.orchestra.run.smithi162.stdout:Installing dependencies: 2024-08-17T16:45:40.839 INFO:teuthology.orchestra.run.smithi162.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-08-17T16:45:40.839 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:40.839 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:45:40.839 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:40.839 INFO:teuthology.orchestra.run.smithi162.stdout:Install 2 Packages 2024-08-17T16:45:40.839 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:40.840 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 931 k 2024-08-17T16:45:40.840 INFO:teuthology.orchestra.run.smithi162.stdout:Installed size: 2.7 M 2024-08-17T16:45:40.840 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:45:41.115 INFO:teuthology.orchestra.run.smithi162.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 507 kB/s | 80 kB 00:00 2024-08-17T16:45:41.120 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:19 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:41.248 INFO:teuthology.orchestra.run.smithi162.stdout:(2/2): ceph-fuse-18.2.4-719.gced649f8.el9.x86_6 2.9 MB/s | 852 kB 00:00 2024-08-17T16:45:41.249 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:41.249 INFO:teuthology.orchestra.run.smithi162.stdout:Total 2.2 MB/s | 931 kB 00:00 2024-08-17T16:45:41.254 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:41.255 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:41.255 INFO:teuthology.orchestra.run.smithi063.stdout: Package Arch Version Repository Size 2024-08-17T16:45:41.255 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:41.255 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:45:41.255 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mgr-cephadm noarch 2:18.2.4-719.gced649f8.el9 ceph-noarch 139 k 2024-08-17T16:45:41.255 INFO:teuthology.orchestra.run.smithi063.stdout:Installing dependencies: 2024-08-17T16:45:41.256 INFO:teuthology.orchestra.run.smithi063.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-08-17T16:45:41.256 INFO:teuthology.orchestra.run.smithi063.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-08-17T16:45:41.256 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-08-17T16:45:41.256 INFO:teuthology.orchestra.run.smithi063.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-08-17T16:45:41.256 INFO:teuthology.orchestra.run.smithi063.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-08-17T16:45:41.256 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:41.256 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:45:41.256 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:41.256 INFO:teuthology.orchestra.run.smithi063.stdout:Install 6 Packages 2024-08-17T16:45:41.256 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:41.257 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 7.0 M 2024-08-17T16:45:41.257 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 33 M 2024-08-17T16:45:41.257 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:45:41.261 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:45:41.283 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:45:41.283 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:45:41.362 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:45:41.363 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:45:41.511 INFO:teuthology.orchestra.run.smithi063.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.7 MB/s | 248 kB 00:00 2024-08-17T16:45:41.554 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:45:41.577 INFO:teuthology.orchestra.run.smithi063.stdout:(2/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 8.0 MB/s | 548 kB 00:00 2024-08-17T16:45:41.636 INFO:teuthology.orchestra.run.smithi063.stdout:(3/6): ceph-mgr-cephadm-18.2.4-719.gced649f8.el 524 kB/s | 139 kB 00:00 2024-08-17T16:45:41.660 INFO:teuthology.orchestra.run.smithi063.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 691 kB/s | 58 kB 00:00 2024-08-17T16:45:41.689 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-08-17T16:45:41.719 INFO:teuthology.orchestra.run.smithi063.stdout:(5/6): python3-typing-extensions-4.12.2-2.el9.n 893 kB/s | 74 kB 00:00 2024-08-17T16:45:41.756 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 2/2 2024-08-17T16:45:41.802 INFO:teuthology.orchestra.run.smithi063.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 14 MB/s | 6.0 MB 00:00 2024-08-17T16:45:41.805 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:41.805 INFO:teuthology.orchestra.run.smithi063.stdout:Total 13 MB/s | 7.0 MB 00:00 2024-08-17T16:45:41.879 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:45:41.891 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:45:41.891 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:45:41.996 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:45:41.996 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:45:42.306 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:45:42.421 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 2/2 2024-08-17T16:45:42.422 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/2 2024-08-17T16:45:42.507 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-08-17T16:45:42.572 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-08-17T16:45:42.670 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-08-17T16:45:42.670 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:42.670 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:45:42.670 INFO:teuthology.orchestra.run.smithi162.stdout: ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-08-17T16:45:42.670 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:42.670 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:42.931 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install librados-devel 2024-08-17T16:45:43.139 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-08-17T16:45:43.228 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-08-17T16:45:43.300 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-08-17T16:45:43.328 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 6/6 2024-08-17T16:45:43.503 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:24 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:43.642 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:43.643 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:43.643 INFO:teuthology.orchestra.run.smithi162.stdout: Package Arch Version Repo Size 2024-08-17T16:45:43.643 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:43.643 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:45:43.643 INFO:teuthology.orchestra.run.smithi162.stdout: librados-devel x86_64 2:18.2.4-719.gced649f8.el9 ceph 127 k 2024-08-17T16:45:43.643 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:43.643 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:45:43.643 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:43.643 INFO:teuthology.orchestra.run.smithi162.stdout:Install 1 Package 2024-08-17T16:45:43.643 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:43.644 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 127 k 2024-08-17T16:45:43.644 INFO:teuthology.orchestra.run.smithi162.stdout:Installed size: 456 k 2024-08-17T16:45:43.644 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:45:43.859 INFO:teuthology.orchestra.run.smithi162.stdout:librados-devel-18.2.4-719.gced649f8.el9.x86_64. 590 kB/s | 127 kB 00:00 2024-08-17T16:45:43.859 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:43.860 INFO:teuthology.orchestra.run.smithi162.stdout:Total 586 kB/s | 127 kB 00:00 2024-08-17T16:45:43.860 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:45:43.865 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:45:43.865 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:45:43.929 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:45:43.929 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:45:44.080 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:45:44.108 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 6/6 2024-08-17T16:45:44.109 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 1/6 2024-08-17T16:45:44.109 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-08-17T16:45:44.109 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-08-17T16:45:44.109 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-08-17T16:45:44.109 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-08-17T16:45:44.176 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:44.363 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-08-17T16:45:44.363 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:44.363 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:45:44.363 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-mgr-cephadm-2:18.2.4-719.gced649f8.el9.noarch 2024-08-17T16:45:44.363 INFO:teuthology.orchestra.run.smithi063.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-08-17T16:45:44.363 INFO:teuthology.orchestra.run.smithi063.stdout: python3-babel-2.9.1-2.el9.noarch 2024-08-17T16:45:44.363 INFO:teuthology.orchestra.run.smithi063.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-08-17T16:45:44.363 INFO:teuthology.orchestra.run.smithi063.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-08-17T16:45:44.363 INFO:teuthology.orchestra.run.smithi063.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-08-17T16:45:44.364 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:44.364 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:44.581 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install ceph-fuse 2024-08-17T16:45:44.610 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:44.861 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:44.861 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:44.862 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:45:44.862 INFO:teuthology.orchestra.run.smithi162.stdout: librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:44.862 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:44.862 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:45.109 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install libcephfs2 2024-08-17T16:45:45.164 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:23 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:45.300 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:45.301 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:45.301 INFO:teuthology.orchestra.run.smithi063.stdout: Package Arch Version Repository Size 2024-08-17T16:45:45.301 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:45.301 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:45:45.301 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-fuse x86_64 2:18.2.4-719.gced649f8.el9 ceph 852 k 2024-08-17T16:45:45.301 INFO:teuthology.orchestra.run.smithi063.stdout:Installing dependencies: 2024-08-17T16:45:45.301 INFO:teuthology.orchestra.run.smithi063.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-08-17T16:45:45.301 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:45.301 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:45:45.301 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:45.302 INFO:teuthology.orchestra.run.smithi063.stdout:Install 2 Packages 2024-08-17T16:45:45.302 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:45.302 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 931 k 2024-08-17T16:45:45.302 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 2.7 M 2024-08-17T16:45:45.302 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:45:45.649 INFO:teuthology.orchestra.run.smithi063.stdout:(1/2): ceph-fuse-18.2.4-719.gced649f8.el9.x86_6 2.7 MB/s | 852 kB 00:00 2024-08-17T16:45:45.664 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:26 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:45.682 INFO:teuthology.orchestra.run.smithi063.stdout:(2/2): fuse-2.9.9-16.el9.x86_64.rpm 234 kB/s | 80 kB 00:00 2024-08-17T16:45:45.683 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:45.683 INFO:teuthology.orchestra.run.smithi063.stdout:Total 2.4 MB/s | 931 kB 00:00 2024-08-17T16:45:45.694 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:45:45.719 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:45:45.719 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:45:45.751 INFO:teuthology.orchestra.run.smithi162.stdout:Package libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:45.794 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:45.795 INFO:teuthology.orchestra.run.smithi162.stdout:Nothing to do. 2024-08-17T16:45:45.795 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:45.802 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:45:45.803 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:45:45.858 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install libcephfs-devel 2024-08-17T16:45:45.996 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:45:46.135 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-08-17T16:45:46.209 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 2/2 2024-08-17T16:45:46.413 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:27 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:46.542 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:46.543 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:46.543 INFO:teuthology.orchestra.run.smithi162.stdout: Package Arch Version Repo Size 2024-08-17T16:45:46.543 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:46.543 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:45:46.543 INFO:teuthology.orchestra.run.smithi162.stdout: libcephfs-devel x86_64 2:18.2.4-719.gced649f8.el9 ceph 31 k 2024-08-17T16:45:46.543 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:46.543 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:45:46.543 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:46.543 INFO:teuthology.orchestra.run.smithi162.stdout:Install 1 Package 2024-08-17T16:45:46.543 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:46.544 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 31 k 2024-08-17T16:45:46.544 INFO:teuthology.orchestra.run.smithi162.stdout:Installed size: 139 k 2024-08-17T16:45:46.544 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:45:46.710 INFO:teuthology.orchestra.run.smithi162.stdout:libcephfs-devel-18.2.4-719.gced649f8.el9.x86_64 187 kB/s | 31 kB 00:00 2024-08-17T16:45:46.710 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:46.710 INFO:teuthology.orchestra.run.smithi162.stdout:Total 186 kB/s | 31 kB 00:00 2024-08-17T16:45:46.711 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:45:46.714 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:45:46.714 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:45:46.739 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:45:46.739 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:45:46.816 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:45:46.863 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 2/2 2024-08-17T16:45:46.863 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/2 2024-08-17T16:45:46.923 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:47.161 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-08-17T16:45:47.161 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:47.161 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:45:47.161 INFO:teuthology.orchestra.run.smithi063.stdout: ceph-fuse-2:18.2.4-719.gced649f8.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-08-17T16:45:47.161 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:47.162 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:47.298 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:47.446 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install librados-devel 2024-08-17T16:45:47.542 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:47.542 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:47.542 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:45:47.542 INFO:teuthology.orchestra.run.smithi162.stdout: libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:47.542 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:47.542 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:47.733 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install librados2 2024-08-17T16:45:48.037 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:26 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:48.170 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:48.171 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:48.171 INFO:teuthology.orchestra.run.smithi063.stdout: Package Arch Version Repo Size 2024-08-17T16:45:48.171 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:48.171 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:45:48.171 INFO:teuthology.orchestra.run.smithi063.stdout: librados-devel x86_64 2:18.2.4-719.gced649f8.el9 ceph 127 k 2024-08-17T16:45:48.171 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:48.172 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:45:48.172 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:48.172 INFO:teuthology.orchestra.run.smithi063.stdout:Install 1 Package 2024-08-17T16:45:48.172 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:48.172 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 127 k 2024-08-17T16:45:48.172 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 456 k 2024-08-17T16:45:48.172 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:45:48.294 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:29 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:48.380 INFO:teuthology.orchestra.run.smithi162.stdout:Package librados2-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:48.399 INFO:teuthology.orchestra.run.smithi063.stdout:librados-devel-18.2.4-719.gced649f8.el9.x86_64. 559 kB/s | 127 kB 00:00 2024-08-17T16:45:48.461 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:48.461 INFO:teuthology.orchestra.run.smithi162.stdout:Nothing to do. 2024-08-17T16:45:48.461 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:48.461 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:48.461 INFO:teuthology.orchestra.run.smithi063.stdout:Total 555 kB/s | 127 kB 00:00 2024-08-17T16:45:48.462 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:45:48.462 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:45:48.462 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:45:48.473 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:45:48.473 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:45:48.491 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install librbd1 2024-08-17T16:45:48.629 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:45:48.734 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:49.044 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:30 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:49.129 INFO:teuthology.orchestra.run.smithi162.stdout:Package librbd1-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:49.172 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:49.173 INFO:teuthology.orchestra.run.smithi162.stdout:Nothing to do. 2024-08-17T16:45:49.173 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:49.192 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:49.241 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install python3-rados 2024-08-17T16:45:49.443 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:49.443 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:49.444 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:45:49.444 INFO:teuthology.orchestra.run.smithi063.stdout: librados-devel-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:49.444 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:49.444 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:49.715 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install libcephfs2 2024-08-17T16:45:49.789 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:30 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:49.875 INFO:teuthology.orchestra.run.smithi162.stdout:Package python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:49.917 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:49.918 INFO:teuthology.orchestra.run.smithi162.stdout:Nothing to do. 2024-08-17T16:45:49.918 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:49.983 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install python3-rgw 2024-08-17T16:45:50.292 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:28 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:50.383 INFO:teuthology.orchestra.run.smithi063.stdout:Package libcephfs2-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:50.428 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:50.429 INFO:teuthology.orchestra.run.smithi063.stdout:Nothing to do. 2024-08-17T16:45:50.429 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:50.487 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install libcephfs-devel 2024-08-17T16:45:50.535 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:31 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:50.621 INFO:teuthology.orchestra.run.smithi162.stdout:Package python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:50.663 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:50.664 INFO:teuthology.orchestra.run.smithi162.stdout:Nothing to do. 2024-08-17T16:45:50.664 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:50.735 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install python3-cephfs 2024-08-17T16:45:51.067 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:29 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:51.200 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:51.201 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:51.201 INFO:teuthology.orchestra.run.smithi063.stdout: Package Arch Version Repo Size 2024-08-17T16:45:51.201 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:51.201 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:45:51.201 INFO:teuthology.orchestra.run.smithi063.stdout: libcephfs-devel x86_64 2:18.2.4-719.gced649f8.el9 ceph 31 k 2024-08-17T16:45:51.202 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:51.202 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:45:51.202 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:51.202 INFO:teuthology.orchestra.run.smithi063.stdout:Install 1 Package 2024-08-17T16:45:51.202 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:51.202 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 31 k 2024-08-17T16:45:51.202 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 139 k 2024-08-17T16:45:51.202 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:45:51.292 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:32 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:51.372 INFO:teuthology.orchestra.run.smithi063.stdout:libcephfs-devel-18.2.4-719.gced649f8.el9.x86_64 183 kB/s | 31 kB 00:00 2024-08-17T16:45:51.372 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:51.372 INFO:teuthology.orchestra.run.smithi063.stdout:Total 182 kB/s | 31 kB 00:00 2024-08-17T16:45:51.372 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:45:51.376 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:45:51.376 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:45:51.378 INFO:teuthology.orchestra.run.smithi162.stdout:Package python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:51.395 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:45:51.395 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:45:51.420 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:51.421 INFO:teuthology.orchestra.run.smithi162.stdout:Nothing to do. 2024-08-17T16:45:51.421 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:51.480 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:45:51.503 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install python3-rbd 2024-08-17T16:45:51.588 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:51.953 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:52.051 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:33 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:52.140 INFO:teuthology.orchestra.run.smithi162.stdout:Package python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:52.176 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:52.176 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:52.177 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:45:52.177 INFO:teuthology.orchestra.run.smithi063.stdout: libcephfs-devel-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:52.177 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:52.177 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:52.182 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:52.183 INFO:teuthology.orchestra.run.smithi162.stdout:Nothing to do. 2024-08-17T16:45:52.183 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:52.257 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install rbd-fuse 2024-08-17T16:45:52.342 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install librados2 2024-08-17T16:45:52.815 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:33 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:52.943 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:30 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:52.944 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:52.944 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:52.944 INFO:teuthology.orchestra.run.smithi162.stdout: Package Architecture Version Repository Size 2024-08-17T16:45:52.944 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:52.944 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:45:52.945 INFO:teuthology.orchestra.run.smithi162.stdout: rbd-fuse x86_64 2:18.2.4-719.gced649f8.el9 ceph 87 k 2024-08-17T16:45:52.945 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:52.945 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:45:52.945 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:52.945 INFO:teuthology.orchestra.run.smithi162.stdout:Install 1 Package 2024-08-17T16:45:52.945 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:52.946 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 87 k 2024-08-17T16:45:52.946 INFO:teuthology.orchestra.run.smithi162.stdout:Installed size: 227 k 2024-08-17T16:45:52.946 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:45:53.045 INFO:teuthology.orchestra.run.smithi063.stdout:Package librados2-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:53.091 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:53.092 INFO:teuthology.orchestra.run.smithi063.stdout:Nothing to do. 2024-08-17T16:45:53.092 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:53.155 INFO:teuthology.orchestra.run.smithi162.stdout:rbd-fuse-18.2.4-719.gced649f8.el9.x86_64.rpm 415 kB/s | 87 kB 00:00 2024-08-17T16:45:53.155 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:53.155 INFO:teuthology.orchestra.run.smithi162.stdout:Total 412 kB/s | 87 kB 00:00 2024-08-17T16:45:53.156 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:45:53.156 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install librbd1 2024-08-17T16:45:53.162 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:45:53.162 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:45:53.225 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:45:53.225 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:45:53.370 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:45:53.460 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:53.757 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:31 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:53.810 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:53.851 INFO:teuthology.orchestra.run.smithi063.stdout:Package librbd1-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:53.896 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:53.897 INFO:teuthology.orchestra.run.smithi063.stdout:Nothing to do. 2024-08-17T16:45:53.897 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:53.955 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install python3-rados 2024-08-17T16:45:54.069 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:54.069 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:54.069 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:45:54.069 INFO:teuthology.orchestra.run.smithi162.stdout: rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:54.069 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:54.069 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:54.298 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install rbd-mirror 2024-08-17T16:45:54.529 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:32 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:54.617 INFO:teuthology.orchestra.run.smithi063.stdout:Package python3-rados-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:54.662 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:54.663 INFO:teuthology.orchestra.run.smithi063.stdout:Nothing to do. 2024-08-17T16:45:54.663 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:54.722 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install python3-rgw 2024-08-17T16:45:54.854 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:35 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:54.984 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:54.984 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:54.984 INFO:teuthology.orchestra.run.smithi162.stdout: Package Arch Version Repo Size 2024-08-17T16:45:54.985 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:54.985 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:45:54.985 INFO:teuthology.orchestra.run.smithi162.stdout: rbd-mirror x86_64 2:18.2.4-719.gced649f8.el9 ceph 3.0 M 2024-08-17T16:45:54.985 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:54.985 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:45:54.985 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:54.985 INFO:teuthology.orchestra.run.smithi162.stdout:Install 1 Package 2024-08-17T16:45:54.985 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:54.985 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 3.0 M 2024-08-17T16:45:54.985 INFO:teuthology.orchestra.run.smithi162.stdout:Installed size: 12 M 2024-08-17T16:45:54.985 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:45:55.298 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:33 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:55.335 INFO:teuthology.orchestra.run.smithi162.stdout:rbd-mirror-18.2.4-719.gced649f8.el9.x86_64.rpm 8.5 MB/s | 3.0 MB 00:00 2024-08-17T16:45:55.336 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:55.337 INFO:teuthology.orchestra.run.smithi162.stdout:Total 8.5 MB/s | 3.0 MB 00:00 2024-08-17T16:45:55.337 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:45:55.346 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:45:55.346 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:45:55.387 INFO:teuthology.orchestra.run.smithi063.stdout:Package python3-rgw-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:55.425 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:45:55.425 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:45:55.431 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:55.433 INFO:teuthology.orchestra.run.smithi063.stdout:Nothing to do. 2024-08-17T16:45:55.433 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:55.498 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install python3-cephfs 2024-08-17T16:45:55.843 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:45:55.911 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:55.936 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:55.936 INFO:teuthology.orchestra.run.smithi162.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:45:55.937 INFO:teuthology.orchestra.run.smithi162.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-08-17T16:45:55.937 INFO:teuthology.orchestra.run.smithi162.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T16:45:55.937 INFO:teuthology.orchestra.run.smithi162.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T16:45:55.937 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:56.074 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:34 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:56.163 INFO:teuthology.orchestra.run.smithi063.stdout:Package python3-cephfs-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:56.208 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:56.209 INFO:teuthology.orchestra.run.smithi063.stdout:Nothing to do. 2024-08-17T16:45:56.209 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:56.273 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install python3-rbd 2024-08-17T16:45:56.844 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:34 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:56.933 INFO:teuthology.orchestra.run.smithi063.stdout:Package python3-rbd-2:18.2.4-719.gced649f8.el9.x86_64 is already installed. 2024-08-17T16:45:56.978 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:56.979 INFO:teuthology.orchestra.run.smithi063.stdout:Nothing to do. 2024-08-17T16:45:56.979 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:57.039 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install rbd-fuse 2024-08-17T16:45:57.083 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:57.083 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:57.083 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:45:57.083 INFO:teuthology.orchestra.run.smithi162.stdout: rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:57.083 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:57.083 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:57.334 DEBUG:teuthology.orchestra.run.smithi162:> sudo yum -y install rbd-nbd 2024-08-17T16:45:57.617 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:35 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:57.751 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:57.752 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:57.752 INFO:teuthology.orchestra.run.smithi063.stdout: Package Architecture Version Repository Size 2024-08-17T16:45:57.752 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:57.752 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:45:57.752 INFO:teuthology.orchestra.run.smithi063.stdout: rbd-fuse x86_64 2:18.2.4-719.gced649f8.el9 ceph 87 k 2024-08-17T16:45:57.752 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:57.752 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:45:57.752 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:57.752 INFO:teuthology.orchestra.run.smithi063.stdout:Install 1 Package 2024-08-17T16:45:57.752 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:57.752 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 87 k 2024-08-17T16:45:57.753 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 227 k 2024-08-17T16:45:57.753 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:45:57.908 INFO:teuthology.orchestra.run.smithi162.stdout:Last metadata expiration check: 0:01:38 ago on Sat 17 Aug 2024 04:44:19 PM UTC. 2024-08-17T16:45:57.966 INFO:teuthology.orchestra.run.smithi063.stdout:rbd-fuse-18.2.4-719.gced649f8.el9.x86_64.rpm 406 kB/s | 87 kB 00:00 2024-08-17T16:45:57.967 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:57.967 INFO:teuthology.orchestra.run.smithi063.stdout:Total 403 kB/s | 87 kB 00:00 2024-08-17T16:45:57.967 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:45:57.974 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:45:57.975 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:45:58.037 INFO:teuthology.orchestra.run.smithi162.stdout:Dependencies resolved. 2024-08-17T16:45:58.037 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:58.037 INFO:teuthology.orchestra.run.smithi162.stdout: Package Architecture Version Repository Size 2024-08-17T16:45:58.038 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:58.038 INFO:teuthology.orchestra.run.smithi162.stdout:Installing: 2024-08-17T16:45:58.038 INFO:teuthology.orchestra.run.smithi162.stdout: rbd-nbd x86_64 2:18.2.4-719.gced649f8.el9 ceph 172 k 2024-08-17T16:45:58.038 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:58.038 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction Summary 2024-08-17T16:45:58.038 INFO:teuthology.orchestra.run.smithi162.stdout:================================================================================ 2024-08-17T16:45:58.038 INFO:teuthology.orchestra.run.smithi162.stdout:Install 1 Package 2024-08-17T16:45:58.038 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:58.038 INFO:teuthology.orchestra.run.smithi162.stdout:Total download size: 172 k 2024-08-17T16:45:58.038 INFO:teuthology.orchestra.run.smithi162.stdout:Installed size: 494 k 2024-08-17T16:45:58.038 INFO:teuthology.orchestra.run.smithi162.stdout:Downloading Packages: 2024-08-17T16:45:58.041 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:45:58.042 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:45:58.196 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:45:58.261 INFO:teuthology.orchestra.run.smithi162.stdout:rbd-nbd-18.2.4-719.gced649f8.el9.x86_64.rpm 775 kB/s | 172 kB 00:00 2024-08-17T16:45:58.261 INFO:teuthology.orchestra.run.smithi162.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:45:58.261 INFO:teuthology.orchestra.run.smithi162.stdout:Total 770 kB/s | 172 kB 00:00 2024-08-17T16:45:58.262 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction check 2024-08-17T16:45:58.269 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction check succeeded. 2024-08-17T16:45:58.269 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction test 2024-08-17T16:45:58.299 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:58.332 INFO:teuthology.orchestra.run.smithi162.stdout:Transaction test succeeded. 2024-08-17T16:45:58.332 INFO:teuthology.orchestra.run.smithi162.stdout:Running transaction 2024-08-17T16:45:58.487 INFO:teuthology.orchestra.run.smithi162.stdout: Preparing : 1/1 2024-08-17T16:45:58.582 INFO:teuthology.orchestra.run.smithi162.stdout: Installing : rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:58.688 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:58.941 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:58.941 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:58.941 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:45:58.941 INFO:teuthology.orchestra.run.smithi063.stdout: rbd-fuse-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:58.941 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:58.941 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:45:59.018 INFO:teuthology.orchestra.run.smithi162.stdout: Running scriptlet: rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:59.153 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install rbd-mirror 2024-08-17T16:45:59.266 INFO:teuthology.orchestra.run.smithi162.stdout: Verifying : rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:45:59.266 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:59.266 INFO:teuthology.orchestra.run.smithi162.stdout:Installed: 2024-08-17T16:45:59.267 INFO:teuthology.orchestra.run.smithi162.stdout: rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:45:59.267 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:45:59.267 INFO:teuthology.orchestra.run.smithi162.stdout:Complete! 2024-08-17T16:45:59.519 DEBUG:teuthology.parallel:result is None 2024-08-17T16:45:59.725 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:37 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:45:59.859 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:45:59.859 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:59.859 INFO:teuthology.orchestra.run.smithi063.stdout: Package Arch Version Repo Size 2024-08-17T16:45:59.860 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:59.860 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:45:59.860 INFO:teuthology.orchestra.run.smithi063.stdout: rbd-mirror x86_64 2:18.2.4-719.gced649f8.el9 ceph 3.0 M 2024-08-17T16:45:59.860 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:59.860 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:45:59.860 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:45:59.860 INFO:teuthology.orchestra.run.smithi063.stdout:Install 1 Package 2024-08-17T16:45:59.860 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:45:59.860 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 3.0 M 2024-08-17T16:45:59.860 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 12 M 2024-08-17T16:45:59.860 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:46:00.194 INFO:teuthology.orchestra.run.smithi063.stdout:rbd-mirror-18.2.4-719.gced649f8.el9.x86_64.rpm 8.9 MB/s | 3.0 MB 00:00 2024-08-17T16:46:00.195 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:46:00.195 INFO:teuthology.orchestra.run.smithi063.stdout:Total 8.9 MB/s | 3.0 MB 00:00 2024-08-17T16:46:00.196 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:46:00.206 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:46:00.206 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:46:00.291 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:46:00.292 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:46:00.728 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:46:00.804 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:46:00.829 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:46:00.830 INFO:teuthology.orchestra.run.smithi063.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-08-17T16:46:00.830 INFO:teuthology.orchestra.run.smithi063.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-08-17T16:46:00.830 INFO:teuthology.orchestra.run.smithi063.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T16:46:00.830 INFO:teuthology.orchestra.run.smithi063.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-08-17T16:46:00.830 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:46:01.902 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:46:01.902 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:46:01.902 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:46:01.903 INFO:teuthology.orchestra.run.smithi063.stdout: rbd-mirror-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:46:01.903 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:46:01.903 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:46:02.130 DEBUG:teuthology.orchestra.run.smithi063:> sudo yum -y install rbd-nbd 2024-08-17T16:46:02.719 INFO:teuthology.orchestra.run.smithi063.stdout:Last metadata expiration check: 0:01:40 ago on Sat 17 Aug 2024 04:44:22 PM UTC. 2024-08-17T16:46:02.854 INFO:teuthology.orchestra.run.smithi063.stdout:Dependencies resolved. 2024-08-17T16:46:02.855 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:46:02.855 INFO:teuthology.orchestra.run.smithi063.stdout: Package Architecture Version Repository Size 2024-08-17T16:46:02.855 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:46:02.855 INFO:teuthology.orchestra.run.smithi063.stdout:Installing: 2024-08-17T16:46:02.855 INFO:teuthology.orchestra.run.smithi063.stdout: rbd-nbd x86_64 2:18.2.4-719.gced649f8.el9 ceph 172 k 2024-08-17T16:46:02.855 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:46:02.856 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction Summary 2024-08-17T16:46:02.856 INFO:teuthology.orchestra.run.smithi063.stdout:================================================================================ 2024-08-17T16:46:02.856 INFO:teuthology.orchestra.run.smithi063.stdout:Install 1 Package 2024-08-17T16:46:02.856 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:46:02.856 INFO:teuthology.orchestra.run.smithi063.stdout:Total download size: 172 k 2024-08-17T16:46:02.856 INFO:teuthology.orchestra.run.smithi063.stdout:Installed size: 494 k 2024-08-17T16:46:02.856 INFO:teuthology.orchestra.run.smithi063.stdout:Downloading Packages: 2024-08-17T16:46:03.086 INFO:teuthology.orchestra.run.smithi063.stdout:rbd-nbd-18.2.4-719.gced649f8.el9.x86_64.rpm 749 kB/s | 172 kB 00:00 2024-08-17T16:46:03.086 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:46:03.087 INFO:teuthology.orchestra.run.smithi063.stdout:Total 745 kB/s | 172 kB 00:00 2024-08-17T16:46:03.087 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction check 2024-08-17T16:46:03.095 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction check succeeded. 2024-08-17T16:46:03.095 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction test 2024-08-17T16:46:03.162 INFO:teuthology.orchestra.run.smithi063.stdout:Transaction test succeeded. 2024-08-17T16:46:03.163 INFO:teuthology.orchestra.run.smithi063.stdout:Running transaction 2024-08-17T16:46:03.323 INFO:teuthology.orchestra.run.smithi063.stdout: Preparing : 1/1 2024-08-17T16:46:03.427 INFO:teuthology.orchestra.run.smithi063.stdout: Installing : rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:46:03.839 INFO:teuthology.orchestra.run.smithi063.stdout: Running scriptlet: rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:46:04.094 INFO:teuthology.orchestra.run.smithi063.stdout: Verifying : rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 1/1 2024-08-17T16:46:04.094 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:46:04.094 INFO:teuthology.orchestra.run.smithi063.stdout:Installed: 2024-08-17T16:46:04.094 INFO:teuthology.orchestra.run.smithi063.stdout: rbd-nbd-2:18.2.4-719.gced649f8.el9.x86_64 2024-08-17T16:46:04.094 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:46:04.094 INFO:teuthology.orchestra.run.smithi063.stdout:Complete! 2024-08-17T16:46:04.314 DEBUG:teuthology.parallel:result is None 2024-08-17T16:46:04.315 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:46:04.315 INFO:teuthology.packaging:ref: None 2024-08-17T16:46:04.315 INFO:teuthology.packaging:tag: None 2024-08-17T16:46:04.315 INFO:teuthology.packaging:branch: reef 2024-08-17T16:46:04.315 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:46:04.315 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-08-17T16:46:04.584 DEBUG:teuthology.orchestra.run.smithi063:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-08-17T16:46:04.606 INFO:teuthology.orchestra.run.smithi063.stdout:18.2.4-719.gced649f8.el9 2024-08-17T16:46:04.607 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-719.gced649f8.el9 2024-08-17T16:46:04.607 INFO:teuthology.task.install:The correct ceph version 18.2.4-719.gced649f8 is installed. 2024-08-17T16:46:04.609 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-08-17T16:46:04.609 INFO:teuthology.packaging:ref: None 2024-08-17T16:46:04.609 INFO:teuthology.packaging:tag: None 2024-08-17T16:46:04.609 INFO:teuthology.packaging:branch: reef 2024-08-17T16:46:04.609 INFO:teuthology.packaging:sha1: cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:46:04.609 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-08-17T16:46:04.873 DEBUG:teuthology.orchestra.run.smithi162:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-08-17T16:46:04.899 INFO:teuthology.orchestra.run.smithi162.stdout:18.2.4-719.gced649f8.el9 2024-08-17T16:46:04.899 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-719.gced649f8.el9 2024-08-17T16:46:04.899 INFO:teuthology.task.install:The correct ceph version 18.2.4-719.gced649f8 is installed. 2024-08-17T16:46:04.901 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-08-17T16:46:04.901 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T16:46:04.901 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-08-17T16:46:04.933 DEBUG:teuthology.orchestra.run.smithi162:> set -ex 2024-08-17T16:46:04.933 DEBUG:teuthology.orchestra.run.smithi162:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-08-17T16:46:04.969 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-08-17T16:46:04.969 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T16:46:04.969 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd of=/usr/bin/daemon-helper 2024-08-17T16:46:05.004 DEBUG:teuthology.orchestra.run.smithi063:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-08-17T16:46:05.072 DEBUG:teuthology.orchestra.run.smithi162:> set -ex 2024-08-17T16:46:05.072 DEBUG:teuthology.orchestra.run.smithi162:> sudo dd of=/usr/bin/daemon-helper 2024-08-17T16:46:05.101 DEBUG:teuthology.orchestra.run.smithi162:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-08-17T16:46:05.169 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-08-17T16:46:05.169 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T16:46:05.169 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd of=/usr/bin/adjust-ulimits 2024-08-17T16:46:05.197 DEBUG:teuthology.orchestra.run.smithi063:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-08-17T16:46:05.266 DEBUG:teuthology.orchestra.run.smithi162:> set -ex 2024-08-17T16:46:05.266 DEBUG:teuthology.orchestra.run.smithi162:> sudo dd of=/usr/bin/adjust-ulimits 2024-08-17T16:46:05.292 DEBUG:teuthology.orchestra.run.smithi162:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-08-17T16:46:05.356 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-08-17T16:46:05.357 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T16:46:05.357 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd of=/usr/bin/stdin-killer 2024-08-17T16:46:05.384 DEBUG:teuthology.orchestra.run.smithi063:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-08-17T16:46:05.451 DEBUG:teuthology.orchestra.run.smithi162:> set -ex 2024-08-17T16:46:05.451 DEBUG:teuthology.orchestra.run.smithi162:> sudo dd of=/usr/bin/stdin-killer 2024-08-17T16:46:05.477 DEBUG:teuthology.orchestra.run.smithi162:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-08-17T16:46:05.551 INFO:teuthology.run_tasks:Running task cephadm... 2024-08-17T16:46:05.644 INFO:tasks.cephadm:Config: {'compiled_cephadm_branch': 'reef', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:reef', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'} 2024-08-17T16:46:05.644 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:reef 2024-08-17T16:46:05.644 INFO:tasks.cephadm:Cluster fsid is 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:46:05.644 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-08-17T16:46:05.645 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.63', 'mon.c': '[v2:172.21.15.63:3301,v1:172.21.15.63:6790]', 'mon.b': '172.21.15.162'} 2024-08-17T16:46:05.645 INFO:tasks.cephadm:First mon is mon.a on smithi063 2024-08-17T16:46:05.645 INFO:tasks.cephadm:First mgr is y 2024-08-17T16:46:05.645 INFO:tasks.cephadm:Normalizing hostnames... 2024-08-17T16:46:05.645 DEBUG:teuthology.orchestra.run.smithi063:> sudo hostname $(hostname -s) 2024-08-17T16:46:05.673 DEBUG:teuthology.orchestra.run.smithi162:> sudo hostname $(hostname -s) 2024-08-17T16:46:05.699 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra for reef 2024-08-17T16:46:05.699 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:46:05.848 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic/55847/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.0.0-5836-gcd8bdbeb', 'node_name': '172.21.5.36+adami06', 'job_name': 'ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic', 'package_manager_version': '19.0.0-5836.gcd8bdbeb'}, 'url': 'https://2.chacra.ceph.com/r/ceph/main/cd8bdbeb9fbdab5733aa3aaadc042c965473e453/centos/9/flavors/default/', 'distro_codename': None, 'modified': '2024-08-15 21:47:48.216668', 'distro_version': '9', 'project': 'ceph', 'flavor': 'default', 'ref': 'main', 'chacra_url': 'https://2.chacra.ceph.com/repos/ceph/main/cd8bdbeb9fbdab5733aa3aaadc042c965473e453/centos/9/flavors/default/', 'archs': ['x86_64', 'source'], 'distro': 'centos'}] 2024-08-17T16:46:06.108 INFO:tasks.util.chacra:got chacra host 3.chacra.ceph.com, ref reef, sha1 ced649f89259a8c5733cc18a50f42f20f41ba352 from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F9%2Fx86_64&flavor=default&ref=reef 2024-08-17T16:46:06.110 INFO:tasks.cephadm:Discovered cachra url: https://3.chacra.ceph.com/binaries/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/x86_64/flavors/default/cephadm 2024-08-17T16:46:06.110 INFO:tasks.cephadm:Downloading cephadm from url: https://3.chacra.ceph.com/binaries/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/x86_64/flavors/default/cephadm 2024-08-17T16:46:06.110 DEBUG:teuthology.orchestra.run.smithi063:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-08-17T16:46:06.482 INFO:teuthology.orchestra.run.smithi063.stdout:-rw-r--r--. 1 ubuntu ubuntu 215594 Aug 17 16:46 /home/ubuntu/cephtest/cephadm 2024-08-17T16:46:06.483 DEBUG:teuthology.orchestra.run.smithi162:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/reef/ced649f89259a8c5733cc18a50f42f20f41ba352/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-08-17T16:46:06.843 INFO:teuthology.orchestra.run.smithi162.stdout:-rw-r--r--. 1 ubuntu ubuntu 215594 Aug 17 16:46 /home/ubuntu/cephtest/cephadm 2024-08-17T16:46:06.843 DEBUG:teuthology.orchestra.run.smithi063:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-08-17T16:46:06.864 DEBUG:teuthology.orchestra.run.smithi162:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-08-17T16:46:06.891 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:reef on all hosts... 2024-08-17T16:46:06.891 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-08-17T16:46:06.906 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-08-17T16:46:07.650 INFO:teuthology.orchestra.run.smithi162.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-08-17T16:46:07.872 INFO:teuthology.orchestra.run.smithi063.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-08-17T16:46:36.692 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T16:46:36.692 INFO:teuthology.orchestra.run.smithi162.stdout: "ceph_version": "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)", 2024-08-17T16:46:36.692 INFO:teuthology.orchestra.run.smithi162.stdout: "image_id": "c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e", 2024-08-17T16:46:36.692 INFO:teuthology.orchestra.run.smithi162.stdout: "repo_digests": [ 2024-08-17T16:46:36.692 INFO:teuthology.orchestra.run.smithi162.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408" 2024-08-17T16:46:36.692 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T16:46:36.692 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T16:46:41.412 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:46:41.412 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph_version": "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)", 2024-08-17T16:46:41.412 INFO:teuthology.orchestra.run.smithi063.stdout: "image_id": "c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e", 2024-08-17T16:46:41.412 INFO:teuthology.orchestra.run.smithi063.stdout: "repo_digests": [ 2024-08-17T16:46:41.413 INFO:teuthology.orchestra.run.smithi063.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408" 2024-08-17T16:46:41.413 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T16:46:41.413 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:46:41.431 DEBUG:teuthology.orchestra.run.smithi063:> sudo mkdir -p /etc/ceph 2024-08-17T16:46:41.477 DEBUG:teuthology.orchestra.run.smithi162:> sudo mkdir -p /etc/ceph 2024-08-17T16:46:41.510 DEBUG:teuthology.orchestra.run.smithi063:> sudo chmod 777 /etc/ceph 2024-08-17T16:46:41.535 DEBUG:teuthology.orchestra.run.smithi162:> sudo chmod 777 /etc/ceph 2024-08-17T16:46:41.577 INFO:tasks.cephadm:Writing seed config... 2024-08-17T16:46:41.579 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-08-17T16:46:41.579 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-08-17T16:46:41.579 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-08-17T16:46:41.579 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-08-17T16:46:41.579 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-08-17T16:46:41.579 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-08-17T16:46:41.579 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-08-17T16:46:41.579 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-08-17T16:46:41.579 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-08-17T16:46:41.579 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-08-17T16:46:41.579 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-08-17T16:46:41.580 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T16:46:41.580 DEBUG:teuthology.orchestra.run.smithi063:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-08-17T16:46:41.596 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 321fa59e-5cb8-11ef-bccf-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-08-17T16:46:41.596 DEBUG:teuthology.orchestra.run.smithi063:mon.a> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.a.service 2024-08-17T16:46:41.638 DEBUG:teuthology.orchestra.run.smithi063:mgr.y> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.y.service 2024-08-17T16:46:41.680 INFO:tasks.cephadm:Bootstrapping... 2024-08-17T16:46:41.681 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef -v bootstrap --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.63 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-08-17T16:46:41.842 INFO:teuthology.orchestra.run.smithi063.stdout:-------------------------------------------------------------------------------- 2024-08-17T16:46:41.842 INFO:teuthology.orchestra.run.smithi063.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:reef', '-v', 'bootstrap', '--fsid', '321fa59e-5cb8-11ef-bccf-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.63', '--skip-admin-label'] 2024-08-17T16:46:41.869 INFO:teuthology.orchestra.run.smithi063.stdout:/bin/podman: stdout 5.1.2 2024-08-17T16:46:41.870 INFO:teuthology.orchestra.run.smithi063.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-08-17T16:46:41.870 INFO:teuthology.orchestra.run.smithi063.stdout:Verifying podman|docker is present... 2024-08-17T16:46:41.894 INFO:teuthology.orchestra.run.smithi063.stdout:/bin/podman: stdout 5.1.2 2024-08-17T16:46:41.894 INFO:teuthology.orchestra.run.smithi063.stdout:Verifying lvm2 is present... 2024-08-17T16:46:41.894 INFO:teuthology.orchestra.run.smithi063.stdout:Verifying time synchronization is in place... 2024-08-17T16:46:41.902 INFO:teuthology.orchestra.run.smithi063.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-08-17T16:46:41.902 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-08-17T16:46:41.909 INFO:teuthology.orchestra.run.smithi063.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-08-17T16:46:41.909 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stdout inactive 2024-08-17T16:46:41.916 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stdout enabled 2024-08-17T16:46:41.923 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stdout active 2024-08-17T16:46:41.923 INFO:teuthology.orchestra.run.smithi063.stdout:Unit chronyd.service is enabled and running 2024-08-17T16:46:41.923 INFO:teuthology.orchestra.run.smithi063.stdout:Repeating the final host check... 2024-08-17T16:46:41.947 INFO:teuthology.orchestra.run.smithi063.stdout:/bin/podman: stdout 5.1.2 2024-08-17T16:46:41.947 INFO:teuthology.orchestra.run.smithi063.stdout:podman (/bin/podman) version 5.1.2 is present 2024-08-17T16:46:41.947 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl is present 2024-08-17T16:46:41.947 INFO:teuthology.orchestra.run.smithi063.stdout:lvcreate is present 2024-08-17T16:46:41.954 INFO:teuthology.orchestra.run.smithi063.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-08-17T16:46:41.954 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-08-17T16:46:41.961 INFO:teuthology.orchestra.run.smithi063.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-08-17T16:46:41.961 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stdout inactive 2024-08-17T16:46:41.968 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stdout enabled 2024-08-17T16:46:41.975 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stdout active 2024-08-17T16:46:41.975 INFO:teuthology.orchestra.run.smithi063.stdout:Unit chronyd.service is enabled and running 2024-08-17T16:46:41.975 INFO:teuthology.orchestra.run.smithi063.stdout:Host looks OK 2024-08-17T16:46:41.975 INFO:teuthology.orchestra.run.smithi063.stdout:Cluster fsid: 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:46:41.975 INFO:teuthology.orchestra.run.smithi063.stdout:Acquiring lock 140538061129472 on /run/cephadm/321fa59e-5cb8-11ef-bccf-c7b262605968.lock 2024-08-17T16:46:41.975 INFO:teuthology.orchestra.run.smithi063.stdout:Lock 140538061129472 acquired on /run/cephadm/321fa59e-5cb8-11ef-bccf-c7b262605968.lock 2024-08-17T16:46:41.976 INFO:teuthology.orchestra.run.smithi063.stdout:Verifying IP 172.21.15.63 port 3300 ... 2024-08-17T16:46:41.976 INFO:teuthology.orchestra.run.smithi063.stdout:Verifying IP 172.21.15.63 port 6789 ... 2024-08-17T16:46:41.976 INFO:teuthology.orchestra.run.smithi063.stdout:Base mon IP(s) is [172.21.15.63:3300, 172.21.15.63:6789], mon addrv is [v2:172.21.15.63:3300,v1:172.21.15.63:6789] 2024-08-17T16:46:41.979 INFO:teuthology.orchestra.run.smithi063.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.63 metric 100 2024-08-17T16:46:41.980 INFO:teuthology.orchestra.run.smithi063.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.63 metric 100 2024-08-17T16:46:41.982 INFO:teuthology.orchestra.run.smithi063.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-08-17T16:46:41.982 INFO:teuthology.orchestra.run.smithi063.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-08-17T16:46:41.982 INFO:teuthology.orchestra.run.smithi063.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1798sec hoplimit 64 pref medium 2024-08-17T16:46:41.985 INFO:teuthology.orchestra.run.smithi063.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-08-17T16:46:41.985 INFO:teuthology.orchestra.run.smithi063.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-08-17T16:46:41.985 INFO:teuthology.orchestra.run.smithi063.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-08-17T16:46:41.985 INFO:teuthology.orchestra.run.smithi063.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-08-17T16:46:41.985 INFO:teuthology.orchestra.run.smithi063.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:feb6:f8d7/64 scope link 2024-08-17T16:46:41.985 INFO:teuthology.orchestra.run.smithi063.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-08-17T16:46:41.986 INFO:teuthology.orchestra.run.smithi063.stdout:Mon IP `172.21.15.63` is in CIDR network `172.21.0.0/20` 2024-08-17T16:46:41.986 INFO:teuthology.orchestra.run.smithi063.stdout:Mon IP `172.21.15.63` is in CIDR network `172.21.0.0/20` 2024-08-17T16:46:41.986 INFO:teuthology.orchestra.run.smithi063.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-08-17T16:46:41.986 INFO:teuthology.orchestra.run.smithi063.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-08-17T16:46:41.987 INFO:teuthology.orchestra.run.smithi063.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-08-17T16:46:42.582 INFO:teuthology.orchestra.run.smithi063.stdout:/bin/podman: stdout c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e 2024-08-17T16:46:42.582 INFO:teuthology.orchestra.run.smithi063.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:reef... 2024-08-17T16:46:42.582 INFO:teuthology.orchestra.run.smithi063.stdout:/bin/podman: stderr Getting image source signatures 2024-08-17T16:46:42.582 INFO:teuthology.orchestra.run.smithi063.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-08-17T16:46:42.582 INFO:teuthology.orchestra.run.smithi063.stdout:/bin/podman: stderr Copying blob sha256:090188efbbf262fdcde0da55307c932266150937ffe226cad16cfadb849e480e 2024-08-17T16:46:42.582 INFO:teuthology.orchestra.run.smithi063.stdout:/bin/podman: stderr Copying config sha256:c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e 2024-08-17T16:46:42.583 INFO:teuthology.orchestra.run.smithi063.stdout:/bin/podman: stderr Writing manifest to image destination 2024-08-17T16:46:43.470 INFO:teuthology.orchestra.run.smithi063.stdout:ceph: stdout ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable) 2024-08-17T16:46:43.470 INFO:teuthology.orchestra.run.smithi063.stdout:Ceph version: ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable) 2024-08-17T16:46:43.470 INFO:teuthology.orchestra.run.smithi063.stdout:Extracting ceph user uid/gid from container image... 2024-08-17T16:46:44.318 INFO:teuthology.orchestra.run.smithi063.stdout:stat: stdout 167 167 2024-08-17T16:46:44.318 INFO:teuthology.orchestra.run.smithi063.stdout:Creating initial keys... 2024-08-17T16:46:45.248 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph-authtool: stdout AQD008BmRVDVJhAA2HZ5Gv1Rg93LgZjV1P/0Ng== 2024-08-17T16:46:46.295 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph-authtool: stdout AQD108BmtSg4JBAA7jMIF5XlRqposM4Pp3djpA== 2024-08-17T16:46:50.833 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph-authtool: stdout AQD208BmprKVMBAAemc5P5X+nDvggrpgQGbPIQ== 2024-08-17T16:46:50.833 INFO:teuthology.orchestra.run.smithi063.stdout:Creating initial monmap... 2024-08-17T16:46:51.864 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-08-17T16:46:51.864 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2024-08-17T16:46:51.864 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:46:51.864 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-08-17T16:46:51.864 INFO:teuthology.orchestra.run.smithi063.stdout:monmaptool for a [v2:172.21.15.63:3300,v1:172.21.15.63:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-08-17T16:46:51.864 INFO:teuthology.orchestra.run.smithi063.stdout:setting min_mon_release = pacific 2024-08-17T16:46:51.864 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/monmaptool: set fsid to 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:46:51.864 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-08-17T16:46:51.864 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:46:51.865 INFO:teuthology.orchestra.run.smithi063.stdout:Creating mon... 2024-08-17T16:46:52.786 INFO:teuthology.orchestra.run.smithi063.stdout:create mon.a on 2024-08-17T16:46:52.987 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-08-17T16:46:53.128 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-08-17T16:46:53.305 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-321fa59e-5cb8-11ef-bccf-c7b262605968.target → /etc/systemd/system/ceph-321fa59e-5cb8-11ef-bccf-c7b262605968.target. 2024-08-17T16:46:53.305 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-321fa59e-5cb8-11ef-bccf-c7b262605968.target → /etc/systemd/system/ceph-321fa59e-5cb8-11ef-bccf-c7b262605968.target. 2024-08-17T16:46:53.547 INFO:teuthology.orchestra.run.smithi063.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.a 2024-08-17T16:46:53.547 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stderr Failed to reset failed state of unit ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.a.service: Unit ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.a.service not loaded. 2024-08-17T16:46:53.706 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-321fa59e-5cb8-11ef-bccf-c7b262605968.target.wants/ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.a.service → /etc/systemd/system/ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@.service. 2024-08-17T16:46:54.222 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:54 smithi063 ceph-mon[30579]: mon.a is new leader, mons a in quorum (ranks 0) 2024-08-17T16:46:54.236 INFO:teuthology.orchestra.run.smithi063.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T16:46:54.236 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stdout disabled 2024-08-17T16:46:54.243 INFO:teuthology.orchestra.run.smithi063.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T16:46:54.243 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stdout inactive 2024-08-17T16:46:54.243 INFO:teuthology.orchestra.run.smithi063.stdout:firewalld.service is not enabled 2024-08-17T16:46:54.243 INFO:teuthology.orchestra.run.smithi063.stdout:Not possible to enable service . firewalld.service is not available 2024-08-17T16:46:54.244 INFO:teuthology.orchestra.run.smithi063.stdout:Waiting for mon to start... 2024-08-17T16:46:54.244 INFO:teuthology.orchestra.run.smithi063.stdout:Waiting for mon... 2024-08-17T16:46:54.475 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:54 smithi063 bash[30542]: 7e617bc2ca964c1b37b6c2ed8a3433d172ad4bfbdd67f8ef6086c10134461aa0 2024-08-17T16:46:54.475 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:54 smithi063 systemd[1]: Started Ceph mon.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:46:55.303 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:55 smithi063 ceph-mon[30579]: mon.a is new leader, mons a in quorum (ranks 0) 2024-08-17T16:46:55.303 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:55 smithi063 ceph-mon[30579]: monmap e1: 1 mons at {a=[v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:46:55.304 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:55 smithi063 ceph-mon[30579]: fsmap 2024-08-17T16:46:55.304 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:55 smithi063 ceph-mon[30579]: osdmap e1: 0 total, 0 up, 0 in 2024-08-17T16:46:55.304 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:55 smithi063 ceph-mon[30579]: mgrmap e1: no daemons active 2024-08-17T16:46:55.304 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:55 smithi063 ceph-mon[30579]: from='client.? 172.21.15.63:0/845114251' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-08-17T16:46:55.305 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout cluster: 2024-08-17T16:46:55.307 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout id: 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:46:55.307 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-08-17T16:46:55.307 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 2024-08-17T16:46:55.308 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout services: 2024-08-17T16:46:55.308 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.586237s) 2024-08-17T16:46:55.308 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-08-17T16:46:55.308 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-08-17T16:46:55.308 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 2024-08-17T16:46:55.308 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout data: 2024-08-17T16:46:55.308 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-08-17T16:46:55.308 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-08-17T16:46:55.308 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-08-17T16:46:55.308 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout pgs: 2024-08-17T16:46:55.308 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 2024-08-17T16:46:55.308 INFO:teuthology.orchestra.run.smithi063.stdout:mon is available 2024-08-17T16:46:55.308 INFO:teuthology.orchestra.run.smithi063.stdout:Assimilating anything we can from ceph.conf... 2024-08-17T16:46:56.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:56 smithi063 ceph-mon[30579]: from='client.? 172.21.15.63:0/926318471' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-08-17T16:46:56.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:56 smithi063 ceph-mon[30579]: from='client.? 172.21.15.63:0/926318471' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-08-17T16:46:56.723 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 2024-08-17T16:46:56.723 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout [global] 2024-08-17T16:46:56.723 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout fsid = 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:46:56.723 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.63:3300,v1:172.21.15.63:6789] 2024-08-17T16:46:56.723 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-08-17T16:46:56.723 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-08-17T16:46:56.723 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-08-17T16:46:56.724 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-08-17T16:46:56.724 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 2024-08-17T16:46:56.724 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout [mgr] 2024-08-17T16:46:56.724 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-08-17T16:46:56.724 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 2024-08-17T16:46:56.724 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout [osd] 2024-08-17T16:46:56.724 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-08-17T16:46:56.724 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-08-17T16:46:56.724 INFO:teuthology.orchestra.run.smithi063.stdout:Generating new minimal ceph.conf... 2024-08-17T16:46:57.457 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:57 smithi063 ceph-mon[30579]: from='client.? 172.21.15.63:0/2915709670' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:46:57.727 INFO:teuthology.orchestra.run.smithi063.stdout:Restarting the monitor... 2024-08-17T16:46:58.014 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:57 smithi063 systemd[1]: Stopping Ceph mon.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:46:58.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:58 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a[30556]: 2024-08-17T16:46:58.011+0000 7f03cee6c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T16:46:58.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:58 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a[30556]: 2024-08-17T16:46:58.011+0000 7f03cee6c640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-08-17T16:46:58.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:58 smithi063 podman[30908]: 2024-08-17 16:46:58.186664622 +0000 UTC m=+0.285513655 container died 7e617bc2ca964c1b37b6c2ed8a3433d172ad4bfbdd67f8ef6086c10134461aa0 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, ceph=True) 2024-08-17T16:46:58.703 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:58 smithi063 podman[30908]: 2024-08-17 16:46:58.459681272 +0000 UTC m=+0.558530308 container cleanup 7e617bc2ca964c1b37b6c2ed8a3433d172ad4bfbdd67f8ef6086c10134461aa0 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a, RELEASE=reef-ced649f, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-08-17T16:46:58.703 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:58 smithi063 bash[30908]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a 2024-08-17T16:46:59.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:58 smithi063 podman[30927]: 2024-08-17 16:46:58.702491343 +0000 UTC m=+0.507785687 container remove 7e617bc2ca964c1b37b6c2ed8a3433d172ad4bfbdd67f8ef6086c10134461aa0 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-08-17T16:46:59.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:58 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.a.service: Deactivated successfully. 2024-08-17T16:46:59.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:58 smithi063 systemd[1]: Stopped Ceph mon.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:46:59.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:58 smithi063 systemd[1]: Starting Ceph mon.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:46:59.307 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 podman[31025]: 2024-08-17 16:46:58.952991782 +0000 UTC m=+0.018001350 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph:reef 2024-08-17T16:46:59.307 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 podman[31025]: 2024-08-17 16:46:59.053299556 +0000 UTC m=+0.118309108 container create 0c71ddd7ada67f493be747616ba47d4bd768d4b951091cc55699858aff6fe2c2 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-08-17T16:46:59.307 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 podman[31025]: 2024-08-17 16:46:59.280858715 +0000 UTC m=+0.345868286 container init 0c71ddd7ada67f493be747616ba47d4bd768d4b951091cc55699858aff6fe2c2 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T16:46:59.307 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 podman[31025]: 2024-08-17 16:46:59.285519907 +0000 UTC m=+0.350529460 container start 0c71ddd7ada67f493be747616ba47d4bd768d4b951091cc55699858aff6fe2c2 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-08-17T16:46:59.386 INFO:teuthology.orchestra.run.smithi063.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-08-17T16:46:59.606 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T16:46:59.606 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable), process ceph-mon, pid 7 2024-08-17T16:46:59.606 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: pidfile_write: ignore empty --pid-file 2024-08-17T16:46:59.606 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: load: jerasure load: lrc 2024-08-17T16:46:59.606 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: RocksDB version: 7.9.2 2024-08-17T16:46:59.606 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Git sha 0 2024-08-17T16:46:59.606 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Compile date 2024-08-17 15:23:20 2024-08-17T16:46:59.607 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: DB SUMMARY 2024-08-17T16:46:59.607 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: DB Session ID: 33ZRV8OPOA7WV8HWDU6I 2024-08-17T16:46:59.607 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: CURRENT file: CURRENT 2024-08-17T16:46:59.607 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T16:46:59.607 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-08-17T16:46:59.607 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-08-17T16:46:59.607 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 84536 ; 2024-08-17T16:46:59.607 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.error_if_exists: 0 2024-08-17T16:46:59.607 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.create_if_missing: 0 2024-08-17T16:46:59.607 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.paranoid_checks: 1 2024-08-17T16:46:59.607 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T16:46:59.607 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T16:46:59.608 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T16:46:59.608 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.env: 0x55ef0d377bc0 2024-08-17T16:46:59.608 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T16:46:59.608 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.info_log: 0x55ef0dcaebe0 2024-08-17T16:46:59.608 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T16:46:59.608 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.statistics: (nil) 2024-08-17T16:46:59.608 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.use_fsync: 0 2024-08-17T16:46:59.608 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_log_file_size: 0 2024-08-17T16:46:59.608 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T16:46:59.608 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T16:46:59.608 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T16:46:59.608 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T16:46:59.608 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.allow_fallocate: 1 2024-08-17T16:46:59.609 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T16:46:59.609 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T16:46:59.609 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.use_direct_reads: 0 2024-08-17T16:46:59.609 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T16:46:59.609 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T16:46:59.609 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.db_log_dir: 2024-08-17T16:46:59.609 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.wal_dir: 2024-08-17T16:46:59.609 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T16:46:59.610 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T16:46:59.611 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T16:46:59.611 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T16:46:59.611 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T16:46:59.611 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T16:46:59.611 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T16:46:59.611 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T16:46:59.611 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.write_buffer_manager: 0x55ef0dc7e140 2024-08-17T16:46:59.611 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T16:46:59.611 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T16:46:59.611 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T16:46:59.612 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T16:46:59.612 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T16:46:59.612 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T16:46:59.612 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T16:46:59.612 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T16:46:59.612 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.unordered_write: 0 2024-08-17T16:46:59.612 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T16:46:59.612 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T16:46:59.612 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T16:46:59.612 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T16:46:59.612 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.row_cache: None 2024-08-17T16:46:59.612 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.wal_filter: None 2024-08-17T16:46:59.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T16:46:59.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T16:46:59.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.two_write_queues: 0 2024-08-17T16:46:59.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T16:46:59.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.wal_compression: 0 2024-08-17T16:46:59.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.atomic_flush: 0 2024-08-17T16:46:59.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T16:46:59.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T16:46:59.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T16:46:59.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.log_readahead_size: 0 2024-08-17T16:46:59.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T16:46:59.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T16:46:59.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T16:46:59.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T16:46:59.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T16:46:59.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T16:46:59.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T16:46:59.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_background_jobs: 2 2024-08-17T16:46:59.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_background_compactions: -1 2024-08-17T16:46:59.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_subcompactions: 1 2024-08-17T16:46:59.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T16:46:59.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T16:46:59.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T16:46:59.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T16:46:59.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T16:46:59.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T16:46:59.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T16:46:59.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T16:46:59.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_open_files: -1 2024-08-17T16:46:59.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T16:46:59.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T16:46:59.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T16:46:59.616 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T16:46:59.616 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_background_flushes: -1 2024-08-17T16:46:59.616 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Compression algorithms supported: 2024-08-17T16:46:59.616 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: kZSTD supported: 0 2024-08-17T16:46:59.616 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: kXpressCompression supported: 0 2024-08-17T16:46:59.616 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: kBZip2Compression supported: 0 2024-08-17T16:46:59.616 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T16:46:59.616 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: kLZ4Compression supported: 1 2024-08-17T16:46:59.616 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: kZlibCompression supported: 1 2024-08-17T16:46:59.616 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T16:46:59.616 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: kSnappyCompression supported: 1 2024-08-17T16:46:59.616 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T16:46:59.616 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T16:46:59.617 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-08-17T16:46:59.617 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T16:46:59.617 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T16:46:59.617 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.merge_operator: 2024-08-17T16:46:59.617 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compaction_filter: None 2024-08-17T16:46:59.617 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T16:46:59.617 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T16:46:59.617 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T16:46:59.617 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T16:46:59.617 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ef0dcae820) 2024-08-17T16:46:59.617 INFO:journalctl@ceph.mon.a.smithi063.stdout: cache_index_and_filter_blocks: 1 2024-08-17T16:46:59.617 INFO:journalctl@ceph.mon.a.smithi063.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T16:46:59.617 INFO:journalctl@ceph.mon.a.smithi063.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T16:46:59.618 INFO:journalctl@ceph.mon.a.smithi063.stdout: pin_top_level_index_and_filter: 1 2024-08-17T16:46:59.618 INFO:journalctl@ceph.mon.a.smithi063.stdout: index_type: 0 2024-08-17T16:46:59.618 INFO:journalctl@ceph.mon.a.smithi063.stdout: data_block_index_type: 0 2024-08-17T16:46:59.618 INFO:journalctl@ceph.mon.a.smithi063.stdout: index_shortening: 1 2024-08-17T16:46:59.618 INFO:journalctl@ceph.mon.a.smithi063.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T16:46:59.618 INFO:journalctl@ceph.mon.a.smithi063.stdout: checksum: 4 2024-08-17T16:46:59.618 INFO:journalctl@ceph.mon.a.smithi063.stdout: no_block_cache: 0 2024-08-17T16:46:59.618 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_cache: 0x55ef0dca3350 2024-08-17T16:46:59.618 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_cache_name: BinnedLRUCache 2024-08-17T16:46:59.618 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_cache_options: 2024-08-17T16:46:59.618 INFO:journalctl@ceph.mon.a.smithi063.stdout: capacity : 536870912 2024-08-17T16:46:59.619 INFO:journalctl@ceph.mon.a.smithi063.stdout: num_shard_bits : 4 2024-08-17T16:46:59.619 INFO:journalctl@ceph.mon.a.smithi063.stdout: strict_capacity_limit : 0 2024-08-17T16:46:59.619 INFO:journalctl@ceph.mon.a.smithi063.stdout: high_pri_pool_ratio: 0.000 2024-08-17T16:46:59.619 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_cache_compressed: (nil) 2024-08-17T16:46:59.619 INFO:journalctl@ceph.mon.a.smithi063.stdout: persistent_cache: (nil) 2024-08-17T16:46:59.619 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_size: 4096 2024-08-17T16:46:59.619 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_size_deviation: 10 2024-08-17T16:46:59.619 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_restart_interval: 16 2024-08-17T16:46:59.619 INFO:journalctl@ceph.mon.a.smithi063.stdout: index_block_restart_interval: 1 2024-08-17T16:46:59.619 INFO:journalctl@ceph.mon.a.smithi063.stdout: metadata_block_size: 4096 2024-08-17T16:46:59.619 INFO:journalctl@ceph.mon.a.smithi063.stdout: partition_filters: 0 2024-08-17T16:46:59.619 INFO:journalctl@ceph.mon.a.smithi063.stdout: use_delta_encoding: 1 2024-08-17T16:46:59.619 INFO:journalctl@ceph.mon.a.smithi063.stdout: filter_policy: bloomfilter 2024-08-17T16:46:59.620 INFO:journalctl@ceph.mon.a.smithi063.stdout: whole_key_filtering: 1 2024-08-17T16:46:59.620 INFO:journalctl@ceph.mon.a.smithi063.stdout: verify_compression: 0 2024-08-17T16:46:59.620 INFO:journalctl@ceph.mon.a.smithi063.stdout: read_amp_bytes_per_bit: 0 2024-08-17T16:46:59.620 INFO:journalctl@ceph.mon.a.smithi063.stdout: format_version: 5 2024-08-17T16:46:59.620 INFO:journalctl@ceph.mon.a.smithi063.stdout: enable_index_compression: 1 2024-08-17T16:46:59.620 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_align: 0 2024-08-17T16:46:59.620 INFO:journalctl@ceph.mon.a.smithi063.stdout: max_auto_readahead_size: 262144 2024-08-17T16:46:59.620 INFO:journalctl@ceph.mon.a.smithi063.stdout: prepopulate_block_cache: 0 2024-08-17T16:46:59.620 INFO:journalctl@ceph.mon.a.smithi063.stdout: initial_auto_readahead_size: 8192 2024-08-17T16:46:59.620 INFO:journalctl@ceph.mon.a.smithi063.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T16:46:59.620 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T16:46:59.620 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T16:46:59.620 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compression: NoCompression 2024-08-17T16:46:59.621 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T16:46:59.621 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T16:46:59.621 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T16:46:59.621 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.num_levels: 7 2024-08-17T16:46:59.621 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T16:46:59.621 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T16:46:59.621 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T16:46:59.621 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T16:46:59.621 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T16:46:59.621 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T16:46:59.621 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T16:46:59.621 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T16:46:59.621 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T16:46:59.621 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T16:46:59.622 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T16:46:59.622 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T16:46:59.622 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T16:46:59.622 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T16:46:59.622 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T16:46:59.622 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T16:46:59.622 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T16:46:59.622 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T16:46:59.622 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T16:46:59.622 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T16:46:59.622 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T16:46:59.622 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T16:46:59.623 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T16:46:59.623 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T16:46:59.623 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T16:46:59.623 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T16:46:59.623 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T16:46:59.623 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T16:46:59.623 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T16:46:59.623 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T16:46:59.623 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T16:46:59.623 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T16:46:59.623 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T16:46:59.623 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T16:46:59.623 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T16:46:59.623 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T16:46:59.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T16:46:59.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T16:46:59.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T16:46:59.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T16:46:59.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T16:46:59.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T16:46:59.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T16:46:59.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T16:46:59.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T16:46:59.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T16:46:59.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T16:46:59.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T16:46:59.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T16:46:59.625 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T16:46:59.625 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T16:46:59.625 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T16:46:59.625 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T16:46:59.625 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.table_properties_collectors: 2024-08-17T16:46:59.625 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.inplace_update_support: 0 2024-08-17T16:46:59.625 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T16:46:59.625 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T16:46:59.625 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T16:46:59.625 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T16:46:59.625 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.bloom_locality: 0 2024-08-17T16:46:59.625 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.max_successive_merges: 0 2024-08-17T16:46:59.625 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T16:46:59.625 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T16:46:59.625 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T16:46:59.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T16:46:59.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.ttl: 2592000 2024-08-17T16:46:59.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T16:46:59.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T16:46:59.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T16:46:59.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.enable_blob_files: false 2024-08-17T16:46:59.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.min_blob_size: 0 2024-08-17T16:46:59.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T16:46:59.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T16:46:59.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T16:46:59.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T16:46:59.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T16:46:59.627 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T16:46:59.627 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T16:46:59.627 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T16:46:59.627 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-08-17T16:46:59.627 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-08-17T16:46:59.627 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 6d2608df-0a6f-415d-a7b9-c671a8b0b588 2024-08-17T16:46:59.627 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913219312363, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-08-17T16:46:59.627 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-08-17T16:46:59.627 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913219313315, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 81060, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 256, "table_properties": {"data_size": 79236, "index_size": 234, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73437, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 215, "num_filter_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723913219, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "6d2608df-0a6f-415d-a7b9-c671a8b0b588", "db_session_id": "33ZRV8OPOA7WV8HWDU6I", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-08-17T16:46:59.627 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913219313389, "job": 1, "event": "recovery_finished"} 2024-08-17T16:46:59.627 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-08-17T16:46:59.627 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T16:46:59.627 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ef0dccce00 2024-08-17T16:46:59.628 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: DB pointer 0x55ef0ddc0000 2024-08-17T16:46:59.628 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T16:46:59.628 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T16:46:59.628 INFO:journalctl@ceph.mon.a.smithi063.stdout: ** DB Stats ** 2024-08-17T16:46:59.628 INFO:journalctl@ceph.mon.a.smithi063.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:46:59.628 INFO:journalctl@ceph.mon.a.smithi063.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T16:46:59.628 INFO:journalctl@ceph.mon.a.smithi063.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T16:46:59.628 INFO:journalctl@ceph.mon.a.smithi063.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T16:46:59.628 INFO:journalctl@ceph.mon.a.smithi063.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T16:46:59.628 INFO:journalctl@ceph.mon.a.smithi063.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T16:46:59.628 INFO:journalctl@ceph.mon.a.smithi063.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T16:46:59.628 INFO:journalctl@ceph.mon.a.smithi063.stdout: 2024-08-17T16:46:59.628 INFO:journalctl@ceph.mon.a.smithi063.stdout: ** Compaction Stats [default] ** 2024-08-17T16:46:59.629 INFO:journalctl@ceph.mon.a.smithi063.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T16:46:59.629 INFO:journalctl@ceph.mon.a.smithi063.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T16:46:59.629 INFO:journalctl@ceph.mon.a.smithi063.stdout: L0 2/0 81.00 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 124.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T16:46:59.629 INFO:journalctl@ceph.mon.a.smithi063.stdout: Sum 2/0 81.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 124.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T16:46:59.629 INFO:journalctl@ceph.mon.a.smithi063.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 124.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T16:46:59.629 INFO:journalctl@ceph.mon.a.smithi063.stdout: 2024-08-17T16:46:59.629 INFO:journalctl@ceph.mon.a.smithi063.stdout: ** Compaction Stats [default] ** 2024-08-17T16:46:59.629 INFO:journalctl@ceph.mon.a.smithi063.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T16:46:59.629 INFO:journalctl@ceph.mon.a.smithi063.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T16:46:59.629 INFO:journalctl@ceph.mon.a.smithi063.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 124.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T16:46:59.629 INFO:journalctl@ceph.mon.a.smithi063.stdout: 2024-08-17T16:46:59.629 INFO:journalctl@ceph.mon.a.smithi063.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T16:46:59.630 INFO:journalctl@ceph.mon.a.smithi063.stdout: 2024-08-17T16:46:59.630 INFO:journalctl@ceph.mon.a.smithi063.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:46:59.630 INFO:journalctl@ceph.mon.a.smithi063.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T16:46:59.630 INFO:journalctl@ceph.mon.a.smithi063.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T16:46:59.630 INFO:journalctl@ceph.mon.a.smithi063.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T16:46:59.630 INFO:journalctl@ceph.mon.a.smithi063.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T16:46:59.630 INFO:journalctl@ceph.mon.a.smithi063.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T16:46:59.630 INFO:journalctl@ceph.mon.a.smithi063.stdout: Cumulative compaction: 0.00 GB write, 14.88 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:46:59.630 INFO:journalctl@ceph.mon.a.smithi063.stdout: Interval compaction: 0.00 GB write, 14.88 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:46:59.630 INFO:journalctl@ceph.mon.a.smithi063.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T16:46:59.630 INFO:journalctl@ceph.mon.a.smithi063.stdout: Block cache BinnedLRUCache@0x55ef0dca3350#7 capacity: 512.00 MB usage: 1.19 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 8e-06 secs_since: 0 2024-08-17T16:46:59.630 INFO:journalctl@ceph.mon.a.smithi063.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,0.77 KB,0.000146031%) IndexBlock(2,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2024-08-17T16:46:59.631 INFO:journalctl@ceph.mon.a.smithi063.stdout: 2024-08-17T16:46:59.631 INFO:journalctl@ceph.mon.a.smithi063.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T16:46:59.631 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: starting mon.a rank 0 at public addrs [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] at bind addrs [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:46:59.631 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: mon.a@-1(???) e1 preinit fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:46:59.631 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: mon.a@-1(???).mds e1 new map 2024-08-17T16:46:59.631 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: mon.a@-1(???).mds e1 print_map 2024-08-17T16:46:59.631 INFO:journalctl@ceph.mon.a.smithi063.stdout: e1 2024-08-17T16:46:59.631 INFO:journalctl@ceph.mon.a.smithi063.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T16:46:59.631 INFO:journalctl@ceph.mon.a.smithi063.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T16:46:59.631 INFO:journalctl@ceph.mon.a.smithi063.stdout: legacy client fscid: -1 2024-08-17T16:46:59.631 INFO:journalctl@ceph.mon.a.smithi063.stdout: 2024-08-17T16:46:59.631 INFO:journalctl@ceph.mon.a.smithi063.stdout: No filesystems configured 2024-08-17T16:46:59.632 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-08-17T16:46:59.632 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:46:59.632 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:46:59.632 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:46:59.632 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-08-17T16:46:59.632 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: mon.a is new leader, mons a in quorum (ranks 0) 2024-08-17T16:46:59.632 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: monmap e1: 1 mons at {a=[v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:46:59.632 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: fsmap 2024-08-17T16:46:59.632 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: osdmap e1: 0 total, 0 up, 0 in 2024-08-17T16:46:59.632 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 ceph-mon[31063]: mgrmap e1: no daemons active 2024-08-17T16:46:59.632 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 bash[31025]: 0c71ddd7ada67f493be747616ba47d4bd768d4b951091cc55699858aff6fe2c2 2024-08-17T16:46:59.632 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:46:59 smithi063 systemd[1]: Started Ceph mon.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:47:00.457 INFO:teuthology.orchestra.run.smithi063.stdout:Wrote config to /etc/ceph/ceph.conf 2024-08-17T16:47:00.478 INFO:teuthology.orchestra.run.smithi063.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-08-17T16:47:00.479 INFO:teuthology.orchestra.run.smithi063.stdout:Creating mgr... 2024-08-17T16:47:00.479 INFO:teuthology.orchestra.run.smithi063.stdout:Verifying port 0.0.0.0:9283 ... 2024-08-17T16:47:00.479 INFO:teuthology.orchestra.run.smithi063.stdout:Verifying port 0.0.0.0:8765 ... 2024-08-17T16:47:00.729 INFO:teuthology.orchestra.run.smithi063.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.y 2024-08-17T16:47:00.729 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stderr Failed to reset failed state of unit ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.y.service: Unit ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.y.service not loaded. 2024-08-17T16:47:00.881 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-321fa59e-5cb8-11ef-bccf-c7b262605968.target.wants/ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@.service. 2024-08-17T16:47:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:00 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/1790774863' entity='client.admin' 2024-08-17T16:47:01.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:00 smithi063 systemd[1]: Starting Ceph mgr.y for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:47:01.459 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:01 smithi063 podman[31300]: 2024-08-17 16:47:01.030604411 +0000 UTC m=+0.017609128 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph:reef 2024-08-17T16:47:01.459 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:01 smithi063 podman[31300]: 2024-08-17 16:47:01.139370631 +0000 UTC m=+0.126375370 container create ba4a2894b308c957b3277cb4f56185e99f5f0219cec4b67d5701bd6948e777b8 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-08-17T16:47:01.459 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:01 smithi063 podman[31300]: 2024-08-17 16:47:01.358488016 +0000 UTC m=+0.345492713 container init ba4a2894b308c957b3277cb4f56185e99f5f0219cec4b67d5701bd6948e777b8 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T16:47:01.459 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:01 smithi063 podman[31300]: 2024-08-17 16:47:01.36349522 +0000 UTC m=+0.350499916 container start ba4a2894b308c957b3277cb4f56185e99f5f0219cec4b67d5701bd6948e777b8 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, ceph=True) 2024-08-17T16:47:01.459 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:01 smithi063 bash[31300]: ba4a2894b308c957b3277cb4f56185e99f5f0219cec4b67d5701bd6948e777b8 2024-08-17T16:47:01.472 INFO:teuthology.orchestra.run.smithi063.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T16:47:01.472 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stdout disabled 2024-08-17T16:47:01.480 INFO:teuthology.orchestra.run.smithi063.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T16:47:01.480 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stdout inactive 2024-08-17T16:47:01.480 INFO:teuthology.orchestra.run.smithi063.stdout:firewalld.service is not enabled 2024-08-17T16:47:01.480 INFO:teuthology.orchestra.run.smithi063.stdout:Not possible to enable service . firewalld.service is not available 2024-08-17T16:47:01.489 INFO:teuthology.orchestra.run.smithi063.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T16:47:01.489 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stdout disabled 2024-08-17T16:47:01.498 INFO:teuthology.orchestra.run.smithi063.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T16:47:01.498 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stdout inactive 2024-08-17T16:47:01.499 INFO:teuthology.orchestra.run.smithi063.stdout:firewalld.service is not enabled 2024-08-17T16:47:01.499 INFO:teuthology.orchestra.run.smithi063.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-08-17T16:47:01.499 INFO:teuthology.orchestra.run.smithi063.stdout:Waiting for mgr to start... 2024-08-17T16:47:01.499 INFO:teuthology.orchestra.run.smithi063.stdout:Waiting for mgr... 2024-08-17T16:47:01.724 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:01 smithi063 systemd[1]: Started Ceph mgr.y for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:47:01.724 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:01 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:01.488+0000 7fb09d47d040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:47:01.724 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:01 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:01.646+0000 7fb09d47d040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:47:01.724 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:01 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:01.721+0000 7fb09d47d040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:47:02.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:02 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/4247316057' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-17T16:47:02.279 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:01 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:01.980+0000 7fb09d47d040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:47:02.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:02 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:02.046+0000 7fb09d47d040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:47:02.587 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 2024-08-17T16:47:02.587 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:47:02.587 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "fsid": "321fa59e-5cb8-11ef-bccf-c7b262605968", 2024-08-17T16:47:02.587 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "health": { 2024-08-17T16:47:02.587 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-08-17T16:47:02.587 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-08-17T16:47:02.587 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-08-17T16:47:02.588 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:02.588 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-08-17T16:47:02.588 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-08-17T16:47:02.588 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 0 2024-08-17T16:47:02.588 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:47:02.588 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-08-17T16:47:02.588 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "a" 2024-08-17T16:47:02.588 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:47:02.588 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-08-17T16:47:02.588 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "monmap": { 2024-08-17T16:47:02.588 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:47:02.588 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-08-17T16:47:02.588 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-08-17T16:47:02.589 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:02.589 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-08-17T16:47:02.589 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:47:02.589 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-08-17T16:47:02.589 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-08-17T16:47:02.589 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-08-17T16:47:02.589 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-08-17T16:47:02.592 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-08-17T16:47:02.592 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-08-17T16:47:02.592 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:02.592 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-08-17T16:47:02.593 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-08-17T16:47:02.593 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-08-17T16:47:02.593 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-08-17T16:47:02.593 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-08-17T16:47:02.593 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-08-17T16:47:02.593 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-08-17T16:47:02.593 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-08-17T16:47:02.593 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-08-17T16:47:02.593 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:02.593 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-08-17T16:47:02.593 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:47:02.593 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-08-17T16:47:02.593 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-08-17T16:47:02.594 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:02.594 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-08-17T16:47:02.594 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "available": false, 2024-08-17T16:47:02.594 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-08-17T16:47:02.594 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "modules": [ 2024-08-17T16:47:02.594 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "iostat", 2024-08-17T16:47:02.594 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "nfs", 2024-08-17T16:47:02.594 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "restful" 2024-08-17T16:47:02.594 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:47:02.594 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:47:02.594 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:02.594 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-08-17T16:47:02.594 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:47:02.594 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "modified": "2024-08-17T16:46:54.156458+0000", 2024-08-17T16:47:02.595 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:47:02.595 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:02.595 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-08-17T16:47:02.595 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:47:02.595 INFO:teuthology.orchestra.run.smithi063.stdout:mgr not available, waiting (1/15)... 2024-08-17T16:47:03.423 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:03.027+0000 7fb09d47d040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:47:03.423 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:03.145+0000 7fb09d47d040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:47:03.423 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:03.203+0000 7fb09d47d040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:47:03.423 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:03.421+0000 7fb09d47d040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:47:03.701 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:03.483+0000 7fb09d47d040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:47:03.702 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:03.551+0000 7fb09d47d040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:47:04.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:03.699+0000 7fb09d47d040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:47:04.456 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:04 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:04.131+0000 7fb09d47d040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:47:04.456 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:04 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:04.454+0000 7fb09d47d040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:47:04.708 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:04 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:04.522+0000 7fb09d47d040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:47:04.708 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:04 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:04.677+0000 7fb09d47d040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:47:05.270 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:05 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/4019690476' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-17T16:47:05.270 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:05 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:05.008+0000 7fb09d47d040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:47:05.271 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:05 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:05.073+0000 7fb09d47d040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:47:05.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:05 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:05.268+0000 7fb09d47d040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:47:05.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:05 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:05.427+0000 7fb09d47d040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-17T16:47:05.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:05 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:05.493+0000 7fb09d47d040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:47:05.646 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 2024-08-17T16:47:05.646 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:47:05.646 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "fsid": "321fa59e-5cb8-11ef-bccf-c7b262605968", 2024-08-17T16:47:05.647 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "health": { 2024-08-17T16:47:05.647 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-08-17T16:47:05.647 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-08-17T16:47:05.647 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-08-17T16:47:05.647 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:05.647 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-08-17T16:47:05.647 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-08-17T16:47:05.647 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 0 2024-08-17T16:47:05.647 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:47:05.647 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-08-17T16:47:05.647 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "a" 2024-08-17T16:47:05.647 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:47:05.647 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-08-17T16:47:05.648 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "monmap": { 2024-08-17T16:47:05.648 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:47:05.648 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-08-17T16:47:05.648 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-08-17T16:47:05.648 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:05.648 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-08-17T16:47:05.648 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:47:05.648 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-08-17T16:47:05.648 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-08-17T16:47:05.648 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-08-17T16:47:05.648 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-08-17T16:47:05.648 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-08-17T16:47:05.648 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-08-17T16:47:05.648 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:05.649 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-08-17T16:47:05.649 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-08-17T16:47:05.649 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-08-17T16:47:05.649 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-08-17T16:47:05.649 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-08-17T16:47:05.649 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-08-17T16:47:05.649 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-08-17T16:47:05.649 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-08-17T16:47:05.649 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-08-17T16:47:05.649 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:05.649 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-08-17T16:47:05.649 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:47:05.649 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-08-17T16:47:05.650 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-08-17T16:47:05.650 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:05.650 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-08-17T16:47:05.650 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "available": false, 2024-08-17T16:47:05.650 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-08-17T16:47:05.650 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "modules": [ 2024-08-17T16:47:05.650 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "iostat", 2024-08-17T16:47:05.650 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "nfs", 2024-08-17T16:47:05.650 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "restful" 2024-08-17T16:47:05.650 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:47:05.650 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:47:05.650 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:05.651 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-08-17T16:47:05.651 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:47:05.651 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "modified": "2024-08-17T16:46:54.156458+0000", 2024-08-17T16:47:05.651 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:47:05.651 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:05.651 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-08-17T16:47:05.651 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:47:05.651 INFO:teuthology.orchestra.run.smithi063.stdout:mgr not available, waiting (2/15)... 2024-08-17T16:47:06.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:05 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:05.579+0000 7fb09d47d040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:47:06.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:05 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:05.641+0000 7fb09d47d040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:47:06.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:06.189+0000 7fb09d47d040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:47:06.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:06.354+0000 7fb09d47d040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:47:06.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:06 smithi063 ceph-mon[31063]: Activating manager daemon y 2024-08-17T16:47:06.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:06 smithi063 ceph-mon[31063]: mgrmap e2: y(active, starting, since 0.00301328s) 2024-08-17T16:47:06.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:06 smithi063 ceph-mon[31063]: from='mgr.14100 172.21.15.63:0/2396398425' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:47:06.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:06 smithi063 ceph-mon[31063]: from='mgr.14100 172.21.15.63:0/2396398425' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:47:06.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:06 smithi063 ceph-mon[31063]: from='mgr.14100 172.21.15.63:0/2396398425' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:47:06.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:06 smithi063 ceph-mon[31063]: from='mgr.14100 172.21.15.63:0/2396398425' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:47:06.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:06 smithi063 ceph-mon[31063]: from='mgr.14100 172.21.15.63:0/2396398425' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:47:06.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:06 smithi063 ceph-mon[31063]: Manager daemon y is now available 2024-08-17T16:47:06.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:06 smithi063 ceph-mon[31063]: from='mgr.14100 172.21.15.63:0/2396398425' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:47:06.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:06 smithi063 ceph-mon[31063]: from='mgr.14100 172.21.15.63:0/2396398425' entity='mgr.y' 2024-08-17T16:47:06.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:06 smithi063 ceph-mon[31063]: from='mgr.14100 172.21.15.63:0/2396398425' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:47:06.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:06 smithi063 ceph-mon[31063]: from='mgr.14100 172.21.15.63:0/2396398425' entity='mgr.y' 2024-08-17T16:47:06.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:06 smithi063 ceph-mon[31063]: from='mgr.14100 172.21.15.63:0/2396398425' entity='mgr.y' 2024-08-17T16:47:08.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:08 smithi063 ceph-mon[31063]: mgrmap e3: y(active, since 1.00801s) 2024-08-17T16:47:08.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:08 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3808452942' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-08-17T16:47:08.900 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 2024-08-17T16:47:08.900 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:47:08.900 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "fsid": "321fa59e-5cb8-11ef-bccf-c7b262605968", 2024-08-17T16:47:08.900 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "health": { 2024-08-17T16:47:08.900 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-08-17T16:47:08.900 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-08-17T16:47:08.900 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-08-17T16:47:08.900 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:08.900 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-08-17T16:47:08.901 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-08-17T16:47:08.901 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 0 2024-08-17T16:47:08.901 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:47:08.901 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-08-17T16:47:08.901 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "a" 2024-08-17T16:47:08.901 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:47:08.901 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2024-08-17T16:47:08.901 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "monmap": { 2024-08-17T16:47:08.901 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:47:08.901 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-08-17T16:47:08.901 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-08-17T16:47:08.901 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:08.901 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-08-17T16:47:08.901 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:47:08.902 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-08-17T16:47:08.902 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-08-17T16:47:08.902 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-08-17T16:47:08.902 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-08-17T16:47:08.905 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-08-17T16:47:08.905 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-08-17T16:47:08.905 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:08.905 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-08-17T16:47:08.905 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-08-17T16:47:08.905 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-08-17T16:47:08.905 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-08-17T16:47:08.906 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-08-17T16:47:08.906 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-08-17T16:47:08.906 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-08-17T16:47:08.906 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-08-17T16:47:08.906 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-08-17T16:47:08.906 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:08.906 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-08-17T16:47:08.906 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:47:08.906 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-08-17T16:47:08.906 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-08-17T16:47:08.906 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:08.906 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-08-17T16:47:08.907 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "available": true, 2024-08-17T16:47:08.907 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-08-17T16:47:08.907 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "modules": [ 2024-08-17T16:47:08.907 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "iostat", 2024-08-17T16:47:08.907 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "nfs", 2024-08-17T16:47:08.907 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "restful" 2024-08-17T16:47:08.907 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout ], 2024-08-17T16:47:08.907 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:47:08.907 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:08.907 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-08-17T16:47:08.907 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-08-17T16:47:08.907 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "modified": "2024-08-17T16:46:54.156458+0000", 2024-08-17T16:47:08.907 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "services": {} 2024-08-17T16:47:08.907 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout }, 2024-08-17T16:47:08.908 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-08-17T16:47:08.908 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:47:08.908 INFO:teuthology.orchestra.run.smithi063.stdout:mgr is available 2024-08-17T16:47:09.523 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:09 smithi063 ceph-mon[31063]: mgrmap e4: y(active, since 2s) 2024-08-17T16:47:10.059 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 2024-08-17T16:47:10.059 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout [global] 2024-08-17T16:47:10.059 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout fsid = 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:47:10.059 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-08-17T16:47:10.060 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-08-17T16:47:10.060 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-08-17T16:47:10.060 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-08-17T16:47:10.060 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 2024-08-17T16:47:10.060 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout [mgr] 2024-08-17T16:47:10.060 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-08-17T16:47:10.060 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 2024-08-17T16:47:10.060 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout [osd] 2024-08-17T16:47:10.060 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-08-17T16:47:10.060 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-08-17T16:47:10.060 INFO:teuthology.orchestra.run.smithi063.stdout:Enabling cephadm module... 2024-08-17T16:47:10.487 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:10 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3776772124' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-08-17T16:47:11.579 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:11 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/942634042' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-08-17T16:47:11.580 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:11 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ignoring --setuser ceph since I am not root 2024-08-17T16:47:11.580 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:11 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ignoring --setgroup ceph since I am not root 2024-08-17T16:47:11.580 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:11 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:11.517+0000 7f20a1ad2040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:47:12.002 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:11 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:11.671+0000 7f20a1ad2040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:47:12.002 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:11 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:11.742+0000 7f20a1ad2040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:47:12.002 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:12 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:12.000+0000 7f20a1ad2040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:47:12.279 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:12 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:12.073+0000 7f20a1ad2040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:47:12.579 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:12 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/942634042' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-08-17T16:47:12.580 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:12 smithi063 ceph-mon[31063]: mgrmap e5: y(active, since 5s) 2024-08-17T16:47:13.357 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:13.054+0000 7f20a1ad2040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:47:13.357 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:13.174+0000 7f20a1ad2040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:47:13.357 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:13.232+0000 7f20a1ad2040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:47:13.359 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:47:13.360 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-08-17T16:47:13.360 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "available": true, 2024-08-17T16:47:13.360 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-08-17T16:47:13.360 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-08-17T16:47:13.360 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:47:13.360 INFO:teuthology.orchestra.run.smithi063.stdout:Waiting for the mgr to restart... 2024-08-17T16:47:13.361 INFO:teuthology.orchestra.run.smithi063.stdout:Waiting for mgr epoch 5... 2024-08-17T16:47:13.629 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:13 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/853318557' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-08-17T16:47:13.630 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:13.460+0000 7f20a1ad2040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:47:13.630 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:13.524+0000 7f20a1ad2040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:47:13.630 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:13.593+0000 7f20a1ad2040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:47:14.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:13.759+0000 7f20a1ad2040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:47:14.519 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:14 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:14.194+0000 7f20a1ad2040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:47:14.519 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:14 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:14.518+0000 7f20a1ad2040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:47:14.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:14 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:14.585+0000 7f20a1ad2040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:47:14.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:14 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:14.737+0000 7f20a1ad2040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:47:15.309 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:15 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:15.054+0000 7f20a1ad2040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:47:15.309 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:15 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:15.117+0000 7f20a1ad2040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:47:15.613 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:15 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:15.307+0000 7f20a1ad2040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:47:15.613 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:15 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:15.461+0000 7f20a1ad2040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-17T16:47:15.613 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:15 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:15.526+0000 7f20a1ad2040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:47:15.614 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:15 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:15.612+0000 7f20a1ad2040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:47:16.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:15 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:15.673+0000 7f20a1ad2040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:47:16.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:16.220+0000 7f20a1ad2040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:47:16.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:16.382+0000 7f20a1ad2040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:47:16.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: Active manager daemon y restarted 2024-08-17T16:47:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: Activating manager daemon y 2024-08-17T16:47:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: osdmap e2: 0 total, 0 up, 0 in 2024-08-17T16:47:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: mgrmap e6: y(active, starting, since 0.00330905s) 2024-08-17T16:47:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:47:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:47:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:47:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:47:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:47:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: Manager daemon y is now available 2024-08-17T16:47:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: Found migration_current of "None". Setting to last migration. 2024-08-17T16:47:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' 2024-08-17T16:47:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' 2024-08-17T16:47:16.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:47:16.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:47:16.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:47:16.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:16 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:47:17.949 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:47:17.950 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-08-17T16:47:17.950 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "initialized": true 2024-08-17T16:47:17.950 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:47:17.950 INFO:teuthology.orchestra.run.smithi063.stdout:mgr epoch 5 is available 2024-08-17T16:47:17.950 INFO:teuthology.orchestra.run.smithi063.stdout:Setting orchestrator backend to cephadm... 2024-08-17T16:47:18.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:18 smithi063 ceph-mon[31063]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-08-17T16:47:18.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:18 smithi063 ceph-mon[31063]: mgrmap e7: y(active, since 1.00524s) 2024-08-17T16:47:18.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:18 smithi063 ceph-mon[31063]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-08-17T16:47:18.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:18 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' 2024-08-17T16:47:18.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:18 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' 2024-08-17T16:47:19.722 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-mon[31063]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:47:19.723 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' 2024-08-17T16:47:19.723 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:47:19.723 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-mon[31063]: mgrmap e8: y(active, since 2s) 2024-08-17T16:47:20.017 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:19.878+0000 7f200f16c640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:16:47:19] ENGINE Error in HTTPServer.serve 2024-08-17T16:47:20.017 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: Traceback (most recent call last): 2024-08-17T16:47:20.017 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:47:20.017 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: self._connections.run(self.expiration_interval) 2024-08-17T16:47:20.017 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:47:20.018 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: self._run(expiration_interval) 2024-08-17T16:47:20.018 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:47:20.018 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:47:20.018 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:47:20.018 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:47:20.018 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:47:20.018 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: s = self.context.wrap_socket( 2024-08-17T16:47:20.018 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:47:20.018 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: return self.sslsocket_class._create( 2024-08-17T16:47:20.018 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:47:20.018 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: self.do_handshake() 2024-08-17T16:47:20.018 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:47:20.019 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: self._sslobj.do_handshake() 2024-08-17T16:47:20.019 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:47:20.019 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:20.262 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout value unchanged 2024-08-17T16:47:20.262 INFO:teuthology.orchestra.run.smithi063.stdout:Generating ssh key... 2024-08-17T16:47:20.712 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:20 smithi063 ceph-mon[31063]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:47:20.712 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:20 smithi063 ceph-mon[31063]: [17/Aug/2024:16:47:19] ENGINE Bus STARTING 2024-08-17T16:47:20.712 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:20 smithi063 ceph-mon[31063]: [17/Aug/2024:16:47:19] ENGINE Serving on https://172.21.15.63:7150 2024-08-17T16:47:20.712 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:20 smithi063 ceph-mon[31063]: [17/Aug/2024:16:47:19] ENGINE Error in HTTPServer.serve 2024-08-17T16:47:20.712 INFO:journalctl@ceph.mon.a.smithi063.stdout: Traceback (most recent call last): 2024-08-17T16:47:20.712 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:47:20.712 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:47:20.712 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:47:20.713 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._run(expiration_interval) 2024-08-17T16:47:20.713 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:47:20.713 INFO:journalctl@ceph.mon.a.smithi063.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:47:20.713 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:47:20.713 INFO:journalctl@ceph.mon.a.smithi063.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:47:20.713 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:47:20.713 INFO:journalctl@ceph.mon.a.smithi063.stdout: s = self.context.wrap_socket( 2024-08-17T16:47:20.713 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:47:20.713 INFO:journalctl@ceph.mon.a.smithi063.stdout: return self.sslsocket_class._create( 2024-08-17T16:47:20.713 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:47:20.713 INFO:journalctl@ceph.mon.a.smithi063.stdout: self.do_handshake() 2024-08-17T16:47:20.713 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:47:20.714 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._sslobj.do_handshake() 2024-08-17T16:47:20.714 INFO:journalctl@ceph.mon.a.smithi063.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:47:20.714 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:20 smithi063 ceph-mon[31063]: [17/Aug/2024:16:47:19] ENGINE Serving on http://172.21.15.63:8765 2024-08-17T16:47:20.714 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:20 smithi063 ceph-mon[31063]: [17/Aug/2024:16:47:19] ENGINE Bus STARTED 2024-08-17T16:47:20.714 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:20 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:47:21.432 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: Generating public/private rsa key pair. 2024-08-17T16:47:21.432 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: Your identification has been saved in /tmp/tmpncvm062e/key 2024-08-17T16:47:21.432 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: Your public key has been saved in /tmp/tmpncvm062e/key.pub 2024-08-17T16:47:21.432 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: The key fingerprint is: 2024-08-17T16:47:21.432 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: SHA256:wN9O2okVCfxwIwmrlDoOQDPWyZZvTECWUPb3nIb9JuQ ceph-321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:47:21.432 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: The key's randomart image is: 2024-08-17T16:47:21.432 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: +---[RSA 3072]----+ 2024-08-17T16:47:21.432 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: | ***= .o.. | 2024-08-17T16:47:21.433 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: |o =B.+ .=.o. | 2024-08-17T16:47:21.433 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: |. . *.+. =o. | 2024-08-17T16:47:21.433 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: |. o =.o=.o. | 2024-08-17T16:47:21.433 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: |. o o .SB+ | 2024-08-17T16:47:21.433 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: | o . +B.. | 2024-08-17T16:47:21.433 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: | . oE+o | 2024-08-17T16:47:21.433 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: | o | 2024-08-17T16:47:21.433 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: | | 2024-08-17T16:47:21.433 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:21 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: +----[SHA256]-----+ 2024-08-17T16:47:22.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:22 smithi063 ceph-mon[31063]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:47:22.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:22 smithi063 ceph-mon[31063]: Generating ssh key... 2024-08-17T16:47:22.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:22 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' 2024-08-17T16:47:22.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:22 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' 2024-08-17T16:47:22.942 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJdYwxHTnNqdehj1I1mB8k4HSCzy2sgQgesp8ftlYwVGzCfQL4m07eTYMqSAmUnIFAOB9W9t3523O4A2zkpZmqKXYtvQDcGSSUeWmMhUPxIBW4N28G/ErMuz02elDM4yfUw6Kasr7oclZtvu3MHCQEeRQiqu2D6279AoIxEW5VVUd4gwOTNHOLBx9lcRaBGOBWYbb6JtrTN7jvHCY5oExGCWicvkTqKTg8U1snNEq57lQ8jgzo5CXcvUZe6NcupfWKptFjM2Rvowgu+XBUjrEVAdzB4e7mYNQhg9qgCagQ/dvony+qjdmWX/txifPa8k7FiYhlknqzY4ciUQfi+sw/c7Rbk6X1nNx5cvXhTg50JjS+dT0x6LKD9/9axMmBv8C8sSWyNfaAnqnk1iAkMcFyUFQS37tf4ccrZo48X5jj0ub0catZB09tkzFheSyKml+DdOD3A0cFIItUSYO5B2B1XhhlJeF/x45FVmzkxEnfAn36nZMpq8UbPpeAQnTAJe8= ceph-321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:47:22.942 INFO:teuthology.orchestra.run.smithi063.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-08-17T16:47:22.942 INFO:teuthology.orchestra.run.smithi063.stdout:Adding key to root@localhost authorized_keys... 2024-08-17T16:47:22.942 INFO:teuthology.orchestra.run.smithi063.stdout:Adding host smithi063... 2024-08-17T16:47:23.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:23 smithi063 ceph-mon[31063]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:47:24.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:24 smithi063 ceph-mon[31063]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi063", "addr": "172.21.15.63", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:47:25.772 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:25 smithi063 ceph-mon[31063]: Deploying cephadm binary to smithi063 2024-08-17T16:47:27.391 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout Added host 'smithi063' with addr '172.21.15.63' 2024-08-17T16:47:27.391 INFO:teuthology.orchestra.run.smithi063.stdout:Deploying unmanaged mon service... 2024-08-17T16:47:28.027 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:27 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' 2024-08-17T16:47:28.027 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:27 smithi063 ceph-mon[31063]: Added host smithi063 2024-08-17T16:47:28.027 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:27 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:47:29.437 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:29 smithi063 ceph-mon[31063]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:47:29.437 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:29 smithi063 ceph-mon[31063]: Saving service mon spec with placement count:5 2024-08-17T16:47:29.438 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:29 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' 2024-08-17T16:47:29.438 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:29 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' 2024-08-17T16:47:29.438 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:29 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' 2024-08-17T16:47:29.438 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-08-17T16:47:29.438 INFO:teuthology.orchestra.run.smithi063.stdout:Deploying unmanaged mgr service... 2024-08-17T16:47:30.889 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-08-17T16:47:31.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:31 smithi063 ceph-mon[31063]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:47:31.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:31 smithi063 ceph-mon[31063]: Saving service mgr spec with placement count:2 2024-08-17T16:47:31.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:31 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' 2024-08-17T16:47:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:32 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' 2024-08-17T16:47:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:32 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3370631755' entity='client.admin' 2024-08-17T16:47:33.610 INFO:teuthology.orchestra.run.smithi063.stdout:Enabling the dashboard module... 2024-08-17T16:47:34.171 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:33 smithi063 ceph-mon[31063]: from='mgr.14118 172.21.15.63:0/348585141' entity='mgr.y' 2024-08-17T16:47:34.172 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:33 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/1692731103' entity='client.admin' 2024-08-17T16:47:35.167 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:34 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/2800888405' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-08-17T16:47:35.168 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:34 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ignoring --setuser ceph since I am not root 2024-08-17T16:47:35.168 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:34 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ignoring --setgroup ceph since I am not root 2024-08-17T16:47:35.168 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:35.046+0000 7faa68063040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:47:35.518 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:35.197+0000 7faa68063040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:47:35.518 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:35.267+0000 7faa68063040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:47:35.518 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:35.517+0000 7faa68063040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:47:35.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:35.579+0000 7faa68063040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:47:36.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:35 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/2800888405' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-08-17T16:47:36.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:35 smithi063 ceph-mon[31063]: mgrmap e9: y(active, since 18s) 2024-08-17T16:47:36.754 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:36.574+0000 7faa68063040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:47:36.755 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:36.694+0000 7faa68063040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:47:36.755 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:36.753+0000 7faa68063040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:47:37.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:36 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/861134591' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-08-17T16:47:37.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:36.973+0000 7faa68063040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:47:37.073 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:47:37.073 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-08-17T16:47:37.073 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "available": true, 2024-08-17T16:47:37.073 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-08-17T16:47:37.074 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-08-17T16:47:37.074 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:47:37.074 INFO:teuthology.orchestra.run.smithi063.stdout:Waiting for the mgr to restart... 2024-08-17T16:47:37.074 INFO:teuthology.orchestra.run.smithi063.stdout:Waiting for mgr epoch 9... 2024-08-17T16:47:37.300 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:37 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:37.036+0000 7faa68063040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:47:37.300 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:37 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:37.113+0000 7faa68063040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:47:37.300 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:37 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:37.264+0000 7faa68063040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:47:38.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:37 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:37.703+0000 7faa68063040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:47:38.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:38.027+0000 7faa68063040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:47:38.279 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:38.095+0000 7faa68063040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:47:38.279 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:38.248+0000 7faa68063040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:47:38.823 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:38.567+0000 7faa68063040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:47:38.823 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:38.630+0000 7faa68063040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:47:39.128 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:38.821+0000 7faa68063040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:47:39.128 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:38.975+0000 7faa68063040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-17T16:47:39.128 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:39.041+0000 7faa68063040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:47:39.128 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:39.127+0000 7faa68063040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:47:39.397 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:39.190+0000 7faa68063040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:47:40.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:39.751+0000 7faa68063040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:47:40.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:39.917+0000 7faa68063040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:47:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-mon[31063]: Active manager daemon y restarted 2024-08-17T16:47:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-mon[31063]: Activating manager daemon y 2024-08-17T16:47:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-mon[31063]: osdmap e3: 0 total, 0 up, 0 in 2024-08-17T16:47:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-mon[31063]: mgrmap e10: y(active, starting, since 0.00332319s) 2024-08-17T16:47:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:47:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:47:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:47:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:47:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:47:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-mon[31063]: Manager daemon y is now available 2024-08-17T16:47:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:47:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:47:40.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:47:41.857 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout { 2024-08-17T16:47:41.857 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-08-17T16:47:41.858 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout "initialized": true 2024-08-17T16:47:41.858 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout } 2024-08-17T16:47:41.858 INFO:teuthology.orchestra.run.smithi063.stdout:mgr epoch 9 is available 2024-08-17T16:47:41.858 INFO:teuthology.orchestra.run.smithi063.stdout:Generating a dashboard self-signed certificate... 2024-08-17T16:47:42.121 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:41 smithi063 ceph-mon[31063]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-08-17T16:47:42.121 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:41 smithi063 ceph-mon[31063]: mgrmap e11: y(active, since 1.00567s) 2024-08-17T16:47:42.121 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:41 smithi063 ceph-mon[31063]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-08-17T16:47:42.121 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:41 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:47:42.121 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:41 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:47:42.626 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:42.520+0000 7fa9d3e6c640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:16:47:42] ENGINE Error in HTTPServer.serve 2024-08-17T16:47:42.627 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: Traceback (most recent call last): 2024-08-17T16:47:42.627 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:47:42.627 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: self._connections.run(self.expiration_interval) 2024-08-17T16:47:42.627 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:47:42.627 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: self._run(expiration_interval) 2024-08-17T16:47:42.627 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:47:42.627 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:47:42.627 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:47:42.627 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:47:42.628 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:47:42.628 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: s = self.context.wrap_socket( 2024-08-17T16:47:42.628 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:47:42.628 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: return self.sslsocket_class._create( 2024-08-17T16:47:42.628 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:47:42.628 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: self.do_handshake() 2024-08-17T16:47:42.628 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:47:42.628 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: self._sslobj.do_handshake() 2024-08-17T16:47:42.628 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:47:42.628 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:47:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:47:43.191 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-08-17T16:47:43.191 INFO:teuthology.orchestra.run.smithi063.stdout:Creating initial admin user... 2024-08-17T16:47:43.970 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:43 smithi063 ceph-mon[31063]: [17/Aug/2024:16:47:42] ENGINE Bus STARTING 2024-08-17T16:47:43.970 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:43 smithi063 ceph-mon[31063]: [17/Aug/2024:16:47:42] ENGINE Serving on https://172.21.15.63:7150 2024-08-17T16:47:43.970 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:43 smithi063 ceph-mon[31063]: [17/Aug/2024:16:47:42] ENGINE Error in HTTPServer.serve 2024-08-17T16:47:43.970 INFO:journalctl@ceph.mon.a.smithi063.stdout: Traceback (most recent call last): 2024-08-17T16:47:43.970 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:47:43.970 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:47:43.970 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:47:43.971 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._run(expiration_interval) 2024-08-17T16:47:43.971 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:47:43.971 INFO:journalctl@ceph.mon.a.smithi063.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:47:43.971 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:47:43.971 INFO:journalctl@ceph.mon.a.smithi063.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:47:43.971 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:47:43.971 INFO:journalctl@ceph.mon.a.smithi063.stdout: s = self.context.wrap_socket( 2024-08-17T16:47:43.971 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:47:43.971 INFO:journalctl@ceph.mon.a.smithi063.stdout: return self.sslsocket_class._create( 2024-08-17T16:47:43.971 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:47:43.971 INFO:journalctl@ceph.mon.a.smithi063.stdout: self.do_handshake() 2024-08-17T16:47:43.972 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:47:43.972 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._sslobj.do_handshake() 2024-08-17T16:47:43.972 INFO:journalctl@ceph.mon.a.smithi063.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:47:43.972 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:43 smithi063 ceph-mon[31063]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:47:43.972 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:43 smithi063 ceph-mon[31063]: [17/Aug/2024:16:47:42] ENGINE Serving on http://172.21.15.63:8765 2024-08-17T16:47:43.972 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:43 smithi063 ceph-mon[31063]: [17/Aug/2024:16:47:42] ENGINE Bus STARTED 2024-08-17T16:47:43.972 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:43 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:47:43.972 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:43 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:47:43.972 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:43 smithi063 ceph-mon[31063]: mgrmap e12: y(active, since 2s) 2024-08-17T16:47:45.554 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$mQj4nuguIH6aU.aQBvY5XOw/lq1SFGa6Zbuw/WNISOEfoS/aeMFK6", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1723913264, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-08-17T16:47:45.554 INFO:teuthology.orchestra.run.smithi063.stdout:Fetching dashboard port number... 2024-08-17T16:47:46.138 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:45 smithi063 ceph-mon[31063]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:47:46.138 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:45 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:47:46.726 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stdout 8443 2024-08-17T16:47:46.736 INFO:teuthology.orchestra.run.smithi063.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-08-17T16:47:46.736 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stdout disabled 2024-08-17T16:47:46.744 INFO:teuthology.orchestra.run.smithi063.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-08-17T16:47:46.744 INFO:teuthology.orchestra.run.smithi063.stdout:systemctl: stdout inactive 2024-08-17T16:47:46.744 INFO:teuthology.orchestra.run.smithi063.stdout:firewalld.service is not enabled 2024-08-17T16:47:46.744 INFO:teuthology.orchestra.run.smithi063.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-08-17T16:47:46.746 INFO:teuthology.orchestra.run.smithi063.stdout:Ceph Dashboard is now available at: 2024-08-17T16:47:46.746 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:47:46.746 INFO:teuthology.orchestra.run.smithi063.stdout: URL: https://smithi063.front.sepia.ceph.com:8443/ 2024-08-17T16:47:46.746 INFO:teuthology.orchestra.run.smithi063.stdout: User: admin 2024-08-17T16:47:46.746 INFO:teuthology.orchestra.run.smithi063.stdout: Password: ddbwft88zn 2024-08-17T16:47:46.746 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:47:46.746 INFO:teuthology.orchestra.run.smithi063.stdout:Saving cluster configuration to /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config directory 2024-08-17T16:47:46.747 INFO:teuthology.orchestra.run.smithi063.stdout:Enabling autotune for osd_memory_target 2024-08-17T16:47:46.983 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:46 smithi063 ceph-mon[31063]: mgrmap e13: y(active, since 6s) 2024-08-17T16:47:46.983 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:46 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3735492656' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-08-17T16:47:49.319 INFO:teuthology.orchestra.run.smithi063.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-08-17T16:47:49.319 INFO:teuthology.orchestra.run.smithi063.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-08-17T16:47:49.319 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:47:49.319 INFO:teuthology.orchestra.run.smithi063.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-08-17T16:47:49.319 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:47:49.319 INFO:teuthology.orchestra.run.smithi063.stdout:Or, if you are only running a single cluster on this host: 2024-08-17T16:47:49.320 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:47:49.320 INFO:teuthology.orchestra.run.smithi063.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-08-17T16:47:49.320 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:47:49.320 INFO:teuthology.orchestra.run.smithi063.stdout:Please consider enabling telemetry to help improve Ceph: 2024-08-17T16:47:49.320 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:47:49.320 INFO:teuthology.orchestra.run.smithi063.stdout: ceph telemetry on 2024-08-17T16:47:49.320 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:47:49.320 INFO:teuthology.orchestra.run.smithi063.stdout:For more information see: 2024-08-17T16:47:49.320 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:47:49.320 INFO:teuthology.orchestra.run.smithi063.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-08-17T16:47:49.320 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:47:49.320 INFO:teuthology.orchestra.run.smithi063.stdout:Bootstrap complete. 2024-08-17T16:47:49.361 INFO:tasks.cephadm:Fetching config... 2024-08-17T16:47:49.362 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T16:47:49.362 DEBUG:teuthology.orchestra.run.smithi063:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-08-17T16:47:49.380 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-08-17T16:47:49.380 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T16:47:49.380 DEBUG:teuthology.orchestra.run.smithi063:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-08-17T16:47:49.437 INFO:tasks.cephadm:Fetching mon keyring... 2024-08-17T16:47:49.437 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T16:47:49.437 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd if=/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/keyring of=/dev/stdout 2024-08-17T16:47:49.505 INFO:tasks.cephadm:Fetching pub ssh key... 2024-08-17T16:47:49.505 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T16:47:49.505 DEBUG:teuthology.orchestra.run.smithi063:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-08-17T16:47:49.561 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-08-17T16:47:49.561 DEBUG:teuthology.orchestra.run.smithi063:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJdYwxHTnNqdehj1I1mB8k4HSCzy2sgQgesp8ftlYwVGzCfQL4m07eTYMqSAmUnIFAOB9W9t3523O4A2zkpZmqKXYtvQDcGSSUeWmMhUPxIBW4N28G/ErMuz02elDM4yfUw6Kasr7oclZtvu3MHCQEeRQiqu2D6279AoIxEW5VVUd4gwOTNHOLBx9lcRaBGOBWYbb6JtrTN7jvHCY5oExGCWicvkTqKTg8U1snNEq57lQ8jgzo5CXcvUZe6NcupfWKptFjM2Rvowgu+XBUjrEVAdzB4e7mYNQhg9qgCagQ/dvony+qjdmWX/txifPa8k7FiYhlknqzY4ciUQfi+sw/c7Rbk6X1nNx5cvXhTg50JjS+dT0x6LKD9/9axMmBv8C8sSWyNfaAnqnk1iAkMcFyUFQS37tf4ccrZo48X5jj0ub0catZB09tkzFheSyKml+DdOD3A0cFIItUSYO5B2B1XhhlJeF/x45FVmzkxEnfAn36nZMpq8UbPpeAQnTAJe8= ceph-321fa59e-5cb8-11ef-bccf-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-08-17T16:47:49.654 INFO:teuthology.orchestra.run.smithi063.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJdYwxHTnNqdehj1I1mB8k4HSCzy2sgQgesp8ftlYwVGzCfQL4m07eTYMqSAmUnIFAOB9W9t3523O4A2zkpZmqKXYtvQDcGSSUeWmMhUPxIBW4N28G/ErMuz02elDM4yfUw6Kasr7oclZtvu3MHCQEeRQiqu2D6279AoIxEW5VVUd4gwOTNHOLBx9lcRaBGOBWYbb6JtrTN7jvHCY5oExGCWicvkTqKTg8U1snNEq57lQ8jgzo5CXcvUZe6NcupfWKptFjM2Rvowgu+XBUjrEVAdzB4e7mYNQhg9qgCagQ/dvony+qjdmWX/txifPa8k7FiYhlknqzY4ciUQfi+sw/c7Rbk6X1nNx5cvXhTg50JjS+dT0x6LKD9/9axMmBv8C8sSWyNfaAnqnk1iAkMcFyUFQS37tf4ccrZo48X5jj0ub0catZB09tkzFheSyKml+DdOD3A0cFIItUSYO5B2B1XhhlJeF/x45FVmzkxEnfAn36nZMpq8UbPpeAQnTAJe8= ceph-321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:47:49.667 DEBUG:teuthology.orchestra.run.smithi162:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJdYwxHTnNqdehj1I1mB8k4HSCzy2sgQgesp8ftlYwVGzCfQL4m07eTYMqSAmUnIFAOB9W9t3523O4A2zkpZmqKXYtvQDcGSSUeWmMhUPxIBW4N28G/ErMuz02elDM4yfUw6Kasr7oclZtvu3MHCQEeRQiqu2D6279AoIxEW5VVUd4gwOTNHOLBx9lcRaBGOBWYbb6JtrTN7jvHCY5oExGCWicvkTqKTg8U1snNEq57lQ8jgzo5CXcvUZe6NcupfWKptFjM2Rvowgu+XBUjrEVAdzB4e7mYNQhg9qgCagQ/dvony+qjdmWX/txifPa8k7FiYhlknqzY4ciUQfi+sw/c7Rbk6X1nNx5cvXhTg50JjS+dT0x6LKD9/9axMmBv8C8sSWyNfaAnqnk1iAkMcFyUFQS37tf4ccrZo48X5jj0ub0catZB09tkzFheSyKml+DdOD3A0cFIItUSYO5B2B1XhhlJeF/x45FVmzkxEnfAn36nZMpq8UbPpeAQnTAJe8= ceph-321fa59e-5cb8-11ef-bccf-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-08-17T16:47:49.709 INFO:teuthology.orchestra.run.smithi162.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDJdYwxHTnNqdehj1I1mB8k4HSCzy2sgQgesp8ftlYwVGzCfQL4m07eTYMqSAmUnIFAOB9W9t3523O4A2zkpZmqKXYtvQDcGSSUeWmMhUPxIBW4N28G/ErMuz02elDM4yfUw6Kasr7oclZtvu3MHCQEeRQiqu2D6279AoIxEW5VVUd4gwOTNHOLBx9lcRaBGOBWYbb6JtrTN7jvHCY5oExGCWicvkTqKTg8U1snNEq57lQ8jgzo5CXcvUZe6NcupfWKptFjM2Rvowgu+XBUjrEVAdzB4e7mYNQhg9qgCagQ/dvony+qjdmWX/txifPa8k7FiYhlknqzY4ciUQfi+sw/c7Rbk6X1nNx5cvXhTg50JjS+dT0x6LKD9/9axMmBv8C8sSWyNfaAnqnk1iAkMcFyUFQS37tf4ccrZo48X5jj0ub0catZB09tkzFheSyKml+DdOD3A0cFIItUSYO5B2B1XhhlJeF/x45FVmzkxEnfAn36nZMpq8UbPpeAQnTAJe8= ceph-321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:47:49.721 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-08-17T16:47:49.935 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:47:50.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:49 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/4157858932' entity='client.admin' 2024-08-17T16:47:53.234 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-08-17T16:47:53.234 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-08-17T16:47:53.773 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:53 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/14523506' entity='client.admin' 2024-08-17T16:47:53.873 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:47:55.003 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:54 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:47:55.004 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:54 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:47:55.004 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:54 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi063", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:47:55.004 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:54 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:47:55.004 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:54 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:47:55.004 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:54 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:47:56.268 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi162 2024-08-17T16:47:56.268 DEBUG:teuthology.orchestra.run.smithi162:> set -ex 2024-08-17T16:47:56.268 DEBUG:teuthology.orchestra.run.smithi162:> dd of=/etc/ceph/ceph.conf 2024-08-17T16:47:56.288 DEBUG:teuthology.orchestra.run.smithi162:> set -ex 2024-08-17T16:47:56.288 DEBUG:teuthology.orchestra.run.smithi162:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:47:56.346 INFO:tasks.cephadm:Adding host smithi162 to orchestrator... 2024-08-17T16:47:56.346 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch host add smithi162 2024-08-17T16:47:56.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:56 smithi063 ceph-mon[31063]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:47:56.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:56 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:47:56.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:56 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:47:56.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:56 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:47:56.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:56 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:47:56.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:56 smithi063 ceph-mon[31063]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:47:56.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:56 smithi063 ceph-mon[31063]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:47:56.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:56 smithi063 ceph-mon[31063]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:47:56.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:56 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:47:56.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:56 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:47:56.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:56 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:47:56.552 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:47:57.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:57 smithi063 ceph-mon[31063]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:47:59.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:47:59 smithi063 ceph-mon[31063]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi162", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:48:01.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:00 smithi063 ceph-mon[31063]: Deploying cephadm binary to smithi162 2024-08-17T16:48:01.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:00 smithi063 ceph-mon[31063]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:01.556 INFO:teuthology.orchestra.run.smithi063.stdout:Added host 'smithi162' with addr '172.21.15.162' 2024-08-17T16:48:02.230 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch host ls --format=json 2024-08-17T16:48:02.433 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:48:02.860 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:02.860 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:02 smithi063 ceph-mon[31063]: Added host smithi162 2024-08-17T16:48:02.860 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:02.860 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:02.860 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:02 smithi063 ceph-mon[31063]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:02.860 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:04.094 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:48:04.094 INFO:teuthology.orchestra.run.smithi063.stdout:[{"addr": "172.21.15.63", "hostname": "smithi063", "labels": [], "status": ""}, {"addr": "172.21.15.162", "hostname": "smithi162", "labels": [], "status": ""}] 2024-08-17T16:48:04.095 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:03 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:04.633 INFO:tasks.cephadm:Setting crush tunables to default 2024-08-17T16:48:04.633 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd crush tunables default 2024-08-17T16:48:04.839 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:48:04.961 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:04 smithi063 ceph-mon[31063]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:05.940 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:05 smithi063 ceph-mon[31063]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:48:05.941 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:05 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:05.941 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:05 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:05.941 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:05 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:05.941 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:05 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:05.941 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:05 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi162", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:48:05.941 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:05 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:05.941 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:05 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:07.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:06 smithi063 ceph-mon[31063]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:48:07.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:06 smithi063 ceph-mon[31063]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:48:07.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:06 smithi063 ceph-mon[31063]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:07.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:06 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:07.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:06 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:07.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:06 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:07.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:06 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3793692558' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-08-17T16:48:07.527 INFO:teuthology.orchestra.run.smithi063.stderr:adjusted tunables profile to default 2024-08-17T16:48:08.130 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:07 smithi063 ceph-mon[31063]: Updating smithi162:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:48:08.131 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:07 smithi063 ceph-mon[31063]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:48:08.131 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:07 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3793692558' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-08-17T16:48:08.131 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:07 smithi063 ceph-mon[31063]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T16:48:08.157 INFO:tasks.cephadm:Adding mon.a on smithi063 2024-08-17T16:48:08.157 INFO:tasks.cephadm:Adding mon.c on smithi063 2024-08-17T16:48:08.157 INFO:tasks.cephadm:Adding mon.b on smithi162 2024-08-17T16:48:08.158 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch apply mon '3;smithi063:172.21.15.63=a;smithi063:[v2:172.21.15.63:3301,v1:172.21.15.63:6790]=c;smithi162:172.21.15.162=b' 2024-08-17T16:48:08.363 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:48:09.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:08 smithi063 ceph-mon[31063]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:09.952 INFO:teuthology.orchestra.run.smithi162.stdout:Scheduled mon update... 2024-08-17T16:48:10.525 DEBUG:teuthology.orchestra.run.smithi063:mon.c> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.c.service 2024-08-17T16:48:10.527 DEBUG:teuthology.orchestra.run.smithi162:mon.b> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.b.service 2024-08-17T16:48:10.529 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-17T16:48:10.530 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph mon dump -f json 2024-08-17T16:48:11.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:10 smithi063 ceph-mon[31063]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:11.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:10 smithi063 ceph-mon[31063]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi063:172.21.15.63=a;smithi063:[v2:172.21.15.63:3301,v1:172.21.15.63:6790]=c;smithi162:172.21.15.162=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:48:11.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:10 smithi063 ceph-mon[31063]: Saving service mon spec with placement smithi063:172.21.15.63=a;smithi063:[v2:172.21.15.63:3301,v1:172.21.15.63:6790]=c;smithi162:172.21.15.162=b;count:3 2024-08-17T16:48:11.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:10 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:11.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:10 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:11.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:10 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:11.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:10 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:11.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:10 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:11.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:10 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:48:11.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:10 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:11.472 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:48:12.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:11 smithi063 ceph-mon[31063]: Deploying daemon mon.b on smithi162 2024-08-17T16:48:13.206 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:48:13.206 INFO:teuthology.orchestra.run.smithi162.stdout:{"epoch":1,"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","modified":"2024-08-17T16:46:51.197040Z","created":"2024-08-17T16:46:51.197040Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:3300","nonce":0},{"type":"v1","addr":"172.21.15.63:6789","nonce":0}]},"addr":"172.21.15.63:6789/0","public_addr":"172.21.15.63:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-08-17T16:48:13.206 INFO:teuthology.orchestra.run.smithi162.stderr:dumped monmap epoch 1 2024-08-17T16:48:13.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:12 smithi063 ceph-mon[31063]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:14.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:13 smithi063 ceph-mon[31063]: from='client.? 172.21.15.162:0/2453930652' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:48:14.992 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-17T16:48:14.993 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph mon dump -f json 2024-08-17T16:48:15.210 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:48:15.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:14 smithi063 ceph-mon[31063]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:15.333 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:15 smithi162 systemd[1]: Starting Ceph mon.b for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:48:16.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:15 smithi063 ceph-mon[31063]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:16.683 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 podman[34575]: 2024-08-17 16:48:16.313458608 +0000 UTC m=+0.346806420 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T16:48:16.683 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 podman[34575]: 2024-08-17 16:48:16.421060423 +0000 UTC m=+0.454408223 container create d4ecbe65a0c8b20989c04ec24bba82fbf5fa2d3353c2f69b0b106933046fbfb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f) 2024-08-17T16:48:16.683 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 podman[34575]: 2024-08-17 16:48:16.606559508 +0000 UTC m=+0.639907413 container init d4ecbe65a0c8b20989c04ec24bba82fbf5fa2d3353c2f69b0b106933046fbfb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0) 2024-08-17T16:48:16.683 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 podman[34575]: 2024-08-17 16:48:16.611199725 +0000 UTC m=+0.644547530 container start d4ecbe65a0c8b20989c04ec24bba82fbf5fa2d3353c2f69b0b106933046fbfb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T16:48:16.683 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T16:48:16.683 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable), process ceph-mon, pid 7 2024-08-17T16:48:16.683 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: pidfile_write: ignore empty --pid-file 2024-08-17T16:48:16.683 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: load: jerasure load: lrc 2024-08-17T16:48:16.684 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: RocksDB version: 7.9.2 2024-08-17T16:48:16.684 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Git sha 0 2024-08-17T16:48:16.684 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Compile date 2024-08-17 15:23:20 2024-08-17T16:48:16.684 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: DB SUMMARY 2024-08-17T16:48:16.684 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: DB Session ID: D92FRRPWAEF9J69MYTDB 2024-08-17T16:48:16.684 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: CURRENT file: CURRENT 2024-08-17T16:48:16.684 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T16:48:16.684 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-08-17T16:48:16.684 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-08-17T16:48:16.684 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-08-17T16:48:16.685 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.error_if_exists: 0 2024-08-17T16:48:16.685 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.create_if_missing: 0 2024-08-17T16:48:16.685 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.paranoid_checks: 1 2024-08-17T16:48:16.685 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T16:48:16.685 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T16:48:16.685 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T16:48:16.685 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.env: 0x559692dcdbc0 2024-08-17T16:48:16.685 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T16:48:16.685 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.info_log: 0x559694e96be0 2024-08-17T16:48:16.685 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T16:48:16.685 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.statistics: (nil) 2024-08-17T16:48:16.685 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.use_fsync: 0 2024-08-17T16:48:16.685 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_log_file_size: 0 2024-08-17T16:48:16.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T16:48:16.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T16:48:16.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T16:48:16.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T16:48:16.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.allow_fallocate: 1 2024-08-17T16:48:16.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T16:48:16.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T16:48:16.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.use_direct_reads: 0 2024-08-17T16:48:16.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T16:48:16.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T16:48:16.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.db_log_dir: 2024-08-17T16:48:16.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.wal_dir: 2024-08-17T16:48:16.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T16:48:16.687 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T16:48:16.687 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T16:48:16.687 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T16:48:16.687 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T16:48:16.687 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T16:48:16.687 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T16:48:16.687 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T16:48:16.687 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.write_buffer_manager: 0x559694e66140 2024-08-17T16:48:16.687 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T16:48:16.687 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T16:48:16.687 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T16:48:16.687 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T16:48:16.687 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T16:48:16.689 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T16:48:16.689 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T16:48:16.689 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T16:48:16.689 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.unordered_write: 0 2024-08-17T16:48:16.689 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T16:48:16.689 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T16:48:16.689 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T16:48:16.689 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T16:48:16.690 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.row_cache: None 2024-08-17T16:48:16.690 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.wal_filter: None 2024-08-17T16:48:16.690 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T16:48:16.690 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T16:48:16.690 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.two_write_queues: 0 2024-08-17T16:48:16.690 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T16:48:16.690 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.wal_compression: 0 2024-08-17T16:48:16.690 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.atomic_flush: 0 2024-08-17T16:48:16.690 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T16:48:16.690 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T16:48:16.690 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T16:48:16.691 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.log_readahead_size: 0 2024-08-17T16:48:16.691 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T16:48:16.691 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T16:48:16.691 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T16:48:16.691 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T16:48:16.691 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T16:48:16.691 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T16:48:16.691 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T16:48:16.691 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_background_jobs: 2 2024-08-17T16:48:16.691 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_background_compactions: -1 2024-08-17T16:48:16.691 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_subcompactions: 1 2024-08-17T16:48:16.691 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T16:48:16.691 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T16:48:16.692 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T16:48:16.692 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T16:48:16.692 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T16:48:16.692 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T16:48:16.692 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T16:48:16.692 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T16:48:16.692 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_open_files: -1 2024-08-17T16:48:16.692 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T16:48:16.692 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T16:48:16.692 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T16:48:16.692 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T16:48:16.692 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_background_flushes: -1 2024-08-17T16:48:16.692 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Compression algorithms supported: 2024-08-17T16:48:16.692 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: kZSTD supported: 0 2024-08-17T16:48:16.692 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: kXpressCompression supported: 0 2024-08-17T16:48:16.693 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: kBZip2Compression supported: 0 2024-08-17T16:48:16.693 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T16:48:16.693 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: kLZ4Compression supported: 1 2024-08-17T16:48:16.693 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: kZlibCompression supported: 1 2024-08-17T16:48:16.693 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T16:48:16.693 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: kSnappyCompression supported: 1 2024-08-17T16:48:16.693 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T16:48:16.693 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T16:48:16.693 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2024-08-17T16:48:16.693 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T16:48:16.693 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T16:48:16.693 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.merge_operator: 2024-08-17T16:48:16.694 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compaction_filter: None 2024-08-17T16:48:16.694 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T16:48:16.694 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T16:48:16.694 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T16:48:16.694 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T16:48:16.694 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559694e96820) 2024-08-17T16:48:16.694 INFO:journalctl@ceph.mon.b.smithi162.stdout: cache_index_and_filter_blocks: 1 2024-08-17T16:48:16.694 INFO:journalctl@ceph.mon.b.smithi162.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T16:48:16.694 INFO:journalctl@ceph.mon.b.smithi162.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T16:48:16.694 INFO:journalctl@ceph.mon.b.smithi162.stdout: pin_top_level_index_and_filter: 1 2024-08-17T16:48:16.694 INFO:journalctl@ceph.mon.b.smithi162.stdout: index_type: 0 2024-08-17T16:48:16.694 INFO:journalctl@ceph.mon.b.smithi162.stdout: data_block_index_type: 0 2024-08-17T16:48:16.695 INFO:journalctl@ceph.mon.b.smithi162.stdout: index_shortening: 1 2024-08-17T16:48:16.695 INFO:journalctl@ceph.mon.b.smithi162.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T16:48:16.695 INFO:journalctl@ceph.mon.b.smithi162.stdout: checksum: 4 2024-08-17T16:48:16.695 INFO:journalctl@ceph.mon.b.smithi162.stdout: no_block_cache: 0 2024-08-17T16:48:16.695 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_cache: 0x559694e8b350 2024-08-17T16:48:16.695 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_cache_name: BinnedLRUCache 2024-08-17T16:48:16.695 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_cache_options: 2024-08-17T16:48:16.695 INFO:journalctl@ceph.mon.b.smithi162.stdout: capacity : 536870912 2024-08-17T16:48:16.695 INFO:journalctl@ceph.mon.b.smithi162.stdout: num_shard_bits : 4 2024-08-17T16:48:16.695 INFO:journalctl@ceph.mon.b.smithi162.stdout: strict_capacity_limit : 0 2024-08-17T16:48:16.695 INFO:journalctl@ceph.mon.b.smithi162.stdout: high_pri_pool_ratio: 0.000 2024-08-17T16:48:16.695 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_cache_compressed: (nil) 2024-08-17T16:48:16.695 INFO:journalctl@ceph.mon.b.smithi162.stdout: persistent_cache: (nil) 2024-08-17T16:48:16.696 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_size: 4096 2024-08-17T16:48:16.696 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_size_deviation: 10 2024-08-17T16:48:16.696 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_restart_interval: 16 2024-08-17T16:48:16.696 INFO:journalctl@ceph.mon.b.smithi162.stdout: index_block_restart_interval: 1 2024-08-17T16:48:16.696 INFO:journalctl@ceph.mon.b.smithi162.stdout: metadata_block_size: 4096 2024-08-17T16:48:16.696 INFO:journalctl@ceph.mon.b.smithi162.stdout: partition_filters: 0 2024-08-17T16:48:16.696 INFO:journalctl@ceph.mon.b.smithi162.stdout: use_delta_encoding: 1 2024-08-17T16:48:16.696 INFO:journalctl@ceph.mon.b.smithi162.stdout: filter_policy: bloomfilter 2024-08-17T16:48:16.696 INFO:journalctl@ceph.mon.b.smithi162.stdout: whole_key_filtering: 1 2024-08-17T16:48:16.696 INFO:journalctl@ceph.mon.b.smithi162.stdout: verify_compression: 0 2024-08-17T16:48:16.696 INFO:journalctl@ceph.mon.b.smithi162.stdout: read_amp_bytes_per_bit: 0 2024-08-17T16:48:16.696 INFO:journalctl@ceph.mon.b.smithi162.stdout: format_version: 5 2024-08-17T16:48:16.696 INFO:journalctl@ceph.mon.b.smithi162.stdout: enable_index_compression: 1 2024-08-17T16:48:16.697 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_align: 0 2024-08-17T16:48:16.697 INFO:journalctl@ceph.mon.b.smithi162.stdout: max_auto_readahead_size: 262144 2024-08-17T16:48:16.697 INFO:journalctl@ceph.mon.b.smithi162.stdout: prepopulate_block_cache: 0 2024-08-17T16:48:16.697 INFO:journalctl@ceph.mon.b.smithi162.stdout: initial_auto_readahead_size: 8192 2024-08-17T16:48:16.697 INFO:journalctl@ceph.mon.b.smithi162.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T16:48:16.697 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T16:48:16.697 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T16:48:16.697 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compression: NoCompression 2024-08-17T16:48:16.697 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T16:48:16.697 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T16:48:16.697 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T16:48:16.697 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.num_levels: 7 2024-08-17T16:48:16.697 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T16:48:16.698 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T16:48:16.698 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T16:48:16.698 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T16:48:16.698 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T16:48:16.698 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T16:48:16.698 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T16:48:16.698 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T16:48:16.698 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T16:48:16.698 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T16:48:16.698 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T16:48:16.698 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T16:48:16.698 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T16:48:16.698 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T16:48:16.699 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T16:48:16.699 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T16:48:16.699 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T16:48:16.699 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T16:48:16.699 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T16:48:16.699 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T16:48:16.699 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T16:48:16.699 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T16:48:16.699 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T16:48:16.699 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T16:48:16.699 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T16:48:16.700 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T16:48:16.700 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T16:48:16.700 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T16:48:16.700 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T16:48:16.700 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T16:48:16.700 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T16:48:16.700 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T16:48:16.700 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T16:48:16.700 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T16:48:16.700 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T16:48:16.700 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T16:48:16.701 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T16:48:16.701 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T16:48:16.701 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T16:48:16.701 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T16:48:16.701 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T16:48:16.701 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T16:48:16.701 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T16:48:16.701 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T16:48:16.701 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T16:48:16.701 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T16:48:16.702 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T16:48:16.702 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T16:48:16.702 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T16:48:16.702 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T16:48:16.702 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T16:48:16.702 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T16:48:16.702 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T16:48:16.702 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.table_properties_collectors: 2024-08-17T16:48:16.702 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.inplace_update_support: 0 2024-08-17T16:48:16.702 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T16:48:16.702 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T16:48:16.702 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T16:48:16.703 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T16:48:16.703 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.bloom_locality: 0 2024-08-17T16:48:16.703 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.max_successive_merges: 0 2024-08-17T16:48:16.703 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T16:48:16.703 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T16:48:16.703 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T16:48:16.703 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T16:48:16.703 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.ttl: 2592000 2024-08-17T16:48:16.703 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T16:48:16.703 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T16:48:16.703 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T16:48:16.703 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.enable_blob_files: false 2024-08-17T16:48:16.703 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.min_blob_size: 0 2024-08-17T16:48:16.704 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T16:48:16.704 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T16:48:16.704 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T16:48:16.704 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T16:48:16.704 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T16:48:16.704 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T16:48:16.705 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T16:48:16.705 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T16:48:16.705 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-17T16:48:16.705 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-08-17T16:48:16.705 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e873a443-d04e-409d-aa61-ba7c22273137 2024-08-17T16:48:16.705 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913296638406, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-08-17T16:48:16.706 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-08-17T16:48:16.706 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913296638889, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1622, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723913296, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e873a443-d04e-409d-aa61-ba7c22273137", "db_session_id": "D92FRRPWAEF9J69MYTDB", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-08-17T16:48:16.706 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913296638983, "job": 1, "event": "recovery_finished"} 2024-08-17T16:48:16.706 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-08-17T16:48:16.706 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T16:48:16.706 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x559694eb4e00 2024-08-17T16:48:16.706 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: DB pointer 0x559694fa8000 2024-08-17T16:48:16.706 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T16:48:16.706 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T16:48:16.706 INFO:journalctl@ceph.mon.b.smithi162.stdout: ** DB Stats ** 2024-08-17T16:48:16.707 INFO:journalctl@ceph.mon.b.smithi162.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:48:16.707 INFO:journalctl@ceph.mon.b.smithi162.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T16:48:16.707 INFO:journalctl@ceph.mon.b.smithi162.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T16:48:16.707 INFO:journalctl@ceph.mon.b.smithi162.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T16:48:16.707 INFO:journalctl@ceph.mon.b.smithi162.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T16:48:16.707 INFO:journalctl@ceph.mon.b.smithi162.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T16:48:16.707 INFO:journalctl@ceph.mon.b.smithi162.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T16:48:16.707 INFO:journalctl@ceph.mon.b.smithi162.stdout: 2024-08-17T16:48:16.707 INFO:journalctl@ceph.mon.b.smithi162.stdout: ** Compaction Stats [default] ** 2024-08-17T16:48:16.707 INFO:journalctl@ceph.mon.b.smithi162.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T16:48:16.707 INFO:journalctl@ceph.mon.b.smithi162.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T16:48:16.707 INFO:journalctl@ceph.mon.b.smithi162.stdout: L0 1/0 1.58 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-17T16:48:16.707 INFO:journalctl@ceph.mon.b.smithi162.stdout: Sum 1/0 1.58 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-17T16:48:16.707 INFO:journalctl@ceph.mon.b.smithi162.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-17T16:48:16.708 INFO:journalctl@ceph.mon.b.smithi162.stdout: 2024-08-17T16:48:16.708 INFO:journalctl@ceph.mon.b.smithi162.stdout: ** Compaction Stats [default] ** 2024-08-17T16:48:16.708 INFO:journalctl@ceph.mon.b.smithi162.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T16:48:16.708 INFO:journalctl@ceph.mon.b.smithi162.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T16:48:16.708 INFO:journalctl@ceph.mon.b.smithi162.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.3 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-17T16:48:16.708 INFO:journalctl@ceph.mon.b.smithi162.stdout: 2024-08-17T16:48:16.708 INFO:journalctl@ceph.mon.b.smithi162.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T16:48:16.708 INFO:journalctl@ceph.mon.b.smithi162.stdout: 2024-08-17T16:48:16.708 INFO:journalctl@ceph.mon.b.smithi162.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:48:16.708 INFO:journalctl@ceph.mon.b.smithi162.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T16:48:16.708 INFO:journalctl@ceph.mon.b.smithi162.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T16:48:16.708 INFO:journalctl@ceph.mon.b.smithi162.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T16:48:16.708 INFO:journalctl@ceph.mon.b.smithi162.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T16:48:16.709 INFO:journalctl@ceph.mon.b.smithi162.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T16:48:16.709 INFO:journalctl@ceph.mon.b.smithi162.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:48:16.709 INFO:journalctl@ceph.mon.b.smithi162.stdout: Interval compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:48:16.709 INFO:journalctl@ceph.mon.b.smithi162.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T16:48:16.709 INFO:journalctl@ceph.mon.b.smithi162.stdout: Block cache BinnedLRUCache@0x559694e8b350#7 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-08-17T16:48:16.709 INFO:journalctl@ceph.mon.b.smithi162.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-08-17T16:48:16.709 INFO:journalctl@ceph.mon.b.smithi162.stdout: 2024-08-17T16:48:16.709 INFO:journalctl@ceph.mon.b.smithi162.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T16:48:16.709 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-08-17T16:48:16.709 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: using public_addr v2:172.21.15.162:0/0 -> [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] 2024-08-17T16:48:16.709 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: starting mon.b rank -1 at public addrs [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] at bind addrs [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:48:16.710 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: mon.b@-1(???) e0 preinit fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:48:16.710 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: mon.b@-1(synchronizing).mds e1 new map 2024-08-17T16:48:16.710 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: mon.b@-1(synchronizing).mds e1 print_map 2024-08-17T16:48:16.710 INFO:journalctl@ceph.mon.b.smithi162.stdout: e1 2024-08-17T16:48:16.710 INFO:journalctl@ceph.mon.b.smithi162.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T16:48:16.710 INFO:journalctl@ceph.mon.b.smithi162.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T16:48:16.710 INFO:journalctl@ceph.mon.b.smithi162.stdout: legacy client fscid: -1 2024-08-17T16:48:16.710 INFO:journalctl@ceph.mon.b.smithi162.stdout: 2024-08-17T16:48:16.711 INFO:journalctl@ceph.mon.b.smithi162.stdout: No filesystems configured 2024-08-17T16:48:16.711 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-08-17T16:48:16.711 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-08-17T16:48:16.711 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-08-17T16:48:16.711 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-08-17T16:48:16.711 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-08-17T16:48:16.711 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-08-17T16:48:16.711 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-08-17T16:48:16.711 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:48:16.711 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:48:16.711 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:48:16.712 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:16.712 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi063:172.21.15.63=a;smithi063:[v2:172.21.15.63:3301,v1:172.21.15.63:6790]=c;smithi162:172.21.15.162=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:48:16.712 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: Saving service mon spec with placement smithi063:172.21.15.63=a;smithi063:[v2:172.21.15.63:3301,v1:172.21.15.63:6790]=c;smithi162:172.21.15.162=b;count:3 2024-08-17T16:48:16.712 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:16.712 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:16.712 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:16.712 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:16.712 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:16.713 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:48:16.713 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:16.713 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: Deploying daemon mon.b on smithi162 2024-08-17T16:48:16.713 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:16.713 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: from='client.? 172.21.15.162:0/2453930652' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:48:16.713 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:16.713 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:16.713 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 ceph-mon[34633]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-08-17T16:48:16.713 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 bash[34575]: d4ecbe65a0c8b20989c04ec24bba82fbf5fa2d3353c2f69b0b106933046fbfb8 2024-08-17T16:48:16.713 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:16 smithi162 systemd[1]: Started Ceph mon.b for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:48:22.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:48:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: mon.a calling monitor election 2024-08-17T16:48:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: mon.b calling monitor election 2024-08-17T16:48:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-17T16:48:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: monmap e2: 2 mons at {a=[v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0],b=[v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:48:22.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: fsmap 2024-08-17T16:48:22.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T16:48:22.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: mgrmap e13: y(active, since 41s) 2024-08-17T16:48:22.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: overall HEALTH_OK 2024-08-17T16:48:22.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:22.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:22.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:22.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:48:22.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:21 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:48:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: mon.a calling monitor election 2024-08-17T16:48:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:22.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: mon.b calling monitor election 2024-08-17T16:48:22.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:22.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:22.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:22.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:22.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-17T16:48:22.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: monmap e2: 2 mons at {a=[v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0],b=[v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:48:22.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: fsmap 2024-08-17T16:48:22.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T16:48:22.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: mgrmap e13: y(active, since 41s) 2024-08-17T16:48:22.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: overall HEALTH_OK 2024-08-17T16:48:22.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:22.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:22.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:22.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:48:22.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:21 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:23.018 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:22 smithi063 ceph-mon[31063]: Deploying daemon mon.c on smithi063 2024-08-17T16:48:23.019 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:22 smithi063 ceph-mon[31063]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:23.019 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:22 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:23.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:22 smithi162 ceph-mon[34633]: Deploying daemon mon.c on smithi063 2024-08-17T16:48:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:22 smithi162 ceph-mon[34633]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:22 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:23.368 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:48:23.369 INFO:teuthology.orchestra.run.smithi162.stdout:{"epoch":2,"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","modified":"2024-08-17T16:48:16.662687Z","created":"2024-08-17T16:46:51.197040Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:3300","nonce":0},{"type":"v1","addr":"172.21.15.63:6789","nonce":0}]},"addr":"172.21.15.63:6789/0","public_addr":"172.21.15.63:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:3300","nonce":0},{"type":"v1","addr":"172.21.15.162:6789","nonce":0}]},"addr":"172.21.15.162:6789/0","public_addr":"172.21.15.162:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-08-17T16:48:23.369 INFO:teuthology.orchestra.run.smithi162.stderr:dumped monmap epoch 2 2024-08-17T16:48:23.950 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:23 smithi063 ceph-mon[31063]: from='client.? 172.21.15.162:0/1058476817' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:48:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:23 smithi162 ceph-mon[34633]: from='client.? 172.21.15.162:0/1058476817' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:48:24.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 systemd[1]: Starting Ceph mon.c for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:48:24.863 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[31063]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:24.863 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 podman[42643]: 2024-08-17 16:48:24.60123069 +0000 UTC m=+0.110251168 container create b3f2c9339c6b2232a39c4cb019d7ffdc58d13951213a6b62caed65f320581d3c (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, ceph=True, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-08-17T16:48:24.864 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 podman[42643]: 2024-08-17 16:48:24.508650596 +0000 UTC m=+0.017671075 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T16:48:24.864 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 podman[42643]: 2024-08-17 16:48:24.837001957 +0000 UTC m=+0.346022446 container init b3f2c9339c6b2232a39c4cb019d7ffdc58d13951213a6b62caed65f320581d3c (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-08-17T16:48:24.864 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 podman[42643]: 2024-08-17 16:48:24.841779236 +0000 UTC m=+0.350799713 container start b3f2c9339c6b2232a39c4cb019d7ffdc58d13951213a6b62caed65f320581d3c (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-08-17T16:48:24.864 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T16:48:24.864 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable), process ceph-mon, pid 7 2024-08-17T16:48:24.909 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-08-17T16:48:24.910 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph mon dump -f json 2024-08-17T16:48:25.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:24 smithi162 ceph-mon[34633]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:25.122 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:48:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: pidfile_write: ignore empty --pid-file 2024-08-17T16:48:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: load: jerasure load: lrc 2024-08-17T16:48:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: RocksDB version: 7.9.2 2024-08-17T16:48:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Git sha 0 2024-08-17T16:48:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Compile date 2024-08-17 15:23:20 2024-08-17T16:48:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: DB SUMMARY 2024-08-17T16:48:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: DB Session ID: N4ELQ4EEL88R6XTCXJRM 2024-08-17T16:48:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: CURRENT file: CURRENT 2024-08-17T16:48:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T16:48:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-08-17T16:48:25.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-08-17T16:48:25.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-08-17T16:48:25.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.error_if_exists: 0 2024-08-17T16:48:25.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.create_if_missing: 0 2024-08-17T16:48:25.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.paranoid_checks: 1 2024-08-17T16:48:25.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T16:48:25.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T16:48:25.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T16:48:25.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.env: 0x56115dd20bc0 2024-08-17T16:48:25.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T16:48:25.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.info_log: 0x56115e7d6be0 2024-08-17T16:48:25.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T16:48:25.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.statistics: (nil) 2024-08-17T16:48:25.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.use_fsync: 0 2024-08-17T16:48:25.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_log_file_size: 0 2024-08-17T16:48:25.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T16:48:25.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T16:48:25.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T16:48:25.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T16:48:25.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.allow_fallocate: 1 2024-08-17T16:48:25.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T16:48:25.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T16:48:25.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.use_direct_reads: 0 2024-08-17T16:48:25.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T16:48:25.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T16:48:25.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.db_log_dir: 2024-08-17T16:48:25.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.wal_dir: 2024-08-17T16:48:25.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T16:48:25.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T16:48:25.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T16:48:25.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T16:48:25.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T16:48:25.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T16:48:25.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T16:48:25.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T16:48:25.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.write_buffer_manager: 0x56115e7a6140 2024-08-17T16:48:25.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T16:48:25.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T16:48:25.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T16:48:25.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T16:48:25.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T16:48:25.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T16:48:25.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T16:48:25.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T16:48:25.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.unordered_write: 0 2024-08-17T16:48:25.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T16:48:25.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T16:48:25.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T16:48:25.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T16:48:25.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.row_cache: None 2024-08-17T16:48:25.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.wal_filter: None 2024-08-17T16:48:25.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T16:48:25.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T16:48:25.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.two_write_queues: 0 2024-08-17T16:48:25.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T16:48:25.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.wal_compression: 0 2024-08-17T16:48:25.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.atomic_flush: 0 2024-08-17T16:48:25.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T16:48:25.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T16:48:25.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T16:48:25.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.log_readahead_size: 0 2024-08-17T16:48:25.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T16:48:25.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T16:48:25.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T16:48:25.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T16:48:25.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T16:48:25.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T16:48:25.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T16:48:25.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_background_jobs: 2 2024-08-17T16:48:25.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_background_compactions: -1 2024-08-17T16:48:25.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_subcompactions: 1 2024-08-17T16:48:25.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T16:48:25.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T16:48:25.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T16:48:25.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T16:48:25.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T16:48:25.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T16:48:25.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T16:48:25.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T16:48:25.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_open_files: -1 2024-08-17T16:48:25.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T16:48:25.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T16:48:25.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T16:48:25.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T16:48:25.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_background_flushes: -1 2024-08-17T16:48:25.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Compression algorithms supported: 2024-08-17T16:48:25.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: kZSTD supported: 0 2024-08-17T16:48:25.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: kXpressCompression supported: 0 2024-08-17T16:48:25.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: kBZip2Compression supported: 0 2024-08-17T16:48:25.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T16:48:25.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: kLZ4Compression supported: 1 2024-08-17T16:48:25.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: kZlibCompression supported: 1 2024-08-17T16:48:25.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T16:48:25.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: kSnappyCompression supported: 1 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.merge_operator: 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compaction_filter: None 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56115e7d6820) 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout: cache_index_and_filter_blocks: 1 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T16:48:25.289 INFO:journalctl@ceph.mon.c.smithi063.stdout: pin_top_level_index_and_filter: 1 2024-08-17T16:48:25.290 INFO:journalctl@ceph.mon.c.smithi063.stdout: index_type: 0 2024-08-17T16:48:25.290 INFO:journalctl@ceph.mon.c.smithi063.stdout: data_block_index_type: 0 2024-08-17T16:48:25.290 INFO:journalctl@ceph.mon.c.smithi063.stdout: index_shortening: 1 2024-08-17T16:48:25.290 INFO:journalctl@ceph.mon.c.smithi063.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T16:48:25.290 INFO:journalctl@ceph.mon.c.smithi063.stdout: checksum: 4 2024-08-17T16:48:25.290 INFO:journalctl@ceph.mon.c.smithi063.stdout: no_block_cache: 0 2024-08-17T16:48:25.290 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_cache: 0x56115e7cb350 2024-08-17T16:48:25.290 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_cache_name: BinnedLRUCache 2024-08-17T16:48:25.290 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_cache_options: 2024-08-17T16:48:25.290 INFO:journalctl@ceph.mon.c.smithi063.stdout: capacity : 536870912 2024-08-17T16:48:25.290 INFO:journalctl@ceph.mon.c.smithi063.stdout: num_shard_bits : 4 2024-08-17T16:48:25.290 INFO:journalctl@ceph.mon.c.smithi063.stdout: strict_capacity_limit : 0 2024-08-17T16:48:25.290 INFO:journalctl@ceph.mon.c.smithi063.stdout: high_pri_pool_ratio: 0.000 2024-08-17T16:48:25.290 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_cache_compressed: (nil) 2024-08-17T16:48:25.291 INFO:journalctl@ceph.mon.c.smithi063.stdout: persistent_cache: (nil) 2024-08-17T16:48:25.291 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_size: 4096 2024-08-17T16:48:25.291 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_size_deviation: 10 2024-08-17T16:48:25.291 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_restart_interval: 16 2024-08-17T16:48:25.291 INFO:journalctl@ceph.mon.c.smithi063.stdout: index_block_restart_interval: 1 2024-08-17T16:48:25.291 INFO:journalctl@ceph.mon.c.smithi063.stdout: metadata_block_size: 4096 2024-08-17T16:48:25.291 INFO:journalctl@ceph.mon.c.smithi063.stdout: partition_filters: 0 2024-08-17T16:48:25.291 INFO:journalctl@ceph.mon.c.smithi063.stdout: use_delta_encoding: 1 2024-08-17T16:48:25.291 INFO:journalctl@ceph.mon.c.smithi063.stdout: filter_policy: bloomfilter 2024-08-17T16:48:25.292 INFO:journalctl@ceph.mon.c.smithi063.stdout: whole_key_filtering: 1 2024-08-17T16:48:25.292 INFO:journalctl@ceph.mon.c.smithi063.stdout: verify_compression: 0 2024-08-17T16:48:25.292 INFO:journalctl@ceph.mon.c.smithi063.stdout: read_amp_bytes_per_bit: 0 2024-08-17T16:48:25.292 INFO:journalctl@ceph.mon.c.smithi063.stdout: format_version: 5 2024-08-17T16:48:25.292 INFO:journalctl@ceph.mon.c.smithi063.stdout: enable_index_compression: 1 2024-08-17T16:48:25.292 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_align: 0 2024-08-17T16:48:25.292 INFO:journalctl@ceph.mon.c.smithi063.stdout: max_auto_readahead_size: 262144 2024-08-17T16:48:25.292 INFO:journalctl@ceph.mon.c.smithi063.stdout: prepopulate_block_cache: 0 2024-08-17T16:48:25.292 INFO:journalctl@ceph.mon.c.smithi063.stdout: initial_auto_readahead_size: 8192 2024-08-17T16:48:25.292 INFO:journalctl@ceph.mon.c.smithi063.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T16:48:25.292 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T16:48:25.292 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T16:48:25.292 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compression: NoCompression 2024-08-17T16:48:25.292 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T16:48:25.292 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T16:48:25.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T16:48:25.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.num_levels: 7 2024-08-17T16:48:25.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T16:48:25.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T16:48:25.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T16:48:25.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T16:48:25.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T16:48:25.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T16:48:25.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T16:48:25.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T16:48:25.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T16:48:25.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T16:48:25.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T16:48:25.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T16:48:25.294 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T16:48:25.294 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T16:48:25.294 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T16:48:25.294 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T16:48:25.294 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T16:48:25.294 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T16:48:25.294 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T16:48:25.294 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T16:48:25.294 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T16:48:25.294 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T16:48:25.294 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T16:48:25.295 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T16:48:25.295 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T16:48:25.295 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T16:48:25.295 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T16:48:25.295 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T16:48:25.295 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T16:48:25.295 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T16:48:25.295 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T16:48:25.295 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T16:48:25.295 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T16:48:25.295 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T16:48:25.295 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T16:48:25.295 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T16:48:25.295 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T16:48:25.296 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T16:48:25.296 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T16:48:25.296 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T16:48:25.296 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T16:48:25.296 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T16:48:25.296 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T16:48:25.296 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T16:48:25.296 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T16:48:25.296 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T16:48:25.296 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T16:48:25.296 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T16:48:25.296 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T16:48:25.296 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T16:48:25.296 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T16:48:25.296 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T16:48:25.297 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T16:48:25.297 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.table_properties_collectors: 2024-08-17T16:48:25.297 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.inplace_update_support: 0 2024-08-17T16:48:25.297 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T16:48:25.297 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T16:48:25.297 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T16:48:25.297 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T16:48:25.297 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.bloom_locality: 0 2024-08-17T16:48:25.297 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.max_successive_merges: 0 2024-08-17T16:48:25.297 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T16:48:25.297 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T16:48:25.297 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T16:48:25.297 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T16:48:25.297 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.ttl: 2592000 2024-08-17T16:48:25.297 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T16:48:25.298 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T16:48:25.298 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T16:48:25.298 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.enable_blob_files: false 2024-08-17T16:48:25.298 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.min_blob_size: 0 2024-08-17T16:48:25.298 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T16:48:25.298 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T16:48:25.298 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T16:48:25.298 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T16:48:25.298 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T16:48:25.298 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T16:48:25.298 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T16:48:25.298 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T16:48:25.298 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-08-17T16:48:25.299 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-08-17T16:48:25.299 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 71e19b9a-502e-47b0-bdec-39d9641c622f 2024-08-17T16:48:25.299 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913304868846, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-08-17T16:48:25.299 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-08-17T16:48:25.299 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913304869245, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1747, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723913304, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "71e19b9a-502e-47b0-bdec-39d9641c622f", "db_session_id": "N4ELQ4EEL88R6XTCXJRM", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-08-17T16:48:25.299 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913304869341, "job": 1, "event": "recovery_finished"} 2024-08-17T16:48:25.299 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-08-17T16:48:25.299 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T16:48:25.299 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56115e7f4e00 2024-08-17T16:48:25.299 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: DB pointer 0x56115e8e8000 2024-08-17T16:48:25.299 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T16:48:25.299 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T16:48:25.299 INFO:journalctl@ceph.mon.c.smithi063.stdout: ** DB Stats ** 2024-08-17T16:48:25.299 INFO:journalctl@ceph.mon.c.smithi063.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:48:25.300 INFO:journalctl@ceph.mon.c.smithi063.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T16:48:25.300 INFO:journalctl@ceph.mon.c.smithi063.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T16:48:25.300 INFO:journalctl@ceph.mon.c.smithi063.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T16:48:25.300 INFO:journalctl@ceph.mon.c.smithi063.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T16:48:25.300 INFO:journalctl@ceph.mon.c.smithi063.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T16:48:25.300 INFO:journalctl@ceph.mon.c.smithi063.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T16:48:25.300 INFO:journalctl@ceph.mon.c.smithi063.stdout: 2024-08-17T16:48:25.300 INFO:journalctl@ceph.mon.c.smithi063.stdout: ** Compaction Stats [default] ** 2024-08-17T16:48:25.300 INFO:journalctl@ceph.mon.c.smithi063.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T16:48:25.300 INFO:journalctl@ceph.mon.c.smithi063.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T16:48:25.300 INFO:journalctl@ceph.mon.c.smithi063.stdout: L0 1/0 1.71 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.4 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-17T16:48:25.300 INFO:journalctl@ceph.mon.c.smithi063.stdout: Sum 1/0 1.71 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.4 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-17T16:48:25.300 INFO:journalctl@ceph.mon.c.smithi063.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.4 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-17T16:48:25.300 INFO:journalctl@ceph.mon.c.smithi063.stdout: 2024-08-17T16:48:25.301 INFO:journalctl@ceph.mon.c.smithi063.stdout: ** Compaction Stats [default] ** 2024-08-17T16:48:25.301 INFO:journalctl@ceph.mon.c.smithi063.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T16:48:25.301 INFO:journalctl@ceph.mon.c.smithi063.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T16:48:25.301 INFO:journalctl@ceph.mon.c.smithi063.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.4 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-08-17T16:48:25.301 INFO:journalctl@ceph.mon.c.smithi063.stdout: 2024-08-17T16:48:25.301 INFO:journalctl@ceph.mon.c.smithi063.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T16:48:25.301 INFO:journalctl@ceph.mon.c.smithi063.stdout: 2024-08-17T16:48:25.301 INFO:journalctl@ceph.mon.c.smithi063.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:48:25.301 INFO:journalctl@ceph.mon.c.smithi063.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T16:48:25.301 INFO:journalctl@ceph.mon.c.smithi063.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T16:48:25.301 INFO:journalctl@ceph.mon.c.smithi063.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T16:48:25.301 INFO:journalctl@ceph.mon.c.smithi063.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T16:48:25.301 INFO:journalctl@ceph.mon.c.smithi063.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T16:48:25.301 INFO:journalctl@ceph.mon.c.smithi063.stdout: Cumulative compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:48:25.301 INFO:journalctl@ceph.mon.c.smithi063.stdout: Interval compaction: 0.00 GB write, 0.64 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:48:25.302 INFO:journalctl@ceph.mon.c.smithi063.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T16:48:25.302 INFO:journalctl@ceph.mon.c.smithi063.stdout: Block cache BinnedLRUCache@0x56115e7cb350#7 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-08-17T16:48:25.302 INFO:journalctl@ceph.mon.c.smithi063.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-08-17T16:48:25.302 INFO:journalctl@ceph.mon.c.smithi063.stdout: 2024-08-17T16:48:25.302 INFO:journalctl@ceph.mon.c.smithi063.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T16:48:25.302 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-08-17T16:48:25.302 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: using public_addrv [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] 2024-08-17T16:48:25.302 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: starting mon.c rank -1 at public addrs [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] at bind addrs [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:48:25.302 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.c@-1(???) e0 preinit fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:48:25.302 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.c@-1(synchronizing).mds e1 new map 2024-08-17T16:48:25.302 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.c@-1(synchronizing).mds e1 print_map 2024-08-17T16:48:25.302 INFO:journalctl@ceph.mon.c.smithi063.stdout: e1 2024-08-17T16:48:25.302 INFO:journalctl@ceph.mon.c.smithi063.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T16:48:25.303 INFO:journalctl@ceph.mon.c.smithi063.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T16:48:25.303 INFO:journalctl@ceph.mon.c.smithi063.stdout: legacy client fscid: -1 2024-08-17T16:48:25.303 INFO:journalctl@ceph.mon.c.smithi063.stdout: 2024-08-17T16:48:25.303 INFO:journalctl@ceph.mon.c.smithi063.stdout: No filesystems configured 2024-08-17T16:48:25.303 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-08-17T16:48:25.303 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-08-17T16:48:25.303 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-08-17T16:48:25.303 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-08-17T16:48:25.303 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-08-17T16:48:25.303 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-08-17T16:48:25.303 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-08-17T16:48:25.303 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:48:25.303 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:48:25.303 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-08-17T16:48:25.304 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:25.304 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi063:172.21.15.63=a;smithi063:[v2:172.21.15.63:3301,v1:172.21.15.63:6790]=c;smithi162:172.21.15.162=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:48:25.304 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: Saving service mon spec with placement smithi063:172.21.15.63=a;smithi063:[v2:172.21.15.63:3301,v1:172.21.15.63:6790]=c;smithi162:172.21.15.162=b;count:3 2024-08-17T16:48:25.304 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:25.304 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:25.304 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:25.304 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:25.304 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:25.304 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:48:25.304 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:25.304 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: Deploying daemon mon.b on smithi162 2024-08-17T16:48:25.304 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:25.304 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='client.? 172.21.15.162:0/2453930652' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:48:25.304 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:25.305 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:25.305 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:48:25.305 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:25.305 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.a calling monitor election 2024-08-17T16:48:25.305 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:25.305 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:25.305 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:25.305 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.b calling monitor election 2024-08-17T16:48:25.305 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:25.305 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:25.305 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:25.305 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:25.305 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-17T16:48:25.305 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: monmap e2: 2 mons at {a=[v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0],b=[v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:48:25.305 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: fsmap 2024-08-17T16:48:25.306 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T16:48:25.306 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mgrmap e13: y(active, since 41s) 2024-08-17T16:48:25.306 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: overall HEALTH_OK 2024-08-17T16:48:25.306 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:25.306 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:25.306 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:25.306 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:48:25.306 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:25.306 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: Deploying daemon mon.c on smithi063 2024-08-17T16:48:25.306 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:25.306 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:25.306 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: from='client.? 172.21.15.162:0/1058476817' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:48:25.306 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:25.306 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 ceph-mon[42680]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-08-17T16:48:25.307 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 bash[42643]: b3f2c9339c6b2232a39c4cb019d7ffdc58d13951213a6b62caed65f320581d3c 2024-08-17T16:48:25.307 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:24 smithi063 systemd[1]: Started Ceph mon.c for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:48:30.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:48:30.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:30.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: mon.a calling monitor election 2024-08-17T16:48:30.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: mon.b calling monitor election 2024-08-17T16:48:30.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:30.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: mon.c calling monitor election 2024-08-17T16:48:30.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:30.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T16:48:30.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: monmap e3: 3 mons at {a=[v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0],b=[v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0],c=[v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:48:30.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: fsmap 2024-08-17T16:48:30.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T16:48:30.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: mgrmap e13: y(active, since 49s) 2024-08-17T16:48:30.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: overall HEALTH_OK 2024-08-17T16:48:30.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:30.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:30.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:30.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:30.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:48:30.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:30.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: mon.a calling monitor election 2024-08-17T16:48:30.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: mon.b calling monitor election 2024-08-17T16:48:30.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:30.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: mon.c calling monitor election 2024-08-17T16:48:30.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:30.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T16:48:30.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: monmap e3: 3 mons at {a=[v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0],b=[v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0],c=[v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:48:30.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: fsmap 2024-08-17T16:48:30.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T16:48:30.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: mgrmap e13: y(active, since 49s) 2024-08-17T16:48:30.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: overall HEALTH_OK 2024-08-17T16:48:30.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:30.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:30.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:30.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:48:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:48:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: mon.a calling monitor election 2024-08-17T16:48:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: mon.b calling monitor election 2024-08-17T16:48:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: mon.c calling monitor election 2024-08-17T16:48:30.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:30.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:30.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T16:48:30.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: monmap e3: 3 mons at {a=[v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0],b=[v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0],c=[v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-08-17T16:48:30.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: fsmap 2024-08-17T16:48:30.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: osdmap e4: 0 total, 0 up, 0 in 2024-08-17T16:48:30.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: mgrmap e13: y(active, since 49s) 2024-08-17T16:48:30.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: overall HEALTH_OK 2024-08-17T16:48:30.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:30.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:30.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:30.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:30.378 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:48:30.379 INFO:teuthology.orchestra.run.smithi162.stdout:{"epoch":3,"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","modified":"2024-08-17T16:48:24.893441Z","created":"2024-08-17T16:46:51.197040Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:3300","nonce":0},{"type":"v1","addr":"172.21.15.63:6789","nonce":0}]},"addr":"172.21.15.63:6789/0","public_addr":"172.21.15.63:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:3300","nonce":0},{"type":"v1","addr":"172.21.15.162:6789","nonce":0}]},"addr":"172.21.15.162:6789/0","public_addr":"172.21.15.162:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:3301","nonce":0},{"type":"v1","addr":"172.21.15.63:6790","nonce":0}]},"addr":"172.21.15.63:6790/0","public_addr":"172.21.15.63:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-08-17T16:48:30.379 INFO:teuthology.orchestra.run.smithi162.stderr:dumped monmap epoch 3 2024-08-17T16:48:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:30 smithi162 ceph-mon[34633]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:30 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:30 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:30 smithi162 ceph-mon[34633]: from='client.? 172.21.15.162:0/4187560993' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:48:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:30 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:31.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:30 smithi063 ceph-mon[31063]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:31.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:30 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:31.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:30 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:31.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:30 smithi063 ceph-mon[31063]: from='client.? 172.21.15.162:0/4187560993' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:48:31.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:30 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:31.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:30 smithi063 ceph-mon[42680]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:31.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:30 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:31.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:30 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:31.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:30 smithi063 ceph-mon[42680]: from='client.? 172.21.15.162:0/4187560993' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-08-17T16:48:31.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:30 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:48:31.576 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-08-17T16:48:31.576 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph config generate-minimal-conf 2024-08-17T16:48:31.794 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:48:32.186 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.186 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.186 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.186 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.186 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:32.186 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:32.186 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:48:32.186 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:48:32.186 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:48:32.186 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:48:32.186 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.187 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.187 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.187 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.187 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.187 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.187 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.187 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.187 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.187 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: Reconfiguring mon.a (unknown last config time)... 2024-08-17T16:48:32.187 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:48:32.187 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:48:32.187 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:32.188 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: Reconfiguring daemon mon.a on smithi063 2024-08-17T16:48:32.188 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[42680]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:32.188 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.188 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.188 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.188 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.188 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:32.188 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:32.188 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:48:32.188 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:48:32.188 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:48:32.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:48:32.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: Reconfiguring mon.a (unknown last config time)... 2024-08-17T16:48:32.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:48:32.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:48:32.190 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:32.190 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: Reconfiguring daemon mon.a on smithi063 2024-08-17T16:48:32.190 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:32 smithi063 ceph-mon[31063]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:48:32.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:48:32.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:48:32.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:48:32.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:32.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: Reconfiguring mon.a (unknown last config time)... 2024-08-17T16:48:32.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:48:32.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:48:32.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:32.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: Reconfiguring daemon mon.a on smithi063 2024-08-17T16:48:32.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:32 smithi162 ceph-mon[34633]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:34.178 INFO:teuthology.orchestra.run.smithi063.stdout:# minimal ceph.conf for 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:48:34.179 INFO:teuthology.orchestra.run.smithi063.stdout:[global] 2024-08-17T16:48:34.179 INFO:teuthology.orchestra.run.smithi063.stdout: fsid = 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:48:34.179 INFO:teuthology.orchestra.run.smithi063.stdout: mon_host = [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] 2024-08-17T16:48:34.179 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:34 smithi063 ceph-mon[31063]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:34.179 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:34 smithi063 ceph-mon[42680]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:34 smithi162 ceph-mon[34633]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:35.263 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3092684673' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:35.263 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:35.263 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:35.263 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[31063]: Reconfiguring mon.c (monmap changed)... 2024-08-17T16:48:35.263 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:48:35.263 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:48:35.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:35.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[31063]: Reconfiguring daemon mon.c on smithi063 2024-08-17T16:48:35.264 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3092684673' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:35.264 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:35.264 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:35.264 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[42680]: Reconfiguring mon.c (monmap changed)... 2024-08-17T16:48:35.264 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:48:35.264 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:48:35.264 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:35.264 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:35 smithi063 ceph-mon[42680]: Reconfiguring daemon mon.c on smithi063 2024-08-17T16:48:35.282 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-08-17T16:48:35.282 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T16:48:35.283 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd of=/etc/ceph/ceph.conf 2024-08-17T16:48:35.311 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T16:48:35.312 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:48:35.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:35 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3092684673' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:35 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:35 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:35 smithi162 ceph-mon[34633]: Reconfiguring mon.c (monmap changed)... 2024-08-17T16:48:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:35 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:48:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:35 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:48:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:35 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:35 smithi162 ceph-mon[34633]: Reconfiguring daemon mon.c on smithi063 2024-08-17T16:48:35.378 DEBUG:teuthology.orchestra.run.smithi162:> set -ex 2024-08-17T16:48:35.378 DEBUG:teuthology.orchestra.run.smithi162:> sudo dd of=/etc/ceph/ceph.conf 2024-08-17T16:48:35.411 DEBUG:teuthology.orchestra.run.smithi162:> set -ex 2024-08-17T16:48:35.411 DEBUG:teuthology.orchestra.run.smithi162:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:48:35.481 INFO:tasks.cephadm:Adding mgr.y on smithi063 2024-08-17T16:48:35.481 INFO:tasks.cephadm:Adding mgr.x on smithi162 2024-08-17T16:48:35.481 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch apply mgr '2;smithi063=y;smithi162=x' 2024-08-17T16:48:35.723 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:48:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:36 smithi162 ceph-mon[34633]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:36.114 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:36 smithi063 ceph-mon[31063]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:36.114 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:36 smithi063 ceph-mon[42680]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:37.313 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:37 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:37.313 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:37 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:37.314 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:37 smithi162 ceph-mon[34633]: Reconfiguring mon.b (monmap changed)... 2024-08-17T16:48:37.314 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:37 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:48:37.314 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:37 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:48:37.314 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:37 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:37.314 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:37 smithi162 ceph-mon[34633]: Reconfiguring daemon mon.b on smithi162 2024-08-17T16:48:37.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:37 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:37.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:37 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:37.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:37 smithi063 ceph-mon[31063]: Reconfiguring mon.b (monmap changed)... 2024-08-17T16:48:37.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:37 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:48:37.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:37 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:48:37.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:37 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:37.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:37 smithi063 ceph-mon[31063]: Reconfiguring daemon mon.b on smithi162 2024-08-17T16:48:37.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:37 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:37.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:37 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:37.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:37 smithi063 ceph-mon[42680]: Reconfiguring mon.b (monmap changed)... 2024-08-17T16:48:37.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:37 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:48:37.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:37 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:48:37.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:37 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:37.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:37 smithi063 ceph-mon[42680]: Reconfiguring daemon mon.b on smithi162 2024-08-17T16:48:37.585 INFO:teuthology.orchestra.run.smithi162.stdout:Scheduled mgr update... 2024-08-17T16:48:38.361 DEBUG:teuthology.orchestra.run.smithi162:mgr.x> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.x.service 2024-08-17T16:48:38.363 DEBUG:tasks.cephadm:set 0 configs 2024-08-17T16:48:38.363 INFO:tasks.cephadm:Deploying OSDs... 2024-08-17T16:48:38.364 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T16:48:38.364 DEBUG:teuthology.orchestra.run.smithi063:> dd if=/scratch_devs of=/dev/stdout 2024-08-17T16:48:38.384 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-17T16:48:38.384 DEBUG:teuthology.orchestra.run.smithi063:> stat /dev/vg_nvme/lv_1 2024-08-17T16:48:38.443 INFO:teuthology.orchestra.run.smithi063.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-17T16:48:38.443 INFO:teuthology.orchestra.run.smithi063.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:48:38.443 INFO:teuthology.orchestra.run.smithi063.stdout:Device: 5h/5d Inode: 979 Links: 1 2024-08-17T16:48:38.443 INFO:teuthology.orchestra.run.smithi063.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:48:38.443 INFO:teuthology.orchestra.run.smithi063.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:48:38.444 INFO:teuthology.orchestra.run.smithi063.stdout:Access: 2024-08-17 16:48:24.321279674 +0000 2024-08-17T16:48:38.444 INFO:teuthology.orchestra.run.smithi063.stdout:Modify: 2024-08-17 16:43:58.314324121 +0000 2024-08-17T16:48:38.444 INFO:teuthology.orchestra.run.smithi063.stdout:Change: 2024-08-17 16:43:58.314324121 +0000 2024-08-17T16:48:38.444 INFO:teuthology.orchestra.run.smithi063.stdout: Birth: 2024-08-17 16:43:58.314324121 +0000 2024-08-17T16:48:38.444 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-17T16:48:38.514 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records in 2024-08-17T16:48:38.514 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records out 2024-08-17T16:48:38.514 INFO:teuthology.orchestra.run.smithi063.stderr:512 bytes copied, 0.000277607 s, 1.8 MB/s 2024-08-17T16:48:38.515 DEBUG:teuthology.orchestra.run.smithi063:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-17T16:48:38.576 DEBUG:teuthology.orchestra.run.smithi063:> stat /dev/vg_nvme/lv_2 2024-08-17T16:48:38.634 INFO:teuthology.orchestra.run.smithi063.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-17T16:48:38.634 INFO:teuthology.orchestra.run.smithi063.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:48:38.634 INFO:teuthology.orchestra.run.smithi063.stdout:Device: 5h/5d Inode: 989 Links: 1 2024-08-17T16:48:38.635 INFO:teuthology.orchestra.run.smithi063.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:48:38.635 INFO:teuthology.orchestra.run.smithi063.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:48:38.635 INFO:teuthology.orchestra.run.smithi063.stdout:Access: 2024-08-17 16:48:24.322279666 +0000 2024-08-17T16:48:38.635 INFO:teuthology.orchestra.run.smithi063.stdout:Modify: 2024-08-17 16:43:58.314324121 +0000 2024-08-17T16:48:38.635 INFO:teuthology.orchestra.run.smithi063.stdout:Change: 2024-08-17 16:43:58.314324121 +0000 2024-08-17T16:48:38.635 INFO:teuthology.orchestra.run.smithi063.stdout: Birth: 2024-08-17 16:43:58.314324121 +0000 2024-08-17T16:48:38.635 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-17T16:48:38.702 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records in 2024-08-17T16:48:38.702 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records out 2024-08-17T16:48:38.702 INFO:teuthology.orchestra.run.smithi063.stderr:512 bytes copied, 0.000207926 s, 2.5 MB/s 2024-08-17T16:48:38.704 DEBUG:teuthology.orchestra.run.smithi063:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-17T16:48:38.764 DEBUG:teuthology.orchestra.run.smithi063:> stat /dev/vg_nvme/lv_3 2024-08-17T16:48:38.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:38 smithi063 ceph-mon[31063]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi063=y;smithi162=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:48:38.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:38 smithi063 ceph-mon[31063]: Saving service mgr spec with placement smithi063=y;smithi162=x;count:2 2024-08-17T16:48:38.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:38 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:38.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:38 smithi063 ceph-mon[31063]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:38.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:38 smithi063 ceph-mon[42680]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi063=y;smithi162=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:48:38.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:38 smithi063 ceph-mon[42680]: Saving service mgr spec with placement smithi063=y;smithi162=x;count:2 2024-08-17T16:48:38.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:38 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:38.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:38 smithi063 ceph-mon[42680]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:38.796 INFO:teuthology.orchestra.run.smithi063.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-17T16:48:38.796 INFO:teuthology.orchestra.run.smithi063.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:48:38.796 INFO:teuthology.orchestra.run.smithi063.stdout:Device: 5h/5d Inode: 976 Links: 1 2024-08-17T16:48:38.796 INFO:teuthology.orchestra.run.smithi063.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:48:38.796 INFO:teuthology.orchestra.run.smithi063.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:48:38.796 INFO:teuthology.orchestra.run.smithi063.stdout:Access: 2024-08-17 16:48:24.322279666 +0000 2024-08-17T16:48:38.796 INFO:teuthology.orchestra.run.smithi063.stdout:Modify: 2024-08-17 16:43:58.313324129 +0000 2024-08-17T16:48:38.796 INFO:teuthology.orchestra.run.smithi063.stdout:Change: 2024-08-17 16:43:58.313324129 +0000 2024-08-17T16:48:38.796 INFO:teuthology.orchestra.run.smithi063.stdout: Birth: 2024-08-17 16:43:58.313324129 +0000 2024-08-17T16:48:38.797 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-17T16:48:38.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:38 smithi162 ceph-mon[34633]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi063=y;smithi162=x", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:48:38.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:38 smithi162 ceph-mon[34633]: Saving service mgr spec with placement smithi063=y;smithi162=x;count:2 2024-08-17T16:48:38.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:38 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:38.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:38 smithi162 ceph-mon[34633]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:38.861 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records in 2024-08-17T16:48:38.861 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records out 2024-08-17T16:48:38.861 INFO:teuthology.orchestra.run.smithi063.stderr:512 bytes copied, 0.000194304 s, 2.6 MB/s 2024-08-17T16:48:38.862 DEBUG:teuthology.orchestra.run.smithi063:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-17T16:48:38.920 DEBUG:teuthology.orchestra.run.smithi063:> stat /dev/vg_nvme/lv_4 2024-08-17T16:48:38.977 INFO:teuthology.orchestra.run.smithi063.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-17T16:48:38.977 INFO:teuthology.orchestra.run.smithi063.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:48:38.978 INFO:teuthology.orchestra.run.smithi063.stdout:Device: 5h/5d Inode: 986 Links: 1 2024-08-17T16:48:38.978 INFO:teuthology.orchestra.run.smithi063.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:48:38.978 INFO:teuthology.orchestra.run.smithi063.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:48:38.978 INFO:teuthology.orchestra.run.smithi063.stdout:Access: 2024-08-17 16:48:24.323279657 +0000 2024-08-17T16:48:38.978 INFO:teuthology.orchestra.run.smithi063.stdout:Modify: 2024-08-17 16:43:58.314324121 +0000 2024-08-17T16:48:38.978 INFO:teuthology.orchestra.run.smithi063.stdout:Change: 2024-08-17 16:43:58.314324121 +0000 2024-08-17T16:48:38.978 INFO:teuthology.orchestra.run.smithi063.stdout: Birth: 2024-08-17 16:43:58.314324121 +0000 2024-08-17T16:48:38.978 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-17T16:48:39.043 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records in 2024-08-17T16:48:39.043 INFO:teuthology.orchestra.run.smithi063.stderr:1+0 records out 2024-08-17T16:48:39.043 INFO:teuthology.orchestra.run.smithi063.stderr:512 bytes copied, 0.000193675 s, 2.6 MB/s 2024-08-17T16:48:39.044 DEBUG:teuthology.orchestra.run.smithi063:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-17T16:48:39.100 DEBUG:teuthology.orchestra.run.smithi162:> set -ex 2024-08-17T16:48:39.101 DEBUG:teuthology.orchestra.run.smithi162:> dd if=/scratch_devs of=/dev/stdout 2024-08-17T16:48:39.117 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-08-17T16:48:39.117 DEBUG:teuthology.orchestra.run.smithi162:> stat /dev/vg_nvme/lv_1 2024-08-17T16:48:39.173 INFO:teuthology.orchestra.run.smithi162.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-08-17T16:48:39.173 INFO:teuthology.orchestra.run.smithi162.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:48:39.173 INFO:teuthology.orchestra.run.smithi162.stdout:Device: 5h/5d Inode: 981 Links: 1 2024-08-17T16:48:39.173 INFO:teuthology.orchestra.run.smithi162.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:48:39.173 INFO:teuthology.orchestra.run.smithi162.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:48:39.173 INFO:teuthology.orchestra.run.smithi162.stdout:Access: 2024-08-17 16:48:15.157296424 +0000 2024-08-17T16:48:39.173 INFO:teuthology.orchestra.run.smithi162.stdout:Modify: 2024-08-17 16:43:57.781230192 +0000 2024-08-17T16:48:39.173 INFO:teuthology.orchestra.run.smithi162.stdout:Change: 2024-08-17 16:43:57.781230192 +0000 2024-08-17T16:48:39.173 INFO:teuthology.orchestra.run.smithi162.stdout: Birth: 2024-08-17 16:43:57.781230192 +0000 2024-08-17T16:48:39.174 DEBUG:teuthology.orchestra.run.smithi162:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-08-17T16:48:39.237 INFO:teuthology.orchestra.run.smithi162.stderr:1+0 records in 2024-08-17T16:48:39.237 INFO:teuthology.orchestra.run.smithi162.stderr:1+0 records out 2024-08-17T16:48:39.237 INFO:teuthology.orchestra.run.smithi162.stderr:512 bytes copied, 0.000190189 s, 2.7 MB/s 2024-08-17T16:48:39.238 DEBUG:teuthology.orchestra.run.smithi162:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-08-17T16:48:39.296 DEBUG:teuthology.orchestra.run.smithi162:> stat /dev/vg_nvme/lv_2 2024-08-17T16:48:39.352 INFO:teuthology.orchestra.run.smithi162.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-08-17T16:48:39.352 INFO:teuthology.orchestra.run.smithi162.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:48:39.352 INFO:teuthology.orchestra.run.smithi162.stdout:Device: 5h/5d Inode: 974 Links: 1 2024-08-17T16:48:39.352 INFO:teuthology.orchestra.run.smithi162.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:48:39.352 INFO:teuthology.orchestra.run.smithi162.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:48:39.353 INFO:teuthology.orchestra.run.smithi162.stdout:Access: 2024-08-17 16:48:15.157296424 +0000 2024-08-17T16:48:39.353 INFO:teuthology.orchestra.run.smithi162.stdout:Modify: 2024-08-17 16:43:57.781230192 +0000 2024-08-17T16:48:39.353 INFO:teuthology.orchestra.run.smithi162.stdout:Change: 2024-08-17 16:43:57.781230192 +0000 2024-08-17T16:48:39.353 INFO:teuthology.orchestra.run.smithi162.stdout: Birth: 2024-08-17 16:43:57.781230192 +0000 2024-08-17T16:48:39.353 DEBUG:teuthology.orchestra.run.smithi162:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-08-17T16:48:39.416 INFO:teuthology.orchestra.run.smithi162.stderr:1+0 records in 2024-08-17T16:48:39.416 INFO:teuthology.orchestra.run.smithi162.stderr:1+0 records out 2024-08-17T16:48:39.417 INFO:teuthology.orchestra.run.smithi162.stderr:512 bytes copied, 0.000170208 s, 3.0 MB/s 2024-08-17T16:48:39.418 DEBUG:teuthology.orchestra.run.smithi162:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-08-17T16:48:39.474 DEBUG:teuthology.orchestra.run.smithi162:> stat /dev/vg_nvme/lv_3 2024-08-17T16:48:39.530 INFO:teuthology.orchestra.run.smithi162.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-08-17T16:48:39.530 INFO:teuthology.orchestra.run.smithi162.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:48:39.530 INFO:teuthology.orchestra.run.smithi162.stdout:Device: 5h/5d Inode: 985 Links: 1 2024-08-17T16:48:39.530 INFO:teuthology.orchestra.run.smithi162.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:48:39.530 INFO:teuthology.orchestra.run.smithi162.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:48:39.530 INFO:teuthology.orchestra.run.smithi162.stdout:Access: 2024-08-17 16:48:15.157296424 +0000 2024-08-17T16:48:39.530 INFO:teuthology.orchestra.run.smithi162.stdout:Modify: 2024-08-17 16:43:57.781230192 +0000 2024-08-17T16:48:39.530 INFO:teuthology.orchestra.run.smithi162.stdout:Change: 2024-08-17 16:43:57.781230192 +0000 2024-08-17T16:48:39.530 INFO:teuthology.orchestra.run.smithi162.stdout: Birth: 2024-08-17 16:43:57.781230192 +0000 2024-08-17T16:48:39.531 DEBUG:teuthology.orchestra.run.smithi162:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-08-17T16:48:39.594 INFO:teuthology.orchestra.run.smithi162.stderr:1+0 records in 2024-08-17T16:48:39.594 INFO:teuthology.orchestra.run.smithi162.stderr:1+0 records out 2024-08-17T16:48:39.594 INFO:teuthology.orchestra.run.smithi162.stderr:512 bytes copied, 0.000178991 s, 2.9 MB/s 2024-08-17T16:48:39.595 DEBUG:teuthology.orchestra.run.smithi162:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-08-17T16:48:39.651 DEBUG:teuthology.orchestra.run.smithi162:> stat /dev/vg_nvme/lv_4 2024-08-17T16:48:39.707 INFO:teuthology.orchestra.run.smithi162.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-08-17T16:48:39.707 INFO:teuthology.orchestra.run.smithi162.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-08-17T16:48:39.707 INFO:teuthology.orchestra.run.smithi162.stdout:Device: 5h/5d Inode: 972 Links: 1 2024-08-17T16:48:39.707 INFO:teuthology.orchestra.run.smithi162.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-08-17T16:48:39.707 INFO:teuthology.orchestra.run.smithi162.stdout:Context: system_u:object_r:device_t:s0 2024-08-17T16:48:39.707 INFO:teuthology.orchestra.run.smithi162.stdout:Access: 2024-08-17 16:48:15.158296405 +0000 2024-08-17T16:48:39.707 INFO:teuthology.orchestra.run.smithi162.stdout:Modify: 2024-08-17 16:43:57.781230192 +0000 2024-08-17T16:48:39.707 INFO:teuthology.orchestra.run.smithi162.stdout:Change: 2024-08-17 16:43:57.781230192 +0000 2024-08-17T16:48:39.707 INFO:teuthology.orchestra.run.smithi162.stdout: Birth: 2024-08-17 16:43:57.781230192 +0000 2024-08-17T16:48:39.707 DEBUG:teuthology.orchestra.run.smithi162:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-08-17T16:48:39.770 INFO:teuthology.orchestra.run.smithi162.stderr:1+0 records in 2024-08-17T16:48:39.770 INFO:teuthology.orchestra.run.smithi162.stderr:1+0 records out 2024-08-17T16:48:39.770 INFO:teuthology.orchestra.run.smithi162.stderr:512 bytes copied, 0.000224623 s, 2.3 MB/s 2024-08-17T16:48:39.771 DEBUG:teuthology.orchestra.run.smithi162:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-08-17T16:48:39.828 INFO:tasks.cephadm:Deploying osd.0 on smithi063 with /dev/vg_nvme/lv_4... 2024-08-17T16:48:39.828 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-08-17T16:48:39.851 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:39.851 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:39.851 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:39.851 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:39.851 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:39.851 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:39.851 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:48:39.851 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-17T16:48:39.851 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:48:39.852 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:39.852 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[42680]: Deploying daemon mgr.x on smithi162 2024-08-17T16:48:39.852 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:39.852 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:39.852 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:39.852 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:39.852 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:39.852 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:39.852 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:48:39.852 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-17T16:48:39.852 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:48:39.853 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:39.853 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:39 smithi063 ceph-mon[31063]: Deploying daemon mgr.x on smithi162 2024-08-17T16:48:40.042 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:48:40.060 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:39 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:40.060 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:39 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:40.060 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:39 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:40.060 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:39 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:40.061 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:39 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:40.061 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:39 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:40.061 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:39 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:48:40.061 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:39 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-08-17T16:48:40.061 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:39 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:48:40.061 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:39 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:40.061 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:39 smithi162 ceph-mon[34633]: Deploying daemon mgr.x on smithi162 2024-08-17T16:48:40.719 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:40 smithi162 podman[36541]: 2024-08-17 16:48:40.718595172 +0000 UTC m=+0.149401690 container create 4add2ba1a53e5a30fc12e2508486c3ff268c4357e29f3d88b5e52394923358a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-08-17T16:48:40.977 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:40 smithi162 ceph-mon[34633]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:40.977 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:40 smithi162 podman[36541]: 2024-08-17 16:48:40.947631386 +0000 UTC m=+0.378437918 container init 4add2ba1a53e5a30fc12e2508486c3ff268c4357e29f3d88b5e52394923358a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f) 2024-08-17T16:48:40.977 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:40 smithi162 podman[36541]: 2024-08-17 16:48:40.952527711 +0000 UTC m=+0.383334230 container start 4add2ba1a53e5a30fc12e2508486c3ff268c4357e29f3d88b5e52394923358a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731) 2024-08-17T16:48:40.978 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:40 smithi162 ceph-mgr[36579]: -- 172.21.15.162:0/3008668886 <== mon.1 v2:172.21.15.162:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563f0c75ad00 con 0x563f0c72e800 2024-08-17T16:48:41.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:40 smithi063 ceph-mon[31063]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:40 smithi063 ceph-mon[42680]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:41.242 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:41 smithi162 bash[36541]: 4add2ba1a53e5a30fc12e2508486c3ff268c4357e29f3d88b5e52394923358a8 2024-08-17T16:48:41.242 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:41 smithi162 systemd[1]: Started Ceph mgr.x for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:48:41.242 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:41 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:41.073+0000 7fe82847a040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:48:41.242 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:41 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:41.240+0000 7fe82847a040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:48:41.580 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:41 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:41.312+0000 7fe82847a040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:48:41.580 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:41 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:41.578+0000 7fe82847a040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:48:41.845 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:41 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:41.637+0000 7fe82847a040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:48:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:42 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:42 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:42 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:42 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:42 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:42 smithi162 ceph-mon[34633]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:42.519 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:42 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:42.519 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:42 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:42.519 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:42 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:42.519 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:42 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:42.519 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:42 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:42.519 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:42 smithi063 ceph-mon[42680]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:42.520 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:42 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:42.520 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:42 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:42.520 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:42 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:42.520 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:42 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:42.520 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:42 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:42.520 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:42 smithi063 ceph-mon[31063]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:42.522 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:48:42.543 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi063:vg_nvme/lv_4 2024-08-17T16:48:42.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:42 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:42.591+0000 7fe82847a040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:48:42.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:42 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:42.703+0000 7fe82847a040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:48:42.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:42 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:42.759+0000 7fe82847a040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:48:43.230 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:42 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:42.966+0000 7fe82847a040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:48:43.230 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:43 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:43.025+0000 7fe82847a040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:48:43.230 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:43 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:43.089+0000 7fe82847a040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:48:43.230 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:43 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:43.228+0000 7fe82847a040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:48:43.516 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:43.516 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:43.516 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:43.516 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:43.516 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:43.516 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[31063]: Reconfiguring mgr.y (unknown last config time)... 2024-08-17T16:48:43.516 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:48:43.516 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:48:43.517 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:43.517 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[31063]: Reconfiguring daemon mgr.y on smithi063 2024-08-17T16:48:43.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:43.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:43.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:43.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:43.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:43.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[42680]: Reconfiguring mgr.y (unknown last config time)... 2024-08-17T16:48:43.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:48:43.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:48:43.518 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:43.518 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:43 smithi063 ceph-mon[42680]: Reconfiguring daemon mgr.y on smithi063 2024-08-17T16:48:43.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:43 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:43.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:43 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:43.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:43 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:43.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:43 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:43.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:43 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:43.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:43 smithi162 ceph-mon[34633]: Reconfiguring mgr.y (unknown last config time)... 2024-08-17T16:48:43.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:43 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:48:43.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:43 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:48:43.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:43 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:43.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:43 smithi162 ceph-mon[34633]: Reconfiguring daemon mgr.y on smithi063 2024-08-17T16:48:43.612 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:48:43.954 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:43 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:43.644+0000 7fe82847a040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:48:43.954 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:43 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:43.952+0000 7fe82847a040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:48:44.345 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:44 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:44.019+0000 7fe82847a040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:48:44.345 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:44 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:44.169+0000 7fe82847a040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:48:44.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:44 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:44.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:44 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:44.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:44 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:44.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:44 smithi063 ceph-mon[42680]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:44.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:44 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:44.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:44 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:44.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:44 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:44.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:44 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:44.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:44 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:44.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:44 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:44.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:44 smithi063 ceph-mon[31063]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:44.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:44 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:44.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:44 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:44.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:44 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:44.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:44 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:44.469+0000 7fe82847a040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:48:44.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:44 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:44.529+0000 7fe82847a040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:48:44.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:44 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:44.713+0000 7fe82847a040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:48:44.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:44 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:44.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:44 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:44.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:44 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:44.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:44 smithi162 ceph-mon[34633]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:44.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:44 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:44.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:44 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:48:44.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:44 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:45.345 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:44 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:44.860+0000 7fe82847a040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-17T16:48:45.346 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:44 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:44.922+0000 7fe82847a040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:48:45.346 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:45.006+0000 7fe82847a040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:48:45.346 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:45.064+0000 7fe82847a040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:48:45.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:45.585+0000 7fe82847a040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:48:45.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:48:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:48:45.739+0000 7fe82847a040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:48:46.027 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:46.027 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[31063]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:48:46.027 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:48:46.027 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:48:46.027 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:46.027 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[31063]: Standby manager daemon x started 2024-08-17T16:48:46.028 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/2017258348' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:48:46.028 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/2017258348' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:48:46.028 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/2017258348' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:48:46.028 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/2017258348' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:48:46.028 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[31063]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:46.028 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:46.028 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[42680]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:48:46.028 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:48:46.028 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:48:46.028 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[42680]: Standby manager daemon x started 2024-08-17T16:48:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/2017258348' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:48:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/2017258348' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:48:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/2017258348' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:48:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/2017258348' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:48:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:45 smithi063 ceph-mon[42680]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:45 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:45 smithi162 ceph-mon[34633]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:48:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:45 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:48:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:45 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:48:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:45 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:45 smithi162 ceph-mon[34633]: Standby manager daemon x started 2024-08-17T16:48:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:45 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/2017258348' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:48:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:45 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/2017258348' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:48:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:45 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/2017258348' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:48:46.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:45 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/2017258348' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:48:46.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:45 smithi162 ceph-mon[34633]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:47.205 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:46 smithi063 ceph-mon[31063]: mgrmap e14: y(active, since 66s), standbys: x 2024-08-17T16:48:47.206 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:46 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:48:47.206 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:46 smithi063 ceph-mon[42680]: mgrmap e14: y(active, since 66s), standbys: x 2024-08-17T16:48:47.206 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:46 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:48:47.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:46 smithi162 ceph-mon[34633]: mgrmap e14: y(active, since 66s), standbys: x 2024-08-17T16:48:47.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:46 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:48:48.240 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:47 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/490183992' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2"}]: dispatch 2024-08-17T16:48:48.240 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:47 smithi063 ceph-mon[31063]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2"}]: dispatch 2024-08-17T16:48:48.240 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:47 smithi063 ceph-mon[31063]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2"}]': finished 2024-08-17T16:48:48.240 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:47 smithi063 ceph-mon[31063]: osdmap e5: 1 total, 0 up, 1 in 2024-08-17T16:48:48.240 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:47 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:48:48.240 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:47 smithi063 ceph-mon[31063]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:48.241 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:47 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/490183992' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2"}]: dispatch 2024-08-17T16:48:48.241 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:47 smithi063 ceph-mon[42680]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2"}]: dispatch 2024-08-17T16:48:48.241 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:47 smithi063 ceph-mon[42680]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2"}]': finished 2024-08-17T16:48:48.241 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:47 smithi063 ceph-mon[42680]: osdmap e5: 1 total, 0 up, 1 in 2024-08-17T16:48:48.241 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:47 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:48:48.241 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:47 smithi063 ceph-mon[42680]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:48.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:47 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/490183992' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2"}]: dispatch 2024-08-17T16:48:48.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:47 smithi162 ceph-mon[34633]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2"}]: dispatch 2024-08-17T16:48:48.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:47 smithi162 ceph-mon[34633]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2"}]': finished 2024-08-17T16:48:48.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:47 smithi162 ceph-mon[34633]: osdmap e5: 1 total, 0 up, 1 in 2024-08-17T16:48:48.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:47 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:48:48.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:47 smithi162 ceph-mon[34633]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:49.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:48 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/2472669324' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:48:49.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:48 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/2472669324' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:48:49.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:48 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/2472669324' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:48:50.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:49 smithi063 ceph-mon[31063]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:50.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:49 smithi063 ceph-mon[42680]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:50.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:49 smithi162 ceph-mon[34633]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:52 smithi063 ceph-mon[31063]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:52 smithi063 ceph-mon[42680]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:52.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:52 smithi162 ceph-mon[34633]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:54.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:53 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-17T16:48:54.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:53 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:54.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:53 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-17T16:48:54.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:53 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:53 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-08-17T16:48:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:53 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:48:55.010 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:54 smithi063 ceph-mon[31063]: Deploying daemon osd.0 on smithi063 2024-08-17T16:48:55.011 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:54 smithi063 ceph-mon[31063]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:55.011 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:54 smithi063 ceph-mon[42680]: Deploying daemon osd.0 on smithi063 2024-08-17T16:48:55.011 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:54 smithi063 ceph-mon[42680]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:54 smithi162 ceph-mon[34633]: Deploying daemon osd.0 on smithi063 2024-08-17T16:48:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:54 smithi162 ceph-mon[34633]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:56.110 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:56 smithi063 ceph-mon[31063]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:56.110 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:56 smithi063 ceph-mon[42680]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:56.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:56 smithi162 ceph-mon[34633]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:58.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:58 smithi063 ceph-mon[42680]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:58.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:58 smithi063 ceph-mon[31063]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:58.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:58 smithi162 ceph-mon[34633]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:48:59.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:59 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:59.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:59 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:59.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:48:59 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:59.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:59 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:59.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:59 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:59.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:48:59 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:59.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:59 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:48:59.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:59 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:48:59.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:48:59 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:00.234 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:00 smithi063 ceph-mon[42680]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:49:00.235 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:00 smithi063 ceph-mon[31063]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:49:00.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:00 smithi162 ceph-mon[34633]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:49:01.566 INFO:teuthology.orchestra.run.smithi063.stdout:Created osd(s) 0 on host 'smithi063' 2024-08-17T16:49:02.151 DEBUG:teuthology.orchestra.run.smithi063:osd.0> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.0.service 2024-08-17T16:49:02.154 INFO:tasks.cephadm:Deploying osd.1 on smithi063 with /dev/vg_nvme/lv_3... 2024-08-17T16:49:02.154 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-08-17T16:49:02.405 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:49:02.816 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:02.816 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:02.816 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:02 smithi063 ceph-mon[31063]: from='osd.0 [v2:172.21.15.63:6802/782866097,v1:172.21.15.63:6803/782866097]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T16:49:02.817 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:02 smithi063 ceph-mon[31063]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T16:49:02.817 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:02 smithi063 ceph-mon[31063]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:49:02.817 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:02.817 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:02.817 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:02.817 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:02.817 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:02 smithi063 ceph-mon[42680]: from='osd.0 [v2:172.21.15.63:6802/782866097,v1:172.21.15.63:6803/782866097]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T16:49:02.817 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:02 smithi063 ceph-mon[42680]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T16:49:02.817 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:02 smithi063 ceph-mon[42680]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:49:02.817 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:02.817 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:02.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:02 smithi162 ceph-mon[34633]: from='osd.0 [v2:172.21.15.63:6802/782866097,v1:172.21.15.63:6803/782866097]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T16:49:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:02 smithi162 ceph-mon[34633]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-08-17T16:49:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:02 smithi162 ceph-mon[34633]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:49:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:03.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:03 smithi063 ceph-mon[42680]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T16:49:03.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:03 smithi063 ceph-mon[42680]: osdmap e6: 1 total, 0 up, 1 in 2024-08-17T16:49:03.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:03 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:49:03.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:03 smithi063 ceph-mon[42680]: from='osd.0 [v2:172.21.15.63:6802/782866097,v1:172.21.15.63:6803/782866097]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:49:03.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:03 smithi063 ceph-mon[42680]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:49:03.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:03 smithi063 ceph-mon[31063]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T16:49:03.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:03 smithi063 ceph-mon[31063]: osdmap e6: 1 total, 0 up, 1 in 2024-08-17T16:49:03.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:03 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:49:03.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:03 smithi063 ceph-mon[31063]: from='osd.0 [v2:172.21.15.63:6802/782866097,v1:172.21.15.63:6803/782866097]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:49:03.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:03 smithi063 ceph-mon[31063]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:49:03.781 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 16:49:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0[47037]: 2024-08-17T16:49:03.586+0000 7f09e7026640 -1 osd.0 0 waiting for initial osdmap 2024-08-17T16:49:03.781 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 16:49:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0[47037]: 2024-08-17T16:49:03.588+0000 7f09e2e4f640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:49:03.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:03 smithi162 ceph-mon[34633]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T16:49:03.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:03 smithi162 ceph-mon[34633]: osdmap e6: 1 total, 0 up, 1 in 2024-08-17T16:49:03.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:03 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:49:03.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:03 smithi162 ceph-mon[34633]: from='osd.0 [v2:172.21.15.63:6802/782866097,v1:172.21.15.63:6803/782866097]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:49:03.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:03 smithi162 ceph-mon[34633]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:49:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:04 smithi162 ceph-mon[34633]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-08-17T16:49:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:04 smithi162 ceph-mon[34633]: osdmap e7: 1 total, 0 up, 1 in 2024-08-17T16:49:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:04 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:49:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:04 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:49:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:04 smithi162 ceph-mon[34633]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:49:04.938 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:04 smithi063 ceph-mon[42680]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-08-17T16:49:04.938 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:04 smithi063 ceph-mon[42680]: osdmap e7: 1 total, 0 up, 1 in 2024-08-17T16:49:04.938 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:04 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:49:04.938 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:04 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:49:04.938 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:04 smithi063 ceph-mon[42680]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:49:04.939 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:04 smithi063 ceph-mon[31063]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-08-17T16:49:04.939 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:04 smithi063 ceph-mon[31063]: osdmap e7: 1 total, 0 up, 1 in 2024-08-17T16:49:04.939 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:04 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:49:04.939 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:04 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:49:04.939 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:04 smithi063 ceph-mon[31063]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-08-17T16:49:05.916 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:05 smithi063 ceph-mon[31063]: purged_snaps scrub starts 2024-08-17T16:49:05.916 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:05 smithi063 ceph-mon[31063]: purged_snaps scrub ok 2024-08-17T16:49:05.917 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:05 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:49:05.917 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:05 smithi063 ceph-mon[31063]: osd.0 [v2:172.21.15.63:6802/782866097,v1:172.21.15.63:6803/782866097] boot 2024-08-17T16:49:05.917 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:05 smithi063 ceph-mon[31063]: osdmap e8: 1 total, 1 up, 1 in 2024-08-17T16:49:05.917 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:05 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:49:05.917 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:05 smithi063 ceph-mon[42680]: purged_snaps scrub starts 2024-08-17T16:49:05.917 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:05 smithi063 ceph-mon[42680]: purged_snaps scrub ok 2024-08-17T16:49:05.917 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:05 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:49:05.917 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:05 smithi063 ceph-mon[42680]: osd.0 [v2:172.21.15.63:6802/782866097,v1:172.21.15.63:6803/782866097] boot 2024-08-17T16:49:05.917 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:05 smithi063 ceph-mon[42680]: osdmap e8: 1 total, 1 up, 1 in 2024-08-17T16:49:05.917 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:05 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:49:05.919 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:49:05.940 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi063:vg_nvme/lv_3 2024-08-17T16:49:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:05 smithi162 ceph-mon[34633]: purged_snaps scrub starts 2024-08-17T16:49:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:05 smithi162 ceph-mon[34633]: purged_snaps scrub ok 2024-08-17T16:49:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:05 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:49:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:05 smithi162 ceph-mon[34633]: osd.0 [v2:172.21.15.63:6802/782866097,v1:172.21.15.63:6803/782866097] boot 2024-08-17T16:49:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:05 smithi162 ceph-mon[34633]: osdmap e8: 1 total, 1 up, 1 in 2024-08-17T16:49:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:05 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:49:06.448 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:49:06.864 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:06 smithi063 ceph-mon[42680]: osdmap e9: 1 total, 1 up, 1 in 2024-08-17T16:49:06.864 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:06 smithi063 ceph-mon[42680]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:06.865 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:06 smithi063 ceph-mon[31063]: osdmap e9: 1 total, 1 up, 1 in 2024-08-17T16:49:06.865 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:06 smithi063 ceph-mon[31063]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:06 smithi162 ceph-mon[34633]: osdmap e9: 1 total, 1 up, 1 in 2024-08-17T16:49:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:06 smithi162 ceph-mon[34633]: pgmap v42: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:09.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[31063]: Detected new or changed devices on smithi063 2024-08-17T16:49:09.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:09.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[31063]: Adjusting osd_memory_target on smithi063 to 16064M 2024-08-17T16:49:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:49:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[31063]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[42680]: Detected new or changed devices on smithi063 2024-08-17T16:49:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[42680]: Adjusting osd_memory_target on smithi063 to 16064M 2024-08-17T16:49:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:49:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[42680]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:09.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:09.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:08 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:08 smithi162 ceph-mon[34633]: Detected new or changed devices on smithi063 2024-08-17T16:49:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:08 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:08 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:08 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:08 smithi162 ceph-mon[34633]: Adjusting osd_memory_target on smithi063 to 16064M 2024-08-17T16:49:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:08 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:08 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:08 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:08 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:08 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:49:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:08 smithi162 ceph-mon[34633]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:08 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:08 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:08 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:09 smithi063 ceph-mon[42680]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:49:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:09 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:49:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:09 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:49:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:09 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:10.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:09 smithi063 ceph-mon[31063]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:49:10.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:09 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:49:10.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:09 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:49:10.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:09 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:09 smithi162 ceph-mon[34633]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:49:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:09 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:49:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:09 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:49:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:09 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:11.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:10 smithi063 ceph-mon[31063]: pgmap v44: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:11.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:10 smithi063 ceph-mon[42680]: pgmap v44: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:10 smithi162 ceph-mon[34633]: pgmap v44: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:11 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/208850648' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ae346e99-4706-4871-990a-f8e737572245"}]: dispatch 2024-08-17T16:49:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:11 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/208850648' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ae346e99-4706-4871-990a-f8e737572245"}]': finished 2024-08-17T16:49:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:11 smithi063 ceph-mon[31063]: osdmap e10: 2 total, 1 up, 2 in 2024-08-17T16:49:12.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:11 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:49:12.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:11 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/1503314312' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:49:12.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:11 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/208850648' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ae346e99-4706-4871-990a-f8e737572245"}]: dispatch 2024-08-17T16:49:12.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:11 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/208850648' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ae346e99-4706-4871-990a-f8e737572245"}]': finished 2024-08-17T16:49:12.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:11 smithi063 ceph-mon[42680]: osdmap e10: 2 total, 1 up, 2 in 2024-08-17T16:49:12.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:11 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:49:12.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:11 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/1503314312' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:49:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:11 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/208850648' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ae346e99-4706-4871-990a-f8e737572245"}]: dispatch 2024-08-17T16:49:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:11 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/208850648' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ae346e99-4706-4871-990a-f8e737572245"}]': finished 2024-08-17T16:49:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:11 smithi162 ceph-mon[34633]: osdmap e10: 2 total, 1 up, 2 in 2024-08-17T16:49:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:11 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:49:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:11 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/1503314312' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:49:13.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:12 smithi063 ceph-mon[31063]: pgmap v46: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:13.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:12 smithi063 ceph-mon[42680]: pgmap v46: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:12 smithi162 ceph-mon[34633]: pgmap v46: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:14.265 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:14 smithi063 ceph-mon[31063]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:14.265 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:14 smithi063 ceph-mon[42680]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:14.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:14 smithi162 ceph-mon[34633]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:16.129 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:16 smithi063 ceph-mon[42680]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:16.129 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:16 smithi063 ceph-mon[31063]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:16.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:16 smithi162 ceph-mon[34633]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:17.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:17 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-17T16:49:17.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:17 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:17.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:17 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-17T16:49:17.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:17 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:17.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:17 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-08-17T16:49:17.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:17 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:18.367 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:18 smithi063 ceph-mon[31063]: Deploying daemon osd.1 on smithi063 2024-08-17T16:49:18.381 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:18 smithi063 ceph-mon[31063]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:18.382 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:18 smithi063 ceph-mon[42680]: Deploying daemon osd.1 on smithi063 2024-08-17T16:49:18.382 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:18 smithi063 ceph-mon[42680]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:18.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:18 smithi162 ceph-mon[34633]: Deploying daemon osd.1 on smithi063 2024-08-17T16:49:18.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:18 smithi162 ceph-mon[34633]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:20.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:20 smithi063 ceph-mon[31063]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:20.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:20 smithi063 ceph-mon[42680]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:20.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:20 smithi162 ceph-mon[34633]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:22.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:22 smithi063 ceph-mon[42680]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:22.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:22 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:49:22.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:22 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:22.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:22 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:22.215 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:22 smithi063 ceph-mon[31063]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:22.215 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:22 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:49:22.216 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:22 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:22.216 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:22 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:22.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:22 smithi162 ceph-mon[34633]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:22.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:22 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:49:22.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:22 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:22.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:22 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:24.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:24 smithi063 ceph-mon[31063]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:24.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:24 smithi063 ceph-mon[42680]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:24.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:24 smithi162 ceph-mon[34633]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:25.361 INFO:teuthology.orchestra.run.smithi063.stdout:Created osd(s) 1 on host 'smithi063' 2024-08-17T16:49:25.916 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:25.917 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:25.917 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:25.917 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:25.917 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:25.917 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[42680]: from='osd.1 [v2:172.21.15.63:6810/944260396,v1:172.21.15.63:6811/944260396]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T16:49:25.917 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:49:25.917 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:25.917 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:25.918 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:25.918 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:25.918 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:25.918 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:25.918 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:25.918 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[31063]: from='osd.1 [v2:172.21.15.63:6810/944260396,v1:172.21.15.63:6811/944260396]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T16:49:25.918 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:49:25.918 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:25.918 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:25 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:25.942 DEBUG:teuthology.orchestra.run.smithi063:osd.1> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.1.service 2024-08-17T16:49:25.945 INFO:tasks.cephadm:Deploying osd.2 on smithi063 with /dev/vg_nvme/lv_2... 2024-08-17T16:49:25.945 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-08-17T16:49:26.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:25 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:25 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:25 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:25 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:25 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:25 smithi162 ceph-mon[34633]: from='osd.1 [v2:172.21.15.63:6810/944260396,v1:172.21.15.63:6811/944260396]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-08-17T16:49:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:25 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:49:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:25 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:25 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:26.593 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:49:26.973 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:26 smithi063 ceph-mon[42680]: from='osd.1 [v2:172.21.15.63:6810/944260396,v1:172.21.15.63:6811/944260396]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T16:49:26.974 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:26 smithi063 ceph-mon[42680]: osdmap e11: 2 total, 1 up, 2 in 2024-08-17T16:49:26.974 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:26 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:49:26.974 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:26 smithi063 ceph-mon[42680]: from='osd.1 [v2:172.21.15.63:6810/944260396,v1:172.21.15.63:6811/944260396]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:49:26.974 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:26 smithi063 ceph-mon[42680]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:26.974 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:26 smithi063 ceph-mon[31063]: from='osd.1 [v2:172.21.15.63:6810/944260396,v1:172.21.15.63:6811/944260396]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T16:49:26.974 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:26 smithi063 ceph-mon[31063]: osdmap e11: 2 total, 1 up, 2 in 2024-08-17T16:49:26.974 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:26 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:49:26.974 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:26 smithi063 ceph-mon[31063]: from='osd.1 [v2:172.21.15.63:6810/944260396,v1:172.21.15.63:6811/944260396]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:49:26.974 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:26 smithi063 ceph-mon[31063]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:26.975 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 16:49:26 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1[52945]: 2024-08-17T16:49:26.664+0000 7fa1b231d640 -1 osd.1 0 waiting for initial osdmap 2024-08-17T16:49:26.975 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 16:49:26 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1[52945]: 2024-08-17T16:49:26.669+0000 7fa1ae146640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:49:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:26 smithi162 ceph-mon[34633]: from='osd.1 [v2:172.21.15.63:6810/944260396,v1:172.21.15.63:6811/944260396]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T16:49:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:26 smithi162 ceph-mon[34633]: osdmap e11: 2 total, 1 up, 2 in 2024-08-17T16:49:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:26 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:49:27.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:26 smithi162 ceph-mon[34633]: from='osd.1 [v2:172.21.15.63:6810/944260396,v1:172.21.15.63:6811/944260396]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:49:27.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:26 smithi162 ceph-mon[34633]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:27.881 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:27 smithi063 ceph-mon[31063]: from='osd.1 [v2:172.21.15.63:6810/944260396,v1:172.21.15.63:6811/944260396]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-08-17T16:49:27.881 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:27 smithi063 ceph-mon[31063]: osdmap e12: 2 total, 1 up, 2 in 2024-08-17T16:49:27.881 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:27 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:49:27.881 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:27 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:49:27.882 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:27 smithi063 ceph-mon[42680]: from='osd.1 [v2:172.21.15.63:6810/944260396,v1:172.21.15.63:6811/944260396]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-08-17T16:49:27.882 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:27 smithi063 ceph-mon[42680]: osdmap e12: 2 total, 1 up, 2 in 2024-08-17T16:49:27.882 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:27 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:49:27.882 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:27 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:49:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:27 smithi162 ceph-mon[34633]: from='osd.1 [v2:172.21.15.63:6810/944260396,v1:172.21.15.63:6811/944260396]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-08-17T16:49:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:27 smithi162 ceph-mon[34633]: osdmap e12: 2 total, 1 up, 2 in 2024-08-17T16:49:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:27 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:49:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:27 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:49:28.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:28 smithi063 ceph-mon[31063]: purged_snaps scrub starts 2024-08-17T16:49:28.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:28 smithi063 ceph-mon[31063]: purged_snaps scrub ok 2024-08-17T16:49:28.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:28 smithi063 ceph-mon[31063]: osd.1 [v2:172.21.15.63:6810/944260396,v1:172.21.15.63:6811/944260396] boot 2024-08-17T16:49:28.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:28 smithi063 ceph-mon[31063]: osdmap e13: 2 total, 2 up, 2 in 2024-08-17T16:49:28.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:28 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:49:28.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:28 smithi063 ceph-mon[31063]: pgmap v57: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:28.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:28 smithi063 ceph-mon[42680]: purged_snaps scrub starts 2024-08-17T16:49:28.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:28 smithi063 ceph-mon[42680]: purged_snaps scrub ok 2024-08-17T16:49:28.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:28 smithi063 ceph-mon[42680]: osd.1 [v2:172.21.15.63:6810/944260396,v1:172.21.15.63:6811/944260396] boot 2024-08-17T16:49:28.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:28 smithi063 ceph-mon[42680]: osdmap e13: 2 total, 2 up, 2 in 2024-08-17T16:49:28.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:28 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:49:28.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:28 smithi063 ceph-mon[42680]: pgmap v57: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:29.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:28 smithi162 ceph-mon[34633]: purged_snaps scrub starts 2024-08-17T16:49:29.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:28 smithi162 ceph-mon[34633]: purged_snaps scrub ok 2024-08-17T16:49:29.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:28 smithi162 ceph-mon[34633]: osd.1 [v2:172.21.15.63:6810/944260396,v1:172.21.15.63:6811/944260396] boot 2024-08-17T16:49:29.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:28 smithi162 ceph-mon[34633]: osdmap e13: 2 total, 2 up, 2 in 2024-08-17T16:49:29.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:28 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:49:29.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:28 smithi162 ceph-mon[34633]: pgmap v57: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-08-17T16:49:29.897 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[31063]: osdmap e14: 2 total, 2 up, 2 in 2024-08-17T16:49:29.898 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:29.898 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:29.898 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:29.898 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:29.898 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:29.898 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:29.898 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:29.898 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:29.899 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[42680]: osdmap e14: 2 total, 2 up, 2 in 2024-08-17T16:49:29.899 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:29.899 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:29.899 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:29.899 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:29.899 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:29.899 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:29.899 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:29.899 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:29 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:29 smithi162 ceph-mon[34633]: osdmap e14: 2 total, 2 up, 2 in 2024-08-17T16:49:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:29 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:30.575 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:49:30.595 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi063:vg_nvme/lv_2 2024-08-17T16:49:30.807 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:49:30.930 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:30 smithi063 ceph-mon[31063]: Detected new or changed devices on smithi063 2024-08-17T16:49:30.930 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:30 smithi063 ceph-mon[31063]: Adjusting osd_memory_target on smithi063 to 8032M 2024-08-17T16:49:30.930 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:30 smithi063 ceph-mon[31063]: pgmap v59: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:30.930 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:30 smithi063 ceph-mon[42680]: Detected new or changed devices on smithi063 2024-08-17T16:49:30.930 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:30 smithi063 ceph-mon[42680]: Adjusting osd_memory_target on smithi063 to 8032M 2024-08-17T16:49:30.930 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:30 smithi063 ceph-mon[42680]: pgmap v59: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:30 smithi162 ceph-mon[34633]: Detected new or changed devices on smithi063 2024-08-17T16:49:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:30 smithi162 ceph-mon[34633]: Adjusting osd_memory_target on smithi063 to 8032M 2024-08-17T16:49:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:30 smithi162 ceph-mon[34633]: pgmap v59: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:32.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:32 smithi063 ceph-mon[31063]: pgmap v60: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:32.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:32 smithi063 ceph-mon[42680]: pgmap v60: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:32.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:32 smithi162 ceph-mon[34633]: pgmap v60: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:33.217 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:33 smithi063 ceph-mon[31063]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:49:33.217 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:33 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:49:33.218 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:33 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:49:33.218 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:33 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:33.218 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:33 smithi063 ceph-mon[42680]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:49:33.218 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:33 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:49:33.218 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:33 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:49:33.218 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:33 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:33 smithi162 ceph-mon[34633]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:49:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:33 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:49:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:33 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:49:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:33 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:34.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:34 smithi063 ceph-mon[42680]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:34.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:34 smithi063 ceph-mon[31063]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:34.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:34 smithi162 ceph-mon[34633]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:35.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:35 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/2228241481' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d85d75f-1280-443f-857d-d322bd60c451"}]: dispatch 2024-08-17T16:49:35.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:35 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/2228241481' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2d85d75f-1280-443f-857d-d322bd60c451"}]': finished 2024-08-17T16:49:35.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:35 smithi063 ceph-mon[31063]: osdmap e15: 3 total, 2 up, 3 in 2024-08-17T16:49:35.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:35 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:35 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/2228241481' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d85d75f-1280-443f-857d-d322bd60c451"}]: dispatch 2024-08-17T16:49:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:35 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/2228241481' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2d85d75f-1280-443f-857d-d322bd60c451"}]': finished 2024-08-17T16:49:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:35 smithi063 ceph-mon[42680]: osdmap e15: 3 total, 2 up, 3 in 2024-08-17T16:49:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:35 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:35.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:35 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/2228241481' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d85d75f-1280-443f-857d-d322bd60c451"}]: dispatch 2024-08-17T16:49:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:35 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/2228241481' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2d85d75f-1280-443f-857d-d322bd60c451"}]': finished 2024-08-17T16:49:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:35 smithi162 ceph-mon[34633]: osdmap e15: 3 total, 2 up, 3 in 2024-08-17T16:49:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:35 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:36.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:36 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/2131482431' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:49:36.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:36 smithi162 ceph-mon[34633]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:36.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:36 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/2131482431' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:49:36.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:36 smithi063 ceph-mon[42680]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:36.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:36 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/2131482431' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:49:36.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:36 smithi063 ceph-mon[31063]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:38.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:38 smithi063 ceph-mon[31063]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:38.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:38 smithi063 ceph-mon[42680]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:38.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:38 smithi162 ceph-mon[34633]: pgmap v64: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:40.133 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:40 smithi063 ceph-mon[31063]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:40.134 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:40 smithi063 ceph-mon[42680]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:40.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:40 smithi162 ceph-mon[34633]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:41.315 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:41 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-17T16:49:41.315 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:41 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:41.315 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:41 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-17T16:49:41.315 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:41 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:41.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:41 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-08-17T16:49:41.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:41 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:42.467 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:42 smithi063 ceph-mon[31063]: Deploying daemon osd.2 on smithi063 2024-08-17T16:49:42.468 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:42 smithi063 ceph-mon[31063]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:42.468 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:42 smithi063 ceph-mon[42680]: Deploying daemon osd.2 on smithi063 2024-08-17T16:49:42.468 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:42 smithi063 ceph-mon[42680]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:42.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:42 smithi162 ceph-mon[34633]: Deploying daemon osd.2 on smithi063 2024-08-17T16:49:42.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:42 smithi162 ceph-mon[34633]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:44.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:44 smithi063 ceph-mon[31063]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:44.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:44 smithi063 ceph-mon[42680]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:44.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:44 smithi162 ceph-mon[34633]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:46.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:46 smithi063 ceph-mon[31063]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:46.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:46 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:49:46.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:46 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:46.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:46 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:46 smithi063 ceph-mon[42680]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:46 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:49:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:46 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:46 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:46 smithi162 ceph-mon[34633]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:46 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:49:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:46 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:46 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:48 smithi063 ceph-mon[42680]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:48.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:48 smithi063 ceph-mon[31063]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:48.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:48 smithi162 ceph-mon[34633]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:49.096 INFO:teuthology.orchestra.run.smithi063.stdout:Created osd(s) 2 on host 'smithi063' 2024-08-17T16:49:49.743 DEBUG:teuthology.orchestra.run.smithi063:osd.2> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.2.service 2024-08-17T16:49:49.746 INFO:tasks.cephadm:Deploying osd.3 on smithi063 with /dev/vg_nvme/lv_1... 2024-08-17T16:49:49.746 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-08-17T16:49:50.005 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:49:50.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:50 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:50.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:50 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:50.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:50 smithi162 ceph-mon[34633]: from='osd.2 [v2:172.21.15.63:6818/3751946332,v1:172.21.15.63:6819/3751946332]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-17T16:49:50.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:50 smithi162 ceph-mon[34633]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-17T16:49:50.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:50 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:50.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:50 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:50.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:50 smithi162 ceph-mon[34633]: pgmap v70: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:50.385 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:50 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:50.385 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:50 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:50.385 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:50 smithi063 ceph-mon[42680]: from='osd.2 [v2:172.21.15.63:6818/3751946332,v1:172.21.15.63:6819/3751946332]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-17T16:49:50.385 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:50 smithi063 ceph-mon[42680]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-17T16:49:50.385 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:50 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:50.385 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:50 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:50.385 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:50 smithi063 ceph-mon[42680]: pgmap v70: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:50.386 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:50 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:50.386 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:50 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:50.386 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:50 smithi063 ceph-mon[31063]: from='osd.2 [v2:172.21.15.63:6818/3751946332,v1:172.21.15.63:6819/3751946332]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-17T16:49:50.386 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:50 smithi063 ceph-mon[31063]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-08-17T16:49:50.386 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:50 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:50.386 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:50 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:50.386 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:50 smithi063 ceph-mon[31063]: pgmap v70: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:51.318 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:51 smithi063 ceph-mon[31063]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T16:49:51.318 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:51 smithi063 ceph-mon[31063]: osdmap e16: 3 total, 2 up, 3 in 2024-08-17T16:49:51.318 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:51 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:51.319 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:51 smithi063 ceph-mon[31063]: from='osd.2 [v2:172.21.15.63:6818/3751946332,v1:172.21.15.63:6819/3751946332]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:49:51.319 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:51 smithi063 ceph-mon[31063]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:49:51.319 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:51 smithi063 ceph-mon[42680]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T16:49:51.319 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:51 smithi063 ceph-mon[42680]: osdmap e16: 3 total, 2 up, 3 in 2024-08-17T16:49:51.319 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:51 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:51.319 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:51 smithi063 ceph-mon[42680]: from='osd.2 [v2:172.21.15.63:6818/3751946332,v1:172.21.15.63:6819/3751946332]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:49:51.319 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:51 smithi063 ceph-mon[42680]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:49:51.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:51 smithi162 ceph-mon[34633]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T16:49:51.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:51 smithi162 ceph-mon[34633]: osdmap e16: 3 total, 2 up, 3 in 2024-08-17T16:49:51.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:51 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:51.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:51 smithi162 ceph-mon[34633]: from='osd.2 [v2:172.21.15.63:6818/3751946332,v1:172.21.15.63:6819/3751946332]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:49:51.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:51 smithi162 ceph-mon[34633]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:49:51.838 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 16:49:51 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2[59187]: 2024-08-17T16:49:51.566+0000 7f89eb8ef640 -1 osd.2 0 waiting for initial osdmap 2024-08-17T16:49:51.839 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 16:49:51 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2[59187]: 2024-08-17T16:49:51.572+0000 7f89e2ef9640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:49:52.457 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[31063]: purged_snaps scrub starts 2024-08-17T16:49:52.458 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[31063]: purged_snaps scrub ok 2024-08-17T16:49:52.458 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[31063]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-08-17T16:49:52.458 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[31063]: osdmap e17: 3 total, 2 up, 3 in 2024-08-17T16:49:52.458 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:52.458 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:52.458 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[31063]: from='osd.2 ' entity='osd.2' 2024-08-17T16:49:52.458 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[31063]: pgmap v73: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:52.458 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:52.458 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[42680]: purged_snaps scrub starts 2024-08-17T16:49:52.459 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[42680]: purged_snaps scrub ok 2024-08-17T16:49:52.459 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[42680]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-08-17T16:49:52.459 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[42680]: osdmap e17: 3 total, 2 up, 3 in 2024-08-17T16:49:52.459 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:52.459 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:52.459 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[42680]: from='osd.2 ' entity='osd.2' 2024-08-17T16:49:52.459 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[42680]: pgmap v73: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:52.459 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:52 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:52.497 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:52 smithi162 ceph-mon[34633]: purged_snaps scrub starts 2024-08-17T16:49:52.497 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:52 smithi162 ceph-mon[34633]: purged_snaps scrub ok 2024-08-17T16:49:52.497 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:52 smithi162 ceph-mon[34633]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-08-17T16:49:52.497 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:52 smithi162 ceph-mon[34633]: osdmap e17: 3 total, 2 up, 3 in 2024-08-17T16:49:52.497 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:52 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:52.497 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:52 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:52.498 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:52 smithi162 ceph-mon[34633]: from='osd.2 ' entity='osd.2' 2024-08-17T16:49:52.498 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:52 smithi162 ceph-mon[34633]: pgmap v73: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-08-17T16:49:52.498 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:52 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:53.172 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:49:53.197 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi063:vg_nvme/lv_1 2024-08-17T16:49:53.721 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:49:53.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:53 smithi063 ceph-mon[31063]: osd.2 [v2:172.21.15.63:6818/3751946332,v1:172.21.15.63:6819/3751946332] boot 2024-08-17T16:49:53.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:53 smithi063 ceph-mon[31063]: osdmap e18: 3 total, 3 up, 3 in 2024-08-17T16:49:53.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:53 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:53.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:53 smithi063 ceph-mon[42680]: osd.2 [v2:172.21.15.63:6818/3751946332,v1:172.21.15.63:6819/3751946332] boot 2024-08-17T16:49:53.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:53 smithi063 ceph-mon[42680]: osdmap e18: 3 total, 3 up, 3 in 2024-08-17T16:49:53.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:53 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:53.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:53 smithi162 ceph-mon[34633]: osd.2 [v2:172.21.15.63:6818/3751946332,v1:172.21.15.63:6819/3751946332] boot 2024-08-17T16:49:53.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:53 smithi162 ceph-mon[34633]: osdmap e18: 3 total, 3 up, 3 in 2024-08-17T16:49:53.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:53 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:49:54.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:54 smithi063 ceph-mon[42680]: osdmap e19: 3 total, 3 up, 3 in 2024-08-17T16:49:54.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:54 smithi063 ceph-mon[42680]: pgmap v76: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:49:54.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:54 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:49:54.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:54 smithi063 ceph-mon[31063]: osdmap e19: 3 total, 3 up, 3 in 2024-08-17T16:49:54.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:54 smithi063 ceph-mon[31063]: pgmap v76: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:49:54.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:54 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:49:54.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:54 smithi162 ceph-mon[34633]: osdmap e19: 3 total, 3 up, 3 in 2024-08-17T16:49:54.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:54 smithi162 ceph-mon[34633]: pgmap v76: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:49:54.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:54 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:49:55.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-08-17T16:49:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: osdmap e20: 3 total, 3 up, 3 in 2024-08-17T16:49:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:49:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:55.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:49:55.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:55.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:55.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:55.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-08-17T16:49:55.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: osdmap e20: 3 total, 3 up, 3 in 2024-08-17T16:49:55.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:49:55.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:55.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:55.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:55.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:55.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:55.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:55.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:55.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:55.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:55.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:49:55.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:55.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:55.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-08-17T16:49:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: osdmap e20: 3 total, 3 up, 3 in 2024-08-17T16:49:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:49:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:49:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:56.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:56.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:49:56.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:56.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:49:56.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:56.846 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[31063]: Detected new or changed devices on smithi063 2024-08-17T16:49:56.846 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[31063]: Adjusting osd_memory_target on smithi063 to 5354M 2024-08-17T16:49:56.846 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-17T16:49:56.846 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[31063]: osdmap e21: 3 total, 3 up, 3 in 2024-08-17T16:49:56.846 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[31063]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:49:56.847 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:49:56.847 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:49:56.847 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:56.847 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[42680]: Detected new or changed devices on smithi063 2024-08-17T16:49:56.847 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[42680]: Adjusting osd_memory_target on smithi063 to 5354M 2024-08-17T16:49:56.847 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-17T16:49:56.847 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[42680]: osdmap e21: 3 total, 3 up, 3 in 2024-08-17T16:49:56.847 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[42680]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:49:56.847 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:49:56.847 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:49:56.848 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:56 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:56.848 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 16:49:56 smithi063 sudo[63824]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T16:49:56.848 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 16:49:56 smithi063 sudo[63824]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:49:56.848 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 16:49:56 smithi063 sudo[63824]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:49:56.848 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 16:49:56 smithi063 sudo[63824]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:49:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:56 smithi162 ceph-mon[34633]: Detected new or changed devices on smithi063 2024-08-17T16:49:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:56 smithi162 ceph-mon[34633]: Adjusting osd_memory_target on smithi063 to 5354M 2024-08-17T16:49:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:56 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-08-17T16:49:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:56 smithi162 ceph-mon[34633]: osdmap e21: 3 total, 3 up, 3 in 2024-08-17T16:49:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:56 smithi162 ceph-mon[34633]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:49:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:56 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:49:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:56 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:49:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:56 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:49:57.279 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 16:49:57 smithi063 sudo[63863]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T16:49:57.279 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 16:49:57 smithi063 sudo[63863]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:49:57.279 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 16:49:57 smithi063 sudo[63863]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:49:57.279 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 16:49:57 smithi063 sudo[63863]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:49:57.575 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 16:49:57 smithi063 sudo[63875]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T16:49:57.575 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 16:49:57 smithi063 sudo[63875]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:49:57.575 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 16:49:57 smithi063 sudo[63875]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:49:57.575 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 16:49:57 smithi063 sudo[63875]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:49:57.828 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:57 smithi063 ceph-mon[31063]: from='client.24194 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:49:57.828 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:57 smithi063 ceph-mon[31063]: osdmap e22: 3 total, 3 up, 3 in 2024-08-17T16:49:57.829 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:57 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:57.829 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:57 smithi063 ceph-mon[42680]: from='client.24194 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:49:57.829 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:57 smithi063 ceph-mon[42680]: osdmap e22: 3 total, 3 up, 3 in 2024-08-17T16:49:57.829 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:57 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:57 smithi162 ceph-mon[34633]: from='client.24194 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi063:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:49:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:57 smithi162 ceph-mon[34633]: osdmap e22: 3 total, 3 up, 3 in 2024-08-17T16:49:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:57 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:49:58.169 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 16:49:57 smithi063 sudo[63920]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T16:49:58.169 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 16:49:57 smithi063 sudo[63920]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:49:58.169 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 16:49:57 smithi063 sudo[63920]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:49:58.169 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 16:49:57 smithi063 sudo[63920]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:49:58.529 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 16:49:58 smithi063 sudo[63968]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T16:49:58.529 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 16:49:58 smithi063 sudo[63968]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:49:58.529 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 16:49:58 smithi063 sudo[63968]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:49:58.530 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 16:49:58 smithi063 sudo[63968]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:49:59.029 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 16:49:58 smithi063 sudo[63986]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T16:49:59.029 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 16:49:58 smithi063 sudo[63986]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:49:59.029 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 16:49:58 smithi063 sudo[63986]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:49:59.029 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 16:49:58 smithi063 sudo[63986]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:49:59.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:58 smithi063 ceph-mon[31063]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:49:59.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:58 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3749426508' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "771f43d2-e98b-43e0-921b-88e56fadda42"}]: dispatch 2024-08-17T16:49:59.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:58 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3749426508' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "771f43d2-e98b-43e0-921b-88e56fadda42"}]': finished 2024-08-17T16:49:59.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:58 smithi063 ceph-mon[31063]: osdmap e23: 4 total, 3 up, 4 in 2024-08-17T16:49:59.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:58 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:49:59.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:58 smithi063 ceph-mon[42680]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:49:59.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:58 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3749426508' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "771f43d2-e98b-43e0-921b-88e56fadda42"}]: dispatch 2024-08-17T16:49:59.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:58 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3749426508' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "771f43d2-e98b-43e0-921b-88e56fadda42"}]': finished 2024-08-17T16:49:59.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:58 smithi063 ceph-mon[42680]: osdmap e23: 4 total, 3 up, 4 in 2024-08-17T16:49:59.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:58 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:49:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:58 smithi162 ceph-mon[34633]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:49:59.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:58 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3749426508' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "771f43d2-e98b-43e0-921b-88e56fadda42"}]: dispatch 2024-08-17T16:49:59.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:58 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3749426508' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "771f43d2-e98b-43e0-921b-88e56fadda42"}]': finished 2024-08-17T16:49:59.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:58 smithi162 ceph-mon[34633]: osdmap e23: 4 total, 3 up, 4 in 2024-08-17T16:49:59.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:58 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:49:59.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:59 smithi063 sudo[64054]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T16:49:59.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:59 smithi063 sudo[64054]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:49:59.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:59 smithi063 sudo[64054]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:49:59.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:59 smithi063 sudo[64054]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:50:00.009 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:59 smithi162 ceph-mon[34633]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-17T16:50:00.010 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:59 smithi162 sudo[37233]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T16:50:00.010 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:59 smithi162 sudo[37233]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:50:00.010 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:49:59 smithi162 sudo[37233]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:50:00.010 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:00 smithi162 sudo[37233]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:50:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:49:59 smithi063 ceph-mon[42680]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-17T16:50:00.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:59 smithi063 sudo[64241]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T16:50:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:59 smithi063 sudo[64241]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:50:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:59 smithi063 sudo[64241]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:50:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:59 smithi063 sudo[64241]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:50:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:49:59 smithi063 ceph-mon[31063]: mgrmap e15: y(active, since 2m), standbys: x 2024-08-17T16:50:00.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:00 smithi162 sudo[37236]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T16:50:00.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:00 smithi162 sudo[37236]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:50:00.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:00 smithi162 sudo[37236]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:50:00.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:00 smithi162 sudo[37236]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:50:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:00 smithi063 sudo[64616]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-08-17T16:50:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:00 smithi063 sudo[64616]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:50:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:00 smithi063 sudo[64616]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:50:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:00 smithi063 sudo[64616]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:50:01.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 sudo[64809]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-08-17T16:50:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 sudo[64809]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-08-17T16:50:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 sudo[64809]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-08-17T16:50:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 sudo[64809]: pam_unix(sudo:session): session closed for user root 2024-08-17T16:50:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:50:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/2210363681' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:50:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:50:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:50:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:50:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:50:01.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:50:01.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:50:01.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:50:01.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:50:01.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:50:01.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: overall HEALTH_OK 2024-08-17T16:50:01.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:50:01.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:01.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:50:01.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:50:01.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:50:01.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:50:01.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/2210363681' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:50:01.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:50:01.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:50:01.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:50:01.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:50:01.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:50:01.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:50:01.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:50:01.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:50:01.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:50:01.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: overall HEALTH_OK 2024-08-17T16:50:01.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:50:01.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:01.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:50:01.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:50:01.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:01 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:50:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:50:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/2210363681' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:50:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:50:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:50:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:50:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:50:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:50:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:50:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:50:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:50:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:50:01.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: overall HEALTH_OK 2024-08-17T16:50:01.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:50:01.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:01.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:50:01.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:50:01.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:01 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:50:02.503 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:02 smithi063 ceph-mon[42680]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:50:02.503 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:02 smithi063 ceph-mon[42680]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:50:02.503 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:02 smithi063 ceph-mon[42680]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:50:02.503 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:02 smithi063 ceph-mon[31063]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:50:02.503 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:02 smithi063 ceph-mon[31063]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:50:02.503 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:02 smithi063 ceph-mon[31063]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:50:02.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:02 smithi162 ceph-mon[34633]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-08-17T16:50:02.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:02 smithi162 ceph-mon[34633]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-08-17T16:50:02.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:02 smithi162 ceph-mon[34633]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:50:04.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:04 smithi063 ceph-mon[31063]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-08-17T16:50:04.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:04 smithi063 ceph-mon[42680]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-08-17T16:50:04.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:04 smithi162 ceph-mon[34633]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-08-17T16:50:05.100 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:05 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-17T16:50:05.100 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:05 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:05.100 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:05 smithi063 ceph-mon[42680]: Deploying daemon osd.3 on smithi063 2024-08-17T16:50:05.100 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:05 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-17T16:50:05.100 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:05 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:05.101 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:05 smithi063 ceph-mon[31063]: Deploying daemon osd.3 on smithi063 2024-08-17T16:50:05.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:05 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-08-17T16:50:05.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:05 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:05.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:05 smithi162 ceph-mon[34633]: Deploying daemon osd.3 on smithi063 2024-08-17T16:50:06.180 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:06 smithi063 ceph-mon[42680]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-08-17T16:50:06.180 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:06 smithi063 ceph-mon[31063]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-08-17T16:50:06.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:06 smithi162 ceph-mon[34633]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-08-17T16:50:08.240 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:08 smithi063 ceph-mon[42680]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-08-17T16:50:08.240 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:08 smithi063 ceph-mon[31063]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-08-17T16:50:08.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:08 smithi162 ceph-mon[34633]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-08-17T16:50:09.778 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:09 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:50:09.778 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:09 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:09.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:09 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:09.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:09 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:50:09.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:09 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:09.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:09 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:09 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:50:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:09 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:09 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:10.667 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:10 smithi063 ceph-mon[42680]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-08-17T16:50:10.667 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:10 smithi063 ceph-mon[31063]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-08-17T16:50:10.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:10 smithi162 ceph-mon[34633]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-08-17T16:50:12.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:12 smithi063 ceph-mon[31063]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-08-17T16:50:12.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:12 smithi063 ceph-mon[42680]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-08-17T16:50:12.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:12 smithi162 ceph-mon[34633]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-08-17T16:50:12.700 INFO:teuthology.orchestra.run.smithi063.stdout:Created osd(s) 3 on host 'smithi063' 2024-08-17T16:50:13.323 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:13.323 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:13.323 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:13.323 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:50:13.323 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:13.323 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[42680]: from='osd.3 [v2:172.21.15.63:6826/2654575936,v1:172.21.15.63:6827/2654575936]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T16:50:13.323 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[42680]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T16:50:13.323 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:50:13.323 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:13.323 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:13.324 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:13.324 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:13.324 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:13.324 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:50:13.324 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:13.324 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[31063]: from='osd.3 [v2:172.21.15.63:6826/2654575936,v1:172.21.15.63:6827/2654575936]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T16:50:13.324 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[31063]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T16:50:13.324 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:50:13.324 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:13.324 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:13 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:13.344 DEBUG:teuthology.orchestra.run.smithi063:osd.3> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.3.service 2024-08-17T16:50:13.347 INFO:tasks.cephadm:Deploying osd.4 on smithi162 with /dev/vg_nvme/lv_4... 2024-08-17T16:50:13.347 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-08-17T16:50:13.375 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:13 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:13.375 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:13 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:13.375 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:13 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:13.375 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:13 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:50:13.375 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:13 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:13.375 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:13 smithi162 ceph-mon[34633]: from='osd.3 [v2:172.21.15.63:6826/2654575936,v1:172.21.15.63:6827/2654575936]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T16:50:13.375 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:13 smithi162 ceph-mon[34633]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-08-17T16:50:13.375 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:13 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:50:13.376 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:13 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:13.376 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:13 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:13.555 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:50:14.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:14 smithi162 ceph-mon[34633]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T16:50:14.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:14 smithi162 ceph-mon[34633]: osdmap e24: 4 total, 3 up, 4 in 2024-08-17T16:50:14.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:14 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:50:14.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:14 smithi162 ceph-mon[34633]: from='osd.3 [v2:172.21.15.63:6826/2654575936,v1:172.21.15.63:6827/2654575936]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:50:14.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:14 smithi162 ceph-mon[34633]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:50:14.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:14 smithi162 ceph-mon[34633]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:50:14.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:14 smithi063 ceph-mon[42680]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T16:50:14.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:14 smithi063 ceph-mon[42680]: osdmap e24: 4 total, 3 up, 4 in 2024-08-17T16:50:14.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:14 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:50:14.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:14 smithi063 ceph-mon[42680]: from='osd.3 [v2:172.21.15.63:6826/2654575936,v1:172.21.15.63:6827/2654575936]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:50:14.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:14 smithi063 ceph-mon[42680]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:50:14.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:14 smithi063 ceph-mon[42680]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:50:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:14 smithi063 ceph-mon[31063]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T16:50:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:14 smithi063 ceph-mon[31063]: osdmap e24: 4 total, 3 up, 4 in 2024-08-17T16:50:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:14 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:50:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:14 smithi063 ceph-mon[31063]: from='osd.3 [v2:172.21.15.63:6826/2654575936,v1:172.21.15.63:6827/2654575936]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:50:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:14 smithi063 ceph-mon[31063]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]: dispatch 2024-08-17T16:50:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:14 smithi063 ceph-mon[31063]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:50:14.852 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 16:50:14 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3[65761]: 2024-08-17T16:50:14.577+0000 7f64d14af640 -1 osd.3 0 waiting for initial osdmap 2024-08-17T16:50:14.852 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 16:50:14 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3[65761]: 2024-08-17T16:50:14.584+0000 7f64c92d0640 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:50:15.325 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:15 smithi162 ceph-mon[34633]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-08-17T16:50:15.325 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:15 smithi162 ceph-mon[34633]: osdmap e25: 4 total, 3 up, 4 in 2024-08-17T16:50:15.325 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:15 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:50:15.325 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:15 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:50:15.326 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:15 smithi162 ceph-mon[34633]: from='osd.3 ' entity='osd.3' 2024-08-17T16:50:15.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:15 smithi063 ceph-mon[31063]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-08-17T16:50:15.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:15 smithi063 ceph-mon[31063]: osdmap e25: 4 total, 3 up, 4 in 2024-08-17T16:50:15.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:15 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:50:15.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:15 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:50:15.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:15 smithi063 ceph-mon[31063]: from='osd.3 ' entity='osd.3' 2024-08-17T16:50:15.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:15 smithi063 ceph-mon[42680]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]}]': finished 2024-08-17T16:50:15.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:15 smithi063 ceph-mon[42680]: osdmap e25: 4 total, 3 up, 4 in 2024-08-17T16:50:15.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:15 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:50:15.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:15 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:50:15.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:15 smithi063 ceph-mon[42680]: from='osd.3 ' entity='osd.3' 2024-08-17T16:50:15.861 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:50:15.880 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi162:vg_nvme/lv_4 2024-08-17T16:50:16.120 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:50:16.242 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:16 smithi162 ceph-mon[34633]: purged_snaps scrub starts 2024-08-17T16:50:16.242 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:16 smithi162 ceph-mon[34633]: purged_snaps scrub ok 2024-08-17T16:50:16.242 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:16 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:50:16.242 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:16 smithi162 ceph-mon[34633]: osd.3 [v2:172.21.15.63:6826/2654575936,v1:172.21.15.63:6827/2654575936] boot 2024-08-17T16:50:16.243 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:16 smithi162 ceph-mon[34633]: osdmap e26: 4 total, 4 up, 4 in 2024-08-17T16:50:16.243 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:16 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:50:16.243 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:16 smithi162 ceph-mon[34633]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:50:16.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:16 smithi063 ceph-mon[31063]: purged_snaps scrub starts 2024-08-17T16:50:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:16 smithi063 ceph-mon[31063]: purged_snaps scrub ok 2024-08-17T16:50:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:16 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:50:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:16 smithi063 ceph-mon[31063]: osd.3 [v2:172.21.15.63:6826/2654575936,v1:172.21.15.63:6827/2654575936] boot 2024-08-17T16:50:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:16 smithi063 ceph-mon[31063]: osdmap e26: 4 total, 4 up, 4 in 2024-08-17T16:50:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:16 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:50:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:16 smithi063 ceph-mon[31063]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:50:16.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:16 smithi063 ceph-mon[42680]: purged_snaps scrub starts 2024-08-17T16:50:16.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:16 smithi063 ceph-mon[42680]: purged_snaps scrub ok 2024-08-17T16:50:16.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:16 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:50:16.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:16 smithi063 ceph-mon[42680]: osd.3 [v2:172.21.15.63:6826/2654575936,v1:172.21.15.63:6827/2654575936] boot 2024-08-17T16:50:16.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:16 smithi063 ceph-mon[42680]: osdmap e26: 4 total, 4 up, 4 in 2024-08-17T16:50:16.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:16 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:50:16.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:16 smithi063 ceph-mon[42680]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-08-17T16:50:17.299 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:17 smithi162 ceph-mon[34633]: Detected new or changed devices on smithi063 2024-08-17T16:50:17.299 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:17 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:17.299 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:17 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:17.299 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:17 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:50:17.299 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:17 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:50:17.299 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:17 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:50:17.300 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:17 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:50:17.300 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:17 smithi162 ceph-mon[34633]: Adjusting osd_memory_target on smithi063 to 4016M 2024-08-17T16:50:17.300 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:17 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:17.300 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:17 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:17.300 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:17 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:50:17.300 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:17 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:17.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[42680]: Detected new or changed devices on smithi063 2024-08-17T16:50:17.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:17.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:17.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:50:17.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:50:17.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:50:17.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:50:17.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[42680]: Adjusting osd_memory_target on smithi063 to 4016M 2024-08-17T16:50:17.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:17.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:17.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:50:17.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:17.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[31063]: Detected new or changed devices on smithi063 2024-08-17T16:50:17.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:17.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:17.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:50:17.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:50:17.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:50:17.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:50:17.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[31063]: Adjusting osd_memory_target on smithi063 to 4016M 2024-08-17T16:50:17.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:17.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:17.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:50:17.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:17 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:18.765 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:18 smithi162 ceph-mon[34633]: osdmap e27: 4 total, 4 up, 4 in 2024-08-17T16:50:18.765 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:18 smithi162 ceph-mon[34633]: from='client.24196 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi162:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:50:18.765 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:18 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:50:18.765 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:18 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:50:18.765 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:18 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:18.766 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:18 smithi162 ceph-mon[34633]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:19.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:18 smithi063 ceph-mon[31063]: osdmap e27: 4 total, 4 up, 4 in 2024-08-17T16:50:19.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:18 smithi063 ceph-mon[31063]: from='client.24196 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi162:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:50:19.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:18 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:50:19.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:18 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:50:19.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:18 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:19.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:18 smithi063 ceph-mon[31063]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:19.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:18 smithi063 ceph-mon[42680]: osdmap e27: 4 total, 4 up, 4 in 2024-08-17T16:50:19.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:18 smithi063 ceph-mon[42680]: from='client.24196 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi162:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:50:19.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:18 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:50:19.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:18 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:50:19.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:18 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:19.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:18 smithi063 ceph-mon[42680]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:20.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:20 smithi063 ceph-mon[31063]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:20.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:20 smithi063 ceph-mon[31063]: from='client.? 172.21.15.162:0/1766681612' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73c3c545-e975-4a6f-a1cb-e4575ee7a9b0"}]: dispatch 2024-08-17T16:50:20.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:20 smithi063 ceph-mon[31063]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73c3c545-e975-4a6f-a1cb-e4575ee7a9b0"}]: dispatch 2024-08-17T16:50:20.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:20 smithi063 ceph-mon[31063]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "73c3c545-e975-4a6f-a1cb-e4575ee7a9b0"}]': finished 2024-08-17T16:50:20.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:20 smithi063 ceph-mon[31063]: osdmap e28: 5 total, 4 up, 5 in 2024-08-17T16:50:20.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:20 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:50:20.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:20 smithi063 ceph-mon[42680]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:20.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:20 smithi063 ceph-mon[42680]: from='client.? 172.21.15.162:0/1766681612' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73c3c545-e975-4a6f-a1cb-e4575ee7a9b0"}]: dispatch 2024-08-17T16:50:20.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:20 smithi063 ceph-mon[42680]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73c3c545-e975-4a6f-a1cb-e4575ee7a9b0"}]: dispatch 2024-08-17T16:50:20.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:20 smithi063 ceph-mon[42680]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "73c3c545-e975-4a6f-a1cb-e4575ee7a9b0"}]': finished 2024-08-17T16:50:20.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:20 smithi063 ceph-mon[42680]: osdmap e28: 5 total, 4 up, 5 in 2024-08-17T16:50:20.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:20 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:50:20.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:20 smithi162 ceph-mon[34633]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:20.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:20 smithi162 ceph-mon[34633]: from='client.? 172.21.15.162:0/1766681612' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73c3c545-e975-4a6f-a1cb-e4575ee7a9b0"}]: dispatch 2024-08-17T16:50:20.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:20 smithi162 ceph-mon[34633]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "73c3c545-e975-4a6f-a1cb-e4575ee7a9b0"}]: dispatch 2024-08-17T16:50:20.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:20 smithi162 ceph-mon[34633]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "73c3c545-e975-4a6f-a1cb-e4575ee7a9b0"}]': finished 2024-08-17T16:50:20.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:20 smithi162 ceph-mon[34633]: osdmap e28: 5 total, 4 up, 5 in 2024-08-17T16:50:20.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:20 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:50:21.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:21 smithi063 ceph-mon[42680]: from='client.? 172.21.15.162:0/3740518661' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:50:21.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:21 smithi063 ceph-mon[31063]: from='client.? 172.21.15.162:0/3740518661' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:50:21.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:21 smithi162 ceph-mon[34633]: from='client.? 172.21.15.162:0/3740518661' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:50:22.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:22 smithi162 ceph-mon[34633]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:22.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:22 smithi063 ceph-mon[42680]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:22.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:22 smithi063 ceph-mon[31063]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:24 smithi162 ceph-mon[34633]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:24.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:24 smithi063 ceph-mon[42680]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:24.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:24 smithi063 ceph-mon[31063]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:26.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:26 smithi063 ceph-mon[42680]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:26.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:26 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-17T16:50:26.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:26 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:26.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:26 smithi063 ceph-mon[42680]: Deploying daemon osd.4 on smithi162 2024-08-17T16:50:26.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:26 smithi063 ceph-mon[31063]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:26.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:26 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-17T16:50:26.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:26 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:26.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:26 smithi063 ceph-mon[31063]: Deploying daemon osd.4 on smithi162 2024-08-17T16:50:26.303 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:26 smithi162 ceph-mon[34633]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:26.303 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:26 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-08-17T16:50:26.303 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:26 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:26.303 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:26 smithi162 ceph-mon[34633]: Deploying daemon osd.4 on smithi162 2024-08-17T16:50:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:28 smithi162 ceph-mon[34633]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:28.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:28 smithi063 ceph-mon[42680]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:28.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:28 smithi063 ceph-mon[31063]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:30.165 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:30 smithi162 ceph-mon[34633]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:30.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:30 smithi063 ceph-mon[31063]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:30.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:30 smithi063 ceph-mon[42680]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:31.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:31 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:50:31.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:31 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:31.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:31 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:31.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:31 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:50:31.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:31 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:31.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:31 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:31.281 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:31 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:50:31.281 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:31 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:31.281 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:31 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:32 smithi162 ceph-mon[34633]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:32.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:32 smithi063 ceph-mon[42680]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:32.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:32 smithi063 ceph-mon[31063]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:33 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:33 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:34.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:33 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:34.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:33 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:50:34.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:33 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:34.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:33 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:34.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:33 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:34.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:33 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:34.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:33 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:50:34.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:33 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:34.065 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:33 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:34.066 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:33 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:34.066 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:33 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:34.066 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:33 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:50:34.066 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:33 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:34.100 INFO:teuthology.orchestra.run.smithi162.stdout:Created osd(s) 4 on host 'smithi162' 2024-08-17T16:50:34.721 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:34 smithi162 ceph-mon[34633]: from='osd.4 [v2:172.21.15.162:6800/1563885208,v1:172.21.15.162:6801/1563885208]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T16:50:34.721 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:34 smithi162 ceph-mon[34633]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T16:50:34.721 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:34 smithi162 ceph-mon[34633]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:34.721 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:34 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:50:34.722 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:34 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:34.722 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:34 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:34.742 DEBUG:teuthology.orchestra.run.smithi162:osd.4> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.4.service 2024-08-17T16:50:34.744 INFO:tasks.cephadm:Deploying osd.5 on smithi162 with /dev/vg_nvme/lv_3... 2024-08-17T16:50:34.744 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-08-17T16:50:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:34 smithi063 ceph-mon[31063]: from='osd.4 [v2:172.21.15.162:6800/1563885208,v1:172.21.15.162:6801/1563885208]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T16:50:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:34 smithi063 ceph-mon[31063]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T16:50:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:34 smithi063 ceph-mon[31063]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:34 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:50:35.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:34 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:35.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:34 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:35.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:34 smithi063 ceph-mon[42680]: from='osd.4 [v2:172.21.15.162:6800/1563885208,v1:172.21.15.162:6801/1563885208]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T16:50:35.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:34 smithi063 ceph-mon[42680]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-08-17T16:50:35.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:34 smithi063 ceph-mon[42680]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:35.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:34 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:50:35.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:34 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:35.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:34 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:35.489 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:50:35.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:35 smithi162 ceph-mon[34633]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-17T16:50:35.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:35 smithi162 ceph-mon[34633]: osdmap e29: 5 total, 4 up, 5 in 2024-08-17T16:50:35.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:35 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:50:35.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:35 smithi162 ceph-mon[34633]: from='osd.4 [v2:172.21.15.162:6800/1563885208,v1:172.21.15.162:6801/1563885208]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:50:35.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:35 smithi162 ceph-mon[34633]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:50:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:35 smithi063 ceph-mon[31063]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-17T16:50:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:35 smithi063 ceph-mon[31063]: osdmap e29: 5 total, 4 up, 5 in 2024-08-17T16:50:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:35 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:50:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:35 smithi063 ceph-mon[31063]: from='osd.4 [v2:172.21.15.162:6800/1563885208,v1:172.21.15.162:6801/1563885208]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:50:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:35 smithi063 ceph-mon[31063]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:50:36.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:35 smithi063 ceph-mon[42680]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-08-17T16:50:36.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:35 smithi063 ceph-mon[42680]: osdmap e29: 5 total, 4 up, 5 in 2024-08-17T16:50:36.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:35 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:50:36.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:35 smithi063 ceph-mon[42680]: from='osd.4 [v2:172.21.15.162:6800/1563885208,v1:172.21.15.162:6801/1563885208]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:50:36.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:35 smithi063 ceph-mon[42680]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:50:36.478 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 16:50:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T16:50:36.139+0000 7f26d1896640 -1 osd.4 0 waiting for initial osdmap 2024-08-17T16:50:36.479 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 16:50:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T16:50:36.143+0000 7f26c96b7640 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:50:36.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:36 smithi162 ceph-mon[34633]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]': finished 2024-08-17T16:50:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:36 smithi162 ceph-mon[34633]: osdmap e30: 5 total, 4 up, 5 in 2024-08-17T16:50:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:36 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:50:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:36 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:50:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:36 smithi162 ceph-mon[34633]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:36 smithi162 ceph-mon[34633]: from='osd.4 ' entity='osd.4' 2024-08-17T16:50:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:36 smithi162 ceph-mon[34633]: osd.4 [v2:172.21.15.162:6800/1563885208,v1:172.21.15.162:6801/1563885208] boot 2024-08-17T16:50:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:36 smithi162 ceph-mon[34633]: osdmap e31: 5 total, 5 up, 5 in 2024-08-17T16:50:36.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:36 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:50:37.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[31063]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]': finished 2024-08-17T16:50:37.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[31063]: osdmap e30: 5 total, 4 up, 5 in 2024-08-17T16:50:37.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:50:37.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:50:37.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[31063]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:37.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[31063]: from='osd.4 ' entity='osd.4' 2024-08-17T16:50:37.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[31063]: osd.4 [v2:172.21.15.162:6800/1563885208,v1:172.21.15.162:6801/1563885208] boot 2024-08-17T16:50:37.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[31063]: osdmap e31: 5 total, 5 up, 5 in 2024-08-17T16:50:37.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:50:37.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[42680]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]': finished 2024-08-17T16:50:37.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[42680]: osdmap e30: 5 total, 4 up, 5 in 2024-08-17T16:50:37.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:50:37.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:50:37.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[42680]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-08-17T16:50:37.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[42680]: from='osd.4 ' entity='osd.4' 2024-08-17T16:50:37.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[42680]: osd.4 [v2:172.21.15.162:6800/1563885208,v1:172.21.15.162:6801/1563885208] boot 2024-08-17T16:50:37.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[42680]: osdmap e31: 5 total, 5 up, 5 in 2024-08-17T16:50:37.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:36 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:50:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:37 smithi063 ceph-mon[31063]: purged_snaps scrub starts 2024-08-17T16:50:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:37 smithi063 ceph-mon[31063]: purged_snaps scrub ok 2024-08-17T16:50:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:37 smithi063 ceph-mon[31063]: osdmap e32: 5 total, 5 up, 5 in 2024-08-17T16:50:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:37 smithi063 ceph-mon[42680]: purged_snaps scrub starts 2024-08-17T16:50:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:37 smithi063 ceph-mon[42680]: purged_snaps scrub ok 2024-08-17T16:50:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:37 smithi063 ceph-mon[42680]: osdmap e32: 5 total, 5 up, 5 in 2024-08-17T16:50:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:37 smithi162 ceph-mon[34633]: purged_snaps scrub starts 2024-08-17T16:50:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:37 smithi162 ceph-mon[34633]: purged_snaps scrub ok 2024-08-17T16:50:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:37 smithi162 ceph-mon[34633]: osdmap e32: 5 total, 5 up, 5 in 2024-08-17T16:50:38.757 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:38 smithi162 ceph-mon[34633]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 407 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:38.758 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:38 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:38.758 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:38 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:38.758 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:38 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:50:38.758 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:38 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:38.758 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:38 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:38.758 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:38 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:50:38.758 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:38 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:38.758 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:38 smithi162 ceph-mon[34633]: osdmap e33: 5 total, 5 up, 5 in 2024-08-17T16:50:39.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[31063]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 407 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:50:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:50:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[31063]: osdmap e33: 5 total, 5 up, 5 in 2024-08-17T16:50:39.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[42680]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 407 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:39.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:39.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:39.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:50:39.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:39.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:39.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:50:39.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:39.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:38 smithi063 ceph-mon[42680]: osdmap e33: 5 total, 5 up, 5 in 2024-08-17T16:50:39.811 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:39 smithi162 ceph-mon[34633]: Detected new or changed devices on smithi162 2024-08-17T16:50:39.812 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:39 smithi162 ceph-mon[34633]: Adjusting osd_memory_target on smithi162 to 17088M 2024-08-17T16:50:39.814 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:50:39.833 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi162:vg_nvme/lv_3 2024-08-17T16:50:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:39 smithi063 ceph-mon[42680]: Detected new or changed devices on smithi162 2024-08-17T16:50:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:39 smithi063 ceph-mon[42680]: Adjusting osd_memory_target on smithi162 to 17088M 2024-08-17T16:50:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:39 smithi063 ceph-mon[31063]: Detected new or changed devices on smithi162 2024-08-17T16:50:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:39 smithi063 ceph-mon[31063]: Adjusting osd_memory_target on smithi162 to 17088M 2024-08-17T16:50:40.036 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:50:40.984 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:40 smithi162 ceph-mon[34633]: osdmap e34: 5 total, 5 up, 5 in 2024-08-17T16:50:40.984 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:40 smithi162 ceph-mon[34633]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 407 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:40.984 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:40 smithi162 ceph-mon[34633]: osdmap e35: 5 total, 5 up, 5 in 2024-08-17T16:50:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:40 smithi063 ceph-mon[42680]: osdmap e34: 5 total, 5 up, 5 in 2024-08-17T16:50:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:40 smithi063 ceph-mon[42680]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 407 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:40 smithi063 ceph-mon[42680]: osdmap e35: 5 total, 5 up, 5 in 2024-08-17T16:50:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:40 smithi063 ceph-mon[31063]: osdmap e34: 5 total, 5 up, 5 in 2024-08-17T16:50:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:40 smithi063 ceph-mon[31063]: pgmap v114: 1 pgs: 1 remapped+peering; 577 KiB data, 407 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:40 smithi063 ceph-mon[31063]: osdmap e35: 5 total, 5 up, 5 in 2024-08-17T16:50:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:41 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:50:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:41 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:50:42.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:41 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:41 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:50:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:41 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:50:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:41 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:42.072 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:41 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:50:42.072 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:41 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:50:42.072 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:41 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:42.909 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:42 smithi162 ceph-mon[34633]: from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi162:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:50:42.909 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:42 smithi162 ceph-mon[34633]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 136 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:42 smithi063 ceph-mon[31063]: from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi162:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:50:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:42 smithi063 ceph-mon[31063]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 136 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:42 smithi063 ceph-mon[42680]: from='client.24223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi162:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:50:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:42 smithi063 ceph-mon[42680]: pgmap v116: 1 pgs: 1 remapped+peering; 577 KiB data, 136 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:44.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:43 smithi063 ceph-mon[31063]: from='client.? 172.21.15.162:0/2307845117' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fae3ac52-3194-4eb4-9510-e3899768353a"}]: dispatch 2024-08-17T16:50:44.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:43 smithi063 ceph-mon[31063]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fae3ac52-3194-4eb4-9510-e3899768353a"}]: dispatch 2024-08-17T16:50:44.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:43 smithi063 ceph-mon[31063]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fae3ac52-3194-4eb4-9510-e3899768353a"}]': finished 2024-08-17T16:50:44.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:43 smithi063 ceph-mon[31063]: osdmap e36: 6 total, 5 up, 6 in 2024-08-17T16:50:44.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:43 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:50:44.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:43 smithi063 ceph-mon[42680]: from='client.? 172.21.15.162:0/2307845117' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fae3ac52-3194-4eb4-9510-e3899768353a"}]: dispatch 2024-08-17T16:50:44.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:43 smithi063 ceph-mon[42680]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fae3ac52-3194-4eb4-9510-e3899768353a"}]: dispatch 2024-08-17T16:50:44.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:43 smithi063 ceph-mon[42680]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fae3ac52-3194-4eb4-9510-e3899768353a"}]': finished 2024-08-17T16:50:44.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:43 smithi063 ceph-mon[42680]: osdmap e36: 6 total, 5 up, 6 in 2024-08-17T16:50:44.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:43 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:50:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:43 smithi162 ceph-mon[34633]: from='client.? 172.21.15.162:0/2307845117' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fae3ac52-3194-4eb4-9510-e3899768353a"}]: dispatch 2024-08-17T16:50:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:43 smithi162 ceph-mon[34633]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fae3ac52-3194-4eb4-9510-e3899768353a"}]: dispatch 2024-08-17T16:50:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:43 smithi162 ceph-mon[34633]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fae3ac52-3194-4eb4-9510-e3899768353a"}]': finished 2024-08-17T16:50:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:43 smithi162 ceph-mon[34633]: osdmap e36: 6 total, 5 up, 6 in 2024-08-17T16:50:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:43 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:50:45.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:44 smithi063 ceph-mon[31063]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 110 KiB/s, 0 objects/s recovering 2024-08-17T16:50:45.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:44 smithi063 ceph-mon[31063]: from='client.? 172.21.15.162:0/517554679' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:50:45.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:44 smithi063 ceph-mon[42680]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 110 KiB/s, 0 objects/s recovering 2024-08-17T16:50:45.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:44 smithi063 ceph-mon[42680]: from='client.? 172.21.15.162:0/517554679' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:50:45.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:44 smithi162 ceph-mon[34633]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 110 KiB/s, 0 objects/s recovering 2024-08-17T16:50:45.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:44 smithi162 ceph-mon[34633]: from='client.? 172.21.15.162:0/517554679' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:50:46.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:46 smithi162 ceph-mon[34633]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-08-17T16:50:46.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:46 smithi063 ceph-mon[31063]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-08-17T16:50:46.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:46 smithi063 ceph-mon[42680]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 92 KiB/s, 0 objects/s recovering 2024-08-17T16:50:48.239 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:48 smithi162 ceph-mon[34633]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T16:50:48.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:48 smithi063 ceph-mon[31063]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T16:50:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:48 smithi063 ceph-mon[42680]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T16:50:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-17T16:50:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-17T16:50:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:50.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-08-17T16:50:50.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:50:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:50 smithi063 ceph-mon[31063]: Deploying daemon osd.5 on smithi162 2024-08-17T16:50:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:50 smithi063 ceph-mon[31063]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-08-17T16:50:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:50 smithi063 ceph-mon[42680]: Deploying daemon osd.5 on smithi162 2024-08-17T16:50:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:50 smithi063 ceph-mon[42680]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-08-17T16:50:51.081 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:50 smithi162 ceph-mon[34633]: Deploying daemon osd.5 on smithi162 2024-08-17T16:50:51.081 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:50 smithi162 ceph-mon[34633]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-08-17T16:50:52.278 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:52 smithi162 ceph-mon[34633]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T16:50:52.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:52 smithi063 ceph-mon[31063]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T16:50:52.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:52 smithi063 ceph-mon[42680]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T16:50:54.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:54 smithi063 ceph-mon[31063]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:54.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:54 smithi063 ceph-mon[42680]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:54.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:54 smithi162 ceph-mon[34633]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:55.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:50:55.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:55.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:55 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:55.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:50:55.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:55.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:55 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:55.321 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:50:55.321 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:55.321 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:55 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:56.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:56 smithi162 ceph-mon[34633]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:56.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:56 smithi063 ceph-mon[31063]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:56.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:56 smithi063 ceph-mon[42680]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:57.733 INFO:teuthology.orchestra.run.smithi162.stdout:Created osd(s) 5 on host 'smithi162' 2024-08-17T16:50:58.326 DEBUG:teuthology.orchestra.run.smithi162:osd.5> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.5.service 2024-08-17T16:50:58.328 INFO:tasks.cephadm:Deploying osd.6 on smithi162 with /dev/vg_nvme/lv_2... 2024-08-17T16:50:58.329 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-08-17T16:50:59.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:58 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:59.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:58 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:59.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:58 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:59.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:58 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:59.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:58 smithi063 ceph-mon[31063]: from='osd.5 [v2:172.21.15.162:6808/1826502179,v1:172.21.15.162:6809/1826502179]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T16:50:59.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:58 smithi063 ceph-mon[31063]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T16:50:59.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:58 smithi063 ceph-mon[31063]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:59.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:58 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:59.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:58 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:59.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:58 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:59.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:58 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:59.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:58 smithi063 ceph-mon[42680]: from='osd.5 [v2:172.21.15.162:6808/1826502179,v1:172.21.15.162:6809/1826502179]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T16:50:59.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:58 smithi063 ceph-mon[42680]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T16:50:59.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:58 smithi063 ceph-mon[42680]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:59.072 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:50:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:58 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:59.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:58 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:59.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:58 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:59.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:58 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:50:59.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:58 smithi162 ceph-mon[34633]: from='osd.5 [v2:172.21.15.162:6808/1826502179,v1:172.21.15.162:6809/1826502179]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T16:50:59.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:58 smithi162 ceph-mon[34633]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-08-17T16:50:59.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:58 smithi162 ceph-mon[34633]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:50:59.978 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:59 smithi162 ceph-mon[34633]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T16:50:59.979 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:59 smithi162 ceph-mon[34633]: osdmap e37: 6 total, 5 up, 6 in 2024-08-17T16:50:59.979 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:59 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:50:59.979 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:59 smithi162 ceph-mon[34633]: from='osd.5 [v2:172.21.15.162:6808/1826502179,v1:172.21.15.162:6809/1826502179]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:50:59.979 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:50:59 smithi162 ceph-mon[34633]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:00.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:59 smithi063 ceph-mon[31063]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T16:51:00.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:59 smithi063 ceph-mon[31063]: osdmap e37: 6 total, 5 up, 6 in 2024-08-17T16:51:00.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:59 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:51:00.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:59 smithi063 ceph-mon[31063]: from='osd.5 [v2:172.21.15.162:6808/1826502179,v1:172.21.15.162:6809/1826502179]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:00.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:50:59 smithi063 ceph-mon[31063]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:00.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:59 smithi063 ceph-mon[42680]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-08-17T16:51:00.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:59 smithi063 ceph-mon[42680]: osdmap e37: 6 total, 5 up, 6 in 2024-08-17T16:51:00.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:59 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:51:00.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:59 smithi063 ceph-mon[42680]: from='osd.5 [v2:172.21.15.162:6808/1826502179,v1:172.21.15.162:6809/1826502179]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:00.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:50:59 smithi063 ceph-mon[42680]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:00.566 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 16:51:00 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T16:51:00.234+0000 7f3e73341640 -1 osd.5 0 waiting for initial osdmap 2024-08-17T16:51:00.567 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 16:51:00 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T16:51:00.240+0000 7f3e6a94b640 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:51:01.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:00 smithi162 ceph-mon[34633]: purged_snaps scrub starts 2024-08-17T16:51:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:00 smithi162 ceph-mon[34633]: purged_snaps scrub ok 2024-08-17T16:51:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:00 smithi162 ceph-mon[34633]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]': finished 2024-08-17T16:51:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:00 smithi162 ceph-mon[34633]: osdmap e38: 6 total, 5 up, 6 in 2024-08-17T16:51:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:00 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:51:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:00 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:51:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:00 smithi162 ceph-mon[34633]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:51:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:00 smithi162 ceph-mon[34633]: from='osd.5 ' entity='osd.5' 2024-08-17T16:51:01.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[31063]: purged_snaps scrub starts 2024-08-17T16:51:01.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[31063]: purged_snaps scrub ok 2024-08-17T16:51:01.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[31063]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]': finished 2024-08-17T16:51:01.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[31063]: osdmap e38: 6 total, 5 up, 6 in 2024-08-17T16:51:01.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:51:01.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:51:01.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[31063]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:51:01.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[31063]: from='osd.5 ' entity='osd.5' 2024-08-17T16:51:01.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[42680]: purged_snaps scrub starts 2024-08-17T16:51:01.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[42680]: purged_snaps scrub ok 2024-08-17T16:51:01.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[42680]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]': finished 2024-08-17T16:51:01.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[42680]: osdmap e38: 6 total, 5 up, 6 in 2024-08-17T16:51:01.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:51:01.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:51:01.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[42680]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-08-17T16:51:01.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:00 smithi063 ceph-mon[42680]: from='osd.5 ' entity='osd.5' 2024-08-17T16:51:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:01 smithi162 ceph-mon[34633]: osd.5 [v2:172.21.15.162:6808/1826502179,v1:172.21.15.162:6809/1826502179] boot 2024-08-17T16:51:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:01 smithi162 ceph-mon[34633]: osdmap e39: 6 total, 6 up, 6 in 2024-08-17T16:51:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:01 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:51:02.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:01 smithi063 ceph-mon[31063]: osd.5 [v2:172.21.15.162:6808/1826502179,v1:172.21.15.162:6809/1826502179] boot 2024-08-17T16:51:02.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:01 smithi063 ceph-mon[31063]: osdmap e39: 6 total, 6 up, 6 in 2024-08-17T16:51:02.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:01 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:51:02.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:01 smithi063 ceph-mon[42680]: osd.5 [v2:172.21.15.162:6808/1826502179,v1:172.21.15.162:6809/1826502179] boot 2024-08-17T16:51:02.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:01 smithi063 ceph-mon[42680]: osdmap e39: 6 total, 6 up, 6 in 2024-08-17T16:51:02.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:01 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:51:03.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:02 smithi162 ceph-mon[34633]: osdmap e40: 6 total, 6 up, 6 in 2024-08-17T16:51:03.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:02 smithi162 ceph-mon[34633]: pgmap v131: 1 pgs: 1 unknown; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:03.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:03.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:03.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:03.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:03.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:03.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:03.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:03.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:03.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:03.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:03.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:03.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:02 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:03.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[31063]: osdmap e40: 6 total, 6 up, 6 in 2024-08-17T16:51:03.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[31063]: pgmap v131: 1 pgs: 1 unknown; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:03.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:03.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:03.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:03.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:03.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:03.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:03.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:03.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:03.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:03.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:03.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:03.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:03.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[42680]: osdmap e40: 6 total, 6 up, 6 in 2024-08-17T16:51:03.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[42680]: pgmap v131: 1 pgs: 1 unknown; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:03.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:03.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:03.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:03.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:03.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:03.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:03.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:03.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:03.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:03.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:03.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:03.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:02 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:03.564 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:51:03.584 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi162:vg_nvme/lv_2 2024-08-17T16:51:03.787 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:51:03.909 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:03 smithi162 ceph-mon[34633]: Detected new or changed devices on smithi162 2024-08-17T16:51:03.909 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:03 smithi162 ceph-mon[34633]: Adjusting osd_memory_target on smithi162 to 8544M 2024-08-17T16:51:03.909 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:03 smithi162 ceph-mon[34633]: osdmap e41: 6 total, 6 up, 6 in 2024-08-17T16:51:04.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:03 smithi063 ceph-mon[31063]: Detected new or changed devices on smithi162 2024-08-17T16:51:04.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:03 smithi063 ceph-mon[31063]: Adjusting osd_memory_target on smithi162 to 8544M 2024-08-17T16:51:04.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:03 smithi063 ceph-mon[31063]: osdmap e41: 6 total, 6 up, 6 in 2024-08-17T16:51:04.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:03 smithi063 ceph-mon[42680]: Detected new or changed devices on smithi162 2024-08-17T16:51:04.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:03 smithi063 ceph-mon[42680]: Adjusting osd_memory_target on smithi162 to 8544M 2024-08-17T16:51:04.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:03 smithi063 ceph-mon[42680]: osdmap e41: 6 total, 6 up, 6 in 2024-08-17T16:51:05.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:04 smithi162 ceph-mon[34633]: osdmap e42: 6 total, 6 up, 6 in 2024-08-17T16:51:05.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:04 smithi162 ceph-mon[34633]: pgmap v134: 1 pgs: 1 unknown; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:05.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:04 smithi063 ceph-mon[31063]: osdmap e42: 6 total, 6 up, 6 in 2024-08-17T16:51:05.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:04 smithi063 ceph-mon[31063]: pgmap v134: 1 pgs: 1 unknown; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:05.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:04 smithi063 ceph-mon[42680]: osdmap e42: 6 total, 6 up, 6 in 2024-08-17T16:51:05.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:04 smithi063 ceph-mon[42680]: pgmap v134: 1 pgs: 1 unknown; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:05 smithi162 ceph-mon[34633]: osdmap e43: 6 total, 6 up, 6 in 2024-08-17T16:51:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:05 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:51:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:05 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:51:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:05 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:06.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:05 smithi063 ceph-mon[31063]: osdmap e43: 6 total, 6 up, 6 in 2024-08-17T16:51:06.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:05 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:51:06.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:05 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:51:06.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:05 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:06.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:05 smithi063 ceph-mon[42680]: osdmap e43: 6 total, 6 up, 6 in 2024-08-17T16:51:06.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:05 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:51:06.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:05 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:51:06.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:05 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:07.120 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:06 smithi162 ceph-mon[34633]: from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi162:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:07.120 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:06 smithi162 ceph-mon[34633]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail; 139 KiB/s, 0 objects/s recovering 2024-08-17T16:51:07.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:06 smithi063 ceph-mon[31063]: from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi162:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:07.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:06 smithi063 ceph-mon[31063]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail; 139 KiB/s, 0 objects/s recovering 2024-08-17T16:51:07.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:06 smithi063 ceph-mon[42680]: from='client.24250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi162:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:07.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:06 smithi063 ceph-mon[42680]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail; 139 KiB/s, 0 objects/s recovering 2024-08-17T16:51:08.183 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:07 smithi162 ceph-mon[34633]: from='client.? 172.21.15.162:0/4064902300' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a8255b37-1be5-4cd4-9b86-889f6bf52902"}]: dispatch 2024-08-17T16:51:08.183 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:07 smithi162 ceph-mon[34633]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a8255b37-1be5-4cd4-9b86-889f6bf52902"}]: dispatch 2024-08-17T16:51:08.183 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:07 smithi162 ceph-mon[34633]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a8255b37-1be5-4cd4-9b86-889f6bf52902"}]': finished 2024-08-17T16:51:08.183 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:07 smithi162 ceph-mon[34633]: osdmap e44: 7 total, 6 up, 7 in 2024-08-17T16:51:08.183 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:07 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:51:08.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:07 smithi063 ceph-mon[42680]: from='client.? 172.21.15.162:0/4064902300' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a8255b37-1be5-4cd4-9b86-889f6bf52902"}]: dispatch 2024-08-17T16:51:08.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:07 smithi063 ceph-mon[42680]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a8255b37-1be5-4cd4-9b86-889f6bf52902"}]: dispatch 2024-08-17T16:51:08.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:07 smithi063 ceph-mon[42680]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a8255b37-1be5-4cd4-9b86-889f6bf52902"}]': finished 2024-08-17T16:51:08.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:07 smithi063 ceph-mon[42680]: osdmap e44: 7 total, 6 up, 7 in 2024-08-17T16:51:08.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:07 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:51:08.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:07 smithi063 ceph-mon[31063]: from='client.? 172.21.15.162:0/4064902300' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a8255b37-1be5-4cd4-9b86-889f6bf52902"}]: dispatch 2024-08-17T16:51:08.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:07 smithi063 ceph-mon[31063]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a8255b37-1be5-4cd4-9b86-889f6bf52902"}]: dispatch 2024-08-17T16:51:08.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:07 smithi063 ceph-mon[31063]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a8255b37-1be5-4cd4-9b86-889f6bf52902"}]': finished 2024-08-17T16:51:08.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:07 smithi063 ceph-mon[31063]: osdmap e44: 7 total, 6 up, 7 in 2024-08-17T16:51:08.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:07 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:51:09.181 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:08 smithi063 ceph-mon[42680]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-08-17T16:51:09.181 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:08 smithi063 ceph-mon[42680]: from='client.? 172.21.15.162:0/1351248582' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:51:09.181 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:08 smithi063 ceph-mon[31063]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-08-17T16:51:09.181 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:08 smithi063 ceph-mon[31063]: from='client.? 172.21.15.162:0/1351248582' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:51:09.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:08 smithi162 ceph-mon[34633]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-08-17T16:51:09.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:08 smithi162 ceph-mon[34633]: from='client.? 172.21.15.162:0/1351248582' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:51:10.315 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:10 smithi162 ceph-mon[34633]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-08-17T16:51:10.433 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:10 smithi063 ceph-mon[31063]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-08-17T16:51:10.433 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:10 smithi063 ceph-mon[42680]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-08-17T16:51:12.297 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:12 smithi162 ceph-mon[34633]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T16:51:12.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:12 smithi063 ceph-mon[31063]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T16:51:12.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:12 smithi063 ceph-mon[42680]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-08-17T16:51:13.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:13 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-17T16:51:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:13 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:13 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-17T16:51:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:13 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:13 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-08-17T16:51:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:13 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:14.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:14 smithi162 ceph-mon[34633]: Deploying daemon osd.6 on smithi162 2024-08-17T16:51:14.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:14 smithi162 ceph-mon[34633]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-08-17T16:51:15.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:14 smithi063 ceph-mon[31063]: Deploying daemon osd.6 on smithi162 2024-08-17T16:51:15.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:14 smithi063 ceph-mon[31063]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-08-17T16:51:15.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:14 smithi063 ceph-mon[42680]: Deploying daemon osd.6 on smithi162 2024-08-17T16:51:15.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:14 smithi063 ceph-mon[42680]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-08-17T16:51:16.311 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:16 smithi162 ceph-mon[34633]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:16.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:16 smithi063 ceph-mon[31063]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:16.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:16 smithi063 ceph-mon[42680]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:18.225 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:18 smithi162 ceph-mon[34633]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:18.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:18 smithi063 ceph-mon[42680]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:18.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:18 smithi063 ceph-mon[31063]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:19.121 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:19 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:19.121 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:19 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:19.121 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:19 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:19.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:19 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:19.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:19 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:19.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:19 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:19.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:19 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:19.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:19 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:19.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:19 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:20.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:20 smithi063 ceph-mon[31063]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:20.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:20 smithi063 ceph-mon[42680]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:20.329 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:20 smithi162 ceph-mon[34633]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:21.516 INFO:teuthology.orchestra.run.smithi162.stdout:Created osd(s) 6 on host 'smithi162' 2024-08-17T16:51:22.113 DEBUG:teuthology.orchestra.run.smithi162:osd.6> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.6.service 2024-08-17T16:51:22.116 INFO:tasks.cephadm:Deploying osd.7 on smithi162 with /dev/vg_nvme/lv_1... 2024-08-17T16:51:22.116 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-08-17T16:51:22.360 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:51:22.737 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:22 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:22.737 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:22 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:22.737 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:22 smithi162 ceph-mon[34633]: from='osd.6 [v2:172.21.15.162:6816/1675904899,v1:172.21.15.162:6817/1675904899]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T16:51:22.737 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:22 smithi162 ceph-mon[34633]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T16:51:22.738 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:22 smithi162 ceph-mon[34633]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:22.738 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:22 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:22.738 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:22 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:22.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:22 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:22.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:22 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:22.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:22 smithi063 ceph-mon[31063]: from='osd.6 [v2:172.21.15.162:6816/1675904899,v1:172.21.15.162:6817/1675904899]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T16:51:22.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:22 smithi063 ceph-mon[31063]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T16:51:22.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:22 smithi063 ceph-mon[31063]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:22.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:22 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:22.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:22 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:22.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:22 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:22.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:22 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:22.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:22 smithi063 ceph-mon[42680]: from='osd.6 [v2:172.21.15.162:6816/1675904899,v1:172.21.15.162:6817/1675904899]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T16:51:22.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:22 smithi063 ceph-mon[42680]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-08-17T16:51:22.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:22 smithi063 ceph-mon[42680]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:22.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:22 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:22.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:22 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:23.738 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:23 smithi162 ceph-mon[34633]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T16:51:23.738 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:23 smithi162 ceph-mon[34633]: osdmap e45: 7 total, 6 up, 7 in 2024-08-17T16:51:23.739 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:23 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:51:23.739 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:23 smithi162 ceph-mon[34633]: from='osd.6 [v2:172.21.15.162:6816/1675904899,v1:172.21.15.162:6817/1675904899]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:23.739 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:23 smithi162 ceph-mon[34633]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:23.739 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 16:51:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[51613]: 2024-08-17T16:51:23.525+0000 7efc1c5e6640 -1 osd.6 0 waiting for initial osdmap 2024-08-17T16:51:23.739 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 16:51:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[51613]: 2024-08-17T16:51:23.534+0000 7efc1840f640 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:51:23.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:23 smithi063 ceph-mon[31063]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T16:51:23.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:23 smithi063 ceph-mon[31063]: osdmap e45: 7 total, 6 up, 7 in 2024-08-17T16:51:23.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:23 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:51:23.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:23 smithi063 ceph-mon[31063]: from='osd.6 [v2:172.21.15.162:6816/1675904899,v1:172.21.15.162:6817/1675904899]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:23.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:23 smithi063 ceph-mon[31063]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:23.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:23 smithi063 ceph-mon[42680]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T16:51:23.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:23 smithi063 ceph-mon[42680]: osdmap e45: 7 total, 6 up, 7 in 2024-08-17T16:51:23.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:23 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:51:23.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:23 smithi063 ceph-mon[42680]: from='osd.6 [v2:172.21.15.162:6816/1675904899,v1:172.21.15.162:6817/1675904899]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:23.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:23 smithi063 ceph-mon[42680]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:24.534 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:24 smithi162 ceph-mon[34633]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]': finished 2024-08-17T16:51:24.534 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:24 smithi162 ceph-mon[34633]: osdmap e46: 7 total, 6 up, 7 in 2024-08-17T16:51:24.534 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:24 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:51:24.534 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:24 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:51:24.534 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:24 smithi162 ceph-mon[34633]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:24.987 INFO:teuthology.orchestra.run.smithi162.stdout: 2024-08-17T16:51:25.008 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch daemon add osd smithi162:vg_nvme/lv_1 2024-08-17T16:51:25.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:24 smithi063 ceph-mon[31063]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]': finished 2024-08-17T16:51:25.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:24 smithi063 ceph-mon[31063]: osdmap e46: 7 total, 6 up, 7 in 2024-08-17T16:51:25.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:24 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:51:25.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:24 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:51:25.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:24 smithi063 ceph-mon[31063]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:25.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:24 smithi063 ceph-mon[42680]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]': finished 2024-08-17T16:51:25.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:24 smithi063 ceph-mon[42680]: osdmap e46: 7 total, 6 up, 7 in 2024-08-17T16:51:25.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:51:25.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:24 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:51:25.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:24 smithi063 ceph-mon[42680]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-08-17T16:51:25.725 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:51:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:25 smithi162 ceph-mon[34633]: purged_snaps scrub starts 2024-08-17T16:51:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:25 smithi162 ceph-mon[34633]: purged_snaps scrub ok 2024-08-17T16:51:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:25 smithi162 ceph-mon[34633]: osd.6 [v2:172.21.15.162:6816/1675904899,v1:172.21.15.162:6817/1675904899] boot 2024-08-17T16:51:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:25 smithi162 ceph-mon[34633]: osdmap e47: 7 total, 7 up, 7 in 2024-08-17T16:51:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:25 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:51:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:25 smithi162 ceph-mon[34633]: osdmap e48: 7 total, 7 up, 7 in 2024-08-17T16:51:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:25 smithi063 ceph-mon[42680]: purged_snaps scrub starts 2024-08-17T16:51:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:25 smithi063 ceph-mon[42680]: purged_snaps scrub ok 2024-08-17T16:51:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:25 smithi063 ceph-mon[42680]: osd.6 [v2:172.21.15.162:6816/1675904899,v1:172.21.15.162:6817/1675904899] boot 2024-08-17T16:51:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:25 smithi063 ceph-mon[42680]: osdmap e47: 7 total, 7 up, 7 in 2024-08-17T16:51:26.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:25 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:51:26.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:25 smithi063 ceph-mon[42680]: osdmap e48: 7 total, 7 up, 7 in 2024-08-17T16:51:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:25 smithi063 ceph-mon[31063]: purged_snaps scrub starts 2024-08-17T16:51:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:25 smithi063 ceph-mon[31063]: purged_snaps scrub ok 2024-08-17T16:51:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:25 smithi063 ceph-mon[31063]: osd.6 [v2:172.21.15.162:6816/1675904899,v1:172.21.15.162:6817/1675904899] boot 2024-08-17T16:51:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:25 smithi063 ceph-mon[31063]: osdmap e47: 7 total, 7 up, 7 in 2024-08-17T16:51:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:25 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:51:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:25 smithi063 ceph-mon[31063]: osdmap e48: 7 total, 7 up, 7 in 2024-08-17T16:51:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:26 smithi063 ceph-mon[31063]: osdmap e49: 7 total, 7 up, 7 in 2024-08-17T16:51:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:26 smithi063 ceph-mon[31063]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:27.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:26 smithi063 ceph-mon[42680]: osdmap e49: 7 total, 7 up, 7 in 2024-08-17T16:51:27.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:26 smithi063 ceph-mon[42680]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:27.070 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:26 smithi162 ceph-mon[34633]: osdmap e49: 7 total, 7 up, 7 in 2024-08-17T16:51:27.070 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:26 smithi162 ceph-mon[34633]: pgmap v152: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:28.018 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:27 smithi162 ceph-mon[34633]: osdmap e50: 7 total, 7 up, 7 in 2024-08-17T16:51:28.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:27 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:28.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:27 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:28.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:27 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:28.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:27 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:28.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:27 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:28.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:27 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:28.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:27 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:28.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:27 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:28.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:27 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:28.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:27 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:28.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:27 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:28.020 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:27 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:28.020 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:27 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:28.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[31063]: osdmap e50: 7 total, 7 up, 7 in 2024-08-17T16:51:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:28.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:28.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[42680]: osdmap e50: 7 total, 7 up, 7 in 2024-08-17T16:51:28.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:28.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:28.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:28.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:28.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:28.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:28.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:28.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:28.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:28.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:28.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:28.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:28.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:27 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:29.041 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:28 smithi162 ceph-mon[34633]: Detected new or changed devices on smithi162 2024-08-17T16:51:29.041 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:28 smithi162 ceph-mon[34633]: Adjusting osd_memory_target on smithi162 to 5696M 2024-08-17T16:51:29.041 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:28 smithi162 ceph-mon[34633]: osdmap e51: 7 total, 7 up, 7 in 2024-08-17T16:51:29.041 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:28 smithi162 ceph-mon[34633]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:29.042 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:28 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:51:29.042 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:28 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:51:29.042 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:28 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:29.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:28 smithi063 ceph-mon[31063]: Detected new or changed devices on smithi162 2024-08-17T16:51:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:28 smithi063 ceph-mon[31063]: Adjusting osd_memory_target on smithi162 to 5696M 2024-08-17T16:51:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:28 smithi063 ceph-mon[31063]: osdmap e51: 7 total, 7 up, 7 in 2024-08-17T16:51:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:28 smithi063 ceph-mon[31063]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:28 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:51:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:28 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:51:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:28 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:29.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:28 smithi063 ceph-mon[42680]: Detected new or changed devices on smithi162 2024-08-17T16:51:29.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:28 smithi063 ceph-mon[42680]: Adjusting osd_memory_target on smithi162 to 5696M 2024-08-17T16:51:29.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:28 smithi063 ceph-mon[42680]: osdmap e51: 7 total, 7 up, 7 in 2024-08-17T16:51:29.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:28 smithi063 ceph-mon[42680]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:29.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:28 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-08-17T16:51:29.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:28 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-08-17T16:51:29.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:28 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:30.030 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:29 smithi162 ceph-mon[34633]: from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi162:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:30.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:29 smithi063 ceph-mon[31063]: from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi162:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:30.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:29 smithi063 ceph-mon[42680]: from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi162:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:51:30.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:30 smithi162 ceph-mon[34633]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-08-17T16:51:31.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:30 smithi063 ceph-mon[31063]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-08-17T16:51:31.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:30 smithi063 ceph-mon[42680]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 111 KiB/s, 0 objects/s recovering 2024-08-17T16:51:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:31 smithi162 ceph-mon[34633]: from='client.? 172.21.15.162:0/3876600904' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3d0bcd6d-171d-4d90-a033-ed17714a9d96"}]: dispatch 2024-08-17T16:51:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:31 smithi162 ceph-mon[34633]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3d0bcd6d-171d-4d90-a033-ed17714a9d96"}]: dispatch 2024-08-17T16:51:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:31 smithi162 ceph-mon[34633]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3d0bcd6d-171d-4d90-a033-ed17714a9d96"}]': finished 2024-08-17T16:51:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:31 smithi162 ceph-mon[34633]: osdmap e52: 8 total, 7 up, 8 in 2024-08-17T16:51:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:31 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:51:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:31 smithi162 ceph-mon[34633]: from='client.? 172.21.15.162:0/715130955' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:51:32.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:31 smithi063 ceph-mon[42680]: from='client.? 172.21.15.162:0/3876600904' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3d0bcd6d-171d-4d90-a033-ed17714a9d96"}]: dispatch 2024-08-17T16:51:32.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:31 smithi063 ceph-mon[42680]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3d0bcd6d-171d-4d90-a033-ed17714a9d96"}]: dispatch 2024-08-17T16:51:32.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:31 smithi063 ceph-mon[42680]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3d0bcd6d-171d-4d90-a033-ed17714a9d96"}]': finished 2024-08-17T16:51:32.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:31 smithi063 ceph-mon[42680]: osdmap e52: 8 total, 7 up, 8 in 2024-08-17T16:51:32.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:31 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:51:32.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:31 smithi063 ceph-mon[42680]: from='client.? 172.21.15.162:0/715130955' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:51:32.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:31 smithi063 ceph-mon[31063]: from='client.? 172.21.15.162:0/3876600904' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3d0bcd6d-171d-4d90-a033-ed17714a9d96"}]: dispatch 2024-08-17T16:51:32.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:31 smithi063 ceph-mon[31063]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3d0bcd6d-171d-4d90-a033-ed17714a9d96"}]: dispatch 2024-08-17T16:51:32.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:31 smithi063 ceph-mon[31063]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3d0bcd6d-171d-4d90-a033-ed17714a9d96"}]': finished 2024-08-17T16:51:32.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:31 smithi063 ceph-mon[31063]: osdmap e52: 8 total, 7 up, 8 in 2024-08-17T16:51:32.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:31 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:51:32.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:31 smithi063 ceph-mon[31063]: from='client.? 172.21.15.162:0/715130955' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-08-17T16:51:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:32 smithi162 ceph-mon[34633]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T16:51:33.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:32 smithi063 ceph-mon[31063]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T16:51:33.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:32 smithi063 ceph-mon[42680]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T16:51:34.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:34 smithi162 ceph-mon[34633]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-08-17T16:51:34.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:34 smithi063 ceph-mon[42680]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-08-17T16:51:34.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:34 smithi063 ceph-mon[31063]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-08-17T16:51:36.230 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:36 smithi162 ceph-mon[34633]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-08-17T16:51:36.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:36 smithi063 ceph-mon[31063]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-08-17T16:51:36.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:36 smithi063 ceph-mon[42680]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 70 KiB/s, 0 objects/s recovering 2024-08-17T16:51:37.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:37 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-17T16:51:37.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:37 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:37.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:37 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-17T16:51:37.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:37 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:37.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:37 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-08-17T16:51:37.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:37 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:38.306 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:38 smithi162 ceph-mon[34633]: Deploying daemon osd.7 on smithi162 2024-08-17T16:51:38.306 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:38 smithi162 ceph-mon[34633]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T16:51:38.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:38 smithi063 ceph-mon[31063]: Deploying daemon osd.7 on smithi162 2024-08-17T16:51:38.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:38 smithi063 ceph-mon[31063]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T16:51:38.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:38 smithi063 ceph-mon[42680]: Deploying daemon osd.7 on smithi162 2024-08-17T16:51:38.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:38 smithi063 ceph-mon[42680]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T16:51:40.252 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:40 smithi162 ceph-mon[34633]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:40.332 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:40 smithi063 ceph-mon[42680]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:40.332 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:40 smithi063 ceph-mon[31063]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:42.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:42 smithi162 ceph-mon[34633]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:42 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:42 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:42.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:42 smithi063 ceph-mon[42680]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:42.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:42 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:42.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:42 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:42.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:42 smithi063 ceph-mon[31063]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:42.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:42 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:42.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:42 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:43.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:43 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:43.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:43 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:43.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:43 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:44.305 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:44 smithi162 ceph-mon[34633]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:44.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:44 smithi063 ceph-mon[31063]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:44.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:44 smithi063 ceph-mon[42680]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:45.093 INFO:teuthology.orchestra.run.smithi162.stdout:Created osd(s) 7 on host 'smithi162' 2024-08-17T16:51:45.657 DEBUG:teuthology.orchestra.run.smithi162:osd.7> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.7.service 2024-08-17T16:51:45.660 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-08-17T16:51:45.660 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd stat -f json 2024-08-17T16:51:45.884 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:51:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:46 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:46 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:46 smithi063 ceph-mon[42680]: from='osd.7 [v2:172.21.15.162:6824/2938230961,v1:172.21.15.162:6825/2938230961]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T16:51:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:46 smithi063 ceph-mon[42680]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T16:51:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:46 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:46 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:46 smithi063 ceph-mon[42680]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:46.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:46 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:46.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:46 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:46.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:46 smithi063 ceph-mon[31063]: from='osd.7 [v2:172.21.15.162:6824/2938230961,v1:172.21.15.162:6825/2938230961]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T16:51:46.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:46 smithi063 ceph-mon[31063]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T16:51:46.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:46 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:46.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:46 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:46.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:46 smithi063 ceph-mon[31063]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:46 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:46 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:46 smithi162 ceph-mon[34633]: from='osd.7 [v2:172.21.15.162:6824/2938230961,v1:172.21.15.162:6825/2938230961]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T16:51:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:46 smithi162 ceph-mon[34633]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-08-17T16:51:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:46 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:46 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:46 smithi162 ceph-mon[34633]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:47.167 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:47 smithi162 ceph-mon[34633]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T16:51:47.168 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:47 smithi162 ceph-mon[34633]: osdmap e53: 8 total, 7 up, 8 in 2024-08-17T16:51:47.168 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:47 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:51:47.168 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:47 smithi162 ceph-mon[34633]: from='osd.7 [v2:172.21.15.162:6824/2938230961,v1:172.21.15.162:6825/2938230961]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:47.168 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:47 smithi162 ceph-mon[34633]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:47.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:47 smithi063 ceph-mon[31063]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T16:51:47.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:47 smithi063 ceph-mon[31063]: osdmap e53: 8 total, 7 up, 8 in 2024-08-17T16:51:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:47 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:51:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:47 smithi063 ceph-mon[31063]: from='osd.7 [v2:172.21.15.162:6824/2938230961,v1:172.21.15.162:6825/2938230961]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:47 smithi063 ceph-mon[31063]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:47.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:47 smithi063 ceph-mon[42680]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T16:51:47.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:47 smithi063 ceph-mon[42680]: osdmap e53: 8 total, 7 up, 8 in 2024-08-17T16:51:47.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:47 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:51:47.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:47 smithi063 ceph-mon[42680]: from='osd.7 [v2:172.21.15.162:6824/2938230961,v1:172.21.15.162:6825/2938230961]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:47.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:47 smithi063 ceph-mon[42680]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]: dispatch 2024-08-17T16:51:47.644 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:51:47.846 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 16:51:47 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[57708]: 2024-08-17T16:51:47.546+0000 7f061a285640 -1 osd.7 0 waiting for initial osdmap 2024-08-17T16:51:47.846 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 16:51:47 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[57708]: 2024-08-17T16:51:47.553+0000 7f06120a6640 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T16:51:48.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:48 smithi063 ceph-mon[31063]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]': finished 2024-08-17T16:51:48.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:48 smithi063 ceph-mon[31063]: osdmap e54: 8 total, 7 up, 8 in 2024-08-17T16:51:48.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:48 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:51:48.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:48 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:51:48.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:48 smithi063 ceph-mon[31063]: from='osd.7 ' entity='osd.7' 2024-08-17T16:51:48.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:48 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3791947269' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T16:51:48.265 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:48 smithi063 ceph-mon[31063]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:48.265 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:48 smithi063 ceph-mon[42680]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]': finished 2024-08-17T16:51:48.265 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:48 smithi063 ceph-mon[42680]: osdmap e54: 8 total, 7 up, 8 in 2024-08-17T16:51:48.265 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:48 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:51:48.265 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:48 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:51:48.265 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:48 smithi063 ceph-mon[42680]: from='osd.7 ' entity='osd.7' 2024-08-17T16:51:48.265 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:48 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3791947269' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T16:51:48.265 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:48 smithi063 ceph-mon[42680]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:48.289 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1723913484,"num_in_osds":8,"osd_in_since":1723913490,"num_remapped_pgs":0} 2024-08-17T16:51:48.482 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:48 smithi162 ceph-mon[34633]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi162", "root=default"]}]': finished 2024-08-17T16:51:48.482 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:48 smithi162 ceph-mon[34633]: osdmap e54: 8 total, 7 up, 8 in 2024-08-17T16:51:48.482 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:48 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:51:48.482 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:48 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:51:48.482 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:48 smithi162 ceph-mon[34633]: from='osd.7 ' entity='osd.7' 2024-08-17T16:51:48.482 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:48 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3791947269' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T16:51:48.483 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:48 smithi162 ceph-mon[34633]: pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-08-17T16:51:49.290 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd stat -f json 2024-08-17T16:51:49.512 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:51:49.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: purged_snaps scrub starts 2024-08-17T16:51:49.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: purged_snaps scrub ok 2024-08-17T16:51:49.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: osd.7 [v2:172.21.15.162:6824/2938230961,v1:172.21.15.162:6825/2938230961] boot 2024-08-17T16:51:49.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: osdmap e55: 8 total, 8 up, 8 in 2024-08-17T16:51:49.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:51:49.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: Detected new or changed devices on smithi162 2024-08-17T16:51:49.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:49.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:49.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:49.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:49.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:49.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:49.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: Adjusting osd_memory_target on smithi162 to 4272M 2024-08-17T16:51:49.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:49.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:49.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:49.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:49.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:49.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:49.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:49.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:49.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: purged_snaps scrub starts 2024-08-17T16:51:49.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: purged_snaps scrub ok 2024-08-17T16:51:49.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: osd.7 [v2:172.21.15.162:6824/2938230961,v1:172.21.15.162:6825/2938230961] boot 2024-08-17T16:51:49.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: osdmap e55: 8 total, 8 up, 8 in 2024-08-17T16:51:49.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:51:49.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: Detected new or changed devices on smithi162 2024-08-17T16:51:49.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:49.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:49.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:49.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:49.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:49.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:49.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: Adjusting osd_memory_target on smithi162 to 4272M 2024-08-17T16:51:49.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:49.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:49.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:49.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:49.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:49.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:49.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:49.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:49 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: purged_snaps scrub starts 2024-08-17T16:51:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: purged_snaps scrub ok 2024-08-17T16:51:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: osd.7 [v2:172.21.15.162:6824/2938230961,v1:172.21.15.162:6825/2938230961] boot 2024-08-17T16:51:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: osdmap e55: 8 total, 8 up, 8 in 2024-08-17T16:51:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:51:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: Detected new or changed devices on smithi162 2024-08-17T16:51:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:49.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:51:49.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: Adjusting osd_memory_target on smithi162 to 4272M 2024-08-17T16:51:49.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:49.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:49.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:49.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:49.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:51:49.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:51:49.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:51:49.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:49 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:51:50.115 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:50 smithi063 ceph-mon[42680]: osdmap e56: 8 total, 8 up, 8 in 2024-08-17T16:51:50.116 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:50 smithi063 ceph-mon[42680]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 352 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:51:50.116 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:50 smithi063 ceph-mon[31063]: osdmap e56: 8 total, 8 up, 8 in 2024-08-17T16:51:50.116 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:50 smithi063 ceph-mon[31063]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 352 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:51:50.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:50 smithi162 ceph-mon[34633]: osdmap e56: 8 total, 8 up, 8 in 2024-08-17T16:51:50.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:50 smithi162 ceph-mon[34633]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 352 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:51:51.120 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:51 smithi063 ceph-mon[31063]: osdmap e57: 8 total, 8 up, 8 in 2024-08-17T16:51:51.226 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:51:51.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:51 smithi063 ceph-mon[42680]: osdmap e57: 8 total, 8 up, 8 in 2024-08-17T16:51:51.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:51 smithi162 ceph-mon[34633]: osdmap e57: 8 total, 8 up, 8 in 2024-08-17T16:51:51.912 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1723913508,"num_in_osds":8,"osd_in_since":1723913490,"num_remapped_pgs":0} 2024-08-17T16:51:51.913 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T16:51:52.134 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:51:52.257 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:52 smithi063 ceph-mon[42680]: osdmap e58: 8 total, 8 up, 8 in 2024-08-17T16:51:52.257 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:52 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/741881521' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T16:51:52.257 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:52 smithi063 ceph-mon[42680]: pgmap v174: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 352 MiB used, 715 GiB / 715 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-08-17T16:51:52.257 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:52 smithi063 ceph-mon[31063]: osdmap e58: 8 total, 8 up, 8 in 2024-08-17T16:51:52.258 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:52 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/741881521' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T16:51:52.258 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:52 smithi063 ceph-mon[31063]: pgmap v174: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 352 MiB used, 715 GiB / 715 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-08-17T16:51:52.497 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:52 smithi162 ceph-mon[34633]: osdmap e58: 8 total, 8 up, 8 in 2024-08-17T16:51:52.497 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:52 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/741881521' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-08-17T16:51:52.497 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:52 smithi162 ceph-mon[34633]: pgmap v174: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 352 MiB used, 715 GiB / 715 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-08-17T16:51:53.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:53 smithi063 ceph-mon[31063]: osdmap e59: 8 total, 8 up, 8 in 2024-08-17T16:51:53.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:53 smithi063 ceph-mon[42680]: osdmap e59: 8 total, 8 up, 8 in 2024-08-17T16:51:53.514 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:53 smithi162 ceph-mon[34633]: osdmap e59: 8 total, 8 up, 8 in 2024-08-17T16:51:53.964 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:51:53.964 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":59,"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:46:54.156006+0000","modified":"2024-08-17T16:51:52.119275+0000","last_up_change":"2024-08-17T16:51:48.092300+0000","last_in_change":"2024-08-17T16:51:30.856523+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:49:54.431240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6802","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6803","nonce":782866097}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6804","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6805","nonce":782866097}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6808","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6809","nonce":782866097}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6806","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6807","nonce":782866097}]},"public_addr":"172.21.15.63:6803/782866097","cluster_addr":"172.21.15.63:6805/782866097","heartbeat_back_addr":"172.21.15.63:6809/782866097","heartbeat_front_addr":"172.21.15.63:6807/782866097","state":["exists","up"]},{"osd":1,"uuid":"ae346e99-4706-4871-990a-f8e737572245","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6810","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6811","nonce":944260396}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6812","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6813","nonce":944260396}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6816","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6817","nonce":944260396}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6814","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6815","nonce":944260396}]},"public_addr":"172.21.15.63:6811/944260396","cluster_addr":"172.21.15.63:6813/944260396","heartbeat_back_addr":"172.21.15.63:6817/944260396","heartbeat_front_addr":"172.21.15.63:6815/944260396","state":["exists","up"]},{"osd":2,"uuid":"2d85d75f-1280-443f-857d-d322bd60c451","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6818","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6819","nonce":3751946332}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6820","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6821","nonce":3751946332}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6824","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6825","nonce":3751946332}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6822","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6823","nonce":3751946332}]},"public_addr":"172.21.15.63:6819/3751946332","cluster_addr":"172.21.15.63:6821/3751946332","heartbeat_back_addr":"172.21.15.63:6825/3751946332","heartbeat_front_addr":"172.21.15.63:6823/3751946332","state":["exists","up"]},{"osd":3,"uuid":"771f43d2-e98b-43e0-921b-88e56fadda42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6826","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6827","nonce":2654575936}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6828","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6829","nonce":2654575936}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6832","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6833","nonce":2654575936}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6830","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6831","nonce":2654575936}]},"public_addr":"172.21.15.63:6827/2654575936","cluster_addr":"172.21.15.63:6829/2654575936","heartbeat_back_addr":"172.21.15.63:6833/2654575936","heartbeat_front_addr":"172.21.15.63:6831/2654575936","state":["exists","up"]},{"osd":4,"uuid":"73c3c545-e975-4a6f-a1cb-e4575ee7a9b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6800","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6801","nonce":1563885208}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6803","nonce":1563885208}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6807","nonce":1563885208}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6805","nonce":1563885208}]},"public_addr":"172.21.15.162:6801/1563885208","cluster_addr":"172.21.15.162:6803/1563885208","heartbeat_back_addr":"172.21.15.162:6807/1563885208","heartbeat_front_addr":"172.21.15.162:6805/1563885208","state":["exists","up"]},{"osd":5,"uuid":"fae3ac52-3194-4eb4-9510-e3899768353a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6809","nonce":1826502179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6810","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6811","nonce":1826502179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6814","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6815","nonce":1826502179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6812","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6813","nonce":1826502179}]},"public_addr":"172.21.15.162:6809/1826502179","cluster_addr":"172.21.15.162:6811/1826502179","heartbeat_back_addr":"172.21.15.162:6815/1826502179","heartbeat_front_addr":"172.21.15.162:6813/1826502179","state":["exists","up"]},{"osd":6,"uuid":"a8255b37-1be5-4cd4-9b86-889f6bf52902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6816","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6817","nonce":1675904899}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6818","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6819","nonce":1675904899}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6822","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6823","nonce":1675904899}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6820","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6821","nonce":1675904899}]},"public_addr":"172.21.15.162:6817/1675904899","cluster_addr":"172.21.15.162:6819/1675904899","heartbeat_back_addr":"172.21.15.162:6823/1675904899","heartbeat_front_addr":"172.21.15.162:6821/1675904899","state":["exists","up"]},{"osd":7,"uuid":"3d0bcd6d-171d-4d90-a033-ed17714a9d96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6824","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6825","nonce":2938230961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6826","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6827","nonce":2938230961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6830","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6831","nonce":2938230961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6828","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6829","nonce":2938230961}]},"public_addr":"172.21.15.162:6825/2938230961","cluster_addr":"172.21.15.162:6827/2938230961","heartbeat_back_addr":"172.21.15.162:6831/2938230961","heartbeat_front_addr":"172.21.15.162:6829/2938230961","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:02.667962+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:26.166590+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:50.153794+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:13.556156+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:34.827805+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:58.789701+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:22.650398+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:46.166424+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.63:0/1986138219":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3226753153":"2024-08-18T16:47:39.919472+0000","172.21.15.63:6800/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/1792393387":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3540337558":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6801/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.63:6801/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6800/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:0/3004218255":"2024-08-18T16:47:16.384724+0000","172.21.15.63:0/84364348":"2024-08-18T16:47:16.384724+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T16:51:54.232 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:54 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/836146914' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:51:54.232 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:54 smithi063 ceph-mon[42680]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 119 KiB/s, 0 objects/s recovering 2024-08-17T16:51:54.232 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:54 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/836146914' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:51:54.232 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:54 smithi063 ceph-mon[31063]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 119 KiB/s, 0 objects/s recovering 2024-08-17T16:51:54.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:54 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/836146914' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:51:54.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:54 smithi162 ceph-mon[34633]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 119 KiB/s, 0 objects/s recovering 2024-08-17T16:51:54.615 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-08-17T16:49:54.431240+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 7.889999866485596, 'score_stable': 7.889999866485596, 'optimal_score': 0.3799999952316284, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-08-17T16:51:54.616 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-08-17T16:51:54.827 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:51:56.229 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:56 smithi063 ceph-mon[31063]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T16:51:56.230 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:56 smithi063 ceph-mon[42680]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T16:51:56.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:56 smithi162 ceph-mon[34633]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-08-17T16:51:56.604 INFO:teuthology.orchestra.run.smithi063.stdout:pg_num: 1 2024-08-17T16:51:57.237 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:57 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/492279706' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-17T16:51:57.237 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:57 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/492279706' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-17T16:51:57.260 INFO:tasks.cephadm:Adding prometheus.a on smithi162 2024-08-17T16:51:57.261 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch apply prometheus '1;smithi162=a' 2024-08-17T16:51:57.293 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:57 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/492279706' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-08-17T16:51:57.478 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:51:58.321 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:51:58 smithi162 ceph-mon[34633]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-08-17T16:51:58.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:51:58 smithi063 ceph-mon[31063]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-08-17T16:51:58.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:51:58 smithi063 ceph-mon[42680]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-08-17T16:51:59.295 INFO:teuthology.orchestra.run.smithi162.stdout:Scheduled prometheus update... 2024-08-17T16:51:59.953 DEBUG:teuthology.orchestra.run.smithi162:prometheus.a> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@prometheus.a.service 2024-08-17T16:51:59.995 INFO:tasks.cephadm:Adding node-exporter.a on smithi063 2024-08-17T16:51:59.995 INFO:tasks.cephadm:Adding node-exporter.b on smithi162 2024-08-17T16:51:59.995 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch apply node-exporter '2;smithi063=a;smithi162=b' 2024-08-17T16:52:00.261 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:52:00.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:00 smithi162 ceph-mon[34633]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi162=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:00.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:00 smithi162 ceph-mon[34633]: Saving service prometheus spec with placement smithi162=a;count:1 2024-08-17T16:52:00.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:00 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:00.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:00 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:00.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:00 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:00.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:00 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:00.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:00 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:00.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:00 smithi162 ceph-mon[34633]: Deploying daemon prometheus.a on smithi162 2024-08-17T16:52:00.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:00 smithi162 ceph-mon[34633]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:00.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[42680]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi162=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:00.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[42680]: Saving service prometheus spec with placement smithi162=a;count:1 2024-08-17T16:52:00.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:00.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:00.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:00.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:00.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:00.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[42680]: Deploying daemon prometheus.a on smithi162 2024-08-17T16:52:00.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[42680]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[31063]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi162=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[31063]: Saving service prometheus spec with placement smithi162=a;count:1 2024-08-17T16:52:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:00.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:00.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:00.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:00.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[31063]: Deploying daemon prometheus.a on smithi162 2024-08-17T16:52:00.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:00 smithi063 ceph-mon[31063]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:02.261 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:02 smithi162 ceph-mon[34633]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:02.412 INFO:teuthology.orchestra.run.smithi162.stdout:Scheduled node-exporter update... 2024-08-17T16:52:02.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:02 smithi063 ceph-mon[42680]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:02.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:02 smithi063 ceph-mon[31063]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:03.592 DEBUG:teuthology.orchestra.run.smithi063:node-exporter.a> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@node-exporter.a.service 2024-08-17T16:52:03.595 DEBUG:teuthology.orchestra.run.smithi162:node-exporter.b> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@node-exporter.b.service 2024-08-17T16:52:03.597 INFO:tasks.cephadm:Adding alertmanager.a on smithi063 2024-08-17T16:52:03.597 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch apply alertmanager '1;smithi063=a' 2024-08-17T16:52:03.661 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:03 smithi162 ceph-mon[34633]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi063=a;smithi162=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:03.661 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:03 smithi162 ceph-mon[34633]: Saving service node-exporter spec with placement smithi063=a;smithi162=b;count:2 2024-08-17T16:52:03.661 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:03 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:03.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:03 smithi063 ceph-mon[31063]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi063=a;smithi162=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:03.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:03 smithi063 ceph-mon[31063]: Saving service node-exporter spec with placement smithi063=a;smithi162=b;count:2 2024-08-17T16:52:03.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:03 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:03.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:03 smithi063 ceph-mon[42680]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi063=a;smithi162=b", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:03.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:03 smithi063 ceph-mon[42680]: Saving service node-exporter spec with placement smithi063=a;smithi162=b;count:2 2024-08-17T16:52:03.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:03 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:03.836 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:52:04.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:04 smithi162 ceph-mon[34633]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:04.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:04 smithi063 ceph-mon[42680]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:04.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:04 smithi063 ceph-mon[31063]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:06.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:06 smithi162 ceph-mon[34633]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:06.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:06 smithi063 ceph-mon[42680]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:06.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:06 smithi063 ceph-mon[31063]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:06.960 INFO:teuthology.orchestra.run.smithi162.stdout:Scheduled alertmanager update... 2024-08-17T16:52:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:07 smithi162 ceph-mon[34633]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi063=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:07 smithi162 ceph-mon[34633]: Saving service alertmanager spec with placement smithi063=a;count:1 2024-08-17T16:52:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:07 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:08.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:07 smithi063 ceph-mon[42680]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi063=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:08.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:07 smithi063 ceph-mon[42680]: Saving service alertmanager spec with placement smithi063=a;count:1 2024-08-17T16:52:08.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:07 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:08.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:07 smithi063 ceph-mon[31063]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi063=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:08.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:07 smithi063 ceph-mon[31063]: Saving service alertmanager spec with placement smithi063=a;count:1 2024-08-17T16:52:08.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:07 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:08.294 DEBUG:teuthology.orchestra.run.smithi063:alertmanager.a> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@alertmanager.a.service 2024-08-17T16:52:08.296 INFO:tasks.cephadm:Adding grafana.a on smithi162 2024-08-17T16:52:08.296 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph orch apply grafana '1;smithi162=a' 2024-08-17T16:52:08.535 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:52:09.218 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:08 smithi162 ceph-mon[34633]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:09.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:08 smithi063 ceph-mon[42680]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:09.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:08 smithi063 ceph-mon[31063]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:10.225 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:10 smithi162 ceph-mon[34633]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:10.465 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:10 smithi063 ceph-mon[42680]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:10.465 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:10 smithi063 ceph-mon[31063]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:12.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:12 smithi162 ceph-mon[34633]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:12.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:12 smithi063 ceph-mon[42680]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:12.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:12 smithi063 ceph-mon[31063]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:12.870 INFO:teuthology.orchestra.run.smithi162.stdout:Scheduled grafana update... 2024-08-17T16:52:13.816 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:13 smithi162 systemd[1]: Starting Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:52:13.837 DEBUG:teuthology.orchestra.run.smithi162:grafana.a> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@grafana.a.service 2024-08-17T16:52:13.839 INFO:tasks.cephadm:Setting up client nodes... 2024-08-17T16:52:13.840 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-08-17T16:52:14.063 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:13 smithi162 ceph-mon[34633]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi162=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:13 smithi162 ceph-mon[34633]: Saving service grafana spec with placement smithi162=a;count:1 2024-08-17T16:52:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:13 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:14.096 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 podman[62851]: 2024-08-17 16:52:14.016096323 +0000 UTC m=+0.104479632 container create ab5c581becf8935c8c499aa24e700a67a9b416c59467195b33d0cce8781cc30a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:52:14.096 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 podman[62851]: 2024-08-17 16:52:13.929691141 +0000 UTC m=+0.018074448 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-17T16:52:14.180 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:13 smithi063 ceph-mon[42680]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi162=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:14.180 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:13 smithi063 ceph-mon[42680]: Saving service grafana spec with placement smithi162=a;count:1 2024-08-17T16:52:14.180 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:13 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:14.180 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:13 smithi063 ceph-mon[31063]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi162=a", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:52:14.180 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:13 smithi063 ceph-mon[31063]: Saving service grafana spec with placement smithi162=a;count:1 2024-08-17T16:52:14.181 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:13 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:14.596 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 podman[62851]: 2024-08-17 16:52:14.235112219 +0000 UTC m=+0.323495531 container init ab5c581becf8935c8c499aa24e700a67a9b416c59467195b33d0cce8781cc30a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:52:14.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 podman[62851]: 2024-08-17 16:52:14.240574948 +0000 UTC m=+0.328958249 container start ab5c581becf8935c8c499aa24e700a67a9b416c59467195b33d0cce8781cc30a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:52:14.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.263Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-17T16:52:14.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.263Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-17T16:52:14.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.263Z caller=main.go:570 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi162 (none))" 2024-08-17T16:52:14.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.264Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T16:52:14.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.264Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T16:52:14.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.265Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T16:52:14.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.266Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-17T16:52:14.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.266Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T16:52:14.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.266Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T16:52:14.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.269Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T16:52:14.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.269Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.342µs 2024-08-17T16:52:14.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.269Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T16:52:14.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.269Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-08-17T16:52:14.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.269Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=25.926µs wal_replay_duration=231.48µs wbl_replay_duration=127ns total_replay_duration=282.171µs 2024-08-17T16:52:14.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.270Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T16:52:14.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.270Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-17T16:52:14.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.270Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T16:52:14.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.289Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.233612ms db_storage=702ns remote_storage=1.024µs web_handler=313ns query_engine=441ns scrape=459.4µs scrape_sd=70.363µs notify=1.079µs notify_sd=1.285µs rules=18.429371ms tracing=7.005µs 2024-08-17T16:52:14.602 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.289Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-17T16:52:14.602 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:52:14.289Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T16:52:14.602 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 bash[62851]: ab5c581becf8935c8c499aa24e700a67a9b416c59467195b33d0cce8781cc30a 2024-08-17T16:52:14.602 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:52:14 smithi162 systemd[1]: Started Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:52:15.214 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:14 smithi063 ceph-mon[31063]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:15.214 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:14 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:15.214 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:14 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:15.214 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:14 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:15.214 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:14 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-17T16:52:15.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:14 smithi063 ceph-mon[42680]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:15.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:14 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:15.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:14 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:15.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:14 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:15.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:14 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-17T16:52:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-mon[34633]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:14 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-08-17T16:52:15.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:15 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ignoring --setuser ceph since I am not root 2024-08-17T16:52:15.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:15 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ignoring --setgroup ceph since I am not root 2024-08-17T16:52:15.685 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:15 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: ignoring --setuser ceph since I am not root 2024-08-17T16:52:15.686 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:15 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: ignoring --setgroup ceph since I am not root 2024-08-17T16:52:15.686 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:15 smithi162 ceph-mgr[36579]: -- 172.21.15.162:0/2782664961 <== mon.1 v2:172.21.15.162:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5633e2108d00 con 0x5633e20dd400 2024-08-17T16:52:15.686 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:15 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:15.542+0000 7f9fa07d5040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:52:15.686 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:15 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:15.684+0000 7f9fa07d5040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:52:15.974 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:15 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:15.551+0000 7f5de7df6040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:52:15.974 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:15 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:15.704+0000 7f5de7df6040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:52:15.974 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:15 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:15.775+0000 7f5de7df6040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:52:15.974 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:15 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:15.751+0000 7f9fa07d5040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:52:15.994 INFO:teuthology.orchestra.run.smithi063.stdout:[client.0] 2024-08-17T16:52:15.994 INFO:teuthology.orchestra.run.smithi063.stdout: key = AQA/1cBmGirpOhAAmsQTtKUbJ8iw3P/9qphfyg== 2024-08-17T16:52:16.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:15 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:16.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:15 smithi063 ceph-mon[42680]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-17T16:52:16.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:15 smithi063 ceph-mon[42680]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-17T16:52:16.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:15 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:16.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:15 smithi063 ceph-mon[31063]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-17T16:52:16.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:15 smithi063 ceph-mon[31063]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-17T16:52:16.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:16 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:16.033+0000 7f5de7df6040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:52:16.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:16 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:16.094+0000 7f5de7df6040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:52:16.345 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:15 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:15.994+0000 7f9fa07d5040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:52:16.346 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:16 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:16.053+0000 7f9fa07d5040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:52:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:15 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' 2024-08-17T16:52:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:15 smithi162 ceph-mon[34633]: from='mgr.14150 172.21.15.63:0/923110384' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-08-17T16:52:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:15 smithi162 ceph-mon[34633]: mgrmap e16: y(active, since 4m), standbys: x 2024-08-17T16:52:16.606 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T16:52:16.606 DEBUG:teuthology.orchestra.run.smithi063:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-08-17T16:52:16.606 DEBUG:teuthology.orchestra.run.smithi063:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-08-17T16:52:16.646 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-08-17T16:52:16.864 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T16:52:17.239 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:16 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/4089601454' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T16:52:17.239 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:16 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/4089601454' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T16:52:17.239 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:16 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/4089601454' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T16:52:17.239 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:16 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/4089601454' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T16:52:17.240 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:17 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:17.060+0000 7f5de7df6040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:52:17.240 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:17 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:17.179+0000 7f5de7df6040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:52:17.280 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:17 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:17.001+0000 7f9fa07d5040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:52:17.280 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:17 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:17.122+0000 7f9fa07d5040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:52:17.280 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:17 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:17.181+0000 7f9fa07d5040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:52:17.280 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:16 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/4089601454' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T16:52:17.280 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:16 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/4089601454' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T16:52:17.518 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:17 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:17.237+0000 7f5de7df6040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:52:17.518 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:17 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:17.455+0000 7f5de7df6040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:52:17.595 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:17 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:17.396+0000 7f9fa07d5040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:52:17.596 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:17 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:17.456+0000 7f9fa07d5040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:52:17.596 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:17 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:17.523+0000 7f9fa07d5040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:52:17.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:17 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:17.517+0000 7f5de7df6040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:52:17.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:17 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:17.585+0000 7f5de7df6040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:52:17.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:17 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:17.734+0000 7f5de7df6040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:52:18.067 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:17 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:17.668+0000 7f9fa07d5040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:52:18.345 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:18 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:18.091+0000 7f9fa07d5040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:52:18.494 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:18 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:18.169+0000 7f5de7df6040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:52:18.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:18 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:18.492+0000 7f5de7df6040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:52:18.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:18 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:18.560+0000 7f5de7df6040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:52:18.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:18 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:18.712+0000 7f5de7df6040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:52:18.845 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:18 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:18.419+0000 7f9fa07d5040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:52:18.845 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:18 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:18.488+0000 7f9fa07d5040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:52:18.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:18 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:18.642+0000 7f9fa07d5040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:52:18.887 INFO:teuthology.orchestra.run.smithi162.stdout:[client.1] 2024-08-17T16:52:18.887 INFO:teuthology.orchestra.run.smithi162.stdout: key = AQBC1cBmosSVNBAAPc/xSzET5U90UKB1MtSXbw== 2024-08-17T16:52:19.191 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:18 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:18.964+0000 7f9fa07d5040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:52:19.191 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:19 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:19.025+0000 7f9fa07d5040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:52:19.191 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:18 smithi162 ceph-mon[34633]: from='client.? 172.21.15.162:0/700355312' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T16:52:19.191 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:18 smithi162 ceph-mon[34633]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T16:52:19.192 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:18 smithi162 ceph-mon[34633]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T16:52:19.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:18 smithi063 ceph-mon[42680]: from='client.? 172.21.15.162:0/700355312' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T16:52:19.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:18 smithi063 ceph-mon[42680]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T16:52:19.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:18 smithi063 ceph-mon[42680]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T16:52:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:18 smithi063 ceph-mon[31063]: from='client.? 172.21.15.162:0/700355312' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T16:52:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:18 smithi063 ceph-mon[31063]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-08-17T16:52:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:18 smithi063 ceph-mon[31063]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-08-17T16:52:19.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:19.034+0000 7f5de7df6040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:52:19.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:19.097+0000 7f5de7df6040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:52:19.512 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:19 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:19.218+0000 7f9fa07d5040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:52:19.513 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:19 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:19.364+0000 7f9fa07d5040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-17T16:52:19.513 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:19 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:19.428+0000 7f9fa07d5040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:52:19.513 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:19 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:19.511+0000 7f9fa07d5040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:52:19.544 DEBUG:teuthology.orchestra.run.smithi162:> set -ex 2024-08-17T16:52:19.545 DEBUG:teuthology.orchestra.run.smithi162:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-08-17T16:52:19.545 DEBUG:teuthology.orchestra.run.smithi162:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-08-17T16:52:19.583 INFO:tasks.cephadm:Creating RBD pool 2024-08-17T16:52:19.583 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create rbd 8 2024-08-17T16:52:19.594 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:19.287+0000 7f5de7df6040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:52:19.594 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:19.441+0000 7f5de7df6040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-17T16:52:19.594 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:19.507+0000 7f5de7df6040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:52:19.594 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:19.593+0000 7f5de7df6040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:52:19.795 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:19.845 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:19 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:19.584+0000 7f9fa07d5040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:52:19.919 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:19.655+0000 7f5de7df6040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:52:20.368 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:20 smithi162 ceph-mon[34633]: Standby manager daemon x restarted 2024-08-17T16:52:20.368 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:20 smithi162 ceph-mon[34633]: Standby manager daemon x started 2024-08-17T16:52:20.368 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:20 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/2850890665' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:52:20.368 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:20 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/2850890665' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:52:20.368 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:20 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/2850890665' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:52:20.369 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:20 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/2850890665' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:52:20.369 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:20.104+0000 7f9fa07d5040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:52:20.369 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:20.259+0000 7f9fa07d5040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:52:20.369 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: [17/Aug/2024:16:52:20] ENGINE Bus STARTING 2024-08-17T16:52:20.369 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: CherryPy Checker: 2024-08-17T16:52:20.369 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: The Application mounted at '' has an empty config. 2024-08-17T16:52:20.369 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: 2024-08-17T16:52:20.369 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: [17/Aug/2024:16:52:20] ENGINE Serving on http://:::9283 2024-08-17T16:52:20.369 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:52:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[36556]: [17/Aug/2024:16:52:20] ENGINE Bus STARTED 2024-08-17T16:52:20.453 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:20.228+0000 7f5de7df6040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:52:20.453 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:20.395+0000 7f5de7df6040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:52:20.453 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:52:20] ENGINE Bus STARTING 2024-08-17T16:52:20.453 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-mon[42680]: Standby manager daemon x restarted 2024-08-17T16:52:20.453 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-mon[42680]: Standby manager daemon x started 2024-08-17T16:52:20.453 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/2850890665' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:52:20.454 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/2850890665' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:52:20.454 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/2850890665' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:52:20.454 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/2850890665' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:52:20.454 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-mon[31063]: Standby manager daemon x restarted 2024-08-17T16:52:20.454 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-mon[31063]: Standby manager daemon x started 2024-08-17T16:52:20.454 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/2850890665' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:52:20.454 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/2850890665' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:52:20.454 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/2850890665' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:52:20.455 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/2850890665' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:52:20.730 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: CherryPy Checker: 2024-08-17T16:52:20.730 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: The Application mounted at '' has an empty config. 2024-08-17T16:52:20.730 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:20.730 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:52:20] ENGINE Serving on http://:::9283 2024-08-17T16:52:20.730 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:20 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:52:20] ENGINE Bus STARTED 2024-08-17T16:52:21.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: mgrmap e17: y(active, since 4m), standbys: x 2024-08-17T16:52:21.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: Active manager daemon y restarted 2024-08-17T16:52:21.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: Activating manager daemon y 2024-08-17T16:52:21.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: osdmap e60: 8 total, 8 up, 8 in 2024-08-17T16:52:21.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: mgrmap e18: y(active, starting, since 0.00711688s), standbys: x 2024-08-17T16:52:21.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:52:21.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:21.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:21.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:52:21.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:52:21.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:21.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:52:21.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:52:21.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:52:21.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:52:21.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:52:21.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:52:21.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:52:21.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:52:21.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:52:21.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:52:21.532 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: Manager daemon y is now available 2024-08-17T16:52:21.532 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:21.532 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:21.532 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:52:21.532 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:52:21.532 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:52:21.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: mgrmap e17: y(active, since 4m), standbys: x 2024-08-17T16:52:21.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: Active manager daemon y restarted 2024-08-17T16:52:21.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: Activating manager daemon y 2024-08-17T16:52:21.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: osdmap e60: 8 total, 8 up, 8 in 2024-08-17T16:52:21.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: mgrmap e18: y(active, starting, since 0.00711688s), standbys: x 2024-08-17T16:52:21.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:52:21.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:21.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:21.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:52:21.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:52:21.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:21.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:52:21.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:52:21.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:52:21.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:52:21.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:52:21.534 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:52:21.534 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:52:21.534 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:52:21.534 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:52:21.534 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:52:21.534 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: Manager daemon y is now available 2024-08-17T16:52:21.534 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:21.534 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:21.534 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:52:21.534 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:52:21.534 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:52:21.586 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: mgrmap e17: y(active, since 4m), standbys: x 2024-08-17T16:52:21.586 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: Active manager daemon y restarted 2024-08-17T16:52:21.587 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: Activating manager daemon y 2024-08-17T16:52:21.587 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: osdmap e60: 8 total, 8 up, 8 in 2024-08-17T16:52:21.587 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: mgrmap e18: y(active, starting, since 0.00711688s), standbys: x 2024-08-17T16:52:21.587 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:52:21.587 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:52:21.587 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:52:21.587 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:52:21.587 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:52:21.587 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:52:21.587 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:52:21.587 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:52:21.587 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:52:21.588 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:52:21.588 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:52:21.588 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:52:21.588 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:52:21.588 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:52:21.588 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:52:21.588 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:52:21.588 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: Manager daemon y is now available 2024-08-17T16:52:21.588 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:52:21.588 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:21.588 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:52:21.588 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:52:21.588 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:52:22.279 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:22.036+0000 7f5d40519640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:16:52:22] ENGINE Error in HTTPServer.serve 2024-08-17T16:52:22.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: Traceback (most recent call last): 2024-08-17T16:52:22.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:52:22.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: self._connections.run(self.expiration_interval) 2024-08-17T16:52:22.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:52:22.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: self._run(expiration_interval) 2024-08-17T16:52:22.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:52:22.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:52:22.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:52:22.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:52:22.281 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:52:22.281 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: s = self.context.wrap_socket( 2024-08-17T16:52:22.281 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:52:22.281 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: return self.sslsocket_class._create( 2024-08-17T16:52:22.281 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:52:22.281 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: self.do_handshake() 2024-08-17T16:52:22.281 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:52:22.281 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: self._sslobj.do_handshake() 2024-08-17T16:52:22.281 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:52:22.281 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:52:22.425 INFO:teuthology.orchestra.run.smithi063.stderr:pool 'rbd' created 2024-08-17T16:52:22.728 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-mon[31063]: mgrmap e19: y(active, since 1.01134s), standbys: x 2024-08-17T16:52:22.728 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-mon[31063]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:22.728 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/1772059187' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T16:52:22.729 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-mon[31063]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T16:52:22.729 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-mon[42680]: mgrmap e19: y(active, since 1.01134s), standbys: x 2024-08-17T16:52:22.729 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-mon[42680]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:22.729 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/1772059187' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T16:52:22.729 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:22 smithi063 ceph-mon[42680]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T16:52:22.747 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:22 smithi162 ceph-mon[34633]: mgrmap e19: y(active, since 1.01134s), standbys: x 2024-08-17T16:52:22.747 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:22 smithi162 ceph-mon[34633]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:22.747 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:22 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/1772059187' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T16:52:22.747 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:22 smithi162 ceph-mon[34633]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-08-17T16:52:23.143 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- rbd pool init rbd 2024-08-17T16:52:23.407 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:23.418 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: [17/Aug/2024:16:52:21] ENGINE Bus STARTING 2024-08-17T16:52:23.418 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: [17/Aug/2024:16:52:21] ENGINE Serving on http://172.21.15.63:8765 2024-08-17T16:52:23.419 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: [17/Aug/2024:16:52:22] ENGINE Serving on https://172.21.15.63:7150 2024-08-17T16:52:23.419 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: [17/Aug/2024:16:52:22] ENGINE Bus STARTED 2024-08-17T16:52:23.419 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: [17/Aug/2024:16:52:22] ENGINE Error in HTTPServer.serve 2024-08-17T16:52:23.419 INFO:journalctl@ceph.mon.c.smithi063.stdout: Traceback (most recent call last): 2024-08-17T16:52:23.419 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:52:23.419 INFO:journalctl@ceph.mon.c.smithi063.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:52:23.419 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:52:23.419 INFO:journalctl@ceph.mon.c.smithi063.stdout: self._run(expiration_interval) 2024-08-17T16:52:23.419 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:52:23.419 INFO:journalctl@ceph.mon.c.smithi063.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:52:23.419 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:52:23.419 INFO:journalctl@ceph.mon.c.smithi063.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:52:23.420 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:52:23.420 INFO:journalctl@ceph.mon.c.smithi063.stdout: s = self.context.wrap_socket( 2024-08-17T16:52:23.420 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:52:23.420 INFO:journalctl@ceph.mon.c.smithi063.stdout: return self.sslsocket_class._create( 2024-08-17T16:52:23.420 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:52:23.420 INFO:journalctl@ceph.mon.c.smithi063.stdout: self.do_handshake() 2024-08-17T16:52:23.420 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:52:23.420 INFO:journalctl@ceph.mon.c.smithi063.stdout: self._sslobj.do_handshake() 2024-08-17T16:52:23.420 INFO:journalctl@ceph.mon.c.smithi063.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:52:23.420 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:23.420 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-08-17T16:52:23.421 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: osdmap e61: 8 total, 8 up, 8 in 2024-08-17T16:52:23.421 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: mgrmap e20: y(active, since 2s), standbys: x 2024-08-17T16:52:23.421 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.421 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.421 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.421 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.421 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.421 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.421 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:52:23.421 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:52:23.421 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: [17/Aug/2024:16:52:21] ENGINE Bus STARTING 2024-08-17T16:52:23.421 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: [17/Aug/2024:16:52:21] ENGINE Serving on http://172.21.15.63:8765 2024-08-17T16:52:23.422 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: [17/Aug/2024:16:52:22] ENGINE Serving on https://172.21.15.63:7150 2024-08-17T16:52:23.422 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: [17/Aug/2024:16:52:22] ENGINE Bus STARTED 2024-08-17T16:52:23.422 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: [17/Aug/2024:16:52:22] ENGINE Error in HTTPServer.serve 2024-08-17T16:52:23.422 INFO:journalctl@ceph.mon.a.smithi063.stdout: Traceback (most recent call last): 2024-08-17T16:52:23.422 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:52:23.422 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:52:23.422 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:52:23.422 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._run(expiration_interval) 2024-08-17T16:52:23.422 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:52:23.422 INFO:journalctl@ceph.mon.a.smithi063.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:52:23.422 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:52:23.422 INFO:journalctl@ceph.mon.a.smithi063.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:52:23.423 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:52:23.423 INFO:journalctl@ceph.mon.a.smithi063.stdout: s = self.context.wrap_socket( 2024-08-17T16:52:23.423 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:52:23.423 INFO:journalctl@ceph.mon.a.smithi063.stdout: return self.sslsocket_class._create( 2024-08-17T16:52:23.423 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:52:23.423 INFO:journalctl@ceph.mon.a.smithi063.stdout: self.do_handshake() 2024-08-17T16:52:23.423 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:52:23.423 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._sslobj.do_handshake() 2024-08-17T16:52:23.423 INFO:journalctl@ceph.mon.a.smithi063.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:52:23.423 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:23.423 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-08-17T16:52:23.423 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: osdmap e61: 8 total, 8 up, 8 in 2024-08-17T16:52:23.424 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: mgrmap e20: y(active, since 2s), standbys: x 2024-08-17T16:52:23.424 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.424 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.424 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.424 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.424 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.424 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.424 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:52:23.424 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:52:23.424 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:52:23.424 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:52:23.424 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.680 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:52:23.680 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:52:23.680 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:23 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: [17/Aug/2024:16:52:21] ENGINE Bus STARTING 2024-08-17T16:52:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: [17/Aug/2024:16:52:21] ENGINE Serving on http://172.21.15.63:8765 2024-08-17T16:52:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: [17/Aug/2024:16:52:22] ENGINE Serving on https://172.21.15.63:7150 2024-08-17T16:52:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: [17/Aug/2024:16:52:22] ENGINE Bus STARTED 2024-08-17T16:52:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: [17/Aug/2024:16:52:22] ENGINE Error in HTTPServer.serve 2024-08-17T16:52:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout: Traceback (most recent call last): 2024-08-17T16:52:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:52:23.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:52:23.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:52:23.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: self._run(expiration_interval) 2024-08-17T16:52:23.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:52:23.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:52:23.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:52:23.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:52:23.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:52:23.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: s = self.context.wrap_socket( 2024-08-17T16:52:23.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:52:23.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: return self.sslsocket_class._create( 2024-08-17T16:52:23.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:52:23.848 INFO:journalctl@ceph.mon.b.smithi162.stdout: self.do_handshake() 2024-08-17T16:52:23.848 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:52:23.848 INFO:journalctl@ceph.mon.b.smithi162.stdout: self._sslobj.do_handshake() 2024-08-17T16:52:23.848 INFO:journalctl@ceph.mon.b.smithi162.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:52:23.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:23.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-08-17T16:52:23.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: osdmap e61: 8 total, 8 up, 8 in 2024-08-17T16:52:23.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: mgrmap e20: y(active, since 2s), standbys: x 2024-08-17T16:52:23.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:23.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:52:23.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:52:23.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:52:23.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:52:23.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:23 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:24.613 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:24 smithi063 ceph-mon[42680]: Adjusting osd_memory_target on smithi162 to 4016M 2024-08-17T16:52:24.613 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:24 smithi063 ceph-mon[42680]: osdmap e62: 8 total, 8 up, 8 in 2024-08-17T16:52:24.613 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:24 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:24.613 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:24 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:24.613 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:24 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:24.613 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:24 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:24.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:24 smithi063 ceph-mon[31063]: Adjusting osd_memory_target on smithi162 to 4016M 2024-08-17T16:52:24.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:24 smithi063 ceph-mon[31063]: osdmap e62: 8 total, 8 up, 8 in 2024-08-17T16:52:24.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:24 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:24.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:24 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:24.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:24 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:24.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:24 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:24.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:24 smithi162 ceph-mon[34633]: Adjusting osd_memory_target on smithi162 to 4016M 2024-08-17T16:52:24.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:24 smithi162 ceph-mon[34633]: osdmap e62: 8 total, 8 up, 8 in 2024-08-17T16:52:24.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:24 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:24.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:24 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:24.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:24 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:52:24.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:24 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:52:25.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[42680]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:52:25.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[42680]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:52:25.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[42680]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:52:25.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[42680]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:52:25.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[42680]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:25.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[42680]: osdmap e63: 8 total, 8 up, 8 in 2024-08-17T16:52:25.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:25.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:25.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:25.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:25.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:25.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/1394495729' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-17T16:52:25.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[42680]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T16:52:25.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[42680]: mgrmap e21: y(active, since 4s), standbys: x 2024-08-17T16:52:25.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[31063]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:52:25.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[31063]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:52:25.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[31063]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:52:25.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[31063]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:52:25.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[31063]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:25.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[31063]: osdmap e63: 8 total, 8 up, 8 in 2024-08-17T16:52:25.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:25.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:25.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:25.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:25.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:25.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/1394495729' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-17T16:52:25.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[31063]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T16:52:25.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:25 smithi063 ceph-mon[31063]: mgrmap e21: y(active, since 4s), standbys: x 2024-08-17T16:52:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:25 smithi162 ceph-mon[34633]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:52:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:25 smithi162 ceph-mon[34633]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:52:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:25 smithi162 ceph-mon[34633]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:52:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:25 smithi162 ceph-mon[34633]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:52:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:25 smithi162 ceph-mon[34633]: pgmap v7: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:25 smithi162 ceph-mon[34633]: osdmap e63: 8 total, 8 up, 8 in 2024-08-17T16:52:25.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:25 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:25.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:25 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:25.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:25 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:25.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:25 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:25.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:25 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:25.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:25 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/1394495729' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-08-17T16:52:25.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:25 smithi162 ceph-mon[34633]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T16:52:25.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:25 smithi162 ceph-mon[34633]: mgrmap e21: y(active, since 4s), standbys: x 2024-08-17T16:52:26.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:26 smithi063 ceph-mon[42680]: Updating smithi162:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:52:26.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:26 smithi063 ceph-mon[42680]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:52:26.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:26 smithi063 ceph-mon[42680]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:52:26.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:26 smithi063 ceph-mon[42680]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:52:26.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:26 smithi063 ceph-mon[42680]: Deploying daemon alertmanager.a on smithi063 2024-08-17T16:52:26.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:26 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/1394495729' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-08-17T16:52:26.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:26 smithi063 ceph-mon[42680]: osdmap e64: 8 total, 8 up, 8 in 2024-08-17T16:52:26.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:26 smithi063 ceph-mon[31063]: Updating smithi162:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:52:26.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:26 smithi063 ceph-mon[31063]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:52:26.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:26 smithi063 ceph-mon[31063]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:52:26.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:26 smithi063 ceph-mon[31063]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:52:26.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:26 smithi063 ceph-mon[31063]: Deploying daemon alertmanager.a on smithi063 2024-08-17T16:52:26.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:26 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/1394495729' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-08-17T16:52:26.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:26 smithi063 ceph-mon[31063]: osdmap e64: 8 total, 8 up, 8 in 2024-08-17T16:52:26.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:26 smithi162 ceph-mon[34633]: Updating smithi162:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:52:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:26 smithi162 ceph-mon[34633]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:52:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:26 smithi162 ceph-mon[34633]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:52:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:26 smithi162 ceph-mon[34633]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:52:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:26 smithi162 ceph-mon[34633]: Deploying daemon alertmanager.a on smithi063 2024-08-17T16:52:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:26 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/1394495729' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-08-17T16:52:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:26 smithi162 ceph-mon[34633]: osdmap e64: 8 total, 8 up, 8 in 2024-08-17T16:52:27.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:27 smithi063 ceph-mon[42680]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:27.787 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:27 smithi063 ceph-mon[42680]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T16:52:27.788 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:27 smithi063 ceph-mon[42680]: Cluster is now healthy 2024-08-17T16:52:27.788 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:27 smithi063 ceph-mon[42680]: osdmap e65: 8 total, 8 up, 8 in 2024-08-17T16:52:27.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:27 smithi063 ceph-mon[31063]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:27.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:27 smithi063 ceph-mon[31063]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T16:52:27.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:27 smithi063 ceph-mon[31063]: Cluster is now healthy 2024-08-17T16:52:27.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:27 smithi063 ceph-mon[31063]: osdmap e65: 8 total, 8 up, 8 in 2024-08-17T16:52:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:27 smithi162 ceph-mon[34633]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:27 smithi162 ceph-mon[34633]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T16:52:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:27 smithi162 ceph-mon[34633]: Cluster is now healthy 2024-08-17T16:52:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:27 smithi162 ceph-mon[34633]: osdmap e65: 8 total, 8 up, 8 in 2024-08-17T16:52:29.466 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:29 smithi063 ceph-mon[42680]: osdmap e66: 8 total, 8 up, 8 in 2024-08-17T16:52:29.466 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:29 smithi063 ceph-mon[42680]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 28 op/s 2024-08-17T16:52:29.466 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:29 smithi063 ceph-mon[31063]: osdmap e66: 8 total, 8 up, 8 in 2024-08-17T16:52:29.466 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:29 smithi063 ceph-mon[31063]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 28 op/s 2024-08-17T16:52:29.488 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph config log 1 --format=json 2024-08-17T16:52:29.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:29 smithi162 ceph-mon[34633]: osdmap e66: 8 total, 8 up, 8 in 2024-08-17T16:52:29.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:29 smithi162 ceph-mon[34633]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 28 op/s 2024-08-17T16:52:29.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:29 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:52:29] "GET /metrics HTTP/1.1" 200 23671 "" "Prometheus/2.43.0" 2024-08-17T16:52:30.515 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:30.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:30 smithi063 ceph-mon[42680]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-08-17T16:52:30.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:30 smithi063 ceph-mon[31063]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-08-17T16:52:30.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:30 smithi162 ceph-mon[34633]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-08-17T16:52:32.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:32 smithi063 ceph-mon[42680]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 323 B/s wr, 18 op/s 2024-08-17T16:52:32.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:32 smithi063 ceph-mon[31063]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 323 B/s wr, 18 op/s 2024-08-17T16:52:32.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:32 smithi162 ceph-mon[34633]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 323 B/s wr, 18 op/s 2024-08-17T16:52:33.591 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:52:33.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:33 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/1177707188' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-08-17T16:52:33.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:33 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/1177707188' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-08-17T16:52:33.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:33 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/1177707188' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-08-17T16:52:34.650 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:34 smithi063 systemd[1]: Starting Ceph alertmanager.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:52:34.741 INFO:teuthology.orchestra.run.smithi063.stdout:[{"version":23,"timestamp":"2024-08-17T16:52:23.359312+0000","name":"","changes":[{"name":"osd/host:smithi162/osd_memory_target","previous_value":"4479564083","new_value":"4211128627"}]}] 2024-08-17T16:52:34.741 INFO:tasks.ceph_manager:config epoch is 23 2024-08-17T16:52:34.741 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-17T16:52:34.742 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-17T16:52:34.742 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph mgr dump --format=json 2024-08-17T16:52:34.942 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:34 smithi063 ceph-mon[42680]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 255 B/s wr, 14 op/s 2024-08-17T16:52:34.942 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:34 smithi063 ceph-mon[31063]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 255 B/s wr, 14 op/s 2024-08-17T16:52:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:34 smithi162 ceph-mon[34633]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 255 B/s wr, 14 op/s 2024-08-17T16:52:35.241 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:34 smithi063 podman[73453]: 2024-08-17 16:52:34.84405965 +0000 UTC m=+0.018294556 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-08-17T16:52:35.242 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:34 smithi063 podman[73453]: 2024-08-17 16:52:34.980571005 +0000 UTC m=+0.154805914 volume create f4a820f68039d8cfdfcaacb7bdbf2e69ca45a203484f3b1d3e622dbcab15e961 2024-08-17T16:52:35.242 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:35 smithi063 podman[73453]: 2024-08-17 16:52:35.054084587 +0000 UTC m=+0.228319554 container create 80384178e8bd35687c8694eafc3e3e0830b0592272e7dca71f068d2d6c0c7b39 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:52:35.496 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:35.625 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:35 smithi063 podman[73453]: 2024-08-17 16:52:35.35666476 +0000 UTC m=+0.530899680 container init 80384178e8bd35687c8694eafc3e3e0830b0592272e7dca71f068d2d6c0c7b39 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:52:35.625 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:35 smithi063 podman[73453]: 2024-08-17 16:52:35.362052843 +0000 UTC m=+0.536287749 container start 80384178e8bd35687c8694eafc3e3e0830b0592272e7dca71f068d2d6c0c7b39 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:52:35.625 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[73479]: ts=2024-08-17T16:52:35.375Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-08-17T16:52:35.625 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[73479]: ts=2024-08-17T16:52:35.375Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-08-17T16:52:35.625 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[73479]: ts=2024-08-17T16:52:35.376Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.63 port=9094 2024-08-17T16:52:35.625 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[73479]: ts=2024-08-17T16:52:35.377Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-17T16:52:35.625 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[73479]: ts=2024-08-17T16:52:35.400Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T16:52:35.625 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[73479]: ts=2024-08-17T16:52:35.401Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T16:52:35.625 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[73479]: ts=2024-08-17T16:52:35.403Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-08-17T16:52:35.625 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[73479]: ts=2024-08-17T16:52:35.403Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-17T16:52:35.626 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:35 smithi063 bash[73453]: 80384178e8bd35687c8694eafc3e3e0830b0592272e7dca71f068d2d6c0c7b39 2024-08-17T16:52:35.626 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:35 smithi063 systemd[1]: Started Ceph alertmanager.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:52:35.893 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:52:35] ENGINE Bus STOPPING 2024-08-17T16:52:35.894 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[42680]: osdmap e67: 8 total, 8 up, 8 in 2024-08-17T16:52:35.894 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:35.894 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:52:35.894 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:35.894 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:35.894 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:35.894 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:35.894 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:35.894 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:35.895 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T16:52:35.895 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:35.895 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[31063]: osdmap e67: 8 total, 8 up, 8 in 2024-08-17T16:52:35.895 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:35.895 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:52:35.895 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:35.895 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:35.895 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:35.895 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:35.895 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:35.896 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:35.896 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T16:52:35.896 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:35 smithi162 ceph-mon[34633]: osdmap e67: 8 total, 8 up, 8 in 2024-08-17T16:52:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:52:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T16:52:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:36.199 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:52:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T16:52:36.199 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:52:36] ENGINE Bus STOPPED 2024-08-17T16:52:36.199 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:52:36] ENGINE Bus STARTING 2024-08-17T16:52:36.199 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:52:36] ENGINE Serving on http://:::9283 2024-08-17T16:52:36.199 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:52:36] ENGINE Bus STARTED 2024-08-17T16:52:36.928 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:36 smithi063 ceph-mon[31063]: Regenerating cephadm self-signed grafana TLS certificates 2024-08-17T16:52:36.928 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:36 smithi063 ceph-mon[31063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T16:52:36.928 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:36 smithi063 ceph-mon[31063]: Deploying daemon grafana.a on smithi162 2024-08-17T16:52:36.928 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:36 smithi063 ceph-mon[31063]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 246 B/s wr, 0 op/s 2024-08-17T16:52:36.928 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:36 smithi063 ceph-mon[42680]: Regenerating cephadm self-signed grafana TLS certificates 2024-08-17T16:52:36.928 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:36 smithi063 ceph-mon[42680]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T16:52:36.928 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:36 smithi063 ceph-mon[42680]: Deploying daemon grafana.a on smithi162 2024-08-17T16:52:36.929 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:36 smithi063 ceph-mon[42680]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 246 B/s wr, 0 op/s 2024-08-17T16:52:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:36 smithi162 ceph-mon[34633]: Regenerating cephadm self-signed grafana TLS certificates 2024-08-17T16:52:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:36 smithi162 ceph-mon[34633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-08-17T16:52:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:36 smithi162 ceph-mon[34633]: Deploying daemon grafana.a on smithi162 2024-08-17T16:52:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:36 smithi162 ceph-mon[34633]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 246 B/s wr, 0 op/s 2024-08-17T16:52:37.371 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:52:37.696 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:37 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[73479]: ts=2024-08-17T16:52:37.378Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000918665s 2024-08-17T16:52:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:37 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/2098042945' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T16:52:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:37 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/2098042945' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T16:52:38.058 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":21,"flags":0,"active_gid":24335,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6800","nonce":987519051},{"type":"v1","addr":"172.21.15.63:6801","nonce":987519051}]},"active_addr":"172.21.15.63:6801/987519051","active_change":"2024-08-17T16:52:20.398656+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24352,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.63:8443/","prometheus":"http://172.21.15.63:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":2696979598}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":3589494813}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":3746443558}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":4067771896}]}]} 2024-08-17T16:52:38.062 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-17T16:52:38.062 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T16:52:38.062 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T16:52:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:37 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/2098042945' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T16:52:38.283 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:38 smithi063 ceph-mon[42680]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-08-17T16:52:39.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:38 smithi063 ceph-mon[31063]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-08-17T16:52:39.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:38 smithi162 ceph-mon[34633]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-08-17T16:52:39.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:52:39] "GET /metrics HTTP/1.1" 200 28503 "" "Prometheus/2.43.0" 2024-08-17T16:52:39.955 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:52:39.955 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":67,"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:46:54.156006+0000","modified":"2024-08-17T16:52:34.776752+0000","last_up_change":"2024-08-17T16:51:48.092300+0000","last_in_change":"2024-08-17T16:51:30.856523+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:49:54.431240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:52:22.131465+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6802","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6803","nonce":782866097}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6804","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6805","nonce":782866097}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6808","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6809","nonce":782866097}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6806","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6807","nonce":782866097}]},"public_addr":"172.21.15.63:6803/782866097","cluster_addr":"172.21.15.63:6805/782866097","heartbeat_back_addr":"172.21.15.63:6809/782866097","heartbeat_front_addr":"172.21.15.63:6807/782866097","state":["exists","up"]},{"osd":1,"uuid":"ae346e99-4706-4871-990a-f8e737572245","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6810","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6811","nonce":944260396}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6812","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6813","nonce":944260396}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6816","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6817","nonce":944260396}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6814","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6815","nonce":944260396}]},"public_addr":"172.21.15.63:6811/944260396","cluster_addr":"172.21.15.63:6813/944260396","heartbeat_back_addr":"172.21.15.63:6817/944260396","heartbeat_front_addr":"172.21.15.63:6815/944260396","state":["exists","up"]},{"osd":2,"uuid":"2d85d75f-1280-443f-857d-d322bd60c451","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6818","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6819","nonce":3751946332}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6820","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6821","nonce":3751946332}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6824","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6825","nonce":3751946332}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6822","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6823","nonce":3751946332}]},"public_addr":"172.21.15.63:6819/3751946332","cluster_addr":"172.21.15.63:6821/3751946332","heartbeat_back_addr":"172.21.15.63:6825/3751946332","heartbeat_front_addr":"172.21.15.63:6823/3751946332","state":["exists","up"]},{"osd":3,"uuid":"771f43d2-e98b-43e0-921b-88e56fadda42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6826","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6827","nonce":2654575936}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6828","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6829","nonce":2654575936}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6832","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6833","nonce":2654575936}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6830","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6831","nonce":2654575936}]},"public_addr":"172.21.15.63:6827/2654575936","cluster_addr":"172.21.15.63:6829/2654575936","heartbeat_back_addr":"172.21.15.63:6833/2654575936","heartbeat_front_addr":"172.21.15.63:6831/2654575936","state":["exists","up"]},{"osd":4,"uuid":"73c3c545-e975-4a6f-a1cb-e4575ee7a9b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6800","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6801","nonce":1563885208}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6803","nonce":1563885208}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6807","nonce":1563885208}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6805","nonce":1563885208}]},"public_addr":"172.21.15.162:6801/1563885208","cluster_addr":"172.21.15.162:6803/1563885208","heartbeat_back_addr":"172.21.15.162:6807/1563885208","heartbeat_front_addr":"172.21.15.162:6805/1563885208","state":["exists","up"]},{"osd":5,"uuid":"fae3ac52-3194-4eb4-9510-e3899768353a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6809","nonce":1826502179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6810","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6811","nonce":1826502179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6814","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6815","nonce":1826502179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6812","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6813","nonce":1826502179}]},"public_addr":"172.21.15.162:6809/1826502179","cluster_addr":"172.21.15.162:6811/1826502179","heartbeat_back_addr":"172.21.15.162:6815/1826502179","heartbeat_front_addr":"172.21.15.162:6813/1826502179","state":["exists","up"]},{"osd":6,"uuid":"a8255b37-1be5-4cd4-9b86-889f6bf52902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6816","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6817","nonce":1675904899}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6818","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6819","nonce":1675904899}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6822","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6823","nonce":1675904899}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6820","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6821","nonce":1675904899}]},"public_addr":"172.21.15.162:6817/1675904899","cluster_addr":"172.21.15.162:6819/1675904899","heartbeat_back_addr":"172.21.15.162:6823/1675904899","heartbeat_front_addr":"172.21.15.162:6821/1675904899","state":["exists","up"]},{"osd":7,"uuid":"3d0bcd6d-171d-4d90-a033-ed17714a9d96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6824","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6825","nonce":2938230961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6826","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6827","nonce":2938230961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6830","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6831","nonce":2938230961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6828","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6829","nonce":2938230961}]},"public_addr":"172.21.15.162:6825/2938230961","cluster_addr":"172.21.15.162:6827/2938230961","heartbeat_back_addr":"172.21.15.162:6831/2938230961","heartbeat_front_addr":"172.21.15.162:6829/2938230961","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:02.667962+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:26.166590+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:50.153794+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:13.556156+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:34.827805+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:58.789701+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:22.650398+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:46.166424+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.63:0/701508812":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/1256300214":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/2601197458":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/84364348":"2024-08-18T16:47:16.384724+0000","172.21.15.63:0/3004218255":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6800/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6801/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6801/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3540337558":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6800/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/1792393387":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3226753153":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/1986138219":"2024-08-18T16:47:39.919472+0000","172.21.15.63:6800/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.63:6801/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/955244129":"2024-08-18T16:52:20.398501+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T16:52:40.249 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:40 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3752468476' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:52:40.249 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:40 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3752468476' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:52:40.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:40 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3752468476' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:52:40.548 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T16:52:40.548 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T16:52:40.758 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:41.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:41 smithi063 ceph-mon[42680]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-08-17T16:52:41.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:41 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:41.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:41 smithi063 ceph-mon[31063]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-08-17T16:52:41.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:41 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:41.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:41 smithi162 ceph-mon[34633]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-08-17T16:52:41.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:41 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:42.628 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:52:42.628 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":67,"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:46:54.156006+0000","modified":"2024-08-17T16:52:34.776752+0000","last_up_change":"2024-08-17T16:51:48.092300+0000","last_in_change":"2024-08-17T16:51:30.856523+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:49:54.431240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:52:22.131465+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6802","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6803","nonce":782866097}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6804","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6805","nonce":782866097}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6808","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6809","nonce":782866097}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6806","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6807","nonce":782866097}]},"public_addr":"172.21.15.63:6803/782866097","cluster_addr":"172.21.15.63:6805/782866097","heartbeat_back_addr":"172.21.15.63:6809/782866097","heartbeat_front_addr":"172.21.15.63:6807/782866097","state":["exists","up"]},{"osd":1,"uuid":"ae346e99-4706-4871-990a-f8e737572245","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6810","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6811","nonce":944260396}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6812","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6813","nonce":944260396}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6816","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6817","nonce":944260396}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6814","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6815","nonce":944260396}]},"public_addr":"172.21.15.63:6811/944260396","cluster_addr":"172.21.15.63:6813/944260396","heartbeat_back_addr":"172.21.15.63:6817/944260396","heartbeat_front_addr":"172.21.15.63:6815/944260396","state":["exists","up"]},{"osd":2,"uuid":"2d85d75f-1280-443f-857d-d322bd60c451","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6818","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6819","nonce":3751946332}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6820","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6821","nonce":3751946332}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6824","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6825","nonce":3751946332}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6822","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6823","nonce":3751946332}]},"public_addr":"172.21.15.63:6819/3751946332","cluster_addr":"172.21.15.63:6821/3751946332","heartbeat_back_addr":"172.21.15.63:6825/3751946332","heartbeat_front_addr":"172.21.15.63:6823/3751946332","state":["exists","up"]},{"osd":3,"uuid":"771f43d2-e98b-43e0-921b-88e56fadda42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6826","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6827","nonce":2654575936}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6828","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6829","nonce":2654575936}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6832","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6833","nonce":2654575936}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6830","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6831","nonce":2654575936}]},"public_addr":"172.21.15.63:6827/2654575936","cluster_addr":"172.21.15.63:6829/2654575936","heartbeat_back_addr":"172.21.15.63:6833/2654575936","heartbeat_front_addr":"172.21.15.63:6831/2654575936","state":["exists","up"]},{"osd":4,"uuid":"73c3c545-e975-4a6f-a1cb-e4575ee7a9b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6800","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6801","nonce":1563885208}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6803","nonce":1563885208}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6807","nonce":1563885208}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6805","nonce":1563885208}]},"public_addr":"172.21.15.162:6801/1563885208","cluster_addr":"172.21.15.162:6803/1563885208","heartbeat_back_addr":"172.21.15.162:6807/1563885208","heartbeat_front_addr":"172.21.15.162:6805/1563885208","state":["exists","up"]},{"osd":5,"uuid":"fae3ac52-3194-4eb4-9510-e3899768353a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6809","nonce":1826502179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6810","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6811","nonce":1826502179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6814","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6815","nonce":1826502179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6812","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6813","nonce":1826502179}]},"public_addr":"172.21.15.162:6809/1826502179","cluster_addr":"172.21.15.162:6811/1826502179","heartbeat_back_addr":"172.21.15.162:6815/1826502179","heartbeat_front_addr":"172.21.15.162:6813/1826502179","state":["exists","up"]},{"osd":6,"uuid":"a8255b37-1be5-4cd4-9b86-889f6bf52902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6816","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6817","nonce":1675904899}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6818","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6819","nonce":1675904899}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6822","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6823","nonce":1675904899}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6820","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6821","nonce":1675904899}]},"public_addr":"172.21.15.162:6817/1675904899","cluster_addr":"172.21.15.162:6819/1675904899","heartbeat_back_addr":"172.21.15.162:6823/1675904899","heartbeat_front_addr":"172.21.15.162:6821/1675904899","state":["exists","up"]},{"osd":7,"uuid":"3d0bcd6d-171d-4d90-a033-ed17714a9d96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6824","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6825","nonce":2938230961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6826","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6827","nonce":2938230961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6830","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6831","nonce":2938230961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6828","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6829","nonce":2938230961}]},"public_addr":"172.21.15.162:6825/2938230961","cluster_addr":"172.21.15.162:6827/2938230961","heartbeat_back_addr":"172.21.15.162:6831/2938230961","heartbeat_front_addr":"172.21.15.162:6829/2938230961","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:02.667962+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:26.166590+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:50.153794+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:13.556156+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:34.827805+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:58.789701+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:22.650398+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:46.166424+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.63:0/701508812":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/1256300214":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/2601197458":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/84364348":"2024-08-18T16:47:16.384724+0000","172.21.15.63:0/3004218255":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6800/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6801/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6801/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3540337558":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6800/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/1792393387":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3226753153":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/1986138219":"2024-08-18T16:47:39.919472+0000","172.21.15.63:6800/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.63:6801/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/955244129":"2024-08-18T16:52:20.398501+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T16:52:42.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:42 smithi063 ceph-mon[42680]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:42.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:42 smithi063 ceph-mon[31063]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:42.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:42 smithi162 ceph-mon[34633]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:43.245 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T16:52:43.245 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T16:52:43.245 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T16:52:43.246 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T16:52:43.246 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T16:52:43.247 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T16:52:43.247 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T16:52:43.247 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T16:52:43.638 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:43.639 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:43.656 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:43.662 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:43.663 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:43.665 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:43.680 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:43.701 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:52:43.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:43 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3495911779' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:52:43.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:43 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3495911779' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:52:43.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:43 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3495911779' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:52:44.612 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:44 smithi063 ceph-mon[42680]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:44.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:44 smithi063 ceph-mon[31063]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:44.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:44 smithi162 ceph-mon[34633]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:45.645 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:52:45 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[73479]: ts=2024-08-17T16:52:45.381Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.00412881s 2024-08-17T16:52:46.652 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:46 smithi063 ceph-mon[42680]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:46.652 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:46 smithi063 ceph-mon[31063]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:46.759 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:46 smithi162 ceph-mon[34633]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:48.283 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 systemd[1]: Starting Ceph grafana.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:52:48.570 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:48 smithi063 ceph-mon[31063]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:48.571 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:48 smithi063 ceph-mon[42680]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:48.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-mon[34633]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:48.596 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 podman[64944]: 2024-08-17 16:52:48.403108693 +0000 UTC m=+0.016430445 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-08-17T16:52:48.596 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 podman[64944]: 2024-08-17 16:52:48.509530999 +0000 UTC m=+0.122852723 container create fc8d623290125c18c7b1c27ec7151f45858bdfc601b198fc4760e73d9df95ac0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a, architecture=x86_64, io.openshift.tags=base rhel8, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, vcs-type=git) 2024-08-17T16:52:48.954 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 podman[64944]: 2024-08-17 16:52:48.703432378 +0000 UTC m=+0.316754116 container init fc8d623290125c18c7b1c27ec7151f45858bdfc601b198fc4760e73d9df95ac0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, name=ubi8, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., io.openshift.expose-services=, description=Ceph Grafana Container) 2024-08-17T16:52:48.954 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 podman[64944]: 2024-08-17 16:52:48.708643729 +0000 UTC m=+0.321965455 container start fc8d623290125c18c7b1c27ec7151f45858bdfc601b198fc4760e73d9df95ac0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a, io.openshift.tags=base rhel8, name=ubi8, vcs-type=git, distribution-scope=public, com.redhat.component=ubi8-container, description=Ceph Grafana Container, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0) 2024-08-17T16:52:48.955 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 bash[64944]: fc8d623290125c18c7b1c27ec7151f45858bdfc601b198fc4760e73d9df95ac0 2024-08-17T16:52:48.955 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 systemd[1]: Started Ceph grafana.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:52:48.955 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=settings t=2024-08-17T16:52:48.873913081Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-08-17T16:52:48.955 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=settings t=2024-08-17T16:52:48.874096077Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-08-17T16:52:48.955 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=settings t=2024-08-17T16:52:48.874114262Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-08-17T16:52:48.955 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=settings t=2024-08-17T16:52:48.874121211Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-08-17T16:52:48.955 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=settings t=2024-08-17T16:52:48.874127322Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-08-17T16:52:48.956 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=settings t=2024-08-17T16:52:48.874134102Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-08-17T16:52:48.956 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=settings t=2024-08-17T16:52:48.874139867Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-08-17T16:52:48.956 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=settings t=2024-08-17T16:52:48.874145844Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-08-17T16:52:48.956 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=settings t=2024-08-17T16:52:48.874151886Z level=info msg="Path Home" path=/usr/share/grafana 2024-08-17T16:52:48.956 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=settings t=2024-08-17T16:52:48.874157664Z level=info msg="Path Data" path=/var/lib/grafana 2024-08-17T16:52:48.956 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=settings t=2024-08-17T16:52:48.87416309Z level=info msg="Path Logs" path=/var/log/grafana 2024-08-17T16:52:48.956 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=settings t=2024-08-17T16:52:48.87416858Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-08-17T16:52:48.956 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=settings t=2024-08-17T16:52:48.874174263Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-08-17T16:52:48.956 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=settings t=2024-08-17T16:52:48.874179994Z level=info msg="App mode production" 2024-08-17T16:52:48.956 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=sqlstore t=2024-08-17T16:52:48.874224581Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-08-17T16:52:48.956 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=sqlstore t=2024-08-17T16:52:48.874240671Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-08-17T16:52:48.957 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.875084505Z level=info msg="Starting DB migrations" 2024-08-17T16:52:48.957 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.875174167Z level=info msg="Executing migration" id="create migration_log table" 2024-08-17T16:52:48.957 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.875995672Z level=info msg="Executing migration" id="create user table" 2024-08-17T16:52:48.957 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.876438272Z level=info msg="Executing migration" id="add unique index user.login" 2024-08-17T16:52:48.957 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.876931591Z level=info msg="Executing migration" id="add unique index user.email" 2024-08-17T16:52:48.957 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.877773444Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-08-17T16:52:48.957 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.878607409Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-08-17T16:52:48.957 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.879511404Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-08-17T16:52:48.957 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.880293145Z level=info msg="Executing migration" id="create user table v2" 2024-08-17T16:52:48.957 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.880716936Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-08-17T16:52:48.957 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.881142629Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-08-17T16:52:48.957 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.88162009Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-08-17T16:52:48.958 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.881974677Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-08-17T16:52:48.958 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.884429429Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-08-17T16:52:48.958 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.884917929Z level=info msg="Executing migration" id="Update user table charset" 2024-08-17T16:52:48.958 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.885282791Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-08-17T16:52:48.958 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.885684383Z level=info msg="Executing migration" id="Add missing user data" 2024-08-17T16:52:48.958 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.886038901Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-08-17T16:52:48.958 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.886463774Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-08-17T16:52:48.958 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.886842026Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-08-17T16:52:48.958 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.887280499Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-08-17T16:52:48.958 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.890645135Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-08-17T16:52:48.958 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.891188422Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-08-17T16:52:48.958 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.891636017Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-08-17T16:52:48.958 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.892044069Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-08-17T16:52:48.959 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.892450051Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-08-17T16:52:48.959 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.892836111Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-08-17T16:52:48.959 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.893140775Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-08-17T16:52:48.959 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.893492071Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-08-17T16:52:48.960 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.893837734Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-08-17T16:52:48.960 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.894880165Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-08-17T16:52:48.960 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.895306647Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-08-17T16:52:48.960 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.896018352Z level=info msg="Executing migration" id="create temp_user v2" 2024-08-17T16:52:48.960 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.896425254Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-08-17T16:52:48.960 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.896811076Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-08-17T16:52:48.960 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.897236267Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-08-17T16:52:48.960 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.897610705Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-08-17T16:52:48.960 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.898015291Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-08-17T16:52:48.960 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.898365118Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-08-17T16:52:48.960 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.898689465Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-08-17T16:52:48.960 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.899037836Z level=info msg="Executing migration" id="create star table" 2024-08-17T16:52:48.961 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.899393975Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-08-17T16:52:48.961 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.899800981Z level=info msg="Executing migration" id="create org table v1" 2024-08-17T16:52:48.961 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.900189882Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-08-17T16:52:48.961 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.900563844Z level=info msg="Executing migration" id="create org_user table v1" 2024-08-17T16:52:48.961 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.900938397Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-08-17T16:52:48.961 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.901337482Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-08-17T16:52:48.961 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.901737697Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-08-17T16:52:48.961 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.90213238Z level=info msg="Executing migration" id="Update org table charset" 2024-08-17T16:52:48.961 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.902964247Z level=info msg="Executing migration" id="Update org_user table charset" 2024-08-17T16:52:48.961 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.903814062Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-08-17T16:52:48.961 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.904715461Z level=info msg="Executing migration" id="create dashboard table" 2024-08-17T16:52:48.961 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.905425437Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-08-17T16:52:48.961 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.906432503Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-08-17T16:52:48.962 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.907114966Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-08-17T16:52:48.962 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.907949376Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-08-17T16:52:48.962 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.908787368Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-08-17T16:52:48.962 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.909697923Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-08-17T16:52:48.962 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.910650986Z level=info msg="Executing migration" id="create dashboard v2" 2024-08-17T16:52:48.962 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.911491397Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-08-17T16:52:48.962 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.912355492Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-08-17T16:52:48.962 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.913182551Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-08-17T16:52:48.962 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.914059565Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-08-17T16:52:48.962 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.915065294Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-08-17T16:52:48.962 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.915712931Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-08-17T16:52:48.962 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.916572796Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-08-17T16:52:48.963 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.91745939Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-08-17T16:52:48.963 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.91836106Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-08-17T16:52:48.963 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.919312091Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-08-17T16:52:48.963 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.920020143Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-08-17T16:52:48.963 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.92089923Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-08-17T16:52:48.963 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.921734605Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-08-17T16:52:48.963 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.92269184Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-08-17T16:52:48.963 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.923435559Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-08-17T16:52:48.963 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.924175859Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-08-17T16:52:48.963 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.924758651Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-08-17T16:52:48.963 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.925683595Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-08-17T16:52:48.963 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.926656636Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-08-17T16:52:48.963 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.92738587Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-08-17T16:52:48.964 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.928275135Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-08-17T16:52:48.964 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.929096218Z level=info msg="Executing migration" id="Update dashboard title length" 2024-08-17T16:52:48.964 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.930065768Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-08-17T16:52:48.964 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.930873895Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-08-17T16:52:48.964 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.931741409Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-08-17T16:52:48.964 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.934423348Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-08-17T16:52:48.964 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.937195236Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-08-17T16:52:48.964 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.940052375Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-08-17T16:52:48.964 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.94294499Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-08-17T16:52:48.964 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.945677958Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-08-17T16:52:48.964 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.948372263Z level=info msg="Executing migration" id="Add check_sum column" 2024-08-17T16:52:48.964 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.951157965Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-08-17T16:52:48.965 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.953954235Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-08-17T16:52:48.965 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.956721031Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-08-17T16:52:49.206 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.959362282Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-08-17T16:52:49.206 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.962291587Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-08-17T16:52:49.206 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.965180974Z level=info msg="Executing migration" id="create data_source table" 2024-08-17T16:52:49.206 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.96792787Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-08-17T16:52:49.206 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.970686204Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-08-17T16:52:49.206 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.971211419Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-08-17T16:52:49.206 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.973973798Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-08-17T16:52:49.206 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.976936388Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-08-17T16:52:49.206 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.979789749Z level=info msg="Executing migration" id="create data_source table v2" 2024-08-17T16:52:49.207 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.98226378Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-08-17T16:52:49.207 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.984851731Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-08-17T16:52:49.207 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.987690394Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-08-17T16:52:49.207 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.990354117Z level=info msg="Executing migration" id="Add column with_credentials" 2024-08-17T16:52:49.207 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.992842681Z level=info msg="Executing migration" id="Add secure json data column" 2024-08-17T16:52:49.207 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.995661521Z level=info msg="Executing migration" id="Update data_source table charset" 2024-08-17T16:52:49.207 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:48 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:48.998230392Z level=info msg="Executing migration" id="Update initial version to 1" 2024-08-17T16:52:49.207 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.000802536Z level=info msg="Executing migration" id="Add read_only data column" 2024-08-17T16:52:49.207 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.003541701Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-08-17T16:52:49.207 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.006218342Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-08-17T16:52:49.208 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.008972335Z level=info msg="Executing migration" id="Add uid column" 2024-08-17T16:52:49.208 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.011501616Z level=info msg="Executing migration" id="Update uid value" 2024-08-17T16:52:49.209 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.014025521Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-08-17T16:52:49.209 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.016785823Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-08-17T16:52:49.209 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.019445463Z level=info msg="Executing migration" id="create api_key table" 2024-08-17T16:52:49.209 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.022022178Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-08-17T16:52:49.209 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.024684361Z level=info msg="Executing migration" id="add index api_key.key" 2024-08-17T16:52:49.209 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.027309325Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-08-17T16:52:49.209 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.028077479Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-08-17T16:52:49.209 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.028989198Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-08-17T16:52:49.209 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.029472613Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-08-17T16:52:49.209 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.030318959Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-08-17T16:52:49.209 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.03160236Z level=info msg="Executing migration" id="create api_key table v2" 2024-08-17T16:52:49.209 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.03201863Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-08-17T16:52:49.210 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.032464428Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-08-17T16:52:49.210 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.032896565Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-08-17T16:52:49.210 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.033662962Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-08-17T16:52:49.210 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.034521394Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-08-17T16:52:49.210 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.03541673Z level=info msg="Executing migration" id="Update api_key table charset" 2024-08-17T16:52:49.210 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.036090749Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-08-17T16:52:49.210 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.036950173Z level=info msg="Executing migration" id="Add service account foreign key" 2024-08-17T16:52:49.210 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.037788918Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-08-17T16:52:49.210 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.038735154Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-08-17T16:52:49.210 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.039469404Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-08-17T16:52:49.210 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.042080938Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-08-17T16:52:49.210 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.044789309Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-08-17T16:52:49.210 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.047809067Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-08-17T16:52:49.211 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.048639192Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-08-17T16:52:49.211 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.051471903Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-08-17T16:52:49.211 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.054132992Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-08-17T16:52:49.211 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.057145398Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-08-17T16:52:49.211 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.057662848Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-08-17T16:52:49.211 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.058443753Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-08-17T16:52:49.211 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.05912872Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-08-17T16:52:49.211 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.05972095Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-08-17T16:52:49.211 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.060559296Z level=info msg="Executing migration" id="create quota table v1" 2024-08-17T16:52:49.212 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.060997268Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-08-17T16:52:49.212 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.061854833Z level=info msg="Executing migration" id="Update quota table charset" 2024-08-17T16:52:49.212 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.06267651Z level=info msg="Executing migration" id="create plugin_setting table" 2024-08-17T16:52:49.212 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.063368922Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-08-17T16:52:49.212 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.063782147Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-08-17T16:52:49.212 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.06457675Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-08-17T16:52:49.212 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.065483574Z level=info msg="Executing migration" id="create session table" 2024-08-17T16:52:49.212 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.066261423Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-08-17T16:52:49.212 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.066571417Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-08-17T16:52:49.212 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.067412922Z level=info msg="Executing migration" id="create playlist table v2" 2024-08-17T16:52:49.212 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.067814651Z level=info msg="Executing migration" id="create playlist item table v2" 2024-08-17T16:52:49.212 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.068265914Z level=info msg="Executing migration" id="Update playlist table charset" 2024-08-17T16:52:49.213 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.069161985Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-08-17T16:52:49.213 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.069691766Z level=info msg="Executing migration" id="drop preferences table v2" 2024-08-17T16:52:49.213 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.069956621Z level=info msg="Executing migration" id="drop preferences table v3" 2024-08-17T16:52:49.213 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.070260485Z level=info msg="Executing migration" id="create preferences table v3" 2024-08-17T16:52:49.213 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.07084708Z level=info msg="Executing migration" id="Update preferences table charset" 2024-08-17T16:52:49.213 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.071707721Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-08-17T16:52:49.213 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.072552634Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-08-17T16:52:49.213 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.072886405Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-08-17T16:52:49.213 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.07373535Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-08-17T16:52:49.213 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.074750663Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-08-17T16:52:49.213 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.07546983Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-08-17T16:52:49.213 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.075904875Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-08-17T16:52:49.214 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.076791784Z level=info msg="Executing migration" id="create alert table v1" 2024-08-17T16:52:49.214 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.077738054Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-08-17T16:52:49.214 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.078538385Z level=info msg="Executing migration" id="add index alert state" 2024-08-17T16:52:49.214 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.078938969Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-08-17T16:52:49.214 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.079780592Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-08-17T16:52:49.214 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.080832895Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-08-17T16:52:49.214 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.083333816Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-08-17T16:52:49.214 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.083766808Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-08-17T16:52:49.214 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.086718407Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-08-17T16:52:49.214 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.08953811Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-08-17T16:52:49.214 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.092267118Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-08-17T16:52:49.215 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.092612233Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-08-17T16:52:49.215 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.095486536Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-08-17T16:52:49.215 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.095923588Z level=info msg="Executing migration" id="Add column is_default" 2024-08-17T16:52:49.215 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.098350027Z level=info msg="Executing migration" id="Add column frequency" 2024-08-17T16:52:49.215 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.101003455Z level=info msg="Executing migration" id="Add column send_reminder" 2024-08-17T16:52:49.215 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.101670136Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-08-17T16:52:49.215 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.104481884Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-08-17T16:52:49.215 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.107347267Z level=info msg="Executing migration" id="Update alert table charset" 2024-08-17T16:52:49.215 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.110136451Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-08-17T16:52:49.215 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.110471741Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-08-17T16:52:49.215 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.113349834Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-08-17T16:52:49.215 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.116129001Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-08-17T16:52:49.216 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.118817018Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-08-17T16:52:49.216 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.119329135Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-08-17T16:52:49.216 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.122159448Z level=info msg="Executing migration" id="Add for to alert table" 2024-08-17T16:52:49.216 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.123017438Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-08-17T16:52:49.216 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.124995984Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-08-17T16:52:49.216 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.127524438Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-08-17T16:52:49.216 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.127979849Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-08-17T16:52:49.216 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.130901484Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-08-17T16:52:49.216 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.133648729Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-08-17T16:52:49.216 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.136212846Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-08-17T16:52:49.216 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.136677254Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-08-17T16:52:49.216 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.13955784Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-08-17T16:52:49.217 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.14221597Z level=info msg="Executing migration" id="create annotation table v5" 2024-08-17T16:52:49.217 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.14498506Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-08-17T16:52:49.217 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.145386274Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-08-17T16:52:49.217 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.147966241Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-08-17T16:52:49.217 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.150464027Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-08-17T16:52:49.217 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.153436037Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-08-17T16:52:49.218 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.153935499Z level=info msg="Executing migration" id="Update annotation table charset" 2024-08-17T16:52:49.218 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.154725269Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-08-17T16:52:49.218 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.155594869Z level=info msg="Executing migration" id="Drop category_id index" 2024-08-17T16:52:49.218 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.156423232Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-08-17T16:52:49.218 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.157107761Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-08-17T16:52:49.218 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.158087102Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-08-17T16:52:49.218 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.158784562Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-08-17T16:52:49.218 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.159603003Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-08-17T16:52:49.218 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.161309163Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-08-17T16:52:49.218 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.162097054Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-08-17T16:52:49.218 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.163069052Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-08-17T16:52:49.219 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.163808543Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-08-17T16:52:49.219 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.164172718Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-08-17T16:52:49.219 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.164984949Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-08-17T16:52:49.219 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.165720725Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-08-17T16:52:49.219 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.166445838Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-08-17T16:52:49.219 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.167232048Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-08-17T16:52:49.219 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.168015055Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-08-17T16:52:49.219 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.168934167Z level=info msg="Executing migration" id="Add epoch_end column" 2024-08-17T16:52:49.219 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.169736775Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-08-17T16:52:49.219 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.170575729Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-08-17T16:52:49.219 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.170936738Z level=info msg="Executing migration" id="Move region to single row" 2024-08-17T16:52:49.219 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.171931865Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-08-17T16:52:49.219 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.172645848Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-08-17T16:52:49.220 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.173526486Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-08-17T16:52:49.220 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.173981052Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-08-17T16:52:49.220 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.174852826Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-08-17T16:52:49.221 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.175698147Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-08-17T16:52:49.221 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.176541038Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-08-17T16:52:49.221 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.176896958Z level=info msg="Executing migration" id="create test_data table" 2024-08-17T16:52:49.221 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.177747449Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-08-17T16:52:49.221 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.178699655Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-08-17T16:52:49.222 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.17943698Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-08-17T16:52:49.222 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.179862562Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-08-17T16:52:49.222 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.180732108Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-08-17T16:52:49.222 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.183523297Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-08-17T16:52:49.222 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.186592522Z level=info msg="Executing migration" id="create team table" 2024-08-17T16:52:49.222 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.18701608Z level=info msg="Executing migration" id="add index team.org_id" 2024-08-17T16:52:49.222 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.189815467Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-08-17T16:52:49.222 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.192655107Z level=info msg="Executing migration" id="create team member table" 2024-08-17T16:52:49.223 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.195406835Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-08-17T16:52:49.223 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.195843172Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-08-17T16:52:49.223 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.198738751Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-08-17T16:52:49.223 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.201563105Z level=info msg="Executing migration" id="Add column email to team table" 2024-08-17T16:52:49.223 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.204227422Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-08-17T16:52:49.223 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.204997313Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-08-17T16:52:49.223 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.207871817Z level=info msg="Executing migration" id="create dashboard acl table" 2024-08-17T16:52:49.223 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.210664376Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-08-17T16:52:49.458 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.213493777Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-08-17T16:52:49.458 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.213940184Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-08-17T16:52:49.458 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.216900862Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-08-17T16:52:49.458 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.219832638Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-08-17T16:52:49.458 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.222836184Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-08-17T16:52:49.458 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.225783449Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-08-17T16:52:49.458 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.22854149Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-08-17T16:52:49.458 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.231306122Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-08-17T16:52:49.459 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.23405839Z level=info msg="Executing migration" id="create tag table" 2024-08-17T16:52:49.459 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.234446788Z level=info msg="Executing migration" id="add index tag.key_value" 2024-08-17T16:52:49.459 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.2354493Z level=info msg="Executing migration" id="create login attempt table" 2024-08-17T16:52:49.459 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.23623775Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-08-17T16:52:49.459 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.237145968Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-08-17T16:52:49.459 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.238020699Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-08-17T16:52:49.459 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.24020911Z level=info msg="Executing migration" id="create login_attempt v2" 2024-08-17T16:52:49.459 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.240975864Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-08-17T16:52:49.459 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.241774845Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-08-17T16:52:49.459 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.242630409Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-08-17T16:52:49.459 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.243599576Z level=info msg="Executing migration" id="create user auth table" 2024-08-17T16:52:49.459 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.24449562Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-08-17T16:52:49.459 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.245268755Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-08-17T16:52:49.460 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.246221425Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-08-17T16:52:49.460 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.247064269Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-08-17T16:52:49.460 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.247942655Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-08-17T16:52:49.460 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.248786859Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-08-17T16:52:49.460 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.249698097Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-08-17T16:52:49.460 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.250531675Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-08-17T16:52:49.460 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.253349148Z level=info msg="Executing migration" id="create server_lock table" 2024-08-17T16:52:49.460 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.255960643Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-08-17T16:52:49.460 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.258981067Z level=info msg="Executing migration" id="create user auth token table" 2024-08-17T16:52:49.460 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.26175332Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-08-17T16:52:49.460 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.264486961Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-08-17T16:52:49.460 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.267140053Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-08-17T16:52:49.460 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.270083756Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-08-17T16:52:49.461 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.272876496Z level=info msg="Executing migration" id="create cache_data table" 2024-08-17T16:52:49.461 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.275611704Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-08-17T16:52:49.461 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.278412089Z level=info msg="Executing migration" id="create short_url table v1" 2024-08-17T16:52:49.461 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.281184837Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-08-17T16:52:49.462 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.284105629Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-08-17T16:52:49.462 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.286745813Z level=info msg="Executing migration" id="delete alert_definition table" 2024-08-17T16:52:49.462 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.287086504Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-08-17T16:52:49.462 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.287568Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-08-17T16:52:49.462 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.290228948Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-08-17T16:52:49.462 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.290697099Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-08-17T16:52:49.462 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.291014248Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-08-17T16:52:49.463 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.291443762Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-08-17T16:52:49.463 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.291830539Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-08-17T16:52:49.463 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.292289864Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-08-17T16:52:49.463 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.292727833Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-08-17T16:52:49.463 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.293556826Z level=info msg="Executing migration" id="drop alert_definition table" 2024-08-17T16:52:49.463 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.293976869Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-08-17T16:52:49.463 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.294278237Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-08-17T16:52:49.463 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.294649295Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-08-17T16:52:49.463 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.295109298Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-08-17T16:52:49.463 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.295548473Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-08-17T16:52:49.463 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.295879781Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-08-17T16:52:49.463 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.296312559Z level=info msg="Executing migration" id="create alert_instance table" 2024-08-17T16:52:49.463 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.296696676Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-08-17T16:52:49.464 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.297138327Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-08-17T16:52:49.464 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.297581819Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-08-17T16:52:49.464 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.29844238Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-08-17T16:52:49.464 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.298887415Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-08-17T16:52:49.464 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.299326548Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-08-17T16:52:49.464 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.302743022Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-08-17T16:52:49.464 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.306081421Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-08-17T16:52:49.464 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.306533686Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-08-17T16:52:49.464 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.30691968Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-08-17T16:52:49.464 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.30774683Z level=info msg="Executing migration" id="create alert_rule table" 2024-08-17T16:52:49.464 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.308196352Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-08-17T16:52:49.464 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.308646543Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-08-17T16:52:49.464 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.309096859Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-08-17T16:52:49.465 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.309535983Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-08-17T16:52:49.465 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.309799081Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-08-17T16:52:49.465 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.310643462Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-08-17T16:52:49.465 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.311497563Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-08-17T16:52:49.465 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.312396753Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-08-17T16:52:49.465 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.312830619Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-08-17T16:52:49.465 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.313288171Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-08-17T16:52:49.465 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.314145028Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-08-17T16:52:49.465 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.314987818Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-08-17T16:52:49.465 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.315431946Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-08-17T16:52:49.465 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.316292765Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-08-17T16:52:49.465 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.31710559Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-08-17T16:52:49.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.317415322Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-08-17T16:52:49.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.317867348Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-08-17T16:52:49.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.320567236Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-08-17T16:52:49.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.321018329Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-08-17T16:52:49.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.321312292Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-08-17T16:52:49.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.324317176Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-08-17T16:52:49.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.327074094Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-08-17T16:52:49.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.330008852Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-08-17T16:52:49.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.332490695Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-08-17T16:52:49.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.335117953Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-08-17T16:52:49.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.33776737Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-08-17T16:52:49.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.340480864Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-08-17T16:52:49.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.343275492Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-08-17T16:52:49.467 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.345927007Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-08-17T16:52:49.467 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.348573066Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-08-17T16:52:49.467 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.35119699Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-08-17T16:52:49.467 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.35384343Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-08-17T16:52:49.467 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.356631601Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-08-17T16:52:49.467 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.359066472Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-08-17T16:52:49.467 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.361625131Z level=info msg="Executing migration" id="create provenance_type table" 2024-08-17T16:52:49.467 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.364292983Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-08-17T16:52:49.467 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.366928533Z level=info msg="Executing migration" id="create alert_image table" 2024-08-17T16:52:49.467 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.36961023Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-08-17T16:52:49.467 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.372241801Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-08-17T16:52:49.467 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.374763946Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-08-17T16:52:49.467 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.375500672Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-08-17T16:52:49.468 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.375961111Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-08-17T16:52:49.468 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.376038811Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-08-17T16:52:49.468 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.376750179Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-08-17T16:52:49.468 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.377631966Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-08-17T16:52:49.468 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.378303187Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-08-17T16:52:49.468 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.378424267Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-08-17T16:52:49.468 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.378540079Z level=info msg="alerts found to migrate" alerts=0 2024-08-17T16:52:49.468 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.379083768Z level=info msg="Executing migration" id="create library_element table v1" 2024-08-17T16:52:49.468 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.380045174Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-08-17T16:52:49.468 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.380750172Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-08-17T16:52:49.468 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.38166061Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-08-17T16:52:49.469 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.382386293Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-08-17T16:52:49.469 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.38316371Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-08-17T16:52:49.469 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.384013342Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-08-17T16:52:49.469 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.384840029Z level=info msg="Executing migration" id="create data_keys table" 2024-08-17T16:52:49.469 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.385747226Z level=info msg="Executing migration" id="create secrets table" 2024-08-17T16:52:49.469 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.386520033Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-08-17T16:52:49.469 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.390888396Z level=info msg="Executing migration" id="add name column into data_keys" 2024-08-17T16:52:49.469 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.391820655Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-08-17T16:52:49.470 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.392752205Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-08-17T16:52:49.470 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.397083276Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-08-17T16:52:49.470 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.401378017Z level=info msg="Executing migration" id="create kv_store table v1" 2024-08-17T16:52:49.470 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.402250633Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-08-17T16:52:49.470 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.403080675Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-08-17T16:52:49.470 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.40394312Z level=info msg="Executing migration" id="create permission table" 2024-08-17T16:52:49.470 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.404947548Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-08-17T16:52:49.470 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.405785973Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-08-17T16:52:49.471 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.406492328Z level=info msg="Executing migration" id="create role table" 2024-08-17T16:52:49.471 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.407477931Z level=info msg="Executing migration" id="add column display_name" 2024-08-17T16:52:49.471 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.408533679Z level=info msg="Executing migration" id="add column group_name" 2024-08-17T16:52:49.471 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.40950253Z level=info msg="Executing migration" id="add index role.org_id" 2024-08-17T16:52:49.471 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.410358016Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-08-17T16:52:49.471 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.41120811Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-08-17T16:52:49.471 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.412099287Z level=info msg="Executing migration" id="create team role table" 2024-08-17T16:52:49.471 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.412949187Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-08-17T16:52:49.472 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.413925873Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-08-17T16:52:49.472 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.414688399Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-08-17T16:52:49.472 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.417533104Z level=info msg="Executing migration" id="create user role table" 2024-08-17T16:52:49.472 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.42019785Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-08-17T16:52:49.472 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.422831213Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-08-17T16:52:49.472 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.42547065Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-08-17T16:52:49.472 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.428223838Z level=info msg="Executing migration" id="create builtin role table" 2024-08-17T16:52:49.472 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.430975223Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-08-17T16:52:49.473 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.433620517Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-08-17T16:52:49.473 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.436054283Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-08-17T16:52:49.473 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.438977261Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-08-17T16:52:49.473 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.441704331Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-08-17T16:52:49.473 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.444445003Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-08-17T16:52:49.473 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.446941427Z level=info msg="Executing migration" id="add unique index role.uid" 2024-08-17T16:52:49.473 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.449606979Z level=info msg="Executing migration" id="create seed assignment table" 2024-08-17T16:52:49.473 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.451927101Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-08-17T16:52:49.473 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.454645334Z level=info msg="Executing migration" id="add column hidden to role table" 2024-08-17T16:52:49.474 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.457246543Z level=info msg="Executing migration" id="create query_history table v1" 2024-08-17T16:52:49.474 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.459877754Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-08-17T16:52:49.836 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:49 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:52:49] "GET /metrics HTTP/1.1" 200 28503 "" "Prometheus/2.43.0" 2024-08-17T16:52:49.846 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.462492778Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-08-17T16:52:49.846 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.465230923Z level=info msg="Executing migration" id="teams permissions migration" 2024-08-17T16:52:49.846 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.466080402Z level=info msg="Executing migration" id="dashboard permissions" 2024-08-17T16:52:49.846 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.46689908Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-08-17T16:52:49.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.467768869Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-08-17T16:52:49.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.468584298Z level=info msg="Executing migration" id="alerting notification permissions" 2024-08-17T16:52:49.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.469457862Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-08-17T16:52:49.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.470323782Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-08-17T16:52:49.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.471156973Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-08-17T16:52:49.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.472143006Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-08-17T16:52:49.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.47299578Z level=info msg="Executing migration" id="create correlation table v1" 2024-08-17T16:52:49.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.473933071Z level=info msg="Executing migration" id="add index correlations.uid" 2024-08-17T16:52:49.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.474635359Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-08-17T16:52:49.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.475422282Z level=info msg="Executing migration" id="add correlation config column" 2024-08-17T16:52:49.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.476432204Z level=info msg="Executing migration" id="create entity_events table" 2024-08-17T16:52:49.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.477395583Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-08-17T16:52:49.848 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.478084776Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-08-17T16:52:49.848 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.478171802Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-08-17T16:52:49.848 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.479046232Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-17T16:52:49.848 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.479255511Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-17T16:52:49.849 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.479742194Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-08-17T16:52:49.849 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.480702241Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-08-17T16:52:49.849 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.481464413Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-08-17T16:52:49.849 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.482347515Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-08-17T16:52:49.849 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.483250057Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-08-17T16:52:49.849 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.484214705Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-08-17T16:52:49.849 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.484927801Z level=info msg="Executing migration" id="Drop public config table" 2024-08-17T16:52:49.849 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.485781356Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-08-17T16:52:49.849 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.486673944Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-08-17T16:52:49.849 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.487503127Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-08-17T16:52:49.849 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.488473391Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-08-17T16:52:49.850 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.489203036Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-08-17T16:52:49.850 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.492275284Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-08-17T16:52:49.850 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.493378989Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-08-17T16:52:49.850 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.494472103Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-08-17T16:52:49.850 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.495302899Z level=info msg="Executing migration" id="add share column" 2024-08-17T16:52:49.850 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.496416842Z level=info msg="Executing migration" id="create default alerting folders" 2024-08-17T16:52:49.850 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.497429379Z level=info msg="Executing migration" id="create file table" 2024-08-17T16:52:49.850 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.498278773Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-08-17T16:52:49.850 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.501098293Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-08-17T16:52:49.850 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.503850098Z level=info msg="Executing migration" id="create file_meta table" 2024-08-17T16:52:49.850 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.506637662Z level=info msg="Executing migration" id="file table idx: path key" 2024-08-17T16:52:49.850 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.50953444Z level=info msg="Executing migration" id="set path collation in file table" 2024-08-17T16:52:49.851 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.512374295Z level=info msg="Executing migration" id="managed permissions migration" 2024-08-17T16:52:49.851 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.515078885Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-08-17T16:52:49.851 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.517963643Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-08-17T16:52:49.851 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.520700532Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-08-17T16:52:49.851 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.523568215Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-08-17T16:52:49.851 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.526279009Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-08-17T16:52:49.851 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.528966896Z level=info msg="Executing migration" id="update group index for alert rules" 2024-08-17T16:52:49.851 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.532056978Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-08-17T16:52:49.851 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.534924746Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-08-17T16:52:49.851 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.537594404Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-08-17T16:52:49.851 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.540495251Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-08-17T16:52:49.851 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.543169Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-08-17T16:52:49.851 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.545954716Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-08-17T16:52:49.852 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.555717396Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-08-17T16:52:49.852 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.558487911Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-08-17T16:52:49.852 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.56118677Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-08-17T16:52:49.852 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.564650163Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-08-17T16:52:49.852 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.567216983Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-08-17T16:52:49.852 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.569685757Z level=info msg="Executing migration" id="create folder table" 2024-08-17T16:52:49.852 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.572311114Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-08-17T16:52:49.852 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.57482515Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-08-17T16:52:49.852 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.57759609Z level=info msg="Executing migration" id="Update folder title length" 2024-08-17T16:52:49.852 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.580240124Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-08-17T16:52:49.852 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=migrator t=2024-08-17T16:52:49.582916038Z level=info msg="migrations completed" performed=485 skipped=0 duration=707.76356ms 2024-08-17T16:52:49.852 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=sqlstore t=2024-08-17T16:52:49.583313416Z level=info msg="Created default organization" 2024-08-17T16:52:49.853 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=plugin.loader t=2024-08-17T16:52:49.640294987Z level=info msg="Plugin registered" pluginID=input 2024-08-17T16:52:49.853 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=plugin.loader t=2024-08-17T16:52:49.666702925Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-08-17T16:52:49.853 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=plugin.loader t=2024-08-17T16:52:49.666722816Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-08-17T16:52:49.853 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=secrets t=2024-08-17T16:52:49.666754374Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-08-17T16:52:49.853 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=query_data t=2024-08-17T16:52:49.668465232Z level=info msg="Query Service initialization" 2024-08-17T16:52:49.853 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=live.push_http t=2024-08-17T16:52:49.671218927Z level=info msg="Live Push Gateway initialization" 2024-08-17T16:52:50.238 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:49 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:50.238 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:49 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:50.238 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:49 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:50.238 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:49 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:50.239 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:49 smithi063 ceph-mon[31063]: Deploying daemon node-exporter.a on smithi063 2024-08-17T16:52:50.239 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:49 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:50.239 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:49 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:50.239 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:49 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:50.239 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:49 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:50.239 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:49 smithi063 ceph-mon[42680]: Deploying daemon node-exporter.a on smithi063 2024-08-17T16:52:50.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:50.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:50.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:50.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:50.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:49 smithi162 ceph-mon[34633]: Deploying daemon node-exporter.a on smithi063 2024-08-17T16:52:50.846 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=infra.usagestats.collector t=2024-08-17T16:52:50.408541579Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-08-17T16:52:50.846 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=provisioning.datasources t=2024-08-17T16:52:50.464133958Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-08-17T16:52:50.846 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=provisioning.datasources t=2024-08-17T16:52:50.496055046Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-08-17T16:52:50.846 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=provisioning.alerting t=2024-08-17T16:52:50.517344346Z level=info msg="starting to provision alerting" 2024-08-17T16:52:50.846 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=provisioning.alerting t=2024-08-17T16:52:50.517365323Z level=info msg="finished to provision alerting" 2024-08-17T16:52:50.846 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=ngalert.state.manager t=2024-08-17T16:52:50.517525352Z level=info msg="Warming state cache for startup" 2024-08-17T16:52:50.846 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=grafanaStorageLogger t=2024-08-17T16:52:50.51754152Z level=info msg="storage starting" 2024-08-17T16:52:50.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=http.server t=2024-08-17T16:52:50.519198726Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-08-17T16:52:50.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=sqlstore.transactions t=2024-08-17T16:52:50.527940465Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T16:52:50.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=ngalert.state.manager t=2024-08-17T16:52:50.527998667Z level=info msg="State cache has been initialized" states=0 duration=10.472262ms 2024-08-17T16:52:50.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=ticker t=2024-08-17T16:52:50.528061635Z level=info msg=starting first_tick=2024-08-17T16:53:00Z 2024-08-17T16:52:50.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=ngalert.multiorg.alertmanager t=2024-08-17T16:52:50.528052465Z level=info msg="starting MultiOrg Alertmanager" 2024-08-17T16:52:50.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=sqlstore.transactions t=2024-08-17T16:52:50.572727534Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T16:52:50.847 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[64960]: logger=sqlstore.transactions t=2024-08-17T16:52:50.609358787Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T16:52:50.987 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:50 smithi063 ceph-mon[42680]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:50.988 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:50 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:52:50.988 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:50 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:50.988 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:50 smithi063 ceph-mon[31063]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:50.988 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:50 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:52:50.988 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:50 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:51.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-mon[34633]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:51.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:52:51.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:50 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:52:52.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:52 smithi063 ceph-mon[42680]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:52.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:52 smithi063 ceph-mon[31063]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:52.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:52 smithi162 ceph-mon[34633]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:53.614 INFO:teuthology.orchestra.run.smithi063.stdout:55834574890 2024-08-17T16:52:53.614 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T16:52:54.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:54 smithi063 ceph-mon[42680]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:54.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:54 smithi063 ceph-mon[31063]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:54.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:54 smithi162 ceph-mon[34633]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:55.107 INFO:teuthology.orchestra.run.smithi063.stdout:133143986205 2024-08-17T16:52:55.108 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T16:52:56.643 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:56 smithi063 ceph-mon[42680]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:56.643 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:56 smithi063 ceph-mon[31063]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:56.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:56 smithi162 ceph-mon[34633]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:57.445 INFO:teuthology.orchestra.run.smithi063.stdout:167503724569 2024-08-17T16:52:57.446 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T16:52:57.932 INFO:teuthology.orchestra.run.smithi063.stdout:201863462932 2024-08-17T16:52:57.933 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T16:52:58.499 INFO:teuthology.orchestra.run.smithi063.stdout:236223201295 2024-08-17T16:52:58.499 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T16:52:58.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:52:58 smithi063 ceph-mon[42680]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:58.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:52:58 smithi063 ceph-mon[31063]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:58.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:52:58 smithi162 ceph-mon[34633]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:52:59.141 INFO:teuthology.orchestra.run.smithi063.stdout:34359738417 2024-08-17T16:52:59.141 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T16:52:59.914 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:52:59 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:52:59] "GET /metrics HTTP/1.1" 200 28509 "" "Prometheus/2.43.0" 2024-08-17T16:52:59.939 INFO:teuthology.orchestra.run.smithi063.stdout:111669149731 2024-08-17T16:52:59.940 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T16:53:00.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:00 smithi063 ceph-mon[42680]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:00.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:00 smithi063 ceph-mon[31063]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:00 smithi162 ceph-mon[34633]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:00.890 INFO:teuthology.orchestra.run.smithi063.stdout:77309411368 2024-08-17T16:53:00.890 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T16:53:00.914 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:00.914 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:00.914 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:00.922 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:00.922 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:00.937 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:00.993 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:02.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:02 smithi063 ceph-mon[42680]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:02.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:02 smithi063 ceph-mon[31063]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:02.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:02 smithi162 ceph-mon[34633]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:04.279 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:04 smithi063 podman[76322]: 2024-08-17 16:53:04.118127267 +0000 UTC m=+3.643636670 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-08-17T16:53:04.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:04 smithi063 ceph-mon[42680]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:04.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:04 smithi063 ceph-mon[31063]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:04 smithi162 ceph-mon[34633]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:05.585 INFO:teuthology.orchestra.run.smithi063.stdout:111669149732 2024-08-17T16:53:05.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:05 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:53:05.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:05 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:53:05.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:05 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:53:06.516 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 podman[76322]: 2024-08-17 16:53:06.169338448 +0000 UTC m=+5.694847881 container create 8958df80ad2443d1ec5136e2e4a034986e37d86e549e4f59994a9fa0d17628db (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T16:53:06.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3564468168' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T16:53:06.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-mon[42680]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:06.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3564468168' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T16:53:06.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-mon[31063]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:06.780 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 podman[76322]: 2024-08-17 16:53:06.657893565 +0000 UTC m=+6.183402971 container init 8958df80ad2443d1ec5136e2e4a034986e37d86e549e4f59994a9fa0d17628db (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T16:53:06.780 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 podman[76322]: 2024-08-17 16:53:06.663564284 +0000 UTC m=+6.189073681 container start 8958df80ad2443d1ec5136e2e4a034986e37d86e549e4f59994a9fa0d17628db (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T16:53:06.780 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.667Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-08-17T16:53:06.780 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.667Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-08-17T16:53:06.781 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.668Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T16:53:06.781 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.668Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T16:53:06.781 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T16:53:06.781 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T16:53:06.781 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T16:53:06.781 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T16:53:06.781 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T16:53:06.781 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T16:53:06.781 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T16:53:06.781 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T16:53:06.781 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T16:53:06.781 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T16:53:06.782 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T16:53:06.782 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T16:53:06.782 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T16:53:06.782 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T16:53:06.782 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T16:53:06.782 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T16:53:06.782 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T16:53:06.782 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T16:53:06.782 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T16:53:06.782 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T16:53:06.783 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T16:53:06.783 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T16:53:06.783 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T16:53:06.783 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T16:53:06.783 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T16:53:06.783 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T16:53:06.783 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T16:53:06.783 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T16:53:06.783 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T16:53:06.783 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T16:53:06.783 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T16:53:06.783 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T16:53:06.783 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T16:53:06.784 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T16:53:06.784 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T16:53:06.784 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T16:53:06.784 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T16:53:06.784 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T16:53:06.784 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T16:53:06.784 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T16:53:06.784 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T16:53:06.784 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T16:53:06.785 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T16:53:06.785 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T16:53:06.785 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T16:53:06.785 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T16:53:06.785 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T16:53:06.785 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-08-17T16:53:06.785 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[76852]: ts=2024-08-17T16:53:06.669Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T16:53:06.785 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 bash[76322]: 8958df80ad2443d1ec5136e2e4a034986e37d86e549e4f59994a9fa0d17628db 2024-08-17T16:53:06.785 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:53:06 smithi063 systemd[1]: Started Ceph node-exporter.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:53:06.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:06 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3564468168' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T16:53:06.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:06 smithi162 ceph-mon[34633]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:07.820 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:07 smithi162 bash[65239]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-08-17T16:53:08.077 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:07 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:08.077 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:07 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:08.077 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:07 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:08.077 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:07 smithi162 ceph-mon[34633]: Deploying daemon node-exporter.b on smithi162 2024-08-17T16:53:08.149 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:07 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:08.150 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:07 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:08.150 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:07 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:08.150 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:07 smithi063 ceph-mon[42680]: Deploying daemon node-exporter.b on smithi162 2024-08-17T16:53:08.150 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:07 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:08.150 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:07 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:08.151 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:07 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:08.151 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:07 smithi063 ceph-mon[31063]: Deploying daemon node-exporter.b on smithi162 2024-08-17T16:53:08.345 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:08 smithi162 bash[65239]: Getting image source signatures 2024-08-17T16:53:08.346 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:08 smithi162 bash[65239]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-08-17T16:53:08.346 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:08 smithi162 bash[65239]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-08-17T16:53:08.346 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:08 smithi162 bash[65239]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-08-17T16:53:09.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:08 smithi063 ceph-mon[42680]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:08 smithi063 ceph-mon[31063]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:08 smithi162 ceph-mon[34633]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:09.702 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:53:09] "GET /metrics HTTP/1.1" 200 28507 "" "Prometheus/2.43.0" 2024-08-17T16:53:09.825 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:09 smithi162 bash[65239]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-08-17T16:53:09.825 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:09 smithi162 bash[65239]: Writing manifest to image destination 2024-08-17T16:53:09.827 INFO:teuthology.orchestra.run.smithi063.stdout:236223201298 2024-08-17T16:53:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:09 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3635551305' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T16:53:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:09 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3635551305' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T16:53:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:09 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3635551305' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T16:53:10.096 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:09 smithi162 podman[65239]: 2024-08-17 16:53:09.785384175 +0000 UTC m=+2.155685968 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-08-17T16:53:10.096 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:09 smithi162 podman[65239]: 2024-08-17 16:53:09.867021815 +0000 UTC m=+2.237323595 container create b1691104d8aced91d63bce7e00f54a88a4870d0dc0f8f5697a12e80e2397e401 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T16:53:10.353 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201295 got 236223201298 for osd.7 2024-08-17T16:53:10.354 DEBUG:teuthology.parallel:result is None 2024-08-17T16:53:10.596 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 podman[65239]: 2024-08-17 16:53:10.127449122 +0000 UTC m=+2.497750913 container init b1691104d8aced91d63bce7e00f54a88a4870d0dc0f8f5697a12e80e2397e401 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T16:53:10.596 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 podman[65239]: 2024-08-17 16:53:10.133109598 +0000 UTC m=+2.503411377 container start b1691104d8aced91d63bce7e00f54a88a4870d0dc0f8f5697a12e80e2397e401 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T16:53:10.596 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.137Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-08-17T16:53:10.596 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.137Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-08-17T16:53:10.596 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.138Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T16:53:10.597 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.138Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T16:53:10.597 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.138Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T16:53:10.597 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.138Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T16:53:10.597 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T16:53:10.597 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T16:53:10.597 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T16:53:10.597 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T16:53:10.597 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T16:53:10.597 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T16:53:10.597 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T16:53:10.597 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T16:53:10.597 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T16:53:10.598 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T16:53:10.598 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T16:53:10.598 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T16:53:10.598 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T16:53:10.598 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T16:53:10.598 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T16:53:10.598 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T16:53:10.598 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T16:53:10.598 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T16:53:10.598 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T16:53:10.598 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T16:53:10.599 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T16:53:10.599 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T16:53:10.599 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T16:53:10.599 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T16:53:10.599 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T16:53:10.599 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T16:53:10.599 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T16:53:10.599 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T16:53:10.599 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T16:53:10.599 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T16:53:10.599 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T16:53:10.600 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T16:53:10.600 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T16:53:10.600 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T16:53:10.600 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T16:53:10.600 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T16:53:10.600 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T16:53:10.600 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T16:53:10.600 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T16:53:10.601 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T16:53:10.601 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T16:53:10.601 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T16:53:10.601 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T16:53:10.601 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T16:53:10.601 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T16:53:10.601 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-08-17T16:53:10.601 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[65290]: ts=2024-08-17T16:53:10.139Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T16:53:10.601 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 bash[65239]: b1691104d8aced91d63bce7e00f54a88a4870d0dc0f8f5697a12e80e2397e401 2024-08-17T16:53:10.601 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:53:10 smithi162 systemd[1]: Started Ceph node-exporter.b for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:53:11.371 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.371 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.371 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.372 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.372 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.372 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:11.372 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[42680]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:11.372 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.372 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.372 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.372 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.373 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.373 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.373 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:11.373 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[31063]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:11.373 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:11 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:11 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:11 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:11 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:11 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:11 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:11.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:11 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:11.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:11 smithi162 ceph-mon[34633]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:11.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:11 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:12.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:12 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/414539964' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T16:53:12.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:12 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/414539964' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T16:53:12.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:12 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/414539964' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T16:53:12.753 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149731 got 111669149732 for osd.3 2024-08-17T16:53:12.754 DEBUG:teuthology.parallel:result is None 2024-08-17T16:53:12.833 INFO:teuthology.orchestra.run.smithi063.stdout:55834574894 2024-08-17T16:53:13.413 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:13 smithi063 ceph-mon[42680]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:13.413 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:13 smithi063 ceph-mon[31063]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:13.439 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574890 got 55834574894 for osd.1 2024-08-17T16:53:13.439 DEBUG:teuthology.parallel:result is None 2024-08-17T16:53:13.473 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:13.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:13 smithi162 ceph-mon[34633]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:13.952 INFO:teuthology.orchestra.run.smithi063.stdout:133143986209 2024-08-17T16:53:14.295 INFO:teuthology.orchestra.run.smithi063.stdout:167503724572 2024-08-17T16:53:14.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:14 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/926538' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T16:53:14.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:14 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:14.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:14 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:14.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:14 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/1901665169' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T16:53:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:14 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/926538' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T16:53:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:14 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:14 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:14 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/1901665169' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T16:53:14.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:14 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/926538' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T16:53:14.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:14 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:14.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:14 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:14.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:14 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/1901665169' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T16:53:14.896 INFO:teuthology.orchestra.run.smithi063.stdout:34359738420 2024-08-17T16:53:15.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:15 smithi063 ceph-mon[42680]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:15.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:15 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3508872019' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T16:53:15.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:15 smithi063 ceph-mon[31063]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:15.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:15 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3508872019' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T16:53:15.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:15 smithi162 ceph-mon[34633]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:15.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:15 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3508872019' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T16:53:16.358 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738417 got 34359738420 for osd.0 2024-08-17T16:53:16.358 DEBUG:teuthology.parallel:result is None 2024-08-17T16:53:16.515 INFO:teuthology.orchestra.run.smithi063.stdout:201863462937 2024-08-17T16:53:16.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:16 smithi063 ceph-mon[31063]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:16.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:16 smithi063 ceph-mon[42680]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:16 smithi162 ceph-mon[34633]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:17.216 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724569 got 167503724572 for osd.5 2024-08-17T16:53:17.216 DEBUG:teuthology.parallel:result is None 2024-08-17T16:53:17.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:17 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3738043941' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T16:53:17.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:17 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3738043941' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T16:53:17.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:17 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3738043941' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T16:53:18.635 INFO:teuthology.orchestra.run.smithi063.stdout:77309411372 2024-08-17T16:53:18.636 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:18 smithi063 ceph-mon[31063]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:18.636 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:18 smithi063 ceph-mon[42680]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:18.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:18 smithi162 ceph-mon[34633]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:18.936 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462932 got 201863462937 for osd.6 2024-08-17T16:53:18.936 DEBUG:teuthology.parallel:result is None 2024-08-17T16:53:19.076 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986205 got 133143986209 for osd.4 2024-08-17T16:53:19.076 DEBUG:teuthology.parallel:result is None 2024-08-17T16:53:19.742 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:19 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/2315787602' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T16:53:19.743 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:53:19] "GET /metrics HTTP/1.1" 200 28507 "" "Prometheus/2.43.0" 2024-08-17T16:53:19.743 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:19 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/2315787602' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T16:53:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:19 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/2315787602' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T16:53:20.051 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411368 got 77309411372 for osd.2 2024-08-17T16:53:20.051 DEBUG:teuthology.parallel:result is None 2024-08-17T16:53:20.051 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T16:53:20.051 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T16:53:20.290 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:20.688 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:20 smithi063 ceph-mon[42680]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:20.688 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:20 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:53:20.688 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:20 smithi063 ceph-mon[31063]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:20.688 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:20 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:53:20.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:20 smithi162 ceph-mon[34633]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:20.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:20 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:53:22.708 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:22 smithi063 ceph-mon[42680]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:22.708 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:22 smithi063 ceph-mon[31063]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:22.750 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:53:22.750 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T16:53:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:22 smithi162 ceph-mon[34633]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:23.577 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":41,"stamp":"2024-08-17T16:53:22.420527+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":223188,"kb_used_data":7028,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":749738028,"statfs":{"total":767960285184,"available":767731740672,"internally_reserved":0,"allocated":7196672,"data_stored":4055922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12742,"internal_metadata":221236794},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001681"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:35.137319+0000","last_change":"2024-08-17T16:52:28.102166+0000","last_active":"2024-08-17T16:52:35.137319+0000","last_peered":"2024-08-17T16:52:35.137319+0000","last_clean":"2024-08-17T16:52:35.137319+0000","last_became_active":"2024-08-17T16:52:23.430739+0000","last_became_peered":"2024-08-17T16:52:23.430739+0000","last_unstale":"2024-08-17T16:52:35.137319+0000","last_undegraded":"2024-08-17T16:52:35.137319+0000","last_fullsized":"2024-08-17T16:52:35.137319+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:19.139627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:34.780003+0000","last_change":"2024-08-17T16:52:28.097477+0000","last_active":"2024-08-17T16:52:34.780003+0000","last_peered":"2024-08-17T16:52:34.780003+0000","last_clean":"2024-08-17T16:52:34.780003+0000","last_became_active":"2024-08-17T16:52:23.816540+0000","last_became_peered":"2024-08-17T16:52:23.816540+0000","last_unstale":"2024-08-17T16:52:34.780003+0000","last_undegraded":"2024-08-17T16:52:34.780003+0000","last_fullsized":"2024-08-17T16:52:34.780003+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:27:49.345586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000334624,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:35.551278+0000","last_change":"2024-08-17T16:52:28.243251+0000","last_active":"2024-08-17T16:52:35.551278+0000","last_peered":"2024-08-17T16:52:35.551278+0000","last_clean":"2024-08-17T16:52:35.551278+0000","last_became_active":"2024-08-17T16:52:23.816512+0000","last_became_peered":"2024-08-17T16:52:23.816512+0000","last_unstale":"2024-08-17T16:52:35.551278+0000","last_undegraded":"2024-08-17T16:52:35.551278+0000","last_fullsized":"2024-08-17T16:52:35.551278+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:59.683846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:34.779906+0000","last_change":"2024-08-17T16:52:28.097582+0000","last_active":"2024-08-17T16:52:34.779906+0000","last_peered":"2024-08-17T16:52:34.779906+0000","last_clean":"2024-08-17T16:52:34.779906+0000","last_became_active":"2024-08-17T16:52:23.739237+0000","last_became_peered":"2024-08-17T16:52:23.739237+0000","last_unstale":"2024-08-17T16:52:34.779906+0000","last_undegraded":"2024-08-17T16:52:34.779906+0000","last_fullsized":"2024-08-17T16:52:34.779906+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:02:16.574623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":22,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:35.551207+0000","last_change":"2024-08-17T16:52:28.101249+0000","last_active":"2024-08-17T16:52:35.551207+0000","last_peered":"2024-08-17T16:52:35.551207+0000","last_clean":"2024-08-17T16:52:35.551207+0000","last_became_active":"2024-08-17T16:52:23.430593+0000","last_became_peered":"2024-08-17T16:52:23.430593+0000","last_unstale":"2024-08-17T16:52:35.551207+0000","last_undegraded":"2024-08-17T16:52:35.551207+0000","last_fullsized":"2024-08-17T16:52:35.551207+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:19.292172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:34.780039+0000","last_change":"2024-08-17T16:52:28.097703+0000","last_active":"2024-08-17T16:52:34.780039+0000","last_peered":"2024-08-17T16:52:34.780039+0000","last_clean":"2024-08-17T16:52:34.780039+0000","last_became_active":"2024-08-17T16:52:23.739224+0000","last_became_peered":"2024-08-17T16:52:23.739224+0000","last_unstale":"2024-08-17T16:52:34.780039+0000","last_undegraded":"2024-08-17T16:52:34.780039+0000","last_fullsized":"2024-08-17T16:52:34.780039+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:33.382141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:35.551201+0000","last_change":"2024-08-17T16:52:28.243231+0000","last_active":"2024-08-17T16:52:35.551201+0000","last_peered":"2024-08-17T16:52:35.551201+0000","last_clean":"2024-08-17T16:52:35.551201+0000","last_became_active":"2024-08-17T16:52:23.739287+0000","last_became_peered":"2024-08-17T16:52:23.739287+0000","last_unstale":"2024-08-17T16:52:35.551201+0000","last_undegraded":"2024-08-17T16:52:35.551201+0000","last_fullsized":"2024-08-17T16:52:35.551201+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:39.119833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:35.551505+0000","last_change":"2024-08-17T16:52:28.100670+0000","last_active":"2024-08-17T16:52:35.551505+0000","last_peered":"2024-08-17T16:52:35.551505+0000","last_clean":"2024-08-17T16:52:35.551505+0000","last_became_active":"2024-08-17T16:52:23.430320+0000","last_became_peered":"2024-08-17T16:52:23.430320+0000","last_unstale":"2024-08-17T16:52:35.551505+0000","last_undegraded":"2024-08-17T16:52:35.551505+0000","last_fullsized":"2024-08-17T16:52:35.551505+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:47.861974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":263,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:52:28.242917+0000","last_change":"2024-08-17T16:51:52.132475+0000","last_active":"2024-08-17T16:52:28.242917+0000","last_peered":"2024-08-17T16:52:28.242917+0000","last_clean":"2024-08-17T16:52:28.242917+0000","last_became_active":"2024-08-17T16:51:52.132088+0000","last_became_peered":"2024-08-17T16:51:52.132088+0000","last_unstale":"2024-08-17T16:52:28.242917+0000","last_undegraded":"2024-08-17T16:52:28.242917+0000","last_fullsized":"2024-08-17T16:52:28.242917+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:13.679985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201301,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28256,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716896,"statfs":{"total":95995035648,"available":95966101504,"internally_reserved":0,"allocated":1265664,"data_stored":875688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.54800000000000004}]},{"osd":1,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.52400000000000002}]},{"osd":2,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.61799999999999999}]},{"osd":3,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.59699999999999998}]},{"osd":4,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.33300000000000002}]},{"osd":5,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.35599999999999998}]},{"osd":6,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.46700000000000003}]}]},{"osd":6,"up_from":47,"seq":201863462938,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28256,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716896,"statfs":{"total":95995035648,"available":95966101504,"internally_reserved":0,"allocated":1265664,"data_stored":875434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":27654601},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:52:27 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.53800000000000003}]},{"osd":1,"last update":"Sat Aug 17 16:52:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.46899999999999997}]},{"osd":2,"last update":"Sat Aug 17 16:52:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.56799999999999995}]},{"osd":3,"last update":"Sat Aug 17 16:52:27 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.504},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.51900000000000002}]},{"osd":4,"last update":"Sat Aug 17 16:52:27 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.315},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.34100000000000003}]},{"osd":5,"last update":"Sat Aug 17 16:52:27 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.36899999999999999}]},{"osd":7,"last update":"Sat Aug 17 16:52:51 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.40100000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724574,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:53:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52300000000000002,"15min":0.52100000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.72099999999999997,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53200000000000003,"15min":0.53600000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.52700000000000002}]},{"osd":1,"last update":"Sat Aug 17 16:53:05 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52700000000000002,"15min":0.52800000000000002},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.32100000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.80900000000000005,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.57599999999999996}]},{"osd":2,"last update":"Sat Aug 17 16:53:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54000000000000004,"15min":0.53900000000000003},"min":{"1min":0.43099999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.69599999999999995,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.53700000000000003,"15min":0.53900000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77200000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.56299999999999994}]},{"osd":3,"last update":"Sat Aug 17 16:53:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54400000000000004,"15min":0.54600000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66800000000000004,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.55800000000000005,"15min":0.56100000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.71799999999999997,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.61799999999999999}]},{"osd":4,"last update":"Sat Aug 17 16:53:05 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.38800000000000001,"15min":0.38400000000000001},"min":{"1min":0.26100000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.56499999999999995,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.41899999999999998,"15min":0.41399999999999998},"min":{"1min":0.33000000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.61299999999999999,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Aug 17 16:52:29 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.02,"5min":1.02,"15min":1.02},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Aug 17 16:52:52 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.39100000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986211,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":285651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:52:38 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.40400000000000003,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.71799999999999997,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.53900000000000003,"15min":0.53600000000000003},"min":{"1min":0.377,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.68899999999999995,"5min":0.749,"15min":0.749},"last":0.68100000000000005}]},{"osd":1,"last update":"Sat Aug 17 16:52:38 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.36299999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77600000000000002,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.53100000000000003,"15min":0.52600000000000002},"min":{"1min":0.42399999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.746,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.66900000000000004}]},{"osd":2,"last update":"Sat Aug 17 16:52:38 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.56399999999999995,"15min":0.55900000000000005},"min":{"1min":0.45800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.79000000000000004,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35899999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.71099999999999997,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.73699999999999999}]},{"osd":3,"last update":"Sat Aug 17 16:52:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.40100000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55700000000000005,"15min":0.55900000000000005},"min":{"1min":0.375,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.69799999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.70399999999999996}]},{"osd":5,"last update":"Sat Aug 17 16:53:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40600000000000003,"15min":0.40699999999999997},"min":{"1min":0.27600000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.51000000000000001,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.41399999999999998,"15min":0.41599999999999998},"min":{"1min":0.25600000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.56699999999999995,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.443}]},{"osd":6,"last update":"Sat Aug 17 16:52:25 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.496},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.51400000000000001}]},{"osd":7,"last update":"Sat Aug 17 16:52:48 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.45800000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149736,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:52:21 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.376,"15min":0.375},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.40100000000000002}]},{"osd":1,"last update":"Sat Aug 17 16:52:21 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.44700000000000001,"15min":0.45100000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.379}]},{"osd":2,"last update":"Sat Aug 17 16:52:21 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.44400000000000001,"15min":0.442},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.439},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42999999999999999,"15min":0.432},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.46200000000000002}]},{"osd":4,"last update":"Sat Aug 17 16:52:41 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.57999999999999996,"15min":0.58199999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.69799999999999995,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.56200000000000006,"15min":0.56399999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.70299999999999996,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.69099999999999995}]},{"osd":5,"last update":"Sat Aug 17 16:53:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.58399999999999996,"15min":0.58799999999999997},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.69099999999999995,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56999999999999995,"15min":0.57099999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.748,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.67200000000000004}]},{"osd":6,"last update":"Sat Aug 17 16:52:29 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.64300000000000002}]},{"osd":7,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.76100000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:52:57 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.39500000000000002,"15min":0.39300000000000002},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40400000000000003,"15min":0.39200000000000002},"min":{"1min":0.26400000000000001,"5min":0.245,"15min":0.245},"max":{"1min":0.59299999999999997,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.51700000000000002}]},{"osd":1,"last update":"Sat Aug 17 16:52:57 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.41699999999999998,"15min":0.40699999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60899999999999999,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.42799999999999999,"15min":0.42399999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.67700000000000005,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.58699999999999997}]},{"osd":3,"last update":"Sat Aug 17 16:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.48299999999999998,"15min":0.48699999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.63,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.48699999999999999,"15min":0.48899999999999999},"min":{"1min":0.34000000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.59499999999999997,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.55600000000000005}]},{"osd":4,"last update":"Sat Aug 17 16:52:42 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.64100000000000001,"15min":0.65800000000000003},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.878,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.86399999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.65600000000000003,"15min":0.67000000000000004},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.86499999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.88700000000000001}]},{"osd":5,"last update":"Sat Aug 17 16:53:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.42699999999999999,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.92900000000000005},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.64000000000000001,"15min":0.65200000000000002},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80700000000000005,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.91400000000000003}]},{"osd":6,"last update":"Sat Aug 17 16:52:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.999,"5min":0.999,"15min":0.999},"last":0.873},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.80100000000000005}]},{"osd":7,"last update":"Sat Aug 17 16:52:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.89700000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:52:33 2024","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.34499999999999997,"15min":0.33900000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.51900000000000002,"5min":0.53300000000000003,"15min":0.53300000000000003},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.36699999999999999,"15min":0.35199999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.46200000000000002}]},{"osd":2,"last update":"Sat Aug 17 16:53:01 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.37,"15min":0.36299999999999999},"min":{"1min":0.254,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.54400000000000004,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.378,"5min":0.36699999999999999,"15min":0.35699999999999998},"min":{"1min":0.26100000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.49299999999999999}]},{"osd":3,"last update":"Sat Aug 17 16:53:19 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.40000000000000002,"15min":0.40500000000000003},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.51500000000000001,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.434,"5min":0.45800000000000002,"15min":0.46800000000000003},"min":{"1min":0.32400000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.75600000000000001}]},{"osd":4,"last update":"Sat Aug 17 16:52:42 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56299999999999994,"15min":0.55900000000000005},"min":{"1min":0.434,"5min":0.315,"15min":0.315},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.53100000000000003,"15min":0.52600000000000002},"min":{"1min":0.39700000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.74099999999999999}]},{"osd":5,"last update":"Sat Aug 17 16:53:01 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59099999999999997,"15min":0.58999999999999997},"min":{"1min":0.46300000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.60099999999999998}]},{"osd":6,"last update":"Sat Aug 17 16:52:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.625},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.81999999999999995}]},{"osd":7,"last update":"Sat Aug 17 16:52:49 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.79400000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28260,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716892,"statfs":{"total":95995035648,"available":95966097408,"internally_reserved":0,"allocated":1269760,"data_stored":876019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 16:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.32600000000000001},"min":{"1min":0.23400000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.58999999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.318},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.35599999999999998,"15min":0.33400000000000002},"min":{"1min":0.27400000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.31}]},{"osd":2,"last update":"Sat Aug 17 16:53:00 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.39600000000000002,"15min":0.39400000000000002},"min":{"1min":0.23100000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.60799999999999998,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39600000000000002,"15min":0.38800000000000001},"min":{"1min":0.253,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.35399999999999998}]},{"osd":3,"last update":"Sat Aug 17 16:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.46899999999999997,"15min":0.46400000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.443,"15min":0.441},"min":{"1min":0.32200000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.48099999999999998}]},{"osd":4,"last update":"Sat Aug 17 16:52:40 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.58699999999999997,"15min":0.59299999999999997},"min":{"1min":0.379,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.752,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.60799999999999998,"15min":0.61199999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77700000000000002,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.45900000000000002}]},{"osd":5,"last update":"Sat Aug 17 16:53:03 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.61199999999999999,"15min":0.61799999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.84999999999999998,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.58699999999999997,"15min":0.59099999999999997},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.50600000000000001}]},{"osd":6,"last update":"Sat Aug 17 16:52:28 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.439},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.42399999999999999}]},{"osd":7,"last update":"Sat Aug 17 16:52:48 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.56200000000000006}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T16:53:23.580 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T16:53:23.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:23 smithi063 ceph-mon[42680]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:53:23.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:23 smithi063 ceph-mon[31063]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:53:23.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:23 smithi162 ceph-mon[34633]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:53:24.123 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:25.228 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:25 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:25.228 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:25 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:25.228 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:25 smithi063 ceph-mon[42680]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:25.229 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:25 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:25.229 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:25 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:25.229 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:25 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:25.229 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:25 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:25.229 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:25 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:25.229 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:25 smithi063 ceph-mon[31063]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:25.229 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:25 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:25.229 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:25 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:25.230 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:25 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:25.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:25 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:25 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:25 smithi162 ceph-mon[34633]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:25 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:25 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:25 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:26.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:26 smithi063 ceph-mon[42680]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T16:53:26.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:26 smithi063 ceph-mon[42680]: Reconfiguring daemon alertmanager.a on smithi063 2024-08-17T16:53:26.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:26 smithi063 ceph-mon[31063]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T16:53:26.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:26 smithi063 ceph-mon[31063]: Reconfiguring daemon alertmanager.a on smithi063 2024-08-17T16:53:26.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:26 smithi162 ceph-mon[34633]: Reconfiguring alertmanager.a (dependencies changed)... 2024-08-17T16:53:26.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:26 smithi162 ceph-mon[34633]: Reconfiguring daemon alertmanager.a on smithi063 2024-08-17T16:53:26.715 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:53:26.716 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T16:53:27.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:27 smithi063 ceph-mon[42680]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:27.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:27 smithi063 ceph-mon[31063]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:27.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:27 smithi162 ceph-mon[34633]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:28.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:28 smithi162 ceph-mon[34633]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:53:28.418 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:28 smithi063 ceph-mon[31063]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:53:28.418 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:28 smithi063 ceph-mon[42680]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:53:28.442 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":43,"stamp":"2024-08-17T16:53:26.421201+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":223188,"kb_used_data":7028,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":749738028,"statfs":{"total":767960285184,"available":767731740672,"internally_reserved":0,"allocated":7196672,"data_stored":4055922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12742,"internal_metadata":221236794},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001738"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:35.137319+0000","last_change":"2024-08-17T16:52:28.102166+0000","last_active":"2024-08-17T16:52:35.137319+0000","last_peered":"2024-08-17T16:52:35.137319+0000","last_clean":"2024-08-17T16:52:35.137319+0000","last_became_active":"2024-08-17T16:52:23.430739+0000","last_became_peered":"2024-08-17T16:52:23.430739+0000","last_unstale":"2024-08-17T16:52:35.137319+0000","last_undegraded":"2024-08-17T16:52:35.137319+0000","last_fullsized":"2024-08-17T16:52:35.137319+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:19.139627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:34.780003+0000","last_change":"2024-08-17T16:52:28.097477+0000","last_active":"2024-08-17T16:52:34.780003+0000","last_peered":"2024-08-17T16:52:34.780003+0000","last_clean":"2024-08-17T16:52:34.780003+0000","last_became_active":"2024-08-17T16:52:23.816540+0000","last_became_peered":"2024-08-17T16:52:23.816540+0000","last_unstale":"2024-08-17T16:52:34.780003+0000","last_undegraded":"2024-08-17T16:52:34.780003+0000","last_fullsized":"2024-08-17T16:52:34.780003+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:27:49.345586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000334624,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:35.551278+0000","last_change":"2024-08-17T16:52:28.243251+0000","last_active":"2024-08-17T16:52:35.551278+0000","last_peered":"2024-08-17T16:52:35.551278+0000","last_clean":"2024-08-17T16:52:35.551278+0000","last_became_active":"2024-08-17T16:52:23.816512+0000","last_became_peered":"2024-08-17T16:52:23.816512+0000","last_unstale":"2024-08-17T16:52:35.551278+0000","last_undegraded":"2024-08-17T16:52:35.551278+0000","last_fullsized":"2024-08-17T16:52:35.551278+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:59.683846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:34.779906+0000","last_change":"2024-08-17T16:52:28.097582+0000","last_active":"2024-08-17T16:52:34.779906+0000","last_peered":"2024-08-17T16:52:34.779906+0000","last_clean":"2024-08-17T16:52:34.779906+0000","last_became_active":"2024-08-17T16:52:23.739237+0000","last_became_peered":"2024-08-17T16:52:23.739237+0000","last_unstale":"2024-08-17T16:52:34.779906+0000","last_undegraded":"2024-08-17T16:52:34.779906+0000","last_fullsized":"2024-08-17T16:52:34.779906+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:02:16.574623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":22,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:35.551207+0000","last_change":"2024-08-17T16:52:28.101249+0000","last_active":"2024-08-17T16:52:35.551207+0000","last_peered":"2024-08-17T16:52:35.551207+0000","last_clean":"2024-08-17T16:52:35.551207+0000","last_became_active":"2024-08-17T16:52:23.430593+0000","last_became_peered":"2024-08-17T16:52:23.430593+0000","last_unstale":"2024-08-17T16:52:35.551207+0000","last_undegraded":"2024-08-17T16:52:35.551207+0000","last_fullsized":"2024-08-17T16:52:35.551207+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:19.292172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:34.780039+0000","last_change":"2024-08-17T16:52:28.097703+0000","last_active":"2024-08-17T16:52:34.780039+0000","last_peered":"2024-08-17T16:52:34.780039+0000","last_clean":"2024-08-17T16:52:34.780039+0000","last_became_active":"2024-08-17T16:52:23.739224+0000","last_became_peered":"2024-08-17T16:52:23.739224+0000","last_unstale":"2024-08-17T16:52:34.780039+0000","last_undegraded":"2024-08-17T16:52:34.780039+0000","last_fullsized":"2024-08-17T16:52:34.780039+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:33.382141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:35.551201+0000","last_change":"2024-08-17T16:52:28.243231+0000","last_active":"2024-08-17T16:52:35.551201+0000","last_peered":"2024-08-17T16:52:35.551201+0000","last_clean":"2024-08-17T16:52:35.551201+0000","last_became_active":"2024-08-17T16:52:23.739287+0000","last_became_peered":"2024-08-17T16:52:23.739287+0000","last_unstale":"2024-08-17T16:52:35.551201+0000","last_undegraded":"2024-08-17T16:52:35.551201+0000","last_fullsized":"2024-08-17T16:52:35.551201+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:39.119833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-08-17T16:52:35.551505+0000","last_change":"2024-08-17T16:52:28.100670+0000","last_active":"2024-08-17T16:52:35.551505+0000","last_peered":"2024-08-17T16:52:35.551505+0000","last_clean":"2024-08-17T16:52:35.551505+0000","last_became_active":"2024-08-17T16:52:23.430320+0000","last_became_peered":"2024-08-17T16:52:23.430320+0000","last_unstale":"2024-08-17T16:52:35.551505+0000","last_undegraded":"2024-08-17T16:52:35.551505+0000","last_fullsized":"2024-08-17T16:52:35.551505+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:47.861974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":263,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-08-17T16:52:28.242917+0000","last_change":"2024-08-17T16:51:52.132475+0000","last_active":"2024-08-17T16:52:28.242917+0000","last_peered":"2024-08-17T16:52:28.242917+0000","last_clean":"2024-08-17T16:52:28.242917+0000","last_became_active":"2024-08-17T16:51:52.132088+0000","last_became_peered":"2024-08-17T16:51:52.132088+0000","last_unstale":"2024-08-17T16:52:28.242917+0000","last_undegraded":"2024-08-17T16:52:28.242917+0000","last_fullsized":"2024-08-17T16:52:28.242917+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:13.679985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201302,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28256,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716896,"statfs":{"total":95995035648,"available":95966101504,"internally_reserved":0,"allocated":1265664,"data_stored":875688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.42399999999999999}]},{"osd":1,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.30599999999999999}]},{"osd":2,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.012,"5min":1.012,"15min":1.012},"last":0.32700000000000001}]},{"osd":3,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.374}]},{"osd":4,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.621,"5min":0.621,"15min":0.621},"last":0.34000000000000002}]},{"osd":5,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.40000000000000002}]},{"osd":6,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.42999999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462939,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28256,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716896,"statfs":{"total":95995035648,"available":95966101504,"internally_reserved":0,"allocated":1265664,"data_stored":875434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":27654601},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:52:27 2024","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.5}]},{"osd":1,"last update":"Sat Aug 17 16:52:27 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.53300000000000003}]},{"osd":2,"last update":"Sat Aug 17 16:52:27 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.40600000000000003}]},{"osd":3,"last update":"Sat Aug 17 16:52:27 2024","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.46400000000000002}]},{"osd":4,"last update":"Sat Aug 17 16:52:27 2024","interfaces":[{"interface":"back","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.64500000000000002}]},{"osd":5,"last update":"Sat Aug 17 16:52:27 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.57399999999999995}]},{"osd":7,"last update":"Sat Aug 17 16:52:51 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.63400000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724574,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:53:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52300000000000002,"15min":0.52100000000000002},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.72099999999999997,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53200000000000003,"15min":0.53600000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.52700000000000002}]},{"osd":1,"last update":"Sat Aug 17 16:53:05 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52700000000000002,"15min":0.52800000000000002},"min":{"1min":0.34899999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.54900000000000004,"15min":0.55100000000000005},"min":{"1min":0.32100000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.80900000000000005,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.57599999999999996}]},{"osd":2,"last update":"Sat Aug 17 16:53:05 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54000000000000004,"15min":0.53900000000000003},"min":{"1min":0.43099999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.69599999999999995,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.53700000000000003,"15min":0.53900000000000003},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77200000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.56299999999999994}]},{"osd":3,"last update":"Sat Aug 17 16:53:05 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.54400000000000004,"15min":0.54600000000000004},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66800000000000004,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.55800000000000005,"15min":0.56100000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.71799999999999997,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.61799999999999999}]},{"osd":4,"last update":"Sat Aug 17 16:53:05 2024","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.38800000000000001,"15min":0.38400000000000001},"min":{"1min":0.26100000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.56499999999999995,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.41899999999999998,"15min":0.41399999999999998},"min":{"1min":0.33000000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.61299999999999999,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Aug 17 16:52:29 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.438,"15min":0.438},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.02,"5min":1.02,"15min":1.02},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Aug 17 16:52:52 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.39100000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986211,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27680,"kb_used_data":660,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717472,"statfs":{"total":95995035648,"available":95966691328,"internally_reserved":0,"allocated":675840,"data_stored":285651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:52:38 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.40400000000000003,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.71799999999999997,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.53900000000000003,"15min":0.53600000000000003},"min":{"1min":0.377,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.68899999999999995,"5min":0.749,"15min":0.749},"last":0.68100000000000005}]},{"osd":1,"last update":"Sat Aug 17 16:52:38 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.36299999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77600000000000002,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.53100000000000003,"15min":0.52600000000000002},"min":{"1min":0.42399999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.746,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.66900000000000004}]},{"osd":2,"last update":"Sat Aug 17 16:52:38 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.56399999999999995,"15min":0.55900000000000005},"min":{"1min":0.45800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.79000000000000004,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.35899999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.71099999999999997,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.73699999999999999}]},{"osd":3,"last update":"Sat Aug 17 16:52:38 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.40100000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55700000000000005,"15min":0.55900000000000005},"min":{"1min":0.375,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.69799999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.70399999999999996}]},{"osd":5,"last update":"Sat Aug 17 16:53:06 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.40600000000000003,"15min":0.40699999999999997},"min":{"1min":0.27600000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.51000000000000001,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.41399999999999998,"15min":0.41599999999999998},"min":{"1min":0.25600000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.56699999999999995,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.443}]},{"osd":6,"last update":"Sat Aug 17 16:52:25 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.496},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.51400000000000001}]},{"osd":7,"last update":"Sat Aug 17 16:52:48 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.45800000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149737,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":0.37,"15min":0.373},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.41199999999999998,"15min":0.41599999999999998},"min":{"1min":0.28199999999999997,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.59799999999999998,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.47299999999999998}]},{"osd":1,"last update":"Sat Aug 17 16:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.41999999999999998,"15min":0.42599999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.61399999999999999,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.439},{"interface":"front","average":{"1min":0.378,"5min":0.432,"15min":0.44600000000000001},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.61399999999999999,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.48899999999999999}]},{"osd":2,"last update":"Sat Aug 17 16:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.39000000000000001,"5min":0.434,"15min":0.438},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.57699999999999996,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.42499999999999999,"15min":0.42999999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.56000000000000005,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.45400000000000001}]},{"osd":4,"last update":"Sat Aug 17 16:52:41 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.57999999999999996,"15min":0.58199999999999996},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.69799999999999995,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.628},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.56200000000000006,"15min":0.56399999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.70299999999999996,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.57399999999999995}]},{"osd":5,"last update":"Sat Aug 17 16:53:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.58399999999999996,"15min":0.58799999999999997},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.69099999999999995,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.56999999999999995,"15min":0.57099999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.748,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.65400000000000003}]},{"osd":6,"last update":"Sat Aug 17 16:52:29 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.70399999999999996}]},{"osd":7,"last update":"Sat Aug 17 16:52:50 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.66600000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:52:57 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.39500000000000002,"15min":0.39300000000000002},"min":{"1min":0.245,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.40400000000000003,"15min":0.39200000000000002},"min":{"1min":0.26400000000000001,"5min":0.245,"15min":0.245},"max":{"1min":0.59299999999999997,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.35499999999999998}]},{"osd":1,"last update":"Sat Aug 17 16:52:57 2024","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.41699999999999998,"15min":0.40699999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.60899999999999999,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.42799999999999999,"15min":0.42399999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.67700000000000005,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.24099999999999999}]},{"osd":3,"last update":"Sat Aug 17 16:53:23 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.47199999999999998,"15min":0.48399999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.65700000000000003,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.436},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.47299999999999998,"15min":0.48399999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.63700000000000001,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.28799999999999998}]},{"osd":4,"last update":"Sat Aug 17 16:52:42 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.64100000000000001,"15min":0.65800000000000003},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.878,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.65600000000000003,"15min":0.67000000000000004},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.86499999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.40799999999999997}]},{"osd":5,"last update":"Sat Aug 17 16:53:05 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.42699999999999999,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.64000000000000001,"15min":0.65200000000000002},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.80700000000000005,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Aug 17 16:52:28 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.999,"5min":0.999,"15min":0.999},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.39200000000000002}]},{"osd":7,"last update":"Sat Aug 17 16:52:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.49399999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:52:33 2024","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.34499999999999997,"15min":0.33900000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.51900000000000002,"5min":0.53300000000000003,"15min":0.53300000000000003},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.36699999999999999,"15min":0.35199999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.505}]},{"osd":2,"last update":"Sat Aug 17 16:53:01 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.37,"15min":0.36299999999999999},"min":{"1min":0.254,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.54400000000000004,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.378,"5min":0.36699999999999999,"15min":0.35699999999999998},"min":{"1min":0.26100000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.46899999999999997}]},{"osd":3,"last update":"Sat Aug 17 16:53:19 2024","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.40000000000000002,"15min":0.40500000000000003},"min":{"1min":0.27000000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.51500000000000001,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.496},{"interface":"front","average":{"1min":0.434,"5min":0.45800000000000002,"15min":0.46800000000000003},"min":{"1min":0.32400000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.53200000000000003}]},{"osd":4,"last update":"Sat Aug 17 16:52:42 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.56299999999999994,"15min":0.55900000000000005},"min":{"1min":0.434,"5min":0.315,"15min":0.315},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.53100000000000003,"15min":0.52600000000000002},"min":{"1min":0.39700000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.83999999999999997}]},{"osd":5,"last update":"Sat Aug 17 16:53:01 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59099999999999997,"15min":0.58999999999999997},"min":{"1min":0.46300000000000002,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.747}]},{"osd":6,"last update":"Sat Aug 17 16:52:30 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.82799999999999996}]},{"osd":7,"last update":"Sat Aug 17 16:52:49 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.88500000000000001,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.86699999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28260,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716892,"statfs":{"total":95995035648,"available":95966097408,"internally_reserved":0,"allocated":1269760,"data_stored":876019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 16:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.34000000000000002,"15min":0.32600000000000001},"min":{"1min":0.23400000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.58999999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.35599999999999998,"15min":0.33400000000000002},"min":{"1min":0.27400000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.41699999999999998}]},{"osd":2,"last update":"Sat Aug 17 16:53:00 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.39600000000000002,"15min":0.39400000000000002},"min":{"1min":0.23100000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.60799999999999998,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.39000000000000001,"5min":0.39600000000000002,"15min":0.38800000000000001},"min":{"1min":0.253,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.39800000000000002}]},{"osd":3,"last update":"Sat Aug 17 16:53:23 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47099999999999997,"15min":0.46500000000000002},"min":{"1min":0.36599999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.054,"5min":1.054,"15min":1.054},"last":0.435},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.44,"15min":0.44},"min":{"1min":0.27400000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.36899999999999999}]},{"osd":4,"last update":"Sat Aug 17 16:52:40 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.58699999999999997,"15min":0.59299999999999997},"min":{"1min":0.379,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.752,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.60799999999999998,"15min":0.61199999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77700000000000002,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.61099999999999999}]},{"osd":5,"last update":"Sat Aug 17 16:53:03 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.61199999999999999,"15min":0.61799999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.84999999999999998,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.58699999999999997,"15min":0.59099999999999997},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.54900000000000004}]},{"osd":6,"last update":"Sat Aug 17 16:52:28 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.58899999999999997}]},{"osd":7,"last update":"Sat Aug 17 16:52:48 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.48699999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T16:53:28.444 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T16:53:28.445 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-17T16:53:28.445 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-17T16:53:28.445 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph health --format=json 2024-08-17T16:53:28.997 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:29.054 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:28 smithi063 systemd[1]: Stopping Ceph alertmanager.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:53:29.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:29 smithi162 ceph-mon[34633]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:29.365 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:29 smithi063 ceph-mon[42680]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:29.366 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:29 smithi063 ceph-mon[31063]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:29.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:29 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:53:29] "GET /metrics HTTP/1.1" 200 28510 "" "Prometheus/2.43.0" 2024-08-17T16:53:29.780 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:29 smithi063 podman[78977]: 2024-08-17 16:53:29.474212246 +0000 UTC m=+0.510962946 container died 6bd349eb2f900ffe3fbf6fa73c25041a854608ad70a49cfcf6c013398efecc56 (image=quay.ceph.io/ceph-ci/ceph:reef, name=serene_ardinghelli, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.36.0, RELEASE=reef-ced649f, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T16:53:30.104 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:29 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[73479]: ts=2024-08-17T16:53:29.779Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-17T16:53:30.104 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:29 smithi063 podman[79069]: 2024-08-17 16:53:29.800392509 +0000 UTC m=+0.136848726 container died 80384178e8bd35687c8694eafc3e3e0830b0592272e7dca71f068d2d6c0c7b39 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:53:30.370 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:30 smithi063 podman[79069]: 2024-08-17 16:53:30.202536752 +0000 UTC m=+0.538992968 container cleanup 80384178e8bd35687c8694eafc3e3e0830b0592272e7dca71f068d2d6c0c7b39 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:53:30.370 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:30 smithi063 bash[79069]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a 2024-08-17T16:53:30.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:30 smithi063 ceph-mon[42680]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:30.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:30 smithi063 ceph-mon[31063]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:30.780 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:30 smithi063 podman[79087]: 2024-08-17 16:53:30.370138724 +0000 UTC m=+0.567417100 container remove 80384178e8bd35687c8694eafc3e3e0830b0592272e7dca71f068d2d6c0c7b39 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:53:30.780 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:30 smithi063 podman[79087]: 2024-08-17 16:53:30.454138898 +0000 UTC m=+0.651417277 volume remove f4a820f68039d8cfdfcaacb7bdbf2e69ca45a203484f3b1d3e622dbcab15e961 2024-08-17T16:53:30.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:30 smithi162 ceph-mon[34633]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:31.617 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:53:31.618 INFO:teuthology.orchestra.run.smithi063.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-17T16:53:31.753 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:31 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/2735908466' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T16:53:31.753 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:31 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/2735908466' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T16:53:31.753 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:31 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-17T16:53:31.753 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:31 smithi063 systemd[1]: Stopped Ceph alertmanager.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:53:31.754 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:31 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@alertmanager.a.service: Consumed 1.009s CPU time. 2024-08-17T16:53:31.754 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:31 smithi063 systemd[1]: Starting Ceph alertmanager.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:53:31.754 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:31 smithi063 podman[79258]: 2024-08-17 16:53:31.652069513 +0000 UTC m=+0.109966963 volume create cb54b9f4476e509ca3bc0e3d9f7f94b73162f13e3b648065b667ae7d550584f4 2024-08-17T16:53:31.754 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:31 smithi063 podman[79258]: 2024-08-17 16:53:31.560535781 +0000 UTC m=+0.018433241 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-08-17T16:53:31.754 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:31 smithi063 podman[79258]: 2024-08-17 16:53:31.752606703 +0000 UTC m=+0.210504153 container create 87286b24f231cfa279dbf4c7540a822b009995a8c2c1551794a462fc2d597cde (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:53:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:31 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/2735908466' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T16:53:32.619 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-17T16:53:32.619 INFO:tasks.cephadm:Setup complete, yielding 2024-08-17T16:53:32.619 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-08-17T16:53:32.629 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi063.front.sepia.ceph.com 2024-08-17T16:53:32.629 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-08-17T16:53:32.901 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:33.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:32 smithi063 ceph-mon[42680]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:33.025 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:32 smithi063 ceph-mon[31063]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:33.025 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:32 smithi063 podman[79258]: 2024-08-17 16:53:32.709560705 +0000 UTC m=+1.167458156 container init 87286b24f231cfa279dbf4c7540a822b009995a8c2c1551794a462fc2d597cde (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:53:33.025 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:32 smithi063 podman[79258]: 2024-08-17 16:53:32.715482137 +0000 UTC m=+1.173379595 container start 87286b24f231cfa279dbf4c7540a822b009995a8c2c1551794a462fc2d597cde (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:53:33.025 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:32 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[79286]: ts=2024-08-17T16:53:32.728Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-08-17T16:53:33.025 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:32 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[79286]: ts=2024-08-17T16:53:32.728Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-08-17T16:53:33.025 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:32 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[79286]: ts=2024-08-17T16:53:32.729Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.63 port=9094 2024-08-17T16:53:33.025 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:32 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[79286]: ts=2024-08-17T16:53:32.730Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-17T16:53:33.026 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:32 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[79286]: ts=2024-08-17T16:53:32.762Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T16:53:33.026 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:32 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[79286]: ts=2024-08-17T16:53:32.763Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T16:53:33.026 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:32 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[79286]: ts=2024-08-17T16:53:32.766Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-08-17T16:53:33.026 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:32 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[79286]: ts=2024-08-17T16:53:32.766Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-17T16:53:33.026 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:32 smithi063 bash[79258]: 87286b24f231cfa279dbf4c7540a822b009995a8c2c1551794a462fc2d597cde 2024-08-17T16:53:33.026 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:32 smithi063 systemd[1]: Started Ceph alertmanager.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:53:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:32 smithi162 ceph-mon[34633]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:34.065 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:33 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:34.065 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:33 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:34.065 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:33 smithi162 ceph-mon[34633]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T16:53:34.065 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:33 smithi162 ceph-mon[34633]: Reconfiguring daemon prometheus.a on smithi162 2024-08-17T16:53:34.133 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:33 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:34.134 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:33 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:34.134 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:33 smithi063 ceph-mon[42680]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T16:53:34.134 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:33 smithi063 ceph-mon[42680]: Reconfiguring daemon prometheus.a on smithi162 2024-08-17T16:53:34.134 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:33 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:34.134 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:33 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:34.134 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:33 smithi063 ceph-mon[31063]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T16:53:34.134 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:33 smithi063 ceph-mon[31063]: Reconfiguring daemon prometheus.a on smithi162 2024-08-17T16:53:35.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:34 smithi063 ceph-mon[42680]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:34 smithi063 ceph-mon[31063]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:35.030 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:34 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[79286]: ts=2024-08-17T16:53:34.730Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000544262s 2024-08-17T16:53:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:34 smithi162 ceph-mon[34633]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:36.269 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:35 smithi162 ceph-mon[34633]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:53:36.269 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-17T16:53:36.270 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:53:36.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:35 smithi063 ceph-mon[42680]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:53:36.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-17T16:53:36.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:53:36.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:35 smithi063 ceph-mon[31063]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:53:36.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-08-17T16:53:36.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:53:36.596 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:36 smithi162 systemd[1]: Stopping Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:53:37.008 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-08-17T16:53:37.008 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-mon[34633]: osdmap e68: 8 total, 8 up, 8 in 2024-08-17T16:53:37.008 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-17T16:53:37.008 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-mon[34633]: pgmap v49: 41 pgs: 32 creating+peering, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:37.008 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:53:36.700Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T16:53:37.008 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:53:36.701Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-17T16:53:37.009 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:53:36.701Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-17T16:53:37.009 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:53:36.701Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T16:53:37.009 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:53:36.701Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-17T16:53:37.009 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:53:36.701Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-17T16:53:37.009 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:53:36.701Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T16:53:37.009 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:53:36.701Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-17T16:53:37.009 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:53:36.701Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-17T16:53:37.009 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:53:36.701Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T16:53:37.009 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:53:36.701Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-17T16:53:37.009 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[62866]: ts=2024-08-17T16:53:36.701Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-17T16:53:37.010 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:36 smithi162 podman[66170]: 2024-08-17 16:53:36.711161836 +0000 UTC m=+0.140915440 container died ab5c581becf8935c8c499aa24e700a67a9b416c59467195b33d0cce8781cc30a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:53:37.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:36 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-08-17T16:53:37.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:36 smithi063 ceph-mon[31063]: osdmap e68: 8 total, 8 up, 8 in 2024-08-17T16:53:37.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:36 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-17T16:53:37.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:36 smithi063 ceph-mon[31063]: pgmap v49: 41 pgs: 32 creating+peering, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:37.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a[31040]: 2024-08-17T16:53:36.879+0000 7f914de80640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T16:53:37.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:36 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-08-17T16:53:37.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:36 smithi063 ceph-mon[42680]: osdmap e68: 8 total, 8 up, 8 in 2024-08-17T16:53:37.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:36 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-08-17T16:53:37.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:36 smithi063 ceph-mon[42680]: pgmap v49: 41 pgs: 32 creating+peering, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:37.345 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:37 smithi162 podman[66170]: 2024-08-17 16:53:37.199149788 +0000 UTC m=+0.628903380 container cleanup ab5c581becf8935c8c499aa24e700a67a9b416c59467195b33d0cce8781cc30a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:53:37.346 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:37 smithi162 bash[66170]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T16:53:37.684 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-08-17T16:53:37.844 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:37 smithi162 podman[66187]: 2024-08-17 16:53:37.450995862 +0000 UTC m=+0.736818215 container remove ab5c581becf8935c8c499aa24e700a67a9b416c59467195b33d0cce8781cc30a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:53:37.844 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:37 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-17T16:53:37.844 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:37 smithi162 systemd[1]: Stopped Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:53:37.844 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:37 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@prometheus.a.service: Consumed 1.184s CPU time. 2024-08-17T16:53:37.844 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:37 smithi162 systemd[1]: Starting Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:53:37.910 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[42680]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T16:53:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-08-17T16:53:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[42680]: osdmap e69: 8 total, 8 up, 8 in 2024-08-17T16:53:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-17T16:53:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[42680]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T16:53:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[42680]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-08-17T16:53:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[42680]: osdmap e70: 8 total, 8 up, 8 in 2024-08-17T16:53:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-08-17T16:53:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[42680]: fsmap foo:0 2024-08-17T16:53:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[42680]: Saving service mds.foo spec with placement count:2 2024-08-17T16:53:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:38.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[31063]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T16:53:38.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-08-17T16:53:38.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[31063]: osdmap e69: 8 total, 8 up, 8 in 2024-08-17T16:53:38.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-17T16:53:38.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[31063]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T16:53:38.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[31063]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-08-17T16:53:38.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[31063]: osdmap e70: 8 total, 8 up, 8 in 2024-08-17T16:53:38.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-08-17T16:53:38.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[31063]: fsmap foo:0 2024-08-17T16:53:38.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[31063]: Saving service mds.foo spec with placement count:2 2024-08-17T16:53:38.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:37 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:38.095 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:37 smithi162 podman[66287]: 2024-08-17 16:53:37.843546214 +0000 UTC m=+0.111891954 container create 015db799d7d403549212d342df5854ba9d1ce55ea85c6a7dfac03d778351fa3a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:53:38.096 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:37 smithi162 podman[66287]: 2024-08-17 16:53:37.75036934 +0000 UTC m=+0.018715081 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-17T16:53:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:37 smithi162 ceph-mon[34633]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T16:53:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:37 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-08-17T16:53:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:37 smithi162 ceph-mon[34633]: osdmap e69: 8 total, 8 up, 8 in 2024-08-17T16:53:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:37 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-08-17T16:53:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:37 smithi162 ceph-mon[34633]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-08-17T16:53:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:37 smithi162 ceph-mon[34633]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-08-17T16:53:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:37 smithi162 ceph-mon[34633]: osdmap e70: 8 total, 8 up, 8 in 2024-08-17T16:53:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:37 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-08-17T16:53:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:37 smithi162 ceph-mon[34633]: fsmap foo:0 2024-08-17T16:53:38.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:37 smithi162 ceph-mon[34633]: Saving service mds.foo spec with placement count:2 2024-08-17T16:53:38.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:37 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:38.596 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 podman[66287]: 2024-08-17 16:53:38.133989567 +0000 UTC m=+0.402335307 container init 015db799d7d403549212d342df5854ba9d1ce55ea85c6a7dfac03d778351fa3a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:53:38.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 podman[66287]: 2024-08-17 16:53:38.138577779 +0000 UTC m=+0.406923517 container start 015db799d7d403549212d342df5854ba9d1ce55ea85c6a7dfac03d778351fa3a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:53:38.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.163Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-17T16:53:38.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.163Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-17T16:53:38.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.163Z caller=main.go:570 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi162 (none))" 2024-08-17T16:53:38.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.163Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T16:53:38.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.163Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T16:53:38.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.164Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T16:53:38.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.164Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-17T16:53:38.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.165Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T16:53:38.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.165Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T16:53:38.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.168Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T16:53:38.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.168Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.056µs 2024-08-17T16:53:38.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.168Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T16:53:38.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.169Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-08-17T16:53:38.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.169Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-08-17T16:53:38.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.169Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.119µs wal_replay_duration=1.417978ms wbl_replay_duration=100ns total_replay_duration=1.457754ms 2024-08-17T16:53:38.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.170Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T16:53:38.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.170Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-17T16:53:38.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.170Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T16:53:38.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.189Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.408847ms db_storage=799ns remote_storage=1.019µs web_handler=469ns query_engine=563ns scrape=552.202µs scrape_sd=85µs notify=16.604µs notify_sd=15.954µs rules=18.432518ms tracing=6.107µs 2024-08-17T16:53:38.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.189Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-17T16:53:38.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:53:38.189Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T16:53:38.598 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 bash[66287]: 015db799d7d403549212d342df5854ba9d1ce55ea85c6a7dfac03d778351fa3a 2024-08-17T16:53:38.599 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:53:38 smithi162 systemd[1]: Started Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:53:38.712 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:38] ENGINE Bus STOPPING 2024-08-17T16:53:38.712 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T16:53:38.712 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:38] ENGINE Bus STOPPED 2024-08-17T16:53:39.022 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:38] ENGINE Bus STARTING 2024-08-17T16:53:39.022 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:38] ENGINE Serving on http://:::9283 2024-08-17T16:53:39.022 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:38] ENGINE Bus STARTED 2024-08-17T16:53:39.022 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:38] ENGINE Bus STOPPING 2024-08-17T16:53:39.023 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: osdmap e71: 8 total, 8 up, 8 in 2024-08-17T16:53:39.023 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:39.023 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:39.023 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:53:39.023 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:53:39.023 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi063.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T16:53:39.023 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi063.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T16:53:39.023 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:39.023 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:53:39.023 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:53:39.024 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi162.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T16:53:39.024 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi162.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T16:53:39.024 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:39.024 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:53:39.024 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:53:39.024 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi162.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T16:53:39.024 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi162.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T16:53:39.024 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:39.024 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:39.024 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[31063]: pgmap v53: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:39.024 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: osdmap e71: 8 total, 8 up, 8 in 2024-08-17T16:53:39.024 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:39.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:39.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:53:39.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:53:39.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi063.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T16:53:39.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi063.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T16:53:39.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:39.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:53:39.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:53:39.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi162.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T16:53:39.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi162.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T16:53:39.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:39.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:53:39.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:53:39.026 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi162.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T16:53:39.026 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi162.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T16:53:39.026 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:39.026 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:39.026 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:38 smithi063 ceph-mon[42680]: pgmap v53: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:39.200 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: osdmap e71: 8 total, 8 up, 8 in 2024-08-17T16:53:39.200 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:39.200 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:39.201 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:53:39.201 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:53:39.201 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi063.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T16:53:39.201 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi063.front.sepia.ceph.com:9093"}]: dispatch 2024-08-17T16:53:39.201 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:39.201 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:53:39.201 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:53:39.201 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi162.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T16:53:39.201 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi162.front.sepia.ceph.com:9095"}]: dispatch 2024-08-17T16:53:39.201 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:39.201 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:53:39.201 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:53:39.202 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi162.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T16:53:39.202 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi162.front.sepia.ceph.com:3000"}]: dispatch 2024-08-17T16:53:39.202 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:39.202 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:39.202 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:38 smithi162 ceph-mon[34633]: pgmap v53: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:39.681 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T16:53:39.681 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:39] ENGINE Bus STOPPED 2024-08-17T16:53:39.681 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:39] ENGINE Bus STARTING 2024-08-17T16:53:39.681 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:39] ENGINE Serving on http://:::9283 2024-08-17T16:53:39.681 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:39] ENGINE Bus STARTED 2024-08-17T16:53:39.681 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:39] ENGINE Bus STOPPING 2024-08-17T16:53:40.027 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:39 smithi063 ceph-mon[31063]: osdmap e72: 8 total, 8 up, 8 in 2024-08-17T16:53:40.027 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:39 smithi063 ceph-mon[31063]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T16:53:40.028 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:39 smithi063 ceph-mon[42680]: osdmap e72: 8 total, 8 up, 8 in 2024-08-17T16:53:40.028 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:39 smithi063 ceph-mon[42680]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T16:53:40.028 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T16:53:40.028 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:39] ENGINE Bus STOPPED 2024-08-17T16:53:40.028 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:39] ENGINE Bus STARTING 2024-08-17T16:53:40.028 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:40 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:40] ENGINE Serving on http://:::9283 2024-08-17T16:53:40.028 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:40 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:53:40] ENGINE Bus STARTED 2024-08-17T16:53:40.161 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:39 smithi162 ceph-mon[34633]: osdmap e72: 8 total, 8 up, 8 in 2024-08-17T16:53:40.162 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:39 smithi162 ceph-mon[34633]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T16:53:40.449 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-08-17T16:53:41.003 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:41.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:40 smithi162 ceph-mon[34633]: pgmap v55: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:41.207 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:40 smithi063 ceph-mon[42680]: pgmap v55: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:41.207 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:40 smithi063 ceph-mon[31063]: pgmap v55: 73 pgs: 32 unknown, 32 creating+peering, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:42.958 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:53:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[79286]: ts=2024-08-17T16:53:42.733Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003432411s 2024-08-17T16:53:42.959 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:42 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:42.959 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:42 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:42.959 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:42 smithi063 ceph-mon[42680]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:42.959 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:42 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:42.959 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:42 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:42.959 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:42 smithi063 ceph-mon[31063]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:42 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:42 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:42 smithi162 ceph-mon[34633]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:44.274 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-08-17T16:53:44.284 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-17T16:53:44.284 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-17T16:53:44.284 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph mgr dump --format=json 2024-08-17T16:53:44.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:44 smithi063 ceph-mon[42680]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:44.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:44 smithi063 ceph-mon[31063]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:44.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:44 smithi162 ceph-mon[34633]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:45.028 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi063.ultpqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T16:53:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi063.ultpqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T16:53:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[42680]: Deploying daemon mds.foo.smithi063.ultpqj on smithi063 2024-08-17T16:53:46.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:46.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:46.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:46.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:46.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:46.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi063.ultpqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T16:53:46.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi063.ultpqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T16:53:46.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:46.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:46 smithi063 ceph-mon[31063]: Deploying daemon mds.foo.smithi063.ultpqj on smithi063 2024-08-17T16:53:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:46 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:46 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:46 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:46 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:53:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:46 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:46 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi063.ultpqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T16:53:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:46 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi063.ultpqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T16:53:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:46 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:46.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:46 smithi162 ceph-mon[34633]: Deploying daemon mds.foo.smithi063.ultpqj on smithi063 2024-08-17T16:53:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:47 smithi063 ceph-mon[31063]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:47.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:47 smithi063 ceph-mon[42680]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:47.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:47 smithi162 ceph-mon[34633]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:48.135 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:53:48.509 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:48 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/1806192265' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T16:53:48.509 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:48 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/1806192265' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T16:53:48.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:48 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/1806192265' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-08-17T16:53:48.782 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":21,"flags":0,"active_gid":24335,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6800","nonce":987519051},{"type":"v1","addr":"172.21.15.63:6801","nonce":987519051}]},"active_addr":"172.21.15.63:6801/987519051","active_change":"2024-08-17T16:52:20.398656+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24352,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.63:8443/","prometheus":"http://172.21.15.63:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":2696979598}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":3589494813}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":3746443558}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":4067771896}]}]} 2024-08-17T16:53:48.787 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-17T16:53:48.787 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T16:53:48.787 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T16:53:49.278 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:49 smithi063 ceph-mon[42680]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:49.278 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:49 smithi063 ceph-mon[31063]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:49.393 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:49.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:49 smithi162 ceph-mon[34633]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:49.814 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:49 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:53:49] "GET /metrics HTTP/1.1" 200 28502 "" "Prometheus/2.43.0" 2024-08-17T16:53:50.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:50.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:50.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:50.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi162.bjkxnv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T16:53:50.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi162.bjkxnv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T16:53:50.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:50.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[42680]: Deploying daemon mds.foo.smithi162.bjkxnv on smithi162 2024-08-17T16:53:50.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[42680]: daemon mds.foo.smithi063.ultpqj assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-08-17T16:53:50.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[42680]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-08-17T16:53:50.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[42680]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-08-17T16:53:50.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[42680]: Cluster is now healthy 2024-08-17T16:53:50.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[42680]: mds.? [v2:172.21.15.63:6834/3222014296,v1:172.21.15.63:6835/3222014296] up:boot 2024-08-17T16:53:50.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[42680]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:creating} 2024-08-17T16:53:50.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi063.ultpqj"}]: dispatch 2024-08-17T16:53:50.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[42680]: daemon mds.foo.smithi063.ultpqj is now active in filesystem foo as rank 0 2024-08-17T16:53:50.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:50.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:50.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:50.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi162.bjkxnv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T16:53:50.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi162.bjkxnv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T16:53:50.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:50.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[31063]: Deploying daemon mds.foo.smithi162.bjkxnv on smithi162 2024-08-17T16:53:50.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[31063]: daemon mds.foo.smithi063.ultpqj assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-08-17T16:53:50.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[31063]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-08-17T16:53:50.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[31063]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-08-17T16:53:50.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[31063]: Cluster is now healthy 2024-08-17T16:53:50.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[31063]: mds.? [v2:172.21.15.63:6834/3222014296,v1:172.21.15.63:6835/3222014296] up:boot 2024-08-17T16:53:50.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[31063]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:creating} 2024-08-17T16:53:50.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi063.ultpqj"}]: dispatch 2024-08-17T16:53:50.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:50 smithi063 ceph-mon[31063]: daemon mds.foo.smithi063.ultpqj is now active in filesystem foo as rank 0 2024-08-17T16:53:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:50 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:50 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:50 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:50 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi162.bjkxnv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-08-17T16:53:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:50 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi162.bjkxnv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-08-17T16:53:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:50 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:53:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:50 smithi162 ceph-mon[34633]: Deploying daemon mds.foo.smithi162.bjkxnv on smithi162 2024-08-17T16:53:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:50 smithi162 ceph-mon[34633]: daemon mds.foo.smithi063.ultpqj assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-08-17T16:53:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:50 smithi162 ceph-mon[34633]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-08-17T16:53:50.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:50 smithi162 ceph-mon[34633]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-08-17T16:53:50.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:50 smithi162 ceph-mon[34633]: Cluster is now healthy 2024-08-17T16:53:50.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:50 smithi162 ceph-mon[34633]: mds.? [v2:172.21.15.63:6834/3222014296,v1:172.21.15.63:6835/3222014296] up:boot 2024-08-17T16:53:50.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:50 smithi162 ceph-mon[34633]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:creating} 2024-08-17T16:53:50.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:50 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi063.ultpqj"}]: dispatch 2024-08-17T16:53:50.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:50 smithi162 ceph-mon[34633]: daemon mds.foo.smithi063.ultpqj is now active in filesystem foo as rank 0 2024-08-17T16:53:51.307 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:53:51.308 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":72,"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:46:54.156006+0000","modified":"2024-08-17T16:53:38.889498+0000","last_up_change":"2024-08-17T16:51:48.092300+0000","last_in_change":"2024-08-17T16:51:30.856523+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:49:54.431240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:52:22.131465+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T16:53:35.328125+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T16:53:36.292966+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6802","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6803","nonce":782866097}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6804","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6805","nonce":782866097}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6808","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6809","nonce":782866097}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6806","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6807","nonce":782866097}]},"public_addr":"172.21.15.63:6803/782866097","cluster_addr":"172.21.15.63:6805/782866097","heartbeat_back_addr":"172.21.15.63:6809/782866097","heartbeat_front_addr":"172.21.15.63:6807/782866097","state":["exists","up"]},{"osd":1,"uuid":"ae346e99-4706-4871-990a-f8e737572245","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6810","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6811","nonce":944260396}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6812","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6813","nonce":944260396}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6816","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6817","nonce":944260396}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6814","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6815","nonce":944260396}]},"public_addr":"172.21.15.63:6811/944260396","cluster_addr":"172.21.15.63:6813/944260396","heartbeat_back_addr":"172.21.15.63:6817/944260396","heartbeat_front_addr":"172.21.15.63:6815/944260396","state":["exists","up"]},{"osd":2,"uuid":"2d85d75f-1280-443f-857d-d322bd60c451","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6818","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6819","nonce":3751946332}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6820","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6821","nonce":3751946332}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6824","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6825","nonce":3751946332}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6822","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6823","nonce":3751946332}]},"public_addr":"172.21.15.63:6819/3751946332","cluster_addr":"172.21.15.63:6821/3751946332","heartbeat_back_addr":"172.21.15.63:6825/3751946332","heartbeat_front_addr":"172.21.15.63:6823/3751946332","state":["exists","up"]},{"osd":3,"uuid":"771f43d2-e98b-43e0-921b-88e56fadda42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6826","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6827","nonce":2654575936}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6828","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6829","nonce":2654575936}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6832","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6833","nonce":2654575936}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6830","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6831","nonce":2654575936}]},"public_addr":"172.21.15.63:6827/2654575936","cluster_addr":"172.21.15.63:6829/2654575936","heartbeat_back_addr":"172.21.15.63:6833/2654575936","heartbeat_front_addr":"172.21.15.63:6831/2654575936","state":["exists","up"]},{"osd":4,"uuid":"73c3c545-e975-4a6f-a1cb-e4575ee7a9b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6800","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6801","nonce":1563885208}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6803","nonce":1563885208}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6807","nonce":1563885208}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6805","nonce":1563885208}]},"public_addr":"172.21.15.162:6801/1563885208","cluster_addr":"172.21.15.162:6803/1563885208","heartbeat_back_addr":"172.21.15.162:6807/1563885208","heartbeat_front_addr":"172.21.15.162:6805/1563885208","state":["exists","up"]},{"osd":5,"uuid":"fae3ac52-3194-4eb4-9510-e3899768353a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6809","nonce":1826502179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6810","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6811","nonce":1826502179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6814","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6815","nonce":1826502179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6812","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6813","nonce":1826502179}]},"public_addr":"172.21.15.162:6809/1826502179","cluster_addr":"172.21.15.162:6811/1826502179","heartbeat_back_addr":"172.21.15.162:6815/1826502179","heartbeat_front_addr":"172.21.15.162:6813/1826502179","state":["exists","up"]},{"osd":6,"uuid":"a8255b37-1be5-4cd4-9b86-889f6bf52902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6816","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6817","nonce":1675904899}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6818","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6819","nonce":1675904899}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6822","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6823","nonce":1675904899}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6820","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6821","nonce":1675904899}]},"public_addr":"172.21.15.162:6817/1675904899","cluster_addr":"172.21.15.162:6819/1675904899","heartbeat_back_addr":"172.21.15.162:6823/1675904899","heartbeat_front_addr":"172.21.15.162:6821/1675904899","state":["exists","up"]},{"osd":7,"uuid":"3d0bcd6d-171d-4d90-a033-ed17714a9d96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6824","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6825","nonce":2938230961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6826","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6827","nonce":2938230961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6830","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6831","nonce":2938230961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6828","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6829","nonce":2938230961}]},"public_addr":"172.21.15.162:6825/2938230961","cluster_addr":"172.21.15.162:6827/2938230961","heartbeat_back_addr":"172.21.15.162:6831/2938230961","heartbeat_front_addr":"172.21.15.162:6829/2938230961","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:02.667962+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:26.166590+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:50.153794+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:13.556156+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:34.827805+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:58.789701+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:22.650398+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:46.166424+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.63:0/701508812":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/1256300214":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/2601197458":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/84364348":"2024-08-18T16:47:16.384724+0000","172.21.15.63:0/3004218255":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6800/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6801/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6801/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3540337558":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6800/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/1792393387":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3226753153":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/1986138219":"2024-08-18T16:47:39.919472+0000","172.21.15.63:6800/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.63:6801/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/955244129":"2024-08-18T16:52:20.398501+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T16:53:51.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:51 smithi162 ceph-mon[34633]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:51.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:51 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:53:51.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:51 smithi162 ceph-mon[34633]: mds.? [v2:172.21.15.63:6834/3222014296,v1:172.21.15.63:6835/3222014296] up:active 2024-08-17T16:53:51.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:51 smithi162 ceph-mon[34633]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 2024-08-17T16:53:51.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:51 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/4098758723' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:53:51.610 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:51 smithi063 ceph-mon[31063]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:51.610 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:51 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:53:51.610 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:51 smithi063 ceph-mon[31063]: mds.? [v2:172.21.15.63:6834/3222014296,v1:172.21.15.63:6835/3222014296] up:active 2024-08-17T16:53:51.610 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:51 smithi063 ceph-mon[31063]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 2024-08-17T16:53:51.610 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:51 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/4098758723' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:53:51.610 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:51 smithi063 ceph-mon[42680]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 222 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:53:51.611 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:51 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:53:51.611 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:51 smithi063 ceph-mon[42680]: mds.? [v2:172.21.15.63:6834/3222014296,v1:172.21.15.63:6835/3222014296] up:active 2024-08-17T16:53:51.611 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:51 smithi063 ceph-mon[42680]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 2024-08-17T16:53:51.611 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:51 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/4098758723' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:53:52.045 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T16:53:52.045 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T16:53:52.269 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:53.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:53.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:53.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:53.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:53.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:53.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:53.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[31063]: mds.? [v2:172.21.15.162:6832/2161520737,v1:172.21.15.162:6833/2161520737] up:boot 2024-08-17T16:53:53.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[31063]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T16:53:53.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi162.bjkxnv"}]: dispatch 2024-08-17T16:53:53.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[31063]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T16:53:53.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[31063]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:53:53.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:53.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:53.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:53.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:53.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:53.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:53.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[42680]: mds.? [v2:172.21.15.162:6832/2161520737,v1:172.21.15.162:6833/2161520737] up:boot 2024-08-17T16:53:53.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[42680]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T16:53:53.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi162.bjkxnv"}]: dispatch 2024-08-17T16:53:53.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[42680]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T16:53:53.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:53 smithi063 ceph-mon[42680]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:53:53.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:53 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:53.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:53 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:53.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:53 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:53.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:53 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:53.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:53 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:53.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:53 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:53:53.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:53 smithi162 ceph-mon[34633]: mds.? [v2:172.21.15.162:6832/2161520737,v1:172.21.15.162:6833/2161520737] up:boot 2024-08-17T16:53:53.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:53 smithi162 ceph-mon[34633]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T16:53:53.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:53 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi162.bjkxnv"}]: dispatch 2024-08-17T16:53:53.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:53 smithi162 ceph-mon[34633]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T16:53:53.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:53 smithi162 ceph-mon[34633]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:53:54.229 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:53:54.230 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":72,"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:46:54.156006+0000","modified":"2024-08-17T16:53:38.889498+0000","last_up_change":"2024-08-17T16:51:48.092300+0000","last_in_change":"2024-08-17T16:51:30.856523+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:49:54.431240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:52:22.131465+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T16:53:35.328125+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T16:53:36.292966+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6802","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6803","nonce":782866097}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6804","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6805","nonce":782866097}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6808","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6809","nonce":782866097}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6806","nonce":782866097},{"type":"v1","addr":"172.21.15.63:6807","nonce":782866097}]},"public_addr":"172.21.15.63:6803/782866097","cluster_addr":"172.21.15.63:6805/782866097","heartbeat_back_addr":"172.21.15.63:6809/782866097","heartbeat_front_addr":"172.21.15.63:6807/782866097","state":["exists","up"]},{"osd":1,"uuid":"ae346e99-4706-4871-990a-f8e737572245","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6810","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6811","nonce":944260396}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6812","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6813","nonce":944260396}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6816","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6817","nonce":944260396}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6814","nonce":944260396},{"type":"v1","addr":"172.21.15.63:6815","nonce":944260396}]},"public_addr":"172.21.15.63:6811/944260396","cluster_addr":"172.21.15.63:6813/944260396","heartbeat_back_addr":"172.21.15.63:6817/944260396","heartbeat_front_addr":"172.21.15.63:6815/944260396","state":["exists","up"]},{"osd":2,"uuid":"2d85d75f-1280-443f-857d-d322bd60c451","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6818","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6819","nonce":3751946332}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6820","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6821","nonce":3751946332}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6824","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6825","nonce":3751946332}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6822","nonce":3751946332},{"type":"v1","addr":"172.21.15.63:6823","nonce":3751946332}]},"public_addr":"172.21.15.63:6819/3751946332","cluster_addr":"172.21.15.63:6821/3751946332","heartbeat_back_addr":"172.21.15.63:6825/3751946332","heartbeat_front_addr":"172.21.15.63:6823/3751946332","state":["exists","up"]},{"osd":3,"uuid":"771f43d2-e98b-43e0-921b-88e56fadda42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6826","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6827","nonce":2654575936}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6828","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6829","nonce":2654575936}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6832","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6833","nonce":2654575936}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6830","nonce":2654575936},{"type":"v1","addr":"172.21.15.63:6831","nonce":2654575936}]},"public_addr":"172.21.15.63:6827/2654575936","cluster_addr":"172.21.15.63:6829/2654575936","heartbeat_back_addr":"172.21.15.63:6833/2654575936","heartbeat_front_addr":"172.21.15.63:6831/2654575936","state":["exists","up"]},{"osd":4,"uuid":"73c3c545-e975-4a6f-a1cb-e4575ee7a9b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6800","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6801","nonce":1563885208}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6803","nonce":1563885208}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6807","nonce":1563885208}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6805","nonce":1563885208}]},"public_addr":"172.21.15.162:6801/1563885208","cluster_addr":"172.21.15.162:6803/1563885208","heartbeat_back_addr":"172.21.15.162:6807/1563885208","heartbeat_front_addr":"172.21.15.162:6805/1563885208","state":["exists","up"]},{"osd":5,"uuid":"fae3ac52-3194-4eb4-9510-e3899768353a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6809","nonce":1826502179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6810","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6811","nonce":1826502179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6814","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6815","nonce":1826502179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6812","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6813","nonce":1826502179}]},"public_addr":"172.21.15.162:6809/1826502179","cluster_addr":"172.21.15.162:6811/1826502179","heartbeat_back_addr":"172.21.15.162:6815/1826502179","heartbeat_front_addr":"172.21.15.162:6813/1826502179","state":["exists","up"]},{"osd":6,"uuid":"a8255b37-1be5-4cd4-9b86-889f6bf52902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6816","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6817","nonce":1675904899}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6818","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6819","nonce":1675904899}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6822","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6823","nonce":1675904899}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6820","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6821","nonce":1675904899}]},"public_addr":"172.21.15.162:6817/1675904899","cluster_addr":"172.21.15.162:6819/1675904899","heartbeat_back_addr":"172.21.15.162:6823/1675904899","heartbeat_front_addr":"172.21.15.162:6821/1675904899","state":["exists","up"]},{"osd":7,"uuid":"3d0bcd6d-171d-4d90-a033-ed17714a9d96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6824","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6825","nonce":2938230961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6826","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6827","nonce":2938230961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6830","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6831","nonce":2938230961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6828","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6829","nonce":2938230961}]},"public_addr":"172.21.15.162:6825/2938230961","cluster_addr":"172.21.15.162:6827/2938230961","heartbeat_back_addr":"172.21.15.162:6831/2938230961","heartbeat_front_addr":"172.21.15.162:6829/2938230961","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:02.667962+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:26.166590+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:50.153794+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:13.556156+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:34.827805+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:58.789701+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:22.650398+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:46.166424+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.63:0/701508812":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/1256300214":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/2601197458":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/84364348":"2024-08-18T16:47:16.384724+0000","172.21.15.63:0/3004218255":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6800/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6801/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6801/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3540337558":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6800/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/1792393387":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3226753153":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/1986138219":"2024-08-18T16:47:39.919472+0000","172.21.15.63:6800/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.63:6801/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/955244129":"2024-08-18T16:52:20.398501+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T16:53:54.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:54 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/2281451955' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:53:54.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:54 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/2281451955' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:53:54.733 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:54 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/2281451955' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-08-17T16:53:55.177 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T16:53:55.178 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T16:53:55.178 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T16:53:55.179 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T16:53:55.179 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T16:53:55.180 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T16:53:55.180 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T16:53:55.180 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T16:53:55.437 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:55 smithi063 ceph-mon[42680]: mds.? [v2:172.21.15.63:6834/3222014296,v1:172.21.15.63:6835/3222014296] up:active 2024-08-17T16:53:55.438 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:55 smithi063 ceph-mon[42680]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T16:53:55.438 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:55 smithi063 ceph-mon[42680]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:53:55.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:55 smithi162 ceph-mon[34633]: mds.? [v2:172.21.15.63:6834/3222014296,v1:172.21.15.63:6835/3222014296] up:active 2024-08-17T16:53:55.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:55 smithi162 ceph-mon[34633]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T16:53:55.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:55 smithi162 ceph-mon[34633]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:53:55.731 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:55.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:55 smithi063 ceph-mon[31063]: mds.? [v2:172.21.15.63:6834/3222014296,v1:172.21.15.63:6835/3222014296] up:active 2024-08-17T16:53:55.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:55 smithi063 ceph-mon[31063]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T16:53:55.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:55 smithi063 ceph-mon[31063]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:53:55.782 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:55.803 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:55.803 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:55.809 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:55.816 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:55.819 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:53:56.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:56 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:56.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:56 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:56.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:56 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:56 smithi063 ceph-mon[42680]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:53:56.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:56 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:56.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:56 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:56.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:56 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:56.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:56 smithi063 ceph-mon[31063]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:53:56.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:56 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:56.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:56 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:56.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:56 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:53:56.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:56 smithi162 ceph-mon[34633]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:53:58.001 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:57 smithi063 ceph-mon[31063]: mds.? [v2:172.21.15.162:6832/2161520737,v1:172.21.15.162:6833/2161520737] up:standby 2024-08-17T16:53:58.001 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:57 smithi063 ceph-mon[31063]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T16:53:58.002 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:57 smithi063 ceph-mon[42680]: mds.? [v2:172.21.15.162:6832/2161520737,v1:172.21.15.162:6833/2161520737] up:standby 2024-08-17T16:53:58.002 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:57 smithi063 ceph-mon[42680]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T16:53:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:57 smithi162 ceph-mon[34633]: mds.? [v2:172.21.15.162:6832/2161520737,v1:172.21.15.162:6833/2161520737] up:standby 2024-08-17T16:53:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:57 smithi162 ceph-mon[34633]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T16:53:58.843 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:53:58 smithi063 ceph-mon[42680]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:53:58.843 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:53:58 smithi063 ceph-mon[31063]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:53:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:53:58 smithi162 ceph-mon[34633]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:53:59.770 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:53:59 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:53:59] "GET /metrics HTTP/1.1" 200 33417 "" "Prometheus/2.43.0" 2024-08-17T16:54:00.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:00 smithi063 ceph-mon[42680]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:54:00.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:00 smithi063 ceph-mon[31063]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:54:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:00 smithi162 ceph-mon[34633]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:54:02.063 INFO:teuthology.orchestra.run.smithi063.stdout:236223201309 2024-08-17T16:54:02.063 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T16:54:02.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:02 smithi063 ceph-mon[42680]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:54:02.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:02 smithi063 ceph-mon[31063]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:54:02.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:02 smithi162 ceph-mon[34633]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-08-17T16:54:04.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:04 smithi063 ceph-mon[31063]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T16:54:04.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:04 smithi063 ceph-mon[42680]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T16:54:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:04 smithi162 ceph-mon[34633]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T16:54:05.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:05 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:54:05.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:05 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:54:05.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:05 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:54:06.733 INFO:teuthology.orchestra.run.smithi063.stdout:55834574906 2024-08-17T16:54:06.734 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T16:54:06.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:06 smithi063 ceph-mon[42680]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T16:54:06.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:06 smithi063 ceph-mon[31063]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T16:54:06.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:06 smithi162 ceph-mon[34633]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T16:54:08.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:08 smithi063 ceph-mon[42680]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:08.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:08 smithi063 ceph-mon[31063]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:08.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:08 smithi162 ceph-mon[34633]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:09.450 INFO:teuthology.orchestra.run.smithi063.stdout:133143986221 2024-08-17T16:54:09.451 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T16:54:09.464 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:09.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:54:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:54:09] "GET /metrics HTTP/1.1" 200 33703 "" "Prometheus/2.43.0" 2024-08-17T16:54:09.847 INFO:teuthology.orchestra.run.smithi063.stdout:34359738432 2024-08-17T16:54:09.847 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T16:54:10.556 INFO:teuthology.orchestra.run.smithi063.stdout:167503724583 2024-08-17T16:54:10.556 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T16:54:10.678 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:10 smithi063 ceph-mon[42680]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:10.679 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:10 smithi063 ceph-mon[31063]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:10.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:10 smithi162 ceph-mon[34633]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:10.974 INFO:teuthology.orchestra.run.smithi063.stdout:111669149746 2024-08-17T16:54:10.974 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T16:54:12.092 INFO:teuthology.orchestra.run.smithi063.stdout:77309411383 2024-08-17T16:54:12.093 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T16:54:12.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:12 smithi063 ceph-mon[31063]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:12 smithi063 ceph-mon[42680]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:12.828 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:12.831 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:12.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:12 smithi162 ceph-mon[34633]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:14.256 INFO:teuthology.orchestra.run.smithi063.stdout:236223201312 2024-08-17T16:54:14.446 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:14 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/1419465188' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T16:54:14.446 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:14 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/1419465188' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T16:54:14.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:14 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/1419465188' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-08-17T16:54:15.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:15 smithi063 ceph-mon[42680]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:15.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:15 smithi063 ceph-mon[31063]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:15.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:15 smithi162 ceph-mon[34633]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:15.948 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201309 got 236223201312 for osd.7 2024-08-17T16:54:15.949 DEBUG:teuthology.parallel:result is None 2024-08-17T16:54:16.586 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:16.767 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:16.768 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:16.768 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:16.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:16 smithi063 ceph-mon[31063]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:16.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:16 smithi063 ceph-mon[42680]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:16 smithi162 ceph-mon[34633]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:17.556 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:18.912 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:18 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:18.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:18 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:18.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:18 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:18.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:18 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:18.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:18 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:18.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:18 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:18.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:18 smithi063 ceph-mon[31063]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:18.913 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:18 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:18.913 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:18 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:18.914 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:18 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:18.914 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:18 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:18.914 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:18 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:18.914 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:18 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:18.914 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:18 smithi063 ceph-mon[42680]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:18 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:18 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:18 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:18 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:18 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:18 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:18 smithi162 ceph-mon[34633]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:19.793 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:54:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:54:19] "GET /metrics HTTP/1.1" 200 33703 "" "Prometheus/2.43.0" 2024-08-17T16:54:20.747 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:20 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-08-17T16:54:20.747 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:20 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-08-17T16:54:20.747 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:20 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-08-17T16:54:20.748 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:20 smithi162 ceph-mon[34633]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:20.748 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:20 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:54:20.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:20 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-08-17T16:54:20.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:20 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-08-17T16:54:20.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:20 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-08-17T16:54:20.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:20 smithi063 ceph-mon[42680]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:20.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:20 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:54:20.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:20 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2024-08-17T16:54:20.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:20 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-08-17T16:54:20.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:20 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-08-17T16:54:20.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:20 smithi063 ceph-mon[31063]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:20.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:20 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:54:21.510 INFO:teuthology.orchestra.run.smithi063.stdout:133143986223 2024-08-17T16:54:21.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-08-17T16:54:21.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-08-17T16:54:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-08-17T16:54:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:21 smithi063 ceph-mon[31063]: osdmap e73: 8 total, 8 up, 8 in 2024-08-17T16:54:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:21 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:21.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-08-17T16:54:21.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-08-17T16:54:21.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-08-17T16:54:21.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:21 smithi063 ceph-mon[42680]: osdmap e73: 8 total, 8 up, 8 in 2024-08-17T16:54:21.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:21.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:21 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-08-17T16:54:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-08-17T16:54:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-08-17T16:54:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:21 smithi162 ceph-mon[34633]: osdmap e73: 8 total, 8 up, 8 in 2024-08-17T16:54:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:21 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:22.083 INFO:teuthology.orchestra.run.smithi063.stdout:201863462950 2024-08-17T16:54:22.084 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T16:54:22.761 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:22 smithi162 ceph-mon[34633]: osdmap e74: 8 total, 8 up, 8 in 2024-08-17T16:54:22.761 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:22 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/4111189813' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T16:54:22.761 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:22 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/639674680' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T16:54:22.761 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:22 smithi162 ceph-mon[34633]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:22.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:22 smithi063 ceph-mon[42680]: osdmap e74: 8 total, 8 up, 8 in 2024-08-17T16:54:22.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:22 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/4111189813' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T16:54:22.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:22 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/639674680' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T16:54:22.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:22 smithi063 ceph-mon[42680]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:22.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:22 smithi063 ceph-mon[31063]: osdmap e74: 8 total, 8 up, 8 in 2024-08-17T16:54:22.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:22 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/4111189813' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-08-17T16:54:22.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:22 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/639674680' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-08-17T16:54:22.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:22 smithi063 ceph-mon[31063]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:23.139 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986221 got 133143986223 for osd.4 2024-08-17T16:54:23.139 DEBUG:teuthology.parallel:result is None 2024-08-17T16:54:23.223 INFO:teuthology.orchestra.run.smithi063.stdout:111669149748 2024-08-17T16:54:23.788 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149746 got 111669149748 for osd.3 2024-08-17T16:54:23.788 DEBUG:teuthology.parallel:result is None 2024-08-17T16:54:24.536 INFO:teuthology.orchestra.run.smithi063.stdout:55834574910 2024-08-17T16:54:24.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:24 smithi063 ceph-mon[42680]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:24.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:24 smithi063 ceph-mon[31063]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:24.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:24 smithi162 ceph-mon[34633]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:25.739 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:25 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3204280608' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T16:54:25.739 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:25 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3204280608' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T16:54:25.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:25 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3204280608' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-08-17T16:54:25.927 INFO:teuthology.orchestra.run.smithi063.stdout:77309411386 2024-08-17T16:54:26.540 INFO:teuthology.orchestra.run.smithi063.stdout:167503724588 2024-08-17T16:54:26.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:26 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/2547998389' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T16:54:26.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:26 smithi063 ceph-mon[42680]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:26.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:26 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/2547998389' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T16:54:26.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:26 smithi063 ceph-mon[31063]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:26.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:26 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/2547998389' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-08-17T16:54:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:26 smithi162 ceph-mon[34633]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:26.884 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411383 got 77309411386 for osd.2 2024-08-17T16:54:26.884 DEBUG:teuthology.parallel:result is None 2024-08-17T16:54:27.577 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724583 got 167503724588 for osd.5 2024-08-17T16:54:27.578 DEBUG:teuthology.parallel:result is None 2024-08-17T16:54:27.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:27 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/2487824286' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T16:54:27.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:27 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/2487824286' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T16:54:27.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:27 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/2487824286' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-08-17T16:54:28.408 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574906 got 55834574910 for osd.1 2024-08-17T16:54:28.408 DEBUG:teuthology.parallel:result is None 2024-08-17T16:54:28.641 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:28 smithi063 ceph-mon[42680]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:28.641 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:28 smithi063 ceph-mon[31063]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:28.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:28 smithi162 ceph-mon[34633]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:28.906 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:29.059 INFO:teuthology.orchestra.run.smithi063.stdout:34359738436 2024-08-17T16:54:29.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:29 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3736651232' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T16:54:29.881 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:29 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3736651232' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T16:54:29.881 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:54:29 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:54:29] "GET /metrics HTTP/1.1" 200 33702 "" "Prometheus/2.43.0" 2024-08-17T16:54:29.882 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:29 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3736651232' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-08-17T16:54:29.910 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738432 got 34359738436 for osd.0 2024-08-17T16:54:29.910 DEBUG:teuthology.parallel:result is None 2024-08-17T16:54:30.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:30 smithi162 ceph-mon[34633]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:30 smithi063 ceph-mon[42680]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:31.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:30 smithi063 ceph-mon[31063]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:32.138 INFO:teuthology.orchestra.run.smithi063.stdout:201863462952 2024-08-17T16:54:32.446 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:32 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/1347869022' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T16:54:32.446 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:32 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/1347869022' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T16:54:32.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:32 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/1347869022' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-08-17T16:54:32.742 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462950 got 201863462952 for osd.6 2024-08-17T16:54:32.743 DEBUG:teuthology.parallel:result is None 2024-08-17T16:54:32.743 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T16:54:32.743 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T16:54:33.450 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:33 smithi162 ceph-mon[34633]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:33.512 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:33.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:33 smithi063 ceph-mon[31063]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:33.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:33 smithi063 ceph-mon[42680]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:34.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:34 smithi063 ceph-mon[31063]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:34.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:34 smithi063 ceph-mon[42680]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:34.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:34 smithi162 ceph-mon[34633]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:35.820 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:35.820 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:35.820 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:35.820 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:35.820 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:35.820 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:54:35.820 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:35.820 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:35.820 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:35.820 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:35.821 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:35.821 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:54:35.972 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:54:35.973 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T16:54:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:54:36.593 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":84,"stamp":"2024-08-17T16:54:34.435355+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":232164,"kb_used_data":7796,"kb_used_omap":12,"kb_used_meta":224243,"kb_avail":749729052,"statfs":{"total":767960285184,"available":767722549248,"internally_reserved":0,"allocated":7983104,"data_stored":4467032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12742,"internal_metadata":229625402},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002698"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486359+0000","last_change":"2024-08-17T16:53:37.119060+0000","last_active":"2024-08-17T16:54:20.486359+0000","last_peered":"2024-08-17T16:54:20.486359+0000","last_clean":"2024-08-17T16:54:20.486359+0000","last_became_active":"2024-08-17T16:53:37.118648+0000","last_became_peered":"2024-08-17T16:53:37.118648+0000","last_unstale":"2024-08-17T16:54:20.486359+0000","last_undegraded":"2024-08-17T16:54:20.486359+0000","last_fullsized":"2024-08-17T16:54:20.486359+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:44.437835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896821+0000","last_change":"2024-08-17T16:53:36.879852+0000","last_active":"2024-08-17T16:54:20.896821+0000","last_peered":"2024-08-17T16:54:20.896821+0000","last_clean":"2024-08-17T16:54:20.896821+0000","last_became_active":"2024-08-17T16:53:36.879528+0000","last_became_peered":"2024-08-17T16:53:36.879528+0000","last_unstale":"2024-08-17T16:54:20.896821+0000","last_undegraded":"2024-08-17T16:54:20.896821+0000","last_fullsized":"2024-08-17T16:54:20.896821+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:02.468160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483478+0000","last_change":"2024-08-17T16:53:37.120604+0000","last_active":"2024-08-17T16:54:20.483478+0000","last_peered":"2024-08-17T16:54:20.483478+0000","last_clean":"2024-08-17T16:54:20.483478+0000","last_became_active":"2024-08-17T16:53:37.120308+0000","last_became_peered":"2024-08-17T16:53:37.120308+0000","last_unstale":"2024-08-17T16:54:20.483478+0000","last_undegraded":"2024-08-17T16:54:20.483478+0000","last_fullsized":"2024-08-17T16:54:20.483478+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:06.306915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.480973+0000","last_change":"2024-08-17T16:53:36.890705+0000","last_active":"2024-08-17T16:54:20.480973+0000","last_peered":"2024-08-17T16:54:20.480973+0000","last_clean":"2024-08-17T16:54:20.480973+0000","last_became_active":"2024-08-17T16:53:36.889646+0000","last_became_peered":"2024-08-17T16:53:36.889646+0000","last_unstale":"2024-08-17T16:54:20.480973+0000","last_undegraded":"2024-08-17T16:54:20.480973+0000","last_fullsized":"2024-08-17T16:54:20.480973+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:24.768849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.481180+0000","last_change":"2024-08-17T16:53:38.252543+0000","last_active":"2024-08-17T16:54:20.481180+0000","last_peered":"2024-08-17T16:54:20.481180+0000","last_clean":"2024-08-17T16:54:20.481180+0000","last_became_active":"2024-08-17T16:53:38.252268+0000","last_became_peered":"2024-08-17T16:53:38.252268+0000","last_unstale":"2024-08-17T16:54:20.481180+0000","last_undegraded":"2024-08-17T16:54:20.481180+0000","last_fullsized":"2024-08-17T16:54:20.481180+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:03:05.590587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896573+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T16:54:20.896573+0000","last_peered":"2024-08-17T16:54:20.896573+0000","last_clean":"2024-08-17T16:54:20.896573+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T16:54:20.896573+0000","last_undegraded":"2024-08-17T16:54:20.896573+0000","last_fullsized":"2024-08-17T16:54:20.896573+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486564+0000","last_change":"2024-08-17T16:53:37.121032+0000","last_active":"2024-08-17T16:54:20.486564+0000","last_peered":"2024-08-17T16:54:20.486564+0000","last_clean":"2024-08-17T16:54:20.486564+0000","last_became_active":"2024-08-17T16:53:37.120512+0000","last_became_peered":"2024-08-17T16:53:37.120512+0000","last_unstale":"2024-08-17T16:54:20.486564+0000","last_undegraded":"2024-08-17T16:54:20.486564+0000","last_fullsized":"2024-08-17T16:54:20.486564+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:58:17.507284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896511+0000","last_change":"2024-08-17T16:53:36.880307+0000","last_active":"2024-08-17T16:54:20.896511+0000","last_peered":"2024-08-17T16:54:20.896511+0000","last_clean":"2024-08-17T16:54:20.896511+0000","last_became_active":"2024-08-17T16:53:36.880009+0000","last_became_peered":"2024-08-17T16:53:36.880009+0000","last_unstale":"2024-08-17T16:54:20.896511+0000","last_undegraded":"2024-08-17T16:54:20.896511+0000","last_fullsized":"2024-08-17T16:54:20.896511+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:10.601837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.897149+0000","last_change":"2024-08-17T16:53:36.883203+0000","last_active":"2024-08-17T16:54:20.897149+0000","last_peered":"2024-08-17T16:54:20.897149+0000","last_clean":"2024-08-17T16:54:20.897149+0000","last_became_active":"2024-08-17T16:53:36.882908+0000","last_became_peered":"2024-08-17T16:53:36.882908+0000","last_unstale":"2024-08-17T16:54:20.897149+0000","last_undegraded":"2024-08-17T16:54:20.897149+0000","last_fullsized":"2024-08-17T16:54:20.897149+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:20.466791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483299+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T16:54:20.483299+0000","last_peered":"2024-08-17T16:54:20.483299+0000","last_clean":"2024-08-17T16:54:20.483299+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T16:54:20.483299+0000","last_undegraded":"2024-08-17T16:54:20.483299+0000","last_fullsized":"2024-08-17T16:54:20.483299+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896834+0000","last_change":"2024-08-17T16:53:36.881778+0000","last_active":"2024-08-17T16:54:20.896834+0000","last_peered":"2024-08-17T16:54:20.896834+0000","last_clean":"2024-08-17T16:54:20.896834+0000","last_became_active":"2024-08-17T16:53:36.881498+0000","last_became_peered":"2024-08-17T16:53:36.881498+0000","last_unstale":"2024-08-17T16:54:20.896834+0000","last_undegraded":"2024-08-17T16:54:20.896834+0000","last_fullsized":"2024-08-17T16:54:20.896834+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:25:06.671817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476836+0000","last_change":"2024-08-17T16:53:36.899945+0000","last_active":"2024-08-17T16:54:20.476836+0000","last_peered":"2024-08-17T16:54:20.476836+0000","last_clean":"2024-08-17T16:54:20.476836+0000","last_became_active":"2024-08-17T16:53:36.899631+0000","last_became_peered":"2024-08-17T16:53:36.899631+0000","last_unstale":"2024-08-17T16:54:20.476836+0000","last_undegraded":"2024-08-17T16:54:20.476836+0000","last_fullsized":"2024-08-17T16:54:20.476836+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:05.769480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896842+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T16:54:20.896842+0000","last_peered":"2024-08-17T16:54:20.896842+0000","last_clean":"2024-08-17T16:54:20.896842+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T16:54:20.896842+0000","last_undegraded":"2024-08-17T16:54:20.896842+0000","last_fullsized":"2024-08-17T16:54:20.896842+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896694+0000","last_change":"2024-08-17T16:53:37.118710+0000","last_active":"2024-08-17T16:54:20.896694+0000","last_peered":"2024-08-17T16:54:20.896694+0000","last_clean":"2024-08-17T16:54:20.896694+0000","last_became_active":"2024-08-17T16:53:37.118379+0000","last_became_peered":"2024-08-17T16:53:37.118379+0000","last_unstale":"2024-08-17T16:54:20.896694+0000","last_undegraded":"2024-08-17T16:54:20.896694+0000","last_fullsized":"2024-08-17T16:54:20.896694+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:27:23.613708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483708+0000","last_change":"2024-08-17T16:53:36.876268+0000","last_active":"2024-08-17T16:54:20.483708+0000","last_peered":"2024-08-17T16:54:20.483708+0000","last_clean":"2024-08-17T16:54:20.483708+0000","last_became_active":"2024-08-17T16:53:36.875953+0000","last_became_peered":"2024-08-17T16:53:36.875953+0000","last_unstale":"2024-08-17T16:54:20.483708+0000","last_undegraded":"2024-08-17T16:54:20.483708+0000","last_fullsized":"2024-08-17T16:54:20.483708+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:30.229916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":23,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.511744+0000","last_change":"2024-08-17T16:54:21.511744+0000","last_active":"2024-08-17T16:54:21.511744+0000","last_peered":"2024-08-17T16:54:21.511744+0000","last_clean":"2024-08-17T16:54:21.511744+0000","last_became_active":"2024-08-17T16:54:21.511445+0000","last_became_peered":"2024-08-17T16:54:21.511445+0000","last_unstale":"2024-08-17T16:54:21.511744+0000","last_undegraded":"2024-08-17T16:54:21.511744+0000","last_fullsized":"2024-08-17T16:54:21.511744+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:27:20.540018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486453+0000","last_change":"2024-08-17T16:53:36.886636+0000","last_active":"2024-08-17T16:54:20.486453+0000","last_peered":"2024-08-17T16:54:20.486453+0000","last_clean":"2024-08-17T16:54:20.486453+0000","last_became_active":"2024-08-17T16:53:36.886350+0000","last_became_peered":"2024-08-17T16:53:36.886350+0000","last_unstale":"2024-08-17T16:54:20.486453+0000","last_undegraded":"2024-08-17T16:54:20.486453+0000","last_fullsized":"2024-08-17T16:54:20.486453+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:35.454091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476973+0000","last_change":"2024-08-17T16:53:36.900748+0000","last_active":"2024-08-17T16:54:20.476973+0000","last_peered":"2024-08-17T16:54:20.476973+0000","last_clean":"2024-08-17T16:54:20.476973+0000","last_became_active":"2024-08-17T16:53:36.900332+0000","last_became_peered":"2024-08-17T16:53:36.900332+0000","last_unstale":"2024-08-17T16:54:20.476973+0000","last_undegraded":"2024-08-17T16:54:20.476973+0000","last_fullsized":"2024-08-17T16:54:20.476973+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:04:24.749007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.897080+0000","last_change":"2024-08-17T16:53:36.883760+0000","last_active":"2024-08-17T16:54:20.897080+0000","last_peered":"2024-08-17T16:54:20.897080+0000","last_clean":"2024-08-17T16:54:20.897080+0000","last_became_active":"2024-08-17T16:53:36.883480+0000","last_became_peered":"2024-08-17T16:53:36.883480+0000","last_unstale":"2024-08-17T16:54:20.897080+0000","last_undegraded":"2024-08-17T16:54:20.897080+0000","last_fullsized":"2024-08-17T16:54:20.897080+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:23.398009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.477067+0000","last_change":"2024-08-17T16:53:36.901528+0000","last_active":"2024-08-17T16:54:20.477067+0000","last_peered":"2024-08-17T16:54:20.477067+0000","last_clean":"2024-08-17T16:54:20.477067+0000","last_became_active":"2024-08-17T16:53:36.901280+0000","last_became_peered":"2024-08-17T16:53:36.901280+0000","last_unstale":"2024-08-17T16:54:20.477067+0000","last_undegraded":"2024-08-17T16:54:20.477067+0000","last_fullsized":"2024-08-17T16:54:20.477067+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:13.363564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476807+0000","last_change":"2024-08-17T16:53:36.881045+0000","last_active":"2024-08-17T16:54:20.476807+0000","last_peered":"2024-08-17T16:54:20.476807+0000","last_clean":"2024-08-17T16:54:20.476807+0000","last_became_active":"2024-08-17T16:53:36.880658+0000","last_became_peered":"2024-08-17T16:53:36.880658+0000","last_unstale":"2024-08-17T16:54:20.476807+0000","last_undegraded":"2024-08-17T16:54:20.476807+0000","last_fullsized":"2024-08-17T16:54:20.476807+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:49.527766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.477029+0000","last_change":"2024-08-17T16:53:36.901686+0000","last_active":"2024-08-17T16:54:20.477029+0000","last_peered":"2024-08-17T16:54:20.477029+0000","last_clean":"2024-08-17T16:54:20.477029+0000","last_became_active":"2024-08-17T16:53:36.901190+0000","last_became_peered":"2024-08-17T16:53:36.901190+0000","last_unstale":"2024-08-17T16:54:20.477029+0000","last_undegraded":"2024-08-17T16:54:20.477029+0000","last_fullsized":"2024-08-17T16:54:20.477029+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:04:50.685231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486012+0000","last_change":"2024-08-17T16:53:36.888957+0000","last_active":"2024-08-17T16:54:20.486012+0000","last_peered":"2024-08-17T16:54:20.486012+0000","last_clean":"2024-08-17T16:54:20.486012+0000","last_became_active":"2024-08-17T16:53:36.888648+0000","last_became_peered":"2024-08-17T16:53:36.888648+0000","last_unstale":"2024-08-17T16:54:20.486012+0000","last_undegraded":"2024-08-17T16:54:20.486012+0000","last_fullsized":"2024-08-17T16:54:20.486012+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:55:30.381593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.613239+0000","last_change":"2024-08-17T16:54:21.613239+0000","last_active":"2024-08-17T16:54:21.613239+0000","last_peered":"2024-08-17T16:54:21.613239+0000","last_clean":"2024-08-17T16:54:21.613239+0000","last_became_active":"2024-08-17T16:54:21.612882+0000","last_became_peered":"2024-08-17T16:54:21.612882+0000","last_unstale":"2024-08-17T16:54:21.613239+0000","last_undegraded":"2024-08-17T16:54:21.613239+0000","last_fullsized":"2024-08-17T16:54:21.613239+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:18.592726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":271,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896513+0000","last_change":"2024-08-17T16:51:52.132475+0000","last_active":"2024-08-17T16:54:20.896513+0000","last_peered":"2024-08-17T16:54:20.896513+0000","last_clean":"2024-08-17T16:54:20.896513+0000","last_became_active":"2024-08-17T16:51:52.132088+0000","last_became_peered":"2024-08-17T16:51:52.132088+0000","last_unstale":"2024-08-17T16:54:20.896513+0000","last_undegraded":"2024-08-17T16:54:20.896513+0000","last_fullsized":"2024-08-17T16:54:20.896513+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:13.679985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":29,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.753832+0000","last_change":"2024-08-17T16:52:28.100670+0000","last_active":"2024-08-17T16:54:21.753832+0000","last_peered":"2024-08-17T16:54:21.753832+0000","last_clean":"2024-08-17T16:54:21.753832+0000","last_became_active":"2024-08-17T16:52:23.430320+0000","last_became_peered":"2024-08-17T16:52:23.430320+0000","last_unstale":"2024-08-17T16:54:21.753832+0000","last_undegraded":"2024-08-17T16:54:21.753832+0000","last_fullsized":"2024-08-17T16:54:21.753832+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:47.861974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486367+0000","last_change":"2024-08-17T16:53:36.884227+0000","last_active":"2024-08-17T16:54:20.486367+0000","last_peered":"2024-08-17T16:54:20.486367+0000","last_clean":"2024-08-17T16:54:20.486367+0000","last_became_active":"2024-08-17T16:53:36.883539+0000","last_became_peered":"2024-08-17T16:53:36.883539+0000","last_unstale":"2024-08-17T16:54:20.486367+0000","last_undegraded":"2024-08-17T16:54:20.486367+0000","last_fullsized":"2024-08-17T16:54:20.486367+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:55:22.330403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896963+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T16:54:20.896963+0000","last_peered":"2024-08-17T16:54:20.896963+0000","last_clean":"2024-08-17T16:54:20.896963+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T16:54:20.896963+0000","last_undegraded":"2024-08-17T16:54:20.896963+0000","last_fullsized":"2024-08-17T16:54:20.896963+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.565607+0000","last_change":"2024-08-17T16:52:28.243231+0000","last_active":"2024-08-17T16:54:21.565607+0000","last_peered":"2024-08-17T16:54:21.565607+0000","last_clean":"2024-08-17T16:54:21.565607+0000","last_became_active":"2024-08-17T16:52:23.739287+0000","last_became_peered":"2024-08-17T16:52:23.739287+0000","last_unstale":"2024-08-17T16:54:21.565607+0000","last_undegraded":"2024-08-17T16:54:21.565607+0000","last_fullsized":"2024-08-17T16:54:21.565607+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:39.119833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.481903+0000","last_change":"2024-08-17T16:53:36.889086+0000","last_active":"2024-08-17T16:54:20.481903+0000","last_peered":"2024-08-17T16:54:20.481903+0000","last_clean":"2024-08-17T16:54:20.481903+0000","last_became_active":"2024-08-17T16:53:36.888137+0000","last_became_peered":"2024-08-17T16:53:36.888137+0000","last_unstale":"2024-08-17T16:54:20.481903+0000","last_undegraded":"2024-08-17T16:54:20.481903+0000","last_fullsized":"2024-08-17T16:54:20.481903+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:11.906601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":23,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.755048+0000","last_change":"2024-08-17T16:54:21.755048+0000","last_active":"2024-08-17T16:54:21.755048+0000","last_peered":"2024-08-17T16:54:21.755048+0000","last_clean":"2024-08-17T16:54:21.755048+0000","last_became_active":"2024-08-17T16:54:21.754695+0000","last_became_peered":"2024-08-17T16:54:21.754695+0000","last_unstale":"2024-08-17T16:54:21.755048+0000","last_undegraded":"2024-08-17T16:54:21.755048+0000","last_fullsized":"2024-08-17T16:54:21.755048+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:23:46.650736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.492865+0000","last_change":"2024-08-17T16:52:28.097703+0000","last_active":"2024-08-17T16:54:21.492865+0000","last_peered":"2024-08-17T16:54:21.492865+0000","last_clean":"2024-08-17T16:54:21.492865+0000","last_became_active":"2024-08-17T16:52:23.739224+0000","last_became_peered":"2024-08-17T16:52:23.739224+0000","last_unstale":"2024-08-17T16:54:21.492865+0000","last_undegraded":"2024-08-17T16:54:21.492865+0000","last_fullsized":"2024-08-17T16:54:21.492865+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:33.382141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483252+0000","last_change":"2024-08-17T16:53:36.889705+0000","last_active":"2024-08-17T16:54:20.483252+0000","last_peered":"2024-08-17T16:54:20.483252+0000","last_clean":"2024-08-17T16:54:20.483252+0000","last_became_active":"2024-08-17T16:53:36.889439+0000","last_became_peered":"2024-08-17T16:53:36.889439+0000","last_unstale":"2024-08-17T16:54:20.483252+0000","last_undegraded":"2024-08-17T16:54:20.483252+0000","last_fullsized":"2024-08-17T16:54:20.483252+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:15:54.587710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483708+0000","last_change":"2024-08-17T16:53:37.996396+0000","last_active":"2024-08-17T16:54:20.483708+0000","last_peered":"2024-08-17T16:54:20.483708+0000","last_clean":"2024-08-17T16:54:20.483708+0000","last_became_active":"2024-08-17T16:53:37.995865+0000","last_became_peered":"2024-08-17T16:53:37.995865+0000","last_unstale":"2024-08-17T16:54:20.483708+0000","last_undegraded":"2024-08-17T16:54:20.483708+0000","last_fullsized":"2024-08-17T16:54:20.483708+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:35.253721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":30,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.753778+0000","last_change":"2024-08-17T16:52:28.101249+0000","last_active":"2024-08-17T16:54:21.753778+0000","last_peered":"2024-08-17T16:54:21.753778+0000","last_clean":"2024-08-17T16:54:21.753778+0000","last_became_active":"2024-08-17T16:52:23.430593+0000","last_became_peered":"2024-08-17T16:52:23.430593+0000","last_unstale":"2024-08-17T16:54:21.753778+0000","last_undegraded":"2024-08-17T16:54:21.753778+0000","last_fullsized":"2024-08-17T16:54:21.753778+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:19.292172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.477094+0000","last_change":"2024-08-17T16:53:36.888745+0000","last_active":"2024-08-17T16:54:20.477094+0000","last_peered":"2024-08-17T16:54:20.477094+0000","last_clean":"2024-08-17T16:54:20.477094+0000","last_became_active":"2024-08-17T16:53:36.888395+0000","last_became_peered":"2024-08-17T16:53:36.888395+0000","last_unstale":"2024-08-17T16:54:20.477094+0000","last_undegraded":"2024-08-17T16:54:20.477094+0000","last_fullsized":"2024-08-17T16:54:20.477094+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:30:49.594838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483106+0000","last_change":"2024-08-17T16:53:38.251602+0000","last_active":"2024-08-17T16:54:20.483106+0000","last_peered":"2024-08-17T16:54:20.483106+0000","last_clean":"2024-08-17T16:54:20.483106+0000","last_became_active":"2024-08-17T16:53:38.251342+0000","last_became_peered":"2024-08-17T16:53:38.251342+0000","last_unstale":"2024-08-17T16:54:20.483106+0000","last_undegraded":"2024-08-17T16:54:20.483106+0000","last_fullsized":"2024-08-17T16:54:20.483106+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:02:16.318917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483648+0000","last_change":"2024-08-17T16:53:36.880034+0000","last_active":"2024-08-17T16:54:20.483648+0000","last_peered":"2024-08-17T16:54:20.483648+0000","last_clean":"2024-08-17T16:54:20.483648+0000","last_became_active":"2024-08-17T16:53:36.879690+0000","last_became_peered":"2024-08-17T16:53:36.879690+0000","last_unstale":"2024-08-17T16:54:20.483648+0000","last_undegraded":"2024-08-17T16:54:20.483648+0000","last_fullsized":"2024-08-17T16:54:20.483648+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:36:46.047979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.492186+0000","last_change":"2024-08-17T16:52:28.097582+0000","last_active":"2024-08-17T16:54:21.492186+0000","last_peered":"2024-08-17T16:54:21.492186+0000","last_clean":"2024-08-17T16:54:21.492186+0000","last_became_active":"2024-08-17T16:52:23.739237+0000","last_became_peered":"2024-08-17T16:52:23.739237+0000","last_unstale":"2024-08-17T16:54:21.492186+0000","last_undegraded":"2024-08-17T16:54:21.492186+0000","last_fullsized":"2024-08-17T16:54:21.492186+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:02:16.574623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.482644+0000","last_change":"2024-08-17T16:53:38.087553+0000","last_active":"2024-08-17T16:54:20.482644+0000","last_peered":"2024-08-17T16:54:20.482644+0000","last_clean":"2024-08-17T16:54:20.482644+0000","last_became_active":"2024-08-17T16:53:38.087152+0000","last_became_peered":"2024-08-17T16:53:38.087152+0000","last_unstale":"2024-08-17T16:54:20.482644+0000","last_undegraded":"2024-08-17T16:54:20.482644+0000","last_fullsized":"2024-08-17T16:54:20.482644+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:33.385166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483289+0000","last_change":"2024-08-17T16:53:36.887881+0000","last_active":"2024-08-17T16:54:20.483289+0000","last_peered":"2024-08-17T16:54:20.483289+0000","last_clean":"2024-08-17T16:54:20.483289+0000","last_became_active":"2024-08-17T16:53:36.887575+0000","last_became_peered":"2024-08-17T16:53:36.887575+0000","last_unstale":"2024-08-17T16:54:20.483289+0000","last_undegraded":"2024-08-17T16:54:20.483289+0000","last_fullsized":"2024-08-17T16:54:20.483289+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:11:03.473197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.565881+0000","last_change":"2024-08-17T16:52:28.243251+0000","last_active":"2024-08-17T16:54:21.565881+0000","last_peered":"2024-08-17T16:54:21.565881+0000","last_clean":"2024-08-17T16:54:21.565881+0000","last_became_active":"2024-08-17T16:52:23.816512+0000","last_became_peered":"2024-08-17T16:52:23.816512+0000","last_unstale":"2024-08-17T16:54:21.565881+0000","last_undegraded":"2024-08-17T16:54:21.565881+0000","last_fullsized":"2024-08-17T16:54:21.565881+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:59.683846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.482952+0000","last_change":"2024-08-17T16:53:37.996145+0000","last_active":"2024-08-17T16:54:20.482952+0000","last_peered":"2024-08-17T16:54:20.482952+0000","last_clean":"2024-08-17T16:54:20.482952+0000","last_became_active":"2024-08-17T16:53:37.995802+0000","last_became_peered":"2024-08-17T16:53:37.995802+0000","last_unstale":"2024-08-17T16:54:20.482952+0000","last_undegraded":"2024-08-17T16:54:20.482952+0000","last_fullsized":"2024-08-17T16:54:20.482952+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:58.123211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486619+0000","last_change":"2024-08-17T16:53:36.885630+0000","last_active":"2024-08-17T16:54:20.486619+0000","last_peered":"2024-08-17T16:54:20.486619+0000","last_clean":"2024-08-17T16:54:20.486619+0000","last_became_active":"2024-08-17T16:53:36.885340+0000","last_became_peered":"2024-08-17T16:53:36.885340+0000","last_unstale":"2024-08-17T16:54:20.486619+0000","last_undegraded":"2024-08-17T16:54:20.486619+0000","last_fullsized":"2024-08-17T16:54:20.486619+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:01.924572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":27,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483494+0000","last_change":"2024-08-17T16:52:28.097477+0000","last_active":"2024-08-17T16:54:20.483494+0000","last_peered":"2024-08-17T16:54:20.483494+0000","last_clean":"2024-08-17T16:54:20.483494+0000","last_became_active":"2024-08-17T16:52:23.816540+0000","last_became_peered":"2024-08-17T16:52:23.816540+0000","last_unstale":"2024-08-17T16:54:20.483494+0000","last_undegraded":"2024-08-17T16:54:20.483494+0000","last_fullsized":"2024-08-17T16:54:20.483494+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:27:49.345586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000334624,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.485895+0000","last_change":"2024-08-17T16:53:37.150235+0000","last_active":"2024-08-17T16:54:20.485895+0000","last_peered":"2024-08-17T16:54:20.485895+0000","last_clean":"2024-08-17T16:54:20.485895+0000","last_became_active":"2024-08-17T16:53:37.149897+0000","last_became_peered":"2024-08-17T16:53:37.149897+0000","last_unstale":"2024-08-17T16:54:20.485895+0000","last_undegraded":"2024-08-17T16:54:20.485895+0000","last_fullsized":"2024-08-17T16:54:20.485895+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:51.578814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.481293+0000","last_change":"2024-08-17T16:53:36.885579+0000","last_active":"2024-08-17T16:54:20.481293+0000","last_peered":"2024-08-17T16:54:20.481293+0000","last_clean":"2024-08-17T16:54:20.481293+0000","last_became_active":"2024-08-17T16:53:36.885271+0000","last_became_peered":"2024-08-17T16:53:36.885271+0000","last_unstale":"2024-08-17T16:54:20.481293+0000","last_undegraded":"2024-08-17T16:54:20.481293+0000","last_fullsized":"2024-08-17T16:54:20.481293+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:27:50.616487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":26,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896608+0000","last_change":"2024-08-17T16:52:28.102166+0000","last_active":"2024-08-17T16:54:20.896608+0000","last_peered":"2024-08-17T16:54:20.896608+0000","last_clean":"2024-08-17T16:54:20.896608+0000","last_became_active":"2024-08-17T16:52:23.430739+0000","last_became_peered":"2024-08-17T16:52:23.430739+0000","last_unstale":"2024-08-17T16:54:20.896608+0000","last_undegraded":"2024-08-17T16:54:20.896608+0000","last_fullsized":"2024-08-17T16:54:20.896608+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:19.139627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476637+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T16:54:20.476637+0000","last_peered":"2024-08-17T16:54:20.476637+0000","last_clean":"2024-08-17T16:54:20.476637+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T16:54:20.476637+0000","last_undegraded":"2024-08-17T16:54:20.476637+0000","last_fullsized":"2024-08-17T16:54:20.476637+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476956+0000","last_change":"2024-08-17T16:53:36.901638+0000","last_active":"2024-08-17T16:54:20.476956+0000","last_peered":"2024-08-17T16:54:20.476956+0000","last_clean":"2024-08-17T16:54:20.476956+0000","last_became_active":"2024-08-17T16:53:36.901269+0000","last_became_peered":"2024-08-17T16:53:36.901269+0000","last_unstale":"2024-08-17T16:54:20.476956+0000","last_undegraded":"2024-08-17T16:54:20.476956+0000","last_fullsized":"2024-08-17T16:54:20.476956+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:31.151603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"72'6","reported_seq":21,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.477113+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T16:54:20.477113+0000","last_peered":"2024-08-17T16:54:20.477113+0000","last_clean":"2024-08-17T16:54:20.477113+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T16:54:20.477113+0000","last_undegraded":"2024-08-17T16:54:20.477113+0000","last_fullsized":"2024-08-17T16:54:20.477113+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483466+0000","last_change":"2024-08-17T16:53:38.251303+0000","last_active":"2024-08-17T16:54:20.483466+0000","last_peered":"2024-08-17T16:54:20.483466+0000","last_clean":"2024-08-17T16:54:20.483466+0000","last_became_active":"2024-08-17T16:53:38.251048+0000","last_became_peered":"2024-08-17T16:53:38.251048+0000","last_unstale":"2024-08-17T16:54:20.483466+0000","last_undegraded":"2024-08-17T16:54:20.483466+0000","last_fullsized":"2024-08-17T16:54:20.483466+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:58.408282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":18,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896713+0000","last_change":"2024-08-17T16:53:36.877296+0000","last_active":"2024-08-17T16:54:20.896713+0000","last_peered":"2024-08-17T16:54:20.896713+0000","last_clean":"2024-08-17T16:54:20.896713+0000","last_became_active":"2024-08-17T16:53:36.877018+0000","last_became_peered":"2024-08-17T16:53:36.877018+0000","last_unstale":"2024-08-17T16:54:20.896713+0000","last_undegraded":"2024-08-17T16:54:20.896713+0000","last_fullsized":"2024-08-17T16:54:20.896713+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:19.839578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483501+0000","last_change":"2024-08-17T16:53:36.883382+0000","last_active":"2024-08-17T16:54:20.483501+0000","last_peered":"2024-08-17T16:54:20.483501+0000","last_clean":"2024-08-17T16:54:20.483501+0000","last_became_active":"2024-08-17T16:53:36.883108+0000","last_became_peered":"2024-08-17T16:53:36.883108+0000","last_unstale":"2024-08-17T16:54:20.483501+0000","last_undegraded":"2024-08-17T16:54:20.483501+0000","last_fullsized":"2024-08-17T16:54:20.483501+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:46:55.973076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486550+0000","last_change":"2024-08-17T16:53:37.119299+0000","last_active":"2024-08-17T16:54:20.486550+0000","last_peered":"2024-08-17T16:54:20.486550+0000","last_clean":"2024-08-17T16:54:20.486550+0000","last_became_active":"2024-08-17T16:53:37.118657+0000","last_became_peered":"2024-08-17T16:53:37.118657+0000","last_unstale":"2024-08-17T16:54:20.486550+0000","last_undegraded":"2024-08-17T16:54:20.486550+0000","last_fullsized":"2024-08-17T16:54:20.486550+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:14:32.942498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.482233+0000","last_change":"2024-08-17T16:53:36.889202+0000","last_active":"2024-08-17T16:54:20.482233+0000","last_peered":"2024-08-17T16:54:20.482233+0000","last_clean":"2024-08-17T16:54:20.482233+0000","last_became_active":"2024-08-17T16:53:36.888179+0000","last_became_peered":"2024-08-17T16:53:36.888179+0000","last_unstale":"2024-08-17T16:54:20.482233+0000","last_undegraded":"2024-08-17T16:54:20.482233+0000","last_fullsized":"2024-08-17T16:54:20.482233+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:49:45.828822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.485835+0000","last_change":"2024-08-17T16:53:36.902422+0000","last_active":"2024-08-17T16:54:20.485835+0000","last_peered":"2024-08-17T16:54:20.485835+0000","last_clean":"2024-08-17T16:54:20.485835+0000","last_became_active":"2024-08-17T16:53:36.902107+0000","last_became_peered":"2024-08-17T16:53:36.902107+0000","last_unstale":"2024-08-17T16:54:20.485835+0000","last_undegraded":"2024-08-17T16:54:20.485835+0000","last_fullsized":"2024-08-17T16:54:20.485835+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:06.822564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896539+0000","last_change":"2024-08-17T16:53:36.901487+0000","last_active":"2024-08-17T16:54:20.896539+0000","last_peered":"2024-08-17T16:54:20.896539+0000","last_clean":"2024-08-17T16:54:20.896539+0000","last_became_active":"2024-08-17T16:53:36.901191+0000","last_became_peered":"2024-08-17T16:53:36.901191+0000","last_unstale":"2024-08-17T16:54:20.896539+0000","last_undegraded":"2024-08-17T16:54:20.896539+0000","last_fullsized":"2024-08-17T16:54:20.896539+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:26.851092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.485950+0000","last_change":"2024-08-17T16:53:36.889630+0000","last_active":"2024-08-17T16:54:20.485950+0000","last_peered":"2024-08-17T16:54:20.485950+0000","last_clean":"2024-08-17T16:54:20.485950+0000","last_became_active":"2024-08-17T16:53:36.889336+0000","last_became_peered":"2024-08-17T16:53:36.889336+0000","last_unstale":"2024-08-17T16:54:20.485950+0000","last_undegraded":"2024-08-17T16:54:20.485950+0000","last_fullsized":"2024-08-17T16:54:20.485950+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:10.013575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.480972+0000","last_change":"2024-08-17T16:53:38.251672+0000","last_active":"2024-08-17T16:54:20.480972+0000","last_peered":"2024-08-17T16:54:20.480972+0000","last_clean":"2024-08-17T16:54:20.480972+0000","last_became_active":"2024-08-17T16:53:38.251398+0000","last_became_peered":"2024-08-17T16:53:38.251398+0000","last_unstale":"2024-08-17T16:54:20.480972+0000","last_undegraded":"2024-08-17T16:54:20.480972+0000","last_fullsized":"2024-08-17T16:54:20.480972+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:37:05.335449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.482469+0000","last_change":"2024-08-17T16:53:36.886233+0000","last_active":"2024-08-17T16:54:20.482469+0000","last_peered":"2024-08-17T16:54:20.482469+0000","last_clean":"2024-08-17T16:54:20.482469+0000","last_became_active":"2024-08-17T16:53:36.885869+0000","last_became_peered":"2024-08-17T16:53:36.885869+0000","last_unstale":"2024-08-17T16:54:20.482469+0000","last_undegraded":"2024-08-17T16:54:20.482469+0000","last_fullsized":"2024-08-17T16:54:20.482469+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:44.305315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486329+0000","last_change":"2024-08-17T16:53:36.904687+0000","last_active":"2024-08-17T16:54:20.486329+0000","last_peered":"2024-08-17T16:54:20.486329+0000","last_clean":"2024-08-17T16:54:20.486329+0000","last_became_active":"2024-08-17T16:53:36.904393+0000","last_became_peered":"2024-08-17T16:53:36.904393+0000","last_unstale":"2024-08-17T16:54:20.486329+0000","last_undegraded":"2024-08-17T16:54:20.486329+0000","last_fullsized":"2024-08-17T16:54:20.486329+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:24:25.019529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476851+0000","last_change":"2024-08-17T16:53:36.882578+0000","last_active":"2024-08-17T16:54:20.476851+0000","last_peered":"2024-08-17T16:54:20.476851+0000","last_clean":"2024-08-17T16:54:20.476851+0000","last_became_active":"2024-08-17T16:53:36.882065+0000","last_became_peered":"2024-08-17T16:53:36.882065+0000","last_unstale":"2024-08-17T16:54:20.476851+0000","last_undegraded":"2024-08-17T16:54:20.476851+0000","last_fullsized":"2024-08-17T16:54:20.476851+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:35.409187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.481537+0000","last_change":"2024-08-17T16:53:36.902424+0000","last_active":"2024-08-17T16:54:20.481537+0000","last_peered":"2024-08-17T16:54:20.481537+0000","last_clean":"2024-08-17T16:54:20.481537+0000","last_became_active":"2024-08-17T16:53:36.902142+0000","last_became_peered":"2024-08-17T16:53:36.902142+0000","last_unstale":"2024-08-17T16:54:20.481537+0000","last_undegraded":"2024-08-17T16:54:20.481537+0000","last_fullsized":"2024-08-17T16:54:20.481537+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:07.703935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483073+0000","last_change":"2024-08-17T16:53:36.886281+0000","last_active":"2024-08-17T16:54:20.483073+0000","last_peered":"2024-08-17T16:54:20.483073+0000","last_clean":"2024-08-17T16:54:20.483073+0000","last_became_active":"2024-08-17T16:53:36.885982+0000","last_became_peered":"2024-08-17T16:53:36.885982+0000","last_unstale":"2024-08-17T16:54:20.483073+0000","last_undegraded":"2024-08-17T16:54:20.483073+0000","last_fullsized":"2024-08-17T16:54:20.483073+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:57:11.167476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476881+0000","last_change":"2024-08-17T16:53:36.902213+0000","last_active":"2024-08-17T16:54:20.476881+0000","last_peered":"2024-08-17T16:54:20.476881+0000","last_clean":"2024-08-17T16:54:20.476881+0000","last_became_active":"2024-08-17T16:53:36.901923+0000","last_became_peered":"2024-08-17T16:53:36.901923+0000","last_unstale":"2024-08-17T16:54:20.476881+0000","last_undegraded":"2024-08-17T16:54:20.476881+0000","last_fullsized":"2024-08-17T16:54:20.476881+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:20:28.471491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483456+0000","last_change":"2024-08-17T16:53:36.882002+0000","last_active":"2024-08-17T16:54:20.483456+0000","last_peered":"2024-08-17T16:54:20.483456+0000","last_clean":"2024-08-17T16:54:20.483456+0000","last_became_active":"2024-08-17T16:53:36.881716+0000","last_became_peered":"2024-08-17T16:53:36.881716+0000","last_unstale":"2024-08-17T16:54:20.483456+0000","last_undegraded":"2024-08-17T16:54:20.483456+0000","last_fullsized":"2024-08-17T16:54:20.483456+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:50:51.350825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476753+0000","last_change":"2024-08-17T16:53:37.150151+0000","last_active":"2024-08-17T16:54:20.476753+0000","last_peered":"2024-08-17T16:54:20.476753+0000","last_clean":"2024-08-17T16:54:20.476753+0000","last_became_active":"2024-08-17T16:53:37.149940+0000","last_became_peered":"2024-08-17T16:53:37.149940+0000","last_unstale":"2024-08-17T16:54:20.476753+0000","last_undegraded":"2024-08-17T16:54:20.476753+0000","last_fullsized":"2024-08-17T16:54:20.476753+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:44.895666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483679+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T16:54:20.483679+0000","last_peered":"2024-08-17T16:54:20.483679+0000","last_clean":"2024-08-17T16:54:20.483679+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T16:54:20.483679+0000","last_undegraded":"2024-08-17T16:54:20.483679+0000","last_fullsized":"2024-08-17T16:54:20.483679+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486359+0000","last_change":"2024-08-17T16:53:37.150694+0000","last_active":"2024-08-17T16:54:20.486359+0000","last_peered":"2024-08-17T16:54:20.486359+0000","last_clean":"2024-08-17T16:54:20.486359+0000","last_became_active":"2024-08-17T16:53:37.150303+0000","last_became_peered":"2024-08-17T16:53:37.150303+0000","last_unstale":"2024-08-17T16:54:20.486359+0000","last_undegraded":"2024-08-17T16:54:20.486359+0000","last_fullsized":"2024-08-17T16:54:20.486359+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:51.129623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486242+0000","last_change":"2024-08-17T16:53:36.884929+0000","last_active":"2024-08-17T16:54:20.486242+0000","last_peered":"2024-08-17T16:54:20.486242+0000","last_clean":"2024-08-17T16:54:20.486242+0000","last_became_active":"2024-08-17T16:53:36.883501+0000","last_became_peered":"2024-08-17T16:53:36.883501+0000","last_unstale":"2024-08-17T16:54:20.486242+0000","last_undegraded":"2024-08-17T16:54:20.486242+0000","last_fullsized":"2024-08-17T16:54:20.486242+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:38:08.920185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476635+0000","last_change":"2024-08-17T16:53:36.899921+0000","last_active":"2024-08-17T16:54:20.476635+0000","last_peered":"2024-08-17T16:54:20.476635+0000","last_clean":"2024-08-17T16:54:20.476635+0000","last_became_active":"2024-08-17T16:53:36.899661+0000","last_became_peered":"2024-08-17T16:53:36.899661+0000","last_unstale":"2024-08-17T16:54:20.476635+0000","last_undegraded":"2024-08-17T16:54:20.476635+0000","last_fullsized":"2024-08-17T16:54:20.476635+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:08:13.805113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":19,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.482141+0000","last_change":"2024-08-17T16:53:36.886775+0000","last_active":"2024-08-17T16:54:20.482141+0000","last_peered":"2024-08-17T16:54:20.482141+0000","last_clean":"2024-08-17T16:54:20.482141+0000","last_became_active":"2024-08-17T16:53:36.886364+0000","last_became_peered":"2024-08-17T16:53:36.886364+0000","last_unstale":"2024-08-17T16:54:20.482141+0000","last_undegraded":"2024-08-17T16:54:20.482141+0000","last_fullsized":"2024-08-17T16:54:20.482141+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:21.249333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201316,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28368,"kb_used_data":1348,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716784,"statfs":{"total":95995035648,"available":95965986816,"internally_reserved":0,"allocated":1380352,"data_stored":927971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.51300000000000001,"15min":0.51700000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.60799999999999998,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.53400000000000003,"15min":0.53600000000000003},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.66900000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.38300000000000001}]},{"osd":1,"last update":"Sat Aug 17 16:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.66400000000000003,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.63700000000000001,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.432}]},{"osd":2,"last update":"Sat Aug 17 16:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.5,"15min":0.497},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.65900000000000003,"5min":1.012,"15min":1.012},"last":0.51200000000000001}]},{"osd":3,"last update":"Sat Aug 17 16:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.52400000000000002,"15min":0.52200000000000002},"min":{"1min":0.38300000000000001,"5min":0.373,"15min":0.373},"max":{"1min":0.65800000000000003,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.499},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.53800000000000003,"15min":0.53500000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.65700000000000003,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.59599999999999997}]},{"osd":4,"last update":"Sat Aug 17 16:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44500000000000001,"15min":0.44600000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45600000000000002,"15min":0.45700000000000002},"min":{"1min":0.33300000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.58499999999999996}]},{"osd":5,"last update":"Sat Aug 17 16:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.46400000000000002,"15min":0.46200000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.48899999999999999,"15min":0.48499999999999999},"min":{"1min":0.35499999999999998,"5min":0.314,"15min":0.314},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.48399999999999999}]},{"osd":6,"last update":"Sat Aug 17 16:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.496,"15min":0.48999999999999999},"min":{"1min":0.35999999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.47999999999999998,"15min":0.47399999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.53600000000000003}]}]},{"osd":6,"up_from":47,"seq":201863462953,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28360,"kb_used_data":1336,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716792,"statfs":{"total":95995035648,"available":95965995008,"internally_reserved":0,"allocated":1368064,"data_stored":927571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":27654601},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:53:28 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.49199999999999999,"15min":0.48899999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.51600000000000001,"15min":0.51500000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.64100000000000001,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.66400000000000003}]},{"osd":1,"last update":"Sat Aug 17 16:53:28 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.65400000000000003,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.48299999999999998,"15min":0.47999999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.60199999999999998,"5min":0.61299999999999999,"15min":0.61299999999999999},"last":0.43099999999999999}]},{"osd":2,"last update":"Sat Aug 17 16:53:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.53400000000000003,"15min":0.53600000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.74299999999999999,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.505,"5min":0.48399999999999999,"15min":0.47999999999999998},"min":{"1min":0.34300000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.70299999999999996,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.52000000000000002}]},{"osd":3,"last update":"Sat Aug 17 16:53:28 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.5,"15min":0.496},"min":{"1min":0.40000000000000002,"5min":0.315,"15min":0.315},"max":{"1min":0.627,"5min":0.65300000000000002,"15min":0.65300000000000002},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.41799999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.65100000000000002,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.53800000000000003}]},{"osd":4,"last update":"Sat Aug 17 16:53:28 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.39600000000000002,"15min":0.38800000000000001},"min":{"1min":0.27200000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.626,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.41599999999999998,"15min":0.40799999999999997},"min":{"1min":0.315,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.505}]},{"osd":5,"last update":"Sat Aug 17 16:53:28 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.40799999999999997,"15min":0.39700000000000002},"min":{"1min":0.35899999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.64500000000000002,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.44700000000000001,"15min":0.441},"min":{"1min":0.34399999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.58699999999999997}]},{"osd":7,"last update":"Sat Aug 17 16:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.47899999999999998,"15min":0.48199999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.66100000000000003,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.53200000000000003,"15min":0.53000000000000003},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.745,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.61499999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724589,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27780,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717372,"statfs":{"total":95995035648,"available":95966588928,"internally_reserved":0,"allocated":774144,"data_stored":336767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.52500000000000002,"15min":0.52100000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65900000000000003,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.52300000000000002,"15min":0.53300000000000003},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.68300000000000005,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.60299999999999998}]},{"osd":1,"last update":"Sat Aug 17 16:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.53500000000000003,"15min":0.54700000000000004},"min":{"1min":0.29199999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.66700000000000004,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.56999999999999995}]},{"osd":2,"last update":"Sat Aug 17 16:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.53700000000000003,"15min":0.53800000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.64100000000000001,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.53900000000000003,"15min":0.54000000000000004},"min":{"1min":0.39700000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.75800000000000001,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.48299999999999998}]},{"osd":3,"last update":"Sat Aug 17 16:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.54100000000000004,"15min":0.54500000000000004},"min":{"1min":0.38,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.76900000000000002,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.55800000000000005,"15min":0.56100000000000005},"min":{"1min":0.378,"5min":0.372,"15min":0.372},"max":{"1min":0.73399999999999999,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.65200000000000002}]},{"osd":4,"last update":"Sat Aug 17 16:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.40400000000000003,"15min":0.38900000000000001},"min":{"1min":0.28499999999999998,"5min":0.251,"15min":0.251},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.433,"15min":0.41899999999999998},"min":{"1min":0.25900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.67500000000000004,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.45100000000000001}]},{"osd":6,"last update":"Sat Aug 17 16:53:31 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.442,"15min":0.439},"min":{"1min":0.36699999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.58299999999999996,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.47399999999999998,"15min":0.46899999999999997},"min":{"1min":0.313,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.70699999999999996,"5min":1.02,"15min":1.02},"last":0.53300000000000003}]},{"osd":7,"last update":"Sat Aug 17 16:53:52 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.47199999999999998,"15min":0.46600000000000003},"min":{"1min":0.36399999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.5},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.46700000000000003,"15min":0.46000000000000002},"min":{"1min":0.31900000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.67500000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986226,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31884,"kb_used_data":768,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713268,"statfs":{"total":95995035648,"available":95962386432,"internally_reserved":0,"allocated":786432,"data_stored":337864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:53:41 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54600000000000004,"15min":0.54800000000000004},"min":{"1min":0.36499999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.67700000000000005,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54200000000000004,"15min":0.53700000000000003},"min":{"1min":0.40300000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":1.1559999999999999}]},{"osd":1,"last update":"Sat Aug 17 16:53:41 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.53900000000000003,"15min":0.54600000000000004},"min":{"1min":0.35399999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.53800000000000003,"15min":0.52800000000000002},"min":{"1min":0.34799999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.69899999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.64500000000000002}]},{"osd":2,"last update":"Sat Aug 17 16:53:41 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.56100000000000005,"15min":0.55800000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":1.0760000000000001},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.53900000000000003,"15min":0.53800000000000003},"min":{"1min":0.35799999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.73699999999999999,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.60399999999999998}]},{"osd":3,"last update":"Sat Aug 17 16:53:41 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.434,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.68200000000000005,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.54900000000000004,"15min":0.55600000000000005},"min":{"1min":0.32100000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.61799999999999999}]},{"osd":5,"last update":"Sat Aug 17 16:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.28599999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.59799999999999998,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.42099999999999999,"15min":0.41799999999999998},"min":{"1min":0.30499999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.45800000000000002}]},{"osd":6,"last update":"Sat Aug 17 16:53:29 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.44500000000000001,"15min":0.44400000000000001},"min":{"1min":0.307,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.307,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.61199999999999999,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.51300000000000001}]},{"osd":7,"last update":"Sat Aug 17 16:53:49 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.49199999999999999,"15min":0.49099999999999999},"min":{"1min":0.378,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.65100000000000002,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.64700000000000002,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.53600000000000003}]}]},{"osd":3,"up_from":26,"seq":111669149751,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27772,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717380,"statfs":{"total":95995035648,"available":95966597120,"internally_reserved":0,"allocated":770048,"data_stored":336517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.36199999999999999,"15min":0.37},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.51300000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.35699999999999998,"5min":0.39900000000000002,"15min":0.41199999999999998},"min":{"1min":0.252,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.47699999999999998,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.41899999999999998}]},{"osd":1,"last update":"Sat Aug 17 16:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.40600000000000003,"15min":0.42199999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.54100000000000004,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.41299999999999998,"15min":0.439},"min":{"1min":0.245,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.503,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.44600000000000001}]},{"osd":2,"last update":"Sat Aug 17 16:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.42599999999999999,"15min":0.436},"min":{"1min":0.26000000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.55500000000000005,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.41599999999999998,"15min":0.42699999999999999},"min":{"1min":0.27200000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.55400000000000005,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.33400000000000002}]},{"osd":4,"last update":"Sat Aug 17 16:53:42 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.57199999999999995,"15min":0.57999999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.73799999999999999,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.58099999999999996}]},{"osd":5,"last update":"Sat Aug 17 16:54:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.58399999999999996,"15min":0.58799999999999997},"min":{"1min":0.38600000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.57699999999999996,"15min":0.57299999999999995},"min":{"1min":0.45700000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.61499999999999999}]},{"osd":6,"last update":"Sat Aug 17 16:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.59699999999999998,"15min":0.60099999999999998},"min":{"1min":0.41299999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.77600000000000002,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.57499999999999996,"15min":0.56999999999999995},"min":{"1min":0.42599999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.89400000000000002,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.63400000000000001}]},{"osd":7,"last update":"Sat Aug 17 16:53:51 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.57599999999999996,"15min":0.56799999999999995},"min":{"1min":0.41899999999999998,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.57299999999999995,"15min":0.56499999999999995},"min":{"1min":0.45000000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.54700000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411388,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31872,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713280,"statfs":{"total":95995035648,"available":95962398720,"internally_reserved":0,"allocated":774144,"data_stored":337043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.33600000000000002,"5min":0.38400000000000001,"15min":0.38900000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.54100000000000004,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.39800000000000002,"15min":0.39000000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.52300000000000002,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.28199999999999997}]},{"osd":1,"last update":"Sat Aug 17 16:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.41299999999999998,"15min":0.40600000000000003},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.57999999999999996,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.20300000000000001},{"interface":"front","average":{"1min":0.379,"5min":0.41999999999999998,"15min":0.42099999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.62,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.251}]},{"osd":3,"last update":"Sat Aug 17 16:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.45000000000000001,"15min":0.47599999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.57599999999999996,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.45300000000000001,"15min":0.47699999999999998},"min":{"1min":0.30599999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.55500000000000005,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.316}]},{"osd":4,"last update":"Sat Aug 17 16:53:45 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.624,"15min":0.65200000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.90600000000000003,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.63800000000000001,"15min":0.66400000000000003},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.92600000000000005,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.499}]},{"osd":5,"last update":"Sat Aug 17 16:54:09 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.58299999999999996,"15min":0.58499999999999996},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.92900000000000005,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.629,"15min":0.64900000000000002},"min":{"1min":0.46500000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.91400000000000003,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.55600000000000005}]},{"osd":6,"last update":"Sat Aug 17 16:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.60299999999999998,"15min":0.623},"min":{"1min":0.38700000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.81299999999999994,"5min":0.999,"15min":0.999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.59399999999999997,"15min":0.60999999999999999},"min":{"1min":0.41499999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.75,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.44800000000000001}]},{"osd":7,"last update":"Sat Aug 17 16:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.57699999999999996,"15min":0.58499999999999996},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.82499999999999996,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.60099999999999998,"15min":0.60499999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.57899999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574912,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27772,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717380,"statfs":{"total":95995035648,"available":95966597120,"internally_reserved":0,"allocated":765952,"data_stored":336173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:53:36 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.34599999999999997,"15min":0.34000000000000002},"min":{"1min":0.223,"5min":0.193,"15min":0.193},"max":{"1min":0.52300000000000002,"5min":0.53300000000000003,"15min":0.53300000000000003},"last":0.374},{"interface":"front","average":{"1min":0.33400000000000002,"5min":0.36399999999999999,"15min":0.35099999999999998},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":0.53100000000000003,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.39300000000000002}]},{"osd":2,"last update":"Sat Aug 17 16:54:04 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.375,"15min":0.36499999999999999},"min":{"1min":0.26900000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.58899999999999997,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.372,"5min":0.371,"15min":0.35899999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.53100000000000003,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.34499999999999997}]},{"osd":3,"last update":"Sat Aug 17 16:54:20 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.40200000000000002,"15min":0.40600000000000003},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.57799999999999996,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.45700000000000002,"15min":0.46700000000000003},"min":{"1min":0.30499999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.753,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.41599999999999998}]},{"osd":4,"last update":"Sat Aug 17 16:53:44 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.56799999999999995,"15min":0.56100000000000005},"min":{"1min":0.36799999999999999,"5min":0.315,"15min":0.315},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.54100000000000004,"15min":0.52900000000000003},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.624}]},{"osd":5,"last update":"Sat Aug 17 16:54:04 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.59499999999999997,"15min":0.59199999999999997},"min":{"1min":0.38500000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.84199999999999997,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.501},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.42199999999999999,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.747,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.59699999999999998}]},{"osd":6,"last update":"Sat Aug 17 16:53:32 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.60899999999999999,"15min":0.60799999999999998},"min":{"1min":0.44800000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.60599999999999998,"15min":0.60099999999999998},"min":{"1min":0.41199999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.58199999999999996}]},{"osd":7,"last update":"Sat Aug 17 16:53:51 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.628,"15min":0.627},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.85599999999999998,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.433,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.86699999999999999,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.53700000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738437,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28356,"kb_used_data":1332,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716796,"statfs":{"total":95995035648,"available":95965999104,"internally_reserved":0,"allocated":1363968,"data_stored":927126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 16:53:34 2024","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.34300000000000003,"15min":0.32700000000000001},"min":{"1min":0.224,"5min":0.193,"15min":0.193},"max":{"1min":0.46100000000000002,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.34499999999999997,"5min":0.36099999999999999,"15min":0.33500000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.53900000000000003,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.30199999999999999}]},{"osd":2,"last update":"Sat Aug 17 16:54:01 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.39100000000000001,"15min":0.39200000000000002},"min":{"1min":0.27800000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.50800000000000001,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39900000000000002,"15min":0.38900000000000001},"min":{"1min":0.249,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.59699999999999998,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.374}]},{"osd":3,"last update":"Sat Aug 17 16:54:27 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.46500000000000002,"15min":0.46300000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.55600000000000005,"5min":1.054,"15min":1.054},"last":0.438},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.42899999999999999,"15min":0.436},"min":{"1min":0.28000000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.51800000000000002,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.39200000000000002}]},{"osd":4,"last update":"Sat Aug 17 16:53:46 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.58799999999999997,"15min":0.59399999999999997},"min":{"1min":0.38700000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.79400000000000004,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60899999999999999,"15min":0.61199999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.81799999999999995,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.52200000000000002}]},{"osd":5,"last update":"Sat Aug 17 16:54:06 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.61399999999999999,"15min":0.61899999999999999},"min":{"1min":0.40400000000000003,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.83599999999999997,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.495},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.59399999999999997,"15min":0.59299999999999997},"min":{"1min":0.41499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.79700000000000004,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat Aug 17 16:53:31 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.56299999999999994,"15min":0.56499999999999995},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.58299999999999996,"15min":0.58699999999999997},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.83899999999999997,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.46200000000000002}]},{"osd":7,"last update":"Sat Aug 17 16:53:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.56299999999999994,"15min":0.55600000000000005},"min":{"1min":0.42599999999999999,"5min":0.313,"15min":0.313},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.57599999999999996,"15min":0.56899999999999995},"min":{"1min":0.48699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77000000000000002,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.58699999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T16:54:36.598 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T16:54:36.811 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:36.825 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:36 smithi063 ceph-mon[31063]: from='client.24668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:54:37.084 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:36 smithi063 ceph-mon[42680]: from='client.24668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:54:37.084 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:36 smithi063 ceph-mon[42680]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:37.085 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:36 smithi063 ceph-mon[31063]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:36 smithi162 ceph-mon[34633]: from='client.24668 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:54:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:36 smithi162 ceph-mon[34633]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:38.616 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:54:38.617 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T16:54:38.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:38 smithi063 ceph-mon[42680]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:38.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:38 smithi063 ceph-mon[31063]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:38.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:38 smithi162 ceph-mon[34633]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:39.343 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":86,"stamp":"2024-08-17T16:54:38.436204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":232164,"kb_used_data":7796,"kb_used_omap":12,"kb_used_meta":224243,"kb_avail":749729052,"statfs":{"total":767960285184,"available":767722549248,"internally_reserved":0,"allocated":7983104,"data_stored":4467032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12742,"internal_metadata":229625402},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002788"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486359+0000","last_change":"2024-08-17T16:53:37.119060+0000","last_active":"2024-08-17T16:54:20.486359+0000","last_peered":"2024-08-17T16:54:20.486359+0000","last_clean":"2024-08-17T16:54:20.486359+0000","last_became_active":"2024-08-17T16:53:37.118648+0000","last_became_peered":"2024-08-17T16:53:37.118648+0000","last_unstale":"2024-08-17T16:54:20.486359+0000","last_undegraded":"2024-08-17T16:54:20.486359+0000","last_fullsized":"2024-08-17T16:54:20.486359+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:13:44.437835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896821+0000","last_change":"2024-08-17T16:53:36.879852+0000","last_active":"2024-08-17T16:54:20.896821+0000","last_peered":"2024-08-17T16:54:20.896821+0000","last_clean":"2024-08-17T16:54:20.896821+0000","last_became_active":"2024-08-17T16:53:36.879528+0000","last_became_peered":"2024-08-17T16:53:36.879528+0000","last_unstale":"2024-08-17T16:54:20.896821+0000","last_undegraded":"2024-08-17T16:54:20.896821+0000","last_fullsized":"2024-08-17T16:54:20.896821+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:02.468160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483478+0000","last_change":"2024-08-17T16:53:37.120604+0000","last_active":"2024-08-17T16:54:20.483478+0000","last_peered":"2024-08-17T16:54:20.483478+0000","last_clean":"2024-08-17T16:54:20.483478+0000","last_became_active":"2024-08-17T16:53:37.120308+0000","last_became_peered":"2024-08-17T16:53:37.120308+0000","last_unstale":"2024-08-17T16:54:20.483478+0000","last_undegraded":"2024-08-17T16:54:20.483478+0000","last_fullsized":"2024-08-17T16:54:20.483478+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:06.306915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.480973+0000","last_change":"2024-08-17T16:53:36.890705+0000","last_active":"2024-08-17T16:54:20.480973+0000","last_peered":"2024-08-17T16:54:20.480973+0000","last_clean":"2024-08-17T16:54:20.480973+0000","last_became_active":"2024-08-17T16:53:36.889646+0000","last_became_peered":"2024-08-17T16:53:36.889646+0000","last_unstale":"2024-08-17T16:54:20.480973+0000","last_undegraded":"2024-08-17T16:54:20.480973+0000","last_fullsized":"2024-08-17T16:54:20.480973+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:24.768849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.481180+0000","last_change":"2024-08-17T16:53:38.252543+0000","last_active":"2024-08-17T16:54:20.481180+0000","last_peered":"2024-08-17T16:54:20.481180+0000","last_clean":"2024-08-17T16:54:20.481180+0000","last_became_active":"2024-08-17T16:53:38.252268+0000","last_became_peered":"2024-08-17T16:53:38.252268+0000","last_unstale":"2024-08-17T16:54:20.481180+0000","last_undegraded":"2024-08-17T16:54:20.481180+0000","last_fullsized":"2024-08-17T16:54:20.481180+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:03:05.590587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896573+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T16:54:20.896573+0000","last_peered":"2024-08-17T16:54:20.896573+0000","last_clean":"2024-08-17T16:54:20.896573+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T16:54:20.896573+0000","last_undegraded":"2024-08-17T16:54:20.896573+0000","last_fullsized":"2024-08-17T16:54:20.896573+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486564+0000","last_change":"2024-08-17T16:53:37.121032+0000","last_active":"2024-08-17T16:54:20.486564+0000","last_peered":"2024-08-17T16:54:20.486564+0000","last_clean":"2024-08-17T16:54:20.486564+0000","last_became_active":"2024-08-17T16:53:37.120512+0000","last_became_peered":"2024-08-17T16:53:37.120512+0000","last_unstale":"2024-08-17T16:54:20.486564+0000","last_undegraded":"2024-08-17T16:54:20.486564+0000","last_fullsized":"2024-08-17T16:54:20.486564+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:58:17.507284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896511+0000","last_change":"2024-08-17T16:53:36.880307+0000","last_active":"2024-08-17T16:54:20.896511+0000","last_peered":"2024-08-17T16:54:20.896511+0000","last_clean":"2024-08-17T16:54:20.896511+0000","last_became_active":"2024-08-17T16:53:36.880009+0000","last_became_peered":"2024-08-17T16:53:36.880009+0000","last_unstale":"2024-08-17T16:54:20.896511+0000","last_undegraded":"2024-08-17T16:54:20.896511+0000","last_fullsized":"2024-08-17T16:54:20.896511+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:10.601837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.897149+0000","last_change":"2024-08-17T16:53:36.883203+0000","last_active":"2024-08-17T16:54:20.897149+0000","last_peered":"2024-08-17T16:54:20.897149+0000","last_clean":"2024-08-17T16:54:20.897149+0000","last_became_active":"2024-08-17T16:53:36.882908+0000","last_became_peered":"2024-08-17T16:53:36.882908+0000","last_unstale":"2024-08-17T16:54:20.897149+0000","last_undegraded":"2024-08-17T16:54:20.897149+0000","last_fullsized":"2024-08-17T16:54:20.897149+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:20.466791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483299+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T16:54:20.483299+0000","last_peered":"2024-08-17T16:54:20.483299+0000","last_clean":"2024-08-17T16:54:20.483299+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T16:54:20.483299+0000","last_undegraded":"2024-08-17T16:54:20.483299+0000","last_fullsized":"2024-08-17T16:54:20.483299+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896834+0000","last_change":"2024-08-17T16:53:36.881778+0000","last_active":"2024-08-17T16:54:20.896834+0000","last_peered":"2024-08-17T16:54:20.896834+0000","last_clean":"2024-08-17T16:54:20.896834+0000","last_became_active":"2024-08-17T16:53:36.881498+0000","last_became_peered":"2024-08-17T16:53:36.881498+0000","last_unstale":"2024-08-17T16:54:20.896834+0000","last_undegraded":"2024-08-17T16:54:20.896834+0000","last_fullsized":"2024-08-17T16:54:20.896834+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:25:06.671817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476836+0000","last_change":"2024-08-17T16:53:36.899945+0000","last_active":"2024-08-17T16:54:20.476836+0000","last_peered":"2024-08-17T16:54:20.476836+0000","last_clean":"2024-08-17T16:54:20.476836+0000","last_became_active":"2024-08-17T16:53:36.899631+0000","last_became_peered":"2024-08-17T16:53:36.899631+0000","last_unstale":"2024-08-17T16:54:20.476836+0000","last_undegraded":"2024-08-17T16:54:20.476836+0000","last_fullsized":"2024-08-17T16:54:20.476836+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:06:05.769480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896842+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T16:54:20.896842+0000","last_peered":"2024-08-17T16:54:20.896842+0000","last_clean":"2024-08-17T16:54:20.896842+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T16:54:20.896842+0000","last_undegraded":"2024-08-17T16:54:20.896842+0000","last_fullsized":"2024-08-17T16:54:20.896842+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896694+0000","last_change":"2024-08-17T16:53:37.118710+0000","last_active":"2024-08-17T16:54:20.896694+0000","last_peered":"2024-08-17T16:54:20.896694+0000","last_clean":"2024-08-17T16:54:20.896694+0000","last_became_active":"2024-08-17T16:53:37.118379+0000","last_became_peered":"2024-08-17T16:53:37.118379+0000","last_unstale":"2024-08-17T16:54:20.896694+0000","last_undegraded":"2024-08-17T16:54:20.896694+0000","last_fullsized":"2024-08-17T16:54:20.896694+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:27:23.613708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483708+0000","last_change":"2024-08-17T16:53:36.876268+0000","last_active":"2024-08-17T16:54:20.483708+0000","last_peered":"2024-08-17T16:54:20.483708+0000","last_clean":"2024-08-17T16:54:20.483708+0000","last_became_active":"2024-08-17T16:53:36.875953+0000","last_became_peered":"2024-08-17T16:53:36.875953+0000","last_unstale":"2024-08-17T16:54:20.483708+0000","last_undegraded":"2024-08-17T16:54:20.483708+0000","last_fullsized":"2024-08-17T16:54:20.483708+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:30.229916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":23,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.511744+0000","last_change":"2024-08-17T16:54:21.511744+0000","last_active":"2024-08-17T16:54:21.511744+0000","last_peered":"2024-08-17T16:54:21.511744+0000","last_clean":"2024-08-17T16:54:21.511744+0000","last_became_active":"2024-08-17T16:54:21.511445+0000","last_became_peered":"2024-08-17T16:54:21.511445+0000","last_unstale":"2024-08-17T16:54:21.511744+0000","last_undegraded":"2024-08-17T16:54:21.511744+0000","last_fullsized":"2024-08-17T16:54:21.511744+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:27:20.540018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486453+0000","last_change":"2024-08-17T16:53:36.886636+0000","last_active":"2024-08-17T16:54:20.486453+0000","last_peered":"2024-08-17T16:54:20.486453+0000","last_clean":"2024-08-17T16:54:20.486453+0000","last_became_active":"2024-08-17T16:53:36.886350+0000","last_became_peered":"2024-08-17T16:53:36.886350+0000","last_unstale":"2024-08-17T16:54:20.486453+0000","last_undegraded":"2024-08-17T16:54:20.486453+0000","last_fullsized":"2024-08-17T16:54:20.486453+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:35.454091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476973+0000","last_change":"2024-08-17T16:53:36.900748+0000","last_active":"2024-08-17T16:54:20.476973+0000","last_peered":"2024-08-17T16:54:20.476973+0000","last_clean":"2024-08-17T16:54:20.476973+0000","last_became_active":"2024-08-17T16:53:36.900332+0000","last_became_peered":"2024-08-17T16:53:36.900332+0000","last_unstale":"2024-08-17T16:54:20.476973+0000","last_undegraded":"2024-08-17T16:54:20.476973+0000","last_fullsized":"2024-08-17T16:54:20.476973+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:04:24.749007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.897080+0000","last_change":"2024-08-17T16:53:36.883760+0000","last_active":"2024-08-17T16:54:20.897080+0000","last_peered":"2024-08-17T16:54:20.897080+0000","last_clean":"2024-08-17T16:54:20.897080+0000","last_became_active":"2024-08-17T16:53:36.883480+0000","last_became_peered":"2024-08-17T16:53:36.883480+0000","last_unstale":"2024-08-17T16:54:20.897080+0000","last_undegraded":"2024-08-17T16:54:20.897080+0000","last_fullsized":"2024-08-17T16:54:20.897080+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:23.398009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.477067+0000","last_change":"2024-08-17T16:53:36.901528+0000","last_active":"2024-08-17T16:54:20.477067+0000","last_peered":"2024-08-17T16:54:20.477067+0000","last_clean":"2024-08-17T16:54:20.477067+0000","last_became_active":"2024-08-17T16:53:36.901280+0000","last_became_peered":"2024-08-17T16:53:36.901280+0000","last_unstale":"2024-08-17T16:54:20.477067+0000","last_undegraded":"2024-08-17T16:54:20.477067+0000","last_fullsized":"2024-08-17T16:54:20.477067+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:16:13.363564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476807+0000","last_change":"2024-08-17T16:53:36.881045+0000","last_active":"2024-08-17T16:54:20.476807+0000","last_peered":"2024-08-17T16:54:20.476807+0000","last_clean":"2024-08-17T16:54:20.476807+0000","last_became_active":"2024-08-17T16:53:36.880658+0000","last_became_peered":"2024-08-17T16:53:36.880658+0000","last_unstale":"2024-08-17T16:54:20.476807+0000","last_undegraded":"2024-08-17T16:54:20.476807+0000","last_fullsized":"2024-08-17T16:54:20.476807+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:29:49.527766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.477029+0000","last_change":"2024-08-17T16:53:36.901686+0000","last_active":"2024-08-17T16:54:20.477029+0000","last_peered":"2024-08-17T16:54:20.477029+0000","last_clean":"2024-08-17T16:54:20.477029+0000","last_became_active":"2024-08-17T16:53:36.901190+0000","last_became_peered":"2024-08-17T16:53:36.901190+0000","last_unstale":"2024-08-17T16:54:20.477029+0000","last_undegraded":"2024-08-17T16:54:20.477029+0000","last_fullsized":"2024-08-17T16:54:20.477029+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:04:50.685231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486012+0000","last_change":"2024-08-17T16:53:36.888957+0000","last_active":"2024-08-17T16:54:20.486012+0000","last_peered":"2024-08-17T16:54:20.486012+0000","last_clean":"2024-08-17T16:54:20.486012+0000","last_became_active":"2024-08-17T16:53:36.888648+0000","last_became_peered":"2024-08-17T16:53:36.888648+0000","last_unstale":"2024-08-17T16:54:20.486012+0000","last_undegraded":"2024-08-17T16:54:20.486012+0000","last_fullsized":"2024-08-17T16:54:20.486012+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:55:30.381593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.613239+0000","last_change":"2024-08-17T16:54:21.613239+0000","last_active":"2024-08-17T16:54:21.613239+0000","last_peered":"2024-08-17T16:54:21.613239+0000","last_clean":"2024-08-17T16:54:21.613239+0000","last_became_active":"2024-08-17T16:54:21.612882+0000","last_became_peered":"2024-08-17T16:54:21.612882+0000","last_unstale":"2024-08-17T16:54:21.613239+0000","last_undegraded":"2024-08-17T16:54:21.613239+0000","last_fullsized":"2024-08-17T16:54:21.613239+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:02:18.592726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":271,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896513+0000","last_change":"2024-08-17T16:51:52.132475+0000","last_active":"2024-08-17T16:54:20.896513+0000","last_peered":"2024-08-17T16:54:20.896513+0000","last_clean":"2024-08-17T16:54:20.896513+0000","last_became_active":"2024-08-17T16:51:52.132088+0000","last_became_peered":"2024-08-17T16:51:52.132088+0000","last_unstale":"2024-08-17T16:54:20.896513+0000","last_undegraded":"2024-08-17T16:54:20.896513+0000","last_fullsized":"2024-08-17T16:54:20.896513+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:13.679985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":29,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.753832+0000","last_change":"2024-08-17T16:52:28.100670+0000","last_active":"2024-08-17T16:54:21.753832+0000","last_peered":"2024-08-17T16:54:21.753832+0000","last_clean":"2024-08-17T16:54:21.753832+0000","last_became_active":"2024-08-17T16:52:23.430320+0000","last_became_peered":"2024-08-17T16:52:23.430320+0000","last_unstale":"2024-08-17T16:54:21.753832+0000","last_undegraded":"2024-08-17T16:54:21.753832+0000","last_fullsized":"2024-08-17T16:54:21.753832+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:47.861974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486367+0000","last_change":"2024-08-17T16:53:36.884227+0000","last_active":"2024-08-17T16:54:20.486367+0000","last_peered":"2024-08-17T16:54:20.486367+0000","last_clean":"2024-08-17T16:54:20.486367+0000","last_became_active":"2024-08-17T16:53:36.883539+0000","last_became_peered":"2024-08-17T16:53:36.883539+0000","last_unstale":"2024-08-17T16:54:20.486367+0000","last_undegraded":"2024-08-17T16:54:20.486367+0000","last_fullsized":"2024-08-17T16:54:20.486367+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T16:55:22.330403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896963+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T16:54:20.896963+0000","last_peered":"2024-08-17T16:54:20.896963+0000","last_clean":"2024-08-17T16:54:20.896963+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T16:54:20.896963+0000","last_undegraded":"2024-08-17T16:54:20.896963+0000","last_fullsized":"2024-08-17T16:54:20.896963+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.565607+0000","last_change":"2024-08-17T16:52:28.243231+0000","last_active":"2024-08-17T16:54:21.565607+0000","last_peered":"2024-08-17T16:54:21.565607+0000","last_clean":"2024-08-17T16:54:21.565607+0000","last_became_active":"2024-08-17T16:52:23.739287+0000","last_became_peered":"2024-08-17T16:52:23.739287+0000","last_unstale":"2024-08-17T16:54:21.565607+0000","last_undegraded":"2024-08-17T16:54:21.565607+0000","last_fullsized":"2024-08-17T16:54:21.565607+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:39.119833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.481903+0000","last_change":"2024-08-17T16:53:36.889086+0000","last_active":"2024-08-17T16:54:20.481903+0000","last_peered":"2024-08-17T16:54:20.481903+0000","last_clean":"2024-08-17T16:54:20.481903+0000","last_became_active":"2024-08-17T16:53:36.888137+0000","last_became_peered":"2024-08-17T16:53:36.888137+0000","last_unstale":"2024-08-17T16:54:20.481903+0000","last_undegraded":"2024-08-17T16:54:20.481903+0000","last_fullsized":"2024-08-17T16:54:20.481903+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:11.906601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":23,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.755048+0000","last_change":"2024-08-17T16:54:21.755048+0000","last_active":"2024-08-17T16:54:21.755048+0000","last_peered":"2024-08-17T16:54:21.755048+0000","last_clean":"2024-08-17T16:54:21.755048+0000","last_became_active":"2024-08-17T16:54:21.754695+0000","last_became_peered":"2024-08-17T16:54:21.754695+0000","last_unstale":"2024-08-17T16:54:21.755048+0000","last_undegraded":"2024-08-17T16:54:21.755048+0000","last_fullsized":"2024-08-17T16:54:21.755048+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:23:46.650736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.492865+0000","last_change":"2024-08-17T16:52:28.097703+0000","last_active":"2024-08-17T16:54:21.492865+0000","last_peered":"2024-08-17T16:54:21.492865+0000","last_clean":"2024-08-17T16:54:21.492865+0000","last_became_active":"2024-08-17T16:52:23.739224+0000","last_became_peered":"2024-08-17T16:52:23.739224+0000","last_unstale":"2024-08-17T16:54:21.492865+0000","last_undegraded":"2024-08-17T16:54:21.492865+0000","last_fullsized":"2024-08-17T16:54:21.492865+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:07:33.382141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483252+0000","last_change":"2024-08-17T16:53:36.889705+0000","last_active":"2024-08-17T16:54:20.483252+0000","last_peered":"2024-08-17T16:54:20.483252+0000","last_clean":"2024-08-17T16:54:20.483252+0000","last_became_active":"2024-08-17T16:53:36.889439+0000","last_became_peered":"2024-08-17T16:53:36.889439+0000","last_unstale":"2024-08-17T16:54:20.483252+0000","last_undegraded":"2024-08-17T16:54:20.483252+0000","last_fullsized":"2024-08-17T16:54:20.483252+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:15:54.587710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483708+0000","last_change":"2024-08-17T16:53:37.996396+0000","last_active":"2024-08-17T16:54:20.483708+0000","last_peered":"2024-08-17T16:54:20.483708+0000","last_clean":"2024-08-17T16:54:20.483708+0000","last_became_active":"2024-08-17T16:53:37.995865+0000","last_became_peered":"2024-08-17T16:53:37.995865+0000","last_unstale":"2024-08-17T16:54:20.483708+0000","last_undegraded":"2024-08-17T16:54:20.483708+0000","last_fullsized":"2024-08-17T16:54:20.483708+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:35.253721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":30,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.753778+0000","last_change":"2024-08-17T16:52:28.101249+0000","last_active":"2024-08-17T16:54:21.753778+0000","last_peered":"2024-08-17T16:54:21.753778+0000","last_clean":"2024-08-17T16:54:21.753778+0000","last_became_active":"2024-08-17T16:52:23.430593+0000","last_became_peered":"2024-08-17T16:52:23.430593+0000","last_unstale":"2024-08-17T16:54:21.753778+0000","last_undegraded":"2024-08-17T16:54:21.753778+0000","last_fullsized":"2024-08-17T16:54:21.753778+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:19.292172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.477094+0000","last_change":"2024-08-17T16:53:36.888745+0000","last_active":"2024-08-17T16:54:20.477094+0000","last_peered":"2024-08-17T16:54:20.477094+0000","last_clean":"2024-08-17T16:54:20.477094+0000","last_became_active":"2024-08-17T16:53:36.888395+0000","last_became_peered":"2024-08-17T16:53:36.888395+0000","last_unstale":"2024-08-17T16:54:20.477094+0000","last_undegraded":"2024-08-17T16:54:20.477094+0000","last_fullsized":"2024-08-17T16:54:20.477094+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:30:49.594838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483106+0000","last_change":"2024-08-17T16:53:38.251602+0000","last_active":"2024-08-17T16:54:20.483106+0000","last_peered":"2024-08-17T16:54:20.483106+0000","last_clean":"2024-08-17T16:54:20.483106+0000","last_became_active":"2024-08-17T16:53:38.251342+0000","last_became_peered":"2024-08-17T16:53:38.251342+0000","last_unstale":"2024-08-17T16:54:20.483106+0000","last_undegraded":"2024-08-17T16:54:20.483106+0000","last_fullsized":"2024-08-17T16:54:20.483106+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:02:16.318917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483648+0000","last_change":"2024-08-17T16:53:36.880034+0000","last_active":"2024-08-17T16:54:20.483648+0000","last_peered":"2024-08-17T16:54:20.483648+0000","last_clean":"2024-08-17T16:54:20.483648+0000","last_became_active":"2024-08-17T16:53:36.879690+0000","last_became_peered":"2024-08-17T16:53:36.879690+0000","last_unstale":"2024-08-17T16:54:20.483648+0000","last_undegraded":"2024-08-17T16:54:20.483648+0000","last_fullsized":"2024-08-17T16:54:20.483648+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:36:46.047979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.492186+0000","last_change":"2024-08-17T16:52:28.097582+0000","last_active":"2024-08-17T16:54:21.492186+0000","last_peered":"2024-08-17T16:54:21.492186+0000","last_clean":"2024-08-17T16:54:21.492186+0000","last_became_active":"2024-08-17T16:52:23.739237+0000","last_became_peered":"2024-08-17T16:52:23.739237+0000","last_unstale":"2024-08-17T16:54:21.492186+0000","last_undegraded":"2024-08-17T16:54:21.492186+0000","last_fullsized":"2024-08-17T16:54:21.492186+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:02:16.574623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.482644+0000","last_change":"2024-08-17T16:53:38.087553+0000","last_active":"2024-08-17T16:54:20.482644+0000","last_peered":"2024-08-17T16:54:20.482644+0000","last_clean":"2024-08-17T16:54:20.482644+0000","last_became_active":"2024-08-17T16:53:38.087152+0000","last_became_peered":"2024-08-17T16:53:38.087152+0000","last_unstale":"2024-08-17T16:54:20.482644+0000","last_undegraded":"2024-08-17T16:54:20.482644+0000","last_fullsized":"2024-08-17T16:54:20.482644+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:33.385166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483289+0000","last_change":"2024-08-17T16:53:36.887881+0000","last_active":"2024-08-17T16:54:20.483289+0000","last_peered":"2024-08-17T16:54:20.483289+0000","last_clean":"2024-08-17T16:54:20.483289+0000","last_became_active":"2024-08-17T16:53:36.887575+0000","last_became_peered":"2024-08-17T16:53:36.887575+0000","last_unstale":"2024-08-17T16:54:20.483289+0000","last_undegraded":"2024-08-17T16:54:20.483289+0000","last_fullsized":"2024-08-17T16:54:20.483289+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:11:03.473197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.565881+0000","last_change":"2024-08-17T16:52:28.243251+0000","last_active":"2024-08-17T16:54:21.565881+0000","last_peered":"2024-08-17T16:54:21.565881+0000","last_clean":"2024-08-17T16:54:21.565881+0000","last_became_active":"2024-08-17T16:52:23.816512+0000","last_became_peered":"2024-08-17T16:52:23.816512+0000","last_unstale":"2024-08-17T16:54:21.565881+0000","last_undegraded":"2024-08-17T16:54:21.565881+0000","last_fullsized":"2024-08-17T16:54:21.565881+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:59.683846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.482952+0000","last_change":"2024-08-17T16:53:37.996145+0000","last_active":"2024-08-17T16:54:20.482952+0000","last_peered":"2024-08-17T16:54:20.482952+0000","last_clean":"2024-08-17T16:54:20.482952+0000","last_became_active":"2024-08-17T16:53:37.995802+0000","last_became_peered":"2024-08-17T16:53:37.995802+0000","last_unstale":"2024-08-17T16:54:20.482952+0000","last_undegraded":"2024-08-17T16:54:20.482952+0000","last_fullsized":"2024-08-17T16:54:20.482952+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:58.123211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486619+0000","last_change":"2024-08-17T16:53:36.885630+0000","last_active":"2024-08-17T16:54:20.486619+0000","last_peered":"2024-08-17T16:54:20.486619+0000","last_clean":"2024-08-17T16:54:20.486619+0000","last_became_active":"2024-08-17T16:53:36.885340+0000","last_became_peered":"2024-08-17T16:53:36.885340+0000","last_unstale":"2024-08-17T16:54:20.486619+0000","last_undegraded":"2024-08-17T16:54:20.486619+0000","last_fullsized":"2024-08-17T16:54:20.486619+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:01.924572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.492270+0000","last_change":"2024-08-17T16:52:28.097477+0000","last_active":"2024-08-17T16:54:21.492270+0000","last_peered":"2024-08-17T16:54:21.492270+0000","last_clean":"2024-08-17T16:54:21.492270+0000","last_became_active":"2024-08-17T16:52:23.816540+0000","last_became_peered":"2024-08-17T16:52:23.816540+0000","last_unstale":"2024-08-17T16:54:21.492270+0000","last_undegraded":"2024-08-17T16:54:21.492270+0000","last_fullsized":"2024-08-17T16:54:21.492270+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:27:49.345586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000334624,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.485895+0000","last_change":"2024-08-17T16:53:37.150235+0000","last_active":"2024-08-17T16:54:20.485895+0000","last_peered":"2024-08-17T16:54:20.485895+0000","last_clean":"2024-08-17T16:54:20.485895+0000","last_became_active":"2024-08-17T16:53:37.149897+0000","last_became_peered":"2024-08-17T16:53:37.149897+0000","last_unstale":"2024-08-17T16:54:20.485895+0000","last_undegraded":"2024-08-17T16:54:20.485895+0000","last_fullsized":"2024-08-17T16:54:20.485895+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:51.578814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.481293+0000","last_change":"2024-08-17T16:53:36.885579+0000","last_active":"2024-08-17T16:54:20.481293+0000","last_peered":"2024-08-17T16:54:20.481293+0000","last_clean":"2024-08-17T16:54:20.481293+0000","last_became_active":"2024-08-17T16:53:36.885271+0000","last_became_peered":"2024-08-17T16:53:36.885271+0000","last_unstale":"2024-08-17T16:54:20.481293+0000","last_undegraded":"2024-08-17T16:54:20.481293+0000","last_fullsized":"2024-08-17T16:54:20.481293+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:27:50.616487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":27,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-08-17T16:54:21.491808+0000","last_change":"2024-08-17T16:52:28.102166+0000","last_active":"2024-08-17T16:54:21.491808+0000","last_peered":"2024-08-17T16:54:21.491808+0000","last_clean":"2024-08-17T16:54:21.491808+0000","last_became_active":"2024-08-17T16:52:23.430739+0000","last_became_peered":"2024-08-17T16:52:23.430739+0000","last_unstale":"2024-08-17T16:54:21.491808+0000","last_undegraded":"2024-08-17T16:54:21.491808+0000","last_fullsized":"2024-08-17T16:54:21.491808+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:19.139627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476637+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T16:54:20.476637+0000","last_peered":"2024-08-17T16:54:20.476637+0000","last_clean":"2024-08-17T16:54:20.476637+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T16:54:20.476637+0000","last_undegraded":"2024-08-17T16:54:20.476637+0000","last_fullsized":"2024-08-17T16:54:20.476637+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476956+0000","last_change":"2024-08-17T16:53:36.901638+0000","last_active":"2024-08-17T16:54:20.476956+0000","last_peered":"2024-08-17T16:54:20.476956+0000","last_clean":"2024-08-17T16:54:20.476956+0000","last_became_active":"2024-08-17T16:53:36.901269+0000","last_became_peered":"2024-08-17T16:53:36.901269+0000","last_unstale":"2024-08-17T16:54:20.476956+0000","last_undegraded":"2024-08-17T16:54:20.476956+0000","last_fullsized":"2024-08-17T16:54:20.476956+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:31.151603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"72'6","reported_seq":21,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.477113+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T16:54:20.477113+0000","last_peered":"2024-08-17T16:54:20.477113+0000","last_clean":"2024-08-17T16:54:20.477113+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T16:54:20.477113+0000","last_undegraded":"2024-08-17T16:54:20.477113+0000","last_fullsized":"2024-08-17T16:54:20.477113+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483466+0000","last_change":"2024-08-17T16:53:38.251303+0000","last_active":"2024-08-17T16:54:20.483466+0000","last_peered":"2024-08-17T16:54:20.483466+0000","last_clean":"2024-08-17T16:54:20.483466+0000","last_became_active":"2024-08-17T16:53:38.251048+0000","last_became_peered":"2024-08-17T16:53:38.251048+0000","last_unstale":"2024-08-17T16:54:20.483466+0000","last_undegraded":"2024-08-17T16:54:20.483466+0000","last_fullsized":"2024-08-17T16:54:20.483466+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:58.408282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":18,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896713+0000","last_change":"2024-08-17T16:53:36.877296+0000","last_active":"2024-08-17T16:54:20.896713+0000","last_peered":"2024-08-17T16:54:20.896713+0000","last_clean":"2024-08-17T16:54:20.896713+0000","last_became_active":"2024-08-17T16:53:36.877018+0000","last_became_peered":"2024-08-17T16:53:36.877018+0000","last_unstale":"2024-08-17T16:54:20.896713+0000","last_undegraded":"2024-08-17T16:54:20.896713+0000","last_fullsized":"2024-08-17T16:54:20.896713+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:19.839578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483501+0000","last_change":"2024-08-17T16:53:36.883382+0000","last_active":"2024-08-17T16:54:20.483501+0000","last_peered":"2024-08-17T16:54:20.483501+0000","last_clean":"2024-08-17T16:54:20.483501+0000","last_became_active":"2024-08-17T16:53:36.883108+0000","last_became_peered":"2024-08-17T16:53:36.883108+0000","last_unstale":"2024-08-17T16:54:20.483501+0000","last_undegraded":"2024-08-17T16:54:20.483501+0000","last_fullsized":"2024-08-17T16:54:20.483501+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:46:55.973076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486550+0000","last_change":"2024-08-17T16:53:37.119299+0000","last_active":"2024-08-17T16:54:20.486550+0000","last_peered":"2024-08-17T16:54:20.486550+0000","last_clean":"2024-08-17T16:54:20.486550+0000","last_became_active":"2024-08-17T16:53:37.118657+0000","last_became_peered":"2024-08-17T16:53:37.118657+0000","last_unstale":"2024-08-17T16:54:20.486550+0000","last_undegraded":"2024-08-17T16:54:20.486550+0000","last_fullsized":"2024-08-17T16:54:20.486550+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:14:32.942498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.482233+0000","last_change":"2024-08-17T16:53:36.889202+0000","last_active":"2024-08-17T16:54:20.482233+0000","last_peered":"2024-08-17T16:54:20.482233+0000","last_clean":"2024-08-17T16:54:20.482233+0000","last_became_active":"2024-08-17T16:53:36.888179+0000","last_became_peered":"2024-08-17T16:53:36.888179+0000","last_unstale":"2024-08-17T16:54:20.482233+0000","last_undegraded":"2024-08-17T16:54:20.482233+0000","last_fullsized":"2024-08-17T16:54:20.482233+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:49:45.828822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.485835+0000","last_change":"2024-08-17T16:53:36.902422+0000","last_active":"2024-08-17T16:54:20.485835+0000","last_peered":"2024-08-17T16:54:20.485835+0000","last_clean":"2024-08-17T16:54:20.485835+0000","last_became_active":"2024-08-17T16:53:36.902107+0000","last_became_peered":"2024-08-17T16:53:36.902107+0000","last_unstale":"2024-08-17T16:54:20.485835+0000","last_undegraded":"2024-08-17T16:54:20.485835+0000","last_fullsized":"2024-08-17T16:54:20.485835+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:06.822564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.896539+0000","last_change":"2024-08-17T16:53:36.901487+0000","last_active":"2024-08-17T16:54:20.896539+0000","last_peered":"2024-08-17T16:54:20.896539+0000","last_clean":"2024-08-17T16:54:20.896539+0000","last_became_active":"2024-08-17T16:53:36.901191+0000","last_became_peered":"2024-08-17T16:53:36.901191+0000","last_unstale":"2024-08-17T16:54:20.896539+0000","last_undegraded":"2024-08-17T16:54:20.896539+0000","last_fullsized":"2024-08-17T16:54:20.896539+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:26.851092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.485950+0000","last_change":"2024-08-17T16:53:36.889630+0000","last_active":"2024-08-17T16:54:20.485950+0000","last_peered":"2024-08-17T16:54:20.485950+0000","last_clean":"2024-08-17T16:54:20.485950+0000","last_became_active":"2024-08-17T16:53:36.889336+0000","last_became_peered":"2024-08-17T16:53:36.889336+0000","last_unstale":"2024-08-17T16:54:20.485950+0000","last_undegraded":"2024-08-17T16:54:20.485950+0000","last_fullsized":"2024-08-17T16:54:20.485950+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:10.013575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.480972+0000","last_change":"2024-08-17T16:53:38.251672+0000","last_active":"2024-08-17T16:54:20.480972+0000","last_peered":"2024-08-17T16:54:20.480972+0000","last_clean":"2024-08-17T16:54:20.480972+0000","last_became_active":"2024-08-17T16:53:38.251398+0000","last_became_peered":"2024-08-17T16:53:38.251398+0000","last_unstale":"2024-08-17T16:54:20.480972+0000","last_undegraded":"2024-08-17T16:54:20.480972+0000","last_fullsized":"2024-08-17T16:54:20.480972+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:37:05.335449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.482469+0000","last_change":"2024-08-17T16:53:36.886233+0000","last_active":"2024-08-17T16:54:20.482469+0000","last_peered":"2024-08-17T16:54:20.482469+0000","last_clean":"2024-08-17T16:54:20.482469+0000","last_became_active":"2024-08-17T16:53:36.885869+0000","last_became_peered":"2024-08-17T16:53:36.885869+0000","last_unstale":"2024-08-17T16:54:20.482469+0000","last_undegraded":"2024-08-17T16:54:20.482469+0000","last_fullsized":"2024-08-17T16:54:20.482469+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:44.305315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486329+0000","last_change":"2024-08-17T16:53:36.904687+0000","last_active":"2024-08-17T16:54:20.486329+0000","last_peered":"2024-08-17T16:54:20.486329+0000","last_clean":"2024-08-17T16:54:20.486329+0000","last_became_active":"2024-08-17T16:53:36.904393+0000","last_became_peered":"2024-08-17T16:53:36.904393+0000","last_unstale":"2024-08-17T16:54:20.486329+0000","last_undegraded":"2024-08-17T16:54:20.486329+0000","last_fullsized":"2024-08-17T16:54:20.486329+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:24:25.019529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476851+0000","last_change":"2024-08-17T16:53:36.882578+0000","last_active":"2024-08-17T16:54:20.476851+0000","last_peered":"2024-08-17T16:54:20.476851+0000","last_clean":"2024-08-17T16:54:20.476851+0000","last_became_active":"2024-08-17T16:53:36.882065+0000","last_became_peered":"2024-08-17T16:53:36.882065+0000","last_unstale":"2024-08-17T16:54:20.476851+0000","last_undegraded":"2024-08-17T16:54:20.476851+0000","last_fullsized":"2024-08-17T16:54:20.476851+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:01:35.409187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.481537+0000","last_change":"2024-08-17T16:53:36.902424+0000","last_active":"2024-08-17T16:54:20.481537+0000","last_peered":"2024-08-17T16:54:20.481537+0000","last_clean":"2024-08-17T16:54:20.481537+0000","last_became_active":"2024-08-17T16:53:36.902142+0000","last_became_peered":"2024-08-17T16:53:36.902142+0000","last_unstale":"2024-08-17T16:54:20.481537+0000","last_undegraded":"2024-08-17T16:54:20.481537+0000","last_fullsized":"2024-08-17T16:54:20.481537+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:11:07.703935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483073+0000","last_change":"2024-08-17T16:53:36.886281+0000","last_active":"2024-08-17T16:54:20.483073+0000","last_peered":"2024-08-17T16:54:20.483073+0000","last_clean":"2024-08-17T16:54:20.483073+0000","last_became_active":"2024-08-17T16:53:36.885982+0000","last_became_peered":"2024-08-17T16:53:36.885982+0000","last_unstale":"2024-08-17T16:54:20.483073+0000","last_undegraded":"2024-08-17T16:54:20.483073+0000","last_fullsized":"2024-08-17T16:54:20.483073+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:57:11.167476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476881+0000","last_change":"2024-08-17T16:53:36.902213+0000","last_active":"2024-08-17T16:54:20.476881+0000","last_peered":"2024-08-17T16:54:20.476881+0000","last_clean":"2024-08-17T16:54:20.476881+0000","last_became_active":"2024-08-17T16:53:36.901923+0000","last_became_peered":"2024-08-17T16:53:36.901923+0000","last_unstale":"2024-08-17T16:54:20.476881+0000","last_undegraded":"2024-08-17T16:54:20.476881+0000","last_fullsized":"2024-08-17T16:54:20.476881+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:20:28.471491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483456+0000","last_change":"2024-08-17T16:53:36.882002+0000","last_active":"2024-08-17T16:54:20.483456+0000","last_peered":"2024-08-17T16:54:20.483456+0000","last_clean":"2024-08-17T16:54:20.483456+0000","last_became_active":"2024-08-17T16:53:36.881716+0000","last_became_peered":"2024-08-17T16:53:36.881716+0000","last_unstale":"2024-08-17T16:54:20.483456+0000","last_undegraded":"2024-08-17T16:54:20.483456+0000","last_fullsized":"2024-08-17T16:54:20.483456+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:50:51.350825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476753+0000","last_change":"2024-08-17T16:53:37.150151+0000","last_active":"2024-08-17T16:54:20.476753+0000","last_peered":"2024-08-17T16:54:20.476753+0000","last_clean":"2024-08-17T16:54:20.476753+0000","last_became_active":"2024-08-17T16:53:37.149940+0000","last_became_peered":"2024-08-17T16:53:37.149940+0000","last_unstale":"2024-08-17T16:54:20.476753+0000","last_undegraded":"2024-08-17T16:54:20.476753+0000","last_fullsized":"2024-08-17T16:54:20.476753+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:44.895666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.483679+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T16:54:20.483679+0000","last_peered":"2024-08-17T16:54:20.483679+0000","last_clean":"2024-08-17T16:54:20.483679+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T16:54:20.483679+0000","last_undegraded":"2024-08-17T16:54:20.483679+0000","last_fullsized":"2024-08-17T16:54:20.483679+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486359+0000","last_change":"2024-08-17T16:53:37.150694+0000","last_active":"2024-08-17T16:54:20.486359+0000","last_peered":"2024-08-17T16:54:20.486359+0000","last_clean":"2024-08-17T16:54:20.486359+0000","last_became_active":"2024-08-17T16:53:37.150303+0000","last_became_peered":"2024-08-17T16:53:37.150303+0000","last_unstale":"2024-08-17T16:54:20.486359+0000","last_undegraded":"2024-08-17T16:54:20.486359+0000","last_fullsized":"2024-08-17T16:54:20.486359+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:51.129623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.486242+0000","last_change":"2024-08-17T16:53:36.884929+0000","last_active":"2024-08-17T16:54:20.486242+0000","last_peered":"2024-08-17T16:54:20.486242+0000","last_clean":"2024-08-17T16:54:20.486242+0000","last_became_active":"2024-08-17T16:53:36.883501+0000","last_became_peered":"2024-08-17T16:53:36.883501+0000","last_unstale":"2024-08-17T16:54:20.486242+0000","last_undegraded":"2024-08-17T16:54:20.486242+0000","last_fullsized":"2024-08-17T16:54:20.486242+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:38:08.920185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.476635+0000","last_change":"2024-08-17T16:53:36.899921+0000","last_active":"2024-08-17T16:54:20.476635+0000","last_peered":"2024-08-17T16:54:20.476635+0000","last_clean":"2024-08-17T16:54:20.476635+0000","last_became_active":"2024-08-17T16:53:36.899661+0000","last_became_peered":"2024-08-17T16:53:36.899661+0000","last_unstale":"2024-08-17T16:54:20.476635+0000","last_undegraded":"2024-08-17T16:54:20.476635+0000","last_fullsized":"2024-08-17T16:54:20.476635+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:08:13.805113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":19,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-08-17T16:54:20.482141+0000","last_change":"2024-08-17T16:53:36.886775+0000","last_active":"2024-08-17T16:54:20.482141+0000","last_peered":"2024-08-17T16:54:20.482141+0000","last_clean":"2024-08-17T16:54:20.482141+0000","last_became_active":"2024-08-17T16:53:36.886364+0000","last_became_peered":"2024-08-17T16:53:36.886364+0000","last_unstale":"2024-08-17T16:54:20.482141+0000","last_undegraded":"2024-08-17T16:54:20.482141+0000","last_fullsized":"2024-08-17T16:54:20.482141+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:21.249333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201317,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28368,"kb_used_data":1348,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716784,"statfs":{"total":95995035648,"available":95965986816,"internally_reserved":0,"allocated":1380352,"data_stored":927971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.51300000000000001,"15min":0.51700000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.60799999999999998,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.318},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.53400000000000003,"15min":0.53600000000000003},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.66900000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.42499999999999999}]},{"osd":1,"last update":"Sat Aug 17 16:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.66400000000000003,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.63700000000000001,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.502}]},{"osd":2,"last update":"Sat Aug 17 16:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.5,"15min":0.497},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.65900000000000003,"5min":1.012,"15min":1.012},"last":0.54200000000000004}]},{"osd":3,"last update":"Sat Aug 17 16:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.52400000000000002,"15min":0.52200000000000002},"min":{"1min":0.38300000000000001,"5min":0.373,"15min":0.373},"max":{"1min":0.65800000000000003,"5min":0.66700000000000004,"15min":0.66700000000000004},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.53800000000000003,"15min":0.53500000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.65700000000000003,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.46999999999999997}]},{"osd":4,"last update":"Sat Aug 17 16:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44500000000000001,"15min":0.44600000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45600000000000002,"15min":0.45700000000000002},"min":{"1min":0.33300000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.56799999999999995}]},{"osd":5,"last update":"Sat Aug 17 16:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.46400000000000002,"15min":0.46200000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.48899999999999999,"15min":0.48499999999999999},"min":{"1min":0.35499999999999998,"5min":0.314,"15min":0.314},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.61899999999999999}]},{"osd":6,"last update":"Sat Aug 17 16:53:54 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.496,"15min":0.48999999999999999},"min":{"1min":0.35999999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.47999999999999998,"15min":0.47399999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.60399999999999998}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28360,"kb_used_data":1336,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716792,"statfs":{"total":95995035648,"available":95965995008,"internally_reserved":0,"allocated":1368064,"data_stored":927571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":27654601},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:54:32 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.505,"15min":0.49299999999999999},"min":{"1min":0.33900000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.71099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.52600000000000002,"15min":0.51800000000000002},"min":{"1min":0.35999999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.72899999999999998,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.57399999999999995}]},{"osd":1,"last update":"Sat Aug 17 16:54:32 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.52400000000000002,"15min":0.52200000000000002},"min":{"1min":0.38100000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.499,"15min":0.48499999999999999},"min":{"1min":0.43099999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.51700000000000002}]},{"osd":2,"last update":"Sat Aug 17 16:54:32 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.53900000000000003,"15min":0.53700000000000003},"min":{"1min":0.39200000000000002,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.73199999999999998,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.496,"15min":0.48399999999999999},"min":{"1min":0.41399999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.71099999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.54200000000000004}]},{"osd":3,"last update":"Sat Aug 17 16:54:32 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.51100000000000001,"15min":0.499},"min":{"1min":0.45900000000000002,"5min":0.315,"15min":0.315},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.53700000000000003,"15min":0.53100000000000003},"min":{"1min":0.42399999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.70499999999999996,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.67600000000000005}]},{"osd":4,"last update":"Sat Aug 17 16:54:32 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.41099999999999998,"15min":0.39300000000000002},"min":{"1min":0.27300000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.68500000000000005,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.42499999999999999,"15min":0.41099999999999998},"min":{"1min":0.32100000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.57799999999999996,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.63100000000000001}]},{"osd":5,"last update":"Sat Aug 17 16:54:32 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.42499999999999999,"15min":0.40300000000000002},"min":{"1min":0.30499999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.46500000000000002,"15min":0.44700000000000001},"min":{"1min":0.35599999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.66000000000000003}]},{"osd":7,"last update":"Sat Aug 17 16:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.47899999999999998,"15min":0.48199999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.66100000000000003,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.53200000000000003,"15min":0.53000000000000003},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.745,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.68799999999999994}]}]},{"osd":5,"up_from":39,"seq":167503724590,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27780,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717372,"statfs":{"total":95995035648,"available":95966588928,"internally_reserved":0,"allocated":774144,"data_stored":336767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.52500000000000002,"15min":0.52100000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.65900000000000003,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.52300000000000002,"15min":0.53300000000000003},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.68300000000000005,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.54200000000000004}]},{"osd":1,"last update":"Sat Aug 17 16:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.53500000000000003,"15min":0.54700000000000004},"min":{"1min":0.29199999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.66700000000000004,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.51900000000000002}]},{"osd":2,"last update":"Sat Aug 17 16:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.53700000000000003,"15min":0.53800000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.64100000000000001,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.53900000000000003,"15min":0.54000000000000004},"min":{"1min":0.39700000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.75800000000000001,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.60699999999999998}]},{"osd":3,"last update":"Sat Aug 17 16:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.54100000000000004,"15min":0.54500000000000004},"min":{"1min":0.38,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.76900000000000002,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.55800000000000005,"15min":0.56100000000000005},"min":{"1min":0.378,"5min":0.372,"15min":0.372},"max":{"1min":0.73399999999999999,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.54600000000000004}]},{"osd":4,"last update":"Sat Aug 17 16:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.40400000000000003,"15min":0.38900000000000001},"min":{"1min":0.28499999999999998,"5min":0.251,"15min":0.251},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.433,"15min":0.41899999999999998},"min":{"1min":0.25900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.67500000000000004,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.39700000000000002}]},{"osd":6,"last update":"Sat Aug 17 16:54:34 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.44800000000000001,"15min":0.441},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.69199999999999995,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.47899999999999998,"15min":0.47099999999999997},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.61299999999999999,"5min":1.02,"15min":1.02},"last":0.443}]},{"osd":7,"last update":"Sat Aug 17 16:53:52 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.47199999999999998,"15min":0.46600000000000003},"min":{"1min":0.36399999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.46700000000000003,"15min":0.46000000000000002},"min":{"1min":0.31900000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.499}]}]},{"osd":4,"up_from":31,"seq":133143986227,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31884,"kb_used_data":768,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713268,"statfs":{"total":95995035648,"available":95962386432,"internally_reserved":0,"allocated":786432,"data_stored":337864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:53:41 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.54600000000000004,"15min":0.54800000000000004},"min":{"1min":0.36499999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.67700000000000005,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54200000000000004,"15min":0.53700000000000003},"min":{"1min":0.40300000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.47899999999999998}]},{"osd":1,"last update":"Sat Aug 17 16:53:41 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.53900000000000003,"15min":0.54600000000000004},"min":{"1min":0.35399999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.441},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.53800000000000003,"15min":0.52800000000000002},"min":{"1min":0.34799999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.69899999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.56699999999999995}]},{"osd":2,"last update":"Sat Aug 17 16:53:41 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.56100000000000005,"15min":0.55800000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.53900000000000003,"15min":0.53800000000000003},"min":{"1min":0.35799999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.73699999999999999,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.42399999999999999}]},{"osd":3,"last update":"Sat Aug 17 16:53:41 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.434,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.68200000000000005,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.378},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.54900000000000004,"15min":0.55600000000000005},"min":{"1min":0.32100000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.54200000000000004}]},{"osd":5,"last update":"Sat Aug 17 16:54:08 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.28599999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.59799999999999998,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.42099999999999999,"15min":0.41799999999999998},"min":{"1min":0.30499999999999999,"5min":0.22,"15min":0.22},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.55700000000000005}]},{"osd":6,"last update":"Sat Aug 17 16:54:32 2024","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.44800000000000001,"15min":0.44500000000000001},"min":{"1min":0.32900000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.45900000000000002,"15min":0.45800000000000002},"min":{"1min":0.33000000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.73199999999999998,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.51600000000000001}]},{"osd":7,"last update":"Sat Aug 17 16:53:49 2024","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.49199999999999999,"15min":0.49099999999999999},"min":{"1min":0.378,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.65100000000000002,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.64700000000000002,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.503}]}]},{"osd":3,"up_from":26,"seq":111669149752,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27772,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717380,"statfs":{"total":95995035648,"available":95966597120,"internally_reserved":0,"allocated":770048,"data_stored":336517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.36199999999999999,"15min":0.37},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.51300000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.35699999999999998,"5min":0.39900000000000002,"15min":0.41199999999999998},"min":{"1min":0.252,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.47699999999999998,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.253}]},{"osd":1,"last update":"Sat Aug 17 16:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.35699999999999998,"5min":0.40600000000000003,"15min":0.42199999999999999},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.54100000000000004,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.35799999999999998,"5min":0.41299999999999998,"15min":0.439},"min":{"1min":0.245,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.503,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.312}]},{"osd":2,"last update":"Sat Aug 17 16:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.42599999999999999,"15min":0.436},"min":{"1min":0.26000000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.55500000000000005,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.41599999999999998,"15min":0.42699999999999999},"min":{"1min":0.27200000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.55400000000000005,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.52100000000000002}]},{"osd":4,"last update":"Sat Aug 17 16:53:42 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.57199999999999995,"15min":0.57999999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.73799999999999999,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.64400000000000002}]},{"osd":5,"last update":"Sat Aug 17 16:54:10 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.58399999999999996,"15min":0.58799999999999997},"min":{"1min":0.38600000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.57699999999999996,"15min":0.57299999999999995},"min":{"1min":0.45700000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.59199999999999997}]},{"osd":6,"last update":"Sat Aug 17 16:54:35 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59699999999999998,"15min":0.60099999999999998},"min":{"1min":0.40899999999999997,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.62,"5min":0.58599999999999997,"15min":0.57399999999999995},"min":{"1min":0.45700000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.875,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.68500000000000005}]},{"osd":7,"last update":"Sat Aug 17 16:53:51 2024","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.57599999999999996,"15min":0.56799999999999995},"min":{"1min":0.41899999999999998,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.57299999999999995,"15min":0.56499999999999995},"min":{"1min":0.45000000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.70599999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411389,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31872,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713280,"statfs":{"total":95995035648,"available":95962398720,"internally_reserved":0,"allocated":774144,"data_stored":337043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.33600000000000002,"5min":0.38400000000000001,"15min":0.38900000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.54100000000000004,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.39800000000000002,"15min":0.39000000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.52300000000000002,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.35299999999999998}]},{"osd":1,"last update":"Sat Aug 17 16:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.41299999999999998,"15min":0.40600000000000003},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.57999999999999996,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.379,"5min":0.41999999999999998,"15min":0.42099999999999999},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.62,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.38200000000000001}]},{"osd":3,"last update":"Sat Aug 17 16:54:24 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.45000000000000001,"15min":0.47599999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.57599999999999996,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.39100000000000001,"5min":0.45300000000000001,"15min":0.47699999999999998},"min":{"1min":0.30599999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.55500000000000005,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.316}]},{"osd":4,"last update":"Sat Aug 17 16:53:45 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.624,"15min":0.65200000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.90600000000000003,"5min":1.0649999999999999,"15min":1.0649999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.63800000000000001,"15min":0.66400000000000003},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.92600000000000005,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.64000000000000001}]},{"osd":5,"last update":"Sat Aug 17 16:54:09 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.58299999999999996,"15min":0.58499999999999996},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.92900000000000005,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.629,"15min":0.64900000000000002},"min":{"1min":0.46500000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.91400000000000003,"5min":1.0069999999999999,"15min":1.0069999999999999},"last":0.625}]},{"osd":6,"last update":"Sat Aug 17 16:54:30 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.60299999999999998,"15min":0.623},"min":{"1min":0.38700000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.81299999999999994,"5min":0.999,"15min":0.999},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.59399999999999997,"15min":0.60999999999999999},"min":{"1min":0.41499999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.75,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.53900000000000003}]},{"osd":7,"last update":"Sat Aug 17 16:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.57699999999999996,"15min":0.58499999999999996},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.82499999999999996,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.60099999999999998,"15min":0.60499999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.51800000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27772,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717380,"statfs":{"total":95995035648,"available":95966597120,"internally_reserved":0,"allocated":765952,"data_stored":336173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Aug 17 16:53:36 2024","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.34599999999999997,"15min":0.34000000000000002},"min":{"1min":0.223,"5min":0.193,"15min":0.193},"max":{"1min":0.52300000000000002,"5min":0.53300000000000003,"15min":0.53300000000000003},"last":0.374},{"interface":"front","average":{"1min":0.33400000000000002,"5min":0.36399999999999999,"15min":0.35099999999999998},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":0.53100000000000003,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.39300000000000002}]},{"osd":2,"last update":"Sat Aug 17 16:54:04 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.375,"15min":0.36499999999999999},"min":{"1min":0.26900000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.58899999999999997,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.372,"5min":0.371,"15min":0.35899999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":0.53100000000000003,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.34499999999999997}]},{"osd":3,"last update":"Sat Aug 17 16:54:20 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.40200000000000002,"15min":0.40600000000000003},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.57799999999999996,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.45700000000000002,"15min":0.46700000000000003},"min":{"1min":0.30499999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.753,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.41599999999999998}]},{"osd":4,"last update":"Sat Aug 17 16:53:44 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.56799999999999995,"15min":0.56100000000000005},"min":{"1min":0.36799999999999999,"5min":0.315,"15min":0.315},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.54100000000000004,"15min":0.52900000000000003},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.624}]},{"osd":5,"last update":"Sat Aug 17 16:54:04 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.59499999999999997,"15min":0.59199999999999997},"min":{"1min":0.38500000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.84199999999999997,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.501},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.42199999999999999,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.747,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.59699999999999998}]},{"osd":6,"last update":"Sat Aug 17 16:53:32 2024","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.60899999999999999,"15min":0.60799999999999998},"min":{"1min":0.44800000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.60599999999999998,"15min":0.60099999999999998},"min":{"1min":0.41199999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.58199999999999996}]},{"osd":7,"last update":"Sat Aug 17 16:53:51 2024","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.628,"15min":0.627},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.85599999999999998,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.433,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.86699999999999999,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.53700000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28356,"kb_used_data":1332,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716796,"statfs":{"total":95995035648,"available":95965999104,"internally_reserved":0,"allocated":1363968,"data_stored":927126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Aug 17 16:53:34 2024","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.34300000000000003,"15min":0.32700000000000001},"min":{"1min":0.224,"5min":0.193,"15min":0.193},"max":{"1min":0.46100000000000002,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.34499999999999997,"5min":0.36099999999999999,"15min":0.33500000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.53900000000000003,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.44400000000000001}]},{"osd":2,"last update":"Sat Aug 17 16:54:01 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.39100000000000001,"15min":0.39200000000000002},"min":{"1min":0.27800000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.50800000000000001,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39900000000000002,"15min":0.38900000000000001},"min":{"1min":0.249,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.59699999999999998,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.495}]},{"osd":3,"last update":"Sat Aug 17 16:54:27 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.46500000000000002,"15min":0.46300000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.55600000000000005,"5min":1.054,"15min":1.054},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.42899999999999999,"15min":0.436},"min":{"1min":0.28000000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.51800000000000002,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.47999999999999998}]},{"osd":4,"last update":"Sat Aug 17 16:53:46 2024","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.58799999999999997,"15min":0.59399999999999997},"min":{"1min":0.38700000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.79400000000000004,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60899999999999999,"15min":0.61199999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.81799999999999995,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.64900000000000002}]},{"osd":5,"last update":"Sat Aug 17 16:54:06 2024","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.61399999999999999,"15min":0.61899999999999999},"min":{"1min":0.40400000000000003,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.83599999999999997,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.59399999999999997,"15min":0.59299999999999997},"min":{"1min":0.41499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.79700000000000004,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.72999999999999998}]},{"osd":6,"last update":"Sat Aug 17 16:54:31 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.56399999999999995,"15min":0.56599999999999995},"min":{"1min":0.433,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79800000000000004,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.625,"5min":0.59099999999999997,"15min":0.58899999999999997},"min":{"1min":0.45800000000000002,"5min":0.374,"15min":0.374},"max":{"1min":0.73999999999999999,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.64100000000000001}]},{"osd":7,"last update":"Sat Aug 17 16:53:50 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.56299999999999994,"15min":0.55600000000000005},"min":{"1min":0.42599999999999999,"5min":0.313,"15min":0.313},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.57599999999999996,"15min":0.56899999999999995},"min":{"1min":0.48699999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77000000000000002,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.67900000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T16:54:39.347 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T16:54:39.347 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-17T16:54:39.347 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-17T16:54:39.348 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph health --format=json 2024-08-17T16:54:39.567 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:39.585 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:39 smithi063 ceph-mon[31063]: from='client.14781 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:54:39.585 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:39 smithi063 ceph-mon[42680]: from='client.14781 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:54:39.839 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:54:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:54:39] "GET /metrics HTTP/1.1" 200 33715 "" "Prometheus/2.43.0" 2024-08-17T16:54:39.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:39 smithi162 ceph-mon[34633]: from='client.14781 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T16:54:40.749 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:40 smithi063 ceph-mon[31063]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:40.750 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:40 smithi063 ceph-mon[42680]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:40.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:40 smithi162 ceph-mon[34633]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:41.470 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T16:54:41.470 INFO:teuthology.orchestra.run.smithi063.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-17T16:54:41.806 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:41 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/428764766' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T16:54:41.806 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:41 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/428764766' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T16:54:41.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:41 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/428764766' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-08-17T16:54:42.179 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-17T16:54:42.179 INFO:teuthology.run_tasks:Running task print... 2024-08-17T16:54:42.189 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-08-17T16:54:42.189 INFO:teuthology.run_tasks:Running task parallel... 2024-08-17T16:54:42.196 INFO:teuthology.task.parallel:starting parallel... 2024-08-17T16:54:42.197 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-08-17T16:54:42.197 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-08-17T16:54:42.198 INFO:teuthology.task.full_sequential:In full_sequential, running task rados... 2024-08-17T16:54:42.201 INFO:tasks.rados:Beginning rados... 2024-08-17T16:54:42.201 INFO:tasks.rados:config is {'clients': ['client.0'], 'objects': 50, 'op_weights': {'delete': 50, 'read': 100, 'rollback': 50, 'snap_create': 50, 'snap_remove': 50, 'write': 100}, 'ops': 4000, 'write_append_excl': False} 2024-08-17T16:54:42.201 INFO:tasks.rados:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '91f88e4d132ac08dc53355fa3d9717a53236a3a9'}} 2024-08-17T16:54:42.201 INFO:tasks.rados:config is {'clients': ['client.0'], 'objects': 50, 'op_weights': {'delete': 50, 'read': 100, 'rollback': 50, 'snap_create': 50, 'snap_remove': 50, 'write': 100}, 'ops': 4000, 'write_append_excl': False} 2024-08-17T16:54:42.202 INFO:tasks.rados:joining rados 2024-08-17T16:54:42.202 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-08-17T16:54:42.202 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-08-17T16:54:42.202 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi063.front.sepia.ceph.com 2024-08-17T16:54:42.202 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-08-17T16:54:42.203 INFO:tasks.rados:clients are ['client.0', 'client.1'] 2024-08-17T16:54:42.203 INFO:tasks.rados:starting run 0 out of 1 2024-08-17T16:54:42.203 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-08-17T16:54:42.203 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_0 16 2024-08-17T16:54:42.442 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:42.442 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:42.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:42 smithi162 ceph-mon[34633]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:42.851 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:42 smithi063 ceph-mon[42680]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:42.851 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:42 smithi063 ceph-mon[31063]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:44.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:44 smithi063 ceph-mon[42680]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:44.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:44 smithi063 ceph-mon[31063]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:44.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:44 smithi162 ceph-mon[34633]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:45.099 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:54:45 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:54:45] ENGINE Bus STOPPING 2024-08-17T16:54:45.099 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:54:45 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:54:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T16:54:45.364 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:54:45 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:54:45] ENGINE Bus STOPPED 2024-08-17T16:54:45.364 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:54:45 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:54:45] ENGINE Bus STARTING 2024-08-17T16:54:45.364 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:54:45 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:54:45] ENGINE Serving on http://:::9283 2024-08-17T16:54:45.364 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:54:45 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:54:45] ENGINE Bus STARTED 2024-08-17T16:54:45.957 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph config set global log_to_journald false --force' 2024-08-17T16:54:46.225 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:46 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3266588928' entity='client.admin' 2024-08-17T16:54:46.225 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:46 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:46.226 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:46 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:46.226 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:46 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:46.226 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:46 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:46.226 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:46 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3266588928' entity='client.admin' 2024-08-17T16:54:46.226 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:46 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:46.226 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:46 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:46.226 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:46 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:46.226 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:46 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:46.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:46 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3266588928' entity='client.admin' 2024-08-17T16:54:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:46 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:54:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:46 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:54:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:46 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:54:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:46 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:46.498 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:47.152 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:47 smithi063 ceph-mon[42680]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:47.152 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:47 smithi063 ceph-mon[31063]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:47.152 INFO:teuthology.orchestra.run.smithi063.stderr:pool 'unique_pool_0' created 2024-08-17T16:54:47.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:47 smithi162 ceph-mon[34633]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:48 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/4190123912' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-08-17T16:54:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:48 smithi063 ceph-mon[42680]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-08-17T16:54:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:48 smithi063 ceph-mon[42680]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-08-17T16:54:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:48 smithi063 ceph-mon[42680]: osdmap e75: 8 total, 8 up, 8 in 2024-08-17T16:54:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:48 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/4190123912' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-08-17T16:54:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:48 smithi063 ceph-mon[31063]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-08-17T16:54:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:48 smithi063 ceph-mon[31063]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-08-17T16:54:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:48 smithi063 ceph-mon[31063]: osdmap e75: 8 total, 8 up, 8 in 2024-08-17T16:54:48.422 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-08-17T16:54:48.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:48 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/4190123912' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-08-17T16:54:48.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:48 smithi162 ceph-mon[34633]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2024-08-17T16:54:48.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:48 smithi162 ceph-mon[34633]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-08-17T16:54:48.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:48 smithi162 ceph-mon[34633]: osdmap e75: 8 total, 8 up, 8 in 2024-08-17T16:54:48.920 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:49.475 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:49 smithi063 ceph-mon[42680]: osdmap e76: 8 total, 8 up, 8 in 2024-08-17T16:54:49.476 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:49 smithi063 ceph-mon[42680]: pgmap v93: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:49.476 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:49 smithi063 ceph-mon[31063]: osdmap e76: 8 total, 8 up, 8 in 2024-08-17T16:54:49.476 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:49 smithi063 ceph-mon[31063]: pgmap v93: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:49 smithi162 ceph-mon[34633]: osdmap e76: 8 total, 8 up, 8 in 2024-08-17T16:54:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:49 smithi162 ceph-mon[34633]: pgmap v93: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:49.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:54:49 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:54:49] "GET /metrics HTTP/1.1" 200 33715 "" "Prometheus/2.43.0" 2024-08-17T16:54:50.255 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:50 smithi063 ceph-mon[31063]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T16:54:50.255 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:50 smithi063 ceph-mon[31063]: osdmap e77: 8 total, 8 up, 8 in 2024-08-17T16:54:50.255 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:50 smithi063 ceph-mon[42680]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T16:54:50.256 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:50 smithi063 ceph-mon[42680]: osdmap e77: 8 total, 8 up, 8 in 2024-08-17T16:54:50.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:50 smithi162 ceph-mon[34633]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T16:54:50.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:50 smithi162 ceph-mon[34633]: osdmap e77: 8 total, 8 up, 8 in 2024-08-17T16:54:50.658 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for mgr daemons to upgrade' 2024-08-17T16:54:51.108 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:51.523 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:51 smithi063 ceph-mon[42680]: pgmap v95: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:51.523 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:51 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:51.523 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:51 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:54:51.523 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:51 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/78541169' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:54:51.523 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:51 smithi063 ceph-mon[31063]: pgmap v95: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:51.524 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:51 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:51.524 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:51 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:54:51.524 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:51 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/78541169' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:54:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:51 smithi162 ceph-mon[34633]: pgmap v95: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:51 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:54:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:51 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:54:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:51 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/78541169' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-08-17T16:54:52.158 INFO:teuthology.orchestra.run.smithi063.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-08-17T16:54:53.300 INFO:teuthology.orchestra.run.smithi063.stdout:wait for mgr daemons to upgrade 2024-08-17T16:54:53.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:53 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/78541169' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T16:54:53.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:53 smithi063 ceph-mon[31063]: osdmap e78: 8 total, 8 up, 8 in 2024-08-17T16:54:53.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:53 smithi063 ceph-mon[31063]: pgmap v97: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:53.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:53 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/78541169' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T16:54:53.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:53 smithi063 ceph-mon[42680]: osdmap e78: 8 total, 8 up, 8 in 2024-08-17T16:54:53.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:53 smithi063 ceph-mon[42680]: pgmap v97: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:53.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:53 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/78541169' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T16:54:53.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:53 smithi162 ceph-mon[34633]: osdmap e78: 8 total, 8 up, 8 in 2024-08-17T16:54:53.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:53 smithi162 ceph-mon[34633]: pgmap v97: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:53.855 DEBUG:teuthology.orchestra.run.smithi063:> CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 100 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 50 --pool unique_pool_0 2024-08-17T16:54:53.911 INFO:tasks.rados.rados.0.smithi063.stdout:adding op weight read -> 100 2024-08-17T16:54:53.911 INFO:tasks.rados.rados.0.smithi063.stdout:adding op weight write -> 100 2024-08-17T16:54:53.911 INFO:tasks.rados.rados.0.smithi063.stdout:adding op weight delete -> 50 2024-08-17T16:54:53.911 INFO:tasks.rados.rados.0.smithi063.stdout:adding op weight snap_create -> 50 2024-08-17T16:54:53.911 INFO:tasks.rados.rados.0.smithi063.stdout:adding op weight snap_remove -> 50 2024-08-17T16:54:53.911 INFO:tasks.rados.rados.0.smithi063.stdout:adding op weight rollback -> 50 2024-08-17T16:54:53.911 INFO:tasks.rados.rados.0.smithi063.stdout:ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable) 2024-08-17T16:54:53.912 INFO:tasks.rados.rados.0.smithi063.stdout:Configuration: 2024-08-17T16:54:53.912 INFO:tasks.rados.rados.0.smithi063.stdout: Number of operations: 4000 2024-08-17T16:54:53.912 INFO:tasks.rados.rados.0.smithi063.stdout: Number of objects: 50 2024-08-17T16:54:53.912 INFO:tasks.rados.rados.0.smithi063.stdout: Max in flight operations: 16 2024-08-17T16:54:53.912 INFO:tasks.rados.rados.0.smithi063.stdout: Object size (in bytes): 4000000 2024-08-17T16:54:53.912 INFO:tasks.rados.rados.0.smithi063.stdout: Write stride min: 400000 2024-08-17T16:54:53.912 INFO:tasks.rados.rados.0.smithi063.stdout: Write stride max: 800000 2024-08-17T16:54:53.980 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-08-17T16:54:54.171 INFO:tasks.rados.rados.0.smithi063.stdout:1: write initial oid 1 2024-08-17T16:54:54.171 INFO:tasks.rados.rados.0.smithi063.stdout:1: seq_num 0 ranges {416807=475249,1542129=474678} 2024-08-17T16:54:54.176 INFO:tasks.rados.rados.0.smithi063.stdout:1: writing smithi06387500-1 from 416807 to 892056 tid 1 2024-08-17T16:54:54.183 INFO:tasks.rados.rados.0.smithi063.stdout:1: writing smithi06387500-1 from 1542129 to 2016807 tid 2 2024-08-17T16:54:54.183 INFO:tasks.rados.rados.0.smithi063.stdout:2: write initial oid 2 2024-08-17T16:54:54.183 INFO:tasks.rados.rados.0.smithi063.stdout:2: seq_num 1 ranges {416807=475249,1542129=474678} 2024-08-17T16:54:54.187 INFO:tasks.rados.rados.0.smithi063.stdout:2: writing smithi06387500-2 from 416807 to 892056 tid 1 2024-08-17T16:54:54.194 INFO:tasks.rados.rados.0.smithi063.stdout:2: writing smithi06387500-2 from 1542129 to 2016807 tid 2 2024-08-17T16:54:54.194 INFO:tasks.rados.rados.0.smithi063.stdout:3: write initial oid 3 2024-08-17T16:54:54.194 INFO:tasks.rados.rados.0.smithi063.stdout:3: seq_num 2 ranges {433614=550498,1600611=433003} 2024-08-17T16:54:54.199 INFO:tasks.rados.rados.0.smithi063.stdout:3: writing smithi06387500-3 from 433614 to 984112 tid 1 2024-08-17T16:54:54.206 INFO:tasks.rados.rados.0.smithi063.stdout:3: writing smithi06387500-3 from 1600611 to 2033614 tid 2 2024-08-17T16:54:54.206 INFO:tasks.rados.rados.0.smithi063.stdout:4: write initial oid 4 2024-08-17T16:54:54.206 INFO:tasks.rados.rados.0.smithi063.stdout:4: seq_num 3 ranges {450421=625747,1659093=391328} 2024-08-17T16:54:54.212 INFO:tasks.rados.rados.0.smithi063.stdout:4: writing smithi06387500-4 from 450421 to 1076168 tid 1 2024-08-17T16:54:54.218 INFO:tasks.rados.rados.0.smithi063.stdout:4: writing smithi06387500-4 from 1659093 to 2050421 tid 2 2024-08-17T16:54:54.218 INFO:tasks.rados.rados.0.smithi063.stdout:5: write initial oid 5 2024-08-17T16:54:54.218 INFO:tasks.rados.rados.0.smithi063.stdout:5: seq_num 4 ranges {467228=700996,1717575=349653} 2024-08-17T16:54:54.224 INFO:tasks.rados.rados.0.smithi063.stdout:5: writing smithi06387500-5 from 467228 to 1168224 tid 1 2024-08-17T16:54:54.229 INFO:tasks.rados.rados.0.smithi063.stdout:5: writing smithi06387500-5 from 1717575 to 2067228 tid 2 2024-08-17T16:54:54.230 INFO:tasks.rados.rados.0.smithi063.stdout:6: write initial oid 6 2024-08-17T16:54:54.230 INFO:tasks.rados.rados.0.smithi063.stdout:6: seq_num 5 ranges {484035=776245,2059704=24331} 2024-08-17T16:54:54.236 INFO:tasks.rados.rados.0.smithi063.stdout:6: writing smithi06387500-6 from 484035 to 1260280 tid 1 2024-08-17T16:54:54.241 INFO:tasks.rados.rados.0.smithi063.stdout:6: writing smithi06387500-6 from 2059704 to 2084035 tid 2 2024-08-17T16:54:54.241 INFO:tasks.rados.rados.0.smithi063.stdout:7: write initial oid 7 2024-08-17T16:54:54.241 INFO:tasks.rados.rados.0.smithi063.stdout:7: seq_num 6 ranges {500842=451494,1718186=382656} 2024-08-17T16:54:54.246 INFO:tasks.rados.rados.0.smithi063.stdout:7: writing smithi06387500-7 from 500842 to 952336 tid 1 2024-08-17T16:54:54.248 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:54:54.254 INFO:tasks.rados.rados.0.smithi063.stdout:7: writing smithi06387500-7 from 1718186 to 2100842 tid 2 2024-08-17T16:54:54.254 INFO:tasks.rados.rados.0.smithi063.stdout:8: write initial oid 8 2024-08-17T16:54:54.254 INFO:tasks.rados.rados.0.smithi063.stdout:1: finishing write tid 1 to smithi06387500-1 2024-08-17T16:54:54.254 INFO:tasks.rados.rados.0.smithi063.stdout:1: finishing write tid 2 to smithi06387500-1 2024-08-17T16:54:54.254 INFO:tasks.rados.rados.0.smithi063.stdout:1: finishing write tid 3 to smithi06387500-1 2024-08-17T16:54:54.254 INFO:tasks.rados.rados.0.smithi063.stdout:1: finishing write tid 4 to smithi06387500-1 2024-08-17T16:54:54.254 INFO:tasks.rados.rados.0.smithi063.stdout:1: oid 1 updating version 0 to 2 2024-08-17T16:54:54.254 INFO:tasks.rados.rados.0.smithi063.stdout:1: oid 1 version 2 is already newer than 1 2024-08-17T16:54:54.254 INFO:tasks.rados.rados.0.smithi063.stdout:1: oid 1 updating version 2 to 3 2024-08-17T16:54:54.254 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 3 (ObjNum 0 snap 0 seq_num 0) dirty exists 2024-08-17T16:54:54.254 INFO:tasks.rados.rados.0.smithi063.stdout:1: left oid 1 (ObjNum 0 snap 0 seq_num 0) 2024-08-17T16:54:54.255 INFO:tasks.rados.rados.0.smithi063.stdout:2: finishing write tid 1 to smithi06387500-2 2024-08-17T16:54:54.255 INFO:tasks.rados.rados.0.smithi063.stdout:8: seq_num 7 ranges {517649=526743,1776668=340981} 2024-08-17T16:54:54.259 INFO:tasks.rados.rados.0.smithi063.stdout:8: writing smithi06387500-8 from 517649 to 1044392 tid 1 2024-08-17T16:54:54.266 INFO:tasks.rados.rados.0.smithi063.stdout:8: writing smithi06387500-8 from 1776668 to 2117649 tid 2 2024-08-17T16:54:54.267 INFO:tasks.rados.rados.0.smithi063.stdout:1: done (7 left) 2024-08-17T16:54:54.267 INFO:tasks.rados.rados.0.smithi063.stdout:9: write initial oid 9 2024-08-17T16:54:54.267 INFO:tasks.rados.rados.0.smithi063.stdout:9: seq_num 8 ranges {534456=718345,1951503=182953} 2024-08-17T16:54:54.274 INFO:tasks.rados.rados.0.smithi063.stdout:9: writing smithi06387500-9 from 534456 to 1252801 tid 1 2024-08-17T16:54:54.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:54 smithi063 ceph-mon[42680]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T16:54:54.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:54 smithi063 ceph-mon[42680]: Cluster is now healthy 2024-08-17T16:54:54.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:54 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/1078629904' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-08-17T16:54:54.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:54 smithi063 ceph-mon[31063]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T16:54:54.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:54 smithi063 ceph-mon[31063]: Cluster is now healthy 2024-08-17T16:54:54.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:54 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/1078629904' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-08-17T16:54:54.280 INFO:tasks.rados.rados.0.smithi063.stdout:9: writing smithi06387500-9 from 1951503 to 2134456 tid 2 2024-08-17T16:54:54.280 INFO:tasks.rados.rados.0.smithi063.stdout:10: write initial oid 10 2024-08-17T16:54:54.280 INFO:tasks.rados.rados.0.smithi063.stdout:10: seq_num 9 ranges {551263=793594,1893632=257631} 2024-08-17T16:54:54.286 INFO:tasks.rados.rados.0.smithi063.stdout:10: writing smithi06387500-10 from 551263 to 1344857 tid 1 2024-08-17T16:54:54.291 INFO:tasks.rados.rados.0.smithi063.stdout:10: writing smithi06387500-10 from 1893632 to 2151263 tid 2 2024-08-17T16:54:54.292 INFO:tasks.rados.rados.0.smithi063.stdout:11: write initial oid 11 2024-08-17T16:54:54.292 INFO:tasks.rados.rados.0.smithi063.stdout:11: seq_num 10 ranges {568070=468843,1552114=615956} 2024-08-17T16:54:54.297 INFO:tasks.rados.rados.0.smithi063.stdout:11: writing smithi06387500-11 from 568070 to 1036913 tid 1 2024-08-17T16:54:54.306 INFO:tasks.rados.rados.0.smithi063.stdout:11: writing smithi06387500-11 from 1552114 to 2168070 tid 2 2024-08-17T16:54:54.306 INFO:tasks.rados.rados.0.smithi063.stdout:12: write initial oid 12 2024-08-17T16:54:54.306 INFO:tasks.rados.rados.0.smithi063.stdout:12: seq_num 11 ranges {584877=544092,1610596=562003} 2024-08-17T16:54:54.310 INFO:tasks.rados.rados.0.smithi063.stdout:12: writing smithi06387500-12 from 584877 to 1128969 tid 1 2024-08-17T16:54:54.315 INFO:tasks.rados.rados.0.smithi063.stdout:12: writing smithi06387500-12 from 1610596 to 2172599 tid 2 2024-08-17T16:54:54.316 INFO:tasks.rados.rados.0.smithi063.stdout:13: write initial oid 13 2024-08-17T16:54:54.316 INFO:tasks.rados.rados.0.smithi063.stdout:13: seq_num 12 ranges {601684=619341,1669078=532606} 2024-08-17T16:54:54.322 INFO:tasks.rados.rados.0.smithi063.stdout:13: writing smithi06387500-13 from 601684 to 1221025 tid 1 2024-08-17T16:54:54.327 INFO:tasks.rados.rados.0.smithi063.stdout:13: writing smithi06387500-13 from 1669078 to 2201684 tid 2 2024-08-17T16:54:54.327 INFO:tasks.rados.rados.0.smithi063.stdout:14: write initial oid 14 2024-08-17T16:54:54.328 INFO:tasks.rados.rados.0.smithi063.stdout:14: seq_num 13 ranges {618491=694590,2011207=207284} 2024-08-17T16:54:54.334 INFO:tasks.rados.rados.0.smithi063.stdout:14: writing smithi06387500-14 from 618491 to 1313081 tid 1 2024-08-17T16:54:54.339 INFO:tasks.rados.rados.0.smithi063.stdout:14: writing smithi06387500-14 from 2011207 to 2218491 tid 2 2024-08-17T16:54:54.340 INFO:tasks.rados.rados.0.smithi063.stdout:15: write initial oid 15 2024-08-17T16:54:54.340 INFO:tasks.rados.rados.0.smithi063.stdout:15: seq_num 14 ranges {635298=769839,2069689=165609} 2024-08-17T16:54:54.347 INFO:tasks.rados.rados.0.smithi063.stdout:15: writing smithi06387500-15 from 635298 to 1405137 tid 1 2024-08-17T16:54:54.352 INFO:tasks.rados.rados.0.smithi063.stdout:15: writing smithi06387500-15 from 2069689 to 2235298 tid 2 2024-08-17T16:54:54.352 INFO:tasks.rados.rados.0.smithi063.stdout:16: write initial oid 16 2024-08-17T16:54:54.352 INFO:tasks.rados.rados.0.smithi063.stdout:16: seq_num 15 ranges {652105=445088,1728171=452988} 2024-08-17T16:54:54.358 INFO:tasks.rados.rados.0.smithi063.stdout:16: writing smithi06387500-16 from 652105 to 1097193 tid 1 2024-08-17T16:54:54.364 INFO:tasks.rados.rados.0.smithi063.stdout:16: writing smithi06387500-16 from 1728171 to 2181159 tid 2 2024-08-17T16:54:54.364 INFO:tasks.rados.rados.0.smithi063.stdout:17: write initial oid 17 2024-08-17T16:54:54.364 INFO:tasks.rados.rados.0.smithi063.stdout:17: seq_num 16 ranges {668912=636690,1903006=365906} 2024-08-17T16:54:54.371 INFO:tasks.rados.rados.0.smithi063.stdout:17: writing smithi06387500-17 from 668912 to 1305602 tid 1 2024-08-17T16:54:54.377 INFO:tasks.rados.rados.0.smithi063.stdout:17: writing smithi06387500-17 from 1903006 to 2268912 tid 2 2024-08-17T16:54:54.377 INFO:tasks.rados.rados.0.smithi063.stdout: waiting on 16 2024-08-17T16:54:54.377 INFO:tasks.rados.rados.0.smithi063.stdout:2: finishing write tid 2 to smithi06387500-2 2024-08-17T16:54:54.377 INFO:tasks.rados.rados.0.smithi063.stdout:2: finishing write tid 3 to smithi06387500-2 2024-08-17T16:54:54.377 INFO:tasks.rados.rados.0.smithi063.stdout:2: finishing write tid 4 to smithi06387500-2 2024-08-17T16:54:54.377 INFO:tasks.rados.rados.0.smithi063.stdout:2: oid 2 updating version 0 to 5 2024-08-17T16:54:54.377 INFO:tasks.rados.rados.0.smithi063.stdout:2: oid 2 version 5 is already newer than 4 2024-08-17T16:54:54.377 INFO:tasks.rados.rados.0.smithi063.stdout:2: oid 2 updating version 5 to 6 2024-08-17T16:54:54.378 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 6 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-08-17T16:54:54.378 INFO:tasks.rados.rados.0.smithi063.stdout:2: left oid 2 (ObjNum 1 snap 0 seq_num 1) 2024-08-17T16:54:54.378 INFO:tasks.rados.rados.0.smithi063.stdout:3: finishing write tid 1 to smithi06387500-3 2024-08-17T16:54:54.378 INFO:tasks.rados.rados.0.smithi063.stdout:3: finishing write tid 2 to smithi06387500-3 2024-08-17T16:54:54.378 INFO:tasks.rados.rados.0.smithi063.stdout:3: finishing write tid 3 to smithi06387500-3 2024-08-17T16:54:54.378 INFO:tasks.rados.rados.0.smithi063.stdout:3: finishing write tid 4 to smithi06387500-3 2024-08-17T16:54:54.378 INFO:tasks.rados.rados.0.smithi063.stdout:3: oid 3 updating version 0 to 2 2024-08-17T16:54:54.378 INFO:tasks.rados.rados.0.smithi063.stdout:3: oid 3 version 2 is already newer than 1 2024-08-17T16:54:54.378 INFO:tasks.rados.rados.0.smithi063.stdout:3: oid 3 updating version 2 to 3 2024-08-17T16:54:54.378 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 3 (ObjNum 2 snap 0 seq_num 2) dirty exists 2024-08-17T16:54:54.378 INFO:tasks.rados.rados.0.smithi063.stdout:3: left oid 3 (ObjNum 2 snap 0 seq_num 2) 2024-08-17T16:54:54.378 INFO:tasks.rados.rados.0.smithi063.stdout:4: finishing write tid 1 to smithi06387500-4 2024-08-17T16:54:54.378 INFO:tasks.rados.rados.0.smithi063.stdout:4: finishing write tid 2 to smithi06387500-4 2024-08-17T16:54:54.379 INFO:tasks.rados.rados.0.smithi063.stdout:4: finishing write tid 3 to smithi06387500-4 2024-08-17T16:54:54.379 INFO:tasks.rados.rados.0.smithi063.stdout:4: finishing write tid 4 to smithi06387500-4 2024-08-17T16:54:54.379 INFO:tasks.rados.rados.0.smithi063.stdout:4: oid 4 updating version 0 to 1 2024-08-17T16:54:54.379 INFO:tasks.rados.rados.0.smithi063.stdout:4: oid 4 updating version 1 to 2 2024-08-17T16:54:54.379 INFO:tasks.rados.rados.0.smithi063.stdout:4: oid 4 updating version 2 to 3 2024-08-17T16:54:54.379 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 3 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-08-17T16:54:54.379 INFO:tasks.rados.rados.0.smithi063.stdout:4: left oid 4 (ObjNum 3 snap 0 seq_num 3) 2024-08-17T16:54:54.379 INFO:tasks.rados.rados.0.smithi063.stdout:5: finishing write tid 1 to smithi06387500-5 2024-08-17T16:54:54.379 INFO:tasks.rados.rados.0.smithi063.stdout:2: done (15 left) 2024-08-17T16:54:54.379 INFO:tasks.rados.rados.0.smithi063.stdout:3: done (14 left) 2024-08-17T16:54:54.379 INFO:tasks.rados.rados.0.smithi063.stdout:4: done (13 left) 2024-08-17T16:54:54.379 INFO:tasks.rados.rados.0.smithi063.stdout:18: write initial oid 18 2024-08-17T16:54:54.379 INFO:tasks.rados.rados.0.smithi063.stdout:18: seq_num 17 ranges {685719=711939,1845135=440584} 2024-08-17T16:54:54.384 INFO:tasks.rados.rados.0.smithi063.stdout:18: writing smithi06387500-18 from 685719 to 1397658 tid 1 2024-08-17T16:54:54.389 INFO:tasks.rados.rados.0.smithi063.stdout:18: writing smithi06387500-18 from 1845135 to 2285719 tid 2 2024-08-17T16:54:54.389 INFO:tasks.rados.rados.0.smithi063.stdout:19: write initial oid 19 2024-08-17T16:54:54.389 INFO:tasks.rados.rados.0.smithi063.stdout:19: seq_num 18 ranges {702526=787188,1903617=398909} 2024-08-17T16:54:54.397 INFO:tasks.rados.rados.0.smithi063.stdout:19: writing smithi06387500-19 from 702526 to 1489714 tid 1 2024-08-17T16:54:54.402 INFO:tasks.rados.rados.0.smithi063.stdout:19: writing smithi06387500-19 from 1903617 to 2302526 tid 2 2024-08-17T16:54:54.402 INFO:tasks.rados.rados.0.smithi063.stdout:20: write initial oid 20 2024-08-17T16:54:54.402 INFO:tasks.rados.rados.0.smithi063.stdout:20: seq_num 19 ranges {719333=462437,1962099=357234} 2024-08-17T16:54:54.408 INFO:tasks.rados.rados.0.smithi063.stdout:20: writing smithi06387500-20 from 719333 to 1181770 tid 1 2024-08-17T16:54:54.415 INFO:tasks.rados.rados.0.smithi063.stdout:20: writing smithi06387500-20 from 1962099 to 2319333 tid 2 2024-08-17T16:54:54.415 INFO:tasks.rados.rados.0.smithi063.stdout: waiting on 16 2024-08-17T16:54:54.415 INFO:tasks.rados.rados.0.smithi063.stdout:5: finishing write tid 2 to smithi06387500-5 2024-08-17T16:54:54.416 INFO:tasks.rados.rados.0.smithi063.stdout:5: finishing write tid 3 to smithi06387500-5 2024-08-17T16:54:54.416 INFO:tasks.rados.rados.0.smithi063.stdout:5: finishing write tid 4 to smithi06387500-5 2024-08-17T16:54:54.416 INFO:tasks.rados.rados.0.smithi063.stdout:5: oid 5 updating version 0 to 8 2024-08-17T16:54:54.416 INFO:tasks.rados.rados.0.smithi063.stdout:5: oid 5 version 8 is already newer than 7 2024-08-17T16:54:54.416 INFO:tasks.rados.rados.0.smithi063.stdout:5: oid 5 updating version 8 to 9 2024-08-17T16:54:54.416 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 9 (ObjNum 4 snap 0 seq_num 4) dirty exists 2024-08-17T16:54:54.416 INFO:tasks.rados.rados.0.smithi063.stdout:5: left oid 5 (ObjNum 4 snap 0 seq_num 4) 2024-08-17T16:54:54.416 INFO:tasks.rados.rados.0.smithi063.stdout:6: finishing write tid 1 to smithi06387500-6 2024-08-17T16:54:54.416 INFO:tasks.rados.rados.0.smithi063.stdout:6: finishing write tid 2 to smithi06387500-6 2024-08-17T16:54:54.416 INFO:tasks.rados.rados.0.smithi063.stdout:6: finishing write tid 3 to smithi06387500-6 2024-08-17T16:54:54.416 INFO:tasks.rados.rados.0.smithi063.stdout:6: finishing write tid 4 to smithi06387500-6 2024-08-17T16:54:54.416 INFO:tasks.rados.rados.0.smithi063.stdout:6: oid 6 updating version 0 to 3 2024-08-17T16:54:54.416 INFO:tasks.rados.rados.0.smithi063.stdout:6: oid 6 version 3 is already newer than 2 2024-08-17T16:54:54.417 INFO:tasks.rados.rados.0.smithi063.stdout:6: oid 6 version 3 is already newer than 1 2024-08-17T16:54:54.417 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 3 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-08-17T16:54:54.417 INFO:tasks.rados.rados.0.smithi063.stdout:6: left oid 6 (ObjNum 5 snap 0 seq_num 5) 2024-08-17T16:54:54.417 INFO:tasks.rados.rados.0.smithi063.stdout:7: finishing write tid 1 to smithi06387500-7 2024-08-17T16:54:54.417 INFO:tasks.rados.rados.0.smithi063.stdout:7: finishing write tid 2 to smithi06387500-7 2024-08-17T16:54:54.417 INFO:tasks.rados.rados.0.smithi063.stdout:5: done (15 left) 2024-08-17T16:54:54.417 INFO:tasks.rados.rados.0.smithi063.stdout:6: done (14 left) 2024-08-17T16:54:54.417 INFO:tasks.rados.rados.0.smithi063.stdout:21: write initial oid 21 2024-08-17T16:54:54.417 INFO:tasks.rados.rados.0.smithi063.stdout:21: seq_num 20 ranges {736140=537686,2020581=315559} 2024-08-17T16:54:54.422 INFO:tasks.rados.rados.0.smithi063.stdout:21: writing smithi06387500-21 from 736140 to 1273826 tid 1 2024-08-17T16:54:54.428 INFO:tasks.rados.rados.0.smithi063.stdout:21: writing smithi06387500-21 from 2020581 to 2336140 tid 2 2024-08-17T16:54:54.428 INFO:tasks.rados.rados.0.smithi063.stdout:22: write initial oid 22 2024-08-17T16:54:54.428 INFO:tasks.rados.rados.0.smithi063.stdout:22: seq_num 21 ranges {752947=612935,1962710=390237} 2024-08-17T16:54:54.435 INFO:tasks.rados.rados.0.smithi063.stdout:22: writing smithi06387500-22 from 752947 to 1365882 tid 1 2024-08-17T16:54:54.441 INFO:tasks.rados.rados.0.smithi063.stdout:22: writing smithi06387500-22 from 1962710 to 2352947 tid 2 2024-08-17T16:54:54.441 INFO:tasks.rados.rados.0.smithi063.stdout: waiting on 16 2024-08-17T16:54:54.441 INFO:tasks.rados.rados.0.smithi063.stdout:7: finishing write tid 3 to smithi06387500-7 2024-08-17T16:54:54.441 INFO:tasks.rados.rados.0.smithi063.stdout:7: finishing write tid 4 to smithi06387500-7 2024-08-17T16:54:54.441 INFO:tasks.rados.rados.0.smithi063.stdout:7: oid 7 updating version 0 to 3 2024-08-17T16:54:54.441 INFO:tasks.rados.rados.0.smithi063.stdout:7: oid 7 version 3 is already newer than 1 2024-08-17T16:54:54.442 INFO:tasks.rados.rados.0.smithi063.stdout:7: oid 7 version 3 is already newer than 2 2024-08-17T16:54:54.442 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 3 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-08-17T16:54:54.442 INFO:tasks.rados.rados.0.smithi063.stdout:7: left oid 7 (ObjNum 6 snap 0 seq_num 6) 2024-08-17T16:54:54.442 INFO:tasks.rados.rados.0.smithi063.stdout:8: finishing write tid 1 to smithi06387500-8 2024-08-17T16:54:54.442 INFO:tasks.rados.rados.0.smithi063.stdout:7: done (15 left) 2024-08-17T16:54:54.442 INFO:tasks.rados.rados.0.smithi063.stdout:23: write initial oid 23 2024-08-17T16:54:54.442 INFO:tasks.rados.rados.0.smithi063.stdout:23: seq_num 22 ranges {769754=688184,2021192=348562} 2024-08-17T16:54:54.449 INFO:tasks.rados.rados.0.smithi063.stdout:23: writing smithi06387500-23 from 769754 to 1457938 tid 1 2024-08-17T16:54:54.454 INFO:tasks.rados.rados.0.smithi063.stdout:23: writing smithi06387500-23 from 2021192 to 2369754 tid 2 2024-08-17T16:54:54.455 INFO:tasks.rados.rados.0.smithi063.stdout: waiting on 16 2024-08-17T16:54:54.455 INFO:tasks.rados.rados.0.smithi063.stdout:8: finishing write tid 2 to smithi06387500-8 2024-08-17T16:54:54.455 INFO:tasks.rados.rados.0.smithi063.stdout:8: finishing write tid 3 to smithi06387500-8 2024-08-17T16:54:54.455 INFO:tasks.rados.rados.0.smithi063.stdout:8: finishing write tid 4 to smithi06387500-8 2024-08-17T16:54:54.455 INFO:tasks.rados.rados.0.smithi063.stdout:8: oid 8 updating version 0 to 4 2024-08-17T16:54:54.455 INFO:tasks.rados.rados.0.smithi063.stdout:8: oid 8 updating version 4 to 6 2024-08-17T16:54:54.455 INFO:tasks.rados.rados.0.smithi063.stdout:8: oid 8 version 6 is already newer than 5 2024-08-17T16:54:54.455 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 6 (ObjNum 7 snap 0 seq_num 7) dirty exists 2024-08-17T16:54:54.455 INFO:tasks.rados.rados.0.smithi063.stdout:8: left oid 8 (ObjNum 7 snap 0 seq_num 7) 2024-08-17T16:54:54.455 INFO:tasks.rados.rados.0.smithi063.stdout:9: finishing write tid 1 to smithi06387500-9 2024-08-17T16:54:54.455 INFO:tasks.rados.rados.0.smithi063.stdout:9: finishing write tid 2 to smithi06387500-9 2024-08-17T16:54:54.455 INFO:tasks.rados.rados.0.smithi063.stdout:9: finishing write tid 3 to smithi06387500-9 2024-08-17T16:54:54.456 INFO:tasks.rados.rados.0.smithi063.stdout:9: finishing write tid 4 to smithi06387500-9 2024-08-17T16:54:54.456 INFO:tasks.rados.rados.0.smithi063.stdout:9: oid 9 updating version 0 to 5 2024-08-17T16:54:54.456 INFO:tasks.rados.rados.0.smithi063.stdout:9: oid 9 updating version 5 to 6 2024-08-17T16:54:54.456 INFO:tasks.rados.rados.0.smithi063.stdout:9: oid 9 version 6 is already newer than 4 2024-08-17T16:54:54.456 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 6 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-08-17T16:54:54.456 INFO:tasks.rados.rados.0.smithi063.stdout:9: left oid 9 (ObjNum 8 snap 0 seq_num 8) 2024-08-17T16:54:54.456 INFO:tasks.rados.rados.0.smithi063.stdout:10: finishing write tid 1 to smithi06387500-10 2024-08-17T16:54:54.456 INFO:tasks.rados.rados.0.smithi063.stdout:10: finishing write tid 2 to smithi06387500-10 2024-08-17T16:54:54.456 INFO:tasks.rados.rados.0.smithi063.stdout:8: done (15 left) 2024-08-17T16:54:54.456 INFO:tasks.rados.rados.0.smithi063.stdout:9: done (14 left) 2024-08-17T16:54:54.456 INFO:tasks.rados.rados.0.smithi063.stdout:24: write initial oid 24 2024-08-17T16:54:54.456 INFO:tasks.rados.rados.0.smithi063.stdout:24: seq_num 23 ranges {786561=479786,1796027=467664} 2024-08-17T16:54:54.461 INFO:tasks.rados.rados.0.smithi063.stdout:24: writing smithi06387500-24 from 786561 to 1266347 tid 1 2024-08-17T16:54:54.466 INFO:tasks.rados.rados.0.smithi063.stdout:24: writing smithi06387500-24 from 1796027 to 2263691 tid 2 2024-08-17T16:54:54.467 INFO:tasks.rados.rados.0.smithi063.stdout:25: write initial oid 25 2024-08-17T16:54:54.467 INFO:tasks.rados.rados.0.smithi063.stdout:25: seq_num 24 ranges {403368=555035,1454509=727675} 2024-08-17T16:54:54.472 INFO:tasks.rados.rados.0.smithi063.stdout:25: writing smithi06387500-25 from 403368 to 958403 tid 1 2024-08-17T16:54:54.479 INFO:tasks.rados.rados.0.smithi063.stdout:25: writing smithi06387500-25 from 1454509 to 2182184 tid 2 2024-08-17T16:54:54.479 INFO:tasks.rados.rados.0.smithi063.stdout: waiting on 16 2024-08-17T16:54:54.479 INFO:tasks.rados.rados.0.smithi063.stdout:10: finishing write tid 3 to smithi06387500-10 2024-08-17T16:54:54.479 INFO:tasks.rados.rados.0.smithi063.stdout:10: finishing write tid 4 to smithi06387500-10 2024-08-17T16:54:54.479 INFO:tasks.rados.rados.0.smithi063.stdout:10: oid 10 updating version 0 to 3 2024-08-17T16:54:54.479 INFO:tasks.rados.rados.0.smithi063.stdout:10: oid 10 version 3 is already newer than 2 2024-08-17T16:54:54.480 INFO:tasks.rados.rados.0.smithi063.stdout:10: oid 10 version 3 is already newer than 1 2024-08-17T16:54:54.480 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 3 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-08-17T16:54:54.480 INFO:tasks.rados.rados.0.smithi063.stdout:10: left oid 10 (ObjNum 9 snap 0 seq_num 9) 2024-08-17T16:54:54.480 INFO:tasks.rados.rados.0.smithi063.stdout:11: finishing write tid 1 to smithi06387500-11 2024-08-17T16:54:54.480 INFO:tasks.rados.rados.0.smithi063.stdout:11: finishing write tid 2 to smithi06387500-11 2024-08-17T16:54:54.480 INFO:tasks.rados.rados.0.smithi063.stdout:11: finishing write tid 3 to smithi06387500-11 2024-08-17T16:54:54.480 INFO:tasks.rados.rados.0.smithi063.stdout:11: finishing write tid 4 to smithi06387500-11 2024-08-17T16:54:54.480 INFO:tasks.rados.rados.0.smithi063.stdout:11: oid 11 updating version 0 to 1 2024-08-17T16:54:54.480 INFO:tasks.rados.rados.0.smithi063.stdout:11: oid 11 updating version 1 to 3 2024-08-17T16:54:54.480 INFO:tasks.rados.rados.0.smithi063.stdout:11: oid 11 version 3 is already newer than 2 2024-08-17T16:54:54.480 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 3 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-08-17T16:54:54.480 INFO:tasks.rados.rados.0.smithi063.stdout:11: left oid 11 (ObjNum 10 snap 0 seq_num 10) 2024-08-17T16:54:54.480 INFO:tasks.rados.rados.0.smithi063.stdout:12: finishing write tid 1 to smithi06387500-12 2024-08-17T16:54:54.481 INFO:tasks.rados.rados.0.smithi063.stdout:10: done (15 left) 2024-08-17T16:54:54.481 INFO:tasks.rados.rados.0.smithi063.stdout:11: done (14 left) 2024-08-17T16:54:54.481 INFO:tasks.rados.rados.0.smithi063.stdout:26: write initial oid 26 2024-08-17T16:54:54.481 INFO:tasks.rados.rados.0.smithi063.stdout:26: seq_num 25 ranges {420175=630284,1796638=471333} 2024-08-17T16:54:54.484 INFO:tasks.rados.rados.0.smithi063.stdout:26: writing smithi06387500-26 from 420175 to 1050459 tid 1 2024-08-17T16:54:54.491 INFO:tasks.rados.rados.0.smithi063.stdout:26: writing smithi06387500-26 from 1796638 to 2267971 tid 2 2024-08-17T16:54:54.492 INFO:tasks.rados.rados.0.smithi063.stdout:27: write initial oid 27 2024-08-17T16:54:54.492 INFO:tasks.rados.rados.0.smithi063.stdout:27: seq_num 26 ranges {436982=705533,1855120=581862} 2024-08-17T16:54:54.497 INFO:tasks.rados.rados.0.smithi063.stdout:27: writing smithi06387500-27 from 436982 to 1142515 tid 1 2024-08-17T16:54:54.505 INFO:tasks.rados.rados.0.smithi063.stdout:27: writing smithi06387500-27 from 1855120 to 2436982 tid 2 2024-08-17T16:54:54.505 INFO:tasks.rados.rados.0.smithi063.stdout: waiting on 16 2024-08-17T16:54:54.505 INFO:tasks.rados.rados.0.smithi063.stdout:12: finishing write tid 2 to smithi06387500-12 2024-08-17T16:54:54.505 INFO:tasks.rados.rados.0.smithi063.stdout:12: finishing write tid 3 to smithi06387500-12 2024-08-17T16:54:54.506 INFO:tasks.rados.rados.0.smithi063.stdout:12: finishing write tid 4 to smithi06387500-12 2024-08-17T16:54:54.506 INFO:tasks.rados.rados.0.smithi063.stdout:12: oid 12 updating version 0 to 1 2024-08-17T16:54:54.506 INFO:tasks.rados.rados.0.smithi063.stdout:12: oid 12 updating version 1 to 2 2024-08-17T16:54:54.506 INFO:tasks.rados.rados.0.smithi063.stdout:12: oid 12 updating version 2 to 3 2024-08-17T16:54:54.506 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 3 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-08-17T16:54:54.506 INFO:tasks.rados.rados.0.smithi063.stdout:12: left oid 12 (ObjNum 11 snap 0 seq_num 11) 2024-08-17T16:54:54.506 INFO:tasks.rados.rados.0.smithi063.stdout:13: finishing write tid 1 to smithi06387500-13 2024-08-17T16:54:54.506 INFO:tasks.rados.rados.0.smithi063.stdout:13: finishing write tid 2 to smithi06387500-13 2024-08-17T16:54:54.506 INFO:tasks.rados.rados.0.smithi063.stdout:13: finishing write tid 3 to smithi06387500-13 2024-08-17T16:54:54.506 INFO:tasks.rados.rados.0.smithi063.stdout:13: finishing write tid 4 to smithi06387500-13 2024-08-17T16:54:54.506 INFO:tasks.rados.rados.0.smithi063.stdout:13: oid 13 updating version 0 to 10 2024-08-17T16:54:54.506 INFO:tasks.rados.rados.0.smithi063.stdout:13: oid 13 updating version 10 to 12 2024-08-17T16:54:54.506 INFO:tasks.rados.rados.0.smithi063.stdout:13: oid 13 version 12 is already newer than 11 2024-08-17T16:54:54.507 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 12 (ObjNum 12 snap 0 seq_num 12) dirty exists 2024-08-17T16:54:54.507 INFO:tasks.rados.rados.0.smithi063.stdout:13: left oid 13 (ObjNum 12 snap 0 seq_num 12) 2024-08-17T16:54:54.507 INFO:tasks.rados.rados.0.smithi063.stdout:14: finishing write tid 1 to smithi06387500-14 2024-08-17T16:54:54.507 INFO:tasks.rados.rados.0.smithi063.stdout:12: done (15 left) 2024-08-17T16:54:54.507 INFO:tasks.rados.rados.0.smithi063.stdout:13: done (14 left) 2024-08-17T16:54:54.507 INFO:tasks.rados.rados.0.smithi063.stdout:28: write initial oid 28 2024-08-17T16:54:54.507 INFO:tasks.rados.rados.0.smithi063.stdout:28: seq_num 27 ranges {453789=780782,1913602=475002} 2024-08-17T16:54:54.513 INFO:tasks.rados.rados.0.smithi063.stdout:28: writing smithi06387500-28 from 453789 to 1234571 tid 1 2024-08-17T16:54:54.519 INFO:tasks.rados.rados.0.smithi063.stdout:28: writing smithi06387500-28 from 1913602 to 2388604 tid 2 2024-08-17T16:54:54.519 INFO:tasks.rados.rados.0.smithi063.stdout:29: write initial oid 29 2024-08-17T16:54:54.519 INFO:tasks.rados.rados.0.smithi063.stdout:29: seq_num 28 ranges {470596=456031,1572084=618660} 2024-08-17T16:54:54.524 INFO:tasks.rados.rados.0.smithi063.stdout:29: writing smithi06387500-29 from 470596 to 926627 tid 1 2024-08-17T16:54:54.532 INFO:tasks.rados.rados.0.smithi063.stdout:29: writing smithi06387500-29 from 1572084 to 2190744 tid 2 2024-08-17T16:54:54.533 INFO:tasks.rados.rados.0.smithi063.stdout: waiting on 16 2024-08-17T16:54:54.533 INFO:tasks.rados.rados.0.smithi063.stdout:14: finishing write tid 2 to smithi06387500-14 2024-08-17T16:54:54.533 INFO:tasks.rados.rados.0.smithi063.stdout:14: finishing write tid 3 to smithi06387500-14 2024-08-17T16:54:54.533 INFO:tasks.rados.rados.0.smithi063.stdout:14: finishing write tid 4 to smithi06387500-14 2024-08-17T16:54:54.533 INFO:tasks.rados.rados.0.smithi063.stdout:14: oid 14 updating version 0 to 6 2024-08-17T16:54:54.533 INFO:tasks.rados.rados.0.smithi063.stdout:14: oid 14 version 6 is already newer than 5 2024-08-17T16:54:54.533 INFO:tasks.rados.rados.0.smithi063.stdout:14: oid 14 version 6 is already newer than 4 2024-08-17T16:54:54.533 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 6 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-08-17T16:54:54.533 INFO:tasks.rados.rados.0.smithi063.stdout:14: left oid 14 (ObjNum 13 snap 0 seq_num 13) 2024-08-17T16:54:54.533 INFO:tasks.rados.rados.0.smithi063.stdout:15: finishing write tid 1 to smithi06387500-15 2024-08-17T16:54:54.533 INFO:tasks.rados.rados.0.smithi063.stdout:15: finishing write tid 2 to smithi06387500-15 2024-08-17T16:54:54.533 INFO:tasks.rados.rados.0.smithi063.stdout:15: finishing write tid 3 to smithi06387500-15 2024-08-17T16:54:54.534 INFO:tasks.rados.rados.0.smithi063.stdout:14: done (15 left) 2024-08-17T16:54:54.534 INFO:tasks.rados.rados.0.smithi063.stdout:30: write initial oid 30 2024-08-17T16:54:54.534 INFO:tasks.rados.rados.0.smithi063.stdout:30: seq_num 29 ranges {487403=531280,1514213=478671} 2024-08-17T16:54:54.538 INFO:tasks.rados.rados.0.smithi063.stdout:30: writing smithi06387500-30 from 487403 to 1018683 tid 1 2024-08-17T16:54:54.543 INFO:tasks.rados.rados.0.smithi063.stdout:30: writing smithi06387500-30 from 1514213 to 1992884 tid 2 2024-08-17T16:54:54.543 INFO:tasks.rados.rados.0.smithi063.stdout: waiting on 16 2024-08-17T16:54:54.543 INFO:tasks.rados.rados.0.smithi063.stdout:15: finishing write tid 4 to smithi06387500-15 2024-08-17T16:54:54.544 INFO:tasks.rados.rados.0.smithi063.stdout:15: oid 15 updating version 0 to 1 2024-08-17T16:54:54.544 INFO:tasks.rados.rados.0.smithi063.stdout:15: oid 15 updating version 1 to 2 2024-08-17T16:54:54.544 INFO:tasks.rados.rados.0.smithi063.stdout:15: oid 15 updating version 2 to 3 2024-08-17T16:54:54.544 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 3 (ObjNum 14 snap 0 seq_num 14) dirty exists 2024-08-17T16:54:54.544 INFO:tasks.rados.rados.0.smithi063.stdout:15: left oid 15 (ObjNum 14 snap 0 seq_num 14) 2024-08-17T16:54:54.544 INFO:tasks.rados.rados.0.smithi063.stdout:15: done (15 left) 2024-08-17T16:54:54.544 INFO:tasks.rados.rados.0.smithi063.stdout:31: write initial oid 31 2024-08-17T16:54:54.544 INFO:tasks.rados.rados.0.smithi063.stdout:31: seq_num 30 ranges {504210=606529,1572695=622329} 2024-08-17T16:54:54.549 INFO:tasks.rados.rados.0.smithi063.stdout:31: writing smithi06387500-31 from 504210 to 1110739 tid 1 2024-08-17T16:54:54.556 INFO:tasks.rados.rados.0.smithi063.stdout:31: writing smithi06387500-31 from 1572695 to 2195024 tid 2 2024-08-17T16:54:54.556 INFO:tasks.rados.rados.0.smithi063.stdout: waiting on 16 2024-08-17T16:54:54.556 INFO:tasks.rados.rados.0.smithi063.stdout:16: finishing write tid 1 to smithi06387500-16 2024-08-17T16:54:54.556 INFO:tasks.rados.rados.0.smithi063.stdout:16: finishing write tid 2 to smithi06387500-16 2024-08-17T16:54:54.556 INFO:tasks.rados.rados.0.smithi063.stdout:16: finishing write tid 3 to smithi06387500-16 2024-08-17T16:54:54.556 INFO:tasks.rados.rados.0.smithi063.stdout:16: finishing write tid 4 to smithi06387500-16 2024-08-17T16:54:54.556 INFO:tasks.rados.rados.0.smithi063.stdout:16: oid 16 updating version 0 to 4 2024-08-17T16:54:54.556 INFO:tasks.rados.rados.0.smithi063.stdout:16: oid 16 updating version 4 to 5 2024-08-17T16:54:54.556 INFO:tasks.rados.rados.0.smithi063.stdout:16: oid 16 updating version 5 to 6 2024-08-17T16:54:54.556 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 6 (ObjNum 15 snap 0 seq_num 15) dirty exists 2024-08-17T16:54:54.556 INFO:tasks.rados.rados.0.smithi063.stdout:16: left oid 16 (ObjNum 15 snap 0 seq_num 15) 2024-08-17T16:54:54.556 INFO:tasks.rados.rados.0.smithi063.stdout:17: finishing write tid 1 to smithi06387500-17 2024-08-17T16:54:54.557 INFO:tasks.rados.rados.0.smithi063.stdout:16: done (15 left) 2024-08-17T16:54:54.557 INFO:tasks.rados.rados.0.smithi063.stdout:32: write initial oid 32 2024-08-17T16:54:54.557 INFO:tasks.rados.rados.0.smithi063.stdout:32: seq_num 31 ranges {521017=798131,1747530=482340} 2024-08-17T16:54:54.562 INFO:tasks.rados.rados.0.smithi063.stdout:32: writing smithi06387500-32 from 521017 to 1319148 tid 1 2024-08-17T16:54:54.568 INFO:tasks.rados.rados.0.smithi063.stdout:32: writing smithi06387500-32 from 1747530 to 2229870 tid 2 2024-08-17T16:54:54.568 INFO:tasks.rados.rados.0.smithi063.stdout: waiting on 16 2024-08-17T16:54:54.568 INFO:tasks.rados.rados.0.smithi063.stdout:17: finishing write tid 2 to smithi06387500-17 2024-08-17T16:54:54.569 INFO:tasks.rados.rados.0.smithi063.stdout:17: finishing write tid 3 to smithi06387500-17 2024-08-17T16:54:54.569 INFO:tasks.rados.rados.0.smithi063.stdout:17: finishing write tid 4 to smithi06387500-17 2024-08-17T16:54:54.569 INFO:tasks.rados.rados.0.smithi063.stdout:17: oid 17 updating version 0 to 4 2024-08-17T16:54:54.569 INFO:tasks.rados.rados.0.smithi063.stdout:17: oid 17 updating version 4 to 6 2024-08-17T16:54:54.569 INFO:tasks.rados.rados.0.smithi063.stdout:17: oid 17 version 6 is already newer than 5 2024-08-17T16:54:54.569 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 6 (ObjNum 16 snap 0 seq_num 16) dirty exists 2024-08-17T16:54:54.569 INFO:tasks.rados.rados.0.smithi063.stdout:17: left oid 17 (ObjNum 16 snap 0 seq_num 16) 2024-08-17T16:54:54.569 INFO:tasks.rados.rados.0.smithi063.stdout:18: finishing write tid 1 to smithi06387500-18 2024-08-17T16:54:54.569 INFO:tasks.rados.rados.0.smithi063.stdout:18: finishing write tid 2 to smithi06387500-18 2024-08-17T16:54:54.569 INFO:tasks.rados.rados.0.smithi063.stdout:18: finishing write tid 3 to smithi06387500-18 2024-08-17T16:54:54.569 INFO:tasks.rados.rados.0.smithi063.stdout:18: finishing write tid 4 to smithi06387500-18 2024-08-17T16:54:54.569 INFO:tasks.rados.rados.0.smithi063.stdout:18: oid 18 updating version 0 to 5 2024-08-17T16:54:54.570 INFO:tasks.rados.rados.0.smithi063.stdout:18: oid 18 version 5 is already newer than 4 2024-08-17T16:54:54.570 INFO:tasks.rados.rados.0.smithi063.stdout:18: oid 18 updating version 5 to 6 2024-08-17T16:54:54.570 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 6 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-08-17T16:54:54.570 INFO:tasks.rados.rados.0.smithi063.stdout:18: left oid 18 (ObjNum 17 snap 0 seq_num 17) 2024-08-17T16:54:54.570 INFO:tasks.rados.rados.0.smithi063.stdout:19: finishing write tid 1 to smithi06387500-19 2024-08-17T16:54:54.570 INFO:tasks.rados.rados.0.smithi063.stdout:19: finishing write tid 2 to smithi06387500-19 2024-08-17T16:54:54.570 INFO:tasks.rados.rados.0.smithi063.stdout:19: finishing write tid 3 to smithi06387500-19 2024-08-17T16:54:54.570 INFO:tasks.rados.rados.0.smithi063.stdout:19: finishing write tid 4 to smithi06387500-19 2024-08-17T16:54:54.570 INFO:tasks.rados.rados.0.smithi063.stdout:19: oid 19 updating version 0 to 3 2024-08-17T16:54:54.570 INFO:tasks.rados.rados.0.smithi063.stdout:19: oid 19 version 3 is already newer than 2 2024-08-17T16:54:54.570 INFO:tasks.rados.rados.0.smithi063.stdout:19: oid 19 version 3 is already newer than 1 2024-08-17T16:54:54.570 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 3 (ObjNum 18 snap 0 seq_num 18) dirty exists 2024-08-17T16:54:54.570 INFO:tasks.rados.rados.0.smithi063.stdout:19: left oid 19 (ObjNum 18 snap 0 seq_num 18) 2024-08-17T16:54:54.571 INFO:tasks.rados.rados.0.smithi063.stdout:20: finishing write tid 1 to smithi06387500-20 2024-08-17T16:54:54.571 INFO:tasks.rados.rados.0.smithi063.stdout:17: done (15 left) 2024-08-17T16:54:54.571 INFO:tasks.rados.rados.0.smithi063.stdout:18: done (14 left) 2024-08-17T16:54:54.571 INFO:tasks.rados.rados.0.smithi063.stdout:19: done (13 left) 2024-08-17T16:54:54.571 INFO:tasks.rados.rados.0.smithi063.stdout:33: write initial oid 33 2024-08-17T16:54:54.571 INFO:tasks.rados.rados.0.smithi063.stdout:33: seq_num 32 ranges {537824=473380,1806012=625998} 2024-08-17T16:54:54.573 INFO:tasks.rados.rados.0.smithi063.stdout:33: writing smithi06387500-33 from 537824 to 1011204 tid 1 2024-08-17T16:54:54.581 INFO:tasks.rados.rados.0.smithi063.stdout:33: writing smithi06387500-33 from 1806012 to 2432010 tid 2 2024-08-17T16:54:54.581 INFO:tasks.rados.rados.0.smithi063.stdout:34: write initial oid 34 2024-08-17T16:54:54.581 INFO:tasks.rados.rados.0.smithi063.stdout:34: seq_num 33 ranges {554631=548629,1748141=486009} 2024-08-17T16:54:54.587 INFO:tasks.rados.rados.0.smithi063.stdout:34: writing smithi06387500-34 from 554631 to 1103260 tid 1 2024-08-17T16:54:54.593 INFO:tasks.rados.rados.0.smithi063.stdout:34: writing smithi06387500-34 from 1748141 to 2234150 tid 2 2024-08-17T16:54:54.594 INFO:tasks.rados.rados.0.smithi063.stdout:35: write initial oid 35 2024-08-17T16:54:54.594 INFO:tasks.rados.rados.0.smithi063.stdout:35: seq_num 34 ranges {571438=623878,1806623=629667} 2024-08-17T16:54:54.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:54 smithi162 ceph-mon[34633]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T16:54:54.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:54 smithi162 ceph-mon[34633]: Cluster is now healthy 2024-08-17T16:54:54.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:54 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/1078629904' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-08-17T16:54:54.600 INFO:tasks.rados.rados.0.smithi063.stdout:35: writing smithi06387500-35 from 571438 to 1195316 tid 1 2024-08-17T16:54:54.608 INFO:tasks.rados.rados.0.smithi063.stdout:35: writing smithi06387500-35 from 1806623 to 2436290 tid 2 2024-08-17T16:54:54.608 INFO:tasks.rados.rados.0.smithi063.stdout: waiting on 16 2024-08-17T16:54:54.608 INFO:tasks.rados.rados.0.smithi063.stdout:20: finishing write tid 2 to smithi06387500-20 2024-08-17T16:54:54.608 INFO:tasks.rados.rados.0.smithi063.stdout:20: finishing write tid 3 to smithi06387500-20 2024-08-17T16:54:54.608 INFO:tasks.rados.rados.0.smithi063.stdout:20: finishing write tid 4 to smithi06387500-20 2024-08-17T16:54:54.608 INFO:tasks.rados.rados.0.smithi063.stdout:20: oid 20 updating version 0 to 3 2024-08-17T16:54:54.608 INFO:tasks.rados.rados.0.smithi063.stdout:20: oid 20 version 3 is already newer than 2 2024-08-17T16:54:54.608 INFO:tasks.rados.rados.0.smithi063.stdout:20: oid 20 version 3 is already newer than 1 2024-08-17T16:54:54.608 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 3 (ObjNum 19 snap 0 seq_num 19) dirty exists 2024-08-17T16:54:54.608 INFO:tasks.rados.rados.0.smithi063.stdout:20: left oid 20 (ObjNum 19 snap 0 seq_num 19) 2024-08-17T16:54:54.609 INFO:tasks.rados.rados.0.smithi063.stdout:21: finishing write tid 1 to smithi06387500-21 2024-08-17T16:54:54.609 INFO:tasks.rados.rados.0.smithi063.stdout:21: finishing write tid 2 to smithi06387500-21 2024-08-17T16:54:54.609 INFO:tasks.rados.rados.0.smithi063.stdout:21: finishing write tid 3 to smithi06387500-21 2024-08-17T16:54:54.609 INFO:tasks.rados.rados.0.smithi063.stdout:21: finishing write tid 4 to smithi06387500-21 2024-08-17T16:54:54.609 INFO:tasks.rados.rados.0.smithi063.stdout:21: oid 21 updating version 0 to 4 2024-08-17T16:54:54.609 INFO:tasks.rados.rados.0.smithi063.stdout:21: oid 21 updating version 4 to 6 2024-08-17T16:54:54.609 INFO:tasks.rados.rados.0.smithi063.stdout:21: oid 21 version 6 is already newer than 5 2024-08-17T16:54:54.609 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 6 (ObjNum 20 snap 0 seq_num 20) dirty exists 2024-08-17T16:54:54.609 INFO:tasks.rados.rados.0.smithi063.stdout:21: left oid 21 (ObjNum 20 snap 0 seq_num 20) 2024-08-17T16:54:54.609 INFO:tasks.rados.rados.0.smithi063.stdout:22: finishing write tid 1 to smithi06387500-22 2024-08-17T16:54:54.609 INFO:tasks.rados.rados.0.smithi063.stdout:22: finishing write tid 2 to smithi06387500-22 2024-08-17T16:54:54.609 INFO:tasks.rados.rados.0.smithi063.stdout:22: finishing write tid 3 to smithi06387500-22 2024-08-17T16:54:54.609 INFO:tasks.rados.rados.0.smithi063.stdout:22: finishing write tid 4 to smithi06387500-22 2024-08-17T16:54:54.609 INFO:tasks.rados.rados.0.smithi063.stdout:22: oid 22 updating version 0 to 5 2024-08-17T16:54:54.609 INFO:tasks.rados.rados.0.smithi063.stdout:22: oid 22 version 5 is already newer than 4 2024-08-17T16:54:54.610 INFO:tasks.rados.rados.0.smithi063.stdout:22: oid 22 updating version 5 to 6 2024-08-17T16:54:54.610 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 6 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-08-17T16:54:54.610 INFO:tasks.rados.rados.0.smithi063.stdout:22: left oid 22 (ObjNum 21 snap 0 seq_num 21) 2024-08-17T16:54:54.610 INFO:tasks.rados.rados.0.smithi063.stdout:23: finishing write tid 1 to smithi06387500-23 2024-08-17T16:54:54.610 INFO:tasks.rados.rados.0.smithi063.stdout:23: finishing write tid 2 to smithi06387500-23 2024-08-17T16:54:54.610 INFO:tasks.rados.rados.0.smithi063.stdout:23: finishing write tid 3 to smithi06387500-23 2024-08-17T16:54:54.610 INFO:tasks.rados.rados.0.smithi063.stdout:23: finishing write tid 4 to smithi06387500-23 2024-08-17T16:54:54.610 INFO:tasks.rados.rados.0.smithi063.stdout:23: oid 23 updating version 0 to 14 2024-08-17T16:54:54.610 INFO:tasks.rados.rados.0.smithi063.stdout:23: oid 23 version 14 is already newer than 13 2024-08-17T16:54:54.610 INFO:tasks.rados.rados.0.smithi063.stdout:23: oid 23 updating version 14 to 15 2024-08-17T16:54:54.610 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 15 (ObjNum 22 snap 0 seq_num 22) dirty exists 2024-08-17T16:54:54.610 INFO:tasks.rados.rados.0.smithi063.stdout:23: left oid 23 (ObjNum 22 snap 0 seq_num 22) 2024-08-17T16:54:54.610 INFO:tasks.rados.rados.0.smithi063.stdout:24: finishing write tid 1 to smithi06387500-24 2024-08-17T16:54:54.610 INFO:tasks.rados.rados.0.smithi063.stdout:24: finishing write tid 2 to smithi06387500-24 2024-08-17T16:54:54.611 INFO:tasks.rados.rados.0.smithi063.stdout:20: done (15 left) 2024-08-17T16:54:54.611 INFO:tasks.rados.rados.0.smithi063.stdout:21: done (14 left) 2024-08-17T16:54:54.611 INFO:tasks.rados.rados.0.smithi063.stdout:22: done (13 left) 2024-08-17T16:54:54.611 INFO:tasks.rados.rados.0.smithi063.stdout:23: done (12 left) 2024-08-17T16:54:54.611 INFO:tasks.rados.rados.0.smithi063.stdout:36: write initial oid 36 2024-08-17T16:54:54.611 INFO:tasks.rados.rados.0.smithi063.stdout:36: seq_num 35 ranges {588245=699127,1865105=489678} 2024-08-17T16:54:54.614 INFO:tasks.rados.rados.0.smithi063.stdout:36: writing smithi06387500-36 from 588245 to 1287372 tid 1 2024-08-17T16:54:54.620 INFO:tasks.rados.rados.0.smithi063.stdout:36: writing smithi06387500-36 from 1865105 to 2354783 tid 2 2024-08-17T16:54:54.620 INFO:tasks.rados.rados.0.smithi063.stdout:37: write initial oid 37 2024-08-17T16:54:54.620 INFO:tasks.rados.rados.0.smithi063.stdout:37: seq_num 36 ranges {605052=774376,1923587=633336} 2024-08-17T16:54:54.627 INFO:tasks.rados.rados.0.smithi063.stdout:37: writing smithi06387500-37 from 605052 to 1379428 tid 1 2024-08-17T16:54:54.633 INFO:tasks.rados.rados.0.smithi063.stdout:37: writing smithi06387500-37 from 1923587 to 2556923 tid 2 2024-08-17T16:54:54.633 INFO:tasks.rados.rados.0.smithi063.stdout:38: write initial oid 38 2024-08-17T16:54:54.634 INFO:tasks.rados.rados.0.smithi063.stdout:38: seq_num 37 ranges {621859=449625,1865716=756143} 2024-08-17T16:54:54.639 INFO:tasks.rados.rados.0.smithi063.stdout:38: writing smithi06387500-38 from 621859 to 1071484 tid 1 2024-08-17T16:54:54.648 INFO:tasks.rados.rados.0.smithi063.stdout:38: writing smithi06387500-38 from 1865716 to 2621859 tid 2 2024-08-17T16:54:54.648 INFO:tasks.rados.rados.0.smithi063.stdout:39: write initial oid 39 2024-08-17T16:54:54.648 INFO:tasks.rados.rados.0.smithi063.stdout:39: seq_num 38 ranges {638666=524874,1924198=637005} 2024-08-17T16:54:54.654 INFO:tasks.rados.rados.0.smithi063.stdout:39: writing smithi06387500-39 from 638666 to 1163540 tid 1 2024-08-17T16:54:54.662 INFO:tasks.rados.rados.0.smithi063.stdout:39: writing smithi06387500-39 from 1924198 to 2561203 tid 2 2024-08-17T16:54:54.663 INFO:tasks.rados.rados.0.smithi063.stdout: waiting on 16 2024-08-17T16:54:54.663 INFO:tasks.rados.rados.0.smithi063.stdout:24: finishing write tid 3 to smithi06387500-24 2024-08-17T16:54:54.663 INFO:tasks.rados.rados.0.smithi063.stdout:24: finishing write tid 4 to smithi06387500-24 2024-08-17T16:54:54.663 INFO:tasks.rados.rados.0.smithi063.stdout:24: oid 24 updating version 0 to 8 2024-08-17T16:54:54.663 INFO:tasks.rados.rados.0.smithi063.stdout:24: oid 24 version 8 is already newer than 7 2024-08-17T16:54:54.663 INFO:tasks.rados.rados.0.smithi063.stdout:24: oid 24 updating version 8 to 9 2024-08-17T16:54:54.663 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 9 (ObjNum 23 snap 0 seq_num 23) dirty exists 2024-08-17T16:54:54.663 INFO:tasks.rados.rados.0.smithi063.stdout:24: left oid 24 (ObjNum 23 snap 0 seq_num 23) 2024-08-17T16:54:54.663 INFO:tasks.rados.rados.0.smithi063.stdout:25: finishing write tid 1 to smithi06387500-25 2024-08-17T16:54:54.663 INFO:tasks.rados.rados.0.smithi063.stdout:25: finishing write tid 2 to smithi06387500-25 2024-08-17T16:54:54.663 INFO:tasks.rados.rados.0.smithi063.stdout:25: finishing write tid 3 to smithi06387500-25 2024-08-17T16:54:54.663 INFO:tasks.rados.rados.0.smithi063.stdout:25: finishing write tid 4 to smithi06387500-25 2024-08-17T16:54:54.663 INFO:tasks.rados.rados.0.smithi063.stdout:25: oid 25 updating version 0 to 8 2024-08-17T16:54:54.664 INFO:tasks.rados.rados.0.smithi063.stdout:25: oid 25 updating version 8 to 9 2024-08-17T16:54:54.664 INFO:tasks.rados.rados.0.smithi063.stdout:25: oid 25 version 9 is already newer than 7 2024-08-17T16:54:54.664 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 9 (ObjNum 24 snap 0 seq_num 24) dirty exists 2024-08-17T16:54:54.664 INFO:tasks.rados.rados.0.smithi063.stdout:25: left oid 25 (ObjNum 24 snap 0 seq_num 24) 2024-08-17T16:54:54.664 INFO:tasks.rados.rados.0.smithi063.stdout:26: finishing write tid 1 to smithi06387500-26 2024-08-17T16:54:54.664 INFO:tasks.rados.rados.0.smithi063.stdout:26: finishing write tid 2 to smithi06387500-26 2024-08-17T16:54:54.664 INFO:tasks.rados.rados.0.smithi063.stdout:26: finishing write tid 3 to smithi06387500-26 2024-08-17T16:54:54.664 INFO:tasks.rados.rados.0.smithi063.stdout:26: finishing write tid 4 to smithi06387500-26 2024-08-17T16:54:54.664 INFO:tasks.rados.rados.0.smithi063.stdout:26: oid 26 updating version 0 to 8 2024-08-17T16:54:54.664 INFO:tasks.rados.rados.0.smithi063.stdout:26: oid 26 version 8 is already newer than 7 2024-08-17T16:54:54.664 INFO:tasks.rados.rados.0.smithi063.stdout:26: oid 26 updating version 8 to 9 2024-08-17T16:54:54.664 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 9 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-08-17T16:54:54.664 INFO:tasks.rados.rados.0.smithi063.stdout:26: left oid 26 (ObjNum 25 snap 0 seq_num 25) 2024-08-17T16:54:54.665 INFO:tasks.rados.rados.0.smithi063.stdout:27: finishing write tid 1 to smithi06387500-27 2024-08-17T16:54:54.665 INFO:tasks.rados.rados.0.smithi063.stdout:27: finishing write tid 2 to smithi06387500-27 2024-08-17T16:54:54.665 INFO:tasks.rados.rados.0.smithi063.stdout:24: done (15 left) 2024-08-17T16:54:54.665 INFO:tasks.rados.rados.0.smithi063.stdout:25: done (14 left) 2024-08-17T16:54:54.665 INFO:tasks.rados.rados.0.smithi063.stdout:26: done (13 left) 2024-08-17T16:54:54.665 INFO:tasks.rados.rados.0.smithi063.stdout:40: write initial oid 40 2024-08-17T16:54:54.665 INFO:tasks.rados.rados.0.smithi063.stdout:40: seq_num 39 ranges {655473=716476,2099033=556440} 2024-08-17T16:54:54.669 INFO:tasks.rados.rados.0.smithi063.stdout:40: writing smithi06387500-40 from 655473 to 1371949 tid 1 2024-08-17T16:54:54.677 INFO:tasks.rados.rados.0.smithi063.stdout:40: writing smithi06387500-40 from 2099033 to 2655473 tid 2 2024-08-17T16:54:54.677 INFO:tasks.rados.rados.0.smithi063.stdout:41: write initial oid 41 2024-08-17T16:54:54.677 INFO:tasks.rados.rados.0.smithi063.stdout:27: finishing write tid 3 to smithi06387500-27 2024-08-17T16:54:54.677 INFO:tasks.rados.rados.0.smithi063.stdout:27: finishing write tid 4 to smithi06387500-27 2024-08-17T16:54:54.677 INFO:tasks.rados.rados.0.smithi063.stdout:27: oid 27 updating version 0 to 3 2024-08-17T16:54:54.677 INFO:tasks.rados.rados.0.smithi063.stdout:27: oid 27 version 3 is already newer than 2 2024-08-17T16:54:54.678 INFO:tasks.rados.rados.0.smithi063.stdout:27: oid 27 version 3 is already newer than 1 2024-08-17T16:54:54.678 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 3 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-08-17T16:54:54.678 INFO:tasks.rados.rados.0.smithi063.stdout:27: left oid 27 (ObjNum 26 snap 0 seq_num 26) 2024-08-17T16:54:54.678 INFO:tasks.rados.rados.0.smithi063.stdout:28: finishing write tid 1 to smithi06387500-28 2024-08-17T16:54:54.678 INFO:tasks.rados.rados.0.smithi063.stdout:28: finishing write tid 2 to smithi06387500-28 2024-08-17T16:54:54.678 INFO:tasks.rados.rados.0.smithi063.stdout:28: finishing write tid 3 to smithi06387500-28 2024-08-17T16:54:54.678 INFO:tasks.rados.rados.0.smithi063.stdout:41: seq_num 40 ranges {672280=791725,2157515=514765} 2024-08-17T16:54:54.685 INFO:tasks.rados.rados.0.smithi063.stdout:41: writing smithi06387500-41 from 672280 to 1464005 tid 1 2024-08-17T16:54:54.691 INFO:tasks.rados.rados.0.smithi063.stdout:41: writing smithi06387500-41 from 2157515 to 2672280 tid 2 2024-08-17T16:54:54.692 INFO:tasks.rados.rados.0.smithi063.stdout:27: done (14 left) 2024-08-17T16:54:54.692 INFO:tasks.rados.rados.0.smithi063.stdout:42: write initial oid 42 2024-08-17T16:54:54.692 INFO:tasks.rados.rados.0.smithi063.stdout:42: seq_num 41 ranges {689087=466974,1699644=784332} 2024-08-17T16:54:54.698 INFO:tasks.rados.rados.0.smithi063.stdout:42: writing smithi06387500-42 from 689087 to 1156061 tid 1 2024-08-17T16:54:54.705 INFO:tasks.rados.rados.0.smithi063.stdout:42: writing smithi06387500-42 from 1699644 to 2483976 tid 2 2024-08-17T16:54:54.705 INFO:tasks.rados.rados.0.smithi063.stdout:43: write initial oid 43 2024-08-17T16:54:54.705 INFO:tasks.rados.rados.0.smithi063.stdout:43: seq_num 42 ranges {705894=542223,1758126=644343} 2024-08-17T16:54:54.712 INFO:tasks.rados.rados.0.smithi063.stdout:43: writing smithi06387500-43 from 705894 to 1248117 tid 1 2024-08-17T16:54:54.719 INFO:tasks.rados.rados.0.smithi063.stdout:43: writing smithi06387500-43 from 1758126 to 2402469 tid 2 2024-08-17T16:54:54.719 INFO:tasks.rados.rados.0.smithi063.stdout: waiting on 16 2024-08-17T16:54:54.719 INFO:tasks.rados.rados.0.smithi063.stdout:28: finishing write tid 4 to smithi06387500-28 2024-08-17T16:54:54.719 INFO:tasks.rados.rados.0.smithi063.stdout:28: oid 28 updating version 0 to 7 2024-08-17T16:54:54.719 INFO:tasks.rados.rados.0.smithi063.stdout:28: oid 28 updating version 7 to 9 2024-08-17T16:54:54.719 INFO:tasks.rados.rados.0.smithi063.stdout:28: oid 28 version 9 is already newer than 8 2024-08-17T16:54:54.719 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 9 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-08-17T16:54:54.719 INFO:tasks.rados.rados.0.smithi063.stdout:28: left oid 28 (ObjNum 27 snap 0 seq_num 27) 2024-08-17T16:54:54.719 INFO:tasks.rados.rados.0.smithi063.stdout:29: finishing write tid 1 to smithi06387500-29 2024-08-17T16:54:54.719 INFO:tasks.rados.rados.0.smithi063.stdout:29: finishing write tid 2 to smithi06387500-29 2024-08-17T16:54:54.720 INFO:tasks.rados.rados.0.smithi063.stdout:29: finishing write tid 3 to smithi06387500-29 2024-08-17T16:54:54.720 INFO:tasks.rados.rados.0.smithi063.stdout:28: done (15 left) 2024-08-17T16:54:54.720 INFO:tasks.rados.rados.0.smithi063.stdout:44: write initial oid 44 2024-08-17T16:54:54.720 INFO:tasks.rados.rados.0.smithi063.stdout:44: seq_num 43 ranges {722701=617472,1816608=788001} 2024-08-17T16:54:54.726 INFO:tasks.rados.rados.0.smithi063.stdout:44: writing smithi06387500-44 from 722701 to 1340173 tid 1 2024-08-17T16:54:54.733 INFO:tasks.rados.rados.0.smithi063.stdout:44: writing smithi06387500-44 from 1816608 to 2604609 tid 2 2024-08-17T16:54:54.733 INFO:tasks.rados.rados.0.smithi063.stdout: waiting on 16 2024-08-17T16:54:54.733 INFO:tasks.rados.rados.0.smithi063.stdout:29: finishing write tid 4 to smithi06387500-29 2024-08-17T16:54:54.733 INFO:tasks.rados.rados.0.smithi063.stdout:29: oid 29 updating version 0 to 4 2024-08-17T16:54:54.733 INFO:tasks.rados.rados.0.smithi063.stdout:29: oid 29 updating version 4 to 6 2024-08-17T16:54:54.733 INFO:tasks.rados.rados.0.smithi063.stdout:29: oid 29 version 6 is already newer than 5 2024-08-17T16:54:54.734 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 6 (ObjNum 28 snap 0 seq_num 28) dirty exists 2024-08-17T16:54:54.734 INFO:tasks.rados.rados.0.smithi063.stdout:29: left oid 29 (ObjNum 28 snap 0 seq_num 28) 2024-08-17T16:54:54.734 INFO:tasks.rados.rados.0.smithi063.stdout:30: finishing write tid 1 to smithi06387500-30 2024-08-17T16:54:54.734 INFO:tasks.rados.rados.0.smithi063.stdout:30: finishing write tid 2 to smithi06387500-30 2024-08-17T16:54:54.734 INFO:tasks.rados.rados.0.smithi063.stdout:30: finishing write tid 3 to smithi06387500-30 2024-08-17T16:54:54.734 INFO:tasks.rados.rados.0.smithi063.stdout:30: finishing write tid 4 to smithi06387500-30 2024-08-17T16:54:54.734 INFO:tasks.rados.rados.0.smithi063.stdout:30: oid 30 updating version 0 to 9 2024-08-17T16:54:54.734 INFO:tasks.rados.rados.0.smithi063.stdout:30: oid 30 version 9 is already newer than 7 2024-08-17T16:54:54.734 INFO:tasks.rados.rados.0.smithi063.stdout:30: oid 30 version 9 is already newer than 8 2024-08-17T16:54:54.734 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 9 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-08-17T16:54:54.734 INFO:tasks.rados.rados.0.smithi063.stdout:30: left oid 30 (ObjNum 29 snap 0 seq_num 29) 2024-08-17T16:54:54.734 INFO:tasks.rados.rados.0.smithi063.stdout:31: finishing write tid 1 to smithi06387500-31 2024-08-17T16:54:54.735 INFO:tasks.rados.rados.0.smithi063.stdout:31: finishing write tid 2 to smithi06387500-31 2024-08-17T16:54:54.735 INFO:tasks.rados.rados.0.smithi063.stdout:31: finishing write tid 3 to smithi06387500-31 2024-08-17T16:54:54.735 INFO:tasks.rados.rados.0.smithi063.stdout:31: finishing write tid 4 to smithi06387500-31 2024-08-17T16:54:54.735 INFO:tasks.rados.rados.0.smithi063.stdout:31: oid 31 updating version 0 to 8 2024-08-17T16:54:54.735 INFO:tasks.rados.rados.0.smithi063.stdout:31: oid 31 version 8 is already newer than 7 2024-08-17T16:54:54.735 INFO:tasks.rados.rados.0.smithi063.stdout:31: oid 31 updating version 8 to 9 2024-08-17T16:54:54.735 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 9 (ObjNum 30 snap 0 seq_num 30) dirty exists 2024-08-17T16:54:54.735 INFO:tasks.rados.rados.0.smithi063.stdout:31: left oid 31 (ObjNum 30 snap 0 seq_num 30) 2024-08-17T16:54:54.735 INFO:tasks.rados.rados.0.smithi063.stdout:29: done (15 left) 2024-08-17T16:54:54.735 INFO:tasks.rados.rados.0.smithi063.stdout:30: done (14 left) 2024-08-17T16:54:54.735 INFO:tasks.rados.rados.0.smithi063.stdout:31: done (13 left) 2024-08-17T16:54:54.735 INFO:tasks.rados.rados.0.smithi063.stdout:45: write initial oid 45 2024-08-17T16:54:54.735 INFO:tasks.rados.rados.0.smithi063.stdout:45: seq_num 44 ranges {739508=692721,1875090=648012} 2024-08-17T16:54:54.740 INFO:tasks.rados.rados.0.smithi063.stdout:45: writing smithi06387500-45 from 739508 to 1432229 tid 1 2024-08-17T16:54:54.746 INFO:tasks.rados.rados.0.smithi063.stdout:45: writing smithi06387500-45 from 1875090 to 2523102 tid 2 2024-08-17T16:54:54.747 INFO:tasks.rados.rados.0.smithi063.stdout:46: write initial oid 46 2024-08-17T16:54:54.747 INFO:tasks.rados.rados.0.smithi063.stdout:46: seq_num 45 ranges {756315=767970,1933572=791670} 2024-08-17T16:54:54.755 INFO:tasks.rados.rados.0.smithi063.stdout:46: writing smithi06387500-46 from 756315 to 1524285 tid 1 2024-08-17T16:54:54.761 INFO:tasks.rados.rados.0.smithi063.stdout:46: writing smithi06387500-46 from 1933572 to 2725242 tid 2 2024-08-17T16:54:54.762 INFO:tasks.rados.rados.0.smithi063.stdout:47: write initial oid 47 2024-08-17T16:54:54.762 INFO:tasks.rados.rados.0.smithi063.stdout:47: seq_num 46 ranges {773122=559572,1992054=651681} 2024-08-17T16:54:54.769 INFO:tasks.rados.rados.0.smithi063.stdout:47: writing smithi06387500-47 from 773122 to 1332694 tid 1 2024-08-17T16:54:54.777 INFO:tasks.rados.rados.0.smithi063.stdout:47: writing smithi06387500-47 from 1992054 to 2643735 tid 2 2024-08-17T16:54:54.777 INFO:tasks.rados.rados.0.smithi063.stdout: waiting on 16 2024-08-17T16:54:54.777 INFO:tasks.rados.rados.0.smithi063.stdout:32: finishing write tid 1 to smithi06387500-32 2024-08-17T16:54:54.777 INFO:tasks.rados.rados.0.smithi063.stdout:32: finishing write tid 2 to smithi06387500-32 2024-08-17T16:54:54.777 INFO:tasks.rados.rados.0.smithi063.stdout:32: finishing write tid 3 to smithi06387500-32 2024-08-17T16:54:54.777 INFO:tasks.rados.rados.0.smithi063.stdout:32: finishing write tid 4 to smithi06387500-32 2024-08-17T16:54:54.777 INFO:tasks.rados.rados.0.smithi063.stdout:32: oid 32 updating version 0 to 1 2024-08-17T16:54:54.777 INFO:tasks.rados.rados.0.smithi063.stdout:32: oid 32 updating version 1 to 3 2024-08-17T16:54:54.777 INFO:tasks.rados.rados.0.smithi063.stdout:32: oid 32 version 3 is already newer than 2 2024-08-17T16:54:54.777 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 3 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-08-17T16:54:54.777 INFO:tasks.rados.rados.0.smithi063.stdout:32: left oid 32 (ObjNum 31 snap 0 seq_num 31) 2024-08-17T16:54:54.778 INFO:tasks.rados.rados.0.smithi063.stdout:33: finishing write tid 1 to smithi06387500-33 2024-08-17T16:54:54.778 INFO:tasks.rados.rados.0.smithi063.stdout:33: finishing write tid 2 to smithi06387500-33 2024-08-17T16:54:54.778 INFO:tasks.rados.rados.0.smithi063.stdout:33: finishing write tid 3 to smithi06387500-33 2024-08-17T16:54:54.778 INFO:tasks.rados.rados.0.smithi063.stdout:33: finishing write tid 4 to smithi06387500-33 2024-08-17T16:54:54.778 INFO:tasks.rados.rados.0.smithi063.stdout:33: oid 33 updating version 0 to 7 2024-08-17T16:54:54.778 INFO:tasks.rados.rados.0.smithi063.stdout:33: oid 33 updating version 7 to 9 2024-08-17T16:54:54.778 INFO:tasks.rados.rados.0.smithi063.stdout:33: oid 33 version 9 is already newer than 8 2024-08-17T16:54:54.778 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 9 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-08-17T16:54:54.778 INFO:tasks.rados.rados.0.smithi063.stdout:33: left oid 33 (ObjNum 32 snap 0 seq_num 32) 2024-08-17T16:54:54.778 INFO:tasks.rados.rados.0.smithi063.stdout:34: finishing write tid 1 to smithi06387500-34 2024-08-17T16:54:54.778 INFO:tasks.rados.rados.0.smithi063.stdout:34: finishing write tid 2 to smithi06387500-34 2024-08-17T16:54:54.778 INFO:tasks.rados.rados.0.smithi063.stdout:34: finishing write tid 3 to smithi06387500-34 2024-08-17T16:54:54.778 INFO:tasks.rados.rados.0.smithi063.stdout:34: finishing write tid 4 to smithi06387500-34 2024-08-17T16:54:54.779 INFO:tasks.rados.rados.0.smithi063.stdout:34: oid 34 updating version 0 to 11 2024-08-17T16:54:54.779 INFO:tasks.rados.rados.0.smithi063.stdout:34: oid 34 version 11 is already newer than 10 2024-08-17T16:54:54.779 INFO:tasks.rados.rados.0.smithi063.stdout:34: oid 34 updating version 11 to 12 2024-08-17T16:54:54.779 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 12 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-08-17T16:54:54.779 INFO:tasks.rados.rados.0.smithi063.stdout:34: left oid 34 (ObjNum 33 snap 0 seq_num 33) 2024-08-17T16:54:54.779 INFO:tasks.rados.rados.0.smithi063.stdout:35: finishing write tid 1 to smithi06387500-35 2024-08-17T16:54:54.779 INFO:tasks.rados.rados.0.smithi063.stdout:35: finishing write tid 2 to smithi06387500-35 2024-08-17T16:54:54.779 INFO:tasks.rados.rados.0.smithi063.stdout:35: finishing write tid 3 to smithi06387500-35 2024-08-17T16:54:54.779 INFO:tasks.rados.rados.0.smithi063.stdout:35: finishing write tid 4 to smithi06387500-35 2024-08-17T16:54:54.779 INFO:tasks.rados.rados.0.smithi063.stdout:35: oid 35 updating version 0 to 10 2024-08-17T16:54:54.779 INFO:tasks.rados.rados.0.smithi063.stdout:35: oid 35 updating version 10 to 11 2024-08-17T16:54:54.779 INFO:tasks.rados.rados.0.smithi063.stdout:35: oid 35 updating version 11 to 12 2024-08-17T16:54:54.779 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 12 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-08-17T16:54:54.779 INFO:tasks.rados.rados.0.smithi063.stdout:35: left oid 35 (ObjNum 34 snap 0 seq_num 34) 2024-08-17T16:54:54.780 INFO:tasks.rados.rados.0.smithi063.stdout:36: finishing write tid 1 to smithi06387500-36 2024-08-17T16:54:54.780 INFO:tasks.rados.rados.0.smithi063.stdout:36: finishing write tid 2 to smithi06387500-36 2024-08-17T16:54:54.780 INFO:tasks.rados.rados.0.smithi063.stdout:36: finishing write tid 3 to smithi06387500-36 2024-08-17T16:54:54.780 INFO:tasks.rados.rados.0.smithi063.stdout:36: finishing write tid 4 to smithi06387500-36 2024-08-17T16:54:54.780 INFO:tasks.rados.rados.0.smithi063.stdout:36: oid 36 updating version 0 to 13 2024-08-17T16:54:54.780 INFO:tasks.rados.rados.0.smithi063.stdout:36: oid 36 updating version 13 to 14 2024-08-17T16:54:54.780 INFO:tasks.rados.rados.0.smithi063.stdout:36: oid 36 updating version 14 to 15 2024-08-17T16:54:54.780 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 15 (ObjNum 35 snap 0 seq_num 35) dirty exists 2024-08-17T16:54:54.780 INFO:tasks.rados.rados.0.smithi063.stdout:36: left oid 36 (ObjNum 35 snap 0 seq_num 35) 2024-08-17T16:54:54.780 INFO:tasks.rados.rados.0.smithi063.stdout:37: finishing write tid 1 to smithi06387500-37 2024-08-17T16:54:54.780 INFO:tasks.rados.rados.0.smithi063.stdout:37: finishing write tid 2 to smithi06387500-37 2024-08-17T16:54:54.780 INFO:tasks.rados.rados.0.smithi063.stdout:37: finishing write tid 3 to smithi06387500-37 2024-08-17T16:54:54.780 INFO:tasks.rados.rados.0.smithi063.stdout:32: done (15 left) 2024-08-17T16:54:54.780 INFO:tasks.rados.rados.0.smithi063.stdout:33: done (14 left) 2024-08-17T16:54:54.781 INFO:tasks.rados.rados.0.smithi063.stdout:34: done (13 left) 2024-08-17T16:54:54.781 INFO:tasks.rados.rados.0.smithi063.stdout:35: done (12 left) 2024-08-17T16:54:54.781 INFO:tasks.rados.rados.0.smithi063.stdout:36: done (11 left) 2024-08-17T16:54:54.781 INFO:tasks.rados.rados.0.smithi063.stdout:48: write initial oid 48 2024-08-17T16:54:54.781 INFO:tasks.rados.rados.0.smithi063.stdout:48: seq_num 47 ranges {789929=634821,2050536=739393} 2024-08-17T16:54:54.784 INFO:tasks.rados.rados.0.smithi063.stdout:48: writing smithi06387500-48 from 789929 to 1424750 tid 1 2024-08-17T16:54:54.791 INFO:tasks.rados.rados.0.smithi063.stdout:48: writing smithi06387500-48 from 2050536 to 2789929 tid 2 2024-08-17T16:54:54.792 INFO:tasks.rados.rados.0.smithi063.stdout:49: write initial oid 49 2024-08-17T16:54:54.792 INFO:tasks.rados.rados.0.smithi063.stdout:49: seq_num 48 ranges {406736=710070,1709018=655350} 2024-08-17T16:54:54.797 INFO:tasks.rados.rados.0.smithi063.stdout:49: writing smithi06387500-49 from 406736 to 1116806 tid 1 2024-08-17T16:54:54.804 INFO:tasks.rados.rados.0.smithi063.stdout:49: writing smithi06387500-49 from 1709018 to 2364368 tid 2 2024-08-17T16:54:54.804 INFO:tasks.rados.rados.0.smithi063.stdout:50: write initial oid 50 2024-08-17T16:54:54.805 INFO:tasks.rados.rados.0.smithi063.stdout:50: seq_num 49 ranges {423543=785319,1767500=799008} 2024-08-17T16:54:54.811 INFO:tasks.rados.rados.0.smithi063.stdout:50: writing smithi06387500-50 from 423543 to 1208862 tid 1 2024-08-17T16:54:54.819 INFO:tasks.rados.rados.0.smithi063.stdout:50: writing smithi06387500-50 from 1767500 to 2566508 tid 2 2024-08-17T16:54:54.819 INFO:tasks.rados.rados.0.smithi063.stdout:51: write oid 8 current snap is 0 2024-08-17T16:54:54.819 INFO:tasks.rados.rados.0.smithi063.stdout:37: finishing write tid 4 to smithi06387500-37 2024-08-17T16:54:54.819 INFO:tasks.rados.rados.0.smithi063.stdout:37: oid 37 updating version 0 to 18 2024-08-17T16:54:54.819 INFO:tasks.rados.rados.0.smithi063.stdout:37: oid 37 version 18 is already newer than 16 2024-08-17T16:54:54.819 INFO:tasks.rados.rados.0.smithi063.stdout:37: oid 37 version 18 is already newer than 17 2024-08-17T16:54:54.819 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 18 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-08-17T16:54:54.819 INFO:tasks.rados.rados.0.smithi063.stdout:37: left oid 37 (ObjNum 36 snap 0 seq_num 36) 2024-08-17T16:54:54.820 INFO:tasks.rados.rados.0.smithi063.stdout:38: finishing write tid 1 to smithi06387500-38 2024-08-17T16:54:54.820 INFO:tasks.rados.rados.0.smithi063.stdout:38: finishing write tid 2 to smithi06387500-38 2024-08-17T16:54:54.820 INFO:tasks.rados.rados.0.smithi063.stdout:38: finishing write tid 3 to smithi06387500-38 2024-08-17T16:54:54.820 INFO:tasks.rados.rados.0.smithi063.stdout:38: finishing write tid 4 to smithi06387500-38 2024-08-17T16:54:54.820 INFO:tasks.rados.rados.0.smithi063.stdout:38: oid 38 updating version 0 to 4 2024-08-17T16:54:54.820 INFO:tasks.rados.rados.0.smithi063.stdout:38: oid 38 updating version 4 to 6 2024-08-17T16:54:54.820 INFO:tasks.rados.rados.0.smithi063.stdout:38: oid 38 version 6 is already newer than 5 2024-08-17T16:54:54.820 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 6 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-08-17T16:54:54.820 INFO:tasks.rados.rados.0.smithi063.stdout:38: left oid 38 (ObjNum 37 snap 0 seq_num 37) 2024-08-17T16:54:54.820 INFO:tasks.rados.rados.0.smithi063.stdout:39: finishing write tid 1 to smithi06387500-39 2024-08-17T16:54:54.820 INFO:tasks.rados.rados.0.smithi063.stdout:51: seq_num 50 ranges {440350=460568,1309629=542666,2323973=516377} 2024-08-17T16:54:54.823 INFO:tasks.rados.rados.0.smithi063.stdout:51: writing smithi06387500-8 from 440350 to 900918 tid 1 2024-08-17T16:54:54.829 INFO:tasks.rados.rados.0.smithi063.stdout:51: writing smithi06387500-8 from 1309629 to 1852295 tid 2 2024-08-17T16:54:54.835 INFO:tasks.rados.rados.0.smithi063.stdout:51: writing smithi06387500-8 from 2323973 to 2840350 tid 3 2024-08-17T16:54:54.835 INFO:tasks.rados.rados.0.smithi063.stdout:37: done (14 left) 2024-08-17T16:54:54.835 INFO:tasks.rados.rados.0.smithi063.stdout:38: done (13 left) 2024-08-17T16:54:54.835 INFO:tasks.rados.rados.0.smithi063.stdout:52: rollback oid 18 current snap is 0 2024-08-17T16:54:54.835 INFO:tasks.rados.rados.0.smithi063.stdout:52: done (13 left) 2024-08-17T16:54:54.835 INFO:tasks.rados.rados.0.smithi063.stdout:53: write oid 7 current snap is 0 2024-08-17T16:54:54.835 INFO:tasks.rados.rados.0.smithi063.stdout:39: finishing write tid 2 to smithi06387500-39 2024-08-17T16:54:54.835 INFO:tasks.rados.rados.0.smithi063.stdout:39: finishing write tid 3 to smithi06387500-39 2024-08-17T16:54:54.835 INFO:tasks.rados.rados.0.smithi063.stdout:39: finishing write tid 4 to smithi06387500-39 2024-08-17T16:54:54.836 INFO:tasks.rados.rados.0.smithi063.stdout:39: oid 39 updating version 0 to 1 2024-08-17T16:54:54.836 INFO:tasks.rados.rados.0.smithi063.stdout:39: oid 39 updating version 1 to 3 2024-08-17T16:54:54.836 INFO:tasks.rados.rados.0.smithi063.stdout:39: oid 39 version 3 is already newer than 2 2024-08-17T16:54:54.836 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 3 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-08-17T16:54:54.836 INFO:tasks.rados.rados.0.smithi063.stdout:39: left oid 39 (ObjNum 38 snap 0 seq_num 38) 2024-08-17T16:54:54.836 INFO:tasks.rados.rados.0.smithi063.stdout:40: finishing write tid 1 to smithi06387500-40 2024-08-17T16:54:54.836 INFO:tasks.rados.rados.0.smithi063.stdout:40: finishing write tid 2 to smithi06387500-40 2024-08-17T16:54:54.836 INFO:tasks.rados.rados.0.smithi063.stdout:53: seq_num 51 ranges {457157=535817,1768111=402677} 2024-08-17T16:54:54.840 INFO:tasks.rados.rados.0.smithi063.stdout:53: writing smithi06387500-7 from 457157 to 992974 tid 1 2024-08-17T16:54:54.847 INFO:tasks.rados.rados.0.smithi063.stdout:53: writing smithi06387500-7 from 1768111 to 2170788 tid 2 2024-08-17T16:54:54.847 INFO:tasks.rados.rados.0.smithi063.stdout:39: done (13 left) 2024-08-17T16:54:54.847 INFO:tasks.rados.rados.0.smithi063.stdout:54: write oid 23 current snap is 0 2024-08-17T16:54:54.847 INFO:tasks.rados.rados.0.smithi063.stdout:40: finishing write tid 3 to smithi06387500-40 2024-08-17T16:54:54.847 INFO:tasks.rados.rados.0.smithi063.stdout:40: finishing write tid 4 to smithi06387500-40 2024-08-17T16:54:54.847 INFO:tasks.rados.rados.0.smithi063.stdout:40: oid 40 updating version 0 to 8 2024-08-17T16:54:54.847 INFO:tasks.rados.rados.0.smithi063.stdout:40: oid 40 version 8 is already newer than 7 2024-08-17T16:54:54.848 INFO:tasks.rados.rados.0.smithi063.stdout:40: oid 40 updating version 8 to 9 2024-08-17T16:54:54.848 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 9 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-08-17T16:54:54.848 INFO:tasks.rados.rados.0.smithi063.stdout:40: left oid 40 (ObjNum 39 snap 0 seq_num 39) 2024-08-17T16:54:54.848 INFO:tasks.rados.rados.0.smithi063.stdout:41: finishing write tid 1 to smithi06387500-41 2024-08-17T16:54:54.848 INFO:tasks.rados.rados.0.smithi063.stdout:41: finishing write tid 2 to smithi06387500-41 2024-08-17T16:54:54.848 INFO:tasks.rados.rados.0.smithi063.stdout:41: finishing write tid 3 to smithi06387500-41 2024-08-17T16:54:54.848 INFO:tasks.rados.rados.0.smithi063.stdout:54: seq_num 52 ranges {473964=611066,1826593=546335,2778819=95145} 2024-08-17T16:54:54.852 INFO:tasks.rados.rados.0.smithi063.stdout:54: writing smithi06387500-23 from 473964 to 1085030 tid 1 2024-08-17T16:54:54.859 INFO:tasks.rados.rados.0.smithi063.stdout:54: writing smithi06387500-23 from 1826593 to 2372928 tid 2 2024-08-17T16:54:54.868 INFO:tasks.rados.rados.0.smithi063.stdout:54: writing smithi06387500-23 from 2778819 to 2873964 tid 3 2024-08-17T16:54:54.868 INFO:tasks.rados.rados.0.smithi063.stdout:40: done (13 left) 2024-08-17T16:54:54.868 INFO:tasks.rados.rados.0.smithi063.stdout:55: delete oid 39 current snap is 0 2024-08-17T16:54:54.869 INFO:tasks.rados.rados.0.smithi063.stdout:41: finishing write tid 4 to smithi06387500-41 2024-08-17T16:54:54.869 INFO:tasks.rados.rados.0.smithi063.stdout:41: oid 41 updating version 0 to 2 2024-08-17T16:54:54.869 INFO:tasks.rados.rados.0.smithi063.stdout:41: oid 41 version 2 is already newer than 1 2024-08-17T16:54:54.869 INFO:tasks.rados.rados.0.smithi063.stdout:41: oid 41 updating version 2 to 3 2024-08-17T16:54:54.869 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 3 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-08-17T16:54:54.869 INFO:tasks.rados.rados.0.smithi063.stdout:41: left oid 41 (ObjNum 40 snap 0 seq_num 40) 2024-08-17T16:54:54.870 INFO:tasks.rados.rados.0.smithi063.stdout:42: finishing write tid 1 to smithi06387500-42 2024-08-17T16:54:54.870 INFO:tasks.rados.rados.0.smithi063.stdout:42: finishing write tid 2 to smithi06387500-42 2024-08-17T16:54:54.870 INFO:tasks.rados.rados.0.smithi063.stdout:42: finishing write tid 3 to smithi06387500-42 2024-08-17T16:54:54.870 INFO:tasks.rados.rados.0.smithi063.stdout:42: finishing write tid 4 to smithi06387500-42 2024-08-17T16:54:54.870 INFO:tasks.rados.rados.0.smithi063.stdout:42: oid 42 updating version 0 to 15 2024-08-17T16:54:54.871 INFO:tasks.rados.rados.0.smithi063.stdout:42: oid 42 version 15 is already newer than 13 2024-08-17T16:54:54.871 INFO:tasks.rados.rados.0.smithi063.stdout:42: oid 42 version 15 is already newer than 14 2024-08-17T16:54:54.871 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 15 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-08-17T16:54:54.871 INFO:tasks.rados.rados.0.smithi063.stdout:42: left oid 42 (ObjNum 41 snap 0 seq_num 41) 2024-08-17T16:54:54.871 INFO:tasks.rados.rados.0.smithi063.stdout:43: finishing write tid 1 to smithi06387500-43 2024-08-17T16:54:54.871 INFO:tasks.rados.rados.0.smithi063.stdout:43: finishing write tid 2 to smithi06387500-43 2024-08-17T16:54:54.871 INFO:tasks.rados.rados.0.smithi063.stdout:43: finishing write tid 3 to smithi06387500-43 2024-08-17T16:54:54.872 INFO:tasks.rados.rados.0.smithi063.stdout:43: finishing write tid 4 to smithi06387500-43 2024-08-17T16:54:54.872 INFO:tasks.rados.rados.0.smithi063.stdout:43: oid 43 updating version 0 to 11 2024-08-17T16:54:54.872 INFO:tasks.rados.rados.0.smithi063.stdout:43: oid 43 updating version 11 to 12 2024-08-17T16:54:54.872 INFO:tasks.rados.rados.0.smithi063.stdout:43: oid 43 version 12 is already newer than 10 2024-08-17T16:54:54.872 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 12 (ObjNum 42 snap 0 seq_num 42) dirty exists 2024-08-17T16:54:54.872 INFO:tasks.rados.rados.0.smithi063.stdout:43: left oid 43 (ObjNum 42 snap 0 seq_num 42) 2024-08-17T16:54:54.872 INFO:tasks.rados.rados.0.smithi063.stdout:44: finishing write tid 1 to smithi06387500-44 2024-08-17T16:54:54.873 INFO:tasks.rados.rados.0.smithi063.stdout:44: finishing write tid 2 to smithi06387500-44 2024-08-17T16:54:54.873 INFO:tasks.rados.rados.0.smithi063.stdout:44: finishing write tid 3 to smithi06387500-44 2024-08-17T16:54:54.873 INFO:tasks.rados.rados.0.smithi063.stdout:44: finishing write tid 4 to smithi06387500-44 2024-08-17T16:54:54.873 INFO:tasks.rados.rados.0.smithi063.stdout:44: oid 44 updating version 0 to 3 2024-08-17T16:54:54.873 INFO:tasks.rados.rados.0.smithi063.stdout:44: oid 44 version 3 is already newer than 1 2024-08-17T16:54:54.873 INFO:tasks.rados.rados.0.smithi063.stdout:44: oid 44 version 3 is already newer than 2 2024-08-17T16:54:54.873 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 3 (ObjNum 43 snap 0 seq_num 43) dirty exists 2024-08-17T16:54:54.874 INFO:tasks.rados.rados.0.smithi063.stdout:44: left oid 44 (ObjNum 43 snap 0 seq_num 43) 2024-08-17T16:54:54.874 INFO:tasks.rados.rados.0.smithi063.stdout:45: finishing write tid 1 to smithi06387500-45 2024-08-17T16:54:54.874 INFO:tasks.rados.rados.0.smithi063.stdout:45: finishing write tid 2 to smithi06387500-45 2024-08-17T16:54:54.874 INFO:tasks.rados.rados.0.smithi063.stdout:45: finishing write tid 3 to smithi06387500-45 2024-08-17T16:54:54.874 INFO:tasks.rados.rados.0.smithi063.stdout:45: finishing write tid 4 to smithi06387500-45 2024-08-17T16:54:54.874 INFO:tasks.rados.rados.0.smithi063.stdout:45: oid 45 updating version 0 to 6 2024-08-17T16:54:54.875 INFO:tasks.rados.rados.0.smithi063.stdout:45: oid 45 version 6 is already newer than 4 2024-08-17T16:54:54.875 INFO:tasks.rados.rados.0.smithi063.stdout:45: oid 45 version 6 is already newer than 5 2024-08-17T16:54:54.875 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 6 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-08-17T16:54:54.875 INFO:tasks.rados.rados.0.smithi063.stdout:45: left oid 45 (ObjNum 44 snap 0 seq_num 44) 2024-08-17T16:54:54.875 INFO:tasks.rados.rados.0.smithi063.stdout:46: finishing write tid 1 to smithi06387500-46 2024-08-17T16:54:54.875 INFO:tasks.rados.rados.0.smithi063.stdout:46: finishing write tid 2 to smithi06387500-46 2024-08-17T16:54:54.876 INFO:tasks.rados.rados.0.smithi063.stdout:46: finishing write tid 3 to smithi06387500-46 2024-08-17T16:54:54.876 INFO:tasks.rados.rados.0.smithi063.stdout:46: finishing write tid 4 to smithi06387500-46 2024-08-17T16:54:54.876 INFO:tasks.rados.rados.0.smithi063.stdout:46: oid 46 updating version 0 to 10 2024-08-17T16:54:54.876 INFO:tasks.rados.rados.0.smithi063.stdout:46: oid 46 updating version 10 to 12 2024-08-17T16:54:54.876 INFO:tasks.rados.rados.0.smithi063.stdout:46: oid 46 version 12 is already newer than 11 2024-08-17T16:54:54.876 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 12 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-08-17T16:54:54.876 INFO:tasks.rados.rados.0.smithi063.stdout:46: left oid 46 (ObjNum 45 snap 0 seq_num 45) 2024-08-17T16:54:54.877 INFO:tasks.rados.rados.0.smithi063.stdout:47: finishing write tid 1 to smithi06387500-47 2024-08-17T16:54:54.877 INFO:tasks.rados.rados.0.smithi063.stdout:47: finishing write tid 2 to smithi06387500-47 2024-08-17T16:54:54.877 INFO:tasks.rados.rados.0.smithi063.stdout:47: finishing write tid 3 to smithi06387500-47 2024-08-17T16:54:54.877 INFO:tasks.rados.rados.0.smithi063.stdout:47: finishing write tid 4 to smithi06387500-47 2024-08-17T16:54:54.877 INFO:tasks.rados.rados.0.smithi063.stdout:47: oid 47 updating version 0 to 5 2024-08-17T16:54:54.877 INFO:tasks.rados.rados.0.smithi063.stdout:47: oid 47 updating version 5 to 6 2024-08-17T16:54:54.877 INFO:tasks.rados.rados.0.smithi063.stdout:47: oid 47 version 6 is already newer than 4 2024-08-17T16:54:54.878 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 6 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-08-17T16:54:54.878 INFO:tasks.rados.rados.0.smithi063.stdout:47: left oid 47 (ObjNum 46 snap 0 seq_num 46) 2024-08-17T16:54:54.878 INFO:tasks.rados.rados.0.smithi063.stdout:48: finishing write tid 1 to smithi06387500-48 2024-08-17T16:54:54.878 INFO:tasks.rados.rados.0.smithi063.stdout:48: finishing write tid 2 to smithi06387500-48 2024-08-17T16:54:54.878 INFO:tasks.rados.rados.0.smithi063.stdout:48: finishing write tid 3 to smithi06387500-48 2024-08-17T16:54:54.878 INFO:tasks.rados.rados.0.smithi063.stdout:48: finishing write tid 4 to smithi06387500-48 2024-08-17T16:54:54.879 INFO:tasks.rados.rados.0.smithi063.stdout:48: oid 48 updating version 0 to 21 2024-08-17T16:54:54.879 INFO:tasks.rados.rados.0.smithi063.stdout:48: oid 48 version 21 is already newer than 19 2024-08-17T16:54:54.879 INFO:tasks.rados.rados.0.smithi063.stdout:48: oid 48 version 21 is already newer than 20 2024-08-17T16:54:54.879 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 21 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-08-17T16:54:54.879 INFO:tasks.rados.rados.0.smithi063.stdout:48: left oid 48 (ObjNum 47 snap 0 seq_num 47) 2024-08-17T16:54:54.879 INFO:tasks.rados.rados.0.smithi063.stdout:49: finishing write tid 1 to smithi06387500-49 2024-08-17T16:54:54.879 INFO:tasks.rados.rados.0.smithi063.stdout:49: finishing write tid 2 to smithi06387500-49 2024-08-17T16:54:54.880 INFO:tasks.rados.rados.0.smithi063.stdout:49: finishing write tid 3 to smithi06387500-49 2024-08-17T16:54:54.880 INFO:tasks.rados.rados.0.smithi063.stdout:49: finishing write tid 4 to smithi06387500-49 2024-08-17T16:54:54.880 INFO:tasks.rados.rados.0.smithi063.stdout:49: oid 49 updating version 0 to 22 2024-08-17T16:54:54.880 INFO:tasks.rados.rados.0.smithi063.stdout:49: oid 49 updating version 22 to 24 2024-08-17T16:54:54.880 INFO:tasks.rados.rados.0.smithi063.stdout:49: oid 49 version 24 is already newer than 23 2024-08-17T16:54:54.880 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 24 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-08-17T16:54:54.880 INFO:tasks.rados.rados.0.smithi063.stdout:49: left oid 49 (ObjNum 48 snap 0 seq_num 48) 2024-08-17T16:54:54.881 INFO:tasks.rados.rados.0.smithi063.stdout:50: finishing write tid 1 to smithi06387500-50 2024-08-17T16:54:54.881 INFO:tasks.rados.rados.0.smithi063.stdout:50: finishing write tid 2 to smithi06387500-50 2024-08-17T16:54:54.881 INFO:tasks.rados.rados.0.smithi063.stdout:50: finishing write tid 3 to smithi06387500-50 2024-08-17T16:54:54.881 INFO:tasks.rados.rados.0.smithi063.stdout:50: finishing write tid 4 to smithi06387500-50 2024-08-17T16:54:54.881 INFO:tasks.rados.rados.0.smithi063.stdout:50: oid 50 updating version 0 to 17 2024-08-17T16:54:54.881 INFO:tasks.rados.rados.0.smithi063.stdout:50: oid 50 updating version 17 to 18 2024-08-17T16:54:54.881 INFO:tasks.rados.rados.0.smithi063.stdout:50: oid 50 version 18 is already newer than 16 2024-08-17T16:54:54.882 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 18 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-08-17T16:54:54.882 INFO:tasks.rados.rados.0.smithi063.stdout:50: left oid 50 (ObjNum 49 snap 0 seq_num 49) 2024-08-17T16:54:54.882 INFO:tasks.rados.rados.0.smithi063.stdout:51: finishing write tid 1 to smithi06387500-8 2024-08-17T16:54:54.882 INFO:tasks.rados.rados.0.smithi063.stdout:51: finishing write tid 2 to smithi06387500-8 2024-08-17T16:54:54.882 INFO:tasks.rados.rados.0.smithi063.stdout:51: finishing write tid 3 to smithi06387500-8 2024-08-17T16:54:54.882 INFO:tasks.rados.rados.0.smithi063.stdout:51: finishing write tid 4 to smithi06387500-8 2024-08-17T16:54:54.883 INFO:tasks.rados.rados.0.smithi063.stdout:51: finishing write tid 5 to smithi06387500-8 2024-08-17T16:54:54.883 INFO:tasks.rados.rados.0.smithi063.stdout:51: oid 8 updating version 0 to 13 2024-08-17T16:54:54.883 INFO:tasks.rados.rados.0.smithi063.stdout:51: oid 8 version 13 is already newer than 10 2024-08-17T16:54:54.883 INFO:tasks.rados.rados.0.smithi063.stdout:51: oid 8 version 13 is already newer than 12 2024-08-17T16:54:54.883 INFO:tasks.rados.rados.0.smithi063.stdout:51: oid 8 version 13 is already newer than 11 2024-08-17T16:54:54.883 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 13 (ObjNum 50 snap 0 seq_num 50) dirty exists 2024-08-17T16:54:54.883 INFO:tasks.rados.rados.0.smithi063.stdout:51: left oid 8 (ObjNum 50 snap 0 seq_num 50) 2024-08-17T16:54:54.883 INFO:tasks.rados.rados.0.smithi063.stdout:53: finishing write tid 1 to smithi06387500-7 2024-08-17T16:54:54.884 INFO:tasks.rados.rados.0.smithi063.stdout:53: finishing write tid 2 to smithi06387500-7 2024-08-17T16:54:54.884 INFO:tasks.rados.rados.0.smithi063.stdout:53: finishing write tid 3 to smithi06387500-7 2024-08-17T16:54:54.884 INFO:tasks.rados.rados.0.smithi063.stdout:53: finishing write tid 4 to smithi06387500-7 2024-08-17T16:54:54.884 INFO:tasks.rados.rados.0.smithi063.stdout:53: oid 7 updating version 0 to 13 2024-08-17T16:54:54.884 INFO:tasks.rados.rados.0.smithi063.stdout:53: oid 7 updating version 13 to 14 2024-08-17T16:54:54.884 INFO:tasks.rados.rados.0.smithi063.stdout:53: oid 7 updating version 14 to 15 2024-08-17T16:54:54.884 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 15 (ObjNum 51 snap 0 seq_num 51) dirty exists 2024-08-17T16:54:54.885 INFO:tasks.rados.rados.0.smithi063.stdout:53: left oid 7 (ObjNum 51 snap 0 seq_num 51) 2024-08-17T16:54:54.885 INFO:tasks.rados.rados.0.smithi063.stdout:54: finishing write tid 1 to smithi06387500-23 2024-08-17T16:54:54.885 INFO:tasks.rados.rados.0.smithi063.stdout:54: finishing write tid 2 to smithi06387500-23 2024-08-17T16:54:54.885 INFO:tasks.rados.rados.0.smithi063.stdout:41: done (13 left) 2024-08-17T16:54:54.885 INFO:tasks.rados.rados.0.smithi063.stdout:42: done (12 left) 2024-08-17T16:54:54.885 INFO:tasks.rados.rados.0.smithi063.stdout:43: done (11 left) 2024-08-17T16:54:54.885 INFO:tasks.rados.rados.0.smithi063.stdout:44: done (10 left) 2024-08-17T16:54:54.886 INFO:tasks.rados.rados.0.smithi063.stdout:45: done (9 left) 2024-08-17T16:54:54.886 INFO:tasks.rados.rados.0.smithi063.stdout:46: done (8 left) 2024-08-17T16:54:54.886 INFO:tasks.rados.rados.0.smithi063.stdout:47: done (7 left) 2024-08-17T16:54:54.886 INFO:tasks.rados.rados.0.smithi063.stdout:48: done (6 left) 2024-08-17T16:54:54.886 INFO:tasks.rados.rados.0.smithi063.stdout:49: done (5 left) 2024-08-17T16:54:54.886 INFO:tasks.rados.rados.0.smithi063.stdout:50: done (4 left) 2024-08-17T16:54:54.887 INFO:tasks.rados.rados.0.smithi063.stdout:51: done (3 left) 2024-08-17T16:54:54.887 INFO:tasks.rados.rados.0.smithi063.stdout:53: done (2 left) 2024-08-17T16:54:54.887 INFO:tasks.rados.rados.0.smithi063.stdout:55: done (1 left) 2024-08-17T16:54:54.887 INFO:tasks.rados.rados.0.smithi063.stdout:56: read oid 15 snap -1 2024-08-17T16:54:54.887 INFO:tasks.rados.rados.0.smithi063.stdout:56: expect (ObjNum 14 snap 0 seq_num 14) 2024-08-17T16:54:54.887 INFO:tasks.rados.rados.0.smithi063.stdout:57: delete oid 13 current snap is 0 2024-08-17T16:54:54.887 INFO:tasks.rados.rados.0.smithi063.stdout:54: finishing write tid 3 to smithi06387500-23 2024-08-17T16:54:54.888 INFO:tasks.rados.rados.0.smithi063.stdout:54: finishing write tid 4 to smithi06387500-23 2024-08-17T16:54:54.888 INFO:tasks.rados.rados.0.smithi063.stdout:54: finishing write tid 5 to smithi06387500-23 2024-08-17T16:54:54.888 INFO:tasks.rados.rados.0.smithi063.stdout:54: oid 23 updating version 0 to 18 2024-08-17T16:54:54.888 INFO:tasks.rados.rados.0.smithi063.stdout:54: oid 23 updating version 18 to 19 2024-08-17T16:54:54.888 INFO:tasks.rados.rados.0.smithi063.stdout:54: oid 23 version 19 is already newer than 16 2024-08-17T16:54:54.888 INFO:tasks.rados.rados.0.smithi063.stdout:54: oid 23 version 19 is already newer than 17 2024-08-17T16:54:54.888 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 19 (ObjNum 52 snap 0 seq_num 52) dirty exists 2024-08-17T16:54:54.889 INFO:tasks.rados.rados.0.smithi063.stdout:54: left oid 23 (ObjNum 52 snap 0 seq_num 52) 2024-08-17T16:54:54.889 INFO:tasks.rados.rados.0.smithi063.stdout:54: done (2 left) 2024-08-17T16:54:54.889 INFO:tasks.rados.rados.0.smithi063.stdout:57: done (1 left) 2024-08-17T16:54:54.889 INFO:tasks.rados.rados.0.smithi063.stdout:58: read oid 3 snap -1 2024-08-17T16:54:54.889 INFO:tasks.rados.rados.0.smithi063.stdout:58: expect (ObjNum 2 snap 0 seq_num 2) 2024-08-17T16:54:54.889 INFO:tasks.rados.rados.0.smithi063.stdout:59: read oid 28 snap -1 2024-08-17T16:54:54.889 INFO:tasks.rados.rados.0.smithi063.stdout:59: expect (ObjNum 27 snap 0 seq_num 27) 2024-08-17T16:54:54.889 INFO:tasks.rados.rados.0.smithi063.stdout:60: read oid 34 snap -1 2024-08-17T16:54:54.890 INFO:tasks.rados.rados.0.smithi063.stdout:60: expect (ObjNum 33 snap 0 seq_num 33) 2024-08-17T16:54:54.890 INFO:tasks.rados.rados.0.smithi063.stdout:61: delete oid 2 current snap is 0 2024-08-17T16:54:54.890 INFO:tasks.rados.rados.0.smithi063.stdout:61: done (4 left) 2024-08-17T16:54:54.890 INFO:tasks.rados.rados.0.smithi063.stdout:62: write oid 7 current snap is 0 2024-08-17T16:54:54.890 INFO:tasks.rados.rados.0.smithi063.stdout:62: seq_num 56 ranges {541192=628415,1660521=553673,2888511=52681} 2024-08-17T16:54:54.891 INFO:tasks.rados.rados.0.smithi063.stdout:62: writing smithi06387500-7 from 541192 to 1169607 tid 1 2024-08-17T16:54:54.892 INFO:tasks.rados.rados.0.smithi063.stdout:62: writing smithi06387500-7 from 1660521 to 2214194 tid 2 2024-08-17T16:54:54.892 INFO:tasks.rados.rados.0.smithi063.stdout:62: writing smithi06387500-7 from 2888511 to 2941192 tid 3 2024-08-17T16:54:54.892 INFO:tasks.rados.rados.0.smithi063.stdout:63: write oid 23 current snap is 0 2024-08-17T16:54:54.892 INFO:tasks.rados.rados.0.smithi063.stdout:56: expect (ObjNum 14 snap 0 seq_num 14) 2024-08-17T16:54:54.959 INFO:tasks.rados.rados.0.smithi063.stdout:60: expect (ObjNum 33 snap 0 seq_num 33) 2024-08-17T16:54:55.046 INFO:tasks.rados.rados.0.smithi063.stdout:59: expect (ObjNum 27 snap 0 seq_num 27) 2024-08-17T16:54:55.125 INFO:tasks.rados.rados.0.smithi063.stdout:58: expect (ObjNum 2 snap 0 seq_num 2) 2024-08-17T16:54:55.193 INFO:tasks.rados.rados.0.smithi063.stdout:62: finishing write tid 1 to smithi06387500-7 2024-08-17T16:54:55.194 INFO:tasks.rados.rados.0.smithi063.stdout:62: finishing write tid 2 to smithi06387500-7 2024-08-17T16:54:55.194 INFO:tasks.rados.rados.0.smithi063.stdout:62: finishing write tid 3 to smithi06387500-7 2024-08-17T16:54:55.194 INFO:tasks.rados.rados.0.smithi063.stdout:62: finishing write tid 4 to smithi06387500-7 2024-08-17T16:54:55.194 INFO:tasks.rados.rados.0.smithi063.stdout:62: finishing write tid 5 to smithi06387500-7 2024-08-17T16:54:55.194 INFO:tasks.rados.rados.0.smithi063.stdout:62: oid 7 updating version 0 to 16 2024-08-17T16:54:55.194 INFO:tasks.rados.rados.0.smithi063.stdout:62: oid 7 updating version 16 to 18 2024-08-17T16:54:55.194 INFO:tasks.rados.rados.0.smithi063.stdout:62: oid 7 version 18 is already newer than 17 2024-08-17T16:54:55.194 INFO:tasks.rados.rados.0.smithi063.stdout:62: oid 7 updating version 18 to 19 2024-08-17T16:54:55.194 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 19 (ObjNum 56 snap 0 seq_num 56) dirty exists 2024-08-17T16:54:55.195 INFO:tasks.rados.rados.0.smithi063.stdout:62: left oid 7 (ObjNum 56 snap 0 seq_num 56) 2024-08-17T16:54:55.195 INFO:tasks.rados.rados.0.smithi063.stdout:63: seq_num 57 ranges {557999=703664,1719003=413684,2632287=325712} 2024-08-17T16:54:55.200 INFO:tasks.rados.rados.0.smithi063.stdout:63: writing smithi06387500-23 from 557999 to 1261663 tid 1 2024-08-17T16:54:55.205 INFO:tasks.rados.rados.0.smithi063.stdout:63: writing smithi06387500-23 from 1719003 to 2132687 tid 2 2024-08-17T16:54:55.210 INFO:tasks.rados.rados.0.smithi063.stdout:63: writing smithi06387500-23 from 2632287 to 2957999 tid 3 2024-08-17T16:54:55.210 INFO:tasks.rados.rados.0.smithi063.stdout:56: done (5 left) 2024-08-17T16:54:55.211 INFO:tasks.rados.rados.0.smithi063.stdout:58: done (4 left) 2024-08-17T16:54:55.211 INFO:tasks.rados.rados.0.smithi063.stdout:59: done (3 left) 2024-08-17T16:54:55.211 INFO:tasks.rados.rados.0.smithi063.stdout:60: done (2 left) 2024-08-17T16:54:55.211 INFO:tasks.rados.rados.0.smithi063.stdout:62: done (1 left) 2024-08-17T16:54:55.211 INFO:tasks.rados.rados.0.smithi063.stdout:64: write oid 48 current snap is 0 2024-08-17T16:54:55.211 INFO:tasks.rados.rados.0.smithi063.stdout:63: finishing write tid 1 to smithi06387500-23 2024-08-17T16:54:55.211 INFO:tasks.rados.rados.0.smithi063.stdout:63: finishing write tid 2 to smithi06387500-23 2024-08-17T16:54:55.211 INFO:tasks.rados.rados.0.smithi063.stdout:64: seq_num 58 ranges {574806=778913,2061132=557342} 2024-08-17T16:54:55.217 INFO:tasks.rados.rados.0.smithi063.stdout:64: writing smithi06387500-48 from 574806 to 1353719 tid 1 2024-08-17T16:54:55.225 INFO:tasks.rados.rados.0.smithi063.stdout:64: writing smithi06387500-48 from 2061132 to 2618474 tid 2 2024-08-17T16:54:55.225 INFO:tasks.rados.rados.0.smithi063.stdout:65: rollback oid 37 current snap is 0 2024-08-17T16:54:55.225 INFO:tasks.rados.rados.0.smithi063.stdout:65: done (2 left) 2024-08-17T16:54:55.225 INFO:tasks.rados.rados.0.smithi063.stdout:66: read oid 29 snap -1 2024-08-17T16:54:55.225 INFO:tasks.rados.rados.0.smithi063.stdout:66: expect (ObjNum 28 snap 0 seq_num 28) 2024-08-17T16:54:55.225 INFO:tasks.rados.rados.0.smithi063.stdout:67: write oid 5 current snap is 0 2024-08-17T16:54:55.225 INFO:tasks.rados.rados.0.smithi063.stdout:63: finishing write tid 3 to smithi06387500-23 2024-08-17T16:54:55.225 INFO:tasks.rados.rados.0.smithi063.stdout:63: finishing write tid 4 to smithi06387500-23 2024-08-17T16:54:55.226 INFO:tasks.rados.rados.0.smithi063.stdout:63: finishing write tid 5 to smithi06387500-23 2024-08-17T16:54:55.226 INFO:tasks.rados.rados.0.smithi063.stdout:63: oid 23 updating version 0 to 22 2024-08-17T16:54:55.226 INFO:tasks.rados.rados.0.smithi063.stdout:63: oid 23 updating version 22 to 24 2024-08-17T16:54:55.226 INFO:tasks.rados.rados.0.smithi063.stdout:63: oid 23 version 24 is already newer than 23 2024-08-17T16:54:55.226 INFO:tasks.rados.rados.0.smithi063.stdout:63: oid 23 updating version 24 to 25 2024-08-17T16:54:55.226 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 25 (ObjNum 57 snap 0 seq_num 57) dirty exists 2024-08-17T16:54:55.226 INFO:tasks.rados.rados.0.smithi063.stdout:63: left oid 23 (ObjNum 57 snap 0 seq_num 57) 2024-08-17T16:54:55.226 INFO:tasks.rados.rados.0.smithi063.stdout:64: finishing write tid 1 to smithi06387500-48 2024-08-17T16:54:55.226 INFO:tasks.rados.rados.0.smithi063.stdout:67: seq_num 59 ranges {591613=454162,1719614=417353,2570780=420833} 2024-08-17T16:54:55.230 INFO:tasks.rados.rados.0.smithi063.stdout:67: writing smithi06387500-5 from 591613 to 1045775 tid 1 2024-08-17T16:54:55.236 INFO:tasks.rados.rados.0.smithi063.stdout:67: writing smithi06387500-5 from 1719614 to 2136967 tid 2 2024-08-17T16:54:55.241 INFO:tasks.rados.rados.0.smithi063.stdout:67: writing smithi06387500-5 from 2570780 to 2991613 tid 3 2024-08-17T16:54:55.241 INFO:tasks.rados.rados.0.smithi063.stdout:63: done (3 left) 2024-08-17T16:54:55.242 INFO:tasks.rados.rados.0.smithi063.stdout:68: write oid 4 current snap is 0 2024-08-17T16:54:55.242 INFO:tasks.rados.rados.0.smithi063.stdout:68: seq_num 60 ranges {608420=529411,1778096=561011,2881850=126570} 2024-08-17T16:54:55.247 INFO:tasks.rados.rados.0.smithi063.stdout:68: writing smithi06387500-4 from 608420 to 1137831 tid 1 2024-08-17T16:54:55.254 INFO:tasks.rados.rados.0.smithi063.stdout:68: writing smithi06387500-4 from 1778096 to 2339107 tid 2 2024-08-17T16:54:55.258 INFO:tasks.rados.rados.0.smithi063.stdout:68: writing smithi06387500-4 from 2881850 to 3008420 tid 3 2024-08-17T16:54:55.258 INFO:tasks.rados.rados.0.smithi063.stdout:69: write oid 42 current snap is 0 2024-08-17T16:54:55.258 INFO:tasks.rados.rados.0.smithi063.stdout:69: seq_num 61 ranges {625227=721013,1952931=704669} 2024-08-17T16:54:55.265 INFO:tasks.rados.rados.0.smithi063.stdout:69: writing smithi06387500-42 from 625227 to 1346240 tid 1 2024-08-17T16:54:55.272 INFO:tasks.rados.rados.0.smithi063.stdout:69: writing smithi06387500-42 from 1952931 to 2657600 tid 2 2024-08-17T16:54:55.273 INFO:tasks.rados.rados.0.smithi063.stdout:70: write oid 21 current snap is 0 2024-08-17T16:54:55.273 INFO:tasks.rados.rados.0.smithi063.stdout:70: seq_num 62 ranges {642034=796262,1895060=564680} 2024-08-17T16:54:55.281 INFO:tasks.rados.rados.0.smithi063.stdout:70: writing smithi06387500-21 from 642034 to 1438296 tid 1 2024-08-17T16:54:55.286 INFO:tasks.rados.rados.0.smithi063.stdout:70: writing smithi06387500-21 from 1895060 to 2459740 tid 2 2024-08-17T16:54:55.287 INFO:tasks.rados.rados.0.smithi063.stdout:71: snap_create 2024-08-17T16:54:55.287 INFO:tasks.rados.rados.0.smithi063.stdout:66: expect (ObjNum 28 snap 0 seq_num 28) 2024-08-17T16:54:55.370 INFO:tasks.rados.rados.0.smithi063.stdout:64: finishing write tid 2 to smithi06387500-48 2024-08-17T16:54:55.371 INFO:tasks.rados.rados.0.smithi063.stdout:64: finishing write tid 3 to smithi06387500-48 2024-08-17T16:54:55.371 INFO:tasks.rados.rados.0.smithi063.stdout:64: finishing write tid 4 to smithi06387500-48 2024-08-17T16:54:55.371 INFO:tasks.rados.rados.0.smithi063.stdout:64: oid 48 updating version 0 to 25 2024-08-17T16:54:55.371 INFO:tasks.rados.rados.0.smithi063.stdout:64: oid 48 updating version 25 to 27 2024-08-17T16:54:55.371 INFO:tasks.rados.rados.0.smithi063.stdout:64: oid 48 version 27 is already newer than 26 2024-08-17T16:54:55.371 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 27 (ObjNum 58 snap 0 seq_num 58) dirty exists 2024-08-17T16:54:55.371 INFO:tasks.rados.rados.0.smithi063.stdout:64: left oid 48 (ObjNum 58 snap 0 seq_num 58) 2024-08-17T16:54:55.371 INFO:tasks.rados.rados.0.smithi063.stdout:67: finishing write tid 1 to smithi06387500-5 2024-08-17T16:54:55.371 INFO:tasks.rados.rados.0.smithi063.stdout:67: finishing write tid 2 to smithi06387500-5 2024-08-17T16:54:55.371 INFO:tasks.rados.rados.0.smithi063.stdout:67: finishing write tid 3 to smithi06387500-5 2024-08-17T16:54:55.371 INFO:tasks.rados.rados.0.smithi063.stdout:67: finishing write tid 4 to smithi06387500-5 2024-08-17T16:54:55.371 INFO:tasks.rados.rados.0.smithi063.stdout:67: finishing write tid 5 to smithi06387500-5 2024-08-17T16:54:55.372 INFO:tasks.rados.rados.0.smithi063.stdout:67: oid 5 updating version 0 to 26 2024-08-17T16:54:55.372 INFO:tasks.rados.rados.0.smithi063.stdout:67: oid 5 updating version 26 to 28 2024-08-17T16:54:55.372 INFO:tasks.rados.rados.0.smithi063.stdout:67: oid 5 version 28 is already newer than 27 2024-08-17T16:54:55.372 INFO:tasks.rados.rados.0.smithi063.stdout:67: oid 5 updating version 28 to 29 2024-08-17T16:54:55.372 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 29 (ObjNum 59 snap 0 seq_num 59) dirty exists 2024-08-17T16:54:55.372 INFO:tasks.rados.rados.0.smithi063.stdout:67: left oid 5 (ObjNum 59 snap 0 seq_num 59) 2024-08-17T16:54:55.372 INFO:tasks.rados.rados.0.smithi063.stdout:68: finishing write tid 1 to smithi06387500-4 2024-08-17T16:54:55.372 INFO:tasks.rados.rados.0.smithi063.stdout:68: finishing write tid 2 to smithi06387500-4 2024-08-17T16:54:55.372 INFO:tasks.rados.rados.0.smithi063.stdout:68: finishing write tid 3 to smithi06387500-4 2024-08-17T16:54:55.372 INFO:tasks.rados.rados.0.smithi063.stdout:68: finishing write tid 4 to smithi06387500-4 2024-08-17T16:54:55.372 INFO:tasks.rados.rados.0.smithi063.stdout:68: finishing write tid 5 to smithi06387500-4 2024-08-17T16:54:55.372 INFO:tasks.rados.rados.0.smithi063.stdout:68: oid 4 updating version 0 to 28 2024-08-17T16:54:55.372 INFO:tasks.rados.rados.0.smithi063.stdout:68: oid 4 updating version 28 to 31 2024-08-17T16:54:55.373 INFO:tasks.rados.rados.0.smithi063.stdout:68: oid 4 version 31 is already newer than 30 2024-08-17T16:54:55.373 INFO:tasks.rados.rados.0.smithi063.stdout:68: oid 4 version 31 is already newer than 29 2024-08-17T16:54:55.373 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 31 (ObjNum 60 snap 0 seq_num 60) dirty exists 2024-08-17T16:54:55.373 INFO:tasks.rados.rados.0.smithi063.stdout:68: left oid 4 (ObjNum 60 snap 0 seq_num 60) 2024-08-17T16:54:55.373 INFO:tasks.rados.rados.0.smithi063.stdout:69: finishing write tid 1 to smithi06387500-42 2024-08-17T16:54:55.373 INFO:tasks.rados.rados.0.smithi063.stdout:69: finishing write tid 2 to smithi06387500-42 2024-08-17T16:54:55.373 INFO:tasks.rados.rados.0.smithi063.stdout:69: finishing write tid 3 to smithi06387500-42 2024-08-17T16:54:55.373 INFO:tasks.rados.rados.0.smithi063.stdout:69: finishing write tid 4 to smithi06387500-42 2024-08-17T16:54:55.373 INFO:tasks.rados.rados.0.smithi063.stdout:69: oid 42 updating version 0 to 21 2024-08-17T16:54:55.373 INFO:tasks.rados.rados.0.smithi063.stdout:69: oid 42 version 21 is already newer than 20 2024-08-17T16:54:55.373 INFO:tasks.rados.rados.0.smithi063.stdout:69: oid 42 version 21 is already newer than 19 2024-08-17T16:54:55.373 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 21 (ObjNum 61 snap 0 seq_num 61) dirty exists 2024-08-17T16:54:55.373 INFO:tasks.rados.rados.0.smithi063.stdout:69: left oid 42 (ObjNum 61 snap 0 seq_num 61) 2024-08-17T16:54:55.373 INFO:tasks.rados.rados.0.smithi063.stdout:70: finishing write tid 1 to smithi06387500-21 2024-08-17T16:54:55.374 INFO:tasks.rados.rados.0.smithi063.stdout:70: finishing write tid 2 to smithi06387500-21 2024-08-17T16:54:55.374 INFO:tasks.rados.rados.0.smithi063.stdout:70: finishing write tid 3 to smithi06387500-21 2024-08-17T16:54:55.374 INFO:tasks.rados.rados.0.smithi063.stdout:70: finishing write tid 4 to smithi06387500-21 2024-08-17T16:54:55.374 INFO:tasks.rados.rados.0.smithi063.stdout:70: oid 21 updating version 0 to 24 2024-08-17T16:54:55.374 INFO:tasks.rados.rados.0.smithi063.stdout:70: oid 21 version 24 is already newer than 22 2024-08-17T16:54:55.374 INFO:tasks.rados.rados.0.smithi063.stdout:70: oid 21 version 24 is already newer than 23 2024-08-17T16:54:55.374 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 24 (ObjNum 62 snap 0 seq_num 62) dirty exists 2024-08-17T16:54:55.374 INFO:tasks.rados.rados.0.smithi063.stdout:70: left oid 21 (ObjNum 62 snap 0 seq_num 62) 2024-08-17T16:54:55.463 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:55 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/1078629904' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-08-17T16:54:55.464 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:55 smithi063 ceph-mon[31063]: osdmap e79: 8 total, 8 up, 8 in 2024-08-17T16:54:55.464 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:55 smithi063 ceph-mon[31063]: pgmap v99: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:55.464 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:55 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/1078629904' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-08-17T16:54:55.464 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:55 smithi063 ceph-mon[42680]: osdmap e79: 8 total, 8 up, 8 in 2024-08-17T16:54:55.464 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:55 smithi063 ceph-mon[42680]: pgmap v99: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:55.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:55 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/1078629904' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-08-17T16:54:55.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:55 smithi162 ceph-mon[34633]: osdmap e79: 8 total, 8 up, 8 in 2024-08-17T16:54:55.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:55 smithi162 ceph-mon[34633]: pgmap v99: 89 pgs: 89 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:54:56.177 INFO:tasks.rados.rados.0.smithi063.stdout:64: done (6 left) 2024-08-17T16:54:56.177 INFO:tasks.rados.rados.0.smithi063.stdout:66: done (5 left) 2024-08-17T16:54:56.177 INFO:tasks.rados.rados.0.smithi063.stdout:67: done (4 left) 2024-08-17T16:54:56.177 INFO:tasks.rados.rados.0.smithi063.stdout:68: done (3 left) 2024-08-17T16:54:56.178 INFO:tasks.rados.rados.0.smithi063.stdout:69: done (2 left) 2024-08-17T16:54:56.178 INFO:tasks.rados.rados.0.smithi063.stdout:70: done (1 left) 2024-08-17T16:54:56.178 INFO:tasks.rados.rados.0.smithi063.stdout:71: done (0 left) 2024-08-17T16:54:56.178 INFO:tasks.rados.rados.0.smithi063.stdout:72: write oid 50 current snap is 1 2024-08-17T16:54:56.178 INFO:tasks.rados.rados.0.smithi063.stdout:72: seq_num 63 ranges {658841=471511,1553542=708338,2964119=94722} 2024-08-17T16:54:56.183 INFO:tasks.rados.rados.0.smithi063.stdout:72: writing smithi06387500-50 from 658841 to 1130352 tid 1 2024-08-17T16:54:56.189 INFO:tasks.rados.rados.0.smithi063.stdout:72: writing smithi06387500-50 from 1553542 to 2261880 tid 2 2024-08-17T16:54:56.194 INFO:tasks.rados.rados.0.smithi063.stdout:72: writing smithi06387500-50 from 2964119 to 3058841 tid 3 2024-08-17T16:54:56.194 INFO:tasks.rados.rados.0.smithi063.stdout:73: rollback oid 38 current snap is 1 2024-08-17T16:54:56.194 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 38 to 0 2024-08-17T16:54:56.194 INFO:tasks.rados.rados.0.smithi063.stdout:72: finishing write tid 1 to smithi06387500-50 2024-08-17T16:54:56.194 INFO:tasks.rados.rados.0.smithi063.stdout:72: finishing write tid 2 to smithi06387500-50 2024-08-17T16:54:56.194 INFO:tasks.rados.rados.0.smithi063.stdout:74: write oid 33 current snap is 1 2024-08-17T16:54:56.195 INFO:tasks.rados.rados.0.smithi063.stdout:74: seq_num 64 ranges {675648=546760,2012024=568349} 2024-08-17T16:54:56.200 INFO:tasks.rados.rados.0.smithi063.stdout:74: writing smithi06387500-33 from 675648 to 1222408 tid 1 2024-08-17T16:54:56.208 INFO:tasks.rados.rados.0.smithi063.stdout:74: writing smithi06387500-33 from 2012024 to 2580373 tid 2 2024-08-17T16:54:56.209 INFO:tasks.rados.rados.0.smithi063.stdout:75: rollback oid 37 current snap is 1 2024-08-17T16:54:56.209 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 0 2024-08-17T16:54:56.209 INFO:tasks.rados.rados.0.smithi063.stdout:72: finishing write tid 3 to smithi06387500-50 2024-08-17T16:54:56.209 INFO:tasks.rados.rados.0.smithi063.stdout:73: finishing rollback tid 0 to smithi06387500-38 2024-08-17T16:54:56.209 INFO:tasks.rados.rados.0.smithi063.stdout:72: finishing write tid 4 to smithi06387500-50 2024-08-17T16:54:56.209 INFO:tasks.rados.rados.0.smithi063.stdout:72: finishing write tid 5 to smithi06387500-50 2024-08-17T16:54:56.209 INFO:tasks.rados.rados.0.smithi063.stdout:72: oid 50 updating version 0 to 26 2024-08-17T16:54:56.209 INFO:tasks.rados.rados.0.smithi063.stdout:72: oid 50 updating version 26 to 28 2024-08-17T16:54:56.209 INFO:tasks.rados.rados.0.smithi063.stdout:72: oid 50 updating version 28 to 29 2024-08-17T16:54:56.209 INFO:tasks.rados.rados.0.smithi063.stdout:72: oid 50 version 29 is already newer than 27 2024-08-17T16:54:56.209 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 29 (ObjNum 63 snap 1 seq_num 63) dirty exists 2024-08-17T16:54:56.209 INFO:tasks.rados.rados.0.smithi063.stdout:72: left oid 50 (ObjNum 63 snap 1 seq_num 63) 2024-08-17T16:54:56.210 INFO:tasks.rados.rados.0.smithi063.stdout:73: finishing rollback tid 1 to smithi06387500-38 2024-08-17T16:54:56.210 INFO:tasks.rados.rados.0.smithi063.stdout:72: done (3 left) 2024-08-17T16:54:56.210 INFO:tasks.rados.rados.0.smithi063.stdout:76: snap_create 2024-08-17T16:54:56.210 INFO:tasks.rados.rados.0.smithi063.stdout:73: finishing rollback tid 2 to smithi06387500-38 2024-08-17T16:54:56.210 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 10 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-08-17T16:54:56.210 INFO:tasks.rados.rados.0.smithi063.stdout:74: finishing write tid 1 to smithi06387500-33 2024-08-17T16:54:56.211 INFO:tasks.rados.rados.0.smithi063.stdout:75: finishing rollback tid 0 to smithi06387500-37 2024-08-17T16:54:56.211 INFO:tasks.rados.rados.0.smithi063.stdout:75: finishing rollback tid 1 to smithi06387500-37 2024-08-17T16:54:56.212 INFO:tasks.rados.rados.0.smithi063.stdout:74: finishing write tid 2 to smithi06387500-33 2024-08-17T16:54:56.212 INFO:tasks.rados.rados.0.smithi063.stdout:74: finishing write tid 3 to smithi06387500-33 2024-08-17T16:54:56.212 INFO:tasks.rados.rados.0.smithi063.stdout:75: finishing rollback tid 2 to smithi06387500-37 2024-08-17T16:54:56.212 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 35 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-08-17T16:54:56.212 INFO:tasks.rados.rados.0.smithi063.stdout:74: finishing write tid 4 to smithi06387500-33 2024-08-17T16:54:56.212 INFO:tasks.rados.rados.0.smithi063.stdout:74: oid 33 updating version 0 to 15 2024-08-17T16:54:56.212 INFO:tasks.rados.rados.0.smithi063.stdout:74: oid 33 updating version 15 to 17 2024-08-17T16:54:56.212 INFO:tasks.rados.rados.0.smithi063.stdout:74: oid 33 version 17 is already newer than 16 2024-08-17T16:54:56.212 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 17 (ObjNum 64 snap 1 seq_num 64) dirty exists 2024-08-17T16:54:56.212 INFO:tasks.rados.rados.0.smithi063.stdout:74: left oid 33 (ObjNum 64 snap 1 seq_num 64) 2024-08-17T16:54:57.182 INFO:tasks.rados.rados.0.smithi063.stdout:73: done (3 left) 2024-08-17T16:54:57.182 INFO:tasks.rados.rados.0.smithi063.stdout:74: done (2 left) 2024-08-17T16:54:57.182 INFO:tasks.rados.rados.0.smithi063.stdout:75: done (1 left) 2024-08-17T16:54:57.182 INFO:tasks.rados.rados.0.smithi063.stdout:76: done (0 left) 2024-08-17T16:54:57.182 INFO:tasks.rados.rados.0.smithi063.stdout:77: read oid 33 snap -1 2024-08-17T16:54:57.182 INFO:tasks.rados.rados.0.smithi063.stdout:77: expect (ObjNum 64 snap 1 seq_num 64) 2024-08-17T16:54:57.182 INFO:tasks.rados.rados.0.smithi063.stdout:78: snap_create 2024-08-17T16:54:57.187 INFO:tasks.rados.rados.0.smithi063.stdout:77: expect (ObjNum 64 snap 1 seq_num 64) 2024-08-17T16:54:57.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:57 smithi063 ceph-mon[42680]: from='client.24716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:54:57.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:57 smithi063 ceph-mon[42680]: osdmap e80: 8 total, 8 up, 8 in 2024-08-17T16:54:57.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:57 smithi063 ceph-mon[42680]: pgmap v101: 89 pgs: 89 active+clean; 117 MiB data, 418 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 11 MiB/s wr, 69 op/s 2024-08-17T16:54:57.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:57 smithi063 ceph-mon[31063]: from='client.24716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:54:57.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:57 smithi063 ceph-mon[31063]: osdmap e80: 8 total, 8 up, 8 in 2024-08-17T16:54:57.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:57 smithi063 ceph-mon[31063]: pgmap v101: 89 pgs: 89 active+clean; 117 MiB data, 418 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 11 MiB/s wr, 69 op/s 2024-08-17T16:54:57.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:57 smithi162 ceph-mon[34633]: from='client.24716 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:54:57.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:57 smithi162 ceph-mon[34633]: osdmap e80: 8 total, 8 up, 8 in 2024-08-17T16:54:57.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:57 smithi162 ceph-mon[34633]: pgmap v101: 89 pgs: 89 active+clean; 117 MiB data, 418 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 11 MiB/s wr, 69 op/s 2024-08-17T16:54:58.183 INFO:tasks.rados.rados.0.smithi063.stdout:77: done (1 left) 2024-08-17T16:54:58.183 INFO:tasks.rados.rados.0.smithi063.stdout:78: done (0 left) 2024-08-17T16:54:58.183 INFO:tasks.rados.rados.0.smithi063.stdout:79: rollback oid 9 current snap is 3 2024-08-17T16:54:58.183 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 9 to 0 2024-08-17T16:54:58.183 INFO:tasks.rados.rados.0.smithi063.stdout:80: snap_create 2024-08-17T16:54:58.188 INFO:tasks.rados.rados.0.smithi063.stdout:79: finishing rollback tid 0 to smithi06387500-9 2024-08-17T16:54:58.189 INFO:tasks.rados.rados.0.smithi063.stdout:79: finishing rollback tid 1 to smithi06387500-9 2024-08-17T16:54:58.189 INFO:tasks.rados.rados.0.smithi063.stdout:79: finishing rollback tid 2 to smithi06387500-9 2024-08-17T16:54:58.189 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 23 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-08-17T16:54:58.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:58 smithi063 ceph-mon[42680]: osdmap e81: 8 total, 8 up, 8 in 2024-08-17T16:54:58.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:58 smithi063 ceph-mon[31063]: osdmap e81: 8 total, 8 up, 8 in 2024-08-17T16:54:58.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:58 smithi162 ceph-mon[34633]: osdmap e81: 8 total, 8 up, 8 in 2024-08-17T16:54:59.183 INFO:tasks.rados.rados.0.smithi063.stdout:79: done (1 left) 2024-08-17T16:54:59.183 INFO:tasks.rados.rados.0.smithi063.stdout:80: done (0 left) 2024-08-17T16:54:59.183 INFO:tasks.rados.rados.0.smithi063.stdout:81: read oid 26 snap -1 2024-08-17T16:54:59.184 INFO:tasks.rados.rados.0.smithi063.stdout:81: expect (ObjNum 25 snap 0 seq_num 25) 2024-08-17T16:54:59.184 INFO:tasks.rados.rados.0.smithi063.stdout:82: read oid 36 snap -1 2024-08-17T16:54:59.184 INFO:tasks.rados.rados.0.smithi063.stdout:82: expect (ObjNum 35 snap 0 seq_num 35) 2024-08-17T16:54:59.184 INFO:tasks.rados.rados.0.smithi063.stdout:83: snap_create 2024-08-17T16:54:59.188 INFO:tasks.rados.rados.0.smithi063.stdout:82: expect (ObjNum 35 snap 0 seq_num 35) 2024-08-17T16:54:59.277 INFO:tasks.rados.rados.0.smithi063.stdout:81: expect (ObjNum 25 snap 0 seq_num 25) 2024-08-17T16:54:59.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:59 smithi063 ceph-mon[42680]: osdmap e82: 8 total, 8 up, 8 in 2024-08-17T16:54:59.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:59 smithi063 ceph-mon[42680]: pgmap v104: 89 pgs: 89 active+clean; 117 MiB data, 418 MiB used, 715 GiB / 715 GiB avail; 9.3 MiB/s rd, 16 MiB/s wr, 97 op/s 2024-08-17T16:54:59.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:54:59 smithi063 ceph-mon[42680]: osdmap e83: 8 total, 8 up, 8 in 2024-08-17T16:54:59.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:59 smithi063 ceph-mon[31063]: osdmap e82: 8 total, 8 up, 8 in 2024-08-17T16:54:59.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:59 smithi063 ceph-mon[31063]: pgmap v104: 89 pgs: 89 active+clean; 117 MiB data, 418 MiB used, 715 GiB / 715 GiB avail; 9.3 MiB/s rd, 16 MiB/s wr, 97 op/s 2024-08-17T16:54:59.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:54:59 smithi063 ceph-mon[31063]: osdmap e83: 8 total, 8 up, 8 in 2024-08-17T16:54:59.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:59 smithi162 ceph-mon[34633]: osdmap e82: 8 total, 8 up, 8 in 2024-08-17T16:54:59.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:59 smithi162 ceph-mon[34633]: pgmap v104: 89 pgs: 89 active+clean; 117 MiB data, 418 MiB used, 715 GiB / 715 GiB avail; 9.3 MiB/s rd, 16 MiB/s wr, 97 op/s 2024-08-17T16:54:59.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:54:59 smithi162 ceph-mon[34633]: osdmap e83: 8 total, 8 up, 8 in 2024-08-17T16:55:00.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:54:59 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:54:59] "GET /metrics HTTP/1.1" 200 36074 "" "Prometheus/2.43.0" 2024-08-17T16:55:00.185 INFO:tasks.rados.rados.0.smithi063.stdout:81: done (2 left) 2024-08-17T16:55:00.185 INFO:tasks.rados.rados.0.smithi063.stdout:82: done (1 left) 2024-08-17T16:55:00.185 INFO:tasks.rados.rados.0.smithi063.stdout:83: done (0 left) 2024-08-17T16:55:00.185 INFO:tasks.rados.rados.0.smithi063.stdout:84: delete oid 7 current snap is 5 2024-08-17T16:55:00.188 INFO:tasks.rados.rados.0.smithi063.stdout:84: done (0 left) 2024-08-17T16:55:00.188 INFO:tasks.rados.rados.0.smithi063.stdout:85: write oid 48 current snap is 5 2024-08-17T16:55:00.188 INFO:tasks.rados.rados.0.smithi063.stdout:85: seq_num 66 ranges {709262=697258,2012635=572018,3046388=62874} 2024-08-17T16:55:00.196 INFO:tasks.rados.rados.0.smithi063.stdout:85: writing smithi06387500-48 from 709262 to 1406520 tid 1 2024-08-17T16:55:00.203 INFO:tasks.rados.rados.0.smithi063.stdout:85: writing smithi06387500-48 from 2012635 to 2584653 tid 2 2024-08-17T16:55:00.206 INFO:tasks.rados.rados.0.smithi063.stdout:85: writing smithi06387500-48 from 3046388 to 3109262 tid 3 2024-08-17T16:55:00.206 INFO:tasks.rados.rados.0.smithi063.stdout:86: snap_create 2024-08-17T16:55:00.206 INFO:tasks.rados.rados.0.smithi063.stdout:85: finishing write tid 1 to smithi06387500-48 2024-08-17T16:55:00.206 INFO:tasks.rados.rados.0.smithi063.stdout:85: finishing write tid 2 to smithi06387500-48 2024-08-17T16:55:00.208 INFO:tasks.rados.rados.0.smithi063.stdout:85: finishing write tid 3 to smithi06387500-48 2024-08-17T16:55:00.208 INFO:tasks.rados.rados.0.smithi063.stdout:85: finishing write tid 4 to smithi06387500-48 2024-08-17T16:55:00.208 INFO:tasks.rados.rados.0.smithi063.stdout:85: finishing write tid 5 to smithi06387500-48 2024-08-17T16:55:00.208 INFO:tasks.rados.rados.0.smithi063.stdout:85: oid 48 updating version 0 to 39 2024-08-17T16:55:00.208 INFO:tasks.rados.rados.0.smithi063.stdout:85: oid 48 version 39 is already newer than 38 2024-08-17T16:55:00.208 INFO:tasks.rados.rados.0.smithi063.stdout:85: oid 48 updating version 39 to 40 2024-08-17T16:55:00.208 INFO:tasks.rados.rados.0.smithi063.stdout:85: oid 48 version 40 is already newer than 37 2024-08-17T16:55:00.208 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 40 (ObjNum 66 snap 5 seq_num 66) dirty exists 2024-08-17T16:55:00.208 INFO:tasks.rados.rados.0.smithi063.stdout:85: left oid 48 (ObjNum 66 snap 5 seq_num 66) 2024-08-17T16:55:01.193 INFO:tasks.rados.rados.0.smithi063.stdout:85: done (1 left) 2024-08-17T16:55:01.193 INFO:tasks.rados.rados.0.smithi063.stdout:86: done (0 left) 2024-08-17T16:55:01.193 INFO:tasks.rados.rados.0.smithi063.stdout:87: rollback oid 10 current snap is 6 2024-08-17T16:55:01.193 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 10 to 2 2024-08-17T16:55:01.193 INFO:tasks.rados.rados.0.smithi063.stdout:88: snap_create 2024-08-17T16:55:01.198 INFO:tasks.rados.rados.0.smithi063.stdout:87: finishing rollback tid 0 to smithi06387500-10 2024-08-17T16:55:01.198 INFO:tasks.rados.rados.0.smithi063.stdout:87: finishing rollback tid 1 to smithi06387500-10 2024-08-17T16:55:01.199 INFO:tasks.rados.rados.0.smithi063.stdout:87: finishing rollback tid 2 to smithi06387500-10 2024-08-17T16:55:01.199 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 13 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-08-17T16:55:01.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:01 smithi063 ceph-mon[31063]: osdmap e84: 8 total, 8 up, 8 in 2024-08-17T16:55:01.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:01 smithi063 ceph-mon[31063]: pgmap v107: 89 pgs: 89 active+clean; 117 MiB data, 418 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:01.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:01 smithi063 ceph-mon[42680]: osdmap e84: 8 total, 8 up, 8 in 2024-08-17T16:55:01.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:01 smithi063 ceph-mon[42680]: pgmap v107: 89 pgs: 89 active+clean; 117 MiB data, 418 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:01.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:01 smithi162 ceph-mon[34633]: osdmap e84: 8 total, 8 up, 8 in 2024-08-17T16:55:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:01 smithi162 ceph-mon[34633]: pgmap v107: 89 pgs: 89 active+clean; 117 MiB data, 418 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:02.195 INFO:tasks.rados.rados.0.smithi063.stdout:87: done (1 left) 2024-08-17T16:55:02.195 INFO:tasks.rados.rados.0.smithi063.stdout:88: done (0 left) 2024-08-17T16:55:02.195 INFO:tasks.rados.rados.0.smithi063.stdout:89: delete oid 23 current snap is 7 2024-08-17T16:55:02.202 INFO:tasks.rados.rados.0.smithi063.stdout:89: done (0 left) 2024-08-17T16:55:02.203 INFO:tasks.rados.rados.0.smithi063.stdout:90: read oid 27 snap 3 2024-08-17T16:55:02.203 INFO:tasks.rados.rados.0.smithi063.stdout:90: expect (ObjNum 26 snap 0 seq_num 26) 2024-08-17T16:55:02.203 INFO:tasks.rados.rados.0.smithi063.stdout:91: write oid 12 current snap is 7 2024-08-17T16:55:02.203 INFO:tasks.rados.rados.0.smithi063.stdout:91: seq_num 68 ranges {742876=447756,1729599=575687,3101234=41642} 2024-08-17T16:55:02.210 INFO:tasks.rados.rados.0.smithi063.stdout:91: writing smithi06387500-12 from 742876 to 1190632 tid 1 2024-08-17T16:55:02.217 INFO:tasks.rados.rados.0.smithi063.stdout:91: writing smithi06387500-12 from 1729599 to 2305286 tid 2 2024-08-17T16:55:02.222 INFO:tasks.rados.rados.0.smithi063.stdout:91: writing smithi06387500-12 from 3101234 to 3142876 tid 3 2024-08-17T16:55:02.222 INFO:tasks.rados.rados.0.smithi063.stdout:92: snap_create 2024-08-17T16:55:02.223 INFO:tasks.rados.rados.0.smithi063.stdout:90: expect (ObjNum 26 snap 0 seq_num 26) 2024-08-17T16:55:02.320 INFO:tasks.rados.rados.0.smithi063.stdout:91: finishing write tid 1 to smithi06387500-12 2024-08-17T16:55:02.320 INFO:tasks.rados.rados.0.smithi063.stdout:91: finishing write tid 2 to smithi06387500-12 2024-08-17T16:55:02.320 INFO:tasks.rados.rados.0.smithi063.stdout:91: finishing write tid 3 to smithi06387500-12 2024-08-17T16:55:02.320 INFO:tasks.rados.rados.0.smithi063.stdout:91: finishing write tid 4 to smithi06387500-12 2024-08-17T16:55:02.321 INFO:tasks.rados.rados.0.smithi063.stdout:91: finishing write tid 5 to smithi06387500-12 2024-08-17T16:55:02.321 INFO:tasks.rados.rados.0.smithi063.stdout:91: oid 12 updating version 0 to 10 2024-08-17T16:55:02.321 INFO:tasks.rados.rados.0.smithi063.stdout:91: oid 12 updating version 10 to 11 2024-08-17T16:55:02.321 INFO:tasks.rados.rados.0.smithi063.stdout:91: oid 12 version 11 is already newer than 9 2024-08-17T16:55:02.321 INFO:tasks.rados.rados.0.smithi063.stdout:91: oid 12 version 11 is already newer than 8 2024-08-17T16:55:02.321 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 11 (ObjNum 68 snap 7 seq_num 68) dirty exists 2024-08-17T16:55:02.321 INFO:tasks.rados.rados.0.smithi063.stdout:91: left oid 12 (ObjNum 68 snap 7 seq_num 68) 2024-08-17T16:55:02.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:02 smithi063 ceph-mon[42680]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T16:55:02.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:02 smithi063 ceph-mon[31063]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T16:55:02.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:02 smithi162 ceph-mon[34633]: osdmap e85: 8 total, 8 up, 8 in 2024-08-17T16:55:03.210 INFO:tasks.rados.rados.0.smithi063.stdout:90: done (2 left) 2024-08-17T16:55:03.210 INFO:tasks.rados.rados.0.smithi063.stdout:91: done (1 left) 2024-08-17T16:55:03.210 INFO:tasks.rados.rados.0.smithi063.stdout:92: done (0 left) 2024-08-17T16:55:03.210 INFO:tasks.rados.rados.0.smithi063.stdout:93: snap_remove snap 7 2024-08-17T16:55:03.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:03 smithi063 ceph-mon[42680]: osdmap e86: 8 total, 8 up, 8 in 2024-08-17T16:55:03.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:03 smithi063 ceph-mon[42680]: pgmap v110: 89 pgs: 89 active+clean; 118 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 325 KiB/s wr, 8 op/s 2024-08-17T16:55:03.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:03 smithi063 ceph-mon[31063]: osdmap e86: 8 total, 8 up, 8 in 2024-08-17T16:55:03.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:03 smithi063 ceph-mon[31063]: pgmap v110: 89 pgs: 89 active+clean; 118 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 325 KiB/s wr, 8 op/s 2024-08-17T16:55:03.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:03 smithi162 ceph-mon[34633]: osdmap e86: 8 total, 8 up, 8 in 2024-08-17T16:55:03.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:03 smithi162 ceph-mon[34633]: pgmap v110: 89 pgs: 89 active+clean; 118 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 325 KiB/s wr, 8 op/s 2024-08-17T16:55:04.218 INFO:tasks.rados.rados.0.smithi063.stdout:93: done (0 left) 2024-08-17T16:55:04.218 INFO:tasks.rados.rados.0.smithi063.stdout:94: write oid 15 current snap is 8 2024-08-17T16:55:04.219 INFO:tasks.rados.rados.0.smithi063.stdout:94: seq_num 69 ranges {759683=639358,1904434=719345,3128657=31026} 2024-08-17T16:55:04.226 INFO:tasks.rados.rados.0.smithi063.stdout:94: writing smithi06387500-15 from 759683 to 1399041 tid 1 2024-08-17T16:55:04.233 INFO:tasks.rados.rados.0.smithi063.stdout:94: writing smithi06387500-15 from 1904434 to 2623779 tid 2 2024-08-17T16:55:04.236 INFO:tasks.rados.rados.0.smithi063.stdout:94: writing smithi06387500-15 from 3128657 to 3159683 tid 3 2024-08-17T16:55:04.236 INFO:tasks.rados.rados.0.smithi063.stdout:95: snap_create 2024-08-17T16:55:04.236 INFO:tasks.rados.rados.0.smithi063.stdout:94: finishing write tid 1 to smithi06387500-15 2024-08-17T16:55:04.236 INFO:tasks.rados.rados.0.smithi063.stdout:94: finishing write tid 2 to smithi06387500-15 2024-08-17T16:55:04.237 INFO:tasks.rados.rados.0.smithi063.stdout:94: finishing write tid 3 to smithi06387500-15 2024-08-17T16:55:04.238 INFO:tasks.rados.rados.0.smithi063.stdout:94: finishing write tid 4 to smithi06387500-15 2024-08-17T16:55:04.238 INFO:tasks.rados.rados.0.smithi063.stdout:94: finishing write tid 5 to smithi06387500-15 2024-08-17T16:55:04.238 INFO:tasks.rados.rados.0.smithi063.stdout:94: oid 15 updating version 0 to 17 2024-08-17T16:55:04.238 INFO:tasks.rados.rados.0.smithi063.stdout:94: oid 15 version 17 is already newer than 16 2024-08-17T16:55:04.238 INFO:tasks.rados.rados.0.smithi063.stdout:94: oid 15 version 17 is already newer than 15 2024-08-17T16:55:04.238 INFO:tasks.rados.rados.0.smithi063.stdout:94: oid 15 version 17 is already newer than 14 2024-08-17T16:55:04.239 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 17 (ObjNum 69 snap 8 seq_num 69) dirty exists 2024-08-17T16:55:04.239 INFO:tasks.rados.rados.0.smithi063.stdout:94: left oid 15 (ObjNum 69 snap 8 seq_num 69) 2024-08-17T16:55:04.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:04 smithi063 ceph-mon[31063]: osdmap e87: 8 total, 8 up, 8 in 2024-08-17T16:55:04.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:04 smithi063 ceph-mon[42680]: osdmap e87: 8 total, 8 up, 8 in 2024-08-17T16:55:04.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:04 smithi162 ceph-mon[34633]: osdmap e87: 8 total, 8 up, 8 in 2024-08-17T16:55:05.223 INFO:tasks.rados.rados.0.smithi063.stdout:94: done (1 left) 2024-08-17T16:55:05.223 INFO:tasks.rados.rados.0.smithi063.stdout:95: done (0 left) 2024-08-17T16:55:05.223 INFO:tasks.rados.rados.0.smithi063.stdout:96: rollback oid 42 current snap is 9 2024-08-17T16:55:05.223 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 42 to 5 2024-08-17T16:55:05.223 INFO:tasks.rados.rados.0.smithi063.stdout:97: write oid 3 current snap is 9 2024-08-17T16:55:05.223 INFO:tasks.rados.rados.0.smithi063.stdout:97: seq_num 70 ranges {776490=714607,2246563=579356} 2024-08-17T16:55:05.231 INFO:tasks.rados.rados.0.smithi063.stdout:97: writing smithi06387500-3 from 776490 to 1491097 tid 1 2024-08-17T16:55:05.238 INFO:tasks.rados.rados.0.smithi063.stdout:97: writing smithi06387500-3 from 2246563 to 2825919 tid 2 2024-08-17T16:55:05.239 INFO:tasks.rados.rados.0.smithi063.stdout:98: delete oid 4 current snap is 9 2024-08-17T16:55:05.239 INFO:tasks.rados.rados.0.smithi063.stdout:96: finishing rollback tid 0 to smithi06387500-42 2024-08-17T16:55:05.239 INFO:tasks.rados.rados.0.smithi063.stdout:96: finishing rollback tid 1 to smithi06387500-42 2024-08-17T16:55:05.239 INFO:tasks.rados.rados.0.smithi063.stdout:96: finishing rollback tid 2 to smithi06387500-42 2024-08-17T16:55:05.239 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 33 (ObjNum 61 snap 0 seq_num 61) dirty exists 2024-08-17T16:55:05.239 INFO:tasks.rados.rados.0.smithi063.stdout:97: finishing write tid 1 to smithi06387500-3 2024-08-17T16:55:05.240 INFO:tasks.rados.rados.0.smithi063.stdout:96: done (2 left) 2024-08-17T16:55:05.241 INFO:tasks.rados.rados.0.smithi063.stdout:98: done (1 left) 2024-08-17T16:55:05.241 INFO:tasks.rados.rados.0.smithi063.stdout:99: read oid 46 snap -1 2024-08-17T16:55:05.241 INFO:tasks.rados.rados.0.smithi063.stdout:99: expect (ObjNum 45 snap 0 seq_num 45) 2024-08-17T16:55:05.241 INFO:tasks.rados.rados.0.smithi063.stdout:100: write oid 36 current snap is 9 2024-08-17T16:55:05.241 INFO:tasks.rados.rados.0.smithi063.stdout:100: seq_num 72 ranges {410104=465105,1563527=583025,2810926=399178} 2024-08-17T16:55:05.246 INFO:tasks.rados.rados.0.smithi063.stdout:100: writing smithi06387500-36 from 410104 to 875209 tid 1 2024-08-17T16:55:05.253 INFO:tasks.rados.rados.0.smithi063.stdout:100: writing smithi06387500-36 from 1563527 to 2146552 tid 2 2024-08-17T16:55:05.259 INFO:tasks.rados.rados.0.smithi063.stdout:100: writing smithi06387500-36 from 2810926 to 3210104 tid 3 2024-08-17T16:55:05.259 INFO:tasks.rados.rados.0.smithi063.stdout:101: snap_remove snap 4 2024-08-17T16:55:05.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:05 smithi063 ceph-mon[42680]: osdmap e88: 8 total, 8 up, 8 in 2024-08-17T16:55:05.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:05 smithi063 ceph-mon[42680]: pgmap v113: 89 pgs: 89 active+clean; 118 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 325 KiB/s wr, 8 op/s 2024-08-17T16:55:05.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:05 smithi063 ceph-mon[31063]: osdmap e88: 8 total, 8 up, 8 in 2024-08-17T16:55:05.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:05 smithi063 ceph-mon[31063]: pgmap v113: 89 pgs: 89 active+clean; 118 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 325 KiB/s wr, 8 op/s 2024-08-17T16:55:05.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:05 smithi162 ceph-mon[34633]: osdmap e88: 8 total, 8 up, 8 in 2024-08-17T16:55:05.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:05 smithi162 ceph-mon[34633]: pgmap v113: 89 pgs: 89 active+clean; 118 MiB data, 404 MiB used, 715 GiB / 715 GiB avail; 3.8 MiB/s rd, 325 KiB/s wr, 8 op/s 2024-08-17T16:55:06.228 INFO:tasks.rados.rados.0.smithi063.stdout:101: done (3 left) 2024-08-17T16:55:06.228 INFO:tasks.rados.rados.0.smithi063.stdout:102: write oid 41 current snap is 9 2024-08-17T16:55:06.229 INFO:tasks.rados.rados.0.smithi063.stdout:102: seq_num 73 ranges {426911=540354,1622009=726683,3121996=104915} 2024-08-17T16:55:06.235 INFO:tasks.rados.rados.0.smithi063.stdout:102: writing smithi06387500-41 from 426911 to 967265 tid 1 2024-08-17T16:55:06.243 INFO:tasks.rados.rados.0.smithi063.stdout:102: writing smithi06387500-41 from 1622009 to 2348692 tid 2 2024-08-17T16:55:06.248 INFO:tasks.rados.rados.0.smithi063.stdout:102: writing smithi06387500-41 from 3121996 to 3226911 tid 3 2024-08-17T16:55:06.248 INFO:tasks.rados.rados.0.smithi063.stdout:103: snap_remove snap 5 2024-08-17T16:55:06.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:06 smithi063 ceph-mon[42680]: osdmap e89: 8 total, 8 up, 8 in 2024-08-17T16:55:06.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:06 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:06.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:06 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:55:06.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:06 smithi063 ceph-mon[31063]: osdmap e89: 8 total, 8 up, 8 in 2024-08-17T16:55:06.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:06 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:06.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:06 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:55:06.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:06 smithi162 ceph-mon[34633]: osdmap e89: 8 total, 8 up, 8 in 2024-08-17T16:55:06.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:06 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:06.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:06 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:55:07.241 INFO:tasks.rados.rados.0.smithi063.stdout:103: done (4 left) 2024-08-17T16:55:07.241 INFO:tasks.rados.rados.0.smithi063.stdout:104: delete oid 15 current snap is 9 2024-08-17T16:55:07.241 INFO:tasks.rados.rados.0.smithi063.stdout:97: finishing write tid 2 to smithi06387500-3 2024-08-17T16:55:07.241 INFO:tasks.rados.rados.0.smithi063.stdout:97: finishing write tid 3 to smithi06387500-3 2024-08-17T16:55:07.241 INFO:tasks.rados.rados.0.smithi063.stdout:97: finishing write tid 4 to smithi06387500-3 2024-08-17T16:55:07.241 INFO:tasks.rados.rados.0.smithi063.stdout:97: oid 3 updating version 0 to 11 2024-08-17T16:55:07.241 INFO:tasks.rados.rados.0.smithi063.stdout:97: oid 3 updating version 11 to 13 2024-08-17T16:55:07.242 INFO:tasks.rados.rados.0.smithi063.stdout:97: oid 3 version 13 is already newer than 12 2024-08-17T16:55:07.242 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 13 (ObjNum 70 snap 9 seq_num 70) dirty exists 2024-08-17T16:55:07.242 INFO:tasks.rados.rados.0.smithi063.stdout:97: left oid 3 (ObjNum 70 snap 9 seq_num 70) 2024-08-17T16:55:07.242 INFO:tasks.rados.rados.0.smithi063.stdout:99: expect (ObjNum 45 snap 0 seq_num 45) 2024-08-17T16:55:07.353 INFO:tasks.rados.rados.0.smithi063.stdout:100: finishing write tid 1 to smithi06387500-36 2024-08-17T16:55:07.353 INFO:tasks.rados.rados.0.smithi063.stdout:100: finishing write tid 2 to smithi06387500-36 2024-08-17T16:55:07.353 INFO:tasks.rados.rados.0.smithi063.stdout:100: finishing write tid 3 to smithi06387500-36 2024-08-17T16:55:07.353 INFO:tasks.rados.rados.0.smithi063.stdout:100: finishing write tid 4 to smithi06387500-36 2024-08-17T16:55:07.353 INFO:tasks.rados.rados.0.smithi063.stdout:100: finishing write tid 5 to smithi06387500-36 2024-08-17T16:55:07.353 INFO:tasks.rados.rados.0.smithi063.stdout:100: oid 36 updating version 0 to 45 2024-08-17T16:55:07.354 INFO:tasks.rados.rados.0.smithi063.stdout:100: oid 36 version 45 is already newer than 44 2024-08-17T16:55:07.354 INFO:tasks.rados.rados.0.smithi063.stdout:100: oid 36 updating version 45 to 47 2024-08-17T16:55:07.354 INFO:tasks.rados.rados.0.smithi063.stdout:100: oid 36 version 47 is already newer than 46 2024-08-17T16:55:07.354 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 47 (ObjNum 72 snap 9 seq_num 72) dirty exists 2024-08-17T16:55:07.354 INFO:tasks.rados.rados.0.smithi063.stdout:100: left oid 36 (ObjNum 72 snap 9 seq_num 72) 2024-08-17T16:55:07.354 INFO:tasks.rados.rados.0.smithi063.stdout:102: finishing write tid 1 to smithi06387500-41 2024-08-17T16:55:07.354 INFO:tasks.rados.rados.0.smithi063.stdout:102: finishing write tid 2 to smithi06387500-41 2024-08-17T16:55:07.354 INFO:tasks.rados.rados.0.smithi063.stdout:102: finishing write tid 3 to smithi06387500-41 2024-08-17T16:55:07.354 INFO:tasks.rados.rados.0.smithi063.stdout:102: finishing write tid 4 to smithi06387500-41 2024-08-17T16:55:07.354 INFO:tasks.rados.rados.0.smithi063.stdout:102: finishing write tid 5 to smithi06387500-41 2024-08-17T16:55:07.354 INFO:tasks.rados.rados.0.smithi063.stdout:102: oid 41 updating version 0 to 6 2024-08-17T16:55:07.354 INFO:tasks.rados.rados.0.smithi063.stdout:102: oid 41 updating version 6 to 8 2024-08-17T16:55:07.354 INFO:tasks.rados.rados.0.smithi063.stdout:102: oid 41 version 8 is already newer than 7 2024-08-17T16:55:07.354 INFO:tasks.rados.rados.0.smithi063.stdout:102: oid 41 version 8 is already newer than 5 2024-08-17T16:55:07.355 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 8 (ObjNum 73 snap 9 seq_num 73) dirty exists 2024-08-17T16:55:07.355 INFO:tasks.rados.rados.0.smithi063.stdout:102: left oid 41 (ObjNum 73 snap 9 seq_num 73) 2024-08-17T16:55:07.355 INFO:tasks.rados.rados.0.smithi063.stdout:97: done (4 left) 2024-08-17T16:55:07.355 INFO:tasks.rados.rados.0.smithi063.stdout:99: done (3 left) 2024-08-17T16:55:07.355 INFO:tasks.rados.rados.0.smithi063.stdout:100: done (2 left) 2024-08-17T16:55:07.355 INFO:tasks.rados.rados.0.smithi063.stdout:102: done (1 left) 2024-08-17T16:55:07.355 INFO:tasks.rados.rados.0.smithi063.stdout:104: done (0 left) 2024-08-17T16:55:07.355 INFO:tasks.rados.rados.0.smithi063.stdout:105: write oid 14 current snap is 9 2024-08-17T16:55:07.355 INFO:tasks.rados.rados.0.smithi063.stdout:105: seq_num 75 ranges {460525=690852,1622620=730352,3060489=200036} 2024-08-17T16:55:07.358 INFO:tasks.rados.rados.0.smithi063.stdout:105: writing smithi06387500-14 from 460525 to 1151377 tid 1 2024-08-17T16:55:07.365 INFO:tasks.rados.rados.0.smithi063.stdout:105: writing smithi06387500-14 from 1622620 to 2352972 tid 2 2024-08-17T16:55:07.370 INFO:tasks.rados.rados.0.smithi063.stdout:105: writing smithi06387500-14 from 3060489 to 3260525 tid 3 2024-08-17T16:55:07.371 INFO:tasks.rados.rados.0.smithi063.stdout:106: rollback oid 30 current snap is 9 2024-08-17T16:55:07.371 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 30 to 2 2024-08-17T16:55:07.371 INFO:tasks.rados.rados.0.smithi063.stdout:105: finishing write tid 1 to smithi06387500-14 2024-08-17T16:55:07.371 INFO:tasks.rados.rados.0.smithi063.stdout:105: finishing write tid 2 to smithi06387500-14 2024-08-17T16:55:07.371 INFO:tasks.rados.rados.0.smithi063.stdout:107: delete oid 10 current snap is 9 2024-08-17T16:55:07.373 INFO:tasks.rados.rados.0.smithi063.stdout:107: done (2 left) 2024-08-17T16:55:07.373 INFO:tasks.rados.rados.0.smithi063.stdout:108: rollback oid 37 current snap is 9 2024-08-17T16:55:07.373 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 2 2024-08-17T16:55:07.373 INFO:tasks.rados.rados.0.smithi063.stdout:109: write oid 13 current snap is 9 2024-08-17T16:55:07.373 INFO:tasks.rados.rados.0.smithi063.stdout:109: seq_num 77 ranges {494139=557703,1455937=734021,2948041=346098} 2024-08-17T16:55:07.379 INFO:tasks.rados.rados.0.smithi063.stdout:109: writing smithi06387500-13 from 494139 to 1051842 tid 1 2024-08-17T16:55:07.385 INFO:tasks.rados.rados.0.smithi063.stdout:109: writing smithi06387500-13 from 1455937 to 2189958 tid 2 2024-08-17T16:55:07.392 INFO:tasks.rados.rados.0.smithi063.stdout:109: writing smithi06387500-13 from 2948041 to 3294139 tid 3 2024-08-17T16:55:07.392 INFO:tasks.rados.rados.0.smithi063.stdout:110: write oid 25 current snap is 9 2024-08-17T16:55:07.392 INFO:tasks.rados.rados.0.smithi063.stdout:105: finishing write tid 3 to smithi06387500-14 2024-08-17T16:55:07.392 INFO:tasks.rados.rados.0.smithi063.stdout:110: seq_num 78 ranges {510946=632952,1798066=477679,2742758=457217} 2024-08-17T16:55:07.398 INFO:tasks.rados.rados.0.smithi063.stdout:110: writing smithi06387500-25 from 510946 to 1143898 tid 1 2024-08-17T16:55:07.404 INFO:tasks.rados.rados.0.smithi063.stdout:110: writing smithi06387500-25 from 1798066 to 2275745 tid 2 2024-08-17T16:55:07.410 INFO:tasks.rados.rados.0.smithi063.stdout:110: writing smithi06387500-25 from 2742758 to 3199975 tid 3 2024-08-17T16:55:07.410 INFO:tasks.rados.rados.0.smithi063.stdout:111: snap_create 2024-08-17T16:55:07.410 INFO:tasks.rados.rados.0.smithi063.stdout:106: finishing rollback tid 0 to smithi06387500-30 2024-08-17T16:55:07.410 INFO:tasks.rados.rados.0.smithi063.stdout:106: finishing rollback tid 1 to smithi06387500-30 2024-08-17T16:55:07.410 INFO:tasks.rados.rados.0.smithi063.stdout:106: finishing rollback tid 2 to smithi06387500-30 2024-08-17T16:55:07.410 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 31 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-08-17T16:55:07.410 INFO:tasks.rados.rados.0.smithi063.stdout:105: finishing write tid 4 to smithi06387500-14 2024-08-17T16:55:07.411 INFO:tasks.rados.rados.0.smithi063.stdout:105: finishing write tid 5 to smithi06387500-14 2024-08-17T16:55:07.411 INFO:tasks.rados.rados.0.smithi063.stdout:105: oid 14 updating version 0 to 61 2024-08-17T16:55:07.411 INFO:tasks.rados.rados.0.smithi063.stdout:105: oid 14 updating version 61 to 62 2024-08-17T16:55:07.411 INFO:tasks.rados.rados.0.smithi063.stdout:105: oid 14 version 62 is already newer than 59 2024-08-17T16:55:07.411 INFO:tasks.rados.rados.0.smithi063.stdout:105: oid 14 version 62 is already newer than 60 2024-08-17T16:55:07.411 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 62 (ObjNum 75 snap 9 seq_num 75) dirty exists 2024-08-17T16:55:07.411 INFO:tasks.rados.rados.0.smithi063.stdout:105: left oid 14 (ObjNum 75 snap 9 seq_num 75) 2024-08-17T16:55:07.411 INFO:tasks.rados.rados.0.smithi063.stdout:108: finishing rollback tid 0 to smithi06387500-37 2024-08-17T16:55:07.411 INFO:tasks.rados.rados.0.smithi063.stdout:108: finishing rollback tid 1 to smithi06387500-37 2024-08-17T16:55:07.411 INFO:tasks.rados.rados.0.smithi063.stdout:108: finishing rollback tid 2 to smithi06387500-37 2024-08-17T16:55:07.411 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 66 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-08-17T16:55:07.411 INFO:tasks.rados.rados.0.smithi063.stdout:109: finishing write tid 1 to smithi06387500-13 2024-08-17T16:55:07.411 INFO:tasks.rados.rados.0.smithi063.stdout:109: finishing write tid 2 to smithi06387500-13 2024-08-17T16:55:07.411 INFO:tasks.rados.rados.0.smithi063.stdout:109: finishing write tid 3 to smithi06387500-13 2024-08-17T16:55:07.412 INFO:tasks.rados.rados.0.smithi063.stdout:109: finishing write tid 4 to smithi06387500-13 2024-08-17T16:55:07.412 INFO:tasks.rados.rados.0.smithi063.stdout:109: finishing write tid 5 to smithi06387500-13 2024-08-17T16:55:07.412 INFO:tasks.rados.rados.0.smithi063.stdout:109: oid 13 updating version 0 to 39 2024-08-17T16:55:07.412 INFO:tasks.rados.rados.0.smithi063.stdout:109: oid 13 version 39 is already newer than 36 2024-08-17T16:55:07.412 INFO:tasks.rados.rados.0.smithi063.stdout:109: oid 13 version 39 is already newer than 37 2024-08-17T16:55:07.412 INFO:tasks.rados.rados.0.smithi063.stdout:109: oid 13 version 39 is already newer than 38 2024-08-17T16:55:07.412 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 39 (ObjNum 77 snap 9 seq_num 77) dirty exists 2024-08-17T16:55:07.412 INFO:tasks.rados.rados.0.smithi063.stdout:109: left oid 13 (ObjNum 77 snap 9 seq_num 77) 2024-08-17T16:55:07.412 INFO:tasks.rados.rados.0.smithi063.stdout:110: finishing write tid 1 to smithi06387500-25 2024-08-17T16:55:07.412 INFO:tasks.rados.rados.0.smithi063.stdout:110: finishing write tid 2 to smithi06387500-25 2024-08-17T16:55:07.413 INFO:tasks.rados.rados.0.smithi063.stdout:110: finishing write tid 3 to smithi06387500-25 2024-08-17T16:55:07.413 INFO:tasks.rados.rados.0.smithi063.stdout:110: finishing write tid 4 to smithi06387500-25 2024-08-17T16:55:07.413 INFO:tasks.rados.rados.0.smithi063.stdout:110: finishing write tid 5 to smithi06387500-25 2024-08-17T16:55:07.413 INFO:tasks.rados.rados.0.smithi063.stdout:110: oid 25 updating version 0 to 71 2024-08-17T16:55:07.413 INFO:tasks.rados.rados.0.smithi063.stdout:110: oid 25 version 71 is already newer than 70 2024-08-17T16:55:07.413 INFO:tasks.rados.rados.0.smithi063.stdout:110: oid 25 version 71 is already newer than 69 2024-08-17T16:55:07.413 INFO:tasks.rados.rados.0.smithi063.stdout:110: oid 25 version 71 is already newer than 68 2024-08-17T16:55:07.413 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 71 (ObjNum 78 snap 9 seq_num 78) dirty exists 2024-08-17T16:55:07.413 INFO:tasks.rados.rados.0.smithi063.stdout:110: left oid 25 (ObjNum 78 snap 9 seq_num 78) 2024-08-17T16:55:07.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:07 smithi063 ceph-mon[31063]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T16:55:07.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:07 smithi063 ceph-mon[31063]: pgmap v116: 89 pgs: 89 active+clean; 127 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-08-17T16:55:07.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:07 smithi063 ceph-mon[42680]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T16:55:07.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:07 smithi063 ceph-mon[42680]: pgmap v116: 89 pgs: 89 active+clean; 127 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-08-17T16:55:07.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:07 smithi162 ceph-mon[34633]: osdmap e90: 8 total, 8 up, 8 in 2024-08-17T16:55:07.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:07 smithi162 ceph-mon[34633]: pgmap v116: 89 pgs: 89 active+clean; 127 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-08-17T16:55:08.244 INFO:tasks.rados.rados.0.smithi063.stdout:105: done (5 left) 2024-08-17T16:55:08.244 INFO:tasks.rados.rados.0.smithi063.stdout:106: done (4 left) 2024-08-17T16:55:08.244 INFO:tasks.rados.rados.0.smithi063.stdout:108: done (3 left) 2024-08-17T16:55:08.244 INFO:tasks.rados.rados.0.smithi063.stdout:109: done (2 left) 2024-08-17T16:55:08.244 INFO:tasks.rados.rados.0.smithi063.stdout:110: done (1 left) 2024-08-17T16:55:08.245 INFO:tasks.rados.rados.0.smithi063.stdout:111: done (0 left) 2024-08-17T16:55:08.245 INFO:tasks.rados.rados.0.smithi063.stdout:112: write oid 47 current snap is 10 2024-08-17T16:55:08.245 INFO:tasks.rados.rados.0.smithi063.stdout:112: seq_num 79 ranges {527753=708201,1856548=737690,3286534=41219} 2024-08-17T16:55:08.251 INFO:tasks.rados.rados.0.smithi063.stdout:112: writing smithi06387500-47 from 527753 to 1235954 tid 1 2024-08-17T16:55:08.259 INFO:tasks.rados.rados.0.smithi063.stdout:112: writing smithi06387500-47 from 1856548 to 2594238 tid 2 2024-08-17T16:55:08.263 INFO:tasks.rados.rados.0.smithi063.stdout:112: writing smithi06387500-47 from 3286534 to 3327753 tid 3 2024-08-17T16:55:08.263 INFO:tasks.rados.rados.0.smithi063.stdout:113: delete oid 17 current snap is 10 2024-08-17T16:55:08.263 INFO:tasks.rados.rados.0.smithi063.stdout:112: finishing write tid 1 to smithi06387500-47 2024-08-17T16:55:08.264 INFO:tasks.rados.rados.0.smithi063.stdout:112: finishing write tid 2 to smithi06387500-47 2024-08-17T16:55:08.265 INFO:tasks.rados.rados.0.smithi063.stdout:112: finishing write tid 3 to smithi06387500-47 2024-08-17T16:55:08.265 INFO:tasks.rados.rados.0.smithi063.stdout:112: finishing write tid 4 to smithi06387500-47 2024-08-17T16:55:08.265 INFO:tasks.rados.rados.0.smithi063.stdout:112: finishing write tid 5 to smithi06387500-47 2024-08-17T16:55:08.265 INFO:tasks.rados.rados.0.smithi063.stdout:112: oid 47 updating version 0 to 11 2024-08-17T16:55:08.265 INFO:tasks.rados.rados.0.smithi063.stdout:112: oid 47 version 11 is already newer than 9 2024-08-17T16:55:08.265 INFO:tasks.rados.rados.0.smithi063.stdout:112: oid 47 version 11 is already newer than 8 2024-08-17T16:55:08.265 INFO:tasks.rados.rados.0.smithi063.stdout:112: oid 47 version 11 is already newer than 10 2024-08-17T16:55:08.265 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 11 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-08-17T16:55:08.265 INFO:tasks.rados.rados.0.smithi063.stdout:112: left oid 47 (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:55:08.267 INFO:tasks.rados.rados.0.smithi063.stdout:112: done (1 left) 2024-08-17T16:55:08.267 INFO:tasks.rados.rados.0.smithi063.stdout:113: done (0 left) 2024-08-17T16:55:08.267 INFO:tasks.rados.rados.0.smithi063.stdout:114: snap_remove snap 2 2024-08-17T16:55:08.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:08 smithi063 ceph-mon[42680]: osdmap e91: 8 total, 8 up, 8 in 2024-08-17T16:55:08.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:08 smithi063 ceph-mon[31063]: osdmap e91: 8 total, 8 up, 8 in 2024-08-17T16:55:08.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:08 smithi162 ceph-mon[34633]: osdmap e91: 8 total, 8 up, 8 in 2024-08-17T16:55:09.248 INFO:tasks.rados.rados.0.smithi063.stdout:114: done (0 left) 2024-08-17T16:55:09.248 INFO:tasks.rados.rados.0.smithi063.stdout:115: write oid 43 current snap is 10 2024-08-17T16:55:09.249 INFO:tasks.rados.rados.0.smithi063.stdout:115: seq_num 81 ranges {561367=458699,1573512=741359,2941380=419987} 2024-08-17T16:55:09.254 INFO:tasks.rados.rados.0.smithi063.stdout:115: writing smithi06387500-43 from 561367 to 1020066 tid 1 2024-08-17T16:55:09.261 INFO:tasks.rados.rados.0.smithi063.stdout:115: writing smithi06387500-43 from 1573512 to 2314871 tid 2 2024-08-17T16:55:09.268 INFO:tasks.rados.rados.0.smithi063.stdout:115: writing smithi06387500-43 from 2941380 to 3361367 tid 3 2024-08-17T16:55:09.268 INFO:tasks.rados.rados.0.smithi063.stdout:116: read oid 11 snap -1 2024-08-17T16:55:09.268 INFO:tasks.rados.rados.0.smithi063.stdout:116: expect (ObjNum 10 snap 0 seq_num 10) 2024-08-17T16:55:09.268 INFO:tasks.rados.rados.0.smithi063.stdout:117: rollback oid 37 current snap is 10 2024-08-17T16:55:09.268 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 1 2024-08-17T16:55:09.268 INFO:tasks.rados.rados.0.smithi063.stdout:115: finishing write tid 1 to smithi06387500-43 2024-08-17T16:55:09.269 INFO:tasks.rados.rados.0.smithi063.stdout:115: finishing write tid 2 to smithi06387500-43 2024-08-17T16:55:09.269 INFO:tasks.rados.rados.0.smithi063.stdout:118: delete oid 38 current snap is 10 2024-08-17T16:55:09.271 INFO:tasks.rados.rados.0.smithi063.stdout:117: finishing rollback tid 0 to smithi06387500-37 2024-08-17T16:55:09.271 INFO:tasks.rados.rados.0.smithi063.stdout:115: finishing write tid 3 to smithi06387500-43 2024-08-17T16:55:09.271 INFO:tasks.rados.rados.0.smithi063.stdout:115: finishing write tid 4 to smithi06387500-43 2024-08-17T16:55:09.271 INFO:tasks.rados.rados.0.smithi063.stdout:115: finishing write tid 5 to smithi06387500-43 2024-08-17T16:55:09.272 INFO:tasks.rados.rados.0.smithi063.stdout:115: oid 43 updating version 0 to 42 2024-08-17T16:55:09.272 INFO:tasks.rados.rados.0.smithi063.stdout:115: oid 43 version 42 is already newer than 40 2024-08-17T16:55:09.272 INFO:tasks.rados.rados.0.smithi063.stdout:115: oid 43 version 42 is already newer than 33 2024-08-17T16:55:09.272 INFO:tasks.rados.rados.0.smithi063.stdout:115: oid 43 version 42 is already newer than 41 2024-08-17T16:55:09.272 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 42 (ObjNum 81 snap 10 seq_num 81) dirty exists 2024-08-17T16:55:09.272 INFO:tasks.rados.rados.0.smithi063.stdout:115: left oid 43 (ObjNum 81 snap 10 seq_num 81) 2024-08-17T16:55:09.272 INFO:tasks.rados.rados.0.smithi063.stdout:117: finishing rollback tid 1 to smithi06387500-37 2024-08-17T16:55:09.272 INFO:tasks.rados.rados.0.smithi063.stdout:117: finishing rollback tid 2 to smithi06387500-37 2024-08-17T16:55:09.272 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 87 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-08-17T16:55:09.276 INFO:tasks.rados.rados.0.smithi063.stdout:116: expect (ObjNum 10 snap 0 seq_num 10) 2024-08-17T16:55:09.350 INFO:tasks.rados.rados.0.smithi063.stdout:115: done (3 left) 2024-08-17T16:55:09.350 INFO:tasks.rados.rados.0.smithi063.stdout:116: done (2 left) 2024-08-17T16:55:09.350 INFO:tasks.rados.rados.0.smithi063.stdout:117: done (1 left) 2024-08-17T16:55:09.350 INFO:tasks.rados.rados.0.smithi063.stdout:118: done (0 left) 2024-08-17T16:55:09.350 INFO:tasks.rados.rados.0.smithi063.stdout:119: read oid 49 snap -1 2024-08-17T16:55:09.350 INFO:tasks.rados.rados.0.smithi063.stdout:119: expect (ObjNum 48 snap 0 seq_num 48) 2024-08-17T16:55:09.350 INFO:tasks.rados.rados.0.smithi063.stdout:120: read oid 13 snap -1 2024-08-17T16:55:09.351 INFO:tasks.rados.rados.0.smithi063.stdout:120: expect (ObjNum 77 snap 9 seq_num 77) 2024-08-17T16:55:09.351 INFO:tasks.rados.rados.0.smithi063.stdout:121: read oid 47 snap -1 2024-08-17T16:55:09.351 INFO:tasks.rados.rados.0.smithi063.stdout:121: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:55:09.351 INFO:tasks.rados.rados.0.smithi063.stdout:122: snap_remove snap 9 2024-08-17T16:55:09.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:09 smithi063 ceph-mon[42680]: osdmap e92: 8 total, 8 up, 8 in 2024-08-17T16:55:09.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:09 smithi063 ceph-mon[42680]: pgmap v119: 89 pgs: 89 active+clean; 127 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-08-17T16:55:09.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:09 smithi063 ceph-mon[31063]: osdmap e92: 8 total, 8 up, 8 in 2024-08-17T16:55:09.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:09 smithi063 ceph-mon[31063]: pgmap v119: 89 pgs: 89 active+clean; 127 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-08-17T16:55:09.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:09 smithi162 ceph-mon[34633]: osdmap e92: 8 total, 8 up, 8 in 2024-08-17T16:55:09.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:09 smithi162 ceph-mon[34633]: pgmap v119: 89 pgs: 89 active+clean; 127 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-08-17T16:55:09.814 INFO:tasks.rados.rados.0.smithi063.stdout:122: done (3 left) 2024-08-17T16:55:09.814 INFO:tasks.rados.rados.0.smithi063.stdout:123: write oid 27 current snap is 10 2024-08-17T16:55:09.815 INFO:tasks.rados.rados.0.smithi063.stdout:123: seq_num 83 ranges {594981=609197,1974123=745028,3279873=115108} 2024-08-17T16:55:09.821 INFO:tasks.rados.rados.0.smithi063.stdout:123: writing smithi06387500-27 from 594981 to 1204178 tid 1 2024-08-17T16:55:09.830 INFO:tasks.rados.rados.0.smithi063.stdout:123: writing smithi06387500-27 from 1974123 to 2719151 tid 2 2024-08-17T16:55:09.835 INFO:tasks.rados.rados.0.smithi063.stdout:123: writing smithi06387500-27 from 3279873 to 3394981 tid 3 2024-08-17T16:55:09.835 INFO:tasks.rados.rados.0.smithi063.stdout:124: write oid 28 current snap is 10 2024-08-17T16:55:09.835 INFO:tasks.rados.rados.0.smithi063.stdout:121: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:55:09.993 INFO:tasks.rados.rados.0.smithi063.stdout:119: expect (ObjNum 48 snap 0 seq_num 48) 2024-08-17T16:55:10.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:55:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:55:09] "GET /metrics HTTP/1.1" 200 36138 "" "Prometheus/2.43.0" 2024-08-17T16:55:10.101 INFO:tasks.rados.rados.0.smithi063.stdout:120: expect (ObjNum 77 snap 9 seq_num 77) 2024-08-17T16:55:10.224 INFO:tasks.rados.rados.0.smithi063.stdout:123: finishing write tid 1 to smithi06387500-27 2024-08-17T16:55:10.224 INFO:tasks.rados.rados.0.smithi063.stdout:123: finishing write tid 2 to smithi06387500-27 2024-08-17T16:55:10.224 INFO:tasks.rados.rados.0.smithi063.stdout:124: seq_num 84 ranges {611788=400799,1748958=488686,2907296=504492} 2024-08-17T16:55:10.229 INFO:tasks.rados.rados.0.smithi063.stdout:124: writing smithi06387500-28 from 611788 to 1012587 tid 1 2024-08-17T16:55:10.236 INFO:tasks.rados.rados.0.smithi063.stdout:124: writing smithi06387500-28 from 1748958 to 2237644 tid 2 2024-08-17T16:55:10.243 INFO:tasks.rados.rados.0.smithi063.stdout:124: writing smithi06387500-28 from 2907296 to 3411788 tid 3 2024-08-17T16:55:10.243 INFO:tasks.rados.rados.0.smithi063.stdout:119: done (4 left) 2024-08-17T16:55:10.243 INFO:tasks.rados.rados.0.smithi063.stdout:120: done (3 left) 2024-08-17T16:55:10.243 INFO:tasks.rados.rados.0.smithi063.stdout:121: done (2 left) 2024-08-17T16:55:10.244 INFO:tasks.rados.rados.0.smithi063.stdout:125: write oid 35 current snap is 10 2024-08-17T16:55:10.244 INFO:tasks.rados.rados.0.smithi063.stdout:123: finishing write tid 3 to smithi06387500-27 2024-08-17T16:55:10.244 INFO:tasks.rados.rados.0.smithi063.stdout:123: finishing write tid 4 to smithi06387500-27 2024-08-17T16:55:10.244 INFO:tasks.rados.rados.0.smithi063.stdout:123: finishing write tid 5 to smithi06387500-27 2024-08-17T16:55:10.244 INFO:tasks.rados.rados.0.smithi063.stdout:123: oid 27 updating version 0 to 6 2024-08-17T16:55:10.244 INFO:tasks.rados.rados.0.smithi063.stdout:123: oid 27 updating version 6 to 7 2024-08-17T16:55:10.244 INFO:tasks.rados.rados.0.smithi063.stdout:123: oid 27 updating version 7 to 8 2024-08-17T16:55:10.244 INFO:tasks.rados.rados.0.smithi063.stdout:123: oid 27 version 8 is already newer than 5 2024-08-17T16:55:10.244 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 8 (ObjNum 83 snap 10 seq_num 83) dirty exists 2024-08-17T16:55:10.244 INFO:tasks.rados.rados.0.smithi063.stdout:123: left oid 27 (ObjNum 83 snap 10 seq_num 83) 2024-08-17T16:55:10.244 INFO:tasks.rados.rados.0.smithi063.stdout:124: finishing write tid 1 to smithi06387500-28 2024-08-17T16:55:10.244 INFO:tasks.rados.rados.0.smithi063.stdout:124: finishing write tid 2 to smithi06387500-28 2024-08-17T16:55:10.245 INFO:tasks.rados.rados.0.smithi063.stdout:125: seq_num 85 ranges {628595=476048,1807440=632344,2934719=452474} 2024-08-17T16:55:10.249 INFO:tasks.rados.rados.0.smithi063.stdout:125: writing smithi06387500-35 from 628595 to 1104643 tid 1 2024-08-17T16:55:10.256 INFO:tasks.rados.rados.0.smithi063.stdout:125: writing smithi06387500-35 from 1807440 to 2439784 tid 2 2024-08-17T16:55:10.263 INFO:tasks.rados.rados.0.smithi063.stdout:125: writing smithi06387500-35 from 2934719 to 3387193 tid 3 2024-08-17T16:55:10.263 INFO:tasks.rados.rados.0.smithi063.stdout:123: done (2 left) 2024-08-17T16:55:10.263 INFO:tasks.rados.rados.0.smithi063.stdout:126: snap_create 2024-08-17T16:55:10.264 INFO:tasks.rados.rados.0.smithi063.stdout:124: finishing write tid 3 to smithi06387500-28 2024-08-17T16:55:10.264 INFO:tasks.rados.rados.0.smithi063.stdout:124: finishing write tid 4 to smithi06387500-28 2024-08-17T16:55:10.264 INFO:tasks.rados.rados.0.smithi063.stdout:124: finishing write tid 5 to smithi06387500-28 2024-08-17T16:55:10.264 INFO:tasks.rados.rados.0.smithi063.stdout:124: oid 28 updating version 0 to 14 2024-08-17T16:55:10.264 INFO:tasks.rados.rados.0.smithi063.stdout:124: oid 28 version 14 is already newer than 13 2024-08-17T16:55:10.264 INFO:tasks.rados.rados.0.smithi063.stdout:124: oid 28 version 14 is already newer than 12 2024-08-17T16:55:10.264 INFO:tasks.rados.rados.0.smithi063.stdout:124: oid 28 version 14 is already newer than 11 2024-08-17T16:55:10.264 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 14 (ObjNum 84 snap 10 seq_num 84) dirty exists 2024-08-17T16:55:10.264 INFO:tasks.rados.rados.0.smithi063.stdout:124: left oid 28 (ObjNum 84 snap 10 seq_num 84) 2024-08-17T16:55:10.264 INFO:tasks.rados.rados.0.smithi063.stdout:125: finishing write tid 1 to smithi06387500-35 2024-08-17T16:55:10.264 INFO:tasks.rados.rados.0.smithi063.stdout:125: finishing write tid 2 to smithi06387500-35 2024-08-17T16:55:10.266 INFO:tasks.rados.rados.0.smithi063.stdout:125: finishing write tid 3 to smithi06387500-35 2024-08-17T16:55:10.266 INFO:tasks.rados.rados.0.smithi063.stdout:125: finishing write tid 4 to smithi06387500-35 2024-08-17T16:55:10.266 INFO:tasks.rados.rados.0.smithi063.stdout:125: finishing write tid 5 to smithi06387500-35 2024-08-17T16:55:10.266 INFO:tasks.rados.rados.0.smithi063.stdout:125: oid 35 updating version 0 to 94 2024-08-17T16:55:10.266 INFO:tasks.rados.rados.0.smithi063.stdout:125: oid 35 version 94 is already newer than 91 2024-08-17T16:55:10.266 INFO:tasks.rados.rados.0.smithi063.stdout:125: oid 35 version 94 is already newer than 93 2024-08-17T16:55:10.266 INFO:tasks.rados.rados.0.smithi063.stdout:125: oid 35 version 94 is already newer than 92 2024-08-17T16:55:10.266 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 94 (ObjNum 85 snap 10 seq_num 85) dirty exists 2024-08-17T16:55:10.266 INFO:tasks.rados.rados.0.smithi063.stdout:125: left oid 35 (ObjNum 85 snap 10 seq_num 85) 2024-08-17T16:55:10.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:10 smithi063 ceph-mon[31063]: osdmap e93: 8 total, 8 up, 8 in 2024-08-17T16:55:10.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:10 smithi063 ceph-mon[31063]: osdmap e94: 8 total, 8 up, 8 in 2024-08-17T16:55:10.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:10 smithi063 ceph-mon[42680]: osdmap e93: 8 total, 8 up, 8 in 2024-08-17T16:55:10.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:10 smithi063 ceph-mon[42680]: osdmap e94: 8 total, 8 up, 8 in 2024-08-17T16:55:10.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:10 smithi162 ceph-mon[34633]: osdmap e93: 8 total, 8 up, 8 in 2024-08-17T16:55:10.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:10 smithi162 ceph-mon[34633]: osdmap e94: 8 total, 8 up, 8 in 2024-08-17T16:55:11.256 INFO:tasks.rados.rados.0.smithi063.stdout:124: done (2 left) 2024-08-17T16:55:11.256 INFO:tasks.rados.rados.0.smithi063.stdout:125: done (1 left) 2024-08-17T16:55:11.256 INFO:tasks.rados.rados.0.smithi063.stdout:126: done (0 left) 2024-08-17T16:55:11.256 INFO:tasks.rados.rados.0.smithi063.stdout:127: write oid 32 current snap is 11 2024-08-17T16:55:11.256 INFO:tasks.rados.rados.0.smithi063.stdout:127: seq_num 86 ranges {645402=551297,1749569=492355,2845789=599613} 2024-08-17T16:55:11.262 INFO:tasks.rados.rados.0.smithi063.stdout:127: writing smithi06387500-32 from 645402 to 1196699 tid 1 2024-08-17T16:55:11.268 INFO:tasks.rados.rados.0.smithi063.stdout:127: writing smithi06387500-32 from 1749569 to 2241924 tid 2 2024-08-17T16:55:11.275 INFO:tasks.rados.rados.0.smithi063.stdout:127: writing smithi06387500-32 from 2845789 to 3445402 tid 3 2024-08-17T16:55:11.275 INFO:tasks.rados.rados.0.smithi063.stdout:128: read oid 35 snap -1 2024-08-17T16:55:11.275 INFO:tasks.rados.rados.0.smithi063.stdout:128: expect (ObjNum 85 snap 10 seq_num 85) 2024-08-17T16:55:11.275 INFO:tasks.rados.rados.0.smithi063.stdout:129: read oid 11 snap -1 2024-08-17T16:55:11.275 INFO:tasks.rados.rados.0.smithi063.stdout:129: expect (ObjNum 10 snap 0 seq_num 10) 2024-08-17T16:55:11.275 INFO:tasks.rados.rados.0.smithi063.stdout:130: write oid 42 current snap is 11 2024-08-17T16:55:11.275 INFO:tasks.rados.rados.0.smithi063.stdout:130: seq_num 87 ranges {662209=626546,1808051=636013,2873212=588997} 2024-08-17T16:55:11.282 INFO:tasks.rados.rados.0.smithi063.stdout:130: writing smithi06387500-42 from 662209 to 1288755 tid 1 2024-08-17T16:55:11.290 INFO:tasks.rados.rados.0.smithi063.stdout:130: writing smithi06387500-42 from 1808051 to 2444064 tid 2 2024-08-17T16:55:11.296 INFO:tasks.rados.rados.0.smithi063.stdout:130: writing smithi06387500-42 from 2873212 to 3462209 tid 3 2024-08-17T16:55:11.296 INFO:tasks.rados.rados.0.smithi063.stdout:131: snap_remove snap 10 2024-08-17T16:55:11.352 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:11 smithi063 ceph-mon[31063]: pgmap v122: 89 pgs: 89 active+clean; 127 MiB data, 424 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:11.352 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:11 smithi063 ceph-mon[42680]: pgmap v122: 89 pgs: 89 active+clean; 127 MiB data, 424 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:11.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:11 smithi162 ceph-mon[34633]: pgmap v122: 89 pgs: 89 active+clean; 127 MiB data, 424 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:12.263 INFO:tasks.rados.rados.0.smithi063.stdout:131: done (4 left) 2024-08-17T16:55:12.263 INFO:tasks.rados.rados.0.smithi063.stdout:132: read oid 16 snap -1 2024-08-17T16:55:12.263 INFO:tasks.rados.rados.0.smithi063.stdout:132: expect (ObjNum 15 snap 0 seq_num 15) 2024-08-17T16:55:12.263 INFO:tasks.rados.rados.0.smithi063.stdout:133: snap_remove snap 3 2024-08-17T16:55:12.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:12 smithi063 ceph-mon[42680]: osdmap e95: 8 total, 8 up, 8 in 2024-08-17T16:55:12.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:12 smithi063 ceph-mon[31063]: osdmap e95: 8 total, 8 up, 8 in 2024-08-17T16:55:12.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:12 smithi162 ceph-mon[34633]: osdmap e95: 8 total, 8 up, 8 in 2024-08-17T16:55:13.268 INFO:tasks.rados.rados.0.smithi063.stdout:133: done (5 left) 2024-08-17T16:55:13.269 INFO:tasks.rados.rados.0.smithi063.stdout:134: write oid 20 current snap is 11 2024-08-17T16:55:13.269 INFO:tasks.rados.rados.0.smithi063.stdout:134: seq_num 88 ranges {679016=701795,1866533=496024,2900635=402643} 2024-08-17T16:55:13.276 INFO:tasks.rados.rados.0.smithi063.stdout:134: writing smithi06387500-20 from 679016 to 1380811 tid 1 2024-08-17T16:55:13.283 INFO:tasks.rados.rados.0.smithi063.stdout:134: writing smithi06387500-20 from 1866533 to 2362557 tid 2 2024-08-17T16:55:13.289 INFO:tasks.rados.rados.0.smithi063.stdout:134: writing smithi06387500-20 from 2900635 to 3303278 tid 3 2024-08-17T16:55:13.289 INFO:tasks.rados.rados.0.smithi063.stdout:135: read oid 10 snap -1 2024-08-17T16:55:13.289 INFO:tasks.rados.rados.0.smithi063.stdout:135: expect deleted 2024-08-17T16:55:13.289 INFO:tasks.rados.rados.0.smithi063.stdout:136: write oid 33 current snap is 11 2024-08-17T16:55:13.289 INFO:tasks.rados.rados.0.smithi063.stdout:127: finishing write tid 1 to smithi06387500-32 2024-08-17T16:55:13.289 INFO:tasks.rados.rados.0.smithi063.stdout:136: seq_num 89 ranges {695823=777044,1925015=639682,3328058=167765} 2024-08-17T16:55:13.297 INFO:tasks.rados.rados.0.smithi063.stdout:136: writing smithi06387500-33 from 695823 to 1472867 tid 1 2024-08-17T16:55:13.304 INFO:tasks.rados.rados.0.smithi063.stdout:136: writing smithi06387500-33 from 1925015 to 2564697 tid 2 2024-08-17T16:55:13.311 INFO:tasks.rados.rados.0.smithi063.stdout:136: writing smithi06387500-33 from 3328058 to 3495823 tid 3 2024-08-17T16:55:13.311 INFO:tasks.rados.rados.0.smithi063.stdout:137: delete oid 1 current snap is 11 2024-08-17T16:55:13.311 INFO:tasks.rados.rados.0.smithi063.stdout:127: finishing write tid 2 to smithi06387500-32 2024-08-17T16:55:13.311 INFO:tasks.rados.rados.0.smithi063.stdout:127: finishing write tid 3 to smithi06387500-32 2024-08-17T16:55:13.311 INFO:tasks.rados.rados.0.smithi063.stdout:127: finishing write tid 4 to smithi06387500-32 2024-08-17T16:55:13.311 INFO:tasks.rados.rados.0.smithi063.stdout:127: finishing write tid 5 to smithi06387500-32 2024-08-17T16:55:13.311 INFO:tasks.rados.rados.0.smithi063.stdout:127: oid 32 updating version 0 to 19 2024-08-17T16:55:13.311 INFO:tasks.rados.rados.0.smithi063.stdout:127: oid 32 version 19 is already newer than 17 2024-08-17T16:55:13.311 INFO:tasks.rados.rados.0.smithi063.stdout:127: oid 32 version 19 is already newer than 18 2024-08-17T16:55:13.311 INFO:tasks.rados.rados.0.smithi063.stdout:127: oid 32 updating version 19 to 20 2024-08-17T16:55:13.312 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 20 (ObjNum 86 snap 11 seq_num 86) dirty exists 2024-08-17T16:55:13.312 INFO:tasks.rados.rados.0.smithi063.stdout:127: left oid 32 (ObjNum 86 snap 11 seq_num 86) 2024-08-17T16:55:13.312 INFO:tasks.rados.rados.0.smithi063.stdout:129: expect (ObjNum 10 snap 0 seq_num 10) 2024-08-17T16:55:13.391 INFO:tasks.rados.rados.0.smithi063.stdout:128: expect (ObjNum 85 snap 10 seq_num 85) 2024-08-17T16:55:13.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:13 smithi063 ceph-mon[31063]: osdmap e96: 8 total, 8 up, 8 in 2024-08-17T16:55:13.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:13 smithi063 ceph-mon[31063]: pgmap v125: 89 pgs: 89 active+clean; 147 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.8 MiB/s wr, 30 op/s 2024-08-17T16:55:13.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:13 smithi063 ceph-mon[42680]: osdmap e96: 8 total, 8 up, 8 in 2024-08-17T16:55:13.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:13 smithi063 ceph-mon[42680]: pgmap v125: 89 pgs: 89 active+clean; 147 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.8 MiB/s wr, 30 op/s 2024-08-17T16:55:13.585 INFO:tasks.rados.rados.0.smithi063.stdout:130: finishing write tid 1 to smithi06387500-42 2024-08-17T16:55:13.585 INFO:tasks.rados.rados.0.smithi063.stdout:130: finishing write tid 2 to smithi06387500-42 2024-08-17T16:55:13.585 INFO:tasks.rados.rados.0.smithi063.stdout:130: finishing write tid 3 to smithi06387500-42 2024-08-17T16:55:13.585 INFO:tasks.rados.rados.0.smithi063.stdout:130: finishing write tid 4 to smithi06387500-42 2024-08-17T16:55:13.585 INFO:tasks.rados.rados.0.smithi063.stdout:130: finishing write tid 5 to smithi06387500-42 2024-08-17T16:55:13.585 INFO:tasks.rados.rados.0.smithi063.stdout:130: oid 42 updating version 0 to 41 2024-08-17T16:55:13.585 INFO:tasks.rados.rados.0.smithi063.stdout:130: oid 42 updating version 41 to 44 2024-08-17T16:55:13.585 INFO:tasks.rados.rados.0.smithi063.stdout:130: oid 42 version 44 is already newer than 43 2024-08-17T16:55:13.586 INFO:tasks.rados.rados.0.smithi063.stdout:130: oid 42 version 44 is already newer than 42 2024-08-17T16:55:13.586 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 44 (ObjNum 87 snap 11 seq_num 87) dirty exists 2024-08-17T16:55:13.586 INFO:tasks.rados.rados.0.smithi063.stdout:130: left oid 42 (ObjNum 87 snap 11 seq_num 87) 2024-08-17T16:55:13.586 INFO:tasks.rados.rados.0.smithi063.stdout:127: done (8 left) 2024-08-17T16:55:13.586 INFO:tasks.rados.rados.0.smithi063.stdout:128: done (7 left) 2024-08-17T16:55:13.586 INFO:tasks.rados.rados.0.smithi063.stdout:129: done (6 left) 2024-08-17T16:55:13.586 INFO:tasks.rados.rados.0.smithi063.stdout:130: done (5 left) 2024-08-17T16:55:13.586 INFO:tasks.rados.rados.0.smithi063.stdout:137: done (4 left) 2024-08-17T16:55:13.586 INFO:tasks.rados.rados.0.smithi063.stdout:138: snap_create 2024-08-17T16:55:13.586 INFO:tasks.rados.rados.0.smithi063.stdout:132: expect (ObjNum 15 snap 0 seq_num 15) 2024-08-17T16:55:13.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:13 smithi162 ceph-mon[34633]: osdmap e96: 8 total, 8 up, 8 in 2024-08-17T16:55:13.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:13 smithi162 ceph-mon[34633]: pgmap v125: 89 pgs: 89 active+clean; 147 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.8 MiB/s wr, 30 op/s 2024-08-17T16:55:13.662 INFO:tasks.rados.rados.0.smithi063.stdout:134: finishing write tid 1 to smithi06387500-20 2024-08-17T16:55:13.662 INFO:tasks.rados.rados.0.smithi063.stdout:134: finishing write tid 2 to smithi06387500-20 2024-08-17T16:55:13.662 INFO:tasks.rados.rados.0.smithi063.stdout:134: finishing write tid 3 to smithi06387500-20 2024-08-17T16:55:13.663 INFO:tasks.rados.rados.0.smithi063.stdout:134: finishing write tid 4 to smithi06387500-20 2024-08-17T16:55:13.663 INFO:tasks.rados.rados.0.smithi063.stdout:134: finishing write tid 5 to smithi06387500-20 2024-08-17T16:55:13.663 INFO:tasks.rados.rados.0.smithi063.stdout:134: oid 20 updating version 0 to 16 2024-08-17T16:55:13.663 INFO:tasks.rados.rados.0.smithi063.stdout:134: oid 20 updating version 16 to 20 2024-08-17T16:55:13.663 INFO:tasks.rados.rados.0.smithi063.stdout:134: oid 20 version 20 is already newer than 19 2024-08-17T16:55:13.663 INFO:tasks.rados.rados.0.smithi063.stdout:134: oid 20 updating version 20 to 21 2024-08-17T16:55:13.663 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 21 (ObjNum 88 snap 11 seq_num 88) dirty exists 2024-08-17T16:55:13.663 INFO:tasks.rados.rados.0.smithi063.stdout:134: left oid 20 (ObjNum 88 snap 11 seq_num 88) 2024-08-17T16:55:13.663 INFO:tasks.rados.rados.0.smithi063.stdout:136: finishing write tid 1 to smithi06387500-33 2024-08-17T16:55:13.663 INFO:tasks.rados.rados.0.smithi063.stdout:136: finishing write tid 2 to smithi06387500-33 2024-08-17T16:55:13.663 INFO:tasks.rados.rados.0.smithi063.stdout:136: finishing write tid 3 to smithi06387500-33 2024-08-17T16:55:13.663 INFO:tasks.rados.rados.0.smithi063.stdout:136: finishing write tid 4 to smithi06387500-33 2024-08-17T16:55:13.663 INFO:tasks.rados.rados.0.smithi063.stdout:136: finishing write tid 5 to smithi06387500-33 2024-08-17T16:55:13.664 INFO:tasks.rados.rados.0.smithi063.stdout:136: oid 33 updating version 0 to 19 2024-08-17T16:55:13.664 INFO:tasks.rados.rados.0.smithi063.stdout:136: oid 33 updating version 19 to 22 2024-08-17T16:55:13.664 INFO:tasks.rados.rados.0.smithi063.stdout:136: oid 33 version 22 is already newer than 21 2024-08-17T16:55:13.664 INFO:tasks.rados.rados.0.smithi063.stdout:136: oid 33 version 22 is already newer than 20 2024-08-17T16:55:13.664 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 22 (ObjNum 89 snap 11 seq_num 89) dirty exists 2024-08-17T16:55:13.664 INFO:tasks.rados.rados.0.smithi063.stdout:136: left oid 33 (ObjNum 89 snap 11 seq_num 89) 2024-08-17T16:55:14.268 INFO:tasks.rados.rados.0.smithi063.stdout:132: done (4 left) 2024-08-17T16:55:14.268 INFO:tasks.rados.rados.0.smithi063.stdout:134: done (3 left) 2024-08-17T16:55:14.268 INFO:tasks.rados.rados.0.smithi063.stdout:135: done (2 left) 2024-08-17T16:55:14.268 INFO:tasks.rados.rados.0.smithi063.stdout:136: done (1 left) 2024-08-17T16:55:14.269 INFO:tasks.rados.rados.0.smithi063.stdout:138: done (0 left) 2024-08-17T16:55:14.269 INFO:tasks.rados.rados.0.smithi063.stdout:139: snap_remove snap 1 2024-08-17T16:55:14.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:14 smithi063 ceph-mon[42680]: osdmap e97: 8 total, 8 up, 8 in 2024-08-17T16:55:14.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:14 smithi063 ceph-mon[31063]: osdmap e97: 8 total, 8 up, 8 in 2024-08-17T16:55:14.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:14 smithi162 ceph-mon[34633]: osdmap e97: 8 total, 8 up, 8 in 2024-08-17T16:55:14.815 INFO:tasks.rados.rados.0.smithi063.stdout:139: done (0 left) 2024-08-17T16:55:14.816 INFO:tasks.rados.rados.0.smithi063.stdout:140: snap_create 2024-08-17T16:55:15.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:15 smithi063 ceph-mon[42680]: osdmap e98: 8 total, 8 up, 8 in 2024-08-17T16:55:15.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:15 smithi063 ceph-mon[42680]: pgmap v128: 89 pgs: 89 active+clean; 147 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.8 MiB/s wr, 30 op/s 2024-08-17T16:55:15.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:15 smithi063 ceph-mon[42680]: osdmap e99: 8 total, 8 up, 8 in 2024-08-17T16:55:15.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:15 smithi063 ceph-mon[31063]: osdmap e98: 8 total, 8 up, 8 in 2024-08-17T16:55:15.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:15 smithi063 ceph-mon[31063]: pgmap v128: 89 pgs: 89 active+clean; 147 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.8 MiB/s wr, 30 op/s 2024-08-17T16:55:15.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:15 smithi063 ceph-mon[31063]: osdmap e99: 8 total, 8 up, 8 in 2024-08-17T16:55:15.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:15 smithi162 ceph-mon[34633]: osdmap e98: 8 total, 8 up, 8 in 2024-08-17T16:55:15.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:15 smithi162 ceph-mon[34633]: pgmap v128: 89 pgs: 89 active+clean; 147 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.8 MiB/s wr, 30 op/s 2024-08-17T16:55:15.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:15 smithi162 ceph-mon[34633]: osdmap e99: 8 total, 8 up, 8 in 2024-08-17T16:55:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:140: done (0 left) 2024-08-17T16:55:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:141: read oid 31 snap -1 2024-08-17T16:55:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:141: expect (ObjNum 30 snap 0 seq_num 30) 2024-08-17T16:55:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:142: read oid 13 snap -1 2024-08-17T16:55:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:142: expect (ObjNum 77 snap 9 seq_num 77) 2024-08-17T16:55:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:143: read oid 7 snap -1 2024-08-17T16:55:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:143: expect deleted 2024-08-17T16:55:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:144: snap_create 2024-08-17T16:55:15.819 INFO:tasks.rados.rados.0.smithi063.stdout:141: expect (ObjNum 30 snap 0 seq_num 30) 2024-08-17T16:55:15.902 INFO:tasks.rados.rados.0.smithi063.stdout:142: expect (ObjNum 77 snap 9 seq_num 77) 2024-08-17T16:55:16.826 INFO:tasks.rados.rados.0.smithi063.stdout:141: done (3 left) 2024-08-17T16:55:16.826 INFO:tasks.rados.rados.0.smithi063.stdout:142: done (2 left) 2024-08-17T16:55:16.826 INFO:tasks.rados.rados.0.smithi063.stdout:143: done (1 left) 2024-08-17T16:55:16.826 INFO:tasks.rados.rados.0.smithi063.stdout:144: done (0 left) 2024-08-17T16:55:16.826 INFO:tasks.rados.rados.0.smithi063.stdout:145: snap_create 2024-08-17T16:55:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:16 smithi162 ceph-mon[34633]: osdmap e100: 8 total, 8 up, 8 in 2024-08-17T16:55:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:16 smithi162 ceph-mon[34633]: pgmap v131: 89 pgs: 89 active+clean; 151 MiB data, 492 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 778 KiB/s wr, 8 op/s 2024-08-17T16:55:17.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:16 smithi063 ceph-mon[42680]: osdmap e100: 8 total, 8 up, 8 in 2024-08-17T16:55:17.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:16 smithi063 ceph-mon[42680]: pgmap v131: 89 pgs: 89 active+clean; 151 MiB data, 492 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 778 KiB/s wr, 8 op/s 2024-08-17T16:55:17.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:16 smithi063 ceph-mon[31063]: osdmap e100: 8 total, 8 up, 8 in 2024-08-17T16:55:17.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:16 smithi063 ceph-mon[31063]: pgmap v131: 89 pgs: 89 active+clean; 151 MiB data, 492 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 778 KiB/s wr, 8 op/s 2024-08-17T16:55:17.825 INFO:tasks.rados.rados.0.smithi063.stdout:145: done (0 left) 2024-08-17T16:55:17.825 INFO:tasks.rados.rados.0.smithi063.stdout:146: write oid 9 current snap is 15 2024-08-17T16:55:17.825 INFO:tasks.rados.rados.0.smithi063.stdout:146: seq_num 91 ranges {729437=527542,1925626=643351,3266551=262886} 2024-08-17T16:55:17.832 INFO:tasks.rados.rados.0.smithi063.stdout:146: writing smithi06387500-9 from 729437 to 1256979 tid 1 2024-08-17T16:55:17.839 INFO:tasks.rados.rados.0.smithi063.stdout:146: writing smithi06387500-9 from 1925626 to 2568977 tid 2 2024-08-17T16:55:17.845 INFO:tasks.rados.rados.0.smithi063.stdout:146: writing smithi06387500-9 from 3266551 to 3529437 tid 3 2024-08-17T16:55:17.845 INFO:tasks.rados.rados.0.smithi063.stdout:147: rollback oid 14 current snap is 15 2024-08-17T16:55:17.845 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 14 to 13 2024-08-17T16:55:17.845 INFO:tasks.rados.rados.0.smithi063.stdout:146: finishing write tid 1 to smithi06387500-9 2024-08-17T16:55:17.845 INFO:tasks.rados.rados.0.smithi063.stdout:146: finishing write tid 2 to smithi06387500-9 2024-08-17T16:55:17.845 INFO:tasks.rados.rados.0.smithi063.stdout:148: write oid 6 current snap is 15 2024-08-17T16:55:17.845 INFO:tasks.rados.rados.0.smithi063.stdout:148: seq_num 92 ranges {746244=719144,2100461=503362,3126680=419564} 2024-08-17T16:55:17.853 INFO:tasks.rados.rados.0.smithi063.stdout:148: writing smithi06387500-6 from 746244 to 1465388 tid 1 2024-08-17T16:55:17.859 INFO:tasks.rados.rados.0.smithi063.stdout:148: writing smithi06387500-6 from 2100461 to 2603823 tid 2 2024-08-17T16:55:17.865 INFO:tasks.rados.rados.0.smithi063.stdout:148: writing smithi06387500-6 from 3126680 to 3546244 tid 3 2024-08-17T16:55:17.865 INFO:tasks.rados.rados.0.smithi063.stdout:149: read oid 42 snap -1 2024-08-17T16:55:17.865 INFO:tasks.rados.rados.0.smithi063.stdout:149: expect (ObjNum 87 snap 11 seq_num 87) 2024-08-17T16:55:17.866 INFO:tasks.rados.rados.0.smithi063.stdout:150: snap_create 2024-08-17T16:55:17.866 INFO:tasks.rados.rados.0.smithi063.stdout:146: finishing write tid 3 to smithi06387500-9 2024-08-17T16:55:17.866 INFO:tasks.rados.rados.0.smithi063.stdout:146: finishing write tid 4 to smithi06387500-9 2024-08-17T16:55:17.866 INFO:tasks.rados.rados.0.smithi063.stdout:146: finishing write tid 5 to smithi06387500-9 2024-08-17T16:55:17.866 INFO:tasks.rados.rados.0.smithi063.stdout:146: oid 9 updating version 0 to 61 2024-08-17T16:55:17.866 INFO:tasks.rados.rados.0.smithi063.stdout:146: oid 9 updating version 61 to 62 2024-08-17T16:55:17.866 INFO:tasks.rados.rados.0.smithi063.stdout:146: oid 9 updating version 62 to 63 2024-08-17T16:55:17.866 INFO:tasks.rados.rados.0.smithi063.stdout:146: oid 9 version 63 is already newer than 60 2024-08-17T16:55:17.866 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 63 (ObjNum 91 snap 15 seq_num 91) dirty exists 2024-08-17T16:55:17.866 INFO:tasks.rados.rados.0.smithi063.stdout:146: left oid 9 (ObjNum 91 snap 15 seq_num 91) 2024-08-17T16:55:17.866 INFO:tasks.rados.rados.0.smithi063.stdout:147: finishing rollback tid 0 to smithi06387500-14 2024-08-17T16:55:17.866 INFO:tasks.rados.rados.0.smithi063.stdout:147: finishing rollback tid 1 to smithi06387500-14 2024-08-17T16:55:17.867 INFO:tasks.rados.rados.0.smithi063.stdout:147: finishing rollback tid 2 to smithi06387500-14 2024-08-17T16:55:17.867 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 126 (ObjNum 75 snap 9 seq_num 75) dirty exists 2024-08-17T16:55:17.867 INFO:tasks.rados.rados.0.smithi063.stdout:148: finishing write tid 1 to smithi06387500-6 2024-08-17T16:55:17.867 INFO:tasks.rados.rados.0.smithi063.stdout:148: finishing write tid 2 to smithi06387500-6 2024-08-17T16:55:17.867 INFO:tasks.rados.rados.0.smithi063.stdout:148: finishing write tid 3 to smithi06387500-6 2024-08-17T16:55:17.867 INFO:tasks.rados.rados.0.smithi063.stdout:148: finishing write tid 4 to smithi06387500-6 2024-08-17T16:55:17.868 INFO:tasks.rados.rados.0.smithi063.stdout:148: finishing write tid 5 to smithi06387500-6 2024-08-17T16:55:17.868 INFO:tasks.rados.rados.0.smithi063.stdout:148: oid 6 updating version 0 to 29 2024-08-17T16:55:17.868 INFO:tasks.rados.rados.0.smithi063.stdout:148: oid 6 version 29 is already newer than 26 2024-08-17T16:55:17.868 INFO:tasks.rados.rados.0.smithi063.stdout:148: oid 6 version 29 is already newer than 28 2024-08-17T16:55:17.868 INFO:tasks.rados.rados.0.smithi063.stdout:148: oid 6 version 29 is already newer than 27 2024-08-17T16:55:17.868 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 29 (ObjNum 92 snap 15 seq_num 92) dirty exists 2024-08-17T16:55:17.868 INFO:tasks.rados.rados.0.smithi063.stdout:148: left oid 6 (ObjNum 92 snap 15 seq_num 92) 2024-08-17T16:55:17.871 INFO:tasks.rados.rados.0.smithi063.stdout:149: expect (ObjNum 87 snap 11 seq_num 87) 2024-08-17T16:55:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:17 smithi162 ceph-mon[34633]: osdmap e101: 8 total, 8 up, 8 in 2024-08-17T16:55:18.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:17 smithi063 ceph-mon[42680]: osdmap e101: 8 total, 8 up, 8 in 2024-08-17T16:55:18.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:17 smithi063 ceph-mon[31063]: osdmap e101: 8 total, 8 up, 8 in 2024-08-17T16:55:18.835 INFO:tasks.rados.rados.0.smithi063.stdout:146: done (4 left) 2024-08-17T16:55:18.835 INFO:tasks.rados.rados.0.smithi063.stdout:147: done (3 left) 2024-08-17T16:55:18.835 INFO:tasks.rados.rados.0.smithi063.stdout:148: done (2 left) 2024-08-17T16:55:18.835 INFO:tasks.rados.rados.0.smithi063.stdout:149: done (1 left) 2024-08-17T16:55:18.835 INFO:tasks.rados.rados.0.smithi063.stdout:150: done (0 left) 2024-08-17T16:55:18.835 INFO:tasks.rados.rados.0.smithi063.stdout:151: write oid 46 current snap is 16 2024-08-17T16:55:18.835 INFO:tasks.rados.rados.0.smithi063.stdout:151: seq_num 93 ranges {763051=794393,2158943=647020,3437750=125301} 2024-08-17T16:55:18.843 INFO:tasks.rados.rados.0.smithi063.stdout:151: writing smithi06387500-46 from 763051 to 1557444 tid 1 2024-08-17T16:55:18.850 INFO:tasks.rados.rados.0.smithi063.stdout:151: writing smithi06387500-46 from 2158943 to 2805963 tid 2 2024-08-17T16:55:18.855 INFO:tasks.rados.rados.0.smithi063.stdout:151: writing smithi06387500-46 from 3437750 to 3563051 tid 3 2024-08-17T16:55:18.855 INFO:tasks.rados.rados.0.smithi063.stdout:152: read oid 25 snap -1 2024-08-17T16:55:18.855 INFO:tasks.rados.rados.0.smithi063.stdout:152: expect (ObjNum 78 snap 9 seq_num 78) 2024-08-17T16:55:18.855 INFO:tasks.rados.rados.0.smithi063.stdout:153: write oid 50 current snap is 16 2024-08-17T16:55:18.855 INFO:tasks.rados.rados.0.smithi063.stdout:151: finishing write tid 1 to smithi06387500-46 2024-08-17T16:55:18.855 INFO:tasks.rados.rados.0.smithi063.stdout:151: finishing write tid 2 to smithi06387500-46 2024-08-17T16:55:18.855 INFO:tasks.rados.rados.0.smithi063.stdout:153: seq_num 94 ranges {779858=469642,1817425=507031,2781526=586628} 2024-08-17T16:55:18.861 INFO:tasks.rados.rados.0.smithi063.stdout:153: writing smithi06387500-50 from 779858 to 1249500 tid 1 2024-08-17T16:55:18.867 INFO:tasks.rados.rados.0.smithi063.stdout:153: writing smithi06387500-50 from 1817425 to 2324456 tid 2 2024-08-17T16:55:18.873 INFO:tasks.rados.rados.0.smithi063.stdout:153: writing smithi06387500-50 from 2781526 to 3368154 tid 3 2024-08-17T16:55:18.873 INFO:tasks.rados.rados.0.smithi063.stdout:154: read oid 6 snap 12 2024-08-17T16:55:18.873 INFO:tasks.rados.rados.0.smithi063.stdout:154: expect (ObjNum 5 snap 0 seq_num 5) 2024-08-17T16:55:18.874 INFO:tasks.rados.rados.0.smithi063.stdout:155: snap_remove snap 8 2024-08-17T16:55:19.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:18 smithi162 ceph-mon[34633]: osdmap e102: 8 total, 8 up, 8 in 2024-08-17T16:55:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:18 smithi162 ceph-mon[34633]: pgmap v134: 89 pgs: 89 active+clean; 151 MiB data, 492 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 778 KiB/s wr, 8 op/s 2024-08-17T16:55:19.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:18 smithi063 ceph-mon[42680]: osdmap e102: 8 total, 8 up, 8 in 2024-08-17T16:55:19.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:18 smithi063 ceph-mon[42680]: pgmap v134: 89 pgs: 89 active+clean; 151 MiB data, 492 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 778 KiB/s wr, 8 op/s 2024-08-17T16:55:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:18 smithi063 ceph-mon[31063]: osdmap e102: 8 total, 8 up, 8 in 2024-08-17T16:55:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:18 smithi063 ceph-mon[31063]: pgmap v134: 89 pgs: 89 active+clean; 151 MiB data, 492 MiB used, 715 GiB / 715 GiB avail; 4.7 MiB/s rd, 778 KiB/s wr, 8 op/s 2024-08-17T16:55:19.818 INFO:tasks.rados.rados.0.smithi063.stdout:151: finishing write tid 3 to smithi06387500-46 2024-08-17T16:55:19.818 INFO:tasks.rados.rados.0.smithi063.stdout:151: finishing write tid 4 to smithi06387500-46 2024-08-17T16:55:19.818 INFO:tasks.rados.rados.0.smithi063.stdout:151: finishing write tid 5 to smithi06387500-46 2024-08-17T16:55:19.818 INFO:tasks.rados.rados.0.smithi063.stdout:151: oid 46 updating version 0 to 34 2024-08-17T16:55:19.818 INFO:tasks.rados.rados.0.smithi063.stdout:151: oid 46 version 34 is already newer than 32 2024-08-17T16:55:19.818 INFO:tasks.rados.rados.0.smithi063.stdout:151: oid 46 version 34 is already newer than 33 2024-08-17T16:55:19.818 INFO:tasks.rados.rados.0.smithi063.stdout:151: oid 46 version 34 is already newer than 31 2024-08-17T16:55:19.819 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 34 (ObjNum 93 snap 16 seq_num 93) dirty exists 2024-08-17T16:55:19.819 INFO:tasks.rados.rados.0.smithi063.stdout:151: left oid 46 (ObjNum 93 snap 16 seq_num 93) 2024-08-17T16:55:19.819 INFO:tasks.rados.rados.0.smithi063.stdout:152: expect (ObjNum 78 snap 9 seq_num 78) 2024-08-17T16:55:19.977 INFO:tasks.rados.rados.0.smithi063.stdout:153: finishing write tid 1 to smithi06387500-50 2024-08-17T16:55:19.977 INFO:tasks.rados.rados.0.smithi063.stdout:153: finishing write tid 2 to smithi06387500-50 2024-08-17T16:55:19.977 INFO:tasks.rados.rados.0.smithi063.stdout:153: finishing write tid 3 to smithi06387500-50 2024-08-17T16:55:19.977 INFO:tasks.rados.rados.0.smithi063.stdout:153: finishing write tid 4 to smithi06387500-50 2024-08-17T16:55:19.978 INFO:tasks.rados.rados.0.smithi063.stdout:153: finishing write tid 5 to smithi06387500-50 2024-08-17T16:55:19.978 INFO:tasks.rados.rados.0.smithi063.stdout:153: oid 50 updating version 0 to 52 2024-08-17T16:55:19.978 INFO:tasks.rados.rados.0.smithi063.stdout:153: oid 50 updating version 52 to 55 2024-08-17T16:55:19.978 INFO:tasks.rados.rados.0.smithi063.stdout:153: oid 50 version 55 is already newer than 54 2024-08-17T16:55:19.978 INFO:tasks.rados.rados.0.smithi063.stdout:153: oid 50 version 55 is already newer than 53 2024-08-17T16:55:19.978 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 55 (ObjNum 94 snap 16 seq_num 94) dirty exists 2024-08-17T16:55:19.978 INFO:tasks.rados.rados.0.smithi063.stdout:153: left oid 50 (ObjNum 94 snap 16 seq_num 94) 2024-08-17T16:55:19.978 INFO:tasks.rados.rados.0.smithi063.stdout:154: expect (ObjNum 5 snap 0 seq_num 5) 2024-08-17T16:55:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:19 smithi063 ceph-mon[31063]: osdmap e103: 8 total, 8 up, 8 in 2024-08-17T16:55:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:19 smithi063 ceph-mon[31063]: osdmap e104: 8 total, 8 up, 8 in 2024-08-17T16:55:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:19 smithi063 ceph-mon[42680]: osdmap e103: 8 total, 8 up, 8 in 2024-08-17T16:55:20.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:19 smithi063 ceph-mon[42680]: osdmap e104: 8 total, 8 up, 8 in 2024-08-17T16:55:20.030 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:55:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:55:19] "GET /metrics HTTP/1.1" 200 36138 "" "Prometheus/2.43.0" 2024-08-17T16:55:20.048 INFO:tasks.rados.rados.0.smithi063.stdout:151: done (4 left) 2024-08-17T16:55:20.048 INFO:tasks.rados.rados.0.smithi063.stdout:152: done (3 left) 2024-08-17T16:55:20.048 INFO:tasks.rados.rados.0.smithi063.stdout:153: done (2 left) 2024-08-17T16:55:20.048 INFO:tasks.rados.rados.0.smithi063.stdout:154: done (1 left) 2024-08-17T16:55:20.049 INFO:tasks.rados.rados.0.smithi063.stdout:155: done (0 left) 2024-08-17T16:55:20.049 INFO:tasks.rados.rados.0.smithi063.stdout:156: snap_remove snap 6 2024-08-17T16:55:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:19 smithi162 ceph-mon[34633]: osdmap e103: 8 total, 8 up, 8 in 2024-08-17T16:55:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:19 smithi162 ceph-mon[34633]: osdmap e104: 8 total, 8 up, 8 in 2024-08-17T16:55:20.849 INFO:tasks.rados.rados.0.smithi063.stdout:156: done (0 left) 2024-08-17T16:55:20.850 INFO:tasks.rados.rados.0.smithi063.stdout:157: delete oid 10 current snap is 16 2024-08-17T16:55:20.855 INFO:tasks.rados.rados.0.smithi063.stdout:157: done (0 left) 2024-08-17T16:55:20.856 INFO:tasks.rados.rados.0.smithi063.stdout:158: write oid 29 current snap is 16 2024-08-17T16:55:20.856 INFO:tasks.rados.rados.0.smithi063.stdout:158: seq_num 96 ranges {413472=620140,1818036=510700,3120019=493453} 2024-08-17T16:55:20.861 INFO:tasks.rados.rados.0.smithi063.stdout:158: writing smithi06387500-29 from 413472 to 1033612 tid 1 2024-08-17T16:55:20.869 INFO:tasks.rados.rados.0.smithi063.stdout:158: writing smithi06387500-29 from 1818036 to 2328736 tid 2 2024-08-17T16:55:20.876 INFO:tasks.rados.rados.0.smithi063.stdout:158: writing smithi06387500-29 from 3120019 to 3613472 tid 3 2024-08-17T16:55:20.876 INFO:tasks.rados.rados.0.smithi063.stdout:159: read oid 41 snap -1 2024-08-17T16:55:20.876 INFO:tasks.rados.rados.0.smithi063.stdout:159: expect (ObjNum 73 snap 9 seq_num 73) 2024-08-17T16:55:20.876 INFO:tasks.rados.rados.0.smithi063.stdout:160: delete oid 28 current snap is 16 2024-08-17T16:55:20.877 INFO:tasks.rados.rados.0.smithi063.stdout:158: finishing write tid 1 to smithi06387500-29 2024-08-17T16:55:20.877 INFO:tasks.rados.rados.0.smithi063.stdout:158: finishing write tid 2 to smithi06387500-29 2024-08-17T16:55:20.878 INFO:tasks.rados.rados.0.smithi063.stdout:160: done (2 left) 2024-08-17T16:55:20.878 INFO:tasks.rados.rados.0.smithi063.stdout:161: write oid 44 current snap is 16 2024-08-17T16:55:20.878 INFO:tasks.rados.rados.0.smithi063.stdout:161: seq_num 98 ranges {447086=770638,1935000=798016,3458512=188574} 2024-08-17T16:55:20.885 INFO:tasks.rados.rados.0.smithi063.stdout:161: writing smithi06387500-44 from 447086 to 1217724 tid 1 2024-08-17T16:55:20.894 INFO:tasks.rados.rados.0.smithi063.stdout:161: writing smithi06387500-44 from 1935000 to 2733016 tid 2 2024-08-17T16:55:20.899 INFO:tasks.rados.rados.0.smithi063.stdout:161: writing smithi06387500-44 from 3458512 to 3647086 tid 3 2024-08-17T16:55:20.899 INFO:tasks.rados.rados.0.smithi063.stdout:162: snap_remove snap 12 2024-08-17T16:55:21.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:20 smithi162 ceph-mon[34633]: pgmap v137: 89 pgs: 89 active+clean; 151 MiB data, 492 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:21.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:20 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:55:21.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:20 smithi063 ceph-mon[31063]: pgmap v137: 89 pgs: 89 active+clean; 151 MiB data, 492 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:21.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:20 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:55:21.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:20 smithi063 ceph-mon[42680]: pgmap v137: 89 pgs: 89 active+clean; 151 MiB data, 492 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:21.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:20 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:55:21.852 INFO:tasks.rados.rados.0.smithi063.stdout:162: done (3 left) 2024-08-17T16:55:21.853 INFO:tasks.rados.rados.0.smithi063.stdout:163: rollback oid 15 current snap is 16 2024-08-17T16:55:21.853 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 15 to 0 2024-08-17T16:55:21.853 INFO:tasks.rados.rados.0.smithi063.stdout:158: finishing write tid 3 to smithi06387500-29 2024-08-17T16:55:21.853 INFO:tasks.rados.rados.0.smithi063.stdout:158: finishing write tid 4 to smithi06387500-29 2024-08-17T16:55:21.853 INFO:tasks.rados.rados.0.smithi063.stdout:158: finishing write tid 5 to smithi06387500-29 2024-08-17T16:55:21.853 INFO:tasks.rados.rados.0.smithi063.stdout:158: oid 29 updating version 0 to 25 2024-08-17T16:55:21.853 INFO:tasks.rados.rados.0.smithi063.stdout:158: oid 29 updating version 25 to 26 2024-08-17T16:55:21.853 INFO:tasks.rados.rados.0.smithi063.stdout:158: oid 29 updating version 26 to 28 2024-08-17T16:55:21.853 INFO:tasks.rados.rados.0.smithi063.stdout:158: oid 29 version 28 is already newer than 27 2024-08-17T16:55:21.853 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 28 (ObjNum 96 snap 16 seq_num 96) dirty exists 2024-08-17T16:55:21.853 INFO:tasks.rados.rados.0.smithi063.stdout:158: left oid 29 (ObjNum 96 snap 16 seq_num 96) 2024-08-17T16:55:21.853 INFO:tasks.rados.rados.0.smithi063.stdout:159: expect (ObjNum 73 snap 9 seq_num 73) 2024-08-17T16:55:21.996 INFO:tasks.rados.rados.0.smithi063.stdout:161: finishing write tid 1 to smithi06387500-44 2024-08-17T16:55:21.996 INFO:tasks.rados.rados.0.smithi063.stdout:161: finishing write tid 2 to smithi06387500-44 2024-08-17T16:55:21.997 INFO:tasks.rados.rados.0.smithi063.stdout:158: done (3 left) 2024-08-17T16:55:21.997 INFO:tasks.rados.rados.0.smithi063.stdout:159: done (2 left) 2024-08-17T16:55:21.997 INFO:tasks.rados.rados.0.smithi063.stdout:164: read oid 30 snap -1 2024-08-17T16:55:21.997 INFO:tasks.rados.rados.0.smithi063.stdout:164: expect (ObjNum 29 snap 0 seq_num 29) 2024-08-17T16:55:21.997 INFO:tasks.rados.rados.0.smithi063.stdout:165: read oid 24 snap 13 2024-08-17T16:55:21.997 INFO:tasks.rados.rados.0.smithi063.stdout:165: expect (ObjNum 23 snap 0 seq_num 23) 2024-08-17T16:55:21.997 INFO:tasks.rados.rados.0.smithi063.stdout:166: read oid 40 snap 13 2024-08-17T16:55:21.997 INFO:tasks.rados.rados.0.smithi063.stdout:166: expect (ObjNum 39 snap 0 seq_num 39) 2024-08-17T16:55:21.997 INFO:tasks.rados.rados.0.smithi063.stdout:167: write oid 5 current snap is 16 2024-08-17T16:55:21.997 INFO:tasks.rados.rados.0.smithi063.stdout:161: finishing write tid 3 to smithi06387500-44 2024-08-17T16:55:21.997 INFO:tasks.rados.rados.0.smithi063.stdout:167: seq_num 99 ranges {463893=562240,1593482=658027,2685935=559341} 2024-08-17T16:55:22.002 INFO:tasks.rados.rados.0.smithi063.stdout:167: writing smithi06387500-5 from 463893 to 1026133 tid 1 2024-08-17T16:55:22.009 INFO:tasks.rados.rados.0.smithi063.stdout:167: writing smithi06387500-5 from 1593482 to 2251509 tid 2 2024-08-17T16:55:22.015 INFO:tasks.rados.rados.0.smithi063.stdout:167: writing smithi06387500-5 from 2685935 to 3245276 tid 3 2024-08-17T16:55:22.015 INFO:tasks.rados.rados.0.smithi063.stdout:168: write oid 10 current snap is 16 2024-08-17T16:55:22.015 INFO:tasks.rados.rados.0.smithi063.stdout:161: finishing write tid 4 to smithi06387500-44 2024-08-17T16:55:22.015 INFO:tasks.rados.rados.0.smithi063.stdout:161: finishing write tid 5 to smithi06387500-44 2024-08-17T16:55:22.015 INFO:tasks.rados.rados.0.smithi063.stdout:161: oid 44 updating version 0 to 5 2024-08-17T16:55:22.015 INFO:tasks.rados.rados.0.smithi063.stdout:161: oid 44 updating version 5 to 7 2024-08-17T16:55:22.015 INFO:tasks.rados.rados.0.smithi063.stdout:161: oid 44 updating version 7 to 8 2024-08-17T16:55:22.015 INFO:tasks.rados.rados.0.smithi063.stdout:161: oid 44 version 8 is already newer than 6 2024-08-17T16:55:22.015 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 8 (ObjNum 98 snap 16 seq_num 98) dirty exists 2024-08-17T16:55:22.015 INFO:tasks.rados.rados.0.smithi063.stdout:161: left oid 44 (ObjNum 98 snap 16 seq_num 98) 2024-08-17T16:55:22.015 INFO:tasks.rados.rados.0.smithi063.stdout:168: seq_num 100 ranges {480700=637489,1651964=401685,2713358=770613} 2024-08-17T16:55:22.021 INFO:tasks.rados.rados.0.smithi063.stdout:168: writing smithi06387500-10 from 480700 to 1118189 tid 1 2024-08-17T16:55:22.026 INFO:tasks.rados.rados.0.smithi063.stdout:168: writing smithi06387500-10 from 1651964 to 2053649 tid 2 2024-08-17T16:55:22.034 INFO:tasks.rados.rados.0.smithi063.stdout:168: writing smithi06387500-10 from 2713358 to 3483971 tid 3 2024-08-17T16:55:22.034 INFO:tasks.rados.rados.0.smithi063.stdout:161: done (6 left) 2024-08-17T16:55:22.034 INFO:tasks.rados.rados.0.smithi063.stdout:169: write oid 18 current snap is 16 2024-08-17T16:55:22.034 INFO:tasks.rados.rados.0.smithi063.stdout:163: finishing rollback tid 1 to smithi06387500-15 2024-08-17T16:55:22.034 INFO:tasks.rados.rados.0.smithi063.stdout:163: finishing rollback tid 2 to smithi06387500-15 2024-08-17T16:55:22.034 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 44 (ObjNum 14 snap 0 seq_num 14) dirty exists 2024-08-17T16:55:22.035 INFO:tasks.rados.rados.0.smithi063.stdout:169: seq_num 101 ranges {497507=712738,1710446=661696,3140781=556726} 2024-08-17T16:55:22.040 INFO:tasks.rados.rados.0.smithi063.stdout:169: writing smithi06387500-18 from 497507 to 1210245 tid 1 2024-08-17T16:55:22.047 INFO:tasks.rados.rados.0.smithi063.stdout:169: writing smithi06387500-18 from 1710446 to 2372142 tid 2 2024-08-17T16:55:22.054 INFO:tasks.rados.rados.0.smithi063.stdout:169: writing smithi06387500-18 from 3140781 to 3697507 tid 3 2024-08-17T16:55:22.055 INFO:tasks.rados.rados.0.smithi063.stdout:163: done (6 left) 2024-08-17T16:55:22.055 INFO:tasks.rados.rados.0.smithi063.stdout:170: write oid 39 current snap is 16 2024-08-17T16:55:22.055 INFO:tasks.rados.rados.0.smithi063.stdout:170: seq_num 102 ranges {514314=787987,1768928=405354,2768204=509510} 2024-08-17T16:55:22.061 INFO:tasks.rados.rados.0.smithi063.stdout:170: writing smithi06387500-39 from 514314 to 1302301 tid 1 2024-08-17T16:55:22.066 INFO:tasks.rados.rados.0.smithi063.stdout:170: writing smithi06387500-39 from 1768928 to 2174282 tid 2 2024-08-17T16:55:22.072 INFO:tasks.rados.rados.0.smithi063.stdout:170: writing smithi06387500-39 from 2768204 to 3277714 tid 3 2024-08-17T16:55:22.072 INFO:tasks.rados.rados.0.smithi063.stdout:171: rollback oid 20 current snap is 16 2024-08-17T16:55:22.073 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 20 to 15 2024-08-17T16:55:22.073 INFO:tasks.rados.rados.0.smithi063.stdout:166: expect (ObjNum 39 snap 0 seq_num 39) 2024-08-17T16:55:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:21 smithi162 ceph-mon[34633]: osdmap e105: 8 total, 8 up, 8 in 2024-08-17T16:55:22.160 INFO:tasks.rados.rados.0.smithi063.stdout:165: expect (ObjNum 23 snap 0 seq_num 23) 2024-08-17T16:55:22.241 INFO:tasks.rados.rados.0.smithi063.stdout:164: expect (ObjNum 29 snap 0 seq_num 29) 2024-08-17T16:55:22.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:21 smithi063 ceph-mon[31063]: osdmap e105: 8 total, 8 up, 8 in 2024-08-17T16:55:22.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:21 smithi063 ceph-mon[42680]: osdmap e105: 8 total, 8 up, 8 in 2024-08-17T16:55:22.327 INFO:tasks.rados.rados.0.smithi063.stdout:167: finishing write tid 1 to smithi06387500-5 2024-08-17T16:55:22.327 INFO:tasks.rados.rados.0.smithi063.stdout:167: finishing write tid 2 to smithi06387500-5 2024-08-17T16:55:22.327 INFO:tasks.rados.rados.0.smithi063.stdout:164: done (7 left) 2024-08-17T16:55:22.327 INFO:tasks.rados.rados.0.smithi063.stdout:165: done (6 left) 2024-08-17T16:55:22.327 INFO:tasks.rados.rados.0.smithi063.stdout:166: done (5 left) 2024-08-17T16:55:22.327 INFO:tasks.rados.rados.0.smithi063.stdout:172: rollback oid 9 current snap is 16 2024-08-17T16:55:22.327 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 9 to 14 2024-08-17T16:55:22.327 INFO:tasks.rados.rados.0.smithi063.stdout:167: finishing write tid 3 to smithi06387500-5 2024-08-17T16:55:22.328 INFO:tasks.rados.rados.0.smithi063.stdout:167: finishing write tid 4 to smithi06387500-5 2024-08-17T16:55:22.328 INFO:tasks.rados.rados.0.smithi063.stdout:167: finishing write tid 5 to smithi06387500-5 2024-08-17T16:55:22.328 INFO:tasks.rados.rados.0.smithi063.stdout:167: oid 5 updating version 0 to 58 2024-08-17T16:55:22.328 INFO:tasks.rados.rados.0.smithi063.stdout:167: oid 5 updating version 58 to 60 2024-08-17T16:55:22.328 INFO:tasks.rados.rados.0.smithi063.stdout:167: oid 5 version 60 is already newer than 59 2024-08-17T16:55:22.328 INFO:tasks.rados.rados.0.smithi063.stdout:167: oid 5 version 60 is already newer than 57 2024-08-17T16:55:22.328 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 60 (ObjNum 99 snap 16 seq_num 99) dirty exists 2024-08-17T16:55:22.328 INFO:tasks.rados.rados.0.smithi063.stdout:167: left oid 5 (ObjNum 99 snap 16 seq_num 99) 2024-08-17T16:55:22.328 INFO:tasks.rados.rados.0.smithi063.stdout:168: finishing write tid 1 to smithi06387500-10 2024-08-17T16:55:22.328 INFO:tasks.rados.rados.0.smithi063.stdout:168: finishing write tid 2 to smithi06387500-10 2024-08-17T16:55:22.328 INFO:tasks.rados.rados.0.smithi063.stdout:168: finishing write tid 3 to smithi06387500-10 2024-08-17T16:55:22.328 INFO:tasks.rados.rados.0.smithi063.stdout:168: finishing write tid 4 to smithi06387500-10 2024-08-17T16:55:22.328 INFO:tasks.rados.rados.0.smithi063.stdout:168: finishing write tid 5 to smithi06387500-10 2024-08-17T16:55:22.329 INFO:tasks.rados.rados.0.smithi063.stdout:168: oid 10 updating version 0 to 33 2024-08-17T16:55:22.329 INFO:tasks.rados.rados.0.smithi063.stdout:168: oid 10 version 33 is already newer than 32 2024-08-17T16:55:22.329 INFO:tasks.rados.rados.0.smithi063.stdout:168: oid 10 updating version 33 to 34 2024-08-17T16:55:22.329 INFO:tasks.rados.rados.0.smithi063.stdout:168: oid 10 version 34 is already newer than 31 2024-08-17T16:55:22.329 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 34 (ObjNum 100 snap 16 seq_num 100) dirty exists 2024-08-17T16:55:22.329 INFO:tasks.rados.rados.0.smithi063.stdout:168: left oid 10 (ObjNum 100 snap 16 seq_num 100) 2024-08-17T16:55:22.329 INFO:tasks.rados.rados.0.smithi063.stdout:169: finishing write tid 1 to smithi06387500-18 2024-08-17T16:55:22.329 INFO:tasks.rados.rados.0.smithi063.stdout:169: finishing write tid 2 to smithi06387500-18 2024-08-17T16:55:22.329 INFO:tasks.rados.rados.0.smithi063.stdout:169: finishing write tid 3 to smithi06387500-18 2024-08-17T16:55:22.329 INFO:tasks.rados.rados.0.smithi063.stdout:169: finishing write tid 4 to smithi06387500-18 2024-08-17T16:55:22.329 INFO:tasks.rados.rados.0.smithi063.stdout:169: finishing write tid 5 to smithi06387500-18 2024-08-17T16:55:22.329 INFO:tasks.rados.rados.0.smithi063.stdout:169: oid 18 updating version 0 to 39 2024-08-17T16:55:22.330 INFO:tasks.rados.rados.0.smithi063.stdout:169: oid 18 version 39 is already newer than 37 2024-08-17T16:55:22.330 INFO:tasks.rados.rados.0.smithi063.stdout:169: oid 18 version 39 is already newer than 36 2024-08-17T16:55:22.330 INFO:tasks.rados.rados.0.smithi063.stdout:169: oid 18 version 39 is already newer than 38 2024-08-17T16:55:22.330 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 39 (ObjNum 101 snap 16 seq_num 101) dirty exists 2024-08-17T16:55:22.330 INFO:tasks.rados.rados.0.smithi063.stdout:169: left oid 18 (ObjNum 101 snap 16 seq_num 101) 2024-08-17T16:55:22.330 INFO:tasks.rados.rados.0.smithi063.stdout:170: finishing write tid 1 to smithi06387500-39 2024-08-17T16:55:22.330 INFO:tasks.rados.rados.0.smithi063.stdout:167: done (5 left) 2024-08-17T16:55:22.330 INFO:tasks.rados.rados.0.smithi063.stdout:168: done (4 left) 2024-08-17T16:55:22.330 INFO:tasks.rados.rados.0.smithi063.stdout:169: done (3 left) 2024-08-17T16:55:22.330 INFO:tasks.rados.rados.0.smithi063.stdout:173: write oid 17 current snap is 16 2024-08-17T16:55:22.330 INFO:tasks.rados.rados.0.smithi063.stdout:170: finishing write tid 2 to smithi06387500-39 2024-08-17T16:55:22.330 INFO:tasks.rados.rados.0.smithi063.stdout:173: seq_num 103 ranges {531121=463236,1711057=665365,3079274=651847} 2024-08-17T16:55:22.331 INFO:tasks.rados.rados.0.smithi063.stdout:173: writing smithi06387500-17 from 531121 to 994357 tid 1 2024-08-17T16:55:22.339 INFO:tasks.rados.rados.0.smithi063.stdout:173: writing smithi06387500-17 from 1711057 to 2376422 tid 2 2024-08-17T16:55:22.347 INFO:tasks.rados.rados.0.smithi063.stdout:173: writing smithi06387500-17 from 3079274 to 3731121 tid 3 2024-08-17T16:55:22.347 INFO:tasks.rados.rados.0.smithi063.stdout:174: read oid 7 snap -1 2024-08-17T16:55:22.347 INFO:tasks.rados.rados.0.smithi063.stdout:174: expect deleted 2024-08-17T16:55:22.347 INFO:tasks.rados.rados.0.smithi063.stdout:175: read oid 5 snap -1 2024-08-17T16:55:22.347 INFO:tasks.rados.rados.0.smithi063.stdout:175: expect (ObjNum 99 snap 16 seq_num 99) 2024-08-17T16:55:22.347 INFO:tasks.rados.rados.0.smithi063.stdout:176: rollback oid 4 current snap is 16 2024-08-17T16:55:22.347 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 4 to 15 2024-08-17T16:55:22.347 INFO:tasks.rados.rados.0.smithi063.stdout:171: finishing rollback tid 0 to smithi06387500-20 2024-08-17T16:55:22.348 INFO:tasks.rados.rados.0.smithi063.stdout:171: finishing rollback tid 1 to smithi06387500-20 2024-08-17T16:55:22.348 INFO:tasks.rados.rados.0.smithi063.stdout:171: finishing rollback tid 2 to smithi06387500-20 2024-08-17T16:55:22.348 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 41 (ObjNum 88 snap 11 seq_num 88) dirty exists 2024-08-17T16:55:22.348 INFO:tasks.rados.rados.0.smithi063.stdout:170: finishing write tid 3 to smithi06387500-39 2024-08-17T16:55:22.348 INFO:tasks.rados.rados.0.smithi063.stdout:171: done (6 left) 2024-08-17T16:55:22.348 INFO:tasks.rados.rados.0.smithi063.stdout:177: write oid 38 current snap is 16 2024-08-17T16:55:22.348 INFO:tasks.rados.rados.0.smithi063.stdout:170: finishing write tid 4 to smithi06387500-39 2024-08-17T16:55:22.348 INFO:tasks.rados.rados.0.smithi063.stdout:170: finishing write tid 5 to smithi06387500-39 2024-08-17T16:55:22.348 INFO:tasks.rados.rados.0.smithi063.stdout:170: oid 39 updating version 0 to 10 2024-08-17T16:55:22.348 INFO:tasks.rados.rados.0.smithi063.stdout:170: oid 39 version 10 is already newer than 9 2024-08-17T16:55:22.349 INFO:tasks.rados.rados.0.smithi063.stdout:170: oid 39 updating version 10 to 11 2024-08-17T16:55:22.349 INFO:tasks.rados.rados.0.smithi063.stdout:170: oid 39 version 11 is already newer than 8 2024-08-17T16:55:22.349 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 11 (ObjNum 102 snap 16 seq_num 102) dirty exists 2024-08-17T16:55:22.349 INFO:tasks.rados.rados.0.smithi063.stdout:170: left oid 39 (ObjNum 102 snap 16 seq_num 102) 2024-08-17T16:55:22.349 INFO:tasks.rados.rados.0.smithi063.stdout:172: finishing rollback tid 0 to smithi06387500-9 2024-08-17T16:55:22.349 INFO:tasks.rados.rados.0.smithi063.stdout:172: finishing rollback tid 1 to smithi06387500-9 2024-08-17T16:55:22.349 INFO:tasks.rados.rados.0.smithi063.stdout:172: finishing rollback tid 2 to smithi06387500-9 2024-08-17T16:55:22.349 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 81 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-08-17T16:55:22.349 INFO:tasks.rados.rados.0.smithi063.stdout:173: finishing write tid 1 to smithi06387500-17 2024-08-17T16:55:22.349 INFO:tasks.rados.rados.0.smithi063.stdout:173: finishing write tid 2 to smithi06387500-17 2024-08-17T16:55:22.349 INFO:tasks.rados.rados.0.smithi063.stdout:177: seq_num 104 ranges {547928=538485,1769539=409023,2706697=532054} 2024-08-17T16:55:22.353 INFO:tasks.rados.rados.0.smithi063.stdout:177: writing smithi06387500-38 from 547928 to 1086413 tid 1 2024-08-17T16:55:22.361 INFO:tasks.rados.rados.0.smithi063.stdout:177: writing smithi06387500-38 from 1769539 to 2178562 tid 2 2024-08-17T16:55:22.368 INFO:tasks.rados.rados.0.smithi063.stdout:177: writing smithi06387500-38 from 2706697 to 3238751 tid 3 2024-08-17T16:55:22.368 INFO:tasks.rados.rados.0.smithi063.stdout:170: done (6 left) 2024-08-17T16:55:22.369 INFO:tasks.rados.rados.0.smithi063.stdout:172: done (5 left) 2024-08-17T16:55:22.369 INFO:tasks.rados.rados.0.smithi063.stdout:178: read oid 20 snap 11 2024-08-17T16:55:22.369 INFO:tasks.rados.rados.0.smithi063.stdout:178: expect (ObjNum 88 snap 11 seq_num 88) 2024-08-17T16:55:22.369 INFO:tasks.rados.rados.0.smithi063.stdout:179: rollback oid 42 current snap is 16 2024-08-17T16:55:22.369 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 42 to 14 2024-08-17T16:55:22.369 INFO:tasks.rados.rados.0.smithi063.stdout:176: finishing rollback tid 1 to smithi06387500-4 2024-08-17T16:55:22.369 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 42 (ObjNum 33 snap 0 seq_num 1786885368) dirty dne 2024-08-17T16:55:22.369 INFO:tasks.rados.rados.0.smithi063.stdout:173: finishing write tid 3 to smithi06387500-17 2024-08-17T16:55:22.369 INFO:tasks.rados.rados.0.smithi063.stdout:173: finishing write tid 4 to smithi06387500-17 2024-08-17T16:55:22.369 INFO:tasks.rados.rados.0.smithi063.stdout:173: finishing write tid 5 to smithi06387500-17 2024-08-17T16:55:22.369 INFO:tasks.rados.rados.0.smithi063.stdout:173: oid 17 updating version 0 to 44 2024-08-17T16:55:22.369 INFO:tasks.rados.rados.0.smithi063.stdout:173: oid 17 updating version 44 to 45 2024-08-17T16:55:22.369 INFO:tasks.rados.rados.0.smithi063.stdout:173: oid 17 version 45 is already newer than 43 2024-08-17T16:55:22.370 INFO:tasks.rados.rados.0.smithi063.stdout:173: oid 17 version 45 is already newer than 42 2024-08-17T16:55:22.370 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 45 (ObjNum 103 snap 16 seq_num 103) dirty exists 2024-08-17T16:55:22.370 INFO:tasks.rados.rados.0.smithi063.stdout:173: left oid 17 (ObjNum 103 snap 16 seq_num 103) 2024-08-17T16:55:22.370 INFO:tasks.rados.rados.0.smithi063.stdout:177: finishing write tid 1 to smithi06387500-38 2024-08-17T16:55:22.370 INFO:tasks.rados.rados.0.smithi063.stdout:175: expect (ObjNum 99 snap 16 seq_num 99) 2024-08-17T16:55:22.577 INFO:tasks.rados.rados.0.smithi063.stdout:177: finishing write tid 2 to smithi06387500-38 2024-08-17T16:55:22.577 INFO:tasks.rados.rados.0.smithi063.stdout:179: finishing rollback tid 0 to smithi06387500-42 2024-08-17T16:55:22.577 INFO:tasks.rados.rados.0.smithi063.stdout:179: finishing rollback tid 1 to smithi06387500-42 2024-08-17T16:55:22.577 INFO:tasks.rados.rados.0.smithi063.stdout:179: finishing rollback tid 2 to smithi06387500-42 2024-08-17T16:55:22.577 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 69 (ObjNum 87 snap 11 seq_num 87) dirty exists 2024-08-17T16:55:22.577 INFO:tasks.rados.rados.0.smithi063.stdout:178: expect (ObjNum 88 snap 11 seq_num 88) 2024-08-17T16:55:22.746 INFO:tasks.rados.rados.0.smithi063.stdout:177: finishing write tid 3 to smithi06387500-38 2024-08-17T16:55:22.746 INFO:tasks.rados.rados.0.smithi063.stdout:177: finishing write tid 4 to smithi06387500-38 2024-08-17T16:55:22.746 INFO:tasks.rados.rados.0.smithi063.stdout:177: finishing write tid 5 to smithi06387500-38 2024-08-17T16:55:22.746 INFO:tasks.rados.rados.0.smithi063.stdout:177: oid 38 updating version 0 to 23 2024-08-17T16:55:22.747 INFO:tasks.rados.rados.0.smithi063.stdout:177: oid 38 updating version 23 to 26 2024-08-17T16:55:22.747 INFO:tasks.rados.rados.0.smithi063.stdout:177: oid 38 version 26 is already newer than 25 2024-08-17T16:55:22.747 INFO:tasks.rados.rados.0.smithi063.stdout:177: oid 38 version 26 is already newer than 24 2024-08-17T16:55:22.747 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 26 (ObjNum 104 snap 16 seq_num 104) dirty exists 2024-08-17T16:55:22.747 INFO:tasks.rados.rados.0.smithi063.stdout:177: left oid 38 (ObjNum 104 snap 16 seq_num 104) 2024-08-17T16:55:22.747 INFO:tasks.rados.rados.0.smithi063.stdout:173: done (6 left) 2024-08-17T16:55:22.747 INFO:tasks.rados.rados.0.smithi063.stdout:174: done (5 left) 2024-08-17T16:55:22.747 INFO:tasks.rados.rados.0.smithi063.stdout:175: done (4 left) 2024-08-17T16:55:22.747 INFO:tasks.rados.rados.0.smithi063.stdout:176: done (3 left) 2024-08-17T16:55:22.747 INFO:tasks.rados.rados.0.smithi063.stdout:177: done (2 left) 2024-08-17T16:55:22.747 INFO:tasks.rados.rados.0.smithi063.stdout:178: done (1 left) 2024-08-17T16:55:22.747 INFO:tasks.rados.rados.0.smithi063.stdout:179: done (0 left) 2024-08-17T16:55:22.748 INFO:tasks.rados.rados.0.smithi063.stdout:180: rollback oid 8 current snap is 16 2024-08-17T16:55:22.748 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 8 to 11 2024-08-17T16:55:22.748 INFO:tasks.rados.rados.0.smithi063.stdout:181: read oid 50 snap 14 2024-08-17T16:55:22.748 INFO:tasks.rados.rados.0.smithi063.stdout:181: expect (ObjNum 63 snap 1 seq_num 63) 2024-08-17T16:55:22.748 INFO:tasks.rados.rados.0.smithi063.stdout:182: read oid 48 snap -1 2024-08-17T16:55:22.748 INFO:tasks.rados.rados.0.smithi063.stdout:182: expect (ObjNum 66 snap 5 seq_num 66) 2024-08-17T16:55:22.748 INFO:tasks.rados.rados.0.smithi063.stdout:183: write oid 16 current snap is 16 2024-08-17T16:55:22.748 INFO:tasks.rados.rados.0.smithi063.stdout:183: seq_num 105 ranges {564735=613734,1828021=669034,3134120=630615} 2024-08-17T16:55:22.753 INFO:tasks.rados.rados.0.smithi063.stdout:183: writing smithi06387500-16 from 564735 to 1178469 tid 1 2024-08-17T16:55:22.760 INFO:tasks.rados.rados.0.smithi063.stdout:183: writing smithi06387500-16 from 1828021 to 2497055 tid 2 2024-08-17T16:55:22.767 INFO:tasks.rados.rados.0.smithi063.stdout:183: writing smithi06387500-16 from 3134120 to 3764735 tid 3 2024-08-17T16:55:22.767 INFO:tasks.rados.rados.0.smithi063.stdout:184: write oid 31 current snap is 16 2024-08-17T16:55:22.767 INFO:tasks.rados.rados.0.smithi063.stdout:180: finishing rollback tid 0 to smithi06387500-8 2024-08-17T16:55:22.767 INFO:tasks.rados.rados.0.smithi063.stdout:180: finishing rollback tid 1 to smithi06387500-8 2024-08-17T16:55:22.767 INFO:tasks.rados.rados.0.smithi063.stdout:180: finishing rollback tid 2 to smithi06387500-8 2024-08-17T16:55:22.767 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 43 (ObjNum 50 snap 0 seq_num 50) dirty exists 2024-08-17T16:55:22.768 INFO:tasks.rados.rados.0.smithi063.stdout:181: expect (ObjNum 63 snap 1 seq_num 63) 2024-08-17T16:55:22.909 INFO:tasks.rados.rados.0.smithi063.stdout:182: expect (ObjNum 66 snap 5 seq_num 66) 2024-08-17T16:55:23.103 INFO:tasks.rados.rados.0.smithi063.stdout:183: finishing write tid 1 to smithi06387500-16 2024-08-17T16:55:23.103 INFO:tasks.rados.rados.0.smithi063.stdout:183: finishing write tid 2 to smithi06387500-16 2024-08-17T16:55:23.103 INFO:tasks.rados.rados.0.smithi063.stdout:184: seq_num 106 ranges {581542=688983,1886503=412692,2761543=670951} 2024-08-17T16:55:23.109 INFO:tasks.rados.rados.0.smithi063.stdout:184: writing smithi06387500-31 from 581542 to 1270525 tid 1 2024-08-17T16:55:23.115 INFO:tasks.rados.rados.0.smithi063.stdout:184: writing smithi06387500-31 from 1886503 to 2299195 tid 2 2024-08-17T16:55:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:184: writing smithi06387500-31 from 2761543 to 3432494 tid 3 2024-08-17T16:55:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:180: done (4 left) 2024-08-17T16:55:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:181: done (3 left) 2024-08-17T16:55:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:182: done (2 left) 2024-08-17T16:55:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:185: delete oid 8 current snap is 16 2024-08-17T16:55:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:183: finishing write tid 3 to smithi06387500-16 2024-08-17T16:55:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:183: finishing write tid 4 to smithi06387500-16 2024-08-17T16:55:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:183: finishing write tid 5 to smithi06387500-16 2024-08-17T16:55:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:183: oid 16 updating version 0 to 49 2024-08-17T16:55:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:183: oid 16 updating version 49 to 50 2024-08-17T16:55:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:183: oid 16 updating version 50 to 51 2024-08-17T16:55:23.123 INFO:tasks.rados.rados.0.smithi063.stdout:183: oid 16 version 51 is already newer than 48 2024-08-17T16:55:23.123 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 51 (ObjNum 105 snap 16 seq_num 105) dirty exists 2024-08-17T16:55:23.123 INFO:tasks.rados.rados.0.smithi063.stdout:183: left oid 16 (ObjNum 105 snap 16 seq_num 105) 2024-08-17T16:55:23.123 INFO:tasks.rados.rados.0.smithi063.stdout:184: finishing write tid 1 to smithi06387500-31 2024-08-17T16:55:23.123 INFO:tasks.rados.rados.0.smithi063.stdout:184: finishing write tid 2 to smithi06387500-31 2024-08-17T16:55:23.124 INFO:tasks.rados.rados.0.smithi063.stdout:183: done (2 left) 2024-08-17T16:55:23.124 INFO:tasks.rados.rados.0.smithi063.stdout:185: done (1 left) 2024-08-17T16:55:23.124 INFO:tasks.rados.rados.0.smithi063.stdout:186: delete oid 35 current snap is 16 2024-08-17T16:55:23.124 INFO:tasks.rados.rados.0.smithi063.stdout:184: finishing write tid 3 to smithi06387500-31 2024-08-17T16:55:23.124 INFO:tasks.rados.rados.0.smithi063.stdout:184: finishing write tid 4 to smithi06387500-31 2024-08-17T16:55:23.124 INFO:tasks.rados.rados.0.smithi063.stdout:184: finishing write tid 5 to smithi06387500-31 2024-08-17T16:55:23.124 INFO:tasks.rados.rados.0.smithi063.stdout:184: oid 31 updating version 0 to 71 2024-08-17T16:55:23.124 INFO:tasks.rados.rados.0.smithi063.stdout:184: oid 31 updating version 71 to 74 2024-08-17T16:55:23.124 INFO:tasks.rados.rados.0.smithi063.stdout:184: oid 31 version 74 is already newer than 72 2024-08-17T16:55:23.124 INFO:tasks.rados.rados.0.smithi063.stdout:184: oid 31 version 74 is already newer than 73 2024-08-17T16:55:23.124 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 74 (ObjNum 106 snap 16 seq_num 106) dirty exists 2024-08-17T16:55:23.125 INFO:tasks.rados.rados.0.smithi063.stdout:184: left oid 31 (ObjNum 106 snap 16 seq_num 106) 2024-08-17T16:55:23.125 INFO:tasks.rados.rados.0.smithi063.stdout:184: done (1 left) 2024-08-17T16:55:23.125 INFO:tasks.rados.rados.0.smithi063.stdout:186: done (0 left) 2024-08-17T16:55:23.125 INFO:tasks.rados.rados.0.smithi063.stdout:187: snap_create 2024-08-17T16:55:23.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:22 smithi063 ceph-mon[42680]: osdmap e106: 8 total, 8 up, 8 in 2024-08-17T16:55:23.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:22 smithi063 ceph-mon[42680]: pgmap v140: 89 pgs: 89 active+clean; 161 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 2.3 MiB/s wr, 18 op/s 2024-08-17T16:55:23.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:22 smithi063 ceph-mon[31063]: osdmap e106: 8 total, 8 up, 8 in 2024-08-17T16:55:23.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:22 smithi063 ceph-mon[31063]: pgmap v140: 89 pgs: 89 active+clean; 161 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 2.3 MiB/s wr, 18 op/s 2024-08-17T16:55:23.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:22 smithi162 ceph-mon[34633]: osdmap e106: 8 total, 8 up, 8 in 2024-08-17T16:55:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:22 smithi162 ceph-mon[34633]: pgmap v140: 89 pgs: 89 active+clean; 161 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 2.3 MiB/s wr, 18 op/s 2024-08-17T16:55:23.861 INFO:tasks.rados.rados.0.smithi063.stdout:187: done (0 left) 2024-08-17T16:55:23.861 INFO:tasks.rados.rados.0.smithi063.stdout:188: read oid 42 snap -1 2024-08-17T16:55:23.861 INFO:tasks.rados.rados.0.smithi063.stdout:188: expect (ObjNum 87 snap 11 seq_num 87) 2024-08-17T16:55:23.861 INFO:tasks.rados.rados.0.smithi063.stdout:189: rollback oid 30 current snap is 17 2024-08-17T16:55:23.862 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 30 to 14 2024-08-17T16:55:23.862 INFO:tasks.rados.rados.0.smithi063.stdout:190: delete oid 20 current snap is 17 2024-08-17T16:55:23.865 INFO:tasks.rados.rados.0.smithi063.stdout:190: done (2 left) 2024-08-17T16:55:23.865 INFO:tasks.rados.rados.0.smithi063.stdout:191: snap_remove snap 0 2024-08-17T16:55:24.821 INFO:tasks.rados.rados.0.smithi063.stdout:191: done (2 left) 2024-08-17T16:55:24.821 INFO:tasks.rados.rados.0.smithi063.stdout:192: snap_remove snap 15 2024-08-17T16:55:25.148 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:24 smithi162 ceph-mon[34633]: osdmap e107: 8 total, 8 up, 8 in 2024-08-17T16:55:25.148 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:24 smithi162 ceph-mon[34633]: pgmap v142: 89 pgs: 89 active+clean; 161 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-08-17T16:55:25.148 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:24 smithi162 ceph-mon[34633]: osdmap e108: 8 total, 8 up, 8 in 2024-08-17T16:55:25.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:24 smithi063 ceph-mon[42680]: osdmap e107: 8 total, 8 up, 8 in 2024-08-17T16:55:25.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:24 smithi063 ceph-mon[42680]: pgmap v142: 89 pgs: 89 active+clean; 161 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-08-17T16:55:25.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:24 smithi063 ceph-mon[42680]: osdmap e108: 8 total, 8 up, 8 in 2024-08-17T16:55:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:24 smithi063 ceph-mon[31063]: osdmap e107: 8 total, 8 up, 8 in 2024-08-17T16:55:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:24 smithi063 ceph-mon[31063]: pgmap v142: 89 pgs: 89 active+clean; 161 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 10 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-08-17T16:55:25.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:24 smithi063 ceph-mon[31063]: osdmap e108: 8 total, 8 up, 8 in 2024-08-17T16:55:25.824 INFO:tasks.rados.rados.0.smithi063.stdout:192: done (2 left) 2024-08-17T16:55:25.824 INFO:tasks.rados.rados.0.smithi063.stdout:193: read oid 11 snap 11 2024-08-17T16:55:25.824 INFO:tasks.rados.rados.0.smithi063.stdout:193: expect (ObjNum 10 snap 0 seq_num 10) 2024-08-17T16:55:25.824 INFO:tasks.rados.rados.0.smithi063.stdout:194: snap_create 2024-08-17T16:55:25.824 INFO:tasks.rados.rados.0.smithi063.stdout:189: finishing rollback tid 0 to smithi06387500-30 2024-08-17T16:55:25.825 INFO:tasks.rados.rados.0.smithi063.stdout:189: finishing rollback tid 1 to smithi06387500-30 2024-08-17T16:55:25.825 INFO:tasks.rados.rados.0.smithi063.stdout:189: finishing rollback tid 2 to smithi06387500-30 2024-08-17T16:55:25.825 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 85 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-08-17T16:55:25.825 INFO:tasks.rados.rados.0.smithi063.stdout:188: expect (ObjNum 87 snap 11 seq_num 87) 2024-08-17T16:55:26.026 INFO:tasks.rados.rados.0.smithi063.stdout:193: expect (ObjNum 10 snap 0 seq_num 10) 2024-08-17T16:55:26.826 INFO:tasks.rados.rados.0.smithi063.stdout:188: done (3 left) 2024-08-17T16:55:26.826 INFO:tasks.rados.rados.0.smithi063.stdout:189: done (2 left) 2024-08-17T16:55:26.826 INFO:tasks.rados.rados.0.smithi063.stdout:193: done (1 left) 2024-08-17T16:55:26.826 INFO:tasks.rados.rados.0.smithi063.stdout:194: done (0 left) 2024-08-17T16:55:26.826 INFO:tasks.rados.rados.0.smithi063.stdout:195: delete oid 45 current snap is 18 2024-08-17T16:55:26.830 INFO:tasks.rados.rados.0.smithi063.stdout:195: done (0 left) 2024-08-17T16:55:26.830 INFO:tasks.rados.rados.0.smithi063.stdout:196: read oid 12 snap 13 2024-08-17T16:55:26.830 INFO:tasks.rados.rados.0.smithi063.stdout:196: expect (ObjNum 68 snap 7 seq_num 68) 2024-08-17T16:55:26.830 INFO:tasks.rados.rados.0.smithi063.stdout:197: rollback oid 44 current snap is 18 2024-08-17T16:55:26.830 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 44 to 17 2024-08-17T16:55:26.830 INFO:tasks.rados.rados.0.smithi063.stdout:198: snap_remove snap 14 2024-08-17T16:55:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:26 smithi162 ceph-mon[34633]: osdmap e109: 8 total, 8 up, 8 in 2024-08-17T16:55:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:26 smithi162 ceph-mon[34633]: pgmap v145: 89 pgs: 89 active+clean; 147 MiB data, 487 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 2.9 MiB/s wr, 30 op/s 2024-08-17T16:55:27.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:26 smithi063 ceph-mon[31063]: osdmap e109: 8 total, 8 up, 8 in 2024-08-17T16:55:27.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:26 smithi063 ceph-mon[31063]: pgmap v145: 89 pgs: 89 active+clean; 147 MiB data, 487 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 2.9 MiB/s wr, 30 op/s 2024-08-17T16:55:27.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:26 smithi063 ceph-mon[42680]: osdmap e109: 8 total, 8 up, 8 in 2024-08-17T16:55:27.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:26 smithi063 ceph-mon[42680]: pgmap v145: 89 pgs: 89 active+clean; 147 MiB data, 487 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 2.9 MiB/s wr, 30 op/s 2024-08-17T16:55:27.831 INFO:tasks.rados.rados.0.smithi063.stdout:198: done (2 left) 2024-08-17T16:55:27.832 INFO:tasks.rados.rados.0.smithi063.stdout:199: delete oid 24 current snap is 18 2024-08-17T16:55:27.832 INFO:tasks.rados.rados.0.smithi063.stdout:197: finishing rollback tid 0 to smithi06387500-44 2024-08-17T16:55:27.832 INFO:tasks.rados.rados.0.smithi063.stdout:197: finishing rollback tid 1 to smithi06387500-44 2024-08-17T16:55:27.832 INFO:tasks.rados.rados.0.smithi063.stdout:197: finishing rollback tid 2 to smithi06387500-44 2024-08-17T16:55:27.832 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 18 (ObjNum 98 snap 16 seq_num 98) dirty exists 2024-08-17T16:55:27.832 INFO:tasks.rados.rados.0.smithi063.stdout:196: expect (ObjNum 68 snap 7 seq_num 68) 2024-08-17T16:55:27.983 INFO:tasks.rados.rados.0.smithi063.stdout:196: done (2 left) 2024-08-17T16:55:27.983 INFO:tasks.rados.rados.0.smithi063.stdout:197: done (1 left) 2024-08-17T16:55:27.984 INFO:tasks.rados.rados.0.smithi063.stdout:199: done (0 left) 2024-08-17T16:55:27.984 INFO:tasks.rados.rados.0.smithi063.stdout:200: read oid 9 snap 13 2024-08-17T16:55:27.984 INFO:tasks.rados.rados.0.smithi063.stdout:200: expect (ObjNum 8 snap 0 seq_num 8) 2024-08-17T16:55:27.984 INFO:tasks.rados.rados.0.smithi063.stdout:201: rollback oid 45 current snap is 18 2024-08-17T16:55:27.984 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 45 to 16 2024-08-17T16:55:27.984 INFO:tasks.rados.rados.0.smithi063.stdout:202: rollback oid 14 current snap is 18 2024-08-17T16:55:27.984 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 14 to 13 2024-08-17T16:55:27.984 INFO:tasks.rados.rados.0.smithi063.stdout:203: write oid 34 current snap is 18 2024-08-17T16:55:27.984 INFO:tasks.rados.rados.0.smithi063.stdout:203: seq_num 112 ranges {682384=456830,1721042=423699,2809728=454936,3731357=151027} 2024-08-17T16:55:27.990 INFO:tasks.rados.rados.0.smithi063.stdout:203: writing smithi06387500-34 from 682384 to 1139214 tid 1 2024-08-17T16:55:27.995 INFO:tasks.rados.rados.0.smithi063.stdout:203: writing smithi06387500-34 from 1721042 to 2144741 tid 2 2024-08-17T16:55:28.001 INFO:tasks.rados.rados.0.smithi063.stdout:203: writing smithi06387500-34 from 2809728 to 3264664 tid 3 2024-08-17T16:55:28.005 INFO:tasks.rados.rados.0.smithi063.stdout:203: writing smithi06387500-34 from 3731357 to 3882384 tid 4 2024-08-17T16:55:28.005 INFO:tasks.rados.rados.0.smithi063.stdout:204: snap_create 2024-08-17T16:55:28.005 INFO:tasks.rados.rados.0.smithi063.stdout:201: finishing rollback tid 1 to smithi06387500-45 2024-08-17T16:55:28.005 INFO:tasks.rados.rados.0.smithi063.stdout:201: finishing rollback tid 2 to smithi06387500-45 2024-08-17T16:55:28.006 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 17 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-08-17T16:55:28.006 INFO:tasks.rados.rados.0.smithi063.stdout:202: finishing rollback tid 0 to smithi06387500-14 2024-08-17T16:55:28.006 INFO:tasks.rados.rados.0.smithi063.stdout:202: finishing rollback tid 1 to smithi06387500-14 2024-08-17T16:55:28.006 INFO:tasks.rados.rados.0.smithi063.stdout:202: finishing rollback tid 2 to smithi06387500-14 2024-08-17T16:55:28.006 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 179 (ObjNum 75 snap 9 seq_num 75) dirty exists 2024-08-17T16:55:28.006 INFO:tasks.rados.rados.0.smithi063.stdout:200: expect (ObjNum 8 snap 0 seq_num 8) 2024-08-17T16:55:28.093 INFO:tasks.rados.rados.0.smithi063.stdout:203: finishing write tid 1 to smithi06387500-34 2024-08-17T16:55:28.093 INFO:tasks.rados.rados.0.smithi063.stdout:203: finishing write tid 2 to smithi06387500-34 2024-08-17T16:55:28.093 INFO:tasks.rados.rados.0.smithi063.stdout:203: finishing write tid 3 to smithi06387500-34 2024-08-17T16:55:28.093 INFO:tasks.rados.rados.0.smithi063.stdout:203: finishing write tid 4 to smithi06387500-34 2024-08-17T16:55:28.093 INFO:tasks.rados.rados.0.smithi063.stdout:203: finishing write tid 5 to smithi06387500-34 2024-08-17T16:55:28.093 INFO:tasks.rados.rados.0.smithi063.stdout:203: finishing write tid 6 to smithi06387500-34 2024-08-17T16:55:28.093 INFO:tasks.rados.rados.0.smithi063.stdout:203: oid 34 updating version 0 to 96 2024-08-17T16:55:28.093 INFO:tasks.rados.rados.0.smithi063.stdout:203: oid 34 updating version 96 to 97 2024-08-17T16:55:28.094 INFO:tasks.rados.rados.0.smithi063.stdout:203: oid 34 updating version 97 to 98 2024-08-17T16:55:28.094 INFO:tasks.rados.rados.0.smithi063.stdout:203: oid 34 version 98 is already newer than 94 2024-08-17T16:55:28.094 INFO:tasks.rados.rados.0.smithi063.stdout:203: oid 34 version 98 is already newer than 95 2024-08-17T16:55:28.094 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 98 (ObjNum 112 snap 18 seq_num 112) dirty exists 2024-08-17T16:55:28.094 INFO:tasks.rados.rados.0.smithi063.stdout:203: left oid 34 (ObjNum 112 snap 18 seq_num 112) 2024-08-17T16:55:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:27 smithi162 ceph-mon[34633]: osdmap e110: 8 total, 8 up, 8 in 2024-08-17T16:55:28.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:27 smithi063 ceph-mon[31063]: osdmap e110: 8 total, 8 up, 8 in 2024-08-17T16:55:28.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:27 smithi063 ceph-mon[42680]: osdmap e110: 8 total, 8 up, 8 in 2024-08-17T16:55:28.843 INFO:tasks.rados.rados.0.smithi063.stdout:200: done (4 left) 2024-08-17T16:55:28.843 INFO:tasks.rados.rados.0.smithi063.stdout:201: done (3 left) 2024-08-17T16:55:28.843 INFO:tasks.rados.rados.0.smithi063.stdout:202: done (2 left) 2024-08-17T16:55:28.843 INFO:tasks.rados.rados.0.smithi063.stdout:203: done (1 left) 2024-08-17T16:55:28.843 INFO:tasks.rados.rados.0.smithi063.stdout:204: done (0 left) 2024-08-17T16:55:28.843 INFO:tasks.rados.rados.0.smithi063.stdout:205: write oid 49 current snap is 19 2024-08-17T16:55:28.843 INFO:tasks.rados.rados.0.smithi063.stdout:205: seq_num 113 ranges {699191=532079,1779524=567357,2837151=666208} 2024-08-17T16:55:28.849 INFO:tasks.rados.rados.0.smithi063.stdout:205: writing smithi06387500-49 from 699191 to 1231270 tid 1 2024-08-17T16:55:28.856 INFO:tasks.rados.rados.0.smithi063.stdout:205: writing smithi06387500-49 from 1779524 to 2346881 tid 2 2024-08-17T16:55:28.863 INFO:tasks.rados.rados.0.smithi063.stdout:205: writing smithi06387500-49 from 2837151 to 3503359 tid 3 2024-08-17T16:55:28.863 INFO:tasks.rados.rados.0.smithi063.stdout:206: snap_remove snap 18 2024-08-17T16:55:29.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:28 smithi162 ceph-mon[34633]: osdmap e111: 8 total, 8 up, 8 in 2024-08-17T16:55:29.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:28 smithi162 ceph-mon[34633]: pgmap v148: 89 pgs: 89 active+clean; 147 MiB data, 487 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 3.4 MiB/s wr, 34 op/s 2024-08-17T16:55:29.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:28 smithi063 ceph-mon[42680]: osdmap e111: 8 total, 8 up, 8 in 2024-08-17T16:55:29.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:28 smithi063 ceph-mon[42680]: pgmap v148: 89 pgs: 89 active+clean; 147 MiB data, 487 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 3.4 MiB/s wr, 34 op/s 2024-08-17T16:55:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:28 smithi063 ceph-mon[31063]: osdmap e111: 8 total, 8 up, 8 in 2024-08-17T16:55:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:28 smithi063 ceph-mon[31063]: pgmap v148: 89 pgs: 89 active+clean; 147 MiB data, 487 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 3.4 MiB/s wr, 34 op/s 2024-08-17T16:55:29.821 INFO:tasks.rados.rados.0.smithi063.stdout:206: done (1 left) 2024-08-17T16:55:29.821 INFO:tasks.rados.rados.0.smithi063.stdout:207: rollback oid 12 current snap is 19 2024-08-17T16:55:29.821 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 12 to 11 2024-08-17T16:55:29.822 INFO:tasks.rados.rados.0.smithi063.stdout:205: finishing write tid 1 to smithi06387500-49 2024-08-17T16:55:29.822 INFO:tasks.rados.rados.0.smithi063.stdout:205: finishing write tid 2 to smithi06387500-49 2024-08-17T16:55:29.822 INFO:tasks.rados.rados.0.smithi063.stdout:205: finishing write tid 3 to smithi06387500-49 2024-08-17T16:55:29.822 INFO:tasks.rados.rados.0.smithi063.stdout:205: finishing write tid 4 to smithi06387500-49 2024-08-17T16:55:29.822 INFO:tasks.rados.rados.0.smithi063.stdout:205: finishing write tid 5 to smithi06387500-49 2024-08-17T16:55:29.822 INFO:tasks.rados.rados.0.smithi063.stdout:205: oid 49 updating version 0 to 182 2024-08-17T16:55:29.822 INFO:tasks.rados.rados.0.smithi063.stdout:205: oid 49 updating version 182 to 183 2024-08-17T16:55:29.822 INFO:tasks.rados.rados.0.smithi063.stdout:205: oid 49 updating version 183 to 184 2024-08-17T16:55:29.822 INFO:tasks.rados.rados.0.smithi063.stdout:205: oid 49 version 184 is already newer than 181 2024-08-17T16:55:29.822 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 184 (ObjNum 113 snap 19 seq_num 113) dirty exists 2024-08-17T16:55:29.822 INFO:tasks.rados.rados.0.smithi063.stdout:205: left oid 49 (ObjNum 113 snap 19 seq_num 113) 2024-08-17T16:55:29.822 INFO:tasks.rados.rados.0.smithi063.stdout:205: done (1 left) 2024-08-17T16:55:29.822 INFO:tasks.rados.rados.0.smithi063.stdout:208: snap_remove snap 13 2024-08-17T16:55:29.834 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:55:29 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:55:29] "GET /metrics HTTP/1.1" 200 36137 "" "Prometheus/2.43.0" 2024-08-17T16:55:29.834 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:29 smithi063 ceph-mon[31063]: osdmap e112: 8 total, 8 up, 8 in 2024-08-17T16:55:29.834 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:29 smithi063 ceph-mon[31063]: osdmap e113: 8 total, 8 up, 8 in 2024-08-17T16:55:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:29 smithi162 ceph-mon[34633]: osdmap e112: 8 total, 8 up, 8 in 2024-08-17T16:55:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:29 smithi162 ceph-mon[34633]: osdmap e113: 8 total, 8 up, 8 in 2024-08-17T16:55:30.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:29 smithi063 ceph-mon[42680]: osdmap e112: 8 total, 8 up, 8 in 2024-08-17T16:55:30.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:29 smithi063 ceph-mon[42680]: osdmap e113: 8 total, 8 up, 8 in 2024-08-17T16:55:30.824 INFO:tasks.rados.rados.0.smithi063.stdout:208: done (1 left) 2024-08-17T16:55:30.824 INFO:tasks.rados.rados.0.smithi063.stdout:209: delete oid 21 current snap is 19 2024-08-17T16:55:30.824 INFO:tasks.rados.rados.0.smithi063.stdout:207: finishing rollback tid 0 to smithi06387500-12 2024-08-17T16:55:30.824 INFO:tasks.rados.rados.0.smithi063.stdout:207: finishing rollback tid 1 to smithi06387500-12 2024-08-17T16:55:30.824 INFO:tasks.rados.rados.0.smithi063.stdout:207: finishing rollback tid 2 to smithi06387500-12 2024-08-17T16:55:30.824 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 42 (ObjNum 68 snap 7 seq_num 68) dirty exists 2024-08-17T16:55:30.837 INFO:tasks.rados.rados.0.smithi063.stdout:207: done (1 left) 2024-08-17T16:55:30.837 INFO:tasks.rados.rados.0.smithi063.stdout:209: done (0 left) 2024-08-17T16:55:30.837 INFO:tasks.rados.rados.0.smithi063.stdout:210: read oid 19 snap -1 2024-08-17T16:55:30.837 INFO:tasks.rados.rados.0.smithi063.stdout:210: expect (ObjNum 18 snap 0 seq_num 18) 2024-08-17T16:55:30.838 INFO:tasks.rados.rados.0.smithi063.stdout:211: snap_remove snap 11 2024-08-17T16:55:31.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:30 smithi063 ceph-mon[42680]: pgmap v151: 89 pgs: 89 active+clean; 147 MiB data, 487 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:31.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:30 smithi063 ceph-mon[42680]: osdmap e114: 8 total, 8 up, 8 in 2024-08-17T16:55:31.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:30 smithi063 ceph-mon[31063]: pgmap v151: 89 pgs: 89 active+clean; 147 MiB data, 487 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:31.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:30 smithi063 ceph-mon[31063]: osdmap e114: 8 total, 8 up, 8 in 2024-08-17T16:55:31.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:30 smithi162 ceph-mon[34633]: pgmap v151: 89 pgs: 89 active+clean; 147 MiB data, 487 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:31.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:30 smithi162 ceph-mon[34633]: osdmap e114: 8 total, 8 up, 8 in 2024-08-17T16:55:31.834 INFO:tasks.rados.rados.0.smithi063.stdout:211: done (1 left) 2024-08-17T16:55:31.834 INFO:tasks.rados.rados.0.smithi063.stdout:212: delete oid 2 current snap is 19 2024-08-17T16:55:31.838 INFO:tasks.rados.rados.0.smithi063.stdout:210: expect (ObjNum 18 snap 0 seq_num 18) 2024-08-17T16:55:31.913 INFO:tasks.rados.rados.0.smithi063.stdout:210: done (1 left) 2024-08-17T16:55:31.913 INFO:tasks.rados.rados.0.smithi063.stdout:212: done (0 left) 2024-08-17T16:55:31.913 INFO:tasks.rados.rados.0.smithi063.stdout:213: snap_create 2024-08-17T16:55:32.838 INFO:tasks.rados.rados.0.smithi063.stdout:213: done (0 left) 2024-08-17T16:55:32.838 INFO:tasks.rados.rados.0.smithi063.stdout:214: delete oid 46 current snap is 20 2024-08-17T16:55:32.841 INFO:tasks.rados.rados.0.smithi063.stdout:214: done (0 left) 2024-08-17T16:55:32.841 INFO:tasks.rados.rados.0.smithi063.stdout:215: write oid 16 current snap is 20 2024-08-17T16:55:32.842 INFO:tasks.rados.rados.0.smithi063.stdout:215: seq_num 117 ranges {766419=549428,2013452=574695,3346843=427649} 2024-08-17T16:55:32.848 INFO:tasks.rados.rados.0.smithi063.stdout:215: writing smithi06387500-16 from 766419 to 1315847 tid 1 2024-08-17T16:55:32.856 INFO:tasks.rados.rados.0.smithi063.stdout:215: writing smithi06387500-16 from 2013452 to 2588147 tid 2 2024-08-17T16:55:32.862 INFO:tasks.rados.rados.0.smithi063.stdout:215: writing smithi06387500-16 from 3346843 to 3774492 tid 3 2024-08-17T16:55:32.863 INFO:tasks.rados.rados.0.smithi063.stdout:216: read oid 30 snap -1 2024-08-17T16:55:32.863 INFO:tasks.rados.rados.0.smithi063.stdout:216: expect (ObjNum 29 snap 0 seq_num 29) 2024-08-17T16:55:32.863 INFO:tasks.rados.rados.0.smithi063.stdout:217: snap_remove snap 16 2024-08-17T16:55:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:32 smithi162 ceph-mon[34633]: osdmap e115: 8 total, 8 up, 8 in 2024-08-17T16:55:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:32 smithi162 ceph-mon[34633]: pgmap v154: 89 pgs: 89 active+clean; 154 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 795 KiB/s wr, 13 op/s 2024-08-17T16:55:33.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:32 smithi063 ceph-mon[42680]: osdmap e115: 8 total, 8 up, 8 in 2024-08-17T16:55:33.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:32 smithi063 ceph-mon[42680]: pgmap v154: 89 pgs: 89 active+clean; 154 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 795 KiB/s wr, 13 op/s 2024-08-17T16:55:33.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:32 smithi063 ceph-mon[31063]: osdmap e115: 8 total, 8 up, 8 in 2024-08-17T16:55:33.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:32 smithi063 ceph-mon[31063]: pgmap v154: 89 pgs: 89 active+clean; 154 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 795 KiB/s wr, 13 op/s 2024-08-17T16:55:33.845 INFO:tasks.rados.rados.0.smithi063.stdout:217: done (2 left) 2024-08-17T16:55:33.845 INFO:tasks.rados.rados.0.smithi063.stdout:218: snap_remove snap 17 2024-08-17T16:55:34.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:33 smithi162 ceph-mon[34633]: osdmap e116: 8 total, 8 up, 8 in 2024-08-17T16:55:34.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:33 smithi063 ceph-mon[42680]: osdmap e116: 8 total, 8 up, 8 in 2024-08-17T16:55:34.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:33 smithi063 ceph-mon[31063]: osdmap e116: 8 total, 8 up, 8 in 2024-08-17T16:55:34.822 INFO:tasks.rados.rados.0.smithi063.stdout:218: done (2 left) 2024-08-17T16:55:34.822 INFO:tasks.rados.rados.0.smithi063.stdout:219: write oid 18 current snap is 20 2024-08-17T16:55:34.823 INFO:tasks.rados.rados.0.smithi063.stdout:219: seq_num 118 ranges {783226=624677,2071934=434706,2974266=638921} 2024-08-17T16:55:34.830 INFO:tasks.rados.rados.0.smithi063.stdout:219: writing smithi06387500-18 from 783226 to 1407903 tid 1 2024-08-17T16:55:34.837 INFO:tasks.rados.rados.0.smithi063.stdout:219: writing smithi06387500-18 from 2071934 to 2506640 tid 2 2024-08-17T16:55:34.844 INFO:tasks.rados.rados.0.smithi063.stdout:219: writing smithi06387500-18 from 2974266 to 3613187 tid 3 2024-08-17T16:55:34.844 INFO:tasks.rados.rados.0.smithi063.stdout:220: rollback oid 7 current snap is 20 2024-08-17T16:55:34.844 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 19 2024-08-17T16:55:34.844 INFO:tasks.rados.rados.0.smithi063.stdout:215: finishing write tid 1 to smithi06387500-16 2024-08-17T16:55:34.844 INFO:tasks.rados.rados.0.smithi063.stdout:215: finishing write tid 2 to smithi06387500-16 2024-08-17T16:55:34.844 INFO:tasks.rados.rados.0.smithi063.stdout:215: finishing write tid 3 to smithi06387500-16 2024-08-17T16:55:34.844 INFO:tasks.rados.rados.0.smithi063.stdout:215: finishing write tid 4 to smithi06387500-16 2024-08-17T16:55:34.844 INFO:tasks.rados.rados.0.smithi063.stdout:215: finishing write tid 5 to smithi06387500-16 2024-08-17T16:55:34.845 INFO:tasks.rados.rados.0.smithi063.stdout:215: oid 16 updating version 0 to 84 2024-08-17T16:55:34.845 INFO:tasks.rados.rados.0.smithi063.stdout:215: oid 16 version 84 is already newer than 83 2024-08-17T16:55:34.845 INFO:tasks.rados.rados.0.smithi063.stdout:215: oid 16 updating version 84 to 86 2024-08-17T16:55:34.845 INFO:tasks.rados.rados.0.smithi063.stdout:215: oid 16 version 86 is already newer than 85 2024-08-17T16:55:34.845 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 86 (ObjNum 117 snap 20 seq_num 117) dirty exists 2024-08-17T16:55:34.845 INFO:tasks.rados.rados.0.smithi063.stdout:215: left oid 16 (ObjNum 117 snap 20 seq_num 117) 2024-08-17T16:55:34.845 INFO:tasks.rados.rados.0.smithi063.stdout:215: done (3 left) 2024-08-17T16:55:34.845 INFO:tasks.rados.rados.0.smithi063.stdout:221: snap_create 2024-08-17T16:55:34.845 INFO:tasks.rados.rados.0.smithi063.stdout:216: expect (ObjNum 29 snap 0 seq_num 29) 2024-08-17T16:55:34.931 INFO:tasks.rados.rados.0.smithi063.stdout:219: finishing write tid 1 to smithi06387500-18 2024-08-17T16:55:34.931 INFO:tasks.rados.rados.0.smithi063.stdout:219: finishing write tid 2 to smithi06387500-18 2024-08-17T16:55:34.931 INFO:tasks.rados.rados.0.smithi063.stdout:220: finishing rollback tid 1 to smithi06387500-7 2024-08-17T16:55:34.931 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 0 (ObjNum 129 snap 0 seq_num 1807182992) dirty dne 2024-08-17T16:55:34.931 INFO:tasks.rados.rados.0.smithi063.stdout:219: finishing write tid 3 to smithi06387500-18 2024-08-17T16:55:34.931 INFO:tasks.rados.rados.0.smithi063.stdout:219: finishing write tid 4 to smithi06387500-18 2024-08-17T16:55:34.931 INFO:tasks.rados.rados.0.smithi063.stdout:219: finishing write tid 5 to smithi06387500-18 2024-08-17T16:55:34.931 INFO:tasks.rados.rados.0.smithi063.stdout:219: oid 18 updating version 0 to 55 2024-08-17T16:55:34.931 INFO:tasks.rados.rados.0.smithi063.stdout:219: oid 18 version 55 is already newer than 53 2024-08-17T16:55:34.932 INFO:tasks.rados.rados.0.smithi063.stdout:219: oid 18 updating version 55 to 56 2024-08-17T16:55:34.932 INFO:tasks.rados.rados.0.smithi063.stdout:219: oid 18 version 56 is already newer than 54 2024-08-17T16:55:34.932 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 56 (ObjNum 118 snap 20 seq_num 118) dirty exists 2024-08-17T16:55:34.932 INFO:tasks.rados.rados.0.smithi063.stdout:219: left oid 18 (ObjNum 118 snap 20 seq_num 118) 2024-08-17T16:55:35.142 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:34 smithi063 ceph-mon[42680]: osdmap e117: 8 total, 8 up, 8 in 2024-08-17T16:55:35.142 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:34 smithi063 ceph-mon[42680]: pgmap v157: 89 pgs: 89 active+clean; 154 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 795 KiB/s wr, 13 op/s 2024-08-17T16:55:35.142 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:34 smithi063 ceph-mon[42680]: osdmap e118: 8 total, 8 up, 8 in 2024-08-17T16:55:35.142 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:34 smithi063 ceph-mon[31063]: osdmap e117: 8 total, 8 up, 8 in 2024-08-17T16:55:35.142 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:34 smithi063 ceph-mon[31063]: pgmap v157: 89 pgs: 89 active+clean; 154 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 795 KiB/s wr, 13 op/s 2024-08-17T16:55:35.143 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:34 smithi063 ceph-mon[31063]: osdmap e118: 8 total, 8 up, 8 in 2024-08-17T16:55:35.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:34 smithi162 ceph-mon[34633]: osdmap e117: 8 total, 8 up, 8 in 2024-08-17T16:55:35.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:34 smithi162 ceph-mon[34633]: pgmap v157: 89 pgs: 89 active+clean; 154 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 795 KiB/s wr, 13 op/s 2024-08-17T16:55:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:34 smithi162 ceph-mon[34633]: osdmap e118: 8 total, 8 up, 8 in 2024-08-17T16:55:35.850 INFO:tasks.rados.rados.0.smithi063.stdout:216: done (3 left) 2024-08-17T16:55:35.850 INFO:tasks.rados.rados.0.smithi063.stdout:219: done (2 left) 2024-08-17T16:55:35.850 INFO:tasks.rados.rados.0.smithi063.stdout:220: done (1 left) 2024-08-17T16:55:35.850 INFO:tasks.rados.rados.0.smithi063.stdout:221: done (0 left) 2024-08-17T16:55:35.850 INFO:tasks.rados.rados.0.smithi063.stdout:222: delete oid 28 current snap is 21 2024-08-17T16:55:35.852 INFO:tasks.rados.rados.0.smithi063.stdout:222: done (0 left) 2024-08-17T16:55:35.852 INFO:tasks.rados.rados.0.smithi063.stdout:223: snap_create 2024-08-17T16:55:36.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:55:36.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:55:36.339 INFO:teuthology.orchestra.run.smithi063.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:55:36.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:55:36.863 INFO:tasks.rados.rados.0.smithi063.stdout:223: done (0 left) 2024-08-17T16:55:36.863 INFO:tasks.rados.rados.0.smithi063.stdout:224: read oid 49 snap -1 2024-08-17T16:55:36.863 INFO:tasks.rados.rados.0.smithi063.stdout:224: expect (ObjNum 113 snap 19 seq_num 113) 2024-08-17T16:55:36.863 INFO:tasks.rados.rados.0.smithi063.stdout:225: read oid 23 snap 21 2024-08-17T16:55:36.864 INFO:tasks.rados.rados.0.smithi063.stdout:225: expect deleted 2024-08-17T16:55:36.864 INFO:tasks.rados.rados.0.smithi063.stdout:226: write oid 7 current snap is 22 2024-08-17T16:55:36.864 INFO:tasks.rados.rados.0.smithi063.stdout:226: seq_num 120 ranges {416840=775175,1672545=344295} 2024-08-17T16:55:36.869 INFO:tasks.rados.rados.0.smithi063.stdout:226: writing smithi06387500-7 from 416840 to 1192015 tid 1 2024-08-17T16:55:36.874 INFO:tasks.rados.rados.0.smithi063.stdout:226: writing smithi06387500-7 from 1672545 to 2016840 tid 2 2024-08-17T16:55:36.874 INFO:tasks.rados.rados.0.smithi063.stdout:227: snap_create 2024-08-17T16:55:36.878 INFO:tasks.rados.rados.0.smithi063.stdout:226: finishing write tid 1 to smithi06387500-7 2024-08-17T16:55:36.878 INFO:tasks.rados.rados.0.smithi063.stdout:226: finishing write tid 2 to smithi06387500-7 2024-08-17T16:55:36.878 INFO:tasks.rados.rados.0.smithi063.stdout:226: finishing write tid 3 to smithi06387500-7 2024-08-17T16:55:36.878 INFO:tasks.rados.rados.0.smithi063.stdout:226: finishing write tid 4 to smithi06387500-7 2024-08-17T16:55:36.878 INFO:tasks.rados.rados.0.smithi063.stdout:226: oid 7 updating version 0 to 113 2024-08-17T16:55:36.878 INFO:tasks.rados.rados.0.smithi063.stdout:226: oid 7 version 113 is already newer than 112 2024-08-17T16:55:36.878 INFO:tasks.rados.rados.0.smithi063.stdout:226: oid 7 updating version 113 to 114 2024-08-17T16:55:36.878 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 114 (ObjNum 120 snap 22 seq_num 120) dirty exists 2024-08-17T16:55:36.879 INFO:tasks.rados.rados.0.smithi063.stdout:226: left oid 7 (ObjNum 120 snap 22 seq_num 120) 2024-08-17T16:55:36.879 INFO:tasks.rados.rados.0.smithi063.stdout:224: expect (ObjNum 113 snap 19 seq_num 113) 2024-08-17T16:55:36.979 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:36 smithi063 ceph-mon[31063]: osdmap e119: 8 total, 8 up, 8 in 2024-08-17T16:55:36.979 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:36 smithi063 ceph-mon[31063]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:55:36.979 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:36 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:36.979 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:36 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:36.979 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:36 smithi063 ceph-mon[31063]: pgmap v160: 89 pgs: 89 active+clean; 124 MiB data, 452 MiB used, 715 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 4 op/s 2024-08-17T16:55:36.979 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:36 smithi063 ceph-mon[42680]: osdmap e119: 8 total, 8 up, 8 in 2024-08-17T16:55:36.979 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:36 smithi063 ceph-mon[42680]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:55:36.979 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:36 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:36.979 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:36 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:36.980 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:36 smithi063 ceph-mon[42680]: pgmap v160: 89 pgs: 89 active+clean; 124 MiB data, 452 MiB used, 715 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 4 op/s 2024-08-17T16:55:37.004 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-17T16:55:37.137 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:36 smithi162 ceph-mon[34633]: osdmap e119: 8 total, 8 up, 8 in 2024-08-17T16:55:37.138 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:36 smithi162 ceph-mon[34633]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:55:37.138 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:36 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:37.138 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:36 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:37.138 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:36 smithi162 ceph-mon[34633]: pgmap v160: 89 pgs: 89 active+clean; 124 MiB data, 452 MiB used, 715 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 4 op/s 2024-08-17T16:55:37.820 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:55:37.875 INFO:tasks.rados.rados.0.smithi063.stdout:224: done (3 left) 2024-08-17T16:55:37.876 INFO:tasks.rados.rados.0.smithi063.stdout:225: done (2 left) 2024-08-17T16:55:37.876 INFO:tasks.rados.rados.0.smithi063.stdout:226: done (1 left) 2024-08-17T16:55:37.876 INFO:tasks.rados.rados.0.smithi063.stdout:227: done (0 left) 2024-08-17T16:55:37.876 INFO:tasks.rados.rados.0.smithi063.stdout:228: rollback oid 47 current snap is 23 2024-08-17T16:55:37.876 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 47 to 19 2024-08-17T16:55:37.876 INFO:tasks.rados.rados.0.smithi063.stdout:229: delete oid 28 current snap is 23 2024-08-17T16:55:37.876 INFO:tasks.rados.rados.0.smithi063.stdout:229: done (1 left) 2024-08-17T16:55:37.877 INFO:tasks.rados.rados.0.smithi063.stdout:230: snap_remove snap 21 2024-08-17T16:55:38.145 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:37 smithi063 ceph-mon[42680]: osdmap e120: 8 total, 8 up, 8 in 2024-08-17T16:55:38.145 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:37 smithi063 ceph-mon[31063]: osdmap e120: 8 total, 8 up, 8 in 2024-08-17T16:55:38.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:37 smithi162 ceph-mon[34633]: osdmap e120: 8 total, 8 up, 8 in 2024-08-17T16:55:38.880 INFO:tasks.rados.rados.0.smithi063.stdout:230: done (1 left) 2024-08-17T16:55:38.881 INFO:tasks.rados.rados.0.smithi063.stdout:231: write oid 28 current snap is 23 2024-08-17T16:55:38.881 INFO:tasks.rados.rados.0.smithi063.stdout:228: finishing rollback tid 0 to smithi06387500-47 2024-08-17T16:55:38.881 INFO:tasks.rados.rados.0.smithi063.stdout:231: seq_num 122 ranges {450454=642026,1505862=544592} 2024-08-17T16:55:38.886 INFO:tasks.rados.rados.0.smithi063.stdout:231: writing smithi06387500-28 from 450454 to 1092480 tid 1 2024-08-17T16:55:38.892 INFO:tasks.rados.rados.0.smithi063.stdout:231: writing smithi06387500-28 from 1505862 to 2050454 tid 2 2024-08-17T16:55:38.892 INFO:tasks.rados.rados.0.smithi063.stdout:232: read oid 40 snap -1 2024-08-17T16:55:38.892 INFO:tasks.rados.rados.0.smithi063.stdout:232: expect (ObjNum 39 snap 0 seq_num 39) 2024-08-17T16:55:38.892 INFO:tasks.rados.rados.0.smithi063.stdout:233: read oid 46 snap -1 2024-08-17T16:55:38.893 INFO:tasks.rados.rados.0.smithi063.stdout:233: expect deleted 2024-08-17T16:55:38.893 INFO:tasks.rados.rados.0.smithi063.stdout:234: read oid 15 snap -1 2024-08-17T16:55:38.893 INFO:tasks.rados.rados.0.smithi063.stdout:234: expect (ObjNum 14 snap 0 seq_num 14) 2024-08-17T16:55:38.893 INFO:tasks.rados.rados.0.smithi063.stdout:235: delete oid 27 current snap is 23 2024-08-17T16:55:38.893 INFO:tasks.rados.rados.0.smithi063.stdout:228: finishing rollback tid 1 to smithi06387500-47 2024-08-17T16:55:38.893 INFO:tasks.rados.rados.0.smithi063.stdout:228: finishing rollback tid 2 to smithi06387500-47 2024-08-17T16:55:38.893 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 46 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-08-17T16:55:38.893 INFO:tasks.rados.rados.0.smithi063.stdout:231: finishing write tid 1 to smithi06387500-28 2024-08-17T16:55:38.898 INFO:tasks.rados.rados.0.smithi063.stdout:228: done (5 left) 2024-08-17T16:55:38.898 INFO:tasks.rados.rados.0.smithi063.stdout:233: done (4 left) 2024-08-17T16:55:38.898 INFO:tasks.rados.rados.0.smithi063.stdout:235: done (3 left) 2024-08-17T16:55:38.898 INFO:tasks.rados.rados.0.smithi063.stdout:236: write oid 8 current snap is 23 2024-08-17T16:55:38.898 INFO:tasks.rados.rados.0.smithi063.stdout:231: finishing write tid 2 to smithi06387500-28 2024-08-17T16:55:38.898 INFO:tasks.rados.rados.0.smithi063.stdout:231: finishing write tid 3 to smithi06387500-28 2024-08-17T16:55:38.898 INFO:tasks.rados.rados.0.smithi063.stdout:236: seq_num 124 ranges {484068=792524,1906473=177595} 2024-08-17T16:55:38.905 INFO:tasks.rados.rados.0.smithi063.stdout:236: writing smithi06387500-8 from 484068 to 1276592 tid 1 2024-08-17T16:55:38.909 INFO:tasks.rados.rados.0.smithi063.stdout:236: writing smithi06387500-8 from 1906473 to 2084068 tid 2 2024-08-17T16:55:38.909 INFO:tasks.rados.rados.0.smithi063.stdout:237: write oid 23 current snap is 23 2024-08-17T16:55:38.910 INFO:tasks.rados.rados.0.smithi063.stdout:237: seq_num 125 ranges {500875=467773,1564955=535920} 2024-08-17T16:55:38.915 INFO:tasks.rados.rados.0.smithi063.stdout:237: writing smithi06387500-23 from 500875 to 968648 tid 1 2024-08-17T16:55:38.922 INFO:tasks.rados.rados.0.smithi063.stdout:237: writing smithi06387500-23 from 1564955 to 2100875 tid 2 2024-08-17T16:55:38.922 INFO:tasks.rados.rados.0.smithi063.stdout:238: snap_create 2024-08-17T16:55:38.922 INFO:tasks.rados.rados.0.smithi063.stdout:231: finishing write tid 4 to smithi06387500-28 2024-08-17T16:55:38.922 INFO:tasks.rados.rados.0.smithi063.stdout:231: oid 28 updating version 0 to 68 2024-08-17T16:55:38.922 INFO:tasks.rados.rados.0.smithi063.stdout:231: oid 28 updating version 68 to 70 2024-08-17T16:55:38.922 INFO:tasks.rados.rados.0.smithi063.stdout:231: oid 28 version 70 is already newer than 69 2024-08-17T16:55:38.922 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 70 (ObjNum 122 snap 23 seq_num 122) dirty exists 2024-08-17T16:55:38.922 INFO:tasks.rados.rados.0.smithi063.stdout:231: left oid 28 (ObjNum 122 snap 23 seq_num 122) 2024-08-17T16:55:38.922 INFO:tasks.rados.rados.0.smithi063.stdout:234: expect (ObjNum 14 snap 0 seq_num 14) 2024-08-17T16:55:39.000 INFO:tasks.rados.rados.0.smithi063.stdout:232: expect (ObjNum 39 snap 0 seq_num 39) 2024-08-17T16:55:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:38 smithi063 ceph-mon[42680]: osdmap e121: 8 total, 8 up, 8 in 2024-08-17T16:55:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:38 smithi063 ceph-mon[42680]: pgmap v163: 89 pgs: 89 active+clean; 124 MiB data, 452 MiB used, 715 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 4 op/s 2024-08-17T16:55:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:38 smithi063 ceph-mon[31063]: osdmap e121: 8 total, 8 up, 8 in 2024-08-17T16:55:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:38 smithi063 ceph-mon[31063]: pgmap v163: 89 pgs: 89 active+clean; 124 MiB data, 452 MiB used, 715 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 4 op/s 2024-08-17T16:55:39.095 INFO:tasks.rados.rados.0.smithi063.stdout:236: finishing write tid 1 to smithi06387500-8 2024-08-17T16:55:39.095 INFO:tasks.rados.rados.0.smithi063.stdout:236: finishing write tid 2 to smithi06387500-8 2024-08-17T16:55:39.095 INFO:tasks.rados.rados.0.smithi063.stdout:236: finishing write tid 3 to smithi06387500-8 2024-08-17T16:55:39.095 INFO:tasks.rados.rados.0.smithi063.stdout:236: finishing write tid 4 to smithi06387500-8 2024-08-17T16:55:39.096 INFO:tasks.rados.rados.0.smithi063.stdout:236: oid 8 updating version 0 to 67 2024-08-17T16:55:39.096 INFO:tasks.rados.rados.0.smithi063.stdout:236: oid 8 version 67 is already newer than 65 2024-08-17T16:55:39.096 INFO:tasks.rados.rados.0.smithi063.stdout:236: oid 8 version 67 is already newer than 66 2024-08-17T16:55:39.096 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 67 (ObjNum 124 snap 23 seq_num 124) dirty exists 2024-08-17T16:55:39.096 INFO:tasks.rados.rados.0.smithi063.stdout:236: left oid 8 (ObjNum 124 snap 23 seq_num 124) 2024-08-17T16:55:39.096 INFO:tasks.rados.rados.0.smithi063.stdout:237: finishing write tid 1 to smithi06387500-23 2024-08-17T16:55:39.096 INFO:tasks.rados.rados.0.smithi063.stdout:237: finishing write tid 2 to smithi06387500-23 2024-08-17T16:55:39.096 INFO:tasks.rados.rados.0.smithi063.stdout:237: finishing write tid 3 to smithi06387500-23 2024-08-17T16:55:39.096 INFO:tasks.rados.rados.0.smithi063.stdout:237: finishing write tid 4 to smithi06387500-23 2024-08-17T16:55:39.096 INFO:tasks.rados.rados.0.smithi063.stdout:237: oid 23 updating version 0 to 77 2024-08-17T16:55:39.096 INFO:tasks.rados.rados.0.smithi063.stdout:237: oid 23 updating version 77 to 78 2024-08-17T16:55:39.096 INFO:tasks.rados.rados.0.smithi063.stdout:237: oid 23 version 78 is already newer than 76 2024-08-17T16:55:39.096 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 78 (ObjNum 125 snap 23 seq_num 125) dirty exists 2024-08-17T16:55:39.097 INFO:tasks.rados.rados.0.smithi063.stdout:237: left oid 23 (ObjNum 125 snap 23 seq_num 125) 2024-08-17T16:55:39.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:38 smithi162 ceph-mon[34633]: osdmap e121: 8 total, 8 up, 8 in 2024-08-17T16:55:39.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:38 smithi162 ceph-mon[34633]: pgmap v163: 89 pgs: 89 active+clean; 124 MiB data, 452 MiB used, 715 GiB / 715 GiB avail; 742 KiB/s rd, 794 KiB/s wr, 4 op/s 2024-08-17T16:55:39.823 INFO:tasks.rados.rados.0.smithi063.stdout:231: done (5 left) 2024-08-17T16:55:39.823 INFO:tasks.rados.rados.0.smithi063.stdout:232: done (4 left) 2024-08-17T16:55:39.823 INFO:tasks.rados.rados.0.smithi063.stdout:234: done (3 left) 2024-08-17T16:55:39.823 INFO:tasks.rados.rados.0.smithi063.stdout:236: done (2 left) 2024-08-17T16:55:39.823 INFO:tasks.rados.rados.0.smithi063.stdout:237: done (1 left) 2024-08-17T16:55:39.823 INFO:tasks.rados.rados.0.smithi063.stdout:238: done (0 left) 2024-08-17T16:55:39.823 INFO:tasks.rados.rados.0.smithi063.stdout:239: read oid 44 snap -1 2024-08-17T16:55:39.823 INFO:tasks.rados.rados.0.smithi063.stdout:239: expect (ObjNum 98 snap 16 seq_num 98) 2024-08-17T16:55:39.823 INFO:tasks.rados.rados.0.smithi063.stdout:240: write oid 22 current snap is 24 2024-08-17T16:55:39.823 INFO:tasks.rados.rados.0.smithi063.stdout:240: seq_num 126 ranges {517682=543022,1623437=494245} 2024-08-17T16:55:39.828 INFO:tasks.rados.rados.0.smithi063.stdout:240: writing smithi06387500-22 from 517682 to 1060704 tid 1 2024-08-17T16:55:39.835 INFO:tasks.rados.rados.0.smithi063.stdout:240: writing smithi06387500-22 from 1623437 to 2117682 tid 2 2024-08-17T16:55:39.835 INFO:tasks.rados.rados.0.smithi063.stdout:241: snap_create 2024-08-17T16:55:39.835 INFO:tasks.rados.rados.0.smithi063.stdout:240: finishing write tid 1 to smithi06387500-22 2024-08-17T16:55:39.835 INFO:tasks.rados.rados.0.smithi063.stdout:239: expect (ObjNum 98 snap 16 seq_num 98) 2024-08-17T16:55:39.846 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:55:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:55:39] "GET /metrics HTTP/1.1" 200 36137 "" "Prometheus/2.43.0" 2024-08-17T16:55:39.940 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T16:55:39.999 INFO:tasks.rados.rados.0.smithi063.stdout:240: finishing write tid 2 to smithi06387500-22 2024-08-17T16:55:39.999 INFO:tasks.rados.rados.0.smithi063.stdout:240: finishing write tid 3 to smithi06387500-22 2024-08-17T16:55:39.999 INFO:tasks.rados.rados.0.smithi063.stdout:240: finishing write tid 4 to smithi06387500-22 2024-08-17T16:55:39.999 INFO:tasks.rados.rados.0.smithi063.stdout:240: oid 22 updating version 0 to 73 2024-08-17T16:55:39.999 INFO:tasks.rados.rados.0.smithi063.stdout:240: oid 22 updating version 73 to 74 2024-08-17T16:55:39.999 INFO:tasks.rados.rados.0.smithi063.stdout:240: oid 22 version 74 is already newer than 72 2024-08-17T16:55:39.999 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 74 (ObjNum 126 snap 24 seq_num 126) dirty exists 2024-08-17T16:55:39.999 INFO:tasks.rados.rados.0.smithi063.stdout:240: left oid 22 (ObjNum 126 snap 24 seq_num 126) 2024-08-17T16:55:40.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:39 smithi063 ceph-mon[42680]: osdmap e122: 8 total, 8 up, 8 in 2024-08-17T16:55:40.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:39 smithi063 ceph-mon[42680]: osdmap e123: 8 total, 8 up, 8 in 2024-08-17T16:55:40.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:39 smithi063 ceph-mon[31063]: osdmap e122: 8 total, 8 up, 8 in 2024-08-17T16:55:40.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:39 smithi063 ceph-mon[31063]: osdmap e123: 8 total, 8 up, 8 in 2024-08-17T16:55:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:39 smithi162 ceph-mon[34633]: osdmap e122: 8 total, 8 up, 8 in 2024-08-17T16:55:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:39 smithi162 ceph-mon[34633]: osdmap e123: 8 total, 8 up, 8 in 2024-08-17T16:55:40.515 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T16:55:40.515 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (2m) 65s ago 3m 26.8M - 0.25.0 c8568f914cd2 87286b24f231 2024-08-17T16:55:40.515 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (2m) 0s ago 2m 86.7M - 9.4.7 954c08fa6188 fc8d62329012 2024-08-17T16:55:40.515 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (111s) 65s ago 111s 14.4M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T16:55:40.515 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (108s) 0s ago 108s 15.5M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T16:55:40.515 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,8765 running (6m) 0s ago 6m 439M - 18.2.4-719-gced649f8 c870753d67c2 4add2ba1a53e 2024-08-17T16:55:40.515 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:9283,8765 running (8m) 65s ago 8m 484M - 18.2.4-719-gced649f8 c870753d67c2 ba4a2894b308 2024-08-17T16:55:40.515 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (8m) 65s ago 8m 46.2M 2048M 18.2.4-719-gced649f8 c870753d67c2 0c71ddd7ada6 2024-08-17T16:55:40.515 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (7m) 0s ago 7m 38.1M 2048M 18.2.4-719-gced649f8 c870753d67c2 d4ecbe65a0c8 2024-08-17T16:55:40.515 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (7m) 65s ago 7m 38.6M 2048M 18.2.4-719-gced649f8 c870753d67c2 b3f2c9339c6b 2024-08-17T16:55:40.516 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 running (2m) 65s ago 2m 15.7M - 1.5.0 0da6a335fe13 8958df80ad24 2024-08-17T16:55:40.516 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (2m) 0s ago 2m 13.1M - 1.5.0 0da6a335fe13 b1691104d8ac 2024-08-17T16:55:40.516 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (6m) 65s ago 6m 44.7M 4016M 18.2.4-719-gced649f8 c870753d67c2 f3b910ed49b8 2024-08-17T16:55:40.516 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (6m) 65s ago 6m 43.3M 4016M 18.2.4-719-gced649f8 c870753d67c2 5d6fdb6000ad 2024-08-17T16:55:40.516 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (5m) 65s ago 5m 48.7M 4016M 18.2.4-719-gced649f8 c870753d67c2 a3cbe8edb49a 2024-08-17T16:55:40.516 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (5m) 65s ago 5m 51.0M 4016M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T16:55:40.516 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (5m) 0s ago 5m 65.5M 4016M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T16:55:40.516 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (4m) 0s ago 4m 61.6M 4016M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T16:55:40.516 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (4m) 0s ago 4m 50.0M 4016M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T16:55:40.516 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (3m) 0s ago 3m 57.9M 4016M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T16:55:40.516 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (2m) 0s ago 3m 44.9M - 2.43.0 a07b618ecd1d 015db799d7d4 2024-08-17T16:55:40.823 INFO:tasks.rados.rados.0.smithi063.stdout:239: done (2 left) 2024-08-17T16:55:40.824 INFO:tasks.rados.rados.0.smithi063.stdout:240: done (1 left) 2024-08-17T16:55:40.824 INFO:tasks.rados.rados.0.smithi063.stdout:241: done (0 left) 2024-08-17T16:55:40.824 INFO:tasks.rados.rados.0.smithi063.stdout:242: rollback oid 28 current snap is 25 2024-08-17T16:55:40.824 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 28 to 24 2024-08-17T16:55:40.824 INFO:tasks.rados.rados.0.smithi063.stdout:243: delete oid 29 current snap is 25 2024-08-17T16:55:40.826 INFO:tasks.rados.rados.0.smithi063.stdout:242: finishing rollback tid 0 to smithi06387500-28 2024-08-17T16:55:40.826 INFO:tasks.rados.rados.0.smithi063.stdout:243: done (1 left) 2024-08-17T16:55:40.826 INFO:tasks.rados.rados.0.smithi063.stdout:244: rollback oid 26 current snap is 25 2024-08-17T16:55:40.826 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 26 to 24 2024-08-17T16:55:40.826 INFO:tasks.rados.rados.0.smithi063.stdout:245: rollback oid 16 current snap is 25 2024-08-17T16:55:40.827 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 16 to 20 2024-08-17T16:55:40.827 INFO:tasks.rados.rados.0.smithi063.stdout:246: snap_create 2024-08-17T16:55:40.827 INFO:tasks.rados.rados.0.smithi063.stdout:242: finishing rollback tid 1 to smithi06387500-28 2024-08-17T16:55:40.827 INFO:tasks.rados.rados.0.smithi063.stdout:242: finishing rollback tid 2 to smithi06387500-28 2024-08-17T16:55:40.827 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 78 (ObjNum 122 snap 23 seq_num 122) dirty exists 2024-08-17T16:55:40.829 INFO:tasks.rados.rados.0.smithi063.stdout:244: finishing rollback tid 0 to smithi06387500-26 2024-08-17T16:55:40.829 INFO:tasks.rados.rados.0.smithi063.stdout:244: finishing rollback tid 1 to smithi06387500-26 2024-08-17T16:55:40.830 INFO:tasks.rados.rados.0.smithi063.stdout:244: finishing rollback tid 2 to smithi06387500-26 2024-08-17T16:55:40.830 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 53 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-08-17T16:55:40.830 INFO:tasks.rados.rados.0.smithi063.stdout:245: finishing rollback tid 0 to smithi06387500-16 2024-08-17T16:55:40.830 INFO:tasks.rados.rados.0.smithi063.stdout:245: finishing rollback tid 1 to smithi06387500-16 2024-08-17T16:55:40.830 INFO:tasks.rados.rados.0.smithi063.stdout:245: finishing rollback tid 2 to smithi06387500-16 2024-08-17T16:55:40.830 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 102 (ObjNum 117 snap 20 seq_num 117) dirty exists 2024-08-17T16:55:40.852 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:55:40.852 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T16:55:40.852 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T16:55:40.852 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:55:40.852 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T16:55:40.852 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T16:55:40.852 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:55:40.853 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T16:55:40.853 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T16:55:40.853 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:55:40.853 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T16:55:40.853 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T16:55:40.853 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:55:40.853 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T16:55:40.853 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 15 2024-08-17T16:55:40.853 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T16:55:40.853 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:55:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:40 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:40 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:40 smithi063 ceph-mon[42680]: from='client.24722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:40 smithi063 ceph-mon[42680]: from='client.14835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:40 smithi063 ceph-mon[42680]: pgmap v166: 89 pgs: 89 active+clean; 124 MiB data, 452 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:40 smithi063 ceph-mon[42680]: osdmap e124: 8 total, 8 up, 8 in 2024-08-17T16:55:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:40 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/1773994332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:55:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:40 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:40 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:40 smithi063 ceph-mon[31063]: from='client.24722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:40 smithi063 ceph-mon[31063]: from='client.14835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:40 smithi063 ceph-mon[31063]: pgmap v166: 89 pgs: 89 active+clean; 124 MiB data, 452 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:40 smithi063 ceph-mon[31063]: osdmap e124: 8 total, 8 up, 8 in 2024-08-17T16:55:41.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:40 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/1773994332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:55:41.149 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:55:41.149 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", 2024-08-17T16:55:41.149 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-08-17T16:55:41.149 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-17T16:55:41.149 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-08-17T16:55:41.149 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "", 2024-08-17T16:55:41.149 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "", 2024-08-17T16:55:41.149 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T16:55:41.150 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:55:41.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:40 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:41.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:40 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:41.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:40 smithi162 ceph-mon[34633]: from='client.24722 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:41.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:40 smithi162 ceph-mon[34633]: from='client.14835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:41.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:40 smithi162 ceph-mon[34633]: pgmap v166: 89 pgs: 89 active+clean; 124 MiB data, 452 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:41.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:40 smithi162 ceph-mon[34633]: osdmap e124: 8 total, 8 up, 8 in 2024-08-17T16:55:41.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:40 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/1773994332' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:55:41.824 INFO:tasks.rados.rados.0.smithi063.stdout:242: done (3 left) 2024-08-17T16:55:41.825 INFO:tasks.rados.rados.0.smithi063.stdout:244: done (2 left) 2024-08-17T16:55:41.825 INFO:tasks.rados.rados.0.smithi063.stdout:245: done (1 left) 2024-08-17T16:55:41.825 INFO:tasks.rados.rados.0.smithi063.stdout:246: done (0 left) 2024-08-17T16:55:41.825 INFO:tasks.rados.rados.0.smithi063.stdout:247: snap_create 2024-08-17T16:55:42.115 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[42680]: from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:42.115 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:42.115 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:42.115 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[42680]: from='client.14853 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:42.115 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:42.115 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:42.115 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:42.116 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[42680]: osdmap e125: 8 total, 8 up, 8 in 2024-08-17T16:55:42.116 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[31063]: from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:42.116 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:42.116 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:42.116 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[31063]: from='client.14853 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:42.116 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:42.116 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:42.116 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:42.116 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:42 smithi063 ceph-mon[31063]: osdmap e125: 8 total, 8 up, 8 in 2024-08-17T16:55:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:42 smithi162 ceph-mon[34633]: from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:42 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:42 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:42 smithi162 ceph-mon[34633]: from='client.14853 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:55:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:42 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:55:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:42 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:55:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:42 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:42 smithi162 ceph-mon[34633]: osdmap e125: 8 total, 8 up, 8 in 2024-08-17T16:55:42.826 INFO:tasks.rados.rados.0.smithi063.stdout:247: done (0 left) 2024-08-17T16:55:42.826 INFO:tasks.rados.rados.0.smithi063.stdout:248: write oid 48 current snap is 27 2024-08-17T16:55:42.826 INFO:tasks.rados.rados.0.smithi063.stdout:248: seq_num 128 ranges {551296=693520,2024048=127248} 2024-08-17T16:55:42.833 INFO:tasks.rados.rados.0.smithi063.stdout:248: writing smithi06387500-48 from 551296 to 1244816 tid 1 2024-08-17T16:55:42.838 INFO:tasks.rados.rados.0.smithi063.stdout:248: writing smithi06387500-48 from 2024048 to 2151296 tid 2 2024-08-17T16:55:42.838 INFO:tasks.rados.rados.0.smithi063.stdout:249: write oid 44 current snap is 27 2024-08-17T16:55:42.838 INFO:tasks.rados.rados.0.smithi063.stdout:249: seq_num 129 ranges {568103=768769,2082530=85573} 2024-08-17T16:55:42.845 INFO:tasks.rados.rados.0.smithi063.stdout:249: writing smithi06387500-44 from 568103 to 1336872 tid 1 2024-08-17T16:55:42.849 INFO:tasks.rados.rados.0.smithi063.stdout:249: writing smithi06387500-44 from 2082530 to 2168103 tid 2 2024-08-17T16:55:42.849 INFO:tasks.rados.rados.0.smithi063.stdout:250: snap_remove snap 26 2024-08-17T16:55:43.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:43 smithi063 ceph-mon[42680]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:55:43.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:43 smithi063 ceph-mon[42680]: pgmap v169: 89 pgs: 89 active+clean; 133 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-08-17T16:55:43.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:43 smithi063 ceph-mon[42680]: osdmap e126: 8 total, 8 up, 8 in 2024-08-17T16:55:43.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:43 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:43.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:43 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:43.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:43 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:43.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:43 smithi063 ceph-mon[31063]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:55:43.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:43 smithi063 ceph-mon[31063]: pgmap v169: 89 pgs: 89 active+clean; 133 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-08-17T16:55:43.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:43 smithi063 ceph-mon[31063]: osdmap e126: 8 total, 8 up, 8 in 2024-08-17T16:55:43.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:43 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:43.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:43 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:43.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:43 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:43.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:43 smithi162 ceph-mon[34633]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:55:43.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:43 smithi162 ceph-mon[34633]: pgmap v169: 89 pgs: 89 active+clean; 133 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-08-17T16:55:43.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:43 smithi162 ceph-mon[34633]: osdmap e126: 8 total, 8 up, 8 in 2024-08-17T16:55:43.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:43 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:43.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:43 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:55:43.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:43 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:55:43.829 INFO:tasks.rados.rados.0.smithi063.stdout:248: finishing write tid 1 to smithi06387500-48 2024-08-17T16:55:43.829 INFO:tasks.rados.rados.0.smithi063.stdout:248: finishing write tid 2 to smithi06387500-48 2024-08-17T16:55:43.829 INFO:tasks.rados.rados.0.smithi063.stdout:248: finishing write tid 3 to smithi06387500-48 2024-08-17T16:55:43.829 INFO:tasks.rados.rados.0.smithi063.stdout:248: finishing write tid 4 to smithi06387500-48 2024-08-17T16:55:43.829 INFO:tasks.rados.rados.0.smithi063.stdout:248: oid 48 updating version 0 to 209 2024-08-17T16:55:43.829 INFO:tasks.rados.rados.0.smithi063.stdout:248: oid 48 updating version 209 to 210 2024-08-17T16:55:43.829 INFO:tasks.rados.rados.0.smithi063.stdout:248: oid 48 version 210 is already newer than 208 2024-08-17T16:55:43.829 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 210 (ObjNum 128 snap 27 seq_num 128) dirty exists 2024-08-17T16:55:43.830 INFO:tasks.rados.rados.0.smithi063.stdout:248: left oid 48 (ObjNum 128 snap 27 seq_num 128) 2024-08-17T16:55:43.830 INFO:tasks.rados.rados.0.smithi063.stdout:249: finishing write tid 1 to smithi06387500-44 2024-08-17T16:55:43.830 INFO:tasks.rados.rados.0.smithi063.stdout:249: finishing write tid 2 to smithi06387500-44 2024-08-17T16:55:43.830 INFO:tasks.rados.rados.0.smithi063.stdout:249: finishing write tid 3 to smithi06387500-44 2024-08-17T16:55:43.830 INFO:tasks.rados.rados.0.smithi063.stdout:249: finishing write tid 4 to smithi06387500-44 2024-08-17T16:55:43.830 INFO:tasks.rados.rados.0.smithi063.stdout:249: oid 44 updating version 0 to 32 2024-08-17T16:55:43.830 INFO:tasks.rados.rados.0.smithi063.stdout:249: oid 44 version 32 is already newer than 31 2024-08-17T16:55:43.830 INFO:tasks.rados.rados.0.smithi063.stdout:249: oid 44 version 32 is already newer than 30 2024-08-17T16:55:43.830 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 32 (ObjNum 129 snap 27 seq_num 129) dirty exists 2024-08-17T16:55:43.830 INFO:tasks.rados.rados.0.smithi063.stdout:249: left oid 44 (ObjNum 129 snap 27 seq_num 129) 2024-08-17T16:55:43.830 INFO:tasks.rados.rados.0.smithi063.stdout:248: done (2 left) 2024-08-17T16:55:43.830 INFO:tasks.rados.rados.0.smithi063.stdout:249: done (1 left) 2024-08-17T16:55:43.831 INFO:tasks.rados.rados.0.smithi063.stdout:250: done (0 left) 2024-08-17T16:55:43.831 INFO:tasks.rados.rados.0.smithi063.stdout:251: write oid 31 current snap is 27 2024-08-17T16:55:43.831 INFO:tasks.rados.rados.0.smithi063.stdout:251: seq_num 130 ranges {584910=560371,1857365=327545} 2024-08-17T16:55:43.835 INFO:tasks.rados.rados.0.smithi063.stdout:251: writing smithi06387500-31 from 584910 to 1145281 tid 1 2024-08-17T16:55:43.841 INFO:tasks.rados.rados.0.smithi063.stdout:251: writing smithi06387500-31 from 1857365 to 2184910 tid 2 2024-08-17T16:55:43.841 INFO:tasks.rados.rados.0.smithi063.stdout:252: snap_create 2024-08-17T16:55:43.841 INFO:tasks.rados.rados.0.smithi063.stdout:251: finishing write tid 1 to smithi06387500-31 2024-08-17T16:55:43.843 INFO:tasks.rados.rados.0.smithi063.stdout:251: finishing write tid 2 to smithi06387500-31 2024-08-17T16:55:43.843 INFO:tasks.rados.rados.0.smithi063.stdout:251: finishing write tid 3 to smithi06387500-31 2024-08-17T16:55:43.843 INFO:tasks.rados.rados.0.smithi063.stdout:251: finishing write tid 4 to smithi06387500-31 2024-08-17T16:55:43.843 INFO:tasks.rados.rados.0.smithi063.stdout:251: oid 31 updating version 0 to 129 2024-08-17T16:55:43.843 INFO:tasks.rados.rados.0.smithi063.stdout:251: oid 31 version 129 is already newer than 128 2024-08-17T16:55:43.844 INFO:tasks.rados.rados.0.smithi063.stdout:251: oid 31 updating version 129 to 130 2024-08-17T16:55:43.844 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 130 (ObjNum 130 snap 27 seq_num 130) dirty exists 2024-08-17T16:55:43.844 INFO:tasks.rados.rados.0.smithi063.stdout:251: left oid 31 (ObjNum 130 snap 27 seq_num 130) 2024-08-17T16:55:44.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:44 smithi162 ceph-mon[34633]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (unknown) 2024-08-17T16:55:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:44 smithi162 ceph-mon[34633]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T16:55:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:44 smithi162 ceph-mon[34633]: Upgrade: Need to upgrade myself (mgr.y) 2024-08-17T16:55:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:44 smithi162 ceph-mon[34633]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d on smithi162 2024-08-17T16:55:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:44 smithi162 ceph-mon[34633]: osdmap e127: 8 total, 8 up, 8 in 2024-08-17T16:55:44.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:44 smithi063 ceph-mon[31063]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (unknown) 2024-08-17T16:55:44.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:44 smithi063 ceph-mon[31063]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T16:55:44.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:44 smithi063 ceph-mon[31063]: Upgrade: Need to upgrade myself (mgr.y) 2024-08-17T16:55:44.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:44 smithi063 ceph-mon[31063]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d on smithi162 2024-08-17T16:55:44.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:44 smithi063 ceph-mon[31063]: osdmap e127: 8 total, 8 up, 8 in 2024-08-17T16:55:44.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:44 smithi063 ceph-mon[42680]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (unknown) 2024-08-17T16:55:44.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:44 smithi063 ceph-mon[42680]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T16:55:44.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:44 smithi063 ceph-mon[42680]: Upgrade: Need to upgrade myself (mgr.y) 2024-08-17T16:55:44.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:44 smithi063 ceph-mon[42680]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d on smithi162 2024-08-17T16:55:44.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:44 smithi063 ceph-mon[42680]: osdmap e127: 8 total, 8 up, 8 in 2024-08-17T16:55:44.824 INFO:tasks.rados.rados.0.smithi063.stdout:251: done (1 left) 2024-08-17T16:55:44.824 INFO:tasks.rados.rados.0.smithi063.stdout:252: done (0 left) 2024-08-17T16:55:44.824 INFO:tasks.rados.rados.0.smithi063.stdout:253: snap_remove snap 27 2024-08-17T16:55:45.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:45 smithi162 ceph-mon[34633]: pgmap v172: 89 pgs: 89 active+clean; 133 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-08-17T16:55:45.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:45 smithi162 ceph-mon[34633]: osdmap e128: 8 total, 8 up, 8 in 2024-08-17T16:55:45.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:45 smithi063 ceph-mon[31063]: pgmap v172: 89 pgs: 89 active+clean; 133 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-08-17T16:55:45.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:45 smithi063 ceph-mon[31063]: osdmap e128: 8 total, 8 up, 8 in 2024-08-17T16:55:45.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:45 smithi063 ceph-mon[42680]: pgmap v172: 89 pgs: 89 active+clean; 133 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.3 MiB/s wr, 17 op/s 2024-08-17T16:55:45.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:45 smithi063 ceph-mon[42680]: osdmap e128: 8 total, 8 up, 8 in 2024-08-17T16:55:45.826 INFO:tasks.rados.rados.0.smithi063.stdout:253: done (0 left) 2024-08-17T16:55:45.826 INFO:tasks.rados.rados.0.smithi063.stdout:254: read oid 29 snap -1 2024-08-17T16:55:45.826 INFO:tasks.rados.rados.0.smithi063.stdout:254: expect deleted 2024-08-17T16:55:45.826 INFO:tasks.rados.rados.0.smithi063.stdout:255: read oid 15 snap -1 2024-08-17T16:55:45.826 INFO:tasks.rados.rados.0.smithi063.stdout:255: expect (ObjNum 14 snap 0 seq_num 14) 2024-08-17T16:55:45.826 INFO:tasks.rados.rados.0.smithi063.stdout:256: delete oid 36 current snap is 28 2024-08-17T16:55:45.828 INFO:tasks.rados.rados.0.smithi063.stdout:256: done (2 left) 2024-08-17T16:55:45.828 INFO:tasks.rados.rados.0.smithi063.stdout:257: delete oid 11 current snap is 28 2024-08-17T16:55:45.831 INFO:tasks.rados.rados.0.smithi063.stdout:255: expect (ObjNum 14 snap 0 seq_num 14) 2024-08-17T16:55:45.908 INFO:tasks.rados.rados.0.smithi063.stdout:254: done (2 left) 2024-08-17T16:55:45.908 INFO:tasks.rados.rados.0.smithi063.stdout:255: done (1 left) 2024-08-17T16:55:45.908 INFO:tasks.rados.rados.0.smithi063.stdout:257: done (0 left) 2024-08-17T16:55:45.908 INFO:tasks.rados.rados.0.smithi063.stdout:258: read oid 29 snap -1 2024-08-17T16:55:45.908 INFO:tasks.rados.rados.0.smithi063.stdout:258: expect deleted 2024-08-17T16:55:45.908 INFO:tasks.rados.rados.0.smithi063.stdout:259: write oid 19 current snap is 28 2024-08-17T16:55:45.908 INFO:tasks.rados.rados.0.smithi063.stdout:259: seq_num 133 ranges {635331=786118,1916458=318873} 2024-08-17T16:55:45.915 INFO:tasks.rados.rados.0.smithi063.stdout:259: writing smithi06387500-19 from 635331 to 1421449 tid 1 2024-08-17T16:55:45.920 INFO:tasks.rados.rados.0.smithi063.stdout:259: writing smithi06387500-19 from 1916458 to 2235331 tid 2 2024-08-17T16:55:45.920 INFO:tasks.rados.rados.0.smithi063.stdout:260: snap_remove snap 19 2024-08-17T16:55:46.833 INFO:tasks.rados.rados.0.smithi063.stdout:260: done (2 left) 2024-08-17T16:55:46.833 INFO:tasks.rados.rados.0.smithi063.stdout:261: write oid 36 current snap is 28 2024-08-17T16:55:46.833 INFO:tasks.rados.rados.0.smithi063.stdout:261: seq_num 134 ranges {652138=461367,1574940=677198} 2024-08-17T16:55:46.840 INFO:tasks.rados.rados.0.smithi063.stdout:261: writing smithi06387500-36 from 652138 to 1113505 tid 1 2024-08-17T16:55:46.847 INFO:tasks.rados.rados.0.smithi063.stdout:261: writing smithi06387500-36 from 1574940 to 2252138 tid 2 2024-08-17T16:55:46.847 INFO:tasks.rados.rados.0.smithi063.stdout:262: read oid 2 snap -1 2024-08-17T16:55:46.847 INFO:tasks.rados.rados.0.smithi063.stdout:262: expect deleted 2024-08-17T16:55:46.847 INFO:tasks.rados.rados.0.smithi063.stdout:263: write oid 26 current snap is 28 2024-08-17T16:55:46.847 INFO:tasks.rados.rados.0.smithi063.stdout:259: finishing write tid 1 to smithi06387500-19 2024-08-17T16:55:46.847 INFO:tasks.rados.rados.0.smithi063.stdout:259: finishing write tid 2 to smithi06387500-19 2024-08-17T16:55:46.847 INFO:tasks.rados.rados.0.smithi063.stdout:259: finishing write tid 3 to smithi06387500-19 2024-08-17T16:55:46.847 INFO:tasks.rados.rados.0.smithi063.stdout:259: finishing write tid 4 to smithi06387500-19 2024-08-17T16:55:46.847 INFO:tasks.rados.rados.0.smithi063.stdout:259: oid 19 updating version 0 to 132 2024-08-17T16:55:46.848 INFO:tasks.rados.rados.0.smithi063.stdout:259: oid 19 updating version 132 to 134 2024-08-17T16:55:46.848 INFO:tasks.rados.rados.0.smithi063.stdout:259: oid 19 version 134 is already newer than 133 2024-08-17T16:55:46.848 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 134 (ObjNum 133 snap 28 seq_num 133) dirty exists 2024-08-17T16:55:46.848 INFO:tasks.rados.rados.0.smithi063.stdout:259: left oid 19 (ObjNum 133 snap 28 seq_num 133) 2024-08-17T16:55:46.848 INFO:tasks.rados.rados.0.smithi063.stdout:263: seq_num 135 ranges {668945=536616,1633422=491363} 2024-08-17T16:55:46.853 INFO:tasks.rados.rados.0.smithi063.stdout:263: writing smithi06387500-26 from 668945 to 1205561 tid 1 2024-08-17T16:55:46.858 INFO:tasks.rados.rados.0.smithi063.stdout:263: writing smithi06387500-26 from 1633422 to 2124785 tid 2 2024-08-17T16:55:46.858 INFO:tasks.rados.rados.0.smithi063.stdout:258: done (4 left) 2024-08-17T16:55:46.858 INFO:tasks.rados.rados.0.smithi063.stdout:259: done (3 left) 2024-08-17T16:55:46.859 INFO:tasks.rados.rados.0.smithi063.stdout:264: read oid 14 snap -1 2024-08-17T16:55:46.859 INFO:tasks.rados.rados.0.smithi063.stdout:264: expect (ObjNum 75 snap 9 seq_num 75) 2024-08-17T16:55:46.859 INFO:tasks.rados.rados.0.smithi063.stdout:265: read oid 21 snap -1 2024-08-17T16:55:46.859 INFO:tasks.rados.rados.0.smithi063.stdout:265: expect deleted 2024-08-17T16:55:46.859 INFO:tasks.rados.rados.0.smithi063.stdout:266: write oid 19 current snap is 28 2024-08-17T16:55:46.859 INFO:tasks.rados.rados.0.smithi063.stdout:266: seq_num 136 ranges {685752=611865,1975551=310201} 2024-08-17T16:55:46.866 INFO:tasks.rados.rados.0.smithi063.stdout:266: writing smithi06387500-19 from 685752 to 1297617 tid 1 2024-08-17T16:55:46.872 INFO:tasks.rados.rados.0.smithi063.stdout:266: writing smithi06387500-19 from 1975551 to 2285752 tid 2 2024-08-17T16:55:46.873 INFO:tasks.rados.rados.0.smithi063.stdout:267: rollback oid 18 current snap is 28 2024-08-17T16:55:46.873 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 18 to 24 2024-08-17T16:55:46.873 INFO:tasks.rados.rados.0.smithi063.stdout:261: finishing write tid 1 to smithi06387500-36 2024-08-17T16:55:46.873 INFO:tasks.rados.rados.0.smithi063.stdout:261: finishing write tid 2 to smithi06387500-36 2024-08-17T16:55:46.873 INFO:tasks.rados.rados.0.smithi063.stdout:261: finishing write tid 3 to smithi06387500-36 2024-08-17T16:55:46.873 INFO:tasks.rados.rados.0.smithi063.stdout:261: finishing write tid 4 to smithi06387500-36 2024-08-17T16:55:46.873 INFO:tasks.rados.rados.0.smithi063.stdout:261: oid 36 updating version 0 to 219 2024-08-17T16:55:46.873 INFO:tasks.rados.rados.0.smithi063.stdout:261: oid 36 updating version 219 to 220 2024-08-17T16:55:46.873 INFO:tasks.rados.rados.0.smithi063.stdout:261: oid 36 updating version 220 to 221 2024-08-17T16:55:46.873 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 221 (ObjNum 134 snap 28 seq_num 134) dirty exists 2024-08-17T16:55:46.874 INFO:tasks.rados.rados.0.smithi063.stdout:261: left oid 36 (ObjNum 134 snap 28 seq_num 134) 2024-08-17T16:55:46.874 INFO:tasks.rados.rados.0.smithi063.stdout:263: finishing write tid 1 to smithi06387500-26 2024-08-17T16:55:46.874 INFO:tasks.rados.rados.0.smithi063.stdout:261: done (6 left) 2024-08-17T16:55:46.874 INFO:tasks.rados.rados.0.smithi063.stdout:262: done (5 left) 2024-08-17T16:55:46.874 INFO:tasks.rados.rados.0.smithi063.stdout:268: write oid 1 current snap is 28 2024-08-17T16:55:46.874 INFO:tasks.rados.rados.0.smithi063.stdout:268: seq_num 137 ranges {702559=403467,1750386=495032} 2024-08-17T16:55:46.878 INFO:tasks.rados.rados.0.smithi063.stdout:268: writing smithi06387500-1 from 702559 to 1106026 tid 1 2024-08-17T16:55:46.885 INFO:tasks.rados.rados.0.smithi063.stdout:268: writing smithi06387500-1 from 1750386 to 2245418 tid 2 2024-08-17T16:55:46.885 INFO:tasks.rados.rados.0.smithi063.stdout:265: done (5 left) 2024-08-17T16:55:46.885 INFO:tasks.rados.rados.0.smithi063.stdout:269: read oid 12 snap -1 2024-08-17T16:55:46.885 INFO:tasks.rados.rados.0.smithi063.stdout:269: expect (ObjNum 68 snap 7 seq_num 68) 2024-08-17T16:55:46.885 INFO:tasks.rados.rados.0.smithi063.stdout:270: rollback oid 5 current snap is 28 2024-08-17T16:55:46.885 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 24 2024-08-17T16:55:46.886 INFO:tasks.rados.rados.0.smithi063.stdout:263: finishing write tid 2 to smithi06387500-26 2024-08-17T16:55:46.886 INFO:tasks.rados.rados.0.smithi063.stdout:263: finishing write tid 3 to smithi06387500-26 2024-08-17T16:55:46.886 INFO:tasks.rados.rados.0.smithi063.stdout:263: finishing write tid 4 to smithi06387500-26 2024-08-17T16:55:46.886 INFO:tasks.rados.rados.0.smithi063.stdout:263: oid 26 updating version 0 to 59 2024-08-17T16:55:46.886 INFO:tasks.rados.rados.0.smithi063.stdout:263: oid 26 version 59 is already newer than 57 2024-08-17T16:55:46.886 INFO:tasks.rados.rados.0.smithi063.stdout:263: oid 26 version 59 is already newer than 58 2024-08-17T16:55:46.886 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 59 (ObjNum 135 snap 28 seq_num 135) dirty exists 2024-08-17T16:55:46.886 INFO:tasks.rados.rados.0.smithi063.stdout:263: left oid 26 (ObjNum 135 snap 28 seq_num 135) 2024-08-17T16:55:46.886 INFO:tasks.rados.rados.0.smithi063.stdout:266: finishing write tid 1 to smithi06387500-19 2024-08-17T16:55:46.886 INFO:tasks.rados.rados.0.smithi063.stdout:264: expect (ObjNum 75 snap 9 seq_num 75) 2024-08-17T16:55:47.029 INFO:tasks.rados.rados.0.smithi063.stdout:266: finishing write tid 2 to smithi06387500-19 2024-08-17T16:55:47.029 INFO:tasks.rados.rados.0.smithi063.stdout:266: finishing write tid 3 to smithi06387500-19 2024-08-17T16:55:47.029 INFO:tasks.rados.rados.0.smithi063.stdout:267: finishing rollback tid 0 to smithi06387500-18 2024-08-17T16:55:47.030 INFO:tasks.rados.rados.0.smithi063.stdout:267: finishing rollback tid 1 to smithi06387500-18 2024-08-17T16:55:47.030 INFO:tasks.rados.rados.0.smithi063.stdout:266: finishing write tid 4 to smithi06387500-19 2024-08-17T16:55:47.030 INFO:tasks.rados.rados.0.smithi063.stdout:266: oid 19 updating version 0 to 140 2024-08-17T16:55:47.030 INFO:tasks.rados.rados.0.smithi063.stdout:266: oid 19 version 140 is already newer than 139 2024-08-17T16:55:47.030 INFO:tasks.rados.rados.0.smithi063.stdout:266: oid 19 updating version 140 to 141 2024-08-17T16:55:47.030 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 141 (ObjNum 136 snap 28 seq_num 136) dirty exists 2024-08-17T16:55:47.030 INFO:tasks.rados.rados.0.smithi063.stdout:266: left oid 19 (ObjNum 136 snap 28 seq_num 136) 2024-08-17T16:55:47.030 INFO:tasks.rados.rados.0.smithi063.stdout:267: finishing rollback tid 2 to smithi06387500-18 2024-08-17T16:55:47.030 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 62 (ObjNum 118 snap 20 seq_num 118) dirty exists 2024-08-17T16:55:47.030 INFO:tasks.rados.rados.0.smithi063.stdout:268: finishing write tid 1 to smithi06387500-1 2024-08-17T16:55:47.030 INFO:tasks.rados.rados.0.smithi063.stdout:263: done (6 left) 2024-08-17T16:55:47.030 INFO:tasks.rados.rados.0.smithi063.stdout:264: done (5 left) 2024-08-17T16:55:47.030 INFO:tasks.rados.rados.0.smithi063.stdout:266: done (4 left) 2024-08-17T16:55:47.031 INFO:tasks.rados.rados.0.smithi063.stdout:267: done (3 left) 2024-08-17T16:55:47.031 INFO:tasks.rados.rados.0.smithi063.stdout:271: snap_create 2024-08-17T16:55:47.031 INFO:tasks.rados.rados.0.smithi063.stdout:268: finishing write tid 2 to smithi06387500-1 2024-08-17T16:55:47.031 INFO:tasks.rados.rados.0.smithi063.stdout:268: finishing write tid 3 to smithi06387500-1 2024-08-17T16:55:47.031 INFO:tasks.rados.rados.0.smithi063.stdout:270: finishing rollback tid 0 to smithi06387500-5 2024-08-17T16:55:47.031 INFO:tasks.rados.rados.0.smithi063.stdout:268: finishing write tid 4 to smithi06387500-1 2024-08-17T16:55:47.031 INFO:tasks.rados.rados.0.smithi063.stdout:268: oid 1 updating version 0 to 79 2024-08-17T16:55:47.031 INFO:tasks.rados.rados.0.smithi063.stdout:268: oid 1 updating version 79 to 81 2024-08-17T16:55:47.032 INFO:tasks.rados.rados.0.smithi063.stdout:268: oid 1 version 81 is already newer than 80 2024-08-17T16:55:47.032 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 81 (ObjNum 137 snap 28 seq_num 137) dirty exists 2024-08-17T16:55:47.032 INFO:tasks.rados.rados.0.smithi063.stdout:268: left oid 1 (ObjNum 137 snap 28 seq_num 137) 2024-08-17T16:55:47.032 INFO:tasks.rados.rados.0.smithi063.stdout:270: finishing rollback tid 1 to smithi06387500-5 2024-08-17T16:55:47.032 INFO:tasks.rados.rados.0.smithi063.stdout:270: finishing rollback tid 2 to smithi06387500-5 2024-08-17T16:55:47.032 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 85 (ObjNum 99 snap 16 seq_num 99) dirty exists 2024-08-17T16:55:47.032 INFO:tasks.rados.rados.0.smithi063.stdout:269: expect (ObjNum 68 snap 7 seq_num 68) 2024-08-17T16:55:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:46 smithi162 ceph-mon[34633]: osdmap e129: 8 total, 8 up, 8 in 2024-08-17T16:55:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:46 smithi162 ceph-mon[34633]: pgmap v175: 89 pgs: 89 active+clean; 136 MiB data, 476 MiB used, 715 GiB / 715 GiB avail; 687 KiB/s rd, 896 KiB/s wr, 7 op/s 2024-08-17T16:55:47.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:46 smithi063 ceph-mon[42680]: osdmap e129: 8 total, 8 up, 8 in 2024-08-17T16:55:47.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:46 smithi063 ceph-mon[42680]: pgmap v175: 89 pgs: 89 active+clean; 136 MiB data, 476 MiB used, 715 GiB / 715 GiB avail; 687 KiB/s rd, 896 KiB/s wr, 7 op/s 2024-08-17T16:55:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:46 smithi063 ceph-mon[31063]: osdmap e129: 8 total, 8 up, 8 in 2024-08-17T16:55:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:46 smithi063 ceph-mon[31063]: pgmap v175: 89 pgs: 89 active+clean; 136 MiB data, 476 MiB used, 715 GiB / 715 GiB avail; 687 KiB/s rd, 896 KiB/s wr, 7 op/s 2024-08-17T16:55:47.837 INFO:tasks.rados.rados.0.smithi063.stdout:268: done (3 left) 2024-08-17T16:55:47.838 INFO:tasks.rados.rados.0.smithi063.stdout:269: done (2 left) 2024-08-17T16:55:47.838 INFO:tasks.rados.rados.0.smithi063.stdout:270: done (1 left) 2024-08-17T16:55:47.838 INFO:tasks.rados.rados.0.smithi063.stdout:271: done (0 left) 2024-08-17T16:55:47.838 INFO:tasks.rados.rados.0.smithi063.stdout:272: snap_create 2024-08-17T16:55:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:47 smithi162 ceph-mon[34633]: osdmap e130: 8 total, 8 up, 8 in 2024-08-17T16:55:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:47 smithi063 ceph-mon[42680]: osdmap e130: 8 total, 8 up, 8 in 2024-08-17T16:55:48.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:47 smithi063 ceph-mon[31063]: osdmap e130: 8 total, 8 up, 8 in 2024-08-17T16:55:48.847 INFO:tasks.rados.rados.0.smithi063.stdout:272: done (0 left) 2024-08-17T16:55:48.847 INFO:tasks.rados.rados.0.smithi063.stdout:273: write oid 31 current snap is 30 2024-08-17T16:55:48.847 INFO:tasks.rados.rados.0.smithi063.stdout:273: seq_num 138 ranges {719366=478716,1808868=510498} 2024-08-17T16:55:48.854 INFO:tasks.rados.rados.0.smithi063.stdout:273: writing smithi06387500-31 from 719366 to 1198082 tid 1 2024-08-17T16:55:48.860 INFO:tasks.rados.rados.0.smithi063.stdout:273: writing smithi06387500-31 from 1808868 to 2319366 tid 2 2024-08-17T16:55:48.860 INFO:tasks.rados.rados.0.smithi063.stdout:274: delete oid 38 current snap is 30 2024-08-17T16:55:48.860 INFO:tasks.rados.rados.0.smithi063.stdout:273: finishing write tid 1 to smithi06387500-31 2024-08-17T16:55:48.863 INFO:tasks.rados.rados.0.smithi063.stdout:273: finishing write tid 2 to smithi06387500-31 2024-08-17T16:55:48.863 INFO:tasks.rados.rados.0.smithi063.stdout:273: finishing write tid 3 to smithi06387500-31 2024-08-17T16:55:48.864 INFO:tasks.rados.rados.0.smithi063.stdout:273: finishing write tid 4 to smithi06387500-31 2024-08-17T16:55:48.864 INFO:tasks.rados.rados.0.smithi063.stdout:273: oid 31 updating version 0 to 145 2024-08-17T16:55:48.864 INFO:tasks.rados.rados.0.smithi063.stdout:273: oid 31 version 145 is already newer than 144 2024-08-17T16:55:48.864 INFO:tasks.rados.rados.0.smithi063.stdout:273: oid 31 version 145 is already newer than 143 2024-08-17T16:55:48.864 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 145 (ObjNum 138 snap 30 seq_num 138) dirty exists 2024-08-17T16:55:48.864 INFO:tasks.rados.rados.0.smithi063.stdout:273: left oid 31 (ObjNum 138 snap 30 seq_num 138) 2024-08-17T16:55:48.866 INFO:tasks.rados.rados.0.smithi063.stdout:273: done (1 left) 2024-08-17T16:55:48.866 INFO:tasks.rados.rados.0.smithi063.stdout:274: done (0 left) 2024-08-17T16:55:48.867 INFO:tasks.rados.rados.0.smithi063.stdout:275: rollback oid 20 current snap is 30 2024-08-17T16:55:48.867 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 20 to 25 2024-08-17T16:55:48.867 INFO:tasks.rados.rados.0.smithi063.stdout:276: snap_create 2024-08-17T16:55:48.867 INFO:tasks.rados.rados.0.smithi063.stdout:275: finishing rollback tid 1 to smithi06387500-20 2024-08-17T16:55:48.867 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-08-17T16:55:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:48 smithi162 ceph-mon[34633]: osdmap e131: 8 total, 8 up, 8 in 2024-08-17T16:55:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:48 smithi162 ceph-mon[34633]: pgmap v178: 89 pgs: 89 active+clean; 136 MiB data, 476 MiB used, 715 GiB / 715 GiB avail; 687 KiB/s rd, 896 KiB/s wr, 7 op/s 2024-08-17T16:55:49.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:48 smithi063 ceph-mon[42680]: osdmap e131: 8 total, 8 up, 8 in 2024-08-17T16:55:49.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:48 smithi063 ceph-mon[42680]: pgmap v178: 89 pgs: 89 active+clean; 136 MiB data, 476 MiB used, 715 GiB / 715 GiB avail; 687 KiB/s rd, 896 KiB/s wr, 7 op/s 2024-08-17T16:55:49.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:48 smithi063 ceph-mon[31063]: osdmap e131: 8 total, 8 up, 8 in 2024-08-17T16:55:49.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:48 smithi063 ceph-mon[31063]: pgmap v178: 89 pgs: 89 active+clean; 136 MiB data, 476 MiB used, 715 GiB / 715 GiB avail; 687 KiB/s rd, 896 KiB/s wr, 7 op/s 2024-08-17T16:55:49.826 INFO:tasks.rados.rados.0.smithi063.stdout:275: done (1 left) 2024-08-17T16:55:49.826 INFO:tasks.rados.rados.0.smithi063.stdout:276: done (0 left) 2024-08-17T16:55:49.826 INFO:tasks.rados.rados.0.smithi063.stdout:277: write oid 19 current snap is 31 2024-08-17T16:55:49.826 INFO:tasks.rados.rados.0.smithi063.stdout:277: seq_num 140 ranges {752980=629214,1809479=543501} 2024-08-17T16:55:49.833 INFO:tasks.rados.rados.0.smithi063.stdout:277: writing smithi06387500-19 from 752980 to 1382194 tid 1 2024-08-17T16:55:49.840 INFO:tasks.rados.rados.0.smithi063.stdout:277: writing smithi06387500-19 from 1809479 to 2352980 tid 2 2024-08-17T16:55:49.840 INFO:tasks.rados.rados.0.smithi063.stdout:278: snap_remove snap 29 2024-08-17T16:55:49.840 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:55:49 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:55:49] "GET /metrics HTTP/1.1" 200 36137 "" "Prometheus/2.43.0" 2024-08-17T16:55:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:49 smithi162 ceph-mon[34633]: osdmap e132: 8 total, 8 up, 8 in 2024-08-17T16:55:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:49 smithi162 ceph-mon[34633]: osdmap e133: 8 total, 8 up, 8 in 2024-08-17T16:55:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:49 smithi063 ceph-mon[42680]: osdmap e132: 8 total, 8 up, 8 in 2024-08-17T16:55:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:49 smithi063 ceph-mon[42680]: osdmap e133: 8 total, 8 up, 8 in 2024-08-17T16:55:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:49 smithi063 ceph-mon[31063]: osdmap e132: 8 total, 8 up, 8 in 2024-08-17T16:55:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:49 smithi063 ceph-mon[31063]: osdmap e133: 8 total, 8 up, 8 in 2024-08-17T16:55:50.828 INFO:tasks.rados.rados.0.smithi063.stdout:278: done (1 left) 2024-08-17T16:55:50.828 INFO:tasks.rados.rados.0.smithi063.stdout:279: delete oid 33 current snap is 31 2024-08-17T16:55:50.828 INFO:tasks.rados.rados.0.smithi063.stdout:277: finishing write tid 1 to smithi06387500-19 2024-08-17T16:55:50.828 INFO:tasks.rados.rados.0.smithi063.stdout:277: finishing write tid 2 to smithi06387500-19 2024-08-17T16:55:50.828 INFO:tasks.rados.rados.0.smithi063.stdout:277: finishing write tid 3 to smithi06387500-19 2024-08-17T16:55:50.828 INFO:tasks.rados.rados.0.smithi063.stdout:277: finishing write tid 4 to smithi06387500-19 2024-08-17T16:55:50.828 INFO:tasks.rados.rados.0.smithi063.stdout:277: oid 19 updating version 0 to 147 2024-08-17T16:55:50.828 INFO:tasks.rados.rados.0.smithi063.stdout:277: oid 19 updating version 147 to 149 2024-08-17T16:55:50.828 INFO:tasks.rados.rados.0.smithi063.stdout:277: oid 19 version 149 is already newer than 148 2024-08-17T16:55:50.828 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 149 (ObjNum 140 snap 31 seq_num 140) dirty exists 2024-08-17T16:55:50.828 INFO:tasks.rados.rados.0.smithi063.stdout:277: left oid 19 (ObjNum 140 snap 31 seq_num 140) 2024-08-17T16:55:50.830 INFO:tasks.rados.rados.0.smithi063.stdout:277: done (1 left) 2024-08-17T16:55:50.830 INFO:tasks.rados.rados.0.smithi063.stdout:279: done (0 left) 2024-08-17T16:55:50.830 INFO:tasks.rados.rados.0.smithi063.stdout:280: snap_remove snap 24 2024-08-17T16:55:51.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:50 smithi063 ceph-mon[42680]: pgmap v181: 89 pgs: 89 active+clean; 136 MiB data, 476 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:51.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:50 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:55:51.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:50 smithi063 ceph-mon[42680]: osdmap e134: 8 total, 8 up, 8 in 2024-08-17T16:55:51.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:50 smithi063 ceph-mon[31063]: pgmap v181: 89 pgs: 89 active+clean; 136 MiB data, 476 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:51.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:50 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:55:51.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:50 smithi063 ceph-mon[31063]: osdmap e134: 8 total, 8 up, 8 in 2024-08-17T16:55:51.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:50 smithi162 ceph-mon[34633]: pgmap v181: 89 pgs: 89 active+clean; 136 MiB data, 476 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:55:51.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:50 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:55:51.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:50 smithi162 ceph-mon[34633]: osdmap e134: 8 total, 8 up, 8 in 2024-08-17T16:55:51.830 INFO:tasks.rados.rados.0.smithi063.stdout:280: done (0 left) 2024-08-17T16:55:51.830 INFO:tasks.rados.rados.0.smithi063.stdout:281: rollback oid 49 current snap is 31 2024-08-17T16:55:51.830 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 49 to 22 2024-08-17T16:55:51.830 INFO:tasks.rados.rados.0.smithi063.stdout:282: snap_remove snap 25 2024-08-17T16:55:52.840 INFO:tasks.rados.rados.0.smithi063.stdout:281: finishing rollback tid 0 to smithi06387500-49 2024-08-17T16:55:52.840 INFO:tasks.rados.rados.0.smithi063.stdout:281: finishing rollback tid 1 to smithi06387500-49 2024-08-17T16:55:52.840 INFO:tasks.rados.rados.0.smithi063.stdout:281: finishing rollback tid 2 to smithi06387500-49 2024-08-17T16:55:52.840 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 225 (ObjNum 113 snap 19 seq_num 113) dirty exists 2024-08-17T16:55:52.840 INFO:tasks.rados.rados.0.smithi063.stdout:281: done (1 left) 2024-08-17T16:55:52.840 INFO:tasks.rados.rados.0.smithi063.stdout:282: done (0 left) 2024-08-17T16:55:52.840 INFO:tasks.rados.rados.0.smithi063.stdout:283: read oid 19 snap -1 2024-08-17T16:55:52.840 INFO:tasks.rados.rados.0.smithi063.stdout:283: expect (ObjNum 140 snap 31 seq_num 140) 2024-08-17T16:55:52.840 INFO:tasks.rados.rados.0.smithi063.stdout:284: snap_create 2024-08-17T16:55:52.847 INFO:tasks.rados.rados.0.smithi063.stdout:283: expect (ObjNum 140 snap 31 seq_num 140) 2024-08-17T16:55:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:52 smithi162 ceph-mon[34633]: osdmap e135: 8 total, 8 up, 8 in 2024-08-17T16:55:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:52 smithi162 ceph-mon[34633]: pgmap v184: 89 pgs: 89 active+clean; 137 MiB data, 482 MiB used, 715 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-08-17T16:55:53.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:52 smithi063 ceph-mon[31063]: osdmap e135: 8 total, 8 up, 8 in 2024-08-17T16:55:53.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:52 smithi063 ceph-mon[31063]: pgmap v184: 89 pgs: 89 active+clean; 137 MiB data, 482 MiB used, 715 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-08-17T16:55:53.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:52 smithi063 ceph-mon[42680]: osdmap e135: 8 total, 8 up, 8 in 2024-08-17T16:55:53.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:52 smithi063 ceph-mon[42680]: pgmap v184: 89 pgs: 89 active+clean; 137 MiB data, 482 MiB used, 715 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-08-17T16:55:53.838 INFO:tasks.rados.rados.0.smithi063.stdout:283: done (1 left) 2024-08-17T16:55:53.838 INFO:tasks.rados.rados.0.smithi063.stdout:284: done (0 left) 2024-08-17T16:55:53.838 INFO:tasks.rados.rados.0.smithi063.stdout:285: snap_create 2024-08-17T16:55:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:53 smithi162 ceph-mon[34633]: osdmap e136: 8 total, 8 up, 8 in 2024-08-17T16:55:54.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:53 smithi063 ceph-mon[42680]: osdmap e136: 8 total, 8 up, 8 in 2024-08-17T16:55:54.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:53 smithi063 ceph-mon[31063]: osdmap e136: 8 total, 8 up, 8 in 2024-08-17T16:55:54.827 INFO:tasks.rados.rados.0.smithi063.stdout:285: done (0 left) 2024-08-17T16:55:54.828 INFO:tasks.rados.rados.0.smithi063.stdout:286: snap_create 2024-08-17T16:55:55.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:54 smithi063 ceph-mon[42680]: osdmap e137: 8 total, 8 up, 8 in 2024-08-17T16:55:55.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:54 smithi063 ceph-mon[42680]: pgmap v187: 89 pgs: 89 active+clean; 137 MiB data, 482 MiB used, 715 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-08-17T16:55:55.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:54 smithi063 ceph-mon[42680]: osdmap e138: 8 total, 8 up, 8 in 2024-08-17T16:55:55.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:54 smithi063 ceph-mon[31063]: osdmap e137: 8 total, 8 up, 8 in 2024-08-17T16:55:55.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:54 smithi063 ceph-mon[31063]: pgmap v187: 89 pgs: 89 active+clean; 137 MiB data, 482 MiB used, 715 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-08-17T16:55:55.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:54 smithi063 ceph-mon[31063]: osdmap e138: 8 total, 8 up, 8 in 2024-08-17T16:55:55.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:54 smithi162 ceph-mon[34633]: osdmap e137: 8 total, 8 up, 8 in 2024-08-17T16:55:55.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:54 smithi162 ceph-mon[34633]: pgmap v187: 89 pgs: 89 active+clean; 137 MiB data, 482 MiB used, 715 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-08-17T16:55:55.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:54 smithi162 ceph-mon[34633]: osdmap e138: 8 total, 8 up, 8 in 2024-08-17T16:55:55.828 INFO:tasks.rados.rados.0.smithi063.stdout:286: done (0 left) 2024-08-17T16:55:55.828 INFO:tasks.rados.rados.0.smithi063.stdout:287: write oid 13 current snap is 34 2024-08-17T16:55:55.828 INFO:tasks.rados.rados.0.smithi063.stdout:287: seq_num 142 ranges {786594=779712,2326443=60151} 2024-08-17T16:55:55.836 INFO:tasks.rados.rados.0.smithi063.stdout:287: writing smithi06387500-13 from 786594 to 1566306 tid 1 2024-08-17T16:55:55.841 INFO:tasks.rados.rados.0.smithi063.stdout:287: writing smithi06387500-13 from 2326443 to 2386594 tid 2 2024-08-17T16:55:55.841 INFO:tasks.rados.rados.0.smithi063.stdout:288: write oid 29 current snap is 34 2024-08-17T16:55:55.841 INFO:tasks.rados.rados.0.smithi063.stdout:288: seq_num 143 ranges {403401=454961,1584925=506039} 2024-08-17T16:55:55.846 INFO:tasks.rados.rados.0.smithi063.stdout:288: writing smithi06387500-29 from 403401 to 858362 tid 1 2024-08-17T16:55:55.853 INFO:tasks.rados.rados.0.smithi063.stdout:288: writing smithi06387500-29 from 1584925 to 2090964 tid 2 2024-08-17T16:55:55.854 INFO:tasks.rados.rados.0.smithi063.stdout:289: read oid 30 snap -1 2024-08-17T16:55:55.854 INFO:tasks.rados.rados.0.smithi063.stdout:289: expect (ObjNum 29 snap 0 seq_num 29) 2024-08-17T16:55:55.854 INFO:tasks.rados.rados.0.smithi063.stdout:290: delete oid 22 current snap is 34 2024-08-17T16:55:55.854 INFO:tasks.rados.rados.0.smithi063.stdout:287: finishing write tid 1 to smithi06387500-13 2024-08-17T16:55:55.854 INFO:tasks.rados.rados.0.smithi063.stdout:287: finishing write tid 2 to smithi06387500-13 2024-08-17T16:55:55.854 INFO:tasks.rados.rados.0.smithi063.stdout:287: finishing write tid 3 to smithi06387500-13 2024-08-17T16:55:55.854 INFO:tasks.rados.rados.0.smithi063.stdout:287: finishing write tid 4 to smithi06387500-13 2024-08-17T16:55:55.854 INFO:tasks.rados.rados.0.smithi063.stdout:287: oid 13 updating version 0 to 92 2024-08-17T16:55:55.854 INFO:tasks.rados.rados.0.smithi063.stdout:287: oid 13 version 92 is already newer than 91 2024-08-17T16:55:55.854 INFO:tasks.rados.rados.0.smithi063.stdout:287: oid 13 updating version 92 to 93 2024-08-17T16:55:55.854 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 93 (ObjNum 142 snap 34 seq_num 142) dirty exists 2024-08-17T16:55:55.854 INFO:tasks.rados.rados.0.smithi063.stdout:287: left oid 13 (ObjNum 142 snap 34 seq_num 142) 2024-08-17T16:55:55.854 INFO:tasks.rados.rados.0.smithi063.stdout:288: finishing write tid 1 to smithi06387500-29 2024-08-17T16:55:55.856 INFO:tasks.rados.rados.0.smithi063.stdout:287: done (3 left) 2024-08-17T16:55:55.856 INFO:tasks.rados.rados.0.smithi063.stdout:290: done (2 left) 2024-08-17T16:55:55.856 INFO:tasks.rados.rados.0.smithi063.stdout:291: write oid 7 current snap is 34 2024-08-17T16:55:55.856 INFO:tasks.rados.rados.0.smithi063.stdout:291: seq_num 145 ranges {437015=721812,1701889=509708} 2024-08-17T16:55:55.862 INFO:tasks.rados.rados.0.smithi063.stdout:291: writing smithi06387500-7 from 437015 to 1158827 tid 1 2024-08-17T16:55:55.868 INFO:tasks.rados.rados.0.smithi063.stdout:291: writing smithi06387500-7 from 1701889 to 2211597 tid 2 2024-08-17T16:55:55.869 INFO:tasks.rados.rados.0.smithi063.stdout:292: delete oid 38 current snap is 34 2024-08-17T16:55:55.869 INFO:tasks.rados.rados.0.smithi063.stdout:288: finishing write tid 2 to smithi06387500-29 2024-08-17T16:55:55.869 INFO:tasks.rados.rados.0.smithi063.stdout:288: finishing write tid 3 to smithi06387500-29 2024-08-17T16:55:55.869 INFO:tasks.rados.rados.0.smithi063.stdout:288: finishing write tid 4 to smithi06387500-29 2024-08-17T16:55:55.869 INFO:tasks.rados.rados.0.smithi063.stdout:288: oid 29 updating version 0 to 63 2024-08-17T16:55:55.869 INFO:tasks.rados.rados.0.smithi063.stdout:288: oid 29 version 63 is already newer than 62 2024-08-17T16:55:55.869 INFO:tasks.rados.rados.0.smithi063.stdout:288: oid 29 version 63 is already newer than 61 2024-08-17T16:55:55.869 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 63 (ObjNum 143 snap 34 seq_num 143) dirty exists 2024-08-17T16:55:55.869 INFO:tasks.rados.rados.0.smithi063.stdout:288: left oid 29 (ObjNum 143 snap 34 seq_num 143) 2024-08-17T16:55:55.869 INFO:tasks.rados.rados.0.smithi063.stdout:289: expect (ObjNum 29 snap 0 seq_num 29) 2024-08-17T16:55:55.953 INFO:tasks.rados.rados.0.smithi063.stdout:291: finishing write tid 1 to smithi06387500-7 2024-08-17T16:55:55.954 INFO:tasks.rados.rados.0.smithi063.stdout:291: finishing write tid 2 to smithi06387500-7 2024-08-17T16:55:55.954 INFO:tasks.rados.rados.0.smithi063.stdout:291: finishing write tid 3 to smithi06387500-7 2024-08-17T16:55:55.954 INFO:tasks.rados.rados.0.smithi063.stdout:291: finishing write tid 4 to smithi06387500-7 2024-08-17T16:55:55.954 INFO:tasks.rados.rados.0.smithi063.stdout:291: oid 7 updating version 0 to 117 2024-08-17T16:55:55.954 INFO:tasks.rados.rados.0.smithi063.stdout:291: oid 7 updating version 117 to 118 2024-08-17T16:55:55.954 INFO:tasks.rados.rados.0.smithi063.stdout:291: oid 7 version 118 is already newer than 116 2024-08-17T16:55:55.954 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 118 (ObjNum 145 snap 34 seq_num 145) dirty exists 2024-08-17T16:55:55.955 INFO:tasks.rados.rados.0.smithi063.stdout:291: left oid 7 (ObjNum 145 snap 34 seq_num 145) 2024-08-17T16:55:55.955 INFO:tasks.rados.rados.0.smithi063.stdout:288: done (3 left) 2024-08-17T16:55:55.955 INFO:tasks.rados.rados.0.smithi063.stdout:289: done (2 left) 2024-08-17T16:55:55.955 INFO:tasks.rados.rados.0.smithi063.stdout:291: done (1 left) 2024-08-17T16:55:55.956 INFO:tasks.rados.rados.0.smithi063.stdout:292: done (0 left) 2024-08-17T16:55:55.956 INFO:tasks.rados.rados.0.smithi063.stdout:293: write oid 28 current snap is 34 2024-08-17T16:55:55.956 INFO:tasks.rados.rados.0.smithi063.stdout:293: seq_num 147 ranges {470629=472310,1418853=513377} 2024-08-17T16:55:55.958 INFO:tasks.rados.rados.0.smithi063.stdout:293: writing smithi06387500-28 from 470629 to 942939 tid 1 2024-08-17T16:55:55.964 INFO:tasks.rados.rados.0.smithi063.stdout:293: writing smithi06387500-28 from 1418853 to 1932230 tid 2 2024-08-17T16:55:55.964 INFO:tasks.rados.rados.0.smithi063.stdout:294: write oid 26 current snap is 34 2024-08-17T16:55:55.964 INFO:tasks.rados.rados.0.smithi063.stdout:293: finishing write tid 1 to smithi06387500-28 2024-08-17T16:55:55.964 INFO:tasks.rados.rados.0.smithi063.stdout:294: seq_num 148 ranges {487436=547559,1760982=657035} 2024-08-17T16:55:55.970 INFO:tasks.rados.rados.0.smithi063.stdout:294: writing smithi06387500-26 from 487436 to 1034995 tid 1 2024-08-17T16:55:55.978 INFO:tasks.rados.rados.0.smithi063.stdout:294: writing smithi06387500-26 from 1760982 to 2418017 tid 2 2024-08-17T16:55:55.978 INFO:tasks.rados.rados.0.smithi063.stdout:295: rollback oid 7 current snap is 34 2024-08-17T16:55:55.978 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 30 2024-08-17T16:55:55.978 INFO:tasks.rados.rados.0.smithi063.stdout:293: finishing write tid 2 to smithi06387500-28 2024-08-17T16:55:55.978 INFO:tasks.rados.rados.0.smithi063.stdout:296: rollback oid 24 current snap is 34 2024-08-17T16:55:55.978 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 24 to 20 2024-08-17T16:55:55.978 INFO:tasks.rados.rados.0.smithi063.stdout:293: finishing write tid 3 to smithi06387500-28 2024-08-17T16:55:55.978 INFO:tasks.rados.rados.0.smithi063.stdout:293: finishing write tid 4 to smithi06387500-28 2024-08-17T16:55:55.978 INFO:tasks.rados.rados.0.smithi063.stdout:293: oid 28 updating version 0 to 86 2024-08-17T16:55:55.978 INFO:tasks.rados.rados.0.smithi063.stdout:293: oid 28 updating version 86 to 87 2024-08-17T16:55:55.978 INFO:tasks.rados.rados.0.smithi063.stdout:293: oid 28 updating version 87 to 88 2024-08-17T16:55:55.979 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 88 (ObjNum 147 snap 34 seq_num 147) dirty exists 2024-08-17T16:55:55.979 INFO:tasks.rados.rados.0.smithi063.stdout:293: left oid 28 (ObjNum 147 snap 34 seq_num 147) 2024-08-17T16:55:55.979 INFO:tasks.rados.rados.0.smithi063.stdout:294: finishing write tid 1 to smithi06387500-26 2024-08-17T16:55:55.979 INFO:tasks.rados.rados.0.smithi063.stdout:293: done (3 left) 2024-08-17T16:55:55.979 INFO:tasks.rados.rados.0.smithi063.stdout:297: delete oid 8 current snap is 34 2024-08-17T16:55:55.979 INFO:tasks.rados.rados.0.smithi063.stdout:296: finishing rollback tid 1 to smithi06387500-24 2024-08-17T16:55:55.979 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 0 (ObjNum 113 snap 0 seq_num 1459010438) dirty dne 2024-08-17T16:55:55.979 INFO:tasks.rados.rados.0.smithi063.stdout:296: done (3 left) 2024-08-17T16:55:55.979 INFO:tasks.rados.rados.0.smithi063.stdout:297: done (2 left) 2024-08-17T16:55:55.979 INFO:tasks.rados.rados.0.smithi063.stdout:298: snap_remove snap 28 2024-08-17T16:55:56.838 INFO:tasks.rados.rados.0.smithi063.stdout:298: done (2 left) 2024-08-17T16:55:56.838 INFO:tasks.rados.rados.0.smithi063.stdout:299: snap_remove snap 32 2024-08-17T16:55:57.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:56 smithi162 ceph-mon[34633]: osdmap e139: 8 total, 8 up, 8 in 2024-08-17T16:55:57.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:56 smithi162 ceph-mon[34633]: pgmap v190: 89 pgs: 89 active+clean; 144 MiB data, 486 MiB used, 715 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-08-17T16:55:57.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:56 smithi063 ceph-mon[42680]: osdmap e139: 8 total, 8 up, 8 in 2024-08-17T16:55:57.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:56 smithi063 ceph-mon[42680]: pgmap v190: 89 pgs: 89 active+clean; 144 MiB data, 486 MiB used, 715 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-08-17T16:55:57.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:56 smithi063 ceph-mon[31063]: osdmap e139: 8 total, 8 up, 8 in 2024-08-17T16:55:57.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:56 smithi063 ceph-mon[31063]: pgmap v190: 89 pgs: 89 active+clean; 144 MiB data, 486 MiB used, 715 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-08-17T16:55:57.835 INFO:tasks.rados.rados.0.smithi063.stdout:299: done (2 left) 2024-08-17T16:55:57.836 INFO:tasks.rados.rados.0.smithi063.stdout:300: rollback oid 5 current snap is 34 2024-08-17T16:55:57.836 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 30 2024-08-17T16:55:57.836 INFO:tasks.rados.rados.0.smithi063.stdout:294: finishing write tid 2 to smithi06387500-26 2024-08-17T16:55:57.836 INFO:tasks.rados.rados.0.smithi063.stdout:294: finishing write tid 3 to smithi06387500-26 2024-08-17T16:55:57.836 INFO:tasks.rados.rados.0.smithi063.stdout:295: finishing rollback tid 0 to smithi06387500-7 2024-08-17T16:55:57.836 INFO:tasks.rados.rados.0.smithi063.stdout:295: finishing rollback tid 1 to smithi06387500-7 2024-08-17T16:55:57.836 INFO:tasks.rados.rados.0.smithi063.stdout:294: finishing write tid 4 to smithi06387500-26 2024-08-17T16:55:57.836 INFO:tasks.rados.rados.0.smithi063.stdout:294: oid 26 updating version 0 to 65 2024-08-17T16:55:57.836 INFO:tasks.rados.rados.0.smithi063.stdout:294: oid 26 updating version 65 to 67 2024-08-17T16:55:57.836 INFO:tasks.rados.rados.0.smithi063.stdout:294: oid 26 version 67 is already newer than 66 2024-08-17T16:55:57.836 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 67 (ObjNum 148 snap 34 seq_num 148) dirty exists 2024-08-17T16:55:57.836 INFO:tasks.rados.rados.0.smithi063.stdout:294: left oid 26 (ObjNum 148 snap 34 seq_num 148) 2024-08-17T16:55:57.837 INFO:tasks.rados.rados.0.smithi063.stdout:295: finishing rollback tid 2 to smithi06387500-7 2024-08-17T16:55:57.837 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 121 (ObjNum 120 snap 22 seq_num 120) dirty exists 2024-08-17T16:55:57.837 INFO:tasks.rados.rados.0.smithi063.stdout:294: done (2 left) 2024-08-17T16:55:57.837 INFO:tasks.rados.rados.0.smithi063.stdout:295: done (1 left) 2024-08-17T16:55:57.837 INFO:tasks.rados.rados.0.smithi063.stdout:301: snap_create 2024-08-17T16:55:57.840 INFO:tasks.rados.rados.0.smithi063.stdout:300: finishing rollback tid 0 to smithi06387500-5 2024-08-17T16:55:57.841 INFO:tasks.rados.rados.0.smithi063.stdout:300: finishing rollback tid 1 to smithi06387500-5 2024-08-17T16:55:57.841 INFO:tasks.rados.rados.0.smithi063.stdout:300: finishing rollback tid 2 to smithi06387500-5 2024-08-17T16:55:57.841 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 99 (ObjNum 99 snap 16 seq_num 99) dirty exists 2024-08-17T16:55:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:57 smithi162 ceph-mon[34633]: osdmap e140: 8 total, 8 up, 8 in 2024-08-17T16:55:58.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:57 smithi063 ceph-mon[42680]: osdmap e140: 8 total, 8 up, 8 in 2024-08-17T16:55:58.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:57 smithi063 ceph-mon[31063]: osdmap e140: 8 total, 8 up, 8 in 2024-08-17T16:55:58.836 INFO:tasks.rados.rados.0.smithi063.stdout:300: done (1 left) 2024-08-17T16:55:58.836 INFO:tasks.rados.rados.0.smithi063.stdout:301: done (0 left) 2024-08-17T16:55:58.836 INFO:tasks.rados.rados.0.smithi063.stdout:302: write oid 3 current snap is 35 2024-08-17T16:55:58.836 INFO:tasks.rados.rados.0.smithi063.stdout:302: seq_num 150 ranges {521050=698057,1877946=643104} 2024-08-17T16:55:58.843 INFO:tasks.rados.rados.0.smithi063.stdout:302: writing smithi06387500-3 from 521050 to 1219107 tid 1 2024-08-17T16:55:58.850 INFO:tasks.rados.rados.0.smithi063.stdout:302: writing smithi06387500-3 from 1877946 to 2521050 tid 2 2024-08-17T16:55:58.850 INFO:tasks.rados.rados.0.smithi063.stdout:303: read oid 25 snap 30 2024-08-17T16:55:58.850 INFO:tasks.rados.rados.0.smithi063.stdout:303: expect (ObjNum 78 snap 9 seq_num 78) 2024-08-17T16:55:58.851 INFO:tasks.rados.rados.0.smithi063.stdout:304: snap_create 2024-08-17T16:55:58.851 INFO:tasks.rados.rados.0.smithi063.stdout:302: finishing write tid 1 to smithi06387500-3 2024-08-17T16:55:58.854 INFO:tasks.rados.rados.0.smithi063.stdout:302: finishing write tid 2 to smithi06387500-3 2024-08-17T16:55:58.854 INFO:tasks.rados.rados.0.smithi063.stdout:302: finishing write tid 3 to smithi06387500-3 2024-08-17T16:55:58.854 INFO:tasks.rados.rados.0.smithi063.stdout:302: finishing write tid 4 to smithi06387500-3 2024-08-17T16:55:58.854 INFO:tasks.rados.rados.0.smithi063.stdout:302: oid 3 updating version 0 to 75 2024-08-17T16:55:58.854 INFO:tasks.rados.rados.0.smithi063.stdout:302: oid 3 version 75 is already newer than 73 2024-08-17T16:55:58.854 INFO:tasks.rados.rados.0.smithi063.stdout:302: oid 3 version 75 is already newer than 74 2024-08-17T16:55:58.854 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 75 (ObjNum 150 snap 35 seq_num 150) dirty exists 2024-08-17T16:55:58.854 INFO:tasks.rados.rados.0.smithi063.stdout:302: left oid 3 (ObjNum 150 snap 35 seq_num 150) 2024-08-17T16:55:58.861 INFO:tasks.rados.rados.0.smithi063.stdout:303: expect (ObjNum 78 snap 9 seq_num 78) 2024-08-17T16:55:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:58 smithi162 ceph-mon[34633]: osdmap e141: 8 total, 8 up, 8 in 2024-08-17T16:55:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:58 smithi162 ceph-mon[34633]: pgmap v193: 89 pgs: 89 active+clean; 144 MiB data, 486 MiB used, 715 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-08-17T16:55:59.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:55:58 smithi162 ceph-mon[34633]: osdmap e142: 8 total, 8 up, 8 in 2024-08-17T16:55:59.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:58 smithi063 ceph-mon[31063]: osdmap e141: 8 total, 8 up, 8 in 2024-08-17T16:55:59.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:58 smithi063 ceph-mon[31063]: pgmap v193: 89 pgs: 89 active+clean; 144 MiB data, 486 MiB used, 715 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-08-17T16:55:59.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:55:58 smithi063 ceph-mon[31063]: osdmap e142: 8 total, 8 up, 8 in 2024-08-17T16:55:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:58 smithi063 ceph-mon[42680]: osdmap e141: 8 total, 8 up, 8 in 2024-08-17T16:55:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:58 smithi063 ceph-mon[42680]: pgmap v193: 89 pgs: 89 active+clean; 144 MiB data, 486 MiB used, 715 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-08-17T16:55:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:55:58 smithi063 ceph-mon[42680]: osdmap e142: 8 total, 8 up, 8 in 2024-08-17T16:55:59.829 INFO:tasks.rados.rados.0.smithi063.stdout:302: done (2 left) 2024-08-17T16:55:59.829 INFO:tasks.rados.rados.0.smithi063.stdout:303: done (1 left) 2024-08-17T16:55:59.829 INFO:tasks.rados.rados.0.smithi063.stdout:304: done (0 left) 2024-08-17T16:55:59.829 INFO:tasks.rados.rados.0.smithi063.stdout:305: snap_remove snap 34 2024-08-17T16:56:00.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:55:59 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:55:59] "GET /metrics HTTP/1.1" 200 36139 "" "Prometheus/2.43.0" 2024-08-17T16:56:00.834 INFO:tasks.rados.rados.0.smithi063.stdout:305: done (0 left) 2024-08-17T16:56:00.835 INFO:tasks.rados.rados.0.smithi063.stdout:306: read oid 30 snap -1 2024-08-17T16:56:00.835 INFO:tasks.rados.rados.0.smithi063.stdout:306: expect (ObjNum 29 snap 0 seq_num 29) 2024-08-17T16:56:00.835 INFO:tasks.rados.rados.0.smithi063.stdout:307: write oid 9 current snap is 36 2024-08-17T16:56:00.835 INFO:tasks.rados.rados.0.smithi063.stdout:307: seq_num 151 ranges {537857=773306,1936428=520715} 2024-08-17T16:56:00.841 INFO:tasks.rados.rados.0.smithi063.stdout:307: writing smithi06387500-9 from 537857 to 1311163 tid 1 2024-08-17T16:56:00.848 INFO:tasks.rados.rados.0.smithi063.stdout:307: writing smithi06387500-9 from 1936428 to 2457143 tid 2 2024-08-17T16:56:00.848 INFO:tasks.rados.rados.0.smithi063.stdout:308: delete oid 1 current snap is 36 2024-08-17T16:56:00.848 INFO:tasks.rados.rados.0.smithi063.stdout:306: expect (ObjNum 29 snap 0 seq_num 29) 2024-08-17T16:56:00.934 INFO:tasks.rados.rados.0.smithi063.stdout:307: finishing write tid 1 to smithi06387500-9 2024-08-17T16:56:00.934 INFO:tasks.rados.rados.0.smithi063.stdout:306: done (2 left) 2024-08-17T16:56:00.934 INFO:tasks.rados.rados.0.smithi063.stdout:308: done (1 left) 2024-08-17T16:56:00.935 INFO:tasks.rados.rados.0.smithi063.stdout:309: write oid 27 current snap is 36 2024-08-17T16:56:00.935 INFO:tasks.rados.rados.0.smithi063.stdout:307: finishing write tid 2 to smithi06387500-9 2024-08-17T16:56:00.935 INFO:tasks.rados.rados.0.smithi063.stdout:309: seq_num 153 ranges {571471=640157,1653392=524384} 2024-08-17T16:56:00.940 INFO:tasks.rados.rados.0.smithi063.stdout:309: writing smithi06387500-27 from 571471 to 1211628 tid 1 2024-08-17T16:56:00.947 INFO:tasks.rados.rados.0.smithi063.stdout:309: writing smithi06387500-27 from 1653392 to 2177776 tid 2 2024-08-17T16:56:00.947 INFO:tasks.rados.rados.0.smithi063.stdout:310: snap_create 2024-08-17T16:56:00.947 INFO:tasks.rados.rados.0.smithi063.stdout:307: finishing write tid 3 to smithi06387500-9 2024-08-17T16:56:00.947 INFO:tasks.rados.rados.0.smithi063.stdout:307: finishing write tid 4 to smithi06387500-9 2024-08-17T16:56:00.947 INFO:tasks.rados.rados.0.smithi063.stdout:307: oid 9 updating version 0 to 129 2024-08-17T16:56:00.947 INFO:tasks.rados.rados.0.smithi063.stdout:307: oid 9 version 129 is already newer than 127 2024-08-17T16:56:00.947 INFO:tasks.rados.rados.0.smithi063.stdout:307: oid 9 version 129 is already newer than 128 2024-08-17T16:56:00.947 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 129 (ObjNum 151 snap 36 seq_num 151) dirty exists 2024-08-17T16:56:00.947 INFO:tasks.rados.rados.0.smithi063.stdout:307: left oid 9 (ObjNum 151 snap 36 seq_num 151) 2024-08-17T16:56:00.947 INFO:tasks.rados.rados.0.smithi063.stdout:309: finishing write tid 1 to smithi06387500-27 2024-08-17T16:56:00.949 INFO:tasks.rados.rados.0.smithi063.stdout:309: finishing write tid 2 to smithi06387500-27 2024-08-17T16:56:00.949 INFO:tasks.rados.rados.0.smithi063.stdout:309: finishing write tid 3 to smithi06387500-27 2024-08-17T16:56:00.949 INFO:tasks.rados.rados.0.smithi063.stdout:309: finishing write tid 4 to smithi06387500-27 2024-08-17T16:56:00.949 INFO:tasks.rados.rados.0.smithi063.stdout:309: oid 27 updating version 0 to 24 2024-08-17T16:56:00.949 INFO:tasks.rados.rados.0.smithi063.stdout:309: oid 27 version 24 is already newer than 23 2024-08-17T16:56:00.949 INFO:tasks.rados.rados.0.smithi063.stdout:309: oid 27 updating version 24 to 25 2024-08-17T16:56:00.949 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 25 (ObjNum 153 snap 36 seq_num 153) dirty exists 2024-08-17T16:56:00.950 INFO:tasks.rados.rados.0.smithi063.stdout:309: left oid 27 (ObjNum 153 snap 36 seq_num 153) 2024-08-17T16:56:01.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:00 smithi162 ceph-mon[34633]: osdmap e143: 8 total, 8 up, 8 in 2024-08-17T16:56:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:00 smithi162 ceph-mon[34633]: pgmap v196: 89 pgs: 89 active+clean; 144 MiB data, 486 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:01.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:00 smithi063 ceph-mon[42680]: osdmap e143: 8 total, 8 up, 8 in 2024-08-17T16:56:01.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:00 smithi063 ceph-mon[42680]: pgmap v196: 89 pgs: 89 active+clean; 144 MiB data, 486 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:01.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:00 smithi063 ceph-mon[31063]: osdmap e143: 8 total, 8 up, 8 in 2024-08-17T16:56:01.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:00 smithi063 ceph-mon[31063]: pgmap v196: 89 pgs: 89 active+clean; 144 MiB data, 486 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:01.839 INFO:tasks.rados.rados.0.smithi063.stdout:307: done (2 left) 2024-08-17T16:56:01.839 INFO:tasks.rados.rados.0.smithi063.stdout:309: done (1 left) 2024-08-17T16:56:01.839 INFO:tasks.rados.rados.0.smithi063.stdout:310: done (0 left) 2024-08-17T16:56:01.839 INFO:tasks.rados.rados.0.smithi063.stdout:311: snap_create 2024-08-17T16:56:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:01 smithi162 ceph-mon[34633]: osdmap e144: 8 total, 8 up, 8 in 2024-08-17T16:56:02.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:01 smithi063 ceph-mon[42680]: osdmap e144: 8 total, 8 up, 8 in 2024-08-17T16:56:02.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:01 smithi063 ceph-mon[31063]: osdmap e144: 8 total, 8 up, 8 in 2024-08-17T16:56:02.844 INFO:tasks.rados.rados.0.smithi063.stdout:311: done (0 left) 2024-08-17T16:56:02.844 INFO:tasks.rados.rados.0.smithi063.stdout:312: read oid 35 snap -1 2024-08-17T16:56:02.844 INFO:tasks.rados.rados.0.smithi063.stdout:312: expect deleted 2024-08-17T16:56:02.844 INFO:tasks.rados.rados.0.smithi063.stdout:313: delete oid 29 current snap is 38 2024-08-17T16:56:02.847 INFO:tasks.rados.rados.0.smithi063.stdout:312: done (1 left) 2024-08-17T16:56:02.847 INFO:tasks.rados.rados.0.smithi063.stdout:313: done (0 left) 2024-08-17T16:56:02.848 INFO:tasks.rados.rados.0.smithi063.stdout:314: snap_remove snap 22 2024-08-17T16:56:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:02 smithi162 ceph-mon[34633]: osdmap e145: 8 total, 8 up, 8 in 2024-08-17T16:56:03.189 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:02 smithi162 ceph-mon[34633]: pgmap v199: 89 pgs: 89 active+clean; 148 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 928 KiB/s wr, 8 op/s 2024-08-17T16:56:03.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:02 smithi063 ceph-mon[42680]: osdmap e145: 8 total, 8 up, 8 in 2024-08-17T16:56:03.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:02 smithi063 ceph-mon[42680]: pgmap v199: 89 pgs: 89 active+clean; 148 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 928 KiB/s wr, 8 op/s 2024-08-17T16:56:03.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:02 smithi063 ceph-mon[31063]: osdmap e145: 8 total, 8 up, 8 in 2024-08-17T16:56:03.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:02 smithi063 ceph-mon[31063]: pgmap v199: 89 pgs: 89 active+clean; 148 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 928 KiB/s wr, 8 op/s 2024-08-17T16:56:03.846 INFO:tasks.rados.rados.0.smithi063.stdout:314: done (0 left) 2024-08-17T16:56:03.846 INFO:tasks.rados.rados.0.smithi063.stdout:315: snap_remove snap 30 2024-08-17T16:56:04.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:03 smithi063 ceph-mon[42680]: osdmap e146: 8 total, 8 up, 8 in 2024-08-17T16:56:04.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:03 smithi063 ceph-mon[31063]: osdmap e146: 8 total, 8 up, 8 in 2024-08-17T16:56:04.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:03 smithi162 ceph-mon[34633]: osdmap e146: 8 total, 8 up, 8 in 2024-08-17T16:56:04.830 INFO:tasks.rados.rados.0.smithi063.stdout:315: done (0 left) 2024-08-17T16:56:04.831 INFO:tasks.rados.rados.0.smithi063.stdout:316: write oid 29 current snap is 38 2024-08-17T16:56:04.831 INFO:tasks.rados.rados.0.smithi063.stdout:316: seq_num 155 ranges {605085=790655,2170356=434729} 2024-08-17T16:56:04.838 INFO:tasks.rados.rados.0.smithi063.stdout:316: writing smithi06387500-29 from 605085 to 1395740 tid 1 2024-08-17T16:56:04.845 INFO:tasks.rados.rados.0.smithi063.stdout:316: writing smithi06387500-29 from 2170356 to 2605085 tid 2 2024-08-17T16:56:04.846 INFO:tasks.rados.rados.0.smithi063.stdout:317: snap_create 2024-08-17T16:56:04.846 INFO:tasks.rados.rados.0.smithi063.stdout:316: finishing write tid 1 to smithi06387500-29 2024-08-17T16:56:04.848 INFO:tasks.rados.rados.0.smithi063.stdout:316: finishing write tid 2 to smithi06387500-29 2024-08-17T16:56:04.848 INFO:tasks.rados.rados.0.smithi063.stdout:316: finishing write tid 3 to smithi06387500-29 2024-08-17T16:56:04.848 INFO:tasks.rados.rados.0.smithi063.stdout:316: finishing write tid 4 to smithi06387500-29 2024-08-17T16:56:04.849 INFO:tasks.rados.rados.0.smithi063.stdout:316: oid 29 updating version 0 to 70 2024-08-17T16:56:04.849 INFO:tasks.rados.rados.0.smithi063.stdout:316: oid 29 version 70 is already newer than 69 2024-08-17T16:56:04.849 INFO:tasks.rados.rados.0.smithi063.stdout:316: oid 29 version 70 is already newer than 68 2024-08-17T16:56:04.849 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 70 (ObjNum 155 snap 38 seq_num 155) dirty exists 2024-08-17T16:56:04.849 INFO:tasks.rados.rados.0.smithi063.stdout:316: left oid 29 (ObjNum 155 snap 38 seq_num 155) 2024-08-17T16:56:05.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:04 smithi063 ceph-mon[42680]: osdmap e147: 8 total, 8 up, 8 in 2024-08-17T16:56:05.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:04 smithi063 ceph-mon[42680]: pgmap v202: 89 pgs: 89 active+clean; 148 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 929 KiB/s wr, 8 op/s 2024-08-17T16:56:05.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:04 smithi063 ceph-mon[42680]: osdmap e148: 8 total, 8 up, 8 in 2024-08-17T16:56:05.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:04 smithi063 ceph-mon[31063]: osdmap e147: 8 total, 8 up, 8 in 2024-08-17T16:56:05.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:04 smithi063 ceph-mon[31063]: pgmap v202: 89 pgs: 89 active+clean; 148 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 929 KiB/s wr, 8 op/s 2024-08-17T16:56:05.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:04 smithi063 ceph-mon[31063]: osdmap e148: 8 total, 8 up, 8 in 2024-08-17T16:56:05.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:04 smithi162 ceph-mon[34633]: osdmap e147: 8 total, 8 up, 8 in 2024-08-17T16:56:05.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:04 smithi162 ceph-mon[34633]: pgmap v202: 89 pgs: 89 active+clean; 148 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 929 KiB/s wr, 8 op/s 2024-08-17T16:56:05.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:04 smithi162 ceph-mon[34633]: osdmap e148: 8 total, 8 up, 8 in 2024-08-17T16:56:05.832 INFO:tasks.rados.rados.0.smithi063.stdout:316: done (1 left) 2024-08-17T16:56:05.832 INFO:tasks.rados.rados.0.smithi063.stdout:317: done (0 left) 2024-08-17T16:56:05.832 INFO:tasks.rados.rados.0.smithi063.stdout:318: write oid 7 current snap is 39 2024-08-17T16:56:05.832 INFO:tasks.rados.rados.0.smithi063.stdout:318: seq_num 156 ranges {621892=465904,1712485=671711} 2024-08-17T16:56:05.838 INFO:tasks.rados.rados.0.smithi063.stdout:318: writing smithi06387500-7 from 621892 to 1087796 tid 1 2024-08-17T16:56:05.845 INFO:tasks.rados.rados.0.smithi063.stdout:318: writing smithi06387500-7 from 1712485 to 2384196 tid 2 2024-08-17T16:56:05.845 INFO:tasks.rados.rados.0.smithi063.stdout:319: rollback oid 50 current snap is 39 2024-08-17T16:56:05.845 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 50 to 36 2024-08-17T16:56:05.846 INFO:tasks.rados.rados.0.smithi063.stdout:318: finishing write tid 1 to smithi06387500-7 2024-08-17T16:56:05.846 INFO:tasks.rados.rados.0.smithi063.stdout:320: read oid 36 snap -1 2024-08-17T16:56:05.846 INFO:tasks.rados.rados.0.smithi063.stdout:320: expect (ObjNum 134 snap 28 seq_num 134) 2024-08-17T16:56:05.846 INFO:tasks.rados.rados.0.smithi063.stdout:321: rollback oid 44 current snap is 39 2024-08-17T16:56:05.846 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 44 to 37 2024-08-17T16:56:05.846 INFO:tasks.rados.rados.0.smithi063.stdout:322: write oid 17 current snap is 39 2024-08-17T16:56:05.846 INFO:tasks.rados.rados.0.smithi063.stdout:322: seq_num 157 ranges {638699=541153,1770967=531722} 2024-08-17T16:56:05.852 INFO:tasks.rados.rados.0.smithi063.stdout:322: writing smithi06387500-17 from 638699 to 1179852 tid 1 2024-08-17T16:56:05.859 INFO:tasks.rados.rados.0.smithi063.stdout:322: writing smithi06387500-17 from 1770967 to 2302689 tid 2 2024-08-17T16:56:05.859 INFO:tasks.rados.rados.0.smithi063.stdout:323: snap_create 2024-08-17T16:56:05.859 INFO:tasks.rados.rados.0.smithi063.stdout:321: finishing rollback tid 0 to smithi06387500-44 2024-08-17T16:56:05.859 INFO:tasks.rados.rados.0.smithi063.stdout:319: finishing rollback tid 0 to smithi06387500-50 2024-08-17T16:56:05.859 INFO:tasks.rados.rados.0.smithi063.stdout:319: finishing rollback tid 1 to smithi06387500-50 2024-08-17T16:56:05.859 INFO:tasks.rados.rados.0.smithi063.stdout:319: finishing rollback tid 2 to smithi06387500-50 2024-08-17T16:56:05.859 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 175 (ObjNum 94 snap 16 seq_num 94) dirty exists 2024-08-17T16:56:05.859 INFO:tasks.rados.rados.0.smithi063.stdout:318: finishing write tid 2 to smithi06387500-7 2024-08-17T16:56:05.860 INFO:tasks.rados.rados.0.smithi063.stdout:318: finishing write tid 3 to smithi06387500-7 2024-08-17T16:56:05.860 INFO:tasks.rados.rados.0.smithi063.stdout:318: finishing write tid 4 to smithi06387500-7 2024-08-17T16:56:05.860 INFO:tasks.rados.rados.0.smithi063.stdout:318: oid 7 updating version 0 to 140 2024-08-17T16:56:05.860 INFO:tasks.rados.rados.0.smithi063.stdout:318: oid 7 version 140 is already newer than 139 2024-08-17T16:56:05.860 INFO:tasks.rados.rados.0.smithi063.stdout:318: oid 7 updating version 140 to 141 2024-08-17T16:56:05.860 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 141 (ObjNum 156 snap 39 seq_num 156) dirty exists 2024-08-17T16:56:05.860 INFO:tasks.rados.rados.0.smithi063.stdout:318: left oid 7 (ObjNum 156 snap 39 seq_num 156) 2024-08-17T16:56:05.860 INFO:tasks.rados.rados.0.smithi063.stdout:321: finishing rollback tid 1 to smithi06387500-44 2024-08-17T16:56:05.860 INFO:tasks.rados.rados.0.smithi063.stdout:321: finishing rollback tid 2 to smithi06387500-44 2024-08-17T16:56:05.860 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 46 (ObjNum 129 snap 27 seq_num 129) dirty exists 2024-08-17T16:56:05.860 INFO:tasks.rados.rados.0.smithi063.stdout:320: expect (ObjNum 134 snap 28 seq_num 134) 2024-08-17T16:56:05.933 INFO:tasks.rados.rados.0.smithi063.stdout:322: finishing write tid 1 to smithi06387500-17 2024-08-17T16:56:05.934 INFO:tasks.rados.rados.0.smithi063.stdout:322: finishing write tid 2 to smithi06387500-17 2024-08-17T16:56:05.934 INFO:tasks.rados.rados.0.smithi063.stdout:322: finishing write tid 3 to smithi06387500-17 2024-08-17T16:56:05.934 INFO:tasks.rados.rados.0.smithi063.stdout:322: finishing write tid 4 to smithi06387500-17 2024-08-17T16:56:05.934 INFO:tasks.rados.rados.0.smithi063.stdout:322: oid 17 updating version 0 to 88 2024-08-17T16:56:05.934 INFO:tasks.rados.rados.0.smithi063.stdout:322: oid 17 updating version 88 to 89 2024-08-17T16:56:05.934 INFO:tasks.rados.rados.0.smithi063.stdout:322: oid 17 version 89 is already newer than 87 2024-08-17T16:56:05.934 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 89 (ObjNum 157 snap 39 seq_num 157) dirty exists 2024-08-17T16:56:05.934 INFO:tasks.rados.rados.0.smithi063.stdout:322: left oid 17 (ObjNum 157 snap 39 seq_num 157) 2024-08-17T16:56:06.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:05 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:06.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:05 smithi063 ceph-mon[31063]: osdmap e149: 8 total, 8 up, 8 in 2024-08-17T16:56:06.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:05 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:06.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:05 smithi063 ceph-mon[42680]: osdmap e149: 8 total, 8 up, 8 in 2024-08-17T16:56:06.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:05 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:06.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:05 smithi162 ceph-mon[34633]: osdmap e149: 8 total, 8 up, 8 in 2024-08-17T16:56:06.868 INFO:tasks.rados.rados.0.smithi063.stdout:318: done (5 left) 2024-08-17T16:56:06.868 INFO:tasks.rados.rados.0.smithi063.stdout:319: done (4 left) 2024-08-17T16:56:06.868 INFO:tasks.rados.rados.0.smithi063.stdout:320: done (3 left) 2024-08-17T16:56:06.869 INFO:tasks.rados.rados.0.smithi063.stdout:321: done (2 left) 2024-08-17T16:56:06.869 INFO:tasks.rados.rados.0.smithi063.stdout:322: done (1 left) 2024-08-17T16:56:06.869 INFO:tasks.rados.rados.0.smithi063.stdout:323: done (0 left) 2024-08-17T16:56:06.869 INFO:tasks.rados.rados.0.smithi063.stdout:324: write oid 40 current snap is 40 2024-08-17T16:56:06.869 INFO:tasks.rados.rados.0.smithi063.stdout:324: seq_num 158 ranges {655506=616402,1829449=675380} 2024-08-17T16:56:06.875 INFO:tasks.rados.rados.0.smithi063.stdout:324: writing smithi06387500-40 from 655506 to 1271908 tid 1 2024-08-17T16:56:06.882 INFO:tasks.rados.rados.0.smithi063.stdout:324: writing smithi06387500-40 from 1829449 to 2504829 tid 2 2024-08-17T16:56:06.882 INFO:tasks.rados.rados.0.smithi063.stdout:325: snap_create 2024-08-17T16:56:06.882 INFO:tasks.rados.rados.0.smithi063.stdout:324: finishing write tid 1 to smithi06387500-40 2024-08-17T16:56:06.885 INFO:tasks.rados.rados.0.smithi063.stdout:324: finishing write tid 2 to smithi06387500-40 2024-08-17T16:56:06.885 INFO:tasks.rados.rados.0.smithi063.stdout:324: finishing write tid 3 to smithi06387500-40 2024-08-17T16:56:06.885 INFO:tasks.rados.rados.0.smithi063.stdout:324: finishing write tid 4 to smithi06387500-40 2024-08-17T16:56:06.885 INFO:tasks.rados.rados.0.smithi063.stdout:324: oid 40 updating version 0 to 71 2024-08-17T16:56:06.886 INFO:tasks.rados.rados.0.smithi063.stdout:324: oid 40 updating version 71 to 72 2024-08-17T16:56:06.886 INFO:tasks.rados.rados.0.smithi063.stdout:324: oid 40 version 72 is already newer than 70 2024-08-17T16:56:06.886 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 72 (ObjNum 158 snap 40 seq_num 158) dirty exists 2024-08-17T16:56:06.886 INFO:tasks.rados.rados.0.smithi063.stdout:324: left oid 40 (ObjNum 158 snap 40 seq_num 158) 2024-08-17T16:56:07.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:06 smithi063 ceph-mon[42680]: pgmap v205: 89 pgs: 89 active+clean; 152 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 839 KiB/s wr, 7 op/s 2024-08-17T16:56:07.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:06 smithi063 ceph-mon[31063]: pgmap v205: 89 pgs: 89 active+clean; 152 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 839 KiB/s wr, 7 op/s 2024-08-17T16:56:07.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:06 smithi162 ceph-mon[34633]: pgmap v205: 89 pgs: 89 active+clean; 152 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 839 KiB/s wr, 7 op/s 2024-08-17T16:56:07.872 INFO:tasks.rados.rados.0.smithi063.stdout:324: done (1 left) 2024-08-17T16:56:07.872 INFO:tasks.rados.rados.0.smithi063.stdout:325: done (0 left) 2024-08-17T16:56:07.872 INFO:tasks.rados.rados.0.smithi063.stdout:326: write oid 33 current snap is 41 2024-08-17T16:56:07.872 INFO:tasks.rados.rados.0.smithi063.stdout:326: seq_num 159 ranges {672313=691651,1887931=419038} 2024-08-17T16:56:07.879 INFO:tasks.rados.rados.0.smithi063.stdout:326: writing smithi06387500-33 from 672313 to 1363964 tid 1 2024-08-17T16:56:07.885 INFO:tasks.rados.rados.0.smithi063.stdout:326: writing smithi06387500-33 from 1887931 to 2306969 tid 2 2024-08-17T16:56:07.885 INFO:tasks.rados.rados.0.smithi063.stdout:327: read oid 18 snap 23 2024-08-17T16:56:07.885 INFO:tasks.rados.rados.0.smithi063.stdout:327: expect (ObjNum 118 snap 20 seq_num 118) 2024-08-17T16:56:07.885 INFO:tasks.rados.rados.0.smithi063.stdout:328: read oid 48 snap 40 2024-08-17T16:56:07.885 INFO:tasks.rados.rados.0.smithi063.stdout:328: expect (ObjNum 128 snap 27 seq_num 128) 2024-08-17T16:56:07.885 INFO:tasks.rados.rados.0.smithi063.stdout:329: write oid 49 current snap is 41 2024-08-17T16:56:07.885 INFO:tasks.rados.rados.0.smithi063.stdout:326: finishing write tid 1 to smithi06387500-33 2024-08-17T16:56:07.885 INFO:tasks.rados.rados.0.smithi063.stdout:329: seq_num 160 ranges {689120=483253,1946413=679049} 2024-08-17T16:56:07.891 INFO:tasks.rados.rados.0.smithi063.stdout:329: writing smithi06387500-49 from 689120 to 1172373 tid 1 2024-08-17T16:56:07.900 INFO:tasks.rados.rados.0.smithi063.stdout:329: writing smithi06387500-49 from 1946413 to 2625462 tid 2 2024-08-17T16:56:07.900 INFO:tasks.rados.rados.0.smithi063.stdout:330: snap_create 2024-08-17T16:56:07.900 INFO:tasks.rados.rados.0.smithi063.stdout:326: finishing write tid 2 to smithi06387500-33 2024-08-17T16:56:07.900 INFO:tasks.rados.rados.0.smithi063.stdout:326: finishing write tid 3 to smithi06387500-33 2024-08-17T16:56:07.900 INFO:tasks.rados.rados.0.smithi063.stdout:326: finishing write tid 4 to smithi06387500-33 2024-08-17T16:56:07.900 INFO:tasks.rados.rados.0.smithi063.stdout:326: oid 33 updating version 0 to 88 2024-08-17T16:56:07.901 INFO:tasks.rados.rados.0.smithi063.stdout:326: oid 33 updating version 88 to 89 2024-08-17T16:56:07.901 INFO:tasks.rados.rados.0.smithi063.stdout:326: oid 33 updating version 89 to 90 2024-08-17T16:56:07.901 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 90 (ObjNum 159 snap 41 seq_num 159) dirty exists 2024-08-17T16:56:07.901 INFO:tasks.rados.rados.0.smithi063.stdout:326: left oid 33 (ObjNum 159 snap 41 seq_num 159) 2024-08-17T16:56:07.901 INFO:tasks.rados.rados.0.smithi063.stdout:328: expect (ObjNum 128 snap 27 seq_num 128) 2024-08-17T16:56:08.027 INFO:tasks.rados.rados.0.smithi063.stdout:327: expect (ObjNum 118 snap 20 seq_num 118) 2024-08-17T16:56:08.254 INFO:tasks.rados.rados.0.smithi063.stdout:329: finishing write tid 1 to smithi06387500-49 2024-08-17T16:56:08.254 INFO:tasks.rados.rados.0.smithi063.stdout:329: finishing write tid 2 to smithi06387500-49 2024-08-17T16:56:08.254 INFO:tasks.rados.rados.0.smithi063.stdout:329: finishing write tid 3 to smithi06387500-49 2024-08-17T16:56:08.254 INFO:tasks.rados.rados.0.smithi063.stdout:329: finishing write tid 4 to smithi06387500-49 2024-08-17T16:56:08.254 INFO:tasks.rados.rados.0.smithi063.stdout:329: oid 49 updating version 0 to 248 2024-08-17T16:56:08.255 INFO:tasks.rados.rados.0.smithi063.stdout:329: oid 49 version 248 is already newer than 247 2024-08-17T16:56:08.255 INFO:tasks.rados.rados.0.smithi063.stdout:329: oid 49 updating version 248 to 249 2024-08-17T16:56:08.255 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 249 (ObjNum 160 snap 41 seq_num 160) dirty exists 2024-08-17T16:56:08.255 INFO:tasks.rados.rados.0.smithi063.stdout:329: left oid 49 (ObjNum 160 snap 41 seq_num 160) 2024-08-17T16:56:08.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:07 smithi063 ceph-mon[42680]: osdmap e150: 8 total, 8 up, 8 in 2024-08-17T16:56:08.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:07 smithi063 ceph-mon[31063]: osdmap e150: 8 total, 8 up, 8 in 2024-08-17T16:56:08.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:07 smithi162 ceph-mon[34633]: osdmap e150: 8 total, 8 up, 8 in 2024-08-17T16:56:08.878 INFO:tasks.rados.rados.0.smithi063.stdout:326: done (4 left) 2024-08-17T16:56:08.878 INFO:tasks.rados.rados.0.smithi063.stdout:327: done (3 left) 2024-08-17T16:56:08.878 INFO:tasks.rados.rados.0.smithi063.stdout:328: done (2 left) 2024-08-17T16:56:08.878 INFO:tasks.rados.rados.0.smithi063.stdout:329: done (1 left) 2024-08-17T16:56:08.878 INFO:tasks.rados.rados.0.smithi063.stdout:330: done (0 left) 2024-08-17T16:56:08.878 INFO:tasks.rados.rados.0.smithi063.stdout:331: snap_remove snap 39 2024-08-17T16:56:09.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:08 smithi063 ceph-mon[42680]: osdmap e151: 8 total, 8 up, 8 in 2024-08-17T16:56:09.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:08 smithi063 ceph-mon[42680]: pgmap v208: 89 pgs: 89 active+clean; 152 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 839 KiB/s wr, 7 op/s 2024-08-17T16:56:09.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:08 smithi063 ceph-mon[31063]: osdmap e151: 8 total, 8 up, 8 in 2024-08-17T16:56:09.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:08 smithi063 ceph-mon[31063]: pgmap v208: 89 pgs: 89 active+clean; 152 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 839 KiB/s wr, 7 op/s 2024-08-17T16:56:09.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:08 smithi162 ceph-mon[34633]: osdmap e151: 8 total, 8 up, 8 in 2024-08-17T16:56:09.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:08 smithi162 ceph-mon[34633]: pgmap v208: 89 pgs: 89 active+clean; 152 MiB data, 506 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 839 KiB/s wr, 7 op/s 2024-08-17T16:56:09.832 INFO:tasks.rados.rados.0.smithi063.stdout:331: done (0 left) 2024-08-17T16:56:09.832 INFO:tasks.rados.rados.0.smithi063.stdout:332: rollback oid 33 current snap is 42 2024-08-17T16:56:09.832 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 33 to 38 2024-08-17T16:56:09.833 INFO:tasks.rados.rados.0.smithi063.stdout:333: snap_remove snap 33 2024-08-17T16:56:09.874 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:56:09] "GET /metrics HTTP/1.1" 200 36136 "" "Prometheus/2.43.0" 2024-08-17T16:56:10.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:09 smithi063 ceph-mon[31063]: osdmap e152: 8 total, 8 up, 8 in 2024-08-17T16:56:10.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:09 smithi063 ceph-mon[31063]: osdmap e153: 8 total, 8 up, 8 in 2024-08-17T16:56:10.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:09 smithi063 ceph-mon[42680]: osdmap e152: 8 total, 8 up, 8 in 2024-08-17T16:56:10.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:09 smithi063 ceph-mon[42680]: osdmap e153: 8 total, 8 up, 8 in 2024-08-17T16:56:10.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:09 smithi162 ceph-mon[34633]: osdmap e152: 8 total, 8 up, 8 in 2024-08-17T16:56:10.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:09 smithi162 ceph-mon[34633]: osdmap e153: 8 total, 8 up, 8 in 2024-08-17T16:56:10.835 INFO:tasks.rados.rados.0.smithi063.stdout:333: done (1 left) 2024-08-17T16:56:10.836 INFO:tasks.rados.rados.0.smithi063.stdout:334: rollback oid 2 current snap is 42 2024-08-17T16:56:10.836 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 2 to 37 2024-08-17T16:56:10.836 INFO:tasks.rados.rados.0.smithi063.stdout:332: finishing rollback tid 0 to smithi06387500-33 2024-08-17T16:56:10.836 INFO:tasks.rados.rados.0.smithi063.stdout:332: finishing rollback tid 1 to smithi06387500-33 2024-08-17T16:56:10.836 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 93 (ObjNum 129 snap 0 seq_num 1807248872) dirty dne 2024-08-17T16:56:10.836 INFO:tasks.rados.rados.0.smithi063.stdout:332: done (1 left) 2024-08-17T16:56:10.836 INFO:tasks.rados.rados.0.smithi063.stdout:335: write oid 48 current snap is 42 2024-08-17T16:56:10.836 INFO:tasks.rados.rados.0.smithi063.stdout:335: seq_num 161 ranges {705927=558502,2004895=422707} 2024-08-17T16:56:10.842 INFO:tasks.rados.rados.0.smithi063.stdout:335: writing smithi06387500-48 from 705927 to 1264429 tid 1 2024-08-17T16:56:10.849 INFO:tasks.rados.rados.0.smithi063.stdout:335: writing smithi06387500-48 from 2004895 to 2427602 tid 2 2024-08-17T16:56:10.849 INFO:tasks.rados.rados.0.smithi063.stdout:336: snap_create 2024-08-17T16:56:10.849 INFO:tasks.rados.rados.0.smithi063.stdout:334: finishing rollback tid 1 to smithi06387500-2 2024-08-17T16:56:10.850 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T16:56:10.850 INFO:tasks.rados.rados.0.smithi063.stdout:335: finishing write tid 1 to smithi06387500-48 2024-08-17T16:56:10.852 INFO:tasks.rados.rados.0.smithi063.stdout:335: finishing write tid 2 to smithi06387500-48 2024-08-17T16:56:10.852 INFO:tasks.rados.rados.0.smithi063.stdout:335: finishing write tid 3 to smithi06387500-48 2024-08-17T16:56:10.852 INFO:tasks.rados.rados.0.smithi063.stdout:335: finishing write tid 4 to smithi06387500-48 2024-08-17T16:56:10.852 INFO:tasks.rados.rados.0.smithi063.stdout:335: oid 48 updating version 0 to 255 2024-08-17T16:56:10.852 INFO:tasks.rados.rados.0.smithi063.stdout:335: oid 48 updating version 255 to 257 2024-08-17T16:56:10.852 INFO:tasks.rados.rados.0.smithi063.stdout:335: oid 48 version 257 is already newer than 256 2024-08-17T16:56:10.852 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 257 (ObjNum 161 snap 42 seq_num 161) dirty exists 2024-08-17T16:56:10.852 INFO:tasks.rados.rados.0.smithi063.stdout:335: left oid 48 (ObjNum 161 snap 42 seq_num 161) 2024-08-17T16:56:11.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:10 smithi063 ceph-mon[31063]: pgmap v211: 89 pgs: 89 active+clean; 152 MiB data, 506 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:11.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:10 smithi063 ceph-mon[31063]: osdmap e154: 8 total, 8 up, 8 in 2024-08-17T16:56:11.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:10 smithi063 ceph-mon[42680]: pgmap v211: 89 pgs: 89 active+clean; 152 MiB data, 506 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:11.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:10 smithi063 ceph-mon[42680]: osdmap e154: 8 total, 8 up, 8 in 2024-08-17T16:56:11.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:10 smithi162 ceph-mon[34633]: pgmap v211: 89 pgs: 89 active+clean; 152 MiB data, 506 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:11.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:10 smithi162 ceph-mon[34633]: osdmap e154: 8 total, 8 up, 8 in 2024-08-17T16:56:11.450 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T16:56:11.835 INFO:tasks.rados.rados.0.smithi063.stdout:334: done (2 left) 2024-08-17T16:56:11.835 INFO:tasks.rados.rados.0.smithi063.stdout:335: done (1 left) 2024-08-17T16:56:11.835 INFO:tasks.rados.rados.0.smithi063.stdout:336: done (0 left) 2024-08-17T16:56:11.835 INFO:tasks.rados.rados.0.smithi063.stdout:337: read oid 36 snap -1 2024-08-17T16:56:11.836 INFO:tasks.rados.rados.0.smithi063.stdout:337: expect (ObjNum 134 snap 28 seq_num 134) 2024-08-17T16:56:11.836 INFO:tasks.rados.rados.0.smithi063.stdout:338: delete oid 35 current snap is 43 2024-08-17T16:56:11.841 INFO:tasks.rados.rados.0.smithi063.stdout:338: done (1 left) 2024-08-17T16:56:11.841 INFO:tasks.rados.rados.0.smithi063.stdout:339: rollback oid 35 current snap is 43 2024-08-17T16:56:11.841 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 35 to 35 2024-08-17T16:56:11.841 INFO:tasks.rados.rados.0.smithi063.stdout:337: expect (ObjNum 134 snap 28 seq_num 134) 2024-08-17T16:56:11.915 INFO:tasks.rados.rados.0.smithi063.stdout:339: finishing rollback tid 1 to smithi06387500-35 2024-08-17T16:56:11.916 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 0 (ObjNum 129 snap 0 seq_num 1806787544) dirty dne 2024-08-17T16:56:11.916 INFO:tasks.rados.rados.0.smithi063.stdout:337: done (1 left) 2024-08-17T16:56:11.916 INFO:tasks.rados.rados.0.smithi063.stdout:339: done (0 left) 2024-08-17T16:56:11.916 INFO:tasks.rados.rados.0.smithi063.stdout:340: write oid 4 current snap is 43 2024-08-17T16:56:11.916 INFO:tasks.rados.rados.0.smithi063.stdout:340: seq_num 163 ranges {739541=709000,2121859=426376} 2024-08-17T16:56:11.925 INFO:tasks.rados.rados.0.smithi063.stdout:340: writing smithi06387500-4 from 739541 to 1448541 tid 1 2024-08-17T16:56:11.932 INFO:tasks.rados.rados.0.smithi063.stdout:340: writing smithi06387500-4 from 2121859 to 2548235 tid 2 2024-08-17T16:56:11.933 INFO:tasks.rados.rados.0.smithi063.stdout:341: rollback oid 37 current snap is 43 2024-08-17T16:56:11.933 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 40 2024-08-17T16:56:11.933 INFO:tasks.rados.rados.0.smithi063.stdout:340: finishing write tid 1 to smithi06387500-4 2024-08-17T16:56:11.933 INFO:tasks.rados.rados.0.smithi063.stdout:342: delete oid 24 current snap is 43 2024-08-17T16:56:11.934 INFO:tasks.rados.rados.0.smithi063.stdout:342: done (2 left) 2024-08-17T16:56:11.934 INFO:tasks.rados.rados.0.smithi063.stdout:343: delete oid 42 current snap is 43 2024-08-17T16:56:11.935 INFO:tasks.rados.rados.0.smithi063.stdout:343: done (2 left) 2024-08-17T16:56:11.935 INFO:tasks.rados.rados.0.smithi063.stdout:344: write oid 18 current snap is 43 2024-08-17T16:56:11.936 INFO:tasks.rados.rados.0.smithi063.stdout:344: seq_num 166 ranges {789962=534747,1780952=690056} 2024-08-17T16:56:11.942 INFO:tasks.rados.rados.0.smithi063.stdout:344: writing smithi06387500-18 from 789962 to 1324709 tid 1 2024-08-17T16:56:11.949 INFO:tasks.rados.rados.0.smithi063.stdout:344: writing smithi06387500-18 from 1780952 to 2471008 tid 2 2024-08-17T16:56:11.949 INFO:tasks.rados.rados.0.smithi063.stdout:345: read oid 23 snap -1 2024-08-17T16:56:11.949 INFO:tasks.rados.rados.0.smithi063.stdout:345: expect (ObjNum 125 snap 23 seq_num 125) 2024-08-17T16:56:11.949 INFO:tasks.rados.rados.0.smithi063.stdout:346: write oid 32 current snap is 43 2024-08-17T16:56:11.949 INFO:tasks.rados.rados.0.smithi063.stdout:340: finishing write tid 2 to smithi06387500-4 2024-08-17T16:56:11.950 INFO:tasks.rados.rados.0.smithi063.stdout:346: seq_num 167 ranges {406769=609996,1439434=433714,2303522=503247} 2024-08-17T16:56:11.955 INFO:tasks.rados.rados.0.smithi063.stdout:346: writing smithi06387500-32 from 406769 to 1016765 tid 1 2024-08-17T16:56:11.959 INFO:tasks.rados.rados.0.smithi063.stdout:346: writing smithi06387500-32 from 1439434 to 1873148 tid 2 2024-08-17T16:56:11.965 INFO:tasks.rados.rados.0.smithi063.stdout:346: writing smithi06387500-32 from 2303522 to 2806769 tid 3 2024-08-17T16:56:11.966 INFO:tasks.rados.rados.0.smithi063.stdout:347: rollback oid 7 current snap is 43 2024-08-17T16:56:11.966 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 40 2024-08-17T16:56:11.966 INFO:tasks.rados.rados.0.smithi063.stdout:340: finishing write tid 3 to smithi06387500-4 2024-08-17T16:56:11.966 INFO:tasks.rados.rados.0.smithi063.stdout:340: finishing write tid 4 to smithi06387500-4 2024-08-17T16:56:11.966 INFO:tasks.rados.rados.0.smithi063.stdout:340: oid 4 updating version 0 to 261 2024-08-17T16:56:11.966 INFO:tasks.rados.rados.0.smithi063.stdout:340: oid 4 version 261 is already newer than 260 2024-08-17T16:56:11.966 INFO:tasks.rados.rados.0.smithi063.stdout:340: oid 4 version 261 is already newer than 259 2024-08-17T16:56:11.966 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 261 (ObjNum 163 snap 43 seq_num 163) dirty exists 2024-08-17T16:56:11.966 INFO:tasks.rados.rados.0.smithi063.stdout:340: left oid 4 (ObjNum 163 snap 43 seq_num 163) 2024-08-17T16:56:11.966 INFO:tasks.rados.rados.0.smithi063.stdout:341: finishing rollback tid 0 to smithi06387500-37 2024-08-17T16:56:11.966 INFO:tasks.rados.rados.0.smithi063.stdout:341: finishing rollback tid 1 to smithi06387500-37 2024-08-17T16:56:11.967 INFO:tasks.rados.rados.0.smithi063.stdout:341: finishing rollback tid 2 to smithi06387500-37 2024-08-17T16:56:11.967 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 265 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-08-17T16:56:11.967 INFO:tasks.rados.rados.0.smithi063.stdout:344: finishing write tid 1 to smithi06387500-18 2024-08-17T16:56:11.967 INFO:tasks.rados.rados.0.smithi063.stdout:340: done (5 left) 2024-08-17T16:56:11.967 INFO:tasks.rados.rados.0.smithi063.stdout:341: done (4 left) 2024-08-17T16:56:11.967 INFO:tasks.rados.rados.0.smithi063.stdout:348: snap_create 2024-08-17T16:56:11.967 INFO:tasks.rados.rados.0.smithi063.stdout:344: finishing write tid 2 to smithi06387500-18 2024-08-17T16:56:11.967 INFO:tasks.rados.rados.0.smithi063.stdout:344: finishing write tid 3 to smithi06387500-18 2024-08-17T16:56:11.967 INFO:tasks.rados.rados.0.smithi063.stdout:344: finishing write tid 4 to smithi06387500-18 2024-08-17T16:56:11.967 INFO:tasks.rados.rados.0.smithi063.stdout:344: oid 18 updating version 0 to 80 2024-08-17T16:56:11.967 INFO:tasks.rados.rados.0.smithi063.stdout:344: oid 18 version 80 is already newer than 79 2024-08-17T16:56:11.967 INFO:tasks.rados.rados.0.smithi063.stdout:344: oid 18 version 80 is already newer than 78 2024-08-17T16:56:11.967 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 80 (ObjNum 166 snap 43 seq_num 166) dirty exists 2024-08-17T16:56:11.968 INFO:tasks.rados.rados.0.smithi063.stdout:344: left oid 18 (ObjNum 166 snap 43 seq_num 166) 2024-08-17T16:56:11.968 INFO:tasks.rados.rados.0.smithi063.stdout:345: expect (ObjNum 125 snap 23 seq_num 125) 2024-08-17T16:56:12.035 INFO:tasks.rados.rados.0.smithi063.stdout:346: finishing write tid 1 to smithi06387500-32 2024-08-17T16:56:12.035 INFO:tasks.rados.rados.0.smithi063.stdout:346: finishing write tid 2 to smithi06387500-32 2024-08-17T16:56:12.035 INFO:tasks.rados.rados.0.smithi063.stdout:347: finishing rollback tid 0 to smithi06387500-7 2024-08-17T16:56:12.035 INFO:tasks.rados.rados.0.smithi063.stdout:347: finishing rollback tid 1 to smithi06387500-7 2024-08-17T16:56:12.036 INFO:tasks.rados.rados.0.smithi063.stdout:347: finishing rollback tid 2 to smithi06387500-7 2024-08-17T16:56:12.036 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 149 (ObjNum 156 snap 39 seq_num 156) dirty exists 2024-08-17T16:56:12.036 INFO:tasks.rados.rados.0.smithi063.stdout:346: finishing write tid 3 to smithi06387500-32 2024-08-17T16:56:12.036 INFO:tasks.rados.rados.0.smithi063.stdout:346: finishing write tid 4 to smithi06387500-32 2024-08-17T16:56:12.036 INFO:tasks.rados.rados.0.smithi063.stdout:346: finishing write tid 5 to smithi06387500-32 2024-08-17T16:56:12.036 INFO:tasks.rados.rados.0.smithi063.stdout:346: oid 32 updating version 0 to 55 2024-08-17T16:56:12.036 INFO:tasks.rados.rados.0.smithi063.stdout:346: oid 32 updating version 55 to 57 2024-08-17T16:56:12.036 INFO:tasks.rados.rados.0.smithi063.stdout:346: oid 32 version 57 is already newer than 54 2024-08-17T16:56:12.036 INFO:tasks.rados.rados.0.smithi063.stdout:346: oid 32 version 57 is already newer than 56 2024-08-17T16:56:12.036 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 57 (ObjNum 167 snap 43 seq_num 167) dirty exists 2024-08-17T16:56:12.036 INFO:tasks.rados.rados.0.smithi063.stdout:346: left oid 32 (ObjNum 167 snap 43 seq_num 167) 2024-08-17T16:56:12.037 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T16:56:12.037 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (2m) 30s ago 3m 26.9M - 0.25.0 c8568f914cd2 87286b24f231 2024-08-17T16:56:12.037 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (3m) 32s ago 3m 86.7M - 9.4.7 954c08fa6188 fc8d62329012 2024-08-17T16:56:12.037 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (2m) 30s ago 2m 15.2M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T16:56:12.037 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (2m) 32s ago 2m 15.5M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T16:56:12.037 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,8765 running (7m) 32s ago 7m 439M - 18.2.4-719-gced649f8 c870753d67c2 4add2ba1a53e 2024-08-17T16:56:12.037 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:9283,8765 running (9m) 30s ago 9m 490M - 18.2.4-719-gced649f8 c870753d67c2 ba4a2894b308 2024-08-17T16:56:12.037 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (9m) 30s ago 9m 46.6M 2048M 18.2.4-719-gced649f8 c870753d67c2 0c71ddd7ada6 2024-08-17T16:56:12.037 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (7m) 32s ago 7m 38.1M 2048M 18.2.4-719-gced649f8 c870753d67c2 d4ecbe65a0c8 2024-08-17T16:56:12.037 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (7m) 30s ago 7m 38.2M 2048M 18.2.4-719-gced649f8 c870753d67c2 b3f2c9339c6b 2024-08-17T16:56:12.037 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 running (3m) 30s ago 3m 15.5M - 1.5.0 0da6a335fe13 8958df80ad24 2024-08-17T16:56:12.038 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (3m) 32s ago 3m 13.1M - 1.5.0 0da6a335fe13 b1691104d8ac 2024-08-17T16:56:12.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (7m) 30s ago 7m 66.0M 4016M 18.2.4-719-gced649f8 c870753d67c2 f3b910ed49b8 2024-08-17T16:56:12.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (6m) 30s ago 6m 51.9M 4016M 18.2.4-719-gced649f8 c870753d67c2 5d6fdb6000ad 2024-08-17T16:56:12.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (6m) 30s ago 6m 62.5M 4016M 18.2.4-719-gced649f8 c870753d67c2 a3cbe8edb49a 2024-08-17T16:56:12.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (6m) 30s ago 6m 59.7M 4016M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T16:56:12.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (5m) 32s ago 5m 65.5M 4016M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T16:56:12.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (5m) 32s ago 5m 61.6M 4016M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T16:56:12.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (4m) 32s ago 4m 50.0M 4016M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T16:56:12.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (4m) 32s ago 4m 57.9M 4016M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T16:56:12.038 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (2m) 32s ago 3m 44.9M - 2.43.0 a07b618ecd1d 015db799d7d4 2024-08-17T16:56:12.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:11 smithi063 ceph-mon[42680]: from='client.14859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:12.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:11 smithi063 ceph-mon[42680]: osdmap e155: 8 total, 8 up, 8 in 2024-08-17T16:56:12.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:11 smithi063 ceph-mon[31063]: from='client.14859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:12.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:11 smithi063 ceph-mon[31063]: osdmap e155: 8 total, 8 up, 8 in 2024-08-17T16:56:12.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:11 smithi162 ceph-mon[34633]: from='client.14859 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:12.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:11 smithi162 ceph-mon[34633]: osdmap e155: 8 total, 8 up, 8 in 2024-08-17T16:56:12.366 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:56:12.366 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T16:56:12.366 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T16:56:12.366 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:56:12.366 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T16:56:12.367 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T16:56:12.367 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:56:12.367 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T16:56:12.367 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T16:56:12.367 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:56:12.367 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T16:56:12.367 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T16:56:12.367 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:56:12.367 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T16:56:12.367 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 15 2024-08-17T16:56:12.367 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T16:56:12.367 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:56:12.648 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:56:12.648 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T16:56:12.648 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-08-17T16:56:12.648 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-17T16:56:12.648 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-08-17T16:56:12.649 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "0/2 daemons upgraded", 2024-08-17T16:56:12.649 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d image on host smithi162", 2024-08-17T16:56:12.649 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T16:56:12.649 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:56:12.893 INFO:tasks.rados.rados.0.smithi063.stdout:344: done (4 left) 2024-08-17T16:56:12.893 INFO:tasks.rados.rados.0.smithi063.stdout:345: done (3 left) 2024-08-17T16:56:12.893 INFO:tasks.rados.rados.0.smithi063.stdout:346: done (2 left) 2024-08-17T16:56:12.893 INFO:tasks.rados.rados.0.smithi063.stdout:347: done (1 left) 2024-08-17T16:56:12.893 INFO:tasks.rados.rados.0.smithi063.stdout:348: done (0 left) 2024-08-17T16:56:12.894 INFO:tasks.rados.rados.0.smithi063.stdout:349: write oid 13 current snap is 44 2024-08-17T16:56:12.894 INFO:tasks.rados.rados.0.smithi063.stdout:349: seq_num 168 ranges {423576=401598,1497916=693725} 2024-08-17T16:56:12.897 INFO:tasks.rados.rados.0.smithi063.stdout:349: writing smithi06387500-13 from 423576 to 825174 tid 1 2024-08-17T16:56:12.905 INFO:tasks.rados.rados.0.smithi063.stdout:349: writing smithi06387500-13 from 1497916 to 2191641 tid 2 2024-08-17T16:56:12.905 INFO:tasks.rados.rados.0.smithi063.stdout:350: write oid 44 current snap is 44 2024-08-17T16:56:12.905 INFO:tasks.rados.rados.0.smithi063.stdout:350: seq_num 169 ranges {440383=476847,1556398=437383,2474721=365662} 2024-08-17T16:56:12.910 INFO:tasks.rados.rados.0.smithi063.stdout:350: writing smithi06387500-44 from 440383 to 917230 tid 1 2024-08-17T16:56:12.916 INFO:tasks.rados.rados.0.smithi063.stdout:350: writing smithi06387500-44 from 1556398 to 1993781 tid 2 2024-08-17T16:56:12.920 INFO:tasks.rados.rados.0.smithi063.stdout:350: writing smithi06387500-44 from 2474721 to 2840383 tid 3 2024-08-17T16:56:12.921 INFO:tasks.rados.rados.0.smithi063.stdout:351: write oid 21 current snap is 44 2024-08-17T16:56:12.921 INFO:tasks.rados.rados.0.smithi063.stdout:351: seq_num 170 ranges {457190=552096,1614880=581041,2785791=71399} 2024-08-17T16:56:12.926 INFO:tasks.rados.rados.0.smithi063.stdout:351: writing smithi06387500-21 from 457190 to 1009286 tid 1 2024-08-17T16:56:12.932 INFO:tasks.rados.rados.0.smithi063.stdout:351: writing smithi06387500-21 from 1614880 to 2195921 tid 2 2024-08-17T16:56:12.936 INFO:tasks.rados.rados.0.smithi063.stdout:351: writing smithi06387500-21 from 2785791 to 2857190 tid 3 2024-08-17T16:56:12.936 INFO:tasks.rados.rados.0.smithi063.stdout:352: write oid 14 current snap is 44 2024-08-17T16:56:12.936 INFO:tasks.rados.rados.0.smithi063.stdout:349: finishing write tid 1 to smithi06387500-13 2024-08-17T16:56:12.936 INFO:tasks.rados.rados.0.smithi063.stdout:349: finishing write tid 2 to smithi06387500-13 2024-08-17T16:56:12.937 INFO:tasks.rados.rados.0.smithi063.stdout:349: finishing write tid 3 to smithi06387500-13 2024-08-17T16:56:12.937 INFO:tasks.rados.rados.0.smithi063.stdout:352: seq_num 171 ranges {473997=627345,1673362=441052,2529567=344430} 2024-08-17T16:56:12.942 INFO:tasks.rados.rados.0.smithi063.stdout:352: writing smithi06387500-14 from 473997 to 1101342 tid 1 2024-08-17T16:56:12.948 INFO:tasks.rados.rados.0.smithi063.stdout:352: writing smithi06387500-14 from 1673362 to 2114414 tid 2 2024-08-17T16:56:12.952 INFO:tasks.rados.rados.0.smithi063.stdout:352: writing smithi06387500-14 from 2529567 to 2873997 tid 3 2024-08-17T16:56:12.953 INFO:tasks.rados.rados.0.smithi063.stdout:353: snap_create 2024-08-17T16:56:12.953 INFO:tasks.rados.rados.0.smithi063.stdout:349: finishing write tid 4 to smithi06387500-13 2024-08-17T16:56:12.953 INFO:tasks.rados.rados.0.smithi063.stdout:349: oid 13 updating version 0 to 123 2024-08-17T16:56:12.953 INFO:tasks.rados.rados.0.smithi063.stdout:349: oid 13 updating version 123 to 124 2024-08-17T16:56:12.953 INFO:tasks.rados.rados.0.smithi063.stdout:349: oid 13 version 124 is already newer than 122 2024-08-17T16:56:12.953 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 124 (ObjNum 168 snap 44 seq_num 168) dirty exists 2024-08-17T16:56:12.953 INFO:tasks.rados.rados.0.smithi063.stdout:349: left oid 13 (ObjNum 168 snap 44 seq_num 168) 2024-08-17T16:56:12.953 INFO:tasks.rados.rados.0.smithi063.stdout:350: finishing write tid 1 to smithi06387500-44 2024-08-17T16:56:12.953 INFO:tasks.rados.rados.0.smithi063.stdout:350: finishing write tid 2 to smithi06387500-44 2024-08-17T16:56:12.953 INFO:tasks.rados.rados.0.smithi063.stdout:350: finishing write tid 3 to smithi06387500-44 2024-08-17T16:56:12.953 INFO:tasks.rados.rados.0.smithi063.stdout:350: finishing write tid 4 to smithi06387500-44 2024-08-17T16:56:12.953 INFO:tasks.rados.rados.0.smithi063.stdout:350: finishing write tid 5 to smithi06387500-44 2024-08-17T16:56:12.953 INFO:tasks.rados.rados.0.smithi063.stdout:350: oid 44 updating version 0 to 52 2024-08-17T16:56:12.954 INFO:tasks.rados.rados.0.smithi063.stdout:350: oid 44 updating version 52 to 53 2024-08-17T16:56:12.954 INFO:tasks.rados.rados.0.smithi063.stdout:350: oid 44 version 53 is already newer than 51 2024-08-17T16:56:12.954 INFO:tasks.rados.rados.0.smithi063.stdout:350: oid 44 version 53 is already newer than 50 2024-08-17T16:56:12.954 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 53 (ObjNum 169 snap 44 seq_num 169) dirty exists 2024-08-17T16:56:12.954 INFO:tasks.rados.rados.0.smithi063.stdout:350: left oid 44 (ObjNum 169 snap 44 seq_num 169) 2024-08-17T16:56:12.954 INFO:tasks.rados.rados.0.smithi063.stdout:351: finishing write tid 1 to smithi06387500-21 2024-08-17T16:56:12.954 INFO:tasks.rados.rados.0.smithi063.stdout:351: finishing write tid 2 to smithi06387500-21 2024-08-17T16:56:12.954 INFO:tasks.rados.rados.0.smithi063.stdout:351: finishing write tid 3 to smithi06387500-21 2024-08-17T16:56:12.954 INFO:tasks.rados.rados.0.smithi063.stdout:351: finishing write tid 4 to smithi06387500-21 2024-08-17T16:56:12.954 INFO:tasks.rados.rados.0.smithi063.stdout:351: finishing write tid 5 to smithi06387500-21 2024-08-17T16:56:12.954 INFO:tasks.rados.rados.0.smithi063.stdout:351: oid 21 updating version 0 to 182 2024-08-17T16:56:12.954 INFO:tasks.rados.rados.0.smithi063.stdout:351: oid 21 updating version 182 to 183 2024-08-17T16:56:12.955 INFO:tasks.rados.rados.0.smithi063.stdout:351: oid 21 version 183 is already newer than 181 2024-08-17T16:56:12.955 INFO:tasks.rados.rados.0.smithi063.stdout:351: oid 21 version 183 is already newer than 180 2024-08-17T16:56:12.955 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 183 (ObjNum 170 snap 44 seq_num 170) dirty exists 2024-08-17T16:56:12.955 INFO:tasks.rados.rados.0.smithi063.stdout:351: left oid 21 (ObjNum 170 snap 44 seq_num 170) 2024-08-17T16:56:12.955 INFO:tasks.rados.rados.0.smithi063.stdout:352: finishing write tid 1 to smithi06387500-14 2024-08-17T16:56:12.955 INFO:tasks.rados.rados.0.smithi063.stdout:352: finishing write tid 2 to smithi06387500-14 2024-08-17T16:56:12.956 INFO:tasks.rados.rados.0.smithi063.stdout:352: finishing write tid 3 to smithi06387500-14 2024-08-17T16:56:12.956 INFO:tasks.rados.rados.0.smithi063.stdout:352: finishing write tid 4 to smithi06387500-14 2024-08-17T16:56:12.956 INFO:tasks.rados.rados.0.smithi063.stdout:352: finishing write tid 5 to smithi06387500-14 2024-08-17T16:56:12.956 INFO:tasks.rados.rados.0.smithi063.stdout:352: oid 14 updating version 0 to 268 2024-08-17T16:56:12.956 INFO:tasks.rados.rados.0.smithi063.stdout:352: oid 14 updating version 268 to 269 2024-08-17T16:56:12.956 INFO:tasks.rados.rados.0.smithi063.stdout:352: oid 14 version 269 is already newer than 267 2024-08-17T16:56:12.956 INFO:tasks.rados.rados.0.smithi063.stdout:352: oid 14 updating version 269 to 270 2024-08-17T16:56:12.956 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 270 (ObjNum 171 snap 44 seq_num 171) dirty exists 2024-08-17T16:56:12.956 INFO:tasks.rados.rados.0.smithi063.stdout:352: left oid 14 (ObjNum 171 snap 44 seq_num 171) 2024-08-17T16:56:13.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:12 smithi063 ceph-mon[42680]: from='client.14865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:13.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:12 smithi063 ceph-mon[42680]: from='client.24764 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:13.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:12 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/4108951741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:56:13.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:12 smithi063 ceph-mon[42680]: pgmap v214: 89 pgs: 89 active+clean; 158 MiB data, 522 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T16:56:13.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:12 smithi063 ceph-mon[31063]: from='client.14865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:13.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:12 smithi063 ceph-mon[31063]: from='client.24764 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:13.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:12 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/4108951741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:56:13.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:12 smithi063 ceph-mon[31063]: pgmap v214: 89 pgs: 89 active+clean; 158 MiB data, 522 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T16:56:13.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:12 smithi162 ceph-mon[34633]: from='client.14865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:13.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:12 smithi162 ceph-mon[34633]: from='client.24764 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:13.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:12 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/4108951741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:56:13.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:12 smithi162 ceph-mon[34633]: pgmap v214: 89 pgs: 89 active+clean; 158 MiB data, 522 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T16:56:13.898 INFO:tasks.rados.rados.0.smithi063.stdout:349: done (4 left) 2024-08-17T16:56:13.898 INFO:tasks.rados.rados.0.smithi063.stdout:350: done (3 left) 2024-08-17T16:56:13.898 INFO:tasks.rados.rados.0.smithi063.stdout:351: done (2 left) 2024-08-17T16:56:13.899 INFO:tasks.rados.rados.0.smithi063.stdout:352: done (1 left) 2024-08-17T16:56:13.899 INFO:tasks.rados.rados.0.smithi063.stdout:353: done (0 left) 2024-08-17T16:56:13.899 INFO:tasks.rados.rados.0.smithi063.stdout:354: rollback oid 19 current snap is 45 2024-08-17T16:56:13.899 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 19 to 38 2024-08-17T16:56:13.899 INFO:tasks.rados.rados.0.smithi063.stdout:355: read oid 36 snap -1 2024-08-17T16:56:13.899 INFO:tasks.rados.rados.0.smithi063.stdout:355: expect (ObjNum 134 snap 28 seq_num 134) 2024-08-17T16:56:13.899 INFO:tasks.rados.rados.0.smithi063.stdout:356: write oid 27 current snap is 45 2024-08-17T16:56:13.899 INFO:tasks.rados.rados.0.smithi063.stdout:356: seq_num 172 ranges {490804=702594,1615491=584710,2724284=166520} 2024-08-17T16:56:13.905 INFO:tasks.rados.rados.0.smithi063.stdout:356: writing smithi06387500-27 from 490804 to 1193398 tid 1 2024-08-17T16:56:13.911 INFO:tasks.rados.rados.0.smithi063.stdout:356: writing smithi06387500-27 from 1615491 to 2200201 tid 2 2024-08-17T16:56:13.915 INFO:tasks.rados.rados.0.smithi063.stdout:356: writing smithi06387500-27 from 2724284 to 2890804 tid 3 2024-08-17T16:56:13.915 INFO:tasks.rados.rados.0.smithi063.stdout:357: snap_create 2024-08-17T16:56:13.915 INFO:tasks.rados.rados.0.smithi063.stdout:354: finishing rollback tid 0 to smithi06387500-19 2024-08-17T16:56:13.915 INFO:tasks.rados.rados.0.smithi063.stdout:354: finishing rollback tid 1 to smithi06387500-19 2024-08-17T16:56:13.915 INFO:tasks.rados.rados.0.smithi063.stdout:354: finishing rollback tid 2 to smithi06387500-19 2024-08-17T16:56:13.916 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 187 (ObjNum 140 snap 31 seq_num 140) dirty exists 2024-08-17T16:56:13.916 INFO:tasks.rados.rados.0.smithi063.stdout:355: expect (ObjNum 134 snap 28 seq_num 134) 2024-08-17T16:56:13.989 INFO:tasks.rados.rados.0.smithi063.stdout:356: finishing write tid 1 to smithi06387500-27 2024-08-17T16:56:13.989 INFO:tasks.rados.rados.0.smithi063.stdout:356: finishing write tid 2 to smithi06387500-27 2024-08-17T16:56:13.989 INFO:tasks.rados.rados.0.smithi063.stdout:356: finishing write tid 3 to smithi06387500-27 2024-08-17T16:56:13.989 INFO:tasks.rados.rados.0.smithi063.stdout:356: finishing write tid 4 to smithi06387500-27 2024-08-17T16:56:13.989 INFO:tasks.rados.rados.0.smithi063.stdout:356: finishing write tid 5 to smithi06387500-27 2024-08-17T16:56:13.989 INFO:tasks.rados.rados.0.smithi063.stdout:356: oid 27 updating version 0 to 32 2024-08-17T16:56:13.989 INFO:tasks.rados.rados.0.smithi063.stdout:356: oid 27 version 32 is already newer than 30 2024-08-17T16:56:13.990 INFO:tasks.rados.rados.0.smithi063.stdout:356: oid 27 version 32 is already newer than 29 2024-08-17T16:56:13.990 INFO:tasks.rados.rados.0.smithi063.stdout:356: oid 27 version 32 is already newer than 31 2024-08-17T16:56:13.990 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 32 (ObjNum 172 snap 45 seq_num 172) dirty exists 2024-08-17T16:56:13.990 INFO:tasks.rados.rados.0.smithi063.stdout:356: left oid 27 (ObjNum 172 snap 45 seq_num 172) 2024-08-17T16:56:14.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:13 smithi063 ceph-mon[42680]: from='client.14883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:14.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:13 smithi063 ceph-mon[42680]: osdmap e156: 8 total, 8 up, 8 in 2024-08-17T16:56:14.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:13 smithi063 ceph-mon[31063]: from='client.14883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:14.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:13 smithi063 ceph-mon[31063]: osdmap e156: 8 total, 8 up, 8 in 2024-08-17T16:56:14.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:13 smithi162 ceph-mon[34633]: from='client.14883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:14.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:13 smithi162 ceph-mon[34633]: osdmap e156: 8 total, 8 up, 8 in 2024-08-17T16:56:14.833 INFO:tasks.rados.rados.0.smithi063.stdout:354: done (3 left) 2024-08-17T16:56:14.833 INFO:tasks.rados.rados.0.smithi063.stdout:355: done (2 left) 2024-08-17T16:56:14.833 INFO:tasks.rados.rados.0.smithi063.stdout:356: done (1 left) 2024-08-17T16:56:14.833 INFO:tasks.rados.rados.0.smithi063.stdout:357: done (0 left) 2024-08-17T16:56:14.833 INFO:tasks.rados.rados.0.smithi063.stdout:358: write oid 28 current snap is 46 2024-08-17T16:56:14.834 INFO:tasks.rados.rados.0.smithi063.stdout:358: seq_num 173 ranges {507611=777843,2073973=444721} 2024-08-17T16:56:14.840 INFO:tasks.rados.rados.0.smithi063.stdout:358: writing smithi06387500-28 from 507611 to 1285454 tid 1 2024-08-17T16:56:14.847 INFO:tasks.rados.rados.0.smithi063.stdout:358: writing smithi06387500-28 from 2073973 to 2518694 tid 2 2024-08-17T16:56:14.847 INFO:tasks.rados.rados.0.smithi063.stdout:359: snap_create 2024-08-17T16:56:14.847 INFO:tasks.rados.rados.0.smithi063.stdout:358: finishing write tid 1 to smithi06387500-28 2024-08-17T16:56:14.849 INFO:tasks.rados.rados.0.smithi063.stdout:358: finishing write tid 2 to smithi06387500-28 2024-08-17T16:56:14.849 INFO:tasks.rados.rados.0.smithi063.stdout:358: finishing write tid 3 to smithi06387500-28 2024-08-17T16:56:14.850 INFO:tasks.rados.rados.0.smithi063.stdout:358: finishing write tid 4 to smithi06387500-28 2024-08-17T16:56:14.850 INFO:tasks.rados.rados.0.smithi063.stdout:358: oid 28 updating version 0 to 110 2024-08-17T16:56:14.850 INFO:tasks.rados.rados.0.smithi063.stdout:358: oid 28 version 110 is already newer than 108 2024-08-17T16:56:14.850 INFO:tasks.rados.rados.0.smithi063.stdout:358: oid 28 version 110 is already newer than 109 2024-08-17T16:56:14.850 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 110 (ObjNum 173 snap 46 seq_num 173) dirty exists 2024-08-17T16:56:14.850 INFO:tasks.rados.rados.0.smithi063.stdout:358: left oid 28 (ObjNum 173 snap 46 seq_num 173) 2024-08-17T16:56:15.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:14 smithi063 ceph-mon[42680]: osdmap e157: 8 total, 8 up, 8 in 2024-08-17T16:56:15.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:14 smithi063 ceph-mon[42680]: pgmap v217: 89 pgs: 89 active+clean; 158 MiB data, 522 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T16:56:15.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:14 smithi063 ceph-mon[42680]: osdmap e158: 8 total, 8 up, 8 in 2024-08-17T16:56:15.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:14 smithi063 ceph-mon[31063]: osdmap e157: 8 total, 8 up, 8 in 2024-08-17T16:56:15.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:14 smithi063 ceph-mon[31063]: pgmap v217: 89 pgs: 89 active+clean; 158 MiB data, 522 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T16:56:15.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:14 smithi063 ceph-mon[31063]: osdmap e158: 8 total, 8 up, 8 in 2024-08-17T16:56:15.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:14 smithi162 ceph-mon[34633]: osdmap e157: 8 total, 8 up, 8 in 2024-08-17T16:56:15.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:14 smithi162 ceph-mon[34633]: pgmap v217: 89 pgs: 89 active+clean; 158 MiB data, 522 MiB used, 715 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T16:56:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:14 smithi162 ceph-mon[34633]: osdmap e158: 8 total, 8 up, 8 in 2024-08-17T16:56:15.839 INFO:tasks.rados.rados.0.smithi063.stdout:358: done (1 left) 2024-08-17T16:56:15.839 INFO:tasks.rados.rados.0.smithi063.stdout:359: done (0 left) 2024-08-17T16:56:15.839 INFO:tasks.rados.rados.0.smithi063.stdout:360: snap_remove snap 45 2024-08-17T16:56:16.849 INFO:tasks.rados.rados.0.smithi063.stdout:360: done (0 left) 2024-08-17T16:56:16.849 INFO:tasks.rados.rados.0.smithi063.stdout:361: read oid 32 snap -1 2024-08-17T16:56:16.849 INFO:tasks.rados.rados.0.smithi063.stdout:361: expect (ObjNum 167 snap 43 seq_num 167) 2024-08-17T16:56:16.850 INFO:tasks.rados.rados.0.smithi063.stdout:362: read oid 10 snap 43 2024-08-17T16:56:16.850 INFO:tasks.rados.rados.0.smithi063.stdout:362: expect (ObjNum 100 snap 16 seq_num 100) 2024-08-17T16:56:16.850 INFO:tasks.rados.rados.0.smithi063.stdout:363: read oid 26 snap -1 2024-08-17T16:56:16.850 INFO:tasks.rados.rados.0.smithi063.stdout:363: expect (ObjNum 148 snap 34 seq_num 148) 2024-08-17T16:56:16.850 INFO:tasks.rados.rados.0.smithi063.stdout:364: read oid 2 snap -1 2024-08-17T16:56:16.850 INFO:tasks.rados.rados.0.smithi063.stdout:364: expect deleted 2024-08-17T16:56:16.850 INFO:tasks.rados.rados.0.smithi063.stdout:365: delete oid 40 current snap is 47 2024-08-17T16:56:16.854 INFO:tasks.rados.rados.0.smithi063.stdout:361: expect (ObjNum 167 snap 43 seq_num 167) 2024-08-17T16:56:16.996 INFO:tasks.rados.rados.0.smithi063.stdout:362: expect (ObjNum 100 snap 16 seq_num 100) 2024-08-17T16:56:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:16 smithi162 ceph-mon[34633]: osdmap e159: 8 total, 8 up, 8 in 2024-08-17T16:56:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:16 smithi162 ceph-mon[34633]: pgmap v220: 89 pgs: 89 active+clean; 174 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.8 MiB/s wr, 20 op/s 2024-08-17T16:56:17.116 INFO:tasks.rados.rados.0.smithi063.stdout:363: expect (ObjNum 148 snap 34 seq_num 148) 2024-08-17T16:56:17.241 INFO:tasks.rados.rados.0.smithi063.stdout:361: done (4 left) 2024-08-17T16:56:17.242 INFO:tasks.rados.rados.0.smithi063.stdout:362: done (3 left) 2024-08-17T16:56:17.242 INFO:tasks.rados.rados.0.smithi063.stdout:363: done (2 left) 2024-08-17T16:56:17.242 INFO:tasks.rados.rados.0.smithi063.stdout:364: done (1 left) 2024-08-17T16:56:17.242 INFO:tasks.rados.rados.0.smithi063.stdout:365: done (0 left) 2024-08-17T16:56:17.242 INFO:tasks.rados.rados.0.smithi063.stdout:366: read oid 10 snap -1 2024-08-17T16:56:17.242 INFO:tasks.rados.rados.0.smithi063.stdout:366: expect (ObjNum 100 snap 16 seq_num 100) 2024-08-17T16:56:17.242 INFO:tasks.rados.rados.0.smithi063.stdout:367: rollback oid 8 current snap is 47 2024-08-17T16:56:17.242 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 8 to 38 2024-08-17T16:56:17.242 INFO:tasks.rados.rados.0.smithi063.stdout:368: rollback oid 40 current snap is 47 2024-08-17T16:56:17.242 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 40 to 43 2024-08-17T16:56:17.242 INFO:tasks.rados.rados.0.smithi063.stdout:369: write oid 50 current snap is 47 2024-08-17T16:56:17.242 INFO:tasks.rados.rados.0.smithi063.stdout:369: seq_num 175 ranges {541225=644694,1907290=448390} 2024-08-17T16:56:17.247 INFO:tasks.rados.rados.0.smithi063.stdout:369: writing smithi06387500-50 from 541225 to 1185919 tid 1 2024-08-17T16:56:17.254 INFO:tasks.rados.rados.0.smithi063.stdout:369: writing smithi06387500-50 from 1907290 to 2355680 tid 2 2024-08-17T16:56:17.254 INFO:tasks.rados.rados.0.smithi063.stdout:370: write oid 35 current snap is 47 2024-08-17T16:56:17.254 INFO:tasks.rados.rados.0.smithi063.stdout:370: seq_num 176 ranges {558032=719943,1849419=592048} 2024-08-17T16:56:17.261 INFO:tasks.rados.rados.0.smithi063.stdout:370: writing smithi06387500-35 from 558032 to 1277975 tid 1 2024-08-17T16:56:17.268 INFO:tasks.rados.rados.0.smithi063.stdout:370: writing smithi06387500-35 from 1849419 to 2441467 tid 2 2024-08-17T16:56:17.268 INFO:tasks.rados.rados.0.smithi063.stdout:367: finishing rollback tid 1 to smithi06387500-8 2024-08-17T16:56:17.268 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 75 (ObjNum 49 snap 0 seq_num 1786887280) dirty dne 2024-08-17T16:56:17.268 INFO:tasks.rados.rados.0.smithi063.stdout:366: expect (ObjNum 100 snap 16 seq_num 100) 2024-08-17T16:56:17.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:16 smithi063 ceph-mon[42680]: osdmap e159: 8 total, 8 up, 8 in 2024-08-17T16:56:17.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:16 smithi063 ceph-mon[42680]: pgmap v220: 89 pgs: 89 active+clean; 174 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.8 MiB/s wr, 20 op/s 2024-08-17T16:56:17.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:16 smithi063 ceph-mon[31063]: osdmap e159: 8 total, 8 up, 8 in 2024-08-17T16:56:17.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:16 smithi063 ceph-mon[31063]: pgmap v220: 89 pgs: 89 active+clean; 174 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.8 MiB/s wr, 20 op/s 2024-08-17T16:56:17.389 INFO:tasks.rados.rados.0.smithi063.stdout:368: finishing rollback tid 1 to smithi06387500-40 2024-08-17T16:56:17.389 INFO:tasks.rados.rados.0.smithi063.stdout:368: finishing rollback tid 2 to smithi06387500-40 2024-08-17T16:56:17.389 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 84 (ObjNum 158 snap 40 seq_num 158) dirty exists 2024-08-17T16:56:17.389 INFO:tasks.rados.rados.0.smithi063.stdout:369: finishing write tid 1 to smithi06387500-50 2024-08-17T16:56:17.389 INFO:tasks.rados.rados.0.smithi063.stdout:369: finishing write tid 2 to smithi06387500-50 2024-08-17T16:56:17.389 INFO:tasks.rados.rados.0.smithi063.stdout:369: finishing write tid 3 to smithi06387500-50 2024-08-17T16:56:17.389 INFO:tasks.rados.rados.0.smithi063.stdout:371: read oid 21 snap -1 2024-08-17T16:56:17.389 INFO:tasks.rados.rados.0.smithi063.stdout:371: expect (ObjNum 170 snap 44 seq_num 170) 2024-08-17T16:56:17.389 INFO:tasks.rados.rados.0.smithi063.stdout:366: done (5 left) 2024-08-17T16:56:17.390 INFO:tasks.rados.rados.0.smithi063.stdout:367: done (4 left) 2024-08-17T16:56:17.390 INFO:tasks.rados.rados.0.smithi063.stdout:368: done (3 left) 2024-08-17T16:56:17.390 INFO:tasks.rados.rados.0.smithi063.stdout:372: delete oid 15 current snap is 47 2024-08-17T16:56:17.390 INFO:tasks.rados.rados.0.smithi063.stdout:369: finishing write tid 4 to smithi06387500-50 2024-08-17T16:56:17.390 INFO:tasks.rados.rados.0.smithi063.stdout:369: oid 50 updating version 0 to 191 2024-08-17T16:56:17.390 INFO:tasks.rados.rados.0.smithi063.stdout:369: oid 50 version 191 is already newer than 190 2024-08-17T16:56:17.390 INFO:tasks.rados.rados.0.smithi063.stdout:369: oid 50 version 191 is already newer than 189 2024-08-17T16:56:17.390 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 191 (ObjNum 175 snap 47 seq_num 175) dirty exists 2024-08-17T16:56:17.390 INFO:tasks.rados.rados.0.smithi063.stdout:369: left oid 50 (ObjNum 175 snap 47 seq_num 175) 2024-08-17T16:56:17.390 INFO:tasks.rados.rados.0.smithi063.stdout:370: finishing write tid 1 to smithi06387500-35 2024-08-17T16:56:17.390 INFO:tasks.rados.rados.0.smithi063.stdout:370: finishing write tid 2 to smithi06387500-35 2024-08-17T16:56:17.390 INFO:tasks.rados.rados.0.smithi063.stdout:370: finishing write tid 3 to smithi06387500-35 2024-08-17T16:56:17.390 INFO:tasks.rados.rados.0.smithi063.stdout:370: finishing write tid 4 to smithi06387500-35 2024-08-17T16:56:17.390 INFO:tasks.rados.rados.0.smithi063.stdout:370: oid 35 updating version 0 to 272 2024-08-17T16:56:17.391 INFO:tasks.rados.rados.0.smithi063.stdout:370: oid 35 updating version 272 to 273 2024-08-17T16:56:17.391 INFO:tasks.rados.rados.0.smithi063.stdout:370: oid 35 version 273 is already newer than 271 2024-08-17T16:56:17.391 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 273 (ObjNum 176 snap 47 seq_num 176) dirty exists 2024-08-17T16:56:17.391 INFO:tasks.rados.rados.0.smithi063.stdout:370: left oid 35 (ObjNum 176 snap 47 seq_num 176) 2024-08-17T16:56:17.391 INFO:tasks.rados.rados.0.smithi063.stdout:369: done (3 left) 2024-08-17T16:56:17.391 INFO:tasks.rados.rados.0.smithi063.stdout:370: done (2 left) 2024-08-17T16:56:17.391 INFO:tasks.rados.rados.0.smithi063.stdout:372: done (1 left) 2024-08-17T16:56:17.392 INFO:tasks.rados.rados.0.smithi063.stdout:373: write oid 6 current snap is 47 2024-08-17T16:56:17.392 INFO:tasks.rados.rados.0.smithi063.stdout:373: seq_num 178 ranges {591646=470441,1566383=595717,2888822=102824} 2024-08-17T16:56:17.396 INFO:tasks.rados.rados.0.smithi063.stdout:373: writing smithi06387500-6 from 591646 to 1062087 tid 1 2024-08-17T16:56:17.402 INFO:tasks.rados.rados.0.smithi063.stdout:373: writing smithi06387500-6 from 1566383 to 2162100 tid 2 2024-08-17T16:56:17.407 INFO:tasks.rados.rados.0.smithi063.stdout:373: writing smithi06387500-6 from 2888822 to 2991646 tid 3 2024-08-17T16:56:17.407 INFO:tasks.rados.rados.0.smithi063.stdout:374: write oid 35 current snap is 47 2024-08-17T16:56:17.407 INFO:tasks.rados.rados.0.smithi063.stdout:374: seq_num 179 ranges {608453=545690,1624865=455728,2632598=375855} 2024-08-17T16:56:17.413 INFO:tasks.rados.rados.0.smithi063.stdout:374: writing smithi06387500-35 from 608453 to 1154143 tid 1 2024-08-17T16:56:17.418 INFO:tasks.rados.rados.0.smithi063.stdout:374: writing smithi06387500-35 from 1624865 to 2080593 tid 2 2024-08-17T16:56:17.423 INFO:tasks.rados.rados.0.smithi063.stdout:374: writing smithi06387500-35 from 2632598 to 3008453 tid 3 2024-08-17T16:56:17.423 INFO:tasks.rados.rados.0.smithi063.stdout:375: rollback oid 33 current snap is 47 2024-08-17T16:56:17.423 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 33 to 46 2024-08-17T16:56:17.423 INFO:tasks.rados.rados.0.smithi063.stdout:371: expect (ObjNum 170 snap 44 seq_num 170) 2024-08-17T16:56:17.520 INFO:tasks.rados.rados.0.smithi063.stdout:373: finishing write tid 1 to smithi06387500-6 2024-08-17T16:56:17.520 INFO:tasks.rados.rados.0.smithi063.stdout:373: finishing write tid 2 to smithi06387500-6 2024-08-17T16:56:17.520 INFO:tasks.rados.rados.0.smithi063.stdout:373: finishing write tid 3 to smithi06387500-6 2024-08-17T16:56:17.520 INFO:tasks.rados.rados.0.smithi063.stdout:373: finishing write tid 4 to smithi06387500-6 2024-08-17T16:56:17.520 INFO:tasks.rados.rados.0.smithi063.stdout:373: finishing write tid 5 to smithi06387500-6 2024-08-17T16:56:17.520 INFO:tasks.rados.rados.0.smithi063.stdout:373: oid 6 updating version 0 to 100 2024-08-17T16:56:17.520 INFO:tasks.rados.rados.0.smithi063.stdout:373: oid 6 updating version 100 to 101 2024-08-17T16:56:17.521 INFO:tasks.rados.rados.0.smithi063.stdout:373: oid 6 version 101 is already newer than 99 2024-08-17T16:56:17.521 INFO:tasks.rados.rados.0.smithi063.stdout:373: oid 6 version 101 is already newer than 98 2024-08-17T16:56:17.521 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 101 (ObjNum 178 snap 47 seq_num 178) dirty exists 2024-08-17T16:56:17.521 INFO:tasks.rados.rados.0.smithi063.stdout:373: left oid 6 (ObjNum 178 snap 47 seq_num 178) 2024-08-17T16:56:17.521 INFO:tasks.rados.rados.0.smithi063.stdout:374: finishing write tid 1 to smithi06387500-35 2024-08-17T16:56:17.521 INFO:tasks.rados.rados.0.smithi063.stdout:374: finishing write tid 2 to smithi06387500-35 2024-08-17T16:56:17.521 INFO:tasks.rados.rados.0.smithi063.stdout:375: finishing rollback tid 1 to smithi06387500-33 2024-08-17T16:56:17.521 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 93 (ObjNum 33 snap 0 seq_num 1786885368) dirty dne 2024-08-17T16:56:17.521 INFO:tasks.rados.rados.0.smithi063.stdout:374: finishing write tid 3 to smithi06387500-35 2024-08-17T16:56:17.521 INFO:tasks.rados.rados.0.smithi063.stdout:374: finishing write tid 4 to smithi06387500-35 2024-08-17T16:56:17.521 INFO:tasks.rados.rados.0.smithi063.stdout:374: finishing write tid 5 to smithi06387500-35 2024-08-17T16:56:17.521 INFO:tasks.rados.rados.0.smithi063.stdout:374: oid 35 updating version 0 to 276 2024-08-17T16:56:17.521 INFO:tasks.rados.rados.0.smithi063.stdout:374: oid 35 version 276 is already newer than 274 2024-08-17T16:56:17.522 INFO:tasks.rados.rados.0.smithi063.stdout:374: oid 35 updating version 276 to 277 2024-08-17T16:56:17.522 INFO:tasks.rados.rados.0.smithi063.stdout:374: oid 35 version 277 is already newer than 275 2024-08-17T16:56:17.522 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 277 (ObjNum 179 snap 47 seq_num 179) dirty exists 2024-08-17T16:56:17.522 INFO:tasks.rados.rados.0.smithi063.stdout:374: left oid 35 (ObjNum 179 snap 47 seq_num 179) 2024-08-17T16:56:17.522 INFO:tasks.rados.rados.0.smithi063.stdout:371: done (3 left) 2024-08-17T16:56:17.522 INFO:tasks.rados.rados.0.smithi063.stdout:373: done (2 left) 2024-08-17T16:56:17.522 INFO:tasks.rados.rados.0.smithi063.stdout:374: done (1 left) 2024-08-17T16:56:17.522 INFO:tasks.rados.rados.0.smithi063.stdout:375: done (0 left) 2024-08-17T16:56:17.522 INFO:tasks.rados.rados.0.smithi063.stdout:376: delete oid 13 current snap is 47 2024-08-17T16:56:17.522 INFO:tasks.rados.rados.0.smithi063.stdout:376: done (0 left) 2024-08-17T16:56:17.523 INFO:tasks.rados.rados.0.smithi063.stdout:377: snap_remove snap 37 2024-08-17T16:56:17.861 INFO:tasks.rados.rados.0.smithi063.stdout:377: done (0 left) 2024-08-17T16:56:17.861 INFO:tasks.rados.rados.0.smithi063.stdout:378: read oid 7 snap -1 2024-08-17T16:56:17.861 INFO:tasks.rados.rados.0.smithi063.stdout:378: expect (ObjNum 156 snap 39 seq_num 156) 2024-08-17T16:56:17.861 INFO:tasks.rados.rados.0.smithi063.stdout:379: delete oid 3 current snap is 47 2024-08-17T16:56:17.870 INFO:tasks.rados.rados.0.smithi063.stdout:378: expect (ObjNum 156 snap 39 seq_num 156) 2024-08-17T16:56:17.992 INFO:tasks.rados.rados.0.smithi063.stdout:378: done (1 left) 2024-08-17T16:56:17.992 INFO:tasks.rados.rados.0.smithi063.stdout:379: done (0 left) 2024-08-17T16:56:17.992 INFO:tasks.rados.rados.0.smithi063.stdout:380: snap_create 2024-08-17T16:56:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:17 smithi162 ceph-mon[34633]: osdmap e160: 8 total, 8 up, 8 in 2024-08-17T16:56:18.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:17 smithi063 ceph-mon[42680]: osdmap e160: 8 total, 8 up, 8 in 2024-08-17T16:56:18.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:17 smithi063 ceph-mon[31063]: osdmap e160: 8 total, 8 up, 8 in 2024-08-17T16:56:18.855 INFO:tasks.rados.rados.0.smithi063.stdout:380: done (0 left) 2024-08-17T16:56:18.855 INFO:tasks.rados.rados.0.smithi063.stdout:381: read oid 34 snap -1 2024-08-17T16:56:18.855 INFO:tasks.rados.rados.0.smithi063.stdout:381: expect (ObjNum 112 snap 18 seq_num 112) 2024-08-17T16:56:18.855 INFO:tasks.rados.rados.0.smithi063.stdout:382: read oid 47 snap -1 2024-08-17T16:56:18.855 INFO:tasks.rados.rados.0.smithi063.stdout:382: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:56:18.855 INFO:tasks.rados.rados.0.smithi063.stdout:383: write oid 25 current snap is 48 2024-08-17T16:56:18.856 INFO:tasks.rados.rados.0.smithi063.stdout:383: seq_num 182 ranges {658874=771437,2083958=603055} 2024-08-17T16:56:18.863 INFO:tasks.rados.rados.0.smithi063.stdout:383: writing smithi06387500-25 from 658874 to 1430311 tid 1 2024-08-17T16:56:18.871 INFO:tasks.rados.rados.0.smithi063.stdout:383: writing smithi06387500-25 from 2083958 to 2687013 tid 2 2024-08-17T16:56:18.871 INFO:tasks.rados.rados.0.smithi063.stdout:384: rollback oid 1 current snap is 48 2024-08-17T16:56:18.871 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 1 to 35 2024-08-17T16:56:18.871 INFO:tasks.rados.rados.0.smithi063.stdout:382: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:56:18.876 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:18 smithi162 ceph-mon[34633]: osdmap e161: 8 total, 8 up, 8 in 2024-08-17T16:56:18.876 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:18 smithi162 ceph-mon[34633]: Upgrade: Updating mgr.x 2024-08-17T16:56:18.876 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:18 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:18.876 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:18 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:56:18.877 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:18 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:56:18.877 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:18 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:56:18.877 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:18 smithi162 ceph-mon[34633]: Deploying daemon mgr.x on smithi162 2024-08-17T16:56:18.877 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:18 smithi162 ceph-mon[34633]: pgmap v223: 89 pgs: 89 active+clean; 174 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.8 MiB/s wr, 20 op/s 2024-08-17T16:56:19.025 INFO:tasks.rados.rados.0.smithi063.stdout:383: finishing write tid 1 to smithi06387500-25 2024-08-17T16:56:19.025 INFO:tasks.rados.rados.0.smithi063.stdout:381: expect (ObjNum 112 snap 18 seq_num 112) 2024-08-17T16:56:19.208 INFO:tasks.rados.rados.0.smithi063.stdout:383: finishing write tid 2 to smithi06387500-25 2024-08-17T16:56:19.208 INFO:tasks.rados.rados.0.smithi063.stdout:384: finishing rollback tid 1 to smithi06387500-1 2024-08-17T16:56:19.208 INFO:tasks.rados.rados.0.smithi063.stdout:384: finishing rollback tid 2 to smithi06387500-1 2024-08-17T16:56:19.209 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 130 (ObjNum 137 snap 28 seq_num 137) dirty exists 2024-08-17T16:56:19.209 INFO:tasks.rados.rados.0.smithi063.stdout:383: finishing write tid 3 to smithi06387500-25 2024-08-17T16:56:19.209 INFO:tasks.rados.rados.0.smithi063.stdout:383: finishing write tid 4 to smithi06387500-25 2024-08-17T16:56:19.209 INFO:tasks.rados.rados.0.smithi063.stdout:383: oid 25 updating version 0 to 288 2024-08-17T16:56:19.209 INFO:tasks.rados.rados.0.smithi063.stdout:383: oid 25 updating version 288 to 289 2024-08-17T16:56:19.209 INFO:tasks.rados.rados.0.smithi063.stdout:383: oid 25 version 289 is already newer than 287 2024-08-17T16:56:19.209 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 289 (ObjNum 182 snap 48 seq_num 182) dirty exists 2024-08-17T16:56:19.209 INFO:tasks.rados.rados.0.smithi063.stdout:383: left oid 25 (ObjNum 182 snap 48 seq_num 182) 2024-08-17T16:56:19.209 INFO:tasks.rados.rados.0.smithi063.stdout:381: done (3 left) 2024-08-17T16:56:19.209 INFO:tasks.rados.rados.0.smithi063.stdout:382: done (2 left) 2024-08-17T16:56:19.209 INFO:tasks.rados.rados.0.smithi063.stdout:383: done (1 left) 2024-08-17T16:56:19.209 INFO:tasks.rados.rados.0.smithi063.stdout:384: done (0 left) 2024-08-17T16:56:19.209 INFO:tasks.rados.rados.0.smithi063.stdout:385: write oid 11 current snap is 48 2024-08-17T16:56:19.210 INFO:tasks.rados.rados.0.smithi063.stdout:385: seq_num 183 ranges {675681=563039,1858793=746713,3025937=49744} 2024-08-17T16:56:19.215 INFO:tasks.rados.rados.0.smithi063.stdout:385: writing smithi06387500-11 from 675681 to 1238720 tid 1 2024-08-17T16:56:19.222 INFO:tasks.rados.rados.0.smithi063.stdout:385: writing smithi06387500-11 from 1858793 to 2605506 tid 2 2024-08-17T16:56:19.225 INFO:tasks.rados.rados.0.smithi063.stdout:385: writing smithi06387500-11 from 3025937 to 3075681 tid 3 2024-08-17T16:56:19.225 INFO:tasks.rados.rados.0.smithi063.stdout:386: snap_create 2024-08-17T16:56:19.225 INFO:tasks.rados.rados.0.smithi063.stdout:385: finishing write tid 1 to smithi06387500-11 2024-08-17T16:56:19.226 INFO:tasks.rados.rados.0.smithi063.stdout:385: finishing write tid 2 to smithi06387500-11 2024-08-17T16:56:19.226 INFO:tasks.rados.rados.0.smithi063.stdout:385: finishing write tid 3 to smithi06387500-11 2024-08-17T16:56:19.226 INFO:tasks.rados.rados.0.smithi063.stdout:385: finishing write tid 4 to smithi06387500-11 2024-08-17T16:56:19.226 INFO:tasks.rados.rados.0.smithi063.stdout:385: finishing write tid 5 to smithi06387500-11 2024-08-17T16:56:19.227 INFO:tasks.rados.rados.0.smithi063.stdout:385: oid 11 updating version 0 to 53 2024-08-17T16:56:19.227 INFO:tasks.rados.rados.0.smithi063.stdout:385: oid 11 updating version 53 to 54 2024-08-17T16:56:19.227 INFO:tasks.rados.rados.0.smithi063.stdout:385: oid 11 version 54 is already newer than 52 2024-08-17T16:56:19.227 INFO:tasks.rados.rados.0.smithi063.stdout:385: oid 11 updating version 54 to 55 2024-08-17T16:56:19.227 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 55 (ObjNum 183 snap 48 seq_num 183) dirty exists 2024-08-17T16:56:19.227 INFO:tasks.rados.rados.0.smithi063.stdout:385: left oid 11 (ObjNum 183 snap 48 seq_num 183) 2024-08-17T16:56:19.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[31063]: osdmap e161: 8 total, 8 up, 8 in 2024-08-17T16:56:19.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[31063]: Upgrade: Updating mgr.x 2024-08-17T16:56:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:56:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:56:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:56:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[31063]: Deploying daemon mgr.x on smithi162 2024-08-17T16:56:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[31063]: pgmap v223: 89 pgs: 89 active+clean; 174 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.8 MiB/s wr, 20 op/s 2024-08-17T16:56:19.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[42680]: osdmap e161: 8 total, 8 up, 8 in 2024-08-17T16:56:19.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[42680]: Upgrade: Updating mgr.x 2024-08-17T16:56:19.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:19.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:56:19.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:56:19.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:56:19.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[42680]: Deploying daemon mgr.x on smithi162 2024-08-17T16:56:19.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:18 smithi063 ceph-mon[42680]: pgmap v223: 89 pgs: 89 active+clean; 174 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.8 MiB/s wr, 20 op/s 2024-08-17T16:56:19.858 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:19 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:56:19] "GET /metrics HTTP/1.1" 200 36136 "" "Prometheus/2.43.0" 2024-08-17T16:56:19.867 INFO:tasks.rados.rados.0.smithi063.stdout:385: done (1 left) 2024-08-17T16:56:19.868 INFO:tasks.rados.rados.0.smithi063.stdout:386: done (0 left) 2024-08-17T16:56:19.868 INFO:tasks.rados.rados.0.smithi063.stdout:387: read oid 39 snap -1 2024-08-17T16:56:19.868 INFO:tasks.rados.rados.0.smithi063.stdout:387: expect (ObjNum 102 snap 16 seq_num 102) 2024-08-17T16:56:19.868 INFO:tasks.rados.rados.0.smithi063.stdout:388: delete oid 22 current snap is 49 2024-08-17T16:56:19.869 INFO:tasks.rados.rados.0.smithi063.stdout:388: done (1 left) 2024-08-17T16:56:19.869 INFO:tasks.rados.rados.0.smithi063.stdout:389: rollback oid 29 current snap is 49 2024-08-17T16:56:19.870 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 29 to 38 2024-08-17T16:56:19.870 INFO:tasks.rados.rados.0.smithi063.stdout:390: read oid 28 snap -1 2024-08-17T16:56:19.870 INFO:tasks.rados.rados.0.smithi063.stdout:390: expect (ObjNum 173 snap 46 seq_num 173) 2024-08-17T16:56:19.870 INFO:tasks.rados.rados.0.smithi063.stdout:391: read oid 33 snap -1 2024-08-17T16:56:19.870 INFO:tasks.rados.rados.0.smithi063.stdout:391: expect deleted 2024-08-17T16:56:19.870 INFO:tasks.rados.rados.0.smithi063.stdout:392: rollback oid 20 current snap is 49 2024-08-17T16:56:19.870 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 20 to 48 2024-08-17T16:56:19.870 INFO:tasks.rados.rados.0.smithi063.stdout:393: snap_remove snap 23 2024-08-17T16:56:19.955 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:19 smithi162 ceph-mon[34633]: osdmap e162: 8 total, 8 up, 8 in 2024-08-17T16:56:20.226 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:19 smithi162 systemd[1]: Stopping Ceph mgr.x for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:56:20.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:19 smithi063 ceph-mon[31063]: osdmap e162: 8 total, 8 up, 8 in 2024-08-17T16:56:20.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:19 smithi063 ceph-mon[42680]: osdmap e162: 8 total, 8 up, 8 in 2024-08-17T16:56:20.504 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:20 smithi162 podman[69703]: 2024-08-17 16:56:20.225560957 +0000 UTC m=+0.123889098 container died 4add2ba1a53e5a30fc12e2508486c3ff268c4357e29f3d88b5e52394923358a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=reef-ced649f, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T16:56:20.845 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:20 smithi162 podman[69703]: 2024-08-17 16:56:20.685790393 +0000 UTC m=+0.584118521 container cleanup 4add2ba1a53e5a30fc12e2508486c3ff268c4357e29f3d88b5e52394923358a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, io.buildah.version=1.36.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-08-17T16:56:20.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:20 smithi162 bash[69703]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x 2024-08-17T16:56:20.879 INFO:tasks.rados.rados.0.smithi063.stdout:393: done (5 left) 2024-08-17T16:56:20.879 INFO:tasks.rados.rados.0.smithi063.stdout:394: write oid 17 current snap is 49 2024-08-17T16:56:20.879 INFO:tasks.rados.rados.0.smithi063.stdout:394: seq_num 185 ranges {709295=713537,1859404=750382} 2024-08-17T16:56:20.886 INFO:tasks.rados.rados.0.smithi063.stdout:394: writing smithi06387500-17 from 709295 to 1422832 tid 1 2024-08-17T16:56:20.893 INFO:tasks.rados.rados.0.smithi063.stdout:394: writing smithi06387500-17 from 1859404 to 2609786 tid 2 2024-08-17T16:56:20.894 INFO:tasks.rados.rados.0.smithi063.stdout:395: write oid 44 current snap is 49 2024-08-17T16:56:20.894 INFO:tasks.rados.rados.0.smithi063.stdout:389: finishing rollback tid 0 to smithi06387500-29 2024-08-17T16:56:20.894 INFO:tasks.rados.rados.0.smithi063.stdout:389: finishing rollback tid 1 to smithi06387500-29 2024-08-17T16:56:20.894 INFO:tasks.rados.rados.0.smithi063.stdout:389: finishing rollback tid 2 to smithi06387500-29 2024-08-17T16:56:20.894 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 76 (ObjNum 155 snap 38 seq_num 155) dirty exists 2024-08-17T16:56:20.894 INFO:tasks.rados.rados.0.smithi063.stdout:395: seq_num 186 ranges {726102=788786,1917886=610393,3108206=17896} 2024-08-17T16:56:20.901 INFO:tasks.rados.rados.0.smithi063.stdout:395: writing smithi06387500-44 from 726102 to 1514888 tid 1 2024-08-17T16:56:20.907 INFO:tasks.rados.rados.0.smithi063.stdout:395: writing smithi06387500-44 from 1917886 to 2528279 tid 2 2024-08-17T16:56:20.911 INFO:tasks.rados.rados.0.smithi063.stdout:395: writing smithi06387500-44 from 3108206 to 3126102 tid 3 2024-08-17T16:56:20.911 INFO:tasks.rados.rados.0.smithi063.stdout:389: done (6 left) 2024-08-17T16:56:20.911 INFO:tasks.rados.rados.0.smithi063.stdout:391: done (5 left) 2024-08-17T16:56:20.911 INFO:tasks.rados.rados.0.smithi063.stdout:396: write oid 21 current snap is 49 2024-08-17T16:56:20.912 INFO:tasks.rados.rados.0.smithi063.stdout:390: expect (ObjNum 173 snap 46 seq_num 173) 2024-08-17T16:56:21.075 INFO:tasks.rados.rados.0.smithi063.stdout:392: finishing rollback tid 1 to smithi06387500-20 2024-08-17T16:56:21.075 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 0 (ObjNum 33 snap 0 seq_num 1786885368) dirty dne 2024-08-17T16:56:21.076 INFO:tasks.rados.rados.0.smithi063.stdout:387: expect (ObjNum 102 snap 16 seq_num 102) 2024-08-17T16:56:21.130 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:20 smithi162 ceph-mon[34633]: osdmap e163: 8 total, 8 up, 8 in 2024-08-17T16:56:21.130 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:20 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:21.130 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:20 smithi162 ceph-mon[34633]: pgmap v226: 89 pgs: 89 active+clean; 174 MiB data, 558 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:21.130 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:20 smithi162 podman[69722]: 2024-08-17 16:56:20.946067788 +0000 UTC m=+0.714543142 container remove 4add2ba1a53e5a30fc12e2508486c3ff268c4357e29f3d88b5e52394923358a8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T16:56:21.130 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:20 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-17T16:56:21.131 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:21 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-17T16:56:21.131 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:21 smithi162 systemd[1]: Stopped Ceph mgr.x for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:56:21.131 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:21 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.x.service: Consumed 11.005s CPU time. 2024-08-17T16:56:21.199 INFO:tasks.rados.rados.0.smithi063.stdout:394: finishing write tid 1 to smithi06387500-17 2024-08-17T16:56:21.199 INFO:tasks.rados.rados.0.smithi063.stdout:394: finishing write tid 2 to smithi06387500-17 2024-08-17T16:56:21.199 INFO:tasks.rados.rados.0.smithi063.stdout:396: seq_num 187 ranges {742909=464035,1976368=754051} 2024-08-17T16:56:21.205 INFO:tasks.rados.rados.0.smithi063.stdout:396: writing smithi06387500-21 from 742909 to 1206944 tid 1 2024-08-17T16:56:21.214 INFO:tasks.rados.rados.0.smithi063.stdout:396: writing smithi06387500-21 from 1976368 to 2730419 tid 2 2024-08-17T16:56:21.214 INFO:tasks.rados.rados.0.smithi063.stdout:387: done (5 left) 2024-08-17T16:56:21.214 INFO:tasks.rados.rados.0.smithi063.stdout:390: done (4 left) 2024-08-17T16:56:21.214 INFO:tasks.rados.rados.0.smithi063.stdout:392: done (3 left) 2024-08-17T16:56:21.214 INFO:tasks.rados.rados.0.smithi063.stdout:397: read oid 15 snap 47 2024-08-17T16:56:21.214 INFO:tasks.rados.rados.0.smithi063.stdout:397: expect deleted 2024-08-17T16:56:21.215 INFO:tasks.rados.rados.0.smithi063.stdout:398: snap_create 2024-08-17T16:56:21.215 INFO:tasks.rados.rados.0.smithi063.stdout:394: finishing write tid 3 to smithi06387500-17 2024-08-17T16:56:21.215 INFO:tasks.rados.rados.0.smithi063.stdout:394: finishing write tid 4 to smithi06387500-17 2024-08-17T16:56:21.215 INFO:tasks.rados.rados.0.smithi063.stdout:394: oid 17 updating version 0 to 107 2024-08-17T16:56:21.215 INFO:tasks.rados.rados.0.smithi063.stdout:394: oid 17 updating version 107 to 108 2024-08-17T16:56:21.215 INFO:tasks.rados.rados.0.smithi063.stdout:394: oid 17 updating version 108 to 109 2024-08-17T16:56:21.215 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 109 (ObjNum 185 snap 49 seq_num 185) dirty exists 2024-08-17T16:56:21.215 INFO:tasks.rados.rados.0.smithi063.stdout:394: left oid 17 (ObjNum 185 snap 49 seq_num 185) 2024-08-17T16:56:21.215 INFO:tasks.rados.rados.0.smithi063.stdout:395: finishing write tid 1 to smithi06387500-44 2024-08-17T16:56:21.215 INFO:tasks.rados.rados.0.smithi063.stdout:395: finishing write tid 2 to smithi06387500-44 2024-08-17T16:56:21.215 INFO:tasks.rados.rados.0.smithi063.stdout:395: finishing write tid 3 to smithi06387500-44 2024-08-17T16:56:21.215 INFO:tasks.rados.rados.0.smithi063.stdout:395: finishing write tid 4 to smithi06387500-44 2024-08-17T16:56:21.215 INFO:tasks.rados.rados.0.smithi063.stdout:395: finishing write tid 5 to smithi06387500-44 2024-08-17T16:56:21.215 INFO:tasks.rados.rados.0.smithi063.stdout:395: oid 44 updating version 0 to 61 2024-08-17T16:56:21.216 INFO:tasks.rados.rados.0.smithi063.stdout:395: oid 44 version 61 is already newer than 60 2024-08-17T16:56:21.216 INFO:tasks.rados.rados.0.smithi063.stdout:395: oid 44 updating version 61 to 62 2024-08-17T16:56:21.216 INFO:tasks.rados.rados.0.smithi063.stdout:395: oid 44 version 62 is already newer than 59 2024-08-17T16:56:21.216 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 62 (ObjNum 186 snap 49 seq_num 186) dirty exists 2024-08-17T16:56:21.216 INFO:tasks.rados.rados.0.smithi063.stdout:395: left oid 44 (ObjNum 186 snap 49 seq_num 186) 2024-08-17T16:56:21.216 INFO:tasks.rados.rados.0.smithi063.stdout:396: finishing write tid 1 to smithi06387500-21 2024-08-17T16:56:21.216 INFO:tasks.rados.rados.0.smithi063.stdout:396: finishing write tid 2 to smithi06387500-21 2024-08-17T16:56:21.217 INFO:tasks.rados.rados.0.smithi063.stdout:396: finishing write tid 3 to smithi06387500-21 2024-08-17T16:56:21.217 INFO:tasks.rados.rados.0.smithi063.stdout:396: finishing write tid 4 to smithi06387500-21 2024-08-17T16:56:21.217 INFO:tasks.rados.rados.0.smithi063.stdout:396: oid 21 updating version 0 to 209 2024-08-17T16:56:21.217 INFO:tasks.rados.rados.0.smithi063.stdout:396: oid 21 version 209 is already newer than 207 2024-08-17T16:56:21.217 INFO:tasks.rados.rados.0.smithi063.stdout:396: oid 21 version 209 is already newer than 208 2024-08-17T16:56:21.217 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 209 (ObjNum 187 snap 49 seq_num 187) dirty exists 2024-08-17T16:56:21.217 INFO:tasks.rados.rados.0.smithi063.stdout:396: left oid 21 (ObjNum 187 snap 49 seq_num 187) 2024-08-17T16:56:21.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:20 smithi063 ceph-mon[42680]: osdmap e163: 8 total, 8 up, 8 in 2024-08-17T16:56:21.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:20 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:21.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:20 smithi063 ceph-mon[42680]: pgmap v226: 89 pgs: 89 active+clean; 174 MiB data, 558 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:21.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:20 smithi063 ceph-mon[31063]: osdmap e163: 8 total, 8 up, 8 in 2024-08-17T16:56:21.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:20 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:21.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:20 smithi063 ceph-mon[31063]: pgmap v226: 89 pgs: 89 active+clean; 174 MiB data, 558 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:21.521 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:21 smithi162 systemd[1]: Starting Ceph mgr.x for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:56:21.781 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:21 smithi162 podman[69858]: 2024-08-17 16:56:21.423370638 +0000 UTC m=+0.017936387 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T16:56:21.781 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:21 smithi162 podman[69858]: 2024-08-17 16:56:21.531133869 +0000 UTC m=+0.125699618 container create 180435c78c80767a84f392ef8a014f287b428d6b2b0e69721258e928d585e98d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-08-17T16:56:21.781 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:21 smithi162 podman[69858]: 2024-08-17 16:56:21.750509795 +0000 UTC m=+0.345075550 container init 180435c78c80767a84f392ef8a014f287b428d6b2b0e69721258e928d585e98d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T16:56:21.781 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:21 smithi162 podman[69858]: 2024-08-17 16:56:21.75584752 +0000 UTC m=+0.350413276 container start 180435c78c80767a84f392ef8a014f287b428d6b2b0e69721258e928d585e98d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-08-17T16:56:21.876 INFO:tasks.rados.rados.0.smithi063.stdout:394: done (4 left) 2024-08-17T16:56:21.877 INFO:tasks.rados.rados.0.smithi063.stdout:395: done (3 left) 2024-08-17T16:56:21.877 INFO:tasks.rados.rados.0.smithi063.stdout:396: done (2 left) 2024-08-17T16:56:21.877 INFO:tasks.rados.rados.0.smithi063.stdout:397: done (1 left) 2024-08-17T16:56:21.877 INFO:tasks.rados.rados.0.smithi063.stdout:398: done (0 left) 2024-08-17T16:56:21.877 INFO:tasks.rados.rados.0.smithi063.stdout:399: read oid 47 snap 46 2024-08-17T16:56:21.877 INFO:tasks.rados.rados.0.smithi063.stdout:399: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:56:21.877 INFO:tasks.rados.rados.0.smithi063.stdout:400: rollback oid 24 current snap is 50 2024-08-17T16:56:21.877 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 24 to 20 2024-08-17T16:56:21.877 INFO:tasks.rados.rados.0.smithi063.stdout:401: delete oid 8 current snap is 50 2024-08-17T16:56:21.879 INFO:tasks.rados.rados.0.smithi063.stdout:401: done (2 left) 2024-08-17T16:56:21.879 INFO:tasks.rados.rados.0.smithi063.stdout:402: snap_remove snap 35 2024-08-17T16:56:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:21 smithi063 ceph-mon[42680]: osdmap e164: 8 total, 8 up, 8 in 2024-08-17T16:56:22.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:21 smithi063 ceph-mon[31063]: osdmap e164: 8 total, 8 up, 8 in 2024-08-17T16:56:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:21 smithi162 ceph-mon[34633]: osdmap e164: 8 total, 8 up, 8 in 2024-08-17T16:56:22.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:21 smithi162 ceph-mgr[69897]: -- 172.21.15.162:0/1434310158 <== mon.1 v2:172.21.15.162:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55622d2bb0e0 con 0x55622d299400 2024-08-17T16:56:22.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:21 smithi162 bash[69858]: 180435c78c80767a84f392ef8a014f287b428d6b2b0e69721258e928d585e98d 2024-08-17T16:56:22.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:21 smithi162 systemd[1]: Started Ceph mgr.x for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:56:22.894 INFO:tasks.rados.rados.0.smithi063.stdout:402: done (2 left) 2024-08-17T16:56:22.894 INFO:tasks.rados.rados.0.smithi063.stdout:403: write oid 46 current snap is 50 2024-08-17T16:56:22.895 INFO:tasks.rados.rados.0.smithi063.stdout:403: seq_num 189 ranges {776523=614533,1976979=757720} 2024-08-17T16:56:22.902 INFO:tasks.rados.rados.0.smithi063.stdout:403: writing smithi06387500-46 from 776523 to 1391056 tid 1 2024-08-17T16:56:22.911 INFO:tasks.rados.rados.0.smithi063.stdout:403: writing smithi06387500-46 from 1976979 to 2734699 tid 2 2024-08-17T16:56:22.911 INFO:tasks.rados.rados.0.smithi063.stdout:404: write oid 39 current snap is 50 2024-08-17T16:56:22.911 INFO:tasks.rados.rados.0.smithi063.stdout:400: finishing rollback tid 1 to smithi06387500-24 2024-08-17T16:56:22.911 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T16:56:22.911 INFO:tasks.rados.rados.0.smithi063.stdout:399: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:56:23.066 INFO:tasks.rados.rados.0.smithi063.stdout:403: finishing write tid 1 to smithi06387500-46 2024-08-17T16:56:23.066 INFO:tasks.rados.rados.0.smithi063.stdout:403: finishing write tid 2 to smithi06387500-46 2024-08-17T16:56:23.066 INFO:tasks.rados.rados.0.smithi063.stdout:403: finishing write tid 3 to smithi06387500-46 2024-08-17T16:56:23.066 INFO:tasks.rados.rados.0.smithi063.stdout:403: finishing write tid 4 to smithi06387500-46 2024-08-17T16:56:23.067 INFO:tasks.rados.rados.0.smithi063.stdout:403: oid 46 updating version 0 to 124 2024-08-17T16:56:23.067 INFO:tasks.rados.rados.0.smithi063.stdout:403: oid 46 updating version 124 to 125 2024-08-17T16:56:23.067 INFO:tasks.rados.rados.0.smithi063.stdout:403: oid 46 version 125 is already newer than 121 2024-08-17T16:56:23.067 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 125 (ObjNum 189 snap 50 seq_num 189) dirty exists 2024-08-17T16:56:23.067 INFO:tasks.rados.rados.0.smithi063.stdout:403: left oid 46 (ObjNum 189 snap 50 seq_num 189) 2024-08-17T16:56:23.067 INFO:tasks.rados.rados.0.smithi063.stdout:404: seq_num 190 ranges {793330=689782,2035461=617731,3101545=91785} 2024-08-17T16:56:23.074 INFO:tasks.rados.rados.0.smithi063.stdout:404: writing smithi06387500-39 from 793330 to 1483112 tid 1 2024-08-17T16:56:23.081 INFO:tasks.rados.rados.0.smithi063.stdout:404: writing smithi06387500-39 from 2035461 to 2653192 tid 2 2024-08-17T16:56:23.084 INFO:tasks.rados.rados.0.smithi063.stdout:404: writing smithi06387500-39 from 3101545 to 3193330 tid 3 2024-08-17T16:56:23.084 INFO:tasks.rados.rados.0.smithi063.stdout:399: done (3 left) 2024-08-17T16:56:23.084 INFO:tasks.rados.rados.0.smithi063.stdout:400: done (2 left) 2024-08-17T16:56:23.084 INFO:tasks.rados.rados.0.smithi063.stdout:403: done (1 left) 2024-08-17T16:56:23.084 INFO:tasks.rados.rados.0.smithi063.stdout:405: write oid 17 current snap is 50 2024-08-17T16:56:23.085 INFO:tasks.rados.rados.0.smithi063.stdout:404: finishing write tid 1 to smithi06387500-39 2024-08-17T16:56:23.085 INFO:tasks.rados.rados.0.smithi063.stdout:404: finishing write tid 2 to smithi06387500-39 2024-08-17T16:56:23.085 INFO:tasks.rados.rados.0.smithi063.stdout:405: seq_num 191 ranges {410137=481384,1410296=761389,2728968=481169} 2024-08-17T16:56:23.089 INFO:tasks.rados.rados.0.smithi063.stdout:405: writing smithi06387500-17 from 410137 to 891521 tid 1 2024-08-17T16:56:23.096 INFO:tasks.rados.rados.0.smithi063.stdout:405: writing smithi06387500-17 from 1410296 to 2171685 tid 2 2024-08-17T16:56:23.102 INFO:tasks.rados.rados.0.smithi063.stdout:405: writing smithi06387500-17 from 2728968 to 3210137 tid 3 2024-08-17T16:56:23.102 INFO:tasks.rados.rados.0.smithi063.stdout:406: rollback oid 38 current snap is 50 2024-08-17T16:56:23.102 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 38 to 38 2024-08-17T16:56:23.102 INFO:tasks.rados.rados.0.smithi063.stdout:404: finishing write tid 3 to smithi06387500-39 2024-08-17T16:56:23.102 INFO:tasks.rados.rados.0.smithi063.stdout:404: finishing write tid 4 to smithi06387500-39 2024-08-17T16:56:23.103 INFO:tasks.rados.rados.0.smithi063.stdout:404: finishing write tid 5 to smithi06387500-39 2024-08-17T16:56:23.103 INFO:tasks.rados.rados.0.smithi063.stdout:404: oid 39 updating version 0 to 29 2024-08-17T16:56:23.103 INFO:tasks.rados.rados.0.smithi063.stdout:404: oid 39 version 29 is already newer than 28 2024-08-17T16:56:23.103 INFO:tasks.rados.rados.0.smithi063.stdout:404: oid 39 updating version 29 to 30 2024-08-17T16:56:23.103 INFO:tasks.rados.rados.0.smithi063.stdout:404: oid 39 version 30 is already newer than 27 2024-08-17T16:56:23.103 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 30 (ObjNum 190 snap 50 seq_num 190) dirty exists 2024-08-17T16:56:23.103 INFO:tasks.rados.rados.0.smithi063.stdout:404: left oid 39 (ObjNum 190 snap 50 seq_num 190) 2024-08-17T16:56:23.103 INFO:tasks.rados.rados.0.smithi063.stdout:405: finishing write tid 1 to smithi06387500-17 2024-08-17T16:56:23.103 INFO:tasks.rados.rados.0.smithi063.stdout:405: finishing write tid 2 to smithi06387500-17 2024-08-17T16:56:23.103 INFO:tasks.rados.rados.0.smithi063.stdout:404: done (2 left) 2024-08-17T16:56:23.103 INFO:tasks.rados.rados.0.smithi063.stdout:407: write oid 4 current snap is 50 2024-08-17T16:56:23.103 INFO:tasks.rados.rados.0.smithi063.stdout:407: seq_num 192 ranges {426944=556633,1468778=621400,2872744=354200} 2024-08-17T16:56:23.107 INFO:tasks.rados.rados.0.smithi063.stdout:407: writing smithi06387500-4 from 426944 to 983577 tid 1 2024-08-17T16:56:23.113 INFO:tasks.rados.rados.0.smithi063.stdout:407: writing smithi06387500-4 from 1468778 to 2090178 tid 2 2024-08-17T16:56:23.119 INFO:tasks.rados.rados.0.smithi063.stdout:407: writing smithi06387500-4 from 2872744 to 3226944 tid 3 2024-08-17T16:56:23.119 INFO:tasks.rados.rados.0.smithi063.stdout:408: snap_create 2024-08-17T16:56:23.119 INFO:tasks.rados.rados.0.smithi063.stdout:406: finishing rollback tid 1 to smithi06387500-38 2024-08-17T16:56:23.120 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 34 (ObjNum 129 snap 0 seq_num 1807090504) dirty dne 2024-08-17T16:56:23.120 INFO:tasks.rados.rados.0.smithi063.stdout:405: finishing write tid 3 to smithi06387500-17 2024-08-17T16:56:23.120 INFO:tasks.rados.rados.0.smithi063.stdout:405: finishing write tid 4 to smithi06387500-17 2024-08-17T16:56:23.120 INFO:tasks.rados.rados.0.smithi063.stdout:405: finishing write tid 5 to smithi06387500-17 2024-08-17T16:56:23.120 INFO:tasks.rados.rados.0.smithi063.stdout:405: oid 17 updating version 0 to 117 2024-08-17T16:56:23.120 INFO:tasks.rados.rados.0.smithi063.stdout:405: oid 17 updating version 117 to 118 2024-08-17T16:56:23.120 INFO:tasks.rados.rados.0.smithi063.stdout:405: oid 17 version 118 is already newer than 116 2024-08-17T16:56:23.120 INFO:tasks.rados.rados.0.smithi063.stdout:405: oid 17 version 118 is already newer than 115 2024-08-17T16:56:23.120 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 118 (ObjNum 191 snap 50 seq_num 191) dirty exists 2024-08-17T16:56:23.120 INFO:tasks.rados.rados.0.smithi063.stdout:405: left oid 17 (ObjNum 191 snap 50 seq_num 191) 2024-08-17T16:56:23.120 INFO:tasks.rados.rados.0.smithi063.stdout:407: finishing write tid 1 to smithi06387500-4 2024-08-17T16:56:23.120 INFO:tasks.rados.rados.0.smithi063.stdout:407: finishing write tid 2 to smithi06387500-4 2024-08-17T16:56:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:407: finishing write tid 3 to smithi06387500-4 2024-08-17T16:56:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:407: finishing write tid 4 to smithi06387500-4 2024-08-17T16:56:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:407: finishing write tid 5 to smithi06387500-4 2024-08-17T16:56:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:407: oid 4 updating version 0 to 316 2024-08-17T16:56:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:407: oid 4 version 316 is already newer than 315 2024-08-17T16:56:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:407: oid 4 version 316 is already newer than 313 2024-08-17T16:56:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:407: oid 4 version 316 is already newer than 314 2024-08-17T16:56:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 316 (ObjNum 192 snap 50 seq_num 192) dirty exists 2024-08-17T16:56:23.122 INFO:tasks.rados.rados.0.smithi063.stdout:407: left oid 4 (ObjNum 192 snap 50 seq_num 192) 2024-08-17T16:56:23.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:22 smithi063 ceph-mon[42680]: osdmap e165: 8 total, 8 up, 8 in 2024-08-17T16:56:23.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:22 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:23.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:22 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:22 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:56:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:22 smithi063 ceph-mon[42680]: pgmap v229: 89 pgs: 89 active+clean; 193 MiB data, 582 MiB used, 715 GiB / 715 GiB avail; 31 MiB/s rd, 2.8 MiB/s wr, 36 op/s 2024-08-17T16:56:23.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:22 smithi063 ceph-mon[31063]: osdmap e165: 8 total, 8 up, 8 in 2024-08-17T16:56:23.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:22 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:23.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:22 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:23.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:22 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:56:23.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:22 smithi063 ceph-mon[31063]: pgmap v229: 89 pgs: 89 active+clean; 193 MiB data, 582 MiB used, 715 GiB / 715 GiB avail; 31 MiB/s rd, 2.8 MiB/s wr, 36 op/s 2024-08-17T16:56:23.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:22 smithi162 ceph-mon[34633]: osdmap e165: 8 total, 8 up, 8 in 2024-08-17T16:56:23.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:22 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:22 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:22 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:56:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:22 smithi162 ceph-mon[34633]: pgmap v229: 89 pgs: 89 active+clean; 193 MiB data, 582 MiB used, 715 GiB / 715 GiB avail; 31 MiB/s rd, 2.8 MiB/s wr, 36 op/s 2024-08-17T16:56:23.950 INFO:tasks.rados.rados.0.smithi063.stdout:405: done (3 left) 2024-08-17T16:56:23.950 INFO:tasks.rados.rados.0.smithi063.stdout:406: done (2 left) 2024-08-17T16:56:23.950 INFO:tasks.rados.rados.0.smithi063.stdout:407: done (1 left) 2024-08-17T16:56:23.950 INFO:tasks.rados.rados.0.smithi063.stdout:408: done (0 left) 2024-08-17T16:56:23.950 INFO:tasks.rados.rados.0.smithi063.stdout:409: snap_create 2024-08-17T16:56:24.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:23 smithi063 ceph-mon[42680]: osdmap e166: 8 total, 8 up, 8 in 2024-08-17T16:56:24.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:23 smithi063 ceph-mon[31063]: osdmap e166: 8 total, 8 up, 8 in 2024-08-17T16:56:24.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:23 smithi162 ceph-mon[34633]: osdmap e166: 8 total, 8 up, 8 in 2024-08-17T16:56:24.841 INFO:tasks.rados.rados.0.smithi063.stdout:409: done (0 left) 2024-08-17T16:56:24.841 INFO:tasks.rados.rados.0.smithi063.stdout:410: write oid 8 current snap is 52 2024-08-17T16:56:24.841 INFO:tasks.rados.rados.0.smithi063.stdout:410: seq_num 193 ranges {443751=631882,1810907=765058,3067461=176290} 2024-08-17T16:56:24.847 INFO:tasks.rados.rados.0.smithi063.stdout:410: writing smithi06387500-8 from 443751 to 1075633 tid 1 2024-08-17T16:56:24.858 INFO:tasks.rados.rados.0.smithi063.stdout:410: writing smithi06387500-8 from 1810907 to 2575965 tid 2 2024-08-17T16:56:24.858 INFO:tasks.rados.rados.0.smithi063.stdout:410: writing smithi06387500-8 from 3067461 to 3243751 tid 3 2024-08-17T16:56:24.859 INFO:tasks.rados.rados.0.smithi063.stdout:411: snap_create 2024-08-17T16:56:24.859 INFO:tasks.rados.rados.0.smithi063.stdout:410: finishing write tid 1 to smithi06387500-8 2024-08-17T16:56:24.859 INFO:tasks.rados.rados.0.smithi063.stdout:410: finishing write tid 2 to smithi06387500-8 2024-08-17T16:56:24.860 INFO:tasks.rados.rados.0.smithi063.stdout:410: finishing write tid 3 to smithi06387500-8 2024-08-17T16:56:24.860 INFO:tasks.rados.rados.0.smithi063.stdout:410: finishing write tid 4 to smithi06387500-8 2024-08-17T16:56:24.861 INFO:tasks.rados.rados.0.smithi063.stdout:410: finishing write tid 5 to smithi06387500-8 2024-08-17T16:56:24.861 INFO:tasks.rados.rados.0.smithi063.stdout:410: oid 8 updating version 0 to 116 2024-08-17T16:56:24.861 INFO:tasks.rados.rados.0.smithi063.stdout:410: oid 8 updating version 116 to 117 2024-08-17T16:56:24.861 INFO:tasks.rados.rados.0.smithi063.stdout:410: oid 8 version 117 is already newer than 115 2024-08-17T16:56:24.861 INFO:tasks.rados.rados.0.smithi063.stdout:410: oid 8 version 117 is already newer than 114 2024-08-17T16:56:24.861 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 117 (ObjNum 193 snap 52 seq_num 193) dirty exists 2024-08-17T16:56:24.861 INFO:tasks.rados.rados.0.smithi063.stdout:410: left oid 8 (ObjNum 193 snap 52 seq_num 193) 2024-08-17T16:56:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:24 smithi063 ceph-mon[31063]: osdmap e167: 8 total, 8 up, 8 in 2024-08-17T16:56:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:24 smithi063 ceph-mon[31063]: pgmap v232: 89 pgs: 89 active+clean; 193 MiB data, 582 MiB used, 715 GiB / 715 GiB avail; 31 MiB/s rd, 2.8 MiB/s wr, 36 op/s 2024-08-17T16:56:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:24 smithi063 ceph-mon[31063]: osdmap e168: 8 total, 8 up, 8 in 2024-08-17T16:56:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:24 smithi063 ceph-mon[42680]: osdmap e167: 8 total, 8 up, 8 in 2024-08-17T16:56:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:24 smithi063 ceph-mon[42680]: pgmap v232: 89 pgs: 89 active+clean; 193 MiB data, 582 MiB used, 715 GiB / 715 GiB avail; 31 MiB/s rd, 2.8 MiB/s wr, 36 op/s 2024-08-17T16:56:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:24 smithi063 ceph-mon[42680]: osdmap e168: 8 total, 8 up, 8 in 2024-08-17T16:56:25.327 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:24 smithi162 ceph-mon[34633]: osdmap e167: 8 total, 8 up, 8 in 2024-08-17T16:56:25.327 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:24 smithi162 ceph-mon[34633]: pgmap v232: 89 pgs: 89 active+clean; 193 MiB data, 582 MiB used, 715 GiB / 715 GiB avail; 31 MiB/s rd, 2.8 MiB/s wr, 36 op/s 2024-08-17T16:56:25.327 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:24 smithi162 ceph-mon[34633]: osdmap e168: 8 total, 8 up, 8 in 2024-08-17T16:56:25.837 INFO:tasks.rados.rados.0.smithi063.stdout:410: done (1 left) 2024-08-17T16:56:25.837 INFO:tasks.rados.rados.0.smithi063.stdout:411: done (0 left) 2024-08-17T16:56:25.837 INFO:tasks.rados.rados.0.smithi063.stdout:412: snap_create 2024-08-17T16:56:26.840 INFO:tasks.rados.rados.0.smithi063.stdout:412: done (0 left) 2024-08-17T16:56:26.840 INFO:tasks.rados.rados.0.smithi063.stdout:413: snap_remove snap 46 2024-08-17T16:56:27.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:26 smithi162 ceph-mon[34633]: osdmap e169: 8 total, 8 up, 8 in 2024-08-17T16:56:27.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:26 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:27.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:26 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:27.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:26 smithi162 ceph-mon[34633]: pgmap v235: 89 pgs: 89 active+clean; 203 MiB data, 616 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-08-17T16:56:27.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:26 smithi063 ceph-mon[42680]: osdmap e169: 8 total, 8 up, 8 in 2024-08-17T16:56:27.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:26 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:27.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:26 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:27.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:26 smithi063 ceph-mon[42680]: pgmap v235: 89 pgs: 89 active+clean; 203 MiB data, 616 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-08-17T16:56:27.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:26 smithi063 ceph-mon[31063]: osdmap e169: 8 total, 8 up, 8 in 2024-08-17T16:56:27.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:26 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:27.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:26 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:27.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:26 smithi063 ceph-mon[31063]: pgmap v235: 89 pgs: 89 active+clean; 203 MiB data, 616 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-08-17T16:56:27.402 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: [17/Aug/2024:16:56:27] ENGINE Bus STARTING 2024-08-17T16:56:27.403 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: CherryPy Checker: 2024-08-17T16:56:27.403 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: The Application mounted at '' has an empty config. 2024-08-17T16:56:27.403 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: 2024-08-17T16:56:27.403 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: [17/Aug/2024:16:56:27] ENGINE Serving on http://:::9283 2024-08-17T16:56:27.403 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: [17/Aug/2024:16:56:27] ENGINE Bus STARTED 2024-08-17T16:56:27.842 INFO:tasks.rados.rados.0.smithi063.stdout:413: done (0 left) 2024-08-17T16:56:27.842 INFO:tasks.rados.rados.0.smithi063.stdout:414: rollback oid 9 current snap is 54 2024-08-17T16:56:27.842 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 9 to 49 2024-08-17T16:56:27.842 INFO:tasks.rados.rados.0.smithi063.stdout:415: rollback oid 11 current snap is 54 2024-08-17T16:56:27.842 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 11 to 40 2024-08-17T16:56:27.842 INFO:tasks.rados.rados.0.smithi063.stdout:416: write oid 25 current snap is 54 2024-08-17T16:56:27.842 INFO:tasks.rados.rados.0.smithi063.stdout:416: seq_num 194 ranges {460558=707131,1869389=508716,3094884=165674} 2024-08-17T16:56:27.849 INFO:tasks.rados.rados.0.smithi063.stdout:416: writing smithi06387500-25 from 460558 to 1167689 tid 1 2024-08-17T16:56:27.856 INFO:tasks.rados.rados.0.smithi063.stdout:416: writing smithi06387500-25 from 1869389 to 2378105 tid 2 2024-08-17T16:56:27.861 INFO:tasks.rados.rados.0.smithi063.stdout:416: writing smithi06387500-25 from 3094884 to 3260558 tid 3 2024-08-17T16:56:27.861 INFO:tasks.rados.rados.0.smithi063.stdout:417: write oid 5 current snap is 54 2024-08-17T16:56:27.861 INFO:tasks.rados.rados.0.smithi063.stdout:414: finishing rollback tid 0 to smithi06387500-9 2024-08-17T16:56:27.861 INFO:tasks.rados.rados.0.smithi063.stdout:415: finishing rollback tid 0 to smithi06387500-11 2024-08-17T16:56:27.861 INFO:tasks.rados.rados.0.smithi063.stdout:415: finishing rollback tid 1 to smithi06387500-11 2024-08-17T16:56:27.862 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 63 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-08-17T16:56:27.862 INFO:tasks.rados.rados.0.smithi063.stdout:414: finishing rollback tid 1 to smithi06387500-9 2024-08-17T16:56:27.862 INFO:tasks.rados.rados.0.smithi063.stdout:414: finishing rollback tid 2 to smithi06387500-9 2024-08-17T16:56:27.862 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 163 (ObjNum 151 snap 36 seq_num 151) dirty exists 2024-08-17T16:56:27.862 INFO:tasks.rados.rados.0.smithi063.stdout:416: finishing write tid 1 to smithi06387500-25 2024-08-17T16:56:27.862 INFO:tasks.rados.rados.0.smithi063.stdout:416: finishing write tid 2 to smithi06387500-25 2024-08-17T16:56:27.862 INFO:tasks.rados.rados.0.smithi063.stdout:417: seq_num 195 ranges {477365=782380,1927871=768727,3122307=155058} 2024-08-17T16:56:27.868 INFO:tasks.rados.rados.0.smithi063.stdout:417: writing smithi06387500-5 from 477365 to 1259745 tid 1 2024-08-17T16:56:27.876 INFO:tasks.rados.rados.0.smithi063.stdout:417: writing smithi06387500-5 from 1927871 to 2696598 tid 2 2024-08-17T16:56:27.880 INFO:tasks.rados.rados.0.smithi063.stdout:417: writing smithi06387500-5 from 3122307 to 3277365 tid 3 2024-08-17T16:56:27.880 INFO:tasks.rados.rados.0.smithi063.stdout:414: done (3 left) 2024-08-17T16:56:27.880 INFO:tasks.rados.rados.0.smithi063.stdout:415: done (2 left) 2024-08-17T16:56:27.880 INFO:tasks.rados.rados.0.smithi063.stdout:418: read oid 12 snap -1 2024-08-17T16:56:27.880 INFO:tasks.rados.rados.0.smithi063.stdout:418: expect (ObjNum 68 snap 7 seq_num 68) 2024-08-17T16:56:27.880 INFO:tasks.rados.rados.0.smithi063.stdout:419: write oid 16 current snap is 54 2024-08-17T16:56:27.880 INFO:tasks.rados.rados.0.smithi063.stdout:416: finishing write tid 3 to smithi06387500-25 2024-08-17T16:56:27.880 INFO:tasks.rados.rados.0.smithi063.stdout:416: finishing write tid 4 to smithi06387500-25 2024-08-17T16:56:27.880 INFO:tasks.rados.rados.0.smithi063.stdout:416: finishing write tid 5 to smithi06387500-25 2024-08-17T16:56:27.880 INFO:tasks.rados.rados.0.smithi063.stdout:416: oid 25 updating version 0 to 322 2024-08-17T16:56:27.880 INFO:tasks.rados.rados.0.smithi063.stdout:416: oid 25 updating version 322 to 323 2024-08-17T16:56:27.880 INFO:tasks.rados.rados.0.smithi063.stdout:416: oid 25 updating version 323 to 324 2024-08-17T16:56:27.880 INFO:tasks.rados.rados.0.smithi063.stdout:416: oid 25 updating version 324 to 325 2024-08-17T16:56:27.881 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 325 (ObjNum 194 snap 54 seq_num 194) dirty exists 2024-08-17T16:56:27.881 INFO:tasks.rados.rados.0.smithi063.stdout:416: left oid 25 (ObjNum 194 snap 54 seq_num 194) 2024-08-17T16:56:27.881 INFO:tasks.rados.rados.0.smithi063.stdout:417: finishing write tid 1 to smithi06387500-5 2024-08-17T16:56:27.881 INFO:tasks.rados.rados.0.smithi063.stdout:419: seq_num 196 ranges {494172=457629,1586353=512385,2749730=544442} 2024-08-17T16:56:27.885 INFO:tasks.rados.rados.0.smithi063.stdout:419: writing smithi06387500-16 from 494172 to 951801 tid 1 2024-08-17T16:56:27.891 INFO:tasks.rados.rados.0.smithi063.stdout:419: writing smithi06387500-16 from 1586353 to 2098738 tid 2 2024-08-17T16:56:27.898 INFO:tasks.rados.rados.0.smithi063.stdout:419: writing smithi06387500-16 from 2749730 to 3294172 tid 3 2024-08-17T16:56:27.898 INFO:tasks.rados.rados.0.smithi063.stdout:416: done (3 left) 2024-08-17T16:56:27.898 INFO:tasks.rados.rados.0.smithi063.stdout:420: rollback oid 28 current snap is 54 2024-08-17T16:56:27.899 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 28 to 36 2024-08-17T16:56:27.899 INFO:tasks.rados.rados.0.smithi063.stdout:417: finishing write tid 2 to smithi06387500-5 2024-08-17T16:56:27.899 INFO:tasks.rados.rados.0.smithi063.stdout:417: finishing write tid 3 to smithi06387500-5 2024-08-17T16:56:27.899 INFO:tasks.rados.rados.0.smithi063.stdout:417: finishing write tid 4 to smithi06387500-5 2024-08-17T16:56:27.899 INFO:tasks.rados.rados.0.smithi063.stdout:417: finishing write tid 5 to smithi06387500-5 2024-08-17T16:56:27.899 INFO:tasks.rados.rados.0.smithi063.stdout:417: oid 5 updating version 0 to 145 2024-08-17T16:56:27.899 INFO:tasks.rados.rados.0.smithi063.stdout:417: oid 5 version 145 is already newer than 142 2024-08-17T16:56:27.899 INFO:tasks.rados.rados.0.smithi063.stdout:417: oid 5 version 145 is already newer than 144 2024-08-17T16:56:27.899 INFO:tasks.rados.rados.0.smithi063.stdout:417: oid 5 version 145 is already newer than 143 2024-08-17T16:56:27.899 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 145 (ObjNum 195 snap 54 seq_num 195) dirty exists 2024-08-17T16:56:27.899 INFO:tasks.rados.rados.0.smithi063.stdout:417: left oid 5 (ObjNum 195 snap 54 seq_num 195) 2024-08-17T16:56:27.899 INFO:tasks.rados.rados.0.smithi063.stdout:418: expect (ObjNum 68 snap 7 seq_num 68) 2024-08-17T16:56:27.995 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-mon[34633]: osdmap e170: 8 total, 8 up, 8 in 2024-08-17T16:56:27.995 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:27.995 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:27.995 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-mon[34633]: Standby manager daemon x restarted 2024-08-17T16:56:27.995 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-mon[34633]: Standby manager daemon x started 2024-08-17T16:56:27.995 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:56:27.995 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:56:27.995 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:56:27.995 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:56:27.995 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:27 smithi162 ceph-mon[34633]: osdmap e171: 8 total, 8 up, 8 in 2024-08-17T16:56:28.044 INFO:tasks.rados.rados.0.smithi063.stdout:419: finishing write tid 1 to smithi06387500-16 2024-08-17T16:56:28.044 INFO:tasks.rados.rados.0.smithi063.stdout:419: finishing write tid 2 to smithi06387500-16 2024-08-17T16:56:28.044 INFO:tasks.rados.rados.0.smithi063.stdout:420: finishing rollback tid 0 to smithi06387500-28 2024-08-17T16:56:28.044 INFO:tasks.rados.rados.0.smithi063.stdout:419: finishing write tid 3 to smithi06387500-16 2024-08-17T16:56:28.044 INFO:tasks.rados.rados.0.smithi063.stdout:420: finishing rollback tid 1 to smithi06387500-28 2024-08-17T16:56:28.044 INFO:tasks.rados.rados.0.smithi063.stdout:419: finishing write tid 4 to smithi06387500-16 2024-08-17T16:56:28.044 INFO:tasks.rados.rados.0.smithi063.stdout:419: finishing write tid 5 to smithi06387500-16 2024-08-17T16:56:28.044 INFO:tasks.rados.rados.0.smithi063.stdout:419: oid 16 updating version 0 to 132 2024-08-17T16:56:28.044 INFO:tasks.rados.rados.0.smithi063.stdout:419: oid 16 version 132 is already newer than 129 2024-08-17T16:56:28.044 INFO:tasks.rados.rados.0.smithi063.stdout:419: oid 16 version 132 is already newer than 131 2024-08-17T16:56:28.044 INFO:tasks.rados.rados.0.smithi063.stdout:419: oid 16 version 132 is already newer than 130 2024-08-17T16:56:28.045 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 132 (ObjNum 196 snap 54 seq_num 196) dirty exists 2024-08-17T16:56:28.045 INFO:tasks.rados.rados.0.smithi063.stdout:419: left oid 16 (ObjNum 196 snap 54 seq_num 196) 2024-08-17T16:56:28.045 INFO:tasks.rados.rados.0.smithi063.stdout:420: finishing rollback tid 2 to smithi06387500-28 2024-08-17T16:56:28.045 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 126 (ObjNum 147 snap 34 seq_num 147) dirty exists 2024-08-17T16:56:28.045 INFO:tasks.rados.rados.0.smithi063.stdout:417: done (3 left) 2024-08-17T16:56:28.045 INFO:tasks.rados.rados.0.smithi063.stdout:418: done (2 left) 2024-08-17T16:56:28.045 INFO:tasks.rados.rados.0.smithi063.stdout:419: done (1 left) 2024-08-17T16:56:28.045 INFO:tasks.rados.rados.0.smithi063.stdout:420: done (0 left) 2024-08-17T16:56:28.045 INFO:tasks.rados.rados.0.smithi063.stdout:421: write oid 13 current snap is 54 2024-08-17T16:56:28.045 INFO:tasks.rados.rados.0.smithi063.stdout:421: seq_num 197 ranges {510979=532878,1528482=772396,3060800=250179} 2024-08-17T16:56:28.049 INFO:tasks.rados.rados.0.smithi063.stdout:421: writing smithi06387500-13 from 510979 to 1043857 tid 1 2024-08-17T16:56:28.056 INFO:tasks.rados.rados.0.smithi063.stdout:421: writing smithi06387500-13 from 1528482 to 2300878 tid 2 2024-08-17T16:56:28.062 INFO:tasks.rados.rados.0.smithi063.stdout:421: writing smithi06387500-13 from 3060800 to 3310979 tid 3 2024-08-17T16:56:28.062 INFO:tasks.rados.rados.0.smithi063.stdout:422: snap_create 2024-08-17T16:56:28.062 INFO:tasks.rados.rados.0.smithi063.stdout:421: finishing write tid 1 to smithi06387500-13 2024-08-17T16:56:28.067 INFO:tasks.rados.rados.0.smithi063.stdout:421: finishing write tid 2 to smithi06387500-13 2024-08-17T16:56:28.067 INFO:tasks.rados.rados.0.smithi063.stdout:421: finishing write tid 3 to smithi06387500-13 2024-08-17T16:56:28.067 INFO:tasks.rados.rados.0.smithi063.stdout:421: finishing write tid 4 to smithi06387500-13 2024-08-17T16:56:28.067 INFO:tasks.rados.rados.0.smithi063.stdout:421: finishing write tid 5 to smithi06387500-13 2024-08-17T16:56:28.067 INFO:tasks.rados.rados.0.smithi063.stdout:421: oid 13 updating version 0 to 148 2024-08-17T16:56:28.067 INFO:tasks.rados.rados.0.smithi063.stdout:421: oid 13 version 148 is already newer than 147 2024-08-17T16:56:28.067 INFO:tasks.rados.rados.0.smithi063.stdout:421: oid 13 updating version 148 to 149 2024-08-17T16:56:28.067 INFO:tasks.rados.rados.0.smithi063.stdout:421: oid 13 version 149 is already newer than 146 2024-08-17T16:56:28.068 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 149 (ObjNum 197 snap 54 seq_num 197) dirty exists 2024-08-17T16:56:28.068 INFO:tasks.rados.rados.0.smithi063.stdout:421: left oid 13 (ObjNum 197 snap 54 seq_num 197) 2024-08-17T16:56:28.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[42680]: osdmap e170: 8 total, 8 up, 8 in 2024-08-17T16:56:28.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:28.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:28.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[42680]: Standby manager daemon x restarted 2024-08-17T16:56:28.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[42680]: Standby manager daemon x started 2024-08-17T16:56:28.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:56:28.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:56:28.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:56:28.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:56:28.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[42680]: osdmap e171: 8 total, 8 up, 8 in 2024-08-17T16:56:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[31063]: osdmap e170: 8 total, 8 up, 8 in 2024-08-17T16:56:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[31063]: Standby manager daemon x restarted 2024-08-17T16:56:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[31063]: Standby manager daemon x started 2024-08-17T16:56:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:56:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:56:28.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:56:28.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:56:28.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:27 smithi063 ceph-mon[31063]: osdmap e171: 8 total, 8 up, 8 in 2024-08-17T16:56:28.922 INFO:tasks.rados.rados.0.smithi063.stdout:421: done (1 left) 2024-08-17T16:56:28.922 INFO:tasks.rados.rados.0.smithi063.stdout:422: done (0 left) 2024-08-17T16:56:28.922 INFO:tasks.rados.rados.0.smithi063.stdout:423: rollback oid 48 current snap is 55 2024-08-17T16:56:28.922 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 48 to 38 2024-08-17T16:56:28.922 INFO:tasks.rados.rados.0.smithi063.stdout:424: snap_create 2024-08-17T16:56:28.927 INFO:tasks.rados.rados.0.smithi063.stdout:423: finishing rollback tid 0 to smithi06387500-48 2024-08-17T16:56:28.927 INFO:tasks.rados.rados.0.smithi063.stdout:423: finishing rollback tid 1 to smithi06387500-48 2024-08-17T16:56:28.927 INFO:tasks.rados.rados.0.smithi063.stdout:423: finishing rollback tid 2 to smithi06387500-48 2024-08-17T16:56:28.927 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 329 (ObjNum 128 snap 27 seq_num 128) dirty exists 2024-08-17T16:56:29.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:28 smithi063 ceph-mon[42680]: mgrmap e22: y(active, since 4m), standbys: x 2024-08-17T16:56:29.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:28 smithi063 ceph-mon[42680]: pgmap v238: 89 pgs: 89 active+clean; 203 MiB data, 616 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-08-17T16:56:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:28 smithi063 ceph-mon[31063]: mgrmap e22: y(active, since 4m), standbys: x 2024-08-17T16:56:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:28 smithi063 ceph-mon[31063]: pgmap v238: 89 pgs: 89 active+clean; 203 MiB data, 616 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-08-17T16:56:29.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:28 smithi162 ceph-mon[34633]: mgrmap e22: y(active, since 4m), standbys: x 2024-08-17T16:56:29.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:28 smithi162 ceph-mon[34633]: pgmap v238: 89 pgs: 89 active+clean; 203 MiB data, 616 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.8 MiB/s wr, 9 op/s 2024-08-17T16:56:29.839 INFO:tasks.rados.rados.0.smithi063.stdout:423: done (1 left) 2024-08-17T16:56:29.839 INFO:tasks.rados.rados.0.smithi063.stdout:424: done (0 left) 2024-08-17T16:56:29.840 INFO:tasks.rados.rados.0.smithi063.stdout:425: snap_remove snap 41 2024-08-17T16:56:29.916 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:29 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:56:29] "GET /metrics HTTP/1.1" 200 36129 "" "Prometheus/2.43.0" 2024-08-17T16:56:29.916 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:29 smithi063 ceph-mon[31063]: osdmap e172: 8 total, 8 up, 8 in 2024-08-17T16:56:29.916 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:29 smithi063 ceph-mon[31063]: osdmap e173: 8 total, 8 up, 8 in 2024-08-17T16:56:30.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:29 smithi063 ceph-mon[42680]: osdmap e172: 8 total, 8 up, 8 in 2024-08-17T16:56:30.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:29 smithi063 ceph-mon[42680]: osdmap e173: 8 total, 8 up, 8 in 2024-08-17T16:56:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:29 smithi162 ceph-mon[34633]: osdmap e172: 8 total, 8 up, 8 in 2024-08-17T16:56:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:29 smithi162 ceph-mon[34633]: osdmap e173: 8 total, 8 up, 8 in 2024-08-17T16:56:30.840 INFO:tasks.rados.rados.0.smithi063.stdout:425: done (0 left) 2024-08-17T16:56:30.840 INFO:tasks.rados.rados.0.smithi063.stdout:426: delete oid 27 current snap is 56 2024-08-17T16:56:30.844 INFO:tasks.rados.rados.0.smithi063.stdout:426: done (0 left) 2024-08-17T16:56:30.845 INFO:tasks.rados.rados.0.smithi063.stdout:427: rollback oid 33 current snap is 56 2024-08-17T16:56:30.845 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 33 to 31 2024-08-17T16:56:30.845 INFO:tasks.rados.rados.0.smithi063.stdout:428: read oid 45 snap -1 2024-08-17T16:56:30.845 INFO:tasks.rados.rados.0.smithi063.stdout:428: expect (ObjNum 44 snap 0 seq_num 44) 2024-08-17T16:56:30.845 INFO:tasks.rados.rados.0.smithi063.stdout:429: read oid 26 snap -1 2024-08-17T16:56:30.845 INFO:tasks.rados.rados.0.smithi063.stdout:429: expect (ObjNum 148 snap 34 seq_num 148) 2024-08-17T16:56:30.846 INFO:tasks.rados.rados.0.smithi063.stdout:430: write oid 43 current snap is 56 2024-08-17T16:56:30.846 INFO:tasks.rados.rados.0.smithi063.stdout:430: seq_num 199 ranges {544593=799729,1761799=776065,2948352=396241} 2024-08-17T16:56:30.853 INFO:tasks.rados.rados.0.smithi063.stdout:430: writing smithi06387500-43 from 544593 to 1344322 tid 1 2024-08-17T16:56:30.860 INFO:tasks.rados.rados.0.smithi063.stdout:430: writing smithi06387500-43 from 1761799 to 2537864 tid 2 2024-08-17T16:56:30.865 INFO:tasks.rados.rados.0.smithi063.stdout:430: writing smithi06387500-43 from 2948352 to 3344593 tid 3 2024-08-17T16:56:30.865 INFO:tasks.rados.rados.0.smithi063.stdout:431: read oid 47 snap -1 2024-08-17T16:56:30.865 INFO:tasks.rados.rados.0.smithi063.stdout:431: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:56:30.865 INFO:tasks.rados.rados.0.smithi063.stdout:432: read oid 31 snap -1 2024-08-17T16:56:30.865 INFO:tasks.rados.rados.0.smithi063.stdout:432: expect (ObjNum 138 snap 30 seq_num 138) 2024-08-17T16:56:30.865 INFO:tasks.rados.rados.0.smithi063.stdout:433: write oid 30 current snap is 56 2024-08-17T16:56:30.865 INFO:tasks.rados.rados.0.smithi063.stdout:433: seq_num 200 ranges {561400=474978,1820281=519723,2859422=457579} 2024-08-17T16:56:30.871 INFO:tasks.rados.rados.0.smithi063.stdout:433: writing smithi06387500-30 from 561400 to 1036378 tid 1 2024-08-17T16:56:30.878 INFO:tasks.rados.rados.0.smithi063.stdout:433: writing smithi06387500-30 from 1820281 to 2340004 tid 2 2024-08-17T16:56:30.883 INFO:tasks.rados.rados.0.smithi063.stdout:433: writing smithi06387500-30 from 2859422 to 3317001 tid 3 2024-08-17T16:56:30.884 INFO:tasks.rados.rados.0.smithi063.stdout:434: write oid 21 current snap is 56 2024-08-17T16:56:30.884 INFO:tasks.rados.rados.0.smithi063.stdout:427: finishing rollback tid 1 to smithi06387500-33 2024-08-17T16:56:30.884 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 93 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T16:56:30.884 INFO:tasks.rados.rados.0.smithi063.stdout:434: seq_num 201 ranges {578207=550227,1762410=779734,3286845=91362} 2024-08-17T16:56:30.889 INFO:tasks.rados.rados.0.smithi063.stdout:434: writing smithi06387500-21 from 578207 to 1128434 tid 1 2024-08-17T16:56:30.897 INFO:tasks.rados.rados.0.smithi063.stdout:434: writing smithi06387500-21 from 1762410 to 2542144 tid 2 2024-08-17T16:56:30.902 INFO:tasks.rados.rados.0.smithi063.stdout:434: writing smithi06387500-21 from 3286845 to 3378207 tid 3 2024-08-17T16:56:30.902 INFO:tasks.rados.rados.0.smithi063.stdout:427: done (7 left) 2024-08-17T16:56:30.902 INFO:tasks.rados.rados.0.smithi063.stdout:435: rollback oid 32 current snap is 56 2024-08-17T16:56:30.902 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 32 to 47 2024-08-17T16:56:30.902 INFO:tasks.rados.rados.0.smithi063.stdout:429: expect (ObjNum 148 snap 34 seq_num 148) 2024-08-17T16:56:31.028 INFO:tasks.rados.rados.0.smithi063.stdout:428: expect (ObjNum 44 snap 0 seq_num 44) 2024-08-17T16:56:31.118 INFO:tasks.rados.rados.0.smithi063.stdout:430: finishing write tid 1 to smithi06387500-43 2024-08-17T16:56:31.118 INFO:tasks.rados.rados.0.smithi063.stdout:430: finishing write tid 2 to smithi06387500-43 2024-08-17T16:56:31.118 INFO:tasks.rados.rados.0.smithi063.stdout:430: finishing write tid 3 to smithi06387500-43 2024-08-17T16:56:31.118 INFO:tasks.rados.rados.0.smithi063.stdout:430: finishing write tid 4 to smithi06387500-43 2024-08-17T16:56:31.119 INFO:tasks.rados.rados.0.smithi063.stdout:430: finishing write tid 5 to smithi06387500-43 2024-08-17T16:56:31.119 INFO:tasks.rados.rados.0.smithi063.stdout:430: oid 43 updating version 0 to 172 2024-08-17T16:56:31.119 INFO:tasks.rados.rados.0.smithi063.stdout:430: oid 43 version 172 is already newer than 171 2024-08-17T16:56:31.119 INFO:tasks.rados.rados.0.smithi063.stdout:430: oid 43 version 172 is already newer than 170 2024-08-17T16:56:31.119 INFO:tasks.rados.rados.0.smithi063.stdout:430: oid 43 version 172 is already newer than 169 2024-08-17T16:56:31.119 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 172 (ObjNum 199 snap 56 seq_num 199) dirty exists 2024-08-17T16:56:31.119 INFO:tasks.rados.rados.0.smithi063.stdout:430: left oid 43 (ObjNum 199 snap 56 seq_num 199) 2024-08-17T16:56:31.119 INFO:tasks.rados.rados.0.smithi063.stdout:431: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:56:31.178 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:30 smithi162 ceph-mon[34633]: pgmap v241: 89 pgs: 89 active+clean; 203 MiB data, 616 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:31.178 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:30 smithi162 ceph-mon[34633]: osdmap e174: 8 total, 8 up, 8 in 2024-08-17T16:56:31.272 INFO:tasks.rados.rados.0.smithi063.stdout:432: expect (ObjNum 138 snap 30 seq_num 138) 2024-08-17T16:56:31.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:30 smithi063 ceph-mon[42680]: pgmap v241: 89 pgs: 89 active+clean; 203 MiB data, 616 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:31.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:30 smithi063 ceph-mon[42680]: osdmap e174: 8 total, 8 up, 8 in 2024-08-17T16:56:31.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:30 smithi063 ceph-mon[31063]: pgmap v241: 89 pgs: 89 active+clean; 203 MiB data, 616 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:31.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:30 smithi063 ceph-mon[31063]: osdmap e174: 8 total, 8 up, 8 in 2024-08-17T16:56:31.406 INFO:tasks.rados.rados.0.smithi063.stdout:433: finishing write tid 1 to smithi06387500-30 2024-08-17T16:56:31.406 INFO:tasks.rados.rados.0.smithi063.stdout:433: finishing write tid 2 to smithi06387500-30 2024-08-17T16:56:31.407 INFO:tasks.rados.rados.0.smithi063.stdout:433: finishing write tid 3 to smithi06387500-30 2024-08-17T16:56:31.407 INFO:tasks.rados.rados.0.smithi063.stdout:433: finishing write tid 4 to smithi06387500-30 2024-08-17T16:56:31.407 INFO:tasks.rados.rados.0.smithi063.stdout:433: finishing write tid 5 to smithi06387500-30 2024-08-17T16:56:31.407 INFO:tasks.rados.rados.0.smithi063.stdout:433: oid 30 updating version 0 to 177 2024-08-17T16:56:31.407 INFO:tasks.rados.rados.0.smithi063.stdout:433: oid 30 version 177 is already newer than 176 2024-08-17T16:56:31.407 INFO:tasks.rados.rados.0.smithi063.stdout:433: oid 30 version 177 is already newer than 175 2024-08-17T16:56:31.407 INFO:tasks.rados.rados.0.smithi063.stdout:433: oid 30 version 177 is already newer than 174 2024-08-17T16:56:31.407 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 177 (ObjNum 200 snap 56 seq_num 200) dirty exists 2024-08-17T16:56:31.407 INFO:tasks.rados.rados.0.smithi063.stdout:433: left oid 30 (ObjNum 200 snap 56 seq_num 200) 2024-08-17T16:56:31.407 INFO:tasks.rados.rados.0.smithi063.stdout:434: finishing write tid 1 to smithi06387500-21 2024-08-17T16:56:31.407 INFO:tasks.rados.rados.0.smithi063.stdout:428: done (7 left) 2024-08-17T16:56:31.408 INFO:tasks.rados.rados.0.smithi063.stdout:429: done (6 left) 2024-08-17T16:56:31.408 INFO:tasks.rados.rados.0.smithi063.stdout:430: done (5 left) 2024-08-17T16:56:31.408 INFO:tasks.rados.rados.0.smithi063.stdout:431: done (4 left) 2024-08-17T16:56:31.408 INFO:tasks.rados.rados.0.smithi063.stdout:432: done (3 left) 2024-08-17T16:56:31.408 INFO:tasks.rados.rados.0.smithi063.stdout:433: done (2 left) 2024-08-17T16:56:31.408 INFO:tasks.rados.rados.0.smithi063.stdout:436: snap_remove snap 53 2024-08-17T16:56:31.930 INFO:tasks.rados.rados.0.smithi063.stdout:436: done (2 left) 2024-08-17T16:56:31.931 INFO:tasks.rados.rados.0.smithi063.stdout:437: write oid 11 current snap is 56 2024-08-17T16:56:31.931 INFO:tasks.rados.rados.0.smithi063.stdout:437: seq_num 202 ranges {595014=625476,1820892=523392,2797915=596476} 2024-08-17T16:56:31.938 INFO:tasks.rados.rados.0.smithi063.stdout:437: writing smithi06387500-11 from 595014 to 1220490 tid 1 2024-08-17T16:56:31.944 INFO:tasks.rados.rados.0.smithi063.stdout:437: writing smithi06387500-11 from 1820892 to 2344284 tid 2 2024-08-17T16:56:31.950 INFO:tasks.rados.rados.0.smithi063.stdout:437: writing smithi06387500-11 from 2797915 to 3394391 tid 3 2024-08-17T16:56:31.950 INFO:tasks.rados.rados.0.smithi063.stdout:438: write oid 1 current snap is 56 2024-08-17T16:56:31.950 INFO:tasks.rados.rados.0.smithi063.stdout:438: seq_num 203 ranges {611821=700725,1879374=783403,3341691=70130} 2024-08-17T16:56:31.957 INFO:tasks.rados.rados.0.smithi063.stdout:438: writing smithi06387500-1 from 611821 to 1312546 tid 1 2024-08-17T16:56:31.964 INFO:tasks.rados.rados.0.smithi063.stdout:438: writing smithi06387500-1 from 1879374 to 2662777 tid 2 2024-08-17T16:56:31.969 INFO:tasks.rados.rados.0.smithi063.stdout:438: writing smithi06387500-1 from 3341691 to 3411821 tid 3 2024-08-17T16:56:31.970 INFO:tasks.rados.rados.0.smithi063.stdout:439: snap_create 2024-08-17T16:56:31.970 INFO:tasks.rados.rados.0.smithi063.stdout:434: finishing write tid 2 to smithi06387500-21 2024-08-17T16:56:31.970 INFO:tasks.rados.rados.0.smithi063.stdout:434: finishing write tid 3 to smithi06387500-21 2024-08-17T16:56:31.970 INFO:tasks.rados.rados.0.smithi063.stdout:434: finishing write tid 4 to smithi06387500-21 2024-08-17T16:56:31.970 INFO:tasks.rados.rados.0.smithi063.stdout:434: finishing write tid 5 to smithi06387500-21 2024-08-17T16:56:31.970 INFO:tasks.rados.rados.0.smithi063.stdout:434: oid 21 updating version 0 to 230 2024-08-17T16:56:31.970 INFO:tasks.rados.rados.0.smithi063.stdout:434: oid 21 version 230 is already newer than 228 2024-08-17T16:56:31.970 INFO:tasks.rados.rados.0.smithi063.stdout:434: oid 21 version 230 is already newer than 229 2024-08-17T16:56:31.970 INFO:tasks.rados.rados.0.smithi063.stdout:434: oid 21 version 230 is already newer than 227 2024-08-17T16:56:31.970 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 230 (ObjNum 201 snap 56 seq_num 201) dirty exists 2024-08-17T16:56:31.970 INFO:tasks.rados.rados.0.smithi063.stdout:434: left oid 21 (ObjNum 201 snap 56 seq_num 201) 2024-08-17T16:56:31.970 INFO:tasks.rados.rados.0.smithi063.stdout:435: finishing rollback tid 0 to smithi06387500-32 2024-08-17T16:56:31.971 INFO:tasks.rados.rados.0.smithi063.stdout:435: finishing rollback tid 1 to smithi06387500-32 2024-08-17T16:56:31.971 INFO:tasks.rados.rados.0.smithi063.stdout:435: finishing rollback tid 2 to smithi06387500-32 2024-08-17T16:56:31.971 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 69 (ObjNum 167 snap 43 seq_num 167) dirty exists 2024-08-17T16:56:31.971 INFO:tasks.rados.rados.0.smithi063.stdout:437: finishing write tid 1 to smithi06387500-11 2024-08-17T16:56:31.971 INFO:tasks.rados.rados.0.smithi063.stdout:437: finishing write tid 2 to smithi06387500-11 2024-08-17T16:56:31.971 INFO:tasks.rados.rados.0.smithi063.stdout:437: finishing write tid 3 to smithi06387500-11 2024-08-17T16:56:31.971 INFO:tasks.rados.rados.0.smithi063.stdout:437: finishing write tid 4 to smithi06387500-11 2024-08-17T16:56:31.971 INFO:tasks.rados.rados.0.smithi063.stdout:437: finishing write tid 5 to smithi06387500-11 2024-08-17T16:56:31.971 INFO:tasks.rados.rados.0.smithi063.stdout:437: oid 11 updating version 0 to 66 2024-08-17T16:56:31.971 INFO:tasks.rados.rados.0.smithi063.stdout:437: oid 11 updating version 66 to 69 2024-08-17T16:56:31.971 INFO:tasks.rados.rados.0.smithi063.stdout:437: oid 11 version 69 is already newer than 67 2024-08-17T16:56:31.971 INFO:tasks.rados.rados.0.smithi063.stdout:437: oid 11 version 69 is already newer than 68 2024-08-17T16:56:31.971 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 69 (ObjNum 202 snap 56 seq_num 202) dirty exists 2024-08-17T16:56:31.972 INFO:tasks.rados.rados.0.smithi063.stdout:437: left oid 11 (ObjNum 202 snap 56 seq_num 202) 2024-08-17T16:56:31.972 INFO:tasks.rados.rados.0.smithi063.stdout:438: finishing write tid 1 to smithi06387500-1 2024-08-17T16:56:31.972 INFO:tasks.rados.rados.0.smithi063.stdout:438: finishing write tid 2 to smithi06387500-1 2024-08-17T16:56:31.972 INFO:tasks.rados.rados.0.smithi063.stdout:438: finishing write tid 3 to smithi06387500-1 2024-08-17T16:56:31.972 INFO:tasks.rados.rados.0.smithi063.stdout:438: finishing write tid 4 to smithi06387500-1 2024-08-17T16:56:31.972 INFO:tasks.rados.rados.0.smithi063.stdout:438: finishing write tid 5 to smithi06387500-1 2024-08-17T16:56:31.973 INFO:tasks.rados.rados.0.smithi063.stdout:438: oid 1 updating version 0 to 160 2024-08-17T16:56:31.973 INFO:tasks.rados.rados.0.smithi063.stdout:438: oid 1 version 160 is already newer than 158 2024-08-17T16:56:31.973 INFO:tasks.rados.rados.0.smithi063.stdout:438: oid 1 version 160 is already newer than 159 2024-08-17T16:56:31.973 INFO:tasks.rados.rados.0.smithi063.stdout:438: oid 1 version 160 is already newer than 155 2024-08-17T16:56:31.973 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 160 (ObjNum 203 snap 56 seq_num 203) dirty exists 2024-08-17T16:56:31.973 INFO:tasks.rados.rados.0.smithi063.stdout:438: left oid 1 (ObjNum 203 snap 56 seq_num 203) 2024-08-17T16:56:32.935 INFO:tasks.rados.rados.0.smithi063.stdout:434: done (4 left) 2024-08-17T16:56:32.935 INFO:tasks.rados.rados.0.smithi063.stdout:435: done (3 left) 2024-08-17T16:56:32.935 INFO:tasks.rados.rados.0.smithi063.stdout:437: done (2 left) 2024-08-17T16:56:32.935 INFO:tasks.rados.rados.0.smithi063.stdout:438: done (1 left) 2024-08-17T16:56:32.935 INFO:tasks.rados.rados.0.smithi063.stdout:439: done (0 left) 2024-08-17T16:56:32.935 INFO:tasks.rados.rados.0.smithi063.stdout:440: snap_remove snap 44 2024-08-17T16:56:33.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:32 smithi063 ceph-mon[42680]: osdmap e175: 8 total, 8 up, 8 in 2024-08-17T16:56:33.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:32 smithi063 ceph-mon[42680]: pgmap v244: 89 pgs: 89 active+clean; 215 MiB data, 639 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 2.6 MiB/s wr, 23 op/s 2024-08-17T16:56:33.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:32 smithi063 ceph-mon[31063]: osdmap e175: 8 total, 8 up, 8 in 2024-08-17T16:56:33.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:32 smithi063 ceph-mon[31063]: pgmap v244: 89 pgs: 89 active+clean; 215 MiB data, 639 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 2.6 MiB/s wr, 23 op/s 2024-08-17T16:56:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:32 smithi162 ceph-mon[34633]: osdmap e175: 8 total, 8 up, 8 in 2024-08-17T16:56:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:32 smithi162 ceph-mon[34633]: pgmap v244: 89 pgs: 89 active+clean; 215 MiB data, 639 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 2.6 MiB/s wr, 23 op/s 2024-08-17T16:56:33.937 INFO:tasks.rados.rados.0.smithi063.stdout:440: done (0 left) 2024-08-17T16:56:33.937 INFO:tasks.rados.rados.0.smithi063.stdout:441: read oid 36 snap -1 2024-08-17T16:56:33.937 INFO:tasks.rados.rados.0.smithi063.stdout:441: expect (ObjNum 134 snap 28 seq_num 134) 2024-08-17T16:56:33.937 INFO:tasks.rados.rados.0.smithi063.stdout:442: read oid 35 snap 49 2024-08-17T16:56:33.937 INFO:tasks.rados.rados.0.smithi063.stdout:442: expect (ObjNum 179 snap 47 seq_num 179) 2024-08-17T16:56:33.938 INFO:tasks.rados.rados.0.smithi063.stdout:443: snap_remove snap 56 2024-08-17T16:56:34.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:33 smithi063 ceph-mon[42680]: osdmap e176: 8 total, 8 up, 8 in 2024-08-17T16:56:34.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:33 smithi063 ceph-mon[31063]: osdmap e176: 8 total, 8 up, 8 in 2024-08-17T16:56:34.288 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:33 smithi162 ceph-mon[34633]: osdmap e176: 8 total, 8 up, 8 in 2024-08-17T16:56:34.840 INFO:tasks.rados.rados.0.smithi063.stdout:443: done (2 left) 2024-08-17T16:56:34.840 INFO:tasks.rados.rados.0.smithi063.stdout:444: rollback oid 37 current snap is 57 2024-08-17T16:56:34.840 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 48 2024-08-17T16:56:34.840 INFO:tasks.rados.rados.0.smithi063.stdout:441: expect (ObjNum 134 snap 28 seq_num 134) 2024-08-17T16:56:34.915 INFO:tasks.rados.rados.0.smithi063.stdout:442: expect (ObjNum 179 snap 47 seq_num 179) 2024-08-17T16:56:35.050 INFO:tasks.rados.rados.0.smithi063.stdout:444: finishing rollback tid 0 to smithi06387500-37 2024-08-17T16:56:35.050 INFO:tasks.rados.rados.0.smithi063.stdout:444: finishing rollback tid 1 to smithi06387500-37 2024-08-17T16:56:35.050 INFO:tasks.rados.rados.0.smithi063.stdout:444: finishing rollback tid 2 to smithi06387500-37 2024-08-17T16:56:35.051 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 351 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-08-17T16:56:35.051 INFO:tasks.rados.rados.0.smithi063.stdout:441: done (2 left) 2024-08-17T16:56:35.051 INFO:tasks.rados.rados.0.smithi063.stdout:442: done (1 left) 2024-08-17T16:56:35.051 INFO:tasks.rados.rados.0.smithi063.stdout:444: done (0 left) 2024-08-17T16:56:35.051 INFO:tasks.rados.rados.0.smithi063.stdout:445: snap_remove snap 48 2024-08-17T16:56:35.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:34 smithi063 ceph-mon[42680]: osdmap e177: 8 total, 8 up, 8 in 2024-08-17T16:56:35.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:34 smithi063 ceph-mon[42680]: pgmap v247: 89 pgs: 89 active+clean; 215 MiB data, 639 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 2.6 MiB/s wr, 23 op/s 2024-08-17T16:56:35.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:34 smithi063 ceph-mon[42680]: osdmap e178: 8 total, 8 up, 8 in 2024-08-17T16:56:35.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:34 smithi063 ceph-mon[31063]: osdmap e177: 8 total, 8 up, 8 in 2024-08-17T16:56:35.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:34 smithi063 ceph-mon[31063]: pgmap v247: 89 pgs: 89 active+clean; 215 MiB data, 639 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 2.6 MiB/s wr, 23 op/s 2024-08-17T16:56:35.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:34 smithi063 ceph-mon[31063]: osdmap e178: 8 total, 8 up, 8 in 2024-08-17T16:56:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:34 smithi162 ceph-mon[34633]: osdmap e177: 8 total, 8 up, 8 in 2024-08-17T16:56:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:34 smithi162 ceph-mon[34633]: pgmap v247: 89 pgs: 89 active+clean; 215 MiB data, 639 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 2.6 MiB/s wr, 23 op/s 2024-08-17T16:56:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:34 smithi162 ceph-mon[34633]: osdmap e178: 8 total, 8 up, 8 in 2024-08-17T16:56:35.950 INFO:tasks.rados.rados.0.smithi063.stdout:445: done (0 left) 2024-08-17T16:56:35.951 INFO:tasks.rados.rados.0.smithi063.stdout:446: write oid 49 current snap is 57 2024-08-17T16:56:35.951 INFO:tasks.rados.rados.0.smithi063.stdout:446: seq_num 204 ranges {628628=775974,1937856=527061,3252761=175867} 2024-08-17T16:56:35.959 INFO:tasks.rados.rados.0.smithi063.stdout:446: writing smithi06387500-49 from 628628 to 1404602 tid 1 2024-08-17T16:56:35.965 INFO:tasks.rados.rados.0.smithi063.stdout:446: writing smithi06387500-49 from 1937856 to 2464917 tid 2 2024-08-17T16:56:35.971 INFO:tasks.rados.rados.0.smithi063.stdout:446: writing smithi06387500-49 from 3252761 to 3428628 tid 3 2024-08-17T16:56:35.971 INFO:tasks.rados.rados.0.smithi063.stdout:447: read oid 34 snap 55 2024-08-17T16:56:35.971 INFO:tasks.rados.rados.0.smithi063.stdout:447: expect (ObjNum 112 snap 18 seq_num 112) 2024-08-17T16:56:35.972 INFO:tasks.rados.rados.0.smithi063.stdout:448: read oid 8 snap -1 2024-08-17T16:56:35.972 INFO:tasks.rados.rados.0.smithi063.stdout:448: expect (ObjNum 193 snap 52 seq_num 193) 2024-08-17T16:56:35.972 INFO:tasks.rados.rados.0.smithi063.stdout:449: write oid 42 current snap is 57 2024-08-17T16:56:35.972 INFO:tasks.rados.rados.0.smithi063.stdout:446: finishing write tid 1 to smithi06387500-49 2024-08-17T16:56:35.972 INFO:tasks.rados.rados.0.smithi063.stdout:449: seq_num 205 ranges {645435=451223,1879985=787072,3280184=165251} 2024-08-17T16:56:35.978 INFO:tasks.rados.rados.0.smithi063.stdout:449: writing smithi06387500-42 from 645435 to 1096658 tid 1 2024-08-17T16:56:35.986 INFO:tasks.rados.rados.0.smithi063.stdout:449: writing smithi06387500-42 from 1879985 to 2667057 tid 2 2024-08-17T16:56:35.991 INFO:tasks.rados.rados.0.smithi063.stdout:449: writing smithi06387500-42 from 3280184 to 3445435 tid 3 2024-08-17T16:56:35.991 INFO:tasks.rados.rados.0.smithi063.stdout:450: snap_create 2024-08-17T16:56:35.991 INFO:tasks.rados.rados.0.smithi063.stdout:446: finishing write tid 2 to smithi06387500-49 2024-08-17T16:56:35.991 INFO:tasks.rados.rados.0.smithi063.stdout:446: finishing write tid 3 to smithi06387500-49 2024-08-17T16:56:35.991 INFO:tasks.rados.rados.0.smithi063.stdout:446: finishing write tid 4 to smithi06387500-49 2024-08-17T16:56:35.992 INFO:tasks.rados.rados.0.smithi063.stdout:446: finishing write tid 5 to smithi06387500-49 2024-08-17T16:56:35.992 INFO:tasks.rados.rados.0.smithi063.stdout:446: oid 49 updating version 0 to 364 2024-08-17T16:56:35.992 INFO:tasks.rados.rados.0.smithi063.stdout:446: oid 49 version 364 is already newer than 363 2024-08-17T16:56:35.992 INFO:tasks.rados.rados.0.smithi063.stdout:446: oid 49 version 364 is already newer than 362 2024-08-17T16:56:35.992 INFO:tasks.rados.rados.0.smithi063.stdout:446: oid 49 version 364 is already newer than 361 2024-08-17T16:56:35.992 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 364 (ObjNum 204 snap 57 seq_num 204) dirty exists 2024-08-17T16:56:35.992 INFO:tasks.rados.rados.0.smithi063.stdout:446: left oid 49 (ObjNum 204 snap 57 seq_num 204) 2024-08-17T16:56:35.992 INFO:tasks.rados.rados.0.smithi063.stdout:447: expect (ObjNum 112 snap 18 seq_num 112) 2024-08-17T16:56:36.175 INFO:tasks.rados.rados.0.smithi063.stdout:448: expect (ObjNum 193 snap 52 seq_num 193) 2024-08-17T16:56:36.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:35 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:36.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:35 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:36.281 INFO:tasks.rados.rados.0.smithi063.stdout:449: finishing write tid 1 to smithi06387500-42 2024-08-17T16:56:36.281 INFO:tasks.rados.rados.0.smithi063.stdout:449: finishing write tid 2 to smithi06387500-42 2024-08-17T16:56:36.281 INFO:tasks.rados.rados.0.smithi063.stdout:449: finishing write tid 3 to smithi06387500-42 2024-08-17T16:56:36.281 INFO:tasks.rados.rados.0.smithi063.stdout:449: finishing write tid 4 to smithi06387500-42 2024-08-17T16:56:36.282 INFO:tasks.rados.rados.0.smithi063.stdout:449: finishing write tid 5 to smithi06387500-42 2024-08-17T16:56:36.282 INFO:tasks.rados.rados.0.smithi063.stdout:449: oid 42 updating version 0 to 244 2024-08-17T16:56:36.282 INFO:tasks.rados.rados.0.smithi063.stdout:449: oid 42 version 244 is already newer than 241 2024-08-17T16:56:36.282 INFO:tasks.rados.rados.0.smithi063.stdout:449: oid 42 version 244 is already newer than 243 2024-08-17T16:56:36.282 INFO:tasks.rados.rados.0.smithi063.stdout:449: oid 42 version 244 is already newer than 242 2024-08-17T16:56:36.282 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 244 (ObjNum 205 snap 57 seq_num 205) dirty exists 2024-08-17T16:56:36.282 INFO:tasks.rados.rados.0.smithi063.stdout:449: left oid 42 (ObjNum 205 snap 57 seq_num 205) 2024-08-17T16:56:36.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:35 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:36.969 INFO:tasks.rados.rados.0.smithi063.stdout:446: done (4 left) 2024-08-17T16:56:36.969 INFO:tasks.rados.rados.0.smithi063.stdout:447: done (3 left) 2024-08-17T16:56:36.969 INFO:tasks.rados.rados.0.smithi063.stdout:448: done (2 left) 2024-08-17T16:56:36.969 INFO:tasks.rados.rados.0.smithi063.stdout:449: done (1 left) 2024-08-17T16:56:36.969 INFO:tasks.rados.rados.0.smithi063.stdout:450: done (0 left) 2024-08-17T16:56:36.969 INFO:tasks.rados.rados.0.smithi063.stdout:451: snap_create 2024-08-17T16:56:37.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:36 smithi063 ceph-mon[42680]: osdmap e179: 8 total, 8 up, 8 in 2024-08-17T16:56:37.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:36 smithi063 ceph-mon[42680]: pgmap v250: 89 pgs: 89 active+clean; 224 MiB data, 655 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-08-17T16:56:37.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:36 smithi063 ceph-mon[31063]: osdmap e179: 8 total, 8 up, 8 in 2024-08-17T16:56:37.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:36 smithi063 ceph-mon[31063]: pgmap v250: 89 pgs: 89 active+clean; 224 MiB data, 655 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-08-17T16:56:37.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:36 smithi162 ceph-mon[34633]: osdmap e179: 8 total, 8 up, 8 in 2024-08-17T16:56:37.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:36 smithi162 ceph-mon[34633]: pgmap v250: 89 pgs: 89 active+clean; 224 MiB data, 655 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-08-17T16:56:37.969 INFO:tasks.rados.rados.0.smithi063.stdout:451: done (0 left) 2024-08-17T16:56:37.969 INFO:tasks.rados.rados.0.smithi063.stdout:452: delete oid 44 current snap is 59 2024-08-17T16:56:37.972 INFO:tasks.rados.rados.0.smithi063.stdout:452: done (0 left) 2024-08-17T16:56:37.972 INFO:tasks.rados.rados.0.smithi063.stdout:453: rollback oid 27 current snap is 59 2024-08-17T16:56:37.972 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 27 to 49 2024-08-17T16:56:37.972 INFO:tasks.rados.rados.0.smithi063.stdout:454: rollback oid 7 current snap is 59 2024-08-17T16:56:37.972 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 42 2024-08-17T16:56:37.972 INFO:tasks.rados.rados.0.smithi063.stdout:455: read oid 25 snap -1 2024-08-17T16:56:37.972 INFO:tasks.rados.rados.0.smithi063.stdout:455: expect (ObjNum 194 snap 54 seq_num 194) 2024-08-17T16:56:37.972 INFO:tasks.rados.rados.0.smithi063.stdout:456: read oid 48 snap -1 2024-08-17T16:56:37.972 INFO:tasks.rados.rados.0.smithi063.stdout:456: expect (ObjNum 128 snap 27 seq_num 128) 2024-08-17T16:56:37.972 INFO:tasks.rados.rados.0.smithi063.stdout:457: write oid 8 current snap is 59 2024-08-17T16:56:37.972 INFO:tasks.rados.rados.0.smithi063.stdout:457: seq_num 207 ranges {679049=718074,2113302=674388,3335030=144019} 2024-08-17T16:56:37.980 INFO:tasks.rados.rados.0.smithi063.stdout:457: writing smithi06387500-8 from 679049 to 1397123 tid 1 2024-08-17T16:56:37.988 INFO:tasks.rados.rados.0.smithi063.stdout:457: writing smithi06387500-8 from 2113302 to 2787690 tid 2 2024-08-17T16:56:37.992 INFO:tasks.rados.rados.0.smithi063.stdout:457: writing smithi06387500-8 from 3335030 to 3479049 tid 3 2024-08-17T16:56:37.992 INFO:tasks.rados.rados.0.smithi063.stdout:458: snap_remove snap 52 2024-08-17T16:56:38.245 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:37 smithi162 ceph-mon[34633]: osdmap e180: 8 total, 8 up, 8 in 2024-08-17T16:56:38.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:37 smithi063 ceph-mon[42680]: osdmap e180: 8 total, 8 up, 8 in 2024-08-17T16:56:38.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:37 smithi063 ceph-mon[31063]: osdmap e180: 8 total, 8 up, 8 in 2024-08-17T16:56:38.428 INFO:tasks.rados.rados.0.smithi063.stdout:458: done (5 left) 2024-08-17T16:56:38.428 INFO:tasks.rados.rados.0.smithi063.stdout:459: write oid 44 current snap is 59 2024-08-17T16:56:38.428 INFO:tasks.rados.rados.0.smithi063.stdout:454: finishing rollback tid 0 to smithi06387500-7 2024-08-17T16:56:38.428 INFO:tasks.rados.rados.0.smithi063.stdout:454: finishing rollback tid 1 to smithi06387500-7 2024-08-17T16:56:38.428 INFO:tasks.rados.rados.0.smithi063.stdout:454: finishing rollback tid 2 to smithi06387500-7 2024-08-17T16:56:38.428 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 199 (ObjNum 156 snap 39 seq_num 156) dirty exists 2024-08-17T16:56:38.428 INFO:tasks.rados.rados.0.smithi063.stdout:453: finishing rollback tid 1 to smithi06387500-27 2024-08-17T16:56:38.428 INFO:tasks.rados.rados.0.smithi063.stdout:453: finishing rollback tid 2 to smithi06387500-27 2024-08-17T16:56:38.428 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 46 (ObjNum 172 snap 45 seq_num 172) dirty exists 2024-08-17T16:56:38.428 INFO:tasks.rados.rados.0.smithi063.stdout:457: finishing write tid 1 to smithi06387500-8 2024-08-17T16:56:38.429 INFO:tasks.rados.rados.0.smithi063.stdout:455: expect (ObjNum 194 snap 54 seq_num 194) 2024-08-17T16:56:38.596 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: [17/Aug/2024:16:56:38] ENGINE Bus STOPPING 2024-08-17T16:56:38.666 INFO:tasks.rados.rados.0.smithi063.stdout:456: expect (ObjNum 128 snap 27 seq_num 128) 2024-08-17T16:56:38.727 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:38.423+0000 7f5de2d96640 -1 mgr handle_mgr_map I was active but no longer am 2024-08-17T16:56:38.727 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ignoring --setuser ceph since I am not root 2024-08-17T16:56:38.727 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ignoring --setgroup ceph since I am not root 2024-08-17T16:56:38.727 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:38.573+0000 7f12489e5040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-08-17T16:56:38.727 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:38.725+0000 7f12489e5040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:56:38.795 INFO:tasks.rados.rados.0.smithi063.stdout:457: finishing write tid 2 to smithi06387500-8 2024-08-17T16:56:38.795 INFO:tasks.rados.rados.0.smithi063.stdout:457: finishing write tid 3 to smithi06387500-8 2024-08-17T16:56:38.795 INFO:tasks.rados.rados.0.smithi063.stdout:457: finishing write tid 4 to smithi06387500-8 2024-08-17T16:56:38.795 INFO:tasks.rados.rados.0.smithi063.stdout:457: finishing write tid 5 to smithi06387500-8 2024-08-17T16:56:38.795 INFO:tasks.rados.rados.0.smithi063.stdout:457: oid 8 updating version 0 to 128 2024-08-17T16:56:38.796 INFO:tasks.rados.rados.0.smithi063.stdout:457: oid 8 updating version 128 to 129 2024-08-17T16:56:38.796 INFO:tasks.rados.rados.0.smithi063.stdout:457: oid 8 updating version 129 to 130 2024-08-17T16:56:38.796 INFO:tasks.rados.rados.0.smithi063.stdout:457: oid 8 updating version 130 to 131 2024-08-17T16:56:38.796 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 131 (ObjNum 207 snap 59 seq_num 207) dirty exists 2024-08-17T16:56:38.796 INFO:tasks.rados.rados.0.smithi063.stdout:457: left oid 8 (ObjNum 207 snap 59 seq_num 207) 2024-08-17T16:56:38.796 INFO:tasks.rados.rados.0.smithi063.stdout:459: seq_num 208 ranges {695856=793323,2171784=534399,3362453=133403} 2024-08-17T16:56:38.803 INFO:tasks.rados.rados.0.smithi063.stdout:459: writing smithi06387500-44 from 695856 to 1489179 tid 1 2024-08-17T16:56:38.810 INFO:tasks.rados.rados.0.smithi063.stdout:459: writing smithi06387500-44 from 2171784 to 2706183 tid 2 2024-08-17T16:56:38.814 INFO:tasks.rados.rados.0.smithi063.stdout:459: writing smithi06387500-44 from 3362453 to 3495856 tid 3 2024-08-17T16:56:38.815 INFO:tasks.rados.rados.0.smithi063.stdout:453: done (5 left) 2024-08-17T16:56:38.815 INFO:tasks.rados.rados.0.smithi063.stdout:454: done (4 left) 2024-08-17T16:56:38.815 INFO:tasks.rados.rados.0.smithi063.stdout:455: done (3 left) 2024-08-17T16:56:38.815 INFO:tasks.rados.rados.0.smithi063.stdout:456: done (2 left) 2024-08-17T16:56:38.815 INFO:tasks.rados.rados.0.smithi063.stdout:457: done (1 left) 2024-08-17T16:56:38.815 INFO:tasks.rados.rados.0.smithi063.stdout:460: delete oid 19 current snap is 59 2024-08-17T16:56:38.815 INFO:tasks.rados.rados.0.smithi063.stdout:459: finishing write tid 1 to smithi06387500-44 2024-08-17T16:56:38.815 INFO:tasks.rados.rados.0.smithi063.stdout:459: finishing write tid 2 to smithi06387500-44 2024-08-17T16:56:38.816 INFO:tasks.rados.rados.0.smithi063.stdout:459: finishing write tid 3 to smithi06387500-44 2024-08-17T16:56:38.816 INFO:tasks.rados.rados.0.smithi063.stdout:460: done (1 left) 2024-08-17T16:56:38.816 INFO:tasks.rados.rados.0.smithi063.stdout:461: rollback oid 10 current snap is 59 2024-08-17T16:56:38.816 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 10 to 55 2024-08-17T16:56:38.816 INFO:tasks.rados.rados.0.smithi063.stdout:459: finishing write tid 4 to smithi06387500-44 2024-08-17T16:56:38.816 INFO:tasks.rados.rados.0.smithi063.stdout:462: rollback oid 20 current snap is 59 2024-08-17T16:56:38.817 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 20 to 50 2024-08-17T16:56:38.817 INFO:tasks.rados.rados.0.smithi063.stdout:463: snap_remove snap 40 2024-08-17T16:56:39.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:38.795+0000 7f12489e5040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-08-17T16:56:39.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: osdmap e181: 8 total, 8 up, 8 in 2024-08-17T16:56:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:56:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:56:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:56:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-17T16:56:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: Activating manager daemon x 2024-08-17T16:56:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:56:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: osdmap e182: 8 total, 8 up, 8 in 2024-08-17T16:56:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:56:39.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:56:39.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi063.ultpqj"}]: dispatch 2024-08-17T16:56:39.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-17T16:56:39.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: mgrmap e23: x(active, starting, since 0.0134028s) 2024-08-17T16:56:39.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi162.bjkxnv"}]: dispatch 2024-08-17T16:56:39.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:56:39.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:56:39.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:56:39.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:56:39.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:56:39.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:56:39.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:56:39.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:56:39.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:56:39.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:56:39.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:56:39.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:56:39.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[31063]: Manager daemon x is now available 2024-08-17T16:56:39.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: osdmap e181: 8 total, 8 up, 8 in 2024-08-17T16:56:39.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:39.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:39.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:56:39.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:56:39.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:39.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:56:39.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-17T16:56:39.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: Activating manager daemon x 2024-08-17T16:56:39.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:56:39.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: osdmap e182: 8 total, 8 up, 8 in 2024-08-17T16:56:39.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:56:39.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:56:39.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi063.ultpqj"}]: dispatch 2024-08-17T16:56:39.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-17T16:56:39.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: mgrmap e23: x(active, starting, since 0.0134028s) 2024-08-17T16:56:39.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi162.bjkxnv"}]: dispatch 2024-08-17T16:56:39.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:56:39.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:56:39.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:56:39.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:56:39.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:56:39.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:56:39.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:56:39.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:56:39.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:56:39.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:56:39.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:56:39.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:56:39.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:38 smithi063 ceph-mon[42680]: Manager daemon x is now available 2024-08-17T16:56:39.095 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: [17/Aug/2024:16:56:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T16:56:39.095 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: [17/Aug/2024:16:56:38] ENGINE Bus STOPPED 2024-08-17T16:56:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: osdmap e181: 8 total, 8 up, 8 in 2024-08-17T16:56:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:56:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:56:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' 2024-08-17T16:56:39.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:56:39.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-08-17T16:56:39.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: Activating manager daemon x 2024-08-17T16:56:39.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:56:39.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: osdmap e182: 8 total, 8 up, 8 in 2024-08-17T16:56:39.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:56:39.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:56:39.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi063.ultpqj"}]: dispatch 2024-08-17T16:56:39.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24335 172.21.15.63:0/4006945076' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-08-17T16:56:39.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: mgrmap e23: x(active, starting, since 0.0134028s) 2024-08-17T16:56:39.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi162.bjkxnv"}]: dispatch 2024-08-17T16:56:39.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:56:39.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:56:39.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:56:39.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:56:39.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:56:39.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:56:39.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:56:39.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:56:39.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:56:39.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:56:39.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:56:39.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:56:39.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:38 smithi162 ceph-mon[34633]: Manager daemon x is now available 2024-08-17T16:56:39.439 INFO:tasks.rados.rados.0.smithi063.stdout:463: done (3 left) 2024-08-17T16:56:39.439 INFO:tasks.rados.rados.0.smithi063.stdout:464: delete oid 34 current snap is 59 2024-08-17T16:56:39.439 INFO:tasks.rados.rados.0.smithi063.stdout:459: finishing write tid 5 to smithi06387500-44 2024-08-17T16:56:39.439 INFO:tasks.rados.rados.0.smithi063.stdout:459: oid 44 updating version 0 to 78 2024-08-17T16:56:39.439 INFO:tasks.rados.rados.0.smithi063.stdout:459: oid 44 version 78 is already newer than 77 2024-08-17T16:56:39.439 INFO:tasks.rados.rados.0.smithi063.stdout:459: oid 44 updating version 78 to 79 2024-08-17T16:56:39.439 INFO:tasks.rados.rados.0.smithi063.stdout:459: oid 44 updating version 79 to 80 2024-08-17T16:56:39.439 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 80 (ObjNum 208 snap 59 seq_num 208) dirty exists 2024-08-17T16:56:39.439 INFO:tasks.rados.rados.0.smithi063.stdout:459: left oid 44 (ObjNum 208 snap 59 seq_num 208) 2024-08-17T16:56:39.439 INFO:tasks.rados.rados.0.smithi063.stdout:462: finishing rollback tid 1 to smithi06387500-20 2024-08-17T16:56:39.439 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 0 (ObjNum 65 snap 0 seq_num 1807178256) dirty dne 2024-08-17T16:56:39.440 INFO:tasks.rados.rados.0.smithi063.stdout:461: finishing rollback tid 0 to smithi06387500-10 2024-08-17T16:56:39.440 INFO:tasks.rados.rados.0.smithi063.stdout:461: finishing rollback tid 1 to smithi06387500-10 2024-08-17T16:56:39.440 INFO:tasks.rados.rados.0.smithi063.stdout:461: finishing rollback tid 2 to smithi06387500-10 2024-08-17T16:56:39.440 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 108 (ObjNum 100 snap 16 seq_num 100) dirty exists 2024-08-17T16:56:39.442 INFO:tasks.rados.rados.0.smithi063.stdout:459: done (3 left) 2024-08-17T16:56:39.442 INFO:tasks.rados.rados.0.smithi063.stdout:461: done (2 left) 2024-08-17T16:56:39.442 INFO:tasks.rados.rados.0.smithi063.stdout:462: done (1 left) 2024-08-17T16:56:39.442 INFO:tasks.rados.rados.0.smithi063.stdout:464: done (0 left) 2024-08-17T16:56:39.442 INFO:tasks.rados.rados.0.smithi063.stdout:465: snap_create 2024-08-17T16:56:39.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:39.051+0000 7f12489e5040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-08-17T16:56:39.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:39.113+0000 7f12489e5040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-08-17T16:56:39.845 INFO:tasks.rados.rados.0.smithi063.stdout:465: done (0 left) 2024-08-17T16:56:39.846 INFO:tasks.rados.rados.0.smithi063.stdout:466: read oid 26 snap -1 2024-08-17T16:56:39.846 INFO:tasks.rados.rados.0.smithi063.stdout:466: expect (ObjNum 148 snap 34 seq_num 148) 2024-08-17T16:56:39.846 INFO:tasks.rados.rados.0.smithi063.stdout:467: delete oid 16 current snap is 60 2024-08-17T16:56:39.848 INFO:tasks.rados.rados.0.smithi063.stdout:467: done (1 left) 2024-08-17T16:56:39.848 INFO:tasks.rados.rados.0.smithi063.stdout:468: snap_remove snap 42 2024-08-17T16:56:40.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: [17/Aug/2024:16:56:39] ENGINE Bus STARTING 2024-08-17T16:56:40.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: 2024-08-17T16:56:39.763+0000 7f0ebbc58640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:56:40.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: 2024-08-17T16:56:39.763+0000 7f0ebbc58640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:56:40.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: 2024-08-17T16:56:39.763+0000 7f0ebbc58640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:56:40.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: 2024-08-17T16:56:39.763+0000 7f0ebbc58640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:56:40.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: 2024-08-17T16:56:39.763+0000 7f0ebbc58640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:56:40.097 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: 2024-08-17T16:56:39.764+0000 7f0ebec5e640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:56:40.097 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: 2024-08-17T16:56:39.764+0000 7f0ebec5e640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:56:40.097 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: 2024-08-17T16:56:39.764+0000 7f0ebec5e640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:56:40.097 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: 2024-08-17T16:56:39.764+0000 7f0ebec5e640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:56:40.097 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: 2024-08-17T16:56:39.764+0000 7f0ebec5e640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:56:40.097 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: [17/Aug/2024:16:56:39] ENGINE Serving on http://:::9283 2024-08-17T16:56:40.097 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: [17/Aug/2024:16:56:39] ENGINE Bus STARTED 2024-08-17T16:56:40.227 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:40.102+0000 7f12489e5040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-08-17T16:56:40.505 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:40.226+0000 7f12489e5040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-08-17T16:56:40.505 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:40.289+0000 7f12489e5040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-08-17T16:56:40.505 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: mgrmap e24: x(active, since 1.01531s) 2024-08-17T16:56:40.505 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: osdmap e183: 8 total, 8 up, 8 in 2024-08-17T16:56:40.505 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:40.506 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:40.506 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: Checking for cert/key for grafana.a 2024-08-17T16:56:40.506 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: Migrating grafana.a cert to cert store 2024-08-17T16:56:40.506 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:40.506 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: Migrating grafana.a key to cert store 2024-08-17T16:56:40.506 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:40.506 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:40.506 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:56:40.506 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:40.506 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:56:40.506 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:56:40.506 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:56:40.506 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:56:40.507 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T16:56:40.507 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T16:56:40.507 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[42680]: osdmap e184: 8 total, 8 up, 8 in 2024-08-17T16:56:40.507 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: mgrmap e24: x(active, since 1.01531s) 2024-08-17T16:56:40.507 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: osdmap e183: 8 total, 8 up, 8 in 2024-08-17T16:56:40.507 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:40.507 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:40.507 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: Checking for cert/key for grafana.a 2024-08-17T16:56:40.507 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: Migrating grafana.a cert to cert store 2024-08-17T16:56:40.507 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:40.507 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: Migrating grafana.a key to cert store 2024-08-17T16:56:40.507 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:40.508 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:40.508 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:56:40.508 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:40.508 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:56:40.508 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:56:40.508 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:56:40.508 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:56:40.508 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T16:56:40.508 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T16:56:40.508 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-mon[31063]: osdmap e184: 8 total, 8 up, 8 in 2024-08-17T16:56:40.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:40.514+0000 7f12489e5040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-08-17T16:56:40.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:40.578+0000 7f12489e5040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-08-17T16:56:40.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:40.650+0000 7f12489e5040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-08-17T16:56:40.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: mgrmap e24: x(active, since 1.01531s) 2024-08-17T16:56:40.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: osdmap e183: 8 total, 8 up, 8 in 2024-08-17T16:56:40.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:40.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:40.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: Checking for cert/key for grafana.a 2024-08-17T16:56:40.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: Migrating grafana.a cert to cert store 2024-08-17T16:56:40.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:40.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: Migrating grafana.a key to cert store 2024-08-17T16:56:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:56:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:56:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:56:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:56:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:56:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T16:56:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-08-17T16:56:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-mon[34633]: osdmap e184: 8 total, 8 up, 8 in 2024-08-17T16:56:40.847 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: 2024-08-17T16:56:40.533+0000 7f0ea7973640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:16:56:40] ENGINE Error in HTTPServer.serve 2024-08-17T16:56:40.848 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: Traceback (most recent call last): 2024-08-17T16:56:40.848 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:56:40.848 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: self._connections.run(self.expiration_interval) 2024-08-17T16:56:40.848 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:56:40.848 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: self._run(expiration_interval) 2024-08-17T16:56:40.848 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:56:40.848 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:56:40.848 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:56:40.848 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:56:40.848 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:56:40.848 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: s = self.context.wrap_socket( 2024-08-17T16:56:40.848 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:56:40.848 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: return self.sslsocket_class._create( 2024-08-17T16:56:40.848 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:56:40.849 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: self.do_handshake() 2024-08-17T16:56:40.849 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:56:40.849 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: self._sslobj.do_handshake() 2024-08-17T16:56:40.849 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:56:40.849 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:56:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: 2024-08-17T16:56:40.849 INFO:tasks.rados.rados.0.smithi063.stdout:468: done (1 left) 2024-08-17T16:56:40.849 INFO:tasks.rados.rados.0.smithi063.stdout:469: snap_remove snap 47 2024-08-17T16:56:41.258 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:40 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:40.804+0000 7f12489e5040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-08-17T16:56:41.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:41.257+0000 7f12489e5040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-08-17T16:56:41.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[31063]: Deploying cephadm binary to smithi162 2024-08-17T16:56:41.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[31063]: Deploying cephadm binary to smithi063 2024-08-17T16:56:41.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[31063]: [17/Aug/2024:16:56:40] ENGINE Bus STARTING 2024-08-17T16:56:41.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[31063]: pgmap v6: 89 pgs: 89 active+clean; 232 MiB data, 667 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:41.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[31063]: [17/Aug/2024:16:56:40] ENGINE Serving on https://172.21.15.162:7150 2024-08-17T16:56:41.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[31063]: [17/Aug/2024:16:56:40] ENGINE Error in HTTPServer.serve 2024-08-17T16:56:41.530 INFO:journalctl@ceph.mon.a.smithi063.stdout: Traceback (most recent call last): 2024-08-17T16:56:41.530 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:56:41.530 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:56:41.530 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:56:41.530 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._run(expiration_interval) 2024-08-17T16:56:41.530 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:56:41.530 INFO:journalctl@ceph.mon.a.smithi063.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:56:41.530 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:56:41.531 INFO:journalctl@ceph.mon.a.smithi063.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:56:41.531 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:56:41.531 INFO:journalctl@ceph.mon.a.smithi063.stdout: s = self.context.wrap_socket( 2024-08-17T16:56:41.531 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:56:41.531 INFO:journalctl@ceph.mon.a.smithi063.stdout: return self.sslsocket_class._create( 2024-08-17T16:56:41.531 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:56:41.531 INFO:journalctl@ceph.mon.a.smithi063.stdout: self.do_handshake() 2024-08-17T16:56:41.531 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:56:41.531 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._sslobj.do_handshake() 2024-08-17T16:56:41.531 INFO:journalctl@ceph.mon.a.smithi063.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:56:41.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[31063]: [17/Aug/2024:16:56:40] ENGINE Serving on http://172.21.15.162:8765 2024-08-17T16:56:41.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[31063]: [17/Aug/2024:16:56:40] ENGINE Bus STARTED 2024-08-17T16:56:41.532 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[31063]: osdmap e185: 8 total, 8 up, 8 in 2024-08-17T16:56:41.532 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[42680]: Deploying cephadm binary to smithi162 2024-08-17T16:56:41.532 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[42680]: Deploying cephadm binary to smithi063 2024-08-17T16:56:41.532 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[42680]: [17/Aug/2024:16:56:40] ENGINE Bus STARTING 2024-08-17T16:56:41.532 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[42680]: pgmap v6: 89 pgs: 89 active+clean; 232 MiB data, 667 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:41.532 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[42680]: [17/Aug/2024:16:56:40] ENGINE Serving on https://172.21.15.162:7150 2024-08-17T16:56:41.532 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[42680]: [17/Aug/2024:16:56:40] ENGINE Error in HTTPServer.serve 2024-08-17T16:56:41.532 INFO:journalctl@ceph.mon.c.smithi063.stdout: Traceback (most recent call last): 2024-08-17T16:56:41.532 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:56:41.532 INFO:journalctl@ceph.mon.c.smithi063.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:56:41.532 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:56:41.532 INFO:journalctl@ceph.mon.c.smithi063.stdout: self._run(expiration_interval) 2024-08-17T16:56:41.533 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:56:41.533 INFO:journalctl@ceph.mon.c.smithi063.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:56:41.533 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:56:41.533 INFO:journalctl@ceph.mon.c.smithi063.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:56:41.533 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:56:41.533 INFO:journalctl@ceph.mon.c.smithi063.stdout: s = self.context.wrap_socket( 2024-08-17T16:56:41.533 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:56:41.533 INFO:journalctl@ceph.mon.c.smithi063.stdout: return self.sslsocket_class._create( 2024-08-17T16:56:41.533 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:56:41.533 INFO:journalctl@ceph.mon.c.smithi063.stdout: self.do_handshake() 2024-08-17T16:56:41.533 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:56:41.533 INFO:journalctl@ceph.mon.c.smithi063.stdout: self._sslobj.do_handshake() 2024-08-17T16:56:41.533 INFO:journalctl@ceph.mon.c.smithi063.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:56:41.533 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[42680]: [17/Aug/2024:16:56:40] ENGINE Serving on http://172.21.15.162:8765 2024-08-17T16:56:41.534 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[42680]: [17/Aug/2024:16:56:40] ENGINE Bus STARTED 2024-08-17T16:56:41.534 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-mon[42680]: osdmap e185: 8 total, 8 up, 8 in 2024-08-17T16:56:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:41 smithi162 ceph-mon[34633]: Deploying cephadm binary to smithi162 2024-08-17T16:56:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:41 smithi162 ceph-mon[34633]: Deploying cephadm binary to smithi063 2024-08-17T16:56:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:41 smithi162 ceph-mon[34633]: [17/Aug/2024:16:56:40] ENGINE Bus STARTING 2024-08-17T16:56:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:41 smithi162 ceph-mon[34633]: pgmap v6: 89 pgs: 89 active+clean; 232 MiB data, 667 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:41 smithi162 ceph-mon[34633]: [17/Aug/2024:16:56:40] ENGINE Serving on https://172.21.15.162:7150 2024-08-17T16:56:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:41 smithi162 ceph-mon[34633]: [17/Aug/2024:16:56:40] ENGINE Error in HTTPServer.serve 2024-08-17T16:56:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout: Traceback (most recent call last): 2024-08-17T16:56:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:56:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:56:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:56:41.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: self._run(expiration_interval) 2024-08-17T16:56:41.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:56:41.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:56:41.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:56:41.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:56:41.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:56:41.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: s = self.context.wrap_socket( 2024-08-17T16:56:41.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:56:41.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: return self.sslsocket_class._create( 2024-08-17T16:56:41.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:56:41.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: self.do_handshake() 2024-08-17T16:56:41.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:56:41.847 INFO:journalctl@ceph.mon.b.smithi162.stdout: self._sslobj.do_handshake() 2024-08-17T16:56:41.848 INFO:journalctl@ceph.mon.b.smithi162.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:56:41.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:41 smithi162 ceph-mon[34633]: [17/Aug/2024:16:56:40] ENGINE Serving on http://172.21.15.162:8765 2024-08-17T16:56:41.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:41 smithi162 ceph-mon[34633]: [17/Aug/2024:16:56:40] ENGINE Bus STARTED 2024-08-17T16:56:41.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:41 smithi162 ceph-mon[34633]: osdmap e185: 8 total, 8 up, 8 in 2024-08-17T16:56:41.855 INFO:tasks.rados.rados.0.smithi063.stdout:469: done (1 left) 2024-08-17T16:56:41.855 INFO:tasks.rados.rados.0.smithi063.stdout:470: snap_create 2024-08-17T16:56:41.855 INFO:tasks.rados.rados.0.smithi063.stdout:466: expect (ObjNum 148 snap 34 seq_num 148) 2024-08-17T16:56:42.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:41.586+0000 7f12489e5040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-08-17T16:56:42.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:41.654+0000 7f12489e5040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-08-17T16:56:42.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:41 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:41.812+0000 7f12489e5040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-08-17T16:56:42.420 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:42.158+0000 7f12489e5040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-08-17T16:56:42.420 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:42.221+0000 7f12489e5040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-08-17T16:56:42.735 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:42.418+0000 7f12489e5040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-08-17T16:56:42.735 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:42.578+0000 7f12489e5040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-08-17T16:56:42.735 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:42.645+0000 7f12489e5040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-08-17T16:56:42.736 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:42.733+0000 7f12489e5040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-08-17T16:56:42.862 INFO:tasks.rados.rados.0.smithi063.stdout:466: done (1 left) 2024-08-17T16:56:42.862 INFO:tasks.rados.rados.0.smithi063.stdout:470: done (0 left) 2024-08-17T16:56:42.862 INFO:tasks.rados.rados.0.smithi063.stdout:471: read oid 27 snap -1 2024-08-17T16:56:42.862 INFO:tasks.rados.rados.0.smithi063.stdout:471: expect (ObjNum 172 snap 45 seq_num 172) 2024-08-17T16:56:42.862 INFO:tasks.rados.rados.0.smithi063.stdout:472: rollback oid 48 current snap is 61 2024-08-17T16:56:42.862 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 48 to 54 2024-08-17T16:56:42.862 INFO:tasks.rados.rados.0.smithi063.stdout:473: read oid 17 snap 55 2024-08-17T16:56:42.862 INFO:tasks.rados.rados.0.smithi063.stdout:473: expect (ObjNum 191 snap 50 seq_num 191) 2024-08-17T16:56:42.862 INFO:tasks.rados.rados.0.smithi063.stdout:474: read oid 14 snap -1 2024-08-17T16:56:42.862 INFO:tasks.rados.rados.0.smithi063.stdout:474: expect (ObjNum 171 snap 44 seq_num 171) 2024-08-17T16:56:42.863 INFO:tasks.rados.rados.0.smithi063.stdout:475: write oid 46 current snap is 61 2024-08-17T16:56:42.863 INFO:tasks.rados.rados.0.smithi063.stdout:475: seq_num 212 ranges {763084=694319,1889359=541737,2955792=541902} 2024-08-17T16:56:42.870 INFO:tasks.rados.rados.0.smithi063.stdout:475: writing smithi06387500-46 from 763084 to 1457403 tid 1 2024-08-17T16:56:42.877 INFO:tasks.rados.rados.0.smithi063.stdout:475: writing smithi06387500-46 from 1889359 to 2431096 tid 2 2024-08-17T16:56:42.884 INFO:tasks.rados.rados.0.smithi063.stdout:475: writing smithi06387500-46 from 2955792 to 3497694 tid 3 2024-08-17T16:56:42.884 INFO:tasks.rados.rados.0.smithi063.stdout:476: snap_remove snap 57 2024-08-17T16:56:43.024 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:42.814+0000 7f12489e5040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-08-17T16:56:43.024 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:42 smithi063 ceph-mon[31063]: mgrmap e25: x(active, since 3s) 2024-08-17T16:56:43.025 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:42 smithi063 ceph-mon[31063]: osdmap e186: 8 total, 8 up, 8 in 2024-08-17T16:56:43.025 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:42 smithi063 ceph-mon[31063]: pgmap v9: 89 pgs: 89 active+clean; 232 MiB data, 667 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:43.025 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T16:56:43.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:42 smithi063 ceph-mon[42680]: mgrmap e25: x(active, since 3s) 2024-08-17T16:56:43.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:42 smithi063 ceph-mon[42680]: osdmap e186: 8 total, 8 up, 8 in 2024-08-17T16:56:43.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:42 smithi063 ceph-mon[42680]: pgmap v9: 89 pgs: 89 active+clean; 232 MiB data, 667 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:42 smithi162 ceph-mon[34633]: mgrmap e25: x(active, since 3s) 2024-08-17T16:56:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:42 smithi162 ceph-mon[34633]: osdmap e186: 8 total, 8 up, 8 in 2024-08-17T16:56:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:42 smithi162 ceph-mon[34633]: pgmap v9: 89 pgs: 89 active+clean; 232 MiB data, 667 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:43.653 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T16:56:43.653 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (3m) 62s ago 4m 26.9M - 0.25.0 c8568f914cd2 87286b24f231 2024-08-17T16:56:43.653 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (3m) 17s ago 3m 86.5M - 9.4.7 954c08fa6188 fc8d62329012 2024-08-17T16:56:43.653 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (2m) 62s ago 2m 15.2M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T16:56:43.653 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (2m) 17s ago 2m 15.7M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T16:56:43.654 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,9283,8765 running (22s) 17s ago 8m 53.9M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 180435c78c80 2024-08-17T16:56:43.654 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:9283,8765 running (9m) 62s ago 9m 490M - 18.2.4-719-gced649f8 c870753d67c2 ba4a2894b308 2024-08-17T16:56:43.654 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (9m) 62s ago 9m 46.6M 2048M 18.2.4-719-gced649f8 c870753d67c2 0c71ddd7ada6 2024-08-17T16:56:43.654 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (8m) 17s ago 8m 40.7M 2048M 18.2.4-719-gced649f8 c870753d67c2 d4ecbe65a0c8 2024-08-17T16:56:43.654 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (8m) 62s ago 8m 38.2M 2048M 18.2.4-719-gced649f8 c870753d67c2 b3f2c9339c6b 2024-08-17T16:56:43.654 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 running (3m) 62s ago 3m 15.5M - 1.5.0 0da6a335fe13 8958df80ad24 2024-08-17T16:56:43.654 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (3m) 17s ago 3m 14.9M - 1.5.0 0da6a335fe13 b1691104d8ac 2024-08-17T16:56:43.654 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (7m) 62s ago 7m 66.0M 4016M 18.2.4-719-gced649f8 c870753d67c2 f3b910ed49b8 2024-08-17T16:56:43.654 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (7m) 62s ago 7m 51.9M 4016M 18.2.4-719-gced649f8 c870753d67c2 5d6fdb6000ad 2024-08-17T16:56:43.654 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (6m) 62s ago 6m 62.5M 4016M 18.2.4-719-gced649f8 c870753d67c2 a3cbe8edb49a 2024-08-17T16:56:43.654 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (6m) 62s ago 6m 59.7M 4016M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T16:56:43.654 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (6m) 17s ago 6m 75.8M 4016M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T16:56:43.654 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (5m) 17s ago 5m 70.0M 4016M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T16:56:43.654 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (5m) 17s ago 5m 54.0M 4016M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T16:56:43.655 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (5m) 17s ago 5m 61.9M 4016M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T16:56:43.655 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (3m) 17s ago 4m 45.4M - 2.43.0 a07b618ecd1d 015db799d7d4 2024-08-17T16:56:43.711 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:43.431+0000 7f12489e5040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-08-17T16:56:43.711 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:43.604+0000 7f12489e5040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-08-17T16:56:43.711 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:56:43] ENGINE Bus STARTING 2024-08-17T16:56:43.711 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: CherryPy Checker: 2024-08-17T16:56:43.712 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: The Application mounted at '' has an empty config. 2024-08-17T16:56:43.712 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: 2024-08-17T16:56:43.712 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:56:43] ENGINE Serving on http://:::9283 2024-08-17T16:56:43.712 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: [17/Aug/2024:16:56:43] ENGINE Bus STARTED 2024-08-17T16:56:43.874 INFO:tasks.rados.rados.0.smithi063.stdout:476: done (5 left) 2024-08-17T16:56:43.875 INFO:tasks.rados.rados.0.smithi063.stdout:477: read oid 4 snap -1 2024-08-17T16:56:43.875 INFO:tasks.rados.rados.0.smithi063.stdout:477: expect (ObjNum 192 snap 50 seq_num 192) 2024-08-17T16:56:43.875 INFO:tasks.rados.rados.0.smithi063.stdout:478: rollback oid 7 current snap is 61 2024-08-17T16:56:43.875 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 49 2024-08-17T16:56:43.875 INFO:tasks.rados.rados.0.smithi063.stdout:479: rollback oid 10 current snap is 61 2024-08-17T16:56:43.875 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 10 to 51 2024-08-17T16:56:43.875 INFO:tasks.rados.rados.0.smithi063.stdout:480: rollback oid 25 current snap is 61 2024-08-17T16:56:43.875 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 25 to 43 2024-08-17T16:56:43.875 INFO:tasks.rados.rados.0.smithi063.stdout:481: snap_remove snap 50 2024-08-17T16:56:44.015 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:56:44.015 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T16:56:44.015 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T16:56:44.015 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:56:44.015 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T16:56:44.015 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 1, 2024-08-17T16:56:44.015 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 1 2024-08-17T16:56:44.015 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:56:44.016 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T16:56:44.016 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T16:56:44.016 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:56:44.016 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T16:56:44.016 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T16:56:44.016 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:56:44.016 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T16:56:44.016 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 14, 2024-08-17T16:56:44.016 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 1 2024-08-17T16:56:44.016 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T16:56:44.016 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:56:44.016 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[31063]: osdmap e187: 8 total, 8 up, 8 in 2024-08-17T16:56:44.017 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[31063]: from='client.24824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:44.017 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[31063]: from='client.24830 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:44.017 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[31063]: Standby manager daemon y started 2024-08-17T16:56:44.017 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.63:0/662588217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T16:56:44.017 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.63:0/662588217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:56:44.017 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.63:0/662588217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T16:56:44.017 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.63:0/662588217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:56:44.017 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[31063]: from='client.24836 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:44.017 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[42680]: osdmap e187: 8 total, 8 up, 8 in 2024-08-17T16:56:44.017 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[42680]: from='client.24824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:44.017 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[42680]: from='client.24830 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:44.018 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[42680]: Standby manager daemon y started 2024-08-17T16:56:44.018 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.63:0/662588217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T16:56:44.018 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.63:0/662588217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:56:44.018 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.63:0/662588217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T16:56:44.018 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.63:0/662588217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:56:44.018 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:43 smithi063 ceph-mon[42680]: from='client.24836 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:44.129 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:43 smithi162 ceph-mon[34633]: osdmap e187: 8 total, 8 up, 8 in 2024-08-17T16:56:44.130 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:43 smithi162 ceph-mon[34633]: from='client.24824 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:44.130 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:43 smithi162 ceph-mon[34633]: from='client.24830 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:44.130 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:43 smithi162 ceph-mon[34633]: Standby manager daemon y started 2024-08-17T16:56:44.130 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:43 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.63:0/662588217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T16:56:44.130 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:43 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.63:0/662588217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:56:44.130 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:43 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.63:0/662588217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T16:56:44.130 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:43 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.63:0/662588217' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:56:44.130 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:43 smithi162 ceph-mon[34633]: from='client.24836 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:44.315 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:56:44.396 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T16:56:44.396 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-08-17T16:56:44.396 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-17T16:56:44.396 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-08-17T16:56:44.396 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "1/2 daemons upgraded", 2024-08-17T16:56:44.396 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "", 2024-08-17T16:56:44.396 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T16:56:44.396 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:56:44.888 INFO:tasks.rados.rados.0.smithi063.stdout:481: done (9 left) 2024-08-17T16:56:44.888 INFO:tasks.rados.rados.0.smithi063.stdout:482: delete oid 15 current snap is 61 2024-08-17T16:56:44.888 INFO:tasks.rados.rados.0.smithi063.stdout:471: expect (ObjNum 172 snap 45 seq_num 172) 2024-08-17T16:56:45.026 INFO:tasks.rados.rados.0.smithi063.stdout:472: finishing rollback tid 0 to smithi06387500-48 2024-08-17T16:56:45.027 INFO:tasks.rados.rados.0.smithi063.stdout:472: finishing rollback tid 1 to smithi06387500-48 2024-08-17T16:56:45.027 INFO:tasks.rados.rados.0.smithi063.stdout:472: finishing rollback tid 2 to smithi06387500-48 2024-08-17T16:56:45.027 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 406 (ObjNum 161 snap 42 seq_num 161) dirty exists 2024-08-17T16:56:45.027 INFO:tasks.rados.rados.0.smithi063.stdout:475: finishing write tid 1 to smithi06387500-46 2024-08-17T16:56:45.027 INFO:tasks.rados.rados.0.smithi063.stdout:471: done (9 left) 2024-08-17T16:56:45.027 INFO:tasks.rados.rados.0.smithi063.stdout:472: done (8 left) 2024-08-17T16:56:45.027 INFO:tasks.rados.rados.0.smithi063.stdout:482: done (7 left) 2024-08-17T16:56:45.027 INFO:tasks.rados.rados.0.smithi063.stdout:483: read oid 34 snap -1 2024-08-17T16:56:45.027 INFO:tasks.rados.rados.0.smithi063.stdout:483: expect deleted 2024-08-17T16:56:45.027 INFO:tasks.rados.rados.0.smithi063.stdout:484: read oid 16 snap 58 2024-08-17T16:56:45.027 INFO:tasks.rados.rados.0.smithi063.stdout:484: expect (ObjNum 196 snap 54 seq_num 196) 2024-08-17T16:56:45.027 INFO:tasks.rados.rados.0.smithi063.stdout:485: snap_remove snap 31 2024-08-17T16:56:45.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[42680]: mgrmap e26: x(active, since 5s), standbys: y 2024-08-17T16:56:45.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[42680]: osdmap e188: 8 total, 8 up, 8 in 2024-08-17T16:56:45.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:56:45.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3299202596' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:56:45.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[42680]: from='client.24848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:45.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[42680]: pgmap v12: 89 pgs: 89 active+clean; 235 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 435 KiB/s wr, 34 op/s 2024-08-17T16:56:45.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:45.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:45.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[31063]: mgrmap e26: x(active, since 5s), standbys: y 2024-08-17T16:56:45.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[31063]: osdmap e188: 8 total, 8 up, 8 in 2024-08-17T16:56:45.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:56:45.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3299202596' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:56:45.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[31063]: from='client.24848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:45.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[31063]: pgmap v12: 89 pgs: 89 active+clean; 235 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 435 KiB/s wr, 34 op/s 2024-08-17T16:56:45.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:45.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:44 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:45.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:44 smithi162 ceph-mon[34633]: mgrmap e26: x(active, since 5s), standbys: y 2024-08-17T16:56:45.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:44 smithi162 ceph-mon[34633]: osdmap e188: 8 total, 8 up, 8 in 2024-08-17T16:56:45.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:44 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:56:45.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:44 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3299202596' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:56:45.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:44 smithi162 ceph-mon[34633]: from='client.24848 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:56:45.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:44 smithi162 ceph-mon[34633]: pgmap v12: 89 pgs: 89 active+clean; 235 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 435 KiB/s wr, 34 op/s 2024-08-17T16:56:45.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:44 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:45.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:44 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:45.890 INFO:tasks.rados.rados.0.smithi063.stdout:485: done (9 left) 2024-08-17T16:56:45.890 INFO:tasks.rados.rados.0.smithi063.stdout:486: write oid 22 current snap is 61 2024-08-17T16:56:45.890 INFO:tasks.rados.rados.0.smithi063.stdout:486: seq_num 214 ranges {796698=561170,2006323=545406,3126991=469707} 2024-08-17T16:56:45.897 INFO:tasks.rados.rados.0.smithi063.stdout:486: writing smithi06387500-22 from 796698 to 1357868 tid 1 2024-08-17T16:56:45.904 INFO:tasks.rados.rados.0.smithi063.stdout:486: writing smithi06387500-22 from 2006323 to 2551729 tid 2 2024-08-17T16:56:45.910 INFO:tasks.rados.rados.0.smithi063.stdout:486: writing smithi06387500-22 from 3126991 to 3596698 tid 3 2024-08-17T16:56:45.910 INFO:tasks.rados.rados.0.smithi063.stdout:487: delete oid 24 current snap is 61 2024-08-17T16:56:45.910 INFO:tasks.rados.rados.0.smithi063.stdout:475: finishing write tid 2 to smithi06387500-46 2024-08-17T16:56:45.911 INFO:tasks.rados.rados.0.smithi063.stdout:474: expect (ObjNum 171 snap 44 seq_num 171) 2024-08-17T16:56:46.075 INFO:tasks.rados.rados.0.smithi063.stdout:473: expect (ObjNum 191 snap 50 seq_num 191) 2024-08-17T16:56:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:45 smithi162 ceph-mon[34633]: osdmap e189: 8 total, 8 up, 8 in 2024-08-17T16:56:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:45 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:45 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:45 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:45 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:45 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:45 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:45 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:45 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:45 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:45 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:45 smithi162 ceph-mon[34633]: Adjusting osd_memory_target on smithi063 to 2480M 2024-08-17T16:56:46.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:45 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:46.252 INFO:tasks.rados.rados.0.smithi063.stdout:475: finishing write tid 3 to smithi06387500-46 2024-08-17T16:56:46.253 INFO:tasks.rados.rados.0.smithi063.stdout:475: finishing write tid 4 to smithi06387500-46 2024-08-17T16:56:46.253 INFO:tasks.rados.rados.0.smithi063.stdout:475: finishing write tid 5 to smithi06387500-46 2024-08-17T16:56:46.253 INFO:tasks.rados.rados.0.smithi063.stdout:475: oid 46 updating version 0 to 162 2024-08-17T16:56:46.253 INFO:tasks.rados.rados.0.smithi063.stdout:475: oid 46 updating version 162 to 163 2024-08-17T16:56:46.253 INFO:tasks.rados.rados.0.smithi063.stdout:475: oid 46 version 163 is already newer than 161 2024-08-17T16:56:46.253 INFO:tasks.rados.rados.0.smithi063.stdout:475: oid 46 version 163 is already newer than 160 2024-08-17T16:56:46.253 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 163 (ObjNum 212 snap 61 seq_num 212) dirty exists 2024-08-17T16:56:46.253 INFO:tasks.rados.rados.0.smithi063.stdout:475: left oid 46 (ObjNum 212 snap 61 seq_num 212) 2024-08-17T16:56:46.253 INFO:tasks.rados.rados.0.smithi063.stdout:479: finishing rollback tid 0 to smithi06387500-10 2024-08-17T16:56:46.253 INFO:tasks.rados.rados.0.smithi063.stdout:479: finishing rollback tid 1 to smithi06387500-10 2024-08-17T16:56:46.253 INFO:tasks.rados.rados.0.smithi063.stdout:478: finishing rollback tid 0 to smithi06387500-7 2024-08-17T16:56:46.254 INFO:tasks.rados.rados.0.smithi063.stdout:479: finishing rollback tid 2 to smithi06387500-10 2024-08-17T16:56:46.254 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 128 (ObjNum 100 snap 16 seq_num 100) dirty exists 2024-08-17T16:56:46.254 INFO:tasks.rados.rados.0.smithi063.stdout:478: finishing rollback tid 1 to smithi06387500-7 2024-08-17T16:56:46.254 INFO:tasks.rados.rados.0.smithi063.stdout:473: done (10 left) 2024-08-17T16:56:46.254 INFO:tasks.rados.rados.0.smithi063.stdout:474: done (9 left) 2024-08-17T16:56:46.254 INFO:tasks.rados.rados.0.smithi063.stdout:475: done (8 left) 2024-08-17T16:56:46.254 INFO:tasks.rados.rados.0.smithi063.stdout:479: done (7 left) 2024-08-17T16:56:46.254 INFO:tasks.rados.rados.0.smithi063.stdout:487: done (6 left) 2024-08-17T16:56:46.254 INFO:tasks.rados.rados.0.smithi063.stdout:488: write oid 20 current snap is 61 2024-08-17T16:56:46.254 INFO:tasks.rados.rados.0.smithi063.stdout:478: finishing rollback tid 2 to smithi06387500-7 2024-08-17T16:56:46.254 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 237 (ObjNum 156 snap 39 seq_num 156) dirty exists 2024-08-17T16:56:46.254 INFO:tasks.rados.rados.0.smithi063.stdout:477: expect (ObjNum 192 snap 50 seq_num 192) 2024-08-17T16:56:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[42680]: osdmap e189: 8 total, 8 up, 8 in 2024-08-17T16:56:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[42680]: Adjusting osd_memory_target on smithi063 to 2480M 2024-08-17T16:56:46.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:46.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[31063]: osdmap e189: 8 total, 8 up, 8 in 2024-08-17T16:56:46.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:46.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:46.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:46.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[31063]: Adjusting osd_memory_target on smithi063 to 2480M 2024-08-17T16:56:46.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:45 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:46.392 INFO:tasks.rados.rados.0.smithi063.stdout:480: finishing rollback tid 0 to smithi06387500-25 2024-08-17T16:56:46.392 INFO:tasks.rados.rados.0.smithi063.stdout:480: finishing rollback tid 1 to smithi06387500-25 2024-08-17T16:56:46.392 INFO:tasks.rados.rados.0.smithi063.stdout:480: finishing rollback tid 2 to smithi06387500-25 2024-08-17T16:56:46.392 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 412 (ObjNum 78 snap 9 seq_num 78) dirty exists 2024-08-17T16:56:46.392 INFO:tasks.rados.rados.0.smithi063.stdout:484: expect (ObjNum 196 snap 54 seq_num 196) 2024-08-17T16:56:46.576 INFO:tasks.rados.rados.0.smithi063.stdout:486: finishing write tid 1 to smithi06387500-22 2024-08-17T16:56:46.576 INFO:tasks.rados.rados.0.smithi063.stdout:486: finishing write tid 2 to smithi06387500-22 2024-08-17T16:56:46.577 INFO:tasks.rados.rados.0.smithi063.stdout:486: finishing write tid 3 to smithi06387500-22 2024-08-17T16:56:46.577 INFO:tasks.rados.rados.0.smithi063.stdout:486: finishing write tid 4 to smithi06387500-22 2024-08-17T16:56:46.577 INFO:tasks.rados.rados.0.smithi063.stdout:486: finishing write tid 5 to smithi06387500-22 2024-08-17T16:56:46.577 INFO:tasks.rados.rados.0.smithi063.stdout:486: oid 22 updating version 0 to 151 2024-08-17T16:56:46.577 INFO:tasks.rados.rados.0.smithi063.stdout:486: oid 22 updating version 151 to 152 2024-08-17T16:56:46.577 INFO:tasks.rados.rados.0.smithi063.stdout:486: oid 22 version 152 is already newer than 150 2024-08-17T16:56:46.577 INFO:tasks.rados.rados.0.smithi063.stdout:486: oid 22 updating version 152 to 153 2024-08-17T16:56:46.577 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 153 (ObjNum 214 snap 61 seq_num 214) dirty exists 2024-08-17T16:56:46.577 INFO:tasks.rados.rados.0.smithi063.stdout:486: left oid 22 (ObjNum 214 snap 61 seq_num 214) 2024-08-17T16:56:46.577 INFO:tasks.rados.rados.0.smithi063.stdout:488: seq_num 216 ranges {430312=711668,1723287=549075,2781837=703343} 2024-08-17T16:56:46.582 INFO:tasks.rados.rados.0.smithi063.stdout:488: writing smithi06387500-20 from 430312 to 1141980 tid 1 2024-08-17T16:56:46.590 INFO:tasks.rados.rados.0.smithi063.stdout:488: writing smithi06387500-20 from 1723287 to 2272362 tid 2 2024-08-17T16:56:46.596 INFO:tasks.rados.rados.0.smithi063.stdout:488: writing smithi06387500-20 from 2781837 to 3485180 tid 3 2024-08-17T16:56:46.596 INFO:tasks.rados.rados.0.smithi063.stdout:477: done (6 left) 2024-08-17T16:56:46.596 INFO:tasks.rados.rados.0.smithi063.stdout:478: done (5 left) 2024-08-17T16:56:46.596 INFO:tasks.rados.rados.0.smithi063.stdout:480: done (4 left) 2024-08-17T16:56:46.596 INFO:tasks.rados.rados.0.smithi063.stdout:483: done (3 left) 2024-08-17T16:56:46.596 INFO:tasks.rados.rados.0.smithi063.stdout:484: done (2 left) 2024-08-17T16:56:46.596 INFO:tasks.rados.rados.0.smithi063.stdout:486: done (1 left) 2024-08-17T16:56:46.596 INFO:tasks.rados.rados.0.smithi063.stdout:489: read oid 21 snap 38 2024-08-17T16:56:46.596 INFO:tasks.rados.rados.0.smithi063.stdout:489: expect deleted 2024-08-17T16:56:46.596 INFO:tasks.rados.rados.0.smithi063.stdout:490: write oid 29 current snap is 61 2024-08-17T16:56:46.596 INFO:tasks.rados.rados.0.smithi063.stdout:488: finishing write tid 1 to smithi06387500-20 2024-08-17T16:56:46.597 INFO:tasks.rados.rados.0.smithi063.stdout:488: finishing write tid 2 to smithi06387500-20 2024-08-17T16:56:46.597 INFO:tasks.rados.rados.0.smithi063.stdout:490: seq_num 217 ranges {447119=786917,1665416=692733,2976554=514615} 2024-08-17T16:56:46.602 INFO:tasks.rados.rados.0.smithi063.stdout:490: writing smithi06387500-29 from 447119 to 1234036 tid 1 2024-08-17T16:56:46.608 INFO:tasks.rados.rados.0.smithi063.stdout:490: writing smithi06387500-29 from 1665416 to 2358149 tid 2 2024-08-17T16:56:46.615 INFO:tasks.rados.rados.0.smithi063.stdout:490: writing smithi06387500-29 from 2976554 to 3491169 tid 3 2024-08-17T16:56:46.615 INFO:tasks.rados.rados.0.smithi063.stdout:491: read oid 28 snap -1 2024-08-17T16:56:46.615 INFO:tasks.rados.rados.0.smithi063.stdout:491: expect (ObjNum 147 snap 34 seq_num 147) 2024-08-17T16:56:46.615 INFO:tasks.rados.rados.0.smithi063.stdout:492: rollback oid 42 current snap is 61 2024-08-17T16:56:46.615 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 42 to 60 2024-08-17T16:56:46.616 INFO:tasks.rados.rados.0.smithi063.stdout:488: finishing write tid 3 to smithi06387500-20 2024-08-17T16:56:46.616 INFO:tasks.rados.rados.0.smithi063.stdout:488: finishing write tid 4 to smithi06387500-20 2024-08-17T16:56:46.616 INFO:tasks.rados.rados.0.smithi063.stdout:488: finishing write tid 5 to smithi06387500-20 2024-08-17T16:56:46.616 INFO:tasks.rados.rados.0.smithi063.stdout:488: oid 20 updating version 0 to 154 2024-08-17T16:56:46.616 INFO:tasks.rados.rados.0.smithi063.stdout:488: oid 20 updating version 154 to 156 2024-08-17T16:56:46.616 INFO:tasks.rados.rados.0.smithi063.stdout:488: oid 20 updating version 156 to 157 2024-08-17T16:56:46.616 INFO:tasks.rados.rados.0.smithi063.stdout:488: oid 20 version 157 is already newer than 155 2024-08-17T16:56:46.616 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 157 (ObjNum 216 snap 61 seq_num 216) dirty exists 2024-08-17T16:56:46.616 INFO:tasks.rados.rados.0.smithi063.stdout:488: left oid 20 (ObjNum 216 snap 61 seq_num 216) 2024-08-17T16:56:46.616 INFO:tasks.rados.rados.0.smithi063.stdout:490: finishing write tid 1 to smithi06387500-29 2024-08-17T16:56:46.616 INFO:tasks.rados.rados.0.smithi063.stdout:490: finishing write tid 2 to smithi06387500-29 2024-08-17T16:56:46.616 INFO:tasks.rados.rados.0.smithi063.stdout:488: done (4 left) 2024-08-17T16:56:46.616 INFO:tasks.rados.rados.0.smithi063.stdout:489: done (3 left) 2024-08-17T16:56:46.616 INFO:tasks.rados.rados.0.smithi063.stdout:493: delete oid 1 current snap is 61 2024-08-17T16:56:46.617 INFO:tasks.rados.rados.0.smithi063.stdout:493: done (3 left) 2024-08-17T16:56:46.617 INFO:tasks.rados.rados.0.smithi063.stdout:494: write oid 13 current snap is 61 2024-08-17T16:56:46.617 INFO:tasks.rados.rados.0.smithi063.stdout:494: seq_num 219 ranges {480733=537415,1782380=696402,3031400=537159} 2024-08-17T16:56:46.623 INFO:tasks.rados.rados.0.smithi063.stdout:494: writing smithi06387500-13 from 480733 to 1018148 tid 1 2024-08-17T16:56:46.630 INFO:tasks.rados.rados.0.smithi063.stdout:494: writing smithi06387500-13 from 1782380 to 2478782 tid 2 2024-08-17T16:56:46.637 INFO:tasks.rados.rados.0.smithi063.stdout:494: writing smithi06387500-13 from 3031400 to 3568559 tid 3 2024-08-17T16:56:46.637 INFO:tasks.rados.rados.0.smithi063.stdout:495: rollback oid 49 current snap is 61 2024-08-17T16:56:46.637 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 49 to 51 2024-08-17T16:56:46.637 INFO:tasks.rados.rados.0.smithi063.stdout:491: expect (ObjNum 147 snap 34 seq_num 147) 2024-08-17T16:56:46.751 INFO:tasks.rados.rados.0.smithi063.stdout:490: finishing write tid 3 to smithi06387500-29 2024-08-17T16:56:46.751 INFO:tasks.rados.rados.0.smithi063.stdout:490: finishing write tid 4 to smithi06387500-29 2024-08-17T16:56:46.751 INFO:tasks.rados.rados.0.smithi063.stdout:491: done (4 left) 2024-08-17T16:56:46.751 INFO:tasks.rados.rados.0.smithi063.stdout:496: write oid 26 current snap is 61 2024-08-17T16:56:46.751 INFO:tasks.rados.rados.0.smithi063.stdout:492: finishing rollback tid 0 to smithi06387500-42 2024-08-17T16:56:46.751 INFO:tasks.rados.rados.0.smithi063.stdout:492: finishing rollback tid 1 to smithi06387500-42 2024-08-17T16:56:46.751 INFO:tasks.rados.rados.0.smithi063.stdout:490: finishing write tid 5 to smithi06387500-29 2024-08-17T16:56:46.752 INFO:tasks.rados.rados.0.smithi063.stdout:490: oid 29 updating version 0 to 98 2024-08-17T16:56:46.752 INFO:tasks.rados.rados.0.smithi063.stdout:490: oid 29 version 98 is already newer than 96 2024-08-17T16:56:46.752 INFO:tasks.rados.rados.0.smithi063.stdout:490: oid 29 version 98 is already newer than 97 2024-08-17T16:56:46.752 INFO:tasks.rados.rados.0.smithi063.stdout:490: oid 29 updating version 98 to 99 2024-08-17T16:56:46.752 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 99 (ObjNum 217 snap 61 seq_num 217) dirty exists 2024-08-17T16:56:46.752 INFO:tasks.rados.rados.0.smithi063.stdout:490: left oid 29 (ObjNum 217 snap 61 seq_num 217) 2024-08-17T16:56:46.752 INFO:tasks.rados.rados.0.smithi063.stdout:492: finishing rollback tid 2 to smithi06387500-42 2024-08-17T16:56:46.752 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 292 (ObjNum 205 snap 57 seq_num 205) dirty exists 2024-08-17T16:56:46.752 INFO:tasks.rados.rados.0.smithi063.stdout:494: finishing write tid 1 to smithi06387500-13 2024-08-17T16:56:46.752 INFO:tasks.rados.rados.0.smithi063.stdout:496: seq_num 220 ranges {497540=612664,1840862=440060,3058823=464784} 2024-08-17T16:56:46.757 INFO:tasks.rados.rados.0.smithi063.stdout:496: writing smithi06387500-26 from 497540 to 1110204 tid 1 2024-08-17T16:56:46.763 INFO:tasks.rados.rados.0.smithi063.stdout:496: writing smithi06387500-26 from 1840862 to 2280922 tid 2 2024-08-17T16:56:46.772 INFO:tasks.rados.rados.0.smithi063.stdout:496: writing smithi06387500-26 from 3058823 to 3523607 tid 3 2024-08-17T16:56:46.772 INFO:tasks.rados.rados.0.smithi063.stdout:490: done (4 left) 2024-08-17T16:56:46.772 INFO:tasks.rados.rados.0.smithi063.stdout:492: done (3 left) 2024-08-17T16:56:46.773 INFO:tasks.rados.rados.0.smithi063.stdout:497: read oid 44 snap -1 2024-08-17T16:56:46.773 INFO:tasks.rados.rados.0.smithi063.stdout:497: expect (ObjNum 208 snap 59 seq_num 208) 2024-08-17T16:56:46.773 INFO:tasks.rados.rados.0.smithi063.stdout:498: write oid 4 current snap is 61 2024-08-17T16:56:46.773 INFO:tasks.rados.rados.0.smithi063.stdout:494: finishing write tid 2 to smithi06387500-13 2024-08-17T16:56:46.773 INFO:tasks.rados.rados.0.smithi063.stdout:495: finishing rollback tid 0 to smithi06387500-49 2024-08-17T16:56:46.773 INFO:tasks.rados.rados.0.smithi063.stdout:495: finishing rollback tid 1 to smithi06387500-49 2024-08-17T16:56:46.773 INFO:tasks.rados.rados.0.smithi063.stdout:494: finishing write tid 3 to smithi06387500-13 2024-08-17T16:56:46.773 INFO:tasks.rados.rados.0.smithi063.stdout:495: finishing rollback tid 2 to smithi06387500-49 2024-08-17T16:56:46.773 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 434 (ObjNum 160 snap 41 seq_num 160) dirty exists 2024-08-17T16:56:46.773 INFO:tasks.rados.rados.0.smithi063.stdout:494: finishing write tid 4 to smithi06387500-13 2024-08-17T16:56:46.773 INFO:tasks.rados.rados.0.smithi063.stdout:494: finishing write tid 5 to smithi06387500-13 2024-08-17T16:56:46.774 INFO:tasks.rados.rados.0.smithi063.stdout:494: oid 13 updating version 0 to 196 2024-08-17T16:56:46.774 INFO:tasks.rados.rados.0.smithi063.stdout:494: oid 13 updating version 196 to 198 2024-08-17T16:56:46.774 INFO:tasks.rados.rados.0.smithi063.stdout:494: oid 13 version 198 is already newer than 197 2024-08-17T16:56:46.774 INFO:tasks.rados.rados.0.smithi063.stdout:494: oid 13 updating version 198 to 199 2024-08-17T16:56:46.774 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 199 (ObjNum 219 snap 61 seq_num 219) dirty exists 2024-08-17T16:56:46.774 INFO:tasks.rados.rados.0.smithi063.stdout:494: left oid 13 (ObjNum 219 snap 61 seq_num 219) 2024-08-17T16:56:46.774 INFO:tasks.rados.rados.0.smithi063.stdout:496: finishing write tid 1 to smithi06387500-26 2024-08-17T16:56:46.774 INFO:tasks.rados.rados.0.smithi063.stdout:496: finishing write tid 2 to smithi06387500-26 2024-08-17T16:56:46.774 INFO:tasks.rados.rados.0.smithi063.stdout:498: seq_num 221 ranges {514347=404266,1499344=700071,2686246=676056} 2024-08-17T16:56:46.775 INFO:tasks.rados.rados.0.smithi063.stdout:498: writing smithi06387500-4 from 514347 to 918613 tid 1 2024-08-17T16:56:46.782 INFO:tasks.rados.rados.0.smithi063.stdout:498: writing smithi06387500-4 from 1499344 to 2199415 tid 2 2024-08-17T16:56:46.789 INFO:tasks.rados.rados.0.smithi063.stdout:498: writing smithi06387500-4 from 2686246 to 3362302 tid 3 2024-08-17T16:56:46.789 INFO:tasks.rados.rados.0.smithi063.stdout:494: done (4 left) 2024-08-17T16:56:46.789 INFO:tasks.rados.rados.0.smithi063.stdout:495: done (3 left) 2024-08-17T16:56:46.789 INFO:tasks.rados.rados.0.smithi063.stdout:499: snap_remove snap 54 2024-08-17T16:56:47.178 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:46 smithi162 ceph-mon[34633]: osdmap e190: 8 total, 8 up, 8 in 2024-08-17T16:56:47.178 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:46 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:47.178 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:46 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:47.178 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:46 smithi162 ceph-mon[34633]: pgmap v15: 89 pgs: 89 active+clean; 235 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 435 KiB/s wr, 34 op/s 2024-08-17T16:56:47.179 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:46 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:47.179 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:46 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:47.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:46 smithi063 ceph-mon[42680]: osdmap e190: 8 total, 8 up, 8 in 2024-08-17T16:56:47.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:46 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:47.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:46 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:47.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:46 smithi063 ceph-mon[42680]: pgmap v15: 89 pgs: 89 active+clean; 235 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 435 KiB/s wr, 34 op/s 2024-08-17T16:56:47.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:46 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:47.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:46 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:46 smithi063 ceph-mon[31063]: osdmap e190: 8 total, 8 up, 8 in 2024-08-17T16:56:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:46 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:46 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:46 smithi063 ceph-mon[31063]: pgmap v15: 89 pgs: 89 active+clean; 235 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 435 KiB/s wr, 34 op/s 2024-08-17T16:56:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:46 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:46 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:47.763 INFO:tasks.rados.rados.0.smithi063.stdout:499: done (3 left) 2024-08-17T16:56:47.763 INFO:tasks.rados.rados.0.smithi063.stdout:500: delete oid 12 current snap is 61 2024-08-17T16:56:47.763 INFO:tasks.rados.rados.0.smithi063.stdout:496: finishing write tid 3 to smithi06387500-26 2024-08-17T16:56:47.763 INFO:tasks.rados.rados.0.smithi063.stdout:496: finishing write tid 4 to smithi06387500-26 2024-08-17T16:56:47.763 INFO:tasks.rados.rados.0.smithi063.stdout:496: finishing write tid 5 to smithi06387500-26 2024-08-17T16:56:47.763 INFO:tasks.rados.rados.0.smithi063.stdout:496: oid 26 updating version 0 to 151 2024-08-17T16:56:47.763 INFO:tasks.rados.rados.0.smithi063.stdout:496: oid 26 version 151 is already newer than 150 2024-08-17T16:56:47.764 INFO:tasks.rados.rados.0.smithi063.stdout:496: oid 26 updating version 151 to 153 2024-08-17T16:56:47.764 INFO:tasks.rados.rados.0.smithi063.stdout:496: oid 26 version 153 is already newer than 152 2024-08-17T16:56:47.764 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 153 (ObjNum 220 snap 61 seq_num 220) dirty exists 2024-08-17T16:56:47.764 INFO:tasks.rados.rados.0.smithi063.stdout:496: left oid 26 (ObjNum 220 snap 61 seq_num 220) 2024-08-17T16:56:47.764 INFO:tasks.rados.rados.0.smithi063.stdout:497: expect (ObjNum 208 snap 59 seq_num 208) 2024-08-17T16:56:47.884 INFO:tasks.rados.rados.0.smithi063.stdout:498: finishing write tid 1 to smithi06387500-4 2024-08-17T16:56:47.884 INFO:tasks.rados.rados.0.smithi063.stdout:498: finishing write tid 2 to smithi06387500-4 2024-08-17T16:56:47.884 INFO:tasks.rados.rados.0.smithi063.stdout:498: finishing write tid 3 to smithi06387500-4 2024-08-17T16:56:47.884 INFO:tasks.rados.rados.0.smithi063.stdout:498: finishing write tid 4 to smithi06387500-4 2024-08-17T16:56:47.884 INFO:tasks.rados.rados.0.smithi063.stdout:498: finishing write tid 5 to smithi06387500-4 2024-08-17T16:56:47.884 INFO:tasks.rados.rados.0.smithi063.stdout:498: oid 4 updating version 0 to 437 2024-08-17T16:56:47.884 INFO:tasks.rados.rados.0.smithi063.stdout:498: oid 4 updating version 437 to 438 2024-08-17T16:56:47.885 INFO:tasks.rados.rados.0.smithi063.stdout:498: oid 4 updating version 438 to 439 2024-08-17T16:56:47.885 INFO:tasks.rados.rados.0.smithi063.stdout:498: oid 4 version 439 is already newer than 436 2024-08-17T16:56:47.885 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 439 (ObjNum 221 snap 61 seq_num 221) dirty exists 2024-08-17T16:56:47.885 INFO:tasks.rados.rados.0.smithi063.stdout:498: left oid 4 (ObjNum 221 snap 61 seq_num 221) 2024-08-17T16:56:47.888 INFO:tasks.rados.rados.0.smithi063.stdout:496: done (3 left) 2024-08-17T16:56:47.888 INFO:tasks.rados.rados.0.smithi063.stdout:497: done (2 left) 2024-08-17T16:56:47.888 INFO:tasks.rados.rados.0.smithi063.stdout:498: done (1 left) 2024-08-17T16:56:47.888 INFO:tasks.rados.rados.0.smithi063.stdout:500: done (0 left) 2024-08-17T16:56:47.888 INFO:tasks.rados.rados.0.smithi063.stdout:501: snap_create 2024-08-17T16:56:48.771 INFO:tasks.rados.rados.0.smithi063.stdout:501: done (0 left) 2024-08-17T16:56:48.771 INFO:tasks.rados.rados.0.smithi063.stdout:502: write oid 32 current snap is 62 2024-08-17T16:56:48.771 INFO:tasks.rados.rados.0.smithi063.stdout:502: seq_num 223 ranges {547961=554764,1616308=703740,2741092=698600} 2024-08-17T16:56:48.777 INFO:tasks.rados.rados.0.smithi063.stdout:502: writing smithi06387500-32 from 547961 to 1102725 tid 1 2024-08-17T16:56:48.783 INFO:tasks.rados.rados.0.smithi063.stdout:502: writing smithi06387500-32 from 1616308 to 2320048 tid 2 2024-08-17T16:56:48.790 INFO:tasks.rados.rados.0.smithi063.stdout:502: writing smithi06387500-32 from 2741092 to 3439692 tid 3 2024-08-17T16:56:48.790 INFO:tasks.rados.rados.0.smithi063.stdout:503: read oid 47 snap -1 2024-08-17T16:56:48.790 INFO:tasks.rados.rados.0.smithi063.stdout:503: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:56:48.790 INFO:tasks.rados.rados.0.smithi063.stdout:504: write oid 21 current snap is 62 2024-08-17T16:56:48.790 INFO:tasks.rados.rados.0.smithi063.stdout:502: finishing write tid 1 to smithi06387500-32 2024-08-17T16:56:48.790 INFO:tasks.rados.rados.0.smithi063.stdout:504: seq_num 224 ranges {564768=630013,1674790=447398,2768515=626225} 2024-08-17T16:56:48.796 INFO:tasks.rados.rados.0.smithi063.stdout:504: writing smithi06387500-21 from 564768 to 1194781 tid 1 2024-08-17T16:56:48.802 INFO:tasks.rados.rados.0.smithi063.stdout:504: writing smithi06387500-21 from 1674790 to 2122188 tid 2 2024-08-17T16:56:48.809 INFO:tasks.rados.rados.0.smithi063.stdout:504: writing smithi06387500-21 from 2768515 to 3394740 tid 3 2024-08-17T16:56:48.809 INFO:tasks.rados.rados.0.smithi063.stdout:505: rollback oid 11 current snap is 62 2024-08-17T16:56:48.809 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 11 to 43 2024-08-17T16:56:48.810 INFO:tasks.rados.rados.0.smithi063.stdout:502: finishing write tid 2 to smithi06387500-32 2024-08-17T16:56:48.810 INFO:tasks.rados.rados.0.smithi063.stdout:502: finishing write tid 3 to smithi06387500-32 2024-08-17T16:56:48.810 INFO:tasks.rados.rados.0.smithi063.stdout:502: finishing write tid 4 to smithi06387500-32 2024-08-17T16:56:48.810 INFO:tasks.rados.rados.0.smithi063.stdout:502: finishing write tid 5 to smithi06387500-32 2024-08-17T16:56:48.810 INFO:tasks.rados.rados.0.smithi063.stdout:502: oid 32 updating version 0 to 92 2024-08-17T16:56:48.810 INFO:tasks.rados.rados.0.smithi063.stdout:502: oid 32 updating version 92 to 93 2024-08-17T16:56:48.810 INFO:tasks.rados.rados.0.smithi063.stdout:502: oid 32 updating version 93 to 94 2024-08-17T16:56:48.810 INFO:tasks.rados.rados.0.smithi063.stdout:502: oid 32 version 94 is already newer than 91 2024-08-17T16:56:48.810 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 94 (ObjNum 223 snap 62 seq_num 223) dirty exists 2024-08-17T16:56:48.810 INFO:tasks.rados.rados.0.smithi063.stdout:502: left oid 32 (ObjNum 223 snap 62 seq_num 223) 2024-08-17T16:56:48.810 INFO:tasks.rados.rados.0.smithi063.stdout:503: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:56:48.966 INFO:tasks.rados.rados.0.smithi063.stdout:504: finishing write tid 1 to smithi06387500-21 2024-08-17T16:56:48.966 INFO:tasks.rados.rados.0.smithi063.stdout:504: finishing write tid 2 to smithi06387500-21 2024-08-17T16:56:48.966 INFO:tasks.rados.rados.0.smithi063.stdout:504: finishing write tid 3 to smithi06387500-21 2024-08-17T16:56:48.966 INFO:tasks.rados.rados.0.smithi063.stdout:504: finishing write tid 4 to smithi06387500-21 2024-08-17T16:56:48.966 INFO:tasks.rados.rados.0.smithi063.stdout:504: finishing write tid 5 to smithi06387500-21 2024-08-17T16:56:48.966 INFO:tasks.rados.rados.0.smithi063.stdout:504: oid 21 updating version 0 to 300 2024-08-17T16:56:48.966 INFO:tasks.rados.rados.0.smithi063.stdout:504: oid 21 updating version 300 to 301 2024-08-17T16:56:48.966 INFO:tasks.rados.rados.0.smithi063.stdout:504: oid 21 updating version 301 to 303 2024-08-17T16:56:48.967 INFO:tasks.rados.rados.0.smithi063.stdout:504: oid 21 version 303 is already newer than 302 2024-08-17T16:56:48.967 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 303 (ObjNum 224 snap 62 seq_num 224) dirty exists 2024-08-17T16:56:48.967 INFO:tasks.rados.rados.0.smithi063.stdout:504: left oid 21 (ObjNum 224 snap 62 seq_num 224) 2024-08-17T16:56:48.967 INFO:tasks.rados.rados.0.smithi063.stdout:505: finishing rollback tid 0 to smithi06387500-11 2024-08-17T16:56:48.967 INFO:tasks.rados.rados.0.smithi063.stdout:505: finishing rollback tid 1 to smithi06387500-11 2024-08-17T16:56:48.967 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 78 (ObjNum 129 snap 0 seq_num 1807121976) dirty dne 2024-08-17T16:56:48.967 INFO:tasks.rados.rados.0.smithi063.stdout:502: done (3 left) 2024-08-17T16:56:48.967 INFO:tasks.rados.rados.0.smithi063.stdout:503: done (2 left) 2024-08-17T16:56:48.967 INFO:tasks.rados.rados.0.smithi063.stdout:504: done (1 left) 2024-08-17T16:56:48.967 INFO:tasks.rados.rados.0.smithi063.stdout:505: done (0 left) 2024-08-17T16:56:48.967 INFO:tasks.rados.rados.0.smithi063.stdout:506: snap_create 2024-08-17T16:56:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:48 smithi063 ceph-mon[42680]: osdmap e191: 8 total, 8 up, 8 in 2024-08-17T16:56:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:48 smithi063 ceph-mon[31063]: osdmap e191: 8 total, 8 up, 8 in 2024-08-17T16:56:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:48 smithi162 ceph-mon[34633]: osdmap e191: 8 total, 8 up, 8 in 2024-08-17T16:56:49.772 INFO:tasks.rados.rados.0.smithi063.stdout:506: done (0 left) 2024-08-17T16:56:49.772 INFO:tasks.rados.rados.0.smithi063.stdout:507: delete oid 45 current snap is 63 2024-08-17T16:56:49.786 INFO:tasks.rados.rados.0.smithi063.stdout:507: done (0 left) 2024-08-17T16:56:49.786 INFO:tasks.rados.rados.0.smithi063.stdout:508: snap_create 2024-08-17T16:56:49.847 INFO:tasks.rados.rados.0.smithi063.stdout:508: done (0 left) 2024-08-17T16:56:49.847 INFO:tasks.rados.rados.0.smithi063.stdout:509: read oid 2 snap -1 2024-08-17T16:56:49.847 INFO:tasks.rados.rados.0.smithi063.stdout:509: expect deleted 2024-08-17T16:56:49.847 INFO:tasks.rados.rados.0.smithi063.stdout:510: rollback oid 25 current snap is 64 2024-08-17T16:56:49.847 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 25 to 38 2024-08-17T16:56:49.847 INFO:tasks.rados.rados.0.smithi063.stdout:511: write oid 50 current snap is 64 2024-08-17T16:56:49.847 INFO:tasks.rados.rados.0.smithi063.stdout:511: seq_num 226 ranges {598382=780511,2075401=451067,3107008=648769} 2024-08-17T16:56:49.855 INFO:tasks.rados.rados.0.smithi063.stdout:511: writing smithi06387500-50 from 598382 to 1378893 tid 1 2024-08-17T16:56:49.862 INFO:tasks.rados.rados.0.smithi063.stdout:511: writing smithi06387500-50 from 2075401 to 2526468 tid 2 2024-08-17T16:56:49.870 INFO:tasks.rados.rados.0.smithi063.stdout:511: writing smithi06387500-50 from 3107008 to 3755777 tid 3 2024-08-17T16:56:49.870 INFO:tasks.rados.rados.0.smithi063.stdout:512: read oid 34 snap -1 2024-08-17T16:56:49.870 INFO:tasks.rados.rados.0.smithi063.stdout:512: expect deleted 2024-08-17T16:56:49.870 INFO:tasks.rados.rados.0.smithi063.stdout:513: read oid 1 snap 60 2024-08-17T16:56:49.870 INFO:tasks.rados.rados.0.smithi063.stdout:513: expect (ObjNum 203 snap 56 seq_num 203) 2024-08-17T16:56:49.870 INFO:tasks.rados.rados.0.smithi063.stdout:514: read oid 39 snap -1 2024-08-17T16:56:49.870 INFO:tasks.rados.rados.0.smithi063.stdout:514: expect (ObjNum 190 snap 50 seq_num 190) 2024-08-17T16:56:49.870 INFO:tasks.rados.rados.0.smithi063.stdout:515: write oid 27 current snap is 64 2024-08-17T16:56:49.870 INFO:tasks.rados.rados.0.smithi063.stdout:515: seq_num 227 ranges {615189=455760,1733883=711078,3134431=460041} 2024-08-17T16:56:49.875 INFO:tasks.rados.rados.0.smithi063.stdout:515: writing smithi06387500-27 from 615189 to 1070949 tid 1 2024-08-17T16:56:49.883 INFO:tasks.rados.rados.0.smithi063.stdout:515: writing smithi06387500-27 from 1733883 to 2444961 tid 2 2024-08-17T16:56:49.890 INFO:tasks.rados.rados.0.smithi063.stdout:515: writing smithi06387500-27 from 3134431 to 3594472 tid 3 2024-08-17T16:56:49.890 INFO:tasks.rados.rados.0.smithi063.stdout:516: write oid 44 current snap is 64 2024-08-17T16:56:49.890 INFO:tasks.rados.rados.0.smithi063.stdout:516: seq_num 228 ranges {631996=531009,1792365=454736,2761854=671313} 2024-08-17T16:56:49.897 INFO:tasks.rados.rados.0.smithi063.stdout:516: writing smithi06387500-44 from 631996 to 1163005 tid 1 2024-08-17T16:56:49.902 INFO:tasks.rados.rados.0.smithi063.stdout:516: writing smithi06387500-44 from 1792365 to 2247101 tid 2 2024-08-17T16:56:49.910 INFO:tasks.rados.rados.0.smithi063.stdout:516: writing smithi06387500-44 from 2761854 to 3433167 tid 3 2024-08-17T16:56:49.910 INFO:tasks.rados.rados.0.smithi063.stdout:517: read oid 3 snap 49 2024-08-17T16:56:49.910 INFO:tasks.rados.rados.0.smithi063.stdout:517: expect deleted 2024-08-17T16:56:49.910 INFO:tasks.rados.rados.0.smithi063.stdout:518: rollback oid 9 current snap is 64 2024-08-17T16:56:49.910 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 9 to 60 2024-08-17T16:56:49.910 INFO:tasks.rados.rados.0.smithi063.stdout:510: finishing rollback tid 0 to smithi06387500-25 2024-08-17T16:56:49.910 INFO:tasks.rados.rados.0.smithi063.stdout:509: done (9 left) 2024-08-17T16:56:49.910 INFO:tasks.rados.rados.0.smithi063.stdout:519: snap_create 2024-08-17T16:56:49.910 INFO:tasks.rados.rados.0.smithi063.stdout:510: finishing rollback tid 1 to smithi06387500-25 2024-08-17T16:56:49.911 INFO:tasks.rados.rados.0.smithi063.stdout:510: finishing rollback tid 2 to smithi06387500-25 2024-08-17T16:56:49.911 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 453 (ObjNum 78 snap 9 seq_num 78) dirty exists 2024-08-17T16:56:49.911 INFO:tasks.rados.rados.0.smithi063.stdout:511: finishing write tid 1 to smithi06387500-50 2024-08-17T16:56:49.911 INFO:tasks.rados.rados.0.smithi063.stdout:511: finishing write tid 2 to smithi06387500-50 2024-08-17T16:56:49.911 INFO:tasks.rados.rados.0.smithi063.stdout:515: finishing write tid 1 to smithi06387500-27 2024-08-17T16:56:49.911 INFO:tasks.rados.rados.0.smithi063.stdout:511: finishing write tid 3 to smithi06387500-50 2024-08-17T16:56:49.911 INFO:tasks.rados.rados.0.smithi063.stdout:511: finishing write tid 4 to smithi06387500-50 2024-08-17T16:56:49.911 INFO:tasks.rados.rados.0.smithi063.stdout:511: finishing write tid 5 to smithi06387500-50 2024-08-17T16:56:49.911 INFO:tasks.rados.rados.0.smithi063.stdout:511: oid 50 updating version 0 to 305 2024-08-17T16:56:49.911 INFO:tasks.rados.rados.0.smithi063.stdout:511: oid 50 updating version 305 to 308 2024-08-17T16:56:49.911 INFO:tasks.rados.rados.0.smithi063.stdout:511: oid 50 version 308 is already newer than 307 2024-08-17T16:56:49.911 INFO:tasks.rados.rados.0.smithi063.stdout:511: oid 50 version 308 is already newer than 306 2024-08-17T16:56:49.912 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 308 (ObjNum 226 snap 64 seq_num 226) dirty exists 2024-08-17T16:56:49.912 INFO:tasks.rados.rados.0.smithi063.stdout:511: left oid 50 (ObjNum 226 snap 64 seq_num 226) 2024-08-17T16:56:49.912 INFO:tasks.rados.rados.0.smithi063.stdout:515: finishing write tid 2 to smithi06387500-27 2024-08-17T16:56:49.912 INFO:tasks.rados.rados.0.smithi063.stdout:513: expect (ObjNum 203 snap 56 seq_num 203) 2024-08-17T16:56:50.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:49 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[31315]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:56:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-08-17T16:56:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:49 smithi063 ceph-mon[42680]: pgmap v17: 89 pgs: 89 active+clean; 235 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 381 KiB/s wr, 30 op/s 2024-08-17T16:56:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:49 smithi063 ceph-mon[42680]: osdmap e192: 8 total, 8 up, 8 in 2024-08-17T16:56:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:49 smithi063 ceph-mon[31063]: pgmap v17: 89 pgs: 89 active+clean; 235 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 381 KiB/s wr, 30 op/s 2024-08-17T16:56:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:49 smithi063 ceph-mon[31063]: osdmap e192: 8 total, 8 up, 8 in 2024-08-17T16:56:50.067 INFO:tasks.rados.rados.0.smithi063.stdout:514: expect (ObjNum 190 snap 50 seq_num 190) 2024-08-17T16:56:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:49 smithi162 ceph-mon[34633]: pgmap v17: 89 pgs: 89 active+clean; 235 MiB data, 677 MiB used, 715 GiB / 715 GiB avail; 15 MiB/s rd, 381 KiB/s wr, 30 op/s 2024-08-17T16:56:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:49 smithi162 ceph-mon[34633]: osdmap e192: 8 total, 8 up, 8 in 2024-08-17T16:56:50.215 INFO:tasks.rados.rados.0.smithi063.stdout:515: finishing write tid 3 to smithi06387500-27 2024-08-17T16:56:50.215 INFO:tasks.rados.rados.0.smithi063.stdout:515: finishing write tid 4 to smithi06387500-27 2024-08-17T16:56:50.215 INFO:tasks.rados.rados.0.smithi063.stdout:515: finishing write tid 5 to smithi06387500-27 2024-08-17T16:56:50.215 INFO:tasks.rados.rados.0.smithi063.stdout:515: oid 27 updating version 0 to 63 2024-08-17T16:56:50.215 INFO:tasks.rados.rados.0.smithi063.stdout:515: oid 27 version 63 is already newer than 62 2024-08-17T16:56:50.215 INFO:tasks.rados.rados.0.smithi063.stdout:515: oid 27 version 63 is already newer than 60 2024-08-17T16:56:50.215 INFO:tasks.rados.rados.0.smithi063.stdout:515: oid 27 version 63 is already newer than 61 2024-08-17T16:56:50.215 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 63 (ObjNum 227 snap 64 seq_num 227) dirty exists 2024-08-17T16:56:50.215 INFO:tasks.rados.rados.0.smithi063.stdout:515: left oid 27 (ObjNum 227 snap 64 seq_num 227) 2024-08-17T16:56:50.215 INFO:tasks.rados.rados.0.smithi063.stdout:516: finishing write tid 1 to smithi06387500-44 2024-08-17T16:56:50.216 INFO:tasks.rados.rados.0.smithi063.stdout:516: finishing write tid 2 to smithi06387500-44 2024-08-17T16:56:50.216 INFO:tasks.rados.rados.0.smithi063.stdout:518: finishing rollback tid 0 to smithi06387500-9 2024-08-17T16:56:50.216 INFO:tasks.rados.rados.0.smithi063.stdout:518: finishing rollback tid 1 to smithi06387500-9 2024-08-17T16:56:50.216 INFO:tasks.rados.rados.0.smithi063.stdout:518: finishing rollback tid 2 to smithi06387500-9 2024-08-17T16:56:50.216 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 263 (ObjNum 151 snap 36 seq_num 151) dirty exists 2024-08-17T16:56:50.216 INFO:tasks.rados.rados.0.smithi063.stdout:516: finishing write tid 3 to smithi06387500-44 2024-08-17T16:56:50.216 INFO:tasks.rados.rados.0.smithi063.stdout:516: finishing write tid 4 to smithi06387500-44 2024-08-17T16:56:50.216 INFO:tasks.rados.rados.0.smithi063.stdout:516: finishing write tid 5 to smithi06387500-44 2024-08-17T16:56:50.216 INFO:tasks.rados.rados.0.smithi063.stdout:516: oid 44 updating version 0 to 98 2024-08-17T16:56:50.216 INFO:tasks.rados.rados.0.smithi063.stdout:516: oid 44 updating version 98 to 99 2024-08-17T16:56:50.216 INFO:tasks.rados.rados.0.smithi063.stdout:516: oid 44 version 99 is already newer than 97 2024-08-17T16:56:50.216 INFO:tasks.rados.rados.0.smithi063.stdout:516: oid 44 version 99 is already newer than 96 2024-08-17T16:56:50.216 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 99 (ObjNum 228 snap 64 seq_num 228) dirty exists 2024-08-17T16:56:50.216 INFO:tasks.rados.rados.0.smithi063.stdout:516: left oid 44 (ObjNum 228 snap 64 seq_num 228) 2024-08-17T16:56:50.853 INFO:tasks.rados.rados.0.smithi063.stdout:510: done (9 left) 2024-08-17T16:56:50.854 INFO:tasks.rados.rados.0.smithi063.stdout:511: done (8 left) 2024-08-17T16:56:50.854 INFO:tasks.rados.rados.0.smithi063.stdout:512: done (7 left) 2024-08-17T16:56:50.854 INFO:tasks.rados.rados.0.smithi063.stdout:513: done (6 left) 2024-08-17T16:56:50.854 INFO:tasks.rados.rados.0.smithi063.stdout:514: done (5 left) 2024-08-17T16:56:50.854 INFO:tasks.rados.rados.0.smithi063.stdout:515: done (4 left) 2024-08-17T16:56:50.854 INFO:tasks.rados.rados.0.smithi063.stdout:516: done (3 left) 2024-08-17T16:56:50.854 INFO:tasks.rados.rados.0.smithi063.stdout:517: done (2 left) 2024-08-17T16:56:50.854 INFO:tasks.rados.rados.0.smithi063.stdout:518: done (1 left) 2024-08-17T16:56:50.854 INFO:tasks.rados.rados.0.smithi063.stdout:519: done (0 left) 2024-08-17T16:56:50.854 INFO:tasks.rados.rados.0.smithi063.stdout:520: delete oid 22 current snap is 65 2024-08-17T16:56:50.855 INFO:tasks.rados.rados.0.smithi063.stdout:520: done (0 left) 2024-08-17T16:56:50.855 INFO:tasks.rados.rados.0.smithi063.stdout:521: snap_create 2024-08-17T16:56:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[31063]: osdmap e193: 8 total, 8 up, 8 in 2024-08-17T16:56:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[31063]: osdmap e194: 8 total, 8 up, 8 in 2024-08-17T16:56:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:56:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:56:51.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[42680]: osdmap e193: 8 total, 8 up, 8 in 2024-08-17T16:56:51.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[42680]: osdmap e194: 8 total, 8 up, 8 in 2024-08-17T16:56:51.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:51.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:51.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:51.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:56:51.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:50 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:56:51.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:50 smithi162 ceph-mon[34633]: osdmap e193: 8 total, 8 up, 8 in 2024-08-17T16:56:51.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:50 smithi162 ceph-mon[34633]: osdmap e194: 8 total, 8 up, 8 in 2024-08-17T16:56:51.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:50 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:51.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:50 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:51.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:50 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:50 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:50 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:50 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:50 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:50 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:50 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:50 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-08-17T16:56:51.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:50 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:51.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:50 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:56:51.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:50 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:56:51.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: Adjusting osd_memory_target on smithi162 to 2480M 2024-08-17T16:56:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:56:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:56:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:56:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:56:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: Updating smithi162:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:56:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:56:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: pgmap v21: 89 pgs: 89 active+clean; 248 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 8.2 MiB/s rd, 3.4 MiB/s wr, 21 op/s 2024-08-17T16:56:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:56:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:56:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: osdmap e195: 8 total, 8 up, 8 in 2024-08-17T16:56:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:51.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:51.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:51.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:51.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:51 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:51.860 INFO:tasks.rados.rados.0.smithi063.stdout:521: done (0 left) 2024-08-17T16:56:51.860 INFO:tasks.rados.rados.0.smithi063.stdout:522: snap_remove snap 55 2024-08-17T16:56:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: Adjusting osd_memory_target on smithi162 to 2480M 2024-08-17T16:56:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:56:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:56:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:56:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:56:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: Updating smithi162:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:56:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:56:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: pgmap v21: 89 pgs: 89 active+clean; 248 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 8.2 MiB/s rd, 3.4 MiB/s wr, 21 op/s 2024-08-17T16:56:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:56:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:56:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: osdmap e195: 8 total, 8 up, 8 in 2024-08-17T16:56:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:52.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:52.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: Adjusting osd_memory_target on smithi162 to 2480M 2024-08-17T16:56:52.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:56:52.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:56:52.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:56:52.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:56:52.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: Updating smithi162:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:56:52.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:56:52.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: pgmap v21: 89 pgs: 89 active+clean; 248 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 8.2 MiB/s rd, 3.4 MiB/s wr, 21 op/s 2024-08-17T16:56:52.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:56:52.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:56:52.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: osdmap e195: 8 total, 8 up, 8 in 2024-08-17T16:56:52.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:52.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:52.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:52.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:52.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:51 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:52.870 INFO:tasks.rados.rados.0.smithi063.stdout:522: done (0 left) 2024-08-17T16:56:52.870 INFO:tasks.rados.rados.0.smithi063.stdout:523: write oid 45 current snap is 66 2024-08-17T16:56:52.870 INFO:tasks.rados.rados.0.smithi063.stdout:523: seq_num 230 ranges {665610=797860,1909329=458405,2816700=410210} 2024-08-17T16:56:52.875 INFO:tasks.rados.rados.0.smithi063.stdout:523: writing smithi06387500-45 from 665610 to 1463470 tid 1 2024-08-17T16:56:52.880 INFO:tasks.rados.rados.0.smithi063.stdout:523: writing smithi06387500-45 from 1909329 to 2367734 tid 2 2024-08-17T16:56:52.885 INFO:tasks.rados.rados.0.smithi063.stdout:523: writing smithi06387500-45 from 2816700 to 3226910 tid 3 2024-08-17T16:56:52.885 INFO:tasks.rados.rados.0.smithi063.stdout:524: snap_remove snap 43 2024-08-17T16:56:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:52 smithi063 ceph-mon[42680]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T16:56:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:52 smithi063 ceph-mon[42680]: Reconfiguring daemon prometheus.a on smithi162 2024-08-17T16:56:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:52 smithi063 ceph-mon[42680]: osdmap e196: 8 total, 8 up, 8 in 2024-08-17T16:56:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:52 smithi063 ceph-mon[31063]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T16:56:53.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:52 smithi063 ceph-mon[31063]: Reconfiguring daemon prometheus.a on smithi162 2024-08-17T16:56:53.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:52 smithi063 ceph-mon[31063]: osdmap e196: 8 total, 8 up, 8 in 2024-08-17T16:56:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:52 smithi162 ceph-mon[34633]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T16:56:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:52 smithi162 ceph-mon[34633]: Reconfiguring daemon prometheus.a on smithi162 2024-08-17T16:56:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:52 smithi162 ceph-mon[34633]: osdmap e196: 8 total, 8 up, 8 in 2024-08-17T16:56:53.869 INFO:tasks.rados.rados.0.smithi063.stdout:523: finishing write tid 1 to smithi06387500-45 2024-08-17T16:56:53.869 INFO:tasks.rados.rados.0.smithi063.stdout:523: finishing write tid 2 to smithi06387500-45 2024-08-17T16:56:53.869 INFO:tasks.rados.rados.0.smithi063.stdout:523: finishing write tid 3 to smithi06387500-45 2024-08-17T16:56:53.869 INFO:tasks.rados.rados.0.smithi063.stdout:523: finishing write tid 4 to smithi06387500-45 2024-08-17T16:56:53.869 INFO:tasks.rados.rados.0.smithi063.stdout:523: finishing write tid 5 to smithi06387500-45 2024-08-17T16:56:53.870 INFO:tasks.rados.rados.0.smithi063.stdout:523: oid 45 updating version 0 to 51 2024-08-17T16:56:53.870 INFO:tasks.rados.rados.0.smithi063.stdout:523: oid 45 updating version 51 to 54 2024-08-17T16:56:53.870 INFO:tasks.rados.rados.0.smithi063.stdout:523: oid 45 version 54 is already newer than 52 2024-08-17T16:56:53.870 INFO:tasks.rados.rados.0.smithi063.stdout:523: oid 45 version 54 is already newer than 53 2024-08-17T16:56:53.870 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 54 (ObjNum 230 snap 66 seq_num 230) dirty exists 2024-08-17T16:56:53.870 INFO:tasks.rados.rados.0.smithi063.stdout:523: left oid 45 (ObjNum 230 snap 66 seq_num 230) 2024-08-17T16:56:53.870 INFO:tasks.rados.rados.0.smithi063.stdout:523: done (1 left) 2024-08-17T16:56:53.870 INFO:tasks.rados.rados.0.smithi063.stdout:524: done (0 left) 2024-08-17T16:56:53.870 INFO:tasks.rados.rados.0.smithi063.stdout:525: rollback oid 11 current snap is 66 2024-08-17T16:56:53.870 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 11 to 64 2024-08-17T16:56:53.870 INFO:tasks.rados.rados.0.smithi063.stdout:526: snap_create 2024-08-17T16:56:53.873 INFO:tasks.rados.rados.0.smithi063.stdout:525: finishing rollback tid 1 to smithi06387500-11 2024-08-17T16:56:53.873 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 78 (ObjNum 289 snap 0 seq_num 1027869619) dirty dne 2024-08-17T16:56:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:53 smithi063 ceph-mon[42680]: pgmap v24: 89 pgs: 89 active+clean; 248 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 8.9 MiB/s rd, 3.7 MiB/s wr, 23 op/s 2024-08-17T16:56:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:53 smithi063 ceph-mon[42680]: osdmap e197: 8 total, 8 up, 8 in 2024-08-17T16:56:54.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:53 smithi063 ceph-mon[31063]: pgmap v24: 89 pgs: 89 active+clean; 248 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 8.9 MiB/s rd, 3.7 MiB/s wr, 23 op/s 2024-08-17T16:56:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:53 smithi063 ceph-mon[31063]: osdmap e197: 8 total, 8 up, 8 in 2024-08-17T16:56:54.081 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:53 smithi162 ceph-mon[34633]: pgmap v24: 89 pgs: 89 active+clean; 248 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 8.9 MiB/s rd, 3.7 MiB/s wr, 23 op/s 2024-08-17T16:56:54.081 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:53 smithi162 ceph-mon[34633]: osdmap e197: 8 total, 8 up, 8 in 2024-08-17T16:56:54.345 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 systemd[1]: Stopping Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:56:54.811 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:56:54.519Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T16:56:54.811 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:56:54.520Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-17T16:56:54.811 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:56:54.520Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-17T16:56:54.811 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:56:54.520Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T16:56:54.811 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:56:54.520Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-17T16:56:54.811 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:56:54.520Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-17T16:56:54.811 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:56:54.520Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T16:56:54.812 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:56:54.520Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-17T16:56:54.812 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:56:54.520Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-17T16:56:54.812 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:56:54.520Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T16:56:54.812 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:56:54.520Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-17T16:56:54.812 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[66302]: ts=2024-08-17T16:56:54.520Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-17T16:56:54.812 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 podman[78214]: 2024-08-17 16:56:54.530044859 +0000 UTC m=+0.110255072 container died 015db799d7d403549212d342df5854ba9d1ce55ea85c6a7dfac03d778351fa3a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:56:54.847 INFO:tasks.rados.rados.0.smithi063.stdout:525: done (1 left) 2024-08-17T16:56:54.847 INFO:tasks.rados.rados.0.smithi063.stdout:526: done (0 left) 2024-08-17T16:56:54.847 INFO:tasks.rados.rados.0.smithi063.stdout:527: rollback oid 7 current snap is 67 2024-08-17T16:56:54.848 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 49 2024-08-17T16:56:54.848 INFO:tasks.rados.rados.0.smithi063.stdout:528: read oid 4 snap -1 2024-08-17T16:56:54.848 INFO:tasks.rados.rados.0.smithi063.stdout:528: expect (ObjNum 221 snap 61 seq_num 221) 2024-08-17T16:56:54.848 INFO:tasks.rados.rados.0.smithi063.stdout:529: write oid 34 current snap is 67 2024-08-17T16:56:54.848 INFO:tasks.rados.rados.0.smithi063.stdout:529: seq_num 231 ranges {682417=473109,1567811=602063,2844123=621482} 2024-08-17T16:56:54.854 INFO:tasks.rados.rados.0.smithi063.stdout:529: writing smithi06387500-34 from 682417 to 1155526 tid 1 2024-08-17T16:56:54.859 INFO:tasks.rados.rados.0.smithi063.stdout:529: writing smithi06387500-34 from 1567811 to 2169874 tid 2 2024-08-17T16:56:54.868 INFO:tasks.rados.rados.0.smithi063.stdout:529: writing smithi06387500-34 from 2844123 to 3465605 tid 3 2024-08-17T16:56:54.868 INFO:tasks.rados.rados.0.smithi063.stdout:530: snap_create 2024-08-17T16:56:54.868 INFO:tasks.rados.rados.0.smithi063.stdout:527: finishing rollback tid 0 to smithi06387500-7 2024-08-17T16:56:54.868 INFO:tasks.rados.rados.0.smithi063.stdout:527: finishing rollback tid 1 to smithi06387500-7 2024-08-17T16:56:54.868 INFO:tasks.rados.rados.0.smithi063.stdout:527: finishing rollback tid 2 to smithi06387500-7 2024-08-17T16:56:54.868 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 283 (ObjNum 156 snap 39 seq_num 156) dirty exists 2024-08-17T16:56:54.868 INFO:tasks.rados.rados.0.smithi063.stdout:529: finishing write tid 1 to smithi06387500-34 2024-08-17T16:56:54.868 INFO:tasks.rados.rados.0.smithi063.stdout:529: finishing write tid 2 to smithi06387500-34 2024-08-17T16:56:54.868 INFO:tasks.rados.rados.0.smithi063.stdout:528: expect (ObjNum 221 snap 61 seq_num 221) 2024-08-17T16:56:55.075 INFO:tasks.rados.rados.0.smithi063.stdout:529: finishing write tid 3 to smithi06387500-34 2024-08-17T16:56:55.075 INFO:tasks.rados.rados.0.smithi063.stdout:529: finishing write tid 4 to smithi06387500-34 2024-08-17T16:56:55.076 INFO:tasks.rados.rados.0.smithi063.stdout:529: finishing write tid 5 to smithi06387500-34 2024-08-17T16:56:55.076 INFO:tasks.rados.rados.0.smithi063.stdout:529: oid 34 updating version 0 to 325 2024-08-17T16:56:55.076 INFO:tasks.rados.rados.0.smithi063.stdout:529: oid 34 updating version 325 to 326 2024-08-17T16:56:55.076 INFO:tasks.rados.rados.0.smithi063.stdout:529: oid 34 version 326 is already newer than 323 2024-08-17T16:56:55.076 INFO:tasks.rados.rados.0.smithi063.stdout:529: oid 34 version 326 is already newer than 324 2024-08-17T16:56:55.076 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 326 (ObjNum 231 snap 67 seq_num 231) dirty exists 2024-08-17T16:56:55.076 INFO:tasks.rados.rados.0.smithi063.stdout:529: left oid 34 (ObjNum 231 snap 67 seq_num 231) 2024-08-17T16:56:55.095 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 podman[78214]: 2024-08-17 16:56:54.91773999 +0000 UTC m=+0.497950207 container cleanup 015db799d7d403549212d342df5854ba9d1ce55ea85c6a7dfac03d778351fa3a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:56:55.096 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:54 smithi162 bash[78214]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T16:56:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-mon[34633]: osdmap e198: 8 total, 8 up, 8 in 2024-08-17T16:56:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-mon[34633]: pgmap v27: 89 pgs: 89 active+clean; 257 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-08-17T16:56:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:54 smithi162 ceph-mon[34633]: osdmap e199: 8 total, 8 up, 8 in 2024-08-17T16:56:55.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:54 smithi063 ceph-mon[42680]: osdmap e198: 8 total, 8 up, 8 in 2024-08-17T16:56:55.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:54 smithi063 ceph-mon[42680]: pgmap v27: 89 pgs: 89 active+clean; 257 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-08-17T16:56:55.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:54 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:55.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:54 smithi063 ceph-mon[42680]: osdmap e199: 8 total, 8 up, 8 in 2024-08-17T16:56:55.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:54 smithi063 ceph-mon[31063]: osdmap e198: 8 total, 8 up, 8 in 2024-08-17T16:56:55.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:54 smithi063 ceph-mon[31063]: pgmap v27: 89 pgs: 89 active+clean; 257 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-08-17T16:56:55.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:54 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:56:55.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:54 smithi063 ceph-mon[31063]: osdmap e199: 8 total, 8 up, 8 in 2024-08-17T16:56:55.512 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 podman[78232]: 2024-08-17 16:56:55.144103367 +0000 UTC m=+0.615972447 container remove 015db799d7d403549212d342df5854ba9d1ce55ea85c6a7dfac03d778351fa3a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:56:55.513 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-17T16:56:55.513 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 systemd[1]: Stopped Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:56:55.513 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@prometheus.a.service: Consumed 1.817s CPU time. 2024-08-17T16:56:55.513 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 systemd[1]: Starting Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:56:55.767 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 podman[78329]: 2024-08-17 16:56:55.415964552 +0000 UTC m=+0.017822227 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-17T16:56:55.768 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 podman[78329]: 2024-08-17 16:56:55.528556508 +0000 UTC m=+0.130414207 container create 039ece187f07d7e763df51afa25c27a55c85a075e2ff1be29b66f8d1d18f36f2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:56:55.768 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 podman[78329]: 2024-08-17 16:56:55.73914125 +0000 UTC m=+0.340998915 container init 039ece187f07d7e763df51afa25c27a55c85a075e2ff1be29b66f8d1d18f36f2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:56:55.768 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 podman[78329]: 2024-08-17 16:56:55.744247169 +0000 UTC m=+0.346104822 container start 039ece187f07d7e763df51afa25c27a55c85a075e2ff1be29b66f8d1d18f36f2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:56:55.768 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.767Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-17T16:56:55.768 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.767Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-17T16:56:55.768 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.767Z caller=main.go:570 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi162 (none))" 2024-08-17T16:56:55.768 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.767Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T16:56:55.768 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.767Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T16:56:55.871 INFO:tasks.rados.rados.0.smithi063.stdout:527: done (3 left) 2024-08-17T16:56:55.871 INFO:tasks.rados.rados.0.smithi063.stdout:528: done (2 left) 2024-08-17T16:56:55.871 INFO:tasks.rados.rados.0.smithi063.stdout:529: done (1 left) 2024-08-17T16:56:55.871 INFO:tasks.rados.rados.0.smithi063.stdout:530: done (0 left) 2024-08-17T16:56:55.871 INFO:tasks.rados.rados.0.smithi063.stdout:531: delete oid 49 current snap is 68 2024-08-17T16:56:55.873 INFO:tasks.rados.rados.0.smithi063.stdout:531: done (0 left) 2024-08-17T16:56:55.873 INFO:tasks.rados.rados.0.smithi063.stdout:532: snap_remove snap 61 2024-08-17T16:56:56.096 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.768Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T16:56:56.096 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.769Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-17T16:56:56.096 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.770Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T16:56:56.096 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.770Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T16:56:56.096 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.772Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T16:56:56.096 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.772Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.74µs 2024-08-17T16:56:56.096 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.772Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T16:56:56.096 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.773Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-08-17T16:56:56.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.781Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-08-17T16:56:56.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.781Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-08-17T16:56:56.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.781Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=19.059µs wal_replay_duration=9.309487ms wbl_replay_duration=128ns total_replay_duration=9.346503ms 2024-08-17T16:56:56.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.782Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T16:56:56.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.782Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-17T16:56:56.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.782Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T16:56:56.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.804Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.249766ms db_storage=889ns remote_storage=1.698µs web_handler=428ns query_engine=743ns scrape=424.359µs scrape_sd=108.84µs notify=12.536µs notify_sd=21.165µs rules=20.34415ms tracing=6.41µs 2024-08-17T16:56:56.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.804Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-17T16:56:56.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:56:55.804Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T16:56:56.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 bash[78329]: 039ece187f07d7e763df51afa25c27a55c85a075e2ff1be29b66f8d1d18f36f2 2024-08-17T16:56:56.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:56:55 smithi162 systemd[1]: Started Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:56:56.873 INFO:tasks.rados.rados.0.smithi063.stdout:532: done (0 left) 2024-08-17T16:56:56.873 INFO:tasks.rados.rados.0.smithi063.stdout:533: write oid 45 current snap is 68 2024-08-17T16:56:56.873 INFO:tasks.rados.rados.0.smithi063.stdout:533: seq_num 233 ranges {716031=623607,2084775=605732,3298969=617062} 2024-08-17T16:56:56.881 INFO:tasks.rados.rados.0.smithi063.stdout:533: writing smithi06387500-45 from 716031 to 1339638 tid 1 2024-08-17T16:56:56.888 INFO:tasks.rados.rados.0.smithi063.stdout:533: writing smithi06387500-45 from 2084775 to 2690507 tid 2 2024-08-17T16:56:56.896 INFO:tasks.rados.rados.0.smithi063.stdout:533: writing smithi06387500-45 from 3298969 to 3916031 tid 3 2024-08-17T16:56:56.896 INFO:tasks.rados.rados.0.smithi063.stdout:534: delete oid 10 current snap is 68 2024-08-17T16:56:56.896 INFO:tasks.rados.rados.0.smithi063.stdout:533: finishing write tid 1 to smithi06387500-45 2024-08-17T16:56:56.896 INFO:tasks.rados.rados.0.smithi063.stdout:533: finishing write tid 2 to smithi06387500-45 2024-08-17T16:56:56.899 INFO:tasks.rados.rados.0.smithi063.stdout:533: finishing write tid 3 to smithi06387500-45 2024-08-17T16:56:56.899 INFO:tasks.rados.rados.0.smithi063.stdout:534: done (1 left) 2024-08-17T16:56:56.899 INFO:tasks.rados.rados.0.smithi063.stdout:535: write oid 48 current snap is 68 2024-08-17T16:56:56.899 INFO:tasks.rados.rados.0.smithi063.stdout:535: seq_num 235 ranges {749645=774105,2085386=609401,3237462=712183} 2024-08-17T16:56:56.908 INFO:tasks.rados.rados.0.smithi063.stdout:535: writing smithi06387500-48 from 749645 to 1523750 tid 1 2024-08-17T16:56:56.915 INFO:tasks.rados.rados.0.smithi063.stdout:535: writing smithi06387500-48 from 2085386 to 2694787 tid 2 2024-08-17T16:56:56.922 INFO:tasks.rados.rados.0.smithi063.stdout:535: writing smithi06387500-48 from 3237462 to 3949645 tid 3 2024-08-17T16:56:56.922 INFO:tasks.rados.rados.0.smithi063.stdout:536: snap_create 2024-08-17T16:56:56.922 INFO:tasks.rados.rados.0.smithi063.stdout:533: finishing write tid 4 to smithi06387500-45 2024-08-17T16:56:56.922 INFO:tasks.rados.rados.0.smithi063.stdout:533: finishing write tid 5 to smithi06387500-45 2024-08-17T16:56:56.923 INFO:tasks.rados.rados.0.smithi063.stdout:533: oid 45 updating version 0 to 62 2024-08-17T16:56:56.923 INFO:tasks.rados.rados.0.smithi063.stdout:533: oid 45 updating version 62 to 64 2024-08-17T16:56:56.923 INFO:tasks.rados.rados.0.smithi063.stdout:533: oid 45 version 64 is already newer than 63 2024-08-17T16:56:56.923 INFO:tasks.rados.rados.0.smithi063.stdout:533: oid 45 updating version 64 to 65 2024-08-17T16:56:56.923 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 65 (ObjNum 233 snap 68 seq_num 233) dirty exists 2024-08-17T16:56:56.923 INFO:tasks.rados.rados.0.smithi063.stdout:533: left oid 45 (ObjNum 233 snap 68 seq_num 233) 2024-08-17T16:56:56.923 INFO:tasks.rados.rados.0.smithi063.stdout:535: finishing write tid 1 to smithi06387500-48 2024-08-17T16:56:56.923 INFO:tasks.rados.rados.0.smithi063.stdout:535: finishing write tid 2 to smithi06387500-48 2024-08-17T16:56:56.925 INFO:tasks.rados.rados.0.smithi063.stdout:535: finishing write tid 3 to smithi06387500-48 2024-08-17T16:56:56.925 INFO:tasks.rados.rados.0.smithi063.stdout:535: finishing write tid 4 to smithi06387500-48 2024-08-17T16:56:56.925 INFO:tasks.rados.rados.0.smithi063.stdout:535: finishing write tid 5 to smithi06387500-48 2024-08-17T16:56:56.926 INFO:tasks.rados.rados.0.smithi063.stdout:535: oid 48 updating version 0 to 483 2024-08-17T16:56:56.926 INFO:tasks.rados.rados.0.smithi063.stdout:535: oid 48 updating version 483 to 485 2024-08-17T16:56:56.926 INFO:tasks.rados.rados.0.smithi063.stdout:535: oid 48 updating version 485 to 486 2024-08-17T16:56:56.926 INFO:tasks.rados.rados.0.smithi063.stdout:535: oid 48 version 486 is already newer than 484 2024-08-17T16:56:56.926 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 486 (ObjNum 235 snap 68 seq_num 235) dirty exists 2024-08-17T16:56:56.926 INFO:tasks.rados.rados.0.smithi063.stdout:535: left oid 48 (ObjNum 235 snap 68 seq_num 235) 2024-08-17T16:56:57.006 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[42680]: osdmap e200: 8 total, 8 up, 8 in 2024-08-17T16:56:57.006 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:57.006 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:57.006 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:56:57.006 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[42680]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:56:57.006 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:56:57.006 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[42680]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T16:56:57.007 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[42680]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T16:56:57.007 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[42680]: pgmap v30: 89 pgs: 89 active+clean; 257 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-08-17T16:56:57.007 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[31063]: osdmap e200: 8 total, 8 up, 8 in 2024-08-17T16:56:57.007 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:57.007 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:57.007 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:56:57.007 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[31063]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:56:57.007 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:56:57.007 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[31063]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T16:56:57.007 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[31063]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T16:56:57.007 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:56 smithi063 ceph-mon[31063]: pgmap v30: 89 pgs: 89 active+clean; 257 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-08-17T16:56:57.313 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:56 smithi162 ceph-mon[34633]: osdmap e200: 8 total, 8 up, 8 in 2024-08-17T16:56:57.313 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:56 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:57.313 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:56 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:57.313 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:56 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:56:57.313 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:56 smithi162 ceph-mon[34633]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:56:57.313 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:56 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:56:57.313 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:56 smithi162 ceph-mon[34633]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T16:56:57.313 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:56 smithi162 ceph-mon[34633]: Upgrade: Need to upgrade myself (mgr.x) 2024-08-17T16:56:57.314 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:56 smithi162 ceph-mon[34633]: pgmap v30: 89 pgs: 89 active+clean; 257 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-08-17T16:56:57.897 INFO:tasks.rados.rados.0.smithi063.stdout:533: done (2 left) 2024-08-17T16:56:57.898 INFO:tasks.rados.rados.0.smithi063.stdout:535: done (1 left) 2024-08-17T16:56:57.898 INFO:tasks.rados.rados.0.smithi063.stdout:536: done (0 left) 2024-08-17T16:56:57.898 INFO:tasks.rados.rados.0.smithi063.stdout:537: read oid 42 snap -1 2024-08-17T16:56:57.898 INFO:tasks.rados.rados.0.smithi063.stdout:537: expect (ObjNum 205 snap 57 seq_num 205) 2024-08-17T16:56:57.899 INFO:tasks.rados.rados.0.smithi063.stdout:538: snap_create 2024-08-17T16:56:57.906 INFO:tasks.rados.rados.0.smithi063.stdout:537: expect (ObjNum 205 snap 57 seq_num 205) 2024-08-17T16:56:58.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[42680]: osdmap e201: 8 total, 8 up, 8 in 2024-08-17T16:56:58.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[42680]: Upgrade: Updating mgr.y 2024-08-17T16:56:58.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:58.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:56:58.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:56:58.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:56:58.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:56:58.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[42680]: Deploying daemon mgr.y on smithi063 2024-08-17T16:56:58.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[31063]: osdmap e201: 8 total, 8 up, 8 in 2024-08-17T16:56:58.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[31063]: Upgrade: Updating mgr.y 2024-08-17T16:56:58.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:58.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:56:58.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:56:58.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:56:58.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:56:58.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:57 smithi063 ceph-mon[31063]: Deploying daemon mgr.y on smithi063 2024-08-17T16:56:58.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:57 smithi162 ceph-mon[34633]: osdmap e201: 8 total, 8 up, 8 in 2024-08-17T16:56:58.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:57 smithi162 ceph-mon[34633]: Upgrade: Updating mgr.y 2024-08-17T16:56:58.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:57 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:56:58.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:57 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:56:58.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:57 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:56:58.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:57 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:56:58.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:57 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:56:58.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:57 smithi162 ceph-mon[34633]: Deploying daemon mgr.y on smithi063 2024-08-17T16:56:58.910 INFO:tasks.rados.rados.0.smithi063.stdout:537: done (1 left) 2024-08-17T16:56:58.910 INFO:tasks.rados.rados.0.smithi063.stdout:538: done (0 left) 2024-08-17T16:56:58.910 INFO:tasks.rados.rados.0.smithi063.stdout:539: delete oid 14 current snap is 70 2024-08-17T16:56:58.912 INFO:tasks.rados.rados.0.smithi063.stdout:539: done (0 left) 2024-08-17T16:56:58.912 INFO:tasks.rados.rados.0.smithi063.stdout:540: write oid 24 current snap is 70 2024-08-17T16:56:58.913 INFO:tasks.rados.rados.0.smithi063.stdout:540: seq_num 237 ranges {783259=640956,1918703=613070,3008661=405467,3821939=161320} 2024-08-17T16:56:58.920 INFO:tasks.rados.rados.0.smithi063.stdout:540: writing smithi06387500-24 from 783259 to 1424215 tid 1 2024-08-17T16:56:58.926 INFO:tasks.rados.rados.0.smithi063.stdout:540: writing smithi06387500-24 from 1918703 to 2531773 tid 2 2024-08-17T16:56:58.932 INFO:tasks.rados.rados.0.smithi063.stdout:540: writing smithi06387500-24 from 3008661 to 3414128 tid 3 2024-08-17T16:56:58.937 INFO:tasks.rados.rados.0.smithi063.stdout:540: writing smithi06387500-24 from 3821939 to 3983259 tid 4 2024-08-17T16:56:58.937 INFO:tasks.rados.rados.0.smithi063.stdout:541: read oid 16 snap 38 2024-08-17T16:56:58.937 INFO:tasks.rados.rados.0.smithi063.stdout:541: expect (ObjNum 117 snap 20 seq_num 117) 2024-08-17T16:56:58.937 INFO:tasks.rados.rados.0.smithi063.stdout:542: snap_remove snap 49 2024-08-17T16:56:59.092 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:58 smithi063 ceph-mon[42680]: osdmap e202: 8 total, 8 up, 8 in 2024-08-17T16:56:59.093 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:58 smithi063 ceph-mon[42680]: pgmap v33: 89 pgs: 89 active+clean; 257 MiB data, 732 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:59.093 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:58 smithi063 ceph-mon[31063]: osdmap e202: 8 total, 8 up, 8 in 2024-08-17T16:56:59.093 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:58 smithi063 ceph-mon[31063]: pgmap v33: 89 pgs: 89 active+clean; 257 MiB data, 732 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:59.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:58 smithi162 ceph-mon[34633]: osdmap e202: 8 total, 8 up, 8 in 2024-08-17T16:56:59.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:58 smithi162 ceph-mon[34633]: pgmap v33: 89 pgs: 89 active+clean; 257 MiB data, 732 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:56:59.409 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:59 smithi063 systemd[1]: Stopping Ceph mgr.y for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:56:59.659 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:59 smithi063 podman[91491]: 2024-08-17 16:56:59.408289932 +0000 UTC m=+0.163693020 container died ba4a2894b308c957b3277cb4f56185e99f5f0219cec4b67d5701bd6948e777b8 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-08-17T16:56:59.915 INFO:tasks.rados.rados.0.smithi063.stdout:542: done (2 left) 2024-08-17T16:56:59.915 INFO:tasks.rados.rados.0.smithi063.stdout:543: snap_create 2024-08-17T16:56:59.915 INFO:tasks.rados.rados.0.smithi063.stdout:540: finishing write tid 1 to smithi06387500-24 2024-08-17T16:56:59.915 INFO:tasks.rados.rados.0.smithi063.stdout:540: finishing write tid 2 to smithi06387500-24 2024-08-17T16:56:59.915 INFO:tasks.rados.rados.0.smithi063.stdout:540: finishing write tid 3 to smithi06387500-24 2024-08-17T16:56:59.915 INFO:tasks.rados.rados.0.smithi063.stdout:540: finishing write tid 4 to smithi06387500-24 2024-08-17T16:56:59.916 INFO:tasks.rados.rados.0.smithi063.stdout:540: finishing write tid 5 to smithi06387500-24 2024-08-17T16:56:59.916 INFO:tasks.rados.rados.0.smithi063.stdout:540: finishing write tid 6 to smithi06387500-24 2024-08-17T16:56:59.916 INFO:tasks.rados.rados.0.smithi063.stdout:540: oid 24 updating version 0 to 190 2024-08-17T16:56:59.916 INFO:tasks.rados.rados.0.smithi063.stdout:540: oid 24 updating version 190 to 191 2024-08-17T16:56:59.916 INFO:tasks.rados.rados.0.smithi063.stdout:540: oid 24 updating version 191 to 192 2024-08-17T16:56:59.916 INFO:tasks.rados.rados.0.smithi063.stdout:540: oid 24 updating version 192 to 194 2024-08-17T16:56:59.916 INFO:tasks.rados.rados.0.smithi063.stdout:540: oid 24 version 194 is already newer than 193 2024-08-17T16:56:59.916 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 194 (ObjNum 237 snap 70 seq_num 237) dirty exists 2024-08-17T16:56:59.916 INFO:tasks.rados.rados.0.smithi063.stdout:540: left oid 24 (ObjNum 237 snap 70 seq_num 237) 2024-08-17T16:56:59.916 INFO:tasks.rados.rados.0.smithi063.stdout:541: expect (ObjNum 117 snap 20 seq_num 117) 2024-08-17T16:56:59.960 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:56:59 smithi063 ceph-mon[31063]: osdmap e203: 8 total, 8 up, 8 in 2024-08-17T16:56:59.960 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:59 smithi063 podman[91491]: 2024-08-17 16:56:59.741601367 +0000 UTC m=+0.497004443 container cleanup ba4a2894b308c957b3277cb4f56185e99f5f0219cec4b67d5701bd6948e777b8 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-ced649f, io.buildah.version=1.36.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-08-17T16:56:59.960 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:59 smithi063 bash[91491]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y 2024-08-17T16:56:59.961 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:56:59 smithi063 ceph-mon[42680]: osdmap e203: 8 total, 8 up, 8 in 2024-08-17T16:57:00.223 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:59 smithi063 podman[91511]: 2024-08-17 16:56:59.959829087 +0000 UTC m=+0.555059247 container remove ba4a2894b308c957b3277cb4f56185e99f5f0219cec4b67d5701bd6948e777b8 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-08-17T16:57:00.223 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:56:59 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-08-17T16:57:00.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:56:59 smithi162 ceph-mon[34633]: osdmap e203: 8 total, 8 up, 8 in 2024-08-17T16:57:00.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:00 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-08-17T16:57:00.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:00 smithi063 systemd[1]: Stopped Ceph mgr.y for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:57:00.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:00 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.y.service: Consumed 48.735s CPU time. 2024-08-17T16:57:00.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:00 smithi063 systemd[1]: Starting Ceph mgr.y for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:57:00.910 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:00 smithi063 podman[91642]: 2024-08-17 16:57:00.551491692 +0000 UTC m=+0.018449533 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T16:57:00.910 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:00 smithi063 podman[91642]: 2024-08-17 16:57:00.652358389 +0000 UTC m=+0.119316222 container create 943e639cede62db3b1275abf348fc58bfaf9f5173172413b82f4a5875f340ed5 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, ceph=True, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-08-17T16:57:00.910 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:00 smithi063 podman[91642]: 2024-08-17 16:57:00.879860742 +0000 UTC m=+0.346818581 container init 943e639cede62db3b1275abf348fc58bfaf9f5173172413b82f4a5875f340ed5 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-08-17T16:57:00.910 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:00 smithi063 podman[91642]: 2024-08-17 16:57:00.884949835 +0000 UTC m=+0.351907665 container start 943e639cede62db3b1275abf348fc58bfaf9f5173172413b82f4a5875f340ed5 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T16:57:00.919 INFO:tasks.rados.rados.0.smithi063.stdout:540: done (2 left) 2024-08-17T16:57:00.920 INFO:tasks.rados.rados.0.smithi063.stdout:541: done (1 left) 2024-08-17T16:57:00.920 INFO:tasks.rados.rados.0.smithi063.stdout:543: done (0 left) 2024-08-17T16:57:00.920 INFO:tasks.rados.rados.0.smithi063.stdout:544: write oid 42 current snap is 71 2024-08-17T16:57:00.920 INFO:tasks.rados.rados.0.smithi063.stdout:544: seq_num 238 ranges {400066=716205,1860832=139234} 2024-08-17T16:57:00.927 INFO:tasks.rados.rados.0.smithi063.stdout:544: writing smithi06387500-42 from 400066 to 1116271 tid 1 2024-08-17T16:57:00.932 INFO:tasks.rados.rados.0.smithi063.stdout:544: writing smithi06387500-42 from 1860832 to 2000066 tid 2 2024-08-17T16:57:00.933 INFO:tasks.rados.rados.0.smithi063.stdout:544: finishing write tid 1 to smithi06387500-42 2024-08-17T16:57:00.933 INFO:tasks.rados.rados.0.smithi063.stdout:545: snap_remove snap 62 2024-08-17T16:57:01.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:00 smithi063 ceph-mon[42680]: osdmap e204: 8 total, 8 up, 8 in 2024-08-17T16:57:01.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:00 smithi063 ceph-mon[42680]: pgmap v36: 89 pgs: 89 active+clean; 269 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T16:57:01.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:00 smithi063 ceph-mon[31063]: osdmap e204: 8 total, 8 up, 8 in 2024-08-17T16:57:01.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:00 smithi063 ceph-mon[31063]: pgmap v36: 89 pgs: 89 active+clean; 269 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T16:57:01.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:00 smithi063 ceph-mgr[91679]: -- 172.21.15.63:0/3393191590 <== mon.0 v2:172.21.15.63:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55edf33b90e0 con 0x55edf3397000 2024-08-17T16:57:01.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:00 smithi063 bash[91642]: 943e639cede62db3b1275abf348fc58bfaf9f5173172413b82f4a5875f340ed5 2024-08-17T16:57:01.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:00 smithi063 systemd[1]: Started Ceph mgr.y for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:57:01.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:00 smithi162 ceph-mon[34633]: osdmap e204: 8 total, 8 up, 8 in 2024-08-17T16:57:01.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:00 smithi162 ceph-mon[34633]: pgmap v36: 89 pgs: 89 active+clean; 269 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T16:57:01.927 INFO:tasks.rados.rados.0.smithi063.stdout:545: done (1 left) 2024-08-17T16:57:01.928 INFO:tasks.rados.rados.0.smithi063.stdout:546: snap_create 2024-08-17T16:57:01.928 INFO:tasks.rados.rados.0.smithi063.stdout:544: finishing write tid 2 to smithi06387500-42 2024-08-17T16:57:01.928 INFO:tasks.rados.rados.0.smithi063.stdout:544: finishing write tid 3 to smithi06387500-42 2024-08-17T16:57:01.928 INFO:tasks.rados.rados.0.smithi063.stdout:544: finishing write tid 4 to smithi06387500-42 2024-08-17T16:57:01.928 INFO:tasks.rados.rados.0.smithi063.stdout:544: oid 42 updating version 0 to 342 2024-08-17T16:57:01.928 INFO:tasks.rados.rados.0.smithi063.stdout:544: oid 42 version 342 is already newer than 341 2024-08-17T16:57:01.928 INFO:tasks.rados.rados.0.smithi063.stdout:544: oid 42 version 342 is already newer than 340 2024-08-17T16:57:01.928 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 342 (ObjNum 238 snap 71 seq_num 238) dirty exists 2024-08-17T16:57:01.928 INFO:tasks.rados.rados.0.smithi063.stdout:544: left oid 42 (ObjNum 238 snap 71 seq_num 238) 2024-08-17T16:57:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:01 smithi162 ceph-mon[34633]: osdmap e205: 8 total, 8 up, 8 in 2024-08-17T16:57:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:01 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:01 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:01 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:02.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:01 smithi063 ceph-mon[31063]: osdmap e205: 8 total, 8 up, 8 in 2024-08-17T16:57:02.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:01 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:02.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:01 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:02.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:01 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:02.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:01 smithi063 ceph-mon[42680]: osdmap e205: 8 total, 8 up, 8 in 2024-08-17T16:57:02.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:01 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:02.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:01 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:02.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:01 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:02.930 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:02 smithi063 ceph-mon[31063]: osdmap e206: 8 total, 8 up, 8 in 2024-08-17T16:57:02.930 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:02 smithi063 ceph-mon[31063]: pgmap v39: 89 pgs: 89 active+clean; 269 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T16:57:02.934 INFO:tasks.rados.rados.0.smithi063.stdout:544: done (1 left) 2024-08-17T16:57:02.934 INFO:tasks.rados.rados.0.smithi063.stdout:546: done (0 left) 2024-08-17T16:57:02.934 INFO:tasks.rados.rados.0.smithi063.stdout:547: snap_remove snap 65 2024-08-17T16:57:03.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:02 smithi063 ceph-mon[42680]: osdmap e206: 8 total, 8 up, 8 in 2024-08-17T16:57:03.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:02 smithi063 ceph-mon[42680]: pgmap v39: 89 pgs: 89 active+clean; 269 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T16:57:03.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:02 smithi162 ceph-mon[34633]: osdmap e206: 8 total, 8 up, 8 in 2024-08-17T16:57:03.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:02 smithi162 ceph-mon[34633]: pgmap v39: 89 pgs: 89 active+clean; 269 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T16:57:03.845 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:03 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:57:03] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.43.0" 2024-08-17T16:57:03.935 INFO:tasks.rados.rados.0.smithi063.stdout:547: done (0 left) 2024-08-17T16:57:03.936 INFO:tasks.rados.rados.0.smithi063.stdout:548: rollback oid 39 current snap is 72 2024-08-17T16:57:03.936 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 39 to 64 2024-08-17T16:57:03.936 INFO:tasks.rados.rados.0.smithi063.stdout:549: read oid 9 snap -1 2024-08-17T16:57:03.936 INFO:tasks.rados.rados.0.smithi063.stdout:549: expect (ObjNum 151 snap 36 seq_num 151) 2024-08-17T16:57:03.936 INFO:tasks.rados.rados.0.smithi063.stdout:550: snap_create 2024-08-17T16:57:03.937 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:03 smithi063 ceph-mon[31063]: osdmap e207: 8 total, 8 up, 8 in 2024-08-17T16:57:03.942 INFO:tasks.rados.rados.0.smithi063.stdout:548: finishing rollback tid 0 to smithi06387500-39 2024-08-17T16:57:03.943 INFO:tasks.rados.rados.0.smithi063.stdout:548: finishing rollback tid 1 to smithi06387500-39 2024-08-17T16:57:03.943 INFO:tasks.rados.rados.0.smithi063.stdout:548: finishing rollback tid 2 to smithi06387500-39 2024-08-17T16:57:03.943 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 75 (ObjNum 190 snap 50 seq_num 190) dirty exists 2024-08-17T16:57:03.946 INFO:tasks.rados.rados.0.smithi063.stdout:549: expect (ObjNum 151 snap 36 seq_num 151) 2024-08-17T16:57:04.252 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:03 smithi063 ceph-mon[42680]: osdmap e207: 8 total, 8 up, 8 in 2024-08-17T16:57:04.266 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:03 smithi162 ceph-mon[34633]: osdmap e207: 8 total, 8 up, 8 in 2024-08-17T16:57:04.850 INFO:tasks.rados.rados.0.smithi063.stdout:548: done (2 left) 2024-08-17T16:57:04.850 INFO:tasks.rados.rados.0.smithi063.stdout:549: done (1 left) 2024-08-17T16:57:04.850 INFO:tasks.rados.rados.0.smithi063.stdout:550: done (0 left) 2024-08-17T16:57:04.850 INFO:tasks.rados.rados.0.smithi063.stdout:551: write oid 40 current snap is 73 2024-08-17T16:57:04.850 INFO:tasks.rados.rados.0.smithi063.stdout:551: seq_num 239 ranges {416873=791454,1919314=97559} 2024-08-17T16:57:04.856 INFO:tasks.rados.rados.0.smithi063.stdout:551: writing smithi06387500-40 from 416873 to 1208327 tid 1 2024-08-17T16:57:04.862 INFO:tasks.rados.rados.0.smithi063.stdout:551: writing smithi06387500-40 from 1919314 to 2016873 tid 2 2024-08-17T16:57:04.862 INFO:tasks.rados.rados.0.smithi063.stdout:552: read oid 4 snap 69 2024-08-17T16:57:04.862 INFO:tasks.rados.rados.0.smithi063.stdout:552: expect (ObjNum 221 snap 61 seq_num 221) 2024-08-17T16:57:04.862 INFO:tasks.rados.rados.0.smithi063.stdout:553: write oid 24 current snap is 73 2024-08-17T16:57:04.862 INFO:tasks.rados.rados.0.smithi063.stdout:553: seq_num 240 ranges {433680=466703,1577796=455884} 2024-08-17T16:57:04.867 INFO:tasks.rados.rados.0.smithi063.stdout:553: writing smithi06387500-24 from 433680 to 900383 tid 1 2024-08-17T16:57:04.875 INFO:tasks.rados.rados.0.smithi063.stdout:553: writing smithi06387500-24 from 1577796 to 2033680 tid 2 2024-08-17T16:57:04.876 INFO:tasks.rados.rados.0.smithi063.stdout:554: rollback oid 23 current snap is 73 2024-08-17T16:57:04.876 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 23 to 36 2024-08-17T16:57:04.876 INFO:tasks.rados.rados.0.smithi063.stdout:551: finishing write tid 1 to smithi06387500-40 2024-08-17T16:57:04.876 INFO:tasks.rados.rados.0.smithi063.stdout:551: finishing write tid 2 to smithi06387500-40 2024-08-17T16:57:04.876 INFO:tasks.rados.rados.0.smithi063.stdout:551: finishing write tid 3 to smithi06387500-40 2024-08-17T16:57:04.876 INFO:tasks.rados.rados.0.smithi063.stdout:551: finishing write tid 4 to smithi06387500-40 2024-08-17T16:57:04.876 INFO:tasks.rados.rados.0.smithi063.stdout:551: oid 40 updating version 0 to 154 2024-08-17T16:57:04.876 INFO:tasks.rados.rados.0.smithi063.stdout:551: oid 40 updating version 154 to 155 2024-08-17T16:57:04.876 INFO:tasks.rados.rados.0.smithi063.stdout:551: oid 40 updating version 155 to 156 2024-08-17T16:57:04.876 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 156 (ObjNum 239 snap 73 seq_num 239) dirty exists 2024-08-17T16:57:04.876 INFO:tasks.rados.rados.0.smithi063.stdout:551: left oid 40 (ObjNum 239 snap 73 seq_num 239) 2024-08-17T16:57:04.877 INFO:tasks.rados.rados.0.smithi063.stdout:551: done (3 left) 2024-08-17T16:57:04.877 INFO:tasks.rados.rados.0.smithi063.stdout:555: read oid 10 snap -1 2024-08-17T16:57:04.877 INFO:tasks.rados.rados.0.smithi063.stdout:555: expect deleted 2024-08-17T16:57:04.877 INFO:tasks.rados.rados.0.smithi063.stdout:556: write oid 20 current snap is 73 2024-08-17T16:57:04.877 INFO:tasks.rados.rados.0.smithi063.stdout:553: finishing write tid 1 to smithi06387500-24 2024-08-17T16:57:04.877 INFO:tasks.rados.rados.0.smithi063.stdout:556: seq_num 241 ranges {450487=541952,1636278=414209} 2024-08-17T16:57:04.881 INFO:tasks.rados.rados.0.smithi063.stdout:556: writing smithi06387500-20 from 450487 to 992439 tid 1 2024-08-17T16:57:04.888 INFO:tasks.rados.rados.0.smithi063.stdout:556: writing smithi06387500-20 from 1636278 to 2050487 tid 2 2024-08-17T16:57:04.889 INFO:tasks.rados.rados.0.smithi063.stdout:557: delete oid 11 current snap is 73 2024-08-17T16:57:04.889 INFO:tasks.rados.rados.0.smithi063.stdout:552: expect (ObjNum 221 snap 61 seq_num 221) 2024-08-17T16:57:05.099 INFO:tasks.rados.rados.0.smithi063.stdout:554: finishing rollback tid 0 to smithi06387500-23 2024-08-17T16:57:05.099 INFO:tasks.rados.rados.0.smithi063.stdout:554: finishing rollback tid 1 to smithi06387500-23 2024-08-17T16:57:05.099 INFO:tasks.rados.rados.0.smithi063.stdout:552: done (5 left) 2024-08-17T16:57:05.100 INFO:tasks.rados.rados.0.smithi063.stdout:555: done (4 left) 2024-08-17T16:57:05.100 INFO:tasks.rados.rados.0.smithi063.stdout:557: done (3 left) 2024-08-17T16:57:05.100 INFO:tasks.rados.rados.0.smithi063.stdout:558: write oid 25 current snap is 73 2024-08-17T16:57:05.100 INFO:tasks.rados.rados.0.smithi063.stdout:554: finishing rollback tid 2 to smithi06387500-23 2024-08-17T16:57:05.100 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 219 (ObjNum 125 snap 23 seq_num 125) dirty exists 2024-08-17T16:57:05.100 INFO:tasks.rados.rados.0.smithi063.stdout:553: finishing write tid 2 to smithi06387500-24 2024-08-17T16:57:05.100 INFO:tasks.rados.rados.0.smithi063.stdout:558: seq_num 243 ranges {484101=692450,1636889=447212} 2024-08-17T16:57:05.107 INFO:tasks.rados.rados.0.smithi063.stdout:558: writing smithi06387500-25 from 484101 to 1176551 tid 1 2024-08-17T16:57:05.113 INFO:tasks.rados.rados.0.smithi063.stdout:558: writing smithi06387500-25 from 1636889 to 2084101 tid 2 2024-08-17T16:57:05.113 INFO:tasks.rados.rados.0.smithi063.stdout:554: done (3 left) 2024-08-17T16:57:05.113 INFO:tasks.rados.rados.0.smithi063.stdout:559: rollback oid 38 current snap is 73 2024-08-17T16:57:05.113 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 38 to 67 2024-08-17T16:57:05.114 INFO:tasks.rados.rados.0.smithi063.stdout:553: finishing write tid 3 to smithi06387500-24 2024-08-17T16:57:05.114 INFO:tasks.rados.rados.0.smithi063.stdout:560: delete oid 32 current snap is 73 2024-08-17T16:57:05.114 INFO:tasks.rados.rados.0.smithi063.stdout:553: finishing write tid 4 to smithi06387500-24 2024-08-17T16:57:05.114 INFO:tasks.rados.rados.0.smithi063.stdout:553: oid 24 updating version 0 to 199 2024-08-17T16:57:05.114 INFO:tasks.rados.rados.0.smithi063.stdout:553: oid 24 updating version 199 to 200 2024-08-17T16:57:05.114 INFO:tasks.rados.rados.0.smithi063.stdout:553: oid 24 version 200 is already newer than 198 2024-08-17T16:57:05.114 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 200 (ObjNum 240 snap 73 seq_num 240) dirty exists 2024-08-17T16:57:05.114 INFO:tasks.rados.rados.0.smithi063.stdout:553: left oid 24 (ObjNum 240 snap 73 seq_num 240) 2024-08-17T16:57:05.114 INFO:tasks.rados.rados.0.smithi063.stdout:556: finishing write tid 1 to smithi06387500-20 2024-08-17T16:57:05.114 INFO:tasks.rados.rados.0.smithi063.stdout:556: finishing write tid 2 to smithi06387500-20 2024-08-17T16:57:05.114 INFO:tasks.rados.rados.0.smithi063.stdout:556: finishing write tid 3 to smithi06387500-20 2024-08-17T16:57:05.114 INFO:tasks.rados.rados.0.smithi063.stdout:556: finishing write tid 4 to smithi06387500-20 2024-08-17T16:57:05.115 INFO:tasks.rados.rados.0.smithi063.stdout:556: oid 20 updating version 0 to 171 2024-08-17T16:57:05.115 INFO:tasks.rados.rados.0.smithi063.stdout:556: oid 20 version 171 is already newer than 169 2024-08-17T16:57:05.115 INFO:tasks.rados.rados.0.smithi063.stdout:556: oid 20 version 171 is already newer than 170 2024-08-17T16:57:05.115 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 171 (ObjNum 241 snap 73 seq_num 241) dirty exists 2024-08-17T16:57:05.115 INFO:tasks.rados.rados.0.smithi063.stdout:556: left oid 20 (ObjNum 241 snap 73 seq_num 241) 2024-08-17T16:57:05.115 INFO:tasks.rados.rados.0.smithi063.stdout:558: finishing write tid 1 to smithi06387500-25 2024-08-17T16:57:05.116 INFO:tasks.rados.rados.0.smithi063.stdout:559: finishing rollback tid 1 to smithi06387500-38 2024-08-17T16:57:05.116 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 34 (ObjNum 574598384 snap 32710 seq_num 304) dirty dne 2024-08-17T16:57:05.116 INFO:tasks.rados.rados.0.smithi063.stdout:553: done (4 left) 2024-08-17T16:57:05.116 INFO:tasks.rados.rados.0.smithi063.stdout:556: done (3 left) 2024-08-17T16:57:05.116 INFO:tasks.rados.rados.0.smithi063.stdout:559: done (2 left) 2024-08-17T16:57:05.116 INFO:tasks.rados.rados.0.smithi063.stdout:560: done (1 left) 2024-08-17T16:57:05.116 INFO:tasks.rados.rados.0.smithi063.stdout:561: delete oid 34 current snap is 73 2024-08-17T16:57:05.116 INFO:tasks.rados.rados.0.smithi063.stdout:558: finishing write tid 2 to smithi06387500-25 2024-08-17T16:57:05.116 INFO:tasks.rados.rados.0.smithi063.stdout:558: finishing write tid 3 to smithi06387500-25 2024-08-17T16:57:05.116 INFO:tasks.rados.rados.0.smithi063.stdout:558: finishing write tid 4 to smithi06387500-25 2024-08-17T16:57:05.117 INFO:tasks.rados.rados.0.smithi063.stdout:558: oid 25 updating version 0 to 516 2024-08-17T16:57:05.117 INFO:tasks.rados.rados.0.smithi063.stdout:558: oid 25 updating version 516 to 517 2024-08-17T16:57:05.117 INFO:tasks.rados.rados.0.smithi063.stdout:558: oid 25 updating version 517 to 518 2024-08-17T16:57:05.117 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 518 (ObjNum 243 snap 73 seq_num 243) dirty exists 2024-08-17T16:57:05.117 INFO:tasks.rados.rados.0.smithi063.stdout:558: left oid 25 (ObjNum 243 snap 73 seq_num 243) 2024-08-17T16:57:05.117 INFO:tasks.rados.rados.0.smithi063.stdout:558: done (1 left) 2024-08-17T16:57:05.117 INFO:tasks.rados.rados.0.smithi063.stdout:561: done (0 left) 2024-08-17T16:57:05.117 INFO:tasks.rados.rados.0.smithi063.stdout:562: delete oid 24 current snap is 73 2024-08-17T16:57:05.118 INFO:tasks.rados.rados.0.smithi063.stdout:562: done (0 left) 2024-08-17T16:57:05.118 INFO:tasks.rados.rados.0.smithi063.stdout:563: write oid 4 current snap is 73 2024-08-17T16:57:05.118 INFO:tasks.rados.rados.0.smithi063.stdout:563: seq_num 247 ranges {551329=709799,1870817=280512} 2024-08-17T16:57:05.125 INFO:tasks.rados.rados.0.smithi063.stdout:563: writing smithi06387500-4 from 551329 to 1261128 tid 1 2024-08-17T16:57:05.130 INFO:tasks.rados.rados.0.smithi063.stdout:563: writing smithi06387500-4 from 1870817 to 2151329 tid 2 2024-08-17T16:57:05.130 INFO:tasks.rados.rados.0.smithi063.stdout:564: read oid 31 snap -1 2024-08-17T16:57:05.131 INFO:tasks.rados.rados.0.smithi063.stdout:564: expect (ObjNum 138 snap 30 seq_num 138) 2024-08-17T16:57:05.131 INFO:tasks.rados.rados.0.smithi063.stdout:565: snap_create 2024-08-17T16:57:05.131 INFO:tasks.rados.rados.0.smithi063.stdout:563: finishing write tid 1 to smithi06387500-4 2024-08-17T16:57:05.133 INFO:tasks.rados.rados.0.smithi063.stdout:563: finishing write tid 2 to smithi06387500-4 2024-08-17T16:57:05.133 INFO:tasks.rados.rados.0.smithi063.stdout:563: finishing write tid 3 to smithi06387500-4 2024-08-17T16:57:05.133 INFO:tasks.rados.rados.0.smithi063.stdout:564: expect (ObjNum 138 snap 30 seq_num 138) 2024-08-17T16:57:05.269 INFO:tasks.rados.rados.0.smithi063.stdout:563: finishing write tid 4 to smithi06387500-4 2024-08-17T16:57:05.269 INFO:tasks.rados.rados.0.smithi063.stdout:563: oid 4 updating version 0 to 521 2024-08-17T16:57:05.269 INFO:tasks.rados.rados.0.smithi063.stdout:563: oid 4 version 521 is already newer than 520 2024-08-17T16:57:05.269 INFO:tasks.rados.rados.0.smithi063.stdout:563: oid 4 updating version 521 to 522 2024-08-17T16:57:05.269 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 522 (ObjNum 247 snap 73 seq_num 247) dirty exists 2024-08-17T16:57:05.269 INFO:tasks.rados.rados.0.smithi063.stdout:563: left oid 4 (ObjNum 247 snap 73 seq_num 247) 2024-08-17T16:57:05.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:04 smithi063 ceph-mon[31063]: osdmap e208: 8 total, 8 up, 8 in 2024-08-17T16:57:05.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:04 smithi063 ceph-mon[31063]: pgmap v42: 89 pgs: 89 active+clean; 268 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 209 KiB/s wr, 3 op/s 2024-08-17T16:57:05.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:04 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:05.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:04 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:05.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:04 smithi063 ceph-mon[31063]: osdmap e209: 8 total, 8 up, 8 in 2024-08-17T16:57:05.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:04 smithi063 ceph-mon[42680]: osdmap e208: 8 total, 8 up, 8 in 2024-08-17T16:57:05.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:04 smithi063 ceph-mon[42680]: pgmap v42: 89 pgs: 89 active+clean; 268 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 209 KiB/s wr, 3 op/s 2024-08-17T16:57:05.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:04 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:05.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:04 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:05.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:04 smithi063 ceph-mon[42680]: osdmap e209: 8 total, 8 up, 8 in 2024-08-17T16:57:05.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:04 smithi162 ceph-mon[34633]: osdmap e208: 8 total, 8 up, 8 in 2024-08-17T16:57:05.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:04 smithi162 ceph-mon[34633]: pgmap v42: 89 pgs: 89 active+clean; 268 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 209 KiB/s wr, 3 op/s 2024-08-17T16:57:05.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:04 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:05.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:04 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:05.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:04 smithi162 ceph-mon[34633]: osdmap e209: 8 total, 8 up, 8 in 2024-08-17T16:57:05.950 INFO:tasks.rados.rados.0.smithi063.stdout:563: done (2 left) 2024-08-17T16:57:05.950 INFO:tasks.rados.rados.0.smithi063.stdout:564: done (1 left) 2024-08-17T16:57:05.950 INFO:tasks.rados.rados.0.smithi063.stdout:565: done (0 left) 2024-08-17T16:57:05.950 INFO:tasks.rados.rados.0.smithi063.stdout:566: write oid 36 current snap is 74 2024-08-17T16:57:05.950 INFO:tasks.rados.rados.0.smithi063.stdout:566: seq_num 248 ranges {568136=785048,1929299=238837} 2024-08-17T16:57:05.957 INFO:tasks.rados.rados.0.smithi063.stdout:566: writing smithi06387500-36 from 568136 to 1353184 tid 1 2024-08-17T16:57:05.961 INFO:tasks.rados.rados.0.smithi063.stdout:566: writing smithi06387500-36 from 1929299 to 2168136 tid 2 2024-08-17T16:57:05.962 INFO:tasks.rados.rados.0.smithi063.stdout:567: rollback oid 12 current snap is 74 2024-08-17T16:57:05.962 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 12 to 73 2024-08-17T16:57:05.962 INFO:tasks.rados.rados.0.smithi063.stdout:568: write oid 18 current snap is 74 2024-08-17T16:57:05.962 INFO:tasks.rados.rados.0.smithi063.stdout:568: seq_num 249 ranges {584943=460297,1587781=597162} 2024-08-17T16:57:05.967 INFO:tasks.rados.rados.0.smithi063.stdout:568: writing smithi06387500-18 from 584943 to 1045240 tid 1 2024-08-17T16:57:05.974 INFO:tasks.rados.rados.0.smithi063.stdout:568: writing smithi06387500-18 from 1587781 to 2184943 tid 2 2024-08-17T16:57:05.974 INFO:tasks.rados.rados.0.smithi063.stdout:569: rollback oid 21 current snap is 74 2024-08-17T16:57:05.974 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 21 to 68 2024-08-17T16:57:05.974 INFO:tasks.rados.rados.0.smithi063.stdout:566: finishing write tid 1 to smithi06387500-36 2024-08-17T16:57:05.974 INFO:tasks.rados.rados.0.smithi063.stdout:566: finishing write tid 2 to smithi06387500-36 2024-08-17T16:57:05.974 INFO:tasks.rados.rados.0.smithi063.stdout:566: finishing write tid 3 to smithi06387500-36 2024-08-17T16:57:05.974 INFO:tasks.rados.rados.0.smithi063.stdout:570: snap_remove snap 64 2024-08-17T16:57:06.499 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:06.499 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:06.499 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:06.499 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:06.499 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-mon[42680]: osdmap e210: 8 total, 8 up, 8 in 2024-08-17T16:57:06.499 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:06.500 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:06.500 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:06.500 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:06.500 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-mon[31063]: osdmap e210: 8 total, 8 up, 8 in 2024-08-17T16:57:06.500 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:57:06] ENGINE Bus STARTING 2024-08-17T16:57:06.500 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: CherryPy Checker: 2024-08-17T16:57:06.500 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: The Application mounted at '' has an empty config. 2024-08-17T16:57:06.500 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:57:06.500 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:57:06] ENGINE Serving on http://:::9283 2024-08-17T16:57:06.501 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:57:06] ENGINE Bus STARTED 2024-08-17T16:57:06.793 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:06 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:06.793 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:06 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:06.793 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:06 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:06.793 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:06 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:06.793 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:06 smithi162 ceph-mon[34633]: osdmap e210: 8 total, 8 up, 8 in 2024-08-17T16:57:06.952 INFO:tasks.rados.rados.0.smithi063.stdout:570: done (4 left) 2024-08-17T16:57:06.952 INFO:tasks.rados.rados.0.smithi063.stdout:571: read oid 48 snap -1 2024-08-17T16:57:06.952 INFO:tasks.rados.rados.0.smithi063.stdout:571: expect (ObjNum 235 snap 68 seq_num 235) 2024-08-17T16:57:06.952 INFO:tasks.rados.rados.0.smithi063.stdout:572: snap_remove snap 70 2024-08-17T16:57:07.514 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[31063]: Standby manager daemon y restarted 2024-08-17T16:57:07.514 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[31063]: Standby manager daemon y started 2024-08-17T16:57:07.514 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T16:57:07.514 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:57:07.514 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T16:57:07.514 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:57:07.514 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[31063]: pgmap v45: 89 pgs: 89 active+clean; 268 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 209 KiB/s wr, 3 op/s 2024-08-17T16:57:07.515 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[31063]: osdmap e211: 8 total, 8 up, 8 in 2024-08-17T16:57:07.515 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[42680]: Standby manager daemon y restarted 2024-08-17T16:57:07.515 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[42680]: Standby manager daemon y started 2024-08-17T16:57:07.515 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T16:57:07.515 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:57:07.515 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T16:57:07.515 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:57:07.515 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[42680]: pgmap v45: 89 pgs: 89 active+clean; 268 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 209 KiB/s wr, 3 op/s 2024-08-17T16:57:07.515 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:07 smithi063 ceph-mon[42680]: osdmap e211: 8 total, 8 up, 8 in 2024-08-17T16:57:07.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:07 smithi162 ceph-mon[34633]: Standby manager daemon y restarted 2024-08-17T16:57:07.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:07 smithi162 ceph-mon[34633]: Standby manager daemon y started 2024-08-17T16:57:07.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:07 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-08-17T16:57:07.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:07 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:57:07.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:07 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-08-17T16:57:07.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:07 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:57:07.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:07 smithi162 ceph-mon[34633]: pgmap v45: 89 pgs: 89 active+clean; 268 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 209 KiB/s wr, 3 op/s 2024-08-17T16:57:07.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:07 smithi162 ceph-mon[34633]: osdmap e211: 8 total, 8 up, 8 in 2024-08-17T16:57:07.951 INFO:tasks.rados.rados.0.smithi063.stdout:572: done (5 left) 2024-08-17T16:57:07.951 INFO:tasks.rados.rados.0.smithi063.stdout:573: snap_create 2024-08-17T16:57:07.951 INFO:tasks.rados.rados.0.smithi063.stdout:566: finishing write tid 4 to smithi06387500-36 2024-08-17T16:57:07.951 INFO:tasks.rados.rados.0.smithi063.stdout:566: oid 36 updating version 0 to 526 2024-08-17T16:57:07.951 INFO:tasks.rados.rados.0.smithi063.stdout:566: oid 36 version 526 is already newer than 525 2024-08-17T16:57:07.952 INFO:tasks.rados.rados.0.smithi063.stdout:566: oid 36 version 526 is already newer than 524 2024-08-17T16:57:07.952 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 526 (ObjNum 248 snap 74 seq_num 248) dirty exists 2024-08-17T16:57:07.952 INFO:tasks.rados.rados.0.smithi063.stdout:566: left oid 36 (ObjNum 248 snap 74 seq_num 248) 2024-08-17T16:57:07.952 INFO:tasks.rados.rados.0.smithi063.stdout:567: finishing rollback tid 1 to smithi06387500-12 2024-08-17T16:57:07.952 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 101 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T16:57:07.952 INFO:tasks.rados.rados.0.smithi063.stdout:568: finishing write tid 1 to smithi06387500-18 2024-08-17T16:57:07.952 INFO:tasks.rados.rados.0.smithi063.stdout:569: finishing rollback tid 0 to smithi06387500-21 2024-08-17T16:57:07.952 INFO:tasks.rados.rados.0.smithi063.stdout:568: finishing write tid 2 to smithi06387500-18 2024-08-17T16:57:07.952 INFO:tasks.rados.rados.0.smithi063.stdout:569: finishing rollback tid 1 to smithi06387500-21 2024-08-17T16:57:07.952 INFO:tasks.rados.rados.0.smithi063.stdout:568: finishing write tid 3 to smithi06387500-18 2024-08-17T16:57:07.952 INFO:tasks.rados.rados.0.smithi063.stdout:569: finishing rollback tid 2 to smithi06387500-21 2024-08-17T16:57:07.952 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 354 (ObjNum 224 snap 62 seq_num 224) dirty exists 2024-08-17T16:57:07.952 INFO:tasks.rados.rados.0.smithi063.stdout:568: finishing write tid 4 to smithi06387500-18 2024-08-17T16:57:07.953 INFO:tasks.rados.rados.0.smithi063.stdout:568: oid 18 updating version 0 to 159 2024-08-17T16:57:07.953 INFO:tasks.rados.rados.0.smithi063.stdout:568: oid 18 version 159 is already newer than 158 2024-08-17T16:57:07.953 INFO:tasks.rados.rados.0.smithi063.stdout:568: oid 18 updating version 159 to 160 2024-08-17T16:57:07.953 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 160 (ObjNum 249 snap 74 seq_num 249) dirty exists 2024-08-17T16:57:07.953 INFO:tasks.rados.rados.0.smithi063.stdout:568: left oid 18 (ObjNum 249 snap 74 seq_num 249) 2024-08-17T16:57:07.953 INFO:tasks.rados.rados.0.smithi063.stdout:571: expect (ObjNum 235 snap 68 seq_num 235) 2024-08-17T16:57:08.693 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:08 smithi063 ceph-mon[42680]: mgrmap e27: x(active, since 28s), standbys: y 2024-08-17T16:57:08.693 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:08 smithi063 ceph-mon[42680]: osdmap e212: 8 total, 8 up, 8 in 2024-08-17T16:57:08.694 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:08 smithi063 ceph-mon[31063]: mgrmap e27: x(active, since 28s), standbys: y 2024-08-17T16:57:08.694 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:08 smithi063 ceph-mon[31063]: osdmap e212: 8 total, 8 up, 8 in 2024-08-17T16:57:08.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:08 smithi162 ceph-mon[34633]: mgrmap e27: x(active, since 28s), standbys: y 2024-08-17T16:57:08.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:08 smithi162 ceph-mon[34633]: osdmap e212: 8 total, 8 up, 8 in 2024-08-17T16:57:08.884 INFO:tasks.rados.rados.0.smithi063.stdout:566: done (5 left) 2024-08-17T16:57:08.884 INFO:tasks.rados.rados.0.smithi063.stdout:567: done (4 left) 2024-08-17T16:57:08.884 INFO:tasks.rados.rados.0.smithi063.stdout:568: done (3 left) 2024-08-17T16:57:08.884 INFO:tasks.rados.rados.0.smithi063.stdout:569: done (2 left) 2024-08-17T16:57:08.884 INFO:tasks.rados.rados.0.smithi063.stdout:571: done (1 left) 2024-08-17T16:57:08.884 INFO:tasks.rados.rados.0.smithi063.stdout:573: done (0 left) 2024-08-17T16:57:08.884 INFO:tasks.rados.rados.0.smithi063.stdout:574: snap_create 2024-08-17T16:57:09.004 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:08 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:57:08] ENGINE Bus STOPPING 2024-08-17T16:57:09.004 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:57:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T16:57:09.279 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:57:09] ENGINE Bus STOPPED 2024-08-17T16:57:09.346 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:08 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: 2024-08-17T16:57:08.882+0000 7f0f508ec640 -1 mgr handle_mgr_map I was active but no longer am 2024-08-17T16:57:09.346 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:08 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: ignoring --setuser ceph since I am not root 2024-08-17T16:57:09.346 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:08 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: ignoring --setgroup ceph since I am not root 2024-08-17T16:57:09.346 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:08 smithi162 ceph-mgr[69897]: -- 172.21.15.162:0/2796552352 <== mon.1 v2:172.21.15.162:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55604a18b0e0 con 0x55604a169400 2024-08-17T16:57:09.885 INFO:tasks.rados.rados.0.smithi063.stdout:574: done (0 left) 2024-08-17T16:57:09.885 INFO:tasks.rados.rados.0.smithi063.stdout:575: snap_remove snap 59 2024-08-17T16:57:09.908 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:57:09.586+0000 7f9442922640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:57:09.908 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:57:09.586+0000 7f9442922640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:57:09.908 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:57:09.586+0000 7f9442922640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:57:09.908 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:57:09.586+0000 7f9442922640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:57:09.908 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:57:09.586+0000 7f9442922640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:57:09.908 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:57:09.586+0000 7f943f11b640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:57:09.908 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:57:09.586+0000 7f943f11b640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:57:09.908 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:57:09.586+0000 7f943f11b640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:57:09.909 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:57:09.586+0000 7f943f11b640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:57:09.909 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:57:09.586+0000 7f943f11b640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:57:09.909 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: pgmap v48: 89 pgs: 89 active+clean; 268 MiB data, 756 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:09.909 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:09.909 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:09.909 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:09.909 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:09.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:09.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:09.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T16:57:09.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T16:57:09.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: Activating manager daemon y 2024-08-17T16:57:09.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: osdmap e213: 8 total, 8 up, 8 in 2024-08-17T16:57:09.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24643 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-17T16:57:09.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: mgrmap e28: y(active, starting, since 0.00933891s) 2024-08-17T16:57:09.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:57:09.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:57:09.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:57:09.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi063.ultpqj"}]: dispatch 2024-08-17T16:57:09.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi162.bjkxnv"}]: dispatch 2024-08-17T16:57:09.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:57:09.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:57:09.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:57:09.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:57:09.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:57:09.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:57:09.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:57:09.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:57:09.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:57:09.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:57:09.912 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:57:09.912 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:57:09.912 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: Manager daemon y is now available 2024-08-17T16:57:09.912 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:09.912 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:57:09.912 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:09.912 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:57:09.912 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:57:09.912 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: pgmap v48: 89 pgs: 89 active+clean; 268 MiB data, 756 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:09.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:09.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:09.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:09.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:09.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:09.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:09.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T16:57:09.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T16:57:09.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: Activating manager daemon y 2024-08-17T16:57:09.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: osdmap e213: 8 total, 8 up, 8 in 2024-08-17T16:57:09.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24643 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-17T16:57:09.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: mgrmap e28: y(active, starting, since 0.00933891s) 2024-08-17T16:57:09.914 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:57:09.914 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:57:09.914 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:57:09.914 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi063.ultpqj"}]: dispatch 2024-08-17T16:57:09.914 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi162.bjkxnv"}]: dispatch 2024-08-17T16:57:09.914 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:57:09.914 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:57:09.914 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:57:09.914 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:57:09.914 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:57:09.914 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:57:09.914 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:57:09.915 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:57:09.915 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:57:09.915 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:57:09.915 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:57:09.915 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:57:09.915 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: Manager daemon y is now available 2024-08-17T16:57:09.915 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:09.915 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:57:09.915 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:09.915 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:57:09.915 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:57:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: pgmap v48: 89 pgs: 89 active+clean; 268 MiB data, 756 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' 2024-08-17T16:57:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24643 172.21.15.162:0/3809553871' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T16:57:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-08-17T16:57:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: Activating manager daemon y 2024-08-17T16:57:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: osdmap e213: 8 total, 8 up, 8 in 2024-08-17T16:57:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24643 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-08-17T16:57:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: mgrmap e28: y(active, starting, since 0.00933891s) 2024-08-17T16:57:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:57:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:57:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:57:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi063.ultpqj"}]: dispatch 2024-08-17T16:57:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi162.bjkxnv"}]: dispatch 2024-08-17T16:57:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:57:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:57:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:57:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:57:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:57:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:57:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:57:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:57:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:57:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:57:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:57:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:57:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: Manager daemon y is now available 2024-08-17T16:57:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:57:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:57:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:57:10.279 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:57:09] ENGINE Bus STARTING 2024-08-17T16:57:10.279 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:57:10] ENGINE Serving on http://:::9283 2024-08-17T16:57:10.280 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:57:10] ENGINE Bus STARTED 2024-08-17T16:57:10.890 INFO:tasks.rados.rados.0.smithi063.stdout:575: done (0 left) 2024-08-17T16:57:10.890 INFO:tasks.rados.rados.0.smithi063.stdout:576: delete oid 39 current snap is 76 2024-08-17T16:57:10.894 INFO:tasks.rados.rados.0.smithi063.stdout:576: done (0 left) 2024-08-17T16:57:10.895 INFO:tasks.rados.rados.0.smithi063.stdout:577: delete oid 19 current snap is 76 2024-08-17T16:57:10.897 INFO:tasks.rados.rados.0.smithi063.stdout:577: done (0 left) 2024-08-17T16:57:10.897 INFO:tasks.rados.rados.0.smithi063.stdout:578: read oid 6 snap -1 2024-08-17T16:57:10.897 INFO:tasks.rados.rados.0.smithi063.stdout:578: expect (ObjNum 178 snap 47 seq_num 178) 2024-08-17T16:57:10.897 INFO:tasks.rados.rados.0.smithi063.stdout:579: write oid 45 current snap is 76 2024-08-17T16:57:10.897 INFO:tasks.rados.rados.0.smithi063.stdout:579: seq_num 252 ranges {635364=402397,1763227=472137} 2024-08-17T16:57:10.903 INFO:tasks.rados.rados.0.smithi063.stdout:579: writing smithi06387500-45 from 635364 to 1037761 tid 1 2024-08-17T16:57:10.910 INFO:tasks.rados.rados.0.smithi063.stdout:579: writing smithi06387500-45 from 1763227 to 2235364 tid 2 2024-08-17T16:57:10.910 INFO:tasks.rados.rados.0.smithi063.stdout:580: snap_create 2024-08-17T16:57:10.910 INFO:tasks.rados.rados.0.smithi063.stdout:578: expect (ObjNum 178 snap 47 seq_num 178) 2024-08-17T16:57:11.087 INFO:tasks.rados.rados.0.smithi063.stdout:579: finishing write tid 1 to smithi06387500-45 2024-08-17T16:57:11.087 INFO:tasks.rados.rados.0.smithi063.stdout:579: finishing write tid 2 to smithi06387500-45 2024-08-17T16:57:11.087 INFO:tasks.rados.rados.0.smithi063.stdout:579: finishing write tid 3 to smithi06387500-45 2024-08-17T16:57:11.087 INFO:tasks.rados.rados.0.smithi063.stdout:579: finishing write tid 4 to smithi06387500-45 2024-08-17T16:57:11.087 INFO:tasks.rados.rados.0.smithi063.stdout:579: oid 45 updating version 0 to 88 2024-08-17T16:57:11.088 INFO:tasks.rados.rados.0.smithi063.stdout:579: oid 45 updating version 88 to 89 2024-08-17T16:57:11.088 INFO:tasks.rados.rados.0.smithi063.stdout:579: oid 45 version 89 is already newer than 87 2024-08-17T16:57:11.088 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 89 (ObjNum 252 snap 76 seq_num 252) dirty exists 2024-08-17T16:57:11.088 INFO:tasks.rados.rados.0.smithi063.stdout:579: left oid 45 (ObjNum 252 snap 76 seq_num 252) 2024-08-17T16:57:11.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:10 smithi162 ceph-mon[34633]: osdmap e214: 8 total, 8 up, 8 in 2024-08-17T16:57:11.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:10 smithi162 ceph-mon[34633]: mgrmap e29: y(active, since 1.02004s) 2024-08-17T16:57:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:10 smithi162 ceph-mon[34633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:57:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:10 smithi162 ceph-mon[34633]: pgmap v4: 89 pgs: 89 active+clean; 274 MiB data, 774 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:11.274 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-mon[42680]: osdmap e214: 8 total, 8 up, 8 in 2024-08-17T16:57:11.275 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-mon[42680]: mgrmap e29: y(active, since 1.02004s) 2024-08-17T16:57:11.275 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-mon[42680]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:57:11.275 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-mon[42680]: pgmap v4: 89 pgs: 89 active+clean; 274 MiB data, 774 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:11.275 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-mon[31063]: osdmap e214: 8 total, 8 up, 8 in 2024-08-17T16:57:11.275 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-mon[31063]: mgrmap e29: y(active, since 1.02004s) 2024-08-17T16:57:11.275 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-mon[31063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:57:11.275 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-mon[31063]: pgmap v4: 89 pgs: 89 active+clean; 274 MiB data, 774 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:11.275 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:57:10.939+0000 7f9425ea6640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:16:57:10] ENGINE Error in HTTPServer.serve 2024-08-17T16:57:11.276 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: Traceback (most recent call last): 2024-08-17T16:57:11.276 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:57:11.276 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: self._connections.run(self.expiration_interval) 2024-08-17T16:57:11.276 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:57:11.276 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: self._run(expiration_interval) 2024-08-17T16:57:11.276 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:57:11.276 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:57:11.276 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:57:11.276 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:57:11.276 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:57:11.276 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: s = self.context.wrap_socket( 2024-08-17T16:57:11.276 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:57:11.277 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: return self.sslsocket_class._create( 2024-08-17T16:57:11.277 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:57:11.277 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: self.do_handshake() 2024-08-17T16:57:11.277 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:57:11.277 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: self._sslobj.do_handshake() 2024-08-17T16:57:11.277 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:57:11.277 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:57:11.907 INFO:tasks.rados.rados.0.smithi063.stdout:578: done (2 left) 2024-08-17T16:57:11.907 INFO:tasks.rados.rados.0.smithi063.stdout:579: done (1 left) 2024-08-17T16:57:11.908 INFO:tasks.rados.rados.0.smithi063.stdout:580: done (0 left) 2024-08-17T16:57:11.908 INFO:tasks.rados.rados.0.smithi063.stdout:581: write oid 2 current snap is 77 2024-08-17T16:57:11.908 INFO:tasks.rados.rados.0.smithi063.stdout:581: seq_num 253 ranges {652171=477646,1821709=430462} 2024-08-17T16:57:11.913 INFO:tasks.rados.rados.0.smithi063.stdout:581: writing smithi06387500-2 from 652171 to 1129817 tid 1 2024-08-17T16:57:11.920 INFO:tasks.rados.rados.0.smithi063.stdout:581: writing smithi06387500-2 from 1821709 to 2252171 tid 2 2024-08-17T16:57:11.920 INFO:tasks.rados.rados.0.smithi063.stdout:582: write oid 17 current snap is 77 2024-08-17T16:57:11.920 INFO:tasks.rados.rados.0.smithi063.stdout:581: finishing write tid 1 to smithi06387500-2 2024-08-17T16:57:11.920 INFO:tasks.rados.rados.0.smithi063.stdout:582: seq_num 254 ranges {668978=552895,1763838=505140} 2024-08-17T16:57:11.927 INFO:tasks.rados.rados.0.smithi063.stdout:582: writing smithi06387500-17 from 668978 to 1221873 tid 1 2024-08-17T16:57:11.933 INFO:tasks.rados.rados.0.smithi063.stdout:582: writing smithi06387500-17 from 1763838 to 2268978 tid 2 2024-08-17T16:57:11.933 INFO:tasks.rados.rados.0.smithi063.stdout:583: write oid 30 current snap is 77 2024-08-17T16:57:11.933 INFO:tasks.rados.rados.0.smithi063.stdout:581: finishing write tid 2 to smithi06387500-2 2024-08-17T16:57:11.933 INFO:tasks.rados.rados.0.smithi063.stdout:583: seq_num 255 ranges {685785=628144,1822320=463465} 2024-08-17T16:57:11.940 INFO:tasks.rados.rados.0.smithi063.stdout:583: writing smithi06387500-30 from 685785 to 1313929 tid 1 2024-08-17T16:57:11.947 INFO:tasks.rados.rados.0.smithi063.stdout:583: writing smithi06387500-30 from 1822320 to 2285785 tid 2 2024-08-17T16:57:11.947 INFO:tasks.rados.rados.0.smithi063.stdout:584: delete oid 23 current snap is 77 2024-08-17T16:57:11.947 INFO:tasks.rados.rados.0.smithi063.stdout:581: finishing write tid 3 to smithi06387500-2 2024-08-17T16:57:11.947 INFO:tasks.rados.rados.0.smithi063.stdout:581: finishing write tid 4 to smithi06387500-2 2024-08-17T16:57:11.947 INFO:tasks.rados.rados.0.smithi063.stdout:581: oid 2 updating version 0 to 231 2024-08-17T16:57:11.948 INFO:tasks.rados.rados.0.smithi063.stdout:581: oid 2 version 231 is already newer than 230 2024-08-17T16:57:11.948 INFO:tasks.rados.rados.0.smithi063.stdout:581: oid 2 updating version 231 to 232 2024-08-17T16:57:11.948 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 232 (ObjNum 253 snap 77 seq_num 253) dirty exists 2024-08-17T16:57:11.948 INFO:tasks.rados.rados.0.smithi063.stdout:581: left oid 2 (ObjNum 253 snap 77 seq_num 253) 2024-08-17T16:57:11.949 INFO:tasks.rados.rados.0.smithi063.stdout:582: finishing write tid 1 to smithi06387500-17 2024-08-17T16:57:11.949 INFO:tasks.rados.rados.0.smithi063.stdout:582: finishing write tid 2 to smithi06387500-17 2024-08-17T16:57:11.949 INFO:tasks.rados.rados.0.smithi063.stdout:582: finishing write tid 3 to smithi06387500-17 2024-08-17T16:57:11.949 INFO:tasks.rados.rados.0.smithi063.stdout:582: finishing write tid 4 to smithi06387500-17 2024-08-17T16:57:11.949 INFO:tasks.rados.rados.0.smithi063.stdout:582: oid 17 updating version 0 to 171 2024-08-17T16:57:11.949 INFO:tasks.rados.rados.0.smithi063.stdout:582: oid 17 updating version 171 to 172 2024-08-17T16:57:11.950 INFO:tasks.rados.rados.0.smithi063.stdout:582: oid 17 updating version 172 to 173 2024-08-17T16:57:11.950 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 173 (ObjNum 254 snap 77 seq_num 254) dirty exists 2024-08-17T16:57:11.950 INFO:tasks.rados.rados.0.smithi063.stdout:582: left oid 17 (ObjNum 254 snap 77 seq_num 254) 2024-08-17T16:57:11.950 INFO:tasks.rados.rados.0.smithi063.stdout:583: finishing write tid 1 to smithi06387500-30 2024-08-17T16:57:11.951 INFO:tasks.rados.rados.0.smithi063.stdout:581: done (3 left) 2024-08-17T16:57:11.951 INFO:tasks.rados.rados.0.smithi063.stdout:582: done (2 left) 2024-08-17T16:57:11.952 INFO:tasks.rados.rados.0.smithi063.stdout:584: done (1 left) 2024-08-17T16:57:11.952 INFO:tasks.rados.rados.0.smithi063.stdout:585: read oid 39 snap 74 2024-08-17T16:57:11.952 INFO:tasks.rados.rados.0.smithi063.stdout:585: expect (ObjNum 190 snap 50 seq_num 190) 2024-08-17T16:57:11.952 INFO:tasks.rados.rados.0.smithi063.stdout:586: snap_remove snap 58 2024-08-17T16:57:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:11 smithi162 ceph-mon[34633]: [17/Aug/2024:16:57:10] ENGINE Bus STARTING 2024-08-17T16:57:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:11 smithi162 ceph-mon[34633]: osdmap e215: 8 total, 8 up, 8 in 2024-08-17T16:57:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:11 smithi162 ceph-mon[34633]: pgmap v6: 89 pgs: 89 active+clean; 274 MiB data, 774 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:11 smithi162 ceph-mon[34633]: mgrmap e30: y(active, since 2s) 2024-08-17T16:57:12.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:11 smithi063 ceph-mon[31063]: [17/Aug/2024:16:57:10] ENGINE Bus STARTING 2024-08-17T16:57:12.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:11 smithi063 ceph-mon[31063]: osdmap e215: 8 total, 8 up, 8 in 2024-08-17T16:57:12.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:11 smithi063 ceph-mon[31063]: pgmap v6: 89 pgs: 89 active+clean; 274 MiB data, 774 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:12.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:11 smithi063 ceph-mon[31063]: mgrmap e30: y(active, since 2s) 2024-08-17T16:57:12.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:11 smithi063 ceph-mon[42680]: [17/Aug/2024:16:57:10] ENGINE Bus STARTING 2024-08-17T16:57:12.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:11 smithi063 ceph-mon[42680]: osdmap e215: 8 total, 8 up, 8 in 2024-08-17T16:57:12.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:11 smithi063 ceph-mon[42680]: pgmap v6: 89 pgs: 89 active+clean; 274 MiB data, 774 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:12.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:11 smithi063 ceph-mon[42680]: mgrmap e30: y(active, since 2s) 2024-08-17T16:57:12.928 INFO:tasks.rados.rados.0.smithi063.stdout:586: done (2 left) 2024-08-17T16:57:12.928 INFO:tasks.rados.rados.0.smithi063.stdout:587: rollback oid 22 current snap is 77 2024-08-17T16:57:12.928 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 22 to 75 2024-08-17T16:57:12.928 INFO:tasks.rados.rados.0.smithi063.stdout:588: rollback oid 28 current snap is 77 2024-08-17T16:57:12.928 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 28 to 71 2024-08-17T16:57:12.929 INFO:tasks.rados.rados.0.smithi063.stdout:589: delete oid 1 current snap is 77 2024-08-17T16:57:12.930 INFO:tasks.rados.rados.0.smithi063.stdout:583: finishing write tid 2 to smithi06387500-30 2024-08-17T16:57:12.931 INFO:tasks.rados.rados.0.smithi063.stdout:583: finishing write tid 3 to smithi06387500-30 2024-08-17T16:57:12.931 INFO:tasks.rados.rados.0.smithi063.stdout:583: finishing write tid 4 to smithi06387500-30 2024-08-17T16:57:12.931 INFO:tasks.rados.rados.0.smithi063.stdout:583: oid 30 updating version 0 to 310 2024-08-17T16:57:12.931 INFO:tasks.rados.rados.0.smithi063.stdout:583: oid 30 version 310 is already newer than 309 2024-08-17T16:57:12.931 INFO:tasks.rados.rados.0.smithi063.stdout:583: oid 30 updating version 310 to 311 2024-08-17T16:57:12.931 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 311 (ObjNum 255 snap 77 seq_num 255) dirty exists 2024-08-17T16:57:12.931 INFO:tasks.rados.rados.0.smithi063.stdout:583: left oid 30 (ObjNum 255 snap 77 seq_num 255) 2024-08-17T16:57:12.931 INFO:tasks.rados.rados.0.smithi063.stdout:585: expect (ObjNum 190 snap 50 seq_num 190) 2024-08-17T16:57:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:587: finishing rollback tid 1 to smithi06387500-22 2024-08-17T16:57:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 159 (ObjNum 49 snap 0 seq_num 1027971309) dirty dne 2024-08-17T16:57:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:588: finishing rollback tid 0 to smithi06387500-28 2024-08-17T16:57:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:583: done (4 left) 2024-08-17T16:57:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:585: done (3 left) 2024-08-17T16:57:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:587: done (2 left) 2024-08-17T16:57:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:589: done (1 left) 2024-08-17T16:57:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:590: read oid 14 snap 38 2024-08-17T16:57:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:590: expect (ObjNum 75 snap 9 seq_num 75) 2024-08-17T16:57:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:591: write oid 25 current snap is 77 2024-08-17T16:57:13.091 INFO:tasks.rados.rados.0.smithi063.stdout:588: finishing rollback tid 1 to smithi06387500-28 2024-08-17T16:57:13.091 INFO:tasks.rados.rados.0.smithi063.stdout:588: finishing rollback tid 2 to smithi06387500-28 2024-08-17T16:57:13.091 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 182 (ObjNum 147 snap 34 seq_num 147) dirty exists 2024-08-17T16:57:13.091 INFO:tasks.rados.rados.0.smithi063.stdout:591: seq_num 258 ranges {736206=453891,1881413=454793} 2024-08-17T16:57:13.096 INFO:tasks.rados.rados.0.smithi063.stdout:591: writing smithi06387500-25 from 736206 to 1190097 tid 1 2024-08-17T16:57:13.102 INFO:tasks.rados.rados.0.smithi063.stdout:591: writing smithi06387500-25 from 1881413 to 2336206 tid 2 2024-08-17T16:57:13.102 INFO:tasks.rados.rados.0.smithi063.stdout:588: done (2 left) 2024-08-17T16:57:13.103 INFO:tasks.rados.rados.0.smithi063.stdout:592: snap_create 2024-08-17T16:57:13.103 INFO:tasks.rados.rados.0.smithi063.stdout:590: expect (ObjNum 75 snap 9 seq_num 75) 2024-08-17T16:57:13.256 INFO:tasks.rados.rados.0.smithi063.stdout:591: finishing write tid 1 to smithi06387500-25 2024-08-17T16:57:13.256 INFO:tasks.rados.rados.0.smithi063.stdout:591: finishing write tid 2 to smithi06387500-25 2024-08-17T16:57:13.256 INFO:tasks.rados.rados.0.smithi063.stdout:591: finishing write tid 3 to smithi06387500-25 2024-08-17T16:57:13.257 INFO:tasks.rados.rados.0.smithi063.stdout:591: finishing write tid 4 to smithi06387500-25 2024-08-17T16:57:13.257 INFO:tasks.rados.rados.0.smithi063.stdout:591: oid 25 updating version 0 to 572 2024-08-17T16:57:13.257 INFO:tasks.rados.rados.0.smithi063.stdout:591: oid 25 version 572 is already newer than 571 2024-08-17T16:57:13.257 INFO:tasks.rados.rados.0.smithi063.stdout:591: oid 25 version 572 is already newer than 570 2024-08-17T16:57:13.257 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 572 (ObjNum 258 snap 77 seq_num 258) dirty exists 2024-08-17T16:57:13.257 INFO:tasks.rados.rados.0.smithi063.stdout:591: left oid 25 (ObjNum 258 snap 77 seq_num 258) 2024-08-17T16:57:13.274 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:12 smithi063 ceph-mon[42680]: [17/Aug/2024:16:57:10] ENGINE Serving on https://172.21.15.63:7150 2024-08-17T16:57:13.275 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:12 smithi063 ceph-mon[42680]: [17/Aug/2024:16:57:10] ENGINE Error in HTTPServer.serve 2024-08-17T16:57:13.275 INFO:journalctl@ceph.mon.c.smithi063.stdout: Traceback (most recent call last): 2024-08-17T16:57:13.275 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:57:13.275 INFO:journalctl@ceph.mon.c.smithi063.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:57:13.275 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:57:13.275 INFO:journalctl@ceph.mon.c.smithi063.stdout: self._run(expiration_interval) 2024-08-17T16:57:13.275 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:57:13.275 INFO:journalctl@ceph.mon.c.smithi063.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:57:13.275 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:57:13.275 INFO:journalctl@ceph.mon.c.smithi063.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:57:13.275 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:57:13.276 INFO:journalctl@ceph.mon.c.smithi063.stdout: s = self.context.wrap_socket( 2024-08-17T16:57:13.276 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:57:13.276 INFO:journalctl@ceph.mon.c.smithi063.stdout: return self.sslsocket_class._create( 2024-08-17T16:57:13.276 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:57:13.276 INFO:journalctl@ceph.mon.c.smithi063.stdout: self.do_handshake() 2024-08-17T16:57:13.276 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:57:13.276 INFO:journalctl@ceph.mon.c.smithi063.stdout: self._sslobj.do_handshake() 2024-08-17T16:57:13.276 INFO:journalctl@ceph.mon.c.smithi063.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:57:13.276 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:12 smithi063 ceph-mon[42680]: [17/Aug/2024:16:57:11] ENGINE Serving on http://172.21.15.63:8765 2024-08-17T16:57:13.276 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:12 smithi063 ceph-mon[42680]: [17/Aug/2024:16:57:11] ENGINE Bus STARTED 2024-08-17T16:57:13.276 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:12 smithi063 ceph-mon[42680]: osdmap e216: 8 total, 8 up, 8 in 2024-08-17T16:57:13.276 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:12 smithi063 ceph-mon[31063]: [17/Aug/2024:16:57:10] ENGINE Serving on https://172.21.15.63:7150 2024-08-17T16:57:13.277 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:12 smithi063 ceph-mon[31063]: [17/Aug/2024:16:57:10] ENGINE Error in HTTPServer.serve 2024-08-17T16:57:13.277 INFO:journalctl@ceph.mon.a.smithi063.stdout: Traceback (most recent call last): 2024-08-17T16:57:13.277 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:57:13.277 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:57:13.277 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:57:13.277 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._run(expiration_interval) 2024-08-17T16:57:13.277 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:57:13.277 INFO:journalctl@ceph.mon.a.smithi063.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:57:13.277 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:57:13.277 INFO:journalctl@ceph.mon.a.smithi063.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:57:13.277 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:57:13.277 INFO:journalctl@ceph.mon.a.smithi063.stdout: s = self.context.wrap_socket( 2024-08-17T16:57:13.278 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:57:13.278 INFO:journalctl@ceph.mon.a.smithi063.stdout: return self.sslsocket_class._create( 2024-08-17T16:57:13.278 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:57:13.278 INFO:journalctl@ceph.mon.a.smithi063.stdout: self.do_handshake() 2024-08-17T16:57:13.278 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:57:13.278 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._sslobj.do_handshake() 2024-08-17T16:57:13.278 INFO:journalctl@ceph.mon.a.smithi063.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:57:13.278 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:12 smithi063 ceph-mon[31063]: [17/Aug/2024:16:57:11] ENGINE Serving on http://172.21.15.63:8765 2024-08-17T16:57:13.278 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:12 smithi063 ceph-mon[31063]: [17/Aug/2024:16:57:11] ENGINE Bus STARTED 2024-08-17T16:57:13.278 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:12 smithi063 ceph-mon[31063]: osdmap e216: 8 total, 8 up, 8 in 2024-08-17T16:57:13.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:12 smithi162 ceph-mon[34633]: [17/Aug/2024:16:57:10] ENGINE Serving on https://172.21.15.63:7150 2024-08-17T16:57:13.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:12 smithi162 ceph-mon[34633]: [17/Aug/2024:16:57:10] ENGINE Error in HTTPServer.serve 2024-08-17T16:57:13.346 INFO:journalctl@ceph.mon.b.smithi162.stdout: Traceback (most recent call last): 2024-08-17T16:57:13.346 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:57:13.346 INFO:journalctl@ceph.mon.b.smithi162.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:57:13.346 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:57:13.346 INFO:journalctl@ceph.mon.b.smithi162.stdout: self._run(expiration_interval) 2024-08-17T16:57:13.346 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:57:13.346 INFO:journalctl@ceph.mon.b.smithi162.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:57:13.346 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:57:13.346 INFO:journalctl@ceph.mon.b.smithi162.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:57:13.347 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:57:13.347 INFO:journalctl@ceph.mon.b.smithi162.stdout: s = self.context.wrap_socket( 2024-08-17T16:57:13.347 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:57:13.347 INFO:journalctl@ceph.mon.b.smithi162.stdout: return self.sslsocket_class._create( 2024-08-17T16:57:13.347 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:57:13.347 INFO:journalctl@ceph.mon.b.smithi162.stdout: self.do_handshake() 2024-08-17T16:57:13.347 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:57:13.347 INFO:journalctl@ceph.mon.b.smithi162.stdout: self._sslobj.do_handshake() 2024-08-17T16:57:13.347 INFO:journalctl@ceph.mon.b.smithi162.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:57:13.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:12 smithi162 ceph-mon[34633]: [17/Aug/2024:16:57:11] ENGINE Serving on http://172.21.15.63:8765 2024-08-17T16:57:13.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:12 smithi162 ceph-mon[34633]: [17/Aug/2024:16:57:11] ENGINE Bus STARTED 2024-08-17T16:57:13.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:12 smithi162 ceph-mon[34633]: osdmap e216: 8 total, 8 up, 8 in 2024-08-17T16:57:13.919 INFO:tasks.rados.rados.0.smithi063.stdout:590: done (2 left) 2024-08-17T16:57:13.919 INFO:tasks.rados.rados.0.smithi063.stdout:591: done (1 left) 2024-08-17T16:57:13.920 INFO:tasks.rados.rados.0.smithi063.stdout:592: done (0 left) 2024-08-17T16:57:13.920 INFO:tasks.rados.rados.0.smithi063.stdout:593: snap_create 2024-08-17T16:57:14.174 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:13 smithi162 ceph-mon[34633]: pgmap v8: 89 pgs: 89 active+clean; 274 MiB data, 774 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:14.174 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:13 smithi162 ceph-mon[34633]: osdmap e217: 8 total, 8 up, 8 in 2024-08-17T16:57:14.174 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:13 smithi162 ceph-mon[34633]: mgrmap e31: y(active, since 4s) 2024-08-17T16:57:14.174 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:13 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:14.174 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:13 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:14.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:13 smithi063 ceph-mon[42680]: pgmap v8: 89 pgs: 89 active+clean; 274 MiB data, 774 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:14.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:13 smithi063 ceph-mon[42680]: osdmap e217: 8 total, 8 up, 8 in 2024-08-17T16:57:14.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:13 smithi063 ceph-mon[42680]: mgrmap e31: y(active, since 4s) 2024-08-17T16:57:14.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:13 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:14.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:13 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:14.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:13 smithi063 ceph-mon[31063]: pgmap v8: 89 pgs: 89 active+clean; 274 MiB data, 774 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:14.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:13 smithi063 ceph-mon[31063]: osdmap e217: 8 total, 8 up, 8 in 2024-08-17T16:57:14.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:13 smithi063 ceph-mon[31063]: mgrmap e31: y(active, since 4s) 2024-08-17T16:57:14.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:13 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:14.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:13 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:14.595 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: [17/Aug/2024:16:57:14] ENGINE Bus STARTING 2024-08-17T16:57:14.596 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: CherryPy Checker: 2024-08-17T16:57:14.596 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: The Application mounted at '' has an empty config. 2024-08-17T16:57:14.596 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: 2024-08-17T16:57:14.596 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: [17/Aug/2024:16:57:14] ENGINE Serving on http://:::9283 2024-08-17T16:57:14.596 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[69873]: [17/Aug/2024:16:57:14] ENGINE Bus STARTED 2024-08-17T16:57:14.649 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T16:57:14.858 INFO:tasks.rados.rados.0.smithi063.stdout:593: done (0 left) 2024-08-17T16:57:14.858 INFO:tasks.rados.rados.0.smithi063.stdout:594: read oid 13 snap -1 2024-08-17T16:57:14.858 INFO:tasks.rados.rados.0.smithi063.stdout:594: expect (ObjNum 219 snap 61 seq_num 219) 2024-08-17T16:57:14.858 INFO:tasks.rados.rados.0.smithi063.stdout:595: snap_remove snap 69 2024-08-17T16:57:15.154 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[31063]: osdmap e218: 8 total, 8 up, 8 in 2024-08-17T16:57:15.154 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[31063]: Standby manager daemon x started 2024-08-17T16:57:15.154 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/1851207511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:57:15.154 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/1851207511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:57:15.154 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/1851207511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:57:15.154 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/1851207511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:57:15.155 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:15.155 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:15.155 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:15.155 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:15.155 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[31063]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:15.155 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[31063]: osdmap e219: 8 total, 8 up, 8 in 2024-08-17T16:57:15.155 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[31063]: pgmap v12: 89 pgs: 89 active+clean; 274 MiB data, 774 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:15.155 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[42680]: osdmap e218: 8 total, 8 up, 8 in 2024-08-17T16:57:15.155 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[42680]: Standby manager daemon x started 2024-08-17T16:57:15.155 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/1851207511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:57:15.155 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/1851207511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:57:15.156 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/1851207511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:57:15.156 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/1851207511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:57:15.156 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:15.156 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:15.156 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:15.156 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:15.156 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[42680]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:15.156 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[42680]: osdmap e219: 8 total, 8 up, 8 in 2024-08-17T16:57:15.156 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:14 smithi063 ceph-mon[42680]: pgmap v12: 89 pgs: 89 active+clean; 274 MiB data, 774 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:15.269 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T16:57:15.269 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (3m) 1s ago 4m 26.9M - 0.25.0 c8568f914cd2 87286b24f231 2024-08-17T16:57:15.269 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (4m) 0s ago 4m 88.1M - 9.4.7 954c08fa6188 fc8d62329012 2024-08-17T16:57:15.269 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (3m) 1s ago 3m 17.0M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T16:57:15.269 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (3m) 0s ago 3m 16.7M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T16:57:15.269 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,9283,8765 running (53s) 0s ago 8m 143M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 180435c78c80 2024-08-17T16:57:15.269 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:8443,9283,8765 running (14s) 1s ago 10m 489M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 943e639cede6 2024-08-17T16:57:15.269 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (10m) 1s ago 10m 53.4M 2048M 18.2.4-719-gced649f8 c870753d67c2 0c71ddd7ada6 2024-08-17T16:57:15.269 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (8m) 0s ago 8m 47.6M 2048M 18.2.4-719-gced649f8 c870753d67c2 d4ecbe65a0c8 2024-08-17T16:57:15.269 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (8m) 1s ago 8m 43.9M 2048M 18.2.4-719-gced649f8 c870753d67c2 b3f2c9339c6b 2024-08-17T16:57:15.269 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 running (4m) 1s ago 4m 16.2M - 1.5.0 0da6a335fe13 8958df80ad24 2024-08-17T16:57:15.270 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (4m) 0s ago 4m 15.6M - 1.5.0 0da6a335fe13 b1691104d8ac 2024-08-17T16:57:15.270 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (8m) 1s ago 8m 82.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 f3b910ed49b8 2024-08-17T16:57:15.270 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (7m) 1s ago 7m 62.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 5d6fdb6000ad 2024-08-17T16:57:15.270 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (7m) 1s ago 7m 75.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 a3cbe8edb49a 2024-08-17T16:57:15.270 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (7m) 1s ago 7m 69.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T16:57:15.270 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (6m) 0s ago 6m 85.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T16:57:15.270 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (6m) 0s ago 6m 79.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T16:57:15.270 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (5m) 0s ago 5m 60.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T16:57:15.270 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (5m) 0s ago 5m 69.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T16:57:15.270 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (19s) 0s ago 5m 36.1M - 2.43.0 a07b618ecd1d 039ece187f07 2024-08-17T16:57:15.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-mon[34633]: osdmap e218: 8 total, 8 up, 8 in 2024-08-17T16:57:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-mon[34633]: Standby manager daemon x started 2024-08-17T16:57:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/1851207511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:57:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/1851207511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:57:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/1851207511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:57:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/1851207511' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:57:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-mon[34633]: from='client.15006 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:15.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-mon[34633]: osdmap e219: 8 total, 8 up, 8 in 2024-08-17T16:57:15.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:14 smithi162 ceph-mon[34633]: pgmap v12: 89 pgs: 89 active+clean; 274 MiB data, 774 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:15.625 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:57:15.625 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T16:57:15.625 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T16:57:15.625 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:57:15.625 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T16:57:15.625 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T16:57:15.625 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:57:15.626 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T16:57:15.626 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T16:57:15.626 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:57:15.626 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T16:57:15.626 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T16:57:15.626 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:57:15.626 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T16:57:15.626 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 13, 2024-08-17T16:57:15.626 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T16:57:15.626 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T16:57:15.626 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:57:15.863 INFO:tasks.rados.rados.0.smithi063.stdout:595: done (1 left) 2024-08-17T16:57:15.864 INFO:tasks.rados.rados.0.smithi063.stdout:596: write oid 12 current snap is 79 2024-08-17T16:57:15.864 INFO:tasks.rados.rados.0.smithi063.stdout:596: seq_num 259 ranges {753013=645493,2056248=296765} 2024-08-17T16:57:15.870 INFO:tasks.rados.rados.0.smithi063.stdout:596: writing smithi06387500-12 from 753013 to 1398506 tid 1 2024-08-17T16:57:15.876 INFO:tasks.rados.rados.0.smithi063.stdout:596: writing smithi06387500-12 from 2056248 to 2353013 tid 2 2024-08-17T16:57:15.876 INFO:tasks.rados.rados.0.smithi063.stdout:597: read oid 41 snap 72 2024-08-17T16:57:15.876 INFO:tasks.rados.rados.0.smithi063.stdout:597: expect (ObjNum 73 snap 9 seq_num 73) 2024-08-17T16:57:15.876 INFO:tasks.rados.rados.0.smithi063.stdout:598: write oid 34 current snap is 79 2024-08-17T16:57:15.877 INFO:tasks.rados.rados.0.smithi063.stdout:598: seq_num 260 ranges {769820=720742,2114730=255090} 2024-08-17T16:57:15.884 INFO:tasks.rados.rados.0.smithi063.stdout:598: writing smithi06387500-34 from 769820 to 1490562 tid 1 2024-08-17T16:57:15.889 INFO:tasks.rados.rados.0.smithi063.stdout:598: writing smithi06387500-34 from 2114730 to 2369820 tid 2 2024-08-17T16:57:15.889 INFO:tasks.rados.rados.0.smithi063.stdout:599: delete oid 8 current snap is 79 2024-08-17T16:57:15.890 INFO:tasks.rados.rados.0.smithi063.stdout:594: expect (ObjNum 219 snap 61 seq_num 219) 2024-08-17T16:57:15.934 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:57:15.935 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T16:57:15.935 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-08-17T16:57:15.935 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-17T16:57:15.935 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-08-17T16:57:15.935 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-08-17T16:57:15.935 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-08-17T16:57:15.935 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "2/2 daemons upgraded", 2024-08-17T16:57:15.935 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "", 2024-08-17T16:57:15.935 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T16:57:15.935 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:57:16.068 INFO:tasks.rados.rados.0.smithi063.stdout:596: finishing write tid 1 to smithi06387500-12 2024-08-17T16:57:16.068 INFO:tasks.rados.rados.0.smithi063.stdout:594: done (4 left) 2024-08-17T16:57:16.068 INFO:tasks.rados.rados.0.smithi063.stdout:599: done (3 left) 2024-08-17T16:57:16.069 INFO:tasks.rados.rados.0.smithi063.stdout:600: snap_remove snap 76 2024-08-17T16:57:16.207 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[42680]: mgrmap e32: y(active, since 6s), standbys: x 2024-08-17T16:57:16.207 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:57:16.207 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[42680]: from='client.15012 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:16.207 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:16.207 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:16.207 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[42680]: from='client.24911 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:16.207 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/257208995' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:16.208 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[42680]: osdmap e220: 8 total, 8 up, 8 in 2024-08-17T16:57:16.208 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[31063]: mgrmap e32: y(active, since 6s), standbys: x 2024-08-17T16:57:16.208 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:57:16.208 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[31063]: from='client.15012 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:16.208 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:16.208 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:16.208 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[31063]: from='client.24911 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:16.208 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/257208995' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:16.208 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:15 smithi063 ceph-mon[31063]: osdmap e220: 8 total, 8 up, 8 in 2024-08-17T16:57:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:15 smithi162 ceph-mon[34633]: mgrmap e32: y(active, since 6s), standbys: x 2024-08-17T16:57:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:15 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:57:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:15 smithi162 ceph-mon[34633]: from='client.15012 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:15 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:15 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:15 smithi162 ceph-mon[34633]: from='client.24911 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:15 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/257208995' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:15 smithi162 ceph-mon[34633]: osdmap e220: 8 total, 8 up, 8 in 2024-08-17T16:57:16.941 INFO:tasks.rados.rados.0.smithi063.stdout:600: done (3 left) 2024-08-17T16:57:16.941 INFO:tasks.rados.rados.0.smithi063.stdout:601: snap_create 2024-08-17T16:57:16.941 INFO:tasks.rados.rados.0.smithi063.stdout:596: finishing write tid 2 to smithi06387500-12 2024-08-17T16:57:16.941 INFO:tasks.rados.rados.0.smithi063.stdout:596: finishing write tid 3 to smithi06387500-12 2024-08-17T16:57:16.942 INFO:tasks.rados.rados.0.smithi063.stdout:596: finishing write tid 4 to smithi06387500-12 2024-08-17T16:57:16.942 INFO:tasks.rados.rados.0.smithi063.stdout:596: oid 12 updating version 0 to 126 2024-08-17T16:57:16.942 INFO:tasks.rados.rados.0.smithi063.stdout:596: oid 12 version 126 is already newer than 125 2024-08-17T16:57:16.942 INFO:tasks.rados.rados.0.smithi063.stdout:596: oid 12 updating version 126 to 127 2024-08-17T16:57:16.942 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 127 (ObjNum 259 snap 79 seq_num 259) dirty exists 2024-08-17T16:57:16.942 INFO:tasks.rados.rados.0.smithi063.stdout:596: left oid 12 (ObjNum 259 snap 79 seq_num 259) 2024-08-17T16:57:16.942 INFO:tasks.rados.rados.0.smithi063.stdout:597: expect (ObjNum 73 snap 9 seq_num 73) 2024-08-17T16:57:17.091 INFO:tasks.rados.rados.0.smithi063.stdout:598: finishing write tid 1 to smithi06387500-34 2024-08-17T16:57:17.091 INFO:tasks.rados.rados.0.smithi063.stdout:598: finishing write tid 2 to smithi06387500-34 2024-08-17T16:57:17.092 INFO:tasks.rados.rados.0.smithi063.stdout:598: finishing write tid 3 to smithi06387500-34 2024-08-17T16:57:17.092 INFO:tasks.rados.rados.0.smithi063.stdout:598: finishing write tid 4 to smithi06387500-34 2024-08-17T16:57:17.092 INFO:tasks.rados.rados.0.smithi063.stdout:598: oid 34 updating version 0 to 388 2024-08-17T16:57:17.092 INFO:tasks.rados.rados.0.smithi063.stdout:598: oid 34 updating version 388 to 390 2024-08-17T16:57:17.092 INFO:tasks.rados.rados.0.smithi063.stdout:598: oid 34 version 390 is already newer than 389 2024-08-17T16:57:17.092 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 390 (ObjNum 260 snap 79 seq_num 260) dirty exists 2024-08-17T16:57:17.092 INFO:tasks.rados.rados.0.smithi063.stdout:598: left oid 34 (ObjNum 260 snap 79 seq_num 260) 2024-08-17T16:57:17.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:16 smithi063 ceph-mon[42680]: from='client.15030 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:17.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:16 smithi063 ceph-mon[42680]: pgmap v14: 89 pgs: 89 active+clean; 277 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.2 MiB/s wr, 33 op/s 2024-08-17T16:57:17.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:16 smithi063 ceph-mon[31063]: from='client.15030 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:17.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:16 smithi063 ceph-mon[31063]: pgmap v14: 89 pgs: 89 active+clean; 277 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.2 MiB/s wr, 33 op/s 2024-08-17T16:57:17.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:16 smithi162 ceph-mon[34633]: from='client.15030 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:17.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:16 smithi162 ceph-mon[34633]: pgmap v14: 89 pgs: 89 active+clean; 277 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.2 MiB/s wr, 33 op/s 2024-08-17T16:57:17.939 INFO:tasks.rados.rados.0.smithi063.stdout:596: done (3 left) 2024-08-17T16:57:17.939 INFO:tasks.rados.rados.0.smithi063.stdout:597: done (2 left) 2024-08-17T16:57:17.940 INFO:tasks.rados.rados.0.smithi063.stdout:598: done (1 left) 2024-08-17T16:57:17.940 INFO:tasks.rados.rados.0.smithi063.stdout:601: done (0 left) 2024-08-17T16:57:17.940 INFO:tasks.rados.rados.0.smithi063.stdout:602: write oid 46 current snap is 80 2024-08-17T16:57:17.940 INFO:tasks.rados.rados.0.smithi063.stdout:602: seq_num 262 ranges {403434=471240,1315341=400756} 2024-08-17T16:57:17.944 INFO:tasks.rados.rados.0.smithi063.stdout:602: writing smithi06387500-46 from 403434 to 874674 tid 1 2024-08-17T16:57:17.949 INFO:tasks.rados.rados.0.smithi063.stdout:602: writing smithi06387500-46 from 1315341 to 1716097 tid 2 2024-08-17T16:57:17.949 INFO:tasks.rados.rados.0.smithi063.stdout:603: read oid 50 snap -1 2024-08-17T16:57:17.949 INFO:tasks.rados.rados.0.smithi063.stdout:603: expect (ObjNum 226 snap 64 seq_num 226) 2024-08-17T16:57:17.950 INFO:tasks.rados.rados.0.smithi063.stdout:604: delete oid 4 current snap is 80 2024-08-17T16:57:17.953 INFO:tasks.rados.rados.0.smithi063.stdout:604: done (2 left) 2024-08-17T16:57:17.953 INFO:tasks.rados.rados.0.smithi063.stdout:605: write oid 27 current snap is 80 2024-08-17T16:57:17.953 INFO:tasks.rados.rados.0.smithi063.stdout:605: seq_num 264 ranges {437048=621738,1832305=404425} 2024-08-17T16:57:17.959 INFO:tasks.rados.rados.0.smithi063.stdout:605: writing smithi06387500-27 from 437048 to 1058786 tid 1 2024-08-17T16:57:17.966 INFO:tasks.rados.rados.0.smithi063.stdout:605: writing smithi06387500-27 from 1832305 to 2236730 tid 2 2024-08-17T16:57:17.967 INFO:tasks.rados.rados.0.smithi063.stdout:606: write oid 18 current snap is 80 2024-08-17T16:57:17.967 INFO:tasks.rados.rados.0.smithi063.stdout:602: finishing write tid 1 to smithi06387500-46 2024-08-17T16:57:17.967 INFO:tasks.rados.rados.0.smithi063.stdout:602: finishing write tid 2 to smithi06387500-46 2024-08-17T16:57:17.967 INFO:tasks.rados.rados.0.smithi063.stdout:602: finishing write tid 3 to smithi06387500-46 2024-08-17T16:57:17.967 INFO:tasks.rados.rados.0.smithi063.stdout:606: seq_num 265 ranges {453855=696987,1890787=548083} 2024-08-17T16:57:17.973 INFO:tasks.rados.rados.0.smithi063.stdout:606: writing smithi06387500-18 from 453855 to 1150842 tid 1 2024-08-17T16:57:17.980 INFO:tasks.rados.rados.0.smithi063.stdout:606: writing smithi06387500-18 from 1890787 to 2438870 tid 2 2024-08-17T16:57:17.980 INFO:tasks.rados.rados.0.smithi063.stdout:607: write oid 44 current snap is 80 2024-08-17T16:57:17.981 INFO:tasks.rados.rados.0.smithi063.stdout:602: finishing write tid 4 to smithi06387500-46 2024-08-17T16:57:17.981 INFO:tasks.rados.rados.0.smithi063.stdout:602: oid 46 updating version 0 to 214 2024-08-17T16:57:17.981 INFO:tasks.rados.rados.0.smithi063.stdout:602: oid 46 updating version 214 to 215 2024-08-17T16:57:17.981 INFO:tasks.rados.rados.0.smithi063.stdout:602: oid 46 version 215 is already newer than 213 2024-08-17T16:57:17.981 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 215 (ObjNum 262 snap 80 seq_num 262) dirty exists 2024-08-17T16:57:17.981 INFO:tasks.rados.rados.0.smithi063.stdout:602: left oid 46 (ObjNum 262 snap 80 seq_num 262) 2024-08-17T16:57:17.982 INFO:tasks.rados.rados.0.smithi063.stdout:603: expect (ObjNum 226 snap 64 seq_num 226) 2024-08-17T16:57:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:17 smithi063 ceph-mon[31063]: osdmap e221: 8 total, 8 up, 8 in 2024-08-17T16:57:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:17 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:17 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:17 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:17 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:17 smithi063 ceph-mon[31063]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:57:18.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:17 smithi063 ceph-mon[31063]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:57:18.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:17 smithi063 ceph-mon[42680]: osdmap e221: 8 total, 8 up, 8 in 2024-08-17T16:57:18.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:17 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:18.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:17 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:18.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:17 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:18.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:17 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:18.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:17 smithi063 ceph-mon[42680]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:57:18.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:17 smithi063 ceph-mon[42680]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:57:18.275 INFO:tasks.rados.rados.0.smithi063.stdout:605: finishing write tid 1 to smithi06387500-27 2024-08-17T16:57:18.276 INFO:tasks.rados.rados.0.smithi063.stdout:605: finishing write tid 2 to smithi06387500-27 2024-08-17T16:57:18.276 INFO:tasks.rados.rados.0.smithi063.stdout:605: finishing write tid 3 to smithi06387500-27 2024-08-17T16:57:18.276 INFO:tasks.rados.rados.0.smithi063.stdout:605: finishing write tid 4 to smithi06387500-27 2024-08-17T16:57:18.276 INFO:tasks.rados.rados.0.smithi063.stdout:605: oid 27 updating version 0 to 79 2024-08-17T16:57:18.276 INFO:tasks.rados.rados.0.smithi063.stdout:605: oid 27 version 79 is already newer than 78 2024-08-17T16:57:18.276 INFO:tasks.rados.rados.0.smithi063.stdout:605: oid 27 version 79 is already newer than 77 2024-08-17T16:57:18.276 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 79 (ObjNum 264 snap 80 seq_num 264) dirty exists 2024-08-17T16:57:18.276 INFO:tasks.rados.rados.0.smithi063.stdout:605: left oid 27 (ObjNum 264 snap 80 seq_num 264) 2024-08-17T16:57:18.276 INFO:tasks.rados.rados.0.smithi063.stdout:606: finishing write tid 1 to smithi06387500-18 2024-08-17T16:57:18.277 INFO:tasks.rados.rados.0.smithi063.stdout:607: seq_num 266 ranges {470662=772236,1832916=408094} 2024-08-17T16:57:18.282 INFO:tasks.rados.rados.0.smithi063.stdout:607: writing smithi06387500-44 from 470662 to 1242898 tid 1 2024-08-17T16:57:18.288 INFO:tasks.rados.rados.0.smithi063.stdout:607: writing smithi06387500-44 from 1832916 to 2241010 tid 2 2024-08-17T16:57:18.288 INFO:tasks.rados.rados.0.smithi063.stdout:602: done (4 left) 2024-08-17T16:57:18.288 INFO:tasks.rados.rados.0.smithi063.stdout:603: done (3 left) 2024-08-17T16:57:18.288 INFO:tasks.rados.rados.0.smithi063.stdout:605: done (2 left) 2024-08-17T16:57:18.288 INFO:tasks.rados.rados.0.smithi063.stdout:608: snap_create 2024-08-17T16:57:18.288 INFO:tasks.rados.rados.0.smithi063.stdout:606: finishing write tid 2 to smithi06387500-18 2024-08-17T16:57:18.288 INFO:tasks.rados.rados.0.smithi063.stdout:606: finishing write tid 3 to smithi06387500-18 2024-08-17T16:57:18.289 INFO:tasks.rados.rados.0.smithi063.stdout:606: finishing write tid 4 to smithi06387500-18 2024-08-17T16:57:18.289 INFO:tasks.rados.rados.0.smithi063.stdout:606: oid 18 updating version 0 to 188 2024-08-17T16:57:18.289 INFO:tasks.rados.rados.0.smithi063.stdout:606: oid 18 updating version 188 to 190 2024-08-17T16:57:18.289 INFO:tasks.rados.rados.0.smithi063.stdout:606: oid 18 version 190 is already newer than 189 2024-08-17T16:57:18.289 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 190 (ObjNum 265 snap 80 seq_num 265) dirty exists 2024-08-17T16:57:18.289 INFO:tasks.rados.rados.0.smithi063.stdout:606: left oid 18 (ObjNum 265 snap 80 seq_num 265) 2024-08-17T16:57:18.289 INFO:tasks.rados.rados.0.smithi063.stdout:607: finishing write tid 1 to smithi06387500-44 2024-08-17T16:57:18.290 INFO:tasks.rados.rados.0.smithi063.stdout:607: finishing write tid 2 to smithi06387500-44 2024-08-17T16:57:18.290 INFO:tasks.rados.rados.0.smithi063.stdout:607: finishing write tid 3 to smithi06387500-44 2024-08-17T16:57:18.290 INFO:tasks.rados.rados.0.smithi063.stdout:607: finishing write tid 4 to smithi06387500-44 2024-08-17T16:57:18.290 INFO:tasks.rados.rados.0.smithi063.stdout:607: oid 44 updating version 0 to 117 2024-08-17T16:57:18.290 INFO:tasks.rados.rados.0.smithi063.stdout:607: oid 44 version 117 is already newer than 116 2024-08-17T16:57:18.290 INFO:tasks.rados.rados.0.smithi063.stdout:607: oid 44 version 117 is already newer than 115 2024-08-17T16:57:18.291 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 117 (ObjNum 266 snap 80 seq_num 266) dirty exists 2024-08-17T16:57:18.291 INFO:tasks.rados.rados.0.smithi063.stdout:607: left oid 44 (ObjNum 266 snap 80 seq_num 266) 2024-08-17T16:57:18.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:17 smithi162 ceph-mon[34633]: osdmap e221: 8 total, 8 up, 8 in 2024-08-17T16:57:18.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:17 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:18.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:17 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:18.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:17 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:18.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:17 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:18.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:17 smithi162 ceph-mon[34633]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:57:18.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:17 smithi162 ceph-mon[34633]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:57:18.949 INFO:tasks.rados.rados.0.smithi063.stdout:606: done (2 left) 2024-08-17T16:57:18.949 INFO:tasks.rados.rados.0.smithi063.stdout:607: done (1 left) 2024-08-17T16:57:18.949 INFO:tasks.rados.rados.0.smithi063.stdout:608: done (0 left) 2024-08-17T16:57:18.950 INFO:tasks.rados.rados.0.smithi063.stdout:609: write oid 25 current snap is 81 2024-08-17T16:57:18.950 INFO:tasks.rados.rados.0.smithi063.stdout:609: seq_num 267 ranges {487469=563838,1607751=551752} 2024-08-17T16:57:18.955 INFO:tasks.rados.rados.0.smithi063.stdout:609: writing smithi06387500-25 from 487469 to 1051307 tid 1 2024-08-17T16:57:18.962 INFO:tasks.rados.rados.0.smithi063.stdout:609: writing smithi06387500-25 from 1607751 to 2159503 tid 2 2024-08-17T16:57:18.962 INFO:tasks.rados.rados.0.smithi063.stdout:610: snap_create 2024-08-17T16:57:18.962 INFO:tasks.rados.rados.0.smithi063.stdout:609: finishing write tid 1 to smithi06387500-25 2024-08-17T16:57:18.965 INFO:tasks.rados.rados.0.smithi063.stdout:609: finishing write tid 2 to smithi06387500-25 2024-08-17T16:57:18.965 INFO:tasks.rados.rados.0.smithi063.stdout:609: finishing write tid 3 to smithi06387500-25 2024-08-17T16:57:18.965 INFO:tasks.rados.rados.0.smithi063.stdout:609: finishing write tid 4 to smithi06387500-25 2024-08-17T16:57:18.965 INFO:tasks.rados.rados.0.smithi063.stdout:609: oid 25 updating version 0 to 586 2024-08-17T16:57:18.965 INFO:tasks.rados.rados.0.smithi063.stdout:609: oid 25 updating version 586 to 587 2024-08-17T16:57:18.965 INFO:tasks.rados.rados.0.smithi063.stdout:609: oid 25 updating version 587 to 588 2024-08-17T16:57:18.965 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 588 (ObjNum 267 snap 81 seq_num 267) dirty exists 2024-08-17T16:57:18.965 INFO:tasks.rados.rados.0.smithi063.stdout:609: left oid 25 (ObjNum 267 snap 81 seq_num 267) 2024-08-17T16:57:19.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[31063]: osdmap e222: 8 total, 8 up, 8 in 2024-08-17T16:57:19.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[31063]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:57:19.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[31063]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:57:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[31063]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:57:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[31063]: Updating smithi162:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:57:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[31063]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:57:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[31063]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:57:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[31063]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T16:57:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[31063]: pgmap v17: 89 pgs: 89 active+clean; 277 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.1 MiB/s wr, 33 op/s 2024-08-17T16:57:19.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[42680]: osdmap e222: 8 total, 8 up, 8 in 2024-08-17T16:57:19.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[42680]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:57:19.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[42680]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:57:19.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[42680]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:57:19.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[42680]: Updating smithi162:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:57:19.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[42680]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:57:19.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[42680]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:57:19.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:19.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:19.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:19.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:19.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:19.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[42680]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T16:57:19.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:18 smithi063 ceph-mon[42680]: pgmap v17: 89 pgs: 89 active+clean; 277 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.1 MiB/s wr, 33 op/s 2024-08-17T16:57:19.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:18 smithi162 ceph-mon[34633]: osdmap e222: 8 total, 8 up, 8 in 2024-08-17T16:57:19.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:18 smithi162 ceph-mon[34633]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:57:19.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:18 smithi162 ceph-mon[34633]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:57:19.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:18 smithi162 ceph-mon[34633]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:57:19.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:18 smithi162 ceph-mon[34633]: Updating smithi162:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:57:19.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:18 smithi162 ceph-mon[34633]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:57:19.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:18 smithi162 ceph-mon[34633]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:57:19.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:18 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:19.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:18 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:19.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:18 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:19.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:18 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:19.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:18 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:19.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:18 smithi162 ceph-mon[34633]: Reconfiguring prometheus.a (dependencies changed)... 2024-08-17T16:57:19.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:18 smithi162 ceph-mon[34633]: pgmap v17: 89 pgs: 89 active+clean; 277 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.1 MiB/s wr, 33 op/s 2024-08-17T16:57:19.852 INFO:tasks.rados.rados.0.smithi063.stdout:609: done (1 left) 2024-08-17T16:57:19.852 INFO:tasks.rados.rados.0.smithi063.stdout:610: done (0 left) 2024-08-17T16:57:19.852 INFO:tasks.rados.rados.0.smithi063.stdout:611: delete oid 25 current snap is 82 2024-08-17T16:57:19.855 INFO:tasks.rados.rados.0.smithi063.stdout:611: done (0 left) 2024-08-17T16:57:19.855 INFO:tasks.rados.rados.0.smithi063.stdout:612: snap_remove snap 79 2024-08-17T16:57:20.255 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:19 smithi162 ceph-mon[34633]: osdmap e223: 8 total, 8 up, 8 in 2024-08-17T16:57:20.255 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:19 smithi162 ceph-mon[34633]: Reconfiguring daemon prometheus.a on smithi162 2024-08-17T16:57:20.255 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:19 smithi162 ceph-mon[34633]: osdmap e224: 8 total, 8 up, 8 in 2024-08-17T16:57:20.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:19 smithi063 ceph-mon[31063]: osdmap e223: 8 total, 8 up, 8 in 2024-08-17T16:57:20.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:19 smithi063 ceph-mon[31063]: Reconfiguring daemon prometheus.a on smithi162 2024-08-17T16:57:20.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:19 smithi063 ceph-mon[31063]: osdmap e224: 8 total, 8 up, 8 in 2024-08-17T16:57:20.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:19 smithi063 ceph-mon[42680]: osdmap e223: 8 total, 8 up, 8 in 2024-08-17T16:57:20.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:19 smithi063 ceph-mon[42680]: Reconfiguring daemon prometheus.a on smithi162 2024-08-17T16:57:20.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:19 smithi063 ceph-mon[42680]: osdmap e224: 8 total, 8 up, 8 in 2024-08-17T16:57:20.855 INFO:tasks.rados.rados.0.smithi063.stdout:612: done (0 left) 2024-08-17T16:57:20.856 INFO:tasks.rados.rados.0.smithi063.stdout:613: snap_create 2024-08-17T16:57:21.855 INFO:tasks.rados.rados.0.smithi063.stdout:613: done (0 left) 2024-08-17T16:57:21.855 INFO:tasks.rados.rados.0.smithi063.stdout:614: snap_create 2024-08-17T16:57:21.955 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:21 smithi162 ceph-mon[34633]: osdmap e225: 8 total, 8 up, 8 in 2024-08-17T16:57:21.955 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:21 smithi162 ceph-mon[34633]: pgmap v21: 89 pgs: 89 active+clean; 287 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T16:57:22.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:21 smithi063 ceph-mon[31063]: osdmap e225: 8 total, 8 up, 8 in 2024-08-17T16:57:22.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:21 smithi063 ceph-mon[31063]: pgmap v21: 89 pgs: 89 active+clean; 287 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T16:57:22.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:21 smithi063 ceph-mon[42680]: osdmap e225: 8 total, 8 up, 8 in 2024-08-17T16:57:22.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:21 smithi063 ceph-mon[42680]: pgmap v21: 89 pgs: 89 active+clean; 287 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T16:57:22.344 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 systemd[1]: Stopping Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:57:22.596 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:57:22.343Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T16:57:22.596 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:57:22.343Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-17T16:57:22.596 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:57:22.343Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-17T16:57:22.596 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:57:22.343Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T16:57:22.596 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:57:22.343Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-17T16:57:22.596 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:57:22.343Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-17T16:57:22.596 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:57:22.343Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T16:57:22.596 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:57:22.343Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-17T16:57:22.596 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:57:22.343Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-17T16:57:22.596 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:57:22.343Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T16:57:22.596 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:57:22.343Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-17T16:57:22.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[78344]: ts=2024-08-17T16:57:22.343Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-17T16:57:22.597 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 podman[80981]: 2024-08-17 16:57:22.353801975 +0000 UTC m=+0.135171163 container died 039ece187f07d7e763df51afa25c27a55c85a075e2ff1be29b66f8d1d18f36f2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:22.856 INFO:tasks.rados.rados.0.smithi063.stdout:614: done (0 left) 2024-08-17T16:57:22.856 INFO:tasks.rados.rados.0.smithi063.stdout:615: write oid 9 current snap is 84 2024-08-17T16:57:22.856 INFO:tasks.rados.rados.0.smithi063.stdout:615: seq_num 269 ranges {521083=714336,1724715=555421} 2024-08-17T16:57:22.863 INFO:tasks.rados.rados.0.smithi063.stdout:615: writing smithi06387500-9 from 521083 to 1235419 tid 1 2024-08-17T16:57:22.870 INFO:tasks.rados.rados.0.smithi063.stdout:615: writing smithi06387500-9 from 1724715 to 2280136 tid 2 2024-08-17T16:57:22.870 INFO:tasks.rados.rados.0.smithi063.stdout:615: finishing write tid 1 to smithi06387500-9 2024-08-17T16:57:22.870 INFO:tasks.rados.rados.0.smithi063.stdout:616: write oid 15 current snap is 84 2024-08-17T16:57:22.870 INFO:tasks.rados.rados.0.smithi063.stdout:616: seq_num 270 ranges {537890=789585,1783197=699079} 2024-08-17T16:57:22.877 INFO:tasks.rados.rados.0.smithi063.stdout:616: writing smithi06387500-15 from 537890 to 1327475 tid 1 2024-08-17T16:57:22.884 INFO:tasks.rados.rados.0.smithi063.stdout:616: writing smithi06387500-15 from 1783197 to 2482276 tid 2 2024-08-17T16:57:22.884 INFO:tasks.rados.rados.0.smithi063.stdout:617: read oid 50 snap -1 2024-08-17T16:57:22.884 INFO:tasks.rados.rados.0.smithi063.stdout:617: expect (ObjNum 226 snap 64 seq_num 226) 2024-08-17T16:57:22.884 INFO:tasks.rados.rados.0.smithi063.stdout:618: read oid 47 snap 51 2024-08-17T16:57:22.884 INFO:tasks.rados.rados.0.smithi063.stdout:618: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:57:22.884 INFO:tasks.rados.rados.0.smithi063.stdout:619: read oid 34 snap -1 2024-08-17T16:57:22.885 INFO:tasks.rados.rados.0.smithi063.stdout:619: expect (ObjNum 260 snap 79 seq_num 260) 2024-08-17T16:57:22.885 INFO:tasks.rados.rados.0.smithi063.stdout:620: write oid 13 current snap is 84 2024-08-17T16:57:22.885 INFO:tasks.rados.rados.0.smithi063.stdout:615: finishing write tid 2 to smithi06387500-9 2024-08-17T16:57:22.885 INFO:tasks.rados.rados.0.smithi063.stdout:615: finishing write tid 3 to smithi06387500-9 2024-08-17T16:57:22.885 INFO:tasks.rados.rados.0.smithi063.stdout:615: finishing write tid 4 to smithi06387500-9 2024-08-17T16:57:22.885 INFO:tasks.rados.rados.0.smithi063.stdout:615: oid 9 updating version 0 to 324 2024-08-17T16:57:22.885 INFO:tasks.rados.rados.0.smithi063.stdout:615: oid 9 version 324 is already newer than 323 2024-08-17T16:57:22.885 INFO:tasks.rados.rados.0.smithi063.stdout:615: oid 9 updating version 324 to 325 2024-08-17T16:57:22.885 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 325 (ObjNum 269 snap 84 seq_num 269) dirty exists 2024-08-17T16:57:22.885 INFO:tasks.rados.rados.0.smithi063.stdout:615: left oid 9 (ObjNum 269 snap 84 seq_num 269) 2024-08-17T16:57:22.885 INFO:tasks.rados.rados.0.smithi063.stdout:616: finishing write tid 1 to smithi06387500-15 2024-08-17T16:57:22.885 INFO:tasks.rados.rados.0.smithi063.stdout:620: seq_num 271 ranges {554697=464834,1725326=559090} 2024-08-17T16:57:22.890 INFO:tasks.rados.rados.0.smithi063.stdout:620: writing smithi06387500-13 from 554697 to 1019531 tid 1 2024-08-17T16:57:22.898 INFO:tasks.rados.rados.0.smithi063.stdout:620: writing smithi06387500-13 from 1725326 to 2284416 tid 2 2024-08-17T16:57:22.898 INFO:tasks.rados.rados.0.smithi063.stdout:615: done (5 left) 2024-08-17T16:57:22.898 INFO:tasks.rados.rados.0.smithi063.stdout:621: read oid 12 snap 36 2024-08-17T16:57:22.898 INFO:tasks.rados.rados.0.smithi063.stdout:621: expect (ObjNum 68 snap 7 seq_num 68) 2024-08-17T16:57:22.898 INFO:tasks.rados.rados.0.smithi063.stdout:622: snap_remove snap 82 2024-08-17T16:57:22.967 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 podman[80981]: 2024-08-17 16:57:22.715902657 +0000 UTC m=+0.497271878 container cleanup 039ece187f07d7e763df51afa25c27a55c85a075e2ff1be29b66f8d1d18f36f2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:22.967 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 bash[80981]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T16:57:22.968 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:22 smithi162 ceph-mon[34633]: osdmap e226: 8 total, 8 up, 8 in 2024-08-17T16:57:22.968 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:22 smithi162 ceph-mon[34633]: osdmap e227: 8 total, 8 up, 8 in 2024-08-17T16:57:23.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:22 smithi063 ceph-mon[31063]: osdmap e226: 8 total, 8 up, 8 in 2024-08-17T16:57:23.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:22 smithi063 ceph-mon[31063]: osdmap e227: 8 total, 8 up, 8 in 2024-08-17T16:57:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:22 smithi063 ceph-mon[42680]: osdmap e226: 8 total, 8 up, 8 in 2024-08-17T16:57:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:22 smithi063 ceph-mon[42680]: osdmap e227: 8 total, 8 up, 8 in 2024-08-17T16:57:23.345 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:22 smithi162 podman[80998]: 2024-08-17 16:57:22.967119092 +0000 UTC m=+0.614567339 container remove 039ece187f07d7e763df51afa25c27a55c85a075e2ff1be29b66f8d1d18f36f2 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:23.346 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-17T16:57:23.346 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 systemd[1]: Stopped Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:57:23.346 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@prometheus.a.service: Consumed 1.121s CPU time. 2024-08-17T16:57:23.346 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 systemd[1]: Starting Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:57:23.645 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 podman[81095]: 2024-08-17 16:57:23.257567726 +0000 UTC m=+0.018222201 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-08-17T16:57:23.645 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 podman[81095]: 2024-08-17 16:57:23.393334077 +0000 UTC m=+0.153988545 container create 5ccd60dff86c609512011cf6ec8af1a35769dc292055a798311aec55aba6514d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:23.870 INFO:tasks.rados.rados.0.smithi063.stdout:622: done (6 left) 2024-08-17T16:57:23.870 INFO:tasks.rados.rados.0.smithi063.stdout:623: snap_remove snap 38 2024-08-17T16:57:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:23 smithi063 ceph-mon[42680]: pgmap v24: 89 pgs: 89 active+clean; 287 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T16:57:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:23 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:23 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:24.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:23 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:57:24.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:23 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:24.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:23 smithi063 ceph-mon[31063]: pgmap v24: 89 pgs: 89 active+clean; 287 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T16:57:24.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:23 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:24.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:23 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:24.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:23 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:57:24.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:23 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:24.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-mon[34633]: pgmap v24: 89 pgs: 89 active+clean; 287 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T16:57:24.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:24.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:24.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:57:24.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:24.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 podman[81095]: 2024-08-17 16:57:23.645198815 +0000 UTC m=+0.405853283 container init 5ccd60dff86c609512011cf6ec8af1a35769dc292055a798311aec55aba6514d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:24.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 podman[81095]: 2024-08-17 16:57:23.650152584 +0000 UTC m=+0.410807054 container start 5ccd60dff86c609512011cf6ec8af1a35769dc292055a798311aec55aba6514d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:24.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.674Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-08-17T16:57:24.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.674Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-08-17T16:57:24.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.674Z caller=main.go:570 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi162 (none))" 2024-08-17T16:57:24.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.674Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T16:57:24.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.674Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T16:57:24.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.676Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T16:57:24.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.676Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-08-17T16:57:24.097 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.677Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T16:57:24.098 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.677Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T16:57:24.098 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.679Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T16:57:24.098 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.679Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.618µs 2024-08-17T16:57:24.098 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.679Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T16:57:24.098 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.681Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-08-17T16:57:24.098 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.688Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-08-17T16:57:24.098 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.689Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-08-17T16:57:24.098 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.689Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-08-17T16:57:24.098 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.689Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=39.517µs wal_replay_duration=10.10236ms wbl_replay_duration=89ns total_replay_duration=10.17051ms 2024-08-17T16:57:24.098 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.691Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T16:57:24.098 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.691Z caller=main.go:1029 level=info msg="TSDB started" 2024-08-17T16:57:24.098 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.691Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T16:57:24.099 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.712Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.356311ms db_storage=760ns remote_storage=1.42µs web_handler=396ns query_engine=678ns scrape=499.603µs scrape_sd=98.168µs notify=22.167µs notify_sd=21.585µs rules=20.369636ms tracing=6.419µs 2024-08-17T16:57:24.099 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.712Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-08-17T16:57:24.099 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:57:23.712Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T16:57:24.099 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 bash[81095]: 5ccd60dff86c609512011cf6ec8af1a35769dc292055a798311aec55aba6514d 2024-08-17T16:57:24.099 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:57:23 smithi162 systemd[1]: Started Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:57:24.854 INFO:tasks.rados.rados.0.smithi063.stdout:623: done (6 left) 2024-08-17T16:57:24.855 INFO:tasks.rados.rados.0.smithi063.stdout:624: rollback oid 45 current snap is 84 2024-08-17T16:57:24.855 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 45 to 75 2024-08-17T16:57:24.855 INFO:tasks.rados.rados.0.smithi063.stdout:616: finishing write tid 2 to smithi06387500-15 2024-08-17T16:57:24.855 INFO:tasks.rados.rados.0.smithi063.stdout:616: finishing write tid 3 to smithi06387500-15 2024-08-17T16:57:24.855 INFO:tasks.rados.rados.0.smithi063.stdout:616: finishing write tid 4 to smithi06387500-15 2024-08-17T16:57:24.855 INFO:tasks.rados.rados.0.smithi063.stdout:616: oid 15 updating version 0 to 220 2024-08-17T16:57:24.855 INFO:tasks.rados.rados.0.smithi063.stdout:616: oid 15 version 220 is already newer than 218 2024-08-17T16:57:24.855 INFO:tasks.rados.rados.0.smithi063.stdout:616: oid 15 version 220 is already newer than 219 2024-08-17T16:57:24.855 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 220 (ObjNum 270 snap 84 seq_num 270) dirty exists 2024-08-17T16:57:24.855 INFO:tasks.rados.rados.0.smithi063.stdout:616: left oid 15 (ObjNum 270 snap 84 seq_num 270) 2024-08-17T16:57:24.855 INFO:tasks.rados.rados.0.smithi063.stdout:618: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:57:25.017 INFO:tasks.rados.rados.0.smithi063.stdout:617: expect (ObjNum 226 snap 64 seq_num 226) 2024-08-17T16:57:25.136 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:24 smithi162 ceph-mon[34633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:57:25.136 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:24 smithi162 ceph-mon[34633]: osdmap e228: 8 total, 8 up, 8 in 2024-08-17T16:57:25.136 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:24 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:25.136 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:24 smithi162 ceph-mon[34633]: osdmap e229: 8 total, 8 up, 8 in 2024-08-17T16:57:25.205 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:24 smithi063 ceph-mon[42680]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:57:25.205 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:24 smithi063 ceph-mon[42680]: osdmap e228: 8 total, 8 up, 8 in 2024-08-17T16:57:25.206 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:24 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:25.206 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:24 smithi063 ceph-mon[42680]: osdmap e229: 8 total, 8 up, 8 in 2024-08-17T16:57:25.206 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:24 smithi063 ceph-mon[31063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:57:25.206 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:24 smithi063 ceph-mon[31063]: osdmap e228: 8 total, 8 up, 8 in 2024-08-17T16:57:25.206 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:24 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:25.206 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:24 smithi063 ceph-mon[31063]: osdmap e229: 8 total, 8 up, 8 in 2024-08-17T16:57:25.312 INFO:tasks.rados.rados.0.smithi063.stdout:619: expect (ObjNum 260 snap 79 seq_num 260) 2024-08-17T16:57:25.394 INFO:tasks.rados.rados.0.smithi063.stdout:620: finishing write tid 1 to smithi06387500-13 2024-08-17T16:57:25.394 INFO:tasks.rados.rados.0.smithi063.stdout:620: finishing write tid 2 to smithi06387500-13 2024-08-17T16:57:25.395 INFO:tasks.rados.rados.0.smithi063.stdout:620: finishing write tid 3 to smithi06387500-13 2024-08-17T16:57:25.395 INFO:tasks.rados.rados.0.smithi063.stdout:616: done (6 left) 2024-08-17T16:57:25.395 INFO:tasks.rados.rados.0.smithi063.stdout:617: done (5 left) 2024-08-17T16:57:25.395 INFO:tasks.rados.rados.0.smithi063.stdout:618: done (4 left) 2024-08-17T16:57:25.395 INFO:tasks.rados.rados.0.smithi063.stdout:619: done (3 left) 2024-08-17T16:57:25.395 INFO:tasks.rados.rados.0.smithi063.stdout:625: snap_remove snap 68 2024-08-17T16:57:25.876 INFO:tasks.rados.rados.0.smithi063.stdout:625: done (3 left) 2024-08-17T16:57:25.876 INFO:tasks.rados.rados.0.smithi063.stdout:626: read oid 15 snap -1 2024-08-17T16:57:25.876 INFO:tasks.rados.rados.0.smithi063.stdout:626: expect (ObjNum 270 snap 84 seq_num 270) 2024-08-17T16:57:25.876 INFO:tasks.rados.rados.0.smithi063.stdout:627: write oid 36 current snap is 84 2024-08-17T16:57:25.876 INFO:tasks.rados.rados.0.smithi063.stdout:627: seq_num 272 ranges {571504=540083,1783808=702748} 2024-08-17T16:57:25.882 INFO:tasks.rados.rados.0.smithi063.stdout:627: writing smithi06387500-36 from 571504 to 1111587 tid 1 2024-08-17T16:57:25.890 INFO:tasks.rados.rados.0.smithi063.stdout:627: writing smithi06387500-36 from 1783808 to 2486556 tid 2 2024-08-17T16:57:25.890 INFO:tasks.rados.rados.0.smithi063.stdout:628: snap_remove snap 78 2024-08-17T16:57:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:25 smithi162 ceph-mon[34633]: pgmap v27: 89 pgs: 89 active+clean; 287 MiB data, 806 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:25 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:57:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:25 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:57:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:25 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:26.196 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:25 smithi063 ceph-mon[31063]: pgmap v27: 89 pgs: 89 active+clean; 287 MiB data, 806 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:26.196 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:25 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:57:26.197 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:25 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:57:26.197 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:25 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:26.197 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:25 smithi063 ceph-mon[42680]: pgmap v27: 89 pgs: 89 active+clean; 287 MiB data, 806 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:26.197 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:25 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-08-17T16:57:26.197 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:25 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-08-17T16:57:26.197 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:25 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:26.877 INFO:tasks.rados.rados.0.smithi063.stdout:620: finishing write tid 4 to smithi06387500-13 2024-08-17T16:57:26.877 INFO:tasks.rados.rados.0.smithi063.stdout:620: oid 13 updating version 0 to 249 2024-08-17T16:57:26.877 INFO:tasks.rados.rados.0.smithi063.stdout:620: oid 13 version 249 is already newer than 247 2024-08-17T16:57:26.877 INFO:tasks.rados.rados.0.smithi063.stdout:620: oid 13 version 249 is already newer than 248 2024-08-17T16:57:26.877 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 249 (ObjNum 271 snap 84 seq_num 271) dirty exists 2024-08-17T16:57:26.877 INFO:tasks.rados.rados.0.smithi063.stdout:620: left oid 13 (ObjNum 271 snap 84 seq_num 271) 2024-08-17T16:57:26.877 INFO:tasks.rados.rados.0.smithi063.stdout:621: expect (ObjNum 68 snap 7 seq_num 68) 2024-08-17T16:57:27.023 INFO:tasks.rados.rados.0.smithi063.stdout:624: finishing rollback tid 0 to smithi06387500-45 2024-08-17T16:57:27.023 INFO:tasks.rados.rados.0.smithi063.stdout:624: finishing rollback tid 1 to smithi06387500-45 2024-08-17T16:57:27.023 INFO:tasks.rados.rados.0.smithi063.stdout:624: finishing rollback tid 2 to smithi06387500-45 2024-08-17T16:57:27.023 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 105 (ObjNum 233 snap 68 seq_num 233) dirty exists 2024-08-17T16:57:27.023 INFO:tasks.rados.rados.0.smithi063.stdout:626: expect (ObjNum 270 snap 84 seq_num 270) 2024-08-17T16:57:27.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:26 smithi162 systemd[1]: Stopping Ceph mgr.x for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:57:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:26 smithi162 ceph-mon[34633]: Upgrade: Updating mgr.x 2024-08-17T16:57:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:26 smithi162 ceph-mon[34633]: Deploying daemon mgr.x on smithi162 2024-08-17T16:57:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:26 smithi162 ceph-mon[34633]: osdmap e230: 8 total, 8 up, 8 in 2024-08-17T16:57:27.103 INFO:tasks.rados.rados.0.smithi063.stdout:627: finishing write tid 1 to smithi06387500-36 2024-08-17T16:57:27.103 INFO:tasks.rados.rados.0.smithi063.stdout:627: finishing write tid 2 to smithi06387500-36 2024-08-17T16:57:27.103 INFO:tasks.rados.rados.0.smithi063.stdout:627: finishing write tid 3 to smithi06387500-36 2024-08-17T16:57:27.103 INFO:tasks.rados.rados.0.smithi063.stdout:627: finishing write tid 4 to smithi06387500-36 2024-08-17T16:57:27.103 INFO:tasks.rados.rados.0.smithi063.stdout:627: oid 36 updating version 0 to 617 2024-08-17T16:57:27.103 INFO:tasks.rados.rados.0.smithi063.stdout:627: oid 36 updating version 617 to 618 2024-08-17T16:57:27.103 INFO:tasks.rados.rados.0.smithi063.stdout:627: oid 36 version 618 is already newer than 616 2024-08-17T16:57:27.103 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 618 (ObjNum 272 snap 84 seq_num 272) dirty exists 2024-08-17T16:57:27.103 INFO:tasks.rados.rados.0.smithi063.stdout:627: left oid 36 (ObjNum 272 snap 84 seq_num 272) 2024-08-17T16:57:27.103 INFO:tasks.rados.rados.0.smithi063.stdout:620: done (5 left) 2024-08-17T16:57:27.103 INFO:tasks.rados.rados.0.smithi063.stdout:621: done (4 left) 2024-08-17T16:57:27.103 INFO:tasks.rados.rados.0.smithi063.stdout:624: done (3 left) 2024-08-17T16:57:27.103 INFO:tasks.rados.rados.0.smithi063.stdout:626: done (2 left) 2024-08-17T16:57:27.104 INFO:tasks.rados.rados.0.smithi063.stdout:627: done (1 left) 2024-08-17T16:57:27.104 INFO:tasks.rados.rados.0.smithi063.stdout:628: done (0 left) 2024-08-17T16:57:27.104 INFO:tasks.rados.rados.0.smithi063.stdout:629: write oid 21 current snap is 84 2024-08-17T16:57:27.104 INFO:tasks.rados.rados.0.smithi063.stdout:629: seq_num 273 ranges {588311=615332,1842290=562759} 2024-08-17T16:57:27.109 INFO:tasks.rados.rados.0.smithi063.stdout:629: writing smithi06387500-21 from 588311 to 1203643 tid 1 2024-08-17T16:57:27.116 INFO:tasks.rados.rados.0.smithi063.stdout:629: writing smithi06387500-21 from 1842290 to 2405049 tid 2 2024-08-17T16:57:27.116 INFO:tasks.rados.rados.0.smithi063.stdout:630: delete oid 45 current snap is 84 2024-08-17T16:57:27.116 INFO:tasks.rados.rados.0.smithi063.stdout:629: finishing write tid 1 to smithi06387500-21 2024-08-17T16:57:27.118 INFO:tasks.rados.rados.0.smithi063.stdout:630: done (1 left) 2024-08-17T16:57:27.118 INFO:tasks.rados.rados.0.smithi063.stdout:631: snap_create 2024-08-17T16:57:27.118 INFO:tasks.rados.rados.0.smithi063.stdout:629: finishing write tid 2 to smithi06387500-21 2024-08-17T16:57:27.118 INFO:tasks.rados.rados.0.smithi063.stdout:629: finishing write tid 3 to smithi06387500-21 2024-08-17T16:57:27.118 INFO:tasks.rados.rados.0.smithi063.stdout:629: finishing write tid 4 to smithi06387500-21 2024-08-17T16:57:27.118 INFO:tasks.rados.rados.0.smithi063.stdout:629: oid 21 updating version 0 to 407 2024-08-17T16:57:27.119 INFO:tasks.rados.rados.0.smithi063.stdout:629: oid 21 updating version 407 to 408 2024-08-17T16:57:27.119 INFO:tasks.rados.rados.0.smithi063.stdout:629: oid 21 version 408 is already newer than 406 2024-08-17T16:57:27.119 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 408 (ObjNum 273 snap 84 seq_num 273) dirty exists 2024-08-17T16:57:27.119 INFO:tasks.rados.rados.0.smithi063.stdout:629: left oid 21 (ObjNum 273 snap 84 seq_num 273) 2024-08-17T16:57:27.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:26 smithi063 ceph-mon[42680]: Upgrade: Updating mgr.x 2024-08-17T16:57:27.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:26 smithi063 ceph-mon[42680]: Deploying daemon mgr.x on smithi162 2024-08-17T16:57:27.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:26 smithi063 ceph-mon[42680]: osdmap e230: 8 total, 8 up, 8 in 2024-08-17T16:57:27.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:26 smithi063 ceph-mon[31063]: Upgrade: Updating mgr.x 2024-08-17T16:57:27.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:26 smithi063 ceph-mon[31063]: Deploying daemon mgr.x on smithi162 2024-08-17T16:57:27.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:26 smithi063 ceph-mon[31063]: osdmap e230: 8 total, 8 up, 8 in 2024-08-17T16:57:27.472 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:27 smithi162 podman[81503]: 2024-08-17 16:57:27.204822674 +0000 UTC m=+0.327315506 container died 180435c78c80767a84f392ef8a014f287b428d6b2b0e69721258e928d585e98d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0) 2024-08-17T16:57:27.839 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:27 smithi162 podman[81503]: 2024-08-17 16:57:27.562455711 +0000 UTC m=+0.684948541 container cleanup 180435c78c80767a84f392ef8a014f287b428d6b2b0e69721258e928d585e98d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-08-17T16:57:27.839 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:27 smithi162 bash[81503]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x 2024-08-17T16:57:27.897 INFO:tasks.rados.rados.0.smithi063.stdout:629: done (1 left) 2024-08-17T16:57:27.897 INFO:tasks.rados.rados.0.smithi063.stdout:631: done (0 left) 2024-08-17T16:57:27.897 INFO:tasks.rados.rados.0.smithi063.stdout:632: rollback oid 19 current snap is 85 2024-08-17T16:57:27.897 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 19 to 60 2024-08-17T16:57:27.898 INFO:tasks.rados.rados.0.smithi063.stdout:633: rollback oid 6 current snap is 85 2024-08-17T16:57:27.898 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 6 to 81 2024-08-17T16:57:27.898 INFO:tasks.rados.rados.0.smithi063.stdout:634: snap_remove snap 66 2024-08-17T16:57:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:27 smithi162 ceph-mon[34633]: osdmap e231: 8 total, 8 up, 8 in 2024-08-17T16:57:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:27 smithi162 ceph-mon[34633]: pgmap v30: 89 pgs: 89 active+clean; 294 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 924 KiB/s wr, 12 op/s 2024-08-17T16:57:28.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:27 smithi162 podman[81522]: 2024-08-17 16:57:27.838687705 +0000 UTC m=+0.632903033 container remove 180435c78c80767a84f392ef8a014f287b428d6b2b0e69721258e928d585e98d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-08-17T16:57:28.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:27 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-17T16:57:28.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:27 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-08-17T16:57:28.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:27 smithi162 systemd[1]: Stopped Ceph mgr.x for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:57:28.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:27 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.x.service: Consumed 17.100s CPU time. 2024-08-17T16:57:28.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:27 smithi063 ceph-mon[42680]: osdmap e231: 8 total, 8 up, 8 in 2024-08-17T16:57:28.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:27 smithi063 ceph-mon[42680]: pgmap v30: 89 pgs: 89 active+clean; 294 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 924 KiB/s wr, 12 op/s 2024-08-17T16:57:28.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:27 smithi063 ceph-mon[31063]: osdmap e231: 8 total, 8 up, 8 in 2024-08-17T16:57:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:27 smithi063 ceph-mon[31063]: pgmap v30: 89 pgs: 89 active+clean; 294 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 924 KiB/s wr, 12 op/s 2024-08-17T16:57:28.420 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:28 smithi162 systemd[1]: Starting Ceph mgr.x for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:57:28.420 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:28 smithi162 podman[81655]: 2024-08-17 16:57:28.38165153 +0000 UTC m=+0.113561564 container create da48034a2895b6b37b4c56937841f60a360da1ab77fe946fc286922cecba9d5d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-cd8bdbe) 2024-08-17T16:57:28.420 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:28 smithi162 podman[81655]: 2024-08-17 16:57:28.286084043 +0000 UTC m=+0.017994088 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T16:57:28.709 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:28 smithi162 podman[81655]: 2024-08-17 16:57:28.609433812 +0000 UTC m=+0.341343845 container init da48034a2895b6b37b4c56937841f60a360da1ab77fe946fc286922cecba9d5d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-08-17T16:57:28.710 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:28 smithi162 podman[81655]: 2024-08-17 16:57:28.614490558 +0000 UTC m=+0.346400595 container start da48034a2895b6b37b4c56937841f60a360da1ab77fe946fc286922cecba9d5d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True) 2024-08-17T16:57:28.710 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:28 smithi162 ceph-mgr[81701]: -- 172.21.15.162:0/3458707235 <== mon.1 v2:172.21.15.162:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55da5ecaf0e0 con 0x55da5ec8d400 2024-08-17T16:57:28.897 INFO:tasks.rados.rados.0.smithi063.stdout:634: done (2 left) 2024-08-17T16:57:28.897 INFO:tasks.rados.rados.0.smithi063.stdout:635: read oid 26 snap -1 2024-08-17T16:57:28.897 INFO:tasks.rados.rados.0.smithi063.stdout:635: expect (ObjNum 220 snap 61 seq_num 220) 2024-08-17T16:57:28.897 INFO:tasks.rados.rados.0.smithi063.stdout:636: rollback oid 14 current snap is 85 2024-08-17T16:57:28.897 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 14 to 71 2024-08-17T16:57:28.898 INFO:tasks.rados.rados.0.smithi063.stdout:633: finishing rollback tid 0 to smithi06387500-6 2024-08-17T16:57:28.898 INFO:tasks.rados.rados.0.smithi063.stdout:633: finishing rollback tid 1 to smithi06387500-6 2024-08-17T16:57:28.898 INFO:tasks.rados.rados.0.smithi063.stdout:633: finishing rollback tid 2 to smithi06387500-6 2024-08-17T16:57:28.898 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 165 (ObjNum 178 snap 47 seq_num 178) dirty exists 2024-08-17T16:57:28.898 INFO:tasks.rados.rados.0.smithi063.stdout:632: finishing rollback tid 1 to smithi06387500-19 2024-08-17T16:57:28.898 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 248 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T16:57:28.898 INFO:tasks.rados.rados.0.smithi063.stdout:632: done (3 left) 2024-08-17T16:57:28.898 INFO:tasks.rados.rados.0.smithi063.stdout:633: done (2 left) 2024-08-17T16:57:28.898 INFO:tasks.rados.rados.0.smithi063.stdout:637: snap_create 2024-08-17T16:57:28.909 INFO:tasks.rados.rados.0.smithi063.stdout:636: finishing rollback tid 1 to smithi06387500-14 2024-08-17T16:57:28.909 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 488 (ObjNum 129 snap 0 seq_num 1806871120) dirty dne 2024-08-17T16:57:28.918 INFO:tasks.rados.rados.0.smithi063.stdout:635: expect (ObjNum 220 snap 61 seq_num 220) 2024-08-17T16:57:29.095 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:28 smithi162 bash[81655]: da48034a2895b6b37b4c56937841f60a360da1ab77fe946fc286922cecba9d5d 2024-08-17T16:57:29.096 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:28 smithi162 systemd[1]: Started Ceph mgr.x for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:57:29.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:28 smithi162 ceph-mon[34633]: osdmap e232: 8 total, 8 up, 8 in 2024-08-17T16:57:29.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:28 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:29.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:28 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:29.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:28 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:29.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:28 smithi063 ceph-mon[42680]: osdmap e232: 8 total, 8 up, 8 in 2024-08-17T16:57:29.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:28 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:29.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:28 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:29.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:28 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:29.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:28 smithi063 ceph-mon[31063]: osdmap e232: 8 total, 8 up, 8 in 2024-08-17T16:57:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:28 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:28 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:28 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:29.856 INFO:tasks.rados.rados.0.smithi063.stdout:635: done (2 left) 2024-08-17T16:57:29.856 INFO:tasks.rados.rados.0.smithi063.stdout:636: done (1 left) 2024-08-17T16:57:29.856 INFO:tasks.rados.rados.0.smithi063.stdout:637: done (0 left) 2024-08-17T16:57:29.856 INFO:tasks.rados.rados.0.smithi063.stdout:638: read oid 18 snap -1 2024-08-17T16:57:29.856 INFO:tasks.rados.rados.0.smithi063.stdout:638: expect (ObjNum 265 snap 80 seq_num 265) 2024-08-17T16:57:29.856 INFO:tasks.rados.rados.0.smithi063.stdout:639: snap_remove snap 83 2024-08-17T16:57:29.903 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:29 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:57:29] "GET /metrics HTTP/1.1" 200 36217 "" "Prometheus/2.43.0" 2024-08-17T16:57:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:29 smithi162 ceph-mon[34633]: osdmap e233: 8 total, 8 up, 8 in 2024-08-17T16:57:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:29 smithi162 ceph-mon[34633]: pgmap v33: 89 pgs: 89 active+clean; 294 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 923 KiB/s wr, 12 op/s 2024-08-17T16:57:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:29 smithi162 ceph-mon[34633]: osdmap e234: 8 total, 8 up, 8 in 2024-08-17T16:57:30.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:29 smithi063 ceph-mon[31063]: osdmap e233: 8 total, 8 up, 8 in 2024-08-17T16:57:30.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:29 smithi063 ceph-mon[31063]: pgmap v33: 89 pgs: 89 active+clean; 294 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 923 KiB/s wr, 12 op/s 2024-08-17T16:57:30.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:29 smithi063 ceph-mon[31063]: osdmap e234: 8 total, 8 up, 8 in 2024-08-17T16:57:30.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:29 smithi063 ceph-mon[42680]: osdmap e233: 8 total, 8 up, 8 in 2024-08-17T16:57:30.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:29 smithi063 ceph-mon[42680]: pgmap v33: 89 pgs: 89 active+clean; 294 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 923 KiB/s wr, 12 op/s 2024-08-17T16:57:30.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:29 smithi063 ceph-mon[42680]: osdmap e234: 8 total, 8 up, 8 in 2024-08-17T16:57:30.858 INFO:tasks.rados.rados.0.smithi063.stdout:639: done (1 left) 2024-08-17T16:57:30.859 INFO:tasks.rados.rados.0.smithi063.stdout:640: read oid 13 snap -1 2024-08-17T16:57:30.859 INFO:tasks.rados.rados.0.smithi063.stdout:640: expect (ObjNum 271 snap 84 seq_num 271) 2024-08-17T16:57:30.859 INFO:tasks.rados.rados.0.smithi063.stdout:641: delete oid 39 current snap is 86 2024-08-17T16:57:30.859 INFO:tasks.rados.rados.0.smithi063.stdout:638: expect (ObjNum 265 snap 80 seq_num 265) 2024-08-17T16:57:31.019 INFO:tasks.rados.rados.0.smithi063.stdout:640: expect (ObjNum 271 snap 84 seq_num 271) 2024-08-17T16:57:31.168 INFO:tasks.rados.rados.0.smithi063.stdout:638: done (2 left) 2024-08-17T16:57:31.169 INFO:tasks.rados.rados.0.smithi063.stdout:640: done (1 left) 2024-08-17T16:57:31.169 INFO:tasks.rados.rados.0.smithi063.stdout:641: done (0 left) 2024-08-17T16:57:31.169 INFO:tasks.rados.rados.0.smithi063.stdout:642: read oid 40 snap -1 2024-08-17T16:57:31.169 INFO:tasks.rados.rados.0.smithi063.stdout:642: expect (ObjNum 239 snap 73 seq_num 239) 2024-08-17T16:57:31.169 INFO:tasks.rados.rados.0.smithi063.stdout:643: read oid 11 snap -1 2024-08-17T16:57:31.169 INFO:tasks.rados.rados.0.smithi063.stdout:643: expect deleted 2024-08-17T16:57:31.169 INFO:tasks.rados.rados.0.smithi063.stdout:644: snap_remove snap 73 2024-08-17T16:57:31.866 INFO:tasks.rados.rados.0.smithi063.stdout:642: expect (ObjNum 239 snap 73 seq_num 239) 2024-08-17T16:57:31.986 INFO:tasks.rados.rados.0.smithi063.stdout:642: done (2 left) 2024-08-17T16:57:31.986 INFO:tasks.rados.rados.0.smithi063.stdout:643: done (1 left) 2024-08-17T16:57:31.986 INFO:tasks.rados.rados.0.smithi063.stdout:644: done (0 left) 2024-08-17T16:57:31.986 INFO:tasks.rados.rados.0.smithi063.stdout:645: read oid 13 snap -1 2024-08-17T16:57:31.986 INFO:tasks.rados.rados.0.smithi063.stdout:645: expect (ObjNum 271 snap 84 seq_num 271) 2024-08-17T16:57:31.986 INFO:tasks.rados.rados.0.smithi063.stdout:646: read oid 31 snap -1 2024-08-17T16:57:31.986 INFO:tasks.rados.rados.0.smithi063.stdout:646: expect (ObjNum 138 snap 30 seq_num 138) 2024-08-17T16:57:31.986 INFO:tasks.rados.rados.0.smithi063.stdout:647: delete oid 16 current snap is 86 2024-08-17T16:57:31.987 INFO:tasks.rados.rados.0.smithi063.stdout:647: done (2 left) 2024-08-17T16:57:31.987 INFO:tasks.rados.rados.0.smithi063.stdout:648: snap_remove snap 67 2024-08-17T16:57:32.149 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:31 smithi162 ceph-mon[34633]: osdmap e235: 8 total, 8 up, 8 in 2024-08-17T16:57:32.149 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:31 smithi162 ceph-mon[34633]: pgmap v36: 89 pgs: 89 active+clean; 294 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 591 KiB/s wr, 9 op/s 2024-08-17T16:57:32.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:31 smithi063 ceph-mon[42680]: osdmap e235: 8 total, 8 up, 8 in 2024-08-17T16:57:32.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:31 smithi063 ceph-mon[42680]: pgmap v36: 89 pgs: 89 active+clean; 294 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 591 KiB/s wr, 9 op/s 2024-08-17T16:57:32.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:31 smithi063 ceph-mon[31063]: osdmap e235: 8 total, 8 up, 8 in 2024-08-17T16:57:32.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:31 smithi063 ceph-mon[31063]: pgmap v36: 89 pgs: 89 active+clean; 294 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 591 KiB/s wr, 9 op/s 2024-08-17T16:57:32.871 INFO:tasks.rados.rados.0.smithi063.stdout:648: done (2 left) 2024-08-17T16:57:32.871 INFO:tasks.rados.rados.0.smithi063.stdout:649: write oid 7 current snap is 86 2024-08-17T16:57:32.871 INFO:tasks.rados.rados.0.smithi063.stdout:649: seq_num 277 ranges {655539=632681,2076218=570097} 2024-08-17T16:57:32.879 INFO:tasks.rados.rados.0.smithi063.stdout:649: writing smithi06387500-7 from 655539 to 1288220 tid 1 2024-08-17T16:57:32.888 INFO:tasks.rados.rados.0.smithi063.stdout:649: writing smithi06387500-7 from 2076218 to 2646315 tid 2 2024-08-17T16:57:32.888 INFO:tasks.rados.rados.0.smithi063.stdout:650: delete oid 14 current snap is 86 2024-08-17T16:57:32.888 INFO:tasks.rados.rados.0.smithi063.stdout:646: expect (ObjNum 138 snap 30 seq_num 138) 2024-08-17T16:57:33.021 INFO:tasks.rados.rados.0.smithi063.stdout:645: expect (ObjNum 271 snap 84 seq_num 271) 2024-08-17T16:57:33.161 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:32 smithi162 ceph-mon[34633]: osdmap e236: 8 total, 8 up, 8 in 2024-08-17T16:57:33.170 INFO:tasks.rados.rados.0.smithi063.stdout:649: finishing write tid 1 to smithi06387500-7 2024-08-17T16:57:33.170 INFO:tasks.rados.rados.0.smithi063.stdout:649: finishing write tid 2 to smithi06387500-7 2024-08-17T16:57:33.170 INFO:tasks.rados.rados.0.smithi063.stdout:649: finishing write tid 3 to smithi06387500-7 2024-08-17T16:57:33.170 INFO:tasks.rados.rados.0.smithi063.stdout:649: finishing write tid 4 to smithi06387500-7 2024-08-17T16:57:33.170 INFO:tasks.rados.rados.0.smithi063.stdout:649: oid 7 updating version 0 to 361 2024-08-17T16:57:33.171 INFO:tasks.rados.rados.0.smithi063.stdout:649: oid 7 version 361 is already newer than 360 2024-08-17T16:57:33.171 INFO:tasks.rados.rados.0.smithi063.stdout:649: oid 7 version 361 is already newer than 359 2024-08-17T16:57:33.171 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 361 (ObjNum 277 snap 86 seq_num 277) dirty exists 2024-08-17T16:57:33.171 INFO:tasks.rados.rados.0.smithi063.stdout:649: left oid 7 (ObjNum 277 snap 86 seq_num 277) 2024-08-17T16:57:33.171 INFO:tasks.rados.rados.0.smithi063.stdout:645: done (3 left) 2024-08-17T16:57:33.171 INFO:tasks.rados.rados.0.smithi063.stdout:646: done (2 left) 2024-08-17T16:57:33.171 INFO:tasks.rados.rados.0.smithi063.stdout:649: done (1 left) 2024-08-17T16:57:33.171 INFO:tasks.rados.rados.0.smithi063.stdout:650: done (0 left) 2024-08-17T16:57:33.171 INFO:tasks.rados.rados.0.smithi063.stdout:651: delete oid 39 current snap is 86 2024-08-17T16:57:33.172 INFO:tasks.rados.rados.0.smithi063.stdout:651: done (0 left) 2024-08-17T16:57:33.172 INFO:tasks.rados.rados.0.smithi063.stdout:652: read oid 4 snap -1 2024-08-17T16:57:33.172 INFO:tasks.rados.rados.0.smithi063.stdout:652: expect deleted 2024-08-17T16:57:33.172 INFO:tasks.rados.rados.0.smithi063.stdout:653: rollback oid 16 current snap is 86 2024-08-17T16:57:33.172 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 16 to 36 2024-08-17T16:57:33.172 INFO:tasks.rados.rados.0.smithi063.stdout:654: write oid 45 current snap is 86 2024-08-17T16:57:33.172 INFO:tasks.rados.rados.0.smithi063.stdout:654: seq_num 280 ranges {705960=458428,1735311=717424} 2024-08-17T16:57:33.178 INFO:tasks.rados.rados.0.smithi063.stdout:654: writing smithi06387500-45 from 705960 to 1164388 tid 1 2024-08-17T16:57:33.185 INFO:tasks.rados.rados.0.smithi063.stdout:654: writing smithi06387500-45 from 1735311 to 2452735 tid 2 2024-08-17T16:57:33.185 INFO:tasks.rados.rados.0.smithi063.stdout:655: delete oid 11 current snap is 86 2024-08-17T16:57:33.185 INFO:tasks.rados.rados.0.smithi063.stdout:653: finishing rollback tid 1 to smithi06387500-16 2024-08-17T16:57:33.185 INFO:tasks.rados.rados.0.smithi063.stdout:653: finishing rollback tid 2 to smithi06387500-16 2024-08-17T16:57:33.186 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 237 (ObjNum 117 snap 20 seq_num 117) dirty exists 2024-08-17T16:57:33.186 INFO:tasks.rados.rados.0.smithi063.stdout:654: finishing write tid 1 to smithi06387500-45 2024-08-17T16:57:33.187 INFO:tasks.rados.rados.0.smithi063.stdout:652: done (3 left) 2024-08-17T16:57:33.187 INFO:tasks.rados.rados.0.smithi063.stdout:653: done (2 left) 2024-08-17T16:57:33.187 INFO:tasks.rados.rados.0.smithi063.stdout:655: done (1 left) 2024-08-17T16:57:33.187 INFO:tasks.rados.rados.0.smithi063.stdout:656: read oid 41 snap -1 2024-08-17T16:57:33.187 INFO:tasks.rados.rados.0.smithi063.stdout:656: expect (ObjNum 73 snap 9 seq_num 73) 2024-08-17T16:57:33.187 INFO:tasks.rados.rados.0.smithi063.stdout:657: read oid 8 snap 63 2024-08-17T16:57:33.187 INFO:tasks.rados.rados.0.smithi063.stdout:657: expect (ObjNum 207 snap 59 seq_num 207) 2024-08-17T16:57:33.187 INFO:tasks.rados.rados.0.smithi063.stdout:658: rollback oid 10 current snap is 86 2024-08-17T16:57:33.187 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 10 to 72 2024-08-17T16:57:33.187 INFO:tasks.rados.rados.0.smithi063.stdout:659: snap_remove snap 36 2024-08-17T16:57:33.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:32 smithi063 ceph-mon[42680]: osdmap e236: 8 total, 8 up, 8 in 2024-08-17T16:57:33.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:32 smithi063 ceph-mon[31063]: osdmap e236: 8 total, 8 up, 8 in 2024-08-17T16:57:33.875 INFO:tasks.rados.rados.0.smithi063.stdout:659: done (4 left) 2024-08-17T16:57:33.876 INFO:tasks.rados.rados.0.smithi063.stdout:660: read oid 11 snap -1 2024-08-17T16:57:33.876 INFO:tasks.rados.rados.0.smithi063.stdout:660: expect deleted 2024-08-17T16:57:33.876 INFO:tasks.rados.rados.0.smithi063.stdout:661: snap_remove snap 77 2024-08-17T16:57:34.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:33 smithi063 ceph-mon[42680]: osdmap e237: 8 total, 8 up, 8 in 2024-08-17T16:57:34.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:33 smithi063 ceph-mon[42680]: pgmap v39: 89 pgs: 89 active+clean; 294 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 591 KiB/s wr, 9 op/s 2024-08-17T16:57:34.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:33 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:34.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:33 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:34.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:33 smithi063 ceph-mon[31063]: osdmap e237: 8 total, 8 up, 8 in 2024-08-17T16:57:34.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:33 smithi063 ceph-mon[31063]: pgmap v39: 89 pgs: 89 active+clean; 294 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 591 KiB/s wr, 9 op/s 2024-08-17T16:57:34.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:33 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:34.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:33 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:33 smithi162 ceph-mon[34633]: osdmap e237: 8 total, 8 up, 8 in 2024-08-17T16:57:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:33 smithi162 ceph-mon[34633]: pgmap v39: 89 pgs: 89 active+clean; 294 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 591 KiB/s wr, 9 op/s 2024-08-17T16:57:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:33 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:33 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:34.346 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:33 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[81670]: [17/Aug/2024:16:57:33] ENGINE Bus STARTING 2024-08-17T16:57:34.346 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:33 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[81670]: CherryPy Checker: 2024-08-17T16:57:34.346 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:33 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[81670]: The Application mounted at '' has an empty config. 2024-08-17T16:57:34.346 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:33 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[81670]: 2024-08-17T16:57:34.347 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:34 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[81670]: [17/Aug/2024:16:57:34] ENGINE Serving on http://:::9283 2024-08-17T16:57:34.347 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:57:34 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[81670]: [17/Aug/2024:16:57:34] ENGINE Bus STARTED 2024-08-17T16:57:34.856 INFO:tasks.rados.rados.0.smithi063.stdout:661: done (5 left) 2024-08-17T16:57:34.857 INFO:tasks.rados.rados.0.smithi063.stdout:662: delete oid 16 current snap is 86 2024-08-17T16:57:34.857 INFO:tasks.rados.rados.0.smithi063.stdout:658: finishing rollback tid 1 to smithi06387500-10 2024-08-17T16:57:34.857 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 146 (ObjNum 129 snap 0 seq_num 1807028800) dirty dne 2024-08-17T16:57:34.857 INFO:tasks.rados.rados.0.smithi063.stdout:654: finishing write tid 2 to smithi06387500-45 2024-08-17T16:57:34.857 INFO:tasks.rados.rados.0.smithi063.stdout:654: finishing write tid 3 to smithi06387500-45 2024-08-17T16:57:34.857 INFO:tasks.rados.rados.0.smithi063.stdout:654: finishing write tid 4 to smithi06387500-45 2024-08-17T16:57:34.857 INFO:tasks.rados.rados.0.smithi063.stdout:654: oid 45 updating version 0 to 130 2024-08-17T16:57:34.857 INFO:tasks.rados.rados.0.smithi063.stdout:654: oid 45 version 130 is already newer than 129 2024-08-17T16:57:34.857 INFO:tasks.rados.rados.0.smithi063.stdout:654: oid 45 updating version 130 to 131 2024-08-17T16:57:34.857 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 131 (ObjNum 280 snap 86 seq_num 280) dirty exists 2024-08-17T16:57:34.857 INFO:tasks.rados.rados.0.smithi063.stdout:654: left oid 45 (ObjNum 280 snap 86 seq_num 280) 2024-08-17T16:57:34.858 INFO:tasks.rados.rados.0.smithi063.stdout:657: expect (ObjNum 207 snap 59 seq_num 207) 2024-08-17T16:57:35.027 INFO:tasks.rados.rados.0.smithi063.stdout:656: expect (ObjNum 73 snap 9 seq_num 73) 2024-08-17T16:57:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:34 smithi162 ceph-mon[34633]: osdmap e238: 8 total, 8 up, 8 in 2024-08-17T16:57:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:34 smithi162 ceph-mon[34633]: Standby manager daemon x restarted 2024-08-17T16:57:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:34 smithi162 ceph-mon[34633]: Standby manager daemon x started 2024-08-17T16:57:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:34 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/1165864336' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:57:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:34 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/1165864336' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:57:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:34 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/1165864336' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:57:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:34 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/1165864336' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:57:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:34 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:34 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:34 smithi162 ceph-mon[34633]: osdmap e239: 8 total, 8 up, 8 in 2024-08-17T16:57:35.169 INFO:tasks.rados.rados.0.smithi063.stdout:654: done (5 left) 2024-08-17T16:57:35.169 INFO:tasks.rados.rados.0.smithi063.stdout:656: done (4 left) 2024-08-17T16:57:35.169 INFO:tasks.rados.rados.0.smithi063.stdout:657: done (3 left) 2024-08-17T16:57:35.169 INFO:tasks.rados.rados.0.smithi063.stdout:658: done (2 left) 2024-08-17T16:57:35.169 INFO:tasks.rados.rados.0.smithi063.stdout:660: done (1 left) 2024-08-17T16:57:35.169 INFO:tasks.rados.rados.0.smithi063.stdout:662: done (0 left) 2024-08-17T16:57:35.169 INFO:tasks.rados.rados.0.smithi063.stdout:663: rollback oid 37 current snap is 86 2024-08-17T16:57:35.169 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 74 2024-08-17T16:57:35.169 INFO:tasks.rados.rados.0.smithi063.stdout:664: rollback oid 21 current snap is 86 2024-08-17T16:57:35.169 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 21 to 63 2024-08-17T16:57:35.169 INFO:tasks.rados.rados.0.smithi063.stdout:665: write oid 40 current snap is 86 2024-08-17T16:57:35.170 INFO:tasks.rados.rados.0.smithi063.stdout:665: seq_num 283 ranges {756381=400528,1910757=464751} 2024-08-17T16:57:35.175 INFO:tasks.rados.rados.0.smithi063.stdout:665: writing smithi06387500-40 from 756381 to 1156909 tid 1 2024-08-17T16:57:35.182 INFO:tasks.rados.rados.0.smithi063.stdout:665: writing smithi06387500-40 from 1910757 to 2375508 tid 2 2024-08-17T16:57:35.182 INFO:tasks.rados.rados.0.smithi063.stdout:666: write oid 42 current snap is 86 2024-08-17T16:57:35.182 INFO:tasks.rados.rados.0.smithi063.stdout:663: finishing rollback tid 0 to smithi06387500-37 2024-08-17T16:57:35.182 INFO:tasks.rados.rados.0.smithi063.stdout:664: finishing rollback tid 0 to smithi06387500-21 2024-08-17T16:57:35.182 INFO:tasks.rados.rados.0.smithi063.stdout:664: finishing rollback tid 1 to smithi06387500-21 2024-08-17T16:57:35.182 INFO:tasks.rados.rados.0.smithi063.stdout:663: finishing rollback tid 1 to smithi06387500-37 2024-08-17T16:57:35.182 INFO:tasks.rados.rados.0.smithi063.stdout:663: finishing rollback tid 2 to smithi06387500-37 2024-08-17T16:57:35.182 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 680 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-08-17T16:57:35.182 INFO:tasks.rados.rados.0.smithi063.stdout:664: finishing rollback tid 2 to smithi06387500-21 2024-08-17T16:57:35.182 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 437 (ObjNum 224 snap 62 seq_num 224) dirty exists 2024-08-17T16:57:35.183 INFO:tasks.rados.rados.0.smithi063.stdout:665: finishing write tid 1 to smithi06387500-40 2024-08-17T16:57:35.183 INFO:tasks.rados.rados.0.smithi063.stdout:666: seq_num 284 ranges {773188=475777,1969239=724762} 2024-08-17T16:57:35.188 INFO:tasks.rados.rados.0.smithi063.stdout:666: writing smithi06387500-42 from 773188 to 1248965 tid 1 2024-08-17T16:57:35.197 INFO:tasks.rados.rados.0.smithi063.stdout:666: writing smithi06387500-42 from 1969239 to 2694001 tid 2 2024-08-17T16:57:35.197 INFO:tasks.rados.rados.0.smithi063.stdout:663: done (3 left) 2024-08-17T16:57:35.197 INFO:tasks.rados.rados.0.smithi063.stdout:664: done (2 left) 2024-08-17T16:57:35.197 INFO:tasks.rados.rados.0.smithi063.stdout:667: snap_create 2024-08-17T16:57:35.197 INFO:tasks.rados.rados.0.smithi063.stdout:665: finishing write tid 2 to smithi06387500-40 2024-08-17T16:57:35.197 INFO:tasks.rados.rados.0.smithi063.stdout:665: finishing write tid 3 to smithi06387500-40 2024-08-17T16:57:35.197 INFO:tasks.rados.rados.0.smithi063.stdout:665: finishing write tid 4 to smithi06387500-40 2024-08-17T16:57:35.197 INFO:tasks.rados.rados.0.smithi063.stdout:665: oid 40 updating version 0 to 230 2024-08-17T16:57:35.197 INFO:tasks.rados.rados.0.smithi063.stdout:665: oid 40 updating version 230 to 231 2024-08-17T16:57:35.197 INFO:tasks.rados.rados.0.smithi063.stdout:665: oid 40 version 231 is already newer than 229 2024-08-17T16:57:35.197 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 231 (ObjNum 283 snap 86 seq_num 283) dirty exists 2024-08-17T16:57:35.197 INFO:tasks.rados.rados.0.smithi063.stdout:665: left oid 40 (ObjNum 283 snap 86 seq_num 283) 2024-08-17T16:57:35.198 INFO:tasks.rados.rados.0.smithi063.stdout:666: finishing write tid 1 to smithi06387500-42 2024-08-17T16:57:35.199 INFO:tasks.rados.rados.0.smithi063.stdout:666: finishing write tid 2 to smithi06387500-42 2024-08-17T16:57:35.199 INFO:tasks.rados.rados.0.smithi063.stdout:666: finishing write tid 3 to smithi06387500-42 2024-08-17T16:57:35.199 INFO:tasks.rados.rados.0.smithi063.stdout:666: finishing write tid 4 to smithi06387500-42 2024-08-17T16:57:35.199 INFO:tasks.rados.rados.0.smithi063.stdout:666: oid 42 updating version 0 to 441 2024-08-17T16:57:35.199 INFO:tasks.rados.rados.0.smithi063.stdout:666: oid 42 version 441 is already newer than 440 2024-08-17T16:57:35.199 INFO:tasks.rados.rados.0.smithi063.stdout:666: oid 42 version 441 is already newer than 439 2024-08-17T16:57:35.199 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 441 (ObjNum 284 snap 86 seq_num 284) dirty exists 2024-08-17T16:57:35.199 INFO:tasks.rados.rados.0.smithi063.stdout:666: left oid 42 (ObjNum 284 snap 86 seq_num 284) 2024-08-17T16:57:35.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[42680]: osdmap e238: 8 total, 8 up, 8 in 2024-08-17T16:57:35.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[42680]: Standby manager daemon x restarted 2024-08-17T16:57:35.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[42680]: Standby manager daemon x started 2024-08-17T16:57:35.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/1165864336' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:57:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/1165864336' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:57:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/1165864336' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:57:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/1165864336' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:57:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[42680]: osdmap e239: 8 total, 8 up, 8 in 2024-08-17T16:57:35.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[31063]: osdmap e238: 8 total, 8 up, 8 in 2024-08-17T16:57:35.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[31063]: Standby manager daemon x restarted 2024-08-17T16:57:35.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[31063]: Standby manager daemon x started 2024-08-17T16:57:35.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/1165864336' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:57:35.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/1165864336' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:57:35.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/1165864336' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:57:35.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[31063]: from='mgr.? 172.21.15.162:0/1165864336' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:57:35.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:35.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:35.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:34 smithi063 ceph-mon[31063]: osdmap e239: 8 total, 8 up, 8 in 2024-08-17T16:57:35.895 INFO:tasks.rados.rados.0.smithi063.stdout:665: done (2 left) 2024-08-17T16:57:35.895 INFO:tasks.rados.rados.0.smithi063.stdout:666: done (1 left) 2024-08-17T16:57:35.895 INFO:tasks.rados.rados.0.smithi063.stdout:667: done (0 left) 2024-08-17T16:57:35.895 INFO:tasks.rados.rados.0.smithi063.stdout:668: rollback oid 28 current snap is 87 2024-08-17T16:57:35.895 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 28 to 20 2024-08-17T16:57:35.895 INFO:tasks.rados.rados.0.smithi063.stdout:669: snap_remove snap 72 2024-08-17T16:57:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:35 smithi063 ceph-mon[31063]: mgrmap e33: y(active, since 26s), standbys: x 2024-08-17T16:57:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:35 smithi063 ceph-mon[31063]: pgmap v42: 89 pgs: 89 active+clean; 294 MiB data, 826 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:35 smithi063 ceph-mon[42680]: mgrmap e33: y(active, since 26s), standbys: x 2024-08-17T16:57:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:35 smithi063 ceph-mon[42680]: pgmap v42: 89 pgs: 89 active+clean; 294 MiB data, 826 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:36.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:35 smithi162 ceph-mon[34633]: mgrmap e33: y(active, since 26s), standbys: x 2024-08-17T16:57:36.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:35 smithi162 ceph-mon[34633]: pgmap v42: 89 pgs: 89 active+clean; 294 MiB data, 826 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:36.910 INFO:tasks.rados.rados.0.smithi063.stdout:669: done (1 left) 2024-08-17T16:57:36.911 INFO:tasks.rados.rados.0.smithi063.stdout:670: delete oid 29 current snap is 87 2024-08-17T16:57:36.911 INFO:tasks.rados.rados.0.smithi063.stdout:668: finishing rollback tid 0 to smithi06387500-28 2024-08-17T16:57:36.911 INFO:tasks.rados.rados.0.smithi063.stdout:668: finishing rollback tid 1 to smithi06387500-28 2024-08-17T16:57:36.911 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 209 (ObjNum 129 snap 0 seq_num 1807145072) dirty dne 2024-08-17T16:57:36.913 INFO:tasks.rados.rados.0.smithi063.stdout:668: done (1 left) 2024-08-17T16:57:36.914 INFO:tasks.rados.rados.0.smithi063.stdout:670: done (0 left) 2024-08-17T16:57:36.914 INFO:tasks.rados.rados.0.smithi063.stdout:671: read oid 5 snap -1 2024-08-17T16:57:36.914 INFO:tasks.rados.rados.0.smithi063.stdout:671: expect (ObjNum 195 snap 54 seq_num 195) 2024-08-17T16:57:36.914 INFO:tasks.rados.rados.0.smithi063.stdout:672: snap_remove snap 71 2024-08-17T16:57:37.177 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:36 smithi162 ceph-mon[34633]: osdmap e240: 8 total, 8 up, 8 in 2024-08-17T16:57:37.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:36 smithi063 ceph-mon[31063]: osdmap e240: 8 total, 8 up, 8 in 2024-08-17T16:57:37.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:36 smithi063 ceph-mon[42680]: osdmap e240: 8 total, 8 up, 8 in 2024-08-17T16:57:37.910 INFO:tasks.rados.rados.0.smithi063.stdout:672: done (1 left) 2024-08-17T16:57:37.910 INFO:tasks.rados.rados.0.smithi063.stdout:673: write oid 24 current snap is 87 2024-08-17T16:57:37.910 INFO:tasks.rados.rados.0.smithi063.stdout:673: seq_num 286 ranges {406802=626275,1686203=728431} 2024-08-17T16:57:37.916 INFO:tasks.rados.rados.0.smithi063.stdout:673: writing smithi06387500-24 from 406802 to 1033077 tid 1 2024-08-17T16:57:37.925 INFO:tasks.rados.rados.0.smithi063.stdout:673: writing smithi06387500-24 from 1686203 to 2414634 tid 2 2024-08-17T16:57:37.925 INFO:tasks.rados.rados.0.smithi063.stdout:674: snap_remove snap 80 2024-08-17T16:57:38.098 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: pgmap v44: 89 pgs: 89 active+clean; 287 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 577 KiB/s wr, 16 op/s 2024-08-17T16:57:38.098 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: osdmap e241: 8 total, 8 up, 8 in 2024-08-17T16:57:38.098 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.098 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.098 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:38.098 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:38.098 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.099 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:38.099 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.099 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.099 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T16:57:38.099 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-17T16:57:38.099 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T16:57:38.099 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-17T16:57:38.099 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.099 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.099 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.099 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.099 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.100 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.100 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.100 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.100 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.100 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.100 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.100 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.100 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.100 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.100 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.100 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.100 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: pgmap v44: 89 pgs: 89 active+clean; 287 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 577 KiB/s wr, 16 op/s 2024-08-17T16:57:38.101 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: osdmap e241: 8 total, 8 up, 8 in 2024-08-17T16:57:38.101 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.101 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.101 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:38.101 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:38.101 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.101 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:38.101 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.101 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.101 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T16:57:38.101 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-17T16:57:38.101 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T16:57:38.101 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-17T16:57:38.102 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.102 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.102 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.102 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.102 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.102 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.102 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.102 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.102 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.102 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.102 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.103 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.103 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.103 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.103 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.103 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: pgmap v44: 89 pgs: 89 active+clean; 287 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 577 KiB/s wr, 16 op/s 2024-08-17T16:57:38.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: osdmap e241: 8 total, 8 up, 8 in 2024-08-17T16:57:38.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:38.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:38.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:38.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-08-17T16:57:38.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-08-17T16:57:38.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-08-17T16:57:38.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-08-17T16:57:38.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:38.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:38.918 INFO:tasks.rados.rados.0.smithi063.stdout:671: expect (ObjNum 195 snap 54 seq_num 195) 2024-08-17T16:57:39.077 INFO:tasks.rados.rados.0.smithi063.stdout:673: finishing write tid 1 to smithi06387500-24 2024-08-17T16:57:39.077 INFO:tasks.rados.rados.0.smithi063.stdout:673: finishing write tid 2 to smithi06387500-24 2024-08-17T16:57:39.077 INFO:tasks.rados.rados.0.smithi063.stdout:671: done (2 left) 2024-08-17T16:57:39.077 INFO:tasks.rados.rados.0.smithi063.stdout:674: done (1 left) 2024-08-17T16:57:39.077 INFO:tasks.rados.rados.0.smithi063.stdout:675: write oid 43 current snap is 87 2024-08-17T16:57:39.077 INFO:tasks.rados.rados.0.smithi063.stdout:673: finishing write tid 3 to smithi06387500-24 2024-08-17T16:57:39.077 INFO:tasks.rados.rados.0.smithi063.stdout:675: seq_num 287 ranges {423609=701524,1628332=472089,2648987=174622} 2024-08-17T16:57:39.083 INFO:tasks.rados.rados.0.smithi063.stdout:675: writing smithi06387500-43 from 423609 to 1125133 tid 1 2024-08-17T16:57:39.088 INFO:tasks.rados.rados.0.smithi063.stdout:675: writing smithi06387500-43 from 1628332 to 2100421 tid 2 2024-08-17T16:57:39.093 INFO:tasks.rados.rados.0.smithi063.stdout:675: writing smithi06387500-43 from 2648987 to 2823609 tid 3 2024-08-17T16:57:39.093 INFO:tasks.rados.rados.0.smithi063.stdout:676: read oid 14 snap 74 2024-08-17T16:57:39.093 INFO:tasks.rados.rados.0.smithi063.stdout:676: expect deleted 2024-08-17T16:57:39.093 INFO:tasks.rados.rados.0.smithi063.stdout:677: read oid 6 snap 60 2024-08-17T16:57:39.093 INFO:tasks.rados.rados.0.smithi063.stdout:677: expect (ObjNum 178 snap 47 seq_num 178) 2024-08-17T16:57:39.093 INFO:tasks.rados.rados.0.smithi063.stdout:678: write oid 50 current snap is 87 2024-08-17T16:57:39.093 INFO:tasks.rados.rados.0.smithi063.stdout:673: finishing write tid 4 to smithi06387500-24 2024-08-17T16:57:39.093 INFO:tasks.rados.rados.0.smithi063.stdout:673: oid 24 updating version 0 to 255 2024-08-17T16:57:39.093 INFO:tasks.rados.rados.0.smithi063.stdout:673: oid 24 version 255 is already newer than 254 2024-08-17T16:57:39.093 INFO:tasks.rados.rados.0.smithi063.stdout:673: oid 24 version 255 is already newer than 253 2024-08-17T16:57:39.094 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 255 (ObjNum 286 snap 87 seq_num 286) dirty exists 2024-08-17T16:57:39.094 INFO:tasks.rados.rados.0.smithi063.stdout:673: left oid 24 (ObjNum 286 snap 87 seq_num 286) 2024-08-17T16:57:39.094 INFO:tasks.rados.rados.0.smithi063.stdout:675: finishing write tid 1 to smithi06387500-43 2024-08-17T16:57:39.094 INFO:tasks.rados.rados.0.smithi063.stdout:675: finishing write tid 2 to smithi06387500-43 2024-08-17T16:57:39.094 INFO:tasks.rados.rados.0.smithi063.stdout:678: seq_num 288 ranges {440416=776773,1686814=732100} 2024-08-17T16:57:39.099 INFO:tasks.rados.rados.0.smithi063.stdout:678: writing smithi06387500-50 from 440416 to 1217189 tid 1 2024-08-17T16:57:39.106 INFO:tasks.rados.rados.0.smithi063.stdout:678: writing smithi06387500-50 from 1686814 to 2418914 tid 2 2024-08-17T16:57:39.106 INFO:tasks.rados.rados.0.smithi063.stdout:673: done (4 left) 2024-08-17T16:57:39.106 INFO:tasks.rados.rados.0.smithi063.stdout:679: read oid 41 snap 81 2024-08-17T16:57:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:679: expect (ObjNum 73 snap 9 seq_num 73) 2024-08-17T16:57:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:680: snap_create 2024-08-17T16:57:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:675: finishing write tid 3 to smithi06387500-43 2024-08-17T16:57:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:675: finishing write tid 4 to smithi06387500-43 2024-08-17T16:57:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:675: finishing write tid 5 to smithi06387500-43 2024-08-17T16:57:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:675: oid 43 updating version 0 to 393 2024-08-17T16:57:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:675: oid 43 version 393 is already newer than 392 2024-08-17T16:57:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:675: oid 43 version 393 is already newer than 391 2024-08-17T16:57:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:675: oid 43 updating version 393 to 394 2024-08-17T16:57:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 394 (ObjNum 287 snap 87 seq_num 287) dirty exists 2024-08-17T16:57:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:675: left oid 43 (ObjNum 287 snap 87 seq_num 287) 2024-08-17T16:57:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:677: expect (ObjNum 178 snap 47 seq_num 178) 2024-08-17T16:57:39.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[42680]: Upgrade: Setting container_image for all mgr 2024-08-17T16:57:39.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[42680]: Upgrade: Setting container_image for all crash 2024-08-17T16:57:39.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[42680]: Upgrade: Setting container_image for all rgw 2024-08-17T16:57:39.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[42680]: Upgrade: Setting container_image for all rbd-mirror 2024-08-17T16:57:39.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[42680]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-17T16:57:39.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[42680]: Upgrade: Setting container_image for all ceph-exporter 2024-08-17T16:57:39.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[42680]: Upgrade: Setting container_image for all iscsi 2024-08-17T16:57:39.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[42680]: Upgrade: Setting container_image for all nfs 2024-08-17T16:57:39.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[42680]: Upgrade: Setting container_image for all nvmeof 2024-08-17T16:57:39.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[42680]: osdmap e242: 8 total, 8 up, 8 in 2024-08-17T16:57:39.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[31063]: Upgrade: Setting container_image for all mgr 2024-08-17T16:57:39.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[31063]: Upgrade: Setting container_image for all crash 2024-08-17T16:57:39.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[31063]: Upgrade: Setting container_image for all rgw 2024-08-17T16:57:39.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[31063]: Upgrade: Setting container_image for all rbd-mirror 2024-08-17T16:57:39.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[31063]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-17T16:57:39.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[31063]: Upgrade: Setting container_image for all ceph-exporter 2024-08-17T16:57:39.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[31063]: Upgrade: Setting container_image for all iscsi 2024-08-17T16:57:39.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[31063]: Upgrade: Setting container_image for all nfs 2024-08-17T16:57:39.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[31063]: Upgrade: Setting container_image for all nvmeof 2024-08-17T16:57:39.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:38 smithi063 ceph-mon[31063]: osdmap e242: 8 total, 8 up, 8 in 2024-08-17T16:57:39.283 INFO:tasks.rados.rados.0.smithi063.stdout:678: finishing write tid 1 to smithi06387500-50 2024-08-17T16:57:39.283 INFO:tasks.rados.rados.0.smithi063.stdout:678: finishing write tid 2 to smithi06387500-50 2024-08-17T16:57:39.283 INFO:tasks.rados.rados.0.smithi063.stdout:678: finishing write tid 3 to smithi06387500-50 2024-08-17T16:57:39.283 INFO:tasks.rados.rados.0.smithi063.stdout:678: finishing write tid 4 to smithi06387500-50 2024-08-17T16:57:39.283 INFO:tasks.rados.rados.0.smithi063.stdout:678: oid 50 updating version 0 to 460 2024-08-17T16:57:39.283 INFO:tasks.rados.rados.0.smithi063.stdout:678: oid 50 updating version 460 to 461 2024-08-17T16:57:39.283 INFO:tasks.rados.rados.0.smithi063.stdout:678: oid 50 version 461 is already newer than 459 2024-08-17T16:57:39.283 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 461 (ObjNum 288 snap 87 seq_num 288) dirty exists 2024-08-17T16:57:39.283 INFO:tasks.rados.rados.0.smithi063.stdout:678: left oid 50 (ObjNum 288 snap 87 seq_num 288) 2024-08-17T16:57:39.283 INFO:tasks.rados.rados.0.smithi063.stdout:679: expect (ObjNum 73 snap 9 seq_num 73) 2024-08-17T16:57:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:38 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all mgr 2024-08-17T16:57:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:38 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all crash 2024-08-17T16:57:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:38 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all rgw 2024-08-17T16:57:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:38 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all rbd-mirror 2024-08-17T16:57:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:38 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-17T16:57:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:38 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all ceph-exporter 2024-08-17T16:57:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:38 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all iscsi 2024-08-17T16:57:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:38 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all nfs 2024-08-17T16:57:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:38 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all nvmeof 2024-08-17T16:57:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:38 smithi162 ceph-mon[34633]: osdmap e242: 8 total, 8 up, 8 in 2024-08-17T16:57:39.585 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:39 smithi063 systemd[1]: Stopping Ceph node-exporter.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:57:39.858 INFO:tasks.rados.rados.0.smithi063.stdout:675: done (5 left) 2024-08-17T16:57:39.858 INFO:tasks.rados.rados.0.smithi063.stdout:676: done (4 left) 2024-08-17T16:57:39.858 INFO:tasks.rados.rados.0.smithi063.stdout:677: done (3 left) 2024-08-17T16:57:39.858 INFO:tasks.rados.rados.0.smithi063.stdout:678: done (2 left) 2024-08-17T16:57:39.858 INFO:tasks.rados.rados.0.smithi063.stdout:679: done (1 left) 2024-08-17T16:57:39.859 INFO:tasks.rados.rados.0.smithi063.stdout:680: done (0 left) 2024-08-17T16:57:39.859 INFO:tasks.rados.rados.0.smithi063.stdout:681: write oid 15 current snap is 88 2024-08-17T16:57:39.859 INFO:tasks.rados.rados.0.smithi063.stdout:681: seq_num 289 ranges {457223=568375,1461649=475758,2420186=437037} 2024-08-17T16:57:39.863 INFO:tasks.rados.rados.0.smithi063.stdout:681: writing smithi06387500-15 from 457223 to 1025598 tid 1 2024-08-17T16:57:39.869 INFO:tasks.rados.rados.0.smithi063.stdout:681: writing smithi06387500-15 from 1461649 to 1937407 tid 2 2024-08-17T16:57:39.874 INFO:tasks.rados.rados.0.smithi063.stdout:681: writing smithi06387500-15 from 2420186 to 2857223 tid 3 2024-08-17T16:57:39.874 INFO:tasks.rados.rados.0.smithi063.stdout:682: rollback oid 5 current snap is 88 2024-08-17T16:57:39.874 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 85 2024-08-17T16:57:39.875 INFO:tasks.rados.rados.0.smithi063.stdout:681: finishing write tid 1 to smithi06387500-15 2024-08-17T16:57:39.875 INFO:tasks.rados.rados.0.smithi063.stdout:681: finishing write tid 2 to smithi06387500-15 2024-08-17T16:57:39.875 INFO:tasks.rados.rados.0.smithi063.stdout:683: read oid 36 snap -1 2024-08-17T16:57:39.875 INFO:tasks.rados.rados.0.smithi063.stdout:683: expect (ObjNum 272 snap 84 seq_num 272) 2024-08-17T16:57:39.875 INFO:tasks.rados.rados.0.smithi063.stdout:684: snap_remove snap 74 2024-08-17T16:57:39.913 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:39 smithi063 podman[100020]: 2024-08-17 16:57:39.759885657 +0000 UTC m=+0.111982037 container died 8958df80ad2443d1ec5136e2e4a034986e37d86e549e4f59994a9fa0d17628db (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:39.913 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:57:39] "GET /metrics HTTP/1.1" 200 36212 "" "Prometheus/2.43.0" 2024-08-17T16:57:40.272 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:39 smithi063 ceph-mon[42680]: pgmap v47: 89 pgs: 89 active+clean; 287 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 575 KiB/s wr, 16 op/s 2024-08-17T16:57:40.273 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:39 smithi063 ceph-mon[42680]: osdmap e243: 8 total, 8 up, 8 in 2024-08-17T16:57:40.273 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:39 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:40.273 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:39 smithi063 ceph-mon[42680]: osdmap e244: 8 total, 8 up, 8 in 2024-08-17T16:57:40.273 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:39 smithi063 ceph-mon[31063]: pgmap v47: 89 pgs: 89 active+clean; 287 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 575 KiB/s wr, 16 op/s 2024-08-17T16:57:40.273 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:39 smithi063 ceph-mon[31063]: osdmap e243: 8 total, 8 up, 8 in 2024-08-17T16:57:40.273 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:39 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:40.273 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:39 smithi063 ceph-mon[31063]: osdmap e244: 8 total, 8 up, 8 in 2024-08-17T16:57:40.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:39 smithi162 ceph-mon[34633]: pgmap v47: 89 pgs: 89 active+clean; 287 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 575 KiB/s wr, 16 op/s 2024-08-17T16:57:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:39 smithi162 ceph-mon[34633]: osdmap e243: 8 total, 8 up, 8 in 2024-08-17T16:57:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:39 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:39 smithi162 ceph-mon[34633]: osdmap e244: 8 total, 8 up, 8 in 2024-08-17T16:57:40.529 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:40 smithi063 podman[100020]: 2024-08-17 16:57:40.272308038 +0000 UTC m=+0.624404416 container cleanup 8958df80ad2443d1ec5136e2e4a034986e37d86e549e4f59994a9fa0d17628db (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:40.529 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:40 smithi063 bash[100020]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a 2024-08-17T16:57:40.529 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:40 smithi063 podman[100040]: 2024-08-17 16:57:40.473787601 +0000 UTC m=+0.714032178 container remove 8958df80ad2443d1ec5136e2e4a034986e37d86e549e4f59994a9fa0d17628db (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:40.529 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:40 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-08-17T16:57:40.859 INFO:tasks.rados.rados.0.smithi063.stdout:681: finishing write tid 3 to smithi06387500-15 2024-08-17T16:57:40.860 INFO:tasks.rados.rados.0.smithi063.stdout:681: finishing write tid 4 to smithi06387500-15 2024-08-17T16:57:40.860 INFO:tasks.rados.rados.0.smithi063.stdout:681: finishing write tid 5 to smithi06387500-15 2024-08-17T16:57:40.860 INFO:tasks.rados.rados.0.smithi063.stdout:681: oid 15 updating version 0 to 258 2024-08-17T16:57:40.860 INFO:tasks.rados.rados.0.smithi063.stdout:681: oid 15 updating version 258 to 259 2024-08-17T16:57:40.860 INFO:tasks.rados.rados.0.smithi063.stdout:681: oid 15 updating version 259 to 260 2024-08-17T16:57:40.860 INFO:tasks.rados.rados.0.smithi063.stdout:681: oid 15 version 260 is already newer than 257 2024-08-17T16:57:40.860 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 260 (ObjNum 289 snap 88 seq_num 289) dirty exists 2024-08-17T16:57:40.860 INFO:tasks.rados.rados.0.smithi063.stdout:681: left oid 15 (ObjNum 289 snap 88 seq_num 289) 2024-08-17T16:57:40.860 INFO:tasks.rados.rados.0.smithi063.stdout:682: finishing rollback tid 0 to smithi06387500-5 2024-08-17T16:57:40.860 INFO:tasks.rados.rados.0.smithi063.stdout:682: finishing rollback tid 1 to smithi06387500-5 2024-08-17T16:57:40.860 INFO:tasks.rados.rados.0.smithi063.stdout:682: finishing rollback tid 2 to smithi06387500-5 2024-08-17T16:57:40.860 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 299 (ObjNum 195 snap 54 seq_num 195) dirty exists 2024-08-17T16:57:40.860 INFO:tasks.rados.rados.0.smithi063.stdout:683: expect (ObjNum 272 snap 84 seq_num 272) 2024-08-17T16:57:41.014 INFO:tasks.rados.rados.0.smithi063.stdout:681: done (3 left) 2024-08-17T16:57:41.014 INFO:tasks.rados.rados.0.smithi063.stdout:682: done (2 left) 2024-08-17T16:57:41.014 INFO:tasks.rados.rados.0.smithi063.stdout:683: done (1 left) 2024-08-17T16:57:41.015 INFO:tasks.rados.rados.0.smithi063.stdout:684: done (0 left) 2024-08-17T16:57:41.015 INFO:tasks.rados.rados.0.smithi063.stdout:685: read oid 45 snap -1 2024-08-17T16:57:41.015 INFO:tasks.rados.rados.0.smithi063.stdout:685: expect (ObjNum 280 snap 86 seq_num 280) 2024-08-17T16:57:41.015 INFO:tasks.rados.rados.0.smithi063.stdout:686: snap_create 2024-08-17T16:57:41.023 INFO:tasks.rados.rados.0.smithi063.stdout:685: expect (ObjNum 280 snap 86 seq_num 280) 2024-08-17T16:57:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:40 smithi063 ceph-mon[42680]: Upgrade: Updating node-exporter.a (1/2) 2024-08-17T16:57:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:40 smithi063 ceph-mon[42680]: Deploying daemon node-exporter.a on smithi063 2024-08-17T16:57:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:40 smithi063 ceph-mon[42680]: osdmap e245: 8 total, 8 up, 8 in 2024-08-17T16:57:41.029 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:40 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-08-17T16:57:41.029 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:40 smithi063 systemd[1]: Stopped Ceph node-exporter.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:57:41.029 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:40 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@node-exporter.a.service: Consumed 2.449s CPU time. 2024-08-17T16:57:41.029 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:40 smithi063 systemd[1]: Starting Ceph node-exporter.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:57:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:40 smithi063 ceph-mon[31063]: Upgrade: Updating node-exporter.a (1/2) 2024-08-17T16:57:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:40 smithi063 ceph-mon[31063]: Deploying daemon node-exporter.a on smithi063 2024-08-17T16:57:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:40 smithi063 ceph-mon[31063]: osdmap e245: 8 total, 8 up, 8 in 2024-08-17T16:57:41.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:40 smithi162 ceph-mon[34633]: Upgrade: Updating node-exporter.a (1/2) 2024-08-17T16:57:41.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:40 smithi162 ceph-mon[34633]: Deploying daemon node-exporter.a on smithi063 2024-08-17T16:57:41.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:40 smithi162 ceph-mon[34633]: osdmap e245: 8 total, 8 up, 8 in 2024-08-17T16:57:41.445 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:41 smithi063 bash[100172]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-08-17T16:57:41.779 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:41 smithi063 bash[100172]: Getting image source signatures 2024-08-17T16:57:41.779 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:41 smithi063 bash[100172]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-08-17T16:57:41.779 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:41 smithi063 bash[100172]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-08-17T16:57:41.779 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:41 smithi063 bash[100172]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-08-17T16:57:41.925 INFO:tasks.rados.rados.0.smithi063.stdout:685: done (1 left) 2024-08-17T16:57:41.925 INFO:tasks.rados.rados.0.smithi063.stdout:686: done (0 left) 2024-08-17T16:57:41.925 INFO:tasks.rados.rados.0.smithi063.stdout:687: write oid 27 current snap is 89 2024-08-17T16:57:41.925 INFO:tasks.rados.rados.0.smithi063.stdout:687: seq_num 290 ranges {474030=643624,1520131=735769} 2024-08-17T16:57:41.931 INFO:tasks.rados.rados.0.smithi063.stdout:687: writing smithi06387500-27 from 474030 to 1117654 tid 1 2024-08-17T16:57:41.938 INFO:tasks.rados.rados.0.smithi063.stdout:687: writing smithi06387500-27 from 1520131 to 2255900 tid 2 2024-08-17T16:57:41.938 INFO:tasks.rados.rados.0.smithi063.stdout:688: read oid 16 snap -1 2024-08-17T16:57:41.938 INFO:tasks.rados.rados.0.smithi063.stdout:688: expect deleted 2024-08-17T16:57:41.938 INFO:tasks.rados.rados.0.smithi063.stdout:689: snap_remove snap 81 2024-08-17T16:57:42.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:41 smithi063 ceph-mon[42680]: pgmap v51: 89 pgs: 89 active+clean; 289 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-08-17T16:57:42.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:41 smithi063 ceph-mon[31063]: pgmap v51: 89 pgs: 89 active+clean; 289 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-08-17T16:57:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:41 smithi162 ceph-mon[34633]: pgmap v51: 89 pgs: 89 active+clean; 289 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-08-17T16:57:42.942 INFO:tasks.rados.rados.0.smithi063.stdout:689: done (2 left) 2024-08-17T16:57:42.942 INFO:tasks.rados.rados.0.smithi063.stdout:690: rollback oid 3 current snap is 89 2024-08-17T16:57:42.942 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 3 to 20 2024-08-17T16:57:42.942 INFO:tasks.rados.rados.0.smithi063.stdout:687: finishing write tid 1 to smithi06387500-27 2024-08-17T16:57:42.942 INFO:tasks.rados.rados.0.smithi063.stdout:687: finishing write tid 2 to smithi06387500-27 2024-08-17T16:57:42.942 INFO:tasks.rados.rados.0.smithi063.stdout:687: finishing write tid 3 to smithi06387500-27 2024-08-17T16:57:42.942 INFO:tasks.rados.rados.0.smithi063.stdout:687: finishing write tid 4 to smithi06387500-27 2024-08-17T16:57:42.942 INFO:tasks.rados.rados.0.smithi063.stdout:687: oid 27 updating version 0 to 106 2024-08-17T16:57:42.942 INFO:tasks.rados.rados.0.smithi063.stdout:687: oid 27 version 106 is already newer than 105 2024-08-17T16:57:42.943 INFO:tasks.rados.rados.0.smithi063.stdout:687: oid 27 updating version 106 to 107 2024-08-17T16:57:42.943 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 107 (ObjNum 290 snap 89 seq_num 290) dirty exists 2024-08-17T16:57:42.943 INFO:tasks.rados.rados.0.smithi063.stdout:687: left oid 27 (ObjNum 290 snap 89 seq_num 290) 2024-08-17T16:57:42.943 INFO:tasks.rados.rados.0.smithi063.stdout:687: done (2 left) 2024-08-17T16:57:42.943 INFO:tasks.rados.rados.0.smithi063.stdout:688: done (1 left) 2024-08-17T16:57:42.943 INFO:tasks.rados.rados.0.smithi063.stdout:691: snap_create 2024-08-17T16:57:42.945 INFO:tasks.rados.rados.0.smithi063.stdout:690: finishing rollback tid 1 to smithi06387500-3 2024-08-17T16:57:42.946 INFO:tasks.rados.rados.0.smithi063.stdout:690: finishing rollback tid 2 to smithi06387500-3 2024-08-17T16:57:42.946 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 209 (ObjNum 70 snap 9 seq_num 70) dirty exists 2024-08-17T16:57:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:42 smithi063 ceph-mon[42680]: osdmap e246: 8 total, 8 up, 8 in 2024-08-17T16:57:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:42 smithi063 ceph-mon[42680]: pgmap v53: 89 pgs: 89 active+clean; 289 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-08-17T16:57:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:42 smithi063 ceph-mon[31063]: osdmap e246: 8 total, 8 up, 8 in 2024-08-17T16:57:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:42 smithi063 ceph-mon[31063]: pgmap v53: 89 pgs: 89 active+clean; 289 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-08-17T16:57:43.030 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:42 smithi063 bash[100172]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-08-17T16:57:43.030 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:42 smithi063 bash[100172]: Writing manifest to image destination 2024-08-17T16:57:43.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:42 smithi162 ceph-mon[34633]: osdmap e246: 8 total, 8 up, 8 in 2024-08-17T16:57:43.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:42 smithi162 ceph-mon[34633]: pgmap v53: 89 pgs: 89 active+clean; 289 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 19 op/s 2024-08-17T16:57:43.481 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 podman[100172]: 2024-08-17 16:57:43.226862324 +0000 UTC m=+2.190709104 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-08-17T16:57:43.481 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 podman[100172]: 2024-08-17 16:57:43.366928481 +0000 UTC m=+2.330775250 container create 34f55f5de2aa20fa1a42390df28c89ee86e5cd1a98e425c12246e8eee0c6222e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:43.938 INFO:tasks.rados.rados.0.smithi063.stdout:690: done (1 left) 2024-08-17T16:57:43.938 INFO:tasks.rados.rados.0.smithi063.stdout:691: done (0 left) 2024-08-17T16:57:43.939 INFO:tasks.rados.rados.0.smithi063.stdout:692: read oid 17 snap 75 2024-08-17T16:57:43.939 INFO:tasks.rados.rados.0.smithi063.stdout:692: expect (ObjNum 191 snap 50 seq_num 191) 2024-08-17T16:57:43.939 INFO:tasks.rados.rados.0.smithi063.stdout:693: rollback oid 3 current snap is 90 2024-08-17T16:57:43.939 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 3 to 85 2024-08-17T16:57:43.939 INFO:tasks.rados.rados.0.smithi063.stdout:694: rollback oid 39 current snap is 90 2024-08-17T16:57:43.939 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 39 to 75 2024-08-17T16:57:43.939 INFO:tasks.rados.rados.0.smithi063.stdout:695: write oid 37 current snap is 90 2024-08-17T16:57:43.939 INFO:tasks.rados.rados.0.smithi063.stdout:695: seq_num 291 ranges {490837=718873,1862260=479427,2875032=15805} 2024-08-17T16:57:43.943 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 podman[100172]: 2024-08-17 16:57:43.610684592 +0000 UTC m=+2.574531390 container init 34f55f5de2aa20fa1a42390df28c89ee86e5cd1a98e425c12246e8eee0c6222e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:43.943 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 podman[100172]: 2024-08-17 16:57:43.617047661 +0000 UTC m=+2.580894431 container start 34f55f5de2aa20fa1a42390df28c89ee86e5cd1a98e425c12246e8eee0c6222e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T16:57:43.943 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.620Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-08-17T16:57:43.944 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.620Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-08-17T16:57:43.944 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.621Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T16:57:43.944 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.621Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T16:57:43.944 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.621Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T16:57:43.944 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.621Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T16:57:43.944 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T16:57:43.944 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T16:57:43.944 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T16:57:43.944 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T16:57:43.944 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T16:57:43.944 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T16:57:43.944 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T16:57:43.945 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T16:57:43.945 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T16:57:43.945 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T16:57:43.945 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T16:57:43.945 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T16:57:43.945 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T16:57:43.945 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T16:57:43.945 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T16:57:43.945 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T16:57:43.945 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T16:57:43.945 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T16:57:43.945 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T16:57:43.945 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T16:57:43.946 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T16:57:43.946 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T16:57:43.946 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T16:57:43.946 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T16:57:43.946 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T16:57:43.946 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T16:57:43.946 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T16:57:43.946 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T16:57:43.946 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T16:57:43.946 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T16:57:43.946 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T16:57:43.947 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T16:57:43.947 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T16:57:43.947 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T16:57:43.947 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T16:57:43.947 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T16:57:43.947 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T16:57:43.947 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T16:57:43.947 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T16:57:43.947 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T16:57:43.947 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T16:57:43.947 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T16:57:43.947 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T16:57:43.948 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T16:57:43.948 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T16:57:43.948 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-08-17T16:57:43.948 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a[100232]: ts=2024-08-17T16:57:43.622Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T16:57:43.948 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 bash[100172]: 34f55f5de2aa20fa1a42390df28c89ee86e5cd1a98e425c12246e8eee0c6222e 2024-08-17T16:57:43.948 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 16:57:43 smithi063 systemd[1]: Started Ceph node-exporter.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:57:43.948 INFO:tasks.rados.rados.0.smithi063.stdout:695: writing smithi06387500-37 from 490837 to 1209710 tid 1 2024-08-17T16:57:43.954 INFO:tasks.rados.rados.0.smithi063.stdout:695: writing smithi06387500-37 from 1862260 to 2341687 tid 2 2024-08-17T16:57:43.957 INFO:tasks.rados.rados.0.smithi063.stdout:695: writing smithi06387500-37 from 2875032 to 2890837 tid 3 2024-08-17T16:57:43.958 INFO:tasks.rados.rados.0.smithi063.stdout:696: snap_create 2024-08-17T16:57:43.958 INFO:tasks.rados.rados.0.smithi063.stdout:694: finishing rollback tid 1 to smithi06387500-39 2024-08-17T16:57:43.958 INFO:tasks.rados.rados.0.smithi063.stdout:694: finishing rollback tid 2 to smithi06387500-39 2024-08-17T16:57:43.958 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 155 (ObjNum 190 snap 50 seq_num 190) dirty exists 2024-08-17T16:57:43.958 INFO:tasks.rados.rados.0.smithi063.stdout:695: finishing write tid 1 to smithi06387500-37 2024-08-17T16:57:43.958 INFO:tasks.rados.rados.0.smithi063.stdout:692: expect (ObjNum 191 snap 50 seq_num 191) 2024-08-17T16:57:44.134 INFO:tasks.rados.rados.0.smithi063.stdout:693: finishing rollback tid 0 to smithi06387500-3 2024-08-17T16:57:44.135 INFO:tasks.rados.rados.0.smithi063.stdout:693: finishing rollback tid 1 to smithi06387500-3 2024-08-17T16:57:44.135 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 212 (ObjNum 81 snap 0 seq_num 1) dirty dne 2024-08-17T16:57:44.135 INFO:tasks.rados.rados.0.smithi063.stdout:695: finishing write tid 2 to smithi06387500-37 2024-08-17T16:57:44.135 INFO:tasks.rados.rados.0.smithi063.stdout:695: finishing write tid 3 to smithi06387500-37 2024-08-17T16:57:44.135 INFO:tasks.rados.rados.0.smithi063.stdout:695: finishing write tid 4 to smithi06387500-37 2024-08-17T16:57:44.135 INFO:tasks.rados.rados.0.smithi063.stdout:695: finishing write tid 5 to smithi06387500-37 2024-08-17T16:57:44.135 INFO:tasks.rados.rados.0.smithi063.stdout:695: oid 37 updating version 0 to 721 2024-08-17T16:57:44.135 INFO:tasks.rados.rados.0.smithi063.stdout:695: oid 37 version 721 is already newer than 719 2024-08-17T16:57:44.135 INFO:tasks.rados.rados.0.smithi063.stdout:695: oid 37 version 721 is already newer than 720 2024-08-17T16:57:44.135 INFO:tasks.rados.rados.0.smithi063.stdout:695: oid 37 version 721 is already newer than 718 2024-08-17T16:57:44.135 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 721 (ObjNum 291 snap 90 seq_num 291) dirty exists 2024-08-17T16:57:44.135 INFO:tasks.rados.rados.0.smithi063.stdout:695: left oid 37 (ObjNum 291 snap 90 seq_num 291) 2024-08-17T16:57:44.194 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-mon[42680]: osdmap e247: 8 total, 8 up, 8 in 2024-08-17T16:57:44.194 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:44.194 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:44.194 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-mon[31063]: osdmap e247: 8 total, 8 up, 8 in 2024-08-17T16:57:44.194 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:44.194 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:43 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:44.272 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:43 smithi162 ceph-mon[34633]: osdmap e247: 8 total, 8 up, 8 in 2024-08-17T16:57:44.272 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:43 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:44.272 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:43 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:44.862 INFO:tasks.rados.rados.0.smithi063.stdout:692: done (4 left) 2024-08-17T16:57:44.862 INFO:tasks.rados.rados.0.smithi063.stdout:693: done (3 left) 2024-08-17T16:57:44.862 INFO:tasks.rados.rados.0.smithi063.stdout:694: done (2 left) 2024-08-17T16:57:44.862 INFO:tasks.rados.rados.0.smithi063.stdout:695: done (1 left) 2024-08-17T16:57:44.862 INFO:tasks.rados.rados.0.smithi063.stdout:696: done (0 left) 2024-08-17T16:57:44.862 INFO:tasks.rados.rados.0.smithi063.stdout:697: rollback oid 45 current snap is 91 2024-08-17T16:57:44.862 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 45 to 85 2024-08-17T16:57:44.862 INFO:tasks.rados.rados.0.smithi063.stdout:698: read oid 9 snap -1 2024-08-17T16:57:44.862 INFO:tasks.rados.rados.0.smithi063.stdout:698: expect (ObjNum 269 snap 84 seq_num 269) 2024-08-17T16:57:44.862 INFO:tasks.rados.rados.0.smithi063.stdout:699: write oid 10 current snap is 91 2024-08-17T16:57:44.863 INFO:tasks.rados.rados.0.smithi063.stdout:699: seq_num 292 ranges {507644=794122,1920742=623085} 2024-08-17T16:57:44.868 INFO:tasks.rados.rados.0.smithi063.stdout:699: writing smithi06387500-10 from 507644 to 1301766 tid 1 2024-08-17T16:57:44.876 INFO:tasks.rados.rados.0.smithi063.stdout:699: writing smithi06387500-10 from 1920742 to 2543827 tid 2 2024-08-17T16:57:44.876 INFO:tasks.rados.rados.0.smithi063.stdout:700: write oid 46 current snap is 91 2024-08-17T16:57:44.876 INFO:tasks.rados.rados.0.smithi063.stdout:700: seq_num 293 ranges {524451=469371,1579224=483096,2529878=394573} 2024-08-17T16:57:44.881 INFO:tasks.rados.rados.0.smithi063.stdout:700: writing smithi06387500-46 from 524451 to 993822 tid 1 2024-08-17T16:57:44.887 INFO:tasks.rados.rados.0.smithi063.stdout:700: writing smithi06387500-46 from 1579224 to 2062320 tid 2 2024-08-17T16:57:44.893 INFO:tasks.rados.rados.0.smithi063.stdout:700: writing smithi06387500-46 from 2529878 to 2924451 tid 3 2024-08-17T16:57:44.893 INFO:tasks.rados.rados.0.smithi063.stdout:701: write oid 42 current snap is 91 2024-08-17T16:57:44.893 INFO:tasks.rados.rados.0.smithi063.stdout:701: seq_num 294 ranges {541258=544620,1637706=626754,2840948=100310} 2024-08-17T16:57:44.899 INFO:tasks.rados.rados.0.smithi063.stdout:701: writing smithi06387500-42 from 541258 to 1085878 tid 1 2024-08-17T16:57:44.908 INFO:tasks.rados.rados.0.smithi063.stdout:701: writing smithi06387500-42 from 1637706 to 2264460 tid 2 2024-08-17T16:57:44.912 INFO:tasks.rados.rados.0.smithi063.stdout:701: writing smithi06387500-42 from 2840948 to 2941258 tid 3 2024-08-17T16:57:44.912 INFO:tasks.rados.rados.0.smithi063.stdout:702: write oid 28 current snap is 91 2024-08-17T16:57:44.912 INFO:tasks.rados.rados.0.smithi063.stdout:697: finishing rollback tid 0 to smithi06387500-45 2024-08-17T16:57:44.912 INFO:tasks.rados.rados.0.smithi063.stdout:699: finishing write tid 1 to smithi06387500-10 2024-08-17T16:57:44.912 INFO:tasks.rados.rados.0.smithi063.stdout:697: finishing rollback tid 1 to smithi06387500-45 2024-08-17T16:57:44.912 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 158 (ObjNum 129 snap 0 seq_num 1806922928) dirty dne 2024-08-17T16:57:44.912 INFO:tasks.rados.rados.0.smithi063.stdout:699: finishing write tid 2 to smithi06387500-10 2024-08-17T16:57:44.913 INFO:tasks.rados.rados.0.smithi063.stdout:699: finishing write tid 3 to smithi06387500-10 2024-08-17T16:57:44.913 INFO:tasks.rados.rados.0.smithi063.stdout:699: finishing write tid 4 to smithi06387500-10 2024-08-17T16:57:44.913 INFO:tasks.rados.rados.0.smithi063.stdout:699: oid 10 updating version 0 to 249 2024-08-17T16:57:44.913 INFO:tasks.rados.rados.0.smithi063.stdout:699: oid 10 updating version 249 to 250 2024-08-17T16:57:44.913 INFO:tasks.rados.rados.0.smithi063.stdout:699: oid 10 version 250 is already newer than 248 2024-08-17T16:57:44.913 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 250 (ObjNum 292 snap 91 seq_num 292) dirty exists 2024-08-17T16:57:44.913 INFO:tasks.rados.rados.0.smithi063.stdout:699: left oid 10 (ObjNum 292 snap 91 seq_num 292) 2024-08-17T16:57:44.913 INFO:tasks.rados.rados.0.smithi063.stdout:700: finishing write tid 1 to smithi06387500-46 2024-08-17T16:57:44.913 INFO:tasks.rados.rados.0.smithi063.stdout:702: seq_num 295 ranges {558065=619869,1579835=486765,2468371=489694} 2024-08-17T16:57:44.918 INFO:tasks.rados.rados.0.smithi063.stdout:702: writing smithi06387500-28 from 558065 to 1177934 tid 1 2024-08-17T16:57:44.923 INFO:tasks.rados.rados.0.smithi063.stdout:702: writing smithi06387500-28 from 1579835 to 2066600 tid 2 2024-08-17T16:57:44.929 INFO:tasks.rados.rados.0.smithi063.stdout:702: writing smithi06387500-28 from 2468371 to 2958065 tid 3 2024-08-17T16:57:44.929 INFO:tasks.rados.rados.0.smithi063.stdout:697: done (5 left) 2024-08-17T16:57:44.929 INFO:tasks.rados.rados.0.smithi063.stdout:699: done (4 left) 2024-08-17T16:57:44.929 INFO:tasks.rados.rados.0.smithi063.stdout:703: read oid 32 snap -1 2024-08-17T16:57:44.929 INFO:tasks.rados.rados.0.smithi063.stdout:703: expect deleted 2024-08-17T16:57:44.929 INFO:tasks.rados.rados.0.smithi063.stdout:704: write oid 5 current snap is 91 2024-08-17T16:57:44.929 INFO:tasks.rados.rados.0.smithi063.stdout:700: finishing write tid 2 to smithi06387500-46 2024-08-17T16:57:44.929 INFO:tasks.rados.rados.0.smithi063.stdout:700: finishing write tid 3 to smithi06387500-46 2024-08-17T16:57:44.929 INFO:tasks.rados.rados.0.smithi063.stdout:700: finishing write tid 4 to smithi06387500-46 2024-08-17T16:57:44.929 INFO:tasks.rados.rados.0.smithi063.stdout:704: seq_num 296 ranges {574872=695118,2038317=630423} 2024-08-17T16:57:44.935 INFO:tasks.rados.rados.0.smithi063.stdout:704: writing smithi06387500-5 from 574872 to 1269990 tid 1 2024-08-17T16:57:44.943 INFO:tasks.rados.rados.0.smithi063.stdout:704: writing smithi06387500-5 from 2038317 to 2668740 tid 2 2024-08-17T16:57:44.943 INFO:tasks.rados.rados.0.smithi063.stdout:705: read oid 39 snap -1 2024-08-17T16:57:44.944 INFO:tasks.rados.rados.0.smithi063.stdout:705: expect (ObjNum 190 snap 50 seq_num 190) 2024-08-17T16:57:44.944 INFO:tasks.rados.rados.0.smithi063.stdout:706: write oid 13 current snap is 91 2024-08-17T16:57:44.944 INFO:tasks.rados.rados.0.smithi063.stdout:700: finishing write tid 5 to smithi06387500-46 2024-08-17T16:57:44.944 INFO:tasks.rados.rados.0.smithi063.stdout:700: oid 46 updating version 0 to 264 2024-08-17T16:57:44.944 INFO:tasks.rados.rados.0.smithi063.stdout:700: oid 46 updating version 264 to 266 2024-08-17T16:57:44.944 INFO:tasks.rados.rados.0.smithi063.stdout:700: oid 46 updating version 266 to 267 2024-08-17T16:57:44.944 INFO:tasks.rados.rados.0.smithi063.stdout:700: oid 46 version 267 is already newer than 265 2024-08-17T16:57:44.944 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 267 (ObjNum 293 snap 91 seq_num 293) dirty exists 2024-08-17T16:57:44.944 INFO:tasks.rados.rados.0.smithi063.stdout:700: left oid 46 (ObjNum 293 snap 91 seq_num 293) 2024-08-17T16:57:44.944 INFO:tasks.rados.rados.0.smithi063.stdout:701: finishing write tid 1 to smithi06387500-42 2024-08-17T16:57:44.944 INFO:tasks.rados.rados.0.smithi063.stdout:698: expect (ObjNum 269 snap 84 seq_num 269) 2024-08-17T16:57:45.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:44 smithi162 ceph-mon[34633]: osdmap e248: 8 total, 8 up, 8 in 2024-08-17T16:57:45.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:44 smithi162 ceph-mon[34633]: osdmap e249: 8 total, 8 up, 8 in 2024-08-17T16:57:45.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:44 smithi162 ceph-mon[34633]: pgmap v57: 89 pgs: 89 active+clean; 289 MiB data, 831 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:45.103 INFO:tasks.rados.rados.0.smithi063.stdout:701: finishing write tid 2 to smithi06387500-42 2024-08-17T16:57:45.104 INFO:tasks.rados.rados.0.smithi063.stdout:701: finishing write tid 3 to smithi06387500-42 2024-08-17T16:57:45.104 INFO:tasks.rados.rados.0.smithi063.stdout:701: finishing write tid 4 to smithi06387500-42 2024-08-17T16:57:45.104 INFO:tasks.rados.rados.0.smithi063.stdout:701: finishing write tid 5 to smithi06387500-42 2024-08-17T16:57:45.104 INFO:tasks.rados.rados.0.smithi063.stdout:701: oid 42 updating version 0 to 476 2024-08-17T16:57:45.104 INFO:tasks.rados.rados.0.smithi063.stdout:701: oid 42 version 476 is already newer than 475 2024-08-17T16:57:45.104 INFO:tasks.rados.rados.0.smithi063.stdout:701: oid 42 updating version 476 to 477 2024-08-17T16:57:45.104 INFO:tasks.rados.rados.0.smithi063.stdout:701: oid 42 updating version 477 to 478 2024-08-17T16:57:45.104 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 478 (ObjNum 294 snap 91 seq_num 294) dirty exists 2024-08-17T16:57:45.104 INFO:tasks.rados.rados.0.smithi063.stdout:701: left oid 42 (ObjNum 294 snap 91 seq_num 294) 2024-08-17T16:57:45.104 INFO:tasks.rados.rados.0.smithi063.stdout:702: finishing write tid 1 to smithi06387500-28 2024-08-17T16:57:45.104 INFO:tasks.rados.rados.0.smithi063.stdout:702: finishing write tid 2 to smithi06387500-28 2024-08-17T16:57:45.104 INFO:tasks.rados.rados.0.smithi063.stdout:702: finishing write tid 3 to smithi06387500-28 2024-08-17T16:57:45.105 INFO:tasks.rados.rados.0.smithi063.stdout:702: finishing write tid 4 to smithi06387500-28 2024-08-17T16:57:45.105 INFO:tasks.rados.rados.0.smithi063.stdout:702: finishing write tid 5 to smithi06387500-28 2024-08-17T16:57:45.105 INFO:tasks.rados.rados.0.smithi063.stdout:702: oid 28 updating version 0 to 227 2024-08-17T16:57:45.105 INFO:tasks.rados.rados.0.smithi063.stdout:702: oid 28 version 227 is already newer than 224 2024-08-17T16:57:45.105 INFO:tasks.rados.rados.0.smithi063.stdout:702: oid 28 version 227 is already newer than 226 2024-08-17T16:57:45.105 INFO:tasks.rados.rados.0.smithi063.stdout:702: oid 28 version 227 is already newer than 225 2024-08-17T16:57:45.105 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 227 (ObjNum 295 snap 91 seq_num 295) dirty exists 2024-08-17T16:57:45.105 INFO:tasks.rados.rados.0.smithi063.stdout:702: left oid 28 (ObjNum 295 snap 91 seq_num 295) 2024-08-17T16:57:45.105 INFO:tasks.rados.rados.0.smithi063.stdout:704: finishing write tid 1 to smithi06387500-5 2024-08-17T16:57:45.105 INFO:tasks.rados.rados.0.smithi063.stdout:704: finishing write tid 2 to smithi06387500-5 2024-08-17T16:57:45.105 INFO:tasks.rados.rados.0.smithi063.stdout:704: finishing write tid 3 to smithi06387500-5 2024-08-17T16:57:45.105 INFO:tasks.rados.rados.0.smithi063.stdout:706: seq_num 297 ranges {591679=486720,1813152=490434} 2024-08-17T16:57:45.109 INFO:tasks.rados.rados.0.smithi063.stdout:706: writing smithi06387500-13 from 591679 to 1078399 tid 1 2024-08-17T16:57:45.116 INFO:tasks.rados.rados.0.smithi063.stdout:706: writing smithi06387500-13 from 1813152 to 2303586 tid 2 2024-08-17T16:57:45.116 INFO:tasks.rados.rados.0.smithi063.stdout:698: done (7 left) 2024-08-17T16:57:45.116 INFO:tasks.rados.rados.0.smithi063.stdout:700: done (6 left) 2024-08-17T16:57:45.116 INFO:tasks.rados.rados.0.smithi063.stdout:701: done (5 left) 2024-08-17T16:57:45.117 INFO:tasks.rados.rados.0.smithi063.stdout:702: done (4 left) 2024-08-17T16:57:45.117 INFO:tasks.rados.rados.0.smithi063.stdout:703: done (3 left) 2024-08-17T16:57:45.117 INFO:tasks.rados.rados.0.smithi063.stdout:707: write oid 41 current snap is 91 2024-08-17T16:57:45.117 INFO:tasks.rados.rados.0.smithi063.stdout:704: finishing write tid 4 to smithi06387500-5 2024-08-17T16:57:45.117 INFO:tasks.rados.rados.0.smithi063.stdout:704: oid 5 updating version 0 to 312 2024-08-17T16:57:45.117 INFO:tasks.rados.rados.0.smithi063.stdout:704: oid 5 version 312 is already newer than 311 2024-08-17T16:57:45.117 INFO:tasks.rados.rados.0.smithi063.stdout:704: oid 5 updating version 312 to 313 2024-08-17T16:57:45.117 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 313 (ObjNum 296 snap 91 seq_num 296) dirty exists 2024-08-17T16:57:45.117 INFO:tasks.rados.rados.0.smithi063.stdout:704: left oid 5 (ObjNum 296 snap 91 seq_num 296) 2024-08-17T16:57:45.117 INFO:tasks.rados.rados.0.smithi063.stdout:705: expect (ObjNum 190 snap 50 seq_num 190) 2024-08-17T16:57:45.262 INFO:tasks.rados.rados.0.smithi063.stdout:706: finishing write tid 1 to smithi06387500-13 2024-08-17T16:57:45.263 INFO:tasks.rados.rados.0.smithi063.stdout:706: finishing write tid 2 to smithi06387500-13 2024-08-17T16:57:45.263 INFO:tasks.rados.rados.0.smithi063.stdout:706: finishing write tid 3 to smithi06387500-13 2024-08-17T16:57:45.263 INFO:tasks.rados.rados.0.smithi063.stdout:706: finishing write tid 4 to smithi06387500-13 2024-08-17T16:57:45.263 INFO:tasks.rados.rados.0.smithi063.stdout:706: oid 13 updating version 0 to 317 2024-08-17T16:57:45.263 INFO:tasks.rados.rados.0.smithi063.stdout:706: oid 13 version 317 is already newer than 315 2024-08-17T16:57:45.263 INFO:tasks.rados.rados.0.smithi063.stdout:706: oid 13 version 317 is already newer than 316 2024-08-17T16:57:45.263 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 317 (ObjNum 297 snap 91 seq_num 297) dirty exists 2024-08-17T16:57:45.263 INFO:tasks.rados.rados.0.smithi063.stdout:706: left oid 13 (ObjNum 297 snap 91 seq_num 297) 2024-08-17T16:57:45.263 INFO:tasks.rados.rados.0.smithi063.stdout:707: seq_num 298 ranges {608486=561969,1871634=634092,2950640=57846} 2024-08-17T16:57:45.268 INFO:tasks.rados.rados.0.smithi063.stdout:707: writing smithi06387500-41 from 608486 to 1170455 tid 1 2024-08-17T16:57:45.276 INFO:tasks.rados.rados.0.smithi063.stdout:707: writing smithi06387500-41 from 1871634 to 2505726 tid 2 2024-08-17T16:57:45.279 INFO:tasks.rados.rados.0.smithi063.stdout:707: writing smithi06387500-41 from 2950640 to 3008486 tid 3 2024-08-17T16:57:45.279 INFO:tasks.rados.rados.0.smithi063.stdout:704: done (3 left) 2024-08-17T16:57:45.279 INFO:tasks.rados.rados.0.smithi063.stdout:705: done (2 left) 2024-08-17T16:57:45.279 INFO:tasks.rados.rados.0.smithi063.stdout:706: done (1 left) 2024-08-17T16:57:45.279 INFO:tasks.rados.rados.0.smithi063.stdout:708: read oid 28 snap -1 2024-08-17T16:57:45.279 INFO:tasks.rados.rados.0.smithi063.stdout:708: expect (ObjNum 295 snap 91 seq_num 295) 2024-08-17T16:57:45.279 INFO:tasks.rados.rados.0.smithi063.stdout:709: delete oid 17 current snap is 91 2024-08-17T16:57:45.280 INFO:tasks.rados.rados.0.smithi063.stdout:707: finishing write tid 1 to smithi06387500-41 2024-08-17T16:57:45.280 INFO:tasks.rados.rados.0.smithi063.stdout:707: finishing write tid 2 to smithi06387500-41 2024-08-17T16:57:45.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:44 smithi063 ceph-mon[31063]: osdmap e248: 8 total, 8 up, 8 in 2024-08-17T16:57:45.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:44 smithi063 ceph-mon[31063]: osdmap e249: 8 total, 8 up, 8 in 2024-08-17T16:57:45.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:44 smithi063 ceph-mon[31063]: pgmap v57: 89 pgs: 89 active+clean; 289 MiB data, 831 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:45.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:44 smithi063 ceph-mon[42680]: osdmap e248: 8 total, 8 up, 8 in 2024-08-17T16:57:45.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:44 smithi063 ceph-mon[42680]: osdmap e249: 8 total, 8 up, 8 in 2024-08-17T16:57:45.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:44 smithi063 ceph-mon[42680]: pgmap v57: 89 pgs: 89 active+clean; 289 MiB data, 831 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:45.281 INFO:tasks.rados.rados.0.smithi063.stdout:707: finishing write tid 3 to smithi06387500-41 2024-08-17T16:57:45.281 INFO:tasks.rados.rados.0.smithi063.stdout:707: finishing write tid 4 to smithi06387500-41 2024-08-17T16:57:45.281 INFO:tasks.rados.rados.0.smithi063.stdout:707: finishing write tid 5 to smithi06387500-41 2024-08-17T16:57:45.282 INFO:tasks.rados.rados.0.smithi063.stdout:707: oid 41 updating version 0 to 27 2024-08-17T16:57:45.282 INFO:tasks.rados.rados.0.smithi063.stdout:707: oid 41 version 27 is already newer than 24 2024-08-17T16:57:45.282 INFO:tasks.rados.rados.0.smithi063.stdout:707: oid 41 version 27 is already newer than 25 2024-08-17T16:57:45.282 INFO:tasks.rados.rados.0.smithi063.stdout:707: oid 41 version 27 is already newer than 26 2024-08-17T16:57:45.282 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 27 (ObjNum 298 snap 91 seq_num 298) dirty exists 2024-08-17T16:57:45.282 INFO:tasks.rados.rados.0.smithi063.stdout:707: left oid 41 (ObjNum 298 snap 91 seq_num 298) 2024-08-17T16:57:45.282 INFO:tasks.rados.rados.0.smithi063.stdout:707: done (2 left) 2024-08-17T16:57:45.282 INFO:tasks.rados.rados.0.smithi063.stdout:709: done (1 left) 2024-08-17T16:57:45.282 INFO:tasks.rados.rados.0.smithi063.stdout:710: snap_remove snap 51 2024-08-17T16:57:45.846 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:45 smithi162 systemd[1]: Stopping Ceph node-exporter.b for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:57:45.949 INFO:tasks.rados.rados.0.smithi063.stdout:710: done (1 left) 2024-08-17T16:57:45.949 INFO:tasks.rados.rados.0.smithi063.stdout:711: read oid 42 snap -1 2024-08-17T16:57:45.949 INFO:tasks.rados.rados.0.smithi063.stdout:711: expect (ObjNum 294 snap 91 seq_num 294) 2024-08-17T16:57:45.950 INFO:tasks.rados.rados.0.smithi063.stdout:712: read oid 43 snap -1 2024-08-17T16:57:45.950 INFO:tasks.rados.rados.0.smithi063.stdout:712: expect (ObjNum 287 snap 87 seq_num 287) 2024-08-17T16:57:45.950 INFO:tasks.rados.rados.0.smithi063.stdout:713: rollback oid 1 current snap is 91 2024-08-17T16:57:45.950 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 1 to 85 2024-08-17T16:57:45.953 INFO:tasks.rados.rados.0.smithi063.stdout:708: expect (ObjNum 295 snap 91 seq_num 295) 2024-08-17T16:57:46.053 INFO:tasks.rados.rados.0.smithi063.stdout:713: finishing rollback tid 1 to smithi06387500-1 2024-08-17T16:57:46.053 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 194 (ObjNum 705 snap 0 seq_num 1806918320) dirty dne 2024-08-17T16:57:46.053 INFO:tasks.rados.rados.0.smithi063.stdout:708: done (3 left) 2024-08-17T16:57:46.053 INFO:tasks.rados.rados.0.smithi063.stdout:713: done (2 left) 2024-08-17T16:57:46.053 INFO:tasks.rados.rados.0.smithi063.stdout:714: write oid 50 current snap is 91 2024-08-17T16:57:46.053 INFO:tasks.rados.rados.0.smithi063.stdout:714: seq_num 300 ranges {642100=712467,1872245=637761} 2024-08-17T16:57:46.060 INFO:tasks.rados.rados.0.smithi063.stdout:714: writing smithi06387500-50 from 642100 to 1354567 tid 1 2024-08-17T16:57:46.066 INFO:tasks.rados.rados.0.smithi063.stdout:714: writing smithi06387500-50 from 1872245 to 2510006 tid 2 2024-08-17T16:57:46.066 INFO:tasks.rados.rados.0.smithi063.stdout:715: write oid 28 current snap is 91 2024-08-17T16:57:46.067 INFO:tasks.rados.rados.0.smithi063.stdout:715: seq_num 301 ranges {658907=787716,1930727=497772,3032909=25998} 2024-08-17T16:57:46.074 INFO:tasks.rados.rados.0.smithi063.stdout:715: writing smithi06387500-28 from 658907 to 1446623 tid 1 2024-08-17T16:57:46.080 INFO:tasks.rados.rados.0.smithi063.stdout:715: writing smithi06387500-28 from 1930727 to 2428499 tid 2 2024-08-17T16:57:46.083 INFO:tasks.rados.rados.0.smithi063.stdout:715: writing smithi06387500-28 from 3032909 to 3058907 tid 3 2024-08-17T16:57:46.083 INFO:tasks.rados.rados.0.smithi063.stdout:716: write oid 33 current snap is 91 2024-08-17T16:57:46.084 INFO:tasks.rados.rados.0.smithi063.stdout:711: expect (ObjNum 294 snap 91 seq_num 294) 2024-08-17T16:57:46.255 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:45 smithi162 podman[85649]: 2024-08-17 16:57:45.970055433 +0000 UTC m=+0.118750809 container died b1691104d8aced91d63bce7e00f54a88a4870d0dc0f8f5697a12e80e2397e401 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T16:57:46.262 INFO:tasks.rados.rados.0.smithi063.stdout:712: expect (ObjNum 287 snap 87 seq_num 287) 2024-08-17T16:57:46.270 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T16:57:46.418 INFO:tasks.rados.rados.0.smithi063.stdout:714: finishing write tid 1 to smithi06387500-50 2024-08-17T16:57:46.418 INFO:tasks.rados.rados.0.smithi063.stdout:714: finishing write tid 2 to smithi06387500-50 2024-08-17T16:57:46.419 INFO:tasks.rados.rados.0.smithi063.stdout:714: finishing write tid 3 to smithi06387500-50 2024-08-17T16:57:46.419 INFO:tasks.rados.rados.0.smithi063.stdout:714: finishing write tid 4 to smithi06387500-50 2024-08-17T16:57:46.419 INFO:tasks.rados.rados.0.smithi063.stdout:714: oid 50 updating version 0 to 489 2024-08-17T16:57:46.419 INFO:tasks.rados.rados.0.smithi063.stdout:714: oid 50 version 489 is already newer than 488 2024-08-17T16:57:46.419 INFO:tasks.rados.rados.0.smithi063.stdout:714: oid 50 updating version 489 to 490 2024-08-17T16:57:46.419 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 490 (ObjNum 300 snap 91 seq_num 300) dirty exists 2024-08-17T16:57:46.419 INFO:tasks.rados.rados.0.smithi063.stdout:714: left oid 50 (ObjNum 300 snap 91 seq_num 300) 2024-08-17T16:57:46.419 INFO:tasks.rados.rados.0.smithi063.stdout:715: finishing write tid 1 to smithi06387500-28 2024-08-17T16:57:46.419 INFO:tasks.rados.rados.0.smithi063.stdout:715: finishing write tid 2 to smithi06387500-28 2024-08-17T16:57:46.419 INFO:tasks.rados.rados.0.smithi063.stdout:715: finishing write tid 3 to smithi06387500-28 2024-08-17T16:57:46.419 INFO:tasks.rados.rados.0.smithi063.stdout:715: finishing write tid 4 to smithi06387500-28 2024-08-17T16:57:46.419 INFO:tasks.rados.rados.0.smithi063.stdout:715: finishing write tid 5 to smithi06387500-28 2024-08-17T16:57:46.419 INFO:tasks.rados.rados.0.smithi063.stdout:715: oid 28 updating version 0 to 232 2024-08-17T16:57:46.420 INFO:tasks.rados.rados.0.smithi063.stdout:715: oid 28 version 232 is already newer than 230 2024-08-17T16:57:46.420 INFO:tasks.rados.rados.0.smithi063.stdout:715: oid 28 version 232 is already newer than 231 2024-08-17T16:57:46.420 INFO:tasks.rados.rados.0.smithi063.stdout:715: oid 28 updating version 232 to 233 2024-08-17T16:57:46.420 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 233 (ObjNum 301 snap 91 seq_num 301) dirty exists 2024-08-17T16:57:46.420 INFO:tasks.rados.rados.0.smithi063.stdout:715: left oid 28 (ObjNum 301 snap 91 seq_num 301) 2024-08-17T16:57:46.420 INFO:tasks.rados.rados.0.smithi063.stdout:716: seq_num 302 ranges {675714=462965,1589209=641430,2943979=131735} 2024-08-17T16:57:46.421 INFO:tasks.rados.rados.0.smithi063.stdout:716: writing smithi06387500-33 from 675714 to 1138679 tid 1 2024-08-17T16:57:46.427 INFO:tasks.rados.rados.0.smithi063.stdout:716: writing smithi06387500-33 from 1589209 to 2230639 tid 2 2024-08-17T16:57:46.432 INFO:tasks.rados.rados.0.smithi063.stdout:716: writing smithi06387500-33 from 2943979 to 3075714 tid 3 2024-08-17T16:57:46.433 INFO:tasks.rados.rados.0.smithi063.stdout:711: done (4 left) 2024-08-17T16:57:46.433 INFO:tasks.rados.rados.0.smithi063.stdout:712: done (3 left) 2024-08-17T16:57:46.433 INFO:tasks.rados.rados.0.smithi063.stdout:714: done (2 left) 2024-08-17T16:57:46.433 INFO:tasks.rados.rados.0.smithi063.stdout:715: done (1 left) 2024-08-17T16:57:46.433 INFO:tasks.rados.rados.0.smithi063.stdout:717: write oid 45 current snap is 91 2024-08-17T16:57:46.433 INFO:tasks.rados.rados.0.smithi063.stdout:716: finishing write tid 1 to smithi06387500-33 2024-08-17T16:57:46.433 INFO:tasks.rados.rados.0.smithi063.stdout:716: finishing write tid 2 to smithi06387500-33 2024-08-17T16:57:46.433 INFO:tasks.rados.rados.0.smithi063.stdout:717: seq_num 303 ranges {692521=538214,1931338=785088} 2024-08-17T16:57:46.439 INFO:tasks.rados.rados.0.smithi063.stdout:717: writing smithi06387500-45 from 692521 to 1230735 tid 1 2024-08-17T16:57:46.448 INFO:tasks.rados.rados.0.smithi063.stdout:717: writing smithi06387500-45 from 1931338 to 2716426 tid 2 2024-08-17T16:57:46.448 INFO:tasks.rados.rados.0.smithi063.stdout:718: snap_create 2024-08-17T16:57:46.448 INFO:tasks.rados.rados.0.smithi063.stdout:716: finishing write tid 3 to smithi06387500-33 2024-08-17T16:57:46.448 INFO:tasks.rados.rados.0.smithi063.stdout:716: finishing write tid 4 to smithi06387500-33 2024-08-17T16:57:46.448 INFO:tasks.rados.rados.0.smithi063.stdout:716: finishing write tid 5 to smithi06387500-33 2024-08-17T16:57:46.448 INFO:tasks.rados.rados.0.smithi063.stdout:716: oid 33 updating version 0 to 205 2024-08-17T16:57:46.448 INFO:tasks.rados.rados.0.smithi063.stdout:716: oid 33 updating version 205 to 207 2024-08-17T16:57:46.448 INFO:tasks.rados.rados.0.smithi063.stdout:716: oid 33 version 207 is already newer than 206 2024-08-17T16:57:46.448 INFO:tasks.rados.rados.0.smithi063.stdout:716: oid 33 version 207 is already newer than 204 2024-08-17T16:57:46.448 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 207 (ObjNum 302 snap 91 seq_num 302) dirty exists 2024-08-17T16:57:46.449 INFO:tasks.rados.rados.0.smithi063.stdout:716: left oid 33 (ObjNum 302 snap 91 seq_num 302) 2024-08-17T16:57:46.449 INFO:tasks.rados.rados.0.smithi063.stdout:717: finishing write tid 1 to smithi06387500-45 2024-08-17T16:57:46.451 INFO:tasks.rados.rados.0.smithi063.stdout:717: finishing write tid 2 to smithi06387500-45 2024-08-17T16:57:46.451 INFO:tasks.rados.rados.0.smithi063.stdout:717: finishing write tid 3 to smithi06387500-45 2024-08-17T16:57:46.451 INFO:tasks.rados.rados.0.smithi063.stdout:717: finishing write tid 4 to smithi06387500-45 2024-08-17T16:57:46.451 INFO:tasks.rados.rados.0.smithi063.stdout:717: oid 45 updating version 0 to 164 2024-08-17T16:57:46.451 INFO:tasks.rados.rados.0.smithi063.stdout:717: oid 45 version 164 is already newer than 163 2024-08-17T16:57:46.451 INFO:tasks.rados.rados.0.smithi063.stdout:717: oid 45 updating version 164 to 165 2024-08-17T16:57:46.451 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 165 (ObjNum 303 snap 91 seq_num 303) dirty exists 2024-08-17T16:57:46.452 INFO:tasks.rados.rados.0.smithi063.stdout:717: left oid 45 (ObjNum 303 snap 91 seq_num 303) 2024-08-17T16:57:46.506 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:46 smithi162 podman[85649]: 2024-08-17 16:57:46.504456391 +0000 UTC m=+0.653151756 container cleanup b1691104d8aced91d63bce7e00f54a88a4870d0dc0f8f5697a12e80e2397e401 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T16:57:46.506 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:46 smithi162 bash[85649]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b 2024-08-17T16:57:46.856 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T16:57:46.856 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (4m) 33s ago 5m 26.9M - 0.25.0 c8568f914cd2 87286b24f231 2024-08-17T16:57:46.857 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (4m) 13s ago 4m 88.2M - 9.4.7 954c08fa6188 fc8d62329012 2024-08-17T16:57:46.857 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (3m) 33s ago 3m 17.0M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T16:57:46.857 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (3m) 13s ago 3m 16.9M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T16:57:46.857 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,9283,8765 running (18s) 13s ago 9m 57.7M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb da48034a2895 2024-08-17T16:57:46.857 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:8443,9283,8765 running (46s) 33s ago 10m 489M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 943e639cede6 2024-08-17T16:57:46.857 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (10m) 33s ago 10m 53.4M 2048M 18.2.4-719-gced649f8 c870753d67c2 0c71ddd7ada6 2024-08-17T16:57:46.857 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (9m) 13s ago 9m 48.4M 2048M 18.2.4-719-gced649f8 c870753d67c2 d4ecbe65a0c8 2024-08-17T16:57:46.857 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (9m) 33s ago 9m 43.9M 2048M 18.2.4-719-gced649f8 c870753d67c2 b3f2c9339c6b 2024-08-17T16:57:46.857 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 starting - - - - 2024-08-17T16:57:46.857 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (4m) 13s ago 4m 15.3M - 1.5.0 0da6a335fe13 b1691104d8ac 2024-08-17T16:57:46.857 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (8m) 33s ago 8m 82.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 f3b910ed49b8 2024-08-17T16:57:46.857 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (8m) 33s ago 8m 62.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 5d6fdb6000ad 2024-08-17T16:57:46.857 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (8m) 33s ago 8m 75.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 a3cbe8edb49a 2024-08-17T16:57:46.858 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (7m) 33s ago 7m 69.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T16:57:46.858 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (7m) 13s ago 7m 85.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T16:57:46.858 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (6m) 13s ago 6m 86.3M 2480M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T16:57:46.858 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (6m) 13s ago 6m 60.2M 2480M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T16:57:46.858 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (6m) 13s ago 6m 71.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T16:57:46.858 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (23s) 13s ago 5m 34.3M - 2.43.0 a07b618ecd1d 5ccd60dff86c 2024-08-17T16:57:46.953 INFO:tasks.rados.rados.0.smithi063.stdout:716: done (2 left) 2024-08-17T16:57:46.953 INFO:tasks.rados.rados.0.smithi063.stdout:717: done (1 left) 2024-08-17T16:57:46.953 INFO:tasks.rados.rados.0.smithi063.stdout:718: done (0 left) 2024-08-17T16:57:46.953 INFO:tasks.rados.rados.0.smithi063.stdout:719: snap_remove snap 84 2024-08-17T16:57:47.076 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:46 smithi162 podman[85665]: 2024-08-17 16:57:46.655580826 +0000 UTC m=+0.686064427 container remove b1691104d8aced91d63bce7e00f54a88a4870d0dc0f8f5697a12e80e2397e401 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T16:57:47.076 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:46 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-08-17T16:57:47.077 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:46 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-08-17T16:57:47.077 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:46 smithi162 systemd[1]: Stopped Ceph node-exporter.b for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:57:47.077 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:46 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@node-exporter.b.service: Consumed 2.339s CPU time. 2024-08-17T16:57:47.077 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:46 smithi162 ceph-mon[34633]: Upgrade: Updating node-exporter.b (2/2) 2024-08-17T16:57:47.077 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:46 smithi162 ceph-mon[34633]: Deploying daemon node-exporter.b on smithi162 2024-08-17T16:57:47.077 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:46 smithi162 ceph-mon[34633]: osdmap e250: 8 total, 8 up, 8 in 2024-08-17T16:57:47.209 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:57:47.209 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T16:57:47.209 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T16:57:47.209 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:57:47.209 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T16:57:47.209 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T16:57:47.209 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:57:47.209 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T16:57:47.209 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T16:57:47.209 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:57:47.209 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T16:57:47.209 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T16:57:47.210 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:57:47.210 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T16:57:47.210 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 13, 2024-08-17T16:57:47.210 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T16:57:47.210 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T16:57:47.210 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:57:47.210 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:46 smithi063 ceph-mon[31063]: Upgrade: Updating node-exporter.b (2/2) 2024-08-17T16:57:47.210 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:46 smithi063 ceph-mon[31063]: Deploying daemon node-exporter.b on smithi162 2024-08-17T16:57:47.210 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:46 smithi063 ceph-mon[31063]: osdmap e250: 8 total, 8 up, 8 in 2024-08-17T16:57:47.211 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:46 smithi063 ceph-mon[42680]: Upgrade: Updating node-exporter.b (2/2) 2024-08-17T16:57:47.211 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:46 smithi063 ceph-mon[42680]: Deploying daemon node-exporter.b on smithi162 2024-08-17T16:57:47.211 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:46 smithi063 ceph-mon[42680]: osdmap e250: 8 total, 8 up, 8 in 2024-08-17T16:57:47.345 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:47 smithi162 systemd[1]: Starting Ceph node-exporter.b for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:57:47.346 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:47 smithi162 bash[85792]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-08-17T16:57:47.507 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:57:47.507 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T16:57:47.507 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-08-17T16:57:47.507 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-17T16:57:47.507 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-08-17T16:57:47.507 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-08-17T16:57:47.507 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-08-17T16:57:47.507 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "2/2 daemons upgraded", 2024-08-17T16:57:47.507 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading node-exporter daemons", 2024-08-17T16:57:47.508 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T16:57:47.508 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:57:47.864 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:47 smithi162 bash[85792]: Getting image source signatures 2024-08-17T16:57:47.864 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:47 smithi162 bash[85792]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-08-17T16:57:47.864 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:47 smithi162 bash[85792]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-08-17T16:57:47.864 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:47 smithi162 bash[85792]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-08-17T16:57:47.951 INFO:tasks.rados.rados.0.smithi063.stdout:719: done (0 left) 2024-08-17T16:57:47.951 INFO:tasks.rados.rados.0.smithi063.stdout:720: read oid 6 snap -1 2024-08-17T16:57:47.951 INFO:tasks.rados.rados.0.smithi063.stdout:720: expect (ObjNum 178 snap 47 seq_num 178) 2024-08-17T16:57:47.951 INFO:tasks.rados.rados.0.smithi063.stdout:721: delete oid 29 current snap is 92 2024-08-17T16:57:47.957 INFO:tasks.rados.rados.0.smithi063.stdout:721: done (1 left) 2024-08-17T16:57:47.957 INFO:tasks.rados.rados.0.smithi063.stdout:722: write oid 46 current snap is 92 2024-08-17T16:57:47.957 INFO:tasks.rados.rados.0.smithi063.stdout:722: seq_num 305 ranges {726135=405065,1764655=788757,3026248=99887} 2024-08-17T16:57:47.963 INFO:tasks.rados.rados.0.smithi063.stdout:722: writing smithi06387500-46 from 726135 to 1131200 tid 1 2024-08-17T16:57:47.971 INFO:tasks.rados.rados.0.smithi063.stdout:722: writing smithi06387500-46 from 1764655 to 2553412 tid 2 2024-08-17T16:57:47.975 INFO:tasks.rados.rados.0.smithi063.stdout:722: writing smithi06387500-46 from 3026248 to 3126135 tid 3 2024-08-17T16:57:47.975 INFO:tasks.rados.rados.0.smithi063.stdout:723: read oid 18 snap -1 2024-08-17T16:57:47.975 INFO:tasks.rados.rados.0.smithi063.stdout:723: expect (ObjNum 265 snap 80 seq_num 265) 2024-08-17T16:57:47.975 INFO:tasks.rados.rados.0.smithi063.stdout:724: rollback oid 35 current snap is 92 2024-08-17T16:57:47.975 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 35 to 60 2024-08-17T16:57:47.975 INFO:tasks.rados.rados.0.smithi063.stdout:720: expect (ObjNum 178 snap 47 seq_num 178) 2024-08-17T16:57:48.151 INFO:tasks.rados.rados.0.smithi063.stdout:722: finishing write tid 1 to smithi06387500-46 2024-08-17T16:57:48.151 INFO:tasks.rados.rados.0.smithi063.stdout:722: finishing write tid 2 to smithi06387500-46 2024-08-17T16:57:48.151 INFO:tasks.rados.rados.0.smithi063.stdout:720: done (3 left) 2024-08-17T16:57:48.151 INFO:tasks.rados.rados.0.smithi063.stdout:725: write oid 31 current snap is 92 2024-08-17T16:57:48.151 INFO:tasks.rados.rados.0.smithi063.stdout:722: finishing write tid 3 to smithi06387500-46 2024-08-17T16:57:48.151 INFO:tasks.rados.rados.0.smithi063.stdout:725: seq_num 306 ranges {742942=480314,1823137=648768} 2024-08-17T16:57:48.157 INFO:tasks.rados.rados.0.smithi063.stdout:725: writing smithi06387500-31 from 742942 to 1223256 tid 1 2024-08-17T16:57:48.164 INFO:tasks.rados.rados.0.smithi063.stdout:725: writing smithi06387500-31 from 1823137 to 2471905 tid 2 2024-08-17T16:57:48.164 INFO:tasks.rados.rados.0.smithi063.stdout:726: write oid 32 current snap is 92 2024-08-17T16:57:48.164 INFO:tasks.rados.rados.0.smithi063.stdout:726: seq_num 307 ranges {759749=555563,1765266=792426,2964741=195008} 2024-08-17T16:57:48.171 INFO:tasks.rados.rados.0.smithi063.stdout:726: writing smithi06387500-32 from 759749 to 1315312 tid 1 2024-08-17T16:57:48.178 INFO:tasks.rados.rados.0.smithi063.stdout:726: writing smithi06387500-32 from 1765266 to 2557692 tid 2 2024-08-17T16:57:48.182 INFO:tasks.rados.rados.0.smithi063.stdout:726: writing smithi06387500-32 from 2964741 to 3159749 tid 3 2024-08-17T16:57:48.182 INFO:tasks.rados.rados.0.smithi063.stdout:727: delete oid 7 current snap is 92 2024-08-17T16:57:48.182 INFO:tasks.rados.rados.0.smithi063.stdout:722: finishing write tid 4 to smithi06387500-46 2024-08-17T16:57:48.182 INFO:tasks.rados.rados.0.smithi063.stdout:722: finishing write tid 5 to smithi06387500-46 2024-08-17T16:57:48.182 INFO:tasks.rados.rados.0.smithi063.stdout:722: oid 46 updating version 0 to 280 2024-08-17T16:57:48.183 INFO:tasks.rados.rados.0.smithi063.stdout:722: oid 46 version 280 is already newer than 277 2024-08-17T16:57:48.183 INFO:tasks.rados.rados.0.smithi063.stdout:722: oid 46 version 280 is already newer than 279 2024-08-17T16:57:48.183 INFO:tasks.rados.rados.0.smithi063.stdout:722: oid 46 version 280 is already newer than 278 2024-08-17T16:57:48.183 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 280 (ObjNum 305 snap 92 seq_num 305) dirty exists 2024-08-17T16:57:48.183 INFO:tasks.rados.rados.0.smithi063.stdout:722: left oid 46 (ObjNum 305 snap 92 seq_num 305) 2024-08-17T16:57:48.183 INFO:tasks.rados.rados.0.smithi063.stdout:724: finishing rollback tid 0 to smithi06387500-35 2024-08-17T16:57:48.183 INFO:tasks.rados.rados.0.smithi063.stdout:724: finishing rollback tid 1 to smithi06387500-35 2024-08-17T16:57:48.183 INFO:tasks.rados.rados.0.smithi063.stdout:724: finishing rollback tid 2 to smithi06387500-35 2024-08-17T16:57:48.183 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 741 (ObjNum 179 snap 47 seq_num 179) dirty exists 2024-08-17T16:57:48.183 INFO:tasks.rados.rados.0.smithi063.stdout:723: expect (ObjNum 265 snap 80 seq_num 265) 2024-08-17T16:57:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:47 smithi063 ceph-mon[42680]: from='client.24932 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:47 smithi063 ceph-mon[42680]: from='client.24938 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:47 smithi063 ceph-mon[42680]: from='client.15051 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:47 smithi063 ceph-mon[42680]: pgmap v59: 89 pgs: 89 active+clean; 300 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-08-17T16:57:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:47 smithi063 ceph-mon[42680]: osdmap e251: 8 total, 8 up, 8 in 2024-08-17T16:57:48.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:47 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/512976880' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:47 smithi063 ceph-mon[31063]: from='client.24932 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:47 smithi063 ceph-mon[31063]: from='client.24938 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:47 smithi063 ceph-mon[31063]: from='client.15051 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:47 smithi063 ceph-mon[31063]: pgmap v59: 89 pgs: 89 active+clean; 300 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-08-17T16:57:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:47 smithi063 ceph-mon[31063]: osdmap e251: 8 total, 8 up, 8 in 2024-08-17T16:57:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:47 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/512976880' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:48.340 INFO:tasks.rados.rados.0.smithi063.stdout:725: finishing write tid 1 to smithi06387500-31 2024-08-17T16:57:48.340 INFO:tasks.rados.rados.0.smithi063.stdout:725: finishing write tid 2 to smithi06387500-31 2024-08-17T16:57:48.340 INFO:tasks.rados.rados.0.smithi063.stdout:725: finishing write tid 3 to smithi06387500-31 2024-08-17T16:57:48.340 INFO:tasks.rados.rados.0.smithi063.stdout:725: finishing write tid 4 to smithi06387500-31 2024-08-17T16:57:48.340 INFO:tasks.rados.rados.0.smithi063.stdout:725: oid 31 updating version 0 to 500 2024-08-17T16:57:48.340 INFO:tasks.rados.rados.0.smithi063.stdout:725: oid 31 version 500 is already newer than 499 2024-08-17T16:57:48.340 INFO:tasks.rados.rados.0.smithi063.stdout:725: oid 31 version 500 is already newer than 498 2024-08-17T16:57:48.340 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 500 (ObjNum 306 snap 92 seq_num 306) dirty exists 2024-08-17T16:57:48.340 INFO:tasks.rados.rados.0.smithi063.stdout:725: left oid 31 (ObjNum 306 snap 92 seq_num 306) 2024-08-17T16:57:48.340 INFO:tasks.rados.rados.0.smithi063.stdout:726: finishing write tid 1 to smithi06387500-32 2024-08-17T16:57:48.341 INFO:tasks.rados.rados.0.smithi063.stdout:726: finishing write tid 2 to smithi06387500-32 2024-08-17T16:57:48.341 INFO:tasks.rados.rados.0.smithi063.stdout:726: finishing write tid 3 to smithi06387500-32 2024-08-17T16:57:48.341 INFO:tasks.rados.rados.0.smithi063.stdout:726: finishing write tid 4 to smithi06387500-32 2024-08-17T16:57:48.341 INFO:tasks.rados.rados.0.smithi063.stdout:726: finishing write tid 5 to smithi06387500-32 2024-08-17T16:57:48.341 INFO:tasks.rados.rados.0.smithi063.stdout:726: oid 32 updating version 0 to 133 2024-08-17T16:57:48.341 INFO:tasks.rados.rados.0.smithi063.stdout:726: oid 32 version 133 is already newer than 132 2024-08-17T16:57:48.341 INFO:tasks.rados.rados.0.smithi063.stdout:726: oid 32 updating version 133 to 134 2024-08-17T16:57:48.341 INFO:tasks.rados.rados.0.smithi063.stdout:726: oid 32 version 134 is already newer than 131 2024-08-17T16:57:48.341 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 134 (ObjNum 307 snap 92 seq_num 307) dirty exists 2024-08-17T16:57:48.341 INFO:tasks.rados.rados.0.smithi063.stdout:726: left oid 32 (ObjNum 307 snap 92 seq_num 307) 2024-08-17T16:57:48.341 INFO:tasks.rados.rados.0.smithi063.stdout:722: done (5 left) 2024-08-17T16:57:48.341 INFO:tasks.rados.rados.0.smithi063.stdout:723: done (4 left) 2024-08-17T16:57:48.342 INFO:tasks.rados.rados.0.smithi063.stdout:724: done (3 left) 2024-08-17T16:57:48.342 INFO:tasks.rados.rados.0.smithi063.stdout:725: done (2 left) 2024-08-17T16:57:48.342 INFO:tasks.rados.rados.0.smithi063.stdout:726: done (1 left) 2024-08-17T16:57:48.342 INFO:tasks.rados.rados.0.smithi063.stdout:727: done (0 left) 2024-08-17T16:57:48.342 INFO:tasks.rados.rados.0.smithi063.stdout:728: snap_remove snap 20 2024-08-17T16:57:48.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:47 smithi162 ceph-mon[34633]: from='client.24932 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:48.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:47 smithi162 ceph-mon[34633]: from='client.24938 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:48.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:47 smithi162 ceph-mon[34633]: from='client.15051 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:48.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:47 smithi162 ceph-mon[34633]: pgmap v59: 89 pgs: 89 active+clean; 300 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-08-17T16:57:48.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:47 smithi162 ceph-mon[34633]: osdmap e251: 8 total, 8 up, 8 in 2024-08-17T16:57:48.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:47 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/512976880' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:48.952 INFO:tasks.rados.rados.0.smithi063.stdout:728: done (0 left) 2024-08-17T16:57:48.952 INFO:tasks.rados.rados.0.smithi063.stdout:729: write oid 45 current snap is 92 2024-08-17T16:57:48.952 INFO:tasks.rados.rados.0.smithi063.stdout:729: seq_num 309 ranges {793363=706061,2282230=796095} 2024-08-17T16:57:48.962 INFO:tasks.rados.rados.0.smithi063.stdout:729: writing smithi06387500-45 from 793363 to 1499424 tid 1 2024-08-17T16:57:48.971 INFO:tasks.rados.rados.0.smithi063.stdout:729: writing smithi06387500-45 from 2282230 to 3078325 tid 2 2024-08-17T16:57:48.971 INFO:tasks.rados.rados.0.smithi063.stdout:730: delete oid 19 current snap is 92 2024-08-17T16:57:48.971 INFO:tasks.rados.rados.0.smithi063.stdout:729: finishing write tid 1 to smithi06387500-45 2024-08-17T16:57:48.973 INFO:tasks.rados.rados.0.smithi063.stdout:730: done (1 left) 2024-08-17T16:57:48.973 INFO:tasks.rados.rados.0.smithi063.stdout:731: write oid 34 current snap is 92 2024-08-17T16:57:48.973 INFO:tasks.rados.rados.0.smithi063.stdout:731: seq_num 311 ranges {426977=456559,1482841=799764,2958080=268897} 2024-08-17T16:57:48.978 INFO:tasks.rados.rados.0.smithi063.stdout:731: writing smithi06387500-34 from 426977 to 883536 tid 1 2024-08-17T16:57:48.986 INFO:tasks.rados.rados.0.smithi063.stdout:731: writing smithi06387500-34 from 1482841 to 2282605 tid 2 2024-08-17T16:57:48.991 INFO:tasks.rados.rados.0.smithi063.stdout:731: writing smithi06387500-34 from 2958080 to 3226977 tid 3 2024-08-17T16:57:48.991 INFO:tasks.rados.rados.0.smithi063.stdout:732: write oid 33 current snap is 92 2024-08-17T16:57:48.991 INFO:tasks.rados.rados.0.smithi063.stdout:729: finishing write tid 2 to smithi06387500-45 2024-08-17T16:57:48.991 INFO:tasks.rados.rados.0.smithi063.stdout:732: seq_num 312 ranges {443784=648161,1657676=659775,2818209=425575} 2024-08-17T16:57:48.997 INFO:tasks.rados.rados.0.smithi063.stdout:732: writing smithi06387500-33 from 443784 to 1091945 tid 1 2024-08-17T16:57:49.004 INFO:tasks.rados.rados.0.smithi063.stdout:732: writing smithi06387500-33 from 1657676 to 2317451 tid 2 2024-08-17T16:57:49.009 INFO:tasks.rados.rados.0.smithi063.stdout:732: writing smithi06387500-33 from 2818209 to 3243784 tid 3 2024-08-17T16:57:49.009 INFO:tasks.rados.rados.0.smithi063.stdout:733: rollback oid 37 current snap is 92 2024-08-17T16:57:49.010 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 88 2024-08-17T16:57:49.010 INFO:tasks.rados.rados.0.smithi063.stdout:729: finishing write tid 3 to smithi06387500-45 2024-08-17T16:57:49.010 INFO:tasks.rados.rados.0.smithi063.stdout:729: finishing write tid 4 to smithi06387500-45 2024-08-17T16:57:49.010 INFO:tasks.rados.rados.0.smithi063.stdout:729: oid 45 updating version 0 to 172 2024-08-17T16:57:49.010 INFO:tasks.rados.rados.0.smithi063.stdout:729: oid 45 updating version 172 to 173 2024-08-17T16:57:49.010 INFO:tasks.rados.rados.0.smithi063.stdout:729: oid 45 version 173 is already newer than 171 2024-08-17T16:57:49.010 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 173 (ObjNum 309 snap 92 seq_num 309) dirty exists 2024-08-17T16:57:49.010 INFO:tasks.rados.rados.0.smithi063.stdout:729: left oid 45 (ObjNum 309 snap 92 seq_num 309) 2024-08-17T16:57:49.010 INFO:tasks.rados.rados.0.smithi063.stdout:731: finishing write tid 1 to smithi06387500-34 2024-08-17T16:57:49.010 INFO:tasks.rados.rados.0.smithi063.stdout:731: finishing write tid 2 to smithi06387500-34 2024-08-17T16:57:49.011 INFO:tasks.rados.rados.0.smithi063.stdout:729: done (3 left) 2024-08-17T16:57:49.011 INFO:tasks.rados.rados.0.smithi063.stdout:734: rollback oid 46 current snap is 92 2024-08-17T16:57:49.011 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 46 to 91 2024-08-17T16:57:49.011 INFO:tasks.rados.rados.0.smithi063.stdout:731: finishing write tid 3 to smithi06387500-34 2024-08-17T16:57:49.011 INFO:tasks.rados.rados.0.smithi063.stdout:731: finishing write tid 4 to smithi06387500-34 2024-08-17T16:57:49.011 INFO:tasks.rados.rados.0.smithi063.stdout:731: finishing write tid 5 to smithi06387500-34 2024-08-17T16:57:49.011 INFO:tasks.rados.rados.0.smithi063.stdout:731: oid 34 updating version 0 to 513 2024-08-17T16:57:49.011 INFO:tasks.rados.rados.0.smithi063.stdout:731: oid 34 updating version 513 to 516 2024-08-17T16:57:49.011 INFO:tasks.rados.rados.0.smithi063.stdout:731: oid 34 version 516 is already newer than 514 2024-08-17T16:57:49.011 INFO:tasks.rados.rados.0.smithi063.stdout:731: oid 34 version 516 is already newer than 515 2024-08-17T16:57:49.011 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 516 (ObjNum 311 snap 92 seq_num 311) dirty exists 2024-08-17T16:57:49.011 INFO:tasks.rados.rados.0.smithi063.stdout:731: left oid 34 (ObjNum 311 snap 92 seq_num 311) 2024-08-17T16:57:49.011 INFO:tasks.rados.rados.0.smithi063.stdout:732: finishing write tid 1 to smithi06387500-33 2024-08-17T16:57:49.011 INFO:tasks.rados.rados.0.smithi063.stdout:731: done (3 left) 2024-08-17T16:57:49.012 INFO:tasks.rados.rados.0.smithi063.stdout:735: read oid 17 snap 91 2024-08-17T16:57:49.012 INFO:tasks.rados.rados.0.smithi063.stdout:735: expect deleted 2024-08-17T16:57:49.012 INFO:tasks.rados.rados.0.smithi063.stdout:736: read oid 22 snap -1 2024-08-17T16:57:49.012 INFO:tasks.rados.rados.0.smithi063.stdout:736: expect deleted 2024-08-17T16:57:49.012 INFO:tasks.rados.rados.0.smithi063.stdout:737: snap_create 2024-08-17T16:57:49.012 INFO:tasks.rados.rados.0.smithi063.stdout:732: finishing write tid 2 to smithi06387500-33 2024-08-17T16:57:49.013 INFO:tasks.rados.rados.0.smithi063.stdout:732: finishing write tid 3 to smithi06387500-33 2024-08-17T16:57:49.013 INFO:tasks.rados.rados.0.smithi063.stdout:734: finishing rollback tid 0 to smithi06387500-46 2024-08-17T16:57:49.013 INFO:tasks.rados.rados.0.smithi063.stdout:734: finishing rollback tid 1 to smithi06387500-46 2024-08-17T16:57:49.014 INFO:tasks.rados.rados.0.smithi063.stdout:732: finishing write tid 4 to smithi06387500-33 2024-08-17T16:57:49.014 INFO:tasks.rados.rados.0.smithi063.stdout:733: finishing rollback tid 0 to smithi06387500-37 2024-08-17T16:57:49.014 INFO:tasks.rados.rados.0.smithi063.stdout:732: finishing write tid 5 to smithi06387500-33 2024-08-17T16:57:49.014 INFO:tasks.rados.rados.0.smithi063.stdout:732: oid 33 updating version 0 to 216 2024-08-17T16:57:49.014 INFO:tasks.rados.rados.0.smithi063.stdout:732: oid 33 version 216 is already newer than 215 2024-08-17T16:57:49.014 INFO:tasks.rados.rados.0.smithi063.stdout:732: oid 33 updating version 216 to 218 2024-08-17T16:57:49.014 INFO:tasks.rados.rados.0.smithi063.stdout:732: oid 33 version 218 is already newer than 217 2024-08-17T16:57:49.014 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 218 (ObjNum 312 snap 92 seq_num 312) dirty exists 2024-08-17T16:57:49.014 INFO:tasks.rados.rados.0.smithi063.stdout:732: left oid 33 (ObjNum 312 snap 92 seq_num 312) 2024-08-17T16:57:49.014 INFO:tasks.rados.rados.0.smithi063.stdout:733: finishing rollback tid 1 to smithi06387500-37 2024-08-17T16:57:49.014 INFO:tasks.rados.rados.0.smithi063.stdout:734: finishing rollback tid 2 to smithi06387500-46 2024-08-17T16:57:49.015 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 287 (ObjNum 293 snap 91 seq_num 293) dirty exists 2024-08-17T16:57:49.015 INFO:tasks.rados.rados.0.smithi063.stdout:733: finishing rollback tid 2 to smithi06387500-37 2024-08-17T16:57:49.015 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 757 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-08-17T16:57:49.195 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:48 smithi162 bash[85792]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-08-17T16:57:49.195 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:48 smithi162 ceph-mon[34633]: from='client.24956 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:49.195 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:48 smithi162 ceph-mon[34633]: osdmap e252: 8 total, 8 up, 8 in 2024-08-17T16:57:49.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:48 smithi063 ceph-mon[42680]: from='client.24956 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:49.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:48 smithi063 ceph-mon[42680]: osdmap e252: 8 total, 8 up, 8 in 2024-08-17T16:57:49.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:48 smithi063 ceph-mon[31063]: from='client.24956 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:57:49.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:48 smithi063 ceph-mon[31063]: osdmap e252: 8 total, 8 up, 8 in 2024-08-17T16:57:49.498 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 bash[85792]: Writing manifest to image destination 2024-08-17T16:57:49.759 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 podman[85792]: 2024-08-17 16:57:49.450311157 +0000 UTC m=+2.275115396 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-08-17T16:57:49.759 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 podman[85792]: 2024-08-17 16:57:49.548972665 +0000 UTC m=+2.373776940 container create 5bb162d0f1a920054bdf68ad9fd3db476dc3fe81968f9a2ad80c09b8531a29af (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T16:57:49.862 INFO:tasks.rados.rados.0.smithi063.stdout:732: done (5 left) 2024-08-17T16:57:49.862 INFO:tasks.rados.rados.0.smithi063.stdout:733: done (4 left) 2024-08-17T16:57:49.862 INFO:tasks.rados.rados.0.smithi063.stdout:734: done (3 left) 2024-08-17T16:57:49.863 INFO:tasks.rados.rados.0.smithi063.stdout:735: done (2 left) 2024-08-17T16:57:49.863 INFO:tasks.rados.rados.0.smithi063.stdout:736: done (1 left) 2024-08-17T16:57:49.863 INFO:tasks.rados.rados.0.smithi063.stdout:737: done (0 left) 2024-08-17T16:57:49.863 INFO:tasks.rados.rados.0.smithi063.stdout:738: snap_create 2024-08-17T16:57:49.869 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:49 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:57:49] "GET /metrics HTTP/1.1" 200 36212 "" "Prometheus/2.43.0" 2024-08-17T16:57:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-mon[34633]: pgmap v62: 89 pgs: 89 active+clean; 300 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-08-17T16:57:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-mon[34633]: osdmap e253: 8 total, 8 up, 8 in 2024-08-17T16:57:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-mon[34633]: osdmap e254: 8 total, 8 up, 8 in 2024-08-17T16:57:50.096 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 podman[85792]: 2024-08-17 16:57:49.75907726 +0000 UTC m=+2.583881507 container init 5bb162d0f1a920054bdf68ad9fd3db476dc3fe81968f9a2ad80c09b8531a29af (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T16:57:50.096 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 podman[85792]: 2024-08-17 16:57:49.765206092 +0000 UTC m=+2.590010326 container start 5bb162d0f1a920054bdf68ad9fd3db476dc3fe81968f9a2ad80c09b8531a29af (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T16:57:50.096 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.769Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-08-17T16:57:50.096 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.769Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-08-17T16:57:50.096 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.769Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-08-17T16:57:50.097 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.769Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-08-17T16:57:50.097 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.769Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-08-17T16:57:50.097 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.769Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-08-17T16:57:50.097 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-08-17T16:57:50.097 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=arp 2024-08-17T16:57:50.097 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=bcache 2024-08-17T16:57:50.097 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=bonding 2024-08-17T16:57:50.097 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=btrfs 2024-08-17T16:57:50.097 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=conntrack 2024-08-17T16:57:50.097 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=cpu 2024-08-17T16:57:50.097 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-08-17T16:57:50.097 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=diskstats 2024-08-17T16:57:50.098 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=dmi 2024-08-17T16:57:50.098 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=edac 2024-08-17T16:57:50.098 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=entropy 2024-08-17T16:57:50.098 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-08-17T16:57:50.098 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=filefd 2024-08-17T16:57:50.098 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=filesystem 2024-08-17T16:57:50.098 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=hwmon 2024-08-17T16:57:50.098 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=infiniband 2024-08-17T16:57:50.098 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=ipvs 2024-08-17T16:57:50.098 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=loadavg 2024-08-17T16:57:50.098 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=mdadm 2024-08-17T16:57:50.098 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=meminfo 2024-08-17T16:57:50.098 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=netclass 2024-08-17T16:57:50.099 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=netdev 2024-08-17T16:57:50.099 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=netstat 2024-08-17T16:57:50.099 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=nfs 2024-08-17T16:57:50.099 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=nfsd 2024-08-17T16:57:50.099 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=nvme 2024-08-17T16:57:50.099 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=os 2024-08-17T16:57:50.099 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-08-17T16:57:50.099 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=pressure 2024-08-17T16:57:50.099 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=rapl 2024-08-17T16:57:50.099 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=schedstat 2024-08-17T16:57:50.099 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=selinux 2024-08-17T16:57:50.100 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=sockstat 2024-08-17T16:57:50.100 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=softnet 2024-08-17T16:57:50.100 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=stat 2024-08-17T16:57:50.100 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=tapestats 2024-08-17T16:57:50.100 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=textfile 2024-08-17T16:57:50.100 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-08-17T16:57:50.100 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=time 2024-08-17T16:57:50.100 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-08-17T16:57:50.100 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=uname 2024-08-17T16:57:50.100 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=vmstat 2024-08-17T16:57:50.100 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=xfs 2024-08-17T16:57:50.100 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.770Z caller=node_exporter.go:117 level=info collector=zfs 2024-08-17T16:57:50.100 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.771Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-08-17T16:57:50.100 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b[85843]: ts=2024-08-17T16:57:49.771Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-08-17T16:57:50.101 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 bash[85792]: 5bb162d0f1a920054bdf68ad9fd3db476dc3fe81968f9a2ad80c09b8531a29af 2024-08-17T16:57:50.101 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 16:57:49 smithi162 systemd[1]: Started Ceph node-exporter.b for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:57:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:49 smithi063 ceph-mon[42680]: pgmap v62: 89 pgs: 89 active+clean; 300 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-08-17T16:57:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:49 smithi063 ceph-mon[42680]: osdmap e253: 8 total, 8 up, 8 in 2024-08-17T16:57:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:49 smithi063 ceph-mon[42680]: osdmap e254: 8 total, 8 up, 8 in 2024-08-17T16:57:50.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:49 smithi063 ceph-mon[31063]: pgmap v62: 89 pgs: 89 active+clean; 300 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-08-17T16:57:50.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:49 smithi063 ceph-mon[31063]: osdmap e253: 8 total, 8 up, 8 in 2024-08-17T16:57:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:49 smithi063 ceph-mon[31063]: osdmap e254: 8 total, 8 up, 8 in 2024-08-17T16:57:50.865 INFO:tasks.rados.rados.0.smithi063.stdout:738: done (0 left) 2024-08-17T16:57:50.865 INFO:tasks.rados.rados.0.smithi063.stdout:739: rollback oid 17 current snap is 94 2024-08-17T16:57:50.865 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 17 to 91 2024-08-17T16:57:50.865 INFO:tasks.rados.rados.0.smithi063.stdout:740: write oid 42 current snap is 94 2024-08-17T16:57:50.865 INFO:tasks.rados.rados.0.smithi063.stdout:740: seq_num 313 ranges {460591=723410,1716158=403433,2729279=440140} 2024-08-17T16:57:50.871 INFO:tasks.rados.rados.0.smithi063.stdout:740: writing smithi06387500-42 from 460591 to 1184001 tid 1 2024-08-17T16:57:50.876 INFO:tasks.rados.rados.0.smithi063.stdout:740: writing smithi06387500-42 from 1716158 to 2119591 tid 2 2024-08-17T16:57:50.883 INFO:tasks.rados.rados.0.smithi063.stdout:740: writing smithi06387500-42 from 2729279 to 3169419 tid 3 2024-08-17T16:57:50.883 INFO:tasks.rados.rados.0.smithi063.stdout:741: rollback oid 47 current snap is 94 2024-08-17T16:57:50.883 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 47 to 63 2024-08-17T16:57:50.883 INFO:tasks.rados.rados.0.smithi063.stdout:739: finishing rollback tid 1 to smithi06387500-17 2024-08-17T16:57:50.883 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 214 (ObjNum 209 snap 0 seq_num 1027632502) dirty dne 2024-08-17T16:57:50.883 INFO:tasks.rados.rados.0.smithi063.stdout:740: finishing write tid 1 to smithi06387500-42 2024-08-17T16:57:50.883 INFO:tasks.rados.rados.0.smithi063.stdout:740: finishing write tid 2 to smithi06387500-42 2024-08-17T16:57:50.883 INFO:tasks.rados.rados.0.smithi063.stdout:739: done (2 left) 2024-08-17T16:57:50.883 INFO:tasks.rados.rados.0.smithi063.stdout:742: write oid 4 current snap is 94 2024-08-17T16:57:50.883 INFO:tasks.rados.rados.0.smithi063.stdout:742: seq_num 314 ranges {477398=798659,1774640=663444,2873055=404343} 2024-08-17T16:57:50.889 INFO:tasks.rados.rados.0.smithi063.stdout:742: writing smithi06387500-4 from 477398 to 1276057 tid 1 2024-08-17T16:57:50.896 INFO:tasks.rados.rados.0.smithi063.stdout:742: writing smithi06387500-4 from 1774640 to 2438084 tid 2 2024-08-17T16:57:50.901 INFO:tasks.rados.rados.0.smithi063.stdout:742: writing smithi06387500-4 from 2873055 to 3277398 tid 3 2024-08-17T16:57:50.901 INFO:tasks.rados.rados.0.smithi063.stdout:743: write oid 36 current snap is 94 2024-08-17T16:57:50.901 INFO:tasks.rados.rados.0.smithi063.stdout:741: finishing rollback tid 0 to smithi06387500-47 2024-08-17T16:57:50.901 INFO:tasks.rados.rados.0.smithi063.stdout:743: seq_num 315 ranges {494205=473908,1433122=407102,2384125=462684} 2024-08-17T16:57:50.906 INFO:tasks.rados.rados.0.smithi063.stdout:743: writing smithi06387500-36 from 494205 to 968113 tid 1 2024-08-17T16:57:50.911 INFO:tasks.rados.rados.0.smithi063.stdout:743: writing smithi06387500-36 from 1433122 to 1840224 tid 2 2024-08-17T16:57:50.917 INFO:tasks.rados.rados.0.smithi063.stdout:743: writing smithi06387500-36 from 2384125 to 2846809 tid 3 2024-08-17T16:57:50.917 INFO:tasks.rados.rados.0.smithi063.stdout:740: finishing write tid 3 to smithi06387500-42 2024-08-17T16:57:50.917 INFO:tasks.rados.rados.0.smithi063.stdout:741: finishing rollback tid 1 to smithi06387500-47 2024-08-17T16:57:50.917 INFO:tasks.rados.rados.0.smithi063.stdout:740: finishing write tid 4 to smithi06387500-42 2024-08-17T16:57:50.917 INFO:tasks.rados.rados.0.smithi063.stdout:740: finishing write tid 5 to smithi06387500-42 2024-08-17T16:57:50.917 INFO:tasks.rados.rados.0.smithi063.stdout:740: oid 42 updating version 0 to 520 2024-08-17T16:57:50.918 INFO:tasks.rados.rados.0.smithi063.stdout:740: oid 42 version 520 is already newer than 519 2024-08-17T16:57:50.918 INFO:tasks.rados.rados.0.smithi063.stdout:740: oid 42 version 520 is already newer than 518 2024-08-17T16:57:50.918 INFO:tasks.rados.rados.0.smithi063.stdout:740: oid 42 updating version 520 to 521 2024-08-17T16:57:50.918 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 521 (ObjNum 313 snap 94 seq_num 313) dirty exists 2024-08-17T16:57:50.918 INFO:tasks.rados.rados.0.smithi063.stdout:740: left oid 42 (ObjNum 313 snap 94 seq_num 313) 2024-08-17T16:57:50.918 INFO:tasks.rados.rados.0.smithi063.stdout:741: finishing rollback tid 2 to smithi06387500-47 2024-08-17T16:57:50.918 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 142 (ObjNum 79 snap 10 seq_num 79) dirty exists 2024-08-17T16:57:50.918 INFO:tasks.rados.rados.0.smithi063.stdout:742: finishing write tid 1 to smithi06387500-4 2024-08-17T16:57:50.918 INFO:tasks.rados.rados.0.smithi063.stdout:742: finishing write tid 2 to smithi06387500-4 2024-08-17T16:57:50.918 INFO:tasks.rados.rados.0.smithi063.stdout:744: read oid 38 snap -1 2024-08-17T16:57:50.918 INFO:tasks.rados.rados.0.smithi063.stdout:744: expect deleted 2024-08-17T16:57:50.918 INFO:tasks.rados.rados.0.smithi063.stdout:740: done (4 left) 2024-08-17T16:57:50.918 INFO:tasks.rados.rados.0.smithi063.stdout:741: done (3 left) 2024-08-17T16:57:50.919 INFO:tasks.rados.rados.0.smithi063.stdout:745: delete oid 17 current snap is 94 2024-08-17T16:57:50.919 INFO:tasks.rados.rados.0.smithi063.stdout:742: finishing write tid 3 to smithi06387500-4 2024-08-17T16:57:50.919 INFO:tasks.rados.rados.0.smithi063.stdout:742: finishing write tid 4 to smithi06387500-4 2024-08-17T16:57:50.919 INFO:tasks.rados.rados.0.smithi063.stdout:742: finishing write tid 5 to smithi06387500-4 2024-08-17T16:57:50.919 INFO:tasks.rados.rados.0.smithi063.stdout:742: oid 4 updating version 0 to 758 2024-08-17T16:57:50.919 INFO:tasks.rados.rados.0.smithi063.stdout:742: oid 4 updating version 758 to 759 2024-08-17T16:57:50.919 INFO:tasks.rados.rados.0.smithi063.stdout:742: oid 4 updating version 759 to 761 2024-08-17T16:57:50.919 INFO:tasks.rados.rados.0.smithi063.stdout:742: oid 4 version 761 is already newer than 760 2024-08-17T16:57:50.919 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 761 (ObjNum 314 snap 94 seq_num 314) dirty exists 2024-08-17T16:57:50.919 INFO:tasks.rados.rados.0.smithi063.stdout:742: left oid 4 (ObjNum 314 snap 94 seq_num 314) 2024-08-17T16:57:50.919 INFO:tasks.rados.rados.0.smithi063.stdout:743: finishing write tid 1 to smithi06387500-36 2024-08-17T16:57:50.919 INFO:tasks.rados.rados.0.smithi063.stdout:743: finishing write tid 2 to smithi06387500-36 2024-08-17T16:57:50.920 INFO:tasks.rados.rados.0.smithi063.stdout:742: done (3 left) 2024-08-17T16:57:50.920 INFO:tasks.rados.rados.0.smithi063.stdout:745: done (2 left) 2024-08-17T16:57:50.920 INFO:tasks.rados.rados.0.smithi063.stdout:746: snap_create 2024-08-17T16:57:50.920 INFO:tasks.rados.rados.0.smithi063.stdout:743: finishing write tid 3 to smithi06387500-36 2024-08-17T16:57:50.921 INFO:tasks.rados.rados.0.smithi063.stdout:743: finishing write tid 4 to smithi06387500-36 2024-08-17T16:57:50.921 INFO:tasks.rados.rados.0.smithi063.stdout:743: finishing write tid 5 to smithi06387500-36 2024-08-17T16:57:50.921 INFO:tasks.rados.rados.0.smithi063.stdout:743: oid 36 updating version 0 to 765 2024-08-17T16:57:50.921 INFO:tasks.rados.rados.0.smithi063.stdout:743: oid 36 version 765 is already newer than 763 2024-08-17T16:57:50.921 INFO:tasks.rados.rados.0.smithi063.stdout:743: oid 36 version 765 is already newer than 764 2024-08-17T16:57:50.921 INFO:tasks.rados.rados.0.smithi063.stdout:743: oid 36 updating version 765 to 766 2024-08-17T16:57:50.921 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 766 (ObjNum 315 snap 94 seq_num 315) dirty exists 2024-08-17T16:57:50.921 INFO:tasks.rados.rados.0.smithi063.stdout:743: left oid 36 (ObjNum 315 snap 94 seq_num 315) 2024-08-17T16:57:51.141 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:50 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:51.142 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:50 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:51.142 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:50 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:51.142 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:50 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:51.142 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:50 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:51.142 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:50 smithi063 ceph-mon[42680]: osdmap e255: 8 total, 8 up, 8 in 2024-08-17T16:57:51.142 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:50 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:51.142 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:50 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:51.142 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:50 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:51.143 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:50 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:51.143 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:50 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:51.143 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:50 smithi063 ceph-mon[31063]: osdmap e255: 8 total, 8 up, 8 in 2024-08-17T16:57:51.332 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:50 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:51.333 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:50 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:51.333 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:50 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:51.333 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:50 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:51.333 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:50 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:51.333 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:50 smithi162 ceph-mon[34633]: osdmap e255: 8 total, 8 up, 8 in 2024-08-17T16:57:51.867 INFO:tasks.rados.rados.0.smithi063.stdout:743: done (2 left) 2024-08-17T16:57:51.867 INFO:tasks.rados.rados.0.smithi063.stdout:744: done (1 left) 2024-08-17T16:57:51.867 INFO:tasks.rados.rados.0.smithi063.stdout:746: done (0 left) 2024-08-17T16:57:51.867 INFO:tasks.rados.rados.0.smithi063.stdout:747: rollback oid 17 current snap is 95 2024-08-17T16:57:51.867 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 17 to 90 2024-08-17T16:57:51.867 INFO:tasks.rados.rados.0.smithi063.stdout:748: snap_remove snap 89 2024-08-17T16:57:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:51 smithi063 ceph-mon[42680]: pgmap v66: 89 pgs: 89 active+clean; 232 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 4.9 MiB/s wr, 36 op/s 2024-08-17T16:57:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:51 smithi063 ceph-mon[42680]: osdmap e256: 8 total, 8 up, 8 in 2024-08-17T16:57:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:51 smithi063 ceph-mon[31063]: pgmap v66: 89 pgs: 89 active+clean; 232 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 4.9 MiB/s wr, 36 op/s 2024-08-17T16:57:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:51 smithi063 ceph-mon[31063]: osdmap e256: 8 total, 8 up, 8 in 2024-08-17T16:57:52.261 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:51 smithi162 ceph-mon[34633]: pgmap v66: 89 pgs: 89 active+clean; 232 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 4.9 MiB/s wr, 36 op/s 2024-08-17T16:57:52.261 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:51 smithi162 ceph-mon[34633]: osdmap e256: 8 total, 8 up, 8 in 2024-08-17T16:57:52.869 INFO:tasks.rados.rados.0.smithi063.stdout:748: done (1 left) 2024-08-17T16:57:52.869 INFO:tasks.rados.rados.0.smithi063.stdout:749: rollback oid 3 current snap is 95 2024-08-17T16:57:52.869 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 3 to 75 2024-08-17T16:57:52.869 INFO:tasks.rados.rados.0.smithi063.stdout:747: finishing rollback tid 1 to smithi06387500-17 2024-08-17T16:57:52.869 INFO:tasks.rados.rados.0.smithi063.stdout:747: finishing rollback tid 2 to smithi06387500-17 2024-08-17T16:57:52.869 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 230 (ObjNum 254 snap 77 seq_num 254) dirty exists 2024-08-17T16:57:52.869 INFO:tasks.rados.rados.0.smithi063.stdout:747: done (1 left) 2024-08-17T16:57:52.869 INFO:tasks.rados.rados.0.smithi063.stdout:750: delete oid 13 current snap is 95 2024-08-17T16:57:52.877 INFO:tasks.rados.rados.0.smithi063.stdout:749: finishing rollback tid 1 to smithi06387500-3 2024-08-17T16:57:52.877 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 212 (ObjNum 33 snap 0 seq_num 2) dirty dne 2024-08-17T16:57:52.878 INFO:tasks.rados.rados.0.smithi063.stdout:749: done (1 left) 2024-08-17T16:57:52.878 INFO:tasks.rados.rados.0.smithi063.stdout:750: done (0 left) 2024-08-17T16:57:52.878 INFO:tasks.rados.rados.0.smithi063.stdout:751: delete oid 17 current snap is 95 2024-08-17T16:57:52.880 INFO:tasks.rados.rados.0.smithi063.stdout:751: done (0 left) 2024-08-17T16:57:52.880 INFO:tasks.rados.rados.0.smithi063.stdout:752: write oid 27 current snap is 95 2024-08-17T16:57:52.880 INFO:tasks.rados.rados.0.smithi063.stdout:752: seq_num 319 ranges {561433=774904,1950697=414440,2777464=583969} 2024-08-17T16:57:52.887 INFO:tasks.rados.rados.0.smithi063.stdout:752: writing smithi06387500-27 from 561433 to 1336337 tid 1 2024-08-17T16:57:52.893 INFO:tasks.rados.rados.0.smithi063.stdout:752: writing smithi06387500-27 from 1950697 to 2365137 tid 2 2024-08-17T16:57:52.899 INFO:tasks.rados.rados.0.smithi063.stdout:752: writing smithi06387500-27 from 2777464 to 3361433 tid 3 2024-08-17T16:57:52.899 INFO:tasks.rados.rados.0.smithi063.stdout:753: delete oid 39 current snap is 95 2024-08-17T16:57:52.899 INFO:tasks.rados.rados.0.smithi063.stdout:752: finishing write tid 1 to smithi06387500-27 2024-08-17T16:57:52.899 INFO:tasks.rados.rados.0.smithi063.stdout:752: finishing write tid 2 to smithi06387500-27 2024-08-17T16:57:52.901 INFO:tasks.rados.rados.0.smithi063.stdout:753: done (1 left) 2024-08-17T16:57:52.901 INFO:tasks.rados.rados.0.smithi063.stdout:754: snap_create 2024-08-17T16:57:52.904 INFO:tasks.rados.rados.0.smithi063.stdout:752: finishing write tid 3 to smithi06387500-27 2024-08-17T16:57:52.904 INFO:tasks.rados.rados.0.smithi063.stdout:752: finishing write tid 4 to smithi06387500-27 2024-08-17T16:57:52.904 INFO:tasks.rados.rados.0.smithi063.stdout:752: finishing write tid 5 to smithi06387500-27 2024-08-17T16:57:52.904 INFO:tasks.rados.rados.0.smithi063.stdout:752: oid 27 updating version 0 to 118 2024-08-17T16:57:52.904 INFO:tasks.rados.rados.0.smithi063.stdout:752: oid 27 updating version 118 to 120 2024-08-17T16:57:52.905 INFO:tasks.rados.rados.0.smithi063.stdout:752: oid 27 version 120 is already newer than 119 2024-08-17T16:57:52.905 INFO:tasks.rados.rados.0.smithi063.stdout:752: oid 27 version 120 is already newer than 117 2024-08-17T16:57:52.905 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 120 (ObjNum 319 snap 95 seq_num 319) dirty exists 2024-08-17T16:57:52.905 INFO:tasks.rados.rados.0.smithi063.stdout:752: left oid 27 (ObjNum 319 snap 95 seq_num 319) 2024-08-17T16:57:53.881 INFO:tasks.rados.rados.0.smithi063.stdout:752: done (1 left) 2024-08-17T16:57:53.881 INFO:tasks.rados.rados.0.smithi063.stdout:754: done (0 left) 2024-08-17T16:57:53.881 INFO:tasks.rados.rados.0.smithi063.stdout:755: delete oid 21 current snap is 96 2024-08-17T16:57:53.887 INFO:tasks.rados.rados.0.smithi063.stdout:755: done (0 left) 2024-08-17T16:57:53.887 INFO:tasks.rados.rados.0.smithi063.stdout:756: snap_remove snap 86 2024-08-17T16:57:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:53 smithi063 ceph-mon[42680]: osdmap e257: 8 total, 8 up, 8 in 2024-08-17T16:57:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:53 smithi063 ceph-mon[42680]: pgmap v69: 89 pgs: 89 active+clean; 232 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 4.9 MiB/s wr, 36 op/s 2024-08-17T16:57:54.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:53 smithi063 ceph-mon[31063]: osdmap e257: 8 total, 8 up, 8 in 2024-08-17T16:57:54.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:53 smithi063 ceph-mon[31063]: pgmap v69: 89 pgs: 89 active+clean; 232 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 4.9 MiB/s wr, 36 op/s 2024-08-17T16:57:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:53 smithi162 ceph-mon[34633]: osdmap e257: 8 total, 8 up, 8 in 2024-08-17T16:57:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:53 smithi162 ceph-mon[34633]: pgmap v69: 89 pgs: 89 active+clean; 232 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 4.9 MiB/s wr, 36 op/s 2024-08-17T16:57:54.870 INFO:tasks.rados.rados.0.smithi063.stdout:756: done (0 left) 2024-08-17T16:57:54.870 INFO:tasks.rados.rados.0.smithi063.stdout:757: read oid 41 snap 85 2024-08-17T16:57:54.871 INFO:tasks.rados.rados.0.smithi063.stdout:757: expect (ObjNum 73 snap 9 seq_num 73) 2024-08-17T16:57:54.871 INFO:tasks.rados.rados.0.smithi063.stdout:758: write oid 14 current snap is 96 2024-08-17T16:57:54.871 INFO:tasks.rados.rados.0.smithi063.stdout:758: seq_num 322 ranges {611854=717004,2126143=561767,3259733=152121} 2024-08-17T16:57:54.875 INFO:tasks.rados.rados.0.smithi063.stdout:758: writing smithi06387500-14 from 611854 to 1328858 tid 1 2024-08-17T16:57:54.884 INFO:tasks.rados.rados.0.smithi063.stdout:758: writing smithi06387500-14 from 2126143 to 2687910 tid 2 2024-08-17T16:57:54.888 INFO:tasks.rados.rados.0.smithi063.stdout:758: writing smithi06387500-14 from 3259733 to 3411854 tid 3 2024-08-17T16:57:54.888 INFO:tasks.rados.rados.0.smithi063.stdout:759: read oid 10 snap 87 2024-08-17T16:57:54.888 INFO:tasks.rados.rados.0.smithi063.stdout:759: expect deleted 2024-08-17T16:57:54.888 INFO:tasks.rados.rados.0.smithi063.stdout:760: read oid 6 snap -1 2024-08-17T16:57:54.888 INFO:tasks.rados.rados.0.smithi063.stdout:760: expect (ObjNum 178 snap 47 seq_num 178) 2024-08-17T16:57:54.888 INFO:tasks.rados.rados.0.smithi063.stdout:761: snap_create 2024-08-17T16:57:54.888 INFO:tasks.rados.rados.0.smithi063.stdout:757: expect (ObjNum 73 snap 9 seq_num 73) 2024-08-17T16:57:55.032 INFO:tasks.rados.rados.0.smithi063.stdout:758: finishing write tid 1 to smithi06387500-14 2024-08-17T16:57:55.032 INFO:tasks.rados.rados.0.smithi063.stdout:758: finishing write tid 2 to smithi06387500-14 2024-08-17T16:57:55.032 INFO:tasks.rados.rados.0.smithi063.stdout:758: finishing write tid 3 to smithi06387500-14 2024-08-17T16:57:55.032 INFO:tasks.rados.rados.0.smithi063.stdout:758: finishing write tid 4 to smithi06387500-14 2024-08-17T16:57:55.032 INFO:tasks.rados.rados.0.smithi063.stdout:758: finishing write tid 5 to smithi06387500-14 2024-08-17T16:57:55.032 INFO:tasks.rados.rados.0.smithi063.stdout:758: oid 14 updating version 0 to 781 2024-08-17T16:57:55.032 INFO:tasks.rados.rados.0.smithi063.stdout:758: oid 14 updating version 781 to 782 2024-08-17T16:57:55.033 INFO:tasks.rados.rados.0.smithi063.stdout:758: oid 14 version 782 is already newer than 779 2024-08-17T16:57:55.033 INFO:tasks.rados.rados.0.smithi063.stdout:758: oid 14 version 782 is already newer than 780 2024-08-17T16:57:55.033 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 782 (ObjNum 322 snap 96 seq_num 322) dirty exists 2024-08-17T16:57:55.033 INFO:tasks.rados.rados.0.smithi063.stdout:758: left oid 14 (ObjNum 322 snap 96 seq_num 322) 2024-08-17T16:57:55.033 INFO:tasks.rados.rados.0.smithi063.stdout:760: expect (ObjNum 178 snap 47 seq_num 178) 2024-08-17T16:57:55.152 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[42680]: osdmap e258: 8 total, 8 up, 8 in 2024-08-17T16:57:55.152 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.152 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.152 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.153 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.153 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:55.153 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.153 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.153 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.153 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.153 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[42680]: osdmap e259: 8 total, 8 up, 8 in 2024-08-17T16:57:55.153 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[31063]: osdmap e258: 8 total, 8 up, 8 in 2024-08-17T16:57:55.153 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.153 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.153 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.154 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.154 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:55.154 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.154 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.154 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.154 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.154 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:54 smithi063 ceph-mon[31063]: osdmap e259: 8 total, 8 up, 8 in 2024-08-17T16:57:55.227 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:54 smithi162 ceph-mon[34633]: osdmap e258: 8 total, 8 up, 8 in 2024-08-17T16:57:55.227 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:54 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.227 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:54 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.227 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:54 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.227 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:54 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.227 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:54 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:57:55.227 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:54 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.228 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:54 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.228 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:54 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.228 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:54 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:55.228 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:54 smithi162 ceph-mon[34633]: osdmap e259: 8 total, 8 up, 8 in 2024-08-17T16:57:55.891 INFO:tasks.rados.rados.0.smithi063.stdout:757: done (4 left) 2024-08-17T16:57:55.891 INFO:tasks.rados.rados.0.smithi063.stdout:758: done (3 left) 2024-08-17T16:57:55.891 INFO:tasks.rados.rados.0.smithi063.stdout:759: done (2 left) 2024-08-17T16:57:55.891 INFO:tasks.rados.rados.0.smithi063.stdout:760: done (1 left) 2024-08-17T16:57:55.891 INFO:tasks.rados.rados.0.smithi063.stdout:761: done (0 left) 2024-08-17T16:57:55.891 INFO:tasks.rados.rados.0.smithi063.stdout:762: snap_create 2024-08-17T16:57:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:55 smithi063 ceph-mon[42680]: pgmap v72: 89 pgs: 89 active+clean; 232 MiB data, 750 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:55 smithi063 ceph-mon[31063]: pgmap v72: 89 pgs: 89 active+clean; 232 MiB data, 750 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:55 smithi162 ceph-mon[34633]: pgmap v72: 89 pgs: 89 active+clean; 232 MiB data, 750 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:57:56.890 INFO:tasks.rados.rados.0.smithi063.stdout:762: done (0 left) 2024-08-17T16:57:56.890 INFO:tasks.rados.rados.0.smithi063.stdout:763: write oid 42 current snap is 98 2024-08-17T16:57:56.890 INFO:tasks.rados.rados.0.smithi063.stdout:763: seq_num 323 ranges {628661=792253,2184625=421778,3403509=25152} 2024-08-17T16:57:56.897 INFO:tasks.rados.rados.0.smithi063.stdout:763: writing smithi06387500-42 from 628661 to 1420914 tid 1 2024-08-17T16:57:56.904 INFO:tasks.rados.rados.0.smithi063.stdout:763: writing smithi06387500-42 from 2184625 to 2606403 tid 2 2024-08-17T16:57:56.909 INFO:tasks.rados.rados.0.smithi063.stdout:763: writing smithi06387500-42 from 3403509 to 3428661 tid 3 2024-08-17T16:57:56.909 INFO:tasks.rados.rados.0.smithi063.stdout:764: read oid 4 snap 60 2024-08-17T16:57:56.909 INFO:tasks.rados.rados.0.smithi063.stdout:764: expect (ObjNum 192 snap 50 seq_num 192) 2024-08-17T16:57:56.910 INFO:tasks.rados.rados.0.smithi063.stdout:765: rollback oid 35 current snap is 98 2024-08-17T16:57:56.910 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 35 to 63 2024-08-17T16:57:56.910 INFO:tasks.rados.rados.0.smithi063.stdout:763: finishing write tid 1 to smithi06387500-42 2024-08-17T16:57:56.910 INFO:tasks.rados.rados.0.smithi063.stdout:763: finishing write tid 2 to smithi06387500-42 2024-08-17T16:57:56.910 INFO:tasks.rados.rados.0.smithi063.stdout:766: snap_create 2024-08-17T16:57:56.911 INFO:tasks.rados.rados.0.smithi063.stdout:763: finishing write tid 3 to smithi06387500-42 2024-08-17T16:57:56.911 INFO:tasks.rados.rados.0.smithi063.stdout:763: finishing write tid 4 to smithi06387500-42 2024-08-17T16:57:56.911 INFO:tasks.rados.rados.0.smithi063.stdout:763: finishing write tid 5 to smithi06387500-42 2024-08-17T16:57:56.911 INFO:tasks.rados.rados.0.smithi063.stdout:763: oid 42 updating version 0 to 546 2024-08-17T16:57:56.911 INFO:tasks.rados.rados.0.smithi063.stdout:763: oid 42 version 546 is already newer than 545 2024-08-17T16:57:56.911 INFO:tasks.rados.rados.0.smithi063.stdout:763: oid 42 version 546 is already newer than 544 2024-08-17T16:57:56.911 INFO:tasks.rados.rados.0.smithi063.stdout:763: oid 42 version 546 is already newer than 543 2024-08-17T16:57:56.911 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 546 (ObjNum 323 snap 98 seq_num 323) dirty exists 2024-08-17T16:57:56.911 INFO:tasks.rados.rados.0.smithi063.stdout:763: left oid 42 (ObjNum 323 snap 98 seq_num 323) 2024-08-17T16:57:56.920 INFO:tasks.rados.rados.0.smithi063.stdout:764: expect (ObjNum 192 snap 50 seq_num 192) 2024-08-17T16:57:57.062 INFO:tasks.rados.rados.0.smithi063.stdout:765: finishing rollback tid 0 to smithi06387500-35 2024-08-17T16:57:57.062 INFO:tasks.rados.rados.0.smithi063.stdout:765: finishing rollback tid 1 to smithi06387500-35 2024-08-17T16:57:57.063 INFO:tasks.rados.rados.0.smithi063.stdout:765: finishing rollback tid 2 to smithi06387500-35 2024-08-17T16:57:57.063 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 786 (ObjNum 179 snap 47 seq_num 179) dirty exists 2024-08-17T16:57:57.144 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:56 smithi063 ceph-mon[31063]: osdmap e260: 8 total, 8 up, 8 in 2024-08-17T16:57:57.144 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:56 smithi063 ceph-mon[42680]: osdmap e260: 8 total, 8 up, 8 in 2024-08-17T16:57:57.157 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:56 smithi162 ceph-mon[34633]: osdmap e260: 8 total, 8 up, 8 in 2024-08-17T16:57:57.903 INFO:tasks.rados.rados.0.smithi063.stdout:763: done (3 left) 2024-08-17T16:57:57.903 INFO:tasks.rados.rados.0.smithi063.stdout:764: done (2 left) 2024-08-17T16:57:57.904 INFO:tasks.rados.rados.0.smithi063.stdout:765: done (1 left) 2024-08-17T16:57:57.904 INFO:tasks.rados.rados.0.smithi063.stdout:766: done (0 left) 2024-08-17T16:57:57.904 INFO:tasks.rados.rados.0.smithi063.stdout:767: write oid 19 current snap is 99 2024-08-17T16:57:57.904 INFO:tasks.rados.rados.0.smithi063.stdout:767: seq_num 324 ranges {645468=467502,1726754=565436,2798226=596838} 2024-08-17T16:57:57.909 INFO:tasks.rados.rados.0.smithi063.stdout:767: writing smithi06387500-19 from 645468 to 1112970 tid 1 2024-08-17T16:57:57.916 INFO:tasks.rados.rados.0.smithi063.stdout:767: writing smithi06387500-19 from 1726754 to 2292190 tid 2 2024-08-17T16:57:57.923 INFO:tasks.rados.rados.0.smithi063.stdout:767: writing smithi06387500-19 from 2798226 to 3395064 tid 3 2024-08-17T16:57:57.923 INFO:tasks.rados.rados.0.smithi063.stdout:768: write oid 44 current snap is 99 2024-08-17T16:57:57.923 INFO:tasks.rados.rados.0.smithi063.stdout:767: finishing write tid 1 to smithi06387500-19 2024-08-17T16:57:57.923 INFO:tasks.rados.rados.0.smithi063.stdout:768: seq_num 325 ranges {662275=542751,1785236=425447,2942002=520273} 2024-08-17T16:57:57.929 INFO:tasks.rados.rados.0.smithi063.stdout:768: writing smithi06387500-44 from 662275 to 1205026 tid 1 2024-08-17T16:57:57.935 INFO:tasks.rados.rados.0.smithi063.stdout:768: writing smithi06387500-44 from 1785236 to 2210683 tid 2 2024-08-17T16:57:57.942 INFO:tasks.rados.rados.0.smithi063.stdout:768: writing smithi06387500-44 from 2942002 to 3462275 tid 3 2024-08-17T16:57:57.942 INFO:tasks.rados.rados.0.smithi063.stdout:769: snap_remove snap 97 2024-08-17T16:57:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:57 smithi162 ceph-mon[34633]: osdmap e261: 8 total, 8 up, 8 in 2024-08-17T16:57:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:57 smithi162 ceph-mon[34633]: pgmap v75: 89 pgs: 89 active+clean; 241 MiB data, 763 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.9 MiB/s wr, 14 op/s 2024-08-17T16:57:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:57 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:57 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:57 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:57 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:57 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:57 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:58.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[42680]: osdmap e261: 8 total, 8 up, 8 in 2024-08-17T16:57:58.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[42680]: pgmap v75: 89 pgs: 89 active+clean; 241 MiB data, 763 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.9 MiB/s wr, 14 op/s 2024-08-17T16:57:58.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:58.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:58.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:58.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:58.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:58.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:58.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[31063]: osdmap e261: 8 total, 8 up, 8 in 2024-08-17T16:57:58.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[31063]: pgmap v75: 89 pgs: 89 active+clean; 241 MiB data, 763 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.9 MiB/s wr, 14 op/s 2024-08-17T16:57:58.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:58.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:58.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:58.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:58.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:57:58.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:57 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:57:58.908 INFO:tasks.rados.rados.0.smithi063.stdout:769: done (2 left) 2024-08-17T16:57:58.908 INFO:tasks.rados.rados.0.smithi063.stdout:770: delete oid 41 current snap is 99 2024-08-17T16:57:58.908 INFO:tasks.rados.rados.0.smithi063.stdout:767: finishing write tid 2 to smithi06387500-19 2024-08-17T16:57:58.908 INFO:tasks.rados.rados.0.smithi063.stdout:767: finishing write tid 3 to smithi06387500-19 2024-08-17T16:57:58.908 INFO:tasks.rados.rados.0.smithi063.stdout:767: finishing write tid 4 to smithi06387500-19 2024-08-17T16:57:58.908 INFO:tasks.rados.rados.0.smithi063.stdout:767: finishing write tid 5 to smithi06387500-19 2024-08-17T16:57:58.908 INFO:tasks.rados.rados.0.smithi063.stdout:767: oid 19 updating version 0 to 550 2024-08-17T16:57:58.908 INFO:tasks.rados.rados.0.smithi063.stdout:767: oid 19 version 550 is already newer than 549 2024-08-17T16:57:58.908 INFO:tasks.rados.rados.0.smithi063.stdout:767: oid 19 version 550 is already newer than 548 2024-08-17T16:57:58.908 INFO:tasks.rados.rados.0.smithi063.stdout:767: oid 19 version 550 is already newer than 547 2024-08-17T16:57:58.908 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 550 (ObjNum 324 snap 99 seq_num 324) dirty exists 2024-08-17T16:57:58.908 INFO:tasks.rados.rados.0.smithi063.stdout:767: left oid 19 (ObjNum 324 snap 99 seq_num 324) 2024-08-17T16:57:58.909 INFO:tasks.rados.rados.0.smithi063.stdout:768: finishing write tid 1 to smithi06387500-44 2024-08-17T16:57:58.909 INFO:tasks.rados.rados.0.smithi063.stdout:768: finishing write tid 2 to smithi06387500-44 2024-08-17T16:57:58.909 INFO:tasks.rados.rados.0.smithi063.stdout:768: finishing write tid 3 to smithi06387500-44 2024-08-17T16:57:58.909 INFO:tasks.rados.rados.0.smithi063.stdout:768: finishing write tid 4 to smithi06387500-44 2024-08-17T16:57:58.909 INFO:tasks.rados.rados.0.smithi063.stdout:768: finishing write tid 5 to smithi06387500-44 2024-08-17T16:57:58.909 INFO:tasks.rados.rados.0.smithi063.stdout:768: oid 44 updating version 0 to 147 2024-08-17T16:57:58.909 INFO:tasks.rados.rados.0.smithi063.stdout:768: oid 44 updating version 147 to 150 2024-08-17T16:57:58.909 INFO:tasks.rados.rados.0.smithi063.stdout:768: oid 44 version 150 is already newer than 148 2024-08-17T16:57:58.909 INFO:tasks.rados.rados.0.smithi063.stdout:768: oid 44 version 150 is already newer than 149 2024-08-17T16:57:58.909 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 150 (ObjNum 325 snap 99 seq_num 325) dirty exists 2024-08-17T16:57:58.909 INFO:tasks.rados.rados.0.smithi063.stdout:768: left oid 44 (ObjNum 325 snap 99 seq_num 325) 2024-08-17T16:57:58.910 INFO:tasks.rados.rados.0.smithi063.stdout:767: done (2 left) 2024-08-17T16:57:58.910 INFO:tasks.rados.rados.0.smithi063.stdout:768: done (1 left) 2024-08-17T16:57:58.910 INFO:tasks.rados.rados.0.smithi063.stdout:770: done (0 left) 2024-08-17T16:57:58.910 INFO:tasks.rados.rados.0.smithi063.stdout:771: snap_remove snap 91 2024-08-17T16:57:59.077 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:58 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:59.077 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:58 smithi162 ceph-mon[34633]: osdmap e262: 8 total, 8 up, 8 in 2024-08-17T16:57:59.077 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:58 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:59.078 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:58 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.078 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:58 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.078 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:58 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.078 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:58 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.078 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:58 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.078 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:58 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.078 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:58 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.078 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:58 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.078 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:58 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.078 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:58 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[42680]: osdmap e262: 8 total, 8 up, 8 in 2024-08-17T16:57:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:57:59.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[31063]: osdmap e262: 8 total, 8 up, 8 in 2024-08-17T16:57:59.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:57:59.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:58 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:57:59.864 INFO:tasks.rados.rados.0.smithi063.stdout:771: done (0 left) 2024-08-17T16:57:59.864 INFO:tasks.rados.rados.0.smithi063.stdout:772: write oid 24 current snap is 99 2024-08-17T16:57:59.864 INFO:tasks.rados.rados.0.smithi063.stdout:772: seq_num 327 ranges {695889=409602,1618553=712763,2996848=499041} 2024-08-17T16:57:59.871 INFO:tasks.rados.rados.0.smithi063.stdout:772: writing smithi06387500-24 from 695889 to 1105491 tid 1 2024-08-17T16:57:59.878 INFO:tasks.rados.rados.0.smithi063.stdout:772: writing smithi06387500-24 from 1618553 to 2331316 tid 2 2024-08-17T16:57:59.885 INFO:tasks.rados.rados.0.smithi063.stdout:772: writing smithi06387500-24 from 2996848 to 3495889 tid 3 2024-08-17T16:57:59.885 INFO:tasks.rados.rados.0.smithi063.stdout:773: read oid 19 snap 88 2024-08-17T16:57:59.885 INFO:tasks.rados.rados.0.smithi063.stdout:773: expect deleted 2024-08-17T16:57:59.885 INFO:tasks.rados.rados.0.smithi063.stdout:774: read oid 3 snap 95 2024-08-17T16:57:59.885 INFO:tasks.rados.rados.0.smithi063.stdout:774: expect deleted 2024-08-17T16:57:59.885 INFO:tasks.rados.rados.0.smithi063.stdout:775: read oid 43 snap -1 2024-08-17T16:57:59.885 INFO:tasks.rados.rados.0.smithi063.stdout:775: expect (ObjNum 287 snap 87 seq_num 287) 2024-08-17T16:57:59.885 INFO:tasks.rados.rados.0.smithi063.stdout:772: finishing write tid 1 to smithi06387500-24 2024-08-17T16:57:59.886 INFO:tasks.rados.rados.0.smithi063.stdout:772: finishing write tid 2 to smithi06387500-24 2024-08-17T16:57:59.886 INFO:tasks.rados.rados.0.smithi063.stdout:776: read oid 46 snap -1 2024-08-17T16:57:59.886 INFO:tasks.rados.rados.0.smithi063.stdout:776: expect (ObjNum 293 snap 91 seq_num 293) 2024-08-17T16:57:59.886 INFO:tasks.rados.rados.0.smithi063.stdout:777: rollback oid 4 current snap is 99 2024-08-17T16:57:59.886 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 4 to 94 2024-08-17T16:57:59.886 INFO:tasks.rados.rados.0.smithi063.stdout:778: write oid 18 current snap is 99 2024-08-17T16:57:59.886 INFO:tasks.rados.rados.0.smithi063.stdout:778: seq_num 328 ranges {712696=484851,1960682=572774,3307918=204778} 2024-08-17T16:57:59.891 INFO:tasks.rados.rados.0.smithi063.stdout:778: writing smithi06387500-18 from 712696 to 1197547 tid 1 2024-08-17T16:57:59.899 INFO:tasks.rados.rados.0.smithi063.stdout:778: writing smithi06387500-18 from 1960682 to 2533456 tid 2 2024-08-17T16:57:59.901 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:57:59 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:57:59] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.43.0" 2024-08-17T16:57:59.905 INFO:tasks.rados.rados.0.smithi063.stdout:778: writing smithi06387500-18 from 3307918 to 3512696 tid 3 2024-08-17T16:57:59.905 INFO:tasks.rados.rados.0.smithi063.stdout:779: snap_create 2024-08-17T16:57:59.905 INFO:tasks.rados.rados.0.smithi063.stdout:777: finishing rollback tid 0 to smithi06387500-4 2024-08-17T16:57:59.905 INFO:tasks.rados.rados.0.smithi063.stdout:777: finishing rollback tid 1 to smithi06387500-4 2024-08-17T16:57:59.905 INFO:tasks.rados.rados.0.smithi063.stdout:777: finishing rollback tid 2 to smithi06387500-4 2024-08-17T16:57:59.905 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 798 (ObjNum 314 snap 94 seq_num 314) dirty exists 2024-08-17T16:57:59.905 INFO:tasks.rados.rados.0.smithi063.stdout:772: finishing write tid 3 to smithi06387500-24 2024-08-17T16:57:59.905 INFO:tasks.rados.rados.0.smithi063.stdout:772: finishing write tid 4 to smithi06387500-24 2024-08-17T16:57:59.905 INFO:tasks.rados.rados.0.smithi063.stdout:772: finishing write tid 5 to smithi06387500-24 2024-08-17T16:57:59.905 INFO:tasks.rados.rados.0.smithi063.stdout:772: oid 24 updating version 0 to 297 2024-08-17T16:57:59.906 INFO:tasks.rados.rados.0.smithi063.stdout:772: oid 24 updating version 297 to 299 2024-08-17T16:57:59.906 INFO:tasks.rados.rados.0.smithi063.stdout:772: oid 24 updating version 299 to 300 2024-08-17T16:57:59.906 INFO:tasks.rados.rados.0.smithi063.stdout:772: oid 24 version 300 is already newer than 298 2024-08-17T16:57:59.906 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 300 (ObjNum 327 snap 99 seq_num 327) dirty exists 2024-08-17T16:57:59.906 INFO:tasks.rados.rados.0.smithi063.stdout:772: left oid 24 (ObjNum 327 snap 99 seq_num 327) 2024-08-17T16:57:59.906 INFO:tasks.rados.rados.0.smithi063.stdout:776: expect (ObjNum 293 snap 91 seq_num 293) 2024-08-17T16:58:00.067 INFO:tasks.rados.rados.0.smithi063.stdout:775: expect (ObjNum 287 snap 87 seq_num 287) 2024-08-17T16:58:00.219 INFO:tasks.rados.rados.0.smithi063.stdout:778: finishing write tid 1 to smithi06387500-18 2024-08-17T16:58:00.219 INFO:tasks.rados.rados.0.smithi063.stdout:778: finishing write tid 2 to smithi06387500-18 2024-08-17T16:58:00.219 INFO:tasks.rados.rados.0.smithi063.stdout:778: finishing write tid 3 to smithi06387500-18 2024-08-17T16:58:00.219 INFO:tasks.rados.rados.0.smithi063.stdout:778: finishing write tid 4 to smithi06387500-18 2024-08-17T16:58:00.219 INFO:tasks.rados.rados.0.smithi063.stdout:778: finishing write tid 5 to smithi06387500-18 2024-08-17T16:58:00.220 INFO:tasks.rados.rados.0.smithi063.stdout:778: oid 18 updating version 0 to 266 2024-08-17T16:58:00.220 INFO:tasks.rados.rados.0.smithi063.stdout:778: oid 18 updating version 266 to 267 2024-08-17T16:58:00.220 INFO:tasks.rados.rados.0.smithi063.stdout:778: oid 18 updating version 267 to 269 2024-08-17T16:58:00.220 INFO:tasks.rados.rados.0.smithi063.stdout:778: oid 18 version 269 is already newer than 268 2024-08-17T16:58:00.220 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 269 (ObjNum 328 snap 99 seq_num 328) dirty exists 2024-08-17T16:58:00.220 INFO:tasks.rados.rados.0.smithi063.stdout:778: left oid 18 (ObjNum 328 snap 99 seq_num 328) 2024-08-17T16:58:00.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:59 smithi063 ceph-mon[42680]: pgmap v77: 89 pgs: 89 active+clean; 241 MiB data, 763 MiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.9 MiB/s wr, 14 op/s 2024-08-17T16:58:00.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:59 smithi063 ceph-mon[42680]: osdmap e263: 8 total, 8 up, 8 in 2024-08-17T16:58:00.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:57:59 smithi063 ceph-mon[42680]: osdmap e264: 8 total, 8 up, 8 in 2024-08-17T16:58:00.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:59 smithi063 ceph-mon[31063]: pgmap v77: 89 pgs: 89 active+clean; 241 MiB data, 763 MiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.9 MiB/s wr, 14 op/s 2024-08-17T16:58:00.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:59 smithi063 ceph-mon[31063]: osdmap e263: 8 total, 8 up, 8 in 2024-08-17T16:58:00.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:57:59 smithi063 ceph-mon[31063]: osdmap e264: 8 total, 8 up, 8 in 2024-08-17T16:58:00.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:59 smithi162 ceph-mon[34633]: pgmap v77: 89 pgs: 89 active+clean; 241 MiB data, 763 MiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 1.9 MiB/s wr, 14 op/s 2024-08-17T16:58:00.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:59 smithi162 ceph-mon[34633]: osdmap e263: 8 total, 8 up, 8 in 2024-08-17T16:58:00.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:57:59 smithi162 ceph-mon[34633]: osdmap e264: 8 total, 8 up, 8 in 2024-08-17T16:58:00.908 INFO:tasks.rados.rados.0.smithi063.stdout:772: done (7 left) 2024-08-17T16:58:00.908 INFO:tasks.rados.rados.0.smithi063.stdout:773: done (6 left) 2024-08-17T16:58:00.908 INFO:tasks.rados.rados.0.smithi063.stdout:774: done (5 left) 2024-08-17T16:58:00.908 INFO:tasks.rados.rados.0.smithi063.stdout:775: done (4 left) 2024-08-17T16:58:00.908 INFO:tasks.rados.rados.0.smithi063.stdout:776: done (3 left) 2024-08-17T16:58:00.908 INFO:tasks.rados.rados.0.smithi063.stdout:777: done (2 left) 2024-08-17T16:58:00.909 INFO:tasks.rados.rados.0.smithi063.stdout:778: done (1 left) 2024-08-17T16:58:00.909 INFO:tasks.rados.rados.0.smithi063.stdout:779: done (0 left) 2024-08-17T16:58:00.909 INFO:tasks.rados.rados.0.smithi063.stdout:780: snap_remove snap 60 2024-08-17T16:58:01.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:00 smithi063 ceph-mon[42680]: Upgrade: Updating prometheus.a 2024-08-17T16:58:01.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:00 smithi063 ceph-mon[42680]: Deploying daemon prometheus.a on smithi162 2024-08-17T16:58:01.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:00 smithi063 ceph-mon[31063]: Upgrade: Updating prometheus.a 2024-08-17T16:58:01.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:00 smithi063 ceph-mon[31063]: Deploying daemon prometheus.a on smithi162 2024-08-17T16:58:01.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:00 smithi162 ceph-mon[34633]: Upgrade: Updating prometheus.a 2024-08-17T16:58:01.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:00 smithi162 ceph-mon[34633]: Deploying daemon prometheus.a on smithi162 2024-08-17T16:58:01.913 INFO:tasks.rados.rados.0.smithi063.stdout:780: done (0 left) 2024-08-17T16:58:01.913 INFO:tasks.rados.rados.0.smithi063.stdout:781: read oid 47 snap 98 2024-08-17T16:58:01.913 INFO:tasks.rados.rados.0.smithi063.stdout:781: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:58:01.913 INFO:tasks.rados.rados.0.smithi063.stdout:782: read oid 28 snap 94 2024-08-17T16:58:01.913 INFO:tasks.rados.rados.0.smithi063.stdout:782: expect (ObjNum 301 snap 91 seq_num 301) 2024-08-17T16:58:01.914 INFO:tasks.rados.rados.0.smithi063.stdout:783: snap_remove snap 85 2024-08-17T16:58:02.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:01 smithi063 ceph-mon[42680]: pgmap v80: 89 pgs: 89 active+clean; 247 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-08-17T16:58:02.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:01 smithi063 ceph-mon[42680]: osdmap e265: 8 total, 8 up, 8 in 2024-08-17T16:58:02.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:01 smithi063 ceph-mon[42680]: osdmap e266: 8 total, 8 up, 8 in 2024-08-17T16:58:02.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:01 smithi063 ceph-mon[31063]: pgmap v80: 89 pgs: 89 active+clean; 247 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-08-17T16:58:02.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:01 smithi063 ceph-mon[31063]: osdmap e265: 8 total, 8 up, 8 in 2024-08-17T16:58:02.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:01 smithi063 ceph-mon[31063]: osdmap e266: 8 total, 8 up, 8 in 2024-08-17T16:58:02.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:01 smithi162 ceph-mon[34633]: pgmap v80: 89 pgs: 89 active+clean; 247 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-08-17T16:58:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:01 smithi162 ceph-mon[34633]: osdmap e265: 8 total, 8 up, 8 in 2024-08-17T16:58:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:01 smithi162 ceph-mon[34633]: osdmap e266: 8 total, 8 up, 8 in 2024-08-17T16:58:02.912 INFO:tasks.rados.rados.0.smithi063.stdout:782: expect (ObjNum 301 snap 91 seq_num 301) 2024-08-17T16:58:03.053 INFO:tasks.rados.rados.0.smithi063.stdout:781: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:58:03.210 INFO:tasks.rados.rados.0.smithi063.stdout:781: done (2 left) 2024-08-17T16:58:03.210 INFO:tasks.rados.rados.0.smithi063.stdout:782: done (1 left) 2024-08-17T16:58:03.210 INFO:tasks.rados.rados.0.smithi063.stdout:783: done (0 left) 2024-08-17T16:58:03.210 INFO:tasks.rados.rados.0.smithi063.stdout:784: write oid 22 current snap is 100 2024-08-17T16:58:03.210 INFO:tasks.rados.rados.0.smithi063.stdout:784: seq_num 329 ranges {729503=560100,2019164=716432,3335341=194162} 2024-08-17T16:58:03.214 INFO:tasks.rados.rados.0.smithi063.stdout:784: writing smithi06387500-22 from 729503 to 1289603 tid 1 2024-08-17T16:58:03.222 INFO:tasks.rados.rados.0.smithi063.stdout:784: writing smithi06387500-22 from 2019164 to 2735596 tid 2 2024-08-17T16:58:03.227 INFO:tasks.rados.rados.0.smithi063.stdout:784: writing smithi06387500-22 from 3335341 to 3529503 tid 3 2024-08-17T16:58:03.227 INFO:tasks.rados.rados.0.smithi063.stdout:785: snap_create 2024-08-17T16:58:03.227 INFO:tasks.rados.rados.0.smithi063.stdout:784: finishing write tid 1 to smithi06387500-22 2024-08-17T16:58:03.227 INFO:tasks.rados.rados.0.smithi063.stdout:784: finishing write tid 2 to smithi06387500-22 2024-08-17T16:58:03.229 INFO:tasks.rados.rados.0.smithi063.stdout:784: finishing write tid 3 to smithi06387500-22 2024-08-17T16:58:03.229 INFO:tasks.rados.rados.0.smithi063.stdout:784: finishing write tid 4 to smithi06387500-22 2024-08-17T16:58:03.229 INFO:tasks.rados.rados.0.smithi063.stdout:784: finishing write tid 5 to smithi06387500-22 2024-08-17T16:58:03.229 INFO:tasks.rados.rados.0.smithi063.stdout:784: oid 22 updating version 0 to 244 2024-08-17T16:58:03.229 INFO:tasks.rados.rados.0.smithi063.stdout:784: oid 22 updating version 244 to 247 2024-08-17T16:58:03.229 INFO:tasks.rados.rados.0.smithi063.stdout:784: oid 22 version 247 is already newer than 245 2024-08-17T16:58:03.229 INFO:tasks.rados.rados.0.smithi063.stdout:784: oid 22 version 247 is already newer than 246 2024-08-17T16:58:03.229 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 247 (ObjNum 329 snap 100 seq_num 329) dirty exists 2024-08-17T16:58:03.229 INFO:tasks.rados.rados.0.smithi063.stdout:784: left oid 22 (ObjNum 329 snap 100 seq_num 329) 2024-08-17T16:58:03.926 INFO:tasks.rados.rados.0.smithi063.stdout:784: done (1 left) 2024-08-17T16:58:03.927 INFO:tasks.rados.rados.0.smithi063.stdout:785: done (0 left) 2024-08-17T16:58:03.927 INFO:tasks.rados.rados.0.smithi063.stdout:786: delete oid 3 current snap is 101 2024-08-17T16:58:03.931 INFO:tasks.rados.rados.0.smithi063.stdout:786: done (0 left) 2024-08-17T16:58:03.931 INFO:tasks.rados.rados.0.smithi063.stdout:787: delete oid 4 current snap is 101 2024-08-17T16:58:03.933 INFO:tasks.rados.rados.0.smithi063.stdout:787: done (0 left) 2024-08-17T16:58:03.934 INFO:tasks.rados.rados.0.smithi063.stdout:788: read oid 46 snap 100 2024-08-17T16:58:03.934 INFO:tasks.rados.rados.0.smithi063.stdout:788: expect (ObjNum 293 snap 91 seq_num 293) 2024-08-17T16:58:03.934 INFO:tasks.rados.rados.0.smithi063.stdout:789: delete oid 9 current snap is 101 2024-08-17T16:58:03.936 INFO:tasks.rados.rados.0.smithi063.stdout:789: done (1 left) 2024-08-17T16:58:03.936 INFO:tasks.rados.rados.0.smithi063.stdout:790: read oid 3 snap 87 2024-08-17T16:58:03.936 INFO:tasks.rados.rados.0.smithi063.stdout:790: expect deleted 2024-08-17T16:58:03.937 INFO:tasks.rados.rados.0.smithi063.stdout:791: read oid 39 snap -1 2024-08-17T16:58:03.937 INFO:tasks.rados.rados.0.smithi063.stdout:791: expect deleted 2024-08-17T16:58:03.937 INFO:tasks.rados.rados.0.smithi063.stdout:792: snap_remove snap 93 2024-08-17T16:58:04.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:03 smithi063 ceph-mon[42680]: pgmap v83: 89 pgs: 89 active+clean; 247 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-08-17T16:58:04.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:03 smithi063 ceph-mon[42680]: osdmap e267: 8 total, 8 up, 8 in 2024-08-17T16:58:04.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:03 smithi063 ceph-mon[31063]: pgmap v83: 89 pgs: 89 active+clean; 247 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-08-17T16:58:04.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:03 smithi063 ceph-mon[31063]: osdmap e267: 8 total, 8 up, 8 in 2024-08-17T16:58:04.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:03 smithi162 ceph-mon[34633]: pgmap v83: 89 pgs: 89 active+clean; 247 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-08-17T16:58:04.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:03 smithi162 ceph-mon[34633]: osdmap e267: 8 total, 8 up, 8 in 2024-08-17T16:58:04.872 INFO:tasks.rados.rados.0.smithi063.stdout:788: expect (ObjNum 293 snap 91 seq_num 293) 2024-08-17T16:58:05.035 INFO:tasks.rados.rados.0.smithi063.stdout:788: done (3 left) 2024-08-17T16:58:05.035 INFO:tasks.rados.rados.0.smithi063.stdout:790: done (2 left) 2024-08-17T16:58:05.035 INFO:tasks.rados.rados.0.smithi063.stdout:791: done (1 left) 2024-08-17T16:58:05.036 INFO:tasks.rados.rados.0.smithi063.stdout:792: done (0 left) 2024-08-17T16:58:05.036 INFO:tasks.rados.rados.0.smithi063.stdout:793: rollback oid 5 current snap is 101 2024-08-17T16:58:05.036 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 88 2024-08-17T16:58:05.036 INFO:tasks.rados.rados.0.smithi063.stdout:794: write oid 8 current snap is 101 2024-08-17T16:58:05.036 INFO:tasks.rados.rados.0.smithi063.stdout:794: seq_num 333 ranges {796731=461096,1736739=723770,2928680=668051} 2024-08-17T16:58:05.042 INFO:tasks.rados.rados.0.smithi063.stdout:794: writing smithi06387500-8 from 796731 to 1257827 tid 1 2024-08-17T16:58:05.048 INFO:tasks.rados.rados.0.smithi063.stdout:794: writing smithi06387500-8 from 1736739 to 2460509 tid 2 2024-08-17T16:58:05.054 INFO:tasks.rados.rados.0.smithi063.stdout:794: writing smithi06387500-8 from 2928680 to 3596731 tid 3 2024-08-17T16:58:05.055 INFO:tasks.rados.rados.0.smithi063.stdout:795: snap_remove snap 94 2024-08-17T16:58:05.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:04 smithi063 ceph-mon[42680]: osdmap e268: 8 total, 8 up, 8 in 2024-08-17T16:58:05.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:04 smithi063 ceph-mon[42680]: osdmap e269: 8 total, 8 up, 8 in 2024-08-17T16:58:05.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:04 smithi063 ceph-mon[42680]: pgmap v87: 89 pgs: 89 active+clean; 247 MiB data, 780 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:58:05.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:04 smithi063 ceph-mon[31063]: osdmap e268: 8 total, 8 up, 8 in 2024-08-17T16:58:05.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:04 smithi063 ceph-mon[31063]: osdmap e269: 8 total, 8 up, 8 in 2024-08-17T16:58:05.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:04 smithi063 ceph-mon[31063]: pgmap v87: 89 pgs: 89 active+clean; 247 MiB data, 780 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:58:05.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:04 smithi162 ceph-mon[34633]: osdmap e268: 8 total, 8 up, 8 in 2024-08-17T16:58:05.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:04 smithi162 ceph-mon[34633]: osdmap e269: 8 total, 8 up, 8 in 2024-08-17T16:58:05.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:04 smithi162 ceph-mon[34633]: pgmap v87: 89 pgs: 89 active+clean; 247 MiB data, 780 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:58:05.931 INFO:tasks.rados.rados.0.smithi063.stdout:795: done (2 left) 2024-08-17T16:58:05.931 INFO:tasks.rados.rados.0.smithi063.stdout:796: write oid 19 current snap is 101 2024-08-17T16:58:05.931 INFO:tasks.rados.rados.0.smithi063.stdout:796: seq_num 334 ranges {413538=536345,1395221=583781,2556103=658617} 2024-08-17T16:58:05.936 INFO:tasks.rados.rados.0.smithi063.stdout:796: writing smithi06387500-19 from 413538 to 949883 tid 1 2024-08-17T16:58:05.942 INFO:tasks.rados.rados.0.smithi063.stdout:796: writing smithi06387500-19 from 1395221 to 1979002 tid 2 2024-08-17T16:58:05.950 INFO:tasks.rados.rados.0.smithi063.stdout:796: writing smithi06387500-19 from 2556103 to 3214720 tid 3 2024-08-17T16:58:05.950 INFO:tasks.rados.rados.0.smithi063.stdout:793: finishing rollback tid 0 to smithi06387500-5 2024-08-17T16:58:05.950 INFO:tasks.rados.rados.0.smithi063.stdout:797: read oid 36 snap 99 2024-08-17T16:58:05.950 INFO:tasks.rados.rados.0.smithi063.stdout:797: expect (ObjNum 315 snap 94 seq_num 315) 2024-08-17T16:58:05.950 INFO:tasks.rados.rados.0.smithi063.stdout:798: snap_remove snap 90 2024-08-17T16:58:05.950 INFO:tasks.rados.rados.0.smithi063.stdout:793: finishing rollback tid 1 to smithi06387500-5 2024-08-17T16:58:05.950 INFO:tasks.rados.rados.0.smithi063.stdout:793: finishing rollback tid 2 to smithi06387500-5 2024-08-17T16:58:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 362 (ObjNum 195 snap 54 seq_num 195) dirty exists 2024-08-17T16:58:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:794: finishing write tid 1 to smithi06387500-8 2024-08-17T16:58:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:794: finishing write tid 2 to smithi06387500-8 2024-08-17T16:58:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:794: finishing write tid 3 to smithi06387500-8 2024-08-17T16:58:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:794: finishing write tid 4 to smithi06387500-8 2024-08-17T16:58:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:794: finishing write tid 5 to smithi06387500-8 2024-08-17T16:58:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:794: oid 8 updating version 0 to 226 2024-08-17T16:58:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:794: oid 8 version 226 is already newer than 225 2024-08-17T16:58:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:794: oid 8 updating version 226 to 227 2024-08-17T16:58:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:794: oid 8 updating version 227 to 228 2024-08-17T16:58:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 228 (ObjNum 333 snap 101 seq_num 333) dirty exists 2024-08-17T16:58:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:794: left oid 8 (ObjNum 333 snap 101 seq_num 333) 2024-08-17T16:58:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:796: finishing write tid 1 to smithi06387500-19 2024-08-17T16:58:06.938 INFO:tasks.rados.rados.0.smithi063.stdout:796: finishing write tid 2 to smithi06387500-19 2024-08-17T16:58:06.938 INFO:tasks.rados.rados.0.smithi063.stdout:796: finishing write tid 3 to smithi06387500-19 2024-08-17T16:58:06.938 INFO:tasks.rados.rados.0.smithi063.stdout:796: finishing write tid 4 to smithi06387500-19 2024-08-17T16:58:06.938 INFO:tasks.rados.rados.0.smithi063.stdout:796: finishing write tid 5 to smithi06387500-19 2024-08-17T16:58:06.938 INFO:tasks.rados.rados.0.smithi063.stdout:796: oid 19 updating version 0 to 589 2024-08-17T16:58:06.938 INFO:tasks.rados.rados.0.smithi063.stdout:796: oid 19 updating version 589 to 590 2024-08-17T16:58:06.938 INFO:tasks.rados.rados.0.smithi063.stdout:796: oid 19 updating version 590 to 591 2024-08-17T16:58:06.938 INFO:tasks.rados.rados.0.smithi063.stdout:796: oid 19 version 591 is already newer than 588 2024-08-17T16:58:06.939 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 591 (ObjNum 334 snap 101 seq_num 334) dirty exists 2024-08-17T16:58:06.939 INFO:tasks.rados.rados.0.smithi063.stdout:796: left oid 19 (ObjNum 334 snap 101 seq_num 334) 2024-08-17T16:58:06.939 INFO:tasks.rados.rados.0.smithi063.stdout:797: expect (ObjNum 315 snap 94 seq_num 315) 2024-08-17T16:58:07.108 INFO:tasks.rados.rados.0.smithi063.stdout:793: done (4 left) 2024-08-17T16:58:07.109 INFO:tasks.rados.rados.0.smithi063.stdout:794: done (3 left) 2024-08-17T16:58:07.109 INFO:tasks.rados.rados.0.smithi063.stdout:796: done (2 left) 2024-08-17T16:58:07.109 INFO:tasks.rados.rados.0.smithi063.stdout:797: done (1 left) 2024-08-17T16:58:07.109 INFO:tasks.rados.rados.0.smithi063.stdout:798: done (0 left) 2024-08-17T16:58:07.109 INFO:tasks.rados.rados.0.smithi063.stdout:799: write oid 29 current snap is 101 2024-08-17T16:58:07.109 INFO:tasks.rados.rados.0.smithi063.stdout:799: seq_num 335 ranges {430345=727947,1570056=727439,2699879=469889} 2024-08-17T16:58:07.115 INFO:tasks.rados.rados.0.smithi063.stdout:799: writing smithi06387500-29 from 430345 to 1158292 tid 1 2024-08-17T16:58:07.121 INFO:tasks.rados.rados.0.smithi063.stdout:799: writing smithi06387500-29 from 1570056 to 2297495 tid 2 2024-08-17T16:58:07.126 INFO:tasks.rados.rados.0.smithi063.stdout:799: writing smithi06387500-29 from 2699879 to 3169768 tid 3 2024-08-17T16:58:07.126 INFO:tasks.rados.rados.0.smithi063.stdout:800: write oid 30 current snap is 101 2024-08-17T16:58:07.126 INFO:tasks.rados.rados.0.smithi063.stdout:799: finishing write tid 1 to smithi06387500-29 2024-08-17T16:58:07.126 INFO:tasks.rados.rados.0.smithi063.stdout:799: finishing write tid 2 to smithi06387500-29 2024-08-17T16:58:07.127 INFO:tasks.rados.rados.0.smithi063.stdout:800: seq_num 336 ranges {447152=403196,1512185=587450,2727302=681161} 2024-08-17T16:58:07.131 INFO:tasks.rados.rados.0.smithi063.stdout:800: writing smithi06387500-30 from 447152 to 850348 tid 1 2024-08-17T16:58:07.134 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:06 smithi162 ceph-mon[34633]: osdmap e270: 8 total, 8 up, 8 in 2024-08-17T16:58:07.134 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:06 smithi162 ceph-mon[34633]: pgmap v89: 89 pgs: 89 active+clean; 223 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 7.8 MiB/s rd, 359 KiB/s wr, 8 op/s 2024-08-17T16:58:07.138 INFO:tasks.rados.rados.0.smithi063.stdout:800: writing smithi06387500-30 from 1512185 to 2099635 tid 2 2024-08-17T16:58:07.145 INFO:tasks.rados.rados.0.smithi063.stdout:800: writing smithi06387500-30 from 2727302 to 3408463 tid 3 2024-08-17T16:58:07.145 INFO:tasks.rados.rados.0.smithi063.stdout:801: snap_create 2024-08-17T16:58:07.145 INFO:tasks.rados.rados.0.smithi063.stdout:799: finishing write tid 3 to smithi06387500-29 2024-08-17T16:58:07.145 INFO:tasks.rados.rados.0.smithi063.stdout:799: finishing write tid 4 to smithi06387500-29 2024-08-17T16:58:07.145 INFO:tasks.rados.rados.0.smithi063.stdout:799: finishing write tid 5 to smithi06387500-29 2024-08-17T16:58:07.146 INFO:tasks.rados.rados.0.smithi063.stdout:799: oid 29 updating version 0 to 167 2024-08-17T16:58:07.146 INFO:tasks.rados.rados.0.smithi063.stdout:799: oid 29 updating version 167 to 168 2024-08-17T16:58:07.146 INFO:tasks.rados.rados.0.smithi063.stdout:799: oid 29 version 168 is already newer than 166 2024-08-17T16:58:07.146 INFO:tasks.rados.rados.0.smithi063.stdout:799: oid 29 version 168 is already newer than 165 2024-08-17T16:58:07.146 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 168 (ObjNum 335 snap 101 seq_num 335) dirty exists 2024-08-17T16:58:07.146 INFO:tasks.rados.rados.0.smithi063.stdout:799: left oid 29 (ObjNum 335 snap 101 seq_num 335) 2024-08-17T16:58:07.146 INFO:tasks.rados.rados.0.smithi063.stdout:800: finishing write tid 1 to smithi06387500-30 2024-08-17T16:58:07.146 INFO:tasks.rados.rados.0.smithi063.stdout:800: finishing write tid 2 to smithi06387500-30 2024-08-17T16:58:07.149 INFO:tasks.rados.rados.0.smithi063.stdout:800: finishing write tid 3 to smithi06387500-30 2024-08-17T16:58:07.149 INFO:tasks.rados.rados.0.smithi063.stdout:800: finishing write tid 4 to smithi06387500-30 2024-08-17T16:58:07.149 INFO:tasks.rados.rados.0.smithi063.stdout:800: finishing write tid 5 to smithi06387500-30 2024-08-17T16:58:07.149 INFO:tasks.rados.rados.0.smithi063.stdout:800: oid 30 updating version 0 to 461 2024-08-17T16:58:07.149 INFO:tasks.rados.rados.0.smithi063.stdout:800: oid 30 updating version 461 to 463 2024-08-17T16:58:07.149 INFO:tasks.rados.rados.0.smithi063.stdout:800: oid 30 version 463 is already newer than 460 2024-08-17T16:58:07.149 INFO:tasks.rados.rados.0.smithi063.stdout:800: oid 30 version 463 is already newer than 462 2024-08-17T16:58:07.150 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 463 (ObjNum 336 snap 101 seq_num 336) dirty exists 2024-08-17T16:58:07.150 INFO:tasks.rados.rados.0.smithi063.stdout:800: left oid 30 (ObjNum 336 snap 101 seq_num 336) 2024-08-17T16:58:07.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:06 smithi063 ceph-mon[42680]: osdmap e270: 8 total, 8 up, 8 in 2024-08-17T16:58:07.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:06 smithi063 ceph-mon[42680]: pgmap v89: 89 pgs: 89 active+clean; 223 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 7.8 MiB/s rd, 359 KiB/s wr, 8 op/s 2024-08-17T16:58:07.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:06 smithi063 ceph-mon[31063]: osdmap e270: 8 total, 8 up, 8 in 2024-08-17T16:58:07.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:06 smithi063 ceph-mon[31063]: pgmap v89: 89 pgs: 89 active+clean; 223 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 7.8 MiB/s rd, 359 KiB/s wr, 8 op/s 2024-08-17T16:58:07.941 INFO:tasks.rados.rados.0.smithi063.stdout:799: done (2 left) 2024-08-17T16:58:07.942 INFO:tasks.rados.rados.0.smithi063.stdout:800: done (1 left) 2024-08-17T16:58:07.942 INFO:tasks.rados.rados.0.smithi063.stdout:801: done (0 left) 2024-08-17T16:58:07.942 INFO:tasks.rados.rados.0.smithi063.stdout:802: delete oid 35 current snap is 102 2024-08-17T16:58:07.944 INFO:tasks.rados.rados.0.smithi063.stdout:802: done (0 left) 2024-08-17T16:58:07.944 INFO:tasks.rados.rados.0.smithi063.stdout:803: write oid 17 current snap is 102 2024-08-17T16:58:07.944 INFO:tasks.rados.rados.0.smithi063.stdout:803: seq_num 338 ranges {480766=553694,1629149=591119,2782148=420058} 2024-08-17T16:58:07.950 INFO:tasks.rados.rados.0.smithi063.stdout:803: writing smithi06387500-17 from 480766 to 1034460 tid 1 2024-08-17T16:58:07.956 INFO:tasks.rados.rados.0.smithi063.stdout:803: writing smithi06387500-17 from 1629149 to 2220268 tid 2 2024-08-17T16:58:07.962 INFO:tasks.rados.rados.0.smithi063.stdout:803: writing smithi06387500-17 from 2782148 to 3202206 tid 3 2024-08-17T16:58:07.962 INFO:tasks.rados.rados.0.smithi063.stdout:804: read oid 50 snap -1 2024-08-17T16:58:07.962 INFO:tasks.rados.rados.0.smithi063.stdout:804: expect (ObjNum 300 snap 91 seq_num 300) 2024-08-17T16:58:07.962 INFO:tasks.rados.rados.0.smithi063.stdout:805: write oid 1 current snap is 102 2024-08-17T16:58:07.962 INFO:tasks.rados.rados.0.smithi063.stdout:803: finishing write tid 1 to smithi06387500-17 2024-08-17T16:58:07.963 INFO:tasks.rados.rados.0.smithi063.stdout:803: finishing write tid 2 to smithi06387500-17 2024-08-17T16:58:07.963 INFO:tasks.rados.rados.0.smithi063.stdout:805: seq_num 339 ranges {497573=628943,1687631=734777,3093218=604355} 2024-08-17T16:58:07.968 INFO:tasks.rados.rados.0.smithi063.stdout:805: writing smithi06387500-1 from 497573 to 1126516 tid 1 2024-08-17T16:58:07.975 INFO:tasks.rados.rados.0.smithi063.stdout:805: writing smithi06387500-1 from 1687631 to 2422408 tid 2 2024-08-17T16:58:07.982 INFO:tasks.rados.rados.0.smithi063.stdout:805: writing smithi06387500-1 from 3093218 to 3697573 tid 3 2024-08-17T16:58:07.983 INFO:tasks.rados.rados.0.smithi063.stdout:806: snap_remove snap 100 2024-08-17T16:58:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:07 smithi162 ceph-mon[34633]: osdmap e271: 8 total, 8 up, 8 in 2024-08-17T16:58:08.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:07 smithi063 ceph-mon[42680]: osdmap e271: 8 total, 8 up, 8 in 2024-08-17T16:58:08.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:07 smithi063 ceph-mon[31063]: osdmap e271: 8 total, 8 up, 8 in 2024-08-17T16:58:08.958 INFO:tasks.rados.rados.0.smithi063.stdout:806: done (3 left) 2024-08-17T16:58:08.958 INFO:tasks.rados.rados.0.smithi063.stdout:807: read oid 14 snap -1 2024-08-17T16:58:08.958 INFO:tasks.rados.rados.0.smithi063.stdout:807: expect (ObjNum 322 snap 96 seq_num 322) 2024-08-17T16:58:08.958 INFO:tasks.rados.rados.0.smithi063.stdout:808: snap_remove snap 92 2024-08-17T16:58:09.080 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:08 smithi162 ceph-mon[34633]: osdmap e272: 8 total, 8 up, 8 in 2024-08-17T16:58:09.080 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:08 smithi162 ceph-mon[34633]: pgmap v92: 89 pgs: 89 active+clean; 223 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 356 KiB/s wr, 8 op/s 2024-08-17T16:58:09.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:08 smithi063 ceph-mon[42680]: osdmap e272: 8 total, 8 up, 8 in 2024-08-17T16:58:09.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:08 smithi063 ceph-mon[42680]: pgmap v92: 89 pgs: 89 active+clean; 223 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 356 KiB/s wr, 8 op/s 2024-08-17T16:58:09.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:08 smithi063 ceph-mon[31063]: osdmap e272: 8 total, 8 up, 8 in 2024-08-17T16:58:09.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:08 smithi063 ceph-mon[31063]: pgmap v92: 89 pgs: 89 active+clean; 223 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 7.7 MiB/s rd, 356 KiB/s wr, 8 op/s 2024-08-17T16:58:09.870 INFO:tasks.rados.rados.0.smithi063.stdout:808: done (4 left) 2024-08-17T16:58:09.870 INFO:tasks.rados.rados.0.smithi063.stdout:809: rollback oid 7 current snap is 102 2024-08-17T16:58:09.871 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 88 2024-08-17T16:58:09.871 INFO:tasks.rados.rados.0.smithi063.stdout:810: read oid 35 snap 98 2024-08-17T16:58:09.871 INFO:tasks.rados.rados.0.smithi063.stdout:810: expect (ObjNum 179 snap 47 seq_num 179) 2024-08-17T16:58:09.871 INFO:tasks.rados.rados.0.smithi063.stdout:811: rollback oid 10 current snap is 102 2024-08-17T16:58:09.871 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 10 to 75 2024-08-17T16:58:09.871 INFO:tasks.rados.rados.0.smithi063.stdout:812: rollback oid 24 current snap is 102 2024-08-17T16:58:09.871 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 24 to 88 2024-08-17T16:58:09.871 INFO:tasks.rados.rados.0.smithi063.stdout:813: snap_remove snap 95 2024-08-17T16:58:09.950 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:58:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:58:09] "GET /metrics HTTP/1.1" 200 36211 "" "Prometheus/2.43.0" 2024-08-17T16:58:10.157 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:09 smithi162 systemd[1]: Stopping Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:58:10.157 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:09 smithi162 ceph-mon[34633]: osdmap e273: 8 total, 8 up, 8 in 2024-08-17T16:58:10.157 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:10.158 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:09 smithi162 ceph-mon[34633]: osdmap e274: 8 total, 8 up, 8 in 2024-08-17T16:58:10.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:09 smithi063 ceph-mon[42680]: osdmap e273: 8 total, 8 up, 8 in 2024-08-17T16:58:10.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:10.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:09 smithi063 ceph-mon[42680]: osdmap e274: 8 total, 8 up, 8 in 2024-08-17T16:58:10.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:09 smithi063 ceph-mon[31063]: osdmap e273: 8 total, 8 up, 8 in 2024-08-17T16:58:10.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:10.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:09 smithi063 ceph-mon[31063]: osdmap e274: 8 total, 8 up, 8 in 2024-08-17T16:58:10.439 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:58:10.156Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T16:58:10.439 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:58:10.156Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-08-17T16:58:10.439 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:58:10.156Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-08-17T16:58:10.439 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:58:10.156Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T16:58:10.439 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:58:10.156Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-08-17T16:58:10.439 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:58:10.156Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-08-17T16:58:10.439 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:58:10.156Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T16:58:10.439 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:58:10.156Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-08-17T16:58:10.439 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:58:10.156Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-08-17T16:58:10.440 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:58:10.156Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T16:58:10.440 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:58:10.156Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-08-17T16:58:10.440 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[81110]: ts=2024-08-17T16:58:10.156Z caller=main.go:1147 level=info msg="See you next time!" 2024-08-17T16:58:10.440 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 podman[89754]: 2024-08-17 16:58:10.166411157 +0000 UTC m=+0.126710492 container died 5ccd60dff86c609512011cf6ec8af1a35769dc292055a798311aec55aba6514d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:58:10.779 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 podman[89754]: 2024-08-17 16:58:10.536547433 +0000 UTC m=+0.496846760 container cleanup 5ccd60dff86c609512011cf6ec8af1a35769dc292055a798311aec55aba6514d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:58:10.780 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 bash[89754]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T16:58:10.870 INFO:tasks.rados.rados.0.smithi063.stdout:813: done (8 left) 2024-08-17T16:58:10.870 INFO:tasks.rados.rados.0.smithi063.stdout:814: write oid 2 current snap is 102 2024-08-17T16:58:10.870 INFO:tasks.rados.rados.0.smithi063.stdout:803: finishing write tid 3 to smithi06387500-17 2024-08-17T16:58:10.870 INFO:tasks.rados.rados.0.smithi063.stdout:803: finishing write tid 4 to smithi06387500-17 2024-08-17T16:58:10.870 INFO:tasks.rados.rados.0.smithi063.stdout:803: finishing write tid 5 to smithi06387500-17 2024-08-17T16:58:10.871 INFO:tasks.rados.rados.0.smithi063.stdout:803: oid 17 updating version 0 to 251 2024-08-17T16:58:10.871 INFO:tasks.rados.rados.0.smithi063.stdout:803: oid 17 version 251 is already newer than 250 2024-08-17T16:58:10.871 INFO:tasks.rados.rados.0.smithi063.stdout:803: oid 17 version 251 is already newer than 248 2024-08-17T16:58:10.871 INFO:tasks.rados.rados.0.smithi063.stdout:803: oid 17 version 251 is already newer than 249 2024-08-17T16:58:10.871 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 251 (ObjNum 338 snap 102 seq_num 338) dirty exists 2024-08-17T16:58:10.871 INFO:tasks.rados.rados.0.smithi063.stdout:803: left oid 17 (ObjNum 338 snap 102 seq_num 338) 2024-08-17T16:58:10.871 INFO:tasks.rados.rados.0.smithi063.stdout:804: expect (ObjNum 300 snap 91 seq_num 300) 2024-08-17T16:58:11.095 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 podman[89771]: 2024-08-17 16:58:10.779310095 +0000 UTC m=+0.614815234 container remove 5ccd60dff86c609512011cf6ec8af1a35769dc292055a798311aec55aba6514d (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:58:11.095 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-08-17T16:58:11.096 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 systemd[1]: Stopped Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:58:11.096 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:10 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@prometheus.a.service: Consumed 1.205s CPU time. 2024-08-17T16:58:11.168 INFO:tasks.rados.rados.0.smithi063.stdout:805: finishing write tid 1 to smithi06387500-1 2024-08-17T16:58:11.169 INFO:tasks.rados.rados.0.smithi063.stdout:805: finishing write tid 2 to smithi06387500-1 2024-08-17T16:58:11.169 INFO:tasks.rados.rados.0.smithi063.stdout:805: finishing write tid 3 to smithi06387500-1 2024-08-17T16:58:11.169 INFO:tasks.rados.rados.0.smithi063.stdout:805: finishing write tid 4 to smithi06387500-1 2024-08-17T16:58:11.169 INFO:tasks.rados.rados.0.smithi063.stdout:805: finishing write tid 5 to smithi06387500-1 2024-08-17T16:58:11.169 INFO:tasks.rados.rados.0.smithi063.stdout:805: oid 1 updating version 0 to 371 2024-08-17T16:58:11.169 INFO:tasks.rados.rados.0.smithi063.stdout:805: oid 1 updating version 371 to 372 2024-08-17T16:58:11.169 INFO:tasks.rados.rados.0.smithi063.stdout:805: oid 1 updating version 372 to 374 2024-08-17T16:58:11.169 INFO:tasks.rados.rados.0.smithi063.stdout:805: oid 1 version 374 is already newer than 373 2024-08-17T16:58:11.169 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 374 (ObjNum 339 snap 102 seq_num 339) dirty exists 2024-08-17T16:58:11.169 INFO:tasks.rados.rados.0.smithi063.stdout:805: left oid 1 (ObjNum 339 snap 102 seq_num 339) 2024-08-17T16:58:11.169 INFO:tasks.rados.rados.0.smithi063.stdout:807: expect (ObjNum 322 snap 96 seq_num 322) 2024-08-17T16:58:11.283 INFO:tasks.rados.rados.0.smithi063.stdout:811: finishing rollback tid 0 to smithi06387500-10 2024-08-17T16:58:11.283 INFO:tasks.rados.rados.0.smithi063.stdout:811: finishing rollback tid 1 to smithi06387500-10 2024-08-17T16:58:11.283 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 288 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T16:58:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:812: finishing rollback tid 0 to smithi06387500-24 2024-08-17T16:58:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:812: finishing rollback tid 1 to smithi06387500-24 2024-08-17T16:58:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:809: finishing rollback tid 1 to smithi06387500-7 2024-08-17T16:58:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:809: finishing rollback tid 2 to smithi06387500-7 2024-08-17T16:58:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 469 (ObjNum 277 snap 86 seq_num 277) dirty exists 2024-08-17T16:58:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:812: finishing rollback tid 2 to smithi06387500-24 2024-08-17T16:58:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 318 (ObjNum 286 snap 87 seq_num 286) dirty exists 2024-08-17T16:58:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:810: expect (ObjNum 179 snap 47 seq_num 179) 2024-08-17T16:58:11.417 INFO:tasks.rados.rados.0.smithi063.stdout:814: seq_num 340 ranges {514380=704192,1629760=478435,2604288=442602} 2024-08-17T16:58:11.424 INFO:tasks.rados.rados.0.smithi063.stdout:814: writing smithi06387500-2 from 514380 to 1218572 tid 1 2024-08-17T16:58:11.430 INFO:tasks.rados.rados.0.smithi063.stdout:814: writing smithi06387500-2 from 1629760 to 2108195 tid 2 2024-08-17T16:58:11.437 INFO:tasks.rados.rados.0.smithi063.stdout:814: writing smithi06387500-2 from 2604288 to 3046890 tid 3 2024-08-17T16:58:11.437 INFO:tasks.rados.rados.0.smithi063.stdout:803: done (8 left) 2024-08-17T16:58:11.437 INFO:tasks.rados.rados.0.smithi063.stdout:804: done (7 left) 2024-08-17T16:58:11.437 INFO:tasks.rados.rados.0.smithi063.stdout:805: done (6 left) 2024-08-17T16:58:11.437 INFO:tasks.rados.rados.0.smithi063.stdout:807: done (5 left) 2024-08-17T16:58:11.437 INFO:tasks.rados.rados.0.smithi063.stdout:809: done (4 left) 2024-08-17T16:58:11.437 INFO:tasks.rados.rados.0.smithi063.stdout:810: done (3 left) 2024-08-17T16:58:11.437 INFO:tasks.rados.rados.0.smithi063.stdout:811: done (2 left) 2024-08-17T16:58:11.437 INFO:tasks.rados.rados.0.smithi063.stdout:812: done (1 left) 2024-08-17T16:58:11.437 INFO:tasks.rados.rados.0.smithi063.stdout:815: snap_remove snap 88 2024-08-17T16:58:11.437 INFO:tasks.rados.rados.0.smithi063.stdout:814: finishing write tid 1 to smithi06387500-2 2024-08-17T16:58:11.437 INFO:tasks.rados.rados.0.smithi063.stdout:814: finishing write tid 2 to smithi06387500-2 2024-08-17T16:58:11.481 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 systemd[1]: Starting Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:58:11.482 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 podman[89906]: 2024-08-17 16:58:11.229514943 +0000 UTC m=+0.018133151 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-08-17T16:58:11.482 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 podman[89906]: 2024-08-17 16:58:11.35546226 +0000 UTC m=+0.144080494 container create dfe66b2c5568ec125e083924cb579fd1178e864127062e6eaadec53c7c746218 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:58:11.846 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 podman[89906]: 2024-08-17 16:58:11.574244806 +0000 UTC m=+0.362863025 container init dfe66b2c5568ec125e083924cb579fd1178e864127062e6eaadec53c7c746218 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:58:11.846 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 podman[89906]: 2024-08-17 16:58:11.578811411 +0000 UTC m=+0.367429604 container start dfe66b2c5568ec125e083924cb579fd1178e864127062e6eaadec53c7c746218 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T16:58:11.846 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.598Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-08-17T16:58:11.846 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.598Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-08-17T16:58:11.846 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.599Z caller=main.go:623 level=info host_details="(Linux 5.14.0-480.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Jul 12 20:45:27 UTC 2024 x86_64 smithi162 (none))" 2024-08-17T16:58:11.846 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.599Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-08-17T16:58:11.847 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.599Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-08-17T16:58:11.847 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.601Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-08-17T16:58:11.847 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.601Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-08-17T16:58:11.847 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.603Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-08-17T16:58:11.847 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.603Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-08-17T16:58:11.847 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.605Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-08-17T16:58:11.847 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.605Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.673µs 2024-08-17T16:58:11.847 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.605Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-08-17T16:58:11.847 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.606Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-08-17T16:58:11.848 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.612Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-08-17T16:58:11.848 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.613Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-08-17T16:58:11.848 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.614Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-08-17T16:58:11.848 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.614Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-08-17T16:58:11.848 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.614Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=26.472µs wal_replay_duration=9.230695ms wbl_replay_duration=120ns total_replay_duration=9.275519ms 2024-08-17T16:58:11.848 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.616Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-08-17T16:58:11.848 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.616Z caller=main.go:1153 level=info msg="TSDB started" 2024-08-17T16:58:11.848 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.616Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-08-17T16:58:11.848 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.632Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.644929ms db_storage=818ns remote_storage=1.161µs web_handler=358ns query_engine=612ns scrape=472.324µs scrape_sd=99.057µs notify=11.219µs notify_sd=7.122µs rules=14.721349ms tracing=5.24µs 2024-08-17T16:58:11.848 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.632Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-08-17T16:58:11.849 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T16:58:11.632Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-08-17T16:58:11.849 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 bash[89906]: dfe66b2c5568ec125e083924cb579fd1178e864127062e6eaadec53c7c746218 2024-08-17T16:58:11.849 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 16:58:11 smithi162 systemd[1]: Started Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:58:11.876 INFO:tasks.rados.rados.0.smithi063.stdout:815: done (1 left) 2024-08-17T16:58:11.876 INFO:tasks.rados.rados.0.smithi063.stdout:816: delete oid 3 current snap is 102 2024-08-17T16:58:11.876 INFO:tasks.rados.rados.0.smithi063.stdout:814: finishing write tid 3 to smithi06387500-2 2024-08-17T16:58:11.876 INFO:tasks.rados.rados.0.smithi063.stdout:814: finishing write tid 4 to smithi06387500-2 2024-08-17T16:58:11.876 INFO:tasks.rados.rados.0.smithi063.stdout:814: finishing write tid 5 to smithi06387500-2 2024-08-17T16:58:11.876 INFO:tasks.rados.rados.0.smithi063.stdout:814: oid 2 updating version 0 to 384 2024-08-17T16:58:11.876 INFO:tasks.rados.rados.0.smithi063.stdout:814: oid 2 updating version 384 to 385 2024-08-17T16:58:11.876 INFO:tasks.rados.rados.0.smithi063.stdout:814: oid 2 updating version 385 to 387 2024-08-17T16:58:11.876 INFO:tasks.rados.rados.0.smithi063.stdout:814: oid 2 version 387 is already newer than 386 2024-08-17T16:58:11.876 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 387 (ObjNum 340 snap 102 seq_num 340) dirty exists 2024-08-17T16:58:11.877 INFO:tasks.rados.rados.0.smithi063.stdout:814: left oid 2 (ObjNum 340 snap 102 seq_num 340) 2024-08-17T16:58:11.881 INFO:tasks.rados.rados.0.smithi063.stdout:814: done (1 left) 2024-08-17T16:58:11.882 INFO:tasks.rados.rados.0.smithi063.stdout:816: done (0 left) 2024-08-17T16:58:11.882 INFO:tasks.rados.rados.0.smithi063.stdout:817: read oid 39 snap 99 2024-08-17T16:58:11.882 INFO:tasks.rados.rados.0.smithi063.stdout:817: expect deleted 2024-08-17T16:58:11.882 INFO:tasks.rados.rados.0.smithi063.stdout:818: read oid 43 snap 101 2024-08-17T16:58:11.882 INFO:tasks.rados.rados.0.smithi063.stdout:818: expect (ObjNum 287 snap 87 seq_num 287) 2024-08-17T16:58:11.882 INFO:tasks.rados.rados.0.smithi063.stdout:819: snap_create 2024-08-17T16:58:11.889 INFO:tasks.rados.rados.0.smithi063.stdout:818: expect (ObjNum 287 snap 87 seq_num 287) 2024-08-17T16:58:12.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:11 smithi063 ceph-mon[31063]: osdmap e275: 8 total, 8 up, 8 in 2024-08-17T16:58:12.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:11 smithi063 ceph-mon[31063]: pgmap v96: 89 pgs: 89 active+clean; 239 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2024-08-17T16:58:12.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:11 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:12.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:11 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:12.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:11 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:12.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:11 smithi063 ceph-mon[42680]: osdmap e275: 8 total, 8 up, 8 in 2024-08-17T16:58:12.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:11 smithi063 ceph-mon[42680]: pgmap v96: 89 pgs: 89 active+clean; 239 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2024-08-17T16:58:12.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:11 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:12.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:11 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:12.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:11 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:12.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-mon[34633]: osdmap e275: 8 total, 8 up, 8 in 2024-08-17T16:58:12.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-mon[34633]: pgmap v96: 89 pgs: 89 active+clean; 239 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2024-08-17T16:58:12.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:12.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:12.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:11 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:12.878 INFO:tasks.rados.rados.0.smithi063.stdout:817: done (2 left) 2024-08-17T16:58:12.878 INFO:tasks.rados.rados.0.smithi063.stdout:818: done (1 left) 2024-08-17T16:58:12.878 INFO:tasks.rados.rados.0.smithi063.stdout:819: done (0 left) 2024-08-17T16:58:12.879 INFO:tasks.rados.rados.0.smithi063.stdout:820: write oid 35 current snap is 103 2024-08-17T16:58:12.879 INFO:tasks.rados.rados.0.smithi063.stdout:820: seq_num 342 ranges {547994=454690,1746724=482104,2659134=465146,3605976=142018} 2024-08-17T16:58:12.883 INFO:tasks.rados.rados.0.smithi063.stdout:820: writing smithi06387500-35 from 547994 to 1002684 tid 1 2024-08-17T16:58:12.890 INFO:tasks.rados.rados.0.smithi063.stdout:820: writing smithi06387500-35 from 1746724 to 2228828 tid 2 2024-08-17T16:58:12.895 INFO:tasks.rados.rados.0.smithi063.stdout:820: writing smithi06387500-35 from 2659134 to 3124280 tid 3 2024-08-17T16:58:12.899 INFO:tasks.rados.rados.0.smithi063.stdout:820: writing smithi06387500-35 from 3605976 to 3747994 tid 4 2024-08-17T16:58:12.899 INFO:tasks.rados.rados.0.smithi063.stdout:821: write oid 48 current snap is 103 2024-08-17T16:58:12.899 INFO:tasks.rados.rados.0.smithi063.stdout:820: finishing write tid 1 to smithi06387500-35 2024-08-17T16:58:12.899 INFO:tasks.rados.rados.0.smithi063.stdout:820: finishing write tid 2 to smithi06387500-35 2024-08-17T16:58:12.899 INFO:tasks.rados.rados.0.smithi063.stdout:820: finishing write tid 3 to smithi06387500-35 2024-08-17T16:58:12.899 INFO:tasks.rados.rados.0.smithi063.stdout:821: seq_num 343 ranges {564801=646292,1921559=742115,3202910=561891} 2024-08-17T16:58:12.905 INFO:tasks.rados.rados.0.smithi063.stdout:821: writing smithi06387500-48 from 564801 to 1211093 tid 1 2024-08-17T16:58:12.913 INFO:tasks.rados.rados.0.smithi063.stdout:821: writing smithi06387500-48 from 1921559 to 2663674 tid 2 2024-08-17T16:58:12.919 INFO:tasks.rados.rados.0.smithi063.stdout:821: writing smithi06387500-48 from 3202910 to 3764801 tid 3 2024-08-17T16:58:12.919 INFO:tasks.rados.rados.0.smithi063.stdout:822: write oid 49 current snap is 103 2024-08-17T16:58:12.919 INFO:tasks.rados.rados.0.smithi063.stdout:820: finishing write tid 4 to smithi06387500-35 2024-08-17T16:58:12.920 INFO:tasks.rados.rados.0.smithi063.stdout:820: finishing write tid 5 to smithi06387500-35 2024-08-17T16:58:12.920 INFO:tasks.rados.rados.0.smithi063.stdout:822: seq_num 344 ranges {581608=721541,1863688=485773,3113980=604043} 2024-08-17T16:58:12.926 INFO:tasks.rados.rados.0.smithi063.stdout:822: writing smithi06387500-49 from 581608 to 1303149 tid 1 2024-08-17T16:58:12.932 INFO:tasks.rados.rados.0.smithi063.stdout:822: writing smithi06387500-49 from 1863688 to 2349461 tid 2 2024-08-17T16:58:12.940 INFO:tasks.rados.rados.0.smithi063.stdout:822: writing smithi06387500-49 from 3113980 to 3718023 tid 3 2024-08-17T16:58:12.940 INFO:tasks.rados.rados.0.smithi063.stdout:823: read oid 32 snap -1 2024-08-17T16:58:12.940 INFO:tasks.rados.rados.0.smithi063.stdout:823: expect (ObjNum 307 snap 92 seq_num 307) 2024-08-17T16:58:12.940 INFO:tasks.rados.rados.0.smithi063.stdout:824: delete oid 27 current snap is 103 2024-08-17T16:58:12.940 INFO:tasks.rados.rados.0.smithi063.stdout:820: finishing write tid 6 to smithi06387500-35 2024-08-17T16:58:12.940 INFO:tasks.rados.rados.0.smithi063.stdout:820: oid 35 updating version 0 to 860 2024-08-17T16:58:12.941 INFO:tasks.rados.rados.0.smithi063.stdout:820: oid 35 version 860 is already newer than 858 2024-08-17T16:58:12.941 INFO:tasks.rados.rados.0.smithi063.stdout:820: oid 35 updating version 860 to 861 2024-08-17T16:58:12.941 INFO:tasks.rados.rados.0.smithi063.stdout:820: oid 35 version 861 is already newer than 857 2024-08-17T16:58:12.941 INFO:tasks.rados.rados.0.smithi063.stdout:820: oid 35 version 861 is already newer than 859 2024-08-17T16:58:12.941 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 861 (ObjNum 342 snap 103 seq_num 342) dirty exists 2024-08-17T16:58:12.941 INFO:tasks.rados.rados.0.smithi063.stdout:820: left oid 35 (ObjNum 342 snap 103 seq_num 342) 2024-08-17T16:58:12.941 INFO:tasks.rados.rados.0.smithi063.stdout:821: finishing write tid 1 to smithi06387500-48 2024-08-17T16:58:12.941 INFO:tasks.rados.rados.0.smithi063.stdout:821: finishing write tid 2 to smithi06387500-48 2024-08-17T16:58:12.941 INFO:tasks.rados.rados.0.smithi063.stdout:821: finishing write tid 3 to smithi06387500-48 2024-08-17T16:58:12.941 INFO:tasks.rados.rados.0.smithi063.stdout:821: finishing write tid 4 to smithi06387500-48 2024-08-17T16:58:12.941 INFO:tasks.rados.rados.0.smithi063.stdout:821: finishing write tid 5 to smithi06387500-48 2024-08-17T16:58:12.941 INFO:tasks.rados.rados.0.smithi063.stdout:821: oid 48 updating version 0 to 864 2024-08-17T16:58:12.942 INFO:tasks.rados.rados.0.smithi063.stdout:821: oid 48 updating version 864 to 865 2024-08-17T16:58:12.942 INFO:tasks.rados.rados.0.smithi063.stdout:821: oid 48 version 865 is already newer than 863 2024-08-17T16:58:12.942 INFO:tasks.rados.rados.0.smithi063.stdout:821: oid 48 updating version 865 to 866 2024-08-17T16:58:12.942 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 866 (ObjNum 343 snap 103 seq_num 343) dirty exists 2024-08-17T16:58:12.942 INFO:tasks.rados.rados.0.smithi063.stdout:821: left oid 48 (ObjNum 343 snap 103 seq_num 343) 2024-08-17T16:58:12.942 INFO:tasks.rados.rados.0.smithi063.stdout:822: finishing write tid 1 to smithi06387500-49 2024-08-17T16:58:12.942 INFO:tasks.rados.rados.0.smithi063.stdout:822: finishing write tid 2 to smithi06387500-49 2024-08-17T16:58:12.943 INFO:tasks.rados.rados.0.smithi063.stdout:822: finishing write tid 3 to smithi06387500-49 2024-08-17T16:58:12.943 INFO:tasks.rados.rados.0.smithi063.stdout:822: finishing write tid 4 to smithi06387500-49 2024-08-17T16:58:12.944 INFO:tasks.rados.rados.0.smithi063.stdout:822: finishing write tid 5 to smithi06387500-49 2024-08-17T16:58:12.944 INFO:tasks.rados.rados.0.smithi063.stdout:822: oid 49 updating version 0 to 869 2024-08-17T16:58:12.944 INFO:tasks.rados.rados.0.smithi063.stdout:822: oid 49 updating version 869 to 870 2024-08-17T16:58:12.944 INFO:tasks.rados.rados.0.smithi063.stdout:822: oid 49 version 870 is already newer than 867 2024-08-17T16:58:12.944 INFO:tasks.rados.rados.0.smithi063.stdout:822: oid 49 version 870 is already newer than 868 2024-08-17T16:58:12.944 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 870 (ObjNum 344 snap 103 seq_num 344) dirty exists 2024-08-17T16:58:12.944 INFO:tasks.rados.rados.0.smithi063.stdout:822: left oid 49 (ObjNum 344 snap 103 seq_num 344) 2024-08-17T16:58:12.944 INFO:tasks.rados.rados.0.smithi063.stdout:820: done (4 left) 2024-08-17T16:58:12.944 INFO:tasks.rados.rados.0.smithi063.stdout:821: done (3 left) 2024-08-17T16:58:12.944 INFO:tasks.rados.rados.0.smithi063.stdout:822: done (2 left) 2024-08-17T16:58:12.944 INFO:tasks.rados.rados.0.smithi063.stdout:824: done (1 left) 2024-08-17T16:58:12.944 INFO:tasks.rados.rados.0.smithi063.stdout:825: write oid 5 current snap is 103 2024-08-17T16:58:12.945 INFO:tasks.rados.rados.0.smithi063.stdout:825: seq_num 346 ranges {615222=472039,1580652=489442,2768826=626587} 2024-08-17T16:58:12.950 INFO:tasks.rados.rados.0.smithi063.stdout:825: writing smithi06387500-5 from 615222 to 1087261 tid 1 2024-08-17T16:58:12.955 INFO:tasks.rados.rados.0.smithi063.stdout:825: writing smithi06387500-5 from 1580652 to 2070094 tid 2 2024-08-17T16:58:12.962 INFO:tasks.rados.rados.0.smithi063.stdout:825: writing smithi06387500-5 from 2768826 to 3395413 tid 3 2024-08-17T16:58:12.962 INFO:tasks.rados.rados.0.smithi063.stdout:826: write oid 42 current snap is 103 2024-08-17T16:58:12.962 INFO:tasks.rados.rados.0.smithi063.stdout:826: seq_num 347 ranges {632029=547288,1639134=749453,2796249=437859,3653524=178505} 2024-08-17T16:58:12.968 INFO:tasks.rados.rados.0.smithi063.stdout:826: writing smithi06387500-42 from 632029 to 1179317 tid 1 2024-08-17T16:58:12.975 INFO:tasks.rados.rados.0.smithi063.stdout:826: writing smithi06387500-42 from 1639134 to 2388587 tid 2 2024-08-17T16:58:12.980 INFO:tasks.rados.rados.0.smithi063.stdout:826: writing smithi06387500-42 from 2796249 to 3234108 tid 3 2024-08-17T16:58:12.984 INFO:tasks.rados.rados.0.smithi063.stdout:826: writing smithi06387500-42 from 3653524 to 3832029 tid 4 2024-08-17T16:58:12.984 INFO:tasks.rados.rados.0.smithi063.stdout:827: read oid 46 snap 96 2024-08-17T16:58:12.984 INFO:tasks.rados.rados.0.smithi063.stdout:827: expect (ObjNum 293 snap 91 seq_num 293) 2024-08-17T16:58:12.984 INFO:tasks.rados.rados.0.smithi063.stdout:828: rollback oid 38 current snap is 103 2024-08-17T16:58:12.984 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 38 to 99 2024-08-17T16:58:12.984 INFO:tasks.rados.rados.0.smithi063.stdout:823: expect (ObjNum 307 snap 92 seq_num 307) 2024-08-17T16:58:13.089 INFO:tasks.rados.rados.0.smithi063.stdout:825: finishing write tid 1 to smithi06387500-5 2024-08-17T16:58:13.089 INFO:tasks.rados.rados.0.smithi063.stdout:825: finishing write tid 2 to smithi06387500-5 2024-08-17T16:58:13.089 INFO:tasks.rados.rados.0.smithi063.stdout:825: finishing write tid 3 to smithi06387500-5 2024-08-17T16:58:13.089 INFO:tasks.rados.rados.0.smithi063.stdout:825: finishing write tid 4 to smithi06387500-5 2024-08-17T16:58:13.089 INFO:tasks.rados.rados.0.smithi063.stdout:823: done (4 left) 2024-08-17T16:58:13.089 INFO:tasks.rados.rados.0.smithi063.stdout:829: delete oid 22 current snap is 103 2024-08-17T16:58:13.089 INFO:tasks.rados.rados.0.smithi063.stdout:825: finishing write tid 5 to smithi06387500-5 2024-08-17T16:58:13.089 INFO:tasks.rados.rados.0.smithi063.stdout:825: oid 5 updating version 0 to 395 2024-08-17T16:58:13.089 INFO:tasks.rados.rados.0.smithi063.stdout:825: oid 5 updating version 395 to 396 2024-08-17T16:58:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:825: oid 5 updating version 396 to 397 2024-08-17T16:58:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:825: oid 5 updating version 397 to 398 2024-08-17T16:58:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 398 (ObjNum 346 snap 103 seq_num 346) dirty exists 2024-08-17T16:58:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:825: left oid 5 (ObjNum 346 snap 103 seq_num 346) 2024-08-17T16:58:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:826: finishing write tid 1 to smithi06387500-42 2024-08-17T16:58:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:826: finishing write tid 2 to smithi06387500-42 2024-08-17T16:58:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:826: finishing write tid 3 to smithi06387500-42 2024-08-17T16:58:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:828: finishing rollback tid 1 to smithi06387500-38 2024-08-17T16:58:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 0 (ObjNum 574598384 snap 32710 seq_num 304) dirty dne 2024-08-17T16:58:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:826: finishing write tid 4 to smithi06387500-42 2024-08-17T16:58:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:826: finishing write tid 5 to smithi06387500-42 2024-08-17T16:58:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:826: finishing write tid 6 to smithi06387500-42 2024-08-17T16:58:13.090 INFO:tasks.rados.rados.0.smithi063.stdout:826: oid 42 updating version 0 to 626 2024-08-17T16:58:13.091 INFO:tasks.rados.rados.0.smithi063.stdout:826: oid 42 version 626 is already newer than 623 2024-08-17T16:58:13.091 INFO:tasks.rados.rados.0.smithi063.stdout:826: oid 42 version 626 is already newer than 625 2024-08-17T16:58:13.091 INFO:tasks.rados.rados.0.smithi063.stdout:826: oid 42 updating version 626 to 627 2024-08-17T16:58:13.091 INFO:tasks.rados.rados.0.smithi063.stdout:826: oid 42 version 627 is already newer than 624 2024-08-17T16:58:13.091 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 627 (ObjNum 347 snap 103 seq_num 347) dirty exists 2024-08-17T16:58:13.091 INFO:tasks.rados.rados.0.smithi063.stdout:826: left oid 42 (ObjNum 347 snap 103 seq_num 347) 2024-08-17T16:58:13.091 INFO:tasks.rados.rados.0.smithi063.stdout:827: expect (ObjNum 293 snap 91 seq_num 293) 2024-08-17T16:58:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:12 smithi162 ceph-mon[34633]: osdmap e276: 8 total, 8 up, 8 in 2024-08-17T16:58:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:12 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:12 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:13.251 INFO:tasks.rados.rados.0.smithi063.stdout:825: done (4 left) 2024-08-17T16:58:13.251 INFO:tasks.rados.rados.0.smithi063.stdout:826: done (3 left) 2024-08-17T16:58:13.251 INFO:tasks.rados.rados.0.smithi063.stdout:827: done (2 left) 2024-08-17T16:58:13.251 INFO:tasks.rados.rados.0.smithi063.stdout:828: done (1 left) 2024-08-17T16:58:13.251 INFO:tasks.rados.rados.0.smithi063.stdout:829: done (0 left) 2024-08-17T16:58:13.251 INFO:tasks.rados.rados.0.smithi063.stdout:830: read oid 43 snap -1 2024-08-17T16:58:13.251 INFO:tasks.rados.rados.0.smithi063.stdout:830: expect (ObjNum 287 snap 87 seq_num 287) 2024-08-17T16:58:13.251 INFO:tasks.rados.rados.0.smithi063.stdout:831: snap_create 2024-08-17T16:58:13.258 INFO:tasks.rados.rados.0.smithi063.stdout:830: expect (ObjNum 287 snap 87 seq_num 287) 2024-08-17T16:58:13.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:12 smithi063 ceph-mon[42680]: osdmap e276: 8 total, 8 up, 8 in 2024-08-17T16:58:13.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:12 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:13.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:12 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:13.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:12 smithi063 ceph-mon[31063]: osdmap e276: 8 total, 8 up, 8 in 2024-08-17T16:58:13.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:12 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:13.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:12 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:13.883 INFO:tasks.rados.rados.0.smithi063.stdout:830: done (1 left) 2024-08-17T16:58:13.884 INFO:tasks.rados.rados.0.smithi063.stdout:831: done (0 left) 2024-08-17T16:58:13.884 INFO:tasks.rados.rados.0.smithi063.stdout:832: rollback oid 18 current snap is 104 2024-08-17T16:58:13.884 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 18 to 103 2024-08-17T16:58:13.884 INFO:tasks.rados.rados.0.smithi063.stdout:833: snap_create 2024-08-17T16:58:13.886 INFO:tasks.rados.rados.0.smithi063.stdout:832: finishing rollback tid 0 to smithi06387500-18 2024-08-17T16:58:13.886 INFO:tasks.rados.rados.0.smithi063.stdout:832: finishing rollback tid 1 to smithi06387500-18 2024-08-17T16:58:13.886 INFO:tasks.rados.rados.0.smithi063.stdout:832: finishing rollback tid 2 to smithi06387500-18 2024-08-17T16:58:13.886 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 300 (ObjNum 328 snap 99 seq_num 328) dirty exists 2024-08-17T16:58:14.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:13 smithi063 ceph-mon[31063]: osdmap e277: 8 total, 8 up, 8 in 2024-08-17T16:58:14.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:13 smithi063 ceph-mon[31063]: pgmap v99: 89 pgs: 89 active+clean; 239 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2024-08-17T16:58:14.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:13 smithi063 ceph-mon[42680]: osdmap e277: 8 total, 8 up, 8 in 2024-08-17T16:58:14.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:13 smithi063 ceph-mon[42680]: pgmap v99: 89 pgs: 89 active+clean; 239 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2024-08-17T16:58:14.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:13 smithi162 ceph-mon[34633]: osdmap e277: 8 total, 8 up, 8 in 2024-08-17T16:58:14.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:13 smithi162 ceph-mon[34633]: pgmap v99: 89 pgs: 89 active+clean; 239 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 19 op/s 2024-08-17T16:58:14.871 INFO:tasks.rados.rados.0.smithi063.stdout:832: done (1 left) 2024-08-17T16:58:14.871 INFO:tasks.rados.rados.0.smithi063.stdout:833: done (0 left) 2024-08-17T16:58:14.871 INFO:tasks.rados.rados.0.smithi063.stdout:834: rollback oid 38 current snap is 105 2024-08-17T16:58:14.871 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 38 to 104 2024-08-17T16:58:14.871 INFO:tasks.rados.rados.0.smithi063.stdout:835: read oid 40 snap 104 2024-08-17T16:58:14.871 INFO:tasks.rados.rados.0.smithi063.stdout:835: expect (ObjNum 283 snap 86 seq_num 283) 2024-08-17T16:58:14.871 INFO:tasks.rados.rados.0.smithi063.stdout:836: write oid 39 current snap is 105 2024-08-17T16:58:14.871 INFO:tasks.rados.rados.0.smithi063.stdout:836: seq_num 349 ranges {665643=697786,2039745=753122,3534742=330901} 2024-08-17T16:58:14.878 INFO:tasks.rados.rados.0.smithi063.stdout:836: writing smithi06387500-39 from 665643 to 1363429 tid 1 2024-08-17T16:58:14.886 INFO:tasks.rados.rados.0.smithi063.stdout:836: writing smithi06387500-39 from 2039745 to 2792867 tid 2 2024-08-17T16:58:14.892 INFO:tasks.rados.rados.0.smithi063.stdout:836: writing smithi06387500-39 from 3534742 to 3865643 tid 3 2024-08-17T16:58:14.893 INFO:tasks.rados.rados.0.smithi063.stdout:837: write oid 36 current snap is 105 2024-08-17T16:58:14.893 INFO:tasks.rados.rados.0.smithi063.stdout:834: finishing rollback tid 1 to smithi06387500-38 2024-08-17T16:58:14.893 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 0 (ObjNum 33 snap 0 seq_num 1786890032) dirty dne 2024-08-17T16:58:14.893 INFO:tasks.rados.rados.0.smithi063.stdout:837: seq_num 350 ranges {682450=489388,1814580=496780,2878518=788028} 2024-08-17T16:58:14.899 INFO:tasks.rados.rados.0.smithi063.stdout:837: writing smithi06387500-36 from 682450 to 1171838 tid 1 2024-08-17T16:58:14.905 INFO:tasks.rados.rados.0.smithi063.stdout:837: writing smithi06387500-36 from 1814580 to 2311360 tid 2 2024-08-17T16:58:14.913 INFO:tasks.rados.rados.0.smithi063.stdout:837: writing smithi06387500-36 from 2878518 to 3666546 tid 3 2024-08-17T16:58:14.913 INFO:tasks.rados.rados.0.smithi063.stdout:834: done (3 left) 2024-08-17T16:58:14.913 INFO:tasks.rados.rados.0.smithi063.stdout:838: snap_remove snap 102 2024-08-17T16:58:15.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:14 smithi063 ceph-mon[31063]: osdmap e278: 8 total, 8 up, 8 in 2024-08-17T16:58:15.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:14 smithi063 ceph-mon[31063]: osdmap e279: 8 total, 8 up, 8 in 2024-08-17T16:58:15.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:14 smithi063 ceph-mon[42680]: osdmap e278: 8 total, 8 up, 8 in 2024-08-17T16:58:15.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:14 smithi063 ceph-mon[42680]: osdmap e279: 8 total, 8 up, 8 in 2024-08-17T16:58:15.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:14 smithi162 ceph-mon[34633]: osdmap e278: 8 total, 8 up, 8 in 2024-08-17T16:58:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:14 smithi162 ceph-mon[34633]: osdmap e279: 8 total, 8 up, 8 in 2024-08-17T16:58:15.891 INFO:tasks.rados.rados.0.smithi063.stdout:838: done (3 left) 2024-08-17T16:58:15.891 INFO:tasks.rados.rados.0.smithi063.stdout:839: write oid 33 current snap is 105 2024-08-17T16:58:15.891 INFO:tasks.rados.rados.0.smithi063.stdout:839: seq_num 351 ranges {699257=564637,1873062=640438,3305941=593316} 2024-08-17T16:58:15.898 INFO:tasks.rados.rados.0.smithi063.stdout:839: writing smithi06387500-33 from 699257 to 1263894 tid 1 2024-08-17T16:58:15.905 INFO:tasks.rados.rados.0.smithi063.stdout:839: writing smithi06387500-33 from 1873062 to 2513500 tid 2 2024-08-17T16:58:15.913 INFO:tasks.rados.rados.0.smithi063.stdout:839: writing smithi06387500-33 from 3305941 to 3899257 tid 3 2024-08-17T16:58:15.913 INFO:tasks.rados.rados.0.smithi063.stdout:840: write oid 5 current snap is 105 2024-08-17T16:58:15.913 INFO:tasks.rados.rados.0.smithi063.stdout:840: seq_num 352 ranges {716064=639886,1815191=500449,2817011=526925} 2024-08-17T16:58:15.921 INFO:tasks.rados.rados.0.smithi063.stdout:840: writing smithi06387500-5 from 716064 to 1355950 tid 1 2024-08-17T16:58:15.926 INFO:tasks.rados.rados.0.smithi063.stdout:840: writing smithi06387500-5 from 1815191 to 2315640 tid 2 2024-08-17T16:58:15.932 INFO:tasks.rados.rados.0.smithi063.stdout:840: writing smithi06387500-5 from 2817011 to 3343936 tid 3 2024-08-17T16:58:15.932 INFO:tasks.rados.rados.0.smithi063.stdout:841: write oid 3 current snap is 105 2024-08-17T16:58:15.932 INFO:tasks.rados.rados.0.smithi063.stdout:835: expect (ObjNum 283 snap 86 seq_num 283) 2024-08-17T16:58:16.114 INFO:tasks.rados.rados.0.smithi063.stdout:836: finishing write tid 1 to smithi06387500-39 2024-08-17T16:58:16.114 INFO:tasks.rados.rados.0.smithi063.stdout:836: finishing write tid 2 to smithi06387500-39 2024-08-17T16:58:16.114 INFO:tasks.rados.rados.0.smithi063.stdout:841: seq_num 353 ranges {732871=715135,1873673=644107,3244434=688437} 2024-08-17T16:58:16.122 INFO:tasks.rados.rados.0.smithi063.stdout:841: writing smithi06387500-3 from 732871 to 1448006 tid 1 2024-08-17T16:58:16.128 INFO:tasks.rados.rados.0.smithi063.stdout:841: writing smithi06387500-3 from 1873673 to 2517780 tid 2 2024-08-17T16:58:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:841: writing smithi06387500-3 from 3244434 to 3932871 tid 3 2024-08-17T16:58:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:835: done (5 left) 2024-08-17T16:58:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:842: snap_create 2024-08-17T16:58:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:836: finishing write tid 3 to smithi06387500-39 2024-08-17T16:58:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:836: finishing write tid 4 to smithi06387500-39 2024-08-17T16:58:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:836: finishing write tid 5 to smithi06387500-39 2024-08-17T16:58:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:836: oid 39 updating version 0 to 201 2024-08-17T16:58:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:836: oid 39 updating version 201 to 203 2024-08-17T16:58:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:836: oid 39 version 203 is already newer than 200 2024-08-17T16:58:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:836: oid 39 version 203 is already newer than 202 2024-08-17T16:58:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 203 (ObjNum 349 snap 105 seq_num 349) dirty exists 2024-08-17T16:58:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:836: left oid 39 (ObjNum 349 snap 105 seq_num 349) 2024-08-17T16:58:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:837: finishing write tid 1 to smithi06387500-36 2024-08-17T16:58:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:837: finishing write tid 2 to smithi06387500-36 2024-08-17T16:58:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:837: finishing write tid 3 to smithi06387500-36 2024-08-17T16:58:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:837: finishing write tid 4 to smithi06387500-36 2024-08-17T16:58:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:837: finishing write tid 5 to smithi06387500-36 2024-08-17T16:58:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:837: oid 36 updating version 0 to 873 2024-08-17T16:58:16.138 INFO:tasks.rados.rados.0.smithi063.stdout:837: oid 36 updating version 873 to 874 2024-08-17T16:58:16.138 INFO:tasks.rados.rados.0.smithi063.stdout:837: oid 36 updating version 874 to 875 2024-08-17T16:58:16.138 INFO:tasks.rados.rados.0.smithi063.stdout:837: oid 36 version 875 is already newer than 872 2024-08-17T16:58:16.138 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 875 (ObjNum 350 snap 105 seq_num 350) dirty exists 2024-08-17T16:58:16.138 INFO:tasks.rados.rados.0.smithi063.stdout:837: left oid 36 (ObjNum 350 snap 105 seq_num 350) 2024-08-17T16:58:16.138 INFO:tasks.rados.rados.0.smithi063.stdout:839: finishing write tid 1 to smithi06387500-33 2024-08-17T16:58:16.138 INFO:tasks.rados.rados.0.smithi063.stdout:839: finishing write tid 2 to smithi06387500-33 2024-08-17T16:58:16.138 INFO:tasks.rados.rados.0.smithi063.stdout:839: finishing write tid 3 to smithi06387500-33 2024-08-17T16:58:16.138 INFO:tasks.rados.rados.0.smithi063.stdout:839: finishing write tid 4 to smithi06387500-33 2024-08-17T16:58:16.138 INFO:tasks.rados.rados.0.smithi063.stdout:839: finishing write tid 5 to smithi06387500-33 2024-08-17T16:58:16.138 INFO:tasks.rados.rados.0.smithi063.stdout:839: oid 33 updating version 0 to 232 2024-08-17T16:58:16.138 INFO:tasks.rados.rados.0.smithi063.stdout:839: oid 33 version 232 is already newer than 230 2024-08-17T16:58:16.139 INFO:tasks.rados.rados.0.smithi063.stdout:839: oid 33 updating version 232 to 233 2024-08-17T16:58:16.139 INFO:tasks.rados.rados.0.smithi063.stdout:839: oid 33 version 233 is already newer than 231 2024-08-17T16:58:16.139 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 233 (ObjNum 351 snap 105 seq_num 351) dirty exists 2024-08-17T16:58:16.139 INFO:tasks.rados.rados.0.smithi063.stdout:839: left oid 33 (ObjNum 351 snap 105 seq_num 351) 2024-08-17T16:58:16.139 INFO:tasks.rados.rados.0.smithi063.stdout:840: finishing write tid 1 to smithi06387500-5 2024-08-17T16:58:16.139 INFO:tasks.rados.rados.0.smithi063.stdout:840: finishing write tid 2 to smithi06387500-5 2024-08-17T16:58:16.139 INFO:tasks.rados.rados.0.smithi063.stdout:840: finishing write tid 3 to smithi06387500-5 2024-08-17T16:58:16.139 INFO:tasks.rados.rados.0.smithi063.stdout:840: finishing write tid 4 to smithi06387500-5 2024-08-17T16:58:16.139 INFO:tasks.rados.rados.0.smithi063.stdout:840: finishing write tid 5 to smithi06387500-5 2024-08-17T16:58:16.139 INFO:tasks.rados.rados.0.smithi063.stdout:840: oid 5 updating version 0 to 402 2024-08-17T16:58:16.139 INFO:tasks.rados.rados.0.smithi063.stdout:840: oid 5 updating version 402 to 404 2024-08-17T16:58:16.139 INFO:tasks.rados.rados.0.smithi063.stdout:840: oid 5 updating version 404 to 405 2024-08-17T16:58:16.139 INFO:tasks.rados.rados.0.smithi063.stdout:840: oid 5 version 405 is already newer than 403 2024-08-17T16:58:16.139 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 405 (ObjNum 352 snap 105 seq_num 352) dirty exists 2024-08-17T16:58:16.140 INFO:tasks.rados.rados.0.smithi063.stdout:840: left oid 5 (ObjNum 352 snap 105 seq_num 352) 2024-08-17T16:58:16.140 INFO:tasks.rados.rados.0.smithi063.stdout:841: finishing write tid 1 to smithi06387500-3 2024-08-17T16:58:16.140 INFO:tasks.rados.rados.0.smithi063.stdout:841: finishing write tid 2 to smithi06387500-3 2024-08-17T16:58:16.140 INFO:tasks.rados.rados.0.smithi063.stdout:841: finishing write tid 3 to smithi06387500-3 2024-08-17T16:58:16.140 INFO:tasks.rados.rados.0.smithi063.stdout:841: finishing write tid 4 to smithi06387500-3 2024-08-17T16:58:16.141 INFO:tasks.rados.rados.0.smithi063.stdout:841: finishing write tid 5 to smithi06387500-3 2024-08-17T16:58:16.141 INFO:tasks.rados.rados.0.smithi063.stdout:841: oid 3 updating version 0 to 258 2024-08-17T16:58:16.141 INFO:tasks.rados.rados.0.smithi063.stdout:841: oid 3 version 258 is already newer than 255 2024-08-17T16:58:16.141 INFO:tasks.rados.rados.0.smithi063.stdout:841: oid 3 version 258 is already newer than 256 2024-08-17T16:58:16.141 INFO:tasks.rados.rados.0.smithi063.stdout:841: oid 3 version 258 is already newer than 257 2024-08-17T16:58:16.141 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 258 (ObjNum 353 snap 105 seq_num 353) dirty exists 2024-08-17T16:58:16.141 INFO:tasks.rados.rados.0.smithi063.stdout:841: left oid 3 (ObjNum 353 snap 105 seq_num 353) 2024-08-17T16:58:16.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:15 smithi063 ceph-mon[42680]: pgmap v102: 89 pgs: 89 active+clean; 239 MiB data, 759 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:58:16.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:15 smithi063 ceph-mon[31063]: pgmap v102: 89 pgs: 89 active+clean; 239 MiB data, 759 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:58:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:15 smithi162 ceph-mon[34633]: pgmap v102: 89 pgs: 89 active+clean; 239 MiB data, 759 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:58:16.893 INFO:tasks.rados.rados.0.smithi063.stdout:836: done (5 left) 2024-08-17T16:58:16.893 INFO:tasks.rados.rados.0.smithi063.stdout:837: done (4 left) 2024-08-17T16:58:16.893 INFO:tasks.rados.rados.0.smithi063.stdout:839: done (3 left) 2024-08-17T16:58:16.893 INFO:tasks.rados.rados.0.smithi063.stdout:840: done (2 left) 2024-08-17T16:58:16.894 INFO:tasks.rados.rados.0.smithi063.stdout:841: done (1 left) 2024-08-17T16:58:16.894 INFO:tasks.rados.rados.0.smithi063.stdout:842: done (0 left) 2024-08-17T16:58:16.894 INFO:tasks.rados.rados.0.smithi063.stdout:843: rollback oid 4 current snap is 106 2024-08-17T16:58:16.894 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 4 to 105 2024-08-17T16:58:16.894 INFO:tasks.rados.rados.0.smithi063.stdout:844: write oid 5 current snap is 106 2024-08-17T16:58:16.894 INFO:tasks.rados.rados.0.smithi063.stdout:844: seq_num 354 ranges {749678=790384,2332155=504118,3271857=549469} 2024-08-17T16:58:16.902 INFO:tasks.rados.rados.0.smithi063.stdout:844: writing smithi06387500-5 from 749678 to 1540062 tid 1 2024-08-17T16:58:16.909 INFO:tasks.rados.rados.0.smithi063.stdout:844: writing smithi06387500-5 from 2332155 to 2836273 tid 2 2024-08-17T16:58:16.915 INFO:tasks.rados.rados.0.smithi063.stdout:844: writing smithi06387500-5 from 3271857 to 3821326 tid 3 2024-08-17T16:58:16.915 INFO:tasks.rados.rados.0.smithi063.stdout:845: snap_create 2024-08-17T16:58:16.915 INFO:tasks.rados.rados.0.smithi063.stdout:843: finishing rollback tid 1 to smithi06387500-4 2024-08-17T16:58:16.915 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 822 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-08-17T16:58:16.915 INFO:tasks.rados.rados.0.smithi063.stdout:844: finishing write tid 1 to smithi06387500-5 2024-08-17T16:58:16.915 INFO:tasks.rados.rados.0.smithi063.stdout:844: finishing write tid 2 to smithi06387500-5 2024-08-17T16:58:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:844: finishing write tid 3 to smithi06387500-5 2024-08-17T16:58:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:844: finishing write tid 4 to smithi06387500-5 2024-08-17T16:58:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:844: finishing write tid 5 to smithi06387500-5 2024-08-17T16:58:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:844: oid 5 updating version 0 to 407 2024-08-17T16:58:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:844: oid 5 updating version 407 to 409 2024-08-17T16:58:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:844: oid 5 version 409 is already newer than 408 2024-08-17T16:58:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:844: oid 5 updating version 409 to 410 2024-08-17T16:58:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 410 (ObjNum 354 snap 106 seq_num 354) dirty exists 2024-08-17T16:58:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:844: left oid 5 (ObjNum 354 snap 106 seq_num 354) 2024-08-17T16:58:17.157 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:16 smithi162 ceph-mon[34633]: osdmap e280: 8 total, 8 up, 8 in 2024-08-17T16:58:17.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:16 smithi063 ceph-mon[42680]: osdmap e280: 8 total, 8 up, 8 in 2024-08-17T16:58:17.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:16 smithi063 ceph-mon[31063]: osdmap e280: 8 total, 8 up, 8 in 2024-08-17T16:58:17.830 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T16:58:17.907 INFO:tasks.rados.rados.0.smithi063.stdout:843: done (2 left) 2024-08-17T16:58:17.907 INFO:tasks.rados.rados.0.smithi063.stdout:844: done (1 left) 2024-08-17T16:58:17.907 INFO:tasks.rados.rados.0.smithi063.stdout:845: done (0 left) 2024-08-17T16:58:17.907 INFO:tasks.rados.rados.0.smithi063.stdout:846: read oid 12 snap -1 2024-08-17T16:58:17.908 INFO:tasks.rados.rados.0.smithi063.stdout:846: expect (ObjNum 259 snap 79 seq_num 259) 2024-08-17T16:58:17.908 INFO:tasks.rados.rados.0.smithi063.stdout:847: rollback oid 9 current snap is 107 2024-08-17T16:58:17.908 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 9 to 98 2024-08-17T16:58:17.908 INFO:tasks.rados.rados.0.smithi063.stdout:848: snap_remove snap 96 2024-08-17T16:58:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:17 smithi162 ceph-mon[34633]: osdmap e281: 8 total, 8 up, 8 in 2024-08-17T16:58:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:17 smithi162 ceph-mon[34633]: pgmap v105: 89 pgs: 89 active+clean; 259 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 23 op/s 2024-08-17T16:58:18.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:17 smithi063 ceph-mon[42680]: osdmap e281: 8 total, 8 up, 8 in 2024-08-17T16:58:18.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:17 smithi063 ceph-mon[42680]: pgmap v105: 89 pgs: 89 active+clean; 259 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 23 op/s 2024-08-17T16:58:18.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:17 smithi063 ceph-mon[31063]: osdmap e281: 8 total, 8 up, 8 in 2024-08-17T16:58:18.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:17 smithi063 ceph-mon[31063]: pgmap v105: 89 pgs: 89 active+clean; 259 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 23 op/s 2024-08-17T16:58:18.418 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T16:58:18.418 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (4m) 24s ago 5m 26.0M - 0.25.0 c8568f914cd2 87286b24f231 2024-08-17T16:58:18.418 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (5m) 0s ago 5m 86.6M - 9.4.7 954c08fa6188 fc8d62329012 2024-08-17T16:58:18.418 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (4m) 24s ago 4m 17.5M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T16:58:18.418 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (4m) 0s ago 4m 17.2M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T16:58:18.418 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,9283,8765 running (50s) 0s ago 9m 453M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb da48034a2895 2024-08-17T16:58:18.418 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:8443,9283,8765 running (77s) 24s ago 11m 511M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 943e639cede6 2024-08-17T16:58:18.418 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (11m) 24s ago 11m 58.3M 2048M 18.2.4-719-gced649f8 c870753d67c2 0c71ddd7ada6 2024-08-17T16:58:18.419 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (10m) 0s ago 10m 52.7M 2048M 18.2.4-719-gced649f8 c870753d67c2 d4ecbe65a0c8 2024-08-17T16:58:18.419 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (9m) 24s ago 9m 49.1M 2048M 18.2.4-719-gced649f8 c870753d67c2 b3f2c9339c6b 2024-08-17T16:58:18.419 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 running (35s) 24s ago 5m 3363k - 1.7.0 72c9c2088986 34f55f5de2aa 2024-08-17T16:58:18.419 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (28s) 0s ago 5m 8090k - 1.7.0 72c9c2088986 5bb162d0f1a9 2024-08-17T16:58:18.419 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (9m) 24s ago 9m 87.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 f3b910ed49b8 2024-08-17T16:58:18.419 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (8m) 24s ago 8m 70.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 5d6fdb6000ad 2024-08-17T16:58:18.419 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (8m) 24s ago 8m 77.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 a3cbe8edb49a 2024-08-17T16:58:18.419 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (8m) 24s ago 8m 70.7M 2480M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T16:58:18.419 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (7m) 0s ago 7m 91.2M 2480M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T16:58:18.419 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (7m) 0s ago 7m 87.6M 2480M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T16:58:18.419 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (7m) 0s ago 6m 64.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T16:58:18.419 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (6m) 0s ago 6m 76.3M 2480M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T16:58:18.419 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (7s) 0s ago 6m 32.9M - 2.51.0 1d3b7f56885b dfe66b2c5568 2024-08-17T16:58:18.768 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:58:18.768 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T16:58:18.768 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T16:58:18.769 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:58:18.769 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T16:58:18.769 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T16:58:18.769 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:58:18.769 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T16:58:18.769 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T16:58:18.769 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:58:18.769 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T16:58:18.769 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T16:58:18.770 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:58:18.770 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T16:58:18.770 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 13, 2024-08-17T16:58:18.770 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T16:58:18.770 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T16:58:18.770 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:58:18.909 INFO:tasks.rados.rados.0.smithi063.stdout:846: expect (ObjNum 259 snap 79 seq_num 259) 2024-08-17T16:58:18.989 INFO:tasks.rados.rados.0.smithi063.stdout:847: finishing rollback tid 1 to smithi06387500-9 2024-08-17T16:58:18.990 INFO:tasks.rados.rados.0.smithi063.stdout:847: finishing rollback tid 2 to smithi06387500-9 2024-08-17T16:58:18.990 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 485 (ObjNum 269 snap 84 seq_num 269) dirty exists 2024-08-17T16:58:18.990 INFO:tasks.rados.rados.0.smithi063.stdout:846: done (2 left) 2024-08-17T16:58:18.990 INFO:tasks.rados.rados.0.smithi063.stdout:847: done (1 left) 2024-08-17T16:58:18.990 INFO:tasks.rados.rados.0.smithi063.stdout:848: done (0 left) 2024-08-17T16:58:18.990 INFO:tasks.rados.rados.0.smithi063.stdout:849: write oid 40 current snap is 107 2024-08-17T16:58:18.990 INFO:tasks.rados.rados.0.smithi063.stdout:849: seq_num 355 ranges {766485=465633,1990637=647776,3299280=667205} 2024-08-17T16:58:18.996 INFO:tasks.rados.rados.0.smithi063.stdout:849: writing smithi06387500-40 from 766485 to 1232118 tid 1 2024-08-17T16:58:19.003 INFO:tasks.rados.rados.0.smithi063.stdout:849: writing smithi06387500-40 from 1990637 to 2638413 tid 2 2024-08-17T16:58:19.011 INFO:tasks.rados.rados.0.smithi063.stdout:849: writing smithi06387500-40 from 3299280 to 3966485 tid 3 2024-08-17T16:58:19.011 INFO:tasks.rados.rados.0.smithi063.stdout:850: delete oid 23 current snap is 107 2024-08-17T16:58:19.011 INFO:tasks.rados.rados.0.smithi063.stdout:849: finishing write tid 1 to smithi06387500-40 2024-08-17T16:58:19.011 INFO:tasks.rados.rados.0.smithi063.stdout:849: finishing write tid 2 to smithi06387500-40 2024-08-17T16:58:19.013 INFO:tasks.rados.rados.0.smithi063.stdout:850: done (1 left) 2024-08-17T16:58:19.013 INFO:tasks.rados.rados.0.smithi063.stdout:851: read oid 5 snap -1 2024-08-17T16:58:19.013 INFO:tasks.rados.rados.0.smithi063.stdout:851: expect (ObjNum 354 snap 106 seq_num 354) 2024-08-17T16:58:19.013 INFO:tasks.rados.rados.0.smithi063.stdout:852: rollback oid 26 current snap is 107 2024-08-17T16:58:19.013 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 26 to 103 2024-08-17T16:58:19.013 INFO:tasks.rados.rados.0.smithi063.stdout:853: snap_create 2024-08-17T16:58:19.014 INFO:tasks.rados.rados.0.smithi063.stdout:849: finishing write tid 3 to smithi06387500-40 2024-08-17T16:58:19.016 INFO:tasks.rados.rados.0.smithi063.stdout:849: finishing write tid 4 to smithi06387500-40 2024-08-17T16:58:19.017 INFO:tasks.rados.rados.0.smithi063.stdout:849: finishing write tid 5 to smithi06387500-40 2024-08-17T16:58:19.017 INFO:tasks.rados.rados.0.smithi063.stdout:849: oid 40 updating version 0 to 308 2024-08-17T16:58:19.017 INFO:tasks.rados.rados.0.smithi063.stdout:849: oid 40 updating version 308 to 309 2024-08-17T16:58:19.017 INFO:tasks.rados.rados.0.smithi063.stdout:849: oid 40 updating version 309 to 310 2024-08-17T16:58:19.017 INFO:tasks.rados.rados.0.smithi063.stdout:849: oid 40 updating version 310 to 311 2024-08-17T16:58:19.017 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 311 (ObjNum 355 snap 107 seq_num 355) dirty exists 2024-08-17T16:58:19.017 INFO:tasks.rados.rados.0.smithi063.stdout:849: left oid 40 (ObjNum 355 snap 107 seq_num 355) 2024-08-17T16:58:19.018 INFO:tasks.rados.rados.0.smithi063.stdout:852: finishing rollback tid 0 to smithi06387500-26 2024-08-17T16:58:19.018 INFO:tasks.rados.rados.0.smithi063.stdout:852: finishing rollback tid 1 to smithi06387500-26 2024-08-17T16:58:19.018 INFO:tasks.rados.rados.0.smithi063.stdout:852: finishing rollback tid 2 to smithi06387500-26 2024-08-17T16:58:19.018 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 262 (ObjNum 220 snap 61 seq_num 220) dirty exists 2024-08-17T16:58:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:18 smithi063 ceph-mon[42680]: from='client.24962 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:18 smithi063 ceph-mon[42680]: osdmap e282: 8 total, 8 up, 8 in 2024-08-17T16:58:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:18 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:18 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:19.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:18 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3828411695' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:19.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:18 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:19.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:18 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:19.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:18 smithi063 ceph-mon[31063]: from='client.24962 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:19.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:18 smithi063 ceph-mon[31063]: osdmap e282: 8 total, 8 up, 8 in 2024-08-17T16:58:19.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:18 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:19.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:18 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:19.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:18 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3828411695' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:19.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:18 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:19.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:18 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:19.031 INFO:tasks.rados.rados.0.smithi063.stdout:851: expect (ObjNum 354 snap 106 seq_num 354) 2024-08-17T16:58:19.076 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:58:19.076 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T16:58:19.076 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-08-17T16:58:19.076 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-17T16:58:19.076 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-08-17T16:58:19.076 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-08-17T16:58:19.076 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-08-17T16:58:19.076 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "2/2 daemons upgraded", 2024-08-17T16:58:19.076 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading prometheus daemons", 2024-08-17T16:58:19.076 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T16:58:19.076 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:58:19.286 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:18 smithi162 ceph-mon[34633]: from='client.24962 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:19.286 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:18 smithi162 ceph-mon[34633]: osdmap e282: 8 total, 8 up, 8 in 2024-08-17T16:58:19.286 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:18 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:19.286 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:18 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:19.287 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:18 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3828411695' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:19.287 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:18 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:19.287 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:18 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:19.870 INFO:tasks.rados.rados.0.smithi063.stdout:849: done (3 left) 2024-08-17T16:58:19.870 INFO:tasks.rados.rados.0.smithi063.stdout:851: done (2 left) 2024-08-17T16:58:19.870 INFO:tasks.rados.rados.0.smithi063.stdout:852: done (1 left) 2024-08-17T16:58:19.870 INFO:tasks.rados.rados.0.smithi063.stdout:853: done (0 left) 2024-08-17T16:58:19.870 INFO:tasks.rados.rados.0.smithi063.stdout:854: read oid 25 snap -1 2024-08-17T16:58:19.870 INFO:tasks.rados.rados.0.smithi063.stdout:854: expect deleted 2024-08-17T16:58:19.870 INFO:tasks.rados.rados.0.smithi063.stdout:855: snap_remove snap 75 2024-08-17T16:58:20.234 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:19 smithi162 ceph-mon[34633]: from='client.24968 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:20.235 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:19 smithi162 ceph-mon[34633]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:20.235 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:19 smithi162 ceph-mon[34633]: osdmap e283: 8 total, 8 up, 8 in 2024-08-17T16:58:20.235 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:19 smithi162 ceph-mon[34633]: pgmap v108: 89 pgs: 89 active+clean; 259 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 23 op/s 2024-08-17T16:58:20.235 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:19 smithi162 ceph-mon[34633]: osdmap e284: 8 total, 8 up, 8 in 2024-08-17T16:58:20.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:19 smithi063 ceph-mon[42680]: from='client.24968 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:20.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:19 smithi063 ceph-mon[42680]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:20.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:19 smithi063 ceph-mon[42680]: osdmap e283: 8 total, 8 up, 8 in 2024-08-17T16:58:20.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:19 smithi063 ceph-mon[42680]: pgmap v108: 89 pgs: 89 active+clean; 259 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 23 op/s 2024-08-17T16:58:20.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:19 smithi063 ceph-mon[42680]: osdmap e284: 8 total, 8 up, 8 in 2024-08-17T16:58:20.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:19 smithi063 ceph-mon[31063]: from='client.24968 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:20.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:19 smithi063 ceph-mon[31063]: from='client.15081 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:20.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:19 smithi063 ceph-mon[31063]: osdmap e283: 8 total, 8 up, 8 in 2024-08-17T16:58:20.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:19 smithi063 ceph-mon[31063]: pgmap v108: 89 pgs: 89 active+clean; 259 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 23 op/s 2024-08-17T16:58:20.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:19 smithi063 ceph-mon[31063]: osdmap e284: 8 total, 8 up, 8 in 2024-08-17T16:58:20.880 INFO:tasks.rados.rados.0.smithi063.stdout:855: done (1 left) 2024-08-17T16:58:20.880 INFO:tasks.rados.rados.0.smithi063.stdout:856: write oid 12 current snap is 108 2024-08-17T16:58:20.880 INFO:tasks.rados.rados.0.smithi063.stdout:856: seq_num 357 ranges {400099=616131,1591248=408851} 2024-08-17T16:58:20.886 INFO:tasks.rados.rados.0.smithi063.stdout:856: writing smithi06387500-12 from 400099 to 1016230 tid 1 2024-08-17T16:58:20.892 INFO:tasks.rados.rados.0.smithi063.stdout:856: writing smithi06387500-12 from 1591248 to 2000099 tid 2 2024-08-17T16:58:20.892 INFO:tasks.rados.rados.0.smithi063.stdout:857: delete oid 11 current snap is 108 2024-08-17T16:58:20.892 INFO:tasks.rados.rados.0.smithi063.stdout:856: finishing write tid 1 to smithi06387500-12 2024-08-17T16:58:20.894 INFO:tasks.rados.rados.0.smithi063.stdout:854: done (2 left) 2024-08-17T16:58:20.894 INFO:tasks.rados.rados.0.smithi063.stdout:857: done (1 left) 2024-08-17T16:58:20.894 INFO:tasks.rados.rados.0.smithi063.stdout:858: read oid 26 snap -1 2024-08-17T16:58:20.894 INFO:tasks.rados.rados.0.smithi063.stdout:858: expect (ObjNum 220 snap 61 seq_num 220) 2024-08-17T16:58:20.894 INFO:tasks.rados.rados.0.smithi063.stdout:859: read oid 9 snap -1 2024-08-17T16:58:20.895 INFO:tasks.rados.rados.0.smithi063.stdout:859: expect (ObjNum 269 snap 84 seq_num 269) 2024-08-17T16:58:20.895 INFO:tasks.rados.rados.0.smithi063.stdout:860: snap_remove snap 103 2024-08-17T16:58:21.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:20 smithi063 ceph-mon[42680]: from='client.24835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:21.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:20 smithi063 ceph-mon[42680]: osdmap e285: 8 total, 8 up, 8 in 2024-08-17T16:58:21.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:20 smithi063 ceph-mon[31063]: from='client.24835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:21.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:20 smithi063 ceph-mon[31063]: osdmap e285: 8 total, 8 up, 8 in 2024-08-17T16:58:21.281 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:20 smithi162 ceph-mon[34633]: from='client.24835 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:21.281 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:20 smithi162 ceph-mon[34633]: osdmap e285: 8 total, 8 up, 8 in 2024-08-17T16:58:21.879 INFO:tasks.rados.rados.0.smithi063.stdout:856: finishing write tid 2 to smithi06387500-12 2024-08-17T16:58:21.879 INFO:tasks.rados.rados.0.smithi063.stdout:856: finishing write tid 3 to smithi06387500-12 2024-08-17T16:58:21.879 INFO:tasks.rados.rados.0.smithi063.stdout:856: finishing write tid 4 to smithi06387500-12 2024-08-17T16:58:21.879 INFO:tasks.rados.rados.0.smithi063.stdout:856: oid 12 updating version 0 to 174 2024-08-17T16:58:21.879 INFO:tasks.rados.rados.0.smithi063.stdout:856: oid 12 version 174 is already newer than 172 2024-08-17T16:58:21.879 INFO:tasks.rados.rados.0.smithi063.stdout:856: oid 12 version 174 is already newer than 173 2024-08-17T16:58:21.879 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 174 (ObjNum 357 snap 108 seq_num 357) dirty exists 2024-08-17T16:58:21.879 INFO:tasks.rados.rados.0.smithi063.stdout:856: left oid 12 (ObjNum 357 snap 108 seq_num 357) 2024-08-17T16:58:21.879 INFO:tasks.rados.rados.0.smithi063.stdout:858: expect (ObjNum 220 snap 61 seq_num 220) 2024-08-17T16:58:22.119 INFO:tasks.rados.rados.0.smithi063.stdout:859: expect (ObjNum 269 snap 84 seq_num 269) 2024-08-17T16:58:22.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:21 smithi063 ceph-mon[31063]: pgmap v111: 89 pgs: 89 active+clean; 273 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.4 MiB/s wr, 13 op/s 2024-08-17T16:58:22.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:21 smithi063 ceph-mon[31063]: osdmap e286: 8 total, 8 up, 8 in 2024-08-17T16:58:22.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:21 smithi063 ceph-mon[42680]: pgmap v111: 89 pgs: 89 active+clean; 273 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.4 MiB/s wr, 13 op/s 2024-08-17T16:58:22.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:21 smithi063 ceph-mon[42680]: osdmap e286: 8 total, 8 up, 8 in 2024-08-17T16:58:22.281 INFO:tasks.rados.rados.0.smithi063.stdout:856: done (3 left) 2024-08-17T16:58:22.281 INFO:tasks.rados.rados.0.smithi063.stdout:858: done (2 left) 2024-08-17T16:58:22.281 INFO:tasks.rados.rados.0.smithi063.stdout:859: done (1 left) 2024-08-17T16:58:22.281 INFO:tasks.rados.rados.0.smithi063.stdout:860: done (0 left) 2024-08-17T16:58:22.281 INFO:tasks.rados.rados.0.smithi063.stdout:861: delete oid 15 current snap is 108 2024-08-17T16:58:22.283 INFO:tasks.rados.rados.0.smithi063.stdout:861: done (0 left) 2024-08-17T16:58:22.283 INFO:tasks.rados.rados.0.smithi063.stdout:862: read oid 11 snap 63 2024-08-17T16:58:22.283 INFO:tasks.rados.rados.0.smithi063.stdout:862: expect deleted 2024-08-17T16:58:22.283 INFO:tasks.rados.rados.0.smithi063.stdout:863: rollback oid 3 current snap is 108 2024-08-17T16:58:22.283 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 3 to 101 2024-08-17T16:58:22.283 INFO:tasks.rados.rados.0.smithi063.stdout:864: rollback oid 41 current snap is 108 2024-08-17T16:58:22.283 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 41 to 63 2024-08-17T16:58:22.283 INFO:tasks.rados.rados.0.smithi063.stdout:865: snap_remove snap 87 2024-08-17T16:58:22.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:21 smithi162 ceph-mon[34633]: pgmap v111: 89 pgs: 89 active+clean; 273 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.4 MiB/s wr, 13 op/s 2024-08-17T16:58:22.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:21 smithi162 ceph-mon[34633]: osdmap e286: 8 total, 8 up, 8 in 2024-08-17T16:58:22.976 INFO:tasks.rados.rados.0.smithi063.stdout:865: done (3 left) 2024-08-17T16:58:22.976 INFO:tasks.rados.rados.0.smithi063.stdout:866: snap_remove snap 99 2024-08-17T16:58:23.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:58:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:58:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[42680]: pgmap v113: 89 pgs: 89 active+clean; 273 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.4 MiB/s wr, 13 op/s 2024-08-17T16:58:23.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:23.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:23.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:23.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:23.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:23.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:58:23.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:58:23.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:23.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:22 smithi063 ceph-mon[31063]: pgmap v113: 89 pgs: 89 active+clean; 273 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.4 MiB/s wr, 13 op/s 2024-08-17T16:58:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:58:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-08-17T16:58:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:23.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:22 smithi162 ceph-mon[34633]: pgmap v113: 89 pgs: 89 active+clean; 273 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 2.4 MiB/s wr, 13 op/s 2024-08-17T16:58:23.666 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:58:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:58:23] "GET /metrics HTTP/1.1" 200 36211 "" "Prometheus/2.51.0" 2024-08-17T16:58:23.986 INFO:tasks.rados.rados.0.smithi063.stdout:866: done (3 left) 2024-08-17T16:58:23.987 INFO:tasks.rados.rados.0.smithi063.stdout:867: write oid 27 current snap is 108 2024-08-17T16:58:23.987 INFO:tasks.rados.rados.0.smithi063.stdout:867: seq_num 360 ranges {450520=558231,1483047=515125} 2024-08-17T16:58:23.992 INFO:tasks.rados.rados.0.smithi063.stdout:867: writing smithi06387500-27 from 450520 to 1008751 tid 1 2024-08-17T16:58:23.999 INFO:tasks.rados.rados.0.smithi063.stdout:867: writing smithi06387500-27 from 1483047 to 1998172 tid 2 2024-08-17T16:58:23.999 INFO:tasks.rados.rados.0.smithi063.stdout:868: read oid 48 snap -1 2024-08-17T16:58:23.999 INFO:tasks.rados.rados.0.smithi063.stdout:868: expect (ObjNum 343 snap 103 seq_num 343) 2024-08-17T16:58:23.999 INFO:tasks.rados.rados.0.smithi063.stdout:869: read oid 4 snap 106 2024-08-17T16:58:23.999 INFO:tasks.rados.rados.0.smithi063.stdout:869: expect deleted 2024-08-17T16:58:23.999 INFO:tasks.rados.rados.0.smithi063.stdout:870: write oid 17 current snap is 108 2024-08-17T16:58:23.999 INFO:tasks.rados.rados.0.smithi063.stdout:870: seq_num 361 ranges {467327=633480,1825176=242151} 2024-08-17T16:58:24.005 INFO:tasks.rados.rados.0.smithi063.stdout:870: writing smithi06387500-17 from 467327 to 1100807 tid 1 2024-08-17T16:58:24.011 INFO:tasks.rados.rados.0.smithi063.stdout:870: writing smithi06387500-17 from 1825176 to 2067327 tid 2 2024-08-17T16:58:24.011 INFO:tasks.rados.rados.0.smithi063.stdout:871: read oid 50 snap -1 2024-08-17T16:58:24.011 INFO:tasks.rados.rados.0.smithi063.stdout:871: expect (ObjNum 300 snap 91 seq_num 300) 2024-08-17T16:58:24.011 INFO:tasks.rados.rados.0.smithi063.stdout:872: read oid 24 snap 106 2024-08-17T16:58:24.011 INFO:tasks.rados.rados.0.smithi063.stdout:872: expect (ObjNum 286 snap 87 seq_num 286) 2024-08-17T16:58:24.011 INFO:tasks.rados.rados.0.smithi063.stdout:873: write oid 25 current snap is 108 2024-08-17T16:58:24.011 INFO:tasks.rados.rados.0.smithi063.stdout:864: finishing rollback tid 1 to smithi06387500-41 2024-08-17T16:58:24.011 INFO:tasks.rados.rados.0.smithi063.stdout:863: finishing rollback tid 0 to smithi06387500-3 2024-08-17T16:58:24.011 INFO:tasks.rados.rados.0.smithi063.stdout:864: finishing rollback tid 2 to smithi06387500-41 2024-08-17T16:58:24.011 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 63 (ObjNum 73 snap 9 seq_num 73) dirty exists 2024-08-17T16:58:24.012 INFO:tasks.rados.rados.0.smithi063.stdout:863: finishing rollback tid 1 to smithi06387500-3 2024-08-17T16:58:24.012 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 271 (ObjNum 33 snap 0 seq_num 1807077376) dirty dne 2024-08-17T16:58:24.012 INFO:tasks.rados.rados.0.smithi063.stdout:867: finishing write tid 1 to smithi06387500-27 2024-08-17T16:58:24.012 INFO:tasks.rados.rados.0.smithi063.stdout:873: seq_num 362 ranges {484134=708729,1883658=200476} 2024-08-17T16:58:24.017 INFO:tasks.rados.rados.0.smithi063.stdout:873: writing smithi06387500-25 from 484134 to 1192863 tid 1 2024-08-17T16:58:24.022 INFO:tasks.rados.rados.0.smithi063.stdout:873: writing smithi06387500-25 from 1883658 to 2084134 tid 2 2024-08-17T16:58:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:862: done (9 left) 2024-08-17T16:58:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:863: done (8 left) 2024-08-17T16:58:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:864: done (7 left) 2024-08-17T16:58:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:874: snap_create 2024-08-17T16:58:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:867: finishing write tid 2 to smithi06387500-27 2024-08-17T16:58:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:867: finishing write tid 3 to smithi06387500-27 2024-08-17T16:58:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:867: finishing write tid 4 to smithi06387500-27 2024-08-17T16:58:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:867: oid 27 updating version 0 to 153 2024-08-17T16:58:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:867: oid 27 version 153 is already newer than 152 2024-08-17T16:58:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:867: oid 27 version 153 is already newer than 151 2024-08-17T16:58:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 153 (ObjNum 360 snap 108 seq_num 360) dirty exists 2024-08-17T16:58:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:867: left oid 27 (ObjNum 360 snap 108 seq_num 360) 2024-08-17T16:58:24.024 INFO:tasks.rados.rados.0.smithi063.stdout:870: finishing write tid 1 to smithi06387500-17 2024-08-17T16:58:24.024 INFO:tasks.rados.rados.0.smithi063.stdout:868: expect (ObjNum 343 snap 103 seq_num 343) 2024-08-17T16:58:24.277 INFO:tasks.rados.rados.0.smithi063.stdout:870: finishing write tid 2 to smithi06387500-17 2024-08-17T16:58:24.277 INFO:tasks.rados.rados.0.smithi063.stdout:871: expect (ObjNum 300 snap 91 seq_num 300) 2024-08-17T16:58:24.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:23 smithi063 ceph-mon[42680]: osdmap e287: 8 total, 8 up, 8 in 2024-08-17T16:58:24.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:23 smithi063 ceph-mon[42680]: Upgrade: Updating alertmanager.a 2024-08-17T16:58:24.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:23 smithi063 ceph-mon[42680]: Deploying daemon alertmanager.a on smithi063 2024-08-17T16:58:24.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:23 smithi063 ceph-mon[31063]: osdmap e287: 8 total, 8 up, 8 in 2024-08-17T16:58:24.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:23 smithi063 ceph-mon[31063]: Upgrade: Updating alertmanager.a 2024-08-17T16:58:24.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:23 smithi063 ceph-mon[31063]: Deploying daemon alertmanager.a on smithi063 2024-08-17T16:58:24.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:23 smithi162 ceph-mon[34633]: osdmap e287: 8 total, 8 up, 8 in 2024-08-17T16:58:24.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:23 smithi162 ceph-mon[34633]: Upgrade: Updating alertmanager.a 2024-08-17T16:58:24.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:23 smithi162 ceph-mon[34633]: Deploying daemon alertmanager.a on smithi063 2024-08-17T16:58:24.574 INFO:tasks.rados.rados.0.smithi063.stdout:870: finishing write tid 3 to smithi06387500-17 2024-08-17T16:58:24.574 INFO:tasks.rados.rados.0.smithi063.stdout:870: finishing write tid 4 to smithi06387500-17 2024-08-17T16:58:24.574 INFO:tasks.rados.rados.0.smithi063.stdout:870: oid 17 updating version 0 to 281 2024-08-17T16:58:24.574 INFO:tasks.rados.rados.0.smithi063.stdout:870: oid 17 version 281 is already newer than 279 2024-08-17T16:58:24.574 INFO:tasks.rados.rados.0.smithi063.stdout:870: oid 17 version 281 is already newer than 280 2024-08-17T16:58:24.574 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 281 (ObjNum 361 snap 108 seq_num 361) dirty exists 2024-08-17T16:58:24.574 INFO:tasks.rados.rados.0.smithi063.stdout:870: left oid 17 (ObjNum 361 snap 108 seq_num 361) 2024-08-17T16:58:24.574 INFO:tasks.rados.rados.0.smithi063.stdout:872: expect (ObjNum 286 snap 87 seq_num 286) 2024-08-17T16:58:24.667 INFO:tasks.rados.rados.0.smithi063.stdout:873: finishing write tid 1 to smithi06387500-25 2024-08-17T16:58:24.667 INFO:tasks.rados.rados.0.smithi063.stdout:873: finishing write tid 2 to smithi06387500-25 2024-08-17T16:58:24.667 INFO:tasks.rados.rados.0.smithi063.stdout:873: finishing write tid 3 to smithi06387500-25 2024-08-17T16:58:24.667 INFO:tasks.rados.rados.0.smithi063.stdout:873: finishing write tid 4 to smithi06387500-25 2024-08-17T16:58:24.667 INFO:tasks.rados.rados.0.smithi063.stdout:873: oid 25 updating version 0 to 920 2024-08-17T16:58:24.668 INFO:tasks.rados.rados.0.smithi063.stdout:873: oid 25 version 920 is already newer than 919 2024-08-17T16:58:24.668 INFO:tasks.rados.rados.0.smithi063.stdout:873: oid 25 updating version 920 to 921 2024-08-17T16:58:24.668 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 921 (ObjNum 362 snap 108 seq_num 362) dirty exists 2024-08-17T16:58:24.668 INFO:tasks.rados.rados.0.smithi063.stdout:873: left oid 25 (ObjNum 362 snap 108 seq_num 362) 2024-08-17T16:58:24.872 INFO:tasks.rados.rados.0.smithi063.stdout:867: done (7 left) 2024-08-17T16:58:24.872 INFO:tasks.rados.rados.0.smithi063.stdout:868: done (6 left) 2024-08-17T16:58:24.872 INFO:tasks.rados.rados.0.smithi063.stdout:869: done (5 left) 2024-08-17T16:58:24.872 INFO:tasks.rados.rados.0.smithi063.stdout:870: done (4 left) 2024-08-17T16:58:24.872 INFO:tasks.rados.rados.0.smithi063.stdout:871: done (3 left) 2024-08-17T16:58:24.872 INFO:tasks.rados.rados.0.smithi063.stdout:872: done (2 left) 2024-08-17T16:58:24.873 INFO:tasks.rados.rados.0.smithi063.stdout:873: done (1 left) 2024-08-17T16:58:24.873 INFO:tasks.rados.rados.0.smithi063.stdout:874: done (0 left) 2024-08-17T16:58:24.873 INFO:tasks.rados.rados.0.smithi063.stdout:875: write oid 45 current snap is 109 2024-08-17T16:58:24.873 INFO:tasks.rados.rados.0.smithi063.stdout:875: seq_num 363 ranges {500941=783978,1942140=158801} 2024-08-17T16:58:24.879 INFO:tasks.rados.rados.0.smithi063.stdout:875: writing smithi06387500-45 from 500941 to 1284919 tid 1 2024-08-17T16:58:24.884 INFO:tasks.rados.rados.0.smithi063.stdout:875: writing smithi06387500-45 from 1942140 to 2100941 tid 2 2024-08-17T16:58:24.884 INFO:tasks.rados.rados.0.smithi063.stdout:876: rollback oid 30 current snap is 109 2024-08-17T16:58:24.884 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 30 to 101 2024-08-17T16:58:24.885 INFO:tasks.rados.rados.0.smithi063.stdout:877: read oid 50 snap -1 2024-08-17T16:58:24.885 INFO:tasks.rados.rados.0.smithi063.stdout:877: expect (ObjNum 300 snap 91 seq_num 300) 2024-08-17T16:58:24.885 INFO:tasks.rados.rados.0.smithi063.stdout:878: delete oid 22 current snap is 109 2024-08-17T16:58:24.886 INFO:tasks.rados.rados.0.smithi063.stdout:878: done (3 left) 2024-08-17T16:58:24.886 INFO:tasks.rados.rados.0.smithi063.stdout:879: snap_create 2024-08-17T16:58:24.887 INFO:tasks.rados.rados.0.smithi063.stdout:875: finishing write tid 1 to smithi06387500-45 2024-08-17T16:58:24.887 INFO:tasks.rados.rados.0.smithi063.stdout:875: finishing write tid 2 to smithi06387500-45 2024-08-17T16:58:24.887 INFO:tasks.rados.rados.0.smithi063.stdout:875: finishing write tid 3 to smithi06387500-45 2024-08-17T16:58:24.889 INFO:tasks.rados.rados.0.smithi063.stdout:875: finishing write tid 4 to smithi06387500-45 2024-08-17T16:58:24.889 INFO:tasks.rados.rados.0.smithi063.stdout:875: oid 45 updating version 0 to 211 2024-08-17T16:58:24.889 INFO:tasks.rados.rados.0.smithi063.stdout:875: oid 45 updating version 211 to 212 2024-08-17T16:58:24.889 INFO:tasks.rados.rados.0.smithi063.stdout:875: oid 45 updating version 212 to 213 2024-08-17T16:58:24.889 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 213 (ObjNum 363 snap 109 seq_num 363) dirty exists 2024-08-17T16:58:24.889 INFO:tasks.rados.rados.0.smithi063.stdout:875: left oid 45 (ObjNum 363 snap 109 seq_num 363) 2024-08-17T16:58:24.889 INFO:tasks.rados.rados.0.smithi063.stdout:876: finishing rollback tid 0 to smithi06387500-30 2024-08-17T16:58:24.889 INFO:tasks.rados.rados.0.smithi063.stdout:876: finishing rollback tid 1 to smithi06387500-30 2024-08-17T16:58:24.889 INFO:tasks.rados.rados.0.smithi063.stdout:876: finishing rollback tid 2 to smithi06387500-30 2024-08-17T16:58:24.890 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 511 (ObjNum 336 snap 101 seq_num 336) dirty exists 2024-08-17T16:58:24.892 INFO:tasks.rados.rados.0.smithi063.stdout:877: expect (ObjNum 300 snap 91 seq_num 300) 2024-08-17T16:58:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:24 smithi063 ceph-mon[31063]: osdmap e288: 8 total, 8 up, 8 in 2024-08-17T16:58:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:24 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:24 smithi063 ceph-mon[31063]: osdmap e289: 8 total, 8 up, 8 in 2024-08-17T16:58:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:24 smithi063 ceph-mon[31063]: pgmap v117: 89 pgs: 89 active+clean; 273 MiB data, 821 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:58:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:24 smithi063 ceph-mon[42680]: osdmap e288: 8 total, 8 up, 8 in 2024-08-17T16:58:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:24 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:24 smithi063 ceph-mon[42680]: osdmap e289: 8 total, 8 up, 8 in 2024-08-17T16:58:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:24 smithi063 ceph-mon[42680]: pgmap v117: 89 pgs: 89 active+clean; 273 MiB data, 821 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:58:25.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:24 smithi162 ceph-mon[34633]: osdmap e288: 8 total, 8 up, 8 in 2024-08-17T16:58:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:24 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:24 smithi162 ceph-mon[34633]: osdmap e289: 8 total, 8 up, 8 in 2024-08-17T16:58:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:24 smithi162 ceph-mon[34633]: pgmap v117: 89 pgs: 89 active+clean; 273 MiB data, 821 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:58:25.876 INFO:tasks.rados.rados.0.smithi063.stdout:875: done (3 left) 2024-08-17T16:58:25.876 INFO:tasks.rados.rados.0.smithi063.stdout:876: done (2 left) 2024-08-17T16:58:25.877 INFO:tasks.rados.rados.0.smithi063.stdout:877: done (1 left) 2024-08-17T16:58:25.877 INFO:tasks.rados.rados.0.smithi063.stdout:879: done (0 left) 2024-08-17T16:58:25.877 INFO:tasks.rados.rados.0.smithi063.stdout:880: snap_create 2024-08-17T16:58:26.878 INFO:tasks.rados.rados.0.smithi063.stdout:880: done (0 left) 2024-08-17T16:58:26.878 INFO:tasks.rados.rados.0.smithi063.stdout:881: snap_create 2024-08-17T16:58:27.148 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:26 smithi162 ceph-mon[34633]: osdmap e290: 8 total, 8 up, 8 in 2024-08-17T16:58:27.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:26 smithi063 ceph-mon[31063]: osdmap e290: 8 total, 8 up, 8 in 2024-08-17T16:58:27.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:26 smithi063 ceph-mon[42680]: osdmap e290: 8 total, 8 up, 8 in 2024-08-17T16:58:27.880 INFO:tasks.rados.rados.0.smithi063.stdout:881: done (0 left) 2024-08-17T16:58:27.880 INFO:tasks.rados.rados.0.smithi063.stdout:882: delete oid 27 current snap is 112 2024-08-17T16:58:27.888 INFO:tasks.rados.rados.0.smithi063.stdout:882: done (0 left) 2024-08-17T16:58:27.888 INFO:tasks.rados.rados.0.smithi063.stdout:883: read oid 34 snap -1 2024-08-17T16:58:27.888 INFO:tasks.rados.rados.0.smithi063.stdout:883: expect (ObjNum 311 snap 92 seq_num 311) 2024-08-17T16:58:27.888 INFO:tasks.rados.rados.0.smithi063.stdout:884: delete oid 17 current snap is 112 2024-08-17T16:58:27.890 INFO:tasks.rados.rados.0.smithi063.stdout:884: done (1 left) 2024-08-17T16:58:27.890 INFO:tasks.rados.rados.0.smithi063.stdout:885: snap_remove snap 105 2024-08-17T16:58:28.164 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:27 smithi162 ceph-mon[34633]: osdmap e291: 8 total, 8 up, 8 in 2024-08-17T16:58:28.164 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:27 smithi162 ceph-mon[34633]: pgmap v120: 89 pgs: 89 active+clean; 252 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-08-17T16:58:28.195 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:27 smithi063 ceph-mon[42680]: osdmap e291: 8 total, 8 up, 8 in 2024-08-17T16:58:28.196 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:27 smithi063 ceph-mon[42680]: pgmap v120: 89 pgs: 89 active+clean; 252 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-08-17T16:58:28.196 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:27 smithi063 ceph-mon[31063]: osdmap e291: 8 total, 8 up, 8 in 2024-08-17T16:58:28.196 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:27 smithi063 ceph-mon[31063]: pgmap v120: 89 pgs: 89 active+clean; 252 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-08-17T16:58:28.910 INFO:tasks.rados.rados.0.smithi063.stdout:885: done (1 left) 2024-08-17T16:58:28.910 INFO:tasks.rados.rados.0.smithi063.stdout:886: read oid 47 snap 111 2024-08-17T16:58:28.910 INFO:tasks.rados.rados.0.smithi063.stdout:886: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:58:28.910 INFO:tasks.rados.rados.0.smithi063.stdout:887: snap_create 2024-08-17T16:58:28.910 INFO:tasks.rados.rados.0.smithi063.stdout:883: expect (ObjNum 311 snap 92 seq_num 311) 2024-08-17T16:58:29.049 INFO:tasks.rados.rados.0.smithi063.stdout:886: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:58:29.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:28 smithi063 ceph-mon[42680]: osdmap e292: 8 total, 8 up, 8 in 2024-08-17T16:58:29.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:28 smithi063 ceph-mon[31063]: osdmap e292: 8 total, 8 up, 8 in 2024-08-17T16:58:29.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:28 smithi162 ceph-mon[34633]: osdmap e292: 8 total, 8 up, 8 in 2024-08-17T16:58:29.875 INFO:tasks.rados.rados.0.smithi063.stdout:883: done (2 left) 2024-08-17T16:58:29.875 INFO:tasks.rados.rados.0.smithi063.stdout:886: done (1 left) 2024-08-17T16:58:29.875 INFO:tasks.rados.rados.0.smithi063.stdout:887: done (0 left) 2024-08-17T16:58:29.875 INFO:tasks.rados.rados.0.smithi063.stdout:888: read oid 5 snap -1 2024-08-17T16:58:29.875 INFO:tasks.rados.rados.0.smithi063.stdout:888: expect (ObjNum 354 snap 106 seq_num 354) 2024-08-17T16:58:29.875 INFO:tasks.rados.rados.0.smithi063.stdout:889: read oid 36 snap -1 2024-08-17T16:58:29.875 INFO:tasks.rados.rados.0.smithi063.stdout:889: expect (ObjNum 350 snap 105 seq_num 350) 2024-08-17T16:58:29.875 INFO:tasks.rados.rados.0.smithi063.stdout:890: snap_remove snap 104 2024-08-17T16:58:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:29 smithi063 ceph-mon[42680]: pgmap v123: 89 pgs: 89 active+clean; 252 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2024-08-17T16:58:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:29 smithi063 ceph-mon[42680]: osdmap e293: 8 total, 8 up, 8 in 2024-08-17T16:58:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:29 smithi063 ceph-mon[42680]: osdmap e294: 8 total, 8 up, 8 in 2024-08-17T16:58:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:29 smithi063 ceph-mon[31063]: pgmap v123: 89 pgs: 89 active+clean; 252 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2024-08-17T16:58:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:29 smithi063 ceph-mon[31063]: osdmap e293: 8 total, 8 up, 8 in 2024-08-17T16:58:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:29 smithi063 ceph-mon[31063]: osdmap e294: 8 total, 8 up, 8 in 2024-08-17T16:58:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:29 smithi162 ceph-mon[34633]: pgmap v123: 89 pgs: 89 active+clean; 252 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.2 MiB/s wr, 19 op/s 2024-08-17T16:58:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:29 smithi162 ceph-mon[34633]: osdmap e293: 8 total, 8 up, 8 in 2024-08-17T16:58:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:29 smithi162 ceph-mon[34633]: osdmap e294: 8 total, 8 up, 8 in 2024-08-17T16:58:30.876 INFO:tasks.rados.rados.0.smithi063.stdout:890: done (2 left) 2024-08-17T16:58:30.876 INFO:tasks.rados.rados.0.smithi063.stdout:891: snap_create 2024-08-17T16:58:30.876 INFO:tasks.rados.rados.0.smithi063.stdout:888: expect (ObjNum 354 snap 106 seq_num 354) 2024-08-17T16:58:31.166 INFO:tasks.rados.rados.0.smithi063.stdout:889: expect (ObjNum 350 snap 105 seq_num 350) 2024-08-17T16:58:31.611 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:31 smithi063 systemd[1]: Stopping Ceph alertmanager.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:58:31.876 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:31 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[79286]: ts=2024-08-17T16:58:31.610Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-17T16:58:31.877 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:31 smithi063 podman[104362]: 2024-08-17 16:58:31.65108491 +0000 UTC m=+0.200508053 container died 87286b24f231cfa279dbf4c7540a822b009995a8c2c1551794a462fc2d597cde (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:58:31.880 INFO:tasks.rados.rados.0.smithi063.stdout:888: done (2 left) 2024-08-17T16:58:31.880 INFO:tasks.rados.rados.0.smithi063.stdout:889: done (1 left) 2024-08-17T16:58:31.880 INFO:tasks.rados.rados.0.smithi063.stdout:891: done (0 left) 2024-08-17T16:58:31.880 INFO:tasks.rados.rados.0.smithi063.stdout:892: write oid 35 current snap is 114 2024-08-17T16:58:31.880 INFO:tasks.rados.rados.0.smithi063.stdout:892: seq_num 367 ranges {568169=401327,1376068=669790} 2024-08-17T16:58:31.885 INFO:tasks.rados.rados.0.smithi063.stdout:892: writing smithi06387500-35 from 568169 to 969496 tid 1 2024-08-17T16:58:31.892 INFO:tasks.rados.rados.0.smithi063.stdout:892: writing smithi06387500-35 from 1376068 to 2045858 tid 2 2024-08-17T16:58:31.892 INFO:tasks.rados.rados.0.smithi063.stdout:893: read oid 45 snap -1 2024-08-17T16:58:31.892 INFO:tasks.rados.rados.0.smithi063.stdout:893: expect (ObjNum 363 snap 109 seq_num 363) 2024-08-17T16:58:31.892 INFO:tasks.rados.rados.0.smithi063.stdout:894: snap_remove snap 108 2024-08-17T16:58:32.134 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:31 smithi063 ceph-mon[31063]: osdmap e295: 8 total, 8 up, 8 in 2024-08-17T16:58:32.134 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:31 smithi063 ceph-mon[31063]: pgmap v126: 89 pgs: 89 active+clean; 251 MiB data, 775 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T16:58:32.134 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:31 smithi063 ceph-mon[42680]: osdmap e295: 8 total, 8 up, 8 in 2024-08-17T16:58:32.134 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:31 smithi063 ceph-mon[42680]: pgmap v126: 89 pgs: 89 active+clean; 251 MiB data, 775 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T16:58:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:31 smithi162 ceph-mon[34633]: osdmap e295: 8 total, 8 up, 8 in 2024-08-17T16:58:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:31 smithi162 ceph-mon[34633]: pgmap v126: 89 pgs: 89 active+clean; 251 MiB data, 775 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T16:58:32.402 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:32 smithi063 podman[104362]: 2024-08-17 16:58:32.13357214 +0000 UTC m=+0.682995324 container cleanup 87286b24f231cfa279dbf4c7540a822b009995a8c2c1551794a462fc2d597cde (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:58:32.402 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:32 smithi063 bash[104362]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a 2024-08-17T16:58:32.402 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:32 smithi063 podman[104382]: 2024-08-17 16:58:32.326769109 +0000 UTC m=+0.668853896 container remove 87286b24f231cfa279dbf4c7540a822b009995a8c2c1551794a462fc2d597cde (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:58:32.403 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:32 smithi063 podman[104382]: 2024-08-17 16:58:32.402401436 +0000 UTC m=+0.744486223 volume remove cb54b9f4476e509ca3bc0e3d9f7f94b73162f13e3b648065b667ae7d550584f4 2024-08-17T16:58:32.882 INFO:tasks.rados.rados.0.smithi063.stdout:894: done (2 left) 2024-08-17T16:58:32.882 INFO:tasks.rados.rados.0.smithi063.stdout:895: write oid 18 current snap is 114 2024-08-17T16:58:32.882 INFO:tasks.rados.rados.0.smithi063.stdout:892: finishing write tid 1 to smithi06387500-35 2024-08-17T16:58:32.882 INFO:tasks.rados.rados.0.smithi063.stdout:895: seq_num 368 ranges {584976=476576,1834550=350426} 2024-08-17T16:58:32.888 INFO:tasks.rados.rados.0.smithi063.stdout:895: writing smithi06387500-18 from 584976 to 1061552 tid 1 2024-08-17T16:58:32.895 INFO:tasks.rados.rados.0.smithi063.stdout:895: writing smithi06387500-18 from 1834550 to 2184976 tid 2 2024-08-17T16:58:32.895 INFO:tasks.rados.rados.0.smithi063.stdout:896: snap_remove snap 112 2024-08-17T16:58:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:32 smithi063 ceph-mon[42680]: osdmap e296: 8 total, 8 up, 8 in 2024-08-17T16:58:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:32 smithi063 ceph-mon[31063]: osdmap e296: 8 total, 8 up, 8 in 2024-08-17T16:58:33.030 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:32 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-17T16:58:33.030 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:32 smithi063 systemd[1]: Stopped Ceph alertmanager.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:58:33.030 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:32 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@alertmanager.a.service: Consumed 1.414s CPU time. 2024-08-17T16:58:33.030 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:32 smithi063 systemd[1]: Starting Ceph alertmanager.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:58:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:32 smithi162 ceph-mon[34633]: osdmap e296: 8 total, 8 up, 8 in 2024-08-17T16:58:33.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:58:33] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.51.0" 2024-08-17T16:58:33.529 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:33 smithi063 podman[104514]: 2024-08-17 16:58:33.050718693 +0000 UTC m=+0.019119512 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-08-17T16:58:33.530 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:33 smithi063 podman[104514]: 2024-08-17 16:58:33.238519519 +0000 UTC m=+0.206920368 volume create 20521155491dafc5c3c76767fa0919e6b43478dc8975b5ee304504cca20dc0d3 2024-08-17T16:58:33.530 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:33 smithi063 podman[104514]: 2024-08-17 16:58:33.330544959 +0000 UTC m=+0.298945808 container create 59f4018b91caa6ebb54ff8d3e8c2b12192aad700bacee2b981f98f63ad9c373c (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:58:33.896 INFO:tasks.rados.rados.0.smithi063.stdout:896: done (3 left) 2024-08-17T16:58:33.896 INFO:tasks.rados.rados.0.smithi063.stdout:897: rollback oid 15 current snap is 114 2024-08-17T16:58:33.896 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 15 to 98 2024-08-17T16:58:33.896 INFO:tasks.rados.rados.0.smithi063.stdout:898: read oid 44 snap -1 2024-08-17T16:58:33.897 INFO:tasks.rados.rados.0.smithi063.stdout:898: expect (ObjNum 325 snap 99 seq_num 325) 2024-08-17T16:58:33.897 INFO:tasks.rados.rados.0.smithi063.stdout:899: snap_create 2024-08-17T16:58:33.897 INFO:tasks.rados.rados.0.smithi063.stdout:892: finishing write tid 2 to smithi06387500-35 2024-08-17T16:58:33.897 INFO:tasks.rados.rados.0.smithi063.stdout:892: finishing write tid 3 to smithi06387500-35 2024-08-17T16:58:33.897 INFO:tasks.rados.rados.0.smithi063.stdout:892: finishing write tid 4 to smithi06387500-35 2024-08-17T16:58:33.897 INFO:tasks.rados.rados.0.smithi063.stdout:892: oid 35 updating version 0 to 927 2024-08-17T16:58:33.897 INFO:tasks.rados.rados.0.smithi063.stdout:892: oid 35 version 927 is already newer than 925 2024-08-17T16:58:33.897 INFO:tasks.rados.rados.0.smithi063.stdout:892: oid 35 version 927 is already newer than 926 2024-08-17T16:58:33.897 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 927 (ObjNum 367 snap 114 seq_num 367) dirty exists 2024-08-17T16:58:33.897 INFO:tasks.rados.rados.0.smithi063.stdout:892: left oid 35 (ObjNum 367 snap 114 seq_num 367) 2024-08-17T16:58:33.897 INFO:tasks.rados.rados.0.smithi063.stdout:893: expect (ObjNum 363 snap 109 seq_num 363) 2024-08-17T16:58:34.021 INFO:tasks.rados.rados.0.smithi063.stdout:895: finishing write tid 1 to smithi06387500-18 2024-08-17T16:58:34.021 INFO:tasks.rados.rados.0.smithi063.stdout:895: finishing write tid 2 to smithi06387500-18 2024-08-17T16:58:34.021 INFO:tasks.rados.rados.0.smithi063.stdout:895: finishing write tid 3 to smithi06387500-18 2024-08-17T16:58:34.021 INFO:tasks.rados.rados.0.smithi063.stdout:895: finishing write tid 4 to smithi06387500-18 2024-08-17T16:58:34.021 INFO:tasks.rados.rados.0.smithi063.stdout:895: oid 18 updating version 0 to 335 2024-08-17T16:58:34.021 INFO:tasks.rados.rados.0.smithi063.stdout:895: oid 18 updating version 335 to 337 2024-08-17T16:58:34.021 INFO:tasks.rados.rados.0.smithi063.stdout:895: oid 18 version 337 is already newer than 336 2024-08-17T16:58:34.021 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 337 (ObjNum 368 snap 114 seq_num 368) dirty exists 2024-08-17T16:58:34.022 INFO:tasks.rados.rados.0.smithi063.stdout:895: left oid 18 (ObjNum 368 snap 114 seq_num 368) 2024-08-17T16:58:34.022 INFO:tasks.rados.rados.0.smithi063.stdout:897: finishing rollback tid 1 to smithi06387500-15 2024-08-17T16:58:34.022 INFO:tasks.rados.rados.0.smithi063.stdout:897: finishing rollback tid 2 to smithi06387500-15 2024-08-17T16:58:34.022 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 348 (ObjNum 289 snap 88 seq_num 289) dirty exists 2024-08-17T16:58:34.022 INFO:tasks.rados.rados.0.smithi063.stdout:898: expect (ObjNum 325 snap 99 seq_num 325) 2024-08-17T16:58:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-mon[42680]: osdmap e297: 8 total, 8 up, 8 in 2024-08-17T16:58:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-mon[42680]: pgmap v129: 89 pgs: 89 active+clean; 251 MiB data, 775 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T16:58:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:34.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-mon[31063]: osdmap e297: 8 total, 8 up, 8 in 2024-08-17T16:58:34.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-mon[31063]: pgmap v129: 89 pgs: 89 active+clean; 251 MiB data, 775 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T16:58:34.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:34.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:34.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:34.030 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:33 smithi063 podman[104514]: 2024-08-17 16:58:33.674876965 +0000 UTC m=+0.643277772 container init 59f4018b91caa6ebb54ff8d3e8c2b12192aad700bacee2b981f98f63ad9c373c (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:58:34.030 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:33 smithi063 podman[104514]: 2024-08-17 16:58:33.680720519 +0000 UTC m=+0.649121334 container start 59f4018b91caa6ebb54ff8d3e8c2b12192aad700bacee2b981f98f63ad9c373c (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T16:58:34.030 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[104529]: ts=2024-08-17T16:58:33.691Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-08-17T16:58:34.030 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[104529]: ts=2024-08-17T16:58:33.691Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-08-17T16:58:34.031 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[104529]: ts=2024-08-17T16:58:33.692Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.63 port=9094 2024-08-17T16:58:34.031 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[104529]: ts=2024-08-17T16:58:33.693Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-08-17T16:58:34.031 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[104529]: ts=2024-08-17T16:58:33.716Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T16:58:34.031 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[104529]: ts=2024-08-17T16:58:33.716Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-08-17T16:58:34.031 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[104529]: ts=2024-08-17T16:58:33.718Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-08-17T16:58:34.031 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[104529]: ts=2024-08-17T16:58:33.718Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-08-17T16:58:34.031 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:33 smithi063 bash[104514]: 59f4018b91caa6ebb54ff8d3e8c2b12192aad700bacee2b981f98f63ad9c373c 2024-08-17T16:58:34.031 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:33 smithi063 systemd[1]: Started Ceph alertmanager.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:58:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:33 smithi162 ceph-mon[34633]: osdmap e297: 8 total, 8 up, 8 in 2024-08-17T16:58:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:33 smithi162 ceph-mon[34633]: pgmap v129: 89 pgs: 89 active+clean; 251 MiB data, 775 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T16:58:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:33 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:33 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:33 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:34.877 INFO:tasks.rados.rados.0.smithi063.stdout:892: done (5 left) 2024-08-17T16:58:34.877 INFO:tasks.rados.rados.0.smithi063.stdout:893: done (4 left) 2024-08-17T16:58:34.877 INFO:tasks.rados.rados.0.smithi063.stdout:895: done (3 left) 2024-08-17T16:58:34.877 INFO:tasks.rados.rados.0.smithi063.stdout:897: done (2 left) 2024-08-17T16:58:34.877 INFO:tasks.rados.rados.0.smithi063.stdout:898: done (1 left) 2024-08-17T16:58:34.877 INFO:tasks.rados.rados.0.smithi063.stdout:899: done (0 left) 2024-08-17T16:58:34.878 INFO:tasks.rados.rados.0.smithi063.stdout:900: write oid 12 current snap is 115 2024-08-17T16:58:34.878 INFO:tasks.rados.rados.0.smithi063.stdout:900: seq_num 369 ranges {601783=551825,1776679=425104} 2024-08-17T16:58:34.883 INFO:tasks.rados.rados.0.smithi063.stdout:900: writing smithi06387500-12 from 601783 to 1153608 tid 1 2024-08-17T16:58:34.889 INFO:tasks.rados.rados.0.smithi063.stdout:900: writing smithi06387500-12 from 1776679 to 2201783 tid 2 2024-08-17T16:58:34.889 INFO:tasks.rados.rados.0.smithi063.stdout:900: finishing write tid 1 to smithi06387500-12 2024-08-17T16:58:34.889 INFO:tasks.rados.rados.0.smithi063.stdout:901: delete oid 36 current snap is 115 2024-08-17T16:58:34.894 INFO:tasks.rados.rados.0.smithi063.stdout:901: done (1 left) 2024-08-17T16:58:34.894 INFO:tasks.rados.rados.0.smithi063.stdout:902: read oid 15 snap -1 2024-08-17T16:58:34.894 INFO:tasks.rados.rados.0.smithi063.stdout:902: expect (ObjNum 289 snap 88 seq_num 289) 2024-08-17T16:58:34.894 INFO:tasks.rados.rados.0.smithi063.stdout:903: delete oid 17 current snap is 115 2024-08-17T16:58:34.894 INFO:tasks.rados.rados.0.smithi063.stdout:900: finishing write tid 2 to smithi06387500-12 2024-08-17T16:58:34.895 INFO:tasks.rados.rados.0.smithi063.stdout:900: finishing write tid 3 to smithi06387500-12 2024-08-17T16:58:34.895 INFO:tasks.rados.rados.0.smithi063.stdout:900: finishing write tid 4 to smithi06387500-12 2024-08-17T16:58:34.895 INFO:tasks.rados.rados.0.smithi063.stdout:900: oid 12 updating version 0 to 187 2024-08-17T16:58:34.895 INFO:tasks.rados.rados.0.smithi063.stdout:900: oid 12 updating version 187 to 188 2024-08-17T16:58:34.895 INFO:tasks.rados.rados.0.smithi063.stdout:900: oid 12 version 188 is already newer than 186 2024-08-17T16:58:34.895 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 188 (ObjNum 369 snap 115 seq_num 369) dirty exists 2024-08-17T16:58:34.895 INFO:tasks.rados.rados.0.smithi063.stdout:900: left oid 12 (ObjNum 369 snap 115 seq_num 369) 2024-08-17T16:58:34.896 INFO:tasks.rados.rados.0.smithi063.stdout:900: done (2 left) 2024-08-17T16:58:34.896 INFO:tasks.rados.rados.0.smithi063.stdout:903: done (1 left) 2024-08-17T16:58:34.896 INFO:tasks.rados.rados.0.smithi063.stdout:904: snap_create 2024-08-17T16:58:34.899 INFO:tasks.rados.rados.0.smithi063.stdout:902: expect (ObjNum 289 snap 88 seq_num 289) 2024-08-17T16:58:35.206 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:34 smithi063 ceph-mon[42680]: osdmap e298: 8 total, 8 up, 8 in 2024-08-17T16:58:35.206 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:34 smithi063 ceph-mon[42680]: osdmap e299: 8 total, 8 up, 8 in 2024-08-17T16:58:35.206 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:34 smithi063 ceph-mon[31063]: osdmap e298: 8 total, 8 up, 8 in 2024-08-17T16:58:35.206 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:34 smithi063 ceph-mon[31063]: osdmap e299: 8 total, 8 up, 8 in 2024-08-17T16:58:35.229 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:34 smithi162 ceph-mon[34633]: osdmap e298: 8 total, 8 up, 8 in 2024-08-17T16:58:35.229 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:34 smithi162 ceph-mon[34633]: osdmap e299: 8 total, 8 up, 8 in 2024-08-17T16:58:35.899 INFO:tasks.rados.rados.0.smithi063.stdout:902: done (1 left) 2024-08-17T16:58:35.900 INFO:tasks.rados.rados.0.smithi063.stdout:904: done (0 left) 2024-08-17T16:58:35.900 INFO:tasks.rados.rados.0.smithi063.stdout:905: delete oid 23 current snap is 116 2024-08-17T16:58:35.907 INFO:tasks.rados.rados.0.smithi063.stdout:905: done (0 left) 2024-08-17T16:58:35.908 INFO:tasks.rados.rados.0.smithi063.stdout:906: delete oid 30 current snap is 116 2024-08-17T16:58:35.910 INFO:tasks.rados.rados.0.smithi063.stdout:906: done (0 left) 2024-08-17T16:58:35.910 INFO:tasks.rados.rados.0.smithi063.stdout:907: write oid 39 current snap is 116 2024-08-17T16:58:35.910 INFO:tasks.rados.rados.0.smithi063.stdout:907: seq_num 374 ranges {685818=644423,2069089=216729} 2024-08-17T16:58:35.917 INFO:tasks.rados.rados.0.smithi063.stdout:907: writing smithi06387500-39 from 685818 to 1330241 tid 1 2024-08-17T16:58:35.922 INFO:tasks.rados.rados.0.smithi063.stdout:907: writing smithi06387500-39 from 2069089 to 2285818 tid 2 2024-08-17T16:58:35.923 INFO:tasks.rados.rados.0.smithi063.stdout:907: finishing write tid 1 to smithi06387500-39 2024-08-17T16:58:35.923 INFO:tasks.rados.rados.0.smithi063.stdout:908: snap_create 2024-08-17T16:58:35.924 INFO:tasks.rados.rados.0.smithi063.stdout:907: finishing write tid 2 to smithi06387500-39 2024-08-17T16:58:35.924 INFO:tasks.rados.rados.0.smithi063.stdout:907: finishing write tid 3 to smithi06387500-39 2024-08-17T16:58:35.924 INFO:tasks.rados.rados.0.smithi063.stdout:907: finishing write tid 4 to smithi06387500-39 2024-08-17T16:58:35.925 INFO:tasks.rados.rados.0.smithi063.stdout:907: oid 39 updating version 0 to 223 2024-08-17T16:58:35.925 INFO:tasks.rados.rados.0.smithi063.stdout:907: oid 39 version 223 is already newer than 222 2024-08-17T16:58:35.925 INFO:tasks.rados.rados.0.smithi063.stdout:907: oid 39 version 223 is already newer than 221 2024-08-17T16:58:35.925 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 223 (ObjNum 374 snap 116 seq_num 374) dirty exists 2024-08-17T16:58:35.925 INFO:tasks.rados.rados.0.smithi063.stdout:907: left oid 39 (ObjNum 374 snap 116 seq_num 374) 2024-08-17T16:58:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:35 smithi063 ceph-mon[42680]: pgmap v132: 89 pgs: 89 active+clean; 251 MiB data, 775 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:58:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:35 smithi063 ceph-mon[31063]: pgmap v132: 89 pgs: 89 active+clean; 251 MiB data, 775 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:58:36.029 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[104529]: ts=2024-08-17T16:58:35.694Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000934065s 2024-08-17T16:58:36.183 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:35 smithi162 ceph-mon[34633]: pgmap v132: 89 pgs: 89 active+clean; 251 MiB data, 775 MiB used, 714 GiB / 715 GiB avail 2024-08-17T16:58:36.904 INFO:tasks.rados.rados.0.smithi063.stdout:907: done (1 left) 2024-08-17T16:58:36.904 INFO:tasks.rados.rados.0.smithi063.stdout:908: done (0 left) 2024-08-17T16:58:36.904 INFO:tasks.rados.rados.0.smithi063.stdout:909: delete oid 43 current snap is 117 2024-08-17T16:58:36.907 INFO:tasks.rados.rados.0.smithi063.stdout:909: done (0 left) 2024-08-17T16:58:36.907 INFO:tasks.rados.rados.0.smithi063.stdout:910: delete oid 5 current snap is 117 2024-08-17T16:58:36.911 INFO:tasks.rados.rados.0.smithi063.stdout:910: done (0 left) 2024-08-17T16:58:36.911 INFO:tasks.rados.rados.0.smithi063.stdout:911: read oid 3 snap -1 2024-08-17T16:58:36.911 INFO:tasks.rados.rados.0.smithi063.stdout:911: expect deleted 2024-08-17T16:58:36.912 INFO:tasks.rados.rados.0.smithi063.stdout:912: rollback oid 37 current snap is 117 2024-08-17T16:58:36.912 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 109 2024-08-17T16:58:36.912 INFO:tasks.rados.rados.0.smithi063.stdout:913: snap_remove snap 101 2024-08-17T16:58:37.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:36 smithi063 ceph-mon[42680]: osdmap e300: 8 total, 8 up, 8 in 2024-08-17T16:58:37.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:36 smithi063 ceph-mon[31063]: osdmap e300: 8 total, 8 up, 8 in 2024-08-17T16:58:37.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:36 smithi162 ceph-mon[34633]: osdmap e300: 8 total, 8 up, 8 in 2024-08-17T16:58:37.917 INFO:tasks.rados.rados.0.smithi063.stdout:913: done (2 left) 2024-08-17T16:58:37.917 INFO:tasks.rados.rados.0.smithi063.stdout:914: rollback oid 40 current snap is 117 2024-08-17T16:58:37.917 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 40 to 114 2024-08-17T16:58:37.917 INFO:tasks.rados.rados.0.smithi063.stdout:912: finishing rollback tid 0 to smithi06387500-37 2024-08-17T16:58:37.917 INFO:tasks.rados.rados.0.smithi063.stdout:912: finishing rollback tid 1 to smithi06387500-37 2024-08-17T16:58:37.917 INFO:tasks.rados.rados.0.smithi063.stdout:912: finishing rollback tid 2 to smithi06387500-37 2024-08-17T16:58:37.917 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 937 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-08-17T16:58:37.917 INFO:tasks.rados.rados.0.smithi063.stdout:911: done (2 left) 2024-08-17T16:58:37.917 INFO:tasks.rados.rados.0.smithi063.stdout:912: done (1 left) 2024-08-17T16:58:37.917 INFO:tasks.rados.rados.0.smithi063.stdout:915: write oid 30 current snap is 117 2024-08-17T16:58:37.918 INFO:tasks.rados.rados.0.smithi063.stdout:915: seq_num 377 ranges {736239=470170,1728182=571782} 2024-08-17T16:58:37.923 INFO:tasks.rados.rados.0.smithi063.stdout:915: writing smithi06387500-30 from 736239 to 1206409 tid 1 2024-08-17T16:58:37.930 INFO:tasks.rados.rados.0.smithi063.stdout:915: writing smithi06387500-30 from 1728182 to 2299964 tid 2 2024-08-17T16:58:37.930 INFO:tasks.rados.rados.0.smithi063.stdout:916: delete oid 26 current snap is 117 2024-08-17T16:58:37.930 INFO:tasks.rados.rados.0.smithi063.stdout:914: finishing rollback tid 0 to smithi06387500-40 2024-08-17T16:58:37.930 INFO:tasks.rados.rados.0.smithi063.stdout:914: finishing rollback tid 1 to smithi06387500-40 2024-08-17T16:58:37.930 INFO:tasks.rados.rados.0.smithi063.stdout:914: finishing rollback tid 2 to smithi06387500-40 2024-08-17T16:58:37.930 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 349 (ObjNum 355 snap 107 seq_num 355) dirty exists 2024-08-17T16:58:37.932 INFO:tasks.rados.rados.0.smithi063.stdout:915: finishing write tid 1 to smithi06387500-30 2024-08-17T16:58:37.934 INFO:tasks.rados.rados.0.smithi063.stdout:915: finishing write tid 2 to smithi06387500-30 2024-08-17T16:58:37.934 INFO:tasks.rados.rados.0.smithi063.stdout:915: finishing write tid 3 to smithi06387500-30 2024-08-17T16:58:37.934 INFO:tasks.rados.rados.0.smithi063.stdout:915: finishing write tid 4 to smithi06387500-30 2024-08-17T16:58:37.934 INFO:tasks.rados.rados.0.smithi063.stdout:915: oid 30 updating version 0 to 528 2024-08-17T16:58:37.934 INFO:tasks.rados.rados.0.smithi063.stdout:915: oid 30 version 528 is already newer than 527 2024-08-17T16:58:37.934 INFO:tasks.rados.rados.0.smithi063.stdout:915: oid 30 version 528 is already newer than 526 2024-08-17T16:58:37.934 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 528 (ObjNum 377 snap 117 seq_num 377) dirty exists 2024-08-17T16:58:37.934 INFO:tasks.rados.rados.0.smithi063.stdout:915: left oid 30 (ObjNum 377 snap 117 seq_num 377) 2024-08-17T16:58:37.934 INFO:tasks.rados.rados.0.smithi063.stdout:914: done (2 left) 2024-08-17T16:58:37.934 INFO:tasks.rados.rados.0.smithi063.stdout:915: done (1 left) 2024-08-17T16:58:37.934 INFO:tasks.rados.rados.0.smithi063.stdout:916: done (0 left) 2024-08-17T16:58:37.935 INFO:tasks.rados.rados.0.smithi063.stdout:917: rollback oid 9 current snap is 117 2024-08-17T16:58:37.935 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 9 to 113 2024-08-17T16:58:37.935 INFO:tasks.rados.rados.0.smithi063.stdout:918: read oid 1 snap -1 2024-08-17T16:58:37.935 INFO:tasks.rados.rados.0.smithi063.stdout:918: expect (ObjNum 339 snap 102 seq_num 339) 2024-08-17T16:58:37.935 INFO:tasks.rados.rados.0.smithi063.stdout:919: read oid 7 snap -1 2024-08-17T16:58:37.935 INFO:tasks.rados.rados.0.smithi063.stdout:919: expect (ObjNum 277 snap 86 seq_num 277) 2024-08-17T16:58:37.935 INFO:tasks.rados.rados.0.smithi063.stdout:920: rollback oid 44 current snap is 117 2024-08-17T16:58:37.935 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 44 to 114 2024-08-17T16:58:37.935 INFO:tasks.rados.rados.0.smithi063.stdout:921: snap_remove snap 63 2024-08-17T16:58:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:37 smithi162 ceph-mon[34633]: osdmap e301: 8 total, 8 up, 8 in 2024-08-17T16:58:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:37 smithi162 ceph-mon[34633]: pgmap v135: 89 pgs: 89 active+clean; 255 MiB data, 777 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 703 KiB/s wr, 10 op/s 2024-08-17T16:58:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:37 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:38.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:37 smithi063 ceph-mon[42680]: osdmap e301: 8 total, 8 up, 8 in 2024-08-17T16:58:38.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:37 smithi063 ceph-mon[42680]: pgmap v135: 89 pgs: 89 active+clean; 255 MiB data, 777 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 703 KiB/s wr, 10 op/s 2024-08-17T16:58:38.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:38.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:37 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:38.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:37 smithi063 ceph-mon[31063]: osdmap e301: 8 total, 8 up, 8 in 2024-08-17T16:58:38.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:37 smithi063 ceph-mon[31063]: pgmap v135: 89 pgs: 89 active+clean; 255 MiB data, 777 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 703 KiB/s wr, 10 op/s 2024-08-17T16:58:38.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:38.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:37 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:38.935 INFO:tasks.rados.rados.0.smithi063.stdout:921: done (4 left) 2024-08-17T16:58:38.935 INFO:tasks.rados.rados.0.smithi063.stdout:922: snap_remove snap 110 2024-08-17T16:58:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:38 smithi063 ceph-mon[42680]: osdmap e302: 8 total, 8 up, 8 in 2024-08-17T16:58:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:38 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:39.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:38 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:39.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:38 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:39.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:38 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:38 smithi063 ceph-mon[31063]: osdmap e302: 8 total, 8 up, 8 in 2024-08-17T16:58:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:38 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:38 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:38 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:38 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:39.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:38 smithi162 ceph-mon[34633]: osdmap e302: 8 total, 8 up, 8 in 2024-08-17T16:58:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:38 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:38 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:38 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:38 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:39.884 INFO:tasks.rados.rados.0.smithi063.stdout:922: done (4 left) 2024-08-17T16:58:39.884 INFO:tasks.rados.rados.0.smithi063.stdout:923: read oid 47 snap -1 2024-08-17T16:58:39.884 INFO:tasks.rados.rados.0.smithi063.stdout:923: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:58:39.884 INFO:tasks.rados.rados.0.smithi063.stdout:924: write oid 31 current snap is 117 2024-08-17T16:58:39.884 INFO:tasks.rados.rados.0.smithi063.stdout:924: seq_num 379 ranges {769853=620668,1845146=524707} 2024-08-17T16:58:39.891 INFO:tasks.rados.rados.0.smithi063.stdout:924: writing smithi06387500-31 from 769853 to 1390521 tid 1 2024-08-17T16:58:39.897 INFO:tasks.rados.rados.0.smithi063.stdout:924: writing smithi06387500-31 from 1845146 to 2369853 tid 2 2024-08-17T16:58:39.897 INFO:tasks.rados.rados.0.smithi063.stdout:925: write oid 14 current snap is 117 2024-08-17T16:58:39.897 INFO:tasks.rados.rados.0.smithi063.stdout:920: finishing rollback tid 0 to smithi06387500-44 2024-08-17T16:58:39.897 INFO:tasks.rados.rados.0.smithi063.stdout:920: finishing rollback tid 1 to smithi06387500-44 2024-08-17T16:58:39.897 INFO:tasks.rados.rados.0.smithi063.stdout:925: seq_num 380 ranges {786660=695917,1903628=435462} 2024-08-17T16:58:39.904 INFO:tasks.rados.rados.0.smithi063.stdout:925: writing smithi06387500-14 from 786660 to 1482577 tid 1 2024-08-17T16:58:39.910 INFO:tasks.rados.rados.0.smithi063.stdout:925: writing smithi06387500-14 from 1903628 to 2339090 tid 2 2024-08-17T16:58:39.910 INFO:tasks.rados.rados.0.smithi063.stdout:926: read oid 42 snap -1 2024-08-17T16:58:39.910 INFO:tasks.rados.rados.0.smithi063.stdout:926: expect (ObjNum 347 snap 103 seq_num 347) 2024-08-17T16:58:39.910 INFO:tasks.rados.rados.0.smithi063.stdout:927: snap_remove snap 98 2024-08-17T16:58:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:39 smithi063 ceph-mon[31063]: pgmap v137: 89 pgs: 89 active+clean; 255 MiB data, 777 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 697 KiB/s wr, 10 op/s 2024-08-17T16:58:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:39 smithi063 ceph-mon[31063]: osdmap e303: 8 total, 8 up, 8 in 2024-08-17T16:58:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:39 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:39 smithi063 ceph-mon[31063]: osdmap e304: 8 total, 8 up, 8 in 2024-08-17T16:58:40.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:39 smithi063 ceph-mon[42680]: pgmap v137: 89 pgs: 89 active+clean; 255 MiB data, 777 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 697 KiB/s wr, 10 op/s 2024-08-17T16:58:40.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:39 smithi063 ceph-mon[42680]: osdmap e303: 8 total, 8 up, 8 in 2024-08-17T16:58:40.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:39 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:40.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:39 smithi063 ceph-mon[42680]: osdmap e304: 8 total, 8 up, 8 in 2024-08-17T16:58:40.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:39 smithi162 ceph-mon[34633]: pgmap v137: 89 pgs: 89 active+clean; 255 MiB data, 777 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 697 KiB/s wr, 10 op/s 2024-08-17T16:58:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:39 smithi162 ceph-mon[34633]: osdmap e303: 8 total, 8 up, 8 in 2024-08-17T16:58:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:39 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:39 smithi162 ceph-mon[34633]: osdmap e304: 8 total, 8 up, 8 in 2024-08-17T16:58:40.915 INFO:tasks.rados.rados.0.smithi063.stdout:920: finishing rollback tid 2 to smithi06387500-44 2024-08-17T16:58:40.915 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 180 (ObjNum 325 snap 99 seq_num 325) dirty exists 2024-08-17T16:58:40.915 INFO:tasks.rados.rados.0.smithi063.stdout:917: finishing rollback tid 0 to smithi06387500-9 2024-08-17T16:58:40.915 INFO:tasks.rados.rados.0.smithi063.stdout:917: finishing rollback tid 1 to smithi06387500-9 2024-08-17T16:58:40.915 INFO:tasks.rados.rados.0.smithi063.stdout:917: finishing rollback tid 2 to smithi06387500-9 2024-08-17T16:58:40.915 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 532 (ObjNum 269 snap 84 seq_num 269) dirty exists 2024-08-17T16:58:40.915 INFO:tasks.rados.rados.0.smithi063.stdout:918: expect (ObjNum 339 snap 102 seq_num 339) 2024-08-17T16:58:41.039 INFO:tasks.rados.rados.0.smithi063.stdout:919: expect (ObjNum 277 snap 86 seq_num 277) 2024-08-17T16:58:41.191 INFO:tasks.rados.rados.0.smithi063.stdout:923: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:58:41.347 INFO:tasks.rados.rados.0.smithi063.stdout:924: finishing write tid 1 to smithi06387500-31 2024-08-17T16:58:41.347 INFO:tasks.rados.rados.0.smithi063.stdout:917: done (8 left) 2024-08-17T16:58:41.347 INFO:tasks.rados.rados.0.smithi063.stdout:918: done (7 left) 2024-08-17T16:58:41.347 INFO:tasks.rados.rados.0.smithi063.stdout:919: done (6 left) 2024-08-17T16:58:41.347 INFO:tasks.rados.rados.0.smithi063.stdout:920: done (5 left) 2024-08-17T16:58:41.347 INFO:tasks.rados.rados.0.smithi063.stdout:923: done (4 left) 2024-08-17T16:58:41.347 INFO:tasks.rados.rados.0.smithi063.stdout:927: done (3 left) 2024-08-17T16:58:41.347 INFO:tasks.rados.rados.0.smithi063.stdout:928: read oid 5 snap -1 2024-08-17T16:58:41.347 INFO:tasks.rados.rados.0.smithi063.stdout:928: expect deleted 2024-08-17T16:58:41.347 INFO:tasks.rados.rados.0.smithi063.stdout:929: snap_create 2024-08-17T16:58:41.348 INFO:tasks.rados.rados.0.smithi063.stdout:924: finishing write tid 2 to smithi06387500-31 2024-08-17T16:58:41.348 INFO:tasks.rados.rados.0.smithi063.stdout:924: finishing write tid 3 to smithi06387500-31 2024-08-17T16:58:41.348 INFO:tasks.rados.rados.0.smithi063.stdout:924: finishing write tid 4 to smithi06387500-31 2024-08-17T16:58:41.348 INFO:tasks.rados.rados.0.smithi063.stdout:924: oid 31 updating version 0 to 666 2024-08-17T16:58:41.348 INFO:tasks.rados.rados.0.smithi063.stdout:924: oid 31 version 666 is already newer than 665 2024-08-17T16:58:41.348 INFO:tasks.rados.rados.0.smithi063.stdout:924: oid 31 updating version 666 to 667 2024-08-17T16:58:41.348 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 667 (ObjNum 379 snap 117 seq_num 379) dirty exists 2024-08-17T16:58:41.348 INFO:tasks.rados.rados.0.smithi063.stdout:924: left oid 31 (ObjNum 379 snap 117 seq_num 379) 2024-08-17T16:58:41.348 INFO:tasks.rados.rados.0.smithi063.stdout:925: finishing write tid 1 to smithi06387500-14 2024-08-17T16:58:41.348 INFO:tasks.rados.rados.0.smithi063.stdout:925: finishing write tid 2 to smithi06387500-14 2024-08-17T16:58:41.348 INFO:tasks.rados.rados.0.smithi063.stdout:925: finishing write tid 3 to smithi06387500-14 2024-08-17T16:58:41.348 INFO:tasks.rados.rados.0.smithi063.stdout:925: finishing write tid 4 to smithi06387500-14 2024-08-17T16:58:41.348 INFO:tasks.rados.rados.0.smithi063.stdout:925: oid 14 updating version 0 to 969 2024-08-17T16:58:41.349 INFO:tasks.rados.rados.0.smithi063.stdout:925: oid 14 updating version 969 to 970 2024-08-17T16:58:41.349 INFO:tasks.rados.rados.0.smithi063.stdout:925: oid 14 updating version 970 to 971 2024-08-17T16:58:41.349 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 971 (ObjNum 380 snap 117 seq_num 380) dirty exists 2024-08-17T16:58:41.349 INFO:tasks.rados.rados.0.smithi063.stdout:925: left oid 14 (ObjNum 380 snap 117 seq_num 380) 2024-08-17T16:58:41.349 INFO:tasks.rados.rados.0.smithi063.stdout:926: expect (ObjNum 347 snap 103 seq_num 347) 2024-08-17T16:58:41.929 INFO:tasks.rados.rados.0.smithi063.stdout:924: done (4 left) 2024-08-17T16:58:41.929 INFO:tasks.rados.rados.0.smithi063.stdout:925: done (3 left) 2024-08-17T16:58:41.929 INFO:tasks.rados.rados.0.smithi063.stdout:926: done (2 left) 2024-08-17T16:58:41.929 INFO:tasks.rados.rados.0.smithi063.stdout:928: done (1 left) 2024-08-17T16:58:41.929 INFO:tasks.rados.rados.0.smithi063.stdout:929: done (0 left) 2024-08-17T16:58:41.929 INFO:tasks.rados.rados.0.smithi063.stdout:930: rollback oid 28 current snap is 118 2024-08-17T16:58:41.929 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 28 to 117 2024-08-17T16:58:41.929 INFO:tasks.rados.rados.0.smithi063.stdout:931: read oid 29 snap -1 2024-08-17T16:58:41.929 INFO:tasks.rados.rados.0.smithi063.stdout:931: expect (ObjNum 335 snap 101 seq_num 335) 2024-08-17T16:58:41.929 INFO:tasks.rados.rados.0.smithi063.stdout:932: rollback oid 42 current snap is 118 2024-08-17T16:58:41.929 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 42 to 114 2024-08-17T16:58:41.929 INFO:tasks.rados.rados.0.smithi063.stdout:933: read oid 46 snap -1 2024-08-17T16:58:41.930 INFO:tasks.rados.rados.0.smithi063.stdout:933: expect (ObjNum 293 snap 91 seq_num 293) 2024-08-17T16:58:41.930 INFO:tasks.rados.rados.0.smithi063.stdout:934: write oid 1 current snap is 118 2024-08-17T16:58:41.930 INFO:tasks.rados.rados.0.smithi063.stdout:934: seq_num 381 ranges {403467=487519,1562110=579120} 2024-08-17T16:58:41.934 INFO:tasks.rados.rados.0.smithi063.stdout:934: writing smithi06387500-1 from 403467 to 890986 tid 1 2024-08-17T16:58:41.941 INFO:tasks.rados.rados.0.smithi063.stdout:934: writing smithi06387500-1 from 1562110 to 2141230 tid 2 2024-08-17T16:58:41.941 INFO:tasks.rados.rados.0.smithi063.stdout:935: read oid 30 snap -1 2024-08-17T16:58:41.942 INFO:tasks.rados.rados.0.smithi063.stdout:935: expect (ObjNum 377 snap 117 seq_num 377) 2024-08-17T16:58:41.942 INFO:tasks.rados.rados.0.smithi063.stdout:936: delete oid 48 current snap is 118 2024-08-17T16:58:41.942 INFO:tasks.rados.rados.0.smithi063.stdout:932: finishing rollback tid 0 to smithi06387500-42 2024-08-17T16:58:41.942 INFO:tasks.rados.rados.0.smithi063.stdout:932: finishing rollback tid 1 to smithi06387500-42 2024-08-17T16:58:41.942 INFO:tasks.rados.rados.0.smithi063.stdout:932: finishing rollback tid 2 to smithi06387500-42 2024-08-17T16:58:41.942 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 675 (ObjNum 347 snap 103 seq_num 347) dirty exists 2024-08-17T16:58:41.942 INFO:tasks.rados.rados.0.smithi063.stdout:930: finishing rollback tid 0 to smithi06387500-28 2024-08-17T16:58:41.942 INFO:tasks.rados.rados.0.smithi063.stdout:930: finishing rollback tid 1 to smithi06387500-28 2024-08-17T16:58:41.942 INFO:tasks.rados.rados.0.smithi063.stdout:930: finishing rollback tid 2 to smithi06387500-28 2024-08-17T16:58:41.942 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 266 (ObjNum 301 snap 91 seq_num 301) dirty exists 2024-08-17T16:58:41.942 INFO:tasks.rados.rados.0.smithi063.stdout:931: expect (ObjNum 335 snap 101 seq_num 335) 2024-08-17T16:58:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: pgmap v141: 89 pgs: 89 active+clean; 174 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1022 KiB/s wr, 17 op/s 2024-08-17T16:58:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: osdmap e305: 8 total, 8 up, 8 in 2024-08-17T16:58:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: mgrmap e34: y(active, since 92s), standbys: x 2024-08-17T16:58:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:58:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: pgmap v141: 89 pgs: 89 active+clean; 174 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1022 KiB/s wr, 17 op/s 2024-08-17T16:58:42.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: osdmap e305: 8 total, 8 up, 8 in 2024-08-17T16:58:42.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: mgrmap e34: y(active, since 92s), standbys: x 2024-08-17T16:58:42.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:42.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:42.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:42.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:42.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:42.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:58:42.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:42.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:41 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.059 INFO:tasks.rados.rados.0.smithi063.stdout:933: expect (ObjNum 293 snap 91 seq_num 293) 2024-08-17T16:58:42.221 INFO:tasks.rados.rados.0.smithi063.stdout:934: finishing write tid 1 to smithi06387500-1 2024-08-17T16:58:42.221 INFO:tasks.rados.rados.0.smithi063.stdout:934: finishing write tid 2 to smithi06387500-1 2024-08-17T16:58:42.221 INFO:tasks.rados.rados.0.smithi063.stdout:934: finishing write tid 3 to smithi06387500-1 2024-08-17T16:58:42.221 INFO:tasks.rados.rados.0.smithi063.stdout:934: finishing write tid 4 to smithi06387500-1 2024-08-17T16:58:42.221 INFO:tasks.rados.rados.0.smithi063.stdout:934: oid 1 updating version 0 to 459 2024-08-17T16:58:42.221 INFO:tasks.rados.rados.0.smithi063.stdout:934: oid 1 version 459 is already newer than 458 2024-08-17T16:58:42.221 INFO:tasks.rados.rados.0.smithi063.stdout:934: oid 1 updating version 459 to 460 2024-08-17T16:58:42.221 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 460 (ObjNum 381 snap 118 seq_num 381) dirty exists 2024-08-17T16:58:42.221 INFO:tasks.rados.rados.0.smithi063.stdout:934: left oid 1 (ObjNum 381 snap 118 seq_num 381) 2024-08-17T16:58:42.221 INFO:tasks.rados.rados.0.smithi063.stdout:930: done (6 left) 2024-08-17T16:58:42.221 INFO:tasks.rados.rados.0.smithi063.stdout:931: done (5 left) 2024-08-17T16:58:42.222 INFO:tasks.rados.rados.0.smithi063.stdout:932: done (4 left) 2024-08-17T16:58:42.222 INFO:tasks.rados.rados.0.smithi063.stdout:933: done (3 left) 2024-08-17T16:58:42.222 INFO:tasks.rados.rados.0.smithi063.stdout:934: done (2 left) 2024-08-17T16:58:42.222 INFO:tasks.rados.rados.0.smithi063.stdout:936: done (1 left) 2024-08-17T16:58:42.222 INFO:tasks.rados.rados.0.smithi063.stdout:937: snap_remove snap 111 2024-08-17T16:58:42.253 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: pgmap v141: 89 pgs: 89 active+clean; 174 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1022 KiB/s wr, 17 op/s 2024-08-17T16:58:42.253 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: osdmap e305: 8 total, 8 up, 8 in 2024-08-17T16:58:42.253 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: mgrmap e34: y(active, since 92s), standbys: x 2024-08-17T16:58:42.254 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:42.254 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:42.254 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:58:42.254 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:58:42.254 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:58:42.254 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:58:42.254 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:58:42.254 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.254 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.254 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.254 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.254 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.254 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.255 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.255 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.255 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.255 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.255 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.255 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:41 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:42.942 INFO:tasks.rados.rados.0.smithi063.stdout:937: done (1 left) 2024-08-17T16:58:42.942 INFO:tasks.rados.rados.0.smithi063.stdout:938: read oid 14 snap -1 2024-08-17T16:58:42.942 INFO:tasks.rados.rados.0.smithi063.stdout:938: expect (ObjNum 380 snap 117 seq_num 380) 2024-08-17T16:58:42.942 INFO:tasks.rados.rados.0.smithi063.stdout:939: delete oid 43 current snap is 118 2024-08-17T16:58:42.943 INFO:tasks.rados.rados.0.smithi063.stdout:935: expect (ObjNum 377 snap 117 seq_num 377) 2024-08-17T16:58:43.024 INFO:tasks.rados.rados.0.smithi063.stdout:938: expect (ObjNum 380 snap 117 seq_num 380) 2024-08-17T16:58:43.139 INFO:tasks.rados.rados.0.smithi063.stdout:935: done (2 left) 2024-08-17T16:58:43.140 INFO:tasks.rados.rados.0.smithi063.stdout:938: done (1 left) 2024-08-17T16:58:43.140 INFO:tasks.rados.rados.0.smithi063.stdout:939: done (0 left) 2024-08-17T16:58:43.140 INFO:tasks.rados.rados.0.smithi063.stdout:940: write oid 9 current snap is 118 2024-08-17T16:58:43.140 INFO:tasks.rados.rados.0.smithi063.stdout:940: seq_num 384 ranges {453888=713266,1737556=442800} 2024-08-17T16:58:43.146 INFO:tasks.rados.rados.0.smithi063.stdout:940: writing smithi06387500-9 from 453888 to 1167154 tid 1 2024-08-17T16:58:43.151 INFO:tasks.rados.rados.0.smithi063.stdout:940: writing smithi06387500-9 from 1737556 to 2180356 tid 2 2024-08-17T16:58:43.151 INFO:tasks.rados.rados.0.smithi063.stdout:941: read oid 42 snap -1 2024-08-17T16:58:43.152 INFO:tasks.rados.rados.0.smithi063.stdout:941: expect (ObjNum 347 snap 103 seq_num 347) 2024-08-17T16:58:43.152 INFO:tasks.rados.rados.0.smithi063.stdout:942: snap_remove snap 114 2024-08-17T16:58:43.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:42 smithi063 ceph-mon[31063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:58:43.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:42 smithi063 ceph-mon[31063]: osdmap e306: 8 total, 8 up, 8 in 2024-08-17T16:58:43.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:42 smithi063 ceph-mon[42680]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:58:43.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:42 smithi063 ceph-mon[42680]: osdmap e306: 8 total, 8 up, 8 in 2024-08-17T16:58:43.332 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:42 smithi162 ceph-mon[34633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-08-17T16:58:43.332 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:42 smithi162 ceph-mon[34633]: osdmap e306: 8 total, 8 up, 8 in 2024-08-17T16:58:43.697 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:58:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:58:43] "GET /metrics HTTP/1.1" 200 36211 "" "Prometheus/2.51.0" 2024-08-17T16:58:43.937 INFO:tasks.rados.rados.0.smithi063.stdout:940: finishing write tid 1 to smithi06387500-9 2024-08-17T16:58:43.937 INFO:tasks.rados.rados.0.smithi063.stdout:941: expect (ObjNum 347 snap 103 seq_num 347) 2024-08-17T16:58:44.029 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 16:58:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[104529]: ts=2024-08-17T16:58:43.697Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003662055s 2024-08-17T16:58:44.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:43 smithi063 ceph-mon[31063]: pgmap v143: 89 pgs: 89 active+clean; 174 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1022 KiB/s wr, 17 op/s 2024-08-17T16:58:44.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:43 smithi063 ceph-mon[31063]: osdmap e307: 8 total, 8 up, 8 in 2024-08-17T16:58:44.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:43 smithi063 ceph-mon[31063]: Upgrade: Updating grafana.a 2024-08-17T16:58:44.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:43 smithi063 ceph-mon[31063]: Deploying daemon grafana.a on smithi162 2024-08-17T16:58:44.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:43 smithi063 ceph-mon[42680]: pgmap v143: 89 pgs: 89 active+clean; 174 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1022 KiB/s wr, 17 op/s 2024-08-17T16:58:44.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:43 smithi063 ceph-mon[42680]: osdmap e307: 8 total, 8 up, 8 in 2024-08-17T16:58:44.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:43 smithi063 ceph-mon[42680]: Upgrade: Updating grafana.a 2024-08-17T16:58:44.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:43 smithi063 ceph-mon[42680]: Deploying daemon grafana.a on smithi162 2024-08-17T16:58:44.303 INFO:tasks.rados.rados.0.smithi063.stdout:940: finishing write tid 2 to smithi06387500-9 2024-08-17T16:58:44.303 INFO:tasks.rados.rados.0.smithi063.stdout:940: finishing write tid 3 to smithi06387500-9 2024-08-17T16:58:44.303 INFO:tasks.rados.rados.0.smithi063.stdout:940: finishing write tid 4 to smithi06387500-9 2024-08-17T16:58:44.303 INFO:tasks.rados.rados.0.smithi063.stdout:940: oid 9 updating version 0 to 561 2024-08-17T16:58:44.303 INFO:tasks.rados.rados.0.smithi063.stdout:940: oid 9 updating version 561 to 562 2024-08-17T16:58:44.303 INFO:tasks.rados.rados.0.smithi063.stdout:940: oid 9 updating version 562 to 563 2024-08-17T16:58:44.303 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 563 (ObjNum 384 snap 118 seq_num 384) dirty exists 2024-08-17T16:58:44.303 INFO:tasks.rados.rados.0.smithi063.stdout:940: left oid 9 (ObjNum 384 snap 118 seq_num 384) 2024-08-17T16:58:44.304 INFO:tasks.rados.rados.0.smithi063.stdout:940: done (2 left) 2024-08-17T16:58:44.304 INFO:tasks.rados.rados.0.smithi063.stdout:941: done (1 left) 2024-08-17T16:58:44.304 INFO:tasks.rados.rados.0.smithi063.stdout:942: done (0 left) 2024-08-17T16:58:44.304 INFO:tasks.rados.rados.0.smithi063.stdout:943: write oid 5 current snap is 118 2024-08-17T16:58:44.304 INFO:tasks.rados.rados.0.smithi063.stdout:943: seq_num 385 ranges {470695=788515,1679685=586458} 2024-08-17T16:58:44.309 INFO:tasks.rados.rados.0.smithi063.stdout:943: writing smithi06387500-5 from 470695 to 1259210 tid 1 2024-08-17T16:58:44.315 INFO:tasks.rados.rados.0.smithi063.stdout:943: writing smithi06387500-5 from 1679685 to 2266143 tid 2 2024-08-17T16:58:44.315 INFO:tasks.rados.rados.0.smithi063.stdout:944: write oid 38 current snap is 118 2024-08-17T16:58:44.315 INFO:tasks.rados.rados.0.smithi063.stdout:944: seq_num 386 ranges {487502=463764,1738167=446469} 2024-08-17T16:58:44.320 INFO:tasks.rados.rados.0.smithi063.stdout:944: writing smithi06387500-38 from 487502 to 951266 tid 1 2024-08-17T16:58:44.328 INFO:tasks.rados.rados.0.smithi063.stdout:944: writing smithi06387500-38 from 1738167 to 2184636 tid 2 2024-08-17T16:58:44.328 INFO:tasks.rados.rados.0.smithi063.stdout:945: rollback oid 24 current snap is 118 2024-08-17T16:58:44.328 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 24 to 107 2024-08-17T16:58:44.328 INFO:tasks.rados.rados.0.smithi063.stdout:943: finishing write tid 1 to smithi06387500-5 2024-08-17T16:58:44.328 INFO:tasks.rados.rados.0.smithi063.stdout:943: finishing write tid 2 to smithi06387500-5 2024-08-17T16:58:44.328 INFO:tasks.rados.rados.0.smithi063.stdout:943: finishing write tid 3 to smithi06387500-5 2024-08-17T16:58:44.328 INFO:tasks.rados.rados.0.smithi063.stdout:943: finishing write tid 4 to smithi06387500-5 2024-08-17T16:58:44.328 INFO:tasks.rados.rados.0.smithi063.stdout:943: oid 5 updating version 0 to 471 2024-08-17T16:58:44.328 INFO:tasks.rados.rados.0.smithi063.stdout:943: oid 5 version 471 is already newer than 470 2024-08-17T16:58:44.328 INFO:tasks.rados.rados.0.smithi063.stdout:943: oid 5 version 471 is already newer than 469 2024-08-17T16:58:44.328 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 471 (ObjNum 385 snap 118 seq_num 385) dirty exists 2024-08-17T16:58:44.328 INFO:tasks.rados.rados.0.smithi063.stdout:943: left oid 5 (ObjNum 385 snap 118 seq_num 385) 2024-08-17T16:58:44.328 INFO:tasks.rados.rados.0.smithi063.stdout:944: finishing write tid 1 to smithi06387500-38 2024-08-17T16:58:44.328 INFO:tasks.rados.rados.0.smithi063.stdout:943: done (2 left) 2024-08-17T16:58:44.329 INFO:tasks.rados.rados.0.smithi063.stdout:946: write oid 6 current snap is 118 2024-08-17T16:58:44.329 INFO:tasks.rados.rados.0.smithi063.stdout:946: seq_num 387 ranges {504309=539013,1796649=590127} 2024-08-17T16:58:44.333 INFO:tasks.rados.rados.0.smithi063.stdout:946: writing smithi06387500-6 from 504309 to 1043322 tid 1 2024-08-17T16:58:44.340 INFO:tasks.rados.rados.0.smithi063.stdout:946: writing smithi06387500-6 from 1796649 to 2386776 tid 2 2024-08-17T16:58:44.340 INFO:tasks.rados.rados.0.smithi063.stdout:947: snap_remove snap 113 2024-08-17T16:58:44.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:43 smithi162 ceph-mon[34633]: pgmap v143: 89 pgs: 89 active+clean; 174 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1022 KiB/s wr, 17 op/s 2024-08-17T16:58:44.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:43 smithi162 ceph-mon[34633]: osdmap e307: 8 total, 8 up, 8 in 2024-08-17T16:58:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:43 smithi162 ceph-mon[34633]: Upgrade: Updating grafana.a 2024-08-17T16:58:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:43 smithi162 ceph-mon[34633]: Deploying daemon grafana.a on smithi162 2024-08-17T16:58:44.880 INFO:tasks.rados.rados.0.smithi063.stdout:947: done (3 left) 2024-08-17T16:58:44.881 INFO:tasks.rados.rados.0.smithi063.stdout:948: snap_create 2024-08-17T16:58:44.881 INFO:tasks.rados.rados.0.smithi063.stdout:944: finishing write tid 2 to smithi06387500-38 2024-08-17T16:58:44.881 INFO:tasks.rados.rados.0.smithi063.stdout:944: finishing write tid 3 to smithi06387500-38 2024-08-17T16:58:44.881 INFO:tasks.rados.rados.0.smithi063.stdout:945: finishing rollback tid 0 to smithi06387500-24 2024-08-17T16:58:44.881 INFO:tasks.rados.rados.0.smithi063.stdout:944: finishing write tid 4 to smithi06387500-38 2024-08-17T16:58:44.881 INFO:tasks.rados.rados.0.smithi063.stdout:944: oid 38 updating version 0 to 101 2024-08-17T16:58:44.881 INFO:tasks.rados.rados.0.smithi063.stdout:944: oid 38 updating version 101 to 103 2024-08-17T16:58:44.881 INFO:tasks.rados.rados.0.smithi063.stdout:944: oid 38 version 103 is already newer than 102 2024-08-17T16:58:44.881 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 103 (ObjNum 386 snap 118 seq_num 386) dirty exists 2024-08-17T16:58:44.881 INFO:tasks.rados.rados.0.smithi063.stdout:944: left oid 38 (ObjNum 386 snap 118 seq_num 386) 2024-08-17T16:58:44.881 INFO:tasks.rados.rados.0.smithi063.stdout:945: finishing rollback tid 1 to smithi06387500-24 2024-08-17T16:58:44.881 INFO:tasks.rados.rados.0.smithi063.stdout:945: finishing rollback tid 2 to smithi06387500-24 2024-08-17T16:58:44.881 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 362 (ObjNum 286 snap 87 seq_num 286) dirty exists 2024-08-17T16:58:44.882 INFO:tasks.rados.rados.0.smithi063.stdout:946: finishing write tid 1 to smithi06387500-6 2024-08-17T16:58:44.882 INFO:tasks.rados.rados.0.smithi063.stdout:946: finishing write tid 2 to smithi06387500-6 2024-08-17T16:58:44.882 INFO:tasks.rados.rados.0.smithi063.stdout:946: finishing write tid 3 to smithi06387500-6 2024-08-17T16:58:44.882 INFO:tasks.rados.rados.0.smithi063.stdout:946: finishing write tid 4 to smithi06387500-6 2024-08-17T16:58:44.882 INFO:tasks.rados.rados.0.smithi063.stdout:946: oid 6 updating version 0 to 257 2024-08-17T16:58:44.882 INFO:tasks.rados.rados.0.smithi063.stdout:946: oid 6 version 257 is already newer than 256 2024-08-17T16:58:44.882 INFO:tasks.rados.rados.0.smithi063.stdout:946: oid 6 version 257 is already newer than 255 2024-08-17T16:58:44.882 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 257 (ObjNum 387 snap 118 seq_num 387) dirty exists 2024-08-17T16:58:44.882 INFO:tasks.rados.rados.0.smithi063.stdout:946: left oid 6 (ObjNum 387 snap 118 seq_num 387) 2024-08-17T16:58:45.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:44 smithi063 ceph-mon[42680]: osdmap e308: 8 total, 8 up, 8 in 2024-08-17T16:58:45.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:44 smithi063 ceph-mon[42680]: osdmap e309: 8 total, 8 up, 8 in 2024-08-17T16:58:45.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:44 smithi063 ceph-mon[42680]: pgmap v147: 89 pgs: 89 active+clean; 174 MiB data, 638 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:58:45.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:44 smithi063 ceph-mon[31063]: osdmap e308: 8 total, 8 up, 8 in 2024-08-17T16:58:45.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:44 smithi063 ceph-mon[31063]: osdmap e309: 8 total, 8 up, 8 in 2024-08-17T16:58:45.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:44 smithi063 ceph-mon[31063]: pgmap v147: 89 pgs: 89 active+clean; 174 MiB data, 638 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:58:45.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:44 smithi162 ceph-mon[34633]: osdmap e308: 8 total, 8 up, 8 in 2024-08-17T16:58:45.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:44 smithi162 ceph-mon[34633]: osdmap e309: 8 total, 8 up, 8 in 2024-08-17T16:58:45.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:44 smithi162 ceph-mon[34633]: pgmap v147: 89 pgs: 89 active+clean; 174 MiB data, 638 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:58:45.884 INFO:tasks.rados.rados.0.smithi063.stdout:944: done (3 left) 2024-08-17T16:58:45.884 INFO:tasks.rados.rados.0.smithi063.stdout:945: done (2 left) 2024-08-17T16:58:45.884 INFO:tasks.rados.rados.0.smithi063.stdout:946: done (1 left) 2024-08-17T16:58:45.884 INFO:tasks.rados.rados.0.smithi063.stdout:948: done (0 left) 2024-08-17T16:58:45.884 INFO:tasks.rados.rados.0.smithi063.stdout:949: snap_remove snap 118 2024-08-17T16:58:46.886 INFO:tasks.rados.rados.0.smithi063.stdout:949: done (0 left) 2024-08-17T16:58:46.886 INFO:tasks.rados.rados.0.smithi063.stdout:950: delete oid 50 current snap is 119 2024-08-17T16:58:46.889 INFO:tasks.rados.rados.0.smithi063.stdout:950: done (0 left) 2024-08-17T16:58:46.889 INFO:tasks.rados.rados.0.smithi063.stdout:951: delete oid 37 current snap is 119 2024-08-17T16:58:46.892 INFO:tasks.rados.rados.0.smithi063.stdout:951: done (0 left) 2024-08-17T16:58:46.893 INFO:tasks.rados.rados.0.smithi063.stdout:952: write oid 39 current snap is 119 2024-08-17T16:58:46.893 INFO:tasks.rados.rados.0.smithi063.stdout:952: seq_num 390 ranges {554730=481113,1572095=737454} 2024-08-17T16:58:46.898 INFO:tasks.rados.rados.0.smithi063.stdout:952: writing smithi06387500-39 from 554730 to 1035843 tid 1 2024-08-17T16:58:46.906 INFO:tasks.rados.rados.0.smithi063.stdout:952: writing smithi06387500-39 from 1572095 to 2309549 tid 2 2024-08-17T16:58:46.906 INFO:tasks.rados.rados.0.smithi063.stdout:953: rollback oid 32 current snap is 119 2024-08-17T16:58:46.906 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 32 to 117 2024-08-17T16:58:46.906 INFO:tasks.rados.rados.0.smithi063.stdout:952: finishing write tid 1 to smithi06387500-39 2024-08-17T16:58:46.906 INFO:tasks.rados.rados.0.smithi063.stdout:954: snap_remove snap 115 2024-08-17T16:58:47.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:46 smithi063 ceph-mon[31063]: osdmap e310: 8 total, 8 up, 8 in 2024-08-17T16:58:47.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:46 smithi063 ceph-mon[42680]: osdmap e310: 8 total, 8 up, 8 in 2024-08-17T16:58:47.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:46 smithi162 ceph-mon[34633]: osdmap e310: 8 total, 8 up, 8 in 2024-08-17T16:58:47.899 INFO:tasks.rados.rados.0.smithi063.stdout:953: finishing rollback tid 0 to smithi06387500-32 2024-08-17T16:58:47.899 INFO:tasks.rados.rados.0.smithi063.stdout:953: finishing rollback tid 1 to smithi06387500-32 2024-08-17T16:58:47.899 INFO:tasks.rados.rados.0.smithi063.stdout:952: finishing write tid 2 to smithi06387500-39 2024-08-17T16:58:47.899 INFO:tasks.rados.rados.0.smithi063.stdout:953: finishing rollback tid 2 to smithi06387500-32 2024-08-17T16:58:47.899 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 174 (ObjNum 307 snap 92 seq_num 307) dirty exists 2024-08-17T16:58:47.899 INFO:tasks.rados.rados.0.smithi063.stdout:952: finishing write tid 3 to smithi06387500-39 2024-08-17T16:58:47.899 INFO:tasks.rados.rados.0.smithi063.stdout:952: finishing write tid 4 to smithi06387500-39 2024-08-17T16:58:47.899 INFO:tasks.rados.rados.0.smithi063.stdout:952: oid 39 updating version 0 to 240 2024-08-17T16:58:47.899 INFO:tasks.rados.rados.0.smithi063.stdout:952: oid 39 version 240 is already newer than 239 2024-08-17T16:58:47.899 INFO:tasks.rados.rados.0.smithi063.stdout:952: oid 39 updating version 240 to 241 2024-08-17T16:58:47.900 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 241 (ObjNum 390 snap 119 seq_num 390) dirty exists 2024-08-17T16:58:47.900 INFO:tasks.rados.rados.0.smithi063.stdout:952: left oid 39 (ObjNum 390 snap 119 seq_num 390) 2024-08-17T16:58:47.900 INFO:tasks.rados.rados.0.smithi063.stdout:952: done (2 left) 2024-08-17T16:58:47.900 INFO:tasks.rados.rados.0.smithi063.stdout:953: done (1 left) 2024-08-17T16:58:47.900 INFO:tasks.rados.rados.0.smithi063.stdout:954: done (0 left) 2024-08-17T16:58:47.900 INFO:tasks.rados.rados.0.smithi063.stdout:955: write oid 1 current snap is 119 2024-08-17T16:58:47.900 INFO:tasks.rados.rados.0.smithi063.stdout:955: seq_num 391 ranges {571537=556362,1630577=597465} 2024-08-17T16:58:47.905 INFO:tasks.rados.rados.0.smithi063.stdout:955: writing smithi06387500-1 from 571537 to 1127899 tid 1 2024-08-17T16:58:47.912 INFO:tasks.rados.rados.0.smithi063.stdout:955: writing smithi06387500-1 from 1630577 to 2228042 tid 2 2024-08-17T16:58:47.912 INFO:tasks.rados.rados.0.smithi063.stdout:956: snap_remove snap 109 2024-08-17T16:58:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:47 smithi063 ceph-mon[42680]: osdmap e311: 8 total, 8 up, 8 in 2024-08-17T16:58:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:47 smithi063 ceph-mon[42680]: pgmap v150: 89 pgs: 89 active+clean; 151 MiB data, 588 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-08-17T16:58:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:47 smithi063 ceph-mon[31063]: osdmap e311: 8 total, 8 up, 8 in 2024-08-17T16:58:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:47 smithi063 ceph-mon[31063]: pgmap v150: 89 pgs: 89 active+clean; 151 MiB data, 588 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-08-17T16:58:48.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:47 smithi162 ceph-mon[34633]: osdmap e311: 8 total, 8 up, 8 in 2024-08-17T16:58:48.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:47 smithi162 ceph-mon[34633]: pgmap v150: 89 pgs: 89 active+clean; 151 MiB data, 588 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 18 op/s 2024-08-17T16:58:48.904 INFO:tasks.rados.rados.0.smithi063.stdout:955: finishing write tid 1 to smithi06387500-1 2024-08-17T16:58:48.904 INFO:tasks.rados.rados.0.smithi063.stdout:955: finishing write tid 2 to smithi06387500-1 2024-08-17T16:58:48.904 INFO:tasks.rados.rados.0.smithi063.stdout:955: finishing write tid 3 to smithi06387500-1 2024-08-17T16:58:48.904 INFO:tasks.rados.rados.0.smithi063.stdout:955: finishing write tid 4 to smithi06387500-1 2024-08-17T16:58:48.904 INFO:tasks.rados.rados.0.smithi063.stdout:955: oid 1 updating version 0 to 481 2024-08-17T16:58:48.904 INFO:tasks.rados.rados.0.smithi063.stdout:955: oid 1 version 481 is already newer than 480 2024-08-17T16:58:48.904 INFO:tasks.rados.rados.0.smithi063.stdout:955: oid 1 updating version 481 to 482 2024-08-17T16:58:48.904 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 482 (ObjNum 391 snap 119 seq_num 391) dirty exists 2024-08-17T16:58:48.904 INFO:tasks.rados.rados.0.smithi063.stdout:955: left oid 1 (ObjNum 391 snap 119 seq_num 391) 2024-08-17T16:58:48.904 INFO:tasks.rados.rados.0.smithi063.stdout:955: done (1 left) 2024-08-17T16:58:48.905 INFO:tasks.rados.rados.0.smithi063.stdout:956: done (0 left) 2024-08-17T16:58:48.905 INFO:tasks.rados.rados.0.smithi063.stdout:957: rollback oid 29 current snap is 119 2024-08-17T16:58:48.905 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 29 to 106 2024-08-17T16:58:48.905 INFO:tasks.rados.rados.0.smithi063.stdout:958: write oid 17 current snap is 119 2024-08-17T16:58:48.905 INFO:tasks.rados.rados.0.smithi063.stdout:958: seq_num 392 ranges {588344=631611,1689059=741123} 2024-08-17T16:58:48.911 INFO:tasks.rados.rados.0.smithi063.stdout:958: writing smithi06387500-17 from 588344 to 1219955 tid 1 2024-08-17T16:58:48.919 INFO:tasks.rados.rados.0.smithi063.stdout:958: writing smithi06387500-17 from 1689059 to 2430182 tid 2 2024-08-17T16:58:48.919 INFO:tasks.rados.rados.0.smithi063.stdout:959: delete oid 5 current snap is 119 2024-08-17T16:58:48.919 INFO:tasks.rados.rados.0.smithi063.stdout:957: finishing rollback tid 0 to smithi06387500-29 2024-08-17T16:58:48.919 INFO:tasks.rados.rados.0.smithi063.stdout:957: finishing rollback tid 1 to smithi06387500-29 2024-08-17T16:58:48.919 INFO:tasks.rados.rados.0.smithi063.stdout:957: finishing rollback tid 2 to smithi06387500-29 2024-08-17T16:58:48.919 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 208 (ObjNum 335 snap 101 seq_num 335) dirty exists 2024-08-17T16:58:48.919 INFO:tasks.rados.rados.0.smithi063.stdout:958: finishing write tid 1 to smithi06387500-17 2024-08-17T16:58:48.921 INFO:tasks.rados.rados.0.smithi063.stdout:957: done (2 left) 2024-08-17T16:58:48.921 INFO:tasks.rados.rados.0.smithi063.stdout:959: done (1 left) 2024-08-17T16:58:48.921 INFO:tasks.rados.rados.0.smithi063.stdout:960: write oid 11 current snap is 119 2024-08-17T16:58:48.921 INFO:tasks.rados.rados.0.smithi063.stdout:960: seq_num 394 ranges {621958=782109,2089670=532288} 2024-08-17T16:58:48.928 INFO:tasks.rados.rados.0.smithi063.stdout:960: writing smithi06387500-11 from 621958 to 1404067 tid 1 2024-08-17T16:58:48.935 INFO:tasks.rados.rados.0.smithi063.stdout:960: writing smithi06387500-11 from 2089670 to 2621958 tid 2 2024-08-17T16:58:48.935 INFO:tasks.rados.rados.0.smithi063.stdout:961: snap_remove snap 116 2024-08-17T16:58:49.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:48 smithi063 ceph-mon[42680]: osdmap e312: 8 total, 8 up, 8 in 2024-08-17T16:58:49.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:48 smithi063 ceph-mon[31063]: osdmap e312: 8 total, 8 up, 8 in 2024-08-17T16:58:49.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:48 smithi162 ceph-mon[34633]: osdmap e312: 8 total, 8 up, 8 in 2024-08-17T16:58:49.395 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T16:58:49.881 INFO:tasks.rados.rados.0.smithi063.stdout:961: done (2 left) 2024-08-17T16:58:49.881 INFO:tasks.rados.rados.0.smithi063.stdout:962: snap_create 2024-08-17T16:58:49.881 INFO:tasks.rados.rados.0.smithi063.stdout:958: finishing write tid 2 to smithi06387500-17 2024-08-17T16:58:49.881 INFO:tasks.rados.rados.0.smithi063.stdout:958: finishing write tid 3 to smithi06387500-17 2024-08-17T16:58:49.882 INFO:tasks.rados.rados.0.smithi063.stdout:958: finishing write tid 4 to smithi06387500-17 2024-08-17T16:58:49.882 INFO:tasks.rados.rados.0.smithi063.stdout:958: oid 17 updating version 0 to 327 2024-08-17T16:58:49.882 INFO:tasks.rados.rados.0.smithi063.stdout:958: oid 17 version 327 is already newer than 326 2024-08-17T16:58:49.882 INFO:tasks.rados.rados.0.smithi063.stdout:958: oid 17 version 327 is already newer than 325 2024-08-17T16:58:49.882 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 327 (ObjNum 392 snap 119 seq_num 392) dirty exists 2024-08-17T16:58:49.882 INFO:tasks.rados.rados.0.smithi063.stdout:958: left oid 17 (ObjNum 392 snap 119 seq_num 392) 2024-08-17T16:58:49.882 INFO:tasks.rados.rados.0.smithi063.stdout:960: finishing write tid 1 to smithi06387500-11 2024-08-17T16:58:49.882 INFO:tasks.rados.rados.0.smithi063.stdout:960: finishing write tid 2 to smithi06387500-11 2024-08-17T16:58:49.883 INFO:tasks.rados.rados.0.smithi063.stdout:960: finishing write tid 3 to smithi06387500-11 2024-08-17T16:58:49.883 INFO:tasks.rados.rados.0.smithi063.stdout:960: finishing write tid 4 to smithi06387500-11 2024-08-17T16:58:49.883 INFO:tasks.rados.rados.0.smithi063.stdout:960: oid 11 updating version 0 to 104 2024-08-17T16:58:49.883 INFO:tasks.rados.rados.0.smithi063.stdout:960: oid 11 updating version 104 to 106 2024-08-17T16:58:49.883 INFO:tasks.rados.rados.0.smithi063.stdout:960: oid 11 version 106 is already newer than 105 2024-08-17T16:58:49.883 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 106 (ObjNum 394 snap 119 seq_num 394) dirty exists 2024-08-17T16:58:49.883 INFO:tasks.rados.rados.0.smithi063.stdout:960: left oid 11 (ObjNum 394 snap 119 seq_num 394) 2024-08-17T16:58:49.983 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T16:58:49.983 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (16s) 12s ago 6m 17.4M - 0.27.0 11f11916f8cd 59f4018b91ca 2024-08-17T16:58:49.983 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (6m) 11s ago 6m 87.0M - 9.4.7 954c08fa6188 fc8d62329012 2024-08-17T16:58:49.983 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (5m) 12s ago 5m 18.0M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T16:58:49.984 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (4m) 11s ago 4m 17.2M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T16:58:49.984 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,9283,8765 running (81s) 11s ago 10m 453M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb da48034a2895 2024-08-17T16:58:49.984 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:8443,9283,8765 running (109s) 12s ago 11m 515M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 943e639cede6 2024-08-17T16:58:49.984 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (11m) 12s ago 11m 61.2M 2048M 18.2.4-719-gced649f8 c870753d67c2 0c71ddd7ada6 2024-08-17T16:58:49.984 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (10m) 11s ago 10m 54.7M 2048M 18.2.4-719-gced649f8 c870753d67c2 d4ecbe65a0c8 2024-08-17T16:58:49.984 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (10m) 12s ago 10m 52.3M 2048M 18.2.4-719-gced649f8 c870753d67c2 b3f2c9339c6b 2024-08-17T16:58:49.984 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 running (66s) 12s ago 5m 9223k - 1.7.0 72c9c2088986 34f55f5de2aa 2024-08-17T16:58:49.984 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (60s) 11s ago 5m 10.0M - 1.7.0 72c9c2088986 5bb162d0f1a9 2024-08-17T16:58:49.984 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (9m) 12s ago 9m 92.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 f3b910ed49b8 2024-08-17T16:58:49.984 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (9m) 12s ago 9m 73.7M 2480M 18.2.4-719-gced649f8 c870753d67c2 5d6fdb6000ad 2024-08-17T16:58:49.984 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (9m) 12s ago 9m 83.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 a3cbe8edb49a 2024-08-17T16:58:49.984 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (8m) 12s ago 8m 73.8M 2480M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T16:58:49.984 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (8m) 11s ago 8m 93.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T16:58:49.984 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (7m) 11s ago 7m 89.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T16:58:49.985 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (7m) 11s ago 7m 66.7M 2480M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T16:58:49.985 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (7m) 11s ago 7m 76.6M 2480M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T16:58:49.985 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (38s) 11s ago 6m 35.5M - 2.51.0 1d3b7f56885b dfe66b2c5568 2024-08-17T16:58:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:49 smithi063 ceph-mon[42680]: osdmap e313: 8 total, 8 up, 8 in 2024-08-17T16:58:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:49 smithi063 ceph-mon[42680]: pgmap v153: 89 pgs: 89 active+clean; 151 MiB data, 588 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 18 op/s 2024-08-17T16:58:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:49 smithi063 ceph-mon[42680]: osdmap e314: 8 total, 8 up, 8 in 2024-08-17T16:58:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:49 smithi063 ceph-mon[31063]: osdmap e313: 8 total, 8 up, 8 in 2024-08-17T16:58:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:49 smithi063 ceph-mon[31063]: pgmap v153: 89 pgs: 89 active+clean; 151 MiB data, 588 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 18 op/s 2024-08-17T16:58:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:49 smithi063 ceph-mon[31063]: osdmap e314: 8 total, 8 up, 8 in 2024-08-17T16:58:50.334 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:58:50.335 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T16:58:50.335 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T16:58:50.335 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:58:50.335 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T16:58:50.335 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T16:58:50.335 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:58:50.335 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T16:58:50.335 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T16:58:50.335 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:58:50.335 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T16:58:50.335 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T16:58:50.335 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:58:50.335 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T16:58:50.336 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 13, 2024-08-17T16:58:50.336 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T16:58:50.336 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T16:58:50.336 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:58:50.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:49 smithi162 ceph-mon[34633]: osdmap e313: 8 total, 8 up, 8 in 2024-08-17T16:58:50.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:49 smithi162 ceph-mon[34633]: pgmap v153: 89 pgs: 89 active+clean; 151 MiB data, 588 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 18 op/s 2024-08-17T16:58:50.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:49 smithi162 ceph-mon[34633]: osdmap e314: 8 total, 8 up, 8 in 2024-08-17T16:58:50.630 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:58:50.630 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T16:58:50.630 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-08-17T16:58:50.631 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-08-17T16:58:50.631 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-08-17T16:58:50.631 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr" 2024-08-17T16:58:50.631 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-08-17T16:58:50.631 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "2/2 daemons upgraded", 2024-08-17T16:58:50.631 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading grafana daemons", 2024-08-17T16:58:50.631 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T16:58:50.631 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:58:50.882 INFO:tasks.rados.rados.0.smithi063.stdout:958: done (2 left) 2024-08-17T16:58:50.882 INFO:tasks.rados.rados.0.smithi063.stdout:960: done (1 left) 2024-08-17T16:58:50.882 INFO:tasks.rados.rados.0.smithi063.stdout:962: done (0 left) 2024-08-17T16:58:50.883 INFO:tasks.rados.rados.0.smithi063.stdout:963: snap_create 2024-08-17T16:58:51.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:50 smithi063 ceph-mon[42680]: from='client.24992 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:51.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:50 smithi063 ceph-mon[42680]: from='client.24998 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:51.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:50 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/258120960' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:51.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:50 smithi063 ceph-mon[42680]: osdmap e315: 8 total, 8 up, 8 in 2024-08-17T16:58:51.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:50 smithi063 ceph-mon[31063]: from='client.24992 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:51.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:50 smithi063 ceph-mon[31063]: from='client.24998 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:51.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:50 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/258120960' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:51.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:50 smithi063 ceph-mon[31063]: osdmap e315: 8 total, 8 up, 8 in 2024-08-17T16:58:51.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:50 smithi162 ceph-mon[34633]: from='client.24992 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:51.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:50 smithi162 ceph-mon[34633]: from='client.24998 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:51.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:50 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/258120960' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:58:51.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:50 smithi162 ceph-mon[34633]: osdmap e315: 8 total, 8 up, 8 in 2024-08-17T16:58:51.886 INFO:tasks.rados.rados.0.smithi063.stdout:963: done (0 left) 2024-08-17T16:58:51.886 INFO:tasks.rados.rados.0.smithi063.stdout:964: write oid 31 current snap is 121 2024-08-17T16:58:51.886 INFO:tasks.rados.rados.0.smithi063.stdout:964: seq_num 395 ranges {638765=457358,1748152=604803} 2024-08-17T16:58:51.892 INFO:tasks.rados.rados.0.smithi063.stdout:964: writing smithi06387500-31 from 638765 to 1096123 tid 1 2024-08-17T16:58:51.899 INFO:tasks.rados.rados.0.smithi063.stdout:964: writing smithi06387500-31 from 1748152 to 2352955 tid 2 2024-08-17T16:58:51.899 INFO:tasks.rados.rados.0.smithi063.stdout:965: write oid 7 current snap is 121 2024-08-17T16:58:51.899 INFO:tasks.rados.rados.0.smithi063.stdout:964: finishing write tid 1 to smithi06387500-31 2024-08-17T16:58:51.899 INFO:tasks.rados.rados.0.smithi063.stdout:965: seq_num 396 ranges {655572=648960,1922987=732585} 2024-08-17T16:58:51.906 INFO:tasks.rados.rados.0.smithi063.stdout:965: writing smithi06387500-7 from 655572 to 1304532 tid 1 2024-08-17T16:58:51.914 INFO:tasks.rados.rados.0.smithi063.stdout:965: writing smithi06387500-7 from 1922987 to 2655572 tid 2 2024-08-17T16:58:51.914 INFO:tasks.rados.rados.0.smithi063.stdout:966: write oid 45 current snap is 121 2024-08-17T16:58:51.914 INFO:tasks.rados.rados.0.smithi063.stdout:964: finishing write tid 2 to smithi06387500-31 2024-08-17T16:58:51.914 INFO:tasks.rados.rados.0.smithi063.stdout:964: finishing write tid 3 to smithi06387500-31 2024-08-17T16:58:51.914 INFO:tasks.rados.rados.0.smithi063.stdout:964: finishing write tid 4 to smithi06387500-31 2024-08-17T16:58:51.914 INFO:tasks.rados.rados.0.smithi063.stdout:964: oid 31 updating version 0 to 711 2024-08-17T16:58:51.915 INFO:tasks.rados.rados.0.smithi063.stdout:964: oid 31 version 711 is already newer than 709 2024-08-17T16:58:51.915 INFO:tasks.rados.rados.0.smithi063.stdout:964: oid 31 version 711 is already newer than 710 2024-08-17T16:58:51.915 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 711 (ObjNum 395 snap 121 seq_num 395) dirty exists 2024-08-17T16:58:51.915 INFO:tasks.rados.rados.0.smithi063.stdout:964: left oid 31 (ObjNum 395 snap 121 seq_num 395) 2024-08-17T16:58:51.915 INFO:tasks.rados.rados.0.smithi063.stdout:965: finishing write tid 1 to smithi06387500-7 2024-08-17T16:58:51.915 INFO:tasks.rados.rados.0.smithi063.stdout:966: seq_num 397 ranges {672379=724209,1865116=608472} 2024-08-17T16:58:51.922 INFO:tasks.rados.rados.0.smithi063.stdout:966: writing smithi06387500-45 from 672379 to 1396588 tid 1 2024-08-17T16:58:51.928 INFO:tasks.rados.rados.0.smithi063.stdout:966: writing smithi06387500-45 from 1865116 to 2473588 tid 2 2024-08-17T16:58:51.928 INFO:tasks.rados.rados.0.smithi063.stdout:964: done (2 left) 2024-08-17T16:58:51.929 INFO:tasks.rados.rados.0.smithi063.stdout:967: read oid 34 snap -1 2024-08-17T16:58:51.929 INFO:tasks.rados.rados.0.smithi063.stdout:967: expect (ObjNum 311 snap 92 seq_num 311) 2024-08-17T16:58:51.929 INFO:tasks.rados.rados.0.smithi063.stdout:968: write oid 36 current snap is 121 2024-08-17T16:58:51.929 INFO:tasks.rados.rados.0.smithi063.stdout:965: finishing write tid 2 to smithi06387500-7 2024-08-17T16:58:51.929 INFO:tasks.rados.rados.0.smithi063.stdout:965: finishing write tid 3 to smithi06387500-7 2024-08-17T16:58:51.929 INFO:tasks.rados.rados.0.smithi063.stdout:965: finishing write tid 4 to smithi06387500-7 2024-08-17T16:58:51.929 INFO:tasks.rados.rados.0.smithi063.stdout:965: oid 7 updating version 0 to 593 2024-08-17T16:58:51.929 INFO:tasks.rados.rados.0.smithi063.stdout:965: oid 7 updating version 593 to 595 2024-08-17T16:58:51.929 INFO:tasks.rados.rados.0.smithi063.stdout:965: oid 7 version 595 is already newer than 594 2024-08-17T16:58:51.929 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 595 (ObjNum 396 snap 121 seq_num 396) dirty exists 2024-08-17T16:58:51.929 INFO:tasks.rados.rados.0.smithi063.stdout:965: left oid 7 (ObjNum 396 snap 121 seq_num 396) 2024-08-17T16:58:51.929 INFO:tasks.rados.rados.0.smithi063.stdout:966: finishing write tid 1 to smithi06387500-45 2024-08-17T16:58:51.929 INFO:tasks.rados.rados.0.smithi063.stdout:968: seq_num 398 ranges {689186=799458,1923598=752130} 2024-08-17T16:58:51.936 INFO:tasks.rados.rados.0.smithi063.stdout:968: writing smithi06387500-36 from 689186 to 1488644 tid 1 2024-08-17T16:58:51.944 INFO:tasks.rados.rados.0.smithi063.stdout:968: writing smithi06387500-36 from 1923598 to 2675728 tid 2 2024-08-17T16:58:51.945 INFO:tasks.rados.rados.0.smithi063.stdout:965: done (3 left) 2024-08-17T16:58:51.945 INFO:tasks.rados.rados.0.smithi063.stdout:969: read oid 11 snap -1 2024-08-17T16:58:51.945 INFO:tasks.rados.rados.0.smithi063.stdout:969: expect (ObjNum 394 snap 119 seq_num 394) 2024-08-17T16:58:51.945 INFO:tasks.rados.rados.0.smithi063.stdout:970: read oid 4 snap -1 2024-08-17T16:58:51.945 INFO:tasks.rados.rados.0.smithi063.stdout:970: expect deleted 2024-08-17T16:58:51.945 INFO:tasks.rados.rados.0.smithi063.stdout:971: rollback oid 8 current snap is 121 2024-08-17T16:58:51.945 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 8 to 107 2024-08-17T16:58:51.945 INFO:tasks.rados.rados.0.smithi063.stdout:966: finishing write tid 2 to smithi06387500-45 2024-08-17T16:58:51.945 INFO:tasks.rados.rados.0.smithi063.stdout:966: finishing write tid 3 to smithi06387500-45 2024-08-17T16:58:51.945 INFO:tasks.rados.rados.0.smithi063.stdout:966: finishing write tid 4 to smithi06387500-45 2024-08-17T16:58:51.945 INFO:tasks.rados.rados.0.smithi063.stdout:966: oid 45 updating version 0 to 249 2024-08-17T16:58:51.946 INFO:tasks.rados.rados.0.smithi063.stdout:966: oid 45 updating version 249 to 251 2024-08-17T16:58:51.946 INFO:tasks.rados.rados.0.smithi063.stdout:966: oid 45 version 251 is already newer than 250 2024-08-17T16:58:51.946 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 251 (ObjNum 397 snap 121 seq_num 397) dirty exists 2024-08-17T16:58:51.946 INFO:tasks.rados.rados.0.smithi063.stdout:966: left oid 45 (ObjNum 397 snap 121 seq_num 397) 2024-08-17T16:58:51.946 INFO:tasks.rados.rados.0.smithi063.stdout:966: done (5 left) 2024-08-17T16:58:51.946 INFO:tasks.rados.rados.0.smithi063.stdout:972: delete oid 49 current snap is 121 2024-08-17T16:58:51.946 INFO:tasks.rados.rados.0.smithi063.stdout:967: expect (ObjNum 311 snap 92 seq_num 311) 2024-08-17T16:58:52.082 INFO:tasks.rados.rados.0.smithi063.stdout:968: finishing write tid 1 to smithi06387500-36 2024-08-17T16:58:52.082 INFO:tasks.rados.rados.0.smithi063.stdout:971: finishing rollback tid 0 to smithi06387500-8 2024-08-17T16:58:52.082 INFO:tasks.rados.rados.0.smithi063.stdout:971: finishing rollback tid 1 to smithi06387500-8 2024-08-17T16:58:52.082 INFO:tasks.rados.rados.0.smithi063.stdout:971: finishing rollback tid 2 to smithi06387500-8 2024-08-17T16:58:52.082 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 269 (ObjNum 333 snap 101 seq_num 333) dirty exists 2024-08-17T16:58:52.082 INFO:tasks.rados.rados.0.smithi063.stdout:968: finishing write tid 2 to smithi06387500-36 2024-08-17T16:58:52.082 INFO:tasks.rados.rados.0.smithi063.stdout:968: finishing write tid 3 to smithi06387500-36 2024-08-17T16:58:52.082 INFO:tasks.rados.rados.0.smithi063.stdout:969: expect (ObjNum 394 snap 119 seq_num 394) 2024-08-17T16:58:52.167 INFO:tasks.rados.rados.0.smithi063.stdout:968: finishing write tid 4 to smithi06387500-36 2024-08-17T16:58:52.168 INFO:tasks.rados.rados.0.smithi063.stdout:968: oid 36 updating version 0 to 1038 2024-08-17T16:58:52.168 INFO:tasks.rados.rados.0.smithi063.stdout:968: oid 36 updating version 1038 to 1040 2024-08-17T16:58:52.168 INFO:tasks.rados.rados.0.smithi063.stdout:968: oid 36 version 1040 is already newer than 1039 2024-08-17T16:58:52.168 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 1040 (ObjNum 398 snap 121 seq_num 398) dirty exists 2024-08-17T16:58:52.168 INFO:tasks.rados.rados.0.smithi063.stdout:968: left oid 36 (ObjNum 398 snap 121 seq_num 398) 2024-08-17T16:58:52.168 INFO:tasks.rados.rados.0.smithi063.stdout:967: done (5 left) 2024-08-17T16:58:52.168 INFO:tasks.rados.rados.0.smithi063.stdout:968: done (4 left) 2024-08-17T16:58:52.168 INFO:tasks.rados.rados.0.smithi063.stdout:969: done (3 left) 2024-08-17T16:58:52.168 INFO:tasks.rados.rados.0.smithi063.stdout:970: done (2 left) 2024-08-17T16:58:52.168 INFO:tasks.rados.rados.0.smithi063.stdout:971: done (1 left) 2024-08-17T16:58:52.168 INFO:tasks.rados.rados.0.smithi063.stdout:972: done (0 left) 2024-08-17T16:58:52.168 INFO:tasks.rados.rados.0.smithi063.stdout:973: read oid 32 snap -1 2024-08-17T16:58:52.169 INFO:tasks.rados.rados.0.smithi063.stdout:973: expect (ObjNum 307 snap 92 seq_num 307) 2024-08-17T16:58:52.169 INFO:tasks.rados.rados.0.smithi063.stdout:974: rollback oid 16 current snap is 121 2024-08-17T16:58:52.169 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 16 to 120 2024-08-17T16:58:52.169 INFO:tasks.rados.rados.0.smithi063.stdout:975: snap_remove snap 117 2024-08-17T16:58:52.207 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:51 smithi162 ceph-mon[34633]: from='client.25004 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:52.207 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:51 smithi162 ceph-mon[34633]: from='client.25016 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:52.207 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:51 smithi162 ceph-mon[34633]: pgmap v156: 89 pgs: 89 active+clean; 152 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 7 op/s 2024-08-17T16:58:52.207 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:51 smithi162 ceph-mon[34633]: osdmap e316: 8 total, 8 up, 8 in 2024-08-17T16:58:52.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:51 smithi063 ceph-mon[42680]: from='client.25004 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:52.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:51 smithi063 ceph-mon[42680]: from='client.25016 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:52.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:51 smithi063 ceph-mon[42680]: pgmap v156: 89 pgs: 89 active+clean; 152 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 7 op/s 2024-08-17T16:58:52.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:51 smithi063 ceph-mon[42680]: osdmap e316: 8 total, 8 up, 8 in 2024-08-17T16:58:52.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:51 smithi063 ceph-mon[31063]: from='client.25004 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:52.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:51 smithi063 ceph-mon[31063]: from='client.25016 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:58:52.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:51 smithi063 ceph-mon[31063]: pgmap v156: 89 pgs: 89 active+clean; 152 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 7 op/s 2024-08-17T16:58:52.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:51 smithi063 ceph-mon[31063]: osdmap e316: 8 total, 8 up, 8 in 2024-08-17T16:58:52.923 INFO:tasks.rados.rados.0.smithi063.stdout:975: done (2 left) 2024-08-17T16:58:52.924 INFO:tasks.rados.rados.0.smithi063.stdout:976: rollback oid 35 current snap is 121 2024-08-17T16:58:52.924 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 35 to 119 2024-08-17T16:58:52.924 INFO:tasks.rados.rados.0.smithi063.stdout:974: finishing rollback tid 1 to smithi06387500-16 2024-08-17T16:58:52.924 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 0 (ObjNum 33 snap 0 seq_num 1786890032) dirty dne 2024-08-17T16:58:52.924 INFO:tasks.rados.rados.0.smithi063.stdout:973: expect (ObjNum 307 snap 92 seq_num 307) 2024-08-17T16:58:53.030 INFO:tasks.rados.rados.0.smithi063.stdout:976: finishing rollback tid 0 to smithi06387500-35 2024-08-17T16:58:53.030 INFO:tasks.rados.rados.0.smithi063.stdout:976: finishing rollback tid 1 to smithi06387500-35 2024-08-17T16:58:53.030 INFO:tasks.rados.rados.0.smithi063.stdout:976: finishing rollback tid 2 to smithi06387500-35 2024-08-17T16:58:53.031 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 1050 (ObjNum 367 snap 114 seq_num 367) dirty exists 2024-08-17T16:58:53.031 INFO:tasks.rados.rados.0.smithi063.stdout:973: done (2 left) 2024-08-17T16:58:53.031 INFO:tasks.rados.rados.0.smithi063.stdout:974: done (1 left) 2024-08-17T16:58:53.031 INFO:tasks.rados.rados.0.smithi063.stdout:976: done (0 left) 2024-08-17T16:58:53.031 INFO:tasks.rados.rados.0.smithi063.stdout:977: snap_remove snap 120 2024-08-17T16:58:53.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:58:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:58:53] "GET /metrics HTTP/1.1" 200 36211 "" "Prometheus/2.51.0" 2024-08-17T16:58:53.928 INFO:tasks.rados.rados.0.smithi063.stdout:977: done (0 left) 2024-08-17T16:58:53.928 INFO:tasks.rados.rados.0.smithi063.stdout:978: snap_remove snap 106 2024-08-17T16:58:54.277 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:53 smithi162 ceph-mon[34633]: pgmap v158: 89 pgs: 89 active+clean; 152 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 1019 B/s rd, 1.2 MiB/s wr, 7 op/s 2024-08-17T16:58:54.277 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:53 smithi162 ceph-mon[34633]: osdmap e317: 8 total, 8 up, 8 in 2024-08-17T16:58:54.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:53 smithi063 ceph-mon[42680]: pgmap v158: 89 pgs: 89 active+clean; 152 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 1019 B/s rd, 1.2 MiB/s wr, 7 op/s 2024-08-17T16:58:54.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:53 smithi063 ceph-mon[42680]: osdmap e317: 8 total, 8 up, 8 in 2024-08-17T16:58:54.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:53 smithi063 ceph-mon[31063]: pgmap v158: 89 pgs: 89 active+clean; 152 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 1019 B/s rd, 1.2 MiB/s wr, 7 op/s 2024-08-17T16:58:54.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:53 smithi063 ceph-mon[31063]: osdmap e317: 8 total, 8 up, 8 in 2024-08-17T16:58:54.844 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:54 smithi162 systemd[1]: Stopping Ceph grafana.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:58:54.884 INFO:tasks.rados.rados.0.smithi063.stdout:978: done (0 left) 2024-08-17T16:58:54.884 INFO:tasks.rados.rados.0.smithi063.stdout:979: delete oid 6 current snap is 121 2024-08-17T16:58:54.891 INFO:tasks.rados.rados.0.smithi063.stdout:979: done (0 left) 2024-08-17T16:58:54.892 INFO:tasks.rados.rados.0.smithi063.stdout:980: snap_remove snap 119 2024-08-17T16:58:55.095 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:54 smithi162 podman[94960]: 2024-08-17 16:58:54.843400283 +0000 UTC m=+0.128305893 container died fc8d623290125c18c7b1c27ec7151f45858bdfc601b198fc4760e73d9df95ac0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a, maintainer=Paul Cuzner , architecture=x86_64, version=8.5, name=ubi8, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.openshift.expose-services=) 2024-08-17T16:58:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:54 smithi162 ceph-mon[34633]: osdmap e318: 8 total, 8 up, 8 in 2024-08-17T16:58:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:54 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:54 smithi162 ceph-mon[34633]: osdmap e319: 8 total, 8 up, 8 in 2024-08-17T16:58:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:54 smithi162 ceph-mon[34633]: pgmap v162: 89 pgs: 89 active+clean; 152 MiB data, 600 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:58:55.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:54 smithi063 ceph-mon[31063]: osdmap e318: 8 total, 8 up, 8 in 2024-08-17T16:58:55.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:54 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:55.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:54 smithi063 ceph-mon[31063]: osdmap e319: 8 total, 8 up, 8 in 2024-08-17T16:58:55.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:54 smithi063 ceph-mon[31063]: pgmap v162: 89 pgs: 89 active+clean; 152 MiB data, 600 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:58:55.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:54 smithi063 ceph-mon[42680]: osdmap e318: 8 total, 8 up, 8 in 2024-08-17T16:58:55.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:54 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:58:55.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:54 smithi063 ceph-mon[42680]: osdmap e319: 8 total, 8 up, 8 in 2024-08-17T16:58:55.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:54 smithi063 ceph-mon[42680]: pgmap v162: 89 pgs: 89 active+clean; 152 MiB data, 600 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:58:55.465 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:55 smithi162 podman[94960]: 2024-08-17 16:58:55.189060188 +0000 UTC m=+0.473965797 container cleanup fc8d623290125c18c7b1c27ec7151f45858bdfc601b198fc4760e73d9df95ac0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-type=git, distribution-scope=public, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, maintainer=Paul Cuzner , architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container) 2024-08-17T16:58:55.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:55 smithi162 bash[94960]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a 2024-08-17T16:58:55.827 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:55 smithi162 podman[94977]: 2024-08-17 16:58:55.46518259 +0000 UTC m=+0.624254873 container remove fc8d623290125c18c7b1c27ec7151f45858bdfc601b198fc4760e73d9df95ac0 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , io.buildah.version=1.32.0, io.openshift.expose-services=, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, distribution-scope=public, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8) 2024-08-17T16:58:55.827 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:55 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-08-17T16:58:55.827 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:55 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-08-17T16:58:55.828 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:55 smithi162 systemd[1]: Stopped Ceph grafana.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:58:55.828 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:55 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@grafana.a.service: Consumed 7.737s CPU time. 2024-08-17T16:58:55.828 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:55 smithi162 systemd[1]: Starting Ceph grafana.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:58:55.902 INFO:tasks.rados.rados.0.smithi063.stdout:980: done (0 left) 2024-08-17T16:58:55.902 INFO:tasks.rados.rados.0.smithi063.stdout:981: snap_remove snap 107 2024-08-17T16:58:56.095 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:56 smithi162 podman[95110]: 2024-08-17 16:58:55.948967909 +0000 UTC m=+0.018518549 image pull c8b91775d855b99270fc5d22f3c6737e8cca01ef4c25c8b0362295e0746fa39b quay.io/ceph/grafana:10.4.0 2024-08-17T16:58:56.845 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:56 smithi162 podman[95110]: 2024-08-17 16:58:56.581780271 +0000 UTC m=+0.651330911 container create 581e03e654a8304f438cf1c39e6c70f9e974c58e0d58d96d1812b8d84f6c30e3 (image=quay.io/ceph/grafana:10.4.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-17T16:58:56.902 INFO:tasks.rados.rados.0.smithi063.stdout:981: done (0 left) 2024-08-17T16:58:56.902 INFO:tasks.rados.rados.0.smithi063.stdout:982: rollback oid 30 current snap is 121 2024-08-17T16:58:56.902 INFO:tasks.rados.rados.0.smithi063.stdout:982: done (0 left) 2024-08-17T16:58:56.902 INFO:tasks.rados.rados.0.smithi063.stdout:983: read oid 17 snap -1 2024-08-17T16:58:56.902 INFO:tasks.rados.rados.0.smithi063.stdout:983: expect (ObjNum 392 snap 119 seq_num 392) 2024-08-17T16:58:56.902 INFO:tasks.rados.rados.0.smithi063.stdout:984: rollback oid 21 current snap is 121 2024-08-17T16:58:56.902 INFO:tasks.rados.rados.0.smithi063.stdout:984: done (1 left) 2024-08-17T16:58:56.902 INFO:tasks.rados.rados.0.smithi063.stdout:985: read oid 22 snap -1 2024-08-17T16:58:56.903 INFO:tasks.rados.rados.0.smithi063.stdout:985: expect deleted 2024-08-17T16:58:56.903 INFO:tasks.rados.rados.0.smithi063.stdout:986: read oid 47 snap -1 2024-08-17T16:58:56.903 INFO:tasks.rados.rados.0.smithi063.stdout:986: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:58:56.903 INFO:tasks.rados.rados.0.smithi063.stdout:987: rollback oid 27 current snap is 121 2024-08-17T16:58:56.903 INFO:tasks.rados.rados.0.smithi063.stdout:987: done (3 left) 2024-08-17T16:58:56.903 INFO:tasks.rados.rados.0.smithi063.stdout:988: write oid 32 current snap is 121 2024-08-17T16:58:56.903 INFO:tasks.rados.rados.0.smithi063.stdout:988: seq_num 401 ranges {739607=625205,1982691=499457} 2024-08-17T16:58:56.911 INFO:tasks.rados.rados.0.smithi063.stdout:988: writing smithi06387500-32 from 739607 to 1364812 tid 1 2024-08-17T16:58:56.919 INFO:tasks.rados.rados.0.smithi063.stdout:988: writing smithi06387500-32 from 1982691 to 2482148 tid 2 2024-08-17T16:58:56.919 INFO:tasks.rados.rados.0.smithi063.stdout:989: snap_create 2024-08-17T16:58:56.919 INFO:tasks.rados.rados.0.smithi063.stdout:986: expect (ObjNum 79 snap 10 seq_num 79) 2024-08-17T16:58:57.075 INFO:tasks.rados.rados.0.smithi063.stdout:983: expect (ObjNum 392 snap 119 seq_num 392) 2024-08-17T16:58:57.158 INFO:tasks.rados.rados.0.smithi063.stdout:988: finishing write tid 1 to smithi06387500-32 2024-08-17T16:58:57.158 INFO:tasks.rados.rados.0.smithi063.stdout:988: finishing write tid 2 to smithi06387500-32 2024-08-17T16:58:57.159 INFO:tasks.rados.rados.0.smithi063.stdout:988: finishing write tid 3 to smithi06387500-32 2024-08-17T16:58:57.159 INFO:tasks.rados.rados.0.smithi063.stdout:988: finishing write tid 4 to smithi06387500-32 2024-08-17T16:58:57.159 INFO:tasks.rados.rados.0.smithi063.stdout:988: oid 32 updating version 0 to 188 2024-08-17T16:58:57.159 INFO:tasks.rados.rados.0.smithi063.stdout:988: oid 32 version 188 is already newer than 187 2024-08-17T16:58:57.159 INFO:tasks.rados.rados.0.smithi063.stdout:988: oid 32 updating version 188 to 189 2024-08-17T16:58:57.159 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 189 (ObjNum 401 snap 121 seq_num 401) dirty exists 2024-08-17T16:58:57.159 INFO:tasks.rados.rados.0.smithi063.stdout:988: left oid 32 (ObjNum 401 snap 121 seq_num 401) 2024-08-17T16:58:57.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:56 smithi063 ceph-mon[31063]: osdmap e320: 8 total, 8 up, 8 in 2024-08-17T16:58:57.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:56 smithi063 ceph-mon[42680]: osdmap e320: 8 total, 8 up, 8 in 2024-08-17T16:58:57.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:56 smithi162 ceph-mon[34633]: osdmap e320: 8 total, 8 up, 8 in 2024-08-17T16:58:57.917 INFO:tasks.rados.rados.0.smithi063.stdout:983: done (4 left) 2024-08-17T16:58:57.917 INFO:tasks.rados.rados.0.smithi063.stdout:985: done (3 left) 2024-08-17T16:58:57.917 INFO:tasks.rados.rados.0.smithi063.stdout:986: done (2 left) 2024-08-17T16:58:57.917 INFO:tasks.rados.rados.0.smithi063.stdout:988: done (1 left) 2024-08-17T16:58:57.917 INFO:tasks.rados.rados.0.smithi063.stdout:989: done (0 left) 2024-08-17T16:58:57.917 INFO:tasks.rados.rados.0.smithi063.stdout:990: read oid 44 snap 121 2024-08-17T16:58:57.917 INFO:tasks.rados.rados.0.smithi063.stdout:990: expect (ObjNum 325 snap 99 seq_num 325) 2024-08-17T16:58:57.918 INFO:tasks.rados.rados.0.smithi063.stdout:991: rollback oid 49 current snap is 122 2024-08-17T16:58:57.918 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 49 to 121 2024-08-17T16:58:57.918 INFO:tasks.rados.rados.0.smithi063.stdout:992: write oid 5 current snap is 122 2024-08-17T16:58:57.918 INFO:tasks.rados.rados.0.smithi063.stdout:992: seq_num 402 ranges {756414=700454,2041173=715241} 2024-08-17T16:58:57.925 INFO:tasks.rados.rados.0.smithi063.stdout:992: writing smithi06387500-5 from 756414 to 1456868 tid 1 2024-08-17T16:58:57.933 INFO:tasks.rados.rados.0.smithi063.stdout:992: writing smithi06387500-5 from 2041173 to 2756414 tid 2 2024-08-17T16:58:57.933 INFO:tasks.rados.rados.0.smithi063.stdout:993: read oid 13 snap -1 2024-08-17T16:58:57.933 INFO:tasks.rados.rados.0.smithi063.stdout:993: expect deleted 2024-08-17T16:58:57.933 INFO:tasks.rados.rados.0.smithi063.stdout:994: write oid 34 current snap is 122 2024-08-17T16:58:57.933 INFO:tasks.rados.rados.0.smithi063.stdout:991: finishing rollback tid 1 to smithi06387500-49 2024-08-17T16:58:57.933 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 0 (ObjNum 369 snap 0 seq_num 1806785216) dirty dne 2024-08-17T16:58:57.933 INFO:tasks.rados.rados.0.smithi063.stdout:994: seq_num 403 ranges {773221=492056,1816008=503126} 2024-08-17T16:58:57.940 INFO:tasks.rados.rados.0.smithi063.stdout:994: writing smithi06387500-34 from 773221 to 1265277 tid 1 2024-08-17T16:58:57.946 INFO:tasks.rados.rados.0.smithi063.stdout:994: writing smithi06387500-34 from 1816008 to 2319134 tid 2 2024-08-17T16:58:57.946 INFO:tasks.rados.rados.0.smithi063.stdout:991: done (4 left) 2024-08-17T16:58:57.946 INFO:tasks.rados.rados.0.smithi063.stdout:995: read oid 50 snap 121 2024-08-17T16:58:57.946 INFO:tasks.rados.rados.0.smithi063.stdout:995: expect deleted 2024-08-17T16:58:57.946 INFO:tasks.rados.rados.0.smithi063.stdout:996: snap_create 2024-08-17T16:58:57.946 INFO:tasks.rados.rados.0.smithi063.stdout:992: finishing write tid 1 to smithi06387500-5 2024-08-17T16:58:57.946 INFO:tasks.rados.rados.0.smithi063.stdout:990: expect (ObjNum 325 snap 99 seq_num 325) 2024-08-17T16:58:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:57 smithi162 ceph-mon[34633]: osdmap e321: 8 total, 8 up, 8 in 2024-08-17T16:58:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:57 smithi162 ceph-mon[34633]: pgmap v165: 89 pgs: 89 active+clean; 157 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-08-17T16:58:58.171 INFO:tasks.rados.rados.0.smithi063.stdout:992: finishing write tid 2 to smithi06387500-5 2024-08-17T16:58:58.171 INFO:tasks.rados.rados.0.smithi063.stdout:992: finishing write tid 3 to smithi06387500-5 2024-08-17T16:58:58.171 INFO:tasks.rados.rados.0.smithi063.stdout:992: finishing write tid 4 to smithi06387500-5 2024-08-17T16:58:58.171 INFO:tasks.rados.rados.0.smithi063.stdout:992: oid 5 updating version 0 to 502 2024-08-17T16:58:58.171 INFO:tasks.rados.rados.0.smithi063.stdout:992: oid 5 updating version 502 to 504 2024-08-17T16:58:58.171 INFO:tasks.rados.rados.0.smithi063.stdout:992: oid 5 version 504 is already newer than 503 2024-08-17T16:58:58.171 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 504 (ObjNum 402 snap 122 seq_num 402) dirty exists 2024-08-17T16:58:58.171 INFO:tasks.rados.rados.0.smithi063.stdout:992: left oid 5 (ObjNum 402 snap 122 seq_num 402) 2024-08-17T16:58:58.171 INFO:tasks.rados.rados.0.smithi063.stdout:994: finishing write tid 1 to smithi06387500-34 2024-08-17T16:58:58.171 INFO:tasks.rados.rados.0.smithi063.stdout:994: finishing write tid 2 to smithi06387500-34 2024-08-17T16:58:58.171 INFO:tasks.rados.rados.0.smithi063.stdout:994: finishing write tid 3 to smithi06387500-34 2024-08-17T16:58:58.171 INFO:tasks.rados.rados.0.smithi063.stdout:994: finishing write tid 4 to smithi06387500-34 2024-08-17T16:58:58.171 INFO:tasks.rados.rados.0.smithi063.stdout:994: oid 34 updating version 0 to 735 2024-08-17T16:58:58.172 INFO:tasks.rados.rados.0.smithi063.stdout:994: oid 34 updating version 735 to 736 2024-08-17T16:58:58.172 INFO:tasks.rados.rados.0.smithi063.stdout:994: oid 34 updating version 736 to 737 2024-08-17T16:58:58.172 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 737 (ObjNum 403 snap 122 seq_num 403) dirty exists 2024-08-17T16:58:58.172 INFO:tasks.rados.rados.0.smithi063.stdout:994: left oid 34 (ObjNum 403 snap 122 seq_num 403) 2024-08-17T16:58:58.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:57 smithi063 ceph-mon[31063]: osdmap e321: 8 total, 8 up, 8 in 2024-08-17T16:58:58.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:57 smithi063 ceph-mon[31063]: pgmap v165: 89 pgs: 89 active+clean; 157 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-08-17T16:58:58.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:57 smithi063 ceph-mon[42680]: osdmap e321: 8 total, 8 up, 8 in 2024-08-17T16:58:58.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:57 smithi063 ceph-mon[42680]: pgmap v165: 89 pgs: 89 active+clean; 157 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-08-17T16:58:58.830 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 podman[95110]: 2024-08-17 16:58:58.578645824 +0000 UTC m=+2.648196472 container init 581e03e654a8304f438cf1c39e6c70f9e974c58e0d58d96d1812b8d84f6c30e3 (image=quay.io/ceph/grafana:10.4.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-17T16:58:58.830 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 podman[95110]: 2024-08-17 16:58:58.583514796 +0000 UTC m=+2.653065434 container start 581e03e654a8304f438cf1c39e6c70f9e974c58e0d58d96d1812b8d84f6c30e3 (image=quay.io/ceph/grafana:10.4.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-17T16:58:58.830 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708069697Z level=info msg="Starting Grafana" version=10.4.0 commit=03f502a94d17f7dc4e6c34acdf8428aedd986e4c branch=HEAD compiled=2024-08-17T16:58:58Z 2024-08-17T16:58:58.830 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708278424Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-08-17T16:58:58.830 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.70828557Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-08-17T16:58:58.830 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708288907Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-08-17T16:58:58.830 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708291823Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-08-17T16:58:58.830 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.70829433Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-08-17T16:58:58.830 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.70829684Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-08-17T16:58:58.831 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708299336Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-08-17T16:58:58.831 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708302004Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-08-17T16:58:58.831 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708304512Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-08-17T16:58:58.831 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708306851Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-08-17T16:58:58.831 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708309231Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-08-17T16:58:58.831 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708312603Z level=info msg=Target target=[all] 2024-08-17T16:58:58.831 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708318396Z level=info msg="Path Home" path=/usr/share/grafana 2024-08-17T16:58:58.831 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708320863Z level=info msg="Path Data" path=/var/lib/grafana 2024-08-17T16:58:58.832 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708323656Z level=info msg="Path Logs" path=/var/log/grafana 2024-08-17T16:58:58.832 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708326179Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-08-17T16:58:58.832 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708328652Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-08-17T16:58:58.832 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=settings t=2024-08-17T16:58:58.708331246Z level=info msg="App mode production" 2024-08-17T16:58:58.832 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=sqlstore t=2024-08-17T16:58:58.708570809Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-08-17T16:58:58.832 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=sqlstore t=2024-08-17T16:58:58.70858639Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-08-17T16:58:58.832 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.709313343Z level=info msg="Starting DB migrations" 2024-08-17T16:58:58.833 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.721256219Z level=info msg="Executing migration" id="Add uid column to user" 2024-08-17T16:58:58.833 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.726832701Z level=info msg="Migration successfully executed" id="Add uid column to user" duration=5.575602ms 2024-08-17T16:58:58.833 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.727275055Z level=info msg="Executing migration" id="Update uid column values for users" 2024-08-17T16:58:58.833 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.727407082Z level=info msg="Migration successfully executed" id="Update uid column values for users" duration=129.008µs 2024-08-17T16:58:58.833 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.727800837Z level=info msg="Executing migration" id="Add unique index user_uid" 2024-08-17T16:58:58.833 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.728530521Z level=info msg="Migration successfully executed" id="Add unique index user_uid" duration=730.164µs 2024-08-17T16:58:58.833 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.729041705Z level=info msg="Executing migration" id="Add playlist column created_at" 2024-08-17T16:58:58.833 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.734174969Z level=info msg="Migration successfully executed" id="Add playlist column created_at" duration=5.132467ms 2024-08-17T16:58:58.833 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.734699477Z level=info msg="Executing migration" id="Add playlist column updated_at" 2024-08-17T16:58:58.833 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.739915719Z level=info msg="Migration successfully executed" id="Add playlist column updated_at" duration=5.215905ms 2024-08-17T16:58:58.833 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.74036081Z level=info msg="Executing migration" id="Add column uid in team" 2024-08-17T16:58:58.833 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.745318971Z level=info msg="Migration successfully executed" id="Add column uid in team" duration=4.95751ms 2024-08-17T16:58:58.834 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.745684244Z level=info msg="Executing migration" id="Update uid column values in team" 2024-08-17T16:58:58.834 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.745799098Z level=info msg="Migration successfully executed" id="Update uid column values in team" duration=115.184µs 2024-08-17T16:58:58.834 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.746562684Z level=info msg="Executing migration" id="Add unique index team_org_id_uid" 2024-08-17T16:58:58.834 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.747307534Z level=info msg="Migration successfully executed" id="Add unique index team_org_id_uid" duration=744.502µs 2024-08-17T16:58:58.834 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.747714346Z level=info msg="Executing migration" id="add index user_auth_token.revoked_at" 2024-08-17T16:58:58.834 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.74842077Z level=info msg="Migration successfully executed" id="add index user_auth_token.revoked_at" duration=705.921µs 2024-08-17T16:58:58.834 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.748817662Z level=info msg="Executing migration" id="add result_fingerprint column to alert_instance" 2024-08-17T16:58:58.834 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.754148466Z level=info msg="Migration successfully executed" id="add result_fingerprint column to alert_instance" duration=5.329265ms 2024-08-17T16:58:58.834 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.754551535Z level=info msg="Executing migration" id="add last_applied column to alert_configuration_history" 2024-08-17T16:58:58.834 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.75944263Z level=info msg="Migration successfully executed" id="add last_applied column to alert_configuration_history" duration=4.890403ms 2024-08-17T16:58:58.834 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.759812394Z level=info msg="Executing migration" id="alter library_element model to mediumtext" 2024-08-17T16:58:58.834 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.759850576Z level=info msg="Migration successfully executed" id="alter library_element model to mediumtext" duration=38.059µs 2024-08-17T16:58:58.835 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.760727671Z level=info msg="Executing migration" id="permission kind migration" 2024-08-17T16:58:58.835 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.765600136Z level=info msg="Migration successfully executed" id="permission kind migration" duration=4.871637ms 2024-08-17T16:58:58.835 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.765965982Z level=info msg="Executing migration" id="permission attribute migration" 2024-08-17T16:58:58.835 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.770932296Z level=info msg="Migration successfully executed" id="permission attribute migration" duration=4.96616ms 2024-08-17T16:58:58.835 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.771297182Z level=info msg="Executing migration" id="permission identifier migration" 2024-08-17T16:58:58.835 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.776596582Z level=info msg="Migration successfully executed" id="permission identifier migration" duration=5.299261ms 2024-08-17T16:58:58.835 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.776966573Z level=info msg="Executing migration" id="add permission identifier index" 2024-08-17T16:58:58.835 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.777728214Z level=info msg="Migration successfully executed" id="add permission identifier index" duration=761.32µs 2024-08-17T16:58:58.835 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.778101252Z level=info msg="Executing migration" id="add permission action scope role_id index" 2024-08-17T16:58:58.835 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.778878699Z level=info msg="Migration successfully executed" id="add permission action scope role_id index" duration=777.651µs 2024-08-17T16:58:58.835 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.779227106Z level=info msg="Executing migration" id="remove permission role_id action scope index" 2024-08-17T16:58:58.835 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.779926988Z level=info msg="Migration successfully executed" id="remove permission role_id action scope index" duration=698.775µs 2024-08-17T16:58:58.836 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.78026793Z level=info msg="Executing migration" id="rbac disabled migrator" 2024-08-17T16:58:58.836 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.780286536Z level=info msg="Migration successfully executed" id="rbac disabled migrator" duration=18.95µs 2024-08-17T16:58:58.836 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.7811344Z level=info msg="Executing migration" id="drop index IDX_correlation_uid - v1" 2024-08-17T16:58:58.836 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.781802948Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_uid - v1" duration=668.182µs 2024-08-17T16:58:58.836 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.78215486Z level=info msg="Executing migration" id="drop index IDX_correlation_source_uid - v1" 2024-08-17T16:58:58.836 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.782822713Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_source_uid - v1" duration=667.615µs 2024-08-17T16:58:58.836 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.783172018Z level=info msg="Executing migration" id="Rename table correlation to correlation_tmp_qwerty - v1" 2024-08-17T16:58:58.836 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.798092966Z level=info msg="Migration successfully executed" id="Rename table correlation to correlation_tmp_qwerty - v1" duration=14.920442ms 2024-08-17T16:58:58.836 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.798513142Z level=info msg="Executing migration" id="create correlation v2" 2024-08-17T16:58:58.836 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.799212441Z level=info msg="Migration successfully executed" id="create correlation v2" duration=698.835µs 2024-08-17T16:58:58.836 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.799582324Z level=info msg="Executing migration" id="create index IDX_correlation_uid - v2" 2024-08-17T16:58:58.836 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.800282136Z level=info msg="Migration successfully executed" id="create index IDX_correlation_uid - v2" duration=699.672µs 2024-08-17T16:58:58.837 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.800642419Z level=info msg="Executing migration" id="create index IDX_correlation_source_uid - v2" 2024-08-17T16:58:58.837 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.801322067Z level=info msg="Migration successfully executed" id="create index IDX_correlation_source_uid - v2" duration=679.357µs 2024-08-17T16:58:58.837 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.803662925Z level=info msg="Executing migration" id="create index IDX_correlation_org_id - v2" 2024-08-17T16:58:58.837 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.804457088Z level=info msg="Migration successfully executed" id="create index IDX_correlation_org_id - v2" duration=798.988µs 2024-08-17T16:58:58.837 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.806089406Z level=info msg="Executing migration" id="copy correlation v1 to v2" 2024-08-17T16:58:58.837 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.806256625Z level=info msg="Migration successfully executed" id="copy correlation v1 to v2" duration=167.586µs 2024-08-17T16:58:58.837 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.809024516Z level=info msg="Executing migration" id="drop correlation_tmp_qwerty" 2024-08-17T16:58:58.837 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.809579986Z level=info msg="Migration successfully executed" id="drop correlation_tmp_qwerty" duration=555.312µs 2024-08-17T16:58:58.837 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.811853162Z level=info msg="Executing migration" id="add provisioning column" 2024-08-17T16:58:58.837 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.816973237Z level=info msg="Migration successfully executed" id="add provisioning column" duration=5.11973ms 2024-08-17T16:58:58.837 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.817357416Z level=info msg="Executing migration" id="backfill empty share column fields with default of public" 2024-08-17T16:58:58.837 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.817489895Z level=info msg="Migration successfully executed" id="backfill empty share column fields with default of public" duration=132.157µs 2024-08-17T16:58:58.837 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.820087068Z level=info msg="Executing migration" id="migrate contents column to mediumblob for MySQL" 2024-08-17T16:58:58.837 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.820142354Z level=info msg="Migration successfully executed" id="migrate contents column to mediumblob for MySQL" duration=55.09µs 2024-08-17T16:58:58.838 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.823151002Z level=info msg="Executing migration" id="add origin column to seed_assignment" 2024-08-17T16:58:58.838 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.828573455Z level=info msg="Migration successfully executed" id="add origin column to seed_assignment" duration=5.4215ms 2024-08-17T16:58:58.838 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.828935712Z level=info msg="Executing migration" id="add origin to plugin seed_assignment" 2024-08-17T16:58:58.838 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.829146605Z level=info msg="Migration successfully executed" id="add origin to plugin seed_assignment" duration=210.882µs 2024-08-17T16:58:58.838 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.829753988Z level=info msg="Executing migration" id="prevent seeding OnCall access" 2024-08-17T16:58:58.838 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.82987384Z level=info msg="Migration successfully executed" id="prevent seeding OnCall access" duration=120.038µs 2024-08-17T16:58:58.838 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.830637227Z level=info msg="Executing migration" id="managed folder permissions library panel actions migration" 2024-08-17T16:58:58.838 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.831602533Z level=info msg="Migration successfully executed" id="managed folder permissions library panel actions migration" duration=965.241µs 2024-08-17T16:58:58.922 INFO:tasks.rados.rados.0.smithi063.stdout:990: done (5 left) 2024-08-17T16:58:58.922 INFO:tasks.rados.rados.0.smithi063.stdout:992: done (4 left) 2024-08-17T16:58:58.922 INFO:tasks.rados.rados.0.smithi063.stdout:993: done (3 left) 2024-08-17T16:58:58.922 INFO:tasks.rados.rados.0.smithi063.stdout:994: done (2 left) 2024-08-17T16:58:58.922 INFO:tasks.rados.rados.0.smithi063.stdout:995: done (1 left) 2024-08-17T16:58:58.922 INFO:tasks.rados.rados.0.smithi063.stdout:996: done (0 left) 2024-08-17T16:58:58.922 INFO:tasks.rados.rados.0.smithi063.stdout:997: snap_create 2024-08-17T16:58:59.041 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-mon[34633]: osdmap e322: 8 total, 8 up, 8 in 2024-08-17T16:58:59.041 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.831949783Z level=info msg="Executing migration" id="Remove unique index for folder.title and folder.parent_uid" 2024-08-17T16:58:59.041 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.832625789Z level=info msg="Migration successfully executed" id="Remove unique index for folder.title and folder.parent_uid" duration=674.163µs 2024-08-17T16:58:59.041 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.832962373Z level=info msg="Executing migration" id="Add unique index for title, parent_uid, and org_id" 2024-08-17T16:58:59.042 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.833706908Z level=info msg="Migration successfully executed" id="Add unique index for title, parent_uid, and org_id" duration=744.149µs 2024-08-17T16:58:59.042 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.834058275Z level=info msg="Executing migration" id="Sync dashboard and folder table" 2024-08-17T16:58:59.042 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.834513382Z level=info msg="Migration successfully executed" id="Sync dashboard and folder table" duration=454.897µs 2024-08-17T16:58:59.042 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.834915125Z level=info msg="Executing migration" id="Remove ghost folders from the folder table" 2024-08-17T16:58:59.042 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.835106624Z level=info msg="Migration successfully executed" id="Remove ghost folders from the folder table" duration=191.419µs 2024-08-17T16:58:59.042 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.835797204Z level=info msg="Executing migration" id="Remove unique index UQE_folder_uid_org_id" 2024-08-17T16:58:59.042 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.836469189Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_uid_org_id" duration=671.897µs 2024-08-17T16:58:59.042 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.836796983Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_uid" 2024-08-17T16:58:59.042 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.837485842Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_uid" duration=688.523µs 2024-08-17T16:58:59.042 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.837821367Z level=info msg="Executing migration" id="Remove unique index UQE_folder_title_parent_uid_org_id" 2024-08-17T16:58:59.042 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.838490655Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_title_parent_uid_org_id" duration=669.116µs 2024-08-17T16:58:59.043 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.838827614Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_parent_uid_title" 2024-08-17T16:58:59.043 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.839520999Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_parent_uid_title" duration=693.054µs 2024-08-17T16:58:59.043 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.839867973Z level=info msg="Executing migration" id="Remove index IDX_folder_parent_uid_org_id" 2024-08-17T16:58:59.043 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.840526059Z level=info msg="Migration successfully executed" id="Remove index IDX_folder_parent_uid_org_id" duration=669.52µs 2024-08-17T16:58:59.043 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.840877887Z level=info msg="Executing migration" id="create anon_device table" 2024-08-17T16:58:59.043 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.841424633Z level=info msg="Migration successfully executed" id="create anon_device table" duration=546.433µs 2024-08-17T16:58:59.043 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.841763763Z level=info msg="Executing migration" id="add unique index anon_device.device_id" 2024-08-17T16:58:59.043 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.842462675Z level=info msg="Migration successfully executed" id="add unique index anon_device.device_id" duration=698.8µs 2024-08-17T16:58:59.043 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.842815218Z level=info msg="Executing migration" id="add index anon_device.updated_at" 2024-08-17T16:58:59.043 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.843648443Z level=info msg="Migration successfully executed" id="add index anon_device.updated_at" duration=832.914µs 2024-08-17T16:58:59.043 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.844038616Z level=info msg="Executing migration" id="create signing_key table" 2024-08-17T16:58:59.043 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.84460046Z level=info msg="Migration successfully executed" id="create signing_key table" duration=561.654µs 2024-08-17T16:58:59.043 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.844977052Z level=info msg="Executing migration" id="add unique index signing_key.key_id" 2024-08-17T16:58:59.043 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.845645208Z level=info msg="Migration successfully executed" id="add unique index signing_key.key_id" duration=666.869µs 2024-08-17T16:58:59.044 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.84602852Z level=info msg="Executing migration" id="set legacy alert migration status in kvstore" 2024-08-17T16:58:59.044 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.846660705Z level=info msg="Migration successfully executed" id="set legacy alert migration status in kvstore" duration=631.24µs 2024-08-17T16:58:59.044 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.846985235Z level=info msg="Executing migration" id="migrate record of created folders during legacy migration to kvstore" 2024-08-17T16:58:59.044 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.847386685Z level=info msg="Migration successfully executed" id="migrate record of created folders during legacy migration to kvstore" duration=401.992µs 2024-08-17T16:58:59.044 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.847876507Z level=info msg="Executing migration" id="Add folder_uid for dashboard" 2024-08-17T16:58:59.044 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.853065244Z level=info msg="Migration successfully executed" id="Add folder_uid for dashboard" duration=5.18829ms 2024-08-17T16:58:59.044 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.853396971Z level=info msg="Executing migration" id="Populate dashboard folder_uid column" 2024-08-17T16:58:59.045 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.855182998Z level=info msg="Migration successfully executed" id="Populate dashboard folder_uid column" duration=1.786762ms 2024-08-17T16:58:59.045 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.855711487Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title" 2024-08-17T16:58:59.045 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.856588235Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title" duration=876.493µs 2024-08-17T16:58:59.045 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.856931573Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_id_title" 2024-08-17T16:58:59.045 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.857618381Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_id_title" duration=686.81µs 2024-08-17T16:58:59.045 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.857982335Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_uid_title" 2024-08-17T16:58:59.045 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.858649769Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_uid_title" duration=667.43µs 2024-08-17T16:58:59.045 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.860866999Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" 2024-08-17T16:58:59.045 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.861718572Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" duration=851.745µs 2024-08-17T16:58:59.045 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.863966397Z level=info msg="Executing migration" id="Restore index for dashboard_org_id_folder_id_title" 2024-08-17T16:58:59.045 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.864753513Z level=info msg="Migration successfully executed" id="Restore index for dashboard_org_id_folder_id_title" duration=789.074µs 2024-08-17T16:58:59.045 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.866653221Z level=info msg="Executing migration" id="create sso_setting table" 2024-08-17T16:58:59.046 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.867399958Z level=info msg="Migration successfully executed" id="create sso_setting table" duration=746.583µs 2024-08-17T16:58:59.046 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.869432797Z level=info msg="Executing migration" id="copy kvstore migration status to each org" 2024-08-17T16:58:59.046 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.870149877Z level=info msg="Migration successfully executed" id="copy kvstore migration status to each org" duration=716.318µs 2024-08-17T16:58:59.046 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.872212326Z level=info msg="Executing migration" id="add back entry for orgid=0 migrated status" 2024-08-17T16:58:59.046 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.872605302Z level=info msg="Migration successfully executed" id="add back entry for orgid=0 migrated status" duration=392.96µs 2024-08-17T16:58:59.046 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.874922667Z level=info msg="Executing migration" id="alter kv_store.value to longtext" 2024-08-17T16:58:59.046 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.874961889Z level=info msg="Migration successfully executed" id="alter kv_store.value to longtext" duration=39.593µs 2024-08-17T16:58:59.046 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.87783039Z level=info msg="Executing migration" id="add notification_settings column to alert_rule table" 2024-08-17T16:58:59.046 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.883208634Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule table" duration=5.380042ms 2024-08-17T16:58:59.046 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.883565846Z level=info msg="Executing migration" id="add notification_settings column to alert_rule_version table" 2024-08-17T16:58:59.046 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.888910459Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule_version table" duration=5.344342ms 2024-08-17T16:58:59.046 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.889248362Z level=info msg="Executing migration" id="removing scope from alert.instances:read action migration" 2024-08-17T16:58:59.046 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.889453158Z level=info msg="Migration successfully executed" id="removing scope from alert.instances:read action migration" duration=204.745µs 2024-08-17T16:58:59.047 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=migrator t=2024-08-17T16:58:58.892075749Z level=info msg="migrations completed" performed=65 skipped=482 duration=170.861533ms 2024-08-17T16:58:59.047 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=sqlstore t=2024-08-17T16:58:58.892767881Z level=info msg="Created default organization" 2024-08-17T16:58:59.047 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=secrets t=2024-08-17T16:58:58.894784362Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-08-17T16:58:59.047 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=plugin.store t=2024-08-17T16:58:58.909806951Z level=info msg="Loading plugins..." 2024-08-17T16:58:59.047 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=local.finder t=2024-08-17T16:58:58.956388541Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-08-17T16:58:59.047 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=plugin.store t=2024-08-17T16:58:58.956407619Z level=info msg="Plugins loaded" count=55 duration=46.601284ms 2024-08-17T16:58:59.047 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=query_data t=2024-08-17T16:58:58.958562517Z level=info msg="Query Service initialization" 2024-08-17T16:58:59.047 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=live.push_http t=2024-08-17T16:58:58.960989331Z level=info msg="Live Push Gateway initialization" 2024-08-17T16:58:59.047 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=ngalert.migration t=2024-08-17T16:58:58.962110365Z level=info msg=Starting 2024-08-17T16:58:59.047 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=ngalert.state.manager t=2024-08-17T16:58:58.967846868Z level=info msg="Running in alternative execution of Error/NoData mode" 2024-08-17T16:58:59.047 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=infra.usagestats.collector t=2024-08-17T16:58:58.96923994Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-08-17T16:58:59.047 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=provisioning.datasources t=2024-08-17T16:58:58.971753023Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-08-17T16:58:59.048 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=provisioning.datasources t=2024-08-17T16:58:58.972025439Z level=info msg="inserting datasource from configuration" name=Dashboard1 uid=P43CA22E17D0F9596 2024-08-17T16:58:59.048 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=provisioning.alerting t=2024-08-17T16:58:59.002043299Z level=info msg="starting to provision alerting" 2024-08-17T16:58:59.048 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=provisioning.alerting t=2024-08-17T16:58:59.002055082Z level=info msg="finished to provision alerting" 2024-08-17T16:58:59.048 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=grafanaStorageLogger t=2024-08-17T16:58:59.002204641Z level=info msg="Storage starting" 2024-08-17T16:58:59.048 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=ngalert.state.manager t=2024-08-17T16:58:59.002419756Z level=info msg="Warming state cache for startup" 2024-08-17T16:58:59.048 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=provisioning.dashboard t=2024-08-17T16:58:59.004049094Z level=info msg="starting to provision dashboards" 2024-08-17T16:58:59.048 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=http.server t=2024-08-17T16:58:59.004845494Z level=info msg="HTTP Server TLS settings" MinTLSVersion=TLS1.2 configuredciphers=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA 2024-08-17T16:58:59.048 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=http.server t=2024-08-17T16:58:59.005158236Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-08-17T16:58:59.048 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=ngalert.multiorg.alertmanager t=2024-08-17T16:58:59.005242992Z level=info msg="Starting MultiOrg Alertmanager" 2024-08-17T16:58:59.048 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=ngalert.state.manager t=2024-08-17T16:58:59.03739278Z level=info msg="State cache has been initialized" states=0 duration=34.969815ms 2024-08-17T16:58:59.048 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=ngalert.scheduler t=2024-08-17T16:58:59.037423262Z level=info msg="Starting scheduler" tickInterval=10s maxAttempts=1 2024-08-17T16:58:59.048 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=ticker t=2024-08-17T16:58:59.037472368Z level=info msg=starting first_tick=2024-08-17T16:59:00Z 2024-08-17T16:58:59.048 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=sqlstore.transactions t=2024-08-17T16:58:59.03983985Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T16:58:59.086 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=grafana.update.checker t=2024-08-17T16:58:59.075609044Z level=info msg="Update check succeeded" duration=72.487444ms 2024-08-17T16:58:59.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:58 smithi063 ceph-mon[31063]: osdmap e322: 8 total, 8 up, 8 in 2024-08-17T16:58:59.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:58 smithi063 ceph-mon[42680]: osdmap e322: 8 total, 8 up, 8 in 2024-08-17T16:58:59.345 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=plugins.update.checker t=2024-08-17T16:58:59.086056347Z level=info msg="Update check succeeded" duration=83.463545ms 2024-08-17T16:58:59.345 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=sqlstore.transactions t=2024-08-17T16:58:59.102099499Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T16:58:59.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=provisioning.dashboard t=2024-08-17T16:58:59.190981858Z level=info msg="finished to provision dashboards" 2024-08-17T16:58:59.845 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=grafana-apiserver t=2024-08-17T16:58:59.425705993Z level=info msg="Adding GroupVersion playlist.grafana.app v0alpha1 to ResourceManager" 2024-08-17T16:58:59.846 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=grafana-apiserver t=2024-08-17T16:58:59.426112901Z level=info msg="Adding GroupVersion featuretoggle.grafana.app v0alpha1 to ResourceManager" 2024-08-17T16:58:59.846 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 bash[95110]: 581e03e654a8304f438cf1c39e6c70f9e974c58e0d58d96d1812b8d84f6c30e3 2024-08-17T16:58:59.846 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 16:58:59 smithi162 systemd[1]: Started Ceph grafana.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:58:59.883 INFO:tasks.rados.rados.0.smithi063.stdout:997: done (0 left) 2024-08-17T16:58:59.883 INFO:tasks.rados.rados.0.smithi063.stdout:998: rollback oid 43 current snap is 124 2024-08-17T16:58:59.883 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 43 to 122 2024-08-17T16:58:59.884 INFO:tasks.rados.rados.0.smithi063.stdout:999: rollback oid 11 current snap is 124 2024-08-17T16:58:59.884 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 11 to 123 2024-08-17T16:58:59.884 INFO:tasks.rados.rados.0.smithi063.stdout:1000: snap_create 2024-08-17T16:58:59.886 INFO:tasks.rados.rados.0.smithi063.stdout:999: finishing rollback tid 0 to smithi06387500-11 2024-08-17T16:58:59.886 INFO:tasks.rados.rados.0.smithi063.stdout:999: finishing rollback tid 1 to smithi06387500-11 2024-08-17T16:58:59.887 INFO:tasks.rados.rados.0.smithi063.stdout:998: finishing rollback tid 1 to smithi06387500-43 2024-08-17T16:58:59.887 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 0 (ObjNum 257 snap 0 seq_num 1806890640) dirty dne 2024-08-17T16:58:59.887 INFO:tasks.rados.rados.0.smithi063.stdout:999: finishing rollback tid 2 to smithi06387500-11 2024-08-17T16:58:59.887 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 110 (ObjNum 394 snap 119 seq_num 394) dirty exists 2024-08-17T16:59:00.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:59 smithi063 ceph-mon[42680]: pgmap v167: 89 pgs: 89 active+clean; 157 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-08-17T16:59:00.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:59 smithi063 ceph-mon[42680]: osdmap e323: 8 total, 8 up, 8 in 2024-08-17T16:59:00.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:59 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:00.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:59 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:00.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:59 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:00.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:58:59 smithi063 ceph-mon[42680]: osdmap e324: 8 total, 8 up, 8 in 2024-08-17T16:59:00.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:59 smithi063 ceph-mon[31063]: pgmap v167: 89 pgs: 89 active+clean; 157 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-08-17T16:59:00.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:59 smithi063 ceph-mon[31063]: osdmap e323: 8 total, 8 up, 8 in 2024-08-17T16:59:00.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:59 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:00.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:59 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:00.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:59 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:00.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:58:59 smithi063 ceph-mon[31063]: osdmap e324: 8 total, 8 up, 8 in 2024-08-17T16:59:00.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-mon[34633]: pgmap v167: 89 pgs: 89 active+clean; 157 MiB data, 611 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-08-17T16:59:00.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-mon[34633]: osdmap e323: 8 total, 8 up, 8 in 2024-08-17T16:59:00.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:00.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:00.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:00.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:58:59 smithi162 ceph-mon[34633]: osdmap e324: 8 total, 8 up, 8 in 2024-08-17T16:59:00.887 INFO:tasks.rados.rados.0.smithi063.stdout:998: done (2 left) 2024-08-17T16:59:00.887 INFO:tasks.rados.rados.0.smithi063.stdout:999: done (1 left) 2024-08-17T16:59:00.887 INFO:tasks.rados.rados.0.smithi063.stdout:1000: done (0 left) 2024-08-17T16:59:00.887 INFO:tasks.rados.rados.0.smithi063.stdout:1001: delete oid 21 current snap is 125 2024-08-17T16:59:00.890 INFO:tasks.rados.rados.0.smithi063.stdout:1001: done (0 left) 2024-08-17T16:59:00.890 INFO:tasks.rados.rados.0.smithi063.stdout:1002: snap_remove snap 122 2024-08-17T16:59:01.890 INFO:tasks.rados.rados.0.smithi063.stdout:1002: done (0 left) 2024-08-17T16:59:01.890 INFO:tasks.rados.rados.0.smithi063.stdout:1003: read oid 31 snap -1 2024-08-17T16:59:01.890 INFO:tasks.rados.rados.0.smithi063.stdout:1003: expect (ObjNum 395 snap 121 seq_num 395) 2024-08-17T16:59:01.890 INFO:tasks.rados.rados.0.smithi063.stdout:1004: snap_create 2024-08-17T16:59:01.900 INFO:tasks.rados.rados.0.smithi063.stdout:1003: expect (ObjNum 395 snap 121 seq_num 395) 2024-08-17T16:59:02.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:01 smithi063 ceph-mon[42680]: osdmap e325: 8 total, 8 up, 8 in 2024-08-17T16:59:02.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:01 smithi063 ceph-mon[42680]: pgmap v171: 89 pgs: 89 active+clean; 95 MiB data, 492 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 864 KiB/s wr, 11 op/s 2024-08-17T16:59:02.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:01 smithi063 ceph-mon[31063]: osdmap e325: 8 total, 8 up, 8 in 2024-08-17T16:59:02.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:01 smithi063 ceph-mon[31063]: pgmap v171: 89 pgs: 89 active+clean; 95 MiB data, 492 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 864 KiB/s wr, 11 op/s 2024-08-17T16:59:02.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:01 smithi162 ceph-mon[34633]: osdmap e325: 8 total, 8 up, 8 in 2024-08-17T16:59:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:01 smithi162 ceph-mon[34633]: pgmap v171: 89 pgs: 89 active+clean; 95 MiB data, 492 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 864 KiB/s wr, 11 op/s 2024-08-17T16:59:02.891 INFO:tasks.rados.rados.0.smithi063.stdout:1003: done (1 left) 2024-08-17T16:59:02.892 INFO:tasks.rados.rados.0.smithi063.stdout:1004: done (0 left) 2024-08-17T16:59:02.892 INFO:tasks.rados.rados.0.smithi063.stdout:1005: read oid 29 snap 123 2024-08-17T16:59:02.892 INFO:tasks.rados.rados.0.smithi063.stdout:1005: expect (ObjNum 335 snap 101 seq_num 335) 2024-08-17T16:59:02.892 INFO:tasks.rados.rados.0.smithi063.stdout:1006: delete oid 9 current snap is 126 2024-08-17T16:59:02.898 INFO:tasks.rados.rados.0.smithi063.stdout:1006: done (1 left) 2024-08-17T16:59:02.899 INFO:tasks.rados.rados.0.smithi063.stdout:1007: read oid 16 snap 125 2024-08-17T16:59:02.899 INFO:tasks.rados.rados.0.smithi063.stdout:1007: expect deleted 2024-08-17T16:59:02.899 INFO:tasks.rados.rados.0.smithi063.stdout:1008: read oid 21 snap -1 2024-08-17T16:59:02.899 INFO:tasks.rados.rados.0.smithi063.stdout:1008: expect deleted 2024-08-17T16:59:02.899 INFO:tasks.rados.rados.0.smithi063.stdout:1009: rollback oid 10 current snap is 126 2024-08-17T16:59:02.899 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 10 to 124 2024-08-17T16:59:02.899 INFO:tasks.rados.rados.0.smithi063.stdout:1010: delete oid 35 current snap is 126 2024-08-17T16:59:02.899 INFO:tasks.rados.rados.0.smithi063.stdout:1009: finishing rollback tid 1 to smithi06387500-10 2024-08-17T16:59:02.899 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 0 (ObjNum 65 snap 0 seq_num 1953066355) dirty dne 2024-08-17T16:59:02.899 INFO:tasks.rados.rados.0.smithi063.stdout:1005: expect (ObjNum 335 snap 101 seq_num 335) 2024-08-17T16:59:03.016 INFO:tasks.rados.rados.0.smithi063.stdout:1005: done (4 left) 2024-08-17T16:59:03.017 INFO:tasks.rados.rados.0.smithi063.stdout:1007: done (3 left) 2024-08-17T16:59:03.017 INFO:tasks.rados.rados.0.smithi063.stdout:1008: done (2 left) 2024-08-17T16:59:03.017 INFO:tasks.rados.rados.0.smithi063.stdout:1009: done (1 left) 2024-08-17T16:59:03.017 INFO:tasks.rados.rados.0.smithi063.stdout:1010: done (0 left) 2024-08-17T16:59:03.017 INFO:tasks.rados.rados.0.smithi063.stdout:1011: read oid 40 snap -1 2024-08-17T16:59:03.017 INFO:tasks.rados.rados.0.smithi063.stdout:1011: expect (ObjNum 355 snap 107 seq_num 355) 2024-08-17T16:59:03.017 INFO:tasks.rados.rados.0.smithi063.stdout:1012: write oid 16 current snap is 126 2024-08-17T16:59:03.017 INFO:tasks.rados.rados.0.smithi063.stdout:1012: seq_num 407 ranges {440449=793052,1933583=510464} 2024-08-17T16:59:03.023 INFO:tasks.rados.rados.0.smithi063.stdout:1012: writing smithi06387500-16 from 440449 to 1233501 tid 1 2024-08-17T16:59:03.030 INFO:tasks.rados.rados.0.smithi063.stdout:1012: writing smithi06387500-16 from 1933583 to 2444047 tid 2 2024-08-17T16:59:03.030 INFO:tasks.rados.rados.0.smithi063.stdout:1013: rollback oid 34 current snap is 126 2024-08-17T16:59:03.030 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 34 to 124 2024-08-17T16:59:03.030 INFO:tasks.rados.rados.0.smithi063.stdout:1011: expect (ObjNum 355 snap 107 seq_num 355) 2024-08-17T16:59:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:02 smithi162 ceph-mon[34633]: osdmap e326: 8 total, 8 up, 8 in 2024-08-17T16:59:03.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:02 smithi063 ceph-mon[42680]: osdmap e326: 8 total, 8 up, 8 in 2024-08-17T16:59:03.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:02 smithi063 ceph-mon[31063]: osdmap e326: 8 total, 8 up, 8 in 2024-08-17T16:59:03.296 INFO:tasks.rados.rados.0.smithi063.stdout:1012: finishing write tid 1 to smithi06387500-16 2024-08-17T16:59:03.296 INFO:tasks.rados.rados.0.smithi063.stdout:1012: finishing write tid 2 to smithi06387500-16 2024-08-17T16:59:03.296 INFO:tasks.rados.rados.0.smithi063.stdout:1012: finishing write tid 3 to smithi06387500-16 2024-08-17T16:59:03.296 INFO:tasks.rados.rados.0.smithi063.stdout:1012: finishing write tid 4 to smithi06387500-16 2024-08-17T16:59:03.296 INFO:tasks.rados.rados.0.smithi063.stdout:1012: oid 16 updating version 0 to 381 2024-08-17T16:59:03.296 INFO:tasks.rados.rados.0.smithi063.stdout:1012: oid 16 updating version 381 to 382 2024-08-17T16:59:03.297 INFO:tasks.rados.rados.0.smithi063.stdout:1012: oid 16 updating version 382 to 383 2024-08-17T16:59:03.297 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 383 (ObjNum 407 snap 126 seq_num 407) dirty exists 2024-08-17T16:59:03.297 INFO:tasks.rados.rados.0.smithi063.stdout:1012: left oid 16 (ObjNum 407 snap 126 seq_num 407) 2024-08-17T16:59:03.297 INFO:tasks.rados.rados.0.smithi063.stdout:1013: finishing rollback tid 0 to smithi06387500-34 2024-08-17T16:59:03.297 INFO:tasks.rados.rados.0.smithi063.stdout:1013: finishing rollback tid 1 to smithi06387500-34 2024-08-17T16:59:03.297 INFO:tasks.rados.rados.0.smithi063.stdout:1013: finishing rollback tid 2 to smithi06387500-34 2024-08-17T16:59:03.297 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 742 (ObjNum 403 snap 122 seq_num 403) dirty exists 2024-08-17T16:59:03.297 INFO:tasks.rados.rados.0.smithi063.stdout:1011: done (2 left) 2024-08-17T16:59:03.297 INFO:tasks.rados.rados.0.smithi063.stdout:1012: done (1 left) 2024-08-17T16:59:03.297 INFO:tasks.rados.rados.0.smithi063.stdout:1013: done (0 left) 2024-08-17T16:59:03.297 INFO:tasks.rados.rados.0.smithi063.stdout:1014: read oid 8 snap 124 2024-08-17T16:59:03.297 INFO:tasks.rados.rados.0.smithi063.stdout:1014: expect (ObjNum 333 snap 101 seq_num 333) 2024-08-17T16:59:03.297 INFO:tasks.rados.rados.0.smithi063.stdout:1015: delete oid 45 current snap is 126 2024-08-17T16:59:03.298 INFO:tasks.rados.rados.0.smithi063.stdout:1015: done (1 left) 2024-08-17T16:59:03.298 INFO:tasks.rados.rados.0.smithi063.stdout:1016: rollback oid 25 current snap is 126 2024-08-17T16:59:03.298 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 25 to 125 2024-08-17T16:59:03.298 INFO:tasks.rados.rados.0.smithi063.stdout:1017: rollback oid 27 current snap is 126 2024-08-17T16:59:03.299 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 27 to 121 2024-08-17T16:59:03.299 INFO:tasks.rados.rados.0.smithi063.stdout:1018: write oid 44 current snap is 126 2024-08-17T16:59:03.299 INFO:tasks.rados.rados.0.smithi063.stdout:1018: seq_num 409 ranges {474063=543550,1650547=514133,2765651=108412} 2024-08-17T16:59:03.303 INFO:tasks.rados.rados.0.smithi063.stdout:1018: writing smithi06387500-44 from 474063 to 1017613 tid 1 2024-08-17T16:59:03.310 INFO:tasks.rados.rados.0.smithi063.stdout:1018: writing smithi06387500-44 from 1650547 to 2164680 tid 2 2024-08-17T16:59:03.314 INFO:tasks.rados.rados.0.smithi063.stdout:1018: writing smithi06387500-44 from 2765651 to 2874063 tid 3 2024-08-17T16:59:03.314 INFO:tasks.rados.rados.0.smithi063.stdout:1019: read oid 14 snap -1 2024-08-17T16:59:03.314 INFO:tasks.rados.rados.0.smithi063.stdout:1019: expect (ObjNum 380 snap 117 seq_num 380) 2024-08-17T16:59:03.314 INFO:tasks.rados.rados.0.smithi063.stdout:1020: snap_remove snap 123 2024-08-17T16:59:03.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:59:03] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.51.0" 2024-08-17T16:59:03.901 INFO:tasks.rados.rados.0.smithi063.stdout:1020: done (5 left) 2024-08-17T16:59:03.901 INFO:tasks.rados.rados.0.smithi063.stdout:1021: write oid 4 current snap is 126 2024-08-17T16:59:03.902 INFO:tasks.rados.rados.0.smithi063.stdout:1017: finishing rollback tid 1 to smithi06387500-27 2024-08-17T16:59:03.902 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 0 (ObjNum 65 snap 0 seq_num 1953066355) dirty dne 2024-08-17T16:59:03.902 INFO:tasks.rados.rados.0.smithi063.stdout:1016: finishing rollback tid 0 to smithi06387500-25 2024-08-17T16:59:03.902 INFO:tasks.rados.rados.0.smithi063.stdout:1016: finishing rollback tid 1 to smithi06387500-25 2024-08-17T16:59:03.902 INFO:tasks.rados.rados.0.smithi063.stdout:1016: finishing rollback tid 2 to smithi06387500-25 2024-08-17T16:59:03.902 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 1090 (ObjNum 362 snap 108 seq_num 362) dirty exists 2024-08-17T16:59:03.902 INFO:tasks.rados.rados.0.smithi063.stdout:1014: expect (ObjNum 333 snap 101 seq_num 333) 2024-08-17T16:59:04.020 INFO:tasks.rados.rados.0.smithi063.stdout:1018: finishing write tid 1 to smithi06387500-44 2024-08-17T16:59:04.020 INFO:tasks.rados.rados.0.smithi063.stdout:1018: finishing write tid 2 to smithi06387500-44 2024-08-17T16:59:04.020 INFO:tasks.rados.rados.0.smithi063.stdout:1019: expect (ObjNum 380 snap 117 seq_num 380) 2024-08-17T16:59:04.134 INFO:tasks.rados.rados.0.smithi063.stdout:1018: finishing write tid 3 to smithi06387500-44 2024-08-17T16:59:04.135 INFO:tasks.rados.rados.0.smithi063.stdout:1018: finishing write tid 4 to smithi06387500-44 2024-08-17T16:59:04.135 INFO:tasks.rados.rados.0.smithi063.stdout:1018: finishing write tid 5 to smithi06387500-44 2024-08-17T16:59:04.135 INFO:tasks.rados.rados.0.smithi063.stdout:1018: oid 44 updating version 0 to 206 2024-08-17T16:59:04.135 INFO:tasks.rados.rados.0.smithi063.stdout:1018: oid 44 updating version 206 to 207 2024-08-17T16:59:04.135 INFO:tasks.rados.rados.0.smithi063.stdout:1018: oid 44 version 207 is already newer than 204 2024-08-17T16:59:04.135 INFO:tasks.rados.rados.0.smithi063.stdout:1018: oid 44 version 207 is already newer than 205 2024-08-17T16:59:04.135 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 207 (ObjNum 409 snap 126 seq_num 409) dirty exists 2024-08-17T16:59:04.135 INFO:tasks.rados.rados.0.smithi063.stdout:1018: left oid 44 (ObjNum 409 snap 126 seq_num 409) 2024-08-17T16:59:04.135 INFO:tasks.rados.rados.0.smithi063.stdout:1021: seq_num 410 ranges {490870=618799,1592676=774144,2793074=97796} 2024-08-17T16:59:04.140 INFO:tasks.rados.rados.0.smithi063.stdout:1021: writing smithi06387500-4 from 490870 to 1109669 tid 1 2024-08-17T16:59:04.147 INFO:tasks.rados.rados.0.smithi063.stdout:1021: writing smithi06387500-4 from 1592676 to 2366820 tid 2 2024-08-17T16:59:04.151 INFO:tasks.rados.rados.0.smithi063.stdout:1021: writing smithi06387500-4 from 2793074 to 2890870 tid 3 2024-08-17T16:59:04.151 INFO:tasks.rados.rados.0.smithi063.stdout:1014: done (5 left) 2024-08-17T16:59:04.151 INFO:tasks.rados.rados.0.smithi063.stdout:1016: done (4 left) 2024-08-17T16:59:04.151 INFO:tasks.rados.rados.0.smithi063.stdout:1017: done (3 left) 2024-08-17T16:59:04.151 INFO:tasks.rados.rados.0.smithi063.stdout:1018: done (2 left) 2024-08-17T16:59:04.151 INFO:tasks.rados.rados.0.smithi063.stdout:1019: done (1 left) 2024-08-17T16:59:04.151 INFO:tasks.rados.rados.0.smithi063.stdout:1022: read oid 15 snap -1 2024-08-17T16:59:04.151 INFO:tasks.rados.rados.0.smithi063.stdout:1022: expect (ObjNum 289 snap 88 seq_num 289) 2024-08-17T16:59:04.152 INFO:tasks.rados.rados.0.smithi063.stdout:1023: delete oid 28 current snap is 126 2024-08-17T16:59:04.152 INFO:tasks.rados.rados.0.smithi063.stdout:1021: finishing write tid 1 to smithi06387500-4 2024-08-17T16:59:04.152 INFO:tasks.rados.rados.0.smithi063.stdout:1021: finishing write tid 2 to smithi06387500-4 2024-08-17T16:59:04.152 INFO:tasks.rados.rados.0.smithi063.stdout:1021: finishing write tid 3 to smithi06387500-4 2024-08-17T16:59:04.152 INFO:tasks.rados.rados.0.smithi063.stdout:1023: done (2 left) 2024-08-17T16:59:04.153 INFO:tasks.rados.rados.0.smithi063.stdout:1024: read oid 9 snap 124 2024-08-17T16:59:04.153 INFO:tasks.rados.rados.0.smithi063.stdout:1024: expect (ObjNum 384 snap 118 seq_num 384) 2024-08-17T16:59:04.153 INFO:tasks.rados.rados.0.smithi063.stdout:1025: write oid 16 current snap is 126 2024-08-17T16:59:04.153 INFO:tasks.rados.rados.0.smithi063.stdout:1021: finishing write tid 4 to smithi06387500-4 2024-08-17T16:59:04.153 INFO:tasks.rados.rados.0.smithi063.stdout:1025: seq_num 412 ranges {524484=485650,1425993=661460,2847920=76564} 2024-08-17T16:59:04.158 INFO:tasks.rados.rados.0.smithi063.stdout:1025: writing smithi06387500-16 from 524484 to 1010134 tid 1 2024-08-17T16:59:04.164 INFO:tasks.rados.rados.0.smithi063.stdout:1025: writing smithi06387500-16 from 1425993 to 2087453 tid 2 2024-08-17T16:59:04.169 INFO:tasks.rados.rados.0.smithi063.stdout:1025: writing smithi06387500-16 from 2847920 to 2924484 tid 3 2024-08-17T16:59:04.169 INFO:tasks.rados.rados.0.smithi063.stdout:1026: read oid 6 snap -1 2024-08-17T16:59:04.170 INFO:tasks.rados.rados.0.smithi063.stdout:1026: expect deleted 2024-08-17T16:59:04.170 INFO:tasks.rados.rados.0.smithi063.stdout:1027: rollback oid 26 current snap is 126 2024-08-17T16:59:04.170 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 26 to 125 2024-08-17T16:59:04.170 INFO:tasks.rados.rados.0.smithi063.stdout:1021: finishing write tid 5 to smithi06387500-4 2024-08-17T16:59:04.170 INFO:tasks.rados.rados.0.smithi063.stdout:1021: oid 4 updating version 0 to 1098 2024-08-17T16:59:04.170 INFO:tasks.rados.rados.0.smithi063.stdout:1021: oid 4 version 1098 is already newer than 1096 2024-08-17T16:59:04.170 INFO:tasks.rados.rados.0.smithi063.stdout:1021: oid 4 version 1098 is already newer than 1095 2024-08-17T16:59:04.170 INFO:tasks.rados.rados.0.smithi063.stdout:1021: oid 4 version 1098 is already newer than 1097 2024-08-17T16:59:04.170 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 1098 (ObjNum 410 snap 126 seq_num 410) dirty exists 2024-08-17T16:59:04.170 INFO:tasks.rados.rados.0.smithi063.stdout:1021: left oid 4 (ObjNum 410 snap 126 seq_num 410) 2024-08-17T16:59:04.170 INFO:tasks.rados.rados.0.smithi063.stdout:1022: expect (ObjNum 289 snap 88 seq_num 289) 2024-08-17T16:59:04.222 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:03 smithi162 ceph-mon[34633]: osdmap e327: 8 total, 8 up, 8 in 2024-08-17T16:59:04.222 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:03 smithi162 ceph-mon[34633]: pgmap v174: 89 pgs: 89 active+clean; 95 MiB data, 492 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 865 KiB/s wr, 11 op/s 2024-08-17T16:59:04.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:03 smithi063 ceph-mon[42680]: osdmap e327: 8 total, 8 up, 8 in 2024-08-17T16:59:04.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:03 smithi063 ceph-mon[42680]: pgmap v174: 89 pgs: 89 active+clean; 95 MiB data, 492 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 865 KiB/s wr, 11 op/s 2024-08-17T16:59:04.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:03 smithi063 ceph-mon[31063]: osdmap e327: 8 total, 8 up, 8 in 2024-08-17T16:59:04.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:03 smithi063 ceph-mon[31063]: pgmap v174: 89 pgs: 89 active+clean; 95 MiB data, 492 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 865 KiB/s wr, 11 op/s 2024-08-17T16:59:04.291 INFO:tasks.rados.rados.0.smithi063.stdout:1025: finishing write tid 1 to smithi06387500-16 2024-08-17T16:59:04.291 INFO:tasks.rados.rados.0.smithi063.stdout:1024: expect (ObjNum 384 snap 118 seq_num 384) 2024-08-17T16:59:04.444 INFO:tasks.rados.rados.0.smithi063.stdout:1025: finishing write tid 2 to smithi06387500-16 2024-08-17T16:59:04.444 INFO:tasks.rados.rados.0.smithi063.stdout:1027: finishing rollback tid 1 to smithi06387500-26 2024-08-17T16:59:04.444 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T16:59:04.444 INFO:tasks.rados.rados.0.smithi063.stdout:1025: finishing write tid 3 to smithi06387500-16 2024-08-17T16:59:04.445 INFO:tasks.rados.rados.0.smithi063.stdout:1025: finishing write tid 4 to smithi06387500-16 2024-08-17T16:59:04.445 INFO:tasks.rados.rados.0.smithi063.stdout:1021: done (5 left) 2024-08-17T16:59:04.445 INFO:tasks.rados.rados.0.smithi063.stdout:1022: done (4 left) 2024-08-17T16:59:04.445 INFO:tasks.rados.rados.0.smithi063.stdout:1024: done (3 left) 2024-08-17T16:59:04.445 INFO:tasks.rados.rados.0.smithi063.stdout:1026: done (2 left) 2024-08-17T16:59:04.445 INFO:tasks.rados.rados.0.smithi063.stdout:1027: done (1 left) 2024-08-17T16:59:04.445 INFO:tasks.rados.rados.0.smithi063.stdout:1028: snap_create 2024-08-17T16:59:04.445 INFO:tasks.rados.rados.0.smithi063.stdout:1025: finishing write tid 5 to smithi06387500-16 2024-08-17T16:59:04.445 INFO:tasks.rados.rados.0.smithi063.stdout:1025: oid 16 updating version 0 to 386 2024-08-17T16:59:04.445 INFO:tasks.rados.rados.0.smithi063.stdout:1025: oid 16 version 386 is already newer than 385 2024-08-17T16:59:04.445 INFO:tasks.rados.rados.0.smithi063.stdout:1025: oid 16 updating version 386 to 387 2024-08-17T16:59:04.445 INFO:tasks.rados.rados.0.smithi063.stdout:1025: oid 16 version 387 is already newer than 384 2024-08-17T16:59:04.445 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 387 (ObjNum 412 snap 126 seq_num 412) dirty exists 2024-08-17T16:59:04.445 INFO:tasks.rados.rados.0.smithi063.stdout:1025: left oid 16 (ObjNum 412 snap 126 seq_num 412) 2024-08-17T16:59:04.885 INFO:tasks.rados.rados.0.smithi063.stdout:1025: done (1 left) 2024-08-17T16:59:04.885 INFO:tasks.rados.rados.0.smithi063.stdout:1028: done (0 left) 2024-08-17T16:59:04.885 INFO:tasks.rados.rados.0.smithi063.stdout:1029: read oid 1 snap 125 2024-08-17T16:59:04.885 INFO:tasks.rados.rados.0.smithi063.stdout:1029: expect (ObjNum 391 snap 119 seq_num 391) 2024-08-17T16:59:04.885 INFO:tasks.rados.rados.0.smithi063.stdout:1030: write oid 28 current snap is 127 2024-08-17T16:59:04.885 INFO:tasks.rados.rados.0.smithi063.stdout:1030: seq_num 413 ranges {541291=560899,1884475=521471} 2024-08-17T16:59:04.890 INFO:tasks.rados.rados.0.smithi063.stdout:1030: writing smithi06387500-28 from 541291 to 1102190 tid 1 2024-08-17T16:59:04.899 INFO:tasks.rados.rados.0.smithi063.stdout:1030: writing smithi06387500-28 from 1884475 to 2405946 tid 2 2024-08-17T16:59:04.899 INFO:tasks.rados.rados.0.smithi063.stdout:1031: write oid 26 current snap is 127 2024-08-17T16:59:04.899 INFO:tasks.rados.rados.0.smithi063.stdout:1031: seq_num 414 ranges {558098=636148,1826604=665129} 2024-08-17T16:59:04.905 INFO:tasks.rados.rados.0.smithi063.stdout:1031: writing smithi06387500-26 from 558098 to 1194246 tid 1 2024-08-17T16:59:04.913 INFO:tasks.rados.rados.0.smithi063.stdout:1031: writing smithi06387500-26 from 1826604 to 2491733 tid 2 2024-08-17T16:59:04.913 INFO:tasks.rados.rados.0.smithi063.stdout:1032: snap_create 2024-08-17T16:59:04.913 INFO:tasks.rados.rados.0.smithi063.stdout:1029: expect (ObjNum 391 snap 119 seq_num 391) 2024-08-17T16:59:05.072 INFO:tasks.rados.rados.0.smithi063.stdout:1030: finishing write tid 1 to smithi06387500-28 2024-08-17T16:59:05.073 INFO:tasks.rados.rados.0.smithi063.stdout:1030: finishing write tid 2 to smithi06387500-28 2024-08-17T16:59:05.073 INFO:tasks.rados.rados.0.smithi063.stdout:1030: finishing write tid 3 to smithi06387500-28 2024-08-17T16:59:05.073 INFO:tasks.rados.rados.0.smithi063.stdout:1030: finishing write tid 4 to smithi06387500-28 2024-08-17T16:59:05.073 INFO:tasks.rados.rados.0.smithi063.stdout:1030: oid 28 updating version 0 to 287 2024-08-17T16:59:05.073 INFO:tasks.rados.rados.0.smithi063.stdout:1030: oid 28 updating version 287 to 288 2024-08-17T16:59:05.073 INFO:tasks.rados.rados.0.smithi063.stdout:1030: oid 28 updating version 288 to 289 2024-08-17T16:59:05.073 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 289 (ObjNum 413 snap 127 seq_num 413) dirty exists 2024-08-17T16:59:05.073 INFO:tasks.rados.rados.0.smithi063.stdout:1030: left oid 28 (ObjNum 413 snap 127 seq_num 413) 2024-08-17T16:59:05.073 INFO:tasks.rados.rados.0.smithi063.stdout:1031: finishing write tid 1 to smithi06387500-26 2024-08-17T16:59:05.073 INFO:tasks.rados.rados.0.smithi063.stdout:1031: finishing write tid 2 to smithi06387500-26 2024-08-17T16:59:05.073 INFO:tasks.rados.rados.0.smithi063.stdout:1031: finishing write tid 3 to smithi06387500-26 2024-08-17T16:59:05.073 INFO:tasks.rados.rados.0.smithi063.stdout:1031: finishing write tid 4 to smithi06387500-26 2024-08-17T16:59:05.074 INFO:tasks.rados.rados.0.smithi063.stdout:1031: oid 26 updating version 0 to 343 2024-08-17T16:59:05.074 INFO:tasks.rados.rados.0.smithi063.stdout:1031: oid 26 version 343 is already newer than 342 2024-08-17T16:59:05.074 INFO:tasks.rados.rados.0.smithi063.stdout:1031: oid 26 updating version 343 to 344 2024-08-17T16:59:05.074 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 344 (ObjNum 414 snap 127 seq_num 414) dirty exists 2024-08-17T16:59:05.074 INFO:tasks.rados.rados.0.smithi063.stdout:1031: left oid 26 (ObjNum 414 snap 127 seq_num 414) 2024-08-17T16:59:05.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:04 smithi063 ceph-mon[42680]: osdmap e328: 8 total, 8 up, 8 in 2024-08-17T16:59:05.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:04 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:05.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:04 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:05.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:04 smithi063 ceph-mon[42680]: osdmap e329: 8 total, 8 up, 8 in 2024-08-17T16:59:05.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:04 smithi063 ceph-mon[31063]: osdmap e328: 8 total, 8 up, 8 in 2024-08-17T16:59:05.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:04 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:05.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:04 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:05.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:04 smithi063 ceph-mon[31063]: osdmap e329: 8 total, 8 up, 8 in 2024-08-17T16:59:05.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:04 smithi162 ceph-mon[34633]: osdmap e328: 8 total, 8 up, 8 in 2024-08-17T16:59:05.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:04 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:05.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:04 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:05.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:04 smithi162 ceph-mon[34633]: osdmap e329: 8 total, 8 up, 8 in 2024-08-17T16:59:05.908 INFO:tasks.rados.rados.0.smithi063.stdout:1029: done (3 left) 2024-08-17T16:59:05.908 INFO:tasks.rados.rados.0.smithi063.stdout:1030: done (2 left) 2024-08-17T16:59:05.908 INFO:tasks.rados.rados.0.smithi063.stdout:1031: done (1 left) 2024-08-17T16:59:05.908 INFO:tasks.rados.rados.0.smithi063.stdout:1032: done (0 left) 2024-08-17T16:59:05.908 INFO:tasks.rados.rados.0.smithi063.stdout:1033: delete oid 26 current snap is 128 2024-08-17T16:59:05.912 INFO:tasks.rados.rados.0.smithi063.stdout:1033: done (0 left) 2024-08-17T16:59:05.912 INFO:tasks.rados.rados.0.smithi063.stdout:1034: write oid 47 current snap is 128 2024-08-17T16:59:05.912 INFO:tasks.rados.rados.0.smithi063.stdout:1034: seq_num 416 ranges {591712=786646,1943568=668798} 2024-08-17T16:59:05.919 INFO:tasks.rados.rados.0.smithi063.stdout:1034: writing smithi06387500-47 from 591712 to 1378358 tid 1 2024-08-17T16:59:05.927 INFO:tasks.rados.rados.0.smithi063.stdout:1034: writing smithi06387500-47 from 1943568 to 2612366 tid 2 2024-08-17T16:59:05.927 INFO:tasks.rados.rados.0.smithi063.stdout:1035: write oid 20 current snap is 128 2024-08-17T16:59:05.927 INFO:tasks.rados.rados.0.smithi063.stdout:1034: finishing write tid 1 to smithi06387500-47 2024-08-17T16:59:05.927 INFO:tasks.rados.rados.0.smithi063.stdout:1035: seq_num 417 ranges {608519=461895,1602050=528809,2585035=423484} 2024-08-17T16:59:05.932 INFO:tasks.rados.rados.0.smithi063.stdout:1035: writing smithi06387500-20 from 608519 to 1070414 tid 1 2024-08-17T16:59:05.938 INFO:tasks.rados.rados.0.smithi063.stdout:1035: writing smithi06387500-20 from 1602050 to 2130859 tid 2 2024-08-17T16:59:05.944 INFO:tasks.rados.rados.0.smithi063.stdout:1035: writing smithi06387500-20 from 2585035 to 3008519 tid 3 2024-08-17T16:59:05.944 INFO:tasks.rados.rados.0.smithi063.stdout:1036: snap_create 2024-08-17T16:59:05.944 INFO:tasks.rados.rados.0.smithi063.stdout:1034: finishing write tid 2 to smithi06387500-47 2024-08-17T16:59:05.944 INFO:tasks.rados.rados.0.smithi063.stdout:1034: finishing write tid 3 to smithi06387500-47 2024-08-17T16:59:05.944 INFO:tasks.rados.rados.0.smithi063.stdout:1034: finishing write tid 4 to smithi06387500-47 2024-08-17T16:59:05.944 INFO:tasks.rados.rados.0.smithi063.stdout:1034: oid 47 updating version 0 to 193 2024-08-17T16:59:05.944 INFO:tasks.rados.rados.0.smithi063.stdout:1034: oid 47 version 193 is already newer than 191 2024-08-17T16:59:05.944 INFO:tasks.rados.rados.0.smithi063.stdout:1034: oid 47 version 193 is already newer than 192 2024-08-17T16:59:05.944 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 193 (ObjNum 416 snap 128 seq_num 416) dirty exists 2024-08-17T16:59:05.944 INFO:tasks.rados.rados.0.smithi063.stdout:1034: left oid 47 (ObjNum 416 snap 128 seq_num 416) 2024-08-17T16:59:05.945 INFO:tasks.rados.rados.0.smithi063.stdout:1035: finishing write tid 1 to smithi06387500-20 2024-08-17T16:59:05.945 INFO:tasks.rados.rados.0.smithi063.stdout:1035: finishing write tid 2 to smithi06387500-20 2024-08-17T16:59:05.946 INFO:tasks.rados.rados.0.smithi063.stdout:1035: finishing write tid 3 to smithi06387500-20 2024-08-17T16:59:05.946 INFO:tasks.rados.rados.0.smithi063.stdout:1035: finishing write tid 4 to smithi06387500-20 2024-08-17T16:59:05.946 INFO:tasks.rados.rados.0.smithi063.stdout:1035: finishing write tid 5 to smithi06387500-20 2024-08-17T16:59:05.946 INFO:tasks.rados.rados.0.smithi063.stdout:1035: oid 20 updating version 0 to 292 2024-08-17T16:59:05.946 INFO:tasks.rados.rados.0.smithi063.stdout:1035: oid 20 updating version 292 to 294 2024-08-17T16:59:05.946 INFO:tasks.rados.rados.0.smithi063.stdout:1035: oid 20 version 294 is already newer than 291 2024-08-17T16:59:05.947 INFO:tasks.rados.rados.0.smithi063.stdout:1035: oid 20 version 294 is already newer than 293 2024-08-17T16:59:05.947 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 294 (ObjNum 417 snap 128 seq_num 417) dirty exists 2024-08-17T16:59:05.947 INFO:tasks.rados.rados.0.smithi063.stdout:1035: left oid 20 (ObjNum 417 snap 128 seq_num 417) 2024-08-17T16:59:06.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:05 smithi162 ceph-mon[34633]: pgmap v177: 89 pgs: 89 active+clean; 95 MiB data, 492 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:59:06.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:05 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:06.020 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:05 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:06.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:05 smithi063 ceph-mon[42680]: pgmap v177: 89 pgs: 89 active+clean; 95 MiB data, 492 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:59:06.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:05 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:06.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:05 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:06.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:05 smithi063 ceph-mon[31063]: pgmap v177: 89 pgs: 89 active+clean; 95 MiB data, 492 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:59:06.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:05 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:06.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:05 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:06.913 INFO:tasks.rados.rados.0.smithi063.stdout:1034: done (2 left) 2024-08-17T16:59:06.914 INFO:tasks.rados.rados.0.smithi063.stdout:1035: done (1 left) 2024-08-17T16:59:06.914 INFO:tasks.rados.rados.0.smithi063.stdout:1036: done (0 left) 2024-08-17T16:59:06.914 INFO:tasks.rados.rados.0.smithi063.stdout:1037: read oid 36 snap 125 2024-08-17T16:59:06.914 INFO:tasks.rados.rados.0.smithi063.stdout:1037: expect (ObjNum 398 snap 121 seq_num 398) 2024-08-17T16:59:06.914 INFO:tasks.rados.rados.0.smithi063.stdout:1038: write oid 8 current snap is 129 2024-08-17T16:59:06.914 INFO:tasks.rados.rados.0.smithi063.stdout:1038: seq_num 418 ranges {625326=537144,1944179=672467} 2024-08-17T16:59:06.919 INFO:tasks.rados.rados.0.smithi063.stdout:1038: writing smithi06387500-8 from 625326 to 1162470 tid 1 2024-08-17T16:59:06.928 INFO:tasks.rados.rados.0.smithi063.stdout:1038: writing smithi06387500-8 from 1944179 to 2616646 tid 2 2024-08-17T16:59:06.928 INFO:tasks.rados.rados.0.smithi063.stdout:1039: write oid 16 current snap is 129 2024-08-17T16:59:06.928 INFO:tasks.rados.rados.0.smithi063.stdout:1039: seq_num 419 ranges {642133=728746,2119014=532478} 2024-08-17T16:59:06.935 INFO:tasks.rados.rados.0.smithi063.stdout:1039: writing smithi06387500-16 from 642133 to 1370879 tid 1 2024-08-17T16:59:06.943 INFO:tasks.rados.rados.0.smithi063.stdout:1039: writing smithi06387500-16 from 2119014 to 2651492 tid 2 2024-08-17T16:59:06.943 INFO:tasks.rados.rados.0.smithi063.stdout:1040: rollback oid 9 current snap is 129 2024-08-17T16:59:06.943 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 9 to 127 2024-08-17T16:59:06.943 INFO:tasks.rados.rados.0.smithi063.stdout:1037: expect (ObjNum 398 snap 121 seq_num 398) 2024-08-17T16:59:07.029 INFO:tasks.rados.rados.0.smithi063.stdout:1038: finishing write tid 1 to smithi06387500-8 2024-08-17T16:59:07.029 INFO:tasks.rados.rados.0.smithi063.stdout:1038: finishing write tid 2 to smithi06387500-8 2024-08-17T16:59:07.029 INFO:tasks.rados.rados.0.smithi063.stdout:1038: finishing write tid 3 to smithi06387500-8 2024-08-17T16:59:07.030 INFO:tasks.rados.rados.0.smithi063.stdout:1038: finishing write tid 4 to smithi06387500-8 2024-08-17T16:59:07.030 INFO:tasks.rados.rados.0.smithi063.stdout:1038: oid 8 updating version 0 to 291 2024-08-17T16:59:07.030 INFO:tasks.rados.rados.0.smithi063.stdout:1038: oid 8 updating version 291 to 293 2024-08-17T16:59:07.030 INFO:tasks.rados.rados.0.smithi063.stdout:1038: oid 8 version 293 is already newer than 292 2024-08-17T16:59:07.030 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 293 (ObjNum 418 snap 129 seq_num 418) dirty exists 2024-08-17T16:59:07.030 INFO:tasks.rados.rados.0.smithi063.stdout:1038: left oid 8 (ObjNum 418 snap 129 seq_num 418) 2024-08-17T16:59:07.030 INFO:tasks.rados.rados.0.smithi063.stdout:1039: finishing write tid 1 to smithi06387500-16 2024-08-17T16:59:07.030 INFO:tasks.rados.rados.0.smithi063.stdout:1037: done (3 left) 2024-08-17T16:59:07.030 INFO:tasks.rados.rados.0.smithi063.stdout:1038: done (2 left) 2024-08-17T16:59:07.030 INFO:tasks.rados.rados.0.smithi063.stdout:1041: read oid 21 snap -1 2024-08-17T16:59:07.030 INFO:tasks.rados.rados.0.smithi063.stdout:1041: expect deleted 2024-08-17T16:59:07.030 INFO:tasks.rados.rados.0.smithi063.stdout:1042: write oid 2 current snap is 129 2024-08-17T16:59:07.030 INFO:tasks.rados.rados.0.smithi063.stdout:1039: finishing write tid 2 to smithi06387500-16 2024-08-17T16:59:07.031 INFO:tasks.rados.rados.0.smithi063.stdout:1040: finishing rollback tid 1 to smithi06387500-9 2024-08-17T16:59:07.031 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 619 (ObjNum 33 snap 0 seq_num 1786885368) dirty dne 2024-08-17T16:59:07.031 INFO:tasks.rados.rados.0.smithi063.stdout:1039: finishing write tid 3 to smithi06387500-16 2024-08-17T16:59:07.031 INFO:tasks.rados.rados.0.smithi063.stdout:1039: finishing write tid 4 to smithi06387500-16 2024-08-17T16:59:07.031 INFO:tasks.rados.rados.0.smithi063.stdout:1039: oid 16 updating version 0 to 390 2024-08-17T16:59:07.031 INFO:tasks.rados.rados.0.smithi063.stdout:1039: oid 16 version 390 is already newer than 389 2024-08-17T16:59:07.031 INFO:tasks.rados.rados.0.smithi063.stdout:1039: oid 16 updating version 390 to 391 2024-08-17T16:59:07.031 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 391 (ObjNum 419 snap 129 seq_num 419) dirty exists 2024-08-17T16:59:07.031 INFO:tasks.rados.rados.0.smithi063.stdout:1039: left oid 16 (ObjNum 419 snap 129 seq_num 419) 2024-08-17T16:59:07.031 INFO:tasks.rados.rados.0.smithi063.stdout:1042: seq_num 420 ranges {658940=403995,1777496=676136,2950951=107989} 2024-08-17T16:59:07.034 INFO:tasks.rados.rados.0.smithi063.stdout:1042: writing smithi06387500-2 from 658940 to 1062935 tid 1 2024-08-17T16:59:07.042 INFO:tasks.rados.rados.0.smithi063.stdout:1042: writing smithi06387500-2 from 1777496 to 2453632 tid 2 2024-08-17T16:59:07.046 INFO:tasks.rados.rados.0.smithi063.stdout:1042: writing smithi06387500-2 from 2950951 to 3058940 tid 3 2024-08-17T16:59:07.046 INFO:tasks.rados.rados.0.smithi063.stdout:1039: done (3 left) 2024-08-17T16:59:07.046 INFO:tasks.rados.rados.0.smithi063.stdout:1040: done (2 left) 2024-08-17T16:59:07.046 INFO:tasks.rados.rados.0.smithi063.stdout:1043: snap_create 2024-08-17T16:59:07.046 INFO:tasks.rados.rados.0.smithi063.stdout:1042: finishing write tid 1 to smithi06387500-2 2024-08-17T16:59:07.046 INFO:tasks.rados.rados.0.smithi063.stdout:1042: finishing write tid 2 to smithi06387500-2 2024-08-17T16:59:07.047 INFO:tasks.rados.rados.0.smithi063.stdout:1042: finishing write tid 3 to smithi06387500-2 2024-08-17T16:59:07.047 INFO:tasks.rados.rados.0.smithi063.stdout:1042: finishing write tid 4 to smithi06387500-2 2024-08-17T16:59:07.047 INFO:tasks.rados.rados.0.smithi063.stdout:1042: finishing write tid 5 to smithi06387500-2 2024-08-17T16:59:07.047 INFO:tasks.rados.rados.0.smithi063.stdout:1042: oid 2 updating version 0 to 508 2024-08-17T16:59:07.047 INFO:tasks.rados.rados.0.smithi063.stdout:1042: oid 2 version 508 is already newer than 506 2024-08-17T16:59:07.048 INFO:tasks.rados.rados.0.smithi063.stdout:1042: oid 2 version 508 is already newer than 507 2024-08-17T16:59:07.048 INFO:tasks.rados.rados.0.smithi063.stdout:1042: oid 2 updating version 508 to 509 2024-08-17T16:59:07.048 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 509 (ObjNum 420 snap 129 seq_num 420) dirty exists 2024-08-17T16:59:07.048 INFO:tasks.rados.rados.0.smithi063.stdout:1042: left oid 2 (ObjNum 420 snap 129 seq_num 420) 2024-08-17T16:59:07.184 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:06 smithi162 ceph-mon[34633]: osdmap e330: 8 total, 8 up, 8 in 2024-08-17T16:59:07.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:06 smithi063 ceph-mon[42680]: osdmap e330: 8 total, 8 up, 8 in 2024-08-17T16:59:07.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:06 smithi063 ceph-mon[31063]: osdmap e330: 8 total, 8 up, 8 in 2024-08-17T16:59:07.919 INFO:tasks.rados.rados.0.smithi063.stdout:1041: done (2 left) 2024-08-17T16:59:07.919 INFO:tasks.rados.rados.0.smithi063.stdout:1042: done (1 left) 2024-08-17T16:59:07.919 INFO:tasks.rados.rados.0.smithi063.stdout:1043: done (0 left) 2024-08-17T16:59:07.919 INFO:tasks.rados.rados.0.smithi063.stdout:1044: rollback oid 27 current snap is 130 2024-08-17T16:59:07.919 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 27 to 129 2024-08-17T16:59:07.919 INFO:tasks.rados.rados.0.smithi063.stdout:1045: write oid 39 current snap is 130 2024-08-17T16:59:07.920 INFO:tasks.rados.rados.0.smithi063.stdout:1045: seq_num 421 ranges {675747=479244,1835978=536147} 2024-08-17T16:59:07.925 INFO:tasks.rados.rados.0.smithi063.stdout:1045: writing smithi06387500-39 from 675747 to 1154991 tid 1 2024-08-17T16:59:07.932 INFO:tasks.rados.rados.0.smithi063.stdout:1045: writing smithi06387500-39 from 1835978 to 2372125 tid 2 2024-08-17T16:59:07.932 INFO:tasks.rados.rados.0.smithi063.stdout:1046: write oid 38 current snap is 130 2024-08-17T16:59:07.932 INFO:tasks.rados.rados.0.smithi063.stdout:1046: seq_num 422 ranges {692554=554493,1778107=679805,2889444=203110} 2024-08-17T16:59:07.939 INFO:tasks.rados.rados.0.smithi063.stdout:1046: writing smithi06387500-38 from 692554 to 1247047 tid 1 2024-08-17T16:59:07.945 INFO:tasks.rados.rados.0.smithi063.stdout:1046: writing smithi06387500-38 from 1778107 to 2457912 tid 2 2024-08-17T16:59:07.949 INFO:tasks.rados.rados.0.smithi063.stdout:1046: writing smithi06387500-38 from 2889444 to 3092554 tid 3 2024-08-17T16:59:07.949 INFO:tasks.rados.rados.0.smithi063.stdout:1047: write oid 32 current snap is 130 2024-08-17T16:59:07.949 INFO:tasks.rados.rados.0.smithi063.stdout:1044: finishing rollback tid 1 to smithi06387500-27 2024-08-17T16:59:07.950 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 0 (ObjNum 574598368 snap 32710 seq_num 304) dirty dne 2024-08-17T16:59:07.950 INFO:tasks.rados.rados.0.smithi063.stdout:1047: seq_num 423 ranges {709361=629742,1836589=539816,3033220=76141} 2024-08-17T16:59:07.956 INFO:tasks.rados.rados.0.smithi063.stdout:1047: writing smithi06387500-32 from 709361 to 1339103 tid 1 2024-08-17T16:59:07.962 INFO:tasks.rados.rados.0.smithi063.stdout:1047: writing smithi06387500-32 from 1836589 to 2376405 tid 2 2024-08-17T16:59:07.967 INFO:tasks.rados.rados.0.smithi063.stdout:1047: writing smithi06387500-32 from 3033220 to 3109361 tid 3 2024-08-17T16:59:07.967 INFO:tasks.rados.rados.0.smithi063.stdout:1044: done (3 left) 2024-08-17T16:59:07.967 INFO:tasks.rados.rados.0.smithi063.stdout:1048: read oid 16 snap 124 2024-08-17T16:59:07.967 INFO:tasks.rados.rados.0.smithi063.stdout:1048: expect deleted 2024-08-17T16:59:07.967 INFO:tasks.rados.rados.0.smithi063.stdout:1049: rollback oid 8 current snap is 130 2024-08-17T16:59:07.967 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 8 to 128 2024-08-17T16:59:07.967 INFO:tasks.rados.rados.0.smithi063.stdout:1045: finishing write tid 1 to smithi06387500-39 2024-08-17T16:59:07.968 INFO:tasks.rados.rados.0.smithi063.stdout:1045: finishing write tid 2 to smithi06387500-39 2024-08-17T16:59:07.968 INFO:tasks.rados.rados.0.smithi063.stdout:1045: finishing write tid 3 to smithi06387500-39 2024-08-17T16:59:07.968 INFO:tasks.rados.rados.0.smithi063.stdout:1045: finishing write tid 4 to smithi06387500-39 2024-08-17T16:59:07.968 INFO:tasks.rados.rados.0.smithi063.stdout:1045: oid 39 updating version 0 to 274 2024-08-17T16:59:07.968 INFO:tasks.rados.rados.0.smithi063.stdout:1045: oid 39 version 274 is already newer than 273 2024-08-17T16:59:07.968 INFO:tasks.rados.rados.0.smithi063.stdout:1045: oid 39 updating version 274 to 275 2024-08-17T16:59:07.968 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 275 (ObjNum 421 snap 130 seq_num 421) dirty exists 2024-08-17T16:59:07.968 INFO:tasks.rados.rados.0.smithi063.stdout:1045: left oid 39 (ObjNum 421 snap 130 seq_num 421) 2024-08-17T16:59:07.968 INFO:tasks.rados.rados.0.smithi063.stdout:1046: finishing write tid 1 to smithi06387500-38 2024-08-17T16:59:07.968 INFO:tasks.rados.rados.0.smithi063.stdout:1046: finishing write tid 2 to smithi06387500-38 2024-08-17T16:59:07.968 INFO:tasks.rados.rados.0.smithi063.stdout:1045: done (4 left) 2024-08-17T16:59:07.969 INFO:tasks.rados.rados.0.smithi063.stdout:1050: snap_create 2024-08-17T16:59:07.969 INFO:tasks.rados.rados.0.smithi063.stdout:1046: finishing write tid 3 to smithi06387500-38 2024-08-17T16:59:07.969 INFO:tasks.rados.rados.0.smithi063.stdout:1046: finishing write tid 4 to smithi06387500-38 2024-08-17T16:59:07.969 INFO:tasks.rados.rados.0.smithi063.stdout:1046: finishing write tid 5 to smithi06387500-38 2024-08-17T16:59:07.969 INFO:tasks.rados.rados.0.smithi063.stdout:1046: oid 38 updating version 0 to 117 2024-08-17T16:59:07.969 INFO:tasks.rados.rados.0.smithi063.stdout:1046: oid 38 updating version 117 to 119 2024-08-17T16:59:07.969 INFO:tasks.rados.rados.0.smithi063.stdout:1046: oid 38 version 119 is already newer than 118 2024-08-17T16:59:07.969 INFO:tasks.rados.rados.0.smithi063.stdout:1046: oid 38 version 119 is already newer than 116 2024-08-17T16:59:07.969 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 119 (ObjNum 422 snap 130 seq_num 422) dirty exists 2024-08-17T16:59:07.969 INFO:tasks.rados.rados.0.smithi063.stdout:1046: left oid 38 (ObjNum 422 snap 130 seq_num 422) 2024-08-17T16:59:07.969 INFO:tasks.rados.rados.0.smithi063.stdout:1047: finishing write tid 1 to smithi06387500-32 2024-08-17T16:59:07.969 INFO:tasks.rados.rados.0.smithi063.stdout:1047: finishing write tid 2 to smithi06387500-32 2024-08-17T16:59:07.970 INFO:tasks.rados.rados.0.smithi063.stdout:1047: finishing write tid 3 to smithi06387500-32 2024-08-17T16:59:07.970 INFO:tasks.rados.rados.0.smithi063.stdout:1047: finishing write tid 4 to smithi06387500-32 2024-08-17T16:59:07.971 INFO:tasks.rados.rados.0.smithi063.stdout:1047: finishing write tid 5 to smithi06387500-32 2024-08-17T16:59:07.971 INFO:tasks.rados.rados.0.smithi063.stdout:1047: oid 32 updating version 0 to 196 2024-08-17T16:59:07.971 INFO:tasks.rados.rados.0.smithi063.stdout:1047: oid 32 version 196 is already newer than 195 2024-08-17T16:59:07.971 INFO:tasks.rados.rados.0.smithi063.stdout:1047: oid 32 updating version 196 to 198 2024-08-17T16:59:07.971 INFO:tasks.rados.rados.0.smithi063.stdout:1047: oid 32 version 198 is already newer than 197 2024-08-17T16:59:07.971 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 198 (ObjNum 423 snap 130 seq_num 423) dirty exists 2024-08-17T16:59:07.971 INFO:tasks.rados.rados.0.smithi063.stdout:1047: left oid 32 (ObjNum 423 snap 130 seq_num 423) 2024-08-17T16:59:07.971 INFO:tasks.rados.rados.0.smithi063.stdout:1049: finishing rollback tid 0 to smithi06387500-8 2024-08-17T16:59:07.971 INFO:tasks.rados.rados.0.smithi063.stdout:1049: finishing rollback tid 1 to smithi06387500-8 2024-08-17T16:59:07.971 INFO:tasks.rados.rados.0.smithi063.stdout:1049: finishing rollback tid 2 to smithi06387500-8 2024-08-17T16:59:07.971 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 297 (ObjNum 333 snap 101 seq_num 333) dirty exists 2024-08-17T16:59:08.196 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:07 smithi162 ceph-mon[34633]: osdmap e331: 8 total, 8 up, 8 in 2024-08-17T16:59:08.196 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:07 smithi162 ceph-mon[34633]: pgmap v180: 89 pgs: 89 active+clean; 106 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 23 MiB/s rd, 1.6 MiB/s wr, 26 op/s 2024-08-17T16:59:08.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:07 smithi063 ceph-mon[42680]: osdmap e331: 8 total, 8 up, 8 in 2024-08-17T16:59:08.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:07 smithi063 ceph-mon[42680]: pgmap v180: 89 pgs: 89 active+clean; 106 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 23 MiB/s rd, 1.6 MiB/s wr, 26 op/s 2024-08-17T16:59:08.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:07 smithi063 ceph-mon[31063]: osdmap e331: 8 total, 8 up, 8 in 2024-08-17T16:59:08.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:07 smithi063 ceph-mon[31063]: pgmap v180: 89 pgs: 89 active+clean; 106 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 23 MiB/s rd, 1.6 MiB/s wr, 26 op/s 2024-08-17T16:59:08.932 INFO:tasks.rados.rados.0.smithi063.stdout:1046: done (4 left) 2024-08-17T16:59:08.932 INFO:tasks.rados.rados.0.smithi063.stdout:1047: done (3 left) 2024-08-17T16:59:08.932 INFO:tasks.rados.rados.0.smithi063.stdout:1048: done (2 left) 2024-08-17T16:59:08.932 INFO:tasks.rados.rados.0.smithi063.stdout:1049: done (1 left) 2024-08-17T16:59:08.932 INFO:tasks.rados.rados.0.smithi063.stdout:1050: done (0 left) 2024-08-17T16:59:08.932 INFO:tasks.rados.rados.0.smithi063.stdout:1051: snap_remove snap 124 2024-08-17T16:59:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: osdmap e332: 8 total, 8 up, 8 in 2024-08-17T16:59:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:59:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-17T16:59:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T16:59:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T16:59:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-17T16:59:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T16:59:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-17T16:59:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-17T16:59:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-17T16:59:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-17T16:59:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-17T16:59:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-17T16:59:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-17T16:59:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-17T16:59:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-17T16:59:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-17T16:59:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-17T16:59:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-17T16:59:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-17T16:59:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T16:59:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-17T16:59:09.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T16:59:09.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:09.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:09.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:09.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:09.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:09.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:09.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:08 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: osdmap e332: 8 total, 8 up, 8 in 2024-08-17T16:59:09.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:09.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:09.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:59:09.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:09.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-17T16:59:09.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T16:59:09.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T16:59:09.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: osdmap e332: 8 total, 8 up, 8 in 2024-08-17T16:59:09.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:09.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:09.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:59:09.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:09.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.283 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.284 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:09.284 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.284 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-17T16:59:09.284 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T16:59:09.284 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.284 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T16:59:09.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-17T16:59:09.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T16:59:09.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-17T16:59:09.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-17T16:59:09.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-17T16:59:09.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-17T16:59:09.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-17T16:59:09.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-17T16:59:09.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-17T16:59:09.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-17T16:59:09.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-17T16:59:09.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-17T16:59:09.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-17T16:59:09.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-17T16:59:09.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-17T16:59:09.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T16:59:09.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.285 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.286 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-17T16:59:09.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T16:59:09.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-17T16:59:09.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-17T16:59:09.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-17T16:59:09.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-17T16:59:09.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-17T16:59:09.287 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-17T16:59:09.287 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.287 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-17T16:59:09.287 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-17T16:59:09.287 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-17T16:59:09.287 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-17T16:59:09.287 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-17T16:59:09.287 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-17T16:59:09.287 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-17T16:59:09.287 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T16:59:09.287 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.287 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.288 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.288 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.288 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-17T16:59:09.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T16:59:09.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:09.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:09.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:09.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:09.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:09.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:09.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T16:59:09.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-17T16:59:09.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T16:59:09.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:09.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:09.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:09.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.290 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:09.290 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:09.290 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:09.290 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:08 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:09.887 INFO:tasks.rados.rados.0.smithi063.stdout:1051: done (0 left) 2024-08-17T16:59:09.887 INFO:tasks.rados.rados.0.smithi063.stdout:1052: snap_create 2024-08-17T16:59:10.181 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[42680]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:59:10.181 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[42680]: Upgrade: Finalizing container_image settings 2024-08-17T16:59:10.181 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[42680]: Upgrade: Complete! 2024-08-17T16:59:10.181 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[42680]: pgmap v182: 89 pgs: 89 active+clean; 106 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 23 MiB/s rd, 1.6 MiB/s wr, 26 op/s 2024-08-17T16:59:10.181 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[42680]: osdmap e333: 8 total, 8 up, 8 in 2024-08-17T16:59:10.181 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:10.181 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:10.181 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[42680]: osdmap e334: 8 total, 8 up, 8 in 2024-08-17T16:59:10.181 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[31063]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:59:10.182 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[31063]: Upgrade: Finalizing container_image settings 2024-08-17T16:59:10.182 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[31063]: Upgrade: Complete! 2024-08-17T16:59:10.182 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[31063]: pgmap v182: 89 pgs: 89 active+clean; 106 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 23 MiB/s rd, 1.6 MiB/s wr, 26 op/s 2024-08-17T16:59:10.182 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[31063]: osdmap e333: 8 total, 8 up, 8 in 2024-08-17T16:59:10.182 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:10.182 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:10.182 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:09 smithi063 ceph-mon[31063]: osdmap e334: 8 total, 8 up, 8 in 2024-08-17T16:59:10.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:09 smithi162 ceph-mon[34633]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-08-17T16:59:10.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:09 smithi162 ceph-mon[34633]: Upgrade: Finalizing container_image settings 2024-08-17T16:59:10.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:09 smithi162 ceph-mon[34633]: Upgrade: Complete! 2024-08-17T16:59:10.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:09 smithi162 ceph-mon[34633]: pgmap v182: 89 pgs: 89 active+clean; 106 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 23 MiB/s rd, 1.6 MiB/s wr, 26 op/s 2024-08-17T16:59:10.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:09 smithi162 ceph-mon[34633]: osdmap e333: 8 total, 8 up, 8 in 2024-08-17T16:59:10.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:10.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:09 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:10.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:09 smithi162 ceph-mon[34633]: osdmap e334: 8 total, 8 up, 8 in 2024-08-17T16:59:10.892 INFO:tasks.rados.rados.0.smithi063.stdout:1052: done (0 left) 2024-08-17T16:59:10.892 INFO:tasks.rados.rados.0.smithi063.stdout:1053: snap_remove snap 131 2024-08-17T16:59:11.897 INFO:tasks.rados.rados.0.smithi063.stdout:1053: done (0 left) 2024-08-17T16:59:11.897 INFO:tasks.rados.rados.0.smithi063.stdout:1054: delete oid 17 current snap is 132 2024-08-17T16:59:11.902 INFO:tasks.rados.rados.0.smithi063.stdout:1054: done (0 left) 2024-08-17T16:59:11.902 INFO:tasks.rados.rados.0.smithi063.stdout:1055: read oid 16 snap -1 2024-08-17T16:59:11.902 INFO:tasks.rados.rados.0.smithi063.stdout:1055: expect (ObjNum 419 snap 129 seq_num 419) 2024-08-17T16:59:11.902 INFO:tasks.rados.rados.0.smithi063.stdout:1056: read oid 42 snap -1 2024-08-17T16:59:11.902 INFO:tasks.rados.rados.0.smithi063.stdout:1056: expect (ObjNum 347 snap 103 seq_num 347) 2024-08-17T16:59:11.902 INFO:tasks.rados.rados.0.smithi063.stdout:1057: snap_remove snap 128 2024-08-17T16:59:12.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:11 smithi063 ceph-mon[31063]: osdmap e335: 8 total, 8 up, 8 in 2024-08-17T16:59:12.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:11 smithi063 ceph-mon[31063]: pgmap v186: 89 pgs: 89 active+clean; 120 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.6 MiB/s wr, 14 op/s 2024-08-17T16:59:12.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:11 smithi063 ceph-mon[42680]: osdmap e335: 8 total, 8 up, 8 in 2024-08-17T16:59:12.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:11 smithi063 ceph-mon[42680]: pgmap v186: 89 pgs: 89 active+clean; 120 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.6 MiB/s wr, 14 op/s 2024-08-17T16:59:12.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:11 smithi162 ceph-mon[34633]: osdmap e335: 8 total, 8 up, 8 in 2024-08-17T16:59:12.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:11 smithi162 ceph-mon[34633]: pgmap v186: 89 pgs: 89 active+clean; 120 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.6 MiB/s wr, 14 op/s 2024-08-17T16:59:12.903 INFO:tasks.rados.rados.0.smithi063.stdout:1057: done (2 left) 2024-08-17T16:59:12.904 INFO:tasks.rados.rados.0.smithi063.stdout:1058: snap_create 2024-08-17T16:59:12.904 INFO:tasks.rados.rados.0.smithi063.stdout:1056: expect (ObjNum 347 snap 103 seq_num 347) 2024-08-17T16:59:13.269 INFO:tasks.rados.rados.0.smithi063.stdout:1055: expect (ObjNum 419 snap 129 seq_num 419) 2024-08-17T16:59:13.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:12 smithi063 ceph-mon[42680]: osdmap e336: 8 total, 8 up, 8 in 2024-08-17T16:59:13.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:12 smithi063 ceph-mon[31063]: osdmap e336: 8 total, 8 up, 8 in 2024-08-17T16:59:13.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:12 smithi162 ceph-mon[34633]: osdmap e336: 8 total, 8 up, 8 in 2024-08-17T16:59:13.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:59:13] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.51.0" 2024-08-17T16:59:13.908 INFO:tasks.rados.rados.0.smithi063.stdout:1055: done (2 left) 2024-08-17T16:59:13.909 INFO:tasks.rados.rados.0.smithi063.stdout:1056: done (1 left) 2024-08-17T16:59:13.909 INFO:tasks.rados.rados.0.smithi063.stdout:1058: done (0 left) 2024-08-17T16:59:13.909 INFO:tasks.rados.rados.0.smithi063.stdout:1059: read oid 27 snap -1 2024-08-17T16:59:13.909 INFO:tasks.rados.rados.0.smithi063.stdout:1059: expect deleted 2024-08-17T16:59:13.909 INFO:tasks.rados.rados.0.smithi063.stdout:1060: read oid 41 snap -1 2024-08-17T16:59:13.909 INFO:tasks.rados.rados.0.smithi063.stdout:1060: expect (ObjNum 73 snap 9 seq_num 73) 2024-08-17T16:59:13.909 INFO:tasks.rados.rados.0.smithi063.stdout:1061: write oid 6 current snap is 133 2024-08-17T16:59:13.909 INFO:tasks.rados.rados.0.smithi063.stdout:1061: seq_num 425 ranges {742975=780240,1953553=427132,2971713=171262} 2024-08-17T16:59:13.917 INFO:tasks.rados.rados.0.smithi063.stdout:1061: writing smithi06387500-6 from 742975 to 1523215 tid 1 2024-08-17T16:59:13.922 INFO:tasks.rados.rados.0.smithi063.stdout:1061: writing smithi06387500-6 from 1953553 to 2380685 tid 2 2024-08-17T16:59:13.926 INFO:tasks.rados.rados.0.smithi063.stdout:1061: writing smithi06387500-6 from 2971713 to 3142975 tid 3 2024-08-17T16:59:13.927 INFO:tasks.rados.rados.0.smithi063.stdout:1062: delete oid 23 current snap is 133 2024-08-17T16:59:13.927 INFO:tasks.rados.rados.0.smithi063.stdout:1060: expect (ObjNum 73 snap 9 seq_num 73) 2024-08-17T16:59:14.069 INFO:tasks.rados.rados.0.smithi063.stdout:1061: finishing write tid 1 to smithi06387500-6 2024-08-17T16:59:14.069 INFO:tasks.rados.rados.0.smithi063.stdout:1061: finishing write tid 2 to smithi06387500-6 2024-08-17T16:59:14.069 INFO:tasks.rados.rados.0.smithi063.stdout:1061: finishing write tid 3 to smithi06387500-6 2024-08-17T16:59:14.069 INFO:tasks.rados.rados.0.smithi063.stdout:1061: finishing write tid 4 to smithi06387500-6 2024-08-17T16:59:14.069 INFO:tasks.rados.rados.0.smithi063.stdout:1061: finishing write tid 5 to smithi06387500-6 2024-08-17T16:59:14.069 INFO:tasks.rados.rados.0.smithi063.stdout:1061: oid 6 updating version 0 to 304 2024-08-17T16:59:14.070 INFO:tasks.rados.rados.0.smithi063.stdout:1061: oid 6 version 304 is already newer than 302 2024-08-17T16:59:14.070 INFO:tasks.rados.rados.0.smithi063.stdout:1061: oid 6 version 304 is already newer than 303 2024-08-17T16:59:14.070 INFO:tasks.rados.rados.0.smithi063.stdout:1061: oid 6 updating version 304 to 305 2024-08-17T16:59:14.070 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 305 (ObjNum 425 snap 133 seq_num 425) dirty exists 2024-08-17T16:59:14.070 INFO:tasks.rados.rados.0.smithi063.stdout:1061: left oid 6 (ObjNum 425 snap 133 seq_num 425) 2024-08-17T16:59:14.070 INFO:tasks.rados.rados.0.smithi063.stdout:1059: done (3 left) 2024-08-17T16:59:14.070 INFO:tasks.rados.rados.0.smithi063.stdout:1060: done (2 left) 2024-08-17T16:59:14.070 INFO:tasks.rados.rados.0.smithi063.stdout:1061: done (1 left) 2024-08-17T16:59:14.070 INFO:tasks.rados.rados.0.smithi063.stdout:1062: done (0 left) 2024-08-17T16:59:14.070 INFO:tasks.rados.rados.0.smithi063.stdout:1063: read oid 37 snap 132 2024-08-17T16:59:14.070 INFO:tasks.rados.rados.0.smithi063.stdout:1063: expect deleted 2024-08-17T16:59:14.070 INFO:tasks.rados.rados.0.smithi063.stdout:1064: read oid 32 snap -1 2024-08-17T16:59:14.071 INFO:tasks.rados.rados.0.smithi063.stdout:1064: expect (ObjNum 423 snap 130 seq_num 423) 2024-08-17T16:59:14.071 INFO:tasks.rados.rados.0.smithi063.stdout:1065: read oid 45 snap -1 2024-08-17T16:59:14.071 INFO:tasks.rados.rados.0.smithi063.stdout:1065: expect deleted 2024-08-17T16:59:14.071 INFO:tasks.rados.rados.0.smithi063.stdout:1066: write oid 24 current snap is 133 2024-08-17T16:59:14.071 INFO:tasks.rados.rados.0.smithi063.stdout:1066: seq_num 427 ranges {776589=647091,2070517=430801,3026559=150030} 2024-08-17T16:59:14.076 INFO:tasks.rados.rados.0.smithi063.stdout:1066: writing smithi06387500-24 from 776589 to 1423680 tid 1 2024-08-17T16:59:14.082 INFO:tasks.rados.rados.0.smithi063.stdout:1066: writing smithi06387500-24 from 2070517 to 2501318 tid 2 2024-08-17T16:59:14.086 INFO:tasks.rados.rados.0.smithi063.stdout:1066: writing smithi06387500-24 from 3026559 to 3176589 tid 3 2024-08-17T16:59:14.087 INFO:tasks.rados.rados.0.smithi063.stdout:1067: read oid 48 snap -1 2024-08-17T16:59:14.087 INFO:tasks.rados.rados.0.smithi063.stdout:1067: expect deleted 2024-08-17T16:59:14.087 INFO:tasks.rados.rados.0.smithi063.stdout:1068: rollback oid 22 current snap is 133 2024-08-17T16:59:14.087 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 22 to 125 2024-08-17T16:59:14.087 INFO:tasks.rados.rados.0.smithi063.stdout:1064: expect (ObjNum 423 snap 130 seq_num 423) 2024-08-17T16:59:14.275 INFO:tasks.rados.rados.0.smithi063.stdout:1066: finishing write tid 1 to smithi06387500-24 2024-08-17T16:59:14.275 INFO:tasks.rados.rados.0.smithi063.stdout:1066: finishing write tid 2 to smithi06387500-24 2024-08-17T16:59:14.276 INFO:tasks.rados.rados.0.smithi063.stdout:1068: finishing rollback tid 1 to smithi06387500-22 2024-08-17T16:59:14.276 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T16:59:14.276 INFO:tasks.rados.rados.0.smithi063.stdout:1066: finishing write tid 3 to smithi06387500-24 2024-08-17T16:59:14.276 INFO:tasks.rados.rados.0.smithi063.stdout:1066: finishing write tid 4 to smithi06387500-24 2024-08-17T16:59:14.276 INFO:tasks.rados.rados.0.smithi063.stdout:1063: done (5 left) 2024-08-17T16:59:14.276 INFO:tasks.rados.rados.0.smithi063.stdout:1064: done (4 left) 2024-08-17T16:59:14.276 INFO:tasks.rados.rados.0.smithi063.stdout:1065: done (3 left) 2024-08-17T16:59:14.276 INFO:tasks.rados.rados.0.smithi063.stdout:1067: done (2 left) 2024-08-17T16:59:14.276 INFO:tasks.rados.rados.0.smithi063.stdout:1068: done (1 left) 2024-08-17T16:59:14.276 INFO:tasks.rados.rados.0.smithi063.stdout:1069: read oid 37 snap -1 2024-08-17T16:59:14.276 INFO:tasks.rados.rados.0.smithi063.stdout:1069: expect deleted 2024-08-17T16:59:14.276 INFO:tasks.rados.rados.0.smithi063.stdout:1070: rollback oid 48 current snap is 133 2024-08-17T16:59:14.276 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 48 to 129 2024-08-17T16:59:14.277 INFO:tasks.rados.rados.0.smithi063.stdout:1066: finishing write tid 5 to smithi06387500-24 2024-08-17T16:59:14.277 INFO:tasks.rados.rados.0.smithi063.stdout:1066: oid 24 updating version 0 to 395 2024-08-17T16:59:14.277 INFO:tasks.rados.rados.0.smithi063.stdout:1066: oid 24 updating version 395 to 396 2024-08-17T16:59:14.277 INFO:tasks.rados.rados.0.smithi063.stdout:1066: oid 24 updating version 396 to 398 2024-08-17T16:59:14.277 INFO:tasks.rados.rados.0.smithi063.stdout:1066: oid 24 version 398 is already newer than 397 2024-08-17T16:59:14.277 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 398 (ObjNum 427 snap 133 seq_num 427) dirty exists 2024-08-17T16:59:14.277 INFO:tasks.rados.rados.0.smithi063.stdout:1066: left oid 24 (ObjNum 427 snap 133 seq_num 427) 2024-08-17T16:59:14.277 INFO:tasks.rados.rados.0.smithi063.stdout:1066: done (2 left) 2024-08-17T16:59:14.277 INFO:tasks.rados.rados.0.smithi063.stdout:1071: write oid 42 current snap is 133 2024-08-17T16:59:14.277 INFO:tasks.rados.rados.0.smithi063.stdout:1071: seq_num 428 ranges {793396=722340,2128999=690812} 2024-08-17T16:59:14.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:13 smithi063 ceph-mon[31063]: osdmap e337: 8 total, 8 up, 8 in 2024-08-17T16:59:14.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:13 smithi063 ceph-mon[31063]: pgmap v189: 89 pgs: 89 active+clean; 120 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.6 MiB/s wr, 14 op/s 2024-08-17T16:59:14.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:13 smithi063 ceph-mon[42680]: osdmap e337: 8 total, 8 up, 8 in 2024-08-17T16:59:14.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:13 smithi063 ceph-mon[42680]: pgmap v189: 89 pgs: 89 active+clean; 120 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.6 MiB/s wr, 14 op/s 2024-08-17T16:59:14.283 INFO:tasks.rados.rados.0.smithi063.stdout:1071: writing smithi06387500-42 from 793396 to 1515736 tid 1 2024-08-17T16:59:14.290 INFO:tasks.rados.rados.0.smithi063.stdout:1071: writing smithi06387500-42 from 2128999 to 2819811 tid 2 2024-08-17T16:59:14.291 INFO:tasks.rados.rados.0.smithi063.stdout:1072: snap_create 2024-08-17T16:59:14.291 INFO:tasks.rados.rados.0.smithi063.stdout:1070: finishing rollback tid 1 to smithi06387500-48 2024-08-17T16:59:14.291 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 0 (ObjNum 528 snap 0 seq_num 304) dirty dne 2024-08-17T16:59:14.291 INFO:tasks.rados.rados.0.smithi063.stdout:1071: finishing write tid 1 to smithi06387500-42 2024-08-17T16:59:14.293 INFO:tasks.rados.rados.0.smithi063.stdout:1071: finishing write tid 2 to smithi06387500-42 2024-08-17T16:59:14.293 INFO:tasks.rados.rados.0.smithi063.stdout:1071: finishing write tid 3 to smithi06387500-42 2024-08-17T16:59:14.293 INFO:tasks.rados.rados.0.smithi063.stdout:1071: finishing write tid 4 to smithi06387500-42 2024-08-17T16:59:14.294 INFO:tasks.rados.rados.0.smithi063.stdout:1071: oid 42 updating version 0 to 748 2024-08-17T16:59:14.294 INFO:tasks.rados.rados.0.smithi063.stdout:1071: oid 42 updating version 748 to 749 2024-08-17T16:59:14.294 INFO:tasks.rados.rados.0.smithi063.stdout:1071: oid 42 updating version 749 to 750 2024-08-17T16:59:14.294 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 750 (ObjNum 428 snap 133 seq_num 428) dirty exists 2024-08-17T16:59:14.294 INFO:tasks.rados.rados.0.smithi063.stdout:1071: left oid 42 (ObjNum 428 snap 133 seq_num 428) 2024-08-17T16:59:14.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:13 smithi162 ceph-mon[34633]: osdmap e337: 8 total, 8 up, 8 in 2024-08-17T16:59:14.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:13 smithi162 ceph-mon[34633]: pgmap v189: 89 pgs: 89 active+clean; 120 MiB data, 531 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.6 MiB/s wr, 14 op/s 2024-08-17T16:59:14.888 INFO:tasks.rados.rados.0.smithi063.stdout:1069: done (3 left) 2024-08-17T16:59:14.889 INFO:tasks.rados.rados.0.smithi063.stdout:1070: done (2 left) 2024-08-17T16:59:14.889 INFO:tasks.rados.rados.0.smithi063.stdout:1071: done (1 left) 2024-08-17T16:59:14.889 INFO:tasks.rados.rados.0.smithi063.stdout:1072: done (0 left) 2024-08-17T16:59:14.889 INFO:tasks.rados.rados.0.smithi063.stdout:1073: snap_create 2024-08-17T16:59:15.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:14 smithi063 ceph-mon[42680]: osdmap e338: 8 total, 8 up, 8 in 2024-08-17T16:59:15.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:14 smithi063 ceph-mon[42680]: osdmap e339: 8 total, 8 up, 8 in 2024-08-17T16:59:15.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:14 smithi063 ceph-mon[31063]: osdmap e338: 8 total, 8 up, 8 in 2024-08-17T16:59:15.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:14 smithi063 ceph-mon[31063]: osdmap e339: 8 total, 8 up, 8 in 2024-08-17T16:59:15.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:14 smithi162 ceph-mon[34633]: osdmap e338: 8 total, 8 up, 8 in 2024-08-17T16:59:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:14 smithi162 ceph-mon[34633]: osdmap e339: 8 total, 8 up, 8 in 2024-08-17T16:59:15.889 INFO:tasks.rados.rados.0.smithi063.stdout:1073: done (0 left) 2024-08-17T16:59:15.889 INFO:tasks.rados.rados.0.smithi063.stdout:1074: read oid 47 snap -1 2024-08-17T16:59:15.889 INFO:tasks.rados.rados.0.smithi063.stdout:1074: expect (ObjNum 416 snap 128 seq_num 416) 2024-08-17T16:59:15.889 INFO:tasks.rados.rados.0.smithi063.stdout:1075: write oid 26 current snap is 135 2024-08-17T16:59:15.889 INFO:tasks.rados.rados.0.smithi063.stdout:1075: seq_num 429 ranges {410203=797589,1787481=434470,2681405=528798} 2024-08-17T16:59:15.896 INFO:tasks.rados.rados.0.smithi063.stdout:1075: writing smithi06387500-26 from 410203 to 1207792 tid 1 2024-08-17T16:59:15.902 INFO:tasks.rados.rados.0.smithi063.stdout:1075: writing smithi06387500-26 from 1787481 to 2221951 tid 2 2024-08-17T16:59:15.907 INFO:tasks.rados.rados.0.smithi063.stdout:1075: writing smithi06387500-26 from 2681405 to 3210203 tid 3 2024-08-17T16:59:15.907 INFO:tasks.rados.rados.0.smithi063.stdout:1076: write oid 49 current snap is 135 2024-08-17T16:59:15.907 INFO:tasks.rados.rados.0.smithi063.stdout:1076: seq_num 430 ranges {427010=472838,1329610=694481,2708828=518182} 2024-08-17T16:59:15.912 INFO:tasks.rados.rados.0.smithi063.stdout:1076: writing smithi06387500-49 from 427010 to 899848 tid 1 2024-08-17T16:59:15.918 INFO:tasks.rados.rados.0.smithi063.stdout:1076: writing smithi06387500-49 from 1329610 to 2024091 tid 2 2024-08-17T16:59:15.925 INFO:tasks.rados.rados.0.smithi063.stdout:1076: writing smithi06387500-49 from 2708828 to 3227010 tid 3 2024-08-17T16:59:15.925 INFO:tasks.rados.rados.0.smithi063.stdout:1077: write oid 2 current snap is 135 2024-08-17T16:59:15.925 INFO:tasks.rados.rados.0.smithi063.stdout:1077: seq_num 431 ranges {443817=548087,1788092=438139,3019898=223919} 2024-08-17T16:59:15.930 INFO:tasks.rados.rados.0.smithi063.stdout:1077: writing smithi06387500-2 from 443817 to 991904 tid 1 2024-08-17T16:59:15.937 INFO:tasks.rados.rados.0.smithi063.stdout:1077: writing smithi06387500-2 from 1788092 to 2226231 tid 2 2024-08-17T16:59:15.943 INFO:tasks.rados.rados.0.smithi063.stdout:1077: writing smithi06387500-2 from 3019898 to 3243817 tid 3 2024-08-17T16:59:15.943 INFO:tasks.rados.rados.0.smithi063.stdout:1078: read oid 36 snap -1 2024-08-17T16:59:15.943 INFO:tasks.rados.rados.0.smithi063.stdout:1078: expect (ObjNum 398 snap 121 seq_num 398) 2024-08-17T16:59:15.943 INFO:tasks.rados.rados.0.smithi063.stdout:1079: read oid 16 snap -1 2024-08-17T16:59:15.943 INFO:tasks.rados.rados.0.smithi063.stdout:1079: expect (ObjNum 419 snap 129 seq_num 419) 2024-08-17T16:59:15.943 INFO:tasks.rados.rados.0.smithi063.stdout:1080: rollback oid 4 current snap is 135 2024-08-17T16:59:15.943 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 4 to 121 2024-08-17T16:59:15.944 INFO:tasks.rados.rados.0.smithi063.stdout:1074: expect (ObjNum 416 snap 128 seq_num 416) 2024-08-17T16:59:16.118 INFO:tasks.rados.rados.0.smithi063.stdout:1075: finishing write tid 1 to smithi06387500-26 2024-08-17T16:59:16.118 INFO:tasks.rados.rados.0.smithi063.stdout:1075: finishing write tid 2 to smithi06387500-26 2024-08-17T16:59:16.118 INFO:tasks.rados.rados.0.smithi063.stdout:1075: finishing write tid 3 to smithi06387500-26 2024-08-17T16:59:16.118 INFO:tasks.rados.rados.0.smithi063.stdout:1075: finishing write tid 4 to smithi06387500-26 2024-08-17T16:59:16.118 INFO:tasks.rados.rados.0.smithi063.stdout:1075: finishing write tid 5 to smithi06387500-26 2024-08-17T16:59:16.118 INFO:tasks.rados.rados.0.smithi063.stdout:1075: oid 26 updating version 0 to 353 2024-08-17T16:59:16.118 INFO:tasks.rados.rados.0.smithi063.stdout:1075: oid 26 version 353 is already newer than 351 2024-08-17T16:59:16.118 INFO:tasks.rados.rados.0.smithi063.stdout:1075: oid 26 updating version 353 to 354 2024-08-17T16:59:16.118 INFO:tasks.rados.rados.0.smithi063.stdout:1075: oid 26 version 354 is already newer than 352 2024-08-17T16:59:16.118 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 354 (ObjNum 429 snap 135 seq_num 429) dirty exists 2024-08-17T16:59:16.118 INFO:tasks.rados.rados.0.smithi063.stdout:1075: left oid 26 (ObjNum 429 snap 135 seq_num 429) 2024-08-17T16:59:16.119 INFO:tasks.rados.rados.0.smithi063.stdout:1076: finishing write tid 1 to smithi06387500-49 2024-08-17T16:59:16.119 INFO:tasks.rados.rados.0.smithi063.stdout:1074: done (6 left) 2024-08-17T16:59:16.119 INFO:tasks.rados.rados.0.smithi063.stdout:1075: done (5 left) 2024-08-17T16:59:16.119 INFO:tasks.rados.rados.0.smithi063.stdout:1081: write oid 21 current snap is 135 2024-08-17T16:59:16.119 INFO:tasks.rados.rados.0.smithi063.stdout:1076: finishing write tid 2 to smithi06387500-49 2024-08-17T16:59:16.119 INFO:tasks.rados.rados.0.smithi063.stdout:1081: seq_num 432 ranges {460624=623336,1846574=698150,3163674=96950} 2024-08-17T16:59:16.123 INFO:tasks.rados.rados.0.smithi063.stdout:1081: writing smithi06387500-21 from 460624 to 1083960 tid 1 2024-08-17T16:59:16.131 INFO:tasks.rados.rados.0.smithi063.stdout:1081: writing smithi06387500-21 from 1846574 to 2544724 tid 2 2024-08-17T16:59:16.135 INFO:tasks.rados.rados.0.smithi063.stdout:1081: writing smithi06387500-21 from 3163674 to 3260624 tid 3 2024-08-17T16:59:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:1082: snap_create 2024-08-17T16:59:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:1076: finishing write tid 3 to smithi06387500-49 2024-08-17T16:59:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:1076: finishing write tid 4 to smithi06387500-49 2024-08-17T16:59:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:1076: finishing write tid 5 to smithi06387500-49 2024-08-17T16:59:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:1076: oid 49 updating version 0 to 1105 2024-08-17T16:59:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:1076: oid 49 version 1105 is already newer than 1104 2024-08-17T16:59:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:1076: oid 49 version 1105 is already newer than 1103 2024-08-17T16:59:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:1076: oid 49 updating version 1105 to 1106 2024-08-17T16:59:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 1106 (ObjNum 430 snap 135 seq_num 430) dirty exists 2024-08-17T16:59:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:1076: left oid 49 (ObjNum 430 snap 135 seq_num 430) 2024-08-17T16:59:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:1077: finishing write tid 1 to smithi06387500-2 2024-08-17T16:59:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:1077: finishing write tid 2 to smithi06387500-2 2024-08-17T16:59:16.136 INFO:tasks.rados.rados.0.smithi063.stdout:1077: finishing write tid 3 to smithi06387500-2 2024-08-17T16:59:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:1077: finishing write tid 4 to smithi06387500-2 2024-08-17T16:59:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:1077: finishing write tid 5 to smithi06387500-2 2024-08-17T16:59:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:1077: oid 2 updating version 0 to 519 2024-08-17T16:59:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:1077: oid 2 version 519 is already newer than 518 2024-08-17T16:59:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:1077: oid 2 version 519 is already newer than 516 2024-08-17T16:59:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:1077: oid 2 version 519 is already newer than 517 2024-08-17T16:59:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 519 (ObjNum 431 snap 135 seq_num 431) dirty exists 2024-08-17T16:59:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:1077: left oid 2 (ObjNum 431 snap 135 seq_num 431) 2024-08-17T16:59:16.137 INFO:tasks.rados.rados.0.smithi063.stdout:1079: expect (ObjNum 419 snap 129 seq_num 419) 2024-08-17T16:59:16.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:15 smithi063 ceph-mon[42680]: pgmap v192: 89 pgs: 89 active+clean; 123 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 3.6 MiB/s rd, 337 KiB/s wr, 3 op/s 2024-08-17T16:59:16.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:15 smithi063 ceph-mon[42680]: osdmap e340: 8 total, 8 up, 8 in 2024-08-17T16:59:16.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:15 smithi063 ceph-mon[31063]: pgmap v192: 89 pgs: 89 active+clean; 123 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 3.6 MiB/s rd, 337 KiB/s wr, 3 op/s 2024-08-17T16:59:16.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:15 smithi063 ceph-mon[31063]: osdmap e340: 8 total, 8 up, 8 in 2024-08-17T16:59:16.320 INFO:tasks.rados.rados.0.smithi063.stdout:1078: expect (ObjNum 398 snap 121 seq_num 398) 2024-08-17T16:59:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:15 smithi162 ceph-mon[34633]: pgmap v192: 89 pgs: 89 active+clean; 123 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 3.6 MiB/s rd, 337 KiB/s wr, 3 op/s 2024-08-17T16:59:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:15 smithi162 ceph-mon[34633]: osdmap e340: 8 total, 8 up, 8 in 2024-08-17T16:59:16.405 INFO:tasks.rados.rados.0.smithi063.stdout:1080: finishing rollback tid 0 to smithi06387500-4 2024-08-17T16:59:16.405 INFO:tasks.rados.rados.0.smithi063.stdout:1080: finishing rollback tid 1 to smithi06387500-4 2024-08-17T16:59:16.406 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 1109 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T16:59:16.406 INFO:tasks.rados.rados.0.smithi063.stdout:1081: finishing write tid 1 to smithi06387500-21 2024-08-17T16:59:16.406 INFO:tasks.rados.rados.0.smithi063.stdout:1081: finishing write tid 2 to smithi06387500-21 2024-08-17T16:59:16.406 INFO:tasks.rados.rados.0.smithi063.stdout:1081: finishing write tid 3 to smithi06387500-21 2024-08-17T16:59:16.406 INFO:tasks.rados.rados.0.smithi063.stdout:1081: finishing write tid 4 to smithi06387500-21 2024-08-17T16:59:16.406 INFO:tasks.rados.rados.0.smithi063.stdout:1081: finishing write tid 5 to smithi06387500-21 2024-08-17T16:59:16.406 INFO:tasks.rados.rados.0.smithi063.stdout:1081: oid 21 updating version 0 to 753 2024-08-17T16:59:16.406 INFO:tasks.rados.rados.0.smithi063.stdout:1081: oid 21 updating version 753 to 754 2024-08-17T16:59:16.406 INFO:tasks.rados.rados.0.smithi063.stdout:1081: oid 21 version 754 is already newer than 752 2024-08-17T16:59:16.406 INFO:tasks.rados.rados.0.smithi063.stdout:1081: oid 21 version 754 is already newer than 751 2024-08-17T16:59:16.406 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 754 (ObjNum 432 snap 135 seq_num 432) dirty exists 2024-08-17T16:59:16.406 INFO:tasks.rados.rados.0.smithi063.stdout:1081: left oid 21 (ObjNum 432 snap 135 seq_num 432) 2024-08-17T16:59:16.917 INFO:tasks.rados.rados.0.smithi063.stdout:1076: done (6 left) 2024-08-17T16:59:16.917 INFO:tasks.rados.rados.0.smithi063.stdout:1077: done (5 left) 2024-08-17T16:59:16.917 INFO:tasks.rados.rados.0.smithi063.stdout:1078: done (4 left) 2024-08-17T16:59:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:1079: done (3 left) 2024-08-17T16:59:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:1080: done (2 left) 2024-08-17T16:59:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:1081: done (1 left) 2024-08-17T16:59:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:1082: done (0 left) 2024-08-17T16:59:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:1083: rollback oid 17 current snap is 136 2024-08-17T16:59:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 17 to 135 2024-08-17T16:59:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:1084: rollback oid 23 current snap is 136 2024-08-17T16:59:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 23 to 126 2024-08-17T16:59:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:1085: read oid 24 snap 129 2024-08-17T16:59:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:1085: expect (ObjNum 286 snap 87 seq_num 286) 2024-08-17T16:59:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:1086: read oid 20 snap -1 2024-08-17T16:59:16.918 INFO:tasks.rados.rados.0.smithi063.stdout:1086: expect (ObjNum 417 snap 128 seq_num 417) 2024-08-17T16:59:16.919 INFO:tasks.rados.rados.0.smithi063.stdout:1087: read oid 21 snap -1 2024-08-17T16:59:16.919 INFO:tasks.rados.rados.0.smithi063.stdout:1087: expect (ObjNum 432 snap 135 seq_num 432) 2024-08-17T16:59:16.919 INFO:tasks.rados.rados.0.smithi063.stdout:1088: read oid 34 snap -1 2024-08-17T16:59:16.919 INFO:tasks.rados.rados.0.smithi063.stdout:1088: expect (ObjNum 403 snap 122 seq_num 403) 2024-08-17T16:59:16.919 INFO:tasks.rados.rados.0.smithi063.stdout:1089: read oid 1 snap -1 2024-08-17T16:59:16.919 INFO:tasks.rados.rados.0.smithi063.stdout:1089: expect (ObjNum 391 snap 119 seq_num 391) 2024-08-17T16:59:16.919 INFO:tasks.rados.rados.0.smithi063.stdout:1090: write oid 48 current snap is 136 2024-08-17T16:59:16.919 INFO:tasks.rados.rados.0.smithi063.stdout:1090: seq_num 433 ranges {477431=698585,1905056=441808,3074744=202687} 2024-08-17T16:59:16.924 INFO:tasks.rados.rados.0.smithi063.stdout:1090: writing smithi06387500-48 from 477431 to 1176016 tid 1 2024-08-17T16:59:16.931 INFO:tasks.rados.rados.0.smithi063.stdout:1090: writing smithi06387500-48 from 1905056 to 2346864 tid 2 2024-08-17T16:59:16.937 INFO:tasks.rados.rados.0.smithi063.stdout:1090: writing smithi06387500-48 from 3074744 to 3277431 tid 3 2024-08-17T16:59:16.937 INFO:tasks.rados.rados.0.smithi063.stdout:1091: rollback oid 2 current snap is 136 2024-08-17T16:59:16.937 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 2 to 125 2024-08-17T16:59:16.937 INFO:tasks.rados.rados.0.smithi063.stdout:1084: finishing rollback tid 1 to smithi06387500-23 2024-08-17T16:59:16.937 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 0 (ObjNum 49 snap 0 seq_num 1806940608) dirty dne 2024-08-17T16:59:16.937 INFO:tasks.rados.rados.0.smithi063.stdout:1084: done (8 left) 2024-08-17T16:59:16.937 INFO:tasks.rados.rados.0.smithi063.stdout:1092: delete oid 42 current snap is 136 2024-08-17T16:59:16.937 INFO:tasks.rados.rados.0.smithi063.stdout:1083: finishing rollback tid 1 to smithi06387500-17 2024-08-17T16:59:16.937 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 348 (ObjNum 574598368 snap 32710 seq_num 304) dirty dne 2024-08-17T16:59:16.937 INFO:tasks.rados.rados.0.smithi063.stdout:1089: expect (ObjNum 391 snap 119 seq_num 391) 2024-08-17T16:59:17.096 INFO:tasks.rados.rados.0.smithi063.stdout:1087: expect (ObjNum 432 snap 135 seq_num 432) 2024-08-17T16:59:17.205 INFO:tasks.rados.rados.0.smithi063.stdout:1085: expect (ObjNum 286 snap 87 seq_num 286) 2024-08-17T16:59:17.297 INFO:tasks.rados.rados.0.smithi063.stdout:1086: expect (ObjNum 417 snap 128 seq_num 417) 2024-08-17T16:59:17.446 INFO:tasks.rados.rados.0.smithi063.stdout:1088: expect (ObjNum 403 snap 122 seq_num 403) 2024-08-17T16:59:17.599 INFO:tasks.rados.rados.0.smithi063.stdout:1090: finishing write tid 1 to smithi06387500-48 2024-08-17T16:59:17.599 INFO:tasks.rados.rados.0.smithi063.stdout:1090: finishing write tid 2 to smithi06387500-48 2024-08-17T16:59:17.599 INFO:tasks.rados.rados.0.smithi063.stdout:1090: finishing write tid 3 to smithi06387500-48 2024-08-17T16:59:17.599 INFO:tasks.rados.rados.0.smithi063.stdout:1090: finishing write tid 4 to smithi06387500-48 2024-08-17T16:59:17.599 INFO:tasks.rados.rados.0.smithi063.stdout:1091: finishing rollback tid 0 to smithi06387500-2 2024-08-17T16:59:17.599 INFO:tasks.rados.rados.0.smithi063.stdout:1090: finishing write tid 5 to smithi06387500-48 2024-08-17T16:59:17.599 INFO:tasks.rados.rados.0.smithi063.stdout:1090: oid 48 updating version 0 to 1112 2024-08-17T16:59:17.599 INFO:tasks.rados.rados.0.smithi063.stdout:1090: oid 48 version 1112 is already newer than 1110 2024-08-17T16:59:17.599 INFO:tasks.rados.rados.0.smithi063.stdout:1090: oid 48 version 1112 is already newer than 1111 2024-08-17T16:59:17.599 INFO:tasks.rados.rados.0.smithi063.stdout:1090: oid 48 updating version 1112 to 1113 2024-08-17T16:59:17.599 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 1113 (ObjNum 433 snap 136 seq_num 433) dirty exists 2024-08-17T16:59:17.600 INFO:tasks.rados.rados.0.smithi063.stdout:1090: left oid 48 (ObjNum 433 snap 136 seq_num 433) 2024-08-17T16:59:17.600 INFO:tasks.rados.rados.0.smithi063.stdout:1091: finishing rollback tid 1 to smithi06387500-2 2024-08-17T16:59:17.600 INFO:tasks.rados.rados.0.smithi063.stdout:1091: finishing rollback tid 2 to smithi06387500-2 2024-08-17T16:59:17.600 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 523 (ObjNum 340 snap 102 seq_num 340) dirty exists 2024-08-17T16:59:17.600 INFO:tasks.rados.rados.0.smithi063.stdout:1083: done (8 left) 2024-08-17T16:59:17.600 INFO:tasks.rados.rados.0.smithi063.stdout:1085: done (7 left) 2024-08-17T16:59:17.600 INFO:tasks.rados.rados.0.smithi063.stdout:1086: done (6 left) 2024-08-17T16:59:17.600 INFO:tasks.rados.rados.0.smithi063.stdout:1087: done (5 left) 2024-08-17T16:59:17.600 INFO:tasks.rados.rados.0.smithi063.stdout:1088: done (4 left) 2024-08-17T16:59:17.600 INFO:tasks.rados.rados.0.smithi063.stdout:1089: done (3 left) 2024-08-17T16:59:17.600 INFO:tasks.rados.rados.0.smithi063.stdout:1090: done (2 left) 2024-08-17T16:59:17.600 INFO:tasks.rados.rados.0.smithi063.stdout:1091: done (1 left) 2024-08-17T16:59:17.600 INFO:tasks.rados.rados.0.smithi063.stdout:1092: done (0 left) 2024-08-17T16:59:17.601 INFO:tasks.rados.rados.0.smithi063.stdout:1093: rollback oid 7 current snap is 136 2024-08-17T16:59:17.601 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 127 2024-08-17T16:59:17.601 INFO:tasks.rados.rados.0.smithi063.stdout:1094: snap_create 2024-08-17T16:59:17.601 INFO:tasks.rados.rados.0.smithi063.stdout:1093: finishing rollback tid 0 to smithi06387500-7 2024-08-17T16:59:17.602 INFO:tasks.rados.rados.0.smithi063.stdout:1093: finishing rollback tid 1 to smithi06387500-7 2024-08-17T16:59:17.602 INFO:tasks.rados.rados.0.smithi063.stdout:1093: finishing rollback tid 2 to smithi06387500-7 2024-08-17T16:59:17.602 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 628 (ObjNum 396 snap 121 seq_num 396) dirty exists 2024-08-17T16:59:17.922 INFO:tasks.rados.rados.0.smithi063.stdout:1093: done (1 left) 2024-08-17T16:59:17.922 INFO:tasks.rados.rados.0.smithi063.stdout:1094: done (0 left) 2024-08-17T16:59:17.922 INFO:tasks.rados.rados.0.smithi063.stdout:1095: rollback oid 11 current snap is 137 2024-08-17T16:59:17.922 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 11 to 126 2024-08-17T16:59:17.922 INFO:tasks.rados.rados.0.smithi063.stdout:1096: snap_remove snap 135 2024-08-17T16:59:18.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:17 smithi063 ceph-mon[42680]: osdmap e341: 8 total, 8 up, 8 in 2024-08-17T16:59:18.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:17 smithi063 ceph-mon[42680]: pgmap v195: 89 pgs: 89 active+clean; 126 MiB data, 539 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 982 KiB/s wr, 12 op/s 2024-08-17T16:59:18.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:17 smithi063 ceph-mon[31063]: osdmap e341: 8 total, 8 up, 8 in 2024-08-17T16:59:18.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:17 smithi063 ceph-mon[31063]: pgmap v195: 89 pgs: 89 active+clean; 126 MiB data, 539 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 982 KiB/s wr, 12 op/s 2024-08-17T16:59:18.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:17 smithi162 ceph-mon[34633]: osdmap e341: 8 total, 8 up, 8 in 2024-08-17T16:59:18.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:17 smithi162 ceph-mon[34633]: pgmap v195: 89 pgs: 89 active+clean; 126 MiB data, 539 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 982 KiB/s wr, 12 op/s 2024-08-17T16:59:18.931 INFO:tasks.rados.rados.0.smithi063.stdout:1096: done (1 left) 2024-08-17T16:59:18.931 INFO:tasks.rados.rados.0.smithi063.stdout:1097: write oid 26 current snap is 137 2024-08-17T16:59:18.931 INFO:tasks.rados.rados.0.smithi063.stdout:1095: finishing rollback tid 0 to smithi06387500-11 2024-08-17T16:59:18.931 INFO:tasks.rados.rados.0.smithi063.stdout:1097: seq_num 435 ranges {511045=565436,1622020=445477,2729590=556855} 2024-08-17T16:59:18.936 INFO:tasks.rados.rados.0.smithi063.stdout:1097: writing smithi06387500-26 from 511045 to 1076481 tid 1 2024-08-17T16:59:18.942 INFO:tasks.rados.rados.0.smithi063.stdout:1097: writing smithi06387500-26 from 1622020 to 2067497 tid 2 2024-08-17T16:59:18.949 INFO:tasks.rados.rados.0.smithi063.stdout:1097: writing smithi06387500-26 from 2729590 to 3286445 tid 3 2024-08-17T16:59:18.949 INFO:tasks.rados.rados.0.smithi063.stdout:1095: finishing rollback tid 1 to smithi06387500-11 2024-08-17T16:59:18.949 INFO:tasks.rados.rados.0.smithi063.stdout:1095: finishing rollback tid 2 to smithi06387500-11 2024-08-17T16:59:18.949 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 127 (ObjNum 394 snap 119 seq_num 394) dirty exists 2024-08-17T16:59:18.949 INFO:tasks.rados.rados.0.smithi063.stdout:1098: read oid 19 snap -1 2024-08-17T16:59:18.949 INFO:tasks.rados.rados.0.smithi063.stdout:1098: expect (ObjNum 334 snap 101 seq_num 334) 2024-08-17T16:59:18.950 INFO:tasks.rados.rados.0.smithi063.stdout:1095: done (2 left) 2024-08-17T16:59:18.950 INFO:tasks.rados.rados.0.smithi063.stdout:1099: write oid 10 current snap is 137 2024-08-17T16:59:18.950 INFO:tasks.rados.rados.0.smithi063.stdout:1097: finishing write tid 1 to smithi06387500-26 2024-08-17T16:59:18.950 INFO:tasks.rados.rados.0.smithi063.stdout:1097: finishing write tid 2 to smithi06387500-26 2024-08-17T16:59:18.950 INFO:tasks.rados.rados.0.smithi063.stdout:1099: seq_num 436 ranges {527852=640685,1680502=589135,2757013=570839} 2024-08-17T16:59:18.955 INFO:tasks.rados.rados.0.smithi063.stdout:1099: writing smithi06387500-10 from 527852 to 1168537 tid 1 2024-08-17T16:59:18.961 INFO:tasks.rados.rados.0.smithi063.stdout:1099: writing smithi06387500-10 from 1680502 to 2269637 tid 2 2024-08-17T16:59:18.967 INFO:tasks.rados.rados.0.smithi063.stdout:1099: writing smithi06387500-10 from 2757013 to 3327852 tid 3 2024-08-17T16:59:18.968 INFO:tasks.rados.rados.0.smithi063.stdout:1100: write oid 5 current snap is 137 2024-08-17T16:59:18.968 INFO:tasks.rados.rados.0.smithi063.stdout:1100: seq_num 437 ranges {544659=715934,1738984=449146,2784436=560223} 2024-08-17T16:59:18.974 INFO:tasks.rados.rados.0.smithi063.stdout:1100: writing smithi06387500-5 from 544659 to 1260593 tid 1 2024-08-17T16:59:18.979 INFO:tasks.rados.rados.0.smithi063.stdout:1100: writing smithi06387500-5 from 1738984 to 2188130 tid 2 2024-08-17T16:59:18.986 INFO:tasks.rados.rados.0.smithi063.stdout:1100: writing smithi06387500-5 from 2784436 to 3344659 tid 3 2024-08-17T16:59:18.986 INFO:tasks.rados.rados.0.smithi063.stdout:1101: read oid 7 snap -1 2024-08-17T16:59:18.986 INFO:tasks.rados.rados.0.smithi063.stdout:1101: expect (ObjNum 396 snap 121 seq_num 396) 2024-08-17T16:59:18.986 INFO:tasks.rados.rados.0.smithi063.stdout:1102: rollback oid 25 current snap is 137 2024-08-17T16:59:18.987 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 25 to 133 2024-08-17T16:59:18.987 INFO:tasks.rados.rados.0.smithi063.stdout:1097: finishing write tid 3 to smithi06387500-26 2024-08-17T16:59:18.987 INFO:tasks.rados.rados.0.smithi063.stdout:1103: delete oid 40 current snap is 137 2024-08-17T16:59:18.987 INFO:tasks.rados.rados.0.smithi063.stdout:1097: finishing write tid 4 to smithi06387500-26 2024-08-17T16:59:18.987 INFO:tasks.rados.rados.0.smithi063.stdout:1097: finishing write tid 5 to smithi06387500-26 2024-08-17T16:59:18.987 INFO:tasks.rados.rados.0.smithi063.stdout:1097: oid 26 updating version 0 to 358 2024-08-17T16:59:18.987 INFO:tasks.rados.rados.0.smithi063.stdout:1097: oid 26 version 358 is already newer than 357 2024-08-17T16:59:18.987 INFO:tasks.rados.rados.0.smithi063.stdout:1097: oid 26 updating version 358 to 360 2024-08-17T16:59:18.987 INFO:tasks.rados.rados.0.smithi063.stdout:1097: oid 26 version 360 is already newer than 359 2024-08-17T16:59:18.987 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 360 (ObjNum 435 snap 137 seq_num 435) dirty exists 2024-08-17T16:59:18.987 INFO:tasks.rados.rados.0.smithi063.stdout:1097: left oid 26 (ObjNum 435 snap 137 seq_num 435) 2024-08-17T16:59:18.987 INFO:tasks.rados.rados.0.smithi063.stdout:1098: expect (ObjNum 334 snap 101 seq_num 334) 2024-08-17T16:59:19.153 INFO:tasks.rados.rados.0.smithi063.stdout:1099: finishing write tid 1 to smithi06387500-10 2024-08-17T16:59:19.153 INFO:tasks.rados.rados.0.smithi063.stdout:1099: finishing write tid 2 to smithi06387500-10 2024-08-17T16:59:19.153 INFO:tasks.rados.rados.0.smithi063.stdout:1099: finishing write tid 3 to smithi06387500-10 2024-08-17T16:59:19.153 INFO:tasks.rados.rados.0.smithi063.stdout:1099: finishing write tid 4 to smithi06387500-10 2024-08-17T16:59:19.153 INFO:tasks.rados.rados.0.smithi063.stdout:1099: finishing write tid 5 to smithi06387500-10 2024-08-17T16:59:19.153 INFO:tasks.rados.rados.0.smithi063.stdout:1099: oid 10 updating version 0 to 395 2024-08-17T16:59:19.153 INFO:tasks.rados.rados.0.smithi063.stdout:1099: oid 10 version 395 is already newer than 392 2024-08-17T16:59:19.153 INFO:tasks.rados.rados.0.smithi063.stdout:1099: oid 10 version 395 is already newer than 393 2024-08-17T16:59:19.154 INFO:tasks.rados.rados.0.smithi063.stdout:1099: oid 10 version 395 is already newer than 394 2024-08-17T16:59:19.154 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 395 (ObjNum 436 snap 137 seq_num 436) dirty exists 2024-08-17T16:59:19.154 INFO:tasks.rados.rados.0.smithi063.stdout:1099: left oid 10 (ObjNum 436 snap 137 seq_num 436) 2024-08-17T16:59:19.154 INFO:tasks.rados.rados.0.smithi063.stdout:1100: finishing write tid 1 to smithi06387500-5 2024-08-17T16:59:19.154 INFO:tasks.rados.rados.0.smithi063.stdout:1097: done (6 left) 2024-08-17T16:59:19.154 INFO:tasks.rados.rados.0.smithi063.stdout:1098: done (5 left) 2024-08-17T16:59:19.154 INFO:tasks.rados.rados.0.smithi063.stdout:1099: done (4 left) 2024-08-17T16:59:19.154 INFO:tasks.rados.rados.0.smithi063.stdout:1103: done (3 left) 2024-08-17T16:59:19.154 INFO:tasks.rados.rados.0.smithi063.stdout:1104: write oid 1 current snap is 137 2024-08-17T16:59:19.154 INFO:tasks.rados.rados.0.smithi063.stdout:1100: finishing write tid 2 to smithi06387500-5 2024-08-17T16:59:19.154 INFO:tasks.rados.rados.0.smithi063.stdout:1104: seq_num 439 ranges {578273=466432,1739595=452815,2722929=655344} 2024-08-17T16:59:19.158 INFO:tasks.rados.rados.0.smithi063.stdout:1104: writing smithi06387500-1 from 578273 to 1044705 tid 1 2024-08-17T16:59:19.165 INFO:tasks.rados.rados.0.smithi063.stdout:1104: writing smithi06387500-1 from 1739595 to 2192410 tid 2 2024-08-17T16:59:19.171 INFO:tasks.rados.rados.0.smithi063.stdout:1104: writing smithi06387500-1 from 2722929 to 3378273 tid 3 2024-08-17T16:59:19.172 INFO:tasks.rados.rados.0.smithi063.stdout:1105: write oid 36 current snap is 137 2024-08-17T16:59:19.172 INFO:tasks.rados.rados.0.smithi063.stdout:1105: seq_num 440 ranges {595080=541681,1798077=596473,3150352=244728} 2024-08-17T16:59:19.177 INFO:tasks.rados.rados.0.smithi063.stdout:1105: writing smithi06387500-36 from 595080 to 1136761 tid 1 2024-08-17T16:59:19.184 INFO:tasks.rados.rados.0.smithi063.stdout:1105: writing smithi06387500-36 from 1798077 to 2394550 tid 2 2024-08-17T16:59:19.191 INFO:tasks.rados.rados.0.smithi063.stdout:1105: writing smithi06387500-36 from 3150352 to 3395080 tid 3 2024-08-17T16:59:19.191 INFO:tasks.rados.rados.0.smithi063.stdout:1106: snap_create 2024-08-17T16:59:19.191 INFO:tasks.rados.rados.0.smithi063.stdout:1100: finishing write tid 3 to smithi06387500-5 2024-08-17T16:59:19.191 INFO:tasks.rados.rados.0.smithi063.stdout:1100: finishing write tid 4 to smithi06387500-5 2024-08-17T16:59:19.191 INFO:tasks.rados.rados.0.smithi063.stdout:1102: finishing rollback tid 0 to smithi06387500-25 2024-08-17T16:59:19.191 INFO:tasks.rados.rados.0.smithi063.stdout:1100: finishing write tid 5 to smithi06387500-5 2024-08-17T16:59:19.191 INFO:tasks.rados.rados.0.smithi063.stdout:1100: oid 5 updating version 0 to 528 2024-08-17T16:59:19.191 INFO:tasks.rados.rados.0.smithi063.stdout:1100: oid 5 version 528 is already newer than 527 2024-08-17T16:59:19.191 INFO:tasks.rados.rados.0.smithi063.stdout:1100: oid 5 updating version 528 to 530 2024-08-17T16:59:19.191 INFO:tasks.rados.rados.0.smithi063.stdout:1100: oid 5 version 530 is already newer than 529 2024-08-17T16:59:19.191 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 530 (ObjNum 437 snap 137 seq_num 437) dirty exists 2024-08-17T16:59:19.192 INFO:tasks.rados.rados.0.smithi063.stdout:1100: left oid 5 (ObjNum 437 snap 137 seq_num 437) 2024-08-17T16:59:19.192 INFO:tasks.rados.rados.0.smithi063.stdout:1102: finishing rollback tid 1 to smithi06387500-25 2024-08-17T16:59:19.192 INFO:tasks.rados.rados.0.smithi063.stdout:1102: finishing rollback tid 2 to smithi06387500-25 2024-08-17T16:59:19.192 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 1117 (ObjNum 362 snap 108 seq_num 362) dirty exists 2024-08-17T16:59:19.192 INFO:tasks.rados.rados.0.smithi063.stdout:1101: expect (ObjNum 396 snap 121 seq_num 396) 2024-08-17T16:59:19.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:18 smithi063 ceph-mon[42680]: osdmap e342: 8 total, 8 up, 8 in 2024-08-17T16:59:19.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:18 smithi063 ceph-mon[31063]: osdmap e342: 8 total, 8 up, 8 in 2024-08-17T16:59:19.340 INFO:tasks.rados.rados.0.smithi063.stdout:1104: finishing write tid 1 to smithi06387500-1 2024-08-17T16:59:19.340 INFO:tasks.rados.rados.0.smithi063.stdout:1104: finishing write tid 2 to smithi06387500-1 2024-08-17T16:59:19.340 INFO:tasks.rados.rados.0.smithi063.stdout:1104: finishing write tid 3 to smithi06387500-1 2024-08-17T16:59:19.340 INFO:tasks.rados.rados.0.smithi063.stdout:1104: finishing write tid 4 to smithi06387500-1 2024-08-17T16:59:19.341 INFO:tasks.rados.rados.0.smithi063.stdout:1104: finishing write tid 5 to smithi06387500-1 2024-08-17T16:59:19.341 INFO:tasks.rados.rados.0.smithi063.stdout:1104: oid 1 updating version 0 to 535 2024-08-17T16:59:19.341 INFO:tasks.rados.rados.0.smithi063.stdout:1104: oid 1 version 535 is already newer than 534 2024-08-17T16:59:19.341 INFO:tasks.rados.rados.0.smithi063.stdout:1104: oid 1 version 535 is already newer than 532 2024-08-17T16:59:19.341 INFO:tasks.rados.rados.0.smithi063.stdout:1104: oid 1 version 535 is already newer than 533 2024-08-17T16:59:19.341 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 535 (ObjNum 439 snap 137 seq_num 439) dirty exists 2024-08-17T16:59:19.341 INFO:tasks.rados.rados.0.smithi063.stdout:1104: left oid 1 (ObjNum 439 snap 137 seq_num 439) 2024-08-17T16:59:19.341 INFO:tasks.rados.rados.0.smithi063.stdout:1105: finishing write tid 1 to smithi06387500-36 2024-08-17T16:59:19.341 INFO:tasks.rados.rados.0.smithi063.stdout:1105: finishing write tid 2 to smithi06387500-36 2024-08-17T16:59:19.341 INFO:tasks.rados.rados.0.smithi063.stdout:1105: finishing write tid 3 to smithi06387500-36 2024-08-17T16:59:19.341 INFO:tasks.rados.rados.0.smithi063.stdout:1105: finishing write tid 4 to smithi06387500-36 2024-08-17T16:59:19.341 INFO:tasks.rados.rados.0.smithi063.stdout:1105: finishing write tid 5 to smithi06387500-36 2024-08-17T16:59:19.342 INFO:tasks.rados.rados.0.smithi063.stdout:1105: oid 36 updating version 0 to 1120 2024-08-17T16:59:19.342 INFO:tasks.rados.rados.0.smithi063.stdout:1105: oid 36 updating version 1120 to 1121 2024-08-17T16:59:19.342 INFO:tasks.rados.rados.0.smithi063.stdout:1105: oid 36 updating version 1121 to 1122 2024-08-17T16:59:19.342 INFO:tasks.rados.rados.0.smithi063.stdout:1105: oid 36 version 1122 is already newer than 1119 2024-08-17T16:59:19.342 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 1122 (ObjNum 440 snap 137 seq_num 440) dirty exists 2024-08-17T16:59:19.342 INFO:tasks.rados.rados.0.smithi063.stdout:1105: left oid 36 (ObjNum 440 snap 137 seq_num 440) 2024-08-17T16:59:19.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:18 smithi162 ceph-mon[34633]: osdmap e342: 8 total, 8 up, 8 in 2024-08-17T16:59:19.893 INFO:tasks.rados.rados.0.smithi063.stdout:1100: done (5 left) 2024-08-17T16:59:19.893 INFO:tasks.rados.rados.0.smithi063.stdout:1101: done (4 left) 2024-08-17T16:59:19.893 INFO:tasks.rados.rados.0.smithi063.stdout:1102: done (3 left) 2024-08-17T16:59:19.894 INFO:tasks.rados.rados.0.smithi063.stdout:1104: done (2 left) 2024-08-17T16:59:19.894 INFO:tasks.rados.rados.0.smithi063.stdout:1105: done (1 left) 2024-08-17T16:59:19.894 INFO:tasks.rados.rados.0.smithi063.stdout:1106: done (0 left) 2024-08-17T16:59:19.894 INFO:tasks.rados.rados.0.smithi063.stdout:1107: read oid 45 snap -1 2024-08-17T16:59:19.894 INFO:tasks.rados.rados.0.smithi063.stdout:1107: expect deleted 2024-08-17T16:59:19.894 INFO:tasks.rados.rados.0.smithi063.stdout:1108: snap_create 2024-08-17T16:59:20.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:19 smithi063 ceph-mon[42680]: pgmap v197: 89 pgs: 89 active+clean; 126 MiB data, 539 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 974 KiB/s wr, 12 op/s 2024-08-17T16:59:20.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:19 smithi063 ceph-mon[42680]: osdmap e343: 8 total, 8 up, 8 in 2024-08-17T16:59:20.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:19 smithi063 ceph-mon[42680]: osdmap e344: 8 total, 8 up, 8 in 2024-08-17T16:59:20.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:19 smithi063 ceph-mon[31063]: pgmap v197: 89 pgs: 89 active+clean; 126 MiB data, 539 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 974 KiB/s wr, 12 op/s 2024-08-17T16:59:20.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:19 smithi063 ceph-mon[31063]: osdmap e343: 8 total, 8 up, 8 in 2024-08-17T16:59:20.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:19 smithi063 ceph-mon[31063]: osdmap e344: 8 total, 8 up, 8 in 2024-08-17T16:59:20.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:19 smithi162 ceph-mon[34633]: pgmap v197: 89 pgs: 89 active+clean; 126 MiB data, 539 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 974 KiB/s wr, 12 op/s 2024-08-17T16:59:20.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:19 smithi162 ceph-mon[34633]: osdmap e343: 8 total, 8 up, 8 in 2024-08-17T16:59:20.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:19 smithi162 ceph-mon[34633]: osdmap e344: 8 total, 8 up, 8 in 2024-08-17T16:59:20.889 INFO:tasks.rados.rados.0.smithi063.stdout:1107: done (1 left) 2024-08-17T16:59:20.889 INFO:tasks.rados.rados.0.smithi063.stdout:1108: done (0 left) 2024-08-17T16:59:20.889 INFO:tasks.rados.rados.0.smithi063.stdout:1109: read oid 37 snap -1 2024-08-17T16:59:20.889 INFO:tasks.rados.rados.0.smithi063.stdout:1109: expect deleted 2024-08-17T16:59:20.890 INFO:tasks.rados.rados.0.smithi063.stdout:1110: rollback oid 38 current snap is 139 2024-08-17T16:59:20.890 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 38 to 136 2024-08-17T16:59:20.890 INFO:tasks.rados.rados.0.smithi063.stdout:1111: write oid 1 current snap is 139 2024-08-17T16:59:20.890 INFO:tasks.rados.rados.0.smithi063.stdout:1111: seq_num 441 ranges {611887=733283,1972912=456484,2894128=517759} 2024-08-17T16:59:20.898 INFO:tasks.rados.rados.0.smithi063.stdout:1111: writing smithi06387500-1 from 611887 to 1345170 tid 1 2024-08-17T16:59:20.905 INFO:tasks.rados.rados.0.smithi063.stdout:1111: writing smithi06387500-1 from 1972912 to 2429396 tid 2 2024-08-17T16:59:20.911 INFO:tasks.rados.rados.0.smithi063.stdout:1111: writing smithi06387500-1 from 2894128 to 3411887 tid 3 2024-08-17T16:59:20.911 INFO:tasks.rados.rados.0.smithi063.stdout:1112: delete oid 7 current snap is 139 2024-08-17T16:59:20.912 INFO:tasks.rados.rados.0.smithi063.stdout:1110: finishing rollback tid 0 to smithi06387500-38 2024-08-17T16:59:20.912 INFO:tasks.rados.rados.0.smithi063.stdout:1110: finishing rollback tid 1 to smithi06387500-38 2024-08-17T16:59:20.912 INFO:tasks.rados.rados.0.smithi063.stdout:1110: finishing rollback tid 2 to smithi06387500-38 2024-08-17T16:59:20.912 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 133 (ObjNum 422 snap 130 seq_num 422) dirty exists 2024-08-17T16:59:20.912 INFO:tasks.rados.rados.0.smithi063.stdout:1111: finishing write tid 1 to smithi06387500-1 2024-08-17T16:59:20.912 INFO:tasks.rados.rados.0.smithi063.stdout:1111: finishing write tid 2 to smithi06387500-1 2024-08-17T16:59:20.913 INFO:tasks.rados.rados.0.smithi063.stdout:1109: done (3 left) 2024-08-17T16:59:20.914 INFO:tasks.rados.rados.0.smithi063.stdout:1110: done (2 left) 2024-08-17T16:59:20.914 INFO:tasks.rados.rados.0.smithi063.stdout:1112: done (1 left) 2024-08-17T16:59:20.914 INFO:tasks.rados.rados.0.smithi063.stdout:1113: rollback oid 48 current snap is 139 2024-08-17T16:59:20.914 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 48 to 125 2024-08-17T16:59:20.914 INFO:tasks.rados.rados.0.smithi063.stdout:1114: write oid 5 current snap is 139 2024-08-17T16:59:20.914 INFO:tasks.rados.rados.0.smithi063.stdout:1114: seq_num 443 ranges {645501=483781,1573523=460153,2548974=763384} 2024-08-17T16:59:20.919 INFO:tasks.rados.rados.0.smithi063.stdout:1114: writing smithi06387500-5 from 645501 to 1129282 tid 1 2024-08-17T16:59:20.924 INFO:tasks.rados.rados.0.smithi063.stdout:1114: writing smithi06387500-5 from 1573523 to 2033676 tid 2 2024-08-17T16:59:20.932 INFO:tasks.rados.rados.0.smithi063.stdout:1114: writing smithi06387500-5 from 2548974 to 3312358 tid 3 2024-08-17T16:59:20.932 INFO:tasks.rados.rados.0.smithi063.stdout:1115: write oid 29 current snap is 139 2024-08-17T16:59:20.932 INFO:tasks.rados.rados.0.smithi063.stdout:1111: finishing write tid 3 to smithi06387500-1 2024-08-17T16:59:20.932 INFO:tasks.rados.rados.0.smithi063.stdout:1111: finishing write tid 4 to smithi06387500-1 2024-08-17T16:59:20.932 INFO:tasks.rados.rados.0.smithi063.stdout:1111: finishing write tid 5 to smithi06387500-1 2024-08-17T16:59:20.932 INFO:tasks.rados.rados.0.smithi063.stdout:1111: oid 1 updating version 0 to 538 2024-08-17T16:59:20.933 INFO:tasks.rados.rados.0.smithi063.stdout:1111: oid 1 version 538 is already newer than 537 2024-08-17T16:59:20.933 INFO:tasks.rados.rados.0.smithi063.stdout:1111: oid 1 updating version 538 to 539 2024-08-17T16:59:20.933 INFO:tasks.rados.rados.0.smithi063.stdout:1111: oid 1 updating version 539 to 540 2024-08-17T16:59:20.933 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 540 (ObjNum 441 snap 139 seq_num 441) dirty exists 2024-08-17T16:59:20.933 INFO:tasks.rados.rados.0.smithi063.stdout:1111: left oid 1 (ObjNum 441 snap 139 seq_num 441) 2024-08-17T16:59:20.933 INFO:tasks.rados.rados.0.smithi063.stdout:1113: finishing rollback tid 0 to smithi06387500-48 2024-08-17T16:59:20.933 INFO:tasks.rados.rados.0.smithi063.stdout:1115: seq_num 444 ranges {662308=559030,1632005=603811,2860044=602264} 2024-08-17T16:59:20.938 INFO:tasks.rados.rados.0.smithi063.stdout:1115: writing smithi06387500-29 from 662308 to 1221338 tid 1 2024-08-17T16:59:20.944 INFO:tasks.rados.rados.0.smithi063.stdout:1115: writing smithi06387500-29 from 1632005 to 2235816 tid 2 2024-08-17T16:59:20.951 INFO:tasks.rados.rados.0.smithi063.stdout:1115: writing smithi06387500-29 from 2860044 to 3462308 tid 3 2024-08-17T16:59:20.951 INFO:tasks.rados.rados.0.smithi063.stdout:1111: done (3 left) 2024-08-17T16:59:20.951 INFO:tasks.rados.rados.0.smithi063.stdout:1116: read oid 17 snap -1 2024-08-17T16:59:20.952 INFO:tasks.rados.rados.0.smithi063.stdout:1116: expect deleted 2024-08-17T16:59:20.952 INFO:tasks.rados.rados.0.smithi063.stdout:1117: read oid 24 snap -1 2024-08-17T16:59:20.952 INFO:tasks.rados.rados.0.smithi063.stdout:1117: expect (ObjNum 427 snap 133 seq_num 427) 2024-08-17T16:59:20.952 INFO:tasks.rados.rados.0.smithi063.stdout:1118: snap_create 2024-08-17T16:59:20.952 INFO:tasks.rados.rados.0.smithi063.stdout:1113: finishing rollback tid 1 to smithi06387500-48 2024-08-17T16:59:20.952 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 1125 (ObjNum 33 snap 0 seq_num 1807061440) dirty dne 2024-08-17T16:59:20.952 INFO:tasks.rados.rados.0.smithi063.stdout:1114: finishing write tid 1 to smithi06387500-5 2024-08-17T16:59:20.952 INFO:tasks.rados.rados.0.smithi063.stdout:1114: finishing write tid 2 to smithi06387500-5 2024-08-17T16:59:20.952 INFO:tasks.rados.rados.0.smithi063.stdout:1114: finishing write tid 3 to smithi06387500-5 2024-08-17T16:59:20.952 INFO:tasks.rados.rados.0.smithi063.stdout:1114: finishing write tid 4 to smithi06387500-5 2024-08-17T16:59:20.952 INFO:tasks.rados.rados.0.smithi063.stdout:1114: finishing write tid 5 to smithi06387500-5 2024-08-17T16:59:20.952 INFO:tasks.rados.rados.0.smithi063.stdout:1114: oid 5 updating version 0 to 542 2024-08-17T16:59:20.952 INFO:tasks.rados.rados.0.smithi063.stdout:1114: oid 5 updating version 542 to 543 2024-08-17T16:59:20.953 INFO:tasks.rados.rados.0.smithi063.stdout:1114: oid 5 updating version 543 to 544 2024-08-17T16:59:20.953 INFO:tasks.rados.rados.0.smithi063.stdout:1114: oid 5 updating version 544 to 545 2024-08-17T16:59:20.953 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 545 (ObjNum 443 snap 139 seq_num 443) dirty exists 2024-08-17T16:59:20.953 INFO:tasks.rados.rados.0.smithi063.stdout:1114: left oid 5 (ObjNum 443 snap 139 seq_num 443) 2024-08-17T16:59:20.953 INFO:tasks.rados.rados.0.smithi063.stdout:1115: finishing write tid 1 to smithi06387500-29 2024-08-17T16:59:20.953 INFO:tasks.rados.rados.0.smithi063.stdout:1115: finishing write tid 2 to smithi06387500-29 2024-08-17T16:59:20.954 INFO:tasks.rados.rados.0.smithi063.stdout:1115: finishing write tid 3 to smithi06387500-29 2024-08-17T16:59:20.954 INFO:tasks.rados.rados.0.smithi063.stdout:1115: finishing write tid 4 to smithi06387500-29 2024-08-17T16:59:20.954 INFO:tasks.rados.rados.0.smithi063.stdout:1115: finishing write tid 5 to smithi06387500-29 2024-08-17T16:59:20.954 INFO:tasks.rados.rados.0.smithi063.stdout:1115: oid 29 updating version 0 to 222 2024-08-17T16:59:20.954 INFO:tasks.rados.rados.0.smithi063.stdout:1115: oid 29 updating version 222 to 225 2024-08-17T16:59:20.954 INFO:tasks.rados.rados.0.smithi063.stdout:1115: oid 29 version 225 is already newer than 223 2024-08-17T16:59:20.954 INFO:tasks.rados.rados.0.smithi063.stdout:1115: oid 29 version 225 is already newer than 224 2024-08-17T16:59:20.954 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 225 (ObjNum 444 snap 139 seq_num 444) dirty exists 2024-08-17T16:59:20.954 INFO:tasks.rados.rados.0.smithi063.stdout:1115: left oid 29 (ObjNum 444 snap 139 seq_num 444) 2024-08-17T16:59:20.956 INFO:tasks.rados.rados.0.smithi063.stdout:1117: expect (ObjNum 427 snap 133 seq_num 427) 2024-08-17T16:59:21.777 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for minority of mons to upgrade' 2024-08-17T16:59:21.896 INFO:tasks.rados.rados.0.smithi063.stdout:1113: done (5 left) 2024-08-17T16:59:21.896 INFO:tasks.rados.rados.0.smithi063.stdout:1114: done (4 left) 2024-08-17T16:59:21.896 INFO:tasks.rados.rados.0.smithi063.stdout:1115: done (3 left) 2024-08-17T16:59:21.896 INFO:tasks.rados.rados.0.smithi063.stdout:1116: done (2 left) 2024-08-17T16:59:21.896 INFO:tasks.rados.rados.0.smithi063.stdout:1117: done (1 left) 2024-08-17T16:59:21.896 INFO:tasks.rados.rados.0.smithi063.stdout:1118: done (0 left) 2024-08-17T16:59:21.896 INFO:tasks.rados.rados.0.smithi063.stdout:1119: rollback oid 34 current snap is 140 2024-08-17T16:59:21.897 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 34 to 127 2024-08-17T16:59:21.897 INFO:tasks.rados.rados.0.smithi063.stdout:1120: snap_create 2024-08-17T16:59:21.900 INFO:tasks.rados.rados.0.smithi063.stdout:1119: finishing rollback tid 0 to smithi06387500-34 2024-08-17T16:59:21.900 INFO:tasks.rados.rados.0.smithi063.stdout:1119: finishing rollback tid 1 to smithi06387500-34 2024-08-17T16:59:21.901 INFO:tasks.rados.rados.0.smithi063.stdout:1119: finishing rollback tid 2 to smithi06387500-34 2024-08-17T16:59:21.901 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 762 (ObjNum 403 snap 122 seq_num 403) dirty exists 2024-08-17T16:59:22.000 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:59:22.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:21 smithi063 ceph-mon[31063]: osdmap e345: 8 total, 8 up, 8 in 2024-08-17T16:59:22.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:21 smithi063 ceph-mon[31063]: pgmap v201: 89 pgs: 89 active+clean; 150 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 29 MiB/s rd, 3.7 MiB/s wr, 39 op/s 2024-08-17T16:59:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:21 smithi063 ceph-mon[42680]: osdmap e345: 8 total, 8 up, 8 in 2024-08-17T16:59:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:21 smithi063 ceph-mon[42680]: pgmap v201: 89 pgs: 89 active+clean; 150 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 29 MiB/s rd, 3.7 MiB/s wr, 39 op/s 2024-08-17T16:59:22.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:21 smithi162 ceph-mon[34633]: osdmap e345: 8 total, 8 up, 8 in 2024-08-17T16:59:22.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:21 smithi162 ceph-mon[34633]: pgmap v201: 89 pgs: 89 active+clean; 150 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 29 MiB/s rd, 3.7 MiB/s wr, 39 op/s 2024-08-17T16:59:22.901 INFO:tasks.rados.rados.0.smithi063.stdout:1119: done (1 left) 2024-08-17T16:59:22.901 INFO:tasks.rados.rados.0.smithi063.stdout:1120: done (0 left) 2024-08-17T16:59:22.901 INFO:tasks.rados.rados.0.smithi063.stdout:1121: snap_create 2024-08-17T16:59:23.170 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:22 smithi063 ceph-mon[42680]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:23.170 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:22 smithi063 ceph-mon[42680]: osdmap e346: 8 total, 8 up, 8 in 2024-08-17T16:59:23.170 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:22 smithi063 ceph-mon[31063]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:23.171 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:22 smithi063 ceph-mon[31063]: osdmap e346: 8 total, 8 up, 8 in 2024-08-17T16:59:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:22 smithi162 ceph-mon[34633]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:23.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:22 smithi162 ceph-mon[34633]: osdmap e346: 8 total, 8 up, 8 in 2024-08-17T16:59:23.462 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:59:23] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.51.0" 2024-08-17T16:59:23.804 INFO:teuthology.orchestra.run.smithi063.stdout:wait for minority of mons to upgrade 2024-08-17T16:59:23.910 INFO:tasks.rados.rados.0.smithi063.stdout:1121: done (0 left) 2024-08-17T16:59:23.910 INFO:tasks.rados.rados.0.smithi063.stdout:1122: write oid 16 current snap is 142 2024-08-17T16:59:23.910 INFO:tasks.rados.rados.0.smithi063.stdout:1122: seq_num 445 ranges {679115=634279,2090487=463822,3003820=475295} 2024-08-17T16:59:23.916 INFO:tasks.rados.rados.0.smithi063.stdout:1122: writing smithi06387500-16 from 679115 to 1313394 tid 1 2024-08-17T16:59:23.924 INFO:tasks.rados.rados.0.smithi063.stdout:1122: writing smithi06387500-16 from 2090487 to 2554309 tid 2 2024-08-17T16:59:23.929 INFO:tasks.rados.rados.0.smithi063.stdout:1122: writing smithi06387500-16 from 3003820 to 3479115 tid 3 2024-08-17T16:59:23.929 INFO:tasks.rados.rados.0.smithi063.stdout:1123: read oid 3 snap -1 2024-08-17T16:59:23.929 INFO:tasks.rados.rados.0.smithi063.stdout:1123: expect deleted 2024-08-17T16:59:23.929 INFO:tasks.rados.rados.0.smithi063.stdout:1124: snap_remove snap 129 2024-08-17T16:59:24.152 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:23 smithi063 ceph-mon[31063]: osdmap e347: 8 total, 8 up, 8 in 2024-08-17T16:59:24.153 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:23 smithi063 ceph-mon[31063]: pgmap v204: 89 pgs: 89 active+clean; 150 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 29 MiB/s rd, 3.7 MiB/s wr, 40 op/s 2024-08-17T16:59:24.153 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:23 smithi063 ceph-mon[42680]: osdmap e347: 8 total, 8 up, 8 in 2024-08-17T16:59:24.153 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:23 smithi063 ceph-mon[42680]: pgmap v204: 89 pgs: 89 active+clean; 150 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 29 MiB/s rd, 3.7 MiB/s wr, 40 op/s 2024-08-17T16:59:24.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:23 smithi162 ceph-mon[34633]: osdmap e347: 8 total, 8 up, 8 in 2024-08-17T16:59:24.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:23 smithi162 ceph-mon[34633]: pgmap v204: 89 pgs: 89 active+clean; 150 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 29 MiB/s rd, 3.7 MiB/s wr, 40 op/s 2024-08-17T16:59:24.579 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-08-17T16:59:24.798 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:59:24.889 INFO:tasks.rados.rados.0.smithi063.stdout:1124: done (2 left) 2024-08-17T16:59:24.890 INFO:tasks.rados.rados.0.smithi063.stdout:1125: read oid 48 snap -1 2024-08-17T16:59:24.890 INFO:tasks.rados.rados.0.smithi063.stdout:1125: expect deleted 2024-08-17T16:59:24.890 INFO:tasks.rados.rados.0.smithi063.stdout:1126: delete oid 45 current snap is 142 2024-08-17T16:59:24.890 INFO:tasks.rados.rados.0.smithi063.stdout:1122: finishing write tid 1 to smithi06387500-16 2024-08-17T16:59:24.890 INFO:tasks.rados.rados.0.smithi063.stdout:1122: finishing write tid 2 to smithi06387500-16 2024-08-17T16:59:24.890 INFO:tasks.rados.rados.0.smithi063.stdout:1122: finishing write tid 3 to smithi06387500-16 2024-08-17T16:59:24.890 INFO:tasks.rados.rados.0.smithi063.stdout:1122: finishing write tid 4 to smithi06387500-16 2024-08-17T16:59:24.890 INFO:tasks.rados.rados.0.smithi063.stdout:1122: finishing write tid 5 to smithi06387500-16 2024-08-17T16:59:24.890 INFO:tasks.rados.rados.0.smithi063.stdout:1122: oid 16 updating version 0 to 403 2024-08-17T16:59:24.891 INFO:tasks.rados.rados.0.smithi063.stdout:1122: oid 16 version 403 is already newer than 400 2024-08-17T16:59:24.891 INFO:tasks.rados.rados.0.smithi063.stdout:1122: oid 16 version 403 is already newer than 402 2024-08-17T16:59:24.891 INFO:tasks.rados.rados.0.smithi063.stdout:1122: oid 16 version 403 is already newer than 401 2024-08-17T16:59:24.891 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 403 (ObjNum 445 snap 142 seq_num 445) dirty exists 2024-08-17T16:59:24.891 INFO:tasks.rados.rados.0.smithi063.stdout:1122: left oid 16 (ObjNum 445 snap 142 seq_num 445) 2024-08-17T16:59:24.901 INFO:tasks.rados.rados.0.smithi063.stdout:1122: done (3 left) 2024-08-17T16:59:24.901 INFO:tasks.rados.rados.0.smithi063.stdout:1123: done (2 left) 2024-08-17T16:59:24.901 INFO:tasks.rados.rados.0.smithi063.stdout:1125: done (1 left) 2024-08-17T16:59:24.901 INFO:tasks.rados.rados.0.smithi063.stdout:1126: done (0 left) 2024-08-17T16:59:24.901 INFO:tasks.rados.rados.0.smithi063.stdout:1127: snap_create 2024-08-17T16:59:25.173 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:24 smithi063 ceph-mon[42680]: osdmap e348: 8 total, 8 up, 8 in 2024-08-17T16:59:25.173 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:24 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:25.173 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:24 smithi063 ceph-mon[42680]: osdmap e349: 8 total, 8 up, 8 in 2024-08-17T16:59:25.174 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:24 smithi063 ceph-mon[31063]: osdmap e348: 8 total, 8 up, 8 in 2024-08-17T16:59:25.174 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:24 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:25.174 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:24 smithi063 ceph-mon[31063]: osdmap e349: 8 total, 8 up, 8 in 2024-08-17T16:59:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:24 smithi162 ceph-mon[34633]: osdmap e348: 8 total, 8 up, 8 in 2024-08-17T16:59:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:24 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:24 smithi162 ceph-mon[34633]: osdmap e349: 8 total, 8 up, 8 in 2024-08-17T16:59:25.891 INFO:tasks.rados.rados.0.smithi063.stdout:1127: done (0 left) 2024-08-17T16:59:25.892 INFO:tasks.rados.rados.0.smithi063.stdout:1128: delete oid 24 current snap is 143 2024-08-17T16:59:25.894 INFO:tasks.rados.rados.0.smithi063.stdout:1128: done (0 left) 2024-08-17T16:59:25.894 INFO:tasks.rados.rados.0.smithi063.stdout:1129: read oid 43 snap 125 2024-08-17T16:59:25.894 INFO:tasks.rados.rados.0.smithi063.stdout:1129: expect deleted 2024-08-17T16:59:25.894 INFO:tasks.rados.rados.0.smithi063.stdout:1130: delete oid 39 current snap is 143 2024-08-17T16:59:25.898 INFO:tasks.rados.rados.0.smithi063.stdout:1129: done (1 left) 2024-08-17T16:59:25.899 INFO:tasks.rados.rados.0.smithi063.stdout:1130: done (0 left) 2024-08-17T16:59:25.899 INFO:tasks.rados.rados.0.smithi063.stdout:1131: snap_create 2024-08-17T16:59:26.171 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:25 smithi063 ceph-mon[42680]: pgmap v207: 89 pgs: 89 active+clean; 156 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-08-17T16:59:26.171 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:25 smithi063 ceph-mon[42680]: osdmap e350: 8 total, 8 up, 8 in 2024-08-17T16:59:26.171 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:25 smithi063 ceph-mon[31063]: pgmap v207: 89 pgs: 89 active+clean; 156 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-08-17T16:59:26.171 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:25 smithi063 ceph-mon[31063]: osdmap e350: 8 total, 8 up, 8 in 2024-08-17T16:59:26.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:25 smithi162 ceph-mon[34633]: pgmap v207: 89 pgs: 89 active+clean; 156 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.6 MiB/s wr, 9 op/s 2024-08-17T16:59:26.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:25 smithi162 ceph-mon[34633]: osdmap e350: 8 total, 8 up, 8 in 2024-08-17T16:59:26.896 INFO:tasks.rados.rados.0.smithi063.stdout:1131: done (0 left) 2024-08-17T16:59:26.896 INFO:tasks.rados.rados.0.smithi063.stdout:1132: read oid 6 snap -1 2024-08-17T16:59:26.896 INFO:tasks.rados.rados.0.smithi063.stdout:1132: expect (ObjNum 425 snap 133 seq_num 425) 2024-08-17T16:59:26.896 INFO:tasks.rados.rados.0.smithi063.stdout:1133: read oid 21 snap -1 2024-08-17T16:59:26.896 INFO:tasks.rados.rados.0.smithi063.stdout:1133: expect (ObjNum 432 snap 135 seq_num 432) 2024-08-17T16:59:26.896 INFO:tasks.rados.rados.0.smithi063.stdout:1134: snap_create 2024-08-17T16:59:26.904 INFO:tasks.rados.rados.0.smithi063.stdout:1132: expect (ObjNum 425 snap 133 seq_num 425) 2024-08-17T16:59:27.019 INFO:tasks.rados.rados.0.smithi063.stdout:1133: expect (ObjNum 432 snap 135 seq_num 432) 2024-08-17T16:59:27.906 INFO:tasks.rados.rados.0.smithi063.stdout:1132: done (2 left) 2024-08-17T16:59:27.906 INFO:tasks.rados.rados.0.smithi063.stdout:1133: done (1 left) 2024-08-17T16:59:27.906 INFO:tasks.rados.rados.0.smithi063.stdout:1134: done (0 left) 2024-08-17T16:59:27.906 INFO:tasks.rados.rados.0.smithi063.stdout:1135: write oid 49 current snap is 145 2024-08-17T16:59:27.906 INFO:tasks.rados.rados.0.smithi063.stdout:1135: seq_num 449 ranges {746343=651628,1924415=754807,3397159=149184} 2024-08-17T16:59:27.914 INFO:tasks.rados.rados.0.smithi063.stdout:1135: writing smithi06387500-49 from 746343 to 1397971 tid 1 2024-08-17T16:59:27.923 INFO:tasks.rados.rados.0.smithi063.stdout:1135: writing smithi06387500-49 from 1924415 to 2679222 tid 2 2024-08-17T16:59:27.928 INFO:tasks.rados.rados.0.smithi063.stdout:1135: writing smithi06387500-49 from 3397159 to 3546343 tid 3 2024-08-17T16:59:27.928 INFO:tasks.rados.rados.0.smithi063.stdout:1136: snap_remove snap 142 2024-08-17T16:59:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:27 smithi063 ceph-mon[42680]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:27 smithi063 ceph-mon[42680]: osdmap e351: 8 total, 8 up, 8 in 2024-08-17T16:59:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:27 smithi063 ceph-mon[42680]: pgmap v210: 89 pgs: 89 active+clean; 156 MiB data, 604 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-08-17T16:59:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:27 smithi063 ceph-mon[31063]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:27 smithi063 ceph-mon[31063]: osdmap e351: 8 total, 8 up, 8 in 2024-08-17T16:59:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:27 smithi063 ceph-mon[31063]: pgmap v210: 89 pgs: 89 active+clean; 156 MiB data, 604 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-08-17T16:59:28.148 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:27 smithi162 ceph-mon[34633]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:28.148 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:27 smithi162 ceph-mon[34633]: osdmap e351: 8 total, 8 up, 8 in 2024-08-17T16:59:28.148 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:27 smithi162 ceph-mon[34633]: pgmap v210: 89 pgs: 89 active+clean; 156 MiB data, 604 MiB used, 715 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-08-17T16:59:28.714 INFO:teuthology.orchestra.run.smithi063.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:59:28.912 INFO:tasks.rados.rados.0.smithi063.stdout:1136: done (1 left) 2024-08-17T16:59:28.912 INFO:tasks.rados.rados.0.smithi063.stdout:1137: read oid 1 snap -1 2024-08-17T16:59:28.912 INFO:tasks.rados.rados.0.smithi063.stdout:1137: expect (ObjNum 441 snap 139 seq_num 441) 2024-08-17T16:59:28.912 INFO:tasks.rados.rados.0.smithi063.stdout:1138: read oid 40 snap 143 2024-08-17T16:59:28.912 INFO:tasks.rados.rados.0.smithi063.stdout:1138: expect deleted 2024-08-17T16:59:28.912 INFO:tasks.rados.rados.0.smithi063.stdout:1139: snap_remove snap 144 2024-08-17T16:59:29.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:28 smithi063 ceph-mon[31063]: osdmap e352: 8 total, 8 up, 8 in 2024-08-17T16:59:29.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:28 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:29.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:28 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:29.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:28 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:29.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:28 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:29.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:28 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:29.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:28 smithi063 ceph-mon[42680]: osdmap e352: 8 total, 8 up, 8 in 2024-08-17T16:59:29.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:28 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:29.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:28 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:29.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:28 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:29.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:28 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:29.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:28 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:29.052 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:28 smithi162 ceph-mon[34633]: osdmap e352: 8 total, 8 up, 8 in 2024-08-17T16:59:29.052 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:28 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:29.052 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:28 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:29.052 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:28 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:29.052 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:28 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:29.052 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:28 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:29.429 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-17T16:59:29.651 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T16:59:29.892 INFO:tasks.rados.rados.0.smithi063.stdout:1139: done (3 left) 2024-08-17T16:59:29.892 INFO:tasks.rados.rados.0.smithi063.stdout:1140: read oid 37 snap 134 2024-08-17T16:59:29.892 INFO:tasks.rados.rados.0.smithi063.stdout:1140: expect deleted 2024-08-17T16:59:29.892 INFO:tasks.rados.rados.0.smithi063.stdout:1141: read oid 4 snap -1 2024-08-17T16:59:29.893 INFO:tasks.rados.rados.0.smithi063.stdout:1141: expect deleted 2024-08-17T16:59:29.893 INFO:tasks.rados.rados.0.smithi063.stdout:1142: read oid 20 snap -1 2024-08-17T16:59:29.893 INFO:tasks.rados.rados.0.smithi063.stdout:1142: expect (ObjNum 417 snap 128 seq_num 417) 2024-08-17T16:59:29.893 INFO:tasks.rados.rados.0.smithi063.stdout:1143: write oid 35 current snap is 145 2024-08-17T16:59:29.893 INFO:tasks.rados.rados.0.smithi063.stdout:1135: finishing write tid 1 to smithi06387500-49 2024-08-17T16:59:29.893 INFO:tasks.rados.rados.0.smithi063.stdout:1135: finishing write tid 2 to smithi06387500-49 2024-08-17T16:59:29.893 INFO:tasks.rados.rados.0.smithi063.stdout:1135: finishing write tid 3 to smithi06387500-49 2024-08-17T16:59:29.893 INFO:tasks.rados.rados.0.smithi063.stdout:1135: finishing write tid 4 to smithi06387500-49 2024-08-17T16:59:29.893 INFO:tasks.rados.rados.0.smithi063.stdout:1135: finishing write tid 5 to smithi06387500-49 2024-08-17T16:59:29.893 INFO:tasks.rados.rados.0.smithi063.stdout:1135: oid 49 updating version 0 to 1136 2024-08-17T16:59:29.893 INFO:tasks.rados.rados.0.smithi063.stdout:1135: oid 49 version 1136 is already newer than 1134 2024-08-17T16:59:29.893 INFO:tasks.rados.rados.0.smithi063.stdout:1135: oid 49 version 1136 is already newer than 1135 2024-08-17T16:59:29.893 INFO:tasks.rados.rados.0.smithi063.stdout:1135: oid 49 version 1136 is already newer than 1133 2024-08-17T16:59:29.894 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 1136 (ObjNum 449 snap 145 seq_num 449) dirty exists 2024-08-17T16:59:29.894 INFO:tasks.rados.rados.0.smithi063.stdout:1135: left oid 49 (ObjNum 449 snap 145 seq_num 449) 2024-08-17T16:59:29.894 INFO:tasks.rados.rados.0.smithi063.stdout:1143: seq_num 450 ranges {763150=726877,1982897=614818,3024582=474994} 2024-08-17T16:59:29.900 INFO:tasks.rados.rados.0.smithi063.stdout:1143: writing smithi06387500-35 from 763150 to 1490027 tid 1 2024-08-17T16:59:29.906 INFO:tasks.rados.rados.0.smithi063.stdout:1143: writing smithi06387500-35 from 1982897 to 2597715 tid 2 2024-08-17T16:59:29.912 INFO:tasks.rados.rados.0.smithi063.stdout:1143: writing smithi06387500-35 from 3024582 to 3499576 tid 3 2024-08-17T16:59:29.912 INFO:tasks.rados.rados.0.smithi063.stdout:1135: done (6 left) 2024-08-17T16:59:29.912 INFO:tasks.rados.rados.0.smithi063.stdout:1138: done (5 left) 2024-08-17T16:59:29.912 INFO:tasks.rados.rados.0.smithi063.stdout:1144: read oid 43 snap -1 2024-08-17T16:59:29.912 INFO:tasks.rados.rados.0.smithi063.stdout:1144: expect deleted 2024-08-17T16:59:29.912 INFO:tasks.rados.rados.0.smithi063.stdout:1145: snap_remove snap 139 2024-08-17T16:59:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:29 smithi063 ceph-mon[42680]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:59:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:29 smithi063 ceph-mon[42680]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:59:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:29 smithi063 ceph-mon[42680]: osdmap e353: 8 total, 8 up, 8 in 2024-08-17T16:59:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:29 smithi063 ceph-mon[42680]: pgmap v213: 89 pgs: 89 active+clean; 156 MiB data, 604 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T16:59:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:29 smithi063 ceph-mon[42680]: osdmap e354: 8 total, 8 up, 8 in 2024-08-17T16:59:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:29 smithi063 ceph-mon[31063]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:59:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:29 smithi063 ceph-mon[31063]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:59:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:29 smithi063 ceph-mon[31063]: osdmap e353: 8 total, 8 up, 8 in 2024-08-17T16:59:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:29 smithi063 ceph-mon[31063]: pgmap v213: 89 pgs: 89 active+clean; 156 MiB data, 604 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T16:59:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:29 smithi063 ceph-mon[31063]: osdmap e354: 8 total, 8 up, 8 in 2024-08-17T16:59:30.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:29 smithi162 ceph-mon[34633]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:59:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:29 smithi162 ceph-mon[34633]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T16:59:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:29 smithi162 ceph-mon[34633]: osdmap e353: 8 total, 8 up, 8 in 2024-08-17T16:59:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:29 smithi162 ceph-mon[34633]: pgmap v213: 89 pgs: 89 active+clean; 156 MiB data, 604 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T16:59:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:29 smithi162 ceph-mon[34633]: osdmap e354: 8 total, 8 up, 8 in 2024-08-17T16:59:30.924 INFO:tasks.rados.rados.0.smithi063.stdout:1145: done (6 left) 2024-08-17T16:59:30.925 INFO:tasks.rados.rados.0.smithi063.stdout:1146: read oid 36 snap -1 2024-08-17T16:59:30.925 INFO:tasks.rados.rados.0.smithi063.stdout:1146: expect (ObjNum 440 snap 137 seq_num 440) 2024-08-17T16:59:30.925 INFO:tasks.rados.rados.0.smithi063.stdout:1147: read oid 24 snap -1 2024-08-17T16:59:30.925 INFO:tasks.rados.rados.0.smithi063.stdout:1147: expect deleted 2024-08-17T16:59:30.925 INFO:tasks.rados.rados.0.smithi063.stdout:1148: delete oid 13 current snap is 145 2024-08-17T16:59:30.925 INFO:tasks.rados.rados.0.smithi063.stdout:1137: expect (ObjNum 441 snap 139 seq_num 441) 2024-08-17T16:59:31.161 INFO:tasks.rados.rados.0.smithi063.stdout:1142: expect (ObjNum 417 snap 128 seq_num 417) 2024-08-17T16:59:31.314 INFO:tasks.rados.rados.0.smithi063.stdout:1143: finishing write tid 1 to smithi06387500-35 2024-08-17T16:59:31.314 INFO:tasks.rados.rados.0.smithi063.stdout:1143: finishing write tid 2 to smithi06387500-35 2024-08-17T16:59:31.315 INFO:tasks.rados.rados.0.smithi063.stdout:1143: finishing write tid 3 to smithi06387500-35 2024-08-17T16:59:31.315 INFO:tasks.rados.rados.0.smithi063.stdout:1143: finishing write tid 4 to smithi06387500-35 2024-08-17T16:59:31.315 INFO:tasks.rados.rados.0.smithi063.stdout:1143: finishing write tid 5 to smithi06387500-35 2024-08-17T16:59:31.315 INFO:tasks.rados.rados.0.smithi063.stdout:1143: oid 35 updating version 0 to 1143 2024-08-17T16:59:31.315 INFO:tasks.rados.rados.0.smithi063.stdout:1143: oid 35 version 1143 is already newer than 1141 2024-08-17T16:59:31.315 INFO:tasks.rados.rados.0.smithi063.stdout:1143: oid 35 version 1143 is already newer than 1142 2024-08-17T16:59:31.315 INFO:tasks.rados.rados.0.smithi063.stdout:1143: oid 35 updating version 1143 to 1144 2024-08-17T16:59:31.315 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 1144 (ObjNum 450 snap 145 seq_num 450) dirty exists 2024-08-17T16:59:31.315 INFO:tasks.rados.rados.0.smithi063.stdout:1143: left oid 35 (ObjNum 450 snap 145 seq_num 450) 2024-08-17T16:59:31.315 INFO:tasks.rados.rados.0.smithi063.stdout:1146: expect (ObjNum 440 snap 137 seq_num 440) 2024-08-17T16:59:31.472 INFO:tasks.rados.rados.0.smithi063.stdout:1137: done (8 left) 2024-08-17T16:59:31.472 INFO:tasks.rados.rados.0.smithi063.stdout:1140: done (7 left) 2024-08-17T16:59:31.472 INFO:tasks.rados.rados.0.smithi063.stdout:1141: done (6 left) 2024-08-17T16:59:31.472 INFO:tasks.rados.rados.0.smithi063.stdout:1142: done (5 left) 2024-08-17T16:59:31.472 INFO:tasks.rados.rados.0.smithi063.stdout:1143: done (4 left) 2024-08-17T16:59:31.472 INFO:tasks.rados.rados.0.smithi063.stdout:1144: done (3 left) 2024-08-17T16:59:31.472 INFO:tasks.rados.rados.0.smithi063.stdout:1146: done (2 left) 2024-08-17T16:59:31.472 INFO:tasks.rados.rados.0.smithi063.stdout:1147: done (1 left) 2024-08-17T16:59:31.472 INFO:tasks.rados.rados.0.smithi063.stdout:1148: done (0 left) 2024-08-17T16:59:31.472 INFO:tasks.rados.rados.0.smithi063.stdout:1149: snap_create 2024-08-17T16:59:31.925 INFO:tasks.rados.rados.0.smithi063.stdout:1149: done (0 left) 2024-08-17T16:59:31.925 INFO:tasks.rados.rados.0.smithi063.stdout:1150: snap_remove snap 143 2024-08-17T16:59:32.192 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:31 smithi063 ceph-mon[31063]: osdmap e355: 8 total, 8 up, 8 in 2024-08-17T16:59:32.193 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:31 smithi063 ceph-mon[31063]: pgmap v216: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 161 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 824 KiB/s wr, 12 op/s 2024-08-17T16:59:32.193 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:31 smithi063 ceph-mon[42680]: osdmap e355: 8 total, 8 up, 8 in 2024-08-17T16:59:32.193 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:31 smithi063 ceph-mon[42680]: pgmap v216: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 161 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 824 KiB/s wr, 12 op/s 2024-08-17T16:59:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:31 smithi162 ceph-mon[34633]: osdmap e355: 8 total, 8 up, 8 in 2024-08-17T16:59:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:31 smithi162 ceph-mon[34633]: pgmap v216: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 161 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 824 KiB/s wr, 12 op/s 2024-08-17T16:59:32.654 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T16:59:32.925 INFO:tasks.rados.rados.0.smithi063.stdout:1150: done (0 left) 2024-08-17T16:59:32.925 INFO:tasks.rados.rados.0.smithi063.stdout:1151: rollback oid 42 current snap is 146 2024-08-17T16:59:32.925 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 42 to 136 2024-08-17T16:59:32.925 INFO:tasks.rados.rados.0.smithi063.stdout:1152: read oid 44 snap -1 2024-08-17T16:59:32.925 INFO:tasks.rados.rados.0.smithi063.stdout:1152: expect (ObjNum 409 snap 126 seq_num 409) 2024-08-17T16:59:32.926 INFO:tasks.rados.rados.0.smithi063.stdout:1153: read oid 43 snap -1 2024-08-17T16:59:32.926 INFO:tasks.rados.rados.0.smithi063.stdout:1153: expect deleted 2024-08-17T16:59:32.926 INFO:tasks.rados.rados.0.smithi063.stdout:1154: write oid 22 current snap is 146 2024-08-17T16:59:32.926 INFO:tasks.rados.rados.0.smithi063.stdout:1154: seq_num 452 ranges {796764=477375,1983508=618487,3363075=233689} 2024-08-17T16:59:32.932 INFO:tasks.rados.rados.0.smithi063.stdout:1154: writing smithi06387500-22 from 796764 to 1274139 tid 1 2024-08-17T16:59:32.940 INFO:tasks.rados.rados.0.smithi063.stdout:1154: writing smithi06387500-22 from 1983508 to 2601995 tid 2 2024-08-17T16:59:32.946 INFO:tasks.rados.rados.0.smithi063.stdout:1154: writing smithi06387500-22 from 3363075 to 3596764 tid 3 2024-08-17T16:59:32.946 INFO:tasks.rados.rados.0.smithi063.stdout:1155: read oid 21 snap -1 2024-08-17T16:59:32.946 INFO:tasks.rados.rados.0.smithi063.stdout:1155: expect (ObjNum 432 snap 135 seq_num 432) 2024-08-17T16:59:32.946 INFO:tasks.rados.rados.0.smithi063.stdout:1156: write oid 48 current snap is 146 2024-08-17T16:59:32.946 INFO:tasks.rados.rados.0.smithi063.stdout:1151: finishing rollback tid 1 to smithi06387500-42 2024-08-17T16:59:32.946 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 756 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-08-17T16:59:32.946 INFO:tasks.rados.rados.0.smithi063.stdout:1152: expect (ObjNum 409 snap 126 seq_num 409) 2024-08-17T16:59:33.193 INFO:tasks.rados.rados.0.smithi063.stdout:1154: finishing write tid 1 to smithi06387500-22 2024-08-17T16:59:33.193 INFO:tasks.rados.rados.0.smithi063.stdout:1154: finishing write tid 2 to smithi06387500-22 2024-08-17T16:59:33.193 INFO:tasks.rados.rados.0.smithi063.stdout:1154: finishing write tid 3 to smithi06387500-22 2024-08-17T16:59:33.193 INFO:tasks.rados.rados.0.smithi063.stdout:1156: seq_num 453 ranges {413571=552624,1641990=762145,2990498=425163} 2024-08-17T16:59:33.198 INFO:tasks.rados.rados.0.smithi063.stdout:1156: writing smithi06387500-48 from 413571 to 966195 tid 1 2024-08-17T16:59:33.206 INFO:tasks.rados.rados.0.smithi063.stdout:1156: writing smithi06387500-48 from 1641990 to 2404135 tid 2 2024-08-17T16:59:33.212 INFO:tasks.rados.rados.0.smithi063.stdout:1156: writing smithi06387500-48 from 2990498 to 3415661 tid 3 2024-08-17T16:59:33.212 INFO:tasks.rados.rados.0.smithi063.stdout:1151: done (5 left) 2024-08-17T16:59:33.212 INFO:tasks.rados.rados.0.smithi063.stdout:1152: done (4 left) 2024-08-17T16:59:33.212 INFO:tasks.rados.rados.0.smithi063.stdout:1153: done (3 left) 2024-08-17T16:59:33.212 INFO:tasks.rados.rados.0.smithi063.stdout:1157: snap_create 2024-08-17T16:59:33.212 INFO:tasks.rados.rados.0.smithi063.stdout:1154: finishing write tid 4 to smithi06387500-22 2024-08-17T16:59:33.213 INFO:tasks.rados.rados.0.smithi063.stdout:1154: finishing write tid 5 to smithi06387500-22 2024-08-17T16:59:33.213 INFO:tasks.rados.rados.0.smithi063.stdout:1154: oid 22 updating version 0 to 301 2024-08-17T16:59:33.213 INFO:tasks.rados.rados.0.smithi063.stdout:1154: oid 22 updating version 301 to 302 2024-08-17T16:59:33.213 INFO:tasks.rados.rados.0.smithi063.stdout:1154: oid 22 version 302 is already newer than 300 2024-08-17T16:59:33.213 INFO:tasks.rados.rados.0.smithi063.stdout:1154: oid 22 version 302 is already newer than 299 2024-08-17T16:59:33.213 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 302 (ObjNum 452 snap 146 seq_num 452) dirty exists 2024-08-17T16:59:33.213 INFO:tasks.rados.rados.0.smithi063.stdout:1154: left oid 22 (ObjNum 452 snap 146 seq_num 452) 2024-08-17T16:59:33.213 INFO:tasks.rados.rados.0.smithi063.stdout:1155: expect (ObjNum 432 snap 135 seq_num 432) 2024-08-17T16:59:33.255 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T16:59:33.255 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (60s) 55s ago 6m 17.4M - 0.27.0 11f11916f8cd 59f4018b91ca 2024-08-17T16:59:33.255 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (37s) 28s ago 6m 64.1M - c8b91775d855 581e03e654a8 2024-08-17T16:59:33.255 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (5m) 55s ago 5m 18.0M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T16:59:33.255 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (5m) 28s ago 5m 17.4M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T16:59:33.255 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,9283,8765 running (2m) 28s ago 10m 454M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb da48034a2895 2024-08-17T16:59:33.255 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:8443,9283,8765 running (2m) 55s ago 12m 515M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 943e639cede6 2024-08-17T16:59:33.256 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (12m) 55s ago 12m 61.2M 2048M 18.2.4-719-gced649f8 c870753d67c2 0c71ddd7ada6 2024-08-17T16:59:33.256 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (11m) 28s ago 11m 56.7M 2048M 18.2.4-719-gced649f8 c870753d67c2 d4ecbe65a0c8 2024-08-17T16:59:33.256 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (11m) 55s ago 11m 52.3M 2048M 18.2.4-719-gced649f8 c870753d67c2 b3f2c9339c6b 2024-08-17T16:59:33.256 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 running (110s) 55s ago 6m 9223k - 1.7.0 72c9c2088986 34f55f5de2aa 2024-08-17T16:59:33.256 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (103s) 28s ago 6m 9777k - 1.7.0 72c9c2088986 5bb162d0f1a9 2024-08-17T16:59:33.256 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (10m) 55s ago 10m 92.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 f3b910ed49b8 2024-08-17T16:59:33.256 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (10m) 55s ago 10m 73.7M 2480M 18.2.4-719-gced649f8 c870753d67c2 5d6fdb6000ad 2024-08-17T16:59:33.256 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (9m) 55s ago 9m 83.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 a3cbe8edb49a 2024-08-17T16:59:33.256 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (9m) 55s ago 9m 73.8M 2480M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T16:59:33.256 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (9m) 28s ago 9m 93.6M 2480M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T16:59:33.256 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (8m) 28s ago 8m 84.7M 2480M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T16:59:33.257 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (8m) 28s ago 8m 63.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T16:59:33.257 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (7m) 28s ago 7m 75.8M 2480M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T16:59:33.257 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (82s) 28s ago 7m 39.3M - 2.51.0 1d3b7f56885b dfe66b2c5568 2024-08-17T16:59:33.257 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:32 smithi063 ceph-mon[42680]: osdmap e356: 8 total, 8 up, 8 in 2024-08-17T16:59:33.257 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:32 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:33.257 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:32 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:33.257 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:32 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:33.257 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:32 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:33.257 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:32 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T16:59:33.258 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:32 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-17T16:59:33.258 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:32 smithi063 ceph-mon[31063]: osdmap e356: 8 total, 8 up, 8 in 2024-08-17T16:59:33.258 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:32 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:33.258 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:32 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:33.258 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:32 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:33.258 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:32 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:33.258 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:32 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T16:59:33.258 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:32 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-17T16:59:33.323 INFO:tasks.rados.rados.0.smithi063.stdout:1156: finishing write tid 1 to smithi06387500-48 2024-08-17T16:59:33.323 INFO:tasks.rados.rados.0.smithi063.stdout:1156: finishing write tid 2 to smithi06387500-48 2024-08-17T16:59:33.323 INFO:tasks.rados.rados.0.smithi063.stdout:1156: finishing write tid 3 to smithi06387500-48 2024-08-17T16:59:33.323 INFO:tasks.rados.rados.0.smithi063.stdout:1156: finishing write tid 4 to smithi06387500-48 2024-08-17T16:59:33.323 INFO:tasks.rados.rados.0.smithi063.stdout:1156: finishing write tid 5 to smithi06387500-48 2024-08-17T16:59:33.323 INFO:tasks.rados.rados.0.smithi063.stdout:1156: oid 48 updating version 0 to 1150 2024-08-17T16:59:33.323 INFO:tasks.rados.rados.0.smithi063.stdout:1156: oid 48 version 1150 is already newer than 1149 2024-08-17T16:59:33.324 INFO:tasks.rados.rados.0.smithi063.stdout:1156: oid 48 updating version 1150 to 1152 2024-08-17T16:59:33.324 INFO:tasks.rados.rados.0.smithi063.stdout:1156: oid 48 version 1152 is already newer than 1151 2024-08-17T16:59:33.324 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 1152 (ObjNum 453 snap 146 seq_num 453) dirty exists 2024-08-17T16:59:33.324 INFO:tasks.rados.rados.0.smithi063.stdout:1156: left oid 48 (ObjNum 453 snap 146 seq_num 453) 2024-08-17T16:59:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:32 smithi162 ceph-mon[34633]: osdmap e356: 8 total, 8 up, 8 in 2024-08-17T16:59:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:32 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:32 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:32 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:32 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:32 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T16:59:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:32 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-08-17T16:59:33.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:59:33] "GET /metrics HTTP/1.1" 200 36221 "" "Prometheus/2.51.0" 2024-08-17T16:59:33.606 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:59:33.607 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T16:59:33.607 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 3 2024-08-17T16:59:33.607 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:59:33.607 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T16:59:33.607 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T16:59:33.607 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:59:33.607 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T16:59:33.607 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T16:59:33.607 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:59:33.607 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T16:59:33.608 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T16:59:33.608 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T16:59:33.608 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T16:59:33.608 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 13, 2024-08-17T16:59:33.608 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T16:59:33.608 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T16:59:33.608 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:59:33.907 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T16:59:33.908 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T16:59:33.908 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-08-17T16:59:33.908 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-08-17T16:59:33.908 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-08-17T16:59:33.908 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "0/3 daemons upgraded", 2024-08-17T16:59:33.908 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading mon daemons", 2024-08-17T16:59:33.908 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T16:59:33.908 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T16:59:33.936 INFO:tasks.rados.rados.0.smithi063.stdout:1154: done (3 left) 2024-08-17T16:59:33.936 INFO:tasks.rados.rados.0.smithi063.stdout:1155: done (2 left) 2024-08-17T16:59:33.936 INFO:tasks.rados.rados.0.smithi063.stdout:1156: done (1 left) 2024-08-17T16:59:33.936 INFO:tasks.rados.rados.0.smithi063.stdout:1157: done (0 left) 2024-08-17T16:59:33.936 INFO:tasks.rados.rados.0.smithi063.stdout:1158: write oid 7 current snap is 147 2024-08-17T16:59:33.936 INFO:tasks.rados.rados.0.smithi063.stdout:1158: seq_num 454 ranges {430378=627873,1700472=622156,3017921=612457} 2024-08-17T16:59:33.946 INFO:tasks.rados.rados.0.smithi063.stdout:1158: writing smithi06387500-7 from 430378 to 1058251 tid 1 2024-08-17T16:59:33.951 INFO:tasks.rados.rados.0.smithi063.stdout:1158: writing smithi06387500-7 from 1700472 to 2322628 tid 2 2024-08-17T16:59:33.960 INFO:tasks.rados.rados.0.smithi063.stdout:1158: writing smithi06387500-7 from 3017921 to 3630378 tid 3 2024-08-17T16:59:33.960 INFO:tasks.rados.rados.0.smithi063.stdout:1159: write oid 19 current snap is 147 2024-08-17T16:59:33.960 INFO:tasks.rados.rados.0.smithi063.stdout:1158: finishing write tid 1 to smithi06387500-7 2024-08-17T16:59:33.960 INFO:tasks.rados.rados.0.smithi063.stdout:1158: finishing write tid 2 to smithi06387500-7 2024-08-17T16:59:33.960 INFO:tasks.rados.rados.0.smithi063.stdout:1159: seq_num 455 ranges {447185=703122,1642601=765814,2928991=447707} 2024-08-17T16:59:33.967 INFO:tasks.rados.rados.0.smithi063.stdout:1159: writing smithi06387500-19 from 447185 to 1150307 tid 1 2024-08-17T16:59:33.975 INFO:tasks.rados.rados.0.smithi063.stdout:1159: writing smithi06387500-19 from 1642601 to 2408415 tid 2 2024-08-17T16:59:33.982 INFO:tasks.rados.rados.0.smithi063.stdout:1159: writing smithi06387500-19 from 2928991 to 3376698 tid 3 2024-08-17T16:59:33.982 INFO:tasks.rados.rados.0.smithi063.stdout:1160: snap_create 2024-08-17T16:59:33.982 INFO:tasks.rados.rados.0.smithi063.stdout:1158: finishing write tid 3 to smithi06387500-7 2024-08-17T16:59:33.982 INFO:tasks.rados.rados.0.smithi063.stdout:1158: finishing write tid 4 to smithi06387500-7 2024-08-17T16:59:33.982 INFO:tasks.rados.rados.0.smithi063.stdout:1158: finishing write tid 5 to smithi06387500-7 2024-08-17T16:59:33.982 INFO:tasks.rados.rados.0.smithi063.stdout:1158: oid 7 updating version 0 to 637 2024-08-17T16:59:33.982 INFO:tasks.rados.rados.0.smithi063.stdout:1158: oid 7 version 637 is already newer than 635 2024-08-17T16:59:33.982 INFO:tasks.rados.rados.0.smithi063.stdout:1158: oid 7 version 637 is already newer than 636 2024-08-17T16:59:33.982 INFO:tasks.rados.rados.0.smithi063.stdout:1158: oid 7 updating version 637 to 638 2024-08-17T16:59:33.982 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 638 (ObjNum 454 snap 147 seq_num 454) dirty exists 2024-08-17T16:59:33.982 INFO:tasks.rados.rados.0.smithi063.stdout:1158: left oid 7 (ObjNum 454 snap 147 seq_num 454) 2024-08-17T16:59:33.982 INFO:tasks.rados.rados.0.smithi063.stdout:1159: finishing write tid 1 to smithi06387500-19 2024-08-17T16:59:33.983 INFO:tasks.rados.rados.0.smithi063.stdout:1159: finishing write tid 2 to smithi06387500-19 2024-08-17T16:59:33.984 INFO:tasks.rados.rados.0.smithi063.stdout:1159: finishing write tid 3 to smithi06387500-19 2024-08-17T16:59:33.984 INFO:tasks.rados.rados.0.smithi063.stdout:1159: finishing write tid 4 to smithi06387500-19 2024-08-17T16:59:33.984 INFO:tasks.rados.rados.0.smithi063.stdout:1159: finishing write tid 5 to smithi06387500-19 2024-08-17T16:59:33.984 INFO:tasks.rados.rados.0.smithi063.stdout:1159: oid 19 updating version 0 to 773 2024-08-17T16:59:33.985 INFO:tasks.rados.rados.0.smithi063.stdout:1159: oid 19 version 773 is already newer than 772 2024-08-17T16:59:33.985 INFO:tasks.rados.rados.0.smithi063.stdout:1159: oid 19 updating version 773 to 774 2024-08-17T16:59:33.985 INFO:tasks.rados.rados.0.smithi063.stdout:1159: oid 19 version 774 is already newer than 771 2024-08-17T16:59:33.985 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 774 (ObjNum 455 snap 147 seq_num 455) dirty exists 2024-08-17T16:59:33.985 INFO:tasks.rados.rados.0.smithi063.stdout:1159: left oid 19 (ObjNum 455 snap 147 seq_num 455) 2024-08-17T16:59:34.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[31063]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T16:59:34.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[31063]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T16:59:34.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[31063]: Upgrade: Setting container_image for all mgr 2024-08-17T16:59:34.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[31063]: Upgrade: It appears safe to stop mon.a 2024-08-17T16:59:34.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[31063]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:34.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[31063]: pgmap v219: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 161 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 824 KiB/s wr, 12 op/s 2024-08-17T16:59:34.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[31063]: osdmap e357: 8 total, 8 up, 8 in 2024-08-17T16:59:34.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[31063]: from='client.? 172.21.15.63:0/3939053019' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:34.265 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:34.265 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:59:34.265 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:59:34.265 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[31063]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:34.265 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[42680]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T16:59:34.265 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[42680]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T16:59:34.265 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[42680]: Upgrade: Setting container_image for all mgr 2024-08-17T16:59:34.265 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[42680]: Upgrade: It appears safe to stop mon.a 2024-08-17T16:59:34.265 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[42680]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:34.266 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[42680]: pgmap v219: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 161 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 824 KiB/s wr, 12 op/s 2024-08-17T16:59:34.266 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[42680]: osdmap e357: 8 total, 8 up, 8 in 2024-08-17T16:59:34.266 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[42680]: from='client.? 172.21.15.63:0/3939053019' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:34.266 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:34.266 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:59:34.266 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:59:34.266 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:33 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:33 smithi162 ceph-mon[34633]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T16:59:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:33 smithi162 ceph-mon[34633]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T16:59:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:33 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all mgr 2024-08-17T16:59:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:33 smithi162 ceph-mon[34633]: Upgrade: It appears safe to stop mon.a 2024-08-17T16:59:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:33 smithi162 ceph-mon[34633]: from='client.15141 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:33 smithi162 ceph-mon[34633]: pgmap v219: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 161 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 824 KiB/s wr, 12 op/s 2024-08-17T16:59:34.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:33 smithi162 ceph-mon[34633]: osdmap e357: 8 total, 8 up, 8 in 2024-08-17T16:59:34.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:33 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3939053019' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:34.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:33 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' 2024-08-17T16:59:34.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:33 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:59:34.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:33 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:59:34.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:33 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:34.894 INFO:tasks.rados.rados.0.smithi063.stdout:1158: done (2 left) 2024-08-17T16:59:34.895 INFO:tasks.rados.rados.0.smithi063.stdout:1159: done (1 left) 2024-08-17T16:59:34.895 INFO:tasks.rados.rados.0.smithi063.stdout:1160: done (0 left) 2024-08-17T16:59:34.895 INFO:tasks.rados.rados.0.smithi063.stdout:1161: read oid 47 snap 132 2024-08-17T16:59:34.895 INFO:tasks.rados.rados.0.smithi063.stdout:1161: expect (ObjNum 416 snap 128 seq_num 416) 2024-08-17T16:59:34.895 INFO:tasks.rados.rados.0.smithi063.stdout:1162: snap_remove snap 133 2024-08-17T16:59:35.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:34 smithi063 ceph-mon[42680]: from='client.25040 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:35.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:34 smithi063 ceph-mon[42680]: from='client.25046 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:35.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:34 smithi063 ceph-mon[42680]: Upgrade: Updating mon.a 2024-08-17T16:59:35.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:34 smithi063 ceph-mon[42680]: Deploying daemon mon.a on smithi063 2024-08-17T16:59:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:34 smithi063 ceph-mon[42680]: from='client.25058 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:34 smithi063 ceph-mon[42680]: osdmap e358: 8 total, 8 up, 8 in 2024-08-17T16:59:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:34 smithi063 ceph-mon[42680]: osdmap e359: 8 total, 8 up, 8 in 2024-08-17T16:59:35.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:34 smithi063 ceph-mon[31063]: from='client.25040 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:35.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:34 smithi063 ceph-mon[31063]: from='client.25046 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:35.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:34 smithi063 ceph-mon[31063]: Upgrade: Updating mon.a 2024-08-17T16:59:35.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:34 smithi063 ceph-mon[31063]: Deploying daemon mon.a on smithi063 2024-08-17T16:59:35.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:34 smithi063 ceph-mon[31063]: from='client.25058 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:35.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:34 smithi063 ceph-mon[31063]: osdmap e358: 8 total, 8 up, 8 in 2024-08-17T16:59:35.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:34 smithi063 ceph-mon[31063]: osdmap e359: 8 total, 8 up, 8 in 2024-08-17T16:59:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:34 smithi162 ceph-mon[34633]: from='client.25040 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:34 smithi162 ceph-mon[34633]: from='client.25046 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:34 smithi162 ceph-mon[34633]: Upgrade: Updating mon.a 2024-08-17T16:59:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:34 smithi162 ceph-mon[34633]: Deploying daemon mon.a on smithi063 2024-08-17T16:59:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:34 smithi162 ceph-mon[34633]: from='client.25058 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T16:59:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:34 smithi162 ceph-mon[34633]: osdmap e358: 8 total, 8 up, 8 in 2024-08-17T16:59:35.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:34 smithi162 ceph-mon[34633]: osdmap e359: 8 total, 8 up, 8 in 2024-08-17T16:59:35.898 INFO:tasks.rados.rados.0.smithi063.stdout:1162: done (1 left) 2024-08-17T16:59:35.899 INFO:tasks.rados.rados.0.smithi063.stdout:1163: snap_remove snap 147 2024-08-17T16:59:35.917 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:35 smithi063 systemd[1]: Stopping Ceph mon.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:59:36.241 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a[31040]: 2024-08-17T16:59:35.915+0000 7f915368b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T16:59:36.242 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a[31040]: 2024-08-17T16:59:35.915+0000 7f915368b640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-08-17T16:59:36.242 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:35 smithi063 podman[109644]: 2024-08-17 16:59:35.9365965 +0000 UTC m=+0.139091146 container died 0c71ddd7ada67f493be747616ba47d4bd768d4b951091cc55699858aff6fe2c2 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-08-17T16:59:36.242 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:59:35] ENGINE Bus STOPPING 2024-08-17T16:59:36.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:36 smithi063 podman[109644]: 2024-08-17 16:59:36.330665402 +0000 UTC m=+0.533160045 container cleanup 0c71ddd7ada67f493be747616ba47d4bd768d4b951091cc55699858aff6fe2c2 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-08-17T16:59:36.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:36 smithi063 bash[109644]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a 2024-08-17T16:59:36.530 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:59:36] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T16:59:36.530 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:59:36] ENGINE Bus STOPPED 2024-08-17T16:59:36.530 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:59:36] ENGINE Bus STARTING 2024-08-17T16:59:36.530 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:59:36] ENGINE Serving on http://:::9283 2024-08-17T16:59:36.530 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:59:36] ENGINE Bus STARTED 2024-08-17T16:59:36.972 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:36 smithi063 podman[109662]: 2024-08-17 16:59:36.599241815 +0000 UTC m=+0.661402415 container remove 0c71ddd7ada67f493be747616ba47d4bd768d4b951091cc55699858aff6fe2c2 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, ceph=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2) 2024-08-17T16:59:36.972 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:36 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.a.service: Deactivated successfully. 2024-08-17T16:59:36.972 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:36 smithi063 systemd[1]: Stopped Ceph mon.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:59:36.972 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:36 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.a.service: Consumed 16.885s CPU time. 2024-08-17T16:59:36.973 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:36 smithi063 systemd[1]: Starting Ceph mon.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:59:37.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 podman[109805]: 2024-08-17 16:59:37.094468672 +0000 UTC m=+0.018188009 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T16:59:37.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 podman[109805]: 2024-08-17 16:59:37.208968199 +0000 UTC m=+0.132687526 container create f0f20fcac9560a4aa8aa604ab9353004e307d744ed2b7303116a54290134b903 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-08-17T16:59:37.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 podman[109805]: 2024-08-17 16:59:37.428415528 +0000 UTC m=+0.352134856 container init f0f20fcac9560a4aa8aa604ab9353004e307d744ed2b7303116a54290134b903 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, ceph=True) 2024-08-17T16:59:37.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 podman[109805]: 2024-08-17 16:59:37.433603767 +0000 UTC m=+0.357323095 container start f0f20fcac9560a4aa8aa604ab9353004e307d744ed2b7303116a54290134b903 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a, ceph=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-08-17T16:59:37.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T16:59:37.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev), process ceph-mon, pid 7 2024-08-17T16:59:37.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: pidfile_write: ignore empty --pid-file 2024-08-17T16:59:37.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: load: jerasure load: lrc 2024-08-17T16:59:37.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: RocksDB version: 7.9.2 2024-08-17T16:59:37.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Git sha 0 2024-08-17T16:59:37.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Compile date 2024-08-15 20:43:31 2024-08-17T16:59:37.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: DB SUMMARY 2024-08-17T16:59:37.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: DB Session ID: GRB5V3R3HUC2AUZ1MCZX 2024-08-17T16:59:37.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: CURRENT file: CURRENT 2024-08-17T16:59:37.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T16:59:37.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1519 Bytes 2024-08-17T16:59:37.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000032.sst 2024-08-17T16:59:37.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000030.log size: 7758307 ; 2024-08-17T16:59:37.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.error_if_exists: 0 2024-08-17T16:59:37.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.create_if_missing: 0 2024-08-17T16:59:37.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.paranoid_checks: 1 2024-08-17T16:59:37.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T16:59:37.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T16:59:37.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T16:59:37.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.env: 0x55a6553cabe0 2024-08-17T16:59:37.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T16:59:37.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.info_log: 0x55a656dc5700 2024-08-17T16:59:37.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T16:59:37.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.statistics: (nil) 2024-08-17T16:59:37.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.use_fsync: 0 2024-08-17T16:59:37.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_log_file_size: 0 2024-08-17T16:59:37.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T16:59:37.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T16:59:37.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T16:59:37.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T16:59:37.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.allow_fallocate: 1 2024-08-17T16:59:37.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T16:59:37.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T16:59:37.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.use_direct_reads: 0 2024-08-17T16:59:37.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T16:59:37.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T16:59:37.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.db_log_dir: 2024-08-17T16:59:37.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.wal_dir: 2024-08-17T16:59:37.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T16:59:37.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T16:59:37.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T16:59:37.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T16:59:37.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T16:59:37.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T16:59:37.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T16:59:37.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T16:59:37.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.write_buffer_manager: 0x55a656dc8960 2024-08-17T16:59:37.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T16:59:37.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T16:59:37.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T16:59:37.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T16:59:37.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T16:59:37.785 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T16:59:37.785 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T16:59:37.785 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T16:59:37.785 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.unordered_write: 0 2024-08-17T16:59:37.785 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T16:59:37.785 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T16:59:37.785 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T16:59:37.785 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T16:59:37.785 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.row_cache: None 2024-08-17T16:59:37.785 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.wal_filter: None 2024-08-17T16:59:37.787 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T16:59:37.787 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T16:59:37.787 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.two_write_queues: 0 2024-08-17T16:59:37.787 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T16:59:37.787 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.wal_compression: 0 2024-08-17T16:59:37.787 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.atomic_flush: 0 2024-08-17T16:59:37.787 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T16:59:37.787 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T16:59:37.787 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T16:59:37.787 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.log_readahead_size: 0 2024-08-17T16:59:37.787 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T16:59:37.787 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T16:59:37.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T16:59:37.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T16:59:37.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T16:59:37.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T16:59:37.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T16:59:37.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_background_jobs: 2 2024-08-17T16:59:37.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_background_compactions: -1 2024-08-17T16:59:37.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_subcompactions: 1 2024-08-17T16:59:37.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T16:59:37.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T16:59:37.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T16:59:37.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T16:59:37.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T16:59:37.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T16:59:37.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T16:59:37.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T16:59:37.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_open_files: -1 2024-08-17T16:59:37.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T16:59:37.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T16:59:37.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T16:59:37.790 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T16:59:37.790 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_background_flushes: -1 2024-08-17T16:59:37.790 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Compression algorithms supported: 2024-08-17T16:59:37.790 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: kZSTD supported: 0 2024-08-17T16:59:37.790 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: kXpressCompression supported: 0 2024-08-17T16:59:37.790 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: kBZip2Compression supported: 0 2024-08-17T16:59:37.790 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T16:59:37.790 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: kLZ4Compression supported: 1 2024-08-17T16:59:37.790 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: kZlibCompression supported: 1 2024-08-17T16:59:37.790 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T16:59:37.790 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: kSnappyCompression supported: 1 2024-08-17T16:59:37.790 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T16:59:37.791 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T16:59:37.791 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-08-17T16:59:37.791 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T16:59:37.791 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T16:59:37.791 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.merge_operator: 2024-08-17T16:59:37.791 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compaction_filter: None 2024-08-17T16:59:37.791 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T16:59:37.791 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T16:59:37.791 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T16:59:37.791 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T16:59:37.791 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a656dc5380) 2024-08-17T16:59:37.791 INFO:journalctl@ceph.mon.a.smithi063.stdout: cache_index_and_filter_blocks: 1 2024-08-17T16:59:37.791 INFO:journalctl@ceph.mon.a.smithi063.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T16:59:37.792 INFO:journalctl@ceph.mon.a.smithi063.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T16:59:37.792 INFO:journalctl@ceph.mon.a.smithi063.stdout: pin_top_level_index_and_filter: 1 2024-08-17T16:59:37.792 INFO:journalctl@ceph.mon.a.smithi063.stdout: index_type: 0 2024-08-17T16:59:37.792 INFO:journalctl@ceph.mon.a.smithi063.stdout: data_block_index_type: 0 2024-08-17T16:59:37.792 INFO:journalctl@ceph.mon.a.smithi063.stdout: index_shortening: 1 2024-08-17T16:59:37.792 INFO:journalctl@ceph.mon.a.smithi063.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T16:59:37.792 INFO:journalctl@ceph.mon.a.smithi063.stdout: checksum: 4 2024-08-17T16:59:37.792 INFO:journalctl@ceph.mon.a.smithi063.stdout: no_block_cache: 0 2024-08-17T16:59:37.792 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_cache: 0x55a656db9610 2024-08-17T16:59:37.792 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_cache_name: BinnedLRUCache 2024-08-17T16:59:37.792 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_cache_options: 2024-08-17T16:59:37.792 INFO:journalctl@ceph.mon.a.smithi063.stdout: capacity : 536870912 2024-08-17T16:59:37.792 INFO:journalctl@ceph.mon.a.smithi063.stdout: num_shard_bits : 4 2024-08-17T16:59:37.793 INFO:journalctl@ceph.mon.a.smithi063.stdout: strict_capacity_limit : 0 2024-08-17T16:59:37.793 INFO:journalctl@ceph.mon.a.smithi063.stdout: high_pri_pool_ratio: 0.000 2024-08-17T16:59:37.793 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_cache_compressed: (nil) 2024-08-17T16:59:37.793 INFO:journalctl@ceph.mon.a.smithi063.stdout: persistent_cache: (nil) 2024-08-17T16:59:37.793 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_size: 4096 2024-08-17T16:59:37.793 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_size_deviation: 10 2024-08-17T16:59:37.793 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_restart_interval: 16 2024-08-17T16:59:37.793 INFO:journalctl@ceph.mon.a.smithi063.stdout: index_block_restart_interval: 1 2024-08-17T16:59:37.793 INFO:journalctl@ceph.mon.a.smithi063.stdout: metadata_block_size: 4096 2024-08-17T16:59:37.793 INFO:journalctl@ceph.mon.a.smithi063.stdout: partition_filters: 0 2024-08-17T16:59:37.793 INFO:journalctl@ceph.mon.a.smithi063.stdout: use_delta_encoding: 1 2024-08-17T16:59:37.793 INFO:journalctl@ceph.mon.a.smithi063.stdout: filter_policy: bloomfilter 2024-08-17T16:59:37.793 INFO:journalctl@ceph.mon.a.smithi063.stdout: whole_key_filtering: 1 2024-08-17T16:59:37.794 INFO:journalctl@ceph.mon.a.smithi063.stdout: verify_compression: 0 2024-08-17T16:59:37.794 INFO:journalctl@ceph.mon.a.smithi063.stdout: read_amp_bytes_per_bit: 0 2024-08-17T16:59:37.794 INFO:journalctl@ceph.mon.a.smithi063.stdout: format_version: 5 2024-08-17T16:59:37.794 INFO:journalctl@ceph.mon.a.smithi063.stdout: enable_index_compression: 1 2024-08-17T16:59:37.794 INFO:journalctl@ceph.mon.a.smithi063.stdout: block_align: 0 2024-08-17T16:59:37.794 INFO:journalctl@ceph.mon.a.smithi063.stdout: max_auto_readahead_size: 262144 2024-08-17T16:59:37.794 INFO:journalctl@ceph.mon.a.smithi063.stdout: prepopulate_block_cache: 0 2024-08-17T16:59:37.794 INFO:journalctl@ceph.mon.a.smithi063.stdout: initial_auto_readahead_size: 8192 2024-08-17T16:59:37.794 INFO:journalctl@ceph.mon.a.smithi063.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T16:59:37.794 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T16:59:37.794 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T16:59:37.794 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compression: NoCompression 2024-08-17T16:59:37.794 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T16:59:37.795 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T16:59:37.795 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T16:59:37.795 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.num_levels: 7 2024-08-17T16:59:37.795 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T16:59:37.795 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T16:59:37.795 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T16:59:37.795 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T16:59:37.795 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T16:59:37.795 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T16:59:37.795 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T16:59:37.795 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T16:59:37.795 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T16:59:37.796 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T16:59:37.796 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T16:59:37.796 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T16:59:37.796 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T16:59:37.796 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T16:59:37.796 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T16:59:37.796 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T16:59:37.796 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T16:59:37.796 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T16:59:37.796 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T16:59:37.796 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T16:59:37.796 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T16:59:37.796 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T16:59:37.797 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T16:59:37.797 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T16:59:37.797 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T16:59:37.797 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T16:59:37.797 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T16:59:37.797 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T16:59:37.797 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T16:59:37.797 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T16:59:37.797 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T16:59:37.797 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T16:59:37.797 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T16:59:37.797 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T16:59:37.797 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T16:59:37.797 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T16:59:37.798 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T16:59:37.798 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T16:59:37.798 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T16:59:37.798 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T16:59:37.798 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T16:59:37.798 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T16:59:37.798 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T16:59:37.798 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T16:59:37.798 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T16:59:37.798 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T16:59:37.798 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T16:59:37.798 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T16:59:37.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T16:59:37.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T16:59:37.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T16:59:37.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T16:59:37.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T16:59:37.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T16:59:37.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.inplace_update_support: 0 2024-08-17T16:59:37.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T16:59:37.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T16:59:37.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T16:59:37.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T16:59:37.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.bloom_locality: 0 2024-08-17T16:59:37.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.max_successive_merges: 0 2024-08-17T16:59:37.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T16:59:37.800 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T16:59:37.800 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T16:59:37.800 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T16:59:37.800 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.ttl: 2592000 2024-08-17T16:59:37.800 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T16:59:37.800 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T16:59:37.800 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T16:59:37.800 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.enable_blob_files: false 2024-08-17T16:59:37.800 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.min_blob_size: 0 2024-08-17T16:59:37.800 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T16:59:37.800 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T16:59:37.800 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T16:59:37.800 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T16:59:37.801 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T16:59:37.801 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T16:59:37.801 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T16:59:37.801 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T16:59:37.801 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T16:59:37.801 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 34, last_sequence is 12747, log_number is 30,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 30 2024-08-17T16:59:37.801 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 30 2024-08-17T16:59:37.801 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 6d2608df-0a6f-415d-a7b9-c671a8b0b588 2024-08-17T16:59:37.801 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913977460547, "job": 1, "event": "recovery_started", "wal_files": [30]} 2024-08-17T16:59:37.801 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #30 mode 2 2024-08-17T16:59:37.801 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913977485351, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 35, "file_size": 7221157, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12748, "largest_seqno": 17569, "table_properties": {"data_size": 7198455, "index_size": 16151, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 5509, "raw_key_size": 52025, "raw_average_key_size": 23, "raw_value_size": 7151314, "raw_average_value_size": 3269, "num_data_blocks": 698, "num_entries": 2187, "num_filter_entries": 2187, "num_deletions": 365, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723913977, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "6d2608df-0a6f-415d-a7b9-c671a8b0b588", "db_session_id": "GRB5V3R3HUC2AUZ1MCZX", "orig_file_number": 35, "seqno_to_time_mapping": "N/A"}} 2024-08-17T16:59:37.801 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723913977485482, "job": 1, "event": "recovery_finished"} 2024-08-17T16:59:37.801 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: [db/version_set.cc:5047] Creating manifest 37 2024-08-17T16:59:37.801 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T16:59:37.802 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000030.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T16:59:37.802 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a656de8e00 2024-08-17T16:59:37.802 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: DB pointer 0x55a656ee6000 2024-08-17T16:59:37.802 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T16:59:37.802 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T16:59:37.802 INFO:journalctl@ceph.mon.a.smithi063.stdout: ** DB Stats ** 2024-08-17T16:59:37.802 INFO:journalctl@ceph.mon.a.smithi063.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:59:37.802 INFO:journalctl@ceph.mon.a.smithi063.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T16:59:37.802 INFO:journalctl@ceph.mon.a.smithi063.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T16:59:37.802 INFO:journalctl@ceph.mon.a.smithi063.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T16:59:37.802 INFO:journalctl@ceph.mon.a.smithi063.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T16:59:37.802 INFO:journalctl@ceph.mon.a.smithi063.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T16:59:37.803 INFO:journalctl@ceph.mon.a.smithi063.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T16:59:37.803 INFO:journalctl@ceph.mon.a.smithi063.stdout: 2024-08-17T16:59:37.803 INFO:journalctl@ceph.mon.a.smithi063.stdout: ** Compaction Stats [default] ** 2024-08-17T16:59:37.803 INFO:journalctl@ceph.mon.a.smithi063.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T16:59:37.803 INFO:journalctl@ceph.mon.a.smithi063.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T16:59:37.803 INFO:journalctl@ceph.mon.a.smithi063.stdout: L0 1/0 6.89 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 534.7 0.01 0.00 1 0.013 0 0 0.0 0.0 2024-08-17T16:59:37.803 INFO:journalctl@ceph.mon.a.smithi063.stdout: L6 1/0 7.35 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-17T16:59:37.803 INFO:journalctl@ceph.mon.a.smithi063.stdout: Sum 2/0 14.24 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 534.7 0.01 0.00 1 0.013 0 0 0.0 0.0 2024-08-17T16:59:37.803 INFO:journalctl@ceph.mon.a.smithi063.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 534.7 0.01 0.00 1 0.013 0 0 0.0 0.0 2024-08-17T16:59:37.803 INFO:journalctl@ceph.mon.a.smithi063.stdout: 2024-08-17T16:59:37.803 INFO:journalctl@ceph.mon.a.smithi063.stdout: ** Compaction Stats [default] ** 2024-08-17T16:59:37.803 INFO:journalctl@ceph.mon.a.smithi063.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T16:59:37.803 INFO:journalctl@ceph.mon.a.smithi063.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T16:59:37.803 INFO:journalctl@ceph.mon.a.smithi063.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 534.7 0.01 0.00 1 0.013 0 0 0.0 0.0 2024-08-17T16:59:37.804 INFO:journalctl@ceph.mon.a.smithi063.stdout: 2024-08-17T16:59:37.804 INFO:journalctl@ceph.mon.a.smithi063.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T16:59:37.804 INFO:journalctl@ceph.mon.a.smithi063.stdout: 2024-08-17T16:59:37.804 INFO:journalctl@ceph.mon.a.smithi063.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T16:59:37.804 INFO:journalctl@ceph.mon.a.smithi063.stdout: Flush(GB): cumulative 0.007, interval 0.007 2024-08-17T16:59:37.804 INFO:journalctl@ceph.mon.a.smithi063.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T16:59:37.804 INFO:journalctl@ceph.mon.a.smithi063.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T16:59:37.804 INFO:journalctl@ceph.mon.a.smithi063.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T16:59:37.804 INFO:journalctl@ceph.mon.a.smithi063.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T16:59:37.804 INFO:journalctl@ceph.mon.a.smithi063.stdout: Cumulative compaction: 0.01 GB write, 246.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:59:37.804 INFO:journalctl@ceph.mon.a.smithi063.stdout: Interval compaction: 0.01 GB write, 246.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T16:59:37.804 INFO:journalctl@ceph.mon.a.smithi063.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T16:59:37.804 INFO:journalctl@ceph.mon.a.smithi063.stdout: Block cache BinnedLRUCache@0x55a656db9610#7 capacity: 512.00 MB usage: 22.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-08-17T16:59:37.805 INFO:journalctl@ceph.mon.a.smithi063.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,6.03 KB,0.00115037%) IndexBlock(1,16.08 KB,0.00306666%) Misc(1,0.00 KB,0%) 2024-08-17T16:59:37.805 INFO:journalctl@ceph.mon.a.smithi063.stdout: 2024-08-17T16:59:37.805 INFO:journalctl@ceph.mon.a.smithi063.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T16:59:37.805 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: starting mon.a rank 0 at public addrs [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] at bind addrs [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:59:37.805 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: mon.a@-1(???) e3 preinit fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:59:37.805 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: mon.a@-1(???).mds e8 new map 2024-08-17T16:59:37.805 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: mon.a@-1(???).mds e8 print_map 2024-08-17T16:59:37.805 INFO:journalctl@ceph.mon.a.smithi063.stdout: e8 2024-08-17T16:59:37.805 INFO:journalctl@ceph.mon.a.smithi063.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-17T16:59:37.805 INFO:journalctl@ceph.mon.a.smithi063.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T16:59:37.805 INFO:journalctl@ceph.mon.a.smithi063.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T16:59:37.805 INFO:journalctl@ceph.mon.a.smithi063.stdout: legacy client fscid: 1 2024-08-17T16:59:37.805 INFO:journalctl@ceph.mon.a.smithi063.stdout: 2024-08-17T16:59:37.806 INFO:journalctl@ceph.mon.a.smithi063.stdout: Filesystem 'foo' (1) 2024-08-17T16:59:37.806 INFO:journalctl@ceph.mon.a.smithi063.stdout: fs_name foo 2024-08-17T16:59:37.806 INFO:journalctl@ceph.mon.a.smithi063.stdout: epoch 7 2024-08-17T16:59:37.806 INFO:journalctl@ceph.mon.a.smithi063.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-08-17T16:59:37.806 INFO:journalctl@ceph.mon.a.smithi063.stdout: created 2024-08-17T16:53:36.880465+0000 2024-08-17T16:59:37.806 INFO:journalctl@ceph.mon.a.smithi063.stdout: modified 2024-08-17T16:53:54.414788+0000 2024-08-17T16:59:37.806 INFO:journalctl@ceph.mon.a.smithi063.stdout: tableserver 0 2024-08-17T16:59:37.806 INFO:journalctl@ceph.mon.a.smithi063.stdout: root 0 2024-08-17T16:59:37.806 INFO:journalctl@ceph.mon.a.smithi063.stdout: session_timeout 60 2024-08-17T16:59:37.806 INFO:journalctl@ceph.mon.a.smithi063.stdout: session_autoclose 300 2024-08-17T16:59:37.806 INFO:journalctl@ceph.mon.a.smithi063.stdout: max_file_size 1099511627776 2024-08-17T16:59:37.806 INFO:journalctl@ceph.mon.a.smithi063.stdout: max_xattr_size 65536 2024-08-17T16:59:37.806 INFO:journalctl@ceph.mon.a.smithi063.stdout: required_client_features {} 2024-08-17T16:59:37.807 INFO:journalctl@ceph.mon.a.smithi063.stdout: last_failure 0 2024-08-17T16:59:37.807 INFO:journalctl@ceph.mon.a.smithi063.stdout: last_failure_osd_epoch 0 2024-08-17T16:59:37.807 INFO:journalctl@ceph.mon.a.smithi063.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T16:59:37.807 INFO:journalctl@ceph.mon.a.smithi063.stdout: max_mds 1 2024-08-17T16:59:37.807 INFO:journalctl@ceph.mon.a.smithi063.stdout: in 0 2024-08-17T16:59:37.807 INFO:journalctl@ceph.mon.a.smithi063.stdout: up {0=14658} 2024-08-17T16:59:37.807 INFO:journalctl@ceph.mon.a.smithi063.stdout: failed 2024-08-17T16:59:37.807 INFO:journalctl@ceph.mon.a.smithi063.stdout: damaged 2024-08-17T16:59:37.807 INFO:journalctl@ceph.mon.a.smithi063.stdout: stopped 2024-08-17T16:59:37.807 INFO:journalctl@ceph.mon.a.smithi063.stdout: data_pools [4] 2024-08-17T16:59:37.807 INFO:journalctl@ceph.mon.a.smithi063.stdout: metadata_pool 3 2024-08-17T16:59:37.807 INFO:journalctl@ceph.mon.a.smithi063.stdout: inline_data disabled 2024-08-17T16:59:37.807 INFO:journalctl@ceph.mon.a.smithi063.stdout: balancer 2024-08-17T16:59:37.808 INFO:journalctl@ceph.mon.a.smithi063.stdout: bal_rank_mask -1 2024-08-17T16:59:37.808 INFO:journalctl@ceph.mon.a.smithi063.stdout: standby_count_wanted 1 2024-08-17T16:59:37.808 INFO:journalctl@ceph.mon.a.smithi063.stdout: qdb_cluster leader: 0 members: 2024-08-17T16:59:37.808 INFO:journalctl@ceph.mon.a.smithi063.stdout: [mds.foo.smithi063.ultpqj{0:14658} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.63:6834/3222014296,v1:172.21.15.63:6835/3222014296] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T16:59:37.808 INFO:journalctl@ceph.mon.a.smithi063.stdout: 2024-08-17T16:59:37.808 INFO:journalctl@ceph.mon.a.smithi063.stdout: 2024-08-17T16:59:37.808 INFO:journalctl@ceph.mon.a.smithi063.stdout: Standby daemons: 2024-08-17T16:59:37.808 INFO:journalctl@ceph.mon.a.smithi063.stdout: 2024-08-17T16:59:37.808 INFO:journalctl@ceph.mon.a.smithi063.stdout: [mds.foo.smithi162.bjkxnv{-1:24529} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.162:6832/2161520737,v1:172.21.15.162:6833/2161520737] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T16:59:37.808 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: mon.a@-1(???).osd e360 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-17T16:59:37.808 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: mon.a@-1(???).osd e360 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T16:59:37.808 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: mon.a@-1(???).osd e360 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T16:59:37.808 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: mon.a@-1(???).osd e360 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T16:59:37.808 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 ceph-mon[109843]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-08-17T16:59:37.809 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 bash[109805]: f0f20fcac9560a4aa8aa604ab9353004e307d744ed2b7303116a54290134b903 2024-08-17T16:59:37.809 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:37 smithi063 systemd[1]: Started Ceph mon.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T16:59:40.579 INFO:tasks.rados.rados.0.smithi063.stdout:1163: done (1 left) 2024-08-17T16:59:40.579 INFO:tasks.rados.rados.0.smithi063.stdout:1164: delete oid 21 current snap is 148 2024-08-17T16:59:40.579 INFO:tasks.rados.rados.0.smithi063.stdout:1161: expect (ObjNum 416 snap 128 seq_num 416) 2024-08-17T16:59:40.754 INFO:tasks.rados.rados.0.smithi063.stdout:1161: done (1 left) 2024-08-17T16:59:40.755 INFO:tasks.rados.rados.0.smithi063.stdout:1164: done (0 left) 2024-08-17T16:59:40.755 INFO:tasks.rados.rados.0.smithi063.stdout:1165: rollback oid 19 current snap is 148 2024-08-17T16:59:40.755 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 19 to 141 2024-08-17T16:59:40.755 INFO:tasks.rados.rados.0.smithi063.stdout:1166: read oid 5 snap -1 2024-08-17T16:59:40.755 INFO:tasks.rados.rados.0.smithi063.stdout:1166: expect (ObjNum 443 snap 139 seq_num 443) 2024-08-17T16:59:40.755 INFO:tasks.rados.rados.0.smithi063.stdout:1167: write oid 1 current snap is 148 2024-08-17T16:59:40.755 INFO:tasks.rados.rados.0.smithi063.stdout:1167: seq_num 457 ranges {480799=569973,1475918=769483,2700190=586604} 2024-08-17T16:59:40.760 INFO:tasks.rados.rados.0.smithi063.stdout:1167: writing smithi06387500-1 from 480799 to 1050772 tid 1 2024-08-17T16:59:40.767 INFO:tasks.rados.rados.0.smithi063.stdout:1167: writing smithi06387500-1 from 1475918 to 2245401 tid 2 2024-08-17T16:59:40.773 INFO:tasks.rados.rados.0.smithi063.stdout:1167: writing smithi06387500-1 from 2700190 to 3286794 tid 3 2024-08-17T16:59:40.773 INFO:tasks.rados.rados.0.smithi063.stdout:1168: rollback oid 10 current snap is 148 2024-08-17T16:59:40.773 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 10 to 146 2024-08-17T16:59:40.774 INFO:tasks.rados.rados.0.smithi063.stdout:1165: finishing rollback tid 0 to smithi06387500-19 2024-08-17T16:59:40.774 INFO:tasks.rados.rados.0.smithi063.stdout:1165: finishing rollback tid 1 to smithi06387500-19 2024-08-17T16:59:40.774 INFO:tasks.rados.rados.0.smithi063.stdout:1165: finishing rollback tid 2 to smithi06387500-19 2024-08-17T16:59:40.774 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 785 (ObjNum 334 snap 101 seq_num 334) dirty exists 2024-08-17T16:59:40.774 INFO:tasks.rados.rados.0.smithi063.stdout:1166: expect (ObjNum 443 snap 139 seq_num 443) 2024-08-17T16:59:40.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[81670]: ignoring --setuser ceph since I am not root 2024-08-17T16:59:40.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[81670]: ignoring --setgroup ceph since I am not root 2024-08-17T16:59:40.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mgr[81701]: -- 172.21.15.162:0/3045164833 <== mon.1 v2:172.21.15.162:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564afa2570e0 con 0x564afa235400 2024-08-17T16:59:40.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:59:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:59:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:59:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:59:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:59:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: pgmap v224: 89 pgs: 89 active+clean; 173 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-08-17T16:59:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: pgmap v225: 89 pgs: 89 active+clean; 173 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T16:59:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:40.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: mon.a calling monitor election 2024-08-17T16:59:40.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T16:59:40.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: monmap epoch 3 2024-08-17T16:59:40.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:59:40.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: last_changed 2024-08-17T16:48:24.893441+0000 2024-08-17T16:59:40.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: created 2024-08-17T16:46:51.197040+0000 2024-08-17T16:59:40.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: min_mon_release 18 (reef) 2024-08-17T16:59:40.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: election_strategy: 1 2024-08-17T16:59:40.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: 0: [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon.a 2024-08-17T16:59:40.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: 1: [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] mon.b 2024-08-17T16:59:40.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: 2: [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] mon.c 2024-08-17T16:59:40.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T16:59:40.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: osdmap e360: 8 total, 8 up, 8 in 2024-08-17T16:59:40.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: mgrmap e34: y(active, since 2m), standbys: x 2024-08-17T16:59:40.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: overall HEALTH_OK 2024-08-17T16:59:40.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: from='mgr.24854 ' entity='' 2024-08-17T16:59:40.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:40 smithi162 ceph-mon[34633]: mgrmap e35: y(active, since 2m), standbys: x 2024-08-17T16:59:40.971 INFO:tasks.rados.rados.0.smithi063.stdout:1167: finishing write tid 1 to smithi06387500-1 2024-08-17T16:59:40.971 INFO:tasks.rados.rados.0.smithi063.stdout:1167: finishing write tid 2 to smithi06387500-1 2024-08-17T16:59:40.972 INFO:tasks.rados.rados.0.smithi063.stdout:1168: finishing rollback tid 0 to smithi06387500-10 2024-08-17T16:59:40.972 INFO:tasks.rados.rados.0.smithi063.stdout:1168: finishing rollback tid 1 to smithi06387500-10 2024-08-17T16:59:40.972 INFO:tasks.rados.rados.0.smithi063.stdout:1168: finishing rollback tid 2 to smithi06387500-10 2024-08-17T16:59:40.972 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 405 (ObjNum 436 snap 137 seq_num 436) dirty exists 2024-08-17T16:59:40.972 INFO:tasks.rados.rados.0.smithi063.stdout:1167: finishing write tid 3 to smithi06387500-1 2024-08-17T16:59:40.972 INFO:tasks.rados.rados.0.smithi063.stdout:1165: done (3 left) 2024-08-17T16:59:40.972 INFO:tasks.rados.rados.0.smithi063.stdout:1166: done (2 left) 2024-08-17T16:59:40.972 INFO:tasks.rados.rados.0.smithi063.stdout:1168: done (1 left) 2024-08-17T16:59:40.972 INFO:tasks.rados.rados.0.smithi063.stdout:1169: read oid 6 snap -1 2024-08-17T16:59:40.972 INFO:tasks.rados.rados.0.smithi063.stdout:1169: expect (ObjNum 425 snap 133 seq_num 425) 2024-08-17T16:59:40.972 INFO:tasks.rados.rados.0.smithi063.stdout:1170: write oid 14 current snap is 148 2024-08-17T16:59:40.972 INFO:tasks.rados.rados.0.smithi063.stdout:1167: finishing write tid 4 to smithi06387500-1 2024-08-17T16:59:40.973 INFO:tasks.rados.rados.0.smithi063.stdout:1170: seq_num 458 ranges {497606=645222,1934400=629494,3243966=453640} 2024-08-17T16:59:40.977 INFO:tasks.rados.rados.0.smithi063.stdout:1170: writing smithi06387500-14 from 497606 to 1142828 tid 1 2024-08-17T16:59:40.985 INFO:tasks.rados.rados.0.smithi063.stdout:1170: writing smithi06387500-14 from 1934400 to 2563894 tid 2 2024-08-17T16:59:40.992 INFO:tasks.rados.rados.0.smithi063.stdout:1170: writing smithi06387500-14 from 3243966 to 3697606 tid 3 2024-08-17T16:59:40.992 INFO:tasks.rados.rados.0.smithi063.stdout:1171: write oid 4 current snap is 148 2024-08-17T16:59:40.992 INFO:tasks.rados.rados.0.smithi063.stdout:1167: finishing write tid 5 to smithi06387500-1 2024-08-17T16:59:40.992 INFO:tasks.rados.rados.0.smithi063.stdout:1167: oid 1 updating version 0 to 563 2024-08-17T16:59:40.992 INFO:tasks.rados.rados.0.smithi063.stdout:1167: oid 1 version 563 is already newer than 562 2024-08-17T16:59:40.993 INFO:tasks.rados.rados.0.smithi063.stdout:1167: oid 1 version 563 is already newer than 560 2024-08-17T16:59:40.993 INFO:tasks.rados.rados.0.smithi063.stdout:1167: oid 1 version 563 is already newer than 561 2024-08-17T16:59:40.993 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 563 (ObjNum 457 snap 148 seq_num 457) dirty exists 2024-08-17T16:59:40.993 INFO:tasks.rados.rados.0.smithi063.stdout:1167: left oid 1 (ObjNum 457 snap 148 seq_num 457) 2024-08-17T16:59:40.993 INFO:tasks.rados.rados.0.smithi063.stdout:1169: expect (ObjNum 425 snap 133 seq_num 425) 2024-08-17T16:59:41.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ignoring --setuser ceph since I am not root 2024-08-17T16:59:41.030 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ignoring --setgroup ceph since I am not root 2024-08-17T16:59:41.030 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mgr[91679]: -- 172.21.15.63:0/3373023655 <== mon.0 v2:172.21.15.63:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c76f3a90e0 con 0x55c76f386800 2024-08-17T16:59:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:59:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:59:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:59:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:59:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:59:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: pgmap v224: 89 pgs: 89 active+clean; 173 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-08-17T16:59:41.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: pgmap v225: 89 pgs: 89 active+clean; 173 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T16:59:41.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:41.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: mon.a calling monitor election 2024-08-17T16:59:41.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T16:59:41.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: monmap epoch 3 2024-08-17T16:59:41.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:59:41.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: last_changed 2024-08-17T16:48:24.893441+0000 2024-08-17T16:59:41.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: created 2024-08-17T16:46:51.197040+0000 2024-08-17T16:59:41.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: min_mon_release 18 (reef) 2024-08-17T16:59:41.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: election_strategy: 1 2024-08-17T16:59:41.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: 0: [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon.a 2024-08-17T16:59:41.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: 1: [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] mon.b 2024-08-17T16:59:41.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: 2: [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] mon.c 2024-08-17T16:59:41.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T16:59:41.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: osdmap e360: 8 total, 8 up, 8 in 2024-08-17T16:59:41.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: mgrmap e34: y(active, since 2m), standbys: x 2024-08-17T16:59:41.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: overall HEALTH_OK 2024-08-17T16:59:41.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: from='mgr.24854 ' entity='' 2024-08-17T16:59:41.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[42680]: mgrmap e35: y(active, since 2m), standbys: x 2024-08-17T16:59:41.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:59:41.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:59:41.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:59:41.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:59:41.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:59:41.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: pgmap v224: 89 pgs: 89 active+clean; 173 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-08-17T16:59:41.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: pgmap v225: 89 pgs: 89 active+clean; 173 MiB data, 631 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T16:59:41.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: from='mgr.24854 172.21.15.63:0/782996418' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:41.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: mon.a calling monitor election 2024-08-17T16:59:41.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T16:59:41.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: monmap epoch 3 2024-08-17T16:59:41.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T16:59:41.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: last_changed 2024-08-17T16:48:24.893441+0000 2024-08-17T16:59:41.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: created 2024-08-17T16:46:51.197040+0000 2024-08-17T16:59:41.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: min_mon_release 18 (reef) 2024-08-17T16:59:41.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: election_strategy: 1 2024-08-17T16:59:41.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: 0: [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon.a 2024-08-17T16:59:41.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: 1: [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] mon.b 2024-08-17T16:59:41.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: 2: [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] mon.c 2024-08-17T16:59:41.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T16:59:41.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: osdmap e360: 8 total, 8 up, 8 in 2024-08-17T16:59:41.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: mgrmap e34: y(active, since 2m), standbys: x 2024-08-17T16:59:41.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: overall HEALTH_OK 2024-08-17T16:59:41.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: from='mgr.24854 ' entity='' 2024-08-17T16:59:41.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:40 smithi063 ceph-mon[109843]: mgrmap e35: y(active, since 2m), standbys: x 2024-08-17T16:59:41.098 INFO:tasks.rados.rados.0.smithi063.stdout:1170: finishing write tid 1 to smithi06387500-14 2024-08-17T16:59:41.098 INFO:tasks.rados.rados.0.smithi063.stdout:1170: finishing write tid 2 to smithi06387500-14 2024-08-17T16:59:41.099 INFO:tasks.rados.rados.0.smithi063.stdout:1171: seq_num 459 ranges {514413=720471,1876529=773152,3438683=275730} 2024-08-17T16:59:41.105 INFO:tasks.rados.rados.0.smithi063.stdout:1171: writing smithi06387500-4 from 514413 to 1234884 tid 1 2024-08-17T16:59:41.112 INFO:tasks.rados.rados.0.smithi063.stdout:1171: writing smithi06387500-4 from 1876529 to 2649681 tid 2 2024-08-17T16:59:41.119 INFO:tasks.rados.rados.0.smithi063.stdout:1171: writing smithi06387500-4 from 3438683 to 3714413 tid 3 2024-08-17T16:59:41.119 INFO:tasks.rados.rados.0.smithi063.stdout:1167: done (3 left) 2024-08-17T16:59:41.119 INFO:tasks.rados.rados.0.smithi063.stdout:1169: done (2 left) 2024-08-17T16:59:41.119 INFO:tasks.rados.rados.0.smithi063.stdout:1172: write oid 18 current snap is 148 2024-08-17T16:59:41.119 INFO:tasks.rados.rados.0.smithi063.stdout:1170: finishing write tid 3 to smithi06387500-14 2024-08-17T16:59:41.119 INFO:tasks.rados.rados.0.smithi063.stdout:1170: finishing write tid 4 to smithi06387500-14 2024-08-17T16:59:41.119 INFO:tasks.rados.rados.0.smithi063.stdout:1170: finishing write tid 5 to smithi06387500-14 2024-08-17T16:59:41.119 INFO:tasks.rados.rados.0.smithi063.stdout:1170: oid 14 updating version 0 to 1163 2024-08-17T16:59:41.119 INFO:tasks.rados.rados.0.smithi063.stdout:1170: oid 14 version 1163 is already newer than 1160 2024-08-17T16:59:41.119 INFO:tasks.rados.rados.0.smithi063.stdout:1170: oid 14 version 1163 is already newer than 1161 2024-08-17T16:59:41.119 INFO:tasks.rados.rados.0.smithi063.stdout:1170: oid 14 version 1163 is already newer than 1162 2024-08-17T16:59:41.119 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 1163 (ObjNum 458 snap 148 seq_num 458) dirty exists 2024-08-17T16:59:41.120 INFO:tasks.rados.rados.0.smithi063.stdout:1170: left oid 14 (ObjNum 458 snap 148 seq_num 458) 2024-08-17T16:59:41.120 INFO:tasks.rados.rados.0.smithi063.stdout:1172: seq_num 460 ranges {531220=795720,1935011=516810,3066106=420420} 2024-08-17T16:59:41.126 INFO:tasks.rados.rados.0.smithi063.stdout:1172: writing smithi06387500-18 from 531220 to 1326940 tid 1 2024-08-17T16:59:41.132 INFO:tasks.rados.rados.0.smithi063.stdout:1172: writing smithi06387500-18 from 1935011 to 2451821 tid 2 2024-08-17T16:59:41.138 INFO:tasks.rados.rados.0.smithi063.stdout:1172: writing smithi06387500-18 from 3066106 to 3486526 tid 3 2024-08-17T16:59:41.138 INFO:tasks.rados.rados.0.smithi063.stdout:1170: done (2 left) 2024-08-17T16:59:41.138 INFO:tasks.rados.rados.0.smithi063.stdout:1173: rollback oid 8 current snap is 148 2024-08-17T16:59:41.138 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 8 to 145 2024-08-17T16:59:41.139 INFO:tasks.rados.rados.0.smithi063.stdout:1171: finishing write tid 1 to smithi06387500-4 2024-08-17T16:59:41.139 INFO:tasks.rados.rados.0.smithi063.stdout:1171: finishing write tid 2 to smithi06387500-4 2024-08-17T16:59:41.139 INFO:tasks.rados.rados.0.smithi063.stdout:1171: finishing write tid 3 to smithi06387500-4 2024-08-17T16:59:41.139 INFO:tasks.rados.rados.0.smithi063.stdout:1171: finishing write tid 4 to smithi06387500-4 2024-08-17T16:59:41.139 INFO:tasks.rados.rados.0.smithi063.stdout:1171: finishing write tid 5 to smithi06387500-4 2024-08-17T16:59:41.139 INFO:tasks.rados.rados.0.smithi063.stdout:1171: oid 4 updating version 0 to 1165 2024-08-17T16:59:41.139 INFO:tasks.rados.rados.0.smithi063.stdout:1171: oid 4 version 1165 is already newer than 1164 2024-08-17T16:59:41.139 INFO:tasks.rados.rados.0.smithi063.stdout:1171: oid 4 updating version 1165 to 1167 2024-08-17T16:59:41.139 INFO:tasks.rados.rados.0.smithi063.stdout:1171: oid 4 version 1167 is already newer than 1166 2024-08-17T16:59:41.139 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 1167 (ObjNum 459 snap 148 seq_num 459) dirty exists 2024-08-17T16:59:41.139 INFO:tasks.rados.rados.0.smithi063.stdout:1171: left oid 4 (ObjNum 459 snap 148 seq_num 459) 2024-08-17T16:59:41.139 INFO:tasks.rados.rados.0.smithi063.stdout:1172: finishing write tid 1 to smithi06387500-18 2024-08-17T16:59:41.139 INFO:tasks.rados.rados.0.smithi063.stdout:1172: finishing write tid 2 to smithi06387500-18 2024-08-17T16:59:41.140 INFO:tasks.rados.rados.0.smithi063.stdout:1171: done (2 left) 2024-08-17T16:59:41.140 INFO:tasks.rados.rados.0.smithi063.stdout:1174: write oid 42 current snap is 148 2024-08-17T16:59:41.140 INFO:tasks.rados.rados.0.smithi063.stdout:1174: seq_num 461 ranges {548027=470969,1593493=776821,3093529=631692} 2024-08-17T16:59:41.143 INFO:tasks.rados.rados.0.smithi063.stdout:1174: writing smithi06387500-42 from 548027 to 1018996 tid 1 2024-08-17T16:59:41.151 INFO:tasks.rados.rados.0.smithi063.stdout:1174: writing smithi06387500-42 from 1593493 to 2370314 tid 2 2024-08-17T16:59:41.159 INFO:tasks.rados.rados.0.smithi063.stdout:1174: writing smithi06387500-42 from 3093529 to 3725221 tid 3 2024-08-17T16:59:41.159 INFO:tasks.rados.rados.0.smithi063.stdout:1175: snap_create 2024-08-17T16:59:41.159 INFO:tasks.rados.rados.0.smithi063.stdout:1172: finishing write tid 3 to smithi06387500-18 2024-08-17T16:59:41.159 INFO:tasks.rados.rados.0.smithi063.stdout:1172: finishing write tid 4 to smithi06387500-18 2024-08-17T16:59:41.159 INFO:tasks.rados.rados.0.smithi063.stdout:1173: finishing rollback tid 0 to smithi06387500-8 2024-08-17T16:59:41.159 INFO:tasks.rados.rados.0.smithi063.stdout:1173: finishing rollback tid 1 to smithi06387500-8 2024-08-17T16:59:41.159 INFO:tasks.rados.rados.0.smithi063.stdout:1172: finishing write tid 5 to smithi06387500-18 2024-08-17T16:59:41.159 INFO:tasks.rados.rados.0.smithi063.stdout:1172: oid 18 updating version 0 to 408 2024-08-17T16:59:41.159 INFO:tasks.rados.rados.0.smithi063.stdout:1172: oid 18 version 408 is already newer than 407 2024-08-17T16:59:41.159 INFO:tasks.rados.rados.0.smithi063.stdout:1172: oid 18 updating version 408 to 409 2024-08-17T16:59:41.160 INFO:tasks.rados.rados.0.smithi063.stdout:1172: oid 18 updating version 409 to 410 2024-08-17T16:59:41.160 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 410 (ObjNum 460 snap 148 seq_num 460) dirty exists 2024-08-17T16:59:41.160 INFO:tasks.rados.rados.0.smithi063.stdout:1172: left oid 18 (ObjNum 460 snap 148 seq_num 460) 2024-08-17T16:59:41.160 INFO:tasks.rados.rados.0.smithi063.stdout:1173: finishing rollback tid 2 to smithi06387500-8 2024-08-17T16:59:41.160 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 311 (ObjNum 333 snap 101 seq_num 333) dirty exists 2024-08-17T16:59:41.160 INFO:tasks.rados.rados.0.smithi063.stdout:1174: finishing write tid 1 to smithi06387500-42 2024-08-17T16:59:41.160 INFO:tasks.rados.rados.0.smithi063.stdout:1174: finishing write tid 2 to smithi06387500-42 2024-08-17T16:59:41.161 INFO:tasks.rados.rados.0.smithi063.stdout:1174: finishing write tid 3 to smithi06387500-42 2024-08-17T16:59:41.161 INFO:tasks.rados.rados.0.smithi063.stdout:1174: finishing write tid 4 to smithi06387500-42 2024-08-17T16:59:41.161 INFO:tasks.rados.rados.0.smithi063.stdout:1174: finishing write tid 5 to smithi06387500-42 2024-08-17T16:59:41.161 INFO:tasks.rados.rados.0.smithi063.stdout:1174: oid 42 updating version 0 to 787 2024-08-17T16:59:41.161 INFO:tasks.rados.rados.0.smithi063.stdout:1174: oid 42 updating version 787 to 788 2024-08-17T16:59:41.162 INFO:tasks.rados.rados.0.smithi063.stdout:1174: oid 42 version 788 is already newer than 786 2024-08-17T16:59:41.162 INFO:tasks.rados.rados.0.smithi063.stdout:1174: oid 42 updating version 788 to 789 2024-08-17T16:59:41.162 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 789 (ObjNum 461 snap 148 seq_num 461) dirty exists 2024-08-17T16:59:41.162 INFO:tasks.rados.rados.0.smithi063.stdout:1174: left oid 42 (ObjNum 461 snap 148 seq_num 461) 2024-08-17T16:59:41.590 INFO:tasks.rados.rados.0.smithi063.stdout:1172: done (3 left) 2024-08-17T16:59:41.590 INFO:tasks.rados.rados.0.smithi063.stdout:1173: done (2 left) 2024-08-17T16:59:41.590 INFO:tasks.rados.rados.0.smithi063.stdout:1174: done (1 left) 2024-08-17T16:59:41.590 INFO:tasks.rados.rados.0.smithi063.stdout:1175: done (0 left) 2024-08-17T16:59:41.590 INFO:tasks.rados.rados.0.smithi063.stdout:1176: rollback oid 13 current snap is 149 2024-08-17T16:59:41.590 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 13 to 125 2024-08-17T16:59:41.590 INFO:tasks.rados.rados.0.smithi063.stdout:1177: read oid 18 snap 136 2024-08-17T16:59:41.590 INFO:tasks.rados.rados.0.smithi063.stdout:1177: expect (ObjNum 368 snap 114 seq_num 368) 2024-08-17T16:59:41.590 INFO:tasks.rados.rados.0.smithi063.stdout:1178: delete oid 24 current snap is 149 2024-08-17T16:59:41.591 INFO:tasks.rados.rados.0.smithi063.stdout:1176: finishing rollback tid 1 to smithi06387500-13 2024-08-17T16:59:41.591 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 0 (ObjNum 0 snap 0 seq_num 304) dirty dne 2024-08-17T16:59:41.593 INFO:tasks.rados.rados.0.smithi063.stdout:1176: done (2 left) 2024-08-17T16:59:41.593 INFO:tasks.rados.rados.0.smithi063.stdout:1178: done (1 left) 2024-08-17T16:59:41.593 INFO:tasks.rados.rados.0.smithi063.stdout:1179: snap_create 2024-08-17T16:59:41.596 INFO:tasks.rados.rados.0.smithi063.stdout:1177: expect (ObjNum 368 snap 114 seq_num 368) 2024-08-17T16:59:41.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:41 smithi162 ceph-mon[34633]: osdmap e361: 8 total, 8 up, 8 in 2024-08-17T16:59:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:41 smithi063 ceph-mon[42680]: osdmap e361: 8 total, 8 up, 8 in 2024-08-17T16:59:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:41 smithi063 ceph-mon[109843]: osdmap e361: 8 total, 8 up, 8 in 2024-08-17T16:59:42.499 INFO:tasks.rados.rados.0.smithi063.stdout:1177: done (1 left) 2024-08-17T16:59:42.499 INFO:tasks.rados.rados.0.smithi063.stdout:1179: done (0 left) 2024-08-17T16:59:42.499 INFO:tasks.rados.rados.0.smithi063.stdout:1180: snap_remove snap 126 2024-08-17T16:59:42.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:42 smithi162 ceph-mon[34633]: osdmap e362: 8 total, 8 up, 8 in 2024-08-17T16:59:42.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:42 smithi162 ceph-mon[34633]: osdmap e363: 8 total, 8 up, 8 in 2024-08-17T16:59:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:42 smithi063 ceph-mon[42680]: osdmap e362: 8 total, 8 up, 8 in 2024-08-17T16:59:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:42 smithi063 ceph-mon[42680]: osdmap e363: 8 total, 8 up, 8 in 2024-08-17T16:59:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:42 smithi063 ceph-mon[109843]: osdmap e362: 8 total, 8 up, 8 in 2024-08-17T16:59:43.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:42 smithi063 ceph-mon[109843]: osdmap e363: 8 total, 8 up, 8 in 2024-08-17T16:59:43.501 INFO:tasks.rados.rados.0.smithi063.stdout:1180: done (0 left) 2024-08-17T16:59:43.501 INFO:tasks.rados.rados.0.smithi063.stdout:1181: snap_remove snap 132 2024-08-17T16:59:44.501 INFO:tasks.rados.rados.0.smithi063.stdout:1181: done (0 left) 2024-08-17T16:59:44.501 INFO:tasks.rados.rados.0.smithi063.stdout:1182: rollback oid 2 current snap is 150 2024-08-17T16:59:44.501 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 2 to 138 2024-08-17T16:59:44.501 INFO:tasks.rados.rados.0.smithi063.stdout:1183: write oid 26 current snap is 150 2024-08-17T16:59:44.501 INFO:tasks.rados.rados.0.smithi063.stdout:1183: seq_num 463 ranges {581641=621467,1994104=780490,3432022=349619} 2024-08-17T16:59:44.508 INFO:tasks.rados.rados.0.smithi063.stdout:1183: writing smithi06387500-26 from 581641 to 1203108 tid 1 2024-08-17T16:59:44.517 INFO:tasks.rados.rados.0.smithi063.stdout:1183: writing smithi06387500-26 from 1994104 to 2774594 tid 2 2024-08-17T16:59:44.523 INFO:tasks.rados.rados.0.smithi063.stdout:1183: writing smithi06387500-26 from 3432022 to 3781641 tid 3 2024-08-17T16:59:44.523 INFO:tasks.rados.rados.0.smithi063.stdout:1184: rollback oid 44 current snap is 150 2024-08-17T16:59:44.523 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 44 to 148 2024-08-17T16:59:44.523 INFO:tasks.rados.rados.0.smithi063.stdout:1182: finishing rollback tid 0 to smithi06387500-2 2024-08-17T16:59:44.523 INFO:tasks.rados.rados.0.smithi063.stdout:1182: finishing rollback tid 1 to smithi06387500-2 2024-08-17T16:59:44.523 INFO:tasks.rados.rados.0.smithi063.stdout:1182: finishing rollback tid 2 to smithi06387500-2 2024-08-17T16:59:44.523 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 573 (ObjNum 340 snap 102 seq_num 340) dirty exists 2024-08-17T16:59:44.524 INFO:tasks.rados.rados.0.smithi063.stdout:1183: finishing write tid 1 to smithi06387500-26 2024-08-17T16:59:44.524 INFO:tasks.rados.rados.0.smithi063.stdout:1182: done (2 left) 2024-08-17T16:59:44.524 INFO:tasks.rados.rados.0.smithi063.stdout:1185: rollback oid 14 current snap is 150 2024-08-17T16:59:44.524 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 14 to 148 2024-08-17T16:59:44.524 INFO:tasks.rados.rados.0.smithi063.stdout:1183: finishing write tid 2 to smithi06387500-26 2024-08-17T16:59:44.524 INFO:tasks.rados.rados.0.smithi063.stdout:1186: snap_remove snap 140 2024-08-17T16:59:44.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:44 smithi063 ceph-mon[42680]: osdmap e364: 8 total, 8 up, 8 in 2024-08-17T16:59:44.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:44 smithi063 ceph-mon[109843]: osdmap e364: 8 total, 8 up, 8 in 2024-08-17T16:59:44.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:44 smithi162 ceph-mon[34633]: osdmap e364: 8 total, 8 up, 8 in 2024-08-17T16:59:45.517 INFO:tasks.rados.rados.0.smithi063.stdout:1186: done (3 left) 2024-08-17T16:59:45.517 INFO:tasks.rados.rados.0.smithi063.stdout:1187: write oid 29 current snap is 150 2024-08-17T16:59:45.517 INFO:tasks.rados.rados.0.smithi063.stdout:1184: finishing rollback tid 0 to smithi06387500-44 2024-08-17T16:59:45.517 INFO:tasks.rados.rados.0.smithi063.stdout:1183: finishing write tid 3 to smithi06387500-26 2024-08-17T16:59:45.518 INFO:tasks.rados.rados.0.smithi063.stdout:1184: finishing rollback tid 1 to smithi06387500-44 2024-08-17T16:59:45.518 INFO:tasks.rados.rados.0.smithi063.stdout:1184: finishing rollback tid 2 to smithi06387500-44 2024-08-17T16:59:45.518 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 215 (ObjNum 409 snap 126 seq_num 409) dirty exists 2024-08-17T16:59:45.518 INFO:tasks.rados.rados.0.smithi063.stdout:1187: seq_num 464 ranges {598448=413069,1768939=524148,2775798=581861,3781488=16960} 2024-08-17T16:59:45.523 INFO:tasks.rados.rados.0.smithi063.stdout:1187: writing smithi06387500-29 from 598448 to 1011517 tid 1 2024-08-17T16:59:45.531 INFO:tasks.rados.rados.0.smithi063.stdout:1187: writing smithi06387500-29 from 1768939 to 2293087 tid 2 2024-08-17T16:59:45.537 INFO:tasks.rados.rados.0.smithi063.stdout:1187: writing smithi06387500-29 from 2775798 to 3357659 tid 3 2024-08-17T16:59:45.540 INFO:tasks.rados.rados.0.smithi063.stdout:1187: writing smithi06387500-29 from 3781488 to 3798448 tid 4 2024-08-17T16:59:45.540 INFO:tasks.rados.rados.0.smithi063.stdout:1184: done (3 left) 2024-08-17T16:59:45.540 INFO:tasks.rados.rados.0.smithi063.stdout:1188: snap_remove snap 127 2024-08-17T16:59:45.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:45 smithi063 ceph-mon[42680]: osdmap e365: 8 total, 8 up, 8 in 2024-08-17T16:59:45.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:45 smithi063 ceph-mon[109843]: osdmap e365: 8 total, 8 up, 8 in 2024-08-17T16:59:45.840 INFO:tasks.rados.rados.0.smithi063.stdout:1183: finishing write tid 4 to smithi06387500-26 2024-08-17T16:59:45.840 INFO:tasks.rados.rados.0.smithi063.stdout:1183: finishing write tid 5 to smithi06387500-26 2024-08-17T16:59:45.840 INFO:tasks.rados.rados.0.smithi063.stdout:1183: oid 26 updating version 0 to 367 2024-08-17T16:59:45.840 INFO:tasks.rados.rados.0.smithi063.stdout:1183: oid 26 updating version 367 to 368 2024-08-17T16:59:45.840 INFO:tasks.rados.rados.0.smithi063.stdout:1183: oid 26 updating version 368 to 369 2024-08-17T16:59:45.840 INFO:tasks.rados.rados.0.smithi063.stdout:1183: oid 26 version 369 is already newer than 366 2024-08-17T16:59:45.841 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 369 (ObjNum 463 snap 150 seq_num 463) dirty exists 2024-08-17T16:59:45.841 INFO:tasks.rados.rados.0.smithi063.stdout:1183: left oid 26 (ObjNum 463 snap 150 seq_num 463) 2024-08-17T16:59:45.841 INFO:tasks.rados.rados.0.smithi063.stdout:1185: finishing rollback tid 0 to smithi06387500-14 2024-08-17T16:59:45.841 INFO:tasks.rados.rados.0.smithi063.stdout:1185: finishing rollback tid 1 to smithi06387500-14 2024-08-17T16:59:45.841 INFO:tasks.rados.rados.0.smithi063.stdout:1185: finishing rollback tid 2 to smithi06387500-14 2024-08-17T16:59:45.841 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 1187 (ObjNum 458 snap 148 seq_num 458) dirty exists 2024-08-17T16:59:45.841 INFO:tasks.rados.rados.0.smithi063.stdout:1187: finishing write tid 1 to smithi06387500-29 2024-08-17T16:59:45.841 INFO:tasks.rados.rados.0.smithi063.stdout:1187: finishing write tid 2 to smithi06387500-29 2024-08-17T16:59:45.841 INFO:tasks.rados.rados.0.smithi063.stdout:1187: finishing write tid 3 to smithi06387500-29 2024-08-17T16:59:45.841 INFO:tasks.rados.rados.0.smithi063.stdout:1187: finishing write tid 4 to smithi06387500-29 2024-08-17T16:59:45.841 INFO:tasks.rados.rados.0.smithi063.stdout:1187: finishing write tid 5 to smithi06387500-29 2024-08-17T16:59:45.841 INFO:tasks.rados.rados.0.smithi063.stdout:1187: finishing write tid 6 to smithi06387500-29 2024-08-17T16:59:45.842 INFO:tasks.rados.rados.0.smithi063.stdout:1187: oid 29 updating version 0 to 238 2024-08-17T16:59:45.842 INFO:tasks.rados.rados.0.smithi063.stdout:1187: oid 29 version 238 is already newer than 235 2024-08-17T16:59:45.842 INFO:tasks.rados.rados.0.smithi063.stdout:1187: oid 29 version 238 is already newer than 237 2024-08-17T16:59:45.842 INFO:tasks.rados.rados.0.smithi063.stdout:1187: oid 29 updating version 238 to 239 2024-08-17T16:59:45.842 INFO:tasks.rados.rados.0.smithi063.stdout:1187: oid 29 version 239 is already newer than 236 2024-08-17T16:59:45.842 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 239 (ObjNum 464 snap 150 seq_num 464) dirty exists 2024-08-17T16:59:45.842 INFO:tasks.rados.rados.0.smithi063.stdout:1187: left oid 29 (ObjNum 464 snap 150 seq_num 464) 2024-08-17T16:59:45.842 INFO:tasks.rados.rados.0.smithi063.stdout:1183: done (3 left) 2024-08-17T16:59:45.842 INFO:tasks.rados.rados.0.smithi063.stdout:1185: done (2 left) 2024-08-17T16:59:45.842 INFO:tasks.rados.rados.0.smithi063.stdout:1187: done (1 left) 2024-08-17T16:59:45.842 INFO:tasks.rados.rados.0.smithi063.stdout:1188: done (0 left) 2024-08-17T16:59:45.842 INFO:tasks.rados.rados.0.smithi063.stdout:1189: rollback oid 37 current snap is 150 2024-08-17T16:59:45.842 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 148 2024-08-17T16:59:45.843 INFO:tasks.rados.rados.0.smithi063.stdout:1190: delete oid 2 current snap is 150 2024-08-17T16:59:45.843 INFO:tasks.rados.rados.0.smithi063.stdout:1189: finishing rollback tid 1 to smithi06387500-37 2024-08-17T16:59:45.843 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 0 (ObjNum 65 snap 0 seq_num 1953066355) dirty dne 2024-08-17T16:59:45.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:45 smithi162 ceph-mon[34633]: osdmap e365: 8 total, 8 up, 8 in 2024-08-17T16:59:45.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:59:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[81670]: [17/Aug/2024:16:59:45] ENGINE Bus STARTING 2024-08-17T16:59:45.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:59:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[81670]: CherryPy Checker: 2024-08-17T16:59:45.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:59:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[81670]: The Application mounted at '' has an empty config. 2024-08-17T16:59:45.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:59:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[81670]: 2024-08-17T16:59:45.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:59:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[81670]: [17/Aug/2024:16:59:45] ENGINE Serving on http://:::9283 2024-08-17T16:59:45.846 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 16:59:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x[81670]: [17/Aug/2024:16:59:45] ENGINE Bus STARTED 2024-08-17T16:59:45.847 INFO:tasks.rados.rados.0.smithi063.stdout:1189: done (1 left) 2024-08-17T16:59:45.848 INFO:tasks.rados.rados.0.smithi063.stdout:1190: done (0 left) 2024-08-17T16:59:45.848 INFO:tasks.rados.rados.0.smithi063.stdout:1191: rollback oid 48 current snap is 150 2024-08-17T16:59:45.848 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 48 to 138 2024-08-17T16:59:45.848 INFO:tasks.rados.rados.0.smithi063.stdout:1192: snap_create 2024-08-17T16:59:45.851 INFO:tasks.rados.rados.0.smithi063.stdout:1191: finishing rollback tid 0 to smithi06387500-48 2024-08-17T16:59:45.851 INFO:tasks.rados.rados.0.smithi063.stdout:1191: finishing rollback tid 1 to smithi06387500-48 2024-08-17T16:59:45.852 INFO:tasks.rados.rados.0.smithi063.stdout:1191: finishing rollback tid 2 to smithi06387500-48 2024-08-17T16:59:45.852 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 1203 (ObjNum 433 snap 136 seq_num 433) dirty exists 2024-08-17T16:59:46.703 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:59:46.344+0000 7f462087b640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:59:46.704 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:59:46.344+0000 7f462087b640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:59:46.704 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:59:46.344+0000 7f462087b640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:59:46.704 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:59:46.344+0000 7f462087b640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:59:46.704 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:59:46.344+0000 7f462087b640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:59:46.704 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:59:46.347+0000 7f462287f640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:59:46.704 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:59:46.347+0000 7f462287f640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:59:46.704 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:59:46.347+0000 7f462287f640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:59:46.704 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:59:46.347+0000 7f462287f640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:59:46.704 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:59:46.347+0000 7f462287f640 -1 client.0 error registering admin socket command: (17) File exists 2024-08-17T16:59:46.705 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: osdmap e366: 8 total, 8 up, 8 in 2024-08-17T16:59:46.705 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: Standby manager daemon x restarted 2024-08-17T16:59:46.705 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: Standby manager daemon x started 2024-08-17T16:59:46.705 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/997057358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:59:46.705 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/997057358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:59:46.705 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/997057358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:59:46.705 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.? 172.21.15.162:0/997057358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:59:46.705 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: Active manager daemon y restarted 2024-08-17T16:59:46.705 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: Activating manager daemon y 2024-08-17T16:59:46.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: osdmap e367: 8 total, 8 up, 8 in 2024-08-17T16:59:46.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: mgrmap e36: y(active, starting, since 0.00704443s), standbys: x 2024-08-17T16:59:46.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:59:46.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:59:46.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:59:46.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi063.ultpqj"}]: dispatch 2024-08-17T16:59:46.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi162.bjkxnv"}]: dispatch 2024-08-17T16:59:46.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:59:46.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:59:46.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:59:46.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:59:46.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:59:46.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:59:46.707 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:59:46.707 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:59:46.707 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:59:46.707 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:59:46.707 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:59:46.707 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:59:46.707 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:59:46.707 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: osdmap e366: 8 total, 8 up, 8 in 2024-08-17T16:59:46.707 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: Standby manager daemon x restarted 2024-08-17T16:59:46.708 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: Standby manager daemon x started 2024-08-17T16:59:46.708 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.? 172.21.15.162:0/997057358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:59:46.708 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.? 172.21.15.162:0/997057358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:59:46.708 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.? 172.21.15.162:0/997057358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:59:46.708 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.? 172.21.15.162:0/997057358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:59:46.708 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: Active manager daemon y restarted 2024-08-17T16:59:46.708 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: Activating manager daemon y 2024-08-17T16:59:46.708 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: osdmap e367: 8 total, 8 up, 8 in 2024-08-17T16:59:46.708 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: mgrmap e36: y(active, starting, since 0.00704443s), standbys: x 2024-08-17T16:59:46.708 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:59:46.708 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:59:46.708 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:59:46.708 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi063.ultpqj"}]: dispatch 2024-08-17T16:59:46.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi162.bjkxnv"}]: dispatch 2024-08-17T16:59:46.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:59:46.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:59:46.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:59:46.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:59:46.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:59:46.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:59:46.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:59:46.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:59:46.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:59:46.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:59:46.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:59:46.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:59:46.710 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:59:46.710 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: Manager daemon y is now available 2024-08-17T16:59:46.710 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:46.710 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:46.710 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:59:46.710 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:59:46.710 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-08-17T16:59:46.710 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:59:46.710 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-08-17T16:59:46.711 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: Manager daemon y is now available 2024-08-17T16:59:46.711 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:46.711 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:46.711 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:59:46.711 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:59:46.711 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-08-17T16:59:46.711 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:59:46.711 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-08-17T16:59:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: osdmap e366: 8 total, 8 up, 8 in 2024-08-17T16:59:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: Standby manager daemon x restarted 2024-08-17T16:59:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: Standby manager daemon x started 2024-08-17T16:59:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/997057358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-08-17T16:59:46.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/997057358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-08-17T16:59:46.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/997057358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-08-17T16:59:46.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.? 172.21.15.162:0/997057358' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-08-17T16:59:46.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: Active manager daemon y restarted 2024-08-17T16:59:46.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: Activating manager daemon y 2024-08-17T16:59:46.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: osdmap e367: 8 total, 8 up, 8 in 2024-08-17T16:59:46.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: mgrmap e36: y(active, starting, since 0.00704443s), standbys: x 2024-08-17T16:59:46.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-08-17T16:59:46.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-08-17T16:59:46.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-08-17T16:59:46.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi063.ultpqj"}]: dispatch 2024-08-17T16:59:46.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi162.bjkxnv"}]: dispatch 2024-08-17T16:59:46.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-08-17T16:59:46.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-08-17T16:59:46.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-08-17T16:59:46.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-08-17T16:59:46.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-08-17T16:59:46.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-08-17T16:59:46.850 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-08-17T16:59:46.850 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-08-17T16:59:46.850 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-08-17T16:59:46.850 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-08-17T16:59:46.850 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-08-17T16:59:46.850 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-08-17T16:59:46.850 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-08-17T16:59:46.850 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: Manager daemon y is now available 2024-08-17T16:59:46.850 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:46.850 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T16:59:46.850 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:59:46.850 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-08-17T16:59:46.851 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-08-17T16:59:46.851 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-08-17T16:59:46.851 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:46 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-08-17T16:59:46.856 INFO:tasks.rados.rados.0.smithi063.stdout:1191: done (1 left) 2024-08-17T16:59:46.856 INFO:tasks.rados.rados.0.smithi063.stdout:1192: done (0 left) 2024-08-17T16:59:46.856 INFO:tasks.rados.rados.0.smithi063.stdout:1193: delete oid 11 current snap is 151 2024-08-17T16:59:46.858 INFO:tasks.rados.rados.0.smithi063.stdout:1193: done (0 left) 2024-08-17T16:59:46.858 INFO:tasks.rados.rados.0.smithi063.stdout:1194: delete oid 37 current snap is 151 2024-08-17T16:59:46.860 INFO:tasks.rados.rados.0.smithi063.stdout:1194: done (0 left) 2024-08-17T16:59:46.860 INFO:tasks.rados.rados.0.smithi063.stdout:1195: read oid 46 snap -1 2024-08-17T16:59:46.860 INFO:tasks.rados.rados.0.smithi063.stdout:1195: expect (ObjNum 293 snap 91 seq_num 293) 2024-08-17T16:59:46.860 INFO:tasks.rados.rados.0.smithi063.stdout:1196: rollback oid 6 current snap is 151 2024-08-17T16:59:46.860 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 6 to 145 2024-08-17T16:59:46.860 INFO:tasks.rados.rados.0.smithi063.stdout:1197: snap_create 2024-08-17T16:59:46.865 INFO:tasks.rados.rados.0.smithi063.stdout:1196: finishing rollback tid 0 to smithi06387500-6 2024-08-17T16:59:46.866 INFO:tasks.rados.rados.0.smithi063.stdout:1195: expect (ObjNum 293 snap 91 seq_num 293) 2024-08-17T16:59:46.957 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:59:46.840+0000 7f4652b73640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-08-17T16:59:46.957 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:59:46] ENGINE Bus STARTING 2024-08-17T16:59:46.958 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: CherryPy Checker: 2024-08-17T16:59:46.958 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: The Application mounted at '' has an empty config. 2024-08-17T16:59:46.958 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:59:46.958 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:59:46] ENGINE Serving on http://:::9283 2024-08-17T16:59:47.030 INFO:tasks.rados.rados.0.smithi063.stdout:1196: finishing rollback tid 1 to smithi06387500-6 2024-08-17T16:59:47.031 INFO:tasks.rados.rados.0.smithi063.stdout:1196: finishing rollback tid 2 to smithi06387500-6 2024-08-17T16:59:47.031 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 323 (ObjNum 425 snap 133 seq_num 425) dirty exists 2024-08-17T16:59:47.279 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:46 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:16:59:46] ENGINE Bus STARTED 2024-08-17T16:59:47.864 INFO:tasks.rados.rados.0.smithi063.stdout:1195: done (2 left) 2024-08-17T16:59:47.864 INFO:tasks.rados.rados.0.smithi063.stdout:1196: done (1 left) 2024-08-17T16:59:47.864 INFO:tasks.rados.rados.0.smithi063.stdout:1197: done (0 left) 2024-08-17T16:59:47.864 INFO:tasks.rados.rados.0.smithi063.stdout:1198: write oid 29 current snap is 152 2024-08-17T16:59:47.864 INFO:tasks.rados.rados.0.smithi063.stdout:1198: seq_num 468 ranges {665676=714065,1886514=531486,3169137=696539} 2024-08-17T16:59:47.871 INFO:tasks.rados.rados.0.smithi063.stdout:1198: writing smithi06387500-29 from 665676 to 1379741 tid 1 2024-08-17T16:59:47.877 INFO:tasks.rados.rados.0.smithi063.stdout:1198: writing smithi06387500-29 from 1886514 to 2418000 tid 2 2024-08-17T16:59:47.885 INFO:tasks.rados.rados.0.smithi063.stdout:1198: writing smithi06387500-29 from 3169137 to 3865676 tid 3 2024-08-17T16:59:47.885 INFO:tasks.rados.rados.0.smithi063.stdout:1199: write oid 41 current snap is 152 2024-08-17T16:59:47.885 INFO:tasks.rados.rados.0.smithi063.stdout:1198: finishing write tid 1 to smithi06387500-29 2024-08-17T16:59:47.886 INFO:tasks.rados.rados.0.smithi063.stdout:1199: seq_num 469 ranges {682483=789314,1944996=791497,3196560=554574} 2024-08-17T16:59:47.893 INFO:tasks.rados.rados.0.smithi063.stdout:1199: writing smithi06387500-41 from 682483 to 1471797 tid 1 2024-08-17T16:59:47.901 INFO:tasks.rados.rados.0.smithi063.stdout:1199: writing smithi06387500-41 from 1944996 to 2736493 tid 2 2024-08-17T16:59:47.906 INFO:tasks.rados.rados.0.smithi063.stdout:1199: writing smithi06387500-41 from 3196560 to 3751134 tid 3 2024-08-17T16:59:47.906 INFO:tasks.rados.rados.0.smithi063.stdout:1200: read oid 24 snap -1 2024-08-17T16:59:47.907 INFO:tasks.rados.rados.0.smithi063.stdout:1200: expect deleted 2024-08-17T16:59:47.907 INFO:tasks.rados.rados.0.smithi063.stdout:1201: snap_remove snap 137 2024-08-17T16:59:48.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:47 smithi162 ceph-mon[34633]: mgrmap e37: y(active, since 1.01066s), standbys: x 2024-08-17T16:59:48.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:47 smithi162 ceph-mon[34633]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:59:48.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:47 smithi162 ceph-mon[34633]: pgmap v4: 89 pgs: 89 active+clean; 189 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 396 GiB/s rd, 67 GiB/s wr, 977.75k op/s; 1024 MiB/s, 3.64k objects/s recovering 2024-08-17T16:59:48.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:47 smithi162 ceph-mon[34633]: osdmap e368: 8 total, 8 up, 8 in 2024-08-17T16:59:48.268 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-mon[42680]: mgrmap e37: y(active, since 1.01066s), standbys: x 2024-08-17T16:59:48.269 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-mon[42680]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:59:48.269 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-mon[42680]: pgmap v4: 89 pgs: 89 active+clean; 189 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 396 GiB/s rd, 67 GiB/s wr, 977.75k op/s; 1024 MiB/s, 3.64k objects/s recovering 2024-08-17T16:59:48.269 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-mon[42680]: osdmap e368: 8 total, 8 up, 8 in 2024-08-17T16:59:48.269 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-mon[109843]: mgrmap e37: y(active, since 1.01066s), standbys: x 2024-08-17T16:59:48.269 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-mon[109843]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T16:59:48.269 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-mon[109843]: pgmap v4: 89 pgs: 89 active+clean; 189 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 396 GiB/s rd, 67 GiB/s wr, 977.75k op/s; 1024 MiB/s, 3.64k objects/s recovering 2024-08-17T16:59:48.269 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-mon[109843]: osdmap e368: 8 total, 8 up, 8 in 2024-08-17T16:59:48.269 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:59:47.897+0000 7f460668d640 -1 log_channel(cephadm) log [ERR] : [17/Aug/2024:16:59:47] ENGINE Error in HTTPServer.serve 2024-08-17T16:59:48.270 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: Traceback (most recent call last): 2024-08-17T16:59:48.270 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:59:48.270 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: self._connections.run(self.expiration_interval) 2024-08-17T16:59:48.270 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:59:48.270 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: self._run(expiration_interval) 2024-08-17T16:59:48.270 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:59:48.270 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:59:48.270 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:59:48.270 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:59:48.270 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:59:48.270 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: s = self.context.wrap_socket( 2024-08-17T16:59:48.270 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:59:48.271 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: return self.sslsocket_class._create( 2024-08-17T16:59:48.271 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:59:48.271 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: self.do_handshake() 2024-08-17T16:59:48.271 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:59:48.271 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: self._sslobj.do_handshake() 2024-08-17T16:59:48.271 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:59:48.271 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:47 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T16:59:48.867 INFO:tasks.rados.rados.0.smithi063.stdout:1201: done (3 left) 2024-08-17T16:59:48.871 INFO:tasks.rados.rados.0.smithi063.stdout:1202: snap_create 2024-08-17T16:59:48.872 INFO:tasks.rados.rados.0.smithi063.stdout:1198: finishing write tid 2 to smithi06387500-29 2024-08-17T16:59:48.872 INFO:tasks.rados.rados.0.smithi063.stdout:1198: finishing write tid 3 to smithi06387500-29 2024-08-17T16:59:48.872 INFO:tasks.rados.rados.0.smithi063.stdout:1198: finishing write tid 4 to smithi06387500-29 2024-08-17T16:59:48.872 INFO:tasks.rados.rados.0.smithi063.stdout:1198: finishing write tid 5 to smithi06387500-29 2024-08-17T16:59:48.872 INFO:tasks.rados.rados.0.smithi063.stdout:1198: oid 29 updating version 0 to 246 2024-08-17T16:59:48.872 INFO:tasks.rados.rados.0.smithi063.stdout:1198: oid 29 version 246 is already newer than 244 2024-08-17T16:59:48.872 INFO:tasks.rados.rados.0.smithi063.stdout:1198: oid 29 version 246 is already newer than 243 2024-08-17T16:59:48.873 INFO:tasks.rados.rados.0.smithi063.stdout:1198: oid 29 version 246 is already newer than 245 2024-08-17T16:59:48.873 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 246 (ObjNum 468 snap 152 seq_num 468) dirty exists 2024-08-17T16:59:48.873 INFO:tasks.rados.rados.0.smithi063.stdout:1198: left oid 29 (ObjNum 468 snap 152 seq_num 468) 2024-08-17T16:59:48.873 INFO:tasks.rados.rados.0.smithi063.stdout:1199: finishing write tid 1 to smithi06387500-41 2024-08-17T16:59:48.873 INFO:tasks.rados.rados.0.smithi063.stdout:1199: finishing write tid 2 to smithi06387500-41 2024-08-17T16:59:48.873 INFO:tasks.rados.rados.0.smithi063.stdout:1199: finishing write tid 3 to smithi06387500-41 2024-08-17T16:59:48.873 INFO:tasks.rados.rados.0.smithi063.stdout:1199: finishing write tid 4 to smithi06387500-41 2024-08-17T16:59:48.874 INFO:tasks.rados.rados.0.smithi063.stdout:1199: finishing write tid 5 to smithi06387500-41 2024-08-17T16:59:48.874 INFO:tasks.rados.rados.0.smithi063.stdout:1199: oid 41 updating version 0 to 71 2024-08-17T16:59:48.874 INFO:tasks.rados.rados.0.smithi063.stdout:1199: oid 41 updating version 71 to 73 2024-08-17T16:59:48.874 INFO:tasks.rados.rados.0.smithi063.stdout:1199: oid 41 version 73 is already newer than 72 2024-08-17T16:59:48.874 INFO:tasks.rados.rados.0.smithi063.stdout:1199: oid 41 updating version 73 to 74 2024-08-17T16:59:48.874 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 74 (ObjNum 469 snap 152 seq_num 469) dirty exists 2024-08-17T16:59:48.875 INFO:tasks.rados.rados.0.smithi063.stdout:1199: left oid 41 (ObjNum 469 snap 152 seq_num 469) 2024-08-17T16:59:49.136 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:48 smithi162 ceph-mon[34633]: [17/Aug/2024:16:59:47] ENGINE Bus STARTING 2024-08-17T16:59:49.136 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:48 smithi162 ceph-mon[34633]: pgmap v5: 89 pgs: 89 active+clean; 189 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 668 MiB/s rd, 114 MiB/s wr, 1.61k op/s; 1.7 MiB/s, 5 objects/s recovering 2024-08-17T16:59:49.136 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:48 smithi162 ceph-mon[34633]: osdmap e369: 8 total, 8 up, 8 in 2024-08-17T16:59:49.137 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:48 smithi162 ceph-mon[34633]: mgrmap e38: y(active, since 2s), standbys: x 2024-08-17T16:59:49.137 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:48 smithi162 ceph-mon[34633]: [17/Aug/2024:16:59:47] ENGINE Serving on https://172.21.15.63:7150 2024-08-17T16:59:49.137 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:48 smithi162 ceph-mon[34633]: [17/Aug/2024:16:59:47] ENGINE Error in HTTPServer.serve 2024-08-17T16:59:49.137 INFO:journalctl@ceph.mon.b.smithi162.stdout: Traceback (most recent call last): 2024-08-17T16:59:49.137 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:59:49.138 INFO:journalctl@ceph.mon.b.smithi162.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:59:49.138 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:59:49.138 INFO:journalctl@ceph.mon.b.smithi162.stdout: self._run(expiration_interval) 2024-08-17T16:59:49.138 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:59:49.138 INFO:journalctl@ceph.mon.b.smithi162.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:59:49.138 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:59:49.138 INFO:journalctl@ceph.mon.b.smithi162.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:59:49.138 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:59:49.138 INFO:journalctl@ceph.mon.b.smithi162.stdout: s = self.context.wrap_socket( 2024-08-17T16:59:49.138 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:59:49.139 INFO:journalctl@ceph.mon.b.smithi162.stdout: return self.sslsocket_class._create( 2024-08-17T16:59:49.139 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:59:49.139 INFO:journalctl@ceph.mon.b.smithi162.stdout: self.do_handshake() 2024-08-17T16:59:49.139 INFO:journalctl@ceph.mon.b.smithi162.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:59:49.139 INFO:journalctl@ceph.mon.b.smithi162.stdout: self._sslobj.do_handshake() 2024-08-17T16:59:49.139 INFO:journalctl@ceph.mon.b.smithi162.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:59:49.139 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:48 smithi162 ceph-mon[34633]: [17/Aug/2024:16:59:47] ENGINE Serving on http://172.21.15.63:8765 2024-08-17T16:59:49.139 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:48 smithi162 ceph-mon[34633]: [17/Aug/2024:16:59:47] ENGINE Bus STARTED 2024-08-17T16:59:49.206 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[42680]: [17/Aug/2024:16:59:47] ENGINE Bus STARTING 2024-08-17T16:59:49.206 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[42680]: pgmap v5: 89 pgs: 89 active+clean; 189 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 668 MiB/s rd, 114 MiB/s wr, 1.61k op/s; 1.7 MiB/s, 5 objects/s recovering 2024-08-17T16:59:49.206 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[42680]: osdmap e369: 8 total, 8 up, 8 in 2024-08-17T16:59:49.206 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[42680]: mgrmap e38: y(active, since 2s), standbys: x 2024-08-17T16:59:49.206 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[42680]: [17/Aug/2024:16:59:47] ENGINE Serving on https://172.21.15.63:7150 2024-08-17T16:59:49.207 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[42680]: [17/Aug/2024:16:59:47] ENGINE Error in HTTPServer.serve 2024-08-17T16:59:49.207 INFO:journalctl@ceph.mon.c.smithi063.stdout: Traceback (most recent call last): 2024-08-17T16:59:49.207 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:59:49.207 INFO:journalctl@ceph.mon.c.smithi063.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:59:49.207 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:59:49.207 INFO:journalctl@ceph.mon.c.smithi063.stdout: self._run(expiration_interval) 2024-08-17T16:59:49.207 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:59:49.207 INFO:journalctl@ceph.mon.c.smithi063.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:59:49.207 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:59:49.207 INFO:journalctl@ceph.mon.c.smithi063.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:59:49.207 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:59:49.207 INFO:journalctl@ceph.mon.c.smithi063.stdout: s = self.context.wrap_socket( 2024-08-17T16:59:49.207 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:59:49.208 INFO:journalctl@ceph.mon.c.smithi063.stdout: return self.sslsocket_class._create( 2024-08-17T16:59:49.208 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:59:49.208 INFO:journalctl@ceph.mon.c.smithi063.stdout: self.do_handshake() 2024-08-17T16:59:49.208 INFO:journalctl@ceph.mon.c.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:59:49.208 INFO:journalctl@ceph.mon.c.smithi063.stdout: self._sslobj.do_handshake() 2024-08-17T16:59:49.208 INFO:journalctl@ceph.mon.c.smithi063.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:59:49.208 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[42680]: [17/Aug/2024:16:59:47] ENGINE Serving on http://172.21.15.63:8765 2024-08-17T16:59:49.208 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[42680]: [17/Aug/2024:16:59:47] ENGINE Bus STARTED 2024-08-17T16:59:49.208 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[109843]: [17/Aug/2024:16:59:47] ENGINE Bus STARTING 2024-08-17T16:59:49.208 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[109843]: pgmap v5: 89 pgs: 89 active+clean; 189 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 668 MiB/s rd, 114 MiB/s wr, 1.61k op/s; 1.7 MiB/s, 5 objects/s recovering 2024-08-17T16:59:49.208 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[109843]: osdmap e369: 8 total, 8 up, 8 in 2024-08-17T16:59:49.209 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[109843]: mgrmap e38: y(active, since 2s), standbys: x 2024-08-17T16:59:49.209 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[109843]: [17/Aug/2024:16:59:47] ENGINE Serving on https://172.21.15.63:7150 2024-08-17T16:59:49.209 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[109843]: [17/Aug/2024:16:59:47] ENGINE Error in HTTPServer.serve 2024-08-17T16:59:49.209 INFO:journalctl@ceph.mon.a.smithi063.stdout: Traceback (most recent call last): 2024-08-17T16:59:49.209 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-08-17T16:59:49.209 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._connections.run(self.expiration_interval) 2024-08-17T16:59:49.209 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-08-17T16:59:49.209 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._run(expiration_interval) 2024-08-17T16:59:49.209 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-08-17T16:59:49.209 INFO:journalctl@ceph.mon.a.smithi063.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-08-17T16:59:49.209 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-08-17T16:59:49.209 INFO:journalctl@ceph.mon.a.smithi063.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-08-17T16:59:49.209 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-08-17T16:59:49.209 INFO:journalctl@ceph.mon.a.smithi063.stdout: s = self.context.wrap_socket( 2024-08-17T16:59:49.210 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-08-17T16:59:49.210 INFO:journalctl@ceph.mon.a.smithi063.stdout: return self.sslsocket_class._create( 2024-08-17T16:59:49.210 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-08-17T16:59:49.210 INFO:journalctl@ceph.mon.a.smithi063.stdout: self.do_handshake() 2024-08-17T16:59:49.210 INFO:journalctl@ceph.mon.a.smithi063.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-08-17T16:59:49.210 INFO:journalctl@ceph.mon.a.smithi063.stdout: self._sslobj.do_handshake() 2024-08-17T16:59:49.210 INFO:journalctl@ceph.mon.a.smithi063.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-08-17T16:59:49.210 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[109843]: [17/Aug/2024:16:59:47] ENGINE Serving on http://172.21.15.63:8765 2024-08-17T16:59:49.210 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:48 smithi063 ceph-mon[109843]: [17/Aug/2024:16:59:47] ENGINE Bus STARTED 2024-08-17T16:59:49.880 INFO:tasks.rados.rados.0.smithi063.stdout:1198: done (3 left) 2024-08-17T16:59:49.881 INFO:tasks.rados.rados.0.smithi063.stdout:1199: done (2 left) 2024-08-17T16:59:49.881 INFO:tasks.rados.rados.0.smithi063.stdout:1200: done (1 left) 2024-08-17T16:59:49.881 INFO:tasks.rados.rados.0.smithi063.stdout:1202: done (0 left) 2024-08-17T16:59:49.881 INFO:tasks.rados.rados.0.smithi063.stdout:1203: read oid 37 snap -1 2024-08-17T16:59:49.881 INFO:tasks.rados.rados.0.smithi063.stdout:1203: expect deleted 2024-08-17T16:59:49.881 INFO:tasks.rados.rados.0.smithi063.stdout:1204: snap_remove snap 136 2024-08-17T16:59:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:49 smithi162 ceph-mon[34633]: osdmap e370: 8 total, 8 up, 8 in 2024-08-17T16:59:50.178 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:49 smithi063 ceph-mon[42680]: osdmap e370: 8 total, 8 up, 8 in 2024-08-17T16:59:50.178 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:49 smithi063 ceph-mon[109843]: osdmap e370: 8 total, 8 up, 8 in 2024-08-17T16:59:50.877 INFO:tasks.rados.rados.0.smithi063.stdout:1204: done (1 left) 2024-08-17T16:59:50.877 INFO:tasks.rados.rados.0.smithi063.stdout:1205: rollback oid 41 current snap is 153 2024-08-17T16:59:50.877 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 41 to 145 2024-08-17T16:59:50.877 INFO:tasks.rados.rados.0.smithi063.stdout:1206: rollback oid 3 current snap is 153 2024-08-17T16:59:50.877 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 3 to 146 2024-08-17T16:59:50.877 INFO:tasks.rados.rados.0.smithi063.stdout:1207: snap_remove snap 138 2024-08-17T16:59:51.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:50 smithi063 ceph-mon[109843]: pgmap v8: 89 pgs: 89 active+clean; 189 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 223 MiB/s rd, 38 MiB/s wr, 537 op/s; 576 KiB/s, 1 objects/s recovering 2024-08-17T16:59:51.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:50 smithi063 ceph-mon[109843]: mgrmap e39: y(active, since 4s), standbys: x 2024-08-17T16:59:51.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:50 smithi063 ceph-mon[109843]: osdmap e371: 8 total, 8 up, 8 in 2024-08-17T16:59:51.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:50 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:51.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:50 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:51.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:50 smithi063 ceph-mon[42680]: pgmap v8: 89 pgs: 89 active+clean; 189 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 223 MiB/s rd, 38 MiB/s wr, 537 op/s; 576 KiB/s, 1 objects/s recovering 2024-08-17T16:59:51.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:50 smithi063 ceph-mon[42680]: mgrmap e39: y(active, since 4s), standbys: x 2024-08-17T16:59:51.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:50 smithi063 ceph-mon[42680]: osdmap e371: 8 total, 8 up, 8 in 2024-08-17T16:59:51.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:50 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:51.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:50 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:51.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:50 smithi162 ceph-mon[34633]: pgmap v8: 89 pgs: 89 active+clean; 189 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 223 MiB/s rd, 38 MiB/s wr, 537 op/s; 576 KiB/s, 1 objects/s recovering 2024-08-17T16:59:51.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:50 smithi162 ceph-mon[34633]: mgrmap e39: y(active, since 4s), standbys: x 2024-08-17T16:59:51.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:50 smithi162 ceph-mon[34633]: osdmap e371: 8 total, 8 up, 8 in 2024-08-17T16:59:51.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:50 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:51.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:50 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:51.884 INFO:tasks.rados.rados.0.smithi063.stdout:1207: done (3 left) 2024-08-17T16:59:51.884 INFO:tasks.rados.rados.0.smithi063.stdout:1208: write oid 32 current snap is 153 2024-08-17T16:59:51.884 INFO:tasks.rados.rados.0.smithi063.stdout:1208: seq_num 470 ranges {699290=464563,1603478=535155,2823983=765846} 2024-08-17T16:59:51.891 INFO:tasks.rados.rados.0.smithi063.stdout:1208: writing smithi06387500-32 from 699290 to 1163853 tid 1 2024-08-17T16:59:51.897 INFO:tasks.rados.rados.0.smithi063.stdout:1208: writing smithi06387500-32 from 1603478 to 2138633 tid 2 2024-08-17T16:59:51.906 INFO:tasks.rados.rados.0.smithi063.stdout:1208: writing smithi06387500-32 from 2823983 to 3589829 tid 3 2024-08-17T16:59:51.906 INFO:tasks.rados.rados.0.smithi063.stdout:1209: rollback oid 16 current snap is 153 2024-08-17T16:59:51.907 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 16 to 152 2024-08-17T16:59:51.907 INFO:tasks.rados.rados.0.smithi063.stdout:1206: finishing rollback tid 1 to smithi06387500-3 2024-08-17T16:59:51.907 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 0 (ObjNum 97 snap 0 seq_num 600181488) dirty dne 2024-08-17T16:59:51.907 INFO:tasks.rados.rados.0.smithi063.stdout:1205: finishing rollback tid 0 to smithi06387500-41 2024-08-17T16:59:51.907 INFO:tasks.rados.rados.0.smithi063.stdout:1205: finishing rollback tid 1 to smithi06387500-41 2024-08-17T16:59:51.907 INFO:tasks.rados.rados.0.smithi063.stdout:1203: done (4 left) 2024-08-17T16:59:51.907 INFO:tasks.rados.rados.0.smithi063.stdout:1206: done (3 left) 2024-08-17T16:59:51.907 INFO:tasks.rados.rados.0.smithi063.stdout:1210: rollback oid 36 current snap is 153 2024-08-17T16:59:51.907 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 36 to 125 2024-08-17T16:59:51.907 INFO:tasks.rados.rados.0.smithi063.stdout:1205: finishing rollback tid 2 to smithi06387500-41 2024-08-17T16:59:51.907 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 82 (ObjNum 73 snap 9 seq_num 73) dirty exists 2024-08-17T16:59:51.908 INFO:tasks.rados.rados.0.smithi063.stdout:1208: finishing write tid 1 to smithi06387500-32 2024-08-17T16:59:51.908 INFO:tasks.rados.rados.0.smithi063.stdout:1205: done (3 left) 2024-08-17T16:59:51.908 INFO:tasks.rados.rados.0.smithi063.stdout:1211: delete oid 20 current snap is 153 2024-08-17T16:59:51.908 INFO:tasks.rados.rados.0.smithi063.stdout:1208: finishing write tid 2 to smithi06387500-32 2024-08-17T16:59:51.909 INFO:tasks.rados.rados.0.smithi063.stdout:1211: done (3 left) 2024-08-17T16:59:51.909 INFO:tasks.rados.rados.0.smithi063.stdout:1212: write oid 8 current snap is 153 2024-08-17T16:59:51.909 INFO:tasks.rados.rados.0.smithi063.stdout:1212: seq_num 472 ranges {732904=731414,2120442=538824,3278829=504743} 2024-08-17T16:59:51.916 INFO:tasks.rados.rados.0.smithi063.stdout:1212: writing smithi06387500-8 from 732904 to 1464318 tid 1 2024-08-17T16:59:51.923 INFO:tasks.rados.rados.0.smithi063.stdout:1212: writing smithi06387500-8 from 2120442 to 2659266 tid 2 2024-08-17T16:59:51.929 INFO:tasks.rados.rados.0.smithi063.stdout:1212: writing smithi06387500-8 from 3278829 to 3783572 tid 3 2024-08-17T16:59:51.929 INFO:tasks.rados.rados.0.smithi063.stdout:1213: snap_remove snap 121 2024-08-17T16:59:52.160 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:51 smithi162 ceph-mon[34633]: osdmap e372: 8 total, 8 up, 8 in 2024-08-17T16:59:52.161 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:51 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.161 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:51 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.161 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:51 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.161 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:51 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.161 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:51 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.161 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:51 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.238 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:51 smithi063 ceph-mon[42680]: osdmap e372: 8 total, 8 up, 8 in 2024-08-17T16:59:52.238 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:51 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.238 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:51 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.238 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:51 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.238 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:51 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.238 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:51 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.238 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:51 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.239 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:51 smithi063 ceph-mon[109843]: osdmap e372: 8 total, 8 up, 8 in 2024-08-17T16:59:52.239 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:51 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.239 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:51 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.239 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:51 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.239 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:51 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.239 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:51 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.239 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:51 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:52.502 INFO:tasks.rados.rados.0.smithi063.stdout:1209: finishing rollback tid 0 to smithi06387500-16 2024-08-17T16:59:52.502 INFO:tasks.rados.rados.0.smithi063.stdout:1209: finishing rollback tid 1 to smithi06387500-16 2024-08-17T16:59:52.502 INFO:tasks.rados.rados.0.smithi063.stdout:1208: finishing write tid 3 to smithi06387500-32 2024-08-17T16:59:52.502 INFO:tasks.rados.rados.0.smithi063.stdout:1209: finishing rollback tid 2 to smithi06387500-16 2024-08-17T16:59:52.502 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 452 (ObjNum 445 snap 142 seq_num 445) dirty exists 2024-08-17T16:59:52.502 INFO:tasks.rados.rados.0.smithi063.stdout:1208: finishing write tid 4 to smithi06387500-32 2024-08-17T16:59:52.503 INFO:tasks.rados.rados.0.smithi063.stdout:1210: finishing rollback tid 0 to smithi06387500-36 2024-08-17T16:59:52.503 INFO:tasks.rados.rados.0.smithi063.stdout:1210: finishing rollback tid 1 to smithi06387500-36 2024-08-17T16:59:52.503 INFO:tasks.rados.rados.0.smithi063.stdout:1210: finishing rollback tid 2 to smithi06387500-36 2024-08-17T16:59:52.503 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 1230 (ObjNum 398 snap 121 seq_num 398) dirty exists 2024-08-17T16:59:52.503 INFO:tasks.rados.rados.0.smithi063.stdout:1208: finishing write tid 5 to smithi06387500-32 2024-08-17T16:59:52.503 INFO:tasks.rados.rados.0.smithi063.stdout:1208: oid 32 updating version 0 to 216 2024-08-17T16:59:52.503 INFO:tasks.rados.rados.0.smithi063.stdout:1208: oid 32 updating version 216 to 218 2024-08-17T16:59:52.503 INFO:tasks.rados.rados.0.smithi063.stdout:1208: oid 32 updating version 218 to 219 2024-08-17T16:59:52.503 INFO:tasks.rados.rados.0.smithi063.stdout:1208: oid 32 version 219 is already newer than 217 2024-08-17T16:59:52.503 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 219 (ObjNum 470 snap 153 seq_num 470) dirty exists 2024-08-17T16:59:52.503 INFO:tasks.rados.rados.0.smithi063.stdout:1208: left oid 32 (ObjNum 470 snap 153 seq_num 470) 2024-08-17T16:59:52.503 INFO:tasks.rados.rados.0.smithi063.stdout:1212: finishing write tid 1 to smithi06387500-8 2024-08-17T16:59:52.503 INFO:tasks.rados.rados.0.smithi063.stdout:1212: finishing write tid 2 to smithi06387500-8 2024-08-17T16:59:52.503 INFO:tasks.rados.rados.0.smithi063.stdout:1212: finishing write tid 3 to smithi06387500-8 2024-08-17T16:59:52.504 INFO:tasks.rados.rados.0.smithi063.stdout:1212: finishing write tid 4 to smithi06387500-8 2024-08-17T16:59:52.504 INFO:tasks.rados.rados.0.smithi063.stdout:1212: finishing write tid 5 to smithi06387500-8 2024-08-17T16:59:52.504 INFO:tasks.rados.rados.0.smithi063.stdout:1212: oid 8 updating version 0 to 337 2024-08-17T16:59:52.504 INFO:tasks.rados.rados.0.smithi063.stdout:1212: oid 8 updating version 337 to 339 2024-08-17T16:59:52.504 INFO:tasks.rados.rados.0.smithi063.stdout:1212: oid 8 updating version 339 to 340 2024-08-17T16:59:52.504 INFO:tasks.rados.rados.0.smithi063.stdout:1212: oid 8 version 340 is already newer than 338 2024-08-17T16:59:52.504 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 340 (ObjNum 472 snap 153 seq_num 472) dirty exists 2024-08-17T16:59:52.504 INFO:tasks.rados.rados.0.smithi063.stdout:1212: left oid 8 (ObjNum 472 snap 153 seq_num 472) 2024-08-17T16:59:52.504 INFO:tasks.rados.rados.0.smithi063.stdout:1208: done (4 left) 2024-08-17T16:59:52.504 INFO:tasks.rados.rados.0.smithi063.stdout:1209: done (3 left) 2024-08-17T16:59:52.504 INFO:tasks.rados.rados.0.smithi063.stdout:1210: done (2 left) 2024-08-17T16:59:52.504 INFO:tasks.rados.rados.0.smithi063.stdout:1212: done (1 left) 2024-08-17T16:59:52.504 INFO:tasks.rados.rados.0.smithi063.stdout:1213: done (0 left) 2024-08-17T16:59:52.504 INFO:tasks.rados.rados.0.smithi063.stdout:1214: write oid 44 current snap is 153 2024-08-17T16:59:52.504 INFO:tasks.rados.rados.0.smithi063.stdout:1214: seq_num 473 ranges {749711=406663,1778924=682482,3189899=716015} 2024-08-17T16:59:52.509 INFO:tasks.rados.rados.0.smithi063.stdout:1214: writing smithi06387500-44 from 749711 to 1156374 tid 1 2024-08-17T16:59:52.517 INFO:tasks.rados.rados.0.smithi063.stdout:1214: writing smithi06387500-44 from 1778924 to 2461406 tid 2 2024-08-17T16:59:52.525 INFO:tasks.rados.rados.0.smithi063.stdout:1214: writing smithi06387500-44 from 3189899 to 3905914 tid 3 2024-08-17T16:59:52.525 INFO:tasks.rados.rados.0.smithi063.stdout:1215: rollback oid 42 current snap is 153 2024-08-17T16:59:52.525 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 42 to 146 2024-08-17T16:59:52.525 INFO:tasks.rados.rados.0.smithi063.stdout:1214: finishing write tid 1 to smithi06387500-44 2024-08-17T16:59:52.526 INFO:tasks.rados.rados.0.smithi063.stdout:1214: finishing write tid 2 to smithi06387500-44 2024-08-17T16:59:52.526 INFO:tasks.rados.rados.0.smithi063.stdout:1216: snap_remove snap 151 2024-08-17T16:59:53.011 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:52 smithi162 ceph-mon[34633]: pgmap v11: 89 pgs: 89 active+clean; 190 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 1.7 MiB/s rd, 475 KiB/s wr, 3 op/s 2024-08-17T16:59:53.011 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:52 smithi162 ceph-mon[34633]: osdmap e373: 8 total, 8 up, 8 in 2024-08-17T16:59:53.011 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:52 smithi162 ceph-mon[34633]: osdmap e374: 8 total, 8 up, 8 in 2024-08-17T16:59:53.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:52 smithi063 ceph-mon[109843]: pgmap v11: 89 pgs: 89 active+clean; 190 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 1.7 MiB/s rd, 475 KiB/s wr, 3 op/s 2024-08-17T16:59:53.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:52 smithi063 ceph-mon[109843]: osdmap e373: 8 total, 8 up, 8 in 2024-08-17T16:59:53.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:52 smithi063 ceph-mon[109843]: osdmap e374: 8 total, 8 up, 8 in 2024-08-17T16:59:53.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:52 smithi063 ceph-mon[42680]: pgmap v11: 89 pgs: 89 active+clean; 190 MiB data, 678 MiB used, 715 GiB / 715 GiB avail; 1.7 MiB/s rd, 475 KiB/s wr, 3 op/s 2024-08-17T16:59:53.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:52 smithi063 ceph-mon[42680]: osdmap e373: 8 total, 8 up, 8 in 2024-08-17T16:59:53.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:52 smithi063 ceph-mon[42680]: osdmap e374: 8 total, 8 up, 8 in 2024-08-17T16:59:53.506 INFO:tasks.rados.rados.0.smithi063.stdout:1216: done (2 left) 2024-08-17T16:59:53.506 INFO:tasks.rados.rados.0.smithi063.stdout:1217: delete oid 45 current snap is 153 2024-08-17T16:59:53.506 INFO:tasks.rados.rados.0.smithi063.stdout:1215: finishing rollback tid 0 to smithi06387500-42 2024-08-17T16:59:53.506 INFO:tasks.rados.rados.0.smithi063.stdout:1215: finishing rollback tid 1 to smithi06387500-42 2024-08-17T16:59:53.506 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 838 (ObjNum 49 snap 0 seq_num 1806783728) dirty dne 2024-08-17T16:59:53.506 INFO:tasks.rados.rados.0.smithi063.stdout:1214: finishing write tid 3 to smithi06387500-44 2024-08-17T16:59:53.507 INFO:tasks.rados.rados.0.smithi063.stdout:1214: finishing write tid 4 to smithi06387500-44 2024-08-17T16:59:53.507 INFO:tasks.rados.rados.0.smithi063.stdout:1214: finishing write tid 5 to smithi06387500-44 2024-08-17T16:59:53.507 INFO:tasks.rados.rados.0.smithi063.stdout:1214: oid 44 updating version 0 to 232 2024-08-17T16:59:53.507 INFO:tasks.rados.rados.0.smithi063.stdout:1214: oid 44 version 232 is already newer than 231 2024-08-17T16:59:53.507 INFO:tasks.rados.rados.0.smithi063.stdout:1214: oid 44 version 232 is already newer than 230 2024-08-17T16:59:53.507 INFO:tasks.rados.rados.0.smithi063.stdout:1214: oid 44 version 232 is already newer than 229 2024-08-17T16:59:53.507 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 232 (ObjNum 473 snap 153 seq_num 473) dirty exists 2024-08-17T16:59:53.507 INFO:tasks.rados.rados.0.smithi063.stdout:1214: left oid 44 (ObjNum 473 snap 153 seq_num 473) 2024-08-17T16:59:53.508 INFO:tasks.rados.rados.0.smithi063.stdout:1214: done (2 left) 2024-08-17T16:59:53.508 INFO:tasks.rados.rados.0.smithi063.stdout:1215: done (1 left) 2024-08-17T16:59:53.508 INFO:tasks.rados.rados.0.smithi063.stdout:1217: done (0 left) 2024-08-17T16:59:53.508 INFO:tasks.rados.rados.0.smithi063.stdout:1218: write oid 50 current snap is 153 2024-08-17T16:59:53.508 INFO:tasks.rados.rados.0.smithi063.stdout:1218: seq_num 475 ranges {783325=557161,1779535=686151,3244745=454912} 2024-08-17T16:59:53.515 INFO:tasks.rados.rados.0.smithi063.stdout:1218: writing smithi06387500-50 from 783325 to 1340486 tid 1 2024-08-17T16:59:53.522 INFO:tasks.rados.rados.0.smithi063.stdout:1218: writing smithi06387500-50 from 1779535 to 2465686 tid 2 2024-08-17T16:59:53.530 INFO:tasks.rados.rados.0.smithi063.stdout:1218: writing smithi06387500-50 from 3244745 to 3699657 tid 3 2024-08-17T16:59:53.530 INFO:tasks.rados.rados.0.smithi063.stdout:1219: read oid 31 snap 134 2024-08-17T16:59:53.530 INFO:tasks.rados.rados.0.smithi063.stdout:1219: expect (ObjNum 395 snap 121 seq_num 395) 2024-08-17T16:59:53.530 INFO:tasks.rados.rados.0.smithi063.stdout:1220: write oid 34 current snap is 153 2024-08-17T16:59:53.531 INFO:tasks.rados.rados.0.smithi063.stdout:1218: finishing write tid 1 to smithi06387500-50 2024-08-17T16:59:53.531 INFO:tasks.rados.rados.0.smithi063.stdout:1220: seq_num 476 ranges {400132=632410,1438017=546162} 2024-08-17T16:59:53.536 INFO:tasks.rados.rados.0.smithi063.stdout:1220: writing smithi06387500-34 from 400132 to 1032542 tid 1 2024-08-17T16:59:53.542 INFO:tasks.rados.rados.0.smithi063.stdout:1220: writing smithi06387500-34 from 1438017 to 1984179 tid 2 2024-08-17T16:59:53.542 INFO:tasks.rados.rados.0.smithi063.stdout:1221: write oid 10 current snap is 153 2024-08-17T16:59:53.542 INFO:tasks.rados.rados.0.smithi063.stdout:1218: finishing write tid 2 to smithi06387500-50 2024-08-17T16:59:53.542 INFO:tasks.rados.rados.0.smithi063.stdout:1221: seq_num 477 ranges {416939=707659,1896499=120440} 2024-08-17T16:59:53.548 INFO:tasks.rados.rados.0.smithi063.stdout:1221: writing smithi06387500-10 from 416939 to 1124598 tid 1 2024-08-17T16:59:53.553 INFO:tasks.rados.rados.0.smithi063.stdout:1221: writing smithi06387500-10 from 1896499 to 2016939 tid 2 2024-08-17T16:59:53.553 INFO:tasks.rados.rados.0.smithi063.stdout:1222: write oid 42 current snap is 153 2024-08-17T16:59:53.553 INFO:tasks.rados.rados.0.smithi063.stdout:1218: finishing write tid 3 to smithi06387500-50 2024-08-17T16:59:53.554 INFO:tasks.rados.rados.0.smithi063.stdout:1218: finishing write tid 4 to smithi06387500-50 2024-08-17T16:59:53.554 INFO:tasks.rados.rados.0.smithi063.stdout:1218: finishing write tid 5 to smithi06387500-50 2024-08-17T16:59:53.554 INFO:tasks.rados.rados.0.smithi063.stdout:1218: oid 50 updating version 0 to 842 2024-08-17T16:59:53.554 INFO:tasks.rados.rados.0.smithi063.stdout:1218: oid 50 updating version 842 to 843 2024-08-17T16:59:53.554 INFO:tasks.rados.rados.0.smithi063.stdout:1218: oid 50 version 843 is already newer than 841 2024-08-17T16:59:53.554 INFO:tasks.rados.rados.0.smithi063.stdout:1218: oid 50 updating version 843 to 844 2024-08-17T16:59:53.554 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 844 (ObjNum 475 snap 153 seq_num 475) dirty exists 2024-08-17T16:59:53.554 INFO:tasks.rados.rados.0.smithi063.stdout:1218: left oid 50 (ObjNum 475 snap 153 seq_num 475) 2024-08-17T16:59:53.554 INFO:tasks.rados.rados.0.smithi063.stdout:1219: expect (ObjNum 395 snap 121 seq_num 395) 2024-08-17T16:59:53.653 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 16:59:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:16:59:53] "GET /metrics HTTP/1.1" 200 33465 "" "Prometheus/2.51.0" 2024-08-17T16:59:53.726 INFO:tasks.rados.rados.0.smithi063.stdout:1220: finishing write tid 1 to smithi06387500-34 2024-08-17T16:59:53.726 INFO:tasks.rados.rados.0.smithi063.stdout:1220: finishing write tid 2 to smithi06387500-34 2024-08-17T16:59:53.726 INFO:tasks.rados.rados.0.smithi063.stdout:1220: finishing write tid 3 to smithi06387500-34 2024-08-17T16:59:53.726 INFO:tasks.rados.rados.0.smithi063.stdout:1220: finishing write tid 4 to smithi06387500-34 2024-08-17T16:59:53.726 INFO:tasks.rados.rados.0.smithi063.stdout:1220: oid 34 updating version 0 to 846 2024-08-17T16:59:53.726 INFO:tasks.rados.rados.0.smithi063.stdout:1220: oid 34 updating version 846 to 847 2024-08-17T16:59:53.726 INFO:tasks.rados.rados.0.smithi063.stdout:1220: oid 34 updating version 847 to 848 2024-08-17T16:59:53.726 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 848 (ObjNum 476 snap 153 seq_num 476) dirty exists 2024-08-17T16:59:53.726 INFO:tasks.rados.rados.0.smithi063.stdout:1220: left oid 34 (ObjNum 476 snap 153 seq_num 476) 2024-08-17T16:59:53.726 INFO:tasks.rados.rados.0.smithi063.stdout:1221: finishing write tid 1 to smithi06387500-10 2024-08-17T16:59:53.727 INFO:tasks.rados.rados.0.smithi063.stdout:1221: finishing write tid 2 to smithi06387500-10 2024-08-17T16:59:53.727 INFO:tasks.rados.rados.0.smithi063.stdout:1221: finishing write tid 3 to smithi06387500-10 2024-08-17T16:59:53.727 INFO:tasks.rados.rados.0.smithi063.stdout:1221: finishing write tid 4 to smithi06387500-10 2024-08-17T16:59:53.727 INFO:tasks.rados.rados.0.smithi063.stdout:1221: oid 10 updating version 0 to 444 2024-08-17T16:59:53.727 INFO:tasks.rados.rados.0.smithi063.stdout:1221: oid 10 updating version 444 to 445 2024-08-17T16:59:53.727 INFO:tasks.rados.rados.0.smithi063.stdout:1221: oid 10 updating version 445 to 446 2024-08-17T16:59:53.727 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 446 (ObjNum 477 snap 153 seq_num 477) dirty exists 2024-08-17T16:59:53.727 INFO:tasks.rados.rados.0.smithi063.stdout:1221: left oid 10 (ObjNum 477 snap 153 seq_num 477) 2024-08-17T16:59:53.727 INFO:tasks.rados.rados.0.smithi063.stdout:1222: seq_num 478 ranges {433746=782908,1954981=78765} 2024-08-17T16:59:53.732 INFO:tasks.rados.rados.0.smithi063.stdout:1222: writing smithi06387500-42 from 433746 to 1216654 tid 1 2024-08-17T16:59:53.737 INFO:tasks.rados.rados.0.smithi063.stdout:1222: writing smithi06387500-42 from 1954981 to 2033746 tid 2 2024-08-17T16:59:53.737 INFO:tasks.rados.rados.0.smithi063.stdout:1218: done (4 left) 2024-08-17T16:59:53.737 INFO:tasks.rados.rados.0.smithi063.stdout:1219: done (3 left) 2024-08-17T16:59:53.737 INFO:tasks.rados.rados.0.smithi063.stdout:1220: done (2 left) 2024-08-17T16:59:53.737 INFO:tasks.rados.rados.0.smithi063.stdout:1221: done (1 left) 2024-08-17T16:59:53.737 INFO:tasks.rados.rados.0.smithi063.stdout:1223: snap_remove snap 134 2024-08-17T16:59:54.515 INFO:tasks.rados.rados.0.smithi063.stdout:1222: finishing write tid 1 to smithi06387500-42 2024-08-17T16:59:54.515 INFO:tasks.rados.rados.0.smithi063.stdout:1222: finishing write tid 2 to smithi06387500-42 2024-08-17T16:59:54.515 INFO:tasks.rados.rados.0.smithi063.stdout:1222: finishing write tid 3 to smithi06387500-42 2024-08-17T16:59:54.515 INFO:tasks.rados.rados.0.smithi063.stdout:1222: finishing write tid 4 to smithi06387500-42 2024-08-17T16:59:54.515 INFO:tasks.rados.rados.0.smithi063.stdout:1222: oid 42 updating version 0 to 850 2024-08-17T16:59:54.515 INFO:tasks.rados.rados.0.smithi063.stdout:1222: oid 42 updating version 850 to 851 2024-08-17T16:59:54.516 INFO:tasks.rados.rados.0.smithi063.stdout:1222: oid 42 version 851 is already newer than 849 2024-08-17T16:59:54.516 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 851 (ObjNum 478 snap 153 seq_num 478) dirty exists 2024-08-17T16:59:54.516 INFO:tasks.rados.rados.0.smithi063.stdout:1222: left oid 42 (ObjNum 478 snap 153 seq_num 478) 2024-08-17T16:59:54.516 INFO:tasks.rados.rados.0.smithi063.stdout:1222: done (1 left) 2024-08-17T16:59:54.516 INFO:tasks.rados.rados.0.smithi063.stdout:1223: done (0 left) 2024-08-17T16:59:54.516 INFO:tasks.rados.rados.0.smithi063.stdout:1224: read oid 35 snap -1 2024-08-17T16:59:54.516 INFO:tasks.rados.rados.0.smithi063.stdout:1224: expect (ObjNum 450 snap 145 seq_num 450) 2024-08-17T16:59:54.516 INFO:tasks.rados.rados.0.smithi063.stdout:1225: delete oid 34 current snap is 153 2024-08-17T16:59:54.520 INFO:tasks.rados.rados.0.smithi063.stdout:1225: done (1 left) 2024-08-17T16:59:54.520 INFO:tasks.rados.rados.0.smithi063.stdout:1226: write oid 38 current snap is 153 2024-08-17T16:59:54.521 INFO:tasks.rados.rados.0.smithi063.stdout:1226: seq_num 480 ranges {467360=649759,1671945=395415} 2024-08-17T16:59:54.527 INFO:tasks.rados.rados.0.smithi063.stdout:1226: writing smithi06387500-38 from 467360 to 1117119 tid 1 2024-08-17T16:59:54.533 INFO:tasks.rados.rados.0.smithi063.stdout:1226: writing smithi06387500-38 from 1671945 to 2067360 tid 2 2024-08-17T16:59:54.533 INFO:tasks.rados.rados.0.smithi063.stdout:1227: snap_remove snap 148 2024-08-17T16:59:54.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:54 smithi162 ceph-mon[34633]: osdmap e375: 8 total, 8 up, 8 in 2024-08-17T16:59:54.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:54 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:54.686 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:54 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:54.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:54 smithi063 ceph-mon[42680]: osdmap e375: 8 total, 8 up, 8 in 2024-08-17T16:59:54.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:54 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:54.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:54 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:54.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:54 smithi063 ceph-mon[109843]: osdmap e375: 8 total, 8 up, 8 in 2024-08-17T16:59:54.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:54 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:54.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:54 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:55.512 INFO:tasks.rados.rados.0.smithi063.stdout:1227: done (2 left) 2024-08-17T16:59:55.512 INFO:tasks.rados.rados.0.smithi063.stdout:1228: rollback oid 17 current snap is 153 2024-08-17T16:59:55.512 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 17 to 125 2024-08-17T16:59:55.512 INFO:tasks.rados.rados.0.smithi063.stdout:1224: expect (ObjNum 450 snap 145 seq_num 450) 2024-08-17T16:59:55.636 INFO:tasks.rados.rados.0.smithi063.stdout:1226: finishing write tid 1 to smithi06387500-38 2024-08-17T16:59:55.636 INFO:tasks.rados.rados.0.smithi063.stdout:1226: finishing write tid 2 to smithi06387500-38 2024-08-17T16:59:55.636 INFO:tasks.rados.rados.0.smithi063.stdout:1226: finishing write tid 3 to smithi06387500-38 2024-08-17T16:59:55.636 INFO:tasks.rados.rados.0.smithi063.stdout:1226: finishing write tid 4 to smithi06387500-38 2024-08-17T16:59:55.636 INFO:tasks.rados.rados.0.smithi063.stdout:1226: oid 38 updating version 0 to 178 2024-08-17T16:59:55.636 INFO:tasks.rados.rados.0.smithi063.stdout:1226: oid 38 updating version 178 to 179 2024-08-17T16:59:55.636 INFO:tasks.rados.rados.0.smithi063.stdout:1226: oid 38 version 179 is already newer than 177 2024-08-17T16:59:55.637 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 179 (ObjNum 480 snap 153 seq_num 480) dirty exists 2024-08-17T16:59:55.637 INFO:tasks.rados.rados.0.smithi063.stdout:1226: left oid 38 (ObjNum 480 snap 153 seq_num 480) 2024-08-17T16:59:55.637 INFO:tasks.rados.rados.0.smithi063.stdout:1224: done (2 left) 2024-08-17T16:59:55.637 INFO:tasks.rados.rados.0.smithi063.stdout:1226: done (1 left) 2024-08-17T16:59:55.637 INFO:tasks.rados.rados.0.smithi063.stdout:1229: read oid 15 snap -1 2024-08-17T16:59:55.637 INFO:tasks.rados.rados.0.smithi063.stdout:1229: expect (ObjNum 289 snap 88 seq_num 289) 2024-08-17T16:59:55.637 INFO:tasks.rados.rados.0.smithi063.stdout:1230: rollback oid 32 current snap is 153 2024-08-17T16:59:55.637 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 32 to 152 2024-08-17T16:59:55.637 INFO:tasks.rados.rados.0.smithi063.stdout:1228: finishing rollback tid 1 to smithi06387500-17 2024-08-17T16:59:55.637 INFO:tasks.rados.rados.0.smithi063.stdout:1228: finishing rollback tid 2 to smithi06387500-17 2024-08-17T16:59:55.637 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 385 (ObjNum 392 snap 119 seq_num 392) dirty exists 2024-08-17T16:59:55.637 INFO:tasks.rados.rados.0.smithi063.stdout:1228: done (2 left) 2024-08-17T16:59:55.637 INFO:tasks.rados.rados.0.smithi063.stdout:1231: read oid 39 snap -1 2024-08-17T16:59:55.638 INFO:tasks.rados.rados.0.smithi063.stdout:1231: expect deleted 2024-08-17T16:59:55.638 INFO:tasks.rados.rados.0.smithi063.stdout:1232: rollback oid 33 current snap is 153 2024-08-17T16:59:55.638 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 33 to 150 2024-08-17T16:59:55.638 INFO:tasks.rados.rados.0.smithi063.stdout:1233: write oid 41 current snap is 153 2024-08-17T16:59:55.638 INFO:tasks.rados.rados.0.smithi063.stdout:1233: seq_num 481 ranges {484167=725008,1730427=353740} 2024-08-17T16:59:55.643 INFO:tasks.rados.rados.0.smithi063.stdout:1233: writing smithi06387500-41 from 484167 to 1209175 tid 1 2024-08-17T16:59:55.648 INFO:tasks.rados.rados.0.smithi063.stdout:1233: writing smithi06387500-41 from 1730427 to 2084167 tid 2 2024-08-17T16:59:55.648 INFO:tasks.rados.rados.0.smithi063.stdout:1234: read oid 29 snap -1 2024-08-17T16:59:55.648 INFO:tasks.rados.rados.0.smithi063.stdout:1234: expect (ObjNum 468 snap 152 seq_num 468) 2024-08-17T16:59:55.648 INFO:tasks.rados.rados.0.smithi063.stdout:1235: write oid 16 current snap is 153 2024-08-17T16:59:55.648 INFO:tasks.rados.rados.0.smithi063.stdout:1235: seq_num 482 ranges {500974=400257,1388909=557169} 2024-08-17T16:59:55.653 INFO:tasks.rados.rados.0.smithi063.stdout:1235: writing smithi06387500-16 from 500974 to 901231 tid 1 2024-08-17T16:59:55.659 INFO:tasks.rados.rados.0.smithi063.stdout:1235: writing smithi06387500-16 from 1388909 to 1946078 tid 2 2024-08-17T16:59:55.660 INFO:tasks.rados.rados.0.smithi063.stdout:1236: write oid 43 current snap is 153 2024-08-17T16:59:55.660 INFO:tasks.rados.rados.0.smithi063.stdout:1236: seq_num 483 ranges {517781=475506,1731038=386743} 2024-08-17T16:59:55.665 INFO:tasks.rados.rados.0.smithi063.stdout:1236: writing smithi06387500-43 from 517781 to 993287 tid 1 2024-08-17T16:59:55.671 INFO:tasks.rados.rados.0.smithi063.stdout:1236: writing smithi06387500-43 from 1731038 to 2117781 tid 2 2024-08-17T16:59:55.672 INFO:tasks.rados.rados.0.smithi063.stdout:1237: write oid 4 current snap is 153 2024-08-17T16:59:55.672 INFO:tasks.rados.rados.0.smithi063.stdout:1230: finishing rollback tid 0 to smithi06387500-32 2024-08-17T16:59:55.672 INFO:tasks.rados.rados.0.smithi063.stdout:1230: finishing rollback tid 1 to smithi06387500-32 2024-08-17T16:59:55.672 INFO:tasks.rados.rados.0.smithi063.stdout:1230: finishing rollback tid 2 to smithi06387500-32 2024-08-17T16:59:55.672 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 232 (ObjNum 423 snap 130 seq_num 423) dirty exists 2024-08-17T16:59:55.672 INFO:tasks.rados.rados.0.smithi063.stdout:1232: finishing rollback tid 0 to smithi06387500-33 2024-08-17T16:59:55.672 INFO:tasks.rados.rados.0.smithi063.stdout:1232: finishing rollback tid 1 to smithi06387500-33 2024-08-17T16:59:55.672 INFO:tasks.rados.rados.0.smithi063.stdout:1232: finishing rollback tid 2 to smithi06387500-33 2024-08-17T16:59:55.672 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 356 (ObjNum 351 snap 105 seq_num 351) dirty exists 2024-08-17T16:59:55.672 INFO:tasks.rados.rados.0.smithi063.stdout:1229: expect (ObjNum 289 snap 88 seq_num 289) 2024-08-17T16:59:55.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:55 smithi063 ceph-mon[109843]: pgmap v15: 89 pgs: 89 active+clean; 194 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 1000 KiB/s wr, 24 op/s 2024-08-17T16:59:55.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:55 smithi063 ceph-mon[109843]: osdmap e376: 8 total, 8 up, 8 in 2024-08-17T16:59:55.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:55 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:55.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:55 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:55.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:55 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:55.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:55 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:55 smithi063 ceph-mon[42680]: pgmap v15: 89 pgs: 89 active+clean; 194 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 1000 KiB/s wr, 24 op/s 2024-08-17T16:59:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:55 smithi063 ceph-mon[42680]: osdmap e376: 8 total, 8 up, 8 in 2024-08-17T16:59:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:55 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:55 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:55 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:55 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:55.796 INFO:tasks.rados.rados.0.smithi063.stdout:1233: finishing write tid 1 to smithi06387500-41 2024-08-17T16:59:55.796 INFO:tasks.rados.rados.0.smithi063.stdout:1233: finishing write tid 2 to smithi06387500-41 2024-08-17T16:59:55.796 INFO:tasks.rados.rados.0.smithi063.stdout:1233: finishing write tid 3 to smithi06387500-41 2024-08-17T16:59:55.796 INFO:tasks.rados.rados.0.smithi063.stdout:1233: finishing write tid 4 to smithi06387500-41 2024-08-17T16:59:55.796 INFO:tasks.rados.rados.0.smithi063.stdout:1233: oid 41 updating version 0 to 95 2024-08-17T16:59:55.796 INFO:tasks.rados.rados.0.smithi063.stdout:1233: oid 41 version 95 is already newer than 94 2024-08-17T16:59:55.796 INFO:tasks.rados.rados.0.smithi063.stdout:1233: oid 41 version 95 is already newer than 93 2024-08-17T16:59:55.796 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 95 (ObjNum 481 snap 153 seq_num 481) dirty exists 2024-08-17T16:59:55.796 INFO:tasks.rados.rados.0.smithi063.stdout:1233: left oid 41 (ObjNum 481 snap 153 seq_num 481) 2024-08-17T16:59:55.796 INFO:tasks.rados.rados.0.smithi063.stdout:1237: seq_num 484 ranges {534588=550755,1789520=345068} 2024-08-17T16:59:55.801 INFO:tasks.rados.rados.0.smithi063.stdout:1237: writing smithi06387500-4 from 534588 to 1085343 tid 1 2024-08-17T16:59:55.807 INFO:tasks.rados.rados.0.smithi063.stdout:1237: writing smithi06387500-4 from 1789520 to 2134588 tid 2 2024-08-17T16:59:55.807 INFO:tasks.rados.rados.0.smithi063.stdout:1229: done (8 left) 2024-08-17T16:59:55.807 INFO:tasks.rados.rados.0.smithi063.stdout:1230: done (7 left) 2024-08-17T16:59:55.808 INFO:tasks.rados.rados.0.smithi063.stdout:1231: done (6 left) 2024-08-17T16:59:55.808 INFO:tasks.rados.rados.0.smithi063.stdout:1232: done (5 left) 2024-08-17T16:59:55.808 INFO:tasks.rados.rados.0.smithi063.stdout:1233: done (4 left) 2024-08-17T16:59:55.808 INFO:tasks.rados.rados.0.smithi063.stdout:1238: snap_create 2024-08-17T16:59:55.808 INFO:tasks.rados.rados.0.smithi063.stdout:1234: expect (ObjNum 468 snap 152 seq_num 468) 2024-08-17T16:59:55.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:55 smithi162 ceph-mon[34633]: pgmap v15: 89 pgs: 89 active+clean; 194 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 1000 KiB/s wr, 24 op/s 2024-08-17T16:59:55.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:55 smithi162 ceph-mon[34633]: osdmap e376: 8 total, 8 up, 8 in 2024-08-17T16:59:55.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:55 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:55.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:55 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:55.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:55 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:55.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:55 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T16:59:56.051 INFO:tasks.rados.rados.0.smithi063.stdout:1235: finishing write tid 1 to smithi06387500-16 2024-08-17T16:59:56.052 INFO:tasks.rados.rados.0.smithi063.stdout:1235: finishing write tid 2 to smithi06387500-16 2024-08-17T16:59:56.052 INFO:tasks.rados.rados.0.smithi063.stdout:1235: finishing write tid 3 to smithi06387500-16 2024-08-17T16:59:56.052 INFO:tasks.rados.rados.0.smithi063.stdout:1235: finishing write tid 4 to smithi06387500-16 2024-08-17T16:59:56.052 INFO:tasks.rados.rados.0.smithi063.stdout:1235: oid 16 updating version 0 to 465 2024-08-17T16:59:56.052 INFO:tasks.rados.rados.0.smithi063.stdout:1235: oid 16 version 465 is already newer than 464 2024-08-17T16:59:56.052 INFO:tasks.rados.rados.0.smithi063.stdout:1235: oid 16 version 465 is already newer than 463 2024-08-17T16:59:56.052 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 465 (ObjNum 482 snap 153 seq_num 482) dirty exists 2024-08-17T16:59:56.052 INFO:tasks.rados.rados.0.smithi063.stdout:1235: left oid 16 (ObjNum 482 snap 153 seq_num 482) 2024-08-17T16:59:56.052 INFO:tasks.rados.rados.0.smithi063.stdout:1236: finishing write tid 1 to smithi06387500-43 2024-08-17T16:59:56.052 INFO:tasks.rados.rados.0.smithi063.stdout:1236: finishing write tid 2 to smithi06387500-43 2024-08-17T16:59:56.052 INFO:tasks.rados.rados.0.smithi063.stdout:1236: finishing write tid 3 to smithi06387500-43 2024-08-17T16:59:56.052 INFO:tasks.rados.rados.0.smithi063.stdout:1236: finishing write tid 4 to smithi06387500-43 2024-08-17T16:59:56.052 INFO:tasks.rados.rados.0.smithi063.stdout:1236: oid 43 updating version 0 to 659 2024-08-17T16:59:56.053 INFO:tasks.rados.rados.0.smithi063.stdout:1236: oid 43 updating version 659 to 661 2024-08-17T16:59:56.053 INFO:tasks.rados.rados.0.smithi063.stdout:1236: oid 43 version 661 is already newer than 660 2024-08-17T16:59:56.053 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 661 (ObjNum 483 snap 153 seq_num 483) dirty exists 2024-08-17T16:59:56.053 INFO:tasks.rados.rados.0.smithi063.stdout:1236: left oid 43 (ObjNum 483 snap 153 seq_num 483) 2024-08-17T16:59:56.053 INFO:tasks.rados.rados.0.smithi063.stdout:1237: finishing write tid 1 to smithi06387500-4 2024-08-17T16:59:56.053 INFO:tasks.rados.rados.0.smithi063.stdout:1237: finishing write tid 2 to smithi06387500-4 2024-08-17T16:59:56.053 INFO:tasks.rados.rados.0.smithi063.stdout:1237: finishing write tid 3 to smithi06387500-4 2024-08-17T16:59:56.053 INFO:tasks.rados.rados.0.smithi063.stdout:1237: finishing write tid 4 to smithi06387500-4 2024-08-17T16:59:56.053 INFO:tasks.rados.rados.0.smithi063.stdout:1237: oid 4 updating version 0 to 1256 2024-08-17T16:59:56.053 INFO:tasks.rados.rados.0.smithi063.stdout:1237: oid 4 updating version 1256 to 1258 2024-08-17T16:59:56.053 INFO:tasks.rados.rados.0.smithi063.stdout:1237: oid 4 version 1258 is already newer than 1257 2024-08-17T16:59:56.053 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 1258 (ObjNum 484 snap 153 seq_num 484) dirty exists 2024-08-17T16:59:56.053 INFO:tasks.rados.rados.0.smithi063.stdout:1237: left oid 4 (ObjNum 484 snap 153 seq_num 484) 2024-08-17T16:59:56.668 INFO:tasks.rados.rados.0.smithi063.stdout:1234: done (4 left) 2024-08-17T16:59:56.668 INFO:tasks.rados.rados.0.smithi063.stdout:1235: done (3 left) 2024-08-17T16:59:56.668 INFO:tasks.rados.rados.0.smithi063.stdout:1236: done (2 left) 2024-08-17T16:59:56.668 INFO:tasks.rados.rados.0.smithi063.stdout:1237: done (1 left) 2024-08-17T16:59:56.669 INFO:tasks.rados.rados.0.smithi063.stdout:1238: done (0 left) 2024-08-17T16:59:56.669 INFO:tasks.rados.rados.0.smithi063.stdout:1239: delete oid 32 current snap is 154 2024-08-17T16:59:56.671 INFO:tasks.rados.rados.0.smithi063.stdout:1239: done (0 left) 2024-08-17T16:59:56.671 INFO:tasks.rados.rados.0.smithi063.stdout:1240: delete oid 41 current snap is 154 2024-08-17T16:59:56.675 INFO:tasks.rados.rados.0.smithi063.stdout:1240: done (0 left) 2024-08-17T16:59:56.675 INFO:tasks.rados.rados.0.smithi063.stdout:1241: snap_create 2024-08-17T16:59:56.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:59:56.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:59:56.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:59:56.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:59:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: Updating smithi162:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:59:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:59:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:59:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:59:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: osdmap e377: 8 total, 8 up, 8 in 2024-08-17T16:59:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T16:59:56.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-08-17T16:59:56.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:59:56.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:59:56.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:59:56.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:59:56.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: Updating smithi162:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:59:56.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:59:56.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:59:56.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:59:56.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: osdmap e377: 8 total, 8 up, 8 in 2024-08-17T16:59:56.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:56.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:56.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:56.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:56.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:56.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:56.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:56.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T16:59:56.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:56 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-08-17T16:59:56.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: Updating smithi063:/etc/ceph/ceph.conf 2024-08-17T16:59:56.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: Updating smithi162:/etc/ceph/ceph.conf 2024-08-17T16:59:56.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:59:56.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.conf 2024-08-17T16:59:56.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: Updating smithi162:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:59:56.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: Updating smithi063:/etc/ceph/ceph.client.admin.keyring 2024-08-17T16:59:56.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: Updating smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:59:56.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: Updating smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/config/ceph.client.admin.keyring 2024-08-17T16:59:56.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: osdmap e377: 8 total, 8 up, 8 in 2024-08-17T16:59:56.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:56.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:56.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:56.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:56.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:56.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T16:59:56.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T16:59:56.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T16:59:56.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:56 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-08-17T16:59:57.669 INFO:tasks.rados.rados.0.smithi063.stdout:1241: done (0 left) 2024-08-17T16:59:57.669 INFO:tasks.rados.rados.0.smithi063.stdout:1242: delete oid 34 current snap is 155 2024-08-17T16:59:57.672 INFO:tasks.rados.rados.0.smithi063.stdout:1242: done (0 left) 2024-08-17T16:59:57.673 INFO:tasks.rados.rados.0.smithi063.stdout:1243: write oid 46 current snap is 155 2024-08-17T16:59:57.673 INFO:tasks.rados.rados.0.smithi063.stdout:1243: seq_num 488 ranges {601816=568104,1623448=451823} 2024-08-17T16:59:57.679 INFO:tasks.rados.rados.0.smithi063.stdout:1243: writing smithi06387500-46 from 601816 to 1169920 tid 1 2024-08-17T16:59:57.685 INFO:tasks.rados.rados.0.smithi063.stdout:1243: writing smithi06387500-46 from 1623448 to 2075271 tid 2 2024-08-17T16:59:57.685 INFO:tasks.rados.rados.0.smithi063.stdout:1243: finishing write tid 1 to smithi06387500-46 2024-08-17T16:59:57.685 INFO:tasks.rados.rados.0.smithi063.stdout:1244: write oid 48 current snap is 155 2024-08-17T16:59:57.685 INFO:tasks.rados.rados.0.smithi063.stdout:1244: seq_num 489 ranges {618623=643353,1681930=536693} 2024-08-17T16:59:57.694 INFO:tasks.rados.rados.0.smithi063.stdout:1244: writing smithi06387500-48 from 618623 to 1261976 tid 1 2024-08-17T16:59:57.702 INFO:tasks.rados.rados.0.smithi063.stdout:1244: writing smithi06387500-48 from 1681930 to 2218623 tid 2 2024-08-17T16:59:57.702 INFO:tasks.rados.rados.0.smithi063.stdout:1245: write oid 18 current snap is 155 2024-08-17T16:59:57.702 INFO:tasks.rados.rados.0.smithi063.stdout:1245: seq_num 490 ranges {635430=718602,2140412=95018} 2024-08-17T16:59:57.709 INFO:tasks.rados.rados.0.smithi063.stdout:1245: writing smithi06387500-18 from 635430 to 1354032 tid 1 2024-08-17T16:59:57.715 INFO:tasks.rados.rados.0.smithi063.stdout:1245: writing smithi06387500-18 from 2140412 to 2235430 tid 2 2024-08-17T16:59:57.715 INFO:tasks.rados.rados.0.smithi063.stdout:1246: snap_remove snap 152 2024-08-17T16:59:57.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:57 smithi063 ceph-mon[42680]: Upgrade: It appears safe to stop mon.c 2024-08-17T16:59:57.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:57 smithi063 ceph-mon[42680]: pgmap v18: 89 pgs: 89 active+clean; 194 MiB data, 680 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:59:57.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:57 smithi063 ceph-mon[42680]: osdmap e378: 8 total, 8 up, 8 in 2024-08-17T16:59:57.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:57 smithi063 ceph-mon[42680]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:57.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:57 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:59:57.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:57 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:59:57.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:57 smithi063 ceph-mon[42680]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:57.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:57 smithi063 ceph-mon[109843]: Upgrade: It appears safe to stop mon.c 2024-08-17T16:59:57.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:57 smithi063 ceph-mon[109843]: pgmap v18: 89 pgs: 89 active+clean; 194 MiB data, 680 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:59:57.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:57 smithi063 ceph-mon[109843]: osdmap e378: 8 total, 8 up, 8 in 2024-08-17T16:59:57.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:57 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:57.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:57 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:59:57.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:57 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:59:57.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:57 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:57.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:57 smithi162 ceph-mon[34633]: Upgrade: It appears safe to stop mon.c 2024-08-17T16:59:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:57 smithi162 ceph-mon[34633]: pgmap v18: 89 pgs: 89 active+clean; 194 MiB data, 680 MiB used, 715 GiB / 715 GiB avail 2024-08-17T16:59:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:57 smithi162 ceph-mon[34633]: osdmap e378: 8 total, 8 up, 8 in 2024-08-17T16:59:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:57 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T16:59:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:57 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T16:59:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:57 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T16:59:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:57 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T16:59:58.676 INFO:tasks.rados.rados.0.smithi063.stdout:1243: finishing write tid 2 to smithi06387500-46 2024-08-17T16:59:58.676 INFO:tasks.rados.rados.0.smithi063.stdout:1243: finishing write tid 3 to smithi06387500-46 2024-08-17T16:59:58.676 INFO:tasks.rados.rados.0.smithi063.stdout:1243: finishing write tid 4 to smithi06387500-46 2024-08-17T16:59:58.676 INFO:tasks.rados.rados.0.smithi063.stdout:1243: oid 46 updating version 0 to 469 2024-08-17T16:59:58.676 INFO:tasks.rados.rados.0.smithi063.stdout:1243: oid 46 version 469 is already newer than 467 2024-08-17T16:59:58.676 INFO:tasks.rados.rados.0.smithi063.stdout:1243: oid 46 version 469 is already newer than 468 2024-08-17T16:59:58.677 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 469 (ObjNum 488 snap 155 seq_num 488) dirty exists 2024-08-17T16:59:58.677 INFO:tasks.rados.rados.0.smithi063.stdout:1243: left oid 46 (ObjNum 488 snap 155 seq_num 488) 2024-08-17T16:59:58.677 INFO:tasks.rados.rados.0.smithi063.stdout:1244: finishing write tid 1 to smithi06387500-48 2024-08-17T16:59:58.677 INFO:tasks.rados.rados.0.smithi063.stdout:1244: finishing write tid 2 to smithi06387500-48 2024-08-17T16:59:58.677 INFO:tasks.rados.rados.0.smithi063.stdout:1244: finishing write tid 3 to smithi06387500-48 2024-08-17T16:59:58.677 INFO:tasks.rados.rados.0.smithi063.stdout:1244: finishing write tid 4 to smithi06387500-48 2024-08-17T16:59:58.677 INFO:tasks.rados.rados.0.smithi063.stdout:1244: oid 48 updating version 0 to 1262 2024-08-17T16:59:58.677 INFO:tasks.rados.rados.0.smithi063.stdout:1244: oid 48 version 1262 is already newer than 1260 2024-08-17T16:59:58.677 INFO:tasks.rados.rados.0.smithi063.stdout:1244: oid 48 version 1262 is already newer than 1261 2024-08-17T16:59:58.677 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 1262 (ObjNum 489 snap 155 seq_num 489) dirty exists 2024-08-17T16:59:58.677 INFO:tasks.rados.rados.0.smithi063.stdout:1244: left oid 48 (ObjNum 489 snap 155 seq_num 489) 2024-08-17T16:59:58.677 INFO:tasks.rados.rados.0.smithi063.stdout:1245: finishing write tid 1 to smithi06387500-18 2024-08-17T16:59:58.677 INFO:tasks.rados.rados.0.smithi063.stdout:1245: finishing write tid 2 to smithi06387500-18 2024-08-17T16:59:58.677 INFO:tasks.rados.rados.0.smithi063.stdout:1245: finishing write tid 3 to smithi06387500-18 2024-08-17T16:59:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:1245: finishing write tid 4 to smithi06387500-18 2024-08-17T16:59:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:1245: oid 18 updating version 0 to 454 2024-08-17T16:59:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:1245: oid 18 updating version 454 to 455 2024-08-17T16:59:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:1245: oid 18 updating version 455 to 456 2024-08-17T16:59:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 456 (ObjNum 490 snap 155 seq_num 490) dirty exists 2024-08-17T16:59:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:1245: left oid 18 (ObjNum 490 snap 155 seq_num 490) 2024-08-17T16:59:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:1243: done (3 left) 2024-08-17T16:59:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:1244: done (2 left) 2024-08-17T16:59:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:1245: done (1 left) 2024-08-17T16:59:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:1246: done (0 left) 2024-08-17T16:59:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:1247: write oid 4 current snap is 155 2024-08-17T16:59:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:1247: seq_num 491 ranges {652237=793851,2198894=53343} 2024-08-17T16:59:58.684 INFO:tasks.rados.rados.0.smithi063.stdout:1247: writing smithi06387500-4 from 652237 to 1446088 tid 1 2024-08-17T16:59:58.689 INFO:tasks.rados.rados.0.smithi063.stdout:1247: writing smithi06387500-4 from 2198894 to 2252237 tid 2 2024-08-17T16:59:58.689 INFO:tasks.rados.rados.0.smithi063.stdout:1247: finishing write tid 1 to smithi06387500-4 2024-08-17T16:59:58.689 INFO:tasks.rados.rados.0.smithi063.stdout:1248: snap_create 2024-08-17T16:59:58.691 INFO:tasks.rados.rados.0.smithi063.stdout:1247: finishing write tid 2 to smithi06387500-4 2024-08-17T16:59:58.691 INFO:tasks.rados.rados.0.smithi063.stdout:1247: finishing write tid 3 to smithi06387500-4 2024-08-17T16:59:58.692 INFO:tasks.rados.rados.0.smithi063.stdout:1247: finishing write tid 4 to smithi06387500-4 2024-08-17T16:59:58.692 INFO:tasks.rados.rados.0.smithi063.stdout:1247: oid 4 updating version 0 to 1270 2024-08-17T16:59:58.692 INFO:tasks.rados.rados.0.smithi063.stdout:1247: oid 4 updating version 1270 to 1272 2024-08-17T16:59:58.692 INFO:tasks.rados.rados.0.smithi063.stdout:1247: oid 4 version 1272 is already newer than 1271 2024-08-17T16:59:58.692 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 1272 (ObjNum 491 snap 155 seq_num 491) dirty exists 2024-08-17T16:59:58.692 INFO:tasks.rados.rados.0.smithi063.stdout:1247: left oid 4 (ObjNum 491 snap 155 seq_num 491) 2024-08-17T16:59:58.814 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:58 smithi063 ceph-mon[109843]: Upgrade: Updating mon.c 2024-08-17T16:59:58.814 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:58 smithi063 ceph-mon[109843]: Deploying daemon mon.c on smithi063 2024-08-17T16:59:58.814 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 16:59:58 smithi063 ceph-mon[109843]: osdmap e379: 8 total, 8 up, 8 in 2024-08-17T16:59:58.815 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:58 smithi063 ceph-mon[42680]: Upgrade: Updating mon.c 2024-08-17T16:59:58.815 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:58 smithi063 ceph-mon[42680]: Deploying daemon mon.c on smithi063 2024-08-17T16:59:58.815 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:58 smithi063 ceph-mon[42680]: osdmap e379: 8 total, 8 up, 8 in 2024-08-17T16:59:58.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:58 smithi162 ceph-mon[34633]: Upgrade: Updating mon.c 2024-08-17T16:59:58.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:58 smithi162 ceph-mon[34633]: Deploying daemon mon.c on smithi063 2024-08-17T16:59:58.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 16:59:58 smithi162 ceph-mon[34633]: osdmap e379: 8 total, 8 up, 8 in 2024-08-17T16:59:59.163 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:58 smithi063 systemd[1]: Stopping Ceph mon.c for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T16:59:59.503 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:59 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c[42657]: 2024-08-17T16:59:59.161+0000 7f18b9e2c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T16:59:59.503 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:59 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c[42657]: 2024-08-17T16:59:59.161+0000 7f18b9e2c640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-08-17T16:59:59.503 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:59 smithi063 podman[114563]: 2024-08-17 16:59:59.243299793 +0000 UTC m=+0.277626955 container died b3f2c9339c6b2232a39c4cb019d7ffdc58d13951213a6b62caed65f320581d3c (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, RELEASE=reef-ced649f, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0) 2024-08-17T16:59:59.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:59 smithi063 podman[114563]: 2024-08-17 16:59:59.600905014 +0000 UTC m=+0.635232175 container cleanup b3f2c9339c6b2232a39c4cb019d7ffdc58d13951213a6b62caed65f320581d3c (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2) 2024-08-17T16:59:59.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:59 smithi063 bash[114563]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c 2024-08-17T17:00:00.246 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 16:59:59 smithi063 podman[114583]: 2024-08-17 16:59:59.88612524 +0000 UTC m=+0.639512184 container remove b3f2c9339c6b2232a39c4cb019d7ffdc58d13951213a6b62caed65f320581d3c (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-08-17T17:00:00.247 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.c.service: Deactivated successfully. 2024-08-17T17:00:00.247 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 systemd[1]: Stopped Ceph mon.c for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:00:00.247 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.c.service: Consumed 9.277s CPU time. 2024-08-17T17:00:00.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 systemd[1]: Starting Ceph mon.c for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:00:00.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 podman[114719]: 2024-08-17 17:00:00.462223817 +0000 UTC m=+0.110773966 container create c484af026acfb0ab96471e564a355dbb89606b66094497857f1ee222e49e7dbc (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-cd8bdbe) 2024-08-17T17:00:00.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 podman[114719]: 2024-08-17 17:00:00.369854527 +0000 UTC m=+0.018404692 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:00:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[109843]: pgmap v21: 89 pgs: 89 active+clean; 205 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.5 MiB/s wr, 28 op/s 2024-08-17T17:00:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[109843]: osdmap e380: 8 total, 8 up, 8 in 2024-08-17T17:00:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 podman[114719]: 2024-08-17 17:00:00.706725291 +0000 UTC m=+0.355275444 container init c484af026acfb0ab96471e564a355dbb89606b66094497857f1ee222e49e7dbc (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.36.0) 2024-08-17T17:00:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 podman[114719]: 2024-08-17 17:00:00.711076914 +0000 UTC m=+0.359627068 container start c484af026acfb0ab96471e564a355dbb89606b66094497857f1ee222e49e7dbc (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-08-17T17:00:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:00:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev), process ceph-mon, pid 7 2024-08-17T17:00:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: pidfile_write: ignore empty --pid-file 2024-08-17T17:00:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: load: jerasure load: lrc 2024-08-17T17:00:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: RocksDB version: 7.9.2 2024-08-17T17:00:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Git sha 0 2024-08-17T17:00:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Compile date 2024-08-15 20:43:31 2024-08-17T17:00:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: DB SUMMARY 2024-08-17T17:00:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: DB Session ID: Z8Z32OD4AQ5KGJH2IA8U 2024-08-17T17:00:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:00:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:00:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1655 Bytes 2024-08-17T17:00:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000030.sst 2024-08-17T17:00:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000028.log size: 78197 ; 2024-08-17T17:00:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:00:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:00:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:00:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T17:00:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:00:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T17:00:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.env: 0x55e09c01cbe0 2024-08-17T17:00:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T17:00:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.info_log: 0x55e09ded5700 2024-08-17T17:00:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:00:01.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.statistics: (nil) 2024-08-17T17:00:01.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.use_fsync: 0 2024-08-17T17:00:01.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:00:01.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:00:01.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:00:01.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:00:01.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:00:01.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:00:01.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:00:01.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:00:01.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:00:01.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:00:01.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:00:01.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.db_log_dir: 2024-08-17T17:00:01.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.wal_dir: 2024-08-17T17:00:01.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:00:01.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:00:01.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:00:01.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:00:01.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:00:01.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:00:01.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:00:01.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:00:01.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.write_buffer_manager: 0x55e09ded8960 2024-08-17T17:00:01.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:00:01.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:00:01.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:00:01.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:00:01.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:00:01.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:00:01.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:00:01.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:00:01.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.unordered_write: 0 2024-08-17T17:00:01.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:00:01.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:00:01.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:00:01.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:00:01.036 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.row_cache: None 2024-08-17T17:00:01.036 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.wal_filter: None 2024-08-17T17:00:01.036 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:00:01.036 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:00:01.036 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:00:01.036 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:00:01.036 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.wal_compression: 0 2024-08-17T17:00:01.036 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:00:01.036 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:00:01.036 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:00:01.036 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:00:01.036 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:00:01.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:00:01.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:00:01.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:00:01.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:00:01.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:00:01.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:00:01.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T17:00:01.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:00:01.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:00:01.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:00:01.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:00:01.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:00:01.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:00:01.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:00:01.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:00:01.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:00:01.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:00:01.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:00:01.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_open_files: -1 2024-08-17T17:00:01.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:00:01.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:00:01.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:00:01.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:00:01.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:00:01.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Compression algorithms supported: 2024-08-17T17:00:01.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: kZSTD supported: 0 2024-08-17T17:00:01.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:00:01.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:00:01.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:00:01.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:00:01.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:00:01.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:00:01.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:00:01.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:00:01.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T17:00:01.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 2024-08-17T17:00:01.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T17:00:01.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:00:01.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.merge_operator: 2024-08-17T17:00:01.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compaction_filter: None 2024-08-17T17:00:01.040 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:00:01.040 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:00:01.040 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:00:01.040 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:00:01.040 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55e09ded5380) 2024-08-17T17:00:01.040 INFO:journalctl@ceph.mon.c.smithi063.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:00:01.040 INFO:journalctl@ceph.mon.c.smithi063.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:00:01.040 INFO:journalctl@ceph.mon.c.smithi063.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:00:01.040 INFO:journalctl@ceph.mon.c.smithi063.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:00:01.040 INFO:journalctl@ceph.mon.c.smithi063.stdout: index_type: 0 2024-08-17T17:00:01.040 INFO:journalctl@ceph.mon.c.smithi063.stdout: data_block_index_type: 0 2024-08-17T17:00:01.040 INFO:journalctl@ceph.mon.c.smithi063.stdout: index_shortening: 1 2024-08-17T17:00:01.040 INFO:journalctl@ceph.mon.c.smithi063.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:00:01.040 INFO:journalctl@ceph.mon.c.smithi063.stdout: checksum: 4 2024-08-17T17:00:01.041 INFO:journalctl@ceph.mon.c.smithi063.stdout: no_block_cache: 0 2024-08-17T17:00:01.041 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_cache: 0x55e09dec9610 2024-08-17T17:00:01.041 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:00:01.041 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_cache_options: 2024-08-17T17:00:01.041 INFO:journalctl@ceph.mon.c.smithi063.stdout: capacity : 536870912 2024-08-17T17:00:01.041 INFO:journalctl@ceph.mon.c.smithi063.stdout: num_shard_bits : 4 2024-08-17T17:00:01.041 INFO:journalctl@ceph.mon.c.smithi063.stdout: strict_capacity_limit : 0 2024-08-17T17:00:01.041 INFO:journalctl@ceph.mon.c.smithi063.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:00:01.041 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_cache_compressed: (nil) 2024-08-17T17:00:01.041 INFO:journalctl@ceph.mon.c.smithi063.stdout: persistent_cache: (nil) 2024-08-17T17:00:01.041 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_size: 4096 2024-08-17T17:00:01.041 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_size_deviation: 10 2024-08-17T17:00:01.041 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_restart_interval: 16 2024-08-17T17:00:01.041 INFO:journalctl@ceph.mon.c.smithi063.stdout: index_block_restart_interval: 1 2024-08-17T17:00:01.041 INFO:journalctl@ceph.mon.c.smithi063.stdout: metadata_block_size: 4096 2024-08-17T17:00:01.042 INFO:journalctl@ceph.mon.c.smithi063.stdout: partition_filters: 0 2024-08-17T17:00:01.042 INFO:journalctl@ceph.mon.c.smithi063.stdout: use_delta_encoding: 1 2024-08-17T17:00:01.042 INFO:journalctl@ceph.mon.c.smithi063.stdout: filter_policy: bloomfilter 2024-08-17T17:00:01.042 INFO:journalctl@ceph.mon.c.smithi063.stdout: whole_key_filtering: 1 2024-08-17T17:00:01.042 INFO:journalctl@ceph.mon.c.smithi063.stdout: verify_compression: 0 2024-08-17T17:00:01.042 INFO:journalctl@ceph.mon.c.smithi063.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:00:01.042 INFO:journalctl@ceph.mon.c.smithi063.stdout: format_version: 5 2024-08-17T17:00:01.042 INFO:journalctl@ceph.mon.c.smithi063.stdout: enable_index_compression: 1 2024-08-17T17:00:01.042 INFO:journalctl@ceph.mon.c.smithi063.stdout: block_align: 0 2024-08-17T17:00:01.042 INFO:journalctl@ceph.mon.c.smithi063.stdout: max_auto_readahead_size: 262144 2024-08-17T17:00:01.042 INFO:journalctl@ceph.mon.c.smithi063.stdout: prepopulate_block_cache: 0 2024-08-17T17:00:01.042 INFO:journalctl@ceph.mon.c.smithi063.stdout: initial_auto_readahead_size: 8192 2024-08-17T17:00:01.042 INFO:journalctl@ceph.mon.c.smithi063.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T17:00:01.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:00:01.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:00:01.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compression: NoCompression 2024-08-17T17:00:01.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:00:01.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:00:01.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:00:01.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.num_levels: 7 2024-08-17T17:00:01.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:00:01.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:00:01.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:00:01.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:00:01.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:00:01.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:00:01.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:00:01.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:00:01.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:00:01.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:00:01.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:00:01.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:00:01.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:00:01.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:00:01.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:00:01.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:00:01.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:00:01.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:00:01.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:00:01.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:00:01.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:00:01.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:00:01.045 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:00:01.045 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:00:01.045 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:00:01.045 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:00:01.045 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:00:01.045 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:00:01.045 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:00:01.045 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:00:01.046 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:00:01.046 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:00:01.046 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:00:01.046 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:00:01.046 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:00:01.046 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:00:01.046 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:00:01.047 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:00:01.047 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T17:00:01.047 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T17:00:01.047 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:00:01.047 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:00:01.047 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:00:01.047 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:00:01.047 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:00:01.047 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:00:01.047 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:00:01.047 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:00:01.047 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:00:01.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:00:01.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:00:01.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:00:01.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:00:01.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T17:00:01.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:00:01.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:00:01.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:00:01.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:00:01.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:00:01.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:00:01.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:00:01.049 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:00:01.049 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:00:01.049 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:00:01.049 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:00:01.049 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.ttl: 2592000 2024-08-17T17:00:01.049 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:00:01.049 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T17:00:01.049 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T17:00:01.049 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:00:01.049 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:00:01.049 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:00:01.049 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:00:01.050 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:00:01.050 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:00:01.050 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T17:00:01.050 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T17:00:01.050 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T17:00:01.050 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T17:00:01.050 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:00:01.050 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 32, last_sequence is 19552, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 28 2024-08-17T17:00:01.050 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-08-17T17:00:01.050 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 71e19b9a-502e-47b0-bdec-39d9641c622f 2024-08-17T17:00:01.050 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914000739644, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-08-17T17:00:01.050 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-08-17T17:00:01.051 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914000740598, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 52973, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 19557, "largest_seqno": 19592, "table_properties": {"data_size": 51713, "index_size": 160, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 538, "raw_average_key_size": 23, "raw_value_size": 51099, "raw_average_value_size": 2221, "num_data_blocks": 7, "num_entries": 23, "num_filter_entries": 23, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723914000, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "71e19b9a-502e-47b0-bdec-39d9641c622f", "db_session_id": "Z8Z32OD4AQ5KGJH2IA8U", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-08-17T17:00:01.051 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914000740738, "job": 1, "event": "recovery_finished"} 2024-08-17T17:00:01.051 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-08-17T17:00:01.051 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:00:01.051 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:00:01.051 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55e09def8e00 2024-08-17T17:00:01.051 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: DB pointer 0x55e09dff6000 2024-08-17T17:00:01.051 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T17:00:01.051 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T17:00:01.051 INFO:journalctl@ceph.mon.c.smithi063.stdout: ** DB Stats ** 2024-08-17T17:00:01.051 INFO:journalctl@ceph.mon.c.smithi063.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:00:01.051 INFO:journalctl@ceph.mon.c.smithi063.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:00:01.051 INFO:journalctl@ceph.mon.c.smithi063.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:00:01.052 INFO:journalctl@ceph.mon.c.smithi063.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:00:01.052 INFO:journalctl@ceph.mon.c.smithi063.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:00:01.052 INFO:journalctl@ceph.mon.c.smithi063.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:00:01.052 INFO:journalctl@ceph.mon.c.smithi063.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:00:01.052 INFO:journalctl@ceph.mon.c.smithi063.stdout: 2024-08-17T17:00:01.052 INFO:journalctl@ceph.mon.c.smithi063.stdout: ** Compaction Stats [default] ** 2024-08-17T17:00:01.052 INFO:journalctl@ceph.mon.c.smithi063.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:00:01.052 INFO:journalctl@ceph.mon.c.smithi063.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T17:00:01.052 INFO:journalctl@ceph.mon.c.smithi063.stdout: L0 1/0 51.73 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 84.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T17:00:01.052 INFO:journalctl@ceph.mon.c.smithi063.stdout: L6 1/0 12.78 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-17T17:00:01.052 INFO:journalctl@ceph.mon.c.smithi063.stdout: Sum 2/0 12.83 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 84.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T17:00:01.052 INFO:journalctl@ceph.mon.c.smithi063.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 84.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T17:00:01.053 INFO:journalctl@ceph.mon.c.smithi063.stdout: 2024-08-17T17:00:01.053 INFO:journalctl@ceph.mon.c.smithi063.stdout: ** Compaction Stats [default] ** 2024-08-17T17:00:01.053 INFO:journalctl@ceph.mon.c.smithi063.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:00:01.053 INFO:journalctl@ceph.mon.c.smithi063.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:00:01.053 INFO:journalctl@ceph.mon.c.smithi063.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 84.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T17:00:01.053 INFO:journalctl@ceph.mon.c.smithi063.stdout: 2024-08-17T17:00:01.053 INFO:journalctl@ceph.mon.c.smithi063.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T17:00:01.053 INFO:journalctl@ceph.mon.c.smithi063.stdout: 2024-08-17T17:00:01.053 INFO:journalctl@ceph.mon.c.smithi063.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:00:01.053 INFO:journalctl@ceph.mon.c.smithi063.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T17:00:01.053 INFO:journalctl@ceph.mon.c.smithi063.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:00:01.053 INFO:journalctl@ceph.mon.c.smithi063.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:00:01.053 INFO:journalctl@ceph.mon.c.smithi063.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:00:01.054 INFO:journalctl@ceph.mon.c.smithi063.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:00:01.054 INFO:journalctl@ceph.mon.c.smithi063.stdout: Cumulative compaction: 0.00 GB write, 12.12 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:00:01.054 INFO:journalctl@ceph.mon.c.smithi063.stdout: Interval compaction: 0.00 GB write, 12.12 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:00:01.054 INFO:journalctl@ceph.mon.c.smithi063.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:00:01.054 INFO:journalctl@ceph.mon.c.smithi063.stdout: Block cache BinnedLRUCache@0x55e09dec9610#7 capacity: 512.00 MB usage: 0.34 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-08-17T17:00:01.054 INFO:journalctl@ceph.mon.c.smithi063.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.23 KB,4.47035e-05%) Misc(1,0.00 KB,0%) 2024-08-17T17:00:01.054 INFO:journalctl@ceph.mon.c.smithi063.stdout: 2024-08-17T17:00:01.054 INFO:journalctl@ceph.mon.c.smithi063.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:00:01.054 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: starting mon.c rank 2 at public addrs [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] at bind addrs [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:00:01.054 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: mon.c@-1(???) e3 preinit fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:00:01.054 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: mon.c@-1(???).mds e8 new map 2024-08-17T17:00:01.054 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: mon.c@-1(???).mds e8 print_map 2024-08-17T17:00:01.055 INFO:journalctl@ceph.mon.c.smithi063.stdout: e8 2024-08-17T17:00:01.055 INFO:journalctl@ceph.mon.c.smithi063.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-17T17:00:01.055 INFO:journalctl@ceph.mon.c.smithi063.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:00:01.055 INFO:journalctl@ceph.mon.c.smithi063.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:00:01.055 INFO:journalctl@ceph.mon.c.smithi063.stdout: legacy client fscid: 1 2024-08-17T17:00:01.055 INFO:journalctl@ceph.mon.c.smithi063.stdout: 2024-08-17T17:00:01.055 INFO:journalctl@ceph.mon.c.smithi063.stdout: Filesystem 'foo' (1) 2024-08-17T17:00:01.055 INFO:journalctl@ceph.mon.c.smithi063.stdout: fs_name foo 2024-08-17T17:00:01.055 INFO:journalctl@ceph.mon.c.smithi063.stdout: epoch 7 2024-08-17T17:00:01.055 INFO:journalctl@ceph.mon.c.smithi063.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-08-17T17:00:01.055 INFO:journalctl@ceph.mon.c.smithi063.stdout: created 2024-08-17T16:53:36.880465+0000 2024-08-17T17:00:01.055 INFO:journalctl@ceph.mon.c.smithi063.stdout: modified 2024-08-17T16:53:54.414788+0000 2024-08-17T17:00:01.055 INFO:journalctl@ceph.mon.c.smithi063.stdout: tableserver 0 2024-08-17T17:00:01.056 INFO:journalctl@ceph.mon.c.smithi063.stdout: root 0 2024-08-17T17:00:01.056 INFO:journalctl@ceph.mon.c.smithi063.stdout: session_timeout 60 2024-08-17T17:00:01.056 INFO:journalctl@ceph.mon.c.smithi063.stdout: session_autoclose 300 2024-08-17T17:00:01.056 INFO:journalctl@ceph.mon.c.smithi063.stdout: max_file_size 1099511627776 2024-08-17T17:00:01.056 INFO:journalctl@ceph.mon.c.smithi063.stdout: max_xattr_size 65536 2024-08-17T17:00:01.056 INFO:journalctl@ceph.mon.c.smithi063.stdout: required_client_features {} 2024-08-17T17:00:01.056 INFO:journalctl@ceph.mon.c.smithi063.stdout: last_failure 0 2024-08-17T17:00:01.056 INFO:journalctl@ceph.mon.c.smithi063.stdout: last_failure_osd_epoch 0 2024-08-17T17:00:01.056 INFO:journalctl@ceph.mon.c.smithi063.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:00:01.056 INFO:journalctl@ceph.mon.c.smithi063.stdout: max_mds 1 2024-08-17T17:00:01.056 INFO:journalctl@ceph.mon.c.smithi063.stdout: in 0 2024-08-17T17:00:01.056 INFO:journalctl@ceph.mon.c.smithi063.stdout: up {0=14658} 2024-08-17T17:00:01.057 INFO:journalctl@ceph.mon.c.smithi063.stdout: failed 2024-08-17T17:00:01.057 INFO:journalctl@ceph.mon.c.smithi063.stdout: damaged 2024-08-17T17:00:01.057 INFO:journalctl@ceph.mon.c.smithi063.stdout: stopped 2024-08-17T17:00:01.057 INFO:journalctl@ceph.mon.c.smithi063.stdout: data_pools [4] 2024-08-17T17:00:01.057 INFO:journalctl@ceph.mon.c.smithi063.stdout: metadata_pool 3 2024-08-17T17:00:01.057 INFO:journalctl@ceph.mon.c.smithi063.stdout: inline_data disabled 2024-08-17T17:00:01.057 INFO:journalctl@ceph.mon.c.smithi063.stdout: balancer 2024-08-17T17:00:01.057 INFO:journalctl@ceph.mon.c.smithi063.stdout: bal_rank_mask -1 2024-08-17T17:00:01.057 INFO:journalctl@ceph.mon.c.smithi063.stdout: standby_count_wanted 1 2024-08-17T17:00:01.057 INFO:journalctl@ceph.mon.c.smithi063.stdout: qdb_cluster leader: 0 members: 2024-08-17T17:00:01.057 INFO:journalctl@ceph.mon.c.smithi063.stdout: [mds.foo.smithi063.ultpqj{0:14658} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.63:6834/3222014296,v1:172.21.15.63:6835/3222014296] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:00:01.057 INFO:journalctl@ceph.mon.c.smithi063.stdout: 2024-08-17T17:00:01.057 INFO:journalctl@ceph.mon.c.smithi063.stdout: 2024-08-17T17:00:01.057 INFO:journalctl@ceph.mon.c.smithi063.stdout: Standby daemons: 2024-08-17T17:00:01.058 INFO:journalctl@ceph.mon.c.smithi063.stdout: 2024-08-17T17:00:01.058 INFO:journalctl@ceph.mon.c.smithi063.stdout: [mds.foo.smithi162.bjkxnv{-1:24529} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.162:6832/2161520737,v1:172.21.15.162:6833/2161520737] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:00:01.058 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: mon.c@-1(???).osd e380 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-17T17:00:01.058 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: mon.c@-1(???).osd e380 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:00:01.058 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: mon.c@-1(???).osd e380 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:00:01.058 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: mon.c@-1(???).osd e380 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:00:01.058 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-08-17T17:00:01.058 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 bash[114719]: c484af026acfb0ab96471e564a355dbb89606b66094497857f1ee222e49e7dbc 2024-08-17T17:00:01.058 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 systemd[1]: Started Ceph mon.c for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:00:01.058 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: pgmap v21: 89 pgs: 89 active+clean; 205 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.5 MiB/s wr, 28 op/s 2024-08-17T17:00:01.058 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:00 smithi063 ceph-mon[114756]: osdmap e380: 8 total, 8 up, 8 in 2024-08-17T17:00:01.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:00 smithi162 ceph-mon[34633]: pgmap v21: 89 pgs: 89 active+clean; 205 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 3.5 MiB/s wr, 28 op/s 2024-08-17T17:00:01.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:00 smithi162 ceph-mon[34633]: osdmap e380: 8 total, 8 up, 8 in 2024-08-17T17:00:01.946 INFO:tasks.rados.rados.0.smithi063.stdout:1247: done (1 left) 2024-08-17T17:00:01.946 INFO:tasks.rados.rados.0.smithi063.stdout:1248: done (0 left) 2024-08-17T17:00:01.946 INFO:tasks.rados.rados.0.smithi063.stdout:1249: snap_remove snap 150 2024-08-17T17:00:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: pgmap v23: 89 pgs: 89 active+clean; 205 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 3.2 MiB/s wr, 26 op/s 2024-08-17T17:00:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: overall HEALTH_OK 2024-08-17T17:00:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: mon.c calling monitor election 2024-08-17T17:00:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: mon.a calling monitor election 2024-08-17T17:00:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:00:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: monmap epoch 3 2024-08-17T17:00:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:00:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: last_changed 2024-08-17T16:48:24.893441+0000 2024-08-17T17:00:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: created 2024-08-17T16:46:51.197040+0000 2024-08-17T17:00:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: min_mon_release 18 (reef) 2024-08-17T17:00:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: election_strategy: 1 2024-08-17T17:00:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: 0: [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon.a 2024-08-17T17:00:02.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: 1: [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] mon.b 2024-08-17T17:00:02.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: 2: [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] mon.c 2024-08-17T17:00:02.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T17:00:02.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: osdmap e380: 8 total, 8 up, 8 in 2024-08-17T17:00:02.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: mgrmap e39: y(active, since 15s), standbys: x 2024-08-17T17:00:02.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: overall HEALTH_OK 2024-08-17T17:00:02.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:02.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:02.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:02.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:02.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:02.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: pgmap v23: 89 pgs: 89 active+clean; 205 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 3.2 MiB/s wr, 26 op/s 2024-08-17T17:00:02.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: overall HEALTH_OK 2024-08-17T17:00:02.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: mon.c calling monitor election 2024-08-17T17:00:02.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: mon.a calling monitor election 2024-08-17T17:00:02.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:00:02.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: monmap epoch 3 2024-08-17T17:00:02.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:00:02.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: last_changed 2024-08-17T16:48:24.893441+0000 2024-08-17T17:00:02.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: created 2024-08-17T16:46:51.197040+0000 2024-08-17T17:00:02.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: min_mon_release 18 (reef) 2024-08-17T17:00:02.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: election_strategy: 1 2024-08-17T17:00:02.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: 0: [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon.a 2024-08-17T17:00:02.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: 1: [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] mon.b 2024-08-17T17:00:02.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: 2: [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] mon.c 2024-08-17T17:00:02.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T17:00:02.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: osdmap e380: 8 total, 8 up, 8 in 2024-08-17T17:00:02.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: mgrmap e39: y(active, since 15s), standbys: x 2024-08-17T17:00:02.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: overall HEALTH_OK 2024-08-17T17:00:02.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:02.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:02.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:02.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:02.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: pgmap v23: 89 pgs: 89 active+clean; 205 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 3.2 MiB/s wr, 26 op/s 2024-08-17T17:00:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: overall HEALTH_OK 2024-08-17T17:00:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: mon.c calling monitor election 2024-08-17T17:00:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: mon.a calling monitor election 2024-08-17T17:00:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:00:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: monmap epoch 3 2024-08-17T17:00:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:00:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: last_changed 2024-08-17T16:48:24.893441+0000 2024-08-17T17:00:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: created 2024-08-17T16:46:51.197040+0000 2024-08-17T17:00:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: min_mon_release 18 (reef) 2024-08-17T17:00:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: election_strategy: 1 2024-08-17T17:00:02.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: 0: [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon.a 2024-08-17T17:00:02.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: 1: [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] mon.b 2024-08-17T17:00:02.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: 2: [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] mon.c 2024-08-17T17:00:02.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T17:00:02.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: osdmap e380: 8 total, 8 up, 8 in 2024-08-17T17:00:02.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: mgrmap e39: y(active, since 15s), standbys: x 2024-08-17T17:00:02.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: overall HEALTH_OK 2024-08-17T17:00:02.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:02.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:02.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:02.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:02.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:01 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:02.504 INFO:tasks.rados.rados.0.smithi063.stdout:1249: done (0 left) 2024-08-17T17:00:02.504 INFO:tasks.rados.rados.0.smithi063.stdout:1250: read oid 18 snap -1 2024-08-17T17:00:02.504 INFO:tasks.rados.rados.0.smithi063.stdout:1250: expect (ObjNum 490 snap 155 seq_num 490) 2024-08-17T17:00:02.504 INFO:tasks.rados.rados.0.smithi063.stdout:1251: snap_remove snap 125 2024-08-17T17:00:03.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:02 smithi063 ceph-mon[114756]: pgmap v24: 89 pgs: 89 active+clean; 207 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 8.6 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2024-08-17T17:00:03.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:02 smithi063 ceph-mon[114756]: osdmap e381: 8 total, 8 up, 8 in 2024-08-17T17:00:03.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:02 smithi063 ceph-mon[114756]: osdmap e382: 8 total, 8 up, 8 in 2024-08-17T17:00:03.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:02 smithi063 ceph-mon[109843]: pgmap v24: 89 pgs: 89 active+clean; 207 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 8.6 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2024-08-17T17:00:03.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:02 smithi063 ceph-mon[109843]: osdmap e381: 8 total, 8 up, 8 in 2024-08-17T17:00:03.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:02 smithi063 ceph-mon[109843]: osdmap e382: 8 total, 8 up, 8 in 2024-08-17T17:00:03.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:02 smithi162 ceph-mon[34633]: pgmap v24: 89 pgs: 89 active+clean; 207 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 8.6 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2024-08-17T17:00:03.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:02 smithi162 ceph-mon[34633]: osdmap e381: 8 total, 8 up, 8 in 2024-08-17T17:00:03.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:02 smithi162 ceph-mon[34633]: osdmap e382: 8 total, 8 up, 8 in 2024-08-17T17:00:03.508 INFO:tasks.rados.rados.0.smithi063.stdout:1251: done (1 left) 2024-08-17T17:00:03.508 INFO:tasks.rados.rados.0.smithi063.stdout:1252: rollback oid 32 current snap is 156 2024-08-17T17:00:03.508 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 32 to 146 2024-08-17T17:00:03.508 INFO:tasks.rados.rados.0.smithi063.stdout:1250: expect (ObjNum 490 snap 155 seq_num 490) 2024-08-17T17:00:03.707 INFO:tasks.rados.rados.0.smithi063.stdout:1252: finishing rollback tid 1 to smithi06387500-32 2024-08-17T17:00:03.707 INFO:tasks.rados.rados.0.smithi063.stdout:1252: finishing rollback tid 2 to smithi06387500-32 2024-08-17T17:00:03.707 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 244 (ObjNum 423 snap 130 seq_num 423) dirty exists 2024-08-17T17:00:03.707 INFO:tasks.rados.rados.0.smithi063.stdout:1250: done (1 left) 2024-08-17T17:00:03.707 INFO:tasks.rados.rados.0.smithi063.stdout:1252: done (0 left) 2024-08-17T17:00:03.707 INFO:tasks.rados.rados.0.smithi063.stdout:1253: write oid 37 current snap is 156 2024-08-17T17:00:03.707 INFO:tasks.rados.rados.0.smithi063.stdout:1253: seq_num 492 ranges {669044=469100,1741023=459161} 2024-08-17T17:00:03.713 INFO:tasks.rados.rados.0.smithi063.stdout:1253: writing smithi06387500-37 from 669044 to 1138144 tid 1 2024-08-17T17:00:03.719 INFO:tasks.rados.rados.0.smithi063.stdout:1253: writing smithi06387500-37 from 1741023 to 2200184 tid 2 2024-08-17T17:00:03.719 INFO:tasks.rados.rados.0.smithi063.stdout:1254: write oid 39 current snap is 156 2024-08-17T17:00:03.719 INFO:tasks.rados.rados.0.smithi063.stdout:1253: finishing write tid 1 to smithi06387500-37 2024-08-17T17:00:03.719 INFO:tasks.rados.rados.0.smithi063.stdout:1254: seq_num 493 ranges {685851=544349,1799505=486346} 2024-08-17T17:00:03.725 INFO:tasks.rados.rados.0.smithi063.stdout:1254: writing smithi06387500-39 from 685851 to 1230200 tid 1 2024-08-17T17:00:03.732 INFO:tasks.rados.rados.0.smithi063.stdout:1254: writing smithi06387500-39 from 1799505 to 2285851 tid 2 2024-08-17T17:00:03.732 INFO:tasks.rados.rados.0.smithi063.stdout:1255: delete oid 38 current snap is 156 2024-08-17T17:00:03.732 INFO:tasks.rados.rados.0.smithi063.stdout:1253: finishing write tid 2 to smithi06387500-37 2024-08-17T17:00:03.732 INFO:tasks.rados.rados.0.smithi063.stdout:1253: finishing write tid 3 to smithi06387500-37 2024-08-17T17:00:03.732 INFO:tasks.rados.rados.0.smithi063.stdout:1253: finishing write tid 4 to smithi06387500-37 2024-08-17T17:00:03.732 INFO:tasks.rados.rados.0.smithi063.stdout:1253: oid 37 updating version 0 to 1289 2024-08-17T17:00:03.732 INFO:tasks.rados.rados.0.smithi063.stdout:1253: oid 37 version 1289 is already newer than 1288 2024-08-17T17:00:03.732 INFO:tasks.rados.rados.0.smithi063.stdout:1253: oid 37 version 1289 is already newer than 1287 2024-08-17T17:00:03.732 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 1289 (ObjNum 492 snap 156 seq_num 492) dirty exists 2024-08-17T17:00:03.732 INFO:tasks.rados.rados.0.smithi063.stdout:1253: left oid 37 (ObjNum 492 snap 156 seq_num 492) 2024-08-17T17:00:03.732 INFO:tasks.rados.rados.0.smithi063.stdout:1254: finishing write tid 1 to smithi06387500-39 2024-08-17T17:00:03.733 INFO:tasks.rados.rados.0.smithi063.stdout:1253: done (2 left) 2024-08-17T17:00:03.734 INFO:tasks.rados.rados.0.smithi063.stdout:1255: done (1 left) 2024-08-17T17:00:03.734 INFO:tasks.rados.rados.0.smithi063.stdout:1256: rollback oid 25 current snap is 156 2024-08-17T17:00:03.734 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 25 to 155 2024-08-17T17:00:03.734 INFO:tasks.rados.rados.0.smithi063.stdout:1257: read oid 7 snap -1 2024-08-17T17:00:03.734 INFO:tasks.rados.rados.0.smithi063.stdout:1257: expect (ObjNum 454 snap 147 seq_num 454) 2024-08-17T17:00:03.734 INFO:tasks.rados.rados.0.smithi063.stdout:1258: delete oid 33 current snap is 156 2024-08-17T17:00:03.735 INFO:tasks.rados.rados.0.smithi063.stdout:1254: finishing write tid 2 to smithi06387500-39 2024-08-17T17:00:03.735 INFO:tasks.rados.rados.0.smithi063.stdout:1254: finishing write tid 3 to smithi06387500-39 2024-08-17T17:00:03.736 INFO:tasks.rados.rados.0.smithi063.stdout:1254: finishing write tid 4 to smithi06387500-39 2024-08-17T17:00:03.736 INFO:tasks.rados.rados.0.smithi063.stdout:1254: oid 39 updating version 0 to 318 2024-08-17T17:00:03.736 INFO:tasks.rados.rados.0.smithi063.stdout:1254: oid 39 updating version 318 to 319 2024-08-17T17:00:03.736 INFO:tasks.rados.rados.0.smithi063.stdout:1254: oid 39 updating version 319 to 320 2024-08-17T17:00:03.736 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 320 (ObjNum 493 snap 156 seq_num 493) dirty exists 2024-08-17T17:00:03.736 INFO:tasks.rados.rados.0.smithi063.stdout:1254: left oid 39 (ObjNum 493 snap 156 seq_num 493) 2024-08-17T17:00:03.736 INFO:tasks.rados.rados.0.smithi063.stdout:1254: done (3 left) 2024-08-17T17:00:03.736 INFO:tasks.rados.rados.0.smithi063.stdout:1258: done (2 left) 2024-08-17T17:00:03.736 INFO:tasks.rados.rados.0.smithi063.stdout:1259: write oid 21 current snap is 156 2024-08-17T17:00:03.736 INFO:tasks.rados.rados.0.smithi063.stdout:1259: seq_num 496 ranges {736272=486449,1974951=361321} 2024-08-17T17:00:03.742 INFO:tasks.rados.rados.0.smithi063.stdout:1259: writing smithi06387500-21 from 736272 to 1222721 tid 1 2024-08-17T17:00:03.748 INFO:tasks.rados.rados.0.smithi063.stdout:1259: writing smithi06387500-21 from 1974951 to 2336272 tid 2 2024-08-17T17:00:03.748 INFO:tasks.rados.rados.0.smithi063.stdout:1260: snap_create 2024-08-17T17:00:03.748 INFO:tasks.rados.rados.0.smithi063.stdout:1256: finishing rollback tid 0 to smithi06387500-25 2024-08-17T17:00:03.749 INFO:tasks.rados.rados.0.smithi063.stdout:1256: finishing rollback tid 1 to smithi06387500-25 2024-08-17T17:00:03.749 INFO:tasks.rados.rados.0.smithi063.stdout:1256: finishing rollback tid 2 to smithi06387500-25 2024-08-17T17:00:03.749 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 1293 (ObjNum 362 snap 108 seq_num 362) dirty exists 2024-08-17T17:00:03.749 INFO:tasks.rados.rados.0.smithi063.stdout:1259: finishing write tid 1 to smithi06387500-21 2024-08-17T17:00:03.749 INFO:tasks.rados.rados.0.smithi063.stdout:1257: expect (ObjNum 454 snap 147 seq_num 454) 2024-08-17T17:00:03.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:00:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:00:03] "GET /metrics HTTP/1.1" 200 36143 "" "Prometheus/2.51.0" 2024-08-17T17:00:03.872 INFO:tasks.rados.rados.0.smithi063.stdout:1259: finishing write tid 2 to smithi06387500-21 2024-08-17T17:00:03.872 INFO:tasks.rados.rados.0.smithi063.stdout:1259: finishing write tid 3 to smithi06387500-21 2024-08-17T17:00:03.872 INFO:tasks.rados.rados.0.smithi063.stdout:1259: finishing write tid 4 to smithi06387500-21 2024-08-17T17:00:03.872 INFO:tasks.rados.rados.0.smithi063.stdout:1259: oid 21 updating version 0 to 880 2024-08-17T17:00:03.872 INFO:tasks.rados.rados.0.smithi063.stdout:1259: oid 21 version 880 is already newer than 878 2024-08-17T17:00:03.873 INFO:tasks.rados.rados.0.smithi063.stdout:1259: oid 21 version 880 is already newer than 879 2024-08-17T17:00:03.873 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 880 (ObjNum 496 snap 156 seq_num 496) dirty exists 2024-08-17T17:00:03.873 INFO:tasks.rados.rados.0.smithi063.stdout:1259: left oid 21 (ObjNum 496 snap 156 seq_num 496) 2024-08-17T17:00:04.223 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T17:00:04.513 INFO:tasks.rados.rados.0.smithi063.stdout:1256: done (3 left) 2024-08-17T17:00:04.514 INFO:tasks.rados.rados.0.smithi063.stdout:1257: done (2 left) 2024-08-17T17:00:04.514 INFO:tasks.rados.rados.0.smithi063.stdout:1259: done (1 left) 2024-08-17T17:00:04.514 INFO:tasks.rados.rados.0.smithi063.stdout:1260: done (0 left) 2024-08-17T17:00:04.514 INFO:tasks.rados.rados.0.smithi063.stdout:1261: write oid 44 current snap is 157 2024-08-17T17:00:04.514 INFO:tasks.rados.rados.0.smithi063.stdout:1261: seq_num 497 ranges {753079=561698,2033433=319646} 2024-08-17T17:00:04.520 INFO:tasks.rados.rados.0.smithi063.stdout:1261: writing smithi06387500-44 from 753079 to 1314777 tid 1 2024-08-17T17:00:04.526 INFO:tasks.rados.rados.0.smithi063.stdout:1261: writing smithi06387500-44 from 2033433 to 2353079 tid 2 2024-08-17T17:00:04.527 INFO:tasks.rados.rados.0.smithi063.stdout:1262: read oid 49 snap 154 2024-08-17T17:00:04.527 INFO:tasks.rados.rados.0.smithi063.stdout:1262: expect (ObjNum 449 snap 145 seq_num 449) 2024-08-17T17:00:04.527 INFO:tasks.rados.rados.0.smithi063.stdout:1263: rollback oid 32 current snap is 157 2024-08-17T17:00:04.527 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 32 to 153 2024-08-17T17:00:04.527 INFO:tasks.rados.rados.0.smithi063.stdout:1264: read oid 5 snap -1 2024-08-17T17:00:04.527 INFO:tasks.rados.rados.0.smithi063.stdout:1264: expect (ObjNum 443 snap 139 seq_num 443) 2024-08-17T17:00:04.527 INFO:tasks.rados.rados.0.smithi063.stdout:1265: snap_remove snap 146 2024-08-17T17:00:04.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:04 smithi063 ceph-mon[114756]: osdmap e383: 8 total, 8 up, 8 in 2024-08-17T17:00:04.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:04 smithi063 ceph-mon[109843]: osdmap e383: 8 total, 8 up, 8 in 2024-08-17T17:00:04.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:04 smithi162 ceph-mon[34633]: osdmap e383: 8 total, 8 up, 8 in 2024-08-17T17:00:04.854 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:00:04.854 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (91s) 0s ago 7m 17.5M - 0.27.0 11f11916f8cd 59f4018b91ca 2024-08-17T17:00:04.854 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (68s) 13s ago 7m 72.9M - c8b91775d855 581e03e654a8 2024-08-17T17:00:04.854 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (6m) 0s ago 6m 19.1M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T17:00:04.854 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (6m) 13s ago 6m 17.7M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T17:00:04.854 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,9283,8765 running (2m) 13s ago 11m 452M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb da48034a2895 2024-08-17T17:00:04.854 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:8443,9283,8765 running (3m) 0s ago 13m 500M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 943e639cede6 2024-08-17T17:00:04.855 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (27s) 0s ago 13m 43.6M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb f0f20fcac956 2024-08-17T17:00:04.855 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (11m) 13s ago 11m 64.9M 2048M 18.2.4-719-gced649f8 c870753d67c2 d4ecbe65a0c8 2024-08-17T17:00:04.855 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (4s) 0s ago 11m 18.3M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb c484af026acf 2024-08-17T17:00:04.855 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 running (2m) 0s ago 6m 11.0M - 1.7.0 72c9c2088986 34f55f5de2aa 2024-08-17T17:00:04.855 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (2m) 13s ago 6m 9.77M - 1.7.0 72c9c2088986 5bb162d0f1a9 2024-08-17T17:00:04.855 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (11m) 0s ago 11m 100M 2480M 18.2.4-719-gced649f8 c870753d67c2 f3b910ed49b8 2024-08-17T17:00:04.855 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (10m) 0s ago 10m 79.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 5d6fdb6000ad 2024-08-17T17:00:04.855 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (10m) 0s ago 10m 96.2M 2480M 18.2.4-719-gced649f8 c870753d67c2 a3cbe8edb49a 2024-08-17T17:00:04.855 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (9m) 0s ago 9m 78.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T17:00:04.855 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (9m) 13s ago 9m 102M 2480M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T17:00:04.855 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (9m) 13s ago 9m 90.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T17:00:04.855 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (8m) 13s ago 8m 70.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T17:00:04.855 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (8m) 13s ago 8m 78.9M 2480M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T17:00:04.855 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (113s) 13s ago 7m 41.4M - 2.51.0 1d3b7f56885b dfe66b2c5568 2024-08-17T17:00:05.225 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:00:05.225 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T17:00:05.225 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 1, 2024-08-17T17:00:05.225 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:00:05.225 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:00:05.225 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T17:00:05.226 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:00:05.226 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:00:05.226 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T17:00:05.226 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:00:05.226 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:00:05.226 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T17:00:05.226 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:00:05.226 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:00:05.226 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T17:00:05.226 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 11, 2024-08-17T17:00:05.226 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 4 2024-08-17T17:00:05.226 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:00:05.226 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:00:05.509 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[109843]: pgmap v28: 89 pgs: 89 active+clean; 207 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 790 B/s rd, 728 KiB/s wr, 3 op/s 2024-08-17T17:00:05.509 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[109843]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:05.509 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[109843]: osdmap e384: 8 total, 8 up, 8 in 2024-08-17T17:00:05.509 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[109843]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:05.509 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:05.528 INFO:tasks.rados.rados.0.smithi063.stdout:1265: done (4 left) 2024-08-17T17:00:05.528 INFO:tasks.rados.rados.0.smithi063.stdout:1266: write oid 21 current snap is 157 2024-08-17T17:00:05.528 INFO:tasks.rados.rados.0.smithi063.stdout:1266: seq_num 498 ranges {769886=636947,2091915=277971} 2024-08-17T17:00:05.536 INFO:tasks.rados.rados.0.smithi063.stdout:1266: writing smithi06387500-21 from 769886 to 1406833 tid 1 2024-08-17T17:00:05.543 INFO:tasks.rados.rados.0.smithi063.stdout:1266: writing smithi06387500-21 from 2091915 to 2369886 tid 2 2024-08-17T17:00:05.543 INFO:tasks.rados.rados.0.smithi063.stdout:1267: rollback oid 31 current snap is 157 2024-08-17T17:00:05.543 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 31 to 156 2024-08-17T17:00:05.543 INFO:tasks.rados.rados.0.smithi063.stdout:1261: finishing write tid 1 to smithi06387500-44 2024-08-17T17:00:05.543 INFO:tasks.rados.rados.0.smithi063.stdout:1261: finishing write tid 2 to smithi06387500-44 2024-08-17T17:00:05.543 INFO:tasks.rados.rados.0.smithi063.stdout:1261: finishing write tid 3 to smithi06387500-44 2024-08-17T17:00:05.543 INFO:tasks.rados.rados.0.smithi063.stdout:1263: finishing rollback tid 0 to smithi06387500-32 2024-08-17T17:00:05.543 INFO:tasks.rados.rados.0.smithi063.stdout:1268: write oid 39 current snap is 157 2024-08-17T17:00:05.544 INFO:tasks.rados.rados.0.smithi063.stdout:1261: finishing write tid 4 to smithi06387500-44 2024-08-17T17:00:05.544 INFO:tasks.rados.rados.0.smithi063.stdout:1261: oid 44 updating version 0 to 246 2024-08-17T17:00:05.544 INFO:tasks.rados.rados.0.smithi063.stdout:1261: oid 44 updating version 246 to 247 2024-08-17T17:00:05.544 INFO:tasks.rados.rados.0.smithi063.stdout:1261: oid 44 updating version 247 to 248 2024-08-17T17:00:05.544 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 248 (ObjNum 497 snap 157 seq_num 497) dirty exists 2024-08-17T17:00:05.544 INFO:tasks.rados.rados.0.smithi063.stdout:1261: left oid 44 (ObjNum 497 snap 157 seq_num 497) 2024-08-17T17:00:05.544 INFO:tasks.rados.rados.0.smithi063.stdout:1263: finishing rollback tid 1 to smithi06387500-32 2024-08-17T17:00:05.544 INFO:tasks.rados.rados.0.smithi063.stdout:1263: finishing rollback tid 2 to smithi06387500-32 2024-08-17T17:00:05.544 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 248 (ObjNum 423 snap 130 seq_num 423) dirty exists 2024-08-17T17:00:05.544 INFO:tasks.rados.rados.0.smithi063.stdout:1264: expect (ObjNum 443 snap 139 seq_num 443) 2024-08-17T17:00:05.551 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:00:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:00:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-08-17T17:00:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (0 remaining).", 2024-08-17T17:00:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-08-17T17:00:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "2/3 daemons upgraded", 2024-08-17T17:00:05.552 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading mon daemons", 2024-08-17T17:00:05.552 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T17:00:05.552 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:00:05.755 INFO:tasks.rados.rados.0.smithi063.stdout:1262: expect (ObjNum 449 snap 145 seq_num 449) 2024-08-17T17:00:05.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T17:00:05.753+0000 7f4652b73640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-08-17T17:00:05.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:05.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/118322213' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:00:05.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:05.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:05.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[114756]: pgmap v28: 89 pgs: 89 active+clean; 207 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 790 B/s rd, 728 KiB/s wr, 3 op/s 2024-08-17T17:00:05.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[114756]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[114756]: osdmap e384: 8 total, 8 up, 8 in 2024-08-17T17:00:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[114756]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/118322213' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:00:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:05 smithi162 ceph-mon[34633]: pgmap v28: 89 pgs: 89 active+clean; 207 MiB data, 689 MiB used, 715 GiB / 715 GiB avail; 790 B/s rd, 728 KiB/s wr, 3 op/s 2024-08-17T17:00:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:05 smithi162 ceph-mon[34633]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:05 smithi162 ceph-mon[34633]: osdmap e384: 8 total, 8 up, 8 in 2024-08-17T17:00:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:05 smithi162 ceph-mon[34633]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:05 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:05 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:05 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/118322213' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:00:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:05 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:05 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:05.910 INFO:tasks.rados.rados.0.smithi063.stdout:1266: finishing write tid 1 to smithi06387500-21 2024-08-17T17:00:05.910 INFO:tasks.rados.rados.0.smithi063.stdout:1266: finishing write tid 2 to smithi06387500-21 2024-08-17T17:00:05.910 INFO:tasks.rados.rados.0.smithi063.stdout:1266: finishing write tid 3 to smithi06387500-21 2024-08-17T17:00:05.910 INFO:tasks.rados.rados.0.smithi063.stdout:1266: finishing write tid 4 to smithi06387500-21 2024-08-17T17:00:05.910 INFO:tasks.rados.rados.0.smithi063.stdout:1266: oid 21 updating version 0 to 890 2024-08-17T17:00:05.910 INFO:tasks.rados.rados.0.smithi063.stdout:1266: oid 21 version 890 is already newer than 889 2024-08-17T17:00:05.910 INFO:tasks.rados.rados.0.smithi063.stdout:1266: oid 21 version 890 is already newer than 888 2024-08-17T17:00:05.910 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 890 (ObjNum 498 snap 157 seq_num 498) dirty exists 2024-08-17T17:00:05.910 INFO:tasks.rados.rados.0.smithi063.stdout:1266: left oid 21 (ObjNum 498 snap 157 seq_num 498) 2024-08-17T17:00:05.910 INFO:tasks.rados.rados.0.smithi063.stdout:1267: finishing rollback tid 0 to smithi06387500-31 2024-08-17T17:00:05.911 INFO:tasks.rados.rados.0.smithi063.stdout:1268: seq_num 499 ranges {786693=712196,2150397=236296} 2024-08-17T17:00:05.918 INFO:tasks.rados.rados.0.smithi063.stdout:1268: writing smithi06387500-39 from 786693 to 1498889 tid 1 2024-08-17T17:00:05.923 INFO:tasks.rados.rados.0.smithi063.stdout:1268: writing smithi06387500-39 from 2150397 to 2386693 tid 2 2024-08-17T17:00:05.923 INFO:tasks.rados.rados.0.smithi063.stdout:1261: done (6 left) 2024-08-17T17:00:05.923 INFO:tasks.rados.rados.0.smithi063.stdout:1262: done (5 left) 2024-08-17T17:00:05.923 INFO:tasks.rados.rados.0.smithi063.stdout:1263: done (4 left) 2024-08-17T17:00:05.923 INFO:tasks.rados.rados.0.smithi063.stdout:1264: done (3 left) 2024-08-17T17:00:05.924 INFO:tasks.rados.rados.0.smithi063.stdout:1266: done (2 left) 2024-08-17T17:00:05.924 INFO:tasks.rados.rados.0.smithi063.stdout:1269: delete oid 7 current snap is 157 2024-08-17T17:00:05.924 INFO:tasks.rados.rados.0.smithi063.stdout:1267: finishing rollback tid 1 to smithi06387500-31 2024-08-17T17:00:05.924 INFO:tasks.rados.rados.0.smithi063.stdout:1267: finishing rollback tid 2 to smithi06387500-31 2024-08-17T17:00:05.924 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 894 (ObjNum 395 snap 121 seq_num 395) dirty exists 2024-08-17T17:00:05.924 INFO:tasks.rados.rados.0.smithi063.stdout:1268: finishing write tid 1 to smithi06387500-39 2024-08-17T17:00:05.925 INFO:tasks.rados.rados.0.smithi063.stdout:1267: done (2 left) 2024-08-17T17:00:05.925 INFO:tasks.rados.rados.0.smithi063.stdout:1269: done (1 left) 2024-08-17T17:00:05.925 INFO:tasks.rados.rados.0.smithi063.stdout:1270: read oid 22 snap 141 2024-08-17T17:00:05.925 INFO:tasks.rados.rados.0.smithi063.stdout:1270: expect deleted 2024-08-17T17:00:05.925 INFO:tasks.rados.rados.0.smithi063.stdout:1271: write oid 4 current snap is 157 2024-08-17T17:00:05.925 INFO:tasks.rados.rados.0.smithi063.stdout:1271: seq_num 501 ranges {420307=462694,1351008=617495} 2024-08-17T17:00:05.930 INFO:tasks.rados.rados.0.smithi063.stdout:1271: writing smithi06387500-4 from 420307 to 883001 tid 1 2024-08-17T17:00:05.936 INFO:tasks.rados.rados.0.smithi063.stdout:1271: writing smithi06387500-4 from 1351008 to 1968503 tid 2 2024-08-17T17:00:05.936 INFO:tasks.rados.rados.0.smithi063.stdout:1272: write oid 36 current snap is 157 2024-08-17T17:00:05.936 INFO:tasks.rados.rados.0.smithi063.stdout:1272: seq_num 502 ranges {437114=654296,1525843=477506} 2024-08-17T17:00:05.942 INFO:tasks.rados.rados.0.smithi063.stdout:1272: writing smithi06387500-36 from 437114 to 1091410 tid 1 2024-08-17T17:00:05.948 INFO:tasks.rados.rados.0.smithi063.stdout:1272: writing smithi06387500-36 from 1525843 to 2003349 tid 2 2024-08-17T17:00:05.948 INFO:tasks.rados.rados.0.smithi063.stdout:1273: snap_create 2024-08-17T17:00:05.948 INFO:tasks.rados.rados.0.smithi063.stdout:1268: finishing write tid 2 to smithi06387500-39 2024-08-17T17:00:05.948 INFO:tasks.rados.rados.0.smithi063.stdout:1268: finishing write tid 3 to smithi06387500-39 2024-08-17T17:00:05.948 INFO:tasks.rados.rados.0.smithi063.stdout:1268: finishing write tid 4 to smithi06387500-39 2024-08-17T17:00:05.948 INFO:tasks.rados.rados.0.smithi063.stdout:1268: oid 39 updating version 0 to 324 2024-08-17T17:00:05.948 INFO:tasks.rados.rados.0.smithi063.stdout:1268: oid 39 version 324 is already newer than 323 2024-08-17T17:00:05.948 INFO:tasks.rados.rados.0.smithi063.stdout:1268: oid 39 version 324 is already newer than 322 2024-08-17T17:00:05.948 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 324 (ObjNum 499 snap 157 seq_num 499) dirty exists 2024-08-17T17:00:05.948 INFO:tasks.rados.rados.0.smithi063.stdout:1268: left oid 39 (ObjNum 499 snap 157 seq_num 499) 2024-08-17T17:00:05.948 INFO:tasks.rados.rados.0.smithi063.stdout:1271: finishing write tid 1 to smithi06387500-4 2024-08-17T17:00:05.949 INFO:tasks.rados.rados.0.smithi063.stdout:1271: finishing write tid 2 to smithi06387500-4 2024-08-17T17:00:05.949 INFO:tasks.rados.rados.0.smithi063.stdout:1271: finishing write tid 3 to smithi06387500-4 2024-08-17T17:00:05.949 INFO:tasks.rados.rados.0.smithi063.stdout:1271: finishing write tid 4 to smithi06387500-4 2024-08-17T17:00:05.949 INFO:tasks.rados.rados.0.smithi063.stdout:1271: oid 4 updating version 0 to 1304 2024-08-17T17:00:05.949 INFO:tasks.rados.rados.0.smithi063.stdout:1271: oid 4 version 1304 is already newer than 1303 2024-08-17T17:00:05.949 INFO:tasks.rados.rados.0.smithi063.stdout:1271: oid 4 updating version 1304 to 1305 2024-08-17T17:00:05.949 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 1305 (ObjNum 501 snap 157 seq_num 501) dirty exists 2024-08-17T17:00:05.949 INFO:tasks.rados.rados.0.smithi063.stdout:1271: left oid 4 (ObjNum 501 snap 157 seq_num 501) 2024-08-17T17:00:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:1272: finishing write tid 1 to smithi06387500-36 2024-08-17T17:00:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:1272: finishing write tid 2 to smithi06387500-36 2024-08-17T17:00:05.951 INFO:tasks.rados.rados.0.smithi063.stdout:1272: finishing write tid 3 to smithi06387500-36 2024-08-17T17:00:05.952 INFO:tasks.rados.rados.0.smithi063.stdout:1272: finishing write tid 4 to smithi06387500-36 2024-08-17T17:00:05.952 INFO:tasks.rados.rados.0.smithi063.stdout:1272: oid 36 updating version 0 to 1309 2024-08-17T17:00:05.952 INFO:tasks.rados.rados.0.smithi063.stdout:1272: oid 36 version 1309 is already newer than 1308 2024-08-17T17:00:05.952 INFO:tasks.rados.rados.0.smithi063.stdout:1272: oid 36 version 1309 is already newer than 1307 2024-08-17T17:00:05.952 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 1309 (ObjNum 502 snap 157 seq_num 502) dirty exists 2024-08-17T17:00:05.952 INFO:tasks.rados.rados.0.smithi063.stdout:1272: left oid 36 (ObjNum 502 snap 157 seq_num 502) 2024-08-17T17:00:06.766 INFO:tasks.rados.rados.0.smithi063.stdout:1268: done (4 left) 2024-08-17T17:00:06.767 INFO:tasks.rados.rados.0.smithi063.stdout:1270: done (3 left) 2024-08-17T17:00:06.767 INFO:tasks.rados.rados.0.smithi063.stdout:1271: done (2 left) 2024-08-17T17:00:06.767 INFO:tasks.rados.rados.0.smithi063.stdout:1272: done (1 left) 2024-08-17T17:00:06.767 INFO:tasks.rados.rados.0.smithi063.stdout:1273: done (0 left) 2024-08-17T17:00:06.767 INFO:tasks.rados.rados.0.smithi063.stdout:1274: rollback oid 42 current snap is 158 2024-08-17T17:00:06.767 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 42 to 154 2024-08-17T17:00:06.767 INFO:tasks.rados.rados.0.smithi063.stdout:1275: rollback oid 39 current snap is 158 2024-08-17T17:00:06.767 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 39 to 156 2024-08-17T17:00:06.767 INFO:tasks.rados.rados.0.smithi063.stdout:1276: write oid 21 current snap is 158 2024-08-17T17:00:06.767 INFO:tasks.rados.rados.0.smithi063.stdout:1276: seq_num 503 ranges {453921=729545,1584325=621164} 2024-08-17T17:00:06.773 INFO:tasks.rados.rados.0.smithi063.stdout:1276: writing smithi06387500-21 from 453921 to 1183466 tid 1 2024-08-17T17:00:06.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:06 smithi063 ceph-mon[109843]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:06.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:06 smithi063 ceph-mon[109843]: osdmap e385: 8 total, 8 up, 8 in 2024-08-17T17:00:06.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:06 smithi063 ceph-mon[109843]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:06.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:06 smithi063 ceph-mon[114756]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:06.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:06 smithi063 ceph-mon[114756]: osdmap e385: 8 total, 8 up, 8 in 2024-08-17T17:00:06.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:06 smithi063 ceph-mon[114756]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:06.780 INFO:tasks.rados.rados.0.smithi063.stdout:1276: writing smithi06387500-21 from 1584325 to 2205489 tid 2 2024-08-17T17:00:06.780 INFO:tasks.rados.rados.0.smithi063.stdout:1277: read oid 50 snap -1 2024-08-17T17:00:06.781 INFO:tasks.rados.rados.0.smithi063.stdout:1277: expect (ObjNum 475 snap 153 seq_num 475) 2024-08-17T17:00:06.781 INFO:tasks.rados.rados.0.smithi063.stdout:1278: write oid 38 current snap is 158 2024-08-17T17:00:06.781 INFO:tasks.rados.rados.0.smithi063.stdout:1275: finishing rollback tid 0 to smithi06387500-39 2024-08-17T17:00:06.781 INFO:tasks.rados.rados.0.smithi063.stdout:1275: finishing rollback tid 1 to smithi06387500-39 2024-08-17T17:00:06.781 INFO:tasks.rados.rados.0.smithi063.stdout:1275: finishing rollback tid 2 to smithi06387500-39 2024-08-17T17:00:06.781 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 328 (ObjNum 493 snap 156 seq_num 493) dirty exists 2024-08-17T17:00:06.781 INFO:tasks.rados.rados.0.smithi063.stdout:1274: finishing rollback tid 0 to smithi06387500-42 2024-08-17T17:00:06.781 INFO:tasks.rados.rados.0.smithi063.stdout:1274: finishing rollback tid 1 to smithi06387500-42 2024-08-17T17:00:06.781 INFO:tasks.rados.rados.0.smithi063.stdout:1274: finishing rollback tid 2 to smithi06387500-42 2024-08-17T17:00:06.781 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 898 (ObjNum 478 snap 153 seq_num 478) dirty exists 2024-08-17T17:00:06.781 INFO:tasks.rados.rados.0.smithi063.stdout:1276: finishing write tid 1 to smithi06387500-21 2024-08-17T17:00:06.781 INFO:tasks.rados.rados.0.smithi063.stdout:1278: seq_num 504 ranges {470728=404794,1526454=481175} 2024-08-17T17:00:06.785 INFO:tasks.rados.rados.0.smithi063.stdout:1278: writing smithi06387500-38 from 470728 to 875522 tid 1 2024-08-17T17:00:06.791 INFO:tasks.rados.rados.0.smithi063.stdout:1278: writing smithi06387500-38 from 1526454 to 2007629 tid 2 2024-08-17T17:00:06.791 INFO:tasks.rados.rados.0.smithi063.stdout:1274: done (4 left) 2024-08-17T17:00:06.791 INFO:tasks.rados.rados.0.smithi063.stdout:1275: done (3 left) 2024-08-17T17:00:06.791 INFO:tasks.rados.rados.0.smithi063.stdout:1279: read oid 8 snap -1 2024-08-17T17:00:06.791 INFO:tasks.rados.rados.0.smithi063.stdout:1279: expect (ObjNum 472 snap 153 seq_num 472) 2024-08-17T17:00:06.791 INFO:tasks.rados.rados.0.smithi063.stdout:1280: write oid 19 current snap is 158 2024-08-17T17:00:06.792 INFO:tasks.rados.rados.0.smithi063.stdout:1278: finishing write tid 1 to smithi06387500-38 2024-08-17T17:00:06.792 INFO:tasks.rados.rados.0.smithi063.stdout:1280: seq_num 505 ranges {487535=480043,1584936=624833} 2024-08-17T17:00:06.796 INFO:tasks.rados.rados.0.smithi063.stdout:1280: writing smithi06387500-19 from 487535 to 967578 tid 1 2024-08-17T17:00:06.803 INFO:tasks.rados.rados.0.smithi063.stdout:1280: writing smithi06387500-19 from 1584936 to 2209769 tid 2 2024-08-17T17:00:06.803 INFO:tasks.rados.rados.0.smithi063.stdout:1281: write oid 23 current snap is 158 2024-08-17T17:00:06.804 INFO:tasks.rados.rados.0.smithi063.stdout:1281: seq_num 506 ranges {504342=555292,1643418=484844} 2024-08-17T17:00:06.809 INFO:tasks.rados.rados.0.smithi063.stdout:1281: writing smithi06387500-23 from 504342 to 1059634 tid 1 2024-08-17T17:00:06.816 INFO:tasks.rados.rados.0.smithi063.stdout:1281: writing smithi06387500-23 from 1643418 to 2128262 tid 2 2024-08-17T17:00:06.816 INFO:tasks.rados.rados.0.smithi063.stdout:1282: read oid 24 snap 145 2024-08-17T17:00:06.816 INFO:tasks.rados.rados.0.smithi063.stdout:1282: expect deleted 2024-08-17T17:00:06.816 INFO:tasks.rados.rados.0.smithi063.stdout:1283: snap_create 2024-08-17T17:00:06.816 INFO:tasks.rados.rados.0.smithi063.stdout:1276: finishing write tid 2 to smithi06387500-21 2024-08-17T17:00:06.816 INFO:tasks.rados.rados.0.smithi063.stdout:1276: finishing write tid 3 to smithi06387500-21 2024-08-17T17:00:06.816 INFO:tasks.rados.rados.0.smithi063.stdout:1276: finishing write tid 4 to smithi06387500-21 2024-08-17T17:00:06.816 INFO:tasks.rados.rados.0.smithi063.stdout:1276: oid 21 updating version 0 to 902 2024-08-17T17:00:06.816 INFO:tasks.rados.rados.0.smithi063.stdout:1276: oid 21 version 902 is already newer than 900 2024-08-17T17:00:06.816 INFO:tasks.rados.rados.0.smithi063.stdout:1276: oid 21 version 902 is already newer than 901 2024-08-17T17:00:06.817 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 902 (ObjNum 503 snap 158 seq_num 503) dirty exists 2024-08-17T17:00:06.817 INFO:tasks.rados.rados.0.smithi063.stdout:1276: left oid 21 (ObjNum 503 snap 158 seq_num 503) 2024-08-17T17:00:06.817 INFO:tasks.rados.rados.0.smithi063.stdout:1277: expect (ObjNum 475 snap 153 seq_num 475) 2024-08-17T17:00:06.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:06 smithi162 ceph-mon[34633]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:06.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:06 smithi162 ceph-mon[34633]: osdmap e385: 8 total, 8 up, 8 in 2024-08-17T17:00:06.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:06 smithi162 ceph-mon[34633]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:06.952 INFO:tasks.rados.rados.0.smithi063.stdout:1278: finishing write tid 2 to smithi06387500-38 2024-08-17T17:00:06.952 INFO:tasks.rados.rados.0.smithi063.stdout:1278: finishing write tid 3 to smithi06387500-38 2024-08-17T17:00:06.952 INFO:tasks.rados.rados.0.smithi063.stdout:1278: finishing write tid 4 to smithi06387500-38 2024-08-17T17:00:06.952 INFO:tasks.rados.rados.0.smithi063.stdout:1278: oid 38 updating version 0 to 200 2024-08-17T17:00:06.952 INFO:tasks.rados.rados.0.smithi063.stdout:1278: oid 38 updating version 200 to 201 2024-08-17T17:00:06.953 INFO:tasks.rados.rados.0.smithi063.stdout:1278: oid 38 updating version 201 to 202 2024-08-17T17:00:06.953 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 202 (ObjNum 504 snap 158 seq_num 504) dirty exists 2024-08-17T17:00:06.953 INFO:tasks.rados.rados.0.smithi063.stdout:1278: left oid 38 (ObjNum 504 snap 158 seq_num 504) 2024-08-17T17:00:06.953 INFO:tasks.rados.rados.0.smithi063.stdout:1279: expect (ObjNum 472 snap 153 seq_num 472) 2024-08-17T17:00:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1280: finishing write tid 1 to smithi06387500-19 2024-08-17T17:00:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1280: finishing write tid 2 to smithi06387500-19 2024-08-17T17:00:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1280: finishing write tid 3 to smithi06387500-19 2024-08-17T17:00:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1280: finishing write tid 4 to smithi06387500-19 2024-08-17T17:00:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1280: oid 19 updating version 0 to 904 2024-08-17T17:00:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1280: oid 19 updating version 904 to 905 2024-08-17T17:00:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1280: oid 19 updating version 905 to 906 2024-08-17T17:00:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 906 (ObjNum 505 snap 158 seq_num 505) dirty exists 2024-08-17T17:00:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1280: left oid 19 (ObjNum 505 snap 158 seq_num 505) 2024-08-17T17:00:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1281: finishing write tid 1 to smithi06387500-23 2024-08-17T17:00:07.140 INFO:tasks.rados.rados.0.smithi063.stdout:1281: finishing write tid 2 to smithi06387500-23 2024-08-17T17:00:07.140 INFO:tasks.rados.rados.0.smithi063.stdout:1281: finishing write tid 3 to smithi06387500-23 2024-08-17T17:00:07.140 INFO:tasks.rados.rados.0.smithi063.stdout:1281: finishing write tid 4 to smithi06387500-23 2024-08-17T17:00:07.140 INFO:tasks.rados.rados.0.smithi063.stdout:1281: oid 23 updating version 0 to 631 2024-08-17T17:00:07.140 INFO:tasks.rados.rados.0.smithi063.stdout:1281: oid 23 updating version 631 to 632 2024-08-17T17:00:07.140 INFO:tasks.rados.rados.0.smithi063.stdout:1281: oid 23 updating version 632 to 633 2024-08-17T17:00:07.140 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 633 (ObjNum 506 snap 158 seq_num 506) dirty exists 2024-08-17T17:00:07.140 INFO:tasks.rados.rados.0.smithi063.stdout:1281: left oid 23 (ObjNum 506 snap 158 seq_num 506) 2024-08-17T17:00:07.771 INFO:tasks.rados.rados.0.smithi063.stdout:1276: done (7 left) 2024-08-17T17:00:07.771 INFO:tasks.rados.rados.0.smithi063.stdout:1277: done (6 left) 2024-08-17T17:00:07.771 INFO:tasks.rados.rados.0.smithi063.stdout:1278: done (5 left) 2024-08-17T17:00:07.771 INFO:tasks.rados.rados.0.smithi063.stdout:1279: done (4 left) 2024-08-17T17:00:07.771 INFO:tasks.rados.rados.0.smithi063.stdout:1280: done (3 left) 2024-08-17T17:00:07.771 INFO:tasks.rados.rados.0.smithi063.stdout:1281: done (2 left) 2024-08-17T17:00:07.771 INFO:tasks.rados.rados.0.smithi063.stdout:1282: done (1 left) 2024-08-17T17:00:07.771 INFO:tasks.rados.rados.0.smithi063.stdout:1283: done (0 left) 2024-08-17T17:00:07.771 INFO:tasks.rados.rados.0.smithi063.stdout:1284: snap_create 2024-08-17T17:00:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:07 smithi063 ceph-mon[114756]: pgmap v31: 89 pgs: 89 active+clean; 207 MiB data, 689 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:00:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:07 smithi063 ceph-mon[114756]: osdmap e386: 8 total, 8 up, 8 in 2024-08-17T17:00:07.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:07 smithi063 ceph-mon[109843]: pgmap v31: 89 pgs: 89 active+clean; 207 MiB data, 689 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:00:07.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:07 smithi063 ceph-mon[109843]: osdmap e386: 8 total, 8 up, 8 in 2024-08-17T17:00:07.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:07 smithi162 ceph-mon[34633]: pgmap v31: 89 pgs: 89 active+clean; 207 MiB data, 689 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:00:07.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:07 smithi162 ceph-mon[34633]: osdmap e386: 8 total, 8 up, 8 in 2024-08-17T17:00:08.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=infra.usagestats t=2024-08-17T17:00:08.010646742Z level=info msg="Usage stats are ready to report" 2024-08-17T17:00:08.771 INFO:tasks.rados.rados.0.smithi063.stdout:1284: done (0 left) 2024-08-17T17:00:08.772 INFO:tasks.rados.rados.0.smithi063.stdout:1285: read oid 13 snap -1 2024-08-17T17:00:08.772 INFO:tasks.rados.rados.0.smithi063.stdout:1285: expect deleted 2024-08-17T17:00:08.772 INFO:tasks.rados.rados.0.smithi063.stdout:1286: read oid 44 snap -1 2024-08-17T17:00:08.772 INFO:tasks.rados.rados.0.smithi063.stdout:1286: expect (ObjNum 497 snap 157 seq_num 497) 2024-08-17T17:00:08.772 INFO:tasks.rados.rados.0.smithi063.stdout:1287: snap_remove snap 145 2024-08-17T17:00:09.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: osdmap e387: 8 total, 8 up, 8 in 2024-08-17T17:00:09.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: pgmap v34: 89 pgs: 89 active+clean; 210 MiB data, 673 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 2.9 MiB/s wr, 31 op/s 2024-08-17T17:00:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-17T17:00:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-17T17:00:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:09.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:09.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:09.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: osdmap e387: 8 total, 8 up, 8 in 2024-08-17T17:00:09.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: pgmap v34: 89 pgs: 89 active+clean; 210 MiB data, 673 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 2.9 MiB/s wr, 31 op/s 2024-08-17T17:00:09.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:09.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:09.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:09.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:09.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:09.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:09.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:09.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:09.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:09.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:09.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-17T17:00:09.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:09.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:09.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-17T17:00:09.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:09.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:09.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:09.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:09.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:09.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:09.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:09.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:09.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:09.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:09.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:09.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:09.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:08 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: osdmap e387: 8 total, 8 up, 8 in 2024-08-17T17:00:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: pgmap v34: 89 pgs: 89 active+clean; 210 MiB data, 673 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 2.9 MiB/s wr, 31 op/s 2024-08-17T17:00:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-08-17T17:00:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:09.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-08-17T17:00:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:09.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:09.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:08 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:09.778 INFO:tasks.rados.rados.0.smithi063.stdout:1287: done (2 left) 2024-08-17T17:00:09.778 INFO:tasks.rados.rados.0.smithi063.stdout:1288: read oid 35 snap -1 2024-08-17T17:00:09.778 INFO:tasks.rados.rados.0.smithi063.stdout:1288: expect (ObjNum 450 snap 145 seq_num 450) 2024-08-17T17:00:09.778 INFO:tasks.rados.rados.0.smithi063.stdout:1289: rollback oid 29 current snap is 160 2024-08-17T17:00:09.778 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 29 to 130 2024-08-17T17:00:09.778 INFO:tasks.rados.rados.0.smithi063.stdout:1286: expect (ObjNum 497 snap 157 seq_num 497) 2024-08-17T17:00:09.949 INFO:tasks.rados.rados.0.smithi063.stdout:1288: expect (ObjNum 450 snap 145 seq_num 450) 2024-08-17T17:00:10.064 INFO:tasks.rados.rados.0.smithi063.stdout:1289: finishing rollback tid 0 to smithi06387500-29 2024-08-17T17:00:10.065 INFO:tasks.rados.rados.0.smithi063.stdout:1289: finishing rollback tid 1 to smithi06387500-29 2024-08-17T17:00:10.065 INFO:tasks.rados.rados.0.smithi063.stdout:1289: finishing rollback tid 2 to smithi06387500-29 2024-08-17T17:00:10.065 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 270 (ObjNum 335 snap 101 seq_num 335) dirty exists 2024-08-17T17:00:10.065 INFO:tasks.rados.rados.0.smithi063.stdout:1285: done (3 left) 2024-08-17T17:00:10.065 INFO:tasks.rados.rados.0.smithi063.stdout:1286: done (2 left) 2024-08-17T17:00:10.065 INFO:tasks.rados.rados.0.smithi063.stdout:1288: done (1 left) 2024-08-17T17:00:10.065 INFO:tasks.rados.rados.0.smithi063.stdout:1289: done (0 left) 2024-08-17T17:00:10.065 INFO:tasks.rados.rados.0.smithi063.stdout:1290: write oid 30 current snap is 160 2024-08-17T17:00:10.065 INFO:tasks.rados.rados.0.smithi063.stdout:1290: seq_num 507 ranges {521149=630541,1701900=628502} 2024-08-17T17:00:10.070 INFO:tasks.rados.rados.0.smithi063.stdout:1290: writing smithi06387500-30 from 521149 to 1151690 tid 1 2024-08-17T17:00:10.078 INFO:tasks.rados.rados.0.smithi063.stdout:1290: writing smithi06387500-30 from 1701900 to 2330402 tid 2 2024-08-17T17:00:10.078 INFO:tasks.rados.rados.0.smithi063.stdout:1291: write oid 33 current snap is 160 2024-08-17T17:00:10.078 INFO:tasks.rados.rados.0.smithi063.stdout:1291: seq_num 508 ranges {537956=705790,1644029=772160} 2024-08-17T17:00:10.084 INFO:tasks.rados.rados.0.smithi063.stdout:1291: writing smithi06387500-33 from 537956 to 1243746 tid 1 2024-08-17T17:00:10.091 INFO:tasks.rados.rados.0.smithi063.stdout:1291: writing smithi06387500-33 from 1644029 to 2416189 tid 2 2024-08-17T17:00:10.091 INFO:tasks.rados.rados.0.smithi063.stdout:1292: write oid 10 current snap is 160 2024-08-17T17:00:10.092 INFO:tasks.rados.rados.0.smithi063.stdout:1292: seq_num 509 ranges {554763=781039,2102511=452252} 2024-08-17T17:00:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:00:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:00:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:00:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:00:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all grafana 2024-08-17T17:00:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: osdmap e388: 8 total, 8 up, 8 in 2024-08-17T17:00:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all loki 2024-08-17T17:00:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:10.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all promtail 2024-08-17T17:00:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: Upgrade: Finalizing container_image settings 2024-08-17T17:00:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-17T17:00:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:00:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:00:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:00:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-17T17:00:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:00:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-17T17:00:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:00:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-17T17:00:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:00:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-17T17:00:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:00:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-17T17:00:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:00:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-17T17:00:10.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:00:10.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-17T17:00:10.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:00:10.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-17T17:00:10.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:00:10.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-17T17:00:10.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:00:10.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:00:10.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: Upgrade: Complete! 2024-08-17T17:00:10.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-17T17:00:10.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:00:10.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:00:10.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:10.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:10.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:10.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:10.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:10.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:10.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:10.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:09 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:10.102 INFO:tasks.rados.rados.0.smithi063.stdout:1292: writing smithi06387500-10 from 554763 to 1335802 tid 1 2024-08-17T17:00:10.107 INFO:tasks.rados.rados.0.smithi063.stdout:1292: writing smithi06387500-10 from 2102511 to 2554763 tid 2 2024-08-17T17:00:10.107 INFO:tasks.rados.rados.0.smithi063.stdout:1293: read oid 18 snap -1 2024-08-17T17:00:10.107 INFO:tasks.rados.rados.0.smithi063.stdout:1293: expect (ObjNum 490 snap 155 seq_num 490) 2024-08-17T17:00:10.107 INFO:tasks.rados.rados.0.smithi063.stdout:1294: rollback oid 48 current snap is 160 2024-08-17T17:00:10.107 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 48 to 130 2024-08-17T17:00:10.107 INFO:tasks.rados.rados.0.smithi063.stdout:1290: finishing write tid 1 to smithi06387500-30 2024-08-17T17:00:10.107 INFO:tasks.rados.rados.0.smithi063.stdout:1295: rollback oid 26 current snap is 160 2024-08-17T17:00:10.107 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 26 to 154 2024-08-17T17:00:10.107 INFO:tasks.rados.rados.0.smithi063.stdout:1290: finishing write tid 2 to smithi06387500-30 2024-08-17T17:00:10.107 INFO:tasks.rados.rados.0.smithi063.stdout:1290: finishing write tid 3 to smithi06387500-30 2024-08-17T17:00:10.107 INFO:tasks.rados.rados.0.smithi063.stdout:1290: finishing write tid 4 to smithi06387500-30 2024-08-17T17:00:10.107 INFO:tasks.rados.rados.0.smithi063.stdout:1290: oid 30 updating version 0 to 670 2024-08-17T17:00:10.108 INFO:tasks.rados.rados.0.smithi063.stdout:1290: oid 30 version 670 is already newer than 669 2024-08-17T17:00:10.108 INFO:tasks.rados.rados.0.smithi063.stdout:1290: oid 30 updating version 670 to 671 2024-08-17T17:00:10.108 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 671 (ObjNum 507 snap 160 seq_num 507) dirty exists 2024-08-17T17:00:10.108 INFO:tasks.rados.rados.0.smithi063.stdout:1290: left oid 30 (ObjNum 507 snap 160 seq_num 507) 2024-08-17T17:00:10.108 INFO:tasks.rados.rados.0.smithi063.stdout:1291: finishing write tid 1 to smithi06387500-33 2024-08-17T17:00:10.108 INFO:tasks.rados.rados.0.smithi063.stdout:1291: finishing write tid 2 to smithi06387500-33 2024-08-17T17:00:10.108 INFO:tasks.rados.rados.0.smithi063.stdout:1291: finishing write tid 3 to smithi06387500-33 2024-08-17T17:00:10.108 INFO:tasks.rados.rados.0.smithi063.stdout:1291: finishing write tid 4 to smithi06387500-33 2024-08-17T17:00:10.108 INFO:tasks.rados.rados.0.smithi063.stdout:1291: oid 33 updating version 0 to 386 2024-08-17T17:00:10.108 INFO:tasks.rados.rados.0.smithi063.stdout:1291: oid 33 version 386 is already newer than 385 2024-08-17T17:00:10.108 INFO:tasks.rados.rados.0.smithi063.stdout:1291: oid 33 updating version 386 to 387 2024-08-17T17:00:10.108 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 387 (ObjNum 508 snap 160 seq_num 508) dirty exists 2024-08-17T17:00:10.109 INFO:tasks.rados.rados.0.smithi063.stdout:1291: left oid 33 (ObjNum 508 snap 160 seq_num 508) 2024-08-17T17:00:10.109 INFO:tasks.rados.rados.0.smithi063.stdout:1292: finishing write tid 1 to smithi06387500-10 2024-08-17T17:00:10.109 INFO:tasks.rados.rados.0.smithi063.stdout:1290: done (5 left) 2024-08-17T17:00:10.109 INFO:tasks.rados.rados.0.smithi063.stdout:1291: done (4 left) 2024-08-17T17:00:10.109 INFO:tasks.rados.rados.0.smithi063.stdout:1296: read oid 37 snap 155 2024-08-17T17:00:10.109 INFO:tasks.rados.rados.0.smithi063.stdout:1296: expect deleted 2024-08-17T17:00:10.109 INFO:tasks.rados.rados.0.smithi063.stdout:1297: read oid 14 snap -1 2024-08-17T17:00:10.109 INFO:tasks.rados.rados.0.smithi063.stdout:1297: expect (ObjNum 458 snap 148 seq_num 458) 2024-08-17T17:00:10.109 INFO:tasks.rados.rados.0.smithi063.stdout:1298: write oid 46 current snap is 160 2024-08-17T17:00:10.109 INFO:tasks.rados.rados.0.smithi063.stdout:1298: seq_num 510 ranges {571570=572641,1877346=694224} 2024-08-17T17:00:10.112 INFO:tasks.rados.rados.0.smithi063.stdout:1298: writing smithi06387500-46 from 571570 to 1144211 tid 1 2024-08-17T17:00:10.122 INFO:tasks.rados.rados.0.smithi063.stdout:1298: writing smithi06387500-46 from 1877346 to 2571570 tid 2 2024-08-17T17:00:10.122 INFO:tasks.rados.rados.0.smithi063.stdout:1299: read oid 20 snap -1 2024-08-17T17:00:10.122 INFO:tasks.rados.rados.0.smithi063.stdout:1299: expect deleted 2024-08-17T17:00:10.122 INFO:tasks.rados.rados.0.smithi063.stdout:1300: read oid 12 snap 153 2024-08-17T17:00:10.122 INFO:tasks.rados.rados.0.smithi063.stdout:1300: expect (ObjNum 369 snap 115 seq_num 369) 2024-08-17T17:00:10.122 INFO:tasks.rados.rados.0.smithi063.stdout:1301: snap_remove snap 149 2024-08-17T17:00:10.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:00:10.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:00:10.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:00:10.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:00:10.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all grafana 2024-08-17T17:00:10.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:10.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:10.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:10.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: osdmap e388: 8 total, 8 up, 8 in 2024-08-17T17:00:10.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all loki 2024-08-17T17:00:10.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:10.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:10.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:10.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:10.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:10.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all promtail 2024-08-17T17:00:10.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:10.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:10.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:10.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:10.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: Upgrade: Finalizing container_image settings 2024-08-17T17:00:10.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-17T17:00:10.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:00:10.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:00:10.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:00:10.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-17T17:00:10.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:00:10.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:00:10.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:00:10.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:00:10.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all grafana 2024-08-17T17:00:10.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:10.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:10.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:10.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: osdmap e388: 8 total, 8 up, 8 in 2024-08-17T17:00:10.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all loki 2024-08-17T17:00:10.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:10.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:10.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:10.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:10.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:00:10.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all promtail 2024-08-17T17:00:10.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-08-17T17:00:10.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-08-17T17:00:10.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-08-17T17:00:10.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-08-17T17:00:10.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: Upgrade: Finalizing container_image settings 2024-08-17T17:00:10.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-08-17T17:00:10.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:00:10.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:00:10.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:00:10.284 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-08-17T17:00:10.285 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:00:10.285 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-17T17:00:10.285 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:00:10.285 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-17T17:00:10.285 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:00:10.285 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-17T17:00:10.285 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:00:10.285 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-17T17:00:10.285 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:00:10.285 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.285 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.285 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-17T17:00:10.285 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:00:10.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-17T17:00:10.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:00:10.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-17T17:00:10.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:00:10.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-17T17:00:10.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:00:10.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:00:10.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.286 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:00:10.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-08-17T17:00:10.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:00:10.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-08-17T17:00:10.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:00:10.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-08-17T17:00:10.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:00:10.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-08-17T17:00:10.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:00:10.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.287 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-08-17T17:00:10.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:00:10.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-08-17T17:00:10.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:00:10.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-08-17T17:00:10.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:00:10.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-08-17T17:00:10.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:00:10.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:00:10.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.288 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: Upgrade: Complete! 2024-08-17T17:00:10.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-17T17:00:10.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:00:10.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:00:10.290 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:10.290 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:10.290 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:10.290 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:10.290 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:10.290 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:10.290 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:10.290 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:10.290 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.290 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.291 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.291 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.291 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.291 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.291 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.291 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-08-17T17:00:10.291 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:00:10.291 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: Upgrade: Complete! 2024-08-17T17:00:10.291 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-08-17T17:00:10.291 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:00:10.291 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:00:10.291 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:10.291 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:10.291 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:10.291 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:10.292 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:00:10.292 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:00:10.292 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:00:10.292 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:10.794 INFO:tasks.rados.rados.0.smithi063.stdout:1301: done (9 left) 2024-08-17T17:00:10.794 INFO:tasks.rados.rados.0.smithi063.stdout:1302: write oid 1 current snap is 160 2024-08-17T17:00:10.795 INFO:tasks.rados.rados.0.smithi063.stdout:1302: seq_num 511 ranges {588377=647890,1935828=635840} 2024-08-17T17:00:10.802 INFO:tasks.rados.rados.0.smithi063.stdout:1302: writing smithi06387500-1 from 588377 to 1236267 tid 1 2024-08-17T17:00:10.810 INFO:tasks.rados.rados.0.smithi063.stdout:1302: writing smithi06387500-1 from 1935828 to 2571668 tid 2 2024-08-17T17:00:10.810 INFO:tasks.rados.rados.0.smithi063.stdout:1303: snap_create 2024-08-17T17:00:10.810 INFO:tasks.rados.rados.0.smithi063.stdout:1295: finishing rollback tid 0 to smithi06387500-26 2024-08-17T17:00:10.810 INFO:tasks.rados.rados.0.smithi063.stdout:1293: expect (ObjNum 490 snap 155 seq_num 490) 2024-08-17T17:00:11.006 INFO:tasks.rados.rados.0.smithi063.stdout:1295: finishing rollback tid 1 to smithi06387500-26 2024-08-17T17:00:11.007 INFO:tasks.rados.rados.0.smithi063.stdout:1295: finishing rollback tid 2 to smithi06387500-26 2024-08-17T17:00:11.007 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 397 (ObjNum 463 snap 150 seq_num 463) dirty exists 2024-08-17T17:00:11.007 INFO:tasks.rados.rados.0.smithi063.stdout:1292: finishing write tid 2 to smithi06387500-10 2024-08-17T17:00:11.007 INFO:tasks.rados.rados.0.smithi063.stdout:1292: finishing write tid 3 to smithi06387500-10 2024-08-17T17:00:11.007 INFO:tasks.rados.rados.0.smithi063.stdout:1292: finishing write tid 4 to smithi06387500-10 2024-08-17T17:00:11.007 INFO:tasks.rados.rados.0.smithi063.stdout:1292: oid 10 updating version 0 to 480 2024-08-17T17:00:11.007 INFO:tasks.rados.rados.0.smithi063.stdout:1292: oid 10 version 480 is already newer than 479 2024-08-17T17:00:11.007 INFO:tasks.rados.rados.0.smithi063.stdout:1292: oid 10 version 480 is already newer than 478 2024-08-17T17:00:11.007 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 480 (ObjNum 509 snap 160 seq_num 509) dirty exists 2024-08-17T17:00:11.007 INFO:tasks.rados.rados.0.smithi063.stdout:1292: left oid 10 (ObjNum 509 snap 160 seq_num 509) 2024-08-17T17:00:11.007 INFO:tasks.rados.rados.0.smithi063.stdout:1298: finishing write tid 1 to smithi06387500-46 2024-08-17T17:00:11.007 INFO:tasks.rados.rados.0.smithi063.stdout:1294: finishing rollback tid 0 to smithi06387500-48 2024-08-17T17:00:11.008 INFO:tasks.rados.rados.0.smithi063.stdout:1294: finishing rollback tid 1 to smithi06387500-48 2024-08-17T17:00:11.008 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 1318 (ObjNum 33 snap 0 seq_num 1807061440) dirty dne 2024-08-17T17:00:11.008 INFO:tasks.rados.rados.0.smithi063.stdout:1297: expect (ObjNum 458 snap 148 seq_num 458) 2024-08-17T17:00:11.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:10 smithi162 ceph-mon[34633]: osdmap e389: 8 total, 8 up, 8 in 2024-08-17T17:00:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:10 smithi162 ceph-mon[34633]: pgmap v37: 89 pgs: 89 active+clean; 210 MiB data, 673 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 2.9 MiB/s wr, 31 op/s 2024-08-17T17:00:11.166 INFO:tasks.rados.rados.0.smithi063.stdout:1298: finishing write tid 2 to smithi06387500-46 2024-08-17T17:00:11.166 INFO:tasks.rados.rados.0.smithi063.stdout:1298: finishing write tid 3 to smithi06387500-46 2024-08-17T17:00:11.166 INFO:tasks.rados.rados.0.smithi063.stdout:1298: finishing write tid 4 to smithi06387500-46 2024-08-17T17:00:11.166 INFO:tasks.rados.rados.0.smithi063.stdout:1298: oid 46 updating version 0 to 493 2024-08-17T17:00:11.166 INFO:tasks.rados.rados.0.smithi063.stdout:1298: oid 46 version 493 is already newer than 492 2024-08-17T17:00:11.166 INFO:tasks.rados.rados.0.smithi063.stdout:1298: oid 46 version 493 is already newer than 491 2024-08-17T17:00:11.166 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 493 (ObjNum 510 snap 160 seq_num 510) dirty exists 2024-08-17T17:00:11.166 INFO:tasks.rados.rados.0.smithi063.stdout:1298: left oid 46 (ObjNum 510 snap 160 seq_num 510) 2024-08-17T17:00:11.166 INFO:tasks.rados.rados.0.smithi063.stdout:1300: expect (ObjNum 369 snap 115 seq_num 369) 2024-08-17T17:00:11.180 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:10 smithi063 ceph-mon[114756]: osdmap e389: 8 total, 8 up, 8 in 2024-08-17T17:00:11.180 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:10 smithi063 ceph-mon[114756]: pgmap v37: 89 pgs: 89 active+clean; 210 MiB data, 673 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 2.9 MiB/s wr, 31 op/s 2024-08-17T17:00:11.180 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:10 smithi063 ceph-mon[109843]: osdmap e389: 8 total, 8 up, 8 in 2024-08-17T17:00:11.181 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:10 smithi063 ceph-mon[109843]: pgmap v37: 89 pgs: 89 active+clean; 210 MiB data, 673 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 2.9 MiB/s wr, 31 op/s 2024-08-17T17:00:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:1302: finishing write tid 1 to smithi06387500-1 2024-08-17T17:00:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:1302: finishing write tid 2 to smithi06387500-1 2024-08-17T17:00:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:1302: finishing write tid 3 to smithi06387500-1 2024-08-17T17:00:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:1302: finishing write tid 4 to smithi06387500-1 2024-08-17T17:00:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:1302: oid 1 updating version 0 to 643 2024-08-17T17:00:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:1302: oid 1 version 643 is already newer than 641 2024-08-17T17:00:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:1302: oid 1 version 643 is already newer than 642 2024-08-17T17:00:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 643 (ObjNum 511 snap 160 seq_num 511) dirty exists 2024-08-17T17:00:11.284 INFO:tasks.rados.rados.0.smithi063.stdout:1302: left oid 1 (ObjNum 511 snap 160 seq_num 511) 2024-08-17T17:00:11.798 INFO:tasks.rados.rados.0.smithi063.stdout:1292: done (10 left) 2024-08-17T17:00:11.799 INFO:tasks.rados.rados.0.smithi063.stdout:1293: done (9 left) 2024-08-17T17:00:11.799 INFO:tasks.rados.rados.0.smithi063.stdout:1294: done (8 left) 2024-08-17T17:00:11.799 INFO:tasks.rados.rados.0.smithi063.stdout:1295: done (7 left) 2024-08-17T17:00:11.799 INFO:tasks.rados.rados.0.smithi063.stdout:1296: done (6 left) 2024-08-17T17:00:11.799 INFO:tasks.rados.rados.0.smithi063.stdout:1297: done (5 left) 2024-08-17T17:00:11.799 INFO:tasks.rados.rados.0.smithi063.stdout:1298: done (4 left) 2024-08-17T17:00:11.799 INFO:tasks.rados.rados.0.smithi063.stdout:1299: done (3 left) 2024-08-17T17:00:11.799 INFO:tasks.rados.rados.0.smithi063.stdout:1300: done (2 left) 2024-08-17T17:00:11.799 INFO:tasks.rados.rados.0.smithi063.stdout:1302: done (1 left) 2024-08-17T17:00:11.799 INFO:tasks.rados.rados.0.smithi063.stdout:1303: done (0 left) 2024-08-17T17:00:11.799 INFO:tasks.rados.rados.0.smithi063.stdout:1304: write oid 25 current snap is 161 2024-08-17T17:00:11.799 INFO:tasks.rados.rados.0.smithi063.stdout:1304: seq_num 512 ranges {605184=723139,1877957=727227} 2024-08-17T17:00:11.805 INFO:tasks.rados.rados.0.smithi063.stdout:1304: writing smithi06387500-25 from 605184 to 1328323 tid 1 2024-08-17T17:00:11.813 INFO:tasks.rados.rados.0.smithi063.stdout:1304: writing smithi06387500-25 from 1877957 to 2605184 tid 2 2024-08-17T17:00:11.813 INFO:tasks.rados.rados.0.smithi063.stdout:1305: read oid 21 snap -1 2024-08-17T17:00:11.813 INFO:tasks.rados.rados.0.smithi063.stdout:1305: expect (ObjNum 503 snap 158 seq_num 503) 2024-08-17T17:00:11.813 INFO:tasks.rados.rados.0.smithi063.stdout:1306: rollback oid 5 current snap is 161 2024-08-17T17:00:11.813 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 141 2024-08-17T17:00:11.814 INFO:tasks.rados.rados.0.smithi063.stdout:1304: finishing write tid 1 to smithi06387500-25 2024-08-17T17:00:11.814 INFO:tasks.rados.rados.0.smithi063.stdout:1307: write oid 26 current snap is 161 2024-08-17T17:00:11.814 INFO:tasks.rados.rados.0.smithi063.stdout:1307: seq_num 513 ranges {621991=798388,1936439=639509} 2024-08-17T17:00:11.821 INFO:tasks.rados.rados.0.smithi063.stdout:1307: writing smithi06387500-26 from 621991 to 1420379 tid 1 2024-08-17T17:00:11.828 INFO:tasks.rados.rados.0.smithi063.stdout:1307: writing smithi06387500-26 from 1936439 to 2575948 tid 2 2024-08-17T17:00:11.828 INFO:tasks.rados.rados.0.smithi063.stdout:1308: write oid 42 current snap is 161 2024-08-17T17:00:11.828 INFO:tasks.rados.rados.0.smithi063.stdout:1308: seq_num 514 ranges {638798=473637,1594921=783167} 2024-08-17T17:00:11.834 INFO:tasks.rados.rados.0.smithi063.stdout:1308: writing smithi06387500-42 from 638798 to 1112435 tid 1 2024-08-17T17:00:11.841 INFO:tasks.rados.rados.0.smithi063.stdout:1308: writing smithi06387500-42 from 1594921 to 2378088 tid 2 2024-08-17T17:00:11.841 INFO:tasks.rados.rados.0.smithi063.stdout:1309: snap_create 2024-08-17T17:00:11.842 INFO:tasks.rados.rados.0.smithi063.stdout:1306: finishing rollback tid 0 to smithi06387500-5 2024-08-17T17:00:11.842 INFO:tasks.rados.rados.0.smithi063.stdout:1306: finishing rollback tid 1 to smithi06387500-5 2024-08-17T17:00:11.842 INFO:tasks.rados.rados.0.smithi063.stdout:1306: finishing rollback tid 2 to smithi06387500-5 2024-08-17T17:00:11.842 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 647 (ObjNum 443 snap 139 seq_num 443) dirty exists 2024-08-17T17:00:11.842 INFO:tasks.rados.rados.0.smithi063.stdout:1304: finishing write tid 2 to smithi06387500-25 2024-08-17T17:00:11.842 INFO:tasks.rados.rados.0.smithi063.stdout:1304: finishing write tid 3 to smithi06387500-25 2024-08-17T17:00:11.842 INFO:tasks.rados.rados.0.smithi063.stdout:1304: finishing write tid 4 to smithi06387500-25 2024-08-17T17:00:11.842 INFO:tasks.rados.rados.0.smithi063.stdout:1304: oid 25 updating version 0 to 1331 2024-08-17T17:00:11.842 INFO:tasks.rados.rados.0.smithi063.stdout:1304: oid 25 updating version 1331 to 1332 2024-08-17T17:00:11.842 INFO:tasks.rados.rados.0.smithi063.stdout:1304: oid 25 version 1332 is already newer than 1330 2024-08-17T17:00:11.842 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 1332 (ObjNum 512 snap 161 seq_num 512) dirty exists 2024-08-17T17:00:11.842 INFO:tasks.rados.rados.0.smithi063.stdout:1304: left oid 25 (ObjNum 512 snap 161 seq_num 512) 2024-08-17T17:00:11.842 INFO:tasks.rados.rados.0.smithi063.stdout:1305: expect (ObjNum 503 snap 158 seq_num 503) 2024-08-17T17:00:11.954 INFO:tasks.rados.rados.0.smithi063.stdout:1307: finishing write tid 1 to smithi06387500-26 2024-08-17T17:00:11.955 INFO:tasks.rados.rados.0.smithi063.stdout:1307: finishing write tid 2 to smithi06387500-26 2024-08-17T17:00:11.955 INFO:tasks.rados.rados.0.smithi063.stdout:1307: finishing write tid 3 to smithi06387500-26 2024-08-17T17:00:11.955 INFO:tasks.rados.rados.0.smithi063.stdout:1307: finishing write tid 4 to smithi06387500-26 2024-08-17T17:00:11.955 INFO:tasks.rados.rados.0.smithi063.stdout:1307: oid 26 updating version 0 to 401 2024-08-17T17:00:11.955 INFO:tasks.rados.rados.0.smithi063.stdout:1307: oid 26 updating version 401 to 403 2024-08-17T17:00:11.955 INFO:tasks.rados.rados.0.smithi063.stdout:1307: oid 26 version 403 is already newer than 402 2024-08-17T17:00:11.955 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 403 (ObjNum 513 snap 161 seq_num 513) dirty exists 2024-08-17T17:00:11.955 INFO:tasks.rados.rados.0.smithi063.stdout:1307: left oid 26 (ObjNum 513 snap 161 seq_num 513) 2024-08-17T17:00:11.955 INFO:tasks.rados.rados.0.smithi063.stdout:1308: finishing write tid 1 to smithi06387500-42 2024-08-17T17:00:11.955 INFO:tasks.rados.rados.0.smithi063.stdout:1308: finishing write tid 2 to smithi06387500-42 2024-08-17T17:00:11.955 INFO:tasks.rados.rados.0.smithi063.stdout:1308: finishing write tid 3 to smithi06387500-42 2024-08-17T17:00:11.955 INFO:tasks.rados.rados.0.smithi063.stdout:1308: finishing write tid 4 to smithi06387500-42 2024-08-17T17:00:11.955 INFO:tasks.rados.rados.0.smithi063.stdout:1308: oid 42 updating version 0 to 925 2024-08-17T17:00:11.956 INFO:tasks.rados.rados.0.smithi063.stdout:1308: oid 42 updating version 925 to 926 2024-08-17T17:00:11.956 INFO:tasks.rados.rados.0.smithi063.stdout:1308: oid 42 version 926 is already newer than 924 2024-08-17T17:00:11.956 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 926 (ObjNum 514 snap 161 seq_num 514) dirty exists 2024-08-17T17:00:11.956 INFO:tasks.rados.rados.0.smithi063.stdout:1308: left oid 42 (ObjNum 514 snap 161 seq_num 514) 2024-08-17T17:00:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:11 smithi162 ceph-mon[34633]: osdmap e390: 8 total, 8 up, 8 in 2024-08-17T17:00:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:11 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:12.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:11 smithi063 ceph-mon[114756]: osdmap e390: 8 total, 8 up, 8 in 2024-08-17T17:00:12.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:11 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:12.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:11 smithi063 ceph-mon[109843]: osdmap e390: 8 total, 8 up, 8 in 2024-08-17T17:00:12.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:11 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:00:12.508 INFO:tasks.rados.rados.0.smithi063.stdout:1304: done (5 left) 2024-08-17T17:00:12.508 INFO:tasks.rados.rados.0.smithi063.stdout:1305: done (4 left) 2024-08-17T17:00:12.508 INFO:tasks.rados.rados.0.smithi063.stdout:1306: done (3 left) 2024-08-17T17:00:12.508 INFO:tasks.rados.rados.0.smithi063.stdout:1307: done (2 left) 2024-08-17T17:00:12.509 INFO:tasks.rados.rados.0.smithi063.stdout:1308: done (1 left) 2024-08-17T17:00:12.509 INFO:tasks.rados.rados.0.smithi063.stdout:1309: done (0 left) 2024-08-17T17:00:12.509 INFO:tasks.rados.rados.0.smithi063.stdout:1310: read oid 24 snap -1 2024-08-17T17:00:12.509 INFO:tasks.rados.rados.0.smithi063.stdout:1310: expect deleted 2024-08-17T17:00:12.509 INFO:tasks.rados.rados.0.smithi063.stdout:1311: delete oid 31 current snap is 162 2024-08-17T17:00:12.512 INFO:tasks.rados.rados.0.smithi063.stdout:1310: done (1 left) 2024-08-17T17:00:12.512 INFO:tasks.rados.rados.0.smithi063.stdout:1311: done (0 left) 2024-08-17T17:00:12.512 INFO:tasks.rados.rados.0.smithi063.stdout:1312: delete oid 41 current snap is 162 2024-08-17T17:00:12.516 INFO:tasks.rados.rados.0.smithi063.stdout:1312: done (0 left) 2024-08-17T17:00:12.516 INFO:tasks.rados.rados.0.smithi063.stdout:1313: snap_remove snap 158 2024-08-17T17:00:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:12 smithi162 ceph-mon[34633]: osdmap e391: 8 total, 8 up, 8 in 2024-08-17T17:00:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:12 smithi162 ceph-mon[34633]: pgmap v40: 89 pgs: 89 active+clean; 214 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 2.5 MiB/s wr, 24 op/s 2024-08-17T17:00:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:12 smithi162 ceph-mon[34633]: osdmap e392: 8 total, 8 up, 8 in 2024-08-17T17:00:13.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:12 smithi063 ceph-mon[114756]: osdmap e391: 8 total, 8 up, 8 in 2024-08-17T17:00:13.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:12 smithi063 ceph-mon[114756]: pgmap v40: 89 pgs: 89 active+clean; 214 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 2.5 MiB/s wr, 24 op/s 2024-08-17T17:00:13.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:12 smithi063 ceph-mon[114756]: osdmap e392: 8 total, 8 up, 8 in 2024-08-17T17:00:13.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:12 smithi063 ceph-mon[109843]: osdmap e391: 8 total, 8 up, 8 in 2024-08-17T17:00:13.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:12 smithi063 ceph-mon[109843]: pgmap v40: 89 pgs: 89 active+clean; 214 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 2.5 MiB/s wr, 24 op/s 2024-08-17T17:00:13.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:12 smithi063 ceph-mon[109843]: osdmap e392: 8 total, 8 up, 8 in 2024-08-17T17:00:13.510 INFO:tasks.rados.rados.0.smithi063.stdout:1313: done (0 left) 2024-08-17T17:00:13.510 INFO:tasks.rados.rados.0.smithi063.stdout:1314: snap_create 2024-08-17T17:00:13.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:00:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:00:13] "GET /metrics HTTP/1.1" 200 36143 "" "Prometheus/2.51.0" 2024-08-17T17:00:14.515 INFO:tasks.rados.rados.0.smithi063.stdout:1314: done (0 left) 2024-08-17T17:00:14.515 INFO:tasks.rados.rados.0.smithi063.stdout:1315: read oid 32 snap -1 2024-08-17T17:00:14.515 INFO:tasks.rados.rados.0.smithi063.stdout:1315: expect (ObjNum 423 snap 130 seq_num 423) 2024-08-17T17:00:14.516 INFO:tasks.rados.rados.0.smithi063.stdout:1316: rollback oid 49 current snap is 163 2024-08-17T17:00:14.516 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 49 to 160 2024-08-17T17:00:14.516 INFO:tasks.rados.rados.0.smithi063.stdout:1317: delete oid 50 current snap is 163 2024-08-17T17:00:14.523 INFO:tasks.rados.rados.0.smithi063.stdout:1316: finishing rollback tid 0 to smithi06387500-49 2024-08-17T17:00:14.523 INFO:tasks.rados.rados.0.smithi063.stdout:1317: done (2 left) 2024-08-17T17:00:14.523 INFO:tasks.rados.rados.0.smithi063.stdout:1318: read oid 37 snap 153 2024-08-17T17:00:14.523 INFO:tasks.rados.rados.0.smithi063.stdout:1318: expect deleted 2024-08-17T17:00:14.523 INFO:tasks.rados.rados.0.smithi063.stdout:1319: delete oid 28 current snap is 163 2024-08-17T17:00:14.523 INFO:tasks.rados.rados.0.smithi063.stdout:1316: finishing rollback tid 1 to smithi06387500-49 2024-08-17T17:00:14.523 INFO:tasks.rados.rados.0.smithi063.stdout:1316: finishing rollback tid 2 to smithi06387500-49 2024-08-17T17:00:14.523 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 1340 (ObjNum 449 snap 145 seq_num 449) dirty exists 2024-08-17T17:00:14.523 INFO:tasks.rados.rados.0.smithi063.stdout:1315: expect (ObjNum 423 snap 130 seq_num 423) 2024-08-17T17:00:14.712 INFO:tasks.rados.rados.0.smithi063.stdout:1315: done (3 left) 2024-08-17T17:00:14.712 INFO:tasks.rados.rados.0.smithi063.stdout:1316: done (2 left) 2024-08-17T17:00:14.712 INFO:tasks.rados.rados.0.smithi063.stdout:1318: done (1 left) 2024-08-17T17:00:14.713 INFO:tasks.rados.rados.0.smithi063.stdout:1319: done (0 left) 2024-08-17T17:00:14.713 INFO:tasks.rados.rados.0.smithi063.stdout:1320: read oid 3 snap 162 2024-08-17T17:00:14.713 INFO:tasks.rados.rados.0.smithi063.stdout:1320: expect deleted 2024-08-17T17:00:14.713 INFO:tasks.rados.rados.0.smithi063.stdout:1321: rollback oid 19 current snap is 163 2024-08-17T17:00:14.713 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 19 to 162 2024-08-17T17:00:14.713 INFO:tasks.rados.rados.0.smithi063.stdout:1322: write oid 23 current snap is 163 2024-08-17T17:00:14.713 INFO:tasks.rados.rados.0.smithi063.stdout:1322: seq_num 519 ranges {722833=566235,1887331=650516} 2024-08-17T17:00:14.719 INFO:tasks.rados.rados.0.smithi063.stdout:1322: writing smithi06387500-23 from 722833 to 1289068 tid 1 2024-08-17T17:00:14.726 INFO:tasks.rados.rados.0.smithi063.stdout:1322: writing smithi06387500-23 from 1887331 to 2537847 tid 2 2024-08-17T17:00:14.726 INFO:tasks.rados.rados.0.smithi063.stdout:1323: snap_create 2024-08-17T17:00:14.726 INFO:tasks.rados.rados.0.smithi063.stdout:1321: finishing rollback tid 0 to smithi06387500-19 2024-08-17T17:00:14.726 INFO:tasks.rados.rados.0.smithi063.stdout:1321: finishing rollback tid 1 to smithi06387500-19 2024-08-17T17:00:14.727 INFO:tasks.rados.rados.0.smithi063.stdout:1321: finishing rollback tid 2 to smithi06387500-19 2024-08-17T17:00:14.727 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 938 (ObjNum 505 snap 158 seq_num 505) dirty exists 2024-08-17T17:00:14.727 INFO:tasks.rados.rados.0.smithi063.stdout:1322: finishing write tid 1 to smithi06387500-23 2024-08-17T17:00:14.730 INFO:tasks.rados.rados.0.smithi063.stdout:1322: finishing write tid 2 to smithi06387500-23 2024-08-17T17:00:14.730 INFO:tasks.rados.rados.0.smithi063.stdout:1322: finishing write tid 3 to smithi06387500-23 2024-08-17T17:00:14.730 INFO:tasks.rados.rados.0.smithi063.stdout:1322: finishing write tid 4 to smithi06387500-23 2024-08-17T17:00:14.730 INFO:tasks.rados.rados.0.smithi063.stdout:1322: oid 23 updating version 0 to 653 2024-08-17T17:00:14.730 INFO:tasks.rados.rados.0.smithi063.stdout:1322: oid 23 updating version 653 to 655 2024-08-17T17:00:14.730 INFO:tasks.rados.rados.0.smithi063.stdout:1322: oid 23 version 655 is already newer than 654 2024-08-17T17:00:14.730 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 655 (ObjNum 519 snap 163 seq_num 519) dirty exists 2024-08-17T17:00:14.731 INFO:tasks.rados.rados.0.smithi063.stdout:1322: left oid 23 (ObjNum 519 snap 163 seq_num 519) 2024-08-17T17:00:14.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:14 smithi063 ceph-mon[114756]: osdmap e393: 8 total, 8 up, 8 in 2024-08-17T17:00:14.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:14 smithi063 ceph-mon[109843]: osdmap e393: 8 total, 8 up, 8 in 2024-08-17T17:00:14.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:14 smithi162 ceph-mon[34633]: osdmap e393: 8 total, 8 up, 8 in 2024-08-17T17:00:15.524 INFO:tasks.rados.rados.0.smithi063.stdout:1320: done (3 left) 2024-08-17T17:00:15.524 INFO:tasks.rados.rados.0.smithi063.stdout:1321: done (2 left) 2024-08-17T17:00:15.524 INFO:tasks.rados.rados.0.smithi063.stdout:1322: done (1 left) 2024-08-17T17:00:15.525 INFO:tasks.rados.rados.0.smithi063.stdout:1323: done (0 left) 2024-08-17T17:00:15.525 INFO:tasks.rados.rados.0.smithi063.stdout:1324: delete oid 41 current snap is 164 2024-08-17T17:00:15.527 INFO:tasks.rados.rados.0.smithi063.stdout:1324: done (0 left) 2024-08-17T17:00:15.527 INFO:tasks.rados.rados.0.smithi063.stdout:1325: delete oid 45 current snap is 164 2024-08-17T17:00:15.529 INFO:tasks.rados.rados.0.smithi063.stdout:1325: done (0 left) 2024-08-17T17:00:15.530 INFO:tasks.rados.rados.0.smithi063.stdout:1326: snap_create 2024-08-17T17:00:15.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:15 smithi063 ceph-mon[114756]: pgmap v43: 89 pgs: 89 active+clean; 214 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 2.5 MiB/s wr, 24 op/s 2024-08-17T17:00:15.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:15 smithi063 ceph-mon[114756]: osdmap e394: 8 total, 8 up, 8 in 2024-08-17T17:00:15.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:15 smithi063 ceph-mon[109843]: pgmap v43: 89 pgs: 89 active+clean; 214 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 2.5 MiB/s wr, 24 op/s 2024-08-17T17:00:15.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:15 smithi063 ceph-mon[109843]: osdmap e394: 8 total, 8 up, 8 in 2024-08-17T17:00:15.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:15 smithi162 ceph-mon[34633]: pgmap v43: 89 pgs: 89 active+clean; 214 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 14 MiB/s rd, 2.5 MiB/s wr, 24 op/s 2024-08-17T17:00:15.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:15 smithi162 ceph-mon[34633]: osdmap e394: 8 total, 8 up, 8 in 2024-08-17T17:00:16.529 INFO:tasks.rados.rados.0.smithi063.stdout:1326: done (0 left) 2024-08-17T17:00:16.529 INFO:tasks.rados.rados.0.smithi063.stdout:1327: write oid 34 current snap is 165 2024-08-17T17:00:16.530 INFO:tasks.rados.rados.0.smithi063.stdout:1327: seq_num 522 ranges {773254=791982,2346424=426830} 2024-08-17T17:00:16.538 INFO:tasks.rados.rados.0.smithi063.stdout:1327: writing smithi06387500-34 from 773254 to 1565236 tid 1 2024-08-17T17:00:16.545 INFO:tasks.rados.rados.0.smithi063.stdout:1327: writing smithi06387500-34 from 2346424 to 2773254 tid 2 2024-08-17T17:00:16.545 INFO:tasks.rados.rados.0.smithi063.stdout:1328: read oid 50 snap -1 2024-08-17T17:00:16.545 INFO:tasks.rados.rados.0.smithi063.stdout:1328: expect deleted 2024-08-17T17:00:16.545 INFO:tasks.rados.rados.0.smithi063.stdout:1329: read oid 19 snap -1 2024-08-17T17:00:16.545 INFO:tasks.rados.rados.0.smithi063.stdout:1329: expect (ObjNum 505 snap 158 seq_num 505) 2024-08-17T17:00:16.545 INFO:tasks.rados.rados.0.smithi063.stdout:1330: snap_remove snap 160 2024-08-17T17:00:16.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:16 smithi063 ceph-mon[114756]: osdmap e395: 8 total, 8 up, 8 in 2024-08-17T17:00:16.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:16 smithi063 ceph-mon[109843]: osdmap e395: 8 total, 8 up, 8 in 2024-08-17T17:00:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:16 smithi162 ceph-mon[34633]: osdmap e395: 8 total, 8 up, 8 in 2024-08-17T17:00:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:16 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:17.532 INFO:tasks.rados.rados.0.smithi063.stdout:1330: done (3 left) 2024-08-17T17:00:17.532 INFO:tasks.rados.rados.0.smithi063.stdout:1331: write oid 8 current snap is 165 2024-08-17T17:00:17.532 INFO:tasks.rados.rados.0.smithi063.stdout:1327: finishing write tid 1 to smithi06387500-34 2024-08-17T17:00:17.532 INFO:tasks.rados.rados.0.smithi063.stdout:1331: seq_num 523 ranges {790061=467231,2004906=541501} 2024-08-17T17:00:17.538 INFO:tasks.rados.rados.0.smithi063.stdout:1331: writing smithi06387500-8 from 790061 to 1257292 tid 1 2024-08-17T17:00:17.547 INFO:tasks.rados.rados.0.smithi063.stdout:1331: writing smithi06387500-8 from 2004906 to 2546407 tid 2 2024-08-17T17:00:17.548 INFO:tasks.rados.rados.0.smithi063.stdout:1327: finishing write tid 2 to smithi06387500-34 2024-08-17T17:00:17.548 INFO:tasks.rados.rados.0.smithi063.stdout:1327: finishing write tid 3 to smithi06387500-34 2024-08-17T17:00:17.548 INFO:tasks.rados.rados.0.smithi063.stdout:1332: read oid 1 snap -1 2024-08-17T17:00:17.548 INFO:tasks.rados.rados.0.smithi063.stdout:1332: expect (ObjNum 511 snap 160 seq_num 511) 2024-08-17T17:00:17.548 INFO:tasks.rados.rados.0.smithi063.stdout:1333: read oid 43 snap -1 2024-08-17T17:00:17.548 INFO:tasks.rados.rados.0.smithi063.stdout:1333: expect (ObjNum 483 snap 153 seq_num 483) 2024-08-17T17:00:17.548 INFO:tasks.rados.rados.0.smithi063.stdout:1327: finishing write tid 4 to smithi06387500-34 2024-08-17T17:00:17.548 INFO:tasks.rados.rados.0.smithi063.stdout:1327: oid 34 updating version 0 to 939 2024-08-17T17:00:17.548 INFO:tasks.rados.rados.0.smithi063.stdout:1327: oid 34 updating version 939 to 941 2024-08-17T17:00:17.548 INFO:tasks.rados.rados.0.smithi063.stdout:1327: oid 34 version 941 is already newer than 940 2024-08-17T17:00:17.548 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 941 (ObjNum 522 snap 165 seq_num 522) dirty exists 2024-08-17T17:00:17.548 INFO:tasks.rados.rados.0.smithi063.stdout:1327: left oid 34 (ObjNum 522 snap 165 seq_num 522) 2024-08-17T17:00:17.548 INFO:tasks.rados.rados.0.smithi063.stdout:1334: read oid 21 snap -1 2024-08-17T17:00:17.549 INFO:tasks.rados.rados.0.smithi063.stdout:1334: expect (ObjNum 503 snap 158 seq_num 503) 2024-08-17T17:00:17.549 INFO:tasks.rados.rados.0.smithi063.stdout:1327: done (6 left) 2024-08-17T17:00:17.549 INFO:tasks.rados.rados.0.smithi063.stdout:1328: done (5 left) 2024-08-17T17:00:17.549 INFO:tasks.rados.rados.0.smithi063.stdout:1335: snap_create 2024-08-17T17:00:17.549 INFO:tasks.rados.rados.0.smithi063.stdout:1329: expect (ObjNum 505 snap 158 seq_num 505) 2024-08-17T17:00:17.691 INFO:tasks.rados.rados.0.smithi063.stdout:1331: finishing write tid 1 to smithi06387500-8 2024-08-17T17:00:17.691 INFO:tasks.rados.rados.0.smithi063.stdout:1331: finishing write tid 2 to smithi06387500-8 2024-08-17T17:00:17.691 INFO:tasks.rados.rados.0.smithi063.stdout:1331: finishing write tid 3 to smithi06387500-8 2024-08-17T17:00:17.691 INFO:tasks.rados.rados.0.smithi063.stdout:1331: finishing write tid 4 to smithi06387500-8 2024-08-17T17:00:17.691 INFO:tasks.rados.rados.0.smithi063.stdout:1331: oid 8 updating version 0 to 396 2024-08-17T17:00:17.691 INFO:tasks.rados.rados.0.smithi063.stdout:1331: oid 8 version 396 is already newer than 395 2024-08-17T17:00:17.691 INFO:tasks.rados.rados.0.smithi063.stdout:1331: oid 8 updating version 396 to 397 2024-08-17T17:00:17.691 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 397 (ObjNum 523 snap 165 seq_num 523) dirty exists 2024-08-17T17:00:17.692 INFO:tasks.rados.rados.0.smithi063.stdout:1331: left oid 8 (ObjNum 523 snap 165 seq_num 523) 2024-08-17T17:00:17.692 INFO:tasks.rados.rados.0.smithi063.stdout:1334: expect (ObjNum 503 snap 158 seq_num 503) 2024-08-17T17:00:17.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:17 smithi063 ceph-mon[114756]: pgmap v46: 89 pgs: 89 active+clean; 214 MiB data, 693 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:00:17.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:17 smithi063 ceph-mon[114756]: osdmap e396: 8 total, 8 up, 8 in 2024-08-17T17:00:17.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:17 smithi063 ceph-mon[109843]: pgmap v46: 89 pgs: 89 active+clean; 214 MiB data, 693 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:00:17.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:17 smithi063 ceph-mon[109843]: osdmap e396: 8 total, 8 up, 8 in 2024-08-17T17:00:17.803 INFO:tasks.rados.rados.0.smithi063.stdout:1332: expect (ObjNum 511 snap 160 seq_num 511) 2024-08-17T17:00:17.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:17 smithi162 ceph-mon[34633]: pgmap v46: 89 pgs: 89 active+clean; 214 MiB data, 693 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:00:17.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:17 smithi162 ceph-mon[34633]: osdmap e396: 8 total, 8 up, 8 in 2024-08-17T17:00:17.981 INFO:tasks.rados.rados.0.smithi063.stdout:1333: expect (ObjNum 483 snap 153 seq_num 483) 2024-08-17T17:00:18.537 INFO:tasks.rados.rados.0.smithi063.stdout:1329: done (5 left) 2024-08-17T17:00:18.537 INFO:tasks.rados.rados.0.smithi063.stdout:1331: done (4 left) 2024-08-17T17:00:18.537 INFO:tasks.rados.rados.0.smithi063.stdout:1332: done (3 left) 2024-08-17T17:00:18.537 INFO:tasks.rados.rados.0.smithi063.stdout:1333: done (2 left) 2024-08-17T17:00:18.537 INFO:tasks.rados.rados.0.smithi063.stdout:1334: done (1 left) 2024-08-17T17:00:18.537 INFO:tasks.rados.rados.0.smithi063.stdout:1335: done (0 left) 2024-08-17T17:00:18.537 INFO:tasks.rados.rados.0.smithi063.stdout:1336: write oid 19 current snap is 166 2024-08-17T17:00:18.537 INFO:tasks.rados.rados.0.smithi063.stdout:1336: seq_num 524 ranges {406868=542480,1547035=401512,2690354=116514} 2024-08-17T17:00:18.541 INFO:tasks.rados.rados.0.smithi063.stdout:1336: writing smithi06387500-19 from 406868 to 949348 tid 1 2024-08-17T17:00:18.547 INFO:tasks.rados.rados.0.smithi063.stdout:1336: writing smithi06387500-19 from 1547035 to 1948547 tid 2 2024-08-17T17:00:18.553 INFO:tasks.rados.rados.0.smithi063.stdout:1336: writing smithi06387500-19 from 2690354 to 2806868 tid 3 2024-08-17T17:00:18.553 INFO:tasks.rados.rados.0.smithi063.stdout:1337: rollback oid 15 current snap is 166 2024-08-17T17:00:18.553 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 15 to 141 2024-08-17T17:00:18.553 INFO:tasks.rados.rados.0.smithi063.stdout:1336: finishing write tid 1 to smithi06387500-19 2024-08-17T17:00:18.553 INFO:tasks.rados.rados.0.smithi063.stdout:1336: finishing write tid 2 to smithi06387500-19 2024-08-17T17:00:18.553 INFO:tasks.rados.rados.0.smithi063.stdout:1338: read oid 38 snap -1 2024-08-17T17:00:18.554 INFO:tasks.rados.rados.0.smithi063.stdout:1338: expect (ObjNum 504 snap 158 seq_num 504) 2024-08-17T17:00:18.554 INFO:tasks.rados.rados.0.smithi063.stdout:1339: read oid 2 snap -1 2024-08-17T17:00:18.554 INFO:tasks.rados.rados.0.smithi063.stdout:1339: expect deleted 2024-08-17T17:00:18.554 INFO:tasks.rados.rados.0.smithi063.stdout:1340: delete oid 5 current snap is 166 2024-08-17T17:00:18.556 INFO:tasks.rados.rados.0.smithi063.stdout:1337: finishing rollback tid 0 to smithi06387500-15 2024-08-17T17:00:18.556 INFO:tasks.rados.rados.0.smithi063.stdout:1336: finishing write tid 3 to smithi06387500-19 2024-08-17T17:00:18.556 INFO:tasks.rados.rados.0.smithi063.stdout:1336: finishing write tid 4 to smithi06387500-19 2024-08-17T17:00:18.556 INFO:tasks.rados.rados.0.smithi063.stdout:1336: finishing write tid 5 to smithi06387500-19 2024-08-17T17:00:18.557 INFO:tasks.rados.rados.0.smithi063.stdout:1336: oid 19 updating version 0 to 954 2024-08-17T17:00:18.557 INFO:tasks.rados.rados.0.smithi063.stdout:1336: oid 19 version 954 is already newer than 953 2024-08-17T17:00:18.557 INFO:tasks.rados.rados.0.smithi063.stdout:1336: oid 19 version 954 is already newer than 951 2024-08-17T17:00:18.557 INFO:tasks.rados.rados.0.smithi063.stdout:1336: oid 19 version 954 is already newer than 952 2024-08-17T17:00:18.557 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 954 (ObjNum 524 snap 166 seq_num 524) dirty exists 2024-08-17T17:00:18.557 INFO:tasks.rados.rados.0.smithi063.stdout:1336: left oid 19 (ObjNum 524 snap 166 seq_num 524) 2024-08-17T17:00:18.557 INFO:tasks.rados.rados.0.smithi063.stdout:1337: finishing rollback tid 1 to smithi06387500-15 2024-08-17T17:00:18.557 INFO:tasks.rados.rados.0.smithi063.stdout:1337: finishing rollback tid 2 to smithi06387500-15 2024-08-17T17:00:18.557 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 503 (ObjNum 289 snap 88 seq_num 289) dirty exists 2024-08-17T17:00:18.558 INFO:tasks.rados.rados.0.smithi063.stdout:1336: done (4 left) 2024-08-17T17:00:18.559 INFO:tasks.rados.rados.0.smithi063.stdout:1337: done (3 left) 2024-08-17T17:00:18.559 INFO:tasks.rados.rados.0.smithi063.stdout:1339: done (2 left) 2024-08-17T17:00:18.559 INFO:tasks.rados.rados.0.smithi063.stdout:1340: done (1 left) 2024-08-17T17:00:18.559 INFO:tasks.rados.rados.0.smithi063.stdout:1341: read oid 2 snap 159 2024-08-17T17:00:18.559 INFO:tasks.rados.rados.0.smithi063.stdout:1341: expect deleted 2024-08-17T17:00:18.559 INFO:tasks.rados.rados.0.smithi063.stdout:1342: read oid 12 snap -1 2024-08-17T17:00:18.559 INFO:tasks.rados.rados.0.smithi063.stdout:1342: expect (ObjNum 369 snap 115 seq_num 369) 2024-08-17T17:00:18.559 INFO:tasks.rados.rados.0.smithi063.stdout:1343: rollback oid 18 current snap is 166 2024-08-17T17:00:18.559 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 18 to 157 2024-08-17T17:00:18.559 INFO:tasks.rados.rados.0.smithi063.stdout:1344: rollback oid 42 current snap is 166 2024-08-17T17:00:18.560 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 42 to 162 2024-08-17T17:00:18.560 INFO:tasks.rados.rados.0.smithi063.stdout:1345: snap_remove snap 156 2024-08-17T17:00:18.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:18 smithi162 ceph-mon[34633]: osdmap e397: 8 total, 8 up, 8 in 2024-08-17T17:00:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:18 smithi063 ceph-mon[114756]: osdmap e397: 8 total, 8 up, 8 in 2024-08-17T17:00:19.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:18 smithi063 ceph-mon[109843]: osdmap e397: 8 total, 8 up, 8 in 2024-08-17T17:00:19.545 INFO:tasks.rados.rados.0.smithi063.stdout:1345: done (5 left) 2024-08-17T17:00:19.545 INFO:tasks.rados.rados.0.smithi063.stdout:1346: delete oid 16 current snap is 166 2024-08-17T17:00:19.545 INFO:tasks.rados.rados.0.smithi063.stdout:1338: expect (ObjNum 504 snap 158 seq_num 504) 2024-08-17T17:00:19.636 INFO:tasks.rados.rados.0.smithi063.stdout:1343: finishing rollback tid 0 to smithi06387500-18 2024-08-17T17:00:19.644 INFO:tasks.rados.rados.0.smithi063.stdout:1343: finishing rollback tid 1 to smithi06387500-18 2024-08-17T17:00:19.644 INFO:tasks.rados.rados.0.smithi063.stdout:1343: finishing rollback tid 2 to smithi06387500-18 2024-08-17T17:00:19.644 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 490 (ObjNum 490 snap 155 seq_num 490) dirty exists 2024-08-17T17:00:19.644 INFO:tasks.rados.rados.0.smithi063.stdout:1342: expect (ObjNum 369 snap 115 seq_num 369) 2024-08-17T17:00:19.753 INFO:tasks.rados.rados.0.smithi063.stdout:1344: finishing rollback tid 0 to smithi06387500-42 2024-08-17T17:00:19.753 INFO:tasks.rados.rados.0.smithi063.stdout:1344: finishing rollback tid 1 to smithi06387500-42 2024-08-17T17:00:19.753 INFO:tasks.rados.rados.0.smithi063.stdout:1344: finishing rollback tid 2 to smithi06387500-42 2024-08-17T17:00:19.753 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 958 (ObjNum 514 snap 161 seq_num 514) dirty exists 2024-08-17T17:00:19.753 INFO:tasks.rados.rados.0.smithi063.stdout:1338: done (5 left) 2024-08-17T17:00:19.753 INFO:tasks.rados.rados.0.smithi063.stdout:1341: done (4 left) 2024-08-17T17:00:19.753 INFO:tasks.rados.rados.0.smithi063.stdout:1342: done (3 left) 2024-08-17T17:00:19.753 INFO:tasks.rados.rados.0.smithi063.stdout:1343: done (2 left) 2024-08-17T17:00:19.753 INFO:tasks.rados.rados.0.smithi063.stdout:1344: done (1 left) 2024-08-17T17:00:19.754 INFO:tasks.rados.rados.0.smithi063.stdout:1346: done (0 left) 2024-08-17T17:00:19.754 INFO:tasks.rados.rados.0.smithi063.stdout:1347: snap_remove snap 155 2024-08-17T17:00:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:19 smithi162 ceph-mon[34633]: pgmap v49: 89 pgs: 89 active+clean; 218 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 595 KiB/s wr, 8 op/s 2024-08-17T17:00:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:19 smithi162 ceph-mon[34633]: osdmap e398: 8 total, 8 up, 8 in 2024-08-17T17:00:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:19 smithi063 ceph-mon[114756]: pgmap v49: 89 pgs: 89 active+clean; 218 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 595 KiB/s wr, 8 op/s 2024-08-17T17:00:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:19 smithi063 ceph-mon[114756]: osdmap e398: 8 total, 8 up, 8 in 2024-08-17T17:00:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:19 smithi063 ceph-mon[109843]: pgmap v49: 89 pgs: 89 active+clean; 218 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 595 KiB/s wr, 8 op/s 2024-08-17T17:00:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:19 smithi063 ceph-mon[109843]: osdmap e398: 8 total, 8 up, 8 in 2024-08-17T17:00:20.554 INFO:tasks.rados.rados.0.smithi063.stdout:1347: done (0 left) 2024-08-17T17:00:20.554 INFO:tasks.rados.rados.0.smithi063.stdout:1348: read oid 19 snap 130 2024-08-17T17:00:20.554 INFO:tasks.rados.rados.0.smithi063.stdout:1348: expect (ObjNum 334 snap 101 seq_num 334) 2024-08-17T17:00:20.554 INFO:tasks.rados.rados.0.smithi063.stdout:1349: write oid 21 current snap is 166 2024-08-17T17:00:20.554 INFO:tasks.rados.rados.0.smithi063.stdout:1349: seq_num 527 ranges {457289=484580,1438834=548839,2772623=84666} 2024-08-17T17:00:20.559 INFO:tasks.rados.rados.0.smithi063.stdout:1349: writing smithi06387500-21 from 457289 to 941869 tid 1 2024-08-17T17:00:20.566 INFO:tasks.rados.rados.0.smithi063.stdout:1349: writing smithi06387500-21 from 1438834 to 1987673 tid 2 2024-08-17T17:00:20.571 INFO:tasks.rados.rados.0.smithi063.stdout:1349: writing smithi06387500-21 from 2772623 to 2857289 tid 3 2024-08-17T17:00:20.571 INFO:tasks.rados.rados.0.smithi063.stdout:1350: write oid 44 current snap is 166 2024-08-17T17:00:20.571 INFO:tasks.rados.rados.0.smithi063.stdout:1350: seq_num 528 ranges {474096=559829,1780963=408850,2800046=74050} 2024-08-17T17:00:20.577 INFO:tasks.rados.rados.0.smithi063.stdout:1350: writing smithi06387500-44 from 474096 to 1033925 tid 1 2024-08-17T17:00:20.583 INFO:tasks.rados.rados.0.smithi063.stdout:1350: writing smithi06387500-44 from 1780963 to 2189813 tid 2 2024-08-17T17:00:20.588 INFO:tasks.rados.rados.0.smithi063.stdout:1350: writing smithi06387500-44 from 2800046 to 2874096 tid 3 2024-08-17T17:00:20.588 INFO:tasks.rados.rados.0.smithi063.stdout:1351: read oid 35 snap -1 2024-08-17T17:00:20.588 INFO:tasks.rados.rados.0.smithi063.stdout:1351: expect (ObjNum 450 snap 145 seq_num 450) 2024-08-17T17:00:20.588 INFO:tasks.rados.rados.0.smithi063.stdout:1352: read oid 50 snap -1 2024-08-17T17:00:20.588 INFO:tasks.rados.rados.0.smithi063.stdout:1352: expect deleted 2024-08-17T17:00:20.588 INFO:tasks.rados.rados.0.smithi063.stdout:1353: write oid 26 current snap is 166 2024-08-17T17:00:20.588 INFO:tasks.rados.rados.0.smithi063.stdout:1353: seq_num 529 ranges {490903=635078,1839445=552508} 2024-08-17T17:00:20.593 INFO:tasks.rados.rados.0.smithi063.stdout:1353: writing smithi06387500-26 from 490903 to 1125981 tid 1 2024-08-17T17:00:20.600 INFO:tasks.rados.rados.0.smithi063.stdout:1353: writing smithi06387500-26 from 1839445 to 2391953 tid 2 2024-08-17T17:00:20.600 INFO:tasks.rados.rados.0.smithi063.stdout:1354: snap_remove snap 153 2024-08-17T17:00:20.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:20 smithi162 ceph-mon[34633]: osdmap e399: 8 total, 8 up, 8 in 2024-08-17T17:00:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:20 smithi063 ceph-mon[114756]: osdmap e399: 8 total, 8 up, 8 in 2024-08-17T17:00:21.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:20 smithi063 ceph-mon[109843]: osdmap e399: 8 total, 8 up, 8 in 2024-08-17T17:00:21.564 INFO:tasks.rados.rados.0.smithi063.stdout:1354: done (6 left) 2024-08-17T17:00:21.565 INFO:tasks.rados.rados.0.smithi063.stdout:1355: rollback oid 29 current snap is 166 2024-08-17T17:00:21.565 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 29 to 165 2024-08-17T17:00:21.565 INFO:tasks.rados.rados.0.smithi063.stdout:1356: read oid 3 snap -1 2024-08-17T17:00:21.565 INFO:tasks.rados.rados.0.smithi063.stdout:1356: expect deleted 2024-08-17T17:00:21.565 INFO:tasks.rados.rados.0.smithi063.stdout:1357: write oid 12 current snap is 166 2024-08-17T17:00:21.565 INFO:tasks.rados.rados.0.smithi063.stdout:1357: seq_num 530 ranges {507710=710327,1897927=412519,2854892=52818} 2024-08-17T17:00:21.572 INFO:tasks.rados.rados.0.smithi063.stdout:1357: writing smithi06387500-12 from 507710 to 1218037 tid 1 2024-08-17T17:00:21.579 INFO:tasks.rados.rados.0.smithi063.stdout:1357: writing smithi06387500-12 from 1897927 to 2310446 tid 2 2024-08-17T17:00:21.583 INFO:tasks.rados.rados.0.smithi063.stdout:1357: writing smithi06387500-12 from 2854892 to 2907710 tid 3 2024-08-17T17:00:21.583 INFO:tasks.rados.rados.0.smithi063.stdout:1358: write oid 31 current snap is 166 2024-08-17T17:00:21.583 INFO:tasks.rados.rados.0.smithi063.stdout:1348: expect (ObjNum 334 snap 101 seq_num 334) 2024-08-17T17:00:21.750 INFO:tasks.rados.rados.0.smithi063.stdout:1349: finishing write tid 1 to smithi06387500-21 2024-08-17T17:00:21.750 INFO:tasks.rados.rados.0.smithi063.stdout:1349: finishing write tid 2 to smithi06387500-21 2024-08-17T17:00:21.750 INFO:tasks.rados.rados.0.smithi063.stdout:1358: seq_num 531 ranges {524517=785576,1956409=556177} 2024-08-17T17:00:21.757 INFO:tasks.rados.rados.0.smithi063.stdout:1358: writing smithi06387500-31 from 524517 to 1310093 tid 1 2024-08-17T17:00:21.764 INFO:tasks.rados.rados.0.smithi063.stdout:1358: writing smithi06387500-31 from 1956409 to 2512586 tid 2 2024-08-17T17:00:21.764 INFO:tasks.rados.rados.0.smithi063.stdout:1348: done (9 left) 2024-08-17T17:00:21.764 INFO:tasks.rados.rados.0.smithi063.stdout:1359: delete oid 25 current snap is 166 2024-08-17T17:00:21.764 INFO:tasks.rados.rados.0.smithi063.stdout:1349: finishing write tid 3 to smithi06387500-21 2024-08-17T17:00:21.764 INFO:tasks.rados.rados.0.smithi063.stdout:1349: finishing write tid 4 to smithi06387500-21 2024-08-17T17:00:21.764 INFO:tasks.rados.rados.0.smithi063.stdout:1349: finishing write tid 5 to smithi06387500-21 2024-08-17T17:00:21.765 INFO:tasks.rados.rados.0.smithi063.stdout:1349: oid 21 updating version 0 to 977 2024-08-17T17:00:21.765 INFO:tasks.rados.rados.0.smithi063.stdout:1349: oid 21 version 977 is already newer than 970 2024-08-17T17:00:21.765 INFO:tasks.rados.rados.0.smithi063.stdout:1349: oid 21 version 977 is already newer than 975 2024-08-17T17:00:21.765 INFO:tasks.rados.rados.0.smithi063.stdout:1349: oid 21 version 977 is already newer than 976 2024-08-17T17:00:21.765 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 977 (ObjNum 527 snap 166 seq_num 527) dirty exists 2024-08-17T17:00:21.765 INFO:tasks.rados.rados.0.smithi063.stdout:1349: left oid 21 (ObjNum 527 snap 166 seq_num 527) 2024-08-17T17:00:21.765 INFO:tasks.rados.rados.0.smithi063.stdout:1350: finishing write tid 1 to smithi06387500-44 2024-08-17T17:00:21.765 INFO:tasks.rados.rados.0.smithi063.stdout:1350: finishing write tid 2 to smithi06387500-44 2024-08-17T17:00:21.765 INFO:tasks.rados.rados.0.smithi063.stdout:1351: expect (ObjNum 450 snap 145 seq_num 450) 2024-08-17T17:00:21.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:21 smithi162 ceph-mon[34633]: pgmap v52: 89 pgs: 89 active+clean; 218 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 595 KiB/s wr, 8 op/s 2024-08-17T17:00:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:21 smithi162 ceph-mon[34633]: osdmap e400: 8 total, 8 up, 8 in 2024-08-17T17:00:21.884 INFO:tasks.rados.rados.0.smithi063.stdout:1350: finishing write tid 3 to smithi06387500-44 2024-08-17T17:00:21.884 INFO:tasks.rados.rados.0.smithi063.stdout:1350: finishing write tid 4 to smithi06387500-44 2024-08-17T17:00:21.884 INFO:tasks.rados.rados.0.smithi063.stdout:1349: done (9 left) 2024-08-17T17:00:21.884 INFO:tasks.rados.rados.0.smithi063.stdout:1351: done (8 left) 2024-08-17T17:00:21.884 INFO:tasks.rados.rados.0.smithi063.stdout:1352: done (7 left) 2024-08-17T17:00:21.884 INFO:tasks.rados.rados.0.smithi063.stdout:1359: done (6 left) 2024-08-17T17:00:21.884 INFO:tasks.rados.rados.0.smithi063.stdout:1360: read oid 27 snap -1 2024-08-17T17:00:21.884 INFO:tasks.rados.rados.0.smithi063.stdout:1360: expect deleted 2024-08-17T17:00:21.884 INFO:tasks.rados.rados.0.smithi063.stdout:1361: read oid 28 snap 165 2024-08-17T17:00:21.884 INFO:tasks.rados.rados.0.smithi063.stdout:1361: expect deleted 2024-08-17T17:00:21.885 INFO:tasks.rados.rados.0.smithi063.stdout:1362: snap_create 2024-08-17T17:00:21.885 INFO:tasks.rados.rados.0.smithi063.stdout:1350: finishing write tid 5 to smithi06387500-44 2024-08-17T17:00:21.885 INFO:tasks.rados.rados.0.smithi063.stdout:1350: oid 44 updating version 0 to 262 2024-08-17T17:00:21.885 INFO:tasks.rados.rados.0.smithi063.stdout:1350: oid 44 updating version 262 to 263 2024-08-17T17:00:21.885 INFO:tasks.rados.rados.0.smithi063.stdout:1350: oid 44 version 263 is already newer than 261 2024-08-17T17:00:21.885 INFO:tasks.rados.rados.0.smithi063.stdout:1350: oid 44 version 263 is already newer than 260 2024-08-17T17:00:21.885 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 263 (ObjNum 528 snap 166 seq_num 528) dirty exists 2024-08-17T17:00:21.885 INFO:tasks.rados.rados.0.smithi063.stdout:1350: left oid 44 (ObjNum 528 snap 166 seq_num 528) 2024-08-17T17:00:21.885 INFO:tasks.rados.rados.0.smithi063.stdout:1353: finishing write tid 1 to smithi06387500-26 2024-08-17T17:00:21.885 INFO:tasks.rados.rados.0.smithi063.stdout:1353: finishing write tid 2 to smithi06387500-26 2024-08-17T17:00:21.885 INFO:tasks.rados.rados.0.smithi063.stdout:1353: finishing write tid 3 to smithi06387500-26 2024-08-17T17:00:21.885 INFO:tasks.rados.rados.0.smithi063.stdout:1353: finishing write tid 4 to smithi06387500-26 2024-08-17T17:00:21.886 INFO:tasks.rados.rados.0.smithi063.stdout:1353: oid 26 updating version 0 to 413 2024-08-17T17:00:21.886 INFO:tasks.rados.rados.0.smithi063.stdout:1353: oid 26 updating version 413 to 414 2024-08-17T17:00:21.886 INFO:tasks.rados.rados.0.smithi063.stdout:1353: oid 26 updating version 414 to 415 2024-08-17T17:00:21.886 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 415 (ObjNum 529 snap 166 seq_num 529) dirty exists 2024-08-17T17:00:21.886 INFO:tasks.rados.rados.0.smithi063.stdout:1353: left oid 26 (ObjNum 529 snap 166 seq_num 529) 2024-08-17T17:00:21.886 INFO:tasks.rados.rados.0.smithi063.stdout:1355: finishing rollback tid 0 to smithi06387500-29 2024-08-17T17:00:21.886 INFO:tasks.rados.rados.0.smithi063.stdout:1355: finishing rollback tid 1 to smithi06387500-29 2024-08-17T17:00:21.886 INFO:tasks.rados.rados.0.smithi063.stdout:1355: finishing rollback tid 2 to smithi06387500-29 2024-08-17T17:00:21.886 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 284 (ObjNum 335 snap 101 seq_num 335) dirty exists 2024-08-17T17:00:21.886 INFO:tasks.rados.rados.0.smithi063.stdout:1357: finishing write tid 1 to smithi06387500-12 2024-08-17T17:00:21.886 INFO:tasks.rados.rados.0.smithi063.stdout:1357: finishing write tid 2 to smithi06387500-12 2024-08-17T17:00:21.886 INFO:tasks.rados.rados.0.smithi063.stdout:1357: finishing write tid 3 to smithi06387500-12 2024-08-17T17:00:21.886 INFO:tasks.rados.rados.0.smithi063.stdout:1357: finishing write tid 4 to smithi06387500-12 2024-08-17T17:00:21.886 INFO:tasks.rados.rados.0.smithi063.stdout:1357: finishing write tid 5 to smithi06387500-12 2024-08-17T17:00:21.887 INFO:tasks.rados.rados.0.smithi063.stdout:1357: oid 12 updating version 0 to 288 2024-08-17T17:00:21.887 INFO:tasks.rados.rados.0.smithi063.stdout:1357: oid 12 updating version 288 to 289 2024-08-17T17:00:21.887 INFO:tasks.rados.rados.0.smithi063.stdout:1357: oid 12 updating version 289 to 291 2024-08-17T17:00:21.887 INFO:tasks.rados.rados.0.smithi063.stdout:1357: oid 12 version 291 is already newer than 290 2024-08-17T17:00:21.887 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 291 (ObjNum 530 snap 166 seq_num 530) dirty exists 2024-08-17T17:00:21.887 INFO:tasks.rados.rados.0.smithi063.stdout:1357: left oid 12 (ObjNum 530 snap 166 seq_num 530) 2024-08-17T17:00:21.887 INFO:tasks.rados.rados.0.smithi063.stdout:1358: finishing write tid 1 to smithi06387500-31 2024-08-17T17:00:21.887 INFO:tasks.rados.rados.0.smithi063.stdout:1358: finishing write tid 2 to smithi06387500-31 2024-08-17T17:00:21.887 INFO:tasks.rados.rados.0.smithi063.stdout:1358: finishing write tid 3 to smithi06387500-31 2024-08-17T17:00:21.887 INFO:tasks.rados.rados.0.smithi063.stdout:1358: finishing write tid 4 to smithi06387500-31 2024-08-17T17:00:21.887 INFO:tasks.rados.rados.0.smithi063.stdout:1358: oid 31 updating version 0 to 992 2024-08-17T17:00:21.887 INFO:tasks.rados.rados.0.smithi063.stdout:1358: oid 31 version 992 is already newer than 991 2024-08-17T17:00:21.888 INFO:tasks.rados.rados.0.smithi063.stdout:1358: oid 31 version 992 is already newer than 990 2024-08-17T17:00:21.888 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 992 (ObjNum 531 snap 166 seq_num 531) dirty exists 2024-08-17T17:00:21.888 INFO:tasks.rados.rados.0.smithi063.stdout:1358: left oid 31 (ObjNum 531 snap 166 seq_num 531) 2024-08-17T17:00:21.908 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:21 smithi063 ceph-mon[114756]: pgmap v52: 89 pgs: 89 active+clean; 218 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 595 KiB/s wr, 8 op/s 2024-08-17T17:00:21.908 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:21 smithi063 ceph-mon[114756]: osdmap e400: 8 total, 8 up, 8 in 2024-08-17T17:00:21.908 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:21 smithi063 ceph-mon[109843]: pgmap v52: 89 pgs: 89 active+clean; 218 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 595 KiB/s wr, 8 op/s 2024-08-17T17:00:21.908 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:21 smithi063 ceph-mon[109843]: osdmap e400: 8 total, 8 up, 8 in 2024-08-17T17:00:22.513 INFO:tasks.rados.rados.0.smithi063.stdout:1350: done (8 left) 2024-08-17T17:00:22.513 INFO:tasks.rados.rados.0.smithi063.stdout:1353: done (7 left) 2024-08-17T17:00:22.513 INFO:tasks.rados.rados.0.smithi063.stdout:1355: done (6 left) 2024-08-17T17:00:22.513 INFO:tasks.rados.rados.0.smithi063.stdout:1356: done (5 left) 2024-08-17T17:00:22.513 INFO:tasks.rados.rados.0.smithi063.stdout:1357: done (4 left) 2024-08-17T17:00:22.513 INFO:tasks.rados.rados.0.smithi063.stdout:1358: done (3 left) 2024-08-17T17:00:22.513 INFO:tasks.rados.rados.0.smithi063.stdout:1360: done (2 left) 2024-08-17T17:00:22.513 INFO:tasks.rados.rados.0.smithi063.stdout:1361: done (1 left) 2024-08-17T17:00:22.513 INFO:tasks.rados.rados.0.smithi063.stdout:1362: done (0 left) 2024-08-17T17:00:22.513 INFO:tasks.rados.rados.0.smithi063.stdout:1363: read oid 17 snap -1 2024-08-17T17:00:22.513 INFO:tasks.rados.rados.0.smithi063.stdout:1363: expect (ObjNum 392 snap 119 seq_num 392) 2024-08-17T17:00:22.514 INFO:tasks.rados.rados.0.smithi063.stdout:1364: write oid 41 current snap is 167 2024-08-17T17:00:22.514 INFO:tasks.rados.rados.0.smithi063.stdout:1364: seq_num 533 ranges {558131=652427,1673373=559846,2820808=137323} 2024-08-17T17:00:22.520 INFO:tasks.rados.rados.0.smithi063.stdout:1364: writing smithi06387500-41 from 558131 to 1210558 tid 1 2024-08-17T17:00:22.526 INFO:tasks.rados.rados.0.smithi063.stdout:1364: writing smithi06387500-41 from 1673373 to 2233219 tid 2 2024-08-17T17:00:22.530 INFO:tasks.rados.rados.0.smithi063.stdout:1364: writing smithi06387500-41 from 2820808 to 2958131 tid 3 2024-08-17T17:00:22.530 INFO:tasks.rados.rados.0.smithi063.stdout:1365: snap_create 2024-08-17T17:00:22.530 INFO:tasks.rados.rados.0.smithi063.stdout:1363: expect (ObjNum 392 snap 119 seq_num 392) 2024-08-17T17:00:22.614 INFO:tasks.rados.rados.0.smithi063.stdout:1364: finishing write tid 1 to smithi06387500-41 2024-08-17T17:00:22.615 INFO:tasks.rados.rados.0.smithi063.stdout:1364: finishing write tid 2 to smithi06387500-41 2024-08-17T17:00:22.615 INFO:tasks.rados.rados.0.smithi063.stdout:1364: finishing write tid 3 to smithi06387500-41 2024-08-17T17:00:22.615 INFO:tasks.rados.rados.0.smithi063.stdout:1364: finishing write tid 4 to smithi06387500-41 2024-08-17T17:00:22.615 INFO:tasks.rados.rados.0.smithi063.stdout:1364: finishing write tid 5 to smithi06387500-41 2024-08-17T17:00:22.615 INFO:tasks.rados.rados.0.smithi063.stdout:1364: oid 41 updating version 0 to 114 2024-08-17T17:00:22.615 INFO:tasks.rados.rados.0.smithi063.stdout:1364: oid 41 updating version 114 to 117 2024-08-17T17:00:22.615 INFO:tasks.rados.rados.0.smithi063.stdout:1364: oid 41 version 117 is already newer than 116 2024-08-17T17:00:22.615 INFO:tasks.rados.rados.0.smithi063.stdout:1364: oid 41 version 117 is already newer than 115 2024-08-17T17:00:22.615 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 117 (ObjNum 533 snap 167 seq_num 533) dirty exists 2024-08-17T17:00:22.615 INFO:tasks.rados.rados.0.smithi063.stdout:1364: left oid 41 (ObjNum 533 snap 167 seq_num 533) 2024-08-17T17:00:22.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:22 smithi063 ceph-mon[109843]: osdmap e401: 8 total, 8 up, 8 in 2024-08-17T17:00:22.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:22 smithi063 ceph-mon[109843]: osdmap e402: 8 total, 8 up, 8 in 2024-08-17T17:00:22.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:22 smithi063 ceph-mon[114756]: osdmap e401: 8 total, 8 up, 8 in 2024-08-17T17:00:22.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:22 smithi063 ceph-mon[114756]: osdmap e402: 8 total, 8 up, 8 in 2024-08-17T17:00:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:22 smithi162 ceph-mon[34633]: osdmap e401: 8 total, 8 up, 8 in 2024-08-17T17:00:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:22 smithi162 ceph-mon[34633]: osdmap e402: 8 total, 8 up, 8 in 2024-08-17T17:00:23.510 INFO:tasks.rados.rados.0.smithi063.stdout:1363: done (2 left) 2024-08-17T17:00:23.510 INFO:tasks.rados.rados.0.smithi063.stdout:1364: done (1 left) 2024-08-17T17:00:23.510 INFO:tasks.rados.rados.0.smithi063.stdout:1365: done (0 left) 2024-08-17T17:00:23.510 INFO:tasks.rados.rados.0.smithi063.stdout:1366: write oid 10 current snap is 168 2024-08-17T17:00:23.510 INFO:tasks.rados.rados.0.smithi063.stdout:1366: seq_num 534 ranges {574938=727676,1731855=703504,2848231=126707} 2024-08-17T17:00:23.516 INFO:tasks.rados.rados.0.smithi063.stdout:1366: writing smithi06387500-10 from 574938 to 1302614 tid 1 2024-08-17T17:00:23.523 INFO:tasks.rados.rados.0.smithi063.stdout:1366: writing smithi06387500-10 from 1731855 to 2435359 tid 2 2024-08-17T17:00:23.526 INFO:tasks.rados.rados.0.smithi063.stdout:1366: writing smithi06387500-10 from 2848231 to 2974938 tid 3 2024-08-17T17:00:23.526 INFO:tasks.rados.rados.0.smithi063.stdout:1367: delete oid 29 current snap is 168 2024-08-17T17:00:23.526 INFO:tasks.rados.rados.0.smithi063.stdout:1366: finishing write tid 1 to smithi06387500-10 2024-08-17T17:00:23.528 INFO:tasks.rados.rados.0.smithi063.stdout:1366: finishing write tid 2 to smithi06387500-10 2024-08-17T17:00:23.528 INFO:tasks.rados.rados.0.smithi063.stdout:1366: finishing write tid 3 to smithi06387500-10 2024-08-17T17:00:23.529 INFO:tasks.rados.rados.0.smithi063.stdout:1366: finishing write tid 4 to smithi06387500-10 2024-08-17T17:00:23.529 INFO:tasks.rados.rados.0.smithi063.stdout:1366: finishing write tid 5 to smithi06387500-10 2024-08-17T17:00:23.529 INFO:tasks.rados.rados.0.smithi063.stdout:1366: oid 10 updating version 0 to 505 2024-08-17T17:00:23.529 INFO:tasks.rados.rados.0.smithi063.stdout:1366: oid 10 updating version 505 to 506 2024-08-17T17:00:23.529 INFO:tasks.rados.rados.0.smithi063.stdout:1366: oid 10 updating version 506 to 507 2024-08-17T17:00:23.529 INFO:tasks.rados.rados.0.smithi063.stdout:1366: oid 10 version 507 is already newer than 504 2024-08-17T17:00:23.529 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 507 (ObjNum 534 snap 168 seq_num 534) dirty exists 2024-08-17T17:00:23.529 INFO:tasks.rados.rados.0.smithi063.stdout:1366: left oid 10 (ObjNum 534 snap 168 seq_num 534) 2024-08-17T17:00:23.530 INFO:tasks.rados.rados.0.smithi063.stdout:1366: done (1 left) 2024-08-17T17:00:23.530 INFO:tasks.rados.rados.0.smithi063.stdout:1367: done (0 left) 2024-08-17T17:00:23.531 INFO:tasks.rados.rados.0.smithi063.stdout:1368: read oid 27 snap -1 2024-08-17T17:00:23.531 INFO:tasks.rados.rados.0.smithi063.stdout:1368: expect deleted 2024-08-17T17:00:23.531 INFO:tasks.rados.rados.0.smithi063.stdout:1369: snap_create 2024-08-17T17:00:23.563 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:00:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:00:23] "GET /metrics HTTP/1.1" 200 36211 "" "Prometheus/2.51.0" 2024-08-17T17:00:23.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:23 smithi162 ceph-mon[34633]: pgmap v55: 89 pgs: 89 active+clean; 221 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 1.9 MiB/s wr, 28 op/s 2024-08-17T17:00:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:23 smithi162 ceph-mon[34633]: osdmap e403: 8 total, 8 up, 8 in 2024-08-17T17:00:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:23 smithi063 ceph-mon[114756]: pgmap v55: 89 pgs: 89 active+clean; 221 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 1.9 MiB/s wr, 28 op/s 2024-08-17T17:00:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:23 smithi063 ceph-mon[114756]: osdmap e403: 8 total, 8 up, 8 in 2024-08-17T17:00:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:23 smithi063 ceph-mon[109843]: pgmap v55: 89 pgs: 89 active+clean; 221 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 1.9 MiB/s wr, 28 op/s 2024-08-17T17:00:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:23 smithi063 ceph-mon[109843]: osdmap e403: 8 total, 8 up, 8 in 2024-08-17T17:00:24.511 INFO:tasks.rados.rados.0.smithi063.stdout:1368: done (1 left) 2024-08-17T17:00:24.511 INFO:tasks.rados.rados.0.smithi063.stdout:1369: done (0 left) 2024-08-17T17:00:24.511 INFO:tasks.rados.rados.0.smithi063.stdout:1370: read oid 41 snap -1 2024-08-17T17:00:24.511 INFO:tasks.rados.rados.0.smithi063.stdout:1370: expect (ObjNum 533 snap 167 seq_num 533) 2024-08-17T17:00:24.511 INFO:tasks.rados.rados.0.smithi063.stdout:1371: delete oid 20 current snap is 169 2024-08-17T17:00:24.513 INFO:tasks.rados.rados.0.smithi063.stdout:1371: done (1 left) 2024-08-17T17:00:24.513 INFO:tasks.rados.rados.0.smithi063.stdout:1372: write oid 17 current snap is 169 2024-08-17T17:00:24.513 INFO:tasks.rados.rados.0.smithi063.stdout:1372: seq_num 537 ranges {625359=553423,1790948=567184,2930500=94859} 2024-08-17T17:00:24.520 INFO:tasks.rados.rados.0.smithi063.stdout:1372: writing smithi06387500-17 from 625359 to 1178782 tid 1 2024-08-17T17:00:24.527 INFO:tasks.rados.rados.0.smithi063.stdout:1372: writing smithi06387500-17 from 1790948 to 2358132 tid 2 2024-08-17T17:00:24.531 INFO:tasks.rados.rados.0.smithi063.stdout:1372: writing smithi06387500-17 from 2930500 to 3025359 tid 3 2024-08-17T17:00:24.531 INFO:tasks.rados.rados.0.smithi063.stdout:1373: write oid 8 current snap is 169 2024-08-17T17:00:24.531 INFO:tasks.rados.rados.0.smithi063.stdout:1373: seq_num 538 ranges {642166=628672,1849430=710842} 2024-08-17T17:00:24.538 INFO:tasks.rados.rados.0.smithi063.stdout:1373: writing smithi06387500-8 from 642166 to 1270838 tid 1 2024-08-17T17:00:24.545 INFO:tasks.rados.rados.0.smithi063.stdout:1373: writing smithi06387500-8 from 1849430 to 2560272 tid 2 2024-08-17T17:00:24.545 INFO:tasks.rados.rados.0.smithi063.stdout:1374: rollback oid 40 current snap is 169 2024-08-17T17:00:24.545 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 40 to 161 2024-08-17T17:00:24.545 INFO:tasks.rados.rados.0.smithi063.stdout:1370: expect (ObjNum 533 snap 167 seq_num 533) 2024-08-17T17:00:24.645 INFO:tasks.rados.rados.0.smithi063.stdout:1372: finishing write tid 1 to smithi06387500-17 2024-08-17T17:00:24.645 INFO:tasks.rados.rados.0.smithi063.stdout:1372: finishing write tid 2 to smithi06387500-17 2024-08-17T17:00:24.645 INFO:tasks.rados.rados.0.smithi063.stdout:1372: finishing write tid 3 to smithi06387500-17 2024-08-17T17:00:24.645 INFO:tasks.rados.rados.0.smithi063.stdout:1372: finishing write tid 4 to smithi06387500-17 2024-08-17T17:00:24.645 INFO:tasks.rados.rados.0.smithi063.stdout:1372: finishing write tid 5 to smithi06387500-17 2024-08-17T17:00:24.645 INFO:tasks.rados.rados.0.smithi063.stdout:1372: oid 17 updating version 0 to 421 2024-08-17T17:00:24.645 INFO:tasks.rados.rados.0.smithi063.stdout:1372: oid 17 updating version 421 to 422 2024-08-17T17:00:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:1372: oid 17 version 422 is already newer than 420 2024-08-17T17:00:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:1372: oid 17 version 422 is already newer than 419 2024-08-17T17:00:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 422 (ObjNum 537 snap 169 seq_num 537) dirty exists 2024-08-17T17:00:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:1372: left oid 17 (ObjNum 537 snap 169 seq_num 537) 2024-08-17T17:00:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:1373: finishing write tid 1 to smithi06387500-8 2024-08-17T17:00:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:1370: done (3 left) 2024-08-17T17:00:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:1372: done (2 left) 2024-08-17T17:00:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:1375: write oid 16 current snap is 169 2024-08-17T17:00:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:1374: finishing rollback tid 1 to smithi06387500-40 2024-08-17T17:00:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 397 (ObjNum 81 snap 0 seq_num 1027902090) dirty dne 2024-08-17T17:00:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:1373: finishing write tid 2 to smithi06387500-8 2024-08-17T17:00:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:1375: seq_num 539 ranges {658973=703921,1907912=570853,2985346=73627} 2024-08-17T17:00:24.652 INFO:tasks.rados.rados.0.smithi063.stdout:1375: writing smithi06387500-16 from 658973 to 1362894 tid 1 2024-08-17T17:00:24.658 INFO:tasks.rados.rados.0.smithi063.stdout:1375: writing smithi06387500-16 from 1907912 to 2478765 tid 2 2024-08-17T17:00:24.662 INFO:tasks.rados.rados.0.smithi063.stdout:1375: writing smithi06387500-16 from 2985346 to 3058973 tid 3 2024-08-17T17:00:24.662 INFO:tasks.rados.rados.0.smithi063.stdout:1374: done (2 left) 2024-08-17T17:00:24.662 INFO:tasks.rados.rados.0.smithi063.stdout:1376: snap_create 2024-08-17T17:00:24.662 INFO:tasks.rados.rados.0.smithi063.stdout:1373: finishing write tid 3 to smithi06387500-8 2024-08-17T17:00:24.662 INFO:tasks.rados.rados.0.smithi063.stdout:1373: finishing write tid 4 to smithi06387500-8 2024-08-17T17:00:24.662 INFO:tasks.rados.rados.0.smithi063.stdout:1373: oid 8 updating version 0 to 411 2024-08-17T17:00:24.662 INFO:tasks.rados.rados.0.smithi063.stdout:1373: oid 8 version 411 is already newer than 409 2024-08-17T17:00:24.663 INFO:tasks.rados.rados.0.smithi063.stdout:1373: oid 8 version 411 is already newer than 410 2024-08-17T17:00:24.663 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 411 (ObjNum 538 snap 169 seq_num 538) dirty exists 2024-08-17T17:00:24.663 INFO:tasks.rados.rados.0.smithi063.stdout:1373: left oid 8 (ObjNum 538 snap 169 seq_num 538) 2024-08-17T17:00:24.663 INFO:tasks.rados.rados.0.smithi063.stdout:1375: finishing write tid 1 to smithi06387500-16 2024-08-17T17:00:24.663 INFO:tasks.rados.rados.0.smithi063.stdout:1375: finishing write tid 2 to smithi06387500-16 2024-08-17T17:00:24.664 INFO:tasks.rados.rados.0.smithi063.stdout:1375: finishing write tid 3 to smithi06387500-16 2024-08-17T17:00:24.664 INFO:tasks.rados.rados.0.smithi063.stdout:1375: finishing write tid 4 to smithi06387500-16 2024-08-17T17:00:24.664 INFO:tasks.rados.rados.0.smithi063.stdout:1375: finishing write tid 5 to smithi06387500-16 2024-08-17T17:00:24.664 INFO:tasks.rados.rados.0.smithi063.stdout:1375: oid 16 updating version 0 to 523 2024-08-17T17:00:24.664 INFO:tasks.rados.rados.0.smithi063.stdout:1375: oid 16 updating version 523 to 525 2024-08-17T17:00:24.664 INFO:tasks.rados.rados.0.smithi063.stdout:1375: oid 16 version 525 is already newer than 522 2024-08-17T17:00:24.664 INFO:tasks.rados.rados.0.smithi063.stdout:1375: oid 16 version 525 is already newer than 524 2024-08-17T17:00:24.664 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 525 (ObjNum 539 snap 169 seq_num 539) dirty exists 2024-08-17T17:00:24.664 INFO:tasks.rados.rados.0.smithi063.stdout:1375: left oid 16 (ObjNum 539 snap 169 seq_num 539) 2024-08-17T17:00:25.570 INFO:tasks.rados.rados.0.smithi063.stdout:1373: done (2 left) 2024-08-17T17:00:25.570 INFO:tasks.rados.rados.0.smithi063.stdout:1375: done (1 left) 2024-08-17T17:00:25.570 INFO:tasks.rados.rados.0.smithi063.stdout:1376: done (0 left) 2024-08-17T17:00:25.570 INFO:tasks.rados.rados.0.smithi063.stdout:1377: delete oid 36 current snap is 170 2024-08-17T17:00:25.575 INFO:tasks.rados.rados.0.smithi063.stdout:1377: done (0 left) 2024-08-17T17:00:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1378: read oid 1 snap 166 2024-08-17T17:00:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1378: expect (ObjNum 511 snap 160 seq_num 511) 2024-08-17T17:00:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1379: write oid 35 current snap is 170 2024-08-17T17:00:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1379: seq_num 541 ranges {692587=570772,2024876=574522,3040192=52395} 2024-08-17T17:00:25.583 INFO:tasks.rados.rados.0.smithi063.stdout:1379: writing smithi06387500-35 from 692587 to 1263359 tid 1 2024-08-17T17:00:25.591 INFO:tasks.rados.rados.0.smithi063.stdout:1379: writing smithi06387500-35 from 2024876 to 2599398 tid 2 2024-08-17T17:00:25.594 INFO:tasks.rados.rados.0.smithi063.stdout:1379: writing smithi06387500-35 from 3040192 to 3092587 tid 3 2024-08-17T17:00:25.594 INFO:tasks.rados.rados.0.smithi063.stdout:1380: read oid 45 snap -1 2024-08-17T17:00:25.594 INFO:tasks.rados.rados.0.smithi063.stdout:1380: expect deleted 2024-08-17T17:00:25.594 INFO:tasks.rados.rados.0.smithi063.stdout:1381: write oid 41 current snap is 170 2024-08-17T17:00:25.594 INFO:tasks.rados.rados.0.smithi063.stdout:1381: seq_num 542 ranges {709394=646021,2083358=718180} 2024-08-17T17:00:25.601 INFO:tasks.rados.rados.0.smithi063.stdout:1381: writing smithi06387500-41 from 709394 to 1355415 tid 1 2024-08-17T17:00:25.609 INFO:tasks.rados.rados.0.smithi063.stdout:1381: writing smithi06387500-41 from 2083358 to 2801538 tid 2 2024-08-17T17:00:25.609 INFO:tasks.rados.rados.0.smithi063.stdout:1382: delete oid 24 current snap is 170 2024-08-17T17:00:25.609 INFO:tasks.rados.rados.0.smithi063.stdout:1378: expect (ObjNum 511 snap 160 seq_num 511) 2024-08-17T17:00:25.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:25 smithi063 ceph-mon[114756]: pgmap v58: 89 pgs: 89 active+clean; 221 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 1.9 MiB/s wr, 28 op/s 2024-08-17T17:00:25.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:25 smithi063 ceph-mon[114756]: osdmap e404: 8 total, 8 up, 8 in 2024-08-17T17:00:25.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:25 smithi063 ceph-mon[109843]: pgmap v58: 89 pgs: 89 active+clean; 221 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 1.9 MiB/s wr, 28 op/s 2024-08-17T17:00:25.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:25 smithi063 ceph-mon[109843]: osdmap e404: 8 total, 8 up, 8 in 2024-08-17T17:00:25.788 INFO:tasks.rados.rados.0.smithi063.stdout:1379: finishing write tid 1 to smithi06387500-35 2024-08-17T17:00:25.789 INFO:tasks.rados.rados.0.smithi063.stdout:1379: finishing write tid 2 to smithi06387500-35 2024-08-17T17:00:25.789 INFO:tasks.rados.rados.0.smithi063.stdout:1379: finishing write tid 3 to smithi06387500-35 2024-08-17T17:00:25.789 INFO:tasks.rados.rados.0.smithi063.stdout:1379: finishing write tid 4 to smithi06387500-35 2024-08-17T17:00:25.789 INFO:tasks.rados.rados.0.smithi063.stdout:1379: finishing write tid 5 to smithi06387500-35 2024-08-17T17:00:25.789 INFO:tasks.rados.rados.0.smithi063.stdout:1379: oid 35 updating version 0 to 1380 2024-08-17T17:00:25.789 INFO:tasks.rados.rados.0.smithi063.stdout:1379: oid 35 updating version 1380 to 1383 2024-08-17T17:00:25.789 INFO:tasks.rados.rados.0.smithi063.stdout:1379: oid 35 version 1383 is already newer than 1381 2024-08-17T17:00:25.789 INFO:tasks.rados.rados.0.smithi063.stdout:1379: oid 35 version 1383 is already newer than 1382 2024-08-17T17:00:25.789 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 1383 (ObjNum 541 snap 170 seq_num 541) dirty exists 2024-08-17T17:00:25.789 INFO:tasks.rados.rados.0.smithi063.stdout:1379: left oid 35 (ObjNum 541 snap 170 seq_num 541) 2024-08-17T17:00:25.789 INFO:tasks.rados.rados.0.smithi063.stdout:1381: finishing write tid 1 to smithi06387500-41 2024-08-17T17:00:25.790 INFO:tasks.rados.rados.0.smithi063.stdout:1381: finishing write tid 2 to smithi06387500-41 2024-08-17T17:00:25.790 INFO:tasks.rados.rados.0.smithi063.stdout:1381: finishing write tid 3 to smithi06387500-41 2024-08-17T17:00:25.790 INFO:tasks.rados.rados.0.smithi063.stdout:1378: done (4 left) 2024-08-17T17:00:25.790 INFO:tasks.rados.rados.0.smithi063.stdout:1379: done (3 left) 2024-08-17T17:00:25.790 INFO:tasks.rados.rados.0.smithi063.stdout:1380: done (2 left) 2024-08-17T17:00:25.790 INFO:tasks.rados.rados.0.smithi063.stdout:1382: done (1 left) 2024-08-17T17:00:25.790 INFO:tasks.rados.rados.0.smithi063.stdout:1383: snap_remove snap 166 2024-08-17T17:00:25.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:25 smithi162 ceph-mon[34633]: pgmap v58: 89 pgs: 89 active+clean; 221 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 1.9 MiB/s wr, 28 op/s 2024-08-17T17:00:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:25 smithi162 ceph-mon[34633]: osdmap e404: 8 total, 8 up, 8 in 2024-08-17T17:00:26.583 INFO:tasks.rados.rados.0.smithi063.stdout:1383: done (1 left) 2024-08-17T17:00:26.583 INFO:tasks.rados.rados.0.smithi063.stdout:1384: snap_create 2024-08-17T17:00:26.583 INFO:tasks.rados.rados.0.smithi063.stdout:1381: finishing write tid 4 to smithi06387500-41 2024-08-17T17:00:26.584 INFO:tasks.rados.rados.0.smithi063.stdout:1381: oid 41 updating version 0 to 121 2024-08-17T17:00:26.584 INFO:tasks.rados.rados.0.smithi063.stdout:1381: oid 41 version 121 is already newer than 119 2024-08-17T17:00:26.584 INFO:tasks.rados.rados.0.smithi063.stdout:1381: oid 41 version 121 is already newer than 120 2024-08-17T17:00:26.584 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 121 (ObjNum 542 snap 170 seq_num 542) dirty exists 2024-08-17T17:00:26.584 INFO:tasks.rados.rados.0.smithi063.stdout:1381: left oid 41 (ObjNum 542 snap 170 seq_num 542) 2024-08-17T17:00:26.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:26 smithi162 ceph-mon[34633]: osdmap e405: 8 total, 8 up, 8 in 2024-08-17T17:00:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:26 smithi063 ceph-mon[109843]: osdmap e405: 8 total, 8 up, 8 in 2024-08-17T17:00:27.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:26 smithi063 ceph-mon[114756]: osdmap e405: 8 total, 8 up, 8 in 2024-08-17T17:00:27.578 INFO:tasks.rados.rados.0.smithi063.stdout:1381: done (1 left) 2024-08-17T17:00:27.578 INFO:tasks.rados.rados.0.smithi063.stdout:1384: done (0 left) 2024-08-17T17:00:27.578 INFO:tasks.rados.rados.0.smithi063.stdout:1385: delete oid 44 current snap is 171 2024-08-17T17:00:27.583 INFO:tasks.rados.rados.0.smithi063.stdout:1385: done (0 left) 2024-08-17T17:00:27.583 INFO:tasks.rados.rados.0.smithi063.stdout:1386: write oid 12 current snap is 171 2024-08-17T17:00:27.583 INFO:tasks.rados.rados.0.smithi063.stdout:1386: seq_num 545 ranges {759815=471768,1742451=465507,2917178=242637} 2024-08-17T17:00:27.590 INFO:tasks.rados.rados.0.smithi063.stdout:1386: writing smithi06387500-12 from 759815 to 1231583 tid 1 2024-08-17T17:00:27.596 INFO:tasks.rados.rados.0.smithi063.stdout:1386: writing smithi06387500-12 from 1742451 to 2207958 tid 2 2024-08-17T17:00:27.602 INFO:tasks.rados.rados.0.smithi063.stdout:1386: writing smithi06387500-12 from 2917178 to 3159815 tid 3 2024-08-17T17:00:27.602 INFO:tasks.rados.rados.0.smithi063.stdout:1386: finishing write tid 1 to smithi06387500-12 2024-08-17T17:00:27.602 INFO:tasks.rados.rados.0.smithi063.stdout:1386: finishing write tid 2 to smithi06387500-12 2024-08-17T17:00:27.602 INFO:tasks.rados.rados.0.smithi063.stdout:1387: read oid 40 snap -1 2024-08-17T17:00:27.602 INFO:tasks.rados.rados.0.smithi063.stdout:1387: expect deleted 2024-08-17T17:00:27.602 INFO:tasks.rados.rados.0.smithi063.stdout:1388: rollback oid 2 current snap is 171 2024-08-17T17:00:27.602 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 2 to 159 2024-08-17T17:00:27.602 INFO:tasks.rados.rados.0.smithi063.stdout:1389: rollback oid 42 current snap is 171 2024-08-17T17:00:27.602 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 42 to 167 2024-08-17T17:00:27.602 INFO:tasks.rados.rados.0.smithi063.stdout:1390: read oid 45 snap -1 2024-08-17T17:00:27.602 INFO:tasks.rados.rados.0.smithi063.stdout:1390: expect deleted 2024-08-17T17:00:27.603 INFO:tasks.rados.rados.0.smithi063.stdout:1391: write oid 23 current snap is 171 2024-08-17T17:00:27.603 INFO:tasks.rados.rados.0.smithi063.stdout:1391: seq_num 546 ranges {776622=547017,1800933=725518,2944601=232021} 2024-08-17T17:00:27.609 INFO:tasks.rados.rados.0.smithi063.stdout:1391: writing smithi06387500-23 from 776622 to 1323639 tid 1 2024-08-17T17:00:27.615 INFO:tasks.rados.rados.0.smithi063.stdout:1391: writing smithi06387500-23 from 1800933 to 2526451 tid 2 2024-08-17T17:00:27.620 INFO:tasks.rados.rados.0.smithi063.stdout:1391: writing smithi06387500-23 from 2944601 to 3176622 tid 3 2024-08-17T17:00:27.620 INFO:tasks.rados.rados.0.smithi063.stdout:1392: snap_remove snap 169 2024-08-17T17:00:27.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:27 smithi162 ceph-mon[34633]: pgmap v61: 89 pgs: 89 active+clean; 221 MiB data, 713 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:00:27.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:27 smithi162 ceph-mon[34633]: osdmap e406: 8 total, 8 up, 8 in 2024-08-17T17:00:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:27 smithi063 ceph-mon[114756]: pgmap v61: 89 pgs: 89 active+clean; 221 MiB data, 713 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:00:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:27 smithi063 ceph-mon[114756]: osdmap e406: 8 total, 8 up, 8 in 2024-08-17T17:00:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:27 smithi063 ceph-mon[109843]: pgmap v61: 89 pgs: 89 active+clean; 221 MiB data, 713 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:00:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:27 smithi063 ceph-mon[109843]: osdmap e406: 8 total, 8 up, 8 in 2024-08-17T17:00:28.589 INFO:tasks.rados.rados.0.smithi063.stdout:1392: done (6 left) 2024-08-17T17:00:28.589 INFO:tasks.rados.rados.0.smithi063.stdout:1393: delete oid 19 current snap is 171 2024-08-17T17:00:28.589 INFO:tasks.rados.rados.0.smithi063.stdout:1386: finishing write tid 3 to smithi06387500-12 2024-08-17T17:00:28.589 INFO:tasks.rados.rados.0.smithi063.stdout:1389: finishing rollback tid 0 to smithi06387500-42 2024-08-17T17:00:28.590 INFO:tasks.rados.rados.0.smithi063.stdout:1388: finishing rollback tid 1 to smithi06387500-2 2024-08-17T17:00:28.590 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 584 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T17:00:28.590 INFO:tasks.rados.rados.0.smithi063.stdout:1386: finishing write tid 4 to smithi06387500-12 2024-08-17T17:00:28.590 INFO:tasks.rados.rados.0.smithi063.stdout:1386: finishing write tid 5 to smithi06387500-12 2024-08-17T17:00:28.590 INFO:tasks.rados.rados.0.smithi063.stdout:1386: oid 12 updating version 0 to 298 2024-08-17T17:00:28.590 INFO:tasks.rados.rados.0.smithi063.stdout:1386: oid 12 updating version 298 to 300 2024-08-17T17:00:28.590 INFO:tasks.rados.rados.0.smithi063.stdout:1386: oid 12 version 300 is already newer than 299 2024-08-17T17:00:28.590 INFO:tasks.rados.rados.0.smithi063.stdout:1386: oid 12 version 300 is already newer than 297 2024-08-17T17:00:28.590 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 300 (ObjNum 545 snap 171 seq_num 545) dirty exists 2024-08-17T17:00:28.590 INFO:tasks.rados.rados.0.smithi063.stdout:1386: left oid 12 (ObjNum 545 snap 171 seq_num 545) 2024-08-17T17:00:28.590 INFO:tasks.rados.rados.0.smithi063.stdout:1389: finishing rollback tid 1 to smithi06387500-42 2024-08-17T17:00:28.590 INFO:tasks.rados.rados.0.smithi063.stdout:1389: finishing rollback tid 2 to smithi06387500-42 2024-08-17T17:00:28.591 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 996 (ObjNum 514 snap 161 seq_num 514) dirty exists 2024-08-17T17:00:28.591 INFO:tasks.rados.rados.0.smithi063.stdout:1391: finishing write tid 1 to smithi06387500-23 2024-08-17T17:00:28.591 INFO:tasks.rados.rados.0.smithi063.stdout:1391: finishing write tid 2 to smithi06387500-23 2024-08-17T17:00:28.591 INFO:tasks.rados.rados.0.smithi063.stdout:1391: finishing write tid 3 to smithi06387500-23 2024-08-17T17:00:28.591 INFO:tasks.rados.rados.0.smithi063.stdout:1391: finishing write tid 4 to smithi06387500-23 2024-08-17T17:00:28.591 INFO:tasks.rados.rados.0.smithi063.stdout:1391: finishing write tid 5 to smithi06387500-23 2024-08-17T17:00:28.591 INFO:tasks.rados.rados.0.smithi063.stdout:1391: oid 23 updating version 0 to 678 2024-08-17T17:00:28.591 INFO:tasks.rados.rados.0.smithi063.stdout:1391: oid 23 version 678 is already newer than 677 2024-08-17T17:00:28.591 INFO:tasks.rados.rados.0.smithi063.stdout:1391: oid 23 updating version 678 to 679 2024-08-17T17:00:28.591 INFO:tasks.rados.rados.0.smithi063.stdout:1391: oid 23 version 679 is already newer than 676 2024-08-17T17:00:28.591 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 679 (ObjNum 546 snap 171 seq_num 546) dirty exists 2024-08-17T17:00:28.591 INFO:tasks.rados.rados.0.smithi063.stdout:1391: left oid 23 (ObjNum 546 snap 171 seq_num 546) 2024-08-17T17:00:28.594 INFO:tasks.rados.rados.0.smithi063.stdout:1386: done (6 left) 2024-08-17T17:00:28.594 INFO:tasks.rados.rados.0.smithi063.stdout:1387: done (5 left) 2024-08-17T17:00:28.594 INFO:tasks.rados.rados.0.smithi063.stdout:1388: done (4 left) 2024-08-17T17:00:28.594 INFO:tasks.rados.rados.0.smithi063.stdout:1389: done (3 left) 2024-08-17T17:00:28.594 INFO:tasks.rados.rados.0.smithi063.stdout:1390: done (2 left) 2024-08-17T17:00:28.594 INFO:tasks.rados.rados.0.smithi063.stdout:1391: done (1 left) 2024-08-17T17:00:28.595 INFO:tasks.rados.rados.0.smithi063.stdout:1393: done (0 left) 2024-08-17T17:00:28.595 INFO:tasks.rados.rados.0.smithi063.stdout:1394: read oid 9 snap -1 2024-08-17T17:00:28.595 INFO:tasks.rados.rados.0.smithi063.stdout:1394: expect deleted 2024-08-17T17:00:28.595 INFO:tasks.rados.rados.0.smithi063.stdout:1395: read oid 16 snap -1 2024-08-17T17:00:28.595 INFO:tasks.rados.rados.0.smithi063.stdout:1395: expect (ObjNum 539 snap 169 seq_num 539) 2024-08-17T17:00:28.595 INFO:tasks.rados.rados.0.smithi063.stdout:1396: write oid 24 current snap is 171 2024-08-17T17:00:28.595 INFO:tasks.rados.rados.0.smithi063.stdout:1396: seq_num 548 ranges {410236=413868,1234250=729187,2715800=423063} 2024-08-17T17:00:28.599 INFO:tasks.rados.rados.0.smithi063.stdout:1396: writing smithi06387500-24 from 410236 to 824104 tid 1 2024-08-17T17:00:28.606 INFO:tasks.rados.rados.0.smithi063.stdout:1396: writing smithi06387500-24 from 1234250 to 1963437 tid 2 2024-08-17T17:00:28.613 INFO:tasks.rados.rados.0.smithi063.stdout:1396: writing smithi06387500-24 from 2715800 to 3138863 tid 3 2024-08-17T17:00:28.613 INFO:tasks.rados.rados.0.smithi063.stdout:1397: snap_remove snap 164 2024-08-17T17:00:28.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:28 smithi162 ceph-mon[34633]: osdmap e407: 8 total, 8 up, 8 in 2024-08-17T17:00:29.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:28 smithi063 ceph-mon[114756]: osdmap e407: 8 total, 8 up, 8 in 2024-08-17T17:00:29.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:28 smithi063 ceph-mon[109843]: osdmap e407: 8 total, 8 up, 8 in 2024-08-17T17:00:29.595 INFO:tasks.rados.rados.0.smithi063.stdout:1397: done (3 left) 2024-08-17T17:00:29.595 INFO:tasks.rados.rados.0.smithi063.stdout:1398: snap_remove snap 170 2024-08-17T17:00:29.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:29 smithi162 ceph-mon[34633]: pgmap v64: 89 pgs: 89 active+clean; 234 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-08-17T17:00:29.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:29 smithi162 ceph-mon[34633]: osdmap e408: 8 total, 8 up, 8 in 2024-08-17T17:00:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:29 smithi063 ceph-mon[114756]: pgmap v64: 89 pgs: 89 active+clean; 234 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-08-17T17:00:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:29 smithi063 ceph-mon[114756]: osdmap e408: 8 total, 8 up, 8 in 2024-08-17T17:00:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:29 smithi063 ceph-mon[109843]: pgmap v64: 89 pgs: 89 active+clean; 234 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-08-17T17:00:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:29 smithi063 ceph-mon[109843]: osdmap e408: 8 total, 8 up, 8 in 2024-08-17T17:00:30.604 INFO:tasks.rados.rados.0.smithi063.stdout:1398: done (3 left) 2024-08-17T17:00:30.604 INFO:tasks.rados.rados.0.smithi063.stdout:1399: write oid 36 current snap is 171 2024-08-17T17:00:30.604 INFO:tasks.rados.rados.0.smithi063.stdout:1399: seq_num 549 ranges {427043=489117,1576379=472845,2626870=600173} 2024-08-17T17:00:30.609 INFO:tasks.rados.rados.0.smithi063.stdout:1399: writing smithi06387500-36 from 427043 to 916160 tid 1 2024-08-17T17:00:30.615 INFO:tasks.rados.rados.0.smithi063.stdout:1399: writing smithi06387500-36 from 1576379 to 2049224 tid 2 2024-08-17T17:00:30.623 INFO:tasks.rados.rados.0.smithi063.stdout:1399: writing smithi06387500-36 from 2626870 to 3227043 tid 3 2024-08-17T17:00:30.623 INFO:tasks.rados.rados.0.smithi063.stdout:1400: snap_create 2024-08-17T17:00:30.623 INFO:tasks.rados.rados.0.smithi063.stdout:1395: expect (ObjNum 539 snap 169 seq_num 539) 2024-08-17T17:00:30.725 INFO:tasks.rados.rados.0.smithi063.stdout:1396: finishing write tid 1 to smithi06387500-24 2024-08-17T17:00:30.725 INFO:tasks.rados.rados.0.smithi063.stdout:1396: finishing write tid 2 to smithi06387500-24 2024-08-17T17:00:30.725 INFO:tasks.rados.rados.0.smithi063.stdout:1396: finishing write tid 3 to smithi06387500-24 2024-08-17T17:00:30.725 INFO:tasks.rados.rados.0.smithi063.stdout:1396: finishing write tid 4 to smithi06387500-24 2024-08-17T17:00:30.725 INFO:tasks.rados.rados.0.smithi063.stdout:1396: finishing write tid 5 to smithi06387500-24 2024-08-17T17:00:30.725 INFO:tasks.rados.rados.0.smithi063.stdout:1396: oid 24 updating version 0 to 527 2024-08-17T17:00:30.725 INFO:tasks.rados.rados.0.smithi063.stdout:1396: oid 24 updating version 527 to 529 2024-08-17T17:00:30.725 INFO:tasks.rados.rados.0.smithi063.stdout:1396: oid 24 version 529 is already newer than 528 2024-08-17T17:00:30.726 INFO:tasks.rados.rados.0.smithi063.stdout:1396: oid 24 updating version 529 to 530 2024-08-17T17:00:30.726 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 530 (ObjNum 548 snap 171 seq_num 548) dirty exists 2024-08-17T17:00:30.726 INFO:tasks.rados.rados.0.smithi063.stdout:1396: left oid 24 (ObjNum 548 snap 171 seq_num 548) 2024-08-17T17:00:30.726 INFO:tasks.rados.rados.0.smithi063.stdout:1399: finishing write tid 1 to smithi06387500-36 2024-08-17T17:00:30.726 INFO:tasks.rados.rados.0.smithi063.stdout:1399: finishing write tid 2 to smithi06387500-36 2024-08-17T17:00:30.726 INFO:tasks.rados.rados.0.smithi063.stdout:1399: finishing write tid 3 to smithi06387500-36 2024-08-17T17:00:30.726 INFO:tasks.rados.rados.0.smithi063.stdout:1399: finishing write tid 4 to smithi06387500-36 2024-08-17T17:00:30.726 INFO:tasks.rados.rados.0.smithi063.stdout:1399: finishing write tid 5 to smithi06387500-36 2024-08-17T17:00:30.726 INFO:tasks.rados.rados.0.smithi063.stdout:1399: oid 36 updating version 0 to 1398 2024-08-17T17:00:30.726 INFO:tasks.rados.rados.0.smithi063.stdout:1399: oid 36 updating version 1398 to 1399 2024-08-17T17:00:30.726 INFO:tasks.rados.rados.0.smithi063.stdout:1399: oid 36 updating version 1399 to 1400 2024-08-17T17:00:30.726 INFO:tasks.rados.rados.0.smithi063.stdout:1399: oid 36 updating version 1400 to 1401 2024-08-17T17:00:30.727 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 1401 (ObjNum 549 snap 171 seq_num 549) dirty exists 2024-08-17T17:00:30.727 INFO:tasks.rados.rados.0.smithi063.stdout:1399: left oid 36 (ObjNum 549 snap 171 seq_num 549) 2024-08-17T17:00:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:30 smithi063 ceph-mon[114756]: osdmap e409: 8 total, 8 up, 8 in 2024-08-17T17:00:31.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:30 smithi063 ceph-mon[109843]: osdmap e409: 8 total, 8 up, 8 in 2024-08-17T17:00:31.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:30 smithi162 ceph-mon[34633]: osdmap e409: 8 total, 8 up, 8 in 2024-08-17T17:00:31.611 INFO:tasks.rados.rados.0.smithi063.stdout:1394: done (4 left) 2024-08-17T17:00:31.611 INFO:tasks.rados.rados.0.smithi063.stdout:1395: done (3 left) 2024-08-17T17:00:31.611 INFO:tasks.rados.rados.0.smithi063.stdout:1396: done (2 left) 2024-08-17T17:00:31.611 INFO:tasks.rados.rados.0.smithi063.stdout:1399: done (1 left) 2024-08-17T17:00:31.611 INFO:tasks.rados.rados.0.smithi063.stdout:1400: done (0 left) 2024-08-17T17:00:31.611 INFO:tasks.rados.rados.0.smithi063.stdout:1401: write oid 24 current snap is 172 2024-08-17T17:00:31.612 INFO:tasks.rados.rados.0.smithi063.stdout:1401: seq_num 550 ranges {443850=564366,1634861=732856,2770646=473204} 2024-08-17T17:00:31.616 INFO:tasks.rados.rados.0.smithi063.stdout:1401: writing smithi06387500-24 from 443850 to 1008216 tid 1 2024-08-17T17:00:31.624 INFO:tasks.rados.rados.0.smithi063.stdout:1401: writing smithi06387500-24 from 1634861 to 2367717 tid 2 2024-08-17T17:00:31.629 INFO:tasks.rados.rados.0.smithi063.stdout:1401: writing smithi06387500-24 from 2770646 to 3243850 tid 3 2024-08-17T17:00:31.630 INFO:tasks.rados.rados.0.smithi063.stdout:1402: read oid 49 snap -1 2024-08-17T17:00:31.630 INFO:tasks.rados.rados.0.smithi063.stdout:1402: expect (ObjNum 449 snap 145 seq_num 449) 2024-08-17T17:00:31.630 INFO:tasks.rados.rados.0.smithi063.stdout:1403: write oid 9 current snap is 172 2024-08-17T17:00:31.630 INFO:tasks.rados.rados.0.smithi063.stdout:1401: finishing write tid 1 to smithi06387500-24 2024-08-17T17:00:31.630 INFO:tasks.rados.rados.0.smithi063.stdout:1401: finishing write tid 2 to smithi06387500-24 2024-08-17T17:00:31.630 INFO:tasks.rados.rados.0.smithi063.stdout:1403: seq_num 551 ranges {460657=639615,1693343=476514,2681716=578941} 2024-08-17T17:00:31.636 INFO:tasks.rados.rados.0.smithi063.stdout:1403: writing smithi06387500-9 from 460657 to 1100272 tid 1 2024-08-17T17:00:31.643 INFO:tasks.rados.rados.0.smithi063.stdout:1403: writing smithi06387500-9 from 1693343 to 2169857 tid 2 2024-08-17T17:00:31.649 INFO:tasks.rados.rados.0.smithi063.stdout:1403: writing smithi06387500-9 from 2681716 to 3260657 tid 3 2024-08-17T17:00:31.649 INFO:tasks.rados.rados.0.smithi063.stdout:1404: read oid 12 snap 161 2024-08-17T17:00:31.650 INFO:tasks.rados.rados.0.smithi063.stdout:1404: expect (ObjNum 369 snap 115 seq_num 369) 2024-08-17T17:00:31.650 INFO:tasks.rados.rados.0.smithi063.stdout:1405: delete oid 10 current snap is 172 2024-08-17T17:00:31.650 INFO:tasks.rados.rados.0.smithi063.stdout:1401: finishing write tid 3 to smithi06387500-24 2024-08-17T17:00:31.650 INFO:tasks.rados.rados.0.smithi063.stdout:1401: finishing write tid 4 to smithi06387500-24 2024-08-17T17:00:31.650 INFO:tasks.rados.rados.0.smithi063.stdout:1401: finishing write tid 5 to smithi06387500-24 2024-08-17T17:00:31.650 INFO:tasks.rados.rados.0.smithi063.stdout:1401: oid 24 updating version 0 to 536 2024-08-17T17:00:31.650 INFO:tasks.rados.rados.0.smithi063.stdout:1401: oid 24 updating version 536 to 537 2024-08-17T17:00:31.650 INFO:tasks.rados.rados.0.smithi063.stdout:1401: oid 24 updating version 537 to 539 2024-08-17T17:00:31.650 INFO:tasks.rados.rados.0.smithi063.stdout:1401: oid 24 version 539 is already newer than 538 2024-08-17T17:00:31.650 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 539 (ObjNum 550 snap 172 seq_num 550) dirty exists 2024-08-17T17:00:31.650 INFO:tasks.rados.rados.0.smithi063.stdout:1401: left oid 24 (ObjNum 550 snap 172 seq_num 550) 2024-08-17T17:00:31.650 INFO:tasks.rados.rados.0.smithi063.stdout:1402: expect (ObjNum 449 snap 145 seq_num 449) 2024-08-17T17:00:31.805 INFO:tasks.rados.rados.0.smithi063.stdout:1403: finishing write tid 1 to smithi06387500-9 2024-08-17T17:00:31.805 INFO:tasks.rados.rados.0.smithi063.stdout:1403: finishing write tid 2 to smithi06387500-9 2024-08-17T17:00:31.805 INFO:tasks.rados.rados.0.smithi063.stdout:1404: expect (ObjNum 369 snap 115 seq_num 369) 2024-08-17T17:00:31.922 INFO:tasks.rados.rados.0.smithi063.stdout:1403: finishing write tid 3 to smithi06387500-9 2024-08-17T17:00:31.922 INFO:tasks.rados.rados.0.smithi063.stdout:1403: finishing write tid 4 to smithi06387500-9 2024-08-17T17:00:31.923 INFO:tasks.rados.rados.0.smithi063.stdout:1403: finishing write tid 5 to smithi06387500-9 2024-08-17T17:00:31.923 INFO:tasks.rados.rados.0.smithi063.stdout:1403: oid 9 updating version 0 to 691 2024-08-17T17:00:31.923 INFO:tasks.rados.rados.0.smithi063.stdout:1403: oid 9 updating version 691 to 692 2024-08-17T17:00:31.923 INFO:tasks.rados.rados.0.smithi063.stdout:1403: oid 9 version 692 is already newer than 690 2024-08-17T17:00:31.923 INFO:tasks.rados.rados.0.smithi063.stdout:1403: oid 9 updating version 692 to 693 2024-08-17T17:00:31.923 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 693 (ObjNum 551 snap 172 seq_num 551) dirty exists 2024-08-17T17:00:31.923 INFO:tasks.rados.rados.0.smithi063.stdout:1403: left oid 9 (ObjNum 551 snap 172 seq_num 551) 2024-08-17T17:00:31.923 INFO:tasks.rados.rados.0.smithi063.stdout:1401: done (4 left) 2024-08-17T17:00:31.923 INFO:tasks.rados.rados.0.smithi063.stdout:1402: done (3 left) 2024-08-17T17:00:31.923 INFO:tasks.rados.rados.0.smithi063.stdout:1403: done (2 left) 2024-08-17T17:00:31.923 INFO:tasks.rados.rados.0.smithi063.stdout:1404: done (1 left) 2024-08-17T17:00:31.923 INFO:tasks.rados.rados.0.smithi063.stdout:1405: done (0 left) 2024-08-17T17:00:31.923 INFO:tasks.rados.rados.0.smithi063.stdout:1406: snap_remove snap 161 2024-08-17T17:00:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:31 smithi063 ceph-mon[114756]: pgmap v67: 89 pgs: 89 active+clean; 234 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-08-17T17:00:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:31 smithi063 ceph-mon[114756]: osdmap e410: 8 total, 8 up, 8 in 2024-08-17T17:00:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:32.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:31 smithi063 ceph-mon[109843]: pgmap v67: 89 pgs: 89 active+clean; 234 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-08-17T17:00:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:31 smithi063 ceph-mon[109843]: osdmap e410: 8 total, 8 up, 8 in 2024-08-17T17:00:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:31 smithi162 ceph-mon[34633]: pgmap v67: 89 pgs: 89 active+clean; 234 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-08-17T17:00:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:31 smithi162 ceph-mon[34633]: osdmap e410: 8 total, 8 up, 8 in 2024-08-17T17:00:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:31 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:32.511 INFO:tasks.rados.rados.0.smithi063.stdout:1406: done (0 left) 2024-08-17T17:00:32.511 INFO:tasks.rados.rados.0.smithi063.stdout:1407: write oid 21 current snap is 172 2024-08-17T17:00:32.511 INFO:tasks.rados.rados.0.smithi063.stdout:1407: seq_num 553 ranges {494271=790113,1693954=480183,2620209=512129} 2024-08-17T17:00:32.519 INFO:tasks.rados.rados.0.smithi063.stdout:1407: writing smithi06387500-21 from 494271 to 1284384 tid 1 2024-08-17T17:00:32.524 INFO:tasks.rados.rados.0.smithi063.stdout:1407: writing smithi06387500-21 from 1693954 to 2174137 tid 2 2024-08-17T17:00:32.530 INFO:tasks.rados.rados.0.smithi063.stdout:1407: writing smithi06387500-21 from 2620209 to 3132338 tid 3 2024-08-17T17:00:32.530 INFO:tasks.rados.rados.0.smithi063.stdout:1408: read oid 25 snap -1 2024-08-17T17:00:32.530 INFO:tasks.rados.rados.0.smithi063.stdout:1408: expect deleted 2024-08-17T17:00:32.530 INFO:tasks.rados.rados.0.smithi063.stdout:1409: delete oid 19 current snap is 172 2024-08-17T17:00:32.530 INFO:tasks.rados.rados.0.smithi063.stdout:1407: finishing write tid 1 to smithi06387500-21 2024-08-17T17:00:32.530 INFO:tasks.rados.rados.0.smithi063.stdout:1407: finishing write tid 2 to smithi06387500-21 2024-08-17T17:00:32.533 INFO:tasks.rados.rados.0.smithi063.stdout:1407: finishing write tid 3 to smithi06387500-21 2024-08-17T17:00:32.533 INFO:tasks.rados.rados.0.smithi063.stdout:1407: finishing write tid 4 to smithi06387500-21 2024-08-17T17:00:32.533 INFO:tasks.rados.rados.0.smithi063.stdout:1407: finishing write tid 5 to smithi06387500-21 2024-08-17T17:00:32.533 INFO:tasks.rados.rados.0.smithi063.stdout:1407: oid 21 updating version 0 to 1021 2024-08-17T17:00:32.533 INFO:tasks.rados.rados.0.smithi063.stdout:1407: oid 21 updating version 1021 to 1025 2024-08-17T17:00:32.533 INFO:tasks.rados.rados.0.smithi063.stdout:1407: oid 21 version 1025 is already newer than 1024 2024-08-17T17:00:32.533 INFO:tasks.rados.rados.0.smithi063.stdout:1407: oid 21 version 1025 is already newer than 1016 2024-08-17T17:00:32.533 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 1025 (ObjNum 553 snap 172 seq_num 553) dirty exists 2024-08-17T17:00:32.534 INFO:tasks.rados.rados.0.smithi063.stdout:1407: left oid 21 (ObjNum 553 snap 172 seq_num 553) 2024-08-17T17:00:32.534 INFO:tasks.rados.rados.0.smithi063.stdout:1407: done (2 left) 2024-08-17T17:00:32.534 INFO:tasks.rados.rados.0.smithi063.stdout:1408: done (1 left) 2024-08-17T17:00:32.534 INFO:tasks.rados.rados.0.smithi063.stdout:1409: done (0 left) 2024-08-17T17:00:32.534 INFO:tasks.rados.rados.0.smithi063.stdout:1410: delete oid 24 current snap is 172 2024-08-17T17:00:32.539 INFO:tasks.rados.rados.0.smithi063.stdout:1410: done (0 left) 2024-08-17T17:00:32.539 INFO:tasks.rados.rados.0.smithi063.stdout:1411: write oid 34 current snap is 172 2024-08-17T17:00:32.539 INFO:tasks.rados.rados.0.smithi063.stdout:1411: seq_num 556 ranges {544692=732213,1985753=743863,3335184=9508} 2024-08-17T17:00:32.545 INFO:tasks.rados.rados.0.smithi063.stdout:1411: writing smithi06387500-34 from 544692 to 1276905 tid 1 2024-08-17T17:00:32.553 INFO:tasks.rados.rados.0.smithi063.stdout:1411: writing smithi06387500-34 from 1985753 to 2729616 tid 2 2024-08-17T17:00:32.557 INFO:tasks.rados.rados.0.smithi063.stdout:1411: writing smithi06387500-34 from 3335184 to 3344692 tid 3 2024-08-17T17:00:32.557 INFO:tasks.rados.rados.0.smithi063.stdout:1412: write oid 8 current snap is 172 2024-08-17T17:00:32.557 INFO:tasks.rados.rados.0.smithi063.stdout:1411: finishing write tid 1 to smithi06387500-34 2024-08-17T17:00:32.557 INFO:tasks.rados.rados.0.smithi063.stdout:1411: finishing write tid 2 to smithi06387500-34 2024-08-17T17:00:32.558 INFO:tasks.rados.rados.0.smithi063.stdout:1412: seq_num 557 ranges {561499=407462,1527882=487521,2729901=557217} 2024-08-17T17:00:32.562 INFO:tasks.rados.rados.0.smithi063.stdout:1412: writing smithi06387500-8 from 561499 to 968961 tid 1 2024-08-17T17:00:32.568 INFO:tasks.rados.rados.0.smithi063.stdout:1412: writing smithi06387500-8 from 1527882 to 2015403 tid 2 2024-08-17T17:00:32.575 INFO:tasks.rados.rados.0.smithi063.stdout:1412: writing smithi06387500-8 from 2729901 to 3287118 tid 3 2024-08-17T17:00:32.575 INFO:tasks.rados.rados.0.smithi063.stdout:1413: read oid 28 snap -1 2024-08-17T17:00:32.576 INFO:tasks.rados.rados.0.smithi063.stdout:1413: expect deleted 2024-08-17T17:00:32.576 INFO:tasks.rados.rados.0.smithi063.stdout:1414: snap_remove snap 163 2024-08-17T17:00:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:32 smithi063 ceph-mon[114756]: osdmap e411: 8 total, 8 up, 8 in 2024-08-17T17:00:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:32 smithi063 ceph-mon[114756]: pgmap v70: 89 pgs: 89 active+clean; 248 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-08-17T17:00:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:32 smithi063 ceph-mon[114756]: osdmap e412: 8 total, 8 up, 8 in 2024-08-17T17:00:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:32 smithi063 ceph-mon[109843]: osdmap e411: 8 total, 8 up, 8 in 2024-08-17T17:00:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:32 smithi063 ceph-mon[109843]: pgmap v70: 89 pgs: 89 active+clean; 248 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-08-17T17:00:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:32 smithi063 ceph-mon[109843]: osdmap e412: 8 total, 8 up, 8 in 2024-08-17T17:00:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:32 smithi162 ceph-mon[34633]: osdmap e411: 8 total, 8 up, 8 in 2024-08-17T17:00:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:32 smithi162 ceph-mon[34633]: pgmap v70: 89 pgs: 89 active+clean; 248 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-08-17T17:00:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:32 smithi162 ceph-mon[34633]: osdmap e412: 8 total, 8 up, 8 in 2024-08-17T17:00:33.512 INFO:tasks.rados.rados.0.smithi063.stdout:1414: done (3 left) 2024-08-17T17:00:33.512 INFO:tasks.rados.rados.0.smithi063.stdout:1415: read oid 17 snap -1 2024-08-17T17:00:33.512 INFO:tasks.rados.rados.0.smithi063.stdout:1415: expect (ObjNum 537 snap 169 seq_num 537) 2024-08-17T17:00:33.513 INFO:tasks.rados.rados.0.smithi063.stdout:1416: snap_remove snap 165 2024-08-17T17:00:33.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:00:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:00:33] "GET /metrics HTTP/1.1" 200 36205 "" "Prometheus/2.51.0" 2024-08-17T17:00:34.524 INFO:tasks.rados.rados.0.smithi063.stdout:1411: finishing write tid 3 to smithi06387500-34 2024-08-17T17:00:34.525 INFO:tasks.rados.rados.0.smithi063.stdout:1411: finishing write tid 4 to smithi06387500-34 2024-08-17T17:00:34.525 INFO:tasks.rados.rados.0.smithi063.stdout:1411: finishing write tid 5 to smithi06387500-34 2024-08-17T17:00:34.525 INFO:tasks.rados.rados.0.smithi063.stdout:1411: oid 34 updating version 0 to 1029 2024-08-17T17:00:34.525 INFO:tasks.rados.rados.0.smithi063.stdout:1411: oid 34 updating version 1029 to 1030 2024-08-17T17:00:34.525 INFO:tasks.rados.rados.0.smithi063.stdout:1411: oid 34 version 1030 is already newer than 1028 2024-08-17T17:00:34.525 INFO:tasks.rados.rados.0.smithi063.stdout:1411: oid 34 updating version 1030 to 1031 2024-08-17T17:00:34.525 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 1031 (ObjNum 556 snap 172 seq_num 556) dirty exists 2024-08-17T17:00:34.525 INFO:tasks.rados.rados.0.smithi063.stdout:1411: left oid 34 (ObjNum 556 snap 172 seq_num 556) 2024-08-17T17:00:34.525 INFO:tasks.rados.rados.0.smithi063.stdout:1412: finishing write tid 1 to smithi06387500-8 2024-08-17T17:00:34.525 INFO:tasks.rados.rados.0.smithi063.stdout:1412: finishing write tid 2 to smithi06387500-8 2024-08-17T17:00:34.525 INFO:tasks.rados.rados.0.smithi063.stdout:1412: finishing write tid 3 to smithi06387500-8 2024-08-17T17:00:34.525 INFO:tasks.rados.rados.0.smithi063.stdout:1412: finishing write tid 4 to smithi06387500-8 2024-08-17T17:00:34.525 INFO:tasks.rados.rados.0.smithi063.stdout:1412: finishing write tid 5 to smithi06387500-8 2024-08-17T17:00:34.526 INFO:tasks.rados.rados.0.smithi063.stdout:1412: oid 8 updating version 0 to 421 2024-08-17T17:00:34.526 INFO:tasks.rados.rados.0.smithi063.stdout:1412: oid 8 updating version 421 to 422 2024-08-17T17:00:34.526 INFO:tasks.rados.rados.0.smithi063.stdout:1412: oid 8 version 422 is already newer than 419 2024-08-17T17:00:34.526 INFO:tasks.rados.rados.0.smithi063.stdout:1412: oid 8 version 422 is already newer than 420 2024-08-17T17:00:34.526 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 422 (ObjNum 557 snap 172 seq_num 557) dirty exists 2024-08-17T17:00:34.526 INFO:tasks.rados.rados.0.smithi063.stdout:1412: left oid 8 (ObjNum 557 snap 172 seq_num 557) 2024-08-17T17:00:34.526 INFO:tasks.rados.rados.0.smithi063.stdout:1415: expect (ObjNum 537 snap 169 seq_num 537) 2024-08-17T17:00:34.668 INFO:tasks.rados.rados.0.smithi063.stdout:1411: done (4 left) 2024-08-17T17:00:34.668 INFO:tasks.rados.rados.0.smithi063.stdout:1412: done (3 left) 2024-08-17T17:00:34.668 INFO:tasks.rados.rados.0.smithi063.stdout:1413: done (2 left) 2024-08-17T17:00:34.668 INFO:tasks.rados.rados.0.smithi063.stdout:1415: done (1 left) 2024-08-17T17:00:34.668 INFO:tasks.rados.rados.0.smithi063.stdout:1416: done (0 left) 2024-08-17T17:00:34.668 INFO:tasks.rados.rados.0.smithi063.stdout:1417: snap_create 2024-08-17T17:00:34.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:34 smithi063 ceph-mon[114756]: osdmap e413: 8 total, 8 up, 8 in 2024-08-17T17:00:34.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:34 smithi063 ceph-mon[109843]: osdmap e413: 8 total, 8 up, 8 in 2024-08-17T17:00:34.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:34 smithi162 ceph-mon[34633]: osdmap e413: 8 total, 8 up, 8 in 2024-08-17T17:00:35.522 INFO:tasks.rados.rados.0.smithi063.stdout:1417: done (0 left) 2024-08-17T17:00:35.522 INFO:tasks.rados.rados.0.smithi063.stdout:1418: delete oid 36 current snap is 173 2024-08-17T17:00:35.529 INFO:tasks.rados.rados.0.smithi063.stdout:1418: done (0 left) 2024-08-17T17:00:35.529 INFO:tasks.rados.rados.0.smithi063.stdout:1419: read oid 29 snap 154 2024-08-17T17:00:35.530 INFO:tasks.rados.rados.0.smithi063.stdout:1419: expect (ObjNum 468 snap 152 seq_num 468) 2024-08-17T17:00:35.530 INFO:tasks.rados.rados.0.smithi063.stdout:1420: write oid 27 current snap is 173 2024-08-17T17:00:35.530 INFO:tasks.rados.rados.0.smithi063.stdout:1420: seq_num 559 ranges {595113=557960,1644846=491190,2784747=610366} 2024-08-17T17:00:35.536 INFO:tasks.rados.rados.0.smithi063.stdout:1420: writing smithi06387500-27 from 595113 to 1153073 tid 1 2024-08-17T17:00:35.542 INFO:tasks.rados.rados.0.smithi063.stdout:1420: writing smithi06387500-27 from 1644846 to 2136036 tid 2 2024-08-17T17:00:35.549 INFO:tasks.rados.rados.0.smithi063.stdout:1420: writing smithi06387500-27 from 2784747 to 3395113 tid 3 2024-08-17T17:00:35.549 INFO:tasks.rados.rados.0.smithi063.stdout:1421: read oid 43 snap -1 2024-08-17T17:00:35.549 INFO:tasks.rados.rados.0.smithi063.stdout:1421: expect (ObjNum 483 snap 153 seq_num 483) 2024-08-17T17:00:35.549 INFO:tasks.rados.rados.0.smithi063.stdout:1422: delete oid 44 current snap is 173 2024-08-17T17:00:35.549 INFO:tasks.rados.rados.0.smithi063.stdout:1419: expect (ObjNum 468 snap 152 seq_num 468) 2024-08-17T17:00:35.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:35 smithi063 ceph-mon[114756]: pgmap v73: 89 pgs: 89 active+clean; 248 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-08-17T17:00:35.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:35 smithi063 ceph-mon[114756]: osdmap e414: 8 total, 8 up, 8 in 2024-08-17T17:00:35.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:35 smithi063 ceph-mon[109843]: pgmap v73: 89 pgs: 89 active+clean; 248 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-08-17T17:00:35.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:35 smithi063 ceph-mon[109843]: osdmap e414: 8 total, 8 up, 8 in 2024-08-17T17:00:35.791 INFO:tasks.rados.rados.0.smithi063.stdout:1420: finishing write tid 1 to smithi06387500-27 2024-08-17T17:00:35.791 INFO:tasks.rados.rados.0.smithi063.stdout:1420: finishing write tid 2 to smithi06387500-27 2024-08-17T17:00:35.791 INFO:tasks.rados.rados.0.smithi063.stdout:1420: finishing write tid 3 to smithi06387500-27 2024-08-17T17:00:35.791 INFO:tasks.rados.rados.0.smithi063.stdout:1420: finishing write tid 4 to smithi06387500-27 2024-08-17T17:00:35.791 INFO:tasks.rados.rados.0.smithi063.stdout:1420: finishing write tid 5 to smithi06387500-27 2024-08-17T17:00:35.791 INFO:tasks.rados.rados.0.smithi063.stdout:1420: oid 27 updating version 0 to 174 2024-08-17T17:00:35.791 INFO:tasks.rados.rados.0.smithi063.stdout:1420: oid 27 version 174 is already newer than 172 2024-08-17T17:00:35.791 INFO:tasks.rados.rados.0.smithi063.stdout:1420: oid 27 version 174 is already newer than 173 2024-08-17T17:00:35.791 INFO:tasks.rados.rados.0.smithi063.stdout:1420: oid 27 version 174 is already newer than 171 2024-08-17T17:00:35.791 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 174 (ObjNum 559 snap 173 seq_num 559) dirty exists 2024-08-17T17:00:35.791 INFO:tasks.rados.rados.0.smithi063.stdout:1420: left oid 27 (ObjNum 559 snap 173 seq_num 559) 2024-08-17T17:00:35.791 INFO:tasks.rados.rados.0.smithi063.stdout:1421: expect (ObjNum 483 snap 153 seq_num 483) 2024-08-17T17:00:35.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:35 smithi162 ceph-mon[34633]: pgmap v73: 89 pgs: 89 active+clean; 248 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-08-17T17:00:35.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:35 smithi162 ceph-mon[34633]: osdmap e414: 8 total, 8 up, 8 in 2024-08-17T17:00:35.863 INFO:tasks.rados.rados.0.smithi063.stdout:1419: done (3 left) 2024-08-17T17:00:35.863 INFO:tasks.rados.rados.0.smithi063.stdout:1420: done (2 left) 2024-08-17T17:00:35.863 INFO:tasks.rados.rados.0.smithi063.stdout:1421: done (1 left) 2024-08-17T17:00:35.863 INFO:tasks.rados.rados.0.smithi063.stdout:1422: done (0 left) 2024-08-17T17:00:35.863 INFO:tasks.rados.rados.0.smithi063.stdout:1423: write oid 9 current snap is 173 2024-08-17T17:00:35.863 INFO:tasks.rados.rados.0.smithi063.stdout:1423: seq_num 561 ranges {628727=708458,2045457=494859,3123240=305487} 2024-08-17T17:00:35.870 INFO:tasks.rados.rados.0.smithi063.stdout:1423: writing smithi06387500-9 from 628727 to 1337185 tid 1 2024-08-17T17:00:35.877 INFO:tasks.rados.rados.0.smithi063.stdout:1423: writing smithi06387500-9 from 2045457 to 2540316 tid 2 2024-08-17T17:00:35.882 INFO:tasks.rados.rados.0.smithi063.stdout:1423: writing smithi06387500-9 from 3123240 to 3428727 tid 3 2024-08-17T17:00:35.882 INFO:tasks.rados.rados.0.smithi063.stdout:1424: snap_remove snap 159 2024-08-17T17:00:36.502 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'sleep 60' 2024-08-17T17:00:36.532 INFO:tasks.rados.rados.0.smithi063.stdout:1424: done (1 left) 2024-08-17T17:00:36.532 INFO:tasks.rados.rados.0.smithi063.stdout:1425: read oid 15 snap -1 2024-08-17T17:00:36.532 INFO:tasks.rados.rados.0.smithi063.stdout:1425: expect (ObjNum 289 snap 88 seq_num 289) 2024-08-17T17:00:36.532 INFO:tasks.rados.rados.0.smithi063.stdout:1426: write oid 19 current snap is 173 2024-08-17T17:00:36.533 INFO:tasks.rados.rados.0.smithi063.stdout:1423: finishing write tid 1 to smithi06387500-9 2024-08-17T17:00:36.533 INFO:tasks.rados.rados.0.smithi063.stdout:1423: finishing write tid 2 to smithi06387500-9 2024-08-17T17:00:36.533 INFO:tasks.rados.rados.0.smithi063.stdout:1423: finishing write tid 3 to smithi06387500-9 2024-08-17T17:00:36.533 INFO:tasks.rados.rados.0.smithi063.stdout:1423: finishing write tid 4 to smithi06387500-9 2024-08-17T17:00:36.533 INFO:tasks.rados.rados.0.smithi063.stdout:1426: seq_num 562 ranges {645534=783707,2103939=638517,3150663=294871} 2024-08-17T17:00:36.541 INFO:tasks.rados.rados.0.smithi063.stdout:1426: writing smithi06387500-19 from 645534 to 1429241 tid 1 2024-08-17T17:00:36.549 INFO:tasks.rados.rados.0.smithi063.stdout:1426: writing smithi06387500-19 from 2103939 to 2742456 tid 2 2024-08-17T17:00:36.553 INFO:tasks.rados.rados.0.smithi063.stdout:1426: writing smithi06387500-19 from 3150663 to 3445534 tid 3 2024-08-17T17:00:36.554 INFO:tasks.rados.rados.0.smithi063.stdout:1427: write oid 47 current snap is 173 2024-08-17T17:00:36.554 INFO:tasks.rados.rados.0.smithi063.stdout:1423: finishing write tid 5 to smithi06387500-9 2024-08-17T17:00:36.554 INFO:tasks.rados.rados.0.smithi063.stdout:1423: oid 9 updating version 0 to 695 2024-08-17T17:00:36.554 INFO:tasks.rados.rados.0.smithi063.stdout:1423: oid 9 updating version 695 to 698 2024-08-17T17:00:36.554 INFO:tasks.rados.rados.0.smithi063.stdout:1423: oid 9 version 698 is already newer than 697 2024-08-17T17:00:36.554 INFO:tasks.rados.rados.0.smithi063.stdout:1423: oid 9 version 698 is already newer than 696 2024-08-17T17:00:36.554 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 698 (ObjNum 561 snap 173 seq_num 561) dirty exists 2024-08-17T17:00:36.554 INFO:tasks.rados.rados.0.smithi063.stdout:1423: left oid 9 (ObjNum 561 snap 173 seq_num 561) 2024-08-17T17:00:36.554 INFO:tasks.rados.rados.0.smithi063.stdout:1427: seq_num 563 ranges {662341=575309,1878774=498528,2894439=457555} 2024-08-17T17:00:36.560 INFO:tasks.rados.rados.0.smithi063.stdout:1427: writing smithi06387500-47 from 662341 to 1237650 tid 1 2024-08-17T17:00:36.567 INFO:tasks.rados.rados.0.smithi063.stdout:1427: writing smithi06387500-47 from 1878774 to 2377302 tid 2 2024-08-17T17:00:36.572 INFO:tasks.rados.rados.0.smithi063.stdout:1427: writing smithi06387500-47 from 2894439 to 3351994 tid 3 2024-08-17T17:00:36.572 INFO:tasks.rados.rados.0.smithi063.stdout:1423: done (3 left) 2024-08-17T17:00:36.572 INFO:tasks.rados.rados.0.smithi063.stdout:1428: read oid 28 snap -1 2024-08-17T17:00:36.572 INFO:tasks.rados.rados.0.smithi063.stdout:1428: expect deleted 2024-08-17T17:00:36.573 INFO:tasks.rados.rados.0.smithi063.stdout:1429: write oid 50 current snap is 173 2024-08-17T17:00:36.573 INFO:tasks.rados.rados.0.smithi063.stdout:1429: seq_num 564 ranges {679148=650558,1937256=642186,3321862=157286} 2024-08-17T17:00:36.579 INFO:tasks.rados.rados.0.smithi063.stdout:1429: writing smithi06387500-50 from 679148 to 1329706 tid 1 2024-08-17T17:00:36.586 INFO:tasks.rados.rados.0.smithi063.stdout:1429: writing smithi06387500-50 from 1937256 to 2579442 tid 2 2024-08-17T17:00:36.591 INFO:tasks.rados.rados.0.smithi063.stdout:1429: writing smithi06387500-50 from 3321862 to 3479148 tid 3 2024-08-17T17:00:36.591 INFO:tasks.rados.rados.0.smithi063.stdout:1430: rollback oid 49 current snap is 173 2024-08-17T17:00:36.592 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 49 to 171 2024-08-17T17:00:36.592 INFO:tasks.rados.rados.0.smithi063.stdout:1425: expect (ObjNum 289 snap 88 seq_num 289) 2024-08-17T17:00:36.715 INFO:tasks.rados.rados.0.smithi063.stdout:1426: finishing write tid 1 to smithi06387500-19 2024-08-17T17:00:36.715 INFO:tasks.rados.rados.0.smithi063.stdout:1426: finishing write tid 2 to smithi06387500-19 2024-08-17T17:00:36.715 INFO:tasks.rados.rados.0.smithi063.stdout:1425: done (5 left) 2024-08-17T17:00:36.715 INFO:tasks.rados.rados.0.smithi063.stdout:1431: write oid 29 current snap is 173 2024-08-17T17:00:36.716 INFO:tasks.rados.rados.0.smithi063.stdout:1426: finishing write tid 3 to smithi06387500-19 2024-08-17T17:00:36.716 INFO:tasks.rados.rados.0.smithi063.stdout:1431: seq_num 565 ranges {695955=725807,1879385=502197,2949285=480099} 2024-08-17T17:00:36.723 INFO:tasks.rados.rados.0.smithi063.stdout:1431: writing smithi06387500-29 from 695955 to 1421762 tid 1 2024-08-17T17:00:36.728 INFO:tasks.rados.rados.0.smithi063.stdout:1431: writing smithi06387500-29 from 1879385 to 2381582 tid 2 2024-08-17T17:00:36.734 INFO:tasks.rados.rados.0.smithi063.stdout:1431: writing smithi06387500-29 from 2949285 to 3429384 tid 3 2024-08-17T17:00:36.734 INFO:tasks.rados.rados.0.smithi063.stdout:1432: write oid 37 current snap is 173 2024-08-17T17:00:36.734 INFO:tasks.rados.rados.0.smithi063.stdout:1432: seq_num 566 ranges {712762=401056,1537867=645855,2860355=652407} 2024-08-17T17:00:36.740 INFO:tasks.rados.rados.0.smithi063.stdout:1432: writing smithi06387500-37 from 712762 to 1113818 tid 1 2024-08-17T17:00:36.746 INFO:tasks.rados.rados.0.smithi063.stdout:1432: writing smithi06387500-37 from 1537867 to 2183722 tid 2 2024-08-17T17:00:36.753 INFO:tasks.rados.rados.0.smithi063.stdout:1432: writing smithi06387500-37 from 2860355 to 3512762 tid 3 2024-08-17T17:00:36.754 INFO:tasks.rados.rados.0.smithi063.stdout:1433: snap_remove snap 167 2024-08-17T17:00:36.763 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:00:36.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:36 smithi063 ceph-mon[114756]: osdmap e415: 8 total, 8 up, 8 in 2024-08-17T17:00:36.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:36 smithi063 ceph-mon[109843]: osdmap e415: 8 total, 8 up, 8 in 2024-08-17T17:00:36.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:36 smithi162 ceph-mon[34633]: osdmap e415: 8 total, 8 up, 8 in 2024-08-17T17:00:37.540 INFO:tasks.rados.rados.0.smithi063.stdout:1433: done (7 left) 2024-08-17T17:00:37.541 INFO:tasks.rados.rados.0.smithi063.stdout:1434: read oid 35 snap -1 2024-08-17T17:00:37.541 INFO:tasks.rados.rados.0.smithi063.stdout:1434: expect (ObjNum 541 snap 170 seq_num 541) 2024-08-17T17:00:37.541 INFO:tasks.rados.rados.0.smithi063.stdout:1435: rollback oid 5 current snap is 173 2024-08-17T17:00:37.541 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 157 2024-08-17T17:00:37.541 INFO:tasks.rados.rados.0.smithi063.stdout:1436: rollback oid 13 current snap is 173 2024-08-17T17:00:37.541 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 13 to 162 2024-08-17T17:00:37.541 INFO:tasks.rados.rados.0.smithi063.stdout:1437: write oid 16 current snap is 173 2024-08-17T17:00:37.541 INFO:tasks.rados.rados.0.smithi063.stdout:1437: seq_num 567 ranges {729569=476305,1996349=505866,3004131=525438} 2024-08-17T17:00:37.548 INFO:tasks.rados.rados.0.smithi063.stdout:1437: writing smithi06387500-16 from 729569 to 1205874 tid 1 2024-08-17T17:00:37.556 INFO:tasks.rados.rados.0.smithi063.stdout:1437: writing smithi06387500-16 from 1996349 to 2502215 tid 2 2024-08-17T17:00:37.563 INFO:tasks.rados.rados.0.smithi063.stdout:1437: writing smithi06387500-16 from 3004131 to 3529569 tid 3 2024-08-17T17:00:37.563 INFO:tasks.rados.rados.0.smithi063.stdout:1438: rollback oid 26 current snap is 173 2024-08-17T17:00:37.563 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 26 to 157 2024-08-17T17:00:37.563 INFO:tasks.rados.rados.0.smithi063.stdout:1426: finishing write tid 4 to smithi06387500-19 2024-08-17T17:00:37.563 INFO:tasks.rados.rados.0.smithi063.stdout:1426: finishing write tid 5 to smithi06387500-19 2024-08-17T17:00:37.564 INFO:tasks.rados.rados.0.smithi063.stdout:1426: oid 19 updating version 0 to 1058 2024-08-17T17:00:37.564 INFO:tasks.rados.rados.0.smithi063.stdout:1426: oid 19 version 1058 is already newer than 1057 2024-08-17T17:00:37.564 INFO:tasks.rados.rados.0.smithi063.stdout:1426: oid 19 updating version 1058 to 1059 2024-08-17T17:00:37.564 INFO:tasks.rados.rados.0.smithi063.stdout:1426: oid 19 version 1059 is already newer than 1050 2024-08-17T17:00:37.564 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 1059 (ObjNum 562 snap 173 seq_num 562) dirty exists 2024-08-17T17:00:37.564 INFO:tasks.rados.rados.0.smithi063.stdout:1426: left oid 19 (ObjNum 562 snap 173 seq_num 562) 2024-08-17T17:00:37.564 INFO:tasks.rados.rados.0.smithi063.stdout:1427: finishing write tid 1 to smithi06387500-47 2024-08-17T17:00:37.564 INFO:tasks.rados.rados.0.smithi063.stdout:1427: finishing write tid 2 to smithi06387500-47 2024-08-17T17:00:37.564 INFO:tasks.rados.rados.0.smithi063.stdout:1427: finishing write tid 3 to smithi06387500-47 2024-08-17T17:00:37.564 INFO:tasks.rados.rados.0.smithi063.stdout:1427: finishing write tid 4 to smithi06387500-47 2024-08-17T17:00:37.564 INFO:tasks.rados.rados.0.smithi063.stdout:1427: finishing write tid 5 to smithi06387500-47 2024-08-17T17:00:37.564 INFO:tasks.rados.rados.0.smithi063.stdout:1427: oid 47 updating version 0 to 263 2024-08-17T17:00:37.564 INFO:tasks.rados.rados.0.smithi063.stdout:1427: oid 47 version 263 is already newer than 260 2024-08-17T17:00:37.564 INFO:tasks.rados.rados.0.smithi063.stdout:1427: oid 47 version 263 is already newer than 261 2024-08-17T17:00:37.564 INFO:tasks.rados.rados.0.smithi063.stdout:1427: oid 47 version 263 is already newer than 262 2024-08-17T17:00:37.565 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 263 (ObjNum 563 snap 173 seq_num 563) dirty exists 2024-08-17T17:00:37.565 INFO:tasks.rados.rados.0.smithi063.stdout:1427: left oid 47 (ObjNum 563 snap 173 seq_num 563) 2024-08-17T17:00:37.565 INFO:tasks.rados.rados.0.smithi063.stdout:1429: finishing write tid 1 to smithi06387500-50 2024-08-17T17:00:37.565 INFO:tasks.rados.rados.0.smithi063.stdout:1429: finishing write tid 2 to smithi06387500-50 2024-08-17T17:00:37.565 INFO:tasks.rados.rados.0.smithi063.stdout:1429: finishing write tid 3 to smithi06387500-50 2024-08-17T17:00:37.565 INFO:tasks.rados.rados.0.smithi063.stdout:1426: done (11 left) 2024-08-17T17:00:37.565 INFO:tasks.rados.rados.0.smithi063.stdout:1427: done (10 left) 2024-08-17T17:00:37.565 INFO:tasks.rados.rados.0.smithi063.stdout:1428: done (9 left) 2024-08-17T17:00:37.565 INFO:tasks.rados.rados.0.smithi063.stdout:1439: rollback oid 3 current snap is 173 2024-08-17T17:00:37.565 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 3 to 141 2024-08-17T17:00:37.565 INFO:tasks.rados.rados.0.smithi063.stdout:1429: finishing write tid 4 to smithi06387500-50 2024-08-17T17:00:37.565 INFO:tasks.rados.rados.0.smithi063.stdout:1429: finishing write tid 5 to smithi06387500-50 2024-08-17T17:00:37.565 INFO:tasks.rados.rados.0.smithi063.stdout:1429: oid 50 updating version 0 to 1062 2024-08-17T17:00:37.566 INFO:tasks.rados.rados.0.smithi063.stdout:1429: oid 50 updating version 1062 to 1063 2024-08-17T17:00:37.566 INFO:tasks.rados.rados.0.smithi063.stdout:1429: oid 50 version 1063 is already newer than 1060 2024-08-17T17:00:37.566 INFO:tasks.rados.rados.0.smithi063.stdout:1429: oid 50 version 1063 is already newer than 1061 2024-08-17T17:00:37.566 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 1063 (ObjNum 564 snap 173 seq_num 564) dirty exists 2024-08-17T17:00:37.566 INFO:tasks.rados.rados.0.smithi063.stdout:1429: left oid 50 (ObjNum 564 snap 173 seq_num 564) 2024-08-17T17:00:37.566 INFO:tasks.rados.rados.0.smithi063.stdout:1430: finishing rollback tid 0 to smithi06387500-49 2024-08-17T17:00:37.566 INFO:tasks.rados.rados.0.smithi063.stdout:1430: finishing rollback tid 1 to smithi06387500-49 2024-08-17T17:00:37.566 INFO:tasks.rados.rados.0.smithi063.stdout:1430: finishing rollback tid 2 to smithi06387500-49 2024-08-17T17:00:37.566 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 1437 (ObjNum 449 snap 145 seq_num 449) dirty exists 2024-08-17T17:00:37.566 INFO:tasks.rados.rados.0.smithi063.stdout:1429: done (9 left) 2024-08-17T17:00:37.566 INFO:tasks.rados.rados.0.smithi063.stdout:1430: done (8 left) 2024-08-17T17:00:37.566 INFO:tasks.rados.rados.0.smithi063.stdout:1440: read oid 2 snap -1 2024-08-17T17:00:37.566 INFO:tasks.rados.rados.0.smithi063.stdout:1440: expect deleted 2024-08-17T17:00:37.566 INFO:tasks.rados.rados.0.smithi063.stdout:1441: snap_remove snap 154 2024-08-17T17:00:37.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:37 smithi063 ceph-mon[114756]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:37.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:37 smithi063 ceph-mon[114756]: pgmap v76: 89 pgs: 89 active+clean; 248 MiB data, 762 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:00:37.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:37 smithi063 ceph-mon[114756]: osdmap e416: 8 total, 8 up, 8 in 2024-08-17T17:00:37.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:37 smithi063 ceph-mon[109843]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:37.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:37 smithi063 ceph-mon[109843]: pgmap v76: 89 pgs: 89 active+clean; 248 MiB data, 762 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:00:37.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:37 smithi063 ceph-mon[109843]: osdmap e416: 8 total, 8 up, 8 in 2024-08-17T17:00:37.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:37 smithi162 ceph-mon[34633]: from='client.34173 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:00:37.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:37 smithi162 ceph-mon[34633]: pgmap v76: 89 pgs: 89 active+clean; 248 MiB data, 762 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:00:37.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:37 smithi162 ceph-mon[34633]: osdmap e416: 8 total, 8 up, 8 in 2024-08-17T17:00:38.548 INFO:tasks.rados.rados.0.smithi063.stdout:1441: done (9 left) 2024-08-17T17:00:38.548 INFO:tasks.rados.rados.0.smithi063.stdout:1442: read oid 30 snap -1 2024-08-17T17:00:38.548 INFO:tasks.rados.rados.0.smithi063.stdout:1442: expect (ObjNum 507 snap 160 seq_num 507) 2024-08-17T17:00:38.548 INFO:tasks.rados.rados.0.smithi063.stdout:1443: snap_create 2024-08-17T17:00:38.548 INFO:tasks.rados.rados.0.smithi063.stdout:1431: finishing write tid 1 to smithi06387500-29 2024-08-17T17:00:38.548 INFO:tasks.rados.rados.0.smithi063.stdout:1431: finishing write tid 2 to smithi06387500-29 2024-08-17T17:00:38.549 INFO:tasks.rados.rados.0.smithi063.stdout:1431: finishing write tid 3 to smithi06387500-29 2024-08-17T17:00:38.549 INFO:tasks.rados.rados.0.smithi063.stdout:1431: finishing write tid 4 to smithi06387500-29 2024-08-17T17:00:38.549 INFO:tasks.rados.rados.0.smithi063.stdout:1431: finishing write tid 5 to smithi06387500-29 2024-08-17T17:00:38.549 INFO:tasks.rados.rados.0.smithi063.stdout:1431: oid 29 updating version 0 to 326 2024-08-17T17:00:38.549 INFO:tasks.rados.rados.0.smithi063.stdout:1431: oid 29 updating version 326 to 327 2024-08-17T17:00:38.549 INFO:tasks.rados.rados.0.smithi063.stdout:1431: oid 29 version 327 is already newer than 325 2024-08-17T17:00:38.549 INFO:tasks.rados.rados.0.smithi063.stdout:1431: oid 29 updating version 327 to 328 2024-08-17T17:00:38.549 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 328 (ObjNum 565 snap 173 seq_num 565) dirty exists 2024-08-17T17:00:38.549 INFO:tasks.rados.rados.0.smithi063.stdout:1431: left oid 29 (ObjNum 565 snap 173 seq_num 565) 2024-08-17T17:00:38.549 INFO:tasks.rados.rados.0.smithi063.stdout:1432: finishing write tid 1 to smithi06387500-37 2024-08-17T17:00:38.549 INFO:tasks.rados.rados.0.smithi063.stdout:1432: finishing write tid 2 to smithi06387500-37 2024-08-17T17:00:38.549 INFO:tasks.rados.rados.0.smithi063.stdout:1432: finishing write tid 3 to smithi06387500-37 2024-08-17T17:00:38.549 INFO:tasks.rados.rados.0.smithi063.stdout:1432: finishing write tid 4 to smithi06387500-37 2024-08-17T17:00:38.549 INFO:tasks.rados.rados.0.smithi063.stdout:1432: finishing write tid 5 to smithi06387500-37 2024-08-17T17:00:38.550 INFO:tasks.rados.rados.0.smithi063.stdout:1432: oid 37 updating version 0 to 1439 2024-08-17T17:00:38.550 INFO:tasks.rados.rados.0.smithi063.stdout:1432: oid 37 updating version 1439 to 1440 2024-08-17T17:00:38.550 INFO:tasks.rados.rados.0.smithi063.stdout:1432: oid 37 updating version 1440 to 1441 2024-08-17T17:00:38.550 INFO:tasks.rados.rados.0.smithi063.stdout:1432: oid 37 updating version 1441 to 1442 2024-08-17T17:00:38.550 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 1442 (ObjNum 566 snap 173 seq_num 566) dirty exists 2024-08-17T17:00:38.550 INFO:tasks.rados.rados.0.smithi063.stdout:1432: left oid 37 (ObjNum 566 snap 173 seq_num 566) 2024-08-17T17:00:38.550 INFO:tasks.rados.rados.0.smithi063.stdout:1435: finishing rollback tid 1 to smithi06387500-5 2024-08-17T17:00:38.550 INFO:tasks.rados.rados.0.smithi063.stdout:1435: finishing rollback tid 2 to smithi06387500-5 2024-08-17T17:00:38.550 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 707 (ObjNum 443 snap 139 seq_num 443) dirty exists 2024-08-17T17:00:38.550 INFO:tasks.rados.rados.0.smithi063.stdout:1436: finishing rollback tid 1 to smithi06387500-13 2024-08-17T17:00:38.550 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T17:00:38.550 INFO:tasks.rados.rados.0.smithi063.stdout:1437: finishing write tid 1 to smithi06387500-16 2024-08-17T17:00:38.550 INFO:tasks.rados.rados.0.smithi063.stdout:1434: expect (ObjNum 541 snap 170 seq_num 541) 2024-08-17T17:00:38.701 INFO:tasks.rados.rados.0.smithi063.stdout:1437: finishing write tid 2 to smithi06387500-16 2024-08-17T17:00:38.701 INFO:tasks.rados.rados.0.smithi063.stdout:1437: finishing write tid 3 to smithi06387500-16 2024-08-17T17:00:38.701 INFO:tasks.rados.rados.0.smithi063.stdout:1437: finishing write tid 4 to smithi06387500-16 2024-08-17T17:00:38.701 INFO:tasks.rados.rados.0.smithi063.stdout:1437: finishing write tid 5 to smithi06387500-16 2024-08-17T17:00:38.701 INFO:tasks.rados.rados.0.smithi063.stdout:1437: oid 16 updating version 0 to 562 2024-08-17T17:00:38.701 INFO:tasks.rados.rados.0.smithi063.stdout:1437: oid 16 version 562 is already newer than 560 2024-08-17T17:00:38.701 INFO:tasks.rados.rados.0.smithi063.stdout:1437: oid 16 updating version 562 to 563 2024-08-17T17:00:38.701 INFO:tasks.rados.rados.0.smithi063.stdout:1437: oid 16 version 563 is already newer than 561 2024-08-17T17:00:38.702 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 563 (ObjNum 567 snap 173 seq_num 567) dirty exists 2024-08-17T17:00:38.702 INFO:tasks.rados.rados.0.smithi063.stdout:1437: left oid 16 (ObjNum 567 snap 173 seq_num 567) 2024-08-17T17:00:38.702 INFO:tasks.rados.rados.0.smithi063.stdout:1438: finishing rollback tid 0 to smithi06387500-26 2024-08-17T17:00:38.702 INFO:tasks.rados.rados.0.smithi063.stdout:1438: finishing rollback tid 1 to smithi06387500-26 2024-08-17T17:00:38.702 INFO:tasks.rados.rados.0.smithi063.stdout:1438: finishing rollback tid 2 to smithi06387500-26 2024-08-17T17:00:38.702 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 450 (ObjNum 463 snap 150 seq_num 463) dirty exists 2024-08-17T17:00:38.702 INFO:tasks.rados.rados.0.smithi063.stdout:1439: finishing rollback tid 1 to smithi06387500-3 2024-08-17T17:00:38.702 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-08-17T17:00:38.702 INFO:tasks.rados.rados.0.smithi063.stdout:1442: expect (ObjNum 507 snap 160 seq_num 507) 2024-08-17T17:00:38.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:38 smithi162 ceph-mon[34633]: osdmap e417: 8 total, 8 up, 8 in 2024-08-17T17:00:39.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:38 smithi063 ceph-mon[109843]: osdmap e417: 8 total, 8 up, 8 in 2024-08-17T17:00:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:38 smithi063 ceph-mon[114756]: osdmap e417: 8 total, 8 up, 8 in 2024-08-17T17:00:39.544 INFO:tasks.rados.rados.0.smithi063.stdout:1431: done (10 left) 2024-08-17T17:00:39.545 INFO:tasks.rados.rados.0.smithi063.stdout:1432: done (9 left) 2024-08-17T17:00:39.545 INFO:tasks.rados.rados.0.smithi063.stdout:1434: done (8 left) 2024-08-17T17:00:39.545 INFO:tasks.rados.rados.0.smithi063.stdout:1435: done (7 left) 2024-08-17T17:00:39.545 INFO:tasks.rados.rados.0.smithi063.stdout:1436: done (6 left) 2024-08-17T17:00:39.545 INFO:tasks.rados.rados.0.smithi063.stdout:1437: done (5 left) 2024-08-17T17:00:39.545 INFO:tasks.rados.rados.0.smithi063.stdout:1438: done (4 left) 2024-08-17T17:00:39.545 INFO:tasks.rados.rados.0.smithi063.stdout:1439: done (3 left) 2024-08-17T17:00:39.545 INFO:tasks.rados.rados.0.smithi063.stdout:1440: done (2 left) 2024-08-17T17:00:39.545 INFO:tasks.rados.rados.0.smithi063.stdout:1442: done (1 left) 2024-08-17T17:00:39.545 INFO:tasks.rados.rados.0.smithi063.stdout:1443: done (0 left) 2024-08-17T17:00:39.545 INFO:tasks.rados.rados.0.smithi063.stdout:1444: snap_create 2024-08-17T17:00:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:39 smithi162 ceph-mon[34633]: pgmap v79: 89 pgs: 89 active+clean; 270 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 3.8 MiB/s wr, 24 op/s 2024-08-17T17:00:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:39 smithi162 ceph-mon[34633]: osdmap e418: 8 total, 8 up, 8 in 2024-08-17T17:00:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:39 smithi063 ceph-mon[109843]: pgmap v79: 89 pgs: 89 active+clean; 270 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 3.8 MiB/s wr, 24 op/s 2024-08-17T17:00:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:39 smithi063 ceph-mon[109843]: osdmap e418: 8 total, 8 up, 8 in 2024-08-17T17:00:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:39 smithi063 ceph-mon[114756]: pgmap v79: 89 pgs: 89 active+clean; 270 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 3.8 MiB/s wr, 24 op/s 2024-08-17T17:00:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:39 smithi063 ceph-mon[114756]: osdmap e418: 8 total, 8 up, 8 in 2024-08-17T17:00:40.556 INFO:tasks.rados.rados.0.smithi063.stdout:1444: done (0 left) 2024-08-17T17:00:40.556 INFO:tasks.rados.rados.0.smithi063.stdout:1445: write oid 17 current snap is 175 2024-08-17T17:00:40.556 INFO:tasks.rados.rados.0.smithi063.stdout:1445: seq_num 568 ranges {746376=551554,2054831=649524,3315201=231175} 2024-08-17T17:00:40.563 INFO:tasks.rados.rados.0.smithi063.stdout:1445: writing smithi06387500-17 from 746376 to 1297930 tid 1 2024-08-17T17:00:40.571 INFO:tasks.rados.rados.0.smithi063.stdout:1445: writing smithi06387500-17 from 2054831 to 2704355 tid 2 2024-08-17T17:00:40.576 INFO:tasks.rados.rados.0.smithi063.stdout:1445: writing smithi06387500-17 from 3315201 to 3546376 tid 3 2024-08-17T17:00:40.576 INFO:tasks.rados.rados.0.smithi063.stdout:1446: delete oid 4 current snap is 175 2024-08-17T17:00:40.576 INFO:tasks.rados.rados.0.smithi063.stdout:1445: finishing write tid 1 to smithi06387500-17 2024-08-17T17:00:40.576 INFO:tasks.rados.rados.0.smithi063.stdout:1445: finishing write tid 2 to smithi06387500-17 2024-08-17T17:00:40.579 INFO:tasks.rados.rados.0.smithi063.stdout:1446: done (1 left) 2024-08-17T17:00:40.579 INFO:tasks.rados.rados.0.smithi063.stdout:1447: snap_create 2024-08-17T17:00:40.579 INFO:tasks.rados.rados.0.smithi063.stdout:1445: finishing write tid 3 to smithi06387500-17 2024-08-17T17:00:40.579 INFO:tasks.rados.rados.0.smithi063.stdout:1445: finishing write tid 4 to smithi06387500-17 2024-08-17T17:00:40.579 INFO:tasks.rados.rados.0.smithi063.stdout:1445: finishing write tid 5 to smithi06387500-17 2024-08-17T17:00:40.580 INFO:tasks.rados.rados.0.smithi063.stdout:1445: oid 17 updating version 0 to 457 2024-08-17T17:00:40.580 INFO:tasks.rados.rados.0.smithi063.stdout:1445: oid 17 updating version 457 to 459 2024-08-17T17:00:40.580 INFO:tasks.rados.rados.0.smithi063.stdout:1445: oid 17 version 459 is already newer than 458 2024-08-17T17:00:40.580 INFO:tasks.rados.rados.0.smithi063.stdout:1445: oid 17 updating version 459 to 460 2024-08-17T17:00:40.580 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 460 (ObjNum 568 snap 175 seq_num 568) dirty exists 2024-08-17T17:00:40.580 INFO:tasks.rados.rados.0.smithi063.stdout:1445: left oid 17 (ObjNum 568 snap 175 seq_num 568) 2024-08-17T17:00:40.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:40 smithi162 ceph-mon[34633]: osdmap e419: 8 total, 8 up, 8 in 2024-08-17T17:00:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:40 smithi063 ceph-mon[114756]: osdmap e419: 8 total, 8 up, 8 in 2024-08-17T17:00:41.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:40 smithi063 ceph-mon[109843]: osdmap e419: 8 total, 8 up, 8 in 2024-08-17T17:00:41.558 INFO:tasks.rados.rados.0.smithi063.stdout:1445: done (1 left) 2024-08-17T17:00:41.558 INFO:tasks.rados.rados.0.smithi063.stdout:1447: done (0 left) 2024-08-17T17:00:41.558 INFO:tasks.rados.rados.0.smithi063.stdout:1448: snap_create 2024-08-17T17:00:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:41 smithi162 ceph-mon[34633]: pgmap v82: 89 pgs: 89 active+clean; 270 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 3.8 MiB/s wr, 24 op/s 2024-08-17T17:00:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:41 smithi162 ceph-mon[34633]: osdmap e420: 8 total, 8 up, 8 in 2024-08-17T17:00:41.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:41 smithi063 ceph-mon[114756]: pgmap v82: 89 pgs: 89 active+clean; 270 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 3.8 MiB/s wr, 24 op/s 2024-08-17T17:00:41.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:41 smithi063 ceph-mon[114756]: osdmap e420: 8 total, 8 up, 8 in 2024-08-17T17:00:41.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:41 smithi063 ceph-mon[109843]: pgmap v82: 89 pgs: 89 active+clean; 270 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 3.8 MiB/s wr, 24 op/s 2024-08-17T17:00:41.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:41 smithi063 ceph-mon[109843]: osdmap e420: 8 total, 8 up, 8 in 2024-08-17T17:00:42.517 INFO:tasks.rados.rados.0.smithi063.stdout:1448: done (0 left) 2024-08-17T17:00:42.518 INFO:tasks.rados.rados.0.smithi063.stdout:1449: rollback oid 17 current snap is 177 2024-08-17T17:00:42.518 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 17 to 173 2024-08-17T17:00:42.518 INFO:tasks.rados.rados.0.smithi063.stdout:1450: read oid 29 snap -1 2024-08-17T17:00:42.518 INFO:tasks.rados.rados.0.smithi063.stdout:1450: expect (ObjNum 565 snap 173 seq_num 565) 2024-08-17T17:00:42.518 INFO:tasks.rados.rados.0.smithi063.stdout:1451: snap_create 2024-08-17T17:00:42.524 INFO:tasks.rados.rados.0.smithi063.stdout:1449: finishing rollback tid 0 to smithi06387500-17 2024-08-17T17:00:42.524 INFO:tasks.rados.rados.0.smithi063.stdout:1449: finishing rollback tid 1 to smithi06387500-17 2024-08-17T17:00:42.524 INFO:tasks.rados.rados.0.smithi063.stdout:1449: finishing rollback tid 2 to smithi06387500-17 2024-08-17T17:00:42.524 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 464 (ObjNum 537 snap 169 seq_num 537) dirty exists 2024-08-17T17:00:42.525 INFO:tasks.rados.rados.0.smithi063.stdout:1450: expect (ObjNum 565 snap 173 seq_num 565) 2024-08-17T17:00:42.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:42 smithi162 ceph-mon[34633]: osdmap e421: 8 total, 8 up, 8 in 2024-08-17T17:00:42.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:42 smithi162 ceph-mon[34633]: osdmap e422: 8 total, 8 up, 8 in 2024-08-17T17:00:42.946 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:42 smithi063 ceph-mon[114756]: osdmap e421: 8 total, 8 up, 8 in 2024-08-17T17:00:42.946 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:42 smithi063 ceph-mon[114756]: osdmap e422: 8 total, 8 up, 8 in 2024-08-17T17:00:42.946 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:42 smithi063 ceph-mon[109843]: osdmap e421: 8 total, 8 up, 8 in 2024-08-17T17:00:42.947 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:42 smithi063 ceph-mon[109843]: osdmap e422: 8 total, 8 up, 8 in 2024-08-17T17:00:43.516 INFO:tasks.rados.rados.0.smithi063.stdout:1449: done (2 left) 2024-08-17T17:00:43.516 INFO:tasks.rados.rados.0.smithi063.stdout:1450: done (1 left) 2024-08-17T17:00:43.516 INFO:tasks.rados.rados.0.smithi063.stdout:1451: done (0 left) 2024-08-17T17:00:43.516 INFO:tasks.rados.rados.0.smithi063.stdout:1452: delete oid 6 current snap is 178 2024-08-17T17:00:43.521 INFO:tasks.rados.rados.0.smithi063.stdout:1452: done (0 left) 2024-08-17T17:00:43.521 INFO:tasks.rados.rados.0.smithi063.stdout:1453: rollback oid 25 current snap is 178 2024-08-17T17:00:43.521 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 25 to 171 2024-08-17T17:00:43.521 INFO:tasks.rados.rados.0.smithi063.stdout:1454: read oid 45 snap -1 2024-08-17T17:00:43.521 INFO:tasks.rados.rados.0.smithi063.stdout:1454: expect deleted 2024-08-17T17:00:43.521 INFO:tasks.rados.rados.0.smithi063.stdout:1455: read oid 21 snap -1 2024-08-17T17:00:43.521 INFO:tasks.rados.rados.0.smithi063.stdout:1455: expect (ObjNum 553 snap 172 seq_num 553) 2024-08-17T17:00:43.521 INFO:tasks.rados.rados.0.smithi063.stdout:1456: read oid 46 snap -1 2024-08-17T17:00:43.521 INFO:tasks.rados.rados.0.smithi063.stdout:1456: expect (ObjNum 510 snap 160 seq_num 510) 2024-08-17T17:00:43.521 INFO:tasks.rados.rados.0.smithi063.stdout:1457: rollback oid 40 current snap is 178 2024-08-17T17:00:43.521 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 40 to 175 2024-08-17T17:00:43.522 INFO:tasks.rados.rados.0.smithi063.stdout:1458: snap_remove snap 174 2024-08-17T17:00:43.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:00:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:00:43] "GET /metrics HTTP/1.1" 200 36205 "" "Prometheus/2.51.0" 2024-08-17T17:00:43.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:43 smithi162 ceph-mon[34633]: pgmap v85: 89 pgs: 89 active+clean; 266 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 719 KiB/s wr, 8 op/s 2024-08-17T17:00:43.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:43 smithi162 ceph-mon[34633]: osdmap e423: 8 total, 8 up, 8 in 2024-08-17T17:00:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:43 smithi063 ceph-mon[114756]: pgmap v85: 89 pgs: 89 active+clean; 266 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 719 KiB/s wr, 8 op/s 2024-08-17T17:00:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:43 smithi063 ceph-mon[114756]: osdmap e423: 8 total, 8 up, 8 in 2024-08-17T17:00:44.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:43 smithi063 ceph-mon[109843]: pgmap v85: 89 pgs: 89 active+clean; 266 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 719 KiB/s wr, 8 op/s 2024-08-17T17:00:44.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:43 smithi063 ceph-mon[109843]: osdmap e423: 8 total, 8 up, 8 in 2024-08-17T17:00:44.517 INFO:tasks.rados.rados.0.smithi063.stdout:1458: done (5 left) 2024-08-17T17:00:44.518 INFO:tasks.rados.rados.0.smithi063.stdout:1459: read oid 49 snap -1 2024-08-17T17:00:44.518 INFO:tasks.rados.rados.0.smithi063.stdout:1459: expect (ObjNum 449 snap 145 seq_num 449) 2024-08-17T17:00:44.518 INFO:tasks.rados.rados.0.smithi063.stdout:1460: snap_remove snap 130 2024-08-17T17:00:45.523 INFO:tasks.rados.rados.0.smithi063.stdout:1460: done (6 left) 2024-08-17T17:00:45.523 INFO:tasks.rados.rados.0.smithi063.stdout:1461: snap_remove snap 172 2024-08-17T17:00:45.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:45 smithi063 ceph-mon[109843]: pgmap v88: 89 pgs: 89 active+clean; 266 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 719 KiB/s wr, 8 op/s 2024-08-17T17:00:45.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:45 smithi063 ceph-mon[109843]: osdmap e424: 8 total, 8 up, 8 in 2024-08-17T17:00:45.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:45 smithi063 ceph-mon[114756]: pgmap v88: 89 pgs: 89 active+clean; 266 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 719 KiB/s wr, 8 op/s 2024-08-17T17:00:45.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:45 smithi063 ceph-mon[114756]: osdmap e424: 8 total, 8 up, 8 in 2024-08-17T17:00:45.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:45 smithi162 ceph-mon[34633]: pgmap v88: 89 pgs: 89 active+clean; 266 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 719 KiB/s wr, 8 op/s 2024-08-17T17:00:45.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:45 smithi162 ceph-mon[34633]: osdmap e424: 8 total, 8 up, 8 in 2024-08-17T17:00:46.542 INFO:tasks.rados.rados.0.smithi063.stdout:1461: done (6 left) 2024-08-17T17:00:46.542 INFO:tasks.rados.rados.0.smithi063.stdout:1462: read oid 7 snap -1 2024-08-17T17:00:46.542 INFO:tasks.rados.rados.0.smithi063.stdout:1462: expect deleted 2024-08-17T17:00:46.542 INFO:tasks.rados.rados.0.smithi063.stdout:1463: snap_remove snap 162 2024-08-17T17:00:46.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:46 smithi063 ceph-mon[114756]: osdmap e425: 8 total, 8 up, 8 in 2024-08-17T17:00:46.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:46 smithi063 ceph-mon[109843]: osdmap e425: 8 total, 8 up, 8 in 2024-08-17T17:00:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:46.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:46 smithi162 ceph-mon[34633]: osdmap e425: 8 total, 8 up, 8 in 2024-08-17T17:00:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:00:47.519 INFO:tasks.rados.rados.0.smithi063.stdout:1463: done (7 left) 2024-08-17T17:00:47.519 INFO:tasks.rados.rados.0.smithi063.stdout:1464: delete oid 19 current snap is 178 2024-08-17T17:00:47.520 INFO:tasks.rados.rados.0.smithi063.stdout:1457: finishing rollback tid 1 to smithi06387500-40 2024-08-17T17:00:47.520 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 397 (ObjNum 129 snap 0 seq_num 1806991488) dirty dne 2024-08-17T17:00:47.520 INFO:tasks.rados.rados.0.smithi063.stdout:1453: finishing rollback tid 1 to smithi06387500-25 2024-08-17T17:00:47.520 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 1376 (ObjNum 129 snap 0 seq_num 1807273584) dirty dne 2024-08-17T17:00:47.520 INFO:tasks.rados.rados.0.smithi063.stdout:1456: expect (ObjNum 510 snap 160 seq_num 510) 2024-08-17T17:00:47.699 INFO:tasks.rados.rados.0.smithi063.stdout:1455: expect (ObjNum 553 snap 172 seq_num 553) 2024-08-17T17:00:47.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:47 smithi063 ceph-mon[114756]: pgmap v91: 89 pgs: 89 active+clean; 266 MiB data, 800 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:00:47.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:47 smithi063 ceph-mon[114756]: osdmap e426: 8 total, 8 up, 8 in 2024-08-17T17:00:47.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:47 smithi063 ceph-mon[114756]: osdmap e427: 8 total, 8 up, 8 in 2024-08-17T17:00:47.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:47 smithi063 ceph-mon[109843]: pgmap v91: 89 pgs: 89 active+clean; 266 MiB data, 800 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:00:47.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:47 smithi063 ceph-mon[109843]: osdmap e426: 8 total, 8 up, 8 in 2024-08-17T17:00:47.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:47 smithi063 ceph-mon[109843]: osdmap e427: 8 total, 8 up, 8 in 2024-08-17T17:00:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:47 smithi162 ceph-mon[34633]: pgmap v91: 89 pgs: 89 active+clean; 266 MiB data, 800 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:00:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:47 smithi162 ceph-mon[34633]: osdmap e426: 8 total, 8 up, 8 in 2024-08-17T17:00:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:47 smithi162 ceph-mon[34633]: osdmap e427: 8 total, 8 up, 8 in 2024-08-17T17:00:47.927 INFO:tasks.rados.rados.0.smithi063.stdout:1459: expect (ObjNum 449 snap 145 seq_num 449) 2024-08-17T17:00:48.082 INFO:tasks.rados.rados.0.smithi063.stdout:1453: done (7 left) 2024-08-17T17:00:48.082 INFO:tasks.rados.rados.0.smithi063.stdout:1454: done (6 left) 2024-08-17T17:00:48.082 INFO:tasks.rados.rados.0.smithi063.stdout:1455: done (5 left) 2024-08-17T17:00:48.082 INFO:tasks.rados.rados.0.smithi063.stdout:1456: done (4 left) 2024-08-17T17:00:48.082 INFO:tasks.rados.rados.0.smithi063.stdout:1457: done (3 left) 2024-08-17T17:00:48.082 INFO:tasks.rados.rados.0.smithi063.stdout:1459: done (2 left) 2024-08-17T17:00:48.082 INFO:tasks.rados.rados.0.smithi063.stdout:1462: done (1 left) 2024-08-17T17:00:48.082 INFO:tasks.rados.rados.0.smithi063.stdout:1464: done (0 left) 2024-08-17T17:00:48.082 INFO:tasks.rados.rados.0.smithi063.stdout:1465: write oid 14 current snap is 178 2024-08-17T17:00:48.082 INFO:tasks.rados.rados.0.smithi063.stdout:1465: seq_num 572 ranges {413604=568903,1488759=656862,2624893=591709} 2024-08-17T17:00:48.087 INFO:tasks.rados.rados.0.smithi063.stdout:1465: writing smithi06387500-14 from 413604 to 982507 tid 1 2024-08-17T17:00:48.093 INFO:tasks.rados.rados.0.smithi063.stdout:1465: writing smithi06387500-14 from 1488759 to 2145621 tid 2 2024-08-17T17:00:48.099 INFO:tasks.rados.rados.0.smithi063.stdout:1465: writing smithi06387500-14 from 2624893 to 3216602 tid 3 2024-08-17T17:00:48.099 INFO:tasks.rados.rados.0.smithi063.stdout:1466: read oid 30 snap -1 2024-08-17T17:00:48.099 INFO:tasks.rados.rados.0.smithi063.stdout:1466: expect (ObjNum 507 snap 160 seq_num 507) 2024-08-17T17:00:48.099 INFO:tasks.rados.rados.0.smithi063.stdout:1467: snap_create 2024-08-17T17:00:48.099 INFO:tasks.rados.rados.0.smithi063.stdout:1465: finishing write tid 1 to smithi06387500-14 2024-08-17T17:00:48.100 INFO:tasks.rados.rados.0.smithi063.stdout:1465: finishing write tid 2 to smithi06387500-14 2024-08-17T17:00:48.103 INFO:tasks.rados.rados.0.smithi063.stdout:1465: finishing write tid 3 to smithi06387500-14 2024-08-17T17:00:48.104 INFO:tasks.rados.rados.0.smithi063.stdout:1465: finishing write tid 4 to smithi06387500-14 2024-08-17T17:00:48.104 INFO:tasks.rados.rados.0.smithi063.stdout:1465: finishing write tid 5 to smithi06387500-14 2024-08-17T17:00:48.104 INFO:tasks.rados.rados.0.smithi063.stdout:1465: oid 14 updating version 0 to 1499 2024-08-17T17:00:48.104 INFO:tasks.rados.rados.0.smithi063.stdout:1465: oid 14 version 1499 is already newer than 1497 2024-08-17T17:00:48.104 INFO:tasks.rados.rados.0.smithi063.stdout:1465: oid 14 updating version 1499 to 1500 2024-08-17T17:00:48.104 INFO:tasks.rados.rados.0.smithi063.stdout:1465: oid 14 version 1500 is already newer than 1498 2024-08-17T17:00:48.104 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 1500 (ObjNum 572 snap 178 seq_num 572) dirty exists 2024-08-17T17:00:48.104 INFO:tasks.rados.rados.0.smithi063.stdout:1465: left oid 14 (ObjNum 572 snap 178 seq_num 572) 2024-08-17T17:00:48.109 INFO:tasks.rados.rados.0.smithi063.stdout:1466: expect (ObjNum 507 snap 160 seq_num 507) 2024-08-17T17:00:48.540 INFO:tasks.rados.rados.0.smithi063.stdout:1465: done (2 left) 2024-08-17T17:00:48.540 INFO:tasks.rados.rados.0.smithi063.stdout:1466: done (1 left) 2024-08-17T17:00:48.540 INFO:tasks.rados.rados.0.smithi063.stdout:1467: done (0 left) 2024-08-17T17:00:48.540 INFO:tasks.rados.rados.0.smithi063.stdout:1468: write oid 32 current snap is 179 2024-08-17T17:00:48.540 INFO:tasks.rados.rados.0.smithi063.stdout:1468: seq_num 573 ranges {430411=644152,1830888=400520,2935963=402981} 2024-08-17T17:00:48.545 INFO:tasks.rados.rados.0.smithi063.stdout:1468: writing smithi06387500-32 from 430411 to 1074563 tid 1 2024-08-17T17:00:48.552 INFO:tasks.rados.rados.0.smithi063.stdout:1468: writing smithi06387500-32 from 1830888 to 2231408 tid 2 2024-08-17T17:00:48.558 INFO:tasks.rados.rados.0.smithi063.stdout:1468: writing smithi06387500-32 from 2935963 to 3338944 tid 3 2024-08-17T17:00:48.558 INFO:tasks.rados.rados.0.smithi063.stdout:1469: snap_create 2024-08-17T17:00:48.558 INFO:tasks.rados.rados.0.smithi063.stdout:1468: finishing write tid 1 to smithi06387500-32 2024-08-17T17:00:48.559 INFO:tasks.rados.rados.0.smithi063.stdout:1468: finishing write tid 2 to smithi06387500-32 2024-08-17T17:00:48.561 INFO:tasks.rados.rados.0.smithi063.stdout:1468: finishing write tid 3 to smithi06387500-32 2024-08-17T17:00:48.561 INFO:tasks.rados.rados.0.smithi063.stdout:1468: finishing write tid 4 to smithi06387500-32 2024-08-17T17:00:48.561 INFO:tasks.rados.rados.0.smithi063.stdout:1468: finishing write tid 5 to smithi06387500-32 2024-08-17T17:00:48.561 INFO:tasks.rados.rados.0.smithi063.stdout:1468: oid 32 updating version 0 to 279 2024-08-17T17:00:48.561 INFO:tasks.rados.rados.0.smithi063.stdout:1468: oid 32 updating version 279 to 280 2024-08-17T17:00:48.561 INFO:tasks.rados.rados.0.smithi063.stdout:1468: oid 32 version 280 is already newer than 278 2024-08-17T17:00:48.561 INFO:tasks.rados.rados.0.smithi063.stdout:1468: oid 32 version 280 is already newer than 277 2024-08-17T17:00:48.561 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 280 (ObjNum 573 snap 179 seq_num 573) dirty exists 2024-08-17T17:00:48.561 INFO:tasks.rados.rados.0.smithi063.stdout:1468: left oid 32 (ObjNum 573 snap 179 seq_num 573) 2024-08-17T17:00:49.546 INFO:tasks.rados.rados.0.smithi063.stdout:1468: done (1 left) 2024-08-17T17:00:49.546 INFO:tasks.rados.rados.0.smithi063.stdout:1469: done (0 left) 2024-08-17T17:00:49.546 INFO:tasks.rados.rados.0.smithi063.stdout:1470: write oid 12 current snap is 180 2024-08-17T17:00:49.546 INFO:tasks.rados.rados.0.smithi063.stdout:1470: seq_num 574 ranges {447218=719401,1889370=660531,2963386=614253} 2024-08-17T17:00:49.552 INFO:tasks.rados.rados.0.smithi063.stdout:1470: writing smithi06387500-12 from 447218 to 1166619 tid 1 2024-08-17T17:00:49.560 INFO:tasks.rados.rados.0.smithi063.stdout:1470: writing smithi06387500-12 from 1889370 to 2549901 tid 2 2024-08-17T17:00:49.565 INFO:tasks.rados.rados.0.smithi063.stdout:1470: writing smithi06387500-12 from 2963386 to 3577639 tid 3 2024-08-17T17:00:49.566 INFO:tasks.rados.rados.0.smithi063.stdout:1471: snap_create 2024-08-17T17:00:49.566 INFO:tasks.rados.rados.0.smithi063.stdout:1470: finishing write tid 1 to smithi06387500-12 2024-08-17T17:00:49.566 INFO:tasks.rados.rados.0.smithi063.stdout:1470: finishing write tid 2 to smithi06387500-12 2024-08-17T17:00:49.570 INFO:tasks.rados.rados.0.smithi063.stdout:1470: finishing write tid 3 to smithi06387500-12 2024-08-17T17:00:49.570 INFO:tasks.rados.rados.0.smithi063.stdout:1470: finishing write tid 4 to smithi06387500-12 2024-08-17T17:00:49.570 INFO:tasks.rados.rados.0.smithi063.stdout:1470: finishing write tid 5 to smithi06387500-12 2024-08-17T17:00:49.571 INFO:tasks.rados.rados.0.smithi063.stdout:1470: oid 12 updating version 0 to 347 2024-08-17T17:00:49.571 INFO:tasks.rados.rados.0.smithi063.stdout:1470: oid 12 updating version 347 to 348 2024-08-17T17:00:49.571 INFO:tasks.rados.rados.0.smithi063.stdout:1470: oid 12 updating version 348 to 349 2024-08-17T17:00:49.571 INFO:tasks.rados.rados.0.smithi063.stdout:1470: oid 12 version 349 is already newer than 346 2024-08-17T17:00:49.571 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 349 (ObjNum 574 snap 180 seq_num 574) dirty exists 2024-08-17T17:00:49.571 INFO:tasks.rados.rados.0.smithi063.stdout:1470: left oid 12 (ObjNum 574 snap 180 seq_num 574) 2024-08-17T17:00:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:49 smithi162 ceph-mon[34633]: pgmap v94: 89 pgs: 89 active+clean; 242 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T17:00:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:49 smithi162 ceph-mon[34633]: osdmap e428: 8 total, 8 up, 8 in 2024-08-17T17:00:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:49 smithi063 ceph-mon[114756]: pgmap v94: 89 pgs: 89 active+clean; 242 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T17:00:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:49 smithi063 ceph-mon[114756]: osdmap e428: 8 total, 8 up, 8 in 2024-08-17T17:00:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:49 smithi063 ceph-mon[109843]: pgmap v94: 89 pgs: 89 active+clean; 242 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T17:00:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:49 smithi063 ceph-mon[109843]: osdmap e428: 8 total, 8 up, 8 in 2024-08-17T17:00:50.556 INFO:tasks.rados.rados.0.smithi063.stdout:1470: done (1 left) 2024-08-17T17:00:50.556 INFO:tasks.rados.rados.0.smithi063.stdout:1471: done (0 left) 2024-08-17T17:00:50.557 INFO:tasks.rados.rados.0.smithi063.stdout:1472: read oid 36 snap -1 2024-08-17T17:00:50.557 INFO:tasks.rados.rados.0.smithi063.stdout:1472: expect deleted 2024-08-17T17:00:50.557 INFO:tasks.rados.rados.0.smithi063.stdout:1473: write oid 10 current snap is 181 2024-08-17T17:00:50.557 INFO:tasks.rados.rados.0.smithi063.stdout:1473: seq_num 575 ranges {464025=794650,1947852=404189,2990809=425525} 2024-08-17T17:00:50.563 INFO:tasks.rados.rados.0.smithi063.stdout:1473: writing smithi06387500-10 from 464025 to 1258675 tid 1 2024-08-17T17:00:50.569 INFO:tasks.rados.rados.0.smithi063.stdout:1473: writing smithi06387500-10 from 1947852 to 2352041 tid 2 2024-08-17T17:00:50.575 INFO:tasks.rados.rados.0.smithi063.stdout:1473: writing smithi06387500-10 from 2990809 to 3416334 tid 3 2024-08-17T17:00:50.575 INFO:tasks.rados.rados.0.smithi063.stdout:1474: rollback oid 48 current snap is 181 2024-08-17T17:00:50.575 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 48 to 178 2024-08-17T17:00:50.575 INFO:tasks.rados.rados.0.smithi063.stdout:1475: snap_remove snap 168 2024-08-17T17:00:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:50 smithi162 ceph-mon[34633]: osdmap e429: 8 total, 8 up, 8 in 2024-08-17T17:00:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:50 smithi063 ceph-mon[114756]: osdmap e429: 8 total, 8 up, 8 in 2024-08-17T17:00:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:50 smithi063 ceph-mon[109843]: osdmap e429: 8 total, 8 up, 8 in 2024-08-17T17:00:51.556 INFO:tasks.rados.rados.0.smithi063.stdout:1475: done (3 left) 2024-08-17T17:00:51.556 INFO:tasks.rados.rados.0.smithi063.stdout:1476: snap_remove snap 179 2024-08-17T17:00:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:51 smithi162 ceph-mon[34633]: pgmap v97: 89 pgs: 89 active+clean; 242 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T17:00:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:51 smithi162 ceph-mon[34633]: osdmap e430: 8 total, 8 up, 8 in 2024-08-17T17:00:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:51 smithi063 ceph-mon[114756]: pgmap v97: 89 pgs: 89 active+clean; 242 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T17:00:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:51 smithi063 ceph-mon[114756]: osdmap e430: 8 total, 8 up, 8 in 2024-08-17T17:00:52.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:51 smithi063 ceph-mon[109843]: pgmap v97: 89 pgs: 89 active+clean; 242 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 0 B/s wr, 8 op/s 2024-08-17T17:00:52.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:51 smithi063 ceph-mon[109843]: osdmap e430: 8 total, 8 up, 8 in 2024-08-17T17:00:52.517 INFO:tasks.rados.rados.0.smithi063.stdout:1476: done (3 left) 2024-08-17T17:00:52.517 INFO:tasks.rados.rados.0.smithi063.stdout:1477: read oid 37 snap -1 2024-08-17T17:00:52.517 INFO:tasks.rados.rados.0.smithi063.stdout:1477: expect (ObjNum 566 snap 173 seq_num 566) 2024-08-17T17:00:52.517 INFO:tasks.rados.rados.0.smithi063.stdout:1478: write oid 9 current snap is 181 2024-08-17T17:00:52.517 INFO:tasks.rados.rados.0.smithi063.stdout:1478: seq_num 576 ranges {480832=469899,1606334=664200,3018232=662600} 2024-08-17T17:00:52.522 INFO:tasks.rados.rados.0.smithi063.stdout:1478: writing smithi06387500-9 from 480832 to 950731 tid 1 2024-08-17T17:00:52.530 INFO:tasks.rados.rados.0.smithi063.stdout:1478: writing smithi06387500-9 from 1606334 to 2270534 tid 2 2024-08-17T17:00:52.539 INFO:tasks.rados.rados.0.smithi063.stdout:1478: writing smithi06387500-9 from 3018232 to 3680832 tid 3 2024-08-17T17:00:52.539 INFO:tasks.rados.rados.0.smithi063.stdout:1479: read oid 43 snap -1 2024-08-17T17:00:52.539 INFO:tasks.rados.rados.0.smithi063.stdout:1479: expect (ObjNum 483 snap 153 seq_num 483) 2024-08-17T17:00:52.539 INFO:tasks.rados.rados.0.smithi063.stdout:1480: snap_create 2024-08-17T17:00:52.539 INFO:tasks.rados.rados.0.smithi063.stdout:1473: finishing write tid 1 to smithi06387500-10 2024-08-17T17:00:52.539 INFO:tasks.rados.rados.0.smithi063.stdout:1473: finishing write tid 2 to smithi06387500-10 2024-08-17T17:00:52.539 INFO:tasks.rados.rados.0.smithi063.stdout:1473: finishing write tid 3 to smithi06387500-10 2024-08-17T17:00:52.539 INFO:tasks.rados.rados.0.smithi063.stdout:1473: finishing write tid 4 to smithi06387500-10 2024-08-17T17:00:52.539 INFO:tasks.rados.rados.0.smithi063.stdout:1474: finishing rollback tid 1 to smithi06387500-48 2024-08-17T17:00:52.539 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 1318 (ObjNum 657 snap 0 seq_num 574599008) dirty dne 2024-08-17T17:00:52.540 INFO:tasks.rados.rados.0.smithi063.stdout:1473: finishing write tid 5 to smithi06387500-10 2024-08-17T17:00:52.540 INFO:tasks.rados.rados.0.smithi063.stdout:1473: oid 10 updating version 0 to 548 2024-08-17T17:00:52.540 INFO:tasks.rados.rados.0.smithi063.stdout:1473: oid 10 updating version 548 to 551 2024-08-17T17:00:52.540 INFO:tasks.rados.rados.0.smithi063.stdout:1473: oid 10 version 551 is already newer than 550 2024-08-17T17:00:52.540 INFO:tasks.rados.rados.0.smithi063.stdout:1473: oid 10 version 551 is already newer than 549 2024-08-17T17:00:52.540 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 551 (ObjNum 575 snap 181 seq_num 575) dirty exists 2024-08-17T17:00:52.540 INFO:tasks.rados.rados.0.smithi063.stdout:1473: left oid 10 (ObjNum 575 snap 181 seq_num 575) 2024-08-17T17:00:52.540 INFO:tasks.rados.rados.0.smithi063.stdout:1478: finishing write tid 1 to smithi06387500-9 2024-08-17T17:00:52.540 INFO:tasks.rados.rados.0.smithi063.stdout:1478: finishing write tid 2 to smithi06387500-9 2024-08-17T17:00:52.540 INFO:tasks.rados.rados.0.smithi063.stdout:1477: expect (ObjNum 566 snap 173 seq_num 566) 2024-08-17T17:00:52.692 INFO:tasks.rados.rados.0.smithi063.stdout:1478: finishing write tid 3 to smithi06387500-9 2024-08-17T17:00:52.692 INFO:tasks.rados.rados.0.smithi063.stdout:1478: finishing write tid 4 to smithi06387500-9 2024-08-17T17:00:52.692 INFO:tasks.rados.rados.0.smithi063.stdout:1478: finishing write tid 5 to smithi06387500-9 2024-08-17T17:00:52.692 INFO:tasks.rados.rados.0.smithi063.stdout:1478: oid 9 updating version 0 to 715 2024-08-17T17:00:52.693 INFO:tasks.rados.rados.0.smithi063.stdout:1478: oid 9 version 715 is already newer than 714 2024-08-17T17:00:52.693 INFO:tasks.rados.rados.0.smithi063.stdout:1478: oid 9 version 715 is already newer than 713 2024-08-17T17:00:52.693 INFO:tasks.rados.rados.0.smithi063.stdout:1478: oid 9 version 715 is already newer than 712 2024-08-17T17:00:52.693 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 715 (ObjNum 576 snap 181 seq_num 576) dirty exists 2024-08-17T17:00:52.693 INFO:tasks.rados.rados.0.smithi063.stdout:1478: left oid 9 (ObjNum 576 snap 181 seq_num 576) 2024-08-17T17:00:52.693 INFO:tasks.rados.rados.0.smithi063.stdout:1479: expect (ObjNum 483 snap 153 seq_num 483) 2024-08-17T17:00:52.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:52 smithi162 ceph-mon[34633]: osdmap e431: 8 total, 8 up, 8 in 2024-08-17T17:00:52.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:52 smithi162 ceph-mon[34633]: osdmap e432: 8 total, 8 up, 8 in 2024-08-17T17:00:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:52 smithi063 ceph-mon[114756]: osdmap e431: 8 total, 8 up, 8 in 2024-08-17T17:00:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:52 smithi063 ceph-mon[114756]: osdmap e432: 8 total, 8 up, 8 in 2024-08-17T17:00:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:52 smithi063 ceph-mon[109843]: osdmap e431: 8 total, 8 up, 8 in 2024-08-17T17:00:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:52 smithi063 ceph-mon[109843]: osdmap e432: 8 total, 8 up, 8 in 2024-08-17T17:00:53.524 INFO:tasks.rados.rados.0.smithi063.stdout:1472: done (6 left) 2024-08-17T17:00:53.524 INFO:tasks.rados.rados.0.smithi063.stdout:1473: done (5 left) 2024-08-17T17:00:53.524 INFO:tasks.rados.rados.0.smithi063.stdout:1474: done (4 left) 2024-08-17T17:00:53.524 INFO:tasks.rados.rados.0.smithi063.stdout:1477: done (3 left) 2024-08-17T17:00:53.524 INFO:tasks.rados.rados.0.smithi063.stdout:1478: done (2 left) 2024-08-17T17:00:53.524 INFO:tasks.rados.rados.0.smithi063.stdout:1479: done (1 left) 2024-08-17T17:00:53.525 INFO:tasks.rados.rados.0.smithi063.stdout:1480: done (0 left) 2024-08-17T17:00:53.525 INFO:tasks.rados.rados.0.smithi063.stdout:1481: write oid 7 current snap is 182 2024-08-17T17:00:53.525 INFO:tasks.rados.rados.0.smithi063.stdout:1481: seq_num 577 ranges {497639=545148,1548463=407858,2529302=564422,3528143=169496} 2024-08-17T17:00:53.529 INFO:tasks.rados.rados.0.smithi063.stdout:1481: writing smithi06387500-7 from 497639 to 1042787 tid 1 2024-08-17T17:00:53.534 INFO:tasks.rados.rados.0.smithi063.stdout:1481: writing smithi06387500-7 from 1548463 to 1956321 tid 2 2024-08-17T17:00:53.541 INFO:tasks.rados.rados.0.smithi063.stdout:1481: writing smithi06387500-7 from 2529302 to 3093724 tid 3 2024-08-17T17:00:53.544 INFO:tasks.rados.rados.0.smithi063.stdout:1481: writing smithi06387500-7 from 3528143 to 3697639 tid 4 2024-08-17T17:00:53.544 INFO:tasks.rados.rados.0.smithi063.stdout:1482: write oid 21 current snap is 182 2024-08-17T17:00:53.545 INFO:tasks.rados.rados.0.smithi063.stdout:1481: finishing write tid 1 to smithi06387500-7 2024-08-17T17:00:53.545 INFO:tasks.rados.rados.0.smithi063.stdout:1482: seq_num 578 ranges {514446=736750,1723298=667869,3073078=641368} 2024-08-17T17:00:53.551 INFO:tasks.rados.rados.0.smithi063.stdout:1482: writing smithi06387500-21 from 514446 to 1251196 tid 1 2024-08-17T17:00:53.557 INFO:tasks.rados.rados.0.smithi063.stdout:1482: writing smithi06387500-21 from 1723298 to 2391167 tid 2 2024-08-17T17:00:53.565 INFO:tasks.rados.rados.0.smithi063.stdout:1482: writing smithi06387500-21 from 3073078 to 3714446 tid 3 2024-08-17T17:00:53.565 INFO:tasks.rados.rados.0.smithi063.stdout:1483: write oid 44 current snap is 182 2024-08-17T17:00:53.565 INFO:tasks.rados.rados.0.smithi063.stdout:1483: seq_num 579 ranges {531253=411999,1381780=411527,2300501=586966,3376974=354279} 2024-08-17T17:00:53.569 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:00:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:00:53] "GET /metrics HTTP/1.1" 200 36203 "" "Prometheus/2.51.0" 2024-08-17T17:00:53.570 INFO:tasks.rados.rados.0.smithi063.stdout:1483: writing smithi06387500-44 from 531253 to 943252 tid 1 2024-08-17T17:00:53.575 INFO:tasks.rados.rados.0.smithi063.stdout:1483: writing smithi06387500-44 from 1381780 to 1793307 tid 2 2024-08-17T17:00:53.581 INFO:tasks.rados.rados.0.smithi063.stdout:1483: writing smithi06387500-44 from 2300501 to 2887467 tid 3 2024-08-17T17:00:53.586 INFO:tasks.rados.rados.0.smithi063.stdout:1483: writing smithi06387500-44 from 3376974 to 3731253 tid 4 2024-08-17T17:00:53.586 INFO:tasks.rados.rados.0.smithi063.stdout:1484: delete oid 12 current snap is 182 2024-08-17T17:00:53.586 INFO:tasks.rados.rados.0.smithi063.stdout:1481: finishing write tid 2 to smithi06387500-7 2024-08-17T17:00:53.586 INFO:tasks.rados.rados.0.smithi063.stdout:1481: finishing write tid 3 to smithi06387500-7 2024-08-17T17:00:53.587 INFO:tasks.rados.rados.0.smithi063.stdout:1481: finishing write tid 4 to smithi06387500-7 2024-08-17T17:00:53.587 INFO:tasks.rados.rados.0.smithi063.stdout:1481: finishing write tid 5 to smithi06387500-7 2024-08-17T17:00:53.587 INFO:tasks.rados.rados.0.smithi063.stdout:1481: finishing write tid 6 to smithi06387500-7 2024-08-17T17:00:53.587 INFO:tasks.rados.rados.0.smithi063.stdout:1481: oid 7 updating version 0 to 717 2024-08-17T17:00:53.587 INFO:tasks.rados.rados.0.smithi063.stdout:1481: oid 7 updating version 717 to 719 2024-08-17T17:00:53.587 INFO:tasks.rados.rados.0.smithi063.stdout:1481: oid 7 version 719 is already newer than 718 2024-08-17T17:00:53.587 INFO:tasks.rados.rados.0.smithi063.stdout:1481: oid 7 version 719 is already newer than 716 2024-08-17T17:00:53.587 INFO:tasks.rados.rados.0.smithi063.stdout:1481: oid 7 updating version 719 to 720 2024-08-17T17:00:53.587 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 720 (ObjNum 577 snap 182 seq_num 577) dirty exists 2024-08-17T17:00:53.587 INFO:tasks.rados.rados.0.smithi063.stdout:1481: left oid 7 (ObjNum 577 snap 182 seq_num 577) 2024-08-17T17:00:53.587 INFO:tasks.rados.rados.0.smithi063.stdout:1482: finishing write tid 1 to smithi06387500-21 2024-08-17T17:00:53.587 INFO:tasks.rados.rados.0.smithi063.stdout:1482: finishing write tid 2 to smithi06387500-21 2024-08-17T17:00:53.588 INFO:tasks.rados.rados.0.smithi063.stdout:1482: finishing write tid 3 to smithi06387500-21 2024-08-17T17:00:53.588 INFO:tasks.rados.rados.0.smithi063.stdout:1482: finishing write tid 4 to smithi06387500-21 2024-08-17T17:00:53.588 INFO:tasks.rados.rados.0.smithi063.stdout:1482: finishing write tid 5 to smithi06387500-21 2024-08-17T17:00:53.588 INFO:tasks.rados.rados.0.smithi063.stdout:1482: oid 21 updating version 0 to 1108 2024-08-17T17:00:53.588 INFO:tasks.rados.rados.0.smithi063.stdout:1482: oid 21 updating version 1108 to 1109 2024-08-17T17:00:53.588 INFO:tasks.rados.rados.0.smithi063.stdout:1482: oid 21 version 1109 is already newer than 1107 2024-08-17T17:00:53.588 INFO:tasks.rados.rados.0.smithi063.stdout:1482: oid 21 updating version 1109 to 1110 2024-08-17T17:00:53.588 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 1110 (ObjNum 578 snap 182 seq_num 578) dirty exists 2024-08-17T17:00:53.588 INFO:tasks.rados.rados.0.smithi063.stdout:1482: left oid 21 (ObjNum 578 snap 182 seq_num 578) 2024-08-17T17:00:53.588 INFO:tasks.rados.rados.0.smithi063.stdout:1483: finishing write tid 1 to smithi06387500-44 2024-08-17T17:00:53.588 INFO:tasks.rados.rados.0.smithi063.stdout:1483: finishing write tid 2 to smithi06387500-44 2024-08-17T17:00:53.588 INFO:tasks.rados.rados.0.smithi063.stdout:1483: finishing write tid 3 to smithi06387500-44 2024-08-17T17:00:53.589 INFO:tasks.rados.rados.0.smithi063.stdout:1481: done (3 left) 2024-08-17T17:00:53.589 INFO:tasks.rados.rados.0.smithi063.stdout:1482: done (2 left) 2024-08-17T17:00:53.589 INFO:tasks.rados.rados.0.smithi063.stdout:1484: done (1 left) 2024-08-17T17:00:53.589 INFO:tasks.rados.rados.0.smithi063.stdout:1485: write oid 39 current snap is 182 2024-08-17T17:00:53.589 INFO:tasks.rados.rados.0.smithi063.stdout:1485: seq_num 581 ranges {564867=562497,1898744=415196,2755347=725863} 2024-08-17T17:00:53.594 INFO:tasks.rados.rados.0.smithi063.stdout:1485: writing smithi06387500-39 from 564867 to 1127364 tid 1 2024-08-17T17:00:53.600 INFO:tasks.rados.rados.0.smithi063.stdout:1485: writing smithi06387500-39 from 1898744 to 2313940 tid 2 2024-08-17T17:00:53.607 INFO:tasks.rados.rados.0.smithi063.stdout:1485: writing smithi06387500-39 from 2755347 to 3481210 tid 3 2024-08-17T17:00:53.607 INFO:tasks.rados.rados.0.smithi063.stdout:1486: snap_remove snap 178 2024-08-17T17:00:53.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:53 smithi162 ceph-mon[34633]: pgmap v100: 89 pgs: 89 active+clean; 236 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2024-08-17T17:00:53.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:53 smithi162 ceph-mon[34633]: osdmap e433: 8 total, 8 up, 8 in 2024-08-17T17:00:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:53 smithi063 ceph-mon[114756]: pgmap v100: 89 pgs: 89 active+clean; 236 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2024-08-17T17:00:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:53 smithi063 ceph-mon[114756]: osdmap e433: 8 total, 8 up, 8 in 2024-08-17T17:00:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:53 smithi063 ceph-mon[109843]: pgmap v100: 89 pgs: 89 active+clean; 236 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2024-08-17T17:00:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:53 smithi063 ceph-mon[109843]: osdmap e433: 8 total, 8 up, 8 in 2024-08-17T17:00:54.578 INFO:tasks.rados.rados.0.smithi063.stdout:1486: done (2 left) 2024-08-17T17:00:54.578 INFO:tasks.rados.rados.0.smithi063.stdout:1487: read oid 41 snap -1 2024-08-17T17:00:54.578 INFO:tasks.rados.rados.0.smithi063.stdout:1487: expect (ObjNum 542 snap 170 seq_num 542) 2024-08-17T17:00:54.578 INFO:tasks.rados.rados.0.smithi063.stdout:1488: write oid 6 current snap is 182 2024-08-17T17:00:54.578 INFO:tasks.rados.rados.0.smithi063.stdout:1483: finishing write tid 4 to smithi06387500-44 2024-08-17T17:00:54.578 INFO:tasks.rados.rados.0.smithi063.stdout:1488: seq_num 582 ranges {581674=637746,1840873=558854,3066417=537135} 2024-08-17T17:00:54.585 INFO:tasks.rados.rados.0.smithi063.stdout:1488: writing smithi06387500-6 from 581674 to 1219420 tid 1 2024-08-17T17:00:54.591 INFO:tasks.rados.rados.0.smithi063.stdout:1488: writing smithi06387500-6 from 1840873 to 2399727 tid 2 2024-08-17T17:00:54.599 INFO:tasks.rados.rados.0.smithi063.stdout:1488: writing smithi06387500-6 from 3066417 to 3603552 tid 3 2024-08-17T17:00:54.599 INFO:tasks.rados.rados.0.smithi063.stdout:1489: snap_create 2024-08-17T17:00:54.599 INFO:tasks.rados.rados.0.smithi063.stdout:1483: finishing write tid 5 to smithi06387500-44 2024-08-17T17:00:54.599 INFO:tasks.rados.rados.0.smithi063.stdout:1483: finishing write tid 6 to smithi06387500-44 2024-08-17T17:00:54.599 INFO:tasks.rados.rados.0.smithi063.stdout:1483: oid 44 updating version 0 to 295 2024-08-17T17:00:54.599 INFO:tasks.rados.rados.0.smithi063.stdout:1483: oid 44 version 295 is already newer than 293 2024-08-17T17:00:54.599 INFO:tasks.rados.rados.0.smithi063.stdout:1483: oid 44 version 295 is already newer than 294 2024-08-17T17:00:54.599 INFO:tasks.rados.rados.0.smithi063.stdout:1483: oid 44 updating version 295 to 297 2024-08-17T17:00:54.599 INFO:tasks.rados.rados.0.smithi063.stdout:1483: oid 44 version 297 is already newer than 296 2024-08-17T17:00:54.599 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 297 (ObjNum 579 snap 182 seq_num 579) dirty exists 2024-08-17T17:00:54.599 INFO:tasks.rados.rados.0.smithi063.stdout:1483: left oid 44 (ObjNum 579 snap 182 seq_num 579) 2024-08-17T17:00:54.600 INFO:tasks.rados.rados.0.smithi063.stdout:1485: finishing write tid 1 to smithi06387500-39 2024-08-17T17:00:54.600 INFO:tasks.rados.rados.0.smithi063.stdout:1485: finishing write tid 2 to smithi06387500-39 2024-08-17T17:00:54.600 INFO:tasks.rados.rados.0.smithi063.stdout:1485: finishing write tid 3 to smithi06387500-39 2024-08-17T17:00:54.600 INFO:tasks.rados.rados.0.smithi063.stdout:1485: finishing write tid 4 to smithi06387500-39 2024-08-17T17:00:54.600 INFO:tasks.rados.rados.0.smithi063.stdout:1485: finishing write tid 5 to smithi06387500-39 2024-08-17T17:00:54.600 INFO:tasks.rados.rados.0.smithi063.stdout:1485: oid 39 updating version 0 to 337 2024-08-17T17:00:54.600 INFO:tasks.rados.rados.0.smithi063.stdout:1485: oid 39 version 337 is already newer than 336 2024-08-17T17:00:54.600 INFO:tasks.rados.rados.0.smithi063.stdout:1485: oid 39 version 337 is already newer than 335 2024-08-17T17:00:54.600 INFO:tasks.rados.rados.0.smithi063.stdout:1485: oid 39 updating version 337 to 338 2024-08-17T17:00:54.600 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 338 (ObjNum 581 snap 182 seq_num 581) dirty exists 2024-08-17T17:00:54.600 INFO:tasks.rados.rados.0.smithi063.stdout:1485: left oid 39 (ObjNum 581 snap 182 seq_num 581) 2024-08-17T17:00:54.601 INFO:tasks.rados.rados.0.smithi063.stdout:1488: finishing write tid 1 to smithi06387500-6 2024-08-17T17:00:54.601 INFO:tasks.rados.rados.0.smithi063.stdout:1488: finishing write tid 2 to smithi06387500-6 2024-08-17T17:00:54.601 INFO:tasks.rados.rados.0.smithi063.stdout:1487: expect (ObjNum 542 snap 170 seq_num 542) 2024-08-17T17:00:54.743 INFO:tasks.rados.rados.0.smithi063.stdout:1488: finishing write tid 3 to smithi06387500-6 2024-08-17T17:00:54.743 INFO:tasks.rados.rados.0.smithi063.stdout:1488: finishing write tid 4 to smithi06387500-6 2024-08-17T17:00:54.743 INFO:tasks.rados.rados.0.smithi063.stdout:1488: finishing write tid 5 to smithi06387500-6 2024-08-17T17:00:54.743 INFO:tasks.rados.rados.0.smithi063.stdout:1488: oid 6 updating version 0 to 456 2024-08-17T17:00:54.743 INFO:tasks.rados.rados.0.smithi063.stdout:1488: oid 6 version 456 is already newer than 455 2024-08-17T17:00:54.744 INFO:tasks.rados.rados.0.smithi063.stdout:1488: oid 6 version 456 is already newer than 454 2024-08-17T17:00:54.744 INFO:tasks.rados.rados.0.smithi063.stdout:1488: oid 6 version 456 is already newer than 453 2024-08-17T17:00:54.744 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 456 (ObjNum 582 snap 182 seq_num 582) dirty exists 2024-08-17T17:00:54.744 INFO:tasks.rados.rados.0.smithi063.stdout:1488: left oid 6 (ObjNum 582 snap 182 seq_num 582) 2024-08-17T17:00:55.589 INFO:tasks.rados.rados.0.smithi063.stdout:1483: done (4 left) 2024-08-17T17:00:55.590 INFO:tasks.rados.rados.0.smithi063.stdout:1485: done (3 left) 2024-08-17T17:00:55.590 INFO:tasks.rados.rados.0.smithi063.stdout:1487: done (2 left) 2024-08-17T17:00:55.590 INFO:tasks.rados.rados.0.smithi063.stdout:1488: done (1 left) 2024-08-17T17:00:55.590 INFO:tasks.rados.rados.0.smithi063.stdout:1489: done (0 left) 2024-08-17T17:00:55.590 INFO:tasks.rados.rados.0.smithi063.stdout:1490: snap_create 2024-08-17T17:00:55.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:55 smithi162 ceph-mon[34633]: pgmap v103: 89 pgs: 89 active+clean; 236 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2024-08-17T17:00:55.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:55 smithi162 ceph-mon[34633]: osdmap e434: 8 total, 8 up, 8 in 2024-08-17T17:00:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:55 smithi063 ceph-mon[114756]: pgmap v103: 89 pgs: 89 active+clean; 236 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2024-08-17T17:00:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:55 smithi063 ceph-mon[114756]: osdmap e434: 8 total, 8 up, 8 in 2024-08-17T17:00:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:55 smithi063 ceph-mon[109843]: pgmap v103: 89 pgs: 89 active+clean; 236 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2024-08-17T17:00:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:55 smithi063 ceph-mon[109843]: osdmap e434: 8 total, 8 up, 8 in 2024-08-17T17:00:56.600 INFO:tasks.rados.rados.0.smithi063.stdout:1490: done (0 left) 2024-08-17T17:00:56.600 INFO:tasks.rados.rados.0.smithi063.stdout:1491: snap_create 2024-08-17T17:00:56.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:56 smithi162 ceph-mon[34633]: osdmap e435: 8 total, 8 up, 8 in 2024-08-17T17:00:57.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:56 smithi063 ceph-mon[114756]: osdmap e435: 8 total, 8 up, 8 in 2024-08-17T17:00:57.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:56 smithi063 ceph-mon[109843]: osdmap e435: 8 total, 8 up, 8 in 2024-08-17T17:00:57.606 INFO:tasks.rados.rados.0.smithi063.stdout:1491: done (0 left) 2024-08-17T17:00:57.606 INFO:tasks.rados.rados.0.smithi063.stdout:1492: snap_remove snap 175 2024-08-17T17:00:57.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:57 smithi162 ceph-mon[34633]: pgmap v106: 89 pgs: 89 active+clean; 236 MiB data, 773 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:00:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:57 smithi162 ceph-mon[34633]: osdmap e436: 8 total, 8 up, 8 in 2024-08-17T17:00:58.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:57 smithi063 ceph-mon[109843]: pgmap v106: 89 pgs: 89 active+clean; 236 MiB data, 773 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:00:58.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:57 smithi063 ceph-mon[109843]: osdmap e436: 8 total, 8 up, 8 in 2024-08-17T17:00:58.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:57 smithi063 ceph-mon[114756]: pgmap v106: 89 pgs: 89 active+clean; 236 MiB data, 773 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:00:58.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:57 smithi063 ceph-mon[114756]: osdmap e436: 8 total, 8 up, 8 in 2024-08-17T17:00:58.609 INFO:tasks.rados.rados.0.smithi063.stdout:1492: done (0 left) 2024-08-17T17:00:58.609 INFO:tasks.rados.rados.0.smithi063.stdout:1493: snap_create 2024-08-17T17:00:59.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:58 smithi063 ceph-mon[109843]: osdmap e437: 8 total, 8 up, 8 in 2024-08-17T17:00:59.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:58 smithi063 ceph-mon[114756]: osdmap e437: 8 total, 8 up, 8 in 2024-08-17T17:00:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:58 smithi162 ceph-mon[34633]: osdmap e437: 8 total, 8 up, 8 in 2024-08-17T17:00:59.615 INFO:tasks.rados.rados.0.smithi063.stdout:1493: done (0 left) 2024-08-17T17:00:59.615 INFO:tasks.rados.rados.0.smithi063.stdout:1494: snap_remove snap 181 2024-08-17T17:01:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:59 smithi063 ceph-mon[114756]: pgmap v109: 89 pgs: 89 active+clean; 252 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-08-17T17:01:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:00:59 smithi063 ceph-mon[114756]: osdmap e438: 8 total, 8 up, 8 in 2024-08-17T17:01:00.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:59 smithi063 ceph-mon[109843]: pgmap v109: 89 pgs: 89 active+clean; 252 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-08-17T17:01:00.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:00:59 smithi063 ceph-mon[109843]: osdmap e438: 8 total, 8 up, 8 in 2024-08-17T17:01:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:59 smithi162 ceph-mon[34633]: pgmap v109: 89 pgs: 89 active+clean; 252 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-08-17T17:01:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:00:59 smithi162 ceph-mon[34633]: osdmap e438: 8 total, 8 up, 8 in 2024-08-17T17:01:00.633 INFO:tasks.rados.rados.0.smithi063.stdout:1494: done (0 left) 2024-08-17T17:01:00.633 INFO:tasks.rados.rados.0.smithi063.stdout:1495: rollback oid 50 current snap is 186 2024-08-17T17:01:00.633 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 50 to 176 2024-08-17T17:01:00.633 INFO:tasks.rados.rados.0.smithi063.stdout:1496: write oid 24 current snap is 186 2024-08-17T17:01:00.634 INFO:tasks.rados.rados.0.smithi063.stdout:1496: seq_num 583 ranges {598481=712995,1899355=418865,3093840=704641} 2024-08-17T17:01:00.641 INFO:tasks.rados.rados.0.smithi063.stdout:1496: writing smithi06387500-24 from 598481 to 1311476 tid 1 2024-08-17T17:01:00.647 INFO:tasks.rados.rados.0.smithi063.stdout:1496: writing smithi06387500-24 from 1899355 to 2318220 tid 2 2024-08-17T17:01:00.655 INFO:tasks.rados.rados.0.smithi063.stdout:1496: writing smithi06387500-24 from 3093840 to 3798481 tid 3 2024-08-17T17:01:00.655 INFO:tasks.rados.rados.0.smithi063.stdout:1497: write oid 49 current snap is 186 2024-08-17T17:01:00.655 INFO:tasks.rados.rados.0.smithi063.stdout:1497: seq_num 584 ranges {615288=788244,1957837=562523,3121263=559679} 2024-08-17T17:01:00.663 INFO:tasks.rados.rados.0.smithi063.stdout:1497: writing smithi06387500-49 from 615288 to 1403532 tid 1 2024-08-17T17:01:00.669 INFO:tasks.rados.rados.0.smithi063.stdout:1497: writing smithi06387500-49 from 1957837 to 2520360 tid 2 2024-08-17T17:01:00.676 INFO:tasks.rados.rados.0.smithi063.stdout:1497: writing smithi06387500-49 from 3121263 to 3680942 tid 3 2024-08-17T17:01:00.676 INFO:tasks.rados.rados.0.smithi063.stdout:1498: snap_remove snap 141 2024-08-17T17:01:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:00 smithi063 ceph-mon[114756]: osdmap e439: 8 total, 8 up, 8 in 2024-08-17T17:01:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:00 smithi063 ceph-mon[114756]: pgmap v112: 89 pgs: 89 active+clean; 252 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-08-17T17:01:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:00 smithi063 ceph-mon[109843]: osdmap e439: 8 total, 8 up, 8 in 2024-08-17T17:01:01.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:00 smithi063 ceph-mon[109843]: pgmap v112: 89 pgs: 89 active+clean; 252 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-08-17T17:01:01.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:00 smithi162 ceph-mon[34633]: osdmap e439: 8 total, 8 up, 8 in 2024-08-17T17:01:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:00 smithi162 ceph-mon[34633]: pgmap v112: 89 pgs: 89 active+clean; 252 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-08-17T17:01:01.645 INFO:tasks.rados.rados.0.smithi063.stdout:1498: done (3 left) 2024-08-17T17:01:01.645 INFO:tasks.rados.rados.0.smithi063.stdout:1499: delete oid 4 current snap is 186 2024-08-17T17:01:01.645 INFO:tasks.rados.rados.0.smithi063.stdout:1495: finishing rollback tid 0 to smithi06387500-50 2024-08-17T17:01:01.646 INFO:tasks.rados.rados.0.smithi063.stdout:1495: finishing rollback tid 1 to smithi06387500-50 2024-08-17T17:01:01.646 INFO:tasks.rados.rados.0.smithi063.stdout:1495: finishing rollback tid 2 to smithi06387500-50 2024-08-17T17:01:01.646 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 1120 (ObjNum 564 snap 173 seq_num 564) dirty exists 2024-08-17T17:01:01.646 INFO:tasks.rados.rados.0.smithi063.stdout:1496: finishing write tid 1 to smithi06387500-24 2024-08-17T17:01:01.646 INFO:tasks.rados.rados.0.smithi063.stdout:1496: finishing write tid 2 to smithi06387500-24 2024-08-17T17:01:01.646 INFO:tasks.rados.rados.0.smithi063.stdout:1496: finishing write tid 3 to smithi06387500-24 2024-08-17T17:01:01.646 INFO:tasks.rados.rados.0.smithi063.stdout:1496: finishing write tid 4 to smithi06387500-24 2024-08-17T17:01:01.646 INFO:tasks.rados.rados.0.smithi063.stdout:1496: finishing write tid 5 to smithi06387500-24 2024-08-17T17:01:01.646 INFO:tasks.rados.rados.0.smithi063.stdout:1496: oid 24 updating version 0 to 585 2024-08-17T17:01:01.646 INFO:tasks.rados.rados.0.smithi063.stdout:1496: oid 24 version 585 is already newer than 584 2024-08-17T17:01:01.646 INFO:tasks.rados.rados.0.smithi063.stdout:1496: oid 24 updating version 585 to 587 2024-08-17T17:01:01.647 INFO:tasks.rados.rados.0.smithi063.stdout:1496: oid 24 version 587 is already newer than 586 2024-08-17T17:01:01.647 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 587 (ObjNum 583 snap 186 seq_num 583) dirty exists 2024-08-17T17:01:01.647 INFO:tasks.rados.rados.0.smithi063.stdout:1496: left oid 24 (ObjNum 583 snap 186 seq_num 583) 2024-08-17T17:01:01.647 INFO:tasks.rados.rados.0.smithi063.stdout:1497: finishing write tid 1 to smithi06387500-49 2024-08-17T17:01:01.647 INFO:tasks.rados.rados.0.smithi063.stdout:1497: finishing write tid 2 to smithi06387500-49 2024-08-17T17:01:01.647 INFO:tasks.rados.rados.0.smithi063.stdout:1497: finishing write tid 3 to smithi06387500-49 2024-08-17T17:01:01.647 INFO:tasks.rados.rados.0.smithi063.stdout:1497: finishing write tid 4 to smithi06387500-49 2024-08-17T17:01:01.647 INFO:tasks.rados.rados.0.smithi063.stdout:1497: finishing write tid 5 to smithi06387500-49 2024-08-17T17:01:01.647 INFO:tasks.rados.rados.0.smithi063.stdout:1497: oid 49 updating version 0 to 1518 2024-08-17T17:01:01.647 INFO:tasks.rados.rados.0.smithi063.stdout:1497: oid 49 updating version 1518 to 1520 2024-08-17T17:01:01.647 INFO:tasks.rados.rados.0.smithi063.stdout:1497: oid 49 version 1520 is already newer than 1519 2024-08-17T17:01:01.647 INFO:tasks.rados.rados.0.smithi063.stdout:1497: oid 49 version 1520 is already newer than 1517 2024-08-17T17:01:01.647 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 1520 (ObjNum 584 snap 186 seq_num 584) dirty exists 2024-08-17T17:01:01.648 INFO:tasks.rados.rados.0.smithi063.stdout:1497: left oid 49 (ObjNum 584 snap 186 seq_num 584) 2024-08-17T17:01:01.655 INFO:tasks.rados.rados.0.smithi063.stdout:1495: done (3 left) 2024-08-17T17:01:01.655 INFO:tasks.rados.rados.0.smithi063.stdout:1496: done (2 left) 2024-08-17T17:01:01.655 INFO:tasks.rados.rados.0.smithi063.stdout:1497: done (1 left) 2024-08-17T17:01:01.655 INFO:tasks.rados.rados.0.smithi063.stdout:1499: done (0 left) 2024-08-17T17:01:01.655 INFO:tasks.rados.rados.0.smithi063.stdout:1500: read oid 1 snap -1 2024-08-17T17:01:01.655 INFO:tasks.rados.rados.0.smithi063.stdout:1500: expect (ObjNum 511 snap 160 seq_num 511) 2024-08-17T17:01:01.655 INFO:tasks.rados.rados.0.smithi063.stdout:1501: read oid 47 snap -1 2024-08-17T17:01:01.655 INFO:tasks.rados.rados.0.smithi063.stdout:1501: expect (ObjNum 563 snap 173 seq_num 563) 2024-08-17T17:01:01.655 INFO:tasks.rados.rados.0.smithi063.stdout:1502: read oid 16 snap -1 2024-08-17T17:01:01.656 INFO:tasks.rados.rados.0.smithi063.stdout:1502: expect (ObjNum 567 snap 173 seq_num 567) 2024-08-17T17:01:01.656 INFO:tasks.rados.rados.0.smithi063.stdout:1503: snap_remove snap 180 2024-08-17T17:01:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:01 smithi063 ceph-mon[114756]: osdmap e440: 8 total, 8 up, 8 in 2024-08-17T17:01:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:01 smithi063 ceph-mon[109843]: osdmap e440: 8 total, 8 up, 8 in 2024-08-17T17:01:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:01 smithi162 ceph-mon[34633]: osdmap e440: 8 total, 8 up, 8 in 2024-08-17T17:01:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:01 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:02.519 INFO:tasks.rados.rados.0.smithi063.stdout:1503: done (3 left) 2024-08-17T17:01:02.519 INFO:tasks.rados.rados.0.smithi063.stdout:1504: write oid 50 current snap is 186 2024-08-17T17:01:02.519 INFO:tasks.rados.rados.0.smithi063.stdout:1502: expect (ObjNum 567 snap 173 seq_num 567) 2024-08-17T17:01:02.686 INFO:tasks.rados.rados.0.smithi063.stdout:1501: expect (ObjNum 563 snap 173 seq_num 563) 2024-08-17T17:01:02.925 INFO:tasks.rados.rados.0.smithi063.stdout:1500: expect (ObjNum 511 snap 160 seq_num 511) 2024-08-17T17:01:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:02 smithi063 ceph-mon[114756]: osdmap e441: 8 total, 8 up, 8 in 2024-08-17T17:01:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:02 smithi063 ceph-mon[114756]: pgmap v115: 89 pgs: 89 active+clean; 203 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 916 KiB/s wr, 9 op/s 2024-08-17T17:01:03.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:02 smithi063 ceph-mon[114756]: osdmap e442: 8 total, 8 up, 8 in 2024-08-17T17:01:03.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:02 smithi063 ceph-mon[109843]: osdmap e441: 8 total, 8 up, 8 in 2024-08-17T17:01:03.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:02 smithi063 ceph-mon[109843]: pgmap v115: 89 pgs: 89 active+clean; 203 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 916 KiB/s wr, 9 op/s 2024-08-17T17:01:03.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:02 smithi063 ceph-mon[109843]: osdmap e442: 8 total, 8 up, 8 in 2024-08-17T17:01:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:02 smithi162 ceph-mon[34633]: osdmap e441: 8 total, 8 up, 8 in 2024-08-17T17:01:03.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:02 smithi162 ceph-mon[34633]: pgmap v115: 89 pgs: 89 active+clean; 203 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 916 KiB/s wr, 9 op/s 2024-08-17T17:01:03.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:02 smithi162 ceph-mon[34633]: osdmap e442: 8 total, 8 up, 8 in 2024-08-17T17:01:03.103 INFO:tasks.rados.rados.0.smithi063.stdout:1504: seq_num 586 ranges {648902=655095,2074801=566192,3176109=672793} 2024-08-17T17:01:03.110 INFO:tasks.rados.rados.0.smithi063.stdout:1504: writing smithi06387500-50 from 648902 to 1303997 tid 1 2024-08-17T17:01:03.118 INFO:tasks.rados.rados.0.smithi063.stdout:1504: writing smithi06387500-50 from 2074801 to 2640993 tid 2 2024-08-17T17:01:03.125 INFO:tasks.rados.rados.0.smithi063.stdout:1504: writing smithi06387500-50 from 3176109 to 3848902 tid 3 2024-08-17T17:01:03.125 INFO:tasks.rados.rados.0.smithi063.stdout:1500: done (3 left) 2024-08-17T17:01:03.125 INFO:tasks.rados.rados.0.smithi063.stdout:1501: done (2 left) 2024-08-17T17:01:03.125 INFO:tasks.rados.rados.0.smithi063.stdout:1502: done (1 left) 2024-08-17T17:01:03.125 INFO:tasks.rados.rados.0.smithi063.stdout:1505: delete oid 40 current snap is 186 2024-08-17T17:01:03.125 INFO:tasks.rados.rados.0.smithi063.stdout:1504: finishing write tid 1 to smithi06387500-50 2024-08-17T17:01:03.125 INFO:tasks.rados.rados.0.smithi063.stdout:1504: finishing write tid 2 to smithi06387500-50 2024-08-17T17:01:03.127 INFO:tasks.rados.rados.0.smithi063.stdout:1505: done (1 left) 2024-08-17T17:01:03.127 INFO:tasks.rados.rados.0.smithi063.stdout:1506: read oid 46 snap -1 2024-08-17T17:01:03.127 INFO:tasks.rados.rados.0.smithi063.stdout:1506: expect (ObjNum 510 snap 160 seq_num 510) 2024-08-17T17:01:03.127 INFO:tasks.rados.rados.0.smithi063.stdout:1507: snap_create 2024-08-17T17:01:03.128 INFO:tasks.rados.rados.0.smithi063.stdout:1504: finishing write tid 3 to smithi06387500-50 2024-08-17T17:01:03.128 INFO:tasks.rados.rados.0.smithi063.stdout:1504: finishing write tid 4 to smithi06387500-50 2024-08-17T17:01:03.128 INFO:tasks.rados.rados.0.smithi063.stdout:1504: finishing write tid 5 to smithi06387500-50 2024-08-17T17:01:03.128 INFO:tasks.rados.rados.0.smithi063.stdout:1504: oid 50 updating version 0 to 1135 2024-08-17T17:01:03.128 INFO:tasks.rados.rados.0.smithi063.stdout:1504: oid 50 updating version 1135 to 1136 2024-08-17T17:01:03.128 INFO:tasks.rados.rados.0.smithi063.stdout:1504: oid 50 updating version 1136 to 1137 2024-08-17T17:01:03.128 INFO:tasks.rados.rados.0.smithi063.stdout:1504: oid 50 updating version 1137 to 1138 2024-08-17T17:01:03.128 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 1138 (ObjNum 586 snap 186 seq_num 586) dirty exists 2024-08-17T17:01:03.128 INFO:tasks.rados.rados.0.smithi063.stdout:1504: left oid 50 (ObjNum 586 snap 186 seq_num 586) 2024-08-17T17:01:03.132 INFO:tasks.rados.rados.0.smithi063.stdout:1506: expect (ObjNum 510 snap 160 seq_num 510) 2024-08-17T17:01:03.647 INFO:tasks.rados.rados.0.smithi063.stdout:1504: done (2 left) 2024-08-17T17:01:03.648 INFO:tasks.rados.rados.0.smithi063.stdout:1506: done (1 left) 2024-08-17T17:01:03.648 INFO:tasks.rados.rados.0.smithi063.stdout:1507: done (0 left) 2024-08-17T17:01:03.648 INFO:tasks.rados.rados.0.smithi063.stdout:1508: delete oid 15 current snap is 187 2024-08-17T17:01:03.650 INFO:tasks.rados.rados.0.smithi063.stdout:1508: done (0 left) 2024-08-17T17:01:03.650 INFO:tasks.rados.rados.0.smithi063.stdout:1509: read oid 24 snap -1 2024-08-17T17:01:03.650 INFO:tasks.rados.rados.0.smithi063.stdout:1509: expect (ObjNum 583 snap 186 seq_num 583) 2024-08-17T17:01:03.650 INFO:tasks.rados.rados.0.smithi063.stdout:1510: read oid 34 snap 171 2024-08-17T17:01:03.650 INFO:tasks.rados.rados.0.smithi063.stdout:1510: expect (ObjNum 522 snap 165 seq_num 522) 2024-08-17T17:01:03.650 INFO:tasks.rados.rados.0.smithi063.stdout:1511: snap_remove snap 183 2024-08-17T17:01:03.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:01:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:01:03] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.51.0" 2024-08-17T17:01:04.654 INFO:tasks.rados.rados.0.smithi063.stdout:1511: done (2 left) 2024-08-17T17:01:04.654 INFO:tasks.rados.rados.0.smithi063.stdout:1512: write oid 29 current snap is 187 2024-08-17T17:01:04.655 INFO:tasks.rados.rados.0.smithi063.stdout:1512: seq_num 589 ranges {699323=480842,1850247=429872,2858378=532392} 2024-08-17T17:01:04.660 INFO:tasks.rados.rados.0.smithi063.stdout:1512: writing smithi06387500-29 from 699323 to 1180165 tid 1 2024-08-17T17:01:04.667 INFO:tasks.rados.rados.0.smithi063.stdout:1512: writing smithi06387500-29 from 1850247 to 2280119 tid 2 2024-08-17T17:01:04.674 INFO:tasks.rados.rados.0.smithi063.stdout:1512: writing smithi06387500-29 from 2858378 to 3390770 tid 3 2024-08-17T17:01:04.674 INFO:tasks.rados.rados.0.smithi063.stdout:1513: read oid 31 snap -1 2024-08-17T17:01:04.674 INFO:tasks.rados.rados.0.smithi063.stdout:1513: expect (ObjNum 531 snap 166 seq_num 531) 2024-08-17T17:01:04.674 INFO:tasks.rados.rados.0.smithi063.stdout:1514: snap_remove snap 186 2024-08-17T17:01:05.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:04 smithi063 ceph-mon[109843]: osdmap e443: 8 total, 8 up, 8 in 2024-08-17T17:01:05.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:04 smithi063 ceph-mon[109843]: pgmap v118: 89 pgs: 89 active+clean; 203 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 915 KiB/s wr, 9 op/s 2024-08-17T17:01:05.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:04 smithi063 ceph-mon[114756]: osdmap e443: 8 total, 8 up, 8 in 2024-08-17T17:01:05.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:04 smithi063 ceph-mon[114756]: pgmap v118: 89 pgs: 89 active+clean; 203 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 915 KiB/s wr, 9 op/s 2024-08-17T17:01:05.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:04 smithi162 ceph-mon[34633]: osdmap e443: 8 total, 8 up, 8 in 2024-08-17T17:01:05.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:04 smithi162 ceph-mon[34633]: pgmap v118: 89 pgs: 89 active+clean; 203 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 915 KiB/s wr, 9 op/s 2024-08-17T17:01:05.655 INFO:tasks.rados.rados.0.smithi063.stdout:1514: done (4 left) 2024-08-17T17:01:05.655 INFO:tasks.rados.rados.0.smithi063.stdout:1515: write oid 4 current snap is 187 2024-08-17T17:01:05.656 INFO:tasks.rados.rados.0.smithi063.stdout:1515: seq_num 590 ranges {716130=556091,1792376=573530,2769448=460017,3821956=94174} 2024-08-17T17:01:05.663 INFO:tasks.rados.rados.0.smithi063.stdout:1515: writing smithi06387500-4 from 716130 to 1272221 tid 1 2024-08-17T17:01:05.669 INFO:tasks.rados.rados.0.smithi063.stdout:1515: writing smithi06387500-4 from 1792376 to 2365906 tid 2 2024-08-17T17:01:05.674 INFO:tasks.rados.rados.0.smithi063.stdout:1515: writing smithi06387500-4 from 2769448 to 3229465 tid 3 2024-08-17T17:01:05.678 INFO:tasks.rados.rados.0.smithi063.stdout:1515: writing smithi06387500-4 from 3821956 to 3916130 tid 4 2024-08-17T17:01:05.678 INFO:tasks.rados.rados.0.smithi063.stdout:1516: read oid 7 snap -1 2024-08-17T17:01:05.678 INFO:tasks.rados.rados.0.smithi063.stdout:1516: expect (ObjNum 577 snap 182 seq_num 577) 2024-08-17T17:01:05.678 INFO:tasks.rados.rados.0.smithi063.stdout:1517: snap_remove snap 182 2024-08-17T17:01:06.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:05 smithi063 ceph-mon[109843]: osdmap e444: 8 total, 8 up, 8 in 2024-08-17T17:01:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:05 smithi063 ceph-mon[114756]: osdmap e444: 8 total, 8 up, 8 in 2024-08-17T17:01:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:05 smithi162 ceph-mon[34633]: osdmap e444: 8 total, 8 up, 8 in 2024-08-17T17:01:06.676 INFO:tasks.rados.rados.0.smithi063.stdout:1517: done (6 left) 2024-08-17T17:01:06.676 INFO:tasks.rados.rados.0.smithi063.stdout:1518: rollback oid 42 current snap is 187 2024-08-17T17:01:06.676 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 42 to 177 2024-08-17T17:01:06.676 INFO:tasks.rados.rados.0.smithi063.stdout:1509: expect (ObjNum 583 snap 186 seq_num 583) 2024-08-17T17:01:06.801 INFO:tasks.rados.rados.0.smithi063.stdout:1510: expect (ObjNum 522 snap 165 seq_num 522) 2024-08-17T17:01:06.896 INFO:tasks.rados.rados.0.smithi063.stdout:1512: finishing write tid 1 to smithi06387500-29 2024-08-17T17:01:06.896 INFO:tasks.rados.rados.0.smithi063.stdout:1512: finishing write tid 2 to smithi06387500-29 2024-08-17T17:01:06.897 INFO:tasks.rados.rados.0.smithi063.stdout:1509: done (6 left) 2024-08-17T17:01:06.897 INFO:tasks.rados.rados.0.smithi063.stdout:1510: done (5 left) 2024-08-17T17:01:06.897 INFO:tasks.rados.rados.0.smithi063.stdout:1519: write oid 33 current snap is 187 2024-08-17T17:01:06.897 INFO:tasks.rados.rados.0.smithi063.stdout:1512: finishing write tid 3 to smithi06387500-29 2024-08-17T17:01:06.897 INFO:tasks.rados.rados.0.smithi063.stdout:1519: seq_num 591 ranges {732937=631340,1850858=433541,2796871=671289,3888195=44742} 2024-08-17T17:01:06.904 INFO:tasks.rados.rados.0.smithi063.stdout:1519: writing smithi06387500-33 from 732937 to 1364277 tid 1 2024-08-17T17:01:06.909 INFO:tasks.rados.rados.0.smithi063.stdout:1519: writing smithi06387500-33 from 1850858 to 2284399 tid 2 2024-08-17T17:01:06.915 INFO:tasks.rados.rados.0.smithi063.stdout:1519: writing smithi06387500-33 from 2796871 to 3468160 tid 3 2024-08-17T17:01:06.918 INFO:tasks.rados.rados.0.smithi063.stdout:1519: writing smithi06387500-33 from 3888195 to 3932937 tid 4 2024-08-17T17:01:06.918 INFO:tasks.rados.rados.0.smithi063.stdout:1520: write oid 35 current snap is 187 2024-08-17T17:01:06.919 INFO:tasks.rados.rados.0.smithi063.stdout:1512: finishing write tid 4 to smithi06387500-29 2024-08-17T17:01:06.919 INFO:tasks.rados.rados.0.smithi063.stdout:1512: finishing write tid 5 to smithi06387500-29 2024-08-17T17:01:06.919 INFO:tasks.rados.rados.0.smithi063.stdout:1512: oid 29 updating version 0 to 372 2024-08-17T17:01:06.919 INFO:tasks.rados.rados.0.smithi063.stdout:1512: oid 29 version 372 is already newer than 370 2024-08-17T17:01:06.919 INFO:tasks.rados.rados.0.smithi063.stdout:1512: oid 29 version 372 is already newer than 371 2024-08-17T17:01:06.919 INFO:tasks.rados.rados.0.smithi063.stdout:1512: oid 29 version 372 is already newer than 369 2024-08-17T17:01:06.919 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 372 (ObjNum 589 snap 187 seq_num 589) dirty exists 2024-08-17T17:01:06.919 INFO:tasks.rados.rados.0.smithi063.stdout:1512: left oid 29 (ObjNum 589 snap 187 seq_num 589) 2024-08-17T17:01:06.919 INFO:tasks.rados.rados.0.smithi063.stdout:1513: expect (ObjNum 531 snap 166 seq_num 531) 2024-08-17T17:01:07.015 INFO:tasks.rados.rados.0.smithi063.stdout:1515: finishing write tid 1 to smithi06387500-4 2024-08-17T17:01:07.016 INFO:tasks.rados.rados.0.smithi063.stdout:1515: finishing write tid 2 to smithi06387500-4 2024-08-17T17:01:07.016 INFO:tasks.rados.rados.0.smithi063.stdout:1515: finishing write tid 3 to smithi06387500-4 2024-08-17T17:01:07.016 INFO:tasks.rados.rados.0.smithi063.stdout:1515: finishing write tid 4 to smithi06387500-4 2024-08-17T17:01:07.016 INFO:tasks.rados.rados.0.smithi063.stdout:1515: finishing write tid 5 to smithi06387500-4 2024-08-17T17:01:07.016 INFO:tasks.rados.rados.0.smithi063.stdout:1515: finishing write tid 6 to smithi06387500-4 2024-08-17T17:01:07.016 INFO:tasks.rados.rados.0.smithi063.stdout:1515: oid 4 updating version 0 to 1538 2024-08-17T17:01:07.016 INFO:tasks.rados.rados.0.smithi063.stdout:1515: oid 4 version 1538 is already newer than 1535 2024-08-17T17:01:07.016 INFO:tasks.rados.rados.0.smithi063.stdout:1515: oid 4 version 1538 is already newer than 1536 2024-08-17T17:01:07.016 INFO:tasks.rados.rados.0.smithi063.stdout:1515: oid 4 version 1538 is already newer than 1537 2024-08-17T17:01:07.016 INFO:tasks.rados.rados.0.smithi063.stdout:1515: oid 4 version 1538 is already newer than 1534 2024-08-17T17:01:07.016 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 1538 (ObjNum 590 snap 187 seq_num 590) dirty exists 2024-08-17T17:01:07.016 INFO:tasks.rados.rados.0.smithi063.stdout:1515: left oid 4 (ObjNum 590 snap 187 seq_num 590) 2024-08-17T17:01:07.016 INFO:tasks.rados.rados.0.smithi063.stdout:1516: expect (ObjNum 577 snap 182 seq_num 577) 2024-08-17T17:01:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:06 smithi063 ceph-mon[114756]: osdmap e445: 8 total, 8 up, 8 in 2024-08-17T17:01:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:06 smithi063 ceph-mon[114756]: pgmap v121: 89 pgs: 89 active+clean; 203 MiB data, 770 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:01:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:06 smithi063 ceph-mon[109843]: osdmap e445: 8 total, 8 up, 8 in 2024-08-17T17:01:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:06 smithi063 ceph-mon[109843]: pgmap v121: 89 pgs: 89 active+clean; 203 MiB data, 770 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:01:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:06 smithi162 ceph-mon[34633]: osdmap e445: 8 total, 8 up, 8 in 2024-08-17T17:01:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:06 smithi162 ceph-mon[34633]: pgmap v121: 89 pgs: 89 active+clean; 203 MiB data, 770 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:01:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1518: finishing rollback tid 0 to smithi06387500-42 2024-08-17T17:01:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1518: finishing rollback tid 1 to smithi06387500-42 2024-08-17T17:01:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1518: finishing rollback tid 2 to smithi06387500-42 2024-08-17T17:01:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 1146 (ObjNum 514 snap 161 seq_num 514) dirty exists 2024-08-17T17:01:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1519: finishing write tid 1 to smithi06387500-33 2024-08-17T17:01:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1519: finishing write tid 2 to smithi06387500-33 2024-08-17T17:01:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1519: finishing write tid 3 to smithi06387500-33 2024-08-17T17:01:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1519: finishing write tid 4 to smithi06387500-33 2024-08-17T17:01:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1519: finishing write tid 5 to smithi06387500-33 2024-08-17T17:01:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1519: finishing write tid 6 to smithi06387500-33 2024-08-17T17:01:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1519: oid 33 updating version 0 to 469 2024-08-17T17:01:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1519: oid 33 version 469 is already newer than 468 2024-08-17T17:01:07.139 INFO:tasks.rados.rados.0.smithi063.stdout:1519: oid 33 version 469 is already newer than 467 2024-08-17T17:01:07.140 INFO:tasks.rados.rados.0.smithi063.stdout:1519: oid 33 version 469 is already newer than 466 2024-08-17T17:01:07.140 INFO:tasks.rados.rados.0.smithi063.stdout:1519: oid 33 updating version 469 to 470 2024-08-17T17:01:07.140 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 470 (ObjNum 591 snap 187 seq_num 591) dirty exists 2024-08-17T17:01:07.140 INFO:tasks.rados.rados.0.smithi063.stdout:1519: left oid 33 (ObjNum 591 snap 187 seq_num 591) 2024-08-17T17:01:07.140 INFO:tasks.rados.rados.0.smithi063.stdout:1520: seq_num 592 ranges {749744=706589,1909340=577199,3224294=482561} 2024-08-17T17:01:07.146 INFO:tasks.rados.rados.0.smithi063.stdout:1520: writing smithi06387500-35 from 749744 to 1456333 tid 1 2024-08-17T17:01:07.152 INFO:tasks.rados.rados.0.smithi063.stdout:1520: writing smithi06387500-35 from 1909340 to 2486539 tid 2 2024-08-17T17:01:07.159 INFO:tasks.rados.rados.0.smithi063.stdout:1520: writing smithi06387500-35 from 3224294 to 3706855 tid 3 2024-08-17T17:01:07.159 INFO:tasks.rados.rados.0.smithi063.stdout:1512: done (6 left) 2024-08-17T17:01:07.159 INFO:tasks.rados.rados.0.smithi063.stdout:1513: done (5 left) 2024-08-17T17:01:07.159 INFO:tasks.rados.rados.0.smithi063.stdout:1515: done (4 left) 2024-08-17T17:01:07.159 INFO:tasks.rados.rados.0.smithi063.stdout:1516: done (3 left) 2024-08-17T17:01:07.159 INFO:tasks.rados.rados.0.smithi063.stdout:1518: done (2 left) 2024-08-17T17:01:07.160 INFO:tasks.rados.rados.0.smithi063.stdout:1519: done (1 left) 2024-08-17T17:01:07.160 INFO:tasks.rados.rados.0.smithi063.stdout:1521: write oid 15 current snap is 187 2024-08-17T17:01:07.160 INFO:tasks.rados.rados.0.smithi063.stdout:1520: finishing write tid 1 to smithi06387500-35 2024-08-17T17:01:07.160 INFO:tasks.rados.rados.0.smithi063.stdout:1520: finishing write tid 2 to smithi06387500-35 2024-08-17T17:01:07.160 INFO:tasks.rados.rados.0.smithi063.stdout:1521: seq_num 593 ranges {766551=498191,1684175=720857,2851717=693833} 2024-08-17T17:01:07.165 INFO:tasks.rados.rados.0.smithi063.stdout:1521: writing smithi06387500-15 from 766551 to 1264742 tid 1 2024-08-17T17:01:07.172 INFO:tasks.rados.rados.0.smithi063.stdout:1521: writing smithi06387500-15 from 1684175 to 2405032 tid 2 2024-08-17T17:01:07.178 INFO:tasks.rados.rados.0.smithi063.stdout:1521: writing smithi06387500-15 from 2851717 to 3545550 tid 3 2024-08-17T17:01:07.178 INFO:tasks.rados.rados.0.smithi063.stdout:1522: snap_remove snap 171 2024-08-17T17:01:07.521 INFO:tasks.rados.rados.0.smithi063.stdout:1522: done (2 left) 2024-08-17T17:01:07.521 INFO:tasks.rados.rados.0.smithi063.stdout:1523: delete oid 7 current snap is 187 2024-08-17T17:01:07.521 INFO:tasks.rados.rados.0.smithi063.stdout:1520: finishing write tid 3 to smithi06387500-35 2024-08-17T17:01:07.521 INFO:tasks.rados.rados.0.smithi063.stdout:1520: finishing write tid 4 to smithi06387500-35 2024-08-17T17:01:07.522 INFO:tasks.rados.rados.0.smithi063.stdout:1520: finishing write tid 5 to smithi06387500-35 2024-08-17T17:01:07.522 INFO:tasks.rados.rados.0.smithi063.stdout:1520: oid 35 updating version 0 to 1543 2024-08-17T17:01:07.522 INFO:tasks.rados.rados.0.smithi063.stdout:1520: oid 35 version 1543 is already newer than 1542 2024-08-17T17:01:07.522 INFO:tasks.rados.rados.0.smithi063.stdout:1520: oid 35 updating version 1543 to 1545 2024-08-17T17:01:07.522 INFO:tasks.rados.rados.0.smithi063.stdout:1520: oid 35 version 1545 is already newer than 1544 2024-08-17T17:01:07.522 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 1545 (ObjNum 592 snap 187 seq_num 592) dirty exists 2024-08-17T17:01:07.522 INFO:tasks.rados.rados.0.smithi063.stdout:1520: left oid 35 (ObjNum 592 snap 187 seq_num 592) 2024-08-17T17:01:07.522 INFO:tasks.rados.rados.0.smithi063.stdout:1521: finishing write tid 1 to smithi06387500-15 2024-08-17T17:01:07.522 INFO:tasks.rados.rados.0.smithi063.stdout:1521: finishing write tid 2 to smithi06387500-15 2024-08-17T17:01:07.522 INFO:tasks.rados.rados.0.smithi063.stdout:1521: finishing write tid 3 to smithi06387500-15 2024-08-17T17:01:07.522 INFO:tasks.rados.rados.0.smithi063.stdout:1521: finishing write tid 4 to smithi06387500-15 2024-08-17T17:01:07.522 INFO:tasks.rados.rados.0.smithi063.stdout:1521: finishing write tid 5 to smithi06387500-15 2024-08-17T17:01:07.522 INFO:tasks.rados.rados.0.smithi063.stdout:1521: oid 15 updating version 0 to 607 2024-08-17T17:01:07.522 INFO:tasks.rados.rados.0.smithi063.stdout:1521: oid 15 version 607 is already newer than 604 2024-08-17T17:01:07.523 INFO:tasks.rados.rados.0.smithi063.stdout:1521: oid 15 version 607 is already newer than 605 2024-08-17T17:01:07.523 INFO:tasks.rados.rados.0.smithi063.stdout:1521: oid 15 version 607 is already newer than 606 2024-08-17T17:01:07.523 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 607 (ObjNum 593 snap 187 seq_num 593) dirty exists 2024-08-17T17:01:07.523 INFO:tasks.rados.rados.0.smithi063.stdout:1521: left oid 15 (ObjNum 593 snap 187 seq_num 593) 2024-08-17T17:01:07.532 INFO:tasks.rados.rados.0.smithi063.stdout:1520: done (2 left) 2024-08-17T17:01:07.532 INFO:tasks.rados.rados.0.smithi063.stdout:1521: done (1 left) 2024-08-17T17:01:07.532 INFO:tasks.rados.rados.0.smithi063.stdout:1523: done (0 left) 2024-08-17T17:01:07.532 INFO:tasks.rados.rados.0.smithi063.stdout:1524: read oid 40 snap -1 2024-08-17T17:01:07.532 INFO:tasks.rados.rados.0.smithi063.stdout:1524: expect deleted 2024-08-17T17:01:07.532 INFO:tasks.rados.rados.0.smithi063.stdout:1525: write oid 14 current snap is 187 2024-08-17T17:01:07.532 INFO:tasks.rados.rados.0.smithi063.stdout:1525: seq_num 595 ranges {400165=648689,1684786=315379} 2024-08-17T17:01:07.537 INFO:tasks.rados.rados.0.smithi063.stdout:1525: writing smithi06387500-14 from 400165 to 1048854 tid 1 2024-08-17T17:01:07.543 INFO:tasks.rados.rados.0.smithi063.stdout:1525: writing smithi06387500-14 from 1684786 to 2000165 tid 2 2024-08-17T17:01:07.543 INFO:tasks.rados.rados.0.smithi063.stdout:1526: snap_create 2024-08-17T17:01:07.543 INFO:tasks.rados.rados.0.smithi063.stdout:1525: finishing write tid 1 to smithi06387500-14 2024-08-17T17:01:07.546 INFO:tasks.rados.rados.0.smithi063.stdout:1525: finishing write tid 2 to smithi06387500-14 2024-08-17T17:01:07.546 INFO:tasks.rados.rados.0.smithi063.stdout:1525: finishing write tid 3 to smithi06387500-14 2024-08-17T17:01:07.546 INFO:tasks.rados.rados.0.smithi063.stdout:1525: finishing write tid 4 to smithi06387500-14 2024-08-17T17:01:07.546 INFO:tasks.rados.rados.0.smithi063.stdout:1525: oid 14 updating version 0 to 1559 2024-08-17T17:01:07.547 INFO:tasks.rados.rados.0.smithi063.stdout:1525: oid 14 updating version 1559 to 1561 2024-08-17T17:01:07.547 INFO:tasks.rados.rados.0.smithi063.stdout:1525: oid 14 version 1561 is already newer than 1560 2024-08-17T17:01:07.547 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 1561 (ObjNum 595 snap 187 seq_num 595) dirty exists 2024-08-17T17:01:07.547 INFO:tasks.rados.rados.0.smithi063.stdout:1525: left oid 14 (ObjNum 595 snap 187 seq_num 595) 2024-08-17T17:01:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:07 smithi063 ceph-mon[109843]: osdmap e446: 8 total, 8 up, 8 in 2024-08-17T17:01:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:07 smithi063 ceph-mon[109843]: osdmap e447: 8 total, 8 up, 8 in 2024-08-17T17:01:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:07 smithi063 ceph-mon[114756]: osdmap e446: 8 total, 8 up, 8 in 2024-08-17T17:01:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:07 smithi063 ceph-mon[114756]: osdmap e447: 8 total, 8 up, 8 in 2024-08-17T17:01:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:07 smithi162 ceph-mon[34633]: osdmap e446: 8 total, 8 up, 8 in 2024-08-17T17:01:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:07 smithi162 ceph-mon[34633]: osdmap e447: 8 total, 8 up, 8 in 2024-08-17T17:01:08.525 INFO:tasks.rados.rados.0.smithi063.stdout:1524: done (2 left) 2024-08-17T17:01:08.525 INFO:tasks.rados.rados.0.smithi063.stdout:1525: done (1 left) 2024-08-17T17:01:08.525 INFO:tasks.rados.rados.0.smithi063.stdout:1526: done (0 left) 2024-08-17T17:01:08.525 INFO:tasks.rados.rados.0.smithi063.stdout:1527: write oid 26 current snap is 188 2024-08-17T17:01:08.526 INFO:tasks.rados.rados.0.smithi063.stdout:1527: seq_num 596 ranges {416972=723938,1743268=273704} 2024-08-17T17:01:08.531 INFO:tasks.rados.rados.0.smithi063.stdout:1527: writing smithi06387500-26 from 416972 to 1140910 tid 1 2024-08-17T17:01:08.536 INFO:tasks.rados.rados.0.smithi063.stdout:1527: writing smithi06387500-26 from 1743268 to 2016972 tid 2 2024-08-17T17:01:08.536 INFO:tasks.rados.rados.0.smithi063.stdout:1528: snap_remove snap 157 2024-08-17T17:01:09.523 INFO:tasks.rados.rados.0.smithi063.stdout:1528: done (1 left) 2024-08-17T17:01:09.524 INFO:tasks.rados.rados.0.smithi063.stdout:1529: delete oid 22 current snap is 188 2024-08-17T17:01:09.524 INFO:tasks.rados.rados.0.smithi063.stdout:1527: finishing write tid 1 to smithi06387500-26 2024-08-17T17:01:09.524 INFO:tasks.rados.rados.0.smithi063.stdout:1527: finishing write tid 2 to smithi06387500-26 2024-08-17T17:01:09.524 INFO:tasks.rados.rados.0.smithi063.stdout:1527: finishing write tid 3 to smithi06387500-26 2024-08-17T17:01:09.524 INFO:tasks.rados.rados.0.smithi063.stdout:1527: finishing write tid 4 to smithi06387500-26 2024-08-17T17:01:09.524 INFO:tasks.rados.rados.0.smithi063.stdout:1527: oid 26 updating version 0 to 490 2024-08-17T17:01:09.524 INFO:tasks.rados.rados.0.smithi063.stdout:1527: oid 26 updating version 490 to 492 2024-08-17T17:01:09.524 INFO:tasks.rados.rados.0.smithi063.stdout:1527: oid 26 version 492 is already newer than 491 2024-08-17T17:01:09.524 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 492 (ObjNum 596 snap 188 seq_num 596) dirty exists 2024-08-17T17:01:09.524 INFO:tasks.rados.rados.0.smithi063.stdout:1527: left oid 26 (ObjNum 596 snap 188 seq_num 596) 2024-08-17T17:01:09.533 INFO:tasks.rados.rados.0.smithi063.stdout:1527: done (1 left) 2024-08-17T17:01:09.533 INFO:tasks.rados.rados.0.smithi063.stdout:1529: done (0 left) 2024-08-17T17:01:09.533 INFO:tasks.rados.rados.0.smithi063.stdout:1530: snap_create 2024-08-17T17:01:09.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:09 smithi063 ceph-mon[114756]: pgmap v124: 89 pgs: 89 active+clean; 208 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-08-17T17:01:09.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:09 smithi063 ceph-mon[114756]: osdmap e448: 8 total, 8 up, 8 in 2024-08-17T17:01:09.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:09.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:09 smithi063 ceph-mon[109843]: pgmap v124: 89 pgs: 89 active+clean; 208 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-08-17T17:01:09.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:09 smithi063 ceph-mon[109843]: osdmap e448: 8 total, 8 up, 8 in 2024-08-17T17:01:09.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:09.824 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:09 smithi162 ceph-mon[34633]: pgmap v124: 89 pgs: 89 active+clean; 208 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-08-17T17:01:09.824 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:09 smithi162 ceph-mon[34633]: osdmap e448: 8 total, 8 up, 8 in 2024-08-17T17:01:09.824 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:09 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:10.535 INFO:tasks.rados.rados.0.smithi063.stdout:1530: done (0 left) 2024-08-17T17:01:10.535 INFO:tasks.rados.rados.0.smithi063.stdout:1531: snap_remove snap 184 2024-08-17T17:01:10.668 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:10 smithi162 ceph-mon[34633]: osdmap e449: 8 total, 8 up, 8 in 2024-08-17T17:01:10.836 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:10 smithi063 ceph-mon[109843]: osdmap e449: 8 total, 8 up, 8 in 2024-08-17T17:01:10.836 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:10 smithi063 ceph-mon[114756]: osdmap e449: 8 total, 8 up, 8 in 2024-08-17T17:01:11.539 INFO:tasks.rados.rados.0.smithi063.stdout:1531: done (0 left) 2024-08-17T17:01:11.539 INFO:tasks.rados.rados.0.smithi063.stdout:1532: read oid 45 snap -1 2024-08-17T17:01:11.539 INFO:tasks.rados.rados.0.smithi063.stdout:1532: expect deleted 2024-08-17T17:01:11.539 INFO:tasks.rados.rados.0.smithi063.stdout:1533: snap_remove snap 185 2024-08-17T17:01:11.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:11 smithi063 ceph-mon[114756]: pgmap v127: 89 pgs: 89 active+clean; 208 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-08-17T17:01:11.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:11 smithi063 ceph-mon[114756]: osdmap e450: 8 total, 8 up, 8 in 2024-08-17T17:01:11.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:11 smithi063 ceph-mon[109843]: pgmap v127: 89 pgs: 89 active+clean; 208 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-08-17T17:01:11.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:11 smithi063 ceph-mon[109843]: osdmap e450: 8 total, 8 up, 8 in 2024-08-17T17:01:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:11 smithi162 ceph-mon[34633]: pgmap v127: 89 pgs: 89 active+clean; 208 MiB data, 713 MiB used, 715 GiB / 715 GiB avail; 19 MiB/s rd, 1.2 MiB/s wr, 17 op/s 2024-08-17T17:01:11.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:11 smithi162 ceph-mon[34633]: osdmap e450: 8 total, 8 up, 8 in 2024-08-17T17:01:12.540 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:12 smithi162 ceph-mon[34633]: osdmap e451: 8 total, 8 up, 8 in 2024-08-17T17:01:12.551 INFO:tasks.rados.rados.0.smithi063.stdout:1533: done (1 left) 2024-08-17T17:01:12.551 INFO:tasks.rados.rados.0.smithi063.stdout:1534: read oid 38 snap -1 2024-08-17T17:01:12.551 INFO:tasks.rados.rados.0.smithi063.stdout:1534: expect (ObjNum 504 snap 158 seq_num 504) 2024-08-17T17:01:12.551 INFO:tasks.rados.rados.0.smithi063.stdout:1535: snap_remove snap 176 2024-08-17T17:01:12.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:12 smithi063 ceph-mon[114756]: osdmap e451: 8 total, 8 up, 8 in 2024-08-17T17:01:12.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:12 smithi063 ceph-mon[109843]: osdmap e451: 8 total, 8 up, 8 in 2024-08-17T17:01:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:01:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:01:13] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.51.0" 2024-08-17T17:01:13.552 INFO:tasks.rados.rados.0.smithi063.stdout:1535: done (2 left) 2024-08-17T17:01:13.552 INFO:tasks.rados.rados.0.smithi063.stdout:1536: read oid 15 snap -1 2024-08-17T17:01:13.552 INFO:tasks.rados.rados.0.smithi063.stdout:1536: expect (ObjNum 593 snap 187 seq_num 593) 2024-08-17T17:01:13.552 INFO:tasks.rados.rados.0.smithi063.stdout:1537: read oid 28 snap 188 2024-08-17T17:01:13.552 INFO:tasks.rados.rados.0.smithi063.stdout:1537: expect deleted 2024-08-17T17:01:13.552 INFO:tasks.rados.rados.0.smithi063.stdout:1538: read oid 7 snap 188 2024-08-17T17:01:13.552 INFO:tasks.rados.rados.0.smithi063.stdout:1538: expect deleted 2024-08-17T17:01:13.552 INFO:tasks.rados.rados.0.smithi063.stdout:1539: read oid 29 snap 177 2024-08-17T17:01:13.553 INFO:tasks.rados.rados.0.smithi063.stdout:1539: expect (ObjNum 565 snap 173 seq_num 565) 2024-08-17T17:01:13.553 INFO:tasks.rados.rados.0.smithi063.stdout:1540: read oid 21 snap -1 2024-08-17T17:01:13.553 INFO:tasks.rados.rados.0.smithi063.stdout:1540: expect (ObjNum 578 snap 182 seq_num 578) 2024-08-17T17:01:13.553 INFO:tasks.rados.rados.0.smithi063.stdout:1541: read oid 46 snap 188 2024-08-17T17:01:13.553 INFO:tasks.rados.rados.0.smithi063.stdout:1541: expect (ObjNum 510 snap 160 seq_num 510) 2024-08-17T17:01:13.553 INFO:tasks.rados.rados.0.smithi063.stdout:1542: read oid 25 snap -1 2024-08-17T17:01:13.553 INFO:tasks.rados.rados.0.smithi063.stdout:1542: expect deleted 2024-08-17T17:01:13.553 INFO:tasks.rados.rados.0.smithi063.stdout:1543: delete oid 47 current snap is 189 2024-08-17T17:01:13.553 INFO:tasks.rados.rados.0.smithi063.stdout:1534: expect (ObjNum 504 snap 158 seq_num 504) 2024-08-17T17:01:13.645 INFO:tasks.rados.rados.0.smithi063.stdout:1532: done (9 left) 2024-08-17T17:01:13.645 INFO:tasks.rados.rados.0.smithi063.stdout:1534: done (8 left) 2024-08-17T17:01:13.645 INFO:tasks.rados.rados.0.smithi063.stdout:1537: done (7 left) 2024-08-17T17:01:13.645 INFO:tasks.rados.rados.0.smithi063.stdout:1538: done (6 left) 2024-08-17T17:01:13.645 INFO:tasks.rados.rados.0.smithi063.stdout:1542: done (5 left) 2024-08-17T17:01:13.645 INFO:tasks.rados.rados.0.smithi063.stdout:1543: done (4 left) 2024-08-17T17:01:13.646 INFO:tasks.rados.rados.0.smithi063.stdout:1544: read oid 40 snap -1 2024-08-17T17:01:13.646 INFO:tasks.rados.rados.0.smithi063.stdout:1544: expect deleted 2024-08-17T17:01:13.646 INFO:tasks.rados.rados.0.smithi063.stdout:1545: write oid 6 current snap is 189 2024-08-17T17:01:13.646 INFO:tasks.rados.rados.0.smithi063.stdout:1539: expect (ObjNum 565 snap 173 seq_num 565) 2024-08-17T17:01:13.783 INFO:tasks.rados.rados.0.smithi063.stdout:1536: expect (ObjNum 593 snap 187 seq_num 593) 2024-08-17T17:01:13.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:13 smithi162 ceph-mon[34633]: pgmap v130: 89 pgs: 89 active+clean; 153 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.8 MiB/s wr, 7 op/s 2024-08-17T17:01:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:13 smithi162 ceph-mon[34633]: osdmap e452: 8 total, 8 up, 8 in 2024-08-17T17:01:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:13 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:13 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:13 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:13 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:13.917 INFO:tasks.rados.rados.0.smithi063.stdout:1541: expect (ObjNum 510 snap 160 seq_num 510) 2024-08-17T17:01:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:13 smithi063 ceph-mon[109843]: pgmap v130: 89 pgs: 89 active+clean; 153 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.8 MiB/s wr, 7 op/s 2024-08-17T17:01:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:13 smithi063 ceph-mon[109843]: osdmap e452: 8 total, 8 up, 8 in 2024-08-17T17:01:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:13 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:13 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:13 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:13 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:14.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:13 smithi063 ceph-mon[114756]: pgmap v130: 89 pgs: 89 active+clean; 153 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.8 MiB/s wr, 7 op/s 2024-08-17T17:01:14.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:13 smithi063 ceph-mon[114756]: osdmap e452: 8 total, 8 up, 8 in 2024-08-17T17:01:14.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:13 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:14.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:13 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:14.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:13 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:14.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:13 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:14.094 INFO:tasks.rados.rados.0.smithi063.stdout:1540: expect (ObjNum 578 snap 182 seq_num 578) 2024-08-17T17:01:14.358 INFO:tasks.rados.rados.0.smithi063.stdout:1545: seq_num 599 ranges {467393=549685,1802361=265032} 2024-08-17T17:01:14.363 INFO:tasks.rados.rados.0.smithi063.stdout:1545: writing smithi06387500-6 from 467393 to 1017078 tid 1 2024-08-17T17:01:14.370 INFO:tasks.rados.rados.0.smithi063.stdout:1545: writing smithi06387500-6 from 1802361 to 2067393 tid 2 2024-08-17T17:01:14.370 INFO:tasks.rados.rados.0.smithi063.stdout:1536: done (5 left) 2024-08-17T17:01:14.370 INFO:tasks.rados.rados.0.smithi063.stdout:1539: done (4 left) 2024-08-17T17:01:14.370 INFO:tasks.rados.rados.0.smithi063.stdout:1540: done (3 left) 2024-08-17T17:01:14.370 INFO:tasks.rados.rados.0.smithi063.stdout:1541: done (2 left) 2024-08-17T17:01:14.370 INFO:tasks.rados.rados.0.smithi063.stdout:1544: done (1 left) 2024-08-17T17:01:14.370 INFO:tasks.rados.rados.0.smithi063.stdout:1546: read oid 22 snap -1 2024-08-17T17:01:14.370 INFO:tasks.rados.rados.0.smithi063.stdout:1546: expect deleted 2024-08-17T17:01:14.370 INFO:tasks.rados.rados.0.smithi063.stdout:1547: write oid 25 current snap is 189 2024-08-17T17:01:14.370 INFO:tasks.rados.rados.0.smithi063.stdout:1545: finishing write tid 1 to smithi06387500-6 2024-08-17T17:01:14.370 INFO:tasks.rados.rados.0.smithi063.stdout:1547: seq_num 600 ranges {484200=624934,1860843=223357} 2024-08-17T17:01:14.375 INFO:tasks.rados.rados.0.smithi063.stdout:1547: writing smithi06387500-25 from 484200 to 1109134 tid 1 2024-08-17T17:01:14.381 INFO:tasks.rados.rados.0.smithi063.stdout:1547: writing smithi06387500-25 from 1860843 to 2084200 tid 2 2024-08-17T17:01:14.381 INFO:tasks.rados.rados.0.smithi063.stdout:1548: delete oid 18 current snap is 189 2024-08-17T17:01:14.381 INFO:tasks.rados.rados.0.smithi063.stdout:1545: finishing write tid 2 to smithi06387500-6 2024-08-17T17:01:14.381 INFO:tasks.rados.rados.0.smithi063.stdout:1545: finishing write tid 3 to smithi06387500-6 2024-08-17T17:01:14.381 INFO:tasks.rados.rados.0.smithi063.stdout:1545: finishing write tid 4 to smithi06387500-6 2024-08-17T17:01:14.381 INFO:tasks.rados.rados.0.smithi063.stdout:1545: oid 6 updating version 0 to 492 2024-08-17T17:01:14.382 INFO:tasks.rados.rados.0.smithi063.stdout:1545: oid 6 version 492 is already newer than 491 2024-08-17T17:01:14.382 INFO:tasks.rados.rados.0.smithi063.stdout:1545: oid 6 version 492 is already newer than 490 2024-08-17T17:01:14.382 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 492 (ObjNum 599 snap 189 seq_num 599) dirty exists 2024-08-17T17:01:14.382 INFO:tasks.rados.rados.0.smithi063.stdout:1545: left oid 6 (ObjNum 599 snap 189 seq_num 599) 2024-08-17T17:01:14.382 INFO:tasks.rados.rados.0.smithi063.stdout:1547: finishing write tid 1 to smithi06387500-25 2024-08-17T17:01:14.383 INFO:tasks.rados.rados.0.smithi063.stdout:1545: done (3 left) 2024-08-17T17:01:14.383 INFO:tasks.rados.rados.0.smithi063.stdout:1546: done (2 left) 2024-08-17T17:01:14.383 INFO:tasks.rados.rados.0.smithi063.stdout:1548: done (1 left) 2024-08-17T17:01:14.383 INFO:tasks.rados.rados.0.smithi063.stdout:1549: read oid 11 snap -1 2024-08-17T17:01:14.383 INFO:tasks.rados.rados.0.smithi063.stdout:1549: expect deleted 2024-08-17T17:01:14.383 INFO:tasks.rados.rados.0.smithi063.stdout:1550: read oid 24 snap -1 2024-08-17T17:01:14.384 INFO:tasks.rados.rados.0.smithi063.stdout:1550: expect (ObjNum 583 snap 186 seq_num 583) 2024-08-17T17:01:14.384 INFO:tasks.rados.rados.0.smithi063.stdout:1551: snap_remove snap 187 2024-08-17T17:01:14.784 INFO:tasks.rados.rados.0.smithi063.stdout:1551: done (3 left) 2024-08-17T17:01:14.785 INFO:tasks.rados.rados.0.smithi063.stdout:1552: delete oid 29 current snap is 189 2024-08-17T17:01:14.785 INFO:tasks.rados.rados.0.smithi063.stdout:1547: finishing write tid 2 to smithi06387500-25 2024-08-17T17:01:14.785 INFO:tasks.rados.rados.0.smithi063.stdout:1547: finishing write tid 3 to smithi06387500-25 2024-08-17T17:01:14.785 INFO:tasks.rados.rados.0.smithi063.stdout:1547: finishing write tid 4 to smithi06387500-25 2024-08-17T17:01:14.785 INFO:tasks.rados.rados.0.smithi063.stdout:1547: oid 25 updating version 0 to 1596 2024-08-17T17:01:14.785 INFO:tasks.rados.rados.0.smithi063.stdout:1547: oid 25 updating version 1596 to 1597 2024-08-17T17:01:14.785 INFO:tasks.rados.rados.0.smithi063.stdout:1547: oid 25 updating version 1597 to 1598 2024-08-17T17:01:14.785 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 1598 (ObjNum 600 snap 189 seq_num 600) dirty exists 2024-08-17T17:01:14.785 INFO:tasks.rados.rados.0.smithi063.stdout:1547: left oid 25 (ObjNum 600 snap 189 seq_num 600) 2024-08-17T17:01:14.785 INFO:tasks.rados.rados.0.smithi063.stdout:1550: expect (ObjNum 583 snap 186 seq_num 583) 2024-08-17T17:01:14.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:14 smithi162 ceph-mon[34633]: osdmap e453: 8 total, 8 up, 8 in 2024-08-17T17:01:14.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:14 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:14.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:14 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:14.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:14 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:14.911 INFO:tasks.rados.rados.0.smithi063.stdout:1547: done (3 left) 2024-08-17T17:01:14.911 INFO:tasks.rados.rados.0.smithi063.stdout:1549: done (2 left) 2024-08-17T17:01:14.911 INFO:tasks.rados.rados.0.smithi063.stdout:1550: done (1 left) 2024-08-17T17:01:14.912 INFO:tasks.rados.rados.0.smithi063.stdout:1552: done (0 left) 2024-08-17T17:01:14.912 INFO:tasks.rados.rados.0.smithi063.stdout:1553: write oid 29 current snap is 189 2024-08-17T17:01:14.912 INFO:tasks.rados.rados.0.smithi063.stdout:1553: seq_num 603 ranges {534621=567034,1636289=498332} 2024-08-17T17:01:14.917 INFO:tasks.rados.rados.0.smithi063.stdout:1553: writing smithi06387500-29 from 534621 to 1101655 tid 1 2024-08-17T17:01:14.923 INFO:tasks.rados.rados.0.smithi063.stdout:1553: writing smithi06387500-29 from 1636289 to 2134621 tid 2 2024-08-17T17:01:14.923 INFO:tasks.rados.rados.0.smithi063.stdout:1554: snap_remove snap 188 2024-08-17T17:01:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:14 smithi063 ceph-mon[114756]: osdmap e453: 8 total, 8 up, 8 in 2024-08-17T17:01:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:14 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:14 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:14 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:15.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:14 smithi063 ceph-mon[109843]: osdmap e453: 8 total, 8 up, 8 in 2024-08-17T17:01:15.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:14 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:15.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:14 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:15.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:14 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:15.783 INFO:tasks.rados.rados.0.smithi063.stdout:1554: done (1 left) 2024-08-17T17:01:15.784 INFO:tasks.rados.rados.0.smithi063.stdout:1555: read oid 46 snap -1 2024-08-17T17:01:15.784 INFO:tasks.rados.rados.0.smithi063.stdout:1555: expect (ObjNum 510 snap 160 seq_num 510) 2024-08-17T17:01:15.784 INFO:tasks.rados.rados.0.smithi063.stdout:1556: rollback oid 22 current snap is 189 2024-08-17T17:01:15.784 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 22 to 173 2024-08-17T17:01:15.784 INFO:tasks.rados.rados.0.smithi063.stdout:1553: finishing write tid 1 to smithi06387500-29 2024-08-17T17:01:15.784 INFO:tasks.rados.rados.0.smithi063.stdout:1557: read oid 9 snap -1 2024-08-17T17:01:15.784 INFO:tasks.rados.rados.0.smithi063.stdout:1557: expect (ObjNum 576 snap 181 seq_num 576) 2024-08-17T17:01:15.784 INFO:tasks.rados.rados.0.smithi063.stdout:1553: finishing write tid 2 to smithi06387500-29 2024-08-17T17:01:15.784 INFO:tasks.rados.rados.0.smithi063.stdout:1553: finishing write tid 3 to smithi06387500-29 2024-08-17T17:01:15.784 INFO:tasks.rados.rados.0.smithi063.stdout:1553: finishing write tid 4 to smithi06387500-29 2024-08-17T17:01:15.784 INFO:tasks.rados.rados.0.smithi063.stdout:1553: oid 29 updating version 0 to 394 2024-08-17T17:01:15.784 INFO:tasks.rados.rados.0.smithi063.stdout:1553: oid 29 updating version 394 to 395 2024-08-17T17:01:15.784 INFO:tasks.rados.rados.0.smithi063.stdout:1553: oid 29 version 395 is already newer than 393 2024-08-17T17:01:15.785 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 395 (ObjNum 603 snap 189 seq_num 603) dirty exists 2024-08-17T17:01:15.785 INFO:tasks.rados.rados.0.smithi063.stdout:1553: left oid 29 (ObjNum 603 snap 189 seq_num 603) 2024-08-17T17:01:15.785 INFO:tasks.rados.rados.0.smithi063.stdout:1553: done (3 left) 2024-08-17T17:01:15.785 INFO:tasks.rados.rados.0.smithi063.stdout:1558: read oid 12 snap -1 2024-08-17T17:01:15.785 INFO:tasks.rados.rados.0.smithi063.stdout:1558: expect deleted 2024-08-17T17:01:15.785 INFO:tasks.rados.rados.0.smithi063.stdout:1559: snap_create 2024-08-17T17:01:15.788 INFO:tasks.rados.rados.0.smithi063.stdout:1556: finishing rollback tid 1 to smithi06387500-22 2024-08-17T17:01:15.788 INFO:tasks.rados.rados.0.smithi063.stdout:1556: finishing rollback tid 2 to smithi06387500-22 2024-08-17T17:01:15.788 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 373 (ObjNum 452 snap 146 seq_num 452) dirty exists 2024-08-17T17:01:15.792 INFO:tasks.rados.rados.0.smithi063.stdout:1555: expect (ObjNum 510 snap 160 seq_num 510) 2024-08-17T17:01:15.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:15 smithi162 ceph-mon[34633]: pgmap v133: 89 pgs: 89 active+clean; 153 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.8 MiB/s wr, 7 op/s 2024-08-17T17:01:15.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:15 smithi162 ceph-mon[34633]: osdmap e454: 8 total, 8 up, 8 in 2024-08-17T17:01:15.970 INFO:tasks.rados.rados.0.smithi063.stdout:1557: expect (ObjNum 576 snap 181 seq_num 576) 2024-08-17T17:01:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:15 smithi063 ceph-mon[109843]: pgmap v133: 89 pgs: 89 active+clean; 153 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.8 MiB/s wr, 7 op/s 2024-08-17T17:01:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:15 smithi063 ceph-mon[109843]: osdmap e454: 8 total, 8 up, 8 in 2024-08-17T17:01:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:15 smithi063 ceph-mon[114756]: pgmap v133: 89 pgs: 89 active+clean; 153 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.8 MiB/s wr, 7 op/s 2024-08-17T17:01:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:15 smithi063 ceph-mon[114756]: osdmap e454: 8 total, 8 up, 8 in 2024-08-17T17:01:16.788 INFO:tasks.rados.rados.0.smithi063.stdout:1555: done (4 left) 2024-08-17T17:01:16.788 INFO:tasks.rados.rados.0.smithi063.stdout:1556: done (3 left) 2024-08-17T17:01:16.788 INFO:tasks.rados.rados.0.smithi063.stdout:1557: done (2 left) 2024-08-17T17:01:16.788 INFO:tasks.rados.rados.0.smithi063.stdout:1558: done (1 left) 2024-08-17T17:01:16.788 INFO:tasks.rados.rados.0.smithi063.stdout:1559: done (0 left) 2024-08-17T17:01:16.788 INFO:tasks.rados.rados.0.smithi063.stdout:1560: snap_create 2024-08-17T17:01:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:16 smithi162 ceph-mon[34633]: osdmap e455: 8 total, 8 up, 8 in 2024-08-17T17:01:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:16 smithi162 ceph-mon[34633]: pgmap v136: 89 pgs: 89 active+clean; 153 MiB data, 615 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:01:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:16 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:17.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:16 smithi063 ceph-mon[114756]: osdmap e455: 8 total, 8 up, 8 in 2024-08-17T17:01:17.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:16 smithi063 ceph-mon[114756]: pgmap v136: 89 pgs: 89 active+clean; 153 MiB data, 615 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:01:17.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:17.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:16 smithi063 ceph-mon[109843]: osdmap e455: 8 total, 8 up, 8 in 2024-08-17T17:01:17.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:16 smithi063 ceph-mon[109843]: pgmap v136: 89 pgs: 89 active+clean; 153 MiB data, 615 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:01:17.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:17.522 INFO:tasks.rados.rados.0.smithi063.stdout:1560: done (0 left) 2024-08-17T17:01:17.522 INFO:tasks.rados.rados.0.smithi063.stdout:1561: rollback oid 6 current snap is 191 2024-08-17T17:01:17.522 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 6 to 173 2024-08-17T17:01:17.522 INFO:tasks.rados.rados.0.smithi063.stdout:1562: write oid 32 current snap is 191 2024-08-17T17:01:17.522 INFO:tasks.rados.rados.0.smithi063.stdout:1562: seq_num 604 ranges {551428=642283,1694771=456657} 2024-08-17T17:01:17.528 INFO:tasks.rados.rados.0.smithi063.stdout:1562: writing smithi06387500-32 from 551428 to 1193711 tid 1 2024-08-17T17:01:17.534 INFO:tasks.rados.rados.0.smithi063.stdout:1562: writing smithi06387500-32 from 1694771 to 2151428 tid 2 2024-08-17T17:01:17.534 INFO:tasks.rados.rados.0.smithi063.stdout:1563: rollback oid 46 current snap is 191 2024-08-17T17:01:17.534 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 46 to 177 2024-08-17T17:01:17.535 INFO:tasks.rados.rados.0.smithi063.stdout:1561: finishing rollback tid 0 to smithi06387500-6 2024-08-17T17:01:17.535 INFO:tasks.rados.rados.0.smithi063.stdout:1561: finishing rollback tid 1 to smithi06387500-6 2024-08-17T17:01:17.535 INFO:tasks.rados.rados.0.smithi063.stdout:1561: finishing rollback tid 2 to smithi06387500-6 2024-08-17T17:01:17.535 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 500 (ObjNum 425 snap 133 seq_num 425) dirty exists 2024-08-17T17:01:17.535 INFO:tasks.rados.rados.0.smithi063.stdout:1562: finishing write tid 1 to smithi06387500-32 2024-08-17T17:01:17.535 INFO:tasks.rados.rados.0.smithi063.stdout:1561: done (2 left) 2024-08-17T17:01:17.535 INFO:tasks.rados.rados.0.smithi063.stdout:1564: read oid 10 snap 190 2024-08-17T17:01:17.535 INFO:tasks.rados.rados.0.smithi063.stdout:1564: expect (ObjNum 575 snap 181 seq_num 575) 2024-08-17T17:01:17.535 INFO:tasks.rados.rados.0.smithi063.stdout:1565: snap_create 2024-08-17T17:01:17.537 INFO:tasks.rados.rados.0.smithi063.stdout:1562: finishing write tid 2 to smithi06387500-32 2024-08-17T17:01:17.537 INFO:tasks.rados.rados.0.smithi063.stdout:1562: finishing write tid 3 to smithi06387500-32 2024-08-17T17:01:17.538 INFO:tasks.rados.rados.0.smithi063.stdout:1562: finishing write tid 4 to smithi06387500-32 2024-08-17T17:01:17.538 INFO:tasks.rados.rados.0.smithi063.stdout:1562: oid 32 updating version 0 to 310 2024-08-17T17:01:17.538 INFO:tasks.rados.rados.0.smithi063.stdout:1562: oid 32 updating version 310 to 311 2024-08-17T17:01:17.538 INFO:tasks.rados.rados.0.smithi063.stdout:1562: oid 32 updating version 311 to 312 2024-08-17T17:01:17.538 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 312 (ObjNum 604 snap 191 seq_num 604) dirty exists 2024-08-17T17:01:17.538 INFO:tasks.rados.rados.0.smithi063.stdout:1562: left oid 32 (ObjNum 604 snap 191 seq_num 604) 2024-08-17T17:01:17.541 INFO:tasks.rados.rados.0.smithi063.stdout:1563: finishing rollback tid 0 to smithi06387500-46 2024-08-17T17:01:17.541 INFO:tasks.rados.rados.0.smithi063.stdout:1563: finishing rollback tid 1 to smithi06387500-46 2024-08-17T17:01:17.541 INFO:tasks.rados.rados.0.smithi063.stdout:1563: finishing rollback tid 2 to smithi06387500-46 2024-08-17T17:01:17.541 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 629 (ObjNum 510 snap 160 seq_num 510) dirty exists 2024-08-17T17:01:17.543 INFO:tasks.rados.rados.0.smithi063.stdout:1564: expect (ObjNum 575 snap 181 seq_num 575) 2024-08-17T17:01:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:17 smithi162 ceph-mon[34633]: osdmap e456: 8 total, 8 up, 8 in 2024-08-17T17:01:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:17 smithi162 ceph-mon[34633]: osdmap e457: 8 total, 8 up, 8 in 2024-08-17T17:01:18.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:17 smithi063 ceph-mon[114756]: osdmap e456: 8 total, 8 up, 8 in 2024-08-17T17:01:18.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:17 smithi063 ceph-mon[114756]: osdmap e457: 8 total, 8 up, 8 in 2024-08-17T17:01:18.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:17 smithi063 ceph-mon[109843]: osdmap e456: 8 total, 8 up, 8 in 2024-08-17T17:01:18.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:17 smithi063 ceph-mon[109843]: osdmap e457: 8 total, 8 up, 8 in 2024-08-17T17:01:18.527 INFO:tasks.rados.rados.0.smithi063.stdout:1562: done (3 left) 2024-08-17T17:01:18.527 INFO:tasks.rados.rados.0.smithi063.stdout:1563: done (2 left) 2024-08-17T17:01:18.527 INFO:tasks.rados.rados.0.smithi063.stdout:1564: done (1 left) 2024-08-17T17:01:18.527 INFO:tasks.rados.rados.0.smithi063.stdout:1565: done (0 left) 2024-08-17T17:01:18.527 INFO:tasks.rados.rados.0.smithi063.stdout:1566: read oid 17 snap 173 2024-08-17T17:01:18.527 INFO:tasks.rados.rados.0.smithi063.stdout:1566: expect (ObjNum 537 snap 169 seq_num 537) 2024-08-17T17:01:18.528 INFO:tasks.rados.rados.0.smithi063.stdout:1567: read oid 36 snap -1 2024-08-17T17:01:18.528 INFO:tasks.rados.rados.0.smithi063.stdout:1567: expect deleted 2024-08-17T17:01:18.528 INFO:tasks.rados.rados.0.smithi063.stdout:1568: snap_remove snap 190 2024-08-17T17:01:19.537 INFO:tasks.rados.rados.0.smithi063.stdout:1568: done (2 left) 2024-08-17T17:01:19.537 INFO:tasks.rados.rados.0.smithi063.stdout:1569: snap_remove snap 177 2024-08-17T17:01:19.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:19 smithi063 ceph-mon[109843]: pgmap v139: 89 pgs: 89 active+clean; 150 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 667 KiB/s wr, 22 op/s 2024-08-17T17:01:19.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:19 smithi063 ceph-mon[109843]: osdmap e458: 8 total, 8 up, 8 in 2024-08-17T17:01:19.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:19 smithi063 ceph-mon[109843]: mgrmap e40: y(active, since 92s), standbys: x 2024-08-17T17:01:19.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:19 smithi063 ceph-mon[114756]: pgmap v139: 89 pgs: 89 active+clean; 150 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 667 KiB/s wr, 22 op/s 2024-08-17T17:01:19.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:19 smithi063 ceph-mon[114756]: osdmap e458: 8 total, 8 up, 8 in 2024-08-17T17:01:19.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:19 smithi063 ceph-mon[114756]: mgrmap e40: y(active, since 92s), standbys: x 2024-08-17T17:01:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:19 smithi162 ceph-mon[34633]: pgmap v139: 89 pgs: 89 active+clean; 150 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 667 KiB/s wr, 22 op/s 2024-08-17T17:01:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:19 smithi162 ceph-mon[34633]: osdmap e458: 8 total, 8 up, 8 in 2024-08-17T17:01:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:19 smithi162 ceph-mon[34633]: mgrmap e40: y(active, since 92s), standbys: x 2024-08-17T17:01:20.541 INFO:tasks.rados.rados.0.smithi063.stdout:1569: done (2 left) 2024-08-17T17:01:20.542 INFO:tasks.rados.rados.0.smithi063.stdout:1570: write oid 7 current snap is 192 2024-08-17T17:01:20.542 INFO:tasks.rados.rados.0.smithi063.stdout:1570: seq_num 605 ranges {568235=717532,1753253=414982} 2024-08-17T17:01:20.548 INFO:tasks.rados.rados.0.smithi063.stdout:1570: writing smithi06387500-7 from 568235 to 1285767 tid 1 2024-08-17T17:01:20.553 INFO:tasks.rados.rados.0.smithi063.stdout:1570: writing smithi06387500-7 from 1753253 to 2168235 tid 2 2024-08-17T17:01:20.553 INFO:tasks.rados.rados.0.smithi063.stdout:1571: write oid 37 current snap is 192 2024-08-17T17:01:20.553 INFO:tasks.rados.rados.0.smithi063.stdout:1571: seq_num 606 ranges {585042=792781,2095382=89660} 2024-08-17T17:01:20.562 INFO:tasks.rados.rados.0.smithi063.stdout:1571: writing smithi06387500-37 from 585042 to 1377823 tid 1 2024-08-17T17:01:20.567 INFO:tasks.rados.rados.0.smithi063.stdout:1571: writing smithi06387500-37 from 2095382 to 2185042 tid 2 2024-08-17T17:01:20.567 INFO:tasks.rados.rados.0.smithi063.stdout:1572: write oid 14 current snap is 192 2024-08-17T17:01:20.567 INFO:tasks.rados.rados.0.smithi063.stdout:1572: seq_num 607 ranges {601849=468030,1753864=447985} 2024-08-17T17:01:20.573 INFO:tasks.rados.rados.0.smithi063.stdout:1572: writing smithi06387500-14 from 601849 to 1069879 tid 1 2024-08-17T17:01:20.579 INFO:tasks.rados.rados.0.smithi063.stdout:1572: writing smithi06387500-14 from 1753864 to 2201849 tid 2 2024-08-17T17:01:20.579 INFO:tasks.rados.rados.0.smithi063.stdout:1573: read oid 24 snap -1 2024-08-17T17:01:20.579 INFO:tasks.rados.rados.0.smithi063.stdout:1573: expect (ObjNum 583 snap 186 seq_num 583) 2024-08-17T17:01:20.579 INFO:tasks.rados.rados.0.smithi063.stdout:1574: rollback oid 50 current snap is 192 2024-08-17T17:01:20.580 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 50 to 173 2024-08-17T17:01:20.580 INFO:tasks.rados.rados.0.smithi063.stdout:1566: expect (ObjNum 537 snap 169 seq_num 537) 2024-08-17T17:01:20.723 INFO:tasks.rados.rados.0.smithi063.stdout:1570: finishing write tid 1 to smithi06387500-7 2024-08-17T17:01:20.724 INFO:tasks.rados.rados.0.smithi063.stdout:1570: finishing write tid 2 to smithi06387500-7 2024-08-17T17:01:20.724 INFO:tasks.rados.rados.0.smithi063.stdout:1570: finishing write tid 3 to smithi06387500-7 2024-08-17T17:01:20.724 INFO:tasks.rados.rados.0.smithi063.stdout:1570: finishing write tid 4 to smithi06387500-7 2024-08-17T17:01:20.724 INFO:tasks.rados.rados.0.smithi063.stdout:1570: oid 7 updating version 0 to 742 2024-08-17T17:01:20.724 INFO:tasks.rados.rados.0.smithi063.stdout:1570: oid 7 version 742 is already newer than 739 2024-08-17T17:01:20.724 INFO:tasks.rados.rados.0.smithi063.stdout:1570: oid 7 updating version 742 to 743 2024-08-17T17:01:20.724 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 743 (ObjNum 605 snap 192 seq_num 605) dirty exists 2024-08-17T17:01:20.724 INFO:tasks.rados.rados.0.smithi063.stdout:1570: left oid 7 (ObjNum 605 snap 192 seq_num 605) 2024-08-17T17:01:20.724 INFO:tasks.rados.rados.0.smithi063.stdout:1571: finishing write tid 1 to smithi06387500-37 2024-08-17T17:01:20.724 INFO:tasks.rados.rados.0.smithi063.stdout:1571: finishing write tid 2 to smithi06387500-37 2024-08-17T17:01:20.724 INFO:tasks.rados.rados.0.smithi063.stdout:1571: finishing write tid 3 to smithi06387500-37 2024-08-17T17:01:20.724 INFO:tasks.rados.rados.0.smithi063.stdout:1566: done (6 left) 2024-08-17T17:01:20.725 INFO:tasks.rados.rados.0.smithi063.stdout:1567: done (5 left) 2024-08-17T17:01:20.725 INFO:tasks.rados.rados.0.smithi063.stdout:1570: done (4 left) 2024-08-17T17:01:20.725 INFO:tasks.rados.rados.0.smithi063.stdout:1575: snap_remove snap 191 2024-08-17T17:01:20.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:20 smithi162 ceph-mon[34633]: osdmap e459: 8 total, 8 up, 8 in 2024-08-17T17:01:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:20 smithi063 ceph-mon[114756]: osdmap e459: 8 total, 8 up, 8 in 2024-08-17T17:01:21.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:20 smithi063 ceph-mon[109843]: osdmap e459: 8 total, 8 up, 8 in 2024-08-17T17:01:21.551 INFO:tasks.rados.rados.0.smithi063.stdout:1575: done (4 left) 2024-08-17T17:01:21.552 INFO:tasks.rados.rados.0.smithi063.stdout:1576: snap_create 2024-08-17T17:01:21.552 INFO:tasks.rados.rados.0.smithi063.stdout:1571: finishing write tid 4 to smithi06387500-37 2024-08-17T17:01:21.552 INFO:tasks.rados.rados.0.smithi063.stdout:1571: oid 37 updating version 0 to 1606 2024-08-17T17:01:21.552 INFO:tasks.rados.rados.0.smithi063.stdout:1571: oid 37 updating version 1606 to 1607 2024-08-17T17:01:21.552 INFO:tasks.rados.rados.0.smithi063.stdout:1571: oid 37 updating version 1607 to 1608 2024-08-17T17:01:21.552 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 1608 (ObjNum 606 snap 192 seq_num 606) dirty exists 2024-08-17T17:01:21.552 INFO:tasks.rados.rados.0.smithi063.stdout:1571: left oid 37 (ObjNum 606 snap 192 seq_num 606) 2024-08-17T17:01:21.552 INFO:tasks.rados.rados.0.smithi063.stdout:1572: finishing write tid 1 to smithi06387500-14 2024-08-17T17:01:21.552 INFO:tasks.rados.rados.0.smithi063.stdout:1572: finishing write tid 2 to smithi06387500-14 2024-08-17T17:01:21.552 INFO:tasks.rados.rados.0.smithi063.stdout:1572: finishing write tid 3 to smithi06387500-14 2024-08-17T17:01:21.552 INFO:tasks.rados.rados.0.smithi063.stdout:1572: finishing write tid 4 to smithi06387500-14 2024-08-17T17:01:21.553 INFO:tasks.rados.rados.0.smithi063.stdout:1572: oid 14 updating version 0 to 1610 2024-08-17T17:01:21.553 INFO:tasks.rados.rados.0.smithi063.stdout:1572: oid 14 updating version 1610 to 1612 2024-08-17T17:01:21.553 INFO:tasks.rados.rados.0.smithi063.stdout:1572: oid 14 version 1612 is already newer than 1611 2024-08-17T17:01:21.553 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 1612 (ObjNum 607 snap 192 seq_num 607) dirty exists 2024-08-17T17:01:21.553 INFO:tasks.rados.rados.0.smithi063.stdout:1572: left oid 14 (ObjNum 607 snap 192 seq_num 607) 2024-08-17T17:01:21.553 INFO:tasks.rados.rados.0.smithi063.stdout:1574: finishing rollback tid 0 to smithi06387500-50 2024-08-17T17:01:21.553 INFO:tasks.rados.rados.0.smithi063.stdout:1574: finishing rollback tid 1 to smithi06387500-50 2024-08-17T17:01:21.553 INFO:tasks.rados.rados.0.smithi063.stdout:1574: finishing rollback tid 2 to smithi06387500-50 2024-08-17T17:01:21.553 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 1190 (ObjNum 564 snap 173 seq_num 564) dirty exists 2024-08-17T17:01:21.553 INFO:tasks.rados.rados.0.smithi063.stdout:1573: expect (ObjNum 583 snap 186 seq_num 583) 2024-08-17T17:01:21.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:21 smithi162 ceph-mon[34633]: pgmap v142: 89 pgs: 89 active+clean; 150 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 667 KiB/s wr, 22 op/s 2024-08-17T17:01:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:21 smithi162 ceph-mon[34633]: osdmap e460: 8 total, 8 up, 8 in 2024-08-17T17:01:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:21 smithi063 ceph-mon[114756]: pgmap v142: 89 pgs: 89 active+clean; 150 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 667 KiB/s wr, 22 op/s 2024-08-17T17:01:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:21 smithi063 ceph-mon[114756]: osdmap e460: 8 total, 8 up, 8 in 2024-08-17T17:01:22.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:21 smithi063 ceph-mon[109843]: pgmap v142: 89 pgs: 89 active+clean; 150 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 667 KiB/s wr, 22 op/s 2024-08-17T17:01:22.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:21 smithi063 ceph-mon[109843]: osdmap e460: 8 total, 8 up, 8 in 2024-08-17T17:01:22.558 INFO:tasks.rados.rados.0.smithi063.stdout:1571: done (4 left) 2024-08-17T17:01:22.558 INFO:tasks.rados.rados.0.smithi063.stdout:1572: done (3 left) 2024-08-17T17:01:22.558 INFO:tasks.rados.rados.0.smithi063.stdout:1573: done (2 left) 2024-08-17T17:01:22.558 INFO:tasks.rados.rados.0.smithi063.stdout:1574: done (1 left) 2024-08-17T17:01:22.558 INFO:tasks.rados.rados.0.smithi063.stdout:1576: done (0 left) 2024-08-17T17:01:22.558 INFO:tasks.rados.rados.0.smithi063.stdout:1577: snap_remove snap 192 2024-08-17T17:01:22.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:22 smithi162 ceph-mon[34633]: osdmap e461: 8 total, 8 up, 8 in 2024-08-17T17:01:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:22 smithi063 ceph-mon[114756]: osdmap e461: 8 total, 8 up, 8 in 2024-08-17T17:01:23.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:22 smithi063 ceph-mon[109843]: osdmap e461: 8 total, 8 up, 8 in 2024-08-17T17:01:23.553 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:01:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:01:23] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.51.0" 2024-08-17T17:01:23.566 INFO:tasks.rados.rados.0.smithi063.stdout:1577: done (0 left) 2024-08-17T17:01:23.566 INFO:tasks.rados.rados.0.smithi063.stdout:1578: read oid 14 snap -1 2024-08-17T17:01:23.566 INFO:tasks.rados.rados.0.smithi063.stdout:1578: expect (ObjNum 607 snap 192 seq_num 607) 2024-08-17T17:01:23.567 INFO:tasks.rados.rados.0.smithi063.stdout:1579: read oid 5 snap -1 2024-08-17T17:01:23.567 INFO:tasks.rados.rados.0.smithi063.stdout:1579: expect (ObjNum 443 snap 139 seq_num 443) 2024-08-17T17:01:23.567 INFO:tasks.rados.rados.0.smithi063.stdout:1580: rollback oid 23 current snap is 193 2024-08-17T17:01:23.567 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 23 to 173 2024-08-17T17:01:23.567 INFO:tasks.rados.rados.0.smithi063.stdout:1581: read oid 1 snap 189 2024-08-17T17:01:23.567 INFO:tasks.rados.rados.0.smithi063.stdout:1581: expect (ObjNum 511 snap 160 seq_num 511) 2024-08-17T17:01:23.567 INFO:tasks.rados.rados.0.smithi063.stdout:1582: write oid 22 current snap is 193 2024-08-17T17:01:23.567 INFO:tasks.rados.rados.0.smithi063.stdout:1582: seq_num 608 ranges {618656=543279,1812346=406310} 2024-08-17T17:01:23.572 INFO:tasks.rados.rados.0.smithi063.stdout:1582: writing smithi06387500-22 from 618656 to 1161935 tid 1 2024-08-17T17:01:23.579 INFO:tasks.rados.rados.0.smithi063.stdout:1582: writing smithi06387500-22 from 1812346 to 2218656 tid 2 2024-08-17T17:01:23.579 INFO:tasks.rados.rados.0.smithi063.stdout:1583: write oid 11 current snap is 193 2024-08-17T17:01:23.579 INFO:tasks.rados.rados.0.smithi063.stdout:1583: seq_num 609 ranges {635463=734881,1987181=248282} 2024-08-17T17:01:23.588 INFO:tasks.rados.rados.0.smithi063.stdout:1583: writing smithi06387500-11 from 635463 to 1370344 tid 1 2024-08-17T17:01:23.593 INFO:tasks.rados.rados.0.smithi063.stdout:1583: writing smithi06387500-11 from 1987181 to 2235463 tid 2 2024-08-17T17:01:23.593 INFO:tasks.rados.rados.0.smithi063.stdout:1584: write oid 27 current snap is 193 2024-08-17T17:01:23.593 INFO:tasks.rados.rados.0.smithi063.stdout:1584: seq_num 610 ranges {652270=410130,1529310=493867} 2024-08-17T17:01:23.599 INFO:tasks.rados.rados.0.smithi063.stdout:1584: writing smithi06387500-27 from 652270 to 1062400 tid 1 2024-08-17T17:01:23.604 INFO:tasks.rados.rados.0.smithi063.stdout:1584: writing smithi06387500-27 from 1529310 to 2023177 tid 2 2024-08-17T17:01:23.604 INFO:tasks.rados.rados.0.smithi063.stdout:1585: delete oid 8 current snap is 193 2024-08-17T17:01:23.604 INFO:tasks.rados.rados.0.smithi063.stdout:1578: expect (ObjNum 607 snap 192 seq_num 607) 2024-08-17T17:01:23.768 INFO:tasks.rados.rados.0.smithi063.stdout:1578: done (7 left) 2024-08-17T17:01:23.768 INFO:tasks.rados.rados.0.smithi063.stdout:1585: done (6 left) 2024-08-17T17:01:23.768 INFO:tasks.rados.rados.0.smithi063.stdout:1586: write oid 12 current snap is 193 2024-08-17T17:01:23.768 INFO:tasks.rados.rados.0.smithi063.stdout:1582: finishing write tid 1 to smithi06387500-22 2024-08-17T17:01:23.768 INFO:tasks.rados.rados.0.smithi063.stdout:1579: expect (ObjNum 443 snap 139 seq_num 443) 2024-08-17T17:01:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:23 smithi162 ceph-mon[34633]: pgmap v145: 89 pgs: 89 active+clean; 159 MiB data, 614 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 985 KiB/s wr, 13 op/s 2024-08-17T17:01:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:23 smithi162 ceph-mon[34633]: osdmap e462: 8 total, 8 up, 8 in 2024-08-17T17:01:23.963 INFO:tasks.rados.rados.0.smithi063.stdout:1582: finishing write tid 2 to smithi06387500-22 2024-08-17T17:01:23.964 INFO:tasks.rados.rados.0.smithi063.stdout:1582: finishing write tid 3 to smithi06387500-22 2024-08-17T17:01:23.964 INFO:tasks.rados.rados.0.smithi063.stdout:1586: seq_num 612 ranges {685884=560628,2046274=239610} 2024-08-17T17:01:23.970 INFO:tasks.rados.rados.0.smithi063.stdout:1586: writing smithi06387500-12 from 685884 to 1246512 tid 1 2024-08-17T17:01:23.976 INFO:tasks.rados.rados.0.smithi063.stdout:1586: writing smithi06387500-12 from 2046274 to 2285884 tid 2 2024-08-17T17:01:23.977 INFO:tasks.rados.rados.0.smithi063.stdout:1579: done (6 left) 2024-08-17T17:01:23.977 INFO:tasks.rados.rados.0.smithi063.stdout:1587: read oid 25 snap 189 2024-08-17T17:01:23.977 INFO:tasks.rados.rados.0.smithi063.stdout:1587: expect (ObjNum 600 snap 189 seq_num 600) 2024-08-17T17:01:23.977 INFO:tasks.rados.rados.0.smithi063.stdout:1588: delete oid 10 current snap is 193 2024-08-17T17:01:23.977 INFO:tasks.rados.rados.0.smithi063.stdout:1582: finishing write tid 4 to smithi06387500-22 2024-08-17T17:01:23.977 INFO:tasks.rados.rados.0.smithi063.stdout:1582: oid 22 updating version 0 to 378 2024-08-17T17:01:23.977 INFO:tasks.rados.rados.0.smithi063.stdout:1582: oid 22 version 378 is already newer than 377 2024-08-17T17:01:23.977 INFO:tasks.rados.rados.0.smithi063.stdout:1582: oid 22 updating version 378 to 379 2024-08-17T17:01:23.977 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 379 (ObjNum 608 snap 193 seq_num 608) dirty exists 2024-08-17T17:01:23.977 INFO:tasks.rados.rados.0.smithi063.stdout:1582: left oid 22 (ObjNum 608 snap 193 seq_num 608) 2024-08-17T17:01:23.977 INFO:tasks.rados.rados.0.smithi063.stdout:1581: expect (ObjNum 511 snap 160 seq_num 511) 2024-08-17T17:01:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:23 smithi063 ceph-mon[114756]: pgmap v145: 89 pgs: 89 active+clean; 159 MiB data, 614 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 985 KiB/s wr, 13 op/s 2024-08-17T17:01:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:23 smithi063 ceph-mon[114756]: osdmap e462: 8 total, 8 up, 8 in 2024-08-17T17:01:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:23 smithi063 ceph-mon[109843]: pgmap v145: 89 pgs: 89 active+clean; 159 MiB data, 614 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 985 KiB/s wr, 13 op/s 2024-08-17T17:01:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:23 smithi063 ceph-mon[109843]: osdmap e462: 8 total, 8 up, 8 in 2024-08-17T17:01:24.156 INFO:tasks.rados.rados.0.smithi063.stdout:1580: finishing rollback tid 0 to smithi06387500-23 2024-08-17T17:01:24.156 INFO:tasks.rados.rados.0.smithi063.stdout:1580: finishing rollback tid 1 to smithi06387500-23 2024-08-17T17:01:24.156 INFO:tasks.rados.rados.0.smithi063.stdout:1580: finishing rollback tid 2 to smithi06387500-23 2024-08-17T17:01:24.156 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 740 (ObjNum 546 snap 171 seq_num 546) dirty exists 2024-08-17T17:01:24.156 INFO:tasks.rados.rados.0.smithi063.stdout:1583: finishing write tid 1 to smithi06387500-11 2024-08-17T17:01:24.157 INFO:tasks.rados.rados.0.smithi063.stdout:1583: finishing write tid 2 to smithi06387500-11 2024-08-17T17:01:24.157 INFO:tasks.rados.rados.0.smithi063.stdout:1583: finishing write tid 3 to smithi06387500-11 2024-08-17T17:01:24.157 INFO:tasks.rados.rados.0.smithi063.stdout:1583: finishing write tid 4 to smithi06387500-11 2024-08-17T17:01:24.157 INFO:tasks.rados.rados.0.smithi063.stdout:1583: oid 11 updating version 0 to 227 2024-08-17T17:01:24.157 INFO:tasks.rados.rados.0.smithi063.stdout:1583: oid 11 version 227 is already newer than 225 2024-08-17T17:01:24.157 INFO:tasks.rados.rados.0.smithi063.stdout:1583: oid 11 version 227 is already newer than 226 2024-08-17T17:01:24.157 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 227 (ObjNum 609 snap 193 seq_num 609) dirty exists 2024-08-17T17:01:24.157 INFO:tasks.rados.rados.0.smithi063.stdout:1583: left oid 11 (ObjNum 609 snap 193 seq_num 609) 2024-08-17T17:01:24.157 INFO:tasks.rados.rados.0.smithi063.stdout:1584: finishing write tid 1 to smithi06387500-27 2024-08-17T17:01:24.157 INFO:tasks.rados.rados.0.smithi063.stdout:1584: finishing write tid 2 to smithi06387500-27 2024-08-17T17:01:24.157 INFO:tasks.rados.rados.0.smithi063.stdout:1584: finishing write tid 3 to smithi06387500-27 2024-08-17T17:01:24.157 INFO:tasks.rados.rados.0.smithi063.stdout:1584: finishing write tid 4 to smithi06387500-27 2024-08-17T17:01:24.157 INFO:tasks.rados.rados.0.smithi063.stdout:1584: oid 27 updating version 0 to 177 2024-08-17T17:01:24.158 INFO:tasks.rados.rados.0.smithi063.stdout:1584: oid 27 updating version 177 to 178 2024-08-17T17:01:24.158 INFO:tasks.rados.rados.0.smithi063.stdout:1584: oid 27 version 178 is already newer than 176 2024-08-17T17:01:24.158 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 178 (ObjNum 610 snap 193 seq_num 610) dirty exists 2024-08-17T17:01:24.158 INFO:tasks.rados.rados.0.smithi063.stdout:1584: left oid 27 (ObjNum 610 snap 193 seq_num 610) 2024-08-17T17:01:24.158 INFO:tasks.rados.rados.0.smithi063.stdout:1580: done (7 left) 2024-08-17T17:01:24.158 INFO:tasks.rados.rados.0.smithi063.stdout:1581: done (6 left) 2024-08-17T17:01:24.158 INFO:tasks.rados.rados.0.smithi063.stdout:1582: done (5 left) 2024-08-17T17:01:24.158 INFO:tasks.rados.rados.0.smithi063.stdout:1583: done (4 left) 2024-08-17T17:01:24.158 INFO:tasks.rados.rados.0.smithi063.stdout:1584: done (3 left) 2024-08-17T17:01:24.158 INFO:tasks.rados.rados.0.smithi063.stdout:1588: done (2 left) 2024-08-17T17:01:24.158 INFO:tasks.rados.rados.0.smithi063.stdout:1589: write oid 11 current snap is 193 2024-08-17T17:01:24.158 INFO:tasks.rados.rados.0.smithi063.stdout:1586: finishing write tid 1 to smithi06387500-12 2024-08-17T17:01:24.158 INFO:tasks.rados.rados.0.smithi063.stdout:1589: seq_num 614 ranges {719498=711126,2046885=272613} 2024-08-17T17:01:24.163 INFO:tasks.rados.rados.0.smithi063.stdout:1589: writing smithi06387500-11 from 719498 to 1430624 tid 1 2024-08-17T17:01:24.169 INFO:tasks.rados.rados.0.smithi063.stdout:1589: writing smithi06387500-11 from 2046885 to 2319498 tid 2 2024-08-17T17:01:24.169 INFO:tasks.rados.rados.0.smithi063.stdout:1590: snap_remove snap 173 2024-08-17T17:01:24.571 INFO:tasks.rados.rados.0.smithi063.stdout:1590: done (3 left) 2024-08-17T17:01:24.572 INFO:tasks.rados.rados.0.smithi063.stdout:1591: delete oid 37 current snap is 193 2024-08-17T17:01:24.572 INFO:tasks.rados.rados.0.smithi063.stdout:1586: finishing write tid 2 to smithi06387500-12 2024-08-17T17:01:24.572 INFO:tasks.rados.rados.0.smithi063.stdout:1586: finishing write tid 3 to smithi06387500-12 2024-08-17T17:01:24.572 INFO:tasks.rados.rados.0.smithi063.stdout:1586: finishing write tid 4 to smithi06387500-12 2024-08-17T17:01:24.572 INFO:tasks.rados.rados.0.smithi063.stdout:1586: oid 12 updating version 0 to 403 2024-08-17T17:01:24.572 INFO:tasks.rados.rados.0.smithi063.stdout:1586: oid 12 updating version 403 to 404 2024-08-17T17:01:24.572 INFO:tasks.rados.rados.0.smithi063.stdout:1586: oid 12 version 404 is already newer than 402 2024-08-17T17:01:24.572 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 404 (ObjNum 612 snap 193 seq_num 612) dirty exists 2024-08-17T17:01:24.572 INFO:tasks.rados.rados.0.smithi063.stdout:1586: left oid 12 (ObjNum 612 snap 193 seq_num 612) 2024-08-17T17:01:24.572 INFO:tasks.rados.rados.0.smithi063.stdout:1587: expect (ObjNum 600 snap 189 seq_num 600) 2024-08-17T17:01:24.647 INFO:tasks.rados.rados.0.smithi063.stdout:1589: finishing write tid 1 to smithi06387500-11 2024-08-17T17:01:24.647 INFO:tasks.rados.rados.0.smithi063.stdout:1589: finishing write tid 2 to smithi06387500-11 2024-08-17T17:01:24.647 INFO:tasks.rados.rados.0.smithi063.stdout:1586: done (3 left) 2024-08-17T17:01:24.647 INFO:tasks.rados.rados.0.smithi063.stdout:1587: done (2 left) 2024-08-17T17:01:24.647 INFO:tasks.rados.rados.0.smithi063.stdout:1591: done (1 left) 2024-08-17T17:01:24.647 INFO:tasks.rados.rados.0.smithi063.stdout:1592: snap_remove snap 189 2024-08-17T17:01:24.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:24 smithi162 ceph-mon[34633]: osdmap e463: 8 total, 8 up, 8 in 2024-08-17T17:01:25.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:24 smithi063 ceph-mon[114756]: osdmap e463: 8 total, 8 up, 8 in 2024-08-17T17:01:25.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:24 smithi063 ceph-mon[109843]: osdmap e463: 8 total, 8 up, 8 in 2024-08-17T17:01:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1592: done (1 left) 2024-08-17T17:01:25.577 INFO:tasks.rados.rados.0.smithi063.stdout:1593: snap_create 2024-08-17T17:01:25.577 INFO:tasks.rados.rados.0.smithi063.stdout:1589: finishing write tid 3 to smithi06387500-11 2024-08-17T17:01:25.577 INFO:tasks.rados.rados.0.smithi063.stdout:1589: finishing write tid 4 to smithi06387500-11 2024-08-17T17:01:25.577 INFO:tasks.rados.rados.0.smithi063.stdout:1589: oid 11 updating version 0 to 230 2024-08-17T17:01:25.577 INFO:tasks.rados.rados.0.smithi063.stdout:1589: oid 11 version 230 is already newer than 229 2024-08-17T17:01:25.577 INFO:tasks.rados.rados.0.smithi063.stdout:1589: oid 11 version 230 is already newer than 228 2024-08-17T17:01:25.577 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 230 (ObjNum 614 snap 193 seq_num 614) dirty exists 2024-08-17T17:01:25.577 INFO:tasks.rados.rados.0.smithi063.stdout:1589: left oid 11 (ObjNum 614 snap 193 seq_num 614) 2024-08-17T17:01:25.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:25 smithi162 ceph-mon[34633]: pgmap v148: 89 pgs: 89 active+clean; 159 MiB data, 614 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 985 KiB/s wr, 13 op/s 2024-08-17T17:01:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:25 smithi162 ceph-mon[34633]: osdmap e464: 8 total, 8 up, 8 in 2024-08-17T17:01:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:25 smithi063 ceph-mon[114756]: pgmap v148: 89 pgs: 89 active+clean; 159 MiB data, 614 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 985 KiB/s wr, 13 op/s 2024-08-17T17:01:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:25 smithi063 ceph-mon[114756]: osdmap e464: 8 total, 8 up, 8 in 2024-08-17T17:01:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:25 smithi063 ceph-mon[109843]: pgmap v148: 89 pgs: 89 active+clean; 159 MiB data, 614 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 985 KiB/s wr, 13 op/s 2024-08-17T17:01:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:25 smithi063 ceph-mon[109843]: osdmap e464: 8 total, 8 up, 8 in 2024-08-17T17:01:26.588 INFO:tasks.rados.rados.0.smithi063.stdout:1589: done (1 left) 2024-08-17T17:01:26.588 INFO:tasks.rados.rados.0.smithi063.stdout:1593: done (0 left) 2024-08-17T17:01:26.588 INFO:tasks.rados.rados.0.smithi063.stdout:1594: rollback oid 25 current snap is 194 2024-08-17T17:01:26.588 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 25 to 193 2024-08-17T17:01:26.588 INFO:tasks.rados.rados.0.smithi063.stdout:1595: rollback oid 38 current snap is 194 2024-08-17T17:01:26.588 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 38 to 193 2024-08-17T17:01:26.589 INFO:tasks.rados.rados.0.smithi063.stdout:1596: rollback oid 43 current snap is 194 2024-08-17T17:01:26.589 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 43 to 193 2024-08-17T17:01:26.589 INFO:tasks.rados.rados.0.smithi063.stdout:1597: rollback oid 6 current snap is 194 2024-08-17T17:01:26.589 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 6 to 193 2024-08-17T17:01:26.589 INFO:tasks.rados.rados.0.smithi063.stdout:1598: read oid 2 snap -1 2024-08-17T17:01:26.589 INFO:tasks.rados.rados.0.smithi063.stdout:1598: expect deleted 2024-08-17T17:01:26.589 INFO:tasks.rados.rados.0.smithi063.stdout:1599: delete oid 9 current snap is 194 2024-08-17T17:01:26.594 INFO:tasks.rados.rados.0.smithi063.stdout:1596: finishing rollback tid 0 to smithi06387500-43 2024-08-17T17:01:26.595 INFO:tasks.rados.rados.0.smithi063.stdout:1596: finishing rollback tid 1 to smithi06387500-43 2024-08-17T17:01:26.595 INFO:tasks.rados.rados.0.smithi063.stdout:1596: finishing rollback tid 2 to smithi06387500-43 2024-08-17T17:01:26.595 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 749 (ObjNum 483 snap 153 seq_num 483) dirty exists 2024-08-17T17:01:26.595 INFO:tasks.rados.rados.0.smithi063.stdout:1594: finishing rollback tid 0 to smithi06387500-25 2024-08-17T17:01:26.595 INFO:tasks.rados.rados.0.smithi063.stdout:1596: done (5 left) 2024-08-17T17:01:26.595 INFO:tasks.rados.rados.0.smithi063.stdout:1598: done (4 left) 2024-08-17T17:01:26.595 INFO:tasks.rados.rados.0.smithi063.stdout:1599: done (3 left) 2024-08-17T17:01:26.596 INFO:tasks.rados.rados.0.smithi063.stdout:1600: write oid 13 current snap is 194 2024-08-17T17:01:26.596 INFO:tasks.rados.rados.0.smithi063.stdout:1600: seq_num 617 ranges {769919=653226,1938684=431235} 2024-08-17T17:01:26.603 INFO:tasks.rados.rados.0.smithi063.stdout:1600: writing smithi06387500-13 from 769919 to 1423145 tid 1 2024-08-17T17:01:26.608 INFO:tasks.rados.rados.0.smithi063.stdout:1600: writing smithi06387500-13 from 1938684 to 2369919 tid 2 2024-08-17T17:01:26.609 INFO:tasks.rados.rados.0.smithi063.stdout:1601: rollback oid 15 current snap is 194 2024-08-17T17:01:26.609 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 15 to 193 2024-08-17T17:01:26.609 INFO:tasks.rados.rados.0.smithi063.stdout:1595: finishing rollback tid 0 to smithi06387500-38 2024-08-17T17:01:26.609 INFO:tasks.rados.rados.0.smithi063.stdout:1595: finishing rollback tid 1 to smithi06387500-38 2024-08-17T17:01:26.609 INFO:tasks.rados.rados.0.smithi063.stdout:1594: finishing rollback tid 1 to smithi06387500-25 2024-08-17T17:01:26.609 INFO:tasks.rados.rados.0.smithi063.stdout:1595: finishing rollback tid 2 to smithi06387500-38 2024-08-17T17:01:26.609 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 234 (ObjNum 504 snap 158 seq_num 504) dirty exists 2024-08-17T17:01:26.609 INFO:tasks.rados.rados.0.smithi063.stdout:1594: finishing rollback tid 2 to smithi06387500-25 2024-08-17T17:01:26.609 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 1635 (ObjNum 600 snap 189 seq_num 600) dirty exists 2024-08-17T17:01:26.609 INFO:tasks.rados.rados.0.smithi063.stdout:1597: finishing rollback tid 0 to smithi06387500-6 2024-08-17T17:01:26.609 INFO:tasks.rados.rados.0.smithi063.stdout:1597: finishing rollback tid 1 to smithi06387500-6 2024-08-17T17:01:26.609 INFO:tasks.rados.rados.0.smithi063.stdout:1597: finishing rollback tid 2 to smithi06387500-6 2024-08-17T17:01:26.610 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 522 (ObjNum 425 snap 133 seq_num 425) dirty exists 2024-08-17T17:01:26.610 INFO:tasks.rados.rados.0.smithi063.stdout:1600: finishing write tid 1 to smithi06387500-13 2024-08-17T17:01:26.610 INFO:tasks.rados.rados.0.smithi063.stdout:1594: done (4 left) 2024-08-17T17:01:26.610 INFO:tasks.rados.rados.0.smithi063.stdout:1595: done (3 left) 2024-08-17T17:01:26.610 INFO:tasks.rados.rados.0.smithi063.stdout:1597: done (2 left) 2024-08-17T17:01:26.610 INFO:tasks.rados.rados.0.smithi063.stdout:1602: snap_create 2024-08-17T17:01:26.611 INFO:tasks.rados.rados.0.smithi063.stdout:1600: finishing write tid 2 to smithi06387500-13 2024-08-17T17:01:26.611 INFO:tasks.rados.rados.0.smithi063.stdout:1601: finishing rollback tid 0 to smithi06387500-15 2024-08-17T17:01:26.612 INFO:tasks.rados.rados.0.smithi063.stdout:1600: finishing write tid 3 to smithi06387500-13 2024-08-17T17:01:26.612 INFO:tasks.rados.rados.0.smithi063.stdout:1601: finishing rollback tid 1 to smithi06387500-15 2024-08-17T17:01:26.612 INFO:tasks.rados.rados.0.smithi063.stdout:1600: finishing write tid 4 to smithi06387500-13 2024-08-17T17:01:26.612 INFO:tasks.rados.rados.0.smithi063.stdout:1600: oid 13 updating version 0 to 745 2024-08-17T17:01:26.612 INFO:tasks.rados.rados.0.smithi063.stdout:1600: oid 13 updating version 745 to 747 2024-08-17T17:01:26.612 INFO:tasks.rados.rados.0.smithi063.stdout:1600: oid 13 version 747 is already newer than 746 2024-08-17T17:01:26.612 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 747 (ObjNum 617 snap 194 seq_num 617) dirty exists 2024-08-17T17:01:26.612 INFO:tasks.rados.rados.0.smithi063.stdout:1600: left oid 13 (ObjNum 617 snap 194 seq_num 617) 2024-08-17T17:01:26.612 INFO:tasks.rados.rados.0.smithi063.stdout:1601: finishing rollback tid 2 to smithi06387500-15 2024-08-17T17:01:26.612 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 645 (ObjNum 593 snap 187 seq_num 593) dirty exists 2024-08-17T17:01:26.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:26 smithi162 ceph-mon[34633]: osdmap e465: 8 total, 8 up, 8 in 2024-08-17T17:01:27.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:26 smithi063 ceph-mon[114756]: osdmap e465: 8 total, 8 up, 8 in 2024-08-17T17:01:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:26 smithi063 ceph-mon[109843]: osdmap e465: 8 total, 8 up, 8 in 2024-08-17T17:01:27.525 INFO:tasks.rados.rados.0.smithi063.stdout:1600: done (2 left) 2024-08-17T17:01:27.525 INFO:tasks.rados.rados.0.smithi063.stdout:1601: done (1 left) 2024-08-17T17:01:27.525 INFO:tasks.rados.rados.0.smithi063.stdout:1602: done (0 left) 2024-08-17T17:01:27.525 INFO:tasks.rados.rados.0.smithi063.stdout:1603: read oid 25 snap -1 2024-08-17T17:01:27.526 INFO:tasks.rados.rados.0.smithi063.stdout:1603: expect (ObjNum 600 snap 189 seq_num 600) 2024-08-17T17:01:27.526 INFO:tasks.rados.rados.0.smithi063.stdout:1604: snap_create 2024-08-17T17:01:27.531 INFO:tasks.rados.rados.0.smithi063.stdout:1603: expect (ObjNum 600 snap 189 seq_num 600) 2024-08-17T17:01:27.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:27 smithi162 ceph-mon[34633]: pgmap v151: 89 pgs: 89 active+clean; 159 MiB data, 614 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:01:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:27 smithi162 ceph-mon[34633]: osdmap e466: 8 total, 8 up, 8 in 2024-08-17T17:01:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:27 smithi162 ceph-mon[34633]: osdmap e467: 8 total, 8 up, 8 in 2024-08-17T17:01:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:27 smithi063 ceph-mon[109843]: pgmap v151: 89 pgs: 89 active+clean; 159 MiB data, 614 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:01:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:27 smithi063 ceph-mon[109843]: osdmap e466: 8 total, 8 up, 8 in 2024-08-17T17:01:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:27 smithi063 ceph-mon[109843]: osdmap e467: 8 total, 8 up, 8 in 2024-08-17T17:01:28.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:27 smithi063 ceph-mon[114756]: pgmap v151: 89 pgs: 89 active+clean; 159 MiB data, 614 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:01:28.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:27 smithi063 ceph-mon[114756]: osdmap e466: 8 total, 8 up, 8 in 2024-08-17T17:01:28.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:27 smithi063 ceph-mon[114756]: osdmap e467: 8 total, 8 up, 8 in 2024-08-17T17:01:28.527 INFO:tasks.rados.rados.0.smithi063.stdout:1603: done (1 left) 2024-08-17T17:01:28.527 INFO:tasks.rados.rados.0.smithi063.stdout:1604: done (0 left) 2024-08-17T17:01:28.527 INFO:tasks.rados.rados.0.smithi063.stdout:1605: delete oid 41 current snap is 196 2024-08-17T17:01:28.532 INFO:tasks.rados.rados.0.smithi063.stdout:1605: done (0 left) 2024-08-17T17:01:28.532 INFO:tasks.rados.rados.0.smithi063.stdout:1606: write oid 41 current snap is 196 2024-08-17T17:01:28.532 INFO:tasks.rados.rados.0.smithi063.stdout:1606: seq_num 619 ranges {403533=403724,1539295=652201} 2024-08-17T17:01:28.536 INFO:tasks.rados.rados.0.smithi063.stdout:1606: writing smithi06387500-41 from 403533 to 807257 tid 1 2024-08-17T17:01:28.544 INFO:tasks.rados.rados.0.smithi063.stdout:1606: writing smithi06387500-41 from 1539295 to 2191496 tid 2 2024-08-17T17:01:28.544 INFO:tasks.rados.rados.0.smithi063.stdout:1607: rollback oid 43 current snap is 196 2024-08-17T17:01:28.544 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 43 to 195 2024-08-17T17:01:28.544 INFO:tasks.rados.rados.0.smithi063.stdout:1606: finishing write tid 1 to smithi06387500-41 2024-08-17T17:01:28.544 INFO:tasks.rados.rados.0.smithi063.stdout:1608: snap_create 2024-08-17T17:01:28.548 INFO:tasks.rados.rados.0.smithi063.stdout:1606: finishing write tid 2 to smithi06387500-41 2024-08-17T17:01:28.548 INFO:tasks.rados.rados.0.smithi063.stdout:1606: finishing write tid 3 to smithi06387500-41 2024-08-17T17:01:28.548 INFO:tasks.rados.rados.0.smithi063.stdout:1606: finishing write tid 4 to smithi06387500-41 2024-08-17T17:01:28.548 INFO:tasks.rados.rados.0.smithi063.stdout:1606: oid 41 updating version 0 to 136 2024-08-17T17:01:28.548 INFO:tasks.rados.rados.0.smithi063.stdout:1606: oid 41 version 136 is already newer than 135 2024-08-17T17:01:28.548 INFO:tasks.rados.rados.0.smithi063.stdout:1606: oid 41 version 136 is already newer than 134 2024-08-17T17:01:28.548 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 136 (ObjNum 619 snap 196 seq_num 619) dirty exists 2024-08-17T17:01:28.548 INFO:tasks.rados.rados.0.smithi063.stdout:1606: left oid 41 (ObjNum 619 snap 196 seq_num 619) 2024-08-17T17:01:28.551 INFO:tasks.rados.rados.0.smithi063.stdout:1607: finishing rollback tid 0 to smithi06387500-43 2024-08-17T17:01:28.551 INFO:tasks.rados.rados.0.smithi063.stdout:1607: finishing rollback tid 1 to smithi06387500-43 2024-08-17T17:01:28.551 INFO:tasks.rados.rados.0.smithi063.stdout:1607: finishing rollback tid 2 to smithi06387500-43 2024-08-17T17:01:28.551 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 755 (ObjNum 483 snap 153 seq_num 483) dirty exists 2024-08-17T17:01:29.531 INFO:tasks.rados.rados.0.smithi063.stdout:1606: done (2 left) 2024-08-17T17:01:29.531 INFO:tasks.rados.rados.0.smithi063.stdout:1607: done (1 left) 2024-08-17T17:01:29.532 INFO:tasks.rados.rados.0.smithi063.stdout:1608: done (0 left) 2024-08-17T17:01:29.532 INFO:tasks.rados.rados.0.smithi063.stdout:1609: write oid 37 current snap is 197 2024-08-17T17:01:29.532 INFO:tasks.rados.rados.0.smithi063.stdout:1609: seq_num 620 ranges {420340=478973,1597777=512212} 2024-08-17T17:01:29.536 INFO:tasks.rados.rados.0.smithi063.stdout:1609: writing smithi06387500-37 from 420340 to 899313 tid 1 2024-08-17T17:01:29.543 INFO:tasks.rados.rados.0.smithi063.stdout:1609: writing smithi06387500-37 from 1597777 to 2109989 tid 2 2024-08-17T17:01:29.543 INFO:tasks.rados.rados.0.smithi063.stdout:1610: snap_remove snap 194 2024-08-17T17:01:29.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:29 smithi162 ceph-mon[34633]: pgmap v154: 89 pgs: 89 active+clean; 102 MiB data, 518 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.4 MiB/s wr, 21 op/s 2024-08-17T17:01:29.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:29 smithi162 ceph-mon[34633]: osdmap e468: 8 total, 8 up, 8 in 2024-08-17T17:01:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:29 smithi063 ceph-mon[114756]: pgmap v154: 89 pgs: 89 active+clean; 102 MiB data, 518 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.4 MiB/s wr, 21 op/s 2024-08-17T17:01:30.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:29 smithi063 ceph-mon[114756]: osdmap e468: 8 total, 8 up, 8 in 2024-08-17T17:01:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:29 smithi063 ceph-mon[109843]: pgmap v154: 89 pgs: 89 active+clean; 102 MiB data, 518 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.4 MiB/s wr, 21 op/s 2024-08-17T17:01:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:29 smithi063 ceph-mon[109843]: osdmap e468: 8 total, 8 up, 8 in 2024-08-17T17:01:30.541 INFO:tasks.rados.rados.0.smithi063.stdout:1610: done (1 left) 2024-08-17T17:01:30.541 INFO:tasks.rados.rados.0.smithi063.stdout:1611: read oid 48 snap -1 2024-08-17T17:01:30.541 INFO:tasks.rados.rados.0.smithi063.stdout:1611: expect deleted 2024-08-17T17:01:30.541 INFO:tasks.rados.rados.0.smithi063.stdout:1612: read oid 19 snap -1 2024-08-17T17:01:30.541 INFO:tasks.rados.rados.0.smithi063.stdout:1612: expect deleted 2024-08-17T17:01:30.542 INFO:tasks.rados.rados.0.smithi063.stdout:1613: snap_remove snap 193 2024-08-17T17:01:30.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:30 smithi162 ceph-mon[34633]: osdmap e469: 8 total, 8 up, 8 in 2024-08-17T17:01:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:30 smithi063 ceph-mon[114756]: osdmap e469: 8 total, 8 up, 8 in 2024-08-17T17:01:31.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:30 smithi063 ceph-mon[109843]: osdmap e469: 8 total, 8 up, 8 in 2024-08-17T17:01:31.547 INFO:tasks.rados.rados.0.smithi063.stdout:1613: done (3 left) 2024-08-17T17:01:31.547 INFO:tasks.rados.rados.0.smithi063.stdout:1614: write oid 41 current snap is 197 2024-08-17T17:01:31.547 INFO:tasks.rados.rados.0.smithi063.stdout:1609: finishing write tid 1 to smithi06387500-37 2024-08-17T17:01:31.547 INFO:tasks.rados.rados.0.smithi063.stdout:1609: finishing write tid 2 to smithi06387500-37 2024-08-17T17:01:31.547 INFO:tasks.rados.rados.0.smithi063.stdout:1609: finishing write tid 3 to smithi06387500-37 2024-08-17T17:01:31.547 INFO:tasks.rados.rados.0.smithi063.stdout:1609: finishing write tid 4 to smithi06387500-37 2024-08-17T17:01:31.547 INFO:tasks.rados.rados.0.smithi063.stdout:1609: oid 37 updating version 0 to 1637 2024-08-17T17:01:31.547 INFO:tasks.rados.rados.0.smithi063.stdout:1609: oid 37 updating version 1637 to 1638 2024-08-17T17:01:31.548 INFO:tasks.rados.rados.0.smithi063.stdout:1609: oid 37 version 1638 is already newer than 1636 2024-08-17T17:01:31.548 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 1638 (ObjNum 620 snap 197 seq_num 620) dirty exists 2024-08-17T17:01:31.548 INFO:tasks.rados.rados.0.smithi063.stdout:1609: left oid 37 (ObjNum 620 snap 197 seq_num 620) 2024-08-17T17:01:31.548 INFO:tasks.rados.rados.0.smithi063.stdout:1614: seq_num 621 ranges {437147=554222,1656259=655870} 2024-08-17T17:01:31.553 INFO:tasks.rados.rados.0.smithi063.stdout:1614: writing smithi06387500-41 from 437147 to 991369 tid 1 2024-08-17T17:01:31.561 INFO:tasks.rados.rados.0.smithi063.stdout:1614: writing smithi06387500-41 from 1656259 to 2312129 tid 2 2024-08-17T17:01:31.561 INFO:tasks.rados.rados.0.smithi063.stdout:1609: done (3 left) 2024-08-17T17:01:31.561 INFO:tasks.rados.rados.0.smithi063.stdout:1615: snap_remove snap 196 2024-08-17T17:01:31.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:31 smithi162 ceph-mon[34633]: pgmap v157: 89 pgs: 89 active+clean; 102 MiB data, 518 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.4 MiB/s wr, 21 op/s 2024-08-17T17:01:31.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:31 smithi162 ceph-mon[34633]: osdmap e470: 8 total, 8 up, 8 in 2024-08-17T17:01:31.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:31 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:31 smithi063 ceph-mon[114756]: pgmap v157: 89 pgs: 89 active+clean; 102 MiB data, 518 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.4 MiB/s wr, 21 op/s 2024-08-17T17:01:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:31 smithi063 ceph-mon[114756]: osdmap e470: 8 total, 8 up, 8 in 2024-08-17T17:01:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:32.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:31 smithi063 ceph-mon[109843]: pgmap v157: 89 pgs: 89 active+clean; 102 MiB data, 518 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 1.4 MiB/s wr, 21 op/s 2024-08-17T17:01:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:31 smithi063 ceph-mon[109843]: osdmap e470: 8 total, 8 up, 8 in 2024-08-17T17:01:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:32.530 INFO:tasks.rados.rados.0.smithi063.stdout:1615: done (3 left) 2024-08-17T17:01:32.530 INFO:tasks.rados.rados.0.smithi063.stdout:1616: read oid 25 snap -1 2024-08-17T17:01:32.530 INFO:tasks.rados.rados.0.smithi063.stdout:1616: expect (ObjNum 600 snap 189 seq_num 600) 2024-08-17T17:01:32.530 INFO:tasks.rados.rados.0.smithi063.stdout:1617: snap_create 2024-08-17T17:01:32.530 INFO:tasks.rados.rados.0.smithi063.stdout:1614: finishing write tid 1 to smithi06387500-41 2024-08-17T17:01:32.530 INFO:tasks.rados.rados.0.smithi063.stdout:1614: finishing write tid 2 to smithi06387500-41 2024-08-17T17:01:32.530 INFO:tasks.rados.rados.0.smithi063.stdout:1614: finishing write tid 3 to smithi06387500-41 2024-08-17T17:01:32.531 INFO:tasks.rados.rados.0.smithi063.stdout:1614: finishing write tid 4 to smithi06387500-41 2024-08-17T17:01:32.531 INFO:tasks.rados.rados.0.smithi063.stdout:1614: oid 41 updating version 0 to 142 2024-08-17T17:01:32.531 INFO:tasks.rados.rados.0.smithi063.stdout:1614: oid 41 updating version 142 to 143 2024-08-17T17:01:32.531 INFO:tasks.rados.rados.0.smithi063.stdout:1614: oid 41 updating version 143 to 144 2024-08-17T17:01:32.531 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 144 (ObjNum 621 snap 197 seq_num 621) dirty exists 2024-08-17T17:01:32.531 INFO:tasks.rados.rados.0.smithi063.stdout:1614: left oid 41 (ObjNum 621 snap 197 seq_num 621) 2024-08-17T17:01:32.537 INFO:tasks.rados.rados.0.smithi063.stdout:1616: expect (ObjNum 600 snap 189 seq_num 600) 2024-08-17T17:01:32.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:32 smithi162 ceph-mon[34633]: osdmap e471: 8 total, 8 up, 8 in 2024-08-17T17:01:32.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:32 smithi162 ceph-mon[34633]: osdmap e472: 8 total, 8 up, 8 in 2024-08-17T17:01:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:32 smithi063 ceph-mon[114756]: osdmap e471: 8 total, 8 up, 8 in 2024-08-17T17:01:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:32 smithi063 ceph-mon[114756]: osdmap e472: 8 total, 8 up, 8 in 2024-08-17T17:01:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:32 smithi063 ceph-mon[109843]: osdmap e471: 8 total, 8 up, 8 in 2024-08-17T17:01:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:32 smithi063 ceph-mon[109843]: osdmap e472: 8 total, 8 up, 8 in 2024-08-17T17:01:33.528 INFO:tasks.rados.rados.0.smithi063.stdout:1611: done (4 left) 2024-08-17T17:01:33.528 INFO:tasks.rados.rados.0.smithi063.stdout:1612: done (3 left) 2024-08-17T17:01:33.528 INFO:tasks.rados.rados.0.smithi063.stdout:1614: done (2 left) 2024-08-17T17:01:33.528 INFO:tasks.rados.rados.0.smithi063.stdout:1616: done (1 left) 2024-08-17T17:01:33.528 INFO:tasks.rados.rados.0.smithi063.stdout:1617: done (0 left) 2024-08-17T17:01:33.529 INFO:tasks.rados.rados.0.smithi063.stdout:1618: snap_remove snap 195 2024-08-17T17:01:33.554 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:01:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:01:33] "GET /metrics HTTP/1.1" 200 36206 "" "Prometheus/2.51.0" 2024-08-17T17:01:33.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:33 smithi162 ceph-mon[34633]: pgmap v160: 89 pgs: 89 active+clean; 104 MiB data, 526 MiB used, 715 GiB / 715 GiB avail; 623 KiB/s rd, 796 KiB/s wr, 6 op/s 2024-08-17T17:01:33.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:33 smithi162 ceph-mon[34633]: osdmap e473: 8 total, 8 up, 8 in 2024-08-17T17:01:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:33 smithi063 ceph-mon[114756]: pgmap v160: 89 pgs: 89 active+clean; 104 MiB data, 526 MiB used, 715 GiB / 715 GiB avail; 623 KiB/s rd, 796 KiB/s wr, 6 op/s 2024-08-17T17:01:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:33 smithi063 ceph-mon[114756]: osdmap e473: 8 total, 8 up, 8 in 2024-08-17T17:01:34.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:33 smithi063 ceph-mon[109843]: pgmap v160: 89 pgs: 89 active+clean; 104 MiB data, 526 MiB used, 715 GiB / 715 GiB avail; 623 KiB/s rd, 796 KiB/s wr, 6 op/s 2024-08-17T17:01:34.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:33 smithi063 ceph-mon[109843]: osdmap e473: 8 total, 8 up, 8 in 2024-08-17T17:01:34.532 INFO:tasks.rados.rados.0.smithi063.stdout:1618: done (0 left) 2024-08-17T17:01:34.532 INFO:tasks.rados.rados.0.smithi063.stdout:1619: read oid 34 snap -1 2024-08-17T17:01:34.532 INFO:tasks.rados.rados.0.smithi063.stdout:1619: expect (ObjNum 556 snap 172 seq_num 556) 2024-08-17T17:01:34.532 INFO:tasks.rados.rados.0.smithi063.stdout:1620: read oid 35 snap -1 2024-08-17T17:01:34.532 INFO:tasks.rados.rados.0.smithi063.stdout:1620: expect (ObjNum 592 snap 187 seq_num 592) 2024-08-17T17:01:34.532 INFO:tasks.rados.rados.0.smithi063.stdout:1621: read oid 30 snap -1 2024-08-17T17:01:34.532 INFO:tasks.rados.rados.0.smithi063.stdout:1621: expect (ObjNum 507 snap 160 seq_num 507) 2024-08-17T17:01:34.532 INFO:tasks.rados.rados.0.smithi063.stdout:1622: delete oid 14 current snap is 198 2024-08-17T17:01:34.542 INFO:tasks.rados.rados.0.smithi063.stdout:1619: expect (ObjNum 556 snap 172 seq_num 556) 2024-08-17T17:01:34.696 INFO:tasks.rados.rados.0.smithi063.stdout:1621: expect (ObjNum 507 snap 160 seq_num 507) 2024-08-17T17:01:34.816 INFO:tasks.rados.rados.0.smithi063.stdout:1620: expect (ObjNum 592 snap 187 seq_num 592) 2024-08-17T17:01:35.054 INFO:tasks.rados.rados.0.smithi063.stdout:1619: done (3 left) 2024-08-17T17:01:35.055 INFO:tasks.rados.rados.0.smithi063.stdout:1620: done (2 left) 2024-08-17T17:01:35.055 INFO:tasks.rados.rados.0.smithi063.stdout:1621: done (1 left) 2024-08-17T17:01:35.055 INFO:tasks.rados.rados.0.smithi063.stdout:1622: done (0 left) 2024-08-17T17:01:35.055 INFO:tasks.rados.rados.0.smithi063.stdout:1623: delete oid 11 current snap is 198 2024-08-17T17:01:35.057 INFO:tasks.rados.rados.0.smithi063.stdout:1623: done (0 left) 2024-08-17T17:01:35.057 INFO:tasks.rados.rados.0.smithi063.stdout:1624: read oid 13 snap -1 2024-08-17T17:01:35.057 INFO:tasks.rados.rados.0.smithi063.stdout:1624: expect (ObjNum 617 snap 194 seq_num 617) 2024-08-17T17:01:35.057 INFO:tasks.rados.rados.0.smithi063.stdout:1625: read oid 46 snap -1 2024-08-17T17:01:35.057 INFO:tasks.rados.rados.0.smithi063.stdout:1625: expect (ObjNum 510 snap 160 seq_num 510) 2024-08-17T17:01:35.057 INFO:tasks.rados.rados.0.smithi063.stdout:1626: write oid 39 current snap is 198 2024-08-17T17:01:35.057 INFO:tasks.rados.rados.0.smithi063.stdout:1626: seq_num 624 ranges {487568=496322,1431705=519550} 2024-08-17T17:01:35.062 INFO:tasks.rados.rados.0.smithi063.stdout:1626: writing smithi06387500-39 from 487568 to 983890 tid 1 2024-08-17T17:01:35.068 INFO:tasks.rados.rados.0.smithi063.stdout:1626: writing smithi06387500-39 from 1431705 to 1951255 tid 2 2024-08-17T17:01:35.068 INFO:tasks.rados.rados.0.smithi063.stdout:1627: delete oid 22 current snap is 198 2024-08-17T17:01:35.068 INFO:tasks.rados.rados.0.smithi063.stdout:1625: expect (ObjNum 510 snap 160 seq_num 510) 2024-08-17T17:01:35.245 INFO:tasks.rados.rados.0.smithi063.stdout:1624: expect (ObjNum 617 snap 194 seq_num 617) 2024-08-17T17:01:35.324 INFO:tasks.rados.rados.0.smithi063.stdout:1626: finishing write tid 1 to smithi06387500-39 2024-08-17T17:01:35.324 INFO:tasks.rados.rados.0.smithi063.stdout:1626: finishing write tid 2 to smithi06387500-39 2024-08-17T17:01:35.324 INFO:tasks.rados.rados.0.smithi063.stdout:1626: finishing write tid 3 to smithi06387500-39 2024-08-17T17:01:35.324 INFO:tasks.rados.rados.0.smithi063.stdout:1626: finishing write tid 4 to smithi06387500-39 2024-08-17T17:01:35.324 INFO:tasks.rados.rados.0.smithi063.stdout:1626: oid 39 updating version 0 to 360 2024-08-17T17:01:35.325 INFO:tasks.rados.rados.0.smithi063.stdout:1626: oid 39 updating version 360 to 361 2024-08-17T17:01:35.325 INFO:tasks.rados.rados.0.smithi063.stdout:1626: oid 39 updating version 361 to 362 2024-08-17T17:01:35.325 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 362 (ObjNum 624 snap 198 seq_num 624) dirty exists 2024-08-17T17:01:35.325 INFO:tasks.rados.rados.0.smithi063.stdout:1626: left oid 39 (ObjNum 624 snap 198 seq_num 624) 2024-08-17T17:01:35.325 INFO:tasks.rados.rados.0.smithi063.stdout:1624: done (3 left) 2024-08-17T17:01:35.325 INFO:tasks.rados.rados.0.smithi063.stdout:1625: done (2 left) 2024-08-17T17:01:35.325 INFO:tasks.rados.rados.0.smithi063.stdout:1626: done (1 left) 2024-08-17T17:01:35.325 INFO:tasks.rados.rados.0.smithi063.stdout:1627: done (0 left) 2024-08-17T17:01:35.325 INFO:tasks.rados.rados.0.smithi063.stdout:1628: read oid 36 snap 197 2024-08-17T17:01:35.325 INFO:tasks.rados.rados.0.smithi063.stdout:1628: expect deleted 2024-08-17T17:01:35.325 INFO:tasks.rados.rados.0.smithi063.stdout:1629: read oid 50 snap -1 2024-08-17T17:01:35.325 INFO:tasks.rados.rados.0.smithi063.stdout:1629: expect (ObjNum 564 snap 173 seq_num 564) 2024-08-17T17:01:35.325 INFO:tasks.rados.rados.0.smithi063.stdout:1630: snap_create 2024-08-17T17:01:35.329 INFO:tasks.rados.rados.0.smithi063.stdout:1629: expect (ObjNum 564 snap 173 seq_num 564) 2024-08-17T17:01:35.560 INFO:tasks.rados.rados.0.smithi063.stdout:1628: done (2 left) 2024-08-17T17:01:35.561 INFO:tasks.rados.rados.0.smithi063.stdout:1629: done (1 left) 2024-08-17T17:01:35.561 INFO:tasks.rados.rados.0.smithi063.stdout:1630: done (0 left) 2024-08-17T17:01:35.561 INFO:tasks.rados.rados.0.smithi063.stdout:1631: read oid 48 snap -1 2024-08-17T17:01:35.561 INFO:tasks.rados.rados.0.smithi063.stdout:1631: expect deleted 2024-08-17T17:01:35.561 INFO:tasks.rados.rados.0.smithi063.stdout:1632: write oid 22 current snap is 199 2024-08-17T17:01:35.561 INFO:tasks.rados.rados.0.smithi063.stdout:1632: seq_num 626 ranges {521182=646820,1948669=523219} 2024-08-17T17:01:35.567 INFO:tasks.rados.rados.0.smithi063.stdout:1632: writing smithi06387500-22 from 521182 to 1168002 tid 1 2024-08-17T17:01:35.574 INFO:tasks.rados.rados.0.smithi063.stdout:1632: writing smithi06387500-22 from 1948669 to 2471888 tid 2 2024-08-17T17:01:35.574 INFO:tasks.rados.rados.0.smithi063.stdout:1633: snap_remove snap 197 2024-08-17T17:01:35.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:35 smithi063 ceph-mon[114756]: pgmap v163: 89 pgs: 89 active+clean; 104 MiB data, 526 MiB used, 715 GiB / 715 GiB avail; 623 KiB/s rd, 796 KiB/s wr, 6 op/s 2024-08-17T17:01:35.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:35 smithi063 ceph-mon[114756]: osdmap e474: 8 total, 8 up, 8 in 2024-08-17T17:01:35.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:35 smithi063 ceph-mon[109843]: pgmap v163: 89 pgs: 89 active+clean; 104 MiB data, 526 MiB used, 715 GiB / 715 GiB avail; 623 KiB/s rd, 796 KiB/s wr, 6 op/s 2024-08-17T17:01:35.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:35 smithi063 ceph-mon[109843]: osdmap e474: 8 total, 8 up, 8 in 2024-08-17T17:01:35.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:35 smithi162 ceph-mon[34633]: pgmap v163: 89 pgs: 89 active+clean; 104 MiB data, 526 MiB used, 715 GiB / 715 GiB avail; 623 KiB/s rd, 796 KiB/s wr, 6 op/s 2024-08-17T17:01:35.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:35 smithi162 ceph-mon[34633]: osdmap e474: 8 total, 8 up, 8 in 2024-08-17T17:01:36.568 INFO:tasks.rados.rados.0.smithi063.stdout:1633: done (2 left) 2024-08-17T17:01:36.568 INFO:tasks.rados.rados.0.smithi063.stdout:1634: read oid 28 snap -1 2024-08-17T17:01:36.568 INFO:tasks.rados.rados.0.smithi063.stdout:1634: expect deleted 2024-08-17T17:01:36.568 INFO:tasks.rados.rados.0.smithi063.stdout:1635: snap_remove snap 198 2024-08-17T17:01:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:36 smithi162 ceph-mon[34633]: osdmap e475: 8 total, 8 up, 8 in 2024-08-17T17:01:37.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:36 smithi063 ceph-mon[109843]: osdmap e475: 8 total, 8 up, 8 in 2024-08-17T17:01:37.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:36 smithi063 ceph-mon[114756]: osdmap e475: 8 total, 8 up, 8 in 2024-08-17T17:01:37.569 INFO:tasks.rados.rados.0.smithi063.stdout:1635: done (3 left) 2024-08-17T17:01:37.569 INFO:tasks.rados.rados.0.smithi063.stdout:1636: delete oid 41 current snap is 199 2024-08-17T17:01:37.569 INFO:tasks.rados.rados.0.smithi063.stdout:1632: finishing write tid 1 to smithi06387500-22 2024-08-17T17:01:37.569 INFO:tasks.rados.rados.0.smithi063.stdout:1632: finishing write tid 2 to smithi06387500-22 2024-08-17T17:01:37.569 INFO:tasks.rados.rados.0.smithi063.stdout:1632: finishing write tid 3 to smithi06387500-22 2024-08-17T17:01:37.569 INFO:tasks.rados.rados.0.smithi063.stdout:1632: finishing write tid 4 to smithi06387500-22 2024-08-17T17:01:37.569 INFO:tasks.rados.rados.0.smithi063.stdout:1632: oid 22 updating version 0 to 387 2024-08-17T17:01:37.569 INFO:tasks.rados.rados.0.smithi063.stdout:1632: oid 22 version 387 is already newer than 386 2024-08-17T17:01:37.569 INFO:tasks.rados.rados.0.smithi063.stdout:1632: oid 22 updating version 387 to 388 2024-08-17T17:01:37.569 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 388 (ObjNum 626 snap 199 seq_num 626) dirty exists 2024-08-17T17:01:37.569 INFO:tasks.rados.rados.0.smithi063.stdout:1632: left oid 22 (ObjNum 626 snap 199 seq_num 626) 2024-08-17T17:01:37.572 INFO:tasks.rados.rados.0.smithi063.stdout:1631: done (3 left) 2024-08-17T17:01:37.572 INFO:tasks.rados.rados.0.smithi063.stdout:1632: done (2 left) 2024-08-17T17:01:37.573 INFO:tasks.rados.rados.0.smithi063.stdout:1634: done (1 left) 2024-08-17T17:01:37.573 INFO:tasks.rados.rados.0.smithi063.stdout:1636: done (0 left) 2024-08-17T17:01:37.573 INFO:tasks.rados.rados.0.smithi063.stdout:1637: read oid 37 snap -1 2024-08-17T17:01:37.573 INFO:tasks.rados.rados.0.smithi063.stdout:1637: expect (ObjNum 620 snap 197 seq_num 620) 2024-08-17T17:01:37.573 INFO:tasks.rados.rados.0.smithi063.stdout:1638: snap_create 2024-08-17T17:01:37.583 INFO:tasks.rados.rados.0.smithi063.stdout:1637: expect (ObjNum 620 snap 197 seq_num 620) 2024-08-17T17:01:37.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:37 smithi162 ceph-mon[34633]: pgmap v166: 89 pgs: 89 active+clean; 104 MiB data, 526 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:01:37.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:37 smithi162 ceph-mon[34633]: osdmap e476: 8 total, 8 up, 8 in 2024-08-17T17:01:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:37 smithi063 ceph-mon[109843]: pgmap v166: 89 pgs: 89 active+clean; 104 MiB data, 526 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:01:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:37 smithi063 ceph-mon[109843]: osdmap e476: 8 total, 8 up, 8 in 2024-08-17T17:01:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:37 smithi063 ceph-mon[114756]: pgmap v166: 89 pgs: 89 active+clean; 104 MiB data, 526 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:01:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:37 smithi063 ceph-mon[114756]: osdmap e476: 8 total, 8 up, 8 in 2024-08-17T17:01:38.583 INFO:tasks.rados.rados.0.smithi063.stdout:1637: done (1 left) 2024-08-17T17:01:38.583 INFO:tasks.rados.rados.0.smithi063.stdout:1638: done (0 left) 2024-08-17T17:01:38.584 INFO:tasks.rados.rados.0.smithi063.stdout:1639: rollback oid 21 current snap is 200 2024-08-17T17:01:38.584 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 21 to 199 2024-08-17T17:01:38.584 INFO:tasks.rados.rados.0.smithi063.stdout:1640: rollback oid 48 current snap is 200 2024-08-17T17:01:38.584 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 48 to 199 2024-08-17T17:01:38.584 INFO:tasks.rados.rados.0.smithi063.stdout:1641: snap_create 2024-08-17T17:01:38.584 INFO:tasks.rados.rados.0.smithi063.stdout:1640: finishing rollback tid 1 to smithi06387500-48 2024-08-17T17:01:38.584 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 0 (ObjNum 129 snap 0 seq_num 1027972347) dirty dne 2024-08-17T17:01:38.589 INFO:tasks.rados.rados.0.smithi063.stdout:1639: finishing rollback tid 0 to smithi06387500-21 2024-08-17T17:01:38.589 INFO:tasks.rados.rados.0.smithi063.stdout:1639: finishing rollback tid 1 to smithi06387500-21 2024-08-17T17:01:38.589 INFO:tasks.rados.rados.0.smithi063.stdout:1639: finishing rollback tid 2 to smithi06387500-21 2024-08-17T17:01:38.589 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 1206 (ObjNum 578 snap 182 seq_num 578) dirty exists 2024-08-17T17:01:38.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:38 smithi063 ceph-mon[114756]: osdmap e477: 8 total, 8 up, 8 in 2024-08-17T17:01:38.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:38 smithi063 ceph-mon[109843]: osdmap e477: 8 total, 8 up, 8 in 2024-08-17T17:01:38.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:38 smithi162 ceph-mon[34633]: osdmap e477: 8 total, 8 up, 8 in 2024-08-17T17:01:38.918 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for majority of mons to upgrade' 2024-08-17T17:01:39.185 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:01:39.588 INFO:tasks.rados.rados.0.smithi063.stdout:1639: done (2 left) 2024-08-17T17:01:39.589 INFO:tasks.rados.rados.0.smithi063.stdout:1640: done (1 left) 2024-08-17T17:01:39.589 INFO:tasks.rados.rados.0.smithi063.stdout:1641: done (0 left) 2024-08-17T17:01:39.589 INFO:tasks.rados.rados.0.smithi063.stdout:1642: write oid 40 current snap is 201 2024-08-17T17:01:39.589 INFO:tasks.rados.rados.0.smithi063.stdout:1642: seq_num 628 ranges {554796=797318,1949280=526888} 2024-08-17T17:01:39.595 INFO:tasks.rados.rados.0.smithi063.stdout:1642: writing smithi06387500-40 from 554796 to 1352114 tid 1 2024-08-17T17:01:39.602 INFO:tasks.rados.rados.0.smithi063.stdout:1642: writing smithi06387500-40 from 1949280 to 2476168 tid 2 2024-08-17T17:01:39.602 INFO:tasks.rados.rados.0.smithi063.stdout:1643: rollback oid 8 current snap is 201 2024-08-17T17:01:39.602 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 8 to 199 2024-08-17T17:01:39.602 INFO:tasks.rados.rados.0.smithi063.stdout:1644: write oid 47 current snap is 201 2024-08-17T17:01:39.602 INFO:tasks.rados.rados.0.smithi063.stdout:1644: seq_num 629 ranges {571603=472567,1607762=670546} 2024-08-17T17:01:39.607 INFO:tasks.rados.rados.0.smithi063.stdout:1644: writing smithi06387500-47 from 571603 to 1044170 tid 1 2024-08-17T17:01:39.615 INFO:tasks.rados.rados.0.smithi063.stdout:1644: writing smithi06387500-47 from 1607762 to 2278308 tid 2 2024-08-17T17:01:39.615 INFO:tasks.rados.rados.0.smithi063.stdout:1645: read oid 7 snap -1 2024-08-17T17:01:39.615 INFO:tasks.rados.rados.0.smithi063.stdout:1645: expect (ObjNum 605 snap 192 seq_num 605) 2024-08-17T17:01:39.615 INFO:tasks.rados.rados.0.smithi063.stdout:1646: write oid 13 current snap is 201 2024-08-17T17:01:39.615 INFO:tasks.rados.rados.0.smithi063.stdout:1642: finishing write tid 1 to smithi06387500-40 2024-08-17T17:01:39.615 INFO:tasks.rados.rados.0.smithi063.stdout:1646: seq_num 630 ranges {588410=547816,1666244=414204,2484603=103807} 2024-08-17T17:01:39.621 INFO:tasks.rados.rados.0.smithi063.stdout:1646: writing smithi06387500-13 from 588410 to 1136226 tid 1 2024-08-17T17:01:39.626 INFO:tasks.rados.rados.0.smithi063.stdout:1646: writing smithi06387500-13 from 1666244 to 2080448 tid 2 2024-08-17T17:01:39.629 INFO:tasks.rados.rados.0.smithi063.stdout:1646: writing smithi06387500-13 from 2484603 to 2588410 tid 3 2024-08-17T17:01:39.630 INFO:tasks.rados.rados.0.smithi063.stdout:1647: write oid 27 current snap is 201 2024-08-17T17:01:39.630 INFO:tasks.rados.rados.0.smithi063.stdout:1643: finishing rollback tid 1 to smithi06387500-8 2024-08-17T17:01:39.630 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 0 (ObjNum 129 snap 0 seq_num 1807213216) dirty dne 2024-08-17T17:01:39.630 INFO:tasks.rados.rados.0.smithi063.stdout:1642: finishing write tid 2 to smithi06387500-40 2024-08-17T17:01:39.630 INFO:tasks.rados.rados.0.smithi063.stdout:1642: finishing write tid 3 to smithi06387500-40 2024-08-17T17:01:39.630 INFO:tasks.rados.rados.0.smithi063.stdout:1642: finishing write tid 4 to smithi06387500-40 2024-08-17T17:01:39.630 INFO:tasks.rados.rados.0.smithi063.stdout:1642: oid 40 updating version 0 to 579 2024-08-17T17:01:39.630 INFO:tasks.rados.rados.0.smithi063.stdout:1642: oid 40 updating version 579 to 581 2024-08-17T17:01:39.630 INFO:tasks.rados.rados.0.smithi063.stdout:1642: oid 40 version 581 is already newer than 580 2024-08-17T17:01:39.630 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 581 (ObjNum 628 snap 201 seq_num 628) dirty exists 2024-08-17T17:01:39.630 INFO:tasks.rados.rados.0.smithi063.stdout:1642: left oid 40 (ObjNum 628 snap 201 seq_num 628) 2024-08-17T17:01:39.631 INFO:tasks.rados.rados.0.smithi063.stdout:1644: finishing write tid 1 to smithi06387500-47 2024-08-17T17:01:39.631 INFO:tasks.rados.rados.0.smithi063.stdout:1647: seq_num 631 ranges {605217=739418,2124726=480491} 2024-08-17T17:01:39.636 INFO:tasks.rados.rados.0.smithi063.stdout:1647: writing smithi06387500-27 from 605217 to 1344635 tid 1 2024-08-17T17:01:39.643 INFO:tasks.rados.rados.0.smithi063.stdout:1647: writing smithi06387500-27 from 2124726 to 2605217 tid 2 2024-08-17T17:01:39.644 INFO:tasks.rados.rados.0.smithi063.stdout:1642: done (5 left) 2024-08-17T17:01:39.644 INFO:tasks.rados.rados.0.smithi063.stdout:1643: done (4 left) 2024-08-17T17:01:39.644 INFO:tasks.rados.rados.0.smithi063.stdout:1648: read oid 36 snap -1 2024-08-17T17:01:39.644 INFO:tasks.rados.rados.0.smithi063.stdout:1648: expect deleted 2024-08-17T17:01:39.644 INFO:tasks.rados.rados.0.smithi063.stdout:1649: snap_remove snap 200 2024-08-17T17:01:39.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:39 smithi162 ceph-mon[34633]: pgmap v169: 89 pgs: 89 active+clean; 97 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 534 KiB/s wr, 18 op/s 2024-08-17T17:01:39.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:39 smithi162 ceph-mon[34633]: osdmap e478: 8 total, 8 up, 8 in 2024-08-17T17:01:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:39 smithi063 ceph-mon[114756]: pgmap v169: 89 pgs: 89 active+clean; 97 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 534 KiB/s wr, 18 op/s 2024-08-17T17:01:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:39 smithi063 ceph-mon[114756]: osdmap e478: 8 total, 8 up, 8 in 2024-08-17T17:01:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:39 smithi063 ceph-mon[109843]: pgmap v169: 89 pgs: 89 active+clean; 97 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 534 KiB/s wr, 18 op/s 2024-08-17T17:01:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:39 smithi063 ceph-mon[109843]: osdmap e478: 8 total, 8 up, 8 in 2024-08-17T17:01:40.595 INFO:tasks.rados.rados.0.smithi063.stdout:1649: done (5 left) 2024-08-17T17:01:40.595 INFO:tasks.rados.rados.0.smithi063.stdout:1650: rollback oid 9 current snap is 201 2024-08-17T17:01:40.595 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 9 to 199 2024-08-17T17:01:40.595 INFO:tasks.rados.rados.0.smithi063.stdout:1644: finishing write tid 2 to smithi06387500-47 2024-08-17T17:01:40.595 INFO:tasks.rados.rados.0.smithi063.stdout:1644: finishing write tid 3 to smithi06387500-47 2024-08-17T17:01:40.595 INFO:tasks.rados.rados.0.smithi063.stdout:1644: finishing write tid 4 to smithi06387500-47 2024-08-17T17:01:40.595 INFO:tasks.rados.rados.0.smithi063.stdout:1644: oid 47 updating version 0 to 327 2024-08-17T17:01:40.596 INFO:tasks.rados.rados.0.smithi063.stdout:1644: oid 47 version 327 is already newer than 325 2024-08-17T17:01:40.596 INFO:tasks.rados.rados.0.smithi063.stdout:1644: oid 47 version 327 is already newer than 326 2024-08-17T17:01:40.596 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 327 (ObjNum 629 snap 201 seq_num 629) dirty exists 2024-08-17T17:01:40.596 INFO:tasks.rados.rados.0.smithi063.stdout:1644: left oid 47 (ObjNum 629 snap 201 seq_num 629) 2024-08-17T17:01:40.596 INFO:tasks.rados.rados.0.smithi063.stdout:1645: expect (ObjNum 605 snap 192 seq_num 605) 2024-08-17T17:01:40.665 INFO:tasks.rados.rados.0.smithi063.stdout:1646: finishing write tid 1 to smithi06387500-13 2024-08-17T17:01:40.665 INFO:tasks.rados.rados.0.smithi063.stdout:1646: finishing write tid 2 to smithi06387500-13 2024-08-17T17:01:40.665 INFO:tasks.rados.rados.0.smithi063.stdout:1646: finishing write tid 3 to smithi06387500-13 2024-08-17T17:01:40.666 INFO:tasks.rados.rados.0.smithi063.stdout:1646: finishing write tid 4 to smithi06387500-13 2024-08-17T17:01:40.666 INFO:tasks.rados.rados.0.smithi063.stdout:1646: finishing write tid 5 to smithi06387500-13 2024-08-17T17:01:40.666 INFO:tasks.rados.rados.0.smithi063.stdout:1646: oid 13 updating version 0 to 751 2024-08-17T17:01:40.666 INFO:tasks.rados.rados.0.smithi063.stdout:1646: oid 13 version 751 is already newer than 750 2024-08-17T17:01:40.666 INFO:tasks.rados.rados.0.smithi063.stdout:1646: oid 13 version 751 is already newer than 749 2024-08-17T17:01:40.666 INFO:tasks.rados.rados.0.smithi063.stdout:1646: oid 13 updating version 751 to 752 2024-08-17T17:01:40.666 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 752 (ObjNum 630 snap 201 seq_num 630) dirty exists 2024-08-17T17:01:40.666 INFO:tasks.rados.rados.0.smithi063.stdout:1646: left oid 13 (ObjNum 630 snap 201 seq_num 630) 2024-08-17T17:01:40.666 INFO:tasks.rados.rados.0.smithi063.stdout:1647: finishing write tid 1 to smithi06387500-27 2024-08-17T17:01:40.666 INFO:tasks.rados.rados.0.smithi063.stdout:1644: done (5 left) 2024-08-17T17:01:40.666 INFO:tasks.rados.rados.0.smithi063.stdout:1645: done (4 left) 2024-08-17T17:01:40.666 INFO:tasks.rados.rados.0.smithi063.stdout:1646: done (3 left) 2024-08-17T17:01:40.667 INFO:tasks.rados.rados.0.smithi063.stdout:1651: read oid 18 snap -1 2024-08-17T17:01:40.667 INFO:tasks.rados.rados.0.smithi063.stdout:1651: expect deleted 2024-08-17T17:01:40.667 INFO:tasks.rados.rados.0.smithi063.stdout:1652: snap_create 2024-08-17T17:01:40.667 INFO:tasks.rados.rados.0.smithi063.stdout:1647: finishing write tid 2 to smithi06387500-27 2024-08-17T17:01:40.667 INFO:tasks.rados.rados.0.smithi063.stdout:1647: finishing write tid 3 to smithi06387500-27 2024-08-17T17:01:40.667 INFO:tasks.rados.rados.0.smithi063.stdout:1647: finishing write tid 4 to smithi06387500-27 2024-08-17T17:01:40.667 INFO:tasks.rados.rados.0.smithi063.stdout:1647: oid 27 updating version 0 to 184 2024-08-17T17:01:40.667 INFO:tasks.rados.rados.0.smithi063.stdout:1647: oid 27 updating version 184 to 185 2024-08-17T17:01:40.667 INFO:tasks.rados.rados.0.smithi063.stdout:1647: oid 27 updating version 185 to 186 2024-08-17T17:01:40.667 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 186 (ObjNum 631 snap 201 seq_num 631) dirty exists 2024-08-17T17:01:40.668 INFO:tasks.rados.rados.0.smithi063.stdout:1647: left oid 27 (ObjNum 631 snap 201 seq_num 631) 2024-08-17T17:01:40.668 INFO:tasks.rados.rados.0.smithi063.stdout:1650: finishing rollback tid 1 to smithi06387500-9 2024-08-17T17:01:40.668 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T17:01:40.791 INFO:teuthology.orchestra.run.smithi063.stdout:wait for majority of mons to upgrade 2024-08-17T17:01:40.792 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:40 smithi063 ceph-mon[114756]: osdmap e479: 8 total, 8 up, 8 in 2024-08-17T17:01:40.792 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:40 smithi063 ceph-mon[109843]: osdmap e479: 8 total, 8 up, 8 in 2024-08-17T17:01:40.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:40 smithi162 ceph-mon[34633]: osdmap e479: 8 total, 8 up, 8 in 2024-08-17T17:01:41.471 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-08-17T17:01:41.594 INFO:tasks.rados.rados.0.smithi063.stdout:1647: done (4 left) 2024-08-17T17:01:41.594 INFO:tasks.rados.rados.0.smithi063.stdout:1648: done (3 left) 2024-08-17T17:01:41.594 INFO:tasks.rados.rados.0.smithi063.stdout:1650: done (2 left) 2024-08-17T17:01:41.594 INFO:tasks.rados.rados.0.smithi063.stdout:1651: done (1 left) 2024-08-17T17:01:41.595 INFO:tasks.rados.rados.0.smithi063.stdout:1652: done (0 left) 2024-08-17T17:01:41.595 INFO:tasks.rados.rados.0.smithi063.stdout:1653: delete oid 27 current snap is 202 2024-08-17T17:01:41.599 INFO:tasks.rados.rados.0.smithi063.stdout:1653: done (0 left) 2024-08-17T17:01:41.599 INFO:tasks.rados.rados.0.smithi063.stdout:1654: delete oid 15 current snap is 202 2024-08-17T17:01:41.602 INFO:tasks.rados.rados.0.smithi063.stdout:1654: done (0 left) 2024-08-17T17:01:41.602 INFO:tasks.rados.rados.0.smithi063.stdout:1655: snap_remove snap 201 2024-08-17T17:01:41.697 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:01:41.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:41 smithi063 ceph-mon[114756]: pgmap v172: 89 pgs: 89 active+clean; 97 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 534 KiB/s wr, 18 op/s 2024-08-17T17:01:41.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:41 smithi063 ceph-mon[114756]: osdmap e480: 8 total, 8 up, 8 in 2024-08-17T17:01:41.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:41 smithi063 ceph-mon[109843]: pgmap v172: 89 pgs: 89 active+clean; 97 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 534 KiB/s wr, 18 op/s 2024-08-17T17:01:41.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:41 smithi063 ceph-mon[109843]: osdmap e480: 8 total, 8 up, 8 in 2024-08-17T17:01:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:41 smithi162 ceph-mon[34633]: pgmap v172: 89 pgs: 89 active+clean; 97 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 21 MiB/s rd, 534 KiB/s wr, 18 op/s 2024-08-17T17:01:42.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:41 smithi162 ceph-mon[34633]: osdmap e480: 8 total, 8 up, 8 in 2024-08-17T17:01:42.532 INFO:tasks.rados.rados.0.smithi063.stdout:1655: done (0 left) 2024-08-17T17:01:42.532 INFO:tasks.rados.rados.0.smithi063.stdout:1656: read oid 37 snap 199 2024-08-17T17:01:42.532 INFO:tasks.rados.rados.0.smithi063.stdout:1656: expect (ObjNum 620 snap 197 seq_num 620) 2024-08-17T17:01:42.532 INFO:tasks.rados.rados.0.smithi063.stdout:1657: rollback oid 14 current snap is 202 2024-08-17T17:01:42.532 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 14 to 199 2024-08-17T17:01:42.532 INFO:tasks.rados.rados.0.smithi063.stdout:1658: write oid 30 current snap is 202 2024-08-17T17:01:42.532 INFO:tasks.rados.rados.0.smithi063.stdout:1658: seq_num 634 ranges {655638=565165,1900172=421542} 2024-08-17T17:01:42.538 INFO:tasks.rados.rados.0.smithi063.stdout:1658: writing smithi06387500-30 from 655638 to 1220803 tid 1 2024-08-17T17:01:42.545 INFO:tasks.rados.rados.0.smithi063.stdout:1658: writing smithi06387500-30 from 1900172 to 2321714 tid 2 2024-08-17T17:01:42.545 INFO:tasks.rados.rados.0.smithi063.stdout:1659: delete oid 9 current snap is 202 2024-08-17T17:01:42.545 INFO:tasks.rados.rados.0.smithi063.stdout:1656: expect (ObjNum 620 snap 197 seq_num 620) 2024-08-17T17:01:42.629 INFO:tasks.rados.rados.0.smithi063.stdout:1657: finishing rollback tid 1 to smithi06387500-14 2024-08-17T17:01:42.630 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 0 (ObjNum 49 snap 0 seq_num 1132768066) dirty dne 2024-08-17T17:01:42.630 INFO:tasks.rados.rados.0.smithi063.stdout:1658: finishing write tid 1 to smithi06387500-30 2024-08-17T17:01:42.630 INFO:tasks.rados.rados.0.smithi063.stdout:1658: finishing write tid 2 to smithi06387500-30 2024-08-17T17:01:42.630 INFO:tasks.rados.rados.0.smithi063.stdout:1658: finishing write tid 3 to smithi06387500-30 2024-08-17T17:01:42.630 INFO:tasks.rados.rados.0.smithi063.stdout:1658: finishing write tid 4 to smithi06387500-30 2024-08-17T17:01:42.630 INFO:tasks.rados.rados.0.smithi063.stdout:1658: oid 30 updating version 0 to 765 2024-08-17T17:01:42.630 INFO:tasks.rados.rados.0.smithi063.stdout:1658: oid 30 updating version 765 to 766 2024-08-17T17:01:42.630 INFO:tasks.rados.rados.0.smithi063.stdout:1658: oid 30 updating version 766 to 767 2024-08-17T17:01:42.630 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 767 (ObjNum 634 snap 202 seq_num 634) dirty exists 2024-08-17T17:01:42.630 INFO:tasks.rados.rados.0.smithi063.stdout:1658: left oid 30 (ObjNum 634 snap 202 seq_num 634) 2024-08-17T17:01:42.630 INFO:tasks.rados.rados.0.smithi063.stdout:1656: done (3 left) 2024-08-17T17:01:42.630 INFO:tasks.rados.rados.0.smithi063.stdout:1657: done (2 left) 2024-08-17T17:01:42.631 INFO:tasks.rados.rados.0.smithi063.stdout:1658: done (1 left) 2024-08-17T17:01:42.631 INFO:tasks.rados.rados.0.smithi063.stdout:1659: done (0 left) 2024-08-17T17:01:42.631 INFO:tasks.rados.rados.0.smithi063.stdout:1660: snap_create 2024-08-17T17:01:42.959 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:42 smithi063 ceph-mon[109843]: osdmap e481: 8 total, 8 up, 8 in 2024-08-17T17:01:42.960 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:42 smithi063 ceph-mon[109843]: osdmap e482: 8 total, 8 up, 8 in 2024-08-17T17:01:42.960 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:42 smithi063 ceph-mon[114756]: osdmap e481: 8 total, 8 up, 8 in 2024-08-17T17:01:42.960 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:42 smithi063 ceph-mon[114756]: osdmap e482: 8 total, 8 up, 8 in 2024-08-17T17:01:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:42 smithi162 ceph-mon[34633]: osdmap e481: 8 total, 8 up, 8 in 2024-08-17T17:01:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:42 smithi162 ceph-mon[34633]: osdmap e482: 8 total, 8 up, 8 in 2024-08-17T17:01:43.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:01:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:01:43] "GET /metrics HTTP/1.1" 200 36206 "" "Prometheus/2.51.0" 2024-08-17T17:01:43.616 INFO:tasks.rados.rados.0.smithi063.stdout:1660: done (0 left) 2024-08-17T17:01:43.616 INFO:tasks.rados.rados.0.smithi063.stdout:1661: write oid 33 current snap is 203 2024-08-17T17:01:43.616 INFO:tasks.rados.rados.0.smithi063.stdout:1661: seq_num 636 ranges {689252=715663,1900783=425211} 2024-08-17T17:01:43.623 INFO:tasks.rados.rados.0.smithi063.stdout:1661: writing smithi06387500-33 from 689252 to 1404915 tid 1 2024-08-17T17:01:43.629 INFO:tasks.rados.rados.0.smithi063.stdout:1661: writing smithi06387500-33 from 1900783 to 2325994 tid 2 2024-08-17T17:01:43.629 INFO:tasks.rados.rados.0.smithi063.stdout:1662: delete oid 15 current snap is 203 2024-08-17T17:01:43.632 INFO:tasks.rados.rados.0.smithi063.stdout:1661: finishing write tid 1 to smithi06387500-33 2024-08-17T17:01:43.632 INFO:tasks.rados.rados.0.smithi063.stdout:1661: finishing write tid 2 to smithi06387500-33 2024-08-17T17:01:43.633 INFO:tasks.rados.rados.0.smithi063.stdout:1661: finishing write tid 3 to smithi06387500-33 2024-08-17T17:01:43.633 INFO:tasks.rados.rados.0.smithi063.stdout:1661: finishing write tid 4 to smithi06387500-33 2024-08-17T17:01:43.633 INFO:tasks.rados.rados.0.smithi063.stdout:1661: oid 33 updating version 0 to 526 2024-08-17T17:01:43.633 INFO:tasks.rados.rados.0.smithi063.stdout:1661: oid 33 updating version 526 to 527 2024-08-17T17:01:43.633 INFO:tasks.rados.rados.0.smithi063.stdout:1661: oid 33 updating version 527 to 528 2024-08-17T17:01:43.633 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 528 (ObjNum 636 snap 203 seq_num 636) dirty exists 2024-08-17T17:01:43.633 INFO:tasks.rados.rados.0.smithi063.stdout:1661: left oid 33 (ObjNum 636 snap 203 seq_num 636) 2024-08-17T17:01:43.633 INFO:tasks.rados.rados.0.smithi063.stdout:1661: done (1 left) 2024-08-17T17:01:43.633 INFO:tasks.rados.rados.0.smithi063.stdout:1662: done (0 left) 2024-08-17T17:01:43.633 INFO:tasks.rados.rados.0.smithi063.stdout:1663: write oid 41 current snap is 203 2024-08-17T17:01:43.633 INFO:tasks.rados.rados.0.smithi063.stdout:1663: seq_num 638 ranges {722866=466161,1617747=428880,2587634=135232} 2024-08-17T17:01:43.639 INFO:tasks.rados.rados.0.smithi063.stdout:1663: writing smithi06387500-41 from 722866 to 1189027 tid 1 2024-08-17T17:01:43.644 INFO:tasks.rados.rados.0.smithi063.stdout:1663: writing smithi06387500-41 from 1617747 to 2046627 tid 2 2024-08-17T17:01:43.648 INFO:tasks.rados.rados.0.smithi063.stdout:1663: writing smithi06387500-41 from 2587634 to 2722866 tid 3 2024-08-17T17:01:43.648 INFO:tasks.rados.rados.0.smithi063.stdout:1664: read oid 15 snap -1 2024-08-17T17:01:43.648 INFO:tasks.rados.rados.0.smithi063.stdout:1664: expect deleted 2024-08-17T17:01:43.648 INFO:tasks.rados.rados.0.smithi063.stdout:1665: write oid 10 current snap is 203 2024-08-17T17:01:43.648 INFO:tasks.rados.rados.0.smithi063.stdout:1663: finishing write tid 1 to smithi06387500-41 2024-08-17T17:01:43.648 INFO:tasks.rados.rados.0.smithi063.stdout:1663: finishing write tid 2 to smithi06387500-41 2024-08-17T17:01:43.648 INFO:tasks.rados.rados.0.smithi063.stdout:1665: seq_num 639 ranges {739673=657763,2076229=663444} 2024-08-17T17:01:43.655 INFO:tasks.rados.rados.0.smithi063.stdout:1665: writing smithi06387500-10 from 739673 to 1397436 tid 1 2024-08-17T17:01:43.663 INFO:tasks.rados.rados.0.smithi063.stdout:1665: writing smithi06387500-10 from 2076229 to 2739673 tid 2 2024-08-17T17:01:43.663 INFO:tasks.rados.rados.0.smithi063.stdout:1666: delete oid 7 current snap is 203 2024-08-17T17:01:43.663 INFO:tasks.rados.rados.0.smithi063.stdout:1663: finishing write tid 3 to smithi06387500-41 2024-08-17T17:01:43.663 INFO:tasks.rados.rados.0.smithi063.stdout:1663: finishing write tid 4 to smithi06387500-41 2024-08-17T17:01:43.663 INFO:tasks.rados.rados.0.smithi063.stdout:1663: finishing write tid 5 to smithi06387500-41 2024-08-17T17:01:43.663 INFO:tasks.rados.rados.0.smithi063.stdout:1663: oid 41 updating version 0 to 151 2024-08-17T17:01:43.663 INFO:tasks.rados.rados.0.smithi063.stdout:1663: oid 41 version 151 is already newer than 150 2024-08-17T17:01:43.663 INFO:tasks.rados.rados.0.smithi063.stdout:1663: oid 41 updating version 151 to 152 2024-08-17T17:01:43.663 INFO:tasks.rados.rados.0.smithi063.stdout:1663: oid 41 updating version 152 to 153 2024-08-17T17:01:43.663 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 153 (ObjNum 638 snap 203 seq_num 638) dirty exists 2024-08-17T17:01:43.663 INFO:tasks.rados.rados.0.smithi063.stdout:1663: left oid 41 (ObjNum 638 snap 203 seq_num 638) 2024-08-17T17:01:43.663 INFO:tasks.rados.rados.0.smithi063.stdout:1665: finishing write tid 1 to smithi06387500-10 2024-08-17T17:01:43.665 INFO:tasks.rados.rados.0.smithi063.stdout:1663: done (3 left) 2024-08-17T17:01:43.665 INFO:tasks.rados.rados.0.smithi063.stdout:1664: done (2 left) 2024-08-17T17:01:43.665 INFO:tasks.rados.rados.0.smithi063.stdout:1666: done (1 left) 2024-08-17T17:01:43.665 INFO:tasks.rados.rados.0.smithi063.stdout:1667: snap_remove snap 202 2024-08-17T17:01:44.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:43 smithi063 ceph-mon[109843]: pgmap v175: 89 pgs: 89 active+clean; 100 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-08-17T17:01:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:43 smithi063 ceph-mon[114756]: pgmap v175: 89 pgs: 89 active+clean; 100 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-08-17T17:01:44.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:43 smithi162 ceph-mon[34633]: pgmap v175: 89 pgs: 89 active+clean; 100 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-08-17T17:01:44.622 INFO:tasks.rados.rados.0.smithi063.stdout:1667: done (1 left) 2024-08-17T17:01:44.622 INFO:tasks.rados.rados.0.smithi063.stdout:1668: rollback oid 2 current snap is 203 2024-08-17T17:01:44.622 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 2 to 199 2024-08-17T17:01:44.622 INFO:tasks.rados.rados.0.smithi063.stdout:1665: finishing write tid 2 to smithi06387500-10 2024-08-17T17:01:44.622 INFO:tasks.rados.rados.0.smithi063.stdout:1665: finishing write tid 3 to smithi06387500-10 2024-08-17T17:01:44.622 INFO:tasks.rados.rados.0.smithi063.stdout:1665: finishing write tid 4 to smithi06387500-10 2024-08-17T17:01:44.622 INFO:tasks.rados.rados.0.smithi063.stdout:1665: oid 10 updating version 0 to 584 2024-08-17T17:01:44.622 INFO:tasks.rados.rados.0.smithi063.stdout:1665: oid 10 version 584 is already newer than 582 2024-08-17T17:01:44.622 INFO:tasks.rados.rados.0.smithi063.stdout:1665: oid 10 version 584 is already newer than 583 2024-08-17T17:01:44.622 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 584 (ObjNum 639 snap 203 seq_num 639) dirty exists 2024-08-17T17:01:44.623 INFO:tasks.rados.rados.0.smithi063.stdout:1665: left oid 10 (ObjNum 639 snap 203 seq_num 639) 2024-08-17T17:01:44.623 INFO:tasks.rados.rados.0.smithi063.stdout:1665: done (1 left) 2024-08-17T17:01:44.623 INFO:tasks.rados.rados.0.smithi063.stdout:1669: read oid 11 snap -1 2024-08-17T17:01:44.623 INFO:tasks.rados.rados.0.smithi063.stdout:1669: expect deleted 2024-08-17T17:01:44.623 INFO:tasks.rados.rados.0.smithi063.stdout:1670: delete oid 44 current snap is 203 2024-08-17T17:01:44.624 INFO:tasks.rados.rados.0.smithi063.stdout:1668: finishing rollback tid 1 to smithi06387500-2 2024-08-17T17:01:44.624 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 0 (ObjNum 273 snap 0 seq_num 1806966224) dirty dne 2024-08-17T17:01:44.626 INFO:tasks.rados.rados.0.smithi063.stdout:1668: done (2 left) 2024-08-17T17:01:44.626 INFO:tasks.rados.rados.0.smithi063.stdout:1669: done (1 left) 2024-08-17T17:01:44.626 INFO:tasks.rados.rados.0.smithi063.stdout:1670: done (0 left) 2024-08-17T17:01:44.627 INFO:tasks.rados.rados.0.smithi063.stdout:1671: snap_create 2024-08-17T17:01:44.751 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:44 smithi063 ceph-mon[114756]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:44.751 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:44 smithi063 ceph-mon[114756]: osdmap e483: 8 total, 8 up, 8 in 2024-08-17T17:01:44.751 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:44 smithi063 ceph-mon[114756]: pgmap v178: 89 pgs: 89 active+clean; 100 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-08-17T17:01:44.751 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:44 smithi063 ceph-mon[109843]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:44.751 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:44 smithi063 ceph-mon[109843]: osdmap e483: 8 total, 8 up, 8 in 2024-08-17T17:01:44.751 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:44 smithi063 ceph-mon[109843]: pgmap v178: 89 pgs: 89 active+clean; 100 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-08-17T17:01:45.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:44 smithi162 ceph-mon[34633]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:45.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:44 smithi162 ceph-mon[34633]: osdmap e483: 8 total, 8 up, 8 in 2024-08-17T17:01:45.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:44 smithi162 ceph-mon[34633]: pgmap v178: 89 pgs: 89 active+clean; 100 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.1 MiB/s wr, 11 op/s 2024-08-17T17:01:45.375 INFO:teuthology.orchestra.run.smithi063.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:01:45.623 INFO:tasks.rados.rados.0.smithi063.stdout:1671: done (0 left) 2024-08-17T17:01:45.623 INFO:tasks.rados.rados.0.smithi063.stdout:1672: snap_create 2024-08-17T17:01:45.959 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[114756]: osdmap e484: 8 total, 8 up, 8 in 2024-08-17T17:01:45.959 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[114756]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:01:45.959 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:45.959 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:45.959 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:45.959 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:45.959 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:45.959 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[114756]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:01:45.960 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[109843]: osdmap e484: 8 total, 8 up, 8 in 2024-08-17T17:01:45.960 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[109843]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:01:45.960 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:45.960 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:45.960 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:45.960 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:45.960 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:45.960 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:45 smithi063 ceph-mon[109843]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:01:45.979 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-17T17:01:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:45 smithi162 ceph-mon[34633]: osdmap e484: 8 total, 8 up, 8 in 2024-08-17T17:01:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:45 smithi162 ceph-mon[34633]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:01:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:45 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:45 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:45 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:45 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-08-17T17:01:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:45 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:45 smithi162 ceph-mon[34633]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:01:46.199 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:01:46.636 INFO:tasks.rados.rados.0.smithi063.stdout:1672: done (0 left) 2024-08-17T17:01:46.636 INFO:tasks.rados.rados.0.smithi063.stdout:1673: snap_remove snap 199 2024-08-17T17:01:46.882 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:46 smithi063 ceph-mon[109843]: osdmap e485: 8 total, 8 up, 8 in 2024-08-17T17:01:46.882 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:46 smithi063 ceph-mon[109843]: pgmap v181: 89 pgs: 89 active+clean; 100 MiB data, 514 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:01:46.882 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:46.882 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:46 smithi063 ceph-mon[114756]: osdmap e485: 8 total, 8 up, 8 in 2024-08-17T17:01:46.882 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:46 smithi063 ceph-mon[114756]: pgmap v181: 89 pgs: 89 active+clean; 100 MiB data, 514 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:01:46.882 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:46 smithi162 ceph-mon[34633]: osdmap e485: 8 total, 8 up, 8 in 2024-08-17T17:01:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:46 smithi162 ceph-mon[34633]: pgmap v181: 89 pgs: 89 active+clean; 100 MiB data, 514 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:01:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:46 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-08-17T17:01:47.640 INFO:tasks.rados.rados.0.smithi063.stdout:1673: done (0 left) 2024-08-17T17:01:47.641 INFO:tasks.rados.rados.0.smithi063.stdout:1674: snap_remove snap 203 2024-08-17T17:01:47.880 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:47 smithi063 ceph-mon[109843]: osdmap e486: 8 total, 8 up, 8 in 2024-08-17T17:01:47.880 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:47 smithi063 ceph-mon[114756]: osdmap e486: 8 total, 8 up, 8 in 2024-08-17T17:01:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:47 smithi162 ceph-mon[34633]: osdmap e486: 8 total, 8 up, 8 in 2024-08-17T17:01:48.653 INFO:tasks.rados.rados.0.smithi063.stdout:1674: done (0 left) 2024-08-17T17:01:48.653 INFO:tasks.rados.rados.0.smithi063.stdout:1675: write oid 25 current snap is 205 2024-08-17T17:01:48.653 INFO:tasks.rados.rados.0.smithi063.stdout:1675: seq_num 642 ranges {790094=483510,1851675=436218} 2024-08-17T17:01:48.660 INFO:tasks.rados.rados.0.smithi063.stdout:1675: writing smithi06387500-25 from 790094 to 1273604 tid 1 2024-08-17T17:01:48.666 INFO:tasks.rados.rados.0.smithi063.stdout:1675: writing smithi06387500-25 from 1851675 to 2287893 tid 2 2024-08-17T17:01:48.666 INFO:tasks.rados.rados.0.smithi063.stdout:1676: write oid 31 current snap is 205 2024-08-17T17:01:48.666 INFO:tasks.rados.rados.0.smithi063.stdout:1675: finishing write tid 1 to smithi06387500-25 2024-08-17T17:01:48.666 INFO:tasks.rados.rados.0.smithi063.stdout:1676: seq_num 643 ranges {406901=558759,1393804=579876,2608396=198505} 2024-08-17T17:01:48.671 INFO:tasks.rados.rados.0.smithi063.stdout:1676: writing smithi06387500-31 from 406901 to 965660 tid 1 2024-08-17T17:01:48.677 INFO:tasks.rados.rados.0.smithi063.stdout:1676: writing smithi06387500-31 from 1393804 to 1973680 tid 2 2024-08-17T17:01:48.683 INFO:tasks.rados.rados.0.smithi063.stdout:1676: writing smithi06387500-31 from 2608396 to 2806901 tid 3 2024-08-17T17:01:48.683 INFO:tasks.rados.rados.0.smithi063.stdout:1677: snap_create 2024-08-17T17:01:48.683 INFO:tasks.rados.rados.0.smithi063.stdout:1675: finishing write tid 2 to smithi06387500-25 2024-08-17T17:01:48.683 INFO:tasks.rados.rados.0.smithi063.stdout:1675: finishing write tid 3 to smithi06387500-25 2024-08-17T17:01:48.683 INFO:tasks.rados.rados.0.smithi063.stdout:1675: finishing write tid 4 to smithi06387500-25 2024-08-17T17:01:48.683 INFO:tasks.rados.rados.0.smithi063.stdout:1675: oid 25 updating version 0 to 1647 2024-08-17T17:01:48.683 INFO:tasks.rados.rados.0.smithi063.stdout:1675: oid 25 updating version 1647 to 1648 2024-08-17T17:01:48.683 INFO:tasks.rados.rados.0.smithi063.stdout:1675: oid 25 version 1648 is already newer than 1646 2024-08-17T17:01:48.683 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 1648 (ObjNum 642 snap 205 seq_num 642) dirty exists 2024-08-17T17:01:48.684 INFO:tasks.rados.rados.0.smithi063.stdout:1675: left oid 25 (ObjNum 642 snap 205 seq_num 642) 2024-08-17T17:01:48.684 INFO:tasks.rados.rados.0.smithi063.stdout:1676: finishing write tid 1 to smithi06387500-31 2024-08-17T17:01:48.684 INFO:tasks.rados.rados.0.smithi063.stdout:1676: finishing write tid 2 to smithi06387500-31 2024-08-17T17:01:48.686 INFO:tasks.rados.rados.0.smithi063.stdout:1676: finishing write tid 3 to smithi06387500-31 2024-08-17T17:01:48.686 INFO:tasks.rados.rados.0.smithi063.stdout:1676: finishing write tid 4 to smithi06387500-31 2024-08-17T17:01:48.686 INFO:tasks.rados.rados.0.smithi063.stdout:1676: finishing write tid 5 to smithi06387500-31 2024-08-17T17:01:48.686 INFO:tasks.rados.rados.0.smithi063.stdout:1676: oid 31 updating version 0 to 1211 2024-08-17T17:01:48.686 INFO:tasks.rados.rados.0.smithi063.stdout:1676: oid 31 updating version 1211 to 1213 2024-08-17T17:01:48.686 INFO:tasks.rados.rados.0.smithi063.stdout:1676: oid 31 version 1213 is already newer than 1210 2024-08-17T17:01:48.686 INFO:tasks.rados.rados.0.smithi063.stdout:1676: oid 31 version 1213 is already newer than 1212 2024-08-17T17:01:48.686 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 1213 (ObjNum 643 snap 205 seq_num 643) dirty exists 2024-08-17T17:01:48.686 INFO:tasks.rados.rados.0.smithi063.stdout:1676: left oid 31 (ObjNum 643 snap 205 seq_num 643) 2024-08-17T17:01:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:48 smithi063 ceph-mon[114756]: osdmap e487: 8 total, 8 up, 8 in 2024-08-17T17:01:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:48 smithi063 ceph-mon[114756]: pgmap v184: 89 pgs: 89 active+clean; 106 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-08-17T17:01:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:48 smithi063 ceph-mon[109843]: osdmap e487: 8 total, 8 up, 8 in 2024-08-17T17:01:49.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:48 smithi063 ceph-mon[109843]: pgmap v184: 89 pgs: 89 active+clean; 106 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-08-17T17:01:49.090 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T17:01:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:48 smithi162 ceph-mon[34633]: osdmap e487: 8 total, 8 up, 8 in 2024-08-17T17:01:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:48 smithi162 ceph-mon[34633]: pgmap v184: 89 pgs: 89 active+clean; 106 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-08-17T17:01:49.650 INFO:tasks.rados.rados.0.smithi063.stdout:1675: done (2 left) 2024-08-17T17:01:49.651 INFO:tasks.rados.rados.0.smithi063.stdout:1676: done (1 left) 2024-08-17T17:01:49.651 INFO:tasks.rados.rados.0.smithi063.stdout:1677: done (0 left) 2024-08-17T17:01:49.651 INFO:tasks.rados.rados.0.smithi063.stdout:1678: snap_remove snap 204 2024-08-17T17:01:49.678 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:01:49.678 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (3m) 36s ago 9m 17.5M - 0.27.0 11f11916f8cd 59f4018b91ca 2024-08-17T17:01:49.678 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (2m) 36s ago 9m 71.3M - c8b91775d855 581e03e654a8 2024-08-17T17:01:49.678 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (8m) 36s ago 8m 19.9M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T17:01:49.678 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (7m) 36s ago 7m 18.1M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T17:01:49.678 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,9283,8765 running (4m) 36s ago 13m 454M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb da48034a2895 2024-08-17T17:01:49.678 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:8443,9283,8765 running (4m) 36s ago 14m 507M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 943e639cede6 2024-08-17T17:01:49.679 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (2m) 36s ago 14m 47.1M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb f0f20fcac956 2024-08-17T17:01:49.679 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (13m) 36s ago 13m 58.7M 2048M 18.2.4-719-gced649f8 c870753d67c2 d4ecbe65a0c8 2024-08-17T17:01:49.679 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (109s) 36s ago 13m 33.2M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb c484af026acf 2024-08-17T17:01:49.679 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 running (4m) 36s ago 8m 11.1M - 1.7.0 72c9c2088986 34f55f5de2aa 2024-08-17T17:01:49.679 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (4m) 36s ago 8m 12.2M - 1.7.0 72c9c2088986 5bb162d0f1a9 2024-08-17T17:01:49.679 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (12m) 36s ago 12m 105M 2480M 18.2.4-719-gced649f8 c870753d67c2 f3b910ed49b8 2024-08-17T17:01:49.679 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (12m) 36s ago 12m 84.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 5d6fdb6000ad 2024-08-17T17:01:49.679 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (12m) 36s ago 12m 95.2M 2480M 18.2.4-719-gced649f8 c870753d67c2 a3cbe8edb49a 2024-08-17T17:01:49.679 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (11m) 36s ago 11m 83.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T17:01:49.679 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (11m) 36s ago 11m 108M 2480M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T17:01:49.679 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (10m) 36s ago 10m 99.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T17:01:49.679 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (10m) 36s ago 10m 76.7M 2480M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T17:01:49.679 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (10m) 36s ago 10m 83.0M 2480M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T17:01:49.679 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (3m) 36s ago 9m 45.7M - 2.51.0 1d3b7f56885b dfe66b2c5568 2024-08-17T17:01:50.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[109843]: osdmap e488: 8 total, 8 up, 8 in 2024-08-17T17:01:50.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:50.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[109843]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:01:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[109843]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:01:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:01:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all mgr 2024-08-17T17:01:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:01:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-08-17T17:01:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[109843]: Upgrade: It appears safe to stop mon.b 2024-08-17T17:01:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[109843]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[109843]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:50.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[114756]: osdmap e488: 8 total, 8 up, 8 in 2024-08-17T17:01:50.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:50.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[114756]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:01:50.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[114756]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:01:50.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:50.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:01:50.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all mgr 2024-08-17T17:01:50.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:50.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:01:50.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-08-17T17:01:50.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[114756]: Upgrade: It appears safe to stop mon.b 2024-08-17T17:01:50.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[114756]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:50.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:49 smithi063 ceph-mon[114756]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:50.032 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:01:50.032 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T17:01:50.032 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 1, 2024-08-17T17:01:50.032 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:01:50.032 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:01:50.032 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T17:01:50.032 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:01:50.032 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:01:50.032 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T17:01:50.032 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:01:50.032 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:01:50.033 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T17:01:50.033 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:01:50.033 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:01:50.033 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T17:01:50.033 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 11, 2024-08-17T17:01:50.033 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 4 2024-08-17T17:01:50.033 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:01:50.033 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:01:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:49 smithi162 ceph-mon[34633]: osdmap e488: 8 total, 8 up, 8 in 2024-08-17T17:01:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:49 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:49 smithi162 ceph-mon[34633]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:01:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:49 smithi162 ceph-mon[34633]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:01:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:49 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-08-17T17:01:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:49 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-08-17T17:01:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:49 smithi162 ceph-mon[34633]: Upgrade: Setting container_image for all mgr 2024-08-17T17:01:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:49 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:49 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-08-17T17:01:50.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:49 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-08-17T17:01:50.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:49 smithi162 ceph-mon[34633]: Upgrade: It appears safe to stop mon.b 2024-08-17T17:01:50.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:49 smithi162 ceph-mon[34633]: from='client.34185 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:50.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:49 smithi162 ceph-mon[34633]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:50.331 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:01:50.331 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:01:50.331 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-08-17T17:01:50.331 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-08-17T17:01:50.331 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-08-17T17:01:50.331 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "2/3 daemons upgraded", 2024-08-17T17:01:50.331 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading mon daemons", 2024-08-17T17:01:50.331 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T17:01:50.331 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:01:50.657 INFO:tasks.rados.rados.0.smithi063.stdout:1678: done (0 left) 2024-08-17T17:01:50.657 INFO:tasks.rados.rados.0.smithi063.stdout:1679: delete oid 40 current snap is 206 2024-08-17T17:01:50.662 INFO:tasks.rados.rados.0.smithi063.stdout:1679: done (0 left) 2024-08-17T17:01:50.662 INFO:tasks.rados.rados.0.smithi063.stdout:1680: read oid 22 snap -1 2024-08-17T17:01:50.662 INFO:tasks.rados.rados.0.smithi063.stdout:1680: expect (ObjNum 626 snap 199 seq_num 626) 2024-08-17T17:01:50.662 INFO:tasks.rados.rados.0.smithi063.stdout:1681: write oid 35 current snap is 206 2024-08-17T17:01:50.663 INFO:tasks.rados.rados.0.smithi063.stdout:1681: seq_num 645 ranges {440515=709257,1910768=583545} 2024-08-17T17:01:50.669 INFO:tasks.rados.rados.0.smithi063.stdout:1681: writing smithi06387500-35 from 440515 to 1149772 tid 1 2024-08-17T17:01:50.677 INFO:tasks.rados.rados.0.smithi063.stdout:1681: writing smithi06387500-35 from 1910768 to 2494313 tid 2 2024-08-17T17:01:50.677 INFO:tasks.rados.rados.0.smithi063.stdout:1682: delete oid 20 current snap is 206 2024-08-17T17:01:50.677 INFO:tasks.rados.rados.0.smithi063.stdout:1680: expect (ObjNum 626 snap 199 seq_num 626) 2024-08-17T17:01:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:1681: finishing write tid 1 to smithi06387500-35 2024-08-17T17:01:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:1681: finishing write tid 2 to smithi06387500-35 2024-08-17T17:01:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:1681: finishing write tid 3 to smithi06387500-35 2024-08-17T17:01:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:1680: done (2 left) 2024-08-17T17:01:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:1682: done (1 left) 2024-08-17T17:01:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:1683: write oid 18 current snap is 206 2024-08-17T17:01:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:1681: finishing write tid 4 to smithi06387500-35 2024-08-17T17:01:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:1681: oid 35 updating version 0 to 1654 2024-08-17T17:01:50.762 INFO:tasks.rados.rados.0.smithi063.stdout:1681: oid 35 version 1654 is already newer than 1652 2024-08-17T17:01:50.762 INFO:tasks.rados.rados.0.smithi063.stdout:1681: oid 35 version 1654 is already newer than 1653 2024-08-17T17:01:50.762 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 1654 (ObjNum 645 snap 206 seq_num 645) dirty exists 2024-08-17T17:01:50.762 INFO:tasks.rados.rados.0.smithi063.stdout:1681: left oid 35 (ObjNum 645 snap 206 seq_num 645) 2024-08-17T17:01:50.762 INFO:tasks.rados.rados.0.smithi063.stdout:1683: seq_num 647 ranges {474129=576108,1627732=587214,2718088=156041} 2024-08-17T17:01:50.766 INFO:tasks.rados.rados.0.smithi063.stdout:1683: writing smithi06387500-18 from 474129 to 1050237 tid 1 2024-08-17T17:01:50.773 INFO:tasks.rados.rados.0.smithi063.stdout:1683: writing smithi06387500-18 from 1627732 to 2214946 tid 2 2024-08-17T17:01:50.777 INFO:tasks.rados.rados.0.smithi063.stdout:1683: writing smithi06387500-18 from 2718088 to 2874129 tid 3 2024-08-17T17:01:50.777 INFO:tasks.rados.rados.0.smithi063.stdout:1681: done (1 left) 2024-08-17T17:01:50.777 INFO:tasks.rados.rados.0.smithi063.stdout:1684: read oid 9 snap -1 2024-08-17T17:01:50.777 INFO:tasks.rados.rados.0.smithi063.stdout:1684: expect deleted 2024-08-17T17:01:50.777 INFO:tasks.rados.rados.0.smithi063.stdout:1685: write oid 23 current snap is 206 2024-08-17T17:01:50.777 INFO:tasks.rados.rados.0.smithi063.stdout:1683: finishing write tid 1 to smithi06387500-18 2024-08-17T17:01:50.777 INFO:tasks.rados.rados.0.smithi063.stdout:1683: finishing write tid 2 to smithi06387500-18 2024-08-17T17:01:50.777 INFO:tasks.rados.rados.0.smithi063.stdout:1685: seq_num 648 ranges {490936=651357,1686214=447225,2861864=29072} 2024-08-17T17:01:50.782 INFO:tasks.rados.rados.0.smithi063.stdout:1685: writing smithi06387500-23 from 490936 to 1142293 tid 1 2024-08-17T17:01:50.788 INFO:tasks.rados.rados.0.smithi063.stdout:1685: writing smithi06387500-23 from 1686214 to 2133439 tid 2 2024-08-17T17:01:50.792 INFO:tasks.rados.rados.0.smithi063.stdout:1685: writing smithi06387500-23 from 2861864 to 2890936 tid 3 2024-08-17T17:01:50.793 INFO:tasks.rados.rados.0.smithi063.stdout:1686: read oid 19 snap -1 2024-08-17T17:01:50.793 INFO:tasks.rados.rados.0.smithi063.stdout:1686: expect deleted 2024-08-17T17:01:50.793 INFO:tasks.rados.rados.0.smithi063.stdout:1687: delete oid 21 current snap is 206 2024-08-17T17:01:50.793 INFO:tasks.rados.rados.0.smithi063.stdout:1683: finishing write tid 3 to smithi06387500-18 2024-08-17T17:01:50.793 INFO:tasks.rados.rados.0.smithi063.stdout:1683: finishing write tid 4 to smithi06387500-18 2024-08-17T17:01:50.793 INFO:tasks.rados.rados.0.smithi063.stdout:1683: finishing write tid 5 to smithi06387500-18 2024-08-17T17:01:50.793 INFO:tasks.rados.rados.0.smithi063.stdout:1683: oid 18 updating version 0 to 587 2024-08-17T17:01:50.793 INFO:tasks.rados.rados.0.smithi063.stdout:1683: oid 18 updating version 587 to 588 2024-08-17T17:01:50.793 INFO:tasks.rados.rados.0.smithi063.stdout:1683: oid 18 updating version 588 to 589 2024-08-17T17:01:50.793 INFO:tasks.rados.rados.0.smithi063.stdout:1683: oid 18 updating version 589 to 590 2024-08-17T17:01:50.793 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 590 (ObjNum 647 snap 206 seq_num 647) dirty exists 2024-08-17T17:01:50.793 INFO:tasks.rados.rados.0.smithi063.stdout:1683: left oid 18 (ObjNum 647 snap 206 seq_num 647) 2024-08-17T17:01:50.794 INFO:tasks.rados.rados.0.smithi063.stdout:1685: finishing write tid 1 to smithi06387500-23 2024-08-17T17:01:50.794 INFO:tasks.rados.rados.0.smithi063.stdout:1685: finishing write tid 2 to smithi06387500-23 2024-08-17T17:01:50.794 INFO:tasks.rados.rados.0.smithi063.stdout:1685: finishing write tid 3 to smithi06387500-23 2024-08-17T17:01:50.795 INFO:tasks.rados.rados.0.smithi063.stdout:1685: finishing write tid 4 to smithi06387500-23 2024-08-17T17:01:50.795 INFO:tasks.rados.rados.0.smithi063.stdout:1683: done (4 left) 2024-08-17T17:01:50.795 INFO:tasks.rados.rados.0.smithi063.stdout:1684: done (3 left) 2024-08-17T17:01:50.795 INFO:tasks.rados.rados.0.smithi063.stdout:1686: done (2 left) 2024-08-17T17:01:50.795 INFO:tasks.rados.rados.0.smithi063.stdout:1687: done (1 left) 2024-08-17T17:01:50.795 INFO:tasks.rados.rados.0.smithi063.stdout:1688: write oid 1 current snap is 206 2024-08-17T17:01:50.795 INFO:tasks.rados.rados.0.smithi063.stdout:1688: seq_num 650 ranges {524550=401855,1403178=450894,2516710=407840} 2024-08-17T17:01:50.800 INFO:tasks.rados.rados.0.smithi063.stdout:1688: writing smithi06387500-1 from 524550 to 926405 tid 1 2024-08-17T17:01:50.805 INFO:tasks.rados.rados.0.smithi063.stdout:1688: writing smithi06387500-1 from 1403178 to 1854072 tid 2 2024-08-17T17:01:50.811 INFO:tasks.rados.rados.0.smithi063.stdout:1688: writing smithi06387500-1 from 2516710 to 2924550 tid 3 2024-08-17T17:01:50.811 INFO:tasks.rados.rados.0.smithi063.stdout:1689: delete oid 37 current snap is 206 2024-08-17T17:01:50.811 INFO:tasks.rados.rados.0.smithi063.stdout:1685: finishing write tid 5 to smithi06387500-23 2024-08-17T17:01:50.811 INFO:tasks.rados.rados.0.smithi063.stdout:1685: oid 23 updating version 0 to 760 2024-08-17T17:01:50.811 INFO:tasks.rados.rados.0.smithi063.stdout:1685: oid 23 updating version 760 to 761 2024-08-17T17:01:50.811 INFO:tasks.rados.rados.0.smithi063.stdout:1685: oid 23 version 761 is already newer than 758 2024-08-17T17:01:50.811 INFO:tasks.rados.rados.0.smithi063.stdout:1685: oid 23 version 761 is already newer than 759 2024-08-17T17:01:50.811 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 761 (ObjNum 648 snap 206 seq_num 648) dirty exists 2024-08-17T17:01:50.811 INFO:tasks.rados.rados.0.smithi063.stdout:1685: left oid 23 (ObjNum 648 snap 206 seq_num 648) 2024-08-17T17:01:50.811 INFO:tasks.rados.rados.0.smithi063.stdout:1688: finishing write tid 1 to smithi06387500-1 2024-08-17T17:01:50.812 INFO:tasks.rados.rados.0.smithi063.stdout:1688: finishing write tid 2 to smithi06387500-1 2024-08-17T17:01:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:1685: done (2 left) 2024-08-17T17:01:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:1689: done (1 left) 2024-08-17T17:01:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:1690: read oid 19 snap -1 2024-08-17T17:01:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:1690: expect deleted 2024-08-17T17:01:50.814 INFO:tasks.rados.rados.0.smithi063.stdout:1691: snap_create 2024-08-17T17:01:50.814 INFO:tasks.rados.rados.0.smithi063.stdout:1688: finishing write tid 3 to smithi06387500-1 2024-08-17T17:01:50.814 INFO:tasks.rados.rados.0.smithi063.stdout:1688: finishing write tid 4 to smithi06387500-1 2024-08-17T17:01:50.814 INFO:tasks.rados.rados.0.smithi063.stdout:1688: finishing write tid 5 to smithi06387500-1 2024-08-17T17:01:50.814 INFO:tasks.rados.rados.0.smithi063.stdout:1688: oid 1 updating version 0 to 763 2024-08-17T17:01:50.814 INFO:tasks.rados.rados.0.smithi063.stdout:1688: oid 1 updating version 763 to 766 2024-08-17T17:01:50.814 INFO:tasks.rados.rados.0.smithi063.stdout:1688: oid 1 version 766 is already newer than 764 2024-08-17T17:01:50.814 INFO:tasks.rados.rados.0.smithi063.stdout:1688: oid 1 version 766 is already newer than 765 2024-08-17T17:01:50.814 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 766 (ObjNum 650 snap 206 seq_num 650) dirty exists 2024-08-17T17:01:50.814 INFO:tasks.rados.rados.0.smithi063.stdout:1688: left oid 1 (ObjNum 650 snap 206 seq_num 650) 2024-08-17T17:01:50.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:50 smithi162 ceph-mon[34633]: osdmap e489: 8 total, 8 up, 8 in 2024-08-17T17:01:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:50 smithi162 ceph-mon[34633]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:50 smithi162 ceph-mon[34633]: pgmap v187: 89 pgs: 89 active+clean; 106 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-08-17T17:01:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:50 smithi162 ceph-mon[34633]: from='client.? 172.21.15.63:0/3129676556' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:01:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:50 smithi162 ceph-mon[34633]: Upgrade: Updating mon.b 2024-08-17T17:01:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:50 smithi162 ceph-mon[34633]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:50 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:01:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:50 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:01:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:50 smithi162 ceph-mon[34633]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:50 smithi162 ceph-mon[34633]: Deploying daemon mon.b on smithi162 2024-08-17T17:01:50.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:50 smithi162 ceph-mon[34633]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[114756]: osdmap e489: 8 total, 8 up, 8 in 2024-08-17T17:01:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[114756]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[114756]: pgmap v187: 89 pgs: 89 active+clean; 106 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-08-17T17:01:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3129676556' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:01:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[114756]: Upgrade: Updating mon.b 2024-08-17T17:01:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:01:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:01:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[114756]: Deploying daemon mon.b on smithi162 2024-08-17T17:01:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[114756]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[109843]: osdmap e489: 8 total, 8 up, 8 in 2024-08-17T17:01:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[109843]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[109843]: pgmap v187: 89 pgs: 89 active+clean; 106 MiB data, 523 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.1 MiB/s wr, 7 op/s 2024-08-17T17:01:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3129676556' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:01:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[109843]: Upgrade: Updating mon.b 2024-08-17T17:01:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:01:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-08-17T17:01:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-08-17T17:01:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-08-17T17:01:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[109843]: Deploying daemon mon.b on smithi162 2024-08-17T17:01:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:50 smithi063 ceph-mon[109843]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:01:51.666 INFO:tasks.rados.rados.0.smithi063.stdout:1688: done (2 left) 2024-08-17T17:01:51.666 INFO:tasks.rados.rados.0.smithi063.stdout:1690: done (1 left) 2024-08-17T17:01:51.666 INFO:tasks.rados.rados.0.smithi063.stdout:1691: done (0 left) 2024-08-17T17:01:51.666 INFO:tasks.rados.rados.0.smithi063.stdout:1692: read oid 48 snap 206 2024-08-17T17:01:51.666 INFO:tasks.rados.rados.0.smithi063.stdout:1692: expect deleted 2024-08-17T17:01:51.666 INFO:tasks.rados.rados.0.smithi063.stdout:1693: rollback oid 3 current snap is 207 2024-08-17T17:01:51.666 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 3 to 206 2024-08-17T17:01:51.667 INFO:tasks.rados.rados.0.smithi063.stdout:1694: read oid 15 snap -1 2024-08-17T17:01:51.667 INFO:tasks.rados.rados.0.smithi063.stdout:1694: expect deleted 2024-08-17T17:01:51.667 INFO:tasks.rados.rados.0.smithi063.stdout:1695: delete oid 42 current snap is 207 2024-08-17T17:01:51.669 INFO:tasks.rados.rados.0.smithi063.stdout:1693: finishing rollback tid 1 to smithi06387500-3 2024-08-17T17:01:51.669 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 0 (ObjNum 33 snap 0 seq_num 1806987088) dirty dne 2024-08-17T17:01:51.670 INFO:tasks.rados.rados.0.smithi063.stdout:1692: done (3 left) 2024-08-17T17:01:51.670 INFO:tasks.rados.rados.0.smithi063.stdout:1693: done (2 left) 2024-08-17T17:01:51.670 INFO:tasks.rados.rados.0.smithi063.stdout:1694: done (1 left) 2024-08-17T17:01:51.670 INFO:tasks.rados.rados.0.smithi063.stdout:1695: done (0 left) 2024-08-17T17:01:51.670 INFO:tasks.rados.rados.0.smithi063.stdout:1696: snap_remove snap 205 2024-08-17T17:01:51.865 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:51 smithi162 ceph-mon[34633]: osdmap e490: 8 total, 8 up, 8 in 2024-08-17T17:01:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:01:51 smithi063 ceph-mon[114756]: osdmap e490: 8 total, 8 up, 8 in 2024-08-17T17:01:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:01:51 smithi063 ceph-mon[109843]: osdmap e490: 8 total, 8 up, 8 in 2024-08-17T17:01:52.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:51 smithi162 systemd[1]: Stopping Ceph mon.b for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:01:52.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:52 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b[34610]: 2024-08-17T17:01:52.110+0000 7ff2d4ab0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:01:52.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:52 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b[34610]: 2024-08-17T17:01:52.110+0000 7ff2d4ab0640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-08-17T17:01:52.400 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:01:52 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:17:01:52] ENGINE Bus STOPPING 2024-08-17T17:01:52.400 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:01:52 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:17:01:52] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:01:52.400 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:01:52 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:17:01:52] ENGINE Bus STOPPED 2024-08-17T17:01:52.401 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:01:52 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:17:01:52] ENGINE Bus STARTING 2024-08-17T17:01:52.401 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:01:52 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:17:01:52] ENGINE Serving on http://:::9283 2024-08-17T17:01:52.401 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:01:52 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:17:01:52] ENGINE Bus STARTED 2024-08-17T17:01:52.728 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:52 smithi162 podman[104475]: 2024-08-17 17:01:52.377047516 +0000 UTC m=+0.362485715 container died d4ecbe65a0c8b20989c04ec24bba82fbf5fa2d3353c2f69b0b106933046fbfb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, GIT_CLEAN=True, ceph=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240731) 2024-08-17T17:01:53.086 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:52 smithi162 podman[104475]: 2024-08-17 17:01:52.9093898 +0000 UTC m=+0.894827997 container cleanup d4ecbe65a0c8b20989c04ec24bba82fbf5fa2d3353c2f69b0b106933046fbfb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0) 2024-08-17T17:01:53.086 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:52 smithi162 bash[104475]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b 2024-08-17T17:01:53.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:53 smithi162 podman[104495]: 2024-08-17 17:01:53.085829196 +0000 UTC m=+0.707017365 container remove d4ecbe65a0c8b20989c04ec24bba82fbf5fa2d3353c2f69b0b106933046fbfb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-08-17T17:01:53.762 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:53 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.b.service: Deactivated successfully. 2024-08-17T17:01:53.762 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:53 smithi162 systemd[1]: Stopped Ceph mon.b for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:01:53.762 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:53 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.b.service: Consumed 12.077s CPU time. 2024-08-17T17:01:53.762 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:53 smithi162 systemd[1]: Starting Ceph mon.b for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:01:53.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:01:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:01:53] "GET /metrics HTTP/1.1" 200 36198 "" "Prometheus/2.51.0" 2024-08-17T17:01:54.016 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:53 smithi162 podman[104629]: 2024-08-17 17:01:53.761646379 +0000 UTC m=+0.113448012 container create 520525c79dbe79052855f0ae2fc8e57188f8d133ac6c42f8200c78ff3ea2ac8e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0) 2024-08-17T17:01:54.016 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:53 smithi162 podman[104629]: 2024-08-17 17:01:53.665928638 +0000 UTC m=+0.017730285 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:01:54.016 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:53 smithi162 podman[104629]: 2024-08-17 17:01:53.989183273 +0000 UTC m=+0.340984925 container init 520525c79dbe79052855f0ae2fc8e57188f8d133ac6c42f8200c78ff3ea2ac8e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.36.0) 2024-08-17T17:01:54.016 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:53 smithi162 podman[104629]: 2024-08-17 17:01:53.994011283 +0000 UTC m=+0.345812920 container start 520525c79dbe79052855f0ae2fc8e57188f8d133ac6c42f8200c78ff3ea2ac8e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True) 2024-08-17T17:01:54.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: set uid:gid to 167:167 (ceph:ceph) 2024-08-17T17:01:54.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev), process ceph-mon, pid 7 2024-08-17T17:01:54.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: pidfile_write: ignore empty --pid-file 2024-08-17T17:01:54.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: load: jerasure load: lrc 2024-08-17T17:01:54.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: RocksDB version: 7.9.2 2024-08-17T17:01:54.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Git sha 0 2024-08-17T17:01:54.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Compile date 2024-08-15 20:43:31 2024-08-17T17:01:54.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: DB SUMMARY 2024-08-17T17:01:54.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: DB Session ID: 3N7HURFHETSKBYTIJ3EA 2024-08-17T17:01:54.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: CURRENT file: CURRENT 2024-08-17T17:01:54.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: IDENTITY file: IDENTITY 2024-08-17T17:01:54.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1899 Bytes 2024-08-17T17:01:54.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000033.sst 2024-08-17T17:01:54.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000031.log size: 313093 ; 2024-08-17T17:01:54.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.error_if_exists: 0 2024-08-17T17:01:54.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.create_if_missing: 0 2024-08-17T17:01:54.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.paranoid_checks: 1 2024-08-17T17:01:54.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.flush_verify_memtable_count: 1 2024-08-17T17:01:54.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-08-17T17:01:54.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-08-17T17:01:54.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.env: 0x555a53757be0 2024-08-17T17:01:54.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.fs: PosixFileSystem 2024-08-17T17:01:54.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.info_log: 0x555a5452d700 2024-08-17T17:01:54.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_file_opening_threads: 16 2024-08-17T17:01:54.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.statistics: (nil) 2024-08-17T17:01:54.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.use_fsync: 0 2024-08-17T17:01:54.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_log_file_size: 0 2024-08-17T17:01:54.349 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-08-17T17:01:54.349 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.log_file_time_to_roll: 0 2024-08-17T17:01:54.349 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.keep_log_file_num: 1000 2024-08-17T17:01:54.349 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.recycle_log_file_num: 0 2024-08-17T17:01:54.349 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.allow_fallocate: 1 2024-08-17T17:01:54.349 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.allow_mmap_reads: 0 2024-08-17T17:01:54.349 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.allow_mmap_writes: 0 2024-08-17T17:01:54.349 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.use_direct_reads: 0 2024-08-17T17:01:54.349 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-08-17T17:01:54.349 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.create_missing_column_families: 0 2024-08-17T17:01:54.349 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.db_log_dir: 2024-08-17T17:01:54.349 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.wal_dir: 2024-08-17T17:01:54.350 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.table_cache_numshardbits: 6 2024-08-17T17:01:54.350 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.WAL_ttl_seconds: 0 2024-08-17T17:01:54.350 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.WAL_size_limit_MB: 0 2024-08-17T17:01:54.350 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-08-17T17:01:54.350 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-08-17T17:01:54.350 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.is_fd_close_on_exec: 1 2024-08-17T17:01:54.350 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.advise_random_on_open: 1 2024-08-17T17:01:54.350 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.db_write_buffer_size: 0 2024-08-17T17:01:54.350 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.write_buffer_manager: 0x555a54530960 2024-08-17T17:01:54.350 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-08-17T17:01:54.350 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-08-17T17:01:54.351 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.use_adaptive_mutex: 0 2024-08-17T17:01:54.351 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.rate_limiter: (nil) 2024-08-17T17:01:54.351 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-08-17T17:01:54.351 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.wal_recovery_mode: 2 2024-08-17T17:01:54.351 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.enable_thread_tracking: 0 2024-08-17T17:01:54.351 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.enable_pipelined_write: 0 2024-08-17T17:01:54.351 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.unordered_write: 0 2024-08-17T17:01:54.351 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-08-17T17:01:54.351 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-08-17T17:01:54.351 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-08-17T17:01:54.351 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-08-17T17:01:54.351 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.row_cache: None 2024-08-17T17:01:54.351 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.wal_filter: None 2024-08-17T17:01:54.352 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-08-17T17:01:54.352 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.allow_ingest_behind: 0 2024-08-17T17:01:54.352 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.two_write_queues: 0 2024-08-17T17:01:54.352 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.manual_wal_flush: 0 2024-08-17T17:01:54.352 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.wal_compression: 0 2024-08-17T17:01:54.352 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.atomic_flush: 0 2024-08-17T17:01:54.352 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-08-17T17:01:54.352 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.persist_stats_to_disk: 0 2024-08-17T17:01:54.352 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.write_dbid_to_manifest: 0 2024-08-17T17:01:54.352 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.log_readahead_size: 0 2024-08-17T17:01:54.352 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-08-17T17:01:54.352 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.best_efforts_recovery: 0 2024-08-17T17:01:54.352 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-08-17T17:01:54.352 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-08-17T17:01:54.353 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.allow_data_in_errors: 0 2024-08-17T17:01:54.353 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.db_host_id: __hostname__ 2024-08-17T17:01:54.354 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.enforce_single_del_contracts: true 2024-08-17T17:01:54.354 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_background_jobs: 2 2024-08-17T17:01:54.354 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_background_compactions: -1 2024-08-17T17:01:54.354 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_subcompactions: 1 2024-08-17T17:01:54.354 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-08-17T17:01:54.354 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-08-17T17:01:54.354 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.delayed_write_rate : 16777216 2024-08-17T17:01:54.355 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_total_wal_size: 0 2024-08-17T17:01:54.355 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-08-17T17:01:54.355 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.stats_dump_period_sec: 600 2024-08-17T17:01:54.355 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.stats_persist_period_sec: 600 2024-08-17T17:01:54.355 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-08-17T17:01:54.355 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_open_files: -1 2024-08-17T17:01:54.355 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.bytes_per_sync: 0 2024-08-17T17:01:54.355 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.wal_bytes_per_sync: 0 2024-08-17T17:01:54.355 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.strict_bytes_per_sync: 0 2024-08-17T17:01:54.355 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compaction_readahead_size: 0 2024-08-17T17:01:54.355 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_background_flushes: -1 2024-08-17T17:01:54.355 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Compression algorithms supported: 2024-08-17T17:01:54.355 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: kZSTD supported: 0 2024-08-17T17:01:54.356 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: kXpressCompression supported: 0 2024-08-17T17:01:54.356 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: kBZip2Compression supported: 0 2024-08-17T17:01:54.356 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-08-17T17:01:54.356 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: kLZ4Compression supported: 1 2024-08-17T17:01:54.356 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: kZlibCompression supported: 1 2024-08-17T17:01:54.356 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: kLZ4HCCompression supported: 1 2024-08-17T17:01:54.356 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: kSnappyCompression supported: 1 2024-08-17T17:01:54.356 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Fast CRC32 supported: Supported on x86 2024-08-17T17:01:54.356 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: DMutex implementation: pthread_mutex_t 2024-08-17T17:01:54.356 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 2024-08-17T17:01:54.356 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-08-17T17:01:54.356 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-08-17T17:01:54.356 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.merge_operator: 2024-08-17T17:01:54.357 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compaction_filter: None 2024-08-17T17:01:54.357 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compaction_filter_factory: None 2024-08-17T17:01:54.357 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.sst_partitioner_factory: None 2024-08-17T17:01:54.357 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.memtable_factory: SkipListFactory 2024-08-17T17:01:54.357 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.table_factory: BlockBasedTable 2024-08-17T17:01:54.357 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555a5452d380) 2024-08-17T17:01:54.357 INFO:journalctl@ceph.mon.b.smithi162.stdout: cache_index_and_filter_blocks: 1 2024-08-17T17:01:54.357 INFO:journalctl@ceph.mon.b.smithi162.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-08-17T17:01:54.357 INFO:journalctl@ceph.mon.b.smithi162.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-08-17T17:01:54.357 INFO:journalctl@ceph.mon.b.smithi162.stdout: pin_top_level_index_and_filter: 1 2024-08-17T17:01:54.357 INFO:journalctl@ceph.mon.b.smithi162.stdout: index_type: 0 2024-08-17T17:01:54.357 INFO:journalctl@ceph.mon.b.smithi162.stdout: data_block_index_type: 0 2024-08-17T17:01:54.358 INFO:journalctl@ceph.mon.b.smithi162.stdout: index_shortening: 1 2024-08-17T17:01:54.358 INFO:journalctl@ceph.mon.b.smithi162.stdout: data_block_hash_table_util_ratio: 0.750000 2024-08-17T17:01:54.358 INFO:journalctl@ceph.mon.b.smithi162.stdout: checksum: 4 2024-08-17T17:01:54.358 INFO:journalctl@ceph.mon.b.smithi162.stdout: no_block_cache: 0 2024-08-17T17:01:54.358 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_cache: 0x555a54521610 2024-08-17T17:01:54.358 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_cache_name: BinnedLRUCache 2024-08-17T17:01:54.358 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_cache_options: 2024-08-17T17:01:54.358 INFO:journalctl@ceph.mon.b.smithi162.stdout: capacity : 536870912 2024-08-17T17:01:54.358 INFO:journalctl@ceph.mon.b.smithi162.stdout: num_shard_bits : 4 2024-08-17T17:01:54.358 INFO:journalctl@ceph.mon.b.smithi162.stdout: strict_capacity_limit : 0 2024-08-17T17:01:54.358 INFO:journalctl@ceph.mon.b.smithi162.stdout: high_pri_pool_ratio: 0.000 2024-08-17T17:01:54.358 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_cache_compressed: (nil) 2024-08-17T17:01:54.358 INFO:journalctl@ceph.mon.b.smithi162.stdout: persistent_cache: (nil) 2024-08-17T17:01:54.359 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_size: 4096 2024-08-17T17:01:54.359 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_size_deviation: 10 2024-08-17T17:01:54.359 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_restart_interval: 16 2024-08-17T17:01:54.359 INFO:journalctl@ceph.mon.b.smithi162.stdout: index_block_restart_interval: 1 2024-08-17T17:01:54.359 INFO:journalctl@ceph.mon.b.smithi162.stdout: metadata_block_size: 4096 2024-08-17T17:01:54.359 INFO:journalctl@ceph.mon.b.smithi162.stdout: partition_filters: 0 2024-08-17T17:01:54.359 INFO:journalctl@ceph.mon.b.smithi162.stdout: use_delta_encoding: 1 2024-08-17T17:01:54.359 INFO:journalctl@ceph.mon.b.smithi162.stdout: filter_policy: bloomfilter 2024-08-17T17:01:54.359 INFO:journalctl@ceph.mon.b.smithi162.stdout: whole_key_filtering: 1 2024-08-17T17:01:54.359 INFO:journalctl@ceph.mon.b.smithi162.stdout: verify_compression: 0 2024-08-17T17:01:54.359 INFO:journalctl@ceph.mon.b.smithi162.stdout: read_amp_bytes_per_bit: 0 2024-08-17T17:01:54.359 INFO:journalctl@ceph.mon.b.smithi162.stdout: format_version: 5 2024-08-17T17:01:54.359 INFO:journalctl@ceph.mon.b.smithi162.stdout: enable_index_compression: 1 2024-08-17T17:01:54.360 INFO:journalctl@ceph.mon.b.smithi162.stdout: block_align: 0 2024-08-17T17:01:54.360 INFO:journalctl@ceph.mon.b.smithi162.stdout: max_auto_readahead_size: 262144 2024-08-17T17:01:54.360 INFO:journalctl@ceph.mon.b.smithi162.stdout: prepopulate_block_cache: 0 2024-08-17T17:01:54.360 INFO:journalctl@ceph.mon.b.smithi162.stdout: initial_auto_readahead_size: 8192 2024-08-17T17:01:54.360 INFO:journalctl@ceph.mon.b.smithi162.stdout: num_file_reads_for_auto_readahead: 2 2024-08-17T17:01:54.360 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.write_buffer_size: 33554432 2024-08-17T17:01:54.360 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_write_buffer_number: 2 2024-08-17T17:01:54.360 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compression: NoCompression 2024-08-17T17:01:54.360 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.bottommost_compression: Disabled 2024-08-17T17:01:54.360 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.prefix_extractor: nullptr 2024-08-17T17:01:54.360 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-08-17T17:01:54.360 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.num_levels: 7 2024-08-17T17:01:54.360 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-08-17T17:01:54.360 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-08-17T17:01:54.361 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-08-17T17:01:54.361 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-08-17T17:01:54.361 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-08-17T17:01:54.361 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-08-17T17:01:54.361 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-08-17T17:01:54.361 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:01:54.361 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-08-17T17:01:54.361 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-08-17T17:01:54.361 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:01:54.361 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:01:54.361 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compression_opts.window_bits: -14 2024-08-17T17:01:54.361 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compression_opts.level: 32767 2024-08-17T17:01:54.362 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compression_opts.strategy: 0 2024-08-17T17:01:54.362 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-08-17T17:01:54.362 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-08-17T17:01:54.362 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-08-17T17:01:54.362 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-08-17T17:01:54.362 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compression_opts.enabled: false 2024-08-17T17:01:54.362 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-08-17T17:01:54.362 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-08-17T17:01:54.362 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-08-17T17:01:54.362 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-08-17T17:01:54.362 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.target_file_size_base: 67108864 2024-08-17T17:01:54.362 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.target_file_size_multiplier: 1 2024-08-17T17:01:54.363 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-08-17T17:01:54.363 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-08-17T17:01:54.363 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-08-17T17:01:54.363 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-08-17T17:01:54.363 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-08-17T17:01:54.363 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-08-17T17:01:54.363 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-08-17T17:01:54.363 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-08-17T17:01:54.363 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-08-17T17:01:54.363 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-08-17T17:01:54.363 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-08-17T17:01:54.363 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-08-17T17:01:54.363 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-08-17T17:01:54.364 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.arena_block_size: 1048576 2024-08-17T17:01:54.364 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-08-17T17:01:54.364 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-08-17T17:01:54.364 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.disable_auto_compactions: 0 2024-08-17T17:01:54.364 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-08-17T17:01:54.364 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-08-17T17:01:54.364 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-08-17T17:01:54.364 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-08-17T17:01:54.364 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-08-17T17:01:54.364 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-08-17T17:01:54.364 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-08-17T17:01:54.364 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-08-17T17:01:54.364 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-08-17T17:01:54.365 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-08-17T17:01:54.365 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-08-17T17:01:54.365 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.inplace_update_support: 0 2024-08-17T17:01:54.365 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.inplace_update_num_locks: 10000 2024-08-17T17:01:54.365 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-08-17T17:01:54.365 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-08-17T17:01:54.365 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.memtable_huge_page_size: 0 2024-08-17T17:01:54.365 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.bloom_locality: 0 2024-08-17T17:01:54.365 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.max_successive_merges: 0 2024-08-17T17:01:54.365 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.optimize_filters_for_hits: 0 2024-08-17T17:01:54.365 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.paranoid_file_checks: 0 2024-08-17T17:01:54.365 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.force_consistency_checks: 1 2024-08-17T17:01:54.365 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.report_bg_io_stats: 0 2024-08-17T17:01:54.366 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.ttl: 2592000 2024-08-17T17:01:54.366 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.periodic_compaction_seconds: 0 2024-08-17T17:01:54.366 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-08-17T17:01:54.366 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-08-17T17:01:54.366 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.enable_blob_files: false 2024-08-17T17:01:54.366 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.min_blob_size: 0 2024-08-17T17:01:54.366 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.blob_file_size: 268435456 2024-08-17T17:01:54.366 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.blob_compression_type: NoCompression 2024-08-17T17:01:54.366 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.enable_blob_garbage_collection: false 2024-08-17T17:01:54.366 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-08-17T17:01:54.366 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-08-17T17:01:54.366 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-08-17T17:01:54.366 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.blob_file_starting_level: 0 2024-08-17T17:01:54.367 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-08-17T17:01:54.367 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:01:54.367 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 35, last_sequence is 22667, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 31 2024-08-17T17:01:54.367 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-08-17T17:01:54.367 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e873a443-d04e-409d-aa61-ba7c22273137 2024-08-17T17:01:54.367 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914114021264, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-08-17T17:01:54.367 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-08-17T17:01:54.367 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914114022769, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 207326, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 22672, "largest_seqno": 22872, "table_properties": {"data_size": 205328, "index_size": 640, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 2598, "raw_average_key_size": 25, "raw_value_size": 202919, "raw_average_value_size": 1970, "num_data_blocks": 28, "num_entries": 103, "num_filter_entries": 103, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1723914114, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e873a443-d04e-409d-aa61-ba7c22273137", "db_session_id": "3N7HURFHETSKBYTIJ3EA", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-08-17T17:01:54.367 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: EVENT_LOG_v1 {"time_micros": 1723914114022873, "job": 1, "event": "recovery_finished"} 2024-08-17T17:01:54.367 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-08-17T17:01:54.367 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-08-17T17:01:54.367 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-08-17T17:01:54.368 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x555a54550e00 2024-08-17T17:01:54.368 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: DB pointer 0x555a5464e000 2024-08-17T17:01:54.368 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-08-17T17:01:54.368 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-08-17T17:01:54.368 INFO:journalctl@ceph.mon.b.smithi162.stdout: ** DB Stats ** 2024-08-17T17:01:54.368 INFO:journalctl@ceph.mon.b.smithi162.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:01:54.368 INFO:journalctl@ceph.mon.b.smithi162.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-08-17T17:01:54.368 INFO:journalctl@ceph.mon.b.smithi162.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:01:54.368 INFO:journalctl@ceph.mon.b.smithi162.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:01:54.368 INFO:journalctl@ceph.mon.b.smithi162.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-08-17T17:01:54.368 INFO:journalctl@ceph.mon.b.smithi162.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-08-17T17:01:54.368 INFO:journalctl@ceph.mon.b.smithi162.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-08-17T17:01:54.369 INFO:journalctl@ceph.mon.b.smithi162.stdout: 2024-08-17T17:01:54.369 INFO:journalctl@ceph.mon.b.smithi162.stdout: ** Compaction Stats [default] ** 2024-08-17T17:01:54.369 INFO:journalctl@ceph.mon.b.smithi162.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:01:54.369 INFO:journalctl@ceph.mon.b.smithi162.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-08-17T17:01:54.369 INFO:journalctl@ceph.mon.b.smithi162.stdout: L0 1/0 202.47 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 214.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T17:01:54.369 INFO:journalctl@ceph.mon.b.smithi162.stdout: L6 1/0 13.57 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-08-17T17:01:54.369 INFO:journalctl@ceph.mon.b.smithi162.stdout: Sum 2/0 13.76 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 214.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T17:01:54.369 INFO:journalctl@ceph.mon.b.smithi162.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 214.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T17:01:54.369 INFO:journalctl@ceph.mon.b.smithi162.stdout: 2024-08-17T17:01:54.370 INFO:journalctl@ceph.mon.b.smithi162.stdout: ** Compaction Stats [default] ** 2024-08-17T17:01:54.370 INFO:journalctl@ceph.mon.b.smithi162.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-08-17T17:01:54.370 INFO:journalctl@ceph.mon.b.smithi162.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-08-17T17:01:54.370 INFO:journalctl@ceph.mon.b.smithi162.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 214.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-08-17T17:01:54.370 INFO:journalctl@ceph.mon.b.smithi162.stdout: 2024-08-17T17:01:54.370 INFO:journalctl@ceph.mon.b.smithi162.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-08-17T17:01:54.370 INFO:journalctl@ceph.mon.b.smithi162.stdout: 2024-08-17T17:01:54.370 INFO:journalctl@ceph.mon.b.smithi162.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-08-17T17:01:54.370 INFO:journalctl@ceph.mon.b.smithi162.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-08-17T17:01:54.370 INFO:journalctl@ceph.mon.b.smithi162.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-08-17T17:01:54.370 INFO:journalctl@ceph.mon.b.smithi162.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-08-17T17:01:54.370 INFO:journalctl@ceph.mon.b.smithi162.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-08-17T17:01:54.371 INFO:journalctl@ceph.mon.b.smithi162.stdout: AddFile(Keys): cumulative 0, interval 0 2024-08-17T17:01:54.371 INFO:journalctl@ceph.mon.b.smithi162.stdout: Cumulative compaction: 0.00 GB write, 53.17 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:01:54.371 INFO:journalctl@ceph.mon.b.smithi162.stdout: Interval compaction: 0.00 GB write, 53.17 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-08-17T17:01:54.371 INFO:journalctl@ceph.mon.b.smithi162.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-08-17T17:01:54.371 INFO:journalctl@ceph.mon.b.smithi162.stdout: Block cache BinnedLRUCache@0x555a54521610#7 capacity: 512.00 MB usage: 1.08 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-08-17T17:01:54.371 INFO:journalctl@ceph.mon.b.smithi162.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.38 KB,7.15256e-05%) IndexBlock(1,0.70 KB,0.00013411%) Misc(1,0.00 KB,0%) 2024-08-17T17:01:54.371 INFO:journalctl@ceph.mon.b.smithi162.stdout: 2024-08-17T17:01:54.371 INFO:journalctl@ceph.mon.b.smithi162.stdout: ** File Read Latency Histogram By Level [default] ** 2024-08-17T17:01:54.371 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: starting mon.b rank 1 at public addrs [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] at bind addrs [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:01:54.371 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: mon.b@-1(???) e3 preinit fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:01:54.371 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: mon.b@-1(???).mds e8 new map 2024-08-17T17:01:54.371 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: mon.b@-1(???).mds e8 print_map 2024-08-17T17:01:54.372 INFO:journalctl@ceph.mon.b.smithi162.stdout: e8 2024-08-17T17:01:54.372 INFO:journalctl@ceph.mon.b.smithi162.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-08-17T17:01:54.372 INFO:journalctl@ceph.mon.b.smithi162.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-08-17T17:01:54.372 INFO:journalctl@ceph.mon.b.smithi162.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:01:54.372 INFO:journalctl@ceph.mon.b.smithi162.stdout: legacy client fscid: 1 2024-08-17T17:01:54.372 INFO:journalctl@ceph.mon.b.smithi162.stdout: 2024-08-17T17:01:54.372 INFO:journalctl@ceph.mon.b.smithi162.stdout: Filesystem 'foo' (1) 2024-08-17T17:01:54.372 INFO:journalctl@ceph.mon.b.smithi162.stdout: fs_name foo 2024-08-17T17:01:54.372 INFO:journalctl@ceph.mon.b.smithi162.stdout: epoch 7 2024-08-17T17:01:54.372 INFO:journalctl@ceph.mon.b.smithi162.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-08-17T17:01:54.372 INFO:journalctl@ceph.mon.b.smithi162.stdout: created 2024-08-17T16:53:36.880465+0000 2024-08-17T17:01:54.372 INFO:journalctl@ceph.mon.b.smithi162.stdout: modified 2024-08-17T16:53:54.414788+0000 2024-08-17T17:01:54.373 INFO:journalctl@ceph.mon.b.smithi162.stdout: tableserver 0 2024-08-17T17:01:54.373 INFO:journalctl@ceph.mon.b.smithi162.stdout: root 0 2024-08-17T17:01:54.373 INFO:journalctl@ceph.mon.b.smithi162.stdout: session_timeout 60 2024-08-17T17:01:54.373 INFO:journalctl@ceph.mon.b.smithi162.stdout: session_autoclose 300 2024-08-17T17:01:54.373 INFO:journalctl@ceph.mon.b.smithi162.stdout: max_file_size 1099511627776 2024-08-17T17:01:54.373 INFO:journalctl@ceph.mon.b.smithi162.stdout: max_xattr_size 65536 2024-08-17T17:01:54.373 INFO:journalctl@ceph.mon.b.smithi162.stdout: required_client_features {} 2024-08-17T17:01:54.373 INFO:journalctl@ceph.mon.b.smithi162.stdout: last_failure 0 2024-08-17T17:01:54.373 INFO:journalctl@ceph.mon.b.smithi162.stdout: last_failure_osd_epoch 0 2024-08-17T17:01:54.373 INFO:journalctl@ceph.mon.b.smithi162.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-08-17T17:01:54.373 INFO:journalctl@ceph.mon.b.smithi162.stdout: max_mds 1 2024-08-17T17:01:54.373 INFO:journalctl@ceph.mon.b.smithi162.stdout: in 0 2024-08-17T17:01:54.373 INFO:journalctl@ceph.mon.b.smithi162.stdout: up {0=14658} 2024-08-17T17:01:54.374 INFO:journalctl@ceph.mon.b.smithi162.stdout: failed 2024-08-17T17:01:54.374 INFO:journalctl@ceph.mon.b.smithi162.stdout: damaged 2024-08-17T17:01:54.374 INFO:journalctl@ceph.mon.b.smithi162.stdout: stopped 2024-08-17T17:01:54.374 INFO:journalctl@ceph.mon.b.smithi162.stdout: data_pools [4] 2024-08-17T17:01:54.374 INFO:journalctl@ceph.mon.b.smithi162.stdout: metadata_pool 3 2024-08-17T17:01:54.374 INFO:journalctl@ceph.mon.b.smithi162.stdout: inline_data disabled 2024-08-17T17:01:54.374 INFO:journalctl@ceph.mon.b.smithi162.stdout: balancer 2024-08-17T17:01:54.374 INFO:journalctl@ceph.mon.b.smithi162.stdout: bal_rank_mask -1 2024-08-17T17:01:54.374 INFO:journalctl@ceph.mon.b.smithi162.stdout: standby_count_wanted 1 2024-08-17T17:01:54.374 INFO:journalctl@ceph.mon.b.smithi162.stdout: qdb_cluster leader: 0 members: 2024-08-17T17:01:54.374 INFO:journalctl@ceph.mon.b.smithi162.stdout: [mds.foo.smithi063.ultpqj{0:14658} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.63:6834/3222014296,v1:172.21.15.63:6835/3222014296] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:01:54.374 INFO:journalctl@ceph.mon.b.smithi162.stdout: 2024-08-17T17:01:54.374 INFO:journalctl@ceph.mon.b.smithi162.stdout: 2024-08-17T17:01:54.375 INFO:journalctl@ceph.mon.b.smithi162.stdout: Standby daemons: 2024-08-17T17:01:54.375 INFO:journalctl@ceph.mon.b.smithi162.stdout: 2024-08-17T17:01:54.375 INFO:journalctl@ceph.mon.b.smithi162.stdout: [mds.foo.smithi162.bjkxnv{-1:24529} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.162:6832/2161520737,v1:172.21.15.162:6833/2161520737] compat {c=[1],r=[1],i=[7ff]}] 2024-08-17T17:01:54.375 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: mon.b@-1(???).osd e491 crush map has features 3314933000854323200, adjusting msgr requires 2024-08-17T17:01:54.375 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: mon.b@-1(???).osd e491 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:01:54.375 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: mon.b@-1(???).osd e491 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:01:54.375 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: mon.b@-1(???).osd e491 crush map has features 432629239337189376, adjusting msgr requires 2024-08-17T17:01:54.375 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 ceph-mon[104668]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-08-17T17:01:54.375 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 bash[104629]: 520525c79dbe79052855f0ae2fc8e57188f8d133ac6c42f8200c78ff3ea2ac8e 2024-08-17T17:01:54.375 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:01:54 smithi162 systemd[1]: Started Ceph mon.b for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:01:55.546 INFO:tasks.rados.rados.0.smithi063.stdout:1696: done (0 left) 2024-08-17T17:01:55.547 INFO:tasks.rados.rados.0.smithi063.stdout:1697: read oid 48 snap 206 2024-08-17T17:01:55.547 INFO:tasks.rados.rados.0.smithi063.stdout:1697: expect deleted 2024-08-17T17:01:55.547 INFO:tasks.rados.rados.0.smithi063.stdout:1698: read oid 31 snap -1 2024-08-17T17:01:55.547 INFO:tasks.rados.rados.0.smithi063.stdout:1698: expect (ObjNum 643 snap 205 seq_num 643) 2024-08-17T17:01:55.547 INFO:tasks.rados.rados.0.smithi063.stdout:1699: rollback oid 7 current snap is 207 2024-08-17T17:01:55.547 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 206 2024-08-17T17:01:55.547 INFO:tasks.rados.rados.0.smithi063.stdout:1700: snap_create 2024-08-17T17:01:55.555 INFO:tasks.rados.rados.0.smithi063.stdout:1699: finishing rollback tid 1 to smithi06387500-7 2024-08-17T17:01:55.555 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 0 (ObjNum 129 snap 0 seq_num 1027984930) dirty dne 2024-08-17T17:01:55.562 INFO:tasks.rados.rados.0.smithi063.stdout:1698: expect (ObjNum 643 snap 205 seq_num 643) 2024-08-17T17:01:59.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:01:59 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T17:01:59.034+0000 7f4652b73640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-08-17T17:02:00.632 INFO:tasks.rados.rados.0.smithi063.stdout:1697: done (3 left) 2024-08-17T17:02:00.632 INFO:tasks.rados.rados.0.smithi063.stdout:1698: done (2 left) 2024-08-17T17:02:00.632 INFO:tasks.rados.rados.0.smithi063.stdout:1699: done (1 left) 2024-08-17T17:02:00.632 INFO:tasks.rados.rados.0.smithi063.stdout:1700: done (0 left) 2024-08-17T17:02:00.632 INFO:tasks.rados.rados.0.smithi063.stdout:1701: write oid 47 current snap is 208 2024-08-17T17:02:00.632 INFO:tasks.rados.rados.0.smithi063.stdout:1701: seq_num 653 ranges {574971=627602,1862271=598221} 2024-08-17T17:02:00.638 INFO:tasks.rados.rados.0.smithi063.stdout:1701: writing smithi06387500-47 from 574971 to 1202573 tid 1 2024-08-17T17:02:00.646 INFO:tasks.rados.rados.0.smithi063.stdout:1701: writing smithi06387500-47 from 1862271 to 2460492 tid 2 2024-08-17T17:02:00.646 INFO:tasks.rados.rados.0.smithi063.stdout:1702: snap_create 2024-08-17T17:02:00.646 INFO:tasks.rados.rados.0.smithi063.stdout:1701: finishing write tid 1 to smithi06387500-47 2024-08-17T17:02:00.648 INFO:tasks.rados.rados.0.smithi063.stdout:1701: finishing write tid 2 to smithi06387500-47 2024-08-17T17:02:00.648 INFO:tasks.rados.rados.0.smithi063.stdout:1701: finishing write tid 3 to smithi06387500-47 2024-08-17T17:02:00.649 INFO:tasks.rados.rados.0.smithi063.stdout:1701: finishing write tid 4 to smithi06387500-47 2024-08-17T17:02:00.649 INFO:tasks.rados.rados.0.smithi063.stdout:1701: oid 47 updating version 0 to 331 2024-08-17T17:02:00.649 INFO:tasks.rados.rados.0.smithi063.stdout:1701: oid 47 version 331 is already newer than 330 2024-08-17T17:02:00.649 INFO:tasks.rados.rados.0.smithi063.stdout:1701: oid 47 version 331 is already newer than 329 2024-08-17T17:02:00.649 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 331 (ObjNum 653 snap 208 seq_num 653) dirty exists 2024-08-17T17:02:00.649 INFO:tasks.rados.rados.0.smithi063.stdout:1701: left oid 47 (ObjNum 653 snap 208 seq_num 653) 2024-08-17T17:02:00.959 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:02:00.959 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: pgmap v191: 89 pgs: 89 active+clean; 97 MiB data, 519 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-08-17T17:02:00.959 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: 2: [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] mon.c 2024-08-17T17:02:00.959 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T17:02:00.959 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: osdmap e492: 8 total, 8 up, 8 in 2024-08-17T17:02:00.959 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: mgrmap e40: y(active, since 2m), standbys: x 2024-08-17T17:02:00.959 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: overall HEALTH_OK 2024-08-17T17:02:00.959 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: mon.a calling monitor election 2024-08-17T17:02:00.959 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: mon.b calling monitor election 2024-08-17T17:02:00.959 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-17T17:02:00.959 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-17T17:02:00.959 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-17T17:02:00.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: mon.c calling monitor election 2024-08-17T17:02:00.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: pgmap v193: 89 pgs: 89 active+clean; 97 MiB data, 519 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-08-17T17:02:00.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: pgmap v194: 89 pgs: 89 active+clean; 84 MiB data, 493 MiB used, 715 GiB / 715 GiB avail; 2.3 MiB/s rd, 982 KiB/s wr, 7 op/s 2024-08-17T17:02:00.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: pgmap v195: 89 pgs: 89 active+clean; 84 MiB data, 493 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 864 KiB/s wr, 6 op/s 2024-08-17T17:02:00.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-17T17:02:00.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: overall HEALTH_OK 2024-08-17T17:02:00.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: mon.a calling monitor election 2024-08-17T17:02:00.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: mon.b calling monitor election 2024-08-17T17:02:00.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:02:00.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: monmap epoch 4 2024-08-17T17:02:00.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:02:00.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: last_changed 2024-08-17T17:01:55.543839+0000 2024-08-17T17:02:00.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: created 2024-08-17T16:46:51.197040+0000 2024-08-17T17:02:00.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: min_mon_release 19 (squid) 2024-08-17T17:02:00.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: election_strategy: 1 2024-08-17T17:02:00.961 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: 0: [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon.a 2024-08-17T17:02:00.961 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: 1: [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] mon.b 2024-08-17T17:02:00.961 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: 2: [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] mon.c 2024-08-17T17:02:00.961 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T17:02:00.961 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: osdmap e492: 8 total, 8 up, 8 in 2024-08-17T17:02:00.961 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: mgrmap e40: y(active, since 2m), standbys: x 2024-08-17T17:02:00.961 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: overall HEALTH_OK 2024-08-17T17:02:00.961 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='' 2024-08-17T17:02:00.961 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:00.961 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:00.961 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:00 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:02:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:02:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: pgmap v191: 89 pgs: 89 active+clean; 97 MiB data, 519 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-08-17T17:02:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: 2: [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] mon.c 2024-08-17T17:02:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T17:02:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: osdmap e492: 8 total, 8 up, 8 in 2024-08-17T17:02:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: mgrmap e40: y(active, since 2m), standbys: x 2024-08-17T17:02:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: overall HEALTH_OK 2024-08-17T17:02:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: mon.a calling monitor election 2024-08-17T17:02:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: mon.b calling monitor election 2024-08-17T17:02:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-17T17:02:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-17T17:02:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-17T17:02:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: mon.c calling monitor election 2024-08-17T17:02:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: pgmap v193: 89 pgs: 89 active+clean; 97 MiB data, 519 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-08-17T17:02:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: pgmap v194: 89 pgs: 89 active+clean; 84 MiB data, 493 MiB used, 715 GiB / 715 GiB avail; 2.3 MiB/s rd, 982 KiB/s wr, 7 op/s 2024-08-17T17:02:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: pgmap v195: 89 pgs: 89 active+clean; 84 MiB data, 493 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 864 KiB/s wr, 6 op/s 2024-08-17T17:02:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-17T17:02:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: overall HEALTH_OK 2024-08-17T17:02:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: mon.a calling monitor election 2024-08-17T17:02:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: mon.b calling monitor election 2024-08-17T17:02:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:02:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: monmap epoch 4 2024-08-17T17:02:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:02:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: last_changed 2024-08-17T17:01:55.543839+0000 2024-08-17T17:02:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: created 2024-08-17T16:46:51.197040+0000 2024-08-17T17:02:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: min_mon_release 19 (squid) 2024-08-17T17:02:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: election_strategy: 1 2024-08-17T17:02:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: 0: [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon.a 2024-08-17T17:02:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: 1: [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] mon.b 2024-08-17T17:02:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: 2: [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] mon.c 2024-08-17T17:02:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T17:02:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: osdmap e492: 8 total, 8 up, 8 in 2024-08-17T17:02:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: mgrmap e40: y(active, since 2m), standbys: x 2024-08-17T17:02:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: overall HEALTH_OK 2024-08-17T17:02:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='' 2024-08-17T17:02:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:01.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:02:01.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-08-17T17:02:01.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: pgmap v191: 89 pgs: 89 active+clean; 97 MiB data, 519 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-08-17T17:02:01.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: 2: [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] mon.c 2024-08-17T17:02:01.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T17:02:01.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: osdmap e492: 8 total, 8 up, 8 in 2024-08-17T17:02:01.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: mgrmap e40: y(active, since 2m), standbys: x 2024-08-17T17:02:01.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: overall HEALTH_OK 2024-08-17T17:02:01.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: mon.a calling monitor election 2024-08-17T17:02:01.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: mon.b calling monitor election 2024-08-17T17:02:01.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-08-17T17:02:01.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-08-17T17:02:01.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-08-17T17:02:01.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: mon.c calling monitor election 2024-08-17T17:02:01.033 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: pgmap v193: 89 pgs: 89 active+clean; 97 MiB data, 519 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.2 MiB/s wr, 7 op/s 2024-08-17T17:02:01.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: pgmap v194: 89 pgs: 89 active+clean; 84 MiB data, 493 MiB used, 715 GiB / 715 GiB avail; 2.3 MiB/s rd, 982 KiB/s wr, 7 op/s 2024-08-17T17:02:01.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: pgmap v195: 89 pgs: 89 active+clean; 84 MiB data, 493 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 864 KiB/s wr, 6 op/s 2024-08-17T17:02:01.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-08-17T17:02:01.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: overall HEALTH_OK 2024-08-17T17:02:01.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: mon.a calling monitor election 2024-08-17T17:02:01.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: mon.b calling monitor election 2024-08-17T17:02:01.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-08-17T17:02:01.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: monmap epoch 4 2024-08-17T17:02:01.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:02:01.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: last_changed 2024-08-17T17:01:55.543839+0000 2024-08-17T17:02:01.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: created 2024-08-17T16:46:51.197040+0000 2024-08-17T17:02:01.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: min_mon_release 19 (squid) 2024-08-17T17:02:01.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: election_strategy: 1 2024-08-17T17:02:01.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: 0: [v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] mon.a 2024-08-17T17:02:01.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: 1: [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] mon.b 2024-08-17T17:02:01.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: 2: [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0] mon.c 2024-08-17T17:02:01.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: fsmap foo:1 {0=foo.smithi063.ultpqj=up:active} 1 up:standby 2024-08-17T17:02:01.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: osdmap e492: 8 total, 8 up, 8 in 2024-08-17T17:02:01.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: mgrmap e40: y(active, since 2m), standbys: x 2024-08-17T17:02:01.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: overall HEALTH_OK 2024-08-17T17:02:01.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='' 2024-08-17T17:02:01.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:01.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:01.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:00 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:02:01.635 INFO:tasks.rados.rados.0.smithi063.stdout:1701: done (1 left) 2024-08-17T17:02:01.635 INFO:tasks.rados.rados.0.smithi063.stdout:1702: done (0 left) 2024-08-17T17:02:01.635 INFO:tasks.rados.rados.0.smithi063.stdout:1703: write oid 37 current snap is 209 2024-08-17T17:02:01.635 INFO:tasks.rados.rados.0.smithi063.stdout:1703: seq_num 654 ranges {591778=419204,1637106=741879,2910049=81729} 2024-08-17T17:02:01.640 INFO:tasks.rados.rados.0.smithi063.stdout:1703: writing smithi06387500-37 from 591778 to 1010982 tid 1 2024-08-17T17:02:01.648 INFO:tasks.rados.rados.0.smithi063.stdout:1703: writing smithi06387500-37 from 1637106 to 2378985 tid 2 2024-08-17T17:02:01.651 INFO:tasks.rados.rados.0.smithi063.stdout:1703: writing smithi06387500-37 from 2910049 to 2991778 tid 3 2024-08-17T17:02:01.651 INFO:tasks.rados.rados.0.smithi063.stdout:1704: snap_remove snap 206 2024-08-17T17:02:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:01 smithi162 ceph-mon[104668]: osdmap e493: 8 total, 8 up, 8 in 2024-08-17T17:02:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:02:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:01 smithi063 ceph-mon[109843]: osdmap e493: 8 total, 8 up, 8 in 2024-08-17T17:02:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:02:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:01 smithi063 ceph-mon[114756]: osdmap e493: 8 total, 8 up, 8 in 2024-08-17T17:02:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:02:02.534 INFO:tasks.rados.rados.0.smithi063.stdout:1703: finishing write tid 1 to smithi06387500-37 2024-08-17T17:02:02.534 INFO:tasks.rados.rados.0.smithi063.stdout:1703: finishing write tid 2 to smithi06387500-37 2024-08-17T17:02:02.534 INFO:tasks.rados.rados.0.smithi063.stdout:1703: finishing write tid 3 to smithi06387500-37 2024-08-17T17:02:02.534 INFO:tasks.rados.rados.0.smithi063.stdout:1703: finishing write tid 4 to smithi06387500-37 2024-08-17T17:02:02.534 INFO:tasks.rados.rados.0.smithi063.stdout:1703: finishing write tid 5 to smithi06387500-37 2024-08-17T17:02:02.535 INFO:tasks.rados.rados.0.smithi063.stdout:1703: oid 37 updating version 0 to 1662 2024-08-17T17:02:02.535 INFO:tasks.rados.rados.0.smithi063.stdout:1703: oid 37 updating version 1662 to 1663 2024-08-17T17:02:02.535 INFO:tasks.rados.rados.0.smithi063.stdout:1703: oid 37 updating version 1663 to 1664 2024-08-17T17:02:02.535 INFO:tasks.rados.rados.0.smithi063.stdout:1703: oid 37 version 1664 is already newer than 1661 2024-08-17T17:02:02.535 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 1664 (ObjNum 654 snap 209 seq_num 654) dirty exists 2024-08-17T17:02:02.535 INFO:tasks.rados.rados.0.smithi063.stdout:1703: left oid 37 (ObjNum 654 snap 209 seq_num 654) 2024-08-17T17:02:02.535 INFO:tasks.rados.rados.0.smithi063.stdout:1703: done (1 left) 2024-08-17T17:02:02.535 INFO:tasks.rados.rados.0.smithi063.stdout:1704: done (0 left) 2024-08-17T17:02:02.535 INFO:tasks.rados.rados.0.smithi063.stdout:1705: write oid 49 current snap is 209 2024-08-17T17:02:02.535 INFO:tasks.rados.rados.0.smithi063.stdout:1705: seq_num 655 ranges {608585=494453,1579235=601890,2821119=187466} 2024-08-17T17:02:02.540 INFO:tasks.rados.rados.0.smithi063.stdout:1705: writing smithi06387500-49 from 608585 to 1103038 tid 1 2024-08-17T17:02:02.546 INFO:tasks.rados.rados.0.smithi063.stdout:1705: writing smithi06387500-49 from 1579235 to 2181125 tid 2 2024-08-17T17:02:02.551 INFO:tasks.rados.rados.0.smithi063.stdout:1705: writing smithi06387500-49 from 2821119 to 3008585 tid 3 2024-08-17T17:02:02.551 INFO:tasks.rados.rados.0.smithi063.stdout:1706: read oid 12 snap -1 2024-08-17T17:02:02.551 INFO:tasks.rados.rados.0.smithi063.stdout:1706: expect (ObjNum 612 snap 193 seq_num 612) 2024-08-17T17:02:02.551 INFO:tasks.rados.rados.0.smithi063.stdout:1707: read oid 6 snap -1 2024-08-17T17:02:02.551 INFO:tasks.rados.rados.0.smithi063.stdout:1707: expect (ObjNum 425 snap 133 seq_num 425) 2024-08-17T17:02:02.552 INFO:tasks.rados.rados.0.smithi063.stdout:1708: delete oid 30 current snap is 209 2024-08-17T17:02:02.552 INFO:tasks.rados.rados.0.smithi063.stdout:1705: finishing write tid 1 to smithi06387500-49 2024-08-17T17:02:02.552 INFO:tasks.rados.rados.0.smithi063.stdout:1705: finishing write tid 2 to smithi06387500-49 2024-08-17T17:02:02.556 INFO:tasks.rados.rados.0.smithi063.stdout:1705: finishing write tid 3 to smithi06387500-49 2024-08-17T17:02:02.557 INFO:tasks.rados.rados.0.smithi063.stdout:1708: done (3 left) 2024-08-17T17:02:02.557 INFO:tasks.rados.rados.0.smithi063.stdout:1709: write oid 39 current snap is 209 2024-08-17T17:02:02.557 INFO:tasks.rados.rados.0.smithi063.stdout:1709: seq_num 657 ranges {642199=644951,1696199=605559,2992318=49881} 2024-08-17T17:02:02.565 INFO:tasks.rados.rados.0.smithi063.stdout:1709: writing smithi06387500-39 from 642199 to 1287150 tid 1 2024-08-17T17:02:02.570 INFO:tasks.rados.rados.0.smithi063.stdout:1709: writing smithi06387500-39 from 1696199 to 2301758 tid 2 2024-08-17T17:02:02.575 INFO:tasks.rados.rados.0.smithi063.stdout:1709: writing smithi06387500-39 from 2992318 to 3042199 tid 3 2024-08-17T17:02:02.575 INFO:tasks.rados.rados.0.smithi063.stdout:1710: delete oid 47 current snap is 209 2024-08-17T17:02:02.575 INFO:tasks.rados.rados.0.smithi063.stdout:1705: finishing write tid 4 to smithi06387500-49 2024-08-17T17:02:02.575 INFO:tasks.rados.rados.0.smithi063.stdout:1705: finishing write tid 5 to smithi06387500-49 2024-08-17T17:02:02.575 INFO:tasks.rados.rados.0.smithi063.stdout:1705: oid 49 updating version 0 to 1666 2024-08-17T17:02:02.575 INFO:tasks.rados.rados.0.smithi063.stdout:1705: oid 49 updating version 1666 to 1669 2024-08-17T17:02:02.575 INFO:tasks.rados.rados.0.smithi063.stdout:1705: oid 49 version 1669 is already newer than 1667 2024-08-17T17:02:02.575 INFO:tasks.rados.rados.0.smithi063.stdout:1705: oid 49 version 1669 is already newer than 1668 2024-08-17T17:02:02.575 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 1669 (ObjNum 655 snap 209 seq_num 655) dirty exists 2024-08-17T17:02:02.575 INFO:tasks.rados.rados.0.smithi063.stdout:1705: left oid 49 (ObjNum 655 snap 209 seq_num 655) 2024-08-17T17:02:02.575 INFO:tasks.rados.rados.0.smithi063.stdout:1707: expect (ObjNum 425 snap 133 seq_num 425) 2024-08-17T17:02:02.681 INFO:tasks.rados.rados.0.smithi063.stdout:1706: expect (ObjNum 612 snap 193 seq_num 612) 2024-08-17T17:02:02.760 INFO:tasks.rados.rados.0.smithi063.stdout:1709: finishing write tid 1 to smithi06387500-39 2024-08-17T17:02:02.760 INFO:tasks.rados.rados.0.smithi063.stdout:1709: finishing write tid 2 to smithi06387500-39 2024-08-17T17:02:02.760 INFO:tasks.rados.rados.0.smithi063.stdout:1709: finishing write tid 3 to smithi06387500-39 2024-08-17T17:02:02.760 INFO:tasks.rados.rados.0.smithi063.stdout:1709: finishing write tid 4 to smithi06387500-39 2024-08-17T17:02:02.760 INFO:tasks.rados.rados.0.smithi063.stdout:1709: finishing write tid 5 to smithi06387500-39 2024-08-17T17:02:02.760 INFO:tasks.rados.rados.0.smithi063.stdout:1709: oid 39 updating version 0 to 366 2024-08-17T17:02:02.760 INFO:tasks.rados.rados.0.smithi063.stdout:1709: oid 39 updating version 366 to 369 2024-08-17T17:02:02.760 INFO:tasks.rados.rados.0.smithi063.stdout:1709: oid 39 version 369 is already newer than 367 2024-08-17T17:02:02.761 INFO:tasks.rados.rados.0.smithi063.stdout:1709: oid 39 version 369 is already newer than 368 2024-08-17T17:02:02.761 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 369 (ObjNum 657 snap 209 seq_num 657) dirty exists 2024-08-17T17:02:02.761 INFO:tasks.rados.rados.0.smithi063.stdout:1709: left oid 39 (ObjNum 657 snap 209 seq_num 657) 2024-08-17T17:02:02.761 INFO:tasks.rados.rados.0.smithi063.stdout:1705: done (4 left) 2024-08-17T17:02:02.761 INFO:tasks.rados.rados.0.smithi063.stdout:1706: done (3 left) 2024-08-17T17:02:02.761 INFO:tasks.rados.rados.0.smithi063.stdout:1707: done (2 left) 2024-08-17T17:02:02.761 INFO:tasks.rados.rados.0.smithi063.stdout:1709: done (1 left) 2024-08-17T17:02:02.761 INFO:tasks.rados.rados.0.smithi063.stdout:1710: done (0 left) 2024-08-17T17:02:02.761 INFO:tasks.rados.rados.0.smithi063.stdout:1711: snap_create 2024-08-17T17:02:03.007 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:02 smithi162 ceph-mon[104668]: osdmap e494: 8 total, 8 up, 8 in 2024-08-17T17:02:03.007 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:02 smithi162 ceph-mon[104668]: pgmap v198: 89 pgs: 89 active+clean; 89 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 1.5 MiB/s rd, 381 KiB/s wr, 2 op/s 2024-08-17T17:02:03.007 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:02 smithi162 ceph-mon[104668]: osdmap e495: 8 total, 8 up, 8 in 2024-08-17T17:02:03.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:02 smithi063 ceph-mon[109843]: osdmap e494: 8 total, 8 up, 8 in 2024-08-17T17:02:03.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:02 smithi063 ceph-mon[109843]: pgmap v198: 89 pgs: 89 active+clean; 89 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 1.5 MiB/s rd, 381 KiB/s wr, 2 op/s 2024-08-17T17:02:03.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:02 smithi063 ceph-mon[109843]: osdmap e495: 8 total, 8 up, 8 in 2024-08-17T17:02:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:02 smithi063 ceph-mon[114756]: osdmap e494: 8 total, 8 up, 8 in 2024-08-17T17:02:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:02 smithi063 ceph-mon[114756]: pgmap v198: 89 pgs: 89 active+clean; 89 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 1.5 MiB/s rd, 381 KiB/s wr, 2 op/s 2024-08-17T17:02:03.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:02 smithi063 ceph-mon[114756]: osdmap e495: 8 total, 8 up, 8 in 2024-08-17T17:02:03.663 INFO:tasks.rados.rados.0.smithi063.stdout:1711: done (0 left) 2024-08-17T17:02:03.663 INFO:tasks.rados.rados.0.smithi063.stdout:1712: snap_remove snap 208 2024-08-17T17:02:03.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:02:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:02:03] "GET /metrics HTTP/1.1" 200 36217 "" "Prometheus/2.51.0" 2024-08-17T17:02:04.671 INFO:tasks.rados.rados.0.smithi063.stdout:1712: done (0 left) 2024-08-17T17:02:04.671 INFO:tasks.rados.rados.0.smithi063.stdout:1713: delete oid 29 current snap is 210 2024-08-17T17:02:04.674 INFO:tasks.rados.rados.0.smithi063.stdout:1713: done (0 left) 2024-08-17T17:02:04.674 INFO:tasks.rados.rados.0.smithi063.stdout:1714: rollback oid 42 current snap is 210 2024-08-17T17:02:04.674 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 42 to 207 2024-08-17T17:02:04.674 INFO:tasks.rados.rados.0.smithi063.stdout:1715: write oid 17 current snap is 210 2024-08-17T17:02:04.674 INFO:tasks.rados.rados.0.smithi063.stdout:1715: seq_num 660 ranges {692620=470698,1755292=752886} 2024-08-17T17:02:04.680 INFO:tasks.rados.rados.0.smithi063.stdout:1715: writing smithi06387500-17 from 692620 to 1163318 tid 1 2024-08-17T17:02:04.689 INFO:tasks.rados.rados.0.smithi063.stdout:1715: writing smithi06387500-17 from 1755292 to 2508178 tid 2 2024-08-17T17:02:04.689 INFO:tasks.rados.rados.0.smithi063.stdout:1716: write oid 23 current snap is 210 2024-08-17T17:02:04.689 INFO:tasks.rados.rados.0.smithi063.stdout:1714: finishing rollback tid 1 to smithi06387500-42 2024-08-17T17:02:04.689 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T17:02:04.689 INFO:tasks.rados.rados.0.smithi063.stdout:1715: finishing write tid 1 to smithi06387500-17 2024-08-17T17:02:04.689 INFO:tasks.rados.rados.0.smithi063.stdout:1716: seq_num 661 ranges {709427=545947,1813774=612897,2985657=123770} 2024-08-17T17:02:04.695 INFO:tasks.rados.rados.0.smithi063.stdout:1716: writing smithi06387500-23 from 709427 to 1255374 tid 1 2024-08-17T17:02:04.702 INFO:tasks.rados.rados.0.smithi063.stdout:1716: writing smithi06387500-23 from 1813774 to 2426671 tid 2 2024-08-17T17:02:04.706 INFO:tasks.rados.rados.0.smithi063.stdout:1716: writing smithi06387500-23 from 2985657 to 3109427 tid 3 2024-08-17T17:02:04.706 INFO:tasks.rados.rados.0.smithi063.stdout:1714: done (2 left) 2024-08-17T17:02:04.706 INFO:tasks.rados.rados.0.smithi063.stdout:1717: write oid 47 current snap is 210 2024-08-17T17:02:04.706 INFO:tasks.rados.rados.0.smithi063.stdout:1715: finishing write tid 2 to smithi06387500-17 2024-08-17T17:02:04.706 INFO:tasks.rados.rados.0.smithi063.stdout:1715: finishing write tid 3 to smithi06387500-17 2024-08-17T17:02:04.706 INFO:tasks.rados.rados.0.smithi063.stdout:1715: finishing write tid 4 to smithi06387500-17 2024-08-17T17:02:04.706 INFO:tasks.rados.rados.0.smithi063.stdout:1715: oid 17 updating version 0 to 516 2024-08-17T17:02:04.706 INFO:tasks.rados.rados.0.smithi063.stdout:1715: oid 17 updating version 516 to 517 2024-08-17T17:02:04.706 INFO:tasks.rados.rados.0.smithi063.stdout:1715: oid 17 version 517 is already newer than 515 2024-08-17T17:02:04.706 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 517 (ObjNum 660 snap 210 seq_num 660) dirty exists 2024-08-17T17:02:04.706 INFO:tasks.rados.rados.0.smithi063.stdout:1715: left oid 17 (ObjNum 660 snap 210 seq_num 660) 2024-08-17T17:02:04.707 INFO:tasks.rados.rados.0.smithi063.stdout:1716: finishing write tid 1 to smithi06387500-23 2024-08-17T17:02:04.707 INFO:tasks.rados.rados.0.smithi063.stdout:1716: finishing write tid 2 to smithi06387500-23 2024-08-17T17:02:04.707 INFO:tasks.rados.rados.0.smithi063.stdout:1717: seq_num 662 ranges {726234=737549,1988609=756555} 2024-08-17T17:02:04.713 INFO:tasks.rados.rados.0.smithi063.stdout:1717: writing smithi06387500-47 from 726234 to 1463783 tid 1 2024-08-17T17:02:04.721 INFO:tasks.rados.rados.0.smithi063.stdout:1717: writing smithi06387500-47 from 1988609 to 2745164 tid 2 2024-08-17T17:02:04.721 INFO:tasks.rados.rados.0.smithi063.stdout:1715: done (2 left) 2024-08-17T17:02:04.721 INFO:tasks.rados.rados.0.smithi063.stdout:1718: read oid 24 snap -1 2024-08-17T17:02:04.721 INFO:tasks.rados.rados.0.smithi063.stdout:1718: expect (ObjNum 583 snap 186 seq_num 583) 2024-08-17T17:02:04.722 INFO:tasks.rados.rados.0.smithi063.stdout:1719: read oid 39 snap -1 2024-08-17T17:02:04.722 INFO:tasks.rados.rados.0.smithi063.stdout:1719: expect (ObjNum 657 snap 209 seq_num 657) 2024-08-17T17:02:04.722 INFO:tasks.rados.rados.0.smithi063.stdout:1720: write oid 38 current snap is 210 2024-08-17T17:02:04.722 INFO:tasks.rados.rados.0.smithi063.stdout:1716: finishing write tid 3 to smithi06387500-23 2024-08-17T17:02:04.722 INFO:tasks.rados.rados.0.smithi063.stdout:1716: finishing write tid 4 to smithi06387500-23 2024-08-17T17:02:04.722 INFO:tasks.rados.rados.0.smithi063.stdout:1716: finishing write tid 5 to smithi06387500-23 2024-08-17T17:02:04.722 INFO:tasks.rados.rados.0.smithi063.stdout:1716: oid 23 updating version 0 to 775 2024-08-17T17:02:04.722 INFO:tasks.rados.rados.0.smithi063.stdout:1716: oid 23 version 775 is already newer than 774 2024-08-17T17:02:04.722 INFO:tasks.rados.rados.0.smithi063.stdout:1716: oid 23 version 775 is already newer than 772 2024-08-17T17:02:04.722 INFO:tasks.rados.rados.0.smithi063.stdout:1716: oid 23 version 775 is already newer than 773 2024-08-17T17:02:04.722 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 775 (ObjNum 661 snap 210 seq_num 661) dirty exists 2024-08-17T17:02:04.722 INFO:tasks.rados.rados.0.smithi063.stdout:1716: left oid 23 (ObjNum 661 snap 210 seq_num 661) 2024-08-17T17:02:04.722 INFO:tasks.rados.rados.0.smithi063.stdout:1717: finishing write tid 1 to smithi06387500-47 2024-08-17T17:02:04.722 INFO:tasks.rados.rados.0.smithi063.stdout:1720: seq_num 663 ranges {743041=412798,1930738=616566,3040503=102538} 2024-08-17T17:02:04.728 INFO:tasks.rados.rados.0.smithi063.stdout:1720: writing smithi06387500-38 from 743041 to 1155839 tid 1 2024-08-17T17:02:04.735 INFO:tasks.rados.rados.0.smithi063.stdout:1720: writing smithi06387500-38 from 1930738 to 2547304 tid 2 2024-08-17T17:02:04.739 INFO:tasks.rados.rados.0.smithi063.stdout:1720: writing smithi06387500-38 from 3040503 to 3143041 tid 3 2024-08-17T17:02:04.739 INFO:tasks.rados.rados.0.smithi063.stdout:1716: done (4 left) 2024-08-17T17:02:04.739 INFO:tasks.rados.rados.0.smithi063.stdout:1721: snap_remove snap 209 2024-08-17T17:02:04.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:04 smithi162 ceph-mon[104668]: osdmap e496: 8 total, 8 up, 8 in 2024-08-17T17:02:04.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:04 smithi162 ceph-mon[104668]: pgmap v201: 89 pgs: 89 active+clean; 89 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 603 KiB/s wr, 2 op/s 2024-08-17T17:02:05.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:04 smithi063 ceph-mon[114756]: osdmap e496: 8 total, 8 up, 8 in 2024-08-17T17:02:05.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:04 smithi063 ceph-mon[114756]: pgmap v201: 89 pgs: 89 active+clean; 89 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 603 KiB/s wr, 2 op/s 2024-08-17T17:02:05.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:04 smithi063 ceph-mon[109843]: osdmap e496: 8 total, 8 up, 8 in 2024-08-17T17:02:05.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:04 smithi063 ceph-mon[109843]: pgmap v201: 89 pgs: 89 active+clean; 89 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 603 KiB/s wr, 2 op/s 2024-08-17T17:02:05.681 INFO:tasks.rados.rados.0.smithi063.stdout:1721: done (4 left) 2024-08-17T17:02:05.681 INFO:tasks.rados.rados.0.smithi063.stdout:1722: snap_remove snap 207 2024-08-17T17:02:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:05 smithi063 ceph-mon[114756]: osdmap e497: 8 total, 8 up, 8 in 2024-08-17T17:02:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:06.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:05 smithi063 ceph-mon[109843]: osdmap e497: 8 total, 8 up, 8 in 2024-08-17T17:02:06.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:06.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:05 smithi162 ceph-mon[104668]: osdmap e497: 8 total, 8 up, 8 in 2024-08-17T17:02:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:06.680 INFO:tasks.rados.rados.0.smithi063.stdout:1717: finishing write tid 2 to smithi06387500-47 2024-08-17T17:02:06.681 INFO:tasks.rados.rados.0.smithi063.stdout:1717: finishing write tid 3 to smithi06387500-47 2024-08-17T17:02:06.681 INFO:tasks.rados.rados.0.smithi063.stdout:1717: finishing write tid 4 to smithi06387500-47 2024-08-17T17:02:06.681 INFO:tasks.rados.rados.0.smithi063.stdout:1717: oid 47 updating version 0 to 340 2024-08-17T17:02:06.681 INFO:tasks.rados.rados.0.smithi063.stdout:1717: oid 47 version 340 is already newer than 339 2024-08-17T17:02:06.681 INFO:tasks.rados.rados.0.smithi063.stdout:1717: oid 47 version 340 is already newer than 338 2024-08-17T17:02:06.681 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 340 (ObjNum 662 snap 210 seq_num 662) dirty exists 2024-08-17T17:02:06.681 INFO:tasks.rados.rados.0.smithi063.stdout:1717: left oid 47 (ObjNum 662 snap 210 seq_num 662) 2024-08-17T17:02:06.681 INFO:tasks.rados.rados.0.smithi063.stdout:1718: expect (ObjNum 583 snap 186 seq_num 583) 2024-08-17T17:02:06.808 INFO:tasks.rados.rados.0.smithi063.stdout:1720: finishing write tid 1 to smithi06387500-38 2024-08-17T17:02:06.808 INFO:tasks.rados.rados.0.smithi063.stdout:1719: expect (ObjNum 657 snap 209 seq_num 657) 2024-08-17T17:02:07.021 INFO:tasks.rados.rados.0.smithi063.stdout:1720: finishing write tid 2 to smithi06387500-38 2024-08-17T17:02:07.021 INFO:tasks.rados.rados.0.smithi063.stdout:1720: finishing write tid 3 to smithi06387500-38 2024-08-17T17:02:07.021 INFO:tasks.rados.rados.0.smithi063.stdout:1720: finishing write tid 4 to smithi06387500-38 2024-08-17T17:02:07.022 INFO:tasks.rados.rados.0.smithi063.stdout:1720: finishing write tid 5 to smithi06387500-38 2024-08-17T17:02:07.022 INFO:tasks.rados.rados.0.smithi063.stdout:1720: oid 38 updating version 0 to 243 2024-08-17T17:02:07.022 INFO:tasks.rados.rados.0.smithi063.stdout:1720: oid 38 updating version 243 to 245 2024-08-17T17:02:07.022 INFO:tasks.rados.rados.0.smithi063.stdout:1720: oid 38 version 245 is already newer than 242 2024-08-17T17:02:07.022 INFO:tasks.rados.rados.0.smithi063.stdout:1720: oid 38 version 245 is already newer than 244 2024-08-17T17:02:07.022 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 245 (ObjNum 663 snap 210 seq_num 663) dirty exists 2024-08-17T17:02:07.022 INFO:tasks.rados.rados.0.smithi063.stdout:1720: left oid 38 (ObjNum 663 snap 210 seq_num 663) 2024-08-17T17:02:07.022 INFO:tasks.rados.rados.0.smithi063.stdout:1717: done (4 left) 2024-08-17T17:02:07.022 INFO:tasks.rados.rados.0.smithi063.stdout:1718: done (3 left) 2024-08-17T17:02:07.022 INFO:tasks.rados.rados.0.smithi063.stdout:1719: done (2 left) 2024-08-17T17:02:07.022 INFO:tasks.rados.rados.0.smithi063.stdout:1720: done (1 left) 2024-08-17T17:02:07.022 INFO:tasks.rados.rados.0.smithi063.stdout:1722: done (0 left) 2024-08-17T17:02:07.022 INFO:tasks.rados.rados.0.smithi063.stdout:1723: rollback oid 44 current snap is 210 2024-08-17T17:02:07.022 INFO:tasks.rados.rados.0.smithi063.stdout:1723: done (0 left) 2024-08-17T17:02:07.023 INFO:tasks.rados.rados.0.smithi063.stdout:1724: read oid 13 snap -1 2024-08-17T17:02:07.023 INFO:tasks.rados.rados.0.smithi063.stdout:1724: expect (ObjNum 630 snap 201 seq_num 630) 2024-08-17T17:02:07.023 INFO:tasks.rados.rados.0.smithi063.stdout:1725: delete oid 35 current snap is 210 2024-08-17T17:02:07.023 INFO:tasks.rados.rados.0.smithi063.stdout:1725: done (1 left) 2024-08-17T17:02:07.023 INFO:tasks.rados.rados.0.smithi063.stdout:1726: read oid 24 snap -1 2024-08-17T17:02:07.023 INFO:tasks.rados.rados.0.smithi063.stdout:1726: expect (ObjNum 583 snap 186 seq_num 583) 2024-08-17T17:02:07.024 INFO:tasks.rados.rados.0.smithi063.stdout:1727: read oid 43 snap -1 2024-08-17T17:02:07.024 INFO:tasks.rados.rados.0.smithi063.stdout:1727: expect (ObjNum 483 snap 153 seq_num 483) 2024-08-17T17:02:07.024 INFO:tasks.rados.rados.0.smithi063.stdout:1728: read oid 45 snap -1 2024-08-17T17:02:07.024 INFO:tasks.rados.rados.0.smithi063.stdout:1728: expect deleted 2024-08-17T17:02:07.024 INFO:tasks.rados.rados.0.smithi063.stdout:1729: write oid 36 current snap is 210 2024-08-17T17:02:07.024 INFO:tasks.rados.rados.0.smithi063.stdout:1729: seq_num 665 ranges {776655=563296,2047702=620235,3095349=81306} 2024-08-17T17:02:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:06 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:06 smithi063 ceph-mon[114756]: osdmap e498: 8 total, 8 up, 8 in 2024-08-17T17:02:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:06 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:06 smithi063 ceph-mon[114756]: pgmap v204: 89 pgs: 89 active+clean; 89 MiB data, 497 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:06 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:06 smithi063 ceph-mon[109843]: osdmap e498: 8 total, 8 up, 8 in 2024-08-17T17:02:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:06 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:06 smithi063 ceph-mon[109843]: pgmap v204: 89 pgs: 89 active+clean; 89 MiB data, 497 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:07.031 INFO:tasks.rados.rados.0.smithi063.stdout:1729: writing smithi06387500-36 from 776655 to 1339951 tid 1 2024-08-17T17:02:07.038 INFO:tasks.rados.rados.0.smithi063.stdout:1729: writing smithi06387500-36 from 2047702 to 2667937 tid 2 2024-08-17T17:02:07.041 INFO:tasks.rados.rados.0.smithi063.stdout:1729: writing smithi06387500-36 from 3095349 to 3176655 tid 3 2024-08-17T17:02:07.041 INFO:tasks.rados.rados.0.smithi063.stdout:1730: read oid 18 snap -1 2024-08-17T17:02:07.041 INFO:tasks.rados.rados.0.smithi063.stdout:1730: expect (ObjNum 647 snap 206 seq_num 647) 2024-08-17T17:02:07.041 INFO:tasks.rados.rados.0.smithi063.stdout:1731: delete oid 50 current snap is 210 2024-08-17T17:02:07.042 INFO:tasks.rados.rados.0.smithi063.stdout:1726: expect (ObjNum 583 snap 186 seq_num 583) 2024-08-17T17:02:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:06 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:06 smithi162 ceph-mon[104668]: osdmap e498: 8 total, 8 up, 8 in 2024-08-17T17:02:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:06 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:06 smithi162 ceph-mon[104668]: pgmap v204: 89 pgs: 89 active+clean; 89 MiB data, 497 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:07.166 INFO:tasks.rados.rados.0.smithi063.stdout:1727: expect (ObjNum 483 snap 153 seq_num 483) 2024-08-17T17:02:07.238 INFO:tasks.rados.rados.0.smithi063.stdout:1724: expect (ObjNum 630 snap 201 seq_num 630) 2024-08-17T17:02:07.357 INFO:tasks.rados.rados.0.smithi063.stdout:1729: finishing write tid 1 to smithi06387500-36 2024-08-17T17:02:07.357 INFO:tasks.rados.rados.0.smithi063.stdout:1729: finishing write tid 2 to smithi06387500-36 2024-08-17T17:02:07.357 INFO:tasks.rados.rados.0.smithi063.stdout:1729: finishing write tid 3 to smithi06387500-36 2024-08-17T17:02:07.357 INFO:tasks.rados.rados.0.smithi063.stdout:1729: finishing write tid 4 to smithi06387500-36 2024-08-17T17:02:07.357 INFO:tasks.rados.rados.0.smithi063.stdout:1729: finishing write tid 5 to smithi06387500-36 2024-08-17T17:02:07.357 INFO:tasks.rados.rados.0.smithi063.stdout:1729: oid 36 updating version 0 to 1677 2024-08-17T17:02:07.357 INFO:tasks.rados.rados.0.smithi063.stdout:1729: oid 36 version 1677 is already newer than 1675 2024-08-17T17:02:07.357 INFO:tasks.rados.rados.0.smithi063.stdout:1729: oid 36 updating version 1677 to 1678 2024-08-17T17:02:07.357 INFO:tasks.rados.rados.0.smithi063.stdout:1729: oid 36 version 1678 is already newer than 1676 2024-08-17T17:02:07.358 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 1678 (ObjNum 665 snap 210 seq_num 665) dirty exists 2024-08-17T17:02:07.358 INFO:tasks.rados.rados.0.smithi063.stdout:1729: left oid 36 (ObjNum 665 snap 210 seq_num 665) 2024-08-17T17:02:07.358 INFO:tasks.rados.rados.0.smithi063.stdout:1730: expect (ObjNum 647 snap 206 seq_num 647) 2024-08-17T17:02:07.452 INFO:tasks.rados.rados.0.smithi063.stdout:1724: done (6 left) 2024-08-17T17:02:07.452 INFO:tasks.rados.rados.0.smithi063.stdout:1726: done (5 left) 2024-08-17T17:02:07.452 INFO:tasks.rados.rados.0.smithi063.stdout:1727: done (4 left) 2024-08-17T17:02:07.452 INFO:tasks.rados.rados.0.smithi063.stdout:1728: done (3 left) 2024-08-17T17:02:07.452 INFO:tasks.rados.rados.0.smithi063.stdout:1729: done (2 left) 2024-08-17T17:02:07.453 INFO:tasks.rados.rados.0.smithi063.stdout:1730: done (1 left) 2024-08-17T17:02:07.453 INFO:tasks.rados.rados.0.smithi063.stdout:1731: done (0 left) 2024-08-17T17:02:07.453 INFO:tasks.rados.rados.0.smithi063.stdout:1732: delete oid 36 current snap is 210 2024-08-17T17:02:07.454 INFO:tasks.rados.rados.0.smithi063.stdout:1732: done (0 left) 2024-08-17T17:02:07.454 INFO:tasks.rados.rados.0.smithi063.stdout:1733: delete oid 27 current snap is 210 2024-08-17T17:02:07.455 INFO:tasks.rados.rados.0.smithi063.stdout:1733: done (0 left) 2024-08-17T17:02:07.455 INFO:tasks.rados.rados.0.smithi063.stdout:1734: read oid 13 snap -1 2024-08-17T17:02:07.455 INFO:tasks.rados.rados.0.smithi063.stdout:1734: expect (ObjNum 630 snap 201 seq_num 630) 2024-08-17T17:02:07.456 INFO:tasks.rados.rados.0.smithi063.stdout:1735: snap_create 2024-08-17T17:02:07.460 INFO:tasks.rados.rados.0.smithi063.stdout:1734: expect (ObjNum 630 snap 201 seq_num 630) 2024-08-17T17:02:07.700 INFO:tasks.rados.rados.0.smithi063.stdout:1734: done (1 left) 2024-08-17T17:02:07.700 INFO:tasks.rados.rados.0.smithi063.stdout:1735: done (0 left) 2024-08-17T17:02:07.700 INFO:tasks.rados.rados.0.smithi063.stdout:1736: snap_create 2024-08-17T17:02:07.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:07 smithi162 ceph-mon[104668]: osdmap e499: 8 total, 8 up, 8 in 2024-08-17T17:02:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:07 smithi063 ceph-mon[114756]: osdmap e499: 8 total, 8 up, 8 in 2024-08-17T17:02:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:07 smithi063 ceph-mon[109843]: osdmap e499: 8 total, 8 up, 8 in 2024-08-17T17:02:08.707 INFO:tasks.rados.rados.0.smithi063.stdout:1736: done (0 left) 2024-08-17T17:02:08.708 INFO:tasks.rados.rados.0.smithi063.stdout:1737: write oid 50 current snap is 212 2024-08-17T17:02:08.708 INFO:tasks.rados.rados.0.smithi063.stdout:1737: seq_num 669 ranges {443883=464292,1365277=511220,2572335=671548} 2024-08-17T17:02:08.712 INFO:tasks.rados.rados.0.smithi063.stdout:1737: writing smithi06387500-50 from 443883 to 908175 tid 1 2024-08-17T17:02:08.718 INFO:tasks.rados.rados.0.smithi063.stdout:1737: writing smithi06387500-50 from 1365277 to 1876497 tid 2 2024-08-17T17:02:08.726 INFO:tasks.rados.rados.0.smithi063.stdout:1737: writing smithi06387500-50 from 2572335 to 3243883 tid 3 2024-08-17T17:02:08.726 INFO:tasks.rados.rados.0.smithi063.stdout:1738: read oid 40 snap -1 2024-08-17T17:02:08.726 INFO:tasks.rados.rados.0.smithi063.stdout:1738: expect deleted 2024-08-17T17:02:08.726 INFO:tasks.rados.rados.0.smithi063.stdout:1739: snap_create 2024-08-17T17:02:08.726 INFO:tasks.rados.rados.0.smithi063.stdout:1737: finishing write tid 1 to smithi06387500-50 2024-08-17T17:02:08.726 INFO:tasks.rados.rados.0.smithi063.stdout:1737: finishing write tid 2 to smithi06387500-50 2024-08-17T17:02:08.729 INFO:tasks.rados.rados.0.smithi063.stdout:1737: finishing write tid 3 to smithi06387500-50 2024-08-17T17:02:08.729 INFO:tasks.rados.rados.0.smithi063.stdout:1737: finishing write tid 4 to smithi06387500-50 2024-08-17T17:02:08.729 INFO:tasks.rados.rados.0.smithi063.stdout:1737: finishing write tid 5 to smithi06387500-50 2024-08-17T17:02:08.729 INFO:tasks.rados.rados.0.smithi063.stdout:1737: oid 50 updating version 0 to 1229 2024-08-17T17:02:08.729 INFO:tasks.rados.rados.0.smithi063.stdout:1737: oid 50 updating version 1229 to 1230 2024-08-17T17:02:08.729 INFO:tasks.rados.rados.0.smithi063.stdout:1737: oid 50 version 1230 is already newer than 1227 2024-08-17T17:02:08.729 INFO:tasks.rados.rados.0.smithi063.stdout:1737: oid 50 version 1230 is already newer than 1228 2024-08-17T17:02:08.729 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 1230 (ObjNum 669 snap 212 seq_num 669) dirty exists 2024-08-17T17:02:08.729 INFO:tasks.rados.rados.0.smithi063.stdout:1737: left oid 50 (ObjNum 669 snap 212 seq_num 669) 2024-08-17T17:02:09.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:08 smithi063 ceph-mon[114756]: osdmap e500: 8 total, 8 up, 8 in 2024-08-17T17:02:09.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:08 smithi063 ceph-mon[114756]: pgmap v207: 89 pgs: 89 active+clean; 81 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-08-17T17:02:09.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:08 smithi063 ceph-mon[109843]: osdmap e500: 8 total, 8 up, 8 in 2024-08-17T17:02:09.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:08 smithi063 ceph-mon[109843]: pgmap v207: 89 pgs: 89 active+clean; 81 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-08-17T17:02:09.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:08 smithi162 ceph-mon[104668]: osdmap e500: 8 total, 8 up, 8 in 2024-08-17T17:02:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:08 smithi162 ceph-mon[104668]: pgmap v207: 89 pgs: 89 active+clean; 81 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-08-17T17:02:09.708 INFO:tasks.rados.rados.0.smithi063.stdout:1737: done (2 left) 2024-08-17T17:02:09.708 INFO:tasks.rados.rados.0.smithi063.stdout:1738: done (1 left) 2024-08-17T17:02:09.708 INFO:tasks.rados.rados.0.smithi063.stdout:1739: done (0 left) 2024-08-17T17:02:09.709 INFO:tasks.rados.rados.0.smithi063.stdout:1740: read oid 25 snap -1 2024-08-17T17:02:09.709 INFO:tasks.rados.rados.0.smithi063.stdout:1740: expect (ObjNum 642 snap 205 seq_num 642) 2024-08-17T17:02:09.709 INFO:tasks.rados.rados.0.smithi063.stdout:1741: read oid 1 snap -1 2024-08-17T17:02:09.709 INFO:tasks.rados.rados.0.smithi063.stdout:1741: expect (ObjNum 650 snap 206 seq_num 650) 2024-08-17T17:02:09.709 INFO:tasks.rados.rados.0.smithi063.stdout:1742: write oid 29 current snap is 213 2024-08-17T17:02:09.709 INFO:tasks.rados.rados.0.smithi063.stdout:1742: seq_num 670 ranges {460690=655894,1540112=771231,2716111=539778} 2024-08-17T17:02:09.714 INFO:tasks.rados.rados.0.smithi063.stdout:1742: writing smithi06387500-29 from 460690 to 1116584 tid 1 2024-08-17T17:02:09.721 INFO:tasks.rados.rados.0.smithi063.stdout:1742: writing smithi06387500-29 from 1540112 to 2311343 tid 2 2024-08-17T17:02:09.726 INFO:tasks.rados.rados.0.smithi063.stdout:1742: writing smithi06387500-29 from 2716111 to 3255889 tid 3 2024-08-17T17:02:09.727 INFO:tasks.rados.rados.0.smithi063.stdout:1743: delete oid 3 current snap is 213 2024-08-17T17:02:09.727 INFO:tasks.rados.rados.0.smithi063.stdout:1742: finishing write tid 1 to smithi06387500-29 2024-08-17T17:02:09.727 INFO:tasks.rados.rados.0.smithi063.stdout:1740: expect (ObjNum 642 snap 205 seq_num 642) 2024-08-17T17:02:09.858 INFO:tasks.rados.rados.0.smithi063.stdout:1741: expect (ObjNum 650 snap 206 seq_num 650) 2024-08-17T17:02:10.017 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[109843]: osdmap e501: 8 total, 8 up, 8 in 2024-08-17T17:02:10.018 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:10.018 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:10.018 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:10.018 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:02:10.018 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:10.018 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[109843]: Reconfiguring mon.a (monmap changed)... 2024-08-17T17:02:10.018 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:02:10.018 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:02:10.018 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:10.018 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[109843]: Reconfiguring daemon mon.a on smithi063 2024-08-17T17:02:10.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[114756]: osdmap e501: 8 total, 8 up, 8 in 2024-08-17T17:02:10.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:10.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:10.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:10.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:02:10.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:10.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[114756]: Reconfiguring mon.a (monmap changed)... 2024-08-17T17:02:10.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:02:10.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:02:10.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:10.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:09 smithi063 ceph-mon[114756]: Reconfiguring daemon mon.a on smithi063 2024-08-17T17:02:10.045 INFO:tasks.rados.rados.0.smithi063.stdout:1742: finishing write tid 2 to smithi06387500-29 2024-08-17T17:02:10.045 INFO:tasks.rados.rados.0.smithi063.stdout:1742: finishing write tid 3 to smithi06387500-29 2024-08-17T17:02:10.045 INFO:tasks.rados.rados.0.smithi063.stdout:1742: finishing write tid 4 to smithi06387500-29 2024-08-17T17:02:10.045 INFO:tasks.rados.rados.0.smithi063.stdout:1742: finishing write tid 5 to smithi06387500-29 2024-08-17T17:02:10.045 INFO:tasks.rados.rados.0.smithi063.stdout:1742: oid 29 updating version 0 to 418 2024-08-17T17:02:10.046 INFO:tasks.rados.rados.0.smithi063.stdout:1742: oid 29 version 418 is already newer than 417 2024-08-17T17:02:10.046 INFO:tasks.rados.rados.0.smithi063.stdout:1742: oid 29 version 418 is already newer than 416 2024-08-17T17:02:10.046 INFO:tasks.rados.rados.0.smithi063.stdout:1742: oid 29 version 418 is already newer than 415 2024-08-17T17:02:10.046 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 418 (ObjNum 670 snap 213 seq_num 670) dirty exists 2024-08-17T17:02:10.046 INFO:tasks.rados.rados.0.smithi063.stdout:1742: left oid 29 (ObjNum 670 snap 213 seq_num 670) 2024-08-17T17:02:10.046 INFO:tasks.rados.rados.0.smithi063.stdout:1740: done (3 left) 2024-08-17T17:02:10.046 INFO:tasks.rados.rados.0.smithi063.stdout:1741: done (2 left) 2024-08-17T17:02:10.046 INFO:tasks.rados.rados.0.smithi063.stdout:1742: done (1 left) 2024-08-17T17:02:10.046 INFO:tasks.rados.rados.0.smithi063.stdout:1743: done (0 left) 2024-08-17T17:02:10.046 INFO:tasks.rados.rados.0.smithi063.stdout:1744: delete oid 44 current snap is 213 2024-08-17T17:02:10.047 INFO:tasks.rados.rados.0.smithi063.stdout:1744: done (0 left) 2024-08-17T17:02:10.047 INFO:tasks.rados.rados.0.smithi063.stdout:1745: write oid 25 current snap is 213 2024-08-17T17:02:10.047 INFO:tasks.rados.rados.0.smithi063.stdout:1745: seq_num 673 ranges {511111=481641,1599205=518558,2682027=489947} 2024-08-17T17:02:10.052 INFO:tasks.rados.rados.0.smithi063.stdout:1745: writing smithi06387500-25 from 511111 to 992752 tid 1 2024-08-17T17:02:10.058 INFO:tasks.rados.rados.0.smithi063.stdout:1745: writing smithi06387500-25 from 1599205 to 2117763 tid 2 2024-08-17T17:02:10.064 INFO:tasks.rados.rados.0.smithi063.stdout:1745: writing smithi06387500-25 from 2682027 to 3171974 tid 3 2024-08-17T17:02:10.064 INFO:tasks.rados.rados.0.smithi063.stdout:1746: snap_remove snap 212 2024-08-17T17:02:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:09 smithi162 ceph-mon[104668]: osdmap e501: 8 total, 8 up, 8 in 2024-08-17T17:02:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:09 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:09 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:09 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:09 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:02:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:09 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:09 smithi162 ceph-mon[104668]: Reconfiguring mon.a (monmap changed)... 2024-08-17T17:02:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:09 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:02:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:09 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:02:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:09 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:09 smithi162 ceph-mon[104668]: Reconfiguring daemon mon.a on smithi063 2024-08-17T17:02:10.713 INFO:tasks.rados.rados.0.smithi063.stdout:1746: done (1 left) 2024-08-17T17:02:10.714 INFO:tasks.rados.rados.0.smithi063.stdout:1747: snap_remove snap 211 2024-08-17T17:02:11.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[109843]: osdmap e502: 8 total, 8 up, 8 in 2024-08-17T17:02:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[109843]: pgmap v210: 89 pgs: 89 active+clean; 81 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-08-17T17:02:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[109843]: Reconfiguring mgr.y (monmap changed)... 2024-08-17T17:02:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:02:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:02:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:02:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[109843]: Reconfiguring daemon mgr.y on smithi063 2024-08-17T17:02:11.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[114756]: osdmap e502: 8 total, 8 up, 8 in 2024-08-17T17:02:11.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[114756]: pgmap v210: 89 pgs: 89 active+clean; 81 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-08-17T17:02:11.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:11.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:11.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[114756]: Reconfiguring mgr.y (monmap changed)... 2024-08-17T17:02:11.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:02:11.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:02:11.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:02:11.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:11.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:10 smithi063 ceph-mon[114756]: Reconfiguring daemon mgr.y on smithi063 2024-08-17T17:02:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:10 smithi162 ceph-mon[104668]: osdmap e502: 8 total, 8 up, 8 in 2024-08-17T17:02:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:10 smithi162 ceph-mon[104668]: pgmap v210: 89 pgs: 89 active+clean; 81 MiB data, 507 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-08-17T17:02:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:10 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:10 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:10 smithi162 ceph-mon[104668]: Reconfiguring mgr.y (monmap changed)... 2024-08-17T17:02:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:10 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:02:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:10 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:02:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:10 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:02:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:10 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:10 smithi162 ceph-mon[104668]: Reconfiguring daemon mgr.y on smithi063 2024-08-17T17:02:11.716 INFO:tasks.rados.rados.0.smithi063.stdout:1747: done (1 left) 2024-08-17T17:02:11.716 INFO:tasks.rados.rados.0.smithi063.stdout:1748: snap_remove snap 210 2024-08-17T17:02:11.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:11 smithi063 ceph-mon[114756]: osdmap e503: 8 total, 8 up, 8 in 2024-08-17T17:02:11.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:11 smithi063 ceph-mon[109843]: osdmap e503: 8 total, 8 up, 8 in 2024-08-17T17:02:12.015 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:11 smithi162 ceph-mon[104668]: osdmap e503: 8 total, 8 up, 8 in 2024-08-17T17:02:12.535 INFO:tasks.rados.rados.0.smithi063.stdout:1748: done (1 left) 2024-08-17T17:02:12.535 INFO:tasks.rados.rados.0.smithi063.stdout:1749: write oid 38 current snap is 213 2024-08-17T17:02:12.535 INFO:tasks.rados.rados.0.smithi063.stdout:1745: finishing write tid 1 to smithi06387500-25 2024-08-17T17:02:12.535 INFO:tasks.rados.rados.0.smithi063.stdout:1749: seq_num 674 ranges {527918=556890,1657687=778569,3225803=102115} 2024-08-17T17:02:12.540 INFO:tasks.rados.rados.0.smithi063.stdout:1749: writing smithi06387500-38 from 527918 to 1084808 tid 1 2024-08-17T17:02:12.548 INFO:tasks.rados.rados.0.smithi063.stdout:1749: writing smithi06387500-38 from 1657687 to 2436256 tid 2 2024-08-17T17:02:12.554 INFO:tasks.rados.rados.0.smithi063.stdout:1749: writing smithi06387500-38 from 3225803 to 3327918 tid 3 2024-08-17T17:02:12.554 INFO:tasks.rados.rados.0.smithi063.stdout:1750: rollback oid 29 current snap is 213 2024-08-17T17:02:12.554 INFO:tasks.rados.rados.0.smithi063.stdout:1750: done (2 left) 2024-08-17T17:02:12.554 INFO:tasks.rados.rados.0.smithi063.stdout:1751: write oid 34 current snap is 213 2024-08-17T17:02:12.554 INFO:tasks.rados.rados.0.smithi063.stdout:1745: finishing write tid 2 to smithi06387500-25 2024-08-17T17:02:12.554 INFO:tasks.rados.rados.0.smithi063.stdout:1745: finishing write tid 3 to smithi06387500-25 2024-08-17T17:02:12.554 INFO:tasks.rados.rados.0.smithi063.stdout:1745: finishing write tid 4 to smithi06387500-25 2024-08-17T17:02:12.555 INFO:tasks.rados.rados.0.smithi063.stdout:1745: finishing write tid 5 to smithi06387500-25 2024-08-17T17:02:12.555 INFO:tasks.rados.rados.0.smithi063.stdout:1745: oid 25 updating version 0 to 1682 2024-08-17T17:02:12.555 INFO:tasks.rados.rados.0.smithi063.stdout:1745: oid 25 version 1682 is already newer than 1681 2024-08-17T17:02:12.555 INFO:tasks.rados.rados.0.smithi063.stdout:1745: oid 25 updating version 1682 to 1683 2024-08-17T17:02:12.555 INFO:tasks.rados.rados.0.smithi063.stdout:1745: oid 25 updating version 1683 to 1684 2024-08-17T17:02:12.555 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 1684 (ObjNum 673 snap 213 seq_num 673) dirty exists 2024-08-17T17:02:12.555 INFO:tasks.rados.rados.0.smithi063.stdout:1745: left oid 25 (ObjNum 673 snap 213 seq_num 673) 2024-08-17T17:02:12.555 INFO:tasks.rados.rados.0.smithi063.stdout:1749: finishing write tid 1 to smithi06387500-38 2024-08-17T17:02:12.555 INFO:tasks.rados.rados.0.smithi063.stdout:1749: finishing write tid 2 to smithi06387500-38 2024-08-17T17:02:12.555 INFO:tasks.rados.rados.0.smithi063.stdout:1751: seq_num 675 ranges {544725=632139,1716169=522227,2736873=512491} 2024-08-17T17:02:12.560 INFO:tasks.rados.rados.0.smithi063.stdout:1751: writing smithi06387500-34 from 544725 to 1176864 tid 1 2024-08-17T17:02:12.567 INFO:tasks.rados.rados.0.smithi063.stdout:1751: writing smithi06387500-34 from 1716169 to 2238396 tid 2 2024-08-17T17:02:12.573 INFO:tasks.rados.rados.0.smithi063.stdout:1751: writing smithi06387500-34 from 2736873 to 3249364 tid 3 2024-08-17T17:02:12.573 INFO:tasks.rados.rados.0.smithi063.stdout:1745: done (2 left) 2024-08-17T17:02:12.573 INFO:tasks.rados.rados.0.smithi063.stdout:1752: read oid 18 snap -1 2024-08-17T17:02:12.573 INFO:tasks.rados.rados.0.smithi063.stdout:1752: expect (ObjNum 647 snap 206 seq_num 647) 2024-08-17T17:02:12.573 INFO:tasks.rados.rados.0.smithi063.stdout:1753: write oid 2 current snap is 213 2024-08-17T17:02:12.573 INFO:tasks.rados.rados.0.smithi063.stdout:1749: finishing write tid 3 to smithi06387500-38 2024-08-17T17:02:12.573 INFO:tasks.rados.rados.0.smithi063.stdout:1753: seq_num 676 ranges {561532=707388,2058298=782238} 2024-08-17T17:02:12.580 INFO:tasks.rados.rados.0.smithi063.stdout:1753: writing smithi06387500-2 from 561532 to 1268920 tid 1 2024-08-17T17:02:12.588 INFO:tasks.rados.rados.0.smithi063.stdout:1753: writing smithi06387500-2 from 2058298 to 2840536 tid 2 2024-08-17T17:02:12.589 INFO:tasks.rados.rados.0.smithi063.stdout:1754: write oid 17 current snap is 213 2024-08-17T17:02:12.589 INFO:tasks.rados.rados.0.smithi063.stdout:1754: seq_num 677 ranges {578339=498990,1833133=525896,2791719=586620} 2024-08-17T17:02:12.594 INFO:tasks.rados.rados.0.smithi063.stdout:1754: writing smithi06387500-17 from 578339 to 1077329 tid 1 2024-08-17T17:02:12.601 INFO:tasks.rados.rados.0.smithi063.stdout:1754: writing smithi06387500-17 from 1833133 to 2359029 tid 2 2024-08-17T17:02:12.608 INFO:tasks.rados.rados.0.smithi063.stdout:1754: writing smithi06387500-17 from 2791719 to 3378339 tid 3 2024-08-17T17:02:12.608 INFO:tasks.rados.rados.0.smithi063.stdout:1755: snap_create 2024-08-17T17:02:12.608 INFO:tasks.rados.rados.0.smithi063.stdout:1749: finishing write tid 4 to smithi06387500-38 2024-08-17T17:02:12.608 INFO:tasks.rados.rados.0.smithi063.stdout:1749: finishing write tid 5 to smithi06387500-38 2024-08-17T17:02:12.608 INFO:tasks.rados.rados.0.smithi063.stdout:1749: oid 38 updating version 0 to 251 2024-08-17T17:02:12.608 INFO:tasks.rados.rados.0.smithi063.stdout:1749: oid 38 updating version 251 to 253 2024-08-17T17:02:12.608 INFO:tasks.rados.rados.0.smithi063.stdout:1749: oid 38 version 253 is already newer than 252 2024-08-17T17:02:12.608 INFO:tasks.rados.rados.0.smithi063.stdout:1749: oid 38 version 253 is already newer than 250 2024-08-17T17:02:12.608 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 253 (ObjNum 674 snap 213 seq_num 674) dirty exists 2024-08-17T17:02:12.608 INFO:tasks.rados.rados.0.smithi063.stdout:1749: left oid 38 (ObjNum 674 snap 213 seq_num 674) 2024-08-17T17:02:12.608 INFO:tasks.rados.rados.0.smithi063.stdout:1751: finishing write tid 1 to smithi06387500-34 2024-08-17T17:02:12.608 INFO:tasks.rados.rados.0.smithi063.stdout:1751: finishing write tid 2 to smithi06387500-34 2024-08-17T17:02:12.608 INFO:tasks.rados.rados.0.smithi063.stdout:1751: finishing write tid 3 to smithi06387500-34 2024-08-17T17:02:12.609 INFO:tasks.rados.rados.0.smithi063.stdout:1751: finishing write tid 4 to smithi06387500-34 2024-08-17T17:02:12.609 INFO:tasks.rados.rados.0.smithi063.stdout:1752: expect (ObjNum 647 snap 206 seq_num 647) 2024-08-17T17:02:12.703 INFO:tasks.rados.rados.0.smithi063.stdout:1751: finishing write tid 5 to smithi06387500-34 2024-08-17T17:02:12.703 INFO:tasks.rados.rados.0.smithi063.stdout:1751: oid 34 updating version 0 to 1234 2024-08-17T17:02:12.703 INFO:tasks.rados.rados.0.smithi063.stdout:1751: oid 34 version 1234 is already newer than 1233 2024-08-17T17:02:12.703 INFO:tasks.rados.rados.0.smithi063.stdout:1751: oid 34 version 1234 is already newer than 1232 2024-08-17T17:02:12.704 INFO:tasks.rados.rados.0.smithi063.stdout:1751: oid 34 version 1234 is already newer than 1231 2024-08-17T17:02:12.704 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 1234 (ObjNum 675 snap 213 seq_num 675) dirty exists 2024-08-17T17:02:12.704 INFO:tasks.rados.rados.0.smithi063.stdout:1751: left oid 34 (ObjNum 675 snap 213 seq_num 675) 2024-08-17T17:02:12.704 INFO:tasks.rados.rados.0.smithi063.stdout:1753: finishing write tid 1 to smithi06387500-2 2024-08-17T17:02:12.704 INFO:tasks.rados.rados.0.smithi063.stdout:1753: finishing write tid 2 to smithi06387500-2 2024-08-17T17:02:12.704 INFO:tasks.rados.rados.0.smithi063.stdout:1753: finishing write tid 3 to smithi06387500-2 2024-08-17T17:02:12.704 INFO:tasks.rados.rados.0.smithi063.stdout:1753: finishing write tid 4 to smithi06387500-2 2024-08-17T17:02:12.704 INFO:tasks.rados.rados.0.smithi063.stdout:1753: oid 2 updating version 0 to 780 2024-08-17T17:02:12.704 INFO:tasks.rados.rados.0.smithi063.stdout:1753: oid 2 updating version 780 to 782 2024-08-17T17:02:12.704 INFO:tasks.rados.rados.0.smithi063.stdout:1753: oid 2 version 782 is already newer than 781 2024-08-17T17:02:12.704 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 782 (ObjNum 676 snap 213 seq_num 676) dirty exists 2024-08-17T17:02:12.704 INFO:tasks.rados.rados.0.smithi063.stdout:1753: left oid 2 (ObjNum 676 snap 213 seq_num 676) 2024-08-17T17:02:12.704 INFO:tasks.rados.rados.0.smithi063.stdout:1754: finishing write tid 1 to smithi06387500-17 2024-08-17T17:02:12.705 INFO:tasks.rados.rados.0.smithi063.stdout:1754: finishing write tid 2 to smithi06387500-17 2024-08-17T17:02:12.705 INFO:tasks.rados.rados.0.smithi063.stdout:1754: finishing write tid 3 to smithi06387500-17 2024-08-17T17:02:12.705 INFO:tasks.rados.rados.0.smithi063.stdout:1754: finishing write tid 4 to smithi06387500-17 2024-08-17T17:02:12.705 INFO:tasks.rados.rados.0.smithi063.stdout:1754: finishing write tid 5 to smithi06387500-17 2024-08-17T17:02:12.705 INFO:tasks.rados.rados.0.smithi063.stdout:1754: oid 17 updating version 0 to 525 2024-08-17T17:02:12.705 INFO:tasks.rados.rados.0.smithi063.stdout:1754: oid 17 version 525 is already newer than 524 2024-08-17T17:02:12.705 INFO:tasks.rados.rados.0.smithi063.stdout:1754: oid 17 version 525 is already newer than 523 2024-08-17T17:02:12.705 INFO:tasks.rados.rados.0.smithi063.stdout:1754: oid 17 updating version 525 to 526 2024-08-17T17:02:12.705 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 526 (ObjNum 677 snap 213 seq_num 677) dirty exists 2024-08-17T17:02:12.705 INFO:tasks.rados.rados.0.smithi063.stdout:1754: left oid 17 (ObjNum 677 snap 213 seq_num 677) 2024-08-17T17:02:12.868 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[114756]: osdmap e504: 8 total, 8 up, 8 in 2024-08-17T17:02:12.868 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[114756]: pgmap v213: 89 pgs: 89 active+clean; 83 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 1.5 MiB/s wr, 21 op/s 2024-08-17T17:02:12.868 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:12.869 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:12.869 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[114756]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:02:12.869 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:02:12.869 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:02:12.869 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:12.869 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[114756]: Reconfiguring daemon mon.c on smithi063 2024-08-17T17:02:12.869 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[114756]: osdmap e505: 8 total, 8 up, 8 in 2024-08-17T17:02:12.869 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[109843]: osdmap e504: 8 total, 8 up, 8 in 2024-08-17T17:02:12.869 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[109843]: pgmap v213: 89 pgs: 89 active+clean; 83 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 1.5 MiB/s wr, 21 op/s 2024-08-17T17:02:12.869 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:12.870 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:12.870 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[109843]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:02:12.870 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:02:12.870 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:02:12.870 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:12.870 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[109843]: Reconfiguring daemon mon.c on smithi063 2024-08-17T17:02:12.870 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:12 smithi063 ceph-mon[109843]: osdmap e505: 8 total, 8 up, 8 in 2024-08-17T17:02:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:12 smithi162 ceph-mon[104668]: osdmap e504: 8 total, 8 up, 8 in 2024-08-17T17:02:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:12 smithi162 ceph-mon[104668]: pgmap v213: 89 pgs: 89 active+clean; 83 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 1.5 MiB/s wr, 21 op/s 2024-08-17T17:02:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:12 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:12 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:12 smithi162 ceph-mon[104668]: Reconfiguring mon.c (monmap changed)... 2024-08-17T17:02:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:12 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:02:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:12 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:02:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:12 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:12 smithi162 ceph-mon[104668]: Reconfiguring daemon mon.c on smithi063 2024-08-17T17:02:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:12 smithi162 ceph-mon[104668]: osdmap e505: 8 total, 8 up, 8 in 2024-08-17T17:02:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:02:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:02:13] "GET /metrics HTTP/1.1" 200 36217 "" "Prometheus/2.51.0" 2024-08-17T17:02:13.537 INFO:tasks.rados.rados.0.smithi063.stdout:1749: done (5 left) 2024-08-17T17:02:13.538 INFO:tasks.rados.rados.0.smithi063.stdout:1751: done (4 left) 2024-08-17T17:02:13.538 INFO:tasks.rados.rados.0.smithi063.stdout:1752: done (3 left) 2024-08-17T17:02:13.538 INFO:tasks.rados.rados.0.smithi063.stdout:1753: done (2 left) 2024-08-17T17:02:13.538 INFO:tasks.rados.rados.0.smithi063.stdout:1754: done (1 left) 2024-08-17T17:02:13.538 INFO:tasks.rados.rados.0.smithi063.stdout:1755: done (0 left) 2024-08-17T17:02:13.538 INFO:tasks.rados.rados.0.smithi063.stdout:1756: snap_remove snap 213 2024-08-17T17:02:14.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:14 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:14.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:14 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:14.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:14 smithi063 ceph-mon[114756]: Reconfiguring osd.0 (monmap changed)... 2024-08-17T17:02:14.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:14 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:02:14.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:14 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:14.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:14 smithi063 ceph-mon[114756]: Reconfiguring daemon osd.0 on smithi063 2024-08-17T17:02:14.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:14 smithi063 ceph-mon[114756]: osdmap e506: 8 total, 8 up, 8 in 2024-08-17T17:02:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:14 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:14 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:14 smithi063 ceph-mon[109843]: Reconfiguring osd.0 (monmap changed)... 2024-08-17T17:02:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:14 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:02:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:14 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:14 smithi063 ceph-mon[109843]: Reconfiguring daemon osd.0 on smithi063 2024-08-17T17:02:14.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:14 smithi063 ceph-mon[109843]: osdmap e506: 8 total, 8 up, 8 in 2024-08-17T17:02:14.543 INFO:tasks.rados.rados.0.smithi063.stdout:1756: done (0 left) 2024-08-17T17:02:14.543 INFO:tasks.rados.rados.0.smithi063.stdout:1757: snap_create 2024-08-17T17:02:14.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:14 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:14.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:14 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:14.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:14 smithi162 ceph-mon[104668]: Reconfiguring osd.0 (monmap changed)... 2024-08-17T17:02:14.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:14 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:02:14.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:14 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:14.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:14 smithi162 ceph-mon[104668]: Reconfiguring daemon osd.0 on smithi063 2024-08-17T17:02:14.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:14 smithi162 ceph-mon[104668]: osdmap e506: 8 total, 8 up, 8 in 2024-08-17T17:02:15.548 INFO:tasks.rados.rados.0.smithi063.stdout:1757: done (0 left) 2024-08-17T17:02:15.548 INFO:tasks.rados.rados.0.smithi063.stdout:1758: read oid 19 snap -1 2024-08-17T17:02:15.548 INFO:tasks.rados.rados.0.smithi063.stdout:1758: expect deleted 2024-08-17T17:02:15.549 INFO:tasks.rados.rados.0.smithi063.stdout:1759: snap_create 2024-08-17T17:02:15.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:15 smithi063 ceph-mon[109843]: pgmap v216: 89 pgs: 89 active+clean; 83 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 1.5 MiB/s wr, 21 op/s 2024-08-17T17:02:15.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:15 smithi063 ceph-mon[109843]: osdmap e507: 8 total, 8 up, 8 in 2024-08-17T17:02:15.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:15 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:15.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:15 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:15.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:15 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:02:15.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:15 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:15.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:15 smithi063 ceph-mon[114756]: pgmap v216: 89 pgs: 89 active+clean; 83 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 1.5 MiB/s wr, 21 op/s 2024-08-17T17:02:15.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:15 smithi063 ceph-mon[114756]: osdmap e507: 8 total, 8 up, 8 in 2024-08-17T17:02:15.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:15 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:15.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:15 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:15.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:15 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:02:15.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:15 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:15.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:15 smithi162 ceph-mon[104668]: pgmap v216: 89 pgs: 89 active+clean; 83 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 1.5 MiB/s wr, 21 op/s 2024-08-17T17:02:15.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:15 smithi162 ceph-mon[104668]: osdmap e507: 8 total, 8 up, 8 in 2024-08-17T17:02:15.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:15 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:15.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:15 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:15.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:15 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:02:15.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:15 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:16.567 INFO:tasks.rados.rados.0.smithi063.stdout:1758: done (1 left) 2024-08-17T17:02:16.567 INFO:tasks.rados.rados.0.smithi063.stdout:1759: done (0 left) 2024-08-17T17:02:16.567 INFO:tasks.rados.rados.0.smithi063.stdout:1760: delete oid 42 current snap is 216 2024-08-17T17:02:16.571 INFO:tasks.rados.rados.0.smithi063.stdout:1760: done (0 left) 2024-08-17T17:02:16.571 INFO:tasks.rados.rados.0.smithi063.stdout:1761: rollback oid 48 current snap is 216 2024-08-17T17:02:16.571 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 48 to 215 2024-08-17T17:02:16.571 INFO:tasks.rados.rados.0.smithi063.stdout:1762: write oid 24 current snap is 216 2024-08-17T17:02:16.571 INFO:tasks.rados.rados.0.smithi063.stdout:1762: seq_num 679 ranges {611953=649488,1950097=529565,3246565=165388} 2024-08-17T17:02:16.578 INFO:tasks.rados.rados.0.smithi063.stdout:1762: writing smithi06387500-24 from 611953 to 1261441 tid 1 2024-08-17T17:02:16.584 INFO:tasks.rados.rados.0.smithi063.stdout:1762: writing smithi06387500-24 from 1950097 to 2479662 tid 2 2024-08-17T17:02:16.590 INFO:tasks.rados.rados.0.smithi063.stdout:1762: writing smithi06387500-24 from 3246565 to 3411953 tid 3 2024-08-17T17:02:16.590 INFO:tasks.rados.rados.0.smithi063.stdout:1763: delete oid 11 current snap is 216 2024-08-17T17:02:16.590 INFO:tasks.rados.rados.0.smithi063.stdout:1761: finishing rollback tid 1 to smithi06387500-48 2024-08-17T17:02:16.590 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 0 (ObjNum 129 snap 0 seq_num 1806935528) dirty dne 2024-08-17T17:02:16.590 INFO:tasks.rados.rados.0.smithi063.stdout:1762: finishing write tid 1 to smithi06387500-24 2024-08-17T17:02:16.591 INFO:tasks.rados.rados.0.smithi063.stdout:1762: finishing write tid 2 to smithi06387500-24 2024-08-17T17:02:16.592 INFO:tasks.rados.rados.0.smithi063.stdout:1761: done (2 left) 2024-08-17T17:02:16.592 INFO:tasks.rados.rados.0.smithi063.stdout:1763: done (1 left) 2024-08-17T17:02:16.592 INFO:tasks.rados.rados.0.smithi063.stdout:1764: read oid 33 snap 215 2024-08-17T17:02:16.592 INFO:tasks.rados.rados.0.smithi063.stdout:1764: expect (ObjNum 636 snap 203 seq_num 636) 2024-08-17T17:02:16.592 INFO:tasks.rados.rados.0.smithi063.stdout:1765: write oid 31 current snap is 216 2024-08-17T17:02:16.592 INFO:tasks.rados.rados.0.smithi063.stdout:1765: seq_num 681 ranges {645567=799986,1950708=533234,3185058=260509} 2024-08-17T17:02:16.600 INFO:tasks.rados.rados.0.smithi063.stdout:1765: writing smithi06387500-31 from 645567 to 1445553 tid 1 2024-08-17T17:02:16.606 INFO:tasks.rados.rados.0.smithi063.stdout:1765: writing smithi06387500-31 from 1950708 to 2483942 tid 2 2024-08-17T17:02:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:1765: writing smithi06387500-31 from 3185058 to 3445567 tid 3 2024-08-17T17:02:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:1766: snap_create 2024-08-17T17:02:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:1762: finishing write tid 3 to smithi06387500-24 2024-08-17T17:02:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:1762: finishing write tid 4 to smithi06387500-24 2024-08-17T17:02:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:1762: finishing write tid 5 to smithi06387500-24 2024-08-17T17:02:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:1762: oid 24 updating version 0 to 656 2024-08-17T17:02:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:1762: oid 24 updating version 656 to 658 2024-08-17T17:02:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:1762: oid 24 updating version 658 to 659 2024-08-17T17:02:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:1762: oid 24 version 659 is already newer than 657 2024-08-17T17:02:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 659 (ObjNum 679 snap 216 seq_num 679) dirty exists 2024-08-17T17:02:16.613 INFO:tasks.rados.rados.0.smithi063.stdout:1762: left oid 24 (ObjNum 679 snap 216 seq_num 679) 2024-08-17T17:02:16.613 INFO:tasks.rados.rados.0.smithi063.stdout:1764: expect (ObjNum 636 snap 203 seq_num 636) 2024-08-17T17:02:16.769 INFO:tasks.rados.rados.0.smithi063.stdout:1765: finishing write tid 1 to smithi06387500-31 2024-08-17T17:02:16.770 INFO:tasks.rados.rados.0.smithi063.stdout:1765: finishing write tid 2 to smithi06387500-31 2024-08-17T17:02:16.770 INFO:tasks.rados.rados.0.smithi063.stdout:1765: finishing write tid 3 to smithi06387500-31 2024-08-17T17:02:16.770 INFO:tasks.rados.rados.0.smithi063.stdout:1765: finishing write tid 4 to smithi06387500-31 2024-08-17T17:02:16.770 INFO:tasks.rados.rados.0.smithi063.stdout:1765: finishing write tid 5 to smithi06387500-31 2024-08-17T17:02:16.770 INFO:tasks.rados.rados.0.smithi063.stdout:1765: oid 31 updating version 0 to 1238 2024-08-17T17:02:16.770 INFO:tasks.rados.rados.0.smithi063.stdout:1765: oid 31 version 1238 is already newer than 1237 2024-08-17T17:02:16.770 INFO:tasks.rados.rados.0.smithi063.stdout:1765: oid 31 updating version 1238 to 1239 2024-08-17T17:02:16.770 INFO:tasks.rados.rados.0.smithi063.stdout:1765: oid 31 updating version 1239 to 1240 2024-08-17T17:02:16.770 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 1240 (ObjNum 681 snap 216 seq_num 681) dirty exists 2024-08-17T17:02:16.770 INFO:tasks.rados.rados.0.smithi063.stdout:1765: left oid 31 (ObjNum 681 snap 216 seq_num 681) 2024-08-17T17:02:16.822 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[114756]: Reconfiguring osd.1 (monmap changed)... 2024-08-17T17:02:16.822 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[114756]: Reconfiguring daemon osd.1 on smithi063 2024-08-17T17:02:16.822 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[114756]: osdmap e508: 8 total, 8 up, 8 in 2024-08-17T17:02:16.822 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:16.822 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:16.822 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:02:16.822 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:02:16.823 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:16.823 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[109843]: Reconfiguring osd.1 (monmap changed)... 2024-08-17T17:02:16.823 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[109843]: Reconfiguring daemon osd.1 on smithi063 2024-08-17T17:02:16.823 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[109843]: osdmap e508: 8 total, 8 up, 8 in 2024-08-17T17:02:16.823 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:16.823 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:16.823 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:02:16.823 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:02:16.823 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:16 smithi162 ceph-mon[104668]: Reconfiguring osd.1 (monmap changed)... 2024-08-17T17:02:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:16 smithi162 ceph-mon[104668]: Reconfiguring daemon osd.1 on smithi063 2024-08-17T17:02:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:16 smithi162 ceph-mon[104668]: osdmap e508: 8 total, 8 up, 8 in 2024-08-17T17:02:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:16 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:16 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:02:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:02:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:17.574 INFO:tasks.rados.rados.0.smithi063.stdout:1762: done (3 left) 2024-08-17T17:02:17.574 INFO:tasks.rados.rados.0.smithi063.stdout:1764: done (2 left) 2024-08-17T17:02:17.574 INFO:tasks.rados.rados.0.smithi063.stdout:1765: done (1 left) 2024-08-17T17:02:17.574 INFO:tasks.rados.rados.0.smithi063.stdout:1766: done (0 left) 2024-08-17T17:02:17.574 INFO:tasks.rados.rados.0.smithi063.stdout:1767: rollback oid 26 current snap is 217 2024-08-17T17:02:17.575 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 26 to 214 2024-08-17T17:02:17.575 INFO:tasks.rados.rados.0.smithi063.stdout:1768: read oid 24 snap -1 2024-08-17T17:02:17.575 INFO:tasks.rados.rados.0.smithi063.stdout:1768: expect (ObjNum 679 snap 216 seq_num 679) 2024-08-17T17:02:17.575 INFO:tasks.rados.rados.0.smithi063.stdout:1769: delete oid 31 current snap is 217 2024-08-17T17:02:17.579 INFO:tasks.rados.rados.0.smithi063.stdout:1768: expect (ObjNum 679 snap 216 seq_num 679) 2024-08-17T17:02:17.747 INFO:tasks.rados.rados.0.smithi063.stdout:1767: finishing rollback tid 0 to smithi06387500-26 2024-08-17T17:02:17.747 INFO:tasks.rados.rados.0.smithi063.stdout:1767: finishing rollback tid 1 to smithi06387500-26 2024-08-17T17:02:17.747 INFO:tasks.rados.rados.0.smithi063.stdout:1767: finishing rollback tid 2 to smithi06387500-26 2024-08-17T17:02:17.747 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 530 (ObjNum 596 snap 188 seq_num 596) dirty exists 2024-08-17T17:02:17.747 INFO:tasks.rados.rados.0.smithi063.stdout:1767: done (2 left) 2024-08-17T17:02:17.747 INFO:tasks.rados.rados.0.smithi063.stdout:1768: done (1 left) 2024-08-17T17:02:17.747 INFO:tasks.rados.rados.0.smithi063.stdout:1769: done (0 left) 2024-08-17T17:02:17.747 INFO:tasks.rados.rados.0.smithi063.stdout:1770: rollback oid 40 current snap is 217 2024-08-17T17:02:17.747 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 40 to 214 2024-08-17T17:02:17.748 INFO:tasks.rados.rados.0.smithi063.stdout:1771: read oid 30 snap -1 2024-08-17T17:02:17.748 INFO:tasks.rados.rados.0.smithi063.stdout:1771: expect deleted 2024-08-17T17:02:17.748 INFO:tasks.rados.rados.0.smithi063.stdout:1772: snap_create 2024-08-17T17:02:17.748 INFO:tasks.rados.rados.0.smithi063.stdout:1770: finishing rollback tid 1 to smithi06387500-40 2024-08-17T17:02:17.748 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 0 (ObjNum 33 snap 0 seq_num 1806987088) dirty dne 2024-08-17T17:02:17.763 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:17 smithi063 ceph-mon[109843]: pgmap v219: 89 pgs: 89 active+clean; 83 MiB data, 495 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:17.763 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:17 smithi063 ceph-mon[109843]: Reconfiguring osd.2 (monmap changed)... 2024-08-17T17:02:17.763 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:17 smithi063 ceph-mon[109843]: Reconfiguring daemon osd.2 on smithi063 2024-08-17T17:02:17.763 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:17 smithi063 ceph-mon[109843]: osdmap e509: 8 total, 8 up, 8 in 2024-08-17T17:02:17.763 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:17 smithi063 ceph-mon[114756]: pgmap v219: 89 pgs: 89 active+clean; 83 MiB data, 495 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:17.763 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:17 smithi063 ceph-mon[114756]: Reconfiguring osd.2 (monmap changed)... 2024-08-17T17:02:17.763 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:17 smithi063 ceph-mon[114756]: Reconfiguring daemon osd.2 on smithi063 2024-08-17T17:02:17.764 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:17 smithi063 ceph-mon[114756]: osdmap e509: 8 total, 8 up, 8 in 2024-08-17T17:02:17.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:17 smithi162 ceph-mon[104668]: pgmap v219: 89 pgs: 89 active+clean; 83 MiB data, 495 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:17.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:17 smithi162 ceph-mon[104668]: Reconfiguring osd.2 (monmap changed)... 2024-08-17T17:02:17.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:17 smithi162 ceph-mon[104668]: Reconfiguring daemon osd.2 on smithi063 2024-08-17T17:02:17.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:17 smithi162 ceph-mon[104668]: osdmap e509: 8 total, 8 up, 8 in 2024-08-17T17:02:18.575 INFO:tasks.rados.rados.0.smithi063.stdout:1770: done (2 left) 2024-08-17T17:02:18.575 INFO:tasks.rados.rados.0.smithi063.stdout:1771: done (1 left) 2024-08-17T17:02:18.575 INFO:tasks.rados.rados.0.smithi063.stdout:1772: done (0 left) 2024-08-17T17:02:18.575 INFO:tasks.rados.rados.0.smithi063.stdout:1773: write oid 35 current snap is 218 2024-08-17T17:02:18.575 INFO:tasks.rados.rados.0.smithi063.stdout:1773: seq_num 683 ranges {679181=550484,1667672=536903,2839904=435373} 2024-08-17T17:02:18.581 INFO:tasks.rados.rados.0.smithi063.stdout:1773: writing smithi06387500-35 from 679181 to 1229665 tid 1 2024-08-17T17:02:18.586 INFO:tasks.rados.rados.0.smithi063.stdout:1773: writing smithi06387500-35 from 1667672 to 2204575 tid 2 2024-08-17T17:02:18.592 INFO:tasks.rados.rados.0.smithi063.stdout:1773: writing smithi06387500-35 from 2839904 to 3275277 tid 3 2024-08-17T17:02:18.593 INFO:tasks.rados.rados.0.smithi063.stdout:1774: write oid 48 current snap is 218 2024-08-17T17:02:18.593 INFO:tasks.rados.rados.0.smithi063.stdout:1773: finishing write tid 1 to smithi06387500-35 2024-08-17T17:02:18.593 INFO:tasks.rados.rados.0.smithi063.stdout:1773: finishing write tid 2 to smithi06387500-35 2024-08-17T17:02:18.593 INFO:tasks.rados.rados.0.smithi063.stdout:1774: seq_num 684 ranges {695988=625733,2009801=680561,3150974=345014} 2024-08-17T17:02:18.600 INFO:tasks.rados.rados.0.smithi063.stdout:1774: writing smithi06387500-48 from 695988 to 1321721 tid 1 2024-08-17T17:02:18.607 INFO:tasks.rados.rados.0.smithi063.stdout:1774: writing smithi06387500-48 from 2009801 to 2690362 tid 2 2024-08-17T17:02:18.613 INFO:tasks.rados.rados.0.smithi063.stdout:1774: writing smithi06387500-48 from 3150974 to 3495988 tid 3 2024-08-17T17:02:18.613 INFO:tasks.rados.rados.0.smithi063.stdout:1775: rollback oid 20 current snap is 218 2024-08-17T17:02:18.613 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 20 to 214 2024-08-17T17:02:18.613 INFO:tasks.rados.rados.0.smithi063.stdout:1773: finishing write tid 3 to smithi06387500-35 2024-08-17T17:02:18.613 INFO:tasks.rados.rados.0.smithi063.stdout:1773: finishing write tid 4 to smithi06387500-35 2024-08-17T17:02:18.613 INFO:tasks.rados.rados.0.smithi063.stdout:1773: finishing write tid 5 to smithi06387500-35 2024-08-17T17:02:18.613 INFO:tasks.rados.rados.0.smithi063.stdout:1773: oid 35 updating version 0 to 1695 2024-08-17T17:02:18.613 INFO:tasks.rados.rados.0.smithi063.stdout:1773: oid 35 version 1695 is already newer than 1693 2024-08-17T17:02:18.613 INFO:tasks.rados.rados.0.smithi063.stdout:1773: oid 35 version 1695 is already newer than 1692 2024-08-17T17:02:18.613 INFO:tasks.rados.rados.0.smithi063.stdout:1773: oid 35 version 1695 is already newer than 1694 2024-08-17T17:02:18.613 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 1695 (ObjNum 683 snap 218 seq_num 683) dirty exists 2024-08-17T17:02:18.613 INFO:tasks.rados.rados.0.smithi063.stdout:1773: left oid 35 (ObjNum 683 snap 218 seq_num 683) 2024-08-17T17:02:18.613 INFO:tasks.rados.rados.0.smithi063.stdout:1774: finishing write tid 1 to smithi06387500-48 2024-08-17T17:02:18.614 INFO:tasks.rados.rados.0.smithi063.stdout:1774: finishing write tid 2 to smithi06387500-48 2024-08-17T17:02:18.614 INFO:tasks.rados.rados.0.smithi063.stdout:1773: done (2 left) 2024-08-17T17:02:18.614 INFO:tasks.rados.rados.0.smithi063.stdout:1776: read oid 35 snap -1 2024-08-17T17:02:18.614 INFO:tasks.rados.rados.0.smithi063.stdout:1776: expect (ObjNum 683 snap 218 seq_num 683) 2024-08-17T17:02:18.614 INFO:tasks.rados.rados.0.smithi063.stdout:1777: snap_create 2024-08-17T17:02:18.614 INFO:tasks.rados.rados.0.smithi063.stdout:1775: finishing rollback tid 1 to smithi06387500-20 2024-08-17T17:02:18.614 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 0 (ObjNum 65 snap 0 seq_num 1953066355) dirty dne 2024-08-17T17:02:18.615 INFO:tasks.rados.rados.0.smithi063.stdout:1774: finishing write tid 3 to smithi06387500-48 2024-08-17T17:02:18.624 INFO:tasks.rados.rados.0.smithi063.stdout:1774: finishing write tid 4 to smithi06387500-48 2024-08-17T17:02:18.624 INFO:tasks.rados.rados.0.smithi063.stdout:1774: finishing write tid 5 to smithi06387500-48 2024-08-17T17:02:18.624 INFO:tasks.rados.rados.0.smithi063.stdout:1774: oid 48 updating version 0 to 1697 2024-08-17T17:02:18.624 INFO:tasks.rados.rados.0.smithi063.stdout:1774: oid 48 updating version 1697 to 1698 2024-08-17T17:02:18.624 INFO:tasks.rados.rados.0.smithi063.stdout:1774: oid 48 updating version 1698 to 1699 2024-08-17T17:02:18.624 INFO:tasks.rados.rados.0.smithi063.stdout:1774: oid 48 version 1699 is already newer than 1696 2024-08-17T17:02:18.624 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 1699 (ObjNum 684 snap 218 seq_num 684) dirty exists 2024-08-17T17:02:18.624 INFO:tasks.rados.rados.0.smithi063.stdout:1774: left oid 48 (ObjNum 684 snap 218 seq_num 684) 2024-08-17T17:02:18.626 INFO:tasks.rados.rados.0.smithi063.stdout:1776: expect (ObjNum 683 snap 218 seq_num 683) 2024-08-17T17:02:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:18 smithi063 ceph-mon[114756]: osdmap e510: 8 total, 8 up, 8 in 2024-08-17T17:02:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:18 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:18 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:18 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:02:18.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:18 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:18 smithi063 ceph-mon[109843]: osdmap e510: 8 total, 8 up, 8 in 2024-08-17T17:02:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:18 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:18 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:18 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:02:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:18 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:18.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:18 smithi162 ceph-mon[104668]: osdmap e510: 8 total, 8 up, 8 in 2024-08-17T17:02:18.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:18 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:18.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:18 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:18.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:18 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:02:18.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:18 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:19.583 INFO:tasks.rados.rados.0.smithi063.stdout:1774: done (3 left) 2024-08-17T17:02:19.584 INFO:tasks.rados.rados.0.smithi063.stdout:1775: done (2 left) 2024-08-17T17:02:19.584 INFO:tasks.rados.rados.0.smithi063.stdout:1776: done (1 left) 2024-08-17T17:02:19.584 INFO:tasks.rados.rados.0.smithi063.stdout:1777: done (0 left) 2024-08-17T17:02:19.584 INFO:tasks.rados.rados.0.smithi063.stdout:1778: delete oid 48 current snap is 219 2024-08-17T17:02:19.588 INFO:tasks.rados.rados.0.smithi063.stdout:1778: done (0 left) 2024-08-17T17:02:19.588 INFO:tasks.rados.rados.0.smithi063.stdout:1779: write oid 37 current snap is 219 2024-08-17T17:02:19.588 INFO:tasks.rados.rados.0.smithi063.stdout:1779: seq_num 686 ranges {729602=492584,1843118=684230,3322173=207429} 2024-08-17T17:02:19.595 INFO:tasks.rados.rados.0.smithi063.stdout:1779: writing smithi06387500-37 from 729602 to 1222186 tid 1 2024-08-17T17:02:19.603 INFO:tasks.rados.rados.0.smithi063.stdout:1779: writing smithi06387500-37 from 1843118 to 2527348 tid 2 2024-08-17T17:02:19.609 INFO:tasks.rados.rados.0.smithi063.stdout:1779: writing smithi06387500-37 from 3322173 to 3529602 tid 3 2024-08-17T17:02:19.609 INFO:tasks.rados.rados.0.smithi063.stdout:1780: snap_create 2024-08-17T17:02:19.609 INFO:tasks.rados.rados.0.smithi063.stdout:1779: finishing write tid 1 to smithi06387500-37 2024-08-17T17:02:19.609 INFO:tasks.rados.rados.0.smithi063.stdout:1779: finishing write tid 2 to smithi06387500-37 2024-08-17T17:02:19.612 INFO:tasks.rados.rados.0.smithi063.stdout:1779: finishing write tid 3 to smithi06387500-37 2024-08-17T17:02:19.612 INFO:tasks.rados.rados.0.smithi063.stdout:1779: finishing write tid 4 to smithi06387500-37 2024-08-17T17:02:19.613 INFO:tasks.rados.rados.0.smithi063.stdout:1779: finishing write tid 5 to smithi06387500-37 2024-08-17T17:02:19.613 INFO:tasks.rados.rados.0.smithi063.stdout:1779: oid 37 updating version 0 to 1705 2024-08-17T17:02:19.613 INFO:tasks.rados.rados.0.smithi063.stdout:1779: oid 37 updating version 1705 to 1706 2024-08-17T17:02:19.613 INFO:tasks.rados.rados.0.smithi063.stdout:1779: oid 37 version 1706 is already newer than 1704 2024-08-17T17:02:19.613 INFO:tasks.rados.rados.0.smithi063.stdout:1779: oid 37 version 1706 is already newer than 1703 2024-08-17T17:02:19.613 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 1706 (ObjNum 686 snap 219 seq_num 686) dirty exists 2024-08-17T17:02:19.613 INFO:tasks.rados.rados.0.smithi063.stdout:1779: left oid 37 (ObjNum 686 snap 219 seq_num 686) 2024-08-17T17:02:19.834 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[109843]: Reconfiguring osd.3 (monmap changed)... 2024-08-17T17:02:19.834 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[109843]: Reconfiguring daemon osd.3 on smithi063 2024-08-17T17:02:19.834 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[109843]: pgmap v222: 89 pgs: 89 active+clean; 89 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-08-17T17:02:19.834 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[109843]: osdmap e511: 8 total, 8 up, 8 in 2024-08-17T17:02:19.835 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:19.835 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:19.835 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi063.ultpqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:02:19.835 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi063.ultpqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:02:19.835 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:19.835 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[114756]: Reconfiguring osd.3 (monmap changed)... 2024-08-17T17:02:19.835 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[114756]: Reconfiguring daemon osd.3 on smithi063 2024-08-17T17:02:19.835 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[114756]: pgmap v222: 89 pgs: 89 active+clean; 89 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-08-17T17:02:19.835 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[114756]: osdmap e511: 8 total, 8 up, 8 in 2024-08-17T17:02:19.835 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:19.836 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:19.836 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi063.ultpqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:02:19.836 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi063.ultpqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:02:19.836 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:19 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:19 smithi162 ceph-mon[104668]: Reconfiguring osd.3 (monmap changed)... 2024-08-17T17:02:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:19 smithi162 ceph-mon[104668]: Reconfiguring daemon osd.3 on smithi063 2024-08-17T17:02:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:19 smithi162 ceph-mon[104668]: pgmap v222: 89 pgs: 89 active+clean; 89 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-08-17T17:02:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:19 smithi162 ceph-mon[104668]: osdmap e511: 8 total, 8 up, 8 in 2024-08-17T17:02:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:19 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:19 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:19 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi063.ultpqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:02:19.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:19 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi063.ultpqj", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:02:19.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:19 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:20.589 INFO:tasks.rados.rados.0.smithi063.stdout:1779: done (1 left) 2024-08-17T17:02:20.589 INFO:tasks.rados.rados.0.smithi063.stdout:1780: done (0 left) 2024-08-17T17:02:20.589 INFO:tasks.rados.rados.0.smithi063.stdout:1781: delete oid 20 current snap is 220 2024-08-17T17:02:20.591 INFO:tasks.rados.rados.0.smithi063.stdout:1781: done (0 left) 2024-08-17T17:02:20.591 INFO:tasks.rados.rados.0.smithi063.stdout:1782: write oid 16 current snap is 220 2024-08-17T17:02:20.591 INFO:tasks.rados.rados.0.smithi063.stdout:1782: seq_num 688 ranges {763216=643082,1843729=687899,3260666=302550} 2024-08-17T17:02:20.599 INFO:tasks.rados.rados.0.smithi063.stdout:1782: writing smithi06387500-16 from 763216 to 1406298 tid 1 2024-08-17T17:02:20.605 INFO:tasks.rados.rados.0.smithi063.stdout:1782: writing smithi06387500-16 from 1843729 to 2531628 tid 2 2024-08-17T17:02:20.611 INFO:tasks.rados.rados.0.smithi063.stdout:1782: writing smithi06387500-16 from 3260666 to 3563216 tid 3 2024-08-17T17:02:20.611 INFO:tasks.rados.rados.0.smithi063.stdout:1783: read oid 11 snap -1 2024-08-17T17:02:20.611 INFO:tasks.rados.rados.0.smithi063.stdout:1783: expect deleted 2024-08-17T17:02:20.612 INFO:tasks.rados.rados.0.smithi063.stdout:1784: read oid 3 snap -1 2024-08-17T17:02:20.612 INFO:tasks.rados.rados.0.smithi063.stdout:1784: expect deleted 2024-08-17T17:02:20.612 INFO:tasks.rados.rados.0.smithi063.stdout:1785: rollback oid 28 current snap is 220 2024-08-17T17:02:20.612 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 28 to 218 2024-08-17T17:02:20.612 INFO:tasks.rados.rados.0.smithi063.stdout:1782: finishing write tid 1 to smithi06387500-16 2024-08-17T17:02:20.612 INFO:tasks.rados.rados.0.smithi063.stdout:1782: finishing write tid 2 to smithi06387500-16 2024-08-17T17:02:20.612 INFO:tasks.rados.rados.0.smithi063.stdout:1786: write oid 26 current snap is 220 2024-08-17T17:02:20.612 INFO:tasks.rados.rados.0.smithi063.stdout:1786: seq_num 689 ranges {780023=718331,1902211=547910,3004442=575581} 2024-08-17T17:02:20.619 INFO:tasks.rados.rados.0.smithi063.stdout:1786: writing smithi06387500-26 from 780023 to 1498354 tid 1 2024-08-17T17:02:20.624 INFO:tasks.rados.rados.0.smithi063.stdout:1786: writing smithi06387500-26 from 1902211 to 2450121 tid 2 2024-08-17T17:02:20.631 INFO:tasks.rados.rados.0.smithi063.stdout:1786: writing smithi06387500-26 from 3004442 to 3580023 tid 3 2024-08-17T17:02:20.631 INFO:tasks.rados.rados.0.smithi063.stdout:1787: rollback oid 45 current snap is 220 2024-08-17T17:02:20.631 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 45 to 216 2024-08-17T17:02:20.631 INFO:tasks.rados.rados.0.smithi063.stdout:1785: finishing rollback tid 1 to smithi06387500-28 2024-08-17T17:02:20.631 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 0 (ObjNum 0 snap 0 seq_num 304) dirty dne 2024-08-17T17:02:20.631 INFO:tasks.rados.rados.0.smithi063.stdout:1783: done (5 left) 2024-08-17T17:02:20.632 INFO:tasks.rados.rados.0.smithi063.stdout:1785: done (4 left) 2024-08-17T17:02:20.632 INFO:tasks.rados.rados.0.smithi063.stdout:1788: write oid 23 current snap is 220 2024-08-17T17:02:20.632 INFO:tasks.rados.rados.0.smithi063.stdout:1782: finishing write tid 3 to smithi06387500-16 2024-08-17T17:02:20.632 INFO:tasks.rados.rados.0.smithi063.stdout:1788: seq_num 690 ranges {796830=793580,2360693=691568} 2024-08-17T17:02:20.639 INFO:tasks.rados.rados.0.smithi063.stdout:1788: writing smithi06387500-23 from 796830 to 1590410 tid 1 2024-08-17T17:02:20.649 INFO:tasks.rados.rados.0.smithi063.stdout:1788: writing smithi06387500-23 from 2360693 to 3052261 tid 2 2024-08-17T17:02:20.649 INFO:tasks.rados.rados.0.smithi063.stdout:1789: read oid 38 snap -1 2024-08-17T17:02:20.649 INFO:tasks.rados.rados.0.smithi063.stdout:1789: expect (ObjNum 674 snap 213 seq_num 674) 2024-08-17T17:02:20.649 INFO:tasks.rados.rados.0.smithi063.stdout:1790: rollback oid 34 current snap is 220 2024-08-17T17:02:20.649 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 34 to 216 2024-08-17T17:02:20.649 INFO:tasks.rados.rados.0.smithi063.stdout:1782: finishing write tid 4 to smithi06387500-16 2024-08-17T17:02:20.649 INFO:tasks.rados.rados.0.smithi063.stdout:1782: finishing write tid 5 to smithi06387500-16 2024-08-17T17:02:20.649 INFO:tasks.rados.rados.0.smithi063.stdout:1782: oid 16 updating version 0 to 664 2024-08-17T17:02:20.649 INFO:tasks.rados.rados.0.smithi063.stdout:1782: oid 16 version 664 is already newer than 663 2024-08-17T17:02:20.650 INFO:tasks.rados.rados.0.smithi063.stdout:1782: oid 16 version 664 is already newer than 662 2024-08-17T17:02:20.650 INFO:tasks.rados.rados.0.smithi063.stdout:1782: oid 16 version 664 is already newer than 661 2024-08-17T17:02:20.650 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 664 (ObjNum 688 snap 220 seq_num 688) dirty exists 2024-08-17T17:02:20.650 INFO:tasks.rados.rados.0.smithi063.stdout:1782: left oid 16 (ObjNum 688 snap 220 seq_num 688) 2024-08-17T17:02:20.650 INFO:tasks.rados.rados.0.smithi063.stdout:1786: finishing write tid 1 to smithi06387500-26 2024-08-17T17:02:20.650 INFO:tasks.rados.rados.0.smithi063.stdout:1782: done (6 left) 2024-08-17T17:02:20.650 INFO:tasks.rados.rados.0.smithi063.stdout:1784: done (5 left) 2024-08-17T17:02:20.650 INFO:tasks.rados.rados.0.smithi063.stdout:1791: read oid 19 snap -1 2024-08-17T17:02:20.650 INFO:tasks.rados.rados.0.smithi063.stdout:1791: expect deleted 2024-08-17T17:02:20.650 INFO:tasks.rados.rados.0.smithi063.stdout:1792: snap_remove snap 214 2024-08-17T17:02:20.657 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T17:02:20.711 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:20 smithi063 ceph-mon[114756]: Reconfiguring mds.foo.smithi063.ultpqj (monmap changed)... 2024-08-17T17:02:20.711 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:20 smithi063 ceph-mon[114756]: Reconfiguring daemon mds.foo.smithi063.ultpqj on smithi063 2024-08-17T17:02:20.711 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:20 smithi063 ceph-mon[114756]: osdmap e512: 8 total, 8 up, 8 in 2024-08-17T17:02:20.711 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:20 smithi063 ceph-mon[109843]: Reconfiguring mds.foo.smithi063.ultpqj (monmap changed)... 2024-08-17T17:02:20.711 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:20 smithi063 ceph-mon[109843]: Reconfiguring daemon mds.foo.smithi063.ultpqj on smithi063 2024-08-17T17:02:20.711 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:20 smithi063 ceph-mon[109843]: osdmap e512: 8 total, 8 up, 8 in 2024-08-17T17:02:20.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:20 smithi162 ceph-mon[104668]: Reconfiguring mds.foo.smithi063.ultpqj (monmap changed)... 2024-08-17T17:02:20.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:20 smithi162 ceph-mon[104668]: Reconfiguring daemon mds.foo.smithi063.ultpqj on smithi063 2024-08-17T17:02:20.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:20 smithi162 ceph-mon[104668]: osdmap e512: 8 total, 8 up, 8 in 2024-08-17T17:02:21.246 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:02:21.247 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (3m) 68s ago 9m 17.5M - 0.27.0 11f11916f8cd 59f4018b91ca 2024-08-17T17:02:21.247 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (3m) 16s ago 9m 75.1M - c8b91775d855 581e03e654a8 2024-08-17T17:02:21.247 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (8m) 68s ago 8m 19.9M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T17:02:21.247 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (8m) 16s ago 8m 18.3M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T17:02:21.247 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,9283,8765 running (4m) 16s ago 13m 455M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb da48034a2895 2024-08-17T17:02:21.247 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:8443,9283,8765 running (5m) 68s ago 15m 507M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 943e639cede6 2024-08-17T17:02:21.247 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (2m) 68s ago 15m 47.1M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb f0f20fcac956 2024-08-17T17:02:21.247 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (27s) 16s ago 14m 20.0M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 520525c79dbe 2024-08-17T17:02:21.247 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (2m) 68s ago 13m 33.2M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb c484af026acf 2024-08-17T17:02:21.247 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 running (4m) 68s ago 9m 11.1M - 1.7.0 72c9c2088986 34f55f5de2aa 2024-08-17T17:02:21.247 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (4m) 16s ago 9m 11.3M - 1.7.0 72c9c2088986 5bb162d0f1a9 2024-08-17T17:02:21.247 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (13m) 68s ago 13m 105M 2480M 18.2.4-719-gced649f8 c870753d67c2 f3b910ed49b8 2024-08-17T17:02:21.248 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (12m) 68s ago 12m 84.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 5d6fdb6000ad 2024-08-17T17:02:21.248 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (12m) 68s ago 12m 95.2M 2480M 18.2.4-719-gced649f8 c870753d67c2 a3cbe8edb49a 2024-08-17T17:02:21.248 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (12m) 68s ago 12m 83.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T17:02:21.248 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (11m) 16s ago 11m 108M 2480M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T17:02:21.248 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (11m) 16s ago 11m 97.3M 2480M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T17:02:21.248 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (11m) 16s ago 11m 77.7M 2480M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T17:02:21.248 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (10m) 16s ago 10m 84.1M 2480M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T17:02:21.248 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (4m) 16s ago 10m 45.8M - 2.51.0 1d3b7f56885b dfe66b2c5568 2024-08-17T17:02:21.597 INFO:tasks.rados.rados.0.smithi063.stdout:1792: done (6 left) 2024-08-17T17:02:21.598 INFO:tasks.rados.rados.0.smithi063.stdout:1793: snap_create 2024-08-17T17:02:21.598 INFO:tasks.rados.rados.0.smithi063.stdout:1786: finishing write tid 2 to smithi06387500-26 2024-08-17T17:02:21.598 INFO:tasks.rados.rados.0.smithi063.stdout:1787: finishing rollback tid 1 to smithi06387500-45 2024-08-17T17:02:21.598 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-08-17T17:02:21.598 INFO:tasks.rados.rados.0.smithi063.stdout:1786: finishing write tid 3 to smithi06387500-26 2024-08-17T17:02:21.598 INFO:tasks.rados.rados.0.smithi063.stdout:1786: finishing write tid 4 to smithi06387500-26 2024-08-17T17:02:21.598 INFO:tasks.rados.rados.0.smithi063.stdout:1786: finishing write tid 5 to smithi06387500-26 2024-08-17T17:02:21.598 INFO:tasks.rados.rados.0.smithi063.stdout:1786: oid 26 updating version 0 to 534 2024-08-17T17:02:21.598 INFO:tasks.rados.rados.0.smithi063.stdout:1786: oid 26 version 534 is already newer than 533 2024-08-17T17:02:21.598 INFO:tasks.rados.rados.0.smithi063.stdout:1786: oid 26 version 534 is already newer than 532 2024-08-17T17:02:21.598 INFO:tasks.rados.rados.0.smithi063.stdout:1786: oid 26 updating version 534 to 535 2024-08-17T17:02:21.598 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 535 (ObjNum 689 snap 220 seq_num 689) dirty exists 2024-08-17T17:02:21.599 INFO:tasks.rados.rados.0.smithi063.stdout:1786: left oid 26 (ObjNum 689 snap 220 seq_num 689) 2024-08-17T17:02:21.599 INFO:tasks.rados.rados.0.smithi063.stdout:1788: finishing write tid 1 to smithi06387500-23 2024-08-17T17:02:21.599 INFO:tasks.rados.rados.0.smithi063.stdout:1788: finishing write tid 2 to smithi06387500-23 2024-08-17T17:02:21.599 INFO:tasks.rados.rados.0.smithi063.stdout:1788: finishing write tid 3 to smithi06387500-23 2024-08-17T17:02:21.599 INFO:tasks.rados.rados.0.smithi063.stdout:1788: finishing write tid 4 to smithi06387500-23 2024-08-17T17:02:21.599 INFO:tasks.rados.rados.0.smithi063.stdout:1788: oid 23 updating version 0 to 786 2024-08-17T17:02:21.599 INFO:tasks.rados.rados.0.smithi063.stdout:1788: oid 23 version 786 is already newer than 784 2024-08-17T17:02:21.599 INFO:tasks.rados.rados.0.smithi063.stdout:1788: oid 23 version 786 is already newer than 785 2024-08-17T17:02:21.599 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 786 (ObjNum 690 snap 220 seq_num 690) dirty exists 2024-08-17T17:02:21.599 INFO:tasks.rados.rados.0.smithi063.stdout:1788: left oid 23 (ObjNum 690 snap 220 seq_num 690) 2024-08-17T17:02:21.599 INFO:tasks.rados.rados.0.smithi063.stdout:1790: finishing rollback tid 0 to smithi06387500-34 2024-08-17T17:02:21.599 INFO:tasks.rados.rados.0.smithi063.stdout:1790: finishing rollback tid 1 to smithi06387500-34 2024-08-17T17:02:21.599 INFO:tasks.rados.rados.0.smithi063.stdout:1790: finishing rollback tid 2 to smithi06387500-34 2024-08-17T17:02:21.599 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 1246 (ObjNum 675 snap 213 seq_num 675) dirty exists 2024-08-17T17:02:21.599 INFO:tasks.rados.rados.0.smithi063.stdout:1789: expect (ObjNum 674 snap 213 seq_num 674) 2024-08-17T17:02:21.602 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:02:21.602 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T17:02:21.602 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:02:21.602 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:02:21.603 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T17:02:21.603 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:02:21.603 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:02:21.603 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T17:02:21.603 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:02:21.603 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:02:21.603 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T17:02:21.603 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:02:21.603 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:02:21.603 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T17:02:21.603 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 10, 2024-08-17T17:02:21.603 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 5 2024-08-17T17:02:21.603 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:02:21.604 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:02:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:21 smithi162 ceph-mon[104668]: pgmap v225: 89 pgs: 89 active+clean; 89 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-08-17T17:02:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:21 smithi162 ceph-mon[104668]: osdmap e513: 8 total, 8 up, 8 in 2024-08-17T17:02:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:21 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:21 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:21 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:02:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:21 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:02:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:21 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:21.908 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:02:21.908 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:02:21.908 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-08-17T17:02:21.909 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (0 remaining).", 2024-08-17T17:02:21.909 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [ 2024-08-17T17:02:21.909 INFO:teuthology.orchestra.run.smithi063.stdout: "mon" 2024-08-17T17:02:21.909 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-08-17T17:02:21.909 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "3/3 daemons upgraded", 2024-08-17T17:02:21.909 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading mon daemons", 2024-08-17T17:02:21.909 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T17:02:21.909 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:02:21.909 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:21 smithi063 ceph-mon[109843]: pgmap v225: 89 pgs: 89 active+clean; 89 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-08-17T17:02:21.909 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:21 smithi063 ceph-mon[109843]: osdmap e513: 8 total, 8 up, 8 in 2024-08-17T17:02:21.910 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:21 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:21.910 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:21 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:21.910 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:21 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:02:21.910 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:21 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:02:21.910 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:21 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:21.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:21 smithi063 ceph-mon[114756]: pgmap v225: 89 pgs: 89 active+clean; 89 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 5.0 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-08-17T17:02:21.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:21 smithi063 ceph-mon[114756]: osdmap e513: 8 total, 8 up, 8 in 2024-08-17T17:02:21.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:21 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:21.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:21 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:21.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:21 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-08-17T17:02:21.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:21 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-08-17T17:02:21.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:21 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:22.540 INFO:tasks.rados.rados.0.smithi063.stdout:1786: done (6 left) 2024-08-17T17:02:22.540 INFO:tasks.rados.rados.0.smithi063.stdout:1787: done (5 left) 2024-08-17T17:02:22.540 INFO:tasks.rados.rados.0.smithi063.stdout:1788: done (4 left) 2024-08-17T17:02:22.540 INFO:tasks.rados.rados.0.smithi063.stdout:1789: done (3 left) 2024-08-17T17:02:22.540 INFO:tasks.rados.rados.0.smithi063.stdout:1790: done (2 left) 2024-08-17T17:02:22.540 INFO:tasks.rados.rados.0.smithi063.stdout:1791: done (1 left) 2024-08-17T17:02:22.540 INFO:tasks.rados.rados.0.smithi063.stdout:1793: done (0 left) 2024-08-17T17:02:22.540 INFO:tasks.rados.rados.0.smithi063.stdout:1794: read oid 43 snap 220 2024-08-17T17:02:22.540 INFO:tasks.rados.rados.0.smithi063.stdout:1794: expect (ObjNum 483 snap 153 seq_num 483) 2024-08-17T17:02:22.540 INFO:tasks.rados.rados.0.smithi063.stdout:1795: write oid 48 current snap is 221 2024-08-17T17:02:22.540 INFO:tasks.rados.rados.0.smithi063.stdout:1795: seq_num 691 ranges {413637=468829,1619175=435226,2542935=758255} 2024-08-17T17:02:22.544 INFO:tasks.rados.rados.0.smithi063.stdout:1795: writing smithi06387500-48 from 413637 to 882466 tid 1 2024-08-17T17:02:22.551 INFO:tasks.rados.rados.0.smithi063.stdout:1795: writing smithi06387500-48 from 1619175 to 2054401 tid 2 2024-08-17T17:02:22.558 INFO:tasks.rados.rados.0.smithi063.stdout:1795: writing smithi06387500-48 from 2542935 to 3301190 tid 3 2024-08-17T17:02:22.558 INFO:tasks.rados.rados.0.smithi063.stdout:1796: write oid 7 current snap is 221 2024-08-17T17:02:22.558 INFO:tasks.rados.rados.0.smithi063.stdout:1796: seq_num 692 ranges {430444=660431,1677657=695237,2970358=569527} 2024-08-17T17:02:22.564 INFO:tasks.rados.rados.0.smithi063.stdout:1796: writing smithi06387500-7 from 430444 to 1090875 tid 1 2024-08-17T17:02:22.571 INFO:tasks.rados.rados.0.smithi063.stdout:1796: writing smithi06387500-7 from 1677657 to 2372894 tid 2 2024-08-17T17:02:22.578 INFO:tasks.rados.rados.0.smithi063.stdout:1796: writing smithi06387500-7 from 2970358 to 3539885 tid 3 2024-08-17T17:02:22.578 INFO:tasks.rados.rados.0.smithi063.stdout:1797: snap_create 2024-08-17T17:02:22.578 INFO:tasks.rados.rados.0.smithi063.stdout:1794: expect (ObjNum 483 snap 153 seq_num 483) 2024-08-17T17:02:22.652 INFO:tasks.rados.rados.0.smithi063.stdout:1795: finishing write tid 1 to smithi06387500-48 2024-08-17T17:02:22.653 INFO:tasks.rados.rados.0.smithi063.stdout:1795: finishing write tid 2 to smithi06387500-48 2024-08-17T17:02:22.653 INFO:tasks.rados.rados.0.smithi063.stdout:1795: finishing write tid 3 to smithi06387500-48 2024-08-17T17:02:22.653 INFO:tasks.rados.rados.0.smithi063.stdout:1795: finishing write tid 4 to smithi06387500-48 2024-08-17T17:02:22.653 INFO:tasks.rados.rados.0.smithi063.stdout:1795: finishing write tid 5 to smithi06387500-48 2024-08-17T17:02:22.653 INFO:tasks.rados.rados.0.smithi063.stdout:1795: oid 48 updating version 0 to 1711 2024-08-17T17:02:22.653 INFO:tasks.rados.rados.0.smithi063.stdout:1795: oid 48 version 1711 is already newer than 1710 2024-08-17T17:02:22.653 INFO:tasks.rados.rados.0.smithi063.stdout:1795: oid 48 updating version 1711 to 1712 2024-08-17T17:02:22.654 INFO:tasks.rados.rados.0.smithi063.stdout:1795: oid 48 version 1712 is already newer than 1709 2024-08-17T17:02:22.654 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 1712 (ObjNum 691 snap 221 seq_num 691) dirty exists 2024-08-17T17:02:22.654 INFO:tasks.rados.rados.0.smithi063.stdout:1795: left oid 48 (ObjNum 691 snap 221 seq_num 691) 2024-08-17T17:02:22.654 INFO:tasks.rados.rados.0.smithi063.stdout:1796: finishing write tid 1 to smithi06387500-7 2024-08-17T17:02:22.654 INFO:tasks.rados.rados.0.smithi063.stdout:1796: finishing write tid 2 to smithi06387500-7 2024-08-17T17:02:22.654 INFO:tasks.rados.rados.0.smithi063.stdout:1796: finishing write tid 3 to smithi06387500-7 2024-08-17T17:02:22.654 INFO:tasks.rados.rados.0.smithi063.stdout:1796: finishing write tid 4 to smithi06387500-7 2024-08-17T17:02:22.654 INFO:tasks.rados.rados.0.smithi063.stdout:1796: finishing write tid 5 to smithi06387500-7 2024-08-17T17:02:22.654 INFO:tasks.rados.rados.0.smithi063.stdout:1796: oid 7 updating version 0 to 786 2024-08-17T17:02:22.654 INFO:tasks.rados.rados.0.smithi063.stdout:1796: oid 7 version 786 is already newer than 785 2024-08-17T17:02:22.654 INFO:tasks.rados.rados.0.smithi063.stdout:1796: oid 7 version 786 is already newer than 783 2024-08-17T17:02:22.655 INFO:tasks.rados.rados.0.smithi063.stdout:1796: oid 7 version 786 is already newer than 784 2024-08-17T17:02:22.655 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 786 (ObjNum 692 snap 221 seq_num 692) dirty exists 2024-08-17T17:02:22.655 INFO:tasks.rados.rados.0.smithi063.stdout:1796: left oid 7 (ObjNum 692 snap 221 seq_num 692) 2024-08-17T17:02:22.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:22 smithi162 ceph-mon[104668]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:22 smithi162 ceph-mon[104668]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:02:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:22 smithi162 ceph-mon[104668]: Reconfiguring daemon mon.b on smithi162 2024-08-17T17:02:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:22 smithi162 ceph-mon[104668]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:22 smithi162 ceph-mon[104668]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:22 smithi162 ceph-mon[104668]: osdmap e514: 8 total, 8 up, 8 in 2024-08-17T17:02:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:22 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/336899097' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:22 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:22 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:22 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:02:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:22 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:02:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:22 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:02:22.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:22 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:22.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:22 smithi162 ceph-mon[104668]: osdmap e515: 8 total, 8 up, 8 in 2024-08-17T17:02:23.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[109843]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[109843]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:02:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[109843]: Reconfiguring daemon mon.b on smithi162 2024-08-17T17:02:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[109843]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[109843]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[109843]: osdmap e514: 8 total, 8 up, 8 in 2024-08-17T17:02:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/336899097' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:02:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:02:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:02:23.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:23.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[109843]: osdmap e515: 8 total, 8 up, 8 in 2024-08-17T17:02:23.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[114756]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:23.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[114756]: Reconfiguring mon.b (monmap changed)... 2024-08-17T17:02:23.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[114756]: Reconfiguring daemon mon.b on smithi162 2024-08-17T17:02:23.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[114756]: from='client.34221 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:23.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[114756]: from='client.44156 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:23.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[114756]: osdmap e514: 8 total, 8 up, 8 in 2024-08-17T17:02:23.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/336899097' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:23.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:23.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:23.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:02:23.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-08-17T17:02:23.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-08-17T17:02:23.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:23.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:22 smithi063 ceph-mon[114756]: osdmap e515: 8 total, 8 up, 8 in 2024-08-17T17:02:23.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:02:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:02:23] "GET /metrics HTTP/1.1" 200 36210 "" "Prometheus/2.51.0" 2024-08-17T17:02:23.540 INFO:tasks.rados.rados.0.smithi063.stdout:1794: done (3 left) 2024-08-17T17:02:23.540 INFO:tasks.rados.rados.0.smithi063.stdout:1795: done (2 left) 2024-08-17T17:02:23.540 INFO:tasks.rados.rados.0.smithi063.stdout:1796: done (1 left) 2024-08-17T17:02:23.540 INFO:tasks.rados.rados.0.smithi063.stdout:1797: done (0 left) 2024-08-17T17:02:23.540 INFO:tasks.rados.rados.0.smithi063.stdout:1798: rollback oid 29 current snap is 222 2024-08-17T17:02:23.540 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 29 to 221 2024-08-17T17:02:23.540 INFO:tasks.rados.rados.0.smithi063.stdout:1799: write oid 18 current snap is 222 2024-08-17T17:02:23.540 INFO:tasks.rados.rados.0.smithi063.stdout:1799: seq_num 693 ranges {447251=735680,1736139=438895,2597781=780799} 2024-08-17T17:02:23.546 INFO:tasks.rados.rados.0.smithi063.stdout:1799: writing smithi06387500-18 from 447251 to 1182931 tid 1 2024-08-17T17:02:23.552 INFO:tasks.rados.rados.0.smithi063.stdout:1799: writing smithi06387500-18 from 1736139 to 2175034 tid 2 2024-08-17T17:02:23.559 INFO:tasks.rados.rados.0.smithi063.stdout:1799: writing smithi06387500-18 from 2597781 to 3378580 tid 3 2024-08-17T17:02:23.559 INFO:tasks.rados.rados.0.smithi063.stdout:1800: write oid 34 current snap is 222 2024-08-17T17:02:23.559 INFO:tasks.rados.rados.0.smithi063.stdout:1798: finishing rollback tid 0 to smithi06387500-29 2024-08-17T17:02:23.559 INFO:tasks.rados.rados.0.smithi063.stdout:1800: seq_num 694 ranges {464058=410929,1394621=698906,2625204=592071,3656107=7951} 2024-08-17T17:02:23.563 INFO:tasks.rados.rados.0.smithi063.stdout:1800: writing smithi06387500-34 from 464058 to 874987 tid 1 2024-08-17T17:02:23.570 INFO:tasks.rados.rados.0.smithi063.stdout:1800: writing smithi06387500-34 from 1394621 to 2093527 tid 2 2024-08-17T17:02:23.576 INFO:tasks.rados.rados.0.smithi063.stdout:1800: writing smithi06387500-34 from 2625204 to 3217275 tid 3 2024-08-17T17:02:23.579 INFO:tasks.rados.rados.0.smithi063.stdout:1800: writing smithi06387500-34 from 3656107 to 3664058 tid 4 2024-08-17T17:02:23.579 INFO:tasks.rados.rados.0.smithi063.stdout:1801: write oid 38 current snap is 222 2024-08-17T17:02:23.579 INFO:tasks.rados.rados.0.smithi063.stdout:1798: finishing rollback tid 1 to smithi06387500-29 2024-08-17T17:02:23.579 INFO:tasks.rados.rados.0.smithi063.stdout:1798: finishing rollback tid 2 to smithi06387500-29 2024-08-17T17:02:23.579 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 422 (ObjNum 670 snap 213 seq_num 670) dirty exists 2024-08-17T17:02:23.580 INFO:tasks.rados.rados.0.smithi063.stdout:1799: finishing write tid 1 to smithi06387500-18 2024-08-17T17:02:23.580 INFO:tasks.rados.rados.0.smithi063.stdout:1799: finishing write tid 2 to smithi06387500-18 2024-08-17T17:02:23.580 INFO:tasks.rados.rados.0.smithi063.stdout:1801: seq_num 695 ranges {480865=486178,1453103=442564,2652627=519696} 2024-08-17T17:02:23.584 INFO:tasks.rados.rados.0.smithi063.stdout:1801: writing smithi06387500-38 from 480865 to 967043 tid 1 2024-08-17T17:02:23.590 INFO:tasks.rados.rados.0.smithi063.stdout:1801: writing smithi06387500-38 from 1453103 to 1895667 tid 2 2024-08-17T17:02:23.597 INFO:tasks.rados.rados.0.smithi063.stdout:1801: writing smithi06387500-38 from 2652627 to 3172323 tid 3 2024-08-17T17:02:23.597 INFO:tasks.rados.rados.0.smithi063.stdout:1798: done (3 left) 2024-08-17T17:02:23.597 INFO:tasks.rados.rados.0.smithi063.stdout:1802: write oid 37 current snap is 222 2024-08-17T17:02:23.597 INFO:tasks.rados.rados.0.smithi063.stdout:1799: finishing write tid 3 to smithi06387500-18 2024-08-17T17:02:23.597 INFO:tasks.rados.rados.0.smithi063.stdout:1799: finishing write tid 4 to smithi06387500-18 2024-08-17T17:02:23.597 INFO:tasks.rados.rados.0.smithi063.stdout:1802: seq_num 696 ranges {497672=561427,1795232=702575,2963697=730968} 2024-08-17T17:02:23.603 INFO:tasks.rados.rados.0.smithi063.stdout:1802: writing smithi06387500-37 from 497672 to 1059099 tid 1 2024-08-17T17:02:23.611 INFO:tasks.rados.rados.0.smithi063.stdout:1802: writing smithi06387500-37 from 1795232 to 2497807 tid 2 2024-08-17T17:02:23.618 INFO:tasks.rados.rados.0.smithi063.stdout:1802: writing smithi06387500-37 from 2963697 to 3694665 tid 3 2024-08-17T17:02:23.618 INFO:tasks.rados.rados.0.smithi063.stdout:1803: snap_create 2024-08-17T17:02:23.618 INFO:tasks.rados.rados.0.smithi063.stdout:1799: finishing write tid 5 to smithi06387500-18 2024-08-17T17:02:23.618 INFO:tasks.rados.rados.0.smithi063.stdout:1799: oid 18 updating version 0 to 594 2024-08-17T17:02:23.618 INFO:tasks.rados.rados.0.smithi063.stdout:1799: oid 18 updating version 594 to 596 2024-08-17T17:02:23.618 INFO:tasks.rados.rados.0.smithi063.stdout:1799: oid 18 version 596 is already newer than 595 2024-08-17T17:02:23.618 INFO:tasks.rados.rados.0.smithi063.stdout:1799: oid 18 updating version 596 to 597 2024-08-17T17:02:23.618 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 597 (ObjNum 693 snap 222 seq_num 693) dirty exists 2024-08-17T17:02:23.618 INFO:tasks.rados.rados.0.smithi063.stdout:1799: left oid 18 (ObjNum 693 snap 222 seq_num 693) 2024-08-17T17:02:23.619 INFO:tasks.rados.rados.0.smithi063.stdout:1800: finishing write tid 1 to smithi06387500-34 2024-08-17T17:02:23.619 INFO:tasks.rados.rados.0.smithi063.stdout:1800: finishing write tid 2 to smithi06387500-34 2024-08-17T17:02:23.619 INFO:tasks.rados.rados.0.smithi063.stdout:1800: finishing write tid 3 to smithi06387500-34 2024-08-17T17:02:23.619 INFO:tasks.rados.rados.0.smithi063.stdout:1800: finishing write tid 4 to smithi06387500-34 2024-08-17T17:02:23.619 INFO:tasks.rados.rados.0.smithi063.stdout:1800: finishing write tid 5 to smithi06387500-34 2024-08-17T17:02:23.619 INFO:tasks.rados.rados.0.smithi063.stdout:1800: finishing write tid 6 to smithi06387500-34 2024-08-17T17:02:23.619 INFO:tasks.rados.rados.0.smithi063.stdout:1800: oid 34 updating version 0 to 1255 2024-08-17T17:02:23.619 INFO:tasks.rados.rados.0.smithi063.stdout:1800: oid 34 version 1255 is already newer than 1252 2024-08-17T17:02:23.619 INFO:tasks.rados.rados.0.smithi063.stdout:1800: oid 34 version 1255 is already newer than 1253 2024-08-17T17:02:23.619 INFO:tasks.rados.rados.0.smithi063.stdout:1800: oid 34 updating version 1255 to 1256 2024-08-17T17:02:23.619 INFO:tasks.rados.rados.0.smithi063.stdout:1800: oid 34 version 1256 is already newer than 1254 2024-08-17T17:02:23.619 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 1256 (ObjNum 694 snap 222 seq_num 694) dirty exists 2024-08-17T17:02:23.619 INFO:tasks.rados.rados.0.smithi063.stdout:1800: left oid 34 (ObjNum 694 snap 222 seq_num 694) 2024-08-17T17:02:23.620 INFO:tasks.rados.rados.0.smithi063.stdout:1801: finishing write tid 1 to smithi06387500-38 2024-08-17T17:02:23.620 INFO:tasks.rados.rados.0.smithi063.stdout:1801: finishing write tid 2 to smithi06387500-38 2024-08-17T17:02:23.620 INFO:tasks.rados.rados.0.smithi063.stdout:1801: finishing write tid 3 to smithi06387500-38 2024-08-17T17:02:23.620 INFO:tasks.rados.rados.0.smithi063.stdout:1801: finishing write tid 4 to smithi06387500-38 2024-08-17T17:02:23.620 INFO:tasks.rados.rados.0.smithi063.stdout:1801: finishing write tid 5 to smithi06387500-38 2024-08-17T17:02:23.620 INFO:tasks.rados.rados.0.smithi063.stdout:1801: oid 38 updating version 0 to 256 2024-08-17T17:02:23.620 INFO:tasks.rados.rados.0.smithi063.stdout:1801: oid 38 updating version 256 to 257 2024-08-17T17:02:23.620 INFO:tasks.rados.rados.0.smithi063.stdout:1801: oid 38 updating version 257 to 259 2024-08-17T17:02:23.620 INFO:tasks.rados.rados.0.smithi063.stdout:1801: oid 38 version 259 is already newer than 258 2024-08-17T17:02:23.620 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 259 (ObjNum 695 snap 222 seq_num 695) dirty exists 2024-08-17T17:02:23.620 INFO:tasks.rados.rados.0.smithi063.stdout:1801: left oid 38 (ObjNum 695 snap 222 seq_num 695) 2024-08-17T17:02:23.620 INFO:tasks.rados.rados.0.smithi063.stdout:1802: finishing write tid 1 to smithi06387500-37 2024-08-17T17:02:23.620 INFO:tasks.rados.rados.0.smithi063.stdout:1802: finishing write tid 2 to smithi06387500-37 2024-08-17T17:02:23.622 INFO:tasks.rados.rados.0.smithi063.stdout:1802: finishing write tid 3 to smithi06387500-37 2024-08-17T17:02:23.622 INFO:tasks.rados.rados.0.smithi063.stdout:1802: finishing write tid 4 to smithi06387500-37 2024-08-17T17:02:23.622 INFO:tasks.rados.rados.0.smithi063.stdout:1802: finishing write tid 5 to smithi06387500-37 2024-08-17T17:02:23.622 INFO:tasks.rados.rados.0.smithi063.stdout:1802: oid 37 updating version 0 to 1714 2024-08-17T17:02:23.622 INFO:tasks.rados.rados.0.smithi063.stdout:1802: oid 37 updating version 1714 to 1715 2024-08-17T17:02:23.622 INFO:tasks.rados.rados.0.smithi063.stdout:1802: oid 37 updating version 1715 to 1716 2024-08-17T17:02:23.622 INFO:tasks.rados.rados.0.smithi063.stdout:1802: oid 37 updating version 1716 to 1717 2024-08-17T17:02:23.622 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 1717 (ObjNum 696 snap 222 seq_num 696) dirty exists 2024-08-17T17:02:23.622 INFO:tasks.rados.rados.0.smithi063.stdout:1802: left oid 37 (ObjNum 696 snap 222 seq_num 696) 2024-08-17T17:02:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:23 smithi162 ceph-mon[104668]: pgmap v228: 89 pgs: 89 active+clean; 103 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-08-17T17:02:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:23 smithi162 ceph-mon[104668]: from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:23 smithi162 ceph-mon[104668]: Reconfiguring mgr.x (monmap changed)... 2024-08-17T17:02:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:23 smithi162 ceph-mon[104668]: Reconfiguring daemon mgr.x on smithi162 2024-08-17T17:02:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:23 smithi162 ceph-mon[104668]: osdmap e516: 8 total, 8 up, 8 in 2024-08-17T17:02:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:23 smithi063 ceph-mon[109843]: pgmap v228: 89 pgs: 89 active+clean; 103 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-08-17T17:02:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:23 smithi063 ceph-mon[109843]: from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:23 smithi063 ceph-mon[109843]: Reconfiguring mgr.x (monmap changed)... 2024-08-17T17:02:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:23 smithi063 ceph-mon[109843]: Reconfiguring daemon mgr.x on smithi162 2024-08-17T17:02:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:23 smithi063 ceph-mon[109843]: osdmap e516: 8 total, 8 up, 8 in 2024-08-17T17:02:24.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:23 smithi063 ceph-mon[114756]: pgmap v228: 89 pgs: 89 active+clean; 103 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-08-17T17:02:24.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:23 smithi063 ceph-mon[114756]: from='client.34239 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:24.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:23 smithi063 ceph-mon[114756]: Reconfiguring mgr.x (monmap changed)... 2024-08-17T17:02:24.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:23 smithi063 ceph-mon[114756]: Reconfiguring daemon mgr.x on smithi162 2024-08-17T17:02:24.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:23 smithi063 ceph-mon[114756]: osdmap e516: 8 total, 8 up, 8 in 2024-08-17T17:02:24.617 INFO:tasks.rados.rados.0.smithi063.stdout:1799: done (4 left) 2024-08-17T17:02:24.617 INFO:tasks.rados.rados.0.smithi063.stdout:1800: done (3 left) 2024-08-17T17:02:24.617 INFO:tasks.rados.rados.0.smithi063.stdout:1801: done (2 left) 2024-08-17T17:02:24.617 INFO:tasks.rados.rados.0.smithi063.stdout:1802: done (1 left) 2024-08-17T17:02:24.617 INFO:tasks.rados.rados.0.smithi063.stdout:1803: done (0 left) 2024-08-17T17:02:24.617 INFO:tasks.rados.rados.0.smithi063.stdout:1804: snap_create 2024-08-17T17:02:25.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[114756]: pgmap v231: 89 pgs: 89 active+clean; 103 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-08-17T17:02:25.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[114756]: Reconfiguring osd.4 (monmap changed)... 2024-08-17T17:02:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-08-17T17:02:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[114756]: Reconfiguring daemon osd.4 on smithi162 2024-08-17T17:02:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[114756]: osdmap e517: 8 total, 8 up, 8 in 2024-08-17T17:02:25.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[109843]: pgmap v231: 89 pgs: 89 active+clean; 103 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-08-17T17:02:25.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:25.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:25.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[109843]: Reconfiguring osd.4 (monmap changed)... 2024-08-17T17:02:25.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-08-17T17:02:25.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:25.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[109843]: Reconfiguring daemon osd.4 on smithi162 2024-08-17T17:02:25.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:24 smithi063 ceph-mon[109843]: osdmap e517: 8 total, 8 up, 8 in 2024-08-17T17:02:25.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:24 smithi162 ceph-mon[104668]: pgmap v231: 89 pgs: 89 active+clean; 103 MiB data, 545 MiB used, 715 GiB / 715 GiB avail; 6.0 MiB/s rd, 2.3 MiB/s wr, 16 op/s 2024-08-17T17:02:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:24 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:24 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:24 smithi162 ceph-mon[104668]: Reconfiguring osd.4 (monmap changed)... 2024-08-17T17:02:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:24 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-08-17T17:02:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:24 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:24 smithi162 ceph-mon[104668]: Reconfiguring daemon osd.4 on smithi162 2024-08-17T17:02:25.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:24 smithi162 ceph-mon[104668]: osdmap e517: 8 total, 8 up, 8 in 2024-08-17T17:02:25.618 INFO:tasks.rados.rados.0.smithi063.stdout:1804: done (0 left) 2024-08-17T17:02:25.618 INFO:tasks.rados.rados.0.smithi063.stdout:1805: read oid 48 snap 222 2024-08-17T17:02:25.618 INFO:tasks.rados.rados.0.smithi063.stdout:1805: expect (ObjNum 691 snap 221 seq_num 691) 2024-08-17T17:02:25.618 INFO:tasks.rados.rados.0.smithi063.stdout:1806: read oid 5 snap 223 2024-08-17T17:02:25.618 INFO:tasks.rados.rados.0.smithi063.stdout:1806: expect (ObjNum 443 snap 139 seq_num 443) 2024-08-17T17:02:25.618 INFO:tasks.rados.rados.0.smithi063.stdout:1807: snap_create 2024-08-17T17:02:25.635 INFO:tasks.rados.rados.0.smithi063.stdout:1806: expect (ObjNum 443 snap 139 seq_num 443) 2024-08-17T17:02:25.831 INFO:tasks.rados.rados.0.smithi063.stdout:1805: expect (ObjNum 691 snap 221 seq_num 691) 2024-08-17T17:02:26.622 INFO:tasks.rados.rados.0.smithi063.stdout:1805: done (2 left) 2024-08-17T17:02:26.623 INFO:tasks.rados.rados.0.smithi063.stdout:1806: done (1 left) 2024-08-17T17:02:26.623 INFO:tasks.rados.rados.0.smithi063.stdout:1807: done (0 left) 2024-08-17T17:02:26.623 INFO:tasks.rados.rados.0.smithi063.stdout:1808: snap_create 2024-08-17T17:02:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:26 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:26 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:26 smithi162 ceph-mon[104668]: Reconfiguring osd.5 (monmap changed)... 2024-08-17T17:02:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:26 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-08-17T17:02:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:26 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:26 smithi162 ceph-mon[104668]: Reconfiguring daemon osd.5 on smithi162 2024-08-17T17:02:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:26 smithi162 ceph-mon[104668]: osdmap e518: 8 total, 8 up, 8 in 2024-08-17T17:02:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:26 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:26 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:27.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:26 smithi063 ceph-mon[109843]: Reconfiguring osd.5 (monmap changed)... 2024-08-17T17:02:27.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:26 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-08-17T17:02:27.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:26 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:27.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:26 smithi063 ceph-mon[109843]: Reconfiguring daemon osd.5 on smithi162 2024-08-17T17:02:27.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:26 smithi063 ceph-mon[109843]: osdmap e518: 8 total, 8 up, 8 in 2024-08-17T17:02:27.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:26 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:27.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:26 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:27.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:26 smithi063 ceph-mon[114756]: Reconfiguring osd.5 (monmap changed)... 2024-08-17T17:02:27.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:26 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-08-17T17:02:27.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:26 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:27.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:26 smithi063 ceph-mon[114756]: Reconfiguring daemon osd.5 on smithi162 2024-08-17T17:02:27.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:26 smithi063 ceph-mon[114756]: osdmap e518: 8 total, 8 up, 8 in 2024-08-17T17:02:27.628 INFO:tasks.rados.rados.0.smithi063.stdout:1808: done (0 left) 2024-08-17T17:02:27.628 INFO:tasks.rados.rados.0.smithi063.stdout:1809: read oid 31 snap 220 2024-08-17T17:02:27.628 INFO:tasks.rados.rados.0.smithi063.stdout:1809: expect deleted 2024-08-17T17:02:27.628 INFO:tasks.rados.rados.0.smithi063.stdout:1810: rollback oid 3 current snap is 226 2024-08-17T17:02:27.628 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 3 to 216 2024-08-17T17:02:27.628 INFO:tasks.rados.rados.0.smithi063.stdout:1811: rollback oid 37 current snap is 226 2024-08-17T17:02:27.628 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 224 2024-08-17T17:02:27.629 INFO:tasks.rados.rados.0.smithi063.stdout:1812: write oid 34 current snap is 226 2024-08-17T17:02:27.629 INFO:tasks.rados.rados.0.smithi063.stdout:1812: seq_num 697 ranges {514479=636676,1853714=446233,2991120=542240} 2024-08-17T17:02:27.635 INFO:tasks.rados.rados.0.smithi063.stdout:1812: writing smithi06387500-34 from 514479 to 1151155 tid 1 2024-08-17T17:02:27.641 INFO:tasks.rados.rados.0.smithi063.stdout:1812: writing smithi06387500-34 from 1853714 to 2299947 tid 2 2024-08-17T17:02:27.648 INFO:tasks.rados.rados.0.smithi063.stdout:1812: writing smithi06387500-34 from 2991120 to 3533360 tid 3 2024-08-17T17:02:27.649 INFO:tasks.rados.rados.0.smithi063.stdout:1813: snap_create 2024-08-17T17:02:27.649 INFO:tasks.rados.rados.0.smithi063.stdout:1812: finishing write tid 1 to smithi06387500-34 2024-08-17T17:02:27.649 INFO:tasks.rados.rados.0.smithi063.stdout:1810: finishing rollback tid 1 to smithi06387500-3 2024-08-17T17:02:27.649 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T17:02:27.649 INFO:tasks.rados.rados.0.smithi063.stdout:1811: finishing rollback tid 0 to smithi06387500-37 2024-08-17T17:02:27.649 INFO:tasks.rados.rados.0.smithi063.stdout:1811: finishing rollback tid 1 to smithi06387500-37 2024-08-17T17:02:27.649 INFO:tasks.rados.rados.0.smithi063.stdout:1811: finishing rollback tid 2 to smithi06387500-37 2024-08-17T17:02:27.649 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 1721 (ObjNum 696 snap 222 seq_num 696) dirty exists 2024-08-17T17:02:27.649 INFO:tasks.rados.rados.0.smithi063.stdout:1812: finishing write tid 2 to smithi06387500-34 2024-08-17T17:02:27.651 INFO:tasks.rados.rados.0.smithi063.stdout:1812: finishing write tid 3 to smithi06387500-34 2024-08-17T17:02:27.651 INFO:tasks.rados.rados.0.smithi063.stdout:1812: finishing write tid 4 to smithi06387500-34 2024-08-17T17:02:27.652 INFO:tasks.rados.rados.0.smithi063.stdout:1812: finishing write tid 5 to smithi06387500-34 2024-08-17T17:02:27.652 INFO:tasks.rados.rados.0.smithi063.stdout:1812: oid 34 updating version 0 to 1261 2024-08-17T17:02:27.652 INFO:tasks.rados.rados.0.smithi063.stdout:1812: oid 34 version 1261 is already newer than 1259 2024-08-17T17:02:27.652 INFO:tasks.rados.rados.0.smithi063.stdout:1812: oid 34 version 1261 is already newer than 1258 2024-08-17T17:02:27.652 INFO:tasks.rados.rados.0.smithi063.stdout:1812: oid 34 version 1261 is already newer than 1260 2024-08-17T17:02:27.652 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 1261 (ObjNum 697 snap 226 seq_num 697) dirty exists 2024-08-17T17:02:27.652 INFO:tasks.rados.rados.0.smithi063.stdout:1812: left oid 34 (ObjNum 697 snap 226 seq_num 697) 2024-08-17T17:02:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[114756]: pgmap v234: 89 pgs: 89 active+clean; 103 MiB data, 545 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[114756]: osdmap e519: 8 total, 8 up, 8 in 2024-08-17T17:02:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:28.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:28.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[114756]: Reconfiguring osd.6 (monmap changed)... 2024-08-17T17:02:28.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-08-17T17:02:28.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:28.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[114756]: Reconfiguring daemon osd.6 on smithi162 2024-08-17T17:02:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[109843]: pgmap v234: 89 pgs: 89 active+clean; 103 MiB data, 545 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[109843]: osdmap e519: 8 total, 8 up, 8 in 2024-08-17T17:02:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[109843]: Reconfiguring osd.6 (monmap changed)... 2024-08-17T17:02:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-08-17T17:02:28.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:28.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:27 smithi063 ceph-mon[109843]: Reconfiguring daemon osd.6 on smithi162 2024-08-17T17:02:28.064 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:27 smithi162 ceph-mon[104668]: pgmap v234: 89 pgs: 89 active+clean; 103 MiB data, 545 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:28.064 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:27 smithi162 ceph-mon[104668]: osdmap e519: 8 total, 8 up, 8 in 2024-08-17T17:02:28.065 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:27 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:28.065 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:27 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:28.065 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:27 smithi162 ceph-mon[104668]: Reconfiguring osd.6 (monmap changed)... 2024-08-17T17:02:28.065 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:27 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-08-17T17:02:28.065 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:27 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:28.065 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:27 smithi162 ceph-mon[104668]: Reconfiguring daemon osd.6 on smithi162 2024-08-17T17:02:28.638 INFO:tasks.rados.rados.0.smithi063.stdout:1809: done (4 left) 2024-08-17T17:02:28.638 INFO:tasks.rados.rados.0.smithi063.stdout:1810: done (3 left) 2024-08-17T17:02:28.638 INFO:tasks.rados.rados.0.smithi063.stdout:1811: done (2 left) 2024-08-17T17:02:28.638 INFO:tasks.rados.rados.0.smithi063.stdout:1812: done (1 left) 2024-08-17T17:02:28.638 INFO:tasks.rados.rados.0.smithi063.stdout:1813: done (0 left) 2024-08-17T17:02:28.639 INFO:tasks.rados.rados.0.smithi063.stdout:1814: rollback oid 49 current snap is 227 2024-08-17T17:02:28.639 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 49 to 220 2024-08-17T17:02:28.639 INFO:tasks.rados.rados.0.smithi063.stdout:1815: snap_create 2024-08-17T17:02:28.644 INFO:tasks.rados.rados.0.smithi063.stdout:1814: finishing rollback tid 0 to smithi06387500-49 2024-08-17T17:02:28.644 INFO:tasks.rados.rados.0.smithi063.stdout:1814: finishing rollback tid 1 to smithi06387500-49 2024-08-17T17:02:28.644 INFO:tasks.rados.rados.0.smithi063.stdout:1814: finishing rollback tid 2 to smithi06387500-49 2024-08-17T17:02:28.644 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 1725 (ObjNum 655 snap 209 seq_num 655) dirty exists 2024-08-17T17:02:28.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:28 smithi162 ceph-mon[104668]: osdmap e520: 8 total, 8 up, 8 in 2024-08-17T17:02:28.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:28 smithi162 ceph-mon[104668]: pgmap v237: 89 pgs: 89 active+clean; 118 MiB data, 581 MiB used, 715 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-08-17T17:02:28.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:28 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:28.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:28 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:28.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:28 smithi162 ceph-mon[104668]: Reconfiguring osd.7 (monmap changed)... 2024-08-17T17:02:28.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:28 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-08-17T17:02:28.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:28 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:28.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:28 smithi162 ceph-mon[104668]: Reconfiguring daemon osd.7 on smithi162 2024-08-17T17:02:29.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[114756]: osdmap e520: 8 total, 8 up, 8 in 2024-08-17T17:02:29.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[114756]: pgmap v237: 89 pgs: 89 active+clean; 118 MiB data, 581 MiB used, 715 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-08-17T17:02:29.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:29.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:29.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[114756]: Reconfiguring osd.7 (monmap changed)... 2024-08-17T17:02:29.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-08-17T17:02:29.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:29.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[114756]: Reconfiguring daemon osd.7 on smithi162 2024-08-17T17:02:29.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[109843]: osdmap e520: 8 total, 8 up, 8 in 2024-08-17T17:02:29.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[109843]: pgmap v237: 89 pgs: 89 active+clean; 118 MiB data, 581 MiB used, 715 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-08-17T17:02:29.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:29.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:29.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[109843]: Reconfiguring osd.7 (monmap changed)... 2024-08-17T17:02:29.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-08-17T17:02:29.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:29.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:28 smithi063 ceph-mon[109843]: Reconfiguring daemon osd.7 on smithi162 2024-08-17T17:02:29.639 INFO:tasks.rados.rados.0.smithi063.stdout:1814: done (1 left) 2024-08-17T17:02:29.639 INFO:tasks.rados.rados.0.smithi063.stdout:1815: done (0 left) 2024-08-17T17:02:29.639 INFO:tasks.rados.rados.0.smithi063.stdout:1816: delete oid 18 current snap is 228 2024-08-17T17:02:29.643 INFO:tasks.rados.rados.0.smithi063.stdout:1816: done (0 left) 2024-08-17T17:02:29.643 INFO:tasks.rados.rados.0.smithi063.stdout:1817: write oid 38 current snap is 228 2024-08-17T17:02:29.643 INFO:tasks.rados.rados.0.smithi063.stdout:1817: seq_num 699 ranges {548093=787174,1970678=449902,3045966=681137} 2024-08-17T17:02:29.650 INFO:tasks.rados.rados.0.smithi063.stdout:1817: writing smithi06387500-38 from 548093 to 1335267 tid 1 2024-08-17T17:02:29.656 INFO:tasks.rados.rados.0.smithi063.stdout:1817: writing smithi06387500-38 from 1970678 to 2420580 tid 2 2024-08-17T17:02:29.663 INFO:tasks.rados.rados.0.smithi063.stdout:1817: writing smithi06387500-38 from 3045966 to 3727103 tid 3 2024-08-17T17:02:29.663 INFO:tasks.rados.rados.0.smithi063.stdout:1818: snap_create 2024-08-17T17:02:29.663 INFO:tasks.rados.rados.0.smithi063.stdout:1817: finishing write tid 1 to smithi06387500-38 2024-08-17T17:02:29.664 INFO:tasks.rados.rados.0.smithi063.stdout:1817: finishing write tid 2 to smithi06387500-38 2024-08-17T17:02:29.667 INFO:tasks.rados.rados.0.smithi063.stdout:1817: finishing write tid 3 to smithi06387500-38 2024-08-17T17:02:29.667 INFO:tasks.rados.rados.0.smithi063.stdout:1817: finishing write tid 4 to smithi06387500-38 2024-08-17T17:02:29.667 INFO:tasks.rados.rados.0.smithi063.stdout:1817: finishing write tid 5 to smithi06387500-38 2024-08-17T17:02:29.667 INFO:tasks.rados.rados.0.smithi063.stdout:1817: oid 38 updating version 0 to 261 2024-08-17T17:02:29.667 INFO:tasks.rados.rados.0.smithi063.stdout:1817: oid 38 updating version 261 to 264 2024-08-17T17:02:29.667 INFO:tasks.rados.rados.0.smithi063.stdout:1817: oid 38 version 264 is already newer than 262 2024-08-17T17:02:29.667 INFO:tasks.rados.rados.0.smithi063.stdout:1817: oid 38 version 264 is already newer than 263 2024-08-17T17:02:29.667 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 264 (ObjNum 699 snap 228 seq_num 699) dirty exists 2024-08-17T17:02:29.667 INFO:tasks.rados.rados.0.smithi063.stdout:1817: left oid 38 (ObjNum 699 snap 228 seq_num 699) 2024-08-17T17:02:29.888 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:29 smithi162 ceph-mon[104668]: osdmap e521: 8 total, 8 up, 8 in 2024-08-17T17:02:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:29 smithi063 ceph-mon[109843]: osdmap e521: 8 total, 8 up, 8 in 2024-08-17T17:02:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:29 smithi063 ceph-mon[114756]: osdmap e521: 8 total, 8 up, 8 in 2024-08-17T17:02:30.661 INFO:tasks.rados.rados.0.smithi063.stdout:1817: done (1 left) 2024-08-17T17:02:30.661 INFO:tasks.rados.rados.0.smithi063.stdout:1818: done (0 left) 2024-08-17T17:02:30.661 INFO:tasks.rados.rados.0.smithi063.stdout:1819: snap_remove snap 223 2024-08-17T17:02:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[114756]: osdmap e522: 8 total, 8 up, 8 in 2024-08-17T17:02:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[114756]: pgmap v240: 89 pgs: 89 active+clean; 118 MiB data, 581 MiB used, 715 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-08-17T17:02:31.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:31.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:31.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[114756]: Reconfiguring mds.foo.smithi162.bjkxnv (monmap changed)... 2024-08-17T17:02:31.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi162.bjkxnv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:02:31.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi162.bjkxnv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:02:31.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:31.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[114756]: Reconfiguring daemon mds.foo.smithi162.bjkxnv on smithi162 2024-08-17T17:02:31.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[109843]: osdmap e522: 8 total, 8 up, 8 in 2024-08-17T17:02:31.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[109843]: pgmap v240: 89 pgs: 89 active+clean; 118 MiB data, 581 MiB used, 715 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-08-17T17:02:31.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:31.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:31.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[109843]: Reconfiguring mds.foo.smithi162.bjkxnv (monmap changed)... 2024-08-17T17:02:31.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi162.bjkxnv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:02:31.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi162.bjkxnv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:02:31.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:31.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:30 smithi063 ceph-mon[109843]: Reconfiguring daemon mds.foo.smithi162.bjkxnv on smithi162 2024-08-17T17:02:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:30 smithi162 ceph-mon[104668]: osdmap e522: 8 total, 8 up, 8 in 2024-08-17T17:02:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:30 smithi162 ceph-mon[104668]: pgmap v240: 89 pgs: 89 active+clean; 118 MiB data, 581 MiB used, 715 GiB / 715 GiB avail; 9.2 MiB/s rd, 2.6 MiB/s wr, 16 op/s 2024-08-17T17:02:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:30 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:30 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:30 smithi162 ceph-mon[104668]: Reconfiguring mds.foo.smithi162.bjkxnv (monmap changed)... 2024-08-17T17:02:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:30 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi162.bjkxnv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:02:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:30 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi162.bjkxnv", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-08-17T17:02:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:30 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:30 smithi162 ceph-mon[104668]: Reconfiguring daemon mds.foo.smithi162.bjkxnv on smithi162 2024-08-17T17:02:31.655 INFO:tasks.rados.rados.0.smithi063.stdout:1819: done (0 left) 2024-08-17T17:02:31.655 INFO:tasks.rados.rados.0.smithi063.stdout:1820: read oid 4 snap -1 2024-08-17T17:02:31.655 INFO:tasks.rados.rados.0.smithi063.stdout:1820: expect (ObjNum 590 snap 187 seq_num 590) 2024-08-17T17:02:31.655 INFO:tasks.rados.rados.0.smithi063.stdout:1821: snap_remove snap 219 2024-08-17T17:02:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: osdmap e523: 8 total, 8 up, 8 in 2024-08-17T17:02:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:02:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:02:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:02:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:02:32.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:32.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-17T17:02:32.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:02:32.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:02:32.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all grafana 2024-08-17T17:02:32.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all loki 2024-08-17T17:02:32.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: osdmap e523: 8 total, 8 up, 8 in 2024-08-17T17:02:32.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:02:32.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:32.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:32.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:02:32.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:02:32.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:32.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:02:32.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:32.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-17T17:02:32.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:02:32.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:02:32.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all grafana 2024-08-17T17:02:32.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all loki 2024-08-17T17:02:32.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all promtail 2024-08-17T17:02:32.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: Upgrade: Finalizing container_image settings 2024-08-17T17:02:32.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:02:32.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:02:32.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:02:32.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:02:32.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:02:32.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:02:32.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:02:32.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:02:32.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:02:32.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:02:32.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:02:32.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:02:32.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:02:32.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:02:32.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:02:32.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all promtail 2024-08-17T17:02:32.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.037 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: Upgrade: Finalizing container_image settings 2024-08-17T17:02:32.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:02:32.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:02:32.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:02:32.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:02:32.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:02:32.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:02:32.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:02:32.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:02:32.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:02:32.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:02:32.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:02:32.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:02:32.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:02:32.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:02:32.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:02:32.039 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:02:32.039 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:02:32.039 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:02:32.039 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:02:32.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:02:32.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:02:32.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:02:32.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:02:32.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: Upgrade: Complete! 2024-08-17T17:02:32.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:02:32.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:02:32.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:02:32.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:02:32.041 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:02:32.041 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:02:32.041 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:02:32.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:02:32.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:02:32.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:02:32.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:02:32.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:02:32.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: Upgrade: Complete! 2024-08-17T17:02:32.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:02:32.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:02:32.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:02:32.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:02:32.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: osdmap e523: 8 total, 8 up, 8 in 2024-08-17T17:02:32.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:02:32.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:32.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:32.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:02:32.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:02:32.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:32.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:02:32.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:32.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-08-17T17:02:32.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:02:32.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:02:32.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all grafana 2024-08-17T17:02:32.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all loki 2024-08-17T17:02:32.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:02:32.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all promtail 2024-08-17T17:02:32.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-08-17T17:02:32.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: Upgrade: Finalizing container_image settings 2024-08-17T17:02:32.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:02:32.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:02:32.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:02:32.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:02:32.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:02:32.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:02:32.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:02:32.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:02:32.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:02:32.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:02:32.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:02:32.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:02:32.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:02:32.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:02:32.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:02:32.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:02:32.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:02:32.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:02:32.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:02:32.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:02:32.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:02:32.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:02:32.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:02:32.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:02:32.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: Upgrade: Complete! 2024-08-17T17:02:32.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:02:32.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:02:32.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:02:32.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:02:32.542 INFO:tasks.rados.rados.0.smithi063.stdout:1821: done (1 left) 2024-08-17T17:02:32.542 INFO:tasks.rados.rados.0.smithi063.stdout:1822: delete oid 17 current snap is 229 2024-08-17T17:02:32.543 INFO:tasks.rados.rados.0.smithi063.stdout:1820: expect (ObjNum 590 snap 187 seq_num 590) 2024-08-17T17:02:32.684 INFO:tasks.rados.rados.0.smithi063.stdout:1820: done (1 left) 2024-08-17T17:02:32.684 INFO:tasks.rados.rados.0.smithi063.stdout:1822: done (0 left) 2024-08-17T17:02:32.684 INFO:tasks.rados.rados.0.smithi063.stdout:1823: read oid 11 snap -1 2024-08-17T17:02:32.684 INFO:tasks.rados.rados.0.smithi063.stdout:1823: expect deleted 2024-08-17T17:02:32.684 INFO:tasks.rados.rados.0.smithi063.stdout:1824: rollback oid 39 current snap is 229 2024-08-17T17:02:32.684 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 39 to 215 2024-08-17T17:02:32.684 INFO:tasks.rados.rados.0.smithi063.stdout:1825: write oid 36 current snap is 229 2024-08-17T17:02:32.684 INFO:tasks.rados.rados.0.smithi063.stdout:1825: seq_num 701 ranges {581707=654025,1687642=453571,2700812=703681} 2024-08-17T17:02:32.690 INFO:tasks.rados.rados.0.smithi063.stdout:1825: writing smithi06387500-36 from 581707 to 1235732 tid 1 2024-08-17T17:02:32.695 INFO:tasks.rados.rados.0.smithi063.stdout:1825: writing smithi06387500-36 from 1687642 to 2141213 tid 2 2024-08-17T17:02:32.702 INFO:tasks.rados.rados.0.smithi063.stdout:1825: writing smithi06387500-36 from 2700812 to 3404493 tid 3 2024-08-17T17:02:32.702 INFO:tasks.rados.rados.0.smithi063.stdout:1826: write oid 9 current snap is 229 2024-08-17T17:02:32.702 INFO:tasks.rados.rados.0.smithi063.stdout:1826: seq_num 702 ranges {598514=729274,1746124=597229,3128235=514953} 2024-08-17T17:02:32.709 INFO:tasks.rados.rados.0.smithi063.stdout:1826: writing smithi06387500-9 from 598514 to 1327788 tid 1 2024-08-17T17:02:32.715 INFO:tasks.rados.rados.0.smithi063.stdout:1826: writing smithi06387500-9 from 1746124 to 2343353 tid 2 2024-08-17T17:02:32.722 INFO:tasks.rados.rados.0.smithi063.stdout:1826: writing smithi06387500-9 from 3128235 to 3643188 tid 3 2024-08-17T17:02:32.722 INFO:tasks.rados.rados.0.smithi063.stdout:1827: read oid 20 snap -1 2024-08-17T17:02:32.723 INFO:tasks.rados.rados.0.smithi063.stdout:1827: expect deleted 2024-08-17T17:02:32.723 INFO:tasks.rados.rados.0.smithi063.stdout:1828: snap_remove snap 225 2024-08-17T17:02:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:32 smithi063 ceph-mon[114756]: osdmap e524: 8 total, 8 up, 8 in 2024-08-17T17:02:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:32 smithi063 ceph-mon[114756]: pgmap v243: 89 pgs: 89 active+clean; 121 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 866 KiB/s wr, 6 op/s 2024-08-17T17:02:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:32 smithi063 ceph-mon[114756]: osdmap e525: 8 total, 8 up, 8 in 2024-08-17T17:02:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:32 smithi063 ceph-mon[109843]: osdmap e524: 8 total, 8 up, 8 in 2024-08-17T17:02:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:32 smithi063 ceph-mon[109843]: pgmap v243: 89 pgs: 89 active+clean; 121 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 866 KiB/s wr, 6 op/s 2024-08-17T17:02:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:32 smithi063 ceph-mon[109843]: osdmap e525: 8 total, 8 up, 8 in 2024-08-17T17:02:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:32 smithi162 ceph-mon[104668]: osdmap e524: 8 total, 8 up, 8 in 2024-08-17T17:02:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:32 smithi162 ceph-mon[104668]: pgmap v243: 89 pgs: 89 active+clean; 121 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 866 KiB/s wr, 6 op/s 2024-08-17T17:02:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:32 smithi162 ceph-mon[104668]: osdmap e525: 8 total, 8 up, 8 in 2024-08-17T17:02:33.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:02:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:02:33] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.51.0" 2024-08-17T17:02:33.670 INFO:tasks.rados.rados.0.smithi063.stdout:1828: done (5 left) 2024-08-17T17:02:33.670 INFO:tasks.rados.rados.0.smithi063.stdout:1829: write oid 44 current snap is 229 2024-08-17T17:02:33.671 INFO:tasks.rados.rados.0.smithi063.stdout:1824: finishing rollback tid 0 to smithi06387500-39 2024-08-17T17:02:33.671 INFO:tasks.rados.rados.0.smithi063.stdout:1824: finishing rollback tid 1 to smithi06387500-39 2024-08-17T17:02:33.671 INFO:tasks.rados.rados.0.smithi063.stdout:1829: seq_num 703 ranges {615321=404523,1804606=457240,2755658=726225} 2024-08-17T17:02:33.677 INFO:tasks.rados.rados.0.smithi063.stdout:1829: writing smithi06387500-44 from 615321 to 1019844 tid 1 2024-08-17T17:02:33.685 INFO:tasks.rados.rados.0.smithi063.stdout:1829: writing smithi06387500-44 from 1804606 to 2261846 tid 2 2024-08-17T17:02:33.693 INFO:tasks.rados.rados.0.smithi063.stdout:1829: writing smithi06387500-44 from 2755658 to 3481883 tid 3 2024-08-17T17:02:33.693 INFO:tasks.rados.rados.0.smithi063.stdout:1823: done (5 left) 2024-08-17T17:02:33.693 INFO:tasks.rados.rados.0.smithi063.stdout:1830: rollback oid 48 current snap is 229 2024-08-17T17:02:33.693 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 48 to 216 2024-08-17T17:02:33.693 INFO:tasks.rados.rados.0.smithi063.stdout:1824: finishing rollback tid 2 to smithi06387500-39 2024-08-17T17:02:33.693 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 377 (ObjNum 657 snap 209 seq_num 657) dirty exists 2024-08-17T17:02:33.693 INFO:tasks.rados.rados.0.smithi063.stdout:1825: finishing write tid 1 to smithi06387500-36 2024-08-17T17:02:33.693 INFO:tasks.rados.rados.0.smithi063.stdout:1825: finishing write tid 2 to smithi06387500-36 2024-08-17T17:02:33.693 INFO:tasks.rados.rados.0.smithi063.stdout:1825: finishing write tid 3 to smithi06387500-36 2024-08-17T17:02:33.693 INFO:tasks.rados.rados.0.smithi063.stdout:1825: finishing write tid 4 to smithi06387500-36 2024-08-17T17:02:33.694 INFO:tasks.rados.rados.0.smithi063.stdout:1825: finishing write tid 5 to smithi06387500-36 2024-08-17T17:02:33.694 INFO:tasks.rados.rados.0.smithi063.stdout:1825: oid 36 updating version 0 to 1736 2024-08-17T17:02:33.694 INFO:tasks.rados.rados.0.smithi063.stdout:1825: oid 36 updating version 1736 to 1737 2024-08-17T17:02:33.694 INFO:tasks.rados.rados.0.smithi063.stdout:1825: oid 36 version 1737 is already newer than 1734 2024-08-17T17:02:33.694 INFO:tasks.rados.rados.0.smithi063.stdout:1825: oid 36 version 1737 is already newer than 1735 2024-08-17T17:02:33.694 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 1737 (ObjNum 701 snap 229 seq_num 701) dirty exists 2024-08-17T17:02:33.694 INFO:tasks.rados.rados.0.smithi063.stdout:1825: left oid 36 (ObjNum 701 snap 229 seq_num 701) 2024-08-17T17:02:33.694 INFO:tasks.rados.rados.0.smithi063.stdout:1826: finishing write tid 1 to smithi06387500-9 2024-08-17T17:02:33.694 INFO:tasks.rados.rados.0.smithi063.stdout:1826: finishing write tid 2 to smithi06387500-9 2024-08-17T17:02:33.694 INFO:tasks.rados.rados.0.smithi063.stdout:1826: finishing write tid 3 to smithi06387500-9 2024-08-17T17:02:33.694 INFO:tasks.rados.rados.0.smithi063.stdout:1826: finishing write tid 4 to smithi06387500-9 2024-08-17T17:02:33.694 INFO:tasks.rados.rados.0.smithi063.stdout:1826: finishing write tid 5 to smithi06387500-9 2024-08-17T17:02:33.695 INFO:tasks.rados.rados.0.smithi063.stdout:1826: oid 9 updating version 0 to 787 2024-08-17T17:02:33.695 INFO:tasks.rados.rados.0.smithi063.stdout:1826: oid 9 updating version 787 to 790 2024-08-17T17:02:33.695 INFO:tasks.rados.rados.0.smithi063.stdout:1826: oid 9 version 790 is already newer than 788 2024-08-17T17:02:33.695 INFO:tasks.rados.rados.0.smithi063.stdout:1826: oid 9 version 790 is already newer than 789 2024-08-17T17:02:33.695 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 790 (ObjNum 702 snap 229 seq_num 702) dirty exists 2024-08-17T17:02:33.695 INFO:tasks.rados.rados.0.smithi063.stdout:1826: left oid 9 (ObjNum 702 snap 229 seq_num 702) 2024-08-17T17:02:33.695 INFO:tasks.rados.rados.0.smithi063.stdout:1829: finishing write tid 1 to smithi06387500-44 2024-08-17T17:02:33.695 INFO:tasks.rados.rados.0.smithi063.stdout:1829: finishing write tid 2 to smithi06387500-44 2024-08-17T17:02:33.695 INFO:tasks.rados.rados.0.smithi063.stdout:1824: done (5 left) 2024-08-17T17:02:33.695 INFO:tasks.rados.rados.0.smithi063.stdout:1825: done (4 left) 2024-08-17T17:02:33.695 INFO:tasks.rados.rados.0.smithi063.stdout:1826: done (3 left) 2024-08-17T17:02:33.695 INFO:tasks.rados.rados.0.smithi063.stdout:1827: done (2 left) 2024-08-17T17:02:33.696 INFO:tasks.rados.rados.0.smithi063.stdout:1831: delete oid 17 current snap is 229 2024-08-17T17:02:33.696 INFO:tasks.rados.rados.0.smithi063.stdout:1831: done (2 left) 2024-08-17T17:02:33.696 INFO:tasks.rados.rados.0.smithi063.stdout:1832: write oid 4 current snap is 229 2024-08-17T17:02:33.696 INFO:tasks.rados.rados.0.smithi063.stdout:1832: seq_num 705 ranges {648935=555021,1805217=460909,2694151=465122,3817442=31493} 2024-08-17T17:02:33.701 INFO:tasks.rados.rados.0.smithi063.stdout:1832: writing smithi06387500-4 from 648935 to 1203956 tid 1 2024-08-17T17:02:33.707 INFO:tasks.rados.rados.0.smithi063.stdout:1832: writing smithi06387500-4 from 1805217 to 2266126 tid 2 2024-08-17T17:02:33.713 INFO:tasks.rados.rados.0.smithi063.stdout:1832: writing smithi06387500-4 from 2694151 to 3159273 tid 3 2024-08-17T17:02:33.717 INFO:tasks.rados.rados.0.smithi063.stdout:1832: writing smithi06387500-4 from 3817442 to 3848935 tid 4 2024-08-17T17:02:33.717 INFO:tasks.rados.rados.0.smithi063.stdout:1833: write oid 16 current snap is 229 2024-08-17T17:02:33.717 INFO:tasks.rados.rados.0.smithi063.stdout:1829: finishing write tid 3 to smithi06387500-44 2024-08-17T17:02:33.718 INFO:tasks.rados.rados.0.smithi063.stdout:1833: seq_num 706 ranges {665742=630270,1863699=604567,3121574=676394} 2024-08-17T17:02:33.724 INFO:tasks.rados.rados.0.smithi063.stdout:1833: writing smithi06387500-16 from 665742 to 1296012 tid 1 2024-08-17T17:02:33.731 INFO:tasks.rados.rados.0.smithi063.stdout:1833: writing smithi06387500-16 from 1863699 to 2468266 tid 2 2024-08-17T17:02:33.739 INFO:tasks.rados.rados.0.smithi063.stdout:1833: writing smithi06387500-16 from 3121574 to 3797968 tid 3 2024-08-17T17:02:33.739 INFO:tasks.rados.rados.0.smithi063.stdout:1834: rollback oid 21 current snap is 229 2024-08-17T17:02:33.739 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 21 to 218 2024-08-17T17:02:33.739 INFO:tasks.rados.rados.0.smithi063.stdout:1829: finishing write tid 4 to smithi06387500-44 2024-08-17T17:02:33.739 INFO:tasks.rados.rados.0.smithi063.stdout:1829: finishing write tid 5 to smithi06387500-44 2024-08-17T17:02:33.739 INFO:tasks.rados.rados.0.smithi063.stdout:1829: oid 44 updating version 0 to 308 2024-08-17T17:02:33.739 INFO:tasks.rados.rados.0.smithi063.stdout:1829: oid 44 updating version 308 to 310 2024-08-17T17:02:33.739 INFO:tasks.rados.rados.0.smithi063.stdout:1829: oid 44 version 310 is already newer than 309 2024-08-17T17:02:33.739 INFO:tasks.rados.rados.0.smithi063.stdout:1829: oid 44 version 310 is already newer than 307 2024-08-17T17:02:33.739 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 310 (ObjNum 703 snap 229 seq_num 703) dirty exists 2024-08-17T17:02:33.740 INFO:tasks.rados.rados.0.smithi063.stdout:1829: left oid 44 (ObjNum 703 snap 229 seq_num 703) 2024-08-17T17:02:33.740 INFO:tasks.rados.rados.0.smithi063.stdout:1830: finishing rollback tid 0 to smithi06387500-48 2024-08-17T17:02:33.740 INFO:tasks.rados.rados.0.smithi063.stdout:1829: done (4 left) 2024-08-17T17:02:33.740 INFO:tasks.rados.rados.0.smithi063.stdout:1835: rollback oid 39 current snap is 229 2024-08-17T17:02:33.740 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 39 to 221 2024-08-17T17:02:33.740 INFO:tasks.rados.rados.0.smithi063.stdout:1830: finishing rollback tid 1 to smithi06387500-48 2024-08-17T17:02:33.740 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 1744 (ObjNum 33 snap 0 seq_num 1807085712) dirty dne 2024-08-17T17:02:33.740 INFO:tasks.rados.rados.0.smithi063.stdout:1832: finishing write tid 1 to smithi06387500-4 2024-08-17T17:02:33.740 INFO:tasks.rados.rados.0.smithi063.stdout:1832: finishing write tid 2 to smithi06387500-4 2024-08-17T17:02:33.740 INFO:tasks.rados.rados.0.smithi063.stdout:1832: finishing write tid 3 to smithi06387500-4 2024-08-17T17:02:33.740 INFO:tasks.rados.rados.0.smithi063.stdout:1832: finishing write tid 4 to smithi06387500-4 2024-08-17T17:02:33.740 INFO:tasks.rados.rados.0.smithi063.stdout:1832: finishing write tid 5 to smithi06387500-4 2024-08-17T17:02:33.740 INFO:tasks.rados.rados.0.smithi063.stdout:1832: finishing write tid 6 to smithi06387500-4 2024-08-17T17:02:33.740 INFO:tasks.rados.rados.0.smithi063.stdout:1832: oid 4 updating version 0 to 1746 2024-08-17T17:02:33.741 INFO:tasks.rados.rados.0.smithi063.stdout:1832: oid 4 updating version 1746 to 1748 2024-08-17T17:02:33.741 INFO:tasks.rados.rados.0.smithi063.stdout:1832: oid 4 updating version 1748 to 1750 2024-08-17T17:02:33.741 INFO:tasks.rados.rados.0.smithi063.stdout:1832: oid 4 version 1750 is already newer than 1747 2024-08-17T17:02:33.741 INFO:tasks.rados.rados.0.smithi063.stdout:1832: oid 4 version 1750 is already newer than 1749 2024-08-17T17:02:33.741 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 1750 (ObjNum 705 snap 229 seq_num 705) dirty exists 2024-08-17T17:02:33.741 INFO:tasks.rados.rados.0.smithi063.stdout:1832: left oid 4 (ObjNum 705 snap 229 seq_num 705) 2024-08-17T17:02:33.741 INFO:tasks.rados.rados.0.smithi063.stdout:1833: finishing write tid 1 to smithi06387500-16 2024-08-17T17:02:33.741 INFO:tasks.rados.rados.0.smithi063.stdout:1833: finishing write tid 2 to smithi06387500-16 2024-08-17T17:02:33.741 INFO:tasks.rados.rados.0.smithi063.stdout:1830: done (4 left) 2024-08-17T17:02:33.741 INFO:tasks.rados.rados.0.smithi063.stdout:1832: done (3 left) 2024-08-17T17:02:33.741 INFO:tasks.rados.rados.0.smithi063.stdout:1836: snap_create 2024-08-17T17:02:33.741 INFO:tasks.rados.rados.0.smithi063.stdout:1834: finishing rollback tid 1 to smithi06387500-21 2024-08-17T17:02:33.742 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-08-17T17:02:33.743 INFO:tasks.rados.rados.0.smithi063.stdout:1835: finishing rollback tid 0 to smithi06387500-39 2024-08-17T17:02:33.743 INFO:tasks.rados.rados.0.smithi063.stdout:1833: finishing write tid 3 to smithi06387500-16 2024-08-17T17:02:33.743 INFO:tasks.rados.rados.0.smithi063.stdout:1835: finishing rollback tid 1 to smithi06387500-39 2024-08-17T17:02:33.743 INFO:tasks.rados.rados.0.smithi063.stdout:1833: finishing write tid 4 to smithi06387500-16 2024-08-17T17:02:33.743 INFO:tasks.rados.rados.0.smithi063.stdout:1835: finishing rollback tid 2 to smithi06387500-39 2024-08-17T17:02:33.743 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 382 (ObjNum 657 snap 209 seq_num 657) dirty exists 2024-08-17T17:02:33.743 INFO:tasks.rados.rados.0.smithi063.stdout:1833: finishing write tid 5 to smithi06387500-16 2024-08-17T17:02:33.743 INFO:tasks.rados.rados.0.smithi063.stdout:1833: oid 16 updating version 0 to 672 2024-08-17T17:02:33.744 INFO:tasks.rados.rados.0.smithi063.stdout:1833: oid 16 updating version 672 to 674 2024-08-17T17:02:33.744 INFO:tasks.rados.rados.0.smithi063.stdout:1833: oid 16 updating version 674 to 675 2024-08-17T17:02:33.744 INFO:tasks.rados.rados.0.smithi063.stdout:1833: oid 16 version 675 is already newer than 673 2024-08-17T17:02:33.744 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 675 (ObjNum 706 snap 229 seq_num 706) dirty exists 2024-08-17T17:02:33.744 INFO:tasks.rados.rados.0.smithi063.stdout:1833: left oid 16 (ObjNum 706 snap 229 seq_num 706) 2024-08-17T17:02:34.674 INFO:tasks.rados.rados.0.smithi063.stdout:1833: done (3 left) 2024-08-17T17:02:34.674 INFO:tasks.rados.rados.0.smithi063.stdout:1834: done (2 left) 2024-08-17T17:02:34.674 INFO:tasks.rados.rados.0.smithi063.stdout:1835: done (1 left) 2024-08-17T17:02:34.674 INFO:tasks.rados.rados.0.smithi063.stdout:1836: done (0 left) 2024-08-17T17:02:34.674 INFO:tasks.rados.rados.0.smithi063.stdout:1837: snap_create 2024-08-17T17:02:34.933 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:34 smithi063 ceph-mon[114756]: osdmap e526: 8 total, 8 up, 8 in 2024-08-17T17:02:34.933 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:34 smithi063 ceph-mon[114756]: pgmap v246: 89 pgs: 89 active+clean; 121 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 866 KiB/s wr, 6 op/s 2024-08-17T17:02:34.934 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:34 smithi063 ceph-mon[109843]: osdmap e526: 8 total, 8 up, 8 in 2024-08-17T17:02:34.934 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:34 smithi063 ceph-mon[109843]: pgmap v246: 89 pgs: 89 active+clean; 121 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 866 KiB/s wr, 6 op/s 2024-08-17T17:02:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:34 smithi162 ceph-mon[104668]: osdmap e526: 8 total, 8 up, 8 in 2024-08-17T17:02:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:34 smithi162 ceph-mon[104668]: pgmap v246: 89 pgs: 89 active+clean; 121 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 866 KiB/s wr, 6 op/s 2024-08-17T17:02:35.676 INFO:tasks.rados.rados.0.smithi063.stdout:1837: done (0 left) 2024-08-17T17:02:35.676 INFO:tasks.rados.rados.0.smithi063.stdout:1838: snap_create 2024-08-17T17:02:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:35 smithi063 ceph-mon[114756]: osdmap e527: 8 total, 8 up, 8 in 2024-08-17T17:02:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:35 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:35 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:35 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:36.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:35 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:02:36.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:35 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:35 smithi063 ceph-mon[109843]: osdmap e527: 8 total, 8 up, 8 in 2024-08-17T17:02:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:35 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:35 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:35 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:35 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:02:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:35 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:35 smithi162 ceph-mon[104668]: osdmap e527: 8 total, 8 up, 8 in 2024-08-17T17:02:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:35 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:35 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:35 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:35 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:02:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:35 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:36.683 INFO:tasks.rados.rados.0.smithi063.stdout:1838: done (0 left) 2024-08-17T17:02:36.683 INFO:tasks.rados.rados.0.smithi063.stdout:1839: read oid 26 snap 230 2024-08-17T17:02:36.683 INFO:tasks.rados.rados.0.smithi063.stdout:1839: expect (ObjNum 689 snap 220 seq_num 689) 2024-08-17T17:02:36.683 INFO:tasks.rados.rados.0.smithi063.stdout:1840: write oid 30 current snap is 232 2024-08-17T17:02:36.683 INFO:tasks.rados.rados.0.smithi063.stdout:1840: seq_num 707 ranges {682549=705519,1922181=464578,3148997=487666} 2024-08-17T17:02:36.690 INFO:tasks.rados.rados.0.smithi063.stdout:1840: writing smithi06387500-30 from 682549 to 1388068 tid 1 2024-08-17T17:02:36.697 INFO:tasks.rados.rados.0.smithi063.stdout:1840: writing smithi06387500-30 from 1922181 to 2386759 tid 2 2024-08-17T17:02:36.705 INFO:tasks.rados.rados.0.smithi063.stdout:1840: writing smithi06387500-30 from 3148997 to 3636663 tid 3 2024-08-17T17:02:36.705 INFO:tasks.rados.rados.0.smithi063.stdout:1841: snap_create 2024-08-17T17:02:36.705 INFO:tasks.rados.rados.0.smithi063.stdout:1839: expect (ObjNum 689 snap 220 seq_num 689) 2024-08-17T17:02:36.889 INFO:tasks.rados.rados.0.smithi063.stdout:1840: finishing write tid 1 to smithi06387500-30 2024-08-17T17:02:36.889 INFO:tasks.rados.rados.0.smithi063.stdout:1840: finishing write tid 2 to smithi06387500-30 2024-08-17T17:02:36.890 INFO:tasks.rados.rados.0.smithi063.stdout:1840: finishing write tid 3 to smithi06387500-30 2024-08-17T17:02:36.890 INFO:tasks.rados.rados.0.smithi063.stdout:1840: finishing write tid 4 to smithi06387500-30 2024-08-17T17:02:36.890 INFO:tasks.rados.rados.0.smithi063.stdout:1840: finishing write tid 5 to smithi06387500-30 2024-08-17T17:02:36.890 INFO:tasks.rados.rados.0.smithi063.stdout:1840: oid 30 updating version 0 to 791 2024-08-17T17:02:36.890 INFO:tasks.rados.rados.0.smithi063.stdout:1840: oid 30 updating version 791 to 792 2024-08-17T17:02:36.890 INFO:tasks.rados.rados.0.smithi063.stdout:1840: oid 30 updating version 792 to 794 2024-08-17T17:02:36.890 INFO:tasks.rados.rados.0.smithi063.stdout:1840: oid 30 version 794 is already newer than 793 2024-08-17T17:02:36.890 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 794 (ObjNum 707 snap 232 seq_num 707) dirty exists 2024-08-17T17:02:36.890 INFO:tasks.rados.rados.0.smithi063.stdout:1840: left oid 30 (ObjNum 707 snap 232 seq_num 707) 2024-08-17T17:02:37.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:36 smithi063 ceph-mon[114756]: osdmap e528: 8 total, 8 up, 8 in 2024-08-17T17:02:37.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:36 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:02:37.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:36 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:37.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:36 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:02:37.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:36 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:37.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:36 smithi063 ceph-mon[114756]: pgmap v249: 89 pgs: 89 active+clean; 121 MiB data, 591 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:37.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:36 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:37.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:36 smithi063 ceph-mon[109843]: osdmap e528: 8 total, 8 up, 8 in 2024-08-17T17:02:37.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:36 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:02:37.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:36 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:37.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:36 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:02:37.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:36 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:37.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:36 smithi063 ceph-mon[109843]: pgmap v249: 89 pgs: 89 active+clean; 121 MiB data, 591 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:37.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:36 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:36 smithi162 ceph-mon[104668]: osdmap e528: 8 total, 8 up, 8 in 2024-08-17T17:02:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:36 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:02:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:36 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:02:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:36 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:02:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:36 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:36 smithi162 ceph-mon[104668]: pgmap v249: 89 pgs: 89 active+clean; 121 MiB data, 591 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:36 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:02:37.694 INFO:tasks.rados.rados.0.smithi063.stdout:1839: done (2 left) 2024-08-17T17:02:37.694 INFO:tasks.rados.rados.0.smithi063.stdout:1840: done (1 left) 2024-08-17T17:02:37.694 INFO:tasks.rados.rados.0.smithi063.stdout:1841: done (0 left) 2024-08-17T17:02:37.694 INFO:tasks.rados.rados.0.smithi063.stdout:1842: delete oid 30 current snap is 233 2024-08-17T17:02:37.699 INFO:tasks.rados.rados.0.smithi063.stdout:1842: done (0 left) 2024-08-17T17:02:37.699 INFO:tasks.rados.rados.0.smithi063.stdout:1843: read oid 24 snap -1 2024-08-17T17:02:37.699 INFO:tasks.rados.rados.0.smithi063.stdout:1843: expect (ObjNum 679 snap 216 seq_num 679) 2024-08-17T17:02:37.700 INFO:tasks.rados.rados.0.smithi063.stdout:1844: read oid 19 snap 226 2024-08-17T17:02:37.700 INFO:tasks.rados.rados.0.smithi063.stdout:1844: expect deleted 2024-08-17T17:02:37.700 INFO:tasks.rados.rados.0.smithi063.stdout:1845: snap_create 2024-08-17T17:02:37.706 INFO:tasks.rados.rados.0.smithi063.stdout:1843: expect (ObjNum 679 snap 216 seq_num 679) 2024-08-17T17:02:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:37 smithi063 ceph-mon[114756]: osdmap e529: 8 total, 8 up, 8 in 2024-08-17T17:02:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:37 smithi063 ceph-mon[109843]: osdmap e529: 8 total, 8 up, 8 in 2024-08-17T17:02:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:37 smithi162 ceph-mon[104668]: osdmap e529: 8 total, 8 up, 8 in 2024-08-17T17:02:38.700 INFO:tasks.rados.rados.0.smithi063.stdout:1843: done (2 left) 2024-08-17T17:02:38.700 INFO:tasks.rados.rados.0.smithi063.stdout:1844: done (1 left) 2024-08-17T17:02:38.700 INFO:tasks.rados.rados.0.smithi063.stdout:1845: done (0 left) 2024-08-17T17:02:38.700 INFO:tasks.rados.rados.0.smithi063.stdout:1846: write oid 21 current snap is 234 2024-08-17T17:02:38.700 INFO:tasks.rados.rados.0.smithi063.stdout:1846: seq_num 709 ranges {716163=572370,2039145=468247,3203843=626563} 2024-08-17T17:02:38.707 INFO:tasks.rados.rados.0.smithi063.stdout:1846: writing smithi06387500-21 from 716163 to 1288533 tid 1 2024-08-17T17:02:38.714 INFO:tasks.rados.rados.0.smithi063.stdout:1846: writing smithi06387500-21 from 2039145 to 2507392 tid 2 2024-08-17T17:02:38.721 INFO:tasks.rados.rados.0.smithi063.stdout:1846: writing smithi06387500-21 from 3203843 to 3830406 tid 3 2024-08-17T17:02:38.721 INFO:tasks.rados.rados.0.smithi063.stdout:1847: write oid 30 current snap is 234 2024-08-17T17:02:38.722 INFO:tasks.rados.rados.0.smithi063.stdout:1847: seq_num 710 ranges {732970=647619,2097627=611905,3231266=437835} 2024-08-17T17:02:38.729 INFO:tasks.rados.rados.0.smithi063.stdout:1847: writing smithi06387500-30 from 732970 to 1380589 tid 1 2024-08-17T17:02:38.736 INFO:tasks.rados.rados.0.smithi063.stdout:1847: writing smithi06387500-30 from 2097627 to 2709532 tid 2 2024-08-17T17:02:38.742 INFO:tasks.rados.rados.0.smithi063.stdout:1847: writing smithi06387500-30 from 3231266 to 3669101 tid 3 2024-08-17T17:02:38.742 INFO:tasks.rados.rados.0.smithi063.stdout:1848: delete oid 4 current snap is 234 2024-08-17T17:02:38.742 INFO:tasks.rados.rados.0.smithi063.stdout:1846: finishing write tid 1 to smithi06387500-21 2024-08-17T17:02:38.743 INFO:tasks.rados.rados.0.smithi063.stdout:1846: finishing write tid 2 to smithi06387500-21 2024-08-17T17:02:38.743 INFO:tasks.rados.rados.0.smithi063.stdout:1846: finishing write tid 3 to smithi06387500-21 2024-08-17T17:02:38.743 INFO:tasks.rados.rados.0.smithi063.stdout:1846: finishing write tid 4 to smithi06387500-21 2024-08-17T17:02:38.743 INFO:tasks.rados.rados.0.smithi063.stdout:1846: finishing write tid 5 to smithi06387500-21 2024-08-17T17:02:38.743 INFO:tasks.rados.rados.0.smithi063.stdout:1846: oid 21 updating version 0 to 1269 2024-08-17T17:02:38.743 INFO:tasks.rados.rados.0.smithi063.stdout:1846: oid 21 updating version 1269 to 1271 2024-08-17T17:02:38.743 INFO:tasks.rados.rados.0.smithi063.stdout:1846: oid 21 version 1271 is already newer than 1268 2024-08-17T17:02:38.743 INFO:tasks.rados.rados.0.smithi063.stdout:1846: oid 21 version 1271 is already newer than 1270 2024-08-17T17:02:38.743 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 1271 (ObjNum 709 snap 234 seq_num 709) dirty exists 2024-08-17T17:02:38.743 INFO:tasks.rados.rados.0.smithi063.stdout:1846: left oid 21 (ObjNum 709 snap 234 seq_num 709) 2024-08-17T17:02:38.743 INFO:tasks.rados.rados.0.smithi063.stdout:1847: finishing write tid 1 to smithi06387500-30 2024-08-17T17:02:38.743 INFO:tasks.rados.rados.0.smithi063.stdout:1847: finishing write tid 2 to smithi06387500-30 2024-08-17T17:02:38.747 INFO:tasks.rados.rados.0.smithi063.stdout:1847: finishing write tid 3 to smithi06387500-30 2024-08-17T17:02:38.747 INFO:tasks.rados.rados.0.smithi063.stdout:1847: finishing write tid 4 to smithi06387500-30 2024-08-17T17:02:38.747 INFO:tasks.rados.rados.0.smithi063.stdout:1847: finishing write tid 5 to smithi06387500-30 2024-08-17T17:02:38.747 INFO:tasks.rados.rados.0.smithi063.stdout:1847: oid 30 updating version 0 to 797 2024-08-17T17:02:38.747 INFO:tasks.rados.rados.0.smithi063.stdout:1847: oid 30 updating version 797 to 800 2024-08-17T17:02:38.747 INFO:tasks.rados.rados.0.smithi063.stdout:1847: oid 30 version 800 is already newer than 798 2024-08-17T17:02:38.747 INFO:tasks.rados.rados.0.smithi063.stdout:1847: oid 30 version 800 is already newer than 799 2024-08-17T17:02:38.747 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 800 (ObjNum 710 snap 234 seq_num 710) dirty exists 2024-08-17T17:02:38.747 INFO:tasks.rados.rados.0.smithi063.stdout:1847: left oid 30 (ObjNum 710 snap 234 seq_num 710) 2024-08-17T17:02:38.748 INFO:tasks.rados.rados.0.smithi063.stdout:1846: done (2 left) 2024-08-17T17:02:38.748 INFO:tasks.rados.rados.0.smithi063.stdout:1847: done (1 left) 2024-08-17T17:02:38.748 INFO:tasks.rados.rados.0.smithi063.stdout:1848: done (0 left) 2024-08-17T17:02:38.748 INFO:tasks.rados.rados.0.smithi063.stdout:1849: snap_create 2024-08-17T17:02:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:38 smithi063 ceph-mon[114756]: osdmap e530: 8 total, 8 up, 8 in 2024-08-17T17:02:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:38 smithi063 ceph-mon[114756]: pgmap v252: 89 pgs: 89 active+clean; 139 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-08-17T17:02:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:38 smithi063 ceph-mon[109843]: osdmap e530: 8 total, 8 up, 8 in 2024-08-17T17:02:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:38 smithi063 ceph-mon[109843]: pgmap v252: 89 pgs: 89 active+clean; 139 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-08-17T17:02:39.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:38 smithi162 ceph-mon[104668]: osdmap e530: 8 total, 8 up, 8 in 2024-08-17T17:02:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:38 smithi162 ceph-mon[104668]: pgmap v252: 89 pgs: 89 active+clean; 139 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-08-17T17:02:39.703 INFO:tasks.rados.rados.0.smithi063.stdout:1849: done (0 left) 2024-08-17T17:02:39.703 INFO:tasks.rados.rados.0.smithi063.stdout:1850: read oid 7 snap -1 2024-08-17T17:02:39.703 INFO:tasks.rados.rados.0.smithi063.stdout:1850: expect (ObjNum 692 snap 221 seq_num 692) 2024-08-17T17:02:39.703 INFO:tasks.rados.rados.0.smithi063.stdout:1851: snap_remove snap 217 2024-08-17T17:02:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:39 smithi063 ceph-mon[114756]: osdmap e531: 8 total, 8 up, 8 in 2024-08-17T17:02:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:39 smithi063 ceph-mon[109843]: osdmap e531: 8 total, 8 up, 8 in 2024-08-17T17:02:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:39 smithi162 ceph-mon[104668]: osdmap e531: 8 total, 8 up, 8 in 2024-08-17T17:02:40.708 INFO:tasks.rados.rados.0.smithi063.stdout:1851: done (1 left) 2024-08-17T17:02:40.709 INFO:tasks.rados.rados.0.smithi063.stdout:1852: snap_create 2024-08-17T17:02:40.709 INFO:tasks.rados.rados.0.smithi063.stdout:1850: expect (ObjNum 692 snap 221 seq_num 692) 2024-08-17T17:02:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:40 smithi063 ceph-mon[114756]: osdmap e532: 8 total, 8 up, 8 in 2024-08-17T17:02:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:40 smithi063 ceph-mon[114756]: pgmap v255: 89 pgs: 89 active+clean; 139 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-08-17T17:02:41.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:40 smithi063 ceph-mon[109843]: osdmap e532: 8 total, 8 up, 8 in 2024-08-17T17:02:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:40 smithi063 ceph-mon[109843]: pgmap v255: 89 pgs: 89 active+clean; 139 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-08-17T17:02:41.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:40 smithi162 ceph-mon[104668]: osdmap e532: 8 total, 8 up, 8 in 2024-08-17T17:02:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:40 smithi162 ceph-mon[104668]: pgmap v255: 89 pgs: 89 active+clean; 139 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 4.0 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-08-17T17:02:41.712 INFO:tasks.rados.rados.0.smithi063.stdout:1850: done (1 left) 2024-08-17T17:02:41.712 INFO:tasks.rados.rados.0.smithi063.stdout:1852: done (0 left) 2024-08-17T17:02:41.713 INFO:tasks.rados.rados.0.smithi063.stdout:1853: read oid 38 snap -1 2024-08-17T17:02:41.713 INFO:tasks.rados.rados.0.smithi063.stdout:1853: expect (ObjNum 699 snap 228 seq_num 699) 2024-08-17T17:02:41.713 INFO:tasks.rados.rados.0.smithi063.stdout:1854: snap_create 2024-08-17T17:02:41.724 INFO:tasks.rados.rados.0.smithi063.stdout:1853: expect (ObjNum 699 snap 228 seq_num 699) 2024-08-17T17:02:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:41 smithi063 ceph-mon[109843]: osdmap e533: 8 total, 8 up, 8 in 2024-08-17T17:02:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:41 smithi063 ceph-mon[114756]: osdmap e533: 8 total, 8 up, 8 in 2024-08-17T17:02:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:41 smithi162 ceph-mon[104668]: osdmap e533: 8 total, 8 up, 8 in 2024-08-17T17:02:42.545 INFO:tasks.rados.rados.0.smithi063.stdout:1853: done (1 left) 2024-08-17T17:02:42.545 INFO:tasks.rados.rados.0.smithi063.stdout:1854: done (0 left) 2024-08-17T17:02:42.545 INFO:tasks.rados.rados.0.smithi063.stdout:1855: snap_create 2024-08-17T17:02:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:42 smithi063 ceph-mon[109843]: osdmap e534: 8 total, 8 up, 8 in 2024-08-17T17:02:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:42 smithi063 ceph-mon[109843]: pgmap v258: 89 pgs: 89 active+clean; 146 MiB data, 649 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s rd, 822 KiB/s wr, 9 op/s 2024-08-17T17:02:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:42 smithi063 ceph-mon[109843]: osdmap e535: 8 total, 8 up, 8 in 2024-08-17T17:02:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:42 smithi063 ceph-mon[114756]: osdmap e534: 8 total, 8 up, 8 in 2024-08-17T17:02:43.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:42 smithi063 ceph-mon[114756]: pgmap v258: 89 pgs: 89 active+clean; 146 MiB data, 649 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s rd, 822 KiB/s wr, 9 op/s 2024-08-17T17:02:43.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:42 smithi063 ceph-mon[114756]: osdmap e535: 8 total, 8 up, 8 in 2024-08-17T17:02:43.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:42 smithi162 ceph-mon[104668]: osdmap e534: 8 total, 8 up, 8 in 2024-08-17T17:02:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:42 smithi162 ceph-mon[104668]: pgmap v258: 89 pgs: 89 active+clean; 146 MiB data, 649 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s rd, 822 KiB/s wr, 9 op/s 2024-08-17T17:02:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:42 smithi162 ceph-mon[104668]: osdmap e535: 8 total, 8 up, 8 in 2024-08-17T17:02:43.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:02:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:02:43] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.51.0" 2024-08-17T17:02:43.546 INFO:tasks.rados.rados.0.smithi063.stdout:1855: done (0 left) 2024-08-17T17:02:43.546 INFO:tasks.rados.rados.0.smithi063.stdout:1856: write oid 5 current snap is 238 2024-08-17T17:02:43.546 INFO:tasks.rados.rados.0.smithi063.stdout:1856: seq_num 712 ranges {766584=798117,2098238=615574,3169759=460379} 2024-08-17T17:02:43.554 INFO:tasks.rados.rados.0.smithi063.stdout:1856: writing smithi06387500-5 from 766584 to 1564701 tid 1 2024-08-17T17:02:43.561 INFO:tasks.rados.rados.0.smithi063.stdout:1856: writing smithi06387500-5 from 2098238 to 2713812 tid 2 2024-08-17T17:02:43.567 INFO:tasks.rados.rados.0.smithi063.stdout:1856: writing smithi06387500-5 from 3169759 to 3630138 tid 3 2024-08-17T17:02:43.567 INFO:tasks.rados.rados.0.smithi063.stdout:1857: rollback oid 4 current snap is 238 2024-08-17T17:02:43.567 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 4 to 216 2024-08-17T17:02:43.567 INFO:tasks.rados.rados.0.smithi063.stdout:1856: finishing write tid 1 to smithi06387500-5 2024-08-17T17:02:43.567 INFO:tasks.rados.rados.0.smithi063.stdout:1856: finishing write tid 2 to smithi06387500-5 2024-08-17T17:02:43.567 INFO:tasks.rados.rados.0.smithi063.stdout:1858: snap_create 2024-08-17T17:02:43.571 INFO:tasks.rados.rados.0.smithi063.stdout:1856: finishing write tid 3 to smithi06387500-5 2024-08-17T17:02:43.571 INFO:tasks.rados.rados.0.smithi063.stdout:1856: finishing write tid 4 to smithi06387500-5 2024-08-17T17:02:43.571 INFO:tasks.rados.rados.0.smithi063.stdout:1856: finishing write tid 5 to smithi06387500-5 2024-08-17T17:02:43.571 INFO:tasks.rados.rados.0.smithi063.stdout:1856: oid 5 updating version 0 to 795 2024-08-17T17:02:43.571 INFO:tasks.rados.rados.0.smithi063.stdout:1856: oid 5 updating version 795 to 797 2024-08-17T17:02:43.571 INFO:tasks.rados.rados.0.smithi063.stdout:1856: oid 5 version 797 is already newer than 796 2024-08-17T17:02:43.571 INFO:tasks.rados.rados.0.smithi063.stdout:1856: oid 5 version 797 is already newer than 794 2024-08-17T17:02:43.571 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 797 (ObjNum 712 snap 238 seq_num 712) dirty exists 2024-08-17T17:02:43.571 INFO:tasks.rados.rados.0.smithi063.stdout:1856: left oid 5 (ObjNum 712 snap 238 seq_num 712) 2024-08-17T17:02:43.573 INFO:tasks.rados.rados.0.smithi063.stdout:1857: finishing rollback tid 1 to smithi06387500-4 2024-08-17T17:02:43.573 INFO:tasks.rados.rados.0.smithi063.stdout:1857: finishing rollback tid 2 to smithi06387500-4 2024-08-17T17:02:43.573 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 1760 (ObjNum 590 snap 187 seq_num 590) dirty exists 2024-08-17T17:02:44.553 INFO:tasks.rados.rados.0.smithi063.stdout:1856: done (2 left) 2024-08-17T17:02:44.553 INFO:tasks.rados.rados.0.smithi063.stdout:1857: done (1 left) 2024-08-17T17:02:44.554 INFO:tasks.rados.rados.0.smithi063.stdout:1858: done (0 left) 2024-08-17T17:02:44.554 INFO:tasks.rados.rados.0.smithi063.stdout:1859: read oid 32 snap -1 2024-08-17T17:02:44.554 INFO:tasks.rados.rados.0.smithi063.stdout:1859: expect (ObjNum 604 snap 191 seq_num 604) 2024-08-17T17:02:44.554 INFO:tasks.rados.rados.0.smithi063.stdout:1860: snap_create 2024-08-17T17:02:44.557 INFO:tasks.rados.rados.0.smithi063.stdout:1859: expect (ObjNum 604 snap 191 seq_num 604) 2024-08-17T17:02:44.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:44 smithi162 ceph-mon[104668]: osdmap e536: 8 total, 8 up, 8 in 2024-08-17T17:02:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:44 smithi063 ceph-mon[114756]: osdmap e536: 8 total, 8 up, 8 in 2024-08-17T17:02:45.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:44 smithi063 ceph-mon[109843]: osdmap e536: 8 total, 8 up, 8 in 2024-08-17T17:02:45.553 INFO:tasks.rados.rados.0.smithi063.stdout:1859: done (1 left) 2024-08-17T17:02:45.553 INFO:tasks.rados.rados.0.smithi063.stdout:1860: done (0 left) 2024-08-17T17:02:45.554 INFO:tasks.rados.rados.0.smithi063.stdout:1861: snap_create 2024-08-17T17:02:45.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:45 smithi162 ceph-mon[104668]: pgmap v261: 89 pgs: 89 active+clean; 146 MiB data, 649 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s rd, 822 KiB/s wr, 9 op/s 2024-08-17T17:02:45.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:45 smithi162 ceph-mon[104668]: osdmap e537: 8 total, 8 up, 8 in 2024-08-17T17:02:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:45 smithi063 ceph-mon[114756]: pgmap v261: 89 pgs: 89 active+clean; 146 MiB data, 649 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s rd, 822 KiB/s wr, 9 op/s 2024-08-17T17:02:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:45 smithi063 ceph-mon[114756]: osdmap e537: 8 total, 8 up, 8 in 2024-08-17T17:02:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:45 smithi063 ceph-mon[109843]: pgmap v261: 89 pgs: 89 active+clean; 146 MiB data, 649 MiB used, 715 GiB / 715 GiB avail; 8.5 MiB/s rd, 822 KiB/s wr, 9 op/s 2024-08-17T17:02:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:45 smithi063 ceph-mon[109843]: osdmap e537: 8 total, 8 up, 8 in 2024-08-17T17:02:46.564 INFO:tasks.rados.rados.0.smithi063.stdout:1861: done (0 left) 2024-08-17T17:02:46.564 INFO:tasks.rados.rados.0.smithi063.stdout:1862: write oid 39 current snap is 241 2024-08-17T17:02:46.564 INFO:tasks.rados.rados.0.smithi063.stdout:1862: seq_num 713 ranges {783391=473366,1756720=475585,2797182=788004} 2024-08-17T17:02:46.571 INFO:tasks.rados.rados.0.smithi063.stdout:1862: writing smithi06387500-39 from 783391 to 1256757 tid 1 2024-08-17T17:02:46.576 INFO:tasks.rados.rados.0.smithi063.stdout:1862: writing smithi06387500-39 from 1756720 to 2232305 tid 2 2024-08-17T17:02:46.584 INFO:tasks.rados.rados.0.smithi063.stdout:1862: writing smithi06387500-39 from 2797182 to 3585186 tid 3 2024-08-17T17:02:46.584 INFO:tasks.rados.rados.0.smithi063.stdout:1863: read oid 32 snap -1 2024-08-17T17:02:46.584 INFO:tasks.rados.rados.0.smithi063.stdout:1863: expect (ObjNum 604 snap 191 seq_num 604) 2024-08-17T17:02:46.584 INFO:tasks.rados.rados.0.smithi063.stdout:1864: snap_create 2024-08-17T17:02:46.584 INFO:tasks.rados.rados.0.smithi063.stdout:1862: finishing write tid 1 to smithi06387500-39 2024-08-17T17:02:46.584 INFO:tasks.rados.rados.0.smithi063.stdout:1862: finishing write tid 2 to smithi06387500-39 2024-08-17T17:02:46.588 INFO:tasks.rados.rados.0.smithi063.stdout:1862: finishing write tid 3 to smithi06387500-39 2024-08-17T17:02:46.588 INFO:tasks.rados.rados.0.smithi063.stdout:1862: finishing write tid 4 to smithi06387500-39 2024-08-17T17:02:46.588 INFO:tasks.rados.rados.0.smithi063.stdout:1862: finishing write tid 5 to smithi06387500-39 2024-08-17T17:02:46.588 INFO:tasks.rados.rados.0.smithi063.stdout:1862: oid 39 updating version 0 to 388 2024-08-17T17:02:46.588 INFO:tasks.rados.rados.0.smithi063.stdout:1862: oid 39 version 388 is already newer than 387 2024-08-17T17:02:46.588 INFO:tasks.rados.rados.0.smithi063.stdout:1862: oid 39 version 388 is already newer than 386 2024-08-17T17:02:46.588 INFO:tasks.rados.rados.0.smithi063.stdout:1862: oid 39 updating version 388 to 389 2024-08-17T17:02:46.588 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 389 (ObjNum 713 snap 241 seq_num 713) dirty exists 2024-08-17T17:02:46.588 INFO:tasks.rados.rados.0.smithi063.stdout:1862: left oid 39 (ObjNum 713 snap 241 seq_num 713) 2024-08-17T17:02:46.590 INFO:tasks.rados.rados.0.smithi063.stdout:1863: expect (ObjNum 604 snap 191 seq_num 604) 2024-08-17T17:02:46.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:46 smithi162 ceph-mon[104668]: osdmap e538: 8 total, 8 up, 8 in 2024-08-17T17:02:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:02:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:46 smithi063 ceph-mon[114756]: osdmap e538: 8 total, 8 up, 8 in 2024-08-17T17:02:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:02:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:46 smithi063 ceph-mon[109843]: osdmap e538: 8 total, 8 up, 8 in 2024-08-17T17:02:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:02:47.545 INFO:tasks.rados.rados.0.smithi063.stdout:1862: done (2 left) 2024-08-17T17:02:47.545 INFO:tasks.rados.rados.0.smithi063.stdout:1863: done (1 left) 2024-08-17T17:02:47.546 INFO:tasks.rados.rados.0.smithi063.stdout:1864: done (0 left) 2024-08-17T17:02:47.546 INFO:tasks.rados.rados.0.smithi063.stdout:1865: snap_create 2024-08-17T17:02:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:47 smithi162 ceph-mon[104668]: pgmap v264: 89 pgs: 89 active+clean; 146 MiB data, 649 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:47 smithi162 ceph-mon[104668]: osdmap e539: 8 total, 8 up, 8 in 2024-08-17T17:02:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:47 smithi162 ceph-mon[104668]: osdmap e540: 8 total, 8 up, 8 in 2024-08-17T17:02:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:47 smithi063 ceph-mon[114756]: pgmap v264: 89 pgs: 89 active+clean; 146 MiB data, 649 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:47 smithi063 ceph-mon[114756]: osdmap e539: 8 total, 8 up, 8 in 2024-08-17T17:02:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:47 smithi063 ceph-mon[114756]: osdmap e540: 8 total, 8 up, 8 in 2024-08-17T17:02:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:47 smithi063 ceph-mon[109843]: pgmap v264: 89 pgs: 89 active+clean; 146 MiB data, 649 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:47 smithi063 ceph-mon[109843]: osdmap e539: 8 total, 8 up, 8 in 2024-08-17T17:02:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:47 smithi063 ceph-mon[109843]: osdmap e540: 8 total, 8 up, 8 in 2024-08-17T17:02:48.549 INFO:tasks.rados.rados.0.smithi063.stdout:1865: done (0 left) 2024-08-17T17:02:48.549 INFO:tasks.rados.rados.0.smithi063.stdout:1866: read oid 35 snap -1 2024-08-17T17:02:48.549 INFO:tasks.rados.rados.0.smithi063.stdout:1866: expect (ObjNum 683 snap 218 seq_num 683) 2024-08-17T17:02:48.549 INFO:tasks.rados.rados.0.smithi063.stdout:1867: snap_remove snap 242 2024-08-17T17:02:49.553 INFO:tasks.rados.rados.0.smithi063.stdout:1867: done (1 left) 2024-08-17T17:02:49.553 INFO:tasks.rados.rados.0.smithi063.stdout:1868: write oid 25 current snap is 243 2024-08-17T17:02:49.553 INFO:tasks.rados.rados.0.smithi063.stdout:1868: seq_num 714 ranges {400198=548615,1415202=584996} 2024-08-17T17:02:49.558 INFO:tasks.rados.rados.0.smithi063.stdout:1868: writing smithi06387500-25 from 400198 to 948813 tid 1 2024-08-17T17:02:49.564 INFO:tasks.rados.rados.0.smithi063.stdout:1868: writing smithi06387500-25 from 1415202 to 2000198 tid 2 2024-08-17T17:02:49.565 INFO:tasks.rados.rados.0.smithi063.stdout:1869: snap_remove snap 215 2024-08-17T17:02:49.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:49 smithi162 ceph-mon[104668]: pgmap v267: 89 pgs: 89 active+clean; 154 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 882 KiB/s wr, 7 op/s 2024-08-17T17:02:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:49 smithi162 ceph-mon[104668]: osdmap e541: 8 total, 8 up, 8 in 2024-08-17T17:02:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:49 smithi063 ceph-mon[114756]: pgmap v267: 89 pgs: 89 active+clean; 154 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 882 KiB/s wr, 7 op/s 2024-08-17T17:02:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:49 smithi063 ceph-mon[114756]: osdmap e541: 8 total, 8 up, 8 in 2024-08-17T17:02:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:49 smithi063 ceph-mon[109843]: pgmap v267: 89 pgs: 89 active+clean; 154 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 882 KiB/s wr, 7 op/s 2024-08-17T17:02:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:49 smithi063 ceph-mon[109843]: osdmap e541: 8 total, 8 up, 8 in 2024-08-17T17:02:50.571 INFO:tasks.rados.rados.0.smithi063.stdout:1869: done (2 left) 2024-08-17T17:02:50.571 INFO:tasks.rados.rados.0.smithi063.stdout:1870: write oid 2 current snap is 243 2024-08-17T17:02:50.571 INFO:tasks.rados.rados.0.smithi063.stdout:1870: seq_num 715 ranges {417005=740217,1590037=426968} 2024-08-17T17:02:50.578 INFO:tasks.rados.rados.0.smithi063.stdout:1870: writing smithi06387500-2 from 417005 to 1157222 tid 1 2024-08-17T17:02:50.584 INFO:tasks.rados.rados.0.smithi063.stdout:1870: writing smithi06387500-2 from 1590037 to 2017005 tid 2 2024-08-17T17:02:50.584 INFO:tasks.rados.rados.0.smithi063.stdout:1871: read oid 6 snap -1 2024-08-17T17:02:50.584 INFO:tasks.rados.rados.0.smithi063.stdout:1871: expect (ObjNum 425 snap 133 seq_num 425) 2024-08-17T17:02:50.584 INFO:tasks.rados.rados.0.smithi063.stdout:1872: snap_remove snap 224 2024-08-17T17:02:50.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:50 smithi162 ceph-mon[104668]: osdmap e542: 8 total, 8 up, 8 in 2024-08-17T17:02:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:50 smithi063 ceph-mon[109843]: osdmap e542: 8 total, 8 up, 8 in 2024-08-17T17:02:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:50 smithi063 ceph-mon[114756]: osdmap e542: 8 total, 8 up, 8 in 2024-08-17T17:02:51.575 INFO:tasks.rados.rados.0.smithi063.stdout:1872: done (4 left) 2024-08-17T17:02:51.575 INFO:tasks.rados.rados.0.smithi063.stdout:1873: snap_create 2024-08-17T17:02:51.575 INFO:tasks.rados.rados.0.smithi063.stdout:1866: expect (ObjNum 683 snap 218 seq_num 683) 2024-08-17T17:02:51.694 INFO:tasks.rados.rados.0.smithi063.stdout:1868: finishing write tid 1 to smithi06387500-25 2024-08-17T17:02:51.694 INFO:tasks.rados.rados.0.smithi063.stdout:1868: finishing write tid 2 to smithi06387500-25 2024-08-17T17:02:51.694 INFO:tasks.rados.rados.0.smithi063.stdout:1868: finishing write tid 3 to smithi06387500-25 2024-08-17T17:02:51.694 INFO:tasks.rados.rados.0.smithi063.stdout:1868: finishing write tid 4 to smithi06387500-25 2024-08-17T17:02:51.694 INFO:tasks.rados.rados.0.smithi063.stdout:1868: oid 25 updating version 0 to 1764 2024-08-17T17:02:51.694 INFO:tasks.rados.rados.0.smithi063.stdout:1868: oid 25 version 1764 is already newer than 1763 2024-08-17T17:02:51.694 INFO:tasks.rados.rados.0.smithi063.stdout:1868: oid 25 version 1764 is already newer than 1762 2024-08-17T17:02:51.694 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 1764 (ObjNum 714 snap 243 seq_num 714) dirty exists 2024-08-17T17:02:51.694 INFO:tasks.rados.rados.0.smithi063.stdout:1868: left oid 25 (ObjNum 714 snap 243 seq_num 714) 2024-08-17T17:02:51.695 INFO:tasks.rados.rados.0.smithi063.stdout:1870: finishing write tid 1 to smithi06387500-2 2024-08-17T17:02:51.695 INFO:tasks.rados.rados.0.smithi063.stdout:1870: finishing write tid 2 to smithi06387500-2 2024-08-17T17:02:51.695 INFO:tasks.rados.rados.0.smithi063.stdout:1870: finishing write tid 3 to smithi06387500-2 2024-08-17T17:02:51.695 INFO:tasks.rados.rados.0.smithi063.stdout:1870: finishing write tid 4 to smithi06387500-2 2024-08-17T17:02:51.695 INFO:tasks.rados.rados.0.smithi063.stdout:1870: oid 2 updating version 0 to 805 2024-08-17T17:02:51.695 INFO:tasks.rados.rados.0.smithi063.stdout:1870: oid 2 version 805 is already newer than 803 2024-08-17T17:02:51.695 INFO:tasks.rados.rados.0.smithi063.stdout:1870: oid 2 version 805 is already newer than 804 2024-08-17T17:02:51.695 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 805 (ObjNum 715 snap 243 seq_num 715) dirty exists 2024-08-17T17:02:51.695 INFO:tasks.rados.rados.0.smithi063.stdout:1870: left oid 2 (ObjNum 715 snap 243 seq_num 715) 2024-08-17T17:02:51.695 INFO:tasks.rados.rados.0.smithi063.stdout:1871: expect (ObjNum 425 snap 133 seq_num 425) 2024-08-17T17:02:51.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:51 smithi162 ceph-mon[104668]: pgmap v270: 89 pgs: 89 active+clean; 154 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 882 KiB/s wr, 7 op/s 2024-08-17T17:02:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:51 smithi162 ceph-mon[104668]: osdmap e543: 8 total, 8 up, 8 in 2024-08-17T17:02:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:51 smithi063 ceph-mon[114756]: pgmap v270: 89 pgs: 89 active+clean; 154 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 882 KiB/s wr, 7 op/s 2024-08-17T17:02:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:51 smithi063 ceph-mon[114756]: osdmap e543: 8 total, 8 up, 8 in 2024-08-17T17:02:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:51 smithi063 ceph-mon[109843]: pgmap v270: 89 pgs: 89 active+clean; 154 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 4.1 MiB/s rd, 882 KiB/s wr, 7 op/s 2024-08-17T17:02:52.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:51 smithi063 ceph-mon[109843]: osdmap e543: 8 total, 8 up, 8 in 2024-08-17T17:02:52.582 INFO:tasks.rados.rados.0.smithi063.stdout:1866: done (4 left) 2024-08-17T17:02:52.582 INFO:tasks.rados.rados.0.smithi063.stdout:1868: done (3 left) 2024-08-17T17:02:52.582 INFO:tasks.rados.rados.0.smithi063.stdout:1870: done (2 left) 2024-08-17T17:02:52.582 INFO:tasks.rados.rados.0.smithi063.stdout:1871: done (1 left) 2024-08-17T17:02:52.583 INFO:tasks.rados.rados.0.smithi063.stdout:1873: done (0 left) 2024-08-17T17:02:52.583 INFO:tasks.rados.rados.0.smithi063.stdout:1874: rollback oid 36 current snap is 244 2024-08-17T17:02:52.583 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 36 to 236 2024-08-17T17:02:52.583 INFO:tasks.rados.rados.0.smithi063.stdout:1875: read oid 34 snap -1 2024-08-17T17:02:52.583 INFO:tasks.rados.rados.0.smithi063.stdout:1875: expect (ObjNum 697 snap 226 seq_num 697) 2024-08-17T17:02:52.583 INFO:tasks.rados.rados.0.smithi063.stdout:1876: rollback oid 46 current snap is 244 2024-08-17T17:02:52.583 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 46 to 218 2024-08-17T17:02:52.583 INFO:tasks.rados.rados.0.smithi063.stdout:1877: rollback oid 8 current snap is 244 2024-08-17T17:02:52.583 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 8 to 216 2024-08-17T17:02:52.583 INFO:tasks.rados.rados.0.smithi063.stdout:1878: read oid 14 snap -1 2024-08-17T17:02:52.583 INFO:tasks.rados.rados.0.smithi063.stdout:1878: expect deleted 2024-08-17T17:02:52.583 INFO:tasks.rados.rados.0.smithi063.stdout:1879: snap_remove snap 226 2024-08-17T17:02:52.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:52 smithi063 ceph-mon[114756]: osdmap e544: 8 total, 8 up, 8 in 2024-08-17T17:02:52.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:52 smithi063 ceph-mon[109843]: osdmap e544: 8 total, 8 up, 8 in 2024-08-17T17:02:52.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:52 smithi162 ceph-mon[104668]: osdmap e544: 8 total, 8 up, 8 in 2024-08-17T17:02:52.868 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'sleep 60' 2024-08-17T17:02:53.081 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:02:53.466 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:02:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:02:53] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.51.0" 2024-08-17T17:02:53.588 INFO:tasks.rados.rados.0.smithi063.stdout:1879: done (5 left) 2024-08-17T17:02:53.588 INFO:tasks.rados.rados.0.smithi063.stdout:1880: write oid 6 current snap is 244 2024-08-17T17:02:53.589 INFO:tasks.rados.rados.0.smithi063.stdout:1877: finishing rollback tid 1 to smithi06387500-8 2024-08-17T17:02:53.589 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T17:02:53.589 INFO:tasks.rados.rados.0.smithi063.stdout:1880: seq_num 716 ranges {433812=415466,1648519=385293} 2024-08-17T17:02:53.593 INFO:tasks.rados.rados.0.smithi063.stdout:1880: writing smithi06387500-6 from 433812 to 849278 tid 1 2024-08-17T17:02:53.601 INFO:tasks.rados.rados.0.smithi063.stdout:1880: writing smithi06387500-6 from 1648519 to 2033812 tid 2 2024-08-17T17:02:53.601 INFO:tasks.rados.rados.0.smithi063.stdout:1877: done (5 left) 2024-08-17T17:02:53.601 INFO:tasks.rados.rados.0.smithi063.stdout:1881: write oid 9 current snap is 244 2024-08-17T17:02:53.601 INFO:tasks.rados.rados.0.smithi063.stdout:1874: finishing rollback tid 0 to smithi06387500-36 2024-08-17T17:02:53.601 INFO:tasks.rados.rados.0.smithi063.stdout:1876: finishing rollback tid 0 to smithi06387500-46 2024-08-17T17:02:53.601 INFO:tasks.rados.rados.0.smithi063.stdout:1874: finishing rollback tid 1 to smithi06387500-36 2024-08-17T17:02:53.601 INFO:tasks.rados.rados.0.smithi063.stdout:1876: finishing rollback tid 1 to smithi06387500-46 2024-08-17T17:02:53.601 INFO:tasks.rados.rados.0.smithi063.stdout:1874: finishing rollback tid 2 to smithi06387500-36 2024-08-17T17:02:53.601 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 1786 (ObjNum 701 snap 229 seq_num 701) dirty exists 2024-08-17T17:02:53.601 INFO:tasks.rados.rados.0.smithi063.stdout:1876: finishing rollback tid 2 to smithi06387500-46 2024-08-17T17:02:53.601 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 687 (ObjNum 510 snap 160 seq_num 510) dirty exists 2024-08-17T17:02:53.602 INFO:tasks.rados.rados.0.smithi063.stdout:1881: seq_num 717 ranges {450619=490715,1590648=459971} 2024-08-17T17:02:53.605 INFO:tasks.rados.rados.0.smithi063.stdout:1881: writing smithi06387500-9 from 450619 to 941334 tid 1 2024-08-17T17:02:53.612 INFO:tasks.rados.rados.0.smithi063.stdout:1881: writing smithi06387500-9 from 1590648 to 2050619 tid 2 2024-08-17T17:02:53.612 INFO:tasks.rados.rados.0.smithi063.stdout:1874: done (5 left) 2024-08-17T17:02:53.612 INFO:tasks.rados.rados.0.smithi063.stdout:1876: done (4 left) 2024-08-17T17:02:53.612 INFO:tasks.rados.rados.0.smithi063.stdout:1878: done (3 left) 2024-08-17T17:02:53.612 INFO:tasks.rados.rados.0.smithi063.stdout:1882: read oid 14 snap -1 2024-08-17T17:02:53.612 INFO:tasks.rados.rados.0.smithi063.stdout:1882: expect deleted 2024-08-17T17:02:53.612 INFO:tasks.rados.rados.0.smithi063.stdout:1883: write oid 41 current snap is 244 2024-08-17T17:02:53.612 INFO:tasks.rados.rados.0.smithi063.stdout:1883: seq_num 718 ranges {467426=565964,1649130=418296} 2024-08-17T17:02:53.617 INFO:tasks.rados.rados.0.smithi063.stdout:1883: writing smithi06387500-41 from 467426 to 1033390 tid 1 2024-08-17T17:02:53.623 INFO:tasks.rados.rados.0.smithi063.stdout:1883: writing smithi06387500-41 from 1649130 to 2067426 tid 2 2024-08-17T17:02:53.623 INFO:tasks.rados.rados.0.smithi063.stdout:1884: read oid 28 snap -1 2024-08-17T17:02:53.623 INFO:tasks.rados.rados.0.smithi063.stdout:1884: expect deleted 2024-08-17T17:02:53.623 INFO:tasks.rados.rados.0.smithi063.stdout:1885: snap_remove snap 222 2024-08-17T17:02:53.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:53 smithi063 ceph-mon[114756]: pgmap v273: 89 pgs: 89 active+clean; 154 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 562 KiB/s wr, 6 op/s 2024-08-17T17:02:53.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:53 smithi063 ceph-mon[114756]: from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:53.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:53 smithi063 ceph-mon[114756]: osdmap e545: 8 total, 8 up, 8 in 2024-08-17T17:02:53.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:53 smithi063 ceph-mon[109843]: pgmap v273: 89 pgs: 89 active+clean; 154 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 562 KiB/s wr, 6 op/s 2024-08-17T17:02:53.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:53 smithi063 ceph-mon[109843]: from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:53.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:53 smithi063 ceph-mon[109843]: osdmap e545: 8 total, 8 up, 8 in 2024-08-17T17:02:53.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:53 smithi162 ceph-mon[104668]: pgmap v273: 89 pgs: 89 active+clean; 154 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 562 KiB/s wr, 6 op/s 2024-08-17T17:02:53.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:53 smithi162 ceph-mon[104668]: from='client.34245 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:02:53.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:53 smithi162 ceph-mon[104668]: osdmap e545: 8 total, 8 up, 8 in 2024-08-17T17:02:54.599 INFO:tasks.rados.rados.0.smithi063.stdout:1885: done (6 left) 2024-08-17T17:02:54.599 INFO:tasks.rados.rados.0.smithi063.stdout:1886: snap_remove snap 241 2024-08-17T17:02:54.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:54 smithi063 ceph-mon[114756]: osdmap e546: 8 total, 8 up, 8 in 2024-08-17T17:02:54.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:54 smithi063 ceph-mon[109843]: osdmap e546: 8 total, 8 up, 8 in 2024-08-17T17:02:54.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:54 smithi162 ceph-mon[104668]: osdmap e546: 8 total, 8 up, 8 in 2024-08-17T17:02:55.604 INFO:tasks.rados.rados.0.smithi063.stdout:1886: done (6 left) 2024-08-17T17:02:55.605 INFO:tasks.rados.rados.0.smithi063.stdout:1887: snap_remove snap 228 2024-08-17T17:02:55.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:55 smithi162 ceph-mon[104668]: pgmap v276: 89 pgs: 89 active+clean; 154 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 562 KiB/s wr, 6 op/s 2024-08-17T17:02:55.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:55 smithi162 ceph-mon[104668]: osdmap e547: 8 total, 8 up, 8 in 2024-08-17T17:02:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:55 smithi063 ceph-mon[114756]: pgmap v276: 89 pgs: 89 active+clean; 154 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 562 KiB/s wr, 6 op/s 2024-08-17T17:02:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:55 smithi063 ceph-mon[114756]: osdmap e547: 8 total, 8 up, 8 in 2024-08-17T17:02:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:55 smithi063 ceph-mon[109843]: pgmap v276: 89 pgs: 89 active+clean; 154 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 562 KiB/s wr, 6 op/s 2024-08-17T17:02:56.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:55 smithi063 ceph-mon[109843]: osdmap e547: 8 total, 8 up, 8 in 2024-08-17T17:02:56.614 INFO:tasks.rados.rados.0.smithi063.stdout:1887: done (6 left) 2024-08-17T17:02:56.614 INFO:tasks.rados.rados.0.smithi063.stdout:1888: rollback oid 15 current snap is 244 2024-08-17T17:02:56.614 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 15 to 232 2024-08-17T17:02:56.614 INFO:tasks.rados.rados.0.smithi063.stdout:1875: expect (ObjNum 697 snap 226 seq_num 697) 2024-08-17T17:02:56.905 INFO:tasks.rados.rados.0.smithi063.stdout:1880: finishing write tid 1 to smithi06387500-6 2024-08-17T17:02:56.905 INFO:tasks.rados.rados.0.smithi063.stdout:1880: finishing write tid 2 to smithi06387500-6 2024-08-17T17:02:56.905 INFO:tasks.rados.rados.0.smithi063.stdout:1875: done (6 left) 2024-08-17T17:02:56.905 INFO:tasks.rados.rados.0.smithi063.stdout:1889: rollback oid 37 current snap is 244 2024-08-17T17:02:56.905 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 237 2024-08-17T17:02:56.905 INFO:tasks.rados.rados.0.smithi063.stdout:1880: finishing write tid 3 to smithi06387500-6 2024-08-17T17:02:56.905 INFO:tasks.rados.rados.0.smithi063.stdout:1880: finishing write tid 4 to smithi06387500-6 2024-08-17T17:02:56.905 INFO:tasks.rados.rados.0.smithi063.stdout:1880: oid 6 updating version 0 to 536 2024-08-17T17:02:56.905 INFO:tasks.rados.rados.0.smithi063.stdout:1880: oid 6 version 536 is already newer than 535 2024-08-17T17:02:56.906 INFO:tasks.rados.rados.0.smithi063.stdout:1880: oid 6 updating version 536 to 537 2024-08-17T17:02:56.906 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 537 (ObjNum 716 snap 244 seq_num 716) dirty exists 2024-08-17T17:02:56.906 INFO:tasks.rados.rados.0.smithi063.stdout:1880: left oid 6 (ObjNum 716 snap 244 seq_num 716) 2024-08-17T17:02:56.906 INFO:tasks.rados.rados.0.smithi063.stdout:1881: finishing write tid 1 to smithi06387500-9 2024-08-17T17:02:56.906 INFO:tasks.rados.rados.0.smithi063.stdout:1881: finishing write tid 2 to smithi06387500-9 2024-08-17T17:02:56.906 INFO:tasks.rados.rados.0.smithi063.stdout:1881: finishing write tid 3 to smithi06387500-9 2024-08-17T17:02:56.906 INFO:tasks.rados.rados.0.smithi063.stdout:1881: finishing write tid 4 to smithi06387500-9 2024-08-17T17:02:56.906 INFO:tasks.rados.rados.0.smithi063.stdout:1881: oid 9 updating version 0 to 803 2024-08-17T17:02:56.906 INFO:tasks.rados.rados.0.smithi063.stdout:1881: oid 9 version 803 is already newer than 802 2024-08-17T17:02:56.906 INFO:tasks.rados.rados.0.smithi063.stdout:1881: oid 9 updating version 803 to 804 2024-08-17T17:02:56.906 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 804 (ObjNum 717 snap 244 seq_num 717) dirty exists 2024-08-17T17:02:56.906 INFO:tasks.rados.rados.0.smithi063.stdout:1881: left oid 9 (ObjNum 717 snap 244 seq_num 717) 2024-08-17T17:02:56.907 INFO:tasks.rados.rados.0.smithi063.stdout:1883: finishing write tid 1 to smithi06387500-41 2024-08-17T17:02:56.907 INFO:tasks.rados.rados.0.smithi063.stdout:1883: finishing write tid 2 to smithi06387500-41 2024-08-17T17:02:56.907 INFO:tasks.rados.rados.0.smithi063.stdout:1883: finishing write tid 3 to smithi06387500-41 2024-08-17T17:02:56.907 INFO:tasks.rados.rados.0.smithi063.stdout:1883: finishing write tid 4 to smithi06387500-41 2024-08-17T17:02:56.907 INFO:tasks.rados.rados.0.smithi063.stdout:1883: oid 41 updating version 0 to 156 2024-08-17T17:02:56.907 INFO:tasks.rados.rados.0.smithi063.stdout:1883: oid 41 updating version 156 to 157 2024-08-17T17:02:56.907 INFO:tasks.rados.rados.0.smithi063.stdout:1883: oid 41 version 157 is already newer than 155 2024-08-17T17:02:56.907 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 157 (ObjNum 718 snap 244 seq_num 718) dirty exists 2024-08-17T17:02:56.907 INFO:tasks.rados.rados.0.smithi063.stdout:1883: left oid 41 (ObjNum 718 snap 244 seq_num 718) 2024-08-17T17:02:56.907 INFO:tasks.rados.rados.0.smithi063.stdout:1888: finishing rollback tid 1 to smithi06387500-15 2024-08-17T17:02:56.907 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T17:02:56.907 INFO:tasks.rados.rados.0.smithi063.stdout:1880: done (6 left) 2024-08-17T17:02:56.908 INFO:tasks.rados.rados.0.smithi063.stdout:1881: done (5 left) 2024-08-17T17:02:56.908 INFO:tasks.rados.rados.0.smithi063.stdout:1882: done (4 left) 2024-08-17T17:02:56.908 INFO:tasks.rados.rados.0.smithi063.stdout:1883: done (3 left) 2024-08-17T17:02:56.908 INFO:tasks.rados.rados.0.smithi063.stdout:1884: done (2 left) 2024-08-17T17:02:56.908 INFO:tasks.rados.rados.0.smithi063.stdout:1888: done (1 left) 2024-08-17T17:02:56.908 INFO:tasks.rados.rados.0.smithi063.stdout:1890: rollback oid 40 current snap is 244 2024-08-17T17:02:56.908 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 40 to 218 2024-08-17T17:02:56.908 INFO:tasks.rados.rados.0.smithi063.stdout:1891: read oid 2 snap -1 2024-08-17T17:02:56.908 INFO:tasks.rados.rados.0.smithi063.stdout:1891: expect (ObjNum 715 snap 243 seq_num 715) 2024-08-17T17:02:56.908 INFO:tasks.rados.rados.0.smithi063.stdout:1892: read oid 3 snap -1 2024-08-17T17:02:56.908 INFO:tasks.rados.rados.0.smithi063.stdout:1892: expect deleted 2024-08-17T17:02:56.908 INFO:tasks.rados.rados.0.smithi063.stdout:1893: read oid 26 snap -1 2024-08-17T17:02:56.908 INFO:tasks.rados.rados.0.smithi063.stdout:1893: expect (ObjNum 689 snap 220 seq_num 689) 2024-08-17T17:02:56.908 INFO:tasks.rados.rados.0.smithi063.stdout:1894: snap_create 2024-08-17T17:02:56.909 INFO:tasks.rados.rados.0.smithi063.stdout:1890: finishing rollback tid 1 to smithi06387500-40 2024-08-17T17:02:56.909 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 0 (ObjNum 289 snap 0 seq_num 1459010368) dirty dne 2024-08-17T17:02:56.910 INFO:tasks.rados.rados.0.smithi063.stdout:1889: finishing rollback tid 0 to smithi06387500-37 2024-08-17T17:02:56.910 INFO:tasks.rados.rados.0.smithi063.stdout:1889: finishing rollback tid 1 to smithi06387500-37 2024-08-17T17:02:56.910 INFO:tasks.rados.rados.0.smithi063.stdout:1889: finishing rollback tid 2 to smithi06387500-37 2024-08-17T17:02:56.910 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 1818 (ObjNum 696 snap 222 seq_num 696) dirty exists 2024-08-17T17:02:56.913 INFO:tasks.rados.rados.0.smithi063.stdout:1891: expect (ObjNum 715 snap 243 seq_num 715) 2024-08-17T17:02:56.997 INFO:tasks.rados.rados.0.smithi063.stdout:1893: expect (ObjNum 689 snap 220 seq_num 689) 2024-08-17T17:02:57.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:56 smithi063 ceph-mon[114756]: osdmap e548: 8 total, 8 up, 8 in 2024-08-17T17:02:57.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:56 smithi063 ceph-mon[109843]: osdmap e548: 8 total, 8 up, 8 in 2024-08-17T17:02:57.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:56 smithi162 ceph-mon[104668]: osdmap e548: 8 total, 8 up, 8 in 2024-08-17T17:02:57.549 INFO:tasks.rados.rados.0.smithi063.stdout:1889: done (5 left) 2024-08-17T17:02:57.550 INFO:tasks.rados.rados.0.smithi063.stdout:1890: done (4 left) 2024-08-17T17:02:57.550 INFO:tasks.rados.rados.0.smithi063.stdout:1891: done (3 left) 2024-08-17T17:02:57.550 INFO:tasks.rados.rados.0.smithi063.stdout:1892: done (2 left) 2024-08-17T17:02:57.550 INFO:tasks.rados.rados.0.smithi063.stdout:1893: done (1 left) 2024-08-17T17:02:57.550 INFO:tasks.rados.rados.0.smithi063.stdout:1894: done (0 left) 2024-08-17T17:02:57.550 INFO:tasks.rados.rados.0.smithi063.stdout:1895: rollback oid 11 current snap is 245 2024-08-17T17:02:57.550 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 11 to 244 2024-08-17T17:02:57.550 INFO:tasks.rados.rados.0.smithi063.stdout:1896: snap_remove snap 234 2024-08-17T17:02:58.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:57 smithi063 ceph-mon[114756]: pgmap v279: 89 pgs: 89 active+clean; 154 MiB data, 643 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:58.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:57 smithi063 ceph-mon[114756]: osdmap e549: 8 total, 8 up, 8 in 2024-08-17T17:02:58.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:57 smithi063 ceph-mon[114756]: osdmap e550: 8 total, 8 up, 8 in 2024-08-17T17:02:58.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:57 smithi063 ceph-mon[109843]: pgmap v279: 89 pgs: 89 active+clean; 154 MiB data, 643 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:58.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:57 smithi063 ceph-mon[109843]: osdmap e549: 8 total, 8 up, 8 in 2024-08-17T17:02:58.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:57 smithi063 ceph-mon[109843]: osdmap e550: 8 total, 8 up, 8 in 2024-08-17T17:02:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:57 smithi162 ceph-mon[104668]: pgmap v279: 89 pgs: 89 active+clean; 154 MiB data, 643 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:02:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:57 smithi162 ceph-mon[104668]: osdmap e549: 8 total, 8 up, 8 in 2024-08-17T17:02:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:57 smithi162 ceph-mon[104668]: osdmap e550: 8 total, 8 up, 8 in 2024-08-17T17:02:58.559 INFO:tasks.rados.rados.0.smithi063.stdout:1895: finishing rollback tid 1 to smithi06387500-11 2024-08-17T17:02:58.559 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T17:02:58.559 INFO:tasks.rados.rados.0.smithi063.stdout:1895: done (1 left) 2024-08-17T17:02:58.559 INFO:tasks.rados.rados.0.smithi063.stdout:1896: done (0 left) 2024-08-17T17:02:58.559 INFO:tasks.rados.rados.0.smithi063.stdout:1897: write oid 48 current snap is 245 2024-08-17T17:02:58.559 INFO:tasks.rados.rados.0.smithi063.stdout:1897: seq_num 719 ranges {484233=641213,1707612=376621} 2024-08-17T17:02:58.565 INFO:tasks.rados.rados.0.smithi063.stdout:1897: writing smithi06387500-48 from 484233 to 1125446 tid 1 2024-08-17T17:02:58.571 INFO:tasks.rados.rados.0.smithi063.stdout:1897: writing smithi06387500-48 from 1707612 to 2084233 tid 2 2024-08-17T17:02:58.571 INFO:tasks.rados.rados.0.smithi063.stdout:1898: snap_remove snap 244 2024-08-17T17:02:59.564 INFO:tasks.rados.rados.0.smithi063.stdout:1898: done (1 left) 2024-08-17T17:02:59.565 INFO:tasks.rados.rados.0.smithi063.stdout:1899: delete oid 40 current snap is 245 2024-08-17T17:02:59.565 INFO:tasks.rados.rados.0.smithi063.stdout:1897: finishing write tid 1 to smithi06387500-48 2024-08-17T17:02:59.565 INFO:tasks.rados.rados.0.smithi063.stdout:1897: finishing write tid 2 to smithi06387500-48 2024-08-17T17:02:59.565 INFO:tasks.rados.rados.0.smithi063.stdout:1897: finishing write tid 3 to smithi06387500-48 2024-08-17T17:02:59.565 INFO:tasks.rados.rados.0.smithi063.stdout:1897: finishing write tid 4 to smithi06387500-48 2024-08-17T17:02:59.565 INFO:tasks.rados.rados.0.smithi063.stdout:1897: oid 48 updating version 0 to 1823 2024-08-17T17:02:59.565 INFO:tasks.rados.rados.0.smithi063.stdout:1897: oid 48 updating version 1823 to 1826 2024-08-17T17:02:59.565 INFO:tasks.rados.rados.0.smithi063.stdout:1897: oid 48 updating version 1826 to 1827 2024-08-17T17:02:59.565 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 1827 (ObjNum 719 snap 245 seq_num 719) dirty exists 2024-08-17T17:02:59.565 INFO:tasks.rados.rados.0.smithi063.stdout:1897: left oid 48 (ObjNum 719 snap 245 seq_num 719) 2024-08-17T17:02:59.568 INFO:tasks.rados.rados.0.smithi063.stdout:1897: done (1 left) 2024-08-17T17:02:59.568 INFO:tasks.rados.rados.0.smithi063.stdout:1899: done (0 left) 2024-08-17T17:02:59.568 INFO:tasks.rados.rados.0.smithi063.stdout:1900: snap_create 2024-08-17T17:02:59.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:59 smithi162 ceph-mon[104668]: pgmap v282: 89 pgs: 89 active+clean; 155 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 669 KiB/s wr, 6 op/s 2024-08-17T17:02:59.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:02:59 smithi162 ceph-mon[104668]: osdmap e551: 8 total, 8 up, 8 in 2024-08-17T17:03:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:59 smithi063 ceph-mon[109843]: pgmap v282: 89 pgs: 89 active+clean; 155 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 669 KiB/s wr, 6 op/s 2024-08-17T17:03:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:02:59 smithi063 ceph-mon[109843]: osdmap e551: 8 total, 8 up, 8 in 2024-08-17T17:03:00.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:59 smithi063 ceph-mon[114756]: pgmap v282: 89 pgs: 89 active+clean; 155 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 669 KiB/s wr, 6 op/s 2024-08-17T17:03:00.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:02:59 smithi063 ceph-mon[114756]: osdmap e551: 8 total, 8 up, 8 in 2024-08-17T17:03:00.578 INFO:tasks.rados.rados.0.smithi063.stdout:1900: done (0 left) 2024-08-17T17:03:00.578 INFO:tasks.rados.rados.0.smithi063.stdout:1901: write oid 7 current snap is 246 2024-08-17T17:03:00.578 INFO:tasks.rados.rados.0.smithi063.stdout:1901: seq_num 721 ranges {517847=791711,2108223=9624} 2024-08-17T17:03:00.584 INFO:tasks.rados.rados.0.smithi063.stdout:1901: writing smithi06387500-7 from 517847 to 1309558 tid 1 2024-08-17T17:03:00.589 INFO:tasks.rados.rados.0.smithi063.stdout:1901: writing smithi06387500-7 from 2108223 to 2117847 tid 2 2024-08-17T17:03:00.590 INFO:tasks.rados.rados.0.smithi063.stdout:1902: rollback oid 45 current snap is 246 2024-08-17T17:03:00.590 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 45 to 221 2024-08-17T17:03:00.590 INFO:tasks.rados.rados.0.smithi063.stdout:1903: read oid 44 snap -1 2024-08-17T17:03:00.590 INFO:tasks.rados.rados.0.smithi063.stdout:1903: expect (ObjNum 703 snap 229 seq_num 703) 2024-08-17T17:03:00.590 INFO:tasks.rados.rados.0.smithi063.stdout:1904: write oid 9 current snap is 246 2024-08-17T17:03:00.590 INFO:tasks.rados.rados.0.smithi063.stdout:1904: seq_num 722 ranges {534654=466960,1766705=367949} 2024-08-17T17:03:00.595 INFO:tasks.rados.rados.0.smithi063.stdout:1904: writing smithi06387500-9 from 534654 to 1001614 tid 1 2024-08-17T17:03:00.602 INFO:tasks.rados.rados.0.smithi063.stdout:1904: writing smithi06387500-9 from 1766705 to 2134654 tid 2 2024-08-17T17:03:00.602 INFO:tasks.rados.rados.0.smithi063.stdout:1905: write oid 3 current snap is 246 2024-08-17T17:03:00.602 INFO:tasks.rados.rados.0.smithi063.stdout:1901: finishing write tid 1 to smithi06387500-7 2024-08-17T17:03:00.602 INFO:tasks.rados.rados.0.smithi063.stdout:1905: seq_num 723 ranges {551461=658562,1941540=209921} 2024-08-17T17:03:00.608 INFO:tasks.rados.rados.0.smithi063.stdout:1905: writing smithi06387500-3 from 551461 to 1210023 tid 1 2024-08-17T17:03:00.614 INFO:tasks.rados.rados.0.smithi063.stdout:1905: writing smithi06387500-3 from 1941540 to 2151461 tid 2 2024-08-17T17:03:00.614 INFO:tasks.rados.rados.0.smithi063.stdout:1906: read oid 15 snap -1 2024-08-17T17:03:00.614 INFO:tasks.rados.rados.0.smithi063.stdout:1906: expect deleted 2024-08-17T17:03:00.614 INFO:tasks.rados.rados.0.smithi063.stdout:1907: rollback oid 47 current snap is 246 2024-08-17T17:03:00.614 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 47 to 240 2024-08-17T17:03:00.614 INFO:tasks.rados.rados.0.smithi063.stdout:1901: finishing write tid 2 to smithi06387500-7 2024-08-17T17:03:00.614 INFO:tasks.rados.rados.0.smithi063.stdout:1901: finishing write tid 3 to smithi06387500-7 2024-08-17T17:03:00.614 INFO:tasks.rados.rados.0.smithi063.stdout:1901: finishing write tid 4 to smithi06387500-7 2024-08-17T17:03:00.614 INFO:tasks.rados.rados.0.smithi063.stdout:1901: oid 7 updating version 0 to 810 2024-08-17T17:03:00.614 INFO:tasks.rados.rados.0.smithi063.stdout:1901: oid 7 updating version 810 to 811 2024-08-17T17:03:00.614 INFO:tasks.rados.rados.0.smithi063.stdout:1901: oid 7 updating version 811 to 812 2024-08-17T17:03:00.615 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 812 (ObjNum 721 snap 246 seq_num 721) dirty exists 2024-08-17T17:03:00.615 INFO:tasks.rados.rados.0.smithi063.stdout:1901: left oid 7 (ObjNum 721 snap 246 seq_num 721) 2024-08-17T17:03:00.615 INFO:tasks.rados.rados.0.smithi063.stdout:1902: finishing rollback tid 1 to smithi06387500-45 2024-08-17T17:03:00.615 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T17:03:00.615 INFO:tasks.rados.rados.0.smithi063.stdout:1901: done (6 left) 2024-08-17T17:03:00.615 INFO:tasks.rados.rados.0.smithi063.stdout:1902: done (5 left) 2024-08-17T17:03:00.615 INFO:tasks.rados.rados.0.smithi063.stdout:1908: write oid 45 current snap is 246 2024-08-17T17:03:00.615 INFO:tasks.rados.rados.0.smithi063.stdout:1904: finishing write tid 1 to smithi06387500-9 2024-08-17T17:03:00.615 INFO:tasks.rados.rados.0.smithi063.stdout:1903: expect (ObjNum 703 snap 229 seq_num 703) 2024-08-17T17:03:00.743 INFO:tasks.rados.rados.0.smithi063.stdout:1904: finishing write tid 2 to smithi06387500-9 2024-08-17T17:03:00.743 INFO:tasks.rados.rados.0.smithi063.stdout:1904: finishing write tid 3 to smithi06387500-9 2024-08-17T17:03:00.743 INFO:tasks.rados.rados.0.smithi063.stdout:1904: finishing write tid 4 to smithi06387500-9 2024-08-17T17:03:00.743 INFO:tasks.rados.rados.0.smithi063.stdout:1904: oid 9 updating version 0 to 815 2024-08-17T17:03:00.743 INFO:tasks.rados.rados.0.smithi063.stdout:1904: oid 9 updating version 815 to 816 2024-08-17T17:03:00.743 INFO:tasks.rados.rados.0.smithi063.stdout:1904: oid 9 version 816 is already newer than 814 2024-08-17T17:03:00.744 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 816 (ObjNum 722 snap 246 seq_num 722) dirty exists 2024-08-17T17:03:00.744 INFO:tasks.rados.rados.0.smithi063.stdout:1904: left oid 9 (ObjNum 722 snap 246 seq_num 722) 2024-08-17T17:03:00.744 INFO:tasks.rados.rados.0.smithi063.stdout:1905: finishing write tid 1 to smithi06387500-3 2024-08-17T17:03:00.744 INFO:tasks.rados.rados.0.smithi063.stdout:1908: seq_num 724 ranges {568268=733811,2000022=168246} 2024-08-17T17:03:00.750 INFO:tasks.rados.rados.0.smithi063.stdout:1908: writing smithi06387500-45 from 568268 to 1302079 tid 1 2024-08-17T17:03:00.755 INFO:tasks.rados.rados.0.smithi063.stdout:1908: writing smithi06387500-45 from 2000022 to 2168268 tid 2 2024-08-17T17:03:00.755 INFO:tasks.rados.rados.0.smithi063.stdout:1903: done (5 left) 2024-08-17T17:03:00.755 INFO:tasks.rados.rados.0.smithi063.stdout:1904: done (4 left) 2024-08-17T17:03:00.755 INFO:tasks.rados.rados.0.smithi063.stdout:1909: write oid 40 current snap is 246 2024-08-17T17:03:00.755 INFO:tasks.rados.rados.0.smithi063.stdout:1909: seq_num 725 ranges {585075=409060,1542151=642924} 2024-08-17T17:03:00.760 INFO:tasks.rados.rados.0.smithi063.stdout:1909: writing smithi06387500-40 from 585075 to 994135 tid 1 2024-08-17T17:03:00.767 INFO:tasks.rados.rados.0.smithi063.stdout:1909: writing smithi06387500-40 from 1542151 to 2185075 tid 2 2024-08-17T17:03:00.767 INFO:tasks.rados.rados.0.smithi063.stdout:1910: delete oid 1 current snap is 246 2024-08-17T17:03:00.767 INFO:tasks.rados.rados.0.smithi063.stdout:1905: finishing write tid 2 to smithi06387500-3 2024-08-17T17:03:00.768 INFO:tasks.rados.rados.0.smithi063.stdout:1905: finishing write tid 3 to smithi06387500-3 2024-08-17T17:03:00.768 INFO:tasks.rados.rados.0.smithi063.stdout:1905: finishing write tid 4 to smithi06387500-3 2024-08-17T17:03:00.768 INFO:tasks.rados.rados.0.smithi063.stdout:1905: oid 3 updating version 0 to 562 2024-08-17T17:03:00.768 INFO:tasks.rados.rados.0.smithi063.stdout:1905: oid 3 updating version 562 to 564 2024-08-17T17:03:00.768 INFO:tasks.rados.rados.0.smithi063.stdout:1905: oid 3 version 564 is already newer than 563 2024-08-17T17:03:00.768 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 564 (ObjNum 723 snap 246 seq_num 723) dirty exists 2024-08-17T17:03:00.768 INFO:tasks.rados.rados.0.smithi063.stdout:1905: left oid 3 (ObjNum 723 snap 246 seq_num 723) 2024-08-17T17:03:00.768 INFO:tasks.rados.rados.0.smithi063.stdout:1907: finishing rollback tid 0 to smithi06387500-47 2024-08-17T17:03:00.768 INFO:tasks.rados.rados.0.smithi063.stdout:1907: finishing rollback tid 1 to smithi06387500-47 2024-08-17T17:03:00.768 INFO:tasks.rados.rados.0.smithi063.stdout:1907: finishing rollback tid 2 to smithi06387500-47 2024-08-17T17:03:00.768 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 346 (ObjNum 662 snap 210 seq_num 662) dirty exists 2024-08-17T17:03:00.768 INFO:tasks.rados.rados.0.smithi063.stdout:1908: finishing write tid 1 to smithi06387500-45 2024-08-17T17:03:00.769 INFO:tasks.rados.rados.0.smithi063.stdout:1908: finishing write tid 2 to smithi06387500-45 2024-08-17T17:03:00.769 INFO:tasks.rados.rados.0.smithi063.stdout:1908: finishing write tid 3 to smithi06387500-45 2024-08-17T17:03:00.769 INFO:tasks.rados.rados.0.smithi063.stdout:1908: finishing write tid 4 to smithi06387500-45 2024-08-17T17:03:00.769 INFO:tasks.rados.rados.0.smithi063.stdout:1908: oid 45 updating version 0 to 403 2024-08-17T17:03:00.769 INFO:tasks.rados.rados.0.smithi063.stdout:1908: oid 45 updating version 403 to 405 2024-08-17T17:03:00.769 INFO:tasks.rados.rados.0.smithi063.stdout:1908: oid 45 version 405 is already newer than 404 2024-08-17T17:03:00.769 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 405 (ObjNum 724 snap 246 seq_num 724) dirty exists 2024-08-17T17:03:00.769 INFO:tasks.rados.rados.0.smithi063.stdout:1908: left oid 45 (ObjNum 724 snap 246 seq_num 724) 2024-08-17T17:03:00.769 INFO:tasks.rados.rados.0.smithi063.stdout:1909: finishing write tid 1 to smithi06387500-40 2024-08-17T17:03:00.770 INFO:tasks.rados.rados.0.smithi063.stdout:1905: done (5 left) 2024-08-17T17:03:00.770 INFO:tasks.rados.rados.0.smithi063.stdout:1906: done (4 left) 2024-08-17T17:03:00.770 INFO:tasks.rados.rados.0.smithi063.stdout:1907: done (3 left) 2024-08-17T17:03:00.770 INFO:tasks.rados.rados.0.smithi063.stdout:1908: done (2 left) 2024-08-17T17:03:00.770 INFO:tasks.rados.rados.0.smithi063.stdout:1910: done (1 left) 2024-08-17T17:03:00.770 INFO:tasks.rados.rados.0.smithi063.stdout:1911: rollback oid 6 current snap is 246 2024-08-17T17:03:00.770 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 6 to 230 2024-08-17T17:03:00.771 INFO:tasks.rados.rados.0.smithi063.stdout:1912: snap_remove snap 243 2024-08-17T17:03:00.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:00 smithi162 ceph-mon[104668]: osdmap e552: 8 total, 8 up, 8 in 2024-08-17T17:03:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:00 smithi063 ceph-mon[109843]: osdmap e552: 8 total, 8 up, 8 in 2024-08-17T17:03:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:00 smithi063 ceph-mon[114756]: osdmap e552: 8 total, 8 up, 8 in 2024-08-17T17:03:01.575 INFO:tasks.rados.rados.0.smithi063.stdout:1912: done (2 left) 2024-08-17T17:03:01.575 INFO:tasks.rados.rados.0.smithi063.stdout:1913: delete oid 28 current snap is 246 2024-08-17T17:03:01.575 INFO:tasks.rados.rados.0.smithi063.stdout:1909: finishing write tid 2 to smithi06387500-40 2024-08-17T17:03:01.575 INFO:tasks.rados.rados.0.smithi063.stdout:1909: finishing write tid 3 to smithi06387500-40 2024-08-17T17:03:01.575 INFO:tasks.rados.rados.0.smithi063.stdout:1909: finishing write tid 4 to smithi06387500-40 2024-08-17T17:03:01.575 INFO:tasks.rados.rados.0.smithi063.stdout:1909: oid 40 updating version 0 to 618 2024-08-17T17:03:01.575 INFO:tasks.rados.rados.0.smithi063.stdout:1909: oid 40 updating version 618 to 619 2024-08-17T17:03:01.575 INFO:tasks.rados.rados.0.smithi063.stdout:1909: oid 40 updating version 619 to 620 2024-08-17T17:03:01.576 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 620 (ObjNum 725 snap 246 seq_num 725) dirty exists 2024-08-17T17:03:01.576 INFO:tasks.rados.rados.0.smithi063.stdout:1909: left oid 40 (ObjNum 725 snap 246 seq_num 725) 2024-08-17T17:03:01.576 INFO:tasks.rados.rados.0.smithi063.stdout:1911: finishing rollback tid 0 to smithi06387500-6 2024-08-17T17:03:01.576 INFO:tasks.rados.rados.0.smithi063.stdout:1911: finishing rollback tid 1 to smithi06387500-6 2024-08-17T17:03:01.576 INFO:tasks.rados.rados.0.smithi063.stdout:1911: finishing rollback tid 2 to smithi06387500-6 2024-08-17T17:03:01.576 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 549 (ObjNum 425 snap 133 seq_num 425) dirty exists 2024-08-17T17:03:01.583 INFO:tasks.rados.rados.0.smithi063.stdout:1909: done (2 left) 2024-08-17T17:03:01.584 INFO:tasks.rados.rados.0.smithi063.stdout:1911: done (1 left) 2024-08-17T17:03:01.584 INFO:tasks.rados.rados.0.smithi063.stdout:1913: done (0 left) 2024-08-17T17:03:01.584 INFO:tasks.rados.rados.0.smithi063.stdout:1914: snap_remove snap 229 2024-08-17T17:03:01.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:01 smithi162 ceph-mon[104668]: pgmap v285: 89 pgs: 89 active+clean; 155 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 669 KiB/s wr, 6 op/s 2024-08-17T17:03:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:01 smithi162 ceph-mon[104668]: osdmap e553: 8 total, 8 up, 8 in 2024-08-17T17:03:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:01 smithi063 ceph-mon[114756]: pgmap v285: 89 pgs: 89 active+clean; 155 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 669 KiB/s wr, 6 op/s 2024-08-17T17:03:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:01 smithi063 ceph-mon[114756]: osdmap e553: 8 total, 8 up, 8 in 2024-08-17T17:03:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:01 smithi063 ceph-mon[109843]: pgmap v285: 89 pgs: 89 active+clean; 155 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 669 KiB/s wr, 6 op/s 2024-08-17T17:03:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:01 smithi063 ceph-mon[109843]: osdmap e553: 8 total, 8 up, 8 in 2024-08-17T17:03:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:02.598 INFO:tasks.rados.rados.0.smithi063.stdout:1914: done (0 left) 2024-08-17T17:03:02.598 INFO:tasks.rados.rados.0.smithi063.stdout:1915: rollback oid 38 current snap is 246 2024-08-17T17:03:02.598 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 38 to 239 2024-08-17T17:03:02.598 INFO:tasks.rados.rados.0.smithi063.stdout:1916: read oid 36 snap -1 2024-08-17T17:03:02.598 INFO:tasks.rados.rados.0.smithi063.stdout:1916: expect (ObjNum 701 snap 229 seq_num 701) 2024-08-17T17:03:02.598 INFO:tasks.rados.rados.0.smithi063.stdout:1917: delete oid 20 current snap is 246 2024-08-17T17:03:02.602 INFO:tasks.rados.rados.0.smithi063.stdout:1917: done (2 left) 2024-08-17T17:03:02.602 INFO:tasks.rados.rados.0.smithi063.stdout:1918: read oid 28 snap -1 2024-08-17T17:03:02.602 INFO:tasks.rados.rados.0.smithi063.stdout:1918: expect deleted 2024-08-17T17:03:02.602 INFO:tasks.rados.rados.0.smithi063.stdout:1919: snap_remove snap 230 2024-08-17T17:03:02.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:02 smithi162 ceph-mon[104668]: osdmap e554: 8 total, 8 up, 8 in 2024-08-17T17:03:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:02 smithi063 ceph-mon[114756]: osdmap e554: 8 total, 8 up, 8 in 2024-08-17T17:03:03.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:02 smithi063 ceph-mon[109843]: osdmap e554: 8 total, 8 up, 8 in 2024-08-17T17:03:03.586 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:03:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:03:03] "GET /metrics HTTP/1.1" 200 36215 "" "Prometheus/2.51.0" 2024-08-17T17:03:03.587 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:03 smithi063 ceph-mon[109843]: pgmap v288: 89 pgs: 89 active+clean; 167 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-08-17T17:03:03.587 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:03 smithi063 ceph-mon[109843]: osdmap e555: 8 total, 8 up, 8 in 2024-08-17T17:03:03.596 INFO:tasks.rados.rados.0.smithi063.stdout:1919: done (3 left) 2024-08-17T17:03:03.596 INFO:tasks.rados.rados.0.smithi063.stdout:1920: delete oid 17 current snap is 246 2024-08-17T17:03:03.596 INFO:tasks.rados.rados.0.smithi063.stdout:1915: finishing rollback tid 0 to smithi06387500-38 2024-08-17T17:03:03.596 INFO:tasks.rados.rados.0.smithi063.stdout:1915: finishing rollback tid 1 to smithi06387500-38 2024-08-17T17:03:03.596 INFO:tasks.rados.rados.0.smithi063.stdout:1915: finishing rollback tid 2 to smithi06387500-38 2024-08-17T17:03:03.597 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 284 (ObjNum 699 snap 228 seq_num 699) dirty exists 2024-08-17T17:03:03.597 INFO:tasks.rados.rados.0.smithi063.stdout:1916: expect (ObjNum 701 snap 229 seq_num 701) 2024-08-17T17:03:03.724 INFO:tasks.rados.rados.0.smithi063.stdout:1915: done (3 left) 2024-08-17T17:03:03.724 INFO:tasks.rados.rados.0.smithi063.stdout:1916: done (2 left) 2024-08-17T17:03:03.724 INFO:tasks.rados.rados.0.smithi063.stdout:1918: done (1 left) 2024-08-17T17:03:03.725 INFO:tasks.rados.rados.0.smithi063.stdout:1920: done (0 left) 2024-08-17T17:03:03.725 INFO:tasks.rados.rados.0.smithi063.stdout:1921: rollback oid 47 current snap is 246 2024-08-17T17:03:03.725 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 47 to 233 2024-08-17T17:03:03.725 INFO:tasks.rados.rados.0.smithi063.stdout:1922: write oid 48 current snap is 246 2024-08-17T17:03:03.725 INFO:tasks.rados.rados.0.smithi063.stdout:1922: seq_num 730 ranges {669110=501658,1834561=434549} 2024-08-17T17:03:03.730 INFO:tasks.rados.rados.0.smithi063.stdout:1922: writing smithi06387500-48 from 669110 to 1170768 tid 1 2024-08-17T17:03:03.737 INFO:tasks.rados.rados.0.smithi063.stdout:1922: writing smithi06387500-48 from 1834561 to 2269110 tid 2 2024-08-17T17:03:03.737 INFO:tasks.rados.rados.0.smithi063.stdout:1923: delete oid 35 current snap is 246 2024-08-17T17:03:03.737 INFO:tasks.rados.rados.0.smithi063.stdout:1921: finishing rollback tid 0 to smithi06387500-47 2024-08-17T17:03:03.737 INFO:tasks.rados.rados.0.smithi063.stdout:1921: finishing rollback tid 1 to smithi06387500-47 2024-08-17T17:03:03.737 INFO:tasks.rados.rados.0.smithi063.stdout:1921: finishing rollback tid 2 to smithi06387500-47 2024-08-17T17:03:03.737 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 355 (ObjNum 662 snap 210 seq_num 662) dirty exists 2024-08-17T17:03:03.737 INFO:tasks.rados.rados.0.smithi063.stdout:1922: finishing write tid 1 to smithi06387500-48 2024-08-17T17:03:03.740 INFO:tasks.rados.rados.0.smithi063.stdout:1922: finishing write tid 2 to smithi06387500-48 2024-08-17T17:03:03.741 INFO:tasks.rados.rados.0.smithi063.stdout:1922: finishing write tid 3 to smithi06387500-48 2024-08-17T17:03:03.741 INFO:tasks.rados.rados.0.smithi063.stdout:1922: finishing write tid 4 to smithi06387500-48 2024-08-17T17:03:03.742 INFO:tasks.rados.rados.0.smithi063.stdout:1922: oid 48 updating version 0 to 1850 2024-08-17T17:03:03.742 INFO:tasks.rados.rados.0.smithi063.stdout:1922: oid 48 updating version 1850 to 1851 2024-08-17T17:03:03.742 INFO:tasks.rados.rados.0.smithi063.stdout:1922: oid 48 version 1851 is already newer than 1849 2024-08-17T17:03:03.742 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 1851 (ObjNum 730 snap 246 seq_num 730) dirty exists 2024-08-17T17:03:03.742 INFO:tasks.rados.rados.0.smithi063.stdout:1922: left oid 48 (ObjNum 730 snap 246 seq_num 730) 2024-08-17T17:03:03.742 INFO:tasks.rados.rados.0.smithi063.stdout:1921: done (2 left) 2024-08-17T17:03:03.742 INFO:tasks.rados.rados.0.smithi063.stdout:1922: done (1 left) 2024-08-17T17:03:03.742 INFO:tasks.rados.rados.0.smithi063.stdout:1923: done (0 left) 2024-08-17T17:03:03.742 INFO:tasks.rados.rados.0.smithi063.stdout:1924: delete oid 18 current snap is 246 2024-08-17T17:03:03.743 INFO:tasks.rados.rados.0.smithi063.stdout:1924: done (0 left) 2024-08-17T17:03:03.743 INFO:tasks.rados.rados.0.smithi063.stdout:1925: snap_create 2024-08-17T17:03:03.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:03 smithi162 ceph-mon[104668]: pgmap v288: 89 pgs: 89 active+clean; 167 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-08-17T17:03:03.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:03 smithi162 ceph-mon[104668]: osdmap e555: 8 total, 8 up, 8 in 2024-08-17T17:03:04.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:03 smithi063 ceph-mon[114756]: pgmap v288: 89 pgs: 89 active+clean; 167 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-08-17T17:03:04.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:03 smithi063 ceph-mon[114756]: osdmap e555: 8 total, 8 up, 8 in 2024-08-17T17:03:04.605 INFO:tasks.rados.rados.0.smithi063.stdout:1925: done (0 left) 2024-08-17T17:03:04.605 INFO:tasks.rados.rados.0.smithi063.stdout:1926: read oid 34 snap -1 2024-08-17T17:03:04.606 INFO:tasks.rados.rados.0.smithi063.stdout:1926: expect (ObjNum 697 snap 226 seq_num 697) 2024-08-17T17:03:04.606 INFO:tasks.rados.rados.0.smithi063.stdout:1927: read oid 33 snap -1 2024-08-17T17:03:04.606 INFO:tasks.rados.rados.0.smithi063.stdout:1927: expect (ObjNum 636 snap 203 seq_num 636) 2024-08-17T17:03:04.606 INFO:tasks.rados.rados.0.smithi063.stdout:1928: rollback oid 12 current snap is 247 2024-08-17T17:03:04.606 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 12 to 221 2024-08-17T17:03:04.606 INFO:tasks.rados.rados.0.smithi063.stdout:1929: write oid 26 current snap is 247 2024-08-17T17:03:04.606 INFO:tasks.rados.rados.0.smithi063.stdout:1929: seq_num 733 ranges {719531=727405,1893654=425877} 2024-08-17T17:03:04.614 INFO:tasks.rados.rados.0.smithi063.stdout:1929: writing smithi06387500-26 from 719531 to 1446936 tid 1 2024-08-17T17:03:04.619 INFO:tasks.rados.rados.0.smithi063.stdout:1929: writing smithi06387500-26 from 1893654 to 2319531 tid 2 2024-08-17T17:03:04.619 INFO:tasks.rados.rados.0.smithi063.stdout:1930: read oid 48 snap -1 2024-08-17T17:03:04.619 INFO:tasks.rados.rados.0.smithi063.stdout:1930: expect (ObjNum 730 snap 246 seq_num 730) 2024-08-17T17:03:04.619 INFO:tasks.rados.rados.0.smithi063.stdout:1931: read oid 9 snap 238 2024-08-17T17:03:04.619 INFO:tasks.rados.rados.0.smithi063.stdout:1931: expect (ObjNum 702 snap 229 seq_num 702) 2024-08-17T17:03:04.619 INFO:tasks.rados.rados.0.smithi063.stdout:1932: read oid 20 snap 237 2024-08-17T17:03:04.619 INFO:tasks.rados.rados.0.smithi063.stdout:1932: expect deleted 2024-08-17T17:03:04.620 INFO:tasks.rados.rados.0.smithi063.stdout:1933: snap_remove snap 227 2024-08-17T17:03:05.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:04 smithi063 ceph-mon[109843]: osdmap e556: 8 total, 8 up, 8 in 2024-08-17T17:03:05.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:04 smithi063 ceph-mon[114756]: osdmap e556: 8 total, 8 up, 8 in 2024-08-17T17:03:05.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:04 smithi162 ceph-mon[104668]: osdmap e556: 8 total, 8 up, 8 in 2024-08-17T17:03:05.612 INFO:tasks.rados.rados.0.smithi063.stdout:1933: done (7 left) 2024-08-17T17:03:05.612 INFO:tasks.rados.rados.0.smithi063.stdout:1934: delete oid 44 current snap is 247 2024-08-17T17:03:05.612 INFO:tasks.rados.rados.0.smithi063.stdout:1927: expect (ObjNum 636 snap 203 seq_num 636) 2024-08-17T17:03:05.771 INFO:tasks.rados.rados.0.smithi063.stdout:1926: expect (ObjNum 697 snap 226 seq_num 697) 2024-08-17T17:03:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:05 smithi063 ceph-mon[114756]: pgmap v291: 89 pgs: 89 active+clean; 167 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-08-17T17:03:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:05 smithi063 ceph-mon[114756]: osdmap e557: 8 total, 8 up, 8 in 2024-08-17T17:03:06.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:05 smithi063 ceph-mon[109843]: pgmap v291: 89 pgs: 89 active+clean; 167 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-08-17T17:03:06.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:05 smithi063 ceph-mon[109843]: osdmap e557: 8 total, 8 up, 8 in 2024-08-17T17:03:06.061 INFO:tasks.rados.rados.0.smithi063.stdout:1928: finishing rollback tid 0 to smithi06387500-12 2024-08-17T17:03:06.061 INFO:tasks.rados.rados.0.smithi063.stdout:1928: finishing rollback tid 1 to smithi06387500-12 2024-08-17T17:03:06.061 INFO:tasks.rados.rados.0.smithi063.stdout:1928: finishing rollback tid 2 to smithi06387500-12 2024-08-17T17:03:06.061 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 432 (ObjNum 612 snap 193 seq_num 612) dirty exists 2024-08-17T17:03:06.061 INFO:tasks.rados.rados.0.smithi063.stdout:1929: finishing write tid 1 to smithi06387500-26 2024-08-17T17:03:06.061 INFO:tasks.rados.rados.0.smithi063.stdout:1926: done (7 left) 2024-08-17T17:03:06.061 INFO:tasks.rados.rados.0.smithi063.stdout:1927: done (6 left) 2024-08-17T17:03:06.062 INFO:tasks.rados.rados.0.smithi063.stdout:1928: done (5 left) 2024-08-17T17:03:06.062 INFO:tasks.rados.rados.0.smithi063.stdout:1934: done (4 left) 2024-08-17T17:03:06.062 INFO:tasks.rados.rados.0.smithi063.stdout:1935: write oid 43 current snap is 247 2024-08-17T17:03:06.062 INFO:tasks.rados.rados.0.smithi063.stdout:1930: expect (ObjNum 730 snap 246 seq_num 730) 2024-08-17T17:03:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:05 smithi162 ceph-mon[104668]: pgmap v291: 89 pgs: 89 active+clean; 167 MiB data, 667 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-08-17T17:03:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:05 smithi162 ceph-mon[104668]: osdmap e557: 8 total, 8 up, 8 in 2024-08-17T17:03:06.163 INFO:tasks.rados.rados.0.smithi063.stdout:1929: finishing write tid 2 to smithi06387500-26 2024-08-17T17:03:06.163 INFO:tasks.rados.rados.0.smithi063.stdout:1929: finishing write tid 3 to smithi06387500-26 2024-08-17T17:03:06.163 INFO:tasks.rados.rados.0.smithi063.stdout:1935: seq_num 735 ranges {753145=477903,2010618=342527} 2024-08-17T17:03:06.170 INFO:tasks.rados.rados.0.smithi063.stdout:1935: writing smithi06387500-43 from 753145 to 1231048 tid 1 2024-08-17T17:03:06.177 INFO:tasks.rados.rados.0.smithi063.stdout:1935: writing smithi06387500-43 from 2010618 to 2353145 tid 2 2024-08-17T17:03:06.177 INFO:tasks.rados.rados.0.smithi063.stdout:1930: done (4 left) 2024-08-17T17:03:06.177 INFO:tasks.rados.rados.0.smithi063.stdout:1932: done (3 left) 2024-08-17T17:03:06.177 INFO:tasks.rados.rados.0.smithi063.stdout:1936: read oid 28 snap -1 2024-08-17T17:03:06.177 INFO:tasks.rados.rados.0.smithi063.stdout:1936: expect deleted 2024-08-17T17:03:06.177 INFO:tasks.rados.rados.0.smithi063.stdout:1937: snap_create 2024-08-17T17:03:06.177 INFO:tasks.rados.rados.0.smithi063.stdout:1929: finishing write tid 4 to smithi06387500-26 2024-08-17T17:03:06.177 INFO:tasks.rados.rados.0.smithi063.stdout:1929: oid 26 updating version 0 to 568 2024-08-17T17:03:06.177 INFO:tasks.rados.rados.0.smithi063.stdout:1929: oid 26 version 568 is already newer than 567 2024-08-17T17:03:06.177 INFO:tasks.rados.rados.0.smithi063.stdout:1929: oid 26 updating version 568 to 569 2024-08-17T17:03:06.177 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 569 (ObjNum 733 snap 247 seq_num 733) dirty exists 2024-08-17T17:03:06.178 INFO:tasks.rados.rados.0.smithi063.stdout:1929: left oid 26 (ObjNum 733 snap 247 seq_num 733) 2024-08-17T17:03:06.178 INFO:tasks.rados.rados.0.smithi063.stdout:1931: expect (ObjNum 702 snap 229 seq_num 702) 2024-08-17T17:03:06.301 INFO:tasks.rados.rados.0.smithi063.stdout:1935: finishing write tid 1 to smithi06387500-43 2024-08-17T17:03:06.301 INFO:tasks.rados.rados.0.smithi063.stdout:1935: finishing write tid 2 to smithi06387500-43 2024-08-17T17:03:06.302 INFO:tasks.rados.rados.0.smithi063.stdout:1935: finishing write tid 3 to smithi06387500-43 2024-08-17T17:03:06.302 INFO:tasks.rados.rados.0.smithi063.stdout:1935: finishing write tid 4 to smithi06387500-43 2024-08-17T17:03:06.302 INFO:tasks.rados.rados.0.smithi063.stdout:1935: oid 43 updating version 0 to 834 2024-08-17T17:03:06.302 INFO:tasks.rados.rados.0.smithi063.stdout:1935: oid 43 version 834 is already newer than 832 2024-08-17T17:03:06.302 INFO:tasks.rados.rados.0.smithi063.stdout:1935: oid 43 version 834 is already newer than 833 2024-08-17T17:03:06.302 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 834 (ObjNum 735 snap 247 seq_num 735) dirty exists 2024-08-17T17:03:06.302 INFO:tasks.rados.rados.0.smithi063.stdout:1935: left oid 43 (ObjNum 735 snap 247 seq_num 735) 2024-08-17T17:03:06.623 INFO:tasks.rados.rados.0.smithi063.stdout:1929: done (4 left) 2024-08-17T17:03:06.624 INFO:tasks.rados.rados.0.smithi063.stdout:1931: done (3 left) 2024-08-17T17:03:06.624 INFO:tasks.rados.rados.0.smithi063.stdout:1935: done (2 left) 2024-08-17T17:03:06.624 INFO:tasks.rados.rados.0.smithi063.stdout:1936: done (1 left) 2024-08-17T17:03:06.624 INFO:tasks.rados.rados.0.smithi063.stdout:1937: done (0 left) 2024-08-17T17:03:06.624 INFO:tasks.rados.rados.0.smithi063.stdout:1938: read oid 49 snap -1 2024-08-17T17:03:06.624 INFO:tasks.rados.rados.0.smithi063.stdout:1938: expect (ObjNum 655 snap 209 seq_num 655) 2024-08-17T17:03:06.624 INFO:tasks.rados.rados.0.smithi063.stdout:1939: snap_create 2024-08-17T17:03:06.637 INFO:tasks.rados.rados.0.smithi063.stdout:1938: expect (ObjNum 655 snap 209 seq_num 655) 2024-08-17T17:03:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:06 smithi063 ceph-mon[114756]: osdmap e558: 8 total, 8 up, 8 in 2024-08-17T17:03:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:06 smithi063 ceph-mon[109843]: osdmap e558: 8 total, 8 up, 8 in 2024-08-17T17:03:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:06 smithi162 ceph-mon[104668]: osdmap e558: 8 total, 8 up, 8 in 2024-08-17T17:03:07.550 INFO:tasks.rados.rados.0.smithi063.stdout:1938: done (1 left) 2024-08-17T17:03:07.550 INFO:tasks.rados.rados.0.smithi063.stdout:1939: done (0 left) 2024-08-17T17:03:07.551 INFO:tasks.rados.rados.0.smithi063.stdout:1940: rollback oid 34 current snap is 249 2024-08-17T17:03:07.551 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 34 to 221 2024-08-17T17:03:07.551 INFO:tasks.rados.rados.0.smithi063.stdout:1941: snap_create 2024-08-17T17:03:07.557 INFO:tasks.rados.rados.0.smithi063.stdout:1940: finishing rollback tid 0 to smithi06387500-34 2024-08-17T17:03:07.557 INFO:tasks.rados.rados.0.smithi063.stdout:1940: finishing rollback tid 1 to smithi06387500-34 2024-08-17T17:03:07.558 INFO:tasks.rados.rados.0.smithi063.stdout:1940: finishing rollback tid 2 to smithi06387500-34 2024-08-17T17:03:07.558 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 1285 (ObjNum 675 snap 213 seq_num 675) dirty exists 2024-08-17T17:03:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:07 smithi063 ceph-mon[114756]: pgmap v294: 89 pgs: 89 active+clean; 167 MiB data, 667 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:07 smithi063 ceph-mon[114756]: osdmap e559: 8 total, 8 up, 8 in 2024-08-17T17:03:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:07 smithi063 ceph-mon[114756]: osdmap e560: 8 total, 8 up, 8 in 2024-08-17T17:03:08.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:07 smithi063 ceph-mon[109843]: pgmap v294: 89 pgs: 89 active+clean; 167 MiB data, 667 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:08.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:07 smithi063 ceph-mon[109843]: osdmap e559: 8 total, 8 up, 8 in 2024-08-17T17:03:08.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:07 smithi063 ceph-mon[109843]: osdmap e560: 8 total, 8 up, 8 in 2024-08-17T17:03:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:07 smithi162 ceph-mon[104668]: pgmap v294: 89 pgs: 89 active+clean; 167 MiB data, 667 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:07 smithi162 ceph-mon[104668]: osdmap e559: 8 total, 8 up, 8 in 2024-08-17T17:03:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:07 smithi162 ceph-mon[104668]: osdmap e560: 8 total, 8 up, 8 in 2024-08-17T17:03:08.553 INFO:tasks.rados.rados.0.smithi063.stdout:1940: done (1 left) 2024-08-17T17:03:08.554 INFO:tasks.rados.rados.0.smithi063.stdout:1941: done (0 left) 2024-08-17T17:03:08.554 INFO:tasks.rados.rados.0.smithi063.stdout:1942: read oid 37 snap -1 2024-08-17T17:03:08.554 INFO:tasks.rados.rados.0.smithi063.stdout:1942: expect (ObjNum 696 snap 222 seq_num 696) 2024-08-17T17:03:08.554 INFO:tasks.rados.rados.0.smithi063.stdout:1943: write oid 18 current snap is 250 2024-08-17T17:03:08.554 INFO:tasks.rados.rados.0.smithi063.stdout:1943: seq_num 736 ranges {769952=553152,2069100=300852} 2024-08-17T17:03:08.560 INFO:tasks.rados.rados.0.smithi063.stdout:1943: writing smithi06387500-18 from 769952 to 1323104 tid 1 2024-08-17T17:03:08.567 INFO:tasks.rados.rados.0.smithi063.stdout:1943: writing smithi06387500-18 from 2069100 to 2369952 tid 2 2024-08-17T17:03:08.567 INFO:tasks.rados.rados.0.smithi063.stdout:1944: delete oid 29 current snap is 250 2024-08-17T17:03:08.567 INFO:tasks.rados.rados.0.smithi063.stdout:1943: finishing write tid 1 to smithi06387500-18 2024-08-17T17:03:08.568 INFO:tasks.rados.rados.0.smithi063.stdout:1942: expect (ObjNum 696 snap 222 seq_num 696) 2024-08-17T17:03:08.792 INFO:tasks.rados.rados.0.smithi063.stdout:1943: finishing write tid 2 to smithi06387500-18 2024-08-17T17:03:08.792 INFO:tasks.rados.rados.0.smithi063.stdout:1943: finishing write tid 3 to smithi06387500-18 2024-08-17T17:03:08.793 INFO:tasks.rados.rados.0.smithi063.stdout:1942: done (2 left) 2024-08-17T17:03:08.793 INFO:tasks.rados.rados.0.smithi063.stdout:1944: done (1 left) 2024-08-17T17:03:08.793 INFO:tasks.rados.rados.0.smithi063.stdout:1945: read oid 45 snap -1 2024-08-17T17:03:08.793 INFO:tasks.rados.rados.0.smithi063.stdout:1945: expect (ObjNum 724 snap 246 seq_num 724) 2024-08-17T17:03:08.793 INFO:tasks.rados.rados.0.smithi063.stdout:1943: finishing write tid 4 to smithi06387500-18 2024-08-17T17:03:08.793 INFO:tasks.rados.rados.0.smithi063.stdout:1943: oid 18 updating version 0 to 624 2024-08-17T17:03:08.793 INFO:tasks.rados.rados.0.smithi063.stdout:1943: oid 18 updating version 624 to 626 2024-08-17T17:03:08.793 INFO:tasks.rados.rados.0.smithi063.stdout:1943: oid 18 version 626 is already newer than 625 2024-08-17T17:03:08.793 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 626 (ObjNum 736 snap 250 seq_num 736) dirty exists 2024-08-17T17:03:08.793 INFO:tasks.rados.rados.0.smithi063.stdout:1943: left oid 18 (ObjNum 736 snap 250 seq_num 736) 2024-08-17T17:03:08.793 INFO:tasks.rados.rados.0.smithi063.stdout:1946: read oid 25 snap 238 2024-08-17T17:03:08.794 INFO:tasks.rados.rados.0.smithi063.stdout:1946: expect (ObjNum 673 snap 213 seq_num 673) 2024-08-17T17:03:08.794 INFO:tasks.rados.rados.0.smithi063.stdout:1943: done (2 left) 2024-08-17T17:03:08.794 INFO:tasks.rados.rados.0.smithi063.stdout:1947: read oid 21 snap -1 2024-08-17T17:03:08.794 INFO:tasks.rados.rados.0.smithi063.stdout:1947: expect (ObjNum 709 snap 234 seq_num 709) 2024-08-17T17:03:08.794 INFO:tasks.rados.rados.0.smithi063.stdout:1948: snap_create 2024-08-17T17:03:08.795 INFO:tasks.rados.rados.0.smithi063.stdout:1947: expect (ObjNum 709 snap 234 seq_num 709) 2024-08-17T17:03:08.926 INFO:tasks.rados.rados.0.smithi063.stdout:1945: expect (ObjNum 724 snap 246 seq_num 724) 2024-08-17T17:03:08.999 INFO:tasks.rados.rados.0.smithi063.stdout:1946: expect (ObjNum 673 snap 213 seq_num 673) 2024-08-17T17:03:09.627 INFO:tasks.rados.rados.0.smithi063.stdout:1945: done (3 left) 2024-08-17T17:03:09.627 INFO:tasks.rados.rados.0.smithi063.stdout:1946: done (2 left) 2024-08-17T17:03:09.627 INFO:tasks.rados.rados.0.smithi063.stdout:1947: done (1 left) 2024-08-17T17:03:09.627 INFO:tasks.rados.rados.0.smithi063.stdout:1948: done (0 left) 2024-08-17T17:03:09.627 INFO:tasks.rados.rados.0.smithi063.stdout:1949: snap_remove snap 231 2024-08-17T17:03:09.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:09 smithi162 ceph-mon[104668]: pgmap v297: 89 pgs: 89 active+clean; 167 MiB data, 652 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 711 KiB/s wr, 17 op/s 2024-08-17T17:03:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:09 smithi162 ceph-mon[104668]: osdmap e561: 8 total, 8 up, 8 in 2024-08-17T17:03:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:09 smithi063 ceph-mon[114756]: pgmap v297: 89 pgs: 89 active+clean; 167 MiB data, 652 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 711 KiB/s wr, 17 op/s 2024-08-17T17:03:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:09 smithi063 ceph-mon[114756]: osdmap e561: 8 total, 8 up, 8 in 2024-08-17T17:03:10.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:09 smithi063 ceph-mon[109843]: pgmap v297: 89 pgs: 89 active+clean; 167 MiB data, 652 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 711 KiB/s wr, 17 op/s 2024-08-17T17:03:10.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:09 smithi063 ceph-mon[109843]: osdmap e561: 8 total, 8 up, 8 in 2024-08-17T17:03:10.639 INFO:tasks.rados.rados.0.smithi063.stdout:1949: done (0 left) 2024-08-17T17:03:10.640 INFO:tasks.rados.rados.0.smithi063.stdout:1950: read oid 43 snap 246 2024-08-17T17:03:10.640 INFO:tasks.rados.rados.0.smithi063.stdout:1950: expect (ObjNum 483 snap 153 seq_num 483) 2024-08-17T17:03:10.640 INFO:tasks.rados.rados.0.smithi063.stdout:1951: delete oid 11 current snap is 251 2024-08-17T17:03:10.643 INFO:tasks.rados.rados.0.smithi063.stdout:1951: done (1 left) 2024-08-17T17:03:10.643 INFO:tasks.rados.rados.0.smithi063.stdout:1952: read oid 21 snap 236 2024-08-17T17:03:10.643 INFO:tasks.rados.rados.0.smithi063.stdout:1952: expect (ObjNum 709 snap 234 seq_num 709) 2024-08-17T17:03:10.643 INFO:tasks.rados.rados.0.smithi063.stdout:1953: snap_remove snap 248 2024-08-17T17:03:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:10 smithi063 ceph-mon[114756]: osdmap e562: 8 total, 8 up, 8 in 2024-08-17T17:03:11.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:10 smithi063 ceph-mon[109843]: osdmap e562: 8 total, 8 up, 8 in 2024-08-17T17:03:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:10 smithi162 ceph-mon[104668]: osdmap e562: 8 total, 8 up, 8 in 2024-08-17T17:03:11.642 INFO:tasks.rados.rados.0.smithi063.stdout:1953: done (2 left) 2024-08-17T17:03:11.643 INFO:tasks.rados.rados.0.smithi063.stdout:1954: snap_remove snap 239 2024-08-17T17:03:11.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:11 smithi063 ceph-mon[114756]: pgmap v300: 89 pgs: 89 active+clean; 167 MiB data, 652 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 711 KiB/s wr, 17 op/s 2024-08-17T17:03:11.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:11 smithi063 ceph-mon[114756]: osdmap e563: 8 total, 8 up, 8 in 2024-08-17T17:03:11.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:11 smithi063 ceph-mon[109843]: pgmap v300: 89 pgs: 89 active+clean; 167 MiB data, 652 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 711 KiB/s wr, 17 op/s 2024-08-17T17:03:11.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:11 smithi063 ceph-mon[109843]: osdmap e563: 8 total, 8 up, 8 in 2024-08-17T17:03:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:11 smithi162 ceph-mon[104668]: pgmap v300: 89 pgs: 89 active+clean; 167 MiB data, 652 MiB used, 715 GiB / 715 GiB avail; 16 MiB/s rd, 711 KiB/s wr, 17 op/s 2024-08-17T17:03:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:11 smithi162 ceph-mon[104668]: osdmap e563: 8 total, 8 up, 8 in 2024-08-17T17:03:12.650 INFO:tasks.rados.rados.0.smithi063.stdout:1954: done (2 left) 2024-08-17T17:03:12.650 INFO:tasks.rados.rados.0.smithi063.stdout:1955: write oid 24 current snap is 251 2024-08-17T17:03:12.650 INFO:tasks.rados.rados.0.smithi063.stdout:1955: seq_num 739 ranges {420373=495252,1444546=690576} 2024-08-17T17:03:12.655 INFO:tasks.rados.rados.0.smithi063.stdout:1955: writing smithi06387500-24 from 420373 to 915625 tid 1 2024-08-17T17:03:12.663 INFO:tasks.rados.rados.0.smithi063.stdout:1955: writing smithi06387500-24 from 1444546 to 2135122 tid 2 2024-08-17T17:03:12.663 INFO:tasks.rados.rados.0.smithi063.stdout:1956: write oid 29 current snap is 251 2024-08-17T17:03:12.663 INFO:tasks.rados.rados.0.smithi063.stdout:1956: seq_num 740 ranges {437180=570501,1503028=550587} 2024-08-17T17:03:12.669 INFO:tasks.rados.rados.0.smithi063.stdout:1956: writing smithi06387500-29 from 437180 to 1007681 tid 1 2024-08-17T17:03:12.675 INFO:tasks.rados.rados.0.smithi063.stdout:1956: writing smithi06387500-29 from 1503028 to 2053615 tid 2 2024-08-17T17:03:12.675 INFO:tasks.rados.rados.0.smithi063.stdout:1957: snap_remove snap 232 2024-08-17T17:03:13.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:12 smithi063 ceph-mon[114756]: osdmap e564: 8 total, 8 up, 8 in 2024-08-17T17:03:13.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:12 smithi063 ceph-mon[114756]: pgmap v303: 89 pgs: 89 active+clean; 171 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 209 KiB/s wr, 15 op/s 2024-08-17T17:03:13.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:12 smithi063 ceph-mon[109843]: osdmap e564: 8 total, 8 up, 8 in 2024-08-17T17:03:13.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:12 smithi063 ceph-mon[109843]: pgmap v303: 89 pgs: 89 active+clean; 171 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 209 KiB/s wr, 15 op/s 2024-08-17T17:03:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:12 smithi162 ceph-mon[104668]: osdmap e564: 8 total, 8 up, 8 in 2024-08-17T17:03:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:12 smithi162 ceph-mon[104668]: pgmap v303: 89 pgs: 89 active+clean; 171 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 209 KiB/s wr, 15 op/s 2024-08-17T17:03:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:03:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:03:13] "GET /metrics HTTP/1.1" 200 36215 "" "Prometheus/2.51.0" 2024-08-17T17:03:13.652 INFO:tasks.rados.rados.0.smithi063.stdout:1957: done (4 left) 2024-08-17T17:03:13.652 INFO:tasks.rados.rados.0.smithi063.stdout:1958: read oid 40 snap 220 2024-08-17T17:03:13.652 INFO:tasks.rados.rados.0.smithi063.stdout:1958: expect deleted 2024-08-17T17:03:13.652 INFO:tasks.rados.rados.0.smithi063.stdout:1959: write oid 2 current snap is 251 2024-08-17T17:03:13.652 INFO:tasks.rados.rados.0.smithi063.stdout:1959: seq_num 741 ranges {453987=645750,1845157=608830} 2024-08-17T17:03:13.659 INFO:tasks.rados.rados.0.smithi063.stdout:1959: writing smithi06387500-2 from 453987 to 1099737 tid 1 2024-08-17T17:03:13.668 INFO:tasks.rados.rados.0.smithi063.stdout:1959: writing smithi06387500-2 from 1845157 to 2453987 tid 2 2024-08-17T17:03:13.668 INFO:tasks.rados.rados.0.smithi063.stdout:1960: write oid 23 current snap is 251 2024-08-17T17:03:13.668 INFO:tasks.rados.rados.0.smithi063.stdout:1952: expect (ObjNum 709 snap 234 seq_num 709) 2024-08-17T17:03:13.799 INFO:tasks.rados.rados.0.smithi063.stdout:1950: expect (ObjNum 483 snap 153 seq_num 483) 2024-08-17T17:03:13.871 INFO:tasks.rados.rados.0.smithi063.stdout:1955: finishing write tid 1 to smithi06387500-24 2024-08-17T17:03:13.871 INFO:tasks.rados.rados.0.smithi063.stdout:1960: seq_num 742 ranges {470794=720999,1903639=554256} 2024-08-17T17:03:13.877 INFO:tasks.rados.rados.0.smithi063.stdout:1960: writing smithi06387500-23 from 470794 to 1191793 tid 1 2024-08-17T17:03:13.885 INFO:tasks.rados.rados.0.smithi063.stdout:1960: writing smithi06387500-23 from 1903639 to 2457895 tid 2 2024-08-17T17:03:13.885 INFO:tasks.rados.rados.0.smithi063.stdout:1950: done (6 left) 2024-08-17T17:03:13.885 INFO:tasks.rados.rados.0.smithi063.stdout:1952: done (5 left) 2024-08-17T17:03:13.885 INFO:tasks.rados.rados.0.smithi063.stdout:1961: snap_create 2024-08-17T17:03:13.885 INFO:tasks.rados.rados.0.smithi063.stdout:1955: finishing write tid 2 to smithi06387500-24 2024-08-17T17:03:13.885 INFO:tasks.rados.rados.0.smithi063.stdout:1955: finishing write tid 3 to smithi06387500-24 2024-08-17T17:03:13.885 INFO:tasks.rados.rados.0.smithi063.stdout:1955: finishing write tid 4 to smithi06387500-24 2024-08-17T17:03:13.885 INFO:tasks.rados.rados.0.smithi063.stdout:1955: oid 24 updating version 0 to 721 2024-08-17T17:03:13.885 INFO:tasks.rados.rados.0.smithi063.stdout:1955: oid 24 version 721 is already newer than 719 2024-08-17T17:03:13.886 INFO:tasks.rados.rados.0.smithi063.stdout:1955: oid 24 version 721 is already newer than 720 2024-08-17T17:03:13.886 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 721 (ObjNum 739 snap 251 seq_num 739) dirty exists 2024-08-17T17:03:13.886 INFO:tasks.rados.rados.0.smithi063.stdout:1955: left oid 24 (ObjNum 739 snap 251 seq_num 739) 2024-08-17T17:03:13.886 INFO:tasks.rados.rados.0.smithi063.stdout:1956: finishing write tid 1 to smithi06387500-29 2024-08-17T17:03:13.886 INFO:tasks.rados.rados.0.smithi063.stdout:1956: finishing write tid 2 to smithi06387500-29 2024-08-17T17:03:13.886 INFO:tasks.rados.rados.0.smithi063.stdout:1956: finishing write tid 3 to smithi06387500-29 2024-08-17T17:03:13.886 INFO:tasks.rados.rados.0.smithi063.stdout:1956: finishing write tid 4 to smithi06387500-29 2024-08-17T17:03:13.886 INFO:tasks.rados.rados.0.smithi063.stdout:1956: oid 29 updating version 0 to 449 2024-08-17T17:03:13.886 INFO:tasks.rados.rados.0.smithi063.stdout:1956: oid 29 version 449 is already newer than 447 2024-08-17T17:03:13.886 INFO:tasks.rados.rados.0.smithi063.stdout:1956: oid 29 version 449 is already newer than 448 2024-08-17T17:03:13.887 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 449 (ObjNum 740 snap 251 seq_num 740) dirty exists 2024-08-17T17:03:13.887 INFO:tasks.rados.rados.0.smithi063.stdout:1956: left oid 29 (ObjNum 740 snap 251 seq_num 740) 2024-08-17T17:03:13.887 INFO:tasks.rados.rados.0.smithi063.stdout:1959: finishing write tid 1 to smithi06387500-2 2024-08-17T17:03:13.887 INFO:tasks.rados.rados.0.smithi063.stdout:1959: finishing write tid 2 to smithi06387500-2 2024-08-17T17:03:13.887 INFO:tasks.rados.rados.0.smithi063.stdout:1959: finishing write tid 3 to smithi06387500-2 2024-08-17T17:03:13.887 INFO:tasks.rados.rados.0.smithi063.stdout:1959: finishing write tid 4 to smithi06387500-2 2024-08-17T17:03:13.887 INFO:tasks.rados.rados.0.smithi063.stdout:1959: oid 2 updating version 0 to 867 2024-08-17T17:03:13.887 INFO:tasks.rados.rados.0.smithi063.stdout:1959: oid 2 updating version 867 to 869 2024-08-17T17:03:13.887 INFO:tasks.rados.rados.0.smithi063.stdout:1959: oid 2 version 869 is already newer than 868 2024-08-17T17:03:13.887 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 869 (ObjNum 741 snap 251 seq_num 741) dirty exists 2024-08-17T17:03:13.887 INFO:tasks.rados.rados.0.smithi063.stdout:1959: left oid 2 (ObjNum 741 snap 251 seq_num 741) 2024-08-17T17:03:13.887 INFO:tasks.rados.rados.0.smithi063.stdout:1960: finishing write tid 1 to smithi06387500-23 2024-08-17T17:03:13.888 INFO:tasks.rados.rados.0.smithi063.stdout:1960: finishing write tid 2 to smithi06387500-23 2024-08-17T17:03:13.888 INFO:tasks.rados.rados.0.smithi063.stdout:1960: finishing write tid 3 to smithi06387500-23 2024-08-17T17:03:13.888 INFO:tasks.rados.rados.0.smithi063.stdout:1960: finishing write tid 4 to smithi06387500-23 2024-08-17T17:03:13.888 INFO:tasks.rados.rados.0.smithi063.stdout:1960: oid 23 updating version 0 to 871 2024-08-17T17:03:13.888 INFO:tasks.rados.rados.0.smithi063.stdout:1960: oid 23 updating version 871 to 873 2024-08-17T17:03:13.888 INFO:tasks.rados.rados.0.smithi063.stdout:1960: oid 23 version 873 is already newer than 872 2024-08-17T17:03:13.888 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 873 (ObjNum 742 snap 251 seq_num 742) dirty exists 2024-08-17T17:03:13.889 INFO:tasks.rados.rados.0.smithi063.stdout:1960: left oid 23 (ObjNum 742 snap 251 seq_num 742) 2024-08-17T17:03:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:13 smithi063 ceph-mon[114756]: osdmap e565: 8 total, 8 up, 8 in 2024-08-17T17:03:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:13 smithi063 ceph-mon[109843]: osdmap e565: 8 total, 8 up, 8 in 2024-08-17T17:03:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:13 smithi162 ceph-mon[104668]: osdmap e565: 8 total, 8 up, 8 in 2024-08-17T17:03:14.665 INFO:tasks.rados.rados.0.smithi063.stdout:1955: done (5 left) 2024-08-17T17:03:14.665 INFO:tasks.rados.rados.0.smithi063.stdout:1956: done (4 left) 2024-08-17T17:03:14.665 INFO:tasks.rados.rados.0.smithi063.stdout:1958: done (3 left) 2024-08-17T17:03:14.665 INFO:tasks.rados.rados.0.smithi063.stdout:1959: done (2 left) 2024-08-17T17:03:14.666 INFO:tasks.rados.rados.0.smithi063.stdout:1960: done (1 left) 2024-08-17T17:03:14.666 INFO:tasks.rados.rados.0.smithi063.stdout:1961: done (0 left) 2024-08-17T17:03:14.666 INFO:tasks.rados.rados.0.smithi063.stdout:1962: snap_remove snap 221 2024-08-17T17:03:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:14 smithi063 ceph-mon[114756]: osdmap e566: 8 total, 8 up, 8 in 2024-08-17T17:03:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:14 smithi063 ceph-mon[114756]: pgmap v306: 89 pgs: 89 active+clean; 171 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 209 KiB/s wr, 15 op/s 2024-08-17T17:03:15.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:14 smithi063 ceph-mon[109843]: osdmap e566: 8 total, 8 up, 8 in 2024-08-17T17:03:15.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:14 smithi063 ceph-mon[109843]: pgmap v306: 89 pgs: 89 active+clean; 171 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 209 KiB/s wr, 15 op/s 2024-08-17T17:03:15.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:14 smithi162 ceph-mon[104668]: osdmap e566: 8 total, 8 up, 8 in 2024-08-17T17:03:15.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:14 smithi162 ceph-mon[104668]: pgmap v306: 89 pgs: 89 active+clean; 171 MiB data, 656 MiB used, 715 GiB / 715 GiB avail; 20 MiB/s rd, 209 KiB/s wr, 15 op/s 2024-08-17T17:03:15.666 INFO:tasks.rados.rados.0.smithi063.stdout:1962: done (0 left) 2024-08-17T17:03:15.666 INFO:tasks.rados.rados.0.smithi063.stdout:1963: snap_create 2024-08-17T17:03:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:15 smithi063 ceph-mon[114756]: osdmap e567: 8 total, 8 up, 8 in 2024-08-17T17:03:16.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:15 smithi063 ceph-mon[109843]: osdmap e567: 8 total, 8 up, 8 in 2024-08-17T17:03:16.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:15 smithi162 ceph-mon[104668]: osdmap e567: 8 total, 8 up, 8 in 2024-08-17T17:03:16.679 INFO:tasks.rados.rados.0.smithi063.stdout:1963: done (0 left) 2024-08-17T17:03:16.679 INFO:tasks.rados.rados.0.smithi063.stdout:1964: read oid 49 snap -1 2024-08-17T17:03:16.679 INFO:tasks.rados.rados.0.smithi063.stdout:1964: expect (ObjNum 655 snap 209 seq_num 655) 2024-08-17T17:03:16.680 INFO:tasks.rados.rados.0.smithi063.stdout:1965: read oid 35 snap 235 2024-08-17T17:03:16.680 INFO:tasks.rados.rados.0.smithi063.stdout:1965: expect (ObjNum 683 snap 218 seq_num 683) 2024-08-17T17:03:16.680 INFO:tasks.rados.rados.0.smithi063.stdout:1966: snap_remove snap 251 2024-08-17T17:03:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:16 smithi063 ceph-mon[114756]: osdmap e568: 8 total, 8 up, 8 in 2024-08-17T17:03:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:16 smithi063 ceph-mon[114756]: pgmap v309: 89 pgs: 89 active+clean; 171 MiB data, 656 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:16 smithi063 ceph-mon[109843]: osdmap e568: 8 total, 8 up, 8 in 2024-08-17T17:03:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:16 smithi063 ceph-mon[109843]: pgmap v309: 89 pgs: 89 active+clean; 171 MiB data, 656 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:16 smithi162 ceph-mon[104668]: osdmap e568: 8 total, 8 up, 8 in 2024-08-17T17:03:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:16 smithi162 ceph-mon[104668]: pgmap v309: 89 pgs: 89 active+clean; 171 MiB data, 656 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:17.555 INFO:tasks.rados.rados.0.smithi063.stdout:1966: done (2 left) 2024-08-17T17:03:17.555 INFO:tasks.rados.rados.0.smithi063.stdout:1967: read oid 28 snap -1 2024-08-17T17:03:17.555 INFO:tasks.rados.rados.0.smithi063.stdout:1967: expect deleted 2024-08-17T17:03:17.555 INFO:tasks.rados.rados.0.smithi063.stdout:1968: write oid 45 current snap is 253 2024-08-17T17:03:17.555 INFO:tasks.rados.rados.0.smithi063.stdout:1968: seq_num 743 ranges {487601=796248,1962121=525480} 2024-08-17T17:03:17.561 INFO:tasks.rados.rados.0.smithi063.stdout:1968: writing smithi06387500-45 from 487601 to 1283849 tid 1 2024-08-17T17:03:17.569 INFO:tasks.rados.rados.0.smithi063.stdout:1968: writing smithi06387500-45 from 1962121 to 2487601 tid 2 2024-08-17T17:03:17.569 INFO:tasks.rados.rados.0.smithi063.stdout:1969: snap_remove snap 237 2024-08-17T17:03:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:17 smithi063 ceph-mon[114756]: osdmap e569: 8 total, 8 up, 8 in 2024-08-17T17:03:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:17 smithi063 ceph-mon[114756]: osdmap e570: 8 total, 8 up, 8 in 2024-08-17T17:03:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:17 smithi063 ceph-mon[109843]: osdmap e569: 8 total, 8 up, 8 in 2024-08-17T17:03:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:17 smithi063 ceph-mon[109843]: osdmap e570: 8 total, 8 up, 8 in 2024-08-17T17:03:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:17 smithi162 ceph-mon[104668]: osdmap e569: 8 total, 8 up, 8 in 2024-08-17T17:03:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:17 smithi162 ceph-mon[104668]: osdmap e570: 8 total, 8 up, 8 in 2024-08-17T17:03:18.558 INFO:tasks.rados.rados.0.smithi063.stdout:1964: expect (ObjNum 655 snap 209 seq_num 655) 2024-08-17T17:03:18.774 INFO:tasks.rados.rados.0.smithi063.stdout:1965: expect (ObjNum 683 snap 218 seq_num 683) 2024-08-17T17:03:18.892 INFO:tasks.rados.rados.0.smithi063.stdout:1968: finishing write tid 1 to smithi06387500-45 2024-08-17T17:03:18.892 INFO:tasks.rados.rados.0.smithi063.stdout:1968: finishing write tid 2 to smithi06387500-45 2024-08-17T17:03:18.892 INFO:tasks.rados.rados.0.smithi063.stdout:1968: finishing write tid 3 to smithi06387500-45 2024-08-17T17:03:18.892 INFO:tasks.rados.rados.0.smithi063.stdout:1968: finishing write tid 4 to smithi06387500-45 2024-08-17T17:03:18.892 INFO:tasks.rados.rados.0.smithi063.stdout:1968: oid 45 updating version 0 to 422 2024-08-17T17:03:18.892 INFO:tasks.rados.rados.0.smithi063.stdout:1968: oid 45 updating version 422 to 423 2024-08-17T17:03:18.893 INFO:tasks.rados.rados.0.smithi063.stdout:1968: oid 45 version 423 is already newer than 421 2024-08-17T17:03:18.893 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 423 (ObjNum 743 snap 253 seq_num 743) dirty exists 2024-08-17T17:03:18.893 INFO:tasks.rados.rados.0.smithi063.stdout:1968: left oid 45 (ObjNum 743 snap 253 seq_num 743) 2024-08-17T17:03:18.893 INFO:tasks.rados.rados.0.smithi063.stdout:1964: done (4 left) 2024-08-17T17:03:18.893 INFO:tasks.rados.rados.0.smithi063.stdout:1965: done (3 left) 2024-08-17T17:03:18.893 INFO:tasks.rados.rados.0.smithi063.stdout:1967: done (2 left) 2024-08-17T17:03:18.893 INFO:tasks.rados.rados.0.smithi063.stdout:1968: done (1 left) 2024-08-17T17:03:18.893 INFO:tasks.rados.rados.0.smithi063.stdout:1969: done (0 left) 2024-08-17T17:03:18.893 INFO:tasks.rados.rados.0.smithi063.stdout:1970: write oid 5 current snap is 253 2024-08-17T17:03:18.893 INFO:tasks.rados.rados.0.smithi063.stdout:1970: seq_num 744 ranges {504408=471497,1620603=557925} 2024-08-17T17:03:18.895 INFO:tasks.rados.rados.0.smithi063.stdout:1970: writing smithi06387500-5 from 504408 to 975905 tid 1 2024-08-17T17:03:18.902 INFO:tasks.rados.rados.0.smithi063.stdout:1970: writing smithi06387500-5 from 1620603 to 2178528 tid 2 2024-08-17T17:03:18.903 INFO:tasks.rados.rados.0.smithi063.stdout:1971: rollback oid 50 current snap is 253 2024-08-17T17:03:18.903 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 50 to 236 2024-08-17T17:03:18.903 INFO:tasks.rados.rados.0.smithi063.stdout:1970: finishing write tid 1 to smithi06387500-5 2024-08-17T17:03:18.903 INFO:tasks.rados.rados.0.smithi063.stdout:1972: snap_create 2024-08-17T17:03:18.906 INFO:tasks.rados.rados.0.smithi063.stdout:1971: finishing rollback tid 0 to smithi06387500-50 2024-08-17T17:03:18.906 INFO:tasks.rados.rados.0.smithi063.stdout:1970: finishing write tid 2 to smithi06387500-5 2024-08-17T17:03:18.906 INFO:tasks.rados.rados.0.smithi063.stdout:1971: finishing rollback tid 1 to smithi06387500-50 2024-08-17T17:03:18.906 INFO:tasks.rados.rados.0.smithi063.stdout:1970: finishing write tid 3 to smithi06387500-5 2024-08-17T17:03:18.906 INFO:tasks.rados.rados.0.smithi063.stdout:1971: finishing rollback tid 2 to smithi06387500-50 2024-08-17T17:03:18.906 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 1301 (ObjNum 669 snap 212 seq_num 669) dirty exists 2024-08-17T17:03:18.907 INFO:tasks.rados.rados.0.smithi063.stdout:1970: finishing write tid 4 to smithi06387500-5 2024-08-17T17:03:18.907 INFO:tasks.rados.rados.0.smithi063.stdout:1970: oid 5 updating version 0 to 893 2024-08-17T17:03:18.907 INFO:tasks.rados.rados.0.smithi063.stdout:1970: oid 5 version 893 is already newer than 891 2024-08-17T17:03:18.907 INFO:tasks.rados.rados.0.smithi063.stdout:1970: oid 5 version 893 is already newer than 892 2024-08-17T17:03:18.907 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 893 (ObjNum 744 snap 253 seq_num 744) dirty exists 2024-08-17T17:03:18.907 INFO:tasks.rados.rados.0.smithi063.stdout:1970: left oid 5 (ObjNum 744 snap 253 seq_num 744) 2024-08-17T17:03:19.687 INFO:tasks.rados.rados.0.smithi063.stdout:1970: done (2 left) 2024-08-17T17:03:19.687 INFO:tasks.rados.rados.0.smithi063.stdout:1971: done (1 left) 2024-08-17T17:03:19.687 INFO:tasks.rados.rados.0.smithi063.stdout:1972: done (0 left) 2024-08-17T17:03:19.687 INFO:tasks.rados.rados.0.smithi063.stdout:1973: read oid 22 snap -1 2024-08-17T17:03:19.687 INFO:tasks.rados.rados.0.smithi063.stdout:1973: expect (ObjNum 626 snap 199 seq_num 626) 2024-08-17T17:03:19.687 INFO:tasks.rados.rados.0.smithi063.stdout:1974: delete oid 21 current snap is 254 2024-08-17T17:03:19.691 INFO:tasks.rados.rados.0.smithi063.stdout:1974: done (1 left) 2024-08-17T17:03:19.691 INFO:tasks.rados.rados.0.smithi063.stdout:1975: delete oid 28 current snap is 254 2024-08-17T17:03:19.692 INFO:tasks.rados.rados.0.smithi063.stdout:1973: expect (ObjNum 626 snap 199 seq_num 626) 2024-08-17T17:03:19.776 INFO:tasks.rados.rados.0.smithi063.stdout:1973: done (1 left) 2024-08-17T17:03:19.777 INFO:tasks.rados.rados.0.smithi063.stdout:1975: done (0 left) 2024-08-17T17:03:19.777 INFO:tasks.rados.rados.0.smithi063.stdout:1976: snap_remove snap 216 2024-08-17T17:03:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:19 smithi162 ceph-mon[104668]: pgmap v312: 89 pgs: 89 active+clean; 170 MiB data, 665 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-08-17T17:03:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:19 smithi162 ceph-mon[104668]: osdmap e571: 8 total, 8 up, 8 in 2024-08-17T17:03:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:19 smithi063 ceph-mon[114756]: pgmap v312: 89 pgs: 89 active+clean; 170 MiB data, 665 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-08-17T17:03:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:19 smithi063 ceph-mon[114756]: osdmap e571: 8 total, 8 up, 8 in 2024-08-17T17:03:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:19 smithi063 ceph-mon[109843]: pgmap v312: 89 pgs: 89 active+clean; 170 MiB data, 665 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-08-17T17:03:20.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:19 smithi063 ceph-mon[109843]: osdmap e571: 8 total, 8 up, 8 in 2024-08-17T17:03:20.692 INFO:tasks.rados.rados.0.smithi063.stdout:1976: done (0 left) 2024-08-17T17:03:20.692 INFO:tasks.rados.rados.0.smithi063.stdout:1977: write oid 6 current snap is 254 2024-08-17T17:03:20.692 INFO:tasks.rados.rados.0.smithi063.stdout:1977: seq_num 747 ranges {554829=413597,1396049=705252} 2024-08-17T17:03:20.697 INFO:tasks.rados.rados.0.smithi063.stdout:1977: writing smithi06387500-6 from 554829 to 968426 tid 1 2024-08-17T17:03:20.705 INFO:tasks.rados.rados.0.smithi063.stdout:1977: writing smithi06387500-6 from 1396049 to 2101301 tid 2 2024-08-17T17:03:20.705 INFO:tasks.rados.rados.0.smithi063.stdout:1978: read oid 41 snap -1 2024-08-17T17:03:20.705 INFO:tasks.rados.rados.0.smithi063.stdout:1978: expect (ObjNum 718 snap 244 seq_num 718) 2024-08-17T17:03:20.705 INFO:tasks.rados.rados.0.smithi063.stdout:1979: read oid 27 snap 246 2024-08-17T17:03:20.705 INFO:tasks.rados.rados.0.smithi063.stdout:1979: expect deleted 2024-08-17T17:03:20.705 INFO:tasks.rados.rados.0.smithi063.stdout:1980: read oid 8 snap -1 2024-08-17T17:03:20.705 INFO:tasks.rados.rados.0.smithi063.stdout:1980: expect deleted 2024-08-17T17:03:20.705 INFO:tasks.rados.rados.0.smithi063.stdout:1981: delete oid 15 current snap is 254 2024-08-17T17:03:20.705 INFO:tasks.rados.rados.0.smithi063.stdout:1977: finishing write tid 1 to smithi06387500-6 2024-08-17T17:03:20.707 INFO:tasks.rados.rados.0.smithi063.stdout:1979: done (4 left) 2024-08-17T17:03:20.707 INFO:tasks.rados.rados.0.smithi063.stdout:1981: done (3 left) 2024-08-17T17:03:20.707 INFO:tasks.rados.rados.0.smithi063.stdout:1982: read oid 39 snap -1 2024-08-17T17:03:20.707 INFO:tasks.rados.rados.0.smithi063.stdout:1982: expect (ObjNum 713 snap 241 seq_num 713) 2024-08-17T17:03:20.707 INFO:tasks.rados.rados.0.smithi063.stdout:1983: read oid 15 snap 246 2024-08-17T17:03:20.707 INFO:tasks.rados.rados.0.smithi063.stdout:1983: expect deleted 2024-08-17T17:03:20.707 INFO:tasks.rados.rados.0.smithi063.stdout:1984: snap_remove snap 250 2024-08-17T17:03:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:20 smithi063 ceph-mon[114756]: osdmap e572: 8 total, 8 up, 8 in 2024-08-17T17:03:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:20 smithi063 ceph-mon[114756]: pgmap v315: 89 pgs: 89 active+clean; 170 MiB data, 665 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-08-17T17:03:21.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:20 smithi063 ceph-mon[109843]: osdmap e572: 8 total, 8 up, 8 in 2024-08-17T17:03:21.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:20 smithi063 ceph-mon[109843]: pgmap v315: 89 pgs: 89 active+clean; 170 MiB data, 665 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-08-17T17:03:21.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:20 smithi162 ceph-mon[104668]: osdmap e572: 8 total, 8 up, 8 in 2024-08-17T17:03:21.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:20 smithi162 ceph-mon[104668]: pgmap v315: 89 pgs: 89 active+clean; 170 MiB data, 665 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-08-17T17:03:21.704 INFO:tasks.rados.rados.0.smithi063.stdout:1984: done (5 left) 2024-08-17T17:03:21.704 INFO:tasks.rados.rados.0.smithi063.stdout:1985: rollback oid 17 current snap is 254 2024-08-17T17:03:21.704 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 17 to 246 2024-08-17T17:03:21.704 INFO:tasks.rados.rados.0.smithi063.stdout:1977: finishing write tid 2 to smithi06387500-6 2024-08-17T17:03:21.704 INFO:tasks.rados.rados.0.smithi063.stdout:1977: finishing write tid 3 to smithi06387500-6 2024-08-17T17:03:21.704 INFO:tasks.rados.rados.0.smithi063.stdout:1977: finishing write tid 4 to smithi06387500-6 2024-08-17T17:03:21.704 INFO:tasks.rados.rados.0.smithi063.stdout:1977: oid 6 updating version 0 to 572 2024-08-17T17:03:21.704 INFO:tasks.rados.rados.0.smithi063.stdout:1977: oid 6 updating version 572 to 573 2024-08-17T17:03:21.705 INFO:tasks.rados.rados.0.smithi063.stdout:1977: oid 6 version 573 is already newer than 571 2024-08-17T17:03:21.705 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 573 (ObjNum 747 snap 254 seq_num 747) dirty exists 2024-08-17T17:03:21.705 INFO:tasks.rados.rados.0.smithi063.stdout:1977: left oid 6 (ObjNum 747 snap 254 seq_num 747) 2024-08-17T17:03:21.705 INFO:tasks.rados.rados.0.smithi063.stdout:1977: done (5 left) 2024-08-17T17:03:21.705 INFO:tasks.rados.rados.0.smithi063.stdout:1980: done (4 left) 2024-08-17T17:03:21.705 INFO:tasks.rados.rados.0.smithi063.stdout:1983: done (3 left) 2024-08-17T17:03:21.705 INFO:tasks.rados.rados.0.smithi063.stdout:1986: read oid 2 snap -1 2024-08-17T17:03:21.705 INFO:tasks.rados.rados.0.smithi063.stdout:1986: expect (ObjNum 741 snap 251 seq_num 741) 2024-08-17T17:03:21.705 INFO:tasks.rados.rados.0.smithi063.stdout:1987: write oid 20 current snap is 254 2024-08-17T17:03:21.705 INFO:tasks.rados.rados.0.smithi063.stdout:1978: expect (ObjNum 718 snap 244 seq_num 718) 2024-08-17T17:03:21.801 INFO:tasks.rados.rados.0.smithi063.stdout:1982: expect (ObjNum 713 snap 241 seq_num 713) 2024-08-17T17:03:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:21 smithi063 ceph-mon[114756]: osdmap e573: 8 total, 8 up, 8 in 2024-08-17T17:03:22.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:21 smithi063 ceph-mon[109843]: osdmap e573: 8 total, 8 up, 8 in 2024-08-17T17:03:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:21 smithi162 ceph-mon[104668]: osdmap e573: 8 total, 8 up, 8 in 2024-08-17T17:03:22.101 INFO:tasks.rados.rados.0.smithi063.stdout:1985: finishing rollback tid 1 to smithi06387500-17 2024-08-17T17:03:22.101 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 542 (ObjNum 289 snap 0 seq_num 1028107813) dirty dne 2024-08-17T17:03:22.101 INFO:tasks.rados.rados.0.smithi063.stdout:1986: expect (ObjNum 741 snap 251 seq_num 741) 2024-08-17T17:03:22.226 INFO:tasks.rados.rados.0.smithi063.stdout:1987: seq_num 749 ranges {588443=564095,1796660=708921} 2024-08-17T17:03:22.232 INFO:tasks.rados.rados.0.smithi063.stdout:1987: writing smithi06387500-20 from 588443 to 1152538 tid 1 2024-08-17T17:03:22.239 INFO:tasks.rados.rados.0.smithi063.stdout:1987: writing smithi06387500-20 from 1796660 to 2505581 tid 2 2024-08-17T17:03:22.240 INFO:tasks.rados.rados.0.smithi063.stdout:1978: done (4 left) 2024-08-17T17:03:22.240 INFO:tasks.rados.rados.0.smithi063.stdout:1982: done (3 left) 2024-08-17T17:03:22.240 INFO:tasks.rados.rados.0.smithi063.stdout:1985: done (2 left) 2024-08-17T17:03:22.240 INFO:tasks.rados.rados.0.smithi063.stdout:1986: done (1 left) 2024-08-17T17:03:22.240 INFO:tasks.rados.rados.0.smithi063.stdout:1988: snap_create 2024-08-17T17:03:22.240 INFO:tasks.rados.rados.0.smithi063.stdout:1987: finishing write tid 1 to smithi06387500-20 2024-08-17T17:03:22.242 INFO:tasks.rados.rados.0.smithi063.stdout:1987: finishing write tid 2 to smithi06387500-20 2024-08-17T17:03:22.242 INFO:tasks.rados.rados.0.smithi063.stdout:1987: finishing write tid 3 to smithi06387500-20 2024-08-17T17:03:22.242 INFO:tasks.rados.rados.0.smithi063.stdout:1987: finishing write tid 4 to smithi06387500-20 2024-08-17T17:03:22.242 INFO:tasks.rados.rados.0.smithi063.stdout:1987: oid 20 updating version 0 to 397 2024-08-17T17:03:22.243 INFO:tasks.rados.rados.0.smithi063.stdout:1987: oid 20 updating version 397 to 398 2024-08-17T17:03:22.243 INFO:tasks.rados.rados.0.smithi063.stdout:1987: oid 20 version 398 is already newer than 396 2024-08-17T17:03:22.243 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 398 (ObjNum 749 snap 254 seq_num 749) dirty exists 2024-08-17T17:03:22.243 INFO:tasks.rados.rados.0.smithi063.stdout:1987: left oid 20 (ObjNum 749 snap 254 seq_num 749) 2024-08-17T17:03:22.553 INFO:tasks.rados.rados.0.smithi063.stdout:1987: done (1 left) 2024-08-17T17:03:22.553 INFO:tasks.rados.rados.0.smithi063.stdout:1988: done (0 left) 2024-08-17T17:03:22.553 INFO:tasks.rados.rados.0.smithi063.stdout:1989: snap_remove snap 218 2024-08-17T17:03:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:22 smithi063 ceph-mon[114756]: osdmap e574: 8 total, 8 up, 8 in 2024-08-17T17:03:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:22 smithi063 ceph-mon[114756]: pgmap v318: 89 pgs: 89 active+clean; 170 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 848 KiB/s wr, 13 op/s 2024-08-17T17:03:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:22 smithi063 ceph-mon[114756]: osdmap e575: 8 total, 8 up, 8 in 2024-08-17T17:03:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:22 smithi063 ceph-mon[109843]: osdmap e574: 8 total, 8 up, 8 in 2024-08-17T17:03:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:22 smithi063 ceph-mon[109843]: pgmap v318: 89 pgs: 89 active+clean; 170 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 848 KiB/s wr, 13 op/s 2024-08-17T17:03:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:22 smithi063 ceph-mon[109843]: osdmap e575: 8 total, 8 up, 8 in 2024-08-17T17:03:23.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:22 smithi162 ceph-mon[104668]: osdmap e574: 8 total, 8 up, 8 in 2024-08-17T17:03:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:22 smithi162 ceph-mon[104668]: pgmap v318: 89 pgs: 89 active+clean; 170 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 848 KiB/s wr, 13 op/s 2024-08-17T17:03:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:22 smithi162 ceph-mon[104668]: osdmap e575: 8 total, 8 up, 8 in 2024-08-17T17:03:23.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:03:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:03:23] "GET /metrics HTTP/1.1" 200 36209 "" "Prometheus/2.51.0" 2024-08-17T17:03:23.555 INFO:tasks.rados.rados.0.smithi063.stdout:1989: done (0 left) 2024-08-17T17:03:23.555 INFO:tasks.rados.rados.0.smithi063.stdout:1990: read oid 40 snap 220 2024-08-17T17:03:23.556 INFO:tasks.rados.rados.0.smithi063.stdout:1990: expect deleted 2024-08-17T17:03:23.556 INFO:tasks.rados.rados.0.smithi063.stdout:1991: delete oid 42 current snap is 255 2024-08-17T17:03:23.564 INFO:tasks.rados.rados.0.smithi063.stdout:1990: done (1 left) 2024-08-17T17:03:23.564 INFO:tasks.rados.rados.0.smithi063.stdout:1991: done (0 left) 2024-08-17T17:03:23.564 INFO:tasks.rados.rados.0.smithi063.stdout:1992: snap_create 2024-08-17T17:03:24.564 INFO:tasks.rados.rados.0.smithi063.stdout:1992: done (0 left) 2024-08-17T17:03:24.564 INFO:tasks.rados.rados.0.smithi063.stdout:1993: rollback oid 31 current snap is 256 2024-08-17T17:03:24.564 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 31 to 238 2024-08-17T17:03:24.565 INFO:tasks.rados.rados.0.smithi063.stdout:1994: delete oid 14 current snap is 256 2024-08-17T17:03:24.570 INFO:tasks.rados.rados.0.smithi063.stdout:1994: done (1 left) 2024-08-17T17:03:24.570 INFO:tasks.rados.rados.0.smithi063.stdout:1995: write oid 32 current snap is 256 2024-08-17T17:03:24.570 INFO:tasks.rados.rados.0.smithi063.stdout:1995: seq_num 752 ranges {638864=789842,1972106=456248} 2024-08-17T17:03:24.577 INFO:tasks.rados.rados.0.smithi063.stdout:1995: writing smithi06387500-32 from 638864 to 1428706 tid 1 2024-08-17T17:03:24.584 INFO:tasks.rados.rados.0.smithi063.stdout:1995: writing smithi06387500-32 from 1972106 to 2428354 tid 2 2024-08-17T17:03:24.584 INFO:tasks.rados.rados.0.smithi063.stdout:1996: write oid 8 current snap is 256 2024-08-17T17:03:24.584 INFO:tasks.rados.rados.0.smithi063.stdout:1993: finishing rollback tid 1 to smithi06387500-31 2024-08-17T17:03:24.584 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 0 (ObjNum 177 snap 0 seq_num 1807304768) dirty dne 2024-08-17T17:03:24.584 INFO:tasks.rados.rados.0.smithi063.stdout:1996: seq_num 753 ranges {655671=581444,2030588=625083} 2024-08-17T17:03:24.591 INFO:tasks.rados.rados.0.smithi063.stdout:1996: writing smithi06387500-8 from 655671 to 1237115 tid 1 2024-08-17T17:03:24.599 INFO:tasks.rados.rados.0.smithi063.stdout:1996: writing smithi06387500-8 from 2030588 to 2655671 tid 2 2024-08-17T17:03:24.599 INFO:tasks.rados.rados.0.smithi063.stdout:1993: done (2 left) 2024-08-17T17:03:24.599 INFO:tasks.rados.rados.0.smithi063.stdout:1997: snap_remove snap 254 2024-08-17T17:03:24.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:24 smithi162 ceph-mon[104668]: osdmap e576: 8 total, 8 up, 8 in 2024-08-17T17:03:25.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:24 smithi063 ceph-mon[114756]: osdmap e576: 8 total, 8 up, 8 in 2024-08-17T17:03:25.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:24 smithi063 ceph-mon[109843]: osdmap e576: 8 total, 8 up, 8 in 2024-08-17T17:03:25.575 INFO:tasks.rados.rados.0.smithi063.stdout:1995: finishing write tid 1 to smithi06387500-32 2024-08-17T17:03:25.575 INFO:tasks.rados.rados.0.smithi063.stdout:1995: finishing write tid 2 to smithi06387500-32 2024-08-17T17:03:25.575 INFO:tasks.rados.rados.0.smithi063.stdout:1995: finishing write tid 3 to smithi06387500-32 2024-08-17T17:03:25.575 INFO:tasks.rados.rados.0.smithi063.stdout:1995: finishing write tid 4 to smithi06387500-32 2024-08-17T17:03:25.575 INFO:tasks.rados.rados.0.smithi063.stdout:1995: oid 32 updating version 0 to 374 2024-08-17T17:03:25.575 INFO:tasks.rados.rados.0.smithi063.stdout:1995: oid 32 version 374 is already newer than 373 2024-08-17T17:03:25.575 INFO:tasks.rados.rados.0.smithi063.stdout:1995: oid 32 updating version 374 to 375 2024-08-17T17:03:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 375 (ObjNum 752 snap 256 seq_num 752) dirty exists 2024-08-17T17:03:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1995: left oid 32 (ObjNum 752 snap 256 seq_num 752) 2024-08-17T17:03:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1996: finishing write tid 1 to smithi06387500-8 2024-08-17T17:03:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1996: finishing write tid 2 to smithi06387500-8 2024-08-17T17:03:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1996: finishing write tid 3 to smithi06387500-8 2024-08-17T17:03:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1996: finishing write tid 4 to smithi06387500-8 2024-08-17T17:03:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1996: oid 8 updating version 0 to 579 2024-08-17T17:03:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1996: oid 8 updating version 579 to 580 2024-08-17T17:03:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1996: oid 8 version 580 is already newer than 578 2024-08-17T17:03:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 580 (ObjNum 753 snap 256 seq_num 753) dirty exists 2024-08-17T17:03:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1996: left oid 8 (ObjNum 753 snap 256 seq_num 753) 2024-08-17T17:03:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1995: done (2 left) 2024-08-17T17:03:25.576 INFO:tasks.rados.rados.0.smithi063.stdout:1996: done (1 left) 2024-08-17T17:03:25.577 INFO:tasks.rados.rados.0.smithi063.stdout:1997: done (0 left) 2024-08-17T17:03:25.577 INFO:tasks.rados.rados.0.smithi063.stdout:1998: delete oid 34 current snap is 256 2024-08-17T17:03:25.577 INFO:tasks.rados.rados.0.smithi063.stdout:1998: done (0 left) 2024-08-17T17:03:25.577 INFO:tasks.rados.rados.0.smithi063.stdout:1999: rollback oid 25 current snap is 256 2024-08-17T17:03:25.578 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 25 to 246 2024-08-17T17:03:25.578 INFO:tasks.rados.rados.0.smithi063.stdout:2000: snap_create 2024-08-17T17:03:25.582 INFO:tasks.rados.rados.0.smithi063.stdout:1999: finishing rollback tid 0 to smithi06387500-25 2024-08-17T17:03:25.582 INFO:tasks.rados.rados.0.smithi063.stdout:1999: finishing rollback tid 1 to smithi06387500-25 2024-08-17T17:03:25.582 INFO:tasks.rados.rados.0.smithi063.stdout:1999: finishing rollback tid 2 to smithi06387500-25 2024-08-17T17:03:25.582 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 1936 (ObjNum 714 snap 243 seq_num 714) dirty exists 2024-08-17T17:03:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:25 smithi162 ceph-mon[104668]: pgmap v321: 89 pgs: 89 active+clean; 170 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 848 KiB/s wr, 13 op/s 2024-08-17T17:03:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:25 smithi162 ceph-mon[104668]: osdmap e577: 8 total, 8 up, 8 in 2024-08-17T17:03:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:25 smithi063 ceph-mon[114756]: pgmap v321: 89 pgs: 89 active+clean; 170 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 848 KiB/s wr, 13 op/s 2024-08-17T17:03:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:25 smithi063 ceph-mon[114756]: osdmap e577: 8 total, 8 up, 8 in 2024-08-17T17:03:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:25 smithi063 ceph-mon[109843]: pgmap v321: 89 pgs: 89 active+clean; 170 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 848 KiB/s wr, 13 op/s 2024-08-17T17:03:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:25 smithi063 ceph-mon[109843]: osdmap e577: 8 total, 8 up, 8 in 2024-08-17T17:03:26.580 INFO:tasks.rados.rados.0.smithi063.stdout:1999: done (1 left) 2024-08-17T17:03:26.581 INFO:tasks.rados.rados.0.smithi063.stdout:2000: done (0 left) 2024-08-17T17:03:26.581 INFO:tasks.rados.rados.0.smithi063.stdout:2001: write oid 42 current snap is 257 2024-08-17T17:03:26.581 INFO:tasks.rados.rados.0.smithi063.stdout:2001: seq_num 755 ranges {689285=731942,2147552=541733} 2024-08-17T17:03:26.588 INFO:tasks.rados.rados.0.smithi063.stdout:2001: writing smithi06387500-42 from 689285 to 1421227 tid 1 2024-08-17T17:03:26.595 INFO:tasks.rados.rados.0.smithi063.stdout:2001: writing smithi06387500-42 from 2147552 to 2689285 tid 2 2024-08-17T17:03:26.595 INFO:tasks.rados.rados.0.smithi063.stdout:2002: delete oid 14 current snap is 257 2024-08-17T17:03:26.596 INFO:tasks.rados.rados.0.smithi063.stdout:2001: finishing write tid 1 to smithi06387500-42 2024-08-17T17:03:26.599 INFO:tasks.rados.rados.0.smithi063.stdout:2001: finishing write tid 2 to smithi06387500-42 2024-08-17T17:03:26.599 INFO:tasks.rados.rados.0.smithi063.stdout:2002: done (1 left) 2024-08-17T17:03:26.599 INFO:tasks.rados.rados.0.smithi063.stdout:2003: read oid 3 snap -1 2024-08-17T17:03:26.599 INFO:tasks.rados.rados.0.smithi063.stdout:2003: expect (ObjNum 723 snap 246 seq_num 723) 2024-08-17T17:03:26.599 INFO:tasks.rados.rados.0.smithi063.stdout:2004: write oid 23 current snap is 257 2024-08-17T17:03:26.599 INFO:tasks.rados.rados.0.smithi063.stdout:2004: seq_num 757 ranges {722899=482440,1748163=723597} 2024-08-17T17:03:26.605 INFO:tasks.rados.rados.0.smithi063.stdout:2004: writing smithi06387500-23 from 722899 to 1205339 tid 1 2024-08-17T17:03:26.612 INFO:tasks.rados.rados.0.smithi063.stdout:2004: writing smithi06387500-23 from 1748163 to 2471760 tid 2 2024-08-17T17:03:26.612 INFO:tasks.rados.rados.0.smithi063.stdout:2005: write oid 27 current snap is 257 2024-08-17T17:03:26.612 INFO:tasks.rados.rados.0.smithi063.stdout:2001: finishing write tid 3 to smithi06387500-42 2024-08-17T17:03:26.613 INFO:tasks.rados.rados.0.smithi063.stdout:2001: finishing write tid 4 to smithi06387500-42 2024-08-17T17:03:26.613 INFO:tasks.rados.rados.0.smithi063.stdout:2001: oid 42 updating version 0 to 1321 2024-08-17T17:03:26.613 INFO:tasks.rados.rados.0.smithi063.stdout:2001: oid 42 updating version 1321 to 1322 2024-08-17T17:03:26.613 INFO:tasks.rados.rados.0.smithi063.stdout:2001: oid 42 updating version 1322 to 1323 2024-08-17T17:03:26.613 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 1323 (ObjNum 755 snap 257 seq_num 755) dirty exists 2024-08-17T17:03:26.613 INFO:tasks.rados.rados.0.smithi063.stdout:2001: left oid 42 (ObjNum 755 snap 257 seq_num 755) 2024-08-17T17:03:26.613 INFO:tasks.rados.rados.0.smithi063.stdout:2003: expect (ObjNum 723 snap 246 seq_num 723) 2024-08-17T17:03:26.686 INFO:tasks.rados.rados.0.smithi063.stdout:2004: finishing write tid 1 to smithi06387500-23 2024-08-17T17:03:26.686 INFO:tasks.rados.rados.0.smithi063.stdout:2004: finishing write tid 2 to smithi06387500-23 2024-08-17T17:03:26.686 INFO:tasks.rados.rados.0.smithi063.stdout:2004: finishing write tid 3 to smithi06387500-23 2024-08-17T17:03:26.687 INFO:tasks.rados.rados.0.smithi063.stdout:2004: finishing write tid 4 to smithi06387500-23 2024-08-17T17:03:26.687 INFO:tasks.rados.rados.0.smithi063.stdout:2004: oid 23 updating version 0 to 919 2024-08-17T17:03:26.687 INFO:tasks.rados.rados.0.smithi063.stdout:2004: oid 23 version 919 is already newer than 917 2024-08-17T17:03:26.687 INFO:tasks.rados.rados.0.smithi063.stdout:2004: oid 23 version 919 is already newer than 918 2024-08-17T17:03:26.687 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 919 (ObjNum 757 snap 257 seq_num 757) dirty exists 2024-08-17T17:03:26.687 INFO:tasks.rados.rados.0.smithi063.stdout:2004: left oid 23 (ObjNum 757 snap 257 seq_num 757) 2024-08-17T17:03:26.687 INFO:tasks.rados.rados.0.smithi063.stdout:2005: seq_num 758 ranges {739706=557689,1806645=467255} 2024-08-17T17:03:26.693 INFO:tasks.rados.rados.0.smithi063.stdout:2005: writing smithi06387500-27 from 739706 to 1297395 tid 1 2024-08-17T17:03:26.699 INFO:tasks.rados.rados.0.smithi063.stdout:2005: writing smithi06387500-27 from 1806645 to 2273900 tid 2 2024-08-17T17:03:26.699 INFO:tasks.rados.rados.0.smithi063.stdout:2001: done (3 left) 2024-08-17T17:03:26.699 INFO:tasks.rados.rados.0.smithi063.stdout:2003: done (2 left) 2024-08-17T17:03:26.699 INFO:tasks.rados.rados.0.smithi063.stdout:2004: done (1 left) 2024-08-17T17:03:26.699 INFO:tasks.rados.rados.0.smithi063.stdout:2006: write oid 46 current snap is 257 2024-08-17T17:03:26.699 INFO:tasks.rados.rados.0.smithi063.stdout:2005: finishing write tid 1 to smithi06387500-27 2024-08-17T17:03:26.699 INFO:tasks.rados.rados.0.smithi063.stdout:2006: seq_num 759 ranges {756513=632938,1865127=727266} 2024-08-17T17:03:26.706 INFO:tasks.rados.rados.0.smithi063.stdout:2006: writing smithi06387500-46 from 756513 to 1389451 tid 1 2024-08-17T17:03:26.713 INFO:tasks.rados.rados.0.smithi063.stdout:2006: writing smithi06387500-46 from 1865127 to 2592393 tid 2 2024-08-17T17:03:26.713 INFO:tasks.rados.rados.0.smithi063.stdout:2007: read oid 41 snap -1 2024-08-17T17:03:26.713 INFO:tasks.rados.rados.0.smithi063.stdout:2007: expect (ObjNum 718 snap 244 seq_num 718) 2024-08-17T17:03:26.713 INFO:tasks.rados.rados.0.smithi063.stdout:2008: snap_create 2024-08-17T17:03:26.714 INFO:tasks.rados.rados.0.smithi063.stdout:2005: finishing write tid 2 to smithi06387500-27 2024-08-17T17:03:26.714 INFO:tasks.rados.rados.0.smithi063.stdout:2005: finishing write tid 3 to smithi06387500-27 2024-08-17T17:03:26.714 INFO:tasks.rados.rados.0.smithi063.stdout:2005: finishing write tid 4 to smithi06387500-27 2024-08-17T17:03:26.714 INFO:tasks.rados.rados.0.smithi063.stdout:2005: oid 27 updating version 0 to 198 2024-08-17T17:03:26.714 INFO:tasks.rados.rados.0.smithi063.stdout:2005: oid 27 version 198 is already newer than 197 2024-08-17T17:03:26.714 INFO:tasks.rados.rados.0.smithi063.stdout:2005: oid 27 version 198 is already newer than 196 2024-08-17T17:03:26.714 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 198 (ObjNum 758 snap 257 seq_num 758) dirty exists 2024-08-17T17:03:26.714 INFO:tasks.rados.rados.0.smithi063.stdout:2005: left oid 27 (ObjNum 758 snap 257 seq_num 758) 2024-08-17T17:03:26.714 INFO:tasks.rados.rados.0.smithi063.stdout:2006: finishing write tid 1 to smithi06387500-46 2024-08-17T17:03:26.716 INFO:tasks.rados.rados.0.smithi063.stdout:2006: finishing write tid 2 to smithi06387500-46 2024-08-17T17:03:26.716 INFO:tasks.rados.rados.0.smithi063.stdout:2006: finishing write tid 3 to smithi06387500-46 2024-08-17T17:03:26.716 INFO:tasks.rados.rados.0.smithi063.stdout:2006: finishing write tid 4 to smithi06387500-46 2024-08-17T17:03:26.716 INFO:tasks.rados.rados.0.smithi063.stdout:2006: oid 46 updating version 0 to 753 2024-08-17T17:03:26.716 INFO:tasks.rados.rados.0.smithi063.stdout:2006: oid 46 updating version 753 to 754 2024-08-17T17:03:26.716 INFO:tasks.rados.rados.0.smithi063.stdout:2006: oid 46 version 754 is already newer than 752 2024-08-17T17:03:26.716 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 754 (ObjNum 759 snap 257 seq_num 759) dirty exists 2024-08-17T17:03:26.717 INFO:tasks.rados.rados.0.smithi063.stdout:2006: left oid 46 (ObjNum 759 snap 257 seq_num 759) 2024-08-17T17:03:26.719 INFO:tasks.rados.rados.0.smithi063.stdout:2007: expect (ObjNum 718 snap 244 seq_num 718) 2024-08-17T17:03:26.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:26 smithi162 ceph-mon[104668]: osdmap e578: 8 total, 8 up, 8 in 2024-08-17T17:03:27.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:26 smithi063 ceph-mon[114756]: osdmap e578: 8 total, 8 up, 8 in 2024-08-17T17:03:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:26 smithi063 ceph-mon[109843]: osdmap e578: 8 total, 8 up, 8 in 2024-08-17T17:03:27.555 INFO:tasks.rados.rados.0.smithi063.stdout:2005: done (3 left) 2024-08-17T17:03:27.555 INFO:tasks.rados.rados.0.smithi063.stdout:2006: done (2 left) 2024-08-17T17:03:27.555 INFO:tasks.rados.rados.0.smithi063.stdout:2007: done (1 left) 2024-08-17T17:03:27.556 INFO:tasks.rados.rados.0.smithi063.stdout:2008: done (0 left) 2024-08-17T17:03:27.556 INFO:tasks.rados.rados.0.smithi063.stdout:2009: snap_remove snap 245 2024-08-17T17:03:27.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:27 smithi162 ceph-mon[104668]: pgmap v324: 89 pgs: 89 active+clean; 170 MiB data, 672 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:27 smithi162 ceph-mon[104668]: osdmap e579: 8 total, 8 up, 8 in 2024-08-17T17:03:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:27 smithi162 ceph-mon[104668]: osdmap e580: 8 total, 8 up, 8 in 2024-08-17T17:03:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:27 smithi063 ceph-mon[114756]: pgmap v324: 89 pgs: 89 active+clean; 170 MiB data, 672 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:27 smithi063 ceph-mon[114756]: osdmap e579: 8 total, 8 up, 8 in 2024-08-17T17:03:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:27 smithi063 ceph-mon[114756]: osdmap e580: 8 total, 8 up, 8 in 2024-08-17T17:03:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:27 smithi063 ceph-mon[109843]: pgmap v324: 89 pgs: 89 active+clean; 170 MiB data, 672 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:27 smithi063 ceph-mon[109843]: osdmap e579: 8 total, 8 up, 8 in 2024-08-17T17:03:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:27 smithi063 ceph-mon[109843]: osdmap e580: 8 total, 8 up, 8 in 2024-08-17T17:03:28.558 INFO:tasks.rados.rados.0.smithi063.stdout:2009: done (0 left) 2024-08-17T17:03:28.559 INFO:tasks.rados.rados.0.smithi063.stdout:2010: delete oid 10 current snap is 258 2024-08-17T17:03:28.563 INFO:tasks.rados.rados.0.smithi063.stdout:2010: done (0 left) 2024-08-17T17:03:28.563 INFO:tasks.rados.rados.0.smithi063.stdout:2011: read oid 18 snap -1 2024-08-17T17:03:28.563 INFO:tasks.rados.rados.0.smithi063.stdout:2011: expect (ObjNum 736 snap 250 seq_num 736) 2024-08-17T17:03:28.563 INFO:tasks.rados.rados.0.smithi063.stdout:2012: read oid 28 snap -1 2024-08-17T17:03:28.563 INFO:tasks.rados.rados.0.smithi063.stdout:2012: expect deleted 2024-08-17T17:03:28.563 INFO:tasks.rados.rados.0.smithi063.stdout:2013: snap_remove snap 233 2024-08-17T17:03:29.569 INFO:tasks.rados.rados.0.smithi063.stdout:2013: done (2 left) 2024-08-17T17:03:29.569 INFO:tasks.rados.rados.0.smithi063.stdout:2014: read oid 10 snap -1 2024-08-17T17:03:29.569 INFO:tasks.rados.rados.0.smithi063.stdout:2014: expect deleted 2024-08-17T17:03:29.569 INFO:tasks.rados.rados.0.smithi063.stdout:2015: write oid 6 current snap is 258 2024-08-17T17:03:29.569 INFO:tasks.rados.rados.0.smithi063.stdout:2015: seq_num 761 ranges {790127=499789,1698444=730935} 2024-08-17T17:03:29.577 INFO:tasks.rados.rados.0.smithi063.stdout:2015: writing smithi06387500-6 from 790127 to 1289916 tid 1 2024-08-17T17:03:29.585 INFO:tasks.rados.rados.0.smithi063.stdout:2015: writing smithi06387500-6 from 1698444 to 2429379 tid 2 2024-08-17T17:03:29.585 INFO:tasks.rados.rados.0.smithi063.stdout:2016: read oid 45 snap -1 2024-08-17T17:03:29.585 INFO:tasks.rados.rados.0.smithi063.stdout:2016: expect (ObjNum 743 snap 253 seq_num 743) 2024-08-17T17:03:29.585 INFO:tasks.rados.rados.0.smithi063.stdout:2017: read oid 49 snap -1 2024-08-17T17:03:29.585 INFO:tasks.rados.rados.0.smithi063.stdout:2017: expect (ObjNum 655 snap 209 seq_num 655) 2024-08-17T17:03:29.585 INFO:tasks.rados.rados.0.smithi063.stdout:2018: snap_create 2024-08-17T17:03:29.585 INFO:tasks.rados.rados.0.smithi063.stdout:2011: expect (ObjNum 736 snap 250 seq_num 736) 2024-08-17T17:03:29.670 INFO:tasks.rados.rados.0.smithi063.stdout:2015: finishing write tid 1 to smithi06387500-6 2024-08-17T17:03:29.670 INFO:tasks.rados.rados.0.smithi063.stdout:2015: finishing write tid 2 to smithi06387500-6 2024-08-17T17:03:29.670 INFO:tasks.rados.rados.0.smithi063.stdout:2015: finishing write tid 3 to smithi06387500-6 2024-08-17T17:03:29.670 INFO:tasks.rados.rados.0.smithi063.stdout:2015: finishing write tid 4 to smithi06387500-6 2024-08-17T17:03:29.670 INFO:tasks.rados.rados.0.smithi063.stdout:2015: oid 6 updating version 0 to 587 2024-08-17T17:03:29.671 INFO:tasks.rados.rados.0.smithi063.stdout:2015: oid 6 updating version 587 to 588 2024-08-17T17:03:29.671 INFO:tasks.rados.rados.0.smithi063.stdout:2015: oid 6 version 588 is already newer than 586 2024-08-17T17:03:29.671 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 588 (ObjNum 761 snap 258 seq_num 761) dirty exists 2024-08-17T17:03:29.671 INFO:tasks.rados.rados.0.smithi063.stdout:2015: left oid 6 (ObjNum 761 snap 258 seq_num 761) 2024-08-17T17:03:29.671 INFO:tasks.rados.rados.0.smithi063.stdout:2017: expect (ObjNum 655 snap 209 seq_num 655) 2024-08-17T17:03:29.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:29 smithi162 ceph-mon[104668]: pgmap v327: 89 pgs: 89 active+clean; 176 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-08-17T17:03:29.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:29 smithi162 ceph-mon[104668]: osdmap e581: 8 total, 8 up, 8 in 2024-08-17T17:03:29.885 INFO:tasks.rados.rados.0.smithi063.stdout:2016: expect (ObjNum 743 snap 253 seq_num 743) 2024-08-17T17:03:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:29 smithi063 ceph-mon[114756]: pgmap v327: 89 pgs: 89 active+clean; 176 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-08-17T17:03:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:29 smithi063 ceph-mon[114756]: osdmap e581: 8 total, 8 up, 8 in 2024-08-17T17:03:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:29 smithi063 ceph-mon[109843]: pgmap v327: 89 pgs: 89 active+clean; 176 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-08-17T17:03:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:29 smithi063 ceph-mon[109843]: osdmap e581: 8 total, 8 up, 8 in 2024-08-17T17:03:30.572 INFO:tasks.rados.rados.0.smithi063.stdout:2011: done (6 left) 2024-08-17T17:03:30.572 INFO:tasks.rados.rados.0.smithi063.stdout:2012: done (5 left) 2024-08-17T17:03:30.572 INFO:tasks.rados.rados.0.smithi063.stdout:2014: done (4 left) 2024-08-17T17:03:30.572 INFO:tasks.rados.rados.0.smithi063.stdout:2015: done (3 left) 2024-08-17T17:03:30.572 INFO:tasks.rados.rados.0.smithi063.stdout:2016: done (2 left) 2024-08-17T17:03:30.572 INFO:tasks.rados.rados.0.smithi063.stdout:2017: done (1 left) 2024-08-17T17:03:30.572 INFO:tasks.rados.rados.0.smithi063.stdout:2018: done (0 left) 2024-08-17T17:03:30.572 INFO:tasks.rados.rados.0.smithi063.stdout:2019: rollback oid 21 current snap is 259 2024-08-17T17:03:30.573 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 21 to 238 2024-08-17T17:03:30.573 INFO:tasks.rados.rados.0.smithi063.stdout:2020: snap_create 2024-08-17T17:03:30.576 INFO:tasks.rados.rados.0.smithi063.stdout:2019: finishing rollback tid 1 to smithi06387500-21 2024-08-17T17:03:30.577 INFO:tasks.rados.rados.0.smithi063.stdout:2019: finishing rollback tid 2 to smithi06387500-21 2024-08-17T17:03:30.577 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 1335 (ObjNum 709 snap 234 seq_num 709) dirty exists 2024-08-17T17:03:30.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:30 smithi162 ceph-mon[104668]: osdmap e582: 8 total, 8 up, 8 in 2024-08-17T17:03:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:30 smithi063 ceph-mon[114756]: osdmap e582: 8 total, 8 up, 8 in 2024-08-17T17:03:31.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:30 smithi063 ceph-mon[109843]: osdmap e582: 8 total, 8 up, 8 in 2024-08-17T17:03:31.578 INFO:tasks.rados.rados.0.smithi063.stdout:2019: done (1 left) 2024-08-17T17:03:31.578 INFO:tasks.rados.rados.0.smithi063.stdout:2020: done (0 left) 2024-08-17T17:03:31.578 INFO:tasks.rados.rados.0.smithi063.stdout:2021: read oid 43 snap -1 2024-08-17T17:03:31.578 INFO:tasks.rados.rados.0.smithi063.stdout:2021: expect (ObjNum 735 snap 247 seq_num 735) 2024-08-17T17:03:31.579 INFO:tasks.rados.rados.0.smithi063.stdout:2022: write oid 8 current snap is 260 2024-08-17T17:03:31.579 INFO:tasks.rados.rados.0.smithi063.stdout:2022: seq_num 762 ranges {406934=575038,1640573=474593,2642791=164143} 2024-08-17T17:03:31.583 INFO:tasks.rados.rados.0.smithi063.stdout:2022: writing smithi06387500-8 from 406934 to 981972 tid 1 2024-08-17T17:03:31.590 INFO:tasks.rados.rados.0.smithi063.stdout:2022: writing smithi06387500-8 from 1640573 to 2115166 tid 2 2024-08-17T17:03:31.594 INFO:tasks.rados.rados.0.smithi063.stdout:2022: writing smithi06387500-8 from 2642791 to 2806934 tid 3 2024-08-17T17:03:31.594 INFO:tasks.rados.rados.0.smithi063.stdout:2023: snap_remove snap 238 2024-08-17T17:03:31.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:31 smithi162 ceph-mon[104668]: pgmap v330: 89 pgs: 89 active+clean; 176 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-08-17T17:03:31.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:31 smithi162 ceph-mon[104668]: osdmap e583: 8 total, 8 up, 8 in 2024-08-17T17:03:31.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:31 smithi063 ceph-mon[114756]: pgmap v330: 89 pgs: 89 active+clean; 176 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-08-17T17:03:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:31 smithi063 ceph-mon[114756]: osdmap e583: 8 total, 8 up, 8 in 2024-08-17T17:03:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:31 smithi063 ceph-mon[109843]: pgmap v330: 89 pgs: 89 active+clean; 176 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-08-17T17:03:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:31 smithi063 ceph-mon[109843]: osdmap e583: 8 total, 8 up, 8 in 2024-08-17T17:03:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:32.589 INFO:tasks.rados.rados.0.smithi063.stdout:2023: done (2 left) 2024-08-17T17:03:32.589 INFO:tasks.rados.rados.0.smithi063.stdout:2024: write oid 25 current snap is 260 2024-08-17T17:03:32.589 INFO:tasks.rados.rados.0.smithi063.stdout:2021: expect (ObjNum 735 snap 247 seq_num 735) 2024-08-17T17:03:32.701 INFO:tasks.rados.rados.0.smithi063.stdout:2022: finishing write tid 1 to smithi06387500-8 2024-08-17T17:03:32.701 INFO:tasks.rados.rados.0.smithi063.stdout:2022: finishing write tid 2 to smithi06387500-8 2024-08-17T17:03:32.701 INFO:tasks.rados.rados.0.smithi063.stdout:2022: finishing write tid 3 to smithi06387500-8 2024-08-17T17:03:32.701 INFO:tasks.rados.rados.0.smithi063.stdout:2022: finishing write tid 4 to smithi06387500-8 2024-08-17T17:03:32.701 INFO:tasks.rados.rados.0.smithi063.stdout:2022: finishing write tid 5 to smithi06387500-8 2024-08-17T17:03:32.701 INFO:tasks.rados.rados.0.smithi063.stdout:2022: oid 8 updating version 0 to 591 2024-08-17T17:03:32.701 INFO:tasks.rados.rados.0.smithi063.stdout:2022: oid 8 updating version 591 to 592 2024-08-17T17:03:32.701 INFO:tasks.rados.rados.0.smithi063.stdout:2022: oid 8 updating version 592 to 593 2024-08-17T17:03:32.701 INFO:tasks.rados.rados.0.smithi063.stdout:2022: oid 8 version 593 is already newer than 590 2024-08-17T17:03:32.701 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 593 (ObjNum 762 snap 260 seq_num 762) dirty exists 2024-08-17T17:03:32.701 INFO:tasks.rados.rados.0.smithi063.stdout:2022: left oid 8 (ObjNum 762 snap 260 seq_num 762) 2024-08-17T17:03:32.702 INFO:tasks.rados.rados.0.smithi063.stdout:2024: seq_num 763 ranges {423741=650287,1699055=618251} 2024-08-17T17:03:32.706 INFO:tasks.rados.rados.0.smithi063.stdout:2024: writing smithi06387500-25 from 423741 to 1074028 tid 1 2024-08-17T17:03:32.713 INFO:tasks.rados.rados.0.smithi063.stdout:2024: writing smithi06387500-25 from 1699055 to 2317306 tid 2 2024-08-17T17:03:32.713 INFO:tasks.rados.rados.0.smithi063.stdout:2021: done (2 left) 2024-08-17T17:03:32.713 INFO:tasks.rados.rados.0.smithi063.stdout:2022: done (1 left) 2024-08-17T17:03:32.714 INFO:tasks.rados.rados.0.smithi063.stdout:2025: rollback oid 18 current snap is 260 2024-08-17T17:03:32.714 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 18 to 253 2024-08-17T17:03:32.714 INFO:tasks.rados.rados.0.smithi063.stdout:2024: finishing write tid 1 to smithi06387500-25 2024-08-17T17:03:32.714 INFO:tasks.rados.rados.0.smithi063.stdout:2026: delete oid 29 current snap is 260 2024-08-17T17:03:32.716 INFO:tasks.rados.rados.0.smithi063.stdout:2025: finishing rollback tid 0 to smithi06387500-18 2024-08-17T17:03:32.716 INFO:tasks.rados.rados.0.smithi063.stdout:2025: finishing rollback tid 1 to smithi06387500-18 2024-08-17T17:03:32.716 INFO:tasks.rados.rados.0.smithi063.stdout:2026: done (2 left) 2024-08-17T17:03:32.716 INFO:tasks.rados.rados.0.smithi063.stdout:2027: snap_remove snap 258 2024-08-17T17:03:32.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:32 smithi162 ceph-mon[104668]: osdmap e584: 8 total, 8 up, 8 in 2024-08-17T17:03:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:32 smithi063 ceph-mon[114756]: osdmap e584: 8 total, 8 up, 8 in 2024-08-17T17:03:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:32 smithi063 ceph-mon[109843]: osdmap e584: 8 total, 8 up, 8 in 2024-08-17T17:03:33.582 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:03:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:03:33] "GET /metrics HTTP/1.1" 200 36211 "" "Prometheus/2.51.0" 2024-08-17T17:03:33.592 INFO:tasks.rados.rados.0.smithi063.stdout:2027: done (2 left) 2024-08-17T17:03:33.592 INFO:tasks.rados.rados.0.smithi063.stdout:2028: write oid 19 current snap is 260 2024-08-17T17:03:33.592 INFO:tasks.rados.rados.0.smithi063.stdout:2025: finishing rollback tid 2 to smithi06387500-18 2024-08-17T17:03:33.593 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 642 (ObjNum 736 snap 250 seq_num 736) dirty exists 2024-08-17T17:03:33.593 INFO:tasks.rados.rados.0.smithi063.stdout:2028: seq_num 765 ranges {457355=400785,1416019=621920,2725060=132295} 2024-08-17T17:03:33.597 INFO:tasks.rados.rados.0.smithi063.stdout:2028: writing smithi06387500-19 from 457355 to 858140 tid 1 2024-08-17T17:03:33.604 INFO:tasks.rados.rados.0.smithi063.stdout:2028: writing smithi06387500-19 from 1416019 to 2037939 tid 2 2024-08-17T17:03:33.609 INFO:tasks.rados.rados.0.smithi063.stdout:2028: writing smithi06387500-19 from 2725060 to 2857355 tid 3 2024-08-17T17:03:33.609 INFO:tasks.rados.rados.0.smithi063.stdout:2025: done (2 left) 2024-08-17T17:03:33.609 INFO:tasks.rados.rados.0.smithi063.stdout:2029: rollback oid 38 current snap is 260 2024-08-17T17:03:33.609 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 38 to 240 2024-08-17T17:03:33.609 INFO:tasks.rados.rados.0.smithi063.stdout:2024: finishing write tid 2 to smithi06387500-25 2024-08-17T17:03:33.609 INFO:tasks.rados.rados.0.smithi063.stdout:2024: finishing write tid 3 to smithi06387500-25 2024-08-17T17:03:33.609 INFO:tasks.rados.rados.0.smithi063.stdout:2024: finishing write tid 4 to smithi06387500-25 2024-08-17T17:03:33.610 INFO:tasks.rados.rados.0.smithi063.stdout:2024: oid 25 updating version 0 to 1965 2024-08-17T17:03:33.610 INFO:tasks.rados.rados.0.smithi063.stdout:2024: oid 25 version 1965 is already newer than 1963 2024-08-17T17:03:33.610 INFO:tasks.rados.rados.0.smithi063.stdout:2024: oid 25 version 1965 is already newer than 1964 2024-08-17T17:03:33.610 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 1965 (ObjNum 763 snap 260 seq_num 763) dirty exists 2024-08-17T17:03:33.610 INFO:tasks.rados.rados.0.smithi063.stdout:2024: left oid 25 (ObjNum 763 snap 260 seq_num 763) 2024-08-17T17:03:33.610 INFO:tasks.rados.rados.0.smithi063.stdout:2024: done (2 left) 2024-08-17T17:03:33.610 INFO:tasks.rados.rados.0.smithi063.stdout:2030: read oid 17 snap -1 2024-08-17T17:03:33.610 INFO:tasks.rados.rados.0.smithi063.stdout:2030: expect deleted 2024-08-17T17:03:33.610 INFO:tasks.rados.rados.0.smithi063.stdout:2031: read oid 33 snap -1 2024-08-17T17:03:33.610 INFO:tasks.rados.rados.0.smithi063.stdout:2031: expect (ObjNum 636 snap 203 seq_num 636) 2024-08-17T17:03:33.610 INFO:tasks.rados.rados.0.smithi063.stdout:2032: snap_create 2024-08-17T17:03:33.610 INFO:tasks.rados.rados.0.smithi063.stdout:2028: finishing write tid 1 to smithi06387500-19 2024-08-17T17:03:33.610 INFO:tasks.rados.rados.0.smithi063.stdout:2028: finishing write tid 2 to smithi06387500-19 2024-08-17T17:03:33.613 INFO:tasks.rados.rados.0.smithi063.stdout:2028: finishing write tid 3 to smithi06387500-19 2024-08-17T17:03:33.613 INFO:tasks.rados.rados.0.smithi063.stdout:2028: finishing write tid 4 to smithi06387500-19 2024-08-17T17:03:33.613 INFO:tasks.rados.rados.0.smithi063.stdout:2031: expect (ObjNum 636 snap 203 seq_num 636) 2024-08-17T17:03:33.771 INFO:tasks.rados.rados.0.smithi063.stdout:2028: finishing write tid 5 to smithi06387500-19 2024-08-17T17:03:33.771 INFO:tasks.rados.rados.0.smithi063.stdout:2028: oid 19 updating version 0 to 1343 2024-08-17T17:03:33.771 INFO:tasks.rados.rados.0.smithi063.stdout:2028: oid 19 updating version 1343 to 1344 2024-08-17T17:03:33.771 INFO:tasks.rados.rados.0.smithi063.stdout:2028: oid 19 updating version 1344 to 1345 2024-08-17T17:03:33.771 INFO:tasks.rados.rados.0.smithi063.stdout:2028: oid 19 version 1345 is already newer than 1342 2024-08-17T17:03:33.771 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 1345 (ObjNum 765 snap 260 seq_num 765) dirty exists 2024-08-17T17:03:33.771 INFO:tasks.rados.rados.0.smithi063.stdout:2028: left oid 19 (ObjNum 765 snap 260 seq_num 765) 2024-08-17T17:03:33.771 INFO:tasks.rados.rados.0.smithi063.stdout:2029: finishing rollback tid 0 to smithi06387500-38 2024-08-17T17:03:33.771 INFO:tasks.rados.rados.0.smithi063.stdout:2029: finishing rollback tid 1 to smithi06387500-38 2024-08-17T17:03:33.771 INFO:tasks.rados.rados.0.smithi063.stdout:2029: finishing rollback tid 2 to smithi06387500-38 2024-08-17T17:03:33.771 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 313 (ObjNum 699 snap 228 seq_num 699) dirty exists 2024-08-17T17:03:33.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:33 smithi162 ceph-mon[104668]: pgmap v333: 89 pgs: 89 active+clean; 179 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 597 KiB/s wr, 11 op/s 2024-08-17T17:03:33.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:33 smithi162 ceph-mon[104668]: osdmap e585: 8 total, 8 up, 8 in 2024-08-17T17:03:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:33 smithi063 ceph-mon[114756]: pgmap v333: 89 pgs: 89 active+clean; 179 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 597 KiB/s wr, 11 op/s 2024-08-17T17:03:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:33 smithi063 ceph-mon[114756]: osdmap e585: 8 total, 8 up, 8 in 2024-08-17T17:03:34.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:33 smithi063 ceph-mon[109843]: pgmap v333: 89 pgs: 89 active+clean; 179 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 597 KiB/s wr, 11 op/s 2024-08-17T17:03:34.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:33 smithi063 ceph-mon[109843]: osdmap e585: 8 total, 8 up, 8 in 2024-08-17T17:03:34.605 INFO:tasks.rados.rados.0.smithi063.stdout:2028: done (4 left) 2024-08-17T17:03:34.605 INFO:tasks.rados.rados.0.smithi063.stdout:2029: done (3 left) 2024-08-17T17:03:34.605 INFO:tasks.rados.rados.0.smithi063.stdout:2030: done (2 left) 2024-08-17T17:03:34.605 INFO:tasks.rados.rados.0.smithi063.stdout:2031: done (1 left) 2024-08-17T17:03:34.605 INFO:tasks.rados.rados.0.smithi063.stdout:2032: done (0 left) 2024-08-17T17:03:34.605 INFO:tasks.rados.rados.0.smithi063.stdout:2033: write oid 20 current snap is 261 2024-08-17T17:03:34.606 INFO:tasks.rados.rados.0.smithi063.stdout:2033: seq_num 766 ranges {474162=476034,1358148=481931,2352483=465303} 2024-08-17T17:03:34.610 INFO:tasks.rados.rados.0.smithi063.stdout:2033: writing smithi06387500-20 from 474162 to 950196 tid 1 2024-08-17T17:03:34.615 INFO:tasks.rados.rados.0.smithi063.stdout:2033: writing smithi06387500-20 from 1358148 to 1840079 tid 2 2024-08-17T17:03:34.621 INFO:tasks.rados.rados.0.smithi063.stdout:2033: writing smithi06387500-20 from 2352483 to 2817786 tid 3 2024-08-17T17:03:34.621 INFO:tasks.rados.rados.0.smithi063.stdout:2034: write oid 36 current snap is 261 2024-08-17T17:03:34.621 INFO:tasks.rados.rados.0.smithi063.stdout:2033: finishing write tid 1 to smithi06387500-20 2024-08-17T17:03:34.621 INFO:tasks.rados.rados.0.smithi063.stdout:2034: seq_num 767 ranges {490969=551283,1816630=625589} 2024-08-17T17:03:34.627 INFO:tasks.rados.rados.0.smithi063.stdout:2034: writing smithi06387500-36 from 490969 to 1042252 tid 1 2024-08-17T17:03:34.635 INFO:tasks.rados.rados.0.smithi063.stdout:2034: writing smithi06387500-36 from 1816630 to 2442219 tid 2 2024-08-17T17:03:34.635 INFO:tasks.rados.rados.0.smithi063.stdout:2035: delete oid 2 current snap is 261 2024-08-17T17:03:34.635 INFO:tasks.rados.rados.0.smithi063.stdout:2033: finishing write tid 2 to smithi06387500-20 2024-08-17T17:03:34.635 INFO:tasks.rados.rados.0.smithi063.stdout:2033: finishing write tid 3 to smithi06387500-20 2024-08-17T17:03:34.635 INFO:tasks.rados.rados.0.smithi063.stdout:2033: finishing write tid 4 to smithi06387500-20 2024-08-17T17:03:34.635 INFO:tasks.rados.rados.0.smithi063.stdout:2033: finishing write tid 5 to smithi06387500-20 2024-08-17T17:03:34.636 INFO:tasks.rados.rados.0.smithi063.stdout:2033: oid 20 updating version 0 to 403 2024-08-17T17:03:34.636 INFO:tasks.rados.rados.0.smithi063.stdout:2033: oid 20 version 403 is already newer than 401 2024-08-17T17:03:34.636 INFO:tasks.rados.rados.0.smithi063.stdout:2033: oid 20 version 403 is already newer than 402 2024-08-17T17:03:34.636 INFO:tasks.rados.rados.0.smithi063.stdout:2033: oid 20 version 403 is already newer than 400 2024-08-17T17:03:34.636 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 403 (ObjNum 766 snap 261 seq_num 766) dirty exists 2024-08-17T17:03:34.636 INFO:tasks.rados.rados.0.smithi063.stdout:2033: left oid 20 (ObjNum 766 snap 261 seq_num 766) 2024-08-17T17:03:34.636 INFO:tasks.rados.rados.0.smithi063.stdout:2034: finishing write tid 1 to smithi06387500-36 2024-08-17T17:03:34.640 INFO:tasks.rados.rados.0.smithi063.stdout:2034: finishing write tid 2 to smithi06387500-36 2024-08-17T17:03:34.641 INFO:tasks.rados.rados.0.smithi063.stdout:2034: finishing write tid 3 to smithi06387500-36 2024-08-17T17:03:34.641 INFO:tasks.rados.rados.0.smithi063.stdout:2034: finishing write tid 4 to smithi06387500-36 2024-08-17T17:03:34.641 INFO:tasks.rados.rados.0.smithi063.stdout:2034: oid 36 updating version 0 to 1970 2024-08-17T17:03:34.641 INFO:tasks.rados.rados.0.smithi063.stdout:2034: oid 36 version 1970 is already newer than 1969 2024-08-17T17:03:34.641 INFO:tasks.rados.rados.0.smithi063.stdout:2034: oid 36 updating version 1970 to 1971 2024-08-17T17:03:34.641 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 1971 (ObjNum 767 snap 261 seq_num 767) dirty exists 2024-08-17T17:03:34.641 INFO:tasks.rados.rados.0.smithi063.stdout:2034: left oid 36 (ObjNum 767 snap 261 seq_num 767) 2024-08-17T17:03:34.641 INFO:tasks.rados.rados.0.smithi063.stdout:2033: done (2 left) 2024-08-17T17:03:34.641 INFO:tasks.rados.rados.0.smithi063.stdout:2034: done (1 left) 2024-08-17T17:03:34.641 INFO:tasks.rados.rados.0.smithi063.stdout:2035: done (0 left) 2024-08-17T17:03:34.641 INFO:tasks.rados.rados.0.smithi063.stdout:2036: read oid 11 snap -1 2024-08-17T17:03:34.641 INFO:tasks.rados.rados.0.smithi063.stdout:2036: expect deleted 2024-08-17T17:03:34.642 INFO:tasks.rados.rados.0.smithi063.stdout:2037: snap_create 2024-08-17T17:03:34.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:34 smithi162 ceph-mon[104668]: osdmap e586: 8 total, 8 up, 8 in 2024-08-17T17:03:35.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:34 smithi063 ceph-mon[114756]: osdmap e586: 8 total, 8 up, 8 in 2024-08-17T17:03:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:34 smithi063 ceph-mon[109843]: osdmap e586: 8 total, 8 up, 8 in 2024-08-17T17:03:35.605 INFO:tasks.rados.rados.0.smithi063.stdout:2036: done (1 left) 2024-08-17T17:03:35.605 INFO:tasks.rados.rados.0.smithi063.stdout:2037: done (0 left) 2024-08-17T17:03:35.605 INFO:tasks.rados.rados.0.smithi063.stdout:2038: read oid 22 snap -1 2024-08-17T17:03:35.605 INFO:tasks.rados.rados.0.smithi063.stdout:2038: expect (ObjNum 626 snap 199 seq_num 626) 2024-08-17T17:03:35.605 INFO:tasks.rados.rados.0.smithi063.stdout:2039: delete oid 41 current snap is 262 2024-08-17T17:03:35.610 INFO:tasks.rados.rados.0.smithi063.stdout:2039: done (1 left) 2024-08-17T17:03:35.610 INFO:tasks.rados.rados.0.smithi063.stdout:2040: snap_create 2024-08-17T17:03:35.610 INFO:tasks.rados.rados.0.smithi063.stdout:2038: expect (ObjNum 626 snap 199 seq_num 626) 2024-08-17T17:03:35.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:35 smithi162 ceph-mon[104668]: pgmap v336: 89 pgs: 89 active+clean; 179 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 597 KiB/s wr, 11 op/s 2024-08-17T17:03:35.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:35 smithi162 ceph-mon[104668]: osdmap e587: 8 total, 8 up, 8 in 2024-08-17T17:03:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:35 smithi063 ceph-mon[114756]: pgmap v336: 89 pgs: 89 active+clean; 179 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 597 KiB/s wr, 11 op/s 2024-08-17T17:03:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:35 smithi063 ceph-mon[114756]: osdmap e587: 8 total, 8 up, 8 in 2024-08-17T17:03:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:35 smithi063 ceph-mon[109843]: pgmap v336: 89 pgs: 89 active+clean; 179 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 9.5 MiB/s rd, 597 KiB/s wr, 11 op/s 2024-08-17T17:03:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:35 smithi063 ceph-mon[109843]: osdmap e587: 8 total, 8 up, 8 in 2024-08-17T17:03:36.610 INFO:tasks.rados.rados.0.smithi063.stdout:2038: done (1 left) 2024-08-17T17:03:36.610 INFO:tasks.rados.rados.0.smithi063.stdout:2040: done (0 left) 2024-08-17T17:03:36.610 INFO:tasks.rados.rados.0.smithi063.stdout:2041: read oid 25 snap -1 2024-08-17T17:03:36.610 INFO:tasks.rados.rados.0.smithi063.stdout:2041: expect (ObjNum 763 snap 260 seq_num 763) 2024-08-17T17:03:36.610 INFO:tasks.rados.rados.0.smithi063.stdout:2042: read oid 11 snap -1 2024-08-17T17:03:36.610 INFO:tasks.rados.rados.0.smithi063.stdout:2042: expect deleted 2024-08-17T17:03:36.610 INFO:tasks.rados.rados.0.smithi063.stdout:2043: snap_remove snap 256 2024-08-17T17:03:36.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:36 smithi063 ceph-mon[114756]: osdmap e588: 8 total, 8 up, 8 in 2024-08-17T17:03:36.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:36 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:03:36.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:36 smithi063 ceph-mon[109843]: osdmap e588: 8 total, 8 up, 8 in 2024-08-17T17:03:36.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:36 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:03:36.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:36 smithi162 ceph-mon[104668]: osdmap e588: 8 total, 8 up, 8 in 2024-08-17T17:03:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:36 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:03:37.557 INFO:tasks.rados.rados.0.smithi063.stdout:2043: done (2 left) 2024-08-17T17:03:37.557 INFO:tasks.rados.rados.0.smithi063.stdout:2044: read oid 31 snap -1 2024-08-17T17:03:37.557 INFO:tasks.rados.rados.0.smithi063.stdout:2044: expect deleted 2024-08-17T17:03:37.557 INFO:tasks.rados.rados.0.smithi063.stdout:2045: read oid 21 snap -1 2024-08-17T17:03:37.558 INFO:tasks.rados.rados.0.smithi063.stdout:2045: expect (ObjNum 709 snap 234 seq_num 709) 2024-08-17T17:03:37.558 INFO:tasks.rados.rados.0.smithi063.stdout:2046: read oid 47 snap -1 2024-08-17T17:03:37.558 INFO:tasks.rados.rados.0.smithi063.stdout:2046: expect (ObjNum 662 snap 210 seq_num 662) 2024-08-17T17:03:37.558 INFO:tasks.rados.rados.0.smithi063.stdout:2047: delete oid 42 current snap is 263 2024-08-17T17:03:37.558 INFO:tasks.rados.rados.0.smithi063.stdout:2041: expect (ObjNum 763 snap 260 seq_num 763) 2024-08-17T17:03:37.707 INFO:tasks.rados.rados.0.smithi063.stdout:2041: done (5 left) 2024-08-17T17:03:37.707 INFO:tasks.rados.rados.0.smithi063.stdout:2042: done (4 left) 2024-08-17T17:03:37.707 INFO:tasks.rados.rados.0.smithi063.stdout:2044: done (3 left) 2024-08-17T17:03:37.707 INFO:tasks.rados.rados.0.smithi063.stdout:2047: done (2 left) 2024-08-17T17:03:37.707 INFO:tasks.rados.rados.0.smithi063.stdout:2048: write oid 27 current snap is 263 2024-08-17T17:03:37.707 INFO:tasks.rados.rados.0.smithi063.stdout:2046: expect (ObjNum 662 snap 210 seq_num 662) 2024-08-17T17:03:37.812 INFO:tasks.rados.rados.0.smithi063.stdout:2045: expect (ObjNum 709 snap 234 seq_num 709) 2024-08-17T17:03:37.945 INFO:tasks.rados.rados.0.smithi063.stdout:2048: seq_num 771 ranges {558197=568632,1650558=632927,2773245=184952} 2024-08-17T17:03:37.952 INFO:tasks.rados.rados.0.smithi063.stdout:2048: writing smithi06387500-27 from 558197 to 1126829 tid 1 2024-08-17T17:03:37.958 INFO:tasks.rados.rados.0.smithi063.stdout:2048: writing smithi06387500-27 from 1650558 to 2283485 tid 2 2024-08-17T17:03:37.962 INFO:tasks.rados.rados.0.smithi063.stdout:2048: writing smithi06387500-27 from 2773245 to 2958197 tid 3 2024-08-17T17:03:37.963 INFO:tasks.rados.rados.0.smithi063.stdout:2045: done (2 left) 2024-08-17T17:03:37.963 INFO:tasks.rados.rados.0.smithi063.stdout:2046: done (1 left) 2024-08-17T17:03:37.963 INFO:tasks.rados.rados.0.smithi063.stdout:2049: read oid 2 snap 247 2024-08-17T17:03:37.963 INFO:tasks.rados.rados.0.smithi063.stdout:2049: expect (ObjNum 715 snap 243 seq_num 715) 2024-08-17T17:03:37.963 INFO:tasks.rados.rados.0.smithi063.stdout:2050: snap_create 2024-08-17T17:03:37.963 INFO:tasks.rados.rados.0.smithi063.stdout:2048: finishing write tid 1 to smithi06387500-27 2024-08-17T17:03:37.963 INFO:tasks.rados.rados.0.smithi063.stdout:2048: finishing write tid 2 to smithi06387500-27 2024-08-17T17:03:37.964 INFO:tasks.rados.rados.0.smithi063.stdout:2048: finishing write tid 3 to smithi06387500-27 2024-08-17T17:03:37.964 INFO:tasks.rados.rados.0.smithi063.stdout:2048: finishing write tid 4 to smithi06387500-27 2024-08-17T17:03:37.964 INFO:tasks.rados.rados.0.smithi063.stdout:2048: finishing write tid 5 to smithi06387500-27 2024-08-17T17:03:37.964 INFO:tasks.rados.rados.0.smithi063.stdout:2048: oid 27 updating version 0 to 203 2024-08-17T17:03:37.964 INFO:tasks.rados.rados.0.smithi063.stdout:2048: oid 27 version 203 is already newer than 202 2024-08-17T17:03:37.965 INFO:tasks.rados.rados.0.smithi063.stdout:2048: oid 27 version 203 is already newer than 200 2024-08-17T17:03:37.965 INFO:tasks.rados.rados.0.smithi063.stdout:2048: oid 27 version 203 is already newer than 201 2024-08-17T17:03:37.965 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 203 (ObjNum 771 snap 263 seq_num 771) dirty exists 2024-08-17T17:03:37.965 INFO:tasks.rados.rados.0.smithi063.stdout:2048: left oid 27 (ObjNum 771 snap 263 seq_num 771) 2024-08-17T17:03:37.970 INFO:tasks.rados.rados.0.smithi063.stdout:2049: expect (ObjNum 715 snap 243 seq_num 715) 2024-08-17T17:03:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:37 smithi063 ceph-mon[114756]: pgmap v339: 89 pgs: 89 active+clean; 179 MiB data, 680 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:37 smithi063 ceph-mon[114756]: osdmap e589: 8 total, 8 up, 8 in 2024-08-17T17:03:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:37 smithi063 ceph-mon[114756]: osdmap e590: 8 total, 8 up, 8 in 2024-08-17T17:03:38.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:37 smithi063 ceph-mon[109843]: pgmap v339: 89 pgs: 89 active+clean; 179 MiB data, 680 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:38.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:37 smithi063 ceph-mon[109843]: osdmap e589: 8 total, 8 up, 8 in 2024-08-17T17:03:38.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:37 smithi063 ceph-mon[109843]: osdmap e590: 8 total, 8 up, 8 in 2024-08-17T17:03:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:37 smithi162 ceph-mon[104668]: pgmap v339: 89 pgs: 89 active+clean; 179 MiB data, 680 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:37 smithi162 ceph-mon[104668]: osdmap e589: 8 total, 8 up, 8 in 2024-08-17T17:03:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:37 smithi162 ceph-mon[104668]: osdmap e590: 8 total, 8 up, 8 in 2024-08-17T17:03:38.615 INFO:tasks.rados.rados.0.smithi063.stdout:2048: done (2 left) 2024-08-17T17:03:38.615 INFO:tasks.rados.rados.0.smithi063.stdout:2049: done (1 left) 2024-08-17T17:03:38.615 INFO:tasks.rados.rados.0.smithi063.stdout:2050: done (0 left) 2024-08-17T17:03:38.615 INFO:tasks.rados.rados.0.smithi063.stdout:2051: snap_remove snap 260 2024-08-17T17:03:39.631 INFO:tasks.rados.rados.0.smithi063.stdout:2051: done (0 left) 2024-08-17T17:03:39.631 INFO:tasks.rados.rados.0.smithi063.stdout:2052: rollback oid 10 current snap is 264 2024-08-17T17:03:39.631 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 10 to 257 2024-08-17T17:03:39.631 INFO:tasks.rados.rados.0.smithi063.stdout:2053: read oid 44 snap 220 2024-08-17T17:03:39.631 INFO:tasks.rados.rados.0.smithi063.stdout:2053: expect deleted 2024-08-17T17:03:39.632 INFO:tasks.rados.rados.0.smithi063.stdout:2054: rollback oid 43 current snap is 264 2024-08-17T17:03:39.632 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 43 to 252 2024-08-17T17:03:39.632 INFO:tasks.rados.rados.0.smithi063.stdout:2055: snap_create 2024-08-17T17:03:39.632 INFO:tasks.rados.rados.0.smithi063.stdout:2052: finishing rollback tid 1 to smithi06387500-10 2024-08-17T17:03:39.632 INFO:tasks.rados.rados.0.smithi063.stdout:2052: finishing rollback tid 2 to smithi06387500-10 2024-08-17T17:03:39.632 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 650 (ObjNum 639 snap 203 seq_num 639) dirty exists 2024-08-17T17:03:39.632 INFO:tasks.rados.rados.0.smithi063.stdout:2054: finishing rollback tid 0 to smithi06387500-43 2024-08-17T17:03:39.632 INFO:tasks.rados.rados.0.smithi063.stdout:2054: finishing rollback tid 1 to smithi06387500-43 2024-08-17T17:03:39.632 INFO:tasks.rados.rados.0.smithi063.stdout:2054: finishing rollback tid 2 to smithi06387500-43 2024-08-17T17:03:39.632 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 890 (ObjNum 735 snap 247 seq_num 735) dirty exists 2024-08-17T17:03:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:39 smithi162 ceph-mon[104668]: pgmap v342: 89 pgs: 89 active+clean; 186 MiB data, 698 MiB used, 715 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-08-17T17:03:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:39 smithi162 ceph-mon[104668]: osdmap e591: 8 total, 8 up, 8 in 2024-08-17T17:03:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:39 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:03:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:39 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:03:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:39 smithi063 ceph-mon[109843]: pgmap v342: 89 pgs: 89 active+clean; 186 MiB data, 698 MiB used, 715 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-08-17T17:03:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:39 smithi063 ceph-mon[109843]: osdmap e591: 8 total, 8 up, 8 in 2024-08-17T17:03:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:39 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:03:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:39 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:03:40.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:39 smithi063 ceph-mon[114756]: pgmap v342: 89 pgs: 89 active+clean; 186 MiB data, 698 MiB used, 715 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-08-17T17:03:40.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:39 smithi063 ceph-mon[114756]: osdmap e591: 8 total, 8 up, 8 in 2024-08-17T17:03:40.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:39 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:03:40.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:39 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:03:40.630 INFO:tasks.rados.rados.0.smithi063.stdout:2052: done (3 left) 2024-08-17T17:03:40.630 INFO:tasks.rados.rados.0.smithi063.stdout:2053: done (2 left) 2024-08-17T17:03:40.630 INFO:tasks.rados.rados.0.smithi063.stdout:2054: done (1 left) 2024-08-17T17:03:40.630 INFO:tasks.rados.rados.0.smithi063.stdout:2055: done (0 left) 2024-08-17T17:03:40.630 INFO:tasks.rados.rados.0.smithi063.stdout:2056: snap_remove snap 263 2024-08-17T17:03:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:40 smithi063 ceph-mon[114756]: osdmap e592: 8 total, 8 up, 8 in 2024-08-17T17:03:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:40 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:03:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:40 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:03:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:40 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:03:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:40 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:03:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:40 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:03:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:40 smithi063 ceph-mon[109843]: osdmap e592: 8 total, 8 up, 8 in 2024-08-17T17:03:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:40 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:03:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:40 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:03:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:40 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:03:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:40 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:03:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:40 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:03:41.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:40 smithi162 ceph-mon[104668]: osdmap e592: 8 total, 8 up, 8 in 2024-08-17T17:03:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:40 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:03:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:40 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:03:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:40 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:03:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:40 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:03:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:40 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:03:41.645 INFO:tasks.rados.rados.0.smithi063.stdout:2056: done (0 left) 2024-08-17T17:03:41.646 INFO:tasks.rados.rados.0.smithi063.stdout:2057: write oid 3 current snap is 265 2024-08-17T17:03:41.646 INFO:tasks.rados.rados.0.smithi063.stdout:2057: seq_num 772 ranges {575004=643881,1709040=492938,2917021=57983} 2024-08-17T17:03:41.652 INFO:tasks.rados.rados.0.smithi063.stdout:2057: writing smithi06387500-3 from 575004 to 1218885 tid 1 2024-08-17T17:03:41.658 INFO:tasks.rados.rados.0.smithi063.stdout:2057: writing smithi06387500-3 from 1709040 to 2201978 tid 2 2024-08-17T17:03:41.662 INFO:tasks.rados.rados.0.smithi063.stdout:2057: writing smithi06387500-3 from 2917021 to 2975004 tid 3 2024-08-17T17:03:41.663 INFO:tasks.rados.rados.0.smithi063.stdout:2058: snap_create 2024-08-17T17:03:41.663 INFO:tasks.rados.rados.0.smithi063.stdout:2057: finishing write tid 1 to smithi06387500-3 2024-08-17T17:03:41.663 INFO:tasks.rados.rados.0.smithi063.stdout:2057: finishing write tid 2 to smithi06387500-3 2024-08-17T17:03:41.664 INFO:tasks.rados.rados.0.smithi063.stdout:2057: finishing write tid 3 to smithi06387500-3 2024-08-17T17:03:41.664 INFO:tasks.rados.rados.0.smithi063.stdout:2057: finishing write tid 4 to smithi06387500-3 2024-08-17T17:03:41.665 INFO:tasks.rados.rados.0.smithi063.stdout:2057: finishing write tid 5 to smithi06387500-3 2024-08-17T17:03:41.665 INFO:tasks.rados.rados.0.smithi063.stdout:2057: oid 3 updating version 0 to 604 2024-08-17T17:03:41.665 INFO:tasks.rados.rados.0.smithi063.stdout:2057: oid 3 version 604 is already newer than 603 2024-08-17T17:03:41.665 INFO:tasks.rados.rados.0.smithi063.stdout:2057: oid 3 updating version 604 to 605 2024-08-17T17:03:41.665 INFO:tasks.rados.rados.0.smithi063.stdout:2057: oid 3 version 605 is already newer than 602 2024-08-17T17:03:41.665 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 605 (ObjNum 772 snap 265 seq_num 772) dirty exists 2024-08-17T17:03:41.665 INFO:tasks.rados.rados.0.smithi063.stdout:2057: left oid 3 (ObjNum 772 snap 265 seq_num 772) 2024-08-17T17:03:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:41 smithi162 ceph-mon[104668]: pgmap v345: 89 pgs: 89 active+clean; 186 MiB data, 698 MiB used, 715 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-08-17T17:03:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:41 smithi162 ceph-mon[104668]: osdmap e593: 8 total, 8 up, 8 in 2024-08-17T17:03:41.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:41 smithi063 ceph-mon[114756]: pgmap v345: 89 pgs: 89 active+clean; 186 MiB data, 698 MiB used, 715 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-08-17T17:03:41.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:41 smithi063 ceph-mon[114756]: osdmap e593: 8 total, 8 up, 8 in 2024-08-17T17:03:41.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:41 smithi063 ceph-mon[109843]: pgmap v345: 89 pgs: 89 active+clean; 186 MiB data, 698 MiB used, 715 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-08-17T17:03:41.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:41 smithi063 ceph-mon[109843]: osdmap e593: 8 total, 8 up, 8 in 2024-08-17T17:03:42.651 INFO:tasks.rados.rados.0.smithi063.stdout:2057: done (1 left) 2024-08-17T17:03:42.651 INFO:tasks.rados.rados.0.smithi063.stdout:2058: done (0 left) 2024-08-17T17:03:42.651 INFO:tasks.rados.rados.0.smithi063.stdout:2059: delete oid 20 current snap is 266 2024-08-17T17:03:42.654 INFO:tasks.rados.rados.0.smithi063.stdout:2059: done (0 left) 2024-08-17T17:03:42.654 INFO:tasks.rados.rados.0.smithi063.stdout:2060: rollback oid 21 current snap is 266 2024-08-17T17:03:42.655 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 21 to 220 2024-08-17T17:03:42.655 INFO:tasks.rados.rados.0.smithi063.stdout:2061: snap_remove snap 246 2024-08-17T17:03:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:42 smithi063 ceph-mon[109843]: osdmap e594: 8 total, 8 up, 8 in 2024-08-17T17:03:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:42 smithi063 ceph-mon[109843]: pgmap v348: 89 pgs: 89 active+clean; 191 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 8.6 MiB/s rd, 631 KiB/s wr, 10 op/s 2024-08-17T17:03:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:42 smithi063 ceph-mon[114756]: osdmap e594: 8 total, 8 up, 8 in 2024-08-17T17:03:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:42 smithi063 ceph-mon[114756]: pgmap v348: 89 pgs: 89 active+clean; 191 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 8.6 MiB/s rd, 631 KiB/s wr, 10 op/s 2024-08-17T17:03:43.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:42 smithi162 ceph-mon[104668]: osdmap e594: 8 total, 8 up, 8 in 2024-08-17T17:03:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:42 smithi162 ceph-mon[104668]: pgmap v348: 89 pgs: 89 active+clean; 191 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 8.6 MiB/s rd, 631 KiB/s wr, 10 op/s 2024-08-17T17:03:43.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:03:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:03:43] "GET /metrics HTTP/1.1" 200 36211 "" "Prometheus/2.51.0" 2024-08-17T17:03:43.658 INFO:tasks.rados.rados.0.smithi063.stdout:2061: done (1 left) 2024-08-17T17:03:43.658 INFO:tasks.rados.rados.0.smithi063.stdout:2062: snap_create 2024-08-17T17:03:43.659 INFO:tasks.rados.rados.0.smithi063.stdout:2060: finishing rollback tid 0 to smithi06387500-21 2024-08-17T17:03:43.659 INFO:tasks.rados.rados.0.smithi063.stdout:2060: finishing rollback tid 1 to smithi06387500-21 2024-08-17T17:03:43.659 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 1352 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T17:03:44.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:43 smithi063 ceph-mon[109843]: osdmap e595: 8 total, 8 up, 8 in 2024-08-17T17:03:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:43 smithi063 ceph-mon[114756]: osdmap e595: 8 total, 8 up, 8 in 2024-08-17T17:03:44.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:43 smithi162 ceph-mon[104668]: osdmap e595: 8 total, 8 up, 8 in 2024-08-17T17:03:44.663 INFO:tasks.rados.rados.0.smithi063.stdout:2060: done (1 left) 2024-08-17T17:03:44.663 INFO:tasks.rados.rados.0.smithi063.stdout:2062: done (0 left) 2024-08-17T17:03:44.664 INFO:tasks.rados.rados.0.smithi063.stdout:2063: write oid 8 current snap is 267 2024-08-17T17:03:44.664 INFO:tasks.rados.rados.0.smithi063.stdout:2063: seq_num 774 ranges {608618=794379,2109651=780254} 2024-08-17T17:03:44.671 INFO:tasks.rados.rados.0.smithi063.stdout:2063: writing smithi06387500-8 from 608618 to 1402997 tid 1 2024-08-17T17:03:44.679 INFO:tasks.rados.rados.0.smithi063.stdout:2063: writing smithi06387500-8 from 2109651 to 2889905 tid 2 2024-08-17T17:03:44.679 INFO:tasks.rados.rados.0.smithi063.stdout:2064: snap_create 2024-08-17T17:03:44.679 INFO:tasks.rados.rados.0.smithi063.stdout:2063: finishing write tid 1 to smithi06387500-8 2024-08-17T17:03:44.683 INFO:tasks.rados.rados.0.smithi063.stdout:2063: finishing write tid 2 to smithi06387500-8 2024-08-17T17:03:44.683 INFO:tasks.rados.rados.0.smithi063.stdout:2063: finishing write tid 3 to smithi06387500-8 2024-08-17T17:03:44.683 INFO:tasks.rados.rados.0.smithi063.stdout:2063: finishing write tid 4 to smithi06387500-8 2024-08-17T17:03:44.683 INFO:tasks.rados.rados.0.smithi063.stdout:2063: oid 8 updating version 0 to 606 2024-08-17T17:03:44.683 INFO:tasks.rados.rados.0.smithi063.stdout:2063: oid 8 updating version 606 to 607 2024-08-17T17:03:44.683 INFO:tasks.rados.rados.0.smithi063.stdout:2063: oid 8 version 607 is already newer than 605 2024-08-17T17:03:44.683 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 607 (ObjNum 774 snap 267 seq_num 774) dirty exists 2024-08-17T17:03:44.683 INFO:tasks.rados.rados.0.smithi063.stdout:2063: left oid 8 (ObjNum 774 snap 267 seq_num 774) 2024-08-17T17:03:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:44 smithi063 ceph-mon[114756]: osdmap e596: 8 total, 8 up, 8 in 2024-08-17T17:03:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:44 smithi063 ceph-mon[114756]: pgmap v351: 89 pgs: 89 active+clean; 191 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 8.6 MiB/s rd, 631 KiB/s wr, 10 op/s 2024-08-17T17:03:45.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:44 smithi063 ceph-mon[109843]: osdmap e596: 8 total, 8 up, 8 in 2024-08-17T17:03:45.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:44 smithi063 ceph-mon[109843]: pgmap v351: 89 pgs: 89 active+clean; 191 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 8.6 MiB/s rd, 631 KiB/s wr, 10 op/s 2024-08-17T17:03:45.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:44 smithi162 ceph-mon[104668]: osdmap e596: 8 total, 8 up, 8 in 2024-08-17T17:03:45.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:44 smithi162 ceph-mon[104668]: pgmap v351: 89 pgs: 89 active+clean; 191 MiB data, 706 MiB used, 715 GiB / 715 GiB avail; 8.6 MiB/s rd, 631 KiB/s wr, 10 op/s 2024-08-17T17:03:45.666 INFO:tasks.rados.rados.0.smithi063.stdout:2063: done (1 left) 2024-08-17T17:03:45.666 INFO:tasks.rados.rados.0.smithi063.stdout:2064: done (0 left) 2024-08-17T17:03:45.666 INFO:tasks.rados.rados.0.smithi063.stdout:2065: write oid 10 current snap is 268 2024-08-17T17:03:45.666 INFO:tasks.rados.rados.0.smithi063.stdout:2065: seq_num 775 ranges {625425=469628,1768133=640265} 2024-08-17T17:03:45.672 INFO:tasks.rados.rados.0.smithi063.stdout:2065: writing smithi06387500-10 from 625425 to 1095053 tid 1 2024-08-17T17:03:45.680 INFO:tasks.rados.rados.0.smithi063.stdout:2065: writing smithi06387500-10 from 1768133 to 2408398 tid 2 2024-08-17T17:03:45.680 INFO:tasks.rados.rados.0.smithi063.stdout:2066: delete oid 47 current snap is 268 2024-08-17T17:03:45.680 INFO:tasks.rados.rados.0.smithi063.stdout:2065: finishing write tid 1 to smithi06387500-10 2024-08-17T17:03:45.682 INFO:tasks.rados.rados.0.smithi063.stdout:2066: done (1 left) 2024-08-17T17:03:45.682 INFO:tasks.rados.rados.0.smithi063.stdout:2067: snap_create 2024-08-17T17:03:45.684 INFO:tasks.rados.rados.0.smithi063.stdout:2065: finishing write tid 2 to smithi06387500-10 2024-08-17T17:03:45.684 INFO:tasks.rados.rados.0.smithi063.stdout:2065: finishing write tid 3 to smithi06387500-10 2024-08-17T17:03:45.684 INFO:tasks.rados.rados.0.smithi063.stdout:2065: finishing write tid 4 to smithi06387500-10 2024-08-17T17:03:45.684 INFO:tasks.rados.rados.0.smithi063.stdout:2065: oid 10 updating version 0 to 656 2024-08-17T17:03:45.684 INFO:tasks.rados.rados.0.smithi063.stdout:2065: oid 10 version 656 is already newer than 655 2024-08-17T17:03:45.685 INFO:tasks.rados.rados.0.smithi063.stdout:2065: oid 10 version 656 is already newer than 654 2024-08-17T17:03:45.685 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 656 (ObjNum 775 snap 268 seq_num 775) dirty exists 2024-08-17T17:03:45.685 INFO:tasks.rados.rados.0.smithi063.stdout:2065: left oid 10 (ObjNum 775 snap 268 seq_num 775) 2024-08-17T17:03:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:45 smithi063 ceph-mon[114756]: osdmap e597: 8 total, 8 up, 8 in 2024-08-17T17:03:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:45 smithi063 ceph-mon[109843]: osdmap e597: 8 total, 8 up, 8 in 2024-08-17T17:03:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:45 smithi162 ceph-mon[104668]: osdmap e597: 8 total, 8 up, 8 in 2024-08-17T17:03:46.679 INFO:tasks.rados.rados.0.smithi063.stdout:2065: done (1 left) 2024-08-17T17:03:46.679 INFO:tasks.rados.rados.0.smithi063.stdout:2067: done (0 left) 2024-08-17T17:03:46.679 INFO:tasks.rados.rados.0.smithi063.stdout:2068: rollback oid 33 current snap is 269 2024-08-17T17:03:46.680 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 33 to 268 2024-08-17T17:03:46.680 INFO:tasks.rados.rados.0.smithi063.stdout:2069: read oid 44 snap -1 2024-08-17T17:03:46.680 INFO:tasks.rados.rados.0.smithi063.stdout:2069: expect deleted 2024-08-17T17:03:46.680 INFO:tasks.rados.rados.0.smithi063.stdout:2070: write oid 23 current snap is 269 2024-08-17T17:03:46.680 INFO:tasks.rados.rados.0.smithi063.stdout:2070: seq_num 777 ranges {659039=736479,2001450=643934} 2024-08-17T17:03:46.687 INFO:tasks.rados.rados.0.smithi063.stdout:2070: writing smithi06387500-23 from 659039 to 1395518 tid 1 2024-08-17T17:03:46.694 INFO:tasks.rados.rados.0.smithi063.stdout:2070: writing smithi06387500-23 from 2001450 to 2645384 tid 2 2024-08-17T17:03:46.694 INFO:tasks.rados.rados.0.smithi063.stdout:2071: snap_remove snap 235 2024-08-17T17:03:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:46 smithi063 ceph-mon[109843]: osdmap e598: 8 total, 8 up, 8 in 2024-08-17T17:03:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:46 smithi063 ceph-mon[109843]: pgmap v354: 89 pgs: 89 active+clean; 191 MiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:46 smithi063 ceph-mon[114756]: osdmap e598: 8 total, 8 up, 8 in 2024-08-17T17:03:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:46 smithi063 ceph-mon[114756]: pgmap v354: 89 pgs: 89 active+clean; 191 MiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:47.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:46 smithi162 ceph-mon[104668]: osdmap e598: 8 total, 8 up, 8 in 2024-08-17T17:03:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:46 smithi162 ceph-mon[104668]: pgmap v354: 89 pgs: 89 active+clean; 191 MiB data, 706 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:03:47.561 INFO:tasks.rados.rados.0.smithi063.stdout:2071: done (3 left) 2024-08-17T17:03:47.561 INFO:tasks.rados.rados.0.smithi063.stdout:2072: read oid 7 snap 252 2024-08-17T17:03:47.561 INFO:tasks.rados.rados.0.smithi063.stdout:2072: expect (ObjNum 721 snap 246 seq_num 721) 2024-08-17T17:03:47.562 INFO:tasks.rados.rados.0.smithi063.stdout:2068: finishing rollback tid 0 to smithi06387500-33 2024-08-17T17:03:47.562 INFO:tasks.rados.rados.0.smithi063.stdout:2068: finishing rollback tid 1 to smithi06387500-33 2024-08-17T17:03:47.562 INFO:tasks.rados.rados.0.smithi063.stdout:2068: finishing rollback tid 2 to smithi06387500-33 2024-08-17T17:03:47.562 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 611 (ObjNum 636 snap 203 seq_num 636) dirty exists 2024-08-17T17:03:47.562 INFO:tasks.rados.rados.0.smithi063.stdout:2070: finishing write tid 1 to smithi06387500-23 2024-08-17T17:03:47.562 INFO:tasks.rados.rados.0.smithi063.stdout:2070: finishing write tid 2 to smithi06387500-23 2024-08-17T17:03:47.562 INFO:tasks.rados.rados.0.smithi063.stdout:2070: finishing write tid 3 to smithi06387500-23 2024-08-17T17:03:47.562 INFO:tasks.rados.rados.0.smithi063.stdout:2070: finishing write tid 4 to smithi06387500-23 2024-08-17T17:03:47.562 INFO:tasks.rados.rados.0.smithi063.stdout:2070: oid 23 updating version 0 to 961 2024-08-17T17:03:47.562 INFO:tasks.rados.rados.0.smithi063.stdout:2070: oid 23 version 961 is already newer than 960 2024-08-17T17:03:47.562 INFO:tasks.rados.rados.0.smithi063.stdout:2070: oid 23 version 961 is already newer than 959 2024-08-17T17:03:47.562 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 961 (ObjNum 777 snap 269 seq_num 777) dirty exists 2024-08-17T17:03:47.562 INFO:tasks.rados.rados.0.smithi063.stdout:2070: left oid 23 (ObjNum 777 snap 269 seq_num 777) 2024-08-17T17:03:47.563 INFO:tasks.rados.rados.0.smithi063.stdout:2068: done (3 left) 2024-08-17T17:03:47.563 INFO:tasks.rados.rados.0.smithi063.stdout:2069: done (2 left) 2024-08-17T17:03:47.563 INFO:tasks.rados.rados.0.smithi063.stdout:2070: done (1 left) 2024-08-17T17:03:47.563 INFO:tasks.rados.rados.0.smithi063.stdout:2073: read oid 2 snap -1 2024-08-17T17:03:47.563 INFO:tasks.rados.rados.0.smithi063.stdout:2073: expect deleted 2024-08-17T17:03:47.563 INFO:tasks.rados.rados.0.smithi063.stdout:2074: delete oid 34 current snap is 269 2024-08-17T17:03:47.571 INFO:tasks.rados.rados.0.smithi063.stdout:2072: expect (ObjNum 721 snap 246 seq_num 721) 2024-08-17T17:03:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:2072: done (2 left) 2024-08-17T17:03:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:2073: done (1 left) 2024-08-17T17:03:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:2074: done (0 left) 2024-08-17T17:03:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:2075: snap_remove snap 264 2024-08-17T17:03:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:47 smithi063 ceph-mon[114756]: osdmap e599: 8 total, 8 up, 8 in 2024-08-17T17:03:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:47 smithi063 ceph-mon[114756]: osdmap e600: 8 total, 8 up, 8 in 2024-08-17T17:03:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:47 smithi063 ceph-mon[109843]: osdmap e599: 8 total, 8 up, 8 in 2024-08-17T17:03:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:47 smithi063 ceph-mon[109843]: osdmap e600: 8 total, 8 up, 8 in 2024-08-17T17:03:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:47 smithi162 ceph-mon[104668]: osdmap e599: 8 total, 8 up, 8 in 2024-08-17T17:03:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:47 smithi162 ceph-mon[104668]: osdmap e600: 8 total, 8 up, 8 in 2024-08-17T17:03:48.686 INFO:tasks.rados.rados.0.smithi063.stdout:2075: done (0 left) 2024-08-17T17:03:48.686 INFO:tasks.rados.rados.0.smithi063.stdout:2076: snap_remove snap 261 2024-08-17T17:03:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:48 smithi063 ceph-mon[114756]: pgmap v357: 89 pgs: 89 active+clean; 196 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 993 KiB/s wr, 5 op/s 2024-08-17T17:03:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:48 smithi063 ceph-mon[109843]: pgmap v357: 89 pgs: 89 active+clean; 196 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 993 KiB/s wr, 5 op/s 2024-08-17T17:03:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:48 smithi162 ceph-mon[104668]: pgmap v357: 89 pgs: 89 active+clean; 196 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 993 KiB/s wr, 5 op/s 2024-08-17T17:03:49.686 INFO:tasks.rados.rados.0.smithi063.stdout:2076: done (0 left) 2024-08-17T17:03:49.686 INFO:tasks.rados.rados.0.smithi063.stdout:2077: write oid 20 current snap is 269 2024-08-17T17:03:49.686 INFO:tasks.rados.rados.0.smithi063.stdout:2077: seq_num 779 ranges {692653=486977,1602061=647603,2992629=100024} 2024-08-17T17:03:49.693 INFO:tasks.rados.rados.0.smithi063.stdout:2077: writing smithi06387500-20 from 692653 to 1179630 tid 1 2024-08-17T17:03:49.699 INFO:tasks.rados.rados.0.smithi063.stdout:2077: writing smithi06387500-20 from 1602061 to 2249664 tid 2 2024-08-17T17:03:49.704 INFO:tasks.rados.rados.0.smithi063.stdout:2077: writing smithi06387500-20 from 2992629 to 3092653 tid 3 2024-08-17T17:03:49.704 INFO:tasks.rados.rados.0.smithi063.stdout:2078: snap_create 2024-08-17T17:03:49.704 INFO:tasks.rados.rados.0.smithi063.stdout:2077: finishing write tid 1 to smithi06387500-20 2024-08-17T17:03:49.704 INFO:tasks.rados.rados.0.smithi063.stdout:2077: finishing write tid 2 to smithi06387500-20 2024-08-17T17:03:49.706 INFO:tasks.rados.rados.0.smithi063.stdout:2077: finishing write tid 3 to smithi06387500-20 2024-08-17T17:03:49.706 INFO:tasks.rados.rados.0.smithi063.stdout:2077: finishing write tid 4 to smithi06387500-20 2024-08-17T17:03:49.707 INFO:tasks.rados.rados.0.smithi063.stdout:2077: finishing write tid 5 to smithi06387500-20 2024-08-17T17:03:49.707 INFO:tasks.rados.rados.0.smithi063.stdout:2077: oid 20 updating version 0 to 415 2024-08-17T17:03:49.707 INFO:tasks.rados.rados.0.smithi063.stdout:2077: oid 20 updating version 415 to 416 2024-08-17T17:03:49.707 INFO:tasks.rados.rados.0.smithi063.stdout:2077: oid 20 updating version 416 to 417 2024-08-17T17:03:49.707 INFO:tasks.rados.rados.0.smithi063.stdout:2077: oid 20 version 417 is already newer than 412 2024-08-17T17:03:49.707 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 417 (ObjNum 779 snap 269 seq_num 779) dirty exists 2024-08-17T17:03:49.707 INFO:tasks.rados.rados.0.smithi063.stdout:2077: left oid 20 (ObjNum 779 snap 269 seq_num 779) 2024-08-17T17:03:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:49 smithi063 ceph-mon[114756]: osdmap e601: 8 total, 8 up, 8 in 2024-08-17T17:03:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:49 smithi063 ceph-mon[109843]: osdmap e601: 8 total, 8 up, 8 in 2024-08-17T17:03:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:49 smithi162 ceph-mon[104668]: osdmap e601: 8 total, 8 up, 8 in 2024-08-17T17:03:50.699 INFO:tasks.rados.rados.0.smithi063.stdout:2077: done (1 left) 2024-08-17T17:03:50.699 INFO:tasks.rados.rados.0.smithi063.stdout:2078: done (0 left) 2024-08-17T17:03:50.699 INFO:tasks.rados.rados.0.smithi063.stdout:2079: snap_remove snap 220 2024-08-17T17:03:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:50 smithi063 ceph-mon[114756]: osdmap e602: 8 total, 8 up, 8 in 2024-08-17T17:03:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:50 smithi063 ceph-mon[114756]: pgmap v360: 89 pgs: 89 active+clean; 196 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 993 KiB/s wr, 5 op/s 2024-08-17T17:03:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:50 smithi063 ceph-mon[109843]: osdmap e602: 8 total, 8 up, 8 in 2024-08-17T17:03:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:50 smithi063 ceph-mon[109843]: pgmap v360: 89 pgs: 89 active+clean; 196 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 993 KiB/s wr, 5 op/s 2024-08-17T17:03:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:50 smithi162 ceph-mon[104668]: osdmap e602: 8 total, 8 up, 8 in 2024-08-17T17:03:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:50 smithi162 ceph-mon[104668]: pgmap v360: 89 pgs: 89 active+clean; 196 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 993 KiB/s wr, 5 op/s 2024-08-17T17:03:51.700 INFO:tasks.rados.rados.0.smithi063.stdout:2079: done (0 left) 2024-08-17T17:03:51.700 INFO:tasks.rados.rados.0.smithi063.stdout:2080: write oid 8 current snap is 270 2024-08-17T17:03:51.700 INFO:tasks.rados.rados.0.smithi063.stdout:2080: seq_num 780 ranges {709460=562226,2060543=791261} 2024-08-17T17:03:51.708 INFO:tasks.rados.rados.0.smithi063.stdout:2080: writing smithi06387500-8 from 709460 to 1271686 tid 1 2024-08-17T17:03:51.718 INFO:tasks.rados.rados.0.smithi063.stdout:2080: writing smithi06387500-8 from 2060543 to 2851804 tid 2 2024-08-17T17:03:51.718 INFO:tasks.rados.rados.0.smithi063.stdout:2081: delete oid 15 current snap is 270 2024-08-17T17:03:51.718 INFO:tasks.rados.rados.0.smithi063.stdout:2080: finishing write tid 1 to smithi06387500-8 2024-08-17T17:03:51.720 INFO:tasks.rados.rados.0.smithi063.stdout:2081: done (1 left) 2024-08-17T17:03:51.720 INFO:tasks.rados.rados.0.smithi063.stdout:2082: delete oid 34 current snap is 270 2024-08-17T17:03:51.721 INFO:tasks.rados.rados.0.smithi063.stdout:2080: finishing write tid 2 to smithi06387500-8 2024-08-17T17:03:51.722 INFO:tasks.rados.rados.0.smithi063.stdout:2080: finishing write tid 3 to smithi06387500-8 2024-08-17T17:03:51.722 INFO:tasks.rados.rados.0.smithi063.stdout:2080: finishing write tid 4 to smithi06387500-8 2024-08-17T17:03:51.722 INFO:tasks.rados.rados.0.smithi063.stdout:2080: oid 8 updating version 0 to 630 2024-08-17T17:03:51.722 INFO:tasks.rados.rados.0.smithi063.stdout:2080: oid 8 updating version 630 to 631 2024-08-17T17:03:51.722 INFO:tasks.rados.rados.0.smithi063.stdout:2080: oid 8 version 631 is already newer than 629 2024-08-17T17:03:51.722 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 631 (ObjNum 780 snap 270 seq_num 780) dirty exists 2024-08-17T17:03:51.722 INFO:tasks.rados.rados.0.smithi063.stdout:2080: left oid 8 (ObjNum 780 snap 270 seq_num 780) 2024-08-17T17:03:51.723 INFO:tasks.rados.rados.0.smithi063.stdout:2080: done (1 left) 2024-08-17T17:03:51.723 INFO:tasks.rados.rados.0.smithi063.stdout:2082: done (0 left) 2024-08-17T17:03:51.723 INFO:tasks.rados.rados.0.smithi063.stdout:2083: write oid 3 current snap is 270 2024-08-17T17:03:51.723 INFO:tasks.rados.rados.0.smithi063.stdout:2083: seq_num 783 ranges {759881=787973,2119636=654941} 2024-08-17T17:03:51.731 INFO:tasks.rados.rados.0.smithi063.stdout:2083: writing smithi06387500-3 from 759881 to 1547854 tid 1 2024-08-17T17:03:51.738 INFO:tasks.rados.rados.0.smithi063.stdout:2083: writing smithi06387500-3 from 2119636 to 2774577 tid 2 2024-08-17T17:03:51.738 INFO:tasks.rados.rados.0.smithi063.stdout:2084: snap_create 2024-08-17T17:03:51.738 INFO:tasks.rados.rados.0.smithi063.stdout:2083: finishing write tid 1 to smithi06387500-3 2024-08-17T17:03:51.742 INFO:tasks.rados.rados.0.smithi063.stdout:2083: finishing write tid 2 to smithi06387500-3 2024-08-17T17:03:51.742 INFO:tasks.rados.rados.0.smithi063.stdout:2083: finishing write tid 3 to smithi06387500-3 2024-08-17T17:03:51.742 INFO:tasks.rados.rados.0.smithi063.stdout:2083: finishing write tid 4 to smithi06387500-3 2024-08-17T17:03:51.742 INFO:tasks.rados.rados.0.smithi063.stdout:2083: oid 3 updating version 0 to 621 2024-08-17T17:03:51.742 INFO:tasks.rados.rados.0.smithi063.stdout:2083: oid 3 updating version 621 to 623 2024-08-17T17:03:51.742 INFO:tasks.rados.rados.0.smithi063.stdout:2083: oid 3 version 623 is already newer than 622 2024-08-17T17:03:51.742 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 623 (ObjNum 783 snap 270 seq_num 783) dirty exists 2024-08-17T17:03:51.742 INFO:tasks.rados.rados.0.smithi063.stdout:2083: left oid 3 (ObjNum 783 snap 270 seq_num 783) 2024-08-17T17:03:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:51 smithi063 ceph-mon[109843]: osdmap e603: 8 total, 8 up, 8 in 2024-08-17T17:03:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:51 smithi063 ceph-mon[114756]: osdmap e603: 8 total, 8 up, 8 in 2024-08-17T17:03:52.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:51 smithi162 ceph-mon[104668]: osdmap e603: 8 total, 8 up, 8 in 2024-08-17T17:03:52.714 INFO:tasks.rados.rados.0.smithi063.stdout:2083: done (1 left) 2024-08-17T17:03:52.714 INFO:tasks.rados.rados.0.smithi063.stdout:2084: done (0 left) 2024-08-17T17:03:52.715 INFO:tasks.rados.rados.0.smithi063.stdout:2085: write oid 18 current snap is 271 2024-08-17T17:03:52.715 INFO:tasks.rados.rados.0.smithi063.stdout:2085: seq_num 784 ranges {776688=579575,1894471=798599} 2024-08-17T17:03:52.721 INFO:tasks.rados.rados.0.smithi063.stdout:2085: writing smithi06387500-18 from 776688 to 1356263 tid 1 2024-08-17T17:03:52.729 INFO:tasks.rados.rados.0.smithi063.stdout:2085: writing smithi06387500-18 from 1894471 to 2693070 tid 2 2024-08-17T17:03:52.729 INFO:tasks.rados.rados.0.smithi063.stdout:2086: snap_remove snap 265 2024-08-17T17:03:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:52 smithi063 ceph-mon[109843]: osdmap e604: 8 total, 8 up, 8 in 2024-08-17T17:03:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:52 smithi063 ceph-mon[109843]: pgmap v363: 89 pgs: 89 active+clean; 184 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 968 KiB/s rd, 985 KiB/s wr, 5 op/s 2024-08-17T17:03:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:52 smithi063 ceph-mon[114756]: osdmap e604: 8 total, 8 up, 8 in 2024-08-17T17:03:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:52 smithi063 ceph-mon[114756]: pgmap v363: 89 pgs: 89 active+clean; 184 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 968 KiB/s rd, 985 KiB/s wr, 5 op/s 2024-08-17T17:03:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:52 smithi162 ceph-mon[104668]: osdmap e604: 8 total, 8 up, 8 in 2024-08-17T17:03:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:52 smithi162 ceph-mon[104668]: pgmap v363: 89 pgs: 89 active+clean; 184 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 968 KiB/s rd, 985 KiB/s wr, 5 op/s 2024-08-17T17:03:53.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:03:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:03:53] "GET /metrics HTTP/1.1" 200 36145 "" "Prometheus/2.51.0" 2024-08-17T17:03:53.718 INFO:tasks.rados.rados.0.smithi063.stdout:2086: done (1 left) 2024-08-17T17:03:53.718 INFO:tasks.rados.rados.0.smithi063.stdout:2087: delete oid 38 current snap is 271 2024-08-17T17:03:53.718 INFO:tasks.rados.rados.0.smithi063.stdout:2085: finishing write tid 1 to smithi06387500-18 2024-08-17T17:03:53.718 INFO:tasks.rados.rados.0.smithi063.stdout:2085: finishing write tid 2 to smithi06387500-18 2024-08-17T17:03:53.719 INFO:tasks.rados.rados.0.smithi063.stdout:2085: finishing write tid 3 to smithi06387500-18 2024-08-17T17:03:53.719 INFO:tasks.rados.rados.0.smithi063.stdout:2085: finishing write tid 4 to smithi06387500-18 2024-08-17T17:03:53.719 INFO:tasks.rados.rados.0.smithi063.stdout:2085: oid 18 updating version 0 to 667 2024-08-17T17:03:53.719 INFO:tasks.rados.rados.0.smithi063.stdout:2085: oid 18 version 667 is already newer than 666 2024-08-17T17:03:53.719 INFO:tasks.rados.rados.0.smithi063.stdout:2085: oid 18 updating version 667 to 668 2024-08-17T17:03:53.719 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 668 (ObjNum 784 snap 271 seq_num 784) dirty exists 2024-08-17T17:03:53.719 INFO:tasks.rados.rados.0.smithi063.stdout:2085: left oid 18 (ObjNum 784 snap 271 seq_num 784) 2024-08-17T17:03:53.726 INFO:tasks.rados.rados.0.smithi063.stdout:2085: done (1 left) 2024-08-17T17:03:53.726 INFO:tasks.rados.rados.0.smithi063.stdout:2087: done (0 left) 2024-08-17T17:03:53.726 INFO:tasks.rados.rados.0.smithi063.stdout:2088: read oid 43 snap -1 2024-08-17T17:03:53.726 INFO:tasks.rados.rados.0.smithi063.stdout:2088: expect (ObjNum 735 snap 247 seq_num 735) 2024-08-17T17:03:53.726 INFO:tasks.rados.rados.0.smithi063.stdout:2089: snap_remove snap 249 2024-08-17T17:03:54.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:53 smithi063 ceph-mon[109843]: osdmap e605: 8 total, 8 up, 8 in 2024-08-17T17:03:54.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:53 smithi063 ceph-mon[114756]: osdmap e605: 8 total, 8 up, 8 in 2024-08-17T17:03:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:53 smithi162 ceph-mon[104668]: osdmap e605: 8 total, 8 up, 8 in 2024-08-17T17:03:54.727 INFO:tasks.rados.rados.0.smithi063.stdout:2088: expect (ObjNum 735 snap 247 seq_num 735) 2024-08-17T17:03:54.836 INFO:tasks.rados.rados.0.smithi063.stdout:2088: done (1 left) 2024-08-17T17:03:54.836 INFO:tasks.rados.rados.0.smithi063.stdout:2089: done (0 left) 2024-08-17T17:03:54.836 INFO:tasks.rados.rados.0.smithi063.stdout:2090: rollback oid 48 current snap is 271 2024-08-17T17:03:54.836 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 48 to 259 2024-08-17T17:03:54.836 INFO:tasks.rados.rados.0.smithi063.stdout:2091: snap_create 2024-08-17T17:03:54.840 INFO:tasks.rados.rados.0.smithi063.stdout:2090: finishing rollback tid 0 to smithi06387500-48 2024-08-17T17:03:54.840 INFO:tasks.rados.rados.0.smithi063.stdout:2090: finishing rollback tid 1 to smithi06387500-48 2024-08-17T17:03:54.840 INFO:tasks.rados.rados.0.smithi063.stdout:2090: finishing rollback tid 2 to smithi06387500-48 2024-08-17T17:03:54.840 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 2007 (ObjNum 730 snap 246 seq_num 730) dirty exists 2024-08-17T17:03:54.918 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:54 smithi063 ceph-mon[114756]: osdmap e606: 8 total, 8 up, 8 in 2024-08-17T17:03:54.918 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:54 smithi063 ceph-mon[114756]: pgmap v366: 89 pgs: 89 active+clean; 184 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 968 KiB/s rd, 985 KiB/s wr, 5 op/s 2024-08-17T17:03:54.918 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:54 smithi063 ceph-mon[109843]: osdmap e606: 8 total, 8 up, 8 in 2024-08-17T17:03:54.918 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:54 smithi063 ceph-mon[109843]: pgmap v366: 89 pgs: 89 active+clean; 184 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 968 KiB/s rd, 985 KiB/s wr, 5 op/s 2024-08-17T17:03:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:54 smithi162 ceph-mon[104668]: osdmap e606: 8 total, 8 up, 8 in 2024-08-17T17:03:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:54 smithi162 ceph-mon[104668]: pgmap v366: 89 pgs: 89 active+clean; 184 MiB data, 722 MiB used, 715 GiB / 715 GiB avail; 968 KiB/s rd, 985 KiB/s wr, 5 op/s 2024-08-17T17:03:55.324 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for all mons to upgrade' 2024-08-17T17:03:55.540 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:03:55.733 INFO:tasks.rados.rados.0.smithi063.stdout:2090: done (1 left) 2024-08-17T17:03:55.733 INFO:tasks.rados.rados.0.smithi063.stdout:2091: done (0 left) 2024-08-17T17:03:55.733 INFO:tasks.rados.rados.0.smithi063.stdout:2092: read oid 24 snap -1 2024-08-17T17:03:55.733 INFO:tasks.rados.rados.0.smithi063.stdout:2092: expect (ObjNum 739 snap 251 seq_num 739) 2024-08-17T17:03:55.733 INFO:tasks.rados.rados.0.smithi063.stdout:2093: read oid 23 snap -1 2024-08-17T17:03:55.733 INFO:tasks.rados.rados.0.smithi063.stdout:2093: expect (ObjNum 777 snap 269 seq_num 777) 2024-08-17T17:03:55.733 INFO:tasks.rados.rados.0.smithi063.stdout:2094: delete oid 46 current snap is 272 2024-08-17T17:03:55.738 INFO:tasks.rados.rados.0.smithi063.stdout:2092: expect (ObjNum 739 snap 251 seq_num 739) 2024-08-17T17:03:55.871 INFO:tasks.rados.rados.0.smithi063.stdout:2093: expect (ObjNum 777 snap 269 seq_num 777) 2024-08-17T17:03:55.914 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:55 smithi063 ceph-mon[114756]: osdmap e607: 8 total, 8 up, 8 in 2024-08-17T17:03:55.914 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:55 smithi063 ceph-mon[109843]: osdmap e607: 8 total, 8 up, 8 in 2024-08-17T17:03:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:55 smithi162 ceph-mon[104668]: osdmap e607: 8 total, 8 up, 8 in 2024-08-17T17:03:56.137 INFO:tasks.rados.rados.0.smithi063.stdout:2092: done (2 left) 2024-08-17T17:03:56.137 INFO:tasks.rados.rados.0.smithi063.stdout:2093: done (1 left) 2024-08-17T17:03:56.137 INFO:tasks.rados.rados.0.smithi063.stdout:2094: done (0 left) 2024-08-17T17:03:56.137 INFO:tasks.rados.rados.0.smithi063.stdout:2095: write oid 24 current snap is 272 2024-08-17T17:03:56.137 INFO:tasks.rados.rados.0.smithi063.stdout:2095: seq_num 787 ranges {427109=405322,1553564=545926,2579307=647802} 2024-08-17T17:03:56.141 INFO:tasks.rados.rados.0.smithi063.stdout:2095: writing smithi06387500-24 from 427109 to 832431 tid 1 2024-08-17T17:03:56.148 INFO:tasks.rados.rados.0.smithi063.stdout:2095: writing smithi06387500-24 from 1553564 to 2099490 tid 2 2024-08-17T17:03:56.154 INFO:tasks.rados.rados.0.smithi063.stdout:2095: writing smithi06387500-24 from 2579307 to 3227109 tid 3 2024-08-17T17:03:56.155 INFO:tasks.rados.rados.0.smithi063.stdout:2096: snap_remove snap 266 2024-08-17T17:03:56.742 INFO:tasks.rados.rados.0.smithi063.stdout:2096: done (1 left) 2024-08-17T17:03:56.742 INFO:tasks.rados.rados.0.smithi063.stdout:2097: snap_remove snap 262 2024-08-17T17:03:56.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:56 smithi063 ceph-mon[114756]: osdmap e608: 8 total, 8 up, 8 in 2024-08-17T17:03:56.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:56 smithi063 ceph-mon[114756]: pgmap v369: 89 pgs: 89 active+clean; 184 MiB data, 722 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:56.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:56 smithi063 ceph-mon[109843]: osdmap e608: 8 total, 8 up, 8 in 2024-08-17T17:03:56.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:56 smithi063 ceph-mon[109843]: pgmap v369: 89 pgs: 89 active+clean; 184 MiB data, 722 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:57.029 INFO:teuthology.orchestra.run.smithi063.stdout:wait for all mons to upgrade 2024-08-17T17:03:57.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:56 smithi162 ceph-mon[104668]: osdmap e608: 8 total, 8 up, 8 in 2024-08-17T17:03:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:56 smithi162 ceph-mon[104668]: pgmap v369: 89 pgs: 89 active+clean; 184 MiB data, 722 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:03:57.556 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon' 2024-08-17T17:03:57.564 INFO:tasks.rados.rados.0.smithi063.stdout:2097: done (1 left) 2024-08-17T17:03:57.564 INFO:tasks.rados.rados.0.smithi063.stdout:2098: rollback oid 33 current snap is 272 2024-08-17T17:03:57.564 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 33 to 236 2024-08-17T17:03:57.564 INFO:tasks.rados.rados.0.smithi063.stdout:2099: write oid 8 current snap is 272 2024-08-17T17:03:57.564 INFO:tasks.rados.rados.0.smithi063.stdout:2099: seq_num 788 ranges {443916=480571,1612046=405937,2606730=495052} 2024-08-17T17:03:57.570 INFO:tasks.rados.rados.0.smithi063.stdout:2099: writing smithi06387500-8 from 443916 to 924487 tid 1 2024-08-17T17:03:57.576 INFO:tasks.rados.rados.0.smithi063.stdout:2099: writing smithi06387500-8 from 1612046 to 2017983 tid 2 2024-08-17T17:03:57.582 INFO:tasks.rados.rados.0.smithi063.stdout:2099: writing smithi06387500-8 from 2606730 to 3101782 tid 3 2024-08-17T17:03:57.582 INFO:tasks.rados.rados.0.smithi063.stdout:2100: snap_remove snap 268 2024-08-17T17:03:57.777 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:03:57.902 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:57 smithi063 ceph-mon[114756]: osdmap e609: 8 total, 8 up, 8 in 2024-08-17T17:03:57.902 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:57 smithi063 ceph-mon[114756]: osdmap e610: 8 total, 8 up, 8 in 2024-08-17T17:03:57.902 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:57 smithi063 ceph-mon[109843]: osdmap e609: 8 total, 8 up, 8 in 2024-08-17T17:03:57.902 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:57 smithi063 ceph-mon[109843]: osdmap e610: 8 total, 8 up, 8 in 2024-08-17T17:03:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:57 smithi162 ceph-mon[104668]: osdmap e609: 8 total, 8 up, 8 in 2024-08-17T17:03:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:57 smithi162 ceph-mon[104668]: osdmap e610: 8 total, 8 up, 8 in 2024-08-17T17:03:58.564 INFO:tasks.rados.rados.0.smithi063.stdout:2100: done (3 left) 2024-08-17T17:03:58.564 INFO:tasks.rados.rados.0.smithi063.stdout:2101: read oid 29 snap -1 2024-08-17T17:03:58.564 INFO:tasks.rados.rados.0.smithi063.stdout:2101: expect deleted 2024-08-17T17:03:58.564 INFO:tasks.rados.rados.0.smithi063.stdout:2102: snap_remove snap 270 2024-08-17T17:03:59.575 INFO:tasks.rados.rados.0.smithi063.stdout:2102: done (4 left) 2024-08-17T17:03:59.575 INFO:tasks.rados.rados.0.smithi063.stdout:2103: read oid 17 snap -1 2024-08-17T17:03:59.575 INFO:tasks.rados.rados.0.smithi063.stdout:2103: expect deleted 2024-08-17T17:03:59.575 INFO:tasks.rados.rados.0.smithi063.stdout:2104: write oid 32 current snap is 272 2024-08-17T17:03:59.575 INFO:tasks.rados.rados.0.smithi063.stdout:2095: finishing write tid 1 to smithi06387500-24 2024-08-17T17:03:59.575 INFO:tasks.rados.rados.0.smithi063.stdout:2095: finishing write tid 2 to smithi06387500-24 2024-08-17T17:03:59.575 INFO:tasks.rados.rados.0.smithi063.stdout:2095: finishing write tid 3 to smithi06387500-24 2024-08-17T17:03:59.575 INFO:tasks.rados.rados.0.smithi063.stdout:2095: finishing write tid 4 to smithi06387500-24 2024-08-17T17:03:59.575 INFO:tasks.rados.rados.0.smithi063.stdout:2095: finishing write tid 5 to smithi06387500-24 2024-08-17T17:03:59.575 INFO:tasks.rados.rados.0.smithi063.stdout:2095: oid 24 updating version 0 to 792 2024-08-17T17:03:59.575 INFO:tasks.rados.rados.0.smithi063.stdout:2095: oid 24 updating version 792 to 793 2024-08-17T17:03:59.576 INFO:tasks.rados.rados.0.smithi063.stdout:2095: oid 24 updating version 793 to 794 2024-08-17T17:03:59.576 INFO:tasks.rados.rados.0.smithi063.stdout:2095: oid 24 version 794 is already newer than 791 2024-08-17T17:03:59.576 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 794 (ObjNum 787 snap 272 seq_num 787) dirty exists 2024-08-17T17:03:59.576 INFO:tasks.rados.rados.0.smithi063.stdout:2095: left oid 24 (ObjNum 787 snap 272 seq_num 787) 2024-08-17T17:03:59.576 INFO:tasks.rados.rados.0.smithi063.stdout:2098: finishing rollback tid 0 to smithi06387500-33 2024-08-17T17:03:59.576 INFO:tasks.rados.rados.0.smithi063.stdout:2098: finishing rollback tid 1 to smithi06387500-33 2024-08-17T17:03:59.576 INFO:tasks.rados.rados.0.smithi063.stdout:2104: seq_num 789 ranges {460723=555820,1670528=549595,2634153=626570} 2024-08-17T17:03:59.580 INFO:tasks.rados.rados.0.smithi063.stdout:2104: writing smithi06387500-32 from 460723 to 1016543 tid 1 2024-08-17T17:03:59.587 INFO:tasks.rados.rados.0.smithi063.stdout:2104: writing smithi06387500-32 from 1670528 to 2220123 tid 2 2024-08-17T17:03:59.593 INFO:tasks.rados.rados.0.smithi063.stdout:2104: writing smithi06387500-32 from 2634153 to 3260723 tid 3 2024-08-17T17:03:59.593 INFO:tasks.rados.rados.0.smithi063.stdout:2095: done (5 left) 2024-08-17T17:03:59.593 INFO:tasks.rados.rados.0.smithi063.stdout:2105: read oid 38 snap -1 2024-08-17T17:03:59.594 INFO:tasks.rados.rados.0.smithi063.stdout:2105: expect deleted 2024-08-17T17:03:59.594 INFO:tasks.rados.rados.0.smithi063.stdout:2106: delete oid 1 current snap is 272 2024-08-17T17:03:59.594 INFO:tasks.rados.rados.0.smithi063.stdout:2098: finishing rollback tid 2 to smithi06387500-33 2024-08-17T17:03:59.594 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 647 (ObjNum 636 snap 203 seq_num 636) dirty exists 2024-08-17T17:03:59.594 INFO:tasks.rados.rados.0.smithi063.stdout:2099: finishing write tid 1 to smithi06387500-8 2024-08-17T17:03:59.594 INFO:tasks.rados.rados.0.smithi063.stdout:2099: finishing write tid 2 to smithi06387500-8 2024-08-17T17:03:59.594 INFO:tasks.rados.rados.0.smithi063.stdout:2099: finishing write tid 3 to smithi06387500-8 2024-08-17T17:03:59.594 INFO:tasks.rados.rados.0.smithi063.stdout:2099: finishing write tid 4 to smithi06387500-8 2024-08-17T17:03:59.594 INFO:tasks.rados.rados.0.smithi063.stdout:2099: finishing write tid 5 to smithi06387500-8 2024-08-17T17:03:59.594 INFO:tasks.rados.rados.0.smithi063.stdout:2099: oid 8 updating version 0 to 653 2024-08-17T17:03:59.594 INFO:tasks.rados.rados.0.smithi063.stdout:2099: oid 8 updating version 653 to 656 2024-08-17T17:03:59.594 INFO:tasks.rados.rados.0.smithi063.stdout:2099: oid 8 version 656 is already newer than 654 2024-08-17T17:03:59.594 INFO:tasks.rados.rados.0.smithi063.stdout:2099: oid 8 version 656 is already newer than 655 2024-08-17T17:03:59.594 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 656 (ObjNum 788 snap 272 seq_num 788) dirty exists 2024-08-17T17:03:59.595 INFO:tasks.rados.rados.0.smithi063.stdout:2099: left oid 8 (ObjNum 788 snap 272 seq_num 788) 2024-08-17T17:03:59.595 INFO:tasks.rados.rados.0.smithi063.stdout:2104: finishing write tid 1 to smithi06387500-32 2024-08-17T17:03:59.595 INFO:tasks.rados.rados.0.smithi063.stdout:2104: finishing write tid 2 to smithi06387500-32 2024-08-17T17:03:59.596 INFO:tasks.rados.rados.0.smithi063.stdout:2098: done (6 left) 2024-08-17T17:03:59.596 INFO:tasks.rados.rados.0.smithi063.stdout:2099: done (5 left) 2024-08-17T17:03:59.596 INFO:tasks.rados.rados.0.smithi063.stdout:2101: done (4 left) 2024-08-17T17:03:59.596 INFO:tasks.rados.rados.0.smithi063.stdout:2103: done (3 left) 2024-08-17T17:03:59.596 INFO:tasks.rados.rados.0.smithi063.stdout:2105: done (2 left) 2024-08-17T17:03:59.597 INFO:tasks.rados.rados.0.smithi063.stdout:2106: done (1 left) 2024-08-17T17:03:59.597 INFO:tasks.rados.rados.0.smithi063.stdout:2107: write oid 3 current snap is 272 2024-08-17T17:03:59.597 INFO:tasks.rados.rados.0.smithi063.stdout:2107: seq_num 791 ranges {494337=422671,1387492=553264,2688999=445221} 2024-08-17T17:03:59.601 INFO:tasks.rados.rados.0.smithi063.stdout:2107: writing smithi06387500-3 from 494337 to 917008 tid 1 2024-08-17T17:03:59.606 INFO:tasks.rados.rados.0.smithi063.stdout:2107: writing smithi06387500-3 from 1387492 to 1940756 tid 2 2024-08-17T17:03:59.614 INFO:tasks.rados.rados.0.smithi063.stdout:2107: writing smithi06387500-3 from 2688999 to 3134220 tid 3 2024-08-17T17:03:59.614 INFO:tasks.rados.rados.0.smithi063.stdout:2108: snap_remove snap 269 2024-08-17T17:03:59.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:59 smithi162 ceph-mon[104668]: pgmap v372: 89 pgs: 89 active+clean; 188 MiB data, 715 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 727 KiB/s wr, 11 op/s 2024-08-17T17:03:59.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:03:59 smithi162 ceph-mon[104668]: osdmap e611: 8 total, 8 up, 8 in 2024-08-17T17:04:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:59 smithi063 ceph-mon[114756]: pgmap v372: 89 pgs: 89 active+clean; 188 MiB data, 715 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 727 KiB/s wr, 11 op/s 2024-08-17T17:04:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:03:59 smithi063 ceph-mon[114756]: osdmap e611: 8 total, 8 up, 8 in 2024-08-17T17:04:00.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:59 smithi063 ceph-mon[109843]: pgmap v372: 89 pgs: 89 active+clean; 188 MiB data, 715 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 727 KiB/s wr, 11 op/s 2024-08-17T17:04:00.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:03:59 smithi063 ceph-mon[109843]: osdmap e611: 8 total, 8 up, 8 in 2024-08-17T17:04:00.577 INFO:tasks.rados.rados.0.smithi063.stdout:2108: done (2 left) 2024-08-17T17:04:00.578 INFO:tasks.rados.rados.0.smithi063.stdout:2109: delete oid 38 current snap is 272 2024-08-17T17:04:00.578 INFO:tasks.rados.rados.0.smithi063.stdout:2104: finishing write tid 3 to smithi06387500-32 2024-08-17T17:04:00.578 INFO:tasks.rados.rados.0.smithi063.stdout:2104: finishing write tid 4 to smithi06387500-32 2024-08-17T17:04:00.578 INFO:tasks.rados.rados.0.smithi063.stdout:2104: finishing write tid 5 to smithi06387500-32 2024-08-17T17:04:00.578 INFO:tasks.rados.rados.0.smithi063.stdout:2104: oid 32 updating version 0 to 418 2024-08-17T17:04:00.578 INFO:tasks.rados.rados.0.smithi063.stdout:2104: oid 32 version 418 is already newer than 417 2024-08-17T17:04:00.578 INFO:tasks.rados.rados.0.smithi063.stdout:2104: oid 32 updating version 418 to 420 2024-08-17T17:04:00.578 INFO:tasks.rados.rados.0.smithi063.stdout:2104: oid 32 version 420 is already newer than 419 2024-08-17T17:04:00.578 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 420 (ObjNum 789 snap 272 seq_num 789) dirty exists 2024-08-17T17:04:00.578 INFO:tasks.rados.rados.0.smithi063.stdout:2104: left oid 32 (ObjNum 789 snap 272 seq_num 789) 2024-08-17T17:04:00.578 INFO:tasks.rados.rados.0.smithi063.stdout:2107: finishing write tid 1 to smithi06387500-3 2024-08-17T17:04:00.579 INFO:tasks.rados.rados.0.smithi063.stdout:2107: finishing write tid 2 to smithi06387500-3 2024-08-17T17:04:00.579 INFO:tasks.rados.rados.0.smithi063.stdout:2107: finishing write tid 3 to smithi06387500-3 2024-08-17T17:04:00.579 INFO:tasks.rados.rados.0.smithi063.stdout:2107: finishing write tid 4 to smithi06387500-3 2024-08-17T17:04:00.579 INFO:tasks.rados.rados.0.smithi063.stdout:2107: finishing write tid 5 to smithi06387500-3 2024-08-17T17:04:00.579 INFO:tasks.rados.rados.0.smithi063.stdout:2107: oid 3 updating version 0 to 635 2024-08-17T17:04:00.579 INFO:tasks.rados.rados.0.smithi063.stdout:2107: oid 3 updating version 635 to 638 2024-08-17T17:04:00.579 INFO:tasks.rados.rados.0.smithi063.stdout:2107: oid 3 version 638 is already newer than 637 2024-08-17T17:04:00.579 INFO:tasks.rados.rados.0.smithi063.stdout:2107: oid 3 version 638 is already newer than 636 2024-08-17T17:04:00.579 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 638 (ObjNum 791 snap 272 seq_num 791) dirty exists 2024-08-17T17:04:00.579 INFO:tasks.rados.rados.0.smithi063.stdout:2107: left oid 3 (ObjNum 791 snap 272 seq_num 791) 2024-08-17T17:04:00.584 INFO:tasks.rados.rados.0.smithi063.stdout:2104: done (2 left) 2024-08-17T17:04:00.584 INFO:tasks.rados.rados.0.smithi063.stdout:2107: done (1 left) 2024-08-17T17:04:00.584 INFO:tasks.rados.rados.0.smithi063.stdout:2109: done (0 left) 2024-08-17T17:04:00.584 INFO:tasks.rados.rados.0.smithi063.stdout:2110: read oid 20 snap -1 2024-08-17T17:04:00.584 INFO:tasks.rados.rados.0.smithi063.stdout:2110: expect (ObjNum 779 snap 269 seq_num 779) 2024-08-17T17:04:00.584 INFO:tasks.rados.rados.0.smithi063.stdout:2111: read oid 33 snap -1 2024-08-17T17:04:00.584 INFO:tasks.rados.rados.0.smithi063.stdout:2111: expect (ObjNum 636 snap 203 seq_num 636) 2024-08-17T17:04:00.584 INFO:tasks.rados.rados.0.smithi063.stdout:2112: read oid 42 snap -1 2024-08-17T17:04:00.584 INFO:tasks.rados.rados.0.smithi063.stdout:2112: expect deleted 2024-08-17T17:04:00.584 INFO:tasks.rados.rados.0.smithi063.stdout:2113: snap_remove snap 259 2024-08-17T17:04:00.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:00 smithi162 ceph-mon[104668]: osdmap e612: 8 total, 8 up, 8 in 2024-08-17T17:04:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:00 smithi162 ceph-mon[104668]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:00 smithi063 ceph-mon[114756]: osdmap e612: 8 total, 8 up, 8 in 2024-08-17T17:04:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:00 smithi063 ceph-mon[114756]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:00 smithi063 ceph-mon[109843]: osdmap e612: 8 total, 8 up, 8 in 2024-08-17T17:04:01.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:00 smithi063 ceph-mon[109843]: from='client.34251 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:01.463 INFO:teuthology.orchestra.run.smithi063.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:04:01.582 INFO:tasks.rados.rados.0.smithi063.stdout:2113: done (3 left) 2024-08-17T17:04:01.582 INFO:tasks.rados.rados.0.smithi063.stdout:2114: delete oid 17 current snap is 272 2024-08-17T17:04:01.582 INFO:tasks.rados.rados.0.smithi063.stdout:2111: expect (ObjNum 636 snap 203 seq_num 636) 2024-08-17T17:04:01.748 INFO:tasks.rados.rados.0.smithi063.stdout:2110: expect (ObjNum 779 snap 269 seq_num 779) 2024-08-17T17:04:01.774 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[109843]: pgmap v375: 89 pgs: 89 active+clean; 188 MiB data, 715 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 727 KiB/s wr, 11 op/s 2024-08-17T17:04:01.775 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[109843]: osdmap e613: 8 total, 8 up, 8 in 2024-08-17T17:04:01.775 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:01.775 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:01.775 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:04:01.775 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:04:01.775 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:04:01.775 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:01.775 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[114756]: pgmap v375: 89 pgs: 89 active+clean; 188 MiB data, 715 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 727 KiB/s wr, 11 op/s 2024-08-17T17:04:01.775 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[114756]: osdmap e613: 8 total, 8 up, 8 in 2024-08-17T17:04:01.775 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:01.776 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:01.776 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:04:01.776 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:04:01.776 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:04:01.776 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:01 smithi162 ceph-mon[104668]: pgmap v375: 89 pgs: 89 active+clean; 188 MiB data, 715 MiB used, 715 GiB / 715 GiB avail; 9.7 MiB/s rd, 727 KiB/s wr, 11 op/s 2024-08-17T17:04:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:01 smithi162 ceph-mon[104668]: osdmap e613: 8 total, 8 up, 8 in 2024-08-17T17:04:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:04:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:04:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:04:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:01.855 INFO:tasks.rados.rados.0.smithi063.stdout:2110: done (3 left) 2024-08-17T17:04:01.856 INFO:tasks.rados.rados.0.smithi063.stdout:2111: done (2 left) 2024-08-17T17:04:01.856 INFO:tasks.rados.rados.0.smithi063.stdout:2112: done (1 left) 2024-08-17T17:04:01.856 INFO:tasks.rados.rados.0.smithi063.stdout:2114: done (0 left) 2024-08-17T17:04:01.856 INFO:tasks.rados.rados.0.smithi063.stdout:2115: snap_remove snap 255 2024-08-17T17:04:02.069 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-17T17:04:02.328 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:02.590 INFO:tasks.rados.rados.0.smithi063.stdout:2115: done (0 left) 2024-08-17T17:04:02.591 INFO:tasks.rados.rados.0.smithi063.stdout:2116: snap_create 2024-08-17T17:04:02.721 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:02 smithi063 ceph-mon[114756]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:04:02.721 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:02 smithi063 ceph-mon[114756]: osdmap e614: 8 total, 8 up, 8 in 2024-08-17T17:04:02.721 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:02 smithi063 ceph-mon[114756]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:04:02.721 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:02 smithi063 ceph-mon[109843]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:04:02.721 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:02 smithi063 ceph-mon[109843]: osdmap e614: 8 total, 8 up, 8 in 2024-08-17T17:04:02.722 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:02 smithi063 ceph-mon[109843]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:04:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:02 smithi162 ceph-mon[104668]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:04:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:02 smithi162 ceph-mon[104668]: osdmap e614: 8 total, 8 up, 8 in 2024-08-17T17:04:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:02 smithi162 ceph-mon[104668]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:04:03.583 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:04:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:04:03] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.51.0" 2024-08-17T17:04:03.597 INFO:tasks.rados.rados.0.smithi063.stdout:2116: done (0 left) 2024-08-17T17:04:03.597 INFO:tasks.rados.rados.0.smithi063.stdout:2117: write oid 17 current snap is 273 2024-08-17T17:04:03.597 INFO:tasks.rados.rados.0.smithi063.stdout:2117: seq_num 794 ranges {544758=648418,1846585=416944,2771268=573490} 2024-08-17T17:04:03.604 INFO:tasks.rados.rados.0.smithi063.stdout:2117: writing smithi06387500-17 from 544758 to 1193176 tid 1 2024-08-17T17:04:03.609 INFO:tasks.rados.rados.0.smithi063.stdout:2117: writing smithi06387500-17 from 1846585 to 2263529 tid 2 2024-08-17T17:04:03.616 INFO:tasks.rados.rados.0.smithi063.stdout:2117: writing smithi06387500-17 from 2771268 to 3344758 tid 3 2024-08-17T17:04:03.616 INFO:tasks.rados.rados.0.smithi063.stdout:2118: write oid 49 current snap is 273 2024-08-17T17:04:03.616 INFO:tasks.rados.rados.0.smithi063.stdout:2117: finishing write tid 1 to smithi06387500-17 2024-08-17T17:04:03.616 INFO:tasks.rados.rados.0.smithi063.stdout:2117: finishing write tid 2 to smithi06387500-17 2024-08-17T17:04:03.616 INFO:tasks.rados.rados.0.smithi063.stdout:2118: seq_num 795 ranges {561565=723667,1905067=560602,3082338=279227} 2024-08-17T17:04:03.622 INFO:tasks.rados.rados.0.smithi063.stdout:2118: writing smithi06387500-49 from 561565 to 1285232 tid 1 2024-08-17T17:04:03.629 INFO:tasks.rados.rados.0.smithi063.stdout:2118: writing smithi06387500-49 from 1905067 to 2465669 tid 2 2024-08-17T17:04:03.634 INFO:tasks.rados.rados.0.smithi063.stdout:2118: writing smithi06387500-49 from 3082338 to 3361565 tid 3 2024-08-17T17:04:03.635 INFO:tasks.rados.rados.0.smithi063.stdout:2119: delete oid 4 current snap is 273 2024-08-17T17:04:03.635 INFO:tasks.rados.rados.0.smithi063.stdout:2117: finishing write tid 3 to smithi06387500-17 2024-08-17T17:04:03.635 INFO:tasks.rados.rados.0.smithi063.stdout:2117: finishing write tid 4 to smithi06387500-17 2024-08-17T17:04:03.635 INFO:tasks.rados.rados.0.smithi063.stdout:2117: finishing write tid 5 to smithi06387500-17 2024-08-17T17:04:03.635 INFO:tasks.rados.rados.0.smithi063.stdout:2117: oid 17 updating version 0 to 649 2024-08-17T17:04:03.635 INFO:tasks.rados.rados.0.smithi063.stdout:2117: oid 17 version 649 is already newer than 648 2024-08-17T17:04:03.635 INFO:tasks.rados.rados.0.smithi063.stdout:2117: oid 17 version 649 is already newer than 646 2024-08-17T17:04:03.635 INFO:tasks.rados.rados.0.smithi063.stdout:2117: oid 17 version 649 is already newer than 647 2024-08-17T17:04:03.635 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 649 (ObjNum 794 snap 273 seq_num 794) dirty exists 2024-08-17T17:04:03.635 INFO:tasks.rados.rados.0.smithi063.stdout:2117: left oid 17 (ObjNum 794 snap 273 seq_num 794) 2024-08-17T17:04:03.635 INFO:tasks.rados.rados.0.smithi063.stdout:2118: finishing write tid 1 to smithi06387500-49 2024-08-17T17:04:03.635 INFO:tasks.rados.rados.0.smithi063.stdout:2118: finishing write tid 2 to smithi06387500-49 2024-08-17T17:04:03.639 INFO:tasks.rados.rados.0.smithi063.stdout:2118: finishing write tid 3 to smithi06387500-49 2024-08-17T17:04:03.639 INFO:tasks.rados.rados.0.smithi063.stdout:2118: finishing write tid 4 to smithi06387500-49 2024-08-17T17:04:03.639 INFO:tasks.rados.rados.0.smithi063.stdout:2118: finishing write tid 5 to smithi06387500-49 2024-08-17T17:04:03.639 INFO:tasks.rados.rados.0.smithi063.stdout:2118: oid 49 updating version 0 to 2032 2024-08-17T17:04:03.639 INFO:tasks.rados.rados.0.smithi063.stdout:2118: oid 49 version 2032 is already newer than 2031 2024-08-17T17:04:03.639 INFO:tasks.rados.rados.0.smithi063.stdout:2118: oid 49 updating version 2032 to 2034 2024-08-17T17:04:03.639 INFO:tasks.rados.rados.0.smithi063.stdout:2118: oid 49 version 2034 is already newer than 2033 2024-08-17T17:04:03.639 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 2034 (ObjNum 795 snap 273 seq_num 795) dirty exists 2024-08-17T17:04:03.640 INFO:tasks.rados.rados.0.smithi063.stdout:2118: left oid 49 (ObjNum 795 snap 273 seq_num 795) 2024-08-17T17:04:03.640 INFO:tasks.rados.rados.0.smithi063.stdout:2117: done (2 left) 2024-08-17T17:04:03.640 INFO:tasks.rados.rados.0.smithi063.stdout:2118: done (1 left) 2024-08-17T17:04:03.640 INFO:tasks.rados.rados.0.smithi063.stdout:2119: done (0 left) 2024-08-17T17:04:03.640 INFO:tasks.rados.rados.0.smithi063.stdout:2120: snap_create 2024-08-17T17:04:03.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:03 smithi162 ceph-mon[104668]: pgmap v378: 89 pgs: 89 active+clean; 187 MiB data, 711 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T17:04:03.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:03 smithi162 ceph-mon[104668]: osdmap e615: 8 total, 8 up, 8 in 2024-08-17T17:04:03.889 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:03 smithi063 ceph-mon[109843]: pgmap v378: 89 pgs: 89 active+clean; 187 MiB data, 711 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T17:04:03.889 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:03 smithi063 ceph-mon[109843]: osdmap e615: 8 total, 8 up, 8 in 2024-08-17T17:04:03.890 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:03 smithi063 ceph-mon[114756]: pgmap v378: 89 pgs: 89 active+clean; 187 MiB data, 711 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T17:04:03.890 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:03 smithi063 ceph-mon[114756]: osdmap e615: 8 total, 8 up, 8 in 2024-08-17T17:04:04.602 INFO:tasks.rados.rados.0.smithi063.stdout:2120: done (0 left) 2024-08-17T17:04:04.602 INFO:tasks.rados.rados.0.smithi063.stdout:2121: read oid 4 snap 271 2024-08-17T17:04:04.602 INFO:tasks.rados.rados.0.smithi063.stdout:2121: expect (ObjNum 590 snap 187 seq_num 590) 2024-08-17T17:04:04.602 INFO:tasks.rados.rados.0.smithi063.stdout:2122: snap_create 2024-08-17T17:04:04.613 INFO:tasks.rados.rados.0.smithi063.stdout:2121: expect (ObjNum 590 snap 187 seq_num 590) 2024-08-17T17:04:04.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:04 smithi063 ceph-mon[109843]: osdmap e616: 8 total, 8 up, 8 in 2024-08-17T17:04:04.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:04 smithi063 ceph-mon[114756]: osdmap e616: 8 total, 8 up, 8 in 2024-08-17T17:04:05.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:04 smithi162 ceph-mon[104668]: osdmap e616: 8 total, 8 up, 8 in 2024-08-17T17:04:05.118 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T17:04:05.118 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:04:05.118 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (5m) 25s ago 11m 18.9M - 0.27.0 11f11916f8cd 59f4018b91ca 2024-08-17T17:04:05.118 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (5m) 25s ago 11m 75.4M - c8b91775d855 581e03e654a8 2024-08-17T17:04:05.119 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (10m) 25s ago 10m 21.5M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T17:04:05.119 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (10m) 25s ago 10m 18.7M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T17:04:05.119 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,9283,8765 running (6m) 25s ago 15m 456M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb da48034a2895 2024-08-17T17:04:05.119 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:8443,9283,8765 running (7m) 25s ago 17m 515M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 943e639cede6 2024-08-17T17:04:05.119 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (4m) 25s ago 17m 58.7M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb f0f20fcac956 2024-08-17T17:04:05.119 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (2m) 25s ago 15m 36.5M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 520525c79dbe 2024-08-17T17:04:05.119 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (4m) 25s ago 15m 45.8M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb c484af026acf 2024-08-17T17:04:05.119 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 running (6m) 25s ago 10m 11.3M - 1.7.0 72c9c2088986 34f55f5de2aa 2024-08-17T17:04:05.119 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (6m) 25s ago 10m 11.5M - 1.7.0 72c9c2088986 5bb162d0f1a9 2024-08-17T17:04:05.119 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (15m) 25s ago 15m 114M 2480M 18.2.4-719-gced649f8 c870753d67c2 f3b910ed49b8 2024-08-17T17:04:05.119 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (14m) 25s ago 14m 89.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 5d6fdb6000ad 2024-08-17T17:04:05.119 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (14m) 25s ago 14m 100M 2480M 18.2.4-719-gced649f8 c870753d67c2 a3cbe8edb49a 2024-08-17T17:04:05.120 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (13m) 25s ago 13m 87.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T17:04:05.120 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (13m) 25s ago 13m 117M 2480M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T17:04:05.120 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (13m) 25s ago 13m 107M 2480M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T17:04:05.120 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (12m) 25s ago 12m 81.3M 2480M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T17:04:05.120 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (12m) 25s ago 12m 91.8M 2480M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T17:04:05.120 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (5m) 25s ago 11m 45.9M - 2.51.0 1d3b7f56885b dfe66b2c5568 2024-08-17T17:04:05.502 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:04:05.502 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T17:04:05.502 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:04:05.502 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:04:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T17:04:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:04:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:04:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T17:04:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:04:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:04:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T17:04:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:04:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:04:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T17:04:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 10, 2024-08-17T17:04:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 5 2024-08-17T17:04:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:04:05.504 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:04:05.604 INFO:tasks.rados.rados.0.smithi063.stdout:2121: done (1 left) 2024-08-17T17:04:05.604 INFO:tasks.rados.rados.0.smithi063.stdout:2122: done (0 left) 2024-08-17T17:04:05.604 INFO:tasks.rados.rados.0.smithi063.stdout:2123: snap_remove snap 236 2024-08-17T17:04:05.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: pgmap v381: 89 pgs: 89 active+clean; 187 MiB data, 711 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T17:04:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='client.34257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: osdmap e617: 8 total, 8 up, 8 in 2024-08-17T17:04:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:04:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:04:05.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:04:05.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:04:05.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: pgmap v381: 89 pgs: 89 active+clean; 187 MiB data, 711 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T17:04:05.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='client.34257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:05.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: osdmap e617: 8 total, 8 up, 8 in 2024-08-17T17:04:05.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:04:05.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.785 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.785 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.785 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:04:05.785 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:04:05.785 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:04:05.785 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.785 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.785 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:04:05.785 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:04:05.786 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:04:05.786 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T17:04:05.786 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:04:05.786 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:04:05.786 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:04:05.786 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:04:05.786 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:04:05.786 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:04:05.786 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-17T17:04:05.786 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:04:05.786 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:04:05.787 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-17T17:04:05.787 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.787 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.787 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:04:05.787 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:04:05.787 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-17T17:04:05.787 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:04:05.787 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:04:05.787 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.787 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:04:05.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:04:05.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:04:05.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T17:04:05.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:04:05.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:04:05.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:04:05.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:04:05.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:04:05.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:04:05.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-17T17:04:05.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:04:05.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:04:05.788 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-17T17:04:05.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:04:05.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:04:05.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-17T17:04:05.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:04:05.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:04:05.789 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-17T17:04:05.790 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:04:05.790 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:04:05.790 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-17T17:04:05.790 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:04:05.790 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:04:05.790 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:04:05.790 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.790 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.790 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.790 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.790 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.791 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.791 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.791 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.791 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.791 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.791 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.791 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.791 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:04:05.791 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:04:05.791 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:04:05.791 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:04:05.791 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:04:05.791 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:04:05.791 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.792 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4281225559' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.792 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-17T17:04:05.792 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:04:05.792 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:04:05.792 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-17T17:04:05.792 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:04:05.792 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:04:05.792 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:04:05.792 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.793 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.793 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.793 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.793 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.793 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.793 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.793 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.793 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.793 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.793 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.793 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:05.793 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:04:05.793 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:04:05.793 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:04:05.794 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:04:05.794 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:04:05.794 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:04:05.794 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:05.794 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:05 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4281225559' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:05.803 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:04:05.803 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": null, 2024-08-17T17:04:05.803 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": false, 2024-08-17T17:04:05.803 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "", 2024-08-17T17:04:05.804 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-08-17T17:04:05.804 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": null, 2024-08-17T17:04:05.804 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "", 2024-08-17T17:04:05.804 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T17:04:05.804 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:04:06.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: pgmap v381: 89 pgs: 89 active+clean; 187 MiB data, 711 MiB used, 715 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T17:04:06.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='client.34257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:06.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: osdmap e617: 8 total, 8 up, 8 in 2024-08-17T17:04:06.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:06.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:04:06.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:06.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:06.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:06.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:06.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:06.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:06.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:06.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:06.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:06.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:04:06.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:04:06.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:04:06.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:04:06.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:04:06.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:04:06.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T17:04:06.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:04:06.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:04:06.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:04:06.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:04:06.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:04:06.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:04:06.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-08-17T17:04:06.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:04:06.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:04:06.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-08-17T17:04:06.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:04:06.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:04:06.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-08-17T17:04:06.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:04:06.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:04:06.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-08-17T17:04:06.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:04:06.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:04:06.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-08-17T17:04:06.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:04:06.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:04:06.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:04:06.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:04:06.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:04:06.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:04:06.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:04:06.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:04:06.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:04:06.103 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:04:06.104 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:06.104 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:05 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4281225559' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:04:06.608 INFO:tasks.rados.rados.0.smithi063.stdout:2123: done (0 left) 2024-08-17T17:04:06.608 INFO:tasks.rados.rados.0.smithi063.stdout:2124: snap_remove snap 271 2024-08-17T17:04:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: from='client.44192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:04:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:04:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all mgr 2024-08-17T17:04:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all mon 2024-08-17T17:04:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all crash 2024-08-17T17:04:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all rgw 2024-08-17T17:04:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all rbd-mirror 2024-08-17T17:04:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-17T17:04:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all ceph-exporter 2024-08-17T17:04:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all iscsi 2024-08-17T17:04:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all nfs 2024-08-17T17:04:07.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:04:07.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:04:07.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: from='client.34269 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:07.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:04:07.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:04:07.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all grafana 2024-08-17T17:04:07.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all loki 2024-08-17T17:04:07.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all promtail 2024-08-17T17:04:07.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Finalizing container_image settings 2024-08-17T17:04:07.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: Upgrade: Complete! 2024-08-17T17:04:07.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: osdmap e618: 8 total, 8 up, 8 in 2024-08-17T17:04:07.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:07.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: from='client.44192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:07.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:04:07.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:04:07.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all mgr 2024-08-17T17:04:07.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all mon 2024-08-17T17:04:07.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all crash 2024-08-17T17:04:07.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all rgw 2024-08-17T17:04:07.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all rbd-mirror 2024-08-17T17:04:07.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-17T17:04:07.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all ceph-exporter 2024-08-17T17:04:07.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all iscsi 2024-08-17T17:04:07.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all nfs 2024-08-17T17:04:07.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:04:07.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:04:07.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: from='client.34269 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:07.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:04:07.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:04:07.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all grafana 2024-08-17T17:04:07.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all loki 2024-08-17T17:04:07.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all promtail 2024-08-17T17:04:07.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Finalizing container_image settings 2024-08-17T17:04:07.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: Upgrade: Complete! 2024-08-17T17:04:07.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: osdmap e618: 8 total, 8 up, 8 in 2024-08-17T17:04:07.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:06 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: from='client.44192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:04:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:04:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all mgr 2024-08-17T17:04:07.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all mon 2024-08-17T17:04:07.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all crash 2024-08-17T17:04:07.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all rgw 2024-08-17T17:04:07.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all rbd-mirror 2024-08-17T17:04:07.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all cephfs-mirror 2024-08-17T17:04:07.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all ceph-exporter 2024-08-17T17:04:07.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all iscsi 2024-08-17T17:04:07.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all nfs 2024-08-17T17:04:07.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:04:07.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all node-exporter 2024-08-17T17:04:07.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: from='client.34269 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:07.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all prometheus 2024-08-17T17:04:07.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all alertmanager 2024-08-17T17:04:07.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all grafana 2024-08-17T17:04:07.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all loki 2024-08-17T17:04:07.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all promtail 2024-08-17T17:04:07.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Finalizing container_image settings 2024-08-17T17:04:07.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: Upgrade: Complete! 2024-08-17T17:04:07.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: osdmap e618: 8 total, 8 up, 8 in 2024-08-17T17:04:07.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:06 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:04:07.572 INFO:tasks.rados.rados.0.smithi063.stdout:2124: done (0 left) 2024-08-17T17:04:07.572 INFO:tasks.rados.rados.0.smithi063.stdout:2125: delete oid 34 current snap is 275 2024-08-17T17:04:07.581 INFO:tasks.rados.rados.0.smithi063.stdout:2125: done (0 left) 2024-08-17T17:04:07.581 INFO:tasks.rados.rados.0.smithi063.stdout:2126: write oid 31 current snap is 275 2024-08-17T17:04:07.581 INFO:tasks.rados.rados.0.smithi063.stdout:2126: seq_num 798 ranges {611986=549414,1564160=424282,2764607=440478} 2024-08-17T17:04:07.587 INFO:tasks.rados.rados.0.smithi063.stdout:2126: writing smithi06387500-31 from 611986 to 1161400 tid 1 2024-08-17T17:04:07.592 INFO:tasks.rados.rados.0.smithi063.stdout:2126: writing smithi06387500-31 from 1564160 to 1988442 tid 2 2024-08-17T17:04:07.599 INFO:tasks.rados.rados.0.smithi063.stdout:2126: writing smithi06387500-31 from 2764607 to 3205085 tid 3 2024-08-17T17:04:07.599 INFO:tasks.rados.rados.0.smithi063.stdout:2127: read oid 23 snap 257 2024-08-17T17:04:07.599 INFO:tasks.rados.rados.0.smithi063.stdout:2127: expect (ObjNum 757 snap 257 seq_num 757) 2024-08-17T17:04:07.599 INFO:tasks.rados.rados.0.smithi063.stdout:2128: snap_create 2024-08-17T17:04:07.599 INFO:tasks.rados.rados.0.smithi063.stdout:2126: finishing write tid 1 to smithi06387500-31 2024-08-17T17:04:07.599 INFO:tasks.rados.rados.0.smithi063.stdout:2126: finishing write tid 2 to smithi06387500-31 2024-08-17T17:04:07.606 INFO:tasks.rados.rados.0.smithi063.stdout:2126: finishing write tid 3 to smithi06387500-31 2024-08-17T17:04:07.611 INFO:tasks.rados.rados.0.smithi063.stdout:2127: expect (ObjNum 757 snap 257 seq_num 757) 2024-08-17T17:04:07.815 INFO:tasks.rados.rados.0.smithi063.stdout:2126: finishing write tid 4 to smithi06387500-31 2024-08-17T17:04:07.815 INFO:tasks.rados.rados.0.smithi063.stdout:2126: finishing write tid 5 to smithi06387500-31 2024-08-17T17:04:07.815 INFO:tasks.rados.rados.0.smithi063.stdout:2126: oid 31 updating version 0 to 1402 2024-08-17T17:04:07.815 INFO:tasks.rados.rados.0.smithi063.stdout:2126: oid 31 updating version 1402 to 1403 2024-08-17T17:04:07.815 INFO:tasks.rados.rados.0.smithi063.stdout:2126: oid 31 updating version 1403 to 1404 2024-08-17T17:04:07.815 INFO:tasks.rados.rados.0.smithi063.stdout:2126: oid 31 updating version 1404 to 1405 2024-08-17T17:04:07.815 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 1405 (ObjNum 798 snap 275 seq_num 798) dirty exists 2024-08-17T17:04:07.815 INFO:tasks.rados.rados.0.smithi063.stdout:2126: left oid 31 (ObjNum 798 snap 275 seq_num 798) 2024-08-17T17:04:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:07 smithi063 ceph-mon[114756]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:07 smithi063 ceph-mon[114756]: pgmap v384: 89 pgs: 89 active+clean; 187 MiB data, 711 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:07 smithi063 ceph-mon[114756]: osdmap e619: 8 total, 8 up, 8 in 2024-08-17T17:04:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:07 smithi063 ceph-mon[114756]: osdmap e620: 8 total, 8 up, 8 in 2024-08-17T17:04:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:07 smithi063 ceph-mon[109843]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:08.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:07 smithi063 ceph-mon[109843]: pgmap v384: 89 pgs: 89 active+clean; 187 MiB data, 711 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:08.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:07 smithi063 ceph-mon[109843]: osdmap e619: 8 total, 8 up, 8 in 2024-08-17T17:04:08.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:07 smithi063 ceph-mon[109843]: osdmap e620: 8 total, 8 up, 8 in 2024-08-17T17:04:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:07 smithi162 ceph-mon[104668]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:07 smithi162 ceph-mon[104668]: pgmap v384: 89 pgs: 89 active+clean; 187 MiB data, 711 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:07 smithi162 ceph-mon[104668]: osdmap e619: 8 total, 8 up, 8 in 2024-08-17T17:04:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:07 smithi162 ceph-mon[104668]: osdmap e620: 8 total, 8 up, 8 in 2024-08-17T17:04:08.580 INFO:tasks.rados.rados.0.smithi063.stdout:2126: done (2 left) 2024-08-17T17:04:08.580 INFO:tasks.rados.rados.0.smithi063.stdout:2127: done (1 left) 2024-08-17T17:04:08.580 INFO:tasks.rados.rados.0.smithi063.stdout:2128: done (0 left) 2024-08-17T17:04:08.581 INFO:tasks.rados.rados.0.smithi063.stdout:2129: write oid 9 current snap is 276 2024-08-17T17:04:08.581 INFO:tasks.rados.rados.0.smithi063.stdout:2129: seq_num 799 ranges {628793=741016,2138995=567940,3192030=236763} 2024-08-17T17:04:08.587 INFO:tasks.rados.rados.0.smithi063.stdout:2129: writing smithi06387500-9 from 628793 to 1369809 tid 1 2024-08-17T17:04:08.595 INFO:tasks.rados.rados.0.smithi063.stdout:2129: writing smithi06387500-9 from 2138995 to 2706935 tid 2 2024-08-17T17:04:08.600 INFO:tasks.rados.rados.0.smithi063.stdout:2129: writing smithi06387500-9 from 3192030 to 3428793 tid 3 2024-08-17T17:04:08.600 INFO:tasks.rados.rados.0.smithi063.stdout:2130: read oid 10 snap 267 2024-08-17T17:04:08.600 INFO:tasks.rados.rados.0.smithi063.stdout:2130: expect (ObjNum 639 snap 203 seq_num 639) 2024-08-17T17:04:08.600 INFO:tasks.rados.rados.0.smithi063.stdout:2131: snap_create 2024-08-17T17:04:08.600 INFO:tasks.rados.rados.0.smithi063.stdout:2129: finishing write tid 1 to smithi06387500-9 2024-08-17T17:04:08.600 INFO:tasks.rados.rados.0.smithi063.stdout:2129: finishing write tid 2 to smithi06387500-9 2024-08-17T17:04:08.602 INFO:tasks.rados.rados.0.smithi063.stdout:2129: finishing write tid 3 to smithi06387500-9 2024-08-17T17:04:08.602 INFO:tasks.rados.rados.0.smithi063.stdout:2129: finishing write tid 4 to smithi06387500-9 2024-08-17T17:04:08.602 INFO:tasks.rados.rados.0.smithi063.stdout:2129: finishing write tid 5 to smithi06387500-9 2024-08-17T17:04:08.603 INFO:tasks.rados.rados.0.smithi063.stdout:2129: oid 9 updating version 0 to 922 2024-08-17T17:04:08.603 INFO:tasks.rados.rados.0.smithi063.stdout:2129: oid 9 version 922 is already newer than 921 2024-08-17T17:04:08.603 INFO:tasks.rados.rados.0.smithi063.stdout:2129: oid 9 updating version 922 to 923 2024-08-17T17:04:08.603 INFO:tasks.rados.rados.0.smithi063.stdout:2129: oid 9 version 923 is already newer than 920 2024-08-17T17:04:08.603 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 923 (ObjNum 799 snap 276 seq_num 799) dirty exists 2024-08-17T17:04:08.603 INFO:tasks.rados.rados.0.smithi063.stdout:2129: left oid 9 (ObjNum 799 snap 276 seq_num 799) 2024-08-17T17:04:08.605 INFO:tasks.rados.rados.0.smithi063.stdout:2130: expect (ObjNum 639 snap 203 seq_num 639) 2024-08-17T17:04:09.590 INFO:tasks.rados.rados.0.smithi063.stdout:2129: done (2 left) 2024-08-17T17:04:09.590 INFO:tasks.rados.rados.0.smithi063.stdout:2130: done (1 left) 2024-08-17T17:04:09.590 INFO:tasks.rados.rados.0.smithi063.stdout:2131: done (0 left) 2024-08-17T17:04:09.590 INFO:tasks.rados.rados.0.smithi063.stdout:2132: read oid 3 snap -1 2024-08-17T17:04:09.590 INFO:tasks.rados.rados.0.smithi063.stdout:2132: expect (ObjNum 791 snap 272 seq_num 791) 2024-08-17T17:04:09.591 INFO:tasks.rados.rados.0.smithi063.stdout:2133: write oid 39 current snap is 277 2024-08-17T17:04:09.591 INFO:tasks.rados.rados.0.smithi063.stdout:2133: seq_num 800 ranges {645600=416265,1797477=711598,3219453=226147} 2024-08-17T17:04:09.595 INFO:tasks.rados.rados.0.smithi063.stdout:2133: writing smithi06387500-39 from 645600 to 1061865 tid 1 2024-08-17T17:04:09.604 INFO:tasks.rados.rados.0.smithi063.stdout:2133: writing smithi06387500-39 from 1797477 to 2509075 tid 2 2024-08-17T17:04:09.609 INFO:tasks.rados.rados.0.smithi063.stdout:2133: writing smithi06387500-39 from 3219453 to 3445600 tid 3 2024-08-17T17:04:09.610 INFO:tasks.rados.rados.0.smithi063.stdout:2134: read oid 5 snap 273 2024-08-17T17:04:09.610 INFO:tasks.rados.rados.0.smithi063.stdout:2134: expect (ObjNum 744 snap 253 seq_num 744) 2024-08-17T17:04:09.610 INFO:tasks.rados.rados.0.smithi063.stdout:2135: snap_create 2024-08-17T17:04:09.610 INFO:tasks.rados.rados.0.smithi063.stdout:2133: finishing write tid 1 to smithi06387500-39 2024-08-17T17:04:09.610 INFO:tasks.rados.rados.0.smithi063.stdout:2132: expect (ObjNum 791 snap 272 seq_num 791) 2024-08-17T17:04:09.809 INFO:tasks.rados.rados.0.smithi063.stdout:2133: finishing write tid 2 to smithi06387500-39 2024-08-17T17:04:09.809 INFO:tasks.rados.rados.0.smithi063.stdout:2133: finishing write tid 3 to smithi06387500-39 2024-08-17T17:04:09.809 INFO:tasks.rados.rados.0.smithi063.stdout:2133: finishing write tid 4 to smithi06387500-39 2024-08-17T17:04:09.809 INFO:tasks.rados.rados.0.smithi063.stdout:2133: finishing write tid 5 to smithi06387500-39 2024-08-17T17:04:09.809 INFO:tasks.rados.rados.0.smithi063.stdout:2133: oid 39 updating version 0 to 448 2024-08-17T17:04:09.810 INFO:tasks.rados.rados.0.smithi063.stdout:2133: oid 39 updating version 448 to 449 2024-08-17T17:04:09.810 INFO:tasks.rados.rados.0.smithi063.stdout:2133: oid 39 updating version 449 to 450 2024-08-17T17:04:09.810 INFO:tasks.rados.rados.0.smithi063.stdout:2133: oid 39 version 450 is already newer than 447 2024-08-17T17:04:09.810 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 450 (ObjNum 800 snap 277 seq_num 800) dirty exists 2024-08-17T17:04:09.810 INFO:tasks.rados.rados.0.smithi063.stdout:2133: left oid 39 (ObjNum 800 snap 277 seq_num 800) 2024-08-17T17:04:09.810 INFO:tasks.rados.rados.0.smithi063.stdout:2134: expect (ObjNum 744 snap 253 seq_num 744) 2024-08-17T17:04:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:09 smithi162 ceph-mon[104668]: pgmap v387: 89 pgs: 89 active+clean; 190 MiB data, 718 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 783 KiB/s wr, 5 op/s 2024-08-17T17:04:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:09 smithi162 ceph-mon[104668]: osdmap e621: 8 total, 8 up, 8 in 2024-08-17T17:04:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:09 smithi063 ceph-mon[109843]: pgmap v387: 89 pgs: 89 active+clean; 190 MiB data, 718 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 783 KiB/s wr, 5 op/s 2024-08-17T17:04:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:09 smithi063 ceph-mon[109843]: osdmap e621: 8 total, 8 up, 8 in 2024-08-17T17:04:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:09 smithi063 ceph-mon[114756]: pgmap v387: 89 pgs: 89 active+clean; 190 MiB data, 718 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 783 KiB/s wr, 5 op/s 2024-08-17T17:04:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:09 smithi063 ceph-mon[114756]: osdmap e621: 8 total, 8 up, 8 in 2024-08-17T17:04:10.595 INFO:tasks.rados.rados.0.smithi063.stdout:2132: done (3 left) 2024-08-17T17:04:10.595 INFO:tasks.rados.rados.0.smithi063.stdout:2133: done (2 left) 2024-08-17T17:04:10.595 INFO:tasks.rados.rados.0.smithi063.stdout:2134: done (1 left) 2024-08-17T17:04:10.595 INFO:tasks.rados.rados.0.smithi063.stdout:2135: done (0 left) 2024-08-17T17:04:10.595 INFO:tasks.rados.rados.0.smithi063.stdout:2136: delete oid 11 current snap is 278 2024-08-17T17:04:10.598 INFO:tasks.rados.rados.0.smithi063.stdout:2136: done (0 left) 2024-08-17T17:04:10.598 INFO:tasks.rados.rados.0.smithi063.stdout:2137: snap_create 2024-08-17T17:04:10.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:10 smithi162 ceph-mon[104668]: osdmap e622: 8 total, 8 up, 8 in 2024-08-17T17:04:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:10 smithi063 ceph-mon[114756]: osdmap e622: 8 total, 8 up, 8 in 2024-08-17T17:04:11.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:10 smithi063 ceph-mon[109843]: osdmap e622: 8 total, 8 up, 8 in 2024-08-17T17:04:11.597 INFO:tasks.rados.rados.0.smithi063.stdout:2137: done (0 left) 2024-08-17T17:04:11.597 INFO:tasks.rados.rados.0.smithi063.stdout:2138: read oid 43 snap -1 2024-08-17T17:04:11.597 INFO:tasks.rados.rados.0.smithi063.stdout:2138: expect (ObjNum 735 snap 247 seq_num 735) 2024-08-17T17:04:11.597 INFO:tasks.rados.rados.0.smithi063.stdout:2139: read oid 1 snap -1 2024-08-17T17:04:11.597 INFO:tasks.rados.rados.0.smithi063.stdout:2139: expect deleted 2024-08-17T17:04:11.597 INFO:tasks.rados.rados.0.smithi063.stdout:2140: write oid 42 current snap is 279 2024-08-17T17:04:11.597 INFO:tasks.rados.rados.0.smithi063.stdout:2140: seq_num 802 ranges {679214=566763,1798088=715267,3157946=321268} 2024-08-17T17:04:11.603 INFO:tasks.rados.rados.0.smithi063.stdout:2140: writing smithi06387500-42 from 679214 to 1245977 tid 1 2024-08-17T17:04:11.611 INFO:tasks.rados.rados.0.smithi063.stdout:2140: writing smithi06387500-42 from 1798088 to 2513355 tid 2 2024-08-17T17:04:11.616 INFO:tasks.rados.rados.0.smithi063.stdout:2140: writing smithi06387500-42 from 3157946 to 3479214 tid 3 2024-08-17T17:04:11.617 INFO:tasks.rados.rados.0.smithi063.stdout:2141: delete oid 34 current snap is 279 2024-08-17T17:04:11.617 INFO:tasks.rados.rados.0.smithi063.stdout:2138: expect (ObjNum 735 snap 247 seq_num 735) 2024-08-17T17:04:11.726 INFO:tasks.rados.rados.0.smithi063.stdout:2140: finishing write tid 1 to smithi06387500-42 2024-08-17T17:04:11.726 INFO:tasks.rados.rados.0.smithi063.stdout:2140: finishing write tid 2 to smithi06387500-42 2024-08-17T17:04:11.726 INFO:tasks.rados.rados.0.smithi063.stdout:2140: finishing write tid 3 to smithi06387500-42 2024-08-17T17:04:11.726 INFO:tasks.rados.rados.0.smithi063.stdout:2140: finishing write tid 4 to smithi06387500-42 2024-08-17T17:04:11.726 INFO:tasks.rados.rados.0.smithi063.stdout:2140: finishing write tid 5 to smithi06387500-42 2024-08-17T17:04:11.726 INFO:tasks.rados.rados.0.smithi063.stdout:2140: oid 42 updating version 0 to 1407 2024-08-17T17:04:11.727 INFO:tasks.rados.rados.0.smithi063.stdout:2140: oid 42 updating version 1407 to 1409 2024-08-17T17:04:11.727 INFO:tasks.rados.rados.0.smithi063.stdout:2140: oid 42 version 1409 is already newer than 1408 2024-08-17T17:04:11.727 INFO:tasks.rados.rados.0.smithi063.stdout:2140: oid 42 version 1409 is already newer than 1406 2024-08-17T17:04:11.727 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 1409 (ObjNum 802 snap 279 seq_num 802) dirty exists 2024-08-17T17:04:11.727 INFO:tasks.rados.rados.0.smithi063.stdout:2140: left oid 42 (ObjNum 802 snap 279 seq_num 802) 2024-08-17T17:04:11.727 INFO:tasks.rados.rados.0.smithi063.stdout:2138: done (3 left) 2024-08-17T17:04:11.727 INFO:tasks.rados.rados.0.smithi063.stdout:2139: done (2 left) 2024-08-17T17:04:11.727 INFO:tasks.rados.rados.0.smithi063.stdout:2140: done (1 left) 2024-08-17T17:04:11.727 INFO:tasks.rados.rados.0.smithi063.stdout:2141: done (0 left) 2024-08-17T17:04:11.727 INFO:tasks.rados.rados.0.smithi063.stdout:2142: rollback oid 37 current snap is 279 2024-08-17T17:04:11.727 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 240 2024-08-17T17:04:11.727 INFO:tasks.rados.rados.0.smithi063.stdout:2143: snap_remove snap 253 2024-08-17T17:04:11.930 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:11 smithi063 ceph-mon[114756]: pgmap v390: 89 pgs: 89 active+clean; 190 MiB data, 718 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 783 KiB/s wr, 5 op/s 2024-08-17T17:04:11.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:11 smithi063 ceph-mon[114756]: osdmap e623: 8 total, 8 up, 8 in 2024-08-17T17:04:11.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:11 smithi063 ceph-mon[109843]: pgmap v390: 89 pgs: 89 active+clean; 190 MiB data, 718 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 783 KiB/s wr, 5 op/s 2024-08-17T17:04:11.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:11 smithi063 ceph-mon[109843]: osdmap e623: 8 total, 8 up, 8 in 2024-08-17T17:04:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:11 smithi162 ceph-mon[104668]: pgmap v390: 89 pgs: 89 active+clean; 190 MiB data, 718 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 783 KiB/s wr, 5 op/s 2024-08-17T17:04:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:11 smithi162 ceph-mon[104668]: osdmap e623: 8 total, 8 up, 8 in 2024-08-17T17:04:12.569 INFO:tasks.rados.rados.0.smithi063.stdout:2143: done (1 left) 2024-08-17T17:04:12.570 INFO:tasks.rados.rados.0.smithi063.stdout:2144: read oid 2 snap -1 2024-08-17T17:04:12.570 INFO:tasks.rados.rados.0.smithi063.stdout:2144: expect deleted 2024-08-17T17:04:12.570 INFO:tasks.rados.rados.0.smithi063.stdout:2145: snap_create 2024-08-17T17:04:12.570 INFO:tasks.rados.rados.0.smithi063.stdout:2142: finishing rollback tid 0 to smithi06387500-37 2024-08-17T17:04:12.570 INFO:tasks.rados.rados.0.smithi063.stdout:2142: finishing rollback tid 1 to smithi06387500-37 2024-08-17T17:04:12.570 INFO:tasks.rados.rados.0.smithi063.stdout:2142: finishing rollback tid 2 to smithi06387500-37 2024-08-17T17:04:12.570 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 2054 (ObjNum 696 snap 222 seq_num 696) dirty exists 2024-08-17T17:04:13.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:12 smithi063 ceph-mon[114756]: osdmap e624: 8 total, 8 up, 8 in 2024-08-17T17:04:13.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:12 smithi063 ceph-mon[114756]: osdmap e625: 8 total, 8 up, 8 in 2024-08-17T17:04:13.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:12 smithi063 ceph-mon[109843]: osdmap e624: 8 total, 8 up, 8 in 2024-08-17T17:04:13.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:12 smithi063 ceph-mon[109843]: osdmap e625: 8 total, 8 up, 8 in 2024-08-17T17:04:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:12 smithi162 ceph-mon[104668]: osdmap e624: 8 total, 8 up, 8 in 2024-08-17T17:04:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:12 smithi162 ceph-mon[104668]: osdmap e625: 8 total, 8 up, 8 in 2024-08-17T17:04:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:04:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:04:13] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.51.0" 2024-08-17T17:04:13.568 INFO:tasks.rados.rados.0.smithi063.stdout:2142: done (2 left) 2024-08-17T17:04:13.569 INFO:tasks.rados.rados.0.smithi063.stdout:2144: done (1 left) 2024-08-17T17:04:13.569 INFO:tasks.rados.rados.0.smithi063.stdout:2145: done (0 left) 2024-08-17T17:04:13.569 INFO:tasks.rados.rados.0.smithi063.stdout:2146: delete oid 13 current snap is 280 2024-08-17T17:04:13.573 INFO:tasks.rados.rados.0.smithi063.stdout:2146: done (0 left) 2024-08-17T17:04:13.574 INFO:tasks.rados.rados.0.smithi063.stdout:2147: write oid 20 current snap is 280 2024-08-17T17:04:13.574 INFO:tasks.rados.rados.0.smithi063.stdout:2147: seq_num 805 ranges {729635=792510,1973534=578947,3240215=289420} 2024-08-17T17:04:13.581 INFO:tasks.rados.rados.0.smithi063.stdout:2147: writing smithi06387500-20 from 729635 to 1522145 tid 1 2024-08-17T17:04:13.587 INFO:tasks.rados.rados.0.smithi063.stdout:2147: writing smithi06387500-20 from 1973534 to 2552481 tid 2 2024-08-17T17:04:13.593 INFO:tasks.rados.rados.0.smithi063.stdout:2147: writing smithi06387500-20 from 3240215 to 3529635 tid 3 2024-08-17T17:04:13.594 INFO:tasks.rados.rados.0.smithi063.stdout:2148: rollback oid 43 current snap is 280 2024-08-17T17:04:13.594 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 43 to 278 2024-08-17T17:04:13.594 INFO:tasks.rados.rados.0.smithi063.stdout:2147: finishing write tid 1 to smithi06387500-20 2024-08-17T17:04:13.594 INFO:tasks.rados.rados.0.smithi063.stdout:2147: finishing write tid 2 to smithi06387500-20 2024-08-17T17:04:13.594 INFO:tasks.rados.rados.0.smithi063.stdout:2149: rollback oid 5 current snap is 280 2024-08-17T17:04:13.594 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 267 2024-08-17T17:04:13.594 INFO:tasks.rados.rados.0.smithi063.stdout:2150: write oid 16 current snap is 280 2024-08-17T17:04:13.594 INFO:tasks.rados.rados.0.smithi063.stdout:2150: seq_num 806 ranges {746442=584112,1748369=722605,2983991=562451} 2024-08-17T17:04:13.600 INFO:tasks.rados.rados.0.smithi063.stdout:2150: writing smithi06387500-16 from 746442 to 1330554 tid 1 2024-08-17T17:04:13.607 INFO:tasks.rados.rados.0.smithi063.stdout:2150: writing smithi06387500-16 from 1748369 to 2470974 tid 2 2024-08-17T17:04:13.614 INFO:tasks.rados.rados.0.smithi063.stdout:2150: writing smithi06387500-16 from 2983991 to 3546442 tid 3 2024-08-17T17:04:13.614 INFO:tasks.rados.rados.0.smithi063.stdout:2151: read oid 42 snap 267 2024-08-17T17:04:13.614 INFO:tasks.rados.rados.0.smithi063.stdout:2151: expect deleted 2024-08-17T17:04:13.614 INFO:tasks.rados.rados.0.smithi063.stdout:2152: write oid 17 current snap is 280 2024-08-17T17:04:13.614 INFO:tasks.rados.rados.0.smithi063.stdout:2147: finishing write tid 3 to smithi06387500-20 2024-08-17T17:04:13.614 INFO:tasks.rados.rados.0.smithi063.stdout:2147: finishing write tid 4 to smithi06387500-20 2024-08-17T17:04:13.614 INFO:tasks.rados.rados.0.smithi063.stdout:2147: finishing write tid 5 to smithi06387500-20 2024-08-17T17:04:13.615 INFO:tasks.rados.rados.0.smithi063.stdout:2147: oid 20 updating version 0 to 430 2024-08-17T17:04:13.615 INFO:tasks.rados.rados.0.smithi063.stdout:2147: oid 20 version 430 is already newer than 428 2024-08-17T17:04:13.615 INFO:tasks.rados.rados.0.smithi063.stdout:2147: oid 20 version 430 is already newer than 427 2024-08-17T17:04:13.615 INFO:tasks.rados.rados.0.smithi063.stdout:2147: oid 20 version 430 is already newer than 429 2024-08-17T17:04:13.615 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 430 (ObjNum 805 snap 280 seq_num 805) dirty exists 2024-08-17T17:04:13.615 INFO:tasks.rados.rados.0.smithi063.stdout:2147: left oid 20 (ObjNum 805 snap 280 seq_num 805) 2024-08-17T17:04:13.615 INFO:tasks.rados.rados.0.smithi063.stdout:2148: finishing rollback tid 0 to smithi06387500-43 2024-08-17T17:04:13.615 INFO:tasks.rados.rados.0.smithi063.stdout:2149: finishing rollback tid 0 to smithi06387500-5 2024-08-17T17:04:13.615 INFO:tasks.rados.rados.0.smithi063.stdout:2149: finishing rollback tid 1 to smithi06387500-5 2024-08-17T17:04:13.615 INFO:tasks.rados.rados.0.smithi063.stdout:2148: finishing rollback tid 1 to smithi06387500-43 2024-08-17T17:04:13.615 INFO:tasks.rados.rados.0.smithi063.stdout:2148: finishing rollback tid 2 to smithi06387500-43 2024-08-17T17:04:13.615 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 931 (ObjNum 735 snap 247 seq_num 735) dirty exists 2024-08-17T17:04:13.615 INFO:tasks.rados.rados.0.smithi063.stdout:2149: finishing rollback tid 2 to smithi06387500-5 2024-08-17T17:04:13.616 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1022 (ObjNum 744 snap 253 seq_num 744) dirty exists 2024-08-17T17:04:13.616 INFO:tasks.rados.rados.0.smithi063.stdout:2150: finishing write tid 1 to smithi06387500-16 2024-08-17T17:04:13.616 INFO:tasks.rados.rados.0.smithi063.stdout:2152: seq_num 807 ranges {763249=659361,2090498=582616,3295061=268188} 2024-08-17T17:04:13.621 INFO:tasks.rados.rados.0.smithi063.stdout:2152: writing smithi06387500-17 from 763249 to 1422610 tid 1 2024-08-17T17:04:13.628 INFO:tasks.rados.rados.0.smithi063.stdout:2152: writing smithi06387500-17 from 2090498 to 2673114 tid 2 2024-08-17T17:04:13.633 INFO:tasks.rados.rados.0.smithi063.stdout:2152: writing smithi06387500-17 from 3295061 to 3563249 tid 3 2024-08-17T17:04:13.633 INFO:tasks.rados.rados.0.smithi063.stdout:2147: done (5 left) 2024-08-17T17:04:13.633 INFO:tasks.rados.rados.0.smithi063.stdout:2148: done (4 left) 2024-08-17T17:04:13.633 INFO:tasks.rados.rados.0.smithi063.stdout:2149: done (3 left) 2024-08-17T17:04:13.634 INFO:tasks.rados.rados.0.smithi063.stdout:2153: write oid 32 current snap is 280 2024-08-17T17:04:13.634 INFO:tasks.rados.rados.0.smithi063.stdout:2150: finishing write tid 2 to smithi06387500-16 2024-08-17T17:04:13.634 INFO:tasks.rados.rados.0.smithi063.stdout:2153: seq_num 808 ranges {780056=734610,2148980=726274,3322484=257572} 2024-08-17T17:04:13.641 INFO:tasks.rados.rados.0.smithi063.stdout:2153: writing smithi06387500-32 from 780056 to 1514666 tid 1 2024-08-17T17:04:13.649 INFO:tasks.rados.rados.0.smithi063.stdout:2153: writing smithi06387500-32 from 2148980 to 2875254 tid 2 2024-08-17T17:04:13.653 INFO:tasks.rados.rados.0.smithi063.stdout:2153: writing smithi06387500-32 from 3322484 to 3580056 tid 3 2024-08-17T17:04:13.653 INFO:tasks.rados.rados.0.smithi063.stdout:2154: write oid 20 current snap is 280 2024-08-17T17:04:13.653 INFO:tasks.rados.rados.0.smithi063.stdout:2154: seq_num 809 ranges {796863=409859,1807462=586285,3066260=530603} 2024-08-17T17:04:13.659 INFO:tasks.rados.rados.0.smithi063.stdout:2154: writing smithi06387500-20 from 796863 to 1206722 tid 1 2024-08-17T17:04:13.666 INFO:tasks.rados.rados.0.smithi063.stdout:2154: writing smithi06387500-20 from 1807462 to 2393747 tid 2 2024-08-17T17:04:13.673 INFO:tasks.rados.rados.0.smithi063.stdout:2154: writing smithi06387500-20 from 3066260 to 3596863 tid 3 2024-08-17T17:04:13.673 INFO:tasks.rados.rados.0.smithi063.stdout:2155: delete oid 5 current snap is 280 2024-08-17T17:04:13.673 INFO:tasks.rados.rados.0.smithi063.stdout:2150: finishing write tid 3 to smithi06387500-16 2024-08-17T17:04:13.673 INFO:tasks.rados.rados.0.smithi063.stdout:2150: finishing write tid 4 to smithi06387500-16 2024-08-17T17:04:13.673 INFO:tasks.rados.rados.0.smithi063.stdout:2150: finishing write tid 5 to smithi06387500-16 2024-08-17T17:04:13.673 INFO:tasks.rados.rados.0.smithi063.stdout:2150: oid 16 updating version 0 to 838 2024-08-17T17:04:13.673 INFO:tasks.rados.rados.0.smithi063.stdout:2150: oid 16 version 838 is already newer than 836 2024-08-17T17:04:13.673 INFO:tasks.rados.rados.0.smithi063.stdout:2150: oid 16 version 838 is already newer than 837 2024-08-17T17:04:13.673 INFO:tasks.rados.rados.0.smithi063.stdout:2150: oid 16 updating version 838 to 839 2024-08-17T17:04:13.673 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 839 (ObjNum 806 snap 280 seq_num 806) dirty exists 2024-08-17T17:04:13.673 INFO:tasks.rados.rados.0.smithi063.stdout:2150: left oid 16 (ObjNum 806 snap 280 seq_num 806) 2024-08-17T17:04:13.674 INFO:tasks.rados.rados.0.smithi063.stdout:2152: finishing write tid 1 to smithi06387500-17 2024-08-17T17:04:13.674 INFO:tasks.rados.rados.0.smithi063.stdout:2152: finishing write tid 2 to smithi06387500-17 2024-08-17T17:04:13.674 INFO:tasks.rados.rados.0.smithi063.stdout:2152: finishing write tid 3 to smithi06387500-17 2024-08-17T17:04:13.674 INFO:tasks.rados.rados.0.smithi063.stdout:2152: finishing write tid 4 to smithi06387500-17 2024-08-17T17:04:13.674 INFO:tasks.rados.rados.0.smithi063.stdout:2152: finishing write tid 5 to smithi06387500-17 2024-08-17T17:04:13.674 INFO:tasks.rados.rados.0.smithi063.stdout:2152: oid 17 updating version 0 to 658 2024-08-17T17:04:13.674 INFO:tasks.rados.rados.0.smithi063.stdout:2152: oid 17 version 658 is already newer than 657 2024-08-17T17:04:13.674 INFO:tasks.rados.rados.0.smithi063.stdout:2152: oid 17 updating version 658 to 660 2024-08-17T17:04:13.674 INFO:tasks.rados.rados.0.smithi063.stdout:2152: oid 17 version 660 is already newer than 659 2024-08-17T17:04:13.674 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 660 (ObjNum 807 snap 280 seq_num 807) dirty exists 2024-08-17T17:04:13.674 INFO:tasks.rados.rados.0.smithi063.stdout:2152: left oid 17 (ObjNum 807 snap 280 seq_num 807) 2024-08-17T17:04:13.674 INFO:tasks.rados.rados.0.smithi063.stdout:2153: finishing write tid 1 to smithi06387500-32 2024-08-17T17:04:13.674 INFO:tasks.rados.rados.0.smithi063.stdout:2153: finishing write tid 2 to smithi06387500-32 2024-08-17T17:04:13.675 INFO:tasks.rados.rados.0.smithi063.stdout:2153: finishing write tid 3 to smithi06387500-32 2024-08-17T17:04:13.675 INFO:tasks.rados.rados.0.smithi063.stdout:2153: finishing write tid 4 to smithi06387500-32 2024-08-17T17:04:13.675 INFO:tasks.rados.rados.0.smithi063.stdout:2153: finishing write tid 5 to smithi06387500-32 2024-08-17T17:04:13.675 INFO:tasks.rados.rados.0.smithi063.stdout:2153: oid 32 updating version 0 to 443 2024-08-17T17:04:13.675 INFO:tasks.rados.rados.0.smithi063.stdout:2153: oid 32 updating version 443 to 444 2024-08-17T17:04:13.675 INFO:tasks.rados.rados.0.smithi063.stdout:2153: oid 32 version 444 is already newer than 442 2024-08-17T17:04:13.675 INFO:tasks.rados.rados.0.smithi063.stdout:2153: oid 32 updating version 444 to 445 2024-08-17T17:04:13.675 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 445 (ObjNum 808 snap 280 seq_num 808) dirty exists 2024-08-17T17:04:13.675 INFO:tasks.rados.rados.0.smithi063.stdout:2153: left oid 32 (ObjNum 808 snap 280 seq_num 808) 2024-08-17T17:04:13.675 INFO:tasks.rados.rados.0.smithi063.stdout:2154: finishing write tid 1 to smithi06387500-20 2024-08-17T17:04:13.675 INFO:tasks.rados.rados.0.smithi063.stdout:2154: finishing write tid 2 to smithi06387500-20 2024-08-17T17:04:13.677 INFO:tasks.rados.rados.0.smithi063.stdout:2150: done (5 left) 2024-08-17T17:04:13.677 INFO:tasks.rados.rados.0.smithi063.stdout:2151: done (4 left) 2024-08-17T17:04:13.677 INFO:tasks.rados.rados.0.smithi063.stdout:2152: done (3 left) 2024-08-17T17:04:13.677 INFO:tasks.rados.rados.0.smithi063.stdout:2153: done (2 left) 2024-08-17T17:04:13.677 INFO:tasks.rados.rados.0.smithi063.stdout:2155: done (1 left) 2024-08-17T17:04:13.677 INFO:tasks.rados.rados.0.smithi063.stdout:2156: delete oid 9 current snap is 280 2024-08-17T17:04:13.677 INFO:tasks.rados.rados.0.smithi063.stdout:2154: finishing write tid 3 to smithi06387500-20 2024-08-17T17:04:13.677 INFO:tasks.rados.rados.0.smithi063.stdout:2154: finishing write tid 4 to smithi06387500-20 2024-08-17T17:04:13.677 INFO:tasks.rados.rados.0.smithi063.stdout:2154: finishing write tid 5 to smithi06387500-20 2024-08-17T17:04:13.677 INFO:tasks.rados.rados.0.smithi063.stdout:2154: oid 20 updating version 0 to 433 2024-08-17T17:04:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2154: oid 20 updating version 433 to 434 2024-08-17T17:04:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2154: oid 20 version 434 is already newer than 431 2024-08-17T17:04:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2154: oid 20 version 434 is already newer than 432 2024-08-17T17:04:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 434 (ObjNum 809 snap 280 seq_num 809) dirty exists 2024-08-17T17:04:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2154: left oid 20 (ObjNum 809 snap 280 seq_num 809) 2024-08-17T17:04:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2154: done (1 left) 2024-08-17T17:04:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2156: done (0 left) 2024-08-17T17:04:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2157: read oid 43 snap -1 2024-08-17T17:04:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2157: expect (ObjNum 735 snap 247 seq_num 735) 2024-08-17T17:04:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2158: read oid 19 snap -1 2024-08-17T17:04:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2158: expect (ObjNum 765 snap 260 seq_num 765) 2024-08-17T17:04:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2159: read oid 35 snap -1 2024-08-17T17:04:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2159: expect deleted 2024-08-17T17:04:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2160: rollback oid 22 current snap is 280 2024-08-17T17:04:13.679 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 22 to 267 2024-08-17T17:04:13.679 INFO:tasks.rados.rados.0.smithi063.stdout:2161: read oid 31 snap -1 2024-08-17T17:04:13.679 INFO:tasks.rados.rados.0.smithi063.stdout:2161: expect (ObjNum 798 snap 275 seq_num 798) 2024-08-17T17:04:13.679 INFO:tasks.rados.rados.0.smithi063.stdout:2162: snap_remove snap 276 2024-08-17T17:04:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:13 smithi063 ceph-mon[114756]: pgmap v393: 89 pgs: 89 active+clean; 199 MiB data, 730 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-08-17T17:04:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:13 smithi063 ceph-mon[114756]: osdmap e626: 8 total, 8 up, 8 in 2024-08-17T17:04:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:13 smithi063 ceph-mon[109843]: pgmap v393: 89 pgs: 89 active+clean; 199 MiB data, 730 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-08-17T17:04:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:13 smithi063 ceph-mon[109843]: osdmap e626: 8 total, 8 up, 8 in 2024-08-17T17:04:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:13 smithi162 ceph-mon[104668]: pgmap v393: 89 pgs: 89 active+clean; 199 MiB data, 730 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-08-17T17:04:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:13 smithi162 ceph-mon[104668]: osdmap e626: 8 total, 8 up, 8 in 2024-08-17T17:04:14.612 INFO:tasks.rados.rados.0.smithi063.stdout:2162: done (5 left) 2024-08-17T17:04:14.612 INFO:tasks.rados.rados.0.smithi063.stdout:2163: read oid 17 snap -1 2024-08-17T17:04:14.612 INFO:tasks.rados.rados.0.smithi063.stdout:2163: expect (ObjNum 807 snap 280 seq_num 807) 2024-08-17T17:04:14.612 INFO:tasks.rados.rados.0.smithi063.stdout:2164: write oid 2 current snap is 280 2024-08-17T17:04:14.612 INFO:tasks.rados.rados.0.smithi063.stdout:2164: seq_num 812 ranges {447284=635606,1866555=733612,3315823=331461} 2024-08-17T17:04:14.618 INFO:tasks.rados.rados.0.smithi063.stdout:2164: writing smithi06387500-2 from 447284 to 1082890 tid 1 2024-08-17T17:04:14.627 INFO:tasks.rados.rados.0.smithi063.stdout:2164: writing smithi06387500-2 from 1866555 to 2600167 tid 2 2024-08-17T17:04:14.633 INFO:tasks.rados.rados.0.smithi063.stdout:2164: writing smithi06387500-2 from 3315823 to 3647284 tid 3 2024-08-17T17:04:14.633 INFO:tasks.rados.rados.0.smithi063.stdout:2165: snap_remove snap 272 2024-08-17T17:04:15.619 INFO:tasks.rados.rados.0.smithi063.stdout:2165: done (7 left) 2024-08-17T17:04:15.620 INFO:tasks.rados.rados.0.smithi063.stdout:2166: delete oid 24 current snap is 280 2024-08-17T17:04:15.620 INFO:tasks.rados.rados.0.smithi063.stdout:2160: finishing rollback tid 0 to smithi06387500-22 2024-08-17T17:04:15.620 INFO:tasks.rados.rados.0.smithi063.stdout:2160: finishing rollback tid 1 to smithi06387500-22 2024-08-17T17:04:15.620 INFO:tasks.rados.rados.0.smithi063.stdout:2160: finishing rollback tid 2 to smithi06387500-22 2024-08-17T17:04:15.620 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 438 (ObjNum 626 snap 199 seq_num 626) dirty exists 2024-08-17T17:04:15.621 INFO:tasks.rados.rados.0.smithi063.stdout:2158: expect (ObjNum 765 snap 260 seq_num 765) 2024-08-17T17:04:15.718 INFO:tasks.rados.rados.0.smithi063.stdout:2161: expect (ObjNum 798 snap 275 seq_num 798) 2024-08-17T17:04:15.833 INFO:tasks.rados.rados.0.smithi063.stdout:2157: expect (ObjNum 735 snap 247 seq_num 735) 2024-08-17T17:04:15.941 INFO:tasks.rados.rados.0.smithi063.stdout:2163: expect (ObjNum 807 snap 280 seq_num 807) 2024-08-17T17:04:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:15 smithi063 ceph-mon[114756]: pgmap v396: 89 pgs: 89 active+clean; 199 MiB data, 730 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-08-17T17:04:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:15 smithi063 ceph-mon[114756]: osdmap e627: 8 total, 8 up, 8 in 2024-08-17T17:04:16.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:15 smithi063 ceph-mon[109843]: pgmap v396: 89 pgs: 89 active+clean; 199 MiB data, 730 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-08-17T17:04:16.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:15 smithi063 ceph-mon[109843]: osdmap e627: 8 total, 8 up, 8 in 2024-08-17T17:04:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:15 smithi162 ceph-mon[104668]: pgmap v396: 89 pgs: 89 active+clean; 199 MiB data, 730 MiB used, 715 GiB / 715 GiB avail; 13 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-08-17T17:04:16.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:15 smithi162 ceph-mon[104668]: osdmap e627: 8 total, 8 up, 8 in 2024-08-17T17:04:16.102 INFO:tasks.rados.rados.0.smithi063.stdout:2164: finishing write tid 1 to smithi06387500-2 2024-08-17T17:04:16.102 INFO:tasks.rados.rados.0.smithi063.stdout:2164: finishing write tid 2 to smithi06387500-2 2024-08-17T17:04:16.102 INFO:tasks.rados.rados.0.smithi063.stdout:2164: finishing write tid 3 to smithi06387500-2 2024-08-17T17:04:16.102 INFO:tasks.rados.rados.0.smithi063.stdout:2164: finishing write tid 4 to smithi06387500-2 2024-08-17T17:04:16.103 INFO:tasks.rados.rados.0.smithi063.stdout:2164: finishing write tid 5 to smithi06387500-2 2024-08-17T17:04:16.103 INFO:tasks.rados.rados.0.smithi063.stdout:2164: oid 2 updating version 0 to 1030 2024-08-17T17:04:16.103 INFO:tasks.rados.rados.0.smithi063.stdout:2164: oid 2 version 1030 is already newer than 1029 2024-08-17T17:04:16.103 INFO:tasks.rados.rados.0.smithi063.stdout:2164: oid 2 updating version 1030 to 1031 2024-08-17T17:04:16.103 INFO:tasks.rados.rados.0.smithi063.stdout:2164: oid 2 version 1031 is already newer than 1024 2024-08-17T17:04:16.103 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 1031 (ObjNum 812 snap 280 seq_num 812) dirty exists 2024-08-17T17:04:16.103 INFO:tasks.rados.rados.0.smithi063.stdout:2164: left oid 2 (ObjNum 812 snap 280 seq_num 812) 2024-08-17T17:04:16.103 INFO:tasks.rados.rados.0.smithi063.stdout:2157: done (7 left) 2024-08-17T17:04:16.103 INFO:tasks.rados.rados.0.smithi063.stdout:2158: done (6 left) 2024-08-17T17:04:16.103 INFO:tasks.rados.rados.0.smithi063.stdout:2159: done (5 left) 2024-08-17T17:04:16.104 INFO:tasks.rados.rados.0.smithi063.stdout:2160: done (4 left) 2024-08-17T17:04:16.104 INFO:tasks.rados.rados.0.smithi063.stdout:2161: done (3 left) 2024-08-17T17:04:16.104 INFO:tasks.rados.rados.0.smithi063.stdout:2163: done (2 left) 2024-08-17T17:04:16.104 INFO:tasks.rados.rados.0.smithi063.stdout:2164: done (1 left) 2024-08-17T17:04:16.104 INFO:tasks.rados.rados.0.smithi063.stdout:2166: done (0 left) 2024-08-17T17:04:16.104 INFO:tasks.rados.rados.0.smithi063.stdout:2167: rollback oid 13 current snap is 280 2024-08-17T17:04:16.104 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 13 to 240 2024-08-17T17:04:16.104 INFO:tasks.rados.rados.0.smithi063.stdout:2168: delete oid 33 current snap is 280 2024-08-17T17:04:16.105 INFO:tasks.rados.rados.0.smithi063.stdout:2168: done (1 left) 2024-08-17T17:04:16.105 INFO:tasks.rados.rados.0.smithi063.stdout:2169: snap_create 2024-08-17T17:04:16.105 INFO:tasks.rados.rados.0.smithi063.stdout:2167: finishing rollback tid 1 to smithi06387500-13 2024-08-17T17:04:16.105 INFO:tasks.rados.rados.0.smithi063.stdout:2167: finishing rollback tid 2 to smithi06387500-13 2024-08-17T17:04:16.105 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1037 (ObjNum 630 snap 201 seq_num 630) dirty exists 2024-08-17T17:04:16.628 INFO:tasks.rados.rados.0.smithi063.stdout:2167: done (1 left) 2024-08-17T17:04:16.628 INFO:tasks.rados.rados.0.smithi063.stdout:2169: done (0 left) 2024-08-17T17:04:16.628 INFO:tasks.rados.rados.0.smithi063.stdout:2170: write oid 4 current snap is 281 2024-08-17T17:04:16.628 INFO:tasks.rados.rados.0.smithi063.stdout:2170: seq_num 815 ranges {497705=577706,1642001=480939,2598092=497514} 2024-08-17T17:04:16.634 INFO:tasks.rados.rados.0.smithi063.stdout:2170: writing smithi06387500-4 from 497705 to 1075411 tid 1 2024-08-17T17:04:16.639 INFO:tasks.rados.rados.0.smithi063.stdout:2170: writing smithi06387500-4 from 1642001 to 2122940 tid 2 2024-08-17T17:04:16.645 INFO:tasks.rados.rados.0.smithi063.stdout:2170: writing smithi06387500-4 from 2598092 to 3095606 tid 3 2024-08-17T17:04:16.645 INFO:tasks.rados.rados.0.smithi063.stdout:2171: read oid 37 snap -1 2024-08-17T17:04:16.645 INFO:tasks.rados.rados.0.smithi063.stdout:2171: expect (ObjNum 696 snap 222 seq_num 696) 2024-08-17T17:04:16.645 INFO:tasks.rados.rados.0.smithi063.stdout:2172: snap_remove snap 267 2024-08-17T17:04:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:16 smithi063 ceph-mon[114756]: osdmap e628: 8 total, 8 up, 8 in 2024-08-17T17:04:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:16 smithi063 ceph-mon[109843]: osdmap e628: 8 total, 8 up, 8 in 2024-08-17T17:04:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:16 smithi162 ceph-mon[104668]: osdmap e628: 8 total, 8 up, 8 in 2024-08-17T17:04:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:17.631 INFO:tasks.rados.rados.0.smithi063.stdout:2172: done (2 left) 2024-08-17T17:04:17.632 INFO:tasks.rados.rados.0.smithi063.stdout:2173: write oid 33 current snap is 281 2024-08-17T17:04:17.632 INFO:tasks.rados.rados.0.smithi063.stdout:2170: finishing write tid 1 to smithi06387500-4 2024-08-17T17:04:17.632 INFO:tasks.rados.rados.0.smithi063.stdout:2170: finishing write tid 2 to smithi06387500-4 2024-08-17T17:04:17.632 INFO:tasks.rados.rados.0.smithi063.stdout:2170: finishing write tid 3 to smithi06387500-4 2024-08-17T17:04:17.632 INFO:tasks.rados.rados.0.smithi063.stdout:2170: finishing write tid 4 to smithi06387500-4 2024-08-17T17:04:17.632 INFO:tasks.rados.rados.0.smithi063.stdout:2170: finishing write tid 5 to smithi06387500-4 2024-08-17T17:04:17.632 INFO:tasks.rados.rados.0.smithi063.stdout:2170: oid 4 updating version 0 to 2078 2024-08-17T17:04:17.632 INFO:tasks.rados.rados.0.smithi063.stdout:2170: oid 4 version 2078 is already newer than 2075 2024-08-17T17:04:17.632 INFO:tasks.rados.rados.0.smithi063.stdout:2170: oid 4 version 2078 is already newer than 2076 2024-08-17T17:04:17.632 INFO:tasks.rados.rados.0.smithi063.stdout:2170: oid 4 version 2078 is already newer than 2077 2024-08-17T17:04:17.632 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 2078 (ObjNum 815 snap 281 seq_num 815) dirty exists 2024-08-17T17:04:17.633 INFO:tasks.rados.rados.0.smithi063.stdout:2170: left oid 4 (ObjNum 815 snap 281 seq_num 815) 2024-08-17T17:04:17.633 INFO:tasks.rados.rados.0.smithi063.stdout:2171: expect (ObjNum 696 snap 222 seq_num 696) 2024-08-17T17:04:17.856 INFO:tasks.rados.rados.0.smithi063.stdout:2173: seq_num 816 ranges {514512=652955,1700483=740950,3025515=688997} 2024-08-17T17:04:17.862 INFO:tasks.rados.rados.0.smithi063.stdout:2173: writing smithi06387500-33 from 514512 to 1167467 tid 1 2024-08-17T17:04:17.869 INFO:tasks.rados.rados.0.smithi063.stdout:2173: writing smithi06387500-33 from 1700483 to 2441433 tid 2 2024-08-17T17:04:17.877 INFO:tasks.rados.rados.0.smithi063.stdout:2173: writing smithi06387500-33 from 3025515 to 3714512 tid 3 2024-08-17T17:04:17.877 INFO:tasks.rados.rados.0.smithi063.stdout:2170: done (2 left) 2024-08-17T17:04:17.877 INFO:tasks.rados.rados.0.smithi063.stdout:2171: done (1 left) 2024-08-17T17:04:17.877 INFO:tasks.rados.rados.0.smithi063.stdout:2174: rollback oid 49 current snap is 281 2024-08-17T17:04:17.877 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 49 to 280 2024-08-17T17:04:17.877 INFO:tasks.rados.rados.0.smithi063.stdout:2173: finishing write tid 1 to smithi06387500-33 2024-08-17T17:04:17.877 INFO:tasks.rados.rados.0.smithi063.stdout:2173: finishing write tid 2 to smithi06387500-33 2024-08-17T17:04:17.877 INFO:tasks.rados.rados.0.smithi063.stdout:2175: snap_create 2024-08-17T17:04:17.879 INFO:tasks.rados.rados.0.smithi063.stdout:2173: finishing write tid 3 to smithi06387500-33 2024-08-17T17:04:17.879 INFO:tasks.rados.rados.0.smithi063.stdout:2173: finishing write tid 4 to smithi06387500-33 2024-08-17T17:04:17.879 INFO:tasks.rados.rados.0.smithi063.stdout:2173: finishing write tid 5 to smithi06387500-33 2024-08-17T17:04:17.880 INFO:tasks.rados.rados.0.smithi063.stdout:2173: oid 33 updating version 0 to 696 2024-08-17T17:04:17.880 INFO:tasks.rados.rados.0.smithi063.stdout:2173: oid 33 updating version 696 to 697 2024-08-17T17:04:17.880 INFO:tasks.rados.rados.0.smithi063.stdout:2173: oid 33 version 697 is already newer than 695 2024-08-17T17:04:17.880 INFO:tasks.rados.rados.0.smithi063.stdout:2173: oid 33 updating version 697 to 698 2024-08-17T17:04:17.880 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 698 (ObjNum 816 snap 281 seq_num 816) dirty exists 2024-08-17T17:04:17.880 INFO:tasks.rados.rados.0.smithi063.stdout:2173: left oid 33 (ObjNum 816 snap 281 seq_num 816) 2024-08-17T17:04:17.881 INFO:tasks.rados.rados.0.smithi063.stdout:2174: finishing rollback tid 0 to smithi06387500-49 2024-08-17T17:04:17.881 INFO:tasks.rados.rados.0.smithi063.stdout:2174: finishing rollback tid 1 to smithi06387500-49 2024-08-17T17:04:17.882 INFO:tasks.rados.rados.0.smithi063.stdout:2174: finishing rollback tid 2 to smithi06387500-49 2024-08-17T17:04:17.882 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 2090 (ObjNum 795 snap 273 seq_num 795) dirty exists 2024-08-17T17:04:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:17 smithi063 ceph-mon[114756]: pgmap v399: 89 pgs: 89 active+clean; 199 MiB data, 730 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:17 smithi063 ceph-mon[114756]: osdmap e629: 8 total, 8 up, 8 in 2024-08-17T17:04:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:17 smithi063 ceph-mon[109843]: pgmap v399: 89 pgs: 89 active+clean; 199 MiB data, 730 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:17 smithi063 ceph-mon[109843]: osdmap e629: 8 total, 8 up, 8 in 2024-08-17T17:04:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:17 smithi162 ceph-mon[104668]: pgmap v399: 89 pgs: 89 active+clean; 199 MiB data, 730 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:17 smithi162 ceph-mon[104668]: osdmap e629: 8 total, 8 up, 8 in 2024-08-17T17:04:18.634 INFO:tasks.rados.rados.0.smithi063.stdout:2173: done (2 left) 2024-08-17T17:04:18.634 INFO:tasks.rados.rados.0.smithi063.stdout:2174: done (1 left) 2024-08-17T17:04:18.635 INFO:tasks.rados.rados.0.smithi063.stdout:2175: done (0 left) 2024-08-17T17:04:18.635 INFO:tasks.rados.rados.0.smithi063.stdout:2176: write oid 29 current snap is 282 2024-08-17T17:04:18.635 INFO:tasks.rados.rados.0.smithi063.stdout:2176: seq_num 817 ranges {531319=728204,1758965=484608,2652938=520058} 2024-08-17T17:04:18.641 INFO:tasks.rados.rados.0.smithi063.stdout:2176: writing smithi06387500-29 from 531319 to 1259523 tid 1 2024-08-17T17:04:18.647 INFO:tasks.rados.rados.0.smithi063.stdout:2176: writing smithi06387500-29 from 1758965 to 2243573 tid 2 2024-08-17T17:04:18.652 INFO:tasks.rados.rados.0.smithi063.stdout:2176: writing smithi06387500-29 from 2652938 to 3172996 tid 3 2024-08-17T17:04:18.653 INFO:tasks.rados.rados.0.smithi063.stdout:2177: read oid 21 snap 247 2024-08-17T17:04:18.653 INFO:tasks.rados.rados.0.smithi063.stdout:2177: expect (ObjNum 709 snap 234 seq_num 709) 2024-08-17T17:04:18.653 INFO:tasks.rados.rados.0.smithi063.stdout:2178: rollback oid 46 current snap is 282 2024-08-17T17:04:18.653 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 46 to 273 2024-08-17T17:04:18.653 INFO:tasks.rados.rados.0.smithi063.stdout:2176: finishing write tid 1 to smithi06387500-29 2024-08-17T17:04:18.653 INFO:tasks.rados.rados.0.smithi063.stdout:2176: finishing write tid 2 to smithi06387500-29 2024-08-17T17:04:18.653 INFO:tasks.rados.rados.0.smithi063.stdout:2179: write oid 34 current snap is 282 2024-08-17T17:04:18.653 INFO:tasks.rados.rados.0.smithi063.stdout:2179: seq_num 818 ranges {548126=403453,1417447=744619,2680361=447683,3680450=67676} 2024-08-17T17:04:18.658 INFO:tasks.rados.rados.0.smithi063.stdout:2179: writing smithi06387500-34 from 548126 to 951579 tid 1 2024-08-17T17:04:18.665 INFO:tasks.rados.rados.0.smithi063.stdout:2179: writing smithi06387500-34 from 1417447 to 2162066 tid 2 2024-08-17T17:04:18.670 INFO:tasks.rados.rados.0.smithi063.stdout:2179: writing smithi06387500-34 from 2680361 to 3128044 tid 3 2024-08-17T17:04:18.674 INFO:tasks.rados.rados.0.smithi063.stdout:2179: writing smithi06387500-34 from 3680450 to 3748126 tid 4 2024-08-17T17:04:18.674 INFO:tasks.rados.rados.0.smithi063.stdout:2180: delete oid 28 current snap is 282 2024-08-17T17:04:18.674 INFO:tasks.rados.rados.0.smithi063.stdout:2178: finishing rollback tid 1 to smithi06387500-46 2024-08-17T17:04:18.674 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 789 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T17:04:18.674 INFO:tasks.rados.rados.0.smithi063.stdout:2176: finishing write tid 3 to smithi06387500-29 2024-08-17T17:04:18.674 INFO:tasks.rados.rados.0.smithi063.stdout:2176: finishing write tid 4 to smithi06387500-29 2024-08-17T17:04:18.675 INFO:tasks.rados.rados.0.smithi063.stdout:2176: finishing write tid 5 to smithi06387500-29 2024-08-17T17:04:18.675 INFO:tasks.rados.rados.0.smithi063.stdout:2176: oid 29 updating version 0 to 515 2024-08-17T17:04:18.675 INFO:tasks.rados.rados.0.smithi063.stdout:2176: oid 29 version 515 is already newer than 513 2024-08-17T17:04:18.675 INFO:tasks.rados.rados.0.smithi063.stdout:2176: oid 29 version 515 is already newer than 514 2024-08-17T17:04:18.675 INFO:tasks.rados.rados.0.smithi063.stdout:2176: oid 29 version 515 is already newer than 512 2024-08-17T17:04:18.675 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 515 (ObjNum 817 snap 282 seq_num 817) dirty exists 2024-08-17T17:04:18.675 INFO:tasks.rados.rados.0.smithi063.stdout:2176: left oid 29 (ObjNum 817 snap 282 seq_num 817) 2024-08-17T17:04:18.675 INFO:tasks.rados.rados.0.smithi063.stdout:2177: expect (ObjNum 709 snap 234 seq_num 709) 2024-08-17T17:04:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2179: finishing write tid 1 to smithi06387500-34 2024-08-17T17:04:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2179: finishing write tid 2 to smithi06387500-34 2024-08-17T17:04:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2179: finishing write tid 3 to smithi06387500-34 2024-08-17T17:04:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2179: finishing write tid 4 to smithi06387500-34 2024-08-17T17:04:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2179: finishing write tid 5 to smithi06387500-34 2024-08-17T17:04:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2179: finishing write tid 6 to smithi06387500-34 2024-08-17T17:04:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2179: oid 34 updating version 0 to 1419 2024-08-17T17:04:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2179: oid 34 version 1419 is already newer than 1416 2024-08-17T17:04:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2179: oid 34 version 1419 is already newer than 1418 2024-08-17T17:04:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2179: oid 34 version 1419 is already newer than 1415 2024-08-17T17:04:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2179: oid 34 version 1419 is already newer than 1417 2024-08-17T17:04:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 1419 (ObjNum 818 snap 282 seq_num 818) dirty exists 2024-08-17T17:04:18.808 INFO:tasks.rados.rados.0.smithi063.stdout:2179: left oid 34 (ObjNum 818 snap 282 seq_num 818) 2024-08-17T17:04:18.808 INFO:tasks.rados.rados.0.smithi063.stdout:2176: done (4 left) 2024-08-17T17:04:18.808 INFO:tasks.rados.rados.0.smithi063.stdout:2177: done (3 left) 2024-08-17T17:04:18.808 INFO:tasks.rados.rados.0.smithi063.stdout:2178: done (2 left) 2024-08-17T17:04:18.808 INFO:tasks.rados.rados.0.smithi063.stdout:2179: done (1 left) 2024-08-17T17:04:18.808 INFO:tasks.rados.rados.0.smithi063.stdout:2180: done (0 left) 2024-08-17T17:04:18.808 INFO:tasks.rados.rados.0.smithi063.stdout:2181: snap_create 2024-08-17T17:04:19.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:18 smithi063 ceph-mon[109843]: osdmap e630: 8 total, 8 up, 8 in 2024-08-17T17:04:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:18 smithi063 ceph-mon[114756]: osdmap e630: 8 total, 8 up, 8 in 2024-08-17T17:04:19.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:18 smithi162 ceph-mon[104668]: osdmap e630: 8 total, 8 up, 8 in 2024-08-17T17:04:19.635 INFO:tasks.rados.rados.0.smithi063.stdout:2181: done (0 left) 2024-08-17T17:04:19.636 INFO:tasks.rados.rados.0.smithi063.stdout:2182: read oid 40 snap -1 2024-08-17T17:04:19.636 INFO:tasks.rados.rados.0.smithi063.stdout:2182: expect (ObjNum 725 snap 246 seq_num 725) 2024-08-17T17:04:19.636 INFO:tasks.rados.rados.0.smithi063.stdout:2183: delete oid 33 current snap is 283 2024-08-17T17:04:19.640 INFO:tasks.rados.rados.0.smithi063.stdout:2182: expect (ObjNum 725 snap 246 seq_num 725) 2024-08-17T17:04:19.713 INFO:tasks.rados.rados.0.smithi063.stdout:2182: done (1 left) 2024-08-17T17:04:19.713 INFO:tasks.rados.rados.0.smithi063.stdout:2183: done (0 left) 2024-08-17T17:04:19.713 INFO:tasks.rados.rados.0.smithi063.stdout:2184: write oid 31 current snap is 283 2024-08-17T17:04:19.713 INFO:tasks.rados.rados.0.smithi063.stdout:2184: seq_num 821 ranges {598547=629200,1876540=491946,3046277=681499} 2024-08-17T17:04:19.719 INFO:tasks.rados.rados.0.smithi063.stdout:2184: writing smithi06387500-31 from 598547 to 1227747 tid 1 2024-08-17T17:04:19.726 INFO:tasks.rados.rados.0.smithi063.stdout:2184: writing smithi06387500-31 from 1876540 to 2368486 tid 2 2024-08-17T17:04:19.733 INFO:tasks.rados.rados.0.smithi063.stdout:2184: writing smithi06387500-31 from 3046277 to 3727776 tid 3 2024-08-17T17:04:19.733 INFO:tasks.rados.rados.0.smithi063.stdout:2185: snap_remove snap 274 2024-08-17T17:04:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:19 smithi063 ceph-mon[114756]: pgmap v402: 89 pgs: 89 active+clean; 215 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-08-17T17:04:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:19 smithi063 ceph-mon[114756]: osdmap e631: 8 total, 8 up, 8 in 2024-08-17T17:04:20.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:19 smithi063 ceph-mon[109843]: pgmap v402: 89 pgs: 89 active+clean; 215 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-08-17T17:04:20.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:19 smithi063 ceph-mon[109843]: osdmap e631: 8 total, 8 up, 8 in 2024-08-17T17:04:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:19 smithi162 ceph-mon[104668]: pgmap v402: 89 pgs: 89 active+clean; 215 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-08-17T17:04:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:19 smithi162 ceph-mon[104668]: osdmap e631: 8 total, 8 up, 8 in 2024-08-17T17:04:20.655 INFO:tasks.rados.rados.0.smithi063.stdout:2184: finishing write tid 1 to smithi06387500-31 2024-08-17T17:04:20.655 INFO:tasks.rados.rados.0.smithi063.stdout:2184: finishing write tid 2 to smithi06387500-31 2024-08-17T17:04:20.655 INFO:tasks.rados.rados.0.smithi063.stdout:2184: finishing write tid 3 to smithi06387500-31 2024-08-17T17:04:20.655 INFO:tasks.rados.rados.0.smithi063.stdout:2184: finishing write tid 4 to smithi06387500-31 2024-08-17T17:04:20.655 INFO:tasks.rados.rados.0.smithi063.stdout:2184: finishing write tid 5 to smithi06387500-31 2024-08-17T17:04:20.655 INFO:tasks.rados.rados.0.smithi063.stdout:2184: oid 31 updating version 0 to 1422 2024-08-17T17:04:20.655 INFO:tasks.rados.rados.0.smithi063.stdout:2184: oid 31 updating version 1422 to 1423 2024-08-17T17:04:20.655 INFO:tasks.rados.rados.0.smithi063.stdout:2184: oid 31 updating version 1423 to 1424 2024-08-17T17:04:20.655 INFO:tasks.rados.rados.0.smithi063.stdout:2184: oid 31 version 1424 is already newer than 1421 2024-08-17T17:04:20.655 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 1424 (ObjNum 821 snap 283 seq_num 821) dirty exists 2024-08-17T17:04:20.656 INFO:tasks.rados.rados.0.smithi063.stdout:2184: left oid 31 (ObjNum 821 snap 283 seq_num 821) 2024-08-17T17:04:20.656 INFO:tasks.rados.rados.0.smithi063.stdout:2184: done (1 left) 2024-08-17T17:04:20.656 INFO:tasks.rados.rados.0.smithi063.stdout:2185: done (0 left) 2024-08-17T17:04:20.656 INFO:tasks.rados.rados.0.smithi063.stdout:2186: read oid 33 snap 280 2024-08-17T17:04:20.656 INFO:tasks.rados.rados.0.smithi063.stdout:2186: expect deleted 2024-08-17T17:04:20.656 INFO:tasks.rados.rados.0.smithi063.stdout:2187: write oid 14 current snap is 283 2024-08-17T17:04:20.656 INFO:tasks.rados.rados.0.smithi063.stdout:2187: seq_num 822 ranges {615354=420802,1651375=751957,3190053=492771} 2024-08-17T17:04:20.661 INFO:tasks.rados.rados.0.smithi063.stdout:2187: writing smithi06387500-14 from 615354 to 1036156 tid 1 2024-08-17T17:04:20.668 INFO:tasks.rados.rados.0.smithi063.stdout:2187: writing smithi06387500-14 from 1651375 to 2403332 tid 2 2024-08-17T17:04:20.676 INFO:tasks.rados.rados.0.smithi063.stdout:2187: writing smithi06387500-14 from 3190053 to 3682824 tid 3 2024-08-17T17:04:20.676 INFO:tasks.rados.rados.0.smithi063.stdout:2188: read oid 31 snap -1 2024-08-17T17:04:20.676 INFO:tasks.rados.rados.0.smithi063.stdout:2188: expect (ObjNum 821 snap 283 seq_num 821) 2024-08-17T17:04:20.676 INFO:tasks.rados.rados.0.smithi063.stdout:2189: write oid 4 current snap is 283 2024-08-17T17:04:20.676 INFO:tasks.rados.rados.0.smithi063.stdout:2189: seq_num 823 ranges {632161=496051,1593504=495615,2701123=420396,3611645=220516} 2024-08-17T17:04:20.682 INFO:tasks.rados.rados.0.smithi063.stdout:2189: writing smithi06387500-4 from 632161 to 1128212 tid 1 2024-08-17T17:04:20.687 INFO:tasks.rados.rados.0.smithi063.stdout:2189: writing smithi06387500-4 from 1593504 to 2089119 tid 2 2024-08-17T17:04:20.693 INFO:tasks.rados.rados.0.smithi063.stdout:2189: writing smithi06387500-4 from 2701123 to 3121519 tid 3 2024-08-17T17:04:20.697 INFO:tasks.rados.rados.0.smithi063.stdout:2189: writing smithi06387500-4 from 3611645 to 3832161 tid 4 2024-08-17T17:04:20.697 INFO:tasks.rados.rados.0.smithi063.stdout:2190: delete oid 45 current snap is 283 2024-08-17T17:04:20.697 INFO:tasks.rados.rados.0.smithi063.stdout:2187: finishing write tid 1 to smithi06387500-14 2024-08-17T17:04:20.697 INFO:tasks.rados.rados.0.smithi063.stdout:2187: finishing write tid 2 to smithi06387500-14 2024-08-17T17:04:20.697 INFO:tasks.rados.rados.0.smithi063.stdout:2187: finishing write tid 3 to smithi06387500-14 2024-08-17T17:04:20.697 INFO:tasks.rados.rados.0.smithi063.stdout:2187: finishing write tid 4 to smithi06387500-14 2024-08-17T17:04:20.697 INFO:tasks.rados.rados.0.smithi063.stdout:2187: finishing write tid 5 to smithi06387500-14 2024-08-17T17:04:20.697 INFO:tasks.rados.rados.0.smithi063.stdout:2187: oid 14 updating version 0 to 2095 2024-08-17T17:04:20.698 INFO:tasks.rados.rados.0.smithi063.stdout:2187: oid 14 updating version 2095 to 2098 2024-08-17T17:04:20.698 INFO:tasks.rados.rados.0.smithi063.stdout:2187: oid 14 version 2098 is already newer than 2096 2024-08-17T17:04:20.698 INFO:tasks.rados.rados.0.smithi063.stdout:2187: oid 14 version 2098 is already newer than 2097 2024-08-17T17:04:20.698 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 2098 (ObjNum 822 snap 283 seq_num 822) dirty exists 2024-08-17T17:04:20.698 INFO:tasks.rados.rados.0.smithi063.stdout:2187: left oid 14 (ObjNum 822 snap 283 seq_num 822) 2024-08-17T17:04:20.698 INFO:tasks.rados.rados.0.smithi063.stdout:2188: expect (ObjNum 821 snap 283 seq_num 821) 2024-08-17T17:04:20.876 INFO:tasks.rados.rados.0.smithi063.stdout:2189: finishing write tid 1 to smithi06387500-4 2024-08-17T17:04:20.876 INFO:tasks.rados.rados.0.smithi063.stdout:2189: finishing write tid 2 to smithi06387500-4 2024-08-17T17:04:20.876 INFO:tasks.rados.rados.0.smithi063.stdout:2189: finishing write tid 3 to smithi06387500-4 2024-08-17T17:04:20.876 INFO:tasks.rados.rados.0.smithi063.stdout:2189: finishing write tid 4 to smithi06387500-4 2024-08-17T17:04:20.876 INFO:tasks.rados.rados.0.smithi063.stdout:2189: finishing write tid 5 to smithi06387500-4 2024-08-17T17:04:20.876 INFO:tasks.rados.rados.0.smithi063.stdout:2189: finishing write tid 6 to smithi06387500-4 2024-08-17T17:04:20.877 INFO:tasks.rados.rados.0.smithi063.stdout:2189: oid 4 updating version 0 to 2103 2024-08-17T17:04:20.877 INFO:tasks.rados.rados.0.smithi063.stdout:2189: oid 4 version 2103 is already newer than 2102 2024-08-17T17:04:20.877 INFO:tasks.rados.rados.0.smithi063.stdout:2189: oid 4 version 2103 is already newer than 2100 2024-08-17T17:04:20.877 INFO:tasks.rados.rados.0.smithi063.stdout:2189: oid 4 version 2103 is already newer than 2101 2024-08-17T17:04:20.877 INFO:tasks.rados.rados.0.smithi063.stdout:2189: oid 4 updating version 2103 to 2104 2024-08-17T17:04:20.877 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 2104 (ObjNum 823 snap 283 seq_num 823) dirty exists 2024-08-17T17:04:20.877 INFO:tasks.rados.rados.0.smithi063.stdout:2189: left oid 4 (ObjNum 823 snap 283 seq_num 823) 2024-08-17T17:04:20.877 INFO:tasks.rados.rados.0.smithi063.stdout:2186: done (4 left) 2024-08-17T17:04:20.877 INFO:tasks.rados.rados.0.smithi063.stdout:2187: done (3 left) 2024-08-17T17:04:20.877 INFO:tasks.rados.rados.0.smithi063.stdout:2188: done (2 left) 2024-08-17T17:04:20.877 INFO:tasks.rados.rados.0.smithi063.stdout:2189: done (1 left) 2024-08-17T17:04:20.877 INFO:tasks.rados.rados.0.smithi063.stdout:2190: done (0 left) 2024-08-17T17:04:20.877 INFO:tasks.rados.rados.0.smithi063.stdout:2191: snap_remove snap 240 2024-08-17T17:04:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:20 smithi063 ceph-mon[114756]: osdmap e632: 8 total, 8 up, 8 in 2024-08-17T17:04:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:20 smithi063 ceph-mon[114756]: pgmap v405: 89 pgs: 89 active+clean; 215 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-08-17T17:04:21.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:20 smithi063 ceph-mon[109843]: osdmap e632: 8 total, 8 up, 8 in 2024-08-17T17:04:21.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:20 smithi063 ceph-mon[109843]: pgmap v405: 89 pgs: 89 active+clean; 215 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-08-17T17:04:21.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:20 smithi162 ceph-mon[104668]: osdmap e632: 8 total, 8 up, 8 in 2024-08-17T17:04:21.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:20 smithi162 ceph-mon[104668]: pgmap v405: 89 pgs: 89 active+clean; 215 MiB data, 762 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-08-17T17:04:21.648 INFO:tasks.rados.rados.0.smithi063.stdout:2191: done (0 left) 2024-08-17T17:04:21.648 INFO:tasks.rados.rados.0.smithi063.stdout:2192: write oid 25 current snap is 283 2024-08-17T17:04:21.649 INFO:tasks.rados.rados.0.smithi063.stdout:2192: seq_num 825 ranges {665775=646549,2110468=499284,3155969=442940} 2024-08-17T17:04:21.656 INFO:tasks.rados.rados.0.smithi063.stdout:2192: writing smithi06387500-25 from 665775 to 1312324 tid 1 2024-08-17T17:04:21.664 INFO:tasks.rados.rados.0.smithi063.stdout:2192: writing smithi06387500-25 from 2110468 to 2609752 tid 2 2024-08-17T17:04:21.671 INFO:tasks.rados.rados.0.smithi063.stdout:2192: writing smithi06387500-25 from 3155969 to 3598909 tid 3 2024-08-17T17:04:21.671 INFO:tasks.rados.rados.0.smithi063.stdout:2193: read oid 33 snap -1 2024-08-17T17:04:21.671 INFO:tasks.rados.rados.0.smithi063.stdout:2193: expect deleted 2024-08-17T17:04:21.671 INFO:tasks.rados.rados.0.smithi063.stdout:2194: write oid 50 current snap is 283 2024-08-17T17:04:21.672 INFO:tasks.rados.rados.0.smithi063.stdout:2194: seq_num 826 ranges {682582=721798,2168950=642942,3583392=299190} 2024-08-17T17:04:21.679 INFO:tasks.rados.rados.0.smithi063.stdout:2194: writing smithi06387500-50 from 682582 to 1404380 tid 1 2024-08-17T17:04:21.687 INFO:tasks.rados.rados.0.smithi063.stdout:2194: writing smithi06387500-50 from 2168950 to 2811892 tid 2 2024-08-17T17:04:21.693 INFO:tasks.rados.rados.0.smithi063.stdout:2194: writing smithi06387500-50 from 3583392 to 3882582 tid 3 2024-08-17T17:04:21.693 INFO:tasks.rados.rados.0.smithi063.stdout:2195: delete oid 42 current snap is 283 2024-08-17T17:04:21.693 INFO:tasks.rados.rados.0.smithi063.stdout:2192: finishing write tid 1 to smithi06387500-25 2024-08-17T17:04:21.693 INFO:tasks.rados.rados.0.smithi063.stdout:2192: finishing write tid 2 to smithi06387500-25 2024-08-17T17:04:21.693 INFO:tasks.rados.rados.0.smithi063.stdout:2192: finishing write tid 3 to smithi06387500-25 2024-08-17T17:04:21.694 INFO:tasks.rados.rados.0.smithi063.stdout:2192: finishing write tid 4 to smithi06387500-25 2024-08-17T17:04:21.694 INFO:tasks.rados.rados.0.smithi063.stdout:2192: finishing write tid 5 to smithi06387500-25 2024-08-17T17:04:21.694 INFO:tasks.rados.rados.0.smithi063.stdout:2192: oid 25 updating version 0 to 2115 2024-08-17T17:04:21.694 INFO:tasks.rados.rados.0.smithi063.stdout:2192: oid 25 updating version 2115 to 2120 2024-08-17T17:04:21.694 INFO:tasks.rados.rados.0.smithi063.stdout:2192: oid 25 version 2120 is already newer than 2110 2024-08-17T17:04:21.694 INFO:tasks.rados.rados.0.smithi063.stdout:2192: oid 25 updating version 2120 to 2121 2024-08-17T17:04:21.694 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 2121 (ObjNum 825 snap 283 seq_num 825) dirty exists 2024-08-17T17:04:21.694 INFO:tasks.rados.rados.0.smithi063.stdout:2192: left oid 25 (ObjNum 825 snap 283 seq_num 825) 2024-08-17T17:04:21.694 INFO:tasks.rados.rados.0.smithi063.stdout:2194: finishing write tid 1 to smithi06387500-50 2024-08-17T17:04:21.694 INFO:tasks.rados.rados.0.smithi063.stdout:2194: finishing write tid 2 to smithi06387500-50 2024-08-17T17:04:21.695 INFO:tasks.rados.rados.0.smithi063.stdout:2194: finishing write tid 3 to smithi06387500-50 2024-08-17T17:04:21.697 INFO:tasks.rados.rados.0.smithi063.stdout:2194: finishing write tid 4 to smithi06387500-50 2024-08-17T17:04:21.697 INFO:tasks.rados.rados.0.smithi063.stdout:2194: finishing write tid 5 to smithi06387500-50 2024-08-17T17:04:21.697 INFO:tasks.rados.rados.0.smithi063.stdout:2194: oid 50 updating version 0 to 1434 2024-08-17T17:04:21.697 INFO:tasks.rados.rados.0.smithi063.stdout:2194: oid 50 version 1434 is already newer than 1433 2024-08-17T17:04:21.698 INFO:tasks.rados.rados.0.smithi063.stdout:2194: oid 50 version 1434 is already newer than 1432 2024-08-17T17:04:21.698 INFO:tasks.rados.rados.0.smithi063.stdout:2194: oid 50 updating version 1434 to 1435 2024-08-17T17:04:21.698 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 1435 (ObjNum 826 snap 283 seq_num 826) dirty exists 2024-08-17T17:04:21.698 INFO:tasks.rados.rados.0.smithi063.stdout:2194: left oid 50 (ObjNum 826 snap 283 seq_num 826) 2024-08-17T17:04:21.698 INFO:tasks.rados.rados.0.smithi063.stdout:2192: done (3 left) 2024-08-17T17:04:21.698 INFO:tasks.rados.rados.0.smithi063.stdout:2193: done (2 left) 2024-08-17T17:04:21.698 INFO:tasks.rados.rados.0.smithi063.stdout:2194: done (1 left) 2024-08-17T17:04:21.698 INFO:tasks.rados.rados.0.smithi063.stdout:2195: done (0 left) 2024-08-17T17:04:21.698 INFO:tasks.rados.rados.0.smithi063.stdout:2196: write oid 12 current snap is 283 2024-08-17T17:04:21.698 INFO:tasks.rados.rados.0.smithi063.stdout:2196: seq_num 828 ranges {716196=472296,1769561=646611,3121885=793109} 2024-08-17T17:04:21.704 INFO:tasks.rados.rados.0.smithi063.stdout:2196: writing smithi06387500-12 from 716196 to 1188492 tid 1 2024-08-17T17:04:21.710 INFO:tasks.rados.rados.0.smithi063.stdout:2196: writing smithi06387500-12 from 1769561 to 2416172 tid 2 2024-08-17T17:04:21.719 INFO:tasks.rados.rados.0.smithi063.stdout:2196: writing smithi06387500-12 from 3121885 to 3914994 tid 3 2024-08-17T17:04:21.719 INFO:tasks.rados.rados.0.smithi063.stdout:2197: rollback oid 38 current snap is 283 2024-08-17T17:04:21.720 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 38 to 282 2024-08-17T17:04:21.720 INFO:tasks.rados.rados.0.smithi063.stdout:2196: finishing write tid 1 to smithi06387500-12 2024-08-17T17:04:21.720 INFO:tasks.rados.rados.0.smithi063.stdout:2196: finishing write tid 2 to smithi06387500-12 2024-08-17T17:04:21.720 INFO:tasks.rados.rados.0.smithi063.stdout:2198: read oid 34 snap 278 2024-08-17T17:04:21.720 INFO:tasks.rados.rados.0.smithi063.stdout:2198: expect deleted 2024-08-17T17:04:21.720 INFO:tasks.rados.rados.0.smithi063.stdout:2199: snap_create 2024-08-17T17:04:21.721 INFO:tasks.rados.rados.0.smithi063.stdout:2197: finishing rollback tid 1 to smithi06387500-38 2024-08-17T17:04:21.721 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 323 (ObjNum 289 snap 0 seq_num 1027751720) dirty dne 2024-08-17T17:04:21.722 INFO:tasks.rados.rados.0.smithi063.stdout:2196: finishing write tid 3 to smithi06387500-12 2024-08-17T17:04:21.722 INFO:tasks.rados.rados.0.smithi063.stdout:2196: finishing write tid 4 to smithi06387500-12 2024-08-17T17:04:21.723 INFO:tasks.rados.rados.0.smithi063.stdout:2196: finishing write tid 5 to smithi06387500-12 2024-08-17T17:04:21.723 INFO:tasks.rados.rados.0.smithi063.stdout:2196: oid 12 updating version 0 to 523 2024-08-17T17:04:21.723 INFO:tasks.rados.rados.0.smithi063.stdout:2196: oid 12 updating version 523 to 524 2024-08-17T17:04:21.723 INFO:tasks.rados.rados.0.smithi063.stdout:2196: oid 12 version 524 is already newer than 521 2024-08-17T17:04:21.723 INFO:tasks.rados.rados.0.smithi063.stdout:2196: oid 12 version 524 is already newer than 522 2024-08-17T17:04:21.723 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 524 (ObjNum 828 snap 283 seq_num 828) dirty exists 2024-08-17T17:04:21.723 INFO:tasks.rados.rados.0.smithi063.stdout:2196: left oid 12 (ObjNum 828 snap 283 seq_num 828) 2024-08-17T17:04:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:21 smithi063 ceph-mon[114756]: osdmap e633: 8 total, 8 up, 8 in 2024-08-17T17:04:22.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:21 smithi063 ceph-mon[109843]: osdmap e633: 8 total, 8 up, 8 in 2024-08-17T17:04:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:21 smithi162 ceph-mon[104668]: osdmap e633: 8 total, 8 up, 8 in 2024-08-17T17:04:22.569 INFO:tasks.rados.rados.0.smithi063.stdout:2196: done (3 left) 2024-08-17T17:04:22.569 INFO:tasks.rados.rados.0.smithi063.stdout:2197: done (2 left) 2024-08-17T17:04:22.569 INFO:tasks.rados.rados.0.smithi063.stdout:2198: done (1 left) 2024-08-17T17:04:22.570 INFO:tasks.rados.rados.0.smithi063.stdout:2199: done (0 left) 2024-08-17T17:04:22.570 INFO:tasks.rados.rados.0.smithi063.stdout:2200: rollback oid 48 current snap is 284 2024-08-17T17:04:22.570 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 48 to 281 2024-08-17T17:04:22.570 INFO:tasks.rados.rados.0.smithi063.stdout:2201: snap_remove snap 275 2024-08-17T17:04:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:22 smithi063 ceph-mon[114756]: osdmap e634: 8 total, 8 up, 8 in 2024-08-17T17:04:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:22 smithi063 ceph-mon[114756]: pgmap v408: 89 pgs: 89 active+clean; 208 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 3.8 MiB/s wr, 21 op/s 2024-08-17T17:04:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:22 smithi063 ceph-mon[114756]: osdmap e635: 8 total, 8 up, 8 in 2024-08-17T17:04:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:22 smithi063 ceph-mon[109843]: osdmap e634: 8 total, 8 up, 8 in 2024-08-17T17:04:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:22 smithi063 ceph-mon[109843]: pgmap v408: 89 pgs: 89 active+clean; 208 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 3.8 MiB/s wr, 21 op/s 2024-08-17T17:04:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:22 smithi063 ceph-mon[109843]: osdmap e635: 8 total, 8 up, 8 in 2024-08-17T17:04:23.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:22 smithi162 ceph-mon[104668]: osdmap e634: 8 total, 8 up, 8 in 2024-08-17T17:04:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:22 smithi162 ceph-mon[104668]: pgmap v408: 89 pgs: 89 active+clean; 208 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 3.8 MiB/s wr, 21 op/s 2024-08-17T17:04:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:22 smithi162 ceph-mon[104668]: osdmap e635: 8 total, 8 up, 8 in 2024-08-17T17:04:23.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:04:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:04:23] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.51.0" 2024-08-17T17:04:23.571 INFO:tasks.rados.rados.0.smithi063.stdout:2201: done (1 left) 2024-08-17T17:04:23.571 INFO:tasks.rados.rados.0.smithi063.stdout:2202: delete oid 33 current snap is 284 2024-08-17T17:04:23.571 INFO:tasks.rados.rados.0.smithi063.stdout:2200: finishing rollback tid 0 to smithi06387500-48 2024-08-17T17:04:23.571 INFO:tasks.rados.rados.0.smithi063.stdout:2200: finishing rollback tid 1 to smithi06387500-48 2024-08-17T17:04:23.571 INFO:tasks.rados.rados.0.smithi063.stdout:2200: finishing rollback tid 2 to smithi06387500-48 2024-08-17T17:04:23.572 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 2125 (ObjNum 730 snap 246 seq_num 730) dirty exists 2024-08-17T17:04:23.579 INFO:tasks.rados.rados.0.smithi063.stdout:2200: done (1 left) 2024-08-17T17:04:23.579 INFO:tasks.rados.rados.0.smithi063.stdout:2202: done (0 left) 2024-08-17T17:04:23.579 INFO:tasks.rados.rados.0.smithi063.stdout:2203: read oid 35 snap -1 2024-08-17T17:04:23.579 INFO:tasks.rados.rados.0.smithi063.stdout:2203: expect deleted 2024-08-17T17:04:23.579 INFO:tasks.rados.rados.0.smithi063.stdout:2204: read oid 30 snap -1 2024-08-17T17:04:23.579 INFO:tasks.rados.rados.0.smithi063.stdout:2204: expect (ObjNum 710 snap 234 seq_num 710) 2024-08-17T17:04:23.579 INFO:tasks.rados.rados.0.smithi063.stdout:2205: snap_create 2024-08-17T17:04:23.593 INFO:tasks.rados.rados.0.smithi063.stdout:2204: expect (ObjNum 710 snap 234 seq_num 710) 2024-08-17T17:04:24.579 INFO:tasks.rados.rados.0.smithi063.stdout:2203: done (2 left) 2024-08-17T17:04:24.580 INFO:tasks.rados.rados.0.smithi063.stdout:2204: done (1 left) 2024-08-17T17:04:24.580 INFO:tasks.rados.rados.0.smithi063.stdout:2205: done (0 left) 2024-08-17T17:04:24.580 INFO:tasks.rados.rados.0.smithi063.stdout:2206: snap_create 2024-08-17T17:04:24.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:24 smithi162 ceph-mon[104668]: osdmap e636: 8 total, 8 up, 8 in 2024-08-17T17:04:25.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:24 smithi063 ceph-mon[114756]: osdmap e636: 8 total, 8 up, 8 in 2024-08-17T17:04:25.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:24 smithi063 ceph-mon[109843]: osdmap e636: 8 total, 8 up, 8 in 2024-08-17T17:04:25.586 INFO:tasks.rados.rados.0.smithi063.stdout:2206: done (0 left) 2024-08-17T17:04:25.588 INFO:tasks.rados.rados.0.smithi063.stdout:2207: rollback oid 14 current snap is 286 2024-08-17T17:04:25.588 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 14 to 283 2024-08-17T17:04:25.588 INFO:tasks.rados.rados.0.smithi063.stdout:2208: snap_create 2024-08-17T17:04:25.593 INFO:tasks.rados.rados.0.smithi063.stdout:2207: finishing rollback tid 0 to smithi06387500-14 2024-08-17T17:04:25.593 INFO:tasks.rados.rados.0.smithi063.stdout:2207: finishing rollback tid 1 to smithi06387500-14 2024-08-17T17:04:25.593 INFO:tasks.rados.rados.0.smithi063.stdout:2207: finishing rollback tid 2 to smithi06387500-14 2024-08-17T17:04:25.593 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 2137 (ObjNum 822 snap 283 seq_num 822) dirty exists 2024-08-17T17:04:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:25 smithi162 ceph-mon[104668]: pgmap v411: 89 pgs: 89 active+clean; 208 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 3.8 MiB/s wr, 21 op/s 2024-08-17T17:04:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:25 smithi162 ceph-mon[104668]: osdmap e637: 8 total, 8 up, 8 in 2024-08-17T17:04:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:25 smithi063 ceph-mon[114756]: pgmap v411: 89 pgs: 89 active+clean; 208 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 3.8 MiB/s wr, 21 op/s 2024-08-17T17:04:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:25 smithi063 ceph-mon[114756]: osdmap e637: 8 total, 8 up, 8 in 2024-08-17T17:04:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:25 smithi063 ceph-mon[109843]: pgmap v411: 89 pgs: 89 active+clean; 208 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 3.8 MiB/s wr, 21 op/s 2024-08-17T17:04:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:25 smithi063 ceph-mon[109843]: osdmap e637: 8 total, 8 up, 8 in 2024-08-17T17:04:26.594 INFO:tasks.rados.rados.0.smithi063.stdout:2207: done (1 left) 2024-08-17T17:04:26.594 INFO:tasks.rados.rados.0.smithi063.stdout:2208: done (0 left) 2024-08-17T17:04:26.594 INFO:tasks.rados.rados.0.smithi063.stdout:2209: rollback oid 41 current snap is 287 2024-08-17T17:04:26.594 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 41 to 282 2024-08-17T17:04:26.594 INFO:tasks.rados.rados.0.smithi063.stdout:2210: snap_remove snap 277 2024-08-17T17:04:26.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:26 smithi162 ceph-mon[104668]: osdmap e638: 8 total, 8 up, 8 in 2024-08-17T17:04:27.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:26 smithi063 ceph-mon[114756]: osdmap e638: 8 total, 8 up, 8 in 2024-08-17T17:04:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:26 smithi063 ceph-mon[109843]: osdmap e638: 8 total, 8 up, 8 in 2024-08-17T17:04:27.576 INFO:tasks.rados.rados.0.smithi063.stdout:2210: done (1 left) 2024-08-17T17:04:27.576 INFO:tasks.rados.rados.0.smithi063.stdout:2211: read oid 21 snap -1 2024-08-17T17:04:27.576 INFO:tasks.rados.rados.0.smithi063.stdout:2211: expect deleted 2024-08-17T17:04:27.576 INFO:tasks.rados.rados.0.smithi063.stdout:2212: write oid 44 current snap is 287 2024-08-17T17:04:27.576 INFO:tasks.rados.rados.0.smithi063.stdout:2212: seq_num 830 ranges {749810=739147,2002878=650280,3293084=415653} 2024-08-17T17:04:27.585 INFO:tasks.rados.rados.0.smithi063.stdout:2212: writing smithi06387500-44 from 749810 to 1488957 tid 1 2024-08-17T17:04:27.592 INFO:tasks.rados.rados.0.smithi063.stdout:2212: writing smithi06387500-44 from 2002878 to 2653158 tid 2 2024-08-17T17:04:27.598 INFO:tasks.rados.rados.0.smithi063.stdout:2212: writing smithi06387500-44 from 3293084 to 3708737 tid 3 2024-08-17T17:04:27.598 INFO:tasks.rados.rados.0.smithi063.stdout:2213: rollback oid 29 current snap is 287 2024-08-17T17:04:27.598 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 29 to 284 2024-08-17T17:04:27.598 INFO:tasks.rados.rados.0.smithi063.stdout:2209: finishing rollback tid 1 to smithi06387500-41 2024-08-17T17:04:27.598 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 193 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T17:04:27.599 INFO:tasks.rados.rados.0.smithi063.stdout:2212: finishing write tid 1 to smithi06387500-44 2024-08-17T17:04:27.599 INFO:tasks.rados.rados.0.smithi063.stdout:2212: finishing write tid 2 to smithi06387500-44 2024-08-17T17:04:27.599 INFO:tasks.rados.rados.0.smithi063.stdout:2209: done (3 left) 2024-08-17T17:04:27.599 INFO:tasks.rados.rados.0.smithi063.stdout:2211: done (2 left) 2024-08-17T17:04:27.599 INFO:tasks.rados.rados.0.smithi063.stdout:2214: write oid 43 current snap is 287 2024-08-17T17:04:27.599 INFO:tasks.rados.rados.0.smithi063.stdout:2214: seq_num 831 ranges {766617=414396,1945007=510291,3204154=626925} 2024-08-17T17:04:27.604 INFO:tasks.rados.rados.0.smithi063.stdout:2214: writing smithi06387500-43 from 766617 to 1181013 tid 1 2024-08-17T17:04:27.611 INFO:tasks.rados.rados.0.smithi063.stdout:2214: writing smithi06387500-43 from 1945007 to 2455298 tid 2 2024-08-17T17:04:27.619 INFO:tasks.rados.rados.0.smithi063.stdout:2214: writing smithi06387500-43 from 3204154 to 3831079 tid 3 2024-08-17T17:04:27.619 INFO:tasks.rados.rados.0.smithi063.stdout:2215: read oid 46 snap -1 2024-08-17T17:04:27.619 INFO:tasks.rados.rados.0.smithi063.stdout:2215: expect deleted 2024-08-17T17:04:27.620 INFO:tasks.rados.rados.0.smithi063.stdout:2216: snap_create 2024-08-17T17:04:27.620 INFO:tasks.rados.rados.0.smithi063.stdout:2212: finishing write tid 3 to smithi06387500-44 2024-08-17T17:04:27.620 INFO:tasks.rados.rados.0.smithi063.stdout:2213: finishing rollback tid 0 to smithi06387500-29 2024-08-17T17:04:27.620 INFO:tasks.rados.rados.0.smithi063.stdout:2212: finishing write tid 4 to smithi06387500-44 2024-08-17T17:04:27.620 INFO:tasks.rados.rados.0.smithi063.stdout:2212: finishing write tid 5 to smithi06387500-44 2024-08-17T17:04:27.620 INFO:tasks.rados.rados.0.smithi063.stdout:2212: oid 44 updating version 0 to 335 2024-08-17T17:04:27.620 INFO:tasks.rados.rados.0.smithi063.stdout:2212: oid 44 updating version 335 to 338 2024-08-17T17:04:27.620 INFO:tasks.rados.rados.0.smithi063.stdout:2212: oid 44 version 338 is already newer than 337 2024-08-17T17:04:27.620 INFO:tasks.rados.rados.0.smithi063.stdout:2212: oid 44 version 338 is already newer than 336 2024-08-17T17:04:27.620 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 338 (ObjNum 830 snap 287 seq_num 830) dirty exists 2024-08-17T17:04:27.620 INFO:tasks.rados.rados.0.smithi063.stdout:2212: left oid 44 (ObjNum 830 snap 287 seq_num 830) 2024-08-17T17:04:27.620 INFO:tasks.rados.rados.0.smithi063.stdout:2213: finishing rollback tid 1 to smithi06387500-29 2024-08-17T17:04:27.620 INFO:tasks.rados.rados.0.smithi063.stdout:2213: finishing rollback tid 2 to smithi06387500-29 2024-08-17T17:04:27.621 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 532 (ObjNum 817 snap 282 seq_num 817) dirty exists 2024-08-17T17:04:27.621 INFO:tasks.rados.rados.0.smithi063.stdout:2214: finishing write tid 1 to smithi06387500-43 2024-08-17T17:04:27.621 INFO:tasks.rados.rados.0.smithi063.stdout:2214: finishing write tid 2 to smithi06387500-43 2024-08-17T17:04:27.623 INFO:tasks.rados.rados.0.smithi063.stdout:2214: finishing write tid 3 to smithi06387500-43 2024-08-17T17:04:27.623 INFO:tasks.rados.rados.0.smithi063.stdout:2214: finishing write tid 4 to smithi06387500-43 2024-08-17T17:04:27.624 INFO:tasks.rados.rados.0.smithi063.stdout:2214: finishing write tid 5 to smithi06387500-43 2024-08-17T17:04:27.624 INFO:tasks.rados.rados.0.smithi063.stdout:2214: oid 43 updating version 0 to 967 2024-08-17T17:04:27.624 INFO:tasks.rados.rados.0.smithi063.stdout:2214: oid 43 updating version 967 to 968 2024-08-17T17:04:27.624 INFO:tasks.rados.rados.0.smithi063.stdout:2214: oid 43 version 968 is already newer than 965 2024-08-17T17:04:27.624 INFO:tasks.rados.rados.0.smithi063.stdout:2214: oid 43 version 968 is already newer than 966 2024-08-17T17:04:27.624 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 968 (ObjNum 831 snap 287 seq_num 831) dirty exists 2024-08-17T17:04:27.624 INFO:tasks.rados.rados.0.smithi063.stdout:2214: left oid 43 (ObjNum 831 snap 287 seq_num 831) 2024-08-17T17:04:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:27 smithi162 ceph-mon[104668]: pgmap v414: 89 pgs: 89 active+clean; 208 MiB data, 773 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:04:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:27 smithi162 ceph-mon[104668]: osdmap e639: 8 total, 8 up, 8 in 2024-08-17T17:04:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:27 smithi162 ceph-mon[104668]: osdmap e640: 8 total, 8 up, 8 in 2024-08-17T17:04:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:27 smithi063 ceph-mon[114756]: pgmap v414: 89 pgs: 89 active+clean; 208 MiB data, 773 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:04:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:27 smithi063 ceph-mon[114756]: osdmap e639: 8 total, 8 up, 8 in 2024-08-17T17:04:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:27 smithi063 ceph-mon[114756]: osdmap e640: 8 total, 8 up, 8 in 2024-08-17T17:04:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:27 smithi063 ceph-mon[109843]: pgmap v414: 89 pgs: 89 active+clean; 208 MiB data, 773 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:04:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:27 smithi063 ceph-mon[109843]: osdmap e639: 8 total, 8 up, 8 in 2024-08-17T17:04:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:27 smithi063 ceph-mon[109843]: osdmap e640: 8 total, 8 up, 8 in 2024-08-17T17:04:28.603 INFO:tasks.rados.rados.0.smithi063.stdout:2212: done (4 left) 2024-08-17T17:04:28.604 INFO:tasks.rados.rados.0.smithi063.stdout:2213: done (3 left) 2024-08-17T17:04:28.604 INFO:tasks.rados.rados.0.smithi063.stdout:2214: done (2 left) 2024-08-17T17:04:28.604 INFO:tasks.rados.rados.0.smithi063.stdout:2215: done (1 left) 2024-08-17T17:04:28.604 INFO:tasks.rados.rados.0.smithi063.stdout:2216: done (0 left) 2024-08-17T17:04:28.604 INFO:tasks.rados.rados.0.smithi063.stdout:2217: write oid 44 current snap is 288 2024-08-17T17:04:28.604 INFO:tasks.rados.rados.0.smithi063.stdout:2217: seq_num 832 ranges {783424=489645,2003489=653949,3231577=554550} 2024-08-17T17:04:28.610 INFO:tasks.rados.rados.0.smithi063.stdout:2217: writing smithi06387500-44 from 783424 to 1273069 tid 1 2024-08-17T17:04:28.618 INFO:tasks.rados.rados.0.smithi063.stdout:2217: writing smithi06387500-44 from 2003489 to 2657438 tid 2 2024-08-17T17:04:28.625 INFO:tasks.rados.rados.0.smithi063.stdout:2217: writing smithi06387500-44 from 3231577 to 3786127 tid 3 2024-08-17T17:04:28.625 INFO:tasks.rados.rados.0.smithi063.stdout:2218: delete oid 13 current snap is 288 2024-08-17T17:04:28.625 INFO:tasks.rados.rados.0.smithi063.stdout:2217: finishing write tid 1 to smithi06387500-44 2024-08-17T17:04:28.625 INFO:tasks.rados.rados.0.smithi063.stdout:2217: finishing write tid 2 to smithi06387500-44 2024-08-17T17:04:28.627 INFO:tasks.rados.rados.0.smithi063.stdout:2218: done (1 left) 2024-08-17T17:04:28.627 INFO:tasks.rados.rados.0.smithi063.stdout:2219: read oid 46 snap -1 2024-08-17T17:04:28.627 INFO:tasks.rados.rados.0.smithi063.stdout:2219: expect deleted 2024-08-17T17:04:28.627 INFO:tasks.rados.rados.0.smithi063.stdout:2220: read oid 4 snap -1 2024-08-17T17:04:28.627 INFO:tasks.rados.rados.0.smithi063.stdout:2220: expect (ObjNum 823 snap 283 seq_num 823) 2024-08-17T17:04:28.627 INFO:tasks.rados.rados.0.smithi063.stdout:2221: snap_remove snap 280 2024-08-17T17:04:29.606 INFO:tasks.rados.rados.0.smithi063.stdout:2221: done (3 left) 2024-08-17T17:04:29.606 INFO:tasks.rados.rados.0.smithi063.stdout:2222: snap_remove snap 247 2024-08-17T17:04:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:29 smithi063 ceph-mon[114756]: pgmap v417: 89 pgs: 89 active+clean; 208 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 0 B/s wr, 2 op/s 2024-08-17T17:04:30.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:29 smithi063 ceph-mon[114756]: osdmap e641: 8 total, 8 up, 8 in 2024-08-17T17:04:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:29 smithi063 ceph-mon[109843]: pgmap v417: 89 pgs: 89 active+clean; 208 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 0 B/s wr, 2 op/s 2024-08-17T17:04:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:29 smithi063 ceph-mon[109843]: osdmap e641: 8 total, 8 up, 8 in 2024-08-17T17:04:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:29 smithi162 ceph-mon[104668]: pgmap v417: 89 pgs: 89 active+clean; 208 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 0 B/s wr, 2 op/s 2024-08-17T17:04:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:29 smithi162 ceph-mon[104668]: osdmap e641: 8 total, 8 up, 8 in 2024-08-17T17:04:30.617 INFO:tasks.rados.rados.0.smithi063.stdout:2217: finishing write tid 3 to smithi06387500-44 2024-08-17T17:04:30.617 INFO:tasks.rados.rados.0.smithi063.stdout:2217: finishing write tid 4 to smithi06387500-44 2024-08-17T17:04:30.617 INFO:tasks.rados.rados.0.smithi063.stdout:2217: finishing write tid 5 to smithi06387500-44 2024-08-17T17:04:30.617 INFO:tasks.rados.rados.0.smithi063.stdout:2217: oid 44 updating version 0 to 341 2024-08-17T17:04:30.617 INFO:tasks.rados.rados.0.smithi063.stdout:2217: oid 44 updating version 341 to 342 2024-08-17T17:04:30.617 INFO:tasks.rados.rados.0.smithi063.stdout:2217: oid 44 updating version 342 to 343 2024-08-17T17:04:30.617 INFO:tasks.rados.rados.0.smithi063.stdout:2217: oid 44 version 343 is already newer than 340 2024-08-17T17:04:30.617 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 343 (ObjNum 832 snap 288 seq_num 832) dirty exists 2024-08-17T17:04:30.617 INFO:tasks.rados.rados.0.smithi063.stdout:2217: left oid 44 (ObjNum 832 snap 288 seq_num 832) 2024-08-17T17:04:30.617 INFO:tasks.rados.rados.0.smithi063.stdout:2220: expect (ObjNum 823 snap 283 seq_num 823) 2024-08-17T17:04:30.787 INFO:tasks.rados.rados.0.smithi063.stdout:2217: done (3 left) 2024-08-17T17:04:30.787 INFO:tasks.rados.rados.0.smithi063.stdout:2219: done (2 left) 2024-08-17T17:04:30.787 INFO:tasks.rados.rados.0.smithi063.stdout:2220: done (1 left) 2024-08-17T17:04:30.787 INFO:tasks.rados.rados.0.smithi063.stdout:2222: done (0 left) 2024-08-17T17:04:30.787 INFO:tasks.rados.rados.0.smithi063.stdout:2223: write oid 18 current snap is 288 2024-08-17T17:04:30.787 INFO:tasks.rados.rados.0.smithi063.stdout:2223: seq_num 834 ranges {417038=640143,1720453=296585} 2024-08-17T17:04:30.792 INFO:tasks.rados.rados.0.smithi063.stdout:2223: writing smithi06387500-18 from 417038 to 1057181 tid 1 2024-08-17T17:04:30.797 INFO:tasks.rados.rados.0.smithi063.stdout:2223: writing smithi06387500-18 from 1720453 to 2017038 tid 2 2024-08-17T17:04:30.797 INFO:tasks.rados.rados.0.smithi063.stdout:2224: snap_remove snap 278 2024-08-17T17:04:30.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:30 smithi162 ceph-mon[104668]: osdmap e642: 8 total, 8 up, 8 in 2024-08-17T17:04:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:30 smithi063 ceph-mon[114756]: osdmap e642: 8 total, 8 up, 8 in 2024-08-17T17:04:31.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:30 smithi063 ceph-mon[109843]: osdmap e642: 8 total, 8 up, 8 in 2024-08-17T17:04:31.620 INFO:tasks.rados.rados.0.smithi063.stdout:2224: done (1 left) 2024-08-17T17:04:31.620 INFO:tasks.rados.rados.0.smithi063.stdout:2225: write oid 34 current snap is 288 2024-08-17T17:04:31.620 INFO:tasks.rados.rados.0.smithi063.stdout:2223: finishing write tid 1 to smithi06387500-18 2024-08-17T17:04:31.620 INFO:tasks.rados.rados.0.smithi063.stdout:2225: seq_num 835 ranges {433845=715392,1662582=371263} 2024-08-17T17:04:31.627 INFO:tasks.rados.rados.0.smithi063.stdout:2225: writing smithi06387500-34 from 433845 to 1149237 tid 1 2024-08-17T17:04:31.633 INFO:tasks.rados.rados.0.smithi063.stdout:2225: writing smithi06387500-34 from 1662582 to 2033845 tid 2 2024-08-17T17:04:31.633 INFO:tasks.rados.rados.0.smithi063.stdout:2226: delete oid 46 current snap is 288 2024-08-17T17:04:31.633 INFO:tasks.rados.rados.0.smithi063.stdout:2223: finishing write tid 2 to smithi06387500-18 2024-08-17T17:04:31.633 INFO:tasks.rados.rados.0.smithi063.stdout:2223: finishing write tid 3 to smithi06387500-18 2024-08-17T17:04:31.633 INFO:tasks.rados.rados.0.smithi063.stdout:2223: finishing write tid 4 to smithi06387500-18 2024-08-17T17:04:31.633 INFO:tasks.rados.rados.0.smithi063.stdout:2223: oid 18 updating version 0 to 710 2024-08-17T17:04:31.633 INFO:tasks.rados.rados.0.smithi063.stdout:2223: oid 18 version 710 is already newer than 709 2024-08-17T17:04:31.634 INFO:tasks.rados.rados.0.smithi063.stdout:2223: oid 18 version 710 is already newer than 708 2024-08-17T17:04:31.634 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 710 (ObjNum 834 snap 288 seq_num 834) dirty exists 2024-08-17T17:04:31.634 INFO:tasks.rados.rados.0.smithi063.stdout:2223: left oid 18 (ObjNum 834 snap 288 seq_num 834) 2024-08-17T17:04:31.636 INFO:tasks.rados.rados.0.smithi063.stdout:2223: done (2 left) 2024-08-17T17:04:31.636 INFO:tasks.rados.rados.0.smithi063.stdout:2226: done (1 left) 2024-08-17T17:04:31.636 INFO:tasks.rados.rados.0.smithi063.stdout:2227: read oid 22 snap -1 2024-08-17T17:04:31.637 INFO:tasks.rados.rados.0.smithi063.stdout:2227: expect (ObjNum 626 snap 199 seq_num 626) 2024-08-17T17:04:31.637 INFO:tasks.rados.rados.0.smithi063.stdout:2228: write oid 8 current snap is 288 2024-08-17T17:04:31.637 INFO:tasks.rados.rados.0.smithi063.stdout:2228: seq_num 837 ranges {467459=582243,1495899=404945} 2024-08-17T17:04:31.642 INFO:tasks.rados.rados.0.smithi063.stdout:2228: writing smithi06387500-8 from 467459 to 1049702 tid 1 2024-08-17T17:04:31.648 INFO:tasks.rados.rados.0.smithi063.stdout:2228: writing smithi06387500-8 from 1495899 to 1900844 tid 2 2024-08-17T17:04:31.648 INFO:tasks.rados.rados.0.smithi063.stdout:2229: snap_create 2024-08-17T17:04:31.648 INFO:tasks.rados.rados.0.smithi063.stdout:2225: finishing write tid 1 to smithi06387500-34 2024-08-17T17:04:31.648 INFO:tasks.rados.rados.0.smithi063.stdout:2225: finishing write tid 2 to smithi06387500-34 2024-08-17T17:04:31.648 INFO:tasks.rados.rados.0.smithi063.stdout:2225: finishing write tid 3 to smithi06387500-34 2024-08-17T17:04:31.648 INFO:tasks.rados.rados.0.smithi063.stdout:2225: finishing write tid 4 to smithi06387500-34 2024-08-17T17:04:31.648 INFO:tasks.rados.rados.0.smithi063.stdout:2225: oid 34 updating version 0 to 1463 2024-08-17T17:04:31.648 INFO:tasks.rados.rados.0.smithi063.stdout:2225: oid 34 updating version 1463 to 1465 2024-08-17T17:04:31.648 INFO:tasks.rados.rados.0.smithi063.stdout:2225: oid 34 version 1465 is already newer than 1464 2024-08-17T17:04:31.648 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 1465 (ObjNum 835 snap 288 seq_num 835) dirty exists 2024-08-17T17:04:31.648 INFO:tasks.rados.rados.0.smithi063.stdout:2225: left oid 34 (ObjNum 835 snap 288 seq_num 835) 2024-08-17T17:04:31.648 INFO:tasks.rados.rados.0.smithi063.stdout:2227: expect (ObjNum 626 snap 199 seq_num 626) 2024-08-17T17:04:31.731 INFO:tasks.rados.rados.0.smithi063.stdout:2228: finishing write tid 1 to smithi06387500-8 2024-08-17T17:04:31.731 INFO:tasks.rados.rados.0.smithi063.stdout:2228: finishing write tid 2 to smithi06387500-8 2024-08-17T17:04:31.732 INFO:tasks.rados.rados.0.smithi063.stdout:2228: finishing write tid 3 to smithi06387500-8 2024-08-17T17:04:31.732 INFO:tasks.rados.rados.0.smithi063.stdout:2228: finishing write tid 4 to smithi06387500-8 2024-08-17T17:04:31.732 INFO:tasks.rados.rados.0.smithi063.stdout:2228: oid 8 updating version 0 to 719 2024-08-17T17:04:31.732 INFO:tasks.rados.rados.0.smithi063.stdout:2228: oid 8 updating version 719 to 721 2024-08-17T17:04:31.732 INFO:tasks.rados.rados.0.smithi063.stdout:2228: oid 8 version 721 is already newer than 720 2024-08-17T17:04:31.732 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 721 (ObjNum 837 snap 288 seq_num 837) dirty exists 2024-08-17T17:04:31.732 INFO:tasks.rados.rados.0.smithi063.stdout:2228: left oid 8 (ObjNum 837 snap 288 seq_num 837) 2024-08-17T17:04:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:31 smithi063 ceph-mon[114756]: pgmap v420: 89 pgs: 89 active+clean; 208 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 0 B/s wr, 2 op/s 2024-08-17T17:04:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:31 smithi063 ceph-mon[114756]: osdmap e643: 8 total, 8 up, 8 in 2024-08-17T17:04:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:31 smithi063 ceph-mon[109843]: pgmap v420: 89 pgs: 89 active+clean; 208 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 0 B/s wr, 2 op/s 2024-08-17T17:04:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:31 smithi063 ceph-mon[109843]: osdmap e643: 8 total, 8 up, 8 in 2024-08-17T17:04:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:31 smithi162 ceph-mon[104668]: pgmap v420: 89 pgs: 89 active+clean; 208 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 0 B/s wr, 2 op/s 2024-08-17T17:04:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:31 smithi162 ceph-mon[104668]: osdmap e643: 8 total, 8 up, 8 in 2024-08-17T17:04:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:32.627 INFO:tasks.rados.rados.0.smithi063.stdout:2225: done (3 left) 2024-08-17T17:04:32.628 INFO:tasks.rados.rados.0.smithi063.stdout:2227: done (2 left) 2024-08-17T17:04:32.628 INFO:tasks.rados.rados.0.smithi063.stdout:2228: done (1 left) 2024-08-17T17:04:32.628 INFO:tasks.rados.rados.0.smithi063.stdout:2229: done (0 left) 2024-08-17T17:04:32.628 INFO:tasks.rados.rados.0.smithi063.stdout:2230: write oid 12 current snap is 289 2024-08-17T17:04:32.628 INFO:tasks.rados.rados.0.smithi063.stdout:2230: seq_num 838 ranges {484266=657492,1554381=529885} 2024-08-17T17:04:32.633 INFO:tasks.rados.rados.0.smithi063.stdout:2230: writing smithi06387500-12 from 484266 to 1141758 tid 1 2024-08-17T17:04:32.639 INFO:tasks.rados.rados.0.smithi063.stdout:2230: writing smithi06387500-12 from 1554381 to 2084266 tid 2 2024-08-17T17:04:32.639 INFO:tasks.rados.rados.0.smithi063.stdout:2231: write oid 38 current snap is 289 2024-08-17T17:04:32.639 INFO:tasks.rados.rados.0.smithi063.stdout:2231: seq_num 839 ranges {501073=732741,1896510=204563} 2024-08-17T17:04:32.646 INFO:tasks.rados.rados.0.smithi063.stdout:2231: writing smithi06387500-38 from 501073 to 1233814 tid 1 2024-08-17T17:04:32.652 INFO:tasks.rados.rados.0.smithi063.stdout:2231: writing smithi06387500-38 from 1896510 to 2101073 tid 2 2024-08-17T17:04:32.652 INFO:tasks.rados.rados.0.smithi063.stdout:2232: delete oid 39 current snap is 289 2024-08-17T17:04:32.652 INFO:tasks.rados.rados.0.smithi063.stdout:2230: finishing write tid 1 to smithi06387500-12 2024-08-17T17:04:32.652 INFO:tasks.rados.rados.0.smithi063.stdout:2230: finishing write tid 2 to smithi06387500-12 2024-08-17T17:04:32.652 INFO:tasks.rados.rados.0.smithi063.stdout:2230: finishing write tid 3 to smithi06387500-12 2024-08-17T17:04:32.652 INFO:tasks.rados.rados.0.smithi063.stdout:2230: finishing write tid 4 to smithi06387500-12 2024-08-17T17:04:32.652 INFO:tasks.rados.rados.0.smithi063.stdout:2230: oid 12 updating version 0 to 542 2024-08-17T17:04:32.653 INFO:tasks.rados.rados.0.smithi063.stdout:2230: oid 12 updating version 542 to 543 2024-08-17T17:04:32.653 INFO:tasks.rados.rados.0.smithi063.stdout:2230: oid 12 updating version 543 to 544 2024-08-17T17:04:32.653 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 544 (ObjNum 838 snap 289 seq_num 838) dirty exists 2024-08-17T17:04:32.653 INFO:tasks.rados.rados.0.smithi063.stdout:2230: left oid 12 (ObjNum 838 snap 289 seq_num 838) 2024-08-17T17:04:32.653 INFO:tasks.rados.rados.0.smithi063.stdout:2231: finishing write tid 1 to smithi06387500-38 2024-08-17T17:04:32.653 INFO:tasks.rados.rados.0.smithi063.stdout:2231: finishing write tid 2 to smithi06387500-38 2024-08-17T17:04:32.654 INFO:tasks.rados.rados.0.smithi063.stdout:2231: finishing write tid 3 to smithi06387500-38 2024-08-17T17:04:32.654 INFO:tasks.rados.rados.0.smithi063.stdout:2230: done (2 left) 2024-08-17T17:04:32.654 INFO:tasks.rados.rados.0.smithi063.stdout:2232: done (1 left) 2024-08-17T17:04:32.654 INFO:tasks.rados.rados.0.smithi063.stdout:2233: write oid 30 current snap is 289 2024-08-17T17:04:32.654 INFO:tasks.rados.rados.0.smithi063.stdout:2233: seq_num 841 ranges {534687=483239,1613474=412283} 2024-08-17T17:04:32.659 INFO:tasks.rados.rados.0.smithi063.stdout:2233: writing smithi06387500-30 from 534687 to 1017926 tid 1 2024-08-17T17:04:32.665 INFO:tasks.rados.rados.0.smithi063.stdout:2233: writing smithi06387500-30 from 1613474 to 2025757 tid 2 2024-08-17T17:04:32.665 INFO:tasks.rados.rados.0.smithi063.stdout:2234: snap_create 2024-08-17T17:04:32.665 INFO:tasks.rados.rados.0.smithi063.stdout:2231: finishing write tid 4 to smithi06387500-38 2024-08-17T17:04:32.665 INFO:tasks.rados.rados.0.smithi063.stdout:2231: oid 38 updating version 0 to 354 2024-08-17T17:04:32.665 INFO:tasks.rados.rados.0.smithi063.stdout:2231: oid 38 version 354 is already newer than 353 2024-08-17T17:04:32.666 INFO:tasks.rados.rados.0.smithi063.stdout:2231: oid 38 updating version 354 to 355 2024-08-17T17:04:32.666 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 355 (ObjNum 839 snap 289 seq_num 839) dirty exists 2024-08-17T17:04:32.666 INFO:tasks.rados.rados.0.smithi063.stdout:2231: left oid 38 (ObjNum 839 snap 289 seq_num 839) 2024-08-17T17:04:32.666 INFO:tasks.rados.rados.0.smithi063.stdout:2233: finishing write tid 1 to smithi06387500-30 2024-08-17T17:04:32.669 INFO:tasks.rados.rados.0.smithi063.stdout:2233: finishing write tid 2 to smithi06387500-30 2024-08-17T17:04:32.669 INFO:tasks.rados.rados.0.smithi063.stdout:2233: finishing write tid 3 to smithi06387500-30 2024-08-17T17:04:32.669 INFO:tasks.rados.rados.0.smithi063.stdout:2233: finishing write tid 4 to smithi06387500-30 2024-08-17T17:04:32.669 INFO:tasks.rados.rados.0.smithi063.stdout:2233: oid 30 updating version 0 to 982 2024-08-17T17:04:32.669 INFO:tasks.rados.rados.0.smithi063.stdout:2233: oid 30 version 982 is already newer than 981 2024-08-17T17:04:32.669 INFO:tasks.rados.rados.0.smithi063.stdout:2233: oid 30 version 982 is already newer than 980 2024-08-17T17:04:32.669 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 982 (ObjNum 841 snap 289 seq_num 841) dirty exists 2024-08-17T17:04:32.669 INFO:tasks.rados.rados.0.smithi063.stdout:2233: left oid 30 (ObjNum 841 snap 289 seq_num 841) 2024-08-17T17:04:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:32 smithi063 ceph-mon[114756]: osdmap e644: 8 total, 8 up, 8 in 2024-08-17T17:04:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:32 smithi063 ceph-mon[109843]: osdmap e644: 8 total, 8 up, 8 in 2024-08-17T17:04:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:32 smithi162 ceph-mon[104668]: osdmap e644: 8 total, 8 up, 8 in 2024-08-17T17:04:33.631 INFO:tasks.rados.rados.0.smithi063.stdout:2231: done (2 left) 2024-08-17T17:04:33.631 INFO:tasks.rados.rados.0.smithi063.stdout:2233: done (1 left) 2024-08-17T17:04:33.631 INFO:tasks.rados.rados.0.smithi063.stdout:2234: done (0 left) 2024-08-17T17:04:33.631 INFO:tasks.rados.rados.0.smithi063.stdout:2235: read oid 46 snap -1 2024-08-17T17:04:33.631 INFO:tasks.rados.rados.0.smithi063.stdout:2235: expect deleted 2024-08-17T17:04:33.631 INFO:tasks.rados.rados.0.smithi063.stdout:2236: write oid 5 current snap is 290 2024-08-17T17:04:33.631 INFO:tasks.rados.rados.0.smithi063.stdout:2236: seq_num 842 ranges {551494=558488,1671956=479538} 2024-08-17T17:04:33.631 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:04:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:04:33] "GET /metrics HTTP/1.1" 200 36218 "" "Prometheus/2.51.0" 2024-08-17T17:04:33.637 INFO:tasks.rados.rados.0.smithi063.stdout:2236: writing smithi06387500-5 from 551494 to 1109982 tid 1 2024-08-17T17:04:33.643 INFO:tasks.rados.rados.0.smithi063.stdout:2236: writing smithi06387500-5 from 1671956 to 2151494 tid 2 2024-08-17T17:04:33.643 INFO:tasks.rados.rados.0.smithi063.stdout:2237: write oid 37 current snap is 290 2024-08-17T17:04:33.643 INFO:tasks.rados.rados.0.smithi063.stdout:2237: seq_num 843 ranges {568301=633737,1614085=415952} 2024-08-17T17:04:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:2237: writing smithi06387500-37 from 568301 to 1202038 tid 1 2024-08-17T17:04:33.654 INFO:tasks.rados.rados.0.smithi063.stdout:2237: writing smithi06387500-37 from 1614085 to 2030037 tid 2 2024-08-17T17:04:33.654 INFO:tasks.rados.rados.0.smithi063.stdout:2238: delete oid 16 current snap is 290 2024-08-17T17:04:33.654 INFO:tasks.rados.rados.0.smithi063.stdout:2236: finishing write tid 1 to smithi06387500-5 2024-08-17T17:04:33.654 INFO:tasks.rados.rados.0.smithi063.stdout:2236: finishing write tid 2 to smithi06387500-5 2024-08-17T17:04:33.654 INFO:tasks.rados.rados.0.smithi063.stdout:2236: finishing write tid 3 to smithi06387500-5 2024-08-17T17:04:33.654 INFO:tasks.rados.rados.0.smithi063.stdout:2236: finishing write tid 4 to smithi06387500-5 2024-08-17T17:04:33.654 INFO:tasks.rados.rados.0.smithi063.stdout:2236: oid 5 updating version 0 to 1082 2024-08-17T17:04:33.655 INFO:tasks.rados.rados.0.smithi063.stdout:2236: oid 5 updating version 1082 to 1083 2024-08-17T17:04:33.655 INFO:tasks.rados.rados.0.smithi063.stdout:2236: oid 5 updating version 1083 to 1084 2024-08-17T17:04:33.655 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1084 (ObjNum 842 snap 290 seq_num 842) dirty exists 2024-08-17T17:04:33.655 INFO:tasks.rados.rados.0.smithi063.stdout:2236: left oid 5 (ObjNum 842 snap 290 seq_num 842) 2024-08-17T17:04:33.657 INFO:tasks.rados.rados.0.smithi063.stdout:2235: done (3 left) 2024-08-17T17:04:33.657 INFO:tasks.rados.rados.0.smithi063.stdout:2236: done (2 left) 2024-08-17T17:04:33.657 INFO:tasks.rados.rados.0.smithi063.stdout:2238: done (1 left) 2024-08-17T17:04:33.657 INFO:tasks.rados.rados.0.smithi063.stdout:2239: read oid 48 snap 252 2024-08-17T17:04:33.657 INFO:tasks.rados.rados.0.smithi063.stdout:2239: expect (ObjNum 730 snap 246 seq_num 730) 2024-08-17T17:04:33.657 INFO:tasks.rados.rados.0.smithi063.stdout:2240: snap_remove snap 257 2024-08-17T17:04:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:33 smithi063 ceph-mon[114756]: pgmap v423: 89 pgs: 89 active+clean; 207 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-08-17T17:04:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:33 smithi063 ceph-mon[114756]: osdmap e645: 8 total, 8 up, 8 in 2024-08-17T17:04:34.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:33 smithi063 ceph-mon[109843]: pgmap v423: 89 pgs: 89 active+clean; 207 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-08-17T17:04:34.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:33 smithi063 ceph-mon[109843]: osdmap e645: 8 total, 8 up, 8 in 2024-08-17T17:04:34.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:33 smithi162 ceph-mon[104668]: pgmap v423: 89 pgs: 89 active+clean; 207 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-08-17T17:04:34.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:33 smithi162 ceph-mon[104668]: osdmap e645: 8 total, 8 up, 8 in 2024-08-17T17:04:34.640 INFO:tasks.rados.rados.0.smithi063.stdout:2240: done (2 left) 2024-08-17T17:04:34.640 INFO:tasks.rados.rados.0.smithi063.stdout:2241: delete oid 49 current snap is 290 2024-08-17T17:04:34.640 INFO:tasks.rados.rados.0.smithi063.stdout:2237: finishing write tid 1 to smithi06387500-37 2024-08-17T17:04:34.640 INFO:tasks.rados.rados.0.smithi063.stdout:2237: finishing write tid 2 to smithi06387500-37 2024-08-17T17:04:34.640 INFO:tasks.rados.rados.0.smithi063.stdout:2237: finishing write tid 3 to smithi06387500-37 2024-08-17T17:04:34.640 INFO:tasks.rados.rados.0.smithi063.stdout:2237: finishing write tid 4 to smithi06387500-37 2024-08-17T17:04:34.640 INFO:tasks.rados.rados.0.smithi063.stdout:2237: oid 37 updating version 0 to 2175 2024-08-17T17:04:34.640 INFO:tasks.rados.rados.0.smithi063.stdout:2237: oid 37 version 2175 is already newer than 2174 2024-08-17T17:04:34.640 INFO:tasks.rados.rados.0.smithi063.stdout:2237: oid 37 version 2175 is already newer than 2173 2024-08-17T17:04:34.640 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 2175 (ObjNum 843 snap 290 seq_num 843) dirty exists 2024-08-17T17:04:34.641 INFO:tasks.rados.rados.0.smithi063.stdout:2237: left oid 37 (ObjNum 843 snap 290 seq_num 843) 2024-08-17T17:04:34.641 INFO:tasks.rados.rados.0.smithi063.stdout:2239: expect (ObjNum 730 snap 246 seq_num 730) 2024-08-17T17:04:34.748 INFO:tasks.rados.rados.0.smithi063.stdout:2237: done (2 left) 2024-08-17T17:04:34.749 INFO:tasks.rados.rados.0.smithi063.stdout:2239: done (1 left) 2024-08-17T17:04:34.749 INFO:tasks.rados.rados.0.smithi063.stdout:2241: done (0 left) 2024-08-17T17:04:34.749 INFO:tasks.rados.rados.0.smithi063.stdout:2242: rollback oid 10 current snap is 290 2024-08-17T17:04:34.749 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 10 to 285 2024-08-17T17:04:34.749 INFO:tasks.rados.rados.0.smithi063.stdout:2243: read oid 16 snap 252 2024-08-17T17:04:34.749 INFO:tasks.rados.rados.0.smithi063.stdout:2243: expect (ObjNum 706 snap 229 seq_num 706) 2024-08-17T17:04:34.749 INFO:tasks.rados.rados.0.smithi063.stdout:2244: snap_remove snap 289 2024-08-17T17:04:35.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:34 smithi063 ceph-mon[114756]: osdmap e646: 8 total, 8 up, 8 in 2024-08-17T17:04:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:34 smithi063 ceph-mon[109843]: osdmap e646: 8 total, 8 up, 8 in 2024-08-17T17:04:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:34 smithi162 ceph-mon[104668]: osdmap e646: 8 total, 8 up, 8 in 2024-08-17T17:04:35.645 INFO:tasks.rados.rados.0.smithi063.stdout:2244: done (2 left) 2024-08-17T17:04:35.645 INFO:tasks.rados.rados.0.smithi063.stdout:2245: delete oid 33 current snap is 290 2024-08-17T17:04:35.646 INFO:tasks.rados.rados.0.smithi063.stdout:2242: finishing rollback tid 0 to smithi06387500-10 2024-08-17T17:04:35.646 INFO:tasks.rados.rados.0.smithi063.stdout:2242: finishing rollback tid 1 to smithi06387500-10 2024-08-17T17:04:35.646 INFO:tasks.rados.rados.0.smithi063.stdout:2242: finishing rollback tid 2 to smithi06387500-10 2024-08-17T17:04:35.646 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 720 (ObjNum 775 snap 268 seq_num 775) dirty exists 2024-08-17T17:04:35.646 INFO:tasks.rados.rados.0.smithi063.stdout:2243: expect (ObjNum 706 snap 229 seq_num 706) 2024-08-17T17:04:35.932 INFO:tasks.rados.rados.0.smithi063.stdout:2242: done (2 left) 2024-08-17T17:04:35.932 INFO:tasks.rados.rados.0.smithi063.stdout:2243: done (1 left) 2024-08-17T17:04:35.932 INFO:tasks.rados.rados.0.smithi063.stdout:2245: done (0 left) 2024-08-17T17:04:35.932 INFO:tasks.rados.rados.0.smithi063.stdout:2246: write oid 39 current snap is 290 2024-08-17T17:04:35.932 INFO:tasks.rados.rados.0.smithi063.stdout:2246: seq_num 847 ranges {635529=651086,1848013=387516} 2024-08-17T17:04:35.939 INFO:tasks.rados.rados.0.smithi063.stdout:2246: writing smithi06387500-39 from 635529 to 1286615 tid 1 2024-08-17T17:04:35.944 INFO:tasks.rados.rados.0.smithi063.stdout:2246: writing smithi06387500-39 from 1848013 to 2235529 tid 2 2024-08-17T17:04:35.944 INFO:tasks.rados.rados.0.smithi063.stdout:2247: rollback oid 17 current snap is 290 2024-08-17T17:04:35.944 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 17 to 288 2024-08-17T17:04:35.945 INFO:tasks.rados.rados.0.smithi063.stdout:2246: finishing write tid 1 to smithi06387500-39 2024-08-17T17:04:35.945 INFO:tasks.rados.rados.0.smithi063.stdout:2248: snap_remove snap 282 2024-08-17T17:04:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:35 smithi063 ceph-mon[114756]: pgmap v426: 89 pgs: 89 active+clean; 207 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-08-17T17:04:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:35 smithi063 ceph-mon[114756]: osdmap e647: 8 total, 8 up, 8 in 2024-08-17T17:04:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:35 smithi063 ceph-mon[109843]: pgmap v426: 89 pgs: 89 active+clean; 207 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-08-17T17:04:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:35 smithi063 ceph-mon[109843]: osdmap e647: 8 total, 8 up, 8 in 2024-08-17T17:04:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:35 smithi162 ceph-mon[104668]: pgmap v426: 89 pgs: 89 active+clean; 207 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.9 MiB/s wr, 12 op/s 2024-08-17T17:04:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:35 smithi162 ceph-mon[104668]: osdmap e647: 8 total, 8 up, 8 in 2024-08-17T17:04:36.664 INFO:tasks.rados.rados.0.smithi063.stdout:2248: done (2 left) 2024-08-17T17:04:36.664 INFO:tasks.rados.rados.0.smithi063.stdout:2249: read oid 2 snap -1 2024-08-17T17:04:36.664 INFO:tasks.rados.rados.0.smithi063.stdout:2249: expect (ObjNum 812 snap 280 seq_num 812) 2024-08-17T17:04:36.664 INFO:tasks.rados.rados.0.smithi063.stdout:2250: write oid 9 current snap is 290 2024-08-17T17:04:36.664 INFO:tasks.rados.rados.0.smithi063.stdout:2250: seq_num 848 ranges {652336=726335,1906495=345841} 2024-08-17T17:04:36.672 INFO:tasks.rados.rados.0.smithi063.stdout:2250: writing smithi06387500-9 from 652336 to 1378671 tid 1 2024-08-17T17:04:36.678 INFO:tasks.rados.rados.0.smithi063.stdout:2250: writing smithi06387500-9 from 1906495 to 2252336 tid 2 2024-08-17T17:04:36.678 INFO:tasks.rados.rados.0.smithi063.stdout:2251: write oid 35 current snap is 290 2024-08-17T17:04:36.678 INFO:tasks.rados.rados.0.smithi063.stdout:2246: finishing write tid 2 to smithi06387500-39 2024-08-17T17:04:36.678 INFO:tasks.rados.rados.0.smithi063.stdout:2246: finishing write tid 3 to smithi06387500-39 2024-08-17T17:04:36.678 INFO:tasks.rados.rados.0.smithi063.stdout:2247: finishing rollback tid 0 to smithi06387500-17 2024-08-17T17:04:36.679 INFO:tasks.rados.rados.0.smithi063.stdout:2246: finishing write tid 4 to smithi06387500-39 2024-08-17T17:04:36.679 INFO:tasks.rados.rados.0.smithi063.stdout:2246: oid 39 updating version 0 to 487 2024-08-17T17:04:36.679 INFO:tasks.rados.rados.0.smithi063.stdout:2246: oid 39 version 487 is already newer than 485 2024-08-17T17:04:36.679 INFO:tasks.rados.rados.0.smithi063.stdout:2246: oid 39 version 487 is already newer than 486 2024-08-17T17:04:36.679 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 487 (ObjNum 847 snap 290 seq_num 847) dirty exists 2024-08-17T17:04:36.679 INFO:tasks.rados.rados.0.smithi063.stdout:2246: left oid 39 (ObjNum 847 snap 290 seq_num 847) 2024-08-17T17:04:36.679 INFO:tasks.rados.rados.0.smithi063.stdout:2247: finishing rollback tid 1 to smithi06387500-17 2024-08-17T17:04:36.679 INFO:tasks.rados.rados.0.smithi063.stdout:2247: finishing rollback tid 2 to smithi06387500-17 2024-08-17T17:04:36.679 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 682 (ObjNum 807 snap 280 seq_num 807) dirty exists 2024-08-17T17:04:36.679 INFO:tasks.rados.rados.0.smithi063.stdout:2251: seq_num 849 ranges {669143=401584,1564977=426959} 2024-08-17T17:04:36.684 INFO:tasks.rados.rados.0.smithi063.stdout:2251: writing smithi06387500-35 from 669143 to 1070727 tid 1 2024-08-17T17:04:36.690 INFO:tasks.rados.rados.0.smithi063.stdout:2251: writing smithi06387500-35 from 1564977 to 1991936 tid 2 2024-08-17T17:04:36.690 INFO:tasks.rados.rados.0.smithi063.stdout:2246: done (4 left) 2024-08-17T17:04:36.690 INFO:tasks.rados.rados.0.smithi063.stdout:2247: done (3 left) 2024-08-17T17:04:36.690 INFO:tasks.rados.rados.0.smithi063.stdout:2252: read oid 48 snap 285 2024-08-17T17:04:36.690 INFO:tasks.rados.rados.0.smithi063.stdout:2252: expect (ObjNum 730 snap 246 seq_num 730) 2024-08-17T17:04:36.691 INFO:tasks.rados.rados.0.smithi063.stdout:2253: rollback oid 30 current snap is 290 2024-08-17T17:04:36.691 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 30 to 288 2024-08-17T17:04:36.691 INFO:tasks.rados.rados.0.smithi063.stdout:2250: finishing write tid 1 to smithi06387500-9 2024-08-17T17:04:36.691 INFO:tasks.rados.rados.0.smithi063.stdout:2250: finishing write tid 2 to smithi06387500-9 2024-08-17T17:04:36.691 INFO:tasks.rados.rados.0.smithi063.stdout:2254: read oid 45 snap -1 2024-08-17T17:04:36.691 INFO:tasks.rados.rados.0.smithi063.stdout:2254: expect deleted 2024-08-17T17:04:36.691 INFO:tasks.rados.rados.0.smithi063.stdout:2255: read oid 7 snap -1 2024-08-17T17:04:36.691 INFO:tasks.rados.rados.0.smithi063.stdout:2255: expect (ObjNum 721 snap 246 seq_num 721) 2024-08-17T17:04:36.691 INFO:tasks.rados.rados.0.smithi063.stdout:2256: read oid 17 snap 273 2024-08-17T17:04:36.691 INFO:tasks.rados.rados.0.smithi063.stdout:2256: expect (ObjNum 794 snap 273 seq_num 794) 2024-08-17T17:04:36.691 INFO:tasks.rados.rados.0.smithi063.stdout:2257: rollback oid 33 current snap is 290 2024-08-17T17:04:36.691 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 33 to 283 2024-08-17T17:04:36.691 INFO:tasks.rados.rados.0.smithi063.stdout:2250: finishing write tid 3 to smithi06387500-9 2024-08-17T17:04:36.692 INFO:tasks.rados.rados.0.smithi063.stdout:2249: expect (ObjNum 812 snap 280 seq_num 812) 2024-08-17T17:04:36.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:36 smithi063 ceph-mon[114756]: osdmap e648: 8 total, 8 up, 8 in 2024-08-17T17:04:36.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:36 smithi063 ceph-mon[114756]: pgmap v429: 89 pgs: 89 active+clean; 207 MiB data, 757 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:04:36.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:36 smithi063 ceph-mon[114756]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:36.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:36 smithi063 ceph-mon[109843]: osdmap e648: 8 total, 8 up, 8 in 2024-08-17T17:04:36.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:36 smithi063 ceph-mon[109843]: pgmap v429: 89 pgs: 89 active+clean; 207 MiB data, 757 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:04:36.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:36 smithi063 ceph-mon[109843]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:36.820 INFO:tasks.rados.rados.0.smithi063.stdout:2250: finishing write tid 4 to smithi06387500-9 2024-08-17T17:04:36.820 INFO:tasks.rados.rados.0.smithi063.stdout:2250: oid 9 updating version 0 to 994 2024-08-17T17:04:36.820 INFO:tasks.rados.rados.0.smithi063.stdout:2250: oid 9 version 994 is already newer than 993 2024-08-17T17:04:36.820 INFO:tasks.rados.rados.0.smithi063.stdout:2250: oid 9 updating version 994 to 995 2024-08-17T17:04:36.820 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 995 (ObjNum 848 snap 290 seq_num 848) dirty exists 2024-08-17T17:04:36.820 INFO:tasks.rados.rados.0.smithi063.stdout:2250: left oid 9 (ObjNum 848 snap 290 seq_num 848) 2024-08-17T17:04:36.821 INFO:tasks.rados.rados.0.smithi063.stdout:2251: finishing write tid 1 to smithi06387500-35 2024-08-17T17:04:36.821 INFO:tasks.rados.rados.0.smithi063.stdout:2249: done (8 left) 2024-08-17T17:04:36.821 INFO:tasks.rados.rados.0.smithi063.stdout:2250: done (7 left) 2024-08-17T17:04:36.821 INFO:tasks.rados.rados.0.smithi063.stdout:2258: write oid 50 current snap is 290 2024-08-17T17:04:36.821 INFO:tasks.rados.rados.0.smithi063.stdout:2257: finishing rollback tid 1 to smithi06387500-33 2024-08-17T17:04:36.821 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 700 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T17:04:36.821 INFO:tasks.rados.rados.0.smithi063.stdout:2253: finishing rollback tid 0 to smithi06387500-30 2024-08-17T17:04:36.821 INFO:tasks.rados.rados.0.smithi063.stdout:2253: finishing rollback tid 1 to smithi06387500-30 2024-08-17T17:04:36.821 INFO:tasks.rados.rados.0.smithi063.stdout:2253: finishing rollback tid 2 to smithi06387500-30 2024-08-17T17:04:36.821 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 998 (ObjNum 710 snap 234 seq_num 710) dirty exists 2024-08-17T17:04:36.821 INFO:tasks.rados.rados.0.smithi063.stdout:2251: finishing write tid 2 to smithi06387500-35 2024-08-17T17:04:36.821 INFO:tasks.rados.rados.0.smithi063.stdout:2251: finishing write tid 3 to smithi06387500-35 2024-08-17T17:04:36.821 INFO:tasks.rados.rados.0.smithi063.stdout:2251: finishing write tid 4 to smithi06387500-35 2024-08-17T17:04:36.821 INFO:tasks.rados.rados.0.smithi063.stdout:2251: oid 35 updating version 0 to 2206 2024-08-17T17:04:36.821 INFO:tasks.rados.rados.0.smithi063.stdout:2251: oid 35 version 2206 is already newer than 2205 2024-08-17T17:04:36.822 INFO:tasks.rados.rados.0.smithi063.stdout:2251: oid 35 version 2206 is already newer than 2204 2024-08-17T17:04:36.822 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 2206 (ObjNum 849 snap 290 seq_num 849) dirty exists 2024-08-17T17:04:36.822 INFO:tasks.rados.rados.0.smithi063.stdout:2251: left oid 35 (ObjNum 849 snap 290 seq_num 849) 2024-08-17T17:04:36.822 INFO:tasks.rados.rados.0.smithi063.stdout:2258: seq_num 850 ranges {685950=476833,1623459=570617} 2024-08-17T17:04:36.823 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'sleep 60' 2024-08-17T17:04:36.826 INFO:tasks.rados.rados.0.smithi063.stdout:2258: writing smithi06387500-50 from 685950 to 1162783 tid 1 2024-08-17T17:04:36.832 INFO:tasks.rados.rados.0.smithi063.stdout:2258: writing smithi06387500-50 from 1623459 to 2194076 tid 2 2024-08-17T17:04:36.833 INFO:tasks.rados.rados.0.smithi063.stdout:2251: done (7 left) 2024-08-17T17:04:36.833 INFO:tasks.rados.rados.0.smithi063.stdout:2253: done (6 left) 2024-08-17T17:04:36.833 INFO:tasks.rados.rados.0.smithi063.stdout:2254: done (5 left) 2024-08-17T17:04:36.833 INFO:tasks.rados.rados.0.smithi063.stdout:2257: done (4 left) 2024-08-17T17:04:36.833 INFO:tasks.rados.rados.0.smithi063.stdout:2259: read oid 37 snap 285 2024-08-17T17:04:36.833 INFO:tasks.rados.rados.0.smithi063.stdout:2259: expect (ObjNum 696 snap 222 seq_num 696) 2024-08-17T17:04:36.833 INFO:tasks.rados.rados.0.smithi063.stdout:2260: read oid 40 snap -1 2024-08-17T17:04:36.833 INFO:tasks.rados.rados.0.smithi063.stdout:2260: expect (ObjNum 725 snap 246 seq_num 725) 2024-08-17T17:04:36.833 INFO:tasks.rados.rados.0.smithi063.stdout:2261: delete oid 36 current snap is 290 2024-08-17T17:04:36.833 INFO:tasks.rados.rados.0.smithi063.stdout:2255: expect (ObjNum 721 snap 246 seq_num 721) 2024-08-17T17:04:36.934 INFO:tasks.rados.rados.0.smithi063.stdout:2252: expect (ObjNum 730 snap 246 seq_num 730) 2024-08-17T17:04:37.053 INFO:tasks.rados.rados.0.smithi063.stdout:2256: expect (ObjNum 794 snap 273 seq_num 794) 2024-08-17T17:04:37.064 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:04:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:36 smithi162 ceph-mon[104668]: osdmap e648: 8 total, 8 up, 8 in 2024-08-17T17:04:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:36 smithi162 ceph-mon[104668]: pgmap v429: 89 pgs: 89 active+clean; 207 MiB data, 757 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:04:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:36 smithi162 ceph-mon[104668]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:04:37.163 INFO:tasks.rados.rados.0.smithi063.stdout:2258: finishing write tid 1 to smithi06387500-50 2024-08-17T17:04:37.163 INFO:tasks.rados.rados.0.smithi063.stdout:2260: expect (ObjNum 725 snap 246 seq_num 725) 2024-08-17T17:04:37.235 INFO:tasks.rados.rados.0.smithi063.stdout:2258: finishing write tid 2 to smithi06387500-50 2024-08-17T17:04:37.235 INFO:tasks.rados.rados.0.smithi063.stdout:2258: finishing write tid 3 to smithi06387500-50 2024-08-17T17:04:37.235 INFO:tasks.rados.rados.0.smithi063.stdout:2258: finishing write tid 4 to smithi06387500-50 2024-08-17T17:04:37.235 INFO:tasks.rados.rados.0.smithi063.stdout:2258: oid 50 updating version 0 to 1479 2024-08-17T17:04:37.235 INFO:tasks.rados.rados.0.smithi063.stdout:2258: oid 50 updating version 1479 to 1480 2024-08-17T17:04:37.235 INFO:tasks.rados.rados.0.smithi063.stdout:2258: oid 50 updating version 1480 to 1481 2024-08-17T17:04:37.235 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 1481 (ObjNum 850 snap 290 seq_num 850) dirty exists 2024-08-17T17:04:37.235 INFO:tasks.rados.rados.0.smithi063.stdout:2258: left oid 50 (ObjNum 850 snap 290 seq_num 850) 2024-08-17T17:04:37.236 INFO:tasks.rados.rados.0.smithi063.stdout:2259: expect (ObjNum 696 snap 222 seq_num 696) 2024-08-17T17:04:37.462 INFO:tasks.rados.rados.0.smithi063.stdout:2252: done (6 left) 2024-08-17T17:04:37.462 INFO:tasks.rados.rados.0.smithi063.stdout:2255: done (5 left) 2024-08-17T17:04:37.462 INFO:tasks.rados.rados.0.smithi063.stdout:2256: done (4 left) 2024-08-17T17:04:37.462 INFO:tasks.rados.rados.0.smithi063.stdout:2258: done (3 left) 2024-08-17T17:04:37.462 INFO:tasks.rados.rados.0.smithi063.stdout:2259: done (2 left) 2024-08-17T17:04:37.462 INFO:tasks.rados.rados.0.smithi063.stdout:2260: done (1 left) 2024-08-17T17:04:37.463 INFO:tasks.rados.rados.0.smithi063.stdout:2261: done (0 left) 2024-08-17T17:04:37.463 INFO:tasks.rados.rados.0.smithi063.stdout:2262: write oid 14 current snap is 290 2024-08-17T17:04:37.463 INFO:tasks.rados.rados.0.smithi063.stdout:2262: seq_num 852 ranges {719564=743684,2140423=179141} 2024-08-17T17:04:37.470 INFO:tasks.rados.rados.0.smithi063.stdout:2262: writing smithi06387500-14 from 719564 to 1463248 tid 1 2024-08-17T17:04:37.475 INFO:tasks.rados.rados.0.smithi063.stdout:2262: writing smithi06387500-14 from 2140423 to 2319564 tid 2 2024-08-17T17:04:37.475 INFO:tasks.rados.rados.0.smithi063.stdout:2263: delete oid 41 current snap is 290 2024-08-17T17:04:37.475 INFO:tasks.rados.rados.0.smithi063.stdout:2262: finishing write tid 1 to smithi06387500-14 2024-08-17T17:04:37.479 INFO:tasks.rados.rados.0.smithi063.stdout:2262: finishing write tid 2 to smithi06387500-14 2024-08-17T17:04:37.479 INFO:tasks.rados.rados.0.smithi063.stdout:2263: done (1 left) 2024-08-17T17:04:37.479 INFO:tasks.rados.rados.0.smithi063.stdout:2264: read oid 46 snap -1 2024-08-17T17:04:37.479 INFO:tasks.rados.rados.0.smithi063.stdout:2264: expect deleted 2024-08-17T17:04:37.479 INFO:tasks.rados.rados.0.smithi063.stdout:2265: rollback oid 43 current snap is 290 2024-08-17T17:04:37.479 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 43 to 288 2024-08-17T17:04:37.479 INFO:tasks.rados.rados.0.smithi063.stdout:2266: read oid 13 snap 286 2024-08-17T17:04:37.479 INFO:tasks.rados.rados.0.smithi063.stdout:2266: expect (ObjNum 630 snap 201 seq_num 630) 2024-08-17T17:04:37.479 INFO:tasks.rados.rados.0.smithi063.stdout:2267: write oid 27 current snap is 290 2024-08-17T17:04:37.480 INFO:tasks.rados.rados.0.smithi063.stdout:2262: finishing write tid 3 to smithi06387500-14 2024-08-17T17:04:37.480 INFO:tasks.rados.rados.0.smithi063.stdout:2267: seq_num 854 ranges {753178=494182,1857387=495791} 2024-08-17T17:04:37.485 INFO:tasks.rados.rados.0.smithi063.stdout:2267: writing smithi06387500-27 from 753178 to 1247360 tid 1 2024-08-17T17:04:37.492 INFO:tasks.rados.rados.0.smithi063.stdout:2267: writing smithi06387500-27 from 1857387 to 2353178 tid 2 2024-08-17T17:04:37.492 INFO:tasks.rados.rados.0.smithi063.stdout:2268: write oid 29 current snap is 290 2024-08-17T17:04:37.492 INFO:tasks.rados.rados.0.smithi063.stdout:2262: finishing write tid 4 to smithi06387500-14 2024-08-17T17:04:37.492 INFO:tasks.rados.rados.0.smithi063.stdout:2262: oid 14 updating version 0 to 2212 2024-08-17T17:04:37.492 INFO:tasks.rados.rados.0.smithi063.stdout:2262: oid 14 version 2212 is already newer than 2210 2024-08-17T17:04:37.493 INFO:tasks.rados.rados.0.smithi063.stdout:2262: oid 14 version 2212 is already newer than 2211 2024-08-17T17:04:37.493 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 2212 (ObjNum 852 snap 290 seq_num 852) dirty exists 2024-08-17T17:04:37.493 INFO:tasks.rados.rados.0.smithi063.stdout:2262: left oid 14 (ObjNum 852 snap 290 seq_num 852) 2024-08-17T17:04:37.493 INFO:tasks.rados.rados.0.smithi063.stdout:2268: seq_num 855 ranges {769985=569431,1915869=437966} 2024-08-17T17:04:37.499 INFO:tasks.rados.rados.0.smithi063.stdout:2268: writing smithi06387500-29 from 769985 to 1339416 tid 1 2024-08-17T17:04:37.505 INFO:tasks.rados.rados.0.smithi063.stdout:2268: writing smithi06387500-29 from 1915869 to 2353835 tid 2 2024-08-17T17:04:37.505 INFO:tasks.rados.rados.0.smithi063.stdout:2262: done (5 left) 2024-08-17T17:04:37.505 INFO:tasks.rados.rados.0.smithi063.stdout:2269: snap_create 2024-08-17T17:04:37.505 INFO:tasks.rados.rados.0.smithi063.stdout:2265: finishing rollback tid 0 to smithi06387500-43 2024-08-17T17:04:37.505 INFO:tasks.rados.rados.0.smithi063.stdout:2265: finishing rollback tid 1 to smithi06387500-43 2024-08-17T17:04:37.505 INFO:tasks.rados.rados.0.smithi063.stdout:2265: finishing rollback tid 2 to smithi06387500-43 2024-08-17T17:04:37.506 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 1002 (ObjNum 831 snap 287 seq_num 831) dirty exists 2024-08-17T17:04:37.506 INFO:tasks.rados.rados.0.smithi063.stdout:2266: expect (ObjNum 630 snap 201 seq_num 630) 2024-08-17T17:04:37.625 INFO:tasks.rados.rados.0.smithi063.stdout:2267: finishing write tid 1 to smithi06387500-27 2024-08-17T17:04:37.625 INFO:tasks.rados.rados.0.smithi063.stdout:2267: finishing write tid 2 to smithi06387500-27 2024-08-17T17:04:37.626 INFO:tasks.rados.rados.0.smithi063.stdout:2267: finishing write tid 3 to smithi06387500-27 2024-08-17T17:04:37.626 INFO:tasks.rados.rados.0.smithi063.stdout:2264: done (5 left) 2024-08-17T17:04:37.626 INFO:tasks.rados.rados.0.smithi063.stdout:2265: done (4 left) 2024-08-17T17:04:37.626 INFO:tasks.rados.rados.0.smithi063.stdout:2266: done (3 left) 2024-08-17T17:04:37.626 INFO:tasks.rados.rados.0.smithi063.stdout:2269: done (2 left) 2024-08-17T17:04:37.626 INFO:tasks.rados.rados.0.smithi063.stdout:2270: snap_create 2024-08-17T17:04:37.626 INFO:tasks.rados.rados.0.smithi063.stdout:2267: finishing write tid 4 to smithi06387500-27 2024-08-17T17:04:37.626 INFO:tasks.rados.rados.0.smithi063.stdout:2267: oid 27 updating version 0 to 215 2024-08-17T17:04:37.626 INFO:tasks.rados.rados.0.smithi063.stdout:2267: oid 27 updating version 215 to 217 2024-08-17T17:04:37.626 INFO:tasks.rados.rados.0.smithi063.stdout:2267: oid 27 version 217 is already newer than 216 2024-08-17T17:04:37.626 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 217 (ObjNum 854 snap 290 seq_num 854) dirty exists 2024-08-17T17:04:37.626 INFO:tasks.rados.rados.0.smithi063.stdout:2267: left oid 27 (ObjNum 854 snap 290 seq_num 854) 2024-08-17T17:04:37.627 INFO:tasks.rados.rados.0.smithi063.stdout:2268: finishing write tid 1 to smithi06387500-29 2024-08-17T17:04:37.627 INFO:tasks.rados.rados.0.smithi063.stdout:2268: finishing write tid 2 to smithi06387500-29 2024-08-17T17:04:37.627 INFO:tasks.rados.rados.0.smithi063.stdout:2268: finishing write tid 3 to smithi06387500-29 2024-08-17T17:04:37.627 INFO:tasks.rados.rados.0.smithi063.stdout:2268: finishing write tid 4 to smithi06387500-29 2024-08-17T17:04:37.627 INFO:tasks.rados.rados.0.smithi063.stdout:2268: oid 29 updating version 0 to 554 2024-08-17T17:04:37.627 INFO:tasks.rados.rados.0.smithi063.stdout:2268: oid 29 updating version 554 to 555 2024-08-17T17:04:37.627 INFO:tasks.rados.rados.0.smithi063.stdout:2268: oid 29 updating version 555 to 556 2024-08-17T17:04:37.627 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 556 (ObjNum 855 snap 290 seq_num 855) dirty exists 2024-08-17T17:04:37.627 INFO:tasks.rados.rados.0.smithi063.stdout:2268: left oid 29 (ObjNum 855 snap 290 seq_num 855) 2024-08-17T17:04:37.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:37 smithi063 ceph-mon[114756]: osdmap e649: 8 total, 8 up, 8 in 2024-08-17T17:04:37.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:37 smithi063 ceph-mon[114756]: osdmap e650: 8 total, 8 up, 8 in 2024-08-17T17:04:37.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:37 smithi063 ceph-mon[109843]: osdmap e649: 8 total, 8 up, 8 in 2024-08-17T17:04:37.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:37 smithi063 ceph-mon[109843]: osdmap e650: 8 total, 8 up, 8 in 2024-08-17T17:04:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:37 smithi162 ceph-mon[104668]: osdmap e649: 8 total, 8 up, 8 in 2024-08-17T17:04:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:37 smithi162 ceph-mon[104668]: osdmap e650: 8 total, 8 up, 8 in 2024-08-17T17:04:38.576 INFO:tasks.rados.rados.0.smithi063.stdout:2267: done (2 left) 2024-08-17T17:04:38.576 INFO:tasks.rados.rados.0.smithi063.stdout:2268: done (1 left) 2024-08-17T17:04:38.576 INFO:tasks.rados.rados.0.smithi063.stdout:2270: done (0 left) 2024-08-17T17:04:38.576 INFO:tasks.rados.rados.0.smithi063.stdout:2271: delete oid 9 current snap is 292 2024-08-17T17:04:38.580 INFO:tasks.rados.rados.0.smithi063.stdout:2271: done (0 left) 2024-08-17T17:04:38.581 INFO:tasks.rados.rados.0.smithi063.stdout:2272: write oid 48 current snap is 292 2024-08-17T17:04:38.581 INFO:tasks.rados.rados.0.smithi063.stdout:2272: seq_num 857 ranges {403599=719929,1916480=441635} 2024-08-17T17:04:38.587 INFO:tasks.rados.rados.0.smithi063.stdout:2272: writing smithi06387500-48 from 403599 to 1123528 tid 1 2024-08-17T17:04:38.593 INFO:tasks.rados.rados.0.smithi063.stdout:2272: writing smithi06387500-48 from 1916480 to 2358115 tid 2 2024-08-17T17:04:38.594 INFO:tasks.rados.rados.0.smithi063.stdout:2273: snap_remove snap 285 2024-08-17T17:04:39.582 INFO:tasks.rados.rados.0.smithi063.stdout:2272: finishing write tid 1 to smithi06387500-48 2024-08-17T17:04:39.582 INFO:tasks.rados.rados.0.smithi063.stdout:2272: finishing write tid 2 to smithi06387500-48 2024-08-17T17:04:39.582 INFO:tasks.rados.rados.0.smithi063.stdout:2272: finishing write tid 3 to smithi06387500-48 2024-08-17T17:04:39.582 INFO:tasks.rados.rados.0.smithi063.stdout:2272: finishing write tid 4 to smithi06387500-48 2024-08-17T17:04:39.582 INFO:tasks.rados.rados.0.smithi063.stdout:2272: oid 48 updating version 0 to 2216 2024-08-17T17:04:39.582 INFO:tasks.rados.rados.0.smithi063.stdout:2272: oid 48 version 2216 is already newer than 2214 2024-08-17T17:04:39.582 INFO:tasks.rados.rados.0.smithi063.stdout:2272: oid 48 version 2216 is already newer than 2215 2024-08-17T17:04:39.583 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 2216 (ObjNum 857 snap 292 seq_num 857) dirty exists 2024-08-17T17:04:39.583 INFO:tasks.rados.rados.0.smithi063.stdout:2272: left oid 48 (ObjNum 857 snap 292 seq_num 857) 2024-08-17T17:04:39.583 INFO:tasks.rados.rados.0.smithi063.stdout:2272: done (1 left) 2024-08-17T17:04:39.583 INFO:tasks.rados.rados.0.smithi063.stdout:2273: done (0 left) 2024-08-17T17:04:39.583 INFO:tasks.rados.rados.0.smithi063.stdout:2274: read oid 7 snap -1 2024-08-17T17:04:39.583 INFO:tasks.rados.rados.0.smithi063.stdout:2274: expect (ObjNum 721 snap 246 seq_num 721) 2024-08-17T17:04:39.583 INFO:tasks.rados.rados.0.smithi063.stdout:2275: snap_remove snap 279 2024-08-17T17:04:39.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:39 smithi162 ceph-mon[104668]: pgmap v432: 89 pgs: 89 active+clean; 212 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.2 MiB/s wr, 30 op/s 2024-08-17T17:04:39.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:39 smithi162 ceph-mon[104668]: osdmap e651: 8 total, 8 up, 8 in 2024-08-17T17:04:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:39 smithi063 ceph-mon[109843]: pgmap v432: 89 pgs: 89 active+clean; 212 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.2 MiB/s wr, 30 op/s 2024-08-17T17:04:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:39 smithi063 ceph-mon[109843]: osdmap e651: 8 total, 8 up, 8 in 2024-08-17T17:04:40.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:39 smithi063 ceph-mon[114756]: pgmap v432: 89 pgs: 89 active+clean; 212 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.2 MiB/s wr, 30 op/s 2024-08-17T17:04:40.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:39 smithi063 ceph-mon[114756]: osdmap e651: 8 total, 8 up, 8 in 2024-08-17T17:04:40.593 INFO:tasks.rados.rados.0.smithi063.stdout:2274: expect (ObjNum 721 snap 246 seq_num 721) 2024-08-17T17:04:40.697 INFO:tasks.rados.rados.0.smithi063.stdout:2274: done (1 left) 2024-08-17T17:04:40.697 INFO:tasks.rados.rados.0.smithi063.stdout:2275: done (0 left) 2024-08-17T17:04:40.697 INFO:tasks.rados.rados.0.smithi063.stdout:2276: snap_remove snap 284 2024-08-17T17:04:40.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:40 smithi162 ceph-mon[104668]: osdmap e652: 8 total, 8 up, 8 in 2024-08-17T17:04:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:40 smithi063 ceph-mon[114756]: osdmap e652: 8 total, 8 up, 8 in 2024-08-17T17:04:41.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:40 smithi063 ceph-mon[109843]: osdmap e652: 8 total, 8 up, 8 in 2024-08-17T17:04:41.592 INFO:tasks.rados.rados.0.smithi063.stdout:2276: done (0 left) 2024-08-17T17:04:41.593 INFO:tasks.rados.rados.0.smithi063.stdout:2277: write oid 25 current snap is 292 2024-08-17T17:04:41.593 INFO:tasks.rados.rados.0.smithi063.stdout:2277: seq_num 858 ranges {420406=795178,1974962=445444} 2024-08-17T17:04:41.600 INFO:tasks.rados.rados.0.smithi063.stdout:2277: writing smithi06387500-25 from 420406 to 1215584 tid 1 2024-08-17T17:04:41.607 INFO:tasks.rados.rados.0.smithi063.stdout:2277: writing smithi06387500-25 from 1974962 to 2420406 tid 2 2024-08-17T17:04:41.607 INFO:tasks.rados.rados.0.smithi063.stdout:2278: snap_remove snap 252 2024-08-17T17:04:41.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:41 smithi162 ceph-mon[104668]: pgmap v435: 89 pgs: 89 active+clean; 212 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.2 MiB/s wr, 30 op/s 2024-08-17T17:04:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:41 smithi162 ceph-mon[104668]: osdmap e653: 8 total, 8 up, 8 in 2024-08-17T17:04:41.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:41 smithi063 ceph-mon[114756]: pgmap v435: 89 pgs: 89 active+clean; 212 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.2 MiB/s wr, 30 op/s 2024-08-17T17:04:41.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:41 smithi063 ceph-mon[114756]: osdmap e653: 8 total, 8 up, 8 in 2024-08-17T17:04:41.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:41 smithi063 ceph-mon[109843]: pgmap v435: 89 pgs: 89 active+clean; 212 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.2 MiB/s wr, 30 op/s 2024-08-17T17:04:41.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:41 smithi063 ceph-mon[109843]: osdmap e653: 8 total, 8 up, 8 in 2024-08-17T17:04:42.607 INFO:tasks.rados.rados.0.smithi063.stdout:2278: done (1 left) 2024-08-17T17:04:42.608 INFO:tasks.rados.rados.0.smithi063.stdout:2279: write oid 17 current snap is 292 2024-08-17T17:04:42.608 INFO:tasks.rados.rados.0.smithi063.stdout:2279: seq_num 859 ranges {437213=470427,1633444=728951} 2024-08-17T17:04:42.613 INFO:tasks.rados.rados.0.smithi063.stdout:2279: writing smithi06387500-17 from 437213 to 907640 tid 1 2024-08-17T17:04:42.623 INFO:tasks.rados.rados.0.smithi063.stdout:2279: writing smithi06387500-17 from 1633444 to 2362395 tid 2 2024-08-17T17:04:42.623 INFO:tasks.rados.rados.0.smithi063.stdout:2280: snap_create 2024-08-17T17:04:42.623 INFO:tasks.rados.rados.0.smithi063.stdout:2277: finishing write tid 1 to smithi06387500-25 2024-08-17T17:04:42.623 INFO:tasks.rados.rados.0.smithi063.stdout:2277: finishing write tid 2 to smithi06387500-25 2024-08-17T17:04:42.623 INFO:tasks.rados.rados.0.smithi063.stdout:2277: finishing write tid 3 to smithi06387500-25 2024-08-17T17:04:42.623 INFO:tasks.rados.rados.0.smithi063.stdout:2277: finishing write tid 4 to smithi06387500-25 2024-08-17T17:04:42.623 INFO:tasks.rados.rados.0.smithi063.stdout:2277: oid 25 updating version 0 to 2249 2024-08-17T17:04:42.623 INFO:tasks.rados.rados.0.smithi063.stdout:2277: oid 25 updating version 2249 to 2250 2024-08-17T17:04:42.623 INFO:tasks.rados.rados.0.smithi063.stdout:2277: oid 25 version 2250 is already newer than 2246 2024-08-17T17:04:42.623 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 2250 (ObjNum 858 snap 292 seq_num 858) dirty exists 2024-08-17T17:04:42.623 INFO:tasks.rados.rados.0.smithi063.stdout:2277: left oid 25 (ObjNum 858 snap 292 seq_num 858) 2024-08-17T17:04:42.624 INFO:tasks.rados.rados.0.smithi063.stdout:2279: finishing write tid 1 to smithi06387500-17 2024-08-17T17:04:42.627 INFO:tasks.rados.rados.0.smithi063.stdout:2279: finishing write tid 2 to smithi06387500-17 2024-08-17T17:04:42.627 INFO:tasks.rados.rados.0.smithi063.stdout:2279: finishing write tid 3 to smithi06387500-17 2024-08-17T17:04:42.627 INFO:tasks.rados.rados.0.smithi063.stdout:2279: finishing write tid 4 to smithi06387500-17 2024-08-17T17:04:42.627 INFO:tasks.rados.rados.0.smithi063.stdout:2279: oid 17 updating version 0 to 696 2024-08-17T17:04:42.627 INFO:tasks.rados.rados.0.smithi063.stdout:2279: oid 17 version 696 is already newer than 695 2024-08-17T17:04:42.627 INFO:tasks.rados.rados.0.smithi063.stdout:2279: oid 17 version 696 is already newer than 694 2024-08-17T17:04:42.627 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 696 (ObjNum 859 snap 292 seq_num 859) dirty exists 2024-08-17T17:04:42.627 INFO:tasks.rados.rados.0.smithi063.stdout:2279: left oid 17 (ObjNum 859 snap 292 seq_num 859) 2024-08-17T17:04:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:42 smithi063 ceph-mon[114756]: osdmap e654: 8 total, 8 up, 8 in 2024-08-17T17:04:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:42 smithi063 ceph-mon[109843]: osdmap e654: 8 total, 8 up, 8 in 2024-08-17T17:04:43.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:42 smithi162 ceph-mon[104668]: osdmap e654: 8 total, 8 up, 8 in 2024-08-17T17:04:43.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:04:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:04:43] "GET /metrics HTTP/1.1" 200 36218 "" "Prometheus/2.51.0" 2024-08-17T17:04:43.608 INFO:tasks.rados.rados.0.smithi063.stdout:2277: done (2 left) 2024-08-17T17:04:43.608 INFO:tasks.rados.rados.0.smithi063.stdout:2279: done (1 left) 2024-08-17T17:04:43.608 INFO:tasks.rados.rados.0.smithi063.stdout:2280: done (0 left) 2024-08-17T17:04:43.608 INFO:tasks.rados.rados.0.smithi063.stdout:2281: delete oid 11 current snap is 293 2024-08-17T17:04:43.611 INFO:tasks.rados.rados.0.smithi063.stdout:2281: done (0 left) 2024-08-17T17:04:43.611 INFO:tasks.rados.rados.0.smithi063.stdout:2282: rollback oid 44 current snap is 293 2024-08-17T17:04:43.612 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 44 to 273 2024-08-17T17:04:43.612 INFO:tasks.rados.rados.0.smithi063.stdout:2283: write oid 45 current snap is 293 2024-08-17T17:04:43.612 INFO:tasks.rados.rados.0.smithi063.stdout:2283: seq_num 861 ranges {470827=737278,1750408=720419} 2024-08-17T17:04:43.618 INFO:tasks.rados.rados.0.smithi063.stdout:2283: writing smithi06387500-45 from 470827 to 1208105 tid 1 2024-08-17T17:04:43.625 INFO:tasks.rados.rados.0.smithi063.stdout:2283: writing smithi06387500-45 from 1750408 to 2470827 tid 2 2024-08-17T17:04:43.625 INFO:tasks.rados.rados.0.smithi063.stdout:2284: rollback oid 41 current snap is 293 2024-08-17T17:04:43.625 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 41 to 286 2024-08-17T17:04:43.626 INFO:tasks.rados.rados.0.smithi063.stdout:2282: finishing rollback tid 0 to smithi06387500-44 2024-08-17T17:04:43.626 INFO:tasks.rados.rados.0.smithi063.stdout:2282: finishing rollback tid 1 to smithi06387500-44 2024-08-17T17:04:43.626 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 346 (ObjNum 129 snap 0 seq_num 1806993944) dirty dne 2024-08-17T17:04:43.626 INFO:tasks.rados.rados.0.smithi063.stdout:2283: finishing write tid 1 to smithi06387500-45 2024-08-17T17:04:43.626 INFO:tasks.rados.rados.0.smithi063.stdout:2282: done (2 left) 2024-08-17T17:04:43.626 INFO:tasks.rados.rados.0.smithi063.stdout:2285: delete oid 17 current snap is 293 2024-08-17T17:04:43.627 INFO:tasks.rados.rados.0.smithi063.stdout:2284: finishing rollback tid 1 to smithi06387500-41 2024-08-17T17:04:43.627 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-08-17T17:04:43.628 INFO:tasks.rados.rados.0.smithi063.stdout:2284: done (2 left) 2024-08-17T17:04:43.628 INFO:tasks.rados.rados.0.smithi063.stdout:2285: done (1 left) 2024-08-17T17:04:43.628 INFO:tasks.rados.rados.0.smithi063.stdout:2286: snap_create 2024-08-17T17:04:43.629 INFO:tasks.rados.rados.0.smithi063.stdout:2283: finishing write tid 2 to smithi06387500-45 2024-08-17T17:04:43.629 INFO:tasks.rados.rados.0.smithi063.stdout:2283: finishing write tid 3 to smithi06387500-45 2024-08-17T17:04:43.629 INFO:tasks.rados.rados.0.smithi063.stdout:2283: finishing write tid 4 to smithi06387500-45 2024-08-17T17:04:43.629 INFO:tasks.rados.rados.0.smithi063.stdout:2283: oid 45 updating version 0 to 506 2024-08-17T17:04:43.629 INFO:tasks.rados.rados.0.smithi063.stdout:2283: oid 45 version 506 is already newer than 505 2024-08-17T17:04:43.629 INFO:tasks.rados.rados.0.smithi063.stdout:2283: oid 45 version 506 is already newer than 504 2024-08-17T17:04:43.629 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 506 (ObjNum 861 snap 293 seq_num 861) dirty exists 2024-08-17T17:04:43.629 INFO:tasks.rados.rados.0.smithi063.stdout:2283: left oid 45 (ObjNum 861 snap 293 seq_num 861) 2024-08-17T17:04:44.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:43 smithi063 ceph-mon[109843]: pgmap v438: 89 pgs: 89 active+clean; 215 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-08-17T17:04:44.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:43 smithi063 ceph-mon[109843]: osdmap e655: 8 total, 8 up, 8 in 2024-08-17T17:04:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:43 smithi063 ceph-mon[114756]: pgmap v438: 89 pgs: 89 active+clean; 215 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-08-17T17:04:44.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:43 smithi063 ceph-mon[114756]: osdmap e655: 8 total, 8 up, 8 in 2024-08-17T17:04:44.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:43 smithi162 ceph-mon[104668]: pgmap v438: 89 pgs: 89 active+clean; 215 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-08-17T17:04:44.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:43 smithi162 ceph-mon[104668]: osdmap e655: 8 total, 8 up, 8 in 2024-08-17T17:04:44.616 INFO:tasks.rados.rados.0.smithi063.stdout:2283: done (1 left) 2024-08-17T17:04:44.616 INFO:tasks.rados.rados.0.smithi063.stdout:2286: done (0 left) 2024-08-17T17:04:44.616 INFO:tasks.rados.rados.0.smithi063.stdout:2287: write oid 22 current snap is 294 2024-08-17T17:04:44.616 INFO:tasks.rados.rados.0.smithi063.stdout:2287: seq_num 863 ranges {504441=487776,1467372=736289} 2024-08-17T17:04:44.621 INFO:tasks.rados.rados.0.smithi063.stdout:2287: writing smithi06387500-22 from 504441 to 992217 tid 1 2024-08-17T17:04:44.629 INFO:tasks.rados.rados.0.smithi063.stdout:2287: writing smithi06387500-22 from 1467372 to 2203661 tid 2 2024-08-17T17:04:44.629 INFO:tasks.rados.rados.0.smithi063.stdout:2288: rollback oid 4 current snap is 294 2024-08-17T17:04:44.629 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 4 to 273 2024-08-17T17:04:44.629 INFO:tasks.rados.rados.0.smithi063.stdout:2287: finishing write tid 1 to smithi06387500-22 2024-08-17T17:04:44.629 INFO:tasks.rados.rados.0.smithi063.stdout:2289: delete oid 42 current snap is 294 2024-08-17T17:04:44.632 INFO:tasks.rados.rados.0.smithi063.stdout:2289: done (2 left) 2024-08-17T17:04:44.632 INFO:tasks.rados.rados.0.smithi063.stdout:2290: write oid 18 current snap is 294 2024-08-17T17:04:44.632 INFO:tasks.rados.rados.0.smithi063.stdout:2290: seq_num 865 ranges {538055=638274,1867983=670072} 2024-08-17T17:04:44.638 INFO:tasks.rados.rados.0.smithi063.stdout:2290: writing smithi06387500-18 from 538055 to 1176329 tid 1 2024-08-17T17:04:44.646 INFO:tasks.rados.rados.0.smithi063.stdout:2290: writing smithi06387500-18 from 1867983 to 2538055 tid 2 2024-08-17T17:04:44.647 INFO:tasks.rados.rados.0.smithi063.stdout:2291: snap_create 2024-08-17T17:04:44.647 INFO:tasks.rados.rados.0.smithi063.stdout:2287: finishing write tid 2 to smithi06387500-22 2024-08-17T17:04:44.647 INFO:tasks.rados.rados.0.smithi063.stdout:2287: finishing write tid 3 to smithi06387500-22 2024-08-17T17:04:44.647 INFO:tasks.rados.rados.0.smithi063.stdout:2287: finishing write tid 4 to smithi06387500-22 2024-08-17T17:04:44.647 INFO:tasks.rados.rados.0.smithi063.stdout:2287: oid 22 updating version 0 to 481 2024-08-17T17:04:44.647 INFO:tasks.rados.rados.0.smithi063.stdout:2287: oid 22 updating version 481 to 483 2024-08-17T17:04:44.647 INFO:tasks.rados.rados.0.smithi063.stdout:2287: oid 22 version 483 is already newer than 482 2024-08-17T17:04:44.647 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 483 (ObjNum 863 snap 294 seq_num 863) dirty exists 2024-08-17T17:04:44.647 INFO:tasks.rados.rados.0.smithi063.stdout:2287: left oid 22 (ObjNum 863 snap 294 seq_num 863) 2024-08-17T17:04:44.647 INFO:tasks.rados.rados.0.smithi063.stdout:2288: finishing rollback tid 0 to smithi06387500-4 2024-08-17T17:04:44.647 INFO:tasks.rados.rados.0.smithi063.stdout:2288: finishing rollback tid 1 to smithi06387500-4 2024-08-17T17:04:44.647 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 2265 (ObjNum 129 snap 0 seq_num 1806979352) dirty dne 2024-08-17T17:04:44.648 INFO:tasks.rados.rados.0.smithi063.stdout:2290: finishing write tid 1 to smithi06387500-18 2024-08-17T17:04:44.650 INFO:tasks.rados.rados.0.smithi063.stdout:2290: finishing write tid 2 to smithi06387500-18 2024-08-17T17:04:44.650 INFO:tasks.rados.rados.0.smithi063.stdout:2290: finishing write tid 3 to smithi06387500-18 2024-08-17T17:04:44.650 INFO:tasks.rados.rados.0.smithi063.stdout:2290: finishing write tid 4 to smithi06387500-18 2024-08-17T17:04:44.650 INFO:tasks.rados.rados.0.smithi063.stdout:2290: oid 18 updating version 0 to 744 2024-08-17T17:04:44.650 INFO:tasks.rados.rados.0.smithi063.stdout:2290: oid 18 updating version 744 to 746 2024-08-17T17:04:44.651 INFO:tasks.rados.rados.0.smithi063.stdout:2290: oid 18 version 746 is already newer than 745 2024-08-17T17:04:44.651 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 746 (ObjNum 865 snap 294 seq_num 865) dirty exists 2024-08-17T17:04:44.651 INFO:tasks.rados.rados.0.smithi063.stdout:2290: left oid 18 (ObjNum 865 snap 294 seq_num 865) 2024-08-17T17:04:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:44 smithi063 ceph-mon[114756]: osdmap e656: 8 total, 8 up, 8 in 2024-08-17T17:04:45.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:44 smithi063 ceph-mon[109843]: osdmap e656: 8 total, 8 up, 8 in 2024-08-17T17:04:45.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:44 smithi162 ceph-mon[104668]: osdmap e656: 8 total, 8 up, 8 in 2024-08-17T17:04:45.632 INFO:tasks.rados.rados.0.smithi063.stdout:2287: done (3 left) 2024-08-17T17:04:45.632 INFO:tasks.rados.rados.0.smithi063.stdout:2288: done (2 left) 2024-08-17T17:04:45.632 INFO:tasks.rados.rados.0.smithi063.stdout:2290: done (1 left) 2024-08-17T17:04:45.632 INFO:tasks.rados.rados.0.smithi063.stdout:2291: done (0 left) 2024-08-17T17:04:45.632 INFO:tasks.rados.rados.0.smithi063.stdout:2292: read oid 3 snap -1 2024-08-17T17:04:45.632 INFO:tasks.rados.rados.0.smithi063.stdout:2292: expect (ObjNum 791 snap 272 seq_num 791) 2024-08-17T17:04:45.632 INFO:tasks.rados.rados.0.smithi063.stdout:2293: read oid 22 snap -1 2024-08-17T17:04:45.633 INFO:tasks.rados.rados.0.smithi063.stdout:2293: expect (ObjNum 863 snap 294 seq_num 863) 2024-08-17T17:04:45.633 INFO:tasks.rados.rados.0.smithi063.stdout:2294: rollback oid 35 current snap is 295 2024-08-17T17:04:45.633 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 35 to 287 2024-08-17T17:04:45.633 INFO:tasks.rados.rados.0.smithi063.stdout:2295: read oid 21 snap -1 2024-08-17T17:04:45.633 INFO:tasks.rados.rados.0.smithi063.stdout:2295: expect deleted 2024-08-17T17:04:45.633 INFO:tasks.rados.rados.0.smithi063.stdout:2296: read oid 8 snap 281 2024-08-17T17:04:45.633 INFO:tasks.rados.rados.0.smithi063.stdout:2296: expect (ObjNum 788 snap 272 seq_num 788) 2024-08-17T17:04:45.633 INFO:tasks.rados.rados.0.smithi063.stdout:2297: snap_create 2024-08-17T17:04:45.640 INFO:tasks.rados.rados.0.smithi063.stdout:2294: finishing rollback tid 0 to smithi06387500-35 2024-08-17T17:04:45.641 INFO:tasks.rados.rados.0.smithi063.stdout:2293: expect (ObjNum 863 snap 294 seq_num 863) 2024-08-17T17:04:45.758 INFO:tasks.rados.rados.0.smithi063.stdout:2294: finishing rollback tid 1 to smithi06387500-35 2024-08-17T17:04:45.758 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 2268 (ObjNum 177 snap 0 seq_num 1027982193) dirty dne 2024-08-17T17:04:45.758 INFO:tasks.rados.rados.0.smithi063.stdout:2296: expect (ObjNum 788 snap 272 seq_num 788) 2024-08-17T17:04:45.987 INFO:tasks.rados.rados.0.smithi063.stdout:2292: expect (ObjNum 791 snap 272 seq_num 791) 2024-08-17T17:04:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:45 smithi063 ceph-mon[114756]: pgmap v441: 89 pgs: 89 active+clean; 215 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-08-17T17:04:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:45 smithi063 ceph-mon[114756]: osdmap e657: 8 total, 8 up, 8 in 2024-08-17T17:04:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:45 smithi063 ceph-mon[109843]: pgmap v441: 89 pgs: 89 active+clean; 215 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-08-17T17:04:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:45 smithi063 ceph-mon[109843]: osdmap e657: 8 total, 8 up, 8 in 2024-08-17T17:04:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:45 smithi162 ceph-mon[104668]: pgmap v441: 89 pgs: 89 active+clean; 215 MiB data, 778 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.3 MiB/s wr, 11 op/s 2024-08-17T17:04:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:45 smithi162 ceph-mon[104668]: osdmap e657: 8 total, 8 up, 8 in 2024-08-17T17:04:46.635 INFO:tasks.rados.rados.0.smithi063.stdout:2292: done (5 left) 2024-08-17T17:04:46.635 INFO:tasks.rados.rados.0.smithi063.stdout:2293: done (4 left) 2024-08-17T17:04:46.635 INFO:tasks.rados.rados.0.smithi063.stdout:2294: done (3 left) 2024-08-17T17:04:46.635 INFO:tasks.rados.rados.0.smithi063.stdout:2295: done (2 left) 2024-08-17T17:04:46.635 INFO:tasks.rados.rados.0.smithi063.stdout:2296: done (1 left) 2024-08-17T17:04:46.635 INFO:tasks.rados.rados.0.smithi063.stdout:2297: done (0 left) 2024-08-17T17:04:46.635 INFO:tasks.rados.rados.0.smithi063.stdout:2298: read oid 48 snap 286 2024-08-17T17:04:46.635 INFO:tasks.rados.rados.0.smithi063.stdout:2298: expect (ObjNum 730 snap 246 seq_num 730) 2024-08-17T17:04:46.635 INFO:tasks.rados.rados.0.smithi063.stdout:2299: delete oid 12 current snap is 296 2024-08-17T17:04:46.638 INFO:tasks.rados.rados.0.smithi063.stdout:2299: done (1 left) 2024-08-17T17:04:46.638 INFO:tasks.rados.rados.0.smithi063.stdout:2300: snap_create 2024-08-17T17:04:46.643 INFO:tasks.rados.rados.0.smithi063.stdout:2298: expect (ObjNum 730 snap 246 seq_num 730) 2024-08-17T17:04:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:46 smithi063 ceph-mon[109843]: osdmap e658: 8 total, 8 up, 8 in 2024-08-17T17:04:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:46 smithi063 ceph-mon[114756]: osdmap e658: 8 total, 8 up, 8 in 2024-08-17T17:04:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:46 smithi162 ceph-mon[104668]: osdmap e658: 8 total, 8 up, 8 in 2024-08-17T17:04:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:04:47.577 INFO:tasks.rados.rados.0.smithi063.stdout:2298: done (1 left) 2024-08-17T17:04:47.577 INFO:tasks.rados.rados.0.smithi063.stdout:2300: done (0 left) 2024-08-17T17:04:47.577 INFO:tasks.rados.rados.0.smithi063.stdout:2301: read oid 45 snap 281 2024-08-17T17:04:47.577 INFO:tasks.rados.rados.0.smithi063.stdout:2301: expect (ObjNum 743 snap 253 seq_num 743) 2024-08-17T17:04:47.577 INFO:tasks.rados.rados.0.smithi063.stdout:2302: delete oid 47 current snap is 297 2024-08-17T17:04:47.579 INFO:tasks.rados.rados.0.smithi063.stdout:2302: done (1 left) 2024-08-17T17:04:47.579 INFO:tasks.rados.rados.0.smithi063.stdout:2303: snap_create 2024-08-17T17:04:47.585 INFO:tasks.rados.rados.0.smithi063.stdout:2301: expect (ObjNum 743 snap 253 seq_num 743) 2024-08-17T17:04:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:47 smithi063 ceph-mon[114756]: pgmap v444: 89 pgs: 89 active+clean; 215 MiB data, 778 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:04:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:47 smithi063 ceph-mon[114756]: osdmap e659: 8 total, 8 up, 8 in 2024-08-17T17:04:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:47 smithi063 ceph-mon[114756]: osdmap e660: 8 total, 8 up, 8 in 2024-08-17T17:04:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:47 smithi063 ceph-mon[109843]: pgmap v444: 89 pgs: 89 active+clean; 215 MiB data, 778 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:04:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:47 smithi063 ceph-mon[109843]: osdmap e659: 8 total, 8 up, 8 in 2024-08-17T17:04:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:47 smithi063 ceph-mon[109843]: osdmap e660: 8 total, 8 up, 8 in 2024-08-17T17:04:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:47 smithi162 ceph-mon[104668]: pgmap v444: 89 pgs: 89 active+clean; 215 MiB data, 778 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:04:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:47 smithi162 ceph-mon[104668]: osdmap e659: 8 total, 8 up, 8 in 2024-08-17T17:04:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:47 smithi162 ceph-mon[104668]: osdmap e660: 8 total, 8 up, 8 in 2024-08-17T17:04:48.579 INFO:tasks.rados.rados.0.smithi063.stdout:2301: done (1 left) 2024-08-17T17:04:48.580 INFO:tasks.rados.rados.0.smithi063.stdout:2303: done (0 left) 2024-08-17T17:04:48.580 INFO:tasks.rados.rados.0.smithi063.stdout:2304: snap_create 2024-08-17T17:04:49.580 INFO:tasks.rados.rados.0.smithi063.stdout:2304: done (0 left) 2024-08-17T17:04:49.581 INFO:tasks.rados.rados.0.smithi063.stdout:2305: read oid 34 snap -1 2024-08-17T17:04:49.581 INFO:tasks.rados.rados.0.smithi063.stdout:2305: expect (ObjNum 835 snap 288 seq_num 835) 2024-08-17T17:04:49.581 INFO:tasks.rados.rados.0.smithi063.stdout:2306: read oid 49 snap -1 2024-08-17T17:04:49.581 INFO:tasks.rados.rados.0.smithi063.stdout:2306: expect deleted 2024-08-17T17:04:49.581 INFO:tasks.rados.rados.0.smithi063.stdout:2307: snap_create 2024-08-17T17:04:49.587 INFO:tasks.rados.rados.0.smithi063.stdout:2305: expect (ObjNum 835 snap 288 seq_num 835) 2024-08-17T17:04:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:49 smithi162 ceph-mon[104668]: pgmap v447: 89 pgs: 89 active+clean; 176 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-08-17T17:04:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:49 smithi162 ceph-mon[104668]: osdmap e661: 8 total, 8 up, 8 in 2024-08-17T17:04:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:49 smithi063 ceph-mon[114756]: pgmap v447: 89 pgs: 89 active+clean; 176 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-08-17T17:04:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:49 smithi063 ceph-mon[114756]: osdmap e661: 8 total, 8 up, 8 in 2024-08-17T17:04:50.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:49 smithi063 ceph-mon[109843]: pgmap v447: 89 pgs: 89 active+clean; 176 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-08-17T17:04:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:49 smithi063 ceph-mon[109843]: osdmap e661: 8 total, 8 up, 8 in 2024-08-17T17:04:50.592 INFO:tasks.rados.rados.0.smithi063.stdout:2305: done (2 left) 2024-08-17T17:04:50.593 INFO:tasks.rados.rados.0.smithi063.stdout:2306: done (1 left) 2024-08-17T17:04:50.593 INFO:tasks.rados.rados.0.smithi063.stdout:2307: done (0 left) 2024-08-17T17:04:50.593 INFO:tasks.rados.rados.0.smithi063.stdout:2308: read oid 40 snap 297 2024-08-17T17:04:50.593 INFO:tasks.rados.rados.0.smithi063.stdout:2308: expect (ObjNum 725 snap 246 seq_num 725) 2024-08-17T17:04:50.593 INFO:tasks.rados.rados.0.smithi063.stdout:2309: write oid 38 current snap is 300 2024-08-17T17:04:50.593 INFO:tasks.rados.rados.0.smithi063.stdout:2309: seq_num 868 ranges {588476=580374,1643429=603638} 2024-08-17T17:04:50.599 INFO:tasks.rados.rados.0.smithi063.stdout:2309: writing smithi06387500-38 from 588476 to 1168850 tid 1 2024-08-17T17:04:50.605 INFO:tasks.rados.rados.0.smithi063.stdout:2309: writing smithi06387500-38 from 1643429 to 2247067 tid 2 2024-08-17T17:04:50.605 INFO:tasks.rados.rados.0.smithi063.stdout:2310: read oid 44 snap 296 2024-08-17T17:04:50.605 INFO:tasks.rados.rados.0.smithi063.stdout:2310: expect deleted 2024-08-17T17:04:50.605 INFO:tasks.rados.rados.0.smithi063.stdout:2311: snap_create 2024-08-17T17:04:50.605 INFO:tasks.rados.rados.0.smithi063.stdout:2308: expect (ObjNum 725 snap 246 seq_num 725) 2024-08-17T17:04:50.677 INFO:tasks.rados.rados.0.smithi063.stdout:2309: finishing write tid 1 to smithi06387500-38 2024-08-17T17:04:50.678 INFO:tasks.rados.rados.0.smithi063.stdout:2309: finishing write tid 2 to smithi06387500-38 2024-08-17T17:04:50.678 INFO:tasks.rados.rados.0.smithi063.stdout:2309: finishing write tid 3 to smithi06387500-38 2024-08-17T17:04:50.678 INFO:tasks.rados.rados.0.smithi063.stdout:2309: finishing write tid 4 to smithi06387500-38 2024-08-17T17:04:50.678 INFO:tasks.rados.rados.0.smithi063.stdout:2309: oid 38 updating version 0 to 362 2024-08-17T17:04:50.678 INFO:tasks.rados.rados.0.smithi063.stdout:2309: oid 38 updating version 362 to 363 2024-08-17T17:04:50.678 INFO:tasks.rados.rados.0.smithi063.stdout:2309: oid 38 updating version 363 to 364 2024-08-17T17:04:50.678 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 364 (ObjNum 868 snap 300 seq_num 868) dirty exists 2024-08-17T17:04:50.678 INFO:tasks.rados.rados.0.smithi063.stdout:2309: left oid 38 (ObjNum 868 snap 300 seq_num 868) 2024-08-17T17:04:50.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:50 smithi162 ceph-mon[104668]: osdmap e662: 8 total, 8 up, 8 in 2024-08-17T17:04:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:50 smithi063 ceph-mon[114756]: osdmap e662: 8 total, 8 up, 8 in 2024-08-17T17:04:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:50 smithi063 ceph-mon[109843]: osdmap e662: 8 total, 8 up, 8 in 2024-08-17T17:04:51.600 INFO:tasks.rados.rados.0.smithi063.stdout:2308: done (3 left) 2024-08-17T17:04:51.600 INFO:tasks.rados.rados.0.smithi063.stdout:2309: done (2 left) 2024-08-17T17:04:51.600 INFO:tasks.rados.rados.0.smithi063.stdout:2310: done (1 left) 2024-08-17T17:04:51.600 INFO:tasks.rados.rados.0.smithi063.stdout:2311: done (0 left) 2024-08-17T17:04:51.600 INFO:tasks.rados.rados.0.smithi063.stdout:2312: snap_create 2024-08-17T17:04:51.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:51 smithi162 ceph-mon[104668]: pgmap v450: 89 pgs: 89 active+clean; 176 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-08-17T17:04:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:51 smithi162 ceph-mon[104668]: osdmap e663: 8 total, 8 up, 8 in 2024-08-17T17:04:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:51 smithi063 ceph-mon[109843]: pgmap v450: 89 pgs: 89 active+clean; 176 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-08-17T17:04:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:51 smithi063 ceph-mon[109843]: osdmap e663: 8 total, 8 up, 8 in 2024-08-17T17:04:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:51 smithi063 ceph-mon[114756]: pgmap v450: 89 pgs: 89 active+clean; 176 MiB data, 710 MiB used, 715 GiB / 715 GiB avail; 11 MiB/s rd, 1.2 MiB/s wr, 15 op/s 2024-08-17T17:04:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:51 smithi063 ceph-mon[114756]: osdmap e663: 8 total, 8 up, 8 in 2024-08-17T17:04:52.581 INFO:tasks.rados.rados.0.smithi063.stdout:2312: done (0 left) 2024-08-17T17:04:52.581 INFO:tasks.rados.rados.0.smithi063.stdout:2313: read oid 36 snap 295 2024-08-17T17:04:52.581 INFO:tasks.rados.rados.0.smithi063.stdout:2313: expect deleted 2024-08-17T17:04:52.581 INFO:tasks.rados.rados.0.smithi063.stdout:2314: write oid 21 current snap is 302 2024-08-17T17:04:52.581 INFO:tasks.rados.rados.0.smithi063.stdout:2314: seq_num 869 ranges {605283=655623,1701911=747296} 2024-08-17T17:04:52.588 INFO:tasks.rados.rados.0.smithi063.stdout:2314: writing smithi06387500-21 from 605283 to 1260906 tid 1 2024-08-17T17:04:52.595 INFO:tasks.rados.rados.0.smithi063.stdout:2314: writing smithi06387500-21 from 1701911 to 2449207 tid 2 2024-08-17T17:04:52.595 INFO:tasks.rados.rados.0.smithi063.stdout:2315: snap_create 2024-08-17T17:04:52.595 INFO:tasks.rados.rados.0.smithi063.stdout:2314: finishing write tid 1 to smithi06387500-21 2024-08-17T17:04:52.600 INFO:tasks.rados.rados.0.smithi063.stdout:2314: finishing write tid 2 to smithi06387500-21 2024-08-17T17:04:52.600 INFO:tasks.rados.rados.0.smithi063.stdout:2314: finishing write tid 3 to smithi06387500-21 2024-08-17T17:04:52.600 INFO:tasks.rados.rados.0.smithi063.stdout:2314: finishing write tid 4 to smithi06387500-21 2024-08-17T17:04:52.600 INFO:tasks.rados.rados.0.smithi063.stdout:2314: oid 21 updating version 0 to 1505 2024-08-17T17:04:52.600 INFO:tasks.rados.rados.0.smithi063.stdout:2314: oid 21 version 1505 is already newer than 1503 2024-08-17T17:04:52.600 INFO:tasks.rados.rados.0.smithi063.stdout:2314: oid 21 version 1505 is already newer than 1504 2024-08-17T17:04:52.600 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 1505 (ObjNum 869 snap 302 seq_num 869) dirty exists 2024-08-17T17:04:52.600 INFO:tasks.rados.rados.0.smithi063.stdout:2314: left oid 21 (ObjNum 869 snap 302 seq_num 869) 2024-08-17T17:04:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:52 smithi063 ceph-mon[114756]: osdmap e664: 8 total, 8 up, 8 in 2024-08-17T17:04:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:52 smithi063 ceph-mon[114756]: osdmap e665: 8 total, 8 up, 8 in 2024-08-17T17:04:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:52 smithi063 ceph-mon[109843]: osdmap e664: 8 total, 8 up, 8 in 2024-08-17T17:04:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:52 smithi063 ceph-mon[109843]: osdmap e665: 8 total, 8 up, 8 in 2024-08-17T17:04:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:52 smithi162 ceph-mon[104668]: osdmap e664: 8 total, 8 up, 8 in 2024-08-17T17:04:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:52 smithi162 ceph-mon[104668]: osdmap e665: 8 total, 8 up, 8 in 2024-08-17T17:04:53.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:04:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:04:53] "GET /metrics HTTP/1.1" 200 36213 "" "Prometheus/2.51.0" 2024-08-17T17:04:53.610 INFO:tasks.rados.rados.0.smithi063.stdout:2313: done (2 left) 2024-08-17T17:04:53.610 INFO:tasks.rados.rados.0.smithi063.stdout:2314: done (1 left) 2024-08-17T17:04:53.610 INFO:tasks.rados.rados.0.smithi063.stdout:2315: done (0 left) 2024-08-17T17:04:53.610 INFO:tasks.rados.rados.0.smithi063.stdout:2316: snap_create 2024-08-17T17:04:54.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:53 smithi063 ceph-mon[109843]: pgmap v453: 89 pgs: 89 active+clean; 178 MiB data, 718 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 289 KiB/s wr, 7 op/s 2024-08-17T17:04:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:53 smithi063 ceph-mon[114756]: pgmap v453: 89 pgs: 89 active+clean; 178 MiB data, 718 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 289 KiB/s wr, 7 op/s 2024-08-17T17:04:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:53 smithi162 ceph-mon[104668]: pgmap v453: 89 pgs: 89 active+clean; 178 MiB data, 718 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 289 KiB/s wr, 7 op/s 2024-08-17T17:04:54.612 INFO:tasks.rados.rados.0.smithi063.stdout:2316: done (0 left) 2024-08-17T17:04:54.612 INFO:tasks.rados.rados.0.smithi063.stdout:2317: snap_remove snap 290 2024-08-17T17:04:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:54 smithi063 ceph-mon[114756]: osdmap e666: 8 total, 8 up, 8 in 2024-08-17T17:04:55.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:54 smithi063 ceph-mon[109843]: osdmap e666: 8 total, 8 up, 8 in 2024-08-17T17:04:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:54 smithi162 ceph-mon[104668]: osdmap e666: 8 total, 8 up, 8 in 2024-08-17T17:04:55.619 INFO:tasks.rados.rados.0.smithi063.stdout:2317: done (0 left) 2024-08-17T17:04:55.619 INFO:tasks.rados.rados.0.smithi063.stdout:2318: snap_remove snap 293 2024-08-17T17:04:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:55 smithi063 ceph-mon[114756]: pgmap v456: 89 pgs: 89 active+clean; 178 MiB data, 718 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 289 KiB/s wr, 7 op/s 2024-08-17T17:04:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:55 smithi063 ceph-mon[114756]: osdmap e667: 8 total, 8 up, 8 in 2024-08-17T17:04:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:55 smithi063 ceph-mon[109843]: pgmap v456: 89 pgs: 89 active+clean; 178 MiB data, 718 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 289 KiB/s wr, 7 op/s 2024-08-17T17:04:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:55 smithi063 ceph-mon[109843]: osdmap e667: 8 total, 8 up, 8 in 2024-08-17T17:04:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:55 smithi162 ceph-mon[104668]: pgmap v456: 89 pgs: 89 active+clean; 178 MiB data, 718 MiB used, 715 GiB / 715 GiB avail; 7.1 MiB/s rd, 289 KiB/s wr, 7 op/s 2024-08-17T17:04:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:55 smithi162 ceph-mon[104668]: osdmap e667: 8 total, 8 up, 8 in 2024-08-17T17:04:56.639 INFO:tasks.rados.rados.0.smithi063.stdout:2318: done (0 left) 2024-08-17T17:04:56.639 INFO:tasks.rados.rados.0.smithi063.stdout:2319: snap_create 2024-08-17T17:04:57.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:56 smithi063 ceph-mon[114756]: osdmap e668: 8 total, 8 up, 8 in 2024-08-17T17:04:57.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:56 smithi063 ceph-mon[109843]: osdmap e668: 8 total, 8 up, 8 in 2024-08-17T17:04:57.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:56 smithi162 ceph-mon[104668]: osdmap e668: 8 total, 8 up, 8 in 2024-08-17T17:04:57.640 INFO:tasks.rados.rados.0.smithi063.stdout:2319: done (0 left) 2024-08-17T17:04:57.641 INFO:tasks.rados.rados.0.smithi063.stdout:2320: snap_remove snap 288 2024-08-17T17:04:58.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:57 smithi063 ceph-mon[114756]: pgmap v459: 89 pgs: 89 active+clean; 178 MiB data, 718 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:58.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:57 smithi063 ceph-mon[114756]: osdmap e669: 8 total, 8 up, 8 in 2024-08-17T17:04:58.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:57 smithi063 ceph-mon[109843]: pgmap v459: 89 pgs: 89 active+clean; 178 MiB data, 718 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:58.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:57 smithi063 ceph-mon[109843]: osdmap e669: 8 total, 8 up, 8 in 2024-08-17T17:04:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:57 smithi162 ceph-mon[104668]: pgmap v459: 89 pgs: 89 active+clean; 178 MiB data, 718 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:04:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:57 smithi162 ceph-mon[104668]: osdmap e669: 8 total, 8 up, 8 in 2024-08-17T17:04:58.660 INFO:tasks.rados.rados.0.smithi063.stdout:2320: done (0 left) 2024-08-17T17:04:58.660 INFO:tasks.rados.rados.0.smithi063.stdout:2321: write oid 50 current snap is 305 2024-08-17T17:04:58.660 INFO:tasks.rados.rados.0.smithi063.stdout:2321: seq_num 870 ranges {622090=730872,1760393=607307} 2024-08-17T17:04:58.668 INFO:tasks.rados.rados.0.smithi063.stdout:2321: writing smithi06387500-50 from 622090 to 1352962 tid 1 2024-08-17T17:04:58.674 INFO:tasks.rados.rados.0.smithi063.stdout:2321: writing smithi06387500-50 from 1760393 to 2367700 tid 2 2024-08-17T17:04:58.675 INFO:tasks.rados.rados.0.smithi063.stdout:2322: read oid 4 snap -1 2024-08-17T17:04:58.675 INFO:tasks.rados.rados.0.smithi063.stdout:2322: expect deleted 2024-08-17T17:04:58.675 INFO:tasks.rados.rados.0.smithi063.stdout:2323: delete oid 12 current snap is 305 2024-08-17T17:04:58.675 INFO:tasks.rados.rados.0.smithi063.stdout:2321: finishing write tid 1 to smithi06387500-50 2024-08-17T17:04:58.677 INFO:tasks.rados.rados.0.smithi063.stdout:2322: done (2 left) 2024-08-17T17:04:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:2323: done (1 left) 2024-08-17T17:04:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:2324: read oid 23 snap -1 2024-08-17T17:04:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:2324: expect (ObjNum 777 snap 269 seq_num 777) 2024-08-17T17:04:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:2325: read oid 49 snap -1 2024-08-17T17:04:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:2325: expect deleted 2024-08-17T17:04:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:2326: read oid 21 snap -1 2024-08-17T17:04:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:2326: expect (ObjNum 869 snap 302 seq_num 869) 2024-08-17T17:04:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:2327: write oid 8 current snap is 305 2024-08-17T17:04:58.678 INFO:tasks.rados.rados.0.smithi063.stdout:2327: seq_num 872 ranges {655704=481370,1761004=494623} 2024-08-17T17:04:58.684 INFO:tasks.rados.rados.0.smithi063.stdout:2327: writing smithi06387500-8 from 655704 to 1137074 tid 1 2024-08-17T17:04:58.691 INFO:tasks.rados.rados.0.smithi063.stdout:2327: writing smithi06387500-8 from 1761004 to 2255627 tid 2 2024-08-17T17:04:58.691 INFO:tasks.rados.rados.0.smithi063.stdout:2328: write oid 9 current snap is 305 2024-08-17T17:04:58.691 INFO:tasks.rados.rados.0.smithi063.stdout:2321: finishing write tid 2 to smithi06387500-50 2024-08-17T17:04:58.691 INFO:tasks.rados.rados.0.smithi063.stdout:2321: finishing write tid 3 to smithi06387500-50 2024-08-17T17:04:58.691 INFO:tasks.rados.rados.0.smithi063.stdout:2321: finishing write tid 4 to smithi06387500-50 2024-08-17T17:04:58.691 INFO:tasks.rados.rados.0.smithi063.stdout:2321: oid 50 updating version 0 to 1511 2024-08-17T17:04:58.691 INFO:tasks.rados.rados.0.smithi063.stdout:2321: oid 50 version 1511 is already newer than 1510 2024-08-17T17:04:58.691 INFO:tasks.rados.rados.0.smithi063.stdout:2321: oid 50 version 1511 is already newer than 1509 2024-08-17T17:04:58.691 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 1511 (ObjNum 870 snap 305 seq_num 870) dirty exists 2024-08-17T17:04:58.692 INFO:tasks.rados.rados.0.smithi063.stdout:2321: left oid 50 (ObjNum 870 snap 305 seq_num 870) 2024-08-17T17:04:58.692 INFO:tasks.rados.rados.0.smithi063.stdout:2326: expect (ObjNum 869 snap 302 seq_num 869) 2024-08-17T17:04:58.775 INFO:tasks.rados.rados.0.smithi063.stdout:2324: expect (ObjNum 777 snap 269 seq_num 777) 2024-08-17T17:04:59.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:58 smithi063 ceph-mon[114756]: osdmap e670: 8 total, 8 up, 8 in 2024-08-17T17:04:59.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:58 smithi063 ceph-mon[114756]: pgmap v462: 89 pgs: 89 active+clean; 180 MiB data, 723 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 343 KiB/s wr, 1 op/s 2024-08-17T17:04:59.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:58 smithi063 ceph-mon[109843]: osdmap e670: 8 total, 8 up, 8 in 2024-08-17T17:04:59.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:58 smithi063 ceph-mon[109843]: pgmap v462: 89 pgs: 89 active+clean; 180 MiB data, 723 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 343 KiB/s wr, 1 op/s 2024-08-17T17:04:59.036 INFO:tasks.rados.rados.0.smithi063.stdout:2327: finishing write tid 1 to smithi06387500-8 2024-08-17T17:04:59.037 INFO:tasks.rados.rados.0.smithi063.stdout:2327: finishing write tid 2 to smithi06387500-8 2024-08-17T17:04:59.037 INFO:tasks.rados.rados.0.smithi063.stdout:2327: finishing write tid 3 to smithi06387500-8 2024-08-17T17:04:59.037 INFO:tasks.rados.rados.0.smithi063.stdout:2327: finishing write tid 4 to smithi06387500-8 2024-08-17T17:04:59.037 INFO:tasks.rados.rados.0.smithi063.stdout:2327: oid 8 updating version 0 to 747 2024-08-17T17:04:59.037 INFO:tasks.rados.rados.0.smithi063.stdout:2327: oid 8 updating version 747 to 749 2024-08-17T17:04:59.037 INFO:tasks.rados.rados.0.smithi063.stdout:2327: oid 8 version 749 is already newer than 748 2024-08-17T17:04:59.037 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 749 (ObjNum 872 snap 305 seq_num 872) dirty exists 2024-08-17T17:04:59.037 INFO:tasks.rados.rados.0.smithi063.stdout:2327: left oid 8 (ObjNum 872 snap 305 seq_num 872) 2024-08-17T17:04:59.037 INFO:tasks.rados.rados.0.smithi063.stdout:2328: seq_num 873 ranges {672511=556619,1819486=754634} 2024-08-17T17:04:59.043 INFO:tasks.rados.rados.0.smithi063.stdout:2328: writing smithi06387500-9 from 672511 to 1229130 tid 1 2024-08-17T17:04:59.051 INFO:tasks.rados.rados.0.smithi063.stdout:2328: writing smithi06387500-9 from 1819486 to 2574120 tid 2 2024-08-17T17:04:59.051 INFO:tasks.rados.rados.0.smithi063.stdout:2321: done (5 left) 2024-08-17T17:04:59.051 INFO:tasks.rados.rados.0.smithi063.stdout:2324: done (4 left) 2024-08-17T17:04:59.051 INFO:tasks.rados.rados.0.smithi063.stdout:2325: done (3 left) 2024-08-17T17:04:59.051 INFO:tasks.rados.rados.0.smithi063.stdout:2326: done (2 left) 2024-08-17T17:04:59.051 INFO:tasks.rados.rados.0.smithi063.stdout:2327: done (1 left) 2024-08-17T17:04:59.051 INFO:tasks.rados.rados.0.smithi063.stdout:2329: snap_remove snap 303 2024-08-17T17:04:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:58 smithi162 ceph-mon[104668]: osdmap e670: 8 total, 8 up, 8 in 2024-08-17T17:04:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:58 smithi162 ceph-mon[104668]: pgmap v462: 89 pgs: 89 active+clean; 180 MiB data, 723 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 343 KiB/s wr, 1 op/s 2024-08-17T17:04:59.660 INFO:tasks.rados.rados.0.smithi063.stdout:2329: done (1 left) 2024-08-17T17:04:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:2330: rollback oid 14 current snap is 305 2024-08-17T17:04:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 14 to 291 2024-08-17T17:04:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:2328: finishing write tid 1 to smithi06387500-9 2024-08-17T17:04:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:2328: finishing write tid 2 to smithi06387500-9 2024-08-17T17:04:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:2328: finishing write tid 3 to smithi06387500-9 2024-08-17T17:04:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:2328: finishing write tid 4 to smithi06387500-9 2024-08-17T17:04:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:2328: oid 9 updating version 0 to 1031 2024-08-17T17:04:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:2328: oid 9 updating version 1031 to 1033 2024-08-17T17:04:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:2328: oid 9 version 1033 is already newer than 1032 2024-08-17T17:04:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 1033 (ObjNum 873 snap 305 seq_num 873) dirty exists 2024-08-17T17:04:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:2328: left oid 9 (ObjNum 873 snap 305 seq_num 873) 2024-08-17T17:04:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:2328: done (1 left) 2024-08-17T17:04:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:2331: rollback oid 43 current snap is 305 2024-08-17T17:04:59.662 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 43 to 302 2024-08-17T17:04:59.662 INFO:tasks.rados.rados.0.smithi063.stdout:2332: rollback oid 6 current snap is 305 2024-08-17T17:04:59.662 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 6 to 299 2024-08-17T17:04:59.662 INFO:tasks.rados.rados.0.smithi063.stdout:2333: read oid 8 snap 294 2024-08-17T17:04:59.662 INFO:tasks.rados.rados.0.smithi063.stdout:2333: expect (ObjNum 837 snap 288 seq_num 837) 2024-08-17T17:04:59.662 INFO:tasks.rados.rados.0.smithi063.stdout:2334: read oid 48 snap 292 2024-08-17T17:04:59.662 INFO:tasks.rados.rados.0.smithi063.stdout:2334: expect (ObjNum 857 snap 292 seq_num 857) 2024-08-17T17:04:59.662 INFO:tasks.rados.rados.0.smithi063.stdout:2335: rollback oid 13 current snap is 305 2024-08-17T17:04:59.662 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 13 to 291 2024-08-17T17:04:59.662 INFO:tasks.rados.rados.0.smithi063.stdout:2336: delete oid 35 current snap is 305 2024-08-17T17:04:59.669 INFO:tasks.rados.rados.0.smithi063.stdout:2335: finishing rollback tid 1 to smithi06387500-13 2024-08-17T17:04:59.669 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1067 (ObjNum 49 snap 0 seq_num 1786885104) dirty dne 2024-08-17T17:04:59.669 INFO:tasks.rados.rados.0.smithi063.stdout:2332: finishing rollback tid 0 to smithi06387500-6 2024-08-17T17:04:59.670 INFO:tasks.rados.rados.0.smithi063.stdout:2331: finishing rollback tid 0 to smithi06387500-43 2024-08-17T17:04:59.670 INFO:tasks.rados.rados.0.smithi063.stdout:2331: finishing rollback tid 1 to smithi06387500-43 2024-08-17T17:04:59.670 INFO:tasks.rados.rados.0.smithi063.stdout:2331: finishing rollback tid 2 to smithi06387500-43 2024-08-17T17:04:59.670 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 1037 (ObjNum 831 snap 287 seq_num 831) dirty exists 2024-08-17T17:04:59.671 INFO:tasks.rados.rados.0.smithi063.stdout:2333: expect (ObjNum 837 snap 288 seq_num 837) 2024-08-17T17:04:59.856 INFO:tasks.rados.rados.0.smithi063.stdout:2332: finishing rollback tid 1 to smithi06387500-6 2024-08-17T17:04:59.856 INFO:tasks.rados.rados.0.smithi063.stdout:2332: finishing rollback tid 2 to smithi06387500-6 2024-08-17T17:04:59.856 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 753 (ObjNum 761 snap 258 seq_num 761) dirty exists 2024-08-17T17:04:59.856 INFO:tasks.rados.rados.0.smithi063.stdout:2330: finishing rollback tid 0 to smithi06387500-14 2024-08-17T17:04:59.857 INFO:tasks.rados.rados.0.smithi063.stdout:2331: done (6 left) 2024-08-17T17:04:59.857 INFO:tasks.rados.rados.0.smithi063.stdout:2332: done (5 left) 2024-08-17T17:04:59.857 INFO:tasks.rados.rados.0.smithi063.stdout:2333: done (4 left) 2024-08-17T17:04:59.857 INFO:tasks.rados.rados.0.smithi063.stdout:2335: done (3 left) 2024-08-17T17:04:59.857 INFO:tasks.rados.rados.0.smithi063.stdout:2336: done (2 left) 2024-08-17T17:04:59.857 INFO:tasks.rados.rados.0.smithi063.stdout:2337: write oid 6 current snap is 305 2024-08-17T17:04:59.857 INFO:tasks.rados.rados.0.smithi063.stdout:2330: finishing rollback tid 1 to smithi06387500-14 2024-08-17T17:04:59.857 INFO:tasks.rados.rados.0.smithi063.stdout:2337: seq_num 875 ranges {706125=423470,1652803=758303} 2024-08-17T17:04:59.862 INFO:tasks.rados.rados.0.smithi063.stdout:2337: writing smithi06387500-6 from 706125 to 1129595 tid 1 2024-08-17T17:04:59.869 INFO:tasks.rados.rados.0.smithi063.stdout:2337: writing smithi06387500-6 from 1652803 to 2411106 tid 2 2024-08-17T17:04:59.869 INFO:tasks.rados.rados.0.smithi063.stdout:2338: rollback oid 25 current snap is 305 2024-08-17T17:04:59.869 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 25 to 291 2024-08-17T17:04:59.869 INFO:tasks.rados.rados.0.smithi063.stdout:2330: finishing rollback tid 2 to smithi06387500-14 2024-08-17T17:04:59.869 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 2298 (ObjNum 852 snap 290 seq_num 852) dirty exists 2024-08-17T17:04:59.870 INFO:tasks.rados.rados.0.smithi063.stdout:2334: expect (ObjNum 857 snap 292 seq_num 857) 2024-08-17T17:05:00.015 INFO:tasks.rados.rados.0.smithi063.stdout:2337: finishing write tid 1 to smithi06387500-6 2024-08-17T17:05:00.015 INFO:tasks.rados.rados.0.smithi063.stdout:2337: finishing write tid 2 to smithi06387500-6 2024-08-17T17:05:00.015 INFO:tasks.rados.rados.0.smithi063.stdout:2337: finishing write tid 3 to smithi06387500-6 2024-08-17T17:05:00.015 INFO:tasks.rados.rados.0.smithi063.stdout:2337: finishing write tid 4 to smithi06387500-6 2024-08-17T17:05:00.015 INFO:tasks.rados.rados.0.smithi063.stdout:2337: oid 6 updating version 0 to 757 2024-08-17T17:05:00.015 INFO:tasks.rados.rados.0.smithi063.stdout:2337: oid 6 version 757 is already newer than 756 2024-08-17T17:05:00.015 INFO:tasks.rados.rados.0.smithi063.stdout:2337: oid 6 updating version 757 to 758 2024-08-17T17:05:00.015 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 758 (ObjNum 875 snap 305 seq_num 875) dirty exists 2024-08-17T17:05:00.015 INFO:tasks.rados.rados.0.smithi063.stdout:2337: left oid 6 (ObjNum 875 snap 305 seq_num 875) 2024-08-17T17:05:00.016 INFO:tasks.rados.rados.0.smithi063.stdout:2338: finishing rollback tid 0 to smithi06387500-25 2024-08-17T17:05:00.016 INFO:tasks.rados.rados.0.smithi063.stdout:2338: finishing rollback tid 1 to smithi06387500-25 2024-08-17T17:05:00.016 INFO:tasks.rados.rados.0.smithi063.stdout:2338: finishing rollback tid 2 to smithi06387500-25 2024-08-17T17:05:00.016 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 2303 (ObjNum 825 snap 283 seq_num 825) dirty exists 2024-08-17T17:05:00.016 INFO:tasks.rados.rados.0.smithi063.stdout:2330: done (3 left) 2024-08-17T17:05:00.016 INFO:tasks.rados.rados.0.smithi063.stdout:2334: done (2 left) 2024-08-17T17:05:00.016 INFO:tasks.rados.rados.0.smithi063.stdout:2337: done (1 left) 2024-08-17T17:05:00.016 INFO:tasks.rados.rados.0.smithi063.stdout:2338: done (0 left) 2024-08-17T17:05:00.016 INFO:tasks.rados.rados.0.smithi063.stdout:2339: snap_remove snap 298 2024-08-17T17:05:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:04:59 smithi063 ceph-mon[114756]: osdmap e671: 8 total, 8 up, 8 in 2024-08-17T17:05:00.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:04:59 smithi063 ceph-mon[109843]: osdmap e671: 8 total, 8 up, 8 in 2024-08-17T17:05:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:04:59 smithi162 ceph-mon[104668]: osdmap e671: 8 total, 8 up, 8 in 2024-08-17T17:05:00.673 INFO:tasks.rados.rados.0.smithi063.stdout:2339: done (0 left) 2024-08-17T17:05:00.673 INFO:tasks.rados.rados.0.smithi063.stdout:2340: write oid 1 current snap is 305 2024-08-17T17:05:00.673 INFO:tasks.rados.rados.0.smithi063.stdout:2340: seq_num 876 ranges {722932=498719,1994932=501961} 2024-08-17T17:05:00.679 INFO:tasks.rados.rados.0.smithi063.stdout:2340: writing smithi06387500-1 from 722932 to 1221651 tid 1 2024-08-17T17:05:00.687 INFO:tasks.rados.rados.0.smithi063.stdout:2340: writing smithi06387500-1 from 1994932 to 2496893 tid 2 2024-08-17T17:05:00.687 INFO:tasks.rados.rados.0.smithi063.stdout:2341: snap_remove snap 297 2024-08-17T17:05:00.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:00 smithi162 ceph-mon[104668]: osdmap e672: 8 total, 8 up, 8 in 2024-08-17T17:05:00.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:00 smithi162 ceph-mon[104668]: pgmap v465: 89 pgs: 89 active+clean; 180 MiB data, 723 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 343 KiB/s wr, 1 op/s 2024-08-17T17:05:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:00 smithi063 ceph-mon[114756]: osdmap e672: 8 total, 8 up, 8 in 2024-08-17T17:05:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:00 smithi063 ceph-mon[114756]: pgmap v465: 89 pgs: 89 active+clean; 180 MiB data, 723 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 343 KiB/s wr, 1 op/s 2024-08-17T17:05:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:00 smithi063 ceph-mon[109843]: osdmap e672: 8 total, 8 up, 8 in 2024-08-17T17:05:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:00 smithi063 ceph-mon[109843]: pgmap v465: 89 pgs: 89 active+clean; 180 MiB data, 723 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 343 KiB/s wr, 1 op/s 2024-08-17T17:05:01.688 INFO:tasks.rados.rados.0.smithi063.stdout:2341: done (1 left) 2024-08-17T17:05:01.688 INFO:tasks.rados.rados.0.smithi063.stdout:2342: snap_create 2024-08-17T17:05:01.688 INFO:tasks.rados.rados.0.smithi063.stdout:2340: finishing write tid 1 to smithi06387500-1 2024-08-17T17:05:01.688 INFO:tasks.rados.rados.0.smithi063.stdout:2340: finishing write tid 2 to smithi06387500-1 2024-08-17T17:05:01.688 INFO:tasks.rados.rados.0.smithi063.stdout:2340: finishing write tid 3 to smithi06387500-1 2024-08-17T17:05:01.688 INFO:tasks.rados.rados.0.smithi063.stdout:2340: finishing write tid 4 to smithi06387500-1 2024-08-17T17:05:01.688 INFO:tasks.rados.rados.0.smithi063.stdout:2340: oid 1 updating version 0 to 1112 2024-08-17T17:05:01.688 INFO:tasks.rados.rados.0.smithi063.stdout:2340: oid 1 updating version 1112 to 1113 2024-08-17T17:05:01.688 INFO:tasks.rados.rados.0.smithi063.stdout:2340: oid 1 updating version 1113 to 1114 2024-08-17T17:05:01.688 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 1114 (ObjNum 876 snap 305 seq_num 876) dirty exists 2024-08-17T17:05:01.688 INFO:tasks.rados.rados.0.smithi063.stdout:2340: left oid 1 (ObjNum 876 snap 305 seq_num 876) 2024-08-17T17:05:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:01 smithi063 ceph-mon[114756]: osdmap e673: 8 total, 8 up, 8 in 2024-08-17T17:05:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:01 smithi063 ceph-mon[109843]: osdmap e673: 8 total, 8 up, 8 in 2024-08-17T17:05:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:01 smithi162 ceph-mon[104668]: osdmap e673: 8 total, 8 up, 8 in 2024-08-17T17:05:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:02.581 INFO:tasks.rados.rados.0.smithi063.stdout:2340: done (1 left) 2024-08-17T17:05:02.581 INFO:tasks.rados.rados.0.smithi063.stdout:2342: done (0 left) 2024-08-17T17:05:02.581 INFO:tasks.rados.rados.0.smithi063.stdout:2343: delete oid 31 current snap is 306 2024-08-17T17:05:02.586 INFO:tasks.rados.rados.0.smithi063.stdout:2343: done (0 left) 2024-08-17T17:05:02.586 INFO:tasks.rados.rados.0.smithi063.stdout:2344: write oid 40 current snap is 306 2024-08-17T17:05:02.586 INFO:tasks.rados.rados.0.smithi063.stdout:2344: seq_num 878 ranges {756546=649217,2111896=505630} 2024-08-17T17:05:02.593 INFO:tasks.rados.rados.0.smithi063.stdout:2344: writing smithi06387500-40 from 756546 to 1405763 tid 1 2024-08-17T17:05:02.601 INFO:tasks.rados.rados.0.smithi063.stdout:2344: writing smithi06387500-40 from 2111896 to 2617526 tid 2 2024-08-17T17:05:02.601 INFO:tasks.rados.rados.0.smithi063.stdout:2345: read oid 6 snap -1 2024-08-17T17:05:02.601 INFO:tasks.rados.rados.0.smithi063.stdout:2345: expect (ObjNum 875 snap 305 seq_num 875) 2024-08-17T17:05:02.601 INFO:tasks.rados.rados.0.smithi063.stdout:2346: write oid 34 current snap is 306 2024-08-17T17:05:02.601 INFO:tasks.rados.rados.0.smithi063.stdout:2344: finishing write tid 1 to smithi06387500-40 2024-08-17T17:05:02.601 INFO:tasks.rados.rados.0.smithi063.stdout:2346: seq_num 879 ranges {773353=724466,2170378=602975} 2024-08-17T17:05:02.609 INFO:tasks.rados.rados.0.smithi063.stdout:2346: writing smithi06387500-34 from 773353 to 1497819 tid 1 2024-08-17T17:05:02.616 INFO:tasks.rados.rados.0.smithi063.stdout:2346: writing smithi06387500-34 from 2170378 to 2773353 tid 2 2024-08-17T17:05:02.616 INFO:tasks.rados.rados.0.smithi063.stdout:2347: read oid 29 snap 292 2024-08-17T17:05:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2347: expect (ObjNum 855 snap 290 seq_num 855) 2024-08-17T17:05:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2348: rollback oid 30 current snap is 306 2024-08-17T17:05:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 30 to 302 2024-08-17T17:05:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2344: finishing write tid 2 to smithi06387500-40 2024-08-17T17:05:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2344: finishing write tid 3 to smithi06387500-40 2024-08-17T17:05:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2344: finishing write tid 4 to smithi06387500-40 2024-08-17T17:05:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2344: oid 40 updating version 0 to 757 2024-08-17T17:05:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2344: oid 40 version 757 is already newer than 756 2024-08-17T17:05:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2344: oid 40 updating version 757 to 758 2024-08-17T17:05:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 758 (ObjNum 878 snap 306 seq_num 878) dirty exists 2024-08-17T17:05:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2344: left oid 40 (ObjNum 878 snap 306 seq_num 878) 2024-08-17T17:05:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2345: expect (ObjNum 875 snap 305 seq_num 875) 2024-08-17T17:05:02.804 INFO:tasks.rados.rados.0.smithi063.stdout:2346: finishing write tid 1 to smithi06387500-34 2024-08-17T17:05:02.805 INFO:tasks.rados.rados.0.smithi063.stdout:2344: done (4 left) 2024-08-17T17:05:02.805 INFO:tasks.rados.rados.0.smithi063.stdout:2345: done (3 left) 2024-08-17T17:05:02.805 INFO:tasks.rados.rados.0.smithi063.stdout:2349: delete oid 27 current snap is 306 2024-08-17T17:05:02.805 INFO:tasks.rados.rados.0.smithi063.stdout:2348: finishing rollback tid 0 to smithi06387500-30 2024-08-17T17:05:02.805 INFO:tasks.rados.rados.0.smithi063.stdout:2348: finishing rollback tid 1 to smithi06387500-30 2024-08-17T17:05:02.805 INFO:tasks.rados.rados.0.smithi063.stdout:2348: finishing rollback tid 2 to smithi06387500-30 2024-08-17T17:05:02.805 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1045 (ObjNum 710 snap 234 seq_num 710) dirty exists 2024-08-17T17:05:02.805 INFO:tasks.rados.rados.0.smithi063.stdout:2346: finishing write tid 2 to smithi06387500-34 2024-08-17T17:05:02.805 INFO:tasks.rados.rados.0.smithi063.stdout:2346: finishing write tid 3 to smithi06387500-34 2024-08-17T17:05:02.805 INFO:tasks.rados.rados.0.smithi063.stdout:2346: finishing write tid 4 to smithi06387500-34 2024-08-17T17:05:02.805 INFO:tasks.rados.rados.0.smithi063.stdout:2346: oid 34 updating version 0 to 1521 2024-08-17T17:05:02.805 INFO:tasks.rados.rados.0.smithi063.stdout:2346: oid 34 updating version 1521 to 1522 2024-08-17T17:05:02.805 INFO:tasks.rados.rados.0.smithi063.stdout:2346: oid 34 updating version 1522 to 1523 2024-08-17T17:05:02.806 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 1523 (ObjNum 879 snap 306 seq_num 879) dirty exists 2024-08-17T17:05:02.806 INFO:tasks.rados.rados.0.smithi063.stdout:2346: left oid 34 (ObjNum 879 snap 306 seq_num 879) 2024-08-17T17:05:02.806 INFO:tasks.rados.rados.0.smithi063.stdout:2347: expect (ObjNum 855 snap 290 seq_num 855) 2024-08-17T17:05:02.910 INFO:tasks.rados.rados.0.smithi063.stdout:2346: done (3 left) 2024-08-17T17:05:02.910 INFO:tasks.rados.rados.0.smithi063.stdout:2347: done (2 left) 2024-08-17T17:05:02.910 INFO:tasks.rados.rados.0.smithi063.stdout:2348: done (1 left) 2024-08-17T17:05:02.910 INFO:tasks.rados.rados.0.smithi063.stdout:2349: done (0 left) 2024-08-17T17:05:02.910 INFO:tasks.rados.rados.0.smithi063.stdout:2350: snap_create 2024-08-17T17:05:03.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:02 smithi063 ceph-mon[109843]: osdmap e674: 8 total, 8 up, 8 in 2024-08-17T17:05:03.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:02 smithi063 ceph-mon[109843]: pgmap v468: 89 pgs: 89 active+clean; 192 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-08-17T17:05:03.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:02 smithi063 ceph-mon[109843]: osdmap e675: 8 total, 8 up, 8 in 2024-08-17T17:05:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:02 smithi063 ceph-mon[114756]: osdmap e674: 8 total, 8 up, 8 in 2024-08-17T17:05:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:02 smithi063 ceph-mon[114756]: pgmap v468: 89 pgs: 89 active+clean; 192 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-08-17T17:05:03.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:02 smithi063 ceph-mon[114756]: osdmap e675: 8 total, 8 up, 8 in 2024-08-17T17:05:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:02 smithi162 ceph-mon[104668]: osdmap e674: 8 total, 8 up, 8 in 2024-08-17T17:05:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:02 smithi162 ceph-mon[104668]: pgmap v468: 89 pgs: 89 active+clean; 192 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-08-17T17:05:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:02 smithi162 ceph-mon[104668]: osdmap e675: 8 total, 8 up, 8 in 2024-08-17T17:05:03.697 INFO:tasks.rados.rados.0.smithi063.stdout:2350: done (0 left) 2024-08-17T17:05:03.697 INFO:tasks.rados.rados.0.smithi063.stdout:2351: read oid 3 snap -1 2024-08-17T17:05:03.697 INFO:tasks.rados.rados.0.smithi063.stdout:2351: expect (ObjNum 791 snap 272 seq_num 791) 2024-08-17T17:05:03.697 INFO:tasks.rados.rados.0.smithi063.stdout:2352: snap_remove snap 286 2024-08-17T17:05:03.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:05:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:05:03] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.51.0" 2024-08-17T17:05:04.699 INFO:tasks.rados.rados.0.smithi063.stdout:2352: done (1 left) 2024-08-17T17:05:04.699 INFO:tasks.rados.rados.0.smithi063.stdout:2353: read oid 10 snap 306 2024-08-17T17:05:04.699 INFO:tasks.rados.rados.0.smithi063.stdout:2353: expect (ObjNum 775 snap 268 seq_num 775) 2024-08-17T17:05:04.699 INFO:tasks.rados.rados.0.smithi063.stdout:2354: snap_remove snap 292 2024-08-17T17:05:05.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:04 smithi063 ceph-mon[114756]: osdmap e676: 8 total, 8 up, 8 in 2024-08-17T17:05:05.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:04 smithi063 ceph-mon[114756]: pgmap v471: 89 pgs: 89 active+clean; 192 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-08-17T17:05:05.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:04 smithi063 ceph-mon[109843]: osdmap e676: 8 total, 8 up, 8 in 2024-08-17T17:05:05.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:04 smithi063 ceph-mon[109843]: pgmap v471: 89 pgs: 89 active+clean; 192 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-08-17T17:05:05.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:04 smithi162 ceph-mon[104668]: osdmap e676: 8 total, 8 up, 8 in 2024-08-17T17:05:05.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:04 smithi162 ceph-mon[104668]: pgmap v471: 89 pgs: 89 active+clean; 192 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.4 MiB/s wr, 16 op/s 2024-08-17T17:05:05.697 INFO:tasks.rados.rados.0.smithi063.stdout:2354: done (2 left) 2024-08-17T17:05:05.698 INFO:tasks.rados.rados.0.smithi063.stdout:2355: snap_remove snap 299 2024-08-17T17:05:05.973 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:05 smithi162 ceph-mon[104668]: osdmap e677: 8 total, 8 up, 8 in 2024-08-17T17:05:05.973 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:06.009 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:05 smithi063 ceph-mon[109843]: osdmap e677: 8 total, 8 up, 8 in 2024-08-17T17:05:06.009 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:06.010 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:05 smithi063 ceph-mon[114756]: osdmap e677: 8 total, 8 up, 8 in 2024-08-17T17:05:06.010 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:06.713 INFO:tasks.rados.rados.0.smithi063.stdout:2355: done (2 left) 2024-08-17T17:05:06.713 INFO:tasks.rados.rados.0.smithi063.stdout:2356: snap_create 2024-08-17T17:05:06.713 INFO:tasks.rados.rados.0.smithi063.stdout:2351: expect (ObjNum 791 snap 272 seq_num 791) 2024-08-17T17:05:06.914 INFO:tasks.rados.rados.0.smithi063.stdout:2353: expect (ObjNum 775 snap 268 seq_num 775) 2024-08-17T17:05:06.961 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:06 smithi162 ceph-mon[104668]: osdmap e678: 8 total, 8 up, 8 in 2024-08-17T17:05:06.961 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:06 smithi162 ceph-mon[104668]: pgmap v474: 89 pgs: 89 active+clean; 192 MiB data, 740 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:05:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:06 smithi063 ceph-mon[114756]: osdmap e678: 8 total, 8 up, 8 in 2024-08-17T17:05:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:06 smithi063 ceph-mon[114756]: pgmap v474: 89 pgs: 89 active+clean; 192 MiB data, 740 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:05:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:06 smithi063 ceph-mon[109843]: osdmap e678: 8 total, 8 up, 8 in 2024-08-17T17:05:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:06 smithi063 ceph-mon[109843]: pgmap v474: 89 pgs: 89 active+clean; 192 MiB data, 740 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:05:07.584 INFO:tasks.rados.rados.0.smithi063.stdout:2351: done (2 left) 2024-08-17T17:05:07.584 INFO:tasks.rados.rados.0.smithi063.stdout:2353: done (1 left) 2024-08-17T17:05:07.584 INFO:tasks.rados.rados.0.smithi063.stdout:2356: done (0 left) 2024-08-17T17:05:07.584 INFO:tasks.rados.rados.0.smithi063.stdout:2357: write oid 41 current snap is 308 2024-08-17T17:05:07.584 INFO:tasks.rados.rados.0.smithi063.stdout:2357: seq_num 881 ranges {406967=474964,1370989=769310,2677186=129781} 2024-08-17T17:05:07.588 INFO:tasks.rados.rados.0.smithi063.stdout:2357: writing smithi06387500-41 from 406967 to 881931 tid 1 2024-08-17T17:05:07.595 INFO:tasks.rados.rados.0.smithi063.stdout:2357: writing smithi06387500-41 from 1370989 to 2140299 tid 2 2024-08-17T17:05:07.599 INFO:tasks.rados.rados.0.smithi063.stdout:2357: writing smithi06387500-41 from 2677186 to 2806967 tid 3 2024-08-17T17:05:07.599 INFO:tasks.rados.rados.0.smithi063.stdout:2358: write oid 32 current snap is 308 2024-08-17T17:05:07.599 INFO:tasks.rados.rados.0.smithi063.stdout:2357: finishing write tid 1 to smithi06387500-41 2024-08-17T17:05:07.599 INFO:tasks.rados.rados.0.smithi063.stdout:2357: finishing write tid 2 to smithi06387500-41 2024-08-17T17:05:07.600 INFO:tasks.rados.rados.0.smithi063.stdout:2358: seq_num 882 ranges {423774=666566,1545824=512968,2704609=119165} 2024-08-17T17:05:07.605 INFO:tasks.rados.rados.0.smithi063.stdout:2358: writing smithi06387500-32 from 423774 to 1090340 tid 1 2024-08-17T17:05:07.611 INFO:tasks.rados.rados.0.smithi063.stdout:2358: writing smithi06387500-32 from 1545824 to 2058792 tid 2 2024-08-17T17:05:07.615 INFO:tasks.rados.rados.0.smithi063.stdout:2358: writing smithi06387500-32 from 2704609 to 2823774 tid 3 2024-08-17T17:05:07.615 INFO:tasks.rados.rados.0.smithi063.stdout:2359: write oid 15 current snap is 308 2024-08-17T17:05:07.615 INFO:tasks.rados.rados.0.smithi063.stdout:2359: seq_num 883 ranges {440581=741815,1604306=656626,2732032=108549} 2024-08-17T17:05:07.621 INFO:tasks.rados.rados.0.smithi063.stdout:2359: writing smithi06387500-15 from 440581 to 1182396 tid 1 2024-08-17T17:05:07.628 INFO:tasks.rados.rados.0.smithi063.stdout:2359: writing smithi06387500-15 from 1604306 to 2260932 tid 2 2024-08-17T17:05:07.632 INFO:tasks.rados.rados.0.smithi063.stdout:2359: writing smithi06387500-15 from 2732032 to 2840581 tid 3 2024-08-17T17:05:07.632 INFO:tasks.rados.rados.0.smithi063.stdout:2360: read oid 38 snap -1 2024-08-17T17:05:07.632 INFO:tasks.rados.rados.0.smithi063.stdout:2360: expect (ObjNum 868 snap 300 seq_num 868) 2024-08-17T17:05:07.632 INFO:tasks.rados.rados.0.smithi063.stdout:2361: snap_create 2024-08-17T17:05:07.632 INFO:tasks.rados.rados.0.smithi063.stdout:2357: finishing write tid 3 to smithi06387500-41 2024-08-17T17:05:07.632 INFO:tasks.rados.rados.0.smithi063.stdout:2357: finishing write tid 4 to smithi06387500-41 2024-08-17T17:05:07.632 INFO:tasks.rados.rados.0.smithi063.stdout:2357: finishing write tid 5 to smithi06387500-41 2024-08-17T17:05:07.632 INFO:tasks.rados.rados.0.smithi063.stdout:2357: oid 41 updating version 0 to 228 2024-08-17T17:05:07.632 INFO:tasks.rados.rados.0.smithi063.stdout:2357: oid 41 version 228 is already newer than 227 2024-08-17T17:05:07.632 INFO:tasks.rados.rados.0.smithi063.stdout:2357: oid 41 updating version 228 to 229 2024-08-17T17:05:07.632 INFO:tasks.rados.rados.0.smithi063.stdout:2357: oid 41 version 229 is already newer than 226 2024-08-17T17:05:07.633 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 229 (ObjNum 881 snap 308 seq_num 881) dirty exists 2024-08-17T17:05:07.633 INFO:tasks.rados.rados.0.smithi063.stdout:2357: left oid 41 (ObjNum 881 snap 308 seq_num 881) 2024-08-17T17:05:07.633 INFO:tasks.rados.rados.0.smithi063.stdout:2358: finishing write tid 1 to smithi06387500-32 2024-08-17T17:05:07.633 INFO:tasks.rados.rados.0.smithi063.stdout:2358: finishing write tid 2 to smithi06387500-32 2024-08-17T17:05:07.633 INFO:tasks.rados.rados.0.smithi063.stdout:2358: finishing write tid 3 to smithi06387500-32 2024-08-17T17:05:07.633 INFO:tasks.rados.rados.0.smithi063.stdout:2358: finishing write tid 4 to smithi06387500-32 2024-08-17T17:05:07.633 INFO:tasks.rados.rados.0.smithi063.stdout:2358: finishing write tid 5 to smithi06387500-32 2024-08-17T17:05:07.633 INFO:tasks.rados.rados.0.smithi063.stdout:2358: oid 32 updating version 0 to 484 2024-08-17T17:05:07.633 INFO:tasks.rados.rados.0.smithi063.stdout:2358: oid 32 version 484 is already newer than 482 2024-08-17T17:05:07.633 INFO:tasks.rados.rados.0.smithi063.stdout:2358: oid 32 version 484 is already newer than 483 2024-08-17T17:05:07.633 INFO:tasks.rados.rados.0.smithi063.stdout:2358: oid 32 updating version 484 to 485 2024-08-17T17:05:07.633 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 485 (ObjNum 882 snap 308 seq_num 882) dirty exists 2024-08-17T17:05:07.634 INFO:tasks.rados.rados.0.smithi063.stdout:2358: left oid 32 (ObjNum 882 snap 308 seq_num 882) 2024-08-17T17:05:07.634 INFO:tasks.rados.rados.0.smithi063.stdout:2359: finishing write tid 1 to smithi06387500-15 2024-08-17T17:05:07.634 INFO:tasks.rados.rados.0.smithi063.stdout:2359: finishing write tid 2 to smithi06387500-15 2024-08-17T17:05:07.634 INFO:tasks.rados.rados.0.smithi063.stdout:2359: finishing write tid 3 to smithi06387500-15 2024-08-17T17:05:07.634 INFO:tasks.rados.rados.0.smithi063.stdout:2359: finishing write tid 4 to smithi06387500-15 2024-08-17T17:05:07.634 INFO:tasks.rados.rados.0.smithi063.stdout:2359: finishing write tid 5 to smithi06387500-15 2024-08-17T17:05:07.635 INFO:tasks.rados.rados.0.smithi063.stdout:2359: oid 15 updating version 0 to 912 2024-08-17T17:05:07.635 INFO:tasks.rados.rados.0.smithi063.stdout:2359: oid 15 updating version 912 to 915 2024-08-17T17:05:07.635 INFO:tasks.rados.rados.0.smithi063.stdout:2359: oid 15 version 915 is already newer than 914 2024-08-17T17:05:07.635 INFO:tasks.rados.rados.0.smithi063.stdout:2359: oid 15 version 915 is already newer than 913 2024-08-17T17:05:07.635 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 915 (ObjNum 883 snap 308 seq_num 883) dirty exists 2024-08-17T17:05:07.635 INFO:tasks.rados.rados.0.smithi063.stdout:2359: left oid 15 (ObjNum 883 snap 308 seq_num 883) 2024-08-17T17:05:07.635 INFO:tasks.rados.rados.0.smithi063.stdout:2360: expect (ObjNum 868 snap 300 seq_num 868) 2024-08-17T17:05:07.967 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:07 smithi063 ceph-mon[114756]: osdmap e679: 8 total, 8 up, 8 in 2024-08-17T17:05:07.967 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:07 smithi063 ceph-mon[114756]: osdmap e680: 8 total, 8 up, 8 in 2024-08-17T17:05:07.967 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:07 smithi063 ceph-mon[109843]: osdmap e679: 8 total, 8 up, 8 in 2024-08-17T17:05:07.968 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:07 smithi063 ceph-mon[109843]: osdmap e680: 8 total, 8 up, 8 in 2024-08-17T17:05:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:07 smithi162 ceph-mon[104668]: osdmap e679: 8 total, 8 up, 8 in 2024-08-17T17:05:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:07 smithi162 ceph-mon[104668]: osdmap e680: 8 total, 8 up, 8 in 2024-08-17T17:05:08.588 INFO:tasks.rados.rados.0.smithi063.stdout:2357: done (4 left) 2024-08-17T17:05:08.588 INFO:tasks.rados.rados.0.smithi063.stdout:2358: done (3 left) 2024-08-17T17:05:08.588 INFO:tasks.rados.rados.0.smithi063.stdout:2359: done (2 left) 2024-08-17T17:05:08.588 INFO:tasks.rados.rados.0.smithi063.stdout:2360: done (1 left) 2024-08-17T17:05:08.588 INFO:tasks.rados.rados.0.smithi063.stdout:2361: done (0 left) 2024-08-17T17:05:08.589 INFO:tasks.rados.rados.0.smithi063.stdout:2362: write oid 34 current snap is 309 2024-08-17T17:05:08.589 INFO:tasks.rados.rados.0.smithi063.stdout:2362: seq_num 884 ranges {457388=417064,1546435=516637,2643102=214286} 2024-08-17T17:05:08.593 INFO:tasks.rados.rados.0.smithi063.stdout:2362: writing smithi06387500-34 from 457388 to 874452 tid 1 2024-08-17T17:05:08.599 INFO:tasks.rados.rados.0.smithi063.stdout:2362: writing smithi06387500-34 from 1546435 to 2063072 tid 2 2024-08-17T17:05:08.606 INFO:tasks.rados.rados.0.smithi063.stdout:2362: writing smithi06387500-34 from 2643102 to 2857388 tid 3 2024-08-17T17:05:08.606 INFO:tasks.rados.rados.0.smithi063.stdout:2363: write oid 13 current snap is 309 2024-08-17T17:05:08.606 INFO:tasks.rados.rados.0.smithi063.stdout:2362: finishing write tid 1 to smithi06387500-34 2024-08-17T17:05:08.606 INFO:tasks.rados.rados.0.smithi063.stdout:2362: finishing write tid 2 to smithi06387500-34 2024-08-17T17:05:08.606 INFO:tasks.rados.rados.0.smithi063.stdout:2363: seq_num 885 ranges {474195=492313,1604917=660295,2670525=203670} 2024-08-17T17:05:08.609 INFO:tasks.rados.rados.0.smithi063.stdout:2363: writing smithi06387500-13 from 474195 to 966508 tid 1 2024-08-17T17:05:08.617 INFO:tasks.rados.rados.0.smithi063.stdout:2363: writing smithi06387500-13 from 1604917 to 2265212 tid 2 2024-08-17T17:05:08.621 INFO:tasks.rados.rados.0.smithi063.stdout:2363: writing smithi06387500-13 from 2670525 to 2874195 tid 3 2024-08-17T17:05:08.621 INFO:tasks.rados.rados.0.smithi063.stdout:2364: write oid 36 current snap is 309 2024-08-17T17:05:08.621 INFO:tasks.rados.rados.0.smithi063.stdout:2364: seq_num 886 ranges {491002=567562,1663399=520306,2814301=76701} 2024-08-17T17:05:08.627 INFO:tasks.rados.rados.0.smithi063.stdout:2364: writing smithi06387500-36 from 491002 to 1058564 tid 1 2024-08-17T17:05:08.633 INFO:tasks.rados.rados.0.smithi063.stdout:2364: writing smithi06387500-36 from 1663399 to 2183705 tid 2 2024-08-17T17:05:08.637 INFO:tasks.rados.rados.0.smithi063.stdout:2364: writing smithi06387500-36 from 2814301 to 2891002 tid 3 2024-08-17T17:05:08.637 INFO:tasks.rados.rados.0.smithi063.stdout:2365: read oid 15 snap -1 2024-08-17T17:05:08.637 INFO:tasks.rados.rados.0.smithi063.stdout:2365: expect (ObjNum 883 snap 308 seq_num 883) 2024-08-17T17:05:08.638 INFO:tasks.rados.rados.0.smithi063.stdout:2366: write oid 17 current snap is 309 2024-08-17T17:05:08.638 INFO:tasks.rados.rados.0.smithi063.stdout:2362: finishing write tid 3 to smithi06387500-34 2024-08-17T17:05:08.638 INFO:tasks.rados.rados.0.smithi063.stdout:2362: finishing write tid 4 to smithi06387500-34 2024-08-17T17:05:08.638 INFO:tasks.rados.rados.0.smithi063.stdout:2366: seq_num 887 ranges {507809=642811,1721881=663964} 2024-08-17T17:05:08.644 INFO:tasks.rados.rados.0.smithi063.stdout:2366: writing smithi06387500-17 from 507809 to 1150620 tid 1 2024-08-17T17:05:08.652 INFO:tasks.rados.rados.0.smithi063.stdout:2366: writing smithi06387500-17 from 1721881 to 2385845 tid 2 2024-08-17T17:05:08.652 INFO:tasks.rados.rados.0.smithi063.stdout:2367: write oid 47 current snap is 309 2024-08-17T17:05:08.652 INFO:tasks.rados.rados.0.smithi063.stdout:2362: finishing write tid 5 to smithi06387500-34 2024-08-17T17:05:08.652 INFO:tasks.rados.rados.0.smithi063.stdout:2362: oid 34 updating version 0 to 1545 2024-08-17T17:05:08.652 INFO:tasks.rados.rados.0.smithi063.stdout:2362: oid 34 version 1545 is already newer than 1544 2024-08-17T17:05:08.652 INFO:tasks.rados.rados.0.smithi063.stdout:2362: oid 34 updating version 1545 to 1546 2024-08-17T17:05:08.652 INFO:tasks.rados.rados.0.smithi063.stdout:2362: oid 34 version 1546 is already newer than 1543 2024-08-17T17:05:08.652 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 1546 (ObjNum 884 snap 309 seq_num 884) dirty exists 2024-08-17T17:05:08.653 INFO:tasks.rados.rados.0.smithi063.stdout:2362: left oid 34 (ObjNum 884 snap 309 seq_num 884) 2024-08-17T17:05:08.653 INFO:tasks.rados.rados.0.smithi063.stdout:2363: finishing write tid 1 to smithi06387500-13 2024-08-17T17:05:08.653 INFO:tasks.rados.rados.0.smithi063.stdout:2363: finishing write tid 2 to smithi06387500-13 2024-08-17T17:05:08.653 INFO:tasks.rados.rados.0.smithi063.stdout:2363: finishing write tid 3 to smithi06387500-13 2024-08-17T17:05:08.653 INFO:tasks.rados.rados.0.smithi063.stdout:2363: finishing write tid 4 to smithi06387500-13 2024-08-17T17:05:08.653 INFO:tasks.rados.rados.0.smithi063.stdout:2363: finishing write tid 5 to smithi06387500-13 2024-08-17T17:05:08.653 INFO:tasks.rados.rados.0.smithi063.stdout:2363: oid 13 updating version 0 to 1120 2024-08-17T17:05:08.653 INFO:tasks.rados.rados.0.smithi063.stdout:2363: oid 13 version 1120 is already newer than 1119 2024-08-17T17:05:08.653 INFO:tasks.rados.rados.0.smithi063.stdout:2363: oid 13 version 1120 is already newer than 1117 2024-08-17T17:05:08.653 INFO:tasks.rados.rados.0.smithi063.stdout:2363: oid 13 version 1120 is already newer than 1118 2024-08-17T17:05:08.653 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1120 (ObjNum 885 snap 309 seq_num 885) dirty exists 2024-08-17T17:05:08.653 INFO:tasks.rados.rados.0.smithi063.stdout:2363: left oid 13 (ObjNum 885 snap 309 seq_num 885) 2024-08-17T17:05:08.654 INFO:tasks.rados.rados.0.smithi063.stdout:2364: finishing write tid 1 to smithi06387500-36 2024-08-17T17:05:08.654 INFO:tasks.rados.rados.0.smithi063.stdout:2364: finishing write tid 2 to smithi06387500-36 2024-08-17T17:05:08.654 INFO:tasks.rados.rados.0.smithi063.stdout:2367: seq_num 888 ranges {524616=718060,1664010=523975,2752794=171822} 2024-08-17T17:05:08.658 INFO:tasks.rados.rados.0.smithi063.stdout:2367: writing smithi06387500-47 from 524616 to 1242676 tid 1 2024-08-17T17:05:08.663 INFO:tasks.rados.rados.0.smithi063.stdout:2367: writing smithi06387500-47 from 1664010 to 2187985 tid 2 2024-08-17T17:05:08.667 INFO:tasks.rados.rados.0.smithi063.stdout:2367: writing smithi06387500-47 from 2752794 to 2924616 tid 3 2024-08-17T17:05:08.668 INFO:tasks.rados.rados.0.smithi063.stdout:2362: done (5 left) 2024-08-17T17:05:08.668 INFO:tasks.rados.rados.0.smithi063.stdout:2363: done (4 left) 2024-08-17T17:05:08.668 INFO:tasks.rados.rados.0.smithi063.stdout:2368: snap_create 2024-08-17T17:05:08.668 INFO:tasks.rados.rados.0.smithi063.stdout:2364: finishing write tid 3 to smithi06387500-36 2024-08-17T17:05:08.668 INFO:tasks.rados.rados.0.smithi063.stdout:2364: finishing write tid 4 to smithi06387500-36 2024-08-17T17:05:08.668 INFO:tasks.rados.rados.0.smithi063.stdout:2364: finishing write tid 5 to smithi06387500-36 2024-08-17T17:05:08.668 INFO:tasks.rados.rados.0.smithi063.stdout:2364: oid 36 updating version 0 to 2339 2024-08-17T17:05:08.668 INFO:tasks.rados.rados.0.smithi063.stdout:2364: oid 36 version 2339 is already newer than 2338 2024-08-17T17:05:08.668 INFO:tasks.rados.rados.0.smithi063.stdout:2364: oid 36 updating version 2339 to 2341 2024-08-17T17:05:08.668 INFO:tasks.rados.rados.0.smithi063.stdout:2364: oid 36 version 2341 is already newer than 2340 2024-08-17T17:05:08.668 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 2341 (ObjNum 886 snap 309 seq_num 886) dirty exists 2024-08-17T17:05:08.668 INFO:tasks.rados.rados.0.smithi063.stdout:2364: left oid 36 (ObjNum 886 snap 309 seq_num 886) 2024-08-17T17:05:08.669 INFO:tasks.rados.rados.0.smithi063.stdout:2365: expect (ObjNum 883 snap 308 seq_num 883) 2024-08-17T17:05:08.766 INFO:tasks.rados.rados.0.smithi063.stdout:2366: finishing write tid 1 to smithi06387500-17 2024-08-17T17:05:08.767 INFO:tasks.rados.rados.0.smithi063.stdout:2366: finishing write tid 2 to smithi06387500-17 2024-08-17T17:05:08.767 INFO:tasks.rados.rados.0.smithi063.stdout:2366: finishing write tid 3 to smithi06387500-17 2024-08-17T17:05:08.767 INFO:tasks.rados.rados.0.smithi063.stdout:2366: finishing write tid 4 to smithi06387500-17 2024-08-17T17:05:08.767 INFO:tasks.rados.rados.0.smithi063.stdout:2366: oid 17 updating version 0 to 708 2024-08-17T17:05:08.767 INFO:tasks.rados.rados.0.smithi063.stdout:2366: oid 17 version 708 is already newer than 707 2024-08-17T17:05:08.767 INFO:tasks.rados.rados.0.smithi063.stdout:2366: oid 17 updating version 708 to 709 2024-08-17T17:05:08.767 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 709 (ObjNum 887 snap 309 seq_num 887) dirty exists 2024-08-17T17:05:08.767 INFO:tasks.rados.rados.0.smithi063.stdout:2366: left oid 17 (ObjNum 887 snap 309 seq_num 887) 2024-08-17T17:05:08.767 INFO:tasks.rados.rados.0.smithi063.stdout:2367: finishing write tid 1 to smithi06387500-47 2024-08-17T17:05:08.767 INFO:tasks.rados.rados.0.smithi063.stdout:2367: finishing write tid 2 to smithi06387500-47 2024-08-17T17:05:08.767 INFO:tasks.rados.rados.0.smithi063.stdout:2367: finishing write tid 3 to smithi06387500-47 2024-08-17T17:05:08.767 INFO:tasks.rados.rados.0.smithi063.stdout:2367: finishing write tid 4 to smithi06387500-47 2024-08-17T17:05:08.768 INFO:tasks.rados.rados.0.smithi063.stdout:2367: finishing write tid 5 to smithi06387500-47 2024-08-17T17:05:08.768 INFO:tasks.rados.rados.0.smithi063.stdout:2367: oid 47 updating version 0 to 488 2024-08-17T17:05:08.768 INFO:tasks.rados.rados.0.smithi063.stdout:2367: oid 47 version 488 is already newer than 486 2024-08-17T17:05:08.768 INFO:tasks.rados.rados.0.smithi063.stdout:2367: oid 47 version 488 is already newer than 487 2024-08-17T17:05:08.768 INFO:tasks.rados.rados.0.smithi063.stdout:2367: oid 47 updating version 488 to 489 2024-08-17T17:05:08.768 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 489 (ObjNum 888 snap 309 seq_num 888) dirty exists 2024-08-17T17:05:08.768 INFO:tasks.rados.rados.0.smithi063.stdout:2367: left oid 47 (ObjNum 888 snap 309 seq_num 888) 2024-08-17T17:05:09.596 INFO:tasks.rados.rados.0.smithi063.stdout:2364: done (4 left) 2024-08-17T17:05:09.596 INFO:tasks.rados.rados.0.smithi063.stdout:2365: done (3 left) 2024-08-17T17:05:09.596 INFO:tasks.rados.rados.0.smithi063.stdout:2366: done (2 left) 2024-08-17T17:05:09.596 INFO:tasks.rados.rados.0.smithi063.stdout:2367: done (1 left) 2024-08-17T17:05:09.596 INFO:tasks.rados.rados.0.smithi063.stdout:2368: done (0 left) 2024-08-17T17:05:09.596 INFO:tasks.rados.rados.0.smithi063.stdout:2369: read oid 25 snap 302 2024-08-17T17:05:09.596 INFO:tasks.rados.rados.0.smithi063.stdout:2369: expect (ObjNum 858 snap 292 seq_num 858) 2024-08-17T17:05:09.596 INFO:tasks.rados.rados.0.smithi063.stdout:2370: write oid 39 current snap is 310 2024-08-17T17:05:09.596 INFO:tasks.rados.rados.0.smithi063.stdout:2370: seq_num 889 ranges {541423=793309,2122492=667633} 2024-08-17T17:05:09.603 INFO:tasks.rados.rados.0.smithi063.stdout:2370: writing smithi06387500-39 from 541423 to 1334732 tid 1 2024-08-17T17:05:09.612 INFO:tasks.rados.rados.0.smithi063.stdout:2370: writing smithi06387500-39 from 2122492 to 2790125 tid 2 2024-08-17T17:05:09.612 INFO:tasks.rados.rados.0.smithi063.stdout:2371: read oid 42 snap -1 2024-08-17T17:05:09.612 INFO:tasks.rados.rados.0.smithi063.stdout:2371: expect deleted 2024-08-17T17:05:09.612 INFO:tasks.rados.rados.0.smithi063.stdout:2372: write oid 33 current snap is 310 2024-08-17T17:05:09.612 INFO:tasks.rados.rados.0.smithi063.stdout:2369: expect (ObjNum 858 snap 292 seq_num 858) 2024-08-17T17:05:09.763 INFO:tasks.rados.rados.0.smithi063.stdout:2370: finishing write tid 1 to smithi06387500-39 2024-08-17T17:05:09.764 INFO:tasks.rados.rados.0.smithi063.stdout:2370: finishing write tid 2 to smithi06387500-39 2024-08-17T17:05:09.764 INFO:tasks.rados.rados.0.smithi063.stdout:2370: finishing write tid 3 to smithi06387500-39 2024-08-17T17:05:09.764 INFO:tasks.rados.rados.0.smithi063.stdout:2370: finishing write tid 4 to smithi06387500-39 2024-08-17T17:05:09.764 INFO:tasks.rados.rados.0.smithi063.stdout:2370: oid 39 updating version 0 to 512 2024-08-17T17:05:09.764 INFO:tasks.rados.rados.0.smithi063.stdout:2370: oid 39 updating version 512 to 514 2024-08-17T17:05:09.764 INFO:tasks.rados.rados.0.smithi063.stdout:2370: oid 39 version 514 is already newer than 513 2024-08-17T17:05:09.764 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 514 (ObjNum 889 snap 310 seq_num 889) dirty exists 2024-08-17T17:05:09.764 INFO:tasks.rados.rados.0.smithi063.stdout:2370: left oid 39 (ObjNum 889 snap 310 seq_num 889) 2024-08-17T17:05:09.764 INFO:tasks.rados.rados.0.smithi063.stdout:2372: seq_num 890 ranges {558230=584911,1897327=527644,2923993=34237} 2024-08-17T17:05:09.769 INFO:tasks.rados.rados.0.smithi063.stdout:2372: writing smithi06387500-33 from 558230 to 1143141 tid 1 2024-08-17T17:05:09.777 INFO:tasks.rados.rados.0.smithi063.stdout:2372: writing smithi06387500-33 from 1897327 to 2424971 tid 2 2024-08-17T17:05:09.780 INFO:tasks.rados.rados.0.smithi063.stdout:2372: writing smithi06387500-33 from 2923993 to 2958230 tid 3 2024-08-17T17:05:09.781 INFO:tasks.rados.rados.0.smithi063.stdout:2369: done (3 left) 2024-08-17T17:05:09.781 INFO:tasks.rados.rados.0.smithi063.stdout:2370: done (2 left) 2024-08-17T17:05:09.781 INFO:tasks.rados.rados.0.smithi063.stdout:2371: done (1 left) 2024-08-17T17:05:09.781 INFO:tasks.rados.rados.0.smithi063.stdout:2373: write oid 26 current snap is 310 2024-08-17T17:05:09.781 INFO:tasks.rados.rados.0.smithi063.stdout:2372: finishing write tid 1 to smithi06387500-33 2024-08-17T17:05:09.781 INFO:tasks.rados.rados.0.smithi063.stdout:2372: finishing write tid 2 to smithi06387500-33 2024-08-17T17:05:09.781 INFO:tasks.rados.rados.0.smithi063.stdout:2373: seq_num 891 ranges {575037=660160,1955809=671302} 2024-08-17T17:05:09.787 INFO:tasks.rados.rados.0.smithi063.stdout:2373: writing smithi06387500-26 from 575037 to 1235197 tid 1 2024-08-17T17:05:09.795 INFO:tasks.rados.rados.0.smithi063.stdout:2373: writing smithi06387500-26 from 1955809 to 2627111 tid 2 2024-08-17T17:05:09.795 INFO:tasks.rados.rados.0.smithi063.stdout:2374: read oid 13 snap -1 2024-08-17T17:05:09.795 INFO:tasks.rados.rados.0.smithi063.stdout:2374: expect (ObjNum 885 snap 309 seq_num 885) 2024-08-17T17:05:09.795 INFO:tasks.rados.rados.0.smithi063.stdout:2375: delete oid 29 current snap is 310 2024-08-17T17:05:09.795 INFO:tasks.rados.rados.0.smithi063.stdout:2372: finishing write tid 3 to smithi06387500-33 2024-08-17T17:05:09.795 INFO:tasks.rados.rados.0.smithi063.stdout:2372: finishing write tid 4 to smithi06387500-33 2024-08-17T17:05:09.795 INFO:tasks.rados.rados.0.smithi063.stdout:2372: finishing write tid 5 to smithi06387500-33 2024-08-17T17:05:09.796 INFO:tasks.rados.rados.0.smithi063.stdout:2372: oid 33 updating version 0 to 780 2024-08-17T17:05:09.796 INFO:tasks.rados.rados.0.smithi063.stdout:2372: oid 33 updating version 780 to 781 2024-08-17T17:05:09.796 INFO:tasks.rados.rados.0.smithi063.stdout:2372: oid 33 version 781 is already newer than 779 2024-08-17T17:05:09.796 INFO:tasks.rados.rados.0.smithi063.stdout:2372: oid 33 updating version 781 to 782 2024-08-17T17:05:09.796 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 782 (ObjNum 890 snap 310 seq_num 890) dirty exists 2024-08-17T17:05:09.796 INFO:tasks.rados.rados.0.smithi063.stdout:2372: left oid 33 (ObjNum 890 snap 310 seq_num 890) 2024-08-17T17:05:09.796 INFO:tasks.rados.rados.0.smithi063.stdout:2373: finishing write tid 1 to smithi06387500-26 2024-08-17T17:05:09.797 INFO:tasks.rados.rados.0.smithi063.stdout:2372: done (3 left) 2024-08-17T17:05:09.797 INFO:tasks.rados.rados.0.smithi063.stdout:2375: done (2 left) 2024-08-17T17:05:09.797 INFO:tasks.rados.rados.0.smithi063.stdout:2376: rollback oid 4 current snap is 310 2024-08-17T17:05:09.797 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 4 to 291 2024-08-17T17:05:09.797 INFO:tasks.rados.rados.0.smithi063.stdout:2377: rollback oid 38 current snap is 310 2024-08-17T17:05:09.797 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 38 to 306 2024-08-17T17:05:09.797 INFO:tasks.rados.rados.0.smithi063.stdout:2378: write oid 49 current snap is 310 2024-08-17T17:05:09.797 INFO:tasks.rados.rados.0.smithi063.stdout:2378: seq_num 893 ranges {608651=410658,1556420=674971,2773556=235095} 2024-08-17T17:05:09.803 INFO:tasks.rados.rados.0.smithi063.stdout:2378: writing smithi06387500-49 from 608651 to 1019309 tid 1 2024-08-17T17:05:09.809 INFO:tasks.rados.rados.0.smithi063.stdout:2378: writing smithi06387500-49 from 1556420 to 2231391 tid 2 2024-08-17T17:05:09.814 INFO:tasks.rados.rados.0.smithi063.stdout:2378: writing smithi06387500-49 from 2773556 to 3008651 tid 3 2024-08-17T17:05:09.815 INFO:tasks.rados.rados.0.smithi063.stdout:2379: delete oid 5 current snap is 310 2024-08-17T17:05:09.815 INFO:tasks.rados.rados.0.smithi063.stdout:2373: finishing write tid 2 to smithi06387500-26 2024-08-17T17:05:09.815 INFO:tasks.rados.rados.0.smithi063.stdout:2373: finishing write tid 3 to smithi06387500-26 2024-08-17T17:05:09.815 INFO:tasks.rados.rados.0.smithi063.stdout:2373: finishing write tid 4 to smithi06387500-26 2024-08-17T17:05:09.815 INFO:tasks.rados.rados.0.smithi063.stdout:2373: oid 26 updating version 0 to 713 2024-08-17T17:05:09.815 INFO:tasks.rados.rados.0.smithi063.stdout:2373: oid 26 version 713 is already newer than 712 2024-08-17T17:05:09.815 INFO:tasks.rados.rados.0.smithi063.stdout:2373: oid 26 version 713 is already newer than 711 2024-08-17T17:05:09.815 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 713 (ObjNum 891 snap 310 seq_num 891) dirty exists 2024-08-17T17:05:09.815 INFO:tasks.rados.rados.0.smithi063.stdout:2373: left oid 26 (ObjNum 891 snap 310 seq_num 891) 2024-08-17T17:05:09.815 INFO:tasks.rados.rados.0.smithi063.stdout:2377: finishing rollback tid 0 to smithi06387500-38 2024-08-17T17:05:09.815 INFO:tasks.rados.rados.0.smithi063.stdout:2377: finishing rollback tid 1 to smithi06387500-38 2024-08-17T17:05:09.815 INFO:tasks.rados.rados.0.smithi063.stdout:2377: finishing rollback tid 2 to smithi06387500-38 2024-08-17T17:05:09.815 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 380 (ObjNum 868 snap 300 seq_num 868) dirty exists 2024-08-17T17:05:09.816 INFO:tasks.rados.rados.0.smithi063.stdout:2376: finishing rollback tid 1 to smithi06387500-4 2024-08-17T17:05:09.816 INFO:tasks.rados.rados.0.smithi063.stdout:2376: finishing rollback tid 2 to smithi06387500-4 2024-08-17T17:05:09.816 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 2343 (ObjNum 823 snap 283 seq_num 823) dirty exists 2024-08-17T17:05:09.816 INFO:tasks.rados.rados.0.smithi063.stdout:2374: expect (ObjNum 885 snap 309 seq_num 885) 2024-08-17T17:05:09.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:09 smithi162 ceph-mon[104668]: pgmap v477: 89 pgs: 89 active+clean; 193 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 9.0 MiB/s rd, 606 KiB/s wr, 11 op/s 2024-08-17T17:05:09.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:09 smithi162 ceph-mon[104668]: osdmap e681: 8 total, 8 up, 8 in 2024-08-17T17:05:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:09 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:09 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:09.909 INFO:tasks.rados.rados.0.smithi063.stdout:2378: finishing write tid 1 to smithi06387500-49 2024-08-17T17:05:09.909 INFO:tasks.rados.rados.0.smithi063.stdout:2378: finishing write tid 2 to smithi06387500-49 2024-08-17T17:05:09.909 INFO:tasks.rados.rados.0.smithi063.stdout:2378: finishing write tid 3 to smithi06387500-49 2024-08-17T17:05:09.910 INFO:tasks.rados.rados.0.smithi063.stdout:2378: finishing write tid 4 to smithi06387500-49 2024-08-17T17:05:09.910 INFO:tasks.rados.rados.0.smithi063.stdout:2378: finishing write tid 5 to smithi06387500-49 2024-08-17T17:05:09.910 INFO:tasks.rados.rados.0.smithi063.stdout:2378: oid 49 updating version 0 to 2345 2024-08-17T17:05:09.910 INFO:tasks.rados.rados.0.smithi063.stdout:2378: oid 49 updating version 2345 to 2346 2024-08-17T17:05:09.910 INFO:tasks.rados.rados.0.smithi063.stdout:2378: oid 49 updating version 2346 to 2347 2024-08-17T17:05:09.910 INFO:tasks.rados.rados.0.smithi063.stdout:2378: oid 49 version 2347 is already newer than 2344 2024-08-17T17:05:09.910 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 2347 (ObjNum 893 snap 310 seq_num 893) dirty exists 2024-08-17T17:05:09.910 INFO:tasks.rados.rados.0.smithi063.stdout:2378: left oid 49 (ObjNum 893 snap 310 seq_num 893) 2024-08-17T17:05:09.910 INFO:tasks.rados.rados.0.smithi063.stdout:2373: done (5 left) 2024-08-17T17:05:09.910 INFO:tasks.rados.rados.0.smithi063.stdout:2374: done (4 left) 2024-08-17T17:05:09.910 INFO:tasks.rados.rados.0.smithi063.stdout:2376: done (3 left) 2024-08-17T17:05:09.910 INFO:tasks.rados.rados.0.smithi063.stdout:2377: done (2 left) 2024-08-17T17:05:09.910 INFO:tasks.rados.rados.0.smithi063.stdout:2378: done (1 left) 2024-08-17T17:05:09.910 INFO:tasks.rados.rados.0.smithi063.stdout:2379: done (0 left) 2024-08-17T17:05:09.911 INFO:tasks.rados.rados.0.smithi063.stdout:2380: write oid 15 current snap is 310 2024-08-17T17:05:09.911 INFO:tasks.rados.rados.0.smithi063.stdout:2380: seq_num 895 ranges {642265=561156,1673384=678640,2828402=213863} 2024-08-17T17:05:09.916 INFO:tasks.rados.rados.0.smithi063.stdout:2380: writing smithi06387500-15 from 642265 to 1203421 tid 1 2024-08-17T17:05:09.922 INFO:tasks.rados.rados.0.smithi063.stdout:2380: writing smithi06387500-15 from 1673384 to 2352024 tid 2 2024-08-17T17:05:09.927 INFO:tasks.rados.rados.0.smithi063.stdout:2380: writing smithi06387500-15 from 2828402 to 3042265 tid 3 2024-08-17T17:05:09.927 INFO:tasks.rados.rados.0.smithi063.stdout:2381: read oid 14 snap -1 2024-08-17T17:05:09.927 INFO:tasks.rados.rados.0.smithi063.stdout:2381: expect (ObjNum 852 snap 290 seq_num 852) 2024-08-17T17:05:09.927 INFO:tasks.rados.rados.0.smithi063.stdout:2382: write oid 49 current snap is 310 2024-08-17T17:05:09.927 INFO:tasks.rados.rados.0.smithi063.stdout:2380: finishing write tid 1 to smithi06387500-15 2024-08-17T17:05:09.927 INFO:tasks.rados.rados.0.smithi063.stdout:2380: finishing write tid 2 to smithi06387500-15 2024-08-17T17:05:09.927 INFO:tasks.rados.rados.0.smithi063.stdout:2382: seq_num 896 ranges {659072=636405,1731866=422298,2855825=203247} 2024-08-17T17:05:09.933 INFO:tasks.rados.rados.0.smithi063.stdout:2382: writing smithi06387500-49 from 659072 to 1295477 tid 1 2024-08-17T17:05:09.938 INFO:tasks.rados.rados.0.smithi063.stdout:2382: writing smithi06387500-49 from 1731866 to 2154164 tid 2 2024-08-17T17:05:09.944 INFO:tasks.rados.rados.0.smithi063.stdout:2382: writing smithi06387500-49 from 2855825 to 3059072 tid 3 2024-08-17T17:05:09.944 INFO:tasks.rados.rados.0.smithi063.stdout:2383: snap_create 2024-08-17T17:05:09.944 INFO:tasks.rados.rados.0.smithi063.stdout:2380: finishing write tid 3 to smithi06387500-15 2024-08-17T17:05:09.944 INFO:tasks.rados.rados.0.smithi063.stdout:2380: finishing write tid 4 to smithi06387500-15 2024-08-17T17:05:09.944 INFO:tasks.rados.rados.0.smithi063.stdout:2380: finishing write tid 5 to smithi06387500-15 2024-08-17T17:05:09.944 INFO:tasks.rados.rados.0.smithi063.stdout:2380: oid 15 updating version 0 to 920 2024-08-17T17:05:09.944 INFO:tasks.rados.rados.0.smithi063.stdout:2380: oid 15 version 920 is already newer than 918 2024-08-17T17:05:09.944 INFO:tasks.rados.rados.0.smithi063.stdout:2380: oid 15 version 920 is already newer than 919 2024-08-17T17:05:09.944 INFO:tasks.rados.rados.0.smithi063.stdout:2380: oid 15 version 920 is already newer than 917 2024-08-17T17:05:09.944 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 920 (ObjNum 895 snap 310 seq_num 895) dirty exists 2024-08-17T17:05:09.944 INFO:tasks.rados.rados.0.smithi063.stdout:2380: left oid 15 (ObjNum 895 snap 310 seq_num 895) 2024-08-17T17:05:09.944 INFO:tasks.rados.rados.0.smithi063.stdout:2381: expect (ObjNum 852 snap 290 seq_num 852) 2024-08-17T17:05:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:09 smithi063 ceph-mon[114756]: pgmap v477: 89 pgs: 89 active+clean; 193 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 9.0 MiB/s rd, 606 KiB/s wr, 11 op/s 2024-08-17T17:05:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:09 smithi063 ceph-mon[114756]: osdmap e681: 8 total, 8 up, 8 in 2024-08-17T17:05:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:09 smithi063 ceph-mon[109843]: pgmap v477: 89 pgs: 89 active+clean; 193 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 9.0 MiB/s rd, 606 KiB/s wr, 11 op/s 2024-08-17T17:05:10.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:09 smithi063 ceph-mon[109843]: osdmap e681: 8 total, 8 up, 8 in 2024-08-17T17:05:10.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:10.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:10.050 INFO:tasks.rados.rados.0.smithi063.stdout:2382: finishing write tid 1 to smithi06387500-49 2024-08-17T17:05:10.051 INFO:tasks.rados.rados.0.smithi063.stdout:2382: finishing write tid 2 to smithi06387500-49 2024-08-17T17:05:10.051 INFO:tasks.rados.rados.0.smithi063.stdout:2382: finishing write tid 3 to smithi06387500-49 2024-08-17T17:05:10.051 INFO:tasks.rados.rados.0.smithi063.stdout:2382: finishing write tid 4 to smithi06387500-49 2024-08-17T17:05:10.051 INFO:tasks.rados.rados.0.smithi063.stdout:2382: finishing write tid 5 to smithi06387500-49 2024-08-17T17:05:10.051 INFO:tasks.rados.rados.0.smithi063.stdout:2382: oid 49 updating version 0 to 2350 2024-08-17T17:05:10.051 INFO:tasks.rados.rados.0.smithi063.stdout:2382: oid 49 version 2350 is already newer than 2348 2024-08-17T17:05:10.051 INFO:tasks.rados.rados.0.smithi063.stdout:2382: oid 49 updating version 2350 to 2351 2024-08-17T17:05:10.051 INFO:tasks.rados.rados.0.smithi063.stdout:2382: oid 49 version 2351 is already newer than 2349 2024-08-17T17:05:10.051 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 2351 (ObjNum 896 snap 310 seq_num 896) dirty exists 2024-08-17T17:05:10.051 INFO:tasks.rados.rados.0.smithi063.stdout:2382: left oid 49 (ObjNum 896 snap 310 seq_num 896) 2024-08-17T17:05:10.652 INFO:tasks.rados.rados.0.smithi063.stdout:2380: done (3 left) 2024-08-17T17:05:10.652 INFO:tasks.rados.rados.0.smithi063.stdout:2381: done (2 left) 2024-08-17T17:05:10.652 INFO:tasks.rados.rados.0.smithi063.stdout:2382: done (1 left) 2024-08-17T17:05:10.652 INFO:tasks.rados.rados.0.smithi063.stdout:2383: done (0 left) 2024-08-17T17:05:10.652 INFO:tasks.rados.rados.0.smithi063.stdout:2384: write oid 9 current snap is 311 2024-08-17T17:05:10.652 INFO:tasks.rados.rados.0.smithi063.stdout:2384: seq_num 897 ranges {675879=711654,2073995=682309} 2024-08-17T17:05:10.659 INFO:tasks.rados.rados.0.smithi063.stdout:2384: writing smithi06387500-9 from 675879 to 1387533 tid 1 2024-08-17T17:05:10.667 INFO:tasks.rados.rados.0.smithi063.stdout:2384: writing smithi06387500-9 from 2073995 to 2756304 tid 2 2024-08-17T17:05:10.668 INFO:tasks.rados.rados.0.smithi063.stdout:2385: delete oid 25 current snap is 311 2024-08-17T17:05:10.668 INFO:tasks.rados.rados.0.smithi063.stdout:2384: finishing write tid 1 to smithi06387500-9 2024-08-17T17:05:10.672 INFO:tasks.rados.rados.0.smithi063.stdout:2384: finishing write tid 2 to smithi06387500-9 2024-08-17T17:05:10.673 INFO:tasks.rados.rados.0.smithi063.stdout:2385: done (1 left) 2024-08-17T17:05:10.673 INFO:tasks.rados.rados.0.smithi063.stdout:2386: read oid 40 snap -1 2024-08-17T17:05:10.673 INFO:tasks.rados.rados.0.smithi063.stdout:2386: expect (ObjNum 878 snap 306 seq_num 878) 2024-08-17T17:05:10.673 INFO:tasks.rados.rados.0.smithi063.stdout:2387: write oid 5 current snap is 311 2024-08-17T17:05:10.673 INFO:tasks.rados.rados.0.smithi063.stdout:2384: finishing write tid 3 to smithi06387500-9 2024-08-17T17:05:10.674 INFO:tasks.rados.rados.0.smithi063.stdout:2387: seq_num 899 ranges {709493=578505,1907312=685978} 2024-08-17T17:05:10.680 INFO:tasks.rados.rados.0.smithi063.stdout:2387: writing smithi06387500-5 from 709493 to 1287998 tid 1 2024-08-17T17:05:10.688 INFO:tasks.rados.rados.0.smithi063.stdout:2387: writing smithi06387500-5 from 1907312 to 2593290 tid 2 2024-08-17T17:05:10.689 INFO:tasks.rados.rados.0.smithi063.stdout:2388: write oid 6 current snap is 311 2024-08-17T17:05:10.689 INFO:tasks.rados.rados.0.smithi063.stdout:2384: finishing write tid 4 to smithi06387500-9 2024-08-17T17:05:10.689 INFO:tasks.rados.rados.0.smithi063.stdout:2384: oid 9 updating version 0 to 1059 2024-08-17T17:05:10.689 INFO:tasks.rados.rados.0.smithi063.stdout:2384: oid 9 updating version 1059 to 1061 2024-08-17T17:05:10.689 INFO:tasks.rados.rados.0.smithi063.stdout:2384: oid 9 version 1061 is already newer than 1060 2024-08-17T17:05:10.689 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 1061 (ObjNum 897 snap 311 seq_num 897) dirty exists 2024-08-17T17:05:10.689 INFO:tasks.rados.rados.0.smithi063.stdout:2384: left oid 9 (ObjNum 897 snap 311 seq_num 897) 2024-08-17T17:05:10.689 INFO:tasks.rados.rados.0.smithi063.stdout:2386: expect (ObjNum 878 snap 306 seq_num 878) 2024-08-17T17:05:10.814 INFO:tasks.rados.rados.0.smithi063.stdout:2387: finishing write tid 1 to smithi06387500-5 2024-08-17T17:05:10.814 INFO:tasks.rados.rados.0.smithi063.stdout:2387: finishing write tid 2 to smithi06387500-5 2024-08-17T17:05:10.815 INFO:tasks.rados.rados.0.smithi063.stdout:2387: finishing write tid 3 to smithi06387500-5 2024-08-17T17:05:10.815 INFO:tasks.rados.rados.0.smithi063.stdout:2387: finishing write tid 4 to smithi06387500-5 2024-08-17T17:05:10.815 INFO:tasks.rados.rados.0.smithi063.stdout:2387: oid 5 updating version 0 to 1125 2024-08-17T17:05:10.815 INFO:tasks.rados.rados.0.smithi063.stdout:2387: oid 5 version 1125 is already newer than 1123 2024-08-17T17:05:10.815 INFO:tasks.rados.rados.0.smithi063.stdout:2387: oid 5 version 1125 is already newer than 1124 2024-08-17T17:05:10.815 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1125 (ObjNum 899 snap 311 seq_num 899) dirty exists 2024-08-17T17:05:10.815 INFO:tasks.rados.rados.0.smithi063.stdout:2387: left oid 5 (ObjNum 899 snap 311 seq_num 899) 2024-08-17T17:05:10.815 INFO:tasks.rados.rados.0.smithi063.stdout:2388: seq_num 900 ranges {726300=653754,1965794=429636,2965517=160783} 2024-08-17T17:05:10.821 INFO:tasks.rados.rados.0.smithi063.stdout:2388: writing smithi06387500-6 from 726300 to 1380054 tid 1 2024-08-17T17:05:10.827 INFO:tasks.rados.rados.0.smithi063.stdout:2388: writing smithi06387500-6 from 1965794 to 2395430 tid 2 2024-08-17T17:05:10.832 INFO:tasks.rados.rados.0.smithi063.stdout:2388: writing smithi06387500-6 from 2965517 to 3126300 tid 3 2024-08-17T17:05:10.832 INFO:tasks.rados.rados.0.smithi063.stdout:2384: done (3 left) 2024-08-17T17:05:10.832 INFO:tasks.rados.rados.0.smithi063.stdout:2386: done (2 left) 2024-08-17T17:05:10.832 INFO:tasks.rados.rados.0.smithi063.stdout:2387: done (1 left) 2024-08-17T17:05:10.832 INFO:tasks.rados.rados.0.smithi063.stdout:2389: read oid 25 snap -1 2024-08-17T17:05:10.832 INFO:tasks.rados.rados.0.smithi063.stdout:2389: expect deleted 2024-08-17T17:05:10.832 INFO:tasks.rados.rados.0.smithi063.stdout:2390: read oid 17 snap -1 2024-08-17T17:05:10.832 INFO:tasks.rados.rados.0.smithi063.stdout:2390: expect (ObjNum 887 snap 309 seq_num 887) 2024-08-17T17:05:10.832 INFO:tasks.rados.rados.0.smithi063.stdout:2391: read oid 41 snap -1 2024-08-17T17:05:10.832 INFO:tasks.rados.rados.0.smithi063.stdout:2391: expect (ObjNum 881 snap 308 seq_num 881) 2024-08-17T17:05:10.833 INFO:tasks.rados.rados.0.smithi063.stdout:2392: write oid 33 current snap is 311 2024-08-17T17:05:10.833 INFO:tasks.rados.rados.0.smithi063.stdout:2388: finishing write tid 1 to smithi06387500-6 2024-08-17T17:05:10.833 INFO:tasks.rados.rados.0.smithi063.stdout:2388: finishing write tid 2 to smithi06387500-6 2024-08-17T17:05:10.833 INFO:tasks.rados.rados.0.smithi063.stdout:2392: seq_num 901 ranges {743107=729003,1907923=689647} 2024-08-17T17:05:10.840 INFO:tasks.rados.rados.0.smithi063.stdout:2392: writing smithi06387500-33 from 743107 to 1472110 tid 1 2024-08-17T17:05:10.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:10 smithi162 ceph-mon[104668]: osdmap e682: 8 total, 8 up, 8 in 2024-08-17T17:05:10.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:10 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:10.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:10 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:10.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:10 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:10.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:10 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:05:10.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:10 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:10.847 INFO:tasks.rados.rados.0.smithi063.stdout:2392: writing smithi06387500-33 from 1907923 to 2597570 tid 2 2024-08-17T17:05:10.847 INFO:tasks.rados.rados.0.smithi063.stdout:2393: delete oid 42 current snap is 311 2024-08-17T17:05:10.847 INFO:tasks.rados.rados.0.smithi063.stdout:2388: finishing write tid 3 to smithi06387500-6 2024-08-17T17:05:10.847 INFO:tasks.rados.rados.0.smithi063.stdout:2388: finishing write tid 4 to smithi06387500-6 2024-08-17T17:05:10.847 INFO:tasks.rados.rados.0.smithi063.stdout:2388: finishing write tid 5 to smithi06387500-6 2024-08-17T17:05:10.847 INFO:tasks.rados.rados.0.smithi063.stdout:2388: oid 6 updating version 0 to 787 2024-08-17T17:05:10.847 INFO:tasks.rados.rados.0.smithi063.stdout:2388: oid 6 version 787 is already newer than 784 2024-08-17T17:05:10.847 INFO:tasks.rados.rados.0.smithi063.stdout:2388: oid 6 version 787 is already newer than 786 2024-08-17T17:05:10.847 INFO:tasks.rados.rados.0.smithi063.stdout:2388: oid 6 version 787 is already newer than 785 2024-08-17T17:05:10.847 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 787 (ObjNum 900 snap 311 seq_num 900) dirty exists 2024-08-17T17:05:10.847 INFO:tasks.rados.rados.0.smithi063.stdout:2388: left oid 6 (ObjNum 900 snap 311 seq_num 900) 2024-08-17T17:05:10.848 INFO:tasks.rados.rados.0.smithi063.stdout:2390: expect (ObjNum 887 snap 309 seq_num 887) 2024-08-17T17:05:10.944 INFO:tasks.rados.rados.0.smithi063.stdout:2392: finishing write tid 1 to smithi06387500-33 2024-08-17T17:05:10.945 INFO:tasks.rados.rados.0.smithi063.stdout:2391: expect (ObjNum 881 snap 308 seq_num 881) 2024-08-17T17:05:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:10 smithi063 ceph-mon[114756]: osdmap e682: 8 total, 8 up, 8 in 2024-08-17T17:05:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:10 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:10 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:10 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:11.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:10 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:05:11.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:10 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:10 smithi063 ceph-mon[109843]: osdmap e682: 8 total, 8 up, 8 in 2024-08-17T17:05:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:10 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:10 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:10 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:10 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:05:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:10 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:11.036 INFO:tasks.rados.rados.0.smithi063.stdout:2392: finishing write tid 2 to smithi06387500-33 2024-08-17T17:05:11.036 INFO:tasks.rados.rados.0.smithi063.stdout:2392: finishing write tid 3 to smithi06387500-33 2024-08-17T17:05:11.036 INFO:tasks.rados.rados.0.smithi063.stdout:2392: finishing write tid 4 to smithi06387500-33 2024-08-17T17:05:11.036 INFO:tasks.rados.rados.0.smithi063.stdout:2392: oid 33 updating version 0 to 789 2024-08-17T17:05:11.037 INFO:tasks.rados.rados.0.smithi063.stdout:2392: oid 33 updating version 789 to 790 2024-08-17T17:05:11.037 INFO:tasks.rados.rados.0.smithi063.stdout:2392: oid 33 updating version 790 to 791 2024-08-17T17:05:11.037 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 791 (ObjNum 901 snap 311 seq_num 901) dirty exists 2024-08-17T17:05:11.037 INFO:tasks.rados.rados.0.smithi063.stdout:2392: left oid 33 (ObjNum 901 snap 311 seq_num 901) 2024-08-17T17:05:11.037 INFO:tasks.rados.rados.0.smithi063.stdout:2388: done (5 left) 2024-08-17T17:05:11.037 INFO:tasks.rados.rados.0.smithi063.stdout:2389: done (4 left) 2024-08-17T17:05:11.037 INFO:tasks.rados.rados.0.smithi063.stdout:2390: done (3 left) 2024-08-17T17:05:11.037 INFO:tasks.rados.rados.0.smithi063.stdout:2391: done (2 left) 2024-08-17T17:05:11.037 INFO:tasks.rados.rados.0.smithi063.stdout:2392: done (1 left) 2024-08-17T17:05:11.037 INFO:tasks.rados.rados.0.smithi063.stdout:2393: done (0 left) 2024-08-17T17:05:11.037 INFO:tasks.rados.rados.0.smithi063.stdout:2394: snap_remove snap 281 2024-08-17T17:05:11.652 INFO:tasks.rados.rados.0.smithi063.stdout:2394: done (0 left) 2024-08-17T17:05:11.693 INFO:tasks.rados.rados.0.smithi063.stdout:2395: read oid 48 snap -1 2024-08-17T17:05:11.693 INFO:tasks.rados.rados.0.smithi063.stdout:2395: expect (ObjNum 857 snap 292 seq_num 857) 2024-08-17T17:05:11.693 INFO:tasks.rados.rados.0.smithi063.stdout:2396: read oid 37 snap -1 2024-08-17T17:05:11.694 INFO:tasks.rados.rados.0.smithi063.stdout:2396: expect (ObjNum 843 snap 290 seq_num 843) 2024-08-17T17:05:11.694 INFO:tasks.rados.rados.0.smithi063.stdout:2397: read oid 31 snap 300 2024-08-17T17:05:11.694 INFO:tasks.rados.rados.0.smithi063.stdout:2397: expect (ObjNum 821 snap 283 seq_num 821) 2024-08-17T17:05:11.694 INFO:tasks.rados.rados.0.smithi063.stdout:2398: delete oid 14 current snap is 311 2024-08-17T17:05:11.694 INFO:tasks.rados.rados.0.smithi063.stdout:2395: expect (ObjNum 857 snap 292 seq_num 857) 2024-08-17T17:05:11.812 INFO:tasks.rados.rados.0.smithi063.stdout:2397: expect (ObjNum 821 snap 283 seq_num 821) 2024-08-17T17:05:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:11 smithi162 ceph-mon[104668]: pgmap v480: 89 pgs: 89 active+clean; 193 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 9.0 MiB/s rd, 606 KiB/s wr, 11 op/s 2024-08-17T17:05:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:11 smithi162 ceph-mon[104668]: osdmap e683: 8 total, 8 up, 8 in 2024-08-17T17:05:11.930 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:11 smithi063 ceph-mon[114756]: pgmap v480: 89 pgs: 89 active+clean; 193 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 9.0 MiB/s rd, 606 KiB/s wr, 11 op/s 2024-08-17T17:05:11.930 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:11 smithi063 ceph-mon[114756]: osdmap e683: 8 total, 8 up, 8 in 2024-08-17T17:05:11.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:11 smithi063 ceph-mon[109843]: pgmap v480: 89 pgs: 89 active+clean; 193 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 9.0 MiB/s rd, 606 KiB/s wr, 11 op/s 2024-08-17T17:05:11.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:11 smithi063 ceph-mon[109843]: osdmap e683: 8 total, 8 up, 8 in 2024-08-17T17:05:11.991 INFO:tasks.rados.rados.0.smithi063.stdout:2396: expect (ObjNum 843 snap 290 seq_num 843) 2024-08-17T17:05:12.114 INFO:tasks.rados.rados.0.smithi063.stdout:2395: done (3 left) 2024-08-17T17:05:12.114 INFO:tasks.rados.rados.0.smithi063.stdout:2396: done (2 left) 2024-08-17T17:05:12.114 INFO:tasks.rados.rados.0.smithi063.stdout:2397: done (1 left) 2024-08-17T17:05:12.114 INFO:tasks.rados.rados.0.smithi063.stdout:2398: done (0 left) 2024-08-17T17:05:12.115 INFO:tasks.rados.rados.0.smithi063.stdout:2399: write oid 33 current snap is 311 2024-08-17T17:05:12.115 INFO:tasks.rados.rados.0.smithi063.stdout:2399: seq_num 904 ranges {793528=554750,2083369=436974,2958856=234672} 2024-08-17T17:05:12.121 INFO:tasks.rados.rados.0.smithi063.stdout:2399: writing smithi06387500-33 from 793528 to 1348278 tid 1 2024-08-17T17:05:12.128 INFO:tasks.rados.rados.0.smithi063.stdout:2399: writing smithi06387500-33 from 2083369 to 2520343 tid 2 2024-08-17T17:05:12.132 INFO:tasks.rados.rados.0.smithi063.stdout:2399: writing smithi06387500-33 from 2958856 to 3193528 tid 3 2024-08-17T17:05:12.132 INFO:tasks.rados.rados.0.smithi063.stdout:2400: read oid 26 snap 304 2024-08-17T17:05:12.132 INFO:tasks.rados.rados.0.smithi063.stdout:2400: expect (ObjNum 733 snap 247 seq_num 733) 2024-08-17T17:05:12.132 INFO:tasks.rados.rados.0.smithi063.stdout:2401: write oid 12 current snap is 311 2024-08-17T17:05:12.132 INFO:tasks.rados.rados.0.smithi063.stdout:2399: finishing write tid 1 to smithi06387500-33 2024-08-17T17:05:12.133 INFO:tasks.rados.rados.0.smithi063.stdout:2399: finishing write tid 2 to smithi06387500-33 2024-08-17T17:05:12.133 INFO:tasks.rados.rados.0.smithi063.stdout:2401: seq_num 905 ranges {410335=746352,1741851=696985,3102632=107703} 2024-08-17T17:05:12.139 INFO:tasks.rados.rados.0.smithi063.stdout:2401: writing smithi06387500-12 from 410335 to 1156687 tid 1 2024-08-17T17:05:12.147 INFO:tasks.rados.rados.0.smithi063.stdout:2401: writing smithi06387500-12 from 1741851 to 2438836 tid 2 2024-08-17T17:05:12.152 INFO:tasks.rados.rados.0.smithi063.stdout:2401: writing smithi06387500-12 from 3102632 to 3210335 tid 3 2024-08-17T17:05:12.152 INFO:tasks.rados.rados.0.smithi063.stdout:2402: rollback oid 50 current snap is 311 2024-08-17T17:05:12.152 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 50 to 296 2024-08-17T17:05:12.152 INFO:tasks.rados.rados.0.smithi063.stdout:2399: finishing write tid 3 to smithi06387500-33 2024-08-17T17:05:12.152 INFO:tasks.rados.rados.0.smithi063.stdout:2403: read oid 47 snap -1 2024-08-17T17:05:12.152 INFO:tasks.rados.rados.0.smithi063.stdout:2403: expect (ObjNum 888 snap 309 seq_num 888) 2024-08-17T17:05:12.152 INFO:tasks.rados.rados.0.smithi063.stdout:2404: delete oid 40 current snap is 311 2024-08-17T17:05:12.152 INFO:tasks.rados.rados.0.smithi063.stdout:2399: finishing write tid 4 to smithi06387500-33 2024-08-17T17:05:12.152 INFO:tasks.rados.rados.0.smithi063.stdout:2399: finishing write tid 5 to smithi06387500-33 2024-08-17T17:05:12.152 INFO:tasks.rados.rados.0.smithi063.stdout:2399: oid 33 updating version 0 to 798 2024-08-17T17:05:12.153 INFO:tasks.rados.rados.0.smithi063.stdout:2399: oid 33 updating version 798 to 799 2024-08-17T17:05:12.153 INFO:tasks.rados.rados.0.smithi063.stdout:2399: oid 33 updating version 799 to 800 2024-08-17T17:05:12.153 INFO:tasks.rados.rados.0.smithi063.stdout:2399: oid 33 updating version 800 to 801 2024-08-17T17:05:12.153 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 801 (ObjNum 904 snap 311 seq_num 904) dirty exists 2024-08-17T17:05:12.153 INFO:tasks.rados.rados.0.smithi063.stdout:2399: left oid 33 (ObjNum 904 snap 311 seq_num 904) 2024-08-17T17:05:12.153 INFO:tasks.rados.rados.0.smithi063.stdout:2400: expect (ObjNum 733 snap 247 seq_num 733) 2024-08-17T17:05:12.311 INFO:tasks.rados.rados.0.smithi063.stdout:2401: finishing write tid 1 to smithi06387500-12 2024-08-17T17:05:12.311 INFO:tasks.rados.rados.0.smithi063.stdout:2401: finishing write tid 2 to smithi06387500-12 2024-08-17T17:05:12.311 INFO:tasks.rados.rados.0.smithi063.stdout:2401: finishing write tid 3 to smithi06387500-12 2024-08-17T17:05:12.311 INFO:tasks.rados.rados.0.smithi063.stdout:2401: finishing write tid 4 to smithi06387500-12 2024-08-17T17:05:12.311 INFO:tasks.rados.rados.0.smithi063.stdout:2401: finishing write tid 5 to smithi06387500-12 2024-08-17T17:05:12.311 INFO:tasks.rados.rados.0.smithi063.stdout:2401: oid 12 updating version 0 to 593 2024-08-17T17:05:12.311 INFO:tasks.rados.rados.0.smithi063.stdout:2401: oid 12 updating version 593 to 595 2024-08-17T17:05:12.312 INFO:tasks.rados.rados.0.smithi063.stdout:2401: oid 12 version 595 is already newer than 594 2024-08-17T17:05:12.312 INFO:tasks.rados.rados.0.smithi063.stdout:2401: oid 12 version 595 is already newer than 592 2024-08-17T17:05:12.312 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 595 (ObjNum 905 snap 311 seq_num 905) dirty exists 2024-08-17T17:05:12.312 INFO:tasks.rados.rados.0.smithi063.stdout:2401: left oid 12 (ObjNum 905 snap 311 seq_num 905) 2024-08-17T17:05:12.312 INFO:tasks.rados.rados.0.smithi063.stdout:2403: expect (ObjNum 888 snap 309 seq_num 888) 2024-08-17T17:05:12.407 INFO:tasks.rados.rados.0.smithi063.stdout:2402: finishing rollback tid 0 to smithi06387500-50 2024-08-17T17:05:12.407 INFO:tasks.rados.rados.0.smithi063.stdout:2402: finishing rollback tid 1 to smithi06387500-50 2024-08-17T17:05:12.407 INFO:tasks.rados.rados.0.smithi063.stdout:2402: finishing rollback tid 2 to smithi06387500-50 2024-08-17T17:05:12.407 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 1557 (ObjNum 850 snap 290 seq_num 850) dirty exists 2024-08-17T17:05:12.409 INFO:tasks.rados.rados.0.smithi063.stdout:2399: done (5 left) 2024-08-17T17:05:12.409 INFO:tasks.rados.rados.0.smithi063.stdout:2400: done (4 left) 2024-08-17T17:05:12.409 INFO:tasks.rados.rados.0.smithi063.stdout:2401: done (3 left) 2024-08-17T17:05:12.409 INFO:tasks.rados.rados.0.smithi063.stdout:2402: done (2 left) 2024-08-17T17:05:12.409 INFO:tasks.rados.rados.0.smithi063.stdout:2403: done (1 left) 2024-08-17T17:05:12.410 INFO:tasks.rados.rados.0.smithi063.stdout:2404: done (0 left) 2024-08-17T17:05:12.410 INFO:tasks.rados.rados.0.smithi063.stdout:2405: snap_remove snap 291 2024-08-17T17:05:12.583 INFO:tasks.rados.rados.0.smithi063.stdout:2405: done (0 left) 2024-08-17T17:05:12.583 INFO:tasks.rados.rados.0.smithi063.stdout:2406: read oid 16 snap 309 2024-08-17T17:05:12.583 INFO:tasks.rados.rados.0.smithi063.stdout:2406: expect deleted 2024-08-17T17:05:12.583 INFO:tasks.rados.rados.0.smithi063.stdout:2407: delete oid 9 current snap is 311 2024-08-17T17:05:12.594 INFO:tasks.rados.rados.0.smithi063.stdout:2406: done (1 left) 2024-08-17T17:05:12.594 INFO:tasks.rados.rados.0.smithi063.stdout:2407: done (0 left) 2024-08-17T17:05:12.594 INFO:tasks.rados.rados.0.smithi063.stdout:2408: rollback oid 37 current snap is 311 2024-08-17T17:05:12.594 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 287 2024-08-17T17:05:12.594 INFO:tasks.rados.rados.0.smithi063.stdout:2409: rollback oid 34 current snap is 311 2024-08-17T17:05:12.594 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 34 to 310 2024-08-17T17:05:12.594 INFO:tasks.rados.rados.0.smithi063.stdout:2410: write oid 50 current snap is 311 2024-08-17T17:05:12.594 INFO:tasks.rados.rados.0.smithi063.stdout:2410: seq_num 908 ranges {460756=572099,1517297=444312,2668548=472870} 2024-08-17T17:05:12.600 INFO:tasks.rados.rados.0.smithi063.stdout:2410: writing smithi06387500-50 from 460756 to 1032855 tid 1 2024-08-17T17:05:12.605 INFO:tasks.rados.rados.0.smithi063.stdout:2410: writing smithi06387500-50 from 1517297 to 1961609 tid 2 2024-08-17T17:05:12.612 INFO:tasks.rados.rados.0.smithi063.stdout:2410: writing smithi06387500-50 from 2668548 to 3141418 tid 3 2024-08-17T17:05:12.612 INFO:tasks.rados.rados.0.smithi063.stdout:2411: snap_create 2024-08-17T17:05:12.612 INFO:tasks.rados.rados.0.smithi063.stdout:2409: finishing rollback tid 0 to smithi06387500-34 2024-08-17T17:05:12.612 INFO:tasks.rados.rados.0.smithi063.stdout:2408: finishing rollback tid 0 to smithi06387500-37 2024-08-17T17:05:12.612 INFO:tasks.rados.rados.0.smithi063.stdout:2409: finishing rollback tid 1 to smithi06387500-34 2024-08-17T17:05:12.612 INFO:tasks.rados.rados.0.smithi063.stdout:2409: finishing rollback tid 2 to smithi06387500-34 2024-08-17T17:05:12.612 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 1567 (ObjNum 884 snap 309 seq_num 884) dirty exists 2024-08-17T17:05:12.612 INFO:tasks.rados.rados.0.smithi063.stdout:2408: finishing rollback tid 1 to smithi06387500-37 2024-08-17T17:05:12.612 INFO:tasks.rados.rados.0.smithi063.stdout:2408: finishing rollback tid 2 to smithi06387500-37 2024-08-17T17:05:12.612 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 2375 (ObjNum 696 snap 222 seq_num 696) dirty exists 2024-08-17T17:05:12.612 INFO:tasks.rados.rados.0.smithi063.stdout:2410: finishing write tid 1 to smithi06387500-50 2024-08-17T17:05:12.612 INFO:tasks.rados.rados.0.smithi063.stdout:2410: finishing write tid 2 to smithi06387500-50 2024-08-17T17:05:12.614 INFO:tasks.rados.rados.0.smithi063.stdout:2410: finishing write tid 3 to smithi06387500-50 2024-08-17T17:05:12.615 INFO:tasks.rados.rados.0.smithi063.stdout:2410: finishing write tid 4 to smithi06387500-50 2024-08-17T17:05:12.615 INFO:tasks.rados.rados.0.smithi063.stdout:2410: finishing write tid 5 to smithi06387500-50 2024-08-17T17:05:12.615 INFO:tasks.rados.rados.0.smithi063.stdout:2410: oid 50 updating version 0 to 1568 2024-08-17T17:05:12.615 INFO:tasks.rados.rados.0.smithi063.stdout:2410: oid 50 updating version 1568 to 1571 2024-08-17T17:05:12.615 INFO:tasks.rados.rados.0.smithi063.stdout:2410: oid 50 version 1571 is already newer than 1569 2024-08-17T17:05:12.615 INFO:tasks.rados.rados.0.smithi063.stdout:2410: oid 50 version 1571 is already newer than 1570 2024-08-17T17:05:12.615 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 1571 (ObjNum 908 snap 311 seq_num 908) dirty exists 2024-08-17T17:05:12.615 INFO:tasks.rados.rados.0.smithi063.stdout:2410: left oid 50 (ObjNum 908 snap 311 seq_num 908) 2024-08-17T17:05:12.942 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:12 smithi063 ceph-mon[114756]: osdmap e684: 8 total, 8 up, 8 in 2024-08-17T17:05:12.942 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:12 smithi063 ceph-mon[114756]: pgmap v483: 89 pgs: 89 active+clean; 224 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 27 MiB/s rd, 5.7 MiB/s wr, 51 op/s 2024-08-17T17:05:12.942 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:12 smithi063 ceph-mon[114756]: osdmap e685: 8 total, 8 up, 8 in 2024-08-17T17:05:12.943 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:12 smithi063 ceph-mon[109843]: osdmap e684: 8 total, 8 up, 8 in 2024-08-17T17:05:12.943 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:12 smithi063 ceph-mon[109843]: pgmap v483: 89 pgs: 89 active+clean; 224 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 27 MiB/s rd, 5.7 MiB/s wr, 51 op/s 2024-08-17T17:05:12.943 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:12 smithi063 ceph-mon[109843]: osdmap e685: 8 total, 8 up, 8 in 2024-08-17T17:05:13.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:12 smithi162 ceph-mon[104668]: osdmap e684: 8 total, 8 up, 8 in 2024-08-17T17:05:13.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:12 smithi162 ceph-mon[104668]: pgmap v483: 89 pgs: 89 active+clean; 224 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 27 MiB/s rd, 5.7 MiB/s wr, 51 op/s 2024-08-17T17:05:13.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:12 smithi162 ceph-mon[104668]: osdmap e685: 8 total, 8 up, 8 in 2024-08-17T17:05:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:05:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:05:13] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.51.0" 2024-08-17T17:05:13.584 INFO:tasks.rados.rados.0.smithi063.stdout:2408: done (3 left) 2024-08-17T17:05:13.584 INFO:tasks.rados.rados.0.smithi063.stdout:2409: done (2 left) 2024-08-17T17:05:13.584 INFO:tasks.rados.rados.0.smithi063.stdout:2410: done (1 left) 2024-08-17T17:05:13.584 INFO:tasks.rados.rados.0.smithi063.stdout:2411: done (0 left) 2024-08-17T17:05:13.584 INFO:tasks.rados.rados.0.smithi063.stdout:2412: delete oid 24 current snap is 312 2024-08-17T17:05:13.587 INFO:tasks.rados.rados.0.smithi063.stdout:2412: done (0 left) 2024-08-17T17:05:13.587 INFO:tasks.rados.rados.0.smithi063.stdout:2413: rollback oid 18 current snap is 312 2024-08-17T17:05:13.587 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 18 to 308 2024-08-17T17:05:13.587 INFO:tasks.rados.rados.0.smithi063.stdout:2414: snap_remove snap 300 2024-08-17T17:05:14.589 INFO:tasks.rados.rados.0.smithi063.stdout:2414: done (1 left) 2024-08-17T17:05:14.589 INFO:tasks.rados.rados.0.smithi063.stdout:2415: write oid 19 current snap is 312 2024-08-17T17:05:14.589 INFO:tasks.rados.rados.0.smithi063.stdout:2413: finishing rollback tid 0 to smithi06387500-18 2024-08-17T17:05:14.589 INFO:tasks.rados.rados.0.smithi063.stdout:2415: seq_num 910 ranges {494370=722597,1917908=447981,3007041=287329} 2024-08-17T17:05:14.596 INFO:tasks.rados.rados.0.smithi063.stdout:2415: writing smithi06387500-19 from 494370 to 1216967 tid 1 2024-08-17T17:05:14.604 INFO:tasks.rados.rados.0.smithi063.stdout:2415: writing smithi06387500-19 from 1917908 to 2365889 tid 2 2024-08-17T17:05:14.609 INFO:tasks.rados.rados.0.smithi063.stdout:2415: writing smithi06387500-19 from 3007041 to 3294370 tid 3 2024-08-17T17:05:14.609 INFO:tasks.rados.rados.0.smithi063.stdout:2416: write oid 43 current snap is 312 2024-08-17T17:05:14.609 INFO:tasks.rados.rados.0.smithi063.stdout:2416: seq_num 911 ranges {511177=797846,1976390=591639,3034464=276713} 2024-08-17T17:05:14.617 INFO:tasks.rados.rados.0.smithi063.stdout:2416: writing smithi06387500-43 from 511177 to 1309023 tid 1 2024-08-17T17:05:14.624 INFO:tasks.rados.rados.0.smithi063.stdout:2416: writing smithi06387500-43 from 1976390 to 2568029 tid 2 2024-08-17T17:05:14.628 INFO:tasks.rados.rados.0.smithi063.stdout:2416: writing smithi06387500-43 from 3034464 to 3311177 tid 3 2024-08-17T17:05:14.628 INFO:tasks.rados.rados.0.smithi063.stdout:2417: read oid 47 snap 296 2024-08-17T17:05:14.629 INFO:tasks.rados.rados.0.smithi063.stdout:2417: expect deleted 2024-08-17T17:05:14.629 INFO:tasks.rados.rados.0.smithi063.stdout:2418: rollback oid 35 current snap is 312 2024-08-17T17:05:14.629 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 35 to 295 2024-08-17T17:05:14.629 INFO:tasks.rados.rados.0.smithi063.stdout:2413: finishing rollback tid 1 to smithi06387500-18 2024-08-17T17:05:14.629 INFO:tasks.rados.rados.0.smithi063.stdout:2413: finishing rollback tid 2 to smithi06387500-18 2024-08-17T17:05:14.629 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 786 (ObjNum 865 snap 294 seq_num 865) dirty exists 2024-08-17T17:05:14.629 INFO:tasks.rados.rados.0.smithi063.stdout:2415: finishing write tid 1 to smithi06387500-19 2024-08-17T17:05:14.629 INFO:tasks.rados.rados.0.smithi063.stdout:2413: done (4 left) 2024-08-17T17:05:14.629 INFO:tasks.rados.rados.0.smithi063.stdout:2419: snap_remove snap 302 2024-08-17T17:05:14.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:14 smithi162 ceph-mon[104668]: osdmap e686: 8 total, 8 up, 8 in 2024-08-17T17:05:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:14 smithi063 ceph-mon[114756]: osdmap e686: 8 total, 8 up, 8 in 2024-08-17T17:05:15.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:14 smithi063 ceph-mon[109843]: osdmap e686: 8 total, 8 up, 8 in 2024-08-17T17:05:15.594 INFO:tasks.rados.rados.0.smithi063.stdout:2419: done (4 left) 2024-08-17T17:05:15.595 INFO:tasks.rados.rados.0.smithi063.stdout:2420: snap_remove snap 309 2024-08-17T17:05:15.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:15 smithi162 ceph-mon[104668]: pgmap v486: 89 pgs: 89 active+clean; 224 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 27 MiB/s rd, 5.7 MiB/s wr, 51 op/s 2024-08-17T17:05:15.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:15 smithi162 ceph-mon[104668]: osdmap e687: 8 total, 8 up, 8 in 2024-08-17T17:05:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:15 smithi063 ceph-mon[114756]: pgmap v486: 89 pgs: 89 active+clean; 224 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 27 MiB/s rd, 5.7 MiB/s wr, 51 op/s 2024-08-17T17:05:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:15 smithi063 ceph-mon[114756]: osdmap e687: 8 total, 8 up, 8 in 2024-08-17T17:05:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:15 smithi063 ceph-mon[109843]: pgmap v486: 89 pgs: 89 active+clean; 224 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 27 MiB/s rd, 5.7 MiB/s wr, 51 op/s 2024-08-17T17:05:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:15 smithi063 ceph-mon[109843]: osdmap e687: 8 total, 8 up, 8 in 2024-08-17T17:05:16.610 INFO:tasks.rados.rados.0.smithi063.stdout:2420: done (4 left) 2024-08-17T17:05:16.611 INFO:tasks.rados.rados.0.smithi063.stdout:2421: rollback oid 23 current snap is 312 2024-08-17T17:05:16.611 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 23 to 310 2024-08-17T17:05:16.611 INFO:tasks.rados.rados.0.smithi063.stdout:2415: finishing write tid 2 to smithi06387500-19 2024-08-17T17:05:16.611 INFO:tasks.rados.rados.0.smithi063.stdout:2415: finishing write tid 3 to smithi06387500-19 2024-08-17T17:05:16.611 INFO:tasks.rados.rados.0.smithi063.stdout:2415: finishing write tid 4 to smithi06387500-19 2024-08-17T17:05:16.611 INFO:tasks.rados.rados.0.smithi063.stdout:2415: finishing write tid 5 to smithi06387500-19 2024-08-17T17:05:16.611 INFO:tasks.rados.rados.0.smithi063.stdout:2415: oid 19 updating version 0 to 1577 2024-08-17T17:05:16.611 INFO:tasks.rados.rados.0.smithi063.stdout:2415: oid 19 updating version 1577 to 1580 2024-08-17T17:05:16.611 INFO:tasks.rados.rados.0.smithi063.stdout:2415: oid 19 updating version 1580 to 1582 2024-08-17T17:05:16.611 INFO:tasks.rados.rados.0.smithi063.stdout:2415: oid 19 version 1582 is already newer than 1581 2024-08-17T17:05:16.611 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 1582 (ObjNum 910 snap 312 seq_num 910) dirty exists 2024-08-17T17:05:16.611 INFO:tasks.rados.rados.0.smithi063.stdout:2415: left oid 19 (ObjNum 910 snap 312 seq_num 910) 2024-08-17T17:05:16.611 INFO:tasks.rados.rados.0.smithi063.stdout:2416: finishing write tid 1 to smithi06387500-43 2024-08-17T17:05:16.611 INFO:tasks.rados.rados.0.smithi063.stdout:2416: finishing write tid 2 to smithi06387500-43 2024-08-17T17:05:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:2416: finishing write tid 3 to smithi06387500-43 2024-08-17T17:05:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:2416: finishing write tid 4 to smithi06387500-43 2024-08-17T17:05:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:2416: finishing write tid 5 to smithi06387500-43 2024-08-17T17:05:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:2416: oid 43 updating version 0 to 1078 2024-08-17T17:05:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:2416: oid 43 updating version 1078 to 1081 2024-08-17T17:05:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:2416: oid 43 version 1081 is already newer than 1080 2024-08-17T17:05:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:2416: oid 43 version 1081 is already newer than 1079 2024-08-17T17:05:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 1081 (ObjNum 911 snap 312 seq_num 911) dirty exists 2024-08-17T17:05:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:2416: left oid 43 (ObjNum 911 snap 312 seq_num 911) 2024-08-17T17:05:16.612 INFO:tasks.rados.rados.0.smithi063.stdout:2418: finishing rollback tid 1 to smithi06387500-35 2024-08-17T17:05:16.613 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 2268 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T17:05:16.613 INFO:tasks.rados.rados.0.smithi063.stdout:2415: done (4 left) 2024-08-17T17:05:16.613 INFO:tasks.rados.rados.0.smithi063.stdout:2416: done (3 left) 2024-08-17T17:05:16.613 INFO:tasks.rados.rados.0.smithi063.stdout:2417: done (2 left) 2024-08-17T17:05:16.613 INFO:tasks.rados.rados.0.smithi063.stdout:2418: done (1 left) 2024-08-17T17:05:16.613 INFO:tasks.rados.rados.0.smithi063.stdout:2422: snap_create 2024-08-17T17:05:16.615 INFO:tasks.rados.rados.0.smithi063.stdout:2421: finishing rollback tid 0 to smithi06387500-23 2024-08-17T17:05:16.616 INFO:tasks.rados.rados.0.smithi063.stdout:2421: finishing rollback tid 1 to smithi06387500-23 2024-08-17T17:05:16.617 INFO:tasks.rados.rados.0.smithi063.stdout:2421: finishing rollback tid 2 to smithi06387500-23 2024-08-17T17:05:16.617 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 1139 (ObjNum 777 snap 269 seq_num 777) dirty exists 2024-08-17T17:05:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:16 smithi063 ceph-mon[114756]: osdmap e688: 8 total, 8 up, 8 in 2024-08-17T17:05:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:17.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:16 smithi063 ceph-mon[109843]: osdmap e688: 8 total, 8 up, 8 in 2024-08-17T17:05:17.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:16 smithi162 ceph-mon[104668]: osdmap e688: 8 total, 8 up, 8 in 2024-08-17T17:05:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:17.585 INFO:tasks.rados.rados.0.smithi063.stdout:2421: done (1 left) 2024-08-17T17:05:17.585 INFO:tasks.rados.rados.0.smithi063.stdout:2422: done (0 left) 2024-08-17T17:05:17.585 INFO:tasks.rados.rados.0.smithi063.stdout:2423: snap_create 2024-08-17T17:05:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:17 smithi063 ceph-mon[114756]: pgmap v489: 89 pgs: 89 active+clean; 224 MiB data, 797 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:05:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:17 smithi063 ceph-mon[114756]: osdmap e689: 8 total, 8 up, 8 in 2024-08-17T17:05:18.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:17 smithi063 ceph-mon[114756]: osdmap e690: 8 total, 8 up, 8 in 2024-08-17T17:05:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:17 smithi063 ceph-mon[109843]: pgmap v489: 89 pgs: 89 active+clean; 224 MiB data, 797 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:05:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:17 smithi063 ceph-mon[109843]: osdmap e689: 8 total, 8 up, 8 in 2024-08-17T17:05:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:17 smithi063 ceph-mon[109843]: osdmap e690: 8 total, 8 up, 8 in 2024-08-17T17:05:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:17 smithi162 ceph-mon[104668]: pgmap v489: 89 pgs: 89 active+clean; 224 MiB data, 797 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:05:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:17 smithi162 ceph-mon[104668]: osdmap e689: 8 total, 8 up, 8 in 2024-08-17T17:05:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:17 smithi162 ceph-mon[104668]: osdmap e690: 8 total, 8 up, 8 in 2024-08-17T17:05:18.587 INFO:tasks.rados.rados.0.smithi063.stdout:2423: done (0 left) 2024-08-17T17:05:18.587 INFO:tasks.rados.rados.0.smithi063.stdout:2424: read oid 23 snap -1 2024-08-17T17:05:18.587 INFO:tasks.rados.rados.0.smithi063.stdout:2424: expect (ObjNum 777 snap 269 seq_num 777) 2024-08-17T17:05:18.587 INFO:tasks.rados.rados.0.smithi063.stdout:2425: rollback oid 29 current snap is 314 2024-08-17T17:05:18.588 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 29 to 304 2024-08-17T17:05:18.588 INFO:tasks.rados.rados.0.smithi063.stdout:2426: write oid 33 current snap is 314 2024-08-17T17:05:18.588 INFO:tasks.rados.rados.0.smithi063.stdout:2426: seq_num 912 ranges {527984=473095,1634872=451650,2661887=634311} 2024-08-17T17:05:18.593 INFO:tasks.rados.rados.0.smithi063.stdout:2426: writing smithi06387500-33 from 527984 to 1001079 tid 1 2024-08-17T17:05:18.599 INFO:tasks.rados.rados.0.smithi063.stdout:2426: writing smithi06387500-33 from 1634872 to 2086522 tid 2 2024-08-17T17:05:18.606 INFO:tasks.rados.rados.0.smithi063.stdout:2426: writing smithi06387500-33 from 2661887 to 3296198 tid 3 2024-08-17T17:05:18.606 INFO:tasks.rados.rados.0.smithi063.stdout:2427: write oid 3 current snap is 314 2024-08-17T17:05:18.607 INFO:tasks.rados.rados.0.smithi063.stdout:2427: seq_num 913 ranges {544791=664697,1693354=595308,2689310=445583} 2024-08-17T17:05:18.613 INFO:tasks.rados.rados.0.smithi063.stdout:2427: writing smithi06387500-3 from 544791 to 1209488 tid 1 2024-08-17T17:05:18.619 INFO:tasks.rados.rados.0.smithi063.stdout:2427: writing smithi06387500-3 from 1693354 to 2288662 tid 2 2024-08-17T17:05:18.624 INFO:tasks.rados.rados.0.smithi063.stdout:2427: writing smithi06387500-3 from 2689310 to 3134893 tid 3 2024-08-17T17:05:18.624 INFO:tasks.rados.rados.0.smithi063.stdout:2428: rollback oid 36 current snap is 314 2024-08-17T17:05:18.624 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 36 to 310 2024-08-17T17:05:18.624 INFO:tasks.rados.rados.0.smithi063.stdout:2425: finishing rollback tid 1 to smithi06387500-29 2024-08-17T17:05:18.624 INFO:tasks.rados.rados.0.smithi063.stdout:2425: finishing rollback tid 2 to smithi06387500-29 2024-08-17T17:05:18.624 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 607 (ObjNum 855 snap 290 seq_num 855) dirty exists 2024-08-17T17:05:18.624 INFO:tasks.rados.rados.0.smithi063.stdout:2424: expect (ObjNum 777 snap 269 seq_num 777) 2024-08-17T17:05:18.886 INFO:tasks.rados.rados.0.smithi063.stdout:2426: finishing write tid 1 to smithi06387500-33 2024-08-17T17:05:18.886 INFO:tasks.rados.rados.0.smithi063.stdout:2426: finishing write tid 2 to smithi06387500-33 2024-08-17T17:05:18.887 INFO:tasks.rados.rados.0.smithi063.stdout:2426: finishing write tid 3 to smithi06387500-33 2024-08-17T17:05:18.887 INFO:tasks.rados.rados.0.smithi063.stdout:2426: finishing write tid 4 to smithi06387500-33 2024-08-17T17:05:18.887 INFO:tasks.rados.rados.0.smithi063.stdout:2426: finishing write tid 5 to smithi06387500-33 2024-08-17T17:05:18.887 INFO:tasks.rados.rados.0.smithi063.stdout:2426: oid 33 updating version 0 to 818 2024-08-17T17:05:18.887 INFO:tasks.rados.rados.0.smithi063.stdout:2426: oid 33 updating version 818 to 819 2024-08-17T17:05:18.887 INFO:tasks.rados.rados.0.smithi063.stdout:2426: oid 33 updating version 819 to 820 2024-08-17T17:05:18.887 INFO:tasks.rados.rados.0.smithi063.stdout:2426: oid 33 version 820 is already newer than 817 2024-08-17T17:05:18.887 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 820 (ObjNum 912 snap 314 seq_num 912) dirty exists 2024-08-17T17:05:18.887 INFO:tasks.rados.rados.0.smithi063.stdout:2426: left oid 33 (ObjNum 912 snap 314 seq_num 912) 2024-08-17T17:05:18.887 INFO:tasks.rados.rados.0.smithi063.stdout:2427: finishing write tid 1 to smithi06387500-3 2024-08-17T17:05:18.887 INFO:tasks.rados.rados.0.smithi063.stdout:2427: finishing write tid 2 to smithi06387500-3 2024-08-17T17:05:18.887 INFO:tasks.rados.rados.0.smithi063.stdout:2427: finishing write tid 3 to smithi06387500-3 2024-08-17T17:05:18.888 INFO:tasks.rados.rados.0.smithi063.stdout:2427: finishing write tid 4 to smithi06387500-3 2024-08-17T17:05:18.888 INFO:tasks.rados.rados.0.smithi063.stdout:2427: finishing write tid 5 to smithi06387500-3 2024-08-17T17:05:18.888 INFO:tasks.rados.rados.0.smithi063.stdout:2427: oid 3 updating version 0 to 730 2024-08-17T17:05:18.888 INFO:tasks.rados.rados.0.smithi063.stdout:2427: oid 3 version 730 is already newer than 729 2024-08-17T17:05:18.888 INFO:tasks.rados.rados.0.smithi063.stdout:2427: oid 3 updating version 730 to 732 2024-08-17T17:05:18.888 INFO:tasks.rados.rados.0.smithi063.stdout:2427: oid 3 version 732 is already newer than 731 2024-08-17T17:05:18.888 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 732 (ObjNum 913 snap 314 seq_num 913) dirty exists 2024-08-17T17:05:18.888 INFO:tasks.rados.rados.0.smithi063.stdout:2427: left oid 3 (ObjNum 913 snap 314 seq_num 913) 2024-08-17T17:05:18.888 INFO:tasks.rados.rados.0.smithi063.stdout:2428: finishing rollback tid 0 to smithi06387500-36 2024-08-17T17:05:18.888 INFO:tasks.rados.rados.0.smithi063.stdout:2428: finishing rollback tid 1 to smithi06387500-36 2024-08-17T17:05:18.888 INFO:tasks.rados.rados.0.smithi063.stdout:2428: finishing rollback tid 2 to smithi06387500-36 2024-08-17T17:05:18.889 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 2404 (ObjNum 886 snap 309 seq_num 886) dirty exists 2024-08-17T17:05:18.889 INFO:tasks.rados.rados.0.smithi063.stdout:2424: done (4 left) 2024-08-17T17:05:18.889 INFO:tasks.rados.rados.0.smithi063.stdout:2425: done (3 left) 2024-08-17T17:05:18.889 INFO:tasks.rados.rados.0.smithi063.stdout:2426: done (2 left) 2024-08-17T17:05:18.889 INFO:tasks.rados.rados.0.smithi063.stdout:2427: done (1 left) 2024-08-17T17:05:18.889 INFO:tasks.rados.rados.0.smithi063.stdout:2428: done (0 left) 2024-08-17T17:05:18.889 INFO:tasks.rados.rados.0.smithi063.stdout:2429: snap_create 2024-08-17T17:05:19.607 INFO:tasks.rados.rados.0.smithi063.stdout:2429: done (0 left) 2024-08-17T17:05:19.607 INFO:tasks.rados.rados.0.smithi063.stdout:2430: read oid 24 snap -1 2024-08-17T17:05:19.607 INFO:tasks.rados.rados.0.smithi063.stdout:2430: expect deleted 2024-08-17T17:05:19.607 INFO:tasks.rados.rados.0.smithi063.stdout:2431: delete oid 2 current snap is 315 2024-08-17T17:05:19.610 INFO:tasks.rados.rados.0.smithi063.stdout:2430: done (1 left) 2024-08-17T17:05:19.610 INFO:tasks.rados.rados.0.smithi063.stdout:2431: done (0 left) 2024-08-17T17:05:19.610 INFO:tasks.rados.rados.0.smithi063.stdout:2432: delete oid 40 current snap is 315 2024-08-17T17:05:19.613 INFO:tasks.rados.rados.0.smithi063.stdout:2432: done (0 left) 2024-08-17T17:05:19.613 INFO:tasks.rados.rados.0.smithi063.stdout:2433: read oid 48 snap -1 2024-08-17T17:05:19.613 INFO:tasks.rados.rados.0.smithi063.stdout:2433: expect (ObjNum 857 snap 292 seq_num 857) 2024-08-17T17:05:19.613 INFO:tasks.rados.rados.0.smithi063.stdout:2434: write oid 41 current snap is 315 2024-08-17T17:05:19.613 INFO:tasks.rados.rados.0.smithi063.stdout:2434: seq_num 916 ranges {595212=490444,1868800=458988,2887932=507280} 2024-08-17T17:05:19.620 INFO:tasks.rados.rados.0.smithi063.stdout:2434: writing smithi06387500-41 from 595212 to 1085656 tid 1 2024-08-17T17:05:19.628 INFO:tasks.rados.rados.0.smithi063.stdout:2434: writing smithi06387500-41 from 1868800 to 2327788 tid 2 2024-08-17T17:05:19.635 INFO:tasks.rados.rados.0.smithi063.stdout:2434: writing smithi06387500-41 from 2887932 to 3395212 tid 3 2024-08-17T17:05:19.635 INFO:tasks.rados.rados.0.smithi063.stdout:2435: write oid 3 current snap is 315 2024-08-17T17:05:19.635 INFO:tasks.rados.rados.0.smithi063.stdout:2435: seq_num 917 ranges {612019=565693,1810929=602646,3082649=329370} 2024-08-17T17:05:19.641 INFO:tasks.rados.rados.0.smithi063.stdout:2435: writing smithi06387500-3 from 612019 to 1177712 tid 1 2024-08-17T17:05:19.648 INFO:tasks.rados.rados.0.smithi063.stdout:2435: writing smithi06387500-3 from 1810929 to 2413575 tid 2 2024-08-17T17:05:19.654 INFO:tasks.rados.rados.0.smithi063.stdout:2435: writing smithi06387500-3 from 3082649 to 3412019 tid 3 2024-08-17T17:05:19.654 INFO:tasks.rados.rados.0.smithi063.stdout:2436: read oid 33 snap -1 2024-08-17T17:05:19.654 INFO:tasks.rados.rados.0.smithi063.stdout:2436: expect (ObjNum 912 snap 314 seq_num 912) 2024-08-17T17:05:19.654 INFO:tasks.rados.rados.0.smithi063.stdout:2437: read oid 32 snap -1 2024-08-17T17:05:19.654 INFO:tasks.rados.rados.0.smithi063.stdout:2437: expect (ObjNum 882 snap 308 seq_num 882) 2024-08-17T17:05:19.654 INFO:tasks.rados.rados.0.smithi063.stdout:2438: rollback oid 43 current snap is 315 2024-08-17T17:05:19.654 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 43 to 287 2024-08-17T17:05:19.655 INFO:tasks.rados.rados.0.smithi063.stdout:2433: expect (ObjNum 857 snap 292 seq_num 857) 2024-08-17T17:05:19.801 INFO:tasks.rados.rados.0.smithi063.stdout:2434: finishing write tid 1 to smithi06387500-41 2024-08-17T17:05:19.801 INFO:tasks.rados.rados.0.smithi063.stdout:2434: finishing write tid 2 to smithi06387500-41 2024-08-17T17:05:19.801 INFO:tasks.rados.rados.0.smithi063.stdout:2433: done (5 left) 2024-08-17T17:05:19.801 INFO:tasks.rados.rados.0.smithi063.stdout:2439: delete oid 50 current snap is 315 2024-08-17T17:05:19.801 INFO:tasks.rados.rados.0.smithi063.stdout:2434: finishing write tid 3 to smithi06387500-41 2024-08-17T17:05:19.801 INFO:tasks.rados.rados.0.smithi063.stdout:2434: finishing write tid 4 to smithi06387500-41 2024-08-17T17:05:19.801 INFO:tasks.rados.rados.0.smithi063.stdout:2434: finishing write tid 5 to smithi06387500-41 2024-08-17T17:05:19.802 INFO:tasks.rados.rados.0.smithi063.stdout:2434: oid 41 updating version 0 to 234 2024-08-17T17:05:19.802 INFO:tasks.rados.rados.0.smithi063.stdout:2434: oid 41 version 234 is already newer than 231 2024-08-17T17:05:19.802 INFO:tasks.rados.rados.0.smithi063.stdout:2434: oid 41 version 234 is already newer than 232 2024-08-17T17:05:19.802 INFO:tasks.rados.rados.0.smithi063.stdout:2434: oid 41 version 234 is already newer than 233 2024-08-17T17:05:19.802 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 234 (ObjNum 916 snap 315 seq_num 916) dirty exists 2024-08-17T17:05:19.802 INFO:tasks.rados.rados.0.smithi063.stdout:2434: left oid 41 (ObjNum 916 snap 315 seq_num 916) 2024-08-17T17:05:19.802 INFO:tasks.rados.rados.0.smithi063.stdout:2435: finishing write tid 1 to smithi06387500-3 2024-08-17T17:05:19.802 INFO:tasks.rados.rados.0.smithi063.stdout:2435: finishing write tid 2 to smithi06387500-3 2024-08-17T17:05:19.802 INFO:tasks.rados.rados.0.smithi063.stdout:2435: finishing write tid 3 to smithi06387500-3 2024-08-17T17:05:19.802 INFO:tasks.rados.rados.0.smithi063.stdout:2435: finishing write tid 4 to smithi06387500-3 2024-08-17T17:05:19.802 INFO:tasks.rados.rados.0.smithi063.stdout:2435: finishing write tid 5 to smithi06387500-3 2024-08-17T17:05:19.802 INFO:tasks.rados.rados.0.smithi063.stdout:2435: oid 3 updating version 0 to 736 2024-08-17T17:05:19.803 INFO:tasks.rados.rados.0.smithi063.stdout:2435: oid 3 updating version 736 to 737 2024-08-17T17:05:19.803 INFO:tasks.rados.rados.0.smithi063.stdout:2435: oid 3 version 737 is already newer than 734 2024-08-17T17:05:19.803 INFO:tasks.rados.rados.0.smithi063.stdout:2435: oid 3 version 737 is already newer than 735 2024-08-17T17:05:19.803 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 737 (ObjNum 917 snap 315 seq_num 917) dirty exists 2024-08-17T17:05:19.803 INFO:tasks.rados.rados.0.smithi063.stdout:2435: left oid 3 (ObjNum 917 snap 315 seq_num 917) 2024-08-17T17:05:19.803 INFO:tasks.rados.rados.0.smithi063.stdout:2438: finishing rollback tid 0 to smithi06387500-43 2024-08-17T17:05:19.803 INFO:tasks.rados.rados.0.smithi063.stdout:2437: expect (ObjNum 882 snap 308 seq_num 882) 2024-08-17T17:05:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:19 smithi162 ceph-mon[104668]: pgmap v492: 89 pgs: 89 active+clean; 229 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T17:05:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:19 smithi162 ceph-mon[104668]: osdmap e691: 8 total, 8 up, 8 in 2024-08-17T17:05:19.960 INFO:tasks.rados.rados.0.smithi063.stdout:2438: finishing rollback tid 1 to smithi06387500-43 2024-08-17T17:05:19.960 INFO:tasks.rados.rados.0.smithi063.stdout:2438: finishing rollback tid 2 to smithi06387500-43 2024-08-17T17:05:19.960 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 1093 (ObjNum 831 snap 287 seq_num 831) dirty exists 2024-08-17T17:05:19.960 INFO:tasks.rados.rados.0.smithi063.stdout:2436: expect (ObjNum 912 snap 314 seq_num 912) 2024-08-17T17:05:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:19 smithi063 ceph-mon[114756]: pgmap v492: 89 pgs: 89 active+clean; 229 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T17:05:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:19 smithi063 ceph-mon[114756]: osdmap e691: 8 total, 8 up, 8 in 2024-08-17T17:05:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:19 smithi063 ceph-mon[109843]: pgmap v492: 89 pgs: 89 active+clean; 229 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T17:05:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:19 smithi063 ceph-mon[109843]: osdmap e691: 8 total, 8 up, 8 in 2024-08-17T17:05:20.154 INFO:tasks.rados.rados.0.smithi063.stdout:2434: done (5 left) 2024-08-17T17:05:20.154 INFO:tasks.rados.rados.0.smithi063.stdout:2435: done (4 left) 2024-08-17T17:05:20.154 INFO:tasks.rados.rados.0.smithi063.stdout:2436: done (3 left) 2024-08-17T17:05:20.154 INFO:tasks.rados.rados.0.smithi063.stdout:2437: done (2 left) 2024-08-17T17:05:20.154 INFO:tasks.rados.rados.0.smithi063.stdout:2438: done (1 left) 2024-08-17T17:05:20.154 INFO:tasks.rados.rados.0.smithi063.stdout:2439: done (0 left) 2024-08-17T17:05:20.154 INFO:tasks.rados.rados.0.smithi063.stdout:2440: read oid 42 snap -1 2024-08-17T17:05:20.155 INFO:tasks.rados.rados.0.smithi063.stdout:2440: expect deleted 2024-08-17T17:05:20.155 INFO:tasks.rados.rados.0.smithi063.stdout:2441: read oid 44 snap 314 2024-08-17T17:05:20.155 INFO:tasks.rados.rados.0.smithi063.stdout:2441: expect deleted 2024-08-17T17:05:20.155 INFO:tasks.rados.rados.0.smithi063.stdout:2442: snap_create 2024-08-17T17:05:20.614 INFO:tasks.rados.rados.0.smithi063.stdout:2440: done (2 left) 2024-08-17T17:05:20.614 INFO:tasks.rados.rados.0.smithi063.stdout:2441: done (1 left) 2024-08-17T17:05:20.614 INFO:tasks.rados.rados.0.smithi063.stdout:2442: done (0 left) 2024-08-17T17:05:20.614 INFO:tasks.rados.rados.0.smithi063.stdout:2443: write oid 17 current snap is 316 2024-08-17T17:05:20.614 INFO:tasks.rados.rados.0.smithi063.stdout:2443: seq_num 919 ranges {645633=716191,1927893=606315,3137495=308138} 2024-08-17T17:05:20.621 INFO:tasks.rados.rados.0.smithi063.stdout:2443: writing smithi06387500-17 from 645633 to 1361824 tid 1 2024-08-17T17:05:20.628 INFO:tasks.rados.rados.0.smithi063.stdout:2443: writing smithi06387500-17 from 1927893 to 2534208 tid 2 2024-08-17T17:05:20.633 INFO:tasks.rados.rados.0.smithi063.stdout:2443: writing smithi06387500-17 from 3137495 to 3445633 tid 3 2024-08-17T17:05:20.633 INFO:tasks.rados.rados.0.smithi063.stdout:2444: read oid 39 snap -1 2024-08-17T17:05:20.633 INFO:tasks.rados.rados.0.smithi063.stdout:2444: expect (ObjNum 889 snap 310 seq_num 889) 2024-08-17T17:05:20.633 INFO:tasks.rados.rados.0.smithi063.stdout:2443: finishing write tid 1 to smithi06387500-17 2024-08-17T17:05:20.633 INFO:tasks.rados.rados.0.smithi063.stdout:2443: finishing write tid 2 to smithi06387500-17 2024-08-17T17:05:20.633 INFO:tasks.rados.rados.0.smithi063.stdout:2445: read oid 42 snap -1 2024-08-17T17:05:20.633 INFO:tasks.rados.rados.0.smithi063.stdout:2445: expect deleted 2024-08-17T17:05:20.634 INFO:tasks.rados.rados.0.smithi063.stdout:2446: delete oid 1 current snap is 316 2024-08-17T17:05:20.636 INFO:tasks.rados.rados.0.smithi063.stdout:2443: finishing write tid 3 to smithi06387500-17 2024-08-17T17:05:20.636 INFO:tasks.rados.rados.0.smithi063.stdout:2445: done (3 left) 2024-08-17T17:05:20.636 INFO:tasks.rados.rados.0.smithi063.stdout:2446: done (2 left) 2024-08-17T17:05:20.637 INFO:tasks.rados.rados.0.smithi063.stdout:2447: delete oid 9 current snap is 316 2024-08-17T17:05:20.637 INFO:tasks.rados.rados.0.smithi063.stdout:2443: finishing write tid 4 to smithi06387500-17 2024-08-17T17:05:20.637 INFO:tasks.rados.rados.0.smithi063.stdout:2443: finishing write tid 5 to smithi06387500-17 2024-08-17T17:05:20.637 INFO:tasks.rados.rados.0.smithi063.stdout:2443: oid 17 updating version 0 to 741 2024-08-17T17:05:20.637 INFO:tasks.rados.rados.0.smithi063.stdout:2443: oid 17 version 741 is already newer than 740 2024-08-17T17:05:20.637 INFO:tasks.rados.rados.0.smithi063.stdout:2443: oid 17 version 741 is already newer than 739 2024-08-17T17:05:20.637 INFO:tasks.rados.rados.0.smithi063.stdout:2443: oid 17 updating version 741 to 742 2024-08-17T17:05:20.638 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 742 (ObjNum 919 snap 316 seq_num 919) dirty exists 2024-08-17T17:05:20.638 INFO:tasks.rados.rados.0.smithi063.stdout:2443: left oid 17 (ObjNum 919 snap 316 seq_num 919) 2024-08-17T17:05:20.638 INFO:tasks.rados.rados.0.smithi063.stdout:2443: done (2 left) 2024-08-17T17:05:20.638 INFO:tasks.rados.rados.0.smithi063.stdout:2447: done (1 left) 2024-08-17T17:05:20.639 INFO:tasks.rados.rados.0.smithi063.stdout:2448: snap_remove snap 283 2024-08-17T17:05:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:20 smithi063 ceph-mon[114756]: osdmap e692: 8 total, 8 up, 8 in 2024-08-17T17:05:21.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:20 smithi063 ceph-mon[109843]: osdmap e692: 8 total, 8 up, 8 in 2024-08-17T17:05:21.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:20 smithi162 ceph-mon[104668]: osdmap e692: 8 total, 8 up, 8 in 2024-08-17T17:05:21.626 INFO:tasks.rados.rados.0.smithi063.stdout:2444: expect (ObjNum 889 snap 310 seq_num 889) 2024-08-17T17:05:21.769 INFO:tasks.rados.rados.0.smithi063.stdout:2444: done (1 left) 2024-08-17T17:05:21.769 INFO:tasks.rados.rados.0.smithi063.stdout:2448: done (0 left) 2024-08-17T17:05:21.769 INFO:tasks.rados.rados.0.smithi063.stdout:2449: snap_remove snap 311 2024-08-17T17:05:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:21 smithi063 ceph-mon[114756]: pgmap v495: 89 pgs: 89 active+clean; 229 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T17:05:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:21 smithi063 ceph-mon[114756]: osdmap e693: 8 total, 8 up, 8 in 2024-08-17T17:05:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:21 smithi063 ceph-mon[109843]: pgmap v495: 89 pgs: 89 active+clean; 229 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T17:05:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:21 smithi063 ceph-mon[109843]: osdmap e693: 8 total, 8 up, 8 in 2024-08-17T17:05:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:21 smithi162 ceph-mon[104668]: pgmap v495: 89 pgs: 89 active+clean; 229 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.8 MiB/s wr, 14 op/s 2024-08-17T17:05:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:21 smithi162 ceph-mon[104668]: osdmap e693: 8 total, 8 up, 8 in 2024-08-17T17:05:22.640 INFO:tasks.rados.rados.0.smithi063.stdout:2449: done (0 left) 2024-08-17T17:05:22.640 INFO:tasks.rados.rados.0.smithi063.stdout:2450: read oid 43 snap -1 2024-08-17T17:05:22.640 INFO:tasks.rados.rados.0.smithi063.stdout:2450: expect (ObjNum 831 snap 287 seq_num 831) 2024-08-17T17:05:22.640 INFO:tasks.rados.rados.0.smithi063.stdout:2451: read oid 19 snap -1 2024-08-17T17:05:22.640 INFO:tasks.rados.rados.0.smithi063.stdout:2451: expect (ObjNum 910 snap 312 seq_num 910) 2024-08-17T17:05:22.640 INFO:tasks.rados.rados.0.smithi063.stdout:2452: read oid 29 snap 310 2024-08-17T17:05:22.640 INFO:tasks.rados.rados.0.smithi063.stdout:2452: expect deleted 2024-08-17T17:05:22.640 INFO:tasks.rados.rados.0.smithi063.stdout:2453: write oid 14 current snap is 316 2024-08-17T17:05:22.640 INFO:tasks.rados.rados.0.smithi063.stdout:2453: seq_num 922 ranges {696054=658291,2103339=753642} 2024-08-17T17:05:22.647 INFO:tasks.rados.rados.0.smithi063.stdout:2453: writing smithi06387500-14 from 696054 to 1354345 tid 1 2024-08-17T17:05:22.657 INFO:tasks.rados.rados.0.smithi063.stdout:2453: writing smithi06387500-14 from 2103339 to 2856981 tid 2 2024-08-17T17:05:22.657 INFO:tasks.rados.rados.0.smithi063.stdout:2454: snap_remove snap 306 2024-08-17T17:05:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:22 smithi063 ceph-mon[114756]: osdmap e694: 8 total, 8 up, 8 in 2024-08-17T17:05:23.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:22 smithi063 ceph-mon[109843]: osdmap e694: 8 total, 8 up, 8 in 2024-08-17T17:05:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:22 smithi162 ceph-mon[104668]: osdmap e694: 8 total, 8 up, 8 in 2024-08-17T17:05:23.636 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:05:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:05:23] "GET /metrics HTTP/1.1" 200 36212 "" "Prometheus/2.51.0" 2024-08-17T17:05:23.648 INFO:tasks.rados.rados.0.smithi063.stdout:2454: done (4 left) 2024-08-17T17:05:23.648 INFO:tasks.rados.rados.0.smithi063.stdout:2455: delete oid 25 current snap is 316 2024-08-17T17:05:23.648 INFO:tasks.rados.rados.0.smithi063.stdout:2451: expect (ObjNum 910 snap 312 seq_num 910) 2024-08-17T17:05:23.799 INFO:tasks.rados.rados.0.smithi063.stdout:2450: expect (ObjNum 831 snap 287 seq_num 831) 2024-08-17T17:05:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:2453: finishing write tid 1 to smithi06387500-14 2024-08-17T17:05:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:2453: finishing write tid 2 to smithi06387500-14 2024-08-17T17:05:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:2453: finishing write tid 3 to smithi06387500-14 2024-08-17T17:05:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:2453: finishing write tid 4 to smithi06387500-14 2024-08-17T17:05:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:2453: oid 14 updating version 0 to 2421 2024-08-17T17:05:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:2453: oid 14 updating version 2421 to 2422 2024-08-17T17:05:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:2453: oid 14 updating version 2422 to 2423 2024-08-17T17:05:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 2423 (ObjNum 922 snap 316 seq_num 922) dirty exists 2024-08-17T17:05:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:2453: left oid 14 (ObjNum 922 snap 316 seq_num 922) 2024-08-17T17:05:24.023 INFO:tasks.rados.rados.0.smithi063.stdout:2450: done (4 left) 2024-08-17T17:05:24.024 INFO:tasks.rados.rados.0.smithi063.stdout:2451: done (3 left) 2024-08-17T17:05:24.024 INFO:tasks.rados.rados.0.smithi063.stdout:2452: done (2 left) 2024-08-17T17:05:24.024 INFO:tasks.rados.rados.0.smithi063.stdout:2453: done (1 left) 2024-08-17T17:05:24.024 INFO:tasks.rados.rados.0.smithi063.stdout:2455: done (0 left) 2024-08-17T17:05:24.024 INFO:tasks.rados.rados.0.smithi063.stdout:2456: write oid 15 current snap is 316 2024-08-17T17:05:24.024 INFO:tasks.rados.rados.0.smithi063.stdout:2456: seq_num 924 ranges {729668=408789,1820303=757311,3274610=255058} 2024-08-17T17:05:24.029 INFO:tasks.rados.rados.0.smithi063.stdout:2456: writing smithi06387500-15 from 729668 to 1138457 tid 1 2024-08-17T17:05:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:23 smithi063 ceph-mon[114756]: pgmap v498: 89 pgs: 89 active+clean; 241 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-08-17T17:05:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:23 smithi063 ceph-mon[114756]: osdmap e695: 8 total, 8 up, 8 in 2024-08-17T17:05:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:23 smithi063 ceph-mon[109843]: pgmap v498: 89 pgs: 89 active+clean; 241 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-08-17T17:05:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:23 smithi063 ceph-mon[109843]: osdmap e695: 8 total, 8 up, 8 in 2024-08-17T17:05:24.037 INFO:tasks.rados.rados.0.smithi063.stdout:2456: writing smithi06387500-15 from 1820303 to 2577614 tid 2 2024-08-17T17:05:24.042 INFO:tasks.rados.rados.0.smithi063.stdout:2456: writing smithi06387500-15 from 3274610 to 3529668 tid 3 2024-08-17T17:05:24.042 INFO:tasks.rados.rados.0.smithi063.stdout:2457: snap_remove snap 313 2024-08-17T17:05:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:23 smithi162 ceph-mon[104668]: pgmap v498: 89 pgs: 89 active+clean; 241 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-08-17T17:05:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:23 smithi162 ceph-mon[104668]: osdmap e695: 8 total, 8 up, 8 in 2024-08-17T17:05:24.659 INFO:tasks.rados.rados.0.smithi063.stdout:2457: done (1 left) 2024-08-17T17:05:24.659 INFO:tasks.rados.rados.0.smithi063.stdout:2458: read oid 11 snap -1 2024-08-17T17:05:24.659 INFO:tasks.rados.rados.0.smithi063.stdout:2458: expect deleted 2024-08-17T17:05:24.659 INFO:tasks.rados.rados.0.smithi063.stdout:2459: delete oid 27 current snap is 316 2024-08-17T17:05:24.659 INFO:tasks.rados.rados.0.smithi063.stdout:2456: finishing write tid 1 to smithi06387500-15 2024-08-17T17:05:24.659 INFO:tasks.rados.rados.0.smithi063.stdout:2456: finishing write tid 2 to smithi06387500-15 2024-08-17T17:05:24.659 INFO:tasks.rados.rados.0.smithi063.stdout:2456: finishing write tid 3 to smithi06387500-15 2024-08-17T17:05:24.659 INFO:tasks.rados.rados.0.smithi063.stdout:2456: finishing write tid 4 to smithi06387500-15 2024-08-17T17:05:24.660 INFO:tasks.rados.rados.0.smithi063.stdout:2456: finishing write tid 5 to smithi06387500-15 2024-08-17T17:05:24.660 INFO:tasks.rados.rados.0.smithi063.stdout:2456: oid 15 updating version 0 to 932 2024-08-17T17:05:24.660 INFO:tasks.rados.rados.0.smithi063.stdout:2456: oid 15 version 932 is already newer than 930 2024-08-17T17:05:24.660 INFO:tasks.rados.rados.0.smithi063.stdout:2456: oid 15 version 932 is already newer than 931 2024-08-17T17:05:24.660 INFO:tasks.rados.rados.0.smithi063.stdout:2456: oid 15 version 932 is already newer than 929 2024-08-17T17:05:24.660 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 932 (ObjNum 924 snap 316 seq_num 924) dirty exists 2024-08-17T17:05:24.660 INFO:tasks.rados.rados.0.smithi063.stdout:2456: left oid 15 (ObjNum 924 snap 316 seq_num 924) 2024-08-17T17:05:24.662 INFO:tasks.rados.rados.0.smithi063.stdout:2456: done (2 left) 2024-08-17T17:05:24.663 INFO:tasks.rados.rados.0.smithi063.stdout:2458: done (1 left) 2024-08-17T17:05:24.663 INFO:tasks.rados.rados.0.smithi063.stdout:2459: done (0 left) 2024-08-17T17:05:24.663 INFO:tasks.rados.rados.0.smithi063.stdout:2460: write oid 2 current snap is 316 2024-08-17T17:05:24.663 INFO:tasks.rados.rados.0.smithi063.stdout:2460: seq_num 926 ranges {763282=559287,1820914=760980,3213103=350179} 2024-08-17T17:05:24.670 INFO:tasks.rados.rados.0.smithi063.stdout:2460: writing smithi06387500-2 from 763282 to 1322569 tid 1 2024-08-17T17:05:24.677 INFO:tasks.rados.rados.0.smithi063.stdout:2460: writing smithi06387500-2 from 1820914 to 2581894 tid 2 2024-08-17T17:05:24.683 INFO:tasks.rados.rados.0.smithi063.stdout:2460: writing smithi06387500-2 from 3213103 to 3563282 tid 3 2024-08-17T17:05:24.683 INFO:tasks.rados.rados.0.smithi063.stdout:2461: delete oid 4 current snap is 316 2024-08-17T17:05:24.683 INFO:tasks.rados.rados.0.smithi063.stdout:2460: finishing write tid 1 to smithi06387500-2 2024-08-17T17:05:24.683 INFO:tasks.rados.rados.0.smithi063.stdout:2460: finishing write tid 2 to smithi06387500-2 2024-08-17T17:05:24.685 INFO:tasks.rados.rados.0.smithi063.stdout:2461: done (1 left) 2024-08-17T17:05:24.686 INFO:tasks.rados.rados.0.smithi063.stdout:2462: read oid 38 snap -1 2024-08-17T17:05:24.686 INFO:tasks.rados.rados.0.smithi063.stdout:2462: expect (ObjNum 868 snap 300 seq_num 868) 2024-08-17T17:05:24.686 INFO:tasks.rados.rados.0.smithi063.stdout:2463: delete oid 33 current snap is 316 2024-08-17T17:05:24.686 INFO:tasks.rados.rados.0.smithi063.stdout:2460: finishing write tid 3 to smithi06387500-2 2024-08-17T17:05:24.686 INFO:tasks.rados.rados.0.smithi063.stdout:2460: finishing write tid 4 to smithi06387500-2 2024-08-17T17:05:24.687 INFO:tasks.rados.rados.0.smithi063.stdout:2460: finishing write tid 5 to smithi06387500-2 2024-08-17T17:05:24.687 INFO:tasks.rados.rados.0.smithi063.stdout:2460: oid 2 updating version 0 to 1168 2024-08-17T17:05:24.687 INFO:tasks.rados.rados.0.smithi063.stdout:2460: oid 2 updating version 1168 to 1171 2024-08-17T17:05:24.687 INFO:tasks.rados.rados.0.smithi063.stdout:2460: oid 2 version 1171 is already newer than 1170 2024-08-17T17:05:24.687 INFO:tasks.rados.rados.0.smithi063.stdout:2460: oid 2 version 1171 is already newer than 1169 2024-08-17T17:05:24.687 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 1171 (ObjNum 926 snap 316 seq_num 926) dirty exists 2024-08-17T17:05:24.687 INFO:tasks.rados.rados.0.smithi063.stdout:2460: left oid 2 (ObjNum 926 snap 316 seq_num 926) 2024-08-17T17:05:24.687 INFO:tasks.rados.rados.0.smithi063.stdout:2460: done (2 left) 2024-08-17T17:05:24.688 INFO:tasks.rados.rados.0.smithi063.stdout:2463: done (1 left) 2024-08-17T17:05:24.688 INFO:tasks.rados.rados.0.smithi063.stdout:2464: delete oid 45 current snap is 316 2024-08-17T17:05:24.689 INFO:tasks.rados.rados.0.smithi063.stdout:2462: expect (ObjNum 868 snap 300 seq_num 868) 2024-08-17T17:05:24.810 INFO:tasks.rados.rados.0.smithi063.stdout:2462: done (1 left) 2024-08-17T17:05:24.810 INFO:tasks.rados.rados.0.smithi063.stdout:2464: done (0 left) 2024-08-17T17:05:24.810 INFO:tasks.rados.rados.0.smithi063.stdout:2465: rollback oid 15 current snap is 316 2024-08-17T17:05:24.811 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 15 to 296 2024-08-17T17:05:24.811 INFO:tasks.rados.rados.0.smithi063.stdout:2466: delete oid 33 current snap is 316 2024-08-17T17:05:24.812 INFO:tasks.rados.rados.0.smithi063.stdout:2466: done (1 left) 2024-08-17T17:05:24.812 INFO:tasks.rados.rados.0.smithi063.stdout:2467: write oid 8 current snap is 316 2024-08-17T17:05:24.812 INFO:tasks.rados.rados.0.smithi063.stdout:2467: seq_num 931 ranges {447317=651885,1713324=628329,3066571=580746} 2024-08-17T17:05:24.818 INFO:tasks.rados.rados.0.smithi063.stdout:2467: writing smithi06387500-8 from 447317 to 1099202 tid 1 2024-08-17T17:05:24.825 INFO:tasks.rados.rados.0.smithi063.stdout:2467: writing smithi06387500-8 from 1713324 to 2341653 tid 2 2024-08-17T17:05:24.832 INFO:tasks.rados.rados.0.smithi063.stdout:2467: writing smithi06387500-8 from 3066571 to 3647317 tid 3 2024-08-17T17:05:24.832 INFO:tasks.rados.rados.0.smithi063.stdout:2468: delete oid 5 current snap is 316 2024-08-17T17:05:24.833 INFO:tasks.rados.rados.0.smithi063.stdout:2465: finishing rollback tid 0 to smithi06387500-15 2024-08-17T17:05:24.833 INFO:tasks.rados.rados.0.smithi063.stdout:2465: finishing rollback tid 1 to smithi06387500-15 2024-08-17T17:05:24.833 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 936 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-08-17T17:05:24.833 INFO:tasks.rados.rados.0.smithi063.stdout:2467: finishing write tid 1 to smithi06387500-8 2024-08-17T17:05:24.833 INFO:tasks.rados.rados.0.smithi063.stdout:2467: finishing write tid 2 to smithi06387500-8 2024-08-17T17:05:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:2465: done (2 left) 2024-08-17T17:05:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:2468: done (1 left) 2024-08-17T17:05:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:2469: rollback oid 21 current snap is 316 2024-08-17T17:05:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 21 to 307 2024-08-17T17:05:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:2467: finishing write tid 3 to smithi06387500-8 2024-08-17T17:05:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:2470: read oid 20 snap 294 2024-08-17T17:05:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:2470: expect (ObjNum 809 snap 280 seq_num 809) 2024-08-17T17:05:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:2471: read oid 17 snap 296 2024-08-17T17:05:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:2471: expect deleted 2024-08-17T17:05:24.836 INFO:tasks.rados.rados.0.smithi063.stdout:2472: write oid 10 current snap is 316 2024-08-17T17:05:24.836 INFO:tasks.rados.rados.0.smithi063.stdout:2467: finishing write tid 4 to smithi06387500-8 2024-08-17T17:05:24.836 INFO:tasks.rados.rados.0.smithi063.stdout:2472: seq_num 933 ranges {480931=402383,1313935=515645,2488711=452788,3443230=237701} 2024-08-17T17:05:24.840 INFO:tasks.rados.rados.0.smithi063.stdout:2472: writing smithi06387500-10 from 480931 to 883314 tid 1 2024-08-17T17:05:24.845 INFO:tasks.rados.rados.0.smithi063.stdout:2472: writing smithi06387500-10 from 1313935 to 1829580 tid 2 2024-08-17T17:05:24.851 INFO:tasks.rados.rados.0.smithi063.stdout:2472: writing smithi06387500-10 from 2488711 to 2941499 tid 3 2024-08-17T17:05:24.855 INFO:tasks.rados.rados.0.smithi063.stdout:2472: writing smithi06387500-10 from 3443230 to 3680931 tid 4 2024-08-17T17:05:24.855 INFO:tasks.rados.rados.0.smithi063.stdout:2473: snap_remove snap 295 2024-08-17T17:05:25.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:24 smithi063 ceph-mon[114756]: osdmap e696: 8 total, 8 up, 8 in 2024-08-17T17:05:25.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:24 smithi063 ceph-mon[114756]: pgmap v501: 89 pgs: 89 active+clean; 241 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-08-17T17:05:25.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:24 smithi063 ceph-mon[109843]: osdmap e696: 8 total, 8 up, 8 in 2024-08-17T17:05:25.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:24 smithi063 ceph-mon[109843]: pgmap v501: 89 pgs: 89 active+clean; 241 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-08-17T17:05:25.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:24 smithi162 ceph-mon[104668]: osdmap e696: 8 total, 8 up, 8 in 2024-08-17T17:05:25.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:24 smithi162 ceph-mon[104668]: pgmap v501: 89 pgs: 89 active+clean; 241 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-08-17T17:05:25.672 INFO:tasks.rados.rados.0.smithi063.stdout:2473: done (5 left) 2024-08-17T17:05:25.672 INFO:tasks.rados.rados.0.smithi063.stdout:2474: snap_create 2024-08-17T17:05:25.672 INFO:tasks.rados.rados.0.smithi063.stdout:2467: finishing write tid 5 to smithi06387500-8 2024-08-17T17:05:25.673 INFO:tasks.rados.rados.0.smithi063.stdout:2467: oid 8 updating version 0 to 837 2024-08-17T17:05:25.673 INFO:tasks.rados.rados.0.smithi063.stdout:2467: oid 8 version 837 is already newer than 836 2024-08-17T17:05:25.673 INFO:tasks.rados.rados.0.smithi063.stdout:2467: oid 8 updating version 837 to 838 2024-08-17T17:05:25.673 INFO:tasks.rados.rados.0.smithi063.stdout:2467: oid 8 version 838 is already newer than 835 2024-08-17T17:05:25.673 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 838 (ObjNum 931 snap 316 seq_num 931) dirty exists 2024-08-17T17:05:25.673 INFO:tasks.rados.rados.0.smithi063.stdout:2467: left oid 8 (ObjNum 931 snap 316 seq_num 931) 2024-08-17T17:05:25.673 INFO:tasks.rados.rados.0.smithi063.stdout:2469: finishing rollback tid 0 to smithi06387500-21 2024-08-17T17:05:25.673 INFO:tasks.rados.rados.0.smithi063.stdout:2469: finishing rollback tid 1 to smithi06387500-21 2024-08-17T17:05:25.673 INFO:tasks.rados.rados.0.smithi063.stdout:2469: finishing rollback tid 2 to smithi06387500-21 2024-08-17T17:05:25.673 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 1622 (ObjNum 869 snap 302 seq_num 869) dirty exists 2024-08-17T17:05:25.673 INFO:tasks.rados.rados.0.smithi063.stdout:2470: expect (ObjNum 809 snap 280 seq_num 809) 2024-08-17T17:05:25.878 INFO:tasks.rados.rados.0.smithi063.stdout:2472: finishing write tid 1 to smithi06387500-10 2024-08-17T17:05:25.878 INFO:tasks.rados.rados.0.smithi063.stdout:2472: finishing write tid 2 to smithi06387500-10 2024-08-17T17:05:25.878 INFO:tasks.rados.rados.0.smithi063.stdout:2472: finishing write tid 3 to smithi06387500-10 2024-08-17T17:05:25.878 INFO:tasks.rados.rados.0.smithi063.stdout:2472: finishing write tid 4 to smithi06387500-10 2024-08-17T17:05:25.878 INFO:tasks.rados.rados.0.smithi063.stdout:2472: finishing write tid 5 to smithi06387500-10 2024-08-17T17:05:25.878 INFO:tasks.rados.rados.0.smithi063.stdout:2472: finishing write tid 6 to smithi06387500-10 2024-08-17T17:05:25.878 INFO:tasks.rados.rados.0.smithi063.stdout:2472: oid 10 updating version 0 to 815 2024-08-17T17:05:25.878 INFO:tasks.rados.rados.0.smithi063.stdout:2472: oid 10 version 815 is already newer than 813 2024-08-17T17:05:25.878 INFO:tasks.rados.rados.0.smithi063.stdout:2472: oid 10 updating version 815 to 816 2024-08-17T17:05:25.879 INFO:tasks.rados.rados.0.smithi063.stdout:2472: oid 10 updating version 816 to 817 2024-08-17T17:05:25.879 INFO:tasks.rados.rados.0.smithi063.stdout:2472: oid 10 version 817 is already newer than 814 2024-08-17T17:05:25.879 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 817 (ObjNum 933 snap 316 seq_num 933) dirty exists 2024-08-17T17:05:25.879 INFO:tasks.rados.rados.0.smithi063.stdout:2472: left oid 10 (ObjNum 933 snap 316 seq_num 933) 2024-08-17T17:05:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:25 smithi063 ceph-mon[114756]: osdmap e697: 8 total, 8 up, 8 in 2024-08-17T17:05:26.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:25 smithi063 ceph-mon[109843]: osdmap e697: 8 total, 8 up, 8 in 2024-08-17T17:05:26.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:25 smithi162 ceph-mon[104668]: osdmap e697: 8 total, 8 up, 8 in 2024-08-17T17:05:26.682 INFO:tasks.rados.rados.0.smithi063.stdout:2467: done (5 left) 2024-08-17T17:05:26.683 INFO:tasks.rados.rados.0.smithi063.stdout:2469: done (4 left) 2024-08-17T17:05:26.683 INFO:tasks.rados.rados.0.smithi063.stdout:2470: done (3 left) 2024-08-17T17:05:26.683 INFO:tasks.rados.rados.0.smithi063.stdout:2471: done (2 left) 2024-08-17T17:05:26.683 INFO:tasks.rados.rados.0.smithi063.stdout:2472: done (1 left) 2024-08-17T17:05:26.683 INFO:tasks.rados.rados.0.smithi063.stdout:2474: done (0 left) 2024-08-17T17:05:26.683 INFO:tasks.rados.rados.0.smithi063.stdout:2475: read oid 7 snap -1 2024-08-17T17:05:26.683 INFO:tasks.rados.rados.0.smithi063.stdout:2475: expect (ObjNum 721 snap 246 seq_num 721) 2024-08-17T17:05:26.683 INFO:tasks.rados.rados.0.smithi063.stdout:2476: rollback oid 40 current snap is 317 2024-08-17T17:05:26.683 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 40 to 310 2024-08-17T17:05:26.683 INFO:tasks.rados.rados.0.smithi063.stdout:2477: snap_remove snap 296 2024-08-17T17:05:27.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:26 smithi063 ceph-mon[114756]: osdmap e698: 8 total, 8 up, 8 in 2024-08-17T17:05:27.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:26 smithi063 ceph-mon[114756]: pgmap v504: 89 pgs: 89 active+clean; 241 MiB data, 814 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:05:27.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:26 smithi063 ceph-mon[109843]: osdmap e698: 8 total, 8 up, 8 in 2024-08-17T17:05:27.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:26 smithi063 ceph-mon[109843]: pgmap v504: 89 pgs: 89 active+clean; 241 MiB data, 814 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:05:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:26 smithi162 ceph-mon[104668]: osdmap e698: 8 total, 8 up, 8 in 2024-08-17T17:05:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:26 smithi162 ceph-mon[104668]: pgmap v504: 89 pgs: 89 active+clean; 241 MiB data, 814 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:05:27.591 INFO:tasks.rados.rados.0.smithi063.stdout:2477: done (2 left) 2024-08-17T17:05:27.591 INFO:tasks.rados.rados.0.smithi063.stdout:2478: snap_remove snap 287 2024-08-17T17:05:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:27 smithi063 ceph-mon[114756]: osdmap e699: 8 total, 8 up, 8 in 2024-08-17T17:05:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:27 smithi063 ceph-mon[114756]: osdmap e700: 8 total, 8 up, 8 in 2024-08-17T17:05:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:27 smithi063 ceph-mon[109843]: osdmap e699: 8 total, 8 up, 8 in 2024-08-17T17:05:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:27 smithi063 ceph-mon[109843]: osdmap e700: 8 total, 8 up, 8 in 2024-08-17T17:05:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:27 smithi162 ceph-mon[104668]: osdmap e699: 8 total, 8 up, 8 in 2024-08-17T17:05:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:27 smithi162 ceph-mon[104668]: osdmap e700: 8 total, 8 up, 8 in 2024-08-17T17:05:28.592 INFO:tasks.rados.rados.0.smithi063.stdout:2478: done (2 left) 2024-08-17T17:05:28.592 INFO:tasks.rados.rados.0.smithi063.stdout:2479: rollback oid 41 current snap is 317 2024-08-17T17:05:28.592 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 41 to 312 2024-08-17T17:05:28.592 INFO:tasks.rados.rados.0.smithi063.stdout:2476: finishing rollback tid 1 to smithi06387500-40 2024-08-17T17:05:28.592 INFO:tasks.rados.rados.0.smithi063.stdout:2476: finishing rollback tid 2 to smithi06387500-40 2024-08-17T17:05:28.592 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 825 (ObjNum 878 snap 306 seq_num 878) dirty exists 2024-08-17T17:05:28.592 INFO:tasks.rados.rados.0.smithi063.stdout:2475: expect (ObjNum 721 snap 246 seq_num 721) 2024-08-17T17:05:28.696 INFO:tasks.rados.rados.0.smithi063.stdout:2479: finishing rollback tid 0 to smithi06387500-41 2024-08-17T17:05:28.697 INFO:tasks.rados.rados.0.smithi063.stdout:2479: finishing rollback tid 1 to smithi06387500-41 2024-08-17T17:05:28.697 INFO:tasks.rados.rados.0.smithi063.stdout:2479: finishing rollback tid 2 to smithi06387500-41 2024-08-17T17:05:28.697 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 242 (ObjNum 881 snap 308 seq_num 881) dirty exists 2024-08-17T17:05:28.697 INFO:tasks.rados.rados.0.smithi063.stdout:2475: done (2 left) 2024-08-17T17:05:28.697 INFO:tasks.rados.rados.0.smithi063.stdout:2476: done (1 left) 2024-08-17T17:05:28.697 INFO:tasks.rados.rados.0.smithi063.stdout:2479: done (0 left) 2024-08-17T17:05:28.697 INFO:tasks.rados.rados.0.smithi063.stdout:2480: read oid 20 snap -1 2024-08-17T17:05:28.697 INFO:tasks.rados.rados.0.smithi063.stdout:2480: expect (ObjNum 809 snap 280 seq_num 809) 2024-08-17T17:05:28.697 INFO:tasks.rados.rados.0.smithi063.stdout:2481: snap_remove snap 308 2024-08-17T17:05:29.688 INFO:tasks.rados.rados.0.smithi063.stdout:2481: done (1 left) 2024-08-17T17:05:29.688 INFO:tasks.rados.rados.0.smithi063.stdout:2482: snap_create 2024-08-17T17:05:29.688 INFO:tasks.rados.rados.0.smithi063.stdout:2480: expect (ObjNum 809 snap 280 seq_num 809) 2024-08-17T17:05:29.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:29 smithi162 ceph-mon[104668]: pgmap v507: 89 pgs: 89 active+clean; 255 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-08-17T17:05:29.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:29 smithi162 ceph-mon[104668]: osdmap e701: 8 total, 8 up, 8 in 2024-08-17T17:05:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:29 smithi063 ceph-mon[114756]: pgmap v507: 89 pgs: 89 active+clean; 255 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-08-17T17:05:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:29 smithi063 ceph-mon[114756]: osdmap e701: 8 total, 8 up, 8 in 2024-08-17T17:05:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:29 smithi063 ceph-mon[109843]: pgmap v507: 89 pgs: 89 active+clean; 255 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-08-17T17:05:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:29 smithi063 ceph-mon[109843]: osdmap e701: 8 total, 8 up, 8 in 2024-08-17T17:05:30.690 INFO:tasks.rados.rados.0.smithi063.stdout:2480: done (1 left) 2024-08-17T17:05:30.690 INFO:tasks.rados.rados.0.smithi063.stdout:2482: done (0 left) 2024-08-17T17:05:30.690 INFO:tasks.rados.rados.0.smithi063.stdout:2483: write oid 39 current snap is 318 2024-08-17T17:05:30.691 INFO:tasks.rados.rados.0.smithi063.stdout:2483: seq_num 934 ranges {497738=477632,1772417=775656,3316134=381604} 2024-08-17T17:05:30.695 INFO:tasks.rados.rados.0.smithi063.stdout:2483: writing smithi06387500-39 from 497738 to 975370 tid 1 2024-08-17T17:05:30.704 INFO:tasks.rados.rados.0.smithi063.stdout:2483: writing smithi06387500-39 from 1772417 to 2548073 tid 2 2024-08-17T17:05:30.710 INFO:tasks.rados.rados.0.smithi063.stdout:2483: writing smithi06387500-39 from 3316134 to 3697738 tid 3 2024-08-17T17:05:30.710 INFO:tasks.rados.rados.0.smithi063.stdout:2484: snap_remove snap 310 2024-08-17T17:05:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:30 smithi063 ceph-mon[114756]: osdmap e702: 8 total, 8 up, 8 in 2024-08-17T17:05:31.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:30 smithi063 ceph-mon[114756]: pgmap v510: 89 pgs: 89 active+clean; 255 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-08-17T17:05:31.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:30 smithi063 ceph-mon[109843]: osdmap e702: 8 total, 8 up, 8 in 2024-08-17T17:05:31.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:30 smithi063 ceph-mon[109843]: pgmap v510: 89 pgs: 89 active+clean; 255 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-08-17T17:05:31.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:30 smithi162 ceph-mon[104668]: osdmap e702: 8 total, 8 up, 8 in 2024-08-17T17:05:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:30 smithi162 ceph-mon[104668]: pgmap v510: 89 pgs: 89 active+clean; 255 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-08-17T17:05:31.696 INFO:tasks.rados.rados.0.smithi063.stdout:2484: done (1 left) 2024-08-17T17:05:31.696 INFO:tasks.rados.rados.0.smithi063.stdout:2485: snap_create 2024-08-17T17:05:31.696 INFO:tasks.rados.rados.0.smithi063.stdout:2483: finishing write tid 1 to smithi06387500-39 2024-08-17T17:05:31.696 INFO:tasks.rados.rados.0.smithi063.stdout:2483: finishing write tid 2 to smithi06387500-39 2024-08-17T17:05:31.696 INFO:tasks.rados.rados.0.smithi063.stdout:2483: finishing write tid 3 to smithi06387500-39 2024-08-17T17:05:31.696 INFO:tasks.rados.rados.0.smithi063.stdout:2483: finishing write tid 4 to smithi06387500-39 2024-08-17T17:05:31.696 INFO:tasks.rados.rados.0.smithi063.stdout:2483: finishing write tid 5 to smithi06387500-39 2024-08-17T17:05:31.696 INFO:tasks.rados.rados.0.smithi063.stdout:2483: oid 39 updating version 0 to 549 2024-08-17T17:05:31.696 INFO:tasks.rados.rados.0.smithi063.stdout:2483: oid 39 version 549 is already newer than 548 2024-08-17T17:05:31.696 INFO:tasks.rados.rados.0.smithi063.stdout:2483: oid 39 updating version 549 to 550 2024-08-17T17:05:31.696 INFO:tasks.rados.rados.0.smithi063.stdout:2483: oid 39 updating version 550 to 551 2024-08-17T17:05:31.697 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 551 (ObjNum 934 snap 318 seq_num 934) dirty exists 2024-08-17T17:05:31.697 INFO:tasks.rados.rados.0.smithi063.stdout:2483: left oid 39 (ObjNum 934 snap 318 seq_num 934) 2024-08-17T17:05:32.012 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:31 smithi162 ceph-mon[104668]: osdmap e703: 8 total, 8 up, 8 in 2024-08-17T17:05:32.012 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:31 smithi063 ceph-mon[114756]: osdmap e703: 8 total, 8 up, 8 in 2024-08-17T17:05:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:32.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:31 smithi063 ceph-mon[109843]: osdmap e703: 8 total, 8 up, 8 in 2024-08-17T17:05:32.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:32.706 INFO:tasks.rados.rados.0.smithi063.stdout:2483: done (1 left) 2024-08-17T17:05:32.707 INFO:tasks.rados.rados.0.smithi063.stdout:2485: done (0 left) 2024-08-17T17:05:32.707 INFO:tasks.rados.rados.0.smithi063.stdout:2486: write oid 13 current snap is 319 2024-08-17T17:05:32.707 INFO:tasks.rados.rados.0.smithi063.stdout:2486: seq_num 935 ranges {514545=552881,1830899=519314,2943557=475332} 2024-08-17T17:05:32.712 INFO:tasks.rados.rados.0.smithi063.stdout:2486: writing smithi06387500-13 from 514545 to 1067426 tid 1 2024-08-17T17:05:32.722 INFO:tasks.rados.rados.0.smithi063.stdout:2486: writing smithi06387500-13 from 1830899 to 2350213 tid 2 2024-08-17T17:05:32.726 INFO:tasks.rados.rados.0.smithi063.stdout:2486: writing smithi06387500-13 from 2943557 to 3418889 tid 3 2024-08-17T17:05:32.726 INFO:tasks.rados.rados.0.smithi063.stdout:2487: delete oid 35 current snap is 319 2024-08-17T17:05:32.726 INFO:tasks.rados.rados.0.smithi063.stdout:2486: finishing write tid 1 to smithi06387500-13 2024-08-17T17:05:32.726 INFO:tasks.rados.rados.0.smithi063.stdout:2486: finishing write tid 2 to smithi06387500-13 2024-08-17T17:05:32.729 INFO:tasks.rados.rados.0.smithi063.stdout:2486: finishing write tid 3 to smithi06387500-13 2024-08-17T17:05:32.729 INFO:tasks.rados.rados.0.smithi063.stdout:2486: finishing write tid 4 to smithi06387500-13 2024-08-17T17:05:32.729 INFO:tasks.rados.rados.0.smithi063.stdout:2486: finishing write tid 5 to smithi06387500-13 2024-08-17T17:05:32.730 INFO:tasks.rados.rados.0.smithi063.stdout:2486: oid 13 updating version 0 to 1207 2024-08-17T17:05:32.730 INFO:tasks.rados.rados.0.smithi063.stdout:2486: oid 13 updating version 1207 to 1209 2024-08-17T17:05:32.730 INFO:tasks.rados.rados.0.smithi063.stdout:2486: oid 13 updating version 1209 to 1210 2024-08-17T17:05:32.730 INFO:tasks.rados.rados.0.smithi063.stdout:2486: oid 13 version 1210 is already newer than 1208 2024-08-17T17:05:32.730 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1210 (ObjNum 935 snap 319 seq_num 935) dirty exists 2024-08-17T17:05:32.730 INFO:tasks.rados.rados.0.smithi063.stdout:2486: left oid 13 (ObjNum 935 snap 319 seq_num 935) 2024-08-17T17:05:32.731 INFO:tasks.rados.rados.0.smithi063.stdout:2486: done (1 left) 2024-08-17T17:05:32.731 INFO:tasks.rados.rados.0.smithi063.stdout:2487: done (0 left) 2024-08-17T17:05:32.731 INFO:tasks.rados.rados.0.smithi063.stdout:2488: snap_create 2024-08-17T17:05:32.950 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:32 smithi162 ceph-mon[104668]: osdmap e704: 8 total, 8 up, 8 in 2024-08-17T17:05:32.950 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:32 smithi162 ceph-mon[104668]: pgmap v513: 89 pgs: 89 active+clean; 227 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 399 KiB/s wr, 4 op/s 2024-08-17T17:05:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:32 smithi063 ceph-mon[114756]: osdmap e704: 8 total, 8 up, 8 in 2024-08-17T17:05:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:32 smithi063 ceph-mon[114756]: pgmap v513: 89 pgs: 89 active+clean; 227 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 399 KiB/s wr, 4 op/s 2024-08-17T17:05:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:32 smithi063 ceph-mon[109843]: osdmap e704: 8 total, 8 up, 8 in 2024-08-17T17:05:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:32 smithi063 ceph-mon[109843]: pgmap v513: 89 pgs: 89 active+clean; 227 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 399 KiB/s wr, 4 op/s 2024-08-17T17:05:33.698 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:05:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:05:33] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.51.0" 2024-08-17T17:05:33.712 INFO:tasks.rados.rados.0.smithi063.stdout:2488: done (0 left) 2024-08-17T17:05:33.712 INFO:tasks.rados.rados.0.smithi063.stdout:2489: snap_remove snap 316 2024-08-17T17:05:34.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:33 smithi063 ceph-mon[109843]: osdmap e705: 8 total, 8 up, 8 in 2024-08-17T17:05:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:33 smithi063 ceph-mon[114756]: osdmap e705: 8 total, 8 up, 8 in 2024-08-17T17:05:34.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:33 smithi162 ceph-mon[104668]: osdmap e705: 8 total, 8 up, 8 in 2024-08-17T17:05:34.721 INFO:tasks.rados.rados.0.smithi063.stdout:2489: done (0 left) 2024-08-17T17:05:34.721 INFO:tasks.rados.rados.0.smithi063.stdout:2490: read oid 30 snap -1 2024-08-17T17:05:34.721 INFO:tasks.rados.rados.0.smithi063.stdout:2490: expect (ObjNum 710 snap 234 seq_num 710) 2024-08-17T17:05:34.721 INFO:tasks.rados.rados.0.smithi063.stdout:2491: read oid 44 snap -1 2024-08-17T17:05:34.722 INFO:tasks.rados.rados.0.smithi063.stdout:2491: expect deleted 2024-08-17T17:05:34.722 INFO:tasks.rados.rados.0.smithi063.stdout:2492: rollback oid 20 current snap is 320 2024-08-17T17:05:34.722 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 20 to 301 2024-08-17T17:05:34.722 INFO:tasks.rados.rados.0.smithi063.stdout:2493: delete oid 23 current snap is 320 2024-08-17T17:05:34.724 INFO:tasks.rados.rados.0.smithi063.stdout:2491: done (3 left) 2024-08-17T17:05:34.724 INFO:tasks.rados.rados.0.smithi063.stdout:2493: done (2 left) 2024-08-17T17:05:34.724 INFO:tasks.rados.rados.0.smithi063.stdout:2494: read oid 45 snap 301 2024-08-17T17:05:34.725 INFO:tasks.rados.rados.0.smithi063.stdout:2494: expect (ObjNum 861 snap 293 seq_num 861) 2024-08-17T17:05:34.725 INFO:tasks.rados.rados.0.smithi063.stdout:2495: snap_create 2024-08-17T17:05:34.728 INFO:tasks.rados.rados.0.smithi063.stdout:2492: finishing rollback tid 0 to smithi06387500-20 2024-08-17T17:05:34.729 INFO:tasks.rados.rados.0.smithi063.stdout:2492: finishing rollback tid 1 to smithi06387500-20 2024-08-17T17:05:34.729 INFO:tasks.rados.rados.0.smithi063.stdout:2492: finishing rollback tid 2 to smithi06387500-20 2024-08-17T17:05:34.729 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 505 (ObjNum 809 snap 280 seq_num 809) dirty exists 2024-08-17T17:05:34.734 INFO:tasks.rados.rados.0.smithi063.stdout:2490: expect (ObjNum 710 snap 234 seq_num 710) 2024-08-17T17:05:34.867 INFO:tasks.rados.rados.0.smithi063.stdout:2494: expect (ObjNum 861 snap 293 seq_num 861) 2024-08-17T17:05:35.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:34 smithi063 ceph-mon[114756]: osdmap e706: 8 total, 8 up, 8 in 2024-08-17T17:05:35.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:34 smithi063 ceph-mon[114756]: pgmap v516: 89 pgs: 89 active+clean; 227 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 399 KiB/s wr, 4 op/s 2024-08-17T17:05:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:34 smithi063 ceph-mon[109843]: osdmap e706: 8 total, 8 up, 8 in 2024-08-17T17:05:35.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:34 smithi063 ceph-mon[109843]: pgmap v516: 89 pgs: 89 active+clean; 227 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 399 KiB/s wr, 4 op/s 2024-08-17T17:05:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:34 smithi162 ceph-mon[104668]: osdmap e706: 8 total, 8 up, 8 in 2024-08-17T17:05:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:34 smithi162 ceph-mon[104668]: pgmap v516: 89 pgs: 89 active+clean; 227 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 399 KiB/s wr, 4 op/s 2024-08-17T17:05:35.725 INFO:tasks.rados.rados.0.smithi063.stdout:2490: done (3 left) 2024-08-17T17:05:35.725 INFO:tasks.rados.rados.0.smithi063.stdout:2492: done (2 left) 2024-08-17T17:05:35.725 INFO:tasks.rados.rados.0.smithi063.stdout:2494: done (1 left) 2024-08-17T17:05:35.725 INFO:tasks.rados.rados.0.smithi063.stdout:2495: done (0 left) 2024-08-17T17:05:35.726 INFO:tasks.rados.rados.0.smithi063.stdout:2496: write oid 21 current snap is 321 2024-08-17T17:05:35.726 INFO:tasks.rados.rados.0.smithi063.stdout:2496: seq_num 938 ranges {564966=494981,1606345=782994,3025826=425501} 2024-08-17T17:05:35.731 INFO:tasks.rados.rados.0.smithi063.stdout:2496: writing smithi06387500-21 from 564966 to 1059947 tid 1 2024-08-17T17:05:35.738 INFO:tasks.rados.rados.0.smithi063.stdout:2496: writing smithi06387500-21 from 1606345 to 2389339 tid 2 2024-08-17T17:05:35.745 INFO:tasks.rados.rados.0.smithi063.stdout:2496: writing smithi06387500-21 from 3025826 to 3451327 tid 3 2024-08-17T17:05:35.745 INFO:tasks.rados.rados.0.smithi063.stdout:2497: write oid 30 current snap is 321 2024-08-17T17:05:35.745 INFO:tasks.rados.rados.0.smithi063.stdout:2496: finishing write tid 1 to smithi06387500-21 2024-08-17T17:05:35.745 INFO:tasks.rados.rados.0.smithi063.stdout:2496: finishing write tid 2 to smithi06387500-21 2024-08-17T17:05:35.745 INFO:tasks.rados.rados.0.smithi063.stdout:2497: seq_num 939 ranges {581773=570230,1664827=526652,2653249=636773} 2024-08-17T17:05:35.751 INFO:tasks.rados.rados.0.smithi063.stdout:2497: writing smithi06387500-30 from 581773 to 1152003 tid 1 2024-08-17T17:05:35.756 INFO:tasks.rados.rados.0.smithi063.stdout:2497: writing smithi06387500-30 from 1664827 to 2191479 tid 2 2024-08-17T17:05:35.763 INFO:tasks.rados.rados.0.smithi063.stdout:2497: writing smithi06387500-30 from 2653249 to 3290022 tid 3 2024-08-17T17:05:35.763 INFO:tasks.rados.rados.0.smithi063.stdout:2498: write oid 17 current snap is 321 2024-08-17T17:05:35.763 INFO:tasks.rados.rados.0.smithi063.stdout:2496: finishing write tid 3 to smithi06387500-21 2024-08-17T17:05:35.763 INFO:tasks.rados.rados.0.smithi063.stdout:2496: finishing write tid 4 to smithi06387500-21 2024-08-17T17:05:35.763 INFO:tasks.rados.rados.0.smithi063.stdout:2496: finishing write tid 5 to smithi06387500-21 2024-08-17T17:05:35.763 INFO:tasks.rados.rados.0.smithi063.stdout:2496: oid 21 updating version 0 to 1665 2024-08-17T17:05:35.763 INFO:tasks.rados.rados.0.smithi063.stdout:2496: oid 21 updating version 1665 to 1666 2024-08-17T17:05:35.764 INFO:tasks.rados.rados.0.smithi063.stdout:2496: oid 21 updating version 1666 to 1667 2024-08-17T17:05:35.764 INFO:tasks.rados.rados.0.smithi063.stdout:2496: oid 21 version 1667 is already newer than 1664 2024-08-17T17:05:35.764 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 1667 (ObjNum 938 snap 321 seq_num 938) dirty exists 2024-08-17T17:05:35.764 INFO:tasks.rados.rados.0.smithi063.stdout:2496: left oid 21 (ObjNum 938 snap 321 seq_num 938) 2024-08-17T17:05:35.764 INFO:tasks.rados.rados.0.smithi063.stdout:2497: finishing write tid 1 to smithi06387500-30 2024-08-17T17:05:35.764 INFO:tasks.rados.rados.0.smithi063.stdout:2497: finishing write tid 2 to smithi06387500-30 2024-08-17T17:05:35.764 INFO:tasks.rados.rados.0.smithi063.stdout:2498: seq_num 940 ranges {598580=645479,1723309=786663,3080672=448045} 2024-08-17T17:05:35.770 INFO:tasks.rados.rados.0.smithi063.stdout:2498: writing smithi06387500-17 from 598580 to 1244059 tid 1 2024-08-17T17:05:35.777 INFO:tasks.rados.rados.0.smithi063.stdout:2498: writing smithi06387500-17 from 1723309 to 2509972 tid 2 2024-08-17T17:05:35.783 INFO:tasks.rados.rados.0.smithi063.stdout:2498: writing smithi06387500-17 from 3080672 to 3528717 tid 3 2024-08-17T17:05:35.783 INFO:tasks.rados.rados.0.smithi063.stdout:2496: done (2 left) 2024-08-17T17:05:35.783 INFO:tasks.rados.rados.0.smithi063.stdout:2499: rollback oid 9 current snap is 321 2024-08-17T17:05:35.783 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 9 to 314 2024-08-17T17:05:35.783 INFO:tasks.rados.rados.0.smithi063.stdout:2497: finishing write tid 3 to smithi06387500-30 2024-08-17T17:05:35.783 INFO:tasks.rados.rados.0.smithi063.stdout:2497: finishing write tid 4 to smithi06387500-30 2024-08-17T17:05:35.783 INFO:tasks.rados.rados.0.smithi063.stdout:2497: finishing write tid 5 to smithi06387500-30 2024-08-17T17:05:35.783 INFO:tasks.rados.rados.0.smithi063.stdout:2497: oid 30 updating version 0 to 1131 2024-08-17T17:05:35.784 INFO:tasks.rados.rados.0.smithi063.stdout:2497: oid 30 version 1131 is already newer than 1129 2024-08-17T17:05:35.784 INFO:tasks.rados.rados.0.smithi063.stdout:2497: oid 30 version 1131 is already newer than 1130 2024-08-17T17:05:35.784 INFO:tasks.rados.rados.0.smithi063.stdout:2497: oid 30 version 1131 is already newer than 1128 2024-08-17T17:05:35.784 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1131 (ObjNum 939 snap 321 seq_num 939) dirty exists 2024-08-17T17:05:35.784 INFO:tasks.rados.rados.0.smithi063.stdout:2497: left oid 30 (ObjNum 939 snap 321 seq_num 939) 2024-08-17T17:05:35.784 INFO:tasks.rados.rados.0.smithi063.stdout:2498: finishing write tid 1 to smithi06387500-17 2024-08-17T17:05:35.784 INFO:tasks.rados.rados.0.smithi063.stdout:2498: finishing write tid 2 to smithi06387500-17 2024-08-17T17:05:35.784 INFO:tasks.rados.rados.0.smithi063.stdout:2497: done (2 left) 2024-08-17T17:05:35.784 INFO:tasks.rados.rados.0.smithi063.stdout:2500: snap_create 2024-08-17T17:05:35.786 INFO:tasks.rados.rados.0.smithi063.stdout:2498: finishing write tid 3 to smithi06387500-17 2024-08-17T17:05:35.791 INFO:tasks.rados.rados.0.smithi063.stdout:2498: finishing write tid 4 to smithi06387500-17 2024-08-17T17:05:35.792 INFO:tasks.rados.rados.0.smithi063.stdout:2499: finishing rollback tid 1 to smithi06387500-9 2024-08-17T17:05:35.792 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 1066 (ObjNum 129 snap 0 seq_num 1806764696) dirty dne 2024-08-17T17:05:35.792 INFO:tasks.rados.rados.0.smithi063.stdout:2498: finishing write tid 5 to smithi06387500-17 2024-08-17T17:05:35.792 INFO:tasks.rados.rados.0.smithi063.stdout:2498: oid 17 updating version 0 to 784 2024-08-17T17:05:35.792 INFO:tasks.rados.rados.0.smithi063.stdout:2498: oid 17 updating version 784 to 787 2024-08-17T17:05:35.792 INFO:tasks.rados.rados.0.smithi063.stdout:2498: oid 17 version 787 is already newer than 786 2024-08-17T17:05:35.792 INFO:tasks.rados.rados.0.smithi063.stdout:2498: oid 17 version 787 is already newer than 785 2024-08-17T17:05:35.792 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 787 (ObjNum 940 snap 321 seq_num 940) dirty exists 2024-08-17T17:05:35.792 INFO:tasks.rados.rados.0.smithi063.stdout:2498: left oid 17 (ObjNum 940 snap 321 seq_num 940) 2024-08-17T17:05:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:35 smithi063 ceph-mon[114756]: osdmap e707: 8 total, 8 up, 8 in 2024-08-17T17:05:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:35 smithi063 ceph-mon[109843]: osdmap e707: 8 total, 8 up, 8 in 2024-08-17T17:05:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:35 smithi162 ceph-mon[104668]: osdmap e707: 8 total, 8 up, 8 in 2024-08-17T17:05:36.731 INFO:tasks.rados.rados.0.smithi063.stdout:2498: done (2 left) 2024-08-17T17:05:36.732 INFO:tasks.rados.rados.0.smithi063.stdout:2499: done (1 left) 2024-08-17T17:05:36.732 INFO:tasks.rados.rados.0.smithi063.stdout:2500: done (0 left) 2024-08-17T17:05:36.732 INFO:tasks.rados.rados.0.smithi063.stdout:2501: snap_create 2024-08-17T17:05:37.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:36 smithi063 ceph-mon[109843]: osdmap e708: 8 total, 8 up, 8 in 2024-08-17T17:05:37.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:36 smithi063 ceph-mon[109843]: pgmap v519: 89 pgs: 89 active+clean; 227 MiB data, 797 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:05:37.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:36 smithi063 ceph-mon[114756]: osdmap e708: 8 total, 8 up, 8 in 2024-08-17T17:05:37.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:36 smithi063 ceph-mon[114756]: pgmap v519: 89 pgs: 89 active+clean; 227 MiB data, 797 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:05:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:36 smithi162 ceph-mon[104668]: osdmap e708: 8 total, 8 up, 8 in 2024-08-17T17:05:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:36 smithi162 ceph-mon[104668]: pgmap v519: 89 pgs: 89 active+clean; 227 MiB data, 797 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:05:37.592 INFO:tasks.rados.rados.0.smithi063.stdout:2501: done (0 left) 2024-08-17T17:05:37.593 INFO:tasks.rados.rados.0.smithi063.stdout:2502: write oid 35 current snap is 323 2024-08-17T17:05:37.593 INFO:tasks.rados.rados.0.smithi063.stdout:2502: seq_num 941 ranges {615387=720728,1781791=530321,3108095=707292} 2024-08-17T17:05:37.599 INFO:tasks.rados.rados.0.smithi063.stdout:2502: writing smithi06387500-35 from 615387 to 1336115 tid 1 2024-08-17T17:05:37.605 INFO:tasks.rados.rados.0.smithi063.stdout:2502: writing smithi06387500-35 from 1781791 to 2312112 tid 2 2024-08-17T17:05:37.614 INFO:tasks.rados.rados.0.smithi063.stdout:2502: writing smithi06387500-35 from 3108095 to 3815387 tid 3 2024-08-17T17:05:37.614 INFO:tasks.rados.rados.0.smithi063.stdout:2503: write oid 26 current snap is 323 2024-08-17T17:05:37.614 INFO:tasks.rados.rados.0.smithi063.stdout:2502: finishing write tid 1 to smithi06387500-35 2024-08-17T17:05:37.614 INFO:tasks.rados.rados.0.smithi063.stdout:2502: finishing write tid 2 to smithi06387500-35 2024-08-17T17:05:37.614 INFO:tasks.rados.rados.0.smithi063.stdout:2503: seq_num 942 ranges {632194=795977,2123920=790332,3419165=413029} 2024-08-17T17:05:37.621 INFO:tasks.rados.rados.0.smithi063.stdout:2503: writing smithi06387500-26 from 632194 to 1428171 tid 1 2024-08-17T17:05:37.630 INFO:tasks.rados.rados.0.smithi063.stdout:2503: writing smithi06387500-26 from 2123920 to 2914252 tid 2 2024-08-17T17:05:37.635 INFO:tasks.rados.rados.0.smithi063.stdout:2503: writing smithi06387500-26 from 3419165 to 3832194 tid 3 2024-08-17T17:05:37.635 INFO:tasks.rados.rados.0.smithi063.stdout:2504: read oid 34 snap 307 2024-08-17T17:05:37.635 INFO:tasks.rados.rados.0.smithi063.stdout:2504: expect (ObjNum 879 snap 306 seq_num 879) 2024-08-17T17:05:37.635 INFO:tasks.rados.rados.0.smithi063.stdout:2505: snap_remove snap 317 2024-08-17T17:05:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:37 smithi063 ceph-mon[114756]: osdmap e709: 8 total, 8 up, 8 in 2024-08-17T17:05:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:37 smithi063 ceph-mon[114756]: osdmap e710: 8 total, 8 up, 8 in 2024-08-17T17:05:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:37 smithi063 ceph-mon[109843]: osdmap e709: 8 total, 8 up, 8 in 2024-08-17T17:05:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:37 smithi063 ceph-mon[109843]: osdmap e710: 8 total, 8 up, 8 in 2024-08-17T17:05:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:37 smithi162 ceph-mon[104668]: osdmap e709: 8 total, 8 up, 8 in 2024-08-17T17:05:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:37 smithi162 ceph-mon[104668]: osdmap e710: 8 total, 8 up, 8 in 2024-08-17T17:05:38.595 INFO:tasks.rados.rados.0.smithi063.stdout:2505: done (3 left) 2024-08-17T17:05:38.595 INFO:tasks.rados.rados.0.smithi063.stdout:2506: rollback oid 14 current snap is 323 2024-08-17T17:05:38.595 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 14 to 315 2024-08-17T17:05:38.595 INFO:tasks.rados.rados.0.smithi063.stdout:2502: finishing write tid 3 to smithi06387500-35 2024-08-17T17:05:38.595 INFO:tasks.rados.rados.0.smithi063.stdout:2502: finishing write tid 4 to smithi06387500-35 2024-08-17T17:05:38.595 INFO:tasks.rados.rados.0.smithi063.stdout:2502: finishing write tid 5 to smithi06387500-35 2024-08-17T17:05:38.595 INFO:tasks.rados.rados.0.smithi063.stdout:2502: oid 35 updating version 0 to 2481 2024-08-17T17:05:38.595 INFO:tasks.rados.rados.0.smithi063.stdout:2502: oid 35 version 2481 is already newer than 2480 2024-08-17T17:05:38.595 INFO:tasks.rados.rados.0.smithi063.stdout:2502: oid 35 version 2481 is already newer than 2479 2024-08-17T17:05:38.596 INFO:tasks.rados.rados.0.smithi063.stdout:2502: oid 35 version 2481 is already newer than 2478 2024-08-17T17:05:38.596 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 2481 (ObjNum 941 snap 323 seq_num 941) dirty exists 2024-08-17T17:05:38.596 INFO:tasks.rados.rados.0.smithi063.stdout:2502: left oid 35 (ObjNum 941 snap 323 seq_num 941) 2024-08-17T17:05:38.596 INFO:tasks.rados.rados.0.smithi063.stdout:2503: finishing write tid 1 to smithi06387500-26 2024-08-17T17:05:38.596 INFO:tasks.rados.rados.0.smithi063.stdout:2503: finishing write tid 2 to smithi06387500-26 2024-08-17T17:05:38.596 INFO:tasks.rados.rados.0.smithi063.stdout:2503: finishing write tid 3 to smithi06387500-26 2024-08-17T17:05:38.596 INFO:tasks.rados.rados.0.smithi063.stdout:2502: done (3 left) 2024-08-17T17:05:38.596 INFO:tasks.rados.rados.0.smithi063.stdout:2507: read oid 21 snap -1 2024-08-17T17:05:38.596 INFO:tasks.rados.rados.0.smithi063.stdout:2507: expect (ObjNum 938 snap 321 seq_num 938) 2024-08-17T17:05:38.596 INFO:tasks.rados.rados.0.smithi063.stdout:2508: rollback oid 19 current snap is 323 2024-08-17T17:05:38.596 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 19 to 319 2024-08-17T17:05:38.596 INFO:tasks.rados.rados.0.smithi063.stdout:2503: finishing write tid 4 to smithi06387500-26 2024-08-17T17:05:38.596 INFO:tasks.rados.rados.0.smithi063.stdout:2503: finishing write tid 5 to smithi06387500-26 2024-08-17T17:05:38.597 INFO:tasks.rados.rados.0.smithi063.stdout:2503: oid 26 updating version 0 to 792 2024-08-17T17:05:38.597 INFO:tasks.rados.rados.0.smithi063.stdout:2503: oid 26 version 792 is already newer than 789 2024-08-17T17:05:38.597 INFO:tasks.rados.rados.0.smithi063.stdout:2503: oid 26 version 792 is already newer than 790 2024-08-17T17:05:38.597 INFO:tasks.rados.rados.0.smithi063.stdout:2503: oid 26 version 792 is already newer than 791 2024-08-17T17:05:38.597 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 792 (ObjNum 942 snap 323 seq_num 942) dirty exists 2024-08-17T17:05:38.597 INFO:tasks.rados.rados.0.smithi063.stdout:2503: left oid 26 (ObjNum 942 snap 323 seq_num 942) 2024-08-17T17:05:38.597 INFO:tasks.rados.rados.0.smithi063.stdout:2504: expect (ObjNum 879 snap 306 seq_num 879) 2024-08-17T17:05:38.739 INFO:tasks.rados.rados.0.smithi063.stdout:2506: finishing rollback tid 0 to smithi06387500-14 2024-08-17T17:05:38.739 INFO:tasks.rados.rados.0.smithi063.stdout:2506: finishing rollback tid 1 to smithi06387500-14 2024-08-17T17:05:38.739 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 2484 (ObjNum 145 snap 0 seq_num 1027690420) dirty dne 2024-08-17T17:05:38.739 INFO:tasks.rados.rados.0.smithi063.stdout:2503: done (4 left) 2024-08-17T17:05:38.739 INFO:tasks.rados.rados.0.smithi063.stdout:2504: done (3 left) 2024-08-17T17:05:38.740 INFO:tasks.rados.rados.0.smithi063.stdout:2506: done (2 left) 2024-08-17T17:05:38.740 INFO:tasks.rados.rados.0.smithi063.stdout:2509: write oid 39 current snap is 323 2024-08-17T17:05:38.740 INFO:tasks.rados.rados.0.smithi063.stdout:2509: seq_num 943 ranges {649001=587579,1898755=533990,3162941=686060} 2024-08-17T17:05:38.745 INFO:tasks.rados.rados.0.smithi063.stdout:2509: writing smithi06387500-39 from 649001 to 1236580 tid 1 2024-08-17T17:05:38.752 INFO:tasks.rados.rados.0.smithi063.stdout:2509: writing smithi06387500-39 from 1898755 to 2432745 tid 2 2024-08-17T17:05:38.760 INFO:tasks.rados.rados.0.smithi063.stdout:2509: writing smithi06387500-39 from 3162941 to 3849001 tid 3 2024-08-17T17:05:38.760 INFO:tasks.rados.rados.0.smithi063.stdout:2510: read oid 27 snap -1 2024-08-17T17:05:38.761 INFO:tasks.rados.rados.0.smithi063.stdout:2510: expect deleted 2024-08-17T17:05:38.761 INFO:tasks.rados.rados.0.smithi063.stdout:2511: read oid 49 snap -1 2024-08-17T17:05:38.761 INFO:tasks.rados.rados.0.smithi063.stdout:2511: expect (ObjNum 896 snap 310 seq_num 896) 2024-08-17T17:05:38.761 INFO:tasks.rados.rados.0.smithi063.stdout:2512: delete oid 1 current snap is 323 2024-08-17T17:05:38.761 INFO:tasks.rados.rados.0.smithi063.stdout:2507: expect (ObjNum 938 snap 321 seq_num 938) 2024-08-17T17:05:38.939 INFO:tasks.rados.rados.0.smithi063.stdout:2508: finishing rollback tid 0 to smithi06387500-19 2024-08-17T17:05:38.940 INFO:tasks.rados.rados.0.smithi063.stdout:2508: finishing rollback tid 1 to smithi06387500-19 2024-08-17T17:05:38.940 INFO:tasks.rados.rados.0.smithi063.stdout:2508: finishing rollback tid 2 to smithi06387500-19 2024-08-17T17:05:38.940 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 1671 (ObjNum 910 snap 312 seq_num 910) dirty exists 2024-08-17T17:05:38.940 INFO:tasks.rados.rados.0.smithi063.stdout:2509: finishing write tid 1 to smithi06387500-39 2024-08-17T17:05:38.940 INFO:tasks.rados.rados.0.smithi063.stdout:2509: finishing write tid 2 to smithi06387500-39 2024-08-17T17:05:38.940 INFO:tasks.rados.rados.0.smithi063.stdout:2507: done (5 left) 2024-08-17T17:05:38.940 INFO:tasks.rados.rados.0.smithi063.stdout:2508: done (4 left) 2024-08-17T17:05:38.940 INFO:tasks.rados.rados.0.smithi063.stdout:2512: done (3 left) 2024-08-17T17:05:38.940 INFO:tasks.rados.rados.0.smithi063.stdout:2513: write oid 20 current snap is 323 2024-08-17T17:05:38.940 INFO:tasks.rados.rados.0.smithi063.stdout:2509: finishing write tid 3 to smithi06387500-39 2024-08-17T17:05:38.940 INFO:tasks.rados.rados.0.smithi063.stdout:2509: finishing write tid 4 to smithi06387500-39 2024-08-17T17:05:38.940 INFO:tasks.rados.rados.0.smithi063.stdout:2509: finishing write tid 5 to smithi06387500-39 2024-08-17T17:05:38.941 INFO:tasks.rados.rados.0.smithi063.stdout:2509: oid 39 updating version 0 to 561 2024-08-17T17:05:38.941 INFO:tasks.rados.rados.0.smithi063.stdout:2509: oid 39 updating version 561 to 564 2024-08-17T17:05:38.941 INFO:tasks.rados.rados.0.smithi063.stdout:2509: oid 39 version 564 is already newer than 563 2024-08-17T17:05:38.941 INFO:tasks.rados.rados.0.smithi063.stdout:2509: oid 39 version 564 is already newer than 562 2024-08-17T17:05:38.941 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 564 (ObjNum 943 snap 323 seq_num 943) dirty exists 2024-08-17T17:05:38.941 INFO:tasks.rados.rados.0.smithi063.stdout:2509: left oid 39 (ObjNum 943 snap 323 seq_num 943) 2024-08-17T17:05:38.941 INFO:tasks.rados.rados.0.smithi063.stdout:2511: expect (ObjNum 896 snap 310 seq_num 896) 2024-08-17T17:05:39.084 INFO:tasks.rados.rados.0.smithi063.stdout:2513: seq_num 945 ranges {682615=738077,2015719=537659,3217787=420758} 2024-08-17T17:05:39.091 INFO:tasks.rados.rados.0.smithi063.stdout:2513: writing smithi06387500-20 from 682615 to 1420692 tid 1 2024-08-17T17:05:39.098 INFO:tasks.rados.rados.0.smithi063.stdout:2513: writing smithi06387500-20 from 2015719 to 2553378 tid 2 2024-08-17T17:05:39.104 INFO:tasks.rados.rados.0.smithi063.stdout:2513: writing smithi06387500-20 from 3217787 to 3638545 tid 3 2024-08-17T17:05:39.104 INFO:tasks.rados.rados.0.smithi063.stdout:2509: done (3 left) 2024-08-17T17:05:39.104 INFO:tasks.rados.rados.0.smithi063.stdout:2510: done (2 left) 2024-08-17T17:05:39.104 INFO:tasks.rados.rados.0.smithi063.stdout:2511: done (1 left) 2024-08-17T17:05:39.104 INFO:tasks.rados.rados.0.smithi063.stdout:2514: delete oid 18 current snap is 323 2024-08-17T17:05:39.105 INFO:tasks.rados.rados.0.smithi063.stdout:2513: finishing write tid 1 to smithi06387500-20 2024-08-17T17:05:39.105 INFO:tasks.rados.rados.0.smithi063.stdout:2513: finishing write tid 2 to smithi06387500-20 2024-08-17T17:05:39.106 INFO:tasks.rados.rados.0.smithi063.stdout:2513: finishing write tid 3 to smithi06387500-20 2024-08-17T17:05:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:2514: done (1 left) 2024-08-17T17:05:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:2515: delete oid 50 current snap is 323 2024-08-17T17:05:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:2513: finishing write tid 4 to smithi06387500-20 2024-08-17T17:05:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:2513: finishing write tid 5 to smithi06387500-20 2024-08-17T17:05:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:2513: oid 20 updating version 0 to 512 2024-08-17T17:05:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:2513: oid 20 version 512 is already newer than 510 2024-08-17T17:05:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:2513: oid 20 version 512 is already newer than 511 2024-08-17T17:05:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:2513: oid 20 version 512 is already newer than 509 2024-08-17T17:05:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 512 (ObjNum 945 snap 323 seq_num 945) dirty exists 2024-08-17T17:05:39.107 INFO:tasks.rados.rados.0.smithi063.stdout:2513: left oid 20 (ObjNum 945 snap 323 seq_num 945) 2024-08-17T17:05:39.109 INFO:tasks.rados.rados.0.smithi063.stdout:2513: done (1 left) 2024-08-17T17:05:39.109 INFO:tasks.rados.rados.0.smithi063.stdout:2515: done (0 left) 2024-08-17T17:05:39.109 INFO:tasks.rados.rados.0.smithi063.stdout:2516: write oid 36 current snap is 323 2024-08-17T17:05:39.109 INFO:tasks.rados.rados.0.smithi063.stdout:2516: seq_num 948 ranges {733036=563824,2074812=684986,3183703=749333} 2024-08-17T17:05:39.116 INFO:tasks.rados.rados.0.smithi063.stdout:2516: writing smithi06387500-36 from 733036 to 1296860 tid 1 2024-08-17T17:05:39.124 INFO:tasks.rados.rados.0.smithi063.stdout:2516: writing smithi06387500-36 from 2074812 to 2759798 tid 2 2024-08-17T17:05:39.130 INFO:tasks.rados.rados.0.smithi063.stdout:2516: writing smithi06387500-36 from 3183703 to 3933036 tid 3 2024-08-17T17:05:39.130 INFO:tasks.rados.rados.0.smithi063.stdout:2517: delete oid 13 current snap is 323 2024-08-17T17:05:39.131 INFO:tasks.rados.rados.0.smithi063.stdout:2516: finishing write tid 1 to smithi06387500-36 2024-08-17T17:05:39.131 INFO:tasks.rados.rados.0.smithi063.stdout:2516: finishing write tid 2 to smithi06387500-36 2024-08-17T17:05:39.133 INFO:tasks.rados.rados.0.smithi063.stdout:2517: done (1 left) 2024-08-17T17:05:39.133 INFO:tasks.rados.rados.0.smithi063.stdout:2518: read oid 45 snap -1 2024-08-17T17:05:39.133 INFO:tasks.rados.rados.0.smithi063.stdout:2518: expect deleted 2024-08-17T17:05:39.133 INFO:tasks.rados.rados.0.smithi063.stdout:2519: snap_remove snap 301 2024-08-17T17:05:39.189 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'echo wait for half of osds to upgrade' 2024-08-17T17:05:39.442 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:39.739 INFO:tasks.rados.rados.0.smithi063.stdout:2519: done (2 left) 2024-08-17T17:05:39.739 INFO:tasks.rados.rados.0.smithi063.stdout:2520: delete oid 34 current snap is 323 2024-08-17T17:05:39.740 INFO:tasks.rados.rados.0.smithi063.stdout:2516: finishing write tid 3 to smithi06387500-36 2024-08-17T17:05:39.740 INFO:tasks.rados.rados.0.smithi063.stdout:2516: finishing write tid 4 to smithi06387500-36 2024-08-17T17:05:39.740 INFO:tasks.rados.rados.0.smithi063.stdout:2516: finishing write tid 5 to smithi06387500-36 2024-08-17T17:05:39.740 INFO:tasks.rados.rados.0.smithi063.stdout:2516: oid 36 updating version 0 to 2488 2024-08-17T17:05:39.740 INFO:tasks.rados.rados.0.smithi063.stdout:2516: oid 36 version 2488 is already newer than 2486 2024-08-17T17:05:39.740 INFO:tasks.rados.rados.0.smithi063.stdout:2516: oid 36 updating version 2488 to 2489 2024-08-17T17:05:39.740 INFO:tasks.rados.rados.0.smithi063.stdout:2516: oid 36 version 2489 is already newer than 2487 2024-08-17T17:05:39.740 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 2489 (ObjNum 948 snap 323 seq_num 948) dirty exists 2024-08-17T17:05:39.740 INFO:tasks.rados.rados.0.smithi063.stdout:2516: left oid 36 (ObjNum 948 snap 323 seq_num 948) 2024-08-17T17:05:39.748 INFO:tasks.rados.rados.0.smithi063.stdout:2516: done (2 left) 2024-08-17T17:05:39.748 INFO:tasks.rados.rados.0.smithi063.stdout:2518: done (1 left) 2024-08-17T17:05:39.748 INFO:tasks.rados.rados.0.smithi063.stdout:2520: done (0 left) 2024-08-17T17:05:39.748 INFO:tasks.rados.rados.0.smithi063.stdout:2521: snap_create 2024-08-17T17:05:39.846 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:39 smithi063 ceph-mon[114756]: pgmap v522: 89 pgs: 89 active+clean; 235 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-08-17T17:05:39.846 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:39 smithi063 ceph-mon[114756]: osdmap e711: 8 total, 8 up, 8 in 2024-08-17T17:05:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:39 smithi162 ceph-mon[104668]: pgmap v522: 89 pgs: 89 active+clean; 235 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-08-17T17:05:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:39 smithi162 ceph-mon[104668]: osdmap e711: 8 total, 8 up, 8 in 2024-08-17T17:05:39.846 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:39 smithi063 ceph-mon[109843]: pgmap v522: 89 pgs: 89 active+clean; 235 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-08-17T17:05:39.847 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:39 smithi063 ceph-mon[109843]: osdmap e711: 8 total, 8 up, 8 in 2024-08-17T17:05:40.748 INFO:tasks.rados.rados.0.smithi063.stdout:2521: done (0 left) 2024-08-17T17:05:40.748 INFO:tasks.rados.rados.0.smithi063.stdout:2522: delete oid 11 current snap is 324 2024-08-17T17:05:40.751 INFO:tasks.rados.rados.0.smithi063.stdout:2522: done (0 left) 2024-08-17T17:05:40.751 INFO:tasks.rados.rados.0.smithi063.stdout:2523: read oid 13 snap -1 2024-08-17T17:05:40.751 INFO:tasks.rados.rados.0.smithi063.stdout:2523: expect deleted 2024-08-17T17:05:40.751 INFO:tasks.rados.rados.0.smithi063.stdout:2524: snap_create 2024-08-17T17:05:40.860 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:40 smithi063 ceph-mon[114756]: osdmap e712: 8 total, 8 up, 8 in 2024-08-17T17:05:40.860 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:40 smithi063 ceph-mon[114756]: pgmap v525: 89 pgs: 89 active+clean; 235 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-08-17T17:05:40.860 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:40 smithi063 ceph-mon[109843]: osdmap e712: 8 total, 8 up, 8 in 2024-08-17T17:05:40.861 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:40 smithi063 ceph-mon[109843]: pgmap v525: 89 pgs: 89 active+clean; 235 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-08-17T17:05:41.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:40 smithi162 ceph-mon[104668]: osdmap e712: 8 total, 8 up, 8 in 2024-08-17T17:05:41.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:40 smithi162 ceph-mon[104668]: pgmap v525: 89 pgs: 89 active+clean; 235 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-08-17T17:05:41.111 INFO:teuthology.orchestra.run.smithi063.stdout:wait for half of osds to upgrade 2024-08-17T17:05:41.741 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4' 2024-08-17T17:05:41.751 INFO:tasks.rados.rados.0.smithi063.stdout:2523: done (1 left) 2024-08-17T17:05:41.751 INFO:tasks.rados.rados.0.smithi063.stdout:2524: done (0 left) 2024-08-17T17:05:41.751 INFO:tasks.rados.rados.0.smithi063.stdout:2525: write oid 7 current snap is 325 2024-08-17T17:05:41.751 INFO:tasks.rados.rados.0.smithi063.stdout:2525: seq_num 952 ranges {400264=581173,1508740=491524} 2024-08-17T17:05:41.756 INFO:tasks.rados.rados.0.smithi063.stdout:2525: writing smithi06387500-7 from 400264 to 981437 tid 1 2024-08-17T17:05:41.763 INFO:tasks.rados.rados.0.smithi063.stdout:2525: writing smithi06387500-7 from 1508740 to 2000264 tid 2 2024-08-17T17:05:41.763 INFO:tasks.rados.rados.0.smithi063.stdout:2526: read oid 10 snap -1 2024-08-17T17:05:41.764 INFO:tasks.rados.rados.0.smithi063.stdout:2526: expect (ObjNum 933 snap 316 seq_num 933) 2024-08-17T17:05:41.764 INFO:tasks.rados.rados.0.smithi063.stdout:2527: snap_remove snap 322 2024-08-17T17:05:41.965 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:41 smithi063 ceph-mon[114756]: osdmap e713: 8 total, 8 up, 8 in 2024-08-17T17:05:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:41 smithi063 ceph-mon[109843]: osdmap e713: 8 total, 8 up, 8 in 2024-08-17T17:05:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:41 smithi162 ceph-mon[104668]: osdmap e713: 8 total, 8 up, 8 in 2024-08-17T17:05:42.759 INFO:tasks.rados.rados.0.smithi063.stdout:2527: done (2 left) 2024-08-17T17:05:42.759 INFO:tasks.rados.rados.0.smithi063.stdout:2528: snap_remove snap 273 2024-08-17T17:05:43.010 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:42 smithi162 ceph-mon[104668]: osdmap e714: 8 total, 8 up, 8 in 2024-08-17T17:05:43.010 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:42 smithi162 ceph-mon[104668]: pgmap v528: 89 pgs: 89 active+clean; 248 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.5 MiB/s wr, 19 op/s 2024-08-17T17:05:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:42 smithi063 ceph-mon[114756]: osdmap e714: 8 total, 8 up, 8 in 2024-08-17T17:05:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:42 smithi063 ceph-mon[114756]: pgmap v528: 89 pgs: 89 active+clean; 248 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.5 MiB/s wr, 19 op/s 2024-08-17T17:05:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:42 smithi063 ceph-mon[109843]: osdmap e714: 8 total, 8 up, 8 in 2024-08-17T17:05:43.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:42 smithi063 ceph-mon[109843]: pgmap v528: 89 pgs: 89 active+clean; 248 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.5 MiB/s wr, 19 op/s 2024-08-17T17:05:43.292 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:05:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:05:43] "GET /metrics HTTP/1.1" 200 36214 "" "Prometheus/2.51.0" 2024-08-17T17:05:43.765 INFO:tasks.rados.rados.0.smithi063.stdout:2528: done (2 left) 2024-08-17T17:05:43.765 INFO:tasks.rados.rados.0.smithi063.stdout:2529: write oid 34 current snap is 325 2024-08-17T17:05:43.765 INFO:tasks.rados.rados.0.smithi063.stdout:2529: seq_num 953 ranges {417071=656422,1567222=449849} 2024-08-17T17:05:43.771 INFO:tasks.rados.rados.0.smithi063.stdout:2529: writing smithi06387500-34 from 417071 to 1073493 tid 1 2024-08-17T17:05:43.778 INFO:tasks.rados.rados.0.smithi063.stdout:2529: writing smithi06387500-34 from 1567222 to 2017071 tid 2 2024-08-17T17:05:43.779 INFO:tasks.rados.rados.0.smithi063.stdout:2530: write oid 20 current snap is 325 2024-08-17T17:05:43.779 INFO:tasks.rados.rados.0.smithi063.stdout:2530: seq_num 954 ranges {433878=731671,1909351=124527} 2024-08-17T17:05:43.786 INFO:tasks.rados.rados.0.smithi063.stdout:2530: writing smithi06387500-20 from 433878 to 1165549 tid 1 2024-08-17T17:05:43.792 INFO:tasks.rados.rados.0.smithi063.stdout:2530: writing smithi06387500-20 from 1909351 to 2033878 tid 2 2024-08-17T17:05:43.792 INFO:tasks.rados.rados.0.smithi063.stdout:2531: read oid 48 snap -1 2024-08-17T17:05:43.793 INFO:tasks.rados.rados.0.smithi063.stdout:2531: expect (ObjNum 857 snap 292 seq_num 857) 2024-08-17T17:05:43.793 INFO:tasks.rados.rados.0.smithi063.stdout:2532: snap_remove snap 312 2024-08-17T17:05:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:43 smithi063 ceph-mon[114756]: osdmap e715: 8 total, 8 up, 8 in 2024-08-17T17:05:44.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:43 smithi063 ceph-mon[109843]: osdmap e715: 8 total, 8 up, 8 in 2024-08-17T17:05:44.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:43 smithi162 ceph-mon[104668]: osdmap e715: 8 total, 8 up, 8 in 2024-08-17T17:05:44.783 INFO:tasks.rados.rados.0.smithi063.stdout:2525: finishing write tid 1 to smithi06387500-7 2024-08-17T17:05:44.784 INFO:tasks.rados.rados.0.smithi063.stdout:2532: done (5 left) 2024-08-17T17:05:44.784 INFO:tasks.rados.rados.0.smithi063.stdout:2533: write oid 16 current snap is 325 2024-08-17T17:05:44.784 INFO:tasks.rados.rados.0.smithi063.stdout:2533: seq_num 955 ranges {450685=406920,1567833=482852} 2024-08-17T17:05:44.788 INFO:tasks.rados.rados.0.smithi063.stdout:2533: writing smithi06387500-16 from 450685 to 857605 tid 1 2024-08-17T17:05:44.795 INFO:tasks.rados.rados.0.smithi063.stdout:2533: writing smithi06387500-16 from 1567833 to 2050685 tid 2 2024-08-17T17:05:44.795 INFO:tasks.rados.rados.0.smithi063.stdout:2534: write oid 30 current snap is 325 2024-08-17T17:05:44.795 INFO:tasks.rados.rados.0.smithi063.stdout:2534: seq_num 956 ranges {467492=482169,1626315=441177} 2024-08-17T17:05:44.800 INFO:tasks.rados.rados.0.smithi063.stdout:2534: writing smithi06387500-30 from 467492 to 949661 tid 1 2024-08-17T17:05:44.807 INFO:tasks.rados.rados.0.smithi063.stdout:2534: writing smithi06387500-30 from 1626315 to 2067492 tid 2 2024-08-17T17:05:44.807 INFO:tasks.rados.rados.0.smithi063.stdout:2535: snap_remove snap 318 2024-08-17T17:05:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:44 smithi063 ceph-mon[114756]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:44 smithi063 ceph-mon[114756]: osdmap e716: 8 total, 8 up, 8 in 2024-08-17T17:05:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:44 smithi063 ceph-mon[114756]: pgmap v531: 89 pgs: 89 active+clean; 248 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.5 MiB/s wr, 19 op/s 2024-08-17T17:05:45.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:44 smithi063 ceph-mon[109843]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:45.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:44 smithi063 ceph-mon[109843]: osdmap e716: 8 total, 8 up, 8 in 2024-08-17T17:05:45.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:44 smithi063 ceph-mon[109843]: pgmap v531: 89 pgs: 89 active+clean; 248 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.5 MiB/s wr, 19 op/s 2024-08-17T17:05:45.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:44 smithi162 ceph-mon[104668]: from='client.44222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:45.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:44 smithi162 ceph-mon[104668]: osdmap e716: 8 total, 8 up, 8 in 2024-08-17T17:05:45.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:44 smithi162 ceph-mon[104668]: pgmap v531: 89 pgs: 89 active+clean; 248 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.5 MiB/s wr, 19 op/s 2024-08-17T17:05:45.537 INFO:teuthology.orchestra.run.smithi063.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:05:45.783 INFO:tasks.rados.rados.0.smithi063.stdout:2535: done (7 left) 2024-08-17T17:05:45.783 INFO:tasks.rados.rados.0.smithi063.stdout:2536: read oid 37 snap -1 2024-08-17T17:05:45.784 INFO:tasks.rados.rados.0.smithi063.stdout:2536: expect (ObjNum 696 snap 222 seq_num 696) 2024-08-17T17:05:45.784 INFO:tasks.rados.rados.0.smithi063.stdout:2537: read oid 1 snap 305 2024-08-17T17:05:45.784 INFO:tasks.rados.rados.0.smithi063.stdout:2537: expect (ObjNum 876 snap 305 seq_num 876) 2024-08-17T17:05:45.784 INFO:tasks.rados.rados.0.smithi063.stdout:2538: write oid 43 current snap is 325 2024-08-17T17:05:45.784 INFO:tasks.rados.rados.0.smithi063.stdout:2538: seq_num 957 ranges {484299=557418,1684797=399502} 2024-08-17T17:05:45.789 INFO:tasks.rados.rados.0.smithi063.stdout:2538: writing smithi06387500-43 from 484299 to 1041717 tid 1 2024-08-17T17:05:45.795 INFO:tasks.rados.rados.0.smithi063.stdout:2538: writing smithi06387500-43 from 1684797 to 2084299 tid 2 2024-08-17T17:05:45.796 INFO:tasks.rados.rados.0.smithi063.stdout:2525: finishing write tid 2 to smithi06387500-7 2024-08-17T17:05:45.796 INFO:tasks.rados.rados.0.smithi063.stdout:2525: finishing write tid 3 to smithi06387500-7 2024-08-17T17:05:45.796 INFO:tasks.rados.rados.0.smithi063.stdout:2525: finishing write tid 4 to smithi06387500-7 2024-08-17T17:05:45.796 INFO:tasks.rados.rados.0.smithi063.stdout:2525: oid 7 updating version 0 to 1140 2024-08-17T17:05:45.796 INFO:tasks.rados.rados.0.smithi063.stdout:2525: oid 7 updating version 1140 to 1141 2024-08-17T17:05:45.796 INFO:tasks.rados.rados.0.smithi063.stdout:2525: oid 7 updating version 1141 to 1142 2024-08-17T17:05:45.796 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1142 (ObjNum 952 snap 325 seq_num 952) dirty exists 2024-08-17T17:05:45.796 INFO:tasks.rados.rados.0.smithi063.stdout:2525: left oid 7 (ObjNum 952 snap 325 seq_num 952) 2024-08-17T17:05:45.797 INFO:tasks.rados.rados.0.smithi063.stdout:2526: expect (ObjNum 933 snap 316 seq_num 933) 2024-08-17T17:05:45.905 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:45 smithi063 ceph-mon[114756]: osdmap e717: 8 total, 8 up, 8 in 2024-08-17T17:05:45.905 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:45 smithi063 ceph-mon[114756]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:05:45.905 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:45.905 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:45.905 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:45.906 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:05:45.906 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:45.906 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:45 smithi063 ceph-mon[109843]: osdmap e717: 8 total, 8 up, 8 in 2024-08-17T17:05:45.906 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:45 smithi063 ceph-mon[109843]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:05:45.906 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:45.906 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:45.906 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:45.906 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:05:45.906 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:46.009 INFO:tasks.rados.rados.0.smithi063.stdout:2529: finishing write tid 1 to smithi06387500-34 2024-08-17T17:05:46.009 INFO:tasks.rados.rados.0.smithi063.stdout:2529: finishing write tid 2 to smithi06387500-34 2024-08-17T17:05:46.009 INFO:tasks.rados.rados.0.smithi063.stdout:2529: finishing write tid 3 to smithi06387500-34 2024-08-17T17:05:46.009 INFO:tasks.rados.rados.0.smithi063.stdout:2529: finishing write tid 4 to smithi06387500-34 2024-08-17T17:05:46.009 INFO:tasks.rados.rados.0.smithi063.stdout:2529: oid 34 updating version 0 to 1690 2024-08-17T17:05:46.009 INFO:tasks.rados.rados.0.smithi063.stdout:2529: oid 34 version 1690 is already newer than 1689 2024-08-17T17:05:46.009 INFO:tasks.rados.rados.0.smithi063.stdout:2529: oid 34 updating version 1690 to 1691 2024-08-17T17:05:46.010 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 1691 (ObjNum 953 snap 325 seq_num 953) dirty exists 2024-08-17T17:05:46.010 INFO:tasks.rados.rados.0.smithi063.stdout:2529: left oid 34 (ObjNum 953 snap 325 seq_num 953) 2024-08-17T17:05:46.010 INFO:tasks.rados.rados.0.smithi063.stdout:2530: finishing write tid 1 to smithi06387500-20 2024-08-17T17:05:46.010 INFO:tasks.rados.rados.0.smithi063.stdout:2530: finishing write tid 2 to smithi06387500-20 2024-08-17T17:05:46.010 INFO:tasks.rados.rados.0.smithi063.stdout:2530: finishing write tid 3 to smithi06387500-20 2024-08-17T17:05:46.010 INFO:tasks.rados.rados.0.smithi063.stdout:2525: done (9 left) 2024-08-17T17:05:46.010 INFO:tasks.rados.rados.0.smithi063.stdout:2526: done (8 left) 2024-08-17T17:05:46.010 INFO:tasks.rados.rados.0.smithi063.stdout:2529: done (7 left) 2024-08-17T17:05:46.010 INFO:tasks.rados.rados.0.smithi063.stdout:2539: write oid 17 current snap is 325 2024-08-17T17:05:46.010 INFO:tasks.rados.rados.0.smithi063.stdout:2530: finishing write tid 4 to smithi06387500-20 2024-08-17T17:05:46.010 INFO:tasks.rados.rados.0.smithi063.stdout:2530: oid 20 updating version 0 to 522 2024-08-17T17:05:46.010 INFO:tasks.rados.rados.0.smithi063.stdout:2530: oid 20 updating version 522 to 524 2024-08-17T17:05:46.011 INFO:tasks.rados.rados.0.smithi063.stdout:2530: oid 20 version 524 is already newer than 523 2024-08-17T17:05:46.011 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 524 (ObjNum 954 snap 325 seq_num 954) dirty exists 2024-08-17T17:05:46.011 INFO:tasks.rados.rados.0.smithi063.stdout:2530: left oid 20 (ObjNum 954 snap 325 seq_num 954) 2024-08-17T17:05:46.011 INFO:tasks.rados.rados.0.smithi063.stdout:2531: expect (ObjNum 857 snap 292 seq_num 857) 2024-08-17T17:05:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:45 smithi162 ceph-mon[104668]: osdmap e717: 8 total, 8 up, 8 in 2024-08-17T17:05:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:45 smithi162 ceph-mon[104668]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:05:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:05:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:46.153 INFO:tasks.rados.rados.0.smithi063.stdout:2533: finishing write tid 1 to smithi06387500-16 2024-08-17T17:05:46.153 INFO:tasks.rados.rados.0.smithi063.stdout:2533: finishing write tid 2 to smithi06387500-16 2024-08-17T17:05:46.153 INFO:tasks.rados.rados.0.smithi063.stdout:2533: finishing write tid 3 to smithi06387500-16 2024-08-17T17:05:46.153 INFO:tasks.rados.rados.0.smithi063.stdout:2533: finishing write tid 4 to smithi06387500-16 2024-08-17T17:05:46.154 INFO:tasks.rados.rados.0.smithi063.stdout:2533: oid 16 updating version 0 to 951 2024-08-17T17:05:46.154 INFO:tasks.rados.rados.0.smithi063.stdout:2533: oid 16 version 951 is already newer than 950 2024-08-17T17:05:46.154 INFO:tasks.rados.rados.0.smithi063.stdout:2533: oid 16 version 951 is already newer than 949 2024-08-17T17:05:46.154 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 951 (ObjNum 955 snap 325 seq_num 955) dirty exists 2024-08-17T17:05:46.154 INFO:tasks.rados.rados.0.smithi063.stdout:2533: left oid 16 (ObjNum 955 snap 325 seq_num 955) 2024-08-17T17:05:46.154 INFO:tasks.rados.rados.0.smithi063.stdout:2534: finishing write tid 1 to smithi06387500-30 2024-08-17T17:05:46.154 INFO:tasks.rados.rados.0.smithi063.stdout:2534: finishing write tid 2 to smithi06387500-30 2024-08-17T17:05:46.154 INFO:tasks.rados.rados.0.smithi063.stdout:2534: finishing write tid 3 to smithi06387500-30 2024-08-17T17:05:46.154 INFO:tasks.rados.rados.0.smithi063.stdout:2534: finishing write tid 4 to smithi06387500-30 2024-08-17T17:05:46.154 INFO:tasks.rados.rados.0.smithi063.stdout:2534: oid 30 updating version 0 to 1162 2024-08-17T17:05:46.154 INFO:tasks.rados.rados.0.smithi063.stdout:2534: oid 30 version 1162 is already newer than 1161 2024-08-17T17:05:46.155 INFO:tasks.rados.rados.0.smithi063.stdout:2534: oid 30 version 1162 is already newer than 1160 2024-08-17T17:05:46.155 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1162 (ObjNum 956 snap 325 seq_num 956) dirty exists 2024-08-17T17:05:46.155 INFO:tasks.rados.rados.0.smithi063.stdout:2534: left oid 30 (ObjNum 956 snap 325 seq_num 956) 2024-08-17T17:05:46.155 INFO:tasks.rados.rados.0.smithi063.stdout:2537: expect (ObjNum 876 snap 305 seq_num 876) 2024-08-17T17:05:46.234 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -e sha1=cd8bdbeb9fbdab5733aa3aaadc042c965473e453 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-08-17T17:05:46.247 INFO:tasks.rados.rados.0.smithi063.stdout:2538: finishing write tid 1 to smithi06387500-43 2024-08-17T17:05:46.247 INFO:tasks.rados.rados.0.smithi063.stdout:2538: finishing write tid 2 to smithi06387500-43 2024-08-17T17:05:46.247 INFO:tasks.rados.rados.0.smithi063.stdout:2539: seq_num 958 ranges {501106=749020,1743279=357827} 2024-08-17T17:05:46.254 INFO:tasks.rados.rados.0.smithi063.stdout:2539: writing smithi06387500-17 from 501106 to 1250126 tid 1 2024-08-17T17:05:46.259 INFO:tasks.rados.rados.0.smithi063.stdout:2539: writing smithi06387500-17 from 1743279 to 2101106 tid 2 2024-08-17T17:05:46.259 INFO:tasks.rados.rados.0.smithi063.stdout:2530: done (7 left) 2024-08-17T17:05:46.259 INFO:tasks.rados.rados.0.smithi063.stdout:2531: done (6 left) 2024-08-17T17:05:46.259 INFO:tasks.rados.rados.0.smithi063.stdout:2533: done (5 left) 2024-08-17T17:05:46.259 INFO:tasks.rados.rados.0.smithi063.stdout:2534: done (4 left) 2024-08-17T17:05:46.259 INFO:tasks.rados.rados.0.smithi063.stdout:2537: done (3 left) 2024-08-17T17:05:46.259 INFO:tasks.rados.rados.0.smithi063.stdout:2540: delete oid 40 current snap is 325 2024-08-17T17:05:46.259 INFO:tasks.rados.rados.0.smithi063.stdout:2538: finishing write tid 3 to smithi06387500-43 2024-08-17T17:05:46.260 INFO:tasks.rados.rados.0.smithi063.stdout:2538: finishing write tid 4 to smithi06387500-43 2024-08-17T17:05:46.260 INFO:tasks.rados.rados.0.smithi063.stdout:2538: oid 43 updating version 0 to 1170 2024-08-17T17:05:46.260 INFO:tasks.rados.rados.0.smithi063.stdout:2538: oid 43 version 1170 is already newer than 1169 2024-08-17T17:05:46.260 INFO:tasks.rados.rados.0.smithi063.stdout:2538: oid 43 version 1170 is already newer than 1168 2024-08-17T17:05:46.260 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 1170 (ObjNum 957 snap 325 seq_num 957) dirty exists 2024-08-17T17:05:46.260 INFO:tasks.rados.rados.0.smithi063.stdout:2538: left oid 43 (ObjNum 957 snap 325 seq_num 957) 2024-08-17T17:05:46.260 INFO:tasks.rados.rados.0.smithi063.stdout:2536: expect (ObjNum 696 snap 222 seq_num 696) 2024-08-17T17:05:46.469 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:05:46.501 INFO:tasks.rados.rados.0.smithi063.stdout:2539: finishing write tid 1 to smithi06387500-17 2024-08-17T17:05:46.501 INFO:tasks.rados.rados.0.smithi063.stdout:2539: finishing write tid 2 to smithi06387500-17 2024-08-17T17:05:46.501 INFO:tasks.rados.rados.0.smithi063.stdout:2539: finishing write tid 3 to smithi06387500-17 2024-08-17T17:05:46.501 INFO:tasks.rados.rados.0.smithi063.stdout:2539: finishing write tid 4 to smithi06387500-17 2024-08-17T17:05:46.501 INFO:tasks.rados.rados.0.smithi063.stdout:2539: oid 17 updating version 0 to 822 2024-08-17T17:05:46.501 INFO:tasks.rados.rados.0.smithi063.stdout:2539: oid 17 version 822 is already newer than 821 2024-08-17T17:05:46.501 INFO:tasks.rados.rados.0.smithi063.stdout:2539: oid 17 version 822 is already newer than 820 2024-08-17T17:05:46.501 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 822 (ObjNum 958 snap 325 seq_num 958) dirty exists 2024-08-17T17:05:46.502 INFO:tasks.rados.rados.0.smithi063.stdout:2539: left oid 17 (ObjNum 958 snap 325 seq_num 958) 2024-08-17T17:05:46.502 INFO:tasks.rados.rados.0.smithi063.stdout:2536: done (3 left) 2024-08-17T17:05:46.502 INFO:tasks.rados.rados.0.smithi063.stdout:2538: done (2 left) 2024-08-17T17:05:46.502 INFO:tasks.rados.rados.0.smithi063.stdout:2539: done (1 left) 2024-08-17T17:05:46.502 INFO:tasks.rados.rados.0.smithi063.stdout:2540: done (0 left) 2024-08-17T17:05:46.502 INFO:tasks.rados.rados.0.smithi063.stdout:2541: read oid 5 snap 307 2024-08-17T17:05:46.502 INFO:tasks.rados.rados.0.smithi063.stdout:2541: expect (ObjNum 842 snap 290 seq_num 842) 2024-08-17T17:05:46.502 INFO:tasks.rados.rados.0.smithi063.stdout:2542: delete oid 1 current snap is 325 2024-08-17T17:05:46.509 INFO:tasks.rados.rados.0.smithi063.stdout:2541: expect (ObjNum 842 snap 290 seq_num 842) 2024-08-17T17:05:46.580 INFO:tasks.rados.rados.0.smithi063.stdout:2541: done (1 left) 2024-08-17T17:05:46.580 INFO:tasks.rados.rados.0.smithi063.stdout:2542: done (0 left) 2024-08-17T17:05:46.581 INFO:tasks.rados.rados.0.smithi063.stdout:2543: read oid 50 snap 321 2024-08-17T17:05:46.581 INFO:tasks.rados.rados.0.smithi063.stdout:2543: expect deleted 2024-08-17T17:05:46.581 INFO:tasks.rados.rados.0.smithi063.stdout:2544: snap_remove snap 304 2024-08-17T17:05:46.797 INFO:tasks.rados.rados.0.smithi063.stdout:2544: done (1 left) 2024-08-17T17:05:46.797 INFO:tasks.rados.rados.0.smithi063.stdout:2545: snap_create 2024-08-17T17:05:46.861 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:46 smithi063 ceph-mon[114756]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:05:46.861 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:46 smithi063 ceph-mon[114756]: osdmap e718: 8 total, 8 up, 8 in 2024-08-17T17:05:46.861 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:46 smithi063 ceph-mon[114756]: pgmap v534: 89 pgs: 89 active+clean; 248 MiB data, 826 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:05:46.861 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:46.862 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:46 smithi063 ceph-mon[109843]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:05:46.862 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:46 smithi063 ceph-mon[109843]: osdmap e718: 8 total, 8 up, 8 in 2024-08-17T17:05:46.862 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:46 smithi063 ceph-mon[109843]: pgmap v534: 89 pgs: 89 active+clean; 248 MiB data, 826 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:05:46.862 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:46 smithi162 ceph-mon[104668]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:cd8bdbeb9fbdab5733aa3aaadc042c965473e453 2024-08-17T17:05:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:46 smithi162 ceph-mon[104668]: osdmap e718: 8 total, 8 up, 8 in 2024-08-17T17:05:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:46 smithi162 ceph-mon[104668]: pgmap v534: 89 pgs: 89 active+clean; 248 MiB data, 826 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:05:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:05:47.594 INFO:tasks.rados.rados.0.smithi063.stdout:2543: done (1 left) 2024-08-17T17:05:47.594 INFO:tasks.rados.rados.0.smithi063.stdout:2545: done (0 left) 2024-08-17T17:05:47.594 INFO:tasks.rados.rados.0.smithi063.stdout:2546: snap_create 2024-08-17T17:05:47.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:47 smithi063 ceph-mon[114756]: osdmap e719: 8 total, 8 up, 8 in 2024-08-17T17:05:47.910 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:47 smithi063 ceph-mon[114756]: osdmap e720: 8 total, 8 up, 8 in 2024-08-17T17:05:47.910 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:47 smithi063 ceph-mon[109843]: osdmap e719: 8 total, 8 up, 8 in 2024-08-17T17:05:47.911 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:47 smithi063 ceph-mon[109843]: osdmap e720: 8 total, 8 up, 8 in 2024-08-17T17:05:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:47 smithi162 ceph-mon[104668]: osdmap e719: 8 total, 8 up, 8 in 2024-08-17T17:05:48.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:47 smithi162 ceph-mon[104668]: osdmap e720: 8 total, 8 up, 8 in 2024-08-17T17:05:48.441 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T17:05:48.594 INFO:tasks.rados.rados.0.smithi063.stdout:2546: done (0 left) 2024-08-17T17:05:48.595 INFO:tasks.rados.rados.0.smithi063.stdout:2547: snap_remove snap 305 2024-08-17T17:05:49.032 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:05:49.032 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (7m) 39s ago 13m 18.9M - 0.27.0 11f11916f8cd 59f4018b91ca 2024-08-17T17:05:49.032 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (6m) 39s ago 13m 78.2M - c8b91775d855 581e03e654a8 2024-08-17T17:05:49.032 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (12m) 39s ago 11m 22.4M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T17:05:49.032 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (11m) 39s ago 11m 19.1M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T17:05:49.033 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,9283,8765 running (8m) 39s ago 17m 456M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb da48034a2895 2024-08-17T17:05:49.033 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:8443,9283,8765 running (8m) 39s ago 18m 516M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 943e639cede6 2024-08-17T17:05:49.033 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (6m) 39s ago 18m 62.4M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb f0f20fcac956 2024-08-17T17:05:49.033 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (3m) 39s ago 17m 39.9M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 520525c79dbe 2024-08-17T17:05:49.033 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (5m) 39s ago 17m 50.0M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb c484af026acf 2024-08-17T17:05:49.033 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 running (8m) 39s ago 12m 11.0M - 1.7.0 72c9c2088986 34f55f5de2aa 2024-08-17T17:05:49.033 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (7m) 39s ago 12m 11.6M - 1.7.0 72c9c2088986 5bb162d0f1a9 2024-08-17T17:05:49.033 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (16m) 39s ago 16m 118M 2480M 18.2.4-719-gced649f8 c870753d67c2 f3b910ed49b8 2024-08-17T17:05:49.033 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (16m) 39s ago 16m 94.7M 2480M 18.2.4-719-gced649f8 c870753d67c2 5d6fdb6000ad 2024-08-17T17:05:49.033 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (16m) 39s ago 16m 105M 2480M 18.2.4-719-gced649f8 c870753d67c2 a3cbe8edb49a 2024-08-17T17:05:49.033 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (15m) 39s ago 15m 88.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T17:05:49.033 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (15m) 39s ago 15m 121M 2480M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T17:05:49.033 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (14m) 39s ago 14m 115M 2480M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T17:05:49.034 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (14m) 39s ago 14m 85.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T17:05:49.034 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (14m) 39s ago 14m 94.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T17:05:49.034 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (7m) 39s ago 13m 48.0M - 2.51.0 1d3b7f56885b dfe66b2c5568 2024-08-17T17:05:49.392 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:05:49.392 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T17:05:49.392 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:05:49.392 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:05:49.393 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T17:05:49.393 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:05:49.393 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:05:49.393 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T17:05:49.393 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8 2024-08-17T17:05:49.393 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:05:49.393 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T17:05:49.393 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:05:49.393 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:05:49.393 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T17:05:49.393 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 10, 2024-08-17T17:05:49.393 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 5 2024-08-17T17:05:49.393 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:05:49.393 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:05:49.595 INFO:tasks.rados.rados.0.smithi063.stdout:2547: done (0 left) 2024-08-17T17:05:49.595 INFO:tasks.rados.rados.0.smithi063.stdout:2548: delete oid 4 current snap is 327 2024-08-17T17:05:49.606 INFO:tasks.rados.rados.0.smithi063.stdout:2548: done (0 left) 2024-08-17T17:05:49.607 INFO:tasks.rados.rados.0.smithi063.stdout:2549: snap_remove snap 294 2024-08-17T17:05:49.711 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:05:49.711 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:05:49.711 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-08-17T17:05:49.711 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (4 remaining).", 2024-08-17T17:05:49.711 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-08-17T17:05:49.711 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "0/8 daemons upgraded", 2024-08-17T17:05:49.711 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading osd daemons", 2024-08-17T17:05:49.711 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T17:05:49.711 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:05:49.887 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[109843]: pgmap v537: 89 pgs: 89 active+clean; 201 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.4 MiB/s wr, 15 op/s 2024-08-17T17:05:49.887 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[109843]: from='client.44228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:49.887 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[109843]: osdmap e721: 8 total, 8 up, 8 in 2024-08-17T17:05:49.887 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:49.887 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:49.888 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:05:49.888 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:49.889 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:05:49.889 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:49.889 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:05:49.889 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:49.889 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-08-17T17:05:49.889 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1362919688' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:05:49.890 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[114756]: pgmap v537: 89 pgs: 89 active+clean; 201 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.4 MiB/s wr, 15 op/s 2024-08-17T17:05:49.890 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[114756]: from='client.44228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:49.890 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[114756]: osdmap e721: 8 total, 8 up, 8 in 2024-08-17T17:05:49.890 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:49.890 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:49.890 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:05:49.890 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:49.890 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:05:49.890 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:49.890 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:05:49.891 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:49.891 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-08-17T17:05:49.891 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:49 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1362919688' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:05:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:49 smithi162 ceph-mon[104668]: pgmap v537: 89 pgs: 89 active+clean; 201 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.4 MiB/s wr, 15 op/s 2024-08-17T17:05:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:49 smithi162 ceph-mon[104668]: from='client.44228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:49 smithi162 ceph-mon[104668]: osdmap e721: 8 total, 8 up, 8 in 2024-08-17T17:05:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:49 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:49 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:05:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:49 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:05:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:49 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:50.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:49 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:05:50.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:49 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:50.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:49 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:05:50.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:49 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:50.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:49 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-08-17T17:05:50.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:49 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1362919688' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:05:50.614 INFO:tasks.rados.rados.0.smithi063.stdout:2549: done (0 left) 2024-08-17T17:05:50.615 INFO:tasks.rados.rados.0.smithi063.stdout:2550: rollback oid 35 current snap is 327 2024-08-17T17:05:50.615 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 35 to 324 2024-08-17T17:05:50.615 INFO:tasks.rados.rados.0.smithi063.stdout:2551: read oid 29 snap -1 2024-08-17T17:05:50.615 INFO:tasks.rados.rados.0.smithi063.stdout:2551: expect (ObjNum 855 snap 290 seq_num 855) 2024-08-17T17:05:50.615 INFO:tasks.rados.rados.0.smithi063.stdout:2552: write oid 44 current snap is 327 2024-08-17T17:05:50.615 INFO:tasks.rados.rados.0.smithi063.stdout:2552: seq_num 962 ranges {568334=650016,1860854=307480} 2024-08-17T17:05:50.622 INFO:tasks.rados.rados.0.smithi063.stdout:2552: writing smithi06387500-44 from 568334 to 1218350 tid 1 2024-08-17T17:05:50.629 INFO:tasks.rados.rados.0.smithi063.stdout:2552: writing smithi06387500-44 from 1860854 to 2168334 tid 2 2024-08-17T17:05:50.629 INFO:tasks.rados.rados.0.smithi063.stdout:2553: write oid 7 current snap is 327 2024-08-17T17:05:50.629 INFO:tasks.rados.rados.0.smithi063.stdout:2553: seq_num 963 ranges {585141=725265,1919336=265805} 2024-08-17T17:05:50.637 INFO:tasks.rados.rados.0.smithi063.stdout:2553: writing smithi06387500-7 from 585141 to 1310406 tid 1 2024-08-17T17:05:50.642 INFO:tasks.rados.rados.0.smithi063.stdout:2553: writing smithi06387500-7 from 1919336 to 2185141 tid 2 2024-08-17T17:05:50.642 INFO:tasks.rados.rados.0.smithi063.stdout:2551: expect (ObjNum 855 snap 290 seq_num 855) 2024-08-17T17:05:50.760 INFO:tasks.rados.rados.0.smithi063.stdout:2552: finishing write tid 1 to smithi06387500-44 2024-08-17T17:05:50.760 INFO:tasks.rados.rados.0.smithi063.stdout:2550: finishing rollback tid 0 to smithi06387500-35 2024-08-17T17:05:50.760 INFO:tasks.rados.rados.0.smithi063.stdout:2550: finishing rollback tid 1 to smithi06387500-35 2024-08-17T17:05:50.760 INFO:tasks.rados.rados.0.smithi063.stdout:2550: finishing rollback tid 2 to smithi06387500-35 2024-08-17T17:05:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 2538 (ObjNum 941 snap 323 seq_num 941) dirty exists 2024-08-17T17:05:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:2552: finishing write tid 2 to smithi06387500-44 2024-08-17T17:05:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:2550: done (3 left) 2024-08-17T17:05:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:2551: done (2 left) 2024-08-17T17:05:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:2554: delete oid 17 current snap is 327 2024-08-17T17:05:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:2552: finishing write tid 3 to smithi06387500-44 2024-08-17T17:05:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:2552: finishing write tid 4 to smithi06387500-44 2024-08-17T17:05:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:2552: oid 44 updating version 0 to 357 2024-08-17T17:05:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:2552: oid 44 updating version 357 to 359 2024-08-17T17:05:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:2552: oid 44 version 359 is already newer than 358 2024-08-17T17:05:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 359 (ObjNum 962 snap 327 seq_num 962) dirty exists 2024-08-17T17:05:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:2552: left oid 44 (ObjNum 962 snap 327 seq_num 962) 2024-08-17T17:05:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:2553: finishing write tid 1 to smithi06387500-7 2024-08-17T17:05:50.761 INFO:tasks.rados.rados.0.smithi063.stdout:2553: finishing write tid 2 to smithi06387500-7 2024-08-17T17:05:50.762 INFO:tasks.rados.rados.0.smithi063.stdout:2553: finishing write tid 3 to smithi06387500-7 2024-08-17T17:05:50.762 INFO:tasks.rados.rados.0.smithi063.stdout:2553: finishing write tid 4 to smithi06387500-7 2024-08-17T17:05:50.762 INFO:tasks.rados.rados.0.smithi063.stdout:2553: oid 7 updating version 0 to 1192 2024-08-17T17:05:50.762 INFO:tasks.rados.rados.0.smithi063.stdout:2553: oid 7 updating version 1192 to 1193 2024-08-17T17:05:50.762 INFO:tasks.rados.rados.0.smithi063.stdout:2553: oid 7 updating version 1193 to 1194 2024-08-17T17:05:50.762 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1194 (ObjNum 963 snap 327 seq_num 963) dirty exists 2024-08-17T17:05:50.762 INFO:tasks.rados.rados.0.smithi063.stdout:2553: left oid 7 (ObjNum 963 snap 327 seq_num 963) 2024-08-17T17:05:50.762 INFO:tasks.rados.rados.0.smithi063.stdout:2552: done (2 left) 2024-08-17T17:05:50.762 INFO:tasks.rados.rados.0.smithi063.stdout:2553: done (1 left) 2024-08-17T17:05:50.762 INFO:tasks.rados.rados.0.smithi063.stdout:2554: done (0 left) 2024-08-17T17:05:50.763 INFO:tasks.rados.rados.0.smithi063.stdout:2555: snap_create 2024-08-17T17:05:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[114756]: from='client.34305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[114756]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[114756]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:05:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[114756]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:05:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all mgr 2024-08-17T17:05:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all mon 2024-08-17T17:05:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all crash 2024-08-17T17:05:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[114756]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-17T17:05:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[114756]: Upgrade: osd.0 is safe to restart 2024-08-17T17:05:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[114756]: osdmap e722: 8 total, 8 up, 8 in 2024-08-17T17:05:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[109843]: from='client.34305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[109843]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[109843]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:05:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[109843]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:05:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all mgr 2024-08-17T17:05:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all mon 2024-08-17T17:05:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all crash 2024-08-17T17:05:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[109843]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-17T17:05:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[109843]: Upgrade: osd.0 is safe to restart 2024-08-17T17:05:51.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:50 smithi063 ceph-mon[109843]: osdmap e722: 8 total, 8 up, 8 in 2024-08-17T17:05:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:50 smithi162 ceph-mon[104668]: from='client.34305 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:50 smithi162 ceph-mon[104668]: from='client.34311 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:50 smithi162 ceph-mon[104668]: Upgrade: Target is version 19.0.0-5836-gcd8bdbeb (squid) 2024-08-17T17:05:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:50 smithi162 ceph-mon[104668]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d'] 2024-08-17T17:05:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:50 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all mgr 2024-08-17T17:05:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:50 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all mon 2024-08-17T17:05:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:50 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all crash 2024-08-17T17:05:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:50 smithi162 ceph-mon[104668]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-08-17T17:05:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:50 smithi162 ceph-mon[104668]: Upgrade: osd.0 is safe to restart 2024-08-17T17:05:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:50 smithi162 ceph-mon[104668]: osdmap e722: 8 total, 8 up, 8 in 2024-08-17T17:05:51.659 INFO:tasks.rados.rados.0.smithi063.stdout:2555: done (0 left) 2024-08-17T17:05:51.659 INFO:tasks.rados.rados.0.smithi063.stdout:2556: write oid 4 current snap is 328 2024-08-17T17:05:51.660 INFO:tasks.rados.rados.0.smithi063.stdout:2556: seq_num 965 ranges {618755=475763,1636300=457996} 2024-08-17T17:05:51.665 INFO:tasks.rados.rados.0.smithi063.stdout:2556: writing smithi06387500-4 from 618755 to 1094518 tid 1 2024-08-17T17:05:51.671 INFO:tasks.rados.rados.0.smithi063.stdout:2556: writing smithi06387500-4 from 1636300 to 2094296 tid 2 2024-08-17T17:05:51.671 INFO:tasks.rados.rados.0.smithi063.stdout:2557: snap_remove snap 324 2024-08-17T17:05:51.981 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:51 smithi063 ceph-mon[114756]: from='client.44252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:51.981 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:51 smithi063 ceph-mon[114756]: pgmap v540: 89 pgs: 89 active+clean; 201 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.4 MiB/s wr, 15 op/s 2024-08-17T17:05:51.981 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:51 smithi063 ceph-mon[114756]: osdmap e723: 8 total, 8 up, 8 in 2024-08-17T17:05:51.981 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:51 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:51.981 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:51 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:05:51.981 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:51 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:51.982 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:51 smithi063 ceph-mon[109843]: from='client.44252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:51.982 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:51 smithi063 ceph-mon[109843]: pgmap v540: 89 pgs: 89 active+clean; 201 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.4 MiB/s wr, 15 op/s 2024-08-17T17:05:51.982 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:51 smithi063 ceph-mon[109843]: osdmap e723: 8 total, 8 up, 8 in 2024-08-17T17:05:51.982 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:51 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:51.982 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:51 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:05:51.982 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:51 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:52.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:51 smithi162 ceph-mon[104668]: from='client.44252 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:05:52.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:51 smithi162 ceph-mon[104668]: pgmap v540: 89 pgs: 89 active+clean; 201 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 1.4 MiB/s wr, 15 op/s 2024-08-17T17:05:52.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:51 smithi162 ceph-mon[104668]: osdmap e723: 8 total, 8 up, 8 in 2024-08-17T17:05:52.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:51 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:05:52.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:51 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-08-17T17:05:52.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:51 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:05:52.594 INFO:tasks.rados.rados.0.smithi063.stdout:2557: done (1 left) 2024-08-17T17:05:52.594 INFO:tasks.rados.rados.0.smithi063.stdout:2558: snap_remove snap 319 2024-08-17T17:05:52.922 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:52 smithi063 ceph-mon[109843]: Upgrade: Updating osd.0 2024-08-17T17:05:52.922 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:52 smithi063 ceph-mon[109843]: Deploying daemon osd.0 on smithi063 2024-08-17T17:05:52.922 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:52 smithi063 ceph-mon[109843]: osdmap e724: 8 total, 8 up, 8 in 2024-08-17T17:05:52.922 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:52 smithi063 ceph-mon[109843]: osdmap e725: 8 total, 8 up, 8 in 2024-08-17T17:05:52.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:52 smithi063 ceph-mon[114756]: Upgrade: Updating osd.0 2024-08-17T17:05:52.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:52 smithi063 ceph-mon[114756]: Deploying daemon osd.0 on smithi063 2024-08-17T17:05:52.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:52 smithi063 ceph-mon[114756]: osdmap e724: 8 total, 8 up, 8 in 2024-08-17T17:05:52.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:52 smithi063 ceph-mon[114756]: osdmap e725: 8 total, 8 up, 8 in 2024-08-17T17:05:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:52 smithi162 ceph-mon[104668]: Upgrade: Updating osd.0 2024-08-17T17:05:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:52 smithi162 ceph-mon[104668]: Deploying daemon osd.0 on smithi063 2024-08-17T17:05:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:52 smithi162 ceph-mon[104668]: osdmap e724: 8 total, 8 up, 8 in 2024-08-17T17:05:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:52 smithi162 ceph-mon[104668]: osdmap e725: 8 total, 8 up, 8 in 2024-08-17T17:05:53.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:05:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:05:53] "GET /metrics HTTP/1.1" 200 36212 "" "Prometheus/2.51.0" 2024-08-17T17:05:53.595 INFO:tasks.rados.rados.0.smithi063.stdout:2558: done (1 left) 2024-08-17T17:05:53.596 INFO:tasks.rados.rados.0.smithi063.stdout:2559: snap_remove snap 314 2024-08-17T17:05:53.803 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:53 smithi063 ceph-mon[114756]: pgmap v543: 89 pgs: 89 active+clean; 189 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 704 KiB/s wr, 6 op/s 2024-08-17T17:05:53.804 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:53 smithi063 ceph-mon[114756]: osdmap e726: 8 total, 8 up, 8 in 2024-08-17T17:05:53.804 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:53 smithi063 ceph-mon[109843]: pgmap v543: 89 pgs: 89 active+clean; 189 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 704 KiB/s wr, 6 op/s 2024-08-17T17:05:53.804 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:53 smithi063 ceph-mon[109843]: osdmap e726: 8 total, 8 up, 8 in 2024-08-17T17:05:53.804 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:53 smithi063 systemd[1]: Stopping Ceph osd.0 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:05:54.062 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0[47037]: 2024-08-17T17:05:53.802+0000 7f09e803e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:05:54.062 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0[47037]: 2024-08-17T17:05:53.802+0000 7f09e803e640 -1 osd.0 726 *** Got signal Terminated *** 2024-08-17T17:05:54.062 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0[47037]: 2024-08-17T17:05:53.802+0000 7f09e803e640 -1 osd.0 726 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:05:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:53 smithi162 ceph-mon[104668]: pgmap v543: 89 pgs: 89 active+clean; 189 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 704 KiB/s wr, 6 op/s 2024-08-17T17:05:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:53 smithi162 ceph-mon[104668]: osdmap e726: 8 total, 8 up, 8 in 2024-08-17T17:05:54.603 INFO:tasks.rados.rados.0.smithi063.stdout:2559: done (1 left) 2024-08-17T17:05:54.603 INFO:tasks.rados.rados.0.smithi063.stdout:2560: write oid 45 current snap is 328 2024-08-17T17:05:54.603 INFO:tasks.rados.rados.0.smithi063.stdout:2560: seq_num 966 ranges {635562=667365,2094782=140780} 2024-08-17T17:05:54.611 INFO:tasks.rados.rados.0.smithi063.stdout:2560: writing smithi06387500-45 from 635562 to 1302927 tid 1 2024-08-17T17:05:54.617 INFO:tasks.rados.rados.0.smithi063.stdout:2560: writing smithi06387500-45 from 2094782 to 2235562 tid 2 2024-08-17T17:05:54.617 INFO:tasks.rados.rados.0.smithi063.stdout:2561: snap_create 2024-08-17T17:05:54.617 INFO:tasks.rados.rados.0.smithi063.stdout:2556: finishing write tid 1 to smithi06387500-4 2024-08-17T17:05:54.617 INFO:tasks.rados.rados.0.smithi063.stdout:2556: finishing write tid 2 to smithi06387500-4 2024-08-17T17:05:54.617 INFO:tasks.rados.rados.0.smithi063.stdout:2556: finishing write tid 3 to smithi06387500-4 2024-08-17T17:05:54.617 INFO:tasks.rados.rados.0.smithi063.stdout:2556: finishing write tid 4 to smithi06387500-4 2024-08-17T17:05:54.617 INFO:tasks.rados.rados.0.smithi063.stdout:2556: oid 4 updating version 0 to 2544 2024-08-17T17:05:54.617 INFO:tasks.rados.rados.0.smithi063.stdout:2556: oid 4 version 2544 is already newer than 2543 2024-08-17T17:05:54.617 INFO:tasks.rados.rados.0.smithi063.stdout:2556: oid 4 updating version 2544 to 2545 2024-08-17T17:05:54.618 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 2545 (ObjNum 965 snap 328 seq_num 965) dirty exists 2024-08-17T17:05:54.618 INFO:tasks.rados.rados.0.smithi063.stdout:2556: left oid 4 (ObjNum 965 snap 328 seq_num 965) 2024-08-17T17:05:54.618 INFO:tasks.rados.rados.0.smithi063.stdout:2560: finishing write tid 1 to smithi06387500-45 2024-08-17T17:05:54.622 INFO:tasks.rados.rados.0.smithi063.stdout:2560: finishing write tid 2 to smithi06387500-45 2024-08-17T17:05:54.622 INFO:tasks.rados.rados.0.smithi063.stdout:2560: finishing write tid 3 to smithi06387500-45 2024-08-17T17:05:54.623 INFO:tasks.rados.rados.0.smithi063.stdout:2560: finishing write tid 4 to smithi06387500-45 2024-08-17T17:05:54.623 INFO:tasks.rados.rados.0.smithi063.stdout:2560: oid 45 updating version 0 to 599 2024-08-17T17:05:54.623 INFO:tasks.rados.rados.0.smithi063.stdout:2560: oid 45 updating version 599 to 600 2024-08-17T17:05:54.623 INFO:tasks.rados.rados.0.smithi063.stdout:2560: oid 45 updating version 600 to 601 2024-08-17T17:05:54.623 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 601 (ObjNum 966 snap 328 seq_num 966) dirty exists 2024-08-17T17:05:54.623 INFO:tasks.rados.rados.0.smithi063.stdout:2560: left oid 45 (ObjNum 966 snap 328 seq_num 966) 2024-08-17T17:05:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:54 smithi063 ceph-mon[114756]: osd.0 marked itself down and dead 2024-08-17T17:05:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:54 smithi063 ceph-mon[114756]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:05:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:54 smithi063 ceph-mon[114756]: osdmap e727: 8 total, 7 up, 8 in 2024-08-17T17:05:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:54 smithi063 ceph-mon[109843]: osd.0 marked itself down and dead 2024-08-17T17:05:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:54 smithi063 ceph-mon[109843]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:05:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:54 smithi063 ceph-mon[109843]: osdmap e727: 8 total, 7 up, 8 in 2024-08-17T17:05:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:54 smithi162 ceph-mon[104668]: osd.0 marked itself down and dead 2024-08-17T17:05:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:54 smithi162 ceph-mon[104668]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:05:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:54 smithi162 ceph-mon[104668]: osdmap e727: 8 total, 7 up, 8 in 2024-08-17T17:05:55.427 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:55 smithi063 podman[127151]: 2024-08-17 17:05:55.143350745 +0000 UTC m=+1.443476241 container died f3b910ed49b8147157bb766e2d49cb4e9fd2293c338e9b666942eae6682ed4b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-08-17T17:05:55.605 INFO:tasks.rados.rados.0.smithi063.stdout:2556: done (2 left) 2024-08-17T17:05:55.605 INFO:tasks.rados.rados.0.smithi063.stdout:2560: done (1 left) 2024-08-17T17:05:55.605 INFO:tasks.rados.rados.0.smithi063.stdout:2561: done (0 left) 2024-08-17T17:05:55.605 INFO:tasks.rados.rados.0.smithi063.stdout:2562: delete oid 44 current snap is 329 2024-08-17T17:05:55.614 INFO:tasks.rados.rados.0.smithi063.stdout:2562: done (0 left) 2024-08-17T17:05:55.614 INFO:tasks.rados.rados.0.smithi063.stdout:2563: write oid 25 current snap is 329 2024-08-17T17:05:55.614 INFO:tasks.rados.rados.0.smithi063.stdout:2563: seq_num 968 ranges {669176=417863,1811746=457430} 2024-08-17T17:05:55.620 INFO:tasks.rados.rados.0.smithi063.stdout:2563: writing smithi06387500-25 from 669176 to 1087039 tid 1 2024-08-17T17:05:55.627 INFO:tasks.rados.rados.0.smithi063.stdout:2563: writing smithi06387500-25 from 1811746 to 2269176 tid 2 2024-08-17T17:05:55.627 INFO:tasks.rados.rados.0.smithi063.stdout:2564: read oid 49 snap -1 2024-08-17T17:05:55.627 INFO:tasks.rados.rados.0.smithi063.stdout:2564: expect (ObjNum 896 snap 310 seq_num 896) 2024-08-17T17:05:55.627 INFO:tasks.rados.rados.0.smithi063.stdout:2565: snap_create 2024-08-17T17:05:55.627 INFO:tasks.rados.rados.0.smithi063.stdout:2563: finishing write tid 1 to smithi06387500-25 2024-08-17T17:05:55.631 INFO:tasks.rados.rados.0.smithi063.stdout:2563: finishing write tid 2 to smithi06387500-25 2024-08-17T17:05:55.639 INFO:tasks.rados.rados.0.smithi063.stdout:2563: finishing write tid 3 to smithi06387500-25 2024-08-17T17:05:55.639 INFO:tasks.rados.rados.0.smithi063.stdout:2563: finishing write tid 4 to smithi06387500-25 2024-08-17T17:05:55.639 INFO:tasks.rados.rados.0.smithi063.stdout:2563: oid 25 updating version 0 to 2557 2024-08-17T17:05:55.639 INFO:tasks.rados.rados.0.smithi063.stdout:2563: oid 25 updating version 2557 to 2558 2024-08-17T17:05:55.639 INFO:tasks.rados.rados.0.smithi063.stdout:2563: oid 25 version 2558 is already newer than 2556 2024-08-17T17:05:55.639 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 2558 (ObjNum 968 snap 329 seq_num 968) dirty exists 2024-08-17T17:05:55.640 INFO:tasks.rados.rados.0.smithi063.stdout:2563: left oid 25 (ObjNum 968 snap 329 seq_num 968) 2024-08-17T17:05:55.645 INFO:tasks.rados.rados.0.smithi063.stdout:2564: expect (ObjNum 896 snap 310 seq_num 896) 2024-08-17T17:05:55.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:55 smithi063 ceph-mon[114756]: pgmap v546: 89 pgs: 89 active+clean; 189 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 704 KiB/s wr, 6 op/s 2024-08-17T17:05:55.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:55 smithi063 ceph-mon[114756]: osdmap e728: 8 total, 7 up, 8 in 2024-08-17T17:05:55.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:55 smithi063 ceph-mon[109843]: pgmap v546: 89 pgs: 89 active+clean; 189 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 704 KiB/s wr, 6 op/s 2024-08-17T17:05:55.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:55 smithi063 ceph-mon[109843]: osdmap e728: 8 total, 7 up, 8 in 2024-08-17T17:05:55.780 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:55 smithi063 podman[127151]: 2024-08-17 17:05:55.525447666 +0000 UTC m=+1.825573152 container cleanup f3b910ed49b8147157bb766e2d49cb4e9fd2293c338e9b666942eae6682ed4b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0) 2024-08-17T17:05:55.780 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:55 smithi063 bash[127151]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0 2024-08-17T17:05:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:55 smithi162 ceph-mon[104668]: pgmap v546: 89 pgs: 89 active+clean; 189 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 704 KiB/s wr, 6 op/s 2024-08-17T17:05:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:55 smithi162 ceph-mon[104668]: osdmap e728: 8 total, 7 up, 8 in 2024-08-17T17:05:56.167 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:55 smithi063 podman[127175]: 2024-08-17 17:05:55.793693658 +0000 UTC m=+0.646394311 container remove f3b910ed49b8147157bb766e2d49cb4e9fd2293c338e9b666942eae6682ed4b5 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.36.0) 2024-08-17T17:05:56.430 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:56 smithi063 podman[127265]: 2024-08-17 17:05:56.069667493 +0000 UTC m=+0.017806640 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:05:56.431 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:56 smithi063 podman[127265]: 2024-08-17 17:05:56.177913062 +0000 UTC m=+0.126052234 container create 03e549834e5faf64fef198dc772188f98ec0d553f4636d56feba2f1f39ad463e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:05:56.625 INFO:tasks.rados.rados.0.smithi063.stdout:2563: done (2 left) 2024-08-17T17:05:56.625 INFO:tasks.rados.rados.0.smithi063.stdout:2564: done (1 left) 2024-08-17T17:05:56.626 INFO:tasks.rados.rados.0.smithi063.stdout:2565: done (0 left) 2024-08-17T17:05:56.626 INFO:tasks.rados.rados.0.smithi063.stdout:2566: snap_create 2024-08-17T17:05:56.779 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:56 smithi063 podman[127265]: 2024-08-17 17:05:56.430647811 +0000 UTC m=+0.378786960 container init 03e549834e5faf64fef198dc772188f98ec0d553f4636d56feba2f1f39ad463e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-08-17T17:05:56.779 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:56 smithi063 podman[127265]: 2024-08-17 17:05:56.435972242 +0000 UTC m=+0.384111373 container start 03e549834e5faf64fef198dc772188f98ec0d553f4636d56feba2f1f39ad463e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731) 2024-08-17T17:05:56.779 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:56 smithi063 podman[127265]: 2024-08-17 17:05:56.538813592 +0000 UTC m=+0.486952723 container attach 03e549834e5faf64fef198dc772188f98ec0d553f4636d56feba2f1f39ad463e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-08-17T17:05:56.780 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:56 smithi063 podman[127265]: 2024-08-17 17:05:56.629899419 +0000 UTC m=+0.578038553 container died 03e549834e5faf64fef198dc772188f98ec0d553f4636d56feba2f1f39ad463e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-08-17T17:05:57.529 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:57 smithi063 podman[127307]: 2024-08-17 17:05:57.209211939 +0000 UTC m=+0.573317912 container remove 03e549834e5faf64fef198dc772188f98ec0d553f4636d56feba2f1f39ad463e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-08-17T17:05:57.529 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:57 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.0.service: Deactivated successfully. 2024-08-17T17:05:57.530 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:57 smithi063 systemd[1]: Stopped Ceph osd.0 for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:05:57.530 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:57 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.0.service: Consumed 20.307s CPU time. 2024-08-17T17:05:57.530 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:57 smithi063 systemd[1]: Starting Ceph osd.0 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:05:57.635 INFO:tasks.rados.rados.0.smithi063.stdout:2566: done (0 left) 2024-08-17T17:05:57.635 INFO:tasks.rados.rados.0.smithi063.stdout:2567: snap_create 2024-08-17T17:05:57.904 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:57 smithi063 ceph-mon[114756]: pgmap v549: 89 pgs: 13 stale+active+clean, 76 active+clean; 189 MiB data, 726 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:57.904 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:57 smithi063 ceph-mon[114756]: osdmap e729: 8 total, 7 up, 8 in 2024-08-17T17:05:57.905 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:57 smithi063 ceph-mon[109843]: pgmap v549: 89 pgs: 13 stale+active+clean, 76 active+clean; 189 MiB data, 726 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:57.905 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:57 smithi063 ceph-mon[109843]: osdmap e729: 8 total, 7 up, 8 in 2024-08-17T17:05:57.905 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:57 smithi063 podman[127412]: 2024-08-17 17:05:57.567705552 +0000 UTC m=+0.017911547 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:05:57.905 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:57 smithi063 podman[127412]: 2024-08-17 17:05:57.676977051 +0000 UTC m=+0.127183026 container create fef1964af1aee5e3a3a93615776939e697a83fb5dbe655eb7a1adecad6baa17c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:05:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:57 smithi162 ceph-mon[104668]: pgmap v549: 89 pgs: 13 stale+active+clean, 76 active+clean; 189 MiB data, 726 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:05:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:57 smithi162 ceph-mon[104668]: osdmap e729: 8 total, 7 up, 8 in 2024-08-17T17:05:58.155 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:57 smithi063 podman[127412]: 2024-08-17 17:05:57.904325698 +0000 UTC m=+0.354531673 container init fef1964af1aee5e3a3a93615776939e697a83fb5dbe655eb7a1adecad6baa17c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-08-17T17:05:58.155 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:57 smithi063 podman[127412]: 2024-08-17 17:05:57.909117415 +0000 UTC m=+0.359323394 container start fef1964af1aee5e3a3a93615776939e697a83fb5dbe655eb7a1adecad6baa17c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:05:58.155 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 podman[127412]: 2024-08-17 17:05:58.01274916 +0000 UTC m=+0.462955142 container attach fef1964af1aee5e3a3a93615776939e697a83fb5dbe655eb7a1adecad6baa17c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-08-17T17:05:58.155 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate[127427]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:05:58.156 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 bash[127412]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:05:58.156 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate[127427]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:05:58.156 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 bash[127412]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:05:58.156 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate[127427]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:05:58.156 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 bash[127412]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:05:58.156 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate[127427]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:05:58.156 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 bash[127412]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:05:58.156 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate[127427]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:05:58.156 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 bash[127412]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:05:58.156 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate[127427]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:05:58.156 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 bash[127412]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:05:58.439 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate[127427]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-17T17:05:58.439 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 bash[127412]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2024-08-17T17:05:58.440 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate[127427]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:05:58.440 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 bash[127412]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:05:58.646 INFO:tasks.rados.rados.0.smithi063.stdout:2567: done (0 left) 2024-08-17T17:05:58.646 INFO:tasks.rados.rados.0.smithi063.stdout:2568: read oid 32 snap -1 2024-08-17T17:05:58.646 INFO:tasks.rados.rados.0.smithi063.stdout:2568: expect (ObjNum 882 snap 308 seq_num 882) 2024-08-17T17:05:58.646 INFO:tasks.rados.rados.0.smithi063.stdout:2569: write oid 41 current snap is 332 2024-08-17T17:05:58.646 INFO:tasks.rados.rados.0.smithi063.stdout:2569: seq_num 969 ranges {685983=493112,1870228=415755} 2024-08-17T17:05:58.652 INFO:tasks.rados.rados.0.smithi063.stdout:2569: writing smithi06387500-41 from 685983 to 1179095 tid 1 2024-08-17T17:05:58.658 INFO:tasks.rados.rados.0.smithi063.stdout:2569: writing smithi06387500-41 from 1870228 to 2285983 tid 2 2024-08-17T17:05:58.659 INFO:tasks.rados.rados.0.smithi063.stdout:2570: snap_create 2024-08-17T17:05:58.659 INFO:tasks.rados.rados.0.smithi063.stdout:2568: expect (ObjNum 882 snap 308 seq_num 882) 2024-08-17T17:05:58.775 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:58 smithi063 ceph-mon[114756]: osdmap e730: 8 total, 7 up, 8 in 2024-08-17T17:05:58.776 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:58 smithi063 ceph-mon[109843]: osdmap e730: 8 total, 7 up, 8 in 2024-08-17T17:05:58.776 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate[127427]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:05:58.776 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 bash[127412]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2024-08-17T17:05:58.776 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate[127427]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-17T17:05:58.776 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 bash[127412]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-08-17T17:05:58.776 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate[127427]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:05:58.777 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 bash[127412]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-08-17T17:05:58.777 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate[127427]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-17T17:05:58.777 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 bash[127412]: --> ceph-volume lvm activate successful for osd ID: 0 2024-08-17T17:05:58.777 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:58 smithi063 podman[127412]: 2024-08-17 17:05:58.468751493 +0000 UTC m=+0.918957478 container died fef1964af1aee5e3a3a93615776939e697a83fb5dbe655eb7a1adecad6baa17c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-08-17T17:05:58.818 INFO:tasks.rados.rados.0.smithi063.stdout:2569: finishing write tid 1 to smithi06387500-41 2024-08-17T17:05:58.818 INFO:tasks.rados.rados.0.smithi063.stdout:2569: finishing write tid 2 to smithi06387500-41 2024-08-17T17:05:58.819 INFO:tasks.rados.rados.0.smithi063.stdout:2569: finishing write tid 3 to smithi06387500-41 2024-08-17T17:05:58.819 INFO:tasks.rados.rados.0.smithi063.stdout:2569: finishing write tid 4 to smithi06387500-41 2024-08-17T17:05:58.819 INFO:tasks.rados.rados.0.smithi063.stdout:2569: oid 41 updating version 0 to 256 2024-08-17T17:05:58.819 INFO:tasks.rados.rados.0.smithi063.stdout:2569: oid 41 version 256 is already newer than 254 2024-08-17T17:05:58.819 INFO:tasks.rados.rados.0.smithi063.stdout:2569: oid 41 version 256 is already newer than 255 2024-08-17T17:05:58.819 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 256 (ObjNum 969 snap 332 seq_num 969) dirty exists 2024-08-17T17:05:58.819 INFO:tasks.rados.rados.0.smithi063.stdout:2569: left oid 41 (ObjNum 969 snap 332 seq_num 969) 2024-08-17T17:05:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:58 smithi162 ceph-mon[104668]: osdmap e730: 8 total, 7 up, 8 in 2024-08-17T17:05:59.403 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:59 smithi063 podman[127552]: 2024-08-17 17:05:59.075074548 +0000 UTC m=+0.596868187 container remove fef1964af1aee5e3a3a93615776939e697a83fb5dbe655eb7a1adecad6baa17c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-activate, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-08-17T17:05:59.403 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:59 smithi063 podman[127617]: 2024-08-17 17:05:59.284310505 +0000 UTC m=+0.108042425 container create 8fa117adbc0a2a8169a1b8c79158a54be6f036a4208aa503d68c13eaedf975db (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, GIT_CLEAN=True, ceph=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:05:59.404 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:59 smithi063 podman[127617]: 2024-08-17 17:05:59.193968309 +0000 UTC m=+0.017700186 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:05:59.660 INFO:tasks.rados.rados.0.smithi063.stdout:2568: done (2 left) 2024-08-17T17:05:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:2569: done (1 left) 2024-08-17T17:05:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:2570: done (0 left) 2024-08-17T17:05:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:2571: write oid 16 current snap is 333 2024-08-17T17:05:59.661 INFO:tasks.rados.rados.0.smithi063.stdout:2571: seq_num 970 ranges {702790=568361,1812357=490433} 2024-08-17T17:05:59.667 INFO:tasks.rados.rados.0.smithi063.stdout:2571: writing smithi06387500-16 from 702790 to 1271151 tid 1 2024-08-17T17:05:59.674 INFO:tasks.rados.rados.0.smithi063.stdout:2571: writing smithi06387500-16 from 1812357 to 2302790 tid 2 2024-08-17T17:05:59.674 INFO:tasks.rados.rados.0.smithi063.stdout:2572: read oid 5 snap -1 2024-08-17T17:05:59.674 INFO:tasks.rados.rados.0.smithi063.stdout:2572: expect deleted 2024-08-17T17:05:59.674 INFO:tasks.rados.rados.0.smithi063.stdout:2573: snap_remove snap 325 2024-08-17T17:05:59.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:59 smithi063 ceph-mon[114756]: pgmap v552: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 191 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 411 KiB/s wr, 4 op/s; 73/384 objects degraded (19.010%) 2024-08-17T17:05:59.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:59 smithi063 ceph-mon[114756]: Health check failed: Degraded data redundancy: 73/384 objects degraded (19.010%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T17:05:59.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:05:59 smithi063 ceph-mon[114756]: osdmap e731: 8 total, 7 up, 8 in 2024-08-17T17:05:59.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:59 smithi063 ceph-mon[109843]: pgmap v552: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 191 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 411 KiB/s wr, 4 op/s; 73/384 objects degraded (19.010%) 2024-08-17T17:05:59.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:59 smithi063 ceph-mon[109843]: Health check failed: Degraded data redundancy: 73/384 objects degraded (19.010%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T17:05:59.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:05:59 smithi063 ceph-mon[109843]: osdmap e731: 8 total, 7 up, 8 in 2024-08-17T17:05:59.780 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:59 smithi063 podman[127617]: 2024-08-17 17:05:59.494979497 +0000 UTC m=+0.318711391 container init 8fa117adbc0a2a8169a1b8c79158a54be6f036a4208aa503d68c13eaedf975db (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0) 2024-08-17T17:05:59.780 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:59 smithi063 podman[127617]: 2024-08-17 17:05:59.500015881 +0000 UTC m=+0.323747756 container start 8fa117adbc0a2a8169a1b8c79158a54be6f036a4208aa503d68c13eaedf975db (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:05:59.780 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:59 smithi063 ceph-osd[127656]: -- 172.21.15.63:0/1914526526 <== mon.0 v2:172.21.15.63:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x557fb476ba40 con 0x557fb5538400 2024-08-17T17:05:59.780 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:59 smithi063 bash[127617]: 8fa117adbc0a2a8169a1b8c79158a54be6f036a4208aa503d68c13eaedf975db 2024-08-17T17:05:59.780 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:05:59 smithi063 systemd[1]: Started Ceph osd.0 for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:06:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:59 smithi162 ceph-mon[104668]: pgmap v552: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 191 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 411 KiB/s wr, 4 op/s; 73/384 objects degraded (19.010%) 2024-08-17T17:06:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:59 smithi162 ceph-mon[104668]: Health check failed: Degraded data redundancy: 73/384 objects degraded (19.010%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:05:59 smithi162 ceph-mon[104668]: osdmap e731: 8 total, 7 up, 8 in 2024-08-17T17:06:00.672 INFO:tasks.rados.rados.0.smithi063.stdout:2573: done (2 left) 2024-08-17T17:06:00.672 INFO:tasks.rados.rados.0.smithi063.stdout:2574: write oid 34 current snap is 333 2024-08-17T17:06:00.672 INFO:tasks.rados.rados.0.smithi063.stdout:2571: finishing write tid 1 to smithi06387500-16 2024-08-17T17:06:00.672 INFO:tasks.rados.rados.0.smithi063.stdout:2574: seq_num 971 ranges {719597=643610,1870839=448758} 2024-08-17T17:06:00.679 INFO:tasks.rados.rados.0.smithi063.stdout:2574: writing smithi06387500-34 from 719597 to 1363207 tid 1 2024-08-17T17:06:00.686 INFO:tasks.rados.rados.0.smithi063.stdout:2574: writing smithi06387500-34 from 1870839 to 2319597 tid 2 2024-08-17T17:06:00.686 INFO:tasks.rados.rados.0.smithi063.stdout:2575: snap_create 2024-08-17T17:06:00.686 INFO:tasks.rados.rados.0.smithi063.stdout:2571: finishing write tid 2 to smithi06387500-16 2024-08-17T17:06:00.686 INFO:tasks.rados.rados.0.smithi063.stdout:2571: finishing write tid 3 to smithi06387500-16 2024-08-17T17:06:00.686 INFO:tasks.rados.rados.0.smithi063.stdout:2571: finishing write tid 4 to smithi06387500-16 2024-08-17T17:06:00.686 INFO:tasks.rados.rados.0.smithi063.stdout:2571: oid 16 updating version 0 to 955 2024-08-17T17:06:00.686 INFO:tasks.rados.rados.0.smithi063.stdout:2571: oid 16 updating version 955 to 956 2024-08-17T17:06:00.686 INFO:tasks.rados.rados.0.smithi063.stdout:2571: oid 16 updating version 956 to 957 2024-08-17T17:06:00.686 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 957 (ObjNum 970 snap 333 seq_num 970) dirty exists 2024-08-17T17:06:00.686 INFO:tasks.rados.rados.0.smithi063.stdout:2571: left oid 16 (ObjNum 970 snap 333 seq_num 970) 2024-08-17T17:06:00.688 INFO:tasks.rados.rados.0.smithi063.stdout:2574: finishing write tid 1 to smithi06387500-34 2024-08-17T17:06:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:2574: finishing write tid 2 to smithi06387500-34 2024-08-17T17:06:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:2574: finishing write tid 3 to smithi06387500-34 2024-08-17T17:06:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:2574: finishing write tid 4 to smithi06387500-34 2024-08-17T17:06:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:2574: oid 34 updating version 0 to 1747 2024-08-17T17:06:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:2574: oid 34 version 1747 is already newer than 1746 2024-08-17T17:06:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:2574: oid 34 version 1747 is already newer than 1745 2024-08-17T17:06:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 1747 (ObjNum 971 snap 333 seq_num 971) dirty exists 2024-08-17T17:06:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:2574: left oid 34 (ObjNum 971 snap 333 seq_num 971) 2024-08-17T17:06:00.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:00 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:00.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:00 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:00.998 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:00 smithi162 ceph-mon[104668]: osdmap e732: 8 total, 7 up, 8 in 2024-08-17T17:06:00.998 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:00 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:00.998 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:00 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:00 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:00 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:00 smithi063 ceph-mon[114756]: osdmap e732: 8 total, 7 up, 8 in 2024-08-17T17:06:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:00 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:00 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:00 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:01.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:00 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:01.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:00 smithi063 ceph-mon[109843]: osdmap e732: 8 total, 7 up, 8 in 2024-08-17T17:06:01.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:00 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:01.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:00 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:01.481 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:06:01 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0[127633]: 2024-08-17T17:06:01.134+0000 7fc58bb24640 -1 Falling back to public interface 2024-08-17T17:06:01.673 INFO:tasks.rados.rados.0.smithi063.stdout:2571: done (3 left) 2024-08-17T17:06:01.673 INFO:tasks.rados.rados.0.smithi063.stdout:2572: done (2 left) 2024-08-17T17:06:01.673 INFO:tasks.rados.rados.0.smithi063.stdout:2574: done (1 left) 2024-08-17T17:06:01.673 INFO:tasks.rados.rados.0.smithi063.stdout:2575: done (0 left) 2024-08-17T17:06:01.673 INFO:tasks.rados.rados.0.smithi063.stdout:2576: write oid 33 current snap is 334 2024-08-17T17:06:01.673 INFO:tasks.rados.rados.0.smithi063.stdout:2576: seq_num 972 ranges {736404=718859,1929321=407083} 2024-08-17T17:06:01.680 INFO:tasks.rados.rados.0.smithi063.stdout:2576: writing smithi06387500-33 from 736404 to 1455263 tid 1 2024-08-17T17:06:01.686 INFO:tasks.rados.rados.0.smithi063.stdout:2576: writing smithi06387500-33 from 1929321 to 2336404 tid 2 2024-08-17T17:06:01.686 INFO:tasks.rados.rados.0.smithi063.stdout:2577: delete oid 20 current snap is 334 2024-08-17T17:06:01.689 INFO:tasks.rados.rados.0.smithi063.stdout:2576: finishing write tid 1 to smithi06387500-33 2024-08-17T17:06:01.689 INFO:tasks.rados.rados.0.smithi063.stdout:2576: finishing write tid 2 to smithi06387500-33 2024-08-17T17:06:01.689 INFO:tasks.rados.rados.0.smithi063.stdout:2576: finishing write tid 3 to smithi06387500-33 2024-08-17T17:06:01.690 INFO:tasks.rados.rados.0.smithi063.stdout:2576: finishing write tid 4 to smithi06387500-33 2024-08-17T17:06:01.690 INFO:tasks.rados.rados.0.smithi063.stdout:2576: oid 33 updating version 0 to 887 2024-08-17T17:06:01.690 INFO:tasks.rados.rados.0.smithi063.stdout:2576: oid 33 updating version 887 to 889 2024-08-17T17:06:01.690 INFO:tasks.rados.rados.0.smithi063.stdout:2576: oid 33 version 889 is already newer than 888 2024-08-17T17:06:01.690 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 889 (ObjNum 972 snap 334 seq_num 972) dirty exists 2024-08-17T17:06:01.690 INFO:tasks.rados.rados.0.smithi063.stdout:2576: left oid 33 (ObjNum 972 snap 334 seq_num 972) 2024-08-17T17:06:01.690 INFO:tasks.rados.rados.0.smithi063.stdout:2576: done (1 left) 2024-08-17T17:06:01.691 INFO:tasks.rados.rados.0.smithi063.stdout:2577: done (0 left) 2024-08-17T17:06:01.691 INFO:tasks.rados.rados.0.smithi063.stdout:2578: snap_create 2024-08-17T17:06:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:01 smithi063 ceph-mon[114756]: pgmap v555: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 191 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 411 KiB/s wr, 4 op/s; 73/384 objects degraded (19.010%) 2024-08-17T17:06:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:01 smithi063 ceph-mon[114756]: osdmap e733: 8 total, 7 up, 8 in 2024-08-17T17:06:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:01.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:01 smithi063 ceph-mon[109843]: pgmap v555: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 191 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 411 KiB/s wr, 4 op/s; 73/384 objects degraded (19.010%) 2024-08-17T17:06:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:01 smithi063 ceph-mon[109843]: osdmap e733: 8 total, 7 up, 8 in 2024-08-17T17:06:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:01 smithi162 ceph-mon[104668]: pgmap v555: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 191 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 411 KiB/s wr, 4 op/s; 73/384 objects degraded (19.010%) 2024-08-17T17:06:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:01 smithi162 ceph-mon[104668]: osdmap e733: 8 total, 7 up, 8 in 2024-08-17T17:06:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:02.596 INFO:tasks.rados.rados.0.smithi063.stdout:2578: done (0 left) 2024-08-17T17:06:02.597 INFO:tasks.rados.rados.0.smithi063.stdout:2579: rollback oid 15 current snap is 335 2024-08-17T17:06:02.597 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 15 to 334 2024-08-17T17:06:02.597 INFO:tasks.rados.rados.0.smithi063.stdout:2580: read oid 28 snap -1 2024-08-17T17:06:02.597 INFO:tasks.rados.rados.0.smithi063.stdout:2580: expect deleted 2024-08-17T17:06:02.597 INFO:tasks.rados.rados.0.smithi063.stdout:2581: read oid 11 snap -1 2024-08-17T17:06:02.597 INFO:tasks.rados.rados.0.smithi063.stdout:2581: expect deleted 2024-08-17T17:06:02.597 INFO:tasks.rados.rados.0.smithi063.stdout:2582: snap_create 2024-08-17T17:06:02.600 INFO:tasks.rados.rados.0.smithi063.stdout:2579: finishing rollback tid 1 to smithi06387500-15 2024-08-17T17:06:02.600 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 936 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T17:06:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:02 smithi063 ceph-mon[114756]: osdmap e734: 8 total, 7 up, 8 in 2024-08-17T17:06:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:02 smithi063 ceph-mon[114756]: pgmap v558: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 196 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 1023 KiB/s wr, 7 op/s; 75/396 objects degraded (18.939%) 2024-08-17T17:06:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:02 smithi063 ceph-mon[114756]: osdmap e735: 8 total, 7 up, 8 in 2024-08-17T17:06:03.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:02 smithi063 ceph-mon[109843]: osdmap e734: 8 total, 7 up, 8 in 2024-08-17T17:06:03.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:02 smithi063 ceph-mon[109843]: pgmap v558: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 196 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 1023 KiB/s wr, 7 op/s; 75/396 objects degraded (18.939%) 2024-08-17T17:06:03.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:02 smithi063 ceph-mon[109843]: osdmap e735: 8 total, 7 up, 8 in 2024-08-17T17:06:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:02 smithi162 ceph-mon[104668]: osdmap e734: 8 total, 7 up, 8 in 2024-08-17T17:06:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:02 smithi162 ceph-mon[104668]: pgmap v558: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 196 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 1023 KiB/s wr, 7 op/s; 75/396 objects degraded (18.939%) 2024-08-17T17:06:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:02 smithi162 ceph-mon[104668]: osdmap e735: 8 total, 7 up, 8 in 2024-08-17T17:06:03.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:06:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:06:03] "GET /metrics HTTP/1.1" 200 36340 "" "Prometheus/2.51.0" 2024-08-17T17:06:03.602 INFO:tasks.rados.rados.0.smithi063.stdout:2579: done (3 left) 2024-08-17T17:06:03.603 INFO:tasks.rados.rados.0.smithi063.stdout:2580: done (2 left) 2024-08-17T17:06:03.603 INFO:tasks.rados.rados.0.smithi063.stdout:2581: done (1 left) 2024-08-17T17:06:03.603 INFO:tasks.rados.rados.0.smithi063.stdout:2582: done (0 left) 2024-08-17T17:06:03.603 INFO:tasks.rados.rados.0.smithi063.stdout:2583: snap_create 2024-08-17T17:06:04.603 INFO:tasks.rados.rados.0.smithi063.stdout:2583: done (0 left) 2024-08-17T17:06:04.603 INFO:tasks.rados.rados.0.smithi063.stdout:2584: read oid 7 snap -1 2024-08-17T17:06:04.604 INFO:tasks.rados.rados.0.smithi063.stdout:2584: expect (ObjNum 963 snap 327 seq_num 963) 2024-08-17T17:06:04.604 INFO:tasks.rados.rados.0.smithi063.stdout:2585: read oid 5 snap -1 2024-08-17T17:06:04.604 INFO:tasks.rados.rados.0.smithi063.stdout:2585: expect deleted 2024-08-17T17:06:04.604 INFO:tasks.rados.rados.0.smithi063.stdout:2586: write oid 35 current snap is 337 2024-08-17T17:06:04.604 INFO:tasks.rados.rados.0.smithi063.stdout:2586: seq_num 974 ranges {770018=585710,2046285=323733} 2024-08-17T17:06:04.611 INFO:tasks.rados.rados.0.smithi063.stdout:2586: writing smithi06387500-35 from 770018 to 1355728 tid 1 2024-08-17T17:06:04.617 INFO:tasks.rados.rados.0.smithi063.stdout:2586: writing smithi06387500-35 from 2046285 to 2370018 tid 2 2024-08-17T17:06:04.617 INFO:tasks.rados.rados.0.smithi063.stdout:2584: expect (ObjNum 963 snap 327 seq_num 963) 2024-08-17T17:06:04.719 INFO:tasks.rados.rados.0.smithi063.stdout:2586: finishing write tid 1 to smithi06387500-35 2024-08-17T17:06:04.719 INFO:tasks.rados.rados.0.smithi063.stdout:2586: finishing write tid 2 to smithi06387500-35 2024-08-17T17:06:04.719 INFO:tasks.rados.rados.0.smithi063.stdout:2584: done (2 left) 2024-08-17T17:06:04.719 INFO:tasks.rados.rados.0.smithi063.stdout:2585: done (1 left) 2024-08-17T17:06:04.719 INFO:tasks.rados.rados.0.smithi063.stdout:2587: write oid 30 current snap is 337 2024-08-17T17:06:04.719 INFO:tasks.rados.rados.0.smithi063.stdout:2586: finishing write tid 3 to smithi06387500-35 2024-08-17T17:06:04.720 INFO:tasks.rados.rados.0.smithi063.stdout:2587: seq_num 975 ranges {786825=660959,2104767=282058} 2024-08-17T17:06:04.726 INFO:tasks.rados.rados.0.smithi063.stdout:2587: writing smithi06387500-30 from 786825 to 1447784 tid 1 2024-08-17T17:06:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:2587: writing smithi06387500-30 from 2104767 to 2386825 tid 2 2024-08-17T17:06:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:2588: delete oid 10 current snap is 337 2024-08-17T17:06:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:2586: finishing write tid 4 to smithi06387500-35 2024-08-17T17:06:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:2586: oid 35 updating version 0 to 2562 2024-08-17T17:06:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:2586: oid 35 updating version 2562 to 2563 2024-08-17T17:06:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:2586: oid 35 updating version 2563 to 2564 2024-08-17T17:06:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 2564 (ObjNum 974 snap 337 seq_num 974) dirty exists 2024-08-17T17:06:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:2586: left oid 35 (ObjNum 974 snap 337 seq_num 974) 2024-08-17T17:06:04.733 INFO:tasks.rados.rados.0.smithi063.stdout:2587: finishing write tid 1 to smithi06387500-30 2024-08-17T17:06:04.735 INFO:tasks.rados.rados.0.smithi063.stdout:2586: done (2 left) 2024-08-17T17:06:04.735 INFO:tasks.rados.rados.0.smithi063.stdout:2588: done (1 left) 2024-08-17T17:06:04.735 INFO:tasks.rados.rados.0.smithi063.stdout:2589: read oid 44 snap -1 2024-08-17T17:06:04.735 INFO:tasks.rados.rados.0.smithi063.stdout:2589: expect deleted 2024-08-17T17:06:04.736 INFO:tasks.rados.rados.0.smithi063.stdout:2590: read oid 23 snap -1 2024-08-17T17:06:04.736 INFO:tasks.rados.rados.0.smithi063.stdout:2590: expect deleted 2024-08-17T17:06:04.736 INFO:tasks.rados.rados.0.smithi063.stdout:2591: read oid 18 snap -1 2024-08-17T17:06:04.736 INFO:tasks.rados.rados.0.smithi063.stdout:2591: expect deleted 2024-08-17T17:06:04.736 INFO:tasks.rados.rados.0.smithi063.stdout:2592: read oid 47 snap 323 2024-08-17T17:06:04.736 INFO:tasks.rados.rados.0.smithi063.stdout:2592: expect (ObjNum 888 snap 309 seq_num 888) 2024-08-17T17:06:04.736 INFO:tasks.rados.rados.0.smithi063.stdout:2593: read oid 34 snap 331 2024-08-17T17:06:04.736 INFO:tasks.rados.rados.0.smithi063.stdout:2593: expect (ObjNum 953 snap 325 seq_num 953) 2024-08-17T17:06:04.736 INFO:tasks.rados.rados.0.smithi063.stdout:2594: read oid 20 snap -1 2024-08-17T17:06:04.736 INFO:tasks.rados.rados.0.smithi063.stdout:2594: expect deleted 2024-08-17T17:06:04.736 INFO:tasks.rados.rados.0.smithi063.stdout:2595: snap_remove snap 330 2024-08-17T17:06:04.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:04 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:04.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:04 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:04.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:04 smithi063 ceph-mon[114756]: osdmap e736: 8 total, 7 up, 8 in 2024-08-17T17:06:04.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:04 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:04.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:04 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:04.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:04 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:04.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:04 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:04.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:04 smithi063 ceph-mon[109843]: osdmap e736: 8 total, 7 up, 8 in 2024-08-17T17:06:04.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:04 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:04.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:04 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:04 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:04 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:04 smithi162 ceph-mon[104668]: osdmap e736: 8 total, 7 up, 8 in 2024-08-17T17:06:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:04 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:04 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:05.250 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:06:04 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0[127633]: 2024-08-17T17:06:04.891+0000 7fc58bb24640 -1 osd.0 726 log_to_monitors true 2024-08-17T17:06:05.608 INFO:tasks.rados.rados.0.smithi063.stdout:2595: done (7 left) 2024-08-17T17:06:05.608 INFO:tasks.rados.rados.0.smithi063.stdout:2596: read oid 7 snap -1 2024-08-17T17:06:05.608 INFO:tasks.rados.rados.0.smithi063.stdout:2596: expect (ObjNum 963 snap 327 seq_num 963) 2024-08-17T17:06:05.608 INFO:tasks.rados.rados.0.smithi063.stdout:2597: write oid 9 current snap is 337 2024-08-17T17:06:05.608 INFO:tasks.rados.rados.0.smithi063.stdout:2587: finishing write tid 2 to smithi06387500-30 2024-08-17T17:06:05.608 INFO:tasks.rados.rados.0.smithi063.stdout:2587: finishing write tid 3 to smithi06387500-30 2024-08-17T17:06:05.608 INFO:tasks.rados.rados.0.smithi063.stdout:2597: seq_num 977 ranges {420439=411457,1421731=480010} 2024-08-17T17:06:05.612 INFO:tasks.rados.rados.0.smithi063.stdout:2597: writing smithi06387500-9 from 420439 to 831896 tid 1 2024-08-17T17:06:05.619 INFO:tasks.rados.rados.0.smithi063.stdout:2597: writing smithi06387500-9 from 1421731 to 1901741 tid 2 2024-08-17T17:06:05.619 INFO:tasks.rados.rados.0.smithi063.stdout:2598: read oid 49 snap -1 2024-08-17T17:06:05.619 INFO:tasks.rados.rados.0.smithi063.stdout:2598: expect (ObjNum 896 snap 310 seq_num 896) 2024-08-17T17:06:05.619 INFO:tasks.rados.rados.0.smithi063.stdout:2599: rollback oid 3 current snap is 337 2024-08-17T17:06:05.619 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 3 to 323 2024-08-17T17:06:05.619 INFO:tasks.rados.rados.0.smithi063.stdout:2587: finishing write tid 4 to smithi06387500-30 2024-08-17T17:06:05.619 INFO:tasks.rados.rados.0.smithi063.stdout:2587: oid 30 updating version 0 to 1209 2024-08-17T17:06:05.620 INFO:tasks.rados.rados.0.smithi063.stdout:2587: oid 30 version 1209 is already newer than 1208 2024-08-17T17:06:05.620 INFO:tasks.rados.rados.0.smithi063.stdout:2587: oid 30 updating version 1209 to 1210 2024-08-17T17:06:05.620 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1210 (ObjNum 975 snap 337 seq_num 975) dirty exists 2024-08-17T17:06:05.620 INFO:tasks.rados.rados.0.smithi063.stdout:2587: left oid 30 (ObjNum 975 snap 337 seq_num 975) 2024-08-17T17:06:05.620 INFO:tasks.rados.rados.0.smithi063.stdout:2593: expect (ObjNum 953 snap 325 seq_num 953) 2024-08-17T17:06:05.687 INFO:tasks.rados.rados.0.smithi063.stdout:2592: expect (ObjNum 888 snap 309 seq_num 888) 2024-08-17T17:06:05.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:05 smithi063 ceph-mon[114756]: pgmap v561: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 196 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 1023 KiB/s wr, 7 op/s; 75/396 objects degraded (18.939%) 2024-08-17T17:06:05.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:05 smithi063 ceph-mon[114756]: osdmap e737: 8 total, 7 up, 8 in 2024-08-17T17:06:05.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:05 smithi063 ceph-mon[114756]: from='osd.0 [v2:172.21.15.63:6802/4285380350,v1:172.21.15.63:6803/4285380350]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:06:05.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:05 smithi063 ceph-mon[114756]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:06:05.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:05 smithi063 ceph-mon[109843]: pgmap v561: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 196 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 1023 KiB/s wr, 7 op/s; 75/396 objects degraded (18.939%) 2024-08-17T17:06:05.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:05 smithi063 ceph-mon[109843]: osdmap e737: 8 total, 7 up, 8 in 2024-08-17T17:06:05.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:05 smithi063 ceph-mon[109843]: from='osd.0 [v2:172.21.15.63:6802/4285380350,v1:172.21.15.63:6803/4285380350]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:06:05.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:05 smithi063 ceph-mon[109843]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:06:05.784 INFO:tasks.rados.rados.0.smithi063.stdout:2596: expect (ObjNum 963 snap 327 seq_num 963) 2024-08-17T17:06:05.882 INFO:tasks.rados.rados.0.smithi063.stdout:2598: expect (ObjNum 896 snap 310 seq_num 896) 2024-08-17T17:06:06.031 INFO:tasks.rados.rados.0.smithi063.stdout:2599: finishing rollback tid 0 to smithi06387500-3 2024-08-17T17:06:06.031 INFO:tasks.rados.rados.0.smithi063.stdout:2599: finishing rollback tid 1 to smithi06387500-3 2024-08-17T17:06:06.031 INFO:tasks.rados.rados.0.smithi063.stdout:2587: done (10 left) 2024-08-17T17:06:06.031 INFO:tasks.rados.rados.0.smithi063.stdout:2589: done (9 left) 2024-08-17T17:06:06.031 INFO:tasks.rados.rados.0.smithi063.stdout:2590: done (8 left) 2024-08-17T17:06:06.031 INFO:tasks.rados.rados.0.smithi063.stdout:2591: done (7 left) 2024-08-17T17:06:06.032 INFO:tasks.rados.rados.0.smithi063.stdout:2592: done (6 left) 2024-08-17T17:06:06.032 INFO:tasks.rados.rados.0.smithi063.stdout:2593: done (5 left) 2024-08-17T17:06:06.032 INFO:tasks.rados.rados.0.smithi063.stdout:2594: done (4 left) 2024-08-17T17:06:06.032 INFO:tasks.rados.rados.0.smithi063.stdout:2596: done (3 left) 2024-08-17T17:06:06.032 INFO:tasks.rados.rados.0.smithi063.stdout:2598: done (2 left) 2024-08-17T17:06:06.032 INFO:tasks.rados.rados.0.smithi063.stdout:2600: snap_remove snap 336 2024-08-17T17:06:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:05 smithi162 ceph-mon[104668]: pgmap v561: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 196 MiB data, 760 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 1023 KiB/s wr, 7 op/s; 75/396 objects degraded (18.939%) 2024-08-17T17:06:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:05 smithi162 ceph-mon[104668]: osdmap e737: 8 total, 7 up, 8 in 2024-08-17T17:06:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:05 smithi162 ceph-mon[104668]: from='osd.0 [v2:172.21.15.63:6802/4285380350,v1:172.21.15.63:6803/4285380350]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:06:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:05 smithi162 ceph-mon[104668]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-08-17T17:06:06.529 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:06:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0[127633]: 2024-08-17T17:06:06.097+0000 7fc57f8c6640 -1 osd.0 726 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:06:06.616 INFO:tasks.rados.rados.0.smithi063.stdout:2600: done (2 left) 2024-08-17T17:06:06.616 INFO:tasks.rados.rados.0.smithi063.stdout:2601: read oid 7 snap 321 2024-08-17T17:06:06.616 INFO:tasks.rados.rados.0.smithi063.stdout:2601: expect (ObjNum 721 snap 246 seq_num 721) 2024-08-17T17:06:06.617 INFO:tasks.rados.rados.0.smithi063.stdout:2599: finishing rollback tid 2 to smithi06387500-3 2024-08-17T17:06:06.617 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 854 (ObjNum 917 snap 315 seq_num 917) dirty exists 2024-08-17T17:06:06.617 INFO:tasks.rados.rados.0.smithi063.stdout:2597: finishing write tid 1 to smithi06387500-9 2024-08-17T17:06:06.617 INFO:tasks.rados.rados.0.smithi063.stdout:2597: finishing write tid 2 to smithi06387500-9 2024-08-17T17:06:06.617 INFO:tasks.rados.rados.0.smithi063.stdout:2597: finishing write tid 3 to smithi06387500-9 2024-08-17T17:06:06.617 INFO:tasks.rados.rados.0.smithi063.stdout:2597: finishing write tid 4 to smithi06387500-9 2024-08-17T17:06:06.617 INFO:tasks.rados.rados.0.smithi063.stdout:2597: oid 9 updating version 0 to 1213 2024-08-17T17:06:06.617 INFO:tasks.rados.rados.0.smithi063.stdout:2597: oid 9 version 1213 is already newer than 1211 2024-08-17T17:06:06.617 INFO:tasks.rados.rados.0.smithi063.stdout:2597: oid 9 version 1213 is already newer than 1212 2024-08-17T17:06:06.617 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 1213 (ObjNum 977 snap 337 seq_num 977) dirty exists 2024-08-17T17:06:06.617 INFO:tasks.rados.rados.0.smithi063.stdout:2597: left oid 9 (ObjNum 977 snap 337 seq_num 977) 2024-08-17T17:06:06.617 INFO:tasks.rados.rados.0.smithi063.stdout:2597: done (2 left) 2024-08-17T17:06:06.618 INFO:tasks.rados.rados.0.smithi063.stdout:2599: done (1 left) 2024-08-17T17:06:06.618 INFO:tasks.rados.rados.0.smithi063.stdout:2602: rollback oid 44 current snap is 337 2024-08-17T17:06:06.618 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 44 to 331 2024-08-17T17:06:06.618 INFO:tasks.rados.rados.0.smithi063.stdout:2603: read oid 27 snap -1 2024-08-17T17:06:06.618 INFO:tasks.rados.rados.0.smithi063.stdout:2603: expect deleted 2024-08-17T17:06:06.618 INFO:tasks.rados.rados.0.smithi063.stdout:2604: read oid 42 snap -1 2024-08-17T17:06:06.618 INFO:tasks.rados.rados.0.smithi063.stdout:2604: expect deleted 2024-08-17T17:06:06.618 INFO:tasks.rados.rados.0.smithi063.stdout:2605: rollback oid 20 current snap is 337 2024-08-17T17:06:06.618 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 20 to 323 2024-08-17T17:06:06.619 INFO:tasks.rados.rados.0.smithi063.stdout:2606: write oid 41 current snap is 337 2024-08-17T17:06:06.619 INFO:tasks.rados.rados.0.smithi063.stdout:2606: seq_num 978 ranges {437246=486706,1363860=623668} 2024-08-17T17:06:06.621 INFO:tasks.rados.rados.0.smithi063.stdout:2606: writing smithi06387500-41 from 437246 to 923952 tid 1 2024-08-17T17:06:06.628 INFO:tasks.rados.rados.0.smithi063.stdout:2606: writing smithi06387500-41 from 1363860 to 1987528 tid 2 2024-08-17T17:06:06.629 INFO:tasks.rados.rados.0.smithi063.stdout:2607: snap_create 2024-08-17T17:06:06.629 INFO:tasks.rados.rados.0.smithi063.stdout:2606: finishing write tid 1 to smithi06387500-41 2024-08-17T17:06:06.632 INFO:tasks.rados.rados.0.smithi063.stdout:2606: finishing write tid 2 to smithi06387500-41 2024-08-17T17:06:06.632 INFO:tasks.rados.rados.0.smithi063.stdout:2606: finishing write tid 3 to smithi06387500-41 2024-08-17T17:06:06.632 INFO:tasks.rados.rados.0.smithi063.stdout:2606: finishing write tid 4 to smithi06387500-41 2024-08-17T17:06:06.632 INFO:tasks.rados.rados.0.smithi063.stdout:2606: oid 41 updating version 0 to 264 2024-08-17T17:06:06.633 INFO:tasks.rados.rados.0.smithi063.stdout:2606: oid 41 version 264 is already newer than 263 2024-08-17T17:06:06.633 INFO:tasks.rados.rados.0.smithi063.stdout:2606: oid 41 version 264 is already newer than 262 2024-08-17T17:06:06.633 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 264 (ObjNum 978 snap 337 seq_num 978) dirty exists 2024-08-17T17:06:06.633 INFO:tasks.rados.rados.0.smithi063.stdout:2606: left oid 41 (ObjNum 978 snap 337 seq_num 978) 2024-08-17T17:06:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:06 smithi063 ceph-mon[109843]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:06:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:06 smithi063 ceph-mon[109843]: osdmap e738: 8 total, 7 up, 8 in 2024-08-17T17:06:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:06 smithi063 ceph-mon[109843]: from='osd.0 [v2:172.21.15.63:6802/4285380350,v1:172.21.15.63:6803/4285380350]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]} : dispatch 2024-08-17T17:06:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:06 smithi063 ceph-mon[109843]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]} : dispatch 2024-08-17T17:06:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:06 smithi063 ceph-mon[109843]: from='osd.0 ' entity='osd.0' 2024-08-17T17:06:07.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:06 smithi063 ceph-mon[114756]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:06:07.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:06 smithi063 ceph-mon[114756]: osdmap e738: 8 total, 7 up, 8 in 2024-08-17T17:06:07.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:06 smithi063 ceph-mon[114756]: from='osd.0 [v2:172.21.15.63:6802/4285380350,v1:172.21.15.63:6803/4285380350]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]} : dispatch 2024-08-17T17:06:07.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:06 smithi063 ceph-mon[114756]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]} : dispatch 2024-08-17T17:06:07.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:06 smithi063 ceph-mon[114756]: from='osd.0 ' entity='osd.0' 2024-08-17T17:06:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:06 smithi162 ceph-mon[104668]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-08-17T17:06:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:06 smithi162 ceph-mon[104668]: osdmap e738: 8 total, 7 up, 8 in 2024-08-17T17:06:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:06 smithi162 ceph-mon[104668]: from='osd.0 [v2:172.21.15.63:6802/4285380350,v1:172.21.15.63:6803/4285380350]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]} : dispatch 2024-08-17T17:06:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:06 smithi162 ceph-mon[104668]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi063", "root=default"]} : dispatch 2024-08-17T17:06:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:06 smithi162 ceph-mon[104668]: from='osd.0 ' entity='osd.0' 2024-08-17T17:06:07.627 INFO:tasks.rados.rados.0.smithi063.stdout:2603: done (6 left) 2024-08-17T17:06:07.627 INFO:tasks.rados.rados.0.smithi063.stdout:2606: done (5 left) 2024-08-17T17:06:07.627 INFO:tasks.rados.rados.0.smithi063.stdout:2607: done (4 left) 2024-08-17T17:06:07.627 INFO:tasks.rados.rados.0.smithi063.stdout:2608: write oid 24 current snap is 338 2024-08-17T17:06:07.627 INFO:tasks.rados.rados.0.smithi063.stdout:2608: seq_num 979 ranges {454053=561955,1422342=483679,2368404=85649} 2024-08-17T17:06:07.633 INFO:tasks.rados.rados.0.smithi063.stdout:2608: writing smithi06387500-24 from 454053 to 1016008 tid 1 2024-08-17T17:06:07.641 INFO:tasks.rados.rados.0.smithi063.stdout:2608: writing smithi06387500-24 from 1422342 to 1906021 tid 2 2024-08-17T17:06:07.645 INFO:tasks.rados.rados.0.smithi063.stdout:2608: writing smithi06387500-24 from 2368404 to 2454053 tid 3 2024-08-17T17:06:07.645 INFO:tasks.rados.rados.0.smithi063.stdout:2609: read oid 34 snap 331 2024-08-17T17:06:07.645 INFO:tasks.rados.rados.0.smithi063.stdout:2609: expect (ObjNum 953 snap 325 seq_num 953) 2024-08-17T17:06:07.645 INFO:tasks.rados.rados.0.smithi063.stdout:2610: rollback oid 6 current snap is 338 2024-08-17T17:06:07.645 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 6 to 331 2024-08-17T17:06:07.645 INFO:tasks.rados.rados.0.smithi063.stdout:2601: expect (ObjNum 721 snap 246 seq_num 721) 2024-08-17T17:06:07.753 INFO:tasks.rados.rados.0.smithi063.stdout:2608: finishing write tid 1 to smithi06387500-24 2024-08-17T17:06:07.753 INFO:tasks.rados.rados.0.smithi063.stdout:2608: finishing write tid 2 to smithi06387500-24 2024-08-17T17:06:07.753 INFO:tasks.rados.rados.0.smithi063.stdout:2608: finishing write tid 3 to smithi06387500-24 2024-08-17T17:06:07.753 INFO:tasks.rados.rados.0.smithi063.stdout:2608: finishing write tid 4 to smithi06387500-24 2024-08-17T17:06:07.753 INFO:tasks.rados.rados.0.smithi063.stdout:2608: finishing write tid 5 to smithi06387500-24 2024-08-17T17:06:07.753 INFO:tasks.rados.rados.0.smithi063.stdout:2608: oid 24 updating version 0 to 964 2024-08-17T17:06:07.753 INFO:tasks.rados.rados.0.smithi063.stdout:2608: oid 24 version 964 is already newer than 963 2024-08-17T17:06:07.753 INFO:tasks.rados.rados.0.smithi063.stdout:2608: oid 24 updating version 964 to 966 2024-08-17T17:06:07.753 INFO:tasks.rados.rados.0.smithi063.stdout:2608: oid 24 version 966 is already newer than 965 2024-08-17T17:06:07.753 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 966 (ObjNum 979 snap 338 seq_num 979) dirty exists 2024-08-17T17:06:07.753 INFO:tasks.rados.rados.0.smithi063.stdout:2608: left oid 24 (ObjNum 979 snap 338 seq_num 979) 2024-08-17T17:06:07.753 INFO:tasks.rados.rados.0.smithi063.stdout:2602: finishing rollback tid 1 to smithi06387500-44 2024-08-17T17:06:07.754 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 361 (ObjNum 33 snap 0 seq_num 1807085712) dirty dne 2024-08-17T17:06:07.754 INFO:tasks.rados.rados.0.smithi063.stdout:2610: finishing rollback tid 0 to smithi06387500-6 2024-08-17T17:06:07.754 INFO:tasks.rados.rados.0.smithi063.stdout:2610: finishing rollback tid 1 to smithi06387500-6 2024-08-17T17:06:07.754 INFO:tasks.rados.rados.0.smithi063.stdout:2610: finishing rollback tid 2 to smithi06387500-6 2024-08-17T17:06:07.754 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 893 (ObjNum 900 snap 311 seq_num 900) dirty exists 2024-08-17T17:06:07.754 INFO:tasks.rados.rados.0.smithi063.stdout:2605: finishing rollback tid 1 to smithi06387500-20 2024-08-17T17:06:07.754 INFO:tasks.rados.rados.0.smithi063.stdout:2601: done (6 left) 2024-08-17T17:06:07.754 INFO:tasks.rados.rados.0.smithi063.stdout:2602: done (5 left) 2024-08-17T17:06:07.754 INFO:tasks.rados.rados.0.smithi063.stdout:2604: done (4 left) 2024-08-17T17:06:07.754 INFO:tasks.rados.rados.0.smithi063.stdout:2608: done (3 left) 2024-08-17T17:06:07.754 INFO:tasks.rados.rados.0.smithi063.stdout:2610: done (2 left) 2024-08-17T17:06:07.754 INFO:tasks.rados.rados.0.smithi063.stdout:2611: read oid 37 snap 321 2024-08-17T17:06:07.754 INFO:tasks.rados.rados.0.smithi063.stdout:2611: expect (ObjNum 696 snap 222 seq_num 696) 2024-08-17T17:06:07.755 INFO:tasks.rados.rados.0.smithi063.stdout:2612: read oid 30 snap 320 2024-08-17T17:06:07.755 INFO:tasks.rados.rados.0.smithi063.stdout:2612: expect (ObjNum 710 snap 234 seq_num 710) 2024-08-17T17:06:07.755 INFO:tasks.rados.rados.0.smithi063.stdout:2613: delete oid 35 current snap is 338 2024-08-17T17:06:07.755 INFO:tasks.rados.rados.0.smithi063.stdout:2605: finishing rollback tid 2 to smithi06387500-20 2024-08-17T17:06:07.755 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 542 (ObjNum 945 snap 323 seq_num 945) dirty exists 2024-08-17T17:06:07.755 INFO:tasks.rados.rados.0.smithi063.stdout:2609: expect (ObjNum 953 snap 325 seq_num 953) 2024-08-17T17:06:07.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T17:06:07.583+0000 7f4652b73640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-08-17T17:06:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: pgmap v564: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 196 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 75/396 objects degraded (18.939%) 2024-08-17T17:06:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:06:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:06:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: osd.0 [v2:172.21.15.63:6802/4285380350,v1:172.21.15.63:6803/4285380350] boot 2024-08-17T17:06:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: osdmap e739: 8 total, 8 up, 8 in 2024-08-17T17:06:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:07.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:07.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:06:07.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:07.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:07.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:07.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:07.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:07.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:06:07.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 75/396 objects degraded (18.939%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:07.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: pgmap v564: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 196 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 75/396 objects degraded (18.939%) 2024-08-17T17:06:07.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:06:07.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:06:07.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: osd.0 [v2:172.21.15.63:6802/4285380350,v1:172.21.15.63:6803/4285380350] boot 2024-08-17T17:06:07.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: osdmap e739: 8 total, 8 up, 8 in 2024-08-17T17:06:07.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:07.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:07.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:07.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:06:07.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:07.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:07.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:07.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:07.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:07.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:06:07.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:07 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 75/396 objects degraded (18.939%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:07.818 INFO:tasks.rados.rados.0.smithi063.stdout:2611: expect (ObjNum 696 snap 222 seq_num 696) 2024-08-17T17:06:08.040 INFO:tasks.rados.rados.0.smithi063.stdout:2612: expect (ObjNum 710 snap 234 seq_num 710) 2024-08-17T17:06:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: pgmap v564: 89 pgs: 19 active+undersized, 15 active+undersized+degraded, 55 active+clean; 196 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 75/396 objects degraded (18.939%) 2024-08-17T17:06:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:06:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-08-17T17:06:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: osd.0 [v2:172.21.15.63:6802/4285380350,v1:172.21.15.63:6803/4285380350] boot 2024-08-17T17:06:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: osdmap e739: 8 total, 8 up, 8 in 2024-08-17T17:06:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:06:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:08.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:06:08.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:07 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 75/396 objects degraded (18.939%), 15 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:08.171 INFO:tasks.rados.rados.0.smithi063.stdout:2605: done (4 left) 2024-08-17T17:06:08.172 INFO:tasks.rados.rados.0.smithi063.stdout:2609: done (3 left) 2024-08-17T17:06:08.172 INFO:tasks.rados.rados.0.smithi063.stdout:2611: done (2 left) 2024-08-17T17:06:08.172 INFO:tasks.rados.rados.0.smithi063.stdout:2612: done (1 left) 2024-08-17T17:06:08.172 INFO:tasks.rados.rados.0.smithi063.stdout:2613: done (0 left) 2024-08-17T17:06:08.172 INFO:tasks.rados.rados.0.smithi063.stdout:2614: snap_remove snap 333 2024-08-17T17:06:08.629 INFO:tasks.rados.rados.0.smithi063.stdout:2614: done (0 left) 2024-08-17T17:06:08.629 INFO:tasks.rados.rados.0.smithi063.stdout:2615: delete oid 44 current snap is 338 2024-08-17T17:06:08.635 INFO:tasks.rados.rados.0.smithi063.stdout:2615: done (0 left) 2024-08-17T17:06:08.635 INFO:tasks.rados.rados.0.smithi063.stdout:2616: read oid 39 snap -1 2024-08-17T17:06:08.636 INFO:tasks.rados.rados.0.smithi063.stdout:2616: expect (ObjNum 943 snap 323 seq_num 943) 2024-08-17T17:06:08.636 INFO:tasks.rados.rados.0.smithi063.stdout:2617: delete oid 24 current snap is 338 2024-08-17T17:06:08.640 INFO:tasks.rados.rados.0.smithi063.stdout:2617: done (1 left) 2024-08-17T17:06:08.640 INFO:tasks.rados.rados.0.smithi063.stdout:2618: delete oid 28 current snap is 338 2024-08-17T17:06:08.642 INFO:tasks.rados.rados.0.smithi063.stdout:2618: done (1 left) 2024-08-17T17:06:08.642 INFO:tasks.rados.rados.0.smithi063.stdout:2619: snap_remove snap 321 2024-08-17T17:06:09.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:08 smithi063 ceph-mon[114756]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:06:09.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:08 smithi063 ceph-mon[114756]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-08-17T17:06:09.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:08 smithi063 ceph-mon[114756]: osdmap e740: 8 total, 8 up, 8 in 2024-08-17T17:06:09.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:08 smithi063 ceph-mon[109843]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:06:09.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:08 smithi063 ceph-mon[109843]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-08-17T17:06:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:08 smithi063 ceph-mon[109843]: osdmap e740: 8 total, 8 up, 8 in 2024-08-17T17:06:09.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:08 smithi162 ceph-mon[104668]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:06:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:08 smithi162 ceph-mon[104668]: Upgrade: unsafe to stop osd(s) at this time (9 PGs are or would become offline) 2024-08-17T17:06:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:08 smithi162 ceph-mon[104668]: osdmap e740: 8 total, 8 up, 8 in 2024-08-17T17:06:09.636 INFO:tasks.rados.rados.0.smithi063.stdout:2619: done (1 left) 2024-08-17T17:06:09.636 INFO:tasks.rados.rados.0.smithi063.stdout:2620: snap_create 2024-08-17T17:06:09.636 INFO:tasks.rados.rados.0.smithi063.stdout:2616: expect (ObjNum 943 snap 323 seq_num 943) 2024-08-17T17:06:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:09 smithi063 ceph-mon[114756]: pgmap v567: 89 pgs: 21 peering, 9 active+undersized, 4 active+undersized+degraded, 55 active+clean; 202 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 942 KiB/s wr, 13 op/s; 24/408 objects degraded (5.882%) 2024-08-17T17:06:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:09 smithi063 ceph-mon[114756]: osdmap e741: 8 total, 8 up, 8 in 2024-08-17T17:06:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:09 smithi063 ceph-mon[109843]: pgmap v567: 89 pgs: 21 peering, 9 active+undersized, 4 active+undersized+degraded, 55 active+clean; 202 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 942 KiB/s wr, 13 op/s; 24/408 objects degraded (5.882%) 2024-08-17T17:06:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:09 smithi063 ceph-mon[109843]: osdmap e741: 8 total, 8 up, 8 in 2024-08-17T17:06:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:09 smithi162 ceph-mon[104668]: pgmap v567: 89 pgs: 21 peering, 9 active+undersized, 4 active+undersized+degraded, 55 active+clean; 202 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 942 KiB/s wr, 13 op/s; 24/408 objects degraded (5.882%) 2024-08-17T17:06:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:09 smithi162 ceph-mon[104668]: osdmap e741: 8 total, 8 up, 8 in 2024-08-17T17:06:10.649 INFO:tasks.rados.rados.0.smithi063.stdout:2616: done (1 left) 2024-08-17T17:06:10.649 INFO:tasks.rados.rados.0.smithi063.stdout:2620: done (0 left) 2024-08-17T17:06:10.649 INFO:tasks.rados.rados.0.smithi063.stdout:2621: snap_remove snap 326 2024-08-17T17:06:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:10 smithi063 ceph-mon[114756]: osdmap e742: 8 total, 8 up, 8 in 2024-08-17T17:06:11.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:10 smithi063 ceph-mon[109843]: osdmap e742: 8 total, 8 up, 8 in 2024-08-17T17:06:11.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:10 smithi162 ceph-mon[104668]: osdmap e742: 8 total, 8 up, 8 in 2024-08-17T17:06:11.650 INFO:tasks.rados.rados.0.smithi063.stdout:2621: done (0 left) 2024-08-17T17:06:11.650 INFO:tasks.rados.rados.0.smithi063.stdout:2622: rollback oid 42 current snap is 339 2024-08-17T17:06:11.650 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 42 to 307 2024-08-17T17:06:11.650 INFO:tasks.rados.rados.0.smithi063.stdout:2623: rollback oid 5 current snap is 339 2024-08-17T17:06:11.650 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 329 2024-08-17T17:06:11.651 INFO:tasks.rados.rados.0.smithi063.stdout:2624: delete oid 47 current snap is 339 2024-08-17T17:06:11.655 INFO:tasks.rados.rados.0.smithi063.stdout:2623: finishing rollback tid 1 to smithi06387500-5 2024-08-17T17:06:11.655 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1173 (ObjNum 49 snap 0 seq_num 1786885104) dirty dne 2024-08-17T17:06:11.656 INFO:tasks.rados.rados.0.smithi063.stdout:2622: finishing rollback tid 1 to smithi06387500-42 2024-08-17T17:06:11.656 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 0 (ObjNum 97 snap 0 seq_num 1806939648) dirty dne 2024-08-17T17:06:11.657 INFO:tasks.rados.rados.0.smithi063.stdout:2622: done (2 left) 2024-08-17T17:06:11.657 INFO:tasks.rados.rados.0.smithi063.stdout:2623: done (1 left) 2024-08-17T17:06:11.657 INFO:tasks.rados.rados.0.smithi063.stdout:2624: done (0 left) 2024-08-17T17:06:11.657 INFO:tasks.rados.rados.0.smithi063.stdout:2625: write oid 1 current snap is 339 2024-08-17T17:06:11.657 INFO:tasks.rados.rados.0.smithi063.stdout:2625: seq_num 985 ranges {554895=729802,1773234=778333} 2024-08-17T17:06:11.664 INFO:tasks.rados.rados.0.smithi063.stdout:2625: writing smithi06387500-1 from 554895 to 1284697 tid 1 2024-08-17T17:06:11.671 INFO:tasks.rados.rados.0.smithi063.stdout:2625: writing smithi06387500-1 from 1773234 to 2551567 tid 2 2024-08-17T17:06:11.671 INFO:tasks.rados.rados.0.smithi063.stdout:2626: rollback oid 35 current snap is 339 2024-08-17T17:06:11.671 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 35 to 334 2024-08-17T17:06:11.671 INFO:tasks.rados.rados.0.smithi063.stdout:2625: finishing write tid 1 to smithi06387500-1 2024-08-17T17:06:11.671 INFO:tasks.rados.rados.0.smithi063.stdout:2627: rollback oid 14 current snap is 339 2024-08-17T17:06:11.672 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 14 to 335 2024-08-17T17:06:11.672 INFO:tasks.rados.rados.0.smithi063.stdout:2628: delete oid 16 current snap is 339 2024-08-17T17:06:11.678 INFO:tasks.rados.rados.0.smithi063.stdout:2625: finishing write tid 2 to smithi06387500-1 2024-08-17T17:06:11.678 INFO:tasks.rados.rados.0.smithi063.stdout:2625: finishing write tid 3 to smithi06387500-1 2024-08-17T17:06:11.678 INFO:tasks.rados.rados.0.smithi063.stdout:2625: finishing write tid 4 to smithi06387500-1 2024-08-17T17:06:11.678 INFO:tasks.rados.rados.0.smithi063.stdout:2625: oid 1 updating version 0 to 1289 2024-08-17T17:06:11.678 INFO:tasks.rados.rados.0.smithi063.stdout:2625: oid 1 updating version 1289 to 1290 2024-08-17T17:06:11.678 INFO:tasks.rados.rados.0.smithi063.stdout:2625: oid 1 version 1290 is already newer than 1288 2024-08-17T17:06:11.678 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 1290 (ObjNum 985 snap 339 seq_num 985) dirty exists 2024-08-17T17:06:11.678 INFO:tasks.rados.rados.0.smithi063.stdout:2625: left oid 1 (ObjNum 985 snap 339 seq_num 985) 2024-08-17T17:06:11.678 INFO:tasks.rados.rados.0.smithi063.stdout:2625: done (3 left) 2024-08-17T17:06:11.678 INFO:tasks.rados.rados.0.smithi063.stdout:2628: done (2 left) 2024-08-17T17:06:11.678 INFO:tasks.rados.rados.0.smithi063.stdout:2629: read oid 38 snap -1 2024-08-17T17:06:11.678 INFO:tasks.rados.rados.0.smithi063.stdout:2629: expect (ObjNum 868 snap 300 seq_num 868) 2024-08-17T17:06:11.679 INFO:tasks.rados.rados.0.smithi063.stdout:2630: write oid 2 current snap is 339 2024-08-17T17:06:11.679 INFO:tasks.rados.rados.0.smithi063.stdout:2626: finishing rollback tid 1 to smithi06387500-35 2024-08-17T17:06:11.679 INFO:tasks.rados.rados.0.smithi063.stdout:2626: finishing rollback tid 2 to smithi06387500-35 2024-08-17T17:06:11.679 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 2580 (ObjNum 941 snap 323 seq_num 941) dirty exists 2024-08-17T17:06:11.679 INFO:tasks.rados.rados.0.smithi063.stdout:2630: seq_num 987 ranges {588509=480300,1773845=782002} 2024-08-17T17:06:11.683 INFO:tasks.rados.rados.0.smithi063.stdout:2630: writing smithi06387500-2 from 588509 to 1068809 tid 1 2024-08-17T17:06:11.692 INFO:tasks.rados.rados.0.smithi063.stdout:2630: writing smithi06387500-2 from 1773845 to 2555847 tid 2 2024-08-17T17:06:11.692 INFO:tasks.rados.rados.0.smithi063.stdout:2626: done (3 left) 2024-08-17T17:06:11.693 INFO:tasks.rados.rados.0.smithi063.stdout:2631: read oid 4 snap -1 2024-08-17T17:06:11.693 INFO:tasks.rados.rados.0.smithi063.stdout:2631: expect (ObjNum 965 snap 328 seq_num 965) 2024-08-17T17:06:11.693 INFO:tasks.rados.rados.0.smithi063.stdout:2632: read oid 35 snap -1 2024-08-17T17:06:11.693 INFO:tasks.rados.rados.0.smithi063.stdout:2632: expect (ObjNum 941 snap 323 seq_num 941) 2024-08-17T17:06:11.693 INFO:tasks.rados.rados.0.smithi063.stdout:2633: snap_create 2024-08-17T17:06:11.693 INFO:tasks.rados.rados.0.smithi063.stdout:2627: finishing rollback tid 1 to smithi06387500-14 2024-08-17T17:06:11.693 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 2484 (ObjNum 33 snap 0 seq_num 1806963040) dirty dne 2024-08-17T17:06:11.693 INFO:tasks.rados.rados.0.smithi063.stdout:2629: expect (ObjNum 868 snap 300 seq_num 868) 2024-08-17T17:06:11.813 INFO:tasks.rados.rados.0.smithi063.stdout:2630: finishing write tid 1 to smithi06387500-2 2024-08-17T17:06:11.813 INFO:tasks.rados.rados.0.smithi063.stdout:2630: finishing write tid 2 to smithi06387500-2 2024-08-17T17:06:11.813 INFO:tasks.rados.rados.0.smithi063.stdout:2630: finishing write tid 3 to smithi06387500-2 2024-08-17T17:06:11.813 INFO:tasks.rados.rados.0.smithi063.stdout:2630: finishing write tid 4 to smithi06387500-2 2024-08-17T17:06:11.813 INFO:tasks.rados.rados.0.smithi063.stdout:2630: oid 2 updating version 0 to 1294 2024-08-17T17:06:11.814 INFO:tasks.rados.rados.0.smithi063.stdout:2630: oid 2 version 1294 is already newer than 1292 2024-08-17T17:06:11.814 INFO:tasks.rados.rados.0.smithi063.stdout:2630: oid 2 version 1294 is already newer than 1293 2024-08-17T17:06:11.814 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 1294 (ObjNum 987 snap 339 seq_num 987) dirty exists 2024-08-17T17:06:11.814 INFO:tasks.rados.rados.0.smithi063.stdout:2630: left oid 2 (ObjNum 987 snap 339 seq_num 987) 2024-08-17T17:06:11.814 INFO:tasks.rados.rados.0.smithi063.stdout:2631: expect (ObjNum 965 snap 328 seq_num 965) 2024-08-17T17:06:11.888 INFO:tasks.rados.rados.0.smithi063.stdout:2632: expect (ObjNum 941 snap 323 seq_num 941) 2024-08-17T17:06:11.930 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:11 smithi063 ceph-mon[114756]: pgmap v570: 89 pgs: 21 peering, 9 active+undersized, 4 active+undersized+degraded, 55 active+clean; 202 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 942 KiB/s wr, 13 op/s; 24/408 objects degraded (5.882%) 2024-08-17T17:06:11.930 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:11 smithi063 ceph-mon[114756]: osdmap e743: 8 total, 8 up, 8 in 2024-08-17T17:06:11.930 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:11 smithi063 ceph-mon[109843]: pgmap v570: 89 pgs: 21 peering, 9 active+undersized, 4 active+undersized+degraded, 55 active+clean; 202 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 942 KiB/s wr, 13 op/s; 24/408 objects degraded (5.882%) 2024-08-17T17:06:11.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:11 smithi063 ceph-mon[109843]: osdmap e743: 8 total, 8 up, 8 in 2024-08-17T17:06:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:11 smithi162 ceph-mon[104668]: pgmap v570: 89 pgs: 21 peering, 9 active+undersized, 4 active+undersized+degraded, 55 active+clean; 202 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 942 KiB/s wr, 13 op/s; 24/408 objects degraded (5.882%) 2024-08-17T17:06:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:11 smithi162 ceph-mon[104668]: osdmap e743: 8 total, 8 up, 8 in 2024-08-17T17:06:12.660 INFO:tasks.rados.rados.0.smithi063.stdout:2627: done (5 left) 2024-08-17T17:06:12.660 INFO:tasks.rados.rados.0.smithi063.stdout:2629: done (4 left) 2024-08-17T17:06:12.660 INFO:tasks.rados.rados.0.smithi063.stdout:2630: done (3 left) 2024-08-17T17:06:12.660 INFO:tasks.rados.rados.0.smithi063.stdout:2631: done (2 left) 2024-08-17T17:06:12.660 INFO:tasks.rados.rados.0.smithi063.stdout:2632: done (1 left) 2024-08-17T17:06:12.660 INFO:tasks.rados.rados.0.smithi063.stdout:2633: done (0 left) 2024-08-17T17:06:12.660 INFO:tasks.rados.rados.0.smithi063.stdout:2634: write oid 20 current snap is 340 2024-08-17T17:06:12.660 INFO:tasks.rados.rados.0.smithi063.stdout:2634: seq_num 988 ranges {605316=555549,1832327=642013} 2024-08-17T17:06:12.666 INFO:tasks.rados.rados.0.smithi063.stdout:2634: writing smithi06387500-20 from 605316 to 1160865 tid 1 2024-08-17T17:06:12.674 INFO:tasks.rados.rados.0.smithi063.stdout:2634: writing smithi06387500-20 from 1832327 to 2474340 tid 2 2024-08-17T17:06:12.674 INFO:tasks.rados.rados.0.smithi063.stdout:2635: rollback oid 39 current snap is 340 2024-08-17T17:06:12.674 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 39 to 338 2024-08-17T17:06:12.674 INFO:tasks.rados.rados.0.smithi063.stdout:2634: finishing write tid 1 to smithi06387500-20 2024-08-17T17:06:12.674 INFO:tasks.rados.rados.0.smithi063.stdout:2636: delete oid 9 current snap is 340 2024-08-17T17:06:12.677 INFO:tasks.rados.rados.0.smithi063.stdout:2634: finishing write tid 2 to smithi06387500-20 2024-08-17T17:06:12.677 INFO:tasks.rados.rados.0.smithi063.stdout:2634: finishing write tid 3 to smithi06387500-20 2024-08-17T17:06:12.677 INFO:tasks.rados.rados.0.smithi063.stdout:2634: finishing write tid 4 to smithi06387500-20 2024-08-17T17:06:12.677 INFO:tasks.rados.rados.0.smithi063.stdout:2634: oid 20 updating version 0 to 556 2024-08-17T17:06:12.677 INFO:tasks.rados.rados.0.smithi063.stdout:2634: oid 20 version 556 is already newer than 555 2024-08-17T17:06:12.677 INFO:tasks.rados.rados.0.smithi063.stdout:2634: oid 20 updating version 556 to 557 2024-08-17T17:06:12.678 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 557 (ObjNum 988 snap 340 seq_num 988) dirty exists 2024-08-17T17:06:12.678 INFO:tasks.rados.rados.0.smithi063.stdout:2634: left oid 20 (ObjNum 988 snap 340 seq_num 988) 2024-08-17T17:06:12.679 INFO:tasks.rados.rados.0.smithi063.stdout:2634: done (2 left) 2024-08-17T17:06:12.679 INFO:tasks.rados.rados.0.smithi063.stdout:2636: done (1 left) 2024-08-17T17:06:12.679 INFO:tasks.rados.rados.0.smithi063.stdout:2637: read oid 12 snap -1 2024-08-17T17:06:12.679 INFO:tasks.rados.rados.0.smithi063.stdout:2637: expect (ObjNum 905 snap 311 seq_num 905) 2024-08-17T17:06:12.679 INFO:tasks.rados.rados.0.smithi063.stdout:2638: write oid 30 current snap is 340 2024-08-17T17:06:12.679 INFO:tasks.rados.rados.0.smithi063.stdout:2638: seq_num 990 ranges {638930=422400,1665644=645682} 2024-08-17T17:06:12.684 INFO:tasks.rados.rados.0.smithi063.stdout:2638: writing smithi06387500-30 from 638930 to 1061330 tid 1 2024-08-17T17:06:12.692 INFO:tasks.rados.rados.0.smithi063.stdout:2638: writing smithi06387500-30 from 1665644 to 2311326 tid 2 2024-08-17T17:06:12.692 INFO:tasks.rados.rados.0.smithi063.stdout:2639: read oid 3 snap -1 2024-08-17T17:06:12.692 INFO:tasks.rados.rados.0.smithi063.stdout:2639: expect (ObjNum 917 snap 315 seq_num 917) 2024-08-17T17:06:12.692 INFO:tasks.rados.rados.0.smithi063.stdout:2640: snap_remove snap 327 2024-08-17T17:06:12.945 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:12 smithi063 ceph-mon[109843]: osdmap e744: 8 total, 8 up, 8 in 2024-08-17T17:06:12.945 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:12 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 24/408 objects degraded (5.882%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:12.946 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:12 smithi063 ceph-mon[114756]: osdmap e744: 8 total, 8 up, 8 in 2024-08-17T17:06:12.946 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:12 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 24/408 objects degraded (5.882%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:12 smithi162 ceph-mon[104668]: osdmap e744: 8 total, 8 up, 8 in 2024-08-17T17:06:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:12 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 24/408 objects degraded (5.882%), 4 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:06:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:06:13] "GET /metrics HTTP/1.1" 200 36340 "" "Prometheus/2.51.0" 2024-08-17T17:06:13.663 INFO:tasks.rados.rados.0.smithi063.stdout:2640: done (4 left) 2024-08-17T17:06:13.664 INFO:tasks.rados.rados.0.smithi063.stdout:2641: write oid 4 current snap is 340 2024-08-17T17:06:13.664 INFO:tasks.rados.rados.0.smithi063.stdout:2641: seq_num 991 ranges {655737=497649,1607773=789340} 2024-08-17T17:06:13.671 INFO:tasks.rados.rados.0.smithi063.stdout:2641: writing smithi06387500-4 from 655737 to 1153386 tid 1 2024-08-17T17:06:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2641: writing smithi06387500-4 from 1607773 to 2397113 tid 2 2024-08-17T17:06:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2642: write oid 25 current snap is 340 2024-08-17T17:06:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2635: finishing rollback tid 0 to smithi06387500-39 2024-08-17T17:06:13.678 INFO:tasks.rados.rados.0.smithi063.stdout:2635: finishing rollback tid 1 to smithi06387500-39 2024-08-17T17:06:13.679 INFO:tasks.rados.rados.0.smithi063.stdout:2635: finishing rollback tid 2 to smithi06387500-39 2024-08-17T17:06:13.679 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 607 (ObjNum 943 snap 323 seq_num 943) dirty exists 2024-08-17T17:06:13.679 INFO:tasks.rados.rados.0.smithi063.stdout:2637: expect (ObjNum 905 snap 311 seq_num 905) 2024-08-17T17:06:13.799 INFO:tasks.rados.rados.0.smithi063.stdout:2638: finishing write tid 1 to smithi06387500-30 2024-08-17T17:06:13.799 INFO:tasks.rados.rados.0.smithi063.stdout:2638: finishing write tid 2 to smithi06387500-30 2024-08-17T17:06:13.799 INFO:tasks.rados.rados.0.smithi063.stdout:2638: finishing write tid 3 to smithi06387500-30 2024-08-17T17:06:13.799 INFO:tasks.rados.rados.0.smithi063.stdout:2639: expect (ObjNum 917 snap 315 seq_num 917) 2024-08-17T17:06:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:13 smithi063 ceph-mon[114756]: pgmap v573: 89 pgs: 89 active+clean; 207 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 953 KiB/s wr, 24 op/s; 2.3 MiB/s, 2 objects/s recovering 2024-08-17T17:06:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:13 smithi063 ceph-mon[114756]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/408 objects degraded (5.882%), 4 pgs degraded) 2024-08-17T17:06:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:13 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:06:14.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:13 smithi063 ceph-mon[114756]: osdmap e745: 8 total, 8 up, 8 in 2024-08-17T17:06:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:13 smithi063 ceph-mon[109843]: pgmap v573: 89 pgs: 89 active+clean; 207 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 953 KiB/s wr, 24 op/s; 2.3 MiB/s, 2 objects/s recovering 2024-08-17T17:06:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:13 smithi063 ceph-mon[109843]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/408 objects degraded (5.882%), 4 pgs degraded) 2024-08-17T17:06:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:13 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:06:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:13 smithi063 ceph-mon[109843]: osdmap e745: 8 total, 8 up, 8 in 2024-08-17T17:06:14.033 INFO:tasks.rados.rados.0.smithi063.stdout:2638: finishing write tid 4 to smithi06387500-30 2024-08-17T17:06:14.033 INFO:tasks.rados.rados.0.smithi063.stdout:2638: oid 30 updating version 0 to 1241 2024-08-17T17:06:14.033 INFO:tasks.rados.rados.0.smithi063.stdout:2638: oid 30 version 1241 is already newer than 1239 2024-08-17T17:06:14.033 INFO:tasks.rados.rados.0.smithi063.stdout:2638: oid 30 version 1241 is already newer than 1240 2024-08-17T17:06:14.033 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1241 (ObjNum 990 snap 340 seq_num 990) dirty exists 2024-08-17T17:06:14.033 INFO:tasks.rados.rados.0.smithi063.stdout:2638: left oid 30 (ObjNum 990 snap 340 seq_num 990) 2024-08-17T17:06:14.033 INFO:tasks.rados.rados.0.smithi063.stdout:2641: finishing write tid 1 to smithi06387500-4 2024-08-17T17:06:14.034 INFO:tasks.rados.rados.0.smithi063.stdout:2641: finishing write tid 2 to smithi06387500-4 2024-08-17T17:06:14.034 INFO:tasks.rados.rados.0.smithi063.stdout:2641: finishing write tid 3 to smithi06387500-4 2024-08-17T17:06:14.034 INFO:tasks.rados.rados.0.smithi063.stdout:2641: finishing write tid 4 to smithi06387500-4 2024-08-17T17:06:14.034 INFO:tasks.rados.rados.0.smithi063.stdout:2641: oid 4 updating version 0 to 2586 2024-08-17T17:06:14.034 INFO:tasks.rados.rados.0.smithi063.stdout:2641: oid 4 version 2586 is already newer than 2585 2024-08-17T17:06:14.034 INFO:tasks.rados.rados.0.smithi063.stdout:2641: oid 4 updating version 2586 to 2587 2024-08-17T17:06:14.034 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 2587 (ObjNum 991 snap 340 seq_num 991) dirty exists 2024-08-17T17:06:14.034 INFO:tasks.rados.rados.0.smithi063.stdout:2641: left oid 4 (ObjNum 991 snap 340 seq_num 991) 2024-08-17T17:06:14.034 INFO:tasks.rados.rados.0.smithi063.stdout:2642: seq_num 992 ranges {672544=572898,1666255=532998} 2024-08-17T17:06:14.040 INFO:tasks.rados.rados.0.smithi063.stdout:2642: writing smithi06387500-25 from 672544 to 1245442 tid 1 2024-08-17T17:06:14.045 INFO:tasks.rados.rados.0.smithi063.stdout:2642: writing smithi06387500-25 from 1666255 to 2199253 tid 2 2024-08-17T17:06:14.045 INFO:tasks.rados.rados.0.smithi063.stdout:2635: done (5 left) 2024-08-17T17:06:14.045 INFO:tasks.rados.rados.0.smithi063.stdout:2637: done (4 left) 2024-08-17T17:06:14.045 INFO:tasks.rados.rados.0.smithi063.stdout:2638: done (3 left) 2024-08-17T17:06:14.045 INFO:tasks.rados.rados.0.smithi063.stdout:2639: done (2 left) 2024-08-17T17:06:14.045 INFO:tasks.rados.rados.0.smithi063.stdout:2641: done (1 left) 2024-08-17T17:06:14.046 INFO:tasks.rados.rados.0.smithi063.stdout:2643: snap_create 2024-08-17T17:06:14.046 INFO:tasks.rados.rados.0.smithi063.stdout:2642: finishing write tid 1 to smithi06387500-25 2024-08-17T17:06:14.050 INFO:tasks.rados.rados.0.smithi063.stdout:2642: finishing write tid 2 to smithi06387500-25 2024-08-17T17:06:14.050 INFO:tasks.rados.rados.0.smithi063.stdout:2642: finishing write tid 3 to smithi06387500-25 2024-08-17T17:06:14.050 INFO:tasks.rados.rados.0.smithi063.stdout:2642: finishing write tid 4 to smithi06387500-25 2024-08-17T17:06:14.050 INFO:tasks.rados.rados.0.smithi063.stdout:2642: oid 25 updating version 0 to 2589 2024-08-17T17:06:14.050 INFO:tasks.rados.rados.0.smithi063.stdout:2642: oid 25 updating version 2589 to 2590 2024-08-17T17:06:14.050 INFO:tasks.rados.rados.0.smithi063.stdout:2642: oid 25 updating version 2590 to 2591 2024-08-17T17:06:14.051 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 2591 (ObjNum 992 snap 340 seq_num 992) dirty exists 2024-08-17T17:06:14.051 INFO:tasks.rados.rados.0.smithi063.stdout:2642: left oid 25 (ObjNum 992 snap 340 seq_num 992) 2024-08-17T17:06:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:13 smithi162 ceph-mon[104668]: pgmap v573: 89 pgs: 89 active+clean; 207 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 953 KiB/s wr, 24 op/s; 2.3 MiB/s, 2 objects/s recovering 2024-08-17T17:06:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:13 smithi162 ceph-mon[104668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/408 objects degraded (5.882%), 4 pgs degraded) 2024-08-17T17:06:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:13 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:06:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:13 smithi162 ceph-mon[104668]: osdmap e745: 8 total, 8 up, 8 in 2024-08-17T17:06:14.668 INFO:tasks.rados.rados.0.smithi063.stdout:2642: done (1 left) 2024-08-17T17:06:14.668 INFO:tasks.rados.rados.0.smithi063.stdout:2643: done (0 left) 2024-08-17T17:06:14.668 INFO:tasks.rados.rados.0.smithi063.stdout:2644: delete oid 37 current snap is 341 2024-08-17T17:06:14.676 INFO:tasks.rados.rados.0.smithi063.stdout:2644: done (0 left) 2024-08-17T17:06:14.676 INFO:tasks.rados.rados.0.smithi063.stdout:2645: snap_create 2024-08-17T17:06:15.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:14 smithi063 ceph-mon[109843]: osdmap e746: 8 total, 8 up, 8 in 2024-08-17T17:06:15.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:14 smithi063 ceph-mon[109843]: pgmap v576: 89 pgs: 89 active+clean; 207 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 953 KiB/s wr, 24 op/s; 2.3 MiB/s, 2 objects/s recovering 2024-08-17T17:06:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:14 smithi063 ceph-mon[114756]: osdmap e746: 8 total, 8 up, 8 in 2024-08-17T17:06:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:14 smithi063 ceph-mon[114756]: pgmap v576: 89 pgs: 89 active+clean; 207 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 953 KiB/s wr, 24 op/s; 2.3 MiB/s, 2 objects/s recovering 2024-08-17T17:06:15.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:14 smithi162 ceph-mon[104668]: osdmap e746: 8 total, 8 up, 8 in 2024-08-17T17:06:15.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:14 smithi162 ceph-mon[104668]: pgmap v576: 89 pgs: 89 active+clean; 207 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 953 KiB/s wr, 24 op/s; 2.3 MiB/s, 2 objects/s recovering 2024-08-17T17:06:15.685 INFO:tasks.rados.rados.0.smithi063.stdout:2645: done (0 left) 2024-08-17T17:06:15.685 INFO:tasks.rados.rados.0.smithi063.stdout:2646: write oid 43 current snap is 342 2024-08-17T17:06:15.685 INFO:tasks.rados.rados.0.smithi063.stdout:2646: seq_num 994 ranges {706158=723396,2183219=522939} 2024-08-17T17:06:15.692 INFO:tasks.rados.rados.0.smithi063.stdout:2646: writing smithi06387500-43 from 706158 to 1429554 tid 1 2024-08-17T17:06:15.700 INFO:tasks.rados.rados.0.smithi063.stdout:2646: writing smithi06387500-43 from 2183219 to 2706158 tid 2 2024-08-17T17:06:15.700 INFO:tasks.rados.rados.0.smithi063.stdout:2647: delete oid 14 current snap is 342 2024-08-17T17:06:15.701 INFO:tasks.rados.rados.0.smithi063.stdout:2646: finishing write tid 1 to smithi06387500-43 2024-08-17T17:06:15.705 INFO:tasks.rados.rados.0.smithi063.stdout:2646: finishing write tid 2 to smithi06387500-43 2024-08-17T17:06:15.705 INFO:tasks.rados.rados.0.smithi063.stdout:2646: finishing write tid 3 to smithi06387500-43 2024-08-17T17:06:15.705 INFO:tasks.rados.rados.0.smithi063.stdout:2646: finishing write tid 4 to smithi06387500-43 2024-08-17T17:06:15.705 INFO:tasks.rados.rados.0.smithi063.stdout:2646: oid 43 updating version 0 to 1247 2024-08-17T17:06:15.705 INFO:tasks.rados.rados.0.smithi063.stdout:2646: oid 43 version 1247 is already newer than 1246 2024-08-17T17:06:15.705 INFO:tasks.rados.rados.0.smithi063.stdout:2646: oid 43 version 1247 is already newer than 1245 2024-08-17T17:06:15.705 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 1247 (ObjNum 994 snap 342 seq_num 994) dirty exists 2024-08-17T17:06:15.705 INFO:tasks.rados.rados.0.smithi063.stdout:2646: left oid 43 (ObjNum 994 snap 342 seq_num 994) 2024-08-17T17:06:15.707 INFO:tasks.rados.rados.0.smithi063.stdout:2646: done (1 left) 2024-08-17T17:06:15.707 INFO:tasks.rados.rados.0.smithi063.stdout:2647: done (0 left) 2024-08-17T17:06:15.707 INFO:tasks.rados.rados.0.smithi063.stdout:2648: read oid 2 snap -1 2024-08-17T17:06:15.707 INFO:tasks.rados.rados.0.smithi063.stdout:2648: expect (ObjNum 987 snap 339 seq_num 987) 2024-08-17T17:06:15.707 INFO:tasks.rados.rados.0.smithi063.stdout:2649: read oid 44 snap -1 2024-08-17T17:06:15.707 INFO:tasks.rados.rados.0.smithi063.stdout:2649: expect deleted 2024-08-17T17:06:15.707 INFO:tasks.rados.rados.0.smithi063.stdout:2650: write oid 20 current snap is 342 2024-08-17T17:06:15.707 INFO:tasks.rados.rados.0.smithi063.stdout:2650: seq_num 996 ranges {739772=590247,1900183=540336} 2024-08-17T17:06:15.714 INFO:tasks.rados.rados.0.smithi063.stdout:2650: writing smithi06387500-20 from 739772 to 1330019 tid 1 2024-08-17T17:06:15.720 INFO:tasks.rados.rados.0.smithi063.stdout:2650: writing smithi06387500-20 from 1900183 to 2440519 tid 2 2024-08-17T17:06:15.721 INFO:tasks.rados.rados.0.smithi063.stdout:2651: delete oid 5 current snap is 342 2024-08-17T17:06:15.721 INFO:tasks.rados.rados.0.smithi063.stdout:2648: expect (ObjNum 987 snap 339 seq_num 987) 2024-08-17T17:06:15.845 INFO:tasks.rados.rados.0.smithi063.stdout:2650: finishing write tid 1 to smithi06387500-20 2024-08-17T17:06:15.846 INFO:tasks.rados.rados.0.smithi063.stdout:2650: finishing write tid 2 to smithi06387500-20 2024-08-17T17:06:15.846 INFO:tasks.rados.rados.0.smithi063.stdout:2650: finishing write tid 3 to smithi06387500-20 2024-08-17T17:06:15.846 INFO:tasks.rados.rados.0.smithi063.stdout:2648: done (3 left) 2024-08-17T17:06:15.846 INFO:tasks.rados.rados.0.smithi063.stdout:2649: done (2 left) 2024-08-17T17:06:15.846 INFO:tasks.rados.rados.0.smithi063.stdout:2651: done (1 left) 2024-08-17T17:06:15.846 INFO:tasks.rados.rados.0.smithi063.stdout:2652: rollback oid 13 current snap is 342 2024-08-17T17:06:15.846 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 13 to 323 2024-08-17T17:06:15.846 INFO:tasks.rados.rados.0.smithi063.stdout:2650: finishing write tid 4 to smithi06387500-20 2024-08-17T17:06:15.846 INFO:tasks.rados.rados.0.smithi063.stdout:2650: oid 20 updating version 0 to 561 2024-08-17T17:06:15.846 INFO:tasks.rados.rados.0.smithi063.stdout:2650: oid 20 updating version 561 to 563 2024-08-17T17:06:15.846 INFO:tasks.rados.rados.0.smithi063.stdout:2650: oid 20 version 563 is already newer than 562 2024-08-17T17:06:15.846 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 563 (ObjNum 996 snap 342 seq_num 996) dirty exists 2024-08-17T17:06:15.847 INFO:tasks.rados.rados.0.smithi063.stdout:2650: left oid 20 (ObjNum 996 snap 342 seq_num 996) 2024-08-17T17:06:15.847 INFO:tasks.rados.rados.0.smithi063.stdout:2650: done (1 left) 2024-08-17T17:06:15.847 INFO:tasks.rados.rados.0.smithi063.stdout:2653: snap_create 2024-08-17T17:06:15.847 INFO:tasks.rados.rados.0.smithi063.stdout:2652: finishing rollback tid 1 to smithi06387500-13 2024-08-17T17:06:15.847 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1221 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-08-17T17:06:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:15 smithi063 ceph-mon[114756]: osdmap e747: 8 total, 8 up, 8 in 2024-08-17T17:06:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:15 smithi063 ceph-mon[109843]: osdmap e747: 8 total, 8 up, 8 in 2024-08-17T17:06:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:15 smithi162 ceph-mon[104668]: osdmap e747: 8 total, 8 up, 8 in 2024-08-17T17:06:16.692 INFO:tasks.rados.rados.0.smithi063.stdout:2652: done (1 left) 2024-08-17T17:06:16.693 INFO:tasks.rados.rados.0.smithi063.stdout:2653: done (0 left) 2024-08-17T17:06:16.693 INFO:tasks.rados.rados.0.smithi063.stdout:2654: read oid 46 snap -1 2024-08-17T17:06:16.693 INFO:tasks.rados.rados.0.smithi063.stdout:2654: expect deleted 2024-08-17T17:06:16.693 INFO:tasks.rados.rados.0.smithi063.stdout:2655: delete oid 17 current snap is 343 2024-08-17T17:06:16.697 INFO:tasks.rados.rados.0.smithi063.stdout:2654: done (1 left) 2024-08-17T17:06:16.697 INFO:tasks.rados.rados.0.smithi063.stdout:2655: done (0 left) 2024-08-17T17:06:16.697 INFO:tasks.rados.rados.0.smithi063.stdout:2656: write oid 38 current snap is 343 2024-08-17T17:06:16.697 INFO:tasks.rados.rados.0.smithi063.stdout:2656: seq_num 999 ranges {790193=415994,1959276=404016} 2024-08-17T17:06:16.703 INFO:tasks.rados.rados.0.smithi063.stdout:2656: writing smithi06387500-38 from 790193 to 1206187 tid 1 2024-08-17T17:06:16.710 INFO:tasks.rados.rados.0.smithi063.stdout:2656: writing smithi06387500-38 from 1959276 to 2363292 tid 2 2024-08-17T17:06:16.710 INFO:tasks.rados.rados.0.smithi063.stdout:2657: write oid 16 current snap is 343 2024-08-17T17:06:16.710 INFO:tasks.rados.rados.0.smithi063.stdout:2657: seq_num 1000 ranges {407000=491243,1617758=547674,2595228=211772} 2024-08-17T17:06:16.715 INFO:tasks.rados.rados.0.smithi063.stdout:2657: writing smithi06387500-16 from 407000 to 898243 tid 1 2024-08-17T17:06:16.722 INFO:tasks.rados.rados.0.smithi063.stdout:2657: writing smithi06387500-16 from 1617758 to 2165432 tid 2 2024-08-17T17:06:16.727 INFO:tasks.rados.rados.0.smithi063.stdout:2657: writing smithi06387500-16 from 2595228 to 2807000 tid 3 2024-08-17T17:06:16.727 INFO:tasks.rados.rados.0.smithi063.stdout:2658: rollback oid 4 current snap is 343 2024-08-17T17:06:16.727 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 4 to 328 2024-08-17T17:06:16.727 INFO:tasks.rados.rados.0.smithi063.stdout:2656: finishing write tid 1 to smithi06387500-38 2024-08-17T17:06:16.727 INFO:tasks.rados.rados.0.smithi063.stdout:2656: finishing write tid 2 to smithi06387500-38 2024-08-17T17:06:16.727 INFO:tasks.rados.rados.0.smithi063.stdout:2656: finishing write tid 3 to smithi06387500-38 2024-08-17T17:06:16.727 INFO:tasks.rados.rados.0.smithi063.stdout:2656: finishing write tid 4 to smithi06387500-38 2024-08-17T17:06:16.727 INFO:tasks.rados.rados.0.smithi063.stdout:2656: oid 38 updating version 0 to 409 2024-08-17T17:06:16.727 INFO:tasks.rados.rados.0.smithi063.stdout:2656: oid 38 version 409 is already newer than 408 2024-08-17T17:06:16.727 INFO:tasks.rados.rados.0.smithi063.stdout:2656: oid 38 version 409 is already newer than 407 2024-08-17T17:06:16.728 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 409 (ObjNum 999 snap 343 seq_num 999) dirty exists 2024-08-17T17:06:16.728 INFO:tasks.rados.rados.0.smithi063.stdout:2656: left oid 38 (ObjNum 999 snap 343 seq_num 999) 2024-08-17T17:06:16.728 INFO:tasks.rados.rados.0.smithi063.stdout:2657: finishing write tid 1 to smithi06387500-16 2024-08-17T17:06:16.728 INFO:tasks.rados.rados.0.smithi063.stdout:2657: finishing write tid 2 to smithi06387500-16 2024-08-17T17:06:16.728 INFO:tasks.rados.rados.0.smithi063.stdout:2656: done (2 left) 2024-08-17T17:06:16.728 INFO:tasks.rados.rados.0.smithi063.stdout:2659: read oid 27 snap 341 2024-08-17T17:06:16.728 INFO:tasks.rados.rados.0.smithi063.stdout:2659: expect deleted 2024-08-17T17:06:16.728 INFO:tasks.rados.rados.0.smithi063.stdout:2660: delete oid 1 current snap is 343 2024-08-17T17:06:16.729 INFO:tasks.rados.rados.0.smithi063.stdout:2657: finishing write tid 3 to smithi06387500-16 2024-08-17T17:06:16.729 INFO:tasks.rados.rados.0.smithi063.stdout:2659: done (3 left) 2024-08-17T17:06:16.730 INFO:tasks.rados.rados.0.smithi063.stdout:2660: done (2 left) 2024-08-17T17:06:16.730 INFO:tasks.rados.rados.0.smithi063.stdout:2661: read oid 50 snap -1 2024-08-17T17:06:16.730 INFO:tasks.rados.rados.0.smithi063.stdout:2661: expect deleted 2024-08-17T17:06:16.730 INFO:tasks.rados.rados.0.smithi063.stdout:2662: read oid 2 snap -1 2024-08-17T17:06:16.730 INFO:tasks.rados.rados.0.smithi063.stdout:2662: expect (ObjNum 987 snap 339 seq_num 987) 2024-08-17T17:06:16.730 INFO:tasks.rados.rados.0.smithi063.stdout:2663: write oid 38 current snap is 343 2024-08-17T17:06:16.730 INFO:tasks.rados.rados.0.smithi063.stdout:2663: seq_num 1002 ranges {440614=641741,1734722=551343} 2024-08-17T17:06:16.735 INFO:tasks.rados.rados.0.smithi063.stdout:2663: writing smithi06387500-38 from 440614 to 1082355 tid 1 2024-08-17T17:06:16.742 INFO:tasks.rados.rados.0.smithi063.stdout:2663: writing smithi06387500-38 from 1734722 to 2286065 tid 2 2024-08-17T17:06:16.742 INFO:tasks.rados.rados.0.smithi063.stdout:2664: write oid 37 current snap is 343 2024-08-17T17:06:16.742 INFO:tasks.rados.rados.0.smithi063.stdout:2664: seq_num 1003 ranges {457421=716990,1676851=411354,2677497=179924} 2024-08-17T17:06:16.748 INFO:tasks.rados.rados.0.smithi063.stdout:2664: writing smithi06387500-37 from 457421 to 1174411 tid 1 2024-08-17T17:06:16.754 INFO:tasks.rados.rados.0.smithi063.stdout:2664: writing smithi06387500-37 from 1676851 to 2088205 tid 2 2024-08-17T17:06:16.759 INFO:tasks.rados.rados.0.smithi063.stdout:2664: writing smithi06387500-37 from 2677497 to 2857421 tid 3 2024-08-17T17:06:16.759 INFO:tasks.rados.rados.0.smithi063.stdout:2665: write oid 35 current snap is 343 2024-08-17T17:06:16.759 INFO:tasks.rados.rados.0.smithi063.stdout:2657: finishing write tid 4 to smithi06387500-16 2024-08-17T17:06:16.759 INFO:tasks.rados.rados.0.smithi063.stdout:2665: seq_num 1004 ranges {474228=508592,1451686=555012,2704920=169308} 2024-08-17T17:06:16.764 INFO:tasks.rados.rados.0.smithi063.stdout:2665: writing smithi06387500-35 from 474228 to 982820 tid 1 2024-08-17T17:06:16.770 INFO:tasks.rados.rados.0.smithi063.stdout:2665: writing smithi06387500-35 from 1451686 to 2006698 tid 2 2024-08-17T17:06:16.774 INFO:tasks.rados.rados.0.smithi063.stdout:2665: writing smithi06387500-35 from 2704920 to 2874228 tid 3 2024-08-17T17:06:16.775 INFO:tasks.rados.rados.0.smithi063.stdout:2666: snap_remove snap 334 2024-08-17T17:06:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:16 smithi063 ceph-mon[114756]: osdmap e748: 8 total, 8 up, 8 in 2024-08-17T17:06:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:16 smithi063 ceph-mon[114756]: pgmap v579: 89 pgs: 89 active+clean; 207 MiB data, 764 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:06:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:16 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:16 smithi063 ceph-mon[109843]: osdmap e748: 8 total, 8 up, 8 in 2024-08-17T17:06:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:16 smithi063 ceph-mon[109843]: pgmap v579: 89 pgs: 89 active+clean; 207 MiB data, 764 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:06:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:16 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:16 smithi162 ceph-mon[104668]: osdmap e748: 8 total, 8 up, 8 in 2024-08-17T17:06:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:16 smithi162 ceph-mon[104668]: pgmap v579: 89 pgs: 89 active+clean; 207 MiB data, 764 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:06:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:16 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:17.602 INFO:tasks.rados.rados.0.smithi063.stdout:2666: done (7 left) 2024-08-17T17:06:17.602 INFO:tasks.rados.rados.0.smithi063.stdout:2667: rollback oid 20 current snap is 343 2024-08-17T17:06:17.602 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 20 to 342 2024-08-17T17:06:17.602 INFO:tasks.rados.rados.0.smithi063.stdout:2657: finishing write tid 5 to smithi06387500-16 2024-08-17T17:06:17.603 INFO:tasks.rados.rados.0.smithi063.stdout:2657: oid 16 updating version 0 to 976 2024-08-17T17:06:17.603 INFO:tasks.rados.rados.0.smithi063.stdout:2657: oid 16 updating version 976 to 977 2024-08-17T17:06:17.603 INFO:tasks.rados.rados.0.smithi063.stdout:2657: oid 16 version 977 is already newer than 974 2024-08-17T17:06:17.603 INFO:tasks.rados.rados.0.smithi063.stdout:2657: oid 16 version 977 is already newer than 975 2024-08-17T17:06:17.603 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 977 (ObjNum 1000 snap 343 seq_num 1000) dirty exists 2024-08-17T17:06:17.603 INFO:tasks.rados.rados.0.smithi063.stdout:2657: left oid 16 (ObjNum 1000 snap 343 seq_num 1000) 2024-08-17T17:06:17.603 INFO:tasks.rados.rados.0.smithi063.stdout:2658: finishing rollback tid 0 to smithi06387500-4 2024-08-17T17:06:17.603 INFO:tasks.rados.rados.0.smithi063.stdout:2658: finishing rollback tid 1 to smithi06387500-4 2024-08-17T17:06:17.603 INFO:tasks.rados.rados.0.smithi063.stdout:2658: finishing rollback tid 2 to smithi06387500-4 2024-08-17T17:06:17.603 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 2598 (ObjNum 965 snap 328 seq_num 965) dirty exists 2024-08-17T17:06:17.603 INFO:tasks.rados.rados.0.smithi063.stdout:2662: expect (ObjNum 987 snap 339 seq_num 987) 2024-08-17T17:06:17.730 INFO:tasks.rados.rados.0.smithi063.stdout:2663: finishing write tid 1 to smithi06387500-38 2024-08-17T17:06:17.731 INFO:tasks.rados.rados.0.smithi063.stdout:2663: finishing write tid 2 to smithi06387500-38 2024-08-17T17:06:17.731 INFO:tasks.rados.rados.0.smithi063.stdout:2663: finishing write tid 3 to smithi06387500-38 2024-08-17T17:06:17.731 INFO:tasks.rados.rados.0.smithi063.stdout:2663: finishing write tid 4 to smithi06387500-38 2024-08-17T17:06:17.731 INFO:tasks.rados.rados.0.smithi063.stdout:2663: oid 38 updating version 0 to 411 2024-08-17T17:06:17.731 INFO:tasks.rados.rados.0.smithi063.stdout:2663: oid 38 version 411 is already newer than 410 2024-08-17T17:06:17.731 INFO:tasks.rados.rados.0.smithi063.stdout:2663: oid 38 updating version 411 to 412 2024-08-17T17:06:17.731 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 412 (ObjNum 1002 snap 343 seq_num 1002) dirty exists 2024-08-17T17:06:17.731 INFO:tasks.rados.rados.0.smithi063.stdout:2663: left oid 38 (ObjNum 1002 snap 343 seq_num 1002) 2024-08-17T17:06:17.731 INFO:tasks.rados.rados.0.smithi063.stdout:2664: finishing write tid 1 to smithi06387500-37 2024-08-17T17:06:17.731 INFO:tasks.rados.rados.0.smithi063.stdout:2664: finishing write tid 2 to smithi06387500-37 2024-08-17T17:06:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:2657: done (7 left) 2024-08-17T17:06:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:2658: done (6 left) 2024-08-17T17:06:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:2661: done (5 left) 2024-08-17T17:06:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:2662: done (4 left) 2024-08-17T17:06:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:2663: done (3 left) 2024-08-17T17:06:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:2668: write oid 33 current snap is 343 2024-08-17T17:06:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:2664: finishing write tid 3 to smithi06387500-37 2024-08-17T17:06:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:2664: finishing write tid 4 to smithi06387500-37 2024-08-17T17:06:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:2664: finishing write tid 5 to smithi06387500-37 2024-08-17T17:06:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:2664: oid 37 updating version 0 to 2602 2024-08-17T17:06:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:2664: oid 37 version 2602 is already newer than 2601 2024-08-17T17:06:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:2664: oid 37 version 2602 is already newer than 2599 2024-08-17T17:06:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:2664: oid 37 version 2602 is already newer than 2600 2024-08-17T17:06:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 2602 (ObjNum 1003 snap 343 seq_num 1003) dirty exists 2024-08-17T17:06:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:2664: left oid 37 (ObjNum 1003 snap 343 seq_num 1003) 2024-08-17T17:06:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:2665: finishing write tid 1 to smithi06387500-35 2024-08-17T17:06:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:2665: finishing write tid 2 to smithi06387500-35 2024-08-17T17:06:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:2665: finishing write tid 3 to smithi06387500-35 2024-08-17T17:06:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:2665: finishing write tid 4 to smithi06387500-35 2024-08-17T17:06:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:2665: finishing write tid 5 to smithi06387500-35 2024-08-17T17:06:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:2665: oid 35 updating version 0 to 2605 2024-08-17T17:06:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:2665: oid 35 version 2605 is already newer than 2604 2024-08-17T17:06:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:2665: oid 35 updating version 2605 to 2606 2024-08-17T17:06:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:2665: oid 35 updating version 2606 to 2607 2024-08-17T17:06:17.734 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 2607 (ObjNum 1004 snap 343 seq_num 1004) dirty exists 2024-08-17T17:06:17.734 INFO:tasks.rados.rados.0.smithi063.stdout:2665: left oid 35 (ObjNum 1004 snap 343 seq_num 1004) 2024-08-17T17:06:17.734 INFO:tasks.rados.rados.0.smithi063.stdout:2667: finishing rollback tid 0 to smithi06387500-20 2024-08-17T17:06:17.734 INFO:tasks.rados.rados.0.smithi063.stdout:2667: finishing rollback tid 1 to smithi06387500-20 2024-08-17T17:06:17.734 INFO:tasks.rados.rados.0.smithi063.stdout:2667: finishing rollback tid 2 to smithi06387500-20 2024-08-17T17:06:17.734 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 567 (ObjNum 996 snap 342 seq_num 996) dirty exists 2024-08-17T17:06:17.734 INFO:tasks.rados.rados.0.smithi063.stdout:2668: seq_num 1005 ranges {491035=583841,1510168=698670,2732343=158692} 2024-08-17T17:06:17.734 INFO:tasks.rados.rados.0.smithi063.stdout:2668: writing smithi06387500-33 from 491035 to 1074876 tid 1 2024-08-17T17:06:17.741 INFO:tasks.rados.rados.0.smithi063.stdout:2668: writing smithi06387500-33 from 1510168 to 2208838 tid 2 2024-08-17T17:06:17.745 INFO:tasks.rados.rados.0.smithi063.stdout:2668: writing smithi06387500-33 from 2732343 to 2891035 tid 3 2024-08-17T17:06:17.745 INFO:tasks.rados.rados.0.smithi063.stdout:2664: done (3 left) 2024-08-17T17:06:17.745 INFO:tasks.rados.rados.0.smithi063.stdout:2665: done (2 left) 2024-08-17T17:06:17.745 INFO:tasks.rados.rados.0.smithi063.stdout:2667: done (1 left) 2024-08-17T17:06:17.745 INFO:tasks.rados.rados.0.smithi063.stdout:2669: snap_remove snap 335 2024-08-17T17:06:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:17 smithi063 ceph-mon[114756]: osdmap e749: 8 total, 8 up, 8 in 2024-08-17T17:06:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:17 smithi063 ceph-mon[114756]: osdmap e750: 8 total, 8 up, 8 in 2024-08-17T17:06:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:17 smithi063 ceph-mon[109843]: osdmap e749: 8 total, 8 up, 8 in 2024-08-17T17:06:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:17 smithi063 ceph-mon[109843]: osdmap e750: 8 total, 8 up, 8 in 2024-08-17T17:06:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:17 smithi162 ceph-mon[104668]: osdmap e749: 8 total, 8 up, 8 in 2024-08-17T17:06:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:17 smithi162 ceph-mon[104668]: osdmap e750: 8 total, 8 up, 8 in 2024-08-17T17:06:18.697 INFO:tasks.rados.rados.0.smithi063.stdout:2669: done (1 left) 2024-08-17T17:06:18.697 INFO:tasks.rados.rados.0.smithi063.stdout:2668: finishing write tid 1 to smithi06387500-33 2024-08-17T17:06:18.697 INFO:tasks.rados.rados.0.smithi063.stdout:2668: finishing write tid 2 to smithi06387500-33 2024-08-17T17:06:18.697 INFO:tasks.rados.rados.0.smithi063.stdout:2668: finishing write tid 3 to smithi06387500-33 2024-08-17T17:06:18.697 INFO:tasks.rados.rados.0.smithi063.stdout:2668: finishing write tid 4 to smithi06387500-33 2024-08-17T17:06:18.697 INFO:tasks.rados.rados.0.smithi063.stdout:2668: finishing write tid 5 to smithi06387500-33 2024-08-17T17:06:18.697 INFO:tasks.rados.rados.0.smithi063.stdout:2668: oid 33 updating version 0 to 911 2024-08-17T17:06:18.697 INFO:tasks.rados.rados.0.smithi063.stdout:2668: oid 33 version 911 is already newer than 909 2024-08-17T17:06:18.698 INFO:tasks.rados.rados.0.smithi063.stdout:2668: oid 33 updating version 911 to 912 2024-08-17T17:06:18.698 INFO:tasks.rados.rados.0.smithi063.stdout:2668: oid 33 version 912 is already newer than 910 2024-08-17T17:06:18.698 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 912 (ObjNum 1005 snap 343 seq_num 1005) dirty exists 2024-08-17T17:06:18.698 INFO:tasks.rados.rados.0.smithi063.stdout:2668: left oid 33 (ObjNum 1005 snap 343 seq_num 1005) 2024-08-17T17:06:18.698 INFO:tasks.rados.rados.0.smithi063.stdout:2670: read oid 11 snap -1 2024-08-17T17:06:18.698 INFO:tasks.rados.rados.0.smithi063.stdout:2670: expect deleted 2024-08-17T17:06:18.698 INFO:tasks.rados.rados.0.smithi063.stdout:2668: done (1 left) 2024-08-17T17:06:18.698 INFO:tasks.rados.rados.0.smithi063.stdout:2671: write oid 40 current snap is 343 2024-08-17T17:06:18.698 INFO:tasks.rados.rados.0.smithi063.stdout:2671: seq_num 1006 ranges {507842=659090,1568650=558681,2759766=148076} 2024-08-17T17:06:18.704 INFO:tasks.rados.rados.0.smithi063.stdout:2671: writing smithi06387500-40 from 507842 to 1166932 tid 1 2024-08-17T17:06:18.710 INFO:tasks.rados.rados.0.smithi063.stdout:2671: writing smithi06387500-40 from 1568650 to 2127331 tid 2 2024-08-17T17:06:18.714 INFO:tasks.rados.rados.0.smithi063.stdout:2671: writing smithi06387500-40 from 2759766 to 2907842 tid 3 2024-08-17T17:06:18.715 INFO:tasks.rados.rados.0.smithi063.stdout:2672: read oid 45 snap 340 2024-08-17T17:06:18.715 INFO:tasks.rados.rados.0.smithi063.stdout:2672: expect (ObjNum 966 snap 328 seq_num 966) 2024-08-17T17:06:18.715 INFO:tasks.rados.rados.0.smithi063.stdout:2673: write oid 14 current snap is 343 2024-08-17T17:06:18.715 INFO:tasks.rados.rados.0.smithi063.stdout:2673: seq_num 1007 ranges {524649=734339,1910779=702339} 2024-08-17T17:06:18.721 INFO:tasks.rados.rados.0.smithi063.stdout:2673: writing smithi06387500-14 from 524649 to 1258988 tid 1 2024-08-17T17:06:18.729 INFO:tasks.rados.rados.0.smithi063.stdout:2673: writing smithi06387500-14 from 1910779 to 2613118 tid 2 2024-08-17T17:06:18.729 INFO:tasks.rados.rados.0.smithi063.stdout:2674: snap_create 2024-08-17T17:06:18.729 INFO:tasks.rados.rados.0.smithi063.stdout:2671: finishing write tid 1 to smithi06387500-40 2024-08-17T17:06:18.729 INFO:tasks.rados.rados.0.smithi063.stdout:2671: finishing write tid 2 to smithi06387500-40 2024-08-17T17:06:18.729 INFO:tasks.rados.rados.0.smithi063.stdout:2671: finishing write tid 3 to smithi06387500-40 2024-08-17T17:06:18.729 INFO:tasks.rados.rados.0.smithi063.stdout:2671: finishing write tid 4 to smithi06387500-40 2024-08-17T17:06:18.730 INFO:tasks.rados.rados.0.smithi063.stdout:2671: finishing write tid 5 to smithi06387500-40 2024-08-17T17:06:18.730 INFO:tasks.rados.rados.0.smithi063.stdout:2671: oid 40 updating version 0 to 924 2024-08-17T17:06:18.730 INFO:tasks.rados.rados.0.smithi063.stdout:2671: oid 40 version 924 is already newer than 923 2024-08-17T17:06:18.730 INFO:tasks.rados.rados.0.smithi063.stdout:2671: oid 40 version 924 is already newer than 922 2024-08-17T17:06:18.730 INFO:tasks.rados.rados.0.smithi063.stdout:2671: oid 40 updating version 924 to 925 2024-08-17T17:06:18.730 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 925 (ObjNum 1006 snap 343 seq_num 1006) dirty exists 2024-08-17T17:06:18.730 INFO:tasks.rados.rados.0.smithi063.stdout:2671: left oid 40 (ObjNum 1006 snap 343 seq_num 1006) 2024-08-17T17:06:18.730 INFO:tasks.rados.rados.0.smithi063.stdout:2672: expect (ObjNum 966 snap 328 seq_num 966) 2024-08-17T17:06:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2673: finishing write tid 1 to smithi06387500-14 2024-08-17T17:06:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2673: finishing write tid 2 to smithi06387500-14 2024-08-17T17:06:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2673: finishing write tid 3 to smithi06387500-14 2024-08-17T17:06:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2673: finishing write tid 4 to smithi06387500-14 2024-08-17T17:06:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2673: oid 14 updating version 0 to 2624 2024-08-17T17:06:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2673: oid 14 updating version 2624 to 2625 2024-08-17T17:06:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2673: oid 14 updating version 2625 to 2626 2024-08-17T17:06:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 2626 (ObjNum 1007 snap 343 seq_num 1007) dirty exists 2024-08-17T17:06:18.807 INFO:tasks.rados.rados.0.smithi063.stdout:2673: left oid 14 (ObjNum 1007 snap 343 seq_num 1007) 2024-08-17T17:06:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:18 smithi063 ceph-mon[114756]: pgmap v582: 89 pgs: 89 active+clean; 224 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.1 MiB/s wr, 24 op/s 2024-08-17T17:06:19.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:18 smithi063 ceph-mon[109843]: pgmap v582: 89 pgs: 89 active+clean; 224 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.1 MiB/s wr, 24 op/s 2024-08-17T17:06:19.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:18 smithi162 ceph-mon[104668]: pgmap v582: 89 pgs: 89 active+clean; 224 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.1 MiB/s wr, 24 op/s 2024-08-17T17:06:19.697 INFO:tasks.rados.rados.0.smithi063.stdout:2670: done (4 left) 2024-08-17T17:06:19.697 INFO:tasks.rados.rados.0.smithi063.stdout:2671: done (3 left) 2024-08-17T17:06:19.697 INFO:tasks.rados.rados.0.smithi063.stdout:2672: done (2 left) 2024-08-17T17:06:19.697 INFO:tasks.rados.rados.0.smithi063.stdout:2673: done (1 left) 2024-08-17T17:06:19.697 INFO:tasks.rados.rados.0.smithi063.stdout:2674: done (0 left) 2024-08-17T17:06:19.697 INFO:tasks.rados.rados.0.smithi063.stdout:2675: delete oid 4 current snap is 344 2024-08-17T17:06:19.703 INFO:tasks.rados.rados.0.smithi063.stdout:2675: done (0 left) 2024-08-17T17:06:19.703 INFO:tasks.rados.rados.0.smithi063.stdout:2676: rollback oid 2 current snap is 344 2024-08-17T17:06:19.703 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 2 to 337 2024-08-17T17:06:19.703 INFO:tasks.rados.rados.0.smithi063.stdout:2677: snap_create 2024-08-17T17:06:19.706 INFO:tasks.rados.rados.0.smithi063.stdout:2676: finishing rollback tid 0 to smithi06387500-2 2024-08-17T17:06:19.707 INFO:tasks.rados.rados.0.smithi063.stdout:2676: finishing rollback tid 1 to smithi06387500-2 2024-08-17T17:06:19.707 INFO:tasks.rados.rados.0.smithi063.stdout:2676: finishing rollback tid 2 to smithi06387500-2 2024-08-17T17:06:19.707 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 1308 (ObjNum 926 snap 316 seq_num 926) dirty exists 2024-08-17T17:06:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:19 smithi063 ceph-mon[114756]: osdmap e751: 8 total, 8 up, 8 in 2024-08-17T17:06:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:19 smithi063 ceph-mon[109843]: osdmap e751: 8 total, 8 up, 8 in 2024-08-17T17:06:20.030 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T17:06:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:19 smithi162 ceph-mon[104668]: osdmap e751: 8 total, 8 up, 8 in 2024-08-17T17:06:20.611 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:06:20.611 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (7m) 17s ago 13m 18.9M - 0.27.0 11f11916f8cd 59f4018b91ca 2024-08-17T17:06:20.611 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (7m) 70s ago 13m 78.2M - c8b91775d855 581e03e654a8 2024-08-17T17:06:20.611 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (12m) 17s ago 12m 23.0M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T17:06:20.611 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (12m) 70s ago 12m 19.1M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T17:06:20.611 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,9283,8765 running (8m) 70s ago 17m 456M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb da48034a2895 2024-08-17T17:06:20.611 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:8443,9283,8765 running (9m) 17s ago 19m 517M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 943e639cede6 2024-08-17T17:06:20.612 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (6m) 17s ago 19m 67.3M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb f0f20fcac956 2024-08-17T17:06:20.612 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (4m) 70s ago 18m 39.9M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 520525c79dbe 2024-08-17T17:06:20.612 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (6m) 17s ago 17m 53.8M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb c484af026acf 2024-08-17T17:06:20.612 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 running (8m) 17s ago 13m 11.7M - 1.7.0 72c9c2088986 34f55f5de2aa 2024-08-17T17:06:20.612 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (8m) 70s ago 13m 11.6M - 1.7.0 72c9c2088986 5bb162d0f1a9 2024-08-17T17:06:20.612 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (21s) 17s ago 17m 12.2M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 8fa117adbc0a 2024-08-17T17:06:20.612 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (16m) 17s ago 16m 99.3M 2480M 18.2.4-719-gced649f8 c870753d67c2 5d6fdb6000ad 2024-08-17T17:06:20.612 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (16m) 17s ago 16m 110M 2480M 18.2.4-719-gced649f8 c870753d67c2 a3cbe8edb49a 2024-08-17T17:06:20.612 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (16m) 17s ago 16m 95.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T17:06:20.612 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (15m) 70s ago 15m 121M 2480M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T17:06:20.612 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (15m) 70s ago 15m 115M 2480M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T17:06:20.612 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (15m) 70s ago 15m 85.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T17:06:20.613 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (14m) 70s ago 14m 94.5M 2480M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T17:06:20.613 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (8m) 70s ago 14m 48.0M - 2.51.0 1d3b7f56885b dfe66b2c5568 2024-08-17T17:06:20.711 INFO:tasks.rados.rados.0.smithi063.stdout:2676: done (1 left) 2024-08-17T17:06:20.711 INFO:tasks.rados.rados.0.smithi063.stdout:2677: done (0 left) 2024-08-17T17:06:20.712 INFO:tasks.rados.rados.0.smithi063.stdout:2678: read oid 35 snap -1 2024-08-17T17:06:20.712 INFO:tasks.rados.rados.0.smithi063.stdout:2678: expect (ObjNum 1004 snap 343 seq_num 1004) 2024-08-17T17:06:20.712 INFO:tasks.rados.rados.0.smithi063.stdout:2679: write oid 26 current snap is 345 2024-08-17T17:06:20.712 INFO:tasks.rados.rados.0.smithi063.stdout:2679: seq_num 1009 ranges {558263=484837,1627743=706008} 2024-08-17T17:06:20.717 INFO:tasks.rados.rados.0.smithi063.stdout:2679: writing smithi06387500-26 from 558263 to 1043100 tid 1 2024-08-17T17:06:20.724 INFO:tasks.rados.rados.0.smithi063.stdout:2679: writing smithi06387500-26 from 1627743 to 2333751 tid 2 2024-08-17T17:06:20.724 INFO:tasks.rados.rados.0.smithi063.stdout:2680: write oid 41 current snap is 345 2024-08-17T17:06:20.724 INFO:tasks.rados.rados.0.smithi063.stdout:2680: seq_num 1010 ranges {575070=560086,1686225=566019,2869458=105612} 2024-08-17T17:06:20.730 INFO:tasks.rados.rados.0.smithi063.stdout:2680: writing smithi06387500-41 from 575070 to 1135156 tid 1 2024-08-17T17:06:20.736 INFO:tasks.rados.rados.0.smithi063.stdout:2680: writing smithi06387500-41 from 1686225 to 2252244 tid 2 2024-08-17T17:06:20.740 INFO:tasks.rados.rados.0.smithi063.stdout:2680: writing smithi06387500-41 from 2869458 to 2975070 tid 3 2024-08-17T17:06:20.740 INFO:tasks.rados.rados.0.smithi063.stdout:2681: read oid 32 snap 340 2024-08-17T17:06:20.740 INFO:tasks.rados.rados.0.smithi063.stdout:2681: expect (ObjNum 882 snap 308 seq_num 882) 2024-08-17T17:06:20.741 INFO:tasks.rados.rados.0.smithi063.stdout:2682: write oid 19 current snap is 345 2024-08-17T17:06:20.741 INFO:tasks.rados.rados.0.smithi063.stdout:2682: seq_num 1011 ranges {591877=635335,1628354=709677} 2024-08-17T17:06:20.747 INFO:tasks.rados.rados.0.smithi063.stdout:2682: writing smithi06387500-19 from 591877 to 1227212 tid 1 2024-08-17T17:06:20.754 INFO:tasks.rados.rados.0.smithi063.stdout:2682: writing smithi06387500-19 from 1628354 to 2338031 tid 2 2024-08-17T17:06:20.754 INFO:tasks.rados.rados.0.smithi063.stdout:2683: read oid 39 snap -1 2024-08-17T17:06:20.754 INFO:tasks.rados.rados.0.smithi063.stdout:2683: expect (ObjNum 943 snap 323 seq_num 943) 2024-08-17T17:06:20.754 INFO:tasks.rados.rados.0.smithi063.stdout:2684: rollback oid 7 current snap is 345 2024-08-17T17:06:20.754 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 329 2024-08-17T17:06:20.754 INFO:tasks.rados.rados.0.smithi063.stdout:2679: finishing write tid 1 to smithi06387500-26 2024-08-17T17:06:20.754 INFO:tasks.rados.rados.0.smithi063.stdout:2678: expect (ObjNum 1004 snap 343 seq_num 1004) 2024-08-17T17:06:20.882 INFO:tasks.rados.rados.0.smithi063.stdout:2679: finishing write tid 2 to smithi06387500-26 2024-08-17T17:06:20.882 INFO:tasks.rados.rados.0.smithi063.stdout:2679: finishing write tid 3 to smithi06387500-26 2024-08-17T17:06:20.882 INFO:tasks.rados.rados.0.smithi063.stdout:2679: finishing write tid 4 to smithi06387500-26 2024-08-17T17:06:20.882 INFO:tasks.rados.rados.0.smithi063.stdout:2679: oid 26 updating version 0 to 878 2024-08-17T17:06:20.882 INFO:tasks.rados.rados.0.smithi063.stdout:2679: oid 26 updating version 878 to 879 2024-08-17T17:06:20.882 INFO:tasks.rados.rados.0.smithi063.stdout:2679: oid 26 version 879 is already newer than 877 2024-08-17T17:06:20.882 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 879 (ObjNum 1009 snap 345 seq_num 1009) dirty exists 2024-08-17T17:06:20.882 INFO:tasks.rados.rados.0.smithi063.stdout:2679: left oid 26 (ObjNum 1009 snap 345 seq_num 1009) 2024-08-17T17:06:20.882 INFO:tasks.rados.rados.0.smithi063.stdout:2680: finishing write tid 1 to smithi06387500-41 2024-08-17T17:06:20.882 INFO:tasks.rados.rados.0.smithi063.stdout:2678: done (6 left) 2024-08-17T17:06:20.883 INFO:tasks.rados.rados.0.smithi063.stdout:2679: done (5 left) 2024-08-17T17:06:20.883 INFO:tasks.rados.rados.0.smithi063.stdout:2685: read oid 47 snap -1 2024-08-17T17:06:20.883 INFO:tasks.rados.rados.0.smithi063.stdout:2685: expect deleted 2024-08-17T17:06:20.883 INFO:tasks.rados.rados.0.smithi063.stdout:2680: finishing write tid 2 to smithi06387500-41 2024-08-17T17:06:20.883 INFO:tasks.rados.rados.0.smithi063.stdout:2680: finishing write tid 3 to smithi06387500-41 2024-08-17T17:06:20.883 INFO:tasks.rados.rados.0.smithi063.stdout:2680: finishing write tid 4 to smithi06387500-41 2024-08-17T17:06:20.883 INFO:tasks.rados.rados.0.smithi063.stdout:2680: finishing write tid 5 to smithi06387500-41 2024-08-17T17:06:20.883 INFO:tasks.rados.rados.0.smithi063.stdout:2680: oid 41 updating version 0 to 278 2024-08-17T17:06:20.883 INFO:tasks.rados.rados.0.smithi063.stdout:2680: oid 41 updating version 278 to 279 2024-08-17T17:06:20.883 INFO:tasks.rados.rados.0.smithi063.stdout:2680: oid 41 updating version 279 to 281 2024-08-17T17:06:20.883 INFO:tasks.rados.rados.0.smithi063.stdout:2680: oid 41 version 281 is already newer than 280 2024-08-17T17:06:20.883 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 281 (ObjNum 1010 snap 345 seq_num 1010) dirty exists 2024-08-17T17:06:20.883 INFO:tasks.rados.rados.0.smithi063.stdout:2680: left oid 41 (ObjNum 1010 snap 345 seq_num 1010) 2024-08-17T17:06:20.884 INFO:tasks.rados.rados.0.smithi063.stdout:2681: expect (ObjNum 882 snap 308 seq_num 882) 2024-08-17T17:06:20.970 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:06:20.971 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T17:06:20.971 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:06:20.971 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:06:20.971 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T17:06:20.971 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:06:20.971 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:06:20.971 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T17:06:20.972 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 7, 2024-08-17T17:06:20.972 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 1 2024-08-17T17:06:20.972 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:06:20.972 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T17:06:20.972 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:06:20.972 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:06:20.972 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T17:06:20.972 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 9, 2024-08-17T17:06:20.972 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 6 2024-08-17T17:06:20.972 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:06:20.972 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:06:20.973 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:20 smithi063 ceph-mon[114756]: osdmap e752: 8 total, 8 up, 8 in 2024-08-17T17:06:20.973 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:20 smithi063 ceph-mon[114756]: pgmap v585: 89 pgs: 89 active+clean; 224 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.1 MiB/s wr, 24 op/s 2024-08-17T17:06:20.973 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:20 smithi063 ceph-mon[114756]: from='client.44264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:20.973 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:20 smithi063 ceph-mon[114756]: from='client.34341 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:20.973 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:20 smithi063 ceph-mon[114756]: from='client.34347 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:20.973 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:20 smithi063 ceph-mon[109843]: osdmap e752: 8 total, 8 up, 8 in 2024-08-17T17:06:20.974 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:20 smithi063 ceph-mon[109843]: pgmap v585: 89 pgs: 89 active+clean; 224 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.1 MiB/s wr, 24 op/s 2024-08-17T17:06:20.974 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:20 smithi063 ceph-mon[109843]: from='client.44264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:20.974 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:20 smithi063 ceph-mon[109843]: from='client.34341 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:20.974 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:20 smithi063 ceph-mon[109843]: from='client.34347 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:21.042 INFO:tasks.rados.rados.0.smithi063.stdout:2682: finishing write tid 1 to smithi06387500-19 2024-08-17T17:06:21.042 INFO:tasks.rados.rados.0.smithi063.stdout:2682: finishing write tid 2 to smithi06387500-19 2024-08-17T17:06:21.042 INFO:tasks.rados.rados.0.smithi063.stdout:2682: finishing write tid 3 to smithi06387500-19 2024-08-17T17:06:21.042 INFO:tasks.rados.rados.0.smithi063.stdout:2686: read oid 38 snap 329 2024-08-17T17:06:21.042 INFO:tasks.rados.rados.0.smithi063.stdout:2686: expect (ObjNum 868 snap 300 seq_num 868) 2024-08-17T17:06:21.042 INFO:tasks.rados.rados.0.smithi063.stdout:2680: done (6 left) 2024-08-17T17:06:21.043 INFO:tasks.rados.rados.0.smithi063.stdout:2681: done (5 left) 2024-08-17T17:06:21.043 INFO:tasks.rados.rados.0.smithi063.stdout:2687: read oid 9 snap -1 2024-08-17T17:06:21.043 INFO:tasks.rados.rados.0.smithi063.stdout:2687: expect deleted 2024-08-17T17:06:21.043 INFO:tasks.rados.rados.0.smithi063.stdout:2688: read oid 35 snap 323 2024-08-17T17:06:21.043 INFO:tasks.rados.rados.0.smithi063.stdout:2688: expect (ObjNum 941 snap 323 seq_num 941) 2024-08-17T17:06:21.043 INFO:tasks.rados.rados.0.smithi063.stdout:2689: write oid 23 current snap is 345 2024-08-17T17:06:21.043 INFO:tasks.rados.rados.0.smithi063.stdout:2684: finishing rollback tid 0 to smithi06387500-7 2024-08-17T17:06:21.043 INFO:tasks.rados.rados.0.smithi063.stdout:2684: finishing rollback tid 1 to smithi06387500-7 2024-08-17T17:06:21.043 INFO:tasks.rados.rados.0.smithi063.stdout:2682: finishing write tid 4 to smithi06387500-19 2024-08-17T17:06:21.043 INFO:tasks.rados.rados.0.smithi063.stdout:2682: oid 19 updating version 0 to 1768 2024-08-17T17:06:21.043 INFO:tasks.rados.rados.0.smithi063.stdout:2682: oid 19 updating version 1768 to 1770 2024-08-17T17:06:21.044 INFO:tasks.rados.rados.0.smithi063.stdout:2682: oid 19 version 1770 is already newer than 1769 2024-08-17T17:06:21.044 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 1770 (ObjNum 1011 snap 345 seq_num 1011) dirty exists 2024-08-17T17:06:21.044 INFO:tasks.rados.rados.0.smithi063.stdout:2682: left oid 19 (ObjNum 1011 snap 345 seq_num 1011) 2024-08-17T17:06:21.044 INFO:tasks.rados.rados.0.smithi063.stdout:2684: finishing rollback tid 2 to smithi06387500-7 2024-08-17T17:06:21.044 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1259 (ObjNum 963 snap 327 seq_num 963) dirty exists 2024-08-17T17:06:21.044 INFO:tasks.rados.rados.0.smithi063.stdout:2683: expect (ObjNum 943 snap 323 seq_num 943) 2024-08-17T17:06:21.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:20 smithi162 ceph-mon[104668]: osdmap e752: 8 total, 8 up, 8 in 2024-08-17T17:06:21.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:20 smithi162 ceph-mon[104668]: pgmap v585: 89 pgs: 89 active+clean; 224 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.1 MiB/s wr, 24 op/s 2024-08-17T17:06:21.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:20 smithi162 ceph-mon[104668]: from='client.44264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:21.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:20 smithi162 ceph-mon[104668]: from='client.34341 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:21.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:20 smithi162 ceph-mon[104668]: from='client.34347 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:21.269 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:06:21.269 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:06:21.269 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-08-17T17:06:21.270 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (3 remaining).", 2024-08-17T17:06:21.270 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-08-17T17:06:21.270 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "1/8 daemons upgraded", 2024-08-17T17:06:21.270 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading osd daemons", 2024-08-17T17:06:21.270 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T17:06:21.270 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:06:21.296 INFO:tasks.rados.rados.0.smithi063.stdout:2686: expect (ObjNum 868 snap 300 seq_num 868) 2024-08-17T17:06:21.417 INFO:tasks.rados.rados.0.smithi063.stdout:2688: expect (ObjNum 941 snap 323 seq_num 941) 2024-08-17T17:06:21.541 INFO:tasks.rados.rados.0.smithi063.stdout:2689: seq_num 1012 ranges {608684=426937,1803189=569688,2924304=84380} 2024-08-17T17:06:21.546 INFO:tasks.rados.rados.0.smithi063.stdout:2689: writing smithi06387500-23 from 608684 to 1035621 tid 1 2024-08-17T17:06:21.554 INFO:tasks.rados.rados.0.smithi063.stdout:2689: writing smithi06387500-23 from 1803189 to 2372877 tid 2 2024-08-17T17:06:21.558 INFO:tasks.rados.rados.0.smithi063.stdout:2689: writing smithi06387500-23 from 2924304 to 3008684 tid 3 2024-08-17T17:06:21.558 INFO:tasks.rados.rados.0.smithi063.stdout:2682: done (7 left) 2024-08-17T17:06:21.558 INFO:tasks.rados.rados.0.smithi063.stdout:2683: done (6 left) 2024-08-17T17:06:21.558 INFO:tasks.rados.rados.0.smithi063.stdout:2684: done (5 left) 2024-08-17T17:06:21.559 INFO:tasks.rados.rados.0.smithi063.stdout:2685: done (4 left) 2024-08-17T17:06:21.559 INFO:tasks.rados.rados.0.smithi063.stdout:2686: done (3 left) 2024-08-17T17:06:21.559 INFO:tasks.rados.rados.0.smithi063.stdout:2687: done (2 left) 2024-08-17T17:06:21.559 INFO:tasks.rados.rados.0.smithi063.stdout:2688: done (1 left) 2024-08-17T17:06:21.559 INFO:tasks.rados.rados.0.smithi063.stdout:2690: snap_remove snap 340 2024-08-17T17:06:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:21 smithi063 ceph-mon[114756]: osdmap e753: 8 total, 8 up, 8 in 2024-08-17T17:06:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:21 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/772437302' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:21 smithi063 ceph-mon[114756]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:21 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:21 smithi063 ceph-mon[109843]: osdmap e753: 8 total, 8 up, 8 in 2024-08-17T17:06:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:21 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/772437302' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:21 smithi063 ceph-mon[109843]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:21 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:21 smithi162 ceph-mon[104668]: osdmap e753: 8 total, 8 up, 8 in 2024-08-17T17:06:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:21 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/772437302' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:21 smithi162 ceph-mon[104668]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:21 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:22.362 INFO:tasks.rados.rados.0.smithi063.stdout:2690: done (1 left) 2024-08-17T17:06:22.362 INFO:tasks.rados.rados.0.smithi063.stdout:2691: read oid 11 snap -1 2024-08-17T17:06:22.362 INFO:tasks.rados.rados.0.smithi063.stdout:2691: expect deleted 2024-08-17T17:06:22.362 INFO:tasks.rados.rados.0.smithi063.stdout:2692: read oid 22 snap -1 2024-08-17T17:06:22.362 INFO:tasks.rados.rados.0.smithi063.stdout:2692: expect (ObjNum 863 snap 294 seq_num 863) 2024-08-17T17:06:22.362 INFO:tasks.rados.rados.0.smithi063.stdout:2693: snap_create 2024-08-17T17:06:22.362 INFO:tasks.rados.rados.0.smithi063.stdout:2689: finishing write tid 1 to smithi06387500-23 2024-08-17T17:06:22.363 INFO:tasks.rados.rados.0.smithi063.stdout:2689: finishing write tid 2 to smithi06387500-23 2024-08-17T17:06:22.363 INFO:tasks.rados.rados.0.smithi063.stdout:2689: finishing write tid 3 to smithi06387500-23 2024-08-17T17:06:22.363 INFO:tasks.rados.rados.0.smithi063.stdout:2689: finishing write tid 4 to smithi06387500-23 2024-08-17T17:06:22.363 INFO:tasks.rados.rados.0.smithi063.stdout:2689: finishing write tid 5 to smithi06387500-23 2024-08-17T17:06:22.363 INFO:tasks.rados.rados.0.smithi063.stdout:2689: oid 23 updating version 0 to 1309 2024-08-17T17:06:22.363 INFO:tasks.rados.rados.0.smithi063.stdout:2689: oid 23 updating version 1309 to 1311 2024-08-17T17:06:22.363 INFO:tasks.rados.rados.0.smithi063.stdout:2689: oid 23 updating version 1311 to 1312 2024-08-17T17:06:22.363 INFO:tasks.rados.rados.0.smithi063.stdout:2689: oid 23 version 1312 is already newer than 1310 2024-08-17T17:06:22.363 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 1312 (ObjNum 1012 snap 345 seq_num 1012) dirty exists 2024-08-17T17:06:22.363 INFO:tasks.rados.rados.0.smithi063.stdout:2689: left oid 23 (ObjNum 1012 snap 345 seq_num 1012) 2024-08-17T17:06:22.374 INFO:tasks.rados.rados.0.smithi063.stdout:2692: expect (ObjNum 863 snap 294 seq_num 863) 2024-08-17T17:06:23.364 INFO:tasks.rados.rados.0.smithi063.stdout:2689: done (3 left) 2024-08-17T17:06:23.364 INFO:tasks.rados.rados.0.smithi063.stdout:2691: done (2 left) 2024-08-17T17:06:23.364 INFO:tasks.rados.rados.0.smithi063.stdout:2692: done (1 left) 2024-08-17T17:06:23.364 INFO:tasks.rados.rados.0.smithi063.stdout:2693: done (0 left) 2024-08-17T17:06:23.364 INFO:tasks.rados.rados.0.smithi063.stdout:2694: write oid 44 current snap is 346 2024-08-17T17:06:23.364 INFO:tasks.rados.rados.0.smithi063.stdout:2694: seq_num 1013 ranges {625491=502186,1861671=713346} 2024-08-17T17:06:23.370 INFO:tasks.rados.rados.0.smithi063.stdout:2694: writing smithi06387500-44 from 625491 to 1127677 tid 1 2024-08-17T17:06:23.378 INFO:tasks.rados.rados.0.smithi063.stdout:2694: writing smithi06387500-44 from 1861671 to 2575017 tid 2 2024-08-17T17:06:23.378 INFO:tasks.rados.rados.0.smithi063.stdout:2695: write oid 47 current snap is 346 2024-08-17T17:06:23.378 INFO:tasks.rados.rados.0.smithi063.stdout:2694: finishing write tid 1 to smithi06387500-44 2024-08-17T17:06:23.378 INFO:tasks.rados.rados.0.smithi063.stdout:2695: seq_num 1014 ranges {642298=577435,1920153=573357,2979150=63148} 2024-08-17T17:06:23.385 INFO:tasks.rados.rados.0.smithi063.stdout:2695: writing smithi06387500-47 from 642298 to 1219733 tid 1 2024-08-17T17:06:23.392 INFO:tasks.rados.rados.0.smithi063.stdout:2695: writing smithi06387500-47 from 1920153 to 2493510 tid 2 2024-08-17T17:06:23.395 INFO:tasks.rados.rados.0.smithi063.stdout:2695: writing smithi06387500-47 from 2979150 to 3042298 tid 3 2024-08-17T17:06:23.395 INFO:tasks.rados.rados.0.smithi063.stdout:2696: read oid 12 snap -1 2024-08-17T17:06:23.395 INFO:tasks.rados.rados.0.smithi063.stdout:2696: expect (ObjNum 905 snap 311 seq_num 905) 2024-08-17T17:06:23.395 INFO:tasks.rados.rados.0.smithi063.stdout:2697: write oid 14 current snap is 346 2024-08-17T17:06:23.395 INFO:tasks.rados.rados.0.smithi063.stdout:2694: finishing write tid 2 to smithi06387500-44 2024-08-17T17:06:23.395 INFO:tasks.rados.rados.0.smithi063.stdout:2697: seq_num 1015 ranges {659105=652684,1862282=717015} 2024-08-17T17:06:23.402 INFO:tasks.rados.rados.0.smithi063.stdout:2697: writing smithi06387500-14 from 659105 to 1311789 tid 1 2024-08-17T17:06:23.409 INFO:tasks.rados.rados.0.smithi063.stdout:2697: writing smithi06387500-14 from 1862282 to 2579297 tid 2 2024-08-17T17:06:23.409 INFO:tasks.rados.rados.0.smithi063.stdout:2698: rollback oid 16 current snap is 346 2024-08-17T17:06:23.409 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 16 to 315 2024-08-17T17:06:23.410 INFO:tasks.rados.rados.0.smithi063.stdout:2694: finishing write tid 3 to smithi06387500-44 2024-08-17T17:06:23.410 INFO:tasks.rados.rados.0.smithi063.stdout:2694: finishing write tid 4 to smithi06387500-44 2024-08-17T17:06:23.410 INFO:tasks.rados.rados.0.smithi063.stdout:2694: oid 44 updating version 0 to 368 2024-08-17T17:06:23.410 INFO:tasks.rados.rados.0.smithi063.stdout:2694: oid 44 version 368 is already newer than 366 2024-08-17T17:06:23.410 INFO:tasks.rados.rados.0.smithi063.stdout:2694: oid 44 version 368 is already newer than 367 2024-08-17T17:06:23.410 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 368 (ObjNum 1013 snap 346 seq_num 1013) dirty exists 2024-08-17T17:06:23.410 INFO:tasks.rados.rados.0.smithi063.stdout:2694: left oid 44 (ObjNum 1013 snap 346 seq_num 1013) 2024-08-17T17:06:23.410 INFO:tasks.rados.rados.0.smithi063.stdout:2695: finishing write tid 1 to smithi06387500-47 2024-08-17T17:06:23.410 INFO:tasks.rados.rados.0.smithi063.stdout:2695: finishing write tid 2 to smithi06387500-47 2024-08-17T17:06:23.410 INFO:tasks.rados.rados.0.smithi063.stdout:2694: done (4 left) 2024-08-17T17:06:23.410 INFO:tasks.rados.rados.0.smithi063.stdout:2699: snap_remove snap 344 2024-08-17T17:06:23.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:06:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:06:23] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.51.0" 2024-08-17T17:06:23.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:23 smithi063 ceph-mon[109843]: pgmap v587: 89 pgs: 89 active+clean; 239 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.7 MiB/s wr, 21 op/s 2024-08-17T17:06:23.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:23 smithi063 ceph-mon[109843]: osdmap e754: 8 total, 8 up, 8 in 2024-08-17T17:06:23.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:23 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:06:23.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:23 smithi063 ceph-mon[109843]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:06:23.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:23 smithi063 ceph-mon[109843]: Upgrade: osd.1 is safe to restart 2024-08-17T17:06:23.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:23 smithi063 ceph-mon[114756]: pgmap v587: 89 pgs: 89 active+clean; 239 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.7 MiB/s wr, 21 op/s 2024-08-17T17:06:23.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:23 smithi063 ceph-mon[114756]: osdmap e754: 8 total, 8 up, 8 in 2024-08-17T17:06:23.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:23 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:06:23.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:23 smithi063 ceph-mon[114756]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:06:23.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:23 smithi063 ceph-mon[114756]: Upgrade: osd.1 is safe to restart 2024-08-17T17:06:23.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:23 smithi162 ceph-mon[104668]: pgmap v587: 89 pgs: 89 active+clean; 239 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.7 MiB/s wr, 21 op/s 2024-08-17T17:06:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:23 smithi162 ceph-mon[104668]: osdmap e754: 8 total, 8 up, 8 in 2024-08-17T17:06:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:23 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-08-17T17:06:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:23 smithi162 ceph-mon[104668]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-08-17T17:06:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:23 smithi162 ceph-mon[104668]: Upgrade: osd.1 is safe to restart 2024-08-17T17:06:24.377 INFO:tasks.rados.rados.0.smithi063.stdout:2699: done (4 left) 2024-08-17T17:06:24.377 INFO:tasks.rados.rados.0.smithi063.stdout:2700: rollback oid 42 current snap is 346 2024-08-17T17:06:24.377 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 42 to 345 2024-08-17T17:06:24.377 INFO:tasks.rados.rados.0.smithi063.stdout:2695: finishing write tid 3 to smithi06387500-47 2024-08-17T17:06:24.377 INFO:tasks.rados.rados.0.smithi063.stdout:2695: finishing write tid 4 to smithi06387500-47 2024-08-17T17:06:24.378 INFO:tasks.rados.rados.0.smithi063.stdout:2695: finishing write tid 5 to smithi06387500-47 2024-08-17T17:06:24.378 INFO:tasks.rados.rados.0.smithi063.stdout:2695: oid 47 updating version 0 to 529 2024-08-17T17:06:24.378 INFO:tasks.rados.rados.0.smithi063.stdout:2695: oid 47 version 529 is already newer than 526 2024-08-17T17:06:24.378 INFO:tasks.rados.rados.0.smithi063.stdout:2695: oid 47 version 529 is already newer than 527 2024-08-17T17:06:24.378 INFO:tasks.rados.rados.0.smithi063.stdout:2695: oid 47 version 529 is already newer than 528 2024-08-17T17:06:24.378 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 529 (ObjNum 1014 snap 346 seq_num 1014) dirty exists 2024-08-17T17:06:24.378 INFO:tasks.rados.rados.0.smithi063.stdout:2695: left oid 47 (ObjNum 1014 snap 346 seq_num 1014) 2024-08-17T17:06:24.378 INFO:tasks.rados.rados.0.smithi063.stdout:2696: expect (ObjNum 905 snap 311 seq_num 905) 2024-08-17T17:06:24.487 INFO:tasks.rados.rados.0.smithi063.stdout:2697: finishing write tid 1 to smithi06387500-14 2024-08-17T17:06:24.487 INFO:tasks.rados.rados.0.smithi063.stdout:2698: finishing rollback tid 0 to smithi06387500-16 2024-08-17T17:06:24.487 INFO:tasks.rados.rados.0.smithi063.stdout:2698: finishing rollback tid 1 to smithi06387500-16 2024-08-17T17:06:24.487 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 984 (ObjNum 33 snap 0 seq_num 1028453044) dirty dne 2024-08-17T17:06:24.487 INFO:tasks.rados.rados.0.smithi063.stdout:2697: finishing write tid 2 to smithi06387500-14 2024-08-17T17:06:24.487 INFO:tasks.rados.rados.0.smithi063.stdout:2697: finishing write tid 3 to smithi06387500-14 2024-08-17T17:06:24.488 INFO:tasks.rados.rados.0.smithi063.stdout:2697: finishing write tid 4 to smithi06387500-14 2024-08-17T17:06:24.488 INFO:tasks.rados.rados.0.smithi063.stdout:2697: oid 14 updating version 0 to 2637 2024-08-17T17:06:24.488 INFO:tasks.rados.rados.0.smithi063.stdout:2697: oid 14 updating version 2637 to 2638 2024-08-17T17:06:24.488 INFO:tasks.rados.rados.0.smithi063.stdout:2697: oid 14 version 2638 is already newer than 2636 2024-08-17T17:06:24.488 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 2638 (ObjNum 1015 snap 346 seq_num 1015) dirty exists 2024-08-17T17:06:24.488 INFO:tasks.rados.rados.0.smithi063.stdout:2697: left oid 14 (ObjNum 1015 snap 346 seq_num 1015) 2024-08-17T17:06:24.488 INFO:tasks.rados.rados.0.smithi063.stdout:2700: finishing rollback tid 1 to smithi06387500-42 2024-08-17T17:06:24.488 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-08-17T17:06:24.488 INFO:tasks.rados.rados.0.smithi063.stdout:2695: done (4 left) 2024-08-17T17:06:24.488 INFO:tasks.rados.rados.0.smithi063.stdout:2696: done (3 left) 2024-08-17T17:06:24.488 INFO:tasks.rados.rados.0.smithi063.stdout:2697: done (2 left) 2024-08-17T17:06:24.489 INFO:tasks.rados.rados.0.smithi063.stdout:2698: done (1 left) 2024-08-17T17:06:24.489 INFO:tasks.rados.rados.0.smithi063.stdout:2700: done (0 left) 2024-08-17T17:06:24.489 INFO:tasks.rados.rados.0.smithi063.stdout:2701: read oid 34 snap -1 2024-08-17T17:06:24.489 INFO:tasks.rados.rados.0.smithi063.stdout:2701: expect (ObjNum 971 snap 333 seq_num 971) 2024-08-17T17:06:24.489 INFO:tasks.rados.rados.0.smithi063.stdout:2702: read oid 11 snap -1 2024-08-17T17:06:24.489 INFO:tasks.rados.rados.0.smithi063.stdout:2702: expect deleted 2024-08-17T17:06:24.489 INFO:tasks.rados.rados.0.smithi063.stdout:2703: rollback oid 8 current snap is 346 2024-08-17T17:06:24.489 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 8 to 328 2024-08-17T17:06:24.489 INFO:tasks.rados.rados.0.smithi063.stdout:2704: snap_remove snap 343 2024-08-17T17:06:24.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:24 smithi063 ceph-mon[114756]: osdmap e755: 8 total, 8 up, 8 in 2024-08-17T17:06:24.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:24 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:24.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:24 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:06:24.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:24 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:24.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:24 smithi063 ceph-mon[109843]: osdmap e755: 8 total, 8 up, 8 in 2024-08-17T17:06:24.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:24 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:24.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:24 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:06:24.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:24 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:24.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:24 smithi162 ceph-mon[104668]: osdmap e755: 8 total, 8 up, 8 in 2024-08-17T17:06:24.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:24 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:24.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:24 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-08-17T17:06:24.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:24 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:25.382 INFO:tasks.rados.rados.0.smithi063.stdout:2704: done (3 left) 2024-08-17T17:06:25.382 INFO:tasks.rados.rados.0.smithi063.stdout:2705: rollback oid 45 current snap is 346 2024-08-17T17:06:25.382 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 45 to 320 2024-08-17T17:06:25.383 INFO:tasks.rados.rados.0.smithi063.stdout:2703: finishing rollback tid 0 to smithi06387500-8 2024-08-17T17:06:25.383 INFO:tasks.rados.rados.0.smithi063.stdout:2703: finishing rollback tid 1 to smithi06387500-8 2024-08-17T17:06:25.383 INFO:tasks.rados.rados.0.smithi063.stdout:2703: finishing rollback tid 2 to smithi06387500-8 2024-08-17T17:06:25.383 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 922 (ObjNum 931 snap 316 seq_num 931) dirty exists 2024-08-17T17:06:25.383 INFO:tasks.rados.rados.0.smithi063.stdout:2701: expect (ObjNum 971 snap 333 seq_num 971) 2024-08-17T17:06:25.484 INFO:tasks.rados.rados.0.smithi063.stdout:2705: finishing rollback tid 0 to smithi06387500-45 2024-08-17T17:06:25.484 INFO:tasks.rados.rados.0.smithi063.stdout:2705: finishing rollback tid 1 to smithi06387500-45 2024-08-17T17:06:25.484 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 616 (ObjNum 129 snap 0 seq_num 1807210960) dirty dne 2024-08-17T17:06:25.484 INFO:tasks.rados.rados.0.smithi063.stdout:2701: done (3 left) 2024-08-17T17:06:25.484 INFO:tasks.rados.rados.0.smithi063.stdout:2702: done (2 left) 2024-08-17T17:06:25.484 INFO:tasks.rados.rados.0.smithi063.stdout:2703: done (1 left) 2024-08-17T17:06:25.485 INFO:tasks.rados.rados.0.smithi063.stdout:2705: done (0 left) 2024-08-17T17:06:25.485 INFO:tasks.rados.rados.0.smithi063.stdout:2706: read oid 23 snap -1 2024-08-17T17:06:25.485 INFO:tasks.rados.rados.0.smithi063.stdout:2706: expect (ObjNum 1012 snap 345 seq_num 1012) 2024-08-17T17:06:25.485 INFO:tasks.rados.rados.0.smithi063.stdout:2707: write oid 31 current snap is 346 2024-08-17T17:06:25.485 INFO:tasks.rados.rados.0.smithi063.stdout:2707: seq_num 1016 ranges {675912=727933,1920764=460673,2801290=274622} 2024-08-17T17:06:25.491 INFO:tasks.rados.rados.0.smithi063.stdout:2707: writing smithi06387500-31 from 675912 to 1403845 tid 1 2024-08-17T17:06:25.497 INFO:tasks.rados.rados.0.smithi063.stdout:2707: writing smithi06387500-31 from 1920764 to 2381437 tid 2 2024-08-17T17:06:25.501 INFO:tasks.rados.rados.0.smithi063.stdout:2707: writing smithi06387500-31 from 2801290 to 3075912 tid 3 2024-08-17T17:06:25.501 INFO:tasks.rados.rados.0.smithi063.stdout:2708: snap_remove snap 338 2024-08-17T17:06:25.659 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:25 smithi063 ceph-mon[109843]: Upgrade: Updating osd.1 2024-08-17T17:06:25.659 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:25 smithi063 ceph-mon[109843]: Deploying daemon osd.1 on smithi063 2024-08-17T17:06:25.659 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:25 smithi063 ceph-mon[109843]: pgmap v590: 89 pgs: 89 active+clean; 239 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.7 MiB/s wr, 22 op/s 2024-08-17T17:06:25.659 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:25 smithi063 ceph-mon[109843]: osdmap e756: 8 total, 8 up, 8 in 2024-08-17T17:06:25.660 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:25 smithi063 ceph-mon[114756]: Upgrade: Updating osd.1 2024-08-17T17:06:25.660 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:25 smithi063 ceph-mon[114756]: Deploying daemon osd.1 on smithi063 2024-08-17T17:06:25.660 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:25 smithi063 ceph-mon[114756]: pgmap v590: 89 pgs: 89 active+clean; 239 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.7 MiB/s wr, 22 op/s 2024-08-17T17:06:25.660 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:25 smithi063 ceph-mon[114756]: osdmap e756: 8 total, 8 up, 8 in 2024-08-17T17:06:25.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:25 smithi162 ceph-mon[104668]: Upgrade: Updating osd.1 2024-08-17T17:06:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:25 smithi162 ceph-mon[104668]: Deploying daemon osd.1 on smithi063 2024-08-17T17:06:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:25 smithi162 ceph-mon[104668]: pgmap v590: 89 pgs: 89 active+clean; 239 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.7 MiB/s wr, 22 op/s 2024-08-17T17:06:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:25 smithi162 ceph-mon[104668]: osdmap e756: 8 total, 8 up, 8 in 2024-08-17T17:06:26.387 INFO:tasks.rados.rados.0.smithi063.stdout:2708: done (2 left) 2024-08-17T17:06:26.387 INFO:tasks.rados.rados.0.smithi063.stdout:2709: delete oid 50 current snap is 346 2024-08-17T17:06:26.387 INFO:tasks.rados.rados.0.smithi063.stdout:2706: expect (ObjNum 1012 snap 345 seq_num 1012) 2024-08-17T17:06:26.479 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:26 smithi063 ceph-mon[109843]: osdmap e757: 8 total, 8 up, 8 in 2024-08-17T17:06:26.479 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:26 smithi063 ceph-mon[114756]: osdmap e757: 8 total, 8 up, 8 in 2024-08-17T17:06:26.494 INFO:tasks.rados.rados.0.smithi063.stdout:2707: finishing write tid 1 to smithi06387500-31 2024-08-17T17:06:26.494 INFO:tasks.rados.rados.0.smithi063.stdout:2707: finishing write tid 2 to smithi06387500-31 2024-08-17T17:06:26.494 INFO:tasks.rados.rados.0.smithi063.stdout:2707: finishing write tid 3 to smithi06387500-31 2024-08-17T17:06:26.495 INFO:tasks.rados.rados.0.smithi063.stdout:2707: finishing write tid 4 to smithi06387500-31 2024-08-17T17:06:26.495 INFO:tasks.rados.rados.0.smithi063.stdout:2707: finishing write tid 5 to smithi06387500-31 2024-08-17T17:06:26.495 INFO:tasks.rados.rados.0.smithi063.stdout:2707: oid 31 updating version 0 to 1779 2024-08-17T17:06:26.495 INFO:tasks.rados.rados.0.smithi063.stdout:2707: oid 31 updating version 1779 to 1781 2024-08-17T17:06:26.495 INFO:tasks.rados.rados.0.smithi063.stdout:2707: oid 31 version 1781 is already newer than 1780 2024-08-17T17:06:26.495 INFO:tasks.rados.rados.0.smithi063.stdout:2707: oid 31 version 1781 is already newer than 1778 2024-08-17T17:06:26.495 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 1781 (ObjNum 1016 snap 346 seq_num 1016) dirty exists 2024-08-17T17:06:26.495 INFO:tasks.rados.rados.0.smithi063.stdout:2707: left oid 31 (ObjNum 1016 snap 346 seq_num 1016) 2024-08-17T17:06:26.495 INFO:tasks.rados.rados.0.smithi063.stdout:2706: done (2 left) 2024-08-17T17:06:26.495 INFO:tasks.rados.rados.0.smithi063.stdout:2707: done (1 left) 2024-08-17T17:06:26.495 INFO:tasks.rados.rados.0.smithi063.stdout:2709: done (0 left) 2024-08-17T17:06:26.495 INFO:tasks.rados.rados.0.smithi063.stdout:2710: read oid 38 snap 331 2024-08-17T17:06:26.495 INFO:tasks.rados.rados.0.smithi063.stdout:2710: expect (ObjNum 868 snap 300 seq_num 868) 2024-08-17T17:06:26.495 INFO:tasks.rados.rados.0.smithi063.stdout:2711: write oid 26 current snap is 346 2024-08-17T17:06:26.496 INFO:tasks.rados.rados.0.smithi063.stdout:2711: seq_num 1018 ranges {709526=478431,1637728=464342,2856136=253390} 2024-08-17T17:06:26.501 INFO:tasks.rados.rados.0.smithi063.stdout:2711: writing smithi06387500-26 from 709526 to 1187957 tid 1 2024-08-17T17:06:26.506 INFO:tasks.rados.rados.0.smithi063.stdout:2711: writing smithi06387500-26 from 1637728 to 2102070 tid 2 2024-08-17T17:06:26.511 INFO:tasks.rados.rados.0.smithi063.stdout:2711: writing smithi06387500-26 from 2856136 to 3109526 tid 3 2024-08-17T17:06:26.512 INFO:tasks.rados.rados.0.smithi063.stdout:2712: read oid 36 snap -1 2024-08-17T17:06:26.512 INFO:tasks.rados.rados.0.smithi063.stdout:2712: expect (ObjNum 948 snap 323 seq_num 948) 2024-08-17T17:06:26.512 INFO:tasks.rados.rados.0.smithi063.stdout:2713: snap_create 2024-08-17T17:06:26.512 INFO:tasks.rados.rados.0.smithi063.stdout:2710: expect (ObjNum 868 snap 300 seq_num 868) 2024-08-17T17:06:26.642 INFO:tasks.rados.rados.0.smithi063.stdout:2711: finishing write tid 1 to smithi06387500-26 2024-08-17T17:06:26.642 INFO:tasks.rados.rados.0.smithi063.stdout:2711: finishing write tid 2 to smithi06387500-26 2024-08-17T17:06:26.642 INFO:tasks.rados.rados.0.smithi063.stdout:2711: finishing write tid 3 to smithi06387500-26 2024-08-17T17:06:26.642 INFO:tasks.rados.rados.0.smithi063.stdout:2711: finishing write tid 4 to smithi06387500-26 2024-08-17T17:06:26.642 INFO:tasks.rados.rados.0.smithi063.stdout:2711: finishing write tid 5 to smithi06387500-26 2024-08-17T17:06:26.642 INFO:tasks.rados.rados.0.smithi063.stdout:2711: oid 26 updating version 0 to 892 2024-08-17T17:06:26.642 INFO:tasks.rados.rados.0.smithi063.stdout:2711: oid 26 version 892 is already newer than 891 2024-08-17T17:06:26.642 INFO:tasks.rados.rados.0.smithi063.stdout:2711: oid 26 version 892 is already newer than 890 2024-08-17T17:06:26.643 INFO:tasks.rados.rados.0.smithi063.stdout:2711: oid 26 version 892 is already newer than 889 2024-08-17T17:06:26.643 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 892 (ObjNum 1018 snap 346 seq_num 1018) dirty exists 2024-08-17T17:06:26.643 INFO:tasks.rados.rados.0.smithi063.stdout:2711: left oid 26 (ObjNum 1018 snap 346 seq_num 1018) 2024-08-17T17:06:26.643 INFO:tasks.rados.rados.0.smithi063.stdout:2712: expect (ObjNum 948 snap 323 seq_num 948) 2024-08-17T17:06:26.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:26 smithi162 ceph-mon[104668]: osdmap e757: 8 total, 8 up, 8 in 2024-08-17T17:06:27.112 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:26 smithi063 systemd[1]: Stopping Ceph osd.1 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:06:27.390 INFO:tasks.rados.rados.0.smithi063.stdout:2710: done (3 left) 2024-08-17T17:06:27.390 INFO:tasks.rados.rados.0.smithi063.stdout:2711: done (2 left) 2024-08-17T17:06:27.390 INFO:tasks.rados.rados.0.smithi063.stdout:2712: done (1 left) 2024-08-17T17:06:27.390 INFO:tasks.rados.rados.0.smithi063.stdout:2713: done (0 left) 2024-08-17T17:06:27.391 INFO:tasks.rados.rados.0.smithi063.stdout:2714: read oid 4 snap -1 2024-08-17T17:06:27.391 INFO:tasks.rados.rados.0.smithi063.stdout:2714: expect deleted 2024-08-17T17:06:27.391 INFO:tasks.rados.rados.0.smithi063.stdout:2715: snap_remove snap 342 2024-08-17T17:06:27.391 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:27 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1[52945]: 2024-08-17T17:06:27.109+0000 7fa1b3335640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:06:27.391 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:27 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1[52945]: 2024-08-17T17:06:27.109+0000 7fa1b3335640 -1 osd.1 758 *** Got signal Terminated *** 2024-08-17T17:06:27.391 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:27 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1[52945]: 2024-08-17T17:06:27.109+0000 7fa1b3335640 -1 osd.1 758 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:06:27.607 INFO:tasks.rados.rados.0.smithi063.stdout:2715: done (1 left) 2024-08-17T17:06:27.608 INFO:tasks.rados.rados.0.smithi063.stdout:2716: snap_remove snap 341 2024-08-17T17:06:27.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:27 smithi063 ceph-mon[114756]: pgmap v593: 89 pgs: 89 active+clean; 243 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 686 KiB/s wr, 4 op/s 2024-08-17T17:06:27.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:27 smithi063 ceph-mon[114756]: osdmap e758: 8 total, 8 up, 8 in 2024-08-17T17:06:27.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:27 smithi063 ceph-mon[114756]: osd.1 marked itself down and dead 2024-08-17T17:06:27.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:27 smithi063 ceph-mon[109843]: pgmap v593: 89 pgs: 89 active+clean; 243 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 686 KiB/s wr, 4 op/s 2024-08-17T17:06:27.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:27 smithi063 ceph-mon[109843]: osdmap e758: 8 total, 8 up, 8 in 2024-08-17T17:06:27.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:27 smithi063 ceph-mon[109843]: osd.1 marked itself down and dead 2024-08-17T17:06:27.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:27 smithi162 ceph-mon[104668]: pgmap v593: 89 pgs: 89 active+clean; 243 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 686 KiB/s wr, 4 op/s 2024-08-17T17:06:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:27 smithi162 ceph-mon[104668]: osdmap e758: 8 total, 8 up, 8 in 2024-08-17T17:06:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:27 smithi162 ceph-mon[104668]: osd.1 marked itself down and dead 2024-08-17T17:06:28.229 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:27 smithi063 podman[132240]: 2024-08-17 17:06:27.927356452 +0000 UTC m=+0.929242174 container died 5d6fdb6000ad5191aeb7fa16ecb383b40d3c47d47d7008e525a7061a8568b2b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-08-17T17:06:28.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:28 smithi063 ceph-mon[114756]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:06:28.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:28 smithi063 ceph-mon[114756]: osdmap e759: 8 total, 7 up, 8 in 2024-08-17T17:06:28.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:28 smithi063 ceph-mon[114756]: osdmap e760: 8 total, 7 up, 8 in 2024-08-17T17:06:28.530 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:28 smithi063 podman[132240]: 2024-08-17 17:06:28.318690122 +0000 UTC m=+1.320575845 container cleanup 5d6fdb6000ad5191aeb7fa16ecb383b40d3c47d47d7008e525a7061a8568b2b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-ced649f, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-08-17T17:06:28.530 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:28 smithi063 bash[132240]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1 2024-08-17T17:06:28.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:28 smithi063 ceph-mon[109843]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:06:28.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:28 smithi063 ceph-mon[109843]: osdmap e759: 8 total, 7 up, 8 in 2024-08-17T17:06:28.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:28 smithi063 ceph-mon[109843]: osdmap e760: 8 total, 7 up, 8 in 2024-08-17T17:06:28.613 INFO:tasks.rados.rados.0.smithi063.stdout:2716: done (1 left) 2024-08-17T17:06:28.613 INFO:tasks.rados.rados.0.smithi063.stdout:2717: write oid 16 current snap is 347 2024-08-17T17:06:28.613 INFO:tasks.rados.rados.0.smithi063.stdout:2717: seq_num 1019 ranges {726333=670033,2096210=724353} 2024-08-17T17:06:28.622 INFO:tasks.rados.rados.0.smithi063.stdout:2717: writing smithi06387500-16 from 726333 to 1396366 tid 1 2024-08-17T17:06:28.631 INFO:tasks.rados.rados.0.smithi063.stdout:2717: writing smithi06387500-16 from 2096210 to 2820563 tid 2 2024-08-17T17:06:28.631 INFO:tasks.rados.rados.0.smithi063.stdout:2718: snap_create 2024-08-17T17:06:28.631 INFO:tasks.rados.rados.0.smithi063.stdout:2717: finishing write tid 1 to smithi06387500-16 2024-08-17T17:06:28.634 INFO:tasks.rados.rados.0.smithi063.stdout:2717: finishing write tid 2 to smithi06387500-16 2024-08-17T17:06:28.634 INFO:tasks.rados.rados.0.smithi063.stdout:2717: finishing write tid 3 to smithi06387500-16 2024-08-17T17:06:28.634 INFO:tasks.rados.rados.0.smithi063.stdout:2717: finishing write tid 4 to smithi06387500-16 2024-08-17T17:06:28.635 INFO:tasks.rados.rados.0.smithi063.stdout:2717: oid 16 updating version 0 to 993 2024-08-17T17:06:28.635 INFO:tasks.rados.rados.0.smithi063.stdout:2717: oid 16 version 993 is already newer than 991 2024-08-17T17:06:28.635 INFO:tasks.rados.rados.0.smithi063.stdout:2717: oid 16 version 993 is already newer than 992 2024-08-17T17:06:28.635 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 993 (ObjNum 1019 snap 347 seq_num 1019) dirty exists 2024-08-17T17:06:28.635 INFO:tasks.rados.rados.0.smithi063.stdout:2717: left oid 16 (ObjNum 1019 snap 347 seq_num 1019) 2024-08-17T17:06:28.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:28 smithi162 ceph-mon[104668]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:06:28.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:28 smithi162 ceph-mon[104668]: osdmap e759: 8 total, 7 up, 8 in 2024-08-17T17:06:28.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:28 smithi162 ceph-mon[104668]: osdmap e760: 8 total, 7 up, 8 in 2024-08-17T17:06:28.923 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:28 smithi063 podman[132262]: 2024-08-17 17:06:28.561081994 +0000 UTC m=+0.628531589 container remove 5d6fdb6000ad5191aeb7fa16ecb383b40d3c47d47d7008e525a7061a8568b2b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, org.label-schema.vendor=CentOS) 2024-08-17T17:06:29.191 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:28 smithi063 podman[132354]: 2024-08-17 17:06:28.825442842 +0000 UTC m=+0.018059153 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:06:29.191 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:28 smithi063 podman[132354]: 2024-08-17 17:06:28.929599568 +0000 UTC m=+0.122215887 container create 36e77f63f0672ec4b24df9db930aafa86fb0c9e01ec5c96a587a21caca2c82bf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:06:29.191 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:29 smithi063 podman[132354]: 2024-08-17 17:06:29.115638029 +0000 UTC m=+0.308254351 container init 36e77f63f0672ec4b24df9db930aafa86fb0c9e01ec5c96a587a21caca2c82bf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe) 2024-08-17T17:06:29.191 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:29 smithi063 podman[132354]: 2024-08-17 17:06:29.120491979 +0000 UTC m=+0.313108289 container start 36e77f63f0672ec4b24df9db930aafa86fb0c9e01ec5c96a587a21caca2c82bf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-08-17T17:06:29.191 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:29 smithi063 podman[132354]: 2024-08-17 17:06:29.190242423 +0000 UTC m=+0.382858744 container attach 36e77f63f0672ec4b24df9db930aafa86fb0c9e01ec5c96a587a21caca2c82bf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-08-17T17:06:29.529 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:29 smithi063 podman[132354]: 2024-08-17 17:06:29.317030005 +0000 UTC m=+0.509646323 container died 36e77f63f0672ec4b24df9db930aafa86fb0c9e01ec5c96a587a21caca2c82bf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-08-17T17:06:29.614 INFO:tasks.rados.rados.0.smithi063.stdout:2714: done (2 left) 2024-08-17T17:06:29.614 INFO:tasks.rados.rados.0.smithi063.stdout:2717: done (1 left) 2024-08-17T17:06:29.614 INFO:tasks.rados.rados.0.smithi063.stdout:2718: done (0 left) 2024-08-17T17:06:29.614 INFO:tasks.rados.rados.0.smithi063.stdout:2719: read oid 27 snap 332 2024-08-17T17:06:29.614 INFO:tasks.rados.rados.0.smithi063.stdout:2719: expect deleted 2024-08-17T17:06:29.614 INFO:tasks.rados.rados.0.smithi063.stdout:2720: write oid 30 current snap is 348 2024-08-17T17:06:29.614 INFO:tasks.rados.rados.0.smithi063.stdout:2720: seq_num 1020 ranges {743140=745282,2154692=468011} 2024-08-17T17:06:29.622 INFO:tasks.rados.rados.0.smithi063.stdout:2720: writing smithi06387500-30 from 743140 to 1488422 tid 1 2024-08-17T17:06:29.629 INFO:tasks.rados.rados.0.smithi063.stdout:2720: writing smithi06387500-30 from 2154692 to 2622703 tid 2 2024-08-17T17:06:29.629 INFO:tasks.rados.rados.0.smithi063.stdout:2721: write oid 24 current snap is 348 2024-08-17T17:06:29.629 INFO:tasks.rados.rados.0.smithi063.stdout:2721: seq_num 1021 ranges {759947=420531,1813174=728022} 2024-08-17T17:06:29.635 INFO:tasks.rados.rados.0.smithi063.stdout:2721: writing smithi06387500-24 from 759947 to 1180478 tid 1 2024-08-17T17:06:29.643 INFO:tasks.rados.rados.0.smithi063.stdout:2721: writing smithi06387500-24 from 1813174 to 2541196 tid 2 2024-08-17T17:06:29.643 INFO:tasks.rados.rados.0.smithi063.stdout:2722: snap_remove snap 331 2024-08-17T17:06:29.813 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:29 smithi063 ceph-mon[114756]: pgmap v597: 89 pgs: 10 stale+active+clean, 79 active+clean; 251 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 2.1 MiB/s wr, 24 op/s 2024-08-17T17:06:29.813 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:29 smithi063 ceph-mon[114756]: osdmap e761: 8 total, 7 up, 8 in 2024-08-17T17:06:29.813 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:29 smithi063 podman[132396]: 2024-08-17 17:06:29.794291001 +0000 UTC m=+0.465945866 container remove 36e77f63f0672ec4b24df9db930aafa86fb0c9e01ec5c96a587a21caca2c82bf (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:06:29.814 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:29 smithi063 ceph-mon[109843]: pgmap v597: 89 pgs: 10 stale+active+clean, 79 active+clean; 251 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 2.1 MiB/s wr, 24 op/s 2024-08-17T17:06:29.814 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:29 smithi063 ceph-mon[109843]: osdmap e761: 8 total, 7 up, 8 in 2024-08-17T17:06:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:29 smithi162 ceph-mon[104668]: pgmap v597: 89 pgs: 10 stale+active+clean, 79 active+clean; 251 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 2.1 MiB/s wr, 24 op/s 2024-08-17T17:06:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:29 smithi162 ceph-mon[104668]: osdmap e761: 8 total, 7 up, 8 in 2024-08-17T17:06:30.221 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:29 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.1.service: Deactivated successfully. 2024-08-17T17:06:30.222 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:29 smithi063 systemd[1]: Stopped Ceph osd.1 for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:06:30.222 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:29 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.1.service: Consumed 17.173s CPU time. 2024-08-17T17:06:30.222 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 systemd[1]: Starting Ceph osd.1 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:06:30.506 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 podman[132500]: 2024-08-17 17:06:30.220711354 +0000 UTC m=+0.091070154 container create 73addcbe2efcaec66d2d8a6ebf1dadf7e9a753275f184013f7d96b9151845137 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD) 2024-08-17T17:06:30.506 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 podman[132500]: 2024-08-17 17:06:30.147899063 +0000 UTC m=+0.018257817 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:06:30.507 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 podman[132500]: 2024-08-17 17:06:30.423750329 +0000 UTC m=+0.294109092 container init 73addcbe2efcaec66d2d8a6ebf1dadf7e9a753275f184013f7d96b9151845137 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:06:30.507 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 podman[132500]: 2024-08-17 17:06:30.428601591 +0000 UTC m=+0.298960354 container start 73addcbe2efcaec66d2d8a6ebf1dadf7e9a753275f184013f7d96b9151845137 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.36.0) 2024-08-17T17:06:30.507 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 podman[132500]: 2024-08-17 17:06:30.506302361 +0000 UTC m=+0.376661116 container attach 73addcbe2efcaec66d2d8a6ebf1dadf7e9a753275f184013f7d96b9151845137 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-08-17T17:06:30.633 INFO:tasks.rados.rados.0.smithi063.stdout:2722: done (3 left) 2024-08-17T17:06:30.633 INFO:tasks.rados.rados.0.smithi063.stdout:2723: read oid 19 snap -1 2024-08-17T17:06:30.633 INFO:tasks.rados.rados.0.smithi063.stdout:2723: expect (ObjNum 1011 snap 345 seq_num 1011) 2024-08-17T17:06:30.633 INFO:tasks.rados.rados.0.smithi063.stdout:2724: read oid 34 snap 315 2024-08-17T17:06:30.633 INFO:tasks.rados.rados.0.smithi063.stdout:2724: expect (ObjNum 884 snap 309 seq_num 884) 2024-08-17T17:06:30.633 INFO:tasks.rados.rados.0.smithi063.stdout:2725: rollback oid 5 current snap is 348 2024-08-17T17:06:30.633 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 337 2024-08-17T17:06:30.634 INFO:tasks.rados.rados.0.smithi063.stdout:2720: finishing write tid 1 to smithi06387500-30 2024-08-17T17:06:30.634 INFO:tasks.rados.rados.0.smithi063.stdout:2720: finishing write tid 2 to smithi06387500-30 2024-08-17T17:06:30.634 INFO:tasks.rados.rados.0.smithi063.stdout:2720: finishing write tid 3 to smithi06387500-30 2024-08-17T17:06:30.634 INFO:tasks.rados.rados.0.smithi063.stdout:2720: finishing write tid 4 to smithi06387500-30 2024-08-17T17:06:30.634 INFO:tasks.rados.rados.0.smithi063.stdout:2720: oid 30 updating version 0 to 1277 2024-08-17T17:06:30.634 INFO:tasks.rados.rados.0.smithi063.stdout:2720: oid 30 updating version 1277 to 1279 2024-08-17T17:06:30.634 INFO:tasks.rados.rados.0.smithi063.stdout:2720: oid 30 version 1279 is already newer than 1278 2024-08-17T17:06:30.634 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1279 (ObjNum 1020 snap 348 seq_num 1020) dirty exists 2024-08-17T17:06:30.634 INFO:tasks.rados.rados.0.smithi063.stdout:2720: left oid 30 (ObjNum 1020 snap 348 seq_num 1020) 2024-08-17T17:06:30.634 INFO:tasks.rados.rados.0.smithi063.stdout:2721: finishing write tid 1 to smithi06387500-24 2024-08-17T17:06:30.634 INFO:tasks.rados.rados.0.smithi063.stdout:2721: finishing write tid 2 to smithi06387500-24 2024-08-17T17:06:30.634 INFO:tasks.rados.rados.0.smithi063.stdout:2721: finishing write tid 3 to smithi06387500-24 2024-08-17T17:06:30.634 INFO:tasks.rados.rados.0.smithi063.stdout:2721: finishing write tid 4 to smithi06387500-24 2024-08-17T17:06:30.635 INFO:tasks.rados.rados.0.smithi063.stdout:2721: oid 24 updating version 0 to 994 2024-08-17T17:06:30.635 INFO:tasks.rados.rados.0.smithi063.stdout:2721: oid 24 updating version 994 to 996 2024-08-17T17:06:30.635 INFO:tasks.rados.rados.0.smithi063.stdout:2721: oid 24 version 996 is already newer than 995 2024-08-17T17:06:30.635 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 996 (ObjNum 1021 snap 348 seq_num 1021) dirty exists 2024-08-17T17:06:30.635 INFO:tasks.rados.rados.0.smithi063.stdout:2721: left oid 24 (ObjNum 1021 snap 348 seq_num 1021) 2024-08-17T17:06:30.635 INFO:tasks.rados.rados.0.smithi063.stdout:2719: done (5 left) 2024-08-17T17:06:30.635 INFO:tasks.rados.rados.0.smithi063.stdout:2720: done (4 left) 2024-08-17T17:06:30.635 INFO:tasks.rados.rados.0.smithi063.stdout:2721: done (3 left) 2024-08-17T17:06:30.635 INFO:tasks.rados.rados.0.smithi063.stdout:2726: snap_create 2024-08-17T17:06:30.638 INFO:tasks.rados.rados.0.smithi063.stdout:2725: finishing rollback tid 1 to smithi06387500-5 2024-08-17T17:06:30.638 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1173 (ObjNum 129 snap 0 seq_num 1807368648) dirty dne 2024-08-17T17:06:30.640 INFO:tasks.rados.rados.0.smithi063.stdout:2723: expect (ObjNum 1011 snap 345 seq_num 1011) 2024-08-17T17:06:30.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:30 smithi063 ceph-mon[109843]: osdmap e762: 8 total, 7 up, 8 in 2024-08-17T17:06:30.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:30 smithi063 ceph-mon[114756]: osdmap e762: 8 total, 7 up, 8 in 2024-08-17T17:06:30.780 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate[132514]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:30.780 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 bash[132500]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:30.780 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate[132514]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:30.780 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 bash[132500]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:30.780 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate[132514]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:06:30.780 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 bash[132500]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:06:30.780 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate[132514]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:30.780 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 bash[132500]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:30.780 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate[132514]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:30.780 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 bash[132500]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:06:30.781 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate[132514]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:06:30.781 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 bash[132500]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:06:30.781 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate[132514]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-08-17T17:06:30.781 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 bash[132500]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2024-08-17T17:06:30.818 INFO:tasks.rados.rados.0.smithi063.stdout:2724: expect (ObjNum 884 snap 309 seq_num 884) 2024-08-17T17:06:30.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:30 smithi162 ceph-mon[104668]: osdmap e762: 8 total, 7 up, 8 in 2024-08-17T17:06:31.279 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate[132514]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:06:31.279 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 bash[132500]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:06:31.279 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate[132514]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:06:31.279 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 bash[132500]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2024-08-17T17:06:31.280 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate[132514]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-17T17:06:31.280 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 bash[132500]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-08-17T17:06:31.280 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate[132514]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:06:31.280 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 bash[132500]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-08-17T17:06:31.280 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate[132514]: --> ceph-volume lvm activate successful for osd ID: 1 2024-08-17T17:06:31.280 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 bash[132500]: --> ceph-volume lvm activate successful for osd ID: 1 2024-08-17T17:06:31.280 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:30 smithi063 podman[132500]: 2024-08-17 17:06:30.998047436 +0000 UTC m=+0.868406190 container died 73addcbe2efcaec66d2d8a6ebf1dadf7e9a753275f184013f7d96b9151845137 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-08-17T17:06:31.637 INFO:tasks.rados.rados.0.smithi063.stdout:2723: done (3 left) 2024-08-17T17:06:31.637 INFO:tasks.rados.rados.0.smithi063.stdout:2724: done (2 left) 2024-08-17T17:06:31.637 INFO:tasks.rados.rados.0.smithi063.stdout:2725: done (1 left) 2024-08-17T17:06:31.637 INFO:tasks.rados.rados.0.smithi063.stdout:2726: done (0 left) 2024-08-17T17:06:31.637 INFO:tasks.rados.rados.0.smithi063.stdout:2727: rollback oid 32 current snap is 349 2024-08-17T17:06:31.637 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 32 to 307 2024-08-17T17:06:31.637 INFO:tasks.rados.rados.0.smithi063.stdout:2728: delete oid 43 current snap is 349 2024-08-17T17:06:31.641 INFO:tasks.rados.rados.0.smithi063.stdout:2728: done (1 left) 2024-08-17T17:06:31.641 INFO:tasks.rados.rados.0.smithi063.stdout:2729: read oid 11 snap -1 2024-08-17T17:06:31.641 INFO:tasks.rados.rados.0.smithi063.stdout:2729: expect deleted 2024-08-17T17:06:31.641 INFO:tasks.rados.rados.0.smithi063.stdout:2730: snap_create 2024-08-17T17:06:31.642 INFO:tasks.rados.rados.0.smithi063.stdout:2727: finishing rollback tid 0 to smithi06387500-32 2024-08-17T17:06:31.642 INFO:tasks.rados.rados.0.smithi063.stdout:2727: finishing rollback tid 1 to smithi06387500-32 2024-08-17T17:06:31.642 INFO:tasks.rados.rados.0.smithi063.stdout:2727: finishing rollback tid 2 to smithi06387500-32 2024-08-17T17:06:31.643 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 535 (ObjNum 808 snap 280 seq_num 808) dirty exists 2024-08-17T17:06:31.868 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:31 smithi063 ceph-mon[114756]: pgmap v600: 89 pgs: 10 stale+active+clean, 79 active+clean; 251 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.3 MiB/s wr, 19 op/s 2024-08-17T17:06:31.868 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:31 smithi063 ceph-mon[114756]: osdmap e763: 8 total, 7 up, 8 in 2024-08-17T17:06:31.868 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:31.868 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:31.868 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:31 smithi063 ceph-mon[109843]: pgmap v600: 89 pgs: 10 stale+active+clean, 79 active+clean; 251 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.3 MiB/s wr, 19 op/s 2024-08-17T17:06:31.868 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:31 smithi063 ceph-mon[109843]: osdmap e763: 8 total, 7 up, 8 in 2024-08-17T17:06:31.868 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:31.868 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:31.869 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:31 smithi063 podman[132651]: 2024-08-17 17:06:31.543631288 +0000 UTC m=+0.538880658 container remove 73addcbe2efcaec66d2d8a6ebf1dadf7e9a753275f184013f7d96b9151845137 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:06:31.869 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:31 smithi063 podman[132719]: 2024-08-17 17:06:31.744723963 +0000 UTC m=+0.095086978 container create 11bee521ecad9e307f63ab6b94f326c236dc9b4dac1f5e4d986f9c3394bea650 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.36.0, ceph=True) 2024-08-17T17:06:31.869 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:31 smithi063 podman[132719]: 2024-08-17 17:06:31.667470462 +0000 UTC m=+0.017833493 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:06:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:31 smithi162 ceph-mon[104668]: pgmap v600: 89 pgs: 10 stale+active+clean, 79 active+clean; 251 MiB data, 849 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 1.3 MiB/s wr, 19 op/s 2024-08-17T17:06:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:31 smithi162 ceph-mon[104668]: osdmap e763: 8 total, 7 up, 8 in 2024-08-17T17:06:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:32.279 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:31 smithi063 podman[132719]: 2024-08-17 17:06:31.964350864 +0000 UTC m=+0.314713895 container init 11bee521ecad9e307f63ab6b94f326c236dc9b4dac1f5e4d986f9c3394bea650 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, ceph=True, org.label-schema.build-date=20240731) 2024-08-17T17:06:32.279 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:31 smithi063 podman[132719]: 2024-08-17 17:06:31.96947407 +0000 UTC m=+0.319837082 container start 11bee521ecad9e307f63ab6b94f326c236dc9b4dac1f5e4d986f9c3394bea650 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-08-17T17:06:32.279 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:31 smithi063 ceph-osd[132756]: -- 172.21.15.63:0/1019712754 <== mon.0 v2:172.21.15.63:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5642e0fb7a40 con 0x5642e1d86400 2024-08-17T17:06:32.279 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:32 smithi063 bash[132719]: 11bee521ecad9e307f63ab6b94f326c236dc9b4dac1f5e4d986f9c3394bea650 2024-08-17T17:06:32.280 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:32 smithi063 systemd[1]: Started Ceph osd.1 for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:06:32.639 INFO:tasks.rados.rados.0.smithi063.stdout:2727: done (2 left) 2024-08-17T17:06:32.639 INFO:tasks.rados.rados.0.smithi063.stdout:2729: done (1 left) 2024-08-17T17:06:32.640 INFO:tasks.rados.rados.0.smithi063.stdout:2730: done (0 left) 2024-08-17T17:06:32.640 INFO:tasks.rados.rados.0.smithi063.stdout:2731: read oid 7 snap 346 2024-08-17T17:06:32.640 INFO:tasks.rados.rados.0.smithi063.stdout:2731: expect (ObjNum 963 snap 327 seq_num 963) 2024-08-17T17:06:32.640 INFO:tasks.rados.rados.0.smithi063.stdout:2732: write oid 17 current snap is 350 2024-08-17T17:06:32.640 INFO:tasks.rados.rados.0.smithi063.stdout:2732: seq_num 1023 ranges {793561=571029,1813785=731691,2993251=200310} 2024-08-17T17:06:32.647 INFO:tasks.rados.rados.0.smithi063.stdout:2732: writing smithi06387500-17 from 793561 to 1364590 tid 1 2024-08-17T17:06:32.653 INFO:tasks.rados.rados.0.smithi063.stdout:2732: writing smithi06387500-17 from 1813785 to 2545476 tid 2 2024-08-17T17:06:32.657 INFO:tasks.rados.rados.0.smithi063.stdout:2732: writing smithi06387500-17 from 2993251 to 3193561 tid 3 2024-08-17T17:06:32.657 INFO:tasks.rados.rados.0.smithi063.stdout:2733: read oid 13 snap -1 2024-08-17T17:06:32.657 INFO:tasks.rados.rados.0.smithi063.stdout:2733: expect deleted 2024-08-17T17:06:32.657 INFO:tasks.rados.rados.0.smithi063.stdout:2734: read oid 40 snap -1 2024-08-17T17:06:32.657 INFO:tasks.rados.rados.0.smithi063.stdout:2734: expect (ObjNum 1006 snap 343 seq_num 1006) 2024-08-17T17:06:32.657 INFO:tasks.rados.rados.0.smithi063.stdout:2735: write oid 11 current snap is 350 2024-08-17T17:06:32.657 INFO:tasks.rados.rados.0.smithi063.stdout:2735: seq_num 1024 ranges {410368=646278,1472267=475349,2504321=405600} 2024-08-17T17:06:32.663 INFO:tasks.rados.rados.0.smithi063.stdout:2735: writing smithi06387500-11 from 410368 to 1056646 tid 1 2024-08-17T17:06:32.668 INFO:tasks.rados.rados.0.smithi063.stdout:2735: writing smithi06387500-11 from 1472267 to 1947616 tid 2 2024-08-17T17:06:32.673 INFO:tasks.rados.rados.0.smithi063.stdout:2735: writing smithi06387500-11 from 2504321 to 2909921 tid 3 2024-08-17T17:06:32.673 INFO:tasks.rados.rados.0.smithi063.stdout:2736: read oid 34 snap -1 2024-08-17T17:06:32.673 INFO:tasks.rados.rados.0.smithi063.stdout:2736: expect (ObjNum 971 snap 333 seq_num 971) 2024-08-17T17:06:32.674 INFO:tasks.rados.rados.0.smithi063.stdout:2737: read oid 35 snap 332 2024-08-17T17:06:32.674 INFO:tasks.rados.rados.0.smithi063.stdout:2737: expect (ObjNum 941 snap 323 seq_num 941) 2024-08-17T17:06:32.674 INFO:tasks.rados.rados.0.smithi063.stdout:2738: write oid 19 current snap is 350 2024-08-17T17:06:32.674 INFO:tasks.rados.rados.0.smithi063.stdout:2731: expect (ObjNum 963 snap 327 seq_num 963) 2024-08-17T17:06:32.776 INFO:tasks.rados.rados.0.smithi063.stdout:2732: finishing write tid 1 to smithi06387500-17 2024-08-17T17:06:32.777 INFO:tasks.rados.rados.0.smithi063.stdout:2732: finishing write tid 2 to smithi06387500-17 2024-08-17T17:06:32.777 INFO:tasks.rados.rados.0.smithi063.stdout:2732: finishing write tid 3 to smithi06387500-17 2024-08-17T17:06:32.777 INFO:tasks.rados.rados.0.smithi063.stdout:2732: finishing write tid 4 to smithi06387500-17 2024-08-17T17:06:32.777 INFO:tasks.rados.rados.0.smithi063.stdout:2732: finishing write tid 5 to smithi06387500-17 2024-08-17T17:06:32.777 INFO:tasks.rados.rados.0.smithi063.stdout:2732: oid 17 updating version 0 to 904 2024-08-17T17:06:32.777 INFO:tasks.rados.rados.0.smithi063.stdout:2732: oid 17 version 904 is already newer than 902 2024-08-17T17:06:32.777 INFO:tasks.rados.rados.0.smithi063.stdout:2732: oid 17 version 904 is already newer than 901 2024-08-17T17:06:32.777 INFO:tasks.rados.rados.0.smithi063.stdout:2732: oid 17 version 904 is already newer than 903 2024-08-17T17:06:32.777 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 904 (ObjNum 1023 snap 350 seq_num 1023) dirty exists 2024-08-17T17:06:32.777 INFO:tasks.rados.rados.0.smithi063.stdout:2732: left oid 17 (ObjNum 1023 snap 350 seq_num 1023) 2024-08-17T17:06:32.777 INFO:tasks.rados.rados.0.smithi063.stdout:2734: expect (ObjNum 1006 snap 343 seq_num 1006) 2024-08-17T17:06:32.874 INFO:tasks.rados.rados.0.smithi063.stdout:2735: finishing write tid 1 to smithi06387500-11 2024-08-17T17:06:32.875 INFO:tasks.rados.rados.0.smithi063.stdout:2735: finishing write tid 2 to smithi06387500-11 2024-08-17T17:06:32.875 INFO:tasks.rados.rados.0.smithi063.stdout:2738: seq_num 1025 ranges {427175=721527,1930749=735360} 2024-08-17T17:06:32.880 INFO:tasks.rados.rados.0.smithi063.stdout:2738: writing smithi06387500-19 from 427175 to 1148702 tid 1 2024-08-17T17:06:32.889 INFO:tasks.rados.rados.0.smithi063.stdout:2738: writing smithi06387500-19 from 1930749 to 2666109 tid 2 2024-08-17T17:06:32.889 INFO:tasks.rados.rados.0.smithi063.stdout:2731: done (7 left) 2024-08-17T17:06:32.889 INFO:tasks.rados.rados.0.smithi063.stdout:2732: done (6 left) 2024-08-17T17:06:32.889 INFO:tasks.rados.rados.0.smithi063.stdout:2733: done (5 left) 2024-08-17T17:06:32.889 INFO:tasks.rados.rados.0.smithi063.stdout:2734: done (4 left) 2024-08-17T17:06:32.889 INFO:tasks.rados.rados.0.smithi063.stdout:2739: snap_remove snap 315 2024-08-17T17:06:32.921 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:32 smithi063 ceph-mon[114756]: osdmap e764: 8 total, 7 up, 8 in 2024-08-17T17:06:32.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:32 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:32.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:32 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:32.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:32 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:32.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:32 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:32.922 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:32 smithi063 ceph-mon[109843]: osdmap e764: 8 total, 7 up, 8 in 2024-08-17T17:06:32.922 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:32 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:32.922 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:32 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:32.922 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:32 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:32.922 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:32 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:32.975 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:32 smithi162 ceph-mon[104668]: osdmap e764: 8 total, 7 up, 8 in 2024-08-17T17:06:32.975 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:32 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:32.975 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:32 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:32.975 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:32 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:32.975 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:32 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:33.620 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:06:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:06:33] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.51.0" 2024-08-17T17:06:33.641 INFO:tasks.rados.rados.0.smithi063.stdout:2739: done (4 left) 2024-08-17T17:06:33.641 INFO:tasks.rados.rados.0.smithi063.stdout:2740: write oid 5 current snap is 350 2024-08-17T17:06:33.641 INFO:tasks.rados.rados.0.smithi063.stdout:2735: finishing write tid 3 to smithi06387500-11 2024-08-17T17:06:33.642 INFO:tasks.rados.rados.0.smithi063.stdout:2735: finishing write tid 4 to smithi06387500-11 2024-08-17T17:06:33.642 INFO:tasks.rados.rados.0.smithi063.stdout:2735: finishing write tid 5 to smithi06387500-11 2024-08-17T17:06:33.642 INFO:tasks.rados.rados.0.smithi063.stdout:2735: oid 11 updating version 0 to 427 2024-08-17T17:06:33.642 INFO:tasks.rados.rados.0.smithi063.stdout:2735: oid 11 updating version 427 to 428 2024-08-17T17:06:33.642 INFO:tasks.rados.rados.0.smithi063.stdout:2735: oid 11 updating version 428 to 429 2024-08-17T17:06:33.642 INFO:tasks.rados.rados.0.smithi063.stdout:2735: oid 11 updating version 429 to 430 2024-08-17T17:06:33.642 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 430 (ObjNum 1024 snap 350 seq_num 1024) dirty exists 2024-08-17T17:06:33.642 INFO:tasks.rados.rados.0.smithi063.stdout:2735: left oid 11 (ObjNum 1024 snap 350 seq_num 1024) 2024-08-17T17:06:33.642 INFO:tasks.rados.rados.0.smithi063.stdout:2736: expect (ObjNum 971 snap 333 seq_num 971) 2024-08-17T17:06:33.745 INFO:tasks.rados.rados.0.smithi063.stdout:2737: expect (ObjNum 941 snap 323 seq_num 941) 2024-08-17T17:06:33.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:33 smithi162 ceph-mon[104668]: pgmap v603: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 259 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 918 KiB/s wr, 8 op/s; 58/501 objects degraded (11.577%) 2024-08-17T17:06:33.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:33 smithi162 ceph-mon[104668]: Health check failed: Degraded data redundancy: 58/501 objects degraded (11.577%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:33.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:33 smithi162 ceph-mon[104668]: osdmap e765: 8 total, 7 up, 8 in 2024-08-17T17:06:33.869 INFO:tasks.rados.rados.0.smithi063.stdout:2738: finishing write tid 1 to smithi06387500-19 2024-08-17T17:06:33.869 INFO:tasks.rados.rados.0.smithi063.stdout:2738: finishing write tid 2 to smithi06387500-19 2024-08-17T17:06:33.869 INFO:tasks.rados.rados.0.smithi063.stdout:2738: finishing write tid 3 to smithi06387500-19 2024-08-17T17:06:33.869 INFO:tasks.rados.rados.0.smithi063.stdout:2738: finishing write tid 4 to smithi06387500-19 2024-08-17T17:06:33.869 INFO:tasks.rados.rados.0.smithi063.stdout:2738: oid 19 updating version 0 to 1787 2024-08-17T17:06:33.869 INFO:tasks.rados.rados.0.smithi063.stdout:2738: oid 19 version 1787 is already newer than 1786 2024-08-17T17:06:33.869 INFO:tasks.rados.rados.0.smithi063.stdout:2738: oid 19 updating version 1787 to 1788 2024-08-17T17:06:33.870 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 1788 (ObjNum 1025 snap 350 seq_num 1025) dirty exists 2024-08-17T17:06:33.870 INFO:tasks.rados.rados.0.smithi063.stdout:2738: left oid 19 (ObjNum 1025 snap 350 seq_num 1025) 2024-08-17T17:06:33.870 INFO:tasks.rados.rados.0.smithi063.stdout:2740: seq_num 1026 ranges {443982=796776,1989231=479018,2959167=284815} 2024-08-17T17:06:33.875 INFO:tasks.rados.rados.0.smithi063.stdout:2740: writing smithi06387500-5 from 443982 to 1240758 tid 1 2024-08-17T17:06:33.882 INFO:tasks.rados.rados.0.smithi063.stdout:2740: writing smithi06387500-5 from 1989231 to 2468249 tid 2 2024-08-17T17:06:33.887 INFO:tasks.rados.rados.0.smithi063.stdout:2740: writing smithi06387500-5 from 2959167 to 3243982 tid 3 2024-08-17T17:06:33.887 INFO:tasks.rados.rados.0.smithi063.stdout:2735: done (4 left) 2024-08-17T17:06:33.887 INFO:tasks.rados.rados.0.smithi063.stdout:2736: done (3 left) 2024-08-17T17:06:33.887 INFO:tasks.rados.rados.0.smithi063.stdout:2737: done (2 left) 2024-08-17T17:06:33.887 INFO:tasks.rados.rados.0.smithi063.stdout:2738: done (1 left) 2024-08-17T17:06:33.887 INFO:tasks.rados.rados.0.smithi063.stdout:2741: rollback oid 1 current snap is 350 2024-08-17T17:06:33.887 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 1 to 349 2024-08-17T17:06:33.887 INFO:tasks.rados.rados.0.smithi063.stdout:2740: finishing write tid 1 to smithi06387500-5 2024-08-17T17:06:33.887 INFO:tasks.rados.rados.0.smithi063.stdout:2740: finishing write tid 2 to smithi06387500-5 2024-08-17T17:06:33.888 INFO:tasks.rados.rados.0.smithi063.stdout:2742: read oid 29 snap -1 2024-08-17T17:06:33.888 INFO:tasks.rados.rados.0.smithi063.stdout:2742: expect (ObjNum 855 snap 290 seq_num 855) 2024-08-17T17:06:33.888 INFO:tasks.rados.rados.0.smithi063.stdout:2743: snap_remove snap 346 2024-08-17T17:06:33.898 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1[132733]: 2024-08-17T17:06:33.618+0000 7f77d1e60640 -1 Falling back to public interface 2024-08-17T17:06:33.898 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:33 smithi063 ceph-mon[114756]: pgmap v603: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 259 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 918 KiB/s wr, 8 op/s; 58/501 objects degraded (11.577%) 2024-08-17T17:06:33.898 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:33 smithi063 ceph-mon[114756]: Health check failed: Degraded data redundancy: 58/501 objects degraded (11.577%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:33.898 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:33 smithi063 ceph-mon[114756]: osdmap e765: 8 total, 7 up, 8 in 2024-08-17T17:06:33.899 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:33 smithi063 ceph-mon[109843]: pgmap v603: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 259 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 918 KiB/s wr, 8 op/s; 58/501 objects degraded (11.577%) 2024-08-17T17:06:33.899 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:33 smithi063 ceph-mon[109843]: Health check failed: Degraded data redundancy: 58/501 objects degraded (11.577%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:33.899 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:33 smithi063 ceph-mon[109843]: osdmap e765: 8 total, 7 up, 8 in 2024-08-17T17:06:34.658 INFO:tasks.rados.rados.0.smithi063.stdout:2743: done (3 left) 2024-08-17T17:06:34.658 INFO:tasks.rados.rados.0.smithi063.stdout:2744: delete oid 31 current snap is 350 2024-08-17T17:06:34.658 INFO:tasks.rados.rados.0.smithi063.stdout:2740: finishing write tid 3 to smithi06387500-5 2024-08-17T17:06:34.658 INFO:tasks.rados.rados.0.smithi063.stdout:2740: finishing write tid 4 to smithi06387500-5 2024-08-17T17:06:34.659 INFO:tasks.rados.rados.0.smithi063.stdout:2740: finishing write tid 5 to smithi06387500-5 2024-08-17T17:06:34.659 INFO:tasks.rados.rados.0.smithi063.stdout:2740: oid 5 updating version 0 to 1342 2024-08-17T17:06:34.659 INFO:tasks.rados.rados.0.smithi063.stdout:2740: oid 5 version 1342 is already newer than 1341 2024-08-17T17:06:34.659 INFO:tasks.rados.rados.0.smithi063.stdout:2740: oid 5 updating version 1342 to 1343 2024-08-17T17:06:34.659 INFO:tasks.rados.rados.0.smithi063.stdout:2740: oid 5 version 1343 is already newer than 1340 2024-08-17T17:06:34.659 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1343 (ObjNum 1026 snap 350 seq_num 1026) dirty exists 2024-08-17T17:06:34.659 INFO:tasks.rados.rados.0.smithi063.stdout:2740: left oid 5 (ObjNum 1026 snap 350 seq_num 1026) 2024-08-17T17:06:34.659 INFO:tasks.rados.rados.0.smithi063.stdout:2741: finishing rollback tid 1 to smithi06387500-1 2024-08-17T17:06:34.659 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 1300 (ObjNum 641 snap 0 seq_num 574598992) dirty dne 2024-08-17T17:06:34.659 INFO:tasks.rados.rados.0.smithi063.stdout:2742: expect (ObjNum 855 snap 290 seq_num 855) 2024-08-17T17:06:34.766 INFO:tasks.rados.rados.0.smithi063.stdout:2740: done (3 left) 2024-08-17T17:06:34.766 INFO:tasks.rados.rados.0.smithi063.stdout:2741: done (2 left) 2024-08-17T17:06:34.766 INFO:tasks.rados.rados.0.smithi063.stdout:2742: done (1 left) 2024-08-17T17:06:34.766 INFO:tasks.rados.rados.0.smithi063.stdout:2744: done (0 left) 2024-08-17T17:06:34.766 INFO:tasks.rados.rados.0.smithi063.stdout:2745: write oid 22 current snap is 350 2024-08-17T17:06:34.766 INFO:tasks.rados.rados.0.smithi063.stdout:2745: seq_num 1028 ranges {477596=663627,1706195=482687,2614013=567041} 2024-08-17T17:06:34.772 INFO:tasks.rados.rados.0.smithi063.stdout:2745: writing smithi06387500-22 from 477596 to 1141223 tid 1 2024-08-17T17:06:34.778 INFO:tasks.rados.rados.0.smithi063.stdout:2745: writing smithi06387500-22 from 1706195 to 2188882 tid 2 2024-08-17T17:06:34.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:34 smithi063 ceph-mon[114756]: osdmap e766: 8 total, 7 up, 8 in 2024-08-17T17:06:34.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:34 smithi063 ceph-mon[109843]: osdmap e766: 8 total, 7 up, 8 in 2024-08-17T17:06:34.784 INFO:tasks.rados.rados.0.smithi063.stdout:2745: writing smithi06387500-22 from 2614013 to 3181054 tid 3 2024-08-17T17:06:34.784 INFO:tasks.rados.rados.0.smithi063.stdout:2746: write oid 40 current snap is 350 2024-08-17T17:06:34.784 INFO:tasks.rados.rados.0.smithi063.stdout:2745: finishing write tid 1 to smithi06387500-22 2024-08-17T17:06:34.784 INFO:tasks.rados.rados.0.smithi063.stdout:2746: seq_num 1029 ranges {494403=738876,1764677=626345,3041436=252967} 2024-08-17T17:06:34.790 INFO:tasks.rados.rados.0.smithi063.stdout:2746: writing smithi06387500-40 from 494403 to 1233279 tid 1 2024-08-17T17:06:34.797 INFO:tasks.rados.rados.0.smithi063.stdout:2746: writing smithi06387500-40 from 1764677 to 2391022 tid 2 2024-08-17T17:06:34.802 INFO:tasks.rados.rados.0.smithi063.stdout:2746: writing smithi06387500-40 from 3041436 to 3294403 tid 3 2024-08-17T17:06:34.802 INFO:tasks.rados.rados.0.smithi063.stdout:2747: delete oid 17 current snap is 350 2024-08-17T17:06:34.802 INFO:tasks.rados.rados.0.smithi063.stdout:2745: finishing write tid 2 to smithi06387500-22 2024-08-17T17:06:34.802 INFO:tasks.rados.rados.0.smithi063.stdout:2745: finishing write tid 3 to smithi06387500-22 2024-08-17T17:06:34.802 INFO:tasks.rados.rados.0.smithi063.stdout:2745: finishing write tid 4 to smithi06387500-22 2024-08-17T17:06:34.802 INFO:tasks.rados.rados.0.smithi063.stdout:2745: finishing write tid 5 to smithi06387500-22 2024-08-17T17:06:34.802 INFO:tasks.rados.rados.0.smithi063.stdout:2745: oid 22 updating version 0 to 575 2024-08-17T17:06:34.803 INFO:tasks.rados.rados.0.smithi063.stdout:2745: oid 22 version 575 is already newer than 574 2024-08-17T17:06:34.803 INFO:tasks.rados.rados.0.smithi063.stdout:2745: oid 22 version 575 is already newer than 573 2024-08-17T17:06:34.803 INFO:tasks.rados.rados.0.smithi063.stdout:2745: oid 22 updating version 575 to 576 2024-08-17T17:06:34.803 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 576 (ObjNum 1028 snap 350 seq_num 1028) dirty exists 2024-08-17T17:06:34.803 INFO:tasks.rados.rados.0.smithi063.stdout:2745: left oid 22 (ObjNum 1028 snap 350 seq_num 1028) 2024-08-17T17:06:34.803 INFO:tasks.rados.rados.0.smithi063.stdout:2746: finishing write tid 1 to smithi06387500-40 2024-08-17T17:06:34.803 INFO:tasks.rados.rados.0.smithi063.stdout:2746: finishing write tid 2 to smithi06387500-40 2024-08-17T17:06:34.803 INFO:tasks.rados.rados.0.smithi063.stdout:2745: done (2 left) 2024-08-17T17:06:34.804 INFO:tasks.rados.rados.0.smithi063.stdout:2747: done (1 left) 2024-08-17T17:06:34.804 INFO:tasks.rados.rados.0.smithi063.stdout:2748: delete oid 19 current snap is 350 2024-08-17T17:06:34.804 INFO:tasks.rados.rados.0.smithi063.stdout:2746: finishing write tid 3 to smithi06387500-40 2024-08-17T17:06:34.804 INFO:tasks.rados.rados.0.smithi063.stdout:2746: finishing write tid 4 to smithi06387500-40 2024-08-17T17:06:34.804 INFO:tasks.rados.rados.0.smithi063.stdout:2746: finishing write tid 5 to smithi06387500-40 2024-08-17T17:06:34.804 INFO:tasks.rados.rados.0.smithi063.stdout:2746: oid 40 updating version 0 to 933 2024-08-17T17:06:34.805 INFO:tasks.rados.rados.0.smithi063.stdout:2746: oid 40 updating version 933 to 935 2024-08-17T17:06:34.805 INFO:tasks.rados.rados.0.smithi063.stdout:2746: oid 40 updating version 935 to 936 2024-08-17T17:06:34.805 INFO:tasks.rados.rados.0.smithi063.stdout:2746: oid 40 version 936 is already newer than 934 2024-08-17T17:06:34.805 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 936 (ObjNum 1029 snap 350 seq_num 1029) dirty exists 2024-08-17T17:06:34.805 INFO:tasks.rados.rados.0.smithi063.stdout:2746: left oid 40 (ObjNum 1029 snap 350 seq_num 1029) 2024-08-17T17:06:34.805 INFO:tasks.rados.rados.0.smithi063.stdout:2746: done (1 left) 2024-08-17T17:06:34.805 INFO:tasks.rados.rados.0.smithi063.stdout:2748: done (0 left) 2024-08-17T17:06:34.805 INFO:tasks.rados.rados.0.smithi063.stdout:2749: snap_remove snap 345 2024-08-17T17:06:34.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:34 smithi162 ceph-mon[104668]: osdmap e766: 8 total, 7 up, 8 in 2024-08-17T17:06:35.656 INFO:tasks.rados.rados.0.smithi063.stdout:2749: done (0 left) 2024-08-17T17:06:35.656 INFO:tasks.rados.rados.0.smithi063.stdout:2750: snap_remove snap 323 2024-08-17T17:06:36.013 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:35 smithi162 ceph-mon[104668]: pgmap v606: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 259 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 917 KiB/s wr, 8 op/s; 58/501 objects degraded (11.577%) 2024-08-17T17:06:36.013 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:35 smithi162 ceph-mon[104668]: osdmap e767: 8 total, 7 up, 8 in 2024-08-17T17:06:36.013 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:35 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:36.013 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:35 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:35 smithi063 ceph-mon[114756]: pgmap v606: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 259 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 917 KiB/s wr, 8 op/s; 58/501 objects degraded (11.577%) 2024-08-17T17:06:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:35 smithi063 ceph-mon[114756]: osdmap e767: 8 total, 7 up, 8 in 2024-08-17T17:06:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:35 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:35 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:35 smithi063 ceph-mon[109843]: pgmap v606: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 259 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 917 KiB/s wr, 8 op/s; 58/501 objects degraded (11.577%) 2024-08-17T17:06:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:35 smithi063 ceph-mon[109843]: osdmap e767: 8 total, 7 up, 8 in 2024-08-17T17:06:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:35 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:35 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:36.669 INFO:tasks.rados.rados.0.smithi063.stdout:2750: done (0 left) 2024-08-17T17:06:36.670 INFO:tasks.rados.rados.0.smithi063.stdout:2751: snap_create 2024-08-17T17:06:36.985 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:36 smithi063 ceph-mon[109843]: osdmap e768: 8 total, 7 up, 8 in 2024-08-17T17:06:36.985 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:36 smithi063 ceph-mon[109843]: pgmap v609: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 260 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 751 KiB/s wr, 7 op/s; 60/510 objects degraded (11.765%) 2024-08-17T17:06:36.985 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:36 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:36.985 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:36 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:36.985 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:36 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:36.986 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:36 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:36.986 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:36 smithi063 ceph-mon[114756]: osdmap e768: 8 total, 7 up, 8 in 2024-08-17T17:06:36.986 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:36 smithi063 ceph-mon[114756]: pgmap v609: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 260 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 751 KiB/s wr, 7 op/s; 60/510 objects degraded (11.765%) 2024-08-17T17:06:36.986 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:36 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:36.986 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:36 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:36.986 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:36 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:36.986 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:36 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:36 smithi162 ceph-mon[104668]: osdmap e768: 8 total, 7 up, 8 in 2024-08-17T17:06:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:36 smithi162 ceph-mon[104668]: pgmap v609: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 260 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 751 KiB/s wr, 7 op/s; 60/510 objects degraded (11.765%) 2024-08-17T17:06:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:36 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:36 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:36 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:36 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:37.529 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:37 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1[132733]: 2024-08-17T17:06:37.250+0000 7f77d1e60640 -1 osd.1 758 log_to_monitors true 2024-08-17T17:06:37.608 INFO:tasks.rados.rados.0.smithi063.stdout:2751: done (0 left) 2024-08-17T17:06:37.608 INFO:tasks.rados.rados.0.smithi063.stdout:2752: read oid 24 snap -1 2024-08-17T17:06:37.609 INFO:tasks.rados.rados.0.smithi063.stdout:2752: expect (ObjNum 1021 snap 348 seq_num 1021) 2024-08-17T17:06:37.609 INFO:tasks.rados.rados.0.smithi063.stdout:2753: snap_remove snap 329 2024-08-17T17:06:38.027 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:37 smithi063 ceph-mon[114756]: osdmap e769: 8 total, 7 up, 8 in 2024-08-17T17:06:38.027 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:37 smithi063 ceph-mon[114756]: from='osd.1 [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-08-17T17:06:38.027 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:37 smithi063 ceph-mon[114756]: from='osd.1 [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:06:38.028 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:37 smithi063 ceph-mon[114756]: osdmap e770: 8 total, 7 up, 8 in 2024-08-17T17:06:38.028 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:37 smithi063 ceph-mon[114756]: from='osd.1 [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]} : dispatch 2024-08-17T17:06:38.028 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:37 smithi063 ceph-mon[109843]: osdmap e769: 8 total, 7 up, 8 in 2024-08-17T17:06:38.028 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:37 smithi063 ceph-mon[109843]: from='osd.1 [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-08-17T17:06:38.028 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:37 smithi063 ceph-mon[109843]: from='osd.1 [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:06:38.028 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:37 smithi063 ceph-mon[109843]: osdmap e770: 8 total, 7 up, 8 in 2024-08-17T17:06:38.028 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:37 smithi063 ceph-mon[109843]: from='osd.1 [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]} : dispatch 2024-08-17T17:06:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:37 smithi162 ceph-mon[104668]: osdmap e769: 8 total, 7 up, 8 in 2024-08-17T17:06:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:37 smithi162 ceph-mon[104668]: from='osd.1 [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-08-17T17:06:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:37 smithi162 ceph-mon[104668]: from='osd.1 [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-08-17T17:06:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:37 smithi162 ceph-mon[104668]: osdmap e770: 8 total, 7 up, 8 in 2024-08-17T17:06:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:37 smithi162 ceph-mon[104668]: from='osd.1 [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi063", "root=default"]} : dispatch 2024-08-17T17:06:38.279 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:06:38 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1[132733]: 2024-08-17T17:06:38.070+0000 7f77c5c02640 -1 osd.1 758 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:06:38.615 INFO:tasks.rados.rados.0.smithi063.stdout:2753: done (1 left) 2024-08-17T17:06:38.615 INFO:tasks.rados.rados.0.smithi063.stdout:2754: write oid 41 current snap is 351 2024-08-17T17:06:38.616 INFO:tasks.rados.rados.0.smithi063.stdout:2754: seq_num 1032 ranges {544824=564623,1823770=490025,3007352=337472} 2024-08-17T17:06:38.622 INFO:tasks.rados.rados.0.smithi063.stdout:2754: writing smithi06387500-41 from 544824 to 1109447 tid 1 2024-08-17T17:06:38.629 INFO:tasks.rados.rados.0.smithi063.stdout:2754: writing smithi06387500-41 from 1823770 to 2313795 tid 2 2024-08-17T17:06:38.635 INFO:tasks.rados.rados.0.smithi063.stdout:2754: writing smithi06387500-41 from 3007352 to 3344824 tid 3 2024-08-17T17:06:38.636 INFO:tasks.rados.rados.0.smithi063.stdout:2755: delete oid 8 current snap is 351 2024-08-17T17:06:38.636 INFO:tasks.rados.rados.0.smithi063.stdout:2752: expect (ObjNum 1021 snap 348 seq_num 1021) 2024-08-17T17:06:38.764 INFO:tasks.rados.rados.0.smithi063.stdout:2754: finishing write tid 1 to smithi06387500-41 2024-08-17T17:06:38.764 INFO:tasks.rados.rados.0.smithi063.stdout:2754: finishing write tid 2 to smithi06387500-41 2024-08-17T17:06:38.764 INFO:tasks.rados.rados.0.smithi063.stdout:2754: finishing write tid 3 to smithi06387500-41 2024-08-17T17:06:38.764 INFO:tasks.rados.rados.0.smithi063.stdout:2754: finishing write tid 4 to smithi06387500-41 2024-08-17T17:06:38.765 INFO:tasks.rados.rados.0.smithi063.stdout:2754: finishing write tid 5 to smithi06387500-41 2024-08-17T17:06:38.765 INFO:tasks.rados.rados.0.smithi063.stdout:2754: oid 41 updating version 0 to 305 2024-08-17T17:06:38.765 INFO:tasks.rados.rados.0.smithi063.stdout:2754: oid 41 version 305 is already newer than 303 2024-08-17T17:06:38.765 INFO:tasks.rados.rados.0.smithi063.stdout:2754: oid 41 version 305 is already newer than 304 2024-08-17T17:06:38.765 INFO:tasks.rados.rados.0.smithi063.stdout:2754: oid 41 updating version 305 to 306 2024-08-17T17:06:38.765 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 306 (ObjNum 1032 snap 351 seq_num 1032) dirty exists 2024-08-17T17:06:38.766 INFO:tasks.rados.rados.0.smithi063.stdout:2754: left oid 41 (ObjNum 1032 snap 351 seq_num 1032) 2024-08-17T17:06:39.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:39 smithi162 ceph-mon[104668]: pgmap v612: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 248 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.2 MiB/s wr, 20 op/s; 61/474 objects degraded (12.869%) 2024-08-17T17:06:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:39 smithi162 ceph-mon[104668]: from='osd.1 [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701]' entity='osd.1' 2024-08-17T17:06:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:39 smithi162 ceph-mon[104668]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:06:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:39 smithi162 ceph-mon[104668]: osd.1 [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701] boot 2024-08-17T17:06:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:39 smithi162 ceph-mon[104668]: osdmap e771: 8 total, 8 up, 8 in 2024-08-17T17:06:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:39 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:06:39.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:39 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 61/474 objects degraded (12.869%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:39.364 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:39 smithi063 ceph-mon[114756]: pgmap v612: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 248 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.2 MiB/s wr, 20 op/s; 61/474 objects degraded (12.869%) 2024-08-17T17:06:39.364 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:39 smithi063 ceph-mon[114756]: from='osd.1 [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701]' entity='osd.1' 2024-08-17T17:06:39.364 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:39 smithi063 ceph-mon[114756]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:06:39.364 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:39 smithi063 ceph-mon[114756]: osd.1 [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701] boot 2024-08-17T17:06:39.364 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:39 smithi063 ceph-mon[114756]: osdmap e771: 8 total, 8 up, 8 in 2024-08-17T17:06:39.364 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:39 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:06:39.364 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:39 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 61/474 objects degraded (12.869%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:39.365 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:39 smithi063 ceph-mon[109843]: pgmap v612: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 248 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.2 MiB/s wr, 20 op/s; 61/474 objects degraded (12.869%) 2024-08-17T17:06:39.365 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:39 smithi063 ceph-mon[109843]: from='osd.1 [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701]' entity='osd.1' 2024-08-17T17:06:39.365 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:39 smithi063 ceph-mon[109843]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:06:39.365 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:39 smithi063 ceph-mon[109843]: osd.1 [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701] boot 2024-08-17T17:06:39.365 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:39 smithi063 ceph-mon[109843]: osdmap e771: 8 total, 8 up, 8 in 2024-08-17T17:06:39.365 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:39 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-08-17T17:06:39.365 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:39 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 61/474 objects degraded (12.869%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:06:39.644 INFO:tasks.rados.rados.0.smithi063.stdout:2752: done (2 left) 2024-08-17T17:06:39.644 INFO:tasks.rados.rados.0.smithi063.stdout:2754: done (1 left) 2024-08-17T17:06:39.644 INFO:tasks.rados.rados.0.smithi063.stdout:2755: done (0 left) 2024-08-17T17:06:39.645 INFO:tasks.rados.rados.0.smithi063.stdout:2756: read oid 41 snap -1 2024-08-17T17:06:39.645 INFO:tasks.rados.rados.0.smithi063.stdout:2756: expect (ObjNum 1032 snap 351 seq_num 1032) 2024-08-17T17:06:39.645 INFO:tasks.rados.rados.0.smithi063.stdout:2757: read oid 4 snap -1 2024-08-17T17:06:39.645 INFO:tasks.rados.rados.0.smithi063.stdout:2757: expect deleted 2024-08-17T17:06:39.645 INFO:tasks.rados.rados.0.smithi063.stdout:2758: rollback oid 45 current snap is 351 2024-08-17T17:06:39.645 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 45 to 349 2024-08-17T17:06:39.645 INFO:tasks.rados.rados.0.smithi063.stdout:2759: snap_create 2024-08-17T17:06:39.646 INFO:tasks.rados.rados.0.smithi063.stdout:2758: finishing rollback tid 1 to smithi06387500-45 2024-08-17T17:06:39.646 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 616 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T17:06:39.657 INFO:tasks.rados.rados.0.smithi063.stdout:2756: expect (ObjNum 1032 snap 351 seq_num 1032) 2024-08-17T17:06:40.029 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:06:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T17:06:39.919+0000 7f4652b73640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-08-17T17:06:40.617 INFO:tasks.rados.rados.0.smithi063.stdout:2756: done (3 left) 2024-08-17T17:06:40.617 INFO:tasks.rados.rados.0.smithi063.stdout:2757: done (2 left) 2024-08-17T17:06:40.617 INFO:tasks.rados.rados.0.smithi063.stdout:2758: done (1 left) 2024-08-17T17:06:40.617 INFO:tasks.rados.rados.0.smithi063.stdout:2759: done (0 left) 2024-08-17T17:06:40.617 INFO:tasks.rados.rados.0.smithi063.stdout:2760: read oid 42 snap -1 2024-08-17T17:06:40.617 INFO:tasks.rados.rados.0.smithi063.stdout:2760: expect deleted 2024-08-17T17:06:40.617 INFO:tasks.rados.rados.0.smithi063.stdout:2761: read oid 29 snap -1 2024-08-17T17:06:40.617 INFO:tasks.rados.rados.0.smithi063.stdout:2761: expect (ObjNum 855 snap 290 seq_num 855) 2024-08-17T17:06:40.617 INFO:tasks.rados.rados.0.smithi063.stdout:2762: read oid 21 snap -1 2024-08-17T17:06:40.618 INFO:tasks.rados.rados.0.smithi063.stdout:2762: expect (ObjNum 938 snap 321 seq_num 938) 2024-08-17T17:06:40.618 INFO:tasks.rados.rados.0.smithi063.stdout:2763: read oid 45 snap -1 2024-08-17T17:06:40.618 INFO:tasks.rados.rados.0.smithi063.stdout:2763: expect deleted 2024-08-17T17:06:40.618 INFO:tasks.rados.rados.0.smithi063.stdout:2764: rollback oid 34 current snap is 352 2024-08-17T17:06:40.618 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 34 to 348 2024-08-17T17:06:40.618 INFO:tasks.rados.rados.0.smithi063.stdout:2765: snap_remove snap 349 2024-08-17T17:06:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[114756]: osdmap e772: 8 total, 8 up, 8 in 2024-08-17T17:06:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:06:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:06:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[109843]: osdmap e772: 8 total, 8 up, 8 in 2024-08-17T17:06:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:41.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:06:41.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:41.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:41.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:41.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:41.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:41.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:40 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:06:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:40 smithi162 ceph-mon[104668]: osdmap e772: 8 total, 8 up, 8 in 2024-08-17T17:06:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:40 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:40 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:40 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:40 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:06:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:40 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:40 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:06:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:40 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:40 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:40 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:41.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:40 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:06:41.624 INFO:tasks.rados.rados.0.smithi063.stdout:2765: done (5 left) 2024-08-17T17:06:41.625 INFO:tasks.rados.rados.0.smithi063.stdout:2766: read oid 28 snap -1 2024-08-17T17:06:41.625 INFO:tasks.rados.rados.0.smithi063.stdout:2766: expect deleted 2024-08-17T17:06:41.625 INFO:tasks.rados.rados.0.smithi063.stdout:2767: write oid 41 current snap is 352 2024-08-17T17:06:41.625 INFO:tasks.rados.rados.0.smithi063.stdout:2767: seq_num 1034 ranges {578438=431474,1657087=493694,2778551=599887} 2024-08-17T17:06:41.630 INFO:tasks.rados.rados.0.smithi063.stdout:2767: writing smithi06387500-41 from 578438 to 1009912 tid 1 2024-08-17T17:06:41.637 INFO:tasks.rados.rados.0.smithi063.stdout:2767: writing smithi06387500-41 from 1657087 to 2150781 tid 2 2024-08-17T17:06:41.644 INFO:tasks.rados.rados.0.smithi063.stdout:2767: writing smithi06387500-41 from 2778551 to 3378438 tid 3 2024-08-17T17:06:41.644 INFO:tasks.rados.rados.0.smithi063.stdout:2768: write oid 47 current snap is 352 2024-08-17T17:06:41.644 INFO:tasks.rados.rados.0.smithi063.stdout:2768: seq_num 1035 ranges {595245=506723,1715569=637352,2805974=562298} 2024-08-17T17:06:41.650 INFO:tasks.rados.rados.0.smithi063.stdout:2768: writing smithi06387500-47 from 595245 to 1101968 tid 1 2024-08-17T17:06:41.657 INFO:tasks.rados.rados.0.smithi063.stdout:2768: writing smithi06387500-47 from 1715569 to 2352921 tid 2 2024-08-17T17:06:41.663 INFO:tasks.rados.rados.0.smithi063.stdout:2768: writing smithi06387500-47 from 2805974 to 3368272 tid 3 2024-08-17T17:06:41.663 INFO:tasks.rados.rados.0.smithi063.stdout:2769: rollback oid 2 current snap is 352 2024-08-17T17:06:41.663 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 2 to 347 2024-08-17T17:06:41.663 INFO:tasks.rados.rados.0.smithi063.stdout:2761: expect (ObjNum 855 snap 290 seq_num 855) 2024-08-17T17:06:41.770 INFO:tasks.rados.rados.0.smithi063.stdout:2762: expect (ObjNum 938 snap 321 seq_num 938) 2024-08-17T17:06:41.930 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:41 smithi063 ceph-mon[114756]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:06:41.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:41 smithi063 ceph-mon[114756]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-08-17T17:06:41.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:41 smithi063 ceph-mon[114756]: pgmap v615: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 248 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.5 MiB/s wr, 12 op/s; 61/474 objects degraded (12.869%) 2024-08-17T17:06:41.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:41 smithi063 ceph-mon[114756]: osdmap e773: 8 total, 8 up, 8 in 2024-08-17T17:06:41.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:41 smithi063 ceph-mon[109843]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:06:41.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:41 smithi063 ceph-mon[109843]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-08-17T17:06:41.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:41 smithi063 ceph-mon[109843]: pgmap v615: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 248 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.5 MiB/s wr, 12 op/s; 61/474 objects degraded (12.869%) 2024-08-17T17:06:41.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:41 smithi063 ceph-mon[109843]: osdmap e773: 8 total, 8 up, 8 in 2024-08-17T17:06:41.949 INFO:tasks.rados.rados.0.smithi063.stdout:2764: finishing rollback tid 0 to smithi06387500-34 2024-08-17T17:06:41.949 INFO:tasks.rados.rados.0.smithi063.stdout:2764: finishing rollback tid 1 to smithi06387500-34 2024-08-17T17:06:41.949 INFO:tasks.rados.rados.0.smithi063.stdout:2764: finishing rollback tid 2 to smithi06387500-34 2024-08-17T17:06:41.949 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 1807 (ObjNum 971 snap 333 seq_num 971) dirty exists 2024-08-17T17:06:41.949 INFO:tasks.rados.rados.0.smithi063.stdout:2760: done (8 left) 2024-08-17T17:06:41.950 INFO:tasks.rados.rados.0.smithi063.stdout:2761: done (7 left) 2024-08-17T17:06:41.950 INFO:tasks.rados.rados.0.smithi063.stdout:2762: done (6 left) 2024-08-17T17:06:41.950 INFO:tasks.rados.rados.0.smithi063.stdout:2763: done (5 left) 2024-08-17T17:06:41.950 INFO:tasks.rados.rados.0.smithi063.stdout:2764: done (4 left) 2024-08-17T17:06:41.950 INFO:tasks.rados.rados.0.smithi063.stdout:2766: done (3 left) 2024-08-17T17:06:41.950 INFO:tasks.rados.rados.0.smithi063.stdout:2770: rollback oid 5 current snap is 352 2024-08-17T17:06:41.950 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 339 2024-08-17T17:06:41.950 INFO:tasks.rados.rados.0.smithi063.stdout:2767: finishing write tid 1 to smithi06387500-41 2024-08-17T17:06:41.950 INFO:tasks.rados.rados.0.smithi063.stdout:2767: finishing write tid 2 to smithi06387500-41 2024-08-17T17:06:41.950 INFO:tasks.rados.rados.0.smithi063.stdout:2767: finishing write tid 3 to smithi06387500-41 2024-08-17T17:06:41.950 INFO:tasks.rados.rados.0.smithi063.stdout:2767: finishing write tid 4 to smithi06387500-41 2024-08-17T17:06:41.950 INFO:tasks.rados.rados.0.smithi063.stdout:2767: finishing write tid 5 to smithi06387500-41 2024-08-17T17:06:41.951 INFO:tasks.rados.rados.0.smithi063.stdout:2767: oid 41 updating version 0 to 312 2024-08-17T17:06:41.951 INFO:tasks.rados.rados.0.smithi063.stdout:2767: oid 41 version 312 is already newer than 311 2024-08-17T17:06:41.951 INFO:tasks.rados.rados.0.smithi063.stdout:2767: oid 41 updating version 312 to 313 2024-08-17T17:06:41.951 INFO:tasks.rados.rados.0.smithi063.stdout:2767: oid 41 version 313 is already newer than 308 2024-08-17T17:06:41.951 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 313 (ObjNum 1034 snap 352 seq_num 1034) dirty exists 2024-08-17T17:06:41.951 INFO:tasks.rados.rados.0.smithi063.stdout:2767: left oid 41 (ObjNum 1034 snap 352 seq_num 1034) 2024-08-17T17:06:41.951 INFO:tasks.rados.rados.0.smithi063.stdout:2768: finishing write tid 1 to smithi06387500-47 2024-08-17T17:06:41.951 INFO:tasks.rados.rados.0.smithi063.stdout:2768: finishing write tid 2 to smithi06387500-47 2024-08-17T17:06:41.951 INFO:tasks.rados.rados.0.smithi063.stdout:2768: finishing write tid 3 to smithi06387500-47 2024-08-17T17:06:41.951 INFO:tasks.rados.rados.0.smithi063.stdout:2768: finishing write tid 4 to smithi06387500-47 2024-08-17T17:06:41.951 INFO:tasks.rados.rados.0.smithi063.stdout:2768: finishing write tid 5 to smithi06387500-47 2024-08-17T17:06:41.951 INFO:tasks.rados.rados.0.smithi063.stdout:2768: oid 47 updating version 0 to 543 2024-08-17T17:06:41.951 INFO:tasks.rados.rados.0.smithi063.stdout:2768: oid 47 updating version 543 to 544 2024-08-17T17:06:41.952 INFO:tasks.rados.rados.0.smithi063.stdout:2768: oid 47 version 544 is already newer than 541 2024-08-17T17:06:41.952 INFO:tasks.rados.rados.0.smithi063.stdout:2768: oid 47 version 544 is already newer than 542 2024-08-17T17:06:41.952 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 544 (ObjNum 1035 snap 352 seq_num 1035) dirty exists 2024-08-17T17:06:41.952 INFO:tasks.rados.rados.0.smithi063.stdout:2768: left oid 47 (ObjNum 1035 snap 352 seq_num 1035) 2024-08-17T17:06:41.952 INFO:tasks.rados.rados.0.smithi063.stdout:2769: finishing rollback tid 0 to smithi06387500-2 2024-08-17T17:06:41.952 INFO:tasks.rados.rados.0.smithi063.stdout:2769: finishing rollback tid 1 to smithi06387500-2 2024-08-17T17:06:41.952 INFO:tasks.rados.rados.0.smithi063.stdout:2769: finishing rollback tid 2 to smithi06387500-2 2024-08-17T17:06:41.952 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 1352 (ObjNum 926 snap 316 seq_num 926) dirty exists 2024-08-17T17:06:41.952 INFO:tasks.rados.rados.0.smithi063.stdout:2767: done (3 left) 2024-08-17T17:06:41.952 INFO:tasks.rados.rados.0.smithi063.stdout:2768: done (2 left) 2024-08-17T17:06:41.952 INFO:tasks.rados.rados.0.smithi063.stdout:2769: done (1 left) 2024-08-17T17:06:41.952 INFO:tasks.rados.rados.0.smithi063.stdout:2771: read oid 11 snap -1 2024-08-17T17:06:41.953 INFO:tasks.rados.rados.0.smithi063.stdout:2771: expect (ObjNum 1024 snap 350 seq_num 1024) 2024-08-17T17:06:41.953 INFO:tasks.rados.rados.0.smithi063.stdout:2772: write oid 2 current snap is 352 2024-08-17T17:06:41.953 INFO:tasks.rados.rados.0.smithi063.stdout:2772: seq_num 1036 ranges {612052=581972,1657698=497363,2717044=695008} 2024-08-17T17:06:41.955 INFO:tasks.rados.rados.0.smithi063.stdout:2772: writing smithi06387500-2 from 612052 to 1194024 tid 1 2024-08-17T17:06:41.961 INFO:tasks.rados.rados.0.smithi063.stdout:2772: writing smithi06387500-2 from 1657698 to 2155061 tid 2 2024-08-17T17:06:41.968 INFO:tasks.rados.rados.0.smithi063.stdout:2772: writing smithi06387500-2 from 2717044 to 3412052 tid 3 2024-08-17T17:06:41.968 INFO:tasks.rados.rados.0.smithi063.stdout:2773: read oid 20 snap -1 2024-08-17T17:06:41.968 INFO:tasks.rados.rados.0.smithi063.stdout:2773: expect (ObjNum 996 snap 342 seq_num 996) 2024-08-17T17:06:41.968 INFO:tasks.rados.rados.0.smithi063.stdout:2774: write oid 39 current snap is 352 2024-08-17T17:06:41.968 INFO:tasks.rados.rados.0.smithi063.stdout:2774: seq_num 1037 ranges {628859=657221,1716180=641021,3144467=284392} 2024-08-17T17:06:41.975 INFO:tasks.rados.rados.0.smithi063.stdout:2774: writing smithi06387500-39 from 628859 to 1286080 tid 1 2024-08-17T17:06:41.981 INFO:tasks.rados.rados.0.smithi063.stdout:2774: writing smithi06387500-39 from 1716180 to 2357201 tid 2 2024-08-17T17:06:41.986 INFO:tasks.rados.rados.0.smithi063.stdout:2774: writing smithi06387500-39 from 3144467 to 3428859 tid 3 2024-08-17T17:06:41.987 INFO:tasks.rados.rados.0.smithi063.stdout:2775: write oid 37 current snap is 352 2024-08-17T17:06:41.987 INFO:tasks.rados.rados.0.smithi063.stdout:2775: seq_num 1038 ranges {645666=732470,2174662=501032,3288243=157423} 2024-08-17T17:06:41.994 INFO:tasks.rados.rados.0.smithi063.stdout:2775: writing smithi06387500-37 from 645666 to 1378136 tid 1 2024-08-17T17:06:42.001 INFO:tasks.rados.rados.0.smithi063.stdout:2775: writing smithi06387500-37 from 2174662 to 2675694 tid 2 2024-08-17T17:06:42.006 INFO:tasks.rados.rados.0.smithi063.stdout:2775: writing smithi06387500-37 from 3288243 to 3445666 tid 3 2024-08-17T17:06:42.006 INFO:tasks.rados.rados.0.smithi063.stdout:2776: read oid 35 snap -1 2024-08-17T17:06:42.006 INFO:tasks.rados.rados.0.smithi063.stdout:2776: expect (ObjNum 1004 snap 343 seq_num 1004) 2024-08-17T17:06:42.006 INFO:tasks.rados.rados.0.smithi063.stdout:2777: rollback oid 47 current snap is 352 2024-08-17T17:06:42.006 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 47 to 350 2024-08-17T17:06:42.006 INFO:tasks.rados.rados.0.smithi063.stdout:2770: finishing rollback tid 0 to smithi06387500-5 2024-08-17T17:06:42.006 INFO:tasks.rados.rados.0.smithi063.stdout:2778: read oid 26 snap -1 2024-08-17T17:06:42.006 INFO:tasks.rados.rados.0.smithi063.stdout:2778: expect (ObjNum 1018 snap 346 seq_num 1018) 2024-08-17T17:06:42.006 INFO:tasks.rados.rados.0.smithi063.stdout:2779: snap_create 2024-08-17T17:06:42.006 INFO:tasks.rados.rados.0.smithi063.stdout:2770: finishing rollback tid 1 to smithi06387500-5 2024-08-17T17:06:42.007 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1355 (ObjNum 129 snap 0 seq_num 1806760504) dirty dne 2024-08-17T17:06:42.007 INFO:tasks.rados.rados.0.smithi063.stdout:2771: expect (ObjNum 1024 snap 350 seq_num 1024) 2024-08-17T17:06:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:41 smithi162 ceph-mon[104668]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:06:42.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:41 smithi162 ceph-mon[104668]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-08-17T17:06:42.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:41 smithi162 ceph-mon[104668]: pgmap v615: 89 pgs: 28 active+undersized, 9 active+undersized+degraded, 52 active+clean; 248 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.5 MiB/s wr, 12 op/s; 61/474 objects degraded (12.869%) 2024-08-17T17:06:42.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:41 smithi162 ceph-mon[104668]: osdmap e773: 8 total, 8 up, 8 in 2024-08-17T17:06:42.112 INFO:tasks.rados.rados.0.smithi063.stdout:2772: finishing write tid 1 to smithi06387500-2 2024-08-17T17:06:42.112 INFO:tasks.rados.rados.0.smithi063.stdout:2772: finishing write tid 2 to smithi06387500-2 2024-08-17T17:06:42.112 INFO:tasks.rados.rados.0.smithi063.stdout:2772: finishing write tid 3 to smithi06387500-2 2024-08-17T17:06:42.112 INFO:tasks.rados.rados.0.smithi063.stdout:2772: finishing write tid 4 to smithi06387500-2 2024-08-17T17:06:42.112 INFO:tasks.rados.rados.0.smithi063.stdout:2772: finishing write tid 5 to smithi06387500-2 2024-08-17T17:06:42.112 INFO:tasks.rados.rados.0.smithi063.stdout:2772: oid 2 updating version 0 to 1356 2024-08-17T17:06:42.112 INFO:tasks.rados.rados.0.smithi063.stdout:2772: oid 2 updating version 1356 to 1357 2024-08-17T17:06:42.112 INFO:tasks.rados.rados.0.smithi063.stdout:2772: oid 2 updating version 1357 to 1358 2024-08-17T17:06:42.112 INFO:tasks.rados.rados.0.smithi063.stdout:2772: oid 2 updating version 1358 to 1359 2024-08-17T17:06:42.112 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 1359 (ObjNum 1036 snap 352 seq_num 1036) dirty exists 2024-08-17T17:06:42.113 INFO:tasks.rados.rados.0.smithi063.stdout:2772: left oid 2 (ObjNum 1036 snap 352 seq_num 1036) 2024-08-17T17:06:42.113 INFO:tasks.rados.rados.0.smithi063.stdout:2773: expect (ObjNum 996 snap 342 seq_num 996) 2024-08-17T17:06:42.364 INFO:tasks.rados.rados.0.smithi063.stdout:2774: finishing write tid 1 to smithi06387500-39 2024-08-17T17:06:42.364 INFO:tasks.rados.rados.0.smithi063.stdout:2774: finishing write tid 2 to smithi06387500-39 2024-08-17T17:06:42.364 INFO:tasks.rados.rados.0.smithi063.stdout:2774: finishing write tid 3 to smithi06387500-39 2024-08-17T17:06:42.364 INFO:tasks.rados.rados.0.smithi063.stdout:2774: finishing write tid 4 to smithi06387500-39 2024-08-17T17:06:42.364 INFO:tasks.rados.rados.0.smithi063.stdout:2774: finishing write tid 5 to smithi06387500-39 2024-08-17T17:06:42.364 INFO:tasks.rados.rados.0.smithi063.stdout:2774: oid 39 updating version 0 to 644 2024-08-17T17:06:42.364 INFO:tasks.rados.rados.0.smithi063.stdout:2774: oid 39 version 644 is already newer than 643 2024-08-17T17:06:42.364 INFO:tasks.rados.rados.0.smithi063.stdout:2774: oid 39 updating version 644 to 646 2024-08-17T17:06:42.364 INFO:tasks.rados.rados.0.smithi063.stdout:2774: oid 39 version 646 is already newer than 645 2024-08-17T17:06:42.364 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 646 (ObjNum 1037 snap 352 seq_num 1037) dirty exists 2024-08-17T17:06:42.365 INFO:tasks.rados.rados.0.smithi063.stdout:2774: left oid 39 (ObjNum 1037 snap 352 seq_num 1037) 2024-08-17T17:06:42.365 INFO:tasks.rados.rados.0.smithi063.stdout:2775: finishing write tid 1 to smithi06387500-37 2024-08-17T17:06:42.365 INFO:tasks.rados.rados.0.smithi063.stdout:2775: finishing write tid 2 to smithi06387500-37 2024-08-17T17:06:42.365 INFO:tasks.rados.rados.0.smithi063.stdout:2777: finishing rollback tid 0 to smithi06387500-47 2024-08-17T17:06:42.365 INFO:tasks.rados.rados.0.smithi063.stdout:2777: finishing rollback tid 1 to smithi06387500-47 2024-08-17T17:06:42.365 INFO:tasks.rados.rados.0.smithi063.stdout:2777: finishing rollback tid 2 to smithi06387500-47 2024-08-17T17:06:42.365 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 547 (ObjNum 1014 snap 346 seq_num 1014) dirty exists 2024-08-17T17:06:42.365 INFO:tasks.rados.rados.0.smithi063.stdout:2775: finishing write tid 3 to smithi06387500-37 2024-08-17T17:06:42.365 INFO:tasks.rados.rados.0.smithi063.stdout:2775: finishing write tid 4 to smithi06387500-37 2024-08-17T17:06:42.365 INFO:tasks.rados.rados.0.smithi063.stdout:2775: finishing write tid 5 to smithi06387500-37 2024-08-17T17:06:42.365 INFO:tasks.rados.rados.0.smithi063.stdout:2775: oid 37 updating version 0 to 2689 2024-08-17T17:06:42.365 INFO:tasks.rados.rados.0.smithi063.stdout:2775: oid 37 updating version 2689 to 2691 2024-08-17T17:06:42.365 INFO:tasks.rados.rados.0.smithi063.stdout:2775: oid 37 version 2691 is already newer than 2690 2024-08-17T17:06:42.365 INFO:tasks.rados.rados.0.smithi063.stdout:2775: oid 37 version 2691 is already newer than 2688 2024-08-17T17:06:42.366 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 2691 (ObjNum 1038 snap 352 seq_num 1038) dirty exists 2024-08-17T17:06:42.366 INFO:tasks.rados.rados.0.smithi063.stdout:2775: left oid 37 (ObjNum 1038 snap 352 seq_num 1038) 2024-08-17T17:06:42.366 INFO:tasks.rados.rados.0.smithi063.stdout:2778: expect (ObjNum 1018 snap 346 seq_num 1018) 2024-08-17T17:06:42.627 INFO:tasks.rados.rados.0.smithi063.stdout:2776: expect (ObjNum 1004 snap 343 seq_num 1004) 2024-08-17T17:06:42.753 INFO:tasks.rados.rados.0.smithi063.stdout:2770: done (9 left) 2024-08-17T17:06:42.753 INFO:tasks.rados.rados.0.smithi063.stdout:2771: done (8 left) 2024-08-17T17:06:42.754 INFO:tasks.rados.rados.0.smithi063.stdout:2772: done (7 left) 2024-08-17T17:06:42.754 INFO:tasks.rados.rados.0.smithi063.stdout:2773: done (6 left) 2024-08-17T17:06:42.754 INFO:tasks.rados.rados.0.smithi063.stdout:2774: done (5 left) 2024-08-17T17:06:42.754 INFO:tasks.rados.rados.0.smithi063.stdout:2775: done (4 left) 2024-08-17T17:06:42.754 INFO:tasks.rados.rados.0.smithi063.stdout:2776: done (3 left) 2024-08-17T17:06:42.754 INFO:tasks.rados.rados.0.smithi063.stdout:2777: done (2 left) 2024-08-17T17:06:42.754 INFO:tasks.rados.rados.0.smithi063.stdout:2778: done (1 left) 2024-08-17T17:06:42.754 INFO:tasks.rados.rados.0.smithi063.stdout:2779: done (0 left) 2024-08-17T17:06:42.754 INFO:tasks.rados.rados.0.smithi063.stdout:2780: delete oid 30 current snap is 353 2024-08-17T17:06:42.759 INFO:tasks.rados.rados.0.smithi063.stdout:2780: done (0 left) 2024-08-17T17:06:42.759 INFO:tasks.rados.rados.0.smithi063.stdout:2781: rollback oid 50 current snap is 353 2024-08-17T17:06:42.759 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 50 to 320 2024-08-17T17:06:42.759 INFO:tasks.rados.rados.0.smithi063.stdout:2782: read oid 9 snap -1 2024-08-17T17:06:42.759 INFO:tasks.rados.rados.0.smithi063.stdout:2782: expect deleted 2024-08-17T17:06:42.759 INFO:tasks.rados.rados.0.smithi063.stdout:2783: write oid 23 current snap is 353 2024-08-17T17:06:42.759 INFO:tasks.rados.rados.0.smithi063.stdout:2783: seq_num 1040 ranges {679280=482968,1775273=788348,3110383=368897} 2024-08-17T17:06:42.765 INFO:tasks.rados.rados.0.smithi063.stdout:2783: writing smithi06387500-23 from 679280 to 1162248 tid 1 2024-08-17T17:06:42.772 INFO:tasks.rados.rados.0.smithi063.stdout:2783: writing smithi06387500-23 from 1775273 to 2563621 tid 2 2024-08-17T17:06:42.777 INFO:tasks.rados.rados.0.smithi063.stdout:2783: writing smithi06387500-23 from 3110383 to 3479280 tid 3 2024-08-17T17:06:42.778 INFO:tasks.rados.rados.0.smithi063.stdout:2784: write oid 13 current snap is 353 2024-08-17T17:06:42.778 INFO:tasks.rados.rados.0.smithi063.stdout:2784: seq_num 1041 ranges {696087=558217,1833755=648359,3137806=358281} 2024-08-17T17:06:42.784 INFO:tasks.rados.rados.0.smithi063.stdout:2784: writing smithi06387500-13 from 696087 to 1254304 tid 1 2024-08-17T17:06:42.793 INFO:tasks.rados.rados.0.smithi063.stdout:2784: writing smithi06387500-13 from 1833755 to 2482114 tid 2 2024-08-17T17:06:42.797 INFO:tasks.rados.rados.0.smithi063.stdout:2784: writing smithi06387500-13 from 3137806 to 3496087 tid 3 2024-08-17T17:06:42.797 INFO:tasks.rados.rados.0.smithi063.stdout:2785: rollback oid 20 current snap is 353 2024-08-17T17:06:42.797 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 20 to 350 2024-08-17T17:06:42.797 INFO:tasks.rados.rados.0.smithi063.stdout:2781: finishing rollback tid 1 to smithi06387500-50 2024-08-17T17:06:42.797 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 1598 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-08-17T17:06:42.797 INFO:tasks.rados.rados.0.smithi063.stdout:2783: finishing write tid 1 to smithi06387500-23 2024-08-17T17:06:42.797 INFO:tasks.rados.rados.0.smithi063.stdout:2783: finishing write tid 2 to smithi06387500-23 2024-08-17T17:06:42.797 INFO:tasks.rados.rados.0.smithi063.stdout:2783: finishing write tid 3 to smithi06387500-23 2024-08-17T17:06:42.797 INFO:tasks.rados.rados.0.smithi063.stdout:2783: finishing write tid 4 to smithi06387500-23 2024-08-17T17:06:42.798 INFO:tasks.rados.rados.0.smithi063.stdout:2783: finishing write tid 5 to smithi06387500-23 2024-08-17T17:06:42.798 INFO:tasks.rados.rados.0.smithi063.stdout:2783: oid 23 updating version 0 to 1364 2024-08-17T17:06:42.798 INFO:tasks.rados.rados.0.smithi063.stdout:2783: oid 23 version 1364 is already newer than 1362 2024-08-17T17:06:42.798 INFO:tasks.rados.rados.0.smithi063.stdout:2783: oid 23 version 1364 is already newer than 1363 2024-08-17T17:06:42.798 INFO:tasks.rados.rados.0.smithi063.stdout:2783: oid 23 version 1364 is already newer than 1361 2024-08-17T17:06:42.798 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 1364 (ObjNum 1040 snap 353 seq_num 1040) dirty exists 2024-08-17T17:06:42.798 INFO:tasks.rados.rados.0.smithi063.stdout:2783: left oid 23 (ObjNum 1040 snap 353 seq_num 1040) 2024-08-17T17:06:42.798 INFO:tasks.rados.rados.0.smithi063.stdout:2781: done (4 left) 2024-08-17T17:06:42.798 INFO:tasks.rados.rados.0.smithi063.stdout:2782: done (3 left) 2024-08-17T17:06:42.798 INFO:tasks.rados.rados.0.smithi063.stdout:2783: done (2 left) 2024-08-17T17:06:42.798 INFO:tasks.rados.rados.0.smithi063.stdout:2786: write oid 42 current snap is 353 2024-08-17T17:06:42.798 INFO:tasks.rados.rados.0.smithi063.stdout:2786: seq_num 1042 ranges {712894=749819,2008590=792017,3281582=231312} 2024-08-17T17:06:42.805 INFO:tasks.rados.rados.0.smithi063.stdout:2786: writing smithi06387500-42 from 712894 to 1462713 tid 1 2024-08-17T17:06:42.812 INFO:tasks.rados.rados.0.smithi063.stdout:2786: writing smithi06387500-42 from 2008590 to 2800607 tid 2 2024-08-17T17:06:42.816 INFO:tasks.rados.rados.0.smithi063.stdout:2786: writing smithi06387500-42 from 3281582 to 3512894 tid 3 2024-08-17T17:06:42.816 INFO:tasks.rados.rados.0.smithi063.stdout:2787: snap_remove snap 320 2024-08-17T17:06:42.942 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:42 smithi063 ceph-mon[109843]: osdmap e774: 8 total, 8 up, 8 in 2024-08-17T17:06:42.943 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:42 smithi063 ceph-mon[114756]: osdmap e774: 8 total, 8 up, 8 in 2024-08-17T17:06:43.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:42 smithi162 ceph-mon[104668]: osdmap e774: 8 total, 8 up, 8 in 2024-08-17T17:06:43.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:06:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:06:43] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.51.0" 2024-08-17T17:06:43.643 INFO:tasks.rados.rados.0.smithi063.stdout:2787: done (3 left) 2024-08-17T17:06:43.643 INFO:tasks.rados.rados.0.smithi063.stdout:2788: snap_remove snap 347 2024-08-17T17:06:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:43 smithi063 ceph-mon[114756]: pgmap v618: 89 pgs: 9 active+undersized, 1 active+undersized+degraded, 79 active+clean; 248 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 14 op/s; 9/477 objects degraded (1.887%); 3.1 MiB/s, 2 objects/s recovering 2024-08-17T17:06:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:43 smithi063 ceph-mon[114756]: osdmap e775: 8 total, 8 up, 8 in 2024-08-17T17:06:44.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:43 smithi063 ceph-mon[109843]: pgmap v618: 89 pgs: 9 active+undersized, 1 active+undersized+degraded, 79 active+clean; 248 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 14 op/s; 9/477 objects degraded (1.887%); 3.1 MiB/s, 2 objects/s recovering 2024-08-17T17:06:44.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:43 smithi063 ceph-mon[109843]: osdmap e775: 8 total, 8 up, 8 in 2024-08-17T17:06:44.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:43 smithi162 ceph-mon[104668]: pgmap v618: 89 pgs: 9 active+undersized, 1 active+undersized+degraded, 79 active+clean; 248 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 14 op/s; 9/477 objects degraded (1.887%); 3.1 MiB/s, 2 objects/s recovering 2024-08-17T17:06:44.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:43 smithi162 ceph-mon[104668]: osdmap e775: 8 total, 8 up, 8 in 2024-08-17T17:06:44.655 INFO:tasks.rados.rados.0.smithi063.stdout:2788: done (3 left) 2024-08-17T17:06:44.655 INFO:tasks.rados.rados.0.smithi063.stdout:2789: write oid 14 current snap is 353 2024-08-17T17:06:44.655 INFO:tasks.rados.rados.0.smithi063.stdout:2789: seq_num 1043 ranges {729701=425068,1667072=652028,2909005=485180} 2024-08-17T17:06:44.662 INFO:tasks.rados.rados.0.smithi063.stdout:2789: writing smithi06387500-14 from 729701 to 1154769 tid 1 2024-08-17T17:06:44.669 INFO:tasks.rados.rados.0.smithi063.stdout:2789: writing smithi06387500-14 from 1667072 to 2319100 tid 2 2024-08-17T17:06:44.675 INFO:tasks.rados.rados.0.smithi063.stdout:2789: writing smithi06387500-14 from 2909005 to 3394185 tid 3 2024-08-17T17:06:44.675 INFO:tasks.rados.rados.0.smithi063.stdout:2790: snap_create 2024-08-17T17:06:44.676 INFO:tasks.rados.rados.0.smithi063.stdout:2784: finishing write tid 1 to smithi06387500-13 2024-08-17T17:06:44.676 INFO:tasks.rados.rados.0.smithi063.stdout:2784: finishing write tid 2 to smithi06387500-13 2024-08-17T17:06:44.676 INFO:tasks.rados.rados.0.smithi063.stdout:2784: finishing write tid 3 to smithi06387500-13 2024-08-17T17:06:44.676 INFO:tasks.rados.rados.0.smithi063.stdout:2784: finishing write tid 4 to smithi06387500-13 2024-08-17T17:06:44.676 INFO:tasks.rados.rados.0.smithi063.stdout:2784: finishing write tid 5 to smithi06387500-13 2024-08-17T17:06:44.676 INFO:tasks.rados.rados.0.smithi063.stdout:2784: oid 13 updating version 0 to 1365 2024-08-17T17:06:44.676 INFO:tasks.rados.rados.0.smithi063.stdout:2784: oid 13 updating version 1365 to 1366 2024-08-17T17:06:44.676 INFO:tasks.rados.rados.0.smithi063.stdout:2784: oid 13 updating version 1366 to 1367 2024-08-17T17:06:44.676 INFO:tasks.rados.rados.0.smithi063.stdout:2784: oid 13 updating version 1367 to 1368 2024-08-17T17:06:44.676 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1368 (ObjNum 1041 snap 353 seq_num 1041) dirty exists 2024-08-17T17:06:44.676 INFO:tasks.rados.rados.0.smithi063.stdout:2784: left oid 13 (ObjNum 1041 snap 353 seq_num 1041) 2024-08-17T17:06:44.677 INFO:tasks.rados.rados.0.smithi063.stdout:2785: finishing rollback tid 0 to smithi06387500-20 2024-08-17T17:06:44.677 INFO:tasks.rados.rados.0.smithi063.stdout:2785: finishing rollback tid 1 to smithi06387500-20 2024-08-17T17:06:44.677 INFO:tasks.rados.rados.0.smithi063.stdout:2785: finishing rollback tid 2 to smithi06387500-20 2024-08-17T17:06:44.677 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 594 (ObjNum 996 snap 342 seq_num 996) dirty exists 2024-08-17T17:06:44.677 INFO:tasks.rados.rados.0.smithi063.stdout:2786: finishing write tid 1 to smithi06387500-42 2024-08-17T17:06:44.677 INFO:tasks.rados.rados.0.smithi063.stdout:2786: finishing write tid 2 to smithi06387500-42 2024-08-17T17:06:44.677 INFO:tasks.rados.rados.0.smithi063.stdout:2786: finishing write tid 3 to smithi06387500-42 2024-08-17T17:06:44.677 INFO:tasks.rados.rados.0.smithi063.stdout:2786: finishing write tid 4 to smithi06387500-42 2024-08-17T17:06:44.677 INFO:tasks.rados.rados.0.smithi063.stdout:2786: finishing write tid 5 to smithi06387500-42 2024-08-17T17:06:44.677 INFO:tasks.rados.rados.0.smithi063.stdout:2786: oid 42 updating version 0 to 1815 2024-08-17T17:06:44.677 INFO:tasks.rados.rados.0.smithi063.stdout:2786: oid 42 updating version 1815 to 1818 2024-08-17T17:06:44.677 INFO:tasks.rados.rados.0.smithi063.stdout:2786: oid 42 version 1818 is already newer than 1816 2024-08-17T17:06:44.678 INFO:tasks.rados.rados.0.smithi063.stdout:2786: oid 42 version 1818 is already newer than 1817 2024-08-17T17:06:44.678 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 1818 (ObjNum 1042 snap 353 seq_num 1042) dirty exists 2024-08-17T17:06:44.678 INFO:tasks.rados.rados.0.smithi063.stdout:2786: left oid 42 (ObjNum 1042 snap 353 seq_num 1042) 2024-08-17T17:06:44.678 INFO:tasks.rados.rados.0.smithi063.stdout:2789: finishing write tid 1 to smithi06387500-14 2024-08-17T17:06:44.678 INFO:tasks.rados.rados.0.smithi063.stdout:2789: finishing write tid 2 to smithi06387500-14 2024-08-17T17:06:44.679 INFO:tasks.rados.rados.0.smithi063.stdout:2789: finishing write tid 3 to smithi06387500-14 2024-08-17T17:06:44.679 INFO:tasks.rados.rados.0.smithi063.stdout:2789: finishing write tid 4 to smithi06387500-14 2024-08-17T17:06:44.679 INFO:tasks.rados.rados.0.smithi063.stdout:2789: finishing write tid 5 to smithi06387500-14 2024-08-17T17:06:44.679 INFO:tasks.rados.rados.0.smithi063.stdout:2789: oid 14 updating version 0 to 2701 2024-08-17T17:06:44.679 INFO:tasks.rados.rados.0.smithi063.stdout:2789: oid 14 updating version 2701 to 2703 2024-08-17T17:06:44.679 INFO:tasks.rados.rados.0.smithi063.stdout:2789: oid 14 version 2703 is already newer than 2702 2024-08-17T17:06:44.679 INFO:tasks.rados.rados.0.smithi063.stdout:2789: oid 14 updating version 2703 to 2704 2024-08-17T17:06:44.679 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 2704 (ObjNum 1043 snap 353 seq_num 1043) dirty exists 2024-08-17T17:06:44.679 INFO:tasks.rados.rados.0.smithi063.stdout:2789: left oid 14 (ObjNum 1043 snap 353 seq_num 1043) 2024-08-17T17:06:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:44 smithi063 ceph-mon[114756]: osdmap e776: 8 total, 8 up, 8 in 2024-08-17T17:06:45.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:44 smithi063 ceph-mon[109843]: osdmap e776: 8 total, 8 up, 8 in 2024-08-17T17:06:45.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:44 smithi162 ceph-mon[104668]: osdmap e776: 8 total, 8 up, 8 in 2024-08-17T17:06:45.659 INFO:tasks.rados.rados.0.smithi063.stdout:2784: done (4 left) 2024-08-17T17:06:45.659 INFO:tasks.rados.rados.0.smithi063.stdout:2785: done (3 left) 2024-08-17T17:06:45.659 INFO:tasks.rados.rados.0.smithi063.stdout:2786: done (2 left) 2024-08-17T17:06:45.659 INFO:tasks.rados.rados.0.smithi063.stdout:2789: done (1 left) 2024-08-17T17:06:45.659 INFO:tasks.rados.rados.0.smithi063.stdout:2790: done (0 left) 2024-08-17T17:06:45.659 INFO:tasks.rados.rados.0.smithi063.stdout:2791: delete oid 8 current snap is 354 2024-08-17T17:06:45.664 INFO:tasks.rados.rados.0.smithi063.stdout:2791: done (0 left) 2024-08-17T17:06:45.664 INFO:tasks.rados.rados.0.smithi063.stdout:2792: read oid 29 snap -1 2024-08-17T17:06:45.664 INFO:tasks.rados.rados.0.smithi063.stdout:2792: expect (ObjNum 855 snap 290 seq_num 855) 2024-08-17T17:06:45.664 INFO:tasks.rados.rados.0.smithi063.stdout:2793: read oid 43 snap -1 2024-08-17T17:06:45.664 INFO:tasks.rados.rados.0.smithi063.stdout:2793: expect deleted 2024-08-17T17:06:45.664 INFO:tasks.rados.rados.0.smithi063.stdout:2794: rollback oid 3 current snap is 354 2024-08-17T17:06:45.664 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 3 to 307 2024-08-17T17:06:45.665 INFO:tasks.rados.rados.0.smithi063.stdout:2795: write oid 44 current snap is 354 2024-08-17T17:06:45.665 INFO:tasks.rados.rados.0.smithi063.stdout:2795: seq_num 1045 ranges {763315=575566,2067683=655697,3247498=315817} 2024-08-17T17:06:45.671 INFO:tasks.rados.rados.0.smithi063.stdout:2795: writing smithi06387500-44 from 763315 to 1338881 tid 1 2024-08-17T17:06:45.679 INFO:tasks.rados.rados.0.smithi063.stdout:2795: writing smithi06387500-44 from 2067683 to 2723380 tid 2 2024-08-17T17:06:45.684 INFO:tasks.rados.rados.0.smithi063.stdout:2795: writing smithi06387500-44 from 3247498 to 3563315 tid 3 2024-08-17T17:06:45.684 INFO:tasks.rados.rados.0.smithi063.stdout:2796: delete oid 37 current snap is 354 2024-08-17T17:06:45.684 INFO:tasks.rados.rados.0.smithi063.stdout:2792: expect (ObjNum 855 snap 290 seq_num 855) 2024-08-17T17:06:45.791 INFO:tasks.rados.rados.0.smithi063.stdout:2794: finishing rollback tid 0 to smithi06387500-3 2024-08-17T17:06:45.791 INFO:tasks.rados.rados.0.smithi063.stdout:2794: finishing rollback tid 1 to smithi06387500-3 2024-08-17T17:06:45.791 INFO:tasks.rados.rados.0.smithi063.stdout:2794: finishing rollback tid 2 to smithi06387500-3 2024-08-17T17:06:45.791 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 933 (ObjNum 791 snap 272 seq_num 791) dirty exists 2024-08-17T17:06:45.791 INFO:tasks.rados.rados.0.smithi063.stdout:2795: finishing write tid 1 to smithi06387500-44 2024-08-17T17:06:45.791 INFO:tasks.rados.rados.0.smithi063.stdout:2795: finishing write tid 2 to smithi06387500-44 2024-08-17T17:06:45.791 INFO:tasks.rados.rados.0.smithi063.stdout:2792: done (4 left) 2024-08-17T17:06:45.791 INFO:tasks.rados.rados.0.smithi063.stdout:2793: done (3 left) 2024-08-17T17:06:45.791 INFO:tasks.rados.rados.0.smithi063.stdout:2794: done (2 left) 2024-08-17T17:06:45.791 INFO:tasks.rados.rados.0.smithi063.stdout:2796: done (1 left) 2024-08-17T17:06:45.792 INFO:tasks.rados.rados.0.smithi063.stdout:2797: write oid 32 current snap is 354 2024-08-17T17:06:45.792 INFO:tasks.rados.rados.0.smithi063.stdout:2795: finishing write tid 3 to smithi06387500-44 2024-08-17T17:06:45.792 INFO:tasks.rados.rados.0.smithi063.stdout:2797: seq_num 1047 ranges {796929=726064,2184647=659366,3302344=294585} 2024-08-17T17:06:45.799 INFO:tasks.rados.rados.0.smithi063.stdout:2797: writing smithi06387500-32 from 796929 to 1522993 tid 1 2024-08-17T17:06:45.806 INFO:tasks.rados.rados.0.smithi063.stdout:2797: writing smithi06387500-32 from 2184647 to 2844013 tid 2 2024-08-17T17:06:45.811 INFO:tasks.rados.rados.0.smithi063.stdout:2797: writing smithi06387500-32 from 3302344 to 3596929 tid 3 2024-08-17T17:06:45.811 INFO:tasks.rados.rados.0.smithi063.stdout:2798: write oid 9 current snap is 354 2024-08-17T17:06:45.811 INFO:tasks.rados.rados.0.smithi063.stdout:2795: finishing write tid 4 to smithi06387500-44 2024-08-17T17:06:45.811 INFO:tasks.rados.rados.0.smithi063.stdout:2795: finishing write tid 5 to smithi06387500-44 2024-08-17T17:06:45.811 INFO:tasks.rados.rados.0.smithi063.stdout:2795: oid 44 updating version 0 to 371 2024-08-17T17:06:45.811 INFO:tasks.rados.rados.0.smithi063.stdout:2795: oid 44 updating version 371 to 373 2024-08-17T17:06:45.811 INFO:tasks.rados.rados.0.smithi063.stdout:2795: oid 44 version 373 is already newer than 370 2024-08-17T17:06:45.811 INFO:tasks.rados.rados.0.smithi063.stdout:2795: oid 44 version 373 is already newer than 372 2024-08-17T17:06:45.811 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 373 (ObjNum 1045 snap 354 seq_num 1045) dirty exists 2024-08-17T17:06:45.811 INFO:tasks.rados.rados.0.smithi063.stdout:2795: left oid 44 (ObjNum 1045 snap 354 seq_num 1045) 2024-08-17T17:06:45.812 INFO:tasks.rados.rados.0.smithi063.stdout:2797: finishing write tid 1 to smithi06387500-32 2024-08-17T17:06:45.812 INFO:tasks.rados.rados.0.smithi063.stdout:2798: seq_num 1048 ranges {413736=401313,1326776=403024,2413414=457893,3438716=175020} 2024-08-17T17:06:45.815 INFO:tasks.rados.rados.0.smithi063.stdout:2798: writing smithi06387500-9 from 413736 to 815049 tid 1 2024-08-17T17:06:45.820 INFO:tasks.rados.rados.0.smithi063.stdout:2798: writing smithi06387500-9 from 1326776 to 1729800 tid 2 2024-08-17T17:06:45.827 INFO:tasks.rados.rados.0.smithi063.stdout:2798: writing smithi06387500-9 from 2413414 to 2871307 tid 3 2024-08-17T17:06:45.832 INFO:tasks.rados.rados.0.smithi063.stdout:2798: writing smithi06387500-9 from 3438716 to 3613736 tid 4 2024-08-17T17:06:45.832 INFO:tasks.rados.rados.0.smithi063.stdout:2795: done (2 left) 2024-08-17T17:06:45.832 INFO:tasks.rados.rados.0.smithi063.stdout:2799: delete oid 48 current snap is 354 2024-08-17T17:06:45.832 INFO:tasks.rados.rados.0.smithi063.stdout:2797: finishing write tid 2 to smithi06387500-32 2024-08-17T17:06:45.832 INFO:tasks.rados.rados.0.smithi063.stdout:2797: finishing write tid 3 to smithi06387500-32 2024-08-17T17:06:45.832 INFO:tasks.rados.rados.0.smithi063.stdout:2797: finishing write tid 4 to smithi06387500-32 2024-08-17T17:06:45.832 INFO:tasks.rados.rados.0.smithi063.stdout:2797: finishing write tid 5 to smithi06387500-32 2024-08-17T17:06:45.832 INFO:tasks.rados.rados.0.smithi063.stdout:2797: oid 32 updating version 0 to 558 2024-08-17T17:06:45.832 INFO:tasks.rados.rados.0.smithi063.stdout:2797: oid 32 version 558 is already newer than 557 2024-08-17T17:06:45.832 INFO:tasks.rados.rados.0.smithi063.stdout:2797: oid 32 updating version 558 to 560 2024-08-17T17:06:45.833 INFO:tasks.rados.rados.0.smithi063.stdout:2797: oid 32 version 560 is already newer than 559 2024-08-17T17:06:45.833 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 560 (ObjNum 1047 snap 354 seq_num 1047) dirty exists 2024-08-17T17:06:45.833 INFO:tasks.rados.rados.0.smithi063.stdout:2797: left oid 32 (ObjNum 1047 snap 354 seq_num 1047) 2024-08-17T17:06:45.833 INFO:tasks.rados.rados.0.smithi063.stdout:2798: finishing write tid 1 to smithi06387500-9 2024-08-17T17:06:45.833 INFO:tasks.rados.rados.0.smithi063.stdout:2798: finishing write tid 2 to smithi06387500-9 2024-08-17T17:06:45.833 INFO:tasks.rados.rados.0.smithi063.stdout:2798: finishing write tid 3 to smithi06387500-9 2024-08-17T17:06:45.835 INFO:tasks.rados.rados.0.smithi063.stdout:2798: finishing write tid 4 to smithi06387500-9 2024-08-17T17:06:45.835 INFO:tasks.rados.rados.0.smithi063.stdout:2798: finishing write tid 5 to smithi06387500-9 2024-08-17T17:06:45.835 INFO:tasks.rados.rados.0.smithi063.stdout:2797: done (2 left) 2024-08-17T17:06:45.835 INFO:tasks.rados.rados.0.smithi063.stdout:2799: done (1 left) 2024-08-17T17:06:45.835 INFO:tasks.rados.rados.0.smithi063.stdout:2800: read oid 47 snap -1 2024-08-17T17:06:45.836 INFO:tasks.rados.rados.0.smithi063.stdout:2800: expect (ObjNum 1014 snap 346 seq_num 1014) 2024-08-17T17:06:45.836 INFO:tasks.rados.rados.0.smithi063.stdout:2801: delete oid 34 current snap is 354 2024-08-17T17:06:45.836 INFO:tasks.rados.rados.0.smithi063.stdout:2798: finishing write tid 6 to smithi06387500-9 2024-08-17T17:06:45.836 INFO:tasks.rados.rados.0.smithi063.stdout:2798: oid 9 updating version 0 to 1315 2024-08-17T17:06:45.836 INFO:tasks.rados.rados.0.smithi063.stdout:2798: oid 9 updating version 1315 to 1316 2024-08-17T17:06:45.837 INFO:tasks.rados.rados.0.smithi063.stdout:2798: oid 9 version 1316 is already newer than 1314 2024-08-17T17:06:45.837 INFO:tasks.rados.rados.0.smithi063.stdout:2798: oid 9 version 1316 is already newer than 1313 2024-08-17T17:06:45.837 INFO:tasks.rados.rados.0.smithi063.stdout:2798: oid 9 version 1316 is already newer than 1312 2024-08-17T17:06:45.837 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 1316 (ObjNum 1048 snap 354 seq_num 1048) dirty exists 2024-08-17T17:06:45.837 INFO:tasks.rados.rados.0.smithi063.stdout:2798: left oid 9 (ObjNum 1048 snap 354 seq_num 1048) 2024-08-17T17:06:45.837 INFO:tasks.rados.rados.0.smithi063.stdout:2798: done (2 left) 2024-08-17T17:06:45.837 INFO:tasks.rados.rados.0.smithi063.stdout:2801: done (1 left) 2024-08-17T17:06:45.837 INFO:tasks.rados.rados.0.smithi063.stdout:2802: snap_create 2024-08-17T17:06:45.840 INFO:tasks.rados.rados.0.smithi063.stdout:2800: expect (ObjNum 1014 snap 346 seq_num 1014) 2024-08-17T17:06:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:45 smithi063 ceph-mon[114756]: pgmap v621: 89 pgs: 89 active+clean; 245 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 14 op/s; 3.1 MiB/s, 2 objects/s recovering 2024-08-17T17:06:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:45 smithi063 ceph-mon[114756]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/477 objects degraded (1.887%), 1 pg degraded) 2024-08-17T17:06:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:45 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:06:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:45 smithi063 ceph-mon[114756]: osdmap e777: 8 total, 8 up, 8 in 2024-08-17T17:06:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:45 smithi063 ceph-mon[109843]: pgmap v621: 89 pgs: 89 active+clean; 245 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 14 op/s; 3.1 MiB/s, 2 objects/s recovering 2024-08-17T17:06:46.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:45 smithi063 ceph-mon[109843]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/477 objects degraded (1.887%), 1 pg degraded) 2024-08-17T17:06:46.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:45 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:06:46.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:45 smithi063 ceph-mon[109843]: osdmap e777: 8 total, 8 up, 8 in 2024-08-17T17:06:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:45 smithi162 ceph-mon[104668]: pgmap v621: 89 pgs: 89 active+clean; 245 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 14 op/s; 3.1 MiB/s, 2 objects/s recovering 2024-08-17T17:06:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:45 smithi162 ceph-mon[104668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/477 objects degraded (1.887%), 1 pg degraded) 2024-08-17T17:06:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:45 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:06:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:45 smithi162 ceph-mon[104668]: osdmap e777: 8 total, 8 up, 8 in 2024-08-17T17:06:46.663 INFO:tasks.rados.rados.0.smithi063.stdout:2800: done (1 left) 2024-08-17T17:06:46.663 INFO:tasks.rados.rados.0.smithi063.stdout:2802: done (0 left) 2024-08-17T17:06:46.663 INFO:tasks.rados.rados.0.smithi063.stdout:2803: snap_remove snap 350 2024-08-17T17:06:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:46 smithi063 ceph-mon[114756]: osdmap e778: 8 total, 8 up, 8 in 2024-08-17T17:06:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:46 smithi063 ceph-mon[114756]: pgmap v624: 89 pgs: 89 active+clean; 244 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 433 KiB/s wr, 1 op/s 2024-08-17T17:06:47.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:47.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:46 smithi063 ceph-mon[109843]: osdmap e778: 8 total, 8 up, 8 in 2024-08-17T17:06:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:46 smithi063 ceph-mon[109843]: pgmap v624: 89 pgs: 89 active+clean; 244 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 433 KiB/s wr, 1 op/s 2024-08-17T17:06:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:46 smithi162 ceph-mon[104668]: osdmap e778: 8 total, 8 up, 8 in 2024-08-17T17:06:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:46 smithi162 ceph-mon[104668]: pgmap v624: 89 pgs: 89 active+clean; 244 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 433 KiB/s wr, 1 op/s 2024-08-17T17:06:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:06:47.614 INFO:tasks.rados.rados.0.smithi063.stdout:2803: done (0 left) 2024-08-17T17:06:47.614 INFO:tasks.rados.rados.0.smithi063.stdout:2804: write oid 4 current snap is 355 2024-08-17T17:06:47.615 INFO:tasks.rados.rados.0.smithi063.stdout:2804: seq_num 1051 ranges {464157=743413,1618575=666704,3012036=408062} 2024-08-17T17:06:47.622 INFO:tasks.rados.rados.0.smithi063.stdout:2804: writing smithi06387500-4 from 464157 to 1207570 tid 1 2024-08-17T17:06:47.629 INFO:tasks.rados.rados.0.smithi063.stdout:2804: writing smithi06387500-4 from 1618575 to 2285279 tid 2 2024-08-17T17:06:47.637 INFO:tasks.rados.rados.0.smithi063.stdout:2804: writing smithi06387500-4 from 3012036 to 3420098 tid 3 2024-08-17T17:06:47.637 INFO:tasks.rados.rados.0.smithi063.stdout:2805: write oid 11 current snap is 355 2024-08-17T17:06:47.637 INFO:tasks.rados.rados.0.smithi063.stdout:2804: finishing write tid 1 to smithi06387500-4 2024-08-17T17:06:47.637 INFO:tasks.rados.rados.0.smithi063.stdout:2805: seq_num 1052 ranges {480964=418662,1560704=410362,2523106=619334} 2024-08-17T17:06:47.642 INFO:tasks.rados.rados.0.smithi063.stdout:2805: writing smithi06387500-11 from 480964 to 899626 tid 1 2024-08-17T17:06:47.648 INFO:tasks.rados.rados.0.smithi063.stdout:2805: writing smithi06387500-11 from 1560704 to 1971066 tid 2 2024-08-17T17:06:47.654 INFO:tasks.rados.rados.0.smithi063.stdout:2805: writing smithi06387500-11 from 2523106 to 3142440 tid 3 2024-08-17T17:06:47.655 INFO:tasks.rados.rados.0.smithi063.stdout:2804: finishing write tid 2 to smithi06387500-4 2024-08-17T17:06:47.655 INFO:tasks.rados.rados.0.smithi063.stdout:2804: finishing write tid 3 to smithi06387500-4 2024-08-17T17:06:47.655 INFO:tasks.rados.rados.0.smithi063.stdout:2804: finishing write tid 4 to smithi06387500-4 2024-08-17T17:06:47.655 INFO:tasks.rados.rados.0.smithi063.stdout:2804: finishing write tid 5 to smithi06387500-4 2024-08-17T17:06:47.655 INFO:tasks.rados.rados.0.smithi063.stdout:2804: oid 4 updating version 0 to 2716 2024-08-17T17:06:47.655 INFO:tasks.rados.rados.0.smithi063.stdout:2804: oid 4 updating version 2716 to 2717 2024-08-17T17:06:47.655 INFO:tasks.rados.rados.0.smithi063.stdout:2804: oid 4 updating version 2717 to 2718 2024-08-17T17:06:47.655 INFO:tasks.rados.rados.0.smithi063.stdout:2804: oid 4 version 2718 is already newer than 2713 2024-08-17T17:06:47.655 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 2718 (ObjNum 1051 snap 355 seq_num 1051) dirty exists 2024-08-17T17:06:47.655 INFO:tasks.rados.rados.0.smithi063.stdout:2804: left oid 4 (ObjNum 1051 snap 355 seq_num 1051) 2024-08-17T17:06:47.655 INFO:tasks.rados.rados.0.smithi063.stdout:2805: finishing write tid 1 to smithi06387500-11 2024-08-17T17:06:47.655 INFO:tasks.rados.rados.0.smithi063.stdout:2805: finishing write tid 2 to smithi06387500-11 2024-08-17T17:06:47.655 INFO:tasks.rados.rados.0.smithi063.stdout:2804: done (1 left) 2024-08-17T17:06:47.655 INFO:tasks.rados.rados.0.smithi063.stdout:2806: write oid 50 current snap is 355 2024-08-17T17:06:47.656 INFO:tasks.rados.rados.0.smithi063.stdout:2806: seq_num 1053 ranges {497771=493911,1619186=554020,2950529=430606} 2024-08-17T17:06:47.660 INFO:tasks.rados.rados.0.smithi063.stdout:2806: writing smithi06387500-50 from 497771 to 991682 tid 1 2024-08-17T17:06:47.666 INFO:tasks.rados.rados.0.smithi063.stdout:2806: writing smithi06387500-50 from 1619186 to 2173206 tid 2 2024-08-17T17:06:47.673 INFO:tasks.rados.rados.0.smithi063.stdout:2806: writing smithi06387500-50 from 2950529 to 3381135 tid 3 2024-08-17T17:06:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:2807: delete oid 12 current snap is 355 2024-08-17T17:06:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:2805: finishing write tid 3 to smithi06387500-11 2024-08-17T17:06:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:2805: finishing write tid 4 to smithi06387500-11 2024-08-17T17:06:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:2805: finishing write tid 5 to smithi06387500-11 2024-08-17T17:06:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:2805: oid 11 updating version 0 to 442 2024-08-17T17:06:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:2805: oid 11 version 442 is already newer than 441 2024-08-17T17:06:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:2805: oid 11 version 442 is already newer than 440 2024-08-17T17:06:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:2805: oid 11 updating version 442 to 443 2024-08-17T17:06:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 443 (ObjNum 1052 snap 355 seq_num 1052) dirty exists 2024-08-17T17:06:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:2805: left oid 11 (ObjNum 1052 snap 355 seq_num 1052) 2024-08-17T17:06:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:2806: finishing write tid 1 to smithi06387500-50 2024-08-17T17:06:47.674 INFO:tasks.rados.rados.0.smithi063.stdout:2806: finishing write tid 2 to smithi06387500-50 2024-08-17T17:06:47.676 INFO:tasks.rados.rados.0.smithi063.stdout:2805: done (2 left) 2024-08-17T17:06:47.676 INFO:tasks.rados.rados.0.smithi063.stdout:2807: done (1 left) 2024-08-17T17:06:47.676 INFO:tasks.rados.rados.0.smithi063.stdout:2808: read oid 5 snap -1 2024-08-17T17:06:47.676 INFO:tasks.rados.rados.0.smithi063.stdout:2808: expect deleted 2024-08-17T17:06:47.677 INFO:tasks.rados.rados.0.smithi063.stdout:2809: write oid 24 current snap is 355 2024-08-17T17:06:47.677 INFO:tasks.rados.rados.0.smithi063.stdout:2809: seq_num 1055 ranges {531385=644409,1736150=557689,3005375=569503} 2024-08-17T17:06:47.682 INFO:tasks.rados.rados.0.smithi063.stdout:2809: writing smithi06387500-24 from 531385 to 1175794 tid 1 2024-08-17T17:06:47.688 INFO:tasks.rados.rados.0.smithi063.stdout:2809: writing smithi06387500-24 from 1736150 to 2293839 tid 2 2024-08-17T17:06:47.696 INFO:tasks.rados.rados.0.smithi063.stdout:2809: writing smithi06387500-24 from 3005375 to 3574878 tid 3 2024-08-17T17:06:47.696 INFO:tasks.rados.rados.0.smithi063.stdout:2810: write oid 33 current snap is 355 2024-08-17T17:06:47.696 INFO:tasks.rados.rados.0.smithi063.stdout:2810: seq_num 1056 ranges {548192=719658,1678279=417700,2516445=780775} 2024-08-17T17:06:47.702 INFO:tasks.rados.rados.0.smithi063.stdout:2810: writing smithi06387500-33 from 548192 to 1267850 tid 1 2024-08-17T17:06:47.707 INFO:tasks.rados.rados.0.smithi063.stdout:2810: writing smithi06387500-33 from 1678279 to 2095979 tid 2 2024-08-17T17:06:47.715 INFO:tasks.rados.rados.0.smithi063.stdout:2810: writing smithi06387500-33 from 2516445 to 3297220 tid 3 2024-08-17T17:06:47.715 INFO:tasks.rados.rados.0.smithi063.stdout:2811: read oid 8 snap -1 2024-08-17T17:06:47.715 INFO:tasks.rados.rados.0.smithi063.stdout:2811: expect deleted 2024-08-17T17:06:47.715 INFO:tasks.rados.rados.0.smithi063.stdout:2812: rollback oid 32 current snap is 355 2024-08-17T17:06:47.715 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 32 to 339 2024-08-17T17:06:47.715 INFO:tasks.rados.rados.0.smithi063.stdout:2806: finishing write tid 3 to smithi06387500-50 2024-08-17T17:06:47.715 INFO:tasks.rados.rados.0.smithi063.stdout:2806: finishing write tid 4 to smithi06387500-50 2024-08-17T17:06:47.715 INFO:tasks.rados.rados.0.smithi063.stdout:2806: finishing write tid 5 to smithi06387500-50 2024-08-17T17:06:47.715 INFO:tasks.rados.rados.0.smithi063.stdout:2806: oid 50 updating version 0 to 1835 2024-08-17T17:06:47.715 INFO:tasks.rados.rados.0.smithi063.stdout:2806: oid 50 updating version 1835 to 1837 2024-08-17T17:06:47.715 INFO:tasks.rados.rados.0.smithi063.stdout:2806: oid 50 updating version 1837 to 1838 2024-08-17T17:06:47.716 INFO:tasks.rados.rados.0.smithi063.stdout:2806: oid 50 version 1838 is already newer than 1836 2024-08-17T17:06:47.716 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 1838 (ObjNum 1053 snap 355 seq_num 1053) dirty exists 2024-08-17T17:06:47.716 INFO:tasks.rados.rados.0.smithi063.stdout:2806: left oid 50 (ObjNum 1053 snap 355 seq_num 1053) 2024-08-17T17:06:47.716 INFO:tasks.rados.rados.0.smithi063.stdout:2809: finishing write tid 1 to smithi06387500-24 2024-08-17T17:06:47.716 INFO:tasks.rados.rados.0.smithi063.stdout:2806: done (5 left) 2024-08-17T17:06:47.716 INFO:tasks.rados.rados.0.smithi063.stdout:2808: done (4 left) 2024-08-17T17:06:47.716 INFO:tasks.rados.rados.0.smithi063.stdout:2813: snap_create 2024-08-17T17:06:47.716 INFO:tasks.rados.rados.0.smithi063.stdout:2809: finishing write tid 2 to smithi06387500-24 2024-08-17T17:06:47.716 INFO:tasks.rados.rados.0.smithi063.stdout:2809: finishing write tid 3 to smithi06387500-24 2024-08-17T17:06:47.716 INFO:tasks.rados.rados.0.smithi063.stdout:2809: finishing write tid 4 to smithi06387500-24 2024-08-17T17:06:47.716 INFO:tasks.rados.rados.0.smithi063.stdout:2809: finishing write tid 5 to smithi06387500-24 2024-08-17T17:06:47.716 INFO:tasks.rados.rados.0.smithi063.stdout:2809: oid 24 updating version 0 to 1008 2024-08-17T17:06:47.716 INFO:tasks.rados.rados.0.smithi063.stdout:2809: oid 24 updating version 1008 to 1009 2024-08-17T17:06:47.717 INFO:tasks.rados.rados.0.smithi063.stdout:2809: oid 24 version 1009 is already newer than 1006 2024-08-17T17:06:47.717 INFO:tasks.rados.rados.0.smithi063.stdout:2809: oid 24 version 1009 is already newer than 1007 2024-08-17T17:06:47.717 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 1009 (ObjNum 1055 snap 355 seq_num 1055) dirty exists 2024-08-17T17:06:47.717 INFO:tasks.rados.rados.0.smithi063.stdout:2809: left oid 24 (ObjNum 1055 snap 355 seq_num 1055) 2024-08-17T17:06:47.717 INFO:tasks.rados.rados.0.smithi063.stdout:2810: finishing write tid 1 to smithi06387500-33 2024-08-17T17:06:47.717 INFO:tasks.rados.rados.0.smithi063.stdout:2810: finishing write tid 2 to smithi06387500-33 2024-08-17T17:06:47.718 INFO:tasks.rados.rados.0.smithi063.stdout:2812: finishing rollback tid 0 to smithi06387500-32 2024-08-17T17:06:47.718 INFO:tasks.rados.rados.0.smithi063.stdout:2812: finishing rollback tid 1 to smithi06387500-32 2024-08-17T17:06:47.718 INFO:tasks.rados.rados.0.smithi063.stdout:2810: finishing write tid 3 to smithi06387500-33 2024-08-17T17:06:47.718 INFO:tasks.rados.rados.0.smithi063.stdout:2810: finishing write tid 4 to smithi06387500-33 2024-08-17T17:06:47.718 INFO:tasks.rados.rados.0.smithi063.stdout:2812: finishing rollback tid 2 to smithi06387500-32 2024-08-17T17:06:47.718 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 568 (ObjNum 882 snap 308 seq_num 882) dirty exists 2024-08-17T17:06:47.719 INFO:tasks.rados.rados.0.smithi063.stdout:2810: finishing write tid 5 to smithi06387500-33 2024-08-17T17:06:47.719 INFO:tasks.rados.rados.0.smithi063.stdout:2810: oid 33 updating version 0 to 953 2024-08-17T17:06:47.719 INFO:tasks.rados.rados.0.smithi063.stdout:2810: oid 33 updating version 953 to 954 2024-08-17T17:06:47.719 INFO:tasks.rados.rados.0.smithi063.stdout:2810: oid 33 updating version 954 to 956 2024-08-17T17:06:47.719 INFO:tasks.rados.rados.0.smithi063.stdout:2810: oid 33 version 956 is already newer than 955 2024-08-17T17:06:47.719 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 956 (ObjNum 1056 snap 355 seq_num 1056) dirty exists 2024-08-17T17:06:47.719 INFO:tasks.rados.rados.0.smithi063.stdout:2810: left oid 33 (ObjNum 1056 snap 355 seq_num 1056) 2024-08-17T17:06:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:47 smithi063 ceph-mon[109843]: osdmap e779: 8 total, 8 up, 8 in 2024-08-17T17:06:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:47 smithi063 ceph-mon[109843]: osdmap e780: 8 total, 8 up, 8 in 2024-08-17T17:06:48.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:47 smithi063 ceph-mon[114756]: osdmap e779: 8 total, 8 up, 8 in 2024-08-17T17:06:48.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:47 smithi063 ceph-mon[114756]: osdmap e780: 8 total, 8 up, 8 in 2024-08-17T17:06:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:47 smithi162 ceph-mon[104668]: osdmap e779: 8 total, 8 up, 8 in 2024-08-17T17:06:48.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:47 smithi162 ceph-mon[104668]: osdmap e780: 8 total, 8 up, 8 in 2024-08-17T17:06:48.669 INFO:tasks.rados.rados.0.smithi063.stdout:2809: done (4 left) 2024-08-17T17:06:48.670 INFO:tasks.rados.rados.0.smithi063.stdout:2810: done (3 left) 2024-08-17T17:06:48.670 INFO:tasks.rados.rados.0.smithi063.stdout:2811: done (2 left) 2024-08-17T17:06:48.670 INFO:tasks.rados.rados.0.smithi063.stdout:2812: done (1 left) 2024-08-17T17:06:48.670 INFO:tasks.rados.rados.0.smithi063.stdout:2813: done (0 left) 2024-08-17T17:06:48.670 INFO:tasks.rados.rados.0.smithi063.stdout:2814: snap_remove snap 351 2024-08-17T17:06:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:48 smithi063 ceph-mon[109843]: pgmap v627: 89 pgs: 89 active+clean; 247 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 3.8 MiB/s wr, 31 op/s 2024-08-17T17:06:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:48 smithi063 ceph-mon[114756]: pgmap v627: 89 pgs: 89 active+clean; 247 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 3.8 MiB/s wr, 31 op/s 2024-08-17T17:06:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:48 smithi162 ceph-mon[104668]: pgmap v627: 89 pgs: 89 active+clean; 247 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 3.8 MiB/s wr, 31 op/s 2024-08-17T17:06:49.679 INFO:tasks.rados.rados.0.smithi063.stdout:2814: done (0 left) 2024-08-17T17:06:49.679 INFO:tasks.rados.rados.0.smithi063.stdout:2815: rollback oid 26 current snap is 356 2024-08-17T17:06:49.679 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 26 to 348 2024-08-17T17:06:49.679 INFO:tasks.rados.rados.0.smithi063.stdout:2816: delete oid 4 current snap is 356 2024-08-17T17:06:49.688 INFO:tasks.rados.rados.0.smithi063.stdout:2815: finishing rollback tid 0 to smithi06387500-26 2024-08-17T17:06:49.688 INFO:tasks.rados.rados.0.smithi063.stdout:2815: finishing rollback tid 1 to smithi06387500-26 2024-08-17T17:06:49.688 INFO:tasks.rados.rados.0.smithi063.stdout:2815: finishing rollback tid 2 to smithi06387500-26 2024-08-17T17:06:49.689 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 939 (ObjNum 1018 snap 346 seq_num 1018) dirty exists 2024-08-17T17:06:49.691 INFO:tasks.rados.rados.0.smithi063.stdout:2815: done (1 left) 2024-08-17T17:06:49.691 INFO:tasks.rados.rados.0.smithi063.stdout:2816: done (0 left) 2024-08-17T17:06:49.691 INFO:tasks.rados.rados.0.smithi063.stdout:2817: read oid 28 snap 348 2024-08-17T17:06:49.691 INFO:tasks.rados.rados.0.smithi063.stdout:2817: expect deleted 2024-08-17T17:06:49.691 INFO:tasks.rados.rados.0.smithi063.stdout:2818: write oid 44 current snap is 356 2024-08-17T17:06:49.691 INFO:tasks.rados.rados.0.smithi063.stdout:2818: seq_num 1058 ranges {581806=586509,1911596=421369,3087644=403319} 2024-08-17T17:06:49.699 INFO:tasks.rados.rados.0.smithi063.stdout:2818: writing smithi06387500-44 from 581806 to 1168315 tid 1 2024-08-17T17:06:49.703 INFO:tasks.rados.rados.0.smithi063.stdout:2818: writing smithi06387500-44 from 1911596 to 2332965 tid 2 2024-08-17T17:06:49.710 INFO:tasks.rados.rados.0.smithi063.stdout:2818: writing smithi06387500-44 from 3087644 to 3490963 tid 3 2024-08-17T17:06:49.710 INFO:tasks.rados.rados.0.smithi063.stdout:2819: snap_create 2024-08-17T17:06:49.710 INFO:tasks.rados.rados.0.smithi063.stdout:2818: finishing write tid 1 to smithi06387500-44 2024-08-17T17:06:49.710 INFO:tasks.rados.rados.0.smithi063.stdout:2818: finishing write tid 2 to smithi06387500-44 2024-08-17T17:06:49.712 INFO:tasks.rados.rados.0.smithi063.stdout:2818: finishing write tid 3 to smithi06387500-44 2024-08-17T17:06:49.713 INFO:tasks.rados.rados.0.smithi063.stdout:2818: finishing write tid 4 to smithi06387500-44 2024-08-17T17:06:49.713 INFO:tasks.rados.rados.0.smithi063.stdout:2818: finishing write tid 5 to smithi06387500-44 2024-08-17T17:06:49.713 INFO:tasks.rados.rados.0.smithi063.stdout:2818: oid 44 updating version 0 to 382 2024-08-17T17:06:49.713 INFO:tasks.rados.rados.0.smithi063.stdout:2818: oid 44 version 382 is already newer than 381 2024-08-17T17:06:49.713 INFO:tasks.rados.rados.0.smithi063.stdout:2818: oid 44 version 382 is already newer than 379 2024-08-17T17:06:49.713 INFO:tasks.rados.rados.0.smithi063.stdout:2818: oid 44 version 382 is already newer than 380 2024-08-17T17:06:49.713 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 382 (ObjNum 1058 snap 356 seq_num 1058) dirty exists 2024-08-17T17:06:49.713 INFO:tasks.rados.rados.0.smithi063.stdout:2818: left oid 44 (ObjNum 1058 snap 356 seq_num 1058) 2024-08-17T17:06:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:49 smithi063 ceph-mon[114756]: osdmap e781: 8 total, 8 up, 8 in 2024-08-17T17:06:50.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:49 smithi063 ceph-mon[109843]: osdmap e781: 8 total, 8 up, 8 in 2024-08-17T17:06:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:49 smithi162 ceph-mon[104668]: osdmap e781: 8 total, 8 up, 8 in 2024-08-17T17:06:50.684 INFO:tasks.rados.rados.0.smithi063.stdout:2817: done (2 left) 2024-08-17T17:06:50.684 INFO:tasks.rados.rados.0.smithi063.stdout:2818: done (1 left) 2024-08-17T17:06:50.684 INFO:tasks.rados.rados.0.smithi063.stdout:2819: done (0 left) 2024-08-17T17:06:50.685 INFO:tasks.rados.rados.0.smithi063.stdout:2820: delete oid 25 current snap is 357 2024-08-17T17:06:50.689 INFO:tasks.rados.rados.0.smithi063.stdout:2820: done (0 left) 2024-08-17T17:06:50.689 INFO:tasks.rados.rados.0.smithi063.stdout:2821: snap_create 2024-08-17T17:06:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:50 smithi063 ceph-mon[114756]: osdmap e782: 8 total, 8 up, 8 in 2024-08-17T17:06:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:50 smithi063 ceph-mon[114756]: pgmap v630: 89 pgs: 89 active+clean; 247 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 3.4 MiB/s wr, 29 op/s 2024-08-17T17:06:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:50 smithi063 ceph-mon[109843]: osdmap e782: 8 total, 8 up, 8 in 2024-08-17T17:06:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:50 smithi063 ceph-mon[109843]: pgmap v630: 89 pgs: 89 active+clean; 247 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 3.4 MiB/s wr, 29 op/s 2024-08-17T17:06:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:50 smithi162 ceph-mon[104668]: osdmap e782: 8 total, 8 up, 8 in 2024-08-17T17:06:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:50 smithi162 ceph-mon[104668]: pgmap v630: 89 pgs: 89 active+clean; 247 MiB data, 813 MiB used, 714 GiB / 715 GiB avail; 16 MiB/s rd, 3.4 MiB/s wr, 29 op/s 2024-08-17T17:06:51.594 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T17:06:51.685 INFO:tasks.rados.rados.0.smithi063.stdout:2821: done (0 left) 2024-08-17T17:06:51.685 INFO:tasks.rados.rados.0.smithi063.stdout:2822: rollback oid 38 current snap is 358 2024-08-17T17:06:51.685 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 38 to 328 2024-08-17T17:06:51.685 INFO:tasks.rados.rados.0.smithi063.stdout:2823: read oid 16 snap 352 2024-08-17T17:06:51.685 INFO:tasks.rados.rados.0.smithi063.stdout:2823: expect (ObjNum 1019 snap 347 seq_num 1019) 2024-08-17T17:06:51.685 INFO:tasks.rados.rados.0.smithi063.stdout:2824: read oid 33 snap -1 2024-08-17T17:06:51.685 INFO:tasks.rados.rados.0.smithi063.stdout:2824: expect (ObjNum 1056 snap 355 seq_num 1056) 2024-08-17T17:06:51.685 INFO:tasks.rados.rados.0.smithi063.stdout:2825: rollback oid 34 current snap is 358 2024-08-17T17:06:51.685 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 34 to 348 2024-08-17T17:06:51.685 INFO:tasks.rados.rados.0.smithi063.stdout:2826: write oid 3 current snap is 358 2024-08-17T17:06:51.686 INFO:tasks.rados.rados.0.smithi063.stdout:2826: seq_num 1060 ranges {615420=737007,1912207=425038,3026137=542216} 2024-08-17T17:06:51.693 INFO:tasks.rados.rados.0.smithi063.stdout:2826: writing smithi06387500-3 from 615420 to 1352427 tid 1 2024-08-17T17:06:51.699 INFO:tasks.rados.rados.0.smithi063.stdout:2826: writing smithi06387500-3 from 1912207 to 2337245 tid 2 2024-08-17T17:06:51.706 INFO:tasks.rados.rados.0.smithi063.stdout:2826: writing smithi06387500-3 from 3026137 to 3568353 tid 3 2024-08-17T17:06:51.706 INFO:tasks.rados.rados.0.smithi063.stdout:2827: write oid 37 current snap is 358 2024-08-17T17:06:51.706 INFO:tasks.rados.rados.0.smithi063.stdout:2827: seq_num 1061 ranges {632227=412256,1570689=568696,2653560=753488} 2024-08-17T17:06:51.712 INFO:tasks.rados.rados.0.smithi063.stdout:2827: writing smithi06387500-37 from 632227 to 1044483 tid 1 2024-08-17T17:06:51.718 INFO:tasks.rados.rados.0.smithi063.stdout:2827: writing smithi06387500-37 from 1570689 to 2139385 tid 2 2024-08-17T17:06:51.725 INFO:tasks.rados.rados.0.smithi063.stdout:2827: writing smithi06387500-37 from 2653560 to 3407048 tid 3 2024-08-17T17:06:51.725 INFO:tasks.rados.rados.0.smithi063.stdout:2828: write oid 35 current snap is 358 2024-08-17T17:06:51.725 INFO:tasks.rados.rados.0.smithi063.stdout:2828: seq_num 1062 ranges {649034=487505,1629171=428707,2680983=564760,3798768=50266} 2024-08-17T17:06:51.731 INFO:tasks.rados.rados.0.smithi063.stdout:2828: writing smithi06387500-35 from 649034 to 1136539 tid 1 2024-08-17T17:06:51.736 INFO:tasks.rados.rados.0.smithi063.stdout:2828: writing smithi06387500-35 from 1629171 to 2057878 tid 2 2024-08-17T17:06:51.743 INFO:tasks.rados.rados.0.smithi063.stdout:2828: writing smithi06387500-35 from 2680983 to 3245743 tid 3 2024-08-17T17:06:51.747 INFO:tasks.rados.rados.0.smithi063.stdout:2828: writing smithi06387500-35 from 3798768 to 3849034 tid 4 2024-08-17T17:06:51.747 INFO:tasks.rados.rados.0.smithi063.stdout:2829: snap_create 2024-08-17T17:06:51.747 INFO:tasks.rados.rados.0.smithi063.stdout:2825: finishing rollback tid 1 to smithi06387500-34 2024-08-17T17:06:51.747 INFO:tasks.rados.rados.0.smithi063.stdout:2825: finishing rollback tid 2 to smithi06387500-34 2024-08-17T17:06:51.747 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 1842 (ObjNum 971 snap 333 seq_num 971) dirty exists 2024-08-17T17:06:51.747 INFO:tasks.rados.rados.0.smithi063.stdout:2822: finishing rollback tid 0 to smithi06387500-38 2024-08-17T17:06:51.747 INFO:tasks.rados.rados.0.smithi063.stdout:2822: finishing rollback tid 1 to smithi06387500-38 2024-08-17T17:06:51.747 INFO:tasks.rados.rados.0.smithi063.stdout:2822: finishing rollback tid 2 to smithi06387500-38 2024-08-17T17:06:51.747 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 449 (ObjNum 868 snap 300 seq_num 868) dirty exists 2024-08-17T17:06:51.748 INFO:tasks.rados.rados.0.smithi063.stdout:2824: expect (ObjNum 1056 snap 355 seq_num 1056) 2024-08-17T17:06:51.954 INFO:tasks.rados.rados.0.smithi063.stdout:2826: finishing write tid 1 to smithi06387500-3 2024-08-17T17:06:51.955 INFO:tasks.rados.rados.0.smithi063.stdout:2823: expect (ObjNum 1019 snap 347 seq_num 1019) 2024-08-17T17:06:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:51 smithi063 ceph-mon[114756]: osdmap e783: 8 total, 8 up, 8 in 2024-08-17T17:06:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:51 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:51 smithi063 ceph-mon[114756]: from='client.44300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:52.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:51 smithi063 ceph-mon[109843]: osdmap e783: 8 total, 8 up, 8 in 2024-08-17T17:06:52.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:51 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:52.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:51 smithi063 ceph-mon[109843]: from='client.44300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:52.060 INFO:tasks.rados.rados.0.smithi063.stdout:2826: finishing write tid 2 to smithi06387500-3 2024-08-17T17:06:52.060 INFO:tasks.rados.rados.0.smithi063.stdout:2826: finishing write tid 3 to smithi06387500-3 2024-08-17T17:06:52.060 INFO:tasks.rados.rados.0.smithi063.stdout:2826: finishing write tid 4 to smithi06387500-3 2024-08-17T17:06:52.060 INFO:tasks.rados.rados.0.smithi063.stdout:2826: finishing write tid 5 to smithi06387500-3 2024-08-17T17:06:52.060 INFO:tasks.rados.rados.0.smithi063.stdout:2826: oid 3 updating version 0 to 946 2024-08-17T17:06:52.060 INFO:tasks.rados.rados.0.smithi063.stdout:2826: oid 3 version 946 is already newer than 944 2024-08-17T17:06:52.060 INFO:tasks.rados.rados.0.smithi063.stdout:2826: oid 3 version 946 is already newer than 943 2024-08-17T17:06:52.060 INFO:tasks.rados.rados.0.smithi063.stdout:2826: oid 3 version 946 is already newer than 945 2024-08-17T17:06:52.061 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 946 (ObjNum 1060 snap 358 seq_num 1060) dirty exists 2024-08-17T17:06:52.061 INFO:tasks.rados.rados.0.smithi063.stdout:2826: left oid 3 (ObjNum 1060 snap 358 seq_num 1060) 2024-08-17T17:06:52.061 INFO:tasks.rados.rados.0.smithi063.stdout:2827: finishing write tid 1 to smithi06387500-37 2024-08-17T17:06:52.061 INFO:tasks.rados.rados.0.smithi063.stdout:2827: finishing write tid 2 to smithi06387500-37 2024-08-17T17:06:52.061 INFO:tasks.rados.rados.0.smithi063.stdout:2827: finishing write tid 3 to smithi06387500-37 2024-08-17T17:06:52.061 INFO:tasks.rados.rados.0.smithi063.stdout:2827: finishing write tid 4 to smithi06387500-37 2024-08-17T17:06:52.061 INFO:tasks.rados.rados.0.smithi063.stdout:2827: finishing write tid 5 to smithi06387500-37 2024-08-17T17:06:52.061 INFO:tasks.rados.rados.0.smithi063.stdout:2827: oid 37 updating version 0 to 2730 2024-08-17T17:06:52.061 INFO:tasks.rados.rados.0.smithi063.stdout:2827: oid 37 version 2730 is already newer than 2729 2024-08-17T17:06:52.061 INFO:tasks.rados.rados.0.smithi063.stdout:2827: oid 37 updating version 2730 to 2731 2024-08-17T17:06:52.061 INFO:tasks.rados.rados.0.smithi063.stdout:2827: oid 37 updating version 2731 to 2732 2024-08-17T17:06:52.061 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 2732 (ObjNum 1061 snap 358 seq_num 1061) dirty exists 2024-08-17T17:06:52.061 INFO:tasks.rados.rados.0.smithi063.stdout:2827: left oid 37 (ObjNum 1061 snap 358 seq_num 1061) 2024-08-17T17:06:52.062 INFO:tasks.rados.rados.0.smithi063.stdout:2828: finishing write tid 1 to smithi06387500-35 2024-08-17T17:06:52.062 INFO:tasks.rados.rados.0.smithi063.stdout:2828: finishing write tid 2 to smithi06387500-35 2024-08-17T17:06:52.062 INFO:tasks.rados.rados.0.smithi063.stdout:2828: finishing write tid 3 to smithi06387500-35 2024-08-17T17:06:52.062 INFO:tasks.rados.rados.0.smithi063.stdout:2828: finishing write tid 4 to smithi06387500-35 2024-08-17T17:06:52.062 INFO:tasks.rados.rados.0.smithi063.stdout:2828: finishing write tid 5 to smithi06387500-35 2024-08-17T17:06:52.062 INFO:tasks.rados.rados.0.smithi063.stdout:2828: finishing write tid 6 to smithi06387500-35 2024-08-17T17:06:52.062 INFO:tasks.rados.rados.0.smithi063.stdout:2828: oid 35 updating version 0 to 2734 2024-08-17T17:06:52.062 INFO:tasks.rados.rados.0.smithi063.stdout:2828: oid 35 updating version 2734 to 2736 2024-08-17T17:06:52.062 INFO:tasks.rados.rados.0.smithi063.stdout:2828: oid 35 version 2736 is already newer than 2735 2024-08-17T17:06:52.062 INFO:tasks.rados.rados.0.smithi063.stdout:2828: oid 35 updating version 2736 to 2737 2024-08-17T17:06:52.062 INFO:tasks.rados.rados.0.smithi063.stdout:2828: oid 35 updating version 2737 to 2738 2024-08-17T17:06:52.062 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 2738 (ObjNum 1062 snap 358 seq_num 1062) dirty exists 2024-08-17T17:06:52.063 INFO:tasks.rados.rados.0.smithi063.stdout:2828: left oid 35 (ObjNum 1062 snap 358 seq_num 1062) 2024-08-17T17:06:52.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:51 smithi162 ceph-mon[104668]: osdmap e783: 8 total, 8 up, 8 in 2024-08-17T17:06:52.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:51 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:52.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:51 smithi162 ceph-mon[104668]: from='client.44300 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:52.195 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:06:52.195 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (8m) 16s ago 14m 19.0M - 0.27.0 11f11916f8cd 59f4018b91ca 2024-08-17T17:06:52.196 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (7m) 15s ago 14m 79.0M - c8b91775d855 581e03e654a8 2024-08-17T17:06:52.196 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (13m) 16s ago 13m 23.3M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T17:06:52.196 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (13m) 15s ago 13m 19.5M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T17:06:52.196 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,9283,8765 running (9m) 15s ago 18m 457M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb da48034a2895 2024-08-17T17:06:52.196 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:8443,9283,8765 running (9m) 16s ago 19m 518M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 943e639cede6 2024-08-17T17:06:52.196 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (7m) 16s ago 19m 71.2M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb f0f20fcac956 2024-08-17T17:06:52.196 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (4m) 15s ago 18m 43.5M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 520525c79dbe 2024-08-17T17:06:52.196 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (6m) 16s ago 18m 54.8M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb c484af026acf 2024-08-17T17:06:52.196 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 running (9m) 16s ago 13m 10.9M - 1.7.0 72c9c2088986 34f55f5de2aa 2024-08-17T17:06:52.196 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (9m) 15s ago 13m 11.8M - 1.7.0 72c9c2088986 5bb162d0f1a9 2024-08-17T17:06:52.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (52s) 16s ago 17m 72.9M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 8fa117adbc0a 2024-08-17T17:06:52.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (20s) 16s ago 17m 12.2M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 11bee521ecad 2024-08-17T17:06:52.197 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (17m) 16s ago 17m 110M 2480M 18.2.4-719-gced649f8 c870753d67c2 a3cbe8edb49a 2024-08-17T17:06:52.197 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (16m) 16s ago 16m 98.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T17:06:52.197 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (16m) 15s ago 16m 134M 2480M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T17:06:52.197 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (15m) 15s ago 15m 121M 2480M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T17:06:52.197 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (15m) 15s ago 15m 90.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T17:06:52.197 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (15m) 15s ago 15m 101M 2480M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T17:06:52.197 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (8m) 15s ago 14m 48.0M - 2.51.0 1d3b7f56885b dfe66b2c5568 2024-08-17T17:06:52.544 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:06:52.544 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T17:06:52.545 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:06:52.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:06:52.545 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T17:06:52.545 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:06:52.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:06:52.545 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T17:06:52.545 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 6, 2024-08-17T17:06:52.545 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:06:52.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:06:52.545 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T17:06:52.545 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:06:52.546 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:06:52.546 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T17:06:52.546 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 8, 2024-08-17T17:06:52.546 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 7 2024-08-17T17:06:52.546 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:06:52.546 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:06:52.616 INFO:tasks.rados.rados.0.smithi063.stdout:2822: done (7 left) 2024-08-17T17:06:52.616 INFO:tasks.rados.rados.0.smithi063.stdout:2823: done (6 left) 2024-08-17T17:06:52.616 INFO:tasks.rados.rados.0.smithi063.stdout:2824: done (5 left) 2024-08-17T17:06:52.616 INFO:tasks.rados.rados.0.smithi063.stdout:2825: done (4 left) 2024-08-17T17:06:52.616 INFO:tasks.rados.rados.0.smithi063.stdout:2826: done (3 left) 2024-08-17T17:06:52.616 INFO:tasks.rados.rados.0.smithi063.stdout:2827: done (2 left) 2024-08-17T17:06:52.616 INFO:tasks.rados.rados.0.smithi063.stdout:2828: done (1 left) 2024-08-17T17:06:52.616 INFO:tasks.rados.rados.0.smithi063.stdout:2829: done (0 left) 2024-08-17T17:06:52.616 INFO:tasks.rados.rados.0.smithi063.stdout:2830: read oid 45 snap -1 2024-08-17T17:06:52.617 INFO:tasks.rados.rados.0.smithi063.stdout:2830: expect deleted 2024-08-17T17:06:52.617 INFO:tasks.rados.rados.0.smithi063.stdout:2831: write oid 8 current snap is 359 2024-08-17T17:06:52.617 INFO:tasks.rados.rados.0.smithi063.stdout:2831: seq_num 1063 ranges {665841=562754,1687653=572365,2708406=776032} 2024-08-17T17:06:52.622 INFO:tasks.rados.rados.0.smithi063.stdout:2831: writing smithi06387500-8 from 665841 to 1228595 tid 1 2024-08-17T17:06:52.628 INFO:tasks.rados.rados.0.smithi063.stdout:2831: writing smithi06387500-8 from 1687653 to 2260018 tid 2 2024-08-17T17:06:52.636 INFO:tasks.rados.rados.0.smithi063.stdout:2831: writing smithi06387500-8 from 2708406 to 3484438 tid 3 2024-08-17T17:06:52.637 INFO:tasks.rados.rados.0.smithi063.stdout:2832: read oid 39 snap 339 2024-08-17T17:06:52.637 INFO:tasks.rados.rados.0.smithi063.stdout:2832: expect (ObjNum 943 snap 323 seq_num 943) 2024-08-17T17:06:52.637 INFO:tasks.rados.rados.0.smithi063.stdout:2833: rollback oid 40 current snap is 359 2024-08-17T17:06:52.637 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 40 to 354 2024-08-17T17:06:52.637 INFO:tasks.rados.rados.0.smithi063.stdout:2831: finishing write tid 1 to smithi06387500-8 2024-08-17T17:06:52.637 INFO:tasks.rados.rados.0.smithi063.stdout:2831: finishing write tid 2 to smithi06387500-8 2024-08-17T17:06:52.637 INFO:tasks.rados.rados.0.smithi063.stdout:2830: done (3 left) 2024-08-17T17:06:52.637 INFO:tasks.rados.rados.0.smithi063.stdout:2834: read oid 49 snap -1 2024-08-17T17:06:52.637 INFO:tasks.rados.rados.0.smithi063.stdout:2834: expect (ObjNum 896 snap 310 seq_num 896) 2024-08-17T17:06:52.637 INFO:tasks.rados.rados.0.smithi063.stdout:2835: read oid 4 snap -1 2024-08-17T17:06:52.637 INFO:tasks.rados.rados.0.smithi063.stdout:2835: expect deleted 2024-08-17T17:06:52.637 INFO:tasks.rados.rados.0.smithi063.stdout:2836: rollback oid 22 current snap is 359 2024-08-17T17:06:52.638 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 22 to 353 2024-08-17T17:06:52.638 INFO:tasks.rados.rados.0.smithi063.stdout:2837: read oid 41 snap 353 2024-08-17T17:06:52.638 INFO:tasks.rados.rados.0.smithi063.stdout:2837: expect (ObjNum 1034 snap 352 seq_num 1034) 2024-08-17T17:06:52.638 INFO:tasks.rados.rados.0.smithi063.stdout:2838: read oid 19 snap -1 2024-08-17T17:06:52.638 INFO:tasks.rados.rados.0.smithi063.stdout:2838: expect deleted 2024-08-17T17:06:52.638 INFO:tasks.rados.rados.0.smithi063.stdout:2839: snap_remove snap 348 2024-08-17T17:06:52.856 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:06:52.856 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:06:52.856 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-08-17T17:06:52.856 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-08-17T17:06:52.856 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-08-17T17:06:52.856 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "2/8 daemons upgraded", 2024-08-17T17:06:52.856 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading osd daemons", 2024-08-17T17:06:52.856 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T17:06:52.856 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:06:52.857 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:52 smithi063 ceph-mon[109843]: osdmap e784: 8 total, 8 up, 8 in 2024-08-17T17:06:52.857 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:52 smithi063 ceph-mon[109843]: from='client.34377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:52.857 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:52 smithi063 ceph-mon[109843]: pgmap v633: 89 pgs: 89 active+clean; 266 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 3.1 MiB/s wr, 17 op/s 2024-08-17T17:06:52.857 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:52 smithi063 ceph-mon[109843]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:52.857 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:52 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1478052306' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:52.857 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:52 smithi063 ceph-mon[109843]: osdmap e785: 8 total, 8 up, 8 in 2024-08-17T17:06:52.857 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:52 smithi063 ceph-mon[114756]: osdmap e784: 8 total, 8 up, 8 in 2024-08-17T17:06:52.857 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:52 smithi063 ceph-mon[114756]: from='client.34377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:52.858 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:52 smithi063 ceph-mon[114756]: pgmap v633: 89 pgs: 89 active+clean; 266 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 3.1 MiB/s wr, 17 op/s 2024-08-17T17:06:52.858 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:52 smithi063 ceph-mon[114756]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:52.858 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:52 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1478052306' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:52.858 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:52 smithi063 ceph-mon[114756]: osdmap e785: 8 total, 8 up, 8 in 2024-08-17T17:06:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:52 smithi162 ceph-mon[104668]: osdmap e784: 8 total, 8 up, 8 in 2024-08-17T17:06:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:52 smithi162 ceph-mon[104668]: from='client.34377 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:52 smithi162 ceph-mon[104668]: pgmap v633: 89 pgs: 89 active+clean; 266 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 3.1 MiB/s wr, 17 op/s 2024-08-17T17:06:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:52 smithi162 ceph-mon[104668]: from='client.44312 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:52 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1478052306' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:06:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:52 smithi162 ceph-mon[104668]: osdmap e785: 8 total, 8 up, 8 in 2024-08-17T17:06:53.615 INFO:tasks.rados.rados.0.smithi063.stdout:2839: done (8 left) 2024-08-17T17:06:53.615 INFO:tasks.rados.rados.0.smithi063.stdout:2840: snap_remove snap 358 2024-08-17T17:06:53.661 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:06:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:06:53] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.51.0" 2024-08-17T17:06:54.628 INFO:tasks.rados.rados.0.smithi063.stdout:2833: finishing rollback tid 0 to smithi06387500-40 2024-08-17T17:06:54.628 INFO:tasks.rados.rados.0.smithi063.stdout:2836: finishing rollback tid 0 to smithi06387500-22 2024-08-17T17:06:54.628 INFO:tasks.rados.rados.0.smithi063.stdout:2836: finishing rollback tid 1 to smithi06387500-22 2024-08-17T17:06:54.628 INFO:tasks.rados.rados.0.smithi063.stdout:2836: finishing rollback tid 2 to smithi06387500-22 2024-08-17T17:06:54.628 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 610 (ObjNum 1028 snap 350 seq_num 1028) dirty exists 2024-08-17T17:06:54.628 INFO:tasks.rados.rados.0.smithi063.stdout:2833: finishing rollback tid 1 to smithi06387500-40 2024-08-17T17:06:54.628 INFO:tasks.rados.rados.0.smithi063.stdout:2833: finishing rollback tid 2 to smithi06387500-40 2024-08-17T17:06:54.628 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 958 (ObjNum 1029 snap 350 seq_num 1029) dirty exists 2024-08-17T17:06:54.628 INFO:tasks.rados.rados.0.smithi063.stdout:2831: finishing write tid 3 to smithi06387500-8 2024-08-17T17:06:54.628 INFO:tasks.rados.rados.0.smithi063.stdout:2831: finishing write tid 4 to smithi06387500-8 2024-08-17T17:06:54.629 INFO:tasks.rados.rados.0.smithi063.stdout:2831: finishing write tid 5 to smithi06387500-8 2024-08-17T17:06:54.629 INFO:tasks.rados.rados.0.smithi063.stdout:2831: oid 8 updating version 0 to 960 2024-08-17T17:06:54.629 INFO:tasks.rados.rados.0.smithi063.stdout:2831: oid 8 updating version 960 to 961 2024-08-17T17:06:54.629 INFO:tasks.rados.rados.0.smithi063.stdout:2831: oid 8 updating version 961 to 962 2024-08-17T17:06:54.629 INFO:tasks.rados.rados.0.smithi063.stdout:2831: oid 8 version 962 is already newer than 959 2024-08-17T17:06:54.629 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 962 (ObjNum 1063 snap 359 seq_num 1063) dirty exists 2024-08-17T17:06:54.629 INFO:tasks.rados.rados.0.smithi063.stdout:2831: left oid 8 (ObjNum 1063 snap 359 seq_num 1063) 2024-08-17T17:06:54.629 INFO:tasks.rados.rados.0.smithi063.stdout:2834: expect (ObjNum 896 snap 310 seq_num 896) 2024-08-17T17:06:54.776 INFO:tasks.rados.rados.0.smithi063.stdout:2832: expect (ObjNum 943 snap 323 seq_num 943) 2024-08-17T17:06:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:54 smithi063 ceph-mon[114756]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:54 smithi063 ceph-mon[114756]: osdmap e786: 8 total, 8 up, 8 in 2024-08-17T17:06:55.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:54 smithi063 ceph-mon[109843]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:55.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:54 smithi063 ceph-mon[109843]: osdmap e786: 8 total, 8 up, 8 in 2024-08-17T17:06:55.030 INFO:tasks.rados.rados.0.smithi063.stdout:2837: expect (ObjNum 1034 snap 352 seq_num 1034) 2024-08-17T17:06:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:54 smithi162 ceph-mon[104668]: from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:06:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:54 smithi162 ceph-mon[104668]: osdmap e786: 8 total, 8 up, 8 in 2024-08-17T17:06:55.277 INFO:tasks.rados.rados.0.smithi063.stdout:2831: done (8 left) 2024-08-17T17:06:55.278 INFO:tasks.rados.rados.0.smithi063.stdout:2832: done (7 left) 2024-08-17T17:06:55.278 INFO:tasks.rados.rados.0.smithi063.stdout:2833: done (6 left) 2024-08-17T17:06:55.278 INFO:tasks.rados.rados.0.smithi063.stdout:2834: done (5 left) 2024-08-17T17:06:55.278 INFO:tasks.rados.rados.0.smithi063.stdout:2835: done (4 left) 2024-08-17T17:06:55.278 INFO:tasks.rados.rados.0.smithi063.stdout:2836: done (3 left) 2024-08-17T17:06:55.278 INFO:tasks.rados.rados.0.smithi063.stdout:2837: done (2 left) 2024-08-17T17:06:55.278 INFO:tasks.rados.rados.0.smithi063.stdout:2838: done (1 left) 2024-08-17T17:06:55.278 INFO:tasks.rados.rados.0.smithi063.stdout:2840: done (0 left) 2024-08-17T17:06:55.278 INFO:tasks.rados.rados.0.smithi063.stdout:2841: read oid 41 snap -1 2024-08-17T17:06:55.278 INFO:tasks.rados.rados.0.smithi063.stdout:2841: expect (ObjNum 1034 snap 352 seq_num 1034) 2024-08-17T17:06:55.278 INFO:tasks.rados.rados.0.smithi063.stdout:2842: write oid 5 current snap is 359 2024-08-17T17:06:55.278 INFO:tasks.rados.rados.0.smithi063.stdout:2842: seq_num 1064 ranges {682648=638003,2029782=432376,3019476=587304} 2024-08-17T17:06:55.284 INFO:tasks.rados.rados.0.smithi063.stdout:2842: writing smithi06387500-5 from 682648 to 1320651 tid 1 2024-08-17T17:06:55.290 INFO:tasks.rados.rados.0.smithi063.stdout:2842: writing smithi06387500-5 from 2029782 to 2462158 tid 2 2024-08-17T17:06:55.297 INFO:tasks.rados.rados.0.smithi063.stdout:2842: writing smithi06387500-5 from 3019476 to 3606780 tid 3 2024-08-17T17:06:55.297 INFO:tasks.rados.rados.0.smithi063.stdout:2843: snap_create 2024-08-17T17:06:55.297 INFO:tasks.rados.rados.0.smithi063.stdout:2841: expect (ObjNum 1034 snap 352 seq_num 1034) 2024-08-17T17:06:55.544 INFO:tasks.rados.rados.0.smithi063.stdout:2842: finishing write tid 1 to smithi06387500-5 2024-08-17T17:06:55.544 INFO:tasks.rados.rados.0.smithi063.stdout:2842: finishing write tid 2 to smithi06387500-5 2024-08-17T17:06:55.544 INFO:tasks.rados.rados.0.smithi063.stdout:2842: finishing write tid 3 to smithi06387500-5 2024-08-17T17:06:55.544 INFO:tasks.rados.rados.0.smithi063.stdout:2842: finishing write tid 4 to smithi06387500-5 2024-08-17T17:06:55.544 INFO:tasks.rados.rados.0.smithi063.stdout:2842: finishing write tid 5 to smithi06387500-5 2024-08-17T17:06:55.544 INFO:tasks.rados.rados.0.smithi063.stdout:2842: oid 5 updating version 0 to 1394 2024-08-17T17:06:55.544 INFO:tasks.rados.rados.0.smithi063.stdout:2842: oid 5 version 1394 is already newer than 1393 2024-08-17T17:06:55.544 INFO:tasks.rados.rados.0.smithi063.stdout:2842: oid 5 updating version 1394 to 1395 2024-08-17T17:06:55.544 INFO:tasks.rados.rados.0.smithi063.stdout:2842: oid 5 updating version 1395 to 1396 2024-08-17T17:06:55.545 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1396 (ObjNum 1064 snap 359 seq_num 1064) dirty exists 2024-08-17T17:06:55.545 INFO:tasks.rados.rados.0.smithi063.stdout:2842: left oid 5 (ObjNum 1064 snap 359 seq_num 1064) 2024-08-17T17:06:55.628 INFO:tasks.rados.rados.0.smithi063.stdout:2841: done (2 left) 2024-08-17T17:06:55.628 INFO:tasks.rados.rados.0.smithi063.stdout:2842: done (1 left) 2024-08-17T17:06:55.628 INFO:tasks.rados.rados.0.smithi063.stdout:2843: done (0 left) 2024-08-17T17:06:55.628 INFO:tasks.rados.rados.0.smithi063.stdout:2844: write oid 12 current snap is 360 2024-08-17T17:06:55.628 INFO:tasks.rados.rados.0.smithi063.stdout:2844: seq_num 1065 ranges {699455=429605,1804617=576034,3163252=514929} 2024-08-17T17:06:55.634 INFO:tasks.rados.rados.0.smithi063.stdout:2844: writing smithi06387500-12 from 699455 to 1129060 tid 1 2024-08-17T17:06:55.642 INFO:tasks.rados.rados.0.smithi063.stdout:2844: writing smithi06387500-12 from 1804617 to 2380651 tid 2 2024-08-17T17:06:55.650 INFO:tasks.rados.rados.0.smithi063.stdout:2844: writing smithi06387500-12 from 3163252 to 3678181 tid 3 2024-08-17T17:06:55.650 INFO:tasks.rados.rados.0.smithi063.stdout:2845: rollback oid 13 current snap is 360 2024-08-17T17:06:55.650 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 13 to 307 2024-08-17T17:06:55.650 INFO:tasks.rados.rados.0.smithi063.stdout:2844: finishing write tid 1 to smithi06387500-12 2024-08-17T17:06:55.650 INFO:tasks.rados.rados.0.smithi063.stdout:2844: finishing write tid 2 to smithi06387500-12 2024-08-17T17:06:55.650 INFO:tasks.rados.rados.0.smithi063.stdout:2846: delete oid 41 current snap is 360 2024-08-17T17:06:55.652 INFO:tasks.rados.rados.0.smithi063.stdout:2844: finishing write tid 3 to smithi06387500-12 2024-08-17T17:06:55.653 INFO:tasks.rados.rados.0.smithi063.stdout:2844: finishing write tid 4 to smithi06387500-12 2024-08-17T17:06:55.653 INFO:tasks.rados.rados.0.smithi063.stdout:2846: done (2 left) 2024-08-17T17:06:55.653 INFO:tasks.rados.rados.0.smithi063.stdout:2847: snap_create 2024-08-17T17:06:55.653 INFO:tasks.rados.rados.0.smithi063.stdout:2844: finishing write tid 5 to smithi06387500-12 2024-08-17T17:06:55.653 INFO:tasks.rados.rados.0.smithi063.stdout:2844: oid 12 updating version 0 to 646 2024-08-17T17:06:55.653 INFO:tasks.rados.rados.0.smithi063.stdout:2844: oid 12 updating version 646 to 647 2024-08-17T17:06:55.653 INFO:tasks.rados.rados.0.smithi063.stdout:2844: oid 12 version 647 is already newer than 644 2024-08-17T17:06:55.653 INFO:tasks.rados.rados.0.smithi063.stdout:2844: oid 12 version 647 is already newer than 645 2024-08-17T17:06:55.653 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 647 (ObjNum 1065 snap 360 seq_num 1065) dirty exists 2024-08-17T17:06:55.653 INFO:tasks.rados.rados.0.smithi063.stdout:2844: left oid 12 (ObjNum 1065 snap 360 seq_num 1065) 2024-08-17T17:06:55.654 INFO:tasks.rados.rados.0.smithi063.stdout:2845: finishing rollback tid 0 to smithi06387500-13 2024-08-17T17:06:55.654 INFO:tasks.rados.rados.0.smithi063.stdout:2845: finishing rollback tid 1 to smithi06387500-13 2024-08-17T17:06:55.654 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1399 (ObjNum 97 snap 0 seq_num 1807083920) dirty dne 2024-08-17T17:06:55.980 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:55 smithi063 ceph-mon[114756]: pgmap v636: 89 pgs: 89 active+clean; 272 MiB data, 858 MiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 4.0 MiB/s wr, 22 op/s 2024-08-17T17:06:55.981 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:55 smithi063 ceph-mon[114756]: osdmap e787: 8 total, 8 up, 8 in 2024-08-17T17:06:55.981 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:55 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:06:55.981 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:55 smithi063 ceph-mon[109843]: pgmap v636: 89 pgs: 89 active+clean; 272 MiB data, 858 MiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 4.0 MiB/s wr, 22 op/s 2024-08-17T17:06:55.981 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:55 smithi063 ceph-mon[109843]: osdmap e787: 8 total, 8 up, 8 in 2024-08-17T17:06:55.981 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:55 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:06:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:55 smithi162 ceph-mon[104668]: pgmap v636: 89 pgs: 89 active+clean; 272 MiB data, 858 MiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 4.0 MiB/s wr, 22 op/s 2024-08-17T17:06:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:55 smithi162 ceph-mon[104668]: osdmap e787: 8 total, 8 up, 8 in 2024-08-17T17:06:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:55 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-08-17T17:06:56.639 INFO:tasks.rados.rados.0.smithi063.stdout:2844: done (2 left) 2024-08-17T17:06:56.639 INFO:tasks.rados.rados.0.smithi063.stdout:2845: done (1 left) 2024-08-17T17:06:56.639 INFO:tasks.rados.rados.0.smithi063.stdout:2847: done (0 left) 2024-08-17T17:06:56.639 INFO:tasks.rados.rados.0.smithi063.stdout:2848: read oid 48 snap -1 2024-08-17T17:06:56.639 INFO:tasks.rados.rados.0.smithi063.stdout:2848: expect deleted 2024-08-17T17:06:56.639 INFO:tasks.rados.rados.0.smithi063.stdout:2849: delete oid 23 current snap is 361 2024-08-17T17:06:56.642 INFO:tasks.rados.rados.0.smithi063.stdout:2848: done (1 left) 2024-08-17T17:06:56.642 INFO:tasks.rados.rados.0.smithi063.stdout:2849: done (0 left) 2024-08-17T17:06:56.642 INFO:tasks.rados.rados.0.smithi063.stdout:2850: read oid 47 snap -1 2024-08-17T17:06:56.642 INFO:tasks.rados.rados.0.smithi063.stdout:2850: expect (ObjNum 1014 snap 346 seq_num 1014) 2024-08-17T17:06:56.642 INFO:tasks.rados.rados.0.smithi063.stdout:2851: read oid 2 snap 339 2024-08-17T17:06:56.642 INFO:tasks.rados.rados.0.smithi063.stdout:2851: expect (ObjNum 987 snap 339 seq_num 987) 2024-08-17T17:06:56.642 INFO:tasks.rados.rados.0.smithi063.stdout:2852: snap_remove snap 332 2024-08-17T17:06:57.002 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:56 smithi063 ceph-mon[114756]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:06:57.002 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:56 smithi063 ceph-mon[114756]: Upgrade: osd.2 is safe to restart 2024-08-17T17:06:57.002 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:56 smithi063 ceph-mon[114756]: osdmap e788: 8 total, 8 up, 8 in 2024-08-17T17:06:57.002 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:56 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:57.002 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:56 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:06:57.002 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:56 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:57.003 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:56 smithi063 ceph-mon[109843]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:06:57.003 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:56 smithi063 ceph-mon[109843]: Upgrade: osd.2 is safe to restart 2024-08-17T17:06:57.003 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:56 smithi063 ceph-mon[109843]: osdmap e788: 8 total, 8 up, 8 in 2024-08-17T17:06:57.003 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:56 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:57.003 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:56 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:06:57.003 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:56 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:57.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:56 smithi162 ceph-mon[104668]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-08-17T17:06:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:56 smithi162 ceph-mon[104668]: Upgrade: osd.2 is safe to restart 2024-08-17T17:06:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:56 smithi162 ceph-mon[104668]: osdmap e788: 8 total, 8 up, 8 in 2024-08-17T17:06:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:56 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:06:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:56 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-08-17T17:06:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:56 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:06:57.621 INFO:tasks.rados.rados.0.smithi063.stdout:2852: done (2 left) 2024-08-17T17:06:57.621 INFO:tasks.rados.rados.0.smithi063.stdout:2853: read oid 25 snap -1 2024-08-17T17:06:57.621 INFO:tasks.rados.rados.0.smithi063.stdout:2853: expect deleted 2024-08-17T17:06:57.621 INFO:tasks.rados.rados.0.smithi063.stdout:2854: write oid 35 current snap is 361 2024-08-17T17:06:57.621 INFO:tasks.rados.rados.0.smithi063.stdout:2854: seq_num 1068 ranges {749876=655352,1863710=723361,3012815=748745} 2024-08-17T17:06:57.629 INFO:tasks.rados.rados.0.smithi063.stdout:2854: writing smithi06387500-35 from 749876 to 1405228 tid 1 2024-08-17T17:06:57.637 INFO:tasks.rados.rados.0.smithi063.stdout:2854: writing smithi06387500-35 from 1863710 to 2587071 tid 2 2024-08-17T17:06:57.644 INFO:tasks.rados.rados.0.smithi063.stdout:2854: writing smithi06387500-35 from 3012815 to 3761560 tid 3 2024-08-17T17:06:57.644 INFO:tasks.rados.rados.0.smithi063.stdout:2855: read oid 1 snap -1 2024-08-17T17:06:57.644 INFO:tasks.rados.rados.0.smithi063.stdout:2855: expect deleted 2024-08-17T17:06:57.644 INFO:tasks.rados.rados.0.smithi063.stdout:2856: snap_remove snap 307 2024-08-17T17:06:57.706 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:57 smithi063 ceph-mon[109843]: pgmap v639: 89 pgs: 89 active+clean; 269 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 935 KiB/s wr, 5 op/s 2024-08-17T17:06:57.706 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:57 smithi063 ceph-mon[109843]: Upgrade: Updating osd.2 2024-08-17T17:06:57.706 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:57 smithi063 ceph-mon[109843]: Deploying daemon osd.2 on smithi063 2024-08-17T17:06:57.706 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:57 smithi063 ceph-mon[109843]: osdmap e789: 8 total, 8 up, 8 in 2024-08-17T17:06:57.706 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:57 smithi063 ceph-mon[109843]: osdmap e790: 8 total, 8 up, 8 in 2024-08-17T17:06:57.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:57 smithi063 ceph-mon[114756]: pgmap v639: 89 pgs: 89 active+clean; 269 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 935 KiB/s wr, 5 op/s 2024-08-17T17:06:57.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:57 smithi063 ceph-mon[114756]: Upgrade: Updating osd.2 2024-08-17T17:06:57.707 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:57 smithi063 ceph-mon[114756]: Deploying daemon osd.2 on smithi063 2024-08-17T17:06:57.707 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:57 smithi063 ceph-mon[114756]: osdmap e789: 8 total, 8 up, 8 in 2024-08-17T17:06:57.707 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:57 smithi063 ceph-mon[114756]: osdmap e790: 8 total, 8 up, 8 in 2024-08-17T17:06:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:57 smithi162 ceph-mon[104668]: pgmap v639: 89 pgs: 89 active+clean; 269 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 935 KiB/s wr, 5 op/s 2024-08-17T17:06:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:57 smithi162 ceph-mon[104668]: Upgrade: Updating osd.2 2024-08-17T17:06:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:57 smithi162 ceph-mon[104668]: Deploying daemon osd.2 on smithi063 2024-08-17T17:06:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:57 smithi162 ceph-mon[104668]: osdmap e789: 8 total, 8 up, 8 in 2024-08-17T17:06:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:57 smithi162 ceph-mon[104668]: osdmap e790: 8 total, 8 up, 8 in 2024-08-17T17:06:58.643 INFO:tasks.rados.rados.0.smithi063.stdout:2856: done (5 left) 2024-08-17T17:06:58.643 INFO:tasks.rados.rados.0.smithi063.stdout:2857: write oid 7 current snap is 361 2024-08-17T17:06:58.643 INFO:tasks.rados.rados.0.smithi063.stdout:2857: seq_num 1069 ranges {766683=730601,1922192=583372,3156591=676370} 2024-08-17T17:06:58.653 INFO:tasks.rados.rados.0.smithi063.stdout:2857: writing smithi06387500-7 from 766683 to 1497284 tid 1 2024-08-17T17:06:58.660 INFO:tasks.rados.rados.0.smithi063.stdout:2857: writing smithi06387500-7 from 1922192 to 2505564 tid 2 2024-08-17T17:06:58.668 INFO:tasks.rados.rados.0.smithi063.stdout:2857: writing smithi06387500-7 from 3156591 to 3832961 tid 3 2024-08-17T17:06:58.668 INFO:tasks.rados.rados.0.smithi063.stdout:2858: read oid 30 snap -1 2024-08-17T17:06:58.668 INFO:tasks.rados.rados.0.smithi063.stdout:2858: expect deleted 2024-08-17T17:06:58.668 INFO:tasks.rados.rados.0.smithi063.stdout:2859: snap_create 2024-08-17T17:06:58.668 INFO:tasks.rados.rados.0.smithi063.stdout:2850: expect (ObjNum 1014 snap 346 seq_num 1014) 2024-08-17T17:06:58.771 INFO:tasks.rados.rados.0.smithi063.stdout:2851: expect (ObjNum 987 snap 339 seq_num 987) 2024-08-17T17:06:58.896 INFO:tasks.rados.rados.0.smithi063.stdout:2854: finishing write tid 1 to smithi06387500-35 2024-08-17T17:06:58.896 INFO:tasks.rados.rados.0.smithi063.stdout:2854: finishing write tid 2 to smithi06387500-35 2024-08-17T17:06:58.896 INFO:tasks.rados.rados.0.smithi063.stdout:2854: finishing write tid 3 to smithi06387500-35 2024-08-17T17:06:58.896 INFO:tasks.rados.rados.0.smithi063.stdout:2854: finishing write tid 4 to smithi06387500-35 2024-08-17T17:06:58.896 INFO:tasks.rados.rados.0.smithi063.stdout:2854: finishing write tid 5 to smithi06387500-35 2024-08-17T17:06:58.896 INFO:tasks.rados.rados.0.smithi063.stdout:2854: oid 35 updating version 0 to 2758 2024-08-17T17:06:58.896 INFO:tasks.rados.rados.0.smithi063.stdout:2854: oid 35 updating version 2758 to 2762 2024-08-17T17:06:58.897 INFO:tasks.rados.rados.0.smithi063.stdout:2854: oid 35 version 2762 is already newer than 2759 2024-08-17T17:06:58.897 INFO:tasks.rados.rados.0.smithi063.stdout:2854: oid 35 updating version 2762 to 2763 2024-08-17T17:06:58.897 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 2763 (ObjNum 1068 snap 361 seq_num 1068) dirty exists 2024-08-17T17:06:58.897 INFO:tasks.rados.rados.0.smithi063.stdout:2854: left oid 35 (ObjNum 1068 snap 361 seq_num 1068) 2024-08-17T17:06:58.897 INFO:tasks.rados.rados.0.smithi063.stdout:2857: finishing write tid 1 to smithi06387500-7 2024-08-17T17:06:58.897 INFO:tasks.rados.rados.0.smithi063.stdout:2857: finishing write tid 2 to smithi06387500-7 2024-08-17T17:06:58.897 INFO:tasks.rados.rados.0.smithi063.stdout:2857: finishing write tid 3 to smithi06387500-7 2024-08-17T17:06:58.897 INFO:tasks.rados.rados.0.smithi063.stdout:2857: finishing write tid 4 to smithi06387500-7 2024-08-17T17:06:58.897 INFO:tasks.rados.rados.0.smithi063.stdout:2857: finishing write tid 5 to smithi06387500-7 2024-08-17T17:06:58.897 INFO:tasks.rados.rados.0.smithi063.stdout:2857: oid 7 updating version 0 to 1336 2024-08-17T17:06:58.897 INFO:tasks.rados.rados.0.smithi063.stdout:2857: oid 7 updating version 1336 to 1343 2024-08-17T17:06:58.897 INFO:tasks.rados.rados.0.smithi063.stdout:2857: oid 7 version 1343 is already newer than 1341 2024-08-17T17:06:58.898 INFO:tasks.rados.rados.0.smithi063.stdout:2857: oid 7 version 1343 is already newer than 1342 2024-08-17T17:06:58.898 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1343 (ObjNum 1069 snap 361 seq_num 1069) dirty exists 2024-08-17T17:06:58.898 INFO:tasks.rados.rados.0.smithi063.stdout:2857: left oid 7 (ObjNum 1069 snap 361 seq_num 1069) 2024-08-17T17:06:59.529 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:06:59 smithi063 systemd[1]: Stopping Ceph osd.2 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:06:59.529 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:06:59 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2[59187]: 2024-08-17T17:06:59.419+0000 7f89ec0f0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:06:59.529 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:06:59 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2[59187]: 2024-08-17T17:06:59.419+0000 7f89ec0f0640 -1 osd.2 791 *** Got signal Terminated *** 2024-08-17T17:06:59.530 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:06:59 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2[59187]: 2024-08-17T17:06:59.419+0000 7f89ec0f0640 -1 osd.2 791 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:06:59.651 INFO:tasks.rados.rados.0.smithi063.stdout:2850: done (7 left) 2024-08-17T17:06:59.651 INFO:tasks.rados.rados.0.smithi063.stdout:2851: done (6 left) 2024-08-17T17:06:59.651 INFO:tasks.rados.rados.0.smithi063.stdout:2853: done (5 left) 2024-08-17T17:06:59.651 INFO:tasks.rados.rados.0.smithi063.stdout:2854: done (4 left) 2024-08-17T17:06:59.651 INFO:tasks.rados.rados.0.smithi063.stdout:2855: done (3 left) 2024-08-17T17:06:59.651 INFO:tasks.rados.rados.0.smithi063.stdout:2857: done (2 left) 2024-08-17T17:06:59.651 INFO:tasks.rados.rados.0.smithi063.stdout:2858: done (1 left) 2024-08-17T17:06:59.652 INFO:tasks.rados.rados.0.smithi063.stdout:2859: done (0 left) 2024-08-17T17:06:59.652 INFO:tasks.rados.rados.0.smithi063.stdout:2860: delete oid 12 current snap is 362 2024-08-17T17:07:00.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:59 smithi063 ceph-mon[109843]: pgmap v642: 89 pgs: 89 active+clean; 270 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 777 KiB/s wr, 18 op/s 2024-08-17T17:07:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:59 smithi063 ceph-mon[109843]: osdmap e791: 8 total, 8 up, 8 in 2024-08-17T17:07:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:06:59 smithi063 ceph-mon[109843]: osd.2 marked itself down and dead 2024-08-17T17:07:00.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:59 smithi063 ceph-mon[114756]: pgmap v642: 89 pgs: 89 active+clean; 270 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 777 KiB/s wr, 18 op/s 2024-08-17T17:07:00.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:59 smithi063 ceph-mon[114756]: osdmap e791: 8 total, 8 up, 8 in 2024-08-17T17:07:00.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:06:59 smithi063 ceph-mon[114756]: osd.2 marked itself down and dead 2024-08-17T17:07:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:59 smithi162 ceph-mon[104668]: pgmap v642: 89 pgs: 89 active+clean; 270 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 777 KiB/s wr, 18 op/s 2024-08-17T17:07:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:59 smithi162 ceph-mon[104668]: osdmap e791: 8 total, 8 up, 8 in 2024-08-17T17:07:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:06:59 smithi162 ceph-mon[104668]: osd.2 marked itself down and dead 2024-08-17T17:07:00.529 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:00 smithi063 podman[137058]: 2024-08-17 17:07:00.186410228 +0000 UTC m=+0.856773065 container died a3cbe8edb49ae7bfae2a98cacadec20f6cf4cb6775aa161d7d5ac553898255ca (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, RELEASE=reef-ced649f, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-08-17T17:07:00.672 INFO:tasks.rados.rados.0.smithi063.stdout:2860: done (0 left) 2024-08-17T17:07:00.672 INFO:tasks.rados.rados.0.smithi063.stdout:2861: write oid 33 current snap is 362 2024-08-17T17:07:00.672 INFO:tasks.rados.rados.0.smithi063.stdout:2861: seq_num 1071 ranges {400297=481099,1639156=361141} 2024-08-17T17:07:00.677 INFO:tasks.rados.rados.0.smithi063.stdout:2861: writing smithi06387500-33 from 400297 to 881396 tid 1 2024-08-17T17:07:00.683 INFO:tasks.rados.rados.0.smithi063.stdout:2861: writing smithi06387500-33 from 1639156 to 2000297 tid 2 2024-08-17T17:07:00.684 INFO:tasks.rados.rados.0.smithi063.stdout:2862: read oid 21 snap -1 2024-08-17T17:07:00.684 INFO:tasks.rados.rados.0.smithi063.stdout:2862: expect (ObjNum 938 snap 321 seq_num 938) 2024-08-17T17:07:00.684 INFO:tasks.rados.rados.0.smithi063.stdout:2863: snap_remove snap 361 2024-08-17T17:07:00.986 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:00 smithi063 ceph-mon[114756]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:07:00.986 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:00 smithi063 ceph-mon[114756]: osdmap e792: 8 total, 7 up, 8 in 2024-08-17T17:07:00.986 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:00 smithi063 ceph-mon[109843]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:07:00.986 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:00 smithi063 ceph-mon[109843]: osdmap e792: 8 total, 7 up, 8 in 2024-08-17T17:07:00.987 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:00 smithi063 podman[137058]: 2024-08-17 17:07:00.692707973 +0000 UTC m=+1.363070812 container cleanup a3cbe8edb49ae7bfae2a98cacadec20f6cf4cb6775aa161d7d5ac553898255ca (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, ceph=True, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-08-17T17:07:00.987 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:00 smithi063 bash[137058]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2 2024-08-17T17:07:00.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:00 smithi162 ceph-mon[104668]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:07:00.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:00 smithi162 ceph-mon[104668]: osdmap e792: 8 total, 7 up, 8 in 2024-08-17T17:07:01.279 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:00 smithi063 podman[137079]: 2024-08-17 17:07:00.985811464 +0000 UTC m=+0.792675076 container remove a3cbe8edb49ae7bfae2a98cacadec20f6cf4cb6775aa161d7d5ac553898255ca (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-ced649f, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:07:01.631 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:01 smithi063 podman[137172]: 2024-08-17 17:07:01.230723619 +0000 UTC m=+0.017926127 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:07:01.631 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:01 smithi063 podman[137172]: 2024-08-17 17:07:01.353582518 +0000 UTC m=+0.140785017 container create 61fa862008ac2586508ccbe075d02993ccf125e0af36c572cdd163ada8ea52de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-08-17T17:07:01.631 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:01 smithi063 podman[137172]: 2024-08-17 17:07:01.55648864 +0000 UTC m=+0.343691164 container init 61fa862008ac2586508ccbe075d02993ccf125e0af36c572cdd163ada8ea52de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=) 2024-08-17T17:07:01.631 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:01 smithi063 podman[137172]: 2024-08-17 17:07:01.561687566 +0000 UTC m=+0.348890065 container start 61fa862008ac2586508ccbe075d02993ccf125e0af36c572cdd163ada8ea52de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-08-17T17:07:01.665 INFO:tasks.rados.rados.0.smithi063.stdout:2863: done (2 left) 2024-08-17T17:07:01.665 INFO:tasks.rados.rados.0.smithi063.stdout:2864: read oid 38 snap -1 2024-08-17T17:07:01.665 INFO:tasks.rados.rados.0.smithi063.stdout:2864: expect (ObjNum 868 snap 300 seq_num 868) 2024-08-17T17:07:01.665 INFO:tasks.rados.rados.0.smithi063.stdout:2865: write oid 27 current snap is 362 2024-08-17T17:07:01.665 INFO:tasks.rados.rados.0.smithi063.stdout:2861: finishing write tid 1 to smithi06387500-33 2024-08-17T17:07:01.666 INFO:tasks.rados.rados.0.smithi063.stdout:2865: seq_num 1072 ranges {417104=672701,1813991=203113} 2024-08-17T17:07:01.671 INFO:tasks.rados.rados.0.smithi063.stdout:2865: writing smithi06387500-27 from 417104 to 1089805 tid 1 2024-08-17T17:07:01.677 INFO:tasks.rados.rados.0.smithi063.stdout:2865: writing smithi06387500-27 from 1813991 to 2017104 tid 2 2024-08-17T17:07:01.677 INFO:tasks.rados.rados.0.smithi063.stdout:2866: read oid 49 snap 354 2024-08-17T17:07:01.677 INFO:tasks.rados.rados.0.smithi063.stdout:2866: expect (ObjNum 896 snap 310 seq_num 896) 2024-08-17T17:07:01.677 INFO:tasks.rados.rados.0.smithi063.stdout:2867: read oid 19 snap -1 2024-08-17T17:07:01.677 INFO:tasks.rados.rados.0.smithi063.stdout:2867: expect deleted 2024-08-17T17:07:01.677 INFO:tasks.rados.rados.0.smithi063.stdout:2861: finishing write tid 2 to smithi06387500-33 2024-08-17T17:07:01.678 INFO:tasks.rados.rados.0.smithi063.stdout:2861: finishing write tid 3 to smithi06387500-33 2024-08-17T17:07:01.678 INFO:tasks.rados.rados.0.smithi063.stdout:2861: finishing write tid 4 to smithi06387500-33 2024-08-17T17:07:01.678 INFO:tasks.rados.rados.0.smithi063.stdout:2861: oid 33 updating version 0 to 978 2024-08-17T17:07:01.678 INFO:tasks.rados.rados.0.smithi063.stdout:2861: oid 33 version 978 is already newer than 976 2024-08-17T17:07:01.678 INFO:tasks.rados.rados.0.smithi063.stdout:2861: oid 33 version 978 is already newer than 977 2024-08-17T17:07:01.678 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 978 (ObjNum 1071 snap 362 seq_num 1071) dirty exists 2024-08-17T17:07:01.678 INFO:tasks.rados.rados.0.smithi063.stdout:2861: left oid 33 (ObjNum 1071 snap 362 seq_num 1071) 2024-08-17T17:07:01.678 INFO:tasks.rados.rados.0.smithi063.stdout:2862: expect (ObjNum 938 snap 321 seq_num 938) 2024-08-17T17:07:01.858 INFO:tasks.rados.rados.0.smithi063.stdout:2864: expect (ObjNum 868 snap 300 seq_num 868) 2024-08-17T17:07:01.979 INFO:tasks.rados.rados.0.smithi063.stdout:2865: finishing write tid 1 to smithi06387500-27 2024-08-17T17:07:01.980 INFO:tasks.rados.rados.0.smithi063.stdout:2865: finishing write tid 2 to smithi06387500-27 2024-08-17T17:07:01.980 INFO:tasks.rados.rados.0.smithi063.stdout:2865: finishing write tid 3 to smithi06387500-27 2024-08-17T17:07:01.980 INFO:tasks.rados.rados.0.smithi063.stdout:2865: finishing write tid 4 to smithi06387500-27 2024-08-17T17:07:01.980 INFO:tasks.rados.rados.0.smithi063.stdout:2865: oid 27 updating version 0 to 262 2024-08-17T17:07:01.980 INFO:tasks.rados.rados.0.smithi063.stdout:2865: oid 27 version 262 is already newer than 261 2024-08-17T17:07:01.980 INFO:tasks.rados.rados.0.smithi063.stdout:2865: oid 27 updating version 262 to 263 2024-08-17T17:07:01.980 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 263 (ObjNum 1072 snap 362 seq_num 1072) dirty exists 2024-08-17T17:07:01.980 INFO:tasks.rados.rados.0.smithi063.stdout:2865: left oid 27 (ObjNum 1072 snap 362 seq_num 1072) 2024-08-17T17:07:01.980 INFO:tasks.rados.rados.0.smithi063.stdout:2861: done (5 left) 2024-08-17T17:07:01.980 INFO:tasks.rados.rados.0.smithi063.stdout:2862: done (4 left) 2024-08-17T17:07:01.981 INFO:tasks.rados.rados.0.smithi063.stdout:2864: done (3 left) 2024-08-17T17:07:01.981 INFO:tasks.rados.rados.0.smithi063.stdout:2865: done (2 left) 2024-08-17T17:07:01.981 INFO:tasks.rados.rados.0.smithi063.stdout:2868: rollback oid 47 current snap is 362 2024-08-17T17:07:01.981 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 47 to 356 2024-08-17T17:07:01.981 INFO:tasks.rados.rados.0.smithi063.stdout:2866: expect (ObjNum 896 snap 310 seq_num 896) 2024-08-17T17:07:02.013 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:01 smithi162 ceph-mon[104668]: pgmap v645: 89 pgs: 8 stale+active+clean, 81 active+clean; 270 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 777 KiB/s wr, 17 op/s 2024-08-17T17:07:02.013 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:01 smithi162 ceph-mon[104668]: osdmap e793: 8 total, 7 up, 8 in 2024-08-17T17:07:02.013 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:02.013 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:01 smithi063 ceph-mon[114756]: pgmap v645: 89 pgs: 8 stale+active+clean, 81 active+clean; 270 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 777 KiB/s wr, 17 op/s 2024-08-17T17:07:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:01 smithi063 ceph-mon[114756]: osdmap e793: 8 total, 7 up, 8 in 2024-08-17T17:07:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:01 smithi063 ceph-mon[109843]: pgmap v645: 89 pgs: 8 stale+active+clean, 81 active+clean; 270 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 777 KiB/s wr, 17 op/s 2024-08-17T17:07:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:01 smithi063 ceph-mon[109843]: osdmap e793: 8 total, 7 up, 8 in 2024-08-17T17:07:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:02.030 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:01 smithi063 podman[137172]: 2024-08-17 17:07:01.630728505 +0000 UTC m=+0.417931005 container attach 61fa862008ac2586508ccbe075d02993ccf125e0af36c572cdd163ada8ea52de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-08-17T17:07:02.030 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:01 smithi063 podman[137172]: 2024-08-17 17:07:01.763194247 +0000 UTC m=+0.550396751 container died 61fa862008ac2586508ccbe075d02993ccf125e0af36c572cdd163ada8ea52de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-08-17T17:07:02.126 INFO:tasks.rados.rados.0.smithi063.stdout:2868: finishing rollback tid 0 to smithi06387500-47 2024-08-17T17:07:02.126 INFO:tasks.rados.rados.0.smithi063.stdout:2868: finishing rollback tid 1 to smithi06387500-47 2024-08-17T17:07:02.126 INFO:tasks.rados.rados.0.smithi063.stdout:2868: finishing rollback tid 2 to smithi06387500-47 2024-08-17T17:07:02.126 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 586 (ObjNum 1014 snap 346 seq_num 1014) dirty exists 2024-08-17T17:07:02.126 INFO:tasks.rados.rados.0.smithi063.stdout:2866: done (2 left) 2024-08-17T17:07:02.126 INFO:tasks.rados.rados.0.smithi063.stdout:2867: done (1 left) 2024-08-17T17:07:02.126 INFO:tasks.rados.rados.0.smithi063.stdout:2868: done (0 left) 2024-08-17T17:07:02.126 INFO:tasks.rados.rados.0.smithi063.stdout:2869: write oid 32 current snap is 362 2024-08-17T17:07:02.127 INFO:tasks.rados.rados.0.smithi063.stdout:2869: seq_num 1073 ranges {433911=747950,1756120=277791} 2024-08-17T17:07:02.132 INFO:tasks.rados.rados.0.smithi063.stdout:2869: writing smithi06387500-32 from 433911 to 1181861 tid 1 2024-08-17T17:07:02.137 INFO:tasks.rados.rados.0.smithi063.stdout:2869: writing smithi06387500-32 from 1756120 to 2033911 tid 2 2024-08-17T17:07:02.137 INFO:tasks.rados.rados.0.smithi063.stdout:2870: read oid 22 snap -1 2024-08-17T17:07:02.137 INFO:tasks.rados.rados.0.smithi063.stdout:2870: expect (ObjNum 1028 snap 350 seq_num 1028) 2024-08-17T17:07:02.137 INFO:tasks.rados.rados.0.smithi063.stdout:2871: delete oid 28 current snap is 362 2024-08-17T17:07:02.137 INFO:tasks.rados.rados.0.smithi063.stdout:2869: finishing write tid 1 to smithi06387500-32 2024-08-17T17:07:02.139 INFO:tasks.rados.rados.0.smithi063.stdout:2869: finishing write tid 2 to smithi06387500-32 2024-08-17T17:07:02.139 INFO:tasks.rados.rados.0.smithi063.stdout:2869: finishing write tid 3 to smithi06387500-32 2024-08-17T17:07:02.139 INFO:tasks.rados.rados.0.smithi063.stdout:2869: finishing write tid 4 to smithi06387500-32 2024-08-17T17:07:02.139 INFO:tasks.rados.rados.0.smithi063.stdout:2869: oid 32 updating version 0 to 589 2024-08-17T17:07:02.139 INFO:tasks.rados.rados.0.smithi063.stdout:2869: oid 32 version 589 is already newer than 588 2024-08-17T17:07:02.139 INFO:tasks.rados.rados.0.smithi063.stdout:2869: oid 32 updating version 589 to 590 2024-08-17T17:07:02.140 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 590 (ObjNum 1073 snap 362 seq_num 1073) dirty exists 2024-08-17T17:07:02.140 INFO:tasks.rados.rados.0.smithi063.stdout:2869: left oid 32 (ObjNum 1073 snap 362 seq_num 1073) 2024-08-17T17:07:02.142 INFO:tasks.rados.rados.0.smithi063.stdout:2870: expect (ObjNum 1028 snap 350 seq_num 1028) 2024-08-17T17:07:02.333 INFO:tasks.rados.rados.0.smithi063.stdout:2869: done (2 left) 2024-08-17T17:07:02.333 INFO:tasks.rados.rados.0.smithi063.stdout:2870: done (1 left) 2024-08-17T17:07:02.333 INFO:tasks.rados.rados.0.smithi063.stdout:2871: done (0 left) 2024-08-17T17:07:02.333 INFO:tasks.rados.rados.0.smithi063.stdout:2872: rollback oid 38 current snap is 362 2024-08-17T17:07:02.333 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 38 to 354 2024-08-17T17:07:02.334 INFO:tasks.rados.rados.0.smithi063.stdout:2873: write oid 1 current snap is 362 2024-08-17T17:07:02.334 INFO:tasks.rados.rados.0.smithi063.stdout:2873: seq_num 1075 ranges {467525=498448,1473084=594379} 2024-08-17T17:07:02.338 INFO:tasks.rados.rados.0.smithi063.stdout:2873: writing smithi06387500-1 from 467525 to 965973 tid 1 2024-08-17T17:07:02.345 INFO:tasks.rados.rados.0.smithi063.stdout:2873: writing smithi06387500-1 from 1473084 to 2067463 tid 2 2024-08-17T17:07:02.345 INFO:tasks.rados.rados.0.smithi063.stdout:2874: snap_remove snap 337 2024-08-17T17:07:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2874: done (2 left) 2024-08-17T17:07:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2875: delete oid 32 current snap is 362 2024-08-17T17:07:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2872: finishing rollback tid 0 to smithi06387500-38 2024-08-17T17:07:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2872: finishing rollback tid 1 to smithi06387500-38 2024-08-17T17:07:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2872: finishing rollback tid 2 to smithi06387500-38 2024-08-17T17:07:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 459 (ObjNum 1002 snap 343 seq_num 1002) dirty exists 2024-08-17T17:07:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2873: finishing write tid 1 to smithi06387500-1 2024-08-17T17:07:02.617 INFO:tasks.rados.rados.0.smithi063.stdout:2873: finishing write tid 2 to smithi06387500-1 2024-08-17T17:07:02.618 INFO:tasks.rados.rados.0.smithi063.stdout:2873: finishing write tid 3 to smithi06387500-1 2024-08-17T17:07:02.618 INFO:tasks.rados.rados.0.smithi063.stdout:2873: finishing write tid 4 to smithi06387500-1 2024-08-17T17:07:02.618 INFO:tasks.rados.rados.0.smithi063.stdout:2873: oid 1 updating version 0 to 1412 2024-08-17T17:07:02.618 INFO:tasks.rados.rados.0.smithi063.stdout:2873: oid 1 updating version 1412 to 1413 2024-08-17T17:07:02.618 INFO:tasks.rados.rados.0.smithi063.stdout:2873: oid 1 updating version 1413 to 1414 2024-08-17T17:07:02.618 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 1414 (ObjNum 1075 snap 362 seq_num 1075) dirty exists 2024-08-17T17:07:02.618 INFO:tasks.rados.rados.0.smithi063.stdout:2873: left oid 1 (ObjNum 1075 snap 362 seq_num 1075) 2024-08-17T17:07:02.625 INFO:tasks.rados.rados.0.smithi063.stdout:2872: done (2 left) 2024-08-17T17:07:02.625 INFO:tasks.rados.rados.0.smithi063.stdout:2873: done (1 left) 2024-08-17T17:07:02.625 INFO:tasks.rados.rados.0.smithi063.stdout:2875: done (0 left) 2024-08-17T17:07:02.625 INFO:tasks.rados.rados.0.smithi063.stdout:2876: snap_remove snap 357 2024-08-17T17:07:02.665 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:02 smithi063 podman[137215]: 2024-08-17 17:07:02.335050653 +0000 UTC m=+0.562063803 container remove 61fa862008ac2586508ccbe075d02993ccf125e0af36c572cdd163ada8ea52de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, GIT_CLEAN=True) 2024-08-17T17:07:02.665 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:02 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.2.service: Deactivated successfully. 2024-08-17T17:07:02.666 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:02 smithi063 systemd[1]: Stopped Ceph osd.2 for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:07:02.666 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:02 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.2.service: Consumed 17.136s CPU time. 2024-08-17T17:07:02.666 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:02 smithi063 systemd[1]: Starting Ceph osd.2 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:07:02.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:02 smithi063 ceph-mon[114756]: osdmap e794: 8 total, 7 up, 8 in 2024-08-17T17:07:02.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:02 smithi063 ceph-mon[114756]: pgmap v648: 89 pgs: 11 active+undersized, 1 stale+active+clean, 9 active+undersized+degraded, 68 active+clean; 221 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.2 MiB/s wr, 10 op/s; 31/414 objects degraded (7.488%) 2024-08-17T17:07:02.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:02 smithi063 ceph-mon[114756]: osdmap e795: 8 total, 7 up, 8 in 2024-08-17T17:07:02.923 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:02 smithi063 ceph-mon[109843]: osdmap e794: 8 total, 7 up, 8 in 2024-08-17T17:07:02.923 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:02 smithi063 ceph-mon[109843]: pgmap v648: 89 pgs: 11 active+undersized, 1 stale+active+clean, 9 active+undersized+degraded, 68 active+clean; 221 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.2 MiB/s wr, 10 op/s; 31/414 objects degraded (7.488%) 2024-08-17T17:07:02.923 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:02 smithi063 ceph-mon[109843]: osdmap e795: 8 total, 7 up, 8 in 2024-08-17T17:07:02.923 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:02 smithi063 podman[137319]: 2024-08-17 17:07:02.68811277 +0000 UTC m=+0.018085673 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:07:02.923 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:02 smithi063 podman[137319]: 2024-08-17 17:07:02.820089674 +0000 UTC m=+0.150062607 container create d5dccf3ffdebf7926afb5c8d932b04477c8f00841b91ad683e374ba1eed91824 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.36.0) 2024-08-17T17:07:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:02 smithi162 ceph-mon[104668]: osdmap e794: 8 total, 7 up, 8 in 2024-08-17T17:07:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:02 smithi162 ceph-mon[104668]: pgmap v648: 89 pgs: 11 active+undersized, 1 stale+active+clean, 9 active+undersized+degraded, 68 active+clean; 221 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.2 MiB/s wr, 10 op/s; 31/414 objects degraded (7.488%) 2024-08-17T17:07:03.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:02 smithi162 ceph-mon[104668]: osdmap e795: 8 total, 7 up, 8 in 2024-08-17T17:07:03.179 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 podman[137319]: 2024-08-17 17:07:03.022539004 +0000 UTC m=+0.352511907 container init d5dccf3ffdebf7926afb5c8d932b04477c8f00841b91ad683e374ba1eed91824 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS) 2024-08-17T17:07:03.179 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 podman[137319]: 2024-08-17 17:07:03.027738486 +0000 UTC m=+0.357711386 container start d5dccf3ffdebf7926afb5c8d932b04477c8f00841b91ad683e374ba1eed91824 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-08-17T17:07:03.179 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 podman[137319]: 2024-08-17 17:07:03.114145583 +0000 UTC m=+0.444118487 container attach d5dccf3ffdebf7926afb5c8d932b04477c8f00841b91ad683e374ba1eed91824 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0) 2024-08-17T17:07:03.179 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate[137334]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:03.179 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 bash[137319]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:03.179 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate[137334]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:03.179 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 bash[137319]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:03.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:07:03] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.51.0" 2024-08-17T17:07:03.530 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate[137334]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:07:03.530 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 bash[137319]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:07:03.530 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate[137334]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:03.530 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 bash[137319]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:03.530 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate[137334]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:03.530 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 bash[137319]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:03.530 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate[137334]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:07:03.530 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 bash[137319]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:07:03.530 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate[137334]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-08-17T17:07:03.530 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 bash[137319]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2024-08-17T17:07:03.617 INFO:tasks.rados.rados.0.smithi063.stdout:2876: done (0 left) 2024-08-17T17:07:03.618 INFO:tasks.rados.rados.0.smithi063.stdout:2877: delete oid 43 current snap is 362 2024-08-17T17:07:03.626 INFO:tasks.rados.rados.0.smithi063.stdout:2877: done (0 left) 2024-08-17T17:07:03.626 INFO:tasks.rados.rados.0.smithi063.stdout:2878: write oid 30 current snap is 362 2024-08-17T17:07:03.626 INFO:tasks.rados.rados.0.smithi063.stdout:2878: seq_num 1078 ranges {517946=724195,1932177=185769} 2024-08-17T17:07:03.629 INFO:tasks.rados.rados.0.smithi063.stdout:2878: writing smithi06387500-30 from 517946 to 1242141 tid 1 2024-08-17T17:07:03.635 INFO:tasks.rados.rados.0.smithi063.stdout:2878: writing smithi06387500-30 from 1932177 to 2117946 tid 2 2024-08-17T17:07:03.635 INFO:tasks.rados.rados.0.smithi063.stdout:2879: write oid 26 current snap is 362 2024-08-17T17:07:03.635 INFO:tasks.rados.rados.0.smithi063.stdout:2879: seq_num 1079 ranges {534753=799444,1990659=144094} 2024-08-17T17:07:03.642 INFO:tasks.rados.rados.0.smithi063.stdout:2879: writing smithi06387500-26 from 534753 to 1334197 tid 1 2024-08-17T17:07:03.647 INFO:tasks.rados.rados.0.smithi063.stdout:2879: writing smithi06387500-26 from 1990659 to 2134753 tid 2 2024-08-17T17:07:03.647 INFO:tasks.rados.rados.0.smithi063.stdout:2880: snap_create 2024-08-17T17:07:03.647 INFO:tasks.rados.rados.0.smithi063.stdout:2878: finishing write tid 1 to smithi06387500-30 2024-08-17T17:07:03.647 INFO:tasks.rados.rados.0.smithi063.stdout:2878: finishing write tid 2 to smithi06387500-30 2024-08-17T17:07:03.647 INFO:tasks.rados.rados.0.smithi063.stdout:2878: finishing write tid 3 to smithi06387500-30 2024-08-17T17:07:03.647 INFO:tasks.rados.rados.0.smithi063.stdout:2878: finishing write tid 4 to smithi06387500-30 2024-08-17T17:07:03.648 INFO:tasks.rados.rados.0.smithi063.stdout:2878: oid 30 updating version 0 to 1356 2024-08-17T17:07:03.648 INFO:tasks.rados.rados.0.smithi063.stdout:2878: oid 30 version 1356 is already newer than 1355 2024-08-17T17:07:03.648 INFO:tasks.rados.rados.0.smithi063.stdout:2878: oid 30 updating version 1356 to 1357 2024-08-17T17:07:03.648 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1357 (ObjNum 1078 snap 362 seq_num 1078) dirty exists 2024-08-17T17:07:03.648 INFO:tasks.rados.rados.0.smithi063.stdout:2878: left oid 30 (ObjNum 1078 snap 362 seq_num 1078) 2024-08-17T17:07:03.648 INFO:tasks.rados.rados.0.smithi063.stdout:2879: finishing write tid 1 to smithi06387500-26 2024-08-17T17:07:03.649 INFO:tasks.rados.rados.0.smithi063.stdout:2879: finishing write tid 2 to smithi06387500-26 2024-08-17T17:07:03.649 INFO:tasks.rados.rados.0.smithi063.stdout:2879: finishing write tid 3 to smithi06387500-26 2024-08-17T17:07:03.649 INFO:tasks.rados.rados.0.smithi063.stdout:2879: finishing write tid 4 to smithi06387500-26 2024-08-17T17:07:03.650 INFO:tasks.rados.rados.0.smithi063.stdout:2879: oid 26 updating version 0 to 967 2024-08-17T17:07:03.650 INFO:tasks.rados.rados.0.smithi063.stdout:2879: oid 26 version 967 is already newer than 966 2024-08-17T17:07:03.650 INFO:tasks.rados.rados.0.smithi063.stdout:2879: oid 26 updating version 967 to 968 2024-08-17T17:07:03.650 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 968 (ObjNum 1079 snap 362 seq_num 1079) dirty exists 2024-08-17T17:07:03.650 INFO:tasks.rados.rados.0.smithi063.stdout:2879: left oid 26 (ObjNum 1079 snap 362 seq_num 1079) 2024-08-17T17:07:03.919 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-mon[114756]: Health check failed: Degraded data redundancy: 31/414 objects degraded (7.488%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:03.919 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-mon[114756]: osdmap e796: 8 total, 7 up, 8 in 2024-08-17T17:07:03.920 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-mon[109843]: Health check failed: Degraded data redundancy: 31/414 objects degraded (7.488%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:03.920 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-mon[109843]: osdmap e796: 8 total, 7 up, 8 in 2024-08-17T17:07:03.920 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate[137334]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:07:03.920 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 bash[137319]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:07:03.920 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate[137334]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:07:03.920 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 bash[137319]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2024-08-17T17:07:03.920 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate[137334]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T17:07:03.920 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 bash[137319]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T17:07:03.920 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate[137334]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:07:03.920 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 bash[137319]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-08-17T17:07:03.921 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate[137334]: --> ceph-volume lvm activate successful for osd ID: 2 2024-08-17T17:07:03.921 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 bash[137319]: --> ceph-volume lvm activate successful for osd ID: 2 2024-08-17T17:07:03.921 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:03 smithi063 podman[137319]: 2024-08-17 17:07:03.586275572 +0000 UTC m=+0.916248463 container died d5dccf3ffdebf7926afb5c8d932b04477c8f00841b91ad683e374ba1eed91824 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-08-17T17:07:04.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:03 smithi162 ceph-mon[104668]: Health check failed: Degraded data redundancy: 31/414 objects degraded (7.488%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:04.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:03 smithi162 ceph-mon[104668]: osdmap e796: 8 total, 7 up, 8 in 2024-08-17T17:07:04.495 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:04 smithi063 podman[137475]: 2024-08-17 17:07:04.227345415 +0000 UTC m=+0.631167543 container remove d5dccf3ffdebf7926afb5c8d932b04477c8f00841b91ad683e374ba1eed91824 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True) 2024-08-17T17:07:04.495 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:04 smithi063 podman[137541]: 2024-08-17 17:07:04.34902634 +0000 UTC m=+0.018018000 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:07:04.495 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:04 smithi063 podman[137541]: 2024-08-17 17:07:04.495035149 +0000 UTC m=+0.164026841 container create 107c38f8799493fe138746caff84da51f6436ba149bf9bbf1267a49fe5acd075 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:07:04.618 INFO:tasks.rados.rados.0.smithi063.stdout:2878: done (2 left) 2024-08-17T17:07:04.619 INFO:tasks.rados.rados.0.smithi063.stdout:2879: done (1 left) 2024-08-17T17:07:04.619 INFO:tasks.rados.rados.0.smithi063.stdout:2880: done (0 left) 2024-08-17T17:07:04.619 INFO:tasks.rados.rados.0.smithi063.stdout:2881: delete oid 34 current snap is 363 2024-08-17T17:07:04.623 INFO:tasks.rados.rados.0.smithi063.stdout:2881: done (0 left) 2024-08-17T17:07:04.623 INFO:tasks.rados.rados.0.smithi063.stdout:2882: read oid 43 snap -1 2024-08-17T17:07:04.623 INFO:tasks.rados.rados.0.smithi063.stdout:2882: expect deleted 2024-08-17T17:07:04.623 INFO:tasks.rados.rados.0.smithi063.stdout:2883: delete oid 36 current snap is 363 2024-08-17T17:07:04.628 INFO:tasks.rados.rados.0.smithi063.stdout:2882: done (1 left) 2024-08-17T17:07:04.628 INFO:tasks.rados.rados.0.smithi063.stdout:2883: done (0 left) 2024-08-17T17:07:04.628 INFO:tasks.rados.rados.0.smithi063.stdout:2884: rollback oid 17 current snap is 363 2024-08-17T17:07:04.628 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 17 to 354 2024-08-17T17:07:04.628 INFO:tasks.rados.rados.0.smithi063.stdout:2885: read oid 28 snap 353 2024-08-17T17:07:04.628 INFO:tasks.rados.rados.0.smithi063.stdout:2885: expect deleted 2024-08-17T17:07:04.628 INFO:tasks.rados.rados.0.smithi063.stdout:2886: rollback oid 31 current snap is 363 2024-08-17T17:07:04.628 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 31 to 362 2024-08-17T17:07:04.628 INFO:tasks.rados.rados.0.smithi063.stdout:2887: rollback oid 30 current snap is 363 2024-08-17T17:07:04.628 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 30 to 352 2024-08-17T17:07:04.628 INFO:tasks.rados.rados.0.smithi063.stdout:2888: write oid 42 current snap is 363 2024-08-17T17:07:04.628 INFO:tasks.rados.rados.0.smithi063.stdout:2888: seq_num 1082 ranges {585174=741544,1766105=419069} 2024-08-17T17:07:04.635 INFO:tasks.rados.rados.0.smithi063.stdout:2888: writing smithi06387500-42 from 585174 to 1326718 tid 1 2024-08-17T17:07:04.640 INFO:tasks.rados.rados.0.smithi063.stdout:2888: writing smithi06387500-42 from 1766105 to 2185174 tid 2 2024-08-17T17:07:04.640 INFO:tasks.rados.rados.0.smithi063.stdout:2889: read oid 18 snap -1 2024-08-17T17:07:04.640 INFO:tasks.rados.rados.0.smithi063.stdout:2889: expect deleted 2024-08-17T17:07:04.640 INFO:tasks.rados.rados.0.smithi063.stdout:2890: write oid 38 current snap is 363 2024-08-17T17:07:04.640 INFO:tasks.rados.rados.0.smithi063.stdout:2890: seq_num 1083 ranges {601981=416793,1424587=492702} 2024-08-17T17:07:04.646 INFO:tasks.rados.rados.0.smithi063.stdout:2890: writing smithi06387500-38 from 601981 to 1018774 tid 1 2024-08-17T17:07:04.651 INFO:tasks.rados.rados.0.smithi063.stdout:2890: writing smithi06387500-38 from 1424587 to 1917289 tid 2 2024-08-17T17:07:04.651 INFO:tasks.rados.rados.0.smithi063.stdout:2891: delete oid 27 current snap is 363 2024-08-17T17:07:04.651 INFO:tasks.rados.rados.0.smithi063.stdout:2884: finishing rollback tid 1 to smithi06387500-17 2024-08-17T17:07:04.651 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T17:07:04.651 INFO:tasks.rados.rados.0.smithi063.stdout:2886: finishing rollback tid 1 to smithi06387500-31 2024-08-17T17:07:04.651 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 0 (ObjNum 129 snap 0 seq_num 1806950624) dirty dne 2024-08-17T17:07:04.652 INFO:tasks.rados.rados.0.smithi063.stdout:2887: finishing rollback tid 0 to smithi06387500-30 2024-08-17T17:07:04.652 INFO:tasks.rados.rados.0.smithi063.stdout:2887: finishing rollback tid 1 to smithi06387500-30 2024-08-17T17:07:04.652 INFO:tasks.rados.rados.0.smithi063.stdout:2887: finishing rollback tid 2 to smithi06387500-30 2024-08-17T17:07:04.652 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1361 (ObjNum 1020 snap 348 seq_num 1020) dirty exists 2024-08-17T17:07:04.652 INFO:tasks.rados.rados.0.smithi063.stdout:2888: finishing write tid 1 to smithi06387500-42 2024-08-17T17:07:04.652 INFO:tasks.rados.rados.0.smithi063.stdout:2888: finishing write tid 2 to smithi06387500-42 2024-08-17T17:07:04.652 INFO:tasks.rados.rados.0.smithi063.stdout:2888: finishing write tid 3 to smithi06387500-42 2024-08-17T17:07:04.652 INFO:tasks.rados.rados.0.smithi063.stdout:2888: finishing write tid 4 to smithi06387500-42 2024-08-17T17:07:04.652 INFO:tasks.rados.rados.0.smithi063.stdout:2888: oid 42 updating version 0 to 1868 2024-08-17T17:07:04.652 INFO:tasks.rados.rados.0.smithi063.stdout:2888: oid 42 updating version 1868 to 1870 2024-08-17T17:07:04.652 INFO:tasks.rados.rados.0.smithi063.stdout:2888: oid 42 version 1870 is already newer than 1869 2024-08-17T17:07:04.652 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 1870 (ObjNum 1082 snap 363 seq_num 1082) dirty exists 2024-08-17T17:07:04.652 INFO:tasks.rados.rados.0.smithi063.stdout:2888: left oid 42 (ObjNum 1082 snap 363 seq_num 1082) 2024-08-17T17:07:04.653 INFO:tasks.rados.rados.0.smithi063.stdout:2890: finishing write tid 1 to smithi06387500-38 2024-08-17T17:07:04.653 INFO:tasks.rados.rados.0.smithi063.stdout:2884: done (7 left) 2024-08-17T17:07:04.653 INFO:tasks.rados.rados.0.smithi063.stdout:2885: done (6 left) 2024-08-17T17:07:04.653 INFO:tasks.rados.rados.0.smithi063.stdout:2886: done (5 left) 2024-08-17T17:07:04.653 INFO:tasks.rados.rados.0.smithi063.stdout:2887: done (4 left) 2024-08-17T17:07:04.653 INFO:tasks.rados.rados.0.smithi063.stdout:2888: done (3 left) 2024-08-17T17:07:04.653 INFO:tasks.rados.rados.0.smithi063.stdout:2889: done (2 left) 2024-08-17T17:07:04.654 INFO:tasks.rados.rados.0.smithi063.stdout:2891: done (1 left) 2024-08-17T17:07:04.654 INFO:tasks.rados.rados.0.smithi063.stdout:2892: write oid 22 current snap is 363 2024-08-17T17:07:04.654 INFO:tasks.rados.rados.0.smithi063.stdout:2892: seq_num 1085 ranges {635595=567291,1825198=410397} 2024-08-17T17:07:04.659 INFO:tasks.rados.rados.0.smithi063.stdout:2892: writing smithi06387500-22 from 635595 to 1202886 tid 1 2024-08-17T17:07:04.665 INFO:tasks.rados.rados.0.smithi063.stdout:2892: writing smithi06387500-22 from 1825198 to 2235595 tid 2 2024-08-17T17:07:04.665 INFO:tasks.rados.rados.0.smithi063.stdout:2893: read oid 3 snap -1 2024-08-17T17:07:04.665 INFO:tasks.rados.rados.0.smithi063.stdout:2893: expect (ObjNum 1060 snap 358 seq_num 1060) 2024-08-17T17:07:04.665 INFO:tasks.rados.rados.0.smithi063.stdout:2894: write oid 49 current snap is 363 2024-08-17T17:07:04.665 INFO:tasks.rados.rados.0.smithi063.stdout:2890: finishing write tid 2 to smithi06387500-38 2024-08-17T17:07:04.666 INFO:tasks.rados.rados.0.smithi063.stdout:2890: finishing write tid 3 to smithi06387500-38 2024-08-17T17:07:04.666 INFO:tasks.rados.rados.0.smithi063.stdout:2890: finishing write tid 4 to smithi06387500-38 2024-08-17T17:07:04.666 INFO:tasks.rados.rados.0.smithi063.stdout:2890: oid 38 updating version 0 to 462 2024-08-17T17:07:04.666 INFO:tasks.rados.rados.0.smithi063.stdout:2890: oid 38 version 462 is already newer than 461 2024-08-17T17:07:04.666 INFO:tasks.rados.rados.0.smithi063.stdout:2890: oid 38 updating version 462 to 463 2024-08-17T17:07:04.666 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 463 (ObjNum 1083 snap 363 seq_num 1083) dirty exists 2024-08-17T17:07:04.666 INFO:tasks.rados.rados.0.smithi063.stdout:2890: left oid 38 (ObjNum 1083 snap 363 seq_num 1083) 2024-08-17T17:07:04.666 INFO:tasks.rados.rados.0.smithi063.stdout:2892: finishing write tid 1 to smithi06387500-22 2024-08-17T17:07:04.666 INFO:tasks.rados.rados.0.smithi063.stdout:2894: seq_num 1086 ranges {652402=642540,1883680=368722} 2024-08-17T17:07:04.672 INFO:tasks.rados.rados.0.smithi063.stdout:2894: writing smithi06387500-49 from 652402 to 1294942 tid 1 2024-08-17T17:07:04.678 INFO:tasks.rados.rados.0.smithi063.stdout:2894: writing smithi06387500-49 from 1883680 to 2252402 tid 2 2024-08-17T17:07:04.679 INFO:tasks.rados.rados.0.smithi063.stdout:2890: done (3 left) 2024-08-17T17:07:04.679 INFO:tasks.rados.rados.0.smithi063.stdout:2895: snap_remove snap 362 2024-08-17T17:07:04.779 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:04 smithi063 podman[137541]: 2024-08-17 17:07:04.714343874 +0000 UTC m=+0.383335525 container init 107c38f8799493fe138746caff84da51f6436ba149bf9bbf1267a49fe5acd075 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-08-17T17:07:04.779 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:04 smithi063 podman[137541]: 2024-08-17 17:07:04.719444875 +0000 UTC m=+0.388436534 container start 107c38f8799493fe138746caff84da51f6436ba149bf9bbf1267a49fe5acd075 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:07:04.779 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:04 smithi063 ceph-osd[137579]: -- 172.21.15.63:0/1143502224 <== mon.0 v2:172.21.15.63:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55aae2fefa40 con 0x55aae2fcdc00 2024-08-17T17:07:05.279 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:04 smithi063 bash[137541]: 107c38f8799493fe138746caff84da51f6436ba149bf9bbf1267a49fe5acd075 2024-08-17T17:07:05.279 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:04 smithi063 systemd[1]: Started Ceph osd.2 for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:07:05.680 INFO:tasks.rados.rados.0.smithi063.stdout:2895: done (3 left) 2024-08-17T17:07:05.681 INFO:tasks.rados.rados.0.smithi063.stdout:2896: snap_remove snap 356 2024-08-17T17:07:05.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:05 smithi063 ceph-mon[109843]: pgmap v651: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 220 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.4 MiB/s wr, 11 op/s; 31/411 objects degraded (7.543%) 2024-08-17T17:07:05.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:05 smithi063 ceph-mon[109843]: osdmap e797: 8 total, 7 up, 8 in 2024-08-17T17:07:05.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:05.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:05.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:05.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:05 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:05 smithi063 ceph-mon[114756]: pgmap v651: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 220 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.4 MiB/s wr, 11 op/s; 31/411 objects degraded (7.543%) 2024-08-17T17:07:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:05 smithi063 ceph-mon[114756]: osdmap e797: 8 total, 7 up, 8 in 2024-08-17T17:07:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:05.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:05 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:05 smithi162 ceph-mon[104668]: pgmap v651: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 220 MiB data, 780 MiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 1.4 MiB/s wr, 11 op/s; 31/411 objects degraded (7.543%) 2024-08-17T17:07:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:05 smithi162 ceph-mon[104668]: osdmap e797: 8 total, 7 up, 8 in 2024-08-17T17:07:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:05 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:06.618 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:06 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2[137556]: 2024-08-17T17:07:06.338+0000 7f66e85fd640 -1 Falling back to public interface 2024-08-17T17:07:06.683 INFO:tasks.rados.rados.0.smithi063.stdout:2896: done (3 left) 2024-08-17T17:07:06.684 INFO:tasks.rados.rados.0.smithi063.stdout:2897: delete oid 13 current snap is 363 2024-08-17T17:07:06.684 INFO:tasks.rados.rados.0.smithi063.stdout:2892: finishing write tid 2 to smithi06387500-22 2024-08-17T17:07:06.684 INFO:tasks.rados.rados.0.smithi063.stdout:2892: finishing write tid 3 to smithi06387500-22 2024-08-17T17:07:06.684 INFO:tasks.rados.rados.0.smithi063.stdout:2892: finishing write tid 4 to smithi06387500-22 2024-08-17T17:07:06.684 INFO:tasks.rados.rados.0.smithi063.stdout:2892: oid 22 updating version 0 to 627 2024-08-17T17:07:06.684 INFO:tasks.rados.rados.0.smithi063.stdout:2892: oid 22 updating version 627 to 629 2024-08-17T17:07:06.684 INFO:tasks.rados.rados.0.smithi063.stdout:2892: oid 22 version 629 is already newer than 628 2024-08-17T17:07:06.684 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 629 (ObjNum 1085 snap 363 seq_num 1085) dirty exists 2024-08-17T17:07:06.684 INFO:tasks.rados.rados.0.smithi063.stdout:2892: left oid 22 (ObjNum 1085 snap 363 seq_num 1085) 2024-08-17T17:07:06.684 INFO:tasks.rados.rados.0.smithi063.stdout:2893: expect (ObjNum 1060 snap 358 seq_num 1060) 2024-08-17T17:07:06.901 INFO:tasks.rados.rados.0.smithi063.stdout:2894: finishing write tid 1 to smithi06387500-49 2024-08-17T17:07:06.901 INFO:tasks.rados.rados.0.smithi063.stdout:2894: finishing write tid 2 to smithi06387500-49 2024-08-17T17:07:06.901 INFO:tasks.rados.rados.0.smithi063.stdout:2894: finishing write tid 3 to smithi06387500-49 2024-08-17T17:07:06.901 INFO:tasks.rados.rados.0.smithi063.stdout:2894: finishing write tid 4 to smithi06387500-49 2024-08-17T17:07:06.901 INFO:tasks.rados.rados.0.smithi063.stdout:2894: oid 49 updating version 0 to 2787 2024-08-17T17:07:06.902 INFO:tasks.rados.rados.0.smithi063.stdout:2894: oid 49 updating version 2787 to 2788 2024-08-17T17:07:06.902 INFO:tasks.rados.rados.0.smithi063.stdout:2894: oid 49 updating version 2788 to 2789 2024-08-17T17:07:06.902 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 2789 (ObjNum 1086 snap 363 seq_num 1086) dirty exists 2024-08-17T17:07:06.902 INFO:tasks.rados.rados.0.smithi063.stdout:2894: left oid 49 (ObjNum 1086 snap 363 seq_num 1086) 2024-08-17T17:07:06.902 INFO:tasks.rados.rados.0.smithi063.stdout:2892: done (3 left) 2024-08-17T17:07:06.902 INFO:tasks.rados.rados.0.smithi063.stdout:2893: done (2 left) 2024-08-17T17:07:06.902 INFO:tasks.rados.rados.0.smithi063.stdout:2894: done (1 left) 2024-08-17T17:07:06.902 INFO:tasks.rados.rados.0.smithi063.stdout:2897: done (0 left) 2024-08-17T17:07:06.902 INFO:tasks.rados.rados.0.smithi063.stdout:2898: read oid 32 snap -1 2024-08-17T17:07:06.902 INFO:tasks.rados.rados.0.smithi063.stdout:2898: expect deleted 2024-08-17T17:07:06.902 INFO:tasks.rados.rados.0.smithi063.stdout:2899: read oid 12 snap -1 2024-08-17T17:07:06.902 INFO:tasks.rados.rados.0.smithi063.stdout:2899: expect deleted 2024-08-17T17:07:06.903 INFO:tasks.rados.rados.0.smithi063.stdout:2900: read oid 39 snap -1 2024-08-17T17:07:06.903 INFO:tasks.rados.rados.0.smithi063.stdout:2900: expect (ObjNum 1037 snap 352 seq_num 1037) 2024-08-17T17:07:06.903 INFO:tasks.rados.rados.0.smithi063.stdout:2901: write oid 50 current snap is 363 2024-08-17T17:07:06.903 INFO:tasks.rados.rados.0.smithi063.stdout:2901: seq_num 1088 ranges {686016=509391,1716997=569019} 2024-08-17T17:07:06.907 INFO:tasks.rados.rados.0.smithi063.stdout:2901: writing smithi06387500-50 from 686016 to 1195407 tid 1 2024-08-17T17:07:06.915 INFO:tasks.rados.rados.0.smithi063.stdout:2901: writing smithi06387500-50 from 1716997 to 2286016 tid 2 2024-08-17T17:07:06.915 INFO:tasks.rados.rados.0.smithi063.stdout:2902: delete oid 11 current snap is 363 2024-08-17T17:07:06.915 INFO:tasks.rados.rados.0.smithi063.stdout:2901: finishing write tid 1 to smithi06387500-50 2024-08-17T17:07:06.916 INFO:tasks.rados.rados.0.smithi063.stdout:2900: expect (ObjNum 1037 snap 352 seq_num 1037) 2024-08-17T17:07:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:06 smithi063 ceph-mon[114756]: osdmap e798: 8 total, 7 up, 8 in 2024-08-17T17:07:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:06 smithi063 ceph-mon[114756]: pgmap v654: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 222 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 728 KiB/s wr, 5 op/s; 32/420 objects degraded (7.619%) 2024-08-17T17:07:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:06 smithi063 ceph-mon[109843]: osdmap e798: 8 total, 7 up, 8 in 2024-08-17T17:07:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:06 smithi063 ceph-mon[109843]: pgmap v654: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 222 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 728 KiB/s wr, 5 op/s; 32/420 objects degraded (7.619%) 2024-08-17T17:07:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:06 smithi162 ceph-mon[104668]: osdmap e798: 8 total, 7 up, 8 in 2024-08-17T17:07:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:06 smithi162 ceph-mon[104668]: pgmap v654: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 222 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 728 KiB/s wr, 5 op/s; 32/420 objects degraded (7.619%) 2024-08-17T17:07:07.199 INFO:tasks.rados.rados.0.smithi063.stdout:2901: finishing write tid 2 to smithi06387500-50 2024-08-17T17:07:07.199 INFO:tasks.rados.rados.0.smithi063.stdout:2901: finishing write tid 3 to smithi06387500-50 2024-08-17T17:07:07.199 INFO:tasks.rados.rados.0.smithi063.stdout:2901: finishing write tid 4 to smithi06387500-50 2024-08-17T17:07:07.199 INFO:tasks.rados.rados.0.smithi063.stdout:2901: oid 50 updating version 0 to 1878 2024-08-17T17:07:07.200 INFO:tasks.rados.rados.0.smithi063.stdout:2901: oid 50 updating version 1878 to 1880 2024-08-17T17:07:07.200 INFO:tasks.rados.rados.0.smithi063.stdout:2901: oid 50 version 1880 is already newer than 1879 2024-08-17T17:07:07.200 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 1880 (ObjNum 1088 snap 363 seq_num 1088) dirty exists 2024-08-17T17:07:07.200 INFO:tasks.rados.rados.0.smithi063.stdout:2901: left oid 50 (ObjNum 1088 snap 363 seq_num 1088) 2024-08-17T17:07:07.200 INFO:tasks.rados.rados.0.smithi063.stdout:2898: done (4 left) 2024-08-17T17:07:07.200 INFO:tasks.rados.rados.0.smithi063.stdout:2899: done (3 left) 2024-08-17T17:07:07.200 INFO:tasks.rados.rados.0.smithi063.stdout:2900: done (2 left) 2024-08-17T17:07:07.200 INFO:tasks.rados.rados.0.smithi063.stdout:2901: done (1 left) 2024-08-17T17:07:07.200 INFO:tasks.rados.rados.0.smithi063.stdout:2902: done (0 left) 2024-08-17T17:07:07.200 INFO:tasks.rados.rados.0.smithi063.stdout:2903: write oid 13 current snap is 363 2024-08-17T17:07:07.200 INFO:tasks.rados.rados.0.smithi063.stdout:2903: seq_num 1090 ranges {719630=659889,2117608=202022} 2024-08-17T17:07:07.206 INFO:tasks.rados.rados.0.smithi063.stdout:2903: writing smithi06387500-13 from 719630 to 1379519 tid 1 2024-08-17T17:07:07.212 INFO:tasks.rados.rados.0.smithi063.stdout:2903: writing smithi06387500-13 from 2117608 to 2319630 tid 2 2024-08-17T17:07:07.212 INFO:tasks.rados.rados.0.smithi063.stdout:2904: write oid 41 current snap is 363 2024-08-17T17:07:07.212 INFO:tasks.rados.rados.0.smithi063.stdout:2903: finishing write tid 1 to smithi06387500-13 2024-08-17T17:07:07.212 INFO:tasks.rados.rados.0.smithi063.stdout:2904: seq_num 1091 ranges {736437=735138,2176090=160347} 2024-08-17T17:07:07.220 INFO:tasks.rados.rados.0.smithi063.stdout:2904: writing smithi06387500-41 from 736437 to 1471575 tid 1 2024-08-17T17:07:07.226 INFO:tasks.rados.rados.0.smithi063.stdout:2904: writing smithi06387500-41 from 2176090 to 2336437 tid 2 2024-08-17T17:07:07.226 INFO:tasks.rados.rados.0.smithi063.stdout:2905: rollback oid 12 current snap is 363 2024-08-17T17:07:07.226 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 12 to 352 2024-08-17T17:07:07.226 INFO:tasks.rados.rados.0.smithi063.stdout:2903: finishing write tid 2 to smithi06387500-13 2024-08-17T17:07:07.226 INFO:tasks.rados.rados.0.smithi063.stdout:2903: finishing write tid 3 to smithi06387500-13 2024-08-17T17:07:07.226 INFO:tasks.rados.rados.0.smithi063.stdout:2903: finishing write tid 4 to smithi06387500-13 2024-08-17T17:07:07.226 INFO:tasks.rados.rados.0.smithi063.stdout:2903: oid 13 updating version 0 to 1427 2024-08-17T17:07:07.226 INFO:tasks.rados.rados.0.smithi063.stdout:2903: oid 13 version 1427 is already newer than 1426 2024-08-17T17:07:07.226 INFO:tasks.rados.rados.0.smithi063.stdout:2903: oid 13 updating version 1427 to 1428 2024-08-17T17:07:07.226 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1428 (ObjNum 1090 snap 363 seq_num 1090) dirty exists 2024-08-17T17:07:07.227 INFO:tasks.rados.rados.0.smithi063.stdout:2903: left oid 13 (ObjNum 1090 snap 363 seq_num 1090) 2024-08-17T17:07:07.227 INFO:tasks.rados.rados.0.smithi063.stdout:2904: finishing write tid 1 to smithi06387500-41 2024-08-17T17:07:07.227 INFO:tasks.rados.rados.0.smithi063.stdout:2903: done (2 left) 2024-08-17T17:07:07.227 INFO:tasks.rados.rados.0.smithi063.stdout:2906: read oid 37 snap -1 2024-08-17T17:07:07.227 INFO:tasks.rados.rados.0.smithi063.stdout:2906: expect (ObjNum 1061 snap 358 seq_num 1061) 2024-08-17T17:07:07.227 INFO:tasks.rados.rados.0.smithi063.stdout:2907: rollback oid 15 current snap is 363 2024-08-17T17:07:07.227 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 15 to 352 2024-08-17T17:07:07.227 INFO:tasks.rados.rados.0.smithi063.stdout:2908: write oid 39 current snap is 363 2024-08-17T17:07:07.227 INFO:tasks.rados.rados.0.smithi063.stdout:2908: seq_num 1092 ranges {753244=410387,1834572=518672} 2024-08-17T17:07:07.231 INFO:tasks.rados.rados.0.smithi063.stdout:2908: writing smithi06387500-39 from 753244 to 1163631 tid 1 2024-08-17T17:07:07.239 INFO:tasks.rados.rados.0.smithi063.stdout:2908: writing smithi06387500-39 from 1834572 to 2353244 tid 2 2024-08-17T17:07:07.239 INFO:tasks.rados.rados.0.smithi063.stdout:2909: rollback oid 24 current snap is 363 2024-08-17T17:07:07.239 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 24 to 354 2024-08-17T17:07:07.239 INFO:tasks.rados.rados.0.smithi063.stdout:2905: finishing rollback tid 1 to smithi06387500-12 2024-08-17T17:07:07.239 INFO:tasks.rados.rados.0.smithi063.stdout:2905: finishing rollback tid 2 to smithi06387500-12 2024-08-17T17:07:07.239 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 655 (ObjNum 905 snap 311 seq_num 905) dirty exists 2024-08-17T17:07:07.239 INFO:tasks.rados.rados.0.smithi063.stdout:2907: finishing rollback tid 1 to smithi06387500-15 2024-08-17T17:07:07.239 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 0 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T17:07:07.239 INFO:tasks.rados.rados.0.smithi063.stdout:2904: finishing write tid 2 to smithi06387500-41 2024-08-17T17:07:07.239 INFO:tasks.rados.rados.0.smithi063.stdout:2904: finishing write tid 3 to smithi06387500-41 2024-08-17T17:07:07.240 INFO:tasks.rados.rados.0.smithi063.stdout:2904: finishing write tid 4 to smithi06387500-41 2024-08-17T17:07:07.240 INFO:tasks.rados.rados.0.smithi063.stdout:2904: oid 41 updating version 0 to 336 2024-08-17T17:07:07.240 INFO:tasks.rados.rados.0.smithi063.stdout:2904: oid 41 version 336 is already newer than 335 2024-08-17T17:07:07.240 INFO:tasks.rados.rados.0.smithi063.stdout:2904: oid 41 version 336 is already newer than 334 2024-08-17T17:07:07.240 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 336 (ObjNum 1091 snap 363 seq_num 1091) dirty exists 2024-08-17T17:07:07.240 INFO:tasks.rados.rados.0.smithi063.stdout:2904: left oid 41 (ObjNum 1091 snap 363 seq_num 1091) 2024-08-17T17:07:07.240 INFO:tasks.rados.rados.0.smithi063.stdout:2908: finishing write tid 1 to smithi06387500-39 2024-08-17T17:07:07.240 INFO:tasks.rados.rados.0.smithi063.stdout:2904: done (5 left) 2024-08-17T17:07:07.240 INFO:tasks.rados.rados.0.smithi063.stdout:2905: done (4 left) 2024-08-17T17:07:07.240 INFO:tasks.rados.rados.0.smithi063.stdout:2907: done (3 left) 2024-08-17T17:07:07.240 INFO:tasks.rados.rados.0.smithi063.stdout:2910: write oid 23 current snap is 363 2024-08-17T17:07:07.240 INFO:tasks.rados.rados.0.smithi063.stdout:2910: seq_num 1093 ranges {770051=485636,1776701=511047} 2024-08-17T17:07:07.245 INFO:tasks.rados.rados.0.smithi063.stdout:2910: writing smithi06387500-23 from 770051 to 1255687 tid 1 2024-08-17T17:07:07.251 INFO:tasks.rados.rados.0.smithi063.stdout:2910: writing smithi06387500-23 from 1776701 to 2287748 tid 2 2024-08-17T17:07:07.251 INFO:tasks.rados.rados.0.smithi063.stdout:2911: write oid 22 current snap is 363 2024-08-17T17:07:07.252 INFO:tasks.rados.rados.0.smithi063.stdout:2906: expect (ObjNum 1061 snap 358 seq_num 1061) 2024-08-17T17:07:07.380 INFO:tasks.rados.rados.0.smithi063.stdout:2908: finishing write tid 2 to smithi06387500-39 2024-08-17T17:07:07.380 INFO:tasks.rados.rados.0.smithi063.stdout:2909: finishing rollback tid 0 to smithi06387500-24 2024-08-17T17:07:07.380 INFO:tasks.rados.rados.0.smithi063.stdout:2908: finishing write tid 3 to smithi06387500-39 2024-08-17T17:07:07.380 INFO:tasks.rados.rados.0.smithi063.stdout:2909: finishing rollback tid 1 to smithi06387500-24 2024-08-17T17:07:07.380 INFO:tasks.rados.rados.0.smithi063.stdout:2911: seq_num 1094 ranges {786858=560885,1835183=551675} 2024-08-17T17:07:07.387 INFO:tasks.rados.rados.0.smithi063.stdout:2911: writing smithi06387500-22 from 786858 to 1347743 tid 1 2024-08-17T17:07:07.393 INFO:tasks.rados.rados.0.smithi063.stdout:2911: writing smithi06387500-22 from 1835183 to 2386858 tid 2 2024-08-17T17:07:07.393 INFO:tasks.rados.rados.0.smithi063.stdout:2906: done (4 left) 2024-08-17T17:07:07.393 INFO:tasks.rados.rados.0.smithi063.stdout:2912: rollback oid 20 current snap is 363 2024-08-17T17:07:07.394 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 20 to 355 2024-08-17T17:07:07.394 INFO:tasks.rados.rados.0.smithi063.stdout:2909: finishing rollback tid 2 to smithi06387500-24 2024-08-17T17:07:07.394 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 1020 (ObjNum 1021 snap 348 seq_num 1021) dirty exists 2024-08-17T17:07:07.394 INFO:tasks.rados.rados.0.smithi063.stdout:2908: finishing write tid 4 to smithi06387500-39 2024-08-17T17:07:07.394 INFO:tasks.rados.rados.0.smithi063.stdout:2908: oid 39 updating version 0 to 666 2024-08-17T17:07:07.394 INFO:tasks.rados.rados.0.smithi063.stdout:2908: oid 39 updating version 666 to 668 2024-08-17T17:07:07.394 INFO:tasks.rados.rados.0.smithi063.stdout:2908: oid 39 version 668 is already newer than 667 2024-08-17T17:07:07.394 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 668 (ObjNum 1092 snap 363 seq_num 1092) dirty exists 2024-08-17T17:07:07.394 INFO:tasks.rados.rados.0.smithi063.stdout:2908: left oid 39 (ObjNum 1092 snap 363 seq_num 1092) 2024-08-17T17:07:07.394 INFO:tasks.rados.rados.0.smithi063.stdout:2910: finishing write tid 1 to smithi06387500-23 2024-08-17T17:07:07.394 INFO:tasks.rados.rados.0.smithi063.stdout:2910: finishing write tid 2 to smithi06387500-23 2024-08-17T17:07:07.394 INFO:tasks.rados.rados.0.smithi063.stdout:2910: finishing write tid 3 to smithi06387500-23 2024-08-17T17:07:07.395 INFO:tasks.rados.rados.0.smithi063.stdout:2910: finishing write tid 4 to smithi06387500-23 2024-08-17T17:07:07.395 INFO:tasks.rados.rados.0.smithi063.stdout:2910: oid 23 updating version 0 to 1431 2024-08-17T17:07:07.395 INFO:tasks.rados.rados.0.smithi063.stdout:2910: oid 23 version 1431 is already newer than 1430 2024-08-17T17:07:07.395 INFO:tasks.rados.rados.0.smithi063.stdout:2910: oid 23 version 1431 is already newer than 1429 2024-08-17T17:07:07.395 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 1431 (ObjNum 1093 snap 363 seq_num 1093) dirty exists 2024-08-17T17:07:07.395 INFO:tasks.rados.rados.0.smithi063.stdout:2910: left oid 23 (ObjNum 1093 snap 363 seq_num 1093) 2024-08-17T17:07:07.395 INFO:tasks.rados.rados.0.smithi063.stdout:2911: finishing write tid 1 to smithi06387500-22 2024-08-17T17:07:07.395 INFO:tasks.rados.rados.0.smithi063.stdout:2908: done (4 left) 2024-08-17T17:07:07.395 INFO:tasks.rados.rados.0.smithi063.stdout:2909: done (3 left) 2024-08-17T17:07:07.395 INFO:tasks.rados.rados.0.smithi063.stdout:2910: done (2 left) 2024-08-17T17:07:07.395 INFO:tasks.rados.rados.0.smithi063.stdout:2913: delete oid 5 current snap is 363 2024-08-17T17:07:07.396 INFO:tasks.rados.rados.0.smithi063.stdout:2913: done (2 left) 2024-08-17T17:07:07.396 INFO:tasks.rados.rados.0.smithi063.stdout:2914: rollback oid 8 current snap is 363 2024-08-17T17:07:07.396 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 8 to 352 2024-08-17T17:07:07.396 INFO:tasks.rados.rados.0.smithi063.stdout:2915: read oid 33 snap -1 2024-08-17T17:07:07.396 INFO:tasks.rados.rados.0.smithi063.stdout:2915: expect (ObjNum 1071 snap 362 seq_num 1071) 2024-08-17T17:07:07.396 INFO:tasks.rados.rados.0.smithi063.stdout:2916: delete oid 12 current snap is 363 2024-08-17T17:07:07.397 INFO:tasks.rados.rados.0.smithi063.stdout:2916: done (4 left) 2024-08-17T17:07:07.397 INFO:tasks.rados.rados.0.smithi063.stdout:2917: snap_create 2024-08-17T17:07:07.397 INFO:tasks.rados.rados.0.smithi063.stdout:2911: finishing write tid 2 to smithi06387500-22 2024-08-17T17:07:07.397 INFO:tasks.rados.rados.0.smithi063.stdout:2911: finishing write tid 3 to smithi06387500-22 2024-08-17T17:07:07.398 INFO:tasks.rados.rados.0.smithi063.stdout:2911: finishing write tid 4 to smithi06387500-22 2024-08-17T17:07:07.398 INFO:tasks.rados.rados.0.smithi063.stdout:2911: oid 22 updating version 0 to 631 2024-08-17T17:07:07.398 INFO:tasks.rados.rados.0.smithi063.stdout:2911: oid 22 version 631 is already newer than 630 2024-08-17T17:07:07.398 INFO:tasks.rados.rados.0.smithi063.stdout:2911: oid 22 updating version 631 to 632 2024-08-17T17:07:07.398 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 632 (ObjNum 1094 snap 363 seq_num 1094) dirty exists 2024-08-17T17:07:07.398 INFO:tasks.rados.rados.0.smithi063.stdout:2911: left oid 22 (ObjNum 1094 snap 363 seq_num 1094) 2024-08-17T17:07:07.398 INFO:tasks.rados.rados.0.smithi063.stdout:2912: finishing rollback tid 0 to smithi06387500-20 2024-08-17T17:07:07.399 INFO:tasks.rados.rados.0.smithi063.stdout:2912: finishing rollback tid 1 to smithi06387500-20 2024-08-17T17:07:07.399 INFO:tasks.rados.rados.0.smithi063.stdout:2912: finishing rollback tid 2 to smithi06387500-20 2024-08-17T17:07:07.399 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 636 (ObjNum 996 snap 342 seq_num 996) dirty exists 2024-08-17T17:07:07.401 INFO:tasks.rados.rados.0.smithi063.stdout:2914: finishing rollback tid 0 to smithi06387500-8 2024-08-17T17:07:07.401 INFO:tasks.rados.rados.0.smithi063.stdout:2914: finishing rollback tid 1 to smithi06387500-8 2024-08-17T17:07:07.401 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 993 (ObjNum 33 snap 0 seq_num 1807085712) dirty dne 2024-08-17T17:07:07.402 INFO:tasks.rados.rados.0.smithi063.stdout:2915: expect (ObjNum 1071 snap 362 seq_num 1071) 2024-08-17T17:07:07.618 INFO:tasks.rados.rados.0.smithi063.stdout:2911: done (4 left) 2024-08-17T17:07:07.618 INFO:tasks.rados.rados.0.smithi063.stdout:2912: done (3 left) 2024-08-17T17:07:07.618 INFO:tasks.rados.rados.0.smithi063.stdout:2914: done (2 left) 2024-08-17T17:07:07.618 INFO:tasks.rados.rados.0.smithi063.stdout:2915: done (1 left) 2024-08-17T17:07:07.618 INFO:tasks.rados.rados.0.smithi063.stdout:2917: done (0 left) 2024-08-17T17:07:07.618 INFO:tasks.rados.rados.0.smithi063.stdout:2918: write oid 3 current snap is 364 2024-08-17T17:07:07.618 INFO:tasks.rados.rados.0.smithi063.stdout:2918: seq_num 1097 ranges {437279=502985,1610629=518385} 2024-08-17T17:07:07.623 INFO:tasks.rados.rados.0.smithi063.stdout:2918: writing smithi06387500-3 from 437279 to 940264 tid 1 2024-08-17T17:07:07.630 INFO:tasks.rados.rados.0.smithi063.stdout:2918: writing smithi06387500-3 from 1610629 to 2129014 tid 2 2024-08-17T17:07:07.630 INFO:tasks.rados.rados.0.smithi063.stdout:2919: snap_create 2024-08-17T17:07:07.630 INFO:tasks.rados.rados.0.smithi063.stdout:2918: finishing write tid 1 to smithi06387500-3 2024-08-17T17:07:07.633 INFO:tasks.rados.rados.0.smithi063.stdout:2918: finishing write tid 2 to smithi06387500-3 2024-08-17T17:07:07.633 INFO:tasks.rados.rados.0.smithi063.stdout:2918: finishing write tid 3 to smithi06387500-3 2024-08-17T17:07:07.634 INFO:tasks.rados.rados.0.smithi063.stdout:2918: finishing write tid 4 to smithi06387500-3 2024-08-17T17:07:07.634 INFO:tasks.rados.rados.0.smithi063.stdout:2918: oid 3 updating version 0 to 976 2024-08-17T17:07:07.634 INFO:tasks.rados.rados.0.smithi063.stdout:2918: oid 3 version 976 is already newer than 974 2024-08-17T17:07:07.634 INFO:tasks.rados.rados.0.smithi063.stdout:2918: oid 3 version 976 is already newer than 975 2024-08-17T17:07:07.634 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 976 (ObjNum 1097 snap 364 seq_num 1097) dirty exists 2024-08-17T17:07:07.634 INFO:tasks.rados.rados.0.smithi063.stdout:2918: left oid 3 (ObjNum 1097 snap 364 seq_num 1097) 2024-08-17T17:07:07.987 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:07 smithi063 ceph-mon[109843]: osdmap e799: 8 total, 7 up, 8 in 2024-08-17T17:07:07.987 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:07 smithi063 ceph-mon[109843]: osdmap e800: 8 total, 7 up, 8 in 2024-08-17T17:07:07.987 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:07 smithi063 ceph-mon[114756]: osdmap e799: 8 total, 7 up, 8 in 2024-08-17T17:07:07.988 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:07 smithi063 ceph-mon[114756]: osdmap e800: 8 total, 7 up, 8 in 2024-08-17T17:07:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:07 smithi162 ceph-mon[104668]: osdmap e799: 8 total, 7 up, 8 in 2024-08-17T17:07:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:07 smithi162 ceph-mon[104668]: osdmap e800: 8 total, 7 up, 8 in 2024-08-17T17:07:08.621 INFO:tasks.rados.rados.0.smithi063.stdout:2918: done (1 left) 2024-08-17T17:07:08.621 INFO:tasks.rados.rados.0.smithi063.stdout:2919: done (0 left) 2024-08-17T17:07:08.621 INFO:tasks.rados.rados.0.smithi063.stdout:2920: delete oid 47 current snap is 365 2024-08-17T17:07:08.623 INFO:tasks.rados.rados.0.smithi063.stdout:2920: done (0 left) 2024-08-17T17:07:08.623 INFO:tasks.rados.rados.0.smithi063.stdout:2921: rollback oid 10 current snap is 365 2024-08-17T17:07:08.623 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 10 to 352 2024-08-17T17:07:08.623 INFO:tasks.rados.rados.0.smithi063.stdout:2922: snap_create 2024-08-17T17:07:08.625 INFO:tasks.rados.rados.0.smithi063.stdout:2921: finishing rollback tid 1 to smithi06387500-10 2024-08-17T17:07:08.625 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 899 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-08-17T17:07:09.627 INFO:tasks.rados.rados.0.smithi063.stdout:2921: done (1 left) 2024-08-17T17:07:09.627 INFO:tasks.rados.rados.0.smithi063.stdout:2922: done (0 left) 2024-08-17T17:07:09.627 INFO:tasks.rados.rados.0.smithi063.stdout:2923: read oid 35 snap -1 2024-08-17T17:07:09.627 INFO:tasks.rados.rados.0.smithi063.stdout:2923: expect (ObjNum 1068 snap 361 seq_num 1068) 2024-08-17T17:07:09.627 INFO:tasks.rados.rados.0.smithi063.stdout:2924: read oid 48 snap 354 2024-08-17T17:07:09.628 INFO:tasks.rados.rados.0.smithi063.stdout:2924: expect deleted 2024-08-17T17:07:09.628 INFO:tasks.rados.rados.0.smithi063.stdout:2925: write oid 46 current snap is 366 2024-08-17T17:07:09.628 INFO:tasks.rados.rados.0.smithi063.stdout:2925: seq_num 1099 ranges {470893=653483,1727593=522054} 2024-08-17T17:07:09.633 INFO:tasks.rados.rados.0.smithi063.stdout:2925: writing smithi06387500-46 from 470893 to 1124376 tid 1 2024-08-17T17:07:09.640 INFO:tasks.rados.rados.0.smithi063.stdout:2925: writing smithi06387500-46 from 1727593 to 2249647 tid 2 2024-08-17T17:07:09.640 INFO:tasks.rados.rados.0.smithi063.stdout:2926: read oid 4 snap 355 2024-08-17T17:07:09.640 INFO:tasks.rados.rados.0.smithi063.stdout:2926: expect (ObjNum 1051 snap 355 seq_num 1051) 2024-08-17T17:07:09.640 INFO:tasks.rados.rados.0.smithi063.stdout:2927: write oid 22 current snap is 366 2024-08-17T17:07:09.640 INFO:tasks.rados.rados.0.smithi063.stdout:2927: seq_num 1100 ranges {487700=728732,1786075=665712} 2024-08-17T17:07:09.646 INFO:tasks.rados.rados.0.smithi063.stdout:2927: writing smithi06387500-22 from 487700 to 1216432 tid 1 2024-08-17T17:07:09.654 INFO:tasks.rados.rados.0.smithi063.stdout:2927: writing smithi06387500-22 from 1786075 to 2451787 tid 2 2024-08-17T17:07:09.654 INFO:tasks.rados.rados.0.smithi063.stdout:2928: write oid 49 current snap is 366 2024-08-17T17:07:09.654 INFO:tasks.rados.rados.0.smithi063.stdout:2928: seq_num 1101 ranges {504507=403981,1328204=409370,2252362=252145} 2024-08-17T17:07:09.659 INFO:tasks.rados.rados.0.smithi063.stdout:2928: writing smithi06387500-49 from 504507 to 908488 tid 1 2024-08-17T17:07:09.663 INFO:tasks.rados.rados.0.smithi063.stdout:2928: writing smithi06387500-49 from 1328204 to 1737574 tid 2 2024-08-17T17:07:09.668 INFO:tasks.rados.rados.0.smithi063.stdout:2928: writing smithi06387500-49 from 2252362 to 2504507 tid 3 2024-08-17T17:07:09.668 INFO:tasks.rados.rados.0.smithi063.stdout:2929: read oid 44 snap -1 2024-08-17T17:07:09.668 INFO:tasks.rados.rados.0.smithi063.stdout:2929: expect (ObjNum 1058 snap 356 seq_num 1058) 2024-08-17T17:07:09.669 INFO:tasks.rados.rados.0.smithi063.stdout:2930: snap_remove snap 352 2024-08-17T17:07:09.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:09 smithi063 ceph-mon[109843]: pgmap v657: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 230 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.9 MiB/s wr, 19 op/s; 35/444 objects degraded (7.883%) 2024-08-17T17:07:09.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:09.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:09.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:09 smithi063 ceph-mon[109843]: osdmap e801: 8 total, 7 up, 8 in 2024-08-17T17:07:09.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:09 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 35/444 objects degraded (7.883%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:09.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:09.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:09.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:09 smithi063 ceph-mon[114756]: pgmap v657: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 230 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.9 MiB/s wr, 19 op/s; 35/444 objects degraded (7.883%) 2024-08-17T17:07:09.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:09.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:09.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:09 smithi063 ceph-mon[114756]: osdmap e801: 8 total, 7 up, 8 in 2024-08-17T17:07:09.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:09 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 35/444 objects degraded (7.883%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:09.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:09.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:09.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:09 smithi162 ceph-mon[104668]: pgmap v657: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 230 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.9 MiB/s wr, 19 op/s; 35/444 objects degraded (7.883%) 2024-08-17T17:07:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:09 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:09 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:09 smithi162 ceph-mon[104668]: osdmap e801: 8 total, 7 up, 8 in 2024-08-17T17:07:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:09 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 35/444 objects degraded (7.883%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:09 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:09 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:10.279 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2[137556]: 2024-08-17T17:07:09.989+0000 7f66e85fd640 -1 osd.2 791 log_to_monitors true 2024-08-17T17:07:10.629 INFO:tasks.rados.rados.0.smithi063.stdout:2930: done (7 left) 2024-08-17T17:07:10.629 INFO:tasks.rados.rados.0.smithi063.stdout:2931: rollback oid 31 current snap is 366 2024-08-17T17:07:10.629 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 31 to 339 2024-08-17T17:07:10.630 INFO:tasks.rados.rados.0.smithi063.stdout:2932: snap_remove snap 364 2024-08-17T17:07:10.984 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:10 smithi063 ceph-mon[114756]: osdmap e802: 8 total, 7 up, 8 in 2024-08-17T17:07:10.984 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:10 smithi063 ceph-mon[114756]: from='osd.2 [v2:172.21.15.63:6818/3903658295,v1:172.21.15.63:6819/3903658295]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T17:07:10.985 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:07:10 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2[137556]: 2024-08-17T17:07:10.656+0000 7f66dc39f640 -1 osd.2 791 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:07:10.985 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:10 smithi063 ceph-mon[109843]: osdmap e802: 8 total, 7 up, 8 in 2024-08-17T17:07:10.985 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:10 smithi063 ceph-mon[109843]: from='osd.2 [v2:172.21.15.63:6818/3903658295,v1:172.21.15.63:6819/3903658295]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T17:07:11.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:10 smithi162 ceph-mon[104668]: osdmap e802: 8 total, 7 up, 8 in 2024-08-17T17:07:11.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:10 smithi162 ceph-mon[104668]: from='osd.2 [v2:172.21.15.63:6818/3903658295,v1:172.21.15.63:6819/3903658295]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-08-17T17:07:11.645 INFO:tasks.rados.rados.0.smithi063.stdout:2932: done (8 left) 2024-08-17T17:07:11.645 INFO:tasks.rados.rados.0.smithi063.stdout:2933: read oid 7 snap -1 2024-08-17T17:07:11.645 INFO:tasks.rados.rados.0.smithi063.stdout:2933: expect (ObjNum 1069 snap 361 seq_num 1069) 2024-08-17T17:07:11.645 INFO:tasks.rados.rados.0.smithi063.stdout:2934: read oid 38 snap -1 2024-08-17T17:07:11.645 INFO:tasks.rados.rados.0.smithi063.stdout:2934: expect (ObjNum 1083 snap 363 seq_num 1083) 2024-08-17T17:07:11.645 INFO:tasks.rados.rados.0.smithi063.stdout:2935: delete oid 14 current snap is 366 2024-08-17T17:07:11.645 INFO:tasks.rados.rados.0.smithi063.stdout:2923: expect (ObjNum 1068 snap 361 seq_num 1068) 2024-08-17T17:07:11.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:11 smithi063 ceph-mon[109843]: pgmap v660: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 230 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 1.4 MiB/s wr, 14 op/s; 35/444 objects degraded (7.883%) 2024-08-17T17:07:11.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:11 smithi063 ceph-mon[109843]: from='osd.2 [v2:172.21.15.63:6818/3903658295,v1:172.21.15.63:6819/3903658295]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:07:11.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:11 smithi063 ceph-mon[109843]: osdmap e803: 8 total, 7 up, 8 in 2024-08-17T17:07:11.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:11 smithi063 ceph-mon[109843]: from='osd.2 [v2:172.21.15.63:6818/3903658295,v1:172.21.15.63:6819/3903658295]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]} : dispatch 2024-08-17T17:07:11.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:11 smithi063 ceph-mon[114756]: pgmap v660: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 230 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 1.4 MiB/s wr, 14 op/s; 35/444 objects degraded (7.883%) 2024-08-17T17:07:11.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:11 smithi063 ceph-mon[114756]: from='osd.2 [v2:172.21.15.63:6818/3903658295,v1:172.21.15.63:6819/3903658295]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:07:11.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:11 smithi063 ceph-mon[114756]: osdmap e803: 8 total, 7 up, 8 in 2024-08-17T17:07:11.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:11 smithi063 ceph-mon[114756]: from='osd.2 [v2:172.21.15.63:6818/3903658295,v1:172.21.15.63:6819/3903658295]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]} : dispatch 2024-08-17T17:07:11.895 INFO:tasks.rados.rados.0.smithi063.stdout:2925: finishing write tid 1 to smithi06387500-46 2024-08-17T17:07:11.895 INFO:tasks.rados.rados.0.smithi063.stdout:2925: finishing write tid 2 to smithi06387500-46 2024-08-17T17:07:11.895 INFO:tasks.rados.rados.0.smithi063.stdout:2925: finishing write tid 3 to smithi06387500-46 2024-08-17T17:07:11.895 INFO:tasks.rados.rados.0.smithi063.stdout:2925: finishing write tid 4 to smithi06387500-46 2024-08-17T17:07:11.895 INFO:tasks.rados.rados.0.smithi063.stdout:2925: oid 46 updating version 0 to 1021 2024-08-17T17:07:11.896 INFO:tasks.rados.rados.0.smithi063.stdout:2925: oid 46 updating version 1021 to 1022 2024-08-17T17:07:11.896 INFO:tasks.rados.rados.0.smithi063.stdout:2925: oid 46 updating version 1022 to 1023 2024-08-17T17:07:11.896 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 1023 (ObjNum 1099 snap 366 seq_num 1099) dirty exists 2024-08-17T17:07:11.896 INFO:tasks.rados.rados.0.smithi063.stdout:2925: left oid 46 (ObjNum 1099 snap 366 seq_num 1099) 2024-08-17T17:07:11.896 INFO:tasks.rados.rados.0.smithi063.stdout:2927: finishing write tid 1 to smithi06387500-22 2024-08-17T17:07:11.896 INFO:tasks.rados.rados.0.smithi063.stdout:2926: expect (ObjNum 1051 snap 355 seq_num 1051) 2024-08-17T17:07:12.017 INFO:tasks.rados.rados.0.smithi063.stdout:2927: finishing write tid 2 to smithi06387500-22 2024-08-17T17:07:12.017 INFO:tasks.rados.rados.0.smithi063.stdout:2927: finishing write tid 3 to smithi06387500-22 2024-08-17T17:07:12.017 INFO:tasks.rados.rados.0.smithi063.stdout:2923: done (10 left) 2024-08-17T17:07:12.017 INFO:tasks.rados.rados.0.smithi063.stdout:2924: done (9 left) 2024-08-17T17:07:12.017 INFO:tasks.rados.rados.0.smithi063.stdout:2925: done (8 left) 2024-08-17T17:07:12.017 INFO:tasks.rados.rados.0.smithi063.stdout:2926: done (7 left) 2024-08-17T17:07:12.017 INFO:tasks.rados.rados.0.smithi063.stdout:2935: done (6 left) 2024-08-17T17:07:12.018 INFO:tasks.rados.rados.0.smithi063.stdout:2936: write oid 20 current snap is 366 2024-08-17T17:07:12.018 INFO:tasks.rados.rados.0.smithi063.stdout:2927: finishing write tid 4 to smithi06387500-22 2024-08-17T17:07:12.018 INFO:tasks.rados.rados.0.smithi063.stdout:2927: oid 22 updating version 0 to 640 2024-08-17T17:07:12.018 INFO:tasks.rados.rados.0.smithi063.stdout:2927: oid 22 version 640 is already newer than 638 2024-08-17T17:07:12.018 INFO:tasks.rados.rados.0.smithi063.stdout:2927: oid 22 version 640 is already newer than 639 2024-08-17T17:07:12.018 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 640 (ObjNum 1100 snap 366 seq_num 1100) dirty exists 2024-08-17T17:07:12.018 INFO:tasks.rados.rados.0.smithi063.stdout:2927: left oid 22 (ObjNum 1100 snap 366 seq_num 1100) 2024-08-17T17:07:12.018 INFO:tasks.rados.rados.0.smithi063.stdout:2928: finishing write tid 1 to smithi06387500-49 2024-08-17T17:07:12.018 INFO:tasks.rados.rados.0.smithi063.stdout:2928: finishing write tid 2 to smithi06387500-49 2024-08-17T17:07:12.018 INFO:tasks.rados.rados.0.smithi063.stdout:2928: finishing write tid 3 to smithi06387500-49 2024-08-17T17:07:12.018 INFO:tasks.rados.rados.0.smithi063.stdout:2928: finishing write tid 4 to smithi06387500-49 2024-08-17T17:07:12.019 INFO:tasks.rados.rados.0.smithi063.stdout:2928: finishing write tid 5 to smithi06387500-49 2024-08-17T17:07:12.019 INFO:tasks.rados.rados.0.smithi063.stdout:2928: oid 49 updating version 0 to 2805 2024-08-17T17:07:12.019 INFO:tasks.rados.rados.0.smithi063.stdout:2928: oid 49 updating version 2805 to 2806 2024-08-17T17:07:12.019 INFO:tasks.rados.rados.0.smithi063.stdout:2928: oid 49 version 2806 is already newer than 2804 2024-08-17T17:07:12.019 INFO:tasks.rados.rados.0.smithi063.stdout:2928: oid 49 version 2806 is already newer than 2803 2024-08-17T17:07:12.019 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 2806 (ObjNum 1101 snap 366 seq_num 1101) dirty exists 2024-08-17T17:07:12.019 INFO:tasks.rados.rados.0.smithi063.stdout:2928: left oid 49 (ObjNum 1101 snap 366 seq_num 1101) 2024-08-17T17:07:12.019 INFO:tasks.rados.rados.0.smithi063.stdout:2929: expect (ObjNum 1058 snap 356 seq_num 1058) 2024-08-17T17:07:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:11 smithi162 ceph-mon[104668]: pgmap v660: 89 pgs: 16 active+undersized, 9 active+undersized+degraded, 64 active+clean; 230 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 1.4 MiB/s wr, 14 op/s; 35/444 objects degraded (7.883%) 2024-08-17T17:07:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:11 smithi162 ceph-mon[104668]: from='osd.2 [v2:172.21.15.63:6818/3903658295,v1:172.21.15.63:6819/3903658295]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-08-17T17:07:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:11 smithi162 ceph-mon[104668]: osdmap e803: 8 total, 7 up, 8 in 2024-08-17T17:07:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:11 smithi162 ceph-mon[104668]: from='osd.2 [v2:172.21.15.63:6818/3903658295,v1:172.21.15.63:6819/3903658295]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi063", "root=default"]} : dispatch 2024-08-17T17:07:12.246 INFO:tasks.rados.rados.0.smithi063.stdout:2931: finishing rollback tid 1 to smithi06387500-31 2024-08-17T17:07:12.246 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 0 (ObjNum 289 snap 0 seq_num 1028317573) dirty dne 2024-08-17T17:07:12.246 INFO:tasks.rados.rados.0.smithi063.stdout:2933: expect (ObjNum 1069 snap 361 seq_num 1069) 2024-08-17T17:07:12.460 INFO:tasks.rados.rados.0.smithi063.stdout:2936: seq_num 1103 ranges {538121=670832,1961521=413039} 2024-08-17T17:07:12.468 INFO:tasks.rados.rados.0.smithi063.stdout:2936: writing smithi06387500-20 from 538121 to 1208953 tid 1 2024-08-17T17:07:12.474 INFO:tasks.rados.rados.0.smithi063.stdout:2936: writing smithi06387500-20 from 1961521 to 2374560 tid 2 2024-08-17T17:07:12.475 INFO:tasks.rados.rados.0.smithi063.stdout:2927: done (6 left) 2024-08-17T17:07:12.475 INFO:tasks.rados.rados.0.smithi063.stdout:2928: done (5 left) 2024-08-17T17:07:12.475 INFO:tasks.rados.rados.0.smithi063.stdout:2929: done (4 left) 2024-08-17T17:07:12.475 INFO:tasks.rados.rados.0.smithi063.stdout:2931: done (3 left) 2024-08-17T17:07:12.475 INFO:tasks.rados.rados.0.smithi063.stdout:2933: done (2 left) 2024-08-17T17:07:12.475 INFO:tasks.rados.rados.0.smithi063.stdout:2937: delete oid 31 current snap is 366 2024-08-17T17:07:12.477 INFO:tasks.rados.rados.0.smithi063.stdout:2937: done (2 left) 2024-08-17T17:07:12.477 INFO:tasks.rados.rados.0.smithi063.stdout:2938: write oid 17 current snap is 366 2024-08-17T17:07:12.477 INFO:tasks.rados.rados.0.smithi063.stdout:2938: seq_num 1105 ranges {571735=421330,1562132=416708} 2024-08-17T17:07:12.482 INFO:tasks.rados.rados.0.smithi063.stdout:2938: writing smithi06387500-17 from 571735 to 993065 tid 1 2024-08-17T17:07:12.488 INFO:tasks.rados.rados.0.smithi063.stdout:2938: writing smithi06387500-17 from 1562132 to 1978840 tid 2 2024-08-17T17:07:12.488 INFO:tasks.rados.rados.0.smithi063.stdout:2939: snap_create 2024-08-17T17:07:12.488 INFO:tasks.rados.rados.0.smithi063.stdout:2938: finishing write tid 1 to smithi06387500-17 2024-08-17T17:07:12.491 INFO:tasks.rados.rados.0.smithi063.stdout:2938: finishing write tid 2 to smithi06387500-17 2024-08-17T17:07:12.491 INFO:tasks.rados.rados.0.smithi063.stdout:2938: finishing write tid 3 to smithi06387500-17 2024-08-17T17:07:12.491 INFO:tasks.rados.rados.0.smithi063.stdout:2938: finishing write tid 4 to smithi06387500-17 2024-08-17T17:07:12.491 INFO:tasks.rados.rados.0.smithi063.stdout:2938: oid 17 updating version 0 to 982 2024-08-17T17:07:12.492 INFO:tasks.rados.rados.0.smithi063.stdout:2938: oid 17 version 982 is already newer than 981 2024-08-17T17:07:12.492 INFO:tasks.rados.rados.0.smithi063.stdout:2938: oid 17 updating version 982 to 983 2024-08-17T17:07:12.492 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 983 (ObjNum 1105 snap 366 seq_num 1105) dirty exists 2024-08-17T17:07:12.492 INFO:tasks.rados.rados.0.smithi063.stdout:2938: left oid 17 (ObjNum 1105 snap 366 seq_num 1105) 2024-08-17T17:07:12.642 INFO:tasks.rados.rados.0.smithi063.stdout:2938: done (3 left) 2024-08-17T17:07:12.643 INFO:tasks.rados.rados.0.smithi063.stdout:2939: done (2 left) 2024-08-17T17:07:12.643 INFO:tasks.rados.rados.0.smithi063.stdout:2940: write oid 24 current snap is 367 2024-08-17T17:07:12.643 INFO:tasks.rados.rados.0.smithi063.stdout:2940: seq_num 1106 ranges {588542=496579,1620614=676719} 2024-08-17T17:07:12.646 INFO:tasks.rados.rados.0.smithi063.stdout:2940: writing smithi06387500-24 from 588542 to 1085121 tid 1 2024-08-17T17:07:12.655 INFO:tasks.rados.rados.0.smithi063.stdout:2940: writing smithi06387500-24 from 1620614 to 2297333 tid 2 2024-08-17T17:07:12.655 INFO:tasks.rados.rados.0.smithi063.stdout:2941: snap_remove snap 328 2024-08-17T17:07:12.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T17:07:12.590+0000 7f4652b73640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-08-17T17:07:12.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[114756]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:07:12.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[114756]: osd.2 [v2:172.21.15.63:6818/3903658295,v1:172.21.15.63:6819/3903658295] boot 2024-08-17T17:07:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[114756]: osdmap e804: 8 total, 8 up, 8 in 2024-08-17T17:07:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:07:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:12.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:07:12.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[109843]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:07:12.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[109843]: osd.2 [v2:172.21.15.63:6818/3903658295,v1:172.21.15.63:6819/3903658295] boot 2024-08-17T17:07:12.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[109843]: osdmap e804: 8 total, 8 up, 8 in 2024-08-17T17:07:12.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:07:12.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:12.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:12.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:12.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:12.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:12.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:12.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:12.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:12.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:12.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:12 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:07:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:12 smithi162 ceph-mon[104668]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:07:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:12 smithi162 ceph-mon[104668]: osd.2 [v2:172.21.15.63:6818/3903658295,v1:172.21.15.63:6819/3903658295] boot 2024-08-17T17:07:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:12 smithi162 ceph-mon[104668]: osdmap e804: 8 total, 8 up, 8 in 2024-08-17T17:07:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:12 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-08-17T17:07:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:12 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:12 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:12 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:12 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:12 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:12 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:12 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:12 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:13.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:12 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:13.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:12 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:07:13.638 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:07:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:07:13] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.51.0" 2024-08-17T17:07:13.648 INFO:tasks.rados.rados.0.smithi063.stdout:2941: done (3 left) 2024-08-17T17:07:13.649 INFO:tasks.rados.rados.0.smithi063.stdout:2942: snap_create 2024-08-17T17:07:13.649 INFO:tasks.rados.rados.0.smithi063.stdout:2934: expect (ObjNum 1083 snap 363 seq_num 1083) 2024-08-17T17:07:13.794 INFO:tasks.rados.rados.0.smithi063.stdout:2940: finishing write tid 1 to smithi06387500-24 2024-08-17T17:07:13.794 INFO:tasks.rados.rados.0.smithi063.stdout:2936: finishing write tid 1 to smithi06387500-20 2024-08-17T17:07:13.794 INFO:tasks.rados.rados.0.smithi063.stdout:2940: finishing write tid 2 to smithi06387500-24 2024-08-17T17:07:13.794 INFO:tasks.rados.rados.0.smithi063.stdout:2936: finishing write tid 2 to smithi06387500-20 2024-08-17T17:07:13.794 INFO:tasks.rados.rados.0.smithi063.stdout:2940: finishing write tid 3 to smithi06387500-24 2024-08-17T17:07:13.794 INFO:tasks.rados.rados.0.smithi063.stdout:2936: finishing write tid 3 to smithi06387500-20 2024-08-17T17:07:13.795 INFO:tasks.rados.rados.0.smithi063.stdout:2940: finishing write tid 4 to smithi06387500-24 2024-08-17T17:07:13.795 INFO:tasks.rados.rados.0.smithi063.stdout:2940: oid 24 updating version 0 to 1025 2024-08-17T17:07:13.795 INFO:tasks.rados.rados.0.smithi063.stdout:2940: oid 24 updating version 1025 to 1026 2024-08-17T17:07:13.795 INFO:tasks.rados.rados.0.smithi063.stdout:2940: oid 24 updating version 1026 to 1027 2024-08-17T17:07:13.795 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 1027 (ObjNum 1106 snap 367 seq_num 1106) dirty exists 2024-08-17T17:07:13.795 INFO:tasks.rados.rados.0.smithi063.stdout:2940: left oid 24 (ObjNum 1106 snap 367 seq_num 1106) 2024-08-17T17:07:13.795 INFO:tasks.rados.rados.0.smithi063.stdout:2936: finishing write tid 4 to smithi06387500-20 2024-08-17T17:07:13.795 INFO:tasks.rados.rados.0.smithi063.stdout:2936: oid 20 updating version 0 to 644 2024-08-17T17:07:13.795 INFO:tasks.rados.rados.0.smithi063.stdout:2936: oid 20 version 644 is already newer than 643 2024-08-17T17:07:13.795 INFO:tasks.rados.rados.0.smithi063.stdout:2936: oid 20 version 644 is already newer than 642 2024-08-17T17:07:13.795 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 644 (ObjNum 1103 snap 366 seq_num 1103) dirty exists 2024-08-17T17:07:13.795 INFO:tasks.rados.rados.0.smithi063.stdout:2936: left oid 20 (ObjNum 1103 snap 366 seq_num 1103) 2024-08-17T17:07:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:13 smithi063 ceph-mon[114756]: pgmap v663: 89 pgs: 10 peering, 9 active+undersized, 6 active+undersized+degraded, 64 active+clean; 238 MiB data, 790 MiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 2.2 MiB/s wr, 27 op/s; 30/462 objects degraded (6.494%) 2024-08-17T17:07:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:13 smithi063 ceph-mon[114756]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:07:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:13 smithi063 ceph-mon[114756]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-08-17T17:07:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:13 smithi063 ceph-mon[114756]: osdmap e805: 8 total, 8 up, 8 in 2024-08-17T17:07:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:13 smithi063 ceph-mon[109843]: pgmap v663: 89 pgs: 10 peering, 9 active+undersized, 6 active+undersized+degraded, 64 active+clean; 238 MiB data, 790 MiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 2.2 MiB/s wr, 27 op/s; 30/462 objects degraded (6.494%) 2024-08-17T17:07:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:13 smithi063 ceph-mon[109843]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:07:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:13 smithi063 ceph-mon[109843]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-08-17T17:07:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:13 smithi063 ceph-mon[109843]: osdmap e805: 8 total, 8 up, 8 in 2024-08-17T17:07:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:13 smithi162 ceph-mon[104668]: pgmap v663: 89 pgs: 10 peering, 9 active+undersized, 6 active+undersized+degraded, 64 active+clean; 238 MiB data, 790 MiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 2.2 MiB/s wr, 27 op/s; 30/462 objects degraded (6.494%) 2024-08-17T17:07:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:13 smithi162 ceph-mon[104668]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:07:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:13 smithi162 ceph-mon[104668]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-08-17T17:07:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:13 smithi162 ceph-mon[104668]: osdmap e805: 8 total, 8 up, 8 in 2024-08-17T17:07:14.653 INFO:tasks.rados.rados.0.smithi063.stdout:2934: done (3 left) 2024-08-17T17:07:14.653 INFO:tasks.rados.rados.0.smithi063.stdout:2936: done (2 left) 2024-08-17T17:07:14.653 INFO:tasks.rados.rados.0.smithi063.stdout:2940: done (1 left) 2024-08-17T17:07:14.653 INFO:tasks.rados.rados.0.smithi063.stdout:2942: done (0 left) 2024-08-17T17:07:14.654 INFO:tasks.rados.rados.0.smithi063.stdout:2943: rollback oid 14 current snap is 368 2024-08-17T17:07:14.654 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 14 to 359 2024-08-17T17:07:14.654 INFO:tasks.rados.rados.0.smithi063.stdout:2944: read oid 17 snap -1 2024-08-17T17:07:14.654 INFO:tasks.rados.rados.0.smithi063.stdout:2944: expect (ObjNum 1105 snap 366 seq_num 1105) 2024-08-17T17:07:14.654 INFO:tasks.rados.rados.0.smithi063.stdout:2945: snap_remove snap 360 2024-08-17T17:07:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:14 smithi063 ceph-mon[114756]: osdmap e806: 8 total, 8 up, 8 in 2024-08-17T17:07:15.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:14 smithi063 ceph-mon[109843]: osdmap e806: 8 total, 8 up, 8 in 2024-08-17T17:07:15.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:14 smithi162 ceph-mon[104668]: osdmap e806: 8 total, 8 up, 8 in 2024-08-17T17:07:15.656 INFO:tasks.rados.rados.0.smithi063.stdout:2945: done (2 left) 2024-08-17T17:07:15.656 INFO:tasks.rados.rados.0.smithi063.stdout:2946: delete oid 46 current snap is 368 2024-08-17T17:07:15.656 INFO:tasks.rados.rados.0.smithi063.stdout:2943: finishing rollback tid 1 to smithi06387500-14 2024-08-17T17:07:15.656 INFO:tasks.rados.rados.0.smithi063.stdout:2943: finishing rollback tid 2 to smithi06387500-14 2024-08-17T17:07:15.656 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 2838 (ObjNum 1043 snap 353 seq_num 1043) dirty exists 2024-08-17T17:07:15.656 INFO:tasks.rados.rados.0.smithi063.stdout:2944: expect (ObjNum 1105 snap 366 seq_num 1105) 2024-08-17T17:07:15.750 INFO:tasks.rados.rados.0.smithi063.stdout:2943: done (2 left) 2024-08-17T17:07:15.750 INFO:tasks.rados.rados.0.smithi063.stdout:2944: done (1 left) 2024-08-17T17:07:15.750 INFO:tasks.rados.rados.0.smithi063.stdout:2946: done (0 left) 2024-08-17T17:07:15.750 INFO:tasks.rados.rados.0.smithi063.stdout:2947: delete oid 24 current snap is 368 2024-08-17T17:07:15.751 INFO:tasks.rados.rados.0.smithi063.stdout:2947: done (0 left) 2024-08-17T17:07:15.751 INFO:tasks.rados.rados.0.smithi063.stdout:2948: delete oid 15 current snap is 368 2024-08-17T17:07:15.753 INFO:tasks.rados.rados.0.smithi063.stdout:2948: done (0 left) 2024-08-17T17:07:15.753 INFO:tasks.rados.rados.0.smithi063.stdout:2949: rollback oid 32 current snap is 368 2024-08-17T17:07:15.753 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 32 to 359 2024-08-17T17:07:15.753 INFO:tasks.rados.rados.0.smithi063.stdout:2950: write oid 48 current snap is 368 2024-08-17T17:07:15.753 INFO:tasks.rados.rados.0.smithi063.stdout:2950: seq_num 1110 ranges {655770=513928,1854542=684057} 2024-08-17T17:07:15.759 INFO:tasks.rados.rados.0.smithi063.stdout:2950: writing smithi06387500-48 from 655770 to 1169698 tid 1 2024-08-17T17:07:15.767 INFO:tasks.rados.rados.0.smithi063.stdout:2950: writing smithi06387500-48 from 1854542 to 2538599 tid 2 2024-08-17T17:07:15.767 INFO:tasks.rados.rados.0.smithi063.stdout:2951: write oid 28 current snap is 368 2024-08-17T17:07:15.767 INFO:tasks.rados.rados.0.smithi063.stdout:2949: finishing rollback tid 1 to smithi06387500-32 2024-08-17T17:07:15.767 INFO:tasks.rados.rados.0.smithi063.stdout:2949: finishing rollback tid 2 to smithi06387500-32 2024-08-17T17:07:15.767 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 621 (ObjNum 882 snap 308 seq_num 882) dirty exists 2024-08-17T17:07:15.767 INFO:tasks.rados.rados.0.smithi063.stdout:2950: finishing write tid 1 to smithi06387500-48 2024-08-17T17:07:15.767 INFO:tasks.rados.rados.0.smithi063.stdout:2951: seq_num 1111 ranges {672577=589177,1913024=427715} 2024-08-17T17:07:15.774 INFO:tasks.rados.rados.0.smithi063.stdout:2951: writing smithi06387500-28 from 672577 to 1261754 tid 1 2024-08-17T17:07:15.780 INFO:tasks.rados.rados.0.smithi063.stdout:2951: writing smithi06387500-28 from 1913024 to 2340739 tid 2 2024-08-17T17:07:15.781 INFO:tasks.rados.rados.0.smithi063.stdout:2949: done (2 left) 2024-08-17T17:07:15.781 INFO:tasks.rados.rados.0.smithi063.stdout:2952: read oid 47 snap -1 2024-08-17T17:07:15.781 INFO:tasks.rados.rados.0.smithi063.stdout:2952: expect deleted 2024-08-17T17:07:15.781 INFO:tasks.rados.rados.0.smithi063.stdout:2953: snap_create 2024-08-17T17:07:15.781 INFO:tasks.rados.rados.0.smithi063.stdout:2950: finishing write tid 2 to smithi06387500-48 2024-08-17T17:07:15.781 INFO:tasks.rados.rados.0.smithi063.stdout:2950: finishing write tid 3 to smithi06387500-48 2024-08-17T17:07:15.781 INFO:tasks.rados.rados.0.smithi063.stdout:2950: finishing write tid 4 to smithi06387500-48 2024-08-17T17:07:15.781 INFO:tasks.rados.rados.0.smithi063.stdout:2950: oid 48 updating version 0 to 2847 2024-08-17T17:07:15.781 INFO:tasks.rados.rados.0.smithi063.stdout:2950: oid 48 updating version 2847 to 2849 2024-08-17T17:07:15.781 INFO:tasks.rados.rados.0.smithi063.stdout:2950: oid 48 version 2849 is already newer than 2848 2024-08-17T17:07:15.782 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 2849 (ObjNum 1110 snap 368 seq_num 1110) dirty exists 2024-08-17T17:07:15.782 INFO:tasks.rados.rados.0.smithi063.stdout:2950: left oid 48 (ObjNum 1110 snap 368 seq_num 1110) 2024-08-17T17:07:15.782 INFO:tasks.rados.rados.0.smithi063.stdout:2951: finishing write tid 1 to smithi06387500-28 2024-08-17T17:07:15.782 INFO:tasks.rados.rados.0.smithi063.stdout:2951: finishing write tid 2 to smithi06387500-28 2024-08-17T17:07:15.782 INFO:tasks.rados.rados.0.smithi063.stdout:2951: finishing write tid 3 to smithi06387500-28 2024-08-17T17:07:15.783 INFO:tasks.rados.rados.0.smithi063.stdout:2951: finishing write tid 4 to smithi06387500-28 2024-08-17T17:07:15.783 INFO:tasks.rados.rados.0.smithi063.stdout:2951: oid 28 updating version 0 to 665 2024-08-17T17:07:15.783 INFO:tasks.rados.rados.0.smithi063.stdout:2951: oid 28 updating version 665 to 666 2024-08-17T17:07:15.783 INFO:tasks.rados.rados.0.smithi063.stdout:2951: oid 28 updating version 666 to 667 2024-08-17T17:07:15.783 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 667 (ObjNum 1111 snap 368 seq_num 1111) dirty exists 2024-08-17T17:07:15.783 INFO:tasks.rados.rados.0.smithi063.stdout:2951: left oid 28 (ObjNum 1111 snap 368 seq_num 1111) 2024-08-17T17:07:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:15 smithi063 ceph-mon[114756]: pgmap v666: 89 pgs: 10 peering, 5 active+undersized, 6 active+undersized+degraded, 68 active+clean; 238 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 2.2 MiB/s wr, 27 op/s; 30/462 objects degraded (6.494%) 2024-08-17T17:07:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:15 smithi063 ceph-mon[114756]: osdmap e807: 8 total, 8 up, 8 in 2024-08-17T17:07:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:15 smithi063 ceph-mon[109843]: pgmap v666: 89 pgs: 10 peering, 5 active+undersized, 6 active+undersized+degraded, 68 active+clean; 238 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 2.2 MiB/s wr, 27 op/s; 30/462 objects degraded (6.494%) 2024-08-17T17:07:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:15 smithi063 ceph-mon[109843]: osdmap e807: 8 total, 8 up, 8 in 2024-08-17T17:07:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:15 smithi162 ceph-mon[104668]: pgmap v666: 89 pgs: 10 peering, 5 active+undersized, 6 active+undersized+degraded, 68 active+clean; 238 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 2.2 MiB/s wr, 27 op/s; 30/462 objects degraded (6.494%) 2024-08-17T17:07:16.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:15 smithi162 ceph-mon[104668]: osdmap e807: 8 total, 8 up, 8 in 2024-08-17T17:07:16.672 INFO:tasks.rados.rados.0.smithi063.stdout:2950: done (3 left) 2024-08-17T17:07:16.673 INFO:tasks.rados.rados.0.smithi063.stdout:2951: done (2 left) 2024-08-17T17:07:16.673 INFO:tasks.rados.rados.0.smithi063.stdout:2952: done (1 left) 2024-08-17T17:07:16.673 INFO:tasks.rados.rados.0.smithi063.stdout:2953: done (0 left) 2024-08-17T17:07:16.673 INFO:tasks.rados.rados.0.smithi063.stdout:2954: write oid 45 current snap is 369 2024-08-17T17:07:16.673 INFO:tasks.rados.rados.0.smithi063.stdout:2954: seq_num 1112 ranges {689384=664426,1971506=687726} 2024-08-17T17:07:16.679 INFO:tasks.rados.rados.0.smithi063.stdout:2954: writing smithi06387500-45 from 689384 to 1353810 tid 1 2024-08-17T17:07:16.687 INFO:tasks.rados.rados.0.smithi063.stdout:2954: writing smithi06387500-45 from 1971506 to 2659232 tid 2 2024-08-17T17:07:16.687 INFO:tasks.rados.rados.0.smithi063.stdout:2955: write oid 4 current snap is 369 2024-08-17T17:07:16.687 INFO:tasks.rados.rados.0.smithi063.stdout:2954: finishing write tid 1 to smithi06387500-45 2024-08-17T17:07:16.687 INFO:tasks.rados.rados.0.smithi063.stdout:2955: seq_num 1113 ranges {706191=739675,1913635=431384} 2024-08-17T17:07:16.695 INFO:tasks.rados.rados.0.smithi063.stdout:2955: writing smithi06387500-4 from 706191 to 1445866 tid 1 2024-08-17T17:07:16.701 INFO:tasks.rados.rados.0.smithi063.stdout:2955: writing smithi06387500-4 from 1913635 to 2345019 tid 2 2024-08-17T17:07:16.701 INFO:tasks.rados.rados.0.smithi063.stdout:2956: read oid 26 snap -1 2024-08-17T17:07:16.701 INFO:tasks.rados.rados.0.smithi063.stdout:2956: expect (ObjNum 1079 snap 362 seq_num 1079) 2024-08-17T17:07:16.701 INFO:tasks.rados.rados.0.smithi063.stdout:2957: delete oid 13 current snap is 369 2024-08-17T17:07:16.701 INFO:tasks.rados.rados.0.smithi063.stdout:2954: finishing write tid 2 to smithi06387500-45 2024-08-17T17:07:16.701 INFO:tasks.rados.rados.0.smithi063.stdout:2954: finishing write tid 3 to smithi06387500-45 2024-08-17T17:07:16.701 INFO:tasks.rados.rados.0.smithi063.stdout:2954: finishing write tid 4 to smithi06387500-45 2024-08-17T17:07:16.701 INFO:tasks.rados.rados.0.smithi063.stdout:2954: oid 45 updating version 0 to 683 2024-08-17T17:07:16.702 INFO:tasks.rados.rados.0.smithi063.stdout:2954: oid 45 version 683 is already newer than 682 2024-08-17T17:07:16.702 INFO:tasks.rados.rados.0.smithi063.stdout:2954: oid 45 version 683 is already newer than 681 2024-08-17T17:07:16.702 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 683 (ObjNum 1112 snap 369 seq_num 1112) dirty exists 2024-08-17T17:07:16.702 INFO:tasks.rados.rados.0.smithi063.stdout:2954: left oid 45 (ObjNum 1112 snap 369 seq_num 1112) 2024-08-17T17:07:16.704 INFO:tasks.rados.rados.0.smithi063.stdout:2954: done (3 left) 2024-08-17T17:07:16.704 INFO:tasks.rados.rados.0.smithi063.stdout:2957: done (2 left) 2024-08-17T17:07:16.704 INFO:tasks.rados.rados.0.smithi063.stdout:2958: rollback oid 6 current snap is 369 2024-08-17T17:07:16.704 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 6 to 339 2024-08-17T17:07:16.704 INFO:tasks.rados.rados.0.smithi063.stdout:2959: delete oid 8 current snap is 369 2024-08-17T17:07:16.706 INFO:tasks.rados.rados.0.smithi063.stdout:2955: finishing write tid 1 to smithi06387500-4 2024-08-17T17:07:16.706 INFO:tasks.rados.rados.0.smithi063.stdout:2955: finishing write tid 2 to smithi06387500-4 2024-08-17T17:07:16.706 INFO:tasks.rados.rados.0.smithi063.stdout:2955: finishing write tid 3 to smithi06387500-4 2024-08-17T17:07:16.707 INFO:tasks.rados.rados.0.smithi063.stdout:2955: finishing write tid 4 to smithi06387500-4 2024-08-17T17:07:16.707 INFO:tasks.rados.rados.0.smithi063.stdout:2955: oid 4 updating version 0 to 2852 2024-08-17T17:07:16.707 INFO:tasks.rados.rados.0.smithi063.stdout:2955: oid 4 version 2852 is already newer than 2850 2024-08-17T17:07:16.707 INFO:tasks.rados.rados.0.smithi063.stdout:2955: oid 4 version 2852 is already newer than 2851 2024-08-17T17:07:16.707 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 2852 (ObjNum 1113 snap 369 seq_num 1113) dirty exists 2024-08-17T17:07:16.707 INFO:tasks.rados.rados.0.smithi063.stdout:2955: left oid 4 (ObjNum 1113 snap 369 seq_num 1113) 2024-08-17T17:07:16.710 INFO:tasks.rados.rados.0.smithi063.stdout:2956: expect (ObjNum 1079 snap 362 seq_num 1079) 2024-08-17T17:07:16.946 INFO:tasks.rados.rados.0.smithi063.stdout:2958: finishing rollback tid 0 to smithi06387500-6 2024-08-17T17:07:16.946 INFO:tasks.rados.rados.0.smithi063.stdout:2958: finishing rollback tid 1 to smithi06387500-6 2024-08-17T17:07:16.946 INFO:tasks.rados.rados.0.smithi063.stdout:2958: finishing rollback tid 2 to smithi06387500-6 2024-08-17T17:07:16.946 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 1007 (ObjNum 900 snap 311 seq_num 900) dirty exists 2024-08-17T17:07:16.946 INFO:tasks.rados.rados.0.smithi063.stdout:2955: done (3 left) 2024-08-17T17:07:16.946 INFO:tasks.rados.rados.0.smithi063.stdout:2956: done (2 left) 2024-08-17T17:07:16.946 INFO:tasks.rados.rados.0.smithi063.stdout:2958: done (1 left) 2024-08-17T17:07:16.946 INFO:tasks.rados.rados.0.smithi063.stdout:2959: done (0 left) 2024-08-17T17:07:16.947 INFO:tasks.rados.rados.0.smithi063.stdout:2960: delete oid 2 current snap is 369 2024-08-17T17:07:16.949 INFO:tasks.rados.rados.0.smithi063.stdout:2960: done (0 left) 2024-08-17T17:07:16.949 INFO:tasks.rados.rados.0.smithi063.stdout:2961: read oid 48 snap -1 2024-08-17T17:07:16.949 INFO:tasks.rados.rados.0.smithi063.stdout:2961: expect (ObjNum 1110 snap 368 seq_num 1110) 2024-08-17T17:07:16.949 INFO:tasks.rados.rados.0.smithi063.stdout:2962: write oid 22 current snap is 369 2024-08-17T17:07:16.949 INFO:tasks.rados.rados.0.smithi063.stdout:2962: seq_num 1117 ranges {773419=640671,2147563=438722} 2024-08-17T17:07:16.956 INFO:tasks.rados.rados.0.smithi063.stdout:2962: writing smithi06387500-22 from 773419 to 1414090 tid 1 2024-08-17T17:07:16.963 INFO:tasks.rados.rados.0.smithi063.stdout:2962: writing smithi06387500-22 from 2147563 to 2586285 tid 2 2024-08-17T17:07:16.963 INFO:tasks.rados.rados.0.smithi063.stdout:2963: snap_create 2024-08-17T17:07:16.963 INFO:tasks.rados.rados.0.smithi063.stdout:2961: expect (ObjNum 1110 snap 368 seq_num 1110) 2024-08-17T17:07:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:16 smithi063 ceph-mon[114756]: osdmap e808: 8 total, 8 up, 8 in 2024-08-17T17:07:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:16 smithi063 ceph-mon[114756]: pgmap v669: 89 pgs: 10 peering, 79 active+clean; 232 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 839 KiB/s wr, 6 op/s; 4.0 MiB/s, 4 objects/s recovering 2024-08-17T17:07:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:16 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:17.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:16 smithi063 ceph-mon[109843]: osdmap e808: 8 total, 8 up, 8 in 2024-08-17T17:07:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:16 smithi063 ceph-mon[109843]: pgmap v669: 89 pgs: 10 peering, 79 active+clean; 232 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 839 KiB/s wr, 6 op/s; 4.0 MiB/s, 4 objects/s recovering 2024-08-17T17:07:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:16 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:17.052 INFO:tasks.rados.rados.0.smithi063.stdout:2962: finishing write tid 1 to smithi06387500-22 2024-08-17T17:07:17.052 INFO:tasks.rados.rados.0.smithi063.stdout:2962: finishing write tid 2 to smithi06387500-22 2024-08-17T17:07:17.052 INFO:tasks.rados.rados.0.smithi063.stdout:2962: finishing write tid 3 to smithi06387500-22 2024-08-17T17:07:17.052 INFO:tasks.rados.rados.0.smithi063.stdout:2962: finishing write tid 4 to smithi06387500-22 2024-08-17T17:07:17.052 INFO:tasks.rados.rados.0.smithi063.stdout:2962: oid 22 updating version 0 to 669 2024-08-17T17:07:17.052 INFO:tasks.rados.rados.0.smithi063.stdout:2962: oid 22 updating version 669 to 670 2024-08-17T17:07:17.053 INFO:tasks.rados.rados.0.smithi063.stdout:2962: oid 22 updating version 670 to 671 2024-08-17T17:07:17.053 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 671 (ObjNum 1117 snap 369 seq_num 1117) dirty exists 2024-08-17T17:07:17.053 INFO:tasks.rados.rados.0.smithi063.stdout:2962: left oid 22 (ObjNum 1117 snap 369 seq_num 1117) 2024-08-17T17:07:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:16 smithi162 ceph-mon[104668]: osdmap e808: 8 total, 8 up, 8 in 2024-08-17T17:07:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:16 smithi162 ceph-mon[104668]: pgmap v669: 89 pgs: 10 peering, 79 active+clean; 232 MiB data, 793 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 839 KiB/s wr, 6 op/s; 4.0 MiB/s, 4 objects/s recovering 2024-08-17T17:07:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:16 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:17.680 INFO:tasks.rados.rados.0.smithi063.stdout:2961: done (2 left) 2024-08-17T17:07:17.680 INFO:tasks.rados.rados.0.smithi063.stdout:2962: done (1 left) 2024-08-17T17:07:17.680 INFO:tasks.rados.rados.0.smithi063.stdout:2963: done (0 left) 2024-08-17T17:07:17.680 INFO:tasks.rados.rados.0.smithi063.stdout:2964: delete oid 36 current snap is 370 2024-08-17T17:07:17.685 INFO:tasks.rados.rados.0.smithi063.stdout:2964: done (0 left) 2024-08-17T17:07:17.686 INFO:tasks.rados.rados.0.smithi063.stdout:2965: delete oid 42 current snap is 370 2024-08-17T17:07:17.690 INFO:tasks.rados.rados.0.smithi063.stdout:2965: done (0 left) 2024-08-17T17:07:17.690 INFO:tasks.rados.rados.0.smithi063.stdout:2966: write oid 47 current snap is 370 2024-08-17T17:07:17.690 INFO:tasks.rados.rados.0.smithi063.stdout:2966: seq_num 1120 ranges {423840=582771,1523009=586049,2657046=166794} 2024-08-17T17:07:17.695 INFO:tasks.rados.rados.0.smithi063.stdout:2966: writing smithi06387500-47 from 423840 to 1006611 tid 1 2024-08-17T17:07:17.701 INFO:tasks.rados.rados.0.smithi063.stdout:2966: writing smithi06387500-47 from 1523009 to 2109058 tid 2 2024-08-17T17:07:17.705 INFO:tasks.rados.rados.0.smithi063.stdout:2966: writing smithi06387500-47 from 2657046 to 2823840 tid 3 2024-08-17T17:07:17.705 INFO:tasks.rados.rados.0.smithi063.stdout:2967: write oid 20 current snap is 370 2024-08-17T17:07:17.706 INFO:tasks.rados.rados.0.smithi063.stdout:2967: seq_num 1121 ranges {440647=658020,1581491=446060,2800822=39825} 2024-08-17T17:07:17.711 INFO:tasks.rados.rados.0.smithi063.stdout:2967: writing smithi06387500-20 from 440647 to 1098667 tid 1 2024-08-17T17:07:17.716 INFO:tasks.rados.rados.0.smithi063.stdout:2967: writing smithi06387500-20 from 1581491 to 2027551 tid 2 2024-08-17T17:07:17.721 INFO:tasks.rados.rados.0.smithi063.stdout:2967: writing smithi06387500-20 from 2800822 to 2840647 tid 3 2024-08-17T17:07:17.721 INFO:tasks.rados.rados.0.smithi063.stdout:2968: read oid 50 snap -1 2024-08-17T17:07:17.722 INFO:tasks.rados.rados.0.smithi063.stdout:2968: expect (ObjNum 1088 snap 363 seq_num 1088) 2024-08-17T17:07:17.722 INFO:tasks.rados.rados.0.smithi063.stdout:2969: write oid 46 current snap is 370 2024-08-17T17:07:17.722 INFO:tasks.rados.rados.0.smithi063.stdout:2969: seq_num 1122 ranges {457454=733269,1923620=589718} 2024-08-17T17:07:17.728 INFO:tasks.rados.rados.0.smithi063.stdout:2969: writing smithi06387500-46 from 457454 to 1190723 tid 1 2024-08-17T17:07:17.736 INFO:tasks.rados.rados.0.smithi063.stdout:2969: writing smithi06387500-46 from 1923620 to 2513338 tid 2 2024-08-17T17:07:17.736 INFO:tasks.rados.rados.0.smithi063.stdout:2970: snap_create 2024-08-17T17:07:17.736 INFO:tasks.rados.rados.0.smithi063.stdout:2966: finishing write tid 1 to smithi06387500-47 2024-08-17T17:07:17.736 INFO:tasks.rados.rados.0.smithi063.stdout:2966: finishing write tid 2 to smithi06387500-47 2024-08-17T17:07:17.736 INFO:tasks.rados.rados.0.smithi063.stdout:2966: finishing write tid 3 to smithi06387500-47 2024-08-17T17:07:17.736 INFO:tasks.rados.rados.0.smithi063.stdout:2966: finishing write tid 4 to smithi06387500-47 2024-08-17T17:07:17.736 INFO:tasks.rados.rados.0.smithi063.stdout:2966: finishing write tid 5 to smithi06387500-47 2024-08-17T17:07:17.736 INFO:tasks.rados.rados.0.smithi063.stdout:2966: oid 47 updating version 0 to 623 2024-08-17T17:07:17.736 INFO:tasks.rados.rados.0.smithi063.stdout:2966: oid 47 updating version 623 to 624 2024-08-17T17:07:17.737 INFO:tasks.rados.rados.0.smithi063.stdout:2966: oid 47 version 624 is already newer than 622 2024-08-17T17:07:17.737 INFO:tasks.rados.rados.0.smithi063.stdout:2966: oid 47 updating version 624 to 625 2024-08-17T17:07:17.737 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 625 (ObjNum 1120 snap 370 seq_num 1120) dirty exists 2024-08-17T17:07:17.737 INFO:tasks.rados.rados.0.smithi063.stdout:2966: left oid 47 (ObjNum 1120 snap 370 seq_num 1120) 2024-08-17T17:07:17.737 INFO:tasks.rados.rados.0.smithi063.stdout:2967: finishing write tid 1 to smithi06387500-20 2024-08-17T17:07:17.737 INFO:tasks.rados.rados.0.smithi063.stdout:2967: finishing write tid 2 to smithi06387500-20 2024-08-17T17:07:17.737 INFO:tasks.rados.rados.0.smithi063.stdout:2967: finishing write tid 3 to smithi06387500-20 2024-08-17T17:07:17.737 INFO:tasks.rados.rados.0.smithi063.stdout:2967: finishing write tid 4 to smithi06387500-20 2024-08-17T17:07:17.737 INFO:tasks.rados.rados.0.smithi063.stdout:2967: finishing write tid 5 to smithi06387500-20 2024-08-17T17:07:17.737 INFO:tasks.rados.rados.0.smithi063.stdout:2967: oid 20 updating version 0 to 675 2024-08-17T17:07:17.738 INFO:tasks.rados.rados.0.smithi063.stdout:2967: oid 20 version 675 is already newer than 674 2024-08-17T17:07:17.738 INFO:tasks.rados.rados.0.smithi063.stdout:2967: oid 20 updating version 675 to 676 2024-08-17T17:07:17.738 INFO:tasks.rados.rados.0.smithi063.stdout:2967: oid 20 version 676 is already newer than 673 2024-08-17T17:07:17.738 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 676 (ObjNum 1121 snap 370 seq_num 1121) dirty exists 2024-08-17T17:07:17.738 INFO:tasks.rados.rados.0.smithi063.stdout:2967: left oid 20 (ObjNum 1121 snap 370 seq_num 1121) 2024-08-17T17:07:17.738 INFO:tasks.rados.rados.0.smithi063.stdout:2968: expect (ObjNum 1088 snap 363 seq_num 1088) 2024-08-17T17:07:17.836 INFO:tasks.rados.rados.0.smithi063.stdout:2969: finishing write tid 1 to smithi06387500-46 2024-08-17T17:07:17.836 INFO:tasks.rados.rados.0.smithi063.stdout:2969: finishing write tid 2 to smithi06387500-46 2024-08-17T17:07:17.836 INFO:tasks.rados.rados.0.smithi063.stdout:2969: finishing write tid 3 to smithi06387500-46 2024-08-17T17:07:17.836 INFO:tasks.rados.rados.0.smithi063.stdout:2969: finishing write tid 4 to smithi06387500-46 2024-08-17T17:07:17.836 INFO:tasks.rados.rados.0.smithi063.stdout:2969: oid 46 updating version 0 to 1042 2024-08-17T17:07:17.836 INFO:tasks.rados.rados.0.smithi063.stdout:2969: oid 46 updating version 1042 to 1043 2024-08-17T17:07:17.836 INFO:tasks.rados.rados.0.smithi063.stdout:2969: oid 46 version 1043 is already newer than 1041 2024-08-17T17:07:17.837 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 1043 (ObjNum 1122 snap 370 seq_num 1122) dirty exists 2024-08-17T17:07:17.837 INFO:tasks.rados.rados.0.smithi063.stdout:2969: left oid 46 (ObjNum 1122 snap 370 seq_num 1122) 2024-08-17T17:07:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:17 smithi063 ceph-mon[114756]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/462 objects degraded (6.494%), 6 pgs degraded) 2024-08-17T17:07:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:17 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:07:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:17 smithi063 ceph-mon[114756]: osdmap e809: 8 total, 8 up, 8 in 2024-08-17T17:07:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:17 smithi063 ceph-mon[109843]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/462 objects degraded (6.494%), 6 pgs degraded) 2024-08-17T17:07:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:17 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:07:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:17 smithi063 ceph-mon[109843]: osdmap e809: 8 total, 8 up, 8 in 2024-08-17T17:07:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:17 smithi162 ceph-mon[104668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/462 objects degraded (6.494%), 6 pgs degraded) 2024-08-17T17:07:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:17 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:07:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:17 smithi162 ceph-mon[104668]: osdmap e809: 8 total, 8 up, 8 in 2024-08-17T17:07:18.684 INFO:tasks.rados.rados.0.smithi063.stdout:2966: done (4 left) 2024-08-17T17:07:18.684 INFO:tasks.rados.rados.0.smithi063.stdout:2967: done (3 left) 2024-08-17T17:07:18.684 INFO:tasks.rados.rados.0.smithi063.stdout:2968: done (2 left) 2024-08-17T17:07:18.685 INFO:tasks.rados.rados.0.smithi063.stdout:2969: done (1 left) 2024-08-17T17:07:18.685 INFO:tasks.rados.rados.0.smithi063.stdout:2970: done (0 left) 2024-08-17T17:07:18.685 INFO:tasks.rados.rados.0.smithi063.stdout:2971: snap_remove snap 365 2024-08-17T17:07:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:18 smithi063 ceph-mon[114756]: osdmap e810: 8 total, 8 up, 8 in 2024-08-17T17:07:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:18 smithi063 ceph-mon[114756]: pgmap v672: 89 pgs: 89 active+clean; 235 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.9 MiB/s wr, 16 op/s; 4.6 MiB/s, 4 objects/s recovering 2024-08-17T17:07:19.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:18 smithi063 ceph-mon[109843]: osdmap e810: 8 total, 8 up, 8 in 2024-08-17T17:07:19.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:18 smithi063 ceph-mon[109843]: pgmap v672: 89 pgs: 89 active+clean; 235 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.9 MiB/s wr, 16 op/s; 4.6 MiB/s, 4 objects/s recovering 2024-08-17T17:07:19.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:18 smithi162 ceph-mon[104668]: osdmap e810: 8 total, 8 up, 8 in 2024-08-17T17:07:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:18 smithi162 ceph-mon[104668]: pgmap v672: 89 pgs: 89 active+clean; 235 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 1.9 MiB/s wr, 16 op/s; 4.6 MiB/s, 4 objects/s recovering 2024-08-17T17:07:19.685 INFO:tasks.rados.rados.0.smithi063.stdout:2971: done (0 left) 2024-08-17T17:07:19.685 INFO:tasks.rados.rados.0.smithi063.stdout:2972: read oid 11 snap -1 2024-08-17T17:07:19.686 INFO:tasks.rados.rados.0.smithi063.stdout:2972: expect deleted 2024-08-17T17:07:19.686 INFO:tasks.rados.rados.0.smithi063.stdout:2973: rollback oid 23 current snap is 371 2024-08-17T17:07:19.686 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 23 to 366 2024-08-17T17:07:19.686 INFO:tasks.rados.rados.0.smithi063.stdout:2974: snap_create 2024-08-17T17:07:19.695 INFO:tasks.rados.rados.0.smithi063.stdout:2973: finishing rollback tid 0 to smithi06387500-23 2024-08-17T17:07:19.695 INFO:tasks.rados.rados.0.smithi063.stdout:2973: finishing rollback tid 1 to smithi06387500-23 2024-08-17T17:07:19.695 INFO:tasks.rados.rados.0.smithi063.stdout:2973: finishing rollback tid 2 to smithi06387500-23 2024-08-17T17:07:19.695 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 1449 (ObjNum 1093 snap 363 seq_num 1093) dirty exists 2024-08-17T17:07:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:19 smithi063 ceph-mon[114756]: osdmap e811: 8 total, 8 up, 8 in 2024-08-17T17:07:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:19 smithi063 ceph-mon[109843]: osdmap e811: 8 total, 8 up, 8 in 2024-08-17T17:07:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:19 smithi162 ceph-mon[104668]: osdmap e811: 8 total, 8 up, 8 in 2024-08-17T17:07:20.698 INFO:tasks.rados.rados.0.smithi063.stdout:2972: done (2 left) 2024-08-17T17:07:20.699 INFO:tasks.rados.rados.0.smithi063.stdout:2973: done (1 left) 2024-08-17T17:07:20.699 INFO:tasks.rados.rados.0.smithi063.stdout:2974: done (0 left) 2024-08-17T17:07:20.699 INFO:tasks.rados.rados.0.smithi063.stdout:2975: rollback oid 9 current snap is 372 2024-08-17T17:07:20.699 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 9 to 368 2024-08-17T17:07:20.699 INFO:tasks.rados.rados.0.smithi063.stdout:2976: write oid 29 current snap is 372 2024-08-17T17:07:20.699 INFO:tasks.rados.rados.0.smithi063.stdout:2976: seq_num 1123 ranges {474261=408518,1582102=449729,2739315=134946} 2024-08-17T17:07:20.704 INFO:tasks.rados.rados.0.smithi063.stdout:2976: writing smithi06387500-29 from 474261 to 882779 tid 1 2024-08-17T17:07:20.710 INFO:tasks.rados.rados.0.smithi063.stdout:2976: writing smithi06387500-29 from 1582102 to 2031831 tid 2 2024-08-17T17:07:20.715 INFO:tasks.rados.rados.0.smithi063.stdout:2976: writing smithi06387500-29 from 2739315 to 2874261 tid 3 2024-08-17T17:07:20.715 INFO:tasks.rados.rados.0.smithi063.stdout:2977: write oid 44 current snap is 372 2024-08-17T17:07:20.715 INFO:tasks.rados.rados.0.smithi063.stdout:2977: seq_num 1124 ranges {491068=483767,1640584=593387,2650385=240683} 2024-08-17T17:07:20.720 INFO:tasks.rados.rados.0.smithi063.stdout:2977: writing smithi06387500-44 from 491068 to 974835 tid 1 2024-08-17T17:07:20.727 INFO:tasks.rados.rados.0.smithi063.stdout:2977: writing smithi06387500-44 from 1640584 to 2233971 tid 2 2024-08-17T17:07:20.731 INFO:tasks.rados.rados.0.smithi063.stdout:2977: writing smithi06387500-44 from 2650385 to 2891068 tid 3 2024-08-17T17:07:20.732 INFO:tasks.rados.rados.0.smithi063.stdout:2978: snap_create 2024-08-17T17:07:20.732 INFO:tasks.rados.rados.0.smithi063.stdout:2975: finishing rollback tid 0 to smithi06387500-9 2024-08-17T17:07:20.732 INFO:tasks.rados.rados.0.smithi063.stdout:2975: finishing rollback tid 1 to smithi06387500-9 2024-08-17T17:07:20.732 INFO:tasks.rados.rados.0.smithi063.stdout:2975: finishing rollback tid 2 to smithi06387500-9 2024-08-17T17:07:20.732 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 1381 (ObjNum 1048 snap 354 seq_num 1048) dirty exists 2024-08-17T17:07:20.732 INFO:tasks.rados.rados.0.smithi063.stdout:2976: finishing write tid 1 to smithi06387500-29 2024-08-17T17:07:20.732 INFO:tasks.rados.rados.0.smithi063.stdout:2976: finishing write tid 2 to smithi06387500-29 2024-08-17T17:07:20.732 INFO:tasks.rados.rados.0.smithi063.stdout:2976: finishing write tid 3 to smithi06387500-29 2024-08-17T17:07:20.732 INFO:tasks.rados.rados.0.smithi063.stdout:2976: finishing write tid 4 to smithi06387500-29 2024-08-17T17:07:20.732 INFO:tasks.rados.rados.0.smithi063.stdout:2976: finishing write tid 5 to smithi06387500-29 2024-08-17T17:07:20.732 INFO:tasks.rados.rados.0.smithi063.stdout:2976: oid 29 updating version 0 to 668 2024-08-17T17:07:20.733 INFO:tasks.rados.rados.0.smithi063.stdout:2976: oid 29 version 668 is already newer than 667 2024-08-17T17:07:20.733 INFO:tasks.rados.rados.0.smithi063.stdout:2976: oid 29 updating version 668 to 669 2024-08-17T17:07:20.733 INFO:tasks.rados.rados.0.smithi063.stdout:2976: oid 29 version 669 is already newer than 666 2024-08-17T17:07:20.733 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 669 (ObjNum 1123 snap 372 seq_num 1123) dirty exists 2024-08-17T17:07:20.733 INFO:tasks.rados.rados.0.smithi063.stdout:2976: left oid 29 (ObjNum 1123 snap 372 seq_num 1123) 2024-08-17T17:07:20.733 INFO:tasks.rados.rados.0.smithi063.stdout:2977: finishing write tid 1 to smithi06387500-44 2024-08-17T17:07:20.733 INFO:tasks.rados.rados.0.smithi063.stdout:2977: finishing write tid 2 to smithi06387500-44 2024-08-17T17:07:20.734 INFO:tasks.rados.rados.0.smithi063.stdout:2977: finishing write tid 3 to smithi06387500-44 2024-08-17T17:07:20.734 INFO:tasks.rados.rados.0.smithi063.stdout:2977: finishing write tid 4 to smithi06387500-44 2024-08-17T17:07:20.734 INFO:tasks.rados.rados.0.smithi063.stdout:2977: finishing write tid 5 to smithi06387500-44 2024-08-17T17:07:20.734 INFO:tasks.rados.rados.0.smithi063.stdout:2977: oid 44 updating version 0 to 392 2024-08-17T17:07:20.734 INFO:tasks.rados.rados.0.smithi063.stdout:2977: oid 44 version 392 is already newer than 390 2024-08-17T17:07:20.734 INFO:tasks.rados.rados.0.smithi063.stdout:2977: oid 44 version 392 is already newer than 391 2024-08-17T17:07:20.734 INFO:tasks.rados.rados.0.smithi063.stdout:2977: oid 44 updating version 392 to 393 2024-08-17T17:07:20.734 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 393 (ObjNum 1124 snap 372 seq_num 1124) dirty exists 2024-08-17T17:07:20.734 INFO:tasks.rados.rados.0.smithi063.stdout:2977: left oid 44 (ObjNum 1124 snap 372 seq_num 1124) 2024-08-17T17:07:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:20 smithi063 ceph-mon[114756]: osdmap e812: 8 total, 8 up, 8 in 2024-08-17T17:07:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:20 smithi063 ceph-mon[114756]: pgmap v675: 89 pgs: 89 active+clean; 235 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.1 MiB/s wr, 10 op/s; 613 KiB/s, 0 objects/s recovering 2024-08-17T17:07:21.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:20 smithi063 ceph-mon[109843]: osdmap e812: 8 total, 8 up, 8 in 2024-08-17T17:07:21.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:20 smithi063 ceph-mon[109843]: pgmap v675: 89 pgs: 89 active+clean; 235 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.1 MiB/s wr, 10 op/s; 613 KiB/s, 0 objects/s recovering 2024-08-17T17:07:21.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:20 smithi162 ceph-mon[104668]: osdmap e812: 8 total, 8 up, 8 in 2024-08-17T17:07:21.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:20 smithi162 ceph-mon[104668]: pgmap v675: 89 pgs: 89 active+clean; 235 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.1 MiB/s wr, 10 op/s; 613 KiB/s, 0 objects/s recovering 2024-08-17T17:07:21.703 INFO:tasks.rados.rados.0.smithi063.stdout:2975: done (3 left) 2024-08-17T17:07:21.703 INFO:tasks.rados.rados.0.smithi063.stdout:2976: done (2 left) 2024-08-17T17:07:21.704 INFO:tasks.rados.rados.0.smithi063.stdout:2977: done (1 left) 2024-08-17T17:07:21.704 INFO:tasks.rados.rados.0.smithi063.stdout:2978: done (0 left) 2024-08-17T17:07:21.704 INFO:tasks.rados.rados.0.smithi063.stdout:2979: snap_remove snap 368 2024-08-17T17:07:22.022 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:21 smithi063 ceph-mon[114756]: osdmap e813: 8 total, 8 up, 8 in 2024-08-17T17:07:22.023 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:21 smithi063 ceph-mon[109843]: osdmap e813: 8 total, 8 up, 8 in 2024-08-17T17:07:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:21 smithi162 ceph-mon[104668]: osdmap e813: 8 total, 8 up, 8 in 2024-08-17T17:07:22.626 INFO:tasks.rados.rados.0.smithi063.stdout:2979: done (0 left) 2024-08-17T17:07:22.626 INFO:tasks.rados.rados.0.smithi063.stdout:2980: snap_create 2024-08-17T17:07:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:22 smithi063 ceph-mon[114756]: osdmap e814: 8 total, 8 up, 8 in 2024-08-17T17:07:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:22 smithi063 ceph-mon[114756]: pgmap v678: 89 pgs: 89 active+clean; 243 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-08-17T17:07:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:22 smithi063 ceph-mon[114756]: osdmap e815: 8 total, 8 up, 8 in 2024-08-17T17:07:23.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:22 smithi063 ceph-mon[109843]: osdmap e814: 8 total, 8 up, 8 in 2024-08-17T17:07:23.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:22 smithi063 ceph-mon[109843]: pgmap v678: 89 pgs: 89 active+clean; 243 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-08-17T17:07:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:22 smithi063 ceph-mon[109843]: osdmap e815: 8 total, 8 up, 8 in 2024-08-17T17:07:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:22 smithi162 ceph-mon[104668]: osdmap e814: 8 total, 8 up, 8 in 2024-08-17T17:07:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:22 smithi162 ceph-mon[104668]: pgmap v678: 89 pgs: 89 active+clean; 243 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-08-17T17:07:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:22 smithi162 ceph-mon[104668]: osdmap e815: 8 total, 8 up, 8 in 2024-08-17T17:07:23.178 INFO:teuthology.orchestra.run.smithi063.stdout:true 2024-08-17T17:07:23.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:07:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:07:23] "GET /metrics HTTP/1.1" 200 36355 "" "Prometheus/2.51.0" 2024-08-17T17:07:23.624 INFO:tasks.rados.rados.0.smithi063.stdout:2980: done (0 left) 2024-08-17T17:07:23.624 INFO:tasks.rados.rados.0.smithi063.stdout:2981: rollback oid 7 current snap is 374 2024-08-17T17:07:23.625 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 366 2024-08-17T17:07:23.625 INFO:tasks.rados.rados.0.smithi063.stdout:2982: write oid 14 current snap is 374 2024-08-17T17:07:23.625 INFO:tasks.rados.rados.0.smithi063.stdout:2982: seq_num 1125 ranges {507875=559016,1699066=737045} 2024-08-17T17:07:23.631 INFO:tasks.rados.rados.0.smithi063.stdout:2982: writing smithi06387500-14 from 507875 to 1066891 tid 1 2024-08-17T17:07:23.641 INFO:tasks.rados.rados.0.smithi063.stdout:2982: writing smithi06387500-14 from 1699066 to 2436111 tid 2 2024-08-17T17:07:23.641 INFO:tasks.rados.rados.0.smithi063.stdout:2983: read oid 39 snap -1 2024-08-17T17:07:23.641 INFO:tasks.rados.rados.0.smithi063.stdout:2983: expect (ObjNum 1092 snap 363 seq_num 1092) 2024-08-17T17:07:23.641 INFO:tasks.rados.rados.0.smithi063.stdout:2984: delete oid 41 current snap is 374 2024-08-17T17:07:23.641 INFO:tasks.rados.rados.0.smithi063.stdout:2981: finishing rollback tid 0 to smithi06387500-7 2024-08-17T17:07:23.641 INFO:tasks.rados.rados.0.smithi063.stdout:2981: finishing rollback tid 1 to smithi06387500-7 2024-08-17T17:07:23.642 INFO:tasks.rados.rados.0.smithi063.stdout:2981: finishing rollback tid 2 to smithi06387500-7 2024-08-17T17:07:23.642 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1387 (ObjNum 1069 snap 361 seq_num 1069) dirty exists 2024-08-17T17:07:23.642 INFO:tasks.rados.rados.0.smithi063.stdout:2982: finishing write tid 1 to smithi06387500-14 2024-08-17T17:07:23.647 INFO:tasks.rados.rados.0.smithi063.stdout:2982: finishing write tid 2 to smithi06387500-14 2024-08-17T17:07:23.647 INFO:tasks.rados.rados.0.smithi063.stdout:2982: finishing write tid 3 to smithi06387500-14 2024-08-17T17:07:23.648 INFO:tasks.rados.rados.0.smithi063.stdout:2982: finishing write tid 4 to smithi06387500-14 2024-08-17T17:07:23.648 INFO:tasks.rados.rados.0.smithi063.stdout:2982: oid 14 updating version 0 to 2861 2024-08-17T17:07:23.648 INFO:tasks.rados.rados.0.smithi063.stdout:2982: oid 14 version 2861 is already newer than 2859 2024-08-17T17:07:23.648 INFO:tasks.rados.rados.0.smithi063.stdout:2982: oid 14 version 2861 is already newer than 2860 2024-08-17T17:07:23.648 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 2861 (ObjNum 1125 snap 374 seq_num 1125) dirty exists 2024-08-17T17:07:23.648 INFO:tasks.rados.rados.0.smithi063.stdout:2982: left oid 14 (ObjNum 1125 snap 374 seq_num 1125) 2024-08-17T17:07:23.649 INFO:tasks.rados.rados.0.smithi063.stdout:2981: done (3 left) 2024-08-17T17:07:23.649 INFO:tasks.rados.rados.0.smithi063.stdout:2982: done (2 left) 2024-08-17T17:07:23.649 INFO:tasks.rados.rados.0.smithi063.stdout:2984: done (1 left) 2024-08-17T17:07:23.649 INFO:tasks.rados.rados.0.smithi063.stdout:2985: write oid 29 current snap is 374 2024-08-17T17:07:23.649 INFO:tasks.rados.rados.0.smithi063.stdout:2983: expect (ObjNum 1092 snap 363 seq_num 1092) 2024-08-17T17:07:23.770 INFO:teuthology.orchestra.run.smithi063.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-08-17T17:07:23.771 INFO:teuthology.orchestra.run.smithi063.stdout:alertmanager.a smithi063 *:9093,9094 running (8m) 15s ago 14m 19.1M - 0.27.0 11f11916f8cd 59f4018b91ca 2024-08-17T17:07:23.771 INFO:teuthology.orchestra.run.smithi063.stdout:grafana.a smithi162 *:3000 running (8m) 47s ago 14m 79.0M - c8b91775d855 581e03e654a8 2024-08-17T17:07:23.771 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi063.ultpqj smithi063 running (13m) 15s ago 13m 23.6M - 18.2.4-719-gced649f8 c870753d67c2 c4400e5c891e 2024-08-17T17:07:23.771 INFO:teuthology.orchestra.run.smithi063.stdout:mds.foo.smithi162.bjkxnv smithi162 running (13m) 47s ago 13m 19.5M - 18.2.4-719-gced649f8 c870753d67c2 a01c12d382e9 2024-08-17T17:07:23.771 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.x smithi162 *:8443,9283,8765 running (9m) 47s ago 18m 457M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb da48034a2895 2024-08-17T17:07:23.771 INFO:teuthology.orchestra.run.smithi063.stdout:mgr.y smithi063 *:8443,9283,8765 running (10m) 15s ago 20m 519M - 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 943e639cede6 2024-08-17T17:07:23.771 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a smithi063 running (7m) 15s ago 20m 73.5M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb f0f20fcac956 2024-08-17T17:07:23.771 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b smithi162 running (5m) 47s ago 19m 43.5M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 520525c79dbe 2024-08-17T17:07:23.771 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c smithi063 running (7m) 15s ago 18m 57.9M 2048M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb c484af026acf 2024-08-17T17:07:23.771 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.a smithi063 *:9100 running (9m) 15s ago 14m 10.2M - 1.7.0 72c9c2088986 34f55f5de2aa 2024-08-17T17:07:23.771 INFO:teuthology.orchestra.run.smithi063.stdout:node-exporter.b smithi162 *:9100 running (9m) 47s ago 14m 11.8M - 1.7.0 72c9c2088986 5bb162d0f1a9 2024-08-17T17:07:23.772 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 smithi063 running (84s) 15s ago 18m 77.1M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 8fa117adbc0a 2024-08-17T17:07:23.772 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 smithi063 running (52s) 15s ago 18m 67.7M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 11bee521ecad 2024-08-17T17:07:23.772 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 smithi063 running (19s) 15s ago 17m 14.4M 2480M 19.0.0-5836-gcd8bdbeb 0566dfeff0cb 107c38f87994 2024-08-17T17:07:23.772 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 smithi063 running (17m) 15s ago 17m 99.6M 2480M 18.2.4-719-gced649f8 c870753d67c2 9da1dbb0ca65 2024-08-17T17:07:23.772 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 smithi162 running (16m) 47s ago 16m 134M 2480M 18.2.4-719-gced649f8 c870753d67c2 a978a99d5919 2024-08-17T17:07:23.772 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 smithi162 running (16m) 47s ago 16m 121M 2480M 18.2.4-719-gced649f8 c870753d67c2 c66546db8132 2024-08-17T17:07:23.772 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 smithi162 running (16m) 47s ago 16m 90.4M 2480M 18.2.4-719-gced649f8 c870753d67c2 af91211ce044 2024-08-17T17:07:23.772 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 smithi162 running (15m) 47s ago 15m 101M 2480M 18.2.4-719-gced649f8 c870753d67c2 b8c383163607 2024-08-17T17:07:23.772 INFO:teuthology.orchestra.run.smithi063.stdout:prometheus.a smithi162 *:9095 running (9m) 47s ago 15m 48.0M - 2.51.0 1d3b7f56885b dfe66b2c5568 2024-08-17T17:07:23.784 INFO:tasks.rados.rados.0.smithi063.stdout:2985: seq_num 1127 ranges {541489=425867,1416030=740714,2732654=208835} 2024-08-17T17:07:23.789 INFO:tasks.rados.rados.0.smithi063.stdout:2985: writing smithi06387500-29 from 541489 to 967356 tid 1 2024-08-17T17:07:23.796 INFO:tasks.rados.rados.0.smithi063.stdout:2985: writing smithi06387500-29 from 1416030 to 2156744 tid 2 2024-08-17T17:07:23.801 INFO:tasks.rados.rados.0.smithi063.stdout:2985: writing smithi06387500-29 from 2732654 to 2941489 tid 3 2024-08-17T17:07:23.801 INFO:tasks.rados.rados.0.smithi063.stdout:2983: done (1 left) 2024-08-17T17:07:23.801 INFO:tasks.rados.rados.0.smithi063.stdout:2986: write oid 36 current snap is 374 2024-08-17T17:07:23.801 INFO:tasks.rados.rados.0.smithi063.stdout:2986: seq_num 1128 ranges {558296=501116,1474512=600725,2760077=198219} 2024-08-17T17:07:23.806 INFO:tasks.rados.rados.0.smithi063.stdout:2986: writing smithi06387500-36 from 558296 to 1059412 tid 1 2024-08-17T17:07:23.812 INFO:tasks.rados.rados.0.smithi063.stdout:2986: writing smithi06387500-36 from 1474512 to 2075237 tid 2 2024-08-17T17:07:23.817 INFO:tasks.rados.rados.0.smithi063.stdout:2986: writing smithi06387500-36 from 2760077 to 2958296 tid 3 2024-08-17T17:07:23.817 INFO:tasks.rados.rados.0.smithi063.stdout:2987: snap_create 2024-08-17T17:07:23.817 INFO:tasks.rados.rados.0.smithi063.stdout:2985: finishing write tid 1 to smithi06387500-29 2024-08-17T17:07:23.817 INFO:tasks.rados.rados.0.smithi063.stdout:2985: finishing write tid 2 to smithi06387500-29 2024-08-17T17:07:23.817 INFO:tasks.rados.rados.0.smithi063.stdout:2985: finishing write tid 3 to smithi06387500-29 2024-08-17T17:07:23.818 INFO:tasks.rados.rados.0.smithi063.stdout:2985: finishing write tid 4 to smithi06387500-29 2024-08-17T17:07:23.818 INFO:tasks.rados.rados.0.smithi063.stdout:2985: finishing write tid 5 to smithi06387500-29 2024-08-17T17:07:23.818 INFO:tasks.rados.rados.0.smithi063.stdout:2985: oid 29 updating version 0 to 676 2024-08-17T17:07:23.818 INFO:tasks.rados.rados.0.smithi063.stdout:2985: oid 29 version 676 is already newer than 673 2024-08-17T17:07:23.818 INFO:tasks.rados.rados.0.smithi063.stdout:2985: oid 29 version 676 is already newer than 675 2024-08-17T17:07:23.818 INFO:tasks.rados.rados.0.smithi063.stdout:2985: oid 29 version 676 is already newer than 674 2024-08-17T17:07:23.818 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 676 (ObjNum 1127 snap 374 seq_num 1127) dirty exists 2024-08-17T17:07:23.818 INFO:tasks.rados.rados.0.smithi063.stdout:2985: left oid 29 (ObjNum 1127 snap 374 seq_num 1127) 2024-08-17T17:07:23.818 INFO:tasks.rados.rados.0.smithi063.stdout:2986: finishing write tid 1 to smithi06387500-36 2024-08-17T17:07:23.818 INFO:tasks.rados.rados.0.smithi063.stdout:2986: finishing write tid 2 to smithi06387500-36 2024-08-17T17:07:23.820 INFO:tasks.rados.rados.0.smithi063.stdout:2986: finishing write tid 3 to smithi06387500-36 2024-08-17T17:07:23.820 INFO:tasks.rados.rados.0.smithi063.stdout:2986: finishing write tid 4 to smithi06387500-36 2024-08-17T17:07:23.820 INFO:tasks.rados.rados.0.smithi063.stdout:2986: finishing write tid 5 to smithi06387500-36 2024-08-17T17:07:23.821 INFO:tasks.rados.rados.0.smithi063.stdout:2986: oid 36 updating version 0 to 2865 2024-08-17T17:07:23.821 INFO:tasks.rados.rados.0.smithi063.stdout:2986: oid 36 version 2865 is already newer than 2864 2024-08-17T17:07:23.821 INFO:tasks.rados.rados.0.smithi063.stdout:2986: oid 36 version 2865 is already newer than 2863 2024-08-17T17:07:23.821 INFO:tasks.rados.rados.0.smithi063.stdout:2986: oid 36 version 2865 is already newer than 2862 2024-08-17T17:07:23.821 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 2865 (ObjNum 1128 snap 374 seq_num 1128) dirty exists 2024-08-17T17:07:23.821 INFO:tasks.rados.rados.0.smithi063.stdout:2986: left oid 36 (ObjNum 1128 snap 374 seq_num 1128) 2024-08-17T17:07:24.125 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:07:24.125 INFO:teuthology.orchestra.run.smithi063.stdout: "mon": { 2024-08-17T17:07:24.125 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:07:24.125 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:07:24.125 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr": { 2024-08-17T17:07:24.125 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 2 2024-08-17T17:07:24.125 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:07:24.125 INFO:teuthology.orchestra.run.smithi063.stdout: "osd": { 2024-08-17T17:07:24.125 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 5, 2024-08-17T17:07:24.126 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 3 2024-08-17T17:07:24.126 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:07:24.126 INFO:teuthology.orchestra.run.smithi063.stdout: "mds": { 2024-08-17T17:07:24.126 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 2 2024-08-17T17:07:24.126 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:07:24.126 INFO:teuthology.orchestra.run.smithi063.stdout: "overall": { 2024-08-17T17:07:24.126 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 18.2.4-719-gced649f8 (ced649f89259a8c5733cc18a50f42f20f41ba352) reef (stable)": 7, 2024-08-17T17:07:24.126 INFO:teuthology.orchestra.run.smithi063.stdout: "ceph version 19.0.0-5836-gcd8bdbeb (cd8bdbeb9fbdab5733aa3aaadc042c965473e453) squid (dev)": 8 2024-08-17T17:07:24.126 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:07:24.126 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:07:24.423 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:07:24.424 INFO:teuthology.orchestra.run.smithi063.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:07:24.424 INFO:teuthology.orchestra.run.smithi063.stdout: "in_progress": true, 2024-08-17T17:07:24.424 INFO:teuthology.orchestra.run.smithi063.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (1 remaining).", 2024-08-17T17:07:24.424 INFO:teuthology.orchestra.run.smithi063.stdout: "services_complete": [], 2024-08-17T17:07:24.424 INFO:teuthology.orchestra.run.smithi063.stdout: "progress": "3/8 daemons upgraded", 2024-08-17T17:07:24.424 INFO:teuthology.orchestra.run.smithi063.stdout: "message": "Currently upgrading osd daemons", 2024-08-17T17:07:24.424 INFO:teuthology.orchestra.run.smithi063.stdout: "is_paused": false 2024-08-17T17:07:24.424 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:07:24.630 INFO:tasks.rados.rados.0.smithi063.stdout:2985: done (2 left) 2024-08-17T17:07:24.630 INFO:tasks.rados.rados.0.smithi063.stdout:2986: done (1 left) 2024-08-17T17:07:24.630 INFO:tasks.rados.rados.0.smithi063.stdout:2987: done (0 left) 2024-08-17T17:07:24.630 INFO:tasks.rados.rados.0.smithi063.stdout:2988: rollback oid 7 current snap is 375 2024-08-17T17:07:24.631 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 373 2024-08-17T17:07:24.631 INFO:tasks.rados.rados.0.smithi063.stdout:2989: write oid 26 current snap is 375 2024-08-17T17:07:24.631 INFO:tasks.rados.rados.0.smithi063.stdout:2989: seq_num 1129 ranges {575103=576365,1932994=744383} 2024-08-17T17:07:24.637 INFO:tasks.rados.rados.0.smithi063.stdout:2989: writing smithi06387500-26 from 575103 to 1151468 tid 1 2024-08-17T17:07:24.645 INFO:tasks.rados.rados.0.smithi063.stdout:2989: writing smithi06387500-26 from 1932994 to 2677377 tid 2 2024-08-17T17:07:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:2990: snap_create 2024-08-17T17:07:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:2988: finishing rollback tid 0 to smithi06387500-7 2024-08-17T17:07:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:2988: finishing rollback tid 1 to smithi06387500-7 2024-08-17T17:07:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:2988: finishing rollback tid 2 to smithi06387500-7 2024-08-17T17:07:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1391 (ObjNum 1069 snap 361 seq_num 1069) dirty exists 2024-08-17T17:07:24.646 INFO:tasks.rados.rados.0.smithi063.stdout:2989: finishing write tid 1 to smithi06387500-26 2024-08-17T17:07:24.650 INFO:tasks.rados.rados.0.smithi063.stdout:2989: finishing write tid 2 to smithi06387500-26 2024-08-17T17:07:24.650 INFO:tasks.rados.rados.0.smithi063.stdout:2989: finishing write tid 3 to smithi06387500-26 2024-08-17T17:07:24.650 INFO:tasks.rados.rados.0.smithi063.stdout:2989: finishing write tid 4 to smithi06387500-26 2024-08-17T17:07:24.650 INFO:tasks.rados.rados.0.smithi063.stdout:2989: oid 26 updating version 0 to 993 2024-08-17T17:07:24.650 INFO:tasks.rados.rados.0.smithi063.stdout:2989: oid 26 updating version 993 to 994 2024-08-17T17:07:24.650 INFO:tasks.rados.rados.0.smithi063.stdout:2989: oid 26 updating version 994 to 995 2024-08-17T17:07:24.650 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 995 (ObjNum 1129 snap 375 seq_num 1129) dirty exists 2024-08-17T17:07:24.650 INFO:tasks.rados.rados.0.smithi063.stdout:2989: left oid 26 (ObjNum 1129 snap 375 seq_num 1129) 2024-08-17T17:07:24.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:24 smithi063 ceph-mon[114756]: from='client.44336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:24.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:24 smithi063 ceph-mon[114756]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:24.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:24 smithi063 ceph-mon[114756]: osdmap e816: 8 total, 8 up, 8 in 2024-08-17T17:07:24.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:24 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/176860038' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:24.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:24 smithi063 ceph-mon[109843]: from='client.44336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:24.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:24 smithi063 ceph-mon[109843]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:24.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:24 smithi063 ceph-mon[109843]: osdmap e816: 8 total, 8 up, 8 in 2024-08-17T17:07:24.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:24 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/176860038' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:25.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:24 smithi162 ceph-mon[104668]: from='client.44336 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:25.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:24 smithi162 ceph-mon[104668]: from='client.34413 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:25.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:24 smithi162 ceph-mon[104668]: osdmap e816: 8 total, 8 up, 8 in 2024-08-17T17:07:25.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:24 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/176860038' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:25.636 INFO:tasks.rados.rados.0.smithi063.stdout:2988: done (2 left) 2024-08-17T17:07:25.636 INFO:tasks.rados.rados.0.smithi063.stdout:2989: done (1 left) 2024-08-17T17:07:25.637 INFO:tasks.rados.rados.0.smithi063.stdout:2990: done (0 left) 2024-08-17T17:07:25.637 INFO:tasks.rados.rados.0.smithi063.stdout:2991: rollback oid 2 current snap is 376 2024-08-17T17:07:25.637 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 2 to 374 2024-08-17T17:07:25.637 INFO:tasks.rados.rados.0.smithi063.stdout:2992: snap_create 2024-08-17T17:07:25.639 INFO:tasks.rados.rados.0.smithi063.stdout:2991: finishing rollback tid 1 to smithi06387500-2 2024-08-17T17:07:25.639 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 1445 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T17:07:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:25 smithi063 ceph-mon[114756]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:25 smithi063 ceph-mon[114756]: pgmap v681: 89 pgs: 89 active+clean; 243 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-08-17T17:07:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:25 smithi063 ceph-mon[114756]: from='client.44360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:25 smithi063 ceph-mon[114756]: osdmap e817: 8 total, 8 up, 8 in 2024-08-17T17:07:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:25 smithi063 ceph-mon[109843]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:25 smithi063 ceph-mon[109843]: pgmap v681: 89 pgs: 89 active+clean; 243 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-08-17T17:07:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:25 smithi063 ceph-mon[109843]: from='client.44360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:25 smithi063 ceph-mon[109843]: osdmap e817: 8 total, 8 up, 8 in 2024-08-17T17:07:26.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:25 smithi162 ceph-mon[104668]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:25 smithi162 ceph-mon[104668]: pgmap v681: 89 pgs: 89 active+clean; 243 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.7 MiB/s wr, 14 op/s 2024-08-17T17:07:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:25 smithi162 ceph-mon[104668]: from='client.44360 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:25 smithi162 ceph-mon[104668]: osdmap e817: 8 total, 8 up, 8 in 2024-08-17T17:07:26.642 INFO:tasks.rados.rados.0.smithi063.stdout:2991: done (1 left) 2024-08-17T17:07:26.642 INFO:tasks.rados.rados.0.smithi063.stdout:2992: done (0 left) 2024-08-17T17:07:26.643 INFO:tasks.rados.rados.0.smithi063.stdout:2993: write oid 18 current snap is 377 2024-08-17T17:07:26.643 INFO:tasks.rados.rados.0.smithi063.stdout:2993: seq_num 1130 ranges {591910=651614,1875123=604394} 2024-08-17T17:07:26.649 INFO:tasks.rados.rados.0.smithi063.stdout:2993: writing smithi06387500-18 from 591910 to 1243524 tid 1 2024-08-17T17:07:26.656 INFO:tasks.rados.rados.0.smithi063.stdout:2993: writing smithi06387500-18 from 1875123 to 2479517 tid 2 2024-08-17T17:07:26.656 INFO:tasks.rados.rados.0.smithi063.stdout:2994: delete oid 49 current snap is 377 2024-08-17T17:07:26.656 INFO:tasks.rados.rados.0.smithi063.stdout:2993: finishing write tid 1 to smithi06387500-18 2024-08-17T17:07:26.660 INFO:tasks.rados.rados.0.smithi063.stdout:2993: finishing write tid 2 to smithi06387500-18 2024-08-17T17:07:26.660 INFO:tasks.rados.rados.0.smithi063.stdout:2993: finishing write tid 3 to smithi06387500-18 2024-08-17T17:07:26.660 INFO:tasks.rados.rados.0.smithi063.stdout:2993: finishing write tid 4 to smithi06387500-18 2024-08-17T17:07:26.660 INFO:tasks.rados.rados.0.smithi063.stdout:2993: oid 18 updating version 0 to 981 2024-08-17T17:07:26.660 INFO:tasks.rados.rados.0.smithi063.stdout:2993: oid 18 updating version 981 to 982 2024-08-17T17:07:26.660 INFO:tasks.rados.rados.0.smithi063.stdout:2993: oid 18 version 982 is already newer than 980 2024-08-17T17:07:26.661 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 982 (ObjNum 1130 snap 377 seq_num 1130) dirty exists 2024-08-17T17:07:26.661 INFO:tasks.rados.rados.0.smithi063.stdout:2993: left oid 18 (ObjNum 1130 snap 377 seq_num 1130) 2024-08-17T17:07:26.661 INFO:tasks.rados.rados.0.smithi063.stdout:2993: done (1 left) 2024-08-17T17:07:26.661 INFO:tasks.rados.rados.0.smithi063.stdout:2994: done (0 left) 2024-08-17T17:07:26.661 INFO:tasks.rados.rados.0.smithi063.stdout:2995: snap_remove snap 370 2024-08-17T17:07:27.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:26 smithi063 ceph-mon[114756]: osdmap e818: 8 total, 8 up, 8 in 2024-08-17T17:07:27.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:26 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:26 smithi063 ceph-mon[109843]: osdmap e818: 8 total, 8 up, 8 in 2024-08-17T17:07:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:26 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:27.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:26 smithi162 ceph-mon[104668]: osdmap e818: 8 total, 8 up, 8 in 2024-08-17T17:07:27.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:26 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:27.628 INFO:tasks.rados.rados.0.smithi063.stdout:2995: done (0 left) 2024-08-17T17:07:27.629 INFO:tasks.rados.rados.0.smithi063.stdout:2996: write oid 42 current snap is 377 2024-08-17T17:07:27.629 INFO:tasks.rados.rados.0.smithi063.stdout:2996: seq_num 1132 ranges {625524=402112,1592087=608063,2869769=155755} 2024-08-17T17:07:27.635 INFO:tasks.rados.rados.0.smithi063.stdout:2996: writing smithi06387500-42 from 625524 to 1027636 tid 1 2024-08-17T17:07:27.642 INFO:tasks.rados.rados.0.smithi063.stdout:2996: writing smithi06387500-42 from 1592087 to 2200150 tid 2 2024-08-17T17:07:27.647 INFO:tasks.rados.rados.0.smithi063.stdout:2996: writing smithi06387500-42 from 2869769 to 3025524 tid 3 2024-08-17T17:07:27.647 INFO:tasks.rados.rados.0.smithi063.stdout:2997: rollback oid 32 current snap is 377 2024-08-17T17:07:27.647 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 32 to 371 2024-08-17T17:07:27.647 INFO:tasks.rados.rados.0.smithi063.stdout:2996: finishing write tid 1 to smithi06387500-42 2024-08-17T17:07:27.648 INFO:tasks.rados.rados.0.smithi063.stdout:2996: finishing write tid 2 to smithi06387500-42 2024-08-17T17:07:27.648 INFO:tasks.rados.rados.0.smithi063.stdout:2998: delete oid 41 current snap is 377 2024-08-17T17:07:27.649 INFO:tasks.rados.rados.0.smithi063.stdout:2998: done (2 left) 2024-08-17T17:07:27.649 INFO:tasks.rados.rados.0.smithi063.stdout:2999: read oid 25 snap -1 2024-08-17T17:07:27.649 INFO:tasks.rados.rados.0.smithi063.stdout:2999: expect deleted 2024-08-17T17:07:27.649 INFO:tasks.rados.rados.0.smithi063.stdout:3000: write oid 46 current snap is 377 2024-08-17T17:07:27.649 INFO:tasks.rados.rados.0.smithi063.stdout:3000: seq_num 1134 ranges {659138=668963,2109051=611732} 2024-08-17T17:07:27.656 INFO:tasks.rados.rados.0.smithi063.stdout:3000: writing smithi06387500-46 from 659138 to 1328101 tid 1 2024-08-17T17:07:27.665 INFO:tasks.rados.rados.0.smithi063.stdout:3000: writing smithi06387500-46 from 2109051 to 2720783 tid 2 2024-08-17T17:07:27.665 INFO:tasks.rados.rados.0.smithi063.stdout:3001: write oid 9 current snap is 377 2024-08-17T17:07:27.665 INFO:tasks.rados.rados.0.smithi063.stdout:3001: seq_num 1135 ranges {675945=744212,2167533=755390} 2024-08-17T17:07:27.673 INFO:tasks.rados.rados.0.smithi063.stdout:3001: writing smithi06387500-9 from 675945 to 1420157 tid 1 2024-08-17T17:07:27.683 INFO:tasks.rados.rados.0.smithi063.stdout:3001: writing smithi06387500-9 from 2167533 to 2922923 tid 2 2024-08-17T17:07:27.683 INFO:tasks.rados.rados.0.smithi063.stdout:3002: rollback oid 37 current snap is 377 2024-08-17T17:07:27.683 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 376 2024-08-17T17:07:27.683 INFO:tasks.rados.rados.0.smithi063.stdout:2996: finishing write tid 3 to smithi06387500-42 2024-08-17T17:07:27.683 INFO:tasks.rados.rados.0.smithi063.stdout:2997: finishing rollback tid 0 to smithi06387500-32 2024-08-17T17:07:27.683 INFO:tasks.rados.rados.0.smithi063.stdout:2996: finishing write tid 4 to smithi06387500-42 2024-08-17T17:07:27.684 INFO:tasks.rados.rados.0.smithi063.stdout:2997: finishing rollback tid 1 to smithi06387500-32 2024-08-17T17:07:27.684 INFO:tasks.rados.rados.0.smithi063.stdout:2997: finishing rollback tid 2 to smithi06387500-32 2024-08-17T17:07:27.684 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 629 (ObjNum 882 snap 308 seq_num 882) dirty exists 2024-08-17T17:07:27.684 INFO:tasks.rados.rados.0.smithi063.stdout:2996: finishing write tid 5 to smithi06387500-42 2024-08-17T17:07:27.684 INFO:tasks.rados.rados.0.smithi063.stdout:2996: oid 42 updating version 0 to 1904 2024-08-17T17:07:27.684 INFO:tasks.rados.rados.0.smithi063.stdout:2996: oid 42 version 1904 is already newer than 1901 2024-08-17T17:07:27.684 INFO:tasks.rados.rados.0.smithi063.stdout:2996: oid 42 version 1904 is already newer than 1902 2024-08-17T17:07:27.684 INFO:tasks.rados.rados.0.smithi063.stdout:2996: oid 42 version 1904 is already newer than 1903 2024-08-17T17:07:27.684 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 1904 (ObjNum 1132 snap 377 seq_num 1132) dirty exists 2024-08-17T17:07:27.684 INFO:tasks.rados.rados.0.smithi063.stdout:2996: left oid 42 (ObjNum 1132 snap 377 seq_num 1132) 2024-08-17T17:07:27.684 INFO:tasks.rados.rados.0.smithi063.stdout:3000: finishing write tid 1 to smithi06387500-46 2024-08-17T17:07:27.684 INFO:tasks.rados.rados.0.smithi063.stdout:2996: done (5 left) 2024-08-17T17:07:27.684 INFO:tasks.rados.rados.0.smithi063.stdout:2997: done (4 left) 2024-08-17T17:07:27.685 INFO:tasks.rados.rados.0.smithi063.stdout:2999: done (3 left) 2024-08-17T17:07:27.685 INFO:tasks.rados.rados.0.smithi063.stdout:3003: rollback oid 35 current snap is 377 2024-08-17T17:07:27.685 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 35 to 369 2024-08-17T17:07:27.685 INFO:tasks.rados.rados.0.smithi063.stdout:3000: finishing write tid 2 to smithi06387500-46 2024-08-17T17:07:27.685 INFO:tasks.rados.rados.0.smithi063.stdout:3000: finishing write tid 3 to smithi06387500-46 2024-08-17T17:07:27.685 INFO:tasks.rados.rados.0.smithi063.stdout:3000: finishing write tid 4 to smithi06387500-46 2024-08-17T17:07:27.685 INFO:tasks.rados.rados.0.smithi063.stdout:3000: oid 46 updating version 0 to 1049 2024-08-17T17:07:27.685 INFO:tasks.rados.rados.0.smithi063.stdout:3000: oid 46 version 1049 is already newer than 1048 2024-08-17T17:07:27.685 INFO:tasks.rados.rados.0.smithi063.stdout:3000: oid 46 version 1049 is already newer than 1047 2024-08-17T17:07:27.685 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 1049 (ObjNum 1134 snap 377 seq_num 1134) dirty exists 2024-08-17T17:07:27.685 INFO:tasks.rados.rados.0.smithi063.stdout:3000: left oid 46 (ObjNum 1134 snap 377 seq_num 1134) 2024-08-17T17:07:27.685 INFO:tasks.rados.rados.0.smithi063.stdout:3001: finishing write tid 1 to smithi06387500-9 2024-08-17T17:07:27.685 INFO:tasks.rados.rados.0.smithi063.stdout:3000: done (3 left) 2024-08-17T17:07:27.686 INFO:tasks.rados.rados.0.smithi063.stdout:3004: read oid 27 snap -1 2024-08-17T17:07:27.686 INFO:tasks.rados.rados.0.smithi063.stdout:3004: expect deleted 2024-08-17T17:07:27.686 INFO:tasks.rados.rados.0.smithi063.stdout:3005: snap_remove snap 353 2024-08-17T17:07:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:27 smithi063 ceph-mon[114756]: pgmap v684: 89 pgs: 89 active+clean; 245 MiB data, 815 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 579 KiB/s wr, 2 op/s 2024-08-17T17:07:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:27 smithi063 ceph-mon[114756]: osdmap e819: 8 total, 8 up, 8 in 2024-08-17T17:07:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:27 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:07:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:27 smithi063 ceph-mon[114756]: osdmap e820: 8 total, 8 up, 8 in 2024-08-17T17:07:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:27 smithi063 ceph-mon[109843]: pgmap v684: 89 pgs: 89 active+clean; 245 MiB data, 815 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 579 KiB/s wr, 2 op/s 2024-08-17T17:07:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:27 smithi063 ceph-mon[109843]: osdmap e819: 8 total, 8 up, 8 in 2024-08-17T17:07:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:27 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:07:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:27 smithi063 ceph-mon[109843]: osdmap e820: 8 total, 8 up, 8 in 2024-08-17T17:07:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:27 smithi162 ceph-mon[104668]: pgmap v684: 89 pgs: 89 active+clean; 245 MiB data, 815 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 579 KiB/s wr, 2 op/s 2024-08-17T17:07:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:27 smithi162 ceph-mon[104668]: osdmap e819: 8 total, 8 up, 8 in 2024-08-17T17:07:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:27 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-08-17T17:07:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:27 smithi162 ceph-mon[104668]: osdmap e820: 8 total, 8 up, 8 in 2024-08-17T17:07:28.654 INFO:tasks.rados.rados.0.smithi063.stdout:3005: done (4 left) 2024-08-17T17:07:28.655 INFO:tasks.rados.rados.0.smithi063.stdout:3006: read oid 10 snap -1 2024-08-17T17:07:28.655 INFO:tasks.rados.rados.0.smithi063.stdout:3006: expect deleted 2024-08-17T17:07:28.655 INFO:tasks.rados.rados.0.smithi063.stdout:3007: read oid 6 snap -1 2024-08-17T17:07:28.655 INFO:tasks.rados.rados.0.smithi063.stdout:3007: expect (ObjNum 900 snap 311 seq_num 900) 2024-08-17T17:07:28.655 INFO:tasks.rados.rados.0.smithi063.stdout:3008: read oid 24 snap -1 2024-08-17T17:07:28.655 INFO:tasks.rados.rados.0.smithi063.stdout:3008: expect deleted 2024-08-17T17:07:28.655 INFO:tasks.rados.rados.0.smithi063.stdout:3009: delete oid 26 current snap is 377 2024-08-17T17:07:28.655 INFO:tasks.rados.rados.0.smithi063.stdout:3001: finishing write tid 2 to smithi06387500-9 2024-08-17T17:07:28.655 INFO:tasks.rados.rados.0.smithi063.stdout:3002: finishing rollback tid 0 to smithi06387500-37 2024-08-17T17:07:28.655 INFO:tasks.rados.rados.0.smithi063.stdout:3001: finishing write tid 3 to smithi06387500-9 2024-08-17T17:07:28.655 INFO:tasks.rados.rados.0.smithi063.stdout:3001: finishing write tid 4 to smithi06387500-9 2024-08-17T17:07:28.655 INFO:tasks.rados.rados.0.smithi063.stdout:3001: oid 9 updating version 0 to 1399 2024-08-17T17:07:28.656 INFO:tasks.rados.rados.0.smithi063.stdout:3001: oid 9 version 1399 is already newer than 1398 2024-08-17T17:07:28.656 INFO:tasks.rados.rados.0.smithi063.stdout:3001: oid 9 version 1399 is already newer than 1397 2024-08-17T17:07:28.656 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 1399 (ObjNum 1135 snap 377 seq_num 1135) dirty exists 2024-08-17T17:07:28.656 INFO:tasks.rados.rados.0.smithi063.stdout:3001: left oid 9 (ObjNum 1135 snap 377 seq_num 1135) 2024-08-17T17:07:28.656 INFO:tasks.rados.rados.0.smithi063.stdout:3002: finishing rollback tid 1 to smithi06387500-37 2024-08-17T17:07:28.656 INFO:tasks.rados.rados.0.smithi063.stdout:3002: finishing rollback tid 2 to smithi06387500-37 2024-08-17T17:07:28.656 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 2875 (ObjNum 1061 snap 358 seq_num 1061) dirty exists 2024-08-17T17:07:28.656 INFO:tasks.rados.rados.0.smithi063.stdout:3003: finishing rollback tid 0 to smithi06387500-35 2024-08-17T17:07:28.656 INFO:tasks.rados.rados.0.smithi063.stdout:3003: finishing rollback tid 1 to smithi06387500-35 2024-08-17T17:07:28.656 INFO:tasks.rados.rados.0.smithi063.stdout:3003: finishing rollback tid 2 to smithi06387500-35 2024-08-17T17:07:28.656 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 2879 (ObjNum 1068 snap 361 seq_num 1068) dirty exists 2024-08-17T17:07:28.663 INFO:tasks.rados.rados.0.smithi063.stdout:3001: done (7 left) 2024-08-17T17:07:28.663 INFO:tasks.rados.rados.0.smithi063.stdout:3002: done (6 left) 2024-08-17T17:07:28.663 INFO:tasks.rados.rados.0.smithi063.stdout:3003: done (5 left) 2024-08-17T17:07:28.663 INFO:tasks.rados.rados.0.smithi063.stdout:3004: done (4 left) 2024-08-17T17:07:28.663 INFO:tasks.rados.rados.0.smithi063.stdout:3006: done (3 left) 2024-08-17T17:07:28.663 INFO:tasks.rados.rados.0.smithi063.stdout:3008: done (2 left) 2024-08-17T17:07:28.663 INFO:tasks.rados.rados.0.smithi063.stdout:3009: done (1 left) 2024-08-17T17:07:28.663 INFO:tasks.rados.rados.0.smithi063.stdout:3010: snap_create 2024-08-17T17:07:28.666 INFO:tasks.rados.rados.0.smithi063.stdout:3007: expect (ObjNum 900 snap 311 seq_num 900) 2024-08-17T17:07:28.947 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:28 smithi063 ceph-mon[114756]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:07:28.947 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:28 smithi063 ceph-mon[114756]: Upgrade: osd.3 is safe to restart 2024-08-17T17:07:28.947 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:28 smithi063 ceph-mon[109843]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:07:28.947 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:28 smithi063 ceph-mon[109843]: Upgrade: osd.3 is safe to restart 2024-08-17T17:07:29.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:28 smithi162 ceph-mon[104668]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-08-17T17:07:29.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:28 smithi162 ceph-mon[104668]: Upgrade: osd.3 is safe to restart 2024-08-17T17:07:29.648 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:29 smithi063 ceph-mon[109843]: pgmap v687: 89 pgs: 89 active+clean; 251 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-08-17T17:07:29.648 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:29 smithi063 ceph-mon[109843]: osdmap e821: 8 total, 8 up, 8 in 2024-08-17T17:07:29.655 INFO:tasks.rados.rados.0.smithi063.stdout:3007: done (1 left) 2024-08-17T17:07:29.655 INFO:tasks.rados.rados.0.smithi063.stdout:3010: done (0 left) 2024-08-17T17:07:29.655 INFO:tasks.rados.rados.0.smithi063.stdout:3011: rollback oid 13 current snap is 378 2024-08-17T17:07:29.655 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 13 to 363 2024-08-17T17:07:29.655 INFO:tasks.rados.rados.0.smithi063.stdout:3012: snap_create 2024-08-17T17:07:29.658 INFO:tasks.rados.rados.0.smithi063.stdout:3011: finishing rollback tid 1 to smithi06387500-13 2024-08-17T17:07:29.658 INFO:tasks.rados.rados.0.smithi063.stdout:3011: finishing rollback tid 2 to smithi06387500-13 2024-08-17T17:07:29.658 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1470 (ObjNum 1090 snap 363 seq_num 1090) dirty exists 2024-08-17T17:07:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:29 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:29 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:07:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:29 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:30.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:29 smithi063 ceph-mon[114756]: pgmap v687: 89 pgs: 89 active+clean; 251 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-08-17T17:07:30.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:29 smithi063 ceph-mon[114756]: osdmap e821: 8 total, 8 up, 8 in 2024-08-17T17:07:30.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:29 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:30.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:29 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:07:30.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:29 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:29 smithi162 ceph-mon[104668]: pgmap v687: 89 pgs: 89 active+clean; 251 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-08-17T17:07:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:29 smithi162 ceph-mon[104668]: osdmap e821: 8 total, 8 up, 8 in 2024-08-17T17:07:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:29 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:29 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-08-17T17:07:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:29 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:30.658 INFO:tasks.rados.rados.0.smithi063.stdout:3011: done (1 left) 2024-08-17T17:07:30.658 INFO:tasks.rados.rados.0.smithi063.stdout:3012: done (0 left) 2024-08-17T17:07:30.658 INFO:tasks.rados.rados.0.smithi063.stdout:3013: read oid 9 snap -1 2024-08-17T17:07:30.658 INFO:tasks.rados.rados.0.smithi063.stdout:3013: expect (ObjNum 1135 snap 377 seq_num 1135) 2024-08-17T17:07:30.658 INFO:tasks.rados.rados.0.smithi063.stdout:3014: write oid 11 current snap is 379 2024-08-17T17:07:30.658 INFO:tasks.rados.rados.0.smithi063.stdout:3014: seq_num 1137 ranges {709559=494710,1884497=759059} 2024-08-17T17:07:30.664 INFO:tasks.rados.rados.0.smithi063.stdout:3014: writing smithi06387500-11 from 709559 to 1204269 tid 1 2024-08-17T17:07:30.672 INFO:tasks.rados.rados.0.smithi063.stdout:3014: writing smithi06387500-11 from 1884497 to 2643556 tid 2 2024-08-17T17:07:30.672 INFO:tasks.rados.rados.0.smithi063.stdout:3015: snap_remove snap 373 2024-08-17T17:07:30.774 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:30 smithi063 ceph-mon[114756]: Upgrade: Updating osd.3 2024-08-17T17:07:30.774 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:30 smithi063 ceph-mon[114756]: Deploying daemon osd.3 on smithi063 2024-08-17T17:07:30.774 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:30 smithi063 ceph-mon[114756]: osdmap e822: 8 total, 8 up, 8 in 2024-08-17T17:07:30.774 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:30 smithi063 ceph-mon[109843]: Upgrade: Updating osd.3 2024-08-17T17:07:30.774 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:30 smithi063 ceph-mon[109843]: Deploying daemon osd.3 on smithi063 2024-08-17T17:07:30.775 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:30 smithi063 ceph-mon[109843]: osdmap e822: 8 total, 8 up, 8 in 2024-08-17T17:07:30.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:30 smithi162 ceph-mon[104668]: Upgrade: Updating osd.3 2024-08-17T17:07:30.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:30 smithi162 ceph-mon[104668]: Deploying daemon osd.3 on smithi063 2024-08-17T17:07:30.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:30 smithi162 ceph-mon[104668]: osdmap e822: 8 total, 8 up, 8 in 2024-08-17T17:07:31.667 INFO:tasks.rados.rados.0.smithi063.stdout:3015: done (2 left) 2024-08-17T17:07:31.667 INFO:tasks.rados.rados.0.smithi063.stdout:3016: write oid 27 current snap is 379 2024-08-17T17:07:31.667 INFO:tasks.rados.rados.0.smithi063.stdout:3016: seq_num 1138 ranges {726366=569959,1826626=502717,2801601=324765} 2024-08-17T17:07:31.674 INFO:tasks.rados.rados.0.smithi063.stdout:3016: writing smithi06387500-27 from 726366 to 1296325 tid 1 2024-08-17T17:07:31.680 INFO:tasks.rados.rados.0.smithi063.stdout:3016: writing smithi06387500-27 from 1826626 to 2329343 tid 2 2024-08-17T17:07:31.685 INFO:tasks.rados.rados.0.smithi063.stdout:3016: writing smithi06387500-27 from 2801601 to 3126366 tid 3 2024-08-17T17:07:31.685 INFO:tasks.rados.rados.0.smithi063.stdout:3017: write oid 5 current snap is 379 2024-08-17T17:07:31.685 INFO:tasks.rados.rados.0.smithi063.stdout:3017: seq_num 1139 ranges {743173=645208,1885108=762728} 2024-08-17T17:07:31.693 INFO:tasks.rados.rados.0.smithi063.stdout:3017: writing smithi06387500-5 from 743173 to 1388381 tid 1 2024-08-17T17:07:31.700 INFO:tasks.rados.rados.0.smithi063.stdout:3017: writing smithi06387500-5 from 1885108 to 2647836 tid 2 2024-08-17T17:07:31.700 INFO:tasks.rados.rados.0.smithi063.stdout:3018: read oid 1 snap -1 2024-08-17T17:07:31.700 INFO:tasks.rados.rados.0.smithi063.stdout:3018: expect (ObjNum 1075 snap 362 seq_num 1075) 2024-08-17T17:07:31.700 INFO:tasks.rados.rados.0.smithi063.stdout:3019: snap_remove snap 376 2024-08-17T17:07:31.779 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:31 smithi063 systemd[1]: Stopping Ceph osd.3 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:07:31.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:31 smithi063 ceph-mon[109843]: pgmap v690: 89 pgs: 89 active+clean; 251 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 957 KiB/s wr, 7 op/s 2024-08-17T17:07:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:31 smithi063 ceph-mon[109843]: osdmap e823: 8 total, 8 up, 8 in 2024-08-17T17:07:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:31.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:31 smithi063 ceph-mon[114756]: pgmap v690: 89 pgs: 89 active+clean; 251 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 957 KiB/s wr, 7 op/s 2024-08-17T17:07:31.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:31 smithi063 ceph-mon[114756]: osdmap e823: 8 total, 8 up, 8 in 2024-08-17T17:07:31.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:31.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:31 smithi162 ceph-mon[104668]: pgmap v690: 89 pgs: 89 active+clean; 251 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 957 KiB/s wr, 7 op/s 2024-08-17T17:07:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:31 smithi162 ceph-mon[104668]: osdmap e823: 8 total, 8 up, 8 in 2024-08-17T17:07:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:32.529 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:32 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3[65761]: 2024-08-17T17:07:32.042+0000 7f64d24c7640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:07:32.529 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:32 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3[65761]: 2024-08-17T17:07:32.042+0000 7f64d24c7640 -1 osd.3 824 *** Got signal Terminated *** 2024-08-17T17:07:32.529 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:32 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3[65761]: 2024-08-17T17:07:32.042+0000 7f64d24c7640 -1 osd.3 824 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:07:32.631 INFO:tasks.rados.rados.0.smithi063.stdout:3014: finishing write tid 1 to smithi06387500-11 2024-08-17T17:07:32.631 INFO:tasks.rados.rados.0.smithi063.stdout:3013: expect (ObjNum 1135 snap 377 seq_num 1135) 2024-08-17T17:07:32.757 INFO:tasks.rados.rados.0.smithi063.stdout:3014: finishing write tid 2 to smithi06387500-11 2024-08-17T17:07:32.758 INFO:tasks.rados.rados.0.smithi063.stdout:3014: finishing write tid 3 to smithi06387500-11 2024-08-17T17:07:32.758 INFO:tasks.rados.rados.0.smithi063.stdout:3014: finishing write tid 4 to smithi06387500-11 2024-08-17T17:07:32.758 INFO:tasks.rados.rados.0.smithi063.stdout:3014: oid 11 updating version 0 to 485 2024-08-17T17:07:32.758 INFO:tasks.rados.rados.0.smithi063.stdout:3014: oid 11 updating version 485 to 486 2024-08-17T17:07:32.758 INFO:tasks.rados.rados.0.smithi063.stdout:3014: oid 11 version 486 is already newer than 484 2024-08-17T17:07:32.758 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 486 (ObjNum 1137 snap 379 seq_num 1137) dirty exists 2024-08-17T17:07:32.758 INFO:tasks.rados.rados.0.smithi063.stdout:3014: left oid 11 (ObjNum 1137 snap 379 seq_num 1137) 2024-08-17T17:07:32.758 INFO:tasks.rados.rados.0.smithi063.stdout:3013: done (5 left) 2024-08-17T17:07:32.758 INFO:tasks.rados.rados.0.smithi063.stdout:3014: done (4 left) 2024-08-17T17:07:32.758 INFO:tasks.rados.rados.0.smithi063.stdout:3019: done (3 left) 2024-08-17T17:07:32.758 INFO:tasks.rados.rados.0.smithi063.stdout:3020: snap_create 2024-08-17T17:07:32.759 INFO:tasks.rados.rados.0.smithi063.stdout:3016: finishing write tid 1 to smithi06387500-27 2024-08-17T17:07:32.759 INFO:tasks.rados.rados.0.smithi063.stdout:3016: finishing write tid 2 to smithi06387500-27 2024-08-17T17:07:32.759 INFO:tasks.rados.rados.0.smithi063.stdout:3016: finishing write tid 3 to smithi06387500-27 2024-08-17T17:07:32.759 INFO:tasks.rados.rados.0.smithi063.stdout:3016: finishing write tid 4 to smithi06387500-27 2024-08-17T17:07:32.759 INFO:tasks.rados.rados.0.smithi063.stdout:3016: finishing write tid 5 to smithi06387500-27 2024-08-17T17:07:32.759 INFO:tasks.rados.rados.0.smithi063.stdout:3016: oid 27 updating version 0 to 270 2024-08-17T17:07:32.759 INFO:tasks.rados.rados.0.smithi063.stdout:3016: oid 27 updating version 270 to 271 2024-08-17T17:07:32.759 INFO:tasks.rados.rados.0.smithi063.stdout:3016: oid 27 version 271 is already newer than 268 2024-08-17T17:07:32.759 INFO:tasks.rados.rados.0.smithi063.stdout:3016: oid 27 version 271 is already newer than 269 2024-08-17T17:07:32.759 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 271 (ObjNum 1138 snap 379 seq_num 1138) dirty exists 2024-08-17T17:07:32.759 INFO:tasks.rados.rados.0.smithi063.stdout:3016: left oid 27 (ObjNum 1138 snap 379 seq_num 1138) 2024-08-17T17:07:32.759 INFO:tasks.rados.rados.0.smithi063.stdout:3017: finishing write tid 1 to smithi06387500-5 2024-08-17T17:07:32.759 INFO:tasks.rados.rados.0.smithi063.stdout:3017: finishing write tid 2 to smithi06387500-5 2024-08-17T17:07:32.759 INFO:tasks.rados.rados.0.smithi063.stdout:3017: finishing write tid 3 to smithi06387500-5 2024-08-17T17:07:32.760 INFO:tasks.rados.rados.0.smithi063.stdout:3017: finishing write tid 4 to smithi06387500-5 2024-08-17T17:07:32.760 INFO:tasks.rados.rados.0.smithi063.stdout:3017: oid 5 updating version 0 to 1471 2024-08-17T17:07:32.760 INFO:tasks.rados.rados.0.smithi063.stdout:3017: oid 5 updating version 1471 to 1473 2024-08-17T17:07:32.760 INFO:tasks.rados.rados.0.smithi063.stdout:3017: oid 5 version 1473 is already newer than 1472 2024-08-17T17:07:32.760 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1473 (ObjNum 1139 snap 379 seq_num 1139) dirty exists 2024-08-17T17:07:32.760 INFO:tasks.rados.rados.0.smithi063.stdout:3017: left oid 5 (ObjNum 1139 snap 379 seq_num 1139) 2024-08-17T17:07:32.760 INFO:tasks.rados.rados.0.smithi063.stdout:3018: expect (ObjNum 1075 snap 362 seq_num 1075) 2024-08-17T17:07:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:32 smithi063 ceph-mon[109843]: osdmap e824: 8 total, 8 up, 8 in 2024-08-17T17:07:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:32 smithi063 ceph-mon[109843]: pgmap v693: 89 pgs: 89 active+clean; 259 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-08-17T17:07:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:32 smithi063 ceph-mon[109843]: osd.3 marked itself down and dead 2024-08-17T17:07:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:32 smithi063 ceph-mon[109843]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:07:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:32 smithi063 ceph-mon[109843]: osdmap e825: 8 total, 7 up, 8 in 2024-08-17T17:07:33.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:32 smithi063 ceph-mon[114756]: osdmap e824: 8 total, 8 up, 8 in 2024-08-17T17:07:33.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:32 smithi063 ceph-mon[114756]: pgmap v693: 89 pgs: 89 active+clean; 259 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-08-17T17:07:33.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:32 smithi063 ceph-mon[114756]: osd.3 marked itself down and dead 2024-08-17T17:07:33.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:32 smithi063 ceph-mon[114756]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:07:33.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:32 smithi063 ceph-mon[114756]: osdmap e825: 8 total, 7 up, 8 in 2024-08-17T17:07:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:32 smithi162 ceph-mon[104668]: osdmap e824: 8 total, 8 up, 8 in 2024-08-17T17:07:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:32 smithi162 ceph-mon[104668]: pgmap v693: 89 pgs: 89 active+clean; 259 MiB data, 836 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-08-17T17:07:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:32 smithi162 ceph-mon[104668]: osd.3 marked itself down and dead 2024-08-17T17:07:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:32 smithi162 ceph-mon[104668]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:07:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:32 smithi162 ceph-mon[104668]: osdmap e825: 8 total, 7 up, 8 in 2024-08-17T17:07:33.418 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:33 smithi063 podman[141888]: 2024-08-17 17:07:33.115729292 +0000 UTC m=+1.235888649 container died 9da1dbb0ca653081d43b71e50c47596e6486127074a5a78561ca70f0beccd9b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=reef-ced649f, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0) 2024-08-17T17:07:33.418 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:07:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:07:33] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.51.0" 2024-08-17T17:07:33.629 INFO:tasks.rados.rados.0.smithi063.stdout:3016: done (3 left) 2024-08-17T17:07:33.630 INFO:tasks.rados.rados.0.smithi063.stdout:3017: done (2 left) 2024-08-17T17:07:33.630 INFO:tasks.rados.rados.0.smithi063.stdout:3018: done (1 left) 2024-08-17T17:07:33.630 INFO:tasks.rados.rados.0.smithi063.stdout:3020: done (0 left) 2024-08-17T17:07:33.630 INFO:tasks.rados.rados.0.smithi063.stdout:3021: write oid 49 current snap is 380 2024-08-17T17:07:33.630 INFO:tasks.rados.rados.0.smithi063.stdout:3021: seq_num 1140 ranges {759980=720457,1943590=506386,2856447=303533} 2024-08-17T17:07:33.639 INFO:tasks.rados.rados.0.smithi063.stdout:3021: writing smithi06387500-49 from 759980 to 1480437 tid 1 2024-08-17T17:07:33.644 INFO:tasks.rados.rados.0.smithi063.stdout:3021: writing smithi06387500-49 from 1943590 to 2449976 tid 2 2024-08-17T17:07:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:3021: writing smithi06387500-49 from 2856447 to 3159980 tid 3 2024-08-17T17:07:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:3022: read oid 38 snap -1 2024-08-17T17:07:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:3022: expect (ObjNum 1083 snap 363 seq_num 1083) 2024-08-17T17:07:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:3023: rollback oid 21 current snap is 380 2024-08-17T17:07:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 21 to 367 2024-08-17T17:07:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:3021: finishing write tid 1 to smithi06387500-49 2024-08-17T17:07:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:3021: finishing write tid 2 to smithi06387500-49 2024-08-17T17:07:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:3024: snap_create 2024-08-17T17:07:33.652 INFO:tasks.rados.rados.0.smithi063.stdout:3022: expect (ObjNum 1083 snap 363 seq_num 1083) 2024-08-17T17:07:33.776 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:33 smithi063 podman[141888]: 2024-08-17 17:07:33.524732723 +0000 UTC m=+1.644892121 container cleanup 9da1dbb0ca653081d43b71e50c47596e6486127074a5a78561ca70f0beccd9b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:07:33.776 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:33 smithi063 bash[141888]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3 2024-08-17T17:07:33.795 INFO:tasks.rados.rados.0.smithi063.stdout:3021: finishing write tid 3 to smithi06387500-49 2024-08-17T17:07:33.795 INFO:tasks.rados.rados.0.smithi063.stdout:3021: finishing write tid 4 to smithi06387500-49 2024-08-17T17:07:33.795 INFO:tasks.rados.rados.0.smithi063.stdout:3021: finishing write tid 5 to smithi06387500-49 2024-08-17T17:07:33.795 INFO:tasks.rados.rados.0.smithi063.stdout:3021: oid 49 updating version 0 to 2904 2024-08-17T17:07:33.795 INFO:tasks.rados.rados.0.smithi063.stdout:3021: oid 49 version 2904 is already newer than 2903 2024-08-17T17:07:33.795 INFO:tasks.rados.rados.0.smithi063.stdout:3021: oid 49 version 2904 is already newer than 2902 2024-08-17T17:07:33.795 INFO:tasks.rados.rados.0.smithi063.stdout:3021: oid 49 updating version 2904 to 2905 2024-08-17T17:07:33.796 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 2905 (ObjNum 1140 snap 380 seq_num 1140) dirty exists 2024-08-17T17:07:33.796 INFO:tasks.rados.rados.0.smithi063.stdout:3021: left oid 49 (ObjNum 1140 snap 380 seq_num 1140) 2024-08-17T17:07:33.796 INFO:tasks.rados.rados.0.smithi063.stdout:3023: finishing rollback tid 0 to smithi06387500-21 2024-08-17T17:07:33.796 INFO:tasks.rados.rados.0.smithi063.stdout:3023: finishing rollback tid 1 to smithi06387500-21 2024-08-17T17:07:33.796 INFO:tasks.rados.rados.0.smithi063.stdout:3023: finishing rollback tid 2 to smithi06387500-21 2024-08-17T17:07:33.796 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 1912 (ObjNum 938 snap 321 seq_num 938) dirty exists 2024-08-17T17:07:34.029 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:33 smithi063 podman[141910]: 2024-08-17 17:07:33.776002837 +0000 UTC m=+0.655550966 container remove 9da1dbb0ca653081d43b71e50c47596e6486127074a5a78561ca70f0beccd9b8 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-08-17T17:07:34.479 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:34 smithi063 podman[142002]: 2024-08-17 17:07:34.185434507 +0000 UTC m=+0.105834352 container create 3fba0018740dd0b36d5693fa0eaec294ed7d8c07071ebea811402511dd50432f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-08-17T17:07:34.480 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:34 smithi063 podman[142002]: 2024-08-17 17:07:34.09730631 +0000 UTC m=+0.017706115 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:07:34.480 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:34 smithi063 podman[142002]: 2024-08-17 17:07:34.388172869 +0000 UTC m=+0.308572692 container init 3fba0018740dd0b36d5693fa0eaec294ed7d8c07071ebea811402511dd50432f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-08-17T17:07:34.480 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:34 smithi063 podman[142002]: 2024-08-17 17:07:34.393379311 +0000 UTC m=+0.313779117 container start 3fba0018740dd0b36d5693fa0eaec294ed7d8c07071ebea811402511dd50432f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, io.buildah.version=1.36.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-08-17T17:07:34.480 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:34 smithi063 podman[142002]: 2024-08-17 17:07:34.479360952 +0000 UTC m=+0.399760762 container attach 3fba0018740dd0b36d5693fa0eaec294ed7d8c07071ebea811402511dd50432f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0) 2024-08-17T17:07:34.638 INFO:tasks.rados.rados.0.smithi063.stdout:3021: done (3 left) 2024-08-17T17:07:34.639 INFO:tasks.rados.rados.0.smithi063.stdout:3022: done (2 left) 2024-08-17T17:07:34.639 INFO:tasks.rados.rados.0.smithi063.stdout:3023: done (1 left) 2024-08-17T17:07:34.639 INFO:tasks.rados.rados.0.smithi063.stdout:3024: done (0 left) 2024-08-17T17:07:34.639 INFO:tasks.rados.rados.0.smithi063.stdout:3025: rollback oid 10 current snap is 381 2024-08-17T17:07:34.639 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 10 to 378 2024-08-17T17:07:34.639 INFO:tasks.rados.rados.0.smithi063.stdout:3026: snap_remove snap 375 2024-08-17T17:07:34.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:34 smithi063 ceph-mon[109843]: osdmap e826: 8 total, 7 up, 8 in 2024-08-17T17:07:34.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:34 smithi063 ceph-mon[114756]: osdmap e826: 8 total, 7 up, 8 in 2024-08-17T17:07:34.780 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:34 smithi063 podman[142002]: 2024-08-17 17:07:34.60096252 +0000 UTC m=+0.521362322 container died 3fba0018740dd0b36d5693fa0eaec294ed7d8c07071ebea811402511dd50432f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=) 2024-08-17T17:07:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:34 smithi162 ceph-mon[104668]: osdmap e826: 8 total, 7 up, 8 in 2024-08-17T17:07:35.530 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:35 smithi063 podman[142045]: 2024-08-17 17:07:35.175094056 +0000 UTC m=+0.568205148 container remove 3fba0018740dd0b36d5693fa0eaec294ed7d8c07071ebea811402511dd50432f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:07:35.530 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:35 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.3.service: Deactivated successfully. 2024-08-17T17:07:35.530 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:35 smithi063 systemd[1]: Stopped Ceph osd.3 for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:07:35.530 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:35 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.3.service: Consumed 16.171s CPU time. 2024-08-17T17:07:35.530 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:35 smithi063 systemd[1]: Starting Ceph osd.3 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:07:35.634 INFO:tasks.rados.rados.0.smithi063.stdout:3026: done (1 left) 2024-08-17T17:07:35.634 INFO:tasks.rados.rados.0.smithi063.stdout:3027: read oid 47 snap -1 2024-08-17T17:07:35.634 INFO:tasks.rados.rados.0.smithi063.stdout:3027: expect (ObjNum 1120 snap 370 seq_num 1120) 2024-08-17T17:07:35.634 INFO:tasks.rados.rados.0.smithi063.stdout:3028: read oid 31 snap 339 2024-08-17T17:07:35.634 INFO:tasks.rados.rados.0.smithi063.stdout:3028: expect deleted 2024-08-17T17:07:35.635 INFO:tasks.rados.rados.0.smithi063.stdout:3029: write oid 20 current snap is 381 2024-08-17T17:07:35.635 INFO:tasks.rados.rados.0.smithi063.stdout:3025: finishing rollback tid 1 to smithi06387500-10 2024-08-17T17:07:35.635 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 0 (ObjNum 129 snap 0 seq_num 1807191816) dirty dne 2024-08-17T17:07:35.635 INFO:tasks.rados.rados.0.smithi063.stdout:3029: seq_num 1141 ranges {776787=512059,1718425=766397,3000223=176564} 2024-08-17T17:07:35.642 INFO:tasks.rados.rados.0.smithi063.stdout:3029: writing smithi06387500-20 from 776787 to 1288846 tid 1 2024-08-17T17:07:35.649 INFO:tasks.rados.rados.0.smithi063.stdout:3029: writing smithi06387500-20 from 1718425 to 2484822 tid 2 2024-08-17T17:07:35.654 INFO:tasks.rados.rados.0.smithi063.stdout:3029: writing smithi06387500-20 from 3000223 to 3176787 tid 3 2024-08-17T17:07:35.654 INFO:tasks.rados.rados.0.smithi063.stdout:3025: done (3 left) 2024-08-17T17:07:35.654 INFO:tasks.rados.rados.0.smithi063.stdout:3030: rollback oid 16 current snap is 381 2024-08-17T17:07:35.654 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 16 to 372 2024-08-17T17:07:35.654 INFO:tasks.rados.rados.0.smithi063.stdout:3027: expect (ObjNum 1120 snap 370 seq_num 1120) 2024-08-17T17:07:35.749 INFO:tasks.rados.rados.0.smithi063.stdout:3029: finishing write tid 1 to smithi06387500-20 2024-08-17T17:07:35.750 INFO:tasks.rados.rados.0.smithi063.stdout:3029: finishing write tid 2 to smithi06387500-20 2024-08-17T17:07:35.750 INFO:tasks.rados.rados.0.smithi063.stdout:3027: done (3 left) 2024-08-17T17:07:35.750 INFO:tasks.rados.rados.0.smithi063.stdout:3028: done (2 left) 2024-08-17T17:07:35.750 INFO:tasks.rados.rados.0.smithi063.stdout:3031: rollback oid 25 current snap is 381 2024-08-17T17:07:35.750 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 25 to 359 2024-08-17T17:07:35.750 INFO:tasks.rados.rados.0.smithi063.stdout:3029: finishing write tid 3 to smithi06387500-20 2024-08-17T17:07:35.750 INFO:tasks.rados.rados.0.smithi063.stdout:3029: finishing write tid 4 to smithi06387500-20 2024-08-17T17:07:35.750 INFO:tasks.rados.rados.0.smithi063.stdout:3029: finishing write tid 5 to smithi06387500-20 2024-08-17T17:07:35.750 INFO:tasks.rados.rados.0.smithi063.stdout:3029: oid 20 updating version 0 to 690 2024-08-17T17:07:35.750 INFO:tasks.rados.rados.0.smithi063.stdout:3029: oid 20 updating version 690 to 692 2024-08-17T17:07:35.751 INFO:tasks.rados.rados.0.smithi063.stdout:3029: oid 20 updating version 692 to 693 2024-08-17T17:07:35.751 INFO:tasks.rados.rados.0.smithi063.stdout:3029: oid 20 version 693 is already newer than 691 2024-08-17T17:07:35.751 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 693 (ObjNum 1141 snap 381 seq_num 1141) dirty exists 2024-08-17T17:07:35.751 INFO:tasks.rados.rados.0.smithi063.stdout:3029: left oid 20 (ObjNum 1141 snap 381 seq_num 1141) 2024-08-17T17:07:35.751 INFO:tasks.rados.rados.0.smithi063.stdout:3030: finishing rollback tid 0 to smithi06387500-16 2024-08-17T17:07:35.751 INFO:tasks.rados.rados.0.smithi063.stdout:3030: finishing rollback tid 1 to smithi06387500-16 2024-08-17T17:07:35.751 INFO:tasks.rados.rados.0.smithi063.stdout:3030: finishing rollback tid 2 to smithi06387500-16 2024-08-17T17:07:35.751 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 1057 (ObjNum 1019 snap 347 seq_num 1019) dirty exists 2024-08-17T17:07:35.751 INFO:tasks.rados.rados.0.smithi063.stdout:3029: done (2 left) 2024-08-17T17:07:35.751 INFO:tasks.rados.rados.0.smithi063.stdout:3030: done (1 left) 2024-08-17T17:07:35.751 INFO:tasks.rados.rados.0.smithi063.stdout:3032: delete oid 22 current snap is 381 2024-08-17T17:07:35.752 INFO:tasks.rados.rados.0.smithi063.stdout:3032: done (1 left) 2024-08-17T17:07:35.752 INFO:tasks.rados.rados.0.smithi063.stdout:3033: read oid 32 snap -1 2024-08-17T17:07:35.752 INFO:tasks.rados.rados.0.smithi063.stdout:3033: expect (ObjNum 882 snap 308 seq_num 882) 2024-08-17T17:07:35.752 INFO:tasks.rados.rados.0.smithi063.stdout:3034: write oid 48 current snap is 381 2024-08-17T17:07:35.752 INFO:tasks.rados.rados.0.smithi063.stdout:3034: seq_num 1143 ranges {410401=662557,1719036=770066,2938716=271685} 2024-08-17T17:07:35.758 INFO:tasks.rados.rados.0.smithi063.stdout:3034: writing smithi06387500-48 from 410401 to 1072958 tid 1 2024-08-17T17:07:35.765 INFO:tasks.rados.rados.0.smithi063.stdout:3034: writing smithi06387500-48 from 1719036 to 2489102 tid 2 2024-08-17T17:07:35.770 INFO:tasks.rados.rados.0.smithi063.stdout:3034: writing smithi06387500-48 from 2938716 to 3210401 tid 3 2024-08-17T17:07:35.770 INFO:tasks.rados.rados.0.smithi063.stdout:3035: read oid 49 snap -1 2024-08-17T17:07:35.770 INFO:tasks.rados.rados.0.smithi063.stdout:3035: expect (ObjNum 1140 snap 380 seq_num 1140) 2024-08-17T17:07:35.770 INFO:tasks.rados.rados.0.smithi063.stdout:3036: snap_create 2024-08-17T17:07:35.770 INFO:tasks.rados.rados.0.smithi063.stdout:3031: finishing rollback tid 1 to smithi06387500-25 2024-08-17T17:07:35.770 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 2728 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T17:07:35.770 INFO:tasks.rados.rados.0.smithi063.stdout:3033: expect (ObjNum 882 snap 308 seq_num 882) 2024-08-17T17:07:35.930 INFO:tasks.rados.rados.0.smithi063.stdout:3034: finishing write tid 1 to smithi06387500-48 2024-08-17T17:07:35.930 INFO:tasks.rados.rados.0.smithi063.stdout:3034: finishing write tid 2 to smithi06387500-48 2024-08-17T17:07:35.930 INFO:tasks.rados.rados.0.smithi063.stdout:3034: finishing write tid 3 to smithi06387500-48 2024-08-17T17:07:35.930 INFO:tasks.rados.rados.0.smithi063.stdout:3034: finishing write tid 4 to smithi06387500-48 2024-08-17T17:07:35.930 INFO:tasks.rados.rados.0.smithi063.stdout:3034: finishing write tid 5 to smithi06387500-48 2024-08-17T17:07:35.930 INFO:tasks.rados.rados.0.smithi063.stdout:3034: oid 48 updating version 0 to 2911 2024-08-17T17:07:35.931 INFO:tasks.rados.rados.0.smithi063.stdout:3034: oid 48 version 2911 is already newer than 2909 2024-08-17T17:07:35.931 INFO:tasks.rados.rados.0.smithi063.stdout:3034: oid 48 version 2911 is already newer than 2908 2024-08-17T17:07:35.931 INFO:tasks.rados.rados.0.smithi063.stdout:3034: oid 48 version 2911 is already newer than 2910 2024-08-17T17:07:35.931 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 2911 (ObjNum 1143 snap 381 seq_num 1143) dirty exists 2024-08-17T17:07:35.931 INFO:tasks.rados.rados.0.smithi063.stdout:3034: left oid 48 (ObjNum 1143 snap 381 seq_num 1143) 2024-08-17T17:07:35.931 INFO:tasks.rados.rados.0.smithi063.stdout:3035: expect (ObjNum 1140 snap 380 seq_num 1140) 2024-08-17T17:07:35.938 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:35 smithi063 ceph-mon[109843]: pgmap v696: 89 pgs: 7 peering, 9 stale+active+clean, 73 active+clean; 259 MiB data, 838 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-08-17T17:07:35.938 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:35 smithi063 ceph-mon[109843]: osdmap e827: 8 total, 7 up, 8 in 2024-08-17T17:07:35.938 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:35 smithi063 ceph-mon[114756]: pgmap v696: 89 pgs: 7 peering, 9 stale+active+clean, 73 active+clean; 259 MiB data, 838 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-08-17T17:07:35.938 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:35 smithi063 ceph-mon[114756]: osdmap e827: 8 total, 7 up, 8 in 2024-08-17T17:07:35.938 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:35 smithi063 podman[142154]: 2024-08-17 17:07:35.524140967 +0000 UTC m=+0.017979734 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:07:35.939 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:35 smithi063 podman[142154]: 2024-08-17 17:07:35.634346357 +0000 UTC m=+0.128185114 container create 6d88e75a4080a98d6e085dd64a35143c8b6ae261bb61e3e23dd464c65b670268 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0) 2024-08-17T17:07:35.939 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:35 smithi063 podman[142154]: 2024-08-17 17:07:35.853906932 +0000 UTC m=+0.347745710 container init 6d88e75a4080a98d6e085dd64a35143c8b6ae261bb61e3e23dd464c65b670268 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-08-17T17:07:35.939 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:35 smithi063 podman[142154]: 2024-08-17 17:07:35.858830664 +0000 UTC m=+0.352669419 container start 6d88e75a4080a98d6e085dd64a35143c8b6ae261bb61e3e23dd464c65b670268 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-08-17T17:07:35.939 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:35 smithi063 podman[142154]: 2024-08-17 17:07:35.937402241 +0000 UTC m=+0.431241009 container attach 6d88e75a4080a98d6e085dd64a35143c8b6ae261bb61e3e23dd464c65b670268 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-08-17T17:07:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:35 smithi162 ceph-mon[104668]: pgmap v696: 89 pgs: 7 peering, 9 stale+active+clean, 73 active+clean; 259 MiB data, 838 MiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.6 MiB/s wr, 14 op/s 2024-08-17T17:07:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:35 smithi162 ceph-mon[104668]: osdmap e827: 8 total, 7 up, 8 in 2024-08-17T17:07:36.279 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate[142169]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:36.280 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:35 smithi063 bash[142154]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:36.280 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate[142169]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:36.280 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 bash[142154]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:36.280 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate[142169]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:07:36.280 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 bash[142154]: --> Failed to activate via raw: activate() takes 1 positional argument but 5 were given 2024-08-17T17:07:36.280 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate[142169]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:36.280 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 bash[142154]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:36.280 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate[142169]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:36.280 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 bash[142154]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-08-17T17:07:36.280 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate[142169]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:07:36.280 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 bash[142154]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:07:36.281 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate[142169]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-08-17T17:07:36.281 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 bash[142154]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2024-08-17T17:07:36.649 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate[142169]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:07:36.649 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 bash[142154]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:07:36.649 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate[142169]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:07:36.649 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 bash[142154]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2024-08-17T17:07:36.649 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate[142169]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:07:36.649 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 bash[142154]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:07:36.649 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate[142169]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:07:36.650 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 bash[142154]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-08-17T17:07:36.650 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate[142169]: --> ceph-volume lvm activate successful for osd ID: 3 2024-08-17T17:07:36.650 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 bash[142154]: --> ceph-volume lvm activate successful for osd ID: 3 2024-08-17T17:07:36.650 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 podman[142154]: 2024-08-17 17:07:36.425897667 +0000 UTC m=+0.919736433 container died 6d88e75a4080a98d6e085dd64a35143c8b6ae261bb61e3e23dd464c65b670268 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:07:36.653 INFO:tasks.rados.rados.0.smithi063.stdout:3031: done (4 left) 2024-08-17T17:07:36.653 INFO:tasks.rados.rados.0.smithi063.stdout:3033: done (3 left) 2024-08-17T17:07:36.653 INFO:tasks.rados.rados.0.smithi063.stdout:3034: done (2 left) 2024-08-17T17:07:36.653 INFO:tasks.rados.rados.0.smithi063.stdout:3035: done (1 left) 2024-08-17T17:07:36.653 INFO:tasks.rados.rados.0.smithi063.stdout:3036: done (0 left) 2024-08-17T17:07:36.653 INFO:tasks.rados.rados.0.smithi063.stdout:3037: snap_create 2024-08-17T17:07:36.958 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:36 smithi063 ceph-mon[114756]: osdmap e828: 8 total, 7 up, 8 in 2024-08-17T17:07:36.959 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:36 smithi063 ceph-mon[109843]: osdmap e828: 8 total, 7 up, 8 in 2024-08-17T17:07:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:36 smithi162 ceph-mon[104668]: osdmap e828: 8 total, 7 up, 8 in 2024-08-17T17:07:37.218 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:36 smithi063 podman[142300]: 2024-08-17 17:07:36.958427463 +0000 UTC m=+0.525766468 container remove 6d88e75a4080a98d6e085dd64a35143c8b6ae261bb61e3e23dd464c65b670268 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-08-17T17:07:37.218 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:37 smithi063 podman[142362]: 2024-08-17 17:07:37.085604851 +0000 UTC m=+0.018323701 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:07:37.218 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:37 smithi063 podman[142362]: 2024-08-17 17:07:37.21782982 +0000 UTC m=+0.150548670 container create e45c75b955ea998dfa90b0ad799cddeba539ecf8778040cb840583eeb19f2628 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS) 2024-08-17T17:07:37.495 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:37 smithi063 podman[142362]: 2024-08-17 17:07:37.403620494 +0000 UTC m=+0.336339356 container init e45c75b955ea998dfa90b0ad799cddeba539ecf8778040cb840583eeb19f2628 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-08-17T17:07:37.495 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:37 smithi063 podman[142362]: 2024-08-17 17:07:37.408729897 +0000 UTC m=+0.341448746 container start e45c75b955ea998dfa90b0ad799cddeba539ecf8778040cb840583eeb19f2628 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-08-17T17:07:37.496 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-osd[142400]: -- 172.21.15.63:0/2111841295 <== mon.0 v2:172.21.15.63:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55aac5bc5a40 con 0x55aac6992000 2024-08-17T17:07:37.631 INFO:tasks.rados.rados.0.smithi063.stdout:3037: done (0 left) 2024-08-17T17:07:37.631 INFO:tasks.rados.rados.0.smithi063.stdout:3038: read oid 22 snap -1 2024-08-17T17:07:37.631 INFO:tasks.rados.rados.0.smithi063.stdout:3038: expect deleted 2024-08-17T17:07:37.631 INFO:tasks.rados.rados.0.smithi063.stdout:3039: snap_create 2024-08-17T17:07:37.779 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:37 smithi063 bash[142362]: e45c75b955ea998dfa90b0ad799cddeba539ecf8778040cb840583eeb19f2628 2024-08-17T17:07:37.779 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:37 smithi063 systemd[1]: Started Ceph osd.3 for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:07:37.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[109843]: pgmap v699: 89 pgs: 10 active+undersized, 7 peering, 6 stale+active+clean, 2 active+undersized+degraded, 64 active+clean; 266 MiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 773 KiB/s wr, 7 op/s; 10/501 objects degraded (1.996%) 2024-08-17T17:07:37.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[109843]: Health check failed: Degraded data redundancy: 10/501 objects degraded (1.996%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:37.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[109843]: osdmap e829: 8 total, 7 up, 8 in 2024-08-17T17:07:37.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:37.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:37.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:37.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:37.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[109843]: osdmap e830: 8 total, 7 up, 8 in 2024-08-17T17:07:37.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[114756]: pgmap v699: 89 pgs: 10 active+undersized, 7 peering, 6 stale+active+clean, 2 active+undersized+degraded, 64 active+clean; 266 MiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 773 KiB/s wr, 7 op/s; 10/501 objects degraded (1.996%) 2024-08-17T17:07:37.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[114756]: Health check failed: Degraded data redundancy: 10/501 objects degraded (1.996%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:37.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[114756]: osdmap e829: 8 total, 7 up, 8 in 2024-08-17T17:07:37.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:37.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:37.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:37.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:37.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:37 smithi063 ceph-mon[114756]: osdmap e830: 8 total, 7 up, 8 in 2024-08-17T17:07:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:37 smithi162 ceph-mon[104668]: pgmap v699: 89 pgs: 10 active+undersized, 7 peering, 6 stale+active+clean, 2 active+undersized+degraded, 64 active+clean; 266 MiB data, 846 MiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 773 KiB/s wr, 7 op/s; 10/501 objects degraded (1.996%) 2024-08-17T17:07:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:37 smithi162 ceph-mon[104668]: Health check failed: Degraded data redundancy: 10/501 objects degraded (1.996%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:37 smithi162 ceph-mon[104668]: osdmap e829: 8 total, 7 up, 8 in 2024-08-17T17:07:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:37 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:37 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:37 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:37 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:37 smithi162 ceph-mon[104668]: osdmap e830: 8 total, 7 up, 8 in 2024-08-17T17:07:38.631 INFO:tasks.rados.rados.0.smithi063.stdout:3038: done (1 left) 2024-08-17T17:07:38.631 INFO:tasks.rados.rados.0.smithi063.stdout:3039: done (0 left) 2024-08-17T17:07:38.631 INFO:tasks.rados.rados.0.smithi063.stdout:3040: read oid 29 snap -1 2024-08-17T17:07:38.631 INFO:tasks.rados.rados.0.smithi063.stdout:3040: expect (ObjNum 1127 snap 374 seq_num 1127) 2024-08-17T17:07:38.631 INFO:tasks.rados.rados.0.smithi063.stdout:3041: snap_create 2024-08-17T17:07:38.636 INFO:tasks.rados.rados.0.smithi063.stdout:3040: expect (ObjNum 1127 snap 374 seq_num 1127) 2024-08-17T17:07:39.631 INFO:tasks.rados.rados.0.smithi063.stdout:3040: done (1 left) 2024-08-17T17:07:39.632 INFO:tasks.rados.rados.0.smithi063.stdout:3041: done (0 left) 2024-08-17T17:07:39.632 INFO:tasks.rados.rados.0.smithi063.stdout:3042: write oid 19 current snap is 385 2024-08-17T17:07:39.632 INFO:tasks.rados.rados.0.smithi063.stdout:3042: seq_num 1144 ranges {427208=737806,1777518=513724,2966139=261069} 2024-08-17T17:07:39.634 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:39 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3[142377]: 2024-08-17T17:07:39.286+0000 7f2f98a49640 -1 Falling back to public interface 2024-08-17T17:07:39.637 INFO:tasks.rados.rados.0.smithi063.stdout:3042: writing smithi06387500-19 from 427208 to 1165014 tid 1 2024-08-17T17:07:39.644 INFO:tasks.rados.rados.0.smithi063.stdout:3042: writing smithi06387500-19 from 1777518 to 2291242 tid 2 2024-08-17T17:07:39.650 INFO:tasks.rados.rados.0.smithi063.stdout:3042: writing smithi06387500-19 from 2966139 to 3227208 tid 3 2024-08-17T17:07:39.650 INFO:tasks.rados.rados.0.smithi063.stdout:3043: delete oid 35 current snap is 385 2024-08-17T17:07:39.650 INFO:tasks.rados.rados.0.smithi063.stdout:3042: finishing write tid 1 to smithi06387500-19 2024-08-17T17:07:39.650 INFO:tasks.rados.rados.0.smithi063.stdout:3042: finishing write tid 2 to smithi06387500-19 2024-08-17T17:07:39.652 INFO:tasks.rados.rados.0.smithi063.stdout:3042: finishing write tid 3 to smithi06387500-19 2024-08-17T17:07:39.652 INFO:tasks.rados.rados.0.smithi063.stdout:3042: finishing write tid 4 to smithi06387500-19 2024-08-17T17:07:39.653 INFO:tasks.rados.rados.0.smithi063.stdout:3042: finishing write tid 5 to smithi06387500-19 2024-08-17T17:07:39.653 INFO:tasks.rados.rados.0.smithi063.stdout:3042: oid 19 updating version 0 to 1918 2024-08-17T17:07:39.653 INFO:tasks.rados.rados.0.smithi063.stdout:3042: oid 19 version 1918 is already newer than 1915 2024-08-17T17:07:39.653 INFO:tasks.rados.rados.0.smithi063.stdout:3042: oid 19 version 1918 is already newer than 1917 2024-08-17T17:07:39.653 INFO:tasks.rados.rados.0.smithi063.stdout:3042: oid 19 version 1918 is already newer than 1916 2024-08-17T17:07:39.653 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 1918 (ObjNum 1144 snap 385 seq_num 1144) dirty exists 2024-08-17T17:07:39.653 INFO:tasks.rados.rados.0.smithi063.stdout:3042: left oid 19 (ObjNum 1144 snap 385 seq_num 1144) 2024-08-17T17:07:39.654 INFO:tasks.rados.rados.0.smithi063.stdout:3042: done (1 left) 2024-08-17T17:07:39.654 INFO:tasks.rados.rados.0.smithi063.stdout:3043: done (0 left) 2024-08-17T17:07:39.654 INFO:tasks.rados.rados.0.smithi063.stdout:3044: write oid 25 current snap is 385 2024-08-17T17:07:39.654 INFO:tasks.rados.rados.0.smithi063.stdout:3044: seq_num 1146 ranges {460822=488304,1378129=517393,2504632=405962} 2024-08-17T17:07:39.658 INFO:tasks.rados.rados.0.smithi063.stdout:3044: writing smithi06387500-25 from 460822 to 949126 tid 1 2024-08-17T17:07:39.663 INFO:tasks.rados.rados.0.smithi063.stdout:3044: writing smithi06387500-25 from 1378129 to 1895522 tid 2 2024-08-17T17:07:39.669 INFO:tasks.rados.rados.0.smithi063.stdout:3044: writing smithi06387500-25 from 2504632 to 2910594 tid 3 2024-08-17T17:07:39.669 INFO:tasks.rados.rados.0.smithi063.stdout:3045: write oid 38 current snap is 385 2024-08-17T17:07:39.669 INFO:tasks.rados.rados.0.smithi063.stdout:3045: seq_num 1147 ranges {477629=563553,1836611=777404,3048408=229221} 2024-08-17T17:07:39.675 INFO:tasks.rados.rados.0.smithi063.stdout:3045: writing smithi06387500-38 from 477629 to 1041182 tid 1 2024-08-17T17:07:39.683 INFO:tasks.rados.rados.0.smithi063.stdout:3045: writing smithi06387500-38 from 1836611 to 2614015 tid 2 2024-08-17T17:07:39.688 INFO:tasks.rados.rados.0.smithi063.stdout:3045: writing smithi06387500-38 from 3048408 to 3277629 tid 3 2024-08-17T17:07:39.688 INFO:tasks.rados.rados.0.smithi063.stdout:3046: snap_create 2024-08-17T17:07:39.688 INFO:tasks.rados.rados.0.smithi063.stdout:3044: finishing write tid 1 to smithi06387500-25 2024-08-17T17:07:39.688 INFO:tasks.rados.rados.0.smithi063.stdout:3044: finishing write tid 2 to smithi06387500-25 2024-08-17T17:07:39.688 INFO:tasks.rados.rados.0.smithi063.stdout:3044: finishing write tid 3 to smithi06387500-25 2024-08-17T17:07:39.688 INFO:tasks.rados.rados.0.smithi063.stdout:3044: finishing write tid 4 to smithi06387500-25 2024-08-17T17:07:39.688 INFO:tasks.rados.rados.0.smithi063.stdout:3044: finishing write tid 5 to smithi06387500-25 2024-08-17T17:07:39.688 INFO:tasks.rados.rados.0.smithi063.stdout:3044: oid 25 updating version 0 to 2915 2024-08-17T17:07:39.688 INFO:tasks.rados.rados.0.smithi063.stdout:3044: oid 25 updating version 2915 to 2916 2024-08-17T17:07:39.688 INFO:tasks.rados.rados.0.smithi063.stdout:3044: oid 25 version 2916 is already newer than 2914 2024-08-17T17:07:39.688 INFO:tasks.rados.rados.0.smithi063.stdout:3044: oid 25 updating version 2916 to 2917 2024-08-17T17:07:39.688 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 2917 (ObjNum 1146 snap 385 seq_num 1146) dirty exists 2024-08-17T17:07:39.689 INFO:tasks.rados.rados.0.smithi063.stdout:3044: left oid 25 (ObjNum 1146 snap 385 seq_num 1146) 2024-08-17T17:07:39.689 INFO:tasks.rados.rados.0.smithi063.stdout:3045: finishing write tid 1 to smithi06387500-38 2024-08-17T17:07:39.689 INFO:tasks.rados.rados.0.smithi063.stdout:3045: finishing write tid 2 to smithi06387500-38 2024-08-17T17:07:39.689 INFO:tasks.rados.rados.0.smithi063.stdout:3045: finishing write tid 3 to smithi06387500-38 2024-08-17T17:07:39.689 INFO:tasks.rados.rados.0.smithi063.stdout:3045: finishing write tid 4 to smithi06387500-38 2024-08-17T17:07:39.690 INFO:tasks.rados.rados.0.smithi063.stdout:3045: finishing write tid 5 to smithi06387500-38 2024-08-17T17:07:39.690 INFO:tasks.rados.rados.0.smithi063.stdout:3045: oid 38 updating version 0 to 489 2024-08-17T17:07:39.690 INFO:tasks.rados.rados.0.smithi063.stdout:3045: oid 38 version 489 is already newer than 488 2024-08-17T17:07:39.690 INFO:tasks.rados.rados.0.smithi063.stdout:3045: oid 38 updating version 489 to 490 2024-08-17T17:07:39.690 INFO:tasks.rados.rados.0.smithi063.stdout:3045: oid 38 updating version 490 to 491 2024-08-17T17:07:39.690 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 491 (ObjNum 1147 snap 385 seq_num 1147) dirty exists 2024-08-17T17:07:39.690 INFO:tasks.rados.rados.0.smithi063.stdout:3045: left oid 38 (ObjNum 1147 snap 385 seq_num 1147) 2024-08-17T17:07:39.953 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:39 smithi162 ceph-mon[104668]: pgmap v702: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 269 MiB data, 852 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.5 MiB/s wr, 14 op/s; 47/501 objects degraded (9.381%) 2024-08-17T17:07:39.953 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:39 smithi162 ceph-mon[104668]: osdmap e831: 8 total, 7 up, 8 in 2024-08-17T17:07:39.989 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:39 smithi063 ceph-mon[114756]: pgmap v702: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 269 MiB data, 852 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.5 MiB/s wr, 14 op/s; 47/501 objects degraded (9.381%) 2024-08-17T17:07:39.989 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:39 smithi063 ceph-mon[114756]: osdmap e831: 8 total, 7 up, 8 in 2024-08-17T17:07:39.989 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:39 smithi063 ceph-mon[109843]: pgmap v702: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 269 MiB data, 852 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.5 MiB/s wr, 14 op/s; 47/501 objects degraded (9.381%) 2024-08-17T17:07:39.989 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:39 smithi063 ceph-mon[109843]: osdmap e831: 8 total, 7 up, 8 in 2024-08-17T17:07:40.645 INFO:tasks.rados.rados.0.smithi063.stdout:3044: done (2 left) 2024-08-17T17:07:40.645 INFO:tasks.rados.rados.0.smithi063.stdout:3045: done (1 left) 2024-08-17T17:07:40.645 INFO:tasks.rados.rados.0.smithi063.stdout:3046: done (0 left) 2024-08-17T17:07:40.645 INFO:tasks.rados.rados.0.smithi063.stdout:3047: snap_create 2024-08-17T17:07:40.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:40 smithi162 ceph-mon[104668]: osdmap e832: 8 total, 7 up, 8 in 2024-08-17T17:07:40.935 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:40 smithi063 ceph-mon[114756]: osdmap e832: 8 total, 7 up, 8 in 2024-08-17T17:07:40.936 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:40 smithi063 ceph-mon[109843]: osdmap e832: 8 total, 7 up, 8 in 2024-08-17T17:07:41.643 INFO:tasks.rados.rados.0.smithi063.stdout:3047: done (0 left) 2024-08-17T17:07:41.643 INFO:tasks.rados.rados.0.smithi063.stdout:3048: snap_create 2024-08-17T17:07:41.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:41 smithi162 ceph-mon[104668]: pgmap v705: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 269 MiB data, 852 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 791 KiB/s wr, 9 op/s; 47/501 objects degraded (9.381%) 2024-08-17T17:07:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:41 smithi162 ceph-mon[104668]: osdmap e833: 8 total, 7 up, 8 in 2024-08-17T17:07:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:41 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:41 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:41 smithi063 ceph-mon[109843]: pgmap v705: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 269 MiB data, 852 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 791 KiB/s wr, 9 op/s; 47/501 objects degraded (9.381%) 2024-08-17T17:07:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:41 smithi063 ceph-mon[109843]: osdmap e833: 8 total, 7 up, 8 in 2024-08-17T17:07:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:41 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:41 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:41 smithi063 ceph-mon[114756]: pgmap v705: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 269 MiB data, 852 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 791 KiB/s wr, 9 op/s; 47/501 objects degraded (9.381%) 2024-08-17T17:07:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:41 smithi063 ceph-mon[114756]: osdmap e833: 8 total, 7 up, 8 in 2024-08-17T17:07:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:41 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:42.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:41 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:42.654 INFO:tasks.rados.rados.0.smithi063.stdout:3048: done (0 left) 2024-08-17T17:07:42.654 INFO:tasks.rados.rados.0.smithi063.stdout:3049: delete oid 36 current snap is 388 2024-08-17T17:07:42.659 INFO:tasks.rados.rados.0.smithi063.stdout:3049: done (0 left) 2024-08-17T17:07:42.659 INFO:tasks.rados.rados.0.smithi063.stdout:3050: read oid 20 snap -1 2024-08-17T17:07:42.659 INFO:tasks.rados.rados.0.smithi063.stdout:3050: expect (ObjNum 1141 snap 381 seq_num 1141) 2024-08-17T17:07:42.659 INFO:tasks.rados.rados.0.smithi063.stdout:3051: snap_remove snap 381 2024-08-17T17:07:42.937 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:42 smithi063 ceph-mon[114756]: osdmap e834: 8 total, 7 up, 8 in 2024-08-17T17:07:42.937 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:42 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:42.937 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:42 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:42.937 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:42 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:42.937 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:42 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:42.937 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:42 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 47/501 objects degraded (9.381%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:42.938 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:42 smithi063 ceph-mon[109843]: osdmap e834: 8 total, 7 up, 8 in 2024-08-17T17:07:42.938 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:42 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:42.938 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:42 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:42.938 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:42 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:42.938 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:42 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:42.938 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:42 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 47/501 objects degraded (9.381%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:43.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:42 smithi162 ceph-mon[104668]: osdmap e834: 8 total, 7 up, 8 in 2024-08-17T17:07:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:42 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:42 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:42 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:42 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:42 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 47/501 objects degraded (9.381%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:43.279 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:42 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3[142377]: 2024-08-17T17:07:42.996+0000 7f2f98a49640 -1 osd.3 824 log_to_monitors true 2024-08-17T17:07:43.573 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:07:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:07:43] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.51.0" 2024-08-17T17:07:43.666 INFO:tasks.rados.rados.0.smithi063.stdout:3051: done (1 left) 2024-08-17T17:07:43.667 INFO:tasks.rados.rados.0.smithi063.stdout:3052: snap_remove snap 382 2024-08-17T17:07:44.029 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:07:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3[142377]: 2024-08-17T17:07:43.682+0000 7f2f907f3640 -1 osd.3 824 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:07:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:43 smithi063 ceph-mon[114756]: pgmap v708: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 278 MiB data, 870 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.3 MiB/s wr, 9 op/s; 49/513 objects degraded (9.552%) 2024-08-17T17:07:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:43 smithi063 ceph-mon[114756]: osdmap e835: 8 total, 7 up, 8 in 2024-08-17T17:07:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:43 smithi063 ceph-mon[114756]: from='osd.3 [v2:172.21.15.63:6826/1764356824,v1:172.21.15.63:6827/1764356824]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:07:44.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:43 smithi063 ceph-mon[109843]: pgmap v708: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 278 MiB data, 870 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.3 MiB/s wr, 9 op/s; 49/513 objects degraded (9.552%) 2024-08-17T17:07:44.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:43 smithi063 ceph-mon[109843]: osdmap e835: 8 total, 7 up, 8 in 2024-08-17T17:07:44.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:43 smithi063 ceph-mon[109843]: from='osd.3 [v2:172.21.15.63:6826/1764356824,v1:172.21.15.63:6827/1764356824]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:07:44.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:43 smithi162 ceph-mon[104668]: pgmap v708: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 278 MiB data, 870 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.3 MiB/s wr, 9 op/s; 49/513 objects degraded (9.552%) 2024-08-17T17:07:44.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:43 smithi162 ceph-mon[104668]: osdmap e835: 8 total, 7 up, 8 in 2024-08-17T17:07:44.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:43 smithi162 ceph-mon[104668]: from='osd.3 [v2:172.21.15.63:6826/1764356824,v1:172.21.15.63:6827/1764356824]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-08-17T17:07:44.672 INFO:tasks.rados.rados.0.smithi063.stdout:3052: done (1 left) 2024-08-17T17:07:44.673 INFO:tasks.rados.rados.0.smithi063.stdout:3053: read oid 48 snap -1 2024-08-17T17:07:44.673 INFO:tasks.rados.rados.0.smithi063.stdout:3053: expect (ObjNum 1143 snap 381 seq_num 1143) 2024-08-17T17:07:44.673 INFO:tasks.rados.rados.0.smithi063.stdout:3054: write oid 11 current snap is 388 2024-08-17T17:07:44.673 INFO:tasks.rados.rados.0.smithi063.stdout:3054: seq_num 1149 ranges {511243=430404,1669928=664720,3103254=207989} 2024-08-17T17:07:44.675 INFO:tasks.rados.rados.0.smithi063.stdout:3054: writing smithi06387500-11 from 511243 to 941647 tid 1 2024-08-17T17:07:44.684 INFO:tasks.rados.rados.0.smithi063.stdout:3054: writing smithi06387500-11 from 1669928 to 2334648 tid 2 2024-08-17T17:07:44.691 INFO:tasks.rados.rados.0.smithi063.stdout:3054: writing smithi06387500-11 from 3103254 to 3311243 tid 3 2024-08-17T17:07:44.691 INFO:tasks.rados.rados.0.smithi063.stdout:3055: read oid 50 snap -1 2024-08-17T17:07:44.691 INFO:tasks.rados.rados.0.smithi063.stdout:3055: expect (ObjNum 1088 snap 363 seq_num 1088) 2024-08-17T17:07:44.692 INFO:tasks.rados.rados.0.smithi063.stdout:3056: delete oid 29 current snap is 388 2024-08-17T17:07:44.692 INFO:tasks.rados.rados.0.smithi063.stdout:3050: expect (ObjNum 1141 snap 381 seq_num 1141) 2024-08-17T17:07:44.966 INFO:tasks.rados.rados.0.smithi063.stdout:3054: finishing write tid 1 to smithi06387500-11 2024-08-17T17:07:44.966 INFO:tasks.rados.rados.0.smithi063.stdout:3054: finishing write tid 2 to smithi06387500-11 2024-08-17T17:07:44.966 INFO:tasks.rados.rados.0.smithi063.stdout:3050: done (4 left) 2024-08-17T17:07:44.967 INFO:tasks.rados.rados.0.smithi063.stdout:3056: done (3 left) 2024-08-17T17:07:44.967 INFO:tasks.rados.rados.0.smithi063.stdout:3057: read oid 32 snap -1 2024-08-17T17:07:44.967 INFO:tasks.rados.rados.0.smithi063.stdout:3057: expect (ObjNum 882 snap 308 seq_num 882) 2024-08-17T17:07:44.967 INFO:tasks.rados.rados.0.smithi063.stdout:3058: snap_create 2024-08-17T17:07:44.967 INFO:tasks.rados.rados.0.smithi063.stdout:3054: finishing write tid 3 to smithi06387500-11 2024-08-17T17:07:44.967 INFO:tasks.rados.rados.0.smithi063.stdout:3054: finishing write tid 4 to smithi06387500-11 2024-08-17T17:07:44.967 INFO:tasks.rados.rados.0.smithi063.stdout:3054: finishing write tid 5 to smithi06387500-11 2024-08-17T17:07:44.967 INFO:tasks.rados.rados.0.smithi063.stdout:3054: oid 11 updating version 0 to 497 2024-08-17T17:07:44.967 INFO:tasks.rados.rados.0.smithi063.stdout:3054: oid 11 updating version 497 to 498 2024-08-17T17:07:44.967 INFO:tasks.rados.rados.0.smithi063.stdout:3054: oid 11 updating version 498 to 499 2024-08-17T17:07:44.967 INFO:tasks.rados.rados.0.smithi063.stdout:3054: oid 11 updating version 499 to 500 2024-08-17T17:07:44.967 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 500 (ObjNum 1149 snap 388 seq_num 1149) dirty exists 2024-08-17T17:07:44.967 INFO:tasks.rados.rados.0.smithi063.stdout:3054: left oid 11 (ObjNum 1149 snap 388 seq_num 1149) 2024-08-17T17:07:44.968 INFO:tasks.rados.rados.0.smithi063.stdout:3055: expect (ObjNum 1088 snap 363 seq_num 1088) 2024-08-17T17:07:45.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:44 smithi063 ceph-mon[114756]: from='osd.3 [v2:172.21.15.63:6826/1764356824,v1:172.21.15.63:6827/1764356824]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:07:45.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:44 smithi063 ceph-mon[114756]: osdmap e836: 8 total, 7 up, 8 in 2024-08-17T17:07:45.020 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:44 smithi063 ceph-mon[114756]: from='osd.3 [v2:172.21.15.63:6826/1764356824,v1:172.21.15.63:6827/1764356824]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]} : dispatch 2024-08-17T17:07:45.020 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:44 smithi063 ceph-mon[109843]: from='osd.3 [v2:172.21.15.63:6826/1764356824,v1:172.21.15.63:6827/1764356824]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:07:45.020 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:44 smithi063 ceph-mon[109843]: osdmap e836: 8 total, 7 up, 8 in 2024-08-17T17:07:45.020 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:44 smithi063 ceph-mon[109843]: from='osd.3 [v2:172.21.15.63:6826/1764356824,v1:172.21.15.63:6827/1764356824]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]} : dispatch 2024-08-17T17:07:45.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:44 smithi162 ceph-mon[104668]: from='osd.3 [v2:172.21.15.63:6826/1764356824,v1:172.21.15.63:6827/1764356824]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-08-17T17:07:45.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:44 smithi162 ceph-mon[104668]: osdmap e836: 8 total, 7 up, 8 in 2024-08-17T17:07:45.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:44 smithi162 ceph-mon[104668]: from='osd.3 [v2:172.21.15.63:6826/1764356824,v1:172.21.15.63:6827/1764356824]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi063", "root=default"]} : dispatch 2024-08-17T17:07:45.672 INFO:tasks.rados.rados.0.smithi063.stdout:3054: done (4 left) 2024-08-17T17:07:45.672 INFO:tasks.rados.rados.0.smithi063.stdout:3055: done (3 left) 2024-08-17T17:07:45.672 INFO:tasks.rados.rados.0.smithi063.stdout:3058: done (2 left) 2024-08-17T17:07:45.673 INFO:tasks.rados.rados.0.smithi063.stdout:3059: rollback oid 36 current snap is 389 2024-08-17T17:07:45.673 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 36 to 366 2024-08-17T17:07:45.673 INFO:tasks.rados.rados.0.smithi063.stdout:3060: delete oid 20 current snap is 389 2024-08-17T17:07:45.687 INFO:tasks.rados.rados.0.smithi063.stdout:3060: done (3 left) 2024-08-17T17:07:45.688 INFO:tasks.rados.rados.0.smithi063.stdout:3061: write oid 27 current snap is 389 2024-08-17T17:07:45.688 INFO:tasks.rados.rados.0.smithi063.stdout:3061: seq_num 1152 ranges {561664=656151,1729021=528400,2669170=692494} 2024-08-17T17:07:45.693 INFO:tasks.rados.rados.0.smithi063.stdout:3061: writing smithi06387500-27 from 561664 to 1217815 tid 1 2024-08-17T17:07:45.699 INFO:tasks.rados.rados.0.smithi063.stdout:3061: writing smithi06387500-27 from 1729021 to 2257421 tid 2 2024-08-17T17:07:45.706 INFO:tasks.rados.rados.0.smithi063.stdout:3061: writing smithi06387500-27 from 2669170 to 3361664 tid 3 2024-08-17T17:07:45.707 INFO:tasks.rados.rados.0.smithi063.stdout:3062: read oid 44 snap -1 2024-08-17T17:07:45.707 INFO:tasks.rados.rados.0.smithi063.stdout:3062: expect (ObjNum 1124 snap 372 seq_num 1124) 2024-08-17T17:07:45.707 INFO:tasks.rados.rados.0.smithi063.stdout:3063: read oid 13 snap -1 2024-08-17T17:07:45.707 INFO:tasks.rados.rados.0.smithi063.stdout:3063: expect (ObjNum 1090 snap 363 seq_num 1090) 2024-08-17T17:07:45.707 INFO:tasks.rados.rados.0.smithi063.stdout:3064: write oid 21 current snap is 389 2024-08-17T17:07:45.707 INFO:tasks.rados.rados.0.smithi063.stdout:3064: seq_num 1153 ranges {578471=731400,1787503=672058,3096593=281878} 2024-08-17T17:07:45.714 INFO:tasks.rados.rados.0.smithi063.stdout:3064: writing smithi06387500-21 from 578471 to 1309871 tid 1 2024-08-17T17:07:45.720 INFO:tasks.rados.rados.0.smithi063.stdout:3064: writing smithi06387500-21 from 1787503 to 2459561 tid 2 2024-08-17T17:07:45.726 INFO:tasks.rados.rados.0.smithi063.stdout:3064: writing smithi06387500-21 from 3096593 to 3378471 tid 3 2024-08-17T17:07:45.726 INFO:tasks.rados.rados.0.smithi063.stdout:3065: write oid 5 current snap is 389 2024-08-17T17:07:45.726 INFO:tasks.rados.rados.0.smithi063.stdout:3065: seq_num 1154 ranges {595278=406649,1729632=532069,3007663=387615} 2024-08-17T17:07:45.731 INFO:tasks.rados.rados.0.smithi063.stdout:3065: writing smithi06387500-5 from 595278 to 1001927 tid 1 2024-08-17T17:07:45.738 INFO:tasks.rados.rados.0.smithi063.stdout:3065: writing smithi06387500-5 from 1729632 to 2261701 tid 2 2024-08-17T17:07:45.745 INFO:tasks.rados.rados.0.smithi063.stdout:3065: writing smithi06387500-5 from 3007663 to 3395278 tid 3 2024-08-17T17:07:45.745 INFO:tasks.rados.rados.0.smithi063.stdout:3066: snap_remove snap 385 2024-08-17T17:07:46.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: pgmap v711: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 278 MiB data, 851 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.3 MiB/s wr, 9 op/s; 49/513 objects degraded (9.552%) 2024-08-17T17:07:46.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:07:46.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: osd.3 [v2:172.21.15.63:6826/1764356824,v1:172.21.15.63:6827/1764356824] boot 2024-08-17T17:07:46.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: osdmap e837: 8 total, 8 up, 8 in 2024-08-17T17:07:46.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:07:46.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:46.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:46.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:46.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:07:46.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: Upgrade: Finalizing container_image settings 2024-08-17T17:07:46.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:07:46.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:07:46.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:07:46.033 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:07:46.034 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:07:46.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: pgmap v711: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 278 MiB data, 851 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.3 MiB/s wr, 9 op/s; 49/513 objects degraded (9.552%) 2024-08-17T17:07:46.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:07:46.034 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: osd.3 [v2:172.21.15.63:6826/1764356824,v1:172.21.15.63:6827/1764356824] boot 2024-08-17T17:07:46.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: osdmap e837: 8 total, 8 up, 8 in 2024-08-17T17:07:46.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:07:46.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:46.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:46.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:46.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:07:46.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.036 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: Upgrade: Finalizing container_image settings 2024-08-17T17:07:46.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:07:46.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:07:46.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:07:46.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:07:46.037 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:07:46.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:07:46.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T17:07:46.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:07:46.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:07:46.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:07:46.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:07:46.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:07:46.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:07:46.038 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:07:46.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:07:46.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:07:46.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:07:46.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:07:46.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:07:46.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:07:46.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:07:46.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:07:46.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:07:46.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:07:46.039 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.040 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:07:46.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T17:07:46.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:07:46.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:07:46.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:07:46.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:07:46.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:07:46.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:07:46.040 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:07:46.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:07:46.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:07:46.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:07:46.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:07:46.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:07:46.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:07:46.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:07:46.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:07:46.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:07:46.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:07:46.041 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.042 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: Upgrade: Complete! 2024-08-17T17:07:46.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:07:46.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:07:46.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:07:46.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:46.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:46.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:46.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:46.043 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:46.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:46.044 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.044 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.044 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.044 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.044 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.044 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.044 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.044 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.044 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.044 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.044 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.045 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: Upgrade: Complete! 2024-08-17T17:07:46.045 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:07:46.045 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:07:46.045 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:07:46.045 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:46.045 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:46.045 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:46.045 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.045 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:46.045 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:46.045 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:46.045 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: pgmap v711: 89 pgs: 28 active+undersized, 6 active+undersized+degraded, 55 active+clean; 278 MiB data, 851 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.3 MiB/s wr, 9 op/s; 49/513 objects degraded (9.552%) 2024-08-17T17:07:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:07:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: osd.3 [v2:172.21.15.63:6826/1764356824,v1:172.21.15.63:6827/1764356824] boot 2024-08-17T17:07:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: osdmap e837: 8 total, 8 up, 8 in 2024-08-17T17:07:46.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-08-17T17:07:46.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:46.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:46.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:46.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: Upgrade: Setting container_image for all nvmeof 2024-08-17T17:07:46.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-08-17T17:07:46.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: Upgrade: Finalizing container_image settings 2024-08-17T17:07:46.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:07:46.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-08-17T17:07:46.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-08-17T17:07:46.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-08-17T17:07:46.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:07:46.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-08-17T17:07:46.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-08-17T17:07:46.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:07:46.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-08-17T17:07:46.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:07:46.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-08-17T17:07:46.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:07:46.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-08-17T17:07:46.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:07:46.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-08-17T17:07:46.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:07:46.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-08-17T17:07:46.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:07:46.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-08-17T17:07:46.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:07:46.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-08-17T17:07:46.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:07:46.100 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-08-17T17:07:46.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-08-17T17:07:46.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-08-17T17:07:46.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: Upgrade: Complete! 2024-08-17T17:07:46.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:07:46.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-08-17T17:07:46.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-08-17T17:07:46.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:46.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:46.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:46.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:07:46.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:07:46.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:07:46.102 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:46.681 INFO:tasks.rados.rados.0.smithi063.stdout:3066: done (8 left) 2024-08-17T17:07:46.682 INFO:tasks.rados.rados.0.smithi063.stdout:3067: snap_create 2024-08-17T17:07:46.682 INFO:tasks.rados.rados.0.smithi063.stdout:3057: expect (ObjNum 882 snap 308 seq_num 882) 2024-08-17T17:07:46.844 INFO:tasks.rados.rados.0.smithi063.stdout:3061: finishing write tid 1 to smithi06387500-27 2024-08-17T17:07:46.844 INFO:tasks.rados.rados.0.smithi063.stdout:3061: finishing write tid 2 to smithi06387500-27 2024-08-17T17:07:46.844 INFO:tasks.rados.rados.0.smithi063.stdout:3053: expect (ObjNum 1143 snap 381 seq_num 1143) 2024-08-17T17:07:46.989 INFO:tasks.rados.rados.0.smithi063.stdout:3059: finishing rollback tid 1 to smithi06387500-36 2024-08-17T17:07:46.989 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 2919 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T17:07:46.990 INFO:tasks.rados.rados.0.smithi063.stdout:3061: finishing write tid 3 to smithi06387500-27 2024-08-17T17:07:46.990 INFO:tasks.rados.rados.0.smithi063.stdout:3061: finishing write tid 4 to smithi06387500-27 2024-08-17T17:07:46.990 INFO:tasks.rados.rados.0.smithi063.stdout:3061: finishing write tid 5 to smithi06387500-27 2024-08-17T17:07:46.990 INFO:tasks.rados.rados.0.smithi063.stdout:3061: oid 27 updating version 0 to 273 2024-08-17T17:07:46.990 INFO:tasks.rados.rados.0.smithi063.stdout:3061: oid 27 updating version 273 to 276 2024-08-17T17:07:46.990 INFO:tasks.rados.rados.0.smithi063.stdout:3061: oid 27 version 276 is already newer than 274 2024-08-17T17:07:46.990 INFO:tasks.rados.rados.0.smithi063.stdout:3061: oid 27 version 276 is already newer than 275 2024-08-17T17:07:46.990 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 276 (ObjNum 1152 snap 389 seq_num 1152) dirty exists 2024-08-17T17:07:46.990 INFO:tasks.rados.rados.0.smithi063.stdout:3061: left oid 27 (ObjNum 1152 snap 389 seq_num 1152) 2024-08-17T17:07:46.990 INFO:tasks.rados.rados.0.smithi063.stdout:3063: expect (ObjNum 1090 snap 363 seq_num 1090) 2024-08-17T17:07:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:46 smithi063 ceph-mon[109843]: osdmap e838: 8 total, 8 up, 8 in 2024-08-17T17:07:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:46 smithi063 ceph-mon[109843]: pgmap v714: 89 pgs: 4 peering, 20 active+undersized, 5 active+undersized+degraded, 60 active+clean; 280 MiB data, 852 MiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 319 KiB/s wr, 5 op/s; 39/519 objects degraded (7.514%) 2024-08-17T17:07:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:46 smithi063 ceph-mon[114756]: osdmap e838: 8 total, 8 up, 8 in 2024-08-17T17:07:47.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:46 smithi063 ceph-mon[114756]: pgmap v714: 89 pgs: 4 peering, 20 active+undersized, 5 active+undersized+degraded, 60 active+clean; 280 MiB data, 852 MiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 319 KiB/s wr, 5 op/s; 39/519 objects degraded (7.514%) 2024-08-17T17:07:47.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:47.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:47.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:47.069 INFO:tasks.rados.rados.0.smithi063.stdout:3064: finishing write tid 1 to smithi06387500-21 2024-08-17T17:07:47.069 INFO:tasks.rados.rados.0.smithi063.stdout:3062: expect (ObjNum 1124 snap 372 seq_num 1124) 2024-08-17T17:07:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:46 smithi162 ceph-mon[104668]: osdmap e838: 8 total, 8 up, 8 in 2024-08-17T17:07:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:46 smithi162 ceph-mon[104668]: pgmap v714: 89 pgs: 4 peering, 20 active+undersized, 5 active+undersized+degraded, 60 active+clean; 280 MiB data, 852 MiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 319 KiB/s wr, 5 op/s; 39/519 objects degraded (7.514%) 2024-08-17T17:07:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:07:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:47.236 INFO:tasks.rados.rados.0.smithi063.stdout:3064: finishing write tid 2 to smithi06387500-21 2024-08-17T17:07:47.237 INFO:tasks.rados.rados.0.smithi063.stdout:3064: finishing write tid 3 to smithi06387500-21 2024-08-17T17:07:47.237 INFO:tasks.rados.rados.0.smithi063.stdout:3064: finishing write tid 4 to smithi06387500-21 2024-08-17T17:07:47.237 INFO:tasks.rados.rados.0.smithi063.stdout:3064: finishing write tid 5 to smithi06387500-21 2024-08-17T17:07:47.237 INFO:tasks.rados.rados.0.smithi063.stdout:3064: oid 21 updating version 0 to 1923 2024-08-17T17:07:47.237 INFO:tasks.rados.rados.0.smithi063.stdout:3064: oid 21 version 1923 is already newer than 1921 2024-08-17T17:07:47.237 INFO:tasks.rados.rados.0.smithi063.stdout:3064: oid 21 version 1923 is already newer than 1922 2024-08-17T17:07:47.237 INFO:tasks.rados.rados.0.smithi063.stdout:3064: oid 21 version 1923 is already newer than 1920 2024-08-17T17:07:47.237 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 1923 (ObjNum 1153 snap 389 seq_num 1153) dirty exists 2024-08-17T17:07:47.237 INFO:tasks.rados.rados.0.smithi063.stdout:3064: left oid 21 (ObjNum 1153 snap 389 seq_num 1153) 2024-08-17T17:07:47.237 INFO:tasks.rados.rados.0.smithi063.stdout:3065: finishing write tid 1 to smithi06387500-5 2024-08-17T17:07:47.237 INFO:tasks.rados.rados.0.smithi063.stdout:3065: finishing write tid 2 to smithi06387500-5 2024-08-17T17:07:47.238 INFO:tasks.rados.rados.0.smithi063.stdout:3065: finishing write tid 3 to smithi06387500-5 2024-08-17T17:07:47.238 INFO:tasks.rados.rados.0.smithi063.stdout:3065: finishing write tid 4 to smithi06387500-5 2024-08-17T17:07:47.238 INFO:tasks.rados.rados.0.smithi063.stdout:3065: finishing write tid 5 to smithi06387500-5 2024-08-17T17:07:47.238 INFO:tasks.rados.rados.0.smithi063.stdout:3065: oid 5 updating version 0 to 1476 2024-08-17T17:07:47.238 INFO:tasks.rados.rados.0.smithi063.stdout:3065: oid 5 updating version 1476 to 1477 2024-08-17T17:07:47.238 INFO:tasks.rados.rados.0.smithi063.stdout:3065: oid 5 updating version 1477 to 1478 2024-08-17T17:07:47.238 INFO:tasks.rados.rados.0.smithi063.stdout:3065: oid 5 version 1478 is already newer than 1475 2024-08-17T17:07:47.238 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1478 (ObjNum 1154 snap 389 seq_num 1154) dirty exists 2024-08-17T17:07:47.238 INFO:tasks.rados.rados.0.smithi063.stdout:3065: left oid 5 (ObjNum 1154 snap 389 seq_num 1154) 2024-08-17T17:07:47.683 INFO:tasks.rados.rados.0.smithi063.stdout:3053: done (8 left) 2024-08-17T17:07:47.683 INFO:tasks.rados.rados.0.smithi063.stdout:3057: done (7 left) 2024-08-17T17:07:47.683 INFO:tasks.rados.rados.0.smithi063.stdout:3059: done (6 left) 2024-08-17T17:07:47.683 INFO:tasks.rados.rados.0.smithi063.stdout:3061: done (5 left) 2024-08-17T17:07:47.684 INFO:tasks.rados.rados.0.smithi063.stdout:3062: done (4 left) 2024-08-17T17:07:47.684 INFO:tasks.rados.rados.0.smithi063.stdout:3063: done (3 left) 2024-08-17T17:07:47.684 INFO:tasks.rados.rados.0.smithi063.stdout:3064: done (2 left) 2024-08-17T17:07:47.684 INFO:tasks.rados.rados.0.smithi063.stdout:3065: done (1 left) 2024-08-17T17:07:47.684 INFO:tasks.rados.rados.0.smithi063.stdout:3067: done (0 left) 2024-08-17T17:07:47.684 INFO:tasks.rados.rados.0.smithi063.stdout:3068: write oid 7 current snap is 390 2024-08-17T17:07:47.684 INFO:tasks.rados.rados.0.smithi063.stdout:3068: seq_num 1155 ranges {612085=481898,1788114=675727,3035086=376999} 2024-08-17T17:07:47.689 INFO:tasks.rados.rados.0.smithi063.stdout:3068: writing smithi06387500-7 from 612085 to 1093983 tid 1 2024-08-17T17:07:47.697 INFO:tasks.rados.rados.0.smithi063.stdout:3068: writing smithi06387500-7 from 1788114 to 2463841 tid 2 2024-08-17T17:07:47.703 INFO:tasks.rados.rados.0.smithi063.stdout:3068: writing smithi06387500-7 from 3035086 to 3412085 tid 3 2024-08-17T17:07:47.703 INFO:tasks.rados.rados.0.smithi063.stdout:3069: read oid 21 snap -1 2024-08-17T17:07:47.703 INFO:tasks.rados.rados.0.smithi063.stdout:3069: expect (ObjNum 1153 snap 389 seq_num 1153) 2024-08-17T17:07:47.703 INFO:tasks.rados.rados.0.smithi063.stdout:3070: rollback oid 17 current snap is 390 2024-08-17T17:07:47.703 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 17 to 369 2024-08-17T17:07:47.703 INFO:tasks.rados.rados.0.smithi063.stdout:3068: finishing write tid 1 to smithi06387500-7 2024-08-17T17:07:47.703 INFO:tasks.rados.rados.0.smithi063.stdout:3071: read oid 44 snap -1 2024-08-17T17:07:47.703 INFO:tasks.rados.rados.0.smithi063.stdout:3071: expect (ObjNum 1124 snap 372 seq_num 1124) 2024-08-17T17:07:47.703 INFO:tasks.rados.rados.0.smithi063.stdout:3072: snap_remove snap 380 2024-08-17T17:07:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:47 smithi063 ceph-mon[114756]: osdmap e839: 8 total, 8 up, 8 in 2024-08-17T17:07:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:47 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 39/519 objects degraded (7.514%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:47 smithi063 ceph-mon[109843]: osdmap e839: 8 total, 8 up, 8 in 2024-08-17T17:07:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:47 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 39/519 objects degraded (7.514%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:47 smithi162 ceph-mon[104668]: osdmap e839: 8 total, 8 up, 8 in 2024-08-17T17:07:48.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:47 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 39/519 objects degraded (7.514%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:07:48.702 INFO:tasks.rados.rados.0.smithi063.stdout:3072: done (4 left) 2024-08-17T17:07:48.702 INFO:tasks.rados.rados.0.smithi063.stdout:3073: read oid 9 snap -1 2024-08-17T17:07:48.702 INFO:tasks.rados.rados.0.smithi063.stdout:3073: expect (ObjNum 1135 snap 377 seq_num 1135) 2024-08-17T17:07:48.702 INFO:tasks.rados.rados.0.smithi063.stdout:3074: rollback oid 6 current snap is 390 2024-08-17T17:07:48.703 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 6 to 387 2024-08-17T17:07:48.703 INFO:tasks.rados.rados.0.smithi063.stdout:3068: finishing write tid 2 to smithi06387500-7 2024-08-17T17:07:48.703 INFO:tasks.rados.rados.0.smithi063.stdout:3068: finishing write tid 3 to smithi06387500-7 2024-08-17T17:07:48.703 INFO:tasks.rados.rados.0.smithi063.stdout:3069: expect (ObjNum 1153 snap 389 seq_num 1153) 2024-08-17T17:07:48.907 INFO:tasks.rados.rados.0.smithi063.stdout:3068: finishing write tid 4 to smithi06387500-7 2024-08-17T17:07:48.907 INFO:tasks.rados.rados.0.smithi063.stdout:3070: finishing rollback tid 0 to smithi06387500-17 2024-08-17T17:07:48.907 INFO:tasks.rados.rados.0.smithi063.stdout:3070: finishing rollback tid 1 to smithi06387500-17 2024-08-17T17:07:48.907 INFO:tasks.rados.rados.0.smithi063.stdout:3069: done (5 left) 2024-08-17T17:07:48.907 INFO:tasks.rados.rados.0.smithi063.stdout:3075: snap_create 2024-08-17T17:07:48.907 INFO:tasks.rados.rados.0.smithi063.stdout:3068: finishing write tid 5 to smithi06387500-7 2024-08-17T17:07:48.907 INFO:tasks.rados.rados.0.smithi063.stdout:3068: oid 7 updating version 0 to 1412 2024-08-17T17:07:48.907 INFO:tasks.rados.rados.0.smithi063.stdout:3068: oid 7 updating version 1412 to 1414 2024-08-17T17:07:48.907 INFO:tasks.rados.rados.0.smithi063.stdout:3068: oid 7 version 1414 is already newer than 1411 2024-08-17T17:07:48.907 INFO:tasks.rados.rados.0.smithi063.stdout:3068: oid 7 version 1414 is already newer than 1413 2024-08-17T17:07:48.908 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1414 (ObjNum 1155 snap 390 seq_num 1155) dirty exists 2024-08-17T17:07:48.908 INFO:tasks.rados.rados.0.smithi063.stdout:3068: left oid 7 (ObjNum 1155 snap 390 seq_num 1155) 2024-08-17T17:07:48.908 INFO:tasks.rados.rados.0.smithi063.stdout:3070: finishing rollback tid 2 to smithi06387500-17 2024-08-17T17:07:48.908 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 1013 (ObjNum 1105 snap 366 seq_num 1105) dirty exists 2024-08-17T17:07:48.908 INFO:tasks.rados.rados.0.smithi063.stdout:3071: expect (ObjNum 1124 snap 372 seq_num 1124) 2024-08-17T17:07:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:48 smithi063 ceph-mon[114756]: osdmap e840: 8 total, 8 up, 8 in 2024-08-17T17:07:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:48 smithi063 ceph-mon[114756]: pgmap v717: 89 pgs: 4 peering, 11 active+undersized, 3 active+undersized+degraded, 71 active+clean; 281 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 643 KiB/s wr, 13 op/s; 10/525 objects degraded (1.905%); 1.7 MiB/s, 1 objects/s recovering 2024-08-17T17:07:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:48 smithi063 ceph-mon[109843]: osdmap e840: 8 total, 8 up, 8 in 2024-08-17T17:07:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:48 smithi063 ceph-mon[109843]: pgmap v717: 89 pgs: 4 peering, 11 active+undersized, 3 active+undersized+degraded, 71 active+clean; 281 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 643 KiB/s wr, 13 op/s; 10/525 objects degraded (1.905%); 1.7 MiB/s, 1 objects/s recovering 2024-08-17T17:07:49.075 INFO:tasks.rados.rados.0.smithi063.stdout:3074: finishing rollback tid 0 to smithi06387500-6 2024-08-17T17:07:49.075 INFO:tasks.rados.rados.0.smithi063.stdout:3074: finishing rollback tid 1 to smithi06387500-6 2024-08-17T17:07:49.075 INFO:tasks.rados.rados.0.smithi063.stdout:3074: finishing rollback tid 2 to smithi06387500-6 2024-08-17T17:07:49.075 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 1020 (ObjNum 900 snap 311 seq_num 900) dirty exists 2024-08-17T17:07:49.076 INFO:tasks.rados.rados.0.smithi063.stdout:3073: expect (ObjNum 1135 snap 377 seq_num 1135) 2024-08-17T17:07:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:48 smithi162 ceph-mon[104668]: osdmap e840: 8 total, 8 up, 8 in 2024-08-17T17:07:49.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:48 smithi162 ceph-mon[104668]: pgmap v717: 89 pgs: 4 peering, 11 active+undersized, 3 active+undersized+degraded, 71 active+clean; 281 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 643 KiB/s wr, 13 op/s; 10/525 objects degraded (1.905%); 1.7 MiB/s, 1 objects/s recovering 2024-08-17T17:07:49.698 INFO:tasks.rados.rados.0.smithi063.stdout:3068: done (5 left) 2024-08-17T17:07:49.698 INFO:tasks.rados.rados.0.smithi063.stdout:3070: done (4 left) 2024-08-17T17:07:49.698 INFO:tasks.rados.rados.0.smithi063.stdout:3071: done (3 left) 2024-08-17T17:07:49.698 INFO:tasks.rados.rados.0.smithi063.stdout:3073: done (2 left) 2024-08-17T17:07:49.698 INFO:tasks.rados.rados.0.smithi063.stdout:3074: done (1 left) 2024-08-17T17:07:49.698 INFO:tasks.rados.rados.0.smithi063.stdout:3075: done (0 left) 2024-08-17T17:07:49.698 INFO:tasks.rados.rados.0.smithi063.stdout:3076: delete oid 11 current snap is 391 2024-08-17T17:07:49.702 INFO:tasks.rados.rados.0.smithi063.stdout:3076: done (0 left) 2024-08-17T17:07:49.702 INFO:tasks.rados.rados.0.smithi063.stdout:3077: snap_remove snap 369 2024-08-17T17:07:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:49 smithi063 ceph-mon[114756]: osdmap e841: 8 total, 8 up, 8 in 2024-08-17T17:07:50.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:49 smithi063 ceph-mon[109843]: osdmap e841: 8 total, 8 up, 8 in 2024-08-17T17:07:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:49 smithi162 ceph-mon[104668]: osdmap e841: 8 total, 8 up, 8 in 2024-08-17T17:07:50.709 INFO:tasks.rados.rados.0.smithi063.stdout:3077: done (0 left) 2024-08-17T17:07:50.709 INFO:tasks.rados.rados.0.smithi063.stdout:3078: rollback oid 30 current snap is 391 2024-08-17T17:07:50.709 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 30 to 354 2024-08-17T17:07:50.710 INFO:tasks.rados.rados.0.smithi063.stdout:3079: delete oid 35 current snap is 391 2024-08-17T17:07:50.717 INFO:tasks.rados.rados.0.smithi063.stdout:3078: finishing rollback tid 0 to smithi06387500-30 2024-08-17T17:07:50.717 INFO:tasks.rados.rados.0.smithi063.stdout:3078: finishing rollback tid 1 to smithi06387500-30 2024-08-17T17:07:50.717 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1423 (ObjNum 113 snap 0 seq_num 1806134752) dirty dne 2024-08-17T17:07:50.719 INFO:tasks.rados.rados.0.smithi063.stdout:3078: done (1 left) 2024-08-17T17:07:50.719 INFO:tasks.rados.rados.0.smithi063.stdout:3079: done (0 left) 2024-08-17T17:07:50.719 INFO:tasks.rados.rados.0.smithi063.stdout:3080: snap_create 2024-08-17T17:07:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:50 smithi063 ceph-mon[109843]: osdmap e842: 8 total, 8 up, 8 in 2024-08-17T17:07:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:50 smithi063 ceph-mon[109843]: pgmap v720: 89 pgs: 4 peering, 85 active+clean; 281 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 324 KiB/s wr, 9 op/s; 1.7 MiB/s, 1 objects/s recovering 2024-08-17T17:07:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:50 smithi063 ceph-mon[114756]: osdmap e842: 8 total, 8 up, 8 in 2024-08-17T17:07:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:50 smithi063 ceph-mon[114756]: pgmap v720: 89 pgs: 4 peering, 85 active+clean; 281 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 324 KiB/s wr, 9 op/s; 1.7 MiB/s, 1 objects/s recovering 2024-08-17T17:07:51.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:50 smithi162 ceph-mon[104668]: osdmap e842: 8 total, 8 up, 8 in 2024-08-17T17:07:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:50 smithi162 ceph-mon[104668]: pgmap v720: 89 pgs: 4 peering, 85 active+clean; 281 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 324 KiB/s wr, 9 op/s; 1.7 MiB/s, 1 objects/s recovering 2024-08-17T17:07:51.715 INFO:tasks.rados.rados.0.smithi063.stdout:3080: done (0 left) 2024-08-17T17:07:51.715 INFO:tasks.rados.rados.0.smithi063.stdout:3081: read oid 34 snap -1 2024-08-17T17:07:51.715 INFO:tasks.rados.rados.0.smithi063.stdout:3081: expect deleted 2024-08-17T17:07:51.715 INFO:tasks.rados.rados.0.smithi063.stdout:3082: write oid 10 current snap is 392 2024-08-17T17:07:51.715 INFO:tasks.rados.rados.0.smithi063.stdout:3082: seq_num 1158 ranges {662506=423998,1563560=539407,2717355=490285} 2024-08-17T17:07:51.720 INFO:tasks.rados.rados.0.smithi063.stdout:3082: writing smithi06387500-10 from 662506 to 1086504 tid 1 2024-08-17T17:07:51.726 INFO:tasks.rados.rados.0.smithi063.stdout:3082: writing smithi06387500-10 from 1563560 to 2102967 tid 2 2024-08-17T17:07:51.732 INFO:tasks.rados.rados.0.smithi063.stdout:3082: writing smithi06387500-10 from 2717355 to 3207640 tid 3 2024-08-17T17:07:51.733 INFO:tasks.rados.rados.0.smithi063.stdout:3083: rollback oid 26 current snap is 392 2024-08-17T17:07:51.733 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 26 to 363 2024-08-17T17:07:51.733 INFO:tasks.rados.rados.0.smithi063.stdout:3082: finishing write tid 1 to smithi06387500-10 2024-08-17T17:07:51.733 INFO:tasks.rados.rados.0.smithi063.stdout:3082: finishing write tid 2 to smithi06387500-10 2024-08-17T17:07:51.733 INFO:tasks.rados.rados.0.smithi063.stdout:3081: done (2 left) 2024-08-17T17:07:51.733 INFO:tasks.rados.rados.0.smithi063.stdout:3084: read oid 50 snap -1 2024-08-17T17:07:51.733 INFO:tasks.rados.rados.0.smithi063.stdout:3084: expect (ObjNum 1088 snap 363 seq_num 1088) 2024-08-17T17:07:51.733 INFO:tasks.rados.rados.0.smithi063.stdout:3085: write oid 25 current snap is 392 2024-08-17T17:07:51.733 INFO:tasks.rados.rados.0.smithi063.stdout:3085: seq_num 1159 ranges {679313=499247,1622042=683065,2744778=701557} 2024-08-17T17:07:51.739 INFO:tasks.rados.rados.0.smithi063.stdout:3085: writing smithi06387500-25 from 679313 to 1178560 tid 1 2024-08-17T17:07:51.745 INFO:tasks.rados.rados.0.smithi063.stdout:3085: writing smithi06387500-25 from 1622042 to 2305107 tid 2 2024-08-17T17:07:51.752 INFO:tasks.rados.rados.0.smithi063.stdout:3085: writing smithi06387500-25 from 2744778 to 3446335 tid 3 2024-08-17T17:07:51.752 INFO:tasks.rados.rados.0.smithi063.stdout:3086: read oid 43 snap -1 2024-08-17T17:07:51.752 INFO:tasks.rados.rados.0.smithi063.stdout:3086: expect deleted 2024-08-17T17:07:51.752 INFO:tasks.rados.rados.0.smithi063.stdout:3087: write oid 9 current snap is 392 2024-08-17T17:07:51.752 INFO:tasks.rados.rados.0.smithi063.stdout:3087: seq_num 1160 ranges {696120=574496,1680524=543076,2888554=512829} 2024-08-17T17:07:51.758 INFO:tasks.rados.rados.0.smithi063.stdout:3087: writing smithi06387500-9 from 696120 to 1270616 tid 1 2024-08-17T17:07:51.764 INFO:tasks.rados.rados.0.smithi063.stdout:3087: writing smithi06387500-9 from 1680524 to 2223600 tid 2 2024-08-17T17:07:51.771 INFO:tasks.rados.rados.0.smithi063.stdout:3087: writing smithi06387500-9 from 2888554 to 3401383 tid 3 2024-08-17T17:07:51.771 INFO:tasks.rados.rados.0.smithi063.stdout:3088: read oid 17 snap 355 2024-08-17T17:07:51.771 INFO:tasks.rados.rados.0.smithi063.stdout:3088: expect deleted 2024-08-17T17:07:51.771 INFO:tasks.rados.rados.0.smithi063.stdout:3089: rollback oid 49 current snap is 392 2024-08-17T17:07:51.771 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 49 to 359 2024-08-17T17:07:51.771 INFO:tasks.rados.rados.0.smithi063.stdout:3084: expect (ObjNum 1088 snap 363 seq_num 1088) 2024-08-17T17:07:51.872 INFO:tasks.rados.rados.0.smithi063.stdout:3082: finishing write tid 3 to smithi06387500-10 2024-08-17T17:07:51.872 INFO:tasks.rados.rados.0.smithi063.stdout:3083: finishing rollback tid 1 to smithi06387500-26 2024-08-17T17:07:51.872 INFO:tasks.rados.rados.0.smithi063.stdout:3082: finishing write tid 4 to smithi06387500-10 2024-08-17T17:07:51.872 INFO:tasks.rados.rados.0.smithi063.stdout:3083: finishing rollback tid 2 to smithi06387500-26 2024-08-17T17:07:51.872 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 1021 (ObjNum 1079 snap 362 seq_num 1079) dirty exists 2024-08-17T17:07:51.872 INFO:tasks.rados.rados.0.smithi063.stdout:3082: finishing write tid 5 to smithi06387500-10 2024-08-17T17:07:51.872 INFO:tasks.rados.rados.0.smithi063.stdout:3082: oid 10 updating version 0 to 985 2024-08-17T17:07:51.872 INFO:tasks.rados.rados.0.smithi063.stdout:3082: oid 10 updating version 985 to 987 2024-08-17T17:07:51.872 INFO:tasks.rados.rados.0.smithi063.stdout:3082: oid 10 version 987 is already newer than 986 2024-08-17T17:07:51.872 INFO:tasks.rados.rados.0.smithi063.stdout:3082: oid 10 updating version 987 to 988 2024-08-17T17:07:51.872 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 988 (ObjNum 1158 snap 392 seq_num 1158) dirty exists 2024-08-17T17:07:51.873 INFO:tasks.rados.rados.0.smithi063.stdout:3082: left oid 10 (ObjNum 1158 snap 392 seq_num 1158) 2024-08-17T17:07:51.873 INFO:tasks.rados.rados.0.smithi063.stdout:3085: finishing write tid 1 to smithi06387500-25 2024-08-17T17:07:51.873 INFO:tasks.rados.rados.0.smithi063.stdout:3085: finishing write tid 2 to smithi06387500-25 2024-08-17T17:07:51.873 INFO:tasks.rados.rados.0.smithi063.stdout:3082: done (7 left) 2024-08-17T17:07:51.873 INFO:tasks.rados.rados.0.smithi063.stdout:3083: done (6 left) 2024-08-17T17:07:51.873 INFO:tasks.rados.rados.0.smithi063.stdout:3084: done (5 left) 2024-08-17T17:07:51.873 INFO:tasks.rados.rados.0.smithi063.stdout:3090: read oid 10 snap 379 2024-08-17T17:07:51.873 INFO:tasks.rados.rados.0.smithi063.stdout:3090: expect deleted 2024-08-17T17:07:51.873 INFO:tasks.rados.rados.0.smithi063.stdout:3091: write oid 50 current snap is 392 2024-08-17T17:07:51.873 INFO:tasks.rados.rados.0.smithi063.stdout:3091: seq_num 1161 ranges {712927=649745,2139006=686734} 2024-08-17T17:07:51.878 INFO:tasks.rados.rados.0.smithi063.stdout:3091: writing smithi06387500-50 from 712927 to 1362672 tid 1 2024-08-17T17:07:51.887 INFO:tasks.rados.rados.0.smithi063.stdout:3091: writing smithi06387500-50 from 2139006 to 2825740 tid 2 2024-08-17T17:07:51.887 INFO:tasks.rados.rados.0.smithi063.stdout:3086: done (6 left) 2024-08-17T17:07:51.887 INFO:tasks.rados.rados.0.smithi063.stdout:3092: rollback oid 30 current snap is 392 2024-08-17T17:07:51.887 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 30 to 359 2024-08-17T17:07:51.887 INFO:tasks.rados.rados.0.smithi063.stdout:3085: finishing write tid 3 to smithi06387500-25 2024-08-17T17:07:51.887 INFO:tasks.rados.rados.0.smithi063.stdout:3085: finishing write tid 4 to smithi06387500-25 2024-08-17T17:07:51.887 INFO:tasks.rados.rados.0.smithi063.stdout:3093: snap_remove snap 367 2024-08-17T17:07:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:51 smithi063 ceph-mon[109843]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/525 objects degraded (1.905%), 3 pgs degraded) 2024-08-17T17:07:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:51 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:07:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:51 smithi063 ceph-mon[109843]: osdmap e843: 8 total, 8 up, 8 in 2024-08-17T17:07:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:51 smithi063 ceph-mon[114756]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/525 objects degraded (1.905%), 3 pgs degraded) 2024-08-17T17:07:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:51 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:07:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:51 smithi063 ceph-mon[114756]: osdmap e843: 8 total, 8 up, 8 in 2024-08-17T17:07:52.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:51 smithi162 ceph-mon[104668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/525 objects degraded (1.905%), 3 pgs degraded) 2024-08-17T17:07:52.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:51 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:07:52.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:51 smithi162 ceph-mon[104668]: osdmap e843: 8 total, 8 up, 8 in 2024-08-17T17:07:52.722 INFO:tasks.rados.rados.0.smithi063.stdout:3093: done (7 left) 2024-08-17T17:07:52.723 INFO:tasks.rados.rados.0.smithi063.stdout:3094: write oid 45 current snap is 392 2024-08-17T17:07:52.723 INFO:tasks.rados.rados.0.smithi063.stdout:3085: finishing write tid 5 to smithi06387500-25 2024-08-17T17:07:52.723 INFO:tasks.rados.rados.0.smithi063.stdout:3085: oid 25 updating version 0 to 2953 2024-08-17T17:07:52.723 INFO:tasks.rados.rados.0.smithi063.stdout:3085: oid 25 version 2953 is already newer than 2951 2024-08-17T17:07:52.723 INFO:tasks.rados.rados.0.smithi063.stdout:3085: oid 25 version 2953 is already newer than 2952 2024-08-17T17:07:52.723 INFO:tasks.rados.rados.0.smithi063.stdout:3085: oid 25 updating version 2953 to 2954 2024-08-17T17:07:52.723 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 2954 (ObjNum 1159 snap 392 seq_num 1159) dirty exists 2024-08-17T17:07:52.723 INFO:tasks.rados.rados.0.smithi063.stdout:3085: left oid 25 (ObjNum 1159 snap 392 seq_num 1159) 2024-08-17T17:07:52.724 INFO:tasks.rados.rados.0.smithi063.stdout:3087: finishing write tid 1 to smithi06387500-9 2024-08-17T17:07:52.724 INFO:tasks.rados.rados.0.smithi063.stdout:3087: finishing write tid 2 to smithi06387500-9 2024-08-17T17:07:52.724 INFO:tasks.rados.rados.0.smithi063.stdout:3087: finishing write tid 3 to smithi06387500-9 2024-08-17T17:07:52.724 INFO:tasks.rados.rados.0.smithi063.stdout:3087: finishing write tid 4 to smithi06387500-9 2024-08-17T17:07:52.724 INFO:tasks.rados.rados.0.smithi063.stdout:3087: finishing write tid 5 to smithi06387500-9 2024-08-17T17:07:52.724 INFO:tasks.rados.rados.0.smithi063.stdout:3087: oid 9 updating version 0 to 1425 2024-08-17T17:07:52.724 INFO:tasks.rados.rados.0.smithi063.stdout:3087: oid 9 updating version 1425 to 1428 2024-08-17T17:07:52.724 INFO:tasks.rados.rados.0.smithi063.stdout:3087: oid 9 version 1428 is already newer than 1427 2024-08-17T17:07:52.724 INFO:tasks.rados.rados.0.smithi063.stdout:3087: oid 9 version 1428 is already newer than 1426 2024-08-17T17:07:52.724 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 1428 (ObjNum 1160 snap 392 seq_num 1160) dirty exists 2024-08-17T17:07:52.724 INFO:tasks.rados.rados.0.smithi063.stdout:3087: left oid 9 (ObjNum 1160 snap 392 seq_num 1160) 2024-08-17T17:07:52.724 INFO:tasks.rados.rados.0.smithi063.stdout:3089: finishing rollback tid 0 to smithi06387500-49 2024-08-17T17:07:52.725 INFO:tasks.rados.rados.0.smithi063.stdout:3089: finishing rollback tid 1 to smithi06387500-49 2024-08-17T17:07:52.725 INFO:tasks.rados.rados.0.smithi063.stdout:3089: finishing rollback tid 2 to smithi06387500-49 2024-08-17T17:07:52.725 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 2958 (ObjNum 896 snap 310 seq_num 896) dirty exists 2024-08-17T17:07:52.725 INFO:tasks.rados.rados.0.smithi063.stdout:3091: finishing write tid 1 to smithi06387500-50 2024-08-17T17:07:52.725 INFO:tasks.rados.rados.0.smithi063.stdout:3092: finishing rollback tid 1 to smithi06387500-30 2024-08-17T17:07:52.725 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1423 (ObjNum 129 snap 0 seq_num 1807328808) dirty dne 2024-08-17T17:07:52.725 INFO:tasks.rados.rados.0.smithi063.stdout:3091: finishing write tid 2 to smithi06387500-50 2024-08-17T17:07:52.725 INFO:tasks.rados.rados.0.smithi063.stdout:3091: finishing write tid 3 to smithi06387500-50 2024-08-17T17:07:52.725 INFO:tasks.rados.rados.0.smithi063.stdout:3091: finishing write tid 4 to smithi06387500-50 2024-08-17T17:07:52.725 INFO:tasks.rados.rados.0.smithi063.stdout:3091: oid 50 updating version 0 to 1933 2024-08-17T17:07:52.725 INFO:tasks.rados.rados.0.smithi063.stdout:3091: oid 50 updating version 1933 to 1935 2024-08-17T17:07:52.725 INFO:tasks.rados.rados.0.smithi063.stdout:3091: oid 50 version 1935 is already newer than 1934 2024-08-17T17:07:52.725 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 1935 (ObjNum 1161 snap 392 seq_num 1161) dirty exists 2024-08-17T17:07:52.726 INFO:tasks.rados.rados.0.smithi063.stdout:3091: left oid 50 (ObjNum 1161 snap 392 seq_num 1161) 2024-08-17T17:07:52.726 INFO:tasks.rados.rados.0.smithi063.stdout:3094: seq_num 1162 ranges {729734=724994,2197488=430392,3227047=302687} 2024-08-17T17:07:52.730 INFO:tasks.rados.rados.0.smithi063.stdout:3094: writing smithi06387500-45 from 729734 to 1454728 tid 1 2024-08-17T17:07:52.737 INFO:tasks.rados.rados.0.smithi063.stdout:3094: writing smithi06387500-45 from 2197488 to 2627880 tid 2 2024-08-17T17:07:52.743 INFO:tasks.rados.rados.0.smithi063.stdout:3094: writing smithi06387500-45 from 3227047 to 3529734 tid 3 2024-08-17T17:07:52.743 INFO:tasks.rados.rados.0.smithi063.stdout:3085: done (7 left) 2024-08-17T17:07:52.743 INFO:tasks.rados.rados.0.smithi063.stdout:3087: done (6 left) 2024-08-17T17:07:52.743 INFO:tasks.rados.rados.0.smithi063.stdout:3088: done (5 left) 2024-08-17T17:07:52.743 INFO:tasks.rados.rados.0.smithi063.stdout:3089: done (4 left) 2024-08-17T17:07:52.743 INFO:tasks.rados.rados.0.smithi063.stdout:3090: done (3 left) 2024-08-17T17:07:52.743 INFO:tasks.rados.rados.0.smithi063.stdout:3091: done (2 left) 2024-08-17T17:07:52.744 INFO:tasks.rados.rados.0.smithi063.stdout:3092: done (1 left) 2024-08-17T17:07:52.744 INFO:tasks.rados.rados.0.smithi063.stdout:3095: write oid 11 current snap is 392 2024-08-17T17:07:52.744 INFO:tasks.rados.rados.0.smithi063.stdout:3094: finishing write tid 1 to smithi06387500-45 2024-08-17T17:07:52.744 INFO:tasks.rados.rados.0.smithi063.stdout:3094: finishing write tid 2 to smithi06387500-45 2024-08-17T17:07:52.744 INFO:tasks.rados.rados.0.smithi063.stdout:3095: seq_num 1163 ranges {746541=400243,1739617=690403,3138117=408424} 2024-08-17T17:07:52.749 INFO:tasks.rados.rados.0.smithi063.stdout:3095: writing smithi06387500-11 from 746541 to 1146784 tid 1 2024-08-17T17:07:52.756 INFO:tasks.rados.rados.0.smithi063.stdout:3095: writing smithi06387500-11 from 1739617 to 2430020 tid 2 2024-08-17T17:07:52.763 INFO:tasks.rados.rados.0.smithi063.stdout:3095: writing smithi06387500-11 from 3138117 to 3546541 tid 3 2024-08-17T17:07:52.763 INFO:tasks.rados.rados.0.smithi063.stdout:3096: read oid 10 snap -1 2024-08-17T17:07:52.763 INFO:tasks.rados.rados.0.smithi063.stdout:3096: expect (ObjNum 1158 snap 392 seq_num 1158) 2024-08-17T17:07:52.763 INFO:tasks.rados.rados.0.smithi063.stdout:3097: rollback oid 36 current snap is 392 2024-08-17T17:07:52.764 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 36 to 391 2024-08-17T17:07:52.764 INFO:tasks.rados.rados.0.smithi063.stdout:3094: finishing write tid 3 to smithi06387500-45 2024-08-17T17:07:52.764 INFO:tasks.rados.rados.0.smithi063.stdout:3094: finishing write tid 4 to smithi06387500-45 2024-08-17T17:07:52.764 INFO:tasks.rados.rados.0.smithi063.stdout:3094: finishing write tid 5 to smithi06387500-45 2024-08-17T17:07:52.764 INFO:tasks.rados.rados.0.smithi063.stdout:3094: oid 45 updating version 0 to 687 2024-08-17T17:07:52.764 INFO:tasks.rados.rados.0.smithi063.stdout:3094: oid 45 updating version 687 to 690 2024-08-17T17:07:52.764 INFO:tasks.rados.rados.0.smithi063.stdout:3094: oid 45 version 690 is already newer than 688 2024-08-17T17:07:52.764 INFO:tasks.rados.rados.0.smithi063.stdout:3094: oid 45 version 690 is already newer than 689 2024-08-17T17:07:52.764 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 690 (ObjNum 1162 snap 392 seq_num 1162) dirty exists 2024-08-17T17:07:52.764 INFO:tasks.rados.rados.0.smithi063.stdout:3094: left oid 45 (ObjNum 1162 snap 392 seq_num 1162) 2024-08-17T17:07:52.764 INFO:tasks.rados.rados.0.smithi063.stdout:3095: finishing write tid 1 to smithi06387500-11 2024-08-17T17:07:52.764 INFO:tasks.rados.rados.0.smithi063.stdout:3094: done (3 left) 2024-08-17T17:07:52.764 INFO:tasks.rados.rados.0.smithi063.stdout:3098: write oid 15 current snap is 392 2024-08-17T17:07:52.765 INFO:tasks.rados.rados.0.smithi063.stdout:3098: seq_num 1164 ranges {763348=591845,1914452=434061,2881893=674270} 2024-08-17T17:07:52.770 INFO:tasks.rados.rados.0.smithi063.stdout:3098: writing smithi06387500-15 from 763348 to 1355193 tid 1 2024-08-17T17:07:52.776 INFO:tasks.rados.rados.0.smithi063.stdout:3098: writing smithi06387500-15 from 1914452 to 2348513 tid 2 2024-08-17T17:07:52.783 INFO:tasks.rados.rados.0.smithi063.stdout:3098: writing smithi06387500-15 from 2881893 to 3556163 tid 3 2024-08-17T17:07:52.783 INFO:tasks.rados.rados.0.smithi063.stdout:3099: write oid 14 current snap is 392 2024-08-17T17:07:52.783 INFO:tasks.rados.rados.0.smithi063.stdout:3095: finishing write tid 2 to smithi06387500-11 2024-08-17T17:07:52.783 INFO:tasks.rados.rados.0.smithi063.stdout:3099: seq_num 1165 ranges {780155=667094,1972934=694072,3309316=270839} 2024-08-17T17:07:52.790 INFO:tasks.rados.rados.0.smithi063.stdout:3099: writing smithi06387500-14 from 780155 to 1447249 tid 1 2024-08-17T17:07:52.797 INFO:tasks.rados.rados.0.smithi063.stdout:3099: writing smithi06387500-14 from 1972934 to 2667006 tid 2 2024-08-17T17:07:52.802 INFO:tasks.rados.rados.0.smithi063.stdout:3099: writing smithi06387500-14 from 3309316 to 3580155 tid 3 2024-08-17T17:07:52.802 INFO:tasks.rados.rados.0.smithi063.stdout:3100: read oid 26 snap -1 2024-08-17T17:07:52.803 INFO:tasks.rados.rados.0.smithi063.stdout:3100: expect (ObjNum 1079 snap 362 seq_num 1079) 2024-08-17T17:07:52.803 INFO:tasks.rados.rados.0.smithi063.stdout:3101: rollback oid 39 current snap is 392 2024-08-17T17:07:52.803 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 39 to 377 2024-08-17T17:07:52.803 INFO:tasks.rados.rados.0.smithi063.stdout:3095: finishing write tid 3 to smithi06387500-11 2024-08-17T17:07:52.803 INFO:tasks.rados.rados.0.smithi063.stdout:3095: finishing write tid 4 to smithi06387500-11 2024-08-17T17:07:52.803 INFO:tasks.rados.rados.0.smithi063.stdout:3096: expect (ObjNum 1158 snap 392 seq_num 1158) 2024-08-17T17:07:52.920 INFO:tasks.rados.rados.0.smithi063.stdout:3095: finishing write tid 5 to smithi06387500-11 2024-08-17T17:07:52.920 INFO:tasks.rados.rados.0.smithi063.stdout:3095: oid 11 updating version 0 to 513 2024-08-17T17:07:52.920 INFO:tasks.rados.rados.0.smithi063.stdout:3095: oid 11 updating version 513 to 516 2024-08-17T17:07:52.920 INFO:tasks.rados.rados.0.smithi063.stdout:3095: oid 11 version 516 is already newer than 514 2024-08-17T17:07:52.920 INFO:tasks.rados.rados.0.smithi063.stdout:3095: oid 11 version 516 is already newer than 515 2024-08-17T17:07:52.921 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 516 (ObjNum 1163 snap 392 seq_num 1163) dirty exists 2024-08-17T17:07:52.921 INFO:tasks.rados.rados.0.smithi063.stdout:3095: left oid 11 (ObjNum 1163 snap 392 seq_num 1163) 2024-08-17T17:07:52.921 INFO:tasks.rados.rados.0.smithi063.stdout:3097: finishing rollback tid 1 to smithi06387500-36 2024-08-17T17:07:52.921 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 2919 (ObjNum 113 snap 0 seq_num 502) dirty dne 2024-08-17T17:07:52.921 INFO:tasks.rados.rados.0.smithi063.stdout:3098: finishing write tid 1 to smithi06387500-15 2024-08-17T17:07:52.921 INFO:tasks.rados.rados.0.smithi063.stdout:3098: finishing write tid 2 to smithi06387500-15 2024-08-17T17:07:52.921 INFO:tasks.rados.rados.0.smithi063.stdout:3095: done (6 left) 2024-08-17T17:07:52.921 INFO:tasks.rados.rados.0.smithi063.stdout:3096: done (5 left) 2024-08-17T17:07:52.921 INFO:tasks.rados.rados.0.smithi063.stdout:3097: done (4 left) 2024-08-17T17:07:52.921 INFO:tasks.rados.rados.0.smithi063.stdout:3102: rollback oid 44 current snap is 392 2024-08-17T17:07:52.921 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 44 to 363 2024-08-17T17:07:52.921 INFO:tasks.rados.rados.0.smithi063.stdout:3098: finishing write tid 3 to smithi06387500-15 2024-08-17T17:07:52.921 INFO:tasks.rados.rados.0.smithi063.stdout:3098: finishing write tid 4 to smithi06387500-15 2024-08-17T17:07:52.922 INFO:tasks.rados.rados.0.smithi063.stdout:3098: finishing write tid 5 to smithi06387500-15 2024-08-17T17:07:52.922 INFO:tasks.rados.rados.0.smithi063.stdout:3098: oid 15 updating version 0 to 1072 2024-08-17T17:07:52.922 INFO:tasks.rados.rados.0.smithi063.stdout:3098: oid 15 version 1072 is already newer than 1070 2024-08-17T17:07:52.922 INFO:tasks.rados.rados.0.smithi063.stdout:3098: oid 15 updating version 1072 to 1073 2024-08-17T17:07:52.922 INFO:tasks.rados.rados.0.smithi063.stdout:3098: oid 15 version 1073 is already newer than 1071 2024-08-17T17:07:52.922 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 1073 (ObjNum 1164 snap 392 seq_num 1164) dirty exists 2024-08-17T17:07:52.922 INFO:tasks.rados.rados.0.smithi063.stdout:3098: left oid 15 (ObjNum 1164 snap 392 seq_num 1164) 2024-08-17T17:07:52.922 INFO:tasks.rados.rados.0.smithi063.stdout:3099: finishing write tid 1 to smithi06387500-14 2024-08-17T17:07:52.922 INFO:tasks.rados.rados.0.smithi063.stdout:3099: finishing write tid 2 to smithi06387500-14 2024-08-17T17:07:52.922 INFO:tasks.rados.rados.0.smithi063.stdout:3101: finishing rollback tid 0 to smithi06387500-39 2024-08-17T17:07:52.922 INFO:tasks.rados.rados.0.smithi063.stdout:3101: finishing rollback tid 1 to smithi06387500-39 2024-08-17T17:07:52.922 INFO:tasks.rados.rados.0.smithi063.stdout:3101: finishing rollback tid 2 to smithi06387500-39 2024-08-17T17:07:52.922 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 694 (ObjNum 1092 snap 363 seq_num 1092) dirty exists 2024-08-17T17:07:52.923 INFO:tasks.rados.rados.0.smithi063.stdout:3099: finishing write tid 3 to smithi06387500-14 2024-08-17T17:07:52.923 INFO:tasks.rados.rados.0.smithi063.stdout:3099: finishing write tid 4 to smithi06387500-14 2024-08-17T17:07:52.923 INFO:tasks.rados.rados.0.smithi063.stdout:3099: finishing write tid 5 to smithi06387500-14 2024-08-17T17:07:52.923 INFO:tasks.rados.rados.0.smithi063.stdout:3099: oid 14 updating version 0 to 2967 2024-08-17T17:07:52.923 INFO:tasks.rados.rados.0.smithi063.stdout:3099: oid 14 updating version 2967 to 2968 2024-08-17T17:07:52.923 INFO:tasks.rados.rados.0.smithi063.stdout:3099: oid 14 updating version 2968 to 2970 2024-08-17T17:07:52.923 INFO:tasks.rados.rados.0.smithi063.stdout:3099: oid 14 version 2970 is already newer than 2969 2024-08-17T17:07:52.923 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 2970 (ObjNum 1165 snap 392 seq_num 1165) dirty exists 2024-08-17T17:07:52.923 INFO:tasks.rados.rados.0.smithi063.stdout:3099: left oid 14 (ObjNum 1165 snap 392 seq_num 1165) 2024-08-17T17:07:52.923 INFO:tasks.rados.rados.0.smithi063.stdout:3100: expect (ObjNum 1079 snap 362 seq_num 1079) 2024-08-17T17:07:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:52 smithi063 ceph-mon[109843]: osdmap e844: 8 total, 8 up, 8 in 2024-08-17T17:07:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:52 smithi063 ceph-mon[109843]: pgmap v723: 89 pgs: 89 active+clean; 294 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 2.1 MiB/s wr, 17 op/s; 340 KiB/s, 0 objects/s recovering 2024-08-17T17:07:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:52 smithi063 ceph-mon[114756]: osdmap e844: 8 total, 8 up, 8 in 2024-08-17T17:07:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:52 smithi063 ceph-mon[114756]: pgmap v723: 89 pgs: 89 active+clean; 294 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 2.1 MiB/s wr, 17 op/s; 340 KiB/s, 0 objects/s recovering 2024-08-17T17:07:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:52 smithi162 ceph-mon[104668]: osdmap e844: 8 total, 8 up, 8 in 2024-08-17T17:07:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:52 smithi162 ceph-mon[104668]: pgmap v723: 89 pgs: 89 active+clean; 294 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 2.1 MiB/s wr, 17 op/s; 340 KiB/s, 0 objects/s recovering 2024-08-17T17:07:53.155 INFO:tasks.rados.rados.0.smithi063.stdout:3102: finishing rollback tid 0 to smithi06387500-44 2024-08-17T17:07:53.155 INFO:tasks.rados.rados.0.smithi063.stdout:3102: finishing rollback tid 1 to smithi06387500-44 2024-08-17T17:07:53.155 INFO:tasks.rados.rados.0.smithi063.stdout:3102: finishing rollback tid 2 to smithi06387500-44 2024-08-17T17:07:53.155 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 407 (ObjNum 1058 snap 356 seq_num 1058) dirty exists 2024-08-17T17:07:53.155 INFO:tasks.rados.rados.0.smithi063.stdout:3098: done (4 left) 2024-08-17T17:07:53.156 INFO:tasks.rados.rados.0.smithi063.stdout:3099: done (3 left) 2024-08-17T17:07:53.156 INFO:tasks.rados.rados.0.smithi063.stdout:3100: done (2 left) 2024-08-17T17:07:53.156 INFO:tasks.rados.rados.0.smithi063.stdout:3101: done (1 left) 2024-08-17T17:07:53.156 INFO:tasks.rados.rados.0.smithi063.stdout:3102: done (0 left) 2024-08-17T17:07:53.156 INFO:tasks.rados.rados.0.smithi063.stdout:3103: snap_create 2024-08-17T17:07:53.723 INFO:tasks.rados.rados.0.smithi063.stdout:3103: done (0 left) 2024-08-17T17:07:53.723 INFO:tasks.rados.rados.0.smithi063.stdout:3104: snap_remove snap 355 2024-08-17T17:07:53.726 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:07:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:07:53] "GET /metrics HTTP/1.1" 200 36195 "" "Prometheus/2.51.0" 2024-08-17T17:07:54.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:53 smithi063 ceph-mon[109843]: osdmap e845: 8 total, 8 up, 8 in 2024-08-17T17:07:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:53 smithi063 ceph-mon[114756]: osdmap e845: 8 total, 8 up, 8 in 2024-08-17T17:07:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:53 smithi162 ceph-mon[104668]: osdmap e845: 8 total, 8 up, 8 in 2024-08-17T17:07:54.736 INFO:tasks.rados.rados.0.smithi063.stdout:3104: done (0 left) 2024-08-17T17:07:54.736 INFO:tasks.rados.rados.0.smithi063.stdout:3105: read oid 35 snap -1 2024-08-17T17:07:54.736 INFO:tasks.rados.rados.0.smithi063.stdout:3105: expect deleted 2024-08-17T17:07:54.737 INFO:tasks.rados.rados.0.smithi063.stdout:3106: read oid 15 snap 377 2024-08-17T17:07:54.737 INFO:tasks.rados.rados.0.smithi063.stdout:3106: expect deleted 2024-08-17T17:07:54.737 INFO:tasks.rados.rados.0.smithi063.stdout:3107: rollback oid 13 current snap is 393 2024-08-17T17:07:54.737 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 13 to 384 2024-08-17T17:07:54.737 INFO:tasks.rados.rados.0.smithi063.stdout:3108: rollback oid 37 current snap is 393 2024-08-17T17:07:54.737 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 354 2024-08-17T17:07:54.737 INFO:tasks.rados.rados.0.smithi063.stdout:3109: read oid 46 snap -1 2024-08-17T17:07:54.737 INFO:tasks.rados.rados.0.smithi063.stdout:3109: expect (ObjNum 1134 snap 377 seq_num 1134) 2024-08-17T17:07:54.737 INFO:tasks.rados.rados.0.smithi063.stdout:3110: rollback oid 8 current snap is 393 2024-08-17T17:07:54.737 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 8 to 366 2024-08-17T17:07:54.737 INFO:tasks.rados.rados.0.smithi063.stdout:3111: write oid 12 current snap is 393 2024-08-17T17:07:54.737 INFO:tasks.rados.rados.0.smithi063.stdout:3111: seq_num 1166 ranges {796962=742343,2031416=437730,2936739=413167} 2024-08-17T17:07:54.745 INFO:tasks.rados.rados.0.smithi063.stdout:3111: writing smithi06387500-12 from 796962 to 1539305 tid 1 2024-08-17T17:07:54.752 INFO:tasks.rados.rados.0.smithi063.stdout:3111: writing smithi06387500-12 from 2031416 to 2469146 tid 2 2024-08-17T17:07:54.758 INFO:tasks.rados.rados.0.smithi063.stdout:3111: writing smithi06387500-12 from 2936739 to 3349906 tid 3 2024-08-17T17:07:54.758 INFO:tasks.rados.rados.0.smithi063.stdout:3112: delete oid 3 current snap is 393 2024-08-17T17:07:54.758 INFO:tasks.rados.rados.0.smithi063.stdout:3107: finishing rollback tid 0 to smithi06387500-13 2024-08-17T17:07:54.759 INFO:tasks.rados.rados.0.smithi063.stdout:3107: finishing rollback tid 1 to smithi06387500-13 2024-08-17T17:07:54.759 INFO:tasks.rados.rados.0.smithi063.stdout:3107: finishing rollback tid 2 to smithi06387500-13 2024-08-17T17:07:54.759 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1498 (ObjNum 1090 snap 363 seq_num 1090) dirty exists 2024-08-17T17:07:54.759 INFO:tasks.rados.rados.0.smithi063.stdout:3110: finishing rollback tid 1 to smithi06387500-8 2024-08-17T17:07:54.759 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 993 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T17:07:54.759 INFO:tasks.rados.rados.0.smithi063.stdout:3108: finishing rollback tid 0 to smithi06387500-37 2024-08-17T17:07:54.759 INFO:tasks.rados.rados.0.smithi063.stdout:3108: finishing rollback tid 1 to smithi06387500-37 2024-08-17T17:07:54.759 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 2977 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-08-17T17:07:54.760 INFO:tasks.rados.rados.0.smithi063.stdout:3111: finishing write tid 1 to smithi06387500-12 2024-08-17T17:07:54.760 INFO:tasks.rados.rados.0.smithi063.stdout:3109: expect (ObjNum 1134 snap 377 seq_num 1134) 2024-08-17T17:07:54.905 INFO:tasks.rados.rados.0.smithi063.stdout:3111: finishing write tid 2 to smithi06387500-12 2024-08-17T17:07:54.905 INFO:tasks.rados.rados.0.smithi063.stdout:3111: finishing write tid 3 to smithi06387500-12 2024-08-17T17:07:54.905 INFO:tasks.rados.rados.0.smithi063.stdout:3111: finishing write tid 4 to smithi06387500-12 2024-08-17T17:07:54.905 INFO:tasks.rados.rados.0.smithi063.stdout:3111: finishing write tid 5 to smithi06387500-12 2024-08-17T17:07:54.905 INFO:tasks.rados.rados.0.smithi063.stdout:3111: oid 12 updating version 0 to 698 2024-08-17T17:07:54.905 INFO:tasks.rados.rados.0.smithi063.stdout:3111: oid 12 updating version 698 to 699 2024-08-17T17:07:54.905 INFO:tasks.rados.rados.0.smithi063.stdout:3111: oid 12 version 699 is already newer than 697 2024-08-17T17:07:54.905 INFO:tasks.rados.rados.0.smithi063.stdout:3111: oid 12 updating version 699 to 700 2024-08-17T17:07:54.905 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 700 (ObjNum 1166 snap 393 seq_num 1166) dirty exists 2024-08-17T17:07:54.905 INFO:tasks.rados.rados.0.smithi063.stdout:3111: left oid 12 (ObjNum 1166 snap 393 seq_num 1166) 2024-08-17T17:07:54.905 INFO:tasks.rados.rados.0.smithi063.stdout:3105: done (7 left) 2024-08-17T17:07:54.906 INFO:tasks.rados.rados.0.smithi063.stdout:3106: done (6 left) 2024-08-17T17:07:54.906 INFO:tasks.rados.rados.0.smithi063.stdout:3107: done (5 left) 2024-08-17T17:07:54.906 INFO:tasks.rados.rados.0.smithi063.stdout:3108: done (4 left) 2024-08-17T17:07:54.906 INFO:tasks.rados.rados.0.smithi063.stdout:3109: done (3 left) 2024-08-17T17:07:54.906 INFO:tasks.rados.rados.0.smithi063.stdout:3110: done (2 left) 2024-08-17T17:07:54.906 INFO:tasks.rados.rados.0.smithi063.stdout:3111: done (1 left) 2024-08-17T17:07:54.906 INFO:tasks.rados.rados.0.smithi063.stdout:3112: done (0 left) 2024-08-17T17:07:54.906 INFO:tasks.rados.rados.0.smithi063.stdout:3113: snap_remove snap 363 2024-08-17T17:07:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:54 smithi063 ceph-mon[114756]: osdmap e846: 8 total, 8 up, 8 in 2024-08-17T17:07:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:54 smithi063 ceph-mon[114756]: pgmap v726: 89 pgs: 89 active+clean; 294 MiB data, 890 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 2.1 MiB/s wr, 16 op/s; 340 KiB/s, 0 objects/s recovering 2024-08-17T17:07:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:54 smithi063 ceph-mon[109843]: osdmap e846: 8 total, 8 up, 8 in 2024-08-17T17:07:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:54 smithi063 ceph-mon[109843]: pgmap v726: 89 pgs: 89 active+clean; 294 MiB data, 890 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 2.1 MiB/s wr, 16 op/s; 340 KiB/s, 0 objects/s recovering 2024-08-17T17:07:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:54 smithi162 ceph-mon[104668]: osdmap e846: 8 total, 8 up, 8 in 2024-08-17T17:07:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:54 smithi162 ceph-mon[104668]: pgmap v726: 89 pgs: 89 active+clean; 294 MiB data, 890 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 2.1 MiB/s wr, 16 op/s; 340 KiB/s, 0 objects/s recovering 2024-08-17T17:07:55.551 DEBUG:teuthology.parallel:result is None 2024-08-17T17:07:55.748 INFO:tasks.rados.rados.0.smithi063.stdout:3113: done (0 left) 2024-08-17T17:07:55.748 INFO:tasks.rados.rados.0.smithi063.stdout:3114: read oid 42 snap 354 2024-08-17T17:07:55.748 INFO:tasks.rados.rados.0.smithi063.stdout:3114: expect (ObjNum 1042 snap 353 seq_num 1042) 2024-08-17T17:07:55.749 INFO:tasks.rados.rados.0.smithi063.stdout:3115: write oid 14 current snap is 393 2024-08-17T17:07:55.749 INFO:tasks.rados.rados.0.smithi063.stdout:3115: seq_num 1168 ranges {430576=492841,1632027=441399,2475232=435711,3365397=265179} 2024-08-17T17:07:55.754 INFO:tasks.rados.rados.0.smithi063.stdout:3115: writing smithi06387500-14 from 430576 to 923417 tid 1 2024-08-17T17:07:55.761 INFO:tasks.rados.rados.0.smithi063.stdout:3115: writing smithi06387500-14 from 1632027 to 2073426 tid 2 2024-08-17T17:07:55.766 INFO:tasks.rados.rados.0.smithi063.stdout:3115: writing smithi06387500-14 from 2475232 to 2910943 tid 3 2024-08-17T17:07:55.770 INFO:tasks.rados.rados.0.smithi063.stdout:3115: writing smithi06387500-14 from 3365397 to 3630576 tid 4 2024-08-17T17:07:55.770 INFO:tasks.rados.rados.0.smithi063.stdout:3116: rollback oid 5 current snap is 393 2024-08-17T17:07:55.770 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 378 2024-08-17T17:07:55.771 INFO:tasks.rados.rados.0.smithi063.stdout:3114: expect (ObjNum 1042 snap 353 seq_num 1042) 2024-08-17T17:07:55.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:55 smithi063 ceph-mon[114756]: osdmap e847: 8 total, 8 up, 8 in 2024-08-17T17:07:55.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:55 smithi063 ceph-mon[114756]: from='client.34443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:55.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:55 smithi063 ceph-mon[109843]: osdmap e847: 8 total, 8 up, 8 in 2024-08-17T17:07:55.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:55 smithi063 ceph-mon[109843]: from='client.34443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:55.885 INFO:tasks.rados.rados.0.smithi063.stdout:3115: finishing write tid 1 to smithi06387500-14 2024-08-17T17:07:55.885 INFO:tasks.rados.rados.0.smithi063.stdout:3115: finishing write tid 2 to smithi06387500-14 2024-08-17T17:07:55.885 INFO:tasks.rados.rados.0.smithi063.stdout:3115: finishing write tid 3 to smithi06387500-14 2024-08-17T17:07:55.885 INFO:tasks.rados.rados.0.smithi063.stdout:3116: finishing rollback tid 0 to smithi06387500-5 2024-08-17T17:07:55.886 INFO:tasks.rados.rados.0.smithi063.stdout:3116: finishing rollback tid 1 to smithi06387500-5 2024-08-17T17:07:55.886 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1509 (ObjNum 49 snap 0 seq_num 1786885104) dirty dne 2024-08-17T17:07:55.886 INFO:tasks.rados.rados.0.smithi063.stdout:3115: finishing write tid 4 to smithi06387500-14 2024-08-17T17:07:55.886 INFO:tasks.rados.rados.0.smithi063.stdout:3114: done (2 left) 2024-08-17T17:07:55.886 INFO:tasks.rados.rados.0.smithi063.stdout:3116: done (1 left) 2024-08-17T17:07:55.886 INFO:tasks.rados.rados.0.smithi063.stdout:3117: snap_create 2024-08-17T17:07:55.886 INFO:tasks.rados.rados.0.smithi063.stdout:3115: finishing write tid 5 to smithi06387500-14 2024-08-17T17:07:55.886 INFO:tasks.rados.rados.0.smithi063.stdout:3115: finishing write tid 6 to smithi06387500-14 2024-08-17T17:07:55.886 INFO:tasks.rados.rados.0.smithi063.stdout:3115: oid 14 updating version 0 to 2998 2024-08-17T17:07:55.886 INFO:tasks.rados.rados.0.smithi063.stdout:3115: oid 14 updating version 2998 to 2999 2024-08-17T17:07:55.886 INFO:tasks.rados.rados.0.smithi063.stdout:3115: oid 14 version 2999 is already newer than 2997 2024-08-17T17:07:55.886 INFO:tasks.rados.rados.0.smithi063.stdout:3115: oid 14 version 2999 is already newer than 2996 2024-08-17T17:07:55.887 INFO:tasks.rados.rados.0.smithi063.stdout:3115: oid 14 version 2999 is already newer than 2991 2024-08-17T17:07:55.887 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 2999 (ObjNum 1168 snap 393 seq_num 1168) dirty exists 2024-08-17T17:07:55.887 INFO:tasks.rados.rados.0.smithi063.stdout:3115: left oid 14 (ObjNum 1168 snap 393 seq_num 1168) 2024-08-17T17:07:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:55 smithi162 ceph-mon[104668]: osdmap e847: 8 total, 8 up, 8 in 2024-08-17T17:07:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:55 smithi162 ceph-mon[104668]: from='client.34443 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:07:56.754 INFO:tasks.rados.rados.0.smithi063.stdout:3115: done (1 left) 2024-08-17T17:07:56.755 INFO:tasks.rados.rados.0.smithi063.stdout:3117: done (0 left) 2024-08-17T17:07:56.755 INFO:tasks.rados.rados.0.smithi063.stdout:3118: read oid 8 snap -1 2024-08-17T17:07:56.755 INFO:tasks.rados.rados.0.smithi063.stdout:3118: expect deleted 2024-08-17T17:07:56.755 INFO:tasks.rados.rados.0.smithi063.stdout:3119: write oid 31 current snap is 394 2024-08-17T17:07:56.755 INFO:tasks.rados.rados.0.smithi063.stdout:3119: seq_num 1169 ranges {447383=568090,1690509=701410,2902655=646983} 2024-08-17T17:07:56.760 INFO:tasks.rados.rados.0.smithi063.stdout:3119: writing smithi06387500-31 from 447383 to 1015473 tid 1 2024-08-17T17:07:56.770 INFO:tasks.rados.rados.0.smithi063.stdout:3119: writing smithi06387500-31 from 1690509 to 2391919 tid 2 2024-08-17T17:07:56.774 INFO:tasks.rados.rados.0.smithi063.stdout:3119: writing smithi06387500-31 from 2902655 to 3549638 tid 3 2024-08-17T17:07:56.775 INFO:tasks.rados.rados.0.smithi063.stdout:3120: snap_create 2024-08-17T17:07:56.775 INFO:tasks.rados.rados.0.smithi063.stdout:3119: finishing write tid 1 to smithi06387500-31 2024-08-17T17:07:56.775 INFO:tasks.rados.rados.0.smithi063.stdout:3119: finishing write tid 2 to smithi06387500-31 2024-08-17T17:07:56.779 INFO:tasks.rados.rados.0.smithi063.stdout:3119: finishing write tid 3 to smithi06387500-31 2024-08-17T17:07:56.779 INFO:tasks.rados.rados.0.smithi063.stdout:3119: finishing write tid 4 to smithi06387500-31 2024-08-17T17:07:56.779 INFO:tasks.rados.rados.0.smithi063.stdout:3119: finishing write tid 5 to smithi06387500-31 2024-08-17T17:07:56.779 INFO:tasks.rados.rados.0.smithi063.stdout:3119: oid 31 updating version 0 to 1954 2024-08-17T17:07:56.779 INFO:tasks.rados.rados.0.smithi063.stdout:3119: oid 31 updating version 1954 to 1955 2024-08-17T17:07:56.779 INFO:tasks.rados.rados.0.smithi063.stdout:3119: oid 31 updating version 1955 to 1957 2024-08-17T17:07:56.779 INFO:tasks.rados.rados.0.smithi063.stdout:3119: oid 31 version 1957 is already newer than 1956 2024-08-17T17:07:56.779 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 1957 (ObjNum 1169 snap 394 seq_num 1169) dirty exists 2024-08-17T17:07:56.779 INFO:tasks.rados.rados.0.smithi063.stdout:3119: left oid 31 (ObjNum 1169 snap 394 seq_num 1169) 2024-08-17T17:07:57.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:56 smithi063 ceph-mon[114756]: osdmap e848: 8 total, 8 up, 8 in 2024-08-17T17:07:57.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:56 smithi063 ceph-mon[114756]: pgmap v729: 89 pgs: 89 active+clean; 308 MiB data, 900 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.8 MiB/s wr, 13 op/s 2024-08-17T17:07:57.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:56 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:57.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:56 smithi063 ceph-mon[109843]: osdmap e848: 8 total, 8 up, 8 in 2024-08-17T17:07:57.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:56 smithi063 ceph-mon[109843]: pgmap v729: 89 pgs: 89 active+clean; 308 MiB data, 900 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.8 MiB/s wr, 13 op/s 2024-08-17T17:07:57.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:56 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:57.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:56 smithi162 ceph-mon[104668]: osdmap e848: 8 total, 8 up, 8 in 2024-08-17T17:07:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:56 smithi162 ceph-mon[104668]: pgmap v729: 89 pgs: 89 active+clean; 308 MiB data, 900 MiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.8 MiB/s wr, 13 op/s 2024-08-17T17:07:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:56 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:07:57.632 INFO:tasks.rados.rados.0.smithi063.stdout:3118: done (2 left) 2024-08-17T17:07:57.632 INFO:tasks.rados.rados.0.smithi063.stdout:3119: done (1 left) 2024-08-17T17:07:57.632 INFO:tasks.rados.rados.0.smithi063.stdout:3120: done (0 left) 2024-08-17T17:07:57.632 INFO:tasks.rados.rados.0.smithi063.stdout:3121: snap_remove snap 389 2024-08-17T17:07:58.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:57 smithi063 ceph-mon[114756]: osdmap e849: 8 total, 8 up, 8 in 2024-08-17T17:07:58.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:57 smithi063 ceph-mon[114756]: osdmap e850: 8 total, 8 up, 8 in 2024-08-17T17:07:58.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:57 smithi063 ceph-mon[109843]: osdmap e849: 8 total, 8 up, 8 in 2024-08-17T17:07:58.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:57 smithi063 ceph-mon[109843]: osdmap e850: 8 total, 8 up, 8 in 2024-08-17T17:07:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:57 smithi162 ceph-mon[104668]: osdmap e849: 8 total, 8 up, 8 in 2024-08-17T17:07:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:57 smithi162 ceph-mon[104668]: osdmap e850: 8 total, 8 up, 8 in 2024-08-17T17:07:58.636 INFO:tasks.rados.rados.0.smithi063.stdout:3121: done (0 left) 2024-08-17T17:07:58.636 INFO:tasks.rados.rados.0.smithi063.stdout:3122: read oid 35 snap -1 2024-08-17T17:07:58.636 INFO:tasks.rados.rados.0.smithi063.stdout:3122: expect deleted 2024-08-17T17:07:58.636 INFO:tasks.rados.rados.0.smithi063.stdout:3123: snap_remove snap 384 2024-08-17T17:07:59.638 INFO:tasks.rados.rados.0.smithi063.stdout:3123: done (1 left) 2024-08-17T17:07:59.638 INFO:tasks.rados.rados.0.smithi063.stdout:3124: write oid 42 current snap is 395 2024-08-17T17:07:59.638 INFO:tasks.rados.rados.0.smithi063.stdout:3124: seq_num 1170 ranges {464190=643339,1748991=445068,2930078=574608} 2024-08-17T17:07:59.645 INFO:tasks.rados.rados.0.smithi063.stdout:3124: writing smithi06387500-42 from 464190 to 1107529 tid 1 2024-08-17T17:07:59.651 INFO:tasks.rados.rados.0.smithi063.stdout:3124: writing smithi06387500-42 from 1748991 to 2194059 tid 2 2024-08-17T17:07:59.659 INFO:tasks.rados.rados.0.smithi063.stdout:3124: writing smithi06387500-42 from 2930078 to 3504686 tid 3 2024-08-17T17:07:59.659 INFO:tasks.rados.rados.0.smithi063.stdout:3125: read oid 37 snap -1 2024-08-17T17:07:59.659 INFO:tasks.rados.rados.0.smithi063.stdout:3125: expect deleted 2024-08-17T17:07:59.659 INFO:tasks.rados.rados.0.smithi063.stdout:3126: read oid 27 snap -1 2024-08-17T17:07:59.659 INFO:tasks.rados.rados.0.smithi063.stdout:3126: expect (ObjNum 1152 snap 389 seq_num 1152) 2024-08-17T17:07:59.659 INFO:tasks.rados.rados.0.smithi063.stdout:3127: write oid 24 current snap is 395 2024-08-17T17:07:59.660 INFO:tasks.rados.rados.0.smithi063.stdout:3124: finishing write tid 1 to smithi06387500-42 2024-08-17T17:07:59.660 INFO:tasks.rados.rados.0.smithi063.stdout:3124: finishing write tid 2 to smithi06387500-42 2024-08-17T17:07:59.660 INFO:tasks.rados.rados.0.smithi063.stdout:3127: seq_num 1171 ranges {480997=434941,1407473=705079,2557501=785880} 2024-08-17T17:07:59.664 INFO:tasks.rados.rados.0.smithi063.stdout:3127: writing smithi06387500-24 from 480997 to 915938 tid 1 2024-08-17T17:07:59.671 INFO:tasks.rados.rados.0.smithi063.stdout:3127: writing smithi06387500-24 from 1407473 to 2112552 tid 2 2024-08-17T17:07:59.679 INFO:tasks.rados.rados.0.smithi063.stdout:3127: writing smithi06387500-24 from 2557501 to 3343381 tid 3 2024-08-17T17:07:59.679 INFO:tasks.rados.rados.0.smithi063.stdout:3122: done (4 left) 2024-08-17T17:07:59.679 INFO:tasks.rados.rados.0.smithi063.stdout:3128: read oid 45 snap 354 2024-08-17T17:07:59.679 INFO:tasks.rados.rados.0.smithi063.stdout:3128: expect deleted 2024-08-17T17:07:59.679 INFO:tasks.rados.rados.0.smithi063.stdout:3129: rollback oid 5 current snap is 395 2024-08-17T17:07:59.679 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 391 2024-08-17T17:07:59.679 INFO:tasks.rados.rados.0.smithi063.stdout:3124: finishing write tid 3 to smithi06387500-42 2024-08-17T17:07:59.679 INFO:tasks.rados.rados.0.smithi063.stdout:3124: finishing write tid 4 to smithi06387500-42 2024-08-17T17:07:59.679 INFO:tasks.rados.rados.0.smithi063.stdout:3124: finishing write tid 5 to smithi06387500-42 2024-08-17T17:07:59.679 INFO:tasks.rados.rados.0.smithi063.stdout:3124: oid 42 updating version 0 to 1967 2024-08-17T17:07:59.679 INFO:tasks.rados.rados.0.smithi063.stdout:3124: oid 42 version 1967 is already newer than 1965 2024-08-17T17:07:59.680 INFO:tasks.rados.rados.0.smithi063.stdout:3124: oid 42 version 1967 is already newer than 1966 2024-08-17T17:07:59.680 INFO:tasks.rados.rados.0.smithi063.stdout:3124: oid 42 updating version 1967 to 1968 2024-08-17T17:07:59.680 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 1968 (ObjNum 1170 snap 395 seq_num 1170) dirty exists 2024-08-17T17:07:59.680 INFO:tasks.rados.rados.0.smithi063.stdout:3124: left oid 42 (ObjNum 1170 snap 395 seq_num 1170) 2024-08-17T17:07:59.680 INFO:tasks.rados.rados.0.smithi063.stdout:3126: expect (ObjNum 1152 snap 389 seq_num 1152) 2024-08-17T17:07:59.823 INFO:tasks.rados.rados.0.smithi063.stdout:3127: finishing write tid 1 to smithi06387500-24 2024-08-17T17:07:59.823 INFO:tasks.rados.rados.0.smithi063.stdout:3127: finishing write tid 2 to smithi06387500-24 2024-08-17T17:07:59.823 INFO:tasks.rados.rados.0.smithi063.stdout:3127: finishing write tid 3 to smithi06387500-24 2024-08-17T17:07:59.823 INFO:tasks.rados.rados.0.smithi063.stdout:3127: finishing write tid 4 to smithi06387500-24 2024-08-17T17:07:59.823 INFO:tasks.rados.rados.0.smithi063.stdout:3129: finishing rollback tid 1 to smithi06387500-5 2024-08-17T17:07:59.823 INFO:tasks.rados.rados.0.smithi063.stdout:3129: finishing rollback tid 2 to smithi06387500-5 2024-08-17T17:07:59.823 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1519 (ObjNum 1154 snap 389 seq_num 1154) dirty exists 2024-08-17T17:07:59.823 INFO:tasks.rados.rados.0.smithi063.stdout:3127: finishing write tid 5 to smithi06387500-24 2024-08-17T17:07:59.824 INFO:tasks.rados.rados.0.smithi063.stdout:3127: oid 24 updating version 0 to 1083 2024-08-17T17:07:59.824 INFO:tasks.rados.rados.0.smithi063.stdout:3127: oid 24 updating version 1083 to 1085 2024-08-17T17:07:59.824 INFO:tasks.rados.rados.0.smithi063.stdout:3127: oid 24 version 1085 is already newer than 1084 2024-08-17T17:07:59.824 INFO:tasks.rados.rados.0.smithi063.stdout:3127: oid 24 version 1085 is already newer than 1082 2024-08-17T17:07:59.824 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 1085 (ObjNum 1171 snap 395 seq_num 1171) dirty exists 2024-08-17T17:07:59.824 INFO:tasks.rados.rados.0.smithi063.stdout:3127: left oid 24 (ObjNum 1171 snap 395 seq_num 1171) 2024-08-17T17:07:59.824 INFO:tasks.rados.rados.0.smithi063.stdout:3124: done (5 left) 2024-08-17T17:07:59.824 INFO:tasks.rados.rados.0.smithi063.stdout:3125: done (4 left) 2024-08-17T17:07:59.824 INFO:tasks.rados.rados.0.smithi063.stdout:3126: done (3 left) 2024-08-17T17:07:59.824 INFO:tasks.rados.rados.0.smithi063.stdout:3127: done (2 left) 2024-08-17T17:07:59.824 INFO:tasks.rados.rados.0.smithi063.stdout:3128: done (1 left) 2024-08-17T17:07:59.824 INFO:tasks.rados.rados.0.smithi063.stdout:3129: done (0 left) 2024-08-17T17:07:59.824 INFO:tasks.rados.rados.0.smithi063.stdout:3130: read oid 49 snap 387 2024-08-17T17:07:59.825 INFO:tasks.rados.rados.0.smithi063.stdout:3130: expect (ObjNum 1140 snap 380 seq_num 1140) 2024-08-17T17:07:59.825 INFO:tasks.rados.rados.0.smithi063.stdout:3131: rollback oid 18 current snap is 395 2024-08-17T17:07:59.825 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 18 to 374 2024-08-17T17:07:59.825 INFO:tasks.rados.rados.0.smithi063.stdout:3132: read oid 25 snap -1 2024-08-17T17:07:59.825 INFO:tasks.rados.rados.0.smithi063.stdout:3132: expect (ObjNum 1159 snap 392 seq_num 1159) 2024-08-17T17:07:59.825 INFO:tasks.rados.rados.0.smithi063.stdout:3133: delete oid 35 current snap is 395 2024-08-17T17:07:59.825 INFO:tasks.rados.rados.0.smithi063.stdout:3131: finishing rollback tid 0 to smithi06387500-18 2024-08-17T17:07:59.825 INFO:tasks.rados.rados.0.smithi063.stdout:3131: finishing rollback tid 1 to smithi06387500-18 2024-08-17T17:07:59.826 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 993 (ObjNum 49 snap 0 seq_num 1786885104) dirty dne 2024-08-17T17:07:59.835 INFO:tasks.rados.rados.0.smithi063.stdout:3130: expect (ObjNum 1140 snap 380 seq_num 1140) 2024-08-17T17:07:59.939 INFO:tasks.rados.rados.0.smithi063.stdout:3132: expect (ObjNum 1159 snap 392 seq_num 1159) 2024-08-17T17:08:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:59 smithi063 ceph-mon[114756]: pgmap v732: 89 pgs: 89 active+clean; 312 MiB data, 916 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.9 MiB/s wr, 22 op/s 2024-08-17T17:08:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:07:59 smithi063 ceph-mon[114756]: osdmap e851: 8 total, 8 up, 8 in 2024-08-17T17:08:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:59 smithi063 ceph-mon[109843]: pgmap v732: 89 pgs: 89 active+clean; 312 MiB data, 916 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.9 MiB/s wr, 22 op/s 2024-08-17T17:08:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:07:59 smithi063 ceph-mon[109843]: osdmap e851: 8 total, 8 up, 8 in 2024-08-17T17:08:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:59 smithi162 ceph-mon[104668]: pgmap v732: 89 pgs: 89 active+clean; 312 MiB data, 916 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 2.9 MiB/s wr, 22 op/s 2024-08-17T17:08:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:07:59 smithi162 ceph-mon[104668]: osdmap e851: 8 total, 8 up, 8 in 2024-08-17T17:08:00.097 INFO:tasks.rados.rados.0.smithi063.stdout:3130: done (3 left) 2024-08-17T17:08:00.097 INFO:tasks.rados.rados.0.smithi063.stdout:3131: done (2 left) 2024-08-17T17:08:00.097 INFO:tasks.rados.rados.0.smithi063.stdout:3132: done (1 left) 2024-08-17T17:08:00.098 INFO:tasks.rados.rados.0.smithi063.stdout:3133: done (0 left) 2024-08-17T17:08:00.098 INFO:tasks.rados.rados.0.smithi063.stdout:3134: snap_remove snap 388 2024-08-17T17:08:00.653 INFO:tasks.rados.rados.0.smithi063.stdout:3134: done (0 left) 2024-08-17T17:08:00.653 INFO:tasks.rados.rados.0.smithi063.stdout:3135: snap_remove snap 379 2024-08-17T17:08:00.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:00 smithi162 ceph-mon[104668]: osdmap e852: 8 total, 8 up, 8 in 2024-08-17T17:08:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:00 smithi063 ceph-mon[109843]: osdmap e852: 8 total, 8 up, 8 in 2024-08-17T17:08:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:00 smithi063 ceph-mon[114756]: osdmap e852: 8 total, 8 up, 8 in 2024-08-17T17:08:01.654 INFO:tasks.rados.rados.0.smithi063.stdout:3135: done (0 left) 2024-08-17T17:08:01.654 INFO:tasks.rados.rados.0.smithi063.stdout:3136: write oid 40 current snap is 395 2024-08-17T17:08:01.654 INFO:tasks.rados.rados.0.smithi063.stdout:3136: seq_num 1173 ranges {514611=585439,1524437=592395,2895994=408424} 2024-08-17T17:08:01.660 INFO:tasks.rados.rados.0.smithi063.stdout:3136: writing smithi06387500-40 from 514611 to 1100050 tid 1 2024-08-17T17:08:01.666 INFO:tasks.rados.rados.0.smithi063.stdout:3136: writing smithi06387500-40 from 1524437 to 2116832 tid 2 2024-08-17T17:08:01.674 INFO:tasks.rados.rados.0.smithi063.stdout:3136: writing smithi06387500-40 from 2895994 to 3304418 tid 3 2024-08-17T17:08:01.674 INFO:tasks.rados.rados.0.smithi063.stdout:3137: delete oid 19 current snap is 395 2024-08-17T17:08:01.675 INFO:tasks.rados.rados.0.smithi063.stdout:3136: finishing write tid 1 to smithi06387500-40 2024-08-17T17:08:01.675 INFO:tasks.rados.rados.0.smithi063.stdout:3136: finishing write tid 2 to smithi06387500-40 2024-08-17T17:08:01.677 INFO:tasks.rados.rados.0.smithi063.stdout:3137: done (1 left) 2024-08-17T17:08:01.677 INFO:tasks.rados.rados.0.smithi063.stdout:3138: delete oid 33 current snap is 395 2024-08-17T17:08:01.677 INFO:tasks.rados.rados.0.smithi063.stdout:3136: finishing write tid 3 to smithi06387500-40 2024-08-17T17:08:01.677 INFO:tasks.rados.rados.0.smithi063.stdout:3136: finishing write tid 4 to smithi06387500-40 2024-08-17T17:08:01.678 INFO:tasks.rados.rados.0.smithi063.stdout:3136: finishing write tid 5 to smithi06387500-40 2024-08-17T17:08:01.678 INFO:tasks.rados.rados.0.smithi063.stdout:3136: oid 40 updating version 0 to 1002 2024-08-17T17:08:01.678 INFO:tasks.rados.rados.0.smithi063.stdout:3136: oid 40 version 1002 is already newer than 1000 2024-08-17T17:08:01.678 INFO:tasks.rados.rados.0.smithi063.stdout:3136: oid 40 version 1002 is already newer than 999 2024-08-17T17:08:01.678 INFO:tasks.rados.rados.0.smithi063.stdout:3136: oid 40 version 1002 is already newer than 1001 2024-08-17T17:08:01.678 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 1002 (ObjNum 1173 snap 395 seq_num 1173) dirty exists 2024-08-17T17:08:01.678 INFO:tasks.rados.rados.0.smithi063.stdout:3136: left oid 40 (ObjNum 1173 snap 395 seq_num 1173) 2024-08-17T17:08:01.679 INFO:tasks.rados.rados.0.smithi063.stdout:3136: done (1 left) 2024-08-17T17:08:01.679 INFO:tasks.rados.rados.0.smithi063.stdout:3138: done (0 left) 2024-08-17T17:08:01.679 INFO:tasks.rados.rados.0.smithi063.stdout:3139: snap_create 2024-08-17T17:08:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:01 smithi063 ceph-mon[114756]: pgmap v735: 89 pgs: 89 active+clean; 312 MiB data, 916 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T17:08:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:01 smithi063 ceph-mon[114756]: osdmap e853: 8 total, 8 up, 8 in 2024-08-17T17:08:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:01 smithi063 ceph-mon[109843]: pgmap v735: 89 pgs: 89 active+clean; 312 MiB data, 916 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T17:08:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:01 smithi063 ceph-mon[109843]: osdmap e853: 8 total, 8 up, 8 in 2024-08-17T17:08:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:01 smithi162 ceph-mon[104668]: pgmap v735: 89 pgs: 89 active+clean; 312 MiB data, 916 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T17:08:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:01 smithi162 ceph-mon[104668]: osdmap e853: 8 total, 8 up, 8 in 2024-08-17T17:08:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:02.635 INFO:tasks.rados.rados.0.smithi063.stdout:3139: done (0 left) 2024-08-17T17:08:02.635 INFO:tasks.rados.rados.0.smithi063.stdout:3140: rollback oid 4 current snap is 396 2024-08-17T17:08:02.635 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 4 to 394 2024-08-17T17:08:02.636 INFO:tasks.rados.rados.0.smithi063.stdout:3141: snap_create 2024-08-17T17:08:02.642 INFO:tasks.rados.rados.0.smithi063.stdout:3140: finishing rollback tid 0 to smithi06387500-4 2024-08-17T17:08:02.642 INFO:tasks.rados.rados.0.smithi063.stdout:3140: finishing rollback tid 1 to smithi06387500-4 2024-08-17T17:08:02.642 INFO:tasks.rados.rados.0.smithi063.stdout:3140: finishing rollback tid 2 to smithi06387500-4 2024-08-17T17:08:02.642 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 3040 (ObjNum 1113 snap 369 seq_num 1113) dirty exists 2024-08-17T17:08:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:02 smithi063 ceph-mon[114756]: osdmap e854: 8 total, 8 up, 8 in 2024-08-17T17:08:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:02 smithi063 ceph-mon[114756]: osdmap e855: 8 total, 8 up, 8 in 2024-08-17T17:08:03.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:02 smithi063 ceph-mon[109843]: osdmap e854: 8 total, 8 up, 8 in 2024-08-17T17:08:03.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:02 smithi063 ceph-mon[109843]: osdmap e855: 8 total, 8 up, 8 in 2024-08-17T17:08:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:02 smithi162 ceph-mon[104668]: osdmap e854: 8 total, 8 up, 8 in 2024-08-17T17:08:03.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:02 smithi162 ceph-mon[104668]: osdmap e855: 8 total, 8 up, 8 in 2024-08-17T17:08:03.640 INFO:tasks.rados.rados.0.smithi063.stdout:3140: done (1 left) 2024-08-17T17:08:03.640 INFO:tasks.rados.rados.0.smithi063.stdout:3141: done (0 left) 2024-08-17T17:08:03.640 INFO:tasks.rados.rados.0.smithi063.stdout:3142: rollback oid 18 current snap is 397 2024-08-17T17:08:03.640 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 18 to 383 2024-08-17T17:08:03.640 INFO:tasks.rados.rados.0.smithi063.stdout:3143: read oid 29 snap 395 2024-08-17T17:08:03.640 INFO:tasks.rados.rados.0.smithi063.stdout:3143: expect deleted 2024-08-17T17:08:03.640 INFO:tasks.rados.rados.0.smithi063.stdout:3144: snap_create 2024-08-17T17:08:03.645 INFO:tasks.rados.rados.0.smithi063.stdout:3142: finishing rollback tid 1 to smithi06387500-18 2024-08-17T17:08:03.645 INFO:tasks.rados.rados.0.smithi063.stdout:3142: finishing rollback tid 2 to smithi06387500-18 2024-08-17T17:08:03.645 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 1004 (ObjNum 1130 snap 377 seq_num 1130) dirty exists 2024-08-17T17:08:03.663 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:08:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:08:03] "GET /metrics HTTP/1.1" 200 36340 "" "Prometheus/2.51.0" 2024-08-17T17:08:04.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:03 smithi063 ceph-mon[114756]: pgmap v738: 89 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 86 active+clean; 323 MiB data, 925 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-08-17T17:08:04.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:03 smithi063 ceph-mon[114756]: osdmap e856: 8 total, 8 up, 8 in 2024-08-17T17:08:04.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:03 smithi063 ceph-mon[109843]: pgmap v738: 89 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 86 active+clean; 323 MiB data, 925 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-08-17T17:08:04.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:03 smithi063 ceph-mon[109843]: osdmap e856: 8 total, 8 up, 8 in 2024-08-17T17:08:04.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:03 smithi162 ceph-mon[104668]: pgmap v738: 89 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 86 active+clean; 323 MiB data, 925 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-08-17T17:08:04.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:03 smithi162 ceph-mon[104668]: osdmap e856: 8 total, 8 up, 8 in 2024-08-17T17:08:04.638 INFO:tasks.rados.rados.0.smithi063.stdout:3142: done (2 left) 2024-08-17T17:08:04.639 INFO:tasks.rados.rados.0.smithi063.stdout:3143: done (1 left) 2024-08-17T17:08:04.639 INFO:tasks.rados.rados.0.smithi063.stdout:3144: done (0 left) 2024-08-17T17:08:04.639 INFO:tasks.rados.rados.0.smithi063.stdout:3145: read oid 26 snap -1 2024-08-17T17:08:04.639 INFO:tasks.rados.rados.0.smithi063.stdout:3145: expect (ObjNum 1079 snap 362 seq_num 1079) 2024-08-17T17:08:04.639 INFO:tasks.rados.rados.0.smithi063.stdout:3146: snap_remove snap 392 2024-08-17T17:08:05.640 INFO:tasks.rados.rados.0.smithi063.stdout:3146: done (1 left) 2024-08-17T17:08:05.640 INFO:tasks.rados.rados.0.smithi063.stdout:3147: write oid 47 current snap is 398 2024-08-17T17:08:05.640 INFO:tasks.rados.rados.0.smithi063.stdout:3147: seq_num 1176 ranges {565032=411186,1583530=456075,2578263=758593} 2024-08-17T17:08:05.645 INFO:tasks.rados.rados.0.smithi063.stdout:3147: writing smithi06387500-47 from 565032 to 976218 tid 1 2024-08-17T17:08:05.652 INFO:tasks.rados.rados.0.smithi063.stdout:3147: writing smithi06387500-47 from 1583530 to 2039605 tid 2 2024-08-17T17:08:05.659 INFO:tasks.rados.rados.0.smithi063.stdout:3147: writing smithi06387500-47 from 2578263 to 3336856 tid 3 2024-08-17T17:08:05.660 INFO:tasks.rados.rados.0.smithi063.stdout:3148: snap_remove snap 391 2024-08-17T17:08:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:05 smithi063 ceph-mon[114756]: pgmap v741: 89 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 86 active+clean; 323 MiB data, 928 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 14 op/s 2024-08-17T17:08:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:05 smithi063 ceph-mon[114756]: osdmap e857: 8 total, 8 up, 8 in 2024-08-17T17:08:06.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:05 smithi063 ceph-mon[109843]: pgmap v741: 89 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 86 active+clean; 323 MiB data, 928 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 14 op/s 2024-08-17T17:08:06.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:05 smithi063 ceph-mon[109843]: osdmap e857: 8 total, 8 up, 8 in 2024-08-17T17:08:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:05 smithi162 ceph-mon[104668]: pgmap v741: 89 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 86 active+clean; 323 MiB data, 928 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.3 MiB/s wr, 14 op/s 2024-08-17T17:08:06.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:05 smithi162 ceph-mon[104668]: osdmap e857: 8 total, 8 up, 8 in 2024-08-17T17:08:06.652 INFO:tasks.rados.rados.0.smithi063.stdout:3148: done (2 left) 2024-08-17T17:08:06.652 INFO:tasks.rados.rados.0.smithi063.stdout:3149: snap_remove snap 395 2024-08-17T17:08:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:06 smithi063 ceph-mon[114756]: osdmap e858: 8 total, 8 up, 8 in 2024-08-17T17:08:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:06 smithi063 ceph-mon[109843]: osdmap e858: 8 total, 8 up, 8 in 2024-08-17T17:08:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:06 smithi162 ceph-mon[104668]: osdmap e858: 8 total, 8 up, 8 in 2024-08-17T17:08:07.636 INFO:tasks.rados.rados.0.smithi063.stdout:3149: done (2 left) 2024-08-17T17:08:07.636 INFO:tasks.rados.rados.0.smithi063.stdout:3150: delete oid 23 current snap is 398 2024-08-17T17:08:07.636 INFO:tasks.rados.rados.0.smithi063.stdout:3145: expect (ObjNum 1079 snap 362 seq_num 1079) 2024-08-17T17:08:07.875 INFO:tasks.rados.rados.0.smithi063.stdout:3147: finishing write tid 1 to smithi06387500-47 2024-08-17T17:08:07.875 INFO:tasks.rados.rados.0.smithi063.stdout:3147: finishing write tid 2 to smithi06387500-47 2024-08-17T17:08:07.876 INFO:tasks.rados.rados.0.smithi063.stdout:3147: finishing write tid 3 to smithi06387500-47 2024-08-17T17:08:07.876 INFO:tasks.rados.rados.0.smithi063.stdout:3147: finishing write tid 4 to smithi06387500-47 2024-08-17T17:08:07.876 INFO:tasks.rados.rados.0.smithi063.stdout:3147: finishing write tid 5 to smithi06387500-47 2024-08-17T17:08:07.876 INFO:tasks.rados.rados.0.smithi063.stdout:3147: oid 47 updating version 0 to 647 2024-08-17T17:08:07.876 INFO:tasks.rados.rados.0.smithi063.stdout:3147: oid 47 updating version 647 to 649 2024-08-17T17:08:07.876 INFO:tasks.rados.rados.0.smithi063.stdout:3147: oid 47 updating version 649 to 650 2024-08-17T17:08:07.876 INFO:tasks.rados.rados.0.smithi063.stdout:3147: oid 47 version 650 is already newer than 648 2024-08-17T17:08:07.876 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 650 (ObjNum 1176 snap 398 seq_num 1176) dirty exists 2024-08-17T17:08:07.876 INFO:tasks.rados.rados.0.smithi063.stdout:3147: left oid 47 (ObjNum 1176 snap 398 seq_num 1176) 2024-08-17T17:08:07.876 INFO:tasks.rados.rados.0.smithi063.stdout:3145: done (2 left) 2024-08-17T17:08:07.876 INFO:tasks.rados.rados.0.smithi063.stdout:3147: done (1 left) 2024-08-17T17:08:07.876 INFO:tasks.rados.rados.0.smithi063.stdout:3150: done (0 left) 2024-08-17T17:08:07.877 INFO:tasks.rados.rados.0.smithi063.stdout:3151: rollback oid 49 current snap is 398 2024-08-17T17:08:07.877 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 49 to 372 2024-08-17T17:08:07.877 INFO:tasks.rados.rados.0.smithi063.stdout:3152: rollback oid 18 current snap is 398 2024-08-17T17:08:07.877 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 18 to 377 2024-08-17T17:08:07.877 INFO:tasks.rados.rados.0.smithi063.stdout:3153: rollback oid 41 current snap is 398 2024-08-17T17:08:07.877 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 41 to 378 2024-08-17T17:08:07.877 INFO:tasks.rados.rados.0.smithi063.stdout:3154: read oid 30 snap -1 2024-08-17T17:08:07.877 INFO:tasks.rados.rados.0.smithi063.stdout:3154: expect deleted 2024-08-17T17:08:07.877 INFO:tasks.rados.rados.0.smithi063.stdout:3155: read oid 25 snap -1 2024-08-17T17:08:07.877 INFO:tasks.rados.rados.0.smithi063.stdout:3155: expect (ObjNum 1159 snap 392 seq_num 1159) 2024-08-17T17:08:07.877 INFO:tasks.rados.rados.0.smithi063.stdout:3156: write oid 37 current snap is 398 2024-08-17T17:08:07.877 INFO:tasks.rados.rados.0.smithi063.stdout:3156: seq_num 1178 ranges {598646=561684,1700494=459744,2633109=781137} 2024-08-17T17:08:07.881 INFO:tasks.rados.rados.0.smithi063.stdout:3156: writing smithi06387500-37 from 598646 to 1160330 tid 1 2024-08-17T17:08:07.887 INFO:tasks.rados.rados.0.smithi063.stdout:3156: writing smithi06387500-37 from 1700494 to 2160238 tid 2 2024-08-17T17:08:07.894 INFO:tasks.rados.rados.0.smithi063.stdout:3156: writing smithi06387500-37 from 2633109 to 3414246 tid 3 2024-08-17T17:08:07.894 INFO:tasks.rados.rados.0.smithi063.stdout:3157: write oid 23 current snap is 398 2024-08-17T17:08:07.894 INFO:tasks.rados.rados.0.smithi063.stdout:3157: seq_num 1179 ranges {615453=753286,2158976=603402,3460532=354921} 2024-08-17T17:08:07.901 INFO:tasks.rados.rados.0.smithi063.stdout:3157: writing smithi06387500-23 from 615453 to 1368739 tid 1 2024-08-17T17:08:07.908 INFO:tasks.rados.rados.0.smithi063.stdout:3157: writing smithi06387500-23 from 2158976 to 2762378 tid 2 2024-08-17T17:08:07.914 INFO:tasks.rados.rados.0.smithi063.stdout:3157: writing smithi06387500-23 from 3460532 to 3815453 tid 3 2024-08-17T17:08:07.914 INFO:tasks.rados.rados.0.smithi063.stdout:3158: write oid 7 current snap is 398 2024-08-17T17:08:07.914 INFO:tasks.rados.rados.0.smithi063.stdout:3158: seq_num 1180 ranges {632260=428535,1817458=463413,2687955=520034,3709324=122936} 2024-08-17T17:08:07.920 INFO:tasks.rados.rados.0.smithi063.stdout:3158: writing smithi06387500-7 from 632260 to 1060795 tid 1 2024-08-17T17:08:07.927 INFO:tasks.rados.rados.0.smithi063.stdout:3158: writing smithi06387500-7 from 1817458 to 2280871 tid 2 2024-08-17T17:08:07.932 INFO:tasks.rados.rados.0.smithi063.stdout:3158: writing smithi06387500-7 from 2687955 to 3207989 tid 3 2024-08-17T17:08:07.936 INFO:tasks.rados.rados.0.smithi063.stdout:3158: writing smithi06387500-7 from 3709324 to 3832260 tid 4 2024-08-17T17:08:07.936 INFO:tasks.rados.rados.0.smithi063.stdout:3159: read oid 38 snap 387 2024-08-17T17:08:07.936 INFO:tasks.rados.rados.0.smithi063.stdout:3159: expect (ObjNum 1147 snap 385 seq_num 1147) 2024-08-17T17:08:07.936 INFO:tasks.rados.rados.0.smithi063.stdout:3160: delete oid 2 current snap is 398 2024-08-17T17:08:07.936 INFO:tasks.rados.rados.0.smithi063.stdout:3152: finishing rollback tid 0 to smithi06387500-18 2024-08-17T17:08:07.936 INFO:tasks.rados.rados.0.smithi063.stdout:3152: finishing rollback tid 1 to smithi06387500-18 2024-08-17T17:08:07.936 INFO:tasks.rados.rados.0.smithi063.stdout:3152: finishing rollback tid 2 to smithi06387500-18 2024-08-17T17:08:07.936 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 1016 (ObjNum 1130 snap 377 seq_num 1130) dirty exists 2024-08-17T17:08:07.936 INFO:tasks.rados.rados.0.smithi063.stdout:3151: finishing rollback tid 0 to smithi06387500-49 2024-08-17T17:08:07.937 INFO:tasks.rados.rados.0.smithi063.stdout:3153: finishing rollback tid 1 to smithi06387500-41 2024-08-17T17:08:07.937 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 342 (ObjNum 33 snap 0 seq_num 1786890032) dirty dne 2024-08-17T17:08:07.937 INFO:tasks.rados.rados.0.smithi063.stdout:3151: finishing rollback tid 1 to smithi06387500-49 2024-08-17T17:08:07.937 INFO:tasks.rados.rados.0.smithi063.stdout:3151: finishing rollback tid 2 to smithi06387500-49 2024-08-17T17:08:07.937 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 3062 (ObjNum 1101 snap 366 seq_num 1101) dirty exists 2024-08-17T17:08:07.937 INFO:tasks.rados.rados.0.smithi063.stdout:3155: expect (ObjNum 1159 snap 392 seq_num 1159) 2024-08-17T17:08:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:07 smithi063 ceph-mon[114756]: pgmap v744: 89 pgs: 89 active+clean; 328 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 387 KiB/s wr, 2 op/s 2024-08-17T17:08:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:07 smithi063 ceph-mon[114756]: osdmap e859: 8 total, 8 up, 8 in 2024-08-17T17:08:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:07 smithi063 ceph-mon[114756]: osdmap e860: 8 total, 8 up, 8 in 2024-08-17T17:08:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:07 smithi063 ceph-mon[109843]: pgmap v744: 89 pgs: 89 active+clean; 328 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 387 KiB/s wr, 2 op/s 2024-08-17T17:08:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:07 smithi063 ceph-mon[109843]: osdmap e859: 8 total, 8 up, 8 in 2024-08-17T17:08:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:07 smithi063 ceph-mon[109843]: osdmap e860: 8 total, 8 up, 8 in 2024-08-17T17:08:08.095 INFO:tasks.rados.rados.0.smithi063.stdout:3156: finishing write tid 1 to smithi06387500-37 2024-08-17T17:08:08.096 INFO:tasks.rados.rados.0.smithi063.stdout:3156: finishing write tid 2 to smithi06387500-37 2024-08-17T17:08:08.096 INFO:tasks.rados.rados.0.smithi063.stdout:3156: finishing write tid 3 to smithi06387500-37 2024-08-17T17:08:08.096 INFO:tasks.rados.rados.0.smithi063.stdout:3156: finishing write tid 4 to smithi06387500-37 2024-08-17T17:08:08.096 INFO:tasks.rados.rados.0.smithi063.stdout:3156: finishing write tid 5 to smithi06387500-37 2024-08-17T17:08:08.096 INFO:tasks.rados.rados.0.smithi063.stdout:3156: oid 37 updating version 0 to 3063 2024-08-17T17:08:08.096 INFO:tasks.rados.rados.0.smithi063.stdout:3156: oid 37 updating version 3063 to 3064 2024-08-17T17:08:08.096 INFO:tasks.rados.rados.0.smithi063.stdout:3156: oid 37 updating version 3064 to 3066 2024-08-17T17:08:08.096 INFO:tasks.rados.rados.0.smithi063.stdout:3156: oid 37 version 3066 is already newer than 3065 2024-08-17T17:08:08.096 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 3066 (ObjNum 1178 snap 398 seq_num 1178) dirty exists 2024-08-17T17:08:08.096 INFO:tasks.rados.rados.0.smithi063.stdout:3156: left oid 37 (ObjNum 1178 snap 398 seq_num 1178) 2024-08-17T17:08:08.096 INFO:tasks.rados.rados.0.smithi063.stdout:3157: finishing write tid 1 to smithi06387500-23 2024-08-17T17:08:08.097 INFO:tasks.rados.rados.0.smithi063.stdout:3151: done (9 left) 2024-08-17T17:08:08.097 INFO:tasks.rados.rados.0.smithi063.stdout:3152: done (8 left) 2024-08-17T17:08:08.097 INFO:tasks.rados.rados.0.smithi063.stdout:3153: done (7 left) 2024-08-17T17:08:08.097 INFO:tasks.rados.rados.0.smithi063.stdout:3154: done (6 left) 2024-08-17T17:08:08.097 INFO:tasks.rados.rados.0.smithi063.stdout:3155: done (5 left) 2024-08-17T17:08:08.097 INFO:tasks.rados.rados.0.smithi063.stdout:3156: done (4 left) 2024-08-17T17:08:08.097 INFO:tasks.rados.rados.0.smithi063.stdout:3160: done (3 left) 2024-08-17T17:08:08.097 INFO:tasks.rados.rados.0.smithi063.stdout:3161: write oid 47 current snap is 398 2024-08-17T17:08:08.097 INFO:tasks.rados.rados.0.smithi063.stdout:3157: finishing write tid 2 to smithi06387500-23 2024-08-17T17:08:08.097 INFO:tasks.rados.rados.0.smithi063.stdout:3161: seq_num 1182 ranges {665874=579033,1934422=467082,3142801=542578} 2024-08-17T17:08:08.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:07 smithi162 ceph-mon[104668]: pgmap v744: 89 pgs: 89 active+clean; 328 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 387 KiB/s wr, 2 op/s 2024-08-17T17:08:08.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:07 smithi162 ceph-mon[104668]: osdmap e859: 8 total, 8 up, 8 in 2024-08-17T17:08:08.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:07 smithi162 ceph-mon[104668]: osdmap e860: 8 total, 8 up, 8 in 2024-08-17T17:08:08.102 INFO:tasks.rados.rados.0.smithi063.stdout:3161: writing smithi06387500-47 from 665874 to 1244907 tid 1 2024-08-17T17:08:08.108 INFO:tasks.rados.rados.0.smithi063.stdout:3161: writing smithi06387500-47 from 1934422 to 2401504 tid 2 2024-08-17T17:08:08.116 INFO:tasks.rados.rados.0.smithi063.stdout:3161: writing smithi06387500-47 from 3142801 to 3685379 tid 3 2024-08-17T17:08:08.116 INFO:tasks.rados.rados.0.smithi063.stdout:3162: read oid 37 snap 396 2024-08-17T17:08:08.116 INFO:tasks.rados.rados.0.smithi063.stdout:3162: expect deleted 2024-08-17T17:08:08.116 INFO:tasks.rados.rados.0.smithi063.stdout:3163: delete oid 22 current snap is 398 2024-08-17T17:08:08.116 INFO:tasks.rados.rados.0.smithi063.stdout:3157: finishing write tid 3 to smithi06387500-23 2024-08-17T17:08:08.116 INFO:tasks.rados.rados.0.smithi063.stdout:3157: finishing write tid 4 to smithi06387500-23 2024-08-17T17:08:08.117 INFO:tasks.rados.rados.0.smithi063.stdout:3157: finishing write tid 5 to smithi06387500-23 2024-08-17T17:08:08.117 INFO:tasks.rados.rados.0.smithi063.stdout:3157: oid 23 updating version 0 to 1541 2024-08-17T17:08:08.117 INFO:tasks.rados.rados.0.smithi063.stdout:3157: oid 23 version 1541 is already newer than 1539 2024-08-17T17:08:08.117 INFO:tasks.rados.rados.0.smithi063.stdout:3157: oid 23 version 1541 is already newer than 1538 2024-08-17T17:08:08.117 INFO:tasks.rados.rados.0.smithi063.stdout:3157: oid 23 version 1541 is already newer than 1540 2024-08-17T17:08:08.117 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 1541 (ObjNum 1179 snap 398 seq_num 1179) dirty exists 2024-08-17T17:08:08.117 INFO:tasks.rados.rados.0.smithi063.stdout:3157: left oid 23 (ObjNum 1179 snap 398 seq_num 1179) 2024-08-17T17:08:08.117 INFO:tasks.rados.rados.0.smithi063.stdout:3158: finishing write tid 1 to smithi06387500-7 2024-08-17T17:08:08.117 INFO:tasks.rados.rados.0.smithi063.stdout:3158: finishing write tid 2 to smithi06387500-7 2024-08-17T17:08:08.118 INFO:tasks.rados.rados.0.smithi063.stdout:3158: finishing write tid 3 to smithi06387500-7 2024-08-17T17:08:08.118 INFO:tasks.rados.rados.0.smithi063.stdout:3158: finishing write tid 4 to smithi06387500-7 2024-08-17T17:08:08.118 INFO:tasks.rados.rados.0.smithi063.stdout:3158: finishing write tid 5 to smithi06387500-7 2024-08-17T17:08:08.118 INFO:tasks.rados.rados.0.smithi063.stdout:3158: finishing write tid 6 to smithi06387500-7 2024-08-17T17:08:08.118 INFO:tasks.rados.rados.0.smithi063.stdout:3158: oid 7 updating version 0 to 1474 2024-08-17T17:08:08.118 INFO:tasks.rados.rados.0.smithi063.stdout:3158: oid 7 version 1474 is already newer than 1473 2024-08-17T17:08:08.118 INFO:tasks.rados.rados.0.smithi063.stdout:3158: oid 7 updating version 1474 to 1475 2024-08-17T17:08:08.118 INFO:tasks.rados.rados.0.smithi063.stdout:3158: oid 7 updating version 1475 to 1476 2024-08-17T17:08:08.118 INFO:tasks.rados.rados.0.smithi063.stdout:3158: oid 7 updating version 1476 to 1477 2024-08-17T17:08:08.119 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1477 (ObjNum 1180 snap 398 seq_num 1180) dirty exists 2024-08-17T17:08:08.119 INFO:tasks.rados.rados.0.smithi063.stdout:3158: left oid 7 (ObjNum 1180 snap 398 seq_num 1180) 2024-08-17T17:08:08.119 INFO:tasks.rados.rados.0.smithi063.stdout:3159: expect (ObjNum 1147 snap 385 seq_num 1147) 2024-08-17T17:08:08.316 INFO:tasks.rados.rados.0.smithi063.stdout:3161: finishing write tid 1 to smithi06387500-47 2024-08-17T17:08:08.316 INFO:tasks.rados.rados.0.smithi063.stdout:3161: finishing write tid 2 to smithi06387500-47 2024-08-17T17:08:08.317 INFO:tasks.rados.rados.0.smithi063.stdout:3161: finishing write tid 3 to smithi06387500-47 2024-08-17T17:08:08.317 INFO:tasks.rados.rados.0.smithi063.stdout:3161: finishing write tid 4 to smithi06387500-47 2024-08-17T17:08:08.317 INFO:tasks.rados.rados.0.smithi063.stdout:3161: finishing write tid 5 to smithi06387500-47 2024-08-17T17:08:08.317 INFO:tasks.rados.rados.0.smithi063.stdout:3161: oid 47 updating version 0 to 657 2024-08-17T17:08:08.317 INFO:tasks.rados.rados.0.smithi063.stdout:3161: oid 47 version 657 is already newer than 655 2024-08-17T17:08:08.317 INFO:tasks.rados.rados.0.smithi063.stdout:3161: oid 47 version 657 is already newer than 656 2024-08-17T17:08:08.317 INFO:tasks.rados.rados.0.smithi063.stdout:3161: oid 47 updating version 657 to 658 2024-08-17T17:08:08.317 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 658 (ObjNum 1182 snap 398 seq_num 1182) dirty exists 2024-08-17T17:08:08.317 INFO:tasks.rados.rados.0.smithi063.stdout:3161: left oid 47 (ObjNum 1182 snap 398 seq_num 1182) 2024-08-17T17:08:08.317 INFO:tasks.rados.rados.0.smithi063.stdout:3157: done (5 left) 2024-08-17T17:08:08.317 INFO:tasks.rados.rados.0.smithi063.stdout:3158: done (4 left) 2024-08-17T17:08:08.317 INFO:tasks.rados.rados.0.smithi063.stdout:3159: done (3 left) 2024-08-17T17:08:08.318 INFO:tasks.rados.rados.0.smithi063.stdout:3161: done (2 left) 2024-08-17T17:08:08.318 INFO:tasks.rados.rados.0.smithi063.stdout:3162: done (1 left) 2024-08-17T17:08:08.318 INFO:tasks.rados.rados.0.smithi063.stdout:3163: done (0 left) 2024-08-17T17:08:08.318 INFO:tasks.rados.rados.0.smithi063.stdout:3164: read oid 7 snap -1 2024-08-17T17:08:08.318 INFO:tasks.rados.rados.0.smithi063.stdout:3164: expect (ObjNum 1180 snap 398 seq_num 1180) 2024-08-17T17:08:08.318 INFO:tasks.rados.rados.0.smithi063.stdout:3165: write oid 4 current snap is 398 2024-08-17T17:08:08.318 INFO:tasks.rados.rados.0.smithi063.stdout:3165: seq_num 1184 ranges {699488=729531,1935033=470751,3081294=681475} 2024-08-17T17:08:08.324 INFO:tasks.rados.rados.0.smithi063.stdout:3165: writing smithi06387500-4 from 699488 to 1429019 tid 1 2024-08-17T17:08:08.329 INFO:tasks.rados.rados.0.smithi063.stdout:3165: writing smithi06387500-4 from 1935033 to 2405784 tid 2 2024-08-17T17:08:08.337 INFO:tasks.rados.rados.0.smithi063.stdout:3165: writing smithi06387500-4 from 3081294 to 3762769 tid 3 2024-08-17T17:08:08.337 INFO:tasks.rados.rados.0.smithi063.stdout:3166: write oid 39 current snap is 398 2024-08-17T17:08:08.337 INFO:tasks.rados.rados.0.smithi063.stdout:3166: seq_num 1185 ranges {716295=404780,1593515=614409,2708717=492747,3640519=275776} 2024-08-17T17:08:08.343 INFO:tasks.rados.rados.0.smithi063.stdout:3166: writing smithi06387500-39 from 716295 to 1121075 tid 1 2024-08-17T17:08:08.349 INFO:tasks.rados.rados.0.smithi063.stdout:3166: writing smithi06387500-39 from 1593515 to 2207924 tid 2 2024-08-17T17:08:08.354 INFO:tasks.rados.rados.0.smithi063.stdout:3166: writing smithi06387500-39 from 2708717 to 3201464 tid 3 2024-08-17T17:08:08.359 INFO:tasks.rados.rados.0.smithi063.stdout:3166: writing smithi06387500-39 from 3640519 to 3916295 tid 4 2024-08-17T17:08:08.359 INFO:tasks.rados.rados.0.smithi063.stdout:3167: read oid 26 snap -1 2024-08-17T17:08:08.359 INFO:tasks.rados.rados.0.smithi063.stdout:3167: expect (ObjNum 1079 snap 362 seq_num 1079) 2024-08-17T17:08:08.359 INFO:tasks.rados.rados.0.smithi063.stdout:3168: write oid 2 current snap is 398 2024-08-17T17:08:08.359 INFO:tasks.rados.rados.0.smithi063.stdout:3168: seq_num 1186 ranges {733102=596382,1768350=758067,3136140=704019} 2024-08-17T17:08:08.366 INFO:tasks.rados.rados.0.smithi063.stdout:3168: writing smithi06387500-2 from 733102 to 1329484 tid 1 2024-08-17T17:08:08.372 INFO:tasks.rados.rados.0.smithi063.stdout:3168: writing smithi06387500-2 from 1768350 to 2526417 tid 2 2024-08-17T17:08:08.380 INFO:tasks.rados.rados.0.smithi063.stdout:3168: writing smithi06387500-2 from 3136140 to 3840159 tid 3 2024-08-17T17:08:08.380 INFO:tasks.rados.rados.0.smithi063.stdout:3169: read oid 18 snap 394 2024-08-17T17:08:08.380 INFO:tasks.rados.rados.0.smithi063.stdout:3169: expect (ObjNum 1130 snap 377 seq_num 1130) 2024-08-17T17:08:08.381 INFO:tasks.rados.rados.0.smithi063.stdout:3170: snap_create 2024-08-17T17:08:08.381 INFO:tasks.rados.rados.0.smithi063.stdout:3164: expect (ObjNum 1180 snap 398 seq_num 1180) 2024-08-17T17:08:08.661 INFO:tasks.rados.rados.0.smithi063.stdout:3165: finishing write tid 1 to smithi06387500-4 2024-08-17T17:08:08.661 INFO:tasks.rados.rados.0.smithi063.stdout:3165: finishing write tid 2 to smithi06387500-4 2024-08-17T17:08:08.661 INFO:tasks.rados.rados.0.smithi063.stdout:3165: finishing write tid 3 to smithi06387500-4 2024-08-17T17:08:08.661 INFO:tasks.rados.rados.0.smithi063.stdout:3165: finishing write tid 4 to smithi06387500-4 2024-08-17T17:08:08.661 INFO:tasks.rados.rados.0.smithi063.stdout:3165: finishing write tid 5 to smithi06387500-4 2024-08-17T17:08:08.661 INFO:tasks.rados.rados.0.smithi063.stdout:3165: oid 4 updating version 0 to 3071 2024-08-17T17:08:08.661 INFO:tasks.rados.rados.0.smithi063.stdout:3165: oid 4 version 3071 is already newer than 3070 2024-08-17T17:08:08.661 INFO:tasks.rados.rados.0.smithi063.stdout:3165: oid 4 version 3071 is already newer than 3068 2024-08-17T17:08:08.662 INFO:tasks.rados.rados.0.smithi063.stdout:3165: oid 4 version 3071 is already newer than 3069 2024-08-17T17:08:08.662 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 3071 (ObjNum 1184 snap 398 seq_num 1184) dirty exists 2024-08-17T17:08:08.662 INFO:tasks.rados.rados.0.smithi063.stdout:3165: left oid 4 (ObjNum 1184 snap 398 seq_num 1184) 2024-08-17T17:08:08.662 INFO:tasks.rados.rados.0.smithi063.stdout:3166: finishing write tid 1 to smithi06387500-39 2024-08-17T17:08:08.662 INFO:tasks.rados.rados.0.smithi063.stdout:3166: finishing write tid 2 to smithi06387500-39 2024-08-17T17:08:08.662 INFO:tasks.rados.rados.0.smithi063.stdout:3166: finishing write tid 3 to smithi06387500-39 2024-08-17T17:08:08.662 INFO:tasks.rados.rados.0.smithi063.stdout:3166: finishing write tid 4 to smithi06387500-39 2024-08-17T17:08:08.662 INFO:tasks.rados.rados.0.smithi063.stdout:3166: finishing write tid 5 to smithi06387500-39 2024-08-17T17:08:08.662 INFO:tasks.rados.rados.0.smithi063.stdout:3166: finishing write tid 6 to smithi06387500-39 2024-08-17T17:08:08.662 INFO:tasks.rados.rados.0.smithi063.stdout:3166: oid 39 updating version 0 to 723 2024-08-17T17:08:08.662 INFO:tasks.rados.rados.0.smithi063.stdout:3166: oid 39 version 723 is already newer than 721 2024-08-17T17:08:08.662 INFO:tasks.rados.rados.0.smithi063.stdout:3166: oid 39 version 723 is already newer than 722 2024-08-17T17:08:08.663 INFO:tasks.rados.rados.0.smithi063.stdout:3166: oid 39 updating version 723 to 724 2024-08-17T17:08:08.663 INFO:tasks.rados.rados.0.smithi063.stdout:3166: oid 39 version 724 is already newer than 720 2024-08-17T17:08:08.663 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 724 (ObjNum 1185 snap 398 seq_num 1185) dirty exists 2024-08-17T17:08:08.663 INFO:tasks.rados.rados.0.smithi063.stdout:3166: left oid 39 (ObjNum 1185 snap 398 seq_num 1185) 2024-08-17T17:08:08.663 INFO:tasks.rados.rados.0.smithi063.stdout:3167: expect (ObjNum 1079 snap 362 seq_num 1079) 2024-08-17T17:08:08.896 INFO:tasks.rados.rados.0.smithi063.stdout:3168: finishing write tid 1 to smithi06387500-2 2024-08-17T17:08:08.896 INFO:tasks.rados.rados.0.smithi063.stdout:3168: finishing write tid 2 to smithi06387500-2 2024-08-17T17:08:08.896 INFO:tasks.rados.rados.0.smithi063.stdout:3168: finishing write tid 3 to smithi06387500-2 2024-08-17T17:08:08.896 INFO:tasks.rados.rados.0.smithi063.stdout:3168: finishing write tid 4 to smithi06387500-2 2024-08-17T17:08:08.896 INFO:tasks.rados.rados.0.smithi063.stdout:3168: finishing write tid 5 to smithi06387500-2 2024-08-17T17:08:08.896 INFO:tasks.rados.rados.0.smithi063.stdout:3168: oid 2 updating version 0 to 1545 2024-08-17T17:08:08.896 INFO:tasks.rados.rados.0.smithi063.stdout:3168: oid 2 version 1545 is already newer than 1544 2024-08-17T17:08:08.897 INFO:tasks.rados.rados.0.smithi063.stdout:3168: oid 2 updating version 1545 to 1546 2024-08-17T17:08:08.897 INFO:tasks.rados.rados.0.smithi063.stdout:3168: oid 2 version 1546 is already newer than 1543 2024-08-17T17:08:08.897 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 1546 (ObjNum 1186 snap 398 seq_num 1186) dirty exists 2024-08-17T17:08:08.897 INFO:tasks.rados.rados.0.smithi063.stdout:3168: left oid 2 (ObjNum 1186 snap 398 seq_num 1186) 2024-08-17T17:08:08.897 INFO:tasks.rados.rados.0.smithi063.stdout:3169: expect (ObjNum 1130 snap 377 seq_num 1130) 2024-08-17T17:08:08.997 INFO:tasks.rados.rados.0.smithi063.stdout:3164: done (6 left) 2024-08-17T17:08:08.997 INFO:tasks.rados.rados.0.smithi063.stdout:3165: done (5 left) 2024-08-17T17:08:08.997 INFO:tasks.rados.rados.0.smithi063.stdout:3166: done (4 left) 2024-08-17T17:08:08.997 INFO:tasks.rados.rados.0.smithi063.stdout:3167: done (3 left) 2024-08-17T17:08:08.997 INFO:tasks.rados.rados.0.smithi063.stdout:3168: done (2 left) 2024-08-17T17:08:08.997 INFO:tasks.rados.rados.0.smithi063.stdout:3169: done (1 left) 2024-08-17T17:08:08.997 INFO:tasks.rados.rados.0.smithi063.stdout:3170: done (0 left) 2024-08-17T17:08:08.997 INFO:tasks.rados.rados.0.smithi063.stdout:3171: write oid 1 current snap is 399 2024-08-17T17:08:08.997 INFO:tasks.rados.rados.0.smithi063.stdout:3171: seq_num 1187 ranges {749909=671631,2110479=618078,3163563=515291} 2024-08-17T17:08:09.004 INFO:tasks.rados.rados.0.smithi063.stdout:3171: writing smithi06387500-1 from 749909 to 1421540 tid 1 2024-08-17T17:08:09.011 INFO:tasks.rados.rados.0.smithi063.stdout:3171: writing smithi06387500-1 from 2110479 to 2728557 tid 2 2024-08-17T17:08:09.017 INFO:tasks.rados.rados.0.smithi063.stdout:3171: writing smithi06387500-1 from 3163563 to 3678854 tid 3 2024-08-17T17:08:09.017 INFO:tasks.rados.rados.0.smithi063.stdout:3172: write oid 38 current snap is 399 2024-08-17T17:08:09.017 INFO:tasks.rados.rados.0.smithi063.stdout:3172: seq_num 1188 ranges {766716=746880,2168961=761736,3474633=442916} 2024-08-17T17:08:09.025 INFO:tasks.rados.rados.0.smithi063.stdout:3172: writing smithi06387500-38 from 766716 to 1513596 tid 1 2024-08-17T17:08:09.033 INFO:tasks.rados.rados.0.smithi063.stdout:3172: writing smithi06387500-38 from 2168961 to 2930697 tid 2 2024-08-17T17:08:09.039 INFO:tasks.rados.rados.0.smithi063.stdout:3172: writing smithi06387500-38 from 3474633 to 3917549 tid 3 2024-08-17T17:08:09.039 INFO:tasks.rados.rados.0.smithi063.stdout:3173: read oid 48 snap -1 2024-08-17T17:08:09.039 INFO:tasks.rados.rados.0.smithi063.stdout:3173: expect (ObjNum 1143 snap 381 seq_num 1143) 2024-08-17T17:08:09.039 INFO:tasks.rados.rados.0.smithi063.stdout:3174: write oid 13 current snap is 399 2024-08-17T17:08:09.039 INFO:tasks.rados.rados.0.smithi063.stdout:3171: finishing write tid 1 to smithi06387500-1 2024-08-17T17:08:09.039 INFO:tasks.rados.rados.0.smithi063.stdout:3171: finishing write tid 2 to smithi06387500-1 2024-08-17T17:08:09.039 INFO:tasks.rados.rados.0.smithi063.stdout:3171: finishing write tid 3 to smithi06387500-1 2024-08-17T17:08:09.039 INFO:tasks.rados.rados.0.smithi063.stdout:3171: finishing write tid 4 to smithi06387500-1 2024-08-17T17:08:09.039 INFO:tasks.rados.rados.0.smithi063.stdout:3171: finishing write tid 5 to smithi06387500-1 2024-08-17T17:08:09.040 INFO:tasks.rados.rados.0.smithi063.stdout:3171: oid 1 updating version 0 to 1550 2024-08-17T17:08:09.040 INFO:tasks.rados.rados.0.smithi063.stdout:3171: oid 1 updating version 1550 to 1551 2024-08-17T17:08:09.040 INFO:tasks.rados.rados.0.smithi063.stdout:3171: oid 1 version 1551 is already newer than 1549 2024-08-17T17:08:09.040 INFO:tasks.rados.rados.0.smithi063.stdout:3171: oid 1 version 1551 is already newer than 1548 2024-08-17T17:08:09.040 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 1551 (ObjNum 1187 snap 399 seq_num 1187) dirty exists 2024-08-17T17:08:09.040 INFO:tasks.rados.rados.0.smithi063.stdout:3171: left oid 1 (ObjNum 1187 snap 399 seq_num 1187) 2024-08-17T17:08:09.040 INFO:tasks.rados.rados.0.smithi063.stdout:3172: finishing write tid 1 to smithi06387500-38 2024-08-17T17:08:09.040 INFO:tasks.rados.rados.0.smithi063.stdout:3172: finishing write tid 2 to smithi06387500-38 2024-08-17T17:08:09.040 INFO:tasks.rados.rados.0.smithi063.stdout:3174: seq_num 1189 ranges {783523=422129,1827443=621747,3218409=654188} 2024-08-17T17:08:09.045 INFO:tasks.rados.rados.0.smithi063.stdout:3174: writing smithi06387500-13 from 783523 to 1205652 tid 1 2024-08-17T17:08:09.052 INFO:tasks.rados.rados.0.smithi063.stdout:3174: writing smithi06387500-13 from 1827443 to 2449190 tid 2 2024-08-17T17:08:09.060 INFO:tasks.rados.rados.0.smithi063.stdout:3174: writing smithi06387500-13 from 3218409 to 3872597 tid 3 2024-08-17T17:08:09.060 INFO:tasks.rados.rados.0.smithi063.stdout:3171: done (3 left) 2024-08-17T17:08:09.060 INFO:tasks.rados.rados.0.smithi063.stdout:3175: read oid 4 snap -1 2024-08-17T17:08:09.060 INFO:tasks.rados.rados.0.smithi063.stdout:3175: expect (ObjNum 1184 snap 398 seq_num 1184) 2024-08-17T17:08:09.060 INFO:tasks.rados.rados.0.smithi063.stdout:3176: snap_create 2024-08-17T17:08:09.061 INFO:tasks.rados.rados.0.smithi063.stdout:3172: finishing write tid 3 to smithi06387500-38 2024-08-17T17:08:09.061 INFO:tasks.rados.rados.0.smithi063.stdout:3172: finishing write tid 4 to smithi06387500-38 2024-08-17T17:08:09.061 INFO:tasks.rados.rados.0.smithi063.stdout:3172: finishing write tid 5 to smithi06387500-38 2024-08-17T17:08:09.061 INFO:tasks.rados.rados.0.smithi063.stdout:3172: oid 38 updating version 0 to 538 2024-08-17T17:08:09.061 INFO:tasks.rados.rados.0.smithi063.stdout:3172: oid 38 version 538 is already newer than 536 2024-08-17T17:08:09.061 INFO:tasks.rados.rados.0.smithi063.stdout:3172: oid 38 updating version 538 to 539 2024-08-17T17:08:09.061 INFO:tasks.rados.rados.0.smithi063.stdout:3172: oid 38 version 539 is already newer than 537 2024-08-17T17:08:09.061 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 539 (ObjNum 1188 snap 399 seq_num 1188) dirty exists 2024-08-17T17:08:09.061 INFO:tasks.rados.rados.0.smithi063.stdout:3172: left oid 38 (ObjNum 1188 snap 399 seq_num 1188) 2024-08-17T17:08:09.061 INFO:tasks.rados.rados.0.smithi063.stdout:3174: finishing write tid 1 to smithi06387500-13 2024-08-17T17:08:09.061 INFO:tasks.rados.rados.0.smithi063.stdout:3173: expect (ObjNum 1143 snap 381 seq_num 1143) 2024-08-17T17:08:09.207 INFO:tasks.rados.rados.0.smithi063.stdout:3174: finishing write tid 2 to smithi06387500-13 2024-08-17T17:08:09.207 INFO:tasks.rados.rados.0.smithi063.stdout:3174: finishing write tid 3 to smithi06387500-13 2024-08-17T17:08:09.207 INFO:tasks.rados.rados.0.smithi063.stdout:3174: finishing write tid 4 to smithi06387500-13 2024-08-17T17:08:09.207 INFO:tasks.rados.rados.0.smithi063.stdout:3174: finishing write tid 5 to smithi06387500-13 2024-08-17T17:08:09.208 INFO:tasks.rados.rados.0.smithi063.stdout:3174: oid 13 updating version 0 to 1553 2024-08-17T17:08:09.208 INFO:tasks.rados.rados.0.smithi063.stdout:3174: oid 13 updating version 1553 to 1554 2024-08-17T17:08:09.208 INFO:tasks.rados.rados.0.smithi063.stdout:3174: oid 13 updating version 1554 to 1556 2024-08-17T17:08:09.208 INFO:tasks.rados.rados.0.smithi063.stdout:3174: oid 13 version 1556 is already newer than 1555 2024-08-17T17:08:09.208 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1556 (ObjNum 1189 snap 399 seq_num 1189) dirty exists 2024-08-17T17:08:09.208 INFO:tasks.rados.rados.0.smithi063.stdout:3174: left oid 13 (ObjNum 1189 snap 399 seq_num 1189) 2024-08-17T17:08:09.208 INFO:tasks.rados.rados.0.smithi063.stdout:3175: expect (ObjNum 1184 snap 398 seq_num 1184) 2024-08-17T17:08:09.664 INFO:tasks.rados.rados.0.smithi063.stdout:3172: done (4 left) 2024-08-17T17:08:09.664 INFO:tasks.rados.rados.0.smithi063.stdout:3173: done (3 left) 2024-08-17T17:08:09.664 INFO:tasks.rados.rados.0.smithi063.stdout:3174: done (2 left) 2024-08-17T17:08:09.665 INFO:tasks.rados.rados.0.smithi063.stdout:3175: done (1 left) 2024-08-17T17:08:09.665 INFO:tasks.rados.rados.0.smithi063.stdout:3176: done (0 left) 2024-08-17T17:08:09.665 INFO:tasks.rados.rados.0.smithi063.stdout:3177: write oid 19 current snap is 400 2024-08-17T17:08:09.665 INFO:tasks.rados.rados.0.smithi063.stdout:3177: seq_num 1190 ranges {400330=497378,1485925=514405} 2024-08-17T17:08:09.668 INFO:tasks.rados.rados.0.smithi063.stdout:3177: writing smithi06387500-19 from 400330 to 897708 tid 1 2024-08-17T17:08:09.675 INFO:tasks.rados.rados.0.smithi063.stdout:3177: writing smithi06387500-19 from 1485925 to 2000330 tid 2 2024-08-17T17:08:09.675 INFO:tasks.rados.rados.0.smithi063.stdout:3178: rollback oid 2 current snap is 400 2024-08-17T17:08:09.675 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 2 to 399 2024-08-17T17:08:09.675 INFO:tasks.rados.rados.0.smithi063.stdout:3177: finishing write tid 1 to smithi06387500-19 2024-08-17T17:08:09.676 INFO:tasks.rados.rados.0.smithi063.stdout:3179: delete oid 15 current snap is 400 2024-08-17T17:08:09.678 INFO:tasks.rados.rados.0.smithi063.stdout:3179: done (2 left) 2024-08-17T17:08:09.678 INFO:tasks.rados.rados.0.smithi063.stdout:3180: delete oid 8 current snap is 400 2024-08-17T17:08:09.679 INFO:tasks.rados.rados.0.smithi063.stdout:3177: finishing write tid 2 to smithi06387500-19 2024-08-17T17:08:09.679 INFO:tasks.rados.rados.0.smithi063.stdout:3177: finishing write tid 3 to smithi06387500-19 2024-08-17T17:08:09.679 INFO:tasks.rados.rados.0.smithi063.stdout:3177: finishing write tid 4 to smithi06387500-19 2024-08-17T17:08:09.679 INFO:tasks.rados.rados.0.smithi063.stdout:3177: oid 19 updating version 0 to 1994 2024-08-17T17:08:09.679 INFO:tasks.rados.rados.0.smithi063.stdout:3177: oid 19 updating version 1994 to 1995 2024-08-17T17:08:09.680 INFO:tasks.rados.rados.0.smithi063.stdout:3177: oid 19 version 1995 is already newer than 1993 2024-08-17T17:08:09.680 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 1995 (ObjNum 1190 snap 400 seq_num 1190) dirty exists 2024-08-17T17:08:09.680 INFO:tasks.rados.rados.0.smithi063.stdout:3177: left oid 19 (ObjNum 1190 snap 400 seq_num 1190) 2024-08-17T17:08:09.680 INFO:tasks.rados.rados.0.smithi063.stdout:3178: finishing rollback tid 0 to smithi06387500-2 2024-08-17T17:08:09.681 INFO:tasks.rados.rados.0.smithi063.stdout:3178: finishing rollback tid 1 to smithi06387500-2 2024-08-17T17:08:09.681 INFO:tasks.rados.rados.0.smithi063.stdout:3177: done (2 left) 2024-08-17T17:08:09.681 INFO:tasks.rados.rados.0.smithi063.stdout:3180: done (1 left) 2024-08-17T17:08:09.681 INFO:tasks.rados.rados.0.smithi063.stdout:3181: write oid 26 current snap is 400 2024-08-17T17:08:09.681 INFO:tasks.rados.rados.0.smithi063.stdout:3181: seq_num 1193 ranges {450751=723125,1945018=105733} 2024-08-17T17:08:09.687 INFO:tasks.rados.rados.0.smithi063.stdout:3181: writing smithi06387500-26 from 450751 to 1173876 tid 1 2024-08-17T17:08:09.692 INFO:tasks.rados.rados.0.smithi063.stdout:3181: writing smithi06387500-26 from 1945018 to 2050751 tid 2 2024-08-17T17:08:09.692 INFO:tasks.rados.rados.0.smithi063.stdout:3182: rollback oid 45 current snap is 400 2024-08-17T17:08:09.692 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 45 to 383 2024-08-17T17:08:09.692 INFO:tasks.rados.rados.0.smithi063.stdout:3178: finishing rollback tid 2 to smithi06387500-2 2024-08-17T17:08:09.692 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 1560 (ObjNum 1186 snap 398 seq_num 1186) dirty exists 2024-08-17T17:08:09.692 INFO:tasks.rados.rados.0.smithi063.stdout:3181: finishing write tid 1 to smithi06387500-26 2024-08-17T17:08:09.693 INFO:tasks.rados.rados.0.smithi063.stdout:3178: done (2 left) 2024-08-17T17:08:09.693 INFO:tasks.rados.rados.0.smithi063.stdout:3183: rollback oid 37 current snap is 400 2024-08-17T17:08:09.693 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 37 to 399 2024-08-17T17:08:09.693 INFO:tasks.rados.rados.0.smithi063.stdout:3184: snap_remove snap 366 2024-08-17T17:08:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:09 smithi063 ceph-mon[114756]: pgmap v747: 89 pgs: 89 active+clean; 327 MiB data, 935 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 387 KiB/s wr, 4 op/s 2024-08-17T17:08:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:09 smithi063 ceph-mon[114756]: osdmap e861: 8 total, 8 up, 8 in 2024-08-17T17:08:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:09 smithi063 ceph-mon[109843]: pgmap v747: 89 pgs: 89 active+clean; 327 MiB data, 935 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 387 KiB/s wr, 4 op/s 2024-08-17T17:08:10.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:09 smithi063 ceph-mon[109843]: osdmap e861: 8 total, 8 up, 8 in 2024-08-17T17:08:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:09 smithi162 ceph-mon[104668]: pgmap v747: 89 pgs: 89 active+clean; 327 MiB data, 935 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 387 KiB/s wr, 4 op/s 2024-08-17T17:08:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:09 smithi162 ceph-mon[104668]: osdmap e861: 8 total, 8 up, 8 in 2024-08-17T17:08:10.677 INFO:tasks.rados.rados.0.smithi063.stdout:3184: done (3 left) 2024-08-17T17:08:10.677 INFO:tasks.rados.rados.0.smithi063.stdout:3185: write oid 43 current snap is 400 2024-08-17T17:08:10.677 INFO:tasks.rados.rados.0.smithi063.stdout:3185: seq_num 1194 ranges {467558=514727,1719853=347705} 2024-08-17T17:08:10.682 INFO:tasks.rados.rados.0.smithi063.stdout:3185: writing smithi06387500-43 from 467558 to 982285 tid 1 2024-08-17T17:08:10.689 INFO:tasks.rados.rados.0.smithi063.stdout:3185: writing smithi06387500-43 from 1719853 to 2067558 tid 2 2024-08-17T17:08:10.690 INFO:tasks.rados.rados.0.smithi063.stdout:3186: snap_remove snap 397 2024-08-17T17:08:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:10 smithi063 ceph-mon[114756]: osdmap e862: 8 total, 8 up, 8 in 2024-08-17T17:08:11.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:10 smithi063 ceph-mon[109843]: osdmap e862: 8 total, 8 up, 8 in 2024-08-17T17:08:11.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:10 smithi162 ceph-mon[104668]: osdmap e862: 8 total, 8 up, 8 in 2024-08-17T17:08:11.682 INFO:tasks.rados.rados.0.smithi063.stdout:3186: done (4 left) 2024-08-17T17:08:11.686 INFO:tasks.rados.rados.0.smithi063.stdout:3187: snap_create 2024-08-17T17:08:11.686 INFO:tasks.rados.rados.0.smithi063.stdout:3181: finishing write tid 2 to smithi06387500-26 2024-08-17T17:08:11.686 INFO:tasks.rados.rados.0.smithi063.stdout:3181: finishing write tid 3 to smithi06387500-26 2024-08-17T17:08:11.686 INFO:tasks.rados.rados.0.smithi063.stdout:3181: finishing write tid 4 to smithi06387500-26 2024-08-17T17:08:11.686 INFO:tasks.rados.rados.0.smithi063.stdout:3181: oid 26 updating version 0 to 1057 2024-08-17T17:08:11.686 INFO:tasks.rados.rados.0.smithi063.stdout:3181: oid 26 updating version 1057 to 1059 2024-08-17T17:08:11.687 INFO:tasks.rados.rados.0.smithi063.stdout:3181: oid 26 version 1059 is already newer than 1058 2024-08-17T17:08:11.687 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 1059 (ObjNum 1193 snap 400 seq_num 1193) dirty exists 2024-08-17T17:08:11.687 INFO:tasks.rados.rados.0.smithi063.stdout:3181: left oid 26 (ObjNum 1193 snap 400 seq_num 1193) 2024-08-17T17:08:11.687 INFO:tasks.rados.rados.0.smithi063.stdout:3183: finishing rollback tid 0 to smithi06387500-37 2024-08-17T17:08:11.687 INFO:tasks.rados.rados.0.smithi063.stdout:3182: finishing rollback tid 0 to smithi06387500-45 2024-08-17T17:08:11.687 INFO:tasks.rados.rados.0.smithi063.stdout:3182: finishing rollback tid 1 to smithi06387500-45 2024-08-17T17:08:11.687 INFO:tasks.rados.rados.0.smithi063.stdout:3182: finishing rollback tid 2 to smithi06387500-45 2024-08-17T17:08:11.687 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 728 (ObjNum 1112 snap 369 seq_num 1112) dirty exists 2024-08-17T17:08:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:3183: finishing rollback tid 1 to smithi06387500-37 2024-08-17T17:08:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:3183: finishing rollback tid 2 to smithi06387500-37 2024-08-17T17:08:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 3075 (ObjNum 1178 snap 398 seq_num 1178) dirty exists 2024-08-17T17:08:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:3185: finishing write tid 1 to smithi06387500-43 2024-08-17T17:08:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:3185: finishing write tid 2 to smithi06387500-43 2024-08-17T17:08:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:3185: finishing write tid 3 to smithi06387500-43 2024-08-17T17:08:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:3185: finishing write tid 4 to smithi06387500-43 2024-08-17T17:08:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:3185: oid 43 updating version 0 to 1486 2024-08-17T17:08:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:3185: oid 43 version 1486 is already newer than 1485 2024-08-17T17:08:11.689 INFO:tasks.rados.rados.0.smithi063.stdout:3185: oid 43 version 1486 is already newer than 1482 2024-08-17T17:08:11.689 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 1486 (ObjNum 1194 snap 400 seq_num 1194) dirty exists 2024-08-17T17:08:11.689 INFO:tasks.rados.rados.0.smithi063.stdout:3185: left oid 43 (ObjNum 1194 snap 400 seq_num 1194) 2024-08-17T17:08:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:11 smithi063 ceph-mon[114756]: pgmap v750: 89 pgs: 89 active+clean; 330 MiB data, 936 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 785 KiB/s wr, 5 op/s 2024-08-17T17:08:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:11 smithi063 ceph-mon[114756]: osdmap e863: 8 total, 8 up, 8 in 2024-08-17T17:08:12.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:11 smithi063 ceph-mon[109843]: pgmap v750: 89 pgs: 89 active+clean; 330 MiB data, 936 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 785 KiB/s wr, 5 op/s 2024-08-17T17:08:12.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:11 smithi063 ceph-mon[109843]: osdmap e863: 8 total, 8 up, 8 in 2024-08-17T17:08:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:11 smithi162 ceph-mon[104668]: pgmap v750: 89 pgs: 89 active+clean; 330 MiB data, 936 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 785 KiB/s wr, 5 op/s 2024-08-17T17:08:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:11 smithi162 ceph-mon[104668]: osdmap e863: 8 total, 8 up, 8 in 2024-08-17T17:08:12.637 INFO:tasks.rados.rados.0.smithi063.stdout:3181: done (4 left) 2024-08-17T17:08:12.637 INFO:tasks.rados.rados.0.smithi063.stdout:3182: done (3 left) 2024-08-17T17:08:12.637 INFO:tasks.rados.rados.0.smithi063.stdout:3183: done (2 left) 2024-08-17T17:08:12.637 INFO:tasks.rados.rados.0.smithi063.stdout:3185: done (1 left) 2024-08-17T17:08:12.637 INFO:tasks.rados.rados.0.smithi063.stdout:3187: done (0 left) 2024-08-17T17:08:12.637 INFO:tasks.rados.rados.0.smithi063.stdout:3188: write oid 45 current snap is 401 2024-08-17T17:08:12.637 INFO:tasks.rados.rados.0.smithi063.stdout:3188: seq_num 1195 ranges {484365=589976,1661982=422383} 2024-08-17T17:08:12.642 INFO:tasks.rados.rados.0.smithi063.stdout:3188: writing smithi06387500-45 from 484365 to 1074341 tid 1 2024-08-17T17:08:12.648 INFO:tasks.rados.rados.0.smithi063.stdout:3188: writing smithi06387500-45 from 1661982 to 2084365 tid 2 2024-08-17T17:08:12.648 INFO:tasks.rados.rados.0.smithi063.stdout:3189: write oid 14 current snap is 401 2024-08-17T17:08:12.648 INFO:tasks.rados.rados.0.smithi063.stdout:3189: seq_num 1196 ranges {501172=665225,1720464=380708} 2024-08-17T17:08:12.655 INFO:tasks.rados.rados.0.smithi063.stdout:3189: writing smithi06387500-14 from 501172 to 1166397 tid 1 2024-08-17T17:08:12.660 INFO:tasks.rados.rados.0.smithi063.stdout:3189: writing smithi06387500-14 from 1720464 to 2101172 tid 2 2024-08-17T17:08:12.661 INFO:tasks.rados.rados.0.smithi063.stdout:3190: snap_create 2024-08-17T17:08:12.661 INFO:tasks.rados.rados.0.smithi063.stdout:3188: finishing write tid 1 to smithi06387500-45 2024-08-17T17:08:12.661 INFO:tasks.rados.rados.0.smithi063.stdout:3188: finishing write tid 2 to smithi06387500-45 2024-08-17T17:08:12.661 INFO:tasks.rados.rados.0.smithi063.stdout:3188: finishing write tid 3 to smithi06387500-45 2024-08-17T17:08:12.661 INFO:tasks.rados.rados.0.smithi063.stdout:3188: finishing write tid 4 to smithi06387500-45 2024-08-17T17:08:12.661 INFO:tasks.rados.rados.0.smithi063.stdout:3188: oid 45 updating version 0 to 738 2024-08-17T17:08:12.662 INFO:tasks.rados.rados.0.smithi063.stdout:3188: oid 45 version 738 is already newer than 736 2024-08-17T17:08:12.662 INFO:tasks.rados.rados.0.smithi063.stdout:3188: oid 45 version 738 is already newer than 737 2024-08-17T17:08:12.662 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 738 (ObjNum 1195 snap 401 seq_num 1195) dirty exists 2024-08-17T17:08:12.662 INFO:tasks.rados.rados.0.smithi063.stdout:3188: left oid 45 (ObjNum 1195 snap 401 seq_num 1195) 2024-08-17T17:08:12.665 INFO:tasks.rados.rados.0.smithi063.stdout:3189: finishing write tid 1 to smithi06387500-14 2024-08-17T17:08:12.665 INFO:tasks.rados.rados.0.smithi063.stdout:3189: finishing write tid 2 to smithi06387500-14 2024-08-17T17:08:12.666 INFO:tasks.rados.rados.0.smithi063.stdout:3189: finishing write tid 3 to smithi06387500-14 2024-08-17T17:08:12.666 INFO:tasks.rados.rados.0.smithi063.stdout:3189: finishing write tid 4 to smithi06387500-14 2024-08-17T17:08:12.666 INFO:tasks.rados.rados.0.smithi063.stdout:3189: oid 14 updating version 0 to 3088 2024-08-17T17:08:12.666 INFO:tasks.rados.rados.0.smithi063.stdout:3189: oid 14 version 3088 is already newer than 3087 2024-08-17T17:08:12.666 INFO:tasks.rados.rados.0.smithi063.stdout:3189: oid 14 updating version 3088 to 3089 2024-08-17T17:08:12.667 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 3089 (ObjNum 1196 snap 401 seq_num 1196) dirty exists 2024-08-17T17:08:12.667 INFO:tasks.rados.rados.0.smithi063.stdout:3189: left oid 14 (ObjNum 1196 snap 401 seq_num 1196) 2024-08-17T17:08:12.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:12 smithi063 ceph-mon[114756]: osdmap e864: 8 total, 8 up, 8 in 2024-08-17T17:08:12.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:12 smithi063 ceph-mon[114756]: pgmap v753: 89 pgs: 89 active+clean; 362 MiB data, 982 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 5.3 MiB/s wr, 38 op/s 2024-08-17T17:08:12.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:12 smithi063 ceph-mon[114756]: osdmap e865: 8 total, 8 up, 8 in 2024-08-17T17:08:12.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:12 smithi063 ceph-mon[109843]: osdmap e864: 8 total, 8 up, 8 in 2024-08-17T17:08:12.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:12 smithi063 ceph-mon[109843]: pgmap v753: 89 pgs: 89 active+clean; 362 MiB data, 982 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 5.3 MiB/s wr, 38 op/s 2024-08-17T17:08:12.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:12 smithi063 ceph-mon[109843]: osdmap e865: 8 total, 8 up, 8 in 2024-08-17T17:08:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:12 smithi162 ceph-mon[104668]: osdmap e864: 8 total, 8 up, 8 in 2024-08-17T17:08:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:12 smithi162 ceph-mon[104668]: pgmap v753: 89 pgs: 89 active+clean; 362 MiB data, 982 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 5.3 MiB/s wr, 38 op/s 2024-08-17T17:08:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:12 smithi162 ceph-mon[104668]: osdmap e865: 8 total, 8 up, 8 in 2024-08-17T17:08:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:08:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:08:13] "GET /metrics HTTP/1.1" 200 36340 "" "Prometheus/2.51.0" 2024-08-17T17:08:13.638 INFO:tasks.rados.rados.0.smithi063.stdout:3188: done (2 left) 2024-08-17T17:08:13.638 INFO:tasks.rados.rados.0.smithi063.stdout:3189: done (1 left) 2024-08-17T17:08:13.638 INFO:tasks.rados.rados.0.smithi063.stdout:3190: done (0 left) 2024-08-17T17:08:13.638 INFO:tasks.rados.rados.0.smithi063.stdout:3191: write oid 31 current snap is 402 2024-08-17T17:08:13.639 INFO:tasks.rados.rados.0.smithi063.stdout:3191: seq_num 1197 ranges {517979=740474,1778946=339033} 2024-08-17T17:08:13.646 INFO:tasks.rados.rados.0.smithi063.stdout:3191: writing smithi06387500-31 from 517979 to 1258453 tid 1 2024-08-17T17:08:13.652 INFO:tasks.rados.rados.0.smithi063.stdout:3191: writing smithi06387500-31 from 1778946 to 2117979 tid 2 2024-08-17T17:08:13.652 INFO:tasks.rados.rados.0.smithi063.stdout:3192: delete oid 2 current snap is 402 2024-08-17T17:08:13.654 INFO:tasks.rados.rados.0.smithi063.stdout:3191: finishing write tid 1 to smithi06387500-31 2024-08-17T17:08:13.655 INFO:tasks.rados.rados.0.smithi063.stdout:3192: done (1 left) 2024-08-17T17:08:13.655 INFO:tasks.rados.rados.0.smithi063.stdout:3193: snap_create 2024-08-17T17:08:13.655 INFO:tasks.rados.rados.0.smithi063.stdout:3191: finishing write tid 2 to smithi06387500-31 2024-08-17T17:08:13.655 INFO:tasks.rados.rados.0.smithi063.stdout:3191: finishing write tid 3 to smithi06387500-31 2024-08-17T17:08:13.655 INFO:tasks.rados.rados.0.smithi063.stdout:3191: finishing write tid 4 to smithi06387500-31 2024-08-17T17:08:13.656 INFO:tasks.rados.rados.0.smithi063.stdout:3191: oid 31 updating version 0 to 2005 2024-08-17T17:08:13.656 INFO:tasks.rados.rados.0.smithi063.stdout:3191: oid 31 version 2005 is already newer than 2004 2024-08-17T17:08:13.656 INFO:tasks.rados.rados.0.smithi063.stdout:3191: oid 31 version 2005 is already newer than 2003 2024-08-17T17:08:13.656 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 2005 (ObjNum 1197 snap 402 seq_num 1197) dirty exists 2024-08-17T17:08:13.656 INFO:tasks.rados.rados.0.smithi063.stdout:3191: left oid 31 (ObjNum 1197 snap 402 seq_num 1197) 2024-08-17T17:08:14.650 INFO:tasks.rados.rados.0.smithi063.stdout:3191: done (1 left) 2024-08-17T17:08:14.651 INFO:tasks.rados.rados.0.smithi063.stdout:3193: done (0 left) 2024-08-17T17:08:14.651 INFO:tasks.rados.rados.0.smithi063.stdout:3194: write oid 13 current snap is 403 2024-08-17T17:08:14.651 INFO:tasks.rados.rados.0.smithi063.stdout:3194: seq_num 1199 ranges {551593=490972,1779557=372036} 2024-08-17T17:08:14.656 INFO:tasks.rados.rados.0.smithi063.stdout:3194: writing smithi06387500-13 from 551593 to 1042565 tid 1 2024-08-17T17:08:14.662 INFO:tasks.rados.rados.0.smithi063.stdout:3194: writing smithi06387500-13 from 1779557 to 2151593 tid 2 2024-08-17T17:08:14.662 INFO:tasks.rados.rados.0.smithi063.stdout:3195: snap_create 2024-08-17T17:08:14.663 INFO:tasks.rados.rados.0.smithi063.stdout:3194: finishing write tid 1 to smithi06387500-13 2024-08-17T17:08:14.666 INFO:tasks.rados.rados.0.smithi063.stdout:3194: finishing write tid 2 to smithi06387500-13 2024-08-17T17:08:14.666 INFO:tasks.rados.rados.0.smithi063.stdout:3194: finishing write tid 3 to smithi06387500-13 2024-08-17T17:08:14.666 INFO:tasks.rados.rados.0.smithi063.stdout:3194: finishing write tid 4 to smithi06387500-13 2024-08-17T17:08:14.666 INFO:tasks.rados.rados.0.smithi063.stdout:3194: oid 13 updating version 0 to 1579 2024-08-17T17:08:14.666 INFO:tasks.rados.rados.0.smithi063.stdout:3194: oid 13 version 1579 is already newer than 1578 2024-08-17T17:08:14.666 INFO:tasks.rados.rados.0.smithi063.stdout:3194: oid 13 updating version 1579 to 1580 2024-08-17T17:08:14.666 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1580 (ObjNum 1199 snap 403 seq_num 1199) dirty exists 2024-08-17T17:08:14.667 INFO:tasks.rados.rados.0.smithi063.stdout:3194: left oid 13 (ObjNum 1199 snap 403 seq_num 1199) 2024-08-17T17:08:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:14 smithi063 ceph-mon[114756]: osdmap e866: 8 total, 8 up, 8 in 2024-08-17T17:08:15.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:14 smithi063 ceph-mon[109843]: osdmap e866: 8 total, 8 up, 8 in 2024-08-17T17:08:15.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:14 smithi162 ceph-mon[104668]: osdmap e866: 8 total, 8 up, 8 in 2024-08-17T17:08:15.657 INFO:tasks.rados.rados.0.smithi063.stdout:3194: done (1 left) 2024-08-17T17:08:15.657 INFO:tasks.rados.rados.0.smithi063.stdout:3195: done (0 left) 2024-08-17T17:08:15.657 INFO:tasks.rados.rados.0.smithi063.stdout:3196: write oid 38 current snap is 404 2024-08-17T17:08:15.657 INFO:tasks.rados.rados.0.smithi063.stdout:3196: seq_num 1200 ranges {568400=566221,1838039=330361} 2024-08-17T17:08:15.663 INFO:tasks.rados.rados.0.smithi063.stdout:3196: writing smithi06387500-38 from 568400 to 1134621 tid 1 2024-08-17T17:08:15.669 INFO:tasks.rados.rados.0.smithi063.stdout:3196: writing smithi06387500-38 from 1838039 to 2168400 tid 2 2024-08-17T17:08:15.669 INFO:tasks.rados.rados.0.smithi063.stdout:3197: write oid 18 current snap is 404 2024-08-17T17:08:15.669 INFO:tasks.rados.rados.0.smithi063.stdout:3197: seq_num 1201 ranges {585207=641470,1896521=288686} 2024-08-17T17:08:15.675 INFO:tasks.rados.rados.0.smithi063.stdout:3197: writing smithi06387500-18 from 585207 to 1226677 tid 1 2024-08-17T17:08:15.682 INFO:tasks.rados.rados.0.smithi063.stdout:3197: writing smithi06387500-18 from 1896521 to 2185207 tid 2 2024-08-17T17:08:15.682 INFO:tasks.rados.rados.0.smithi063.stdout:3198: rollback oid 40 current snap is 404 2024-08-17T17:08:15.682 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 40 to 378 2024-08-17T17:08:15.682 INFO:tasks.rados.rados.0.smithi063.stdout:3196: finishing write tid 1 to smithi06387500-38 2024-08-17T17:08:15.682 INFO:tasks.rados.rados.0.smithi063.stdout:3199: read oid 22 snap 398 2024-08-17T17:08:15.682 INFO:tasks.rados.rados.0.smithi063.stdout:3199: expect deleted 2024-08-17T17:08:15.682 INFO:tasks.rados.rados.0.smithi063.stdout:3200: read oid 12 snap -1 2024-08-17T17:08:15.682 INFO:tasks.rados.rados.0.smithi063.stdout:3200: expect (ObjNum 1166 snap 393 seq_num 1166) 2024-08-17T17:08:15.683 INFO:tasks.rados.rados.0.smithi063.stdout:3201: rollback oid 41 current snap is 404 2024-08-17T17:08:15.683 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 41 to 372 2024-08-17T17:08:15.683 INFO:tasks.rados.rados.0.smithi063.stdout:3196: finishing write tid 2 to smithi06387500-38 2024-08-17T17:08:15.683 INFO:tasks.rados.rados.0.smithi063.stdout:3196: finishing write tid 3 to smithi06387500-38 2024-08-17T17:08:15.683 INFO:tasks.rados.rados.0.smithi063.stdout:3196: finishing write tid 4 to smithi06387500-38 2024-08-17T17:08:15.683 INFO:tasks.rados.rados.0.smithi063.stdout:3196: oid 38 updating version 0 to 545 2024-08-17T17:08:15.683 INFO:tasks.rados.rados.0.smithi063.stdout:3196: oid 38 updating version 545 to 547 2024-08-17T17:08:15.683 INFO:tasks.rados.rados.0.smithi063.stdout:3196: oid 38 version 547 is already newer than 546 2024-08-17T17:08:15.683 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 547 (ObjNum 1200 snap 404 seq_num 1200) dirty exists 2024-08-17T17:08:15.683 INFO:tasks.rados.rados.0.smithi063.stdout:3196: left oid 38 (ObjNum 1200 snap 404 seq_num 1200) 2024-08-17T17:08:15.683 INFO:tasks.rados.rados.0.smithi063.stdout:3197: finishing write tid 1 to smithi06387500-18 2024-08-17T17:08:15.683 INFO:tasks.rados.rados.0.smithi063.stdout:3196: done (5 left) 2024-08-17T17:08:15.683 INFO:tasks.rados.rados.0.smithi063.stdout:3202: write oid 1 current snap is 404 2024-08-17T17:08:15.683 INFO:tasks.rados.rados.0.smithi063.stdout:3202: seq_num 1202 ranges {602014=433072,1671356=530658} 2024-08-17T17:08:15.688 INFO:tasks.rados.rados.0.smithi063.stdout:3202: writing smithi06387500-1 from 602014 to 1035086 tid 1 2024-08-17T17:08:15.694 INFO:tasks.rados.rados.0.smithi063.stdout:3202: writing smithi06387500-1 from 1671356 to 2202014 tid 2 2024-08-17T17:08:15.695 INFO:tasks.rados.rados.0.smithi063.stdout:3203: delete oid 17 current snap is 404 2024-08-17T17:08:15.695 INFO:tasks.rados.rados.0.smithi063.stdout:3201: finishing rollback tid 1 to smithi06387500-41 2024-08-17T17:08:15.695 INFO:tasks.rados.rados.0.smithi063.stdout:3201: finishing rollback tid 2 to smithi06387500-41 2024-08-17T17:08:15.695 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 362 (ObjNum 1091 snap 363 seq_num 1091) dirty exists 2024-08-17T17:08:15.695 INFO:tasks.rados.rados.0.smithi063.stdout:3200: expect (ObjNum 1166 snap 393 seq_num 1166) 2024-08-17T17:08:15.816 INFO:tasks.rados.rados.0.smithi063.stdout:3197: finishing write tid 2 to smithi06387500-18 2024-08-17T17:08:15.816 INFO:tasks.rados.rados.0.smithi063.stdout:3197: finishing write tid 3 to smithi06387500-18 2024-08-17T17:08:15.816 INFO:tasks.rados.rados.0.smithi063.stdout:3197: finishing write tid 4 to smithi06387500-18 2024-08-17T17:08:15.816 INFO:tasks.rados.rados.0.smithi063.stdout:3197: oid 18 updating version 0 to 1024 2024-08-17T17:08:15.816 INFO:tasks.rados.rados.0.smithi063.stdout:3197: oid 18 version 1024 is already newer than 1022 2024-08-17T17:08:15.816 INFO:tasks.rados.rados.0.smithi063.stdout:3197: oid 18 version 1024 is already newer than 1023 2024-08-17T17:08:15.816 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 1024 (ObjNum 1201 snap 404 seq_num 1201) dirty exists 2024-08-17T17:08:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:3197: left oid 18 (ObjNum 1201 snap 404 seq_num 1201) 2024-08-17T17:08:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:3198: finishing rollback tid 0 to smithi06387500-40 2024-08-17T17:08:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:3197: done (6 left) 2024-08-17T17:08:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:3199: done (5 left) 2024-08-17T17:08:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:3200: done (4 left) 2024-08-17T17:08:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:3201: done (3 left) 2024-08-17T17:08:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:3203: done (2 left) 2024-08-17T17:08:15.817 INFO:tasks.rados.rados.0.smithi063.stdout:3204: snap_remove snap 374 2024-08-17T17:08:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:15 smithi063 ceph-mon[114756]: pgmap v756: 89 pgs: 89 active+clean; 362 MiB data, 988 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-08-17T17:08:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:15 smithi063 ceph-mon[114756]: osdmap e867: 8 total, 8 up, 8 in 2024-08-17T17:08:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:15 smithi063 ceph-mon[109843]: pgmap v756: 89 pgs: 89 active+clean; 362 MiB data, 988 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-08-17T17:08:16.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:15 smithi063 ceph-mon[109843]: osdmap e867: 8 total, 8 up, 8 in 2024-08-17T17:08:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:15 smithi162 ceph-mon[104668]: pgmap v756: 89 pgs: 89 active+clean; 362 MiB data, 988 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-08-17T17:08:16.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:15 smithi162 ceph-mon[104668]: osdmap e867: 8 total, 8 up, 8 in 2024-08-17T17:08:16.660 INFO:tasks.rados.rados.0.smithi063.stdout:3204: done (2 left) 2024-08-17T17:08:16.660 INFO:tasks.rados.rados.0.smithi063.stdout:3205: rollback oid 27 current snap is 404 2024-08-17T17:08:16.661 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 27 to 354 2024-08-17T17:08:16.661 INFO:tasks.rados.rados.0.smithi063.stdout:3198: finishing rollback tid 1 to smithi06387500-40 2024-08-17T17:08:16.661 INFO:tasks.rados.rados.0.smithi063.stdout:3198: finishing rollback tid 2 to smithi06387500-40 2024-08-17T17:08:16.661 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 1028 (ObjNum 1029 snap 350 seq_num 1029) dirty exists 2024-08-17T17:08:16.661 INFO:tasks.rados.rados.0.smithi063.stdout:3202: finishing write tid 1 to smithi06387500-1 2024-08-17T17:08:16.661 INFO:tasks.rados.rados.0.smithi063.stdout:3202: finishing write tid 2 to smithi06387500-1 2024-08-17T17:08:16.661 INFO:tasks.rados.rados.0.smithi063.stdout:3202: finishing write tid 3 to smithi06387500-1 2024-08-17T17:08:16.661 INFO:tasks.rados.rados.0.smithi063.stdout:3202: finishing write tid 4 to smithi06387500-1 2024-08-17T17:08:16.661 INFO:tasks.rados.rados.0.smithi063.stdout:3202: oid 1 updating version 0 to 1583 2024-08-17T17:08:16.661 INFO:tasks.rados.rados.0.smithi063.stdout:3202: oid 1 version 1583 is already newer than 1582 2024-08-17T17:08:16.661 INFO:tasks.rados.rados.0.smithi063.stdout:3202: oid 1 updating version 1583 to 1584 2024-08-17T17:08:16.661 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 1584 (ObjNum 1202 snap 404 seq_num 1202) dirty exists 2024-08-17T17:08:16.661 INFO:tasks.rados.rados.0.smithi063.stdout:3202: left oid 1 (ObjNum 1202 snap 404 seq_num 1202) 2024-08-17T17:08:16.662 INFO:tasks.rados.rados.0.smithi063.stdout:3198: done (2 left) 2024-08-17T17:08:16.662 INFO:tasks.rados.rados.0.smithi063.stdout:3202: done (1 left) 2024-08-17T17:08:16.662 INFO:tasks.rados.rados.0.smithi063.stdout:3206: read oid 20 snap -1 2024-08-17T17:08:16.662 INFO:tasks.rados.rados.0.smithi063.stdout:3206: expect deleted 2024-08-17T17:08:16.662 INFO:tasks.rados.rados.0.smithi063.stdout:3207: delete oid 9 current snap is 404 2024-08-17T17:08:16.668 INFO:tasks.rados.rados.0.smithi063.stdout:3205: finishing rollback tid 0 to smithi06387500-27 2024-08-17T17:08:16.668 INFO:tasks.rados.rados.0.smithi063.stdout:3205: finishing rollback tid 1 to smithi06387500-27 2024-08-17T17:08:16.668 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 289 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-08-17T17:08:16.675 INFO:tasks.rados.rados.0.smithi063.stdout:3205: done (2 left) 2024-08-17T17:08:16.675 INFO:tasks.rados.rados.0.smithi063.stdout:3206: done (1 left) 2024-08-17T17:08:16.675 INFO:tasks.rados.rados.0.smithi063.stdout:3207: done (0 left) 2024-08-17T17:08:16.675 INFO:tasks.rados.rados.0.smithi063.stdout:3208: snap_remove snap 403 2024-08-17T17:08:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:16 smithi063 ceph-mon[114756]: osdmap e868: 8 total, 8 up, 8 in 2024-08-17T17:08:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:16 smithi063 ceph-mon[109843]: osdmap e868: 8 total, 8 up, 8 in 2024-08-17T17:08:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:16 smithi162 ceph-mon[104668]: osdmap e868: 8 total, 8 up, 8 in 2024-08-17T17:08:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:17.637 INFO:tasks.rados.rados.0.smithi063.stdout:3208: done (0 left) 2024-08-17T17:08:17.637 INFO:tasks.rados.rados.0.smithi063.stdout:3209: read oid 21 snap -1 2024-08-17T17:08:17.637 INFO:tasks.rados.rados.0.smithi063.stdout:3209: expect (ObjNum 1153 snap 389 seq_num 1153) 2024-08-17T17:08:17.637 INFO:tasks.rados.rados.0.smithi063.stdout:3210: snap_remove snap 372 2024-08-17T17:08:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:17 smithi063 ceph-mon[114756]: pgmap v759: 89 pgs: 89 active+clean; 358 MiB data, 986 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 491 KiB/s wr, 2 op/s 2024-08-17T17:08:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:17 smithi063 ceph-mon[114756]: osdmap e869: 8 total, 8 up, 8 in 2024-08-17T17:08:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:17 smithi063 ceph-mon[114756]: osdmap e870: 8 total, 8 up, 8 in 2024-08-17T17:08:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:17 smithi063 ceph-mon[109843]: pgmap v759: 89 pgs: 89 active+clean; 358 MiB data, 986 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 491 KiB/s wr, 2 op/s 2024-08-17T17:08:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:17 smithi063 ceph-mon[109843]: osdmap e869: 8 total, 8 up, 8 in 2024-08-17T17:08:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:17 smithi063 ceph-mon[109843]: osdmap e870: 8 total, 8 up, 8 in 2024-08-17T17:08:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:17 smithi162 ceph-mon[104668]: pgmap v759: 89 pgs: 89 active+clean; 358 MiB data, 986 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 491 KiB/s wr, 2 op/s 2024-08-17T17:08:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:17 smithi162 ceph-mon[104668]: osdmap e869: 8 total, 8 up, 8 in 2024-08-17T17:08:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:17 smithi162 ceph-mon[104668]: osdmap e870: 8 total, 8 up, 8 in 2024-08-17T17:08:18.639 INFO:tasks.rados.rados.0.smithi063.stdout:3210: done (1 left) 2024-08-17T17:08:18.640 INFO:tasks.rados.rados.0.smithi063.stdout:3211: write oid 13 current snap is 404 2024-08-17T17:08:18.640 INFO:tasks.rados.rados.0.smithi063.stdout:3211: seq_num 1205 ranges {652435=658819,1730449=521986} 2024-08-17T17:08:18.647 INFO:tasks.rados.rados.0.smithi063.stdout:3211: writing smithi06387500-13 from 652435 to 1311254 tid 1 2024-08-17T17:08:18.654 INFO:tasks.rados.rados.0.smithi063.stdout:3211: writing smithi06387500-13 from 1730449 to 2252435 tid 2 2024-08-17T17:08:18.654 INFO:tasks.rados.rados.0.smithi063.stdout:3212: delete oid 31 current snap is 404 2024-08-17T17:08:18.654 INFO:tasks.rados.rados.0.smithi063.stdout:3209: expect (ObjNum 1153 snap 389 seq_num 1153) 2024-08-17T17:08:18.856 INFO:tasks.rados.rados.0.smithi063.stdout:3211: finishing write tid 1 to smithi06387500-13 2024-08-17T17:08:18.856 INFO:tasks.rados.rados.0.smithi063.stdout:3211: finishing write tid 2 to smithi06387500-13 2024-08-17T17:08:18.856 INFO:tasks.rados.rados.0.smithi063.stdout:3209: done (2 left) 2024-08-17T17:08:18.856 INFO:tasks.rados.rados.0.smithi063.stdout:3212: done (1 left) 2024-08-17T17:08:18.856 INFO:tasks.rados.rados.0.smithi063.stdout:3213: write oid 30 current snap is 404 2024-08-17T17:08:18.856 INFO:tasks.rados.rados.0.smithi063.stdout:3211: finishing write tid 3 to smithi06387500-13 2024-08-17T17:08:18.857 INFO:tasks.rados.rados.0.smithi063.stdout:3211: finishing write tid 4 to smithi06387500-13 2024-08-17T17:08:18.857 INFO:tasks.rados.rados.0.smithi063.stdout:3211: oid 13 updating version 0 to 1597 2024-08-17T17:08:18.857 INFO:tasks.rados.rados.0.smithi063.stdout:3211: oid 13 version 1597 is already newer than 1596 2024-08-17T17:08:18.857 INFO:tasks.rados.rados.0.smithi063.stdout:3211: oid 13 version 1597 is already newer than 1595 2024-08-17T17:08:18.857 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1597 (ObjNum 1205 snap 404 seq_num 1205) dirty exists 2024-08-17T17:08:18.857 INFO:tasks.rados.rados.0.smithi063.stdout:3211: left oid 13 (ObjNum 1205 snap 404 seq_num 1205) 2024-08-17T17:08:18.857 INFO:tasks.rados.rados.0.smithi063.stdout:3213: seq_num 1207 ranges {686049=409317,1847413=438636} 2024-08-17T17:08:18.861 INFO:tasks.rados.rados.0.smithi063.stdout:3213: writing smithi06387500-30 from 686049 to 1095366 tid 1 2024-08-17T17:08:18.868 INFO:tasks.rados.rados.0.smithi063.stdout:3213: writing smithi06387500-30 from 1847413 to 2286049 tid 2 2024-08-17T17:08:18.868 INFO:tasks.rados.rados.0.smithi063.stdout:3211: done (1 left) 2024-08-17T17:08:18.869 INFO:tasks.rados.rados.0.smithi063.stdout:3214: rollback oid 19 current snap is 404 2024-08-17T17:08:18.869 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 19 to 400 2024-08-17T17:08:18.869 INFO:tasks.rados.rados.0.smithi063.stdout:3213: finishing write tid 1 to smithi06387500-30 2024-08-17T17:08:18.869 INFO:tasks.rados.rados.0.smithi063.stdout:3215: read oid 8 snap -1 2024-08-17T17:08:18.869 INFO:tasks.rados.rados.0.smithi063.stdout:3215: expect deleted 2024-08-17T17:08:18.869 INFO:tasks.rados.rados.0.smithi063.stdout:3216: write oid 32 current snap is 404 2024-08-17T17:08:18.869 INFO:tasks.rados.rados.0.smithi063.stdout:3216: seq_num 1208 ranges {702856=484566,1789542=513314} 2024-08-17T17:08:18.875 INFO:tasks.rados.rados.0.smithi063.stdout:3216: writing smithi06387500-32 from 702856 to 1187422 tid 1 2024-08-17T17:08:18.881 INFO:tasks.rados.rados.0.smithi063.stdout:3216: writing smithi06387500-32 from 1789542 to 2302856 tid 2 2024-08-17T17:08:18.881 INFO:tasks.rados.rados.0.smithi063.stdout:3217: read oid 41 snap -1 2024-08-17T17:08:18.881 INFO:tasks.rados.rados.0.smithi063.stdout:3217: expect (ObjNum 1091 snap 363 seq_num 1091) 2024-08-17T17:08:18.881 INFO:tasks.rados.rados.0.smithi063.stdout:3218: snap_remove snap 383 2024-08-17T17:08:19.666 INFO:tasks.rados.rados.0.smithi063.stdout:3218: done (5 left) 2024-08-17T17:08:19.666 INFO:tasks.rados.rados.0.smithi063.stdout:3219: delete oid 3 current snap is 404 2024-08-17T17:08:19.666 INFO:tasks.rados.rados.0.smithi063.stdout:3213: finishing write tid 2 to smithi06387500-30 2024-08-17T17:08:19.666 INFO:tasks.rados.rados.0.smithi063.stdout:3213: finishing write tid 3 to smithi06387500-30 2024-08-17T17:08:19.666 INFO:tasks.rados.rados.0.smithi063.stdout:3214: finishing rollback tid 0 to smithi06387500-19 2024-08-17T17:08:19.667 INFO:tasks.rados.rados.0.smithi063.stdout:3213: finishing write tid 4 to smithi06387500-30 2024-08-17T17:08:19.667 INFO:tasks.rados.rados.0.smithi063.stdout:3213: oid 30 updating version 0 to 1505 2024-08-17T17:08:19.667 INFO:tasks.rados.rados.0.smithi063.stdout:3213: oid 30 updating version 1505 to 1506 2024-08-17T17:08:19.667 INFO:tasks.rados.rados.0.smithi063.stdout:3213: oid 30 updating version 1506 to 1507 2024-08-17T17:08:19.667 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1507 (ObjNum 1207 snap 404 seq_num 1207) dirty exists 2024-08-17T17:08:19.667 INFO:tasks.rados.rados.0.smithi063.stdout:3213: left oid 30 (ObjNum 1207 snap 404 seq_num 1207) 2024-08-17T17:08:19.667 INFO:tasks.rados.rados.0.smithi063.stdout:3214: finishing rollback tid 1 to smithi06387500-19 2024-08-17T17:08:19.667 INFO:tasks.rados.rados.0.smithi063.stdout:3214: finishing rollback tid 2 to smithi06387500-19 2024-08-17T17:08:19.667 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 2019 (ObjNum 1190 snap 400 seq_num 1190) dirty exists 2024-08-17T17:08:19.667 INFO:tasks.rados.rados.0.smithi063.stdout:3216: finishing write tid 1 to smithi06387500-32 2024-08-17T17:08:19.667 INFO:tasks.rados.rados.0.smithi063.stdout:3216: finishing write tid 2 to smithi06387500-32 2024-08-17T17:08:19.667 INFO:tasks.rados.rados.0.smithi063.stdout:3216: finishing write tid 3 to smithi06387500-32 2024-08-17T17:08:19.667 INFO:tasks.rados.rados.0.smithi063.stdout:3216: finishing write tid 4 to smithi06387500-32 2024-08-17T17:08:19.668 INFO:tasks.rados.rados.0.smithi063.stdout:3216: oid 32 updating version 0 to 671 2024-08-17T17:08:19.668 INFO:tasks.rados.rados.0.smithi063.stdout:3216: oid 32 version 671 is already newer than 670 2024-08-17T17:08:19.668 INFO:tasks.rados.rados.0.smithi063.stdout:3216: oid 32 updating version 671 to 672 2024-08-17T17:08:19.668 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 672 (ObjNum 1208 snap 404 seq_num 1208) dirty exists 2024-08-17T17:08:19.668 INFO:tasks.rados.rados.0.smithi063.stdout:3216: left oid 32 (ObjNum 1208 snap 404 seq_num 1208) 2024-08-17T17:08:19.668 INFO:tasks.rados.rados.0.smithi063.stdout:3217: expect (ObjNum 1091 snap 363 seq_num 1091) 2024-08-17T17:08:19.749 INFO:tasks.rados.rados.0.smithi063.stdout:3213: done (5 left) 2024-08-17T17:08:19.750 INFO:tasks.rados.rados.0.smithi063.stdout:3214: done (4 left) 2024-08-17T17:08:19.750 INFO:tasks.rados.rados.0.smithi063.stdout:3215: done (3 left) 2024-08-17T17:08:19.750 INFO:tasks.rados.rados.0.smithi063.stdout:3216: done (2 left) 2024-08-17T17:08:19.750 INFO:tasks.rados.rados.0.smithi063.stdout:3217: done (1 left) 2024-08-17T17:08:19.750 INFO:tasks.rados.rados.0.smithi063.stdout:3219: done (0 left) 2024-08-17T17:08:19.750 INFO:tasks.rados.rados.0.smithi063.stdout:3220: write oid 50 current snap is 404 2024-08-17T17:08:19.750 INFO:tasks.rados.rados.0.smithi063.stdout:3220: seq_num 1210 ranges {736470=751417,2022859=313611} 2024-08-17T17:08:19.757 INFO:tasks.rados.rados.0.smithi063.stdout:3220: writing smithi06387500-50 from 736470 to 1487887 tid 1 2024-08-17T17:08:19.762 INFO:tasks.rados.rados.0.smithi063.stdout:3220: writing smithi06387500-50 from 2022859 to 2336470 tid 2 2024-08-17T17:08:19.762 INFO:tasks.rados.rados.0.smithi063.stdout:3221: snap_create 2024-08-17T17:08:19.762 INFO:tasks.rados.rados.0.smithi063.stdout:3220: finishing write tid 1 to smithi06387500-50 2024-08-17T17:08:19.765 INFO:tasks.rados.rados.0.smithi063.stdout:3220: finishing write tid 2 to smithi06387500-50 2024-08-17T17:08:19.765 INFO:tasks.rados.rados.0.smithi063.stdout:3220: finishing write tid 3 to smithi06387500-50 2024-08-17T17:08:19.766 INFO:tasks.rados.rados.0.smithi063.stdout:3220: finishing write tid 4 to smithi06387500-50 2024-08-17T17:08:19.766 INFO:tasks.rados.rados.0.smithi063.stdout:3220: oid 50 updating version 0 to 2029 2024-08-17T17:08:19.766 INFO:tasks.rados.rados.0.smithi063.stdout:3220: oid 50 version 2029 is already newer than 2027 2024-08-17T17:08:19.766 INFO:tasks.rados.rados.0.smithi063.stdout:3220: oid 50 version 2029 is already newer than 2028 2024-08-17T17:08:19.766 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 2029 (ObjNum 1210 snap 404 seq_num 1210) dirty exists 2024-08-17T17:08:19.766 INFO:tasks.rados.rados.0.smithi063.stdout:3220: left oid 50 (ObjNum 1210 snap 404 seq_num 1210) 2024-08-17T17:08:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:19 smithi063 ceph-mon[114756]: pgmap v762: 89 pgs: 89 active+clean; 364 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.4 MiB/s wr, 9 op/s 2024-08-17T17:08:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:19 smithi063 ceph-mon[114756]: osdmap e871: 8 total, 8 up, 8 in 2024-08-17T17:08:20.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:19 smithi063 ceph-mon[109843]: pgmap v762: 89 pgs: 89 active+clean; 364 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.4 MiB/s wr, 9 op/s 2024-08-17T17:08:20.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:19 smithi063 ceph-mon[109843]: osdmap e871: 8 total, 8 up, 8 in 2024-08-17T17:08:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:19 smithi162 ceph-mon[104668]: pgmap v762: 89 pgs: 89 active+clean; 364 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.4 MiB/s wr, 9 op/s 2024-08-17T17:08:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:19 smithi162 ceph-mon[104668]: osdmap e871: 8 total, 8 up, 8 in 2024-08-17T17:08:20.672 INFO:tasks.rados.rados.0.smithi063.stdout:3220: done (1 left) 2024-08-17T17:08:20.672 INFO:tasks.rados.rados.0.smithi063.stdout:3221: done (0 left) 2024-08-17T17:08:20.672 INFO:tasks.rados.rados.0.smithi063.stdout:3222: rollback oid 11 current snap is 405 2024-08-17T17:08:20.672 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 11 to 354 2024-08-17T17:08:20.673 INFO:tasks.rados.rados.0.smithi063.stdout:3223: read oid 22 snap 386 2024-08-17T17:08:20.673 INFO:tasks.rados.rados.0.smithi063.stdout:3223: expect deleted 2024-08-17T17:08:20.673 INFO:tasks.rados.rados.0.smithi063.stdout:3224: snap_remove snap 377 2024-08-17T17:08:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:20 smithi063 ceph-mon[114756]: osdmap e872: 8 total, 8 up, 8 in 2024-08-17T17:08:21.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:20 smithi063 ceph-mon[109843]: osdmap e872: 8 total, 8 up, 8 in 2024-08-17T17:08:21.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:20 smithi162 ceph-mon[104668]: osdmap e872: 8 total, 8 up, 8 in 2024-08-17T17:08:21.674 INFO:tasks.rados.rados.0.smithi063.stdout:3224: done (2 left) 2024-08-17T17:08:21.674 INFO:tasks.rados.rados.0.smithi063.stdout:3225: snap_remove snap 371 2024-08-17T17:08:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:21 smithi063 ceph-mon[114756]: pgmap v765: 89 pgs: 89 active+clean; 364 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 949 KiB/s wr, 6 op/s 2024-08-17T17:08:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:21 smithi063 ceph-mon[114756]: osdmap e873: 8 total, 8 up, 8 in 2024-08-17T17:08:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:21 smithi063 ceph-mon[109843]: pgmap v765: 89 pgs: 89 active+clean; 364 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 949 KiB/s wr, 6 op/s 2024-08-17T17:08:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:21 smithi063 ceph-mon[109843]: osdmap e873: 8 total, 8 up, 8 in 2024-08-17T17:08:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:21 smithi162 ceph-mon[104668]: pgmap v765: 89 pgs: 89 active+clean; 364 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 949 KiB/s wr, 6 op/s 2024-08-17T17:08:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:21 smithi162 ceph-mon[104668]: osdmap e873: 8 total, 8 up, 8 in 2024-08-17T17:08:22.687 INFO:tasks.rados.rados.0.smithi063.stdout:3225: done (2 left) 2024-08-17T17:08:22.688 INFO:tasks.rados.rados.0.smithi063.stdout:3226: write oid 10 current snap is 405 2024-08-17T17:08:22.688 INFO:tasks.rados.rados.0.smithi063.stdout:3226: seq_num 1211 ranges {753277=426666,1681341=545753} 2024-08-17T17:08:22.694 INFO:tasks.rados.rados.0.smithi063.stdout:3226: writing smithi06387500-10 from 753277 to 1179943 tid 1 2024-08-17T17:08:22.701 INFO:tasks.rados.rados.0.smithi063.stdout:3226: writing smithi06387500-10 from 1681341 to 2227094 tid 2 2024-08-17T17:08:22.701 INFO:tasks.rados.rados.0.smithi063.stdout:3227: snap_create 2024-08-17T17:08:22.701 INFO:tasks.rados.rados.0.smithi063.stdout:3222: finishing rollback tid 0 to smithi06387500-11 2024-08-17T17:08:22.701 INFO:tasks.rados.rados.0.smithi063.stdout:3222: finishing rollback tid 1 to smithi06387500-11 2024-08-17T17:08:22.702 INFO:tasks.rados.rados.0.smithi063.stdout:3222: finishing rollback tid 2 to smithi06387500-11 2024-08-17T17:08:22.702 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 561 (ObjNum 1024 snap 350 seq_num 1024) dirty exists 2024-08-17T17:08:22.702 INFO:tasks.rados.rados.0.smithi063.stdout:3226: finishing write tid 1 to smithi06387500-10 2024-08-17T17:08:22.705 INFO:tasks.rados.rados.0.smithi063.stdout:3226: finishing write tid 2 to smithi06387500-10 2024-08-17T17:08:22.705 INFO:tasks.rados.rados.0.smithi063.stdout:3226: finishing write tid 3 to smithi06387500-10 2024-08-17T17:08:22.705 INFO:tasks.rados.rados.0.smithi063.stdout:3226: finishing write tid 4 to smithi06387500-10 2024-08-17T17:08:22.705 INFO:tasks.rados.rados.0.smithi063.stdout:3226: oid 10 updating version 0 to 1048 2024-08-17T17:08:22.706 INFO:tasks.rados.rados.0.smithi063.stdout:3226: oid 10 updating version 1048 to 1050 2024-08-17T17:08:22.706 INFO:tasks.rados.rados.0.smithi063.stdout:3226: oid 10 version 1050 is already newer than 1049 2024-08-17T17:08:22.706 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 1050 (ObjNum 1211 snap 405 seq_num 1211) dirty exists 2024-08-17T17:08:22.706 INFO:tasks.rados.rados.0.smithi063.stdout:3226: left oid 10 (ObjNum 1211 snap 405 seq_num 1211) 2024-08-17T17:08:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:22 smithi063 ceph-mon[114756]: osdmap e874: 8 total, 8 up, 8 in 2024-08-17T17:08:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:22 smithi063 ceph-mon[114756]: pgmap v768: 89 pgs: 89 active+clean; 367 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 975 KiB/s wr, 12 op/s 2024-08-17T17:08:23.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:22 smithi063 ceph-mon[109843]: osdmap e874: 8 total, 8 up, 8 in 2024-08-17T17:08:23.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:22 smithi063 ceph-mon[109843]: pgmap v768: 89 pgs: 89 active+clean; 367 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 975 KiB/s wr, 12 op/s 2024-08-17T17:08:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:22 smithi162 ceph-mon[104668]: osdmap e874: 8 total, 8 up, 8 in 2024-08-17T17:08:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:22 smithi162 ceph-mon[104668]: pgmap v768: 89 pgs: 89 active+clean; 367 MiB data, 1008 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 975 KiB/s wr, 12 op/s 2024-08-17T17:08:23.676 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:08:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:08:23] "GET /metrics HTTP/1.1" 200 36341 "" "Prometheus/2.51.0" 2024-08-17T17:08:23.689 INFO:tasks.rados.rados.0.smithi063.stdout:3222: done (3 left) 2024-08-17T17:08:23.690 INFO:tasks.rados.rados.0.smithi063.stdout:3223: done (2 left) 2024-08-17T17:08:23.690 INFO:tasks.rados.rados.0.smithi063.stdout:3226: done (1 left) 2024-08-17T17:08:23.690 INFO:tasks.rados.rados.0.smithi063.stdout:3227: done (0 left) 2024-08-17T17:08:23.690 INFO:tasks.rados.rados.0.smithi063.stdout:3228: write oid 2 current snap is 406 2024-08-17T17:08:23.690 INFO:tasks.rados.rados.0.smithi063.stdout:3228: seq_num 1212 ranges {770084=501915,2023470=346614} 2024-08-17T17:08:23.696 INFO:tasks.rados.rados.0.smithi063.stdout:3228: writing smithi06387500-2 from 770084 to 1271999 tid 1 2024-08-17T17:08:23.702 INFO:tasks.rados.rados.0.smithi063.stdout:3228: writing smithi06387500-2 from 2023470 to 2370084 tid 2 2024-08-17T17:08:23.702 INFO:tasks.rados.rados.0.smithi063.stdout:3229: read oid 41 snap 398 2024-08-17T17:08:23.703 INFO:tasks.rados.rados.0.smithi063.stdout:3229: expect deleted 2024-08-17T17:08:23.703 INFO:tasks.rados.rados.0.smithi063.stdout:3230: read oid 12 snap -1 2024-08-17T17:08:23.703 INFO:tasks.rados.rados.0.smithi063.stdout:3230: expect (ObjNum 1166 snap 393 seq_num 1166) 2024-08-17T17:08:23.703 INFO:tasks.rados.rados.0.smithi063.stdout:3231: snap_remove snap 387 2024-08-17T17:08:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:23 smithi063 ceph-mon[109843]: osdmap e875: 8 total, 8 up, 8 in 2024-08-17T17:08:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:23 smithi063 ceph-mon[114756]: osdmap e875: 8 total, 8 up, 8 in 2024-08-17T17:08:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:23 smithi162 ceph-mon[104668]: osdmap e875: 8 total, 8 up, 8 in 2024-08-17T17:08:24.697 INFO:tasks.rados.rados.0.smithi063.stdout:3231: done (3 left) 2024-08-17T17:08:24.697 INFO:tasks.rados.rados.0.smithi063.stdout:3232: write oid 23 current snap is 406 2024-08-17T17:08:24.697 INFO:tasks.rados.rados.0.smithi063.stdout:3228: finishing write tid 1 to smithi06387500-2 2024-08-17T17:08:24.697 INFO:tasks.rados.rados.0.smithi063.stdout:3232: seq_num 1213 ranges {786891=577164,2081952=304939} 2024-08-17T17:08:24.705 INFO:tasks.rados.rados.0.smithi063.stdout:3232: writing smithi06387500-23 from 786891 to 1364055 tid 1 2024-08-17T17:08:24.712 INFO:tasks.rados.rados.0.smithi063.stdout:3232: writing smithi06387500-23 from 2081952 to 2386891 tid 2 2024-08-17T17:08:24.712 INFO:tasks.rados.rados.0.smithi063.stdout:3233: delete oid 27 current snap is 406 2024-08-17T17:08:24.713 INFO:tasks.rados.rados.0.smithi063.stdout:3228: finishing write tid 2 to smithi06387500-2 2024-08-17T17:08:24.713 INFO:tasks.rados.rados.0.smithi063.stdout:3228: finishing write tid 3 to smithi06387500-2 2024-08-17T17:08:24.713 INFO:tasks.rados.rados.0.smithi063.stdout:3228: finishing write tid 4 to smithi06387500-2 2024-08-17T17:08:24.713 INFO:tasks.rados.rados.0.smithi063.stdout:3228: oid 2 updating version 0 to 1614 2024-08-17T17:08:24.713 INFO:tasks.rados.rados.0.smithi063.stdout:3228: oid 2 updating version 1614 to 1616 2024-08-17T17:08:24.713 INFO:tasks.rados.rados.0.smithi063.stdout:3228: oid 2 version 1616 is already newer than 1615 2024-08-17T17:08:24.713 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 1616 (ObjNum 1212 snap 406 seq_num 1212) dirty exists 2024-08-17T17:08:24.713 INFO:tasks.rados.rados.0.smithi063.stdout:3228: left oid 2 (ObjNum 1212 snap 406 seq_num 1212) 2024-08-17T17:08:24.713 INFO:tasks.rados.rados.0.smithi063.stdout:3230: expect (ObjNum 1166 snap 393 seq_num 1166) 2024-08-17T17:08:24.834 INFO:tasks.rados.rados.0.smithi063.stdout:3232: finishing write tid 1 to smithi06387500-23 2024-08-17T17:08:24.834 INFO:tasks.rados.rados.0.smithi063.stdout:3232: finishing write tid 2 to smithi06387500-23 2024-08-17T17:08:24.834 INFO:tasks.rados.rados.0.smithi063.stdout:3232: finishing write tid 3 to smithi06387500-23 2024-08-17T17:08:24.834 INFO:tasks.rados.rados.0.smithi063.stdout:3232: finishing write tid 4 to smithi06387500-23 2024-08-17T17:08:24.834 INFO:tasks.rados.rados.0.smithi063.stdout:3232: oid 23 updating version 0 to 1628 2024-08-17T17:08:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:3232: oid 23 version 1628 is already newer than 1626 2024-08-17T17:08:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:3232: oid 23 version 1628 is already newer than 1627 2024-08-17T17:08:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 1628 (ObjNum 1213 snap 406 seq_num 1213) dirty exists 2024-08-17T17:08:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:3232: left oid 23 (ObjNum 1213 snap 406 seq_num 1213) 2024-08-17T17:08:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:3228: done (4 left) 2024-08-17T17:08:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:3229: done (3 left) 2024-08-17T17:08:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:3230: done (2 left) 2024-08-17T17:08:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:3232: done (1 left) 2024-08-17T17:08:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:3233: done (0 left) 2024-08-17T17:08:24.835 INFO:tasks.rados.rados.0.smithi063.stdout:3234: snap_remove snap 390 2024-08-17T17:08:25.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:24 smithi063 ceph-mon[114756]: osdmap e876: 8 total, 8 up, 8 in 2024-08-17T17:08:25.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:24 smithi063 ceph-mon[114756]: pgmap v771: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 368 MiB data, 1011 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-08-17T17:08:25.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:24 smithi063 ceph-mon[109843]: osdmap e876: 8 total, 8 up, 8 in 2024-08-17T17:08:25.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:24 smithi063 ceph-mon[109843]: pgmap v771: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 368 MiB data, 1011 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-08-17T17:08:25.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:24 smithi162 ceph-mon[104668]: osdmap e876: 8 total, 8 up, 8 in 2024-08-17T17:08:25.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:24 smithi162 ceph-mon[104668]: pgmap v771: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 368 MiB data, 1011 MiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 1.2 MiB/s wr, 13 op/s 2024-08-17T17:08:25.702 INFO:tasks.rados.rados.0.smithi063.stdout:3234: done (0 left) 2024-08-17T17:08:25.703 INFO:tasks.rados.rados.0.smithi063.stdout:3235: snap_create 2024-08-17T17:08:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:25 smithi063 ceph-mon[114756]: osdmap e877: 8 total, 8 up, 8 in 2024-08-17T17:08:26.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:25 smithi063 ceph-mon[109843]: osdmap e877: 8 total, 8 up, 8 in 2024-08-17T17:08:26.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:25 smithi162 ceph-mon[104668]: osdmap e877: 8 total, 8 up, 8 in 2024-08-17T17:08:26.715 INFO:tasks.rados.rados.0.smithi063.stdout:3235: done (0 left) 2024-08-17T17:08:26.715 INFO:tasks.rados.rados.0.smithi063.stdout:3236: write oid 30 current snap is 407 2024-08-17T17:08:26.715 INFO:tasks.rados.rados.0.smithi063.stdout:3236: seq_num 1215 ranges {420505=727662,1798916=553091} 2024-08-17T17:08:26.721 INFO:tasks.rados.rados.0.smithi063.stdout:3236: writing smithi06387500-30 from 420505 to 1148167 tid 1 2024-08-17T17:08:26.729 INFO:tasks.rados.rados.0.smithi063.stdout:3236: writing smithi06387500-30 from 1798916 to 2352007 tid 2 2024-08-17T17:08:26.729 INFO:tasks.rados.rados.0.smithi063.stdout:3237: read oid 44 snap 386 2024-08-17T17:08:26.729 INFO:tasks.rados.rados.0.smithi063.stdout:3237: expect (ObjNum 1124 snap 372 seq_num 1124) 2024-08-17T17:08:26.729 INFO:tasks.rados.rados.0.smithi063.stdout:3238: write oid 24 current snap is 407 2024-08-17T17:08:26.730 INFO:tasks.rados.rados.0.smithi063.stdout:3236: finishing write tid 1 to smithi06387500-30 2024-08-17T17:08:26.730 INFO:tasks.rados.rados.0.smithi063.stdout:3238: seq_num 1216 ranges {437312=402911,1341045=696749} 2024-08-17T17:08:26.734 INFO:tasks.rados.rados.0.smithi063.stdout:3238: writing smithi06387500-24 from 437312 to 840223 tid 1 2024-08-17T17:08:26.743 INFO:tasks.rados.rados.0.smithi063.stdout:3238: writing smithi06387500-24 from 1341045 to 2037794 tid 2 2024-08-17T17:08:26.743 INFO:tasks.rados.rados.0.smithi063.stdout:3239: read oid 36 snap -1 2024-08-17T17:08:26.743 INFO:tasks.rados.rados.0.smithi063.stdout:3239: expect deleted 2024-08-17T17:08:26.743 INFO:tasks.rados.rados.0.smithi063.stdout:3240: delete oid 16 current snap is 407 2024-08-17T17:08:26.743 INFO:tasks.rados.rados.0.smithi063.stdout:3236: finishing write tid 2 to smithi06387500-30 2024-08-17T17:08:26.744 INFO:tasks.rados.rados.0.smithi063.stdout:3236: finishing write tid 3 to smithi06387500-30 2024-08-17T17:08:26.744 INFO:tasks.rados.rados.0.smithi063.stdout:3236: finishing write tid 4 to smithi06387500-30 2024-08-17T17:08:26.744 INFO:tasks.rados.rados.0.smithi063.stdout:3236: oid 30 updating version 0 to 1539 2024-08-17T17:08:26.744 INFO:tasks.rados.rados.0.smithi063.stdout:3236: oid 30 updating version 1539 to 1541 2024-08-17T17:08:26.744 INFO:tasks.rados.rados.0.smithi063.stdout:3236: oid 30 version 1541 is already newer than 1540 2024-08-17T17:08:26.744 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1541 (ObjNum 1215 snap 407 seq_num 1215) dirty exists 2024-08-17T17:08:26.744 INFO:tasks.rados.rados.0.smithi063.stdout:3236: left oid 30 (ObjNum 1215 snap 407 seq_num 1215) 2024-08-17T17:08:26.744 INFO:tasks.rados.rados.0.smithi063.stdout:3238: finishing write tid 1 to smithi06387500-24 2024-08-17T17:08:26.744 INFO:tasks.rados.rados.0.smithi063.stdout:3237: expect (ObjNum 1124 snap 372 seq_num 1124) 2024-08-17T17:08:26.911 INFO:tasks.rados.rados.0.smithi063.stdout:3238: finishing write tid 2 to smithi06387500-24 2024-08-17T17:08:26.912 INFO:tasks.rados.rados.0.smithi063.stdout:3238: finishing write tid 3 to smithi06387500-24 2024-08-17T17:08:26.912 INFO:tasks.rados.rados.0.smithi063.stdout:3238: finishing write tid 4 to smithi06387500-24 2024-08-17T17:08:26.912 INFO:tasks.rados.rados.0.smithi063.stdout:3238: oid 24 updating version 0 to 1113 2024-08-17T17:08:26.912 INFO:tasks.rados.rados.0.smithi063.stdout:3238: oid 24 updating version 1113 to 1115 2024-08-17T17:08:26.912 INFO:tasks.rados.rados.0.smithi063.stdout:3238: oid 24 version 1115 is already newer than 1114 2024-08-17T17:08:26.913 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 1115 (ObjNum 1216 snap 407 seq_num 1216) dirty exists 2024-08-17T17:08:26.913 INFO:tasks.rados.rados.0.smithi063.stdout:3238: left oid 24 (ObjNum 1216 snap 407 seq_num 1216) 2024-08-17T17:08:26.913 INFO:tasks.rados.rados.0.smithi063.stdout:3236: done (4 left) 2024-08-17T17:08:26.913 INFO:tasks.rados.rados.0.smithi063.stdout:3237: done (3 left) 2024-08-17T17:08:26.913 INFO:tasks.rados.rados.0.smithi063.stdout:3238: done (2 left) 2024-08-17T17:08:26.914 INFO:tasks.rados.rados.0.smithi063.stdout:3239: done (1 left) 2024-08-17T17:08:26.914 INFO:tasks.rados.rados.0.smithi063.stdout:3240: done (0 left) 2024-08-17T17:08:26.914 INFO:tasks.rados.rados.0.smithi063.stdout:3241: snap_remove snap 359 2024-08-17T17:08:27.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:26 smithi063 ceph-mon[114756]: osdmap e878: 8 total, 8 up, 8 in 2024-08-17T17:08:27.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:26 smithi063 ceph-mon[114756]: pgmap v774: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 371 MiB data, 1016 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 481 KiB/s wr, 4 op/s 2024-08-17T17:08:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:26 smithi063 ceph-mon[109843]: osdmap e878: 8 total, 8 up, 8 in 2024-08-17T17:08:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:26 smithi063 ceph-mon[109843]: pgmap v774: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 371 MiB data, 1016 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 481 KiB/s wr, 4 op/s 2024-08-17T17:08:27.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:26 smithi162 ceph-mon[104668]: osdmap e878: 8 total, 8 up, 8 in 2024-08-17T17:08:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:26 smithi162 ceph-mon[104668]: pgmap v774: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 371 MiB data, 1016 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 481 KiB/s wr, 4 op/s 2024-08-17T17:08:27.640 INFO:tasks.rados.rados.0.smithi063.stdout:3241: done (0 left) 2024-08-17T17:08:27.640 INFO:tasks.rados.rados.0.smithi063.stdout:3242: write oid 44 current snap is 407 2024-08-17T17:08:27.640 INFO:tasks.rados.rados.0.smithi063.stdout:3242: seq_num 1218 ranges {470926=669762,1574362=700418} 2024-08-17T17:08:27.647 INFO:tasks.rados.rados.0.smithi063.stdout:3242: writing smithi06387500-44 from 470926 to 1140688 tid 1 2024-08-17T17:08:27.655 INFO:tasks.rados.rados.0.smithi063.stdout:3242: writing smithi06387500-44 from 1574362 to 2274780 tid 2 2024-08-17T17:08:27.655 INFO:tasks.rados.rados.0.smithi063.stdout:3243: write oid 17 current snap is 407 2024-08-17T17:08:27.655 INFO:tasks.rados.rados.0.smithi063.stdout:3243: seq_num 1219 ranges {487733=745011,1632844=560429} 2024-08-17T17:08:27.662 INFO:tasks.rados.rados.0.smithi063.stdout:3243: writing smithi06387500-17 from 487733 to 1232744 tid 1 2024-08-17T17:08:27.668 INFO:tasks.rados.rados.0.smithi063.stdout:3243: writing smithi06387500-17 from 1632844 to 2193273 tid 2 2024-08-17T17:08:27.668 INFO:tasks.rados.rados.0.smithi063.stdout:3244: read oid 28 snap -1 2024-08-17T17:08:27.668 INFO:tasks.rados.rados.0.smithi063.stdout:3244: expect (ObjNum 1111 snap 368 seq_num 1111) 2024-08-17T17:08:27.668 INFO:tasks.rados.rados.0.smithi063.stdout:3245: write oid 21 current snap is 407 2024-08-17T17:08:27.668 INFO:tasks.rados.rados.0.smithi063.stdout:3242: finishing write tid 1 to smithi06387500-44 2024-08-17T17:08:27.668 INFO:tasks.rados.rados.0.smithi063.stdout:3245: seq_num 1220 ranges {504540=420260,1574973=704087} 2024-08-17T17:08:27.673 INFO:tasks.rados.rados.0.smithi063.stdout:3245: writing smithi06387500-21 from 504540 to 924800 tid 1 2024-08-17T17:08:27.681 INFO:tasks.rados.rados.0.smithi063.stdout:3245: writing smithi06387500-21 from 1574973 to 2279060 tid 2 2024-08-17T17:08:27.681 INFO:tasks.rados.rados.0.smithi063.stdout:3246: write oid 8 current snap is 407 2024-08-17T17:08:27.681 INFO:tasks.rados.rados.0.smithi063.stdout:3242: finishing write tid 2 to smithi06387500-44 2024-08-17T17:08:27.681 INFO:tasks.rados.rados.0.smithi063.stdout:3242: finishing write tid 3 to smithi06387500-44 2024-08-17T17:08:27.681 INFO:tasks.rados.rados.0.smithi063.stdout:3242: finishing write tid 4 to smithi06387500-44 2024-08-17T17:08:27.681 INFO:tasks.rados.rados.0.smithi063.stdout:3242: oid 44 updating version 0 to 442 2024-08-17T17:08:27.681 INFO:tasks.rados.rados.0.smithi063.stdout:3242: oid 44 updating version 442 to 443 2024-08-17T17:08:27.682 INFO:tasks.rados.rados.0.smithi063.stdout:3242: oid 44 version 443 is already newer than 441 2024-08-17T17:08:27.682 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 443 (ObjNum 1218 snap 407 seq_num 1218) dirty exists 2024-08-17T17:08:27.682 INFO:tasks.rados.rados.0.smithi063.stdout:3242: left oid 44 (ObjNum 1218 snap 407 seq_num 1218) 2024-08-17T17:08:27.682 INFO:tasks.rados.rados.0.smithi063.stdout:3243: finishing write tid 1 to smithi06387500-17 2024-08-17T17:08:27.682 INFO:tasks.rados.rados.0.smithi063.stdout:3246: seq_num 1221 ranges {521347=495509,1633455=564098} 2024-08-17T17:08:27.686 INFO:tasks.rados.rados.0.smithi063.stdout:3246: writing smithi06387500-8 from 521347 to 1016856 tid 1 2024-08-17T17:08:27.693 INFO:tasks.rados.rados.0.smithi063.stdout:3246: writing smithi06387500-8 from 1633455 to 2197553 tid 2 2024-08-17T17:08:27.694 INFO:tasks.rados.rados.0.smithi063.stdout:3242: done (4 left) 2024-08-17T17:08:27.694 INFO:tasks.rados.rados.0.smithi063.stdout:3247: delete oid 9 current snap is 407 2024-08-17T17:08:27.694 INFO:tasks.rados.rados.0.smithi063.stdout:3243: finishing write tid 2 to smithi06387500-17 2024-08-17T17:08:27.694 INFO:tasks.rados.rados.0.smithi063.stdout:3243: finishing write tid 3 to smithi06387500-17 2024-08-17T17:08:27.694 INFO:tasks.rados.rados.0.smithi063.stdout:3243: finishing write tid 4 to smithi06387500-17 2024-08-17T17:08:27.694 INFO:tasks.rados.rados.0.smithi063.stdout:3243: oid 17 updating version 0 to 1112 2024-08-17T17:08:27.694 INFO:tasks.rados.rados.0.smithi063.stdout:3243: oid 17 version 1112 is already newer than 1111 2024-08-17T17:08:27.694 INFO:tasks.rados.rados.0.smithi063.stdout:3243: oid 17 updating version 1112 to 1113 2024-08-17T17:08:27.694 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 1113 (ObjNum 1219 snap 407 seq_num 1219) dirty exists 2024-08-17T17:08:27.694 INFO:tasks.rados.rados.0.smithi063.stdout:3243: left oid 17 (ObjNum 1219 snap 407 seq_num 1219) 2024-08-17T17:08:27.694 INFO:tasks.rados.rados.0.smithi063.stdout:3244: expect (ObjNum 1111 snap 368 seq_num 1111) 2024-08-17T17:08:27.786 INFO:tasks.rados.rados.0.smithi063.stdout:3245: finishing write tid 1 to smithi06387500-21 2024-08-17T17:08:27.786 INFO:tasks.rados.rados.0.smithi063.stdout:3245: finishing write tid 2 to smithi06387500-21 2024-08-17T17:08:27.786 INFO:tasks.rados.rados.0.smithi063.stdout:3243: done (4 left) 2024-08-17T17:08:27.786 INFO:tasks.rados.rados.0.smithi063.stdout:3244: done (3 left) 2024-08-17T17:08:27.786 INFO:tasks.rados.rados.0.smithi063.stdout:3247: done (2 left) 2024-08-17T17:08:27.786 INFO:tasks.rados.rados.0.smithi063.stdout:3248: write oid 41 current snap is 407 2024-08-17T17:08:27.787 INFO:tasks.rados.rados.0.smithi063.stdout:3245: finishing write tid 3 to smithi06387500-21 2024-08-17T17:08:27.787 INFO:tasks.rados.rados.0.smithi063.stdout:3245: finishing write tid 4 to smithi06387500-21 2024-08-17T17:08:27.787 INFO:tasks.rados.rados.0.smithi063.stdout:3245: oid 21 updating version 0 to 2064 2024-08-17T17:08:27.787 INFO:tasks.rados.rados.0.smithi063.stdout:3245: oid 21 updating version 2064 to 2065 2024-08-17T17:08:27.787 INFO:tasks.rados.rados.0.smithi063.stdout:3245: oid 21 version 2065 is already newer than 2063 2024-08-17T17:08:27.787 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 2065 (ObjNum 1220 snap 407 seq_num 1220) dirty exists 2024-08-17T17:08:27.787 INFO:tasks.rados.rados.0.smithi063.stdout:3245: left oid 21 (ObjNum 1220 snap 407 seq_num 1220) 2024-08-17T17:08:27.787 INFO:tasks.rados.rados.0.smithi063.stdout:3246: finishing write tid 1 to smithi06387500-8 2024-08-17T17:08:27.787 INFO:tasks.rados.rados.0.smithi063.stdout:3248: seq_num 1223 ranges {554961=646007,1750419=451414} 2024-08-17T17:08:27.792 INFO:tasks.rados.rados.0.smithi063.stdout:3248: writing smithi06387500-41 from 554961 to 1200968 tid 1 2024-08-17T17:08:27.797 INFO:tasks.rados.rados.0.smithi063.stdout:3248: writing smithi06387500-41 from 1750419 to 2201833 tid 2 2024-08-17T17:08:27.797 INFO:tasks.rados.rados.0.smithi063.stdout:3245: done (2 left) 2024-08-17T17:08:27.798 INFO:tasks.rados.rados.0.smithi063.stdout:3249: snap_remove snap 401 2024-08-17T17:08:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:27 smithi063 ceph-mon[114756]: osdmap e879: 8 total, 8 up, 8 in 2024-08-17T17:08:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:27 smithi063 ceph-mon[114756]: osdmap e880: 8 total, 8 up, 8 in 2024-08-17T17:08:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:27 smithi063 ceph-mon[109843]: osdmap e879: 8 total, 8 up, 8 in 2024-08-17T17:08:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:27 smithi063 ceph-mon[109843]: osdmap e880: 8 total, 8 up, 8 in 2024-08-17T17:08:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:27 smithi162 ceph-mon[104668]: osdmap e879: 8 total, 8 up, 8 in 2024-08-17T17:08:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:27 smithi162 ceph-mon[104668]: osdmap e880: 8 total, 8 up, 8 in 2024-08-17T17:08:28.730 INFO:tasks.rados.rados.0.smithi063.stdout:3249: done (2 left) 2024-08-17T17:08:28.730 INFO:tasks.rados.rados.0.smithi063.stdout:3250: write oid 40 current snap is 407 2024-08-17T17:08:28.730 INFO:tasks.rados.rados.0.smithi063.stdout:3246: finishing write tid 2 to smithi06387500-8 2024-08-17T17:08:28.730 INFO:tasks.rados.rados.0.smithi063.stdout:3246: finishing write tid 3 to smithi06387500-8 2024-08-17T17:08:28.730 INFO:tasks.rados.rados.0.smithi063.stdout:3246: finishing write tid 4 to smithi06387500-8 2024-08-17T17:08:28.731 INFO:tasks.rados.rados.0.smithi063.stdout:3246: oid 8 updating version 0 to 1084 2024-08-17T17:08:28.731 INFO:tasks.rados.rados.0.smithi063.stdout:3246: oid 8 updating version 1084 to 1085 2024-08-17T17:08:28.731 INFO:tasks.rados.rados.0.smithi063.stdout:3246: oid 8 version 1085 is already newer than 1083 2024-08-17T17:08:28.731 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 1085 (ObjNum 1221 snap 407 seq_num 1221) dirty exists 2024-08-17T17:08:28.731 INFO:tasks.rados.rados.0.smithi063.stdout:3246: left oid 8 (ObjNum 1221 snap 407 seq_num 1221) 2024-08-17T17:08:28.731 INFO:tasks.rados.rados.0.smithi063.stdout:3248: finishing write tid 1 to smithi06387500-41 2024-08-17T17:08:28.731 INFO:tasks.rados.rados.0.smithi063.stdout:3248: finishing write tid 2 to smithi06387500-41 2024-08-17T17:08:28.731 INFO:tasks.rados.rados.0.smithi063.stdout:3248: finishing write tid 3 to smithi06387500-41 2024-08-17T17:08:28.731 INFO:tasks.rados.rados.0.smithi063.stdout:3248: finishing write tid 4 to smithi06387500-41 2024-08-17T17:08:28.731 INFO:tasks.rados.rados.0.smithi063.stdout:3248: oid 41 updating version 0 to 372 2024-08-17T17:08:28.731 INFO:tasks.rados.rados.0.smithi063.stdout:3248: oid 41 version 372 is already newer than 371 2024-08-17T17:08:28.731 INFO:tasks.rados.rados.0.smithi063.stdout:3248: oid 41 version 372 is already newer than 370 2024-08-17T17:08:28.731 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 372 (ObjNum 1223 snap 407 seq_num 1223) dirty exists 2024-08-17T17:08:28.732 INFO:tasks.rados.rados.0.smithi063.stdout:3248: left oid 41 (ObjNum 1223 snap 407 seq_num 1223) 2024-08-17T17:08:28.732 INFO:tasks.rados.rados.0.smithi063.stdout:3250: seq_num 1224 ranges {571768=437609,1808901=711425} 2024-08-17T17:08:28.736 INFO:tasks.rados.rados.0.smithi063.stdout:3250: writing smithi06387500-40 from 571768 to 1009377 tid 1 2024-08-17T17:08:28.744 INFO:tasks.rados.rados.0.smithi063.stdout:3250: writing smithi06387500-40 from 1808901 to 2520326 tid 2 2024-08-17T17:08:28.744 INFO:tasks.rados.rados.0.smithi063.stdout:3246: done (2 left) 2024-08-17T17:08:28.744 INFO:tasks.rados.rados.0.smithi063.stdout:3248: done (1 left) 2024-08-17T17:08:28.745 INFO:tasks.rados.rados.0.smithi063.stdout:3251: snap_remove snap 405 2024-08-17T17:08:29.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:28 smithi063 ceph-mon[114756]: pgmap v777: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 363 MiB data, 1022 MiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T17:08:29.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:28 smithi063 ceph-mon[109843]: pgmap v777: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 363 MiB data, 1022 MiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T17:08:29.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:28 smithi162 ceph-mon[104668]: pgmap v777: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 363 MiB data, 1022 MiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T17:08:29.727 INFO:tasks.rados.rados.0.smithi063.stdout:3250: finishing write tid 1 to smithi06387500-40 2024-08-17T17:08:29.727 INFO:tasks.rados.rados.0.smithi063.stdout:3250: finishing write tid 2 to smithi06387500-40 2024-08-17T17:08:29.727 INFO:tasks.rados.rados.0.smithi063.stdout:3250: finishing write tid 3 to smithi06387500-40 2024-08-17T17:08:29.727 INFO:tasks.rados.rados.0.smithi063.stdout:3250: finishing write tid 4 to smithi06387500-40 2024-08-17T17:08:29.727 INFO:tasks.rados.rados.0.smithi063.stdout:3250: oid 40 updating version 0 to 1070 2024-08-17T17:08:29.727 INFO:tasks.rados.rados.0.smithi063.stdout:3250: oid 40 version 1070 is already newer than 1068 2024-08-17T17:08:29.727 INFO:tasks.rados.rados.0.smithi063.stdout:3250: oid 40 version 1070 is already newer than 1069 2024-08-17T17:08:29.727 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 1070 (ObjNum 1224 snap 407 seq_num 1224) dirty exists 2024-08-17T17:08:29.727 INFO:tasks.rados.rados.0.smithi063.stdout:3250: left oid 40 (ObjNum 1224 snap 407 seq_num 1224) 2024-08-17T17:08:29.728 INFO:tasks.rados.rados.0.smithi063.stdout:3250: done (1 left) 2024-08-17T17:08:29.728 INFO:tasks.rados.rados.0.smithi063.stdout:3251: done (0 left) 2024-08-17T17:08:29.728 INFO:tasks.rados.rados.0.smithi063.stdout:3252: read oid 45 snap -1 2024-08-17T17:08:29.728 INFO:tasks.rados.rados.0.smithi063.stdout:3252: expect (ObjNum 1195 snap 401 seq_num 1195) 2024-08-17T17:08:29.728 INFO:tasks.rados.rados.0.smithi063.stdout:3253: write oid 12 current snap is 407 2024-08-17T17:08:29.728 INFO:tasks.rados.rados.0.smithi063.stdout:3253: seq_num 1225 ranges {588575=512858,1867383=455083} 2024-08-17T17:08:29.733 INFO:tasks.rados.rados.0.smithi063.stdout:3253: writing smithi06387500-12 from 588575 to 1101433 tid 1 2024-08-17T17:08:29.741 INFO:tasks.rados.rados.0.smithi063.stdout:3253: writing smithi06387500-12 from 1867383 to 2322466 tid 2 2024-08-17T17:08:29.741 INFO:tasks.rados.rados.0.smithi063.stdout:3254: rollback oid 7 current snap is 407 2024-08-17T17:08:29.741 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 354 2024-08-17T17:08:29.742 INFO:tasks.rados.rados.0.smithi063.stdout:3252: expect (ObjNum 1195 snap 401 seq_num 1195) 2024-08-17T17:08:29.829 INFO:tasks.rados.rados.0.smithi063.stdout:3253: finishing write tid 1 to smithi06387500-12 2024-08-17T17:08:29.830 INFO:tasks.rados.rados.0.smithi063.stdout:3253: finishing write tid 2 to smithi06387500-12 2024-08-17T17:08:29.830 INFO:tasks.rados.rados.0.smithi063.stdout:3253: finishing write tid 3 to smithi06387500-12 2024-08-17T17:08:29.830 INFO:tasks.rados.rados.0.smithi063.stdout:3253: finishing write tid 4 to smithi06387500-12 2024-08-17T17:08:29.830 INFO:tasks.rados.rados.0.smithi063.stdout:3253: oid 12 updating version 0 to 726 2024-08-17T17:08:29.830 INFO:tasks.rados.rados.0.smithi063.stdout:3253: oid 12 version 726 is already newer than 725 2024-08-17T17:08:29.830 INFO:tasks.rados.rados.0.smithi063.stdout:3253: oid 12 version 726 is already newer than 724 2024-08-17T17:08:29.830 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 726 (ObjNum 1225 snap 407 seq_num 1225) dirty exists 2024-08-17T17:08:29.830 INFO:tasks.rados.rados.0.smithi063.stdout:3253: left oid 12 (ObjNum 1225 snap 407 seq_num 1225) 2024-08-17T17:08:29.830 INFO:tasks.rados.rados.0.smithi063.stdout:3254: finishing rollback tid 0 to smithi06387500-7 2024-08-17T17:08:29.830 INFO:tasks.rados.rados.0.smithi063.stdout:3252: done (2 left) 2024-08-17T17:08:29.830 INFO:tasks.rados.rados.0.smithi063.stdout:3253: done (1 left) 2024-08-17T17:08:29.830 INFO:tasks.rados.rados.0.smithi063.stdout:3255: read oid 41 snap -1 2024-08-17T17:08:29.830 INFO:tasks.rados.rados.0.smithi063.stdout:3255: expect (ObjNum 1223 snap 407 seq_num 1223) 2024-08-17T17:08:29.831 INFO:tasks.rados.rados.0.smithi063.stdout:3256: snap_create 2024-08-17T17:08:29.831 INFO:tasks.rados.rados.0.smithi063.stdout:3254: finishing rollback tid 1 to smithi06387500-7 2024-08-17T17:08:29.831 INFO:tasks.rados.rados.0.smithi063.stdout:3254: finishing rollback tid 2 to smithi06387500-7 2024-08-17T17:08:29.831 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1554 (ObjNum 963 snap 327 seq_num 963) dirty exists 2024-08-17T17:08:29.838 INFO:tasks.rados.rados.0.smithi063.stdout:3255: expect (ObjNum 1223 snap 407 seq_num 1223) 2024-08-17T17:08:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:29 smithi063 ceph-mon[114756]: osdmap e881: 8 total, 8 up, 8 in 2024-08-17T17:08:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:29 smithi063 ceph-mon[109843]: osdmap e881: 8 total, 8 up, 8 in 2024-08-17T17:08:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:29 smithi162 ceph-mon[104668]: osdmap e881: 8 total, 8 up, 8 in 2024-08-17T17:08:30.743 INFO:tasks.rados.rados.0.smithi063.stdout:3254: done (2 left) 2024-08-17T17:08:30.744 INFO:tasks.rados.rados.0.smithi063.stdout:3255: done (1 left) 2024-08-17T17:08:30.744 INFO:tasks.rados.rados.0.smithi063.stdout:3256: done (0 left) 2024-08-17T17:08:30.744 INFO:tasks.rados.rados.0.smithi063.stdout:3257: snap_create 2024-08-17T17:08:30.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:30 smithi162 ceph-mon[104668]: osdmap e882: 8 total, 8 up, 8 in 2024-08-17T17:08:30.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:30 smithi162 ceph-mon[104668]: pgmap v780: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 360 MiB data, 1021 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 857 KiB/s wr, 6 op/s 2024-08-17T17:08:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:30 smithi063 ceph-mon[114756]: osdmap e882: 8 total, 8 up, 8 in 2024-08-17T17:08:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:30 smithi063 ceph-mon[114756]: pgmap v780: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 360 MiB data, 1021 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 857 KiB/s wr, 6 op/s 2024-08-17T17:08:31.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:30 smithi063 ceph-mon[109843]: osdmap e882: 8 total, 8 up, 8 in 2024-08-17T17:08:31.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:30 smithi063 ceph-mon[109843]: pgmap v780: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 360 MiB data, 1021 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 857 KiB/s wr, 6 op/s 2024-08-17T17:08:31.744 INFO:tasks.rados.rados.0.smithi063.stdout:3257: done (0 left) 2024-08-17T17:08:31.744 INFO:tasks.rados.rados.0.smithi063.stdout:3258: read oid 12 snap -1 2024-08-17T17:08:31.744 INFO:tasks.rados.rados.0.smithi063.stdout:3258: expect (ObjNum 1225 snap 407 seq_num 1225) 2024-08-17T17:08:31.744 INFO:tasks.rados.rados.0.smithi063.stdout:3259: write oid 27 current snap is 409 2024-08-17T17:08:31.744 INFO:tasks.rados.rados.0.smithi063.stdout:3259: seq_num 1226 ranges {605382=588107,1925865=679517} 2024-08-17T17:08:31.751 INFO:tasks.rados.rados.0.smithi063.stdout:3259: writing smithi06387500-27 from 605382 to 1193489 tid 1 2024-08-17T17:08:31.759 INFO:tasks.rados.rados.0.smithi063.stdout:3259: writing smithi06387500-27 from 1925865 to 2605382 tid 2 2024-08-17T17:08:31.759 INFO:tasks.rados.rados.0.smithi063.stdout:3260: rollback oid 3 current snap is 409 2024-08-17T17:08:31.759 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 3 to 393 2024-08-17T17:08:31.759 INFO:tasks.rados.rados.0.smithi063.stdout:3259: finishing write tid 1 to smithi06387500-27 2024-08-17T17:08:31.760 INFO:tasks.rados.rados.0.smithi063.stdout:3258: expect (ObjNum 1225 snap 407 seq_num 1225) 2024-08-17T17:08:31.883 INFO:tasks.rados.rados.0.smithi063.stdout:3260: finishing rollback tid 1 to smithi06387500-3 2024-08-17T17:08:31.883 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 1031 (ObjNum 129 snap 0 seq_num 1807117280) dirty dne 2024-08-17T17:08:31.883 INFO:tasks.rados.rados.0.smithi063.stdout:3259: finishing write tid 2 to smithi06387500-27 2024-08-17T17:08:31.883 INFO:tasks.rados.rados.0.smithi063.stdout:3258: done (2 left) 2024-08-17T17:08:31.883 INFO:tasks.rados.rados.0.smithi063.stdout:3260: done (1 left) 2024-08-17T17:08:31.883 INFO:tasks.rados.rados.0.smithi063.stdout:3261: rollback oid 17 current snap is 409 2024-08-17T17:08:31.883 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 17 to 399 2024-08-17T17:08:31.883 INFO:tasks.rados.rados.0.smithi063.stdout:3259: finishing write tid 3 to smithi06387500-27 2024-08-17T17:08:31.883 INFO:tasks.rados.rados.0.smithi063.stdout:3259: finishing write tid 4 to smithi06387500-27 2024-08-17T17:08:31.884 INFO:tasks.rados.rados.0.smithi063.stdout:3259: oid 27 updating version 0 to 303 2024-08-17T17:08:31.884 INFO:tasks.rados.rados.0.smithi063.stdout:3259: oid 27 version 303 is already newer than 301 2024-08-17T17:08:31.884 INFO:tasks.rados.rados.0.smithi063.stdout:3259: oid 27 version 303 is already newer than 302 2024-08-17T17:08:31.884 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 303 (ObjNum 1226 snap 409 seq_num 1226) dirty exists 2024-08-17T17:08:31.884 INFO:tasks.rados.rados.0.smithi063.stdout:3259: left oid 27 (ObjNum 1226 snap 409 seq_num 1226) 2024-08-17T17:08:31.884 INFO:tasks.rados.rados.0.smithi063.stdout:3259: done (1 left) 2024-08-17T17:08:31.884 INFO:tasks.rados.rados.0.smithi063.stdout:3262: snap_remove snap 400 2024-08-17T17:08:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:31 smithi063 ceph-mon[114756]: osdmap e883: 8 total, 8 up, 8 in 2024-08-17T17:08:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:32.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:31 smithi063 ceph-mon[109843]: osdmap e883: 8 total, 8 up, 8 in 2024-08-17T17:08:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:31 smithi162 ceph-mon[104668]: osdmap e883: 8 total, 8 up, 8 in 2024-08-17T17:08:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:32.645 INFO:tasks.rados.rados.0.smithi063.stdout:3262: done (1 left) 2024-08-17T17:08:32.645 INFO:tasks.rados.rados.0.smithi063.stdout:3263: read oid 49 snap -1 2024-08-17T17:08:32.645 INFO:tasks.rados.rados.0.smithi063.stdout:3263: expect (ObjNum 1101 snap 366 seq_num 1101) 2024-08-17T17:08:32.645 INFO:tasks.rados.rados.0.smithi063.stdout:3264: read oid 27 snap -1 2024-08-17T17:08:32.645 INFO:tasks.rados.rados.0.smithi063.stdout:3264: expect (ObjNum 1226 snap 409 seq_num 1226) 2024-08-17T17:08:32.646 INFO:tasks.rados.rados.0.smithi063.stdout:3265: rollback oid 39 current snap is 409 2024-08-17T17:08:32.646 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 39 to 339 2024-08-17T17:08:32.646 INFO:tasks.rados.rados.0.smithi063.stdout:3261: finishing rollback tid 0 to smithi06387500-17 2024-08-17T17:08:32.646 INFO:tasks.rados.rados.0.smithi063.stdout:3261: finishing rollback tid 1 to smithi06387500-17 2024-08-17T17:08:32.646 INFO:tasks.rados.rados.0.smithi063.stdout:3261: finishing rollback tid 2 to smithi06387500-17 2024-08-17T17:08:32.646 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 1120 (ObjNum 1105 snap 366 seq_num 1105) dirty exists 2024-08-17T17:08:32.646 INFO:tasks.rados.rados.0.smithi063.stdout:3261: done (3 left) 2024-08-17T17:08:32.646 INFO:tasks.rados.rados.0.smithi063.stdout:3266: read oid 45 snap 404 2024-08-17T17:08:32.646 INFO:tasks.rados.rados.0.smithi063.stdout:3266: expect (ObjNum 1195 snap 401 seq_num 1195) 2024-08-17T17:08:32.646 INFO:tasks.rados.rados.0.smithi063.stdout:3267: read oid 29 snap -1 2024-08-17T17:08:32.646 INFO:tasks.rados.rados.0.smithi063.stdout:3267: expect deleted 2024-08-17T17:08:32.647 INFO:tasks.rados.rados.0.smithi063.stdout:3268: snap_create 2024-08-17T17:08:32.655 INFO:tasks.rados.rados.0.smithi063.stdout:3264: expect (ObjNum 1226 snap 409 seq_num 1226) 2024-08-17T17:08:32.743 INFO:tasks.rados.rados.0.smithi063.stdout:3263: expect (ObjNum 1101 snap 366 seq_num 1101) 2024-08-17T17:08:32.885 INFO:tasks.rados.rados.0.smithi063.stdout:3266: expect (ObjNum 1195 snap 401 seq_num 1195) 2024-08-17T17:08:32.971 INFO:tasks.rados.rados.0.smithi063.stdout:3265: finishing rollback tid 0 to smithi06387500-39 2024-08-17T17:08:32.972 INFO:tasks.rados.rados.0.smithi063.stdout:3265: finishing rollback tid 1 to smithi06387500-39 2024-08-17T17:08:32.972 INFO:tasks.rados.rados.0.smithi063.stdout:3265: finishing rollback tid 2 to smithi06387500-39 2024-08-17T17:08:32.972 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 774 (ObjNum 943 snap 323 seq_num 943) dirty exists 2024-08-17T17:08:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:32 smithi063 ceph-mon[114756]: osdmap e884: 8 total, 8 up, 8 in 2024-08-17T17:08:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:32 smithi063 ceph-mon[114756]: pgmap v783: 89 pgs: 89 active+clean; 355 MiB data, 1023 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.9 MiB/s wr, 15 op/s 2024-08-17T17:08:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:32 smithi063 ceph-mon[114756]: osdmap e885: 8 total, 8 up, 8 in 2024-08-17T17:08:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:32 smithi063 ceph-mon[109843]: osdmap e884: 8 total, 8 up, 8 in 2024-08-17T17:08:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:32 smithi063 ceph-mon[109843]: pgmap v783: 89 pgs: 89 active+clean; 355 MiB data, 1023 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.9 MiB/s wr, 15 op/s 2024-08-17T17:08:33.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:32 smithi063 ceph-mon[109843]: osdmap e885: 8 total, 8 up, 8 in 2024-08-17T17:08:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:32 smithi162 ceph-mon[104668]: osdmap e884: 8 total, 8 up, 8 in 2024-08-17T17:08:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:32 smithi162 ceph-mon[104668]: pgmap v783: 89 pgs: 89 active+clean; 355 MiB data, 1023 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.9 MiB/s wr, 15 op/s 2024-08-17T17:08:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:32 smithi162 ceph-mon[104668]: osdmap e885: 8 total, 8 up, 8 in 2024-08-17T17:08:33.643 INFO:tasks.rados.rados.0.smithi063.stdout:3263: done (5 left) 2024-08-17T17:08:33.643 INFO:tasks.rados.rados.0.smithi063.stdout:3264: done (4 left) 2024-08-17T17:08:33.643 INFO:tasks.rados.rados.0.smithi063.stdout:3265: done (3 left) 2024-08-17T17:08:33.643 INFO:tasks.rados.rados.0.smithi063.stdout:3266: done (2 left) 2024-08-17T17:08:33.644 INFO:tasks.rados.rados.0.smithi063.stdout:3267: done (1 left) 2024-08-17T17:08:33.644 INFO:tasks.rados.rados.0.smithi063.stdout:3268: done (0 left) 2024-08-17T17:08:33.644 INFO:tasks.rados.rados.0.smithi063.stdout:3269: delete oid 47 current snap is 410 2024-08-17T17:08:33.648 INFO:tasks.rados.rados.0.smithi063.stdout:3269: done (0 left) 2024-08-17T17:08:33.648 INFO:tasks.rados.rados.0.smithi063.stdout:3270: rollback oid 17 current snap is 410 2024-08-17T17:08:33.648 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 17 to 407 2024-08-17T17:08:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:3271: read oid 18 snap -1 2024-08-17T17:08:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:3271: expect (ObjNum 1201 snap 404 seq_num 1201) 2024-08-17T17:08:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:3272: read oid 19 snap -1 2024-08-17T17:08:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:3272: expect (ObjNum 1190 snap 400 seq_num 1190) 2024-08-17T17:08:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:3273: read oid 23 snap -1 2024-08-17T17:08:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:3273: expect (ObjNum 1213 snap 406 seq_num 1213) 2024-08-17T17:08:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:3274: rollback oid 40 current snap is 410 2024-08-17T17:08:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 40 to 396 2024-08-17T17:08:33.649 INFO:tasks.rados.rados.0.smithi063.stdout:3275: snap_create 2024-08-17T17:08:33.653 INFO:tasks.rados.rados.0.smithi063.stdout:3272: expect (ObjNum 1190 snap 400 seq_num 1190) 2024-08-17T17:08:33.721 INFO:tasks.rados.rados.0.smithi063.stdout:3271: expect (ObjNum 1201 snap 404 seq_num 1201) 2024-08-17T17:08:33.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:08:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:08:33] "GET /metrics HTTP/1.1" 200 36343 "" "Prometheus/2.51.0" 2024-08-17T17:08:33.825 INFO:tasks.rados.rados.0.smithi063.stdout:3270: finishing rollback tid 0 to smithi06387500-17 2024-08-17T17:08:33.826 INFO:tasks.rados.rados.0.smithi063.stdout:3270: finishing rollback tid 1 to smithi06387500-17 2024-08-17T17:08:33.826 INFO:tasks.rados.rados.0.smithi063.stdout:3270: finishing rollback tid 2 to smithi06387500-17 2024-08-17T17:08:33.826 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 1126 (ObjNum 1219 snap 407 seq_num 1219) dirty exists 2024-08-17T17:08:33.826 INFO:tasks.rados.rados.0.smithi063.stdout:3274: finishing rollback tid 0 to smithi06387500-40 2024-08-17T17:08:33.826 INFO:tasks.rados.rados.0.smithi063.stdout:3274: finishing rollback tid 1 to smithi06387500-40 2024-08-17T17:08:33.826 INFO:tasks.rados.rados.0.smithi063.stdout:3274: finishing rollback tid 2 to smithi06387500-40 2024-08-17T17:08:33.826 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 1082 (ObjNum 1173 snap 395 seq_num 1173) dirty exists 2024-08-17T17:08:33.826 INFO:tasks.rados.rados.0.smithi063.stdout:3273: expect (ObjNum 1213 snap 406 seq_num 1213) 2024-08-17T17:08:34.650 INFO:tasks.rados.rados.0.smithi063.stdout:3270: done (5 left) 2024-08-17T17:08:34.651 INFO:tasks.rados.rados.0.smithi063.stdout:3271: done (4 left) 2024-08-17T17:08:34.651 INFO:tasks.rados.rados.0.smithi063.stdout:3272: done (3 left) 2024-08-17T17:08:34.651 INFO:tasks.rados.rados.0.smithi063.stdout:3273: done (2 left) 2024-08-17T17:08:34.651 INFO:tasks.rados.rados.0.smithi063.stdout:3274: done (1 left) 2024-08-17T17:08:34.651 INFO:tasks.rados.rados.0.smithi063.stdout:3275: done (0 left) 2024-08-17T17:08:34.651 INFO:tasks.rados.rados.0.smithi063.stdout:3276: write oid 17 current snap is 411 2024-08-17T17:08:34.651 INFO:tasks.rados.rados.0.smithi063.stdout:3276: seq_num 1228 ranges {638996=738605,1926476=712520} 2024-08-17T17:08:34.658 INFO:tasks.rados.rados.0.smithi063.stdout:3276: writing smithi06387500-17 from 638996 to 1377601 tid 1 2024-08-17T17:08:34.667 INFO:tasks.rados.rados.0.smithi063.stdout:3276: writing smithi06387500-17 from 1926476 to 2638996 tid 2 2024-08-17T17:08:34.668 INFO:tasks.rados.rados.0.smithi063.stdout:3277: write oid 21 current snap is 411 2024-08-17T17:08:34.668 INFO:tasks.rados.rados.0.smithi063.stdout:3276: finishing write tid 1 to smithi06387500-17 2024-08-17T17:08:34.668 INFO:tasks.rados.rados.0.smithi063.stdout:3277: seq_num 1229 ranges {655803=413854,1584958=462421} 2024-08-17T17:08:34.673 INFO:tasks.rados.rados.0.smithi063.stdout:3277: writing smithi06387500-21 from 655803 to 1069657 tid 1 2024-08-17T17:08:34.681 INFO:tasks.rados.rados.0.smithi063.stdout:3277: writing smithi06387500-21 from 1584958 to 2047379 tid 2 2024-08-17T17:08:34.681 INFO:tasks.rados.rados.0.smithi063.stdout:3278: write oid 47 current snap is 411 2024-08-17T17:08:34.681 INFO:tasks.rados.rados.0.smithi063.stdout:3276: finishing write tid 2 to smithi06387500-17 2024-08-17T17:08:34.681 INFO:tasks.rados.rados.0.smithi063.stdout:3276: finishing write tid 3 to smithi06387500-17 2024-08-17T17:08:34.681 INFO:tasks.rados.rados.0.smithi063.stdout:3276: finishing write tid 4 to smithi06387500-17 2024-08-17T17:08:34.681 INFO:tasks.rados.rados.0.smithi063.stdout:3276: oid 17 updating version 0 to 1130 2024-08-17T17:08:34.681 INFO:tasks.rados.rados.0.smithi063.stdout:3276: oid 17 version 1130 is already newer than 1129 2024-08-17T17:08:34.681 INFO:tasks.rados.rados.0.smithi063.stdout:3276: oid 17 version 1130 is already newer than 1128 2024-08-17T17:08:34.681 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 1130 (ObjNum 1228 snap 411 seq_num 1228) dirty exists 2024-08-17T17:08:34.681 INFO:tasks.rados.rados.0.smithi063.stdout:3276: left oid 17 (ObjNum 1228 snap 411 seq_num 1228) 2024-08-17T17:08:34.681 INFO:tasks.rados.rados.0.smithi063.stdout:3277: finishing write tid 1 to smithi06387500-21 2024-08-17T17:08:34.682 INFO:tasks.rados.rados.0.smithi063.stdout:3278: seq_num 1230 ranges {672610=489103,1643440=722432} 2024-08-17T17:08:34.688 INFO:tasks.rados.rados.0.smithi063.stdout:3278: writing smithi06387500-47 from 672610 to 1161713 tid 1 2024-08-17T17:08:34.695 INFO:tasks.rados.rados.0.smithi063.stdout:3278: writing smithi06387500-47 from 1643440 to 2365872 tid 2 2024-08-17T17:08:34.696 INFO:tasks.rados.rados.0.smithi063.stdout:3276: done (2 left) 2024-08-17T17:08:34.696 INFO:tasks.rados.rados.0.smithi063.stdout:3279: delete oid 10 current snap is 411 2024-08-17T17:08:34.696 INFO:tasks.rados.rados.0.smithi063.stdout:3277: finishing write tid 2 to smithi06387500-21 2024-08-17T17:08:34.696 INFO:tasks.rados.rados.0.smithi063.stdout:3277: finishing write tid 3 to smithi06387500-21 2024-08-17T17:08:34.696 INFO:tasks.rados.rados.0.smithi063.stdout:3277: finishing write tid 4 to smithi06387500-21 2024-08-17T17:08:34.697 INFO:tasks.rados.rados.0.smithi063.stdout:3277: oid 21 updating version 0 to 2087 2024-08-17T17:08:34.697 INFO:tasks.rados.rados.0.smithi063.stdout:3277: oid 21 version 2087 is already newer than 2086 2024-08-17T17:08:34.697 INFO:tasks.rados.rados.0.smithi063.stdout:3277: oid 21 version 2087 is already newer than 2085 2024-08-17T17:08:34.697 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 2087 (ObjNum 1229 snap 411 seq_num 1229) dirty exists 2024-08-17T17:08:34.697 INFO:tasks.rados.rados.0.smithi063.stdout:3277: left oid 21 (ObjNum 1229 snap 411 seq_num 1229) 2024-08-17T17:08:34.717 INFO:tasks.rados.rados.0.smithi063.stdout:3277: done (2 left) 2024-08-17T17:08:34.717 INFO:tasks.rados.rados.0.smithi063.stdout:3279: done (1 left) 2024-08-17T17:08:34.717 INFO:tasks.rados.rados.0.smithi063.stdout:3280: delete oid 31 current snap is 411 2024-08-17T17:08:34.720 INFO:tasks.rados.rados.0.smithi063.stdout:3280: done (1 left) 2024-08-17T17:08:34.720 INFO:tasks.rados.rados.0.smithi063.stdout:3281: snap_create 2024-08-17T17:08:34.722 INFO:tasks.rados.rados.0.smithi063.stdout:3278: finishing write tid 1 to smithi06387500-47 2024-08-17T17:08:34.722 INFO:tasks.rados.rados.0.smithi063.stdout:3278: finishing write tid 2 to smithi06387500-47 2024-08-17T17:08:34.722 INFO:tasks.rados.rados.0.smithi063.stdout:3278: finishing write tid 3 to smithi06387500-47 2024-08-17T17:08:34.722 INFO:tasks.rados.rados.0.smithi063.stdout:3278: finishing write tid 4 to smithi06387500-47 2024-08-17T17:08:34.723 INFO:tasks.rados.rados.0.smithi063.stdout:3278: oid 47 updating version 0 to 704 2024-08-17T17:08:34.723 INFO:tasks.rados.rados.0.smithi063.stdout:3278: oid 47 version 704 is already newer than 703 2024-08-17T17:08:34.723 INFO:tasks.rados.rados.0.smithi063.stdout:3278: oid 47 updating version 704 to 705 2024-08-17T17:08:34.723 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 705 (ObjNum 1230 snap 411 seq_num 1230) dirty exists 2024-08-17T17:08:34.723 INFO:tasks.rados.rados.0.smithi063.stdout:3278: left oid 47 (ObjNum 1230 snap 411 seq_num 1230) 2024-08-17T17:08:35.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:34 smithi063 ceph-mon[114756]: osdmap e886: 8 total, 8 up, 8 in 2024-08-17T17:08:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:34 smithi063 ceph-mon[109843]: osdmap e886: 8 total, 8 up, 8 in 2024-08-17T17:08:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:34 smithi162 ceph-mon[104668]: osdmap e886: 8 total, 8 up, 8 in 2024-08-17T17:08:35.657 INFO:tasks.rados.rados.0.smithi063.stdout:3278: done (1 left) 2024-08-17T17:08:35.657 INFO:tasks.rados.rados.0.smithi063.stdout:3281: done (0 left) 2024-08-17T17:08:35.657 INFO:tasks.rados.rados.0.smithi063.stdout:3282: snap_create 2024-08-17T17:08:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:35 smithi063 ceph-mon[114756]: pgmap v786: 89 pgs: 89 active+clean; 356 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2024-08-17T17:08:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:35 smithi063 ceph-mon[114756]: osdmap e887: 8 total, 8 up, 8 in 2024-08-17T17:08:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:35 smithi063 ceph-mon[109843]: pgmap v786: 89 pgs: 89 active+clean; 356 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2024-08-17T17:08:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:35 smithi063 ceph-mon[109843]: osdmap e887: 8 total, 8 up, 8 in 2024-08-17T17:08:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:35 smithi162 ceph-mon[104668]: pgmap v786: 89 pgs: 89 active+clean; 356 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 2.1 MiB/s wr, 17 op/s 2024-08-17T17:08:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:35 smithi162 ceph-mon[104668]: osdmap e887: 8 total, 8 up, 8 in 2024-08-17T17:08:36.665 INFO:tasks.rados.rados.0.smithi063.stdout:3282: done (0 left) 2024-08-17T17:08:36.665 INFO:tasks.rados.rados.0.smithi063.stdout:3283: read oid 26 snap -1 2024-08-17T17:08:36.665 INFO:tasks.rados.rados.0.smithi063.stdout:3283: expect (ObjNum 1193 snap 400 seq_num 1193) 2024-08-17T17:08:36.665 INFO:tasks.rados.rados.0.smithi063.stdout:3284: snap_remove snap 411 2024-08-17T17:08:37.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:36 smithi063 ceph-mon[114756]: osdmap e888: 8 total, 8 up, 8 in 2024-08-17T17:08:37.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:36 smithi063 ceph-mon[109843]: osdmap e888: 8 total, 8 up, 8 in 2024-08-17T17:08:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:36 smithi162 ceph-mon[104668]: osdmap e888: 8 total, 8 up, 8 in 2024-08-17T17:08:37.646 INFO:tasks.rados.rados.0.smithi063.stdout:3284: done (1 left) 2024-08-17T17:08:37.646 INFO:tasks.rados.rados.0.smithi063.stdout:3285: write oid 35 current snap is 413 2024-08-17T17:08:37.646 INFO:tasks.rados.rados.0.smithi063.stdout:3285: seq_num 1233 ranges {723031=431203,1818886=469759} 2024-08-17T17:08:37.652 INFO:tasks.rados.rados.0.smithi063.stdout:3285: writing smithi06387500-35 from 723031 to 1154234 tid 1 2024-08-17T17:08:37.659 INFO:tasks.rados.rados.0.smithi063.stdout:3285: writing smithi06387500-35 from 1818886 to 2288645 tid 2 2024-08-17T17:08:37.659 INFO:tasks.rados.rados.0.smithi063.stdout:3286: rollback oid 46 current snap is 413 2024-08-17T17:08:37.659 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 46 to 386 2024-08-17T17:08:37.659 INFO:tasks.rados.rados.0.smithi063.stdout:3283: expect (ObjNum 1193 snap 400 seq_num 1193) 2024-08-17T17:08:37.914 INFO:tasks.rados.rados.0.smithi063.stdout:3285: finishing write tid 1 to smithi06387500-35 2024-08-17T17:08:37.914 INFO:tasks.rados.rados.0.smithi063.stdout:3286: finishing rollback tid 0 to smithi06387500-46 2024-08-17T17:08:37.914 INFO:tasks.rados.rados.0.smithi063.stdout:3286: finishing rollback tid 1 to smithi06387500-46 2024-08-17T17:08:37.914 INFO:tasks.rados.rados.0.smithi063.stdout:3283: done (2 left) 2024-08-17T17:08:37.914 INFO:tasks.rados.rados.0.smithi063.stdout:3287: rollback oid 22 current snap is 413 2024-08-17T17:08:37.915 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 22 to 394 2024-08-17T17:08:37.915 INFO:tasks.rados.rados.0.smithi063.stdout:3286: finishing rollback tid 2 to smithi06387500-46 2024-08-17T17:08:37.915 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 1137 (ObjNum 1134 snap 377 seq_num 1134) dirty exists 2024-08-17T17:08:37.915 INFO:tasks.rados.rados.0.smithi063.stdout:3285: finishing write tid 2 to smithi06387500-35 2024-08-17T17:08:37.915 INFO:tasks.rados.rados.0.smithi063.stdout:3285: finishing write tid 3 to smithi06387500-35 2024-08-17T17:08:37.915 INFO:tasks.rados.rados.0.smithi063.stdout:3285: finishing write tid 4 to smithi06387500-35 2024-08-17T17:08:37.915 INFO:tasks.rados.rados.0.smithi063.stdout:3285: oid 35 updating version 0 to 3186 2024-08-17T17:08:37.915 INFO:tasks.rados.rados.0.smithi063.stdout:3285: oid 35 updating version 3186 to 3188 2024-08-17T17:08:37.915 INFO:tasks.rados.rados.0.smithi063.stdout:3285: oid 35 version 3188 is already newer than 3187 2024-08-17T17:08:37.915 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 3188 (ObjNum 1233 snap 413 seq_num 1233) dirty exists 2024-08-17T17:08:37.915 INFO:tasks.rados.rados.0.smithi063.stdout:3285: left oid 35 (ObjNum 1233 snap 413 seq_num 1233) 2024-08-17T17:08:37.916 INFO:tasks.rados.rados.0.smithi063.stdout:3285: done (2 left) 2024-08-17T17:08:37.916 INFO:tasks.rados.rados.0.smithi063.stdout:3286: done (1 left) 2024-08-17T17:08:37.916 INFO:tasks.rados.rados.0.smithi063.stdout:3288: delete oid 8 current snap is 413 2024-08-17T17:08:37.916 INFO:tasks.rados.rados.0.smithi063.stdout:3287: finishing rollback tid 1 to smithi06387500-22 2024-08-17T17:08:37.916 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 695 (ObjNum 129 snap 0 seq_num 1028474639) dirty dne 2024-08-17T17:08:37.918 INFO:tasks.rados.rados.0.smithi063.stdout:3287: done (1 left) 2024-08-17T17:08:37.918 INFO:tasks.rados.rados.0.smithi063.stdout:3288: done (0 left) 2024-08-17T17:08:37.918 INFO:tasks.rados.rados.0.smithi063.stdout:3289: snap_remove snap 394 2024-08-17T17:08:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:37 smithi063 ceph-mon[114756]: pgmap v789: 89 pgs: 89 active+clean; 362 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 662 KiB/s wr, 13 op/s 2024-08-17T17:08:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:37 smithi063 ceph-mon[114756]: osdmap e889: 8 total, 8 up, 8 in 2024-08-17T17:08:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:37 smithi063 ceph-mon[114756]: osdmap e890: 8 total, 8 up, 8 in 2024-08-17T17:08:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:37 smithi063 ceph-mon[109843]: pgmap v789: 89 pgs: 89 active+clean; 362 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 662 KiB/s wr, 13 op/s 2024-08-17T17:08:38.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:37 smithi063 ceph-mon[109843]: osdmap e889: 8 total, 8 up, 8 in 2024-08-17T17:08:38.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:37 smithi063 ceph-mon[109843]: osdmap e890: 8 total, 8 up, 8 in 2024-08-17T17:08:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:37 smithi162 ceph-mon[104668]: pgmap v789: 89 pgs: 89 active+clean; 362 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 662 KiB/s wr, 13 op/s 2024-08-17T17:08:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:37 smithi162 ceph-mon[104668]: osdmap e889: 8 total, 8 up, 8 in 2024-08-17T17:08:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:37 smithi162 ceph-mon[104668]: osdmap e890: 8 total, 8 up, 8 in 2024-08-17T17:08:38.680 INFO:tasks.rados.rados.0.smithi063.stdout:3289: done (0 left) 2024-08-17T17:08:38.680 INFO:tasks.rados.rados.0.smithi063.stdout:3290: snap_remove snap 412 2024-08-17T17:08:39.678 INFO:tasks.rados.rados.0.smithi063.stdout:3290: done (0 left) 2024-08-17T17:08:39.678 INFO:tasks.rados.rados.0.smithi063.stdout:3291: snap_remove snap 407 2024-08-17T17:08:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:39 smithi063 ceph-mon[114756]: pgmap v792: 89 pgs: 89 active+clean; 368 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 878 KiB/s wr, 22 op/s 2024-08-17T17:08:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:39 smithi063 ceph-mon[114756]: osdmap e891: 8 total, 8 up, 8 in 2024-08-17T17:08:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:39 smithi063 ceph-mon[109843]: pgmap v792: 89 pgs: 89 active+clean; 368 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 878 KiB/s wr, 22 op/s 2024-08-17T17:08:40.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:39 smithi063 ceph-mon[109843]: osdmap e891: 8 total, 8 up, 8 in 2024-08-17T17:08:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:39 smithi162 ceph-mon[104668]: pgmap v792: 89 pgs: 89 active+clean; 368 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 878 KiB/s wr, 22 op/s 2024-08-17T17:08:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:39 smithi162 ceph-mon[104668]: osdmap e891: 8 total, 8 up, 8 in 2024-08-17T17:08:40.691 INFO:tasks.rados.rados.0.smithi063.stdout:3291: done (0 left) 2024-08-17T17:08:40.691 INFO:tasks.rados.rados.0.smithi063.stdout:3292: rollback oid 5 current snap is 413 2024-08-17T17:08:40.691 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 354 2024-08-17T17:08:40.691 INFO:tasks.rados.rados.0.smithi063.stdout:3293: rollback oid 33 current snap is 413 2024-08-17T17:08:40.691 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 33 to 393 2024-08-17T17:08:40.691 INFO:tasks.rados.rados.0.smithi063.stdout:3294: rollback oid 15 current snap is 413 2024-08-17T17:08:40.692 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 15 to 378 2024-08-17T17:08:40.692 INFO:tasks.rados.rados.0.smithi063.stdout:3295: snap_remove snap 339 2024-08-17T17:08:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:40 smithi063 ceph-mon[114756]: osdmap e892: 8 total, 8 up, 8 in 2024-08-17T17:08:41.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:40 smithi063 ceph-mon[109843]: osdmap e892: 8 total, 8 up, 8 in 2024-08-17T17:08:41.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:40 smithi162 ceph-mon[104668]: osdmap e892: 8 total, 8 up, 8 in 2024-08-17T17:08:41.693 INFO:tasks.rados.rados.0.smithi063.stdout:3295: done (3 left) 2024-08-17T17:08:41.693 INFO:tasks.rados.rados.0.smithi063.stdout:3296: delete oid 2 current snap is 413 2024-08-17T17:08:41.693 INFO:tasks.rados.rados.0.smithi063.stdout:3292: finishing rollback tid 0 to smithi06387500-5 2024-08-17T17:08:41.693 INFO:tasks.rados.rados.0.smithi063.stdout:3294: finishing rollback tid 1 to smithi06387500-15 2024-08-17T17:08:41.694 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 1089 (ObjNum 177 snap 0 seq_num 1807146256) dirty dne 2024-08-17T17:08:41.694 INFO:tasks.rados.rados.0.smithi063.stdout:3292: finishing rollback tid 1 to smithi06387500-5 2024-08-17T17:08:41.694 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1671 (ObjNum 65 snap 0 seq_num 1953066355) dirty dne 2024-08-17T17:08:41.694 INFO:tasks.rados.rados.0.smithi063.stdout:3293: finishing rollback tid 1 to smithi06387500-33 2024-08-17T17:08:41.694 INFO:tasks.rados.rados.0.smithi063.stdout:3293: finishing rollback tid 2 to smithi06387500-33 2024-08-17T17:08:41.694 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 1093 (ObjNum 1071 snap 362 seq_num 1071) dirty exists 2024-08-17T17:08:41.706 INFO:tasks.rados.rados.0.smithi063.stdout:3292: done (3 left) 2024-08-17T17:08:41.706 INFO:tasks.rados.rados.0.smithi063.stdout:3293: done (2 left) 2024-08-17T17:08:41.706 INFO:tasks.rados.rados.0.smithi063.stdout:3294: done (1 left) 2024-08-17T17:08:41.706 INFO:tasks.rados.rados.0.smithi063.stdout:3296: done (0 left) 2024-08-17T17:08:41.706 INFO:tasks.rados.rados.0.smithi063.stdout:3297: snap_remove snap 396 2024-08-17T17:08:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:41 smithi063 ceph-mon[114756]: pgmap v795: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 371 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 650 KiB/s wr, 12 op/s 2024-08-17T17:08:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:41 smithi063 ceph-mon[114756]: osdmap e893: 8 total, 8 up, 8 in 2024-08-17T17:08:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:41 smithi063 ceph-mon[109843]: pgmap v795: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 371 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 650 KiB/s wr, 12 op/s 2024-08-17T17:08:42.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:41 smithi063 ceph-mon[109843]: osdmap e893: 8 total, 8 up, 8 in 2024-08-17T17:08:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:41 smithi162 ceph-mon[104668]: pgmap v795: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 371 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 650 KiB/s wr, 12 op/s 2024-08-17T17:08:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:41 smithi162 ceph-mon[104668]: osdmap e893: 8 total, 8 up, 8 in 2024-08-17T17:08:42.646 INFO:tasks.rados.rados.0.smithi063.stdout:3297: done (0 left) 2024-08-17T17:08:42.646 INFO:tasks.rados.rados.0.smithi063.stdout:3298: rollback oid 4 current snap is 413 2024-08-17T17:08:42.646 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 4 to 399 2024-08-17T17:08:42.646 INFO:tasks.rados.rados.0.smithi063.stdout:3299: rollback oid 39 current snap is 413 2024-08-17T17:08:42.646 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 39 to 408 2024-08-17T17:08:42.646 INFO:tasks.rados.rados.0.smithi063.stdout:3300: write oid 29 current snap is 413 2024-08-17T17:08:42.647 INFO:tasks.rados.rados.0.smithi063.stdout:3300: seq_num 1236 ranges {773452=656950,1877979=617086} 2024-08-17T17:08:42.655 INFO:tasks.rados.rados.0.smithi063.stdout:3300: writing smithi06387500-29 from 773452 to 1430402 tid 1 2024-08-17T17:08:42.662 INFO:tasks.rados.rados.0.smithi063.stdout:3300: writing smithi06387500-29 from 1877979 to 2495065 tid 2 2024-08-17T17:08:42.662 INFO:tasks.rados.rados.0.smithi063.stdout:3301: delete oid 45 current snap is 413 2024-08-17T17:08:42.662 INFO:tasks.rados.rados.0.smithi063.stdout:3299: finishing rollback tid 0 to smithi06387500-39 2024-08-17T17:08:42.663 INFO:tasks.rados.rados.0.smithi063.stdout:3298: finishing rollback tid 0 to smithi06387500-4 2024-08-17T17:08:42.663 INFO:tasks.rados.rados.0.smithi063.stdout:3299: finishing rollback tid 1 to smithi06387500-39 2024-08-17T17:08:42.663 INFO:tasks.rados.rados.0.smithi063.stdout:3299: finishing rollback tid 2 to smithi06387500-39 2024-08-17T17:08:42.663 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 788 (ObjNum 1185 snap 398 seq_num 1185) dirty exists 2024-08-17T17:08:42.663 INFO:tasks.rados.rados.0.smithi063.stdout:3298: finishing rollback tid 1 to smithi06387500-4 2024-08-17T17:08:42.663 INFO:tasks.rados.rados.0.smithi063.stdout:3298: finishing rollback tid 2 to smithi06387500-4 2024-08-17T17:08:42.663 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 3216 (ObjNum 1184 snap 398 seq_num 1184) dirty exists 2024-08-17T17:08:42.665 INFO:tasks.rados.rados.0.smithi063.stdout:3298: done (3 left) 2024-08-17T17:08:42.665 INFO:tasks.rados.rados.0.smithi063.stdout:3299: done (2 left) 2024-08-17T17:08:42.665 INFO:tasks.rados.rados.0.smithi063.stdout:3301: done (1 left) 2024-08-17T17:08:42.665 INFO:tasks.rados.rados.0.smithi063.stdout:3302: write oid 27 current snap is 413 2024-08-17T17:08:42.666 INFO:tasks.rados.rados.0.smithi063.stdout:3302: seq_num 1238 ranges {407066=407448,1594943=620755} 2024-08-17T17:08:42.670 INFO:tasks.rados.rados.0.smithi063.stdout:3302: writing smithi06387500-27 from 407066 to 814514 tid 1 2024-08-17T17:08:42.678 INFO:tasks.rados.rados.0.smithi063.stdout:3302: writing smithi06387500-27 from 1594943 to 2215698 tid 2 2024-08-17T17:08:42.678 INFO:tasks.rados.rados.0.smithi063.stdout:3303: delete oid 7 current snap is 413 2024-08-17T17:08:42.678 INFO:tasks.rados.rados.0.smithi063.stdout:3300: finishing write tid 1 to smithi06387500-29 2024-08-17T17:08:42.678 INFO:tasks.rados.rados.0.smithi063.stdout:3300: finishing write tid 2 to smithi06387500-29 2024-08-17T17:08:42.678 INFO:tasks.rados.rados.0.smithi063.stdout:3300: finishing write tid 3 to smithi06387500-29 2024-08-17T17:08:42.678 INFO:tasks.rados.rados.0.smithi063.stdout:3300: finishing write tid 4 to smithi06387500-29 2024-08-17T17:08:42.679 INFO:tasks.rados.rados.0.smithi063.stdout:3300: oid 29 updating version 0 to 739 2024-08-17T17:08:42.679 INFO:tasks.rados.rados.0.smithi063.stdout:3300: oid 29 version 739 is already newer than 737 2024-08-17T17:08:42.679 INFO:tasks.rados.rados.0.smithi063.stdout:3300: oid 29 version 739 is already newer than 738 2024-08-17T17:08:42.679 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 739 (ObjNum 1236 snap 413 seq_num 1236) dirty exists 2024-08-17T17:08:42.679 INFO:tasks.rados.rados.0.smithi063.stdout:3300: left oid 29 (ObjNum 1236 snap 413 seq_num 1236) 2024-08-17T17:08:42.679 INFO:tasks.rados.rados.0.smithi063.stdout:3302: finishing write tid 1 to smithi06387500-27 2024-08-17T17:08:42.680 INFO:tasks.rados.rados.0.smithi063.stdout:3300: done (2 left) 2024-08-17T17:08:42.680 INFO:tasks.rados.rados.0.smithi063.stdout:3303: done (1 left) 2024-08-17T17:08:42.680 INFO:tasks.rados.rados.0.smithi063.stdout:3304: snap_remove snap 386 2024-08-17T17:08:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:42 smithi063 ceph-mon[109843]: osdmap e894: 8 total, 8 up, 8 in 2024-08-17T17:08:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:42 smithi063 ceph-mon[109843]: pgmap v798: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 342 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 516 KiB/s wr, 3 op/s 2024-08-17T17:08:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:42 smithi063 ceph-mon[109843]: osdmap e895: 8 total, 8 up, 8 in 2024-08-17T17:08:43.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:42 smithi063 ceph-mon[114756]: osdmap e894: 8 total, 8 up, 8 in 2024-08-17T17:08:43.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:42 smithi063 ceph-mon[114756]: pgmap v798: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 342 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 516 KiB/s wr, 3 op/s 2024-08-17T17:08:43.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:42 smithi063 ceph-mon[114756]: osdmap e895: 8 total, 8 up, 8 in 2024-08-17T17:08:43.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:42 smithi162 ceph-mon[104668]: osdmap e894: 8 total, 8 up, 8 in 2024-08-17T17:08:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:42 smithi162 ceph-mon[104668]: pgmap v798: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 342 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 516 KiB/s wr, 3 op/s 2024-08-17T17:08:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:42 smithi162 ceph-mon[104668]: osdmap e895: 8 total, 8 up, 8 in 2024-08-17T17:08:43.649 INFO:tasks.rados.rados.0.smithi063.stdout:3304: done (1 left) 2024-08-17T17:08:43.649 INFO:tasks.rados.rados.0.smithi063.stdout:3305: snap_remove snap 404 2024-08-17T17:08:43.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:08:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:08:43] "GET /metrics HTTP/1.1" 200 36343 "" "Prometheus/2.51.0" 2024-08-17T17:08:44.655 INFO:tasks.rados.rados.0.smithi063.stdout:3305: done (1 left) 2024-08-17T17:08:44.655 INFO:tasks.rados.rados.0.smithi063.stdout:3306: snap_create 2024-08-17T17:08:44.655 INFO:tasks.rados.rados.0.smithi063.stdout:3302: finishing write tid 2 to smithi06387500-27 2024-08-17T17:08:44.655 INFO:tasks.rados.rados.0.smithi063.stdout:3302: finishing write tid 3 to smithi06387500-27 2024-08-17T17:08:44.655 INFO:tasks.rados.rados.0.smithi063.stdout:3302: finishing write tid 4 to smithi06387500-27 2024-08-17T17:08:44.655 INFO:tasks.rados.rados.0.smithi063.stdout:3302: oid 27 updating version 0 to 312 2024-08-17T17:08:44.655 INFO:tasks.rados.rados.0.smithi063.stdout:3302: oid 27 updating version 312 to 313 2024-08-17T17:08:44.655 INFO:tasks.rados.rados.0.smithi063.stdout:3302: oid 27 version 313 is already newer than 311 2024-08-17T17:08:44.655 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 313 (ObjNum 1238 snap 413 seq_num 1238) dirty exists 2024-08-17T17:08:44.656 INFO:tasks.rados.rados.0.smithi063.stdout:3302: left oid 27 (ObjNum 1238 snap 413 seq_num 1238) 2024-08-17T17:08:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:44 smithi063 ceph-mon[114756]: osdmap e896: 8 total, 8 up, 8 in 2024-08-17T17:08:45.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:44 smithi063 ceph-mon[109843]: osdmap e896: 8 total, 8 up, 8 in 2024-08-17T17:08:45.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:44 smithi162 ceph-mon[104668]: osdmap e896: 8 total, 8 up, 8 in 2024-08-17T17:08:45.655 INFO:tasks.rados.rados.0.smithi063.stdout:3302: done (1 left) 2024-08-17T17:08:45.655 INFO:tasks.rados.rados.0.smithi063.stdout:3306: done (0 left) 2024-08-17T17:08:45.655 INFO:tasks.rados.rados.0.smithi063.stdout:3307: snap_create 2024-08-17T17:08:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:45 smithi063 ceph-mon[114756]: pgmap v801: 89 pgs: 89 active+clean; 337 MiB data, 1021 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 220 KiB/s wr, 2 op/s 2024-08-17T17:08:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:45 smithi063 ceph-mon[114756]: osdmap e897: 8 total, 8 up, 8 in 2024-08-17T17:08:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:45 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:08:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:45 smithi063 ceph-mon[109843]: pgmap v801: 89 pgs: 89 active+clean; 337 MiB data, 1021 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 220 KiB/s wr, 2 op/s 2024-08-17T17:08:46.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:45 smithi063 ceph-mon[109843]: osdmap e897: 8 total, 8 up, 8 in 2024-08-17T17:08:46.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:45 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:08:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:45 smithi162 ceph-mon[104668]: pgmap v801: 89 pgs: 89 active+clean; 337 MiB data, 1021 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 220 KiB/s wr, 2 op/s 2024-08-17T17:08:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:45 smithi162 ceph-mon[104668]: osdmap e897: 8 total, 8 up, 8 in 2024-08-17T17:08:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:45 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:08:46.667 INFO:tasks.rados.rados.0.smithi063.stdout:3307: done (0 left) 2024-08-17T17:08:46.667 INFO:tasks.rados.rados.0.smithi063.stdout:3308: rollback oid 7 current snap is 415 2024-08-17T17:08:46.667 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 402 2024-08-17T17:08:46.667 INFO:tasks.rados.rados.0.smithi063.stdout:3309: read oid 21 snap -1 2024-08-17T17:08:46.667 INFO:tasks.rados.rados.0.smithi063.stdout:3309: expect (ObjNum 1229 snap 411 seq_num 1229) 2024-08-17T17:08:46.667 INFO:tasks.rados.rados.0.smithi063.stdout:3310: read oid 27 snap -1 2024-08-17T17:08:46.668 INFO:tasks.rados.rados.0.smithi063.stdout:3310: expect (ObjNum 1238 snap 413 seq_num 1238) 2024-08-17T17:08:46.668 INFO:tasks.rados.rados.0.smithi063.stdout:3311: write oid 15 current snap is 415 2024-08-17T17:08:46.668 INFO:tasks.rados.rados.0.smithi063.stdout:3311: seq_num 1240 ranges {440680=674299,1711907=624424} 2024-08-17T17:08:46.673 INFO:tasks.rados.rados.0.smithi063.stdout:3311: writing smithi06387500-15 from 440680 to 1114979 tid 1 2024-08-17T17:08:46.680 INFO:tasks.rados.rados.0.smithi063.stdout:3311: writing smithi06387500-15 from 1711907 to 2336331 tid 2 2024-08-17T17:08:46.681 INFO:tasks.rados.rados.0.smithi063.stdout:3312: snap_create 2024-08-17T17:08:46.681 INFO:tasks.rados.rados.0.smithi063.stdout:3308: finishing rollback tid 1 to smithi06387500-7 2024-08-17T17:08:46.681 INFO:tasks.rados.rados.0.smithi063.stdout:3308: finishing rollback tid 2 to smithi06387500-7 2024-08-17T17:08:46.681 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1588 (ObjNum 1180 snap 398 seq_num 1180) dirty exists 2024-08-17T17:08:46.681 INFO:tasks.rados.rados.0.smithi063.stdout:3310: expect (ObjNum 1238 snap 413 seq_num 1238) 2024-08-17T17:08:46.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:46 smithi063 ceph-mon[114756]: osdmap e898: 8 total, 8 up, 8 in 2024-08-17T17:08:46.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:46.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:46 smithi063 ceph-mon[109843]: osdmap e898: 8 total, 8 up, 8 in 2024-08-17T17:08:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:46.810 INFO:tasks.rados.rados.0.smithi063.stdout:3309: expect (ObjNum 1229 snap 411 seq_num 1229) 2024-08-17T17:08:46.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:46 smithi162 ceph-mon[104668]: osdmap e898: 8 total, 8 up, 8 in 2024-08-17T17:08:46.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:08:46.977 INFO:tasks.rados.rados.0.smithi063.stdout:3311: finishing write tid 1 to smithi06387500-15 2024-08-17T17:08:46.977 INFO:tasks.rados.rados.0.smithi063.stdout:3311: finishing write tid 2 to smithi06387500-15 2024-08-17T17:08:46.978 INFO:tasks.rados.rados.0.smithi063.stdout:3311: finishing write tid 3 to smithi06387500-15 2024-08-17T17:08:46.978 INFO:tasks.rados.rados.0.smithi063.stdout:3311: finishing write tid 4 to smithi06387500-15 2024-08-17T17:08:46.978 INFO:tasks.rados.rados.0.smithi063.stdout:3311: oid 15 updating version 0 to 1169 2024-08-17T17:08:46.978 INFO:tasks.rados.rados.0.smithi063.stdout:3311: oid 15 version 1169 is already newer than 1168 2024-08-17T17:08:46.978 INFO:tasks.rados.rados.0.smithi063.stdout:3311: oid 15 version 1169 is already newer than 1167 2024-08-17T17:08:46.978 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 1169 (ObjNum 1240 snap 415 seq_num 1240) dirty exists 2024-08-17T17:08:46.978 INFO:tasks.rados.rados.0.smithi063.stdout:3311: left oid 15 (ObjNum 1240 snap 415 seq_num 1240) 2024-08-17T17:08:47.648 INFO:tasks.rados.rados.0.smithi063.stdout:3308: done (4 left) 2024-08-17T17:08:47.648 INFO:tasks.rados.rados.0.smithi063.stdout:3309: done (3 left) 2024-08-17T17:08:47.648 INFO:tasks.rados.rados.0.smithi063.stdout:3310: done (2 left) 2024-08-17T17:08:47.649 INFO:tasks.rados.rados.0.smithi063.stdout:3311: done (1 left) 2024-08-17T17:08:47.649 INFO:tasks.rados.rados.0.smithi063.stdout:3312: done (0 left) 2024-08-17T17:08:47.649 INFO:tasks.rados.rados.0.smithi063.stdout:3313: write oid 20 current snap is 416 2024-08-17T17:08:47.649 INFO:tasks.rados.rados.0.smithi063.stdout:3313: seq_num 1241 ranges {457487=749548,1770389=484435,2746287=111200} 2024-08-17T17:08:47.654 INFO:tasks.rados.rados.0.smithi063.stdout:3313: writing smithi06387500-20 from 457487 to 1207035 tid 1 2024-08-17T17:08:47.661 INFO:tasks.rados.rados.0.smithi063.stdout:3313: writing smithi06387500-20 from 1770389 to 2254824 tid 2 2024-08-17T17:08:47.664 INFO:tasks.rados.rados.0.smithi063.stdout:3313: writing smithi06387500-20 from 2746287 to 2857487 tid 3 2024-08-17T17:08:47.664 INFO:tasks.rados.rados.0.smithi063.stdout:3314: read oid 32 snap -1 2024-08-17T17:08:47.665 INFO:tasks.rados.rados.0.smithi063.stdout:3314: expect (ObjNum 1208 snap 404 seq_num 1208) 2024-08-17T17:08:47.665 INFO:tasks.rados.rados.0.smithi063.stdout:3315: snap_create 2024-08-17T17:08:47.665 INFO:tasks.rados.rados.0.smithi063.stdout:3313: finishing write tid 1 to smithi06387500-20 2024-08-17T17:08:47.665 INFO:tasks.rados.rados.0.smithi063.stdout:3313: finishing write tid 2 to smithi06387500-20 2024-08-17T17:08:47.666 INFO:tasks.rados.rados.0.smithi063.stdout:3313: finishing write tid 3 to smithi06387500-20 2024-08-17T17:08:47.667 INFO:tasks.rados.rados.0.smithi063.stdout:3313: finishing write tid 4 to smithi06387500-20 2024-08-17T17:08:47.667 INFO:tasks.rados.rados.0.smithi063.stdout:3313: finishing write tid 5 to smithi06387500-20 2024-08-17T17:08:47.667 INFO:tasks.rados.rados.0.smithi063.stdout:3313: oid 20 updating version 0 to 767 2024-08-17T17:08:47.667 INFO:tasks.rados.rados.0.smithi063.stdout:3313: oid 20 version 767 is already newer than 766 2024-08-17T17:08:47.667 INFO:tasks.rados.rados.0.smithi063.stdout:3313: oid 20 version 767 is already newer than 765 2024-08-17T17:08:47.668 INFO:tasks.rados.rados.0.smithi063.stdout:3313: oid 20 version 767 is already newer than 764 2024-08-17T17:08:47.668 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 767 (ObjNum 1241 snap 416 seq_num 1241) dirty exists 2024-08-17T17:08:47.668 INFO:tasks.rados.rados.0.smithi063.stdout:3313: left oid 20 (ObjNum 1241 snap 416 seq_num 1241) 2024-08-17T17:08:47.672 INFO:tasks.rados.rados.0.smithi063.stdout:3314: expect (ObjNum 1208 snap 404 seq_num 1208) 2024-08-17T17:08:48.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:47 smithi162 ceph-mon[104668]: pgmap v804: 89 pgs: 89 active+clean; 325 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 562 KiB/s wr, 2 op/s 2024-08-17T17:08:48.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:47 smithi162 ceph-mon[104668]: osdmap e899: 8 total, 8 up, 8 in 2024-08-17T17:08:48.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:47 smithi162 ceph-mon[104668]: osdmap e900: 8 total, 8 up, 8 in 2024-08-17T17:08:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:47 smithi063 ceph-mon[114756]: pgmap v804: 89 pgs: 89 active+clean; 325 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 562 KiB/s wr, 2 op/s 2024-08-17T17:08:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:47 smithi063 ceph-mon[114756]: osdmap e899: 8 total, 8 up, 8 in 2024-08-17T17:08:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:47 smithi063 ceph-mon[114756]: osdmap e900: 8 total, 8 up, 8 in 2024-08-17T17:08:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:47 smithi063 ceph-mon[109843]: pgmap v804: 89 pgs: 89 active+clean; 325 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 562 KiB/s wr, 2 op/s 2024-08-17T17:08:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:47 smithi063 ceph-mon[109843]: osdmap e899: 8 total, 8 up, 8 in 2024-08-17T17:08:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:47 smithi063 ceph-mon[109843]: osdmap e900: 8 total, 8 up, 8 in 2024-08-17T17:08:48.674 INFO:tasks.rados.rados.0.smithi063.stdout:3313: done (2 left) 2024-08-17T17:08:48.674 INFO:tasks.rados.rados.0.smithi063.stdout:3314: done (1 left) 2024-08-17T17:08:48.674 INFO:tasks.rados.rados.0.smithi063.stdout:3315: done (0 left) 2024-08-17T17:08:48.675 INFO:tasks.rados.rados.0.smithi063.stdout:3316: rollback oid 49 current snap is 417 2024-08-17T17:08:48.675 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 49 to 398 2024-08-17T17:08:48.675 INFO:tasks.rados.rados.0.smithi063.stdout:3317: delete oid 35 current snap is 417 2024-08-17T17:08:48.680 INFO:tasks.rados.rados.0.smithi063.stdout:3316: finishing rollback tid 0 to smithi06387500-49 2024-08-17T17:08:48.681 INFO:tasks.rados.rados.0.smithi063.stdout:3316: finishing rollback tid 1 to smithi06387500-49 2024-08-17T17:08:48.681 INFO:tasks.rados.rados.0.smithi063.stdout:3316: finishing rollback tid 2 to smithi06387500-49 2024-08-17T17:08:48.681 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 3236 (ObjNum 1101 snap 366 seq_num 1101) dirty exists 2024-08-17T17:08:48.682 INFO:tasks.rados.rados.0.smithi063.stdout:3316: done (1 left) 2024-08-17T17:08:48.682 INFO:tasks.rados.rados.0.smithi063.stdout:3317: done (0 left) 2024-08-17T17:08:48.682 INFO:tasks.rados.rados.0.smithi063.stdout:3318: rollback oid 46 current snap is 417 2024-08-17T17:08:48.682 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 46 to 414 2024-08-17T17:08:48.682 INFO:tasks.rados.rados.0.smithi063.stdout:3319: snap_create 2024-08-17T17:08:48.686 INFO:tasks.rados.rados.0.smithi063.stdout:3318: finishing rollback tid 0 to smithi06387500-46 2024-08-17T17:08:48.686 INFO:tasks.rados.rados.0.smithi063.stdout:3318: finishing rollback tid 1 to smithi06387500-46 2024-08-17T17:08:48.687 INFO:tasks.rados.rados.0.smithi063.stdout:3318: finishing rollback tid 2 to smithi06387500-46 2024-08-17T17:08:48.687 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 1173 (ObjNum 1134 snap 377 seq_num 1134) dirty exists 2024-08-17T17:08:49.673 INFO:tasks.rados.rados.0.smithi063.stdout:3318: done (1 left) 2024-08-17T17:08:49.673 INFO:tasks.rados.rados.0.smithi063.stdout:3319: done (0 left) 2024-08-17T17:08:49.674 INFO:tasks.rados.rados.0.smithi063.stdout:3320: snap_create 2024-08-17T17:08:49.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:49 smithi162 ceph-mon[104668]: pgmap v807: 89 pgs: 89 active+clean; 324 MiB data, 960 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 562 KiB/s wr, 4 op/s 2024-08-17T17:08:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:49 smithi162 ceph-mon[104668]: osdmap e901: 8 total, 8 up, 8 in 2024-08-17T17:08:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:49 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:49 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:49 smithi063 ceph-mon[114756]: pgmap v807: 89 pgs: 89 active+clean; 324 MiB data, 960 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 562 KiB/s wr, 4 op/s 2024-08-17T17:08:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:49 smithi063 ceph-mon[114756]: osdmap e901: 8 total, 8 up, 8 in 2024-08-17T17:08:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:49 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:50.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:49 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:49 smithi063 ceph-mon[109843]: pgmap v807: 89 pgs: 89 active+clean; 324 MiB data, 960 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 562 KiB/s wr, 4 op/s 2024-08-17T17:08:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:49 smithi063 ceph-mon[109843]: osdmap e901: 8 total, 8 up, 8 in 2024-08-17T17:08:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:49 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:49 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:50.681 INFO:tasks.rados.rados.0.smithi063.stdout:3320: done (0 left) 2024-08-17T17:08:50.681 INFO:tasks.rados.rados.0.smithi063.stdout:3321: snap_remove snap 409 2024-08-17T17:08:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:50 smithi063 ceph-mon[109843]: osdmap e902: 8 total, 8 up, 8 in 2024-08-17T17:08:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:50 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:50 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:50 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:08:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:50 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:08:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:50 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:50 smithi063 ceph-mon[114756]: osdmap e902: 8 total, 8 up, 8 in 2024-08-17T17:08:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:50 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:50 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:50 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:08:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:50 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:08:51.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:50 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:51.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:50 smithi162 ceph-mon[104668]: osdmap e902: 8 total, 8 up, 8 in 2024-08-17T17:08:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:50 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:50 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:50 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:08:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:50 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:08:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:50 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:08:51.685 INFO:tasks.rados.rados.0.smithi063.stdout:3321: done (0 left) 2024-08-17T17:08:51.685 INFO:tasks.rados.rados.0.smithi063.stdout:3322: snap_remove snap 408 2024-08-17T17:08:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:51 smithi063 ceph-mon[109843]: pgmap v810: 89 pgs: 89 active+clean; 324 MiB data, 959 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 0 B/s wr, 4 op/s 2024-08-17T17:08:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:51 smithi063 ceph-mon[109843]: osdmap e903: 8 total, 8 up, 8 in 2024-08-17T17:08:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:51 smithi063 ceph-mon[114756]: pgmap v810: 89 pgs: 89 active+clean; 324 MiB data, 959 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 0 B/s wr, 4 op/s 2024-08-17T17:08:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:51 smithi063 ceph-mon[114756]: osdmap e903: 8 total, 8 up, 8 in 2024-08-17T17:08:52.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:51 smithi162 ceph-mon[104668]: pgmap v810: 89 pgs: 89 active+clean; 324 MiB data, 959 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 0 B/s wr, 4 op/s 2024-08-17T17:08:52.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:51 smithi162 ceph-mon[104668]: osdmap e903: 8 total, 8 up, 8 in 2024-08-17T17:08:52.650 INFO:tasks.rados.rados.0.smithi063.stdout:3322: done (0 left) 2024-08-17T17:08:52.650 INFO:tasks.rados.rados.0.smithi063.stdout:3323: rollback oid 50 current snap is 419 2024-08-17T17:08:52.650 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 50 to 378 2024-08-17T17:08:52.651 INFO:tasks.rados.rados.0.smithi063.stdout:3324: rollback oid 12 current snap is 419 2024-08-17T17:08:52.651 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 12 to 399 2024-08-17T17:08:52.651 INFO:tasks.rados.rados.0.smithi063.stdout:3325: delete oid 33 current snap is 419 2024-08-17T17:08:52.658 INFO:tasks.rados.rados.0.smithi063.stdout:3323: finishing rollback tid 0 to smithi06387500-50 2024-08-17T17:08:52.658 INFO:tasks.rados.rados.0.smithi063.stdout:3325: done (2 left) 2024-08-17T17:08:52.658 INFO:tasks.rados.rados.0.smithi063.stdout:3326: snap_create 2024-08-17T17:08:52.660 INFO:tasks.rados.rados.0.smithi063.stdout:3323: finishing rollback tid 1 to smithi06387500-50 2024-08-17T17:08:52.660 INFO:tasks.rados.rados.0.smithi063.stdout:3323: finishing rollback tid 2 to smithi06387500-50 2024-08-17T17:08:52.660 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 2130 (ObjNum 1088 snap 363 seq_num 1088) dirty exists 2024-08-17T17:08:52.661 INFO:tasks.rados.rados.0.smithi063.stdout:3324: finishing rollback tid 0 to smithi06387500-12 2024-08-17T17:08:52.661 INFO:tasks.rados.rados.0.smithi063.stdout:3324: finishing rollback tid 1 to smithi06387500-12 2024-08-17T17:08:52.662 INFO:tasks.rados.rados.0.smithi063.stdout:3324: finishing rollback tid 2 to smithi06387500-12 2024-08-17T17:08:52.662 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 747 (ObjNum 1166 snap 393 seq_num 1166) dirty exists 2024-08-17T17:08:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:52 smithi063 ceph-mon[114756]: osdmap e904: 8 total, 8 up, 8 in 2024-08-17T17:08:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:52 smithi063 ceph-mon[114756]: pgmap v813: 89 pgs: 89 active+clean; 327 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 646 KiB/s wr, 10 op/s 2024-08-17T17:08:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:52 smithi063 ceph-mon[114756]: osdmap e905: 8 total, 8 up, 8 in 2024-08-17T17:08:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:52 smithi063 ceph-mon[109843]: osdmap e904: 8 total, 8 up, 8 in 2024-08-17T17:08:53.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:52 smithi063 ceph-mon[109843]: pgmap v813: 89 pgs: 89 active+clean; 327 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 646 KiB/s wr, 10 op/s 2024-08-17T17:08:53.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:52 smithi063 ceph-mon[109843]: osdmap e905: 8 total, 8 up, 8 in 2024-08-17T17:08:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:52 smithi162 ceph-mon[104668]: osdmap e904: 8 total, 8 up, 8 in 2024-08-17T17:08:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:52 smithi162 ceph-mon[104668]: pgmap v813: 89 pgs: 89 active+clean; 327 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 646 KiB/s wr, 10 op/s 2024-08-17T17:08:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:52 smithi162 ceph-mon[104668]: osdmap e905: 8 total, 8 up, 8 in 2024-08-17T17:08:53.654 INFO:tasks.rados.rados.0.smithi063.stdout:3323: done (2 left) 2024-08-17T17:08:53.655 INFO:tasks.rados.rados.0.smithi063.stdout:3324: done (1 left) 2024-08-17T17:08:53.655 INFO:tasks.rados.rados.0.smithi063.stdout:3326: done (0 left) 2024-08-17T17:08:53.655 INFO:tasks.rados.rados.0.smithi063.stdout:3327: snap_create 2024-08-17T17:08:53.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:08:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:08:53] "GET /metrics HTTP/1.1" 200 36342 "" "Prometheus/2.51.0" 2024-08-17T17:08:54.663 INFO:tasks.rados.rados.0.smithi063.stdout:3327: done (0 left) 2024-08-17T17:08:54.663 INFO:tasks.rados.rados.0.smithi063.stdout:3328: snap_remove snap 416 2024-08-17T17:08:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:54 smithi063 ceph-mon[114756]: osdmap e906: 8 total, 8 up, 8 in 2024-08-17T17:08:55.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:54 smithi063 ceph-mon[109843]: osdmap e906: 8 total, 8 up, 8 in 2024-08-17T17:08:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:54 smithi162 ceph-mon[104668]: osdmap e906: 8 total, 8 up, 8 in 2024-08-17T17:08:55.663 INFO:tasks.rados.rados.0.smithi063.stdout:3328: done (0 left) 2024-08-17T17:08:55.663 INFO:tasks.rados.rados.0.smithi063.stdout:3329: snap_remove snap 414 2024-08-17T17:08:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:55 smithi063 ceph-mon[114756]: pgmap v816: 89 pgs: 89 active+clean; 327 MiB data, 966 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 646 KiB/s wr, 8 op/s 2024-08-17T17:08:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:55 smithi063 ceph-mon[114756]: osdmap e907: 8 total, 8 up, 8 in 2024-08-17T17:08:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:55 smithi063 ceph-mon[109843]: pgmap v816: 89 pgs: 89 active+clean; 327 MiB data, 966 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 646 KiB/s wr, 8 op/s 2024-08-17T17:08:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:55 smithi063 ceph-mon[109843]: osdmap e907: 8 total, 8 up, 8 in 2024-08-17T17:08:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:55 smithi162 ceph-mon[104668]: pgmap v816: 89 pgs: 89 active+clean; 327 MiB data, 966 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 646 KiB/s wr, 8 op/s 2024-08-17T17:08:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:55 smithi162 ceph-mon[104668]: osdmap e907: 8 total, 8 up, 8 in 2024-08-17T17:08:56.678 INFO:tasks.rados.rados.0.smithi063.stdout:3329: done (0 left) 2024-08-17T17:08:56.678 INFO:tasks.rados.rados.0.smithi063.stdout:3330: snap_create 2024-08-17T17:08:57.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:56 smithi063 ceph-mon[114756]: osdmap e908: 8 total, 8 up, 8 in 2024-08-17T17:08:57.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:56 smithi063 ceph-mon[109843]: osdmap e908: 8 total, 8 up, 8 in 2024-08-17T17:08:57.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:56 smithi162 ceph-mon[104668]: osdmap e908: 8 total, 8 up, 8 in 2024-08-17T17:08:57.652 INFO:tasks.rados.rados.0.smithi063.stdout:3330: done (0 left) 2024-08-17T17:08:57.653 INFO:tasks.rados.rados.0.smithi063.stdout:3331: snap_remove snap 415 2024-08-17T17:08:58.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:57 smithi063 ceph-mon[109843]: pgmap v819: 89 pgs: 89 active+clean; 328 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2024-08-17T17:08:58.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:57 smithi063 ceph-mon[109843]: osdmap e909: 8 total, 8 up, 8 in 2024-08-17T17:08:58.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:57 smithi063 ceph-mon[109843]: osdmap e910: 8 total, 8 up, 8 in 2024-08-17T17:08:58.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:57 smithi063 ceph-mon[114756]: pgmap v819: 89 pgs: 89 active+clean; 328 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2024-08-17T17:08:58.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:57 smithi063 ceph-mon[114756]: osdmap e909: 8 total, 8 up, 8 in 2024-08-17T17:08:58.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:57 smithi063 ceph-mon[114756]: osdmap e910: 8 total, 8 up, 8 in 2024-08-17T17:08:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:57 smithi162 ceph-mon[104668]: pgmap v819: 89 pgs: 89 active+clean; 328 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 1 op/s 2024-08-17T17:08:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:57 smithi162 ceph-mon[104668]: osdmap e909: 8 total, 8 up, 8 in 2024-08-17T17:08:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:57 smithi162 ceph-mon[104668]: osdmap e910: 8 total, 8 up, 8 in 2024-08-17T17:08:58.665 INFO:tasks.rados.rados.0.smithi063.stdout:3331: done (0 left) 2024-08-17T17:08:58.665 INFO:tasks.rados.rados.0.smithi063.stdout:3332: read oid 42 snap -1 2024-08-17T17:08:58.666 INFO:tasks.rados.rados.0.smithi063.stdout:3332: expect (ObjNum 1170 snap 395 seq_num 1170) 2024-08-17T17:08:58.666 INFO:tasks.rados.rados.0.smithi063.stdout:3333: rollback oid 23 current snap is 422 2024-08-17T17:08:58.666 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 23 to 419 2024-08-17T17:08:58.666 INFO:tasks.rados.rados.0.smithi063.stdout:3334: delete oid 41 current snap is 422 2024-08-17T17:08:58.672 INFO:tasks.rados.rados.0.smithi063.stdout:3332: expect (ObjNum 1170 snap 395 seq_num 1170) 2024-08-17T17:08:58.850 INFO:tasks.rados.rados.0.smithi063.stdout:3333: finishing rollback tid 0 to smithi06387500-23 2024-08-17T17:08:58.850 INFO:tasks.rados.rados.0.smithi063.stdout:3333: finishing rollback tid 1 to smithi06387500-23 2024-08-17T17:08:58.850 INFO:tasks.rados.rados.0.smithi063.stdout:3333: finishing rollback tid 2 to smithi06387500-23 2024-08-17T17:08:58.850 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 1709 (ObjNum 1213 snap 406 seq_num 1213) dirty exists 2024-08-17T17:08:58.850 INFO:tasks.rados.rados.0.smithi063.stdout:3332: done (2 left) 2024-08-17T17:08:58.851 INFO:tasks.rados.rados.0.smithi063.stdout:3333: done (1 left) 2024-08-17T17:08:58.851 INFO:tasks.rados.rados.0.smithi063.stdout:3334: done (0 left) 2024-08-17T17:08:58.851 INFO:tasks.rados.rados.0.smithi063.stdout:3335: delete oid 46 current snap is 422 2024-08-17T17:08:58.852 INFO:tasks.rados.rados.0.smithi063.stdout:3335: done (0 left) 2024-08-17T17:08:58.852 INFO:tasks.rados.rados.0.smithi063.stdout:3336: read oid 31 snap 398 2024-08-17T17:08:58.852 INFO:tasks.rados.rados.0.smithi063.stdout:3336: expect (ObjNum 1169 snap 394 seq_num 1169) 2024-08-17T17:08:58.852 INFO:tasks.rados.rados.0.smithi063.stdout:3337: delete oid 8 current snap is 422 2024-08-17T17:08:58.854 INFO:tasks.rados.rados.0.smithi063.stdout:3337: done (1 left) 2024-08-17T17:08:58.854 INFO:tasks.rados.rados.0.smithi063.stdout:3338: write oid 40 current snap is 422 2024-08-17T17:08:58.854 INFO:tasks.rados.rados.0.smithi063.stdout:3338: seq_num 1247 ranges {558329=517395,1721281=779089} 2024-08-17T17:08:58.859 INFO:tasks.rados.rados.0.smithi063.stdout:3338: writing smithi06387500-40 from 558329 to 1075724 tid 1 2024-08-17T17:08:58.867 INFO:tasks.rados.rados.0.smithi063.stdout:3338: writing smithi06387500-40 from 1721281 to 2500370 tid 2 2024-08-17T17:08:58.868 INFO:tasks.rados.rados.0.smithi063.stdout:3339: snap_create 2024-08-17T17:08:58.868 INFO:tasks.rados.rados.0.smithi063.stdout:3336: expect (ObjNum 1169 snap 394 seq_num 1169) 2024-08-17T17:08:58.986 INFO:tasks.rados.rados.0.smithi063.stdout:3338: finishing write tid 1 to smithi06387500-40 2024-08-17T17:08:58.987 INFO:tasks.rados.rados.0.smithi063.stdout:3338: finishing write tid 2 to smithi06387500-40 2024-08-17T17:08:58.987 INFO:tasks.rados.rados.0.smithi063.stdout:3338: finishing write tid 3 to smithi06387500-40 2024-08-17T17:08:58.987 INFO:tasks.rados.rados.0.smithi063.stdout:3338: finishing write tid 4 to smithi06387500-40 2024-08-17T17:08:58.987 INFO:tasks.rados.rados.0.smithi063.stdout:3338: oid 40 updating version 0 to 1118 2024-08-17T17:08:58.987 INFO:tasks.rados.rados.0.smithi063.stdout:3338: oid 40 version 1118 is already newer than 1117 2024-08-17T17:08:58.987 INFO:tasks.rados.rados.0.smithi063.stdout:3338: oid 40 version 1118 is already newer than 1116 2024-08-17T17:08:58.987 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 1118 (ObjNum 1247 snap 422 seq_num 1247) dirty exists 2024-08-17T17:08:58.987 INFO:tasks.rados.rados.0.smithi063.stdout:3338: left oid 40 (ObjNum 1247 snap 422 seq_num 1247) 2024-08-17T17:08:59.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:08:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=cleanup t=2024-08-17T17:08:59.006931406Z level=info msg="Completed cleanup jobs" duration=2.868026ms 2024-08-17T17:08:59.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:08:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=sqlstore.transactions t=2024-08-17T17:08:59.014296399Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T17:08:59.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:08:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=grafana.update.checker t=2024-08-17T17:08:59.140087619Z level=info msg="Update check succeeded" duration=64.103486ms 2024-08-17T17:08:59.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:08:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=plugins.update.checker t=2024-08-17T17:08:59.155497059Z level=info msg="Update check succeeded" duration=69.3952ms 2024-08-17T17:08:59.675 INFO:tasks.rados.rados.0.smithi063.stdout:3336: done (2 left) 2024-08-17T17:08:59.675 INFO:tasks.rados.rados.0.smithi063.stdout:3338: done (1 left) 2024-08-17T17:08:59.675 INFO:tasks.rados.rados.0.smithi063.stdout:3339: done (0 left) 2024-08-17T17:08:59.675 INFO:tasks.rados.rados.0.smithi063.stdout:3340: read oid 33 snap 419 2024-08-17T17:08:59.676 INFO:tasks.rados.rados.0.smithi063.stdout:3340: expect deleted 2024-08-17T17:08:59.676 INFO:tasks.rados.rados.0.smithi063.stdout:3341: read oid 22 snap -1 2024-08-17T17:08:59.676 INFO:tasks.rados.rados.0.smithi063.stdout:3341: expect deleted 2024-08-17T17:08:59.676 INFO:tasks.rados.rados.0.smithi063.stdout:3342: write oid 8 current snap is 423 2024-08-17T17:08:59.676 INFO:tasks.rados.rados.0.smithi063.stdout:3342: seq_num 1248 ranges {575136=592644,1663410=639100,2705542=269594} 2024-08-17T17:08:59.681 INFO:tasks.rados.rados.0.smithi063.stdout:3342: writing smithi06387500-8 from 575136 to 1167780 tid 1 2024-08-17T17:08:59.687 INFO:tasks.rados.rados.0.smithi063.stdout:3342: writing smithi06387500-8 from 1663410 to 2302510 tid 2 2024-08-17T17:08:59.692 INFO:tasks.rados.rados.0.smithi063.stdout:3342: writing smithi06387500-8 from 2705542 to 2975136 tid 3 2024-08-17T17:08:59.692 INFO:tasks.rados.rados.0.smithi063.stdout:3343: rollback oid 42 current snap is 423 2024-08-17T17:08:59.692 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 42 to 410 2024-08-17T17:08:59.692 INFO:tasks.rados.rados.0.smithi063.stdout:3342: finishing write tid 1 to smithi06387500-8 2024-08-17T17:08:59.692 INFO:tasks.rados.rados.0.smithi063.stdout:3342: finishing write tid 2 to smithi06387500-8 2024-08-17T17:08:59.692 INFO:tasks.rados.rados.0.smithi063.stdout:3340: done (3 left) 2024-08-17T17:08:59.692 INFO:tasks.rados.rados.0.smithi063.stdout:3341: done (2 left) 2024-08-17T17:08:59.692 INFO:tasks.rados.rados.0.smithi063.stdout:3344: read oid 38 snap -1 2024-08-17T17:08:59.692 INFO:tasks.rados.rados.0.smithi063.stdout:3344: expect (ObjNum 1200 snap 404 seq_num 1200) 2024-08-17T17:08:59.693 INFO:tasks.rados.rados.0.smithi063.stdout:3345: snap_remove snap 399 2024-08-17T17:09:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:59 smithi063 ceph-mon[114756]: pgmap v822: 89 pgs: 89 active+clean; 326 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:09:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:08:59 smithi063 ceph-mon[114756]: osdmap e911: 8 total, 8 up, 8 in 2024-08-17T17:09:00.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:59 smithi063 ceph-mon[109843]: pgmap v822: 89 pgs: 89 active+clean; 326 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:09:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:08:59 smithi063 ceph-mon[109843]: osdmap e911: 8 total, 8 up, 8 in 2024-08-17T17:09:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:59 smithi162 ceph-mon[104668]: pgmap v822: 89 pgs: 89 active+clean; 326 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-08-17T17:09:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:08:59 smithi162 ceph-mon[104668]: osdmap e911: 8 total, 8 up, 8 in 2024-08-17T17:09:00.688 INFO:tasks.rados.rados.0.smithi063.stdout:3345: done (3 left) 2024-08-17T17:09:00.688 INFO:tasks.rados.rados.0.smithi063.stdout:3346: rollback oid 11 current snap is 423 2024-08-17T17:09:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 11 to 402 2024-08-17T17:09:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:3342: finishing write tid 3 to smithi06387500-8 2024-08-17T17:09:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:3342: finishing write tid 4 to smithi06387500-8 2024-08-17T17:09:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:3342: finishing write tid 5 to smithi06387500-8 2024-08-17T17:09:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:3342: oid 8 updating version 0 to 1119 2024-08-17T17:09:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:3342: oid 8 updating version 1119 to 1121 2024-08-17T17:09:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:3342: oid 8 updating version 1121 to 1122 2024-08-17T17:09:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:3342: oid 8 version 1122 is already newer than 1120 2024-08-17T17:09:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 1122 (ObjNum 1248 snap 423 seq_num 1248) dirty exists 2024-08-17T17:09:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:3342: left oid 8 (ObjNum 1248 snap 423 seq_num 1248) 2024-08-17T17:09:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:3343: finishing rollback tid 0 to smithi06387500-42 2024-08-17T17:09:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:3344: expect (ObjNum 1200 snap 404 seq_num 1200) 2024-08-17T17:09:00.848 INFO:tasks.rados.rados.0.smithi063.stdout:3343: finishing rollback tid 1 to smithi06387500-42 2024-08-17T17:09:00.848 INFO:tasks.rados.rados.0.smithi063.stdout:3343: finishing rollback tid 2 to smithi06387500-42 2024-08-17T17:09:00.848 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 2140 (ObjNum 1170 snap 395 seq_num 1170) dirty exists 2024-08-17T17:09:00.848 INFO:tasks.rados.rados.0.smithi063.stdout:3342: done (3 left) 2024-08-17T17:09:00.848 INFO:tasks.rados.rados.0.smithi063.stdout:3343: done (2 left) 2024-08-17T17:09:00.848 INFO:tasks.rados.rados.0.smithi063.stdout:3344: done (1 left) 2024-08-17T17:09:00.848 INFO:tasks.rados.rados.0.smithi063.stdout:3347: read oid 47 snap -1 2024-08-17T17:09:00.848 INFO:tasks.rados.rados.0.smithi063.stdout:3347: expect (ObjNum 1230 snap 411 seq_num 1230) 2024-08-17T17:09:00.849 INFO:tasks.rados.rados.0.smithi063.stdout:3348: rollback oid 34 current snap is 423 2024-08-17T17:09:00.849 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 34 to 398 2024-08-17T17:09:00.849 INFO:tasks.rados.rados.0.smithi063.stdout:3346: finishing rollback tid 0 to smithi06387500-11 2024-08-17T17:09:00.849 INFO:tasks.rados.rados.0.smithi063.stdout:3346: finishing rollback tid 1 to smithi06387500-11 2024-08-17T17:09:00.849 INFO:tasks.rados.rados.0.smithi063.stdout:3346: finishing rollback tid 2 to smithi06387500-11 2024-08-17T17:09:00.849 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 609 (ObjNum 1163 snap 392 seq_num 1163) dirty exists 2024-08-17T17:09:00.849 INFO:tasks.rados.rados.0.smithi063.stdout:3346: done (2 left) 2024-08-17T17:09:00.849 INFO:tasks.rados.rados.0.smithi063.stdout:3349: read oid 44 snap -1 2024-08-17T17:09:00.849 INFO:tasks.rados.rados.0.smithi063.stdout:3349: expect (ObjNum 1218 snap 407 seq_num 1218) 2024-08-17T17:09:00.849 INFO:tasks.rados.rados.0.smithi063.stdout:3350: read oid 33 snap -1 2024-08-17T17:09:00.849 INFO:tasks.rados.rados.0.smithi063.stdout:3350: expect deleted 2024-08-17T17:09:00.849 INFO:tasks.rados.rados.0.smithi063.stdout:3351: delete oid 30 current snap is 423 2024-08-17T17:09:00.850 INFO:tasks.rados.rados.0.smithi063.stdout:3348: finishing rollback tid 1 to smithi06387500-34 2024-08-17T17:09:00.850 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-08-17T17:09:00.850 INFO:tasks.rados.rados.0.smithi063.stdout:3348: done (4 left) 2024-08-17T17:09:00.850 INFO:tasks.rados.rados.0.smithi063.stdout:3350: done (3 left) 2024-08-17T17:09:00.850 INFO:tasks.rados.rados.0.smithi063.stdout:3351: done (2 left) 2024-08-17T17:09:00.850 INFO:tasks.rados.rados.0.smithi063.stdout:3352: delete oid 12 current snap is 423 2024-08-17T17:09:00.850 INFO:tasks.rados.rados.0.smithi063.stdout:3347: expect (ObjNum 1230 snap 411 seq_num 1230) 2024-08-17T17:09:00.939 INFO:tasks.rados.rados.0.smithi063.stdout:3349: expect (ObjNum 1218 snap 407 seq_num 1218) 2024-08-17T17:09:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:00 smithi063 ceph-mon[114756]: osdmap e912: 8 total, 8 up, 8 in 2024-08-17T17:09:01.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:00 smithi063 ceph-mon[109843]: osdmap e912: 8 total, 8 up, 8 in 2024-08-17T17:09:01.091 INFO:tasks.rados.rados.0.smithi063.stdout:3347: done (2 left) 2024-08-17T17:09:01.091 INFO:tasks.rados.rados.0.smithi063.stdout:3349: done (1 left) 2024-08-17T17:09:01.091 INFO:tasks.rados.rados.0.smithi063.stdout:3352: done (0 left) 2024-08-17T17:09:01.091 INFO:tasks.rados.rados.0.smithi063.stdout:3353: snap_remove snap 417 2024-08-17T17:09:01.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:00 smithi162 ceph-mon[104668]: osdmap e912: 8 total, 8 up, 8 in 2024-08-17T17:09:01.689 INFO:tasks.rados.rados.0.smithi063.stdout:3353: done (0 left) 2024-08-17T17:09:01.689 INFO:tasks.rados.rados.0.smithi063.stdout:3354: snap_remove snap 393 2024-08-17T17:09:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:01 smithi063 ceph-mon[114756]: pgmap v825: 89 pgs: 89 active+clean; 326 MiB data, 963 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:09:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:01 smithi063 ceph-mon[114756]: osdmap e913: 8 total, 8 up, 8 in 2024-08-17T17:09:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:01 smithi063 ceph-mon[109843]: pgmap v825: 89 pgs: 89 active+clean; 326 MiB data, 963 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:09:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:01 smithi063 ceph-mon[109843]: osdmap e913: 8 total, 8 up, 8 in 2024-08-17T17:09:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:01 smithi162 ceph-mon[104668]: pgmap v825: 89 pgs: 89 active+clean; 326 MiB data, 963 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:09:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:01 smithi162 ceph-mon[104668]: osdmap e913: 8 total, 8 up, 8 in 2024-08-17T17:09:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:02.655 INFO:tasks.rados.rados.0.smithi063.stdout:3354: done (0 left) 2024-08-17T17:09:02.655 INFO:tasks.rados.rados.0.smithi063.stdout:3355: delete oid 3 current snap is 423 2024-08-17T17:09:02.659 INFO:tasks.rados.rados.0.smithi063.stdout:3355: done (0 left) 2024-08-17T17:09:02.659 INFO:tasks.rados.rados.0.smithi063.stdout:3356: write oid 22 current snap is 423 2024-08-17T17:09:02.659 INFO:tasks.rados.rados.0.smithi063.stdout:3356: seq_num 1252 ranges {642364=493640,1897338=646438} 2024-08-17T17:09:02.666 INFO:tasks.rados.rados.0.smithi063.stdout:3356: writing smithi06387500-22 from 642364 to 1136004 tid 1 2024-08-17T17:09:02.675 INFO:tasks.rados.rados.0.smithi063.stdout:3356: writing smithi06387500-22 from 1897338 to 2543776 tid 2 2024-08-17T17:09:02.675 INFO:tasks.rados.rados.0.smithi063.stdout:3357: write oid 45 current snap is 423 2024-08-17T17:09:02.675 INFO:tasks.rados.rados.0.smithi063.stdout:3357: seq_num 1253 ranges {659171=568889,1839467=790096} 2024-08-17T17:09:02.681 INFO:tasks.rados.rados.0.smithi063.stdout:3357: writing smithi06387500-45 from 659171 to 1228060 tid 1 2024-08-17T17:09:02.689 INFO:tasks.rados.rados.0.smithi063.stdout:3357: writing smithi06387500-45 from 1839467 to 2629563 tid 2 2024-08-17T17:09:02.690 INFO:tasks.rados.rados.0.smithi063.stdout:3358: write oid 10 current snap is 423 2024-08-17T17:09:02.690 INFO:tasks.rados.rados.0.smithi063.stdout:3356: finishing write tid 1 to smithi06387500-22 2024-08-17T17:09:02.690 INFO:tasks.rados.rados.0.smithi063.stdout:3358: seq_num 1254 ranges {675978=644138,1897949=650107} 2024-08-17T17:09:02.697 INFO:tasks.rados.rados.0.smithi063.stdout:3358: writing smithi06387500-10 from 675978 to 1320116 tid 1 2024-08-17T17:09:02.704 INFO:tasks.rados.rados.0.smithi063.stdout:3358: writing smithi06387500-10 from 1897949 to 2548056 tid 2 2024-08-17T17:09:02.704 INFO:tasks.rados.rados.0.smithi063.stdout:3359: write oid 36 current snap is 423 2024-08-17T17:09:02.704 INFO:tasks.rados.rados.0.smithi063.stdout:3359: seq_num 1255 ranges {692785=435740,1672784=793765,2897503=195282} 2024-08-17T17:09:02.710 INFO:tasks.rados.rados.0.smithi063.stdout:3359: writing smithi06387500-36 from 692785 to 1128525 tid 1 2024-08-17T17:09:02.717 INFO:tasks.rados.rados.0.smithi063.stdout:3359: writing smithi06387500-36 from 1672784 to 2466549 tid 2 2024-08-17T17:09:02.722 INFO:tasks.rados.rados.0.smithi063.stdout:3359: writing smithi06387500-36 from 2897503 to 3092785 tid 3 2024-08-17T17:09:02.722 INFO:tasks.rados.rados.0.smithi063.stdout:3360: read oid 5 snap 422 2024-08-17T17:09:02.722 INFO:tasks.rados.rados.0.smithi063.stdout:3360: expect deleted 2024-08-17T17:09:02.722 INFO:tasks.rados.rados.0.smithi063.stdout:3361: delete oid 38 current snap is 423 2024-08-17T17:09:02.722 INFO:tasks.rados.rados.0.smithi063.stdout:3356: finishing write tid 2 to smithi06387500-22 2024-08-17T17:09:02.722 INFO:tasks.rados.rados.0.smithi063.stdout:3356: finishing write tid 3 to smithi06387500-22 2024-08-17T17:09:02.722 INFO:tasks.rados.rados.0.smithi063.stdout:3356: finishing write tid 4 to smithi06387500-22 2024-08-17T17:09:02.722 INFO:tasks.rados.rados.0.smithi063.stdout:3356: oid 22 updating version 0 to 768 2024-08-17T17:09:02.722 INFO:tasks.rados.rados.0.smithi063.stdout:3356: oid 22 updating version 768 to 769 2024-08-17T17:09:02.723 INFO:tasks.rados.rados.0.smithi063.stdout:3356: oid 22 updating version 769 to 770 2024-08-17T17:09:02.723 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 770 (ObjNum 1252 snap 423 seq_num 1252) dirty exists 2024-08-17T17:09:02.723 INFO:tasks.rados.rados.0.smithi063.stdout:3356: left oid 22 (ObjNum 1252 snap 423 seq_num 1252) 2024-08-17T17:09:02.723 INFO:tasks.rados.rados.0.smithi063.stdout:3357: finishing write tid 1 to smithi06387500-45 2024-08-17T17:09:02.723 INFO:tasks.rados.rados.0.smithi063.stdout:3357: finishing write tid 2 to smithi06387500-45 2024-08-17T17:09:02.723 INFO:tasks.rados.rados.0.smithi063.stdout:3357: finishing write tid 3 to smithi06387500-45 2024-08-17T17:09:02.723 INFO:tasks.rados.rados.0.smithi063.stdout:3357: finishing write tid 4 to smithi06387500-45 2024-08-17T17:09:02.723 INFO:tasks.rados.rados.0.smithi063.stdout:3357: oid 45 updating version 0 to 820 2024-08-17T17:09:02.723 INFO:tasks.rados.rados.0.smithi063.stdout:3357: oid 45 updating version 820 to 821 2024-08-17T17:09:02.723 INFO:tasks.rados.rados.0.smithi063.stdout:3357: oid 45 version 821 is already newer than 819 2024-08-17T17:09:02.723 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 821 (ObjNum 1253 snap 423 seq_num 1253) dirty exists 2024-08-17T17:09:02.723 INFO:tasks.rados.rados.0.smithi063.stdout:3357: left oid 45 (ObjNum 1253 snap 423 seq_num 1253) 2024-08-17T17:09:02.723 INFO:tasks.rados.rados.0.smithi063.stdout:3358: finishing write tid 1 to smithi06387500-10 2024-08-17T17:09:02.723 INFO:tasks.rados.rados.0.smithi063.stdout:3358: finishing write tid 2 to smithi06387500-10 2024-08-17T17:09:02.724 INFO:tasks.rados.rados.0.smithi063.stdout:3358: finishing write tid 3 to smithi06387500-10 2024-08-17T17:09:02.724 INFO:tasks.rados.rados.0.smithi063.stdout:3358: finishing write tid 4 to smithi06387500-10 2024-08-17T17:09:02.724 INFO:tasks.rados.rados.0.smithi063.stdout:3358: oid 10 updating version 0 to 1133 2024-08-17T17:09:02.724 INFO:tasks.rados.rados.0.smithi063.stdout:3358: oid 10 updating version 1133 to 1135 2024-08-17T17:09:02.724 INFO:tasks.rados.rados.0.smithi063.stdout:3358: oid 10 version 1135 is already newer than 1134 2024-08-17T17:09:02.724 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 1135 (ObjNum 1254 snap 423 seq_num 1254) dirty exists 2024-08-17T17:09:02.724 INFO:tasks.rados.rados.0.smithi063.stdout:3358: left oid 10 (ObjNum 1254 snap 423 seq_num 1254) 2024-08-17T17:09:02.724 INFO:tasks.rados.rados.0.smithi063.stdout:3359: finishing write tid 1 to smithi06387500-36 2024-08-17T17:09:02.724 INFO:tasks.rados.rados.0.smithi063.stdout:3359: finishing write tid 2 to smithi06387500-36 2024-08-17T17:09:02.725 INFO:tasks.rados.rados.0.smithi063.stdout:3356: done (5 left) 2024-08-17T17:09:02.725 INFO:tasks.rados.rados.0.smithi063.stdout:3357: done (4 left) 2024-08-17T17:09:02.725 INFO:tasks.rados.rados.0.smithi063.stdout:3358: done (3 left) 2024-08-17T17:09:02.725 INFO:tasks.rados.rados.0.smithi063.stdout:3360: done (2 left) 2024-08-17T17:09:02.725 INFO:tasks.rados.rados.0.smithi063.stdout:3361: done (1 left) 2024-08-17T17:09:02.725 INFO:tasks.rados.rados.0.smithi063.stdout:3362: read oid 50 snap 402 2024-08-17T17:09:02.725 INFO:tasks.rados.rados.0.smithi063.stdout:3362: expect (ObjNum 1161 snap 392 seq_num 1161) 2024-08-17T17:09:02.726 INFO:tasks.rados.rados.0.smithi063.stdout:3363: write oid 16 current snap is 423 2024-08-17T17:09:02.726 INFO:tasks.rados.rados.0.smithi063.stdout:3363: seq_num 1257 ranges {726399=586238,2073395=797434} 2024-08-17T17:09:02.730 INFO:tasks.rados.rados.0.smithi063.stdout:3363: writing smithi06387500-16 from 726399 to 1312637 tid 1 2024-08-17T17:09:02.739 INFO:tasks.rados.rados.0.smithi063.stdout:3363: writing smithi06387500-16 from 2073395 to 2870829 tid 2 2024-08-17T17:09:02.739 INFO:tasks.rados.rados.0.smithi063.stdout:3364: snap_create 2024-08-17T17:09:02.739 INFO:tasks.rados.rados.0.smithi063.stdout:3359: finishing write tid 3 to smithi06387500-36 2024-08-17T17:09:02.740 INFO:tasks.rados.rados.0.smithi063.stdout:3359: finishing write tid 4 to smithi06387500-36 2024-08-17T17:09:02.740 INFO:tasks.rados.rados.0.smithi063.stdout:3359: finishing write tid 5 to smithi06387500-36 2024-08-17T17:09:02.740 INFO:tasks.rados.rados.0.smithi063.stdout:3359: oid 36 updating version 0 to 3275 2024-08-17T17:09:02.740 INFO:tasks.rados.rados.0.smithi063.stdout:3359: oid 36 version 3275 is already newer than 3274 2024-08-17T17:09:02.740 INFO:tasks.rados.rados.0.smithi063.stdout:3359: oid 36 updating version 3275 to 3276 2024-08-17T17:09:02.740 INFO:tasks.rados.rados.0.smithi063.stdout:3359: oid 36 version 3276 is already newer than 3273 2024-08-17T17:09:02.740 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 3276 (ObjNum 1255 snap 423 seq_num 1255) dirty exists 2024-08-17T17:09:02.740 INFO:tasks.rados.rados.0.smithi063.stdout:3359: left oid 36 (ObjNum 1255 snap 423 seq_num 1255) 2024-08-17T17:09:02.740 INFO:tasks.rados.rados.0.smithi063.stdout:3362: expect (ObjNum 1161 snap 392 seq_num 1161) 2024-08-17T17:09:02.925 INFO:tasks.rados.rados.0.smithi063.stdout:3363: finishing write tid 1 to smithi06387500-16 2024-08-17T17:09:02.925 INFO:tasks.rados.rados.0.smithi063.stdout:3363: finishing write tid 2 to smithi06387500-16 2024-08-17T17:09:02.925 INFO:tasks.rados.rados.0.smithi063.stdout:3363: finishing write tid 3 to smithi06387500-16 2024-08-17T17:09:02.925 INFO:tasks.rados.rados.0.smithi063.stdout:3363: finishing write tid 4 to smithi06387500-16 2024-08-17T17:09:02.925 INFO:tasks.rados.rados.0.smithi063.stdout:3363: oid 16 updating version 0 to 1202 2024-08-17T17:09:02.925 INFO:tasks.rados.rados.0.smithi063.stdout:3363: oid 16 updating version 1202 to 1203 2024-08-17T17:09:02.925 INFO:tasks.rados.rados.0.smithi063.stdout:3363: oid 16 updating version 1203 to 1204 2024-08-17T17:09:02.925 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 1204 (ObjNum 1257 snap 423 seq_num 1257) dirty exists 2024-08-17T17:09:02.925 INFO:tasks.rados.rados.0.smithi063.stdout:3363: left oid 16 (ObjNum 1257 snap 423 seq_num 1257) 2024-08-17T17:09:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:02 smithi063 ceph-mon[114756]: osdmap e914: 8 total, 8 up, 8 in 2024-08-17T17:09:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:02 smithi063 ceph-mon[114756]: pgmap v828: 89 pgs: 89 active+clean; 327 MiB data, 974 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 317 KiB/s wr, 11 op/s 2024-08-17T17:09:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:02 smithi063 ceph-mon[114756]: osdmap e915: 8 total, 8 up, 8 in 2024-08-17T17:09:03.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:02 smithi063 ceph-mon[109843]: osdmap e914: 8 total, 8 up, 8 in 2024-08-17T17:09:03.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:02 smithi063 ceph-mon[109843]: pgmap v828: 89 pgs: 89 active+clean; 327 MiB data, 974 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 317 KiB/s wr, 11 op/s 2024-08-17T17:09:03.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:02 smithi063 ceph-mon[109843]: osdmap e915: 8 total, 8 up, 8 in 2024-08-17T17:09:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:02 smithi162 ceph-mon[104668]: osdmap e914: 8 total, 8 up, 8 in 2024-08-17T17:09:03.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:02 smithi162 ceph-mon[104668]: pgmap v828: 89 pgs: 89 active+clean; 327 MiB data, 974 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 317 KiB/s wr, 11 op/s 2024-08-17T17:09:03.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:02 smithi162 ceph-mon[104668]: osdmap e915: 8 total, 8 up, 8 in 2024-08-17T17:09:03.700 INFO:tasks.rados.rados.0.smithi063.stdout:3359: done (3 left) 2024-08-17T17:09:03.701 INFO:tasks.rados.rados.0.smithi063.stdout:3362: done (2 left) 2024-08-17T17:09:03.701 INFO:tasks.rados.rados.0.smithi063.stdout:3363: done (1 left) 2024-08-17T17:09:03.701 INFO:tasks.rados.rados.0.smithi063.stdout:3364: done (0 left) 2024-08-17T17:09:03.701 INFO:tasks.rados.rados.0.smithi063.stdout:3365: snap_remove snap 398 2024-08-17T17:09:03.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:09:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:09:03] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.51.0" 2024-08-17T17:09:04.720 INFO:tasks.rados.rados.0.smithi063.stdout:3365: done (0 left) 2024-08-17T17:09:04.720 INFO:tasks.rados.rados.0.smithi063.stdout:3366: read oid 7 snap 419 2024-08-17T17:09:04.720 INFO:tasks.rados.rados.0.smithi063.stdout:3366: expect (ObjNum 1180 snap 398 seq_num 1180) 2024-08-17T17:09:04.720 INFO:tasks.rados.rados.0.smithi063.stdout:3367: snap_create 2024-08-17T17:09:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:3366: expect (ObjNum 1180 snap 398 seq_num 1180) 2024-08-17T17:09:05.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:04 smithi063 ceph-mon[114756]: osdmap e916: 8 total, 8 up, 8 in 2024-08-17T17:09:05.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:04 smithi063 ceph-mon[114756]: pgmap v831: 89 pgs: 89 active+clean; 330 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 684 KiB/s wr, 14 op/s 2024-08-17T17:09:05.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:04 smithi063 ceph-mon[109843]: osdmap e916: 8 total, 8 up, 8 in 2024-08-17T17:09:05.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:04 smithi063 ceph-mon[109843]: pgmap v831: 89 pgs: 89 active+clean; 330 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 684 KiB/s wr, 14 op/s 2024-08-17T17:09:05.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:04 smithi162 ceph-mon[104668]: osdmap e916: 8 total, 8 up, 8 in 2024-08-17T17:09:05.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:04 smithi162 ceph-mon[104668]: pgmap v831: 89 pgs: 89 active+clean; 330 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 684 KiB/s wr, 14 op/s 2024-08-17T17:09:05.713 INFO:tasks.rados.rados.0.smithi063.stdout:3366: done (1 left) 2024-08-17T17:09:05.713 INFO:tasks.rados.rados.0.smithi063.stdout:3367: done (0 left) 2024-08-17T17:09:05.713 INFO:tasks.rados.rados.0.smithi063.stdout:3368: read oid 18 snap -1 2024-08-17T17:09:05.713 INFO:tasks.rados.rados.0.smithi063.stdout:3368: expect (ObjNum 1201 snap 404 seq_num 1201) 2024-08-17T17:09:05.713 INFO:tasks.rados.rados.0.smithi063.stdout:3369: read oid 27 snap -1 2024-08-17T17:09:05.713 INFO:tasks.rados.rados.0.smithi063.stdout:3369: expect (ObjNum 1238 snap 413 seq_num 1238) 2024-08-17T17:09:05.713 INFO:tasks.rados.rados.0.smithi063.stdout:3370: read oid 38 snap -1 2024-08-17T17:09:05.714 INFO:tasks.rados.rados.0.smithi063.stdout:3370: expect deleted 2024-08-17T17:09:05.714 INFO:tasks.rados.rados.0.smithi063.stdout:3371: read oid 25 snap 406 2024-08-17T17:09:05.714 INFO:tasks.rados.rados.0.smithi063.stdout:3371: expect (ObjNum 1159 snap 392 seq_num 1159) 2024-08-17T17:09:05.714 INFO:tasks.rados.rados.0.smithi063.stdout:3372: read oid 49 snap 406 2024-08-17T17:09:05.714 INFO:tasks.rados.rados.0.smithi063.stdout:3372: expect (ObjNum 1101 snap 366 seq_num 1101) 2024-08-17T17:09:05.714 INFO:tasks.rados.rados.0.smithi063.stdout:3373: snap_remove snap 420 2024-08-17T17:09:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:05 smithi063 ceph-mon[114756]: osdmap e917: 8 total, 8 up, 8 in 2024-08-17T17:09:06.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:05 smithi063 ceph-mon[109843]: osdmap e917: 8 total, 8 up, 8 in 2024-08-17T17:09:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:05 smithi162 ceph-mon[104668]: osdmap e917: 8 total, 8 up, 8 in 2024-08-17T17:09:06.730 INFO:tasks.rados.rados.0.smithi063.stdout:3373: done (5 left) 2024-08-17T17:09:06.730 INFO:tasks.rados.rados.0.smithi063.stdout:3374: rollback oid 20 current snap is 425 2024-08-17T17:09:06.730 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 20 to 354 2024-08-17T17:09:06.730 INFO:tasks.rados.rados.0.smithi063.stdout:3369: expect (ObjNum 1238 snap 413 seq_num 1238) 2024-08-17T17:09:06.862 INFO:tasks.rados.rados.0.smithi063.stdout:3368: expect (ObjNum 1201 snap 404 seq_num 1201) 2024-08-17T17:09:06.966 INFO:tasks.rados.rados.0.smithi063.stdout:3371: expect (ObjNum 1159 snap 392 seq_num 1159) 2024-08-17T17:09:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:06 smithi063 ceph-mon[114756]: osdmap e918: 8 total, 8 up, 8 in 2024-08-17T17:09:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:06 smithi063 ceph-mon[114756]: pgmap v834: 89 pgs: 89 active+clean; 332 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-08-17T17:09:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:06 smithi063 ceph-mon[109843]: osdmap e918: 8 total, 8 up, 8 in 2024-08-17T17:09:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:06 smithi063 ceph-mon[109843]: pgmap v834: 89 pgs: 89 active+clean; 332 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-08-17T17:09:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:06 smithi162 ceph-mon[104668]: osdmap e918: 8 total, 8 up, 8 in 2024-08-17T17:09:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:06 smithi162 ceph-mon[104668]: pgmap v834: 89 pgs: 89 active+clean; 332 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.3 MiB/s wr, 10 op/s 2024-08-17T17:09:07.125 INFO:tasks.rados.rados.0.smithi063.stdout:3372: expect (ObjNum 1101 snap 366 seq_num 1101) 2024-08-17T17:09:07.265 INFO:tasks.rados.rados.0.smithi063.stdout:3374: finishing rollback tid 0 to smithi06387500-20 2024-08-17T17:09:07.265 INFO:tasks.rados.rados.0.smithi063.stdout:3374: finishing rollback tid 1 to smithi06387500-20 2024-08-17T17:09:07.265 INFO:tasks.rados.rados.0.smithi063.stdout:3368: done (5 left) 2024-08-17T17:09:07.265 INFO:tasks.rados.rados.0.smithi063.stdout:3369: done (4 left) 2024-08-17T17:09:07.265 INFO:tasks.rados.rados.0.smithi063.stdout:3370: done (3 left) 2024-08-17T17:09:07.266 INFO:tasks.rados.rados.0.smithi063.stdout:3371: done (2 left) 2024-08-17T17:09:07.266 INFO:tasks.rados.rados.0.smithi063.stdout:3372: done (1 left) 2024-08-17T17:09:07.266 INFO:tasks.rados.rados.0.smithi063.stdout:3375: rollback oid 40 current snap is 425 2024-08-17T17:09:07.266 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 40 to 413 2024-08-17T17:09:07.266 INFO:tasks.rados.rados.0.smithi063.stdout:3374: finishing rollback tid 2 to smithi06387500-20 2024-08-17T17:09:07.266 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 774 (ObjNum 996 snap 342 seq_num 996) dirty exists 2024-08-17T17:09:07.266 INFO:tasks.rados.rados.0.smithi063.stdout:3374: done (1 left) 2024-08-17T17:09:07.266 INFO:tasks.rados.rados.0.smithi063.stdout:3376: snap_remove snap 419 2024-08-17T17:09:07.658 INFO:tasks.rados.rados.0.smithi063.stdout:3376: done (1 left) 2024-08-17T17:09:07.658 INFO:tasks.rados.rados.0.smithi063.stdout:3377: snap_remove snap 354 2024-08-17T17:09:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:07 smithi063 ceph-mon[114756]: osdmap e919: 8 total, 8 up, 8 in 2024-08-17T17:09:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:07 smithi063 ceph-mon[114756]: osdmap e920: 8 total, 8 up, 8 in 2024-08-17T17:09:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:07 smithi063 ceph-mon[109843]: osdmap e919: 8 total, 8 up, 8 in 2024-08-17T17:09:08.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:07 smithi063 ceph-mon[109843]: osdmap e920: 8 total, 8 up, 8 in 2024-08-17T17:09:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:07 smithi162 ceph-mon[104668]: osdmap e919: 8 total, 8 up, 8 in 2024-08-17T17:09:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:07 smithi162 ceph-mon[104668]: osdmap e920: 8 total, 8 up, 8 in 2024-08-17T17:09:08.657 INFO:tasks.rados.rados.0.smithi063.stdout:3377: done (1 left) 2024-08-17T17:09:08.657 INFO:tasks.rados.rados.0.smithi063.stdout:3378: read oid 28 snap -1 2024-08-17T17:09:08.657 INFO:tasks.rados.rados.0.smithi063.stdout:3378: expect (ObjNum 1111 snap 368 seq_num 1111) 2024-08-17T17:09:08.658 INFO:tasks.rados.rados.0.smithi063.stdout:3379: write oid 37 current snap is 425 2024-08-17T17:09:08.658 INFO:tasks.rados.rados.0.smithi063.stdout:3375: finishing rollback tid 0 to smithi06387500-40 2024-08-17T17:09:08.658 INFO:tasks.rados.rados.0.smithi063.stdout:3375: finishing rollback tid 1 to smithi06387500-40 2024-08-17T17:09:08.658 INFO:tasks.rados.rados.0.smithi063.stdout:3375: finishing rollback tid 2 to smithi06387500-40 2024-08-17T17:09:08.658 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 1147 (ObjNum 1173 snap 395 seq_num 1173) dirty exists 2024-08-17T17:09:08.658 INFO:tasks.rados.rados.0.smithi063.stdout:3379: seq_num 1258 ranges {743206=661487,2131877=541092} 2024-08-17T17:09:08.666 INFO:tasks.rados.rados.0.smithi063.stdout:3379: writing smithi06387500-37 from 743206 to 1404693 tid 1 2024-08-17T17:09:08.674 INFO:tasks.rados.rados.0.smithi063.stdout:3379: writing smithi06387500-37 from 2131877 to 2672969 tid 2 2024-08-17T17:09:08.674 INFO:tasks.rados.rados.0.smithi063.stdout:3375: done (2 left) 2024-08-17T17:09:08.674 INFO:tasks.rados.rados.0.smithi063.stdout:3380: rollback oid 7 current snap is 425 2024-08-17T17:09:08.675 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 424 2024-08-17T17:09:08.675 INFO:tasks.rados.rados.0.smithi063.stdout:3381: rollback oid 29 current snap is 425 2024-08-17T17:09:08.675 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 29 to 410 2024-08-17T17:09:08.675 INFO:tasks.rados.rados.0.smithi063.stdout:3382: snap_remove snap 421 2024-08-17T17:09:09.671 INFO:tasks.rados.rados.0.smithi063.stdout:3382: done (4 left) 2024-08-17T17:09:09.671 INFO:tasks.rados.rados.0.smithi063.stdout:3383: delete oid 5 current snap is 425 2024-08-17T17:09:09.671 INFO:tasks.rados.rados.0.smithi063.stdout:3378: expect (ObjNum 1111 snap 368 seq_num 1111) 2024-08-17T17:09:09.765 INFO:tasks.rados.rados.0.smithi063.stdout:3381: finishing rollback tid 0 to smithi06387500-29 2024-08-17T17:09:09.765 INFO:tasks.rados.rados.0.smithi063.stdout:3381: finishing rollback tid 1 to smithi06387500-29 2024-08-17T17:09:09.765 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 774 (ObjNum 8209 snap 0 seq_num 765070094) dirty dne 2024-08-17T17:09:09.765 INFO:tasks.rados.rados.0.smithi063.stdout:3380: finishing rollback tid 0 to smithi06387500-7 2024-08-17T17:09:09.765 INFO:tasks.rados.rados.0.smithi063.stdout:3380: finishing rollback tid 1 to smithi06387500-7 2024-08-17T17:09:09.766 INFO:tasks.rados.rados.0.smithi063.stdout:3380: finishing rollback tid 2 to smithi06387500-7 2024-08-17T17:09:09.766 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1620 (ObjNum 1180 snap 398 seq_num 1180) dirty exists 2024-08-17T17:09:09.766 INFO:tasks.rados.rados.0.smithi063.stdout:3379: finishing write tid 1 to smithi06387500-37 2024-08-17T17:09:09.766 INFO:tasks.rados.rados.0.smithi063.stdout:3379: finishing write tid 2 to smithi06387500-37 2024-08-17T17:09:09.766 INFO:tasks.rados.rados.0.smithi063.stdout:3378: done (4 left) 2024-08-17T17:09:09.766 INFO:tasks.rados.rados.0.smithi063.stdout:3380: done (3 left) 2024-08-17T17:09:09.766 INFO:tasks.rados.rados.0.smithi063.stdout:3381: done (2 left) 2024-08-17T17:09:09.767 INFO:tasks.rados.rados.0.smithi063.stdout:3383: done (1 left) 2024-08-17T17:09:09.767 INFO:tasks.rados.rados.0.smithi063.stdout:3384: delete oid 25 current snap is 425 2024-08-17T17:09:09.767 INFO:tasks.rados.rados.0.smithi063.stdout:3379: finishing write tid 3 to smithi06387500-37 2024-08-17T17:09:09.767 INFO:tasks.rados.rados.0.smithi063.stdout:3379: finishing write tid 4 to smithi06387500-37 2024-08-17T17:09:09.767 INFO:tasks.rados.rados.0.smithi063.stdout:3379: oid 37 updating version 0 to 3290 2024-08-17T17:09:09.767 INFO:tasks.rados.rados.0.smithi063.stdout:3379: oid 37 updating version 3290 to 3295 2024-08-17T17:09:09.767 INFO:tasks.rados.rados.0.smithi063.stdout:3379: oid 37 updating version 3295 to 3296 2024-08-17T17:09:09.767 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 3296 (ObjNum 1258 snap 425 seq_num 1258) dirty exists 2024-08-17T17:09:09.767 INFO:tasks.rados.rados.0.smithi063.stdout:3379: left oid 37 (ObjNum 1258 snap 425 seq_num 1258) 2024-08-17T17:09:09.767 INFO:tasks.rados.rados.0.smithi063.stdout:3379: done (1 left) 2024-08-17T17:09:09.768 INFO:tasks.rados.rados.0.smithi063.stdout:3384: done (0 left) 2024-08-17T17:09:09.768 INFO:tasks.rados.rados.0.smithi063.stdout:3385: snap_create 2024-08-17T17:09:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:09 smithi063 ceph-mon[114756]: pgmap v837: 89 pgs: 89 active+clean; 325 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-08-17T17:09:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:09 smithi063 ceph-mon[114756]: osdmap e921: 8 total, 8 up, 8 in 2024-08-17T17:09:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:09 smithi063 ceph-mon[109843]: pgmap v837: 89 pgs: 89 active+clean; 325 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-08-17T17:09:10.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:09 smithi063 ceph-mon[109843]: osdmap e921: 8 total, 8 up, 8 in 2024-08-17T17:09:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:09 smithi162 ceph-mon[104668]: pgmap v837: 89 pgs: 89 active+clean; 325 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.6 MiB/s wr, 15 op/s 2024-08-17T17:09:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:09 smithi162 ceph-mon[104668]: osdmap e921: 8 total, 8 up, 8 in 2024-08-17T17:09:10.678 INFO:tasks.rados.rados.0.smithi063.stdout:3385: done (0 left) 2024-08-17T17:09:10.679 INFO:tasks.rados.rados.0.smithi063.stdout:3386: write oid 4 current snap is 426 2024-08-17T17:09:10.679 INFO:tasks.rados.rados.0.smithi063.stdout:3386: seq_num 1261 ranges {793627=487234,1790970=404772,2829335=364292} 2024-08-17T17:09:10.685 INFO:tasks.rados.rados.0.smithi063.stdout:3386: writing smithi06387500-4 from 793627 to 1280861 tid 1 2024-08-17T17:09:10.690 INFO:tasks.rados.rados.0.smithi063.stdout:3386: writing smithi06387500-4 from 1790970 to 2195742 tid 2 2024-08-17T17:09:10.696 INFO:tasks.rados.rados.0.smithi063.stdout:3386: writing smithi06387500-4 from 2829335 to 3193627 tid 3 2024-08-17T17:09:10.696 INFO:tasks.rados.rados.0.smithi063.stdout:3387: read oid 29 snap -1 2024-08-17T17:09:10.697 INFO:tasks.rados.rados.0.smithi063.stdout:3387: expect deleted 2024-08-17T17:09:10.697 INFO:tasks.rados.rados.0.smithi063.stdout:3388: write oid 19 current snap is 426 2024-08-17T17:09:10.697 INFO:tasks.rados.rados.0.smithi063.stdout:3386: finishing write tid 1 to smithi06387500-4 2024-08-17T17:09:10.697 INFO:tasks.rados.rados.0.smithi063.stdout:3386: finishing write tid 2 to smithi06387500-4 2024-08-17T17:09:10.697 INFO:tasks.rados.rados.0.smithi063.stdout:3388: seq_num 1262 ranges {410434=678836,1565805=548430,2573111=637323} 2024-08-17T17:09:10.703 INFO:tasks.rados.rados.0.smithi063.stdout:3388: writing smithi06387500-19 from 410434 to 1089270 tid 1 2024-08-17T17:09:10.709 INFO:tasks.rados.rados.0.smithi063.stdout:3388: writing smithi06387500-19 from 1565805 to 2114235 tid 2 2024-08-17T17:09:10.716 INFO:tasks.rados.rados.0.smithi063.stdout:3388: writing smithi06387500-19 from 2573111 to 3210434 tid 3 2024-08-17T17:09:10.716 INFO:tasks.rados.rados.0.smithi063.stdout:3389: read oid 49 snap -1 2024-08-17T17:09:10.716 INFO:tasks.rados.rados.0.smithi063.stdout:3389: expect (ObjNum 1101 snap 366 seq_num 1101) 2024-08-17T17:09:10.716 INFO:tasks.rados.rados.0.smithi063.stdout:3390: rollback oid 30 current snap is 426 2024-08-17T17:09:10.716 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 30 to 410 2024-08-17T17:09:10.716 INFO:tasks.rados.rados.0.smithi063.stdout:3386: finishing write tid 3 to smithi06387500-4 2024-08-17T17:09:10.716 INFO:tasks.rados.rados.0.smithi063.stdout:3386: finishing write tid 4 to smithi06387500-4 2024-08-17T17:09:10.716 INFO:tasks.rados.rados.0.smithi063.stdout:3386: finishing write tid 5 to smithi06387500-4 2024-08-17T17:09:10.716 INFO:tasks.rados.rados.0.smithi063.stdout:3386: oid 4 updating version 0 to 3305 2024-08-17T17:09:10.717 INFO:tasks.rados.rados.0.smithi063.stdout:3386: oid 4 updating version 3305 to 3307 2024-08-17T17:09:10.717 INFO:tasks.rados.rados.0.smithi063.stdout:3386: oid 4 version 3307 is already newer than 3304 2024-08-17T17:09:10.717 INFO:tasks.rados.rados.0.smithi063.stdout:3386: oid 4 version 3307 is already newer than 3306 2024-08-17T17:09:10.717 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 3307 (ObjNum 1261 snap 426 seq_num 1261) dirty exists 2024-08-17T17:09:10.717 INFO:tasks.rados.rados.0.smithi063.stdout:3386: left oid 4 (ObjNum 1261 snap 426 seq_num 1261) 2024-08-17T17:09:10.717 INFO:tasks.rados.rados.0.smithi063.stdout:3388: finishing write tid 1 to smithi06387500-19 2024-08-17T17:09:10.717 INFO:tasks.rados.rados.0.smithi063.stdout:3388: finishing write tid 2 to smithi06387500-19 2024-08-17T17:09:10.717 INFO:tasks.rados.rados.0.smithi063.stdout:3386: done (4 left) 2024-08-17T17:09:10.717 INFO:tasks.rados.rados.0.smithi063.stdout:3387: done (3 left) 2024-08-17T17:09:10.718 INFO:tasks.rados.rados.0.smithi063.stdout:3391: snap_create 2024-08-17T17:09:10.720 INFO:tasks.rados.rados.0.smithi063.stdout:3388: finishing write tid 3 to smithi06387500-19 2024-08-17T17:09:10.720 INFO:tasks.rados.rados.0.smithi063.stdout:3388: finishing write tid 4 to smithi06387500-19 2024-08-17T17:09:10.721 INFO:tasks.rados.rados.0.smithi063.stdout:3388: finishing write tid 5 to smithi06387500-19 2024-08-17T17:09:10.721 INFO:tasks.rados.rados.0.smithi063.stdout:3388: oid 19 updating version 0 to 2183 2024-08-17T17:09:10.721 INFO:tasks.rados.rados.0.smithi063.stdout:3388: oid 19 version 2183 is already newer than 2181 2024-08-17T17:09:10.721 INFO:tasks.rados.rados.0.smithi063.stdout:3388: oid 19 version 2183 is already newer than 2182 2024-08-17T17:09:10.721 INFO:tasks.rados.rados.0.smithi063.stdout:3388: oid 19 version 2183 is already newer than 2180 2024-08-17T17:09:10.721 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 2183 (ObjNum 1262 snap 426 seq_num 1262) dirty exists 2024-08-17T17:09:10.722 INFO:tasks.rados.rados.0.smithi063.stdout:3388: left oid 19 (ObjNum 1262 snap 426 seq_num 1262) 2024-08-17T17:09:10.722 INFO:tasks.rados.rados.0.smithi063.stdout:3390: finishing rollback tid 1 to smithi06387500-30 2024-08-17T17:09:10.722 INFO:tasks.rados.rados.0.smithi063.stdout:3390: finishing rollback tid 2 to smithi06387500-30 2024-08-17T17:09:10.722 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1626 (ObjNum 1215 snap 407 seq_num 1215) dirty exists 2024-08-17T17:09:10.722 INFO:tasks.rados.rados.0.smithi063.stdout:3389: expect (ObjNum 1101 snap 366 seq_num 1101) 2024-08-17T17:09:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:10 smithi063 ceph-mon[114756]: osdmap e922: 8 total, 8 up, 8 in 2024-08-17T17:09:11.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:10 smithi063 ceph-mon[109843]: osdmap e922: 8 total, 8 up, 8 in 2024-08-17T17:09:11.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:10 smithi162 ceph-mon[104668]: osdmap e922: 8 total, 8 up, 8 in 2024-08-17T17:09:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:3388: done (3 left) 2024-08-17T17:09:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:3389: done (2 left) 2024-08-17T17:09:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:3390: done (1 left) 2024-08-17T17:09:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:3391: done (0 left) 2024-08-17T17:09:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:3392: read oid 3 snap 413 2024-08-17T17:09:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:3392: expect deleted 2024-08-17T17:09:11.688 INFO:tasks.rados.rados.0.smithi063.stdout:3393: snap_create 2024-08-17T17:09:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:11 smithi063 ceph-mon[114756]: pgmap v840: 89 pgs: 89 active+clean; 319 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 680 KiB/s wr, 8 op/s 2024-08-17T17:09:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:11 smithi063 ceph-mon[114756]: osdmap e923: 8 total, 8 up, 8 in 2024-08-17T17:09:12.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:11 smithi063 ceph-mon[109843]: pgmap v840: 89 pgs: 89 active+clean; 319 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 680 KiB/s wr, 8 op/s 2024-08-17T17:09:12.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:11 smithi063 ceph-mon[109843]: osdmap e923: 8 total, 8 up, 8 in 2024-08-17T17:09:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:11 smithi162 ceph-mon[104668]: pgmap v840: 89 pgs: 89 active+clean; 319 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 680 KiB/s wr, 8 op/s 2024-08-17T17:09:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:11 smithi162 ceph-mon[104668]: osdmap e923: 8 total, 8 up, 8 in 2024-08-17T17:09:12.655 INFO:tasks.rados.rados.0.smithi063.stdout:3392: done (1 left) 2024-08-17T17:09:12.655 INFO:tasks.rados.rados.0.smithi063.stdout:3393: done (0 left) 2024-08-17T17:09:12.655 INFO:tasks.rados.rados.0.smithi063.stdout:3394: read oid 2 snap -1 2024-08-17T17:09:12.655 INFO:tasks.rados.rados.0.smithi063.stdout:3394: expect deleted 2024-08-17T17:09:12.655 INFO:tasks.rados.rados.0.smithi063.stdout:3395: delete oid 48 current snap is 428 2024-08-17T17:09:12.661 INFO:tasks.rados.rados.0.smithi063.stdout:3394: done (1 left) 2024-08-17T17:09:12.661 INFO:tasks.rados.rados.0.smithi063.stdout:3395: done (0 left) 2024-08-17T17:09:12.661 INFO:tasks.rados.rados.0.smithi063.stdout:3396: delete oid 45 current snap is 428 2024-08-17T17:09:12.663 INFO:tasks.rados.rados.0.smithi063.stdout:3396: done (0 left) 2024-08-17T17:09:12.664 INFO:tasks.rados.rados.0.smithi063.stdout:3397: rollback oid 39 current snap is 428 2024-08-17T17:09:12.664 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 39 to 378 2024-08-17T17:09:12.664 INFO:tasks.rados.rados.0.smithi063.stdout:3398: write oid 5 current snap is 428 2024-08-17T17:09:12.664 INFO:tasks.rados.rados.0.smithi063.stdout:3398: seq_num 1265 ranges {460855=504583,1624898=412110,2539027=572508} 2024-08-17T17:09:12.669 INFO:tasks.rados.rados.0.smithi063.stdout:3398: writing smithi06387500-5 from 460855 to 965438 tid 1 2024-08-17T17:09:12.675 INFO:tasks.rados.rados.0.smithi063.stdout:3398: writing smithi06387500-5 from 1624898 to 2037008 tid 2 2024-08-17T17:09:12.681 INFO:tasks.rados.rados.0.smithi063.stdout:3398: writing smithi06387500-5 from 2539027 to 3111535 tid 3 2024-08-17T17:09:12.681 INFO:tasks.rados.rados.0.smithi063.stdout:3399: snap_create 2024-08-17T17:09:12.681 INFO:tasks.rados.rados.0.smithi063.stdout:3397: finishing rollback tid 0 to smithi06387500-39 2024-08-17T17:09:12.681 INFO:tasks.rados.rados.0.smithi063.stdout:3397: finishing rollback tid 1 to smithi06387500-39 2024-08-17T17:09:12.681 INFO:tasks.rados.rados.0.smithi063.stdout:3397: finishing rollback tid 2 to smithi06387500-39 2024-08-17T17:09:12.681 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 833 (ObjNum 1092 snap 363 seq_num 1092) dirty exists 2024-08-17T17:09:12.681 INFO:tasks.rados.rados.0.smithi063.stdout:3398: finishing write tid 1 to smithi06387500-5 2024-08-17T17:09:12.681 INFO:tasks.rados.rados.0.smithi063.stdout:3398: finishing write tid 2 to smithi06387500-5 2024-08-17T17:09:12.685 INFO:tasks.rados.rados.0.smithi063.stdout:3398: finishing write tid 3 to smithi06387500-5 2024-08-17T17:09:12.685 INFO:tasks.rados.rados.0.smithi063.stdout:3398: finishing write tid 4 to smithi06387500-5 2024-08-17T17:09:12.685 INFO:tasks.rados.rados.0.smithi063.stdout:3398: finishing write tid 5 to smithi06387500-5 2024-08-17T17:09:12.686 INFO:tasks.rados.rados.0.smithi063.stdout:3398: oid 5 updating version 0 to 1754 2024-08-17T17:09:12.686 INFO:tasks.rados.rados.0.smithi063.stdout:3398: oid 5 version 1754 is already newer than 1753 2024-08-17T17:09:12.686 INFO:tasks.rados.rados.0.smithi063.stdout:3398: oid 5 version 1754 is already newer than 1752 2024-08-17T17:09:12.686 INFO:tasks.rados.rados.0.smithi063.stdout:3398: oid 5 version 1754 is already newer than 1751 2024-08-17T17:09:12.686 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1754 (ObjNum 1265 snap 428 seq_num 1265) dirty exists 2024-08-17T17:09:12.686 INFO:tasks.rados.rados.0.smithi063.stdout:3398: left oid 5 (ObjNum 1265 snap 428 seq_num 1265) 2024-08-17T17:09:13.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:12 smithi063 ceph-mon[109843]: osdmap e924: 8 total, 8 up, 8 in 2024-08-17T17:09:13.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:12 smithi063 ceph-mon[109843]: pgmap v843: 89 pgs: 89 active+clean; 270 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 900 KiB/s wr, 13 op/s 2024-08-17T17:09:13.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:12 smithi063 ceph-mon[109843]: osdmap e925: 8 total, 8 up, 8 in 2024-08-17T17:09:13.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:12 smithi063 ceph-mon[114756]: osdmap e924: 8 total, 8 up, 8 in 2024-08-17T17:09:13.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:12 smithi063 ceph-mon[114756]: pgmap v843: 89 pgs: 89 active+clean; 270 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 900 KiB/s wr, 13 op/s 2024-08-17T17:09:13.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:12 smithi063 ceph-mon[114756]: osdmap e925: 8 total, 8 up, 8 in 2024-08-17T17:09:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:12 smithi162 ceph-mon[104668]: osdmap e924: 8 total, 8 up, 8 in 2024-08-17T17:09:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:12 smithi162 ceph-mon[104668]: pgmap v843: 89 pgs: 89 active+clean; 270 MiB data, 884 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 900 KiB/s wr, 13 op/s 2024-08-17T17:09:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:12 smithi162 ceph-mon[104668]: osdmap e925: 8 total, 8 up, 8 in 2024-08-17T17:09:13.694 INFO:tasks.rados.rados.0.smithi063.stdout:3397: done (2 left) 2024-08-17T17:09:13.694 INFO:tasks.rados.rados.0.smithi063.stdout:3398: done (1 left) 2024-08-17T17:09:13.694 INFO:tasks.rados.rados.0.smithi063.stdout:3399: done (0 left) 2024-08-17T17:09:13.694 INFO:tasks.rados.rados.0.smithi063.stdout:3400: read oid 46 snap -1 2024-08-17T17:09:13.694 INFO:tasks.rados.rados.0.smithi063.stdout:3400: expect deleted 2024-08-17T17:09:13.695 INFO:tasks.rados.rados.0.smithi063.stdout:3401: snap_create 2024-08-17T17:09:13.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:09:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:09:13] "GET /metrics HTTP/1.1" 200 36344 "" "Prometheus/2.51.0" 2024-08-17T17:09:14.694 INFO:tasks.rados.rados.0.smithi063.stdout:3400: done (1 left) 2024-08-17T17:09:14.694 INFO:tasks.rados.rados.0.smithi063.stdout:3401: done (0 left) 2024-08-17T17:09:14.694 INFO:tasks.rados.rados.0.smithi063.stdout:3402: rollback oid 32 current snap is 430 2024-08-17T17:09:14.694 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 32 to 427 2024-08-17T17:09:14.694 INFO:tasks.rados.rados.0.smithi063.stdout:3403: delete oid 6 current snap is 430 2024-08-17T17:09:14.700 INFO:tasks.rados.rados.0.smithi063.stdout:3403: done (1 left) 2024-08-17T17:09:14.701 INFO:tasks.rados.rados.0.smithi063.stdout:3404: snap_create 2024-08-17T17:09:14.701 INFO:tasks.rados.rados.0.smithi063.stdout:3402: finishing rollback tid 0 to smithi06387500-32 2024-08-17T17:09:14.701 INFO:tasks.rados.rados.0.smithi063.stdout:3402: finishing rollback tid 1 to smithi06387500-32 2024-08-17T17:09:14.701 INFO:tasks.rados.rados.0.smithi063.stdout:3402: finishing rollback tid 2 to smithi06387500-32 2024-08-17T17:09:14.701 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 747 (ObjNum 1208 snap 404 seq_num 1208) dirty exists 2024-08-17T17:09:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:14 smithi063 ceph-mon[114756]: osdmap e926: 8 total, 8 up, 8 in 2024-08-17T17:09:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:14 smithi063 ceph-mon[114756]: pgmap v846: 89 pgs: 89 active+clean; 268 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 900 KiB/s wr, 13 op/s 2024-08-17T17:09:15.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:14 smithi063 ceph-mon[109843]: osdmap e926: 8 total, 8 up, 8 in 2024-08-17T17:09:15.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:14 smithi063 ceph-mon[109843]: pgmap v846: 89 pgs: 89 active+clean; 268 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 900 KiB/s wr, 13 op/s 2024-08-17T17:09:15.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:14 smithi162 ceph-mon[104668]: osdmap e926: 8 total, 8 up, 8 in 2024-08-17T17:09:15.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:14 smithi162 ceph-mon[104668]: pgmap v846: 89 pgs: 89 active+clean; 268 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 900 KiB/s wr, 13 op/s 2024-08-17T17:09:15.708 INFO:tasks.rados.rados.0.smithi063.stdout:3402: done (1 left) 2024-08-17T17:09:15.709 INFO:tasks.rados.rados.0.smithi063.stdout:3404: done (0 left) 2024-08-17T17:09:15.709 INFO:tasks.rados.rados.0.smithi063.stdout:3405: read oid 2 snap -1 2024-08-17T17:09:15.709 INFO:tasks.rados.rados.0.smithi063.stdout:3405: expect deleted 2024-08-17T17:09:15.709 INFO:tasks.rados.rados.0.smithi063.stdout:3406: read oid 23 snap 410 2024-08-17T17:09:15.709 INFO:tasks.rados.rados.0.smithi063.stdout:3406: expect (ObjNum 1213 snap 406 seq_num 1213) 2024-08-17T17:09:15.709 INFO:tasks.rados.rados.0.smithi063.stdout:3407: delete oid 47 current snap is 431 2024-08-17T17:09:15.717 INFO:tasks.rados.rados.0.smithi063.stdout:3405: done (2 left) 2024-08-17T17:09:15.717 INFO:tasks.rados.rados.0.smithi063.stdout:3407: done (1 left) 2024-08-17T17:09:15.717 INFO:tasks.rados.rados.0.smithi063.stdout:3408: read oid 48 snap 425 2024-08-17T17:09:15.718 INFO:tasks.rados.rados.0.smithi063.stdout:3408: expect (ObjNum 1143 snap 381 seq_num 1143) 2024-08-17T17:09:15.718 INFO:tasks.rados.rados.0.smithi063.stdout:3409: snap_remove snap 413 2024-08-17T17:09:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:15 smithi063 ceph-mon[114756]: osdmap e927: 8 total, 8 up, 8 in 2024-08-17T17:09:16.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:15 smithi063 ceph-mon[109843]: osdmap e927: 8 total, 8 up, 8 in 2024-08-17T17:09:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:15 smithi162 ceph-mon[104668]: osdmap e927: 8 total, 8 up, 8 in 2024-08-17T17:09:16.725 INFO:tasks.rados.rados.0.smithi063.stdout:3409: done (2 left) 2024-08-17T17:09:16.725 INFO:tasks.rados.rados.0.smithi063.stdout:3410: read oid 18 snap -1 2024-08-17T17:09:16.725 INFO:tasks.rados.rados.0.smithi063.stdout:3410: expect (ObjNum 1201 snap 404 seq_num 1201) 2024-08-17T17:09:16.725 INFO:tasks.rados.rados.0.smithi063.stdout:3411: snap_create 2024-08-17T17:09:16.725 INFO:tasks.rados.rados.0.smithi063.stdout:3406: expect (ObjNum 1213 snap 406 seq_num 1213) 2024-08-17T17:09:16.831 INFO:tasks.rados.rados.0.smithi063.stdout:3408: expect (ObjNum 1143 snap 381 seq_num 1143) 2024-08-17T17:09:16.977 INFO:tasks.rados.rados.0.smithi063.stdout:3410: expect (ObjNum 1201 snap 404 seq_num 1201) 2024-08-17T17:09:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:16 smithi063 ceph-mon[114756]: osdmap e928: 8 total, 8 up, 8 in 2024-08-17T17:09:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:16 smithi063 ceph-mon[114756]: pgmap v849: 89 pgs: 89 active+clean; 269 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 156 KiB/s wr, 0 op/s 2024-08-17T17:09:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:17.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:16 smithi063 ceph-mon[109843]: osdmap e928: 8 total, 8 up, 8 in 2024-08-17T17:09:17.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:16 smithi063 ceph-mon[109843]: pgmap v849: 89 pgs: 89 active+clean; 269 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 156 KiB/s wr, 0 op/s 2024-08-17T17:09:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:16 smithi162 ceph-mon[104668]: osdmap e928: 8 total, 8 up, 8 in 2024-08-17T17:09:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:16 smithi162 ceph-mon[104668]: pgmap v849: 89 pgs: 89 active+clean; 269 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 156 KiB/s wr, 0 op/s 2024-08-17T17:09:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:17.656 INFO:tasks.rados.rados.0.smithi063.stdout:3406: done (3 left) 2024-08-17T17:09:17.656 INFO:tasks.rados.rados.0.smithi063.stdout:3408: done (2 left) 2024-08-17T17:09:17.656 INFO:tasks.rados.rados.0.smithi063.stdout:3410: done (1 left) 2024-08-17T17:09:17.656 INFO:tasks.rados.rados.0.smithi063.stdout:3411: done (0 left) 2024-08-17T17:09:17.656 INFO:tasks.rados.rados.0.smithi063.stdout:3412: read oid 6 snap -1 2024-08-17T17:09:17.656 INFO:tasks.rados.rados.0.smithi063.stdout:3412: expect deleted 2024-08-17T17:09:17.657 INFO:tasks.rados.rados.0.smithi063.stdout:3413: delete oid 45 current snap is 432 2024-08-17T17:09:17.661 INFO:tasks.rados.rados.0.smithi063.stdout:3412: done (1 left) 2024-08-17T17:09:17.661 INFO:tasks.rados.rados.0.smithi063.stdout:3413: done (0 left) 2024-08-17T17:09:17.661 INFO:tasks.rados.rados.0.smithi063.stdout:3414: read oid 44 snap -1 2024-08-17T17:09:17.662 INFO:tasks.rados.rados.0.smithi063.stdout:3414: expect (ObjNum 1218 snap 407 seq_num 1218) 2024-08-17T17:09:17.662 INFO:tasks.rados.rados.0.smithi063.stdout:3415: snap_create 2024-08-17T17:09:17.667 INFO:tasks.rados.rados.0.smithi063.stdout:3414: expect (ObjNum 1218 snap 407 seq_num 1218) 2024-08-17T17:09:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:17 smithi063 ceph-mon[114756]: osdmap e929: 8 total, 8 up, 8 in 2024-08-17T17:09:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:17 smithi063 ceph-mon[114756]: osdmap e930: 8 total, 8 up, 8 in 2024-08-17T17:09:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:17 smithi063 ceph-mon[109843]: osdmap e929: 8 total, 8 up, 8 in 2024-08-17T17:09:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:17 smithi063 ceph-mon[109843]: osdmap e930: 8 total, 8 up, 8 in 2024-08-17T17:09:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:17 smithi162 ceph-mon[104668]: osdmap e929: 8 total, 8 up, 8 in 2024-08-17T17:09:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:17 smithi162 ceph-mon[104668]: osdmap e930: 8 total, 8 up, 8 in 2024-08-17T17:09:18.659 INFO:tasks.rados.rados.0.smithi063.stdout:3414: done (1 left) 2024-08-17T17:09:18.659 INFO:tasks.rados.rados.0.smithi063.stdout:3415: done (0 left) 2024-08-17T17:09:18.659 INFO:tasks.rados.rados.0.smithi063.stdout:3416: delete oid 8 current snap is 433 2024-08-17T17:09:18.662 INFO:tasks.rados.rados.0.smithi063.stdout:3416: done (0 left) 2024-08-17T17:09:18.662 INFO:tasks.rados.rados.0.smithi063.stdout:3417: delete oid 35 current snap is 433 2024-08-17T17:09:18.666 INFO:tasks.rados.rados.0.smithi063.stdout:3417: done (0 left) 2024-08-17T17:09:18.666 INFO:tasks.rados.rados.0.smithi063.stdout:3418: write oid 45 current snap is 433 2024-08-17T17:09:18.666 INFO:tasks.rados.rados.0.smithi063.stdout:3418: seq_num 1271 ranges {561697=672430,1975790=706764,3103565=258132} 2024-08-17T17:09:18.672 INFO:tasks.rados.rados.0.smithi063.stdout:3418: writing smithi06387500-45 from 561697 to 1234127 tid 1 2024-08-17T17:09:18.681 INFO:tasks.rados.rados.0.smithi063.stdout:3418: writing smithi06387500-45 from 1975790 to 2682554 tid 2 2024-08-17T17:09:18.685 INFO:tasks.rados.rados.0.smithi063.stdout:3418: writing smithi06387500-45 from 3103565 to 3361697 tid 3 2024-08-17T17:09:18.685 INFO:tasks.rados.rados.0.smithi063.stdout:3419: snap_create 2024-08-17T17:09:18.685 INFO:tasks.rados.rados.0.smithi063.stdout:3418: finishing write tid 1 to smithi06387500-45 2024-08-17T17:09:18.685 INFO:tasks.rados.rados.0.smithi063.stdout:3418: finishing write tid 2 to smithi06387500-45 2024-08-17T17:09:18.687 INFO:tasks.rados.rados.0.smithi063.stdout:3418: finishing write tid 3 to smithi06387500-45 2024-08-17T17:09:18.687 INFO:tasks.rados.rados.0.smithi063.stdout:3418: finishing write tid 4 to smithi06387500-45 2024-08-17T17:09:18.687 INFO:tasks.rados.rados.0.smithi063.stdout:3418: finishing write tid 5 to smithi06387500-45 2024-08-17T17:09:18.687 INFO:tasks.rados.rados.0.smithi063.stdout:3418: oid 45 updating version 0 to 838 2024-08-17T17:09:18.687 INFO:tasks.rados.rados.0.smithi063.stdout:3418: oid 45 updating version 838 to 840 2024-08-17T17:09:18.687 INFO:tasks.rados.rados.0.smithi063.stdout:3418: oid 45 version 840 is already newer than 839 2024-08-17T17:09:18.687 INFO:tasks.rados.rados.0.smithi063.stdout:3418: oid 45 version 840 is already newer than 837 2024-08-17T17:09:18.687 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 840 (ObjNum 1271 snap 433 seq_num 1271) dirty exists 2024-08-17T17:09:18.688 INFO:tasks.rados.rados.0.smithi063.stdout:3418: left oid 45 (ObjNum 1271 snap 433 seq_num 1271) 2024-08-17T17:09:19.659 INFO:tasks.rados.rados.0.smithi063.stdout:3418: done (1 left) 2024-08-17T17:09:19.659 INFO:tasks.rados.rados.0.smithi063.stdout:3419: done (0 left) 2024-08-17T17:09:19.660 INFO:tasks.rados.rados.0.smithi063.stdout:3420: write oid 37 current snap is 434 2024-08-17T17:09:19.660 INFO:tasks.rados.rados.0.smithi063.stdout:3420: seq_num 1272 ranges {578504=747679,2034272=566775,3130988=247516} 2024-08-17T17:09:19.666 INFO:tasks.rados.rados.0.smithi063.stdout:3420: writing smithi06387500-37 from 578504 to 1326183 tid 1 2024-08-17T17:09:19.673 INFO:tasks.rados.rados.0.smithi063.stdout:3420: writing smithi06387500-37 from 2034272 to 2601047 tid 2 2024-08-17T17:09:19.678 INFO:tasks.rados.rados.0.smithi063.stdout:3420: writing smithi06387500-37 from 3130988 to 3378504 tid 3 2024-08-17T17:09:19.678 INFO:tasks.rados.rados.0.smithi063.stdout:3421: write oid 25 current snap is 434 2024-08-17T17:09:19.678 INFO:tasks.rados.rados.0.smithi063.stdout:3420: finishing write tid 1 to smithi06387500-37 2024-08-17T17:09:19.678 INFO:tasks.rados.rados.0.smithi063.stdout:3421: seq_num 1273 ranges {595311=422928,1576401=710433,3042058=353253} 2024-08-17T17:09:19.683 INFO:tasks.rados.rados.0.smithi063.stdout:3421: writing smithi06387500-25 from 595311 to 1018239 tid 1 2024-08-17T17:09:19.690 INFO:tasks.rados.rados.0.smithi063.stdout:3421: writing smithi06387500-25 from 1576401 to 2286834 tid 2 2024-08-17T17:09:19.697 INFO:tasks.rados.rados.0.smithi063.stdout:3421: writing smithi06387500-25 from 3042058 to 3395311 tid 3 2024-08-17T17:09:19.697 INFO:tasks.rados.rados.0.smithi063.stdout:3422: rollback oid 18 current snap is 434 2024-08-17T17:09:19.697 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 18 to 428 2024-08-17T17:09:19.697 INFO:tasks.rados.rados.0.smithi063.stdout:3420: finishing write tid 2 to smithi06387500-37 2024-08-17T17:09:19.697 INFO:tasks.rados.rados.0.smithi063.stdout:3420: finishing write tid 3 to smithi06387500-37 2024-08-17T17:09:19.698 INFO:tasks.rados.rados.0.smithi063.stdout:3420: finishing write tid 4 to smithi06387500-37 2024-08-17T17:09:19.698 INFO:tasks.rados.rados.0.smithi063.stdout:3420: finishing write tid 5 to smithi06387500-37 2024-08-17T17:09:19.698 INFO:tasks.rados.rados.0.smithi063.stdout:3420: oid 37 updating version 0 to 3325 2024-08-17T17:09:19.698 INFO:tasks.rados.rados.0.smithi063.stdout:3420: oid 37 updating version 3325 to 3327 2024-08-17T17:09:19.698 INFO:tasks.rados.rados.0.smithi063.stdout:3420: oid 37 version 3327 is already newer than 3326 2024-08-17T17:09:19.698 INFO:tasks.rados.rados.0.smithi063.stdout:3420: oid 37 version 3327 is already newer than 3324 2024-08-17T17:09:19.698 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 3327 (ObjNum 1272 snap 434 seq_num 1272) dirty exists 2024-08-17T17:09:19.698 INFO:tasks.rados.rados.0.smithi063.stdout:3420: left oid 37 (ObjNum 1272 snap 434 seq_num 1272) 2024-08-17T17:09:19.698 INFO:tasks.rados.rados.0.smithi063.stdout:3421: finishing write tid 1 to smithi06387500-25 2024-08-17T17:09:19.698 INFO:tasks.rados.rados.0.smithi063.stdout:3420: done (2 left) 2024-08-17T17:09:19.698 INFO:tasks.rados.rados.0.smithi063.stdout:3423: read oid 13 snap -1 2024-08-17T17:09:19.699 INFO:tasks.rados.rados.0.smithi063.stdout:3423: expect (ObjNum 1205 snap 404 seq_num 1205) 2024-08-17T17:09:19.699 INFO:tasks.rados.rados.0.smithi063.stdout:3424: write oid 3 current snap is 434 2024-08-17T17:09:19.699 INFO:tasks.rados.rados.0.smithi063.stdout:3424: seq_num 1274 ranges {612118=498177,1634883=570444,2669481=706662} 2024-08-17T17:09:19.703 INFO:tasks.rados.rados.0.smithi063.stdout:3424: writing smithi06387500-3 from 612118 to 1110295 tid 1 2024-08-17T17:09:19.709 INFO:tasks.rados.rados.0.smithi063.stdout:3424: writing smithi06387500-3 from 1634883 to 2205327 tid 2 2024-08-17T17:09:19.716 INFO:tasks.rados.rados.0.smithi063.stdout:3424: writing smithi06387500-3 from 2669481 to 3376143 tid 3 2024-08-17T17:09:19.716 INFO:tasks.rados.rados.0.smithi063.stdout:3425: write oid 5 current snap is 434 2024-08-17T17:09:19.716 INFO:tasks.rados.rados.0.smithi063.stdout:3425: seq_num 1275 ranges {628925=573426,1693365=714102,3096904=332021} 2024-08-17T17:09:19.722 INFO:tasks.rados.rados.0.smithi063.stdout:3425: writing smithi06387500-5 from 628925 to 1202351 tid 1 2024-08-17T17:09:19.729 INFO:tasks.rados.rados.0.smithi063.stdout:3425: writing smithi06387500-5 from 1693365 to 2407467 tid 2 2024-08-17T17:09:19.735 INFO:tasks.rados.rados.0.smithi063.stdout:3425: writing smithi06387500-5 from 3096904 to 3428925 tid 3 2024-08-17T17:09:19.735 INFO:tasks.rados.rados.0.smithi063.stdout:3426: rollback oid 38 current snap is 434 2024-08-17T17:09:19.735 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 38 to 378 2024-08-17T17:09:19.735 INFO:tasks.rados.rados.0.smithi063.stdout:3421: finishing write tid 2 to smithi06387500-25 2024-08-17T17:09:19.735 INFO:tasks.rados.rados.0.smithi063.stdout:3421: finishing write tid 3 to smithi06387500-25 2024-08-17T17:09:19.735 INFO:tasks.rados.rados.0.smithi063.stdout:3421: finishing write tid 4 to smithi06387500-25 2024-08-17T17:09:19.736 INFO:tasks.rados.rados.0.smithi063.stdout:3421: finishing write tid 5 to smithi06387500-25 2024-08-17T17:09:19.736 INFO:tasks.rados.rados.0.smithi063.stdout:3421: oid 25 updating version 0 to 3328 2024-08-17T17:09:19.736 INFO:tasks.rados.rados.0.smithi063.stdout:3421: oid 25 updating version 3328 to 3331 2024-08-17T17:09:19.736 INFO:tasks.rados.rados.0.smithi063.stdout:3421: oid 25 version 3331 is already newer than 3330 2024-08-17T17:09:19.736 INFO:tasks.rados.rados.0.smithi063.stdout:3421: oid 25 version 3331 is already newer than 3329 2024-08-17T17:09:19.736 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 3331 (ObjNum 1273 snap 434 seq_num 1273) dirty exists 2024-08-17T17:09:19.736 INFO:tasks.rados.rados.0.smithi063.stdout:3421: left oid 25 (ObjNum 1273 snap 434 seq_num 1273) 2024-08-17T17:09:19.736 INFO:tasks.rados.rados.0.smithi063.stdout:3422: finishing rollback tid 0 to smithi06387500-18 2024-08-17T17:09:19.736 INFO:tasks.rados.rados.0.smithi063.stdout:3422: finishing rollback tid 1 to smithi06387500-18 2024-08-17T17:09:19.736 INFO:tasks.rados.rados.0.smithi063.stdout:3423: expect (ObjNum 1205 snap 404 seq_num 1205) 2024-08-17T17:09:19.840 INFO:tasks.rados.rados.0.smithi063.stdout:3422: finishing rollback tid 2 to smithi06387500-18 2024-08-17T17:09:19.840 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 1159 (ObjNum 1201 snap 404 seq_num 1201) dirty exists 2024-08-17T17:09:19.840 INFO:tasks.rados.rados.0.smithi063.stdout:3424: finishing write tid 1 to smithi06387500-3 2024-08-17T17:09:19.840 INFO:tasks.rados.rados.0.smithi063.stdout:3424: finishing write tid 2 to smithi06387500-3 2024-08-17T17:09:19.840 INFO:tasks.rados.rados.0.smithi063.stdout:3424: finishing write tid 3 to smithi06387500-3 2024-08-17T17:09:19.841 INFO:tasks.rados.rados.0.smithi063.stdout:3424: finishing write tid 4 to smithi06387500-3 2024-08-17T17:09:19.841 INFO:tasks.rados.rados.0.smithi063.stdout:3421: done (5 left) 2024-08-17T17:09:19.841 INFO:tasks.rados.rados.0.smithi063.stdout:3422: done (4 left) 2024-08-17T17:09:19.841 INFO:tasks.rados.rados.0.smithi063.stdout:3423: done (3 left) 2024-08-17T17:09:19.841 INFO:tasks.rados.rados.0.smithi063.stdout:3427: read oid 12 snap -1 2024-08-17T17:09:19.841 INFO:tasks.rados.rados.0.smithi063.stdout:3427: expect deleted 2024-08-17T17:09:19.841 INFO:tasks.rados.rados.0.smithi063.stdout:3428: delete oid 42 current snap is 434 2024-08-17T17:09:19.841 INFO:tasks.rados.rados.0.smithi063.stdout:3424: finishing write tid 5 to smithi06387500-3 2024-08-17T17:09:19.841 INFO:tasks.rados.rados.0.smithi063.stdout:3424: oid 3 updating version 0 to 1173 2024-08-17T17:09:19.842 INFO:tasks.rados.rados.0.smithi063.stdout:3424: oid 3 version 1173 is already newer than 1170 2024-08-17T17:09:19.842 INFO:tasks.rados.rados.0.smithi063.stdout:3424: oid 3 version 1173 is already newer than 1171 2024-08-17T17:09:19.842 INFO:tasks.rados.rados.0.smithi063.stdout:3424: oid 3 version 1173 is already newer than 1172 2024-08-17T17:09:19.842 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 1173 (ObjNum 1274 snap 434 seq_num 1274) dirty exists 2024-08-17T17:09:19.842 INFO:tasks.rados.rados.0.smithi063.stdout:3424: left oid 3 (ObjNum 1274 snap 434 seq_num 1274) 2024-08-17T17:09:19.842 INFO:tasks.rados.rados.0.smithi063.stdout:3425: finishing write tid 1 to smithi06387500-5 2024-08-17T17:09:19.842 INFO:tasks.rados.rados.0.smithi063.stdout:3425: finishing write tid 2 to smithi06387500-5 2024-08-17T17:09:19.842 INFO:tasks.rados.rados.0.smithi063.stdout:3425: finishing write tid 3 to smithi06387500-5 2024-08-17T17:09:19.842 INFO:tasks.rados.rados.0.smithi063.stdout:3425: finishing write tid 4 to smithi06387500-5 2024-08-17T17:09:19.842 INFO:tasks.rados.rados.0.smithi063.stdout:3425: finishing write tid 5 to smithi06387500-5 2024-08-17T17:09:19.842 INFO:tasks.rados.rados.0.smithi063.stdout:3425: oid 5 updating version 0 to 1759 2024-08-17T17:09:19.843 INFO:tasks.rados.rados.0.smithi063.stdout:3425: oid 5 updating version 1759 to 1760 2024-08-17T17:09:19.843 INFO:tasks.rados.rados.0.smithi063.stdout:3425: oid 5 version 1760 is already newer than 1758 2024-08-17T17:09:19.843 INFO:tasks.rados.rados.0.smithi063.stdout:3425: oid 5 updating version 1760 to 1761 2024-08-17T17:09:19.843 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1761 (ObjNum 1275 snap 434 seq_num 1275) dirty exists 2024-08-17T17:09:19.843 INFO:tasks.rados.rados.0.smithi063.stdout:3425: left oid 5 (ObjNum 1275 snap 434 seq_num 1275) 2024-08-17T17:09:19.843 INFO:tasks.rados.rados.0.smithi063.stdout:3426: finishing rollback tid 1 to smithi06387500-38 2024-08-17T17:09:19.843 INFO:tasks.rados.rados.0.smithi063.stdout:3426: finishing rollback tid 2 to smithi06387500-38 2024-08-17T17:09:19.843 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 643 (ObjNum 1083 snap 363 seq_num 1083) dirty exists 2024-08-17T17:09:19.844 INFO:tasks.rados.rados.0.smithi063.stdout:3424: done (4 left) 2024-08-17T17:09:19.844 INFO:tasks.rados.rados.0.smithi063.stdout:3425: done (3 left) 2024-08-17T17:09:19.844 INFO:tasks.rados.rados.0.smithi063.stdout:3426: done (2 left) 2024-08-17T17:09:19.844 INFO:tasks.rados.rados.0.smithi063.stdout:3427: done (1 left) 2024-08-17T17:09:19.844 INFO:tasks.rados.rados.0.smithi063.stdout:3428: done (0 left) 2024-08-17T17:09:19.844 INFO:tasks.rados.rados.0.smithi063.stdout:3429: write oid 38 current snap is 434 2024-08-17T17:09:19.844 INFO:tasks.rados.rados.0.smithi063.stdout:3429: seq_num 1277 ranges {662539=723924,2093976=717771,3435397=27142} 2024-08-17T17:09:19.849 INFO:tasks.rados.rados.0.smithi063.stdout:3429: writing smithi06387500-38 from 662539 to 1386463 tid 1 2024-08-17T17:09:19.857 INFO:tasks.rados.rados.0.smithi063.stdout:3429: writing smithi06387500-38 from 2093976 to 2811747 tid 2 2024-08-17T17:09:19.861 INFO:tasks.rados.rados.0.smithi063.stdout:3429: writing smithi06387500-38 from 3435397 to 3462539 tid 3 2024-08-17T17:09:19.861 INFO:tasks.rados.rados.0.smithi063.stdout:3430: read oid 12 snap 378 2024-08-17T17:09:19.861 INFO:tasks.rados.rados.0.smithi063.stdout:3430: expect deleted 2024-08-17T17:09:19.861 INFO:tasks.rados.rados.0.smithi063.stdout:3431: write oid 37 current snap is 434 2024-08-17T17:09:19.861 INFO:tasks.rados.rados.0.smithi063.stdout:3429: finishing write tid 1 to smithi06387500-38 2024-08-17T17:09:19.861 INFO:tasks.rados.rados.0.smithi063.stdout:3429: finishing write tid 2 to smithi06387500-38 2024-08-17T17:09:19.861 INFO:tasks.rados.rados.0.smithi063.stdout:3431: seq_num 1278 ranges {679346=515526,1868811=577782,3179173=300173} 2024-08-17T17:09:19.867 INFO:tasks.rados.rados.0.smithi063.stdout:3431: writing smithi06387500-37 from 679346 to 1194872 tid 1 2024-08-17T17:09:19.874 INFO:tasks.rados.rados.0.smithi063.stdout:3431: writing smithi06387500-37 from 1868811 to 2446593 tid 2 2024-08-17T17:09:19.879 INFO:tasks.rados.rados.0.smithi063.stdout:3431: writing smithi06387500-37 from 3179173 to 3479346 tid 3 2024-08-17T17:09:19.879 INFO:tasks.rados.rados.0.smithi063.stdout:3432: snap_remove snap 425 2024-08-17T17:09:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:19 smithi063 ceph-mon[114756]: pgmap v852: 89 pgs: 89 active+clean; 271 MiB data, 871 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 520 KiB/s wr, 8 op/s 2024-08-17T17:09:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:19 smithi063 ceph-mon[114756]: osdmap e931: 8 total, 8 up, 8 in 2024-08-17T17:09:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:19 smithi063 ceph-mon[109843]: pgmap v852: 89 pgs: 89 active+clean; 271 MiB data, 871 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 520 KiB/s wr, 8 op/s 2024-08-17T17:09:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:19 smithi063 ceph-mon[109843]: osdmap e931: 8 total, 8 up, 8 in 2024-08-17T17:09:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:19 smithi162 ceph-mon[104668]: pgmap v852: 89 pgs: 89 active+clean; 271 MiB data, 871 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 520 KiB/s wr, 8 op/s 2024-08-17T17:09:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:19 smithi162 ceph-mon[104668]: osdmap e931: 8 total, 8 up, 8 in 2024-08-17T17:09:20.672 INFO:tasks.rados.rados.0.smithi063.stdout:3432: done (3 left) 2024-08-17T17:09:20.672 INFO:tasks.rados.rados.0.smithi063.stdout:3433: write oid 7 current snap is 434 2024-08-17T17:09:20.672 INFO:tasks.rados.rados.0.smithi063.stdout:3433: seq_num 1279 ranges {696153=590775,1927293=721440,3206596=289557} 2024-08-17T17:09:20.680 INFO:tasks.rados.rados.0.smithi063.stdout:3433: writing smithi06387500-7 from 696153 to 1286928 tid 1 2024-08-17T17:09:20.687 INFO:tasks.rados.rados.0.smithi063.stdout:3433: writing smithi06387500-7 from 1927293 to 2648733 tid 2 2024-08-17T17:09:20.693 INFO:tasks.rados.rados.0.smithi063.stdout:3433: writing smithi06387500-7 from 3206596 to 3496153 tid 3 2024-08-17T17:09:20.693 INFO:tasks.rados.rados.0.smithi063.stdout:3434: snap_create 2024-08-17T17:09:20.693 INFO:tasks.rados.rados.0.smithi063.stdout:3429: finishing write tid 3 to smithi06387500-38 2024-08-17T17:09:20.693 INFO:tasks.rados.rados.0.smithi063.stdout:3429: finishing write tid 4 to smithi06387500-38 2024-08-17T17:09:20.693 INFO:tasks.rados.rados.0.smithi063.stdout:3429: finishing write tid 5 to smithi06387500-38 2024-08-17T17:09:20.693 INFO:tasks.rados.rados.0.smithi063.stdout:3429: oid 38 updating version 0 to 645 2024-08-17T17:09:20.693 INFO:tasks.rados.rados.0.smithi063.stdout:3429: oid 38 updating version 645 to 647 2024-08-17T17:09:20.693 INFO:tasks.rados.rados.0.smithi063.stdout:3429: oid 38 version 647 is already newer than 646 2024-08-17T17:09:20.693 INFO:tasks.rados.rados.0.smithi063.stdout:3429: oid 38 version 647 is already newer than 644 2024-08-17T17:09:20.693 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 647 (ObjNum 1277 snap 434 seq_num 1277) dirty exists 2024-08-17T17:09:20.694 INFO:tasks.rados.rados.0.smithi063.stdout:3429: left oid 38 (ObjNum 1277 snap 434 seq_num 1277) 2024-08-17T17:09:20.694 INFO:tasks.rados.rados.0.smithi063.stdout:3431: finishing write tid 1 to smithi06387500-37 2024-08-17T17:09:20.694 INFO:tasks.rados.rados.0.smithi063.stdout:3431: finishing write tid 2 to smithi06387500-37 2024-08-17T17:09:20.694 INFO:tasks.rados.rados.0.smithi063.stdout:3431: finishing write tid 3 to smithi06387500-37 2024-08-17T17:09:20.694 INFO:tasks.rados.rados.0.smithi063.stdout:3431: finishing write tid 4 to smithi06387500-37 2024-08-17T17:09:20.694 INFO:tasks.rados.rados.0.smithi063.stdout:3431: finishing write tid 5 to smithi06387500-37 2024-08-17T17:09:20.694 INFO:tasks.rados.rados.0.smithi063.stdout:3431: oid 37 updating version 0 to 3332 2024-08-17T17:09:20.694 INFO:tasks.rados.rados.0.smithi063.stdout:3431: oid 37 updating version 3332 to 3335 2024-08-17T17:09:20.694 INFO:tasks.rados.rados.0.smithi063.stdout:3431: oid 37 version 3335 is already newer than 3334 2024-08-17T17:09:20.694 INFO:tasks.rados.rados.0.smithi063.stdout:3431: oid 37 version 3335 is already newer than 3333 2024-08-17T17:09:20.694 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 3335 (ObjNum 1278 snap 434 seq_num 1278) dirty exists 2024-08-17T17:09:20.695 INFO:tasks.rados.rados.0.smithi063.stdout:3431: left oid 37 (ObjNum 1278 snap 434 seq_num 1278) 2024-08-17T17:09:20.695 INFO:tasks.rados.rados.0.smithi063.stdout:3433: finishing write tid 1 to smithi06387500-7 2024-08-17T17:09:20.695 INFO:tasks.rados.rados.0.smithi063.stdout:3433: finishing write tid 2 to smithi06387500-7 2024-08-17T17:09:20.695 INFO:tasks.rados.rados.0.smithi063.stdout:3433: finishing write tid 3 to smithi06387500-7 2024-08-17T17:09:20.695 INFO:tasks.rados.rados.0.smithi063.stdout:3433: finishing write tid 4 to smithi06387500-7 2024-08-17T17:09:20.695 INFO:tasks.rados.rados.0.smithi063.stdout:3433: finishing write tid 5 to smithi06387500-7 2024-08-17T17:09:20.696 INFO:tasks.rados.rados.0.smithi063.stdout:3433: oid 7 updating version 0 to 1631 2024-08-17T17:09:20.696 INFO:tasks.rados.rados.0.smithi063.stdout:3433: oid 7 updating version 1631 to 1632 2024-08-17T17:09:20.696 INFO:tasks.rados.rados.0.smithi063.stdout:3433: oid 7 updating version 1632 to 1633 2024-08-17T17:09:20.696 INFO:tasks.rados.rados.0.smithi063.stdout:3433: oid 7 version 1633 is already newer than 1630 2024-08-17T17:09:20.696 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1633 (ObjNum 1279 snap 434 seq_num 1279) dirty exists 2024-08-17T17:09:20.696 INFO:tasks.rados.rados.0.smithi063.stdout:3433: left oid 7 (ObjNum 1279 snap 434 seq_num 1279) 2024-08-17T17:09:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:20 smithi063 ceph-mon[114756]: osdmap e932: 8 total, 8 up, 8 in 2024-08-17T17:09:21.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:20 smithi063 ceph-mon[109843]: osdmap e932: 8 total, 8 up, 8 in 2024-08-17T17:09:21.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:20 smithi162 ceph-mon[104668]: osdmap e932: 8 total, 8 up, 8 in 2024-08-17T17:09:21.682 INFO:tasks.rados.rados.0.smithi063.stdout:3429: done (4 left) 2024-08-17T17:09:21.682 INFO:tasks.rados.rados.0.smithi063.stdout:3430: done (3 left) 2024-08-17T17:09:21.682 INFO:tasks.rados.rados.0.smithi063.stdout:3431: done (2 left) 2024-08-17T17:09:21.682 INFO:tasks.rados.rados.0.smithi063.stdout:3433: done (1 left) 2024-08-17T17:09:21.682 INFO:tasks.rados.rados.0.smithi063.stdout:3434: done (0 left) 2024-08-17T17:09:21.682 INFO:tasks.rados.rados.0.smithi063.stdout:3435: write oid 16 current snap is 435 2024-08-17T17:09:21.682 INFO:tasks.rados.rados.0.smithi063.stdout:3435: seq_num 1280 ranges {712960=666024,1985775=581451,3234019=278941} 2024-08-17T17:09:21.689 INFO:tasks.rados.rados.0.smithi063.stdout:3435: writing smithi06387500-16 from 712960 to 1378984 tid 1 2024-08-17T17:09:21.697 INFO:tasks.rados.rados.0.smithi063.stdout:3435: writing smithi06387500-16 from 1985775 to 2567226 tid 2 2024-08-17T17:09:21.702 INFO:tasks.rados.rados.0.smithi063.stdout:3435: writing smithi06387500-16 from 3234019 to 3512960 tid 3 2024-08-17T17:09:21.702 INFO:tasks.rados.rados.0.smithi063.stdout:3436: write oid 24 current snap is 435 2024-08-17T17:09:21.702 INFO:tasks.rados.rados.0.smithi063.stdout:3435: finishing write tid 1 to smithi06387500-16 2024-08-17T17:09:21.702 INFO:tasks.rados.rados.0.smithi063.stdout:3435: finishing write tid 2 to smithi06387500-16 2024-08-17T17:09:21.703 INFO:tasks.rados.rados.0.smithi063.stdout:3436: seq_num 1281 ranges {729767=741273,1927904=725109,3145089=384678} 2024-08-17T17:09:21.710 INFO:tasks.rados.rados.0.smithi063.stdout:3436: writing smithi06387500-24 from 729767 to 1471040 tid 1 2024-08-17T17:09:21.716 INFO:tasks.rados.rados.0.smithi063.stdout:3436: writing smithi06387500-24 from 1927904 to 2653013 tid 2 2024-08-17T17:09:21.722 INFO:tasks.rados.rados.0.smithi063.stdout:3436: writing smithi06387500-24 from 3145089 to 3529767 tid 3 2024-08-17T17:09:21.722 INFO:tasks.rados.rados.0.smithi063.stdout:3437: rollback oid 35 current snap is 435 2024-08-17T17:09:21.722 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 35 to 427 2024-08-17T17:09:21.722 INFO:tasks.rados.rados.0.smithi063.stdout:3435: finishing write tid 3 to smithi06387500-16 2024-08-17T17:09:21.722 INFO:tasks.rados.rados.0.smithi063.stdout:3435: finishing write tid 4 to smithi06387500-16 2024-08-17T17:09:21.722 INFO:tasks.rados.rados.0.smithi063.stdout:3435: finishing write tid 5 to smithi06387500-16 2024-08-17T17:09:21.722 INFO:tasks.rados.rados.0.smithi063.stdout:3435: oid 16 updating version 0 to 1227 2024-08-17T17:09:21.722 INFO:tasks.rados.rados.0.smithi063.stdout:3435: oid 16 updating version 1227 to 1229 2024-08-17T17:09:21.722 INFO:tasks.rados.rados.0.smithi063.stdout:3435: oid 16 version 1229 is already newer than 1226 2024-08-17T17:09:21.723 INFO:tasks.rados.rados.0.smithi063.stdout:3435: oid 16 version 1229 is already newer than 1228 2024-08-17T17:09:21.723 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 1229 (ObjNum 1280 snap 435 seq_num 1280) dirty exists 2024-08-17T17:09:21.723 INFO:tasks.rados.rados.0.smithi063.stdout:3435: left oid 16 (ObjNum 1280 snap 435 seq_num 1280) 2024-08-17T17:09:21.723 INFO:tasks.rados.rados.0.smithi063.stdout:3436: finishing write tid 1 to smithi06387500-24 2024-08-17T17:09:21.723 INFO:tasks.rados.rados.0.smithi063.stdout:3436: finishing write tid 2 to smithi06387500-24 2024-08-17T17:09:21.723 INFO:tasks.rados.rados.0.smithi063.stdout:3435: done (2 left) 2024-08-17T17:09:21.723 INFO:tasks.rados.rados.0.smithi063.stdout:3438: write oid 45 current snap is 435 2024-08-17T17:09:21.723 INFO:tasks.rados.rados.0.smithi063.stdout:3438: seq_num 1282 ranges {746574=416522,1586386=468767,2772512=629544} 2024-08-17T17:09:21.728 INFO:tasks.rados.rados.0.smithi063.stdout:3438: writing smithi06387500-45 from 746574 to 1163096 tid 1 2024-08-17T17:09:21.733 INFO:tasks.rados.rados.0.smithi063.stdout:3438: writing smithi06387500-45 from 1586386 to 2055153 tid 2 2024-08-17T17:09:21.741 INFO:tasks.rados.rados.0.smithi063.stdout:3438: writing smithi06387500-45 from 2772512 to 3402056 tid 3 2024-08-17T17:09:21.741 INFO:tasks.rados.rados.0.smithi063.stdout:3439: snap_create 2024-08-17T17:09:21.741 INFO:tasks.rados.rados.0.smithi063.stdout:3436: finishing write tid 3 to smithi06387500-24 2024-08-17T17:09:21.741 INFO:tasks.rados.rados.0.smithi063.stdout:3436: finishing write tid 4 to smithi06387500-24 2024-08-17T17:09:21.741 INFO:tasks.rados.rados.0.smithi063.stdout:3436: finishing write tid 5 to smithi06387500-24 2024-08-17T17:09:21.741 INFO:tasks.rados.rados.0.smithi063.stdout:3436: oid 24 updating version 0 to 1233 2024-08-17T17:09:21.741 INFO:tasks.rados.rados.0.smithi063.stdout:3436: oid 24 version 1233 is already newer than 1232 2024-08-17T17:09:21.741 INFO:tasks.rados.rados.0.smithi063.stdout:3436: oid 24 updating version 1233 to 1234 2024-08-17T17:09:21.741 INFO:tasks.rados.rados.0.smithi063.stdout:3436: oid 24 version 1234 is already newer than 1231 2024-08-17T17:09:21.742 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 1234 (ObjNum 1281 snap 435 seq_num 1281) dirty exists 2024-08-17T17:09:21.742 INFO:tasks.rados.rados.0.smithi063.stdout:3436: left oid 24 (ObjNum 1281 snap 435 seq_num 1281) 2024-08-17T17:09:21.742 INFO:tasks.rados.rados.0.smithi063.stdout:3437: finishing rollback tid 1 to smithi06387500-35 2024-08-17T17:09:21.742 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 3238 (ObjNum 129 snap 0 seq_num 1806975408) dirty dne 2024-08-17T17:09:21.742 INFO:tasks.rados.rados.0.smithi063.stdout:3438: finishing write tid 1 to smithi06387500-45 2024-08-17T17:09:21.742 INFO:tasks.rados.rados.0.smithi063.stdout:3438: finishing write tid 2 to smithi06387500-45 2024-08-17T17:09:21.745 INFO:tasks.rados.rados.0.smithi063.stdout:3438: finishing write tid 3 to smithi06387500-45 2024-08-17T17:09:21.745 INFO:tasks.rados.rados.0.smithi063.stdout:3438: finishing write tid 4 to smithi06387500-45 2024-08-17T17:09:21.745 INFO:tasks.rados.rados.0.smithi063.stdout:3438: finishing write tid 5 to smithi06387500-45 2024-08-17T17:09:21.745 INFO:tasks.rados.rados.0.smithi063.stdout:3438: oid 45 updating version 0 to 849 2024-08-17T17:09:21.746 INFO:tasks.rados.rados.0.smithi063.stdout:3438: oid 45 version 849 is already newer than 846 2024-08-17T17:09:21.746 INFO:tasks.rados.rados.0.smithi063.stdout:3438: oid 45 version 849 is already newer than 847 2024-08-17T17:09:21.746 INFO:tasks.rados.rados.0.smithi063.stdout:3438: oid 45 version 849 is already newer than 848 2024-08-17T17:09:21.746 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 849 (ObjNum 1282 snap 435 seq_num 1282) dirty exists 2024-08-17T17:09:21.746 INFO:tasks.rados.rados.0.smithi063.stdout:3438: left oid 45 (ObjNum 1282 snap 435 seq_num 1282) 2024-08-17T17:09:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:21 smithi063 ceph-mon[114756]: pgmap v855: 89 pgs: 89 active+clean; 271 MiB data, 870 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 364 KiB/s wr, 7 op/s 2024-08-17T17:09:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:21 smithi063 ceph-mon[114756]: osdmap e933: 8 total, 8 up, 8 in 2024-08-17T17:09:22.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:21 smithi063 ceph-mon[109843]: pgmap v855: 89 pgs: 89 active+clean; 271 MiB data, 870 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 364 KiB/s wr, 7 op/s 2024-08-17T17:09:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:21 smithi063 ceph-mon[109843]: osdmap e933: 8 total, 8 up, 8 in 2024-08-17T17:09:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:21 smithi162 ceph-mon[104668]: pgmap v855: 89 pgs: 89 active+clean; 271 MiB data, 870 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 364 KiB/s wr, 7 op/s 2024-08-17T17:09:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:21 smithi162 ceph-mon[104668]: osdmap e933: 8 total, 8 up, 8 in 2024-08-17T17:09:22.691 INFO:tasks.rados.rados.0.smithi063.stdout:3436: done (3 left) 2024-08-17T17:09:22.691 INFO:tasks.rados.rados.0.smithi063.stdout:3437: done (2 left) 2024-08-17T17:09:22.691 INFO:tasks.rados.rados.0.smithi063.stdout:3438: done (1 left) 2024-08-17T17:09:22.691 INFO:tasks.rados.rados.0.smithi063.stdout:3439: done (0 left) 2024-08-17T17:09:22.691 INFO:tasks.rados.rados.0.smithi063.stdout:3440: snap_remove snap 378 2024-08-17T17:09:23.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:22 smithi063 ceph-mon[109843]: osdmap e934: 8 total, 8 up, 8 in 2024-08-17T17:09:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:22 smithi063 ceph-mon[114756]: osdmap e934: 8 total, 8 up, 8 in 2024-08-17T17:09:23.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:22 smithi162 ceph-mon[104668]: osdmap e934: 8 total, 8 up, 8 in 2024-08-17T17:09:23.685 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:09:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:09:23] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.51.0" 2024-08-17T17:09:23.701 INFO:tasks.rados.rados.0.smithi063.stdout:3440: done (0 left) 2024-08-17T17:09:23.701 INFO:tasks.rados.rados.0.smithi063.stdout:3441: read oid 31 snap 427 2024-08-17T17:09:23.701 INFO:tasks.rados.rados.0.smithi063.stdout:3441: expect deleted 2024-08-17T17:09:23.701 INFO:tasks.rados.rados.0.smithi063.stdout:3442: write oid 25 current snap is 436 2024-08-17T17:09:23.701 INFO:tasks.rados.rados.0.smithi063.stdout:3442: seq_num 1283 ranges {763381=491771,2044868=728778,3199935=363446} 2024-08-17T17:09:23.708 INFO:tasks.rados.rados.0.smithi063.stdout:3442: writing smithi06387500-25 from 763381 to 1255152 tid 1 2024-08-17T17:09:23.718 INFO:tasks.rados.rados.0.smithi063.stdout:3442: writing smithi06387500-25 from 2044868 to 2773646 tid 2 2024-08-17T17:09:23.722 INFO:tasks.rados.rados.0.smithi063.stdout:3442: writing smithi06387500-25 from 3199935 to 3563381 tid 3 2024-08-17T17:09:23.722 INFO:tasks.rados.rados.0.smithi063.stdout:3443: snap_remove snap 429 2024-08-17T17:09:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:23 smithi063 ceph-mon[109843]: pgmap v858: 89 pgs: 89 active+clean; 290 MiB data, 900 MiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 3.4 MiB/s wr, 21 op/s 2024-08-17T17:09:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:23 smithi063 ceph-mon[109843]: osdmap e935: 8 total, 8 up, 8 in 2024-08-17T17:09:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:23 smithi063 ceph-mon[114756]: pgmap v858: 89 pgs: 89 active+clean; 290 MiB data, 900 MiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 3.4 MiB/s wr, 21 op/s 2024-08-17T17:09:24.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:23 smithi063 ceph-mon[114756]: osdmap e935: 8 total, 8 up, 8 in 2024-08-17T17:09:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:23 smithi162 ceph-mon[104668]: pgmap v858: 89 pgs: 89 active+clean; 290 MiB data, 900 MiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 3.4 MiB/s wr, 21 op/s 2024-08-17T17:09:24.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:23 smithi162 ceph-mon[104668]: osdmap e935: 8 total, 8 up, 8 in 2024-08-17T17:09:24.703 INFO:tasks.rados.rados.0.smithi063.stdout:3442: finishing write tid 1 to smithi06387500-25 2024-08-17T17:09:24.703 INFO:tasks.rados.rados.0.smithi063.stdout:3441: done (2 left) 2024-08-17T17:09:24.703 INFO:tasks.rados.rados.0.smithi063.stdout:3443: done (1 left) 2024-08-17T17:09:24.703 INFO:tasks.rados.rados.0.smithi063.stdout:3442: finishing write tid 2 to smithi06387500-25 2024-08-17T17:09:24.703 INFO:tasks.rados.rados.0.smithi063.stdout:3442: finishing write tid 3 to smithi06387500-25 2024-08-17T17:09:24.703 INFO:tasks.rados.rados.0.smithi063.stdout:3442: finishing write tid 4 to smithi06387500-25 2024-08-17T17:09:24.703 INFO:tasks.rados.rados.0.smithi063.stdout:3442: finishing write tid 5 to smithi06387500-25 2024-08-17T17:09:24.703 INFO:tasks.rados.rados.0.smithi063.stdout:3442: oid 25 updating version 0 to 3359 2024-08-17T17:09:24.703 INFO:tasks.rados.rados.0.smithi063.stdout:3442: oid 25 version 3359 is already newer than 3358 2024-08-17T17:09:24.703 INFO:tasks.rados.rados.0.smithi063.stdout:3442: oid 25 version 3359 is already newer than 3352 2024-08-17T17:09:24.703 INFO:tasks.rados.rados.0.smithi063.stdout:3442: oid 25 version 3359 is already newer than 3357 2024-08-17T17:09:24.703 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 3359 (ObjNum 1283 snap 436 seq_num 1283) dirty exists 2024-08-17T17:09:24.704 INFO:tasks.rados.rados.0.smithi063.stdout:3442: left oid 25 (ObjNum 1283 snap 436 seq_num 1283) 2024-08-17T17:09:24.704 INFO:tasks.rados.rados.0.smithi063.stdout:3444: read oid 27 snap -1 2024-08-17T17:09:24.704 INFO:tasks.rados.rados.0.smithi063.stdout:3444: expect (ObjNum 1238 snap 413 seq_num 1238) 2024-08-17T17:09:24.704 INFO:tasks.rados.rados.0.smithi063.stdout:3442: done (1 left) 2024-08-17T17:09:24.704 INFO:tasks.rados.rados.0.smithi063.stdout:3445: snap_create 2024-08-17T17:09:24.712 INFO:tasks.rados.rados.0.smithi063.stdout:3444: expect (ObjNum 1238 snap 413 seq_num 1238) 2024-08-17T17:09:25.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:24 smithi063 ceph-mon[114756]: osdmap e936: 8 total, 8 up, 8 in 2024-08-17T17:09:25.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:24 smithi063 ceph-mon[114756]: pgmap v861: 89 pgs: 89 active+clean; 290 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 3.4 MiB/s wr, 20 op/s 2024-08-17T17:09:25.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:24 smithi063 ceph-mon[109843]: osdmap e936: 8 total, 8 up, 8 in 2024-08-17T17:09:25.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:24 smithi063 ceph-mon[109843]: pgmap v861: 89 pgs: 89 active+clean; 290 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 3.4 MiB/s wr, 20 op/s 2024-08-17T17:09:25.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:24 smithi162 ceph-mon[104668]: osdmap e936: 8 total, 8 up, 8 in 2024-08-17T17:09:25.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:24 smithi162 ceph-mon[104668]: pgmap v861: 89 pgs: 89 active+clean; 290 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 3.4 MiB/s wr, 20 op/s 2024-08-17T17:09:25.702 INFO:tasks.rados.rados.0.smithi063.stdout:3444: done (1 left) 2024-08-17T17:09:25.703 INFO:tasks.rados.rados.0.smithi063.stdout:3445: done (0 left) 2024-08-17T17:09:25.703 INFO:tasks.rados.rados.0.smithi063.stdout:3446: snap_remove snap 424 2024-08-17T17:09:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:25 smithi063 ceph-mon[114756]: osdmap e937: 8 total, 8 up, 8 in 2024-08-17T17:09:26.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:25 smithi063 ceph-mon[109843]: osdmap e937: 8 total, 8 up, 8 in 2024-08-17T17:09:26.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:25 smithi162 ceph-mon[104668]: osdmap e937: 8 total, 8 up, 8 in 2024-08-17T17:09:26.715 INFO:tasks.rados.rados.0.smithi063.stdout:3446: done (0 left) 2024-08-17T17:09:26.715 INFO:tasks.rados.rados.0.smithi063.stdout:3447: rollback oid 11 current snap is 437 2024-08-17T17:09:26.715 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 11 to 426 2024-08-17T17:09:26.715 INFO:tasks.rados.rados.0.smithi063.stdout:3448: snap_remove snap 402 2024-08-17T17:09:27.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:26 smithi063 ceph-mon[114756]: osdmap e938: 8 total, 8 up, 8 in 2024-08-17T17:09:27.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:26 smithi063 ceph-mon[114756]: pgmap v864: 89 pgs: 89 active+clean; 276 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 825 KiB/s wr, 5 op/s 2024-08-17T17:09:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:26 smithi063 ceph-mon[109843]: osdmap e938: 8 total, 8 up, 8 in 2024-08-17T17:09:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:26 smithi063 ceph-mon[109843]: pgmap v864: 89 pgs: 89 active+clean; 276 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 825 KiB/s wr, 5 op/s 2024-08-17T17:09:27.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:26 smithi162 ceph-mon[104668]: osdmap e938: 8 total, 8 up, 8 in 2024-08-17T17:09:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:26 smithi162 ceph-mon[104668]: pgmap v864: 89 pgs: 89 active+clean; 276 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 825 KiB/s wr, 5 op/s 2024-08-17T17:09:27.660 INFO:tasks.rados.rados.0.smithi063.stdout:3448: done (1 left) 2024-08-17T17:09:27.660 INFO:tasks.rados.rados.0.smithi063.stdout:3449: rollback oid 47 current snap is 437 2024-08-17T17:09:27.661 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 47 to 431 2024-08-17T17:09:27.661 INFO:tasks.rados.rados.0.smithi063.stdout:3447: finishing rollback tid 0 to smithi06387500-11 2024-08-17T17:09:27.661 INFO:tasks.rados.rados.0.smithi063.stdout:3447: finishing rollback tid 1 to smithi06387500-11 2024-08-17T17:09:27.661 INFO:tasks.rados.rados.0.smithi063.stdout:3447: finishing rollback tid 2 to smithi06387500-11 2024-08-17T17:09:27.661 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 653 (ObjNum 1163 snap 392 seq_num 1163) dirty exists 2024-08-17T17:09:27.661 INFO:tasks.rados.rados.0.smithi063.stdout:3447: done (1 left) 2024-08-17T17:09:27.661 INFO:tasks.rados.rados.0.smithi063.stdout:3450: delete oid 2 current snap is 437 2024-08-17T17:09:27.669 INFO:tasks.rados.rados.0.smithi063.stdout:3450: done (1 left) 2024-08-17T17:09:27.669 INFO:tasks.rados.rados.0.smithi063.stdout:3451: snap_create 2024-08-17T17:09:27.670 INFO:tasks.rados.rados.0.smithi063.stdout:3449: finishing rollback tid 1 to smithi06387500-47 2024-08-17T17:09:27.670 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 749 (ObjNum 657 snap 0 seq_num 1806905760) dirty dne 2024-08-17T17:09:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:27 smithi063 ceph-mon[114756]: osdmap e939: 8 total, 8 up, 8 in 2024-08-17T17:09:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:27 smithi063 ceph-mon[114756]: osdmap e940: 8 total, 8 up, 8 in 2024-08-17T17:09:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:27 smithi063 ceph-mon[109843]: osdmap e939: 8 total, 8 up, 8 in 2024-08-17T17:09:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:27 smithi063 ceph-mon[109843]: osdmap e940: 8 total, 8 up, 8 in 2024-08-17T17:09:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:27 smithi162 ceph-mon[104668]: osdmap e939: 8 total, 8 up, 8 in 2024-08-17T17:09:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:27 smithi162 ceph-mon[104668]: osdmap e940: 8 total, 8 up, 8 in 2024-08-17T17:09:28.659 INFO:tasks.rados.rados.0.smithi063.stdout:3449: done (1 left) 2024-08-17T17:09:28.660 INFO:tasks.rados.rados.0.smithi063.stdout:3451: done (0 left) 2024-08-17T17:09:28.660 INFO:tasks.rados.rados.0.smithi063.stdout:3452: snap_remove snap 423 2024-08-17T17:09:29.662 INFO:tasks.rados.rados.0.smithi063.stdout:3452: done (0 left) 2024-08-17T17:09:29.662 INFO:tasks.rados.rados.0.smithi063.stdout:3453: rollback oid 36 current snap is 438 2024-08-17T17:09:29.662 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 36 to 431 2024-08-17T17:09:29.662 INFO:tasks.rados.rados.0.smithi063.stdout:3454: snap_create 2024-08-17T17:09:29.673 INFO:tasks.rados.rados.0.smithi063.stdout:3453: finishing rollback tid 0 to smithi06387500-36 2024-08-17T17:09:29.677 INFO:tasks.rados.rados.0.smithi063.stdout:3453: finishing rollback tid 1 to smithi06387500-36 2024-08-17T17:09:29.677 INFO:tasks.rados.rados.0.smithi063.stdout:3453: finishing rollback tid 2 to smithi06387500-36 2024-08-17T17:09:29.677 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 3387 (ObjNum 1255 snap 423 seq_num 1255) dirty exists 2024-08-17T17:09:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:29 smithi063 ceph-mon[114756]: pgmap v867: 89 pgs: 89 active+clean; 244 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.2 MiB/s wr, 6 op/s 2024-08-17T17:09:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:29 smithi063 ceph-mon[114756]: osdmap e941: 8 total, 8 up, 8 in 2024-08-17T17:09:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:29 smithi063 ceph-mon[109843]: pgmap v867: 89 pgs: 89 active+clean; 244 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.2 MiB/s wr, 6 op/s 2024-08-17T17:09:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:29 smithi063 ceph-mon[109843]: osdmap e941: 8 total, 8 up, 8 in 2024-08-17T17:09:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:29 smithi162 ceph-mon[104668]: pgmap v867: 89 pgs: 89 active+clean; 244 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.2 MiB/s wr, 6 op/s 2024-08-17T17:09:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:29 smithi162 ceph-mon[104668]: osdmap e941: 8 total, 8 up, 8 in 2024-08-17T17:09:30.672 INFO:tasks.rados.rados.0.smithi063.stdout:3453: done (1 left) 2024-08-17T17:09:30.672 INFO:tasks.rados.rados.0.smithi063.stdout:3454: done (0 left) 2024-08-17T17:09:30.672 INFO:tasks.rados.rados.0.smithi063.stdout:3455: rollback oid 33 current snap is 439 2024-08-17T17:09:30.672 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 33 to 410 2024-08-17T17:09:30.672 INFO:tasks.rados.rados.0.smithi063.stdout:3456: write oid 20 current snap is 439 2024-08-17T17:09:30.672 INFO:tasks.rados.rados.0.smithi063.stdout:3456: seq_num 1285 ranges {796995=758622,2161832=732447} 2024-08-17T17:09:30.680 INFO:tasks.rados.rados.0.smithi063.stdout:3456: writing smithi06387500-20 from 796995 to 1555617 tid 1 2024-08-17T17:09:30.688 INFO:tasks.rados.rados.0.smithi063.stdout:3456: writing smithi06387500-20 from 2161832 to 2894279 tid 2 2024-08-17T17:09:30.689 INFO:tasks.rados.rados.0.smithi063.stdout:3457: write oid 18 current snap is 439 2024-08-17T17:09:30.689 INFO:tasks.rados.rados.0.smithi063.stdout:3455: finishing rollback tid 1 to smithi06387500-33 2024-08-17T17:09:30.689 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 1111 (ObjNum 65 snap 0 seq_num 1) dirty dne 2024-08-17T17:09:30.689 INFO:tasks.rados.rados.0.smithi063.stdout:3456: finishing write tid 1 to smithi06387500-20 2024-08-17T17:09:30.689 INFO:tasks.rados.rados.0.smithi063.stdout:3457: seq_num 1286 ranges {413802=433871,1420314=476105,2482204=790985} 2024-08-17T17:09:30.693 INFO:tasks.rados.rados.0.smithi063.stdout:3457: writing smithi06387500-18 from 413802 to 847673 tid 1 2024-08-17T17:09:30.699 INFO:tasks.rados.rados.0.smithi063.stdout:3457: writing smithi06387500-18 from 1420314 to 1896419 tid 2 2024-08-17T17:09:30.707 INFO:tasks.rados.rados.0.smithi063.stdout:3457: writing smithi06387500-18 from 2482204 to 3273189 tid 3 2024-08-17T17:09:30.707 INFO:tasks.rados.rados.0.smithi063.stdout:3455: done (2 left) 2024-08-17T17:09:30.707 INFO:tasks.rados.rados.0.smithi063.stdout:3458: snap_create 2024-08-17T17:09:30.707 INFO:tasks.rados.rados.0.smithi063.stdout:3456: finishing write tid 2 to smithi06387500-20 2024-08-17T17:09:30.707 INFO:tasks.rados.rados.0.smithi063.stdout:3456: finishing write tid 3 to smithi06387500-20 2024-08-17T17:09:30.707 INFO:tasks.rados.rados.0.smithi063.stdout:3456: finishing write tid 4 to smithi06387500-20 2024-08-17T17:09:30.707 INFO:tasks.rados.rados.0.smithi063.stdout:3456: oid 20 updating version 0 to 792 2024-08-17T17:09:30.707 INFO:tasks.rados.rados.0.smithi063.stdout:3456: oid 20 updating version 792 to 793 2024-08-17T17:09:30.707 INFO:tasks.rados.rados.0.smithi063.stdout:3456: oid 20 updating version 793 to 794 2024-08-17T17:09:30.707 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 794 (ObjNum 1285 snap 439 seq_num 1285) dirty exists 2024-08-17T17:09:30.708 INFO:tasks.rados.rados.0.smithi063.stdout:3456: left oid 20 (ObjNum 1285 snap 439 seq_num 1285) 2024-08-17T17:09:30.708 INFO:tasks.rados.rados.0.smithi063.stdout:3457: finishing write tid 1 to smithi06387500-18 2024-08-17T17:09:30.708 INFO:tasks.rados.rados.0.smithi063.stdout:3457: finishing write tid 2 to smithi06387500-18 2024-08-17T17:09:30.711 INFO:tasks.rados.rados.0.smithi063.stdout:3457: finishing write tid 3 to smithi06387500-18 2024-08-17T17:09:30.711 INFO:tasks.rados.rados.0.smithi063.stdout:3457: finishing write tid 4 to smithi06387500-18 2024-08-17T17:09:30.711 INFO:tasks.rados.rados.0.smithi063.stdout:3457: finishing write tid 5 to smithi06387500-18 2024-08-17T17:09:30.711 INFO:tasks.rados.rados.0.smithi063.stdout:3457: oid 18 updating version 0 to 1183 2024-08-17T17:09:30.711 INFO:tasks.rados.rados.0.smithi063.stdout:3457: oid 18 updating version 1183 to 1185 2024-08-17T17:09:30.711 INFO:tasks.rados.rados.0.smithi063.stdout:3457: oid 18 version 1185 is already newer than 1184 2024-08-17T17:09:30.711 INFO:tasks.rados.rados.0.smithi063.stdout:3457: oid 18 updating version 1185 to 1186 2024-08-17T17:09:30.711 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 1186 (ObjNum 1286 snap 439 seq_num 1286) dirty exists 2024-08-17T17:09:30.711 INFO:tasks.rados.rados.0.smithi063.stdout:3457: left oid 18 (ObjNum 1286 snap 439 seq_num 1286) 2024-08-17T17:09:30.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:30 smithi162 ceph-mon[104668]: osdmap e942: 8 total, 8 up, 8 in 2024-08-17T17:09:31.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:30 smithi063 ceph-mon[109843]: osdmap e942: 8 total, 8 up, 8 in 2024-08-17T17:09:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:30 smithi063 ceph-mon[114756]: osdmap e942: 8 total, 8 up, 8 in 2024-08-17T17:09:31.676 INFO:tasks.rados.rados.0.smithi063.stdout:3456: done (2 left) 2024-08-17T17:09:31.677 INFO:tasks.rados.rados.0.smithi063.stdout:3457: done (1 left) 2024-08-17T17:09:31.677 INFO:tasks.rados.rados.0.smithi063.stdout:3458: done (0 left) 2024-08-17T17:09:31.677 INFO:tasks.rados.rados.0.smithi063.stdout:3459: write oid 49 current snap is 440 2024-08-17T17:09:31.677 INFO:tasks.rados.rados.0.smithi063.stdout:3459: seq_num 1287 ranges {430609=509120,1478796=736116,2909627=602257} 2024-08-17T17:09:31.681 INFO:tasks.rados.rados.0.smithi063.stdout:3459: writing smithi06387500-49 from 430609 to 939729 tid 1 2024-08-17T17:09:31.689 INFO:tasks.rados.rados.0.smithi063.stdout:3459: writing smithi06387500-49 from 1478796 to 2214912 tid 2 2024-08-17T17:09:31.697 INFO:tasks.rados.rados.0.smithi063.stdout:3459: writing smithi06387500-49 from 2909627 to 3511884 tid 3 2024-08-17T17:09:31.697 INFO:tasks.rados.rados.0.smithi063.stdout:3460: write oid 11 current snap is 440 2024-08-17T17:09:31.697 INFO:tasks.rados.rados.0.smithi063.stdout:3459: finishing write tid 1 to smithi06387500-49 2024-08-17T17:09:31.697 INFO:tasks.rados.rados.0.smithi063.stdout:3459: finishing write tid 2 to smithi06387500-49 2024-08-17T17:09:31.697 INFO:tasks.rados.rados.0.smithi063.stdout:3460: seq_num 1288 ranges {447416=584369,1537278=479774,2537050=529882,3524784=122632} 2024-08-17T17:09:31.702 INFO:tasks.rados.rados.0.smithi063.stdout:3460: writing smithi06387500-11 from 447416 to 1031785 tid 1 2024-08-17T17:09:31.707 INFO:tasks.rados.rados.0.smithi063.stdout:3460: writing smithi06387500-11 from 1537278 to 2017052 tid 2 2024-08-17T17:09:31.713 INFO:tasks.rados.rados.0.smithi063.stdout:3460: writing smithi06387500-11 from 2537050 to 3066932 tid 3 2024-08-17T17:09:31.717 INFO:tasks.rados.rados.0.smithi063.stdout:3460: writing smithi06387500-11 from 3524784 to 3647416 tid 4 2024-08-17T17:09:31.717 INFO:tasks.rados.rados.0.smithi063.stdout:3461: write oid 7 current snap is 440 2024-08-17T17:09:31.717 INFO:tasks.rados.rados.0.smithi063.stdout:3461: seq_num 1289 ranges {464223=659618,1879407=739785,3248120=416103} 2024-08-17T17:09:31.723 INFO:tasks.rados.rados.0.smithi063.stdout:3461: writing smithi06387500-7 from 464223 to 1123841 tid 1 2024-08-17T17:09:31.731 INFO:tasks.rados.rados.0.smithi063.stdout:3461: writing smithi06387500-7 from 1879407 to 2619192 tid 2 2024-08-17T17:09:31.738 INFO:tasks.rados.rados.0.smithi063.stdout:3461: writing smithi06387500-7 from 3248120 to 3664223 tid 3 2024-08-17T17:09:31.738 INFO:tasks.rados.rados.0.smithi063.stdout:3462: delete oid 28 current snap is 440 2024-08-17T17:09:31.738 INFO:tasks.rados.rados.0.smithi063.stdout:3459: finishing write tid 3 to smithi06387500-49 2024-08-17T17:09:31.738 INFO:tasks.rados.rados.0.smithi063.stdout:3459: finishing write tid 4 to smithi06387500-49 2024-08-17T17:09:31.738 INFO:tasks.rados.rados.0.smithi063.stdout:3459: finishing write tid 5 to smithi06387500-49 2024-08-17T17:09:31.738 INFO:tasks.rados.rados.0.smithi063.stdout:3459: oid 49 updating version 0 to 3393 2024-08-17T17:09:31.739 INFO:tasks.rados.rados.0.smithi063.stdout:3459: oid 49 updating version 3393 to 3394 2024-08-17T17:09:31.739 INFO:tasks.rados.rados.0.smithi063.stdout:3459: oid 49 updating version 3394 to 3395 2024-08-17T17:09:31.739 INFO:tasks.rados.rados.0.smithi063.stdout:3459: oid 49 updating version 3395 to 3396 2024-08-17T17:09:31.739 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 3396 (ObjNum 1287 snap 440 seq_num 1287) dirty exists 2024-08-17T17:09:31.739 INFO:tasks.rados.rados.0.smithi063.stdout:3459: left oid 49 (ObjNum 1287 snap 440 seq_num 1287) 2024-08-17T17:09:31.739 INFO:tasks.rados.rados.0.smithi063.stdout:3460: finishing write tid 1 to smithi06387500-11 2024-08-17T17:09:31.739 INFO:tasks.rados.rados.0.smithi063.stdout:3460: finishing write tid 2 to smithi06387500-11 2024-08-17T17:09:31.739 INFO:tasks.rados.rados.0.smithi063.stdout:3460: finishing write tid 3 to smithi06387500-11 2024-08-17T17:09:31.739 INFO:tasks.rados.rados.0.smithi063.stdout:3460: finishing write tid 4 to smithi06387500-11 2024-08-17T17:09:31.739 INFO:tasks.rados.rados.0.smithi063.stdout:3460: finishing write tid 5 to smithi06387500-11 2024-08-17T17:09:31.739 INFO:tasks.rados.rados.0.smithi063.stdout:3460: finishing write tid 6 to smithi06387500-11 2024-08-17T17:09:31.739 INFO:tasks.rados.rados.0.smithi063.stdout:3460: oid 11 updating version 0 to 663 2024-08-17T17:09:31.740 INFO:tasks.rados.rados.0.smithi063.stdout:3460: oid 11 updating version 663 to 665 2024-08-17T17:09:31.740 INFO:tasks.rados.rados.0.smithi063.stdout:3460: oid 11 version 665 is already newer than 661 2024-08-17T17:09:31.740 INFO:tasks.rados.rados.0.smithi063.stdout:3460: oid 11 version 665 is already newer than 662 2024-08-17T17:09:31.740 INFO:tasks.rados.rados.0.smithi063.stdout:3460: oid 11 version 665 is already newer than 664 2024-08-17T17:09:31.740 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 665 (ObjNum 1288 snap 440 seq_num 1288) dirty exists 2024-08-17T17:09:31.740 INFO:tasks.rados.rados.0.smithi063.stdout:3460: left oid 11 (ObjNum 1288 snap 440 seq_num 1288) 2024-08-17T17:09:31.740 INFO:tasks.rados.rados.0.smithi063.stdout:3461: finishing write tid 1 to smithi06387500-7 2024-08-17T17:09:31.740 INFO:tasks.rados.rados.0.smithi063.stdout:3461: finishing write tid 2 to smithi06387500-7 2024-08-17T17:09:31.741 INFO:tasks.rados.rados.0.smithi063.stdout:3459: done (3 left) 2024-08-17T17:09:31.741 INFO:tasks.rados.rados.0.smithi063.stdout:3460: done (2 left) 2024-08-17T17:09:31.741 INFO:tasks.rados.rados.0.smithi063.stdout:3462: done (1 left) 2024-08-17T17:09:31.741 INFO:tasks.rados.rados.0.smithi063.stdout:3463: snap_create 2024-08-17T17:09:31.742 INFO:tasks.rados.rados.0.smithi063.stdout:3461: finishing write tid 3 to smithi06387500-7 2024-08-17T17:09:31.742 INFO:tasks.rados.rados.0.smithi063.stdout:3461: finishing write tid 4 to smithi06387500-7 2024-08-17T17:09:31.742 INFO:tasks.rados.rados.0.smithi063.stdout:3461: finishing write tid 5 to smithi06387500-7 2024-08-17T17:09:31.743 INFO:tasks.rados.rados.0.smithi063.stdout:3461: oid 7 updating version 0 to 1654 2024-08-17T17:09:31.743 INFO:tasks.rados.rados.0.smithi063.stdout:3461: oid 7 version 1654 is already newer than 1651 2024-08-17T17:09:31.743 INFO:tasks.rados.rados.0.smithi063.stdout:3461: oid 7 version 1654 is already newer than 1653 2024-08-17T17:09:31.743 INFO:tasks.rados.rados.0.smithi063.stdout:3461: oid 7 version 1654 is already newer than 1652 2024-08-17T17:09:31.743 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1654 (ObjNum 1289 snap 440 seq_num 1289) dirty exists 2024-08-17T17:09:31.743 INFO:tasks.rados.rados.0.smithi063.stdout:3461: left oid 7 (ObjNum 1289 snap 440 seq_num 1289) 2024-08-17T17:09:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:31 smithi063 ceph-mon[114756]: pgmap v870: 89 pgs: 89 active+clean; 241 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 387 KiB/s wr, 1 op/s 2024-08-17T17:09:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:31 smithi063 ceph-mon[114756]: osdmap e943: 8 total, 8 up, 8 in 2024-08-17T17:09:32.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:31 smithi063 ceph-mon[109843]: pgmap v870: 89 pgs: 89 active+clean; 241 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 387 KiB/s wr, 1 op/s 2024-08-17T17:09:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:31 smithi063 ceph-mon[109843]: osdmap e943: 8 total, 8 up, 8 in 2024-08-17T17:09:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:31 smithi162 ceph-mon[104668]: pgmap v870: 89 pgs: 89 active+clean; 241 MiB data, 848 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 387 KiB/s wr, 1 op/s 2024-08-17T17:09:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:31 smithi162 ceph-mon[104668]: osdmap e943: 8 total, 8 up, 8 in 2024-08-17T17:09:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:32.660 INFO:tasks.rados.rados.0.smithi063.stdout:3461: done (1 left) 2024-08-17T17:09:32.660 INFO:tasks.rados.rados.0.smithi063.stdout:3463: done (0 left) 2024-08-17T17:09:32.660 INFO:tasks.rados.rados.0.smithi063.stdout:3464: snap_remove snap 435 2024-08-17T17:09:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:32 smithi063 ceph-mon[114756]: osdmap e944: 8 total, 8 up, 8 in 2024-08-17T17:09:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:32 smithi063 ceph-mon[114756]: osdmap e945: 8 total, 8 up, 8 in 2024-08-17T17:09:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:32 smithi063 ceph-mon[109843]: osdmap e944: 8 total, 8 up, 8 in 2024-08-17T17:09:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:32 smithi063 ceph-mon[109843]: osdmap e945: 8 total, 8 up, 8 in 2024-08-17T17:09:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:32 smithi162 ceph-mon[104668]: osdmap e944: 8 total, 8 up, 8 in 2024-08-17T17:09:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:32 smithi162 ceph-mon[104668]: osdmap e945: 8 total, 8 up, 8 in 2024-08-17T17:09:33.662 INFO:tasks.rados.rados.0.smithi063.stdout:3464: done (0 left) 2024-08-17T17:09:33.662 INFO:tasks.rados.rados.0.smithi063.stdout:3465: snap_remove snap 439 2024-08-17T17:09:33.683 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:09:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:09:33] "GET /metrics HTTP/1.1" 200 36337 "" "Prometheus/2.51.0" 2024-08-17T17:09:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:33 smithi063 ceph-mon[114756]: pgmap v873: 89 pgs: 89 active+clean; 215 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.3 MiB/s wr, 5 op/s 2024-08-17T17:09:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:33 smithi063 ceph-mon[114756]: osdmap e946: 8 total, 8 up, 8 in 2024-08-17T17:09:34.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:33 smithi063 ceph-mon[109843]: pgmap v873: 89 pgs: 89 active+clean; 215 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.3 MiB/s wr, 5 op/s 2024-08-17T17:09:34.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:33 smithi063 ceph-mon[109843]: osdmap e946: 8 total, 8 up, 8 in 2024-08-17T17:09:34.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:33 smithi162 ceph-mon[104668]: pgmap v873: 89 pgs: 89 active+clean; 215 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.3 MiB/s wr, 5 op/s 2024-08-17T17:09:34.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:33 smithi162 ceph-mon[104668]: osdmap e946: 8 total, 8 up, 8 in 2024-08-17T17:09:34.663 INFO:tasks.rados.rados.0.smithi063.stdout:3465: done (0 left) 2024-08-17T17:09:34.663 INFO:tasks.rados.rados.0.smithi063.stdout:3466: delete oid 7 current snap is 441 2024-08-17T17:09:34.670 INFO:tasks.rados.rados.0.smithi063.stdout:3466: done (0 left) 2024-08-17T17:09:34.671 INFO:tasks.rados.rados.0.smithi063.stdout:3467: write oid 7 current snap is 441 2024-08-17T17:09:34.671 INFO:tasks.rados.rados.0.smithi063.stdout:3467: seq_num 1292 ranges {514644=485365,1654853=487112,2930389=574970} 2024-08-17T17:09:34.676 INFO:tasks.rados.rados.0.smithi063.stdout:3467: writing smithi06387500-7 from 514644 to 1000009 tid 1 2024-08-17T17:09:34.682 INFO:tasks.rados.rados.0.smithi063.stdout:3467: writing smithi06387500-7 from 1654853 to 2141965 tid 2 2024-08-17T17:09:34.690 INFO:tasks.rados.rados.0.smithi063.stdout:3467: writing smithi06387500-7 from 2930389 to 3505359 tid 3 2024-08-17T17:09:34.690 INFO:tasks.rados.rados.0.smithi063.stdout:3468: snap_remove snap 406 2024-08-17T17:09:35.691 INFO:tasks.rados.rados.0.smithi063.stdout:3468: done (1 left) 2024-08-17T17:09:35.691 INFO:tasks.rados.rados.0.smithi063.stdout:3469: write oid 5 current snap is 441 2024-08-17T17:09:35.691 INFO:tasks.rados.rados.0.smithi063.stdout:3467: finishing write tid 1 to smithi06387500-7 2024-08-17T17:09:35.691 INFO:tasks.rados.rados.0.smithi063.stdout:3469: seq_num 1293 ranges {531451=676967,1713335=747123,2957812=502595} 2024-08-17T17:09:35.698 INFO:tasks.rados.rados.0.smithi063.stdout:3469: writing smithi06387500-5 from 531451 to 1208418 tid 1 2024-08-17T17:09:35.707 INFO:tasks.rados.rados.0.smithi063.stdout:3469: writing smithi06387500-5 from 1713335 to 2460458 tid 2 2024-08-17T17:09:35.713 INFO:tasks.rados.rados.0.smithi063.stdout:3469: writing smithi06387500-5 from 2957812 to 3460407 tid 3 2024-08-17T17:09:35.713 INFO:tasks.rados.rados.0.smithi063.stdout:3470: snap_remove snap 410 2024-08-17T17:09:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:35 smithi063 ceph-mon[114756]: pgmap v876: 89 pgs: 89 active+clean; 215 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.3 MiB/s wr, 5 op/s 2024-08-17T17:09:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:35 smithi063 ceph-mon[114756]: osdmap e947: 8 total, 8 up, 8 in 2024-08-17T17:09:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:35 smithi063 ceph-mon[109843]: pgmap v876: 89 pgs: 89 active+clean; 215 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.3 MiB/s wr, 5 op/s 2024-08-17T17:09:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:35 smithi063 ceph-mon[109843]: osdmap e947: 8 total, 8 up, 8 in 2024-08-17T17:09:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:35 smithi162 ceph-mon[104668]: pgmap v876: 89 pgs: 89 active+clean; 215 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.3 MiB/s wr, 5 op/s 2024-08-17T17:09:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:35 smithi162 ceph-mon[104668]: osdmap e947: 8 total, 8 up, 8 in 2024-08-17T17:09:36.705 INFO:tasks.rados.rados.0.smithi063.stdout:3467: finishing write tid 2 to smithi06387500-7 2024-08-17T17:09:36.705 INFO:tasks.rados.rados.0.smithi063.stdout:3467: finishing write tid 3 to smithi06387500-7 2024-08-17T17:09:36.705 INFO:tasks.rados.rados.0.smithi063.stdout:3467: finishing write tid 4 to smithi06387500-7 2024-08-17T17:09:36.706 INFO:tasks.rados.rados.0.smithi063.stdout:3467: finishing write tid 5 to smithi06387500-7 2024-08-17T17:09:36.706 INFO:tasks.rados.rados.0.smithi063.stdout:3467: oid 7 updating version 0 to 1663 2024-08-17T17:09:36.706 INFO:tasks.rados.rados.0.smithi063.stdout:3467: oid 7 version 1663 is already newer than 1661 2024-08-17T17:09:36.706 INFO:tasks.rados.rados.0.smithi063.stdout:3467: oid 7 updating version 1663 to 1664 2024-08-17T17:09:36.706 INFO:tasks.rados.rados.0.smithi063.stdout:3467: oid 7 version 1664 is already newer than 1662 2024-08-17T17:09:36.706 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1664 (ObjNum 1292 snap 441 seq_num 1292) dirty exists 2024-08-17T17:09:36.706 INFO:tasks.rados.rados.0.smithi063.stdout:3467: left oid 7 (ObjNum 1292 snap 441 seq_num 1292) 2024-08-17T17:09:36.706 INFO:tasks.rados.rados.0.smithi063.stdout:3469: finishing write tid 1 to smithi06387500-5 2024-08-17T17:09:36.706 INFO:tasks.rados.rados.0.smithi063.stdout:3469: finishing write tid 2 to smithi06387500-5 2024-08-17T17:09:36.706 INFO:tasks.rados.rados.0.smithi063.stdout:3469: finishing write tid 3 to smithi06387500-5 2024-08-17T17:09:36.706 INFO:tasks.rados.rados.0.smithi063.stdout:3469: finishing write tid 4 to smithi06387500-5 2024-08-17T17:09:36.706 INFO:tasks.rados.rados.0.smithi063.stdout:3469: finishing write tid 5 to smithi06387500-5 2024-08-17T17:09:36.706 INFO:tasks.rados.rados.0.smithi063.stdout:3469: oid 5 updating version 0 to 1788 2024-08-17T17:09:36.707 INFO:tasks.rados.rados.0.smithi063.stdout:3469: oid 5 version 1788 is already newer than 1786 2024-08-17T17:09:36.707 INFO:tasks.rados.rados.0.smithi063.stdout:3469: oid 5 updating version 1788 to 1789 2024-08-17T17:09:36.707 INFO:tasks.rados.rados.0.smithi063.stdout:3469: oid 5 version 1789 is already newer than 1787 2024-08-17T17:09:36.707 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1789 (ObjNum 1293 snap 441 seq_num 1293) dirty exists 2024-08-17T17:09:36.707 INFO:tasks.rados.rados.0.smithi063.stdout:3469: left oid 5 (ObjNum 1293 snap 441 seq_num 1293) 2024-08-17T17:09:36.707 INFO:tasks.rados.rados.0.smithi063.stdout:3467: done (2 left) 2024-08-17T17:09:36.707 INFO:tasks.rados.rados.0.smithi063.stdout:3469: done (1 left) 2024-08-17T17:09:36.707 INFO:tasks.rados.rados.0.smithi063.stdout:3470: done (0 left) 2024-08-17T17:09:36.707 INFO:tasks.rados.rados.0.smithi063.stdout:3471: snap_create 2024-08-17T17:09:37.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:36 smithi063 ceph-mon[114756]: osdmap e948: 8 total, 8 up, 8 in 2024-08-17T17:09:37.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:36 smithi063 ceph-mon[114756]: pgmap v879: 89 pgs: 89 active+clean; 215 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 783 KiB/s wr, 3 op/s 2024-08-17T17:09:37.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:36 smithi063 ceph-mon[109843]: osdmap e948: 8 total, 8 up, 8 in 2024-08-17T17:09:37.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:36 smithi063 ceph-mon[109843]: pgmap v879: 89 pgs: 89 active+clean; 215 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 783 KiB/s wr, 3 op/s 2024-08-17T17:09:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:36 smithi162 ceph-mon[104668]: osdmap e948: 8 total, 8 up, 8 in 2024-08-17T17:09:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:36 smithi162 ceph-mon[104668]: pgmap v879: 89 pgs: 89 active+clean; 215 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 783 KiB/s wr, 3 op/s 2024-08-17T17:09:37.662 INFO:tasks.rados.rados.0.smithi063.stdout:3471: done (0 left) 2024-08-17T17:09:37.663 INFO:tasks.rados.rados.0.smithi063.stdout:3472: read oid 25 snap -1 2024-08-17T17:09:37.663 INFO:tasks.rados.rados.0.smithi063.stdout:3472: expect (ObjNum 1283 snap 436 seq_num 1283) 2024-08-17T17:09:37.663 INFO:tasks.rados.rados.0.smithi063.stdout:3473: read oid 23 snap -1 2024-08-17T17:09:37.663 INFO:tasks.rados.rados.0.smithi063.stdout:3473: expect (ObjNum 1213 snap 406 seq_num 1213) 2024-08-17T17:09:37.663 INFO:tasks.rados.rados.0.smithi063.stdout:3474: delete oid 45 current snap is 442 2024-08-17T17:09:37.667 INFO:tasks.rados.rados.0.smithi063.stdout:3473: expect (ObjNum 1213 snap 406 seq_num 1213) 2024-08-17T17:09:37.772 INFO:tasks.rados.rados.0.smithi063.stdout:3472: expect (ObjNum 1283 snap 436 seq_num 1283) 2024-08-17T17:09:37.935 INFO:tasks.rados.rados.0.smithi063.stdout:3472: done (2 left) 2024-08-17T17:09:37.985 INFO:tasks.rados.rados.0.smithi063.stdout:3473: done (1 left) 2024-08-17T17:09:37.985 INFO:tasks.rados.rados.0.smithi063.stdout:3474: done (0 left) 2024-08-17T17:09:37.985 INFO:tasks.rados.rados.0.smithi063.stdout:3475: write oid 34 current snap is 442 2024-08-17T17:09:37.985 INFO:tasks.rados.rados.0.smithi063.stdout:3475: seq_num 1295 ranges {565065=427465,1430299=634439,2496305=525139,3472104=292961} 2024-08-17T17:09:37.987 INFO:tasks.rados.rados.0.smithi063.stdout:3475: writing smithi06387500-34 from 565065 to 992530 tid 1 2024-08-17T17:09:37.987 INFO:tasks.rados.rados.0.smithi063.stdout:3475: writing smithi06387500-34 from 1430299 to 2064738 tid 2 2024-08-17T17:09:37.987 INFO:tasks.rados.rados.0.smithi063.stdout:3475: writing smithi06387500-34 from 2496305 to 3021444 tid 3 2024-08-17T17:09:37.987 INFO:tasks.rados.rados.0.smithi063.stdout:3475: writing smithi06387500-34 from 3472104 to 3765065 tid 4 2024-08-17T17:09:37.987 INFO:tasks.rados.rados.0.smithi063.stdout:3476: read oid 6 snap -1 2024-08-17T17:09:37.987 INFO:tasks.rados.rados.0.smithi063.stdout:3476: expect deleted 2024-08-17T17:09:37.987 INFO:tasks.rados.rados.0.smithi063.stdout:3477: write oid 36 current snap is 442 2024-08-17T17:09:37.988 INFO:tasks.rados.rados.0.smithi063.stdout:3475: finishing write tid 1 to smithi06387500-34 2024-08-17T17:09:37.988 INFO:tasks.rados.rados.0.smithi063.stdout:3475: finishing write tid 2 to smithi06387500-34 2024-08-17T17:09:37.988 INFO:tasks.rados.rados.0.smithi063.stdout:3475: finishing write tid 3 to smithi06387500-34 2024-08-17T17:09:37.988 INFO:tasks.rados.rados.0.smithi063.stdout:3477: seq_num 1296 ranges {581872=502714,1488781=494450,2640081=736411} 2024-08-17T17:09:37.988 INFO:tasks.rados.rados.0.smithi063.stdout:3477: writing smithi06387500-36 from 581872 to 1084586 tid 1 2024-08-17T17:09:37.988 INFO:tasks.rados.rados.0.smithi063.stdout:3477: writing smithi06387500-36 from 1488781 to 1983231 tid 2 2024-08-17T17:09:37.988 INFO:tasks.rados.rados.0.smithi063.stdout:3477: writing smithi06387500-36 from 2640081 to 3376492 tid 3 2024-08-17T17:09:37.988 INFO:tasks.rados.rados.0.smithi063.stdout:3478: write oid 1 current snap is 442 2024-08-17T17:09:37.988 INFO:tasks.rados.rados.0.smithi063.stdout:3478: seq_num 1297 ranges {598679=577963,1947263=638108,3351151=447528} 2024-08-17T17:09:37.988 INFO:tasks.rados.rados.0.smithi063.stdout:3478: writing smithi06387500-1 from 598679 to 1176642 tid 1 2024-08-17T17:09:37.989 INFO:tasks.rados.rados.0.smithi063.stdout:3478: writing smithi06387500-1 from 1947263 to 2585371 tid 2 2024-08-17T17:09:37.995 INFO:tasks.rados.rados.0.smithi063.stdout:3478: writing smithi06387500-1 from 3351151 to 3798679 tid 3 2024-08-17T17:09:37.995 INFO:tasks.rados.rados.0.smithi063.stdout:3479: write oid 9 current snap is 442 2024-08-17T17:09:37.995 INFO:tasks.rados.rados.0.smithi063.stdout:3479: seq_num 1298 ranges {615486=653212,1889392=498119,2978574=475308} 2024-08-17T17:09:38.002 INFO:tasks.rados.rados.0.smithi063.stdout:3479: writing smithi06387500-9 from 615486 to 1268698 tid 1 2024-08-17T17:09:38.008 INFO:tasks.rados.rados.0.smithi063.stdout:3479: writing smithi06387500-9 from 1889392 to 2387511 tid 2 2024-08-17T17:09:38.014 INFO:tasks.rados.rados.0.smithi063.stdout:3479: writing smithi06387500-9 from 2978574 to 3453882 tid 3 2024-08-17T17:09:38.014 INFO:tasks.rados.rados.0.smithi063.stdout:3480: snap_remove snap 428 2024-08-17T17:09:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:37 smithi063 ceph-mon[109843]: osdmap e949: 8 total, 8 up, 8 in 2024-08-17T17:09:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:37 smithi063 ceph-mon[109843]: osdmap e950: 8 total, 8 up, 8 in 2024-08-17T17:09:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:37 smithi063 ceph-mon[114756]: osdmap e949: 8 total, 8 up, 8 in 2024-08-17T17:09:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:37 smithi063 ceph-mon[114756]: osdmap e950: 8 total, 8 up, 8 in 2024-08-17T17:09:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:37 smithi162 ceph-mon[104668]: osdmap e949: 8 total, 8 up, 8 in 2024-08-17T17:09:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:37 smithi162 ceph-mon[104668]: osdmap e950: 8 total, 8 up, 8 in 2024-08-17T17:09:38.711 INFO:tasks.rados.rados.0.smithi063.stdout:3480: done (5 left) 2024-08-17T17:09:38.712 INFO:tasks.rados.rados.0.smithi063.stdout:3481: read oid 17 snap -1 2024-08-17T17:09:38.712 INFO:tasks.rados.rados.0.smithi063.stdout:3481: expect (ObjNum 1228 snap 411 seq_num 1228) 2024-08-17T17:09:38.712 INFO:tasks.rados.rados.0.smithi063.stdout:3482: read oid 46 snap -1 2024-08-17T17:09:38.712 INFO:tasks.rados.rados.0.smithi063.stdout:3482: expect deleted 2024-08-17T17:09:38.712 INFO:tasks.rados.rados.0.smithi063.stdout:3483: read oid 47 snap -1 2024-08-17T17:09:38.712 INFO:tasks.rados.rados.0.smithi063.stdout:3483: expect deleted 2024-08-17T17:09:38.712 INFO:tasks.rados.rados.0.smithi063.stdout:3484: rollback oid 49 current snap is 442 2024-08-17T17:09:38.712 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 49 to 432 2024-08-17T17:09:38.712 INFO:tasks.rados.rados.0.smithi063.stdout:3475: finishing write tid 4 to smithi06387500-34 2024-08-17T17:09:38.712 INFO:tasks.rados.rados.0.smithi063.stdout:3475: finishing write tid 5 to smithi06387500-34 2024-08-17T17:09:38.712 INFO:tasks.rados.rados.0.smithi063.stdout:3475: finishing write tid 6 to smithi06387500-34 2024-08-17T17:09:38.712 INFO:tasks.rados.rados.0.smithi063.stdout:3475: oid 34 updating version 0 to 2241 2024-08-17T17:09:38.713 INFO:tasks.rados.rados.0.smithi063.stdout:3475: oid 34 version 2241 is already newer than 2238 2024-08-17T17:09:38.713 INFO:tasks.rados.rados.0.smithi063.stdout:3475: oid 34 version 2241 is already newer than 2240 2024-08-17T17:09:38.713 INFO:tasks.rados.rados.0.smithi063.stdout:3475: oid 34 updating version 2241 to 2242 2024-08-17T17:09:38.713 INFO:tasks.rados.rados.0.smithi063.stdout:3475: oid 34 version 2242 is already newer than 2239 2024-08-17T17:09:38.713 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 2242 (ObjNum 1295 snap 442 seq_num 1295) dirty exists 2024-08-17T17:09:38.713 INFO:tasks.rados.rados.0.smithi063.stdout:3475: left oid 34 (ObjNum 1295 snap 442 seq_num 1295) 2024-08-17T17:09:38.713 INFO:tasks.rados.rados.0.smithi063.stdout:3477: finishing write tid 1 to smithi06387500-36 2024-08-17T17:09:38.713 INFO:tasks.rados.rados.0.smithi063.stdout:3477: finishing write tid 2 to smithi06387500-36 2024-08-17T17:09:38.713 INFO:tasks.rados.rados.0.smithi063.stdout:3477: finishing write tid 3 to smithi06387500-36 2024-08-17T17:09:38.713 INFO:tasks.rados.rados.0.smithi063.stdout:3477: finishing write tid 4 to smithi06387500-36 2024-08-17T17:09:38.713 INFO:tasks.rados.rados.0.smithi063.stdout:3477: finishing write tid 5 to smithi06387500-36 2024-08-17T17:09:38.713 INFO:tasks.rados.rados.0.smithi063.stdout:3477: oid 36 updating version 0 to 3426 2024-08-17T17:09:38.714 INFO:tasks.rados.rados.0.smithi063.stdout:3477: oid 36 updating version 3426 to 3427 2024-08-17T17:09:38.714 INFO:tasks.rados.rados.0.smithi063.stdout:3477: oid 36 updating version 3427 to 3428 2024-08-17T17:09:38.714 INFO:tasks.rados.rados.0.smithi063.stdout:3477: oid 36 updating version 3428 to 3429 2024-08-17T17:09:38.714 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 3429 (ObjNum 1296 snap 442 seq_num 1296) dirty exists 2024-08-17T17:09:38.714 INFO:tasks.rados.rados.0.smithi063.stdout:3477: left oid 36 (ObjNum 1296 snap 442 seq_num 1296) 2024-08-17T17:09:38.714 INFO:tasks.rados.rados.0.smithi063.stdout:3478: finishing write tid 1 to smithi06387500-1 2024-08-17T17:09:38.714 INFO:tasks.rados.rados.0.smithi063.stdout:3478: finishing write tid 2 to smithi06387500-1 2024-08-17T17:09:38.714 INFO:tasks.rados.rados.0.smithi063.stdout:3478: finishing write tid 3 to smithi06387500-1 2024-08-17T17:09:38.714 INFO:tasks.rados.rados.0.smithi063.stdout:3478: finishing write tid 4 to smithi06387500-1 2024-08-17T17:09:38.714 INFO:tasks.rados.rados.0.smithi063.stdout:3478: finishing write tid 5 to smithi06387500-1 2024-08-17T17:09:38.714 INFO:tasks.rados.rados.0.smithi063.stdout:3478: oid 1 updating version 0 to 1800 2024-08-17T17:09:38.714 INFO:tasks.rados.rados.0.smithi063.stdout:3478: oid 1 version 1800 is already newer than 1797 2024-08-17T17:09:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3478: oid 1 version 1800 is already newer than 1799 2024-08-17T17:09:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3478: oid 1 version 1800 is already newer than 1798 2024-08-17T17:09:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 1800 (ObjNum 1297 snap 442 seq_num 1297) dirty exists 2024-08-17T17:09:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3478: left oid 1 (ObjNum 1297 snap 442 seq_num 1297) 2024-08-17T17:09:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3479: finishing write tid 1 to smithi06387500-9 2024-08-17T17:09:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3479: finishing write tid 2 to smithi06387500-9 2024-08-17T17:09:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3479: finishing write tid 3 to smithi06387500-9 2024-08-17T17:09:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3479: finishing write tid 4 to smithi06387500-9 2024-08-17T17:09:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3479: finishing write tid 5 to smithi06387500-9 2024-08-17T17:09:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3479: oid 9 updating version 0 to 1673 2024-08-17T17:09:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3479: oid 9 updating version 1673 to 1675 2024-08-17T17:09:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3479: oid 9 updating version 1675 to 1676 2024-08-17T17:09:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3479: oid 9 version 1676 is already newer than 1674 2024-08-17T17:09:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 1676 (ObjNum 1298 snap 442 seq_num 1298) dirty exists 2024-08-17T17:09:38.716 INFO:tasks.rados.rados.0.smithi063.stdout:3479: left oid 9 (ObjNum 1298 snap 442 seq_num 1298) 2024-08-17T17:09:38.716 INFO:tasks.rados.rados.0.smithi063.stdout:3475: done (8 left) 2024-08-17T17:09:38.716 INFO:tasks.rados.rados.0.smithi063.stdout:3476: done (7 left) 2024-08-17T17:09:38.716 INFO:tasks.rados.rados.0.smithi063.stdout:3477: done (6 left) 2024-08-17T17:09:38.716 INFO:tasks.rados.rados.0.smithi063.stdout:3478: done (5 left) 2024-08-17T17:09:38.716 INFO:tasks.rados.rados.0.smithi063.stdout:3479: done (4 left) 2024-08-17T17:09:38.716 INFO:tasks.rados.rados.0.smithi063.stdout:3485: read oid 16 snap -1 2024-08-17T17:09:38.716 INFO:tasks.rados.rados.0.smithi063.stdout:3485: expect (ObjNum 1280 snap 435 seq_num 1280) 2024-08-17T17:09:38.716 INFO:tasks.rados.rados.0.smithi063.stdout:3486: write oid 36 current snap is 442 2024-08-17T17:09:38.716 INFO:tasks.rados.rados.0.smithi063.stdout:3486: seq_num 1299 ranges {632293=728461,1947874=641777,3005997=686580} 2024-08-17T17:09:38.719 INFO:tasks.rados.rados.0.smithi063.stdout:3486: writing smithi06387500-36 from 632293 to 1360754 tid 1 2024-08-17T17:09:38.727 INFO:tasks.rados.rados.0.smithi063.stdout:3486: writing smithi06387500-36 from 1947874 to 2589651 tid 2 2024-08-17T17:09:38.733 INFO:tasks.rados.rados.0.smithi063.stdout:3486: writing smithi06387500-36 from 3005997 to 3692577 tid 3 2024-08-17T17:09:38.734 INFO:tasks.rados.rados.0.smithi063.stdout:3487: delete oid 43 current snap is 442 2024-08-17T17:09:38.734 INFO:tasks.rados.rados.0.smithi063.stdout:3484: finishing rollback tid 0 to smithi06387500-49 2024-08-17T17:09:38.734 INFO:tasks.rados.rados.0.smithi063.stdout:3484: finishing rollback tid 1 to smithi06387500-49 2024-08-17T17:09:38.734 INFO:tasks.rados.rados.0.smithi063.stdout:3484: finishing rollback tid 2 to smithi06387500-49 2024-08-17T17:09:38.734 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 49 v 3437 (ObjNum 1101 snap 366 seq_num 1101) dirty exists 2024-08-17T17:09:38.734 INFO:tasks.rados.rados.0.smithi063.stdout:3481: expect (ObjNum 1228 snap 411 seq_num 1228) 2024-08-17T17:09:38.844 INFO:tasks.rados.rados.0.smithi063.stdout:3485: expect (ObjNum 1280 snap 435 seq_num 1280) 2024-08-17T17:09:39.009 INFO:tasks.rados.rados.0.smithi063.stdout:3486: finishing write tid 1 to smithi06387500-36 2024-08-17T17:09:39.009 INFO:tasks.rados.rados.0.smithi063.stdout:3486: finishing write tid 2 to smithi06387500-36 2024-08-17T17:09:39.009 INFO:tasks.rados.rados.0.smithi063.stdout:3486: finishing write tid 3 to smithi06387500-36 2024-08-17T17:09:39.010 INFO:tasks.rados.rados.0.smithi063.stdout:3486: finishing write tid 4 to smithi06387500-36 2024-08-17T17:09:39.010 INFO:tasks.rados.rados.0.smithi063.stdout:3486: finishing write tid 5 to smithi06387500-36 2024-08-17T17:09:39.010 INFO:tasks.rados.rados.0.smithi063.stdout:3486: oid 36 updating version 0 to 3442 2024-08-17T17:09:39.010 INFO:tasks.rados.rados.0.smithi063.stdout:3486: oid 36 version 3442 is already newer than 3441 2024-08-17T17:09:39.010 INFO:tasks.rados.rados.0.smithi063.stdout:3486: oid 36 version 3442 is already newer than 3438 2024-08-17T17:09:39.010 INFO:tasks.rados.rados.0.smithi063.stdout:3486: oid 36 updating version 3442 to 3443 2024-08-17T17:09:39.010 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 3443 (ObjNum 1299 snap 442 seq_num 1299) dirty exists 2024-08-17T17:09:39.010 INFO:tasks.rados.rados.0.smithi063.stdout:3486: left oid 36 (ObjNum 1299 snap 442 seq_num 1299) 2024-08-17T17:09:39.010 INFO:tasks.rados.rados.0.smithi063.stdout:3481: done (6 left) 2024-08-17T17:09:39.010 INFO:tasks.rados.rados.0.smithi063.stdout:3482: done (5 left) 2024-08-17T17:09:39.010 INFO:tasks.rados.rados.0.smithi063.stdout:3483: done (4 left) 2024-08-17T17:09:39.010 INFO:tasks.rados.rados.0.smithi063.stdout:3484: done (3 left) 2024-08-17T17:09:39.011 INFO:tasks.rados.rados.0.smithi063.stdout:3485: done (2 left) 2024-08-17T17:09:39.011 INFO:tasks.rados.rados.0.smithi063.stdout:3486: done (1 left) 2024-08-17T17:09:39.011 INFO:tasks.rados.rados.0.smithi063.stdout:3487: done (0 left) 2024-08-17T17:09:39.011 INFO:tasks.rados.rados.0.smithi063.stdout:3488: read oid 32 snap -1 2024-08-17T17:09:39.011 INFO:tasks.rados.rados.0.smithi063.stdout:3488: expect (ObjNum 1208 snap 404 seq_num 1208) 2024-08-17T17:09:39.011 INFO:tasks.rados.rados.0.smithi063.stdout:3489: write oid 6 current snap is 442 2024-08-17T17:09:39.011 INFO:tasks.rados.rados.0.smithi063.stdout:3489: seq_num 1301 ranges {665907=595312,1781191=645446,3177196=688711} 2024-08-17T17:09:39.016 INFO:tasks.rados.rados.0.smithi063.stdout:3489: writing smithi06387500-6 from 665907 to 1261219 tid 1 2024-08-17T17:09:39.022 INFO:tasks.rados.rados.0.smithi063.stdout:3489: writing smithi06387500-6 from 1781191 to 2426637 tid 2 2024-08-17T17:09:39.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:38 smithi063 ceph-mon[109843]: pgmap v882: 89 pgs: 89 active+clean; 194 MiB data, 803 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.6 MiB/s wr, 6 op/s 2024-08-17T17:09:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:38 smithi063 ceph-mon[114756]: pgmap v882: 89 pgs: 89 active+clean; 194 MiB data, 803 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.6 MiB/s wr, 6 op/s 2024-08-17T17:09:39.030 INFO:tasks.rados.rados.0.smithi063.stdout:3489: writing smithi06387500-6 from 3177196 to 3865907 tid 3 2024-08-17T17:09:39.030 INFO:tasks.rados.rados.0.smithi063.stdout:3490: write oid 44 current snap is 442 2024-08-17T17:09:39.030 INFO:tasks.rados.rados.0.smithi063.stdout:3490: seq_num 1302 ranges {682714=670561,2123320=505457,3088266=636749} 2024-08-17T17:09:39.038 INFO:tasks.rados.rados.0.smithi063.stdout:3490: writing smithi06387500-44 from 682714 to 1353275 tid 1 2024-08-17T17:09:39.045 INFO:tasks.rados.rados.0.smithi063.stdout:3490: writing smithi06387500-44 from 2123320 to 2628777 tid 2 2024-08-17T17:09:39.051 INFO:tasks.rados.rados.0.smithi063.stdout:3490: writing smithi06387500-44 from 3088266 to 3725015 tid 3 2024-08-17T17:09:39.051 INFO:tasks.rados.rados.0.smithi063.stdout:3491: write oid 29 current snap is 442 2024-08-17T17:09:39.051 INFO:tasks.rados.rados.0.smithi063.stdout:3488: expect (ObjNum 1208 snap 404 seq_num 1208) 2024-08-17T17:09:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:38 smithi162 ceph-mon[104668]: pgmap v882: 89 pgs: 89 active+clean; 194 MiB data, 803 MiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.6 MiB/s wr, 6 op/s 2024-08-17T17:09:39.171 INFO:tasks.rados.rados.0.smithi063.stdout:3489: finishing write tid 1 to smithi06387500-6 2024-08-17T17:09:39.172 INFO:tasks.rados.rados.0.smithi063.stdout:3489: finishing write tid 2 to smithi06387500-6 2024-08-17T17:09:39.172 INFO:tasks.rados.rados.0.smithi063.stdout:3489: finishing write tid 3 to smithi06387500-6 2024-08-17T17:09:39.172 INFO:tasks.rados.rados.0.smithi063.stdout:3489: finishing write tid 4 to smithi06387500-6 2024-08-17T17:09:39.172 INFO:tasks.rados.rados.0.smithi063.stdout:3489: finishing write tid 5 to smithi06387500-6 2024-08-17T17:09:39.172 INFO:tasks.rados.rados.0.smithi063.stdout:3489: oid 6 updating version 0 to 1172 2024-08-17T17:09:39.172 INFO:tasks.rados.rados.0.smithi063.stdout:3489: oid 6 updating version 1172 to 1175 2024-08-17T17:09:39.172 INFO:tasks.rados.rados.0.smithi063.stdout:3489: oid 6 version 1175 is already newer than 1174 2024-08-17T17:09:39.172 INFO:tasks.rados.rados.0.smithi063.stdout:3489: oid 6 version 1175 is already newer than 1173 2024-08-17T17:09:39.172 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 1175 (ObjNum 1301 snap 442 seq_num 1301) dirty exists 2024-08-17T17:09:39.172 INFO:tasks.rados.rados.0.smithi063.stdout:3489: left oid 6 (ObjNum 1301 snap 442 seq_num 1301) 2024-08-17T17:09:39.172 INFO:tasks.rados.rados.0.smithi063.stdout:3490: finishing write tid 1 to smithi06387500-44 2024-08-17T17:09:39.172 INFO:tasks.rados.rados.0.smithi063.stdout:3490: finishing write tid 2 to smithi06387500-44 2024-08-17T17:09:39.172 INFO:tasks.rados.rados.0.smithi063.stdout:3490: finishing write tid 3 to smithi06387500-44 2024-08-17T17:09:39.173 INFO:tasks.rados.rados.0.smithi063.stdout:3490: finishing write tid 4 to smithi06387500-44 2024-08-17T17:09:39.173 INFO:tasks.rados.rados.0.smithi063.stdout:3490: finishing write tid 5 to smithi06387500-44 2024-08-17T17:09:39.173 INFO:tasks.rados.rados.0.smithi063.stdout:3490: oid 44 updating version 0 to 476 2024-08-17T17:09:39.173 INFO:tasks.rados.rados.0.smithi063.stdout:3490: oid 44 version 476 is already newer than 474 2024-08-17T17:09:39.173 INFO:tasks.rados.rados.0.smithi063.stdout:3490: oid 44 version 476 is already newer than 475 2024-08-17T17:09:39.173 INFO:tasks.rados.rados.0.smithi063.stdout:3490: oid 44 version 476 is already newer than 473 2024-08-17T17:09:39.173 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 476 (ObjNum 1302 snap 442 seq_num 1302) dirty exists 2024-08-17T17:09:39.173 INFO:tasks.rados.rados.0.smithi063.stdout:3490: left oid 44 (ObjNum 1302 snap 442 seq_num 1302) 2024-08-17T17:09:39.173 INFO:tasks.rados.rados.0.smithi063.stdout:3491: seq_num 1303 ranges {699521=745810,2181802=649115,3515689=383832} 2024-08-17T17:09:39.179 INFO:tasks.rados.rados.0.smithi063.stdout:3491: writing smithi06387500-29 from 699521 to 1445331 tid 1 2024-08-17T17:09:39.187 INFO:tasks.rados.rados.0.smithi063.stdout:3491: writing smithi06387500-29 from 2181802 to 2830917 tid 2 2024-08-17T17:09:39.193 INFO:tasks.rados.rados.0.smithi063.stdout:3491: writing smithi06387500-29 from 3515689 to 3899521 tid 3 2024-08-17T17:09:39.193 INFO:tasks.rados.rados.0.smithi063.stdout:3488: done (3 left) 2024-08-17T17:09:39.193 INFO:tasks.rados.rados.0.smithi063.stdout:3489: done (2 left) 2024-08-17T17:09:39.193 INFO:tasks.rados.rados.0.smithi063.stdout:3490: done (1 left) 2024-08-17T17:09:39.193 INFO:tasks.rados.rados.0.smithi063.stdout:3492: read oid 35 snap -1 2024-08-17T17:09:39.193 INFO:tasks.rados.rados.0.smithi063.stdout:3492: expect deleted 2024-08-17T17:09:39.193 INFO:tasks.rados.rados.0.smithi063.stdout:3493: snap_create 2024-08-17T17:09:39.193 INFO:tasks.rados.rados.0.smithi063.stdout:3491: finishing write tid 1 to smithi06387500-29 2024-08-17T17:09:39.194 INFO:tasks.rados.rados.0.smithi063.stdout:3491: finishing write tid 2 to smithi06387500-29 2024-08-17T17:09:39.195 INFO:tasks.rados.rados.0.smithi063.stdout:3491: finishing write tid 3 to smithi06387500-29 2024-08-17T17:09:39.195 INFO:tasks.rados.rados.0.smithi063.stdout:3491: finishing write tid 4 to smithi06387500-29 2024-08-17T17:09:39.196 INFO:tasks.rados.rados.0.smithi063.stdout:3491: finishing write tid 5 to smithi06387500-29 2024-08-17T17:09:39.196 INFO:tasks.rados.rados.0.smithi063.stdout:3491: oid 29 updating version 0 to 795 2024-08-17T17:09:39.196 INFO:tasks.rados.rados.0.smithi063.stdout:3491: oid 29 updating version 795 to 798 2024-08-17T17:09:39.196 INFO:tasks.rados.rados.0.smithi063.stdout:3491: oid 29 version 798 is already newer than 796 2024-08-17T17:09:39.196 INFO:tasks.rados.rados.0.smithi063.stdout:3491: oid 29 version 798 is already newer than 797 2024-08-17T17:09:39.197 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 798 (ObjNum 1303 snap 442 seq_num 1303) dirty exists 2024-08-17T17:09:39.197 INFO:tasks.rados.rados.0.smithi063.stdout:3491: left oid 29 (ObjNum 1303 snap 442 seq_num 1303) 2024-08-17T17:09:39.714 INFO:tasks.rados.rados.0.smithi063.stdout:3491: done (2 left) 2024-08-17T17:09:39.714 INFO:tasks.rados.rados.0.smithi063.stdout:3492: done (1 left) 2024-08-17T17:09:39.714 INFO:tasks.rados.rados.0.smithi063.stdout:3493: done (0 left) 2024-08-17T17:09:39.715 INFO:tasks.rados.rados.0.smithi063.stdout:3494: write oid 42 current snap is 443 2024-08-17T17:09:39.715 INFO:tasks.rados.rados.0.smithi063.stdout:3494: seq_num 1304 ranges {716328=421059,1840284=509126,3143112=659293} 2024-08-17T17:09:39.720 INFO:tasks.rados.rados.0.smithi063.stdout:3494: writing smithi06387500-42 from 716328 to 1137387 tid 1 2024-08-17T17:09:39.727 INFO:tasks.rados.rados.0.smithi063.stdout:3494: writing smithi06387500-42 from 1840284 to 2349410 tid 2 2024-08-17T17:09:39.735 INFO:tasks.rados.rados.0.smithi063.stdout:3494: writing smithi06387500-42 from 3143112 to 3802405 tid 3 2024-08-17T17:09:39.736 INFO:tasks.rados.rados.0.smithi063.stdout:3495: read oid 26 snap -1 2024-08-17T17:09:39.736 INFO:tasks.rados.rados.0.smithi063.stdout:3495: expect (ObjNum 1193 snap 400 seq_num 1193) 2024-08-17T17:09:39.736 INFO:tasks.rados.rados.0.smithi063.stdout:3496: write oid 2 current snap is 443 2024-08-17T17:09:39.736 INFO:tasks.rados.rados.0.smithi063.stdout:3494: finishing write tid 1 to smithi06387500-42 2024-08-17T17:09:39.736 INFO:tasks.rados.rados.0.smithi063.stdout:3494: finishing write tid 2 to smithi06387500-42 2024-08-17T17:09:39.736 INFO:tasks.rados.rados.0.smithi063.stdout:3496: seq_num 1305 ranges {733135=496308,1898766=652784,3170535=470565} 2024-08-17T17:09:39.742 INFO:tasks.rados.rados.0.smithi063.stdout:3496: writing smithi06387500-2 from 733135 to 1229443 tid 1 2024-08-17T17:09:39.749 INFO:tasks.rados.rados.0.smithi063.stdout:3496: writing smithi06387500-2 from 1898766 to 2551550 tid 2 2024-08-17T17:09:39.755 INFO:tasks.rados.rados.0.smithi063.stdout:3496: writing smithi06387500-2 from 3170535 to 3641100 tid 3 2024-08-17T17:09:39.755 INFO:tasks.rados.rados.0.smithi063.stdout:3497: write oid 36 current snap is 443 2024-08-17T17:09:39.755 INFO:tasks.rados.rados.0.smithi063.stdout:3497: seq_num 1306 ranges {749942=571557,1840895=796442,3365252=584690} 2024-08-17T17:09:39.762 INFO:tasks.rados.rados.0.smithi063.stdout:3497: writing smithi06387500-36 from 749942 to 1321499 tid 1 2024-08-17T17:09:39.769 INFO:tasks.rados.rados.0.smithi063.stdout:3497: writing smithi06387500-36 from 1840895 to 2637337 tid 2 2024-08-17T17:09:39.777 INFO:tasks.rados.rados.0.smithi063.stdout:3497: writing smithi06387500-36 from 3365252 to 3949942 tid 3 2024-08-17T17:09:39.777 INFO:tasks.rados.rados.0.smithi063.stdout:3498: read oid 14 snap -1 2024-08-17T17:09:39.777 INFO:tasks.rados.rados.0.smithi063.stdout:3498: expect (ObjNum 1196 snap 401 seq_num 1196) 2024-08-17T17:09:39.777 INFO:tasks.rados.rados.0.smithi063.stdout:3499: rollback oid 19 current snap is 443 2024-08-17T17:09:39.778 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 19 to 441 2024-08-17T17:09:39.778 INFO:tasks.rados.rados.0.smithi063.stdout:3494: finishing write tid 3 to smithi06387500-42 2024-08-17T17:09:39.778 INFO:tasks.rados.rados.0.smithi063.stdout:3494: finishing write tid 4 to smithi06387500-42 2024-08-17T17:09:39.778 INFO:tasks.rados.rados.0.smithi063.stdout:3494: finishing write tid 5 to smithi06387500-42 2024-08-17T17:09:39.778 INFO:tasks.rados.rados.0.smithi063.stdout:3494: oid 42 updating version 0 to 2247 2024-08-17T17:09:39.778 INFO:tasks.rados.rados.0.smithi063.stdout:3494: oid 42 version 2247 is already newer than 2246 2024-08-17T17:09:39.778 INFO:tasks.rados.rados.0.smithi063.stdout:3494: oid 42 updating version 2247 to 2248 2024-08-17T17:09:39.778 INFO:tasks.rados.rados.0.smithi063.stdout:3494: oid 42 version 2248 is already newer than 2245 2024-08-17T17:09:39.779 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 42 v 2248 (ObjNum 1304 snap 443 seq_num 1304) dirty exists 2024-08-17T17:09:39.779 INFO:tasks.rados.rados.0.smithi063.stdout:3494: left oid 42 (ObjNum 1304 snap 443 seq_num 1304) 2024-08-17T17:09:39.779 INFO:tasks.rados.rados.0.smithi063.stdout:3495: expect (ObjNum 1193 snap 400 seq_num 1193) 2024-08-17T17:09:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:39 smithi063 ceph-mon[114756]: osdmap e951: 8 total, 8 up, 8 in 2024-08-17T17:09:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:39 smithi063 ceph-mon[109843]: osdmap e951: 8 total, 8 up, 8 in 2024-08-17T17:09:40.031 INFO:tasks.rados.rados.0.smithi063.stdout:3496: finishing write tid 1 to smithi06387500-2 2024-08-17T17:09:40.032 INFO:tasks.rados.rados.0.smithi063.stdout:3496: finishing write tid 2 to smithi06387500-2 2024-08-17T17:09:40.032 INFO:tasks.rados.rados.0.smithi063.stdout:3496: finishing write tid 3 to smithi06387500-2 2024-08-17T17:09:40.032 INFO:tasks.rados.rados.0.smithi063.stdout:3496: finishing write tid 4 to smithi06387500-2 2024-08-17T17:09:40.032 INFO:tasks.rados.rados.0.smithi063.stdout:3496: finishing write tid 5 to smithi06387500-2 2024-08-17T17:09:40.032 INFO:tasks.rados.rados.0.smithi063.stdout:3496: oid 2 updating version 0 to 1805 2024-08-17T17:09:40.032 INFO:tasks.rados.rados.0.smithi063.stdout:3496: oid 2 updating version 1805 to 1807 2024-08-17T17:09:40.032 INFO:tasks.rados.rados.0.smithi063.stdout:3496: oid 2 updating version 1807 to 1808 2024-08-17T17:09:40.032 INFO:tasks.rados.rados.0.smithi063.stdout:3496: oid 2 version 1808 is already newer than 1806 2024-08-17T17:09:40.032 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 1808 (ObjNum 1305 snap 443 seq_num 1305) dirty exists 2024-08-17T17:09:40.032 INFO:tasks.rados.rados.0.smithi063.stdout:3496: left oid 2 (ObjNum 1305 snap 443 seq_num 1305) 2024-08-17T17:09:40.032 INFO:tasks.rados.rados.0.smithi063.stdout:3497: finishing write tid 1 to smithi06387500-36 2024-08-17T17:09:40.032 INFO:tasks.rados.rados.0.smithi063.stdout:3497: finishing write tid 2 to smithi06387500-36 2024-08-17T17:09:40.032 INFO:tasks.rados.rados.0.smithi063.stdout:3494: done (5 left) 2024-08-17T17:09:40.033 INFO:tasks.rados.rados.0.smithi063.stdout:3495: done (4 left) 2024-08-17T17:09:40.033 INFO:tasks.rados.rados.0.smithi063.stdout:3496: done (3 left) 2024-08-17T17:09:40.033 INFO:tasks.rados.rados.0.smithi063.stdout:3500: read oid 43 snap -1 2024-08-17T17:09:40.033 INFO:tasks.rados.rados.0.smithi063.stdout:3500: expect deleted 2024-08-17T17:09:40.033 INFO:tasks.rados.rados.0.smithi063.stdout:3501: write oid 41 current snap is 443 2024-08-17T17:09:40.033 INFO:tasks.rados.rados.0.smithi063.stdout:3499: finishing rollback tid 0 to smithi06387500-19 2024-08-17T17:09:40.033 INFO:tasks.rados.rados.0.smithi063.stdout:3501: seq_num 1307 ranges {766749=646806,1899377=656453,3109028=609462} 2024-08-17T17:09:40.039 INFO:tasks.rados.rados.0.smithi063.stdout:3501: writing smithi06387500-41 from 766749 to 1413555 tid 1 2024-08-17T17:09:40.045 INFO:tasks.rados.rados.0.smithi063.stdout:3501: writing smithi06387500-41 from 1899377 to 2555830 tid 2 2024-08-17T17:09:40.051 INFO:tasks.rados.rados.0.smithi063.stdout:3501: writing smithi06387500-41 from 3109028 to 3718490 tid 3 2024-08-17T17:09:40.051 INFO:tasks.rados.rados.0.smithi063.stdout:3502: read oid 16 snap -1 2024-08-17T17:09:40.051 INFO:tasks.rados.rados.0.smithi063.stdout:3502: expect (ObjNum 1280 snap 435 seq_num 1280) 2024-08-17T17:09:40.052 INFO:tasks.rados.rados.0.smithi063.stdout:3503: snap_remove snap 442 2024-08-17T17:09:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:39 smithi162 ceph-mon[104668]: osdmap e951: 8 total, 8 up, 8 in 2024-08-17T17:09:40.726 INFO:tasks.rados.rados.0.smithi063.stdout:3503: done (6 left) 2024-08-17T17:09:40.727 INFO:tasks.rados.rados.0.smithi063.stdout:3504: write oid 25 current snap is 443 2024-08-17T17:09:40.727 INFO:tasks.rados.rados.0.smithi063.stdout:3499: finishing rollback tid 1 to smithi06387500-19 2024-08-17T17:09:40.727 INFO:tasks.rados.rados.0.smithi063.stdout:3499: finishing rollback tid 2 to smithi06387500-19 2024-08-17T17:09:40.727 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 2252 (ObjNum 1262 snap 426 seq_num 1262) dirty exists 2024-08-17T17:09:40.727 INFO:tasks.rados.rados.0.smithi063.stdout:3497: finishing write tid 3 to smithi06387500-36 2024-08-17T17:09:40.727 INFO:tasks.rados.rados.0.smithi063.stdout:3497: finishing write tid 4 to smithi06387500-36 2024-08-17T17:09:40.727 INFO:tasks.rados.rados.0.smithi063.stdout:3504: seq_num 1308 ranges {783556=438408,1674212=400111,2736451=420734,3882270=101286} 2024-08-17T17:09:40.733 INFO:tasks.rados.rados.0.smithi063.stdout:3504: writing smithi06387500-25 from 783556 to 1221964 tid 1 2024-08-17T17:09:40.737 INFO:tasks.rados.rados.0.smithi063.stdout:3504: writing smithi06387500-25 from 1674212 to 2074323 tid 2 2024-08-17T17:09:40.743 INFO:tasks.rados.rados.0.smithi063.stdout:3504: writing smithi06387500-25 from 2736451 to 3157185 tid 3 2024-08-17T17:09:40.748 INFO:tasks.rados.rados.0.smithi063.stdout:3504: writing smithi06387500-25 from 3882270 to 3983556 tid 4 2024-08-17T17:09:40.748 INFO:tasks.rados.rados.0.smithi063.stdout:3499: done (6 left) 2024-08-17T17:09:40.749 INFO:tasks.rados.rados.0.smithi063.stdout:3505: write oid 17 current snap is 443 2024-08-17T17:09:40.749 INFO:tasks.rados.rados.0.smithi063.stdout:3505: seq_num 1309 ranges {400363=513657,1332694=660122} 2024-08-17T17:09:40.753 INFO:tasks.rados.rados.0.smithi063.stdout:3505: writing smithi06387500-17 from 400363 to 914020 tid 1 2024-08-17T17:09:40.759 INFO:tasks.rados.rados.0.smithi063.stdout:3505: writing smithi06387500-17 from 1332694 to 1992816 tid 2 2024-08-17T17:09:40.759 INFO:tasks.rados.rados.0.smithi063.stdout:3506: write oid 5 current snap is 443 2024-08-17T17:09:40.759 INFO:tasks.rados.rados.0.smithi063.stdout:3506: seq_num 1310 ranges {417170=588906,1674823=342347} 2024-08-17T17:09:40.764 INFO:tasks.rados.rados.0.smithi063.stdout:3506: writing smithi06387500-5 from 417170 to 1006076 tid 1 2024-08-17T17:09:40.771 INFO:tasks.rados.rados.0.smithi063.stdout:3506: writing smithi06387500-5 from 1674823 to 2017170 tid 2 2024-08-17T17:09:40.771 INFO:tasks.rados.rados.0.smithi063.stdout:3507: write oid 33 current snap is 443 2024-08-17T17:09:40.771 INFO:tasks.rados.rados.0.smithi063.stdout:3497: finishing write tid 5 to smithi06387500-36 2024-08-17T17:09:40.771 INFO:tasks.rados.rados.0.smithi063.stdout:3497: oid 36 updating version 0 to 3446 2024-08-17T17:09:40.771 INFO:tasks.rados.rados.0.smithi063.stdout:3497: oid 36 version 3446 is already newer than 3445 2024-08-17T17:09:40.771 INFO:tasks.rados.rados.0.smithi063.stdout:3497: oid 36 updating version 3446 to 3448 2024-08-17T17:09:40.771 INFO:tasks.rados.rados.0.smithi063.stdout:3497: oid 36 version 3448 is already newer than 3447 2024-08-17T17:09:40.771 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 3448 (ObjNum 1306 snap 443 seq_num 1306) dirty exists 2024-08-17T17:09:40.771 INFO:tasks.rados.rados.0.smithi063.stdout:3497: left oid 36 (ObjNum 1306 snap 443 seq_num 1306) 2024-08-17T17:09:40.772 INFO:tasks.rados.rados.0.smithi063.stdout:3498: expect (ObjNum 1196 snap 401 seq_num 1196) 2024-08-17T17:09:40.885 INFO:tasks.rados.rados.0.smithi063.stdout:3501: finishing write tid 1 to smithi06387500-41 2024-08-17T17:09:40.886 INFO:tasks.rados.rados.0.smithi063.stdout:3501: finishing write tid 2 to smithi06387500-41 2024-08-17T17:09:40.886 INFO:tasks.rados.rados.0.smithi063.stdout:3501: finishing write tid 3 to smithi06387500-41 2024-08-17T17:09:40.886 INFO:tasks.rados.rados.0.smithi063.stdout:3501: finishing write tid 4 to smithi06387500-41 2024-08-17T17:09:40.886 INFO:tasks.rados.rados.0.smithi063.stdout:3501: finishing write tid 5 to smithi06387500-41 2024-08-17T17:09:40.886 INFO:tasks.rados.rados.0.smithi063.stdout:3501: oid 41 updating version 0 to 400 2024-08-17T17:09:40.886 INFO:tasks.rados.rados.0.smithi063.stdout:3501: oid 41 version 400 is already newer than 398 2024-08-17T17:09:40.886 INFO:tasks.rados.rados.0.smithi063.stdout:3501: oid 41 version 400 is already newer than 399 2024-08-17T17:09:40.886 INFO:tasks.rados.rados.0.smithi063.stdout:3501: oid 41 version 400 is already newer than 397 2024-08-17T17:09:40.886 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 400 (ObjNum 1307 snap 443 seq_num 1307) dirty exists 2024-08-17T17:09:40.886 INFO:tasks.rados.rados.0.smithi063.stdout:3501: left oid 41 (ObjNum 1307 snap 443 seq_num 1307) 2024-08-17T17:09:40.886 INFO:tasks.rados.rados.0.smithi063.stdout:3502: expect (ObjNum 1280 snap 435 seq_num 1280) 2024-08-17T17:09:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:40 smithi063 ceph-mon[114756]: osdmap e952: 8 total, 8 up, 8 in 2024-08-17T17:09:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:40 smithi063 ceph-mon[114756]: pgmap v885: 89 pgs: 89 active+clean; 190 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 849 KiB/s wr, 3 op/s 2024-08-17T17:09:41.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:40 smithi063 ceph-mon[109843]: osdmap e952: 8 total, 8 up, 8 in 2024-08-17T17:09:41.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:40 smithi063 ceph-mon[109843]: pgmap v885: 89 pgs: 89 active+clean; 190 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 849 KiB/s wr, 3 op/s 2024-08-17T17:09:41.051 INFO:tasks.rados.rados.0.smithi063.stdout:3504: finishing write tid 1 to smithi06387500-25 2024-08-17T17:09:41.051 INFO:tasks.rados.rados.0.smithi063.stdout:3504: finishing write tid 2 to smithi06387500-25 2024-08-17T17:09:41.051 INFO:tasks.rados.rados.0.smithi063.stdout:3504: finishing write tid 3 to smithi06387500-25 2024-08-17T17:09:41.051 INFO:tasks.rados.rados.0.smithi063.stdout:3504: finishing write tid 4 to smithi06387500-25 2024-08-17T17:09:41.051 INFO:tasks.rados.rados.0.smithi063.stdout:3504: finishing write tid 5 to smithi06387500-25 2024-08-17T17:09:41.051 INFO:tasks.rados.rados.0.smithi063.stdout:3504: finishing write tid 6 to smithi06387500-25 2024-08-17T17:09:41.051 INFO:tasks.rados.rados.0.smithi063.stdout:3504: oid 25 updating version 0 to 3453 2024-08-17T17:09:41.051 INFO:tasks.rados.rados.0.smithi063.stdout:3504: oid 25 updating version 3453 to 3455 2024-08-17T17:09:41.051 INFO:tasks.rados.rados.0.smithi063.stdout:3504: oid 25 version 3455 is already newer than 3450 2024-08-17T17:09:41.051 INFO:tasks.rados.rados.0.smithi063.stdout:3504: oid 25 updating version 3455 to 3456 2024-08-17T17:09:41.052 INFO:tasks.rados.rados.0.smithi063.stdout:3504: oid 25 version 3456 is already newer than 3454 2024-08-17T17:09:41.052 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 3456 (ObjNum 1308 snap 443 seq_num 1308) dirty exists 2024-08-17T17:09:41.052 INFO:tasks.rados.rados.0.smithi063.stdout:3504: left oid 25 (ObjNum 1308 snap 443 seq_num 1308) 2024-08-17T17:09:41.052 INFO:tasks.rados.rados.0.smithi063.stdout:3505: finishing write tid 1 to smithi06387500-17 2024-08-17T17:09:41.052 INFO:tasks.rados.rados.0.smithi063.stdout:3505: finishing write tid 2 to smithi06387500-17 2024-08-17T17:09:41.052 INFO:tasks.rados.rados.0.smithi063.stdout:3505: finishing write tid 3 to smithi06387500-17 2024-08-17T17:09:41.052 INFO:tasks.rados.rados.0.smithi063.stdout:3507: seq_num 1311 ranges {433977=664155,1733305=300672} 2024-08-17T17:09:41.056 INFO:tasks.rados.rados.0.smithi063.stdout:3507: writing smithi06387500-33 from 433977 to 1098132 tid 1 2024-08-17T17:09:41.062 INFO:tasks.rados.rados.0.smithi063.stdout:3507: writing smithi06387500-33 from 1733305 to 2033977 tid 2 2024-08-17T17:09:41.063 INFO:tasks.rados.rados.0.smithi063.stdout:3497: done (8 left) 2024-08-17T17:09:41.063 INFO:tasks.rados.rados.0.smithi063.stdout:3498: done (7 left) 2024-08-17T17:09:41.063 INFO:tasks.rados.rados.0.smithi063.stdout:3500: done (6 left) 2024-08-17T17:09:41.063 INFO:tasks.rados.rados.0.smithi063.stdout:3501: done (5 left) 2024-08-17T17:09:41.063 INFO:tasks.rados.rados.0.smithi063.stdout:3502: done (4 left) 2024-08-17T17:09:41.063 INFO:tasks.rados.rados.0.smithi063.stdout:3504: done (3 left) 2024-08-17T17:09:41.063 INFO:tasks.rados.rados.0.smithi063.stdout:3508: rollback oid 23 current snap is 443 2024-08-17T17:09:41.063 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 23 to 440 2024-08-17T17:09:41.063 INFO:tasks.rados.rados.0.smithi063.stdout:3505: finishing write tid 4 to smithi06387500-17 2024-08-17T17:09:41.063 INFO:tasks.rados.rados.0.smithi063.stdout:3505: oid 17 updating version 0 to 1184 2024-08-17T17:09:41.063 INFO:tasks.rados.rados.0.smithi063.stdout:3505: oid 17 updating version 1184 to 1185 2024-08-17T17:09:41.064 INFO:tasks.rados.rados.0.smithi063.stdout:3505: oid 17 version 1185 is already newer than 1183 2024-08-17T17:09:41.064 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 1185 (ObjNum 1309 snap 443 seq_num 1309) dirty exists 2024-08-17T17:09:41.064 INFO:tasks.rados.rados.0.smithi063.stdout:3505: left oid 17 (ObjNum 1309 snap 443 seq_num 1309) 2024-08-17T17:09:41.064 INFO:tasks.rados.rados.0.smithi063.stdout:3506: finishing write tid 1 to smithi06387500-5 2024-08-17T17:09:41.064 INFO:tasks.rados.rados.0.smithi063.stdout:3506: finishing write tid 2 to smithi06387500-5 2024-08-17T17:09:41.064 INFO:tasks.rados.rados.0.smithi063.stdout:3506: finishing write tid 3 to smithi06387500-5 2024-08-17T17:09:41.064 INFO:tasks.rados.rados.0.smithi063.stdout:3506: finishing write tid 4 to smithi06387500-5 2024-08-17T17:09:41.064 INFO:tasks.rados.rados.0.smithi063.stdout:3506: oid 5 updating version 0 to 1810 2024-08-17T17:09:41.064 INFO:tasks.rados.rados.0.smithi063.stdout:3506: oid 5 updating version 1810 to 1811 2024-08-17T17:09:41.064 INFO:tasks.rados.rados.0.smithi063.stdout:3506: oid 5 updating version 1811 to 1812 2024-08-17T17:09:41.064 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1812 (ObjNum 1310 snap 443 seq_num 1310) dirty exists 2024-08-17T17:09:41.064 INFO:tasks.rados.rados.0.smithi063.stdout:3506: left oid 5 (ObjNum 1310 snap 443 seq_num 1310) 2024-08-17T17:09:41.064 INFO:tasks.rados.rados.0.smithi063.stdout:3507: finishing write tid 1 to smithi06387500-33 2024-08-17T17:09:41.065 INFO:tasks.rados.rados.0.smithi063.stdout:3505: done (3 left) 2024-08-17T17:09:41.065 INFO:tasks.rados.rados.0.smithi063.stdout:3506: done (2 left) 2024-08-17T17:09:41.065 INFO:tasks.rados.rados.0.smithi063.stdout:3509: write oid 22 current snap is 443 2024-08-17T17:09:41.065 INFO:tasks.rados.rados.0.smithi063.stdout:3509: seq_num 1312 ranges {450784=739404,1791787=258997} 2024-08-17T17:09:41.068 INFO:tasks.rados.rados.0.smithi063.stdout:3509: writing smithi06387500-22 from 450784 to 1190188 tid 1 2024-08-17T17:09:41.074 INFO:tasks.rados.rados.0.smithi063.stdout:3509: writing smithi06387500-22 from 1791787 to 2050784 tid 2 2024-08-17T17:09:41.074 INFO:tasks.rados.rados.0.smithi063.stdout:3507: finishing write tid 2 to smithi06387500-33 2024-08-17T17:09:41.074 INFO:tasks.rados.rados.0.smithi063.stdout:3507: finishing write tid 3 to smithi06387500-33 2024-08-17T17:09:41.074 INFO:tasks.rados.rados.0.smithi063.stdout:3507: finishing write tid 4 to smithi06387500-33 2024-08-17T17:09:41.074 INFO:tasks.rados.rados.0.smithi063.stdout:3507: oid 33 updating version 0 to 1176 2024-08-17T17:09:41.074 INFO:tasks.rados.rados.0.smithi063.stdout:3507: oid 33 updating version 1176 to 1177 2024-08-17T17:09:41.075 INFO:tasks.rados.rados.0.smithi063.stdout:3507: oid 33 updating version 1177 to 1178 2024-08-17T17:09:41.075 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 1178 (ObjNum 1311 snap 443 seq_num 1311) dirty exists 2024-08-17T17:09:41.075 INFO:tasks.rados.rados.0.smithi063.stdout:3507: left oid 33 (ObjNum 1311 snap 443 seq_num 1311) 2024-08-17T17:09:41.075 INFO:tasks.rados.rados.0.smithi063.stdout:3508: finishing rollback tid 0 to smithi06387500-23 2024-08-17T17:09:41.075 INFO:tasks.rados.rados.0.smithi063.stdout:3508: finishing rollback tid 1 to smithi06387500-23 2024-08-17T17:09:41.075 INFO:tasks.rados.rados.0.smithi063.stdout:3508: finishing rollback tid 2 to smithi06387500-23 2024-08-17T17:09:41.075 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 1816 (ObjNum 1213 snap 406 seq_num 1213) dirty exists 2024-08-17T17:09:41.075 INFO:tasks.rados.rados.0.smithi063.stdout:3509: finishing write tid 1 to smithi06387500-22 2024-08-17T17:09:41.075 INFO:tasks.rados.rados.0.smithi063.stdout:3507: done (2 left) 2024-08-17T17:09:41.075 INFO:tasks.rados.rados.0.smithi063.stdout:3508: done (1 left) 2024-08-17T17:09:41.075 INFO:tasks.rados.rados.0.smithi063.stdout:3510: rollback oid 35 current snap is 443 2024-08-17T17:09:41.075 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 35 to 430 2024-08-17T17:09:41.075 INFO:tasks.rados.rados.0.smithi063.stdout:3511: read oid 11 snap -1 2024-08-17T17:09:41.076 INFO:tasks.rados.rados.0.smithi063.stdout:3511: expect (ObjNum 1288 snap 440 seq_num 1288) 2024-08-17T17:09:41.076 INFO:tasks.rados.rados.0.smithi063.stdout:3512: snap_create 2024-08-17T17:09:41.076 INFO:tasks.rados.rados.0.smithi063.stdout:3510: finishing rollback tid 1 to smithi06387500-35 2024-08-17T17:09:41.076 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 0 (ObjNum 225 snap 0 seq_num 574598576) dirty dne 2024-08-17T17:09:41.077 INFO:tasks.rados.rados.0.smithi063.stdout:3509: finishing write tid 2 to smithi06387500-22 2024-08-17T17:09:41.077 INFO:tasks.rados.rados.0.smithi063.stdout:3509: finishing write tid 3 to smithi06387500-22 2024-08-17T17:09:41.077 INFO:tasks.rados.rados.0.smithi063.stdout:3509: finishing write tid 4 to smithi06387500-22 2024-08-17T17:09:41.077 INFO:tasks.rados.rados.0.smithi063.stdout:3509: oid 22 updating version 0 to 814 2024-08-17T17:09:41.077 INFO:tasks.rados.rados.0.smithi063.stdout:3509: oid 22 version 814 is already newer than 813 2024-08-17T17:09:41.077 INFO:tasks.rados.rados.0.smithi063.stdout:3509: oid 22 version 814 is already newer than 812 2024-08-17T17:09:41.077 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 814 (ObjNum 1312 snap 443 seq_num 1312) dirty exists 2024-08-17T17:09:41.078 INFO:tasks.rados.rados.0.smithi063.stdout:3509: left oid 22 (ObjNum 1312 snap 443 seq_num 1312) 2024-08-17T17:09:41.080 INFO:tasks.rados.rados.0.smithi063.stdout:3511: expect (ObjNum 1288 snap 440 seq_num 1288) 2024-08-17T17:09:41.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:40 smithi162 ceph-mon[104668]: osdmap e952: 8 total, 8 up, 8 in 2024-08-17T17:09:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:40 smithi162 ceph-mon[104668]: pgmap v885: 89 pgs: 89 active+clean; 190 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 849 KiB/s wr, 3 op/s 2024-08-17T17:09:41.726 INFO:tasks.rados.rados.0.smithi063.stdout:3509: done (3 left) 2024-08-17T17:09:41.726 INFO:tasks.rados.rados.0.smithi063.stdout:3510: done (2 left) 2024-08-17T17:09:41.726 INFO:tasks.rados.rados.0.smithi063.stdout:3511: done (1 left) 2024-08-17T17:09:41.726 INFO:tasks.rados.rados.0.smithi063.stdout:3512: done (0 left) 2024-08-17T17:09:41.726 INFO:tasks.rados.rados.0.smithi063.stdout:3513: rollback oid 36 current snap is 444 2024-08-17T17:09:41.726 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 36 to 427 2024-08-17T17:09:41.726 INFO:tasks.rados.rados.0.smithi063.stdout:3514: rollback oid 29 current snap is 444 2024-08-17T17:09:41.727 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 29 to 438 2024-08-17T17:09:41.727 INFO:tasks.rados.rados.0.smithi063.stdout:3515: snap_create 2024-08-17T17:09:41.730 INFO:tasks.rados.rados.0.smithi063.stdout:3514: finishing rollback tid 0 to smithi06387500-29 2024-08-17T17:09:41.731 INFO:tasks.rados.rados.0.smithi063.stdout:3514: finishing rollback tid 1 to smithi06387500-29 2024-08-17T17:09:41.731 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 801 (ObjNum 49 snap 0 seq_num 1786885104) dirty dne 2024-08-17T17:09:41.734 INFO:tasks.rados.rados.0.smithi063.stdout:3513: finishing rollback tid 0 to smithi06387500-36 2024-08-17T17:09:41.734 INFO:tasks.rados.rados.0.smithi063.stdout:3513: finishing rollback tid 1 to smithi06387500-36 2024-08-17T17:09:41.734 INFO:tasks.rados.rados.0.smithi063.stdout:3513: finishing rollback tid 2 to smithi06387500-36 2024-08-17T17:09:41.734 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 3460 (ObjNum 1255 snap 423 seq_num 1255) dirty exists 2024-08-17T17:09:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:41 smithi063 ceph-mon[114756]: osdmap e953: 8 total, 8 up, 8 in 2024-08-17T17:09:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:41 smithi063 ceph-mon[109843]: osdmap e953: 8 total, 8 up, 8 in 2024-08-17T17:09:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:41 smithi162 ceph-mon[104668]: osdmap e953: 8 total, 8 up, 8 in 2024-08-17T17:09:42.669 INFO:tasks.rados.rados.0.smithi063.stdout:3513: done (2 left) 2024-08-17T17:09:42.669 INFO:tasks.rados.rados.0.smithi063.stdout:3514: done (1 left) 2024-08-17T17:09:42.669 INFO:tasks.rados.rados.0.smithi063.stdout:3515: done (0 left) 2024-08-17T17:09:42.669 INFO:tasks.rados.rados.0.smithi063.stdout:3516: snap_create 2024-08-17T17:09:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:42 smithi063 ceph-mon[109843]: osdmap e954: 8 total, 8 up, 8 in 2024-08-17T17:09:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:42 smithi063 ceph-mon[109843]: pgmap v888: 89 pgs: 89 active+clean; 216 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 5.5 MiB/s wr, 38 op/s 2024-08-17T17:09:43.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:42 smithi063 ceph-mon[109843]: osdmap e955: 8 total, 8 up, 8 in 2024-08-17T17:09:43.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:42 smithi063 ceph-mon[114756]: osdmap e954: 8 total, 8 up, 8 in 2024-08-17T17:09:43.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:42 smithi063 ceph-mon[114756]: pgmap v888: 89 pgs: 89 active+clean; 216 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 5.5 MiB/s wr, 38 op/s 2024-08-17T17:09:43.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:42 smithi063 ceph-mon[114756]: osdmap e955: 8 total, 8 up, 8 in 2024-08-17T17:09:43.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:42 smithi162 ceph-mon[104668]: osdmap e954: 8 total, 8 up, 8 in 2024-08-17T17:09:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:42 smithi162 ceph-mon[104668]: pgmap v888: 89 pgs: 89 active+clean; 216 MiB data, 818 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 5.5 MiB/s wr, 38 op/s 2024-08-17T17:09:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:42 smithi162 ceph-mon[104668]: osdmap e955: 8 total, 8 up, 8 in 2024-08-17T17:09:43.665 INFO:tasks.rados.rados.0.smithi063.stdout:3516: done (0 left) 2024-08-17T17:09:43.665 INFO:tasks.rados.rados.0.smithi063.stdout:3517: delete oid 2 current snap is 446 2024-08-17T17:09:43.668 INFO:tasks.rados.rados.0.smithi063.stdout:3517: done (0 left) 2024-08-17T17:09:43.668 INFO:tasks.rados.rados.0.smithi063.stdout:3518: rollback oid 20 current snap is 446 2024-08-17T17:09:43.668 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 20 to 440 2024-08-17T17:09:43.668 INFO:tasks.rados.rados.0.smithi063.stdout:3519: write oid 39 current snap is 446 2024-08-17T17:09:43.668 INFO:tasks.rados.rados.0.smithi063.stdout:3519: seq_num 1314 ranges {484398=489902,1392398=411118} 2024-08-17T17:09:43.674 INFO:tasks.rados.rados.0.smithi063.stdout:3519: writing smithi06387500-39 from 484398 to 974300 tid 1 2024-08-17T17:09:43.679 INFO:tasks.rados.rados.0.smithi063.stdout:3519: writing smithi06387500-39 from 1392398 to 1803516 tid 2 2024-08-17T17:09:43.679 INFO:tasks.rados.rados.0.smithi063.stdout:3520: snap_remove snap 445 2024-08-17T17:09:43.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:09:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:09:43] "GET /metrics HTTP/1.1" 200 36337 "" "Prometheus/2.51.0" 2024-08-17T17:09:44.674 INFO:tasks.rados.rados.0.smithi063.stdout:3520: done (2 left) 2024-08-17T17:09:44.675 INFO:tasks.rados.rados.0.smithi063.stdout:3521: write oid 7 current snap is 446 2024-08-17T17:09:44.675 INFO:tasks.rados.rados.0.smithi063.stdout:3518: finishing rollback tid 0 to smithi06387500-20 2024-08-17T17:09:44.675 INFO:tasks.rados.rados.0.smithi063.stdout:3521: seq_num 1315 ranges {501205=565151,1850880=250325} 2024-08-17T17:09:44.681 INFO:tasks.rados.rados.0.smithi063.stdout:3521: writing smithi06387500-7 from 501205 to 1066356 tid 1 2024-08-17T17:09:44.688 INFO:tasks.rados.rados.0.smithi063.stdout:3521: writing smithi06387500-7 from 1850880 to 2101205 tid 2 2024-08-17T17:09:44.688 INFO:tasks.rados.rados.0.smithi063.stdout:3522: rollback oid 26 current snap is 446 2024-08-17T17:09:44.688 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 26 to 431 2024-08-17T17:09:44.688 INFO:tasks.rados.rados.0.smithi063.stdout:3518: finishing rollback tid 1 to smithi06387500-20 2024-08-17T17:09:44.688 INFO:tasks.rados.rados.0.smithi063.stdout:3518: finishing rollback tid 2 to smithi06387500-20 2024-08-17T17:09:44.688 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 818 (ObjNum 1285 snap 439 seq_num 1285) dirty exists 2024-08-17T17:09:44.688 INFO:tasks.rados.rados.0.smithi063.stdout:3519: finishing write tid 1 to smithi06387500-39 2024-08-17T17:09:44.688 INFO:tasks.rados.rados.0.smithi063.stdout:3519: finishing write tid 2 to smithi06387500-39 2024-08-17T17:09:44.688 INFO:tasks.rados.rados.0.smithi063.stdout:3519: finishing write tid 3 to smithi06387500-39 2024-08-17T17:09:44.689 INFO:tasks.rados.rados.0.smithi063.stdout:3519: finishing write tid 4 to smithi06387500-39 2024-08-17T17:09:44.689 INFO:tasks.rados.rados.0.smithi063.stdout:3519: oid 39 updating version 0 to 879 2024-08-17T17:09:44.689 INFO:tasks.rados.rados.0.smithi063.stdout:3519: oid 39 version 879 is already newer than 878 2024-08-17T17:09:44.689 INFO:tasks.rados.rados.0.smithi063.stdout:3519: oid 39 version 879 is already newer than 877 2024-08-17T17:09:44.689 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 879 (ObjNum 1314 snap 446 seq_num 1314) dirty exists 2024-08-17T17:09:44.689 INFO:tasks.rados.rados.0.smithi063.stdout:3519: left oid 39 (ObjNum 1314 snap 446 seq_num 1314) 2024-08-17T17:09:44.689 INFO:tasks.rados.rados.0.smithi063.stdout:3521: finishing write tid 1 to smithi06387500-7 2024-08-17T17:09:44.689 INFO:tasks.rados.rados.0.smithi063.stdout:3518: done (3 left) 2024-08-17T17:09:44.689 INFO:tasks.rados.rados.0.smithi063.stdout:3519: done (2 left) 2024-08-17T17:09:44.689 INFO:tasks.rados.rados.0.smithi063.stdout:3523: snap_create 2024-08-17T17:09:44.691 INFO:tasks.rados.rados.0.smithi063.stdout:3521: finishing write tid 2 to smithi06387500-7 2024-08-17T17:09:44.691 INFO:tasks.rados.rados.0.smithi063.stdout:3521: finishing write tid 3 to smithi06387500-7 2024-08-17T17:09:44.691 INFO:tasks.rados.rados.0.smithi063.stdout:3521: finishing write tid 4 to smithi06387500-7 2024-08-17T17:09:44.692 INFO:tasks.rados.rados.0.smithi063.stdout:3521: oid 7 updating version 0 to 1682 2024-08-17T17:09:44.692 INFO:tasks.rados.rados.0.smithi063.stdout:3521: oid 7 updating version 1682 to 1683 2024-08-17T17:09:44.692 INFO:tasks.rados.rados.0.smithi063.stdout:3521: oid 7 updating version 1683 to 1684 2024-08-17T17:09:44.692 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1684 (ObjNum 1315 snap 446 seq_num 1315) dirty exists 2024-08-17T17:09:44.692 INFO:tasks.rados.rados.0.smithi063.stdout:3521: left oid 7 (ObjNum 1315 snap 446 seq_num 1315) 2024-08-17T17:09:44.694 INFO:tasks.rados.rados.0.smithi063.stdout:3522: finishing rollback tid 0 to smithi06387500-26 2024-08-17T17:09:44.694 INFO:tasks.rados.rados.0.smithi063.stdout:3522: finishing rollback tid 1 to smithi06387500-26 2024-08-17T17:09:44.694 INFO:tasks.rados.rados.0.smithi063.stdout:3522: finishing rollback tid 2 to smithi06387500-26 2024-08-17T17:09:44.694 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 1189 (ObjNum 1193 snap 400 seq_num 1193) dirty exists 2024-08-17T17:09:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:44 smithi063 ceph-mon[114756]: osdmap e956: 8 total, 8 up, 8 in 2024-08-17T17:09:45.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:44 smithi063 ceph-mon[109843]: osdmap e956: 8 total, 8 up, 8 in 2024-08-17T17:09:45.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:44 smithi162 ceph-mon[104668]: osdmap e956: 8 total, 8 up, 8 in 2024-08-17T17:09:45.680 INFO:tasks.rados.rados.0.smithi063.stdout:3521: done (2 left) 2024-08-17T17:09:45.681 INFO:tasks.rados.rados.0.smithi063.stdout:3522: done (1 left) 2024-08-17T17:09:45.681 INFO:tasks.rados.rados.0.smithi063.stdout:3523: done (0 left) 2024-08-17T17:09:45.681 INFO:tasks.rados.rados.0.smithi063.stdout:3524: delete oid 23 current snap is 447 2024-08-17T17:09:45.684 INFO:tasks.rados.rados.0.smithi063.stdout:3524: done (0 left) 2024-08-17T17:09:45.684 INFO:tasks.rados.rados.0.smithi063.stdout:3525: read oid 34 snap -1 2024-08-17T17:09:45.684 INFO:tasks.rados.rados.0.smithi063.stdout:3525: expect (ObjNum 1295 snap 442 seq_num 1295) 2024-08-17T17:09:45.684 INFO:tasks.rados.rados.0.smithi063.stdout:3526: snap_remove snap 437 2024-08-17T17:09:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:45 smithi063 ceph-mon[114756]: pgmap v891: 89 pgs: 89 active+clean; 222 MiB data, 830 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 6.2 MiB/s wr, 43 op/s 2024-08-17T17:09:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:45 smithi063 ceph-mon[114756]: osdmap e957: 8 total, 8 up, 8 in 2024-08-17T17:09:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:45 smithi063 ceph-mon[109843]: pgmap v891: 89 pgs: 89 active+clean; 222 MiB data, 830 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 6.2 MiB/s wr, 43 op/s 2024-08-17T17:09:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:45 smithi063 ceph-mon[109843]: osdmap e957: 8 total, 8 up, 8 in 2024-08-17T17:09:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:45 smithi162 ceph-mon[104668]: pgmap v891: 89 pgs: 89 active+clean; 222 MiB data, 830 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 6.2 MiB/s wr, 43 op/s 2024-08-17T17:09:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:45 smithi162 ceph-mon[104668]: osdmap e957: 8 total, 8 up, 8 in 2024-08-17T17:09:46.688 INFO:tasks.rados.rados.0.smithi063.stdout:3526: done (1 left) 2024-08-17T17:09:46.689 INFO:tasks.rados.rados.0.smithi063.stdout:3527: write oid 22 current snap is 447 2024-08-17T17:09:46.689 INFO:tasks.rados.rados.0.smithi063.stdout:3527: seq_num 1317 ranges {534819=432002,1684197=450622} 2024-08-17T17:09:46.694 INFO:tasks.rados.rados.0.smithi063.stdout:3527: writing smithi06387500-22 from 534819 to 966821 tid 1 2024-08-17T17:09:46.702 INFO:tasks.rados.rados.0.smithi063.stdout:3527: writing smithi06387500-22 from 1684197 to 2134819 tid 2 2024-08-17T17:09:46.702 INFO:tasks.rados.rados.0.smithi063.stdout:3528: delete oid 7 current snap is 447 2024-08-17T17:09:46.702 INFO:tasks.rados.rados.0.smithi063.stdout:3525: expect (ObjNum 1295 snap 442 seq_num 1295) 2024-08-17T17:09:46.826 INFO:tasks.rados.rados.0.smithi063.stdout:3527: finishing write tid 1 to smithi06387500-22 2024-08-17T17:09:46.826 INFO:tasks.rados.rados.0.smithi063.stdout:3527: finishing write tid 2 to smithi06387500-22 2024-08-17T17:09:46.826 INFO:tasks.rados.rados.0.smithi063.stdout:3527: finishing write tid 3 to smithi06387500-22 2024-08-17T17:09:46.826 INFO:tasks.rados.rados.0.smithi063.stdout:3527: finishing write tid 4 to smithi06387500-22 2024-08-17T17:09:46.827 INFO:tasks.rados.rados.0.smithi063.stdout:3527: oid 22 updating version 0 to 826 2024-08-17T17:09:46.827 INFO:tasks.rados.rados.0.smithi063.stdout:3527: oid 22 updating version 826 to 829 2024-08-17T17:09:46.827 INFO:tasks.rados.rados.0.smithi063.stdout:3527: oid 22 updating version 829 to 830 2024-08-17T17:09:46.827 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 830 (ObjNum 1317 snap 447 seq_num 1317) dirty exists 2024-08-17T17:09:46.827 INFO:tasks.rados.rados.0.smithi063.stdout:3527: left oid 22 (ObjNum 1317 snap 447 seq_num 1317) 2024-08-17T17:09:46.827 INFO:tasks.rados.rados.0.smithi063.stdout:3525: done (2 left) 2024-08-17T17:09:46.827 INFO:tasks.rados.rados.0.smithi063.stdout:3527: done (1 left) 2024-08-17T17:09:46.827 INFO:tasks.rados.rados.0.smithi063.stdout:3528: done (0 left) 2024-08-17T17:09:46.827 INFO:tasks.rados.rados.0.smithi063.stdout:3529: read oid 28 snap -1 2024-08-17T17:09:46.827 INFO:tasks.rados.rados.0.smithi063.stdout:3529: expect deleted 2024-08-17T17:09:46.827 INFO:tasks.rados.rados.0.smithi063.stdout:3530: write oid 48 current snap is 447 2024-08-17T17:09:46.828 INFO:tasks.rados.rados.0.smithi063.stdout:3530: seq_num 1319 ranges {568433=582500,1684808=483625} 2024-08-17T17:09:46.832 INFO:tasks.rados.rados.0.smithi063.stdout:3530: writing smithi06387500-48 from 568433 to 1150933 tid 1 2024-08-17T17:09:46.838 INFO:tasks.rados.rados.0.smithi063.stdout:3530: writing smithi06387500-48 from 1684808 to 2168433 tid 2 2024-08-17T17:09:46.838 INFO:tasks.rados.rados.0.smithi063.stdout:3531: delete oid 39 current snap is 447 2024-08-17T17:09:46.838 INFO:tasks.rados.rados.0.smithi063.stdout:3530: finishing write tid 1 to smithi06387500-48 2024-08-17T17:09:46.840 INFO:tasks.rados.rados.0.smithi063.stdout:3529: done (2 left) 2024-08-17T17:09:46.840 INFO:tasks.rados.rados.0.smithi063.stdout:3531: done (1 left) 2024-08-17T17:09:46.840 INFO:tasks.rados.rados.0.smithi063.stdout:3532: write oid 41 current snap is 447 2024-08-17T17:09:46.840 INFO:tasks.rados.rados.0.smithi063.stdout:3532: seq_num 1321 ranges {602047=732998,1801772=400275} 2024-08-17T17:09:46.847 INFO:tasks.rados.rados.0.smithi063.stdout:3532: writing smithi06387500-41 from 602047 to 1335045 tid 1 2024-08-17T17:09:46.853 INFO:tasks.rados.rados.0.smithi063.stdout:3532: writing smithi06387500-41 from 1801772 to 2202047 tid 2 2024-08-17T17:09:46.853 INFO:tasks.rados.rados.0.smithi063.stdout:3533: read oid 26 snap 422 2024-08-17T17:09:46.853 INFO:tasks.rados.rados.0.smithi063.stdout:3533: expect (ObjNum 1193 snap 400 seq_num 1193) 2024-08-17T17:09:46.854 INFO:tasks.rados.rados.0.smithi063.stdout:3534: rollback oid 5 current snap is 447 2024-08-17T17:09:46.854 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 440 2024-08-17T17:09:46.854 INFO:tasks.rados.rados.0.smithi063.stdout:3530: finishing write tid 2 to smithi06387500-48 2024-08-17T17:09:46.854 INFO:tasks.rados.rados.0.smithi063.stdout:3530: finishing write tid 3 to smithi06387500-48 2024-08-17T17:09:46.854 INFO:tasks.rados.rados.0.smithi063.stdout:3530: finishing write tid 4 to smithi06387500-48 2024-08-17T17:09:46.854 INFO:tasks.rados.rados.0.smithi063.stdout:3530: oid 48 updating version 0 to 3468 2024-08-17T17:09:46.854 INFO:tasks.rados.rados.0.smithi063.stdout:3530: oid 48 version 3468 is already newer than 3467 2024-08-17T17:09:46.854 INFO:tasks.rados.rados.0.smithi063.stdout:3530: oid 48 updating version 3468 to 3469 2024-08-17T17:09:46.854 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 3469 (ObjNum 1319 snap 447 seq_num 1319) dirty exists 2024-08-17T17:09:46.854 INFO:tasks.rados.rados.0.smithi063.stdout:3530: left oid 48 (ObjNum 1319 snap 447 seq_num 1319) 2024-08-17T17:09:46.854 INFO:tasks.rados.rados.0.smithi063.stdout:3532: finishing write tid 1 to smithi06387500-41 2024-08-17T17:09:46.855 INFO:tasks.rados.rados.0.smithi063.stdout:3530: done (3 left) 2024-08-17T17:09:46.855 INFO:tasks.rados.rados.0.smithi063.stdout:3535: read oid 47 snap 427 2024-08-17T17:09:46.855 INFO:tasks.rados.rados.0.smithi063.stdout:3535: expect (ObjNum 1230 snap 411 seq_num 1230) 2024-08-17T17:09:46.855 INFO:tasks.rados.rados.0.smithi063.stdout:3536: rollback oid 12 current snap is 447 2024-08-17T17:09:46.855 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 12 to 444 2024-08-17T17:09:46.855 INFO:tasks.rados.rados.0.smithi063.stdout:3537: read oid 11 snap -1 2024-08-17T17:09:46.855 INFO:tasks.rados.rados.0.smithi063.stdout:3537: expect (ObjNum 1288 snap 440 seq_num 1288) 2024-08-17T17:09:46.855 INFO:tasks.rados.rados.0.smithi063.stdout:3538: rollback oid 29 current snap is 447 2024-08-17T17:09:46.855 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 29 to 426 2024-08-17T17:09:46.855 INFO:tasks.rados.rados.0.smithi063.stdout:3539: write oid 1 current snap is 447 2024-08-17T17:09:46.855 INFO:tasks.rados.rados.0.smithi063.stdout:3539: seq_num 1322 ranges {618854=408247,1743901=425794} 2024-08-17T17:09:46.859 INFO:tasks.rados.rados.0.smithi063.stdout:3539: writing smithi06387500-1 from 618854 to 1027101 tid 1 2024-08-17T17:09:46.865 INFO:tasks.rados.rados.0.smithi063.stdout:3539: writing smithi06387500-1 from 1743901 to 2169695 tid 2 2024-08-17T17:09:46.866 INFO:tasks.rados.rados.0.smithi063.stdout:3540: snap_remove snap 432 2024-08-17T17:09:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:46 smithi063 ceph-mon[114756]: osdmap e958: 8 total, 8 up, 8 in 2024-08-17T17:09:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:46 smithi063 ceph-mon[109843]: osdmap e958: 8 total, 8 up, 8 in 2024-08-17T17:09:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:46 smithi162 ceph-mon[104668]: osdmap e958: 8 total, 8 up, 8 in 2024-08-17T17:09:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:09:47.668 INFO:tasks.rados.rados.0.smithi063.stdout:3540: done (8 left) 2024-08-17T17:09:47.668 INFO:tasks.rados.rados.0.smithi063.stdout:3541: read oid 42 snap -1 2024-08-17T17:09:47.668 INFO:tasks.rados.rados.0.smithi063.stdout:3541: expect (ObjNum 1304 snap 443 seq_num 1304) 2024-08-17T17:09:47.668 INFO:tasks.rados.rados.0.smithi063.stdout:3542: snap_create 2024-08-17T17:09:47.668 INFO:tasks.rados.rados.0.smithi063.stdout:3532: finishing write tid 2 to smithi06387500-41 2024-08-17T17:09:47.669 INFO:tasks.rados.rados.0.smithi063.stdout:3532: finishing write tid 3 to smithi06387500-41 2024-08-17T17:09:47.669 INFO:tasks.rados.rados.0.smithi063.stdout:3532: finishing write tid 4 to smithi06387500-41 2024-08-17T17:09:47.669 INFO:tasks.rados.rados.0.smithi063.stdout:3532: oid 41 updating version 0 to 402 2024-08-17T17:09:47.669 INFO:tasks.rados.rados.0.smithi063.stdout:3532: oid 41 updating version 402 to 404 2024-08-17T17:09:47.669 INFO:tasks.rados.rados.0.smithi063.stdout:3532: oid 41 version 404 is already newer than 403 2024-08-17T17:09:47.669 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 404 (ObjNum 1321 snap 447 seq_num 1321) dirty exists 2024-08-17T17:09:47.669 INFO:tasks.rados.rados.0.smithi063.stdout:3532: left oid 41 (ObjNum 1321 snap 447 seq_num 1321) 2024-08-17T17:09:47.669 INFO:tasks.rados.rados.0.smithi063.stdout:3537: expect (ObjNum 1288 snap 440 seq_num 1288) 2024-08-17T17:09:47.840 INFO:tasks.rados.rados.0.smithi063.stdout:3536: finishing rollback tid 1 to smithi06387500-12 2024-08-17T17:09:47.840 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 757 (ObjNum 33 snap 0 seq_num 1027769440) dirty dne 2024-08-17T17:09:47.841 INFO:tasks.rados.rados.0.smithi063.stdout:3538: finishing rollback tid 1 to smithi06387500-29 2024-08-17T17:09:47.841 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 801 (ObjNum 33 snap 0 seq_num 1806131088) dirty dne 2024-08-17T17:09:47.841 INFO:tasks.rados.rados.0.smithi063.stdout:3535: expect (ObjNum 1230 snap 411 seq_num 1230) 2024-08-17T17:09:47.929 INFO:tasks.rados.rados.0.smithi063.stdout:3534: finishing rollback tid 0 to smithi06387500-5 2024-08-17T17:09:47.929 INFO:tasks.rados.rados.0.smithi063.stdout:3534: finishing rollback tid 1 to smithi06387500-5 2024-08-17T17:09:47.929 INFO:tasks.rados.rados.0.smithi063.stdout:3534: finishing rollback tid 2 to smithi06387500-5 2024-08-17T17:09:47.930 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1832 (ObjNum 1275 snap 434 seq_num 1275) dirty exists 2024-08-17T17:09:47.930 INFO:tasks.rados.rados.0.smithi063.stdout:3533: expect (ObjNum 1193 snap 400 seq_num 1193) 2024-08-17T17:09:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:47 smithi063 ceph-mon[114756]: pgmap v894: 89 pgs: 89 active+clean; 223 MiB data, 835 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 951 KiB/s wr, 10 op/s 2024-08-17T17:09:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:47 smithi063 ceph-mon[114756]: osdmap e959: 8 total, 8 up, 8 in 2024-08-17T17:09:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:47 smithi063 ceph-mon[114756]: osdmap e960: 8 total, 8 up, 8 in 2024-08-17T17:09:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:47 smithi063 ceph-mon[109843]: pgmap v894: 89 pgs: 89 active+clean; 223 MiB data, 835 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 951 KiB/s wr, 10 op/s 2024-08-17T17:09:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:47 smithi063 ceph-mon[109843]: osdmap e959: 8 total, 8 up, 8 in 2024-08-17T17:09:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:47 smithi063 ceph-mon[109843]: osdmap e960: 8 total, 8 up, 8 in 2024-08-17T17:09:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:47 smithi162 ceph-mon[104668]: pgmap v894: 89 pgs: 89 active+clean; 223 MiB data, 835 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 951 KiB/s wr, 10 op/s 2024-08-17T17:09:48.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:47 smithi162 ceph-mon[104668]: osdmap e959: 8 total, 8 up, 8 in 2024-08-17T17:09:48.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:47 smithi162 ceph-mon[104668]: osdmap e960: 8 total, 8 up, 8 in 2024-08-17T17:09:48.182 INFO:tasks.rados.rados.0.smithi063.stdout:3539: finishing write tid 1 to smithi06387500-1 2024-08-17T17:09:48.182 INFO:tasks.rados.rados.0.smithi063.stdout:3539: finishing write tid 2 to smithi06387500-1 2024-08-17T17:09:48.182 INFO:tasks.rados.rados.0.smithi063.stdout:3539: finishing write tid 3 to smithi06387500-1 2024-08-17T17:09:48.182 INFO:tasks.rados.rados.0.smithi063.stdout:3539: finishing write tid 4 to smithi06387500-1 2024-08-17T17:09:48.182 INFO:tasks.rados.rados.0.smithi063.stdout:3539: oid 1 updating version 0 to 1835 2024-08-17T17:09:48.183 INFO:tasks.rados.rados.0.smithi063.stdout:3539: oid 1 version 1835 is already newer than 1834 2024-08-17T17:09:48.183 INFO:tasks.rados.rados.0.smithi063.stdout:3539: oid 1 updating version 1835 to 1836 2024-08-17T17:09:48.183 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 1836 (ObjNum 1322 snap 447 seq_num 1322) dirty exists 2024-08-17T17:09:48.183 INFO:tasks.rados.rados.0.smithi063.stdout:3539: left oid 1 (ObjNum 1322 snap 447 seq_num 1322) 2024-08-17T17:09:48.183 INFO:tasks.rados.rados.0.smithi063.stdout:3541: expect (ObjNum 1304 snap 443 seq_num 1304) 2024-08-17T17:09:48.667 INFO:tasks.rados.rados.0.smithi063.stdout:3532: done (9 left) 2024-08-17T17:09:48.667 INFO:tasks.rados.rados.0.smithi063.stdout:3533: done (8 left) 2024-08-17T17:09:48.667 INFO:tasks.rados.rados.0.smithi063.stdout:3534: done (7 left) 2024-08-17T17:09:48.667 INFO:tasks.rados.rados.0.smithi063.stdout:3535: done (6 left) 2024-08-17T17:09:48.667 INFO:tasks.rados.rados.0.smithi063.stdout:3536: done (5 left) 2024-08-17T17:09:48.667 INFO:tasks.rados.rados.0.smithi063.stdout:3537: done (4 left) 2024-08-17T17:09:48.667 INFO:tasks.rados.rados.0.smithi063.stdout:3538: done (3 left) 2024-08-17T17:09:48.667 INFO:tasks.rados.rados.0.smithi063.stdout:3539: done (2 left) 2024-08-17T17:09:48.667 INFO:tasks.rados.rados.0.smithi063.stdout:3541: done (1 left) 2024-08-17T17:09:48.668 INFO:tasks.rados.rados.0.smithi063.stdout:3542: done (0 left) 2024-08-17T17:09:48.668 INFO:tasks.rados.rados.0.smithi063.stdout:3543: read oid 49 snap -1 2024-08-17T17:09:48.668 INFO:tasks.rados.rados.0.smithi063.stdout:3543: expect (ObjNum 1101 snap 366 seq_num 1101) 2024-08-17T17:09:48.668 INFO:tasks.rados.rados.0.smithi063.stdout:3544: snap_create 2024-08-17T17:09:48.678 INFO:tasks.rados.rados.0.smithi063.stdout:3543: expect (ObjNum 1101 snap 366 seq_num 1101) 2024-08-17T17:09:49.669 INFO:tasks.rados.rados.0.smithi063.stdout:3543: done (1 left) 2024-08-17T17:09:49.669 INFO:tasks.rados.rados.0.smithi063.stdout:3544: done (0 left) 2024-08-17T17:09:49.669 INFO:tasks.rados.rados.0.smithi063.stdout:3545: write oid 33 current snap is 449 2024-08-17T17:09:49.670 INFO:tasks.rados.rados.0.smithi063.stdout:3545: seq_num 1323 ranges {635661=599849,1918736=316925} 2024-08-17T17:09:49.676 INFO:tasks.rados.rados.0.smithi063.stdout:3545: writing smithi06387500-33 from 635661 to 1235510 tid 1 2024-08-17T17:09:49.682 INFO:tasks.rados.rados.0.smithi063.stdout:3545: writing smithi06387500-33 from 1918736 to 2235661 tid 2 2024-08-17T17:09:49.682 INFO:tasks.rados.rados.0.smithi063.stdout:3546: write oid 11 current snap is 449 2024-08-17T17:09:49.682 INFO:tasks.rados.rados.0.smithi063.stdout:3546: seq_num 1324 ranges {652468=675098,1977218=275250} 2024-08-17T17:09:49.689 INFO:tasks.rados.rados.0.smithi063.stdout:3546: writing smithi06387500-11 from 652468 to 1327566 tid 1 2024-08-17T17:09:49.695 INFO:tasks.rados.rados.0.smithi063.stdout:3546: writing smithi06387500-11 from 1977218 to 2252468 tid 2 2024-08-17T17:09:49.695 INFO:tasks.rados.rados.0.smithi063.stdout:3547: rollback oid 15 current snap is 449 2024-08-17T17:09:49.695 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 15 to 444 2024-08-17T17:09:49.695 INFO:tasks.rados.rados.0.smithi063.stdout:3545: finishing write tid 1 to smithi06387500-33 2024-08-17T17:09:49.695 INFO:tasks.rados.rados.0.smithi063.stdout:3545: finishing write tid 2 to smithi06387500-33 2024-08-17T17:09:49.695 INFO:tasks.rados.rados.0.smithi063.stdout:3545: finishing write tid 3 to smithi06387500-33 2024-08-17T17:09:49.695 INFO:tasks.rados.rados.0.smithi063.stdout:3545: finishing write tid 4 to smithi06387500-33 2024-08-17T17:09:49.695 INFO:tasks.rados.rados.0.smithi063.stdout:3545: oid 33 updating version 0 to 1182 2024-08-17T17:09:49.696 INFO:tasks.rados.rados.0.smithi063.stdout:3545: oid 33 updating version 1182 to 1184 2024-08-17T17:09:49.696 INFO:tasks.rados.rados.0.smithi063.stdout:3545: oid 33 version 1184 is already newer than 1183 2024-08-17T17:09:49.696 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 1184 (ObjNum 1323 snap 449 seq_num 1323) dirty exists 2024-08-17T17:09:49.696 INFO:tasks.rados.rados.0.smithi063.stdout:3545: left oid 33 (ObjNum 1323 snap 449 seq_num 1323) 2024-08-17T17:09:49.696 INFO:tasks.rados.rados.0.smithi063.stdout:3545: done (2 left) 2024-08-17T17:09:49.696 INFO:tasks.rados.rados.0.smithi063.stdout:3548: write oid 13 current snap is 449 2024-08-17T17:09:49.696 INFO:tasks.rados.rados.0.smithi063.stdout:3548: seq_num 1325 ranges {669275=750347,2035700=233575} 2024-08-17T17:09:49.702 INFO:tasks.rados.rados.0.smithi063.stdout:3548: writing smithi06387500-13 from 669275 to 1419622 tid 1 2024-08-17T17:09:49.708 INFO:tasks.rados.rados.0.smithi063.stdout:3548: writing smithi06387500-13 from 2035700 to 2269275 tid 2 2024-08-17T17:09:49.708 INFO:tasks.rados.rados.0.smithi063.stdout:3549: rollback oid 6 current snap is 449 2024-08-17T17:09:49.709 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 6 to 446 2024-08-17T17:09:49.709 INFO:tasks.rados.rados.0.smithi063.stdout:3547: finishing rollback tid 0 to smithi06387500-15 2024-08-17T17:09:49.709 INFO:tasks.rados.rados.0.smithi063.stdout:3546: finishing write tid 1 to smithi06387500-11 2024-08-17T17:09:49.709 INFO:tasks.rados.rados.0.smithi063.stdout:3546: finishing write tid 2 to smithi06387500-11 2024-08-17T17:09:49.709 INFO:tasks.rados.rados.0.smithi063.stdout:3546: finishing write tid 3 to smithi06387500-11 2024-08-17T17:09:49.709 INFO:tasks.rados.rados.0.smithi063.stdout:3546: finishing write tid 4 to smithi06387500-11 2024-08-17T17:09:49.709 INFO:tasks.rados.rados.0.smithi063.stdout:3546: oid 11 updating version 0 to 686 2024-08-17T17:09:49.709 INFO:tasks.rados.rados.0.smithi063.stdout:3546: oid 11 version 686 is already newer than 685 2024-08-17T17:09:49.709 INFO:tasks.rados.rados.0.smithi063.stdout:3546: oid 11 updating version 686 to 687 2024-08-17T17:09:49.709 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 687 (ObjNum 1324 snap 449 seq_num 1324) dirty exists 2024-08-17T17:09:49.709 INFO:tasks.rados.rados.0.smithi063.stdout:3546: left oid 11 (ObjNum 1324 snap 449 seq_num 1324) 2024-08-17T17:09:49.709 INFO:tasks.rados.rados.0.smithi063.stdout:3547: finishing rollback tid 1 to smithi06387500-15 2024-08-17T17:09:49.709 INFO:tasks.rados.rados.0.smithi063.stdout:3547: finishing rollback tid 2 to smithi06387500-15 2024-08-17T17:09:49.710 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 1278 (ObjNum 1240 snap 415 seq_num 1240) dirty exists 2024-08-17T17:09:49.710 INFO:tasks.rados.rados.0.smithi063.stdout:3546: done (3 left) 2024-08-17T17:09:49.710 INFO:tasks.rados.rados.0.smithi063.stdout:3547: done (2 left) 2024-08-17T17:09:49.710 INFO:tasks.rados.rados.0.smithi063.stdout:3550: delete oid 31 current snap is 449 2024-08-17T17:09:49.711 INFO:tasks.rados.rados.0.smithi063.stdout:3550: done (2 left) 2024-08-17T17:09:49.711 INFO:tasks.rados.rados.0.smithi063.stdout:3551: write oid 8 current snap is 449 2024-08-17T17:09:49.711 INFO:tasks.rados.rados.0.smithi063.stdout:3551: seq_num 1327 ranges {702889=500845,1636311=576790} 2024-08-17T17:09:49.718 INFO:tasks.rados.rados.0.smithi063.stdout:3551: writing smithi06387500-8 from 702889 to 1203734 tid 1 2024-08-17T17:09:49.723 INFO:tasks.rados.rados.0.smithi063.stdout:3551: writing smithi06387500-8 from 1636311 to 2213101 tid 2 2024-08-17T17:09:49.724 INFO:tasks.rados.rados.0.smithi063.stdout:3552: write oid 40 current snap is 449 2024-08-17T17:09:49.724 INFO:tasks.rados.rados.0.smithi063.stdout:3552: seq_num 1328 ranges {719696=576094,2094793=224903} 2024-08-17T17:09:49.730 INFO:tasks.rados.rados.0.smithi063.stdout:3552: writing smithi06387500-40 from 719696 to 1295790 tid 1 2024-08-17T17:09:49.736 INFO:tasks.rados.rados.0.smithi063.stdout:3552: writing smithi06387500-40 from 2094793 to 2319696 tid 2 2024-08-17T17:09:49.736 INFO:tasks.rados.rados.0.smithi063.stdout:3553: read oid 41 snap -1 2024-08-17T17:09:49.737 INFO:tasks.rados.rados.0.smithi063.stdout:3553: expect (ObjNum 1321 snap 447 seq_num 1321) 2024-08-17T17:09:49.737 INFO:tasks.rados.rados.0.smithi063.stdout:3554: snap_remove snap 447 2024-08-17T17:09:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:49 smithi063 ceph-mon[114756]: pgmap v897: 89 pgs: 89 active+clean; 231 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-08-17T17:09:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:49 smithi063 ceph-mon[114756]: osdmap e961: 8 total, 8 up, 8 in 2024-08-17T17:09:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:49 smithi063 ceph-mon[109843]: pgmap v897: 89 pgs: 89 active+clean; 231 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-08-17T17:09:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:49 smithi063 ceph-mon[109843]: osdmap e961: 8 total, 8 up, 8 in 2024-08-17T17:09:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:49 smithi162 ceph-mon[104668]: pgmap v897: 89 pgs: 89 active+clean; 231 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 1.4 MiB/s wr, 17 op/s 2024-08-17T17:09:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:49 smithi162 ceph-mon[104668]: osdmap e961: 8 total, 8 up, 8 in 2024-08-17T17:09:50.679 INFO:tasks.rados.rados.0.smithi063.stdout:3554: done (5 left) 2024-08-17T17:09:50.679 INFO:tasks.rados.rados.0.smithi063.stdout:3555: rollback oid 19 current snap is 449 2024-08-17T17:09:50.679 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 19 to 446 2024-08-17T17:09:50.680 INFO:tasks.rados.rados.0.smithi063.stdout:3548: finishing write tid 1 to smithi06387500-13 2024-08-17T17:09:50.680 INFO:tasks.rados.rados.0.smithi063.stdout:3549: finishing rollback tid 0 to smithi06387500-6 2024-08-17T17:09:50.680 INFO:tasks.rados.rados.0.smithi063.stdout:3548: finishing write tid 2 to smithi06387500-13 2024-08-17T17:09:50.680 INFO:tasks.rados.rados.0.smithi063.stdout:3549: finishing rollback tid 1 to smithi06387500-6 2024-08-17T17:09:50.680 INFO:tasks.rados.rados.0.smithi063.stdout:3548: finishing write tid 3 to smithi06387500-13 2024-08-17T17:09:50.680 INFO:tasks.rados.rados.0.smithi063.stdout:3549: finishing rollback tid 2 to smithi06387500-6 2024-08-17T17:09:50.680 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 1188 (ObjNum 1301 snap 442 seq_num 1301) dirty exists 2024-08-17T17:09:50.680 INFO:tasks.rados.rados.0.smithi063.stdout:3548: finishing write tid 4 to smithi06387500-13 2024-08-17T17:09:50.680 INFO:tasks.rados.rados.0.smithi063.stdout:3548: oid 13 updating version 0 to 1846 2024-08-17T17:09:50.680 INFO:tasks.rados.rados.0.smithi063.stdout:3548: oid 13 version 1846 is already newer than 1844 2024-08-17T17:09:50.680 INFO:tasks.rados.rados.0.smithi063.stdout:3548: oid 13 version 1846 is already newer than 1845 2024-08-17T17:09:50.680 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1846 (ObjNum 1325 snap 449 seq_num 1325) dirty exists 2024-08-17T17:09:50.680 INFO:tasks.rados.rados.0.smithi063.stdout:3548: left oid 13 (ObjNum 1325 snap 449 seq_num 1325) 2024-08-17T17:09:50.681 INFO:tasks.rados.rados.0.smithi063.stdout:3551: finishing write tid 1 to smithi06387500-8 2024-08-17T17:09:50.681 INFO:tasks.rados.rados.0.smithi063.stdout:3551: finishing write tid 2 to smithi06387500-8 2024-08-17T17:09:50.681 INFO:tasks.rados.rados.0.smithi063.stdout:3551: finishing write tid 3 to smithi06387500-8 2024-08-17T17:09:50.681 INFO:tasks.rados.rados.0.smithi063.stdout:3551: finishing write tid 4 to smithi06387500-8 2024-08-17T17:09:50.681 INFO:tasks.rados.rados.0.smithi063.stdout:3551: oid 8 updating version 0 to 1191 2024-08-17T17:09:50.681 INFO:tasks.rados.rados.0.smithi063.stdout:3551: oid 8 version 1191 is already newer than 1190 2024-08-17T17:09:50.681 INFO:tasks.rados.rados.0.smithi063.stdout:3551: oid 8 version 1191 is already newer than 1189 2024-08-17T17:09:50.681 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 1191 (ObjNum 1327 snap 449 seq_num 1327) dirty exists 2024-08-17T17:09:50.681 INFO:tasks.rados.rados.0.smithi063.stdout:3551: left oid 8 (ObjNum 1327 snap 449 seq_num 1327) 2024-08-17T17:09:50.681 INFO:tasks.rados.rados.0.smithi063.stdout:3552: finishing write tid 1 to smithi06387500-40 2024-08-17T17:09:50.681 INFO:tasks.rados.rados.0.smithi063.stdout:3552: finishing write tid 2 to smithi06387500-40 2024-08-17T17:09:50.681 INFO:tasks.rados.rados.0.smithi063.stdout:3552: finishing write tid 3 to smithi06387500-40 2024-08-17T17:09:50.682 INFO:tasks.rados.rados.0.smithi063.stdout:3552: finishing write tid 4 to smithi06387500-40 2024-08-17T17:09:50.682 INFO:tasks.rados.rados.0.smithi063.stdout:3552: oid 40 updating version 0 to 1210 2024-08-17T17:09:50.682 INFO:tasks.rados.rados.0.smithi063.stdout:3552: oid 40 version 1210 is already newer than 1209 2024-08-17T17:09:50.682 INFO:tasks.rados.rados.0.smithi063.stdout:3552: oid 40 version 1210 is already newer than 1208 2024-08-17T17:09:50.682 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 1210 (ObjNum 1328 snap 449 seq_num 1328) dirty exists 2024-08-17T17:09:50.682 INFO:tasks.rados.rados.0.smithi063.stdout:3552: left oid 40 (ObjNum 1328 snap 449 seq_num 1328) 2024-08-17T17:09:50.682 INFO:tasks.rados.rados.0.smithi063.stdout:3553: expect (ObjNum 1321 snap 447 seq_num 1321) 2024-08-17T17:09:50.780 INFO:tasks.rados.rados.0.smithi063.stdout:3555: finishing rollback tid 0 to smithi06387500-19 2024-08-17T17:09:50.780 INFO:tasks.rados.rados.0.smithi063.stdout:3555: finishing rollback tid 1 to smithi06387500-19 2024-08-17T17:09:50.780 INFO:tasks.rados.rados.0.smithi063.stdout:3555: finishing rollback tid 2 to smithi06387500-19 2024-08-17T17:09:50.780 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 2265 (ObjNum 1262 snap 426 seq_num 1262) dirty exists 2024-08-17T17:09:50.780 INFO:tasks.rados.rados.0.smithi063.stdout:3548: done (5 left) 2024-08-17T17:09:50.780 INFO:tasks.rados.rados.0.smithi063.stdout:3549: done (4 left) 2024-08-17T17:09:50.780 INFO:tasks.rados.rados.0.smithi063.stdout:3551: done (3 left) 2024-08-17T17:09:50.781 INFO:tasks.rados.rados.0.smithi063.stdout:3552: done (2 left) 2024-08-17T17:09:50.781 INFO:tasks.rados.rados.0.smithi063.stdout:3553: done (1 left) 2024-08-17T17:09:50.781 INFO:tasks.rados.rados.0.smithi063.stdout:3555: done (0 left) 2024-08-17T17:09:50.781 INFO:tasks.rados.rados.0.smithi063.stdout:3556: write oid 32 current snap is 449 2024-08-17T17:09:50.781 INFO:tasks.rados.rados.0.smithi063.stdout:3556: seq_num 1329 ranges {736503=651343,2153275=183228} 2024-08-17T17:09:50.787 INFO:tasks.rados.rados.0.smithi063.stdout:3556: writing smithi06387500-32 from 736503 to 1387846 tid 1 2024-08-17T17:09:50.793 INFO:tasks.rados.rados.0.smithi063.stdout:3556: writing smithi06387500-32 from 2153275 to 2336503 tid 2 2024-08-17T17:09:50.793 INFO:tasks.rados.rados.0.smithi063.stdout:3557: delete oid 6 current snap is 449 2024-08-17T17:09:50.793 INFO:tasks.rados.rados.0.smithi063.stdout:3556: finishing write tid 1 to smithi06387500-32 2024-08-17T17:09:50.795 INFO:tasks.rados.rados.0.smithi063.stdout:3557: done (1 left) 2024-08-17T17:09:50.795 INFO:tasks.rados.rados.0.smithi063.stdout:3558: write oid 45 current snap is 449 2024-08-17T17:09:50.795 INFO:tasks.rados.rados.0.smithi063.stdout:3558: seq_num 1331 ranges {770117=518194,1870239=499878} 2024-08-17T17:09:50.803 INFO:tasks.rados.rados.0.smithi063.stdout:3558: writing smithi06387500-45 from 770117 to 1288311 tid 1 2024-08-17T17:09:50.810 INFO:tasks.rados.rados.0.smithi063.stdout:3558: writing smithi06387500-45 from 1870239 to 2370117 tid 2 2024-08-17T17:09:50.810 INFO:tasks.rados.rados.0.smithi063.stdout:3559: delete oid 5 current snap is 449 2024-08-17T17:09:50.810 INFO:tasks.rados.rados.0.smithi063.stdout:3556: finishing write tid 2 to smithi06387500-32 2024-08-17T17:09:50.810 INFO:tasks.rados.rados.0.smithi063.stdout:3556: finishing write tid 3 to smithi06387500-32 2024-08-17T17:09:50.810 INFO:tasks.rados.rados.0.smithi063.stdout:3556: finishing write tid 4 to smithi06387500-32 2024-08-17T17:09:50.810 INFO:tasks.rados.rados.0.smithi063.stdout:3556: oid 32 updating version 0 to 790 2024-08-17T17:09:50.810 INFO:tasks.rados.rados.0.smithi063.stdout:3556: oid 32 updating version 790 to 791 2024-08-17T17:09:50.810 INFO:tasks.rados.rados.0.smithi063.stdout:3556: oid 32 updating version 791 to 792 2024-08-17T17:09:50.810 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 792 (ObjNum 1329 snap 449 seq_num 1329) dirty exists 2024-08-17T17:09:50.810 INFO:tasks.rados.rados.0.smithi063.stdout:3556: left oid 32 (ObjNum 1329 snap 449 seq_num 1329) 2024-08-17T17:09:50.811 INFO:tasks.rados.rados.0.smithi063.stdout:3558: finishing write tid 1 to smithi06387500-45 2024-08-17T17:09:50.812 INFO:tasks.rados.rados.0.smithi063.stdout:3556: done (2 left) 2024-08-17T17:09:50.812 INFO:tasks.rados.rados.0.smithi063.stdout:3559: done (1 left) 2024-08-17T17:09:50.812 INFO:tasks.rados.rados.0.smithi063.stdout:3560: snap_create 2024-08-17T17:09:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3558: finishing write tid 2 to smithi06387500-45 2024-08-17T17:09:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3558: finishing write tid 3 to smithi06387500-45 2024-08-17T17:09:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3558: finishing write tid 4 to smithi06387500-45 2024-08-17T17:09:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3558: oid 45 updating version 0 to 892 2024-08-17T17:09:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3558: oid 45 version 892 is already newer than 891 2024-08-17T17:09:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3558: oid 45 version 892 is already newer than 890 2024-08-17T17:09:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 892 (ObjNum 1331 snap 449 seq_num 1331) dirty exists 2024-08-17T17:09:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3558: left oid 45 (ObjNum 1331 snap 449 seq_num 1331) 2024-08-17T17:09:50.993 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:50 smithi063 ceph-mon[109843]: osdmap e962: 8 total, 8 up, 8 in 2024-08-17T17:09:50.994 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:50 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:09:50.994 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:50 smithi063 ceph-mon[114756]: osdmap e962: 8 total, 8 up, 8 in 2024-08-17T17:09:50.994 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:50 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:09:51.023 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:50 smithi162 ceph-mon[104668]: osdmap e962: 8 total, 8 up, 8 in 2024-08-17T17:09:51.023 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:50 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:09:51.692 INFO:tasks.rados.rados.0.smithi063.stdout:3558: done (1 left) 2024-08-17T17:09:51.692 INFO:tasks.rados.rados.0.smithi063.stdout:3560: done (0 left) 2024-08-17T17:09:51.692 INFO:tasks.rados.rados.0.smithi063.stdout:3561: snap_create 2024-08-17T17:09:51.938 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:51 smithi063 ceph-mon[114756]: pgmap v900: 89 pgs: 89 active+clean; 231 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.1 MiB/s wr, 12 op/s 2024-08-17T17:09:51.938 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:51 smithi063 ceph-mon[114756]: osdmap e963: 8 total, 8 up, 8 in 2024-08-17T17:09:51.938 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:51 smithi063 ceph-mon[109843]: pgmap v900: 89 pgs: 89 active+clean; 231 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.1 MiB/s wr, 12 op/s 2024-08-17T17:09:51.938 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:51 smithi063 ceph-mon[109843]: osdmap e963: 8 total, 8 up, 8 in 2024-08-17T17:09:51.972 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:51 smithi162 ceph-mon[104668]: pgmap v900: 89 pgs: 89 active+clean; 231 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 1.1 MiB/s wr, 12 op/s 2024-08-17T17:09:51.972 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:51 smithi162 ceph-mon[104668]: osdmap e963: 8 total, 8 up, 8 in 2024-08-17T17:09:52.669 INFO:tasks.rados.rados.0.smithi063.stdout:3561: done (0 left) 2024-08-17T17:09:52.669 INFO:tasks.rados.rados.0.smithi063.stdout:3562: write oid 17 current snap is 451 2024-08-17T17:09:52.669 INFO:tasks.rados.rados.0.smithi063.stdout:3562: seq_num 1333 ranges {403731=668692,1587203=587797} 2024-08-17T17:09:52.675 INFO:tasks.rados.rados.0.smithi063.stdout:3562: writing smithi06387500-17 from 403731 to 1072423 tid 1 2024-08-17T17:09:52.682 INFO:tasks.rados.rados.0.smithi063.stdout:3562: writing smithi06387500-17 from 1587203 to 2175000 tid 2 2024-08-17T17:09:52.682 INFO:tasks.rados.rados.0.smithi063.stdout:3563: rollback oid 8 current snap is 451 2024-08-17T17:09:52.682 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 8 to 418 2024-08-17T17:09:52.682 INFO:tasks.rados.rados.0.smithi063.stdout:3562: finishing write tid 1 to smithi06387500-17 2024-08-17T17:09:52.682 INFO:tasks.rados.rados.0.smithi063.stdout:3564: write oid 34 current snap is 451 2024-08-17T17:09:52.682 INFO:tasks.rados.rados.0.smithi063.stdout:3564: seq_num 1334 ranges {420538=743941,1929332=491206} 2024-08-17T17:09:52.688 INFO:tasks.rados.rados.0.smithi063.stdout:3564: writing smithi06387500-34 from 420538 to 1164479 tid 1 2024-08-17T17:09:52.695 INFO:tasks.rados.rados.0.smithi063.stdout:3564: writing smithi06387500-34 from 1929332 to 2420538 tid 2 2024-08-17T17:09:52.696 INFO:tasks.rados.rados.0.smithi063.stdout:3565: delete oid 11 current snap is 451 2024-08-17T17:09:52.696 INFO:tasks.rados.rados.0.smithi063.stdout:3563: finishing rollback tid 0 to smithi06387500-8 2024-08-17T17:09:52.696 INFO:tasks.rados.rados.0.smithi063.stdout:3562: finishing write tid 2 to smithi06387500-17 2024-08-17T17:09:52.696 INFO:tasks.rados.rados.0.smithi063.stdout:3562: finishing write tid 3 to smithi06387500-17 2024-08-17T17:09:52.696 INFO:tasks.rados.rados.0.smithi063.stdout:3563: finishing rollback tid 1 to smithi06387500-8 2024-08-17T17:09:52.696 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 1199 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T17:09:52.696 INFO:tasks.rados.rados.0.smithi063.stdout:3562: finishing write tid 4 to smithi06387500-17 2024-08-17T17:09:52.696 INFO:tasks.rados.rados.0.smithi063.stdout:3562: oid 17 updating version 0 to 1201 2024-08-17T17:09:52.696 INFO:tasks.rados.rados.0.smithi063.stdout:3562: oid 17 version 1201 is already newer than 1200 2024-08-17T17:09:52.696 INFO:tasks.rados.rados.0.smithi063.stdout:3562: oid 17 version 1201 is already newer than 1199 2024-08-17T17:09:52.696 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 1201 (ObjNum 1333 snap 451 seq_num 1333) dirty exists 2024-08-17T17:09:52.697 INFO:tasks.rados.rados.0.smithi063.stdout:3562: left oid 17 (ObjNum 1333 snap 451 seq_num 1333) 2024-08-17T17:09:52.697 INFO:tasks.rados.rados.0.smithi063.stdout:3564: finishing write tid 1 to smithi06387500-34 2024-08-17T17:09:52.700 INFO:tasks.rados.rados.0.smithi063.stdout:3564: finishing write tid 2 to smithi06387500-34 2024-08-17T17:09:52.700 INFO:tasks.rados.rados.0.smithi063.stdout:3564: finishing write tid 3 to smithi06387500-34 2024-08-17T17:09:52.701 INFO:tasks.rados.rados.0.smithi063.stdout:3564: finishing write tid 4 to smithi06387500-34 2024-08-17T17:09:52.701 INFO:tasks.rados.rados.0.smithi063.stdout:3564: oid 34 updating version 0 to 2269 2024-08-17T17:09:52.701 INFO:tasks.rados.rados.0.smithi063.stdout:3564: oid 34 version 2269 is already newer than 2268 2024-08-17T17:09:52.701 INFO:tasks.rados.rados.0.smithi063.stdout:3564: oid 34 version 2269 is already newer than 2267 2024-08-17T17:09:52.702 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 2269 (ObjNum 1334 snap 451 seq_num 1334) dirty exists 2024-08-17T17:09:52.702 INFO:tasks.rados.rados.0.smithi063.stdout:3564: left oid 34 (ObjNum 1334 snap 451 seq_num 1334) 2024-08-17T17:09:52.702 INFO:tasks.rados.rados.0.smithi063.stdout:3562: done (3 left) 2024-08-17T17:09:52.702 INFO:tasks.rados.rados.0.smithi063.stdout:3563: done (2 left) 2024-08-17T17:09:52.702 INFO:tasks.rados.rados.0.smithi063.stdout:3564: done (1 left) 2024-08-17T17:09:52.702 INFO:tasks.rados.rados.0.smithi063.stdout:3565: done (0 left) 2024-08-17T17:09:52.702 INFO:tasks.rados.rados.0.smithi063.stdout:3566: read oid 38 snap -1 2024-08-17T17:09:52.702 INFO:tasks.rados.rados.0.smithi063.stdout:3566: expect (ObjNum 1277 snap 434 seq_num 1277) 2024-08-17T17:09:52.702 INFO:tasks.rados.rados.0.smithi063.stdout:3567: snap_remove snap 433 2024-08-17T17:09:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:52 smithi063 ceph-mon[114756]: osdmap e964: 8 total, 8 up, 8 in 2024-08-17T17:09:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:52 smithi063 ceph-mon[114756]: pgmap v903: 89 pgs: 89 active+clean; 236 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 17 op/s 2024-08-17T17:09:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:52 smithi063 ceph-mon[114756]: osdmap e965: 8 total, 8 up, 8 in 2024-08-17T17:09:53.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:52 smithi063 ceph-mon[109843]: osdmap e964: 8 total, 8 up, 8 in 2024-08-17T17:09:53.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:52 smithi063 ceph-mon[109843]: pgmap v903: 89 pgs: 89 active+clean; 236 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 17 op/s 2024-08-17T17:09:53.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:52 smithi063 ceph-mon[109843]: osdmap e965: 8 total, 8 up, 8 in 2024-08-17T17:09:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:52 smithi162 ceph-mon[104668]: osdmap e964: 8 total, 8 up, 8 in 2024-08-17T17:09:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:52 smithi162 ceph-mon[104668]: pgmap v903: 89 pgs: 89 active+clean; 236 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.1 MiB/s wr, 17 op/s 2024-08-17T17:09:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:52 smithi162 ceph-mon[104668]: osdmap e965: 8 total, 8 up, 8 in 2024-08-17T17:09:53.538 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:09:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:09:53] "GET /metrics HTTP/1.1" 200 36338 "" "Prometheus/2.51.0" 2024-08-17T17:09:53.706 INFO:tasks.rados.rados.0.smithi063.stdout:3567: done (1 left) 2024-08-17T17:09:53.706 INFO:tasks.rados.rados.0.smithi063.stdout:3568: delete oid 18 current snap is 451 2024-08-17T17:09:53.706 INFO:tasks.rados.rados.0.smithi063.stdout:3566: expect (ObjNum 1277 snap 434 seq_num 1277) 2024-08-17T17:09:53.893 INFO:tasks.rados.rados.0.smithi063.stdout:3566: done (1 left) 2024-08-17T17:09:53.893 INFO:tasks.rados.rados.0.smithi063.stdout:3568: done (0 left) 2024-08-17T17:09:53.893 INFO:tasks.rados.rados.0.smithi063.stdout:3569: read oid 21 snap -1 2024-08-17T17:09:53.893 INFO:tasks.rados.rados.0.smithi063.stdout:3569: expect (ObjNum 1229 snap 411 seq_num 1229) 2024-08-17T17:09:53.893 INFO:tasks.rados.rados.0.smithi063.stdout:3570: delete oid 36 current snap is 451 2024-08-17T17:09:53.895 INFO:tasks.rados.rados.0.smithi063.stdout:3570: done (1 left) 2024-08-17T17:09:53.896 INFO:tasks.rados.rados.0.smithi063.stdout:3571: read oid 31 snap -1 2024-08-17T17:09:53.896 INFO:tasks.rados.rados.0.smithi063.stdout:3571: expect deleted 2024-08-17T17:09:53.896 INFO:tasks.rados.rados.0.smithi063.stdout:3572: write oid 44 current snap is 451 2024-08-17T17:09:53.896 INFO:tasks.rados.rados.0.smithi063.stdout:3572: seq_num 1338 ranges {487766=644937,1646907=738793} 2024-08-17T17:09:53.902 INFO:tasks.rados.rados.0.smithi063.stdout:3572: writing smithi06387500-44 from 487766 to 1132703 tid 1 2024-08-17T17:09:53.910 INFO:tasks.rados.rados.0.smithi063.stdout:3572: writing smithi06387500-44 from 1646907 to 2385700 tid 2 2024-08-17T17:09:53.910 INFO:tasks.rados.rados.0.smithi063.stdout:3573: read oid 24 snap -1 2024-08-17T17:09:53.910 INFO:tasks.rados.rados.0.smithi063.stdout:3573: expect (ObjNum 1281 snap 435 seq_num 1281) 2024-08-17T17:09:53.910 INFO:tasks.rados.rados.0.smithi063.stdout:3574: read oid 17 snap -1 2024-08-17T17:09:53.910 INFO:tasks.rados.rados.0.smithi063.stdout:3574: expect (ObjNum 1333 snap 451 seq_num 1333) 2024-08-17T17:09:53.910 INFO:tasks.rados.rados.0.smithi063.stdout:3575: read oid 32 snap -1 2024-08-17T17:09:53.910 INFO:tasks.rados.rados.0.smithi063.stdout:3575: expect (ObjNum 1329 snap 449 seq_num 1329) 2024-08-17T17:09:53.910 INFO:tasks.rados.rados.0.smithi063.stdout:3576: read oid 9 snap -1 2024-08-17T17:09:53.910 INFO:tasks.rados.rados.0.smithi063.stdout:3576: expect (ObjNum 1298 snap 442 seq_num 1298) 2024-08-17T17:09:53.911 INFO:tasks.rados.rados.0.smithi063.stdout:3577: rollback oid 47 current snap is 451 2024-08-17T17:09:53.911 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 47 to 422 2024-08-17T17:09:53.911 INFO:tasks.rados.rados.0.smithi063.stdout:3569: expect (ObjNum 1229 snap 411 seq_num 1229) 2024-08-17T17:09:54.088 INFO:tasks.rados.rados.0.smithi063.stdout:3569: done (7 left) 2024-08-17T17:09:54.088 INFO:tasks.rados.rados.0.smithi063.stdout:3571: done (6 left) 2024-08-17T17:09:54.088 INFO:tasks.rados.rados.0.smithi063.stdout:3578: read oid 31 snap -1 2024-08-17T17:09:54.088 INFO:tasks.rados.rados.0.smithi063.stdout:3578: expect deleted 2024-08-17T17:09:54.088 INFO:tasks.rados.rados.0.smithi063.stdout:3579: snap_remove snap 449 2024-08-17T17:09:54.759 INFO:tasks.rados.rados.0.smithi063.stdout:3579: done (7 left) 2024-08-17T17:09:54.759 INFO:tasks.rados.rados.0.smithi063.stdout:3580: write oid 3 current snap is 451 2024-08-17T17:09:54.759 INFO:tasks.rados.rados.0.smithi063.stdout:3580: seq_num 1339 ranges {504573=436539,1421742=598804,2437505=67068} 2024-08-17T17:09:54.764 INFO:tasks.rados.rados.0.smithi063.stdout:3580: writing smithi06387500-3 from 504573 to 941112 tid 1 2024-08-17T17:09:54.770 INFO:tasks.rados.rados.0.smithi063.stdout:3580: writing smithi06387500-3 from 1421742 to 2020546 tid 2 2024-08-17T17:09:54.773 INFO:tasks.rados.rados.0.smithi063.stdout:3580: writing smithi06387500-3 from 2437505 to 2504573 tid 3 2024-08-17T17:09:54.773 INFO:tasks.rados.rados.0.smithi063.stdout:3581: rollback oid 12 current snap is 451 2024-08-17T17:09:54.773 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 12 to 441 2024-08-17T17:09:54.773 INFO:tasks.rados.rados.0.smithi063.stdout:3572: finishing write tid 1 to smithi06387500-44 2024-08-17T17:09:54.773 INFO:tasks.rados.rados.0.smithi063.stdout:3572: finishing write tid 2 to smithi06387500-44 2024-08-17T17:09:54.773 INFO:tasks.rados.rados.0.smithi063.stdout:3572: finishing write tid 3 to smithi06387500-44 2024-08-17T17:09:54.773 INFO:tasks.rados.rados.0.smithi063.stdout:3582: snap_remove snap 426 2024-08-17T17:09:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:54 smithi063 ceph-mon[114756]: osdmap e966: 8 total, 8 up, 8 in 2024-08-17T17:09:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:54 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:09:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:54 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:09:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:54 smithi063 ceph-mon[114756]: pgmap v906: 89 pgs: 89 active+clean; 240 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.8 MiB/s wr, 20 op/s 2024-08-17T17:09:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:54 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:09:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:54 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:09:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:54 smithi063 ceph-mon[109843]: osdmap e966: 8 total, 8 up, 8 in 2024-08-17T17:09:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:54 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:09:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:54 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:09:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:54 smithi063 ceph-mon[109843]: pgmap v906: 89 pgs: 89 active+clean; 240 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.8 MiB/s wr, 20 op/s 2024-08-17T17:09:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:54 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:09:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:54 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:09:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:54 smithi162 ceph-mon[104668]: osdmap e966: 8 total, 8 up, 8 in 2024-08-17T17:09:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:54 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:09:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:54 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:09:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:54 smithi162 ceph-mon[104668]: pgmap v906: 89 pgs: 89 active+clean; 240 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.8 MiB/s wr, 20 op/s 2024-08-17T17:09:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:54 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:09:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:54 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:09:55.765 INFO:tasks.rados.rados.0.smithi063.stdout:3582: done (9 left) 2024-08-17T17:09:55.766 INFO:tasks.rados.rados.0.smithi063.stdout:3583: write oid 23 current snap is 451 2024-08-17T17:09:55.766 INFO:tasks.rados.rados.0.smithi063.stdout:3583: seq_num 1340 ranges {521380=511788,1480224=742462} 2024-08-17T17:09:55.771 INFO:tasks.rados.rados.0.smithi063.stdout:3583: writing smithi06387500-23 from 521380 to 1033168 tid 1 2024-08-17T17:09:55.779 INFO:tasks.rados.rados.0.smithi063.stdout:3583: writing smithi06387500-23 from 1480224 to 2222686 tid 2 2024-08-17T17:09:55.779 INFO:tasks.rados.rados.0.smithi063.stdout:3584: snap_remove snap 431 2024-08-17T17:09:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:55 smithi063 ceph-mon[114756]: osdmap e967: 8 total, 8 up, 8 in 2024-08-17T17:09:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:55 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:09:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:55 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:09:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:55 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:09:56.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:55 smithi063 ceph-mon[109843]: osdmap e967: 8 total, 8 up, 8 in 2024-08-17T17:09:56.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:55 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:09:56.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:55 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:09:56.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:55 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:09:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:55 smithi162 ceph-mon[104668]: osdmap e967: 8 total, 8 up, 8 in 2024-08-17T17:09:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:55 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:09:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:55 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:09:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:55 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:09:56.774 INFO:tasks.rados.rados.0.smithi063.stdout:3584: done (10 left) 2024-08-17T17:09:56.774 INFO:tasks.rados.rados.0.smithi063.stdout:3585: write oid 21 current snap is 451 2024-08-17T17:09:56.775 INFO:tasks.rados.rados.0.smithi063.stdout:3572: finishing write tid 4 to smithi06387500-44 2024-08-17T17:09:56.775 INFO:tasks.rados.rados.0.smithi063.stdout:3572: oid 44 updating version 0 to 485 2024-08-17T17:09:56.775 INFO:tasks.rados.rados.0.smithi063.stdout:3572: oid 44 updating version 485 to 486 2024-08-17T17:09:56.775 INFO:tasks.rados.rados.0.smithi063.stdout:3572: oid 44 version 486 is already newer than 484 2024-08-17T17:09:56.775 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 486 (ObjNum 1338 snap 451 seq_num 1338) dirty exists 2024-08-17T17:09:56.775 INFO:tasks.rados.rados.0.smithi063.stdout:3572: left oid 44 (ObjNum 1338 snap 451 seq_num 1338) 2024-08-17T17:09:56.775 INFO:tasks.rados.rados.0.smithi063.stdout:3575: expect (ObjNum 1329 snap 449 seq_num 1329) 2024-08-17T17:09:56.933 INFO:tasks.rados.rados.0.smithi063.stdout:3577: finishing rollback tid 1 to smithi06387500-47 2024-08-17T17:09:56.934 INFO:tasks.rados.rados.0.smithi063.stdout:3577: finishing rollback tid 2 to smithi06387500-47 2024-08-17T17:09:56.934 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 796 (ObjNum 1230 snap 411 seq_num 1230) dirty exists 2024-08-17T17:09:56.934 INFO:tasks.rados.rados.0.smithi063.stdout:3574: expect (ObjNum 1333 snap 451 seq_num 1333) 2024-08-17T17:09:57.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:56 smithi063 ceph-mon[114756]: osdmap e968: 8 total, 8 up, 8 in 2024-08-17T17:09:57.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:56 smithi063 ceph-mon[114756]: pgmap v909: 89 pgs: 89 active+clean; 241 MiB data, 872 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 993 KiB/s wr, 14 op/s 2024-08-17T17:09:57.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:56 smithi063 ceph-mon[109843]: osdmap e968: 8 total, 8 up, 8 in 2024-08-17T17:09:57.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:56 smithi063 ceph-mon[109843]: pgmap v909: 89 pgs: 89 active+clean; 241 MiB data, 872 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 993 KiB/s wr, 14 op/s 2024-08-17T17:09:57.062 INFO:tasks.rados.rados.0.smithi063.stdout:3573: expect (ObjNum 1281 snap 435 seq_num 1281) 2024-08-17T17:09:57.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:56 smithi162 ceph-mon[104668]: osdmap e968: 8 total, 8 up, 8 in 2024-08-17T17:09:57.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:56 smithi162 ceph-mon[104668]: pgmap v909: 89 pgs: 89 active+clean; 241 MiB data, 872 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 993 KiB/s wr, 14 op/s 2024-08-17T17:09:57.209 INFO:tasks.rados.rados.0.smithi063.stdout:3576: expect (ObjNum 1298 snap 442 seq_num 1298) 2024-08-17T17:09:57.337 INFO:tasks.rados.rados.0.smithi063.stdout:3580: finishing write tid 1 to smithi06387500-3 2024-08-17T17:09:57.337 INFO:tasks.rados.rados.0.smithi063.stdout:3580: finishing write tid 2 to smithi06387500-3 2024-08-17T17:09:57.338 INFO:tasks.rados.rados.0.smithi063.stdout:3580: finishing write tid 3 to smithi06387500-3 2024-08-17T17:09:57.338 INFO:tasks.rados.rados.0.smithi063.stdout:3580: finishing write tid 4 to smithi06387500-3 2024-08-17T17:09:57.338 INFO:tasks.rados.rados.0.smithi063.stdout:3580: finishing write tid 5 to smithi06387500-3 2024-08-17T17:09:57.338 INFO:tasks.rados.rados.0.smithi063.stdout:3580: oid 3 updating version 0 to 1212 2024-08-17T17:09:57.338 INFO:tasks.rados.rados.0.smithi063.stdout:3580: oid 3 version 1212 is already newer than 1210 2024-08-17T17:09:57.338 INFO:tasks.rados.rados.0.smithi063.stdout:3580: oid 3 version 1212 is already newer than 1209 2024-08-17T17:09:57.338 INFO:tasks.rados.rados.0.smithi063.stdout:3580: oid 3 version 1212 is already newer than 1211 2024-08-17T17:09:57.338 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 1212 (ObjNum 1339 snap 451 seq_num 1339) dirty exists 2024-08-17T17:09:57.338 INFO:tasks.rados.rados.0.smithi063.stdout:3580: left oid 3 (ObjNum 1339 snap 451 seq_num 1339) 2024-08-17T17:09:57.338 INFO:tasks.rados.rados.0.smithi063.stdout:3581: finishing rollback tid 1 to smithi06387500-12 2024-08-17T17:09:57.338 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 757 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T17:09:57.338 INFO:tasks.rados.rados.0.smithi063.stdout:3583: finishing write tid 1 to smithi06387500-23 2024-08-17T17:09:57.339 INFO:tasks.rados.rados.0.smithi063.stdout:3583: finishing write tid 2 to smithi06387500-23 2024-08-17T17:09:57.339 INFO:tasks.rados.rados.0.smithi063.stdout:3583: finishing write tid 3 to smithi06387500-23 2024-08-17T17:09:57.339 INFO:tasks.rados.rados.0.smithi063.stdout:3583: finishing write tid 4 to smithi06387500-23 2024-08-17T17:09:57.339 INFO:tasks.rados.rados.0.smithi063.stdout:3583: oid 23 updating version 0 to 1866 2024-08-17T17:09:57.339 INFO:tasks.rados.rados.0.smithi063.stdout:3583: oid 23 version 1866 is already newer than 1865 2024-08-17T17:09:57.339 INFO:tasks.rados.rados.0.smithi063.stdout:3583: oid 23 updating version 1866 to 1867 2024-08-17T17:09:57.339 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 1867 (ObjNum 1340 snap 451 seq_num 1340) dirty exists 2024-08-17T17:09:57.339 INFO:tasks.rados.rados.0.smithi063.stdout:3583: left oid 23 (ObjNum 1340 snap 451 seq_num 1340) 2024-08-17T17:09:57.339 INFO:tasks.rados.rados.0.smithi063.stdout:3585: seq_num 1341 ranges {538187=587037,1538706=602473} 2024-08-17T17:09:57.343 INFO:tasks.rados.rados.0.smithi063.stdout:3585: writing smithi06387500-21 from 538187 to 1125224 tid 1 2024-08-17T17:09:57.349 INFO:tasks.rados.rados.0.smithi063.stdout:3585: writing smithi06387500-21 from 1538706 to 2141179 tid 2 2024-08-17T17:09:57.350 INFO:tasks.rados.rados.0.smithi063.stdout:3572: done (10 left) 2024-08-17T17:09:57.350 INFO:tasks.rados.rados.0.smithi063.stdout:3573: done (9 left) 2024-08-17T17:09:57.350 INFO:tasks.rados.rados.0.smithi063.stdout:3574: done (8 left) 2024-08-17T17:09:57.350 INFO:tasks.rados.rados.0.smithi063.stdout:3575: done (7 left) 2024-08-17T17:09:57.350 INFO:tasks.rados.rados.0.smithi063.stdout:3576: done (6 left) 2024-08-17T17:09:57.350 INFO:tasks.rados.rados.0.smithi063.stdout:3577: done (5 left) 2024-08-17T17:09:57.350 INFO:tasks.rados.rados.0.smithi063.stdout:3578: done (4 left) 2024-08-17T17:09:57.350 INFO:tasks.rados.rados.0.smithi063.stdout:3580: done (3 left) 2024-08-17T17:09:57.350 INFO:tasks.rados.rados.0.smithi063.stdout:3581: done (2 left) 2024-08-17T17:09:57.350 INFO:tasks.rados.rados.0.smithi063.stdout:3583: done (1 left) 2024-08-17T17:09:57.350 INFO:tasks.rados.rados.0.smithi063.stdout:3586: read oid 10 snap -1 2024-08-17T17:09:57.350 INFO:tasks.rados.rados.0.smithi063.stdout:3586: expect (ObjNum 1254 snap 423 seq_num 1254) 2024-08-17T17:09:57.350 INFO:tasks.rados.rados.0.smithi063.stdout:3587: write oid 39 current snap is 451 2024-08-17T17:09:57.351 INFO:tasks.rados.rados.0.smithi063.stdout:3587: seq_num 1342 ranges {554994=662286,1997188=557806} 2024-08-17T17:09:57.356 INFO:tasks.rados.rados.0.smithi063.stdout:3587: writing smithi06387500-39 from 554994 to 1217280 tid 1 2024-08-17T17:09:57.363 INFO:tasks.rados.rados.0.smithi063.stdout:3587: writing smithi06387500-39 from 1997188 to 2554994 tid 2 2024-08-17T17:09:57.364 INFO:tasks.rados.rados.0.smithi063.stdout:3588: write oid 28 current snap is 451 2024-08-17T17:09:57.364 INFO:tasks.rados.rados.0.smithi063.stdout:3585: finishing write tid 1 to smithi06387500-21 2024-08-17T17:09:57.364 INFO:tasks.rados.rados.0.smithi063.stdout:3588: seq_num 1343 ranges {571801=737535,1939317=489789} 2024-08-17T17:09:57.370 INFO:tasks.rados.rados.0.smithi063.stdout:3588: writing smithi06387500-28 from 571801 to 1309336 tid 1 2024-08-17T17:09:57.377 INFO:tasks.rados.rados.0.smithi063.stdout:3588: writing smithi06387500-28 from 1939317 to 2429106 tid 2 2024-08-17T17:09:57.377 INFO:tasks.rados.rados.0.smithi063.stdout:3589: delete oid 8 current snap is 451 2024-08-17T17:09:57.377 INFO:tasks.rados.rados.0.smithi063.stdout:3586: expect (ObjNum 1254 snap 423 seq_num 1254) 2024-08-17T17:09:57.481 INFO:tasks.rados.rados.0.smithi063.stdout:3585: finishing write tid 2 to smithi06387500-21 2024-08-17T17:09:57.481 INFO:tasks.rados.rados.0.smithi063.stdout:3585: finishing write tid 3 to smithi06387500-21 2024-08-17T17:09:57.481 INFO:tasks.rados.rados.0.smithi063.stdout:3585: finishing write tid 4 to smithi06387500-21 2024-08-17T17:09:57.481 INFO:tasks.rados.rados.0.smithi063.stdout:3585: oid 21 updating version 0 to 2285 2024-08-17T17:09:57.482 INFO:tasks.rados.rados.0.smithi063.stdout:3585: oid 21 updating version 2285 to 2287 2024-08-17T17:09:57.482 INFO:tasks.rados.rados.0.smithi063.stdout:3585: oid 21 version 2287 is already newer than 2286 2024-08-17T17:09:57.482 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 2287 (ObjNum 1341 snap 451 seq_num 1341) dirty exists 2024-08-17T17:09:57.482 INFO:tasks.rados.rados.0.smithi063.stdout:3585: left oid 21 (ObjNum 1341 snap 451 seq_num 1341) 2024-08-17T17:09:57.482 INFO:tasks.rados.rados.0.smithi063.stdout:3587: finishing write tid 1 to smithi06387500-39 2024-08-17T17:09:57.482 INFO:tasks.rados.rados.0.smithi063.stdout:3585: done (4 left) 2024-08-17T17:09:57.482 INFO:tasks.rados.rados.0.smithi063.stdout:3586: done (3 left) 2024-08-17T17:09:57.482 INFO:tasks.rados.rados.0.smithi063.stdout:3589: done (2 left) 2024-08-17T17:09:57.482 INFO:tasks.rados.rados.0.smithi063.stdout:3590: snap_create 2024-08-17T17:09:57.482 INFO:tasks.rados.rados.0.smithi063.stdout:3587: finishing write tid 2 to smithi06387500-39 2024-08-17T17:09:57.482 INFO:tasks.rados.rados.0.smithi063.stdout:3587: finishing write tid 3 to smithi06387500-39 2024-08-17T17:09:57.482 INFO:tasks.rados.rados.0.smithi063.stdout:3587: finishing write tid 4 to smithi06387500-39 2024-08-17T17:09:57.483 INFO:tasks.rados.rados.0.smithi063.stdout:3587: oid 39 updating version 0 to 904 2024-08-17T17:09:57.483 INFO:tasks.rados.rados.0.smithi063.stdout:3587: oid 39 updating version 904 to 905 2024-08-17T17:09:57.483 INFO:tasks.rados.rados.0.smithi063.stdout:3587: oid 39 version 905 is already newer than 903 2024-08-17T17:09:57.483 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 905 (ObjNum 1342 snap 451 seq_num 1342) dirty exists 2024-08-17T17:09:57.483 INFO:tasks.rados.rados.0.smithi063.stdout:3587: left oid 39 (ObjNum 1342 snap 451 seq_num 1342) 2024-08-17T17:09:57.483 INFO:tasks.rados.rados.0.smithi063.stdout:3588: finishing write tid 1 to smithi06387500-28 2024-08-17T17:09:57.483 INFO:tasks.rados.rados.0.smithi063.stdout:3588: finishing write tid 2 to smithi06387500-28 2024-08-17T17:09:57.483 INFO:tasks.rados.rados.0.smithi063.stdout:3588: finishing write tid 3 to smithi06387500-28 2024-08-17T17:09:57.483 INFO:tasks.rados.rados.0.smithi063.stdout:3588: finishing write tid 4 to smithi06387500-28 2024-08-17T17:09:57.483 INFO:tasks.rados.rados.0.smithi063.stdout:3588: oid 28 updating version 0 to 855 2024-08-17T17:09:57.483 INFO:tasks.rados.rados.0.smithi063.stdout:3588: oid 28 updating version 855 to 856 2024-08-17T17:09:57.483 INFO:tasks.rados.rados.0.smithi063.stdout:3588: oid 28 updating version 856 to 857 2024-08-17T17:09:57.484 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 857 (ObjNum 1343 snap 451 seq_num 1343) dirty exists 2024-08-17T17:09:57.484 INFO:tasks.rados.rados.0.smithi063.stdout:3588: left oid 28 (ObjNum 1343 snap 451 seq_num 1343) 2024-08-17T17:09:57.667 INFO:tasks.rados.rados.0.smithi063.stdout:3587: done (2 left) 2024-08-17T17:09:57.667 INFO:tasks.rados.rados.0.smithi063.stdout:3588: done (1 left) 2024-08-17T17:09:57.667 INFO:tasks.rados.rados.0.smithi063.stdout:3590: done (0 left) 2024-08-17T17:09:57.667 INFO:tasks.rados.rados.0.smithi063.stdout:3591: read oid 9 snap -1 2024-08-17T17:09:57.667 INFO:tasks.rados.rados.0.smithi063.stdout:3591: expect (ObjNum 1298 snap 442 seq_num 1298) 2024-08-17T17:09:57.667 INFO:tasks.rados.rados.0.smithi063.stdout:3592: snap_create 2024-08-17T17:09:57.680 INFO:tasks.rados.rados.0.smithi063.stdout:3591: expect (ObjNum 1298 snap 442 seq_num 1298) 2024-08-17T17:09:58.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:57 smithi063 ceph-mon[114756]: osdmap e969: 8 total, 8 up, 8 in 2024-08-17T17:09:58.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:57 smithi063 ceph-mon[114756]: osdmap e970: 8 total, 8 up, 8 in 2024-08-17T17:09:58.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:57 smithi063 ceph-mon[109843]: osdmap e969: 8 total, 8 up, 8 in 2024-08-17T17:09:58.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:57 smithi063 ceph-mon[109843]: osdmap e970: 8 total, 8 up, 8 in 2024-08-17T17:09:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:57 smithi162 ceph-mon[104668]: osdmap e969: 8 total, 8 up, 8 in 2024-08-17T17:09:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:57 smithi162 ceph-mon[104668]: osdmap e970: 8 total, 8 up, 8 in 2024-08-17T17:09:58.669 INFO:tasks.rados.rados.0.smithi063.stdout:3591: done (1 left) 2024-08-17T17:09:58.669 INFO:tasks.rados.rados.0.smithi063.stdout:3592: done (0 left) 2024-08-17T17:09:58.669 INFO:tasks.rados.rados.0.smithi063.stdout:3593: snap_create 2024-08-17T17:09:59.670 INFO:tasks.rados.rados.0.smithi063.stdout:3593: done (0 left) 2024-08-17T17:09:59.670 INFO:tasks.rados.rados.0.smithi063.stdout:3594: snap_remove snap 422 2024-08-17T17:10:00.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:59 smithi063 ceph-mon[109843]: pgmap v912: 89 pgs: 89 active+clean; 247 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2024-08-17T17:10:00.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:09:59 smithi063 ceph-mon[109843]: osdmap e971: 8 total, 8 up, 8 in 2024-08-17T17:10:00.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:59 smithi063 ceph-mon[114756]: pgmap v912: 89 pgs: 89 active+clean; 247 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2024-08-17T17:10:00.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:09:59 smithi063 ceph-mon[114756]: osdmap e971: 8 total, 8 up, 8 in 2024-08-17T17:10:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:59 smithi162 ceph-mon[104668]: pgmap v912: 89 pgs: 89 active+clean; 247 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 1.5 MiB/s wr, 18 op/s 2024-08-17T17:10:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:09:59 smithi162 ceph-mon[104668]: osdmap e971: 8 total, 8 up, 8 in 2024-08-17T17:10:00.682 INFO:tasks.rados.rados.0.smithi063.stdout:3594: done (0 left) 2024-08-17T17:10:00.683 INFO:tasks.rados.rados.0.smithi063.stdout:3595: write oid 1 current snap is 454 2024-08-17T17:10:00.683 INFO:tasks.rados.rados.0.smithi063.stdout:3595: seq_num 1345 ranges {605415=488033,1656281=493458} 2024-08-17T17:10:00.689 INFO:tasks.rados.rados.0.smithi063.stdout:3595: writing smithi06387500-1 from 605415 to 1093448 tid 1 2024-08-17T17:10:00.696 INFO:tasks.rados.rados.0.smithi063.stdout:3595: writing smithi06387500-1 from 1656281 to 2149739 tid 2 2024-08-17T17:10:00.697 INFO:tasks.rados.rados.0.smithi063.stdout:3596: read oid 2 snap -1 2024-08-17T17:10:00.697 INFO:tasks.rados.rados.0.smithi063.stdout:3596: expect deleted 2024-08-17T17:10:00.697 INFO:tasks.rados.rados.0.smithi063.stdout:3597: write oid 13 current snap is 454 2024-08-17T17:10:00.698 INFO:tasks.rados.rados.0.smithi063.stdout:3597: seq_num 1346 ranges {622222=679635,1831116=753469} 2024-08-17T17:10:00.704 INFO:tasks.rados.rados.0.smithi063.stdout:3597: writing smithi06387500-13 from 622222 to 1301857 tid 1 2024-08-17T17:10:00.712 INFO:tasks.rados.rados.0.smithi063.stdout:3597: writing smithi06387500-13 from 1831116 to 2584585 tid 2 2024-08-17T17:10:00.712 INFO:tasks.rados.rados.0.smithi063.stdout:3598: delete oid 34 current snap is 454 2024-08-17T17:10:00.712 INFO:tasks.rados.rados.0.smithi063.stdout:3595: finishing write tid 1 to smithi06387500-1 2024-08-17T17:10:00.712 INFO:tasks.rados.rados.0.smithi063.stdout:3595: finishing write tid 2 to smithi06387500-1 2024-08-17T17:10:00.712 INFO:tasks.rados.rados.0.smithi063.stdout:3595: finishing write tid 3 to smithi06387500-1 2024-08-17T17:10:00.712 INFO:tasks.rados.rados.0.smithi063.stdout:3595: finishing write tid 4 to smithi06387500-1 2024-08-17T17:10:00.712 INFO:tasks.rados.rados.0.smithi063.stdout:3595: oid 1 updating version 0 to 1883 2024-08-17T17:10:00.712 INFO:tasks.rados.rados.0.smithi063.stdout:3595: oid 1 updating version 1883 to 1884 2024-08-17T17:10:00.712 INFO:tasks.rados.rados.0.smithi063.stdout:3595: oid 1 updating version 1884 to 1885 2024-08-17T17:10:00.712 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 1885 (ObjNum 1345 snap 454 seq_num 1345) dirty exists 2024-08-17T17:10:00.712 INFO:tasks.rados.rados.0.smithi063.stdout:3595: left oid 1 (ObjNum 1345 snap 454 seq_num 1345) 2024-08-17T17:10:00.713 INFO:tasks.rados.rados.0.smithi063.stdout:3597: finishing write tid 1 to smithi06387500-13 2024-08-17T17:10:00.714 INFO:tasks.rados.rados.0.smithi063.stdout:3595: done (3 left) 2024-08-17T17:10:00.714 INFO:tasks.rados.rados.0.smithi063.stdout:3596: done (2 left) 2024-08-17T17:10:00.715 INFO:tasks.rados.rados.0.smithi063.stdout:3598: done (1 left) 2024-08-17T17:10:00.715 INFO:tasks.rados.rados.0.smithi063.stdout:3599: snap_remove snap 418 2024-08-17T17:10:00.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:00 smithi162 ceph-mon[104668]: osdmap e972: 8 total, 8 up, 8 in 2024-08-17T17:10:00.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:00 smithi162 ceph-mon[104668]: overall HEALTH_OK 2024-08-17T17:10:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:00 smithi063 ceph-mon[109843]: osdmap e972: 8 total, 8 up, 8 in 2024-08-17T17:10:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:00 smithi063 ceph-mon[109843]: overall HEALTH_OK 2024-08-17T17:10:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:00 smithi063 ceph-mon[114756]: osdmap e972: 8 total, 8 up, 8 in 2024-08-17T17:10:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:00 smithi063 ceph-mon[114756]: overall HEALTH_OK 2024-08-17T17:10:01.698 INFO:tasks.rados.rados.0.smithi063.stdout:3599: done (1 left) 2024-08-17T17:10:01.698 INFO:tasks.rados.rados.0.smithi063.stdout:3600: rollback oid 41 current snap is 454 2024-08-17T17:10:01.698 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 41 to 446 2024-08-17T17:10:01.698 INFO:tasks.rados.rados.0.smithi063.stdout:3597: finishing write tid 2 to smithi06387500-13 2024-08-17T17:10:01.698 INFO:tasks.rados.rados.0.smithi063.stdout:3597: finishing write tid 3 to smithi06387500-13 2024-08-17T17:10:01.698 INFO:tasks.rados.rados.0.smithi063.stdout:3597: finishing write tid 4 to smithi06387500-13 2024-08-17T17:10:01.698 INFO:tasks.rados.rados.0.smithi063.stdout:3597: oid 13 updating version 0 to 1888 2024-08-17T17:10:01.698 INFO:tasks.rados.rados.0.smithi063.stdout:3597: oid 13 updating version 1888 to 1889 2024-08-17T17:10:01.698 INFO:tasks.rados.rados.0.smithi063.stdout:3597: oid 13 version 1889 is already newer than 1887 2024-08-17T17:10:01.698 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1889 (ObjNum 1346 snap 454 seq_num 1346) dirty exists 2024-08-17T17:10:01.699 INFO:tasks.rados.rados.0.smithi063.stdout:3597: left oid 13 (ObjNum 1346 snap 454 seq_num 1346) 2024-08-17T17:10:01.699 INFO:tasks.rados.rados.0.smithi063.stdout:3597: done (1 left) 2024-08-17T17:10:01.699 INFO:tasks.rados.rados.0.smithi063.stdout:3601: write oid 16 current snap is 454 2024-08-17T17:10:01.699 INFO:tasks.rados.rados.0.smithi063.stdout:3601: seq_num 1348 ranges {655836=430133,1831727=757138} 2024-08-17T17:10:01.704 INFO:tasks.rados.rados.0.smithi063.stdout:3601: writing smithi06387500-16 from 655836 to 1085969 tid 1 2024-08-17T17:10:01.714 INFO:tasks.rados.rados.0.smithi063.stdout:3601: writing smithi06387500-16 from 1831727 to 2588865 tid 2 2024-08-17T17:10:01.714 INFO:tasks.rados.rados.0.smithi063.stdout:3602: delete oid 22 current snap is 454 2024-08-17T17:10:01.714 INFO:tasks.rados.rados.0.smithi063.stdout:3600: finishing rollback tid 0 to smithi06387500-41 2024-08-17T17:10:01.715 INFO:tasks.rados.rados.0.smithi063.stdout:3600: finishing rollback tid 1 to smithi06387500-41 2024-08-17T17:10:01.715 INFO:tasks.rados.rados.0.smithi063.stdout:3600: finishing rollback tid 2 to smithi06387500-41 2024-08-17T17:10:01.715 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 410 (ObjNum 1307 snap 443 seq_num 1307) dirty exists 2024-08-17T17:10:01.715 INFO:tasks.rados.rados.0.smithi063.stdout:3601: finishing write tid 1 to smithi06387500-16 2024-08-17T17:10:01.717 INFO:tasks.rados.rados.0.smithi063.stdout:3600: done (2 left) 2024-08-17T17:10:01.717 INFO:tasks.rados.rados.0.smithi063.stdout:3602: done (1 left) 2024-08-17T17:10:01.717 INFO:tasks.rados.rados.0.smithi063.stdout:3603: read oid 19 snap 441 2024-08-17T17:10:01.717 INFO:tasks.rados.rados.0.smithi063.stdout:3603: expect (ObjNum 1262 snap 426 seq_num 1262) 2024-08-17T17:10:01.717 INFO:tasks.rados.rados.0.smithi063.stdout:3604: snap_remove snap 434 2024-08-17T17:10:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:01 smithi063 ceph-mon[114756]: pgmap v915: 89 pgs: 89 active+clean; 250 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-08-17T17:10:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:01 smithi063 ceph-mon[114756]: osdmap e973: 8 total, 8 up, 8 in 2024-08-17T17:10:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:01 smithi063 ceph-mon[109843]: pgmap v915: 89 pgs: 89 active+clean; 250 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-08-17T17:10:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:01 smithi063 ceph-mon[109843]: osdmap e973: 8 total, 8 up, 8 in 2024-08-17T17:10:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:01 smithi162 ceph-mon[104668]: pgmap v915: 89 pgs: 89 active+clean; 250 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-08-17T17:10:02.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:01 smithi162 ceph-mon[104668]: osdmap e973: 8 total, 8 up, 8 in 2024-08-17T17:10:02.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:02.671 INFO:tasks.rados.rados.0.smithi063.stdout:3601: finishing write tid 2 to smithi06387500-16 2024-08-17T17:10:02.671 INFO:tasks.rados.rados.0.smithi063.stdout:3601: finishing write tid 3 to smithi06387500-16 2024-08-17T17:10:02.671 INFO:tasks.rados.rados.0.smithi063.stdout:3601: finishing write tid 4 to smithi06387500-16 2024-08-17T17:10:02.672 INFO:tasks.rados.rados.0.smithi063.stdout:3601: oid 16 updating version 0 to 1310 2024-08-17T17:10:02.672 INFO:tasks.rados.rados.0.smithi063.stdout:3601: oid 16 updating version 1310 to 1312 2024-08-17T17:10:02.672 INFO:tasks.rados.rados.0.smithi063.stdout:3601: oid 16 version 1312 is already newer than 1311 2024-08-17T17:10:02.672 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 1312 (ObjNum 1348 snap 454 seq_num 1348) dirty exists 2024-08-17T17:10:02.672 INFO:tasks.rados.rados.0.smithi063.stdout:3601: left oid 16 (ObjNum 1348 snap 454 seq_num 1348) 2024-08-17T17:10:02.672 INFO:tasks.rados.rados.0.smithi063.stdout:3603: expect (ObjNum 1262 snap 426 seq_num 1262) 2024-08-17T17:10:02.811 INFO:tasks.rados.rados.0.smithi063.stdout:3601: done (2 left) 2024-08-17T17:10:02.812 INFO:tasks.rados.rados.0.smithi063.stdout:3603: done (1 left) 2024-08-17T17:10:02.812 INFO:tasks.rados.rados.0.smithi063.stdout:3604: done (0 left) 2024-08-17T17:10:02.812 INFO:tasks.rados.rados.0.smithi063.stdout:3605: read oid 33 snap -1 2024-08-17T17:10:02.812 INFO:tasks.rados.rados.0.smithi063.stdout:3605: expect (ObjNum 1323 snap 449 seq_num 1323) 2024-08-17T17:10:02.812 INFO:tasks.rados.rados.0.smithi063.stdout:3606: read oid 36 snap -1 2024-08-17T17:10:02.812 INFO:tasks.rados.rados.0.smithi063.stdout:3606: expect deleted 2024-08-17T17:10:02.812 INFO:tasks.rados.rados.0.smithi063.stdout:3607: write oid 6 current snap is 454 2024-08-17T17:10:02.812 INFO:tasks.rados.rados.0.smithi063.stdout:3607: seq_num 1350 ranges {689450=580631,1948691=740759} 2024-08-17T17:10:02.818 INFO:tasks.rados.rados.0.smithi063.stdout:3607: writing smithi06387500-6 from 689450 to 1270081 tid 1 2024-08-17T17:10:02.826 INFO:tasks.rados.rados.0.smithi063.stdout:3607: writing smithi06387500-6 from 1948691 to 2689450 tid 2 2024-08-17T17:10:02.826 INFO:tasks.rados.rados.0.smithi063.stdout:3608: write oid 31 current snap is 454 2024-08-17T17:10:02.826 INFO:tasks.rados.rados.0.smithi063.stdout:3608: seq_num 1351 ranges {706257=655880,1890820=504465} 2024-08-17T17:10:02.833 INFO:tasks.rados.rados.0.smithi063.stdout:3608: writing smithi06387500-31 from 706257 to 1362137 tid 1 2024-08-17T17:10:02.840 INFO:tasks.rados.rados.0.smithi063.stdout:3608: writing smithi06387500-31 from 1890820 to 2395285 tid 2 2024-08-17T17:10:02.840 INFO:tasks.rados.rados.0.smithi063.stdout:3609: delete oid 25 current snap is 454 2024-08-17T17:10:02.840 INFO:tasks.rados.rados.0.smithi063.stdout:3605: expect (ObjNum 1323 snap 449 seq_num 1323) 2024-08-17T17:10:02.943 INFO:tasks.rados.rados.0.smithi063.stdout:3607: finishing write tid 1 to smithi06387500-6 2024-08-17T17:10:02.943 INFO:tasks.rados.rados.0.smithi063.stdout:3607: finishing write tid 2 to smithi06387500-6 2024-08-17T17:10:02.944 INFO:tasks.rados.rados.0.smithi063.stdout:3607: finishing write tid 3 to smithi06387500-6 2024-08-17T17:10:02.944 INFO:tasks.rados.rados.0.smithi063.stdout:3607: finishing write tid 4 to smithi06387500-6 2024-08-17T17:10:02.944 INFO:tasks.rados.rados.0.smithi063.stdout:3607: oid 6 updating version 0 to 1215 2024-08-17T17:10:02.944 INFO:tasks.rados.rados.0.smithi063.stdout:3607: oid 6 updating version 1215 to 1216 2024-08-17T17:10:02.944 INFO:tasks.rados.rados.0.smithi063.stdout:3607: oid 6 updating version 1216 to 1217 2024-08-17T17:10:02.944 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 1217 (ObjNum 1350 snap 454 seq_num 1350) dirty exists 2024-08-17T17:10:02.944 INFO:tasks.rados.rados.0.smithi063.stdout:3607: left oid 6 (ObjNum 1350 snap 454 seq_num 1350) 2024-08-17T17:10:02.944 INFO:tasks.rados.rados.0.smithi063.stdout:3608: finishing write tid 1 to smithi06387500-31 2024-08-17T17:10:02.945 INFO:tasks.rados.rados.0.smithi063.stdout:3605: done (4 left) 2024-08-17T17:10:02.945 INFO:tasks.rados.rados.0.smithi063.stdout:3606: done (3 left) 2024-08-17T17:10:02.945 INFO:tasks.rados.rados.0.smithi063.stdout:3607: done (2 left) 2024-08-17T17:10:02.945 INFO:tasks.rados.rados.0.smithi063.stdout:3609: done (1 left) 2024-08-17T17:10:02.945 INFO:tasks.rados.rados.0.smithi063.stdout:3610: read oid 10 snap -1 2024-08-17T17:10:02.945 INFO:tasks.rados.rados.0.smithi063.stdout:3610: expect (ObjNum 1254 snap 423 seq_num 1254) 2024-08-17T17:10:02.945 INFO:tasks.rados.rados.0.smithi063.stdout:3611: rollback oid 43 current snap is 454 2024-08-17T17:10:02.946 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 43 to 443 2024-08-17T17:10:02.946 INFO:tasks.rados.rados.0.smithi063.stdout:3608: finishing write tid 2 to smithi06387500-31 2024-08-17T17:10:02.946 INFO:tasks.rados.rados.0.smithi063.stdout:3608: finishing write tid 3 to smithi06387500-31 2024-08-17T17:10:02.946 INFO:tasks.rados.rados.0.smithi063.stdout:3608: finishing write tid 4 to smithi06387500-31 2024-08-17T17:10:02.946 INFO:tasks.rados.rados.0.smithi063.stdout:3608: oid 31 updating version 0 to 2310 2024-08-17T17:10:02.946 INFO:tasks.rados.rados.0.smithi063.stdout:3608: oid 31 version 2310 is already newer than 2308 2024-08-17T17:10:02.946 INFO:tasks.rados.rados.0.smithi063.stdout:3608: oid 31 version 2310 is already newer than 2309 2024-08-17T17:10:02.946 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 2310 (ObjNum 1351 snap 454 seq_num 1351) dirty exists 2024-08-17T17:10:02.947 INFO:tasks.rados.rados.0.smithi063.stdout:3608: left oid 31 (ObjNum 1351 snap 454 seq_num 1351) 2024-08-17T17:10:02.947 INFO:tasks.rados.rados.0.smithi063.stdout:3608: done (2 left) 2024-08-17T17:10:02.947 INFO:tasks.rados.rados.0.smithi063.stdout:3612: snap_create 2024-08-17T17:10:02.947 INFO:tasks.rados.rados.0.smithi063.stdout:3610: expect (ObjNum 1254 snap 423 seq_num 1254) 2024-08-17T17:10:03.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:02 smithi063 ceph-mon[109843]: osdmap e974: 8 total, 8 up, 8 in 2024-08-17T17:10:03.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:02 smithi063 ceph-mon[109843]: osdmap e975: 8 total, 8 up, 8 in 2024-08-17T17:10:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:02 smithi063 ceph-mon[114756]: osdmap e974: 8 total, 8 up, 8 in 2024-08-17T17:10:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:02 smithi063 ceph-mon[114756]: osdmap e975: 8 total, 8 up, 8 in 2024-08-17T17:10:03.047 INFO:tasks.rados.rados.0.smithi063.stdout:3611: finishing rollback tid 1 to smithi06387500-43 2024-08-17T17:10:03.048 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 1680 (ObjNum 305 snap 0 seq_num 0) dirty dne 2024-08-17T17:10:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:02 smithi162 ceph-mon[104668]: osdmap e974: 8 total, 8 up, 8 in 2024-08-17T17:10:03.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:02 smithi162 ceph-mon[104668]: osdmap e975: 8 total, 8 up, 8 in 2024-08-17T17:10:03.696 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:10:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:10:03] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.51.0" 2024-08-17T17:10:03.705 INFO:tasks.rados.rados.0.smithi063.stdout:3610: done (2 left) 2024-08-17T17:10:03.705 INFO:tasks.rados.rados.0.smithi063.stdout:3611: done (1 left) 2024-08-17T17:10:03.705 INFO:tasks.rados.rados.0.smithi063.stdout:3612: done (0 left) 2024-08-17T17:10:03.705 INFO:tasks.rados.rados.0.smithi063.stdout:3613: read oid 2 snap -1 2024-08-17T17:10:03.706 INFO:tasks.rados.rados.0.smithi063.stdout:3613: expect deleted 2024-08-17T17:10:03.706 INFO:tasks.rados.rados.0.smithi063.stdout:3614: snap_remove snap 444 2024-08-17T17:10:04.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:03 smithi063 ceph-mon[114756]: pgmap v918: 89 pgs: 89 active+clean; 249 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-08-17T17:10:04.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:03 smithi063 ceph-mon[109843]: pgmap v918: 89 pgs: 89 active+clean; 249 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-08-17T17:10:04.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:03 smithi162 ceph-mon[104668]: pgmap v918: 89 pgs: 89 active+clean; 249 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 1.4 MiB/s wr, 13 op/s 2024-08-17T17:10:04.713 INFO:tasks.rados.rados.0.smithi063.stdout:3614: done (1 left) 2024-08-17T17:10:04.713 INFO:tasks.rados.rados.0.smithi063.stdout:3615: write oid 18 current snap is 455 2024-08-17T17:10:04.713 INFO:tasks.rados.rados.0.smithi063.stdout:3615: seq_num 1353 ranges {739871=406378,1607784=508134} 2024-08-17T17:10:04.719 INFO:tasks.rados.rados.0.smithi063.stdout:3615: writing smithi06387500-18 from 739871 to 1146249 tid 1 2024-08-17T17:10:04.726 INFO:tasks.rados.rados.0.smithi063.stdout:3615: writing smithi06387500-18 from 1607784 to 2115918 tid 2 2024-08-17T17:10:04.726 INFO:tasks.rados.rados.0.smithi063.stdout:3615: finishing write tid 1 to smithi06387500-18 2024-08-17T17:10:04.726 INFO:tasks.rados.rados.0.smithi063.stdout:3613: done (1 left) 2024-08-17T17:10:04.726 INFO:tasks.rados.rados.0.smithi063.stdout:3616: delete oid 10 current snap is 455 2024-08-17T17:10:04.730 INFO:tasks.rados.rados.0.smithi063.stdout:3615: finishing write tid 2 to smithi06387500-18 2024-08-17T17:10:04.731 INFO:tasks.rados.rados.0.smithi063.stdout:3615: finishing write tid 3 to smithi06387500-18 2024-08-17T17:10:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:3615: finishing write tid 4 to smithi06387500-18 2024-08-17T17:10:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:3615: oid 18 updating version 0 to 1247 2024-08-17T17:10:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:3615: oid 18 version 1247 is already newer than 1246 2024-08-17T17:10:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:3615: oid 18 version 1247 is already newer than 1245 2024-08-17T17:10:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 1247 (ObjNum 1353 snap 455 seq_num 1353) dirty exists 2024-08-17T17:10:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:3615: left oid 18 (ObjNum 1353 snap 455 seq_num 1353) 2024-08-17T17:10:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:3615: done (1 left) 2024-08-17T17:10:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:3616: done (0 left) 2024-08-17T17:10:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:3617: read oid 7 snap -1 2024-08-17T17:10:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:3617: expect deleted 2024-08-17T17:10:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:3618: write oid 37 current snap is 455 2024-08-17T17:10:04.732 INFO:tasks.rados.rados.0.smithi063.stdout:3618: seq_num 1355 ranges {773485=673229,2124748=511803} 2024-08-17T17:10:04.739 INFO:tasks.rados.rados.0.smithi063.stdout:3618: writing smithi06387500-37 from 773485 to 1446714 tid 1 2024-08-17T17:10:04.746 INFO:tasks.rados.rados.0.smithi063.stdout:3618: writing smithi06387500-37 from 2124748 to 2636551 tid 2 2024-08-17T17:10:04.746 INFO:tasks.rados.rados.0.smithi063.stdout:3619: delete oid 17 current snap is 455 2024-08-17T17:10:04.746 INFO:tasks.rados.rados.0.smithi063.stdout:3618: finishing write tid 1 to smithi06387500-37 2024-08-17T17:10:04.748 INFO:tasks.rados.rados.0.smithi063.stdout:3617: done (2 left) 2024-08-17T17:10:04.749 INFO:tasks.rados.rados.0.smithi063.stdout:3619: done (1 left) 2024-08-17T17:10:04.749 INFO:tasks.rados.rados.0.smithi063.stdout:3620: delete oid 29 current snap is 455 2024-08-17T17:10:04.750 INFO:tasks.rados.rados.0.smithi063.stdout:3620: done (1 left) 2024-08-17T17:10:04.750 INFO:tasks.rados.rados.0.smithi063.stdout:3621: write oid 41 current snap is 455 2024-08-17T17:10:04.751 INFO:tasks.rados.rados.0.smithi063.stdout:3621: seq_num 1358 ranges {423906=498976,1500194=659130,2609483=214423} 2024-08-17T17:10:04.755 INFO:tasks.rados.rados.0.smithi063.stdout:3621: writing smithi06387500-41 from 423906 to 922882 tid 1 2024-08-17T17:10:04.762 INFO:tasks.rados.rados.0.smithi063.stdout:3621: writing smithi06387500-41 from 1500194 to 2159324 tid 2 2024-08-17T17:10:04.766 INFO:tasks.rados.rados.0.smithi063.stdout:3621: writing smithi06387500-41 from 2609483 to 2823906 tid 3 2024-08-17T17:10:04.766 INFO:tasks.rados.rados.0.smithi063.stdout:3622: write oid 10 current snap is 455 2024-08-17T17:10:04.767 INFO:tasks.rados.rados.0.smithi063.stdout:3622: seq_num 1359 ranges {440713=574225,1442323=519141,2636906=203807} 2024-08-17T17:10:04.771 INFO:tasks.rados.rados.0.smithi063.stdout:3622: writing smithi06387500-10 from 440713 to 1014938 tid 1 2024-08-17T17:10:04.777 INFO:tasks.rados.rados.0.smithi063.stdout:3622: writing smithi06387500-10 from 1442323 to 1961464 tid 2 2024-08-17T17:10:04.781 INFO:tasks.rados.rados.0.smithi063.stdout:3622: writing smithi06387500-10 from 2636906 to 2840713 tid 3 2024-08-17T17:10:04.782 INFO:tasks.rados.rados.0.smithi063.stdout:3623: snap_remove snap 452 2024-08-17T17:10:05.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:04 smithi063 ceph-mon[114756]: osdmap e976: 8 total, 8 up, 8 in 2024-08-17T17:10:05.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:04 smithi063 ceph-mon[114756]: pgmap v921: 89 pgs: 89 active+clean; 247 MiB data, 888 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-08-17T17:10:05.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:04 smithi063 ceph-mon[109843]: osdmap e976: 8 total, 8 up, 8 in 2024-08-17T17:10:05.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:04 smithi063 ceph-mon[109843]: pgmap v921: 89 pgs: 89 active+clean; 247 MiB data, 888 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-08-17T17:10:05.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:04 smithi162 ceph-mon[104668]: osdmap e976: 8 total, 8 up, 8 in 2024-08-17T17:10:05.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:04 smithi162 ceph-mon[104668]: pgmap v921: 89 pgs: 89 active+clean; 247 MiB data, 888 MiB used, 714 GiB / 715 GiB avail; 4.7 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-08-17T17:10:05.716 INFO:tasks.rados.rados.0.smithi063.stdout:3623: done (3 left) 2024-08-17T17:10:05.716 INFO:tasks.rados.rados.0.smithi063.stdout:3624: snap_create 2024-08-17T17:10:05.716 INFO:tasks.rados.rados.0.smithi063.stdout:3618: finishing write tid 2 to smithi06387500-37 2024-08-17T17:10:05.716 INFO:tasks.rados.rados.0.smithi063.stdout:3618: finishing write tid 3 to smithi06387500-37 2024-08-17T17:10:05.716 INFO:tasks.rados.rados.0.smithi063.stdout:3618: finishing write tid 4 to smithi06387500-37 2024-08-17T17:10:05.716 INFO:tasks.rados.rados.0.smithi063.stdout:3618: oid 37 updating version 0 to 3534 2024-08-17T17:10:05.717 INFO:tasks.rados.rados.0.smithi063.stdout:3618: oid 37 updating version 3534 to 3535 2024-08-17T17:10:05.717 INFO:tasks.rados.rados.0.smithi063.stdout:3618: oid 37 version 3535 is already newer than 3533 2024-08-17T17:10:05.717 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 3535 (ObjNum 1355 snap 455 seq_num 1355) dirty exists 2024-08-17T17:10:05.717 INFO:tasks.rados.rados.0.smithi063.stdout:3618: left oid 37 (ObjNum 1355 snap 455 seq_num 1355) 2024-08-17T17:10:05.717 INFO:tasks.rados.rados.0.smithi063.stdout:3621: finishing write tid 1 to smithi06387500-41 2024-08-17T17:10:05.717 INFO:tasks.rados.rados.0.smithi063.stdout:3621: finishing write tid 2 to smithi06387500-41 2024-08-17T17:10:05.717 INFO:tasks.rados.rados.0.smithi063.stdout:3621: finishing write tid 3 to smithi06387500-41 2024-08-17T17:10:05.717 INFO:tasks.rados.rados.0.smithi063.stdout:3621: finishing write tid 4 to smithi06387500-41 2024-08-17T17:10:05.717 INFO:tasks.rados.rados.0.smithi063.stdout:3621: finishing write tid 5 to smithi06387500-41 2024-08-17T17:10:05.717 INFO:tasks.rados.rados.0.smithi063.stdout:3621: oid 41 updating version 0 to 416 2024-08-17T17:10:05.717 INFO:tasks.rados.rados.0.smithi063.stdout:3621: oid 41 version 416 is already newer than 415 2024-08-17T17:10:05.717 INFO:tasks.rados.rados.0.smithi063.stdout:3621: oid 41 version 416 is already newer than 414 2024-08-17T17:10:05.717 INFO:tasks.rados.rados.0.smithi063.stdout:3621: oid 41 updating version 416 to 417 2024-08-17T17:10:05.718 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 417 (ObjNum 1358 snap 455 seq_num 1358) dirty exists 2024-08-17T17:10:05.718 INFO:tasks.rados.rados.0.smithi063.stdout:3621: left oid 41 (ObjNum 1358 snap 455 seq_num 1358) 2024-08-17T17:10:05.718 INFO:tasks.rados.rados.0.smithi063.stdout:3622: finishing write tid 1 to smithi06387500-10 2024-08-17T17:10:05.718 INFO:tasks.rados.rados.0.smithi063.stdout:3622: finishing write tid 2 to smithi06387500-10 2024-08-17T17:10:05.718 INFO:tasks.rados.rados.0.smithi063.stdout:3622: finishing write tid 3 to smithi06387500-10 2024-08-17T17:10:05.718 INFO:tasks.rados.rados.0.smithi063.stdout:3622: finishing write tid 4 to smithi06387500-10 2024-08-17T17:10:05.718 INFO:tasks.rados.rados.0.smithi063.stdout:3622: finishing write tid 5 to smithi06387500-10 2024-08-17T17:10:05.718 INFO:tasks.rados.rados.0.smithi063.stdout:3622: oid 10 updating version 0 to 1251 2024-08-17T17:10:05.718 INFO:tasks.rados.rados.0.smithi063.stdout:3622: oid 10 version 1251 is already newer than 1250 2024-08-17T17:10:05.718 INFO:tasks.rados.rados.0.smithi063.stdout:3622: oid 10 updating version 1251 to 1253 2024-08-17T17:10:05.718 INFO:tasks.rados.rados.0.smithi063.stdout:3622: oid 10 version 1253 is already newer than 1252 2024-08-17T17:10:05.718 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 10 v 1253 (ObjNum 1359 snap 455 seq_num 1359) dirty exists 2024-08-17T17:10:05.718 INFO:tasks.rados.rados.0.smithi063.stdout:3622: left oid 10 (ObjNum 1359 snap 455 seq_num 1359) 2024-08-17T17:10:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:05 smithi063 ceph-mon[114756]: osdmap e977: 8 total, 8 up, 8 in 2024-08-17T17:10:06.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:05 smithi063 ceph-mon[109843]: osdmap e977: 8 total, 8 up, 8 in 2024-08-17T17:10:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:05 smithi162 ceph-mon[104668]: osdmap e977: 8 total, 8 up, 8 in 2024-08-17T17:10:06.726 INFO:tasks.rados.rados.0.smithi063.stdout:3618: done (3 left) 2024-08-17T17:10:06.726 INFO:tasks.rados.rados.0.smithi063.stdout:3621: done (2 left) 2024-08-17T17:10:06.726 INFO:tasks.rados.rados.0.smithi063.stdout:3622: done (1 left) 2024-08-17T17:10:06.726 INFO:tasks.rados.rados.0.smithi063.stdout:3624: done (0 left) 2024-08-17T17:10:06.726 INFO:tasks.rados.rados.0.smithi063.stdout:3625: snap_create 2024-08-17T17:10:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:06 smithi063 ceph-mon[114756]: osdmap e978: 8 total, 8 up, 8 in 2024-08-17T17:10:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:06 smithi063 ceph-mon[114756]: pgmap v924: 89 pgs: 89 active+clean; 234 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T17:10:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:06 smithi063 ceph-mon[109843]: osdmap e978: 8 total, 8 up, 8 in 2024-08-17T17:10:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:06 smithi063 ceph-mon[109843]: pgmap v924: 89 pgs: 89 active+clean; 234 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T17:10:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:06 smithi162 ceph-mon[104668]: osdmap e978: 8 total, 8 up, 8 in 2024-08-17T17:10:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:06 smithi162 ceph-mon[104668]: pgmap v924: 89 pgs: 89 active+clean; 234 MiB data, 877 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-08-17T17:10:07.672 INFO:tasks.rados.rados.0.smithi063.stdout:3625: done (0 left) 2024-08-17T17:10:07.672 INFO:tasks.rados.rados.0.smithi063.stdout:3626: write oid 9 current snap is 457 2024-08-17T17:10:07.673 INFO:tasks.rados.rados.0.smithi063.stdout:3626: seq_num 1360 ranges {457520=649474,1900805=662799} 2024-08-17T17:10:07.678 INFO:tasks.rados.rados.0.smithi063.stdout:3626: writing smithi06387500-9 from 457520 to 1106994 tid 1 2024-08-17T17:10:07.687 INFO:tasks.rados.rados.0.smithi063.stdout:3626: writing smithi06387500-9 from 1900805 to 2563604 tid 2 2024-08-17T17:10:07.687 INFO:tasks.rados.rados.0.smithi063.stdout:3627: write oid 7 current snap is 457 2024-08-17T17:10:07.687 INFO:tasks.rados.rados.0.smithi063.stdout:3626: finishing write tid 1 to smithi06387500-9 2024-08-17T17:10:07.687 INFO:tasks.rados.rados.0.smithi063.stdout:3627: seq_num 1361 ranges {474327=441076,1675640=522810,2808105=66222} 2024-08-17T17:10:07.692 INFO:tasks.rados.rados.0.smithi063.stdout:3627: writing smithi06387500-7 from 474327 to 915403 tid 1 2024-08-17T17:10:07.699 INFO:tasks.rados.rados.0.smithi063.stdout:3627: writing smithi06387500-7 from 1675640 to 2198450 tid 2 2024-08-17T17:10:07.703 INFO:tasks.rados.rados.0.smithi063.stdout:3627: writing smithi06387500-7 from 2808105 to 2874327 tid 3 2024-08-17T17:10:07.703 INFO:tasks.rados.rados.0.smithi063.stdout:3628: delete oid 18 current snap is 457 2024-08-17T17:10:07.703 INFO:tasks.rados.rados.0.smithi063.stdout:3626: finishing write tid 2 to smithi06387500-9 2024-08-17T17:10:07.703 INFO:tasks.rados.rados.0.smithi063.stdout:3626: finishing write tid 3 to smithi06387500-9 2024-08-17T17:10:07.703 INFO:tasks.rados.rados.0.smithi063.stdout:3626: finishing write tid 4 to smithi06387500-9 2024-08-17T17:10:07.703 INFO:tasks.rados.rados.0.smithi063.stdout:3626: oid 9 updating version 0 to 1728 2024-08-17T17:10:07.703 INFO:tasks.rados.rados.0.smithi063.stdout:3626: oid 9 updating version 1728 to 1729 2024-08-17T17:10:07.704 INFO:tasks.rados.rados.0.smithi063.stdout:3626: oid 9 version 1729 is already newer than 1727 2024-08-17T17:10:07.704 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 1729 (ObjNum 1360 snap 457 seq_num 1360) dirty exists 2024-08-17T17:10:07.704 INFO:tasks.rados.rados.0.smithi063.stdout:3626: left oid 9 (ObjNum 1360 snap 457 seq_num 1360) 2024-08-17T17:10:07.704 INFO:tasks.rados.rados.0.smithi063.stdout:3627: finishing write tid 1 to smithi06387500-7 2024-08-17T17:10:07.704 INFO:tasks.rados.rados.0.smithi063.stdout:3627: finishing write tid 2 to smithi06387500-7 2024-08-17T17:10:07.706 INFO:tasks.rados.rados.0.smithi063.stdout:3626: done (2 left) 2024-08-17T17:10:07.706 INFO:tasks.rados.rados.0.smithi063.stdout:3628: done (1 left) 2024-08-17T17:10:07.706 INFO:tasks.rados.rados.0.smithi063.stdout:3629: delete oid 31 current snap is 457 2024-08-17T17:10:07.706 INFO:tasks.rados.rados.0.smithi063.stdout:3627: finishing write tid 3 to smithi06387500-7 2024-08-17T17:10:07.706 INFO:tasks.rados.rados.0.smithi063.stdout:3627: finishing write tid 4 to smithi06387500-7 2024-08-17T17:10:07.706 INFO:tasks.rados.rados.0.smithi063.stdout:3627: finishing write tid 5 to smithi06387500-7 2024-08-17T17:10:07.706 INFO:tasks.rados.rados.0.smithi063.stdout:3627: oid 7 updating version 0 to 1733 2024-08-17T17:10:07.706 INFO:tasks.rados.rados.0.smithi063.stdout:3627: oid 7 version 1733 is already newer than 1732 2024-08-17T17:10:07.706 INFO:tasks.rados.rados.0.smithi063.stdout:3627: oid 7 version 1733 is already newer than 1731 2024-08-17T17:10:07.706 INFO:tasks.rados.rados.0.smithi063.stdout:3627: oid 7 version 1733 is already newer than 1730 2024-08-17T17:10:07.706 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1733 (ObjNum 1361 snap 457 seq_num 1361) dirty exists 2024-08-17T17:10:07.707 INFO:tasks.rados.rados.0.smithi063.stdout:3627: left oid 7 (ObjNum 1361 snap 457 seq_num 1361) 2024-08-17T17:10:07.708 INFO:tasks.rados.rados.0.smithi063.stdout:3627: done (1 left) 2024-08-17T17:10:07.708 INFO:tasks.rados.rados.0.smithi063.stdout:3629: done (0 left) 2024-08-17T17:10:07.708 INFO:tasks.rados.rados.0.smithi063.stdout:3630: snap_create 2024-08-17T17:10:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:07 smithi063 ceph-mon[114756]: osdmap e979: 8 total, 8 up, 8 in 2024-08-17T17:10:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:07 smithi063 ceph-mon[114756]: osdmap e980: 8 total, 8 up, 8 in 2024-08-17T17:10:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:07 smithi063 ceph-mon[109843]: osdmap e979: 8 total, 8 up, 8 in 2024-08-17T17:10:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:07 smithi063 ceph-mon[109843]: osdmap e980: 8 total, 8 up, 8 in 2024-08-17T17:10:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:07 smithi162 ceph-mon[104668]: osdmap e979: 8 total, 8 up, 8 in 2024-08-17T17:10:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:07 smithi162 ceph-mon[104668]: osdmap e980: 8 total, 8 up, 8 in 2024-08-17T17:10:08.672 INFO:tasks.rados.rados.0.smithi063.stdout:3630: done (0 left) 2024-08-17T17:10:08.673 INFO:tasks.rados.rados.0.smithi063.stdout:3631: snap_remove snap 438 2024-08-17T17:10:09.675 INFO:tasks.rados.rados.0.smithi063.stdout:3631: done (0 left) 2024-08-17T17:10:09.675 INFO:tasks.rados.rados.0.smithi063.stdout:3632: rollback oid 39 current snap is 458 2024-08-17T17:10:09.675 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 39 to 450 2024-08-17T17:10:09.675 INFO:tasks.rados.rados.0.smithi063.stdout:3633: delete oid 14 current snap is 458 2024-08-17T17:10:09.685 INFO:tasks.rados.rados.0.smithi063.stdout:3632: finishing rollback tid 0 to smithi06387500-39 2024-08-17T17:10:09.685 INFO:tasks.rados.rados.0.smithi063.stdout:3632: finishing rollback tid 1 to smithi06387500-39 2024-08-17T17:10:09.685 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 922 (ObjNum 129 snap 0 seq_num 1807101872) dirty dne 2024-08-17T17:10:09.687 INFO:tasks.rados.rados.0.smithi063.stdout:3632: done (1 left) 2024-08-17T17:10:09.687 INFO:tasks.rados.rados.0.smithi063.stdout:3633: done (0 left) 2024-08-17T17:10:09.687 INFO:tasks.rados.rados.0.smithi063.stdout:3634: snap_create 2024-08-17T17:10:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:09 smithi063 ceph-mon[109843]: pgmap v927: 89 pgs: 89 active+clean; 236 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-08-17T17:10:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:09 smithi063 ceph-mon[109843]: osdmap e981: 8 total, 8 up, 8 in 2024-08-17T17:10:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:09 smithi063 ceph-mon[114756]: pgmap v927: 89 pgs: 89 active+clean; 236 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-08-17T17:10:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:09 smithi063 ceph-mon[114756]: osdmap e981: 8 total, 8 up, 8 in 2024-08-17T17:10:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:09 smithi162 ceph-mon[104668]: pgmap v927: 89 pgs: 89 active+clean; 236 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-08-17T17:10:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:09 smithi162 ceph-mon[104668]: osdmap e981: 8 total, 8 up, 8 in 2024-08-17T17:10:10.690 INFO:tasks.rados.rados.0.smithi063.stdout:3634: done (0 left) 2024-08-17T17:10:10.690 INFO:tasks.rados.rados.0.smithi063.stdout:3635: read oid 43 snap 427 2024-08-17T17:10:10.690 INFO:tasks.rados.rados.0.smithi063.stdout:3635: expect (ObjNum 1194 snap 400 seq_num 1194) 2024-08-17T17:10:10.691 INFO:tasks.rados.rados.0.smithi063.stdout:3636: read oid 9 snap -1 2024-08-17T17:10:10.691 INFO:tasks.rados.rados.0.smithi063.stdout:3636: expect (ObjNum 1360 snap 457 seq_num 1360) 2024-08-17T17:10:10.691 INFO:tasks.rados.rados.0.smithi063.stdout:3637: read oid 10 snap 458 2024-08-17T17:10:10.691 INFO:tasks.rados.rados.0.smithi063.stdout:3637: expect (ObjNum 1359 snap 455 seq_num 1359) 2024-08-17T17:10:10.691 INFO:tasks.rados.rados.0.smithi063.stdout:3638: snap_create 2024-08-17T17:10:10.698 INFO:tasks.rados.rados.0.smithi063.stdout:3637: expect (ObjNum 1359 snap 455 seq_num 1359) 2024-08-17T17:10:10.794 INFO:tasks.rados.rados.0.smithi063.stdout:3635: expect (ObjNum 1194 snap 400 seq_num 1194) 2024-08-17T17:10:10.864 INFO:tasks.rados.rados.0.smithi063.stdout:3636: expect (ObjNum 1360 snap 457 seq_num 1360) 2024-08-17T17:10:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:10 smithi063 ceph-mon[114756]: osdmap e982: 8 total, 8 up, 8 in 2024-08-17T17:10:11.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:10 smithi063 ceph-mon[109843]: osdmap e982: 8 total, 8 up, 8 in 2024-08-17T17:10:11.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:10 smithi162 ceph-mon[104668]: osdmap e982: 8 total, 8 up, 8 in 2024-08-17T17:10:11.691 INFO:tasks.rados.rados.0.smithi063.stdout:3635: done (3 left) 2024-08-17T17:10:11.692 INFO:tasks.rados.rados.0.smithi063.stdout:3636: done (2 left) 2024-08-17T17:10:11.692 INFO:tasks.rados.rados.0.smithi063.stdout:3637: done (1 left) 2024-08-17T17:10:11.692 INFO:tasks.rados.rados.0.smithi063.stdout:3638: done (0 left) 2024-08-17T17:10:11.692 INFO:tasks.rados.rados.0.smithi063.stdout:3639: read oid 34 snap -1 2024-08-17T17:10:11.692 INFO:tasks.rados.rados.0.smithi063.stdout:3639: expect deleted 2024-08-17T17:10:11.692 INFO:tasks.rados.rados.0.smithi063.stdout:3640: delete oid 28 current snap is 460 2024-08-17T17:10:11.698 INFO:tasks.rados.rados.0.smithi063.stdout:3639: done (1 left) 2024-08-17T17:10:11.698 INFO:tasks.rados.rados.0.smithi063.stdout:3640: done (0 left) 2024-08-17T17:10:11.699 INFO:tasks.rados.rados.0.smithi063.stdout:3641: write oid 5 current snap is 460 2024-08-17T17:10:11.699 INFO:tasks.rados.rados.0.smithi063.stdout:3641: seq_num 1366 ranges {558362=417321,1451697=673806,2712514=245848} 2024-08-17T17:10:11.703 INFO:tasks.rados.rados.0.smithi063.stdout:3641: writing smithi06387500-5 from 558362 to 975683 tid 1 2024-08-17T17:10:11.710 INFO:tasks.rados.rados.0.smithi063.stdout:3641: writing smithi06387500-5 from 1451697 to 2125503 tid 2 2024-08-17T17:10:11.715 INFO:tasks.rados.rados.0.smithi063.stdout:3641: writing smithi06387500-5 from 2712514 to 2958362 tid 3 2024-08-17T17:10:11.715 INFO:tasks.rados.rados.0.smithi063.stdout:3642: read oid 15 snap -1 2024-08-17T17:10:11.715 INFO:tasks.rados.rados.0.smithi063.stdout:3642: expect (ObjNum 1240 snap 415 seq_num 1240) 2024-08-17T17:10:11.715 INFO:tasks.rados.rados.0.smithi063.stdout:3643: snap_remove snap 451 2024-08-17T17:10:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:11 smithi063 ceph-mon[114756]: pgmap v930: 89 pgs: 89 active+clean; 236 MiB data, 871 MiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 948 KiB/s wr, 7 op/s 2024-08-17T17:10:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:11 smithi063 ceph-mon[114756]: osdmap e983: 8 total, 8 up, 8 in 2024-08-17T17:10:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:11 smithi063 ceph-mon[109843]: pgmap v930: 89 pgs: 89 active+clean; 236 MiB data, 871 MiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 948 KiB/s wr, 7 op/s 2024-08-17T17:10:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:11 smithi063 ceph-mon[109843]: osdmap e983: 8 total, 8 up, 8 in 2024-08-17T17:10:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:11 smithi162 ceph-mon[104668]: pgmap v930: 89 pgs: 89 active+clean; 236 MiB data, 871 MiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 948 KiB/s wr, 7 op/s 2024-08-17T17:10:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:11 smithi162 ceph-mon[104668]: osdmap e983: 8 total, 8 up, 8 in 2024-08-17T17:10:12.672 INFO:tasks.rados.rados.0.smithi063.stdout:3643: done (2 left) 2024-08-17T17:10:12.672 INFO:tasks.rados.rados.0.smithi063.stdout:3644: snap_create 2024-08-17T17:10:12.672 INFO:tasks.rados.rados.0.smithi063.stdout:3641: finishing write tid 1 to smithi06387500-5 2024-08-17T17:10:12.672 INFO:tasks.rados.rados.0.smithi063.stdout:3641: finishing write tid 2 to smithi06387500-5 2024-08-17T17:10:12.673 INFO:tasks.rados.rados.0.smithi063.stdout:3641: finishing write tid 3 to smithi06387500-5 2024-08-17T17:10:12.673 INFO:tasks.rados.rados.0.smithi063.stdout:3641: finishing write tid 4 to smithi06387500-5 2024-08-17T17:10:12.673 INFO:tasks.rados.rados.0.smithi063.stdout:3641: finishing write tid 5 to smithi06387500-5 2024-08-17T17:10:12.673 INFO:tasks.rados.rados.0.smithi063.stdout:3641: oid 5 updating version 0 to 1928 2024-08-17T17:10:12.673 INFO:tasks.rados.rados.0.smithi063.stdout:3641: oid 5 updating version 1928 to 1929 2024-08-17T17:10:12.673 INFO:tasks.rados.rados.0.smithi063.stdout:3641: oid 5 version 1929 is already newer than 1927 2024-08-17T17:10:12.673 INFO:tasks.rados.rados.0.smithi063.stdout:3641: oid 5 version 1929 is already newer than 1926 2024-08-17T17:10:12.673 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1929 (ObjNum 1366 snap 460 seq_num 1366) dirty exists 2024-08-17T17:10:12.673 INFO:tasks.rados.rados.0.smithi063.stdout:3641: left oid 5 (ObjNum 1366 snap 460 seq_num 1366) 2024-08-17T17:10:12.673 INFO:tasks.rados.rados.0.smithi063.stdout:3642: expect (ObjNum 1240 snap 415 seq_num 1240) 2024-08-17T17:10:12.951 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:12 smithi063 ceph-mon[114756]: osdmap e984: 8 total, 8 up, 8 in 2024-08-17T17:10:12.951 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:12 smithi063 ceph-mon[114756]: pgmap v933: 89 pgs: 89 active+clean; 243 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 899 KiB/s wr, 11 op/s 2024-08-17T17:10:12.952 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:12 smithi063 ceph-mon[114756]: osdmap e985: 8 total, 8 up, 8 in 2024-08-17T17:10:12.952 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:12 smithi063 ceph-mon[109843]: osdmap e984: 8 total, 8 up, 8 in 2024-08-17T17:10:12.952 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:12 smithi063 ceph-mon[109843]: pgmap v933: 89 pgs: 89 active+clean; 243 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 899 KiB/s wr, 11 op/s 2024-08-17T17:10:12.952 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:12 smithi063 ceph-mon[109843]: osdmap e985: 8 total, 8 up, 8 in 2024-08-17T17:10:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:12 smithi162 ceph-mon[104668]: osdmap e984: 8 total, 8 up, 8 in 2024-08-17T17:10:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:12 smithi162 ceph-mon[104668]: pgmap v933: 89 pgs: 89 active+clean; 243 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 899 KiB/s wr, 11 op/s 2024-08-17T17:10:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:12 smithi162 ceph-mon[104668]: osdmap e985: 8 total, 8 up, 8 in 2024-08-17T17:10:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:10:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:10:13] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.51.0" 2024-08-17T17:10:13.675 INFO:tasks.rados.rados.0.smithi063.stdout:3641: done (2 left) 2024-08-17T17:10:13.675 INFO:tasks.rados.rados.0.smithi063.stdout:3642: done (1 left) 2024-08-17T17:10:13.675 INFO:tasks.rados.rados.0.smithi063.stdout:3644: done (0 left) 2024-08-17T17:10:13.675 INFO:tasks.rados.rados.0.smithi063.stdout:3645: delete oid 6 current snap is 461 2024-08-17T17:10:13.679 INFO:tasks.rados.rados.0.smithi063.stdout:3645: done (0 left) 2024-08-17T17:10:13.679 INFO:tasks.rados.rados.0.smithi063.stdout:3646: delete oid 32 current snap is 461 2024-08-17T17:10:13.683 INFO:tasks.rados.rados.0.smithi063.stdout:3646: done (0 left) 2024-08-17T17:10:13.683 INFO:tasks.rados.rados.0.smithi063.stdout:3647: read oid 33 snap -1 2024-08-17T17:10:13.683 INFO:tasks.rados.rados.0.smithi063.stdout:3647: expect (ObjNum 1323 snap 449 seq_num 1323) 2024-08-17T17:10:13.683 INFO:tasks.rados.rados.0.smithi063.stdout:3648: write oid 16 current snap is 461 2024-08-17T17:10:13.684 INFO:tasks.rados.rados.0.smithi063.stdout:3648: seq_num 1369 ranges {608783=759421,2027143=421133} 2024-08-17T17:10:13.690 INFO:tasks.rados.rados.0.smithi063.stdout:3648: writing smithi06387500-16 from 608783 to 1368204 tid 1 2024-08-17T17:10:13.696 INFO:tasks.rados.rados.0.smithi063.stdout:3648: writing smithi06387500-16 from 2027143 to 2448276 tid 2 2024-08-17T17:10:13.697 INFO:tasks.rados.rados.0.smithi063.stdout:3649: snap_remove snap 436 2024-08-17T17:10:14.681 INFO:tasks.rados.rados.0.smithi063.stdout:3649: done (2 left) 2024-08-17T17:10:14.681 INFO:tasks.rados.rados.0.smithi063.stdout:3650: snap_remove snap 430 2024-08-17T17:10:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:14 smithi063 ceph-mon[114756]: osdmap e986: 8 total, 8 up, 8 in 2024-08-17T17:10:15.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:14 smithi063 ceph-mon[109843]: osdmap e986: 8 total, 8 up, 8 in 2024-08-17T17:10:15.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:14 smithi162 ceph-mon[104668]: osdmap e986: 8 total, 8 up, 8 in 2024-08-17T17:10:15.689 INFO:tasks.rados.rados.0.smithi063.stdout:3650: done (2 left) 2024-08-17T17:10:15.689 INFO:tasks.rados.rados.0.smithi063.stdout:3651: delete oid 48 current snap is 461 2024-08-17T17:10:15.689 INFO:tasks.rados.rados.0.smithi063.stdout:3647: expect (ObjNum 1323 snap 449 seq_num 1323) 2024-08-17T17:10:15.796 INFO:tasks.rados.rados.0.smithi063.stdout:3648: finishing write tid 1 to smithi06387500-16 2024-08-17T17:10:15.797 INFO:tasks.rados.rados.0.smithi063.stdout:3648: finishing write tid 2 to smithi06387500-16 2024-08-17T17:10:15.797 INFO:tasks.rados.rados.0.smithi063.stdout:3648: finishing write tid 3 to smithi06387500-16 2024-08-17T17:10:15.797 INFO:tasks.rados.rados.0.smithi063.stdout:3648: finishing write tid 4 to smithi06387500-16 2024-08-17T17:10:15.797 INFO:tasks.rados.rados.0.smithi063.stdout:3648: oid 16 updating version 0 to 1333 2024-08-17T17:10:15.797 INFO:tasks.rados.rados.0.smithi063.stdout:3648: oid 16 updating version 1333 to 1334 2024-08-17T17:10:15.797 INFO:tasks.rados.rados.0.smithi063.stdout:3648: oid 16 version 1334 is already newer than 1332 2024-08-17T17:10:15.797 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 1334 (ObjNum 1369 snap 461 seq_num 1369) dirty exists 2024-08-17T17:10:15.797 INFO:tasks.rados.rados.0.smithi063.stdout:3648: left oid 16 (ObjNum 1369 snap 461 seq_num 1369) 2024-08-17T17:10:15.797 INFO:tasks.rados.rados.0.smithi063.stdout:3647: done (2 left) 2024-08-17T17:10:15.797 INFO:tasks.rados.rados.0.smithi063.stdout:3648: done (1 left) 2024-08-17T17:10:15.797 INFO:tasks.rados.rados.0.smithi063.stdout:3651: done (0 left) 2024-08-17T17:10:15.798 INFO:tasks.rados.rados.0.smithi063.stdout:3652: snap_create 2024-08-17T17:10:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:15 smithi063 ceph-mon[114756]: pgmap v936: 89 pgs: 89 active+clean; 243 MiB data, 862 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 899 KiB/s wr, 12 op/s 2024-08-17T17:10:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:15 smithi063 ceph-mon[114756]: osdmap e987: 8 total, 8 up, 8 in 2024-08-17T17:10:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:15 smithi063 ceph-mon[109843]: pgmap v936: 89 pgs: 89 active+clean; 243 MiB data, 862 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 899 KiB/s wr, 12 op/s 2024-08-17T17:10:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:15 smithi063 ceph-mon[109843]: osdmap e987: 8 total, 8 up, 8 in 2024-08-17T17:10:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:15 smithi162 ceph-mon[104668]: pgmap v936: 89 pgs: 89 active+clean; 243 MiB data, 862 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 899 KiB/s wr, 12 op/s 2024-08-17T17:10:16.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:15 smithi162 ceph-mon[104668]: osdmap e987: 8 total, 8 up, 8 in 2024-08-17T17:10:16.697 INFO:tasks.rados.rados.0.smithi063.stdout:3652: done (0 left) 2024-08-17T17:10:16.698 INFO:tasks.rados.rados.0.smithi063.stdout:3653: read oid 47 snap -1 2024-08-17T17:10:16.698 INFO:tasks.rados.rados.0.smithi063.stdout:3653: expect (ObjNum 1230 snap 411 seq_num 1230) 2024-08-17T17:10:16.698 INFO:tasks.rados.rados.0.smithi063.stdout:3654: write oid 4 current snap is 462 2024-08-17T17:10:16.698 INFO:tasks.rados.rados.0.smithi063.stdout:3654: seq_num 1371 ranges {642397=509919,1627754=424802,2733276=309121} 2024-08-17T17:10:16.704 INFO:tasks.rados.rados.0.smithi063.stdout:3654: writing smithi06387500-4 from 642397 to 1152316 tid 1 2024-08-17T17:10:16.709 INFO:tasks.rados.rados.0.smithi063.stdout:3654: writing smithi06387500-4 from 1627754 to 2052556 tid 2 2024-08-17T17:10:16.715 INFO:tasks.rados.rados.0.smithi063.stdout:3654: writing smithi06387500-4 from 2733276 to 3042397 tid 3 2024-08-17T17:10:16.715 INFO:tasks.rados.rados.0.smithi063.stdout:3655: write oid 13 current snap is 462 2024-08-17T17:10:16.715 INFO:tasks.rados.rados.0.smithi063.stdout:3655: seq_num 1372 ranges {659204=585168,1686236=684813} 2024-08-17T17:10:16.722 INFO:tasks.rados.rados.0.smithi063.stdout:3655: writing smithi06387500-13 from 659204 to 1244372 tid 1 2024-08-17T17:10:16.728 INFO:tasks.rados.rados.0.smithi063.stdout:3655: writing smithi06387500-13 from 1686236 to 2371049 tid 2 2024-08-17T17:10:16.728 INFO:tasks.rados.rados.0.smithi063.stdout:3656: write oid 14 current snap is 462 2024-08-17T17:10:16.728 INFO:tasks.rados.rados.0.smithi063.stdout:3656: seq_num 1373 ranges {676011=660417,1744718=428471,2788122=287889} 2024-08-17T17:10:16.736 INFO:tasks.rados.rados.0.smithi063.stdout:3656: writing smithi06387500-14 from 676011 to 1336428 tid 1 2024-08-17T17:10:16.740 INFO:tasks.rados.rados.0.smithi063.stdout:3656: writing smithi06387500-14 from 1744718 to 2173189 tid 2 2024-08-17T17:10:16.746 INFO:tasks.rados.rados.0.smithi063.stdout:3656: writing smithi06387500-14 from 2788122 to 3076011 tid 3 2024-08-17T17:10:16.746 INFO:tasks.rados.rados.0.smithi063.stdout:3657: write oid 16 current snap is 462 2024-08-17T17:10:16.746 INFO:tasks.rados.rados.0.smithi063.stdout:3657: seq_num 1374 ranges {692818=735666,2203200=688482} 2024-08-17T17:10:16.753 INFO:tasks.rados.rados.0.smithi063.stdout:3657: writing smithi06387500-16 from 692818 to 1428484 tid 1 2024-08-17T17:10:16.762 INFO:tasks.rados.rados.0.smithi063.stdout:3657: writing smithi06387500-16 from 2203200 to 2891682 tid 2 2024-08-17T17:10:16.762 INFO:tasks.rados.rados.0.smithi063.stdout:3658: snap_create 2024-08-17T17:10:16.763 INFO:tasks.rados.rados.0.smithi063.stdout:3653: expect (ObjNum 1230 snap 411 seq_num 1230) 2024-08-17T17:10:16.851 INFO:tasks.rados.rados.0.smithi063.stdout:3654: finishing write tid 1 to smithi06387500-4 2024-08-17T17:10:16.851 INFO:tasks.rados.rados.0.smithi063.stdout:3654: finishing write tid 2 to smithi06387500-4 2024-08-17T17:10:16.852 INFO:tasks.rados.rados.0.smithi063.stdout:3654: finishing write tid 3 to smithi06387500-4 2024-08-17T17:10:16.852 INFO:tasks.rados.rados.0.smithi063.stdout:3654: finishing write tid 4 to smithi06387500-4 2024-08-17T17:10:16.852 INFO:tasks.rados.rados.0.smithi063.stdout:3654: finishing write tid 5 to smithi06387500-4 2024-08-17T17:10:16.852 INFO:tasks.rados.rados.0.smithi063.stdout:3654: oid 4 updating version 0 to 3580 2024-08-17T17:10:16.852 INFO:tasks.rados.rados.0.smithi063.stdout:3654: oid 4 version 3580 is already newer than 3577 2024-08-17T17:10:16.852 INFO:tasks.rados.rados.0.smithi063.stdout:3654: oid 4 version 3580 is already newer than 3579 2024-08-17T17:10:16.852 INFO:tasks.rados.rados.0.smithi063.stdout:3654: oid 4 version 3580 is already newer than 3578 2024-08-17T17:10:16.852 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 3580 (ObjNum 1371 snap 462 seq_num 1371) dirty exists 2024-08-17T17:10:16.852 INFO:tasks.rados.rados.0.smithi063.stdout:3654: left oid 4 (ObjNum 1371 snap 462 seq_num 1371) 2024-08-17T17:10:16.852 INFO:tasks.rados.rados.0.smithi063.stdout:3655: finishing write tid 1 to smithi06387500-13 2024-08-17T17:10:16.852 INFO:tasks.rados.rados.0.smithi063.stdout:3655: finishing write tid 2 to smithi06387500-13 2024-08-17T17:10:16.852 INFO:tasks.rados.rados.0.smithi063.stdout:3655: finishing write tid 3 to smithi06387500-13 2024-08-17T17:10:16.853 INFO:tasks.rados.rados.0.smithi063.stdout:3655: finishing write tid 4 to smithi06387500-13 2024-08-17T17:10:16.853 INFO:tasks.rados.rados.0.smithi063.stdout:3655: oid 13 updating version 0 to 1951 2024-08-17T17:10:16.853 INFO:tasks.rados.rados.0.smithi063.stdout:3655: oid 13 updating version 1951 to 1952 2024-08-17T17:10:16.853 INFO:tasks.rados.rados.0.smithi063.stdout:3655: oid 13 updating version 1952 to 1953 2024-08-17T17:10:16.853 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 1953 (ObjNum 1372 snap 462 seq_num 1372) dirty exists 2024-08-17T17:10:16.853 INFO:tasks.rados.rados.0.smithi063.stdout:3655: left oid 13 (ObjNum 1372 snap 462 seq_num 1372) 2024-08-17T17:10:16.853 INFO:tasks.rados.rados.0.smithi063.stdout:3656: finishing write tid 1 to smithi06387500-14 2024-08-17T17:10:16.853 INFO:tasks.rados.rados.0.smithi063.stdout:3656: finishing write tid 2 to smithi06387500-14 2024-08-17T17:10:16.853 INFO:tasks.rados.rados.0.smithi063.stdout:3656: finishing write tid 3 to smithi06387500-14 2024-08-17T17:10:16.853 INFO:tasks.rados.rados.0.smithi063.stdout:3656: finishing write tid 4 to smithi06387500-14 2024-08-17T17:10:16.853 INFO:tasks.rados.rados.0.smithi063.stdout:3656: finishing write tid 5 to smithi06387500-14 2024-08-17T17:10:16.853 INFO:tasks.rados.rados.0.smithi063.stdout:3656: oid 14 updating version 0 to 3583 2024-08-17T17:10:16.853 INFO:tasks.rados.rados.0.smithi063.stdout:3656: oid 14 version 3583 is already newer than 3581 2024-08-17T17:10:16.854 INFO:tasks.rados.rados.0.smithi063.stdout:3656: oid 14 version 3583 is already newer than 3582 2024-08-17T17:10:16.854 INFO:tasks.rados.rados.0.smithi063.stdout:3656: oid 14 updating version 3583 to 3584 2024-08-17T17:10:16.854 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 3584 (ObjNum 1373 snap 462 seq_num 1373) dirty exists 2024-08-17T17:10:16.854 INFO:tasks.rados.rados.0.smithi063.stdout:3656: left oid 14 (ObjNum 1373 snap 462 seq_num 1373) 2024-08-17T17:10:16.854 INFO:tasks.rados.rados.0.smithi063.stdout:3657: finishing write tid 1 to smithi06387500-16 2024-08-17T17:10:16.854 INFO:tasks.rados.rados.0.smithi063.stdout:3657: finishing write tid 2 to smithi06387500-16 2024-08-17T17:10:16.854 INFO:tasks.rados.rados.0.smithi063.stdout:3657: finishing write tid 3 to smithi06387500-16 2024-08-17T17:10:16.854 INFO:tasks.rados.rados.0.smithi063.stdout:3657: finishing write tid 4 to smithi06387500-16 2024-08-17T17:10:16.854 INFO:tasks.rados.rados.0.smithi063.stdout:3657: oid 16 updating version 0 to 1348 2024-08-17T17:10:16.854 INFO:tasks.rados.rados.0.smithi063.stdout:3657: oid 16 version 1348 is already newer than 1346 2024-08-17T17:10:16.854 INFO:tasks.rados.rados.0.smithi063.stdout:3657: oid 16 version 1348 is already newer than 1347 2024-08-17T17:10:16.854 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 1348 (ObjNum 1374 snap 462 seq_num 1374) dirty exists 2024-08-17T17:10:16.855 INFO:tasks.rados.rados.0.smithi063.stdout:3657: left oid 16 (ObjNum 1374 snap 462 seq_num 1374) 2024-08-17T17:10:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:16 smithi063 ceph-mon[114756]: osdmap e988: 8 total, 8 up, 8 in 2024-08-17T17:10:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:16 smithi063 ceph-mon[109843]: osdmap e988: 8 total, 8 up, 8 in 2024-08-17T17:10:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:16 smithi162 ceph-mon[104668]: osdmap e988: 8 total, 8 up, 8 in 2024-08-17T17:10:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:17.674 INFO:tasks.rados.rados.0.smithi063.stdout:3653: done (5 left) 2024-08-17T17:10:17.674 INFO:tasks.rados.rados.0.smithi063.stdout:3654: done (4 left) 2024-08-17T17:10:17.674 INFO:tasks.rados.rados.0.smithi063.stdout:3655: done (3 left) 2024-08-17T17:10:17.674 INFO:tasks.rados.rados.0.smithi063.stdout:3656: done (2 left) 2024-08-17T17:10:17.675 INFO:tasks.rados.rados.0.smithi063.stdout:3657: done (1 left) 2024-08-17T17:10:17.675 INFO:tasks.rados.rados.0.smithi063.stdout:3658: done (0 left) 2024-08-17T17:10:17.675 INFO:tasks.rados.rados.0.smithi063.stdout:3659: delete oid 14 current snap is 463 2024-08-17T17:10:17.680 INFO:tasks.rados.rados.0.smithi063.stdout:3659: done (0 left) 2024-08-17T17:10:17.680 INFO:tasks.rados.rados.0.smithi063.stdout:3660: rollback oid 7 current snap is 463 2024-08-17T17:10:17.680 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 7 to 427 2024-08-17T17:10:17.680 INFO:tasks.rados.rados.0.smithi063.stdout:3661: write oid 20 current snap is 463 2024-08-17T17:10:17.680 INFO:tasks.rados.rados.0.smithi063.stdout:3661: seq_num 1376 ranges {726432=486164,1803811=692151} 2024-08-17T17:10:17.687 INFO:tasks.rados.rados.0.smithi063.stdout:3661: writing smithi06387500-20 from 726432 to 1212596 tid 1 2024-08-17T17:10:17.695 INFO:tasks.rados.rados.0.smithi063.stdout:3661: writing smithi06387500-20 from 1803811 to 2495962 tid 2 2024-08-17T17:10:17.695 INFO:tasks.rados.rados.0.smithi063.stdout:3662: snap_remove snap 462 2024-08-17T17:10:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:17 smithi063 ceph-mon[114756]: pgmap v939: 89 pgs: 89 active+clean; 244 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 953 KiB/s rd, 288 KiB/s wr, 3 op/s 2024-08-17T17:10:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:17 smithi063 ceph-mon[114756]: osdmap e989: 8 total, 8 up, 8 in 2024-08-17T17:10:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:17 smithi063 ceph-mon[114756]: osdmap e990: 8 total, 8 up, 8 in 2024-08-17T17:10:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:17 smithi063 ceph-mon[109843]: pgmap v939: 89 pgs: 89 active+clean; 244 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 953 KiB/s rd, 288 KiB/s wr, 3 op/s 2024-08-17T17:10:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:17 smithi063 ceph-mon[109843]: osdmap e989: 8 total, 8 up, 8 in 2024-08-17T17:10:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:17 smithi063 ceph-mon[109843]: osdmap e990: 8 total, 8 up, 8 in 2024-08-17T17:10:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:17 smithi162 ceph-mon[104668]: pgmap v939: 89 pgs: 89 active+clean; 244 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 953 KiB/s rd, 288 KiB/s wr, 3 op/s 2024-08-17T17:10:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:17 smithi162 ceph-mon[104668]: osdmap e989: 8 total, 8 up, 8 in 2024-08-17T17:10:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:17 smithi162 ceph-mon[104668]: osdmap e990: 8 total, 8 up, 8 in 2024-08-17T17:10:18.704 INFO:tasks.rados.rados.0.smithi063.stdout:3662: done (2 left) 2024-08-17T17:10:18.704 INFO:tasks.rados.rados.0.smithi063.stdout:3663: write oid 18 current snap is 463 2024-08-17T17:10:18.704 INFO:tasks.rados.rados.0.smithi063.stdout:3660: finishing rollback tid 0 to smithi06387500-7 2024-08-17T17:10:18.704 INFO:tasks.rados.rados.0.smithi063.stdout:3663: seq_num 1377 ranges {743239=677766,1978646=435809,2897814=245425} 2024-08-17T17:10:18.711 INFO:tasks.rados.rados.0.smithi063.stdout:3663: writing smithi06387500-18 from 743239 to 1421005 tid 1 2024-08-17T17:10:18.718 INFO:tasks.rados.rados.0.smithi063.stdout:3663: writing smithi06387500-18 from 1978646 to 2414455 tid 2 2024-08-17T17:10:18.721 INFO:tasks.rados.rados.0.smithi063.stdout:3663: writing smithi06387500-18 from 2897814 to 3143239 tid 3 2024-08-17T17:10:18.721 INFO:tasks.rados.rados.0.smithi063.stdout:3664: delete oid 36 current snap is 463 2024-08-17T17:10:18.721 INFO:tasks.rados.rados.0.smithi063.stdout:3660: finishing rollback tid 1 to smithi06387500-7 2024-08-17T17:10:18.721 INFO:tasks.rados.rados.0.smithi063.stdout:3660: finishing rollback tid 2 to smithi06387500-7 2024-08-17T17:10:18.722 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 7 v 1751 (ObjNum 1180 snap 398 seq_num 1180) dirty exists 2024-08-17T17:10:18.722 INFO:tasks.rados.rados.0.smithi063.stdout:3661: finishing write tid 1 to smithi06387500-20 2024-08-17T17:10:18.722 INFO:tasks.rados.rados.0.smithi063.stdout:3661: finishing write tid 2 to smithi06387500-20 2024-08-17T17:10:18.722 INFO:tasks.rados.rados.0.smithi063.stdout:3661: finishing write tid 3 to smithi06387500-20 2024-08-17T17:10:18.722 INFO:tasks.rados.rados.0.smithi063.stdout:3661: finishing write tid 4 to smithi06387500-20 2024-08-17T17:10:18.722 INFO:tasks.rados.rados.0.smithi063.stdout:3661: oid 20 updating version 0 to 906 2024-08-17T17:10:18.722 INFO:tasks.rados.rados.0.smithi063.stdout:3661: oid 20 updating version 906 to 907 2024-08-17T17:10:18.722 INFO:tasks.rados.rados.0.smithi063.stdout:3661: oid 20 version 907 is already newer than 905 2024-08-17T17:10:18.722 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 907 (ObjNum 1376 snap 463 seq_num 1376) dirty exists 2024-08-17T17:10:18.722 INFO:tasks.rados.rados.0.smithi063.stdout:3661: left oid 20 (ObjNum 1376 snap 463 seq_num 1376) 2024-08-17T17:10:18.722 INFO:tasks.rados.rados.0.smithi063.stdout:3663: finishing write tid 1 to smithi06387500-18 2024-08-17T17:10:18.722 INFO:tasks.rados.rados.0.smithi063.stdout:3663: finishing write tid 2 to smithi06387500-18 2024-08-17T17:10:18.724 INFO:tasks.rados.rados.0.smithi063.stdout:3663: finishing write tid 3 to smithi06387500-18 2024-08-17T17:10:18.724 INFO:tasks.rados.rados.0.smithi063.stdout:3663: finishing write tid 4 to smithi06387500-18 2024-08-17T17:10:18.725 INFO:tasks.rados.rados.0.smithi063.stdout:3663: finishing write tid 5 to smithi06387500-18 2024-08-17T17:10:18.725 INFO:tasks.rados.rados.0.smithi063.stdout:3663: oid 18 updating version 0 to 1280 2024-08-17T17:10:18.725 INFO:tasks.rados.rados.0.smithi063.stdout:3663: oid 18 updating version 1280 to 1281 2024-08-17T17:10:18.725 INFO:tasks.rados.rados.0.smithi063.stdout:3663: oid 18 version 1281 is already newer than 1279 2024-08-17T17:10:18.725 INFO:tasks.rados.rados.0.smithi063.stdout:3663: oid 18 version 1281 is already newer than 1278 2024-08-17T17:10:18.725 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 1281 (ObjNum 1377 snap 463 seq_num 1377) dirty exists 2024-08-17T17:10:18.725 INFO:tasks.rados.rados.0.smithi063.stdout:3663: left oid 18 (ObjNum 1377 snap 463 seq_num 1377) 2024-08-17T17:10:18.730 INFO:tasks.rados.rados.0.smithi063.stdout:3660: done (3 left) 2024-08-17T17:10:18.730 INFO:tasks.rados.rados.0.smithi063.stdout:3661: done (2 left) 2024-08-17T17:10:18.730 INFO:tasks.rados.rados.0.smithi063.stdout:3663: done (1 left) 2024-08-17T17:10:18.730 INFO:tasks.rados.rados.0.smithi063.stdout:3664: done (0 left) 2024-08-17T17:10:18.730 INFO:tasks.rados.rados.0.smithi063.stdout:3665: write oid 18 current snap is 463 2024-08-17T17:10:18.730 INFO:tasks.rados.rados.0.smithi063.stdout:3665: seq_num 1379 ranges {776853=428264,1979257=439478,2836307=340546} 2024-08-17T17:10:18.736 INFO:tasks.rados.rados.0.smithi063.stdout:3665: writing smithi06387500-18 from 776853 to 1205117 tid 1 2024-08-17T17:10:18.743 INFO:tasks.rados.rados.0.smithi063.stdout:3665: writing smithi06387500-18 from 1979257 to 2418735 tid 2 2024-08-17T17:10:18.748 INFO:tasks.rados.rados.0.smithi063.stdout:3665: writing smithi06387500-18 from 2836307 to 3176853 tid 3 2024-08-17T17:10:18.748 INFO:tasks.rados.rados.0.smithi063.stdout:3666: read oid 30 snap 453 2024-08-17T17:10:18.748 INFO:tasks.rados.rados.0.smithi063.stdout:3666: expect (ObjNum 1215 snap 407 seq_num 1215) 2024-08-17T17:10:18.748 INFO:tasks.rados.rados.0.smithi063.stdout:3667: rollback oid 14 current snap is 463 2024-08-17T17:10:18.748 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 14 to 450 2024-08-17T17:10:18.748 INFO:tasks.rados.rados.0.smithi063.stdout:3665: finishing write tid 1 to smithi06387500-18 2024-08-17T17:10:18.748 INFO:tasks.rados.rados.0.smithi063.stdout:3665: finishing write tid 2 to smithi06387500-18 2024-08-17T17:10:18.748 INFO:tasks.rados.rados.0.smithi063.stdout:3668: write oid 47 current snap is 463 2024-08-17T17:10:18.748 INFO:tasks.rados.rados.0.smithi063.stdout:3668: seq_num 1380 ranges {793660=503513,2037739=583136} 2024-08-17T17:10:18.755 INFO:tasks.rados.rados.0.smithi063.stdout:3668: writing smithi06387500-47 from 793660 to 1297173 tid 1 2024-08-17T17:10:18.762 INFO:tasks.rados.rados.0.smithi063.stdout:3668: writing smithi06387500-47 from 2037739 to 2620875 tid 2 2024-08-17T17:10:18.763 INFO:tasks.rados.rados.0.smithi063.stdout:3669: read oid 42 snap -1 2024-08-17T17:10:18.763 INFO:tasks.rados.rados.0.smithi063.stdout:3669: expect (ObjNum 1304 snap 443 seq_num 1304) 2024-08-17T17:10:18.763 INFO:tasks.rados.rados.0.smithi063.stdout:3670: write oid 9 current snap is 463 2024-08-17T17:10:18.763 INFO:tasks.rados.rados.0.smithi063.stdout:3665: finishing write tid 3 to smithi06387500-18 2024-08-17T17:10:18.763 INFO:tasks.rados.rados.0.smithi063.stdout:3665: finishing write tid 4 to smithi06387500-18 2024-08-17T17:10:18.763 INFO:tasks.rados.rados.0.smithi063.stdout:3665: finishing write tid 5 to smithi06387500-18 2024-08-17T17:10:18.763 INFO:tasks.rados.rados.0.smithi063.stdout:3665: oid 18 updating version 0 to 1285 2024-08-17T17:10:18.763 INFO:tasks.rados.rados.0.smithi063.stdout:3665: oid 18 version 1285 is already newer than 1282 2024-08-17T17:10:18.763 INFO:tasks.rados.rados.0.smithi063.stdout:3665: oid 18 version 1285 is already newer than 1283 2024-08-17T17:10:18.763 INFO:tasks.rados.rados.0.smithi063.stdout:3665: oid 18 version 1285 is already newer than 1284 2024-08-17T17:10:18.763 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 1285 (ObjNum 1379 snap 463 seq_num 1379) dirty exists 2024-08-17T17:10:18.763 INFO:tasks.rados.rados.0.smithi063.stdout:3665: left oid 18 (ObjNum 1379 snap 463 seq_num 1379) 2024-08-17T17:10:18.764 INFO:tasks.rados.rados.0.smithi063.stdout:3667: finishing rollback tid 1 to smithi06387500-14 2024-08-17T17:10:18.764 INFO:tasks.rados.rados.0.smithi063.stdout:3670: seq_num 1381 ranges {410467=578762,1696221=443147,2891153=319314} 2024-08-17T17:10:18.768 INFO:tasks.rados.rados.0.smithi063.stdout:3670: writing smithi06387500-9 from 410467 to 989229 tid 1 2024-08-17T17:10:18.774 INFO:tasks.rados.rados.0.smithi063.stdout:3670: writing smithi06387500-9 from 1696221 to 2139368 tid 2 2024-08-17T17:10:18.781 INFO:tasks.rados.rados.0.smithi063.stdout:3670: writing smithi06387500-9 from 2891153 to 3210467 tid 3 2024-08-17T17:10:18.781 INFO:tasks.rados.rados.0.smithi063.stdout:3665: done (5 left) 2024-08-17T17:10:18.781 INFO:tasks.rados.rados.0.smithi063.stdout:3671: read oid 36 snap -1 2024-08-17T17:10:18.781 INFO:tasks.rados.rados.0.smithi063.stdout:3671: expect deleted 2024-08-17T17:10:18.781 INFO:tasks.rados.rados.0.smithi063.stdout:3672: snap_create 2024-08-17T17:10:18.781 INFO:tasks.rados.rados.0.smithi063.stdout:3667: finishing rollback tid 2 to smithi06387500-14 2024-08-17T17:10:18.781 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 3591 (ObjNum 1196 snap 401 seq_num 1196) dirty exists 2024-08-17T17:10:18.781 INFO:tasks.rados.rados.0.smithi063.stdout:3666: expect (ObjNum 1215 snap 407 seq_num 1215) 2024-08-17T17:10:18.897 INFO:tasks.rados.rados.0.smithi063.stdout:3668: finishing write tid 1 to smithi06387500-47 2024-08-17T17:10:18.898 INFO:tasks.rados.rados.0.smithi063.stdout:3668: finishing write tid 2 to smithi06387500-47 2024-08-17T17:10:18.898 INFO:tasks.rados.rados.0.smithi063.stdout:3668: finishing write tid 3 to smithi06387500-47 2024-08-17T17:10:18.898 INFO:tasks.rados.rados.0.smithi063.stdout:3668: finishing write tid 4 to smithi06387500-47 2024-08-17T17:10:18.898 INFO:tasks.rados.rados.0.smithi063.stdout:3668: oid 47 updating version 0 to 827 2024-08-17T17:10:18.898 INFO:tasks.rados.rados.0.smithi063.stdout:3668: oid 47 updating version 827 to 828 2024-08-17T17:10:18.898 INFO:tasks.rados.rados.0.smithi063.stdout:3668: oid 47 version 828 is already newer than 826 2024-08-17T17:10:18.898 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 828 (ObjNum 1380 snap 463 seq_num 1380) dirty exists 2024-08-17T17:10:18.898 INFO:tasks.rados.rados.0.smithi063.stdout:3668: left oid 47 (ObjNum 1380 snap 463 seq_num 1380) 2024-08-17T17:10:18.898 INFO:tasks.rados.rados.0.smithi063.stdout:3669: expect (ObjNum 1304 snap 443 seq_num 1304) 2024-08-17T17:10:19.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:18 smithi063 ceph-mon[109843]: pgmap v942: 89 pgs: 89 active+clean; 247 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2024-08-17T17:10:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:18 smithi063 ceph-mon[114756]: pgmap v942: 89 pgs: 89 active+clean; 247 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2024-08-17T17:10:19.030 INFO:tasks.rados.rados.0.smithi063.stdout:3670: finishing write tid 1 to smithi06387500-9 2024-08-17T17:10:19.030 INFO:tasks.rados.rados.0.smithi063.stdout:3670: finishing write tid 2 to smithi06387500-9 2024-08-17T17:10:19.030 INFO:tasks.rados.rados.0.smithi063.stdout:3670: finishing write tid 3 to smithi06387500-9 2024-08-17T17:10:19.030 INFO:tasks.rados.rados.0.smithi063.stdout:3670: finishing write tid 4 to smithi06387500-9 2024-08-17T17:10:19.030 INFO:tasks.rados.rados.0.smithi063.stdout:3670: finishing write tid 5 to smithi06387500-9 2024-08-17T17:10:19.030 INFO:tasks.rados.rados.0.smithi063.stdout:3670: oid 9 updating version 0 to 1757 2024-08-17T17:10:19.030 INFO:tasks.rados.rados.0.smithi063.stdout:3670: oid 9 version 1757 is already newer than 1755 2024-08-17T17:10:19.030 INFO:tasks.rados.rados.0.smithi063.stdout:3670: oid 9 updating version 1757 to 1758 2024-08-17T17:10:19.030 INFO:tasks.rados.rados.0.smithi063.stdout:3670: oid 9 version 1758 is already newer than 1756 2024-08-17T17:10:19.030 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 1758 (ObjNum 1381 snap 463 seq_num 1381) dirty exists 2024-08-17T17:10:19.031 INFO:tasks.rados.rados.0.smithi063.stdout:3670: left oid 9 (ObjNum 1381 snap 463 seq_num 1381) 2024-08-17T17:10:19.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:18 smithi162 ceph-mon[104668]: pgmap v942: 89 pgs: 89 active+clean; 247 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.2 MiB/s wr, 10 op/s 2024-08-17T17:10:19.709 INFO:tasks.rados.rados.0.smithi063.stdout:3666: done (6 left) 2024-08-17T17:10:19.709 INFO:tasks.rados.rados.0.smithi063.stdout:3667: done (5 left) 2024-08-17T17:10:19.709 INFO:tasks.rados.rados.0.smithi063.stdout:3668: done (4 left) 2024-08-17T17:10:19.709 INFO:tasks.rados.rados.0.smithi063.stdout:3669: done (3 left) 2024-08-17T17:10:19.709 INFO:tasks.rados.rados.0.smithi063.stdout:3670: done (2 left) 2024-08-17T17:10:19.709 INFO:tasks.rados.rados.0.smithi063.stdout:3671: done (1 left) 2024-08-17T17:10:19.709 INFO:tasks.rados.rados.0.smithi063.stdout:3672: done (0 left) 2024-08-17T17:10:19.709 INFO:tasks.rados.rados.0.smithi063.stdout:3673: write oid 16 current snap is 464 2024-08-17T17:10:19.710 INFO:tasks.rados.rados.0.smithi063.stdout:3673: seq_num 1382 ranges {427274=654011,1754703=586805,2918576=308698} 2024-08-17T17:10:19.714 INFO:tasks.rados.rados.0.smithi063.stdout:3673: writing smithi06387500-16 from 427274 to 1081285 tid 1 2024-08-17T17:10:19.721 INFO:tasks.rados.rados.0.smithi063.stdout:3673: writing smithi06387500-16 from 1754703 to 2341508 tid 2 2024-08-17T17:10:19.726 INFO:tasks.rados.rados.0.smithi063.stdout:3673: writing smithi06387500-16 from 2918576 to 3227274 tid 3 2024-08-17T17:10:19.726 INFO:tasks.rados.rados.0.smithi063.stdout:3674: snap_remove snap 457 2024-08-17T17:10:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:19 smithi063 ceph-mon[109843]: osdmap e991: 8 total, 8 up, 8 in 2024-08-17T17:10:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:19 smithi063 ceph-mon[114756]: osdmap e991: 8 total, 8 up, 8 in 2024-08-17T17:10:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:19 smithi162 ceph-mon[104668]: osdmap e991: 8 total, 8 up, 8 in 2024-08-17T17:10:20.720 INFO:tasks.rados.rados.0.smithi063.stdout:3674: done (1 left) 2024-08-17T17:10:20.720 INFO:tasks.rados.rados.0.smithi063.stdout:3675: delete oid 28 current snap is 464 2024-08-17T17:10:20.720 INFO:tasks.rados.rados.0.smithi063.stdout:3673: finishing write tid 1 to smithi06387500-16 2024-08-17T17:10:20.720 INFO:tasks.rados.rados.0.smithi063.stdout:3673: finishing write tid 2 to smithi06387500-16 2024-08-17T17:10:20.721 INFO:tasks.rados.rados.0.smithi063.stdout:3673: finishing write tid 3 to smithi06387500-16 2024-08-17T17:10:20.721 INFO:tasks.rados.rados.0.smithi063.stdout:3673: finishing write tid 4 to smithi06387500-16 2024-08-17T17:10:20.721 INFO:tasks.rados.rados.0.smithi063.stdout:3673: finishing write tid 5 to smithi06387500-16 2024-08-17T17:10:20.721 INFO:tasks.rados.rados.0.smithi063.stdout:3673: oid 16 updating version 0 to 1352 2024-08-17T17:10:20.721 INFO:tasks.rados.rados.0.smithi063.stdout:3673: oid 16 version 1352 is already newer than 1351 2024-08-17T17:10:20.721 INFO:tasks.rados.rados.0.smithi063.stdout:3673: oid 16 updating version 1352 to 1353 2024-08-17T17:10:20.721 INFO:tasks.rados.rados.0.smithi063.stdout:3673: oid 16 version 1353 is already newer than 1350 2024-08-17T17:10:20.721 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 1353 (ObjNum 1382 snap 464 seq_num 1382) dirty exists 2024-08-17T17:10:20.721 INFO:tasks.rados.rados.0.smithi063.stdout:3673: left oid 16 (ObjNum 1382 snap 464 seq_num 1382) 2024-08-17T17:10:20.723 INFO:tasks.rados.rados.0.smithi063.stdout:3673: done (1 left) 2024-08-17T17:10:20.723 INFO:tasks.rados.rados.0.smithi063.stdout:3675: done (0 left) 2024-08-17T17:10:20.723 INFO:tasks.rados.rados.0.smithi063.stdout:3676: snap_create 2024-08-17T17:10:21.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:20 smithi063 ceph-mon[109843]: osdmap e992: 8 total, 8 up, 8 in 2024-08-17T17:10:21.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:20 smithi063 ceph-mon[109843]: pgmap v945: 89 pgs: 89 active+clean; 247 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 950 KiB/s wr, 9 op/s 2024-08-17T17:10:21.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:20 smithi063 ceph-mon[114756]: osdmap e992: 8 total, 8 up, 8 in 2024-08-17T17:10:21.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:20 smithi063 ceph-mon[114756]: pgmap v945: 89 pgs: 89 active+clean; 247 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 950 KiB/s wr, 9 op/s 2024-08-17T17:10:21.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:20 smithi162 ceph-mon[104668]: osdmap e992: 8 total, 8 up, 8 in 2024-08-17T17:10:21.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:20 smithi162 ceph-mon[104668]: pgmap v945: 89 pgs: 89 active+clean; 247 MiB data, 869 MiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 950 KiB/s wr, 9 op/s 2024-08-17T17:10:21.720 INFO:tasks.rados.rados.0.smithi063.stdout:3676: done (0 left) 2024-08-17T17:10:21.720 INFO:tasks.rados.rados.0.smithi063.stdout:3677: rollback oid 30 current snap is 465 2024-08-17T17:10:21.720 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 30 to 464 2024-08-17T17:10:21.721 INFO:tasks.rados.rados.0.smithi063.stdout:3678: snap_remove snap 446 2024-08-17T17:10:22.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:21 smithi063 ceph-mon[109843]: osdmap e993: 8 total, 8 up, 8 in 2024-08-17T17:10:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:21 smithi063 ceph-mon[114756]: osdmap e993: 8 total, 8 up, 8 in 2024-08-17T17:10:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:21 smithi162 ceph-mon[104668]: osdmap e993: 8 total, 8 up, 8 in 2024-08-17T17:10:22.674 INFO:tasks.rados.rados.0.smithi063.stdout:3678: done (1 left) 2024-08-17T17:10:22.675 INFO:tasks.rados.rados.0.smithi063.stdout:3679: read oid 9 snap 443 2024-08-17T17:10:22.675 INFO:tasks.rados.rados.0.smithi063.stdout:3679: expect (ObjNum 1298 snap 442 seq_num 1298) 2024-08-17T17:10:22.675 INFO:tasks.rados.rados.0.smithi063.stdout:3680: rollback oid 3 current snap is 465 2024-08-17T17:10:22.675 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 3 to 461 2024-08-17T17:10:22.675 INFO:tasks.rados.rados.0.smithi063.stdout:3677: finishing rollback tid 0 to smithi06387500-30 2024-08-17T17:10:22.675 INFO:tasks.rados.rados.0.smithi063.stdout:3677: finishing rollback tid 1 to smithi06387500-30 2024-08-17T17:10:22.675 INFO:tasks.rados.rados.0.smithi063.stdout:3677: finishing rollback tid 2 to smithi06387500-30 2024-08-17T17:10:22.675 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1766 (ObjNum 1215 snap 407 seq_num 1215) dirty exists 2024-08-17T17:10:22.675 INFO:tasks.rados.rados.0.smithi063.stdout:3677: done (2 left) 2024-08-17T17:10:22.675 INFO:tasks.rados.rados.0.smithi063.stdout:3681: rollback oid 33 current snap is 465 2024-08-17T17:10:22.675 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 33 to 450 2024-08-17T17:10:22.675 INFO:tasks.rados.rados.0.smithi063.stdout:3682: snap_remove snap 455 2024-08-17T17:10:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:22 smithi063 ceph-mon[114756]: osdmap e994: 8 total, 8 up, 8 in 2024-08-17T17:10:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:22 smithi063 ceph-mon[114756]: pgmap v948: 89 pgs: 89 active+clean; 259 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-08-17T17:10:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:22 smithi063 ceph-mon[114756]: osdmap e995: 8 total, 8 up, 8 in 2024-08-17T17:10:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:22 smithi063 ceph-mon[109843]: osdmap e994: 8 total, 8 up, 8 in 2024-08-17T17:10:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:22 smithi063 ceph-mon[109843]: pgmap v948: 89 pgs: 89 active+clean; 259 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-08-17T17:10:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:22 smithi063 ceph-mon[109843]: osdmap e995: 8 total, 8 up, 8 in 2024-08-17T17:10:23.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:22 smithi162 ceph-mon[104668]: osdmap e994: 8 total, 8 up, 8 in 2024-08-17T17:10:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:22 smithi162 ceph-mon[104668]: pgmap v948: 89 pgs: 89 active+clean; 259 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 1.9 MiB/s wr, 16 op/s 2024-08-17T17:10:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:22 smithi162 ceph-mon[104668]: osdmap e995: 8 total, 8 up, 8 in 2024-08-17T17:10:23.662 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:10:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:10:23] "GET /metrics HTTP/1.1" 200 36337 "" "Prometheus/2.51.0" 2024-08-17T17:10:23.678 INFO:tasks.rados.rados.0.smithi063.stdout:3682: done (3 left) 2024-08-17T17:10:23.678 INFO:tasks.rados.rados.0.smithi063.stdout:3683: rollback oid 48 current snap is 465 2024-08-17T17:10:23.678 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 48 to 461 2024-08-17T17:10:23.678 INFO:tasks.rados.rados.0.smithi063.stdout:3684: read oid 50 snap 453 2024-08-17T17:10:23.678 INFO:tasks.rados.rados.0.smithi063.stdout:3684: expect (ObjNum 1088 snap 363 seq_num 1088) 2024-08-17T17:10:23.678 INFO:tasks.rados.rados.0.smithi063.stdout:3685: write oid 36 current snap is 465 2024-08-17T17:10:23.678 INFO:tasks.rados.rados.0.smithi063.stdout:3685: seq_num 1384 ranges {460888=520862,1471667=590474,2573422=455407} 2024-08-17T17:10:23.684 INFO:tasks.rados.rados.0.smithi063.stdout:3685: writing smithi06387500-36 from 460888 to 981750 tid 1 2024-08-17T17:10:23.691 INFO:tasks.rados.rados.0.smithi063.stdout:3685: writing smithi06387500-36 from 1471667 to 2062141 tid 2 2024-08-17T17:10:23.697 INFO:tasks.rados.rados.0.smithi063.stdout:3685: writing smithi06387500-36 from 2573422 to 3028829 tid 3 2024-08-17T17:10:23.697 INFO:tasks.rados.rados.0.smithi063.stdout:3686: write oid 14 current snap is 465 2024-08-17T17:10:23.697 INFO:tasks.rados.rados.0.smithi063.stdout:3681: finishing rollback tid 0 to smithi06387500-33 2024-08-17T17:10:23.697 INFO:tasks.rados.rados.0.smithi063.stdout:3681: finishing rollback tid 1 to smithi06387500-33 2024-08-17T17:10:23.698 INFO:tasks.rados.rados.0.smithi063.stdout:3686: seq_num 1385 ranges {477695=596111,1530149=450485,2600845=666679} 2024-08-17T17:10:23.703 INFO:tasks.rados.rados.0.smithi063.stdout:3686: writing smithi06387500-14 from 477695 to 1073806 tid 1 2024-08-17T17:10:23.708 INFO:tasks.rados.rados.0.smithi063.stdout:3686: writing smithi06387500-14 from 1530149 to 1980634 tid 2 2024-08-17T17:10:23.715 INFO:tasks.rados.rados.0.smithi063.stdout:3686: writing smithi06387500-14 from 2600845 to 3267524 tid 3 2024-08-17T17:10:23.715 INFO:tasks.rados.rados.0.smithi063.stdout:3687: write oid 4 current snap is 465 2024-08-17T17:10:23.715 INFO:tasks.rados.rados.0.smithi063.stdout:3687: seq_num 1386 ranges {494502=671360,1588631=594143,2628268=477951} 2024-08-17T17:10:23.721 INFO:tasks.rados.rados.0.smithi063.stdout:3687: writing smithi06387500-4 from 494502 to 1165862 tid 1 2024-08-17T17:10:23.727 INFO:tasks.rados.rados.0.smithi063.stdout:3687: writing smithi06387500-4 from 1588631 to 2182774 tid 2 2024-08-17T17:10:23.732 INFO:tasks.rados.rados.0.smithi063.stdout:3687: writing smithi06387500-4 from 2628268 to 3106219 tid 3 2024-08-17T17:10:23.733 INFO:tasks.rados.rados.0.smithi063.stdout:3688: read oid 19 snap 450 2024-08-17T17:10:23.733 INFO:tasks.rados.rados.0.smithi063.stdout:3688: expect (ObjNum 1262 snap 426 seq_num 1262) 2024-08-17T17:10:23.733 INFO:tasks.rados.rados.0.smithi063.stdout:3689: snap_remove snap 427 2024-08-17T17:10:24.685 INFO:tasks.rados.rados.0.smithi063.stdout:3689: done (9 left) 2024-08-17T17:10:24.685 INFO:tasks.rados.rados.0.smithi063.stdout:3690: snap_remove snap 448 2024-08-17T17:10:25.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:24 smithi063 ceph-mon[114756]: osdmap e996: 8 total, 8 up, 8 in 2024-08-17T17:10:25.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:24 smithi063 ceph-mon[109843]: osdmap e996: 8 total, 8 up, 8 in 2024-08-17T17:10:25.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:24 smithi162 ceph-mon[104668]: osdmap e996: 8 total, 8 up, 8 in 2024-08-17T17:10:25.692 INFO:tasks.rados.rados.0.smithi063.stdout:3690: done (9 left) 2024-08-17T17:10:25.692 INFO:tasks.rados.rados.0.smithi063.stdout:3691: read oid 17 snap -1 2024-08-17T17:10:25.692 INFO:tasks.rados.rados.0.smithi063.stdout:3691: expect deleted 2024-08-17T17:10:25.693 INFO:tasks.rados.rados.0.smithi063.stdout:3692: read oid 25 snap -1 2024-08-17T17:10:25.693 INFO:tasks.rados.rados.0.smithi063.stdout:3692: expect deleted 2024-08-17T17:10:25.693 INFO:tasks.rados.rados.0.smithi063.stdout:3693: write oid 43 current snap is 465 2024-08-17T17:10:25.693 INFO:tasks.rados.rados.0.smithi063.stdout:3693: seq_num 1387 ranges {511309=746609,2047113=454154,3055691=255618} 2024-08-17T17:10:25.700 INFO:tasks.rados.rados.0.smithi063.stdout:3693: writing smithi06387500-43 from 511309 to 1257918 tid 1 2024-08-17T17:10:25.708 INFO:tasks.rados.rados.0.smithi063.stdout:3693: writing smithi06387500-43 from 2047113 to 2501267 tid 2 2024-08-17T17:10:25.712 INFO:tasks.rados.rados.0.smithi063.stdout:3693: writing smithi06387500-43 from 3055691 to 3311309 tid 3 2024-08-17T17:10:25.713 INFO:tasks.rados.rados.0.smithi063.stdout:3694: snap_remove snap 454 2024-08-17T17:10:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:25 smithi063 ceph-mon[114756]: pgmap v951: 89 pgs: 89 active+clean; 260 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-08-17T17:10:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:25 smithi063 ceph-mon[114756]: osdmap e997: 8 total, 8 up, 8 in 2024-08-17T17:10:26.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:25 smithi063 ceph-mon[109843]: pgmap v951: 89 pgs: 89 active+clean; 260 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-08-17T17:10:26.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:25 smithi063 ceph-mon[109843]: osdmap e997: 8 total, 8 up, 8 in 2024-08-17T17:10:26.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:25 smithi162 ceph-mon[104668]: pgmap v951: 89 pgs: 89 active+clean; 260 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 2.2 MiB/s wr, 16 op/s 2024-08-17T17:10:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:25 smithi162 ceph-mon[104668]: osdmap e997: 8 total, 8 up, 8 in 2024-08-17T17:10:26.700 INFO:tasks.rados.rados.0.smithi063.stdout:3694: done (12 left) 2024-08-17T17:10:26.701 INFO:tasks.rados.rados.0.smithi063.stdout:3695: read oid 5 snap 440 2024-08-17T17:10:26.701 INFO:tasks.rados.rados.0.smithi063.stdout:3695: expect (ObjNum 1275 snap 434 seq_num 1275) 2024-08-17T17:10:26.701 INFO:tasks.rados.rados.0.smithi063.stdout:3696: snap_remove snap 458 2024-08-17T17:10:27.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:26 smithi063 ceph-mon[114756]: osdmap e998: 8 total, 8 up, 8 in 2024-08-17T17:10:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:26 smithi063 ceph-mon[109843]: osdmap e998: 8 total, 8 up, 8 in 2024-08-17T17:10:27.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:26 smithi162 ceph-mon[104668]: osdmap e998: 8 total, 8 up, 8 in 2024-08-17T17:10:27.678 INFO:tasks.rados.rados.0.smithi063.stdout:3696: done (13 left) 2024-08-17T17:10:27.678 INFO:tasks.rados.rados.0.smithi063.stdout:3697: write oid 16 current snap is 465 2024-08-17T17:10:27.678 INFO:tasks.rados.rados.0.smithi063.stdout:3697: seq_num 1388 ranges {528116=421858,1589242=597812,2966761=361355} 2024-08-17T17:10:27.684 INFO:tasks.rados.rados.0.smithi063.stdout:3697: writing smithi06387500-16 from 528116 to 949974 tid 1 2024-08-17T17:10:27.692 INFO:tasks.rados.rados.0.smithi063.stdout:3697: writing smithi06387500-16 from 1589242 to 2187054 tid 2 2024-08-17T17:10:27.699 INFO:tasks.rados.rados.0.smithi063.stdout:3697: writing smithi06387500-16 from 2966761 to 3328116 tid 3 2024-08-17T17:10:27.699 INFO:tasks.rados.rados.0.smithi063.stdout:3698: rollback oid 35 current snap is 465 2024-08-17T17:10:27.699 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 35 to 460 2024-08-17T17:10:27.699 INFO:tasks.rados.rados.0.smithi063.stdout:3681: finishing rollback tid 2 to smithi06387500-33 2024-08-17T17:10:27.699 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 1231 (ObjNum 1323 snap 449 seq_num 1323) dirty exists 2024-08-17T17:10:27.699 INFO:tasks.rados.rados.0.smithi063.stdout:3679: expect (ObjNum 1298 snap 442 seq_num 1298) 2024-08-17T17:10:27.829 INFO:tasks.rados.rados.0.smithi063.stdout:3680: finishing rollback tid 0 to smithi06387500-3 2024-08-17T17:10:27.829 INFO:tasks.rados.rados.0.smithi063.stdout:3680: finishing rollback tid 1 to smithi06387500-3 2024-08-17T17:10:27.829 INFO:tasks.rados.rados.0.smithi063.stdout:3680: finishing rollback tid 2 to smithi06387500-3 2024-08-17T17:10:27.829 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 1266 (ObjNum 1339 snap 451 seq_num 1339) dirty exists 2024-08-17T17:10:27.829 INFO:tasks.rados.rados.0.smithi063.stdout:3683: finishing rollback tid 1 to smithi06387500-48 2024-08-17T17:10:27.829 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 3567 (ObjNum 33 snap 0 seq_num 1806787856) dirty dne 2024-08-17T17:10:27.829 INFO:tasks.rados.rados.0.smithi063.stdout:3684: expect (ObjNum 1088 snap 363 seq_num 1088) 2024-08-17T17:10:27.929 INFO:tasks.rados.rados.0.smithi063.stdout:3685: finishing write tid 1 to smithi06387500-36 2024-08-17T17:10:27.929 INFO:tasks.rados.rados.0.smithi063.stdout:3685: finishing write tid 2 to smithi06387500-36 2024-08-17T17:10:27.929 INFO:tasks.rados.rados.0.smithi063.stdout:3685: finishing write tid 3 to smithi06387500-36 2024-08-17T17:10:27.929 INFO:tasks.rados.rados.0.smithi063.stdout:3685: finishing write tid 4 to smithi06387500-36 2024-08-17T17:10:27.929 INFO:tasks.rados.rados.0.smithi063.stdout:3685: finishing write tid 5 to smithi06387500-36 2024-08-17T17:10:27.929 INFO:tasks.rados.rados.0.smithi063.stdout:3685: oid 36 updating version 0 to 3617 2024-08-17T17:10:27.930 INFO:tasks.rados.rados.0.smithi063.stdout:3685: oid 36 version 3617 is already newer than 3613 2024-08-17T17:10:27.930 INFO:tasks.rados.rados.0.smithi063.stdout:3685: oid 36 updating version 3617 to 3618 2024-08-17T17:10:27.930 INFO:tasks.rados.rados.0.smithi063.stdout:3685: oid 36 version 3618 is already newer than 3614 2024-08-17T17:10:27.930 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 3618 (ObjNum 1384 snap 465 seq_num 1384) dirty exists 2024-08-17T17:10:27.930 INFO:tasks.rados.rados.0.smithi063.stdout:3685: left oid 36 (ObjNum 1384 snap 465 seq_num 1384) 2024-08-17T17:10:27.930 INFO:tasks.rados.rados.0.smithi063.stdout:3686: finishing write tid 1 to smithi06387500-14 2024-08-17T17:10:27.930 INFO:tasks.rados.rados.0.smithi063.stdout:3679: done (14 left) 2024-08-17T17:10:27.930 INFO:tasks.rados.rados.0.smithi063.stdout:3680: done (13 left) 2024-08-17T17:10:27.930 INFO:tasks.rados.rados.0.smithi063.stdout:3681: done (12 left) 2024-08-17T17:10:27.930 INFO:tasks.rados.rados.0.smithi063.stdout:3683: done (11 left) 2024-08-17T17:10:27.931 INFO:tasks.rados.rados.0.smithi063.stdout:3684: done (10 left) 2024-08-17T17:10:27.931 INFO:tasks.rados.rados.0.smithi063.stdout:3685: done (9 left) 2024-08-17T17:10:27.931 INFO:tasks.rados.rados.0.smithi063.stdout:3699: write oid 44 current snap is 465 2024-08-17T17:10:27.931 INFO:tasks.rados.rados.0.smithi063.stdout:3686: finishing write tid 2 to smithi06387500-14 2024-08-17T17:10:27.931 INFO:tasks.rados.rados.0.smithi063.stdout:3699: seq_num 1389 ranges {544923=497107,1647724=457823,2710537=428120} 2024-08-17T17:10:27.934 INFO:tasks.rados.rados.0.smithi063.stdout:3699: writing smithi06387500-44 from 544923 to 1042030 tid 1 2024-08-17T17:10:27.942 INFO:tasks.rados.rados.0.smithi063.stdout:3699: writing smithi06387500-44 from 1647724 to 2105547 tid 2 2024-08-17T17:10:27.947 INFO:tasks.rados.rados.0.smithi063.stdout:3699: writing smithi06387500-44 from 2710537 to 3138657 tid 3 2024-08-17T17:10:27.947 INFO:tasks.rados.rados.0.smithi063.stdout:3700: delete oid 21 current snap is 465 2024-08-17T17:10:27.947 INFO:tasks.rados.rados.0.smithi063.stdout:3686: finishing write tid 3 to smithi06387500-14 2024-08-17T17:10:27.947 INFO:tasks.rados.rados.0.smithi063.stdout:3686: finishing write tid 4 to smithi06387500-14 2024-08-17T17:10:27.947 INFO:tasks.rados.rados.0.smithi063.stdout:3686: finishing write tid 5 to smithi06387500-14 2024-08-17T17:10:27.947 INFO:tasks.rados.rados.0.smithi063.stdout:3686: oid 14 updating version 0 to 3620 2024-08-17T17:10:27.947 INFO:tasks.rados.rados.0.smithi063.stdout:3686: oid 14 updating version 3620 to 3621 2024-08-17T17:10:27.947 INFO:tasks.rados.rados.0.smithi063.stdout:3686: oid 14 updating version 3621 to 3622 2024-08-17T17:10:27.947 INFO:tasks.rados.rados.0.smithi063.stdout:3686: oid 14 updating version 3622 to 3623 2024-08-17T17:10:27.948 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 3623 (ObjNum 1385 snap 465 seq_num 1385) dirty exists 2024-08-17T17:10:27.948 INFO:tasks.rados.rados.0.smithi063.stdout:3686: left oid 14 (ObjNum 1385 snap 465 seq_num 1385) 2024-08-17T17:10:27.948 INFO:tasks.rados.rados.0.smithi063.stdout:3687: finishing write tid 1 to smithi06387500-4 2024-08-17T17:10:27.948 INFO:tasks.rados.rados.0.smithi063.stdout:3687: finishing write tid 2 to smithi06387500-4 2024-08-17T17:10:27.948 INFO:tasks.rados.rados.0.smithi063.stdout:3687: finishing write tid 3 to smithi06387500-4 2024-08-17T17:10:27.948 INFO:tasks.rados.rados.0.smithi063.stdout:3687: finishing write tid 4 to smithi06387500-4 2024-08-17T17:10:27.948 INFO:tasks.rados.rados.0.smithi063.stdout:3687: finishing write tid 5 to smithi06387500-4 2024-08-17T17:10:27.948 INFO:tasks.rados.rados.0.smithi063.stdout:3687: oid 4 updating version 0 to 3628 2024-08-17T17:10:27.948 INFO:tasks.rados.rados.0.smithi063.stdout:3687: oid 4 version 3628 is already newer than 3626 2024-08-17T17:10:27.948 INFO:tasks.rados.rados.0.smithi063.stdout:3687: oid 4 version 3628 is already newer than 3625 2024-08-17T17:10:27.948 INFO:tasks.rados.rados.0.smithi063.stdout:3687: oid 4 version 3628 is already newer than 3627 2024-08-17T17:10:27.948 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 3628 (ObjNum 1386 snap 465 seq_num 1386) dirty exists 2024-08-17T17:10:27.948 INFO:tasks.rados.rados.0.smithi063.stdout:3687: left oid 4 (ObjNum 1386 snap 465 seq_num 1386) 2024-08-17T17:10:27.949 INFO:tasks.rados.rados.0.smithi063.stdout:3688: expect (ObjNum 1262 snap 426 seq_num 1262) 2024-08-17T17:10:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:27 smithi063 ceph-mon[114756]: pgmap v954: 89 pgs: 89 active+clean; 248 MiB data, 887 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 265 KiB/s wr, 6 op/s 2024-08-17T17:10:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:27 smithi063 ceph-mon[114756]: osdmap e999: 8 total, 8 up, 8 in 2024-08-17T17:10:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:27 smithi063 ceph-mon[114756]: osdmap e1000: 8 total, 8 up, 8 in 2024-08-17T17:10:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:27 smithi063 ceph-mon[109843]: pgmap v954: 89 pgs: 89 active+clean; 248 MiB data, 887 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 265 KiB/s wr, 6 op/s 2024-08-17T17:10:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:27 smithi063 ceph-mon[109843]: osdmap e999: 8 total, 8 up, 8 in 2024-08-17T17:10:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:27 smithi063 ceph-mon[109843]: osdmap e1000: 8 total, 8 up, 8 in 2024-08-17T17:10:28.081 INFO:tasks.rados.rados.0.smithi063.stdout:3693: finishing write tid 1 to smithi06387500-43 2024-08-17T17:10:28.081 INFO:tasks.rados.rados.0.smithi063.stdout:3693: finishing write tid 2 to smithi06387500-43 2024-08-17T17:10:28.082 INFO:tasks.rados.rados.0.smithi063.stdout:3693: finishing write tid 3 to smithi06387500-43 2024-08-17T17:10:28.082 INFO:tasks.rados.rados.0.smithi063.stdout:3693: finishing write tid 4 to smithi06387500-43 2024-08-17T17:10:28.082 INFO:tasks.rados.rados.0.smithi063.stdout:3693: finishing write tid 5 to smithi06387500-43 2024-08-17T17:10:28.082 INFO:tasks.rados.rados.0.smithi063.stdout:3693: oid 43 updating version 0 to 1788 2024-08-17T17:10:28.082 INFO:tasks.rados.rados.0.smithi063.stdout:3693: oid 43 updating version 1788 to 1790 2024-08-17T17:10:28.082 INFO:tasks.rados.rados.0.smithi063.stdout:3693: oid 43 version 1790 is already newer than 1787 2024-08-17T17:10:28.082 INFO:tasks.rados.rados.0.smithi063.stdout:3693: oid 43 version 1790 is already newer than 1789 2024-08-17T17:10:28.082 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 1790 (ObjNum 1387 snap 465 seq_num 1387) dirty exists 2024-08-17T17:10:28.082 INFO:tasks.rados.rados.0.smithi063.stdout:3693: left oid 43 (ObjNum 1387 snap 465 seq_num 1387) 2024-08-17T17:10:28.082 INFO:tasks.rados.rados.0.smithi063.stdout:3695: expect (ObjNum 1275 snap 434 seq_num 1275) 2024-08-17T17:10:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:27 smithi162 ceph-mon[104668]: pgmap v954: 89 pgs: 89 active+clean; 248 MiB data, 887 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 265 KiB/s wr, 6 op/s 2024-08-17T17:10:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:27 smithi162 ceph-mon[104668]: osdmap e999: 8 total, 8 up, 8 in 2024-08-17T17:10:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:27 smithi162 ceph-mon[104668]: osdmap e1000: 8 total, 8 up, 8 in 2024-08-17T17:10:28.235 INFO:tasks.rados.rados.0.smithi063.stdout:3697: finishing write tid 1 to smithi06387500-16 2024-08-17T17:10:28.235 INFO:tasks.rados.rados.0.smithi063.stdout:3697: finishing write tid 2 to smithi06387500-16 2024-08-17T17:10:28.235 INFO:tasks.rados.rados.0.smithi063.stdout:3697: finishing write tid 3 to smithi06387500-16 2024-08-17T17:10:28.235 INFO:tasks.rados.rados.0.smithi063.stdout:3697: finishing write tid 4 to smithi06387500-16 2024-08-17T17:10:28.235 INFO:tasks.rados.rados.0.smithi063.stdout:3697: finishing write tid 5 to smithi06387500-16 2024-08-17T17:10:28.235 INFO:tasks.rados.rados.0.smithi063.stdout:3697: oid 16 updating version 0 to 1378 2024-08-17T17:10:28.236 INFO:tasks.rados.rados.0.smithi063.stdout:3697: oid 16 updating version 1378 to 1379 2024-08-17T17:10:28.236 INFO:tasks.rados.rados.0.smithi063.stdout:3697: oid 16 updating version 1379 to 1380 2024-08-17T17:10:28.236 INFO:tasks.rados.rados.0.smithi063.stdout:3697: oid 16 version 1380 is already newer than 1375 2024-08-17T17:10:28.236 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 1380 (ObjNum 1388 snap 465 seq_num 1388) dirty exists 2024-08-17T17:10:28.236 INFO:tasks.rados.rados.0.smithi063.stdout:3697: left oid 16 (ObjNum 1388 snap 465 seq_num 1388) 2024-08-17T17:10:28.236 INFO:tasks.rados.rados.0.smithi063.stdout:3698: finishing rollback tid 1 to smithi06387500-35 2024-08-17T17:10:28.236 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-08-17T17:10:28.236 INFO:tasks.rados.rados.0.smithi063.stdout:3699: finishing write tid 1 to smithi06387500-44 2024-08-17T17:10:28.236 INFO:tasks.rados.rados.0.smithi063.stdout:3699: finishing write tid 2 to smithi06387500-44 2024-08-17T17:10:28.236 INFO:tasks.rados.rados.0.smithi063.stdout:3686: done (10 left) 2024-08-17T17:10:28.236 INFO:tasks.rados.rados.0.smithi063.stdout:3687: done (9 left) 2024-08-17T17:10:28.237 INFO:tasks.rados.rados.0.smithi063.stdout:3688: done (8 left) 2024-08-17T17:10:28.237 INFO:tasks.rados.rados.0.smithi063.stdout:3691: done (7 left) 2024-08-17T17:10:28.237 INFO:tasks.rados.rados.0.smithi063.stdout:3692: done (6 left) 2024-08-17T17:10:28.237 INFO:tasks.rados.rados.0.smithi063.stdout:3693: done (5 left) 2024-08-17T17:10:28.237 INFO:tasks.rados.rados.0.smithi063.stdout:3695: done (4 left) 2024-08-17T17:10:28.237 INFO:tasks.rados.rados.0.smithi063.stdout:3697: done (3 left) 2024-08-17T17:10:28.237 INFO:tasks.rados.rados.0.smithi063.stdout:3698: done (2 left) 2024-08-17T17:10:28.237 INFO:tasks.rados.rados.0.smithi063.stdout:3700: done (1 left) 2024-08-17T17:10:28.237 INFO:tasks.rados.rados.0.smithi063.stdout:3701: snap_create 2024-08-17T17:10:28.237 INFO:tasks.rados.rados.0.smithi063.stdout:3699: finishing write tid 3 to smithi06387500-44 2024-08-17T17:10:28.237 INFO:tasks.rados.rados.0.smithi063.stdout:3699: finishing write tid 4 to smithi06387500-44 2024-08-17T17:10:28.237 INFO:tasks.rados.rados.0.smithi063.stdout:3699: finishing write tid 5 to smithi06387500-44 2024-08-17T17:10:28.238 INFO:tasks.rados.rados.0.smithi063.stdout:3699: oid 44 updating version 0 to 516 2024-08-17T17:10:28.238 INFO:tasks.rados.rados.0.smithi063.stdout:3699: oid 44 updating version 516 to 517 2024-08-17T17:10:28.238 INFO:tasks.rados.rados.0.smithi063.stdout:3699: oid 44 version 517 is already newer than 515 2024-08-17T17:10:28.238 INFO:tasks.rados.rados.0.smithi063.stdout:3699: oid 44 version 517 is already newer than 514 2024-08-17T17:10:28.238 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 517 (ObjNum 1389 snap 465 seq_num 1389) dirty exists 2024-08-17T17:10:28.238 INFO:tasks.rados.rados.0.smithi063.stdout:3699: left oid 44 (ObjNum 1389 snap 465 seq_num 1389) 2024-08-17T17:10:28.707 INFO:tasks.rados.rados.0.smithi063.stdout:3699: done (1 left) 2024-08-17T17:10:28.707 INFO:tasks.rados.rados.0.smithi063.stdout:3701: done (0 left) 2024-08-17T17:10:28.707 INFO:tasks.rados.rados.0.smithi063.stdout:3702: read oid 18 snap -1 2024-08-17T17:10:28.707 INFO:tasks.rados.rados.0.smithi063.stdout:3702: expect (ObjNum 1379 snap 463 seq_num 1379) 2024-08-17T17:10:28.707 INFO:tasks.rados.rados.0.smithi063.stdout:3703: rollback oid 32 current snap is 466 2024-08-17T17:10:28.707 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 32 to 461 2024-08-17T17:10:28.707 INFO:tasks.rados.rados.0.smithi063.stdout:3704: write oid 8 current snap is 466 2024-08-17T17:10:28.708 INFO:tasks.rados.rados.0.smithi063.stdout:3704: seq_num 1391 ranges {578537=647605,1764688=745139,3049030=329507} 2024-08-17T17:10:28.713 INFO:tasks.rados.rados.0.smithi063.stdout:3704: writing smithi06387500-8 from 578537 to 1226142 tid 1 2024-08-17T17:10:28.720 INFO:tasks.rados.rados.0.smithi063.stdout:3704: writing smithi06387500-8 from 1764688 to 2509827 tid 2 2024-08-17T17:10:28.726 INFO:tasks.rados.rados.0.smithi063.stdout:3704: writing smithi06387500-8 from 3049030 to 3378537 tid 3 2024-08-17T17:10:28.726 INFO:tasks.rados.rados.0.smithi063.stdout:3705: read oid 16 snap -1 2024-08-17T17:10:28.726 INFO:tasks.rados.rados.0.smithi063.stdout:3705: expect (ObjNum 1388 snap 465 seq_num 1388) 2024-08-17T17:10:28.726 INFO:tasks.rados.rados.0.smithi063.stdout:3706: delete oid 24 current snap is 466 2024-08-17T17:10:28.726 INFO:tasks.rados.rados.0.smithi063.stdout:3703: finishing rollback tid 1 to smithi06387500-32 2024-08-17T17:10:28.726 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 818 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-08-17T17:10:28.726 INFO:tasks.rados.rados.0.smithi063.stdout:3702: expect (ObjNum 1379 snap 463 seq_num 1379) 2024-08-17T17:10:28.865 INFO:tasks.rados.rados.0.smithi063.stdout:3704: finishing write tid 1 to smithi06387500-8 2024-08-17T17:10:28.865 INFO:tasks.rados.rados.0.smithi063.stdout:3704: finishing write tid 2 to smithi06387500-8 2024-08-17T17:10:28.865 INFO:tasks.rados.rados.0.smithi063.stdout:3704: finishing write tid 3 to smithi06387500-8 2024-08-17T17:10:28.865 INFO:tasks.rados.rados.0.smithi063.stdout:3704: finishing write tid 4 to smithi06387500-8 2024-08-17T17:10:28.865 INFO:tasks.rados.rados.0.smithi063.stdout:3704: finishing write tid 5 to smithi06387500-8 2024-08-17T17:10:28.865 INFO:tasks.rados.rados.0.smithi063.stdout:3704: oid 8 updating version 0 to 1257 2024-08-17T17:10:28.865 INFO:tasks.rados.rados.0.smithi063.stdout:3704: oid 8 updating version 1257 to 1259 2024-08-17T17:10:28.865 INFO:tasks.rados.rados.0.smithi063.stdout:3704: oid 8 version 1259 is already newer than 1258 2024-08-17T17:10:28.865 INFO:tasks.rados.rados.0.smithi063.stdout:3704: oid 8 version 1259 is already newer than 1256 2024-08-17T17:10:28.866 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 8 v 1259 (ObjNum 1391 snap 466 seq_num 1391) dirty exists 2024-08-17T17:10:28.866 INFO:tasks.rados.rados.0.smithi063.stdout:3704: left oid 8 (ObjNum 1391 snap 466 seq_num 1391) 2024-08-17T17:10:28.866 INFO:tasks.rados.rados.0.smithi063.stdout:3705: expect (ObjNum 1388 snap 465 seq_num 1388) 2024-08-17T17:10:29.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:28 smithi063 ceph-mon[114756]: pgmap v957: 89 pgs: 89 active+clean; 233 MiB data, 872 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-08-17T17:10:29.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:28 smithi063 ceph-mon[109843]: pgmap v957: 89 pgs: 89 active+clean; 233 MiB data, 872 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-08-17T17:10:29.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:28 smithi162 ceph-mon[104668]: pgmap v957: 89 pgs: 89 active+clean; 233 MiB data, 872 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.5 MiB/s wr, 14 op/s 2024-08-17T17:10:29.103 INFO:tasks.rados.rados.0.smithi063.stdout:3702: done (4 left) 2024-08-17T17:10:29.103 INFO:tasks.rados.rados.0.smithi063.stdout:3703: done (3 left) 2024-08-17T17:10:29.103 INFO:tasks.rados.rados.0.smithi063.stdout:3704: done (2 left) 2024-08-17T17:10:29.103 INFO:tasks.rados.rados.0.smithi063.stdout:3705: done (1 left) 2024-08-17T17:10:29.103 INFO:tasks.rados.rados.0.smithi063.stdout:3706: done (0 left) 2024-08-17T17:10:29.103 INFO:tasks.rados.rados.0.smithi063.stdout:3707: rollback oid 5 current snap is 466 2024-08-17T17:10:29.103 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 441 2024-08-17T17:10:29.103 INFO:tasks.rados.rados.0.smithi063.stdout:3708: rollback oid 20 current snap is 466 2024-08-17T17:10:29.103 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 20 to 464 2024-08-17T17:10:29.103 INFO:tasks.rados.rados.0.smithi063.stdout:3709: write oid 23 current snap is 466 2024-08-17T17:10:29.103 INFO:tasks.rados.rados.0.smithi063.stdout:3709: seq_num 1393 ranges {612151=514456,1881652=748808,3103876=308275} 2024-08-17T17:10:29.109 INFO:tasks.rados.rados.0.smithi063.stdout:3709: writing smithi06387500-23 from 612151 to 1126607 tid 1 2024-08-17T17:10:29.117 INFO:tasks.rados.rados.0.smithi063.stdout:3709: writing smithi06387500-23 from 1881652 to 2630460 tid 2 2024-08-17T17:10:29.122 INFO:tasks.rados.rados.0.smithi063.stdout:3709: writing smithi06387500-23 from 3103876 to 3412151 tid 3 2024-08-17T17:10:29.122 INFO:tasks.rados.rados.0.smithi063.stdout:3710: read oid 34 snap -1 2024-08-17T17:10:29.122 INFO:tasks.rados.rados.0.smithi063.stdout:3710: expect deleted 2024-08-17T17:10:29.122 INFO:tasks.rados.rados.0.smithi063.stdout:3711: write oid 44 current snap is 466 2024-08-17T17:10:29.122 INFO:tasks.rados.rados.0.smithi063.stdout:3708: finishing rollback tid 0 to smithi06387500-20 2024-08-17T17:10:29.122 INFO:tasks.rados.rados.0.smithi063.stdout:3711: seq_num 1394 ranges {628958=589705,1940134=608819,3131299=297659} 2024-08-17T17:10:29.128 INFO:tasks.rados.rados.0.smithi063.stdout:3711: writing smithi06387500-44 from 628958 to 1218663 tid 1 2024-08-17T17:10:29.136 INFO:tasks.rados.rados.0.smithi063.stdout:3711: writing smithi06387500-44 from 1940134 to 2548953 tid 2 2024-08-17T17:10:29.141 INFO:tasks.rados.rados.0.smithi063.stdout:3711: writing smithi06387500-44 from 3131299 to 3428958 tid 3 2024-08-17T17:10:29.141 INFO:tasks.rados.rados.0.smithi063.stdout:3712: delete oid 7 current snap is 466 2024-08-17T17:10:29.141 INFO:tasks.rados.rados.0.smithi063.stdout:3708: finishing rollback tid 1 to smithi06387500-20 2024-08-17T17:10:29.141 INFO:tasks.rados.rados.0.smithi063.stdout:3708: finishing rollback tid 2 to smithi06387500-20 2024-08-17T17:10:29.142 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 944 (ObjNum 1376 snap 463 seq_num 1376) dirty exists 2024-08-17T17:10:29.142 INFO:tasks.rados.rados.0.smithi063.stdout:3707: finishing rollback tid 0 to smithi06387500-5 2024-08-17T17:10:29.142 INFO:tasks.rados.rados.0.smithi063.stdout:3707: finishing rollback tid 1 to smithi06387500-5 2024-08-17T17:10:29.142 INFO:tasks.rados.rados.0.smithi063.stdout:3707: finishing rollback tid 2 to smithi06387500-5 2024-08-17T17:10:29.142 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 1985 (ObjNum 1293 snap 441 seq_num 1293) dirty exists 2024-08-17T17:10:29.142 INFO:tasks.rados.rados.0.smithi063.stdout:3709: finishing write tid 1 to smithi06387500-23 2024-08-17T17:10:29.142 INFO:tasks.rados.rados.0.smithi063.stdout:3709: finishing write tid 2 to smithi06387500-23 2024-08-17T17:10:29.142 INFO:tasks.rados.rados.0.smithi063.stdout:3709: finishing write tid 3 to smithi06387500-23 2024-08-17T17:10:29.142 INFO:tasks.rados.rados.0.smithi063.stdout:3709: finishing write tid 4 to smithi06387500-23 2024-08-17T17:10:29.142 INFO:tasks.rados.rados.0.smithi063.stdout:3709: finishing write tid 5 to smithi06387500-23 2024-08-17T17:10:29.142 INFO:tasks.rados.rados.0.smithi063.stdout:3709: oid 23 updating version 0 to 1989 2024-08-17T17:10:29.143 INFO:tasks.rados.rados.0.smithi063.stdout:3709: oid 23 version 1989 is already newer than 1988 2024-08-17T17:10:29.143 INFO:tasks.rados.rados.0.smithi063.stdout:3709: oid 23 updating version 1989 to 1990 2024-08-17T17:10:29.143 INFO:tasks.rados.rados.0.smithi063.stdout:3709: oid 23 version 1990 is already newer than 1987 2024-08-17T17:10:29.143 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 1990 (ObjNum 1393 snap 466 seq_num 1393) dirty exists 2024-08-17T17:10:29.143 INFO:tasks.rados.rados.0.smithi063.stdout:3709: left oid 23 (ObjNum 1393 snap 466 seq_num 1393) 2024-08-17T17:10:29.143 INFO:tasks.rados.rados.0.smithi063.stdout:3711: finishing write tid 1 to smithi06387500-44 2024-08-17T17:10:29.143 INFO:tasks.rados.rados.0.smithi063.stdout:3711: finishing write tid 2 to smithi06387500-44 2024-08-17T17:10:29.144 INFO:tasks.rados.rados.0.smithi063.stdout:3711: finishing write tid 3 to smithi06387500-44 2024-08-17T17:10:29.144 INFO:tasks.rados.rados.0.smithi063.stdout:3707: done (5 left) 2024-08-17T17:10:29.144 INFO:tasks.rados.rados.0.smithi063.stdout:3708: done (4 left) 2024-08-17T17:10:29.144 INFO:tasks.rados.rados.0.smithi063.stdout:3709: done (3 left) 2024-08-17T17:10:29.144 INFO:tasks.rados.rados.0.smithi063.stdout:3710: done (2 left) 2024-08-17T17:10:29.144 INFO:tasks.rados.rados.0.smithi063.stdout:3712: done (1 left) 2024-08-17T17:10:29.145 INFO:tasks.rados.rados.0.smithi063.stdout:3713: rollback oid 17 current snap is 466 2024-08-17T17:10:29.145 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 17 to 453 2024-08-17T17:10:29.145 INFO:tasks.rados.rados.0.smithi063.stdout:3714: snap_remove snap 450 2024-08-17T17:10:29.720 INFO:tasks.rados.rados.0.smithi063.stdout:3711: finishing write tid 4 to smithi06387500-44 2024-08-17T17:10:29.720 INFO:tasks.rados.rados.0.smithi063.stdout:3711: finishing write tid 5 to smithi06387500-44 2024-08-17T17:10:29.720 INFO:tasks.rados.rados.0.smithi063.stdout:3711: oid 44 updating version 0 to 522 2024-08-17T17:10:29.720 INFO:tasks.rados.rados.0.smithi063.stdout:3711: oid 44 version 522 is already newer than 520 2024-08-17T17:10:29.720 INFO:tasks.rados.rados.0.smithi063.stdout:3711: oid 44 version 522 is already newer than 521 2024-08-17T17:10:29.720 INFO:tasks.rados.rados.0.smithi063.stdout:3711: oid 44 version 522 is already newer than 519 2024-08-17T17:10:29.721 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 522 (ObjNum 1394 snap 466 seq_num 1394) dirty exists 2024-08-17T17:10:29.721 INFO:tasks.rados.rados.0.smithi063.stdout:3711: left oid 44 (ObjNum 1394 snap 466 seq_num 1394) 2024-08-17T17:10:29.721 INFO:tasks.rados.rados.0.smithi063.stdout:3713: finishing rollback tid 1 to smithi06387500-17 2024-08-17T17:10:29.721 INFO:tasks.rados.rados.0.smithi063.stdout:3713: finishing rollback tid 2 to smithi06387500-17 2024-08-17T17:10:29.721 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 1288 (ObjNum 1333 snap 451 seq_num 1333) dirty exists 2024-08-17T17:10:29.721 INFO:tasks.rados.rados.0.smithi063.stdout:3711: done (2 left) 2024-08-17T17:10:29.721 INFO:tasks.rados.rados.0.smithi063.stdout:3713: done (1 left) 2024-08-17T17:10:29.721 INFO:tasks.rados.rados.0.smithi063.stdout:3714: done (0 left) 2024-08-17T17:10:29.721 INFO:tasks.rados.rados.0.smithi063.stdout:3715: read oid 20 snap -1 2024-08-17T17:10:29.721 INFO:tasks.rados.rados.0.smithi063.stdout:3715: expect (ObjNum 1376 snap 463 seq_num 1376) 2024-08-17T17:10:29.721 INFO:tasks.rados.rados.0.smithi063.stdout:3716: snap_remove snap 464 2024-08-17T17:10:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:29 smithi063 ceph-mon[114756]: osdmap e1001: 8 total, 8 up, 8 in 2024-08-17T17:10:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:29 smithi063 ceph-mon[109843]: osdmap e1001: 8 total, 8 up, 8 in 2024-08-17T17:10:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:29 smithi162 ceph-mon[104668]: osdmap e1001: 8 total, 8 up, 8 in 2024-08-17T17:10:30.731 INFO:tasks.rados.rados.0.smithi063.stdout:3716: done (1 left) 2024-08-17T17:10:30.732 INFO:tasks.rados.rados.0.smithi063.stdout:3717: delete oid 39 current snap is 466 2024-08-17T17:10:30.732 INFO:tasks.rados.rados.0.smithi063.stdout:3715: expect (ObjNum 1376 snap 463 seq_num 1376) 2024-08-17T17:10:30.964 INFO:tasks.rados.rados.0.smithi063.stdout:3715: done (1 left) 2024-08-17T17:10:30.964 INFO:tasks.rados.rados.0.smithi063.stdout:3717: done (0 left) 2024-08-17T17:10:30.964 INFO:tasks.rados.rados.0.smithi063.stdout:3718: snap_create 2024-08-17T17:10:30.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:30 smithi162 ceph-mon[104668]: osdmap e1002: 8 total, 8 up, 8 in 2024-08-17T17:10:30.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:30 smithi162 ceph-mon[104668]: pgmap v960: 89 pgs: 89 active+clean; 230 MiB data, 873 MiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-08-17T17:10:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:30 smithi063 ceph-mon[114756]: osdmap e1002: 8 total, 8 up, 8 in 2024-08-17T17:10:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:30 smithi063 ceph-mon[114756]: pgmap v960: 89 pgs: 89 active+clean; 230 MiB data, 873 MiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-08-17T17:10:31.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:30 smithi063 ceph-mon[109843]: osdmap e1002: 8 total, 8 up, 8 in 2024-08-17T17:10:31.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:30 smithi063 ceph-mon[109843]: pgmap v960: 89 pgs: 89 active+clean; 230 MiB data, 873 MiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 1.5 MiB/s wr, 10 op/s 2024-08-17T17:10:31.734 INFO:tasks.rados.rados.0.smithi063.stdout:3718: done (0 left) 2024-08-17T17:10:31.734 INFO:tasks.rados.rados.0.smithi063.stdout:3719: rollback oid 3 current snap is 467 2024-08-17T17:10:31.734 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 3 to 466 2024-08-17T17:10:31.734 INFO:tasks.rados.rados.0.smithi063.stdout:3720: rollback oid 16 current snap is 467 2024-08-17T17:10:31.734 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 16 to 461 2024-08-17T17:10:31.734 INFO:tasks.rados.rados.0.smithi063.stdout:3721: rollback oid 33 current snap is 467 2024-08-17T17:10:31.734 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 33 to 443 2024-08-17T17:10:31.735 INFO:tasks.rados.rados.0.smithi063.stdout:3722: write oid 27 current snap is 467 2024-08-17T17:10:31.735 INFO:tasks.rados.rados.0.smithi063.stdout:3722: seq_num 1397 ranges {679379=415452,1599227=756146,3097215=382164} 2024-08-17T17:10:31.740 INFO:tasks.rados.rados.0.smithi063.stdout:3722: writing smithi06387500-27 from 679379 to 1094831 tid 1 2024-08-17T17:10:31.748 INFO:tasks.rados.rados.0.smithi063.stdout:3722: writing smithi06387500-27 from 1599227 to 2355373 tid 2 2024-08-17T17:10:31.754 INFO:tasks.rados.rados.0.smithi063.stdout:3722: writing smithi06387500-27 from 3097215 to 3479379 tid 3 2024-08-17T17:10:31.755 INFO:tasks.rados.rados.0.smithi063.stdout:3723: snap_create 2024-08-17T17:10:31.755 INFO:tasks.rados.rados.0.smithi063.stdout:3719: finishing rollback tid 0 to smithi06387500-3 2024-08-17T17:10:31.755 INFO:tasks.rados.rados.0.smithi063.stdout:3719: finishing rollback tid 1 to smithi06387500-3 2024-08-17T17:10:31.755 INFO:tasks.rados.rados.0.smithi063.stdout:3719: finishing rollback tid 2 to smithi06387500-3 2024-08-17T17:10:31.755 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 1298 (ObjNum 1339 snap 451 seq_num 1339) dirty exists 2024-08-17T17:10:31.755 INFO:tasks.rados.rados.0.smithi063.stdout:3720: finishing rollback tid 0 to smithi06387500-16 2024-08-17T17:10:31.755 INFO:tasks.rados.rados.0.smithi063.stdout:3720: finishing rollback tid 1 to smithi06387500-16 2024-08-17T17:10:31.755 INFO:tasks.rados.rados.0.smithi063.stdout:3722: finishing write tid 1 to smithi06387500-27 2024-08-17T17:10:31.755 INFO:tasks.rados.rados.0.smithi063.stdout:3720: finishing rollback tid 2 to smithi06387500-16 2024-08-17T17:10:31.755 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 1394 (ObjNum 1369 snap 461 seq_num 1369) dirty exists 2024-08-17T17:10:31.755 INFO:tasks.rados.rados.0.smithi063.stdout:3721: finishing rollback tid 0 to smithi06387500-33 2024-08-17T17:10:31.756 INFO:tasks.rados.rados.0.smithi063.stdout:3721: finishing rollback tid 1 to smithi06387500-33 2024-08-17T17:10:31.756 INFO:tasks.rados.rados.0.smithi063.stdout:3721: finishing rollback tid 2 to smithi06387500-33 2024-08-17T17:10:31.756 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 1269 (ObjNum 1311 snap 443 seq_num 1311) dirty exists 2024-08-17T17:10:31.756 INFO:tasks.rados.rados.0.smithi063.stdout:3722: finishing write tid 2 to smithi06387500-27 2024-08-17T17:10:31.757 INFO:tasks.rados.rados.0.smithi063.stdout:3722: finishing write tid 3 to smithi06387500-27 2024-08-17T17:10:31.757 INFO:tasks.rados.rados.0.smithi063.stdout:3722: finishing write tid 4 to smithi06387500-27 2024-08-17T17:10:31.757 INFO:tasks.rados.rados.0.smithi063.stdout:3722: finishing write tid 5 to smithi06387500-27 2024-08-17T17:10:31.757 INFO:tasks.rados.rados.0.smithi063.stdout:3722: oid 27 updating version 0 to 330 2024-08-17T17:10:31.757 INFO:tasks.rados.rados.0.smithi063.stdout:3722: oid 27 updating version 330 to 332 2024-08-17T17:10:31.757 INFO:tasks.rados.rados.0.smithi063.stdout:3722: oid 27 version 332 is already newer than 331 2024-08-17T17:10:31.758 INFO:tasks.rados.rados.0.smithi063.stdout:3722: oid 27 version 332 is already newer than 329 2024-08-17T17:10:31.758 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 332 (ObjNum 1397 snap 467 seq_num 1397) dirty exists 2024-08-17T17:10:31.758 INFO:tasks.rados.rados.0.smithi063.stdout:3722: left oid 27 (ObjNum 1397 snap 467 seq_num 1397) 2024-08-17T17:10:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:31 smithi063 ceph-mon[114756]: osdmap e1003: 8 total, 8 up, 8 in 2024-08-17T17:10:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:32.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:31 smithi063 ceph-mon[109843]: osdmap e1003: 8 total, 8 up, 8 in 2024-08-17T17:10:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:31 smithi162 ceph-mon[104668]: osdmap e1003: 8 total, 8 up, 8 in 2024-08-17T17:10:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:32.678 INFO:tasks.rados.rados.0.smithi063.stdout:3719: done (4 left) 2024-08-17T17:10:32.678 INFO:tasks.rados.rados.0.smithi063.stdout:3720: done (3 left) 2024-08-17T17:10:32.678 INFO:tasks.rados.rados.0.smithi063.stdout:3721: done (2 left) 2024-08-17T17:10:32.679 INFO:tasks.rados.rados.0.smithi063.stdout:3722: done (1 left) 2024-08-17T17:10:32.679 INFO:tasks.rados.rados.0.smithi063.stdout:3723: done (0 left) 2024-08-17T17:10:32.679 INFO:tasks.rados.rados.0.smithi063.stdout:3724: snap_create 2024-08-17T17:10:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:32 smithi063 ceph-mon[114756]: osdmap e1004: 8 total, 8 up, 8 in 2024-08-17T17:10:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:32 smithi063 ceph-mon[114756]: pgmap v963: 89 pgs: 89 active+clean; 239 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-08-17T17:10:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:32 smithi063 ceph-mon[114756]: osdmap e1005: 8 total, 8 up, 8 in 2024-08-17T17:10:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:32 smithi063 ceph-mon[109843]: osdmap e1004: 8 total, 8 up, 8 in 2024-08-17T17:10:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:32 smithi063 ceph-mon[109843]: pgmap v963: 89 pgs: 89 active+clean; 239 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-08-17T17:10:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:32 smithi063 ceph-mon[109843]: osdmap e1005: 8 total, 8 up, 8 in 2024-08-17T17:10:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:32 smithi162 ceph-mon[104668]: osdmap e1004: 8 total, 8 up, 8 in 2024-08-17T17:10:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:32 smithi162 ceph-mon[104668]: pgmap v963: 89 pgs: 89 active+clean; 239 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.4 MiB/s wr, 14 op/s 2024-08-17T17:10:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:32 smithi162 ceph-mon[104668]: osdmap e1005: 8 total, 8 up, 8 in 2024-08-17T17:10:33.682 INFO:tasks.rados.rados.0.smithi063.stdout:3724: done (0 left) 2024-08-17T17:10:33.682 INFO:tasks.rados.rados.0.smithi063.stdout:3725: write oid 6 current snap is 469 2024-08-17T17:10:33.682 INFO:tasks.rados.rados.0.smithi063.stdout:3725: seq_num 1398 ranges {696186=490701,1657709=616157,2724638=562274} 2024-08-17T17:10:33.689 INFO:tasks.rados.rados.0.smithi063.stdout:3725: writing smithi06387500-6 from 696186 to 1186887 tid 1 2024-08-17T17:10:33.695 INFO:tasks.rados.rados.0.smithi063.stdout:3725: writing smithi06387500-6 from 1657709 to 2273866 tid 2 2024-08-17T17:10:33.702 INFO:tasks.rados.rados.0.smithi063.stdout:3725: writing smithi06387500-6 from 2724638 to 3286912 tid 3 2024-08-17T17:10:33.702 INFO:tasks.rados.rados.0.smithi063.stdout:3726: read oid 23 snap -1 2024-08-17T17:10:33.702 INFO:tasks.rados.rados.0.smithi063.stdout:3726: expect (ObjNum 1393 snap 466 seq_num 1393) 2024-08-17T17:10:33.702 INFO:tasks.rados.rados.0.smithi063.stdout:3727: snap_remove snap 465 2024-08-17T17:10:33.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:10:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:10:33] "GET /metrics HTTP/1.1" 200 36337 "" "Prometheus/2.51.0" 2024-08-17T17:10:34.689 INFO:tasks.rados.rados.0.smithi063.stdout:3727: done (2 left) 2024-08-17T17:10:34.690 INFO:tasks.rados.rados.0.smithi063.stdout:3728: rollback oid 48 current snap is 469 2024-08-17T17:10:34.690 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 48 to 440 2024-08-17T17:10:34.690 INFO:tasks.rados.rados.0.smithi063.stdout:3725: finishing write tid 1 to smithi06387500-6 2024-08-17T17:10:34.690 INFO:tasks.rados.rados.0.smithi063.stdout:3725: finishing write tid 2 to smithi06387500-6 2024-08-17T17:10:34.690 INFO:tasks.rados.rados.0.smithi063.stdout:3725: finishing write tid 3 to smithi06387500-6 2024-08-17T17:10:34.690 INFO:tasks.rados.rados.0.smithi063.stdout:3725: finishing write tid 4 to smithi06387500-6 2024-08-17T17:10:34.690 INFO:tasks.rados.rados.0.smithi063.stdout:3725: finishing write tid 5 to smithi06387500-6 2024-08-17T17:10:34.690 INFO:tasks.rados.rados.0.smithi063.stdout:3725: oid 6 updating version 0 to 1271 2024-08-17T17:10:34.690 INFO:tasks.rados.rados.0.smithi063.stdout:3725: oid 6 updating version 1271 to 1272 2024-08-17T17:10:34.690 INFO:tasks.rados.rados.0.smithi063.stdout:3725: oid 6 updating version 1272 to 1273 2024-08-17T17:10:34.690 INFO:tasks.rados.rados.0.smithi063.stdout:3725: oid 6 version 1273 is already newer than 1270 2024-08-17T17:10:34.691 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 1273 (ObjNum 1398 snap 469 seq_num 1398) dirty exists 2024-08-17T17:10:34.691 INFO:tasks.rados.rados.0.smithi063.stdout:3725: left oid 6 (ObjNum 1398 snap 469 seq_num 1398) 2024-08-17T17:10:34.691 INFO:tasks.rados.rados.0.smithi063.stdout:3726: expect (ObjNum 1393 snap 466 seq_num 1393) 2024-08-17T17:10:34.841 INFO:tasks.rados.rados.0.smithi063.stdout:3728: finishing rollback tid 1 to smithi06387500-48 2024-08-17T17:10:34.842 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 3567 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-08-17T17:10:34.842 INFO:tasks.rados.rados.0.smithi063.stdout:3725: done (2 left) 2024-08-17T17:10:34.842 INFO:tasks.rados.rados.0.smithi063.stdout:3726: done (1 left) 2024-08-17T17:10:34.842 INFO:tasks.rados.rados.0.smithi063.stdout:3728: done (0 left) 2024-08-17T17:10:34.842 INFO:tasks.rados.rados.0.smithi063.stdout:3729: snap_remove snap 466 2024-08-17T17:10:35.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:34 smithi063 ceph-mon[114756]: osdmap e1006: 8 total, 8 up, 8 in 2024-08-17T17:10:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:34 smithi063 ceph-mon[109843]: osdmap e1006: 8 total, 8 up, 8 in 2024-08-17T17:10:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:34 smithi162 ceph-mon[104668]: osdmap e1006: 8 total, 8 up, 8 in 2024-08-17T17:10:35.687 INFO:tasks.rados.rados.0.smithi063.stdout:3729: done (0 left) 2024-08-17T17:10:35.687 INFO:tasks.rados.rados.0.smithi063.stdout:3730: rollback oid 39 current snap is 469 2024-08-17T17:10:35.687 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 39 to 443 2024-08-17T17:10:35.687 INFO:tasks.rados.rados.0.smithi063.stdout:3731: snap_create 2024-08-17T17:10:35.696 INFO:tasks.rados.rados.0.smithi063.stdout:3730: finishing rollback tid 1 to smithi06387500-39 2024-08-17T17:10:35.696 INFO:tasks.rados.rados.0.smithi063.stdout:3730: finishing rollback tid 2 to smithi06387500-39 2024-08-17T17:10:35.696 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 945 (ObjNum 1092 snap 363 seq_num 1092) dirty exists 2024-08-17T17:10:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:35 smithi063 ceph-mon[109843]: pgmap v966: 89 pgs: 89 active+clean; 241 MiB data, 895 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.8 MiB/s wr, 16 op/s 2024-08-17T17:10:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:35 smithi063 ceph-mon[109843]: osdmap e1007: 8 total, 8 up, 8 in 2024-08-17T17:10:36.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:35 smithi063 ceph-mon[114756]: pgmap v966: 89 pgs: 89 active+clean; 241 MiB data, 895 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.8 MiB/s wr, 16 op/s 2024-08-17T17:10:36.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:35 smithi063 ceph-mon[114756]: osdmap e1007: 8 total, 8 up, 8 in 2024-08-17T17:10:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:35 smithi162 ceph-mon[104668]: pgmap v966: 89 pgs: 89 active+clean; 241 MiB data, 895 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 1.8 MiB/s wr, 16 op/s 2024-08-17T17:10:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:35 smithi162 ceph-mon[104668]: osdmap e1007: 8 total, 8 up, 8 in 2024-08-17T17:10:36.701 INFO:tasks.rados.rados.0.smithi063.stdout:3730: done (1 left) 2024-08-17T17:10:36.701 INFO:tasks.rados.rados.0.smithi063.stdout:3731: done (0 left) 2024-08-17T17:10:36.701 INFO:tasks.rados.rados.0.smithi063.stdout:3732: snap_remove snap 460 2024-08-17T17:10:37.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:36 smithi063 ceph-mon[109843]: osdmap e1008: 8 total, 8 up, 8 in 2024-08-17T17:10:37.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:36 smithi063 ceph-mon[109843]: pgmap v969: 89 pgs: 89 active+clean; 244 MiB data, 897 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 800 KiB/s wr, 3 op/s 2024-08-17T17:10:37.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:36 smithi063 ceph-mon[114756]: osdmap e1008: 8 total, 8 up, 8 in 2024-08-17T17:10:37.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:36 smithi063 ceph-mon[114756]: pgmap v969: 89 pgs: 89 active+clean; 244 MiB data, 897 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 800 KiB/s wr, 3 op/s 2024-08-17T17:10:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:36 smithi162 ceph-mon[104668]: osdmap e1008: 8 total, 8 up, 8 in 2024-08-17T17:10:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:36 smithi162 ceph-mon[104668]: pgmap v969: 89 pgs: 89 active+clean; 244 MiB data, 897 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 800 KiB/s wr, 3 op/s 2024-08-17T17:10:37.681 INFO:tasks.rados.rados.0.smithi063.stdout:3732: done (0 left) 2024-08-17T17:10:37.681 INFO:tasks.rados.rados.0.smithi063.stdout:3733: delete oid 41 current snap is 470 2024-08-17T17:10:37.687 INFO:tasks.rados.rados.0.smithi063.stdout:3733: done (0 left) 2024-08-17T17:10:37.687 INFO:tasks.rados.rados.0.smithi063.stdout:3734: write oid 36 current snap is 470 2024-08-17T17:10:37.687 INFO:tasks.rados.rados.0.smithi063.stdout:3734: seq_num 1400 ranges {729800=757552,2174673=619826} 2024-08-17T17:10:37.696 INFO:tasks.rados.rados.0.smithi063.stdout:3734: writing smithi06387500-36 from 729800 to 1487352 tid 1 2024-08-17T17:10:37.703 INFO:tasks.rados.rados.0.smithi063.stdout:3734: writing smithi06387500-36 from 2174673 to 2794499 tid 2 2024-08-17T17:10:37.703 INFO:tasks.rados.rados.0.smithi063.stdout:3735: write oid 9 current snap is 470 2024-08-17T17:10:37.703 INFO:tasks.rados.rados.0.smithi063.stdout:3734: finishing write tid 1 to smithi06387500-36 2024-08-17T17:10:37.703 INFO:tasks.rados.rados.0.smithi063.stdout:3735: seq_num 1401 ranges {746607=432801,1833155=763484,3206907=339700} 2024-08-17T17:10:37.709 INFO:tasks.rados.rados.0.smithi063.stdout:3735: writing smithi06387500-9 from 746607 to 1179408 tid 1 2024-08-17T17:10:37.717 INFO:tasks.rados.rados.0.smithi063.stdout:3735: writing smithi06387500-9 from 1833155 to 2596639 tid 2 2024-08-17T17:10:37.723 INFO:tasks.rados.rados.0.smithi063.stdout:3735: writing smithi06387500-9 from 3206907 to 3546607 tid 3 2024-08-17T17:10:37.723 INFO:tasks.rados.rados.0.smithi063.stdout:3736: write oid 3 current snap is 470 2024-08-17T17:10:37.723 INFO:tasks.rados.rados.0.smithi063.stdout:3734: finishing write tid 2 to smithi06387500-36 2024-08-17T17:10:37.723 INFO:tasks.rados.rados.0.smithi063.stdout:3734: finishing write tid 3 to smithi06387500-36 2024-08-17T17:10:37.723 INFO:tasks.rados.rados.0.smithi063.stdout:3736: seq_num 1402 ranges {763414=508050,1891637=623495,3234330=329084} 2024-08-17T17:10:37.729 INFO:tasks.rados.rados.0.smithi063.stdout:3736: writing smithi06387500-3 from 763414 to 1271464 tid 1 2024-08-17T17:10:37.736 INFO:tasks.rados.rados.0.smithi063.stdout:3736: writing smithi06387500-3 from 1891637 to 2515132 tid 2 2024-08-17T17:10:37.742 INFO:tasks.rados.rados.0.smithi063.stdout:3736: writing smithi06387500-3 from 3234330 to 3563414 tid 3 2024-08-17T17:10:37.742 INFO:tasks.rados.rados.0.smithi063.stdout:3737: write oid 27 current snap is 470 2024-08-17T17:10:37.742 INFO:tasks.rados.rados.0.smithi063.stdout:3737: seq_num 1403 ranges {780221=583299,1950119=767153,3261753=318468} 2024-08-17T17:10:37.749 INFO:tasks.rados.rados.0.smithi063.stdout:3737: writing smithi06387500-27 from 780221 to 1363520 tid 1 2024-08-17T17:10:37.758 INFO:tasks.rados.rados.0.smithi063.stdout:3737: writing smithi06387500-27 from 1950119 to 2717272 tid 2 2024-08-17T17:10:37.763 INFO:tasks.rados.rados.0.smithi063.stdout:3737: writing smithi06387500-27 from 3261753 to 3580221 tid 3 2024-08-17T17:10:37.763 INFO:tasks.rados.rados.0.smithi063.stdout:3738: delete oid 4 current snap is 470 2024-08-17T17:10:37.763 INFO:tasks.rados.rados.0.smithi063.stdout:3734: finishing write tid 4 to smithi06387500-36 2024-08-17T17:10:37.764 INFO:tasks.rados.rados.0.smithi063.stdout:3734: oid 36 updating version 0 to 3688 2024-08-17T17:10:37.764 INFO:tasks.rados.rados.0.smithi063.stdout:3734: oid 36 updating version 3688 to 3689 2024-08-17T17:10:37.764 INFO:tasks.rados.rados.0.smithi063.stdout:3734: oid 36 updating version 3689 to 3690 2024-08-17T17:10:37.764 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 3690 (ObjNum 1400 snap 470 seq_num 1400) dirty exists 2024-08-17T17:10:37.764 INFO:tasks.rados.rados.0.smithi063.stdout:3734: left oid 36 (ObjNum 1400 snap 470 seq_num 1400) 2024-08-17T17:10:37.764 INFO:tasks.rados.rados.0.smithi063.stdout:3735: finishing write tid 1 to smithi06387500-9 2024-08-17T17:10:37.764 INFO:tasks.rados.rados.0.smithi063.stdout:3735: finishing write tid 2 to smithi06387500-9 2024-08-17T17:10:37.764 INFO:tasks.rados.rados.0.smithi063.stdout:3735: finishing write tid 3 to smithi06387500-9 2024-08-17T17:10:37.764 INFO:tasks.rados.rados.0.smithi063.stdout:3735: finishing write tid 4 to smithi06387500-9 2024-08-17T17:10:37.764 INFO:tasks.rados.rados.0.smithi063.stdout:3735: finishing write tid 5 to smithi06387500-9 2024-08-17T17:10:37.764 INFO:tasks.rados.rados.0.smithi063.stdout:3735: oid 9 updating version 0 to 1820 2024-08-17T17:10:37.764 INFO:tasks.rados.rados.0.smithi063.stdout:3735: oid 9 updating version 1820 to 1822 2024-08-17T17:10:37.764 INFO:tasks.rados.rados.0.smithi063.stdout:3735: oid 9 version 1822 is already newer than 1821 2024-08-17T17:10:37.765 INFO:tasks.rados.rados.0.smithi063.stdout:3735: oid 9 updating version 1822 to 1823 2024-08-17T17:10:37.765 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 1823 (ObjNum 1401 snap 470 seq_num 1401) dirty exists 2024-08-17T17:10:37.765 INFO:tasks.rados.rados.0.smithi063.stdout:3735: left oid 9 (ObjNum 1401 snap 470 seq_num 1401) 2024-08-17T17:10:37.765 INFO:tasks.rados.rados.0.smithi063.stdout:3736: finishing write tid 1 to smithi06387500-3 2024-08-17T17:10:37.765 INFO:tasks.rados.rados.0.smithi063.stdout:3736: finishing write tid 2 to smithi06387500-3 2024-08-17T17:10:37.765 INFO:tasks.rados.rados.0.smithi063.stdout:3736: finishing write tid 3 to smithi06387500-3 2024-08-17T17:10:37.765 INFO:tasks.rados.rados.0.smithi063.stdout:3736: finishing write tid 4 to smithi06387500-3 2024-08-17T17:10:37.765 INFO:tasks.rados.rados.0.smithi063.stdout:3736: finishing write tid 5 to smithi06387500-3 2024-08-17T17:10:37.765 INFO:tasks.rados.rados.0.smithi063.stdout:3736: oid 3 updating version 0 to 1307 2024-08-17T17:10:37.765 INFO:tasks.rados.rados.0.smithi063.stdout:3736: oid 3 updating version 1307 to 1308 2024-08-17T17:10:37.765 INFO:tasks.rados.rados.0.smithi063.stdout:3736: oid 3 updating version 1308 to 1309 2024-08-17T17:10:37.765 INFO:tasks.rados.rados.0.smithi063.stdout:3736: oid 3 version 1309 is already newer than 1306 2024-08-17T17:10:37.765 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 1309 (ObjNum 1402 snap 470 seq_num 1402) dirty exists 2024-08-17T17:10:37.766 INFO:tasks.rados.rados.0.smithi063.stdout:3736: left oid 3 (ObjNum 1402 snap 470 seq_num 1402) 2024-08-17T17:10:37.766 INFO:tasks.rados.rados.0.smithi063.stdout:3737: finishing write tid 1 to smithi06387500-27 2024-08-17T17:10:37.766 INFO:tasks.rados.rados.0.smithi063.stdout:3737: finishing write tid 2 to smithi06387500-27 2024-08-17T17:10:37.766 INFO:tasks.rados.rados.0.smithi063.stdout:3737: finishing write tid 3 to smithi06387500-27 2024-08-17T17:10:37.766 INFO:tasks.rados.rados.0.smithi063.stdout:3737: finishing write tid 4 to smithi06387500-27 2024-08-17T17:10:37.767 INFO:tasks.rados.rados.0.smithi063.stdout:3737: finishing write tid 5 to smithi06387500-27 2024-08-17T17:10:37.767 INFO:tasks.rados.rados.0.smithi063.stdout:3737: oid 27 updating version 0 to 341 2024-08-17T17:10:37.767 INFO:tasks.rados.rados.0.smithi063.stdout:3737: oid 27 updating version 341 to 342 2024-08-17T17:10:37.767 INFO:tasks.rados.rados.0.smithi063.stdout:3737: oid 27 version 342 is already newer than 338 2024-08-17T17:10:37.767 INFO:tasks.rados.rados.0.smithi063.stdout:3737: oid 27 updating version 342 to 343 2024-08-17T17:10:37.767 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 343 (ObjNum 1403 snap 470 seq_num 1403) dirty exists 2024-08-17T17:10:37.767 INFO:tasks.rados.rados.0.smithi063.stdout:3737: left oid 27 (ObjNum 1403 snap 470 seq_num 1403) 2024-08-17T17:10:37.767 INFO:tasks.rados.rados.0.smithi063.stdout:3734: done (4 left) 2024-08-17T17:10:37.767 INFO:tasks.rados.rados.0.smithi063.stdout:3735: done (3 left) 2024-08-17T17:10:37.767 INFO:tasks.rados.rados.0.smithi063.stdout:3736: done (2 left) 2024-08-17T17:10:37.767 INFO:tasks.rados.rados.0.smithi063.stdout:3737: done (1 left) 2024-08-17T17:10:37.767 INFO:tasks.rados.rados.0.smithi063.stdout:3738: done (0 left) 2024-08-17T17:10:37.768 INFO:tasks.rados.rados.0.smithi063.stdout:3739: snap_remove snap 467 2024-08-17T17:10:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:37 smithi063 ceph-mon[114756]: osdmap e1009: 8 total, 8 up, 8 in 2024-08-17T17:10:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:37 smithi063 ceph-mon[114756]: osdmap e1010: 8 total, 8 up, 8 in 2024-08-17T17:10:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:37 smithi063 ceph-mon[109843]: osdmap e1009: 8 total, 8 up, 8 in 2024-08-17T17:10:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:37 smithi063 ceph-mon[109843]: osdmap e1010: 8 total, 8 up, 8 in 2024-08-17T17:10:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:37 smithi162 ceph-mon[104668]: osdmap e1009: 8 total, 8 up, 8 in 2024-08-17T17:10:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:37 smithi162 ceph-mon[104668]: osdmap e1010: 8 total, 8 up, 8 in 2024-08-17T17:10:38.709 INFO:tasks.rados.rados.0.smithi063.stdout:3739: done (0 left) 2024-08-17T17:10:38.709 INFO:tasks.rados.rados.0.smithi063.stdout:3740: read oid 46 snap -1 2024-08-17T17:10:38.709 INFO:tasks.rados.rados.0.smithi063.stdout:3740: expect deleted 2024-08-17T17:10:38.710 INFO:tasks.rados.rados.0.smithi063.stdout:3741: read oid 21 snap -1 2024-08-17T17:10:38.710 INFO:tasks.rados.rados.0.smithi063.stdout:3741: expect deleted 2024-08-17T17:10:38.710 INFO:tasks.rados.rados.0.smithi063.stdout:3742: delete oid 22 current snap is 470 2024-08-17T17:10:38.714 INFO:tasks.rados.rados.0.smithi063.stdout:3740: done (2 left) 2024-08-17T17:10:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3741: done (1 left) 2024-08-17T17:10:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3742: done (0 left) 2024-08-17T17:10:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3743: read oid 33 snap -1 2024-08-17T17:10:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3743: expect (ObjNum 1311 snap 443 seq_num 1311) 2024-08-17T17:10:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3744: write oid 11 current snap is 470 2024-08-17T17:10:38.715 INFO:tasks.rados.rados.0.smithi063.stdout:3744: seq_num 1406 ranges {430642=409046,1609212=514480,2827669=768803} 2024-08-17T17:10:38.719 INFO:tasks.rados.rados.0.smithi063.stdout:3744: writing smithi06387500-11 from 430642 to 839688 tid 1 2024-08-17T17:10:38.726 INFO:tasks.rados.rados.0.smithi063.stdout:3744: writing smithi06387500-11 from 1609212 to 2123692 tid 2 2024-08-17T17:10:38.735 INFO:tasks.rados.rados.0.smithi063.stdout:3744: writing smithi06387500-11 from 2827669 to 3596472 tid 3 2024-08-17T17:10:38.735 INFO:tasks.rados.rados.0.smithi063.stdout:3745: delete oid 34 current snap is 470 2024-08-17T17:10:38.735 INFO:tasks.rados.rados.0.smithi063.stdout:3743: expect (ObjNum 1311 snap 443 seq_num 1311) 2024-08-17T17:10:38.803 INFO:tasks.rados.rados.0.smithi063.stdout:3744: finishing write tid 1 to smithi06387500-11 2024-08-17T17:10:38.804 INFO:tasks.rados.rados.0.smithi063.stdout:3744: finishing write tid 2 to smithi06387500-11 2024-08-17T17:10:38.804 INFO:tasks.rados.rados.0.smithi063.stdout:3743: done (2 left) 2024-08-17T17:10:38.804 INFO:tasks.rados.rados.0.smithi063.stdout:3745: done (1 left) 2024-08-17T17:10:38.805 INFO:tasks.rados.rados.0.smithi063.stdout:3746: read oid 44 snap -1 2024-08-17T17:10:38.805 INFO:tasks.rados.rados.0.smithi063.stdout:3746: expect (ObjNum 1394 snap 466 seq_num 1394) 2024-08-17T17:10:38.805 INFO:tasks.rados.rados.0.smithi063.stdout:3747: read oid 33 snap 440 2024-08-17T17:10:38.805 INFO:tasks.rados.rados.0.smithi063.stdout:3747: expect deleted 2024-08-17T17:10:38.805 INFO:tasks.rados.rados.0.smithi063.stdout:3748: snap_remove snap 461 2024-08-17T17:10:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:38 smithi063 ceph-mon[114756]: pgmap v972: 89 pgs: 89 active+clean; 249 MiB data, 918 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 787 KiB/s wr, 6 op/s 2024-08-17T17:10:39.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:38 smithi063 ceph-mon[109843]: pgmap v972: 89 pgs: 89 active+clean; 249 MiB data, 918 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 787 KiB/s wr, 6 op/s 2024-08-17T17:10:39.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:38 smithi162 ceph-mon[104668]: pgmap v972: 89 pgs: 89 active+clean; 249 MiB data, 918 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 787 KiB/s wr, 6 op/s 2024-08-17T17:10:39.717 INFO:tasks.rados.rados.0.smithi063.stdout:3748: done (3 left) 2024-08-17T17:10:39.718 INFO:tasks.rados.rados.0.smithi063.stdout:3749: delete oid 3 current snap is 470 2024-08-17T17:10:39.718 INFO:tasks.rados.rados.0.smithi063.stdout:3744: finishing write tid 3 to smithi06387500-11 2024-08-17T17:10:39.718 INFO:tasks.rados.rados.0.smithi063.stdout:3744: finishing write tid 4 to smithi06387500-11 2024-08-17T17:10:39.718 INFO:tasks.rados.rados.0.smithi063.stdout:3744: finishing write tid 5 to smithi06387500-11 2024-08-17T17:10:39.718 INFO:tasks.rados.rados.0.smithi063.stdout:3744: oid 11 updating version 0 to 726 2024-08-17T17:10:39.718 INFO:tasks.rados.rados.0.smithi063.stdout:3744: oid 11 updating version 726 to 729 2024-08-17T17:10:39.718 INFO:tasks.rados.rados.0.smithi063.stdout:3744: oid 11 version 729 is already newer than 728 2024-08-17T17:10:39.718 INFO:tasks.rados.rados.0.smithi063.stdout:3744: oid 11 version 729 is already newer than 727 2024-08-17T17:10:39.718 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 729 (ObjNum 1406 snap 470 seq_num 1406) dirty exists 2024-08-17T17:10:39.718 INFO:tasks.rados.rados.0.smithi063.stdout:3744: left oid 11 (ObjNum 1406 snap 470 seq_num 1406) 2024-08-17T17:10:39.718 INFO:tasks.rados.rados.0.smithi063.stdout:3746: expect (ObjNum 1394 snap 466 seq_num 1394) 2024-08-17T17:10:39.962 INFO:tasks.rados.rados.0.smithi063.stdout:3744: done (3 left) 2024-08-17T17:10:39.963 INFO:tasks.rados.rados.0.smithi063.stdout:3746: done (2 left) 2024-08-17T17:10:39.963 INFO:tasks.rados.rados.0.smithi063.stdout:3747: done (1 left) 2024-08-17T17:10:39.963 INFO:tasks.rados.rados.0.smithi063.stdout:3749: done (0 left) 2024-08-17T17:10:39.963 INFO:tasks.rados.rados.0.smithi063.stdout:3750: write oid 15 current snap is 470 2024-08-17T17:10:39.963 INFO:tasks.rados.rados.0.smithi063.stdout:3750: seq_num 1409 ranges {481063=751146,1784658=778160,3309938=371125} 2024-08-17T17:10:39.969 INFO:tasks.rados.rados.0.smithi063.stdout:3750: writing smithi06387500-15 from 481063 to 1232209 tid 1 2024-08-17T17:10:39.976 INFO:tasks.rados.rados.0.smithi063.stdout:3750: writing smithi06387500-15 from 1784658 to 2562818 tid 2 2024-08-17T17:10:39.983 INFO:tasks.rados.rados.0.smithi063.stdout:3750: writing smithi06387500-15 from 3309938 to 3681063 tid 3 2024-08-17T17:10:39.983 INFO:tasks.rados.rados.0.smithi063.stdout:3751: write oid 30 current snap is 470 2024-08-17T17:10:39.983 INFO:tasks.rados.rados.0.smithi063.stdout:3750: finishing write tid 1 to smithi06387500-15 2024-08-17T17:10:39.983 INFO:tasks.rados.rados.0.smithi063.stdout:3750: finishing write tid 2 to smithi06387500-15 2024-08-17T17:10:39.983 INFO:tasks.rados.rados.0.smithi063.stdout:3751: seq_num 1410 ranges {497870=426395,1443140=521818,2537361=530244,3583943=113927} 2024-08-17T17:10:39.988 INFO:tasks.rados.rados.0.smithi063.stdout:3751: writing smithi06387500-30 from 497870 to 924265 tid 1 2024-08-17T17:10:39.994 INFO:tasks.rados.rados.0.smithi063.stdout:3751: writing smithi06387500-30 from 1443140 to 1964958 tid 2 2024-08-17T17:10:40.000 INFO:tasks.rados.rados.0.smithi063.stdout:3751: writing smithi06387500-30 from 2537361 to 3067605 tid 3 2024-08-17T17:10:40.003 INFO:tasks.rados.rados.0.smithi063.stdout:3751: writing smithi06387500-30 from 3583943 to 3697870 tid 4 2024-08-17T17:10:40.004 INFO:tasks.rados.rados.0.smithi063.stdout:3752: snap_remove snap 440 2024-08-17T17:10:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:39 smithi063 ceph-mon[109843]: osdmap e1011: 8 total, 8 up, 8 in 2024-08-17T17:10:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:39 smithi063 ceph-mon[114756]: osdmap e1011: 8 total, 8 up, 8 in 2024-08-17T17:10:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:39 smithi162 ceph-mon[104668]: osdmap e1011: 8 total, 8 up, 8 in 2024-08-17T17:10:40.721 INFO:tasks.rados.rados.0.smithi063.stdout:3752: done (2 left) 2024-08-17T17:10:40.721 INFO:tasks.rados.rados.0.smithi063.stdout:3753: snap_create 2024-08-17T17:10:40.721 INFO:tasks.rados.rados.0.smithi063.stdout:3750: finishing write tid 3 to smithi06387500-15 2024-08-17T17:10:40.721 INFO:tasks.rados.rados.0.smithi063.stdout:3750: finishing write tid 4 to smithi06387500-15 2024-08-17T17:10:40.721 INFO:tasks.rados.rados.0.smithi063.stdout:3750: finishing write tid 5 to smithi06387500-15 2024-08-17T17:10:40.722 INFO:tasks.rados.rados.0.smithi063.stdout:3750: oid 15 updating version 0 to 1412 2024-08-17T17:10:40.722 INFO:tasks.rados.rados.0.smithi063.stdout:3750: oid 15 version 1412 is already newer than 1411 2024-08-17T17:10:40.722 INFO:tasks.rados.rados.0.smithi063.stdout:3750: oid 15 updating version 1412 to 1413 2024-08-17T17:10:40.722 INFO:tasks.rados.rados.0.smithi063.stdout:3750: oid 15 version 1413 is already newer than 1410 2024-08-17T17:10:40.722 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 1413 (ObjNum 1409 snap 470 seq_num 1409) dirty exists 2024-08-17T17:10:40.722 INFO:tasks.rados.rados.0.smithi063.stdout:3750: left oid 15 (ObjNum 1409 snap 470 seq_num 1409) 2024-08-17T17:10:40.722 INFO:tasks.rados.rados.0.smithi063.stdout:3751: finishing write tid 1 to smithi06387500-30 2024-08-17T17:10:40.722 INFO:tasks.rados.rados.0.smithi063.stdout:3751: finishing write tid 2 to smithi06387500-30 2024-08-17T17:10:40.722 INFO:tasks.rados.rados.0.smithi063.stdout:3751: finishing write tid 3 to smithi06387500-30 2024-08-17T17:10:40.722 INFO:tasks.rados.rados.0.smithi063.stdout:3751: finishing write tid 4 to smithi06387500-30 2024-08-17T17:10:40.722 INFO:tasks.rados.rados.0.smithi063.stdout:3751: finishing write tid 5 to smithi06387500-30 2024-08-17T17:10:40.722 INFO:tasks.rados.rados.0.smithi063.stdout:3751: finishing write tid 6 to smithi06387500-30 2024-08-17T17:10:40.723 INFO:tasks.rados.rados.0.smithi063.stdout:3751: oid 30 updating version 0 to 1837 2024-08-17T17:10:40.723 INFO:tasks.rados.rados.0.smithi063.stdout:3751: oid 30 version 1837 is already newer than 1835 2024-08-17T17:10:40.723 INFO:tasks.rados.rados.0.smithi063.stdout:3751: oid 30 version 1837 is already newer than 1833 2024-08-17T17:10:40.723 INFO:tasks.rados.rados.0.smithi063.stdout:3751: oid 30 version 1837 is already newer than 1836 2024-08-17T17:10:40.723 INFO:tasks.rados.rados.0.smithi063.stdout:3751: oid 30 version 1837 is already newer than 1834 2024-08-17T17:10:40.723 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1837 (ObjNum 1410 snap 470 seq_num 1410) dirty exists 2024-08-17T17:10:40.723 INFO:tasks.rados.rados.0.smithi063.stdout:3751: left oid 30 (ObjNum 1410 snap 470 seq_num 1410) 2024-08-17T17:10:41.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:40 smithi063 ceph-mon[109843]: osdmap e1012: 8 total, 8 up, 8 in 2024-08-17T17:10:41.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:40 smithi063 ceph-mon[109843]: pgmap v975: 89 pgs: 89 active+clean; 249 MiB data, 918 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 408 KiB/s wr, 4 op/s 2024-08-17T17:10:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:40 smithi063 ceph-mon[114756]: osdmap e1012: 8 total, 8 up, 8 in 2024-08-17T17:10:41.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:40 smithi063 ceph-mon[114756]: pgmap v975: 89 pgs: 89 active+clean; 249 MiB data, 918 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 408 KiB/s wr, 4 op/s 2024-08-17T17:10:41.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:40 smithi162 ceph-mon[104668]: osdmap e1012: 8 total, 8 up, 8 in 2024-08-17T17:10:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:40 smithi162 ceph-mon[104668]: pgmap v975: 89 pgs: 89 active+clean; 249 MiB data, 918 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 408 KiB/s wr, 4 op/s 2024-08-17T17:10:41.722 INFO:tasks.rados.rados.0.smithi063.stdout:3750: done (2 left) 2024-08-17T17:10:41.723 INFO:tasks.rados.rados.0.smithi063.stdout:3751: done (1 left) 2024-08-17T17:10:41.723 INFO:tasks.rados.rados.0.smithi063.stdout:3753: done (0 left) 2024-08-17T17:10:41.723 INFO:tasks.rados.rados.0.smithi063.stdout:3754: read oid 2 snap -1 2024-08-17T17:10:41.723 INFO:tasks.rados.rados.0.smithi063.stdout:3754: expect deleted 2024-08-17T17:10:41.723 INFO:tasks.rados.rados.0.smithi063.stdout:3755: snap_create 2024-08-17T17:10:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:41 smithi063 ceph-mon[109843]: osdmap e1013: 8 total, 8 up, 8 in 2024-08-17T17:10:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:41 smithi063 ceph-mon[114756]: osdmap e1013: 8 total, 8 up, 8 in 2024-08-17T17:10:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:41 smithi162 ceph-mon[104668]: osdmap e1013: 8 total, 8 up, 8 in 2024-08-17T17:10:42.684 INFO:tasks.rados.rados.0.smithi063.stdout:3754: done (1 left) 2024-08-17T17:10:42.684 INFO:tasks.rados.rados.0.smithi063.stdout:3755: done (0 left) 2024-08-17T17:10:42.684 INFO:tasks.rados.rados.0.smithi063.stdout:3756: snap_create 2024-08-17T17:10:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:42 smithi063 ceph-mon[114756]: osdmap e1014: 8 total, 8 up, 8 in 2024-08-17T17:10:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:42 smithi063 ceph-mon[114756]: pgmap v978: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 253 MiB data, 917 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.7 MiB/s wr, 13 op/s 2024-08-17T17:10:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:42 smithi063 ceph-mon[114756]: osdmap e1015: 8 total, 8 up, 8 in 2024-08-17T17:10:43.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:42 smithi063 ceph-mon[109843]: osdmap e1014: 8 total, 8 up, 8 in 2024-08-17T17:10:43.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:42 smithi063 ceph-mon[109843]: pgmap v978: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 253 MiB data, 917 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.7 MiB/s wr, 13 op/s 2024-08-17T17:10:43.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:42 smithi063 ceph-mon[109843]: osdmap e1015: 8 total, 8 up, 8 in 2024-08-17T17:10:43.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:42 smithi162 ceph-mon[104668]: osdmap e1014: 8 total, 8 up, 8 in 2024-08-17T17:10:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:42 smithi162 ceph-mon[104668]: pgmap v978: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 253 MiB data, 917 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.7 MiB/s wr, 13 op/s 2024-08-17T17:10:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:42 smithi162 ceph-mon[104668]: osdmap e1015: 8 total, 8 up, 8 in 2024-08-17T17:10:43.684 INFO:tasks.rados.rados.0.smithi063.stdout:3756: done (0 left) 2024-08-17T17:10:43.685 INFO:tasks.rados.rados.0.smithi063.stdout:3757: snap_remove snap 468 2024-08-17T17:10:43.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:10:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:10:43] "GET /metrics HTTP/1.1" 200 36337 "" "Prometheus/2.51.0" 2024-08-17T17:10:44.696 INFO:tasks.rados.rados.0.smithi063.stdout:3757: done (0 left) 2024-08-17T17:10:44.696 INFO:tasks.rados.rados.0.smithi063.stdout:3758: snap_remove snap 456 2024-08-17T17:10:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:44 smithi063 ceph-mon[114756]: osdmap e1016: 8 total, 8 up, 8 in 2024-08-17T17:10:45.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:44 smithi063 ceph-mon[109843]: osdmap e1016: 8 total, 8 up, 8 in 2024-08-17T17:10:45.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:44 smithi162 ceph-mon[104668]: osdmap e1016: 8 total, 8 up, 8 in 2024-08-17T17:10:45.696 INFO:tasks.rados.rados.0.smithi063.stdout:3758: done (0 left) 2024-08-17T17:10:45.696 INFO:tasks.rados.rados.0.smithi063.stdout:3759: write oid 39 current snap is 473 2024-08-17T17:10:45.696 INFO:tasks.rados.rados.0.smithi063.stdout:3759: seq_num 1411 ranges {514677=501644,1501622=781829,2964784=741516} 2024-08-17T17:10:45.702 INFO:tasks.rados.rados.0.smithi063.stdout:3759: writing smithi06387500-39 from 514677 to 1016321 tid 1 2024-08-17T17:10:45.710 INFO:tasks.rados.rados.0.smithi063.stdout:3759: writing smithi06387500-39 from 1501622 to 2283451 tid 2 2024-08-17T17:10:45.719 INFO:tasks.rados.rados.0.smithi063.stdout:3759: writing smithi06387500-39 from 2964784 to 3706300 tid 3 2024-08-17T17:10:45.719 INFO:tasks.rados.rados.0.smithi063.stdout:3760: write oid 30 current snap is 473 2024-08-17T17:10:45.719 INFO:tasks.rados.rados.0.smithi063.stdout:3759: finishing write tid 1 to smithi06387500-39 2024-08-17T17:10:45.719 INFO:tasks.rados.rados.0.smithi063.stdout:3759: finishing write tid 2 to smithi06387500-39 2024-08-17T17:10:45.719 INFO:tasks.rados.rados.0.smithi063.stdout:3760: seq_num 1412 ranges {531484=576893,1843751=525487,2875854=669141} 2024-08-17T17:10:45.725 INFO:tasks.rados.rados.0.smithi063.stdout:3760: writing smithi06387500-30 from 531484 to 1108377 tid 1 2024-08-17T17:10:45.732 INFO:tasks.rados.rados.0.smithi063.stdout:3760: writing smithi06387500-30 from 1843751 to 2369238 tid 2 2024-08-17T17:10:45.739 INFO:tasks.rados.rados.0.smithi063.stdout:3760: writing smithi06387500-30 from 2875854 to 3544995 tid 3 2024-08-17T17:10:45.739 INFO:tasks.rados.rados.0.smithi063.stdout:3761: write oid 1 current snap is 473 2024-08-17T17:10:45.739 INFO:tasks.rados.rados.0.smithi063.stdout:3759: finishing write tid 3 to smithi06387500-39 2024-08-17T17:10:45.739 INFO:tasks.rados.rados.0.smithi063.stdout:3759: finishing write tid 4 to smithi06387500-39 2024-08-17T17:10:45.739 INFO:tasks.rados.rados.0.smithi063.stdout:3759: finishing write tid 5 to smithi06387500-39 2024-08-17T17:10:45.739 INFO:tasks.rados.rados.0.smithi063.stdout:3759: oid 39 updating version 0 to 955 2024-08-17T17:10:45.739 INFO:tasks.rados.rados.0.smithi063.stdout:3759: oid 39 updating version 955 to 956 2024-08-17T17:10:45.739 INFO:tasks.rados.rados.0.smithi063.stdout:3759: oid 39 version 956 is already newer than 953 2024-08-17T17:10:45.739 INFO:tasks.rados.rados.0.smithi063.stdout:3759: oid 39 version 956 is already newer than 954 2024-08-17T17:10:45.739 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 956 (ObjNum 1411 snap 473 seq_num 1411) dirty exists 2024-08-17T17:10:45.739 INFO:tasks.rados.rados.0.smithi063.stdout:3759: left oid 39 (ObjNum 1411 snap 473 seq_num 1411) 2024-08-17T17:10:45.739 INFO:tasks.rados.rados.0.smithi063.stdout:3760: finishing write tid 1 to smithi06387500-30 2024-08-17T17:10:45.740 INFO:tasks.rados.rados.0.smithi063.stdout:3760: finishing write tid 2 to smithi06387500-30 2024-08-17T17:10:45.740 INFO:tasks.rados.rados.0.smithi063.stdout:3761: seq_num 1413 ranges {548291=652142,1902233=785498,3303277=445014} 2024-08-17T17:10:45.745 INFO:tasks.rados.rados.0.smithi063.stdout:3761: writing smithi06387500-1 from 548291 to 1200433 tid 1 2024-08-17T17:10:45.753 INFO:tasks.rados.rados.0.smithi063.stdout:3761: writing smithi06387500-1 from 1902233 to 2687731 tid 2 2024-08-17T17:10:45.759 INFO:tasks.rados.rados.0.smithi063.stdout:3761: writing smithi06387500-1 from 3303277 to 3748291 tid 3 2024-08-17T17:10:45.759 INFO:tasks.rados.rados.0.smithi063.stdout:3759: done (2 left) 2024-08-17T17:10:45.759 INFO:tasks.rados.rados.0.smithi063.stdout:3762: snap_remove snap 471 2024-08-17T17:10:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:45 smithi063 ceph-mon[114756]: pgmap v981: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 253 MiB data, 920 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.7 MiB/s wr, 15 op/s 2024-08-17T17:10:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:45 smithi063 ceph-mon[114756]: osdmap e1017: 8 total, 8 up, 8 in 2024-08-17T17:10:46.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:45 smithi063 ceph-mon[109843]: pgmap v981: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 253 MiB data, 920 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.7 MiB/s wr, 15 op/s 2024-08-17T17:10:46.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:45 smithi063 ceph-mon[109843]: osdmap e1017: 8 total, 8 up, 8 in 2024-08-17T17:10:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:45 smithi162 ceph-mon[104668]: pgmap v981: 89 pgs: 2 active+clean+snaptrim, 87 active+clean; 253 MiB data, 920 MiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.7 MiB/s wr, 15 op/s 2024-08-17T17:10:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:45 smithi162 ceph-mon[104668]: osdmap e1017: 8 total, 8 up, 8 in 2024-08-17T17:10:46.707 INFO:tasks.rados.rados.0.smithi063.stdout:3762: done (2 left) 2024-08-17T17:10:46.707 INFO:tasks.rados.rados.0.smithi063.stdout:3763: snap_create 2024-08-17T17:10:46.707 INFO:tasks.rados.rados.0.smithi063.stdout:3760: finishing write tid 3 to smithi06387500-30 2024-08-17T17:10:46.708 INFO:tasks.rados.rados.0.smithi063.stdout:3760: finishing write tid 4 to smithi06387500-30 2024-08-17T17:10:46.708 INFO:tasks.rados.rados.0.smithi063.stdout:3760: finishing write tid 5 to smithi06387500-30 2024-08-17T17:10:46.708 INFO:tasks.rados.rados.0.smithi063.stdout:3760: oid 30 updating version 0 to 1856 2024-08-17T17:10:46.708 INFO:tasks.rados.rados.0.smithi063.stdout:3760: oid 30 version 1856 is already newer than 1853 2024-08-17T17:10:46.708 INFO:tasks.rados.rados.0.smithi063.stdout:3760: oid 30 version 1856 is already newer than 1854 2024-08-17T17:10:46.708 INFO:tasks.rados.rados.0.smithi063.stdout:3760: oid 30 version 1856 is already newer than 1855 2024-08-17T17:10:46.708 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1856 (ObjNum 1412 snap 473 seq_num 1412) dirty exists 2024-08-17T17:10:46.708 INFO:tasks.rados.rados.0.smithi063.stdout:3760: left oid 30 (ObjNum 1412 snap 473 seq_num 1412) 2024-08-17T17:10:46.708 INFO:tasks.rados.rados.0.smithi063.stdout:3761: finishing write tid 1 to smithi06387500-1 2024-08-17T17:10:46.708 INFO:tasks.rados.rados.0.smithi063.stdout:3761: finishing write tid 2 to smithi06387500-1 2024-08-17T17:10:46.708 INFO:tasks.rados.rados.0.smithi063.stdout:3761: finishing write tid 3 to smithi06387500-1 2024-08-17T17:10:46.708 INFO:tasks.rados.rados.0.smithi063.stdout:3761: finishing write tid 4 to smithi06387500-1 2024-08-17T17:10:46.708 INFO:tasks.rados.rados.0.smithi063.stdout:3761: finishing write tid 5 to smithi06387500-1 2024-08-17T17:10:46.709 INFO:tasks.rados.rados.0.smithi063.stdout:3761: oid 1 updating version 0 to 2031 2024-08-17T17:10:46.709 INFO:tasks.rados.rados.0.smithi063.stdout:3761: oid 1 version 2031 is already newer than 2029 2024-08-17T17:10:46.709 INFO:tasks.rados.rados.0.smithi063.stdout:3761: oid 1 version 2031 is already newer than 2030 2024-08-17T17:10:46.709 INFO:tasks.rados.rados.0.smithi063.stdout:3761: oid 1 version 2031 is already newer than 2028 2024-08-17T17:10:46.709 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 2031 (ObjNum 1413 snap 473 seq_num 1413) dirty exists 2024-08-17T17:10:46.709 INFO:tasks.rados.rados.0.smithi063.stdout:3761: left oid 1 (ObjNum 1413 snap 473 seq_num 1413) 2024-08-17T17:10:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:46 smithi063 ceph-mon[114756]: osdmap e1018: 8 total, 8 up, 8 in 2024-08-17T17:10:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:46 smithi063 ceph-mon[114756]: pgmap v984: 89 pgs: 89 active+clean; 248 MiB data, 916 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-08-17T17:10:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:46 smithi063 ceph-mon[109843]: osdmap e1018: 8 total, 8 up, 8 in 2024-08-17T17:10:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:46 smithi063 ceph-mon[109843]: pgmap v984: 89 pgs: 89 active+clean; 248 MiB data, 916 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-08-17T17:10:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:46 smithi162 ceph-mon[104668]: osdmap e1018: 8 total, 8 up, 8 in 2024-08-17T17:10:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:46 smithi162 ceph-mon[104668]: pgmap v984: 89 pgs: 89 active+clean; 248 MiB data, 916 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-08-17T17:10:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:10:47.683 INFO:tasks.rados.rados.0.smithi063.stdout:3760: done (2 left) 2024-08-17T17:10:47.683 INFO:tasks.rados.rados.0.smithi063.stdout:3761: done (1 left) 2024-08-17T17:10:47.683 INFO:tasks.rados.rados.0.smithi063.stdout:3763: done (0 left) 2024-08-17T17:10:47.684 INFO:tasks.rados.rados.0.smithi063.stdout:3764: write oid 24 current snap is 474 2024-08-17T17:10:47.684 INFO:tasks.rados.rados.0.smithi063.stdout:3764: seq_num 1414 ranges {565098=727391,1960715=529156,2930700=691685} 2024-08-17T17:10:47.690 INFO:tasks.rados.rados.0.smithi063.stdout:3764: writing smithi06387500-24 from 565098 to 1292489 tid 1 2024-08-17T17:10:47.697 INFO:tasks.rados.rados.0.smithi063.stdout:3764: writing smithi06387500-24 from 1960715 to 2489871 tid 2 2024-08-17T17:10:47.704 INFO:tasks.rados.rados.0.smithi063.stdout:3764: writing smithi06387500-24 from 2930700 to 3622385 tid 3 2024-08-17T17:10:47.704 INFO:tasks.rados.rados.0.smithi063.stdout:3765: rollback oid 4 current snap is 474 2024-08-17T17:10:47.705 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 4 to 472 2024-08-17T17:10:47.705 INFO:tasks.rados.rados.0.smithi063.stdout:3764: finishing write tid 1 to smithi06387500-24 2024-08-17T17:10:47.705 INFO:tasks.rados.rados.0.smithi063.stdout:3764: finishing write tid 2 to smithi06387500-24 2024-08-17T17:10:47.705 INFO:tasks.rados.rados.0.smithi063.stdout:3766: write oid 12 current snap is 474 2024-08-17T17:10:47.705 INFO:tasks.rados.rados.0.smithi063.stdout:3766: seq_num 1415 ranges {581905=518993,1735550=672814,2958123=502957} 2024-08-17T17:10:47.710 INFO:tasks.rados.rados.0.smithi063.stdout:3766: writing smithi06387500-12 from 581905 to 1100898 tid 1 2024-08-17T17:10:47.718 INFO:tasks.rados.rados.0.smithi063.stdout:3766: writing smithi06387500-12 from 1735550 to 2408364 tid 2 2024-08-17T17:10:47.724 INFO:tasks.rados.rados.0.smithi063.stdout:3766: writing smithi06387500-12 from 2958123 to 3461080 tid 3 2024-08-17T17:10:47.724 INFO:tasks.rados.rados.0.smithi063.stdout:3767: write oid 29 current snap is 474 2024-08-17T17:10:47.724 INFO:tasks.rados.rados.0.smithi063.stdout:3764: finishing write tid 3 to smithi06387500-24 2024-08-17T17:10:47.724 INFO:tasks.rados.rados.0.smithi063.stdout:3764: finishing write tid 4 to smithi06387500-24 2024-08-17T17:10:47.725 INFO:tasks.rados.rados.0.smithi063.stdout:3764: finishing write tid 5 to smithi06387500-24 2024-08-17T17:10:47.725 INFO:tasks.rados.rados.0.smithi063.stdout:3764: oid 24 updating version 0 to 1431 2024-08-17T17:10:47.725 INFO:tasks.rados.rados.0.smithi063.stdout:3764: oid 24 version 1431 is already newer than 1430 2024-08-17T17:10:47.725 INFO:tasks.rados.rados.0.smithi063.stdout:3764: oid 24 version 1431 is already newer than 1429 2024-08-17T17:10:47.725 INFO:tasks.rados.rados.0.smithi063.stdout:3764: oid 24 version 1431 is already newer than 1428 2024-08-17T17:10:47.725 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 24 v 1431 (ObjNum 1414 snap 474 seq_num 1414) dirty exists 2024-08-17T17:10:47.725 INFO:tasks.rados.rados.0.smithi063.stdout:3764: left oid 24 (ObjNum 1414 snap 474 seq_num 1414) 2024-08-17T17:10:47.725 INFO:tasks.rados.rados.0.smithi063.stdout:3765: finishing rollback tid 1 to smithi06387500-4 2024-08-17T17:10:47.725 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 4 v 3692 (ObjNum 225 snap 0 seq_num 574598576) dirty dne 2024-08-17T17:10:47.725 INFO:tasks.rados.rados.0.smithi063.stdout:3766: finishing write tid 1 to smithi06387500-12 2024-08-17T17:10:47.725 INFO:tasks.rados.rados.0.smithi063.stdout:3766: finishing write tid 2 to smithi06387500-12 2024-08-17T17:10:47.725 INFO:tasks.rados.rados.0.smithi063.stdout:3767: seq_num 1416 ranges {598712=594242,1677679=532825,2985546=430582} 2024-08-17T17:10:47.730 INFO:tasks.rados.rados.0.smithi063.stdout:3767: writing smithi06387500-29 from 598712 to 1192954 tid 1 2024-08-17T17:10:47.736 INFO:tasks.rados.rados.0.smithi063.stdout:3767: writing smithi06387500-29 from 1677679 to 2210504 tid 2 2024-08-17T17:10:47.743 INFO:tasks.rados.rados.0.smithi063.stdout:3767: writing smithi06387500-29 from 2985546 to 3416128 tid 3 2024-08-17T17:10:47.743 INFO:tasks.rados.rados.0.smithi063.stdout:3764: done (3 left) 2024-08-17T17:10:47.743 INFO:tasks.rados.rados.0.smithi063.stdout:3765: done (2 left) 2024-08-17T17:10:47.743 INFO:tasks.rados.rados.0.smithi063.stdout:3768: write oid 40 current snap is 474 2024-08-17T17:10:47.743 INFO:tasks.rados.rados.0.smithi063.stdout:3766: finishing write tid 3 to smithi06387500-12 2024-08-17T17:10:47.743 INFO:tasks.rados.rados.0.smithi063.stdout:3766: finishing write tid 4 to smithi06387500-12 2024-08-17T17:10:47.743 INFO:tasks.rados.rados.0.smithi063.stdout:3766: finishing write tid 5 to smithi06387500-12 2024-08-17T17:10:47.743 INFO:tasks.rados.rados.0.smithi063.stdout:3766: oid 12 updating version 0 to 816 2024-08-17T17:10:47.744 INFO:tasks.rados.rados.0.smithi063.stdout:3766: oid 12 version 816 is already newer than 814 2024-08-17T17:10:47.744 INFO:tasks.rados.rados.0.smithi063.stdout:3766: oid 12 updating version 816 to 817 2024-08-17T17:10:47.744 INFO:tasks.rados.rados.0.smithi063.stdout:3766: oid 12 version 817 is already newer than 815 2024-08-17T17:10:47.744 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 817 (ObjNum 1415 snap 474 seq_num 1415) dirty exists 2024-08-17T17:10:47.744 INFO:tasks.rados.rados.0.smithi063.stdout:3766: left oid 12 (ObjNum 1415 snap 474 seq_num 1415) 2024-08-17T17:10:47.744 INFO:tasks.rados.rados.0.smithi063.stdout:3768: seq_num 1417 ranges {615519=669491,1736161=676483,2896616=641854} 2024-08-17T17:10:47.750 INFO:tasks.rados.rados.0.smithi063.stdout:3768: writing smithi06387500-40 from 615519 to 1285010 tid 1 2024-08-17T17:10:47.756 INFO:tasks.rados.rados.0.smithi063.stdout:3768: writing smithi06387500-40 from 1736161 to 2412644 tid 2 2024-08-17T17:10:47.763 INFO:tasks.rados.rados.0.smithi063.stdout:3768: writing smithi06387500-40 from 2896616 to 3538470 tid 3 2024-08-17T17:10:47.763 INFO:tasks.rados.rados.0.smithi063.stdout:3766: done (2 left) 2024-08-17T17:10:47.763 INFO:tasks.rados.rados.0.smithi063.stdout:3769: read oid 46 snap -1 2024-08-17T17:10:47.763 INFO:tasks.rados.rados.0.smithi063.stdout:3769: expect deleted 2024-08-17T17:10:47.763 INFO:tasks.rados.rados.0.smithi063.stdout:3770: write oid 30 current snap is 474 2024-08-17T17:10:47.764 INFO:tasks.rados.rados.0.smithi063.stdout:3767: finishing write tid 1 to smithi06387500-29 2024-08-17T17:10:47.764 INFO:tasks.rados.rados.0.smithi063.stdout:3767: finishing write tid 2 to smithi06387500-29 2024-08-17T17:10:47.764 INFO:tasks.rados.rados.0.smithi063.stdout:3770: seq_num 1418 ranges {632326=744740,1794643=536494,3040392=453126} 2024-08-17T17:10:47.770 INFO:tasks.rados.rados.0.smithi063.stdout:3770: writing smithi06387500-30 from 632326 to 1377066 tid 1 2024-08-17T17:10:47.777 INFO:tasks.rados.rados.0.smithi063.stdout:3770: writing smithi06387500-30 from 1794643 to 2331137 tid 2 2024-08-17T17:10:47.784 INFO:tasks.rados.rados.0.smithi063.stdout:3770: writing smithi06387500-30 from 3040392 to 3493518 tid 3 2024-08-17T17:10:47.784 INFO:tasks.rados.rados.0.smithi063.stdout:3771: snap_create 2024-08-17T17:10:47.784 INFO:tasks.rados.rados.0.smithi063.stdout:3767: finishing write tid 3 to smithi06387500-29 2024-08-17T17:10:47.784 INFO:tasks.rados.rados.0.smithi063.stdout:3767: finishing write tid 4 to smithi06387500-29 2024-08-17T17:10:47.784 INFO:tasks.rados.rados.0.smithi063.stdout:3767: finishing write tid 5 to smithi06387500-29 2024-08-17T17:10:47.784 INFO:tasks.rados.rados.0.smithi063.stdout:3767: oid 29 updating version 0 to 818 2024-08-17T17:10:47.784 INFO:tasks.rados.rados.0.smithi063.stdout:3767: oid 29 updating version 818 to 821 2024-08-17T17:10:47.784 INFO:tasks.rados.rados.0.smithi063.stdout:3767: oid 29 version 821 is already newer than 820 2024-08-17T17:10:47.784 INFO:tasks.rados.rados.0.smithi063.stdout:3767: oid 29 version 821 is already newer than 819 2024-08-17T17:10:47.785 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 821 (ObjNum 1416 snap 474 seq_num 1416) dirty exists 2024-08-17T17:10:47.785 INFO:tasks.rados.rados.0.smithi063.stdout:3767: left oid 29 (ObjNum 1416 snap 474 seq_num 1416) 2024-08-17T17:10:47.785 INFO:tasks.rados.rados.0.smithi063.stdout:3768: finishing write tid 1 to smithi06387500-40 2024-08-17T17:10:47.785 INFO:tasks.rados.rados.0.smithi063.stdout:3768: finishing write tid 2 to smithi06387500-40 2024-08-17T17:10:47.785 INFO:tasks.rados.rados.0.smithi063.stdout:3768: finishing write tid 3 to smithi06387500-40 2024-08-17T17:10:47.785 INFO:tasks.rados.rados.0.smithi063.stdout:3768: finishing write tid 4 to smithi06387500-40 2024-08-17T17:10:47.785 INFO:tasks.rados.rados.0.smithi063.stdout:3768: finishing write tid 5 to smithi06387500-40 2024-08-17T17:10:47.785 INFO:tasks.rados.rados.0.smithi063.stdout:3768: oid 40 updating version 0 to 1324 2024-08-17T17:10:47.785 INFO:tasks.rados.rados.0.smithi063.stdout:3768: oid 40 version 1324 is already newer than 1322 2024-08-17T17:10:47.785 INFO:tasks.rados.rados.0.smithi063.stdout:3768: oid 40 version 1324 is already newer than 1321 2024-08-17T17:10:47.785 INFO:tasks.rados.rados.0.smithi063.stdout:3768: oid 40 version 1324 is already newer than 1323 2024-08-17T17:10:47.785 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 1324 (ObjNum 1417 snap 474 seq_num 1417) dirty exists 2024-08-17T17:10:47.785 INFO:tasks.rados.rados.0.smithi063.stdout:3768: left oid 40 (ObjNum 1417 snap 474 seq_num 1417) 2024-08-17T17:10:47.786 INFO:tasks.rados.rados.0.smithi063.stdout:3770: finishing write tid 1 to smithi06387500-30 2024-08-17T17:10:47.786 INFO:tasks.rados.rados.0.smithi063.stdout:3770: finishing write tid 2 to smithi06387500-30 2024-08-17T17:10:47.787 INFO:tasks.rados.rados.0.smithi063.stdout:3770: finishing write tid 3 to smithi06387500-30 2024-08-17T17:10:47.787 INFO:tasks.rados.rados.0.smithi063.stdout:3770: finishing write tid 4 to smithi06387500-30 2024-08-17T17:10:47.787 INFO:tasks.rados.rados.0.smithi063.stdout:3770: finishing write tid 5 to smithi06387500-30 2024-08-17T17:10:47.787 INFO:tasks.rados.rados.0.smithi063.stdout:3770: oid 30 updating version 0 to 1862 2024-08-17T17:10:47.787 INFO:tasks.rados.rados.0.smithi063.stdout:3770: oid 30 version 1862 is already newer than 1861 2024-08-17T17:10:47.787 INFO:tasks.rados.rados.0.smithi063.stdout:3770: oid 30 updating version 1862 to 1863 2024-08-17T17:10:47.787 INFO:tasks.rados.rados.0.smithi063.stdout:3770: oid 30 version 1863 is already newer than 1860 2024-08-17T17:10:47.787 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1863 (ObjNum 1418 snap 474 seq_num 1418) dirty exists 2024-08-17T17:10:47.788 INFO:tasks.rados.rados.0.smithi063.stdout:3770: left oid 30 (ObjNum 1418 snap 474 seq_num 1418) 2024-08-17T17:10:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:47 smithi063 ceph-mon[114756]: osdmap e1019: 8 total, 8 up, 8 in 2024-08-17T17:10:48.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:47 smithi063 ceph-mon[114756]: osdmap e1020: 8 total, 8 up, 8 in 2024-08-17T17:10:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:47 smithi063 ceph-mon[109843]: osdmap e1019: 8 total, 8 up, 8 in 2024-08-17T17:10:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:47 smithi063 ceph-mon[109843]: osdmap e1020: 8 total, 8 up, 8 in 2024-08-17T17:10:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:47 smithi162 ceph-mon[104668]: osdmap e1019: 8 total, 8 up, 8 in 2024-08-17T17:10:48.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:47 smithi162 ceph-mon[104668]: osdmap e1020: 8 total, 8 up, 8 in 2024-08-17T17:10:48.712 INFO:tasks.rados.rados.0.smithi063.stdout:3767: done (4 left) 2024-08-17T17:10:48.712 INFO:tasks.rados.rados.0.smithi063.stdout:3768: done (3 left) 2024-08-17T17:10:48.712 INFO:tasks.rados.rados.0.smithi063.stdout:3769: done (2 left) 2024-08-17T17:10:48.712 INFO:tasks.rados.rados.0.smithi063.stdout:3770: done (1 left) 2024-08-17T17:10:48.712 INFO:tasks.rados.rados.0.smithi063.stdout:3771: done (0 left) 2024-08-17T17:10:48.712 INFO:tasks.rados.rados.0.smithi063.stdout:3772: read oid 33 snap -1 2024-08-17T17:10:48.712 INFO:tasks.rados.rados.0.smithi063.stdout:3772: expect (ObjNum 1311 snap 443 seq_num 1311) 2024-08-17T17:10:48.712 INFO:tasks.rados.rados.0.smithi063.stdout:3773: snap_remove snap 453 2024-08-17T17:10:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:48 smithi063 ceph-mon[114756]: pgmap v987: 89 pgs: 89 active+clean; 255 MiB data, 923 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.4 MiB/s wr, 4 op/s 2024-08-17T17:10:49.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:48 smithi063 ceph-mon[109843]: pgmap v987: 89 pgs: 89 active+clean; 255 MiB data, 923 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.4 MiB/s wr, 4 op/s 2024-08-17T17:10:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:48 smithi162 ceph-mon[104668]: pgmap v987: 89 pgs: 89 active+clean; 255 MiB data, 923 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.4 MiB/s wr, 4 op/s 2024-08-17T17:10:49.714 INFO:tasks.rados.rados.0.smithi063.stdout:3773: done (1 left) 2024-08-17T17:10:49.714 INFO:tasks.rados.rados.0.smithi063.stdout:3774: snap_remove snap 443 2024-08-17T17:10:50.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:49 smithi063 ceph-mon[109843]: osdmap e1021: 8 total, 8 up, 8 in 2024-08-17T17:10:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:49 smithi063 ceph-mon[114756]: osdmap e1021: 8 total, 8 up, 8 in 2024-08-17T17:10:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:49 smithi162 ceph-mon[104668]: osdmap e1021: 8 total, 8 up, 8 in 2024-08-17T17:10:50.720 INFO:tasks.rados.rados.0.smithi063.stdout:3774: done (1 left) 2024-08-17T17:10:50.720 INFO:tasks.rados.rados.0.smithi063.stdout:3775: read oid 4 snap -1 2024-08-17T17:10:50.720 INFO:tasks.rados.rados.0.smithi063.stdout:3775: expect deleted 2024-08-17T17:10:50.720 INFO:tasks.rados.rados.0.smithi063.stdout:3776: write oid 1 current snap is 475 2024-08-17T17:10:50.720 INFO:tasks.rados.rados.0.smithi063.stdout:3776: seq_num 1419 ranges {649133=419989,1853125=680152,2951462=664398} 2024-08-17T17:10:50.727 INFO:tasks.rados.rados.0.smithi063.stdout:3776: writing smithi06387500-1 from 649133 to 1069122 tid 1 2024-08-17T17:10:50.736 INFO:tasks.rados.rados.0.smithi063.stdout:3776: writing smithi06387500-1 from 1853125 to 2533277 tid 2 2024-08-17T17:10:50.743 INFO:tasks.rados.rados.0.smithi063.stdout:3776: writing smithi06387500-1 from 2951462 to 3615860 tid 3 2024-08-17T17:10:50.743 INFO:tasks.rados.rados.0.smithi063.stdout:3777: rollback oid 44 current snap is 475 2024-08-17T17:10:50.743 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 44 to 441 2024-08-17T17:10:50.743 INFO:tasks.rados.rados.0.smithi063.stdout:3772: expect (ObjNum 1311 snap 443 seq_num 1311) 2024-08-17T17:10:50.812 INFO:tasks.rados.rados.0.smithi063.stdout:3776: finishing write tid 1 to smithi06387500-1 2024-08-17T17:10:50.812 INFO:tasks.rados.rados.0.smithi063.stdout:3776: finishing write tid 2 to smithi06387500-1 2024-08-17T17:10:50.812 INFO:tasks.rados.rados.0.smithi063.stdout:3777: finishing rollback tid 0 to smithi06387500-44 2024-08-17T17:10:50.812 INFO:tasks.rados.rados.0.smithi063.stdout:3777: finishing rollback tid 1 to smithi06387500-44 2024-08-17T17:10:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3776: finishing write tid 3 to smithi06387500-1 2024-08-17T17:10:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3776: finishing write tid 4 to smithi06387500-1 2024-08-17T17:10:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3777: finishing rollback tid 2 to smithi06387500-44 2024-08-17T17:10:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 544 (ObjNum 1218 snap 407 seq_num 1218) dirty exists 2024-08-17T17:10:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3776: finishing write tid 5 to smithi06387500-1 2024-08-17T17:10:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3776: oid 1 updating version 0 to 2050 2024-08-17T17:10:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3776: oid 1 version 2050 is already newer than 2049 2024-08-17T17:10:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3776: oid 1 updating version 2050 to 2053 2024-08-17T17:10:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3776: oid 1 updating version 2053 to 2054 2024-08-17T17:10:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 1 v 2054 (ObjNum 1419 snap 475 seq_num 1419) dirty exists 2024-08-17T17:10:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3776: left oid 1 (ObjNum 1419 snap 475 seq_num 1419) 2024-08-17T17:10:50.813 INFO:tasks.rados.rados.0.smithi063.stdout:3772: done (3 left) 2024-08-17T17:10:50.814 INFO:tasks.rados.rados.0.smithi063.stdout:3775: done (2 left) 2024-08-17T17:10:50.814 INFO:tasks.rados.rados.0.smithi063.stdout:3776: done (1 left) 2024-08-17T17:10:50.814 INFO:tasks.rados.rados.0.smithi063.stdout:3777: done (0 left) 2024-08-17T17:10:50.814 INFO:tasks.rados.rados.0.smithi063.stdout:3778: write oid 11 current snap is 475 2024-08-17T17:10:50.814 INFO:tasks.rados.rados.0.smithi063.stdout:3778: seq_num 1420 ranges {665940=495238,1795254=540163,2978885=475670} 2024-08-17T17:10:50.818 INFO:tasks.rados.rados.0.smithi063.stdout:3778: writing smithi06387500-11 from 665940 to 1161178 tid 1 2024-08-17T17:10:50.824 INFO:tasks.rados.rados.0.smithi063.stdout:3778: writing smithi06387500-11 from 1795254 to 2335417 tid 2 2024-08-17T17:10:50.831 INFO:tasks.rados.rados.0.smithi063.stdout:3778: writing smithi06387500-11 from 2978885 to 3454555 tid 3 2024-08-17T17:10:50.831 INFO:tasks.rados.rados.0.smithi063.stdout:3779: write oid 15 current snap is 475 2024-08-17T17:10:50.831 INFO:tasks.rados.rados.0.smithi063.stdout:3779: seq_num 1421 ranges {682747=570487,1853736=683821,3006308=403295} 2024-08-17T17:10:50.838 INFO:tasks.rados.rados.0.smithi063.stdout:3779: writing smithi06387500-15 from 682747 to 1253234 tid 1 2024-08-17T17:10:50.845 INFO:tasks.rados.rados.0.smithi063.stdout:3779: writing smithi06387500-15 from 1853736 to 2537557 tid 2 2024-08-17T17:10:50.850 INFO:tasks.rados.rados.0.smithi063.stdout:3779: writing smithi06387500-15 from 3006308 to 3409603 tid 3 2024-08-17T17:10:50.850 INFO:tasks.rados.rados.0.smithi063.stdout:3780: snap_create 2024-08-17T17:10:50.850 INFO:tasks.rados.rados.0.smithi063.stdout:3778: finishing write tid 1 to smithi06387500-11 2024-08-17T17:10:50.850 INFO:tasks.rados.rados.0.smithi063.stdout:3778: finishing write tid 2 to smithi06387500-11 2024-08-17T17:10:50.850 INFO:tasks.rados.rados.0.smithi063.stdout:3778: finishing write tid 3 to smithi06387500-11 2024-08-17T17:10:50.851 INFO:tasks.rados.rados.0.smithi063.stdout:3778: finishing write tid 4 to smithi06387500-11 2024-08-17T17:10:50.851 INFO:tasks.rados.rados.0.smithi063.stdout:3778: finishing write tid 5 to smithi06387500-11 2024-08-17T17:10:50.851 INFO:tasks.rados.rados.0.smithi063.stdout:3778: oid 11 updating version 0 to 735 2024-08-17T17:10:50.851 INFO:tasks.rados.rados.0.smithi063.stdout:3778: oid 11 updating version 735 to 737 2024-08-17T17:10:50.851 INFO:tasks.rados.rados.0.smithi063.stdout:3778: oid 11 updating version 737 to 738 2024-08-17T17:10:50.851 INFO:tasks.rados.rados.0.smithi063.stdout:3778: oid 11 version 738 is already newer than 736 2024-08-17T17:10:50.851 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 738 (ObjNum 1420 snap 475 seq_num 1420) dirty exists 2024-08-17T17:10:50.851 INFO:tasks.rados.rados.0.smithi063.stdout:3778: left oid 11 (ObjNum 1420 snap 475 seq_num 1420) 2024-08-17T17:10:50.853 INFO:tasks.rados.rados.0.smithi063.stdout:3779: finishing write tid 1 to smithi06387500-15 2024-08-17T17:10:50.853 INFO:tasks.rados.rados.0.smithi063.stdout:3779: finishing write tid 2 to smithi06387500-15 2024-08-17T17:10:50.854 INFO:tasks.rados.rados.0.smithi063.stdout:3779: finishing write tid 3 to smithi06387500-15 2024-08-17T17:10:50.854 INFO:tasks.rados.rados.0.smithi063.stdout:3779: finishing write tid 4 to smithi06387500-15 2024-08-17T17:10:50.854 INFO:tasks.rados.rados.0.smithi063.stdout:3779: finishing write tid 5 to smithi06387500-15 2024-08-17T17:10:50.855 INFO:tasks.rados.rados.0.smithi063.stdout:3779: oid 15 updating version 0 to 1445 2024-08-17T17:10:50.855 INFO:tasks.rados.rados.0.smithi063.stdout:3779: oid 15 updating version 1445 to 1448 2024-08-17T17:10:50.855 INFO:tasks.rados.rados.0.smithi063.stdout:3779: oid 15 version 1448 is already newer than 1446 2024-08-17T17:10:50.855 INFO:tasks.rados.rados.0.smithi063.stdout:3779: oid 15 version 1448 is already newer than 1447 2024-08-17T17:10:50.855 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 1448 (ObjNum 1421 snap 475 seq_num 1421) dirty exists 2024-08-17T17:10:50.855 INFO:tasks.rados.rados.0.smithi063.stdout:3779: left oid 15 (ObjNum 1421 snap 475 seq_num 1421) 2024-08-17T17:10:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:50 smithi063 ceph-mon[114756]: osdmap e1022: 8 total, 8 up, 8 in 2024-08-17T17:10:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:50 smithi063 ceph-mon[114756]: pgmap v990: 89 pgs: 89 active+clean; 255 MiB data, 925 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.4 MiB/s wr, 4 op/s 2024-08-17T17:10:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:50 smithi063 ceph-mon[114756]: osdmap e1023: 8 total, 8 up, 8 in 2024-08-17T17:10:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:50 smithi063 ceph-mon[109843]: osdmap e1022: 8 total, 8 up, 8 in 2024-08-17T17:10:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:50 smithi063 ceph-mon[109843]: pgmap v990: 89 pgs: 89 active+clean; 255 MiB data, 925 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.4 MiB/s wr, 4 op/s 2024-08-17T17:10:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:50 smithi063 ceph-mon[109843]: osdmap e1023: 8 total, 8 up, 8 in 2024-08-17T17:10:51.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:50 smithi162 ceph-mon[104668]: osdmap e1022: 8 total, 8 up, 8 in 2024-08-17T17:10:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:50 smithi162 ceph-mon[104668]: pgmap v990: 89 pgs: 89 active+clean; 255 MiB data, 925 MiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.4 MiB/s wr, 4 op/s 2024-08-17T17:10:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:50 smithi162 ceph-mon[104668]: osdmap e1023: 8 total, 8 up, 8 in 2024-08-17T17:10:51.742 INFO:tasks.rados.rados.0.smithi063.stdout:3778: done (2 left) 2024-08-17T17:10:51.742 INFO:tasks.rados.rados.0.smithi063.stdout:3779: done (1 left) 2024-08-17T17:10:51.743 INFO:tasks.rados.rados.0.smithi063.stdout:3780: done (0 left) 2024-08-17T17:10:51.743 INFO:tasks.rados.rados.0.smithi063.stdout:3781: read oid 30 snap 474 2024-08-17T17:10:51.743 INFO:tasks.rados.rados.0.smithi063.stdout:3781: expect (ObjNum 1418 snap 474 seq_num 1418) 2024-08-17T17:10:51.743 INFO:tasks.rados.rados.0.smithi063.stdout:3782: write oid 29 current snap is 476 2024-08-17T17:10:51.743 INFO:tasks.rados.rados.0.smithi063.stdout:3782: seq_num 1422 ranges {699554=762089,2028571=543832,3150084=614567} 2024-08-17T17:10:51.750 INFO:tasks.rados.rados.0.smithi063.stdout:3782: writing smithi06387500-29 from 699554 to 1461643 tid 1 2024-08-17T17:10:51.756 INFO:tasks.rados.rados.0.smithi063.stdout:3782: writing smithi06387500-29 from 2028571 to 2572403 tid 2 2024-08-17T17:10:51.763 INFO:tasks.rados.rados.0.smithi063.stdout:3782: writing smithi06387500-29 from 3150084 to 3764651 tid 3 2024-08-17T17:10:51.763 INFO:tasks.rados.rados.0.smithi063.stdout:3783: snap_remove snap 441 2024-08-17T17:10:52.687 INFO:tasks.rados.rados.0.smithi063.stdout:3783: done (2 left) 2024-08-17T17:10:52.687 INFO:tasks.rados.rados.0.smithi063.stdout:3784: write oid 11 current snap is 476 2024-08-17T17:10:52.687 INFO:tasks.rados.rados.0.smithi063.stdout:3784: seq_num 1423 ranges {716361=437338,1687053=687490,2777507=425839,3877756=38605} 2024-08-17T17:10:52.694 INFO:tasks.rados.rados.0.smithi063.stdout:3784: writing smithi06387500-11 from 716361 to 1153699 tid 1 2024-08-17T17:10:52.702 INFO:tasks.rados.rados.0.smithi063.stdout:3784: writing smithi06387500-11 from 1687053 to 2374543 tid 2 2024-08-17T17:10:52.707 INFO:tasks.rados.rados.0.smithi063.stdout:3784: writing smithi06387500-11 from 2777507 to 3203346 tid 3 2024-08-17T17:10:52.711 INFO:tasks.rados.rados.0.smithi063.stdout:3784: writing smithi06387500-11 from 3877756 to 3916361 tid 4 2024-08-17T17:10:52.711 INFO:tasks.rados.rados.0.smithi063.stdout:3785: write oid 35 current snap is 476 2024-08-17T17:10:52.711 INFO:tasks.rados.rados.0.smithi063.stdout:3785: seq_num 1424 ranges {733168=512587,2029182=547501,3088577=637111} 2024-08-17T17:10:52.718 INFO:tasks.rados.rados.0.smithi063.stdout:3785: writing smithi06387500-35 from 733168 to 1245755 tid 1 2024-08-17T17:10:52.725 INFO:tasks.rados.rados.0.smithi063.stdout:3785: writing smithi06387500-35 from 2029182 to 2576683 tid 2 2024-08-17T17:10:52.732 INFO:tasks.rados.rados.0.smithi063.stdout:3785: writing smithi06387500-35 from 3088577 to 3725688 tid 3 2024-08-17T17:10:52.732 INFO:tasks.rados.rados.0.smithi063.stdout:3786: write oid 25 current snap is 476 2024-08-17T17:10:52.732 INFO:tasks.rados.rados.0.smithi063.stdout:3782: finishing write tid 1 to smithi06387500-29 2024-08-17T17:10:52.732 INFO:tasks.rados.rados.0.smithi063.stdout:3781: expect (ObjNum 1418 snap 474 seq_num 1418) 2024-08-17T17:10:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:52 smithi063 ceph-mon[114756]: osdmap e1024: 8 total, 8 up, 8 in 2024-08-17T17:10:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:52 smithi063 ceph-mon[114756]: pgmap v993: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 241 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.5 MiB/s wr, 11 op/s 2024-08-17T17:10:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:52 smithi063 ceph-mon[114756]: osdmap e1025: 8 total, 8 up, 8 in 2024-08-17T17:10:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:52 smithi063 ceph-mon[109843]: osdmap e1024: 8 total, 8 up, 8 in 2024-08-17T17:10:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:52 smithi063 ceph-mon[109843]: pgmap v993: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 241 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.5 MiB/s wr, 11 op/s 2024-08-17T17:10:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:52 smithi063 ceph-mon[109843]: osdmap e1025: 8 total, 8 up, 8 in 2024-08-17T17:10:53.061 INFO:tasks.rados.rados.0.smithi063.stdout:3782: finishing write tid 2 to smithi06387500-29 2024-08-17T17:10:53.061 INFO:tasks.rados.rados.0.smithi063.stdout:3782: finishing write tid 3 to smithi06387500-29 2024-08-17T17:10:53.061 INFO:tasks.rados.rados.0.smithi063.stdout:3786: seq_num 1425 ranges {749975=587836,2087664=691159,3516000=433975} 2024-08-17T17:10:53.068 INFO:tasks.rados.rados.0.smithi063.stdout:3786: writing smithi06387500-25 from 749975 to 1337811 tid 1 2024-08-17T17:10:53.076 INFO:tasks.rados.rados.0.smithi063.stdout:3786: writing smithi06387500-25 from 2087664 to 2778823 tid 2 2024-08-17T17:10:53.084 INFO:tasks.rados.rados.0.smithi063.stdout:3786: writing smithi06387500-25 from 3516000 to 3949975 tid 3 2024-08-17T17:10:53.084 INFO:tasks.rados.rados.0.smithi063.stdout:3781: done (4 left) 2024-08-17T17:10:53.084 INFO:tasks.rados.rados.0.smithi063.stdout:3787: rollback oid 39 current snap is 476 2024-08-17T17:10:53.084 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 39 to 475 2024-08-17T17:10:53.084 INFO:tasks.rados.rados.0.smithi063.stdout:3782: finishing write tid 4 to smithi06387500-29 2024-08-17T17:10:53.084 INFO:tasks.rados.rados.0.smithi063.stdout:3782: finishing write tid 5 to smithi06387500-29 2024-08-17T17:10:53.084 INFO:tasks.rados.rados.0.smithi063.stdout:3782: oid 29 updating version 0 to 825 2024-08-17T17:10:53.084 INFO:tasks.rados.rados.0.smithi063.stdout:3782: oid 29 updating version 825 to 828 2024-08-17T17:10:53.084 INFO:tasks.rados.rados.0.smithi063.stdout:3782: oid 29 version 828 is already newer than 826 2024-08-17T17:10:53.084 INFO:tasks.rados.rados.0.smithi063.stdout:3782: oid 29 version 828 is already newer than 827 2024-08-17T17:10:53.084 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 828 (ObjNum 1422 snap 476 seq_num 1422) dirty exists 2024-08-17T17:10:53.085 INFO:tasks.rados.rados.0.smithi063.stdout:3782: left oid 29 (ObjNum 1422 snap 476 seq_num 1422) 2024-08-17T17:10:53.085 INFO:tasks.rados.rados.0.smithi063.stdout:3784: finishing write tid 1 to smithi06387500-11 2024-08-17T17:10:53.085 INFO:tasks.rados.rados.0.smithi063.stdout:3784: finishing write tid 2 to smithi06387500-11 2024-08-17T17:10:53.085 INFO:tasks.rados.rados.0.smithi063.stdout:3782: done (4 left) 2024-08-17T17:10:53.085 INFO:tasks.rados.rados.0.smithi063.stdout:3788: delete oid 29 current snap is 476 2024-08-17T17:10:53.085 INFO:tasks.rados.rados.0.smithi063.stdout:3784: finishing write tid 3 to smithi06387500-11 2024-08-17T17:10:53.085 INFO:tasks.rados.rados.0.smithi063.stdout:3784: finishing write tid 4 to smithi06387500-11 2024-08-17T17:10:53.085 INFO:tasks.rados.rados.0.smithi063.stdout:3784: finishing write tid 5 to smithi06387500-11 2024-08-17T17:10:53.085 INFO:tasks.rados.rados.0.smithi063.stdout:3784: finishing write tid 6 to smithi06387500-11 2024-08-17T17:10:53.085 INFO:tasks.rados.rados.0.smithi063.stdout:3784: oid 11 updating version 0 to 746 2024-08-17T17:10:53.085 INFO:tasks.rados.rados.0.smithi063.stdout:3784: oid 11 version 746 is already newer than 740 2024-08-17T17:10:53.085 INFO:tasks.rados.rados.0.smithi063.stdout:3784: oid 11 version 746 is already newer than 744 2024-08-17T17:10:53.085 INFO:tasks.rados.rados.0.smithi063.stdout:3784: oid 11 version 746 is already newer than 745 2024-08-17T17:10:53.085 INFO:tasks.rados.rados.0.smithi063.stdout:3784: oid 11 version 746 is already newer than 743 2024-08-17T17:10:53.085 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 11 v 746 (ObjNum 1423 snap 476 seq_num 1423) dirty exists 2024-08-17T17:10:53.086 INFO:tasks.rados.rados.0.smithi063.stdout:3784: left oid 11 (ObjNum 1423 snap 476 seq_num 1423) 2024-08-17T17:10:53.086 INFO:tasks.rados.rados.0.smithi063.stdout:3785: finishing write tid 1 to smithi06387500-35 2024-08-17T17:10:53.086 INFO:tasks.rados.rados.0.smithi063.stdout:3785: finishing write tid 2 to smithi06387500-35 2024-08-17T17:10:53.086 INFO:tasks.rados.rados.0.smithi063.stdout:3785: finishing write tid 3 to smithi06387500-35 2024-08-17T17:10:53.086 INFO:tasks.rados.rados.0.smithi063.stdout:3785: finishing write tid 4 to smithi06387500-35 2024-08-17T17:10:53.086 INFO:tasks.rados.rados.0.smithi063.stdout:3785: finishing write tid 5 to smithi06387500-35 2024-08-17T17:10:53.086 INFO:tasks.rados.rados.0.smithi063.stdout:3785: oid 35 updating version 0 to 3757 2024-08-17T17:10:53.086 INFO:tasks.rados.rados.0.smithi063.stdout:3785: oid 35 version 3757 is already newer than 3754 2024-08-17T17:10:53.086 INFO:tasks.rados.rados.0.smithi063.stdout:3785: oid 35 version 3757 is already newer than 3756 2024-08-17T17:10:53.086 INFO:tasks.rados.rados.0.smithi063.stdout:3785: oid 35 version 3757 is already newer than 3755 2024-08-17T17:10:53.086 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 3757 (ObjNum 1424 snap 476 seq_num 1424) dirty exists 2024-08-17T17:10:53.086 INFO:tasks.rados.rados.0.smithi063.stdout:3785: left oid 35 (ObjNum 1424 snap 476 seq_num 1424) 2024-08-17T17:10:53.086 INFO:tasks.rados.rados.0.smithi063.stdout:3786: finishing write tid 1 to smithi06387500-25 2024-08-17T17:10:53.087 INFO:tasks.rados.rados.0.smithi063.stdout:3786: finishing write tid 2 to smithi06387500-25 2024-08-17T17:10:53.087 INFO:tasks.rados.rados.0.smithi063.stdout:3784: done (4 left) 2024-08-17T17:10:53.087 INFO:tasks.rados.rados.0.smithi063.stdout:3785: done (3 left) 2024-08-17T17:10:53.087 INFO:tasks.rados.rados.0.smithi063.stdout:3788: done (2 left) 2024-08-17T17:10:53.087 INFO:tasks.rados.rados.0.smithi063.stdout:3789: write oid 22 current snap is 476 2024-08-17T17:10:53.087 INFO:tasks.rados.rados.0.smithi063.stdout:3789: seq_num 1427 ranges {783589=738334,2204628=694828,3570846=412743} 2024-08-17T17:10:53.093 INFO:tasks.rados.rados.0.smithi063.stdout:3789: writing smithi06387500-22 from 783589 to 1521923 tid 1 2024-08-17T17:10:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:52 smithi162 ceph-mon[104668]: osdmap e1024: 8 total, 8 up, 8 in 2024-08-17T17:10:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:52 smithi162 ceph-mon[104668]: pgmap v993: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 241 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.5 MiB/s wr, 11 op/s 2024-08-17T17:10:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:52 smithi162 ceph-mon[104668]: osdmap e1025: 8 total, 8 up, 8 in 2024-08-17T17:10:53.100 INFO:tasks.rados.rados.0.smithi063.stdout:3789: writing smithi06387500-22 from 2204628 to 2899456 tid 2 2024-08-17T17:10:53.107 INFO:tasks.rados.rados.0.smithi063.stdout:3789: writing smithi06387500-22 from 3570846 to 3983589 tid 3 2024-08-17T17:10:53.108 INFO:tasks.rados.rados.0.smithi063.stdout:3790: rollback oid 50 current snap is 476 2024-08-17T17:10:53.108 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 50 to 474 2024-08-17T17:10:53.108 INFO:tasks.rados.rados.0.smithi063.stdout:3787: finishing rollback tid 0 to smithi06387500-39 2024-08-17T17:10:53.108 INFO:tasks.rados.rados.0.smithi063.stdout:3787: finishing rollback tid 1 to smithi06387500-39 2024-08-17T17:10:53.108 INFO:tasks.rados.rados.0.smithi063.stdout:3787: finishing rollback tid 2 to smithi06387500-39 2024-08-17T17:10:53.108 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 970 (ObjNum 1411 snap 473 seq_num 1411) dirty exists 2024-08-17T17:10:53.108 INFO:tasks.rados.rados.0.smithi063.stdout:3786: finishing write tid 3 to smithi06387500-25 2024-08-17T17:10:53.108 INFO:tasks.rados.rados.0.smithi063.stdout:3786: finishing write tid 4 to smithi06387500-25 2024-08-17T17:10:53.108 INFO:tasks.rados.rados.0.smithi063.stdout:3786: finishing write tid 5 to smithi06387500-25 2024-08-17T17:10:53.108 INFO:tasks.rados.rados.0.smithi063.stdout:3786: oid 25 updating version 0 to 3760 2024-08-17T17:10:53.108 INFO:tasks.rados.rados.0.smithi063.stdout:3786: oid 25 version 3760 is already newer than 3759 2024-08-17T17:10:53.108 INFO:tasks.rados.rados.0.smithi063.stdout:3786: oid 25 updating version 3760 to 3761 2024-08-17T17:10:53.109 INFO:tasks.rados.rados.0.smithi063.stdout:3786: oid 25 version 3761 is already newer than 3758 2024-08-17T17:10:53.109 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 3761 (ObjNum 1425 snap 476 seq_num 1425) dirty exists 2024-08-17T17:10:53.109 INFO:tasks.rados.rados.0.smithi063.stdout:3786: left oid 25 (ObjNum 1425 snap 476 seq_num 1425) 2024-08-17T17:10:53.109 INFO:tasks.rados.rados.0.smithi063.stdout:3789: finishing write tid 1 to smithi06387500-22 2024-08-17T17:10:53.109 INFO:tasks.rados.rados.0.smithi063.stdout:3786: done (3 left) 2024-08-17T17:10:53.109 INFO:tasks.rados.rados.0.smithi063.stdout:3787: done (2 left) 2024-08-17T17:10:53.109 INFO:tasks.rados.rados.0.smithi063.stdout:3791: read oid 11 snap 473 2024-08-17T17:10:53.109 INFO:tasks.rados.rados.0.smithi063.stdout:3791: expect (ObjNum 1406 snap 470 seq_num 1406) 2024-08-17T17:10:53.109 INFO:tasks.rados.rados.0.smithi063.stdout:3792: read oid 8 snap -1 2024-08-17T17:10:53.109 INFO:tasks.rados.rados.0.smithi063.stdout:3792: expect (ObjNum 1391 snap 466 seq_num 1391) 2024-08-17T17:10:53.109 INFO:tasks.rados.rados.0.smithi063.stdout:3793: rollback oid 31 current snap is 476 2024-08-17T17:10:53.109 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 31 to 470 2024-08-17T17:10:53.109 INFO:tasks.rados.rados.0.smithi063.stdout:3789: finishing write tid 2 to smithi06387500-22 2024-08-17T17:10:53.109 INFO:tasks.rados.rados.0.smithi063.stdout:3794: read oid 9 snap 470 2024-08-17T17:10:53.109 INFO:tasks.rados.rados.0.smithi063.stdout:3794: expect (ObjNum 1401 snap 470 seq_num 1401) 2024-08-17T17:10:53.110 INFO:tasks.rados.rados.0.smithi063.stdout:3795: snap_create 2024-08-17T17:10:53.110 INFO:tasks.rados.rados.0.smithi063.stdout:3789: finishing write tid 3 to smithi06387500-22 2024-08-17T17:10:53.111 INFO:tasks.rados.rados.0.smithi063.stdout:3789: finishing write tid 4 to smithi06387500-22 2024-08-17T17:10:53.112 INFO:tasks.rados.rados.0.smithi063.stdout:3791: expect (ObjNum 1406 snap 470 seq_num 1406) 2024-08-17T17:10:53.233 INFO:tasks.rados.rados.0.smithi063.stdout:3789: finishing write tid 5 to smithi06387500-22 2024-08-17T17:10:53.233 INFO:tasks.rados.rados.0.smithi063.stdout:3789: oid 22 updating version 0 to 980 2024-08-17T17:10:53.233 INFO:tasks.rados.rados.0.smithi063.stdout:3789: oid 22 updating version 980 to 981 2024-08-17T17:10:53.233 INFO:tasks.rados.rados.0.smithi063.stdout:3789: oid 22 updating version 981 to 982 2024-08-17T17:10:53.234 INFO:tasks.rados.rados.0.smithi063.stdout:3789: oid 22 updating version 982 to 983 2024-08-17T17:10:53.234 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 983 (ObjNum 1427 snap 476 seq_num 1427) dirty exists 2024-08-17T17:10:53.234 INFO:tasks.rados.rados.0.smithi063.stdout:3789: left oid 22 (ObjNum 1427 snap 476 seq_num 1427) 2024-08-17T17:10:53.234 INFO:tasks.rados.rados.0.smithi063.stdout:3793: finishing rollback tid 1 to smithi06387500-31 2024-08-17T17:10:53.234 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-08-17T17:10:53.234 INFO:tasks.rados.rados.0.smithi063.stdout:3790: finishing rollback tid 0 to smithi06387500-50 2024-08-17T17:10:53.234 INFO:tasks.rados.rados.0.smithi063.stdout:3790: finishing rollback tid 1 to smithi06387500-50 2024-08-17T17:10:53.234 INFO:tasks.rados.rados.0.smithi063.stdout:3790: finishing rollback tid 2 to smithi06387500-50 2024-08-17T17:10:53.234 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 2397 (ObjNum 1088 snap 363 seq_num 1088) dirty exists 2024-08-17T17:10:53.234 INFO:tasks.rados.rados.0.smithi063.stdout:3792: expect (ObjNum 1391 snap 466 seq_num 1391) 2024-08-17T17:10:53.343 INFO:tasks.rados.rados.0.smithi063.stdout:3794: expect (ObjNum 1401 snap 470 seq_num 1401) 2024-08-17T17:10:53.746 INFO:tasks.rados.rados.0.smithi063.stdout:3789: done (6 left) 2024-08-17T17:10:53.746 INFO:tasks.rados.rados.0.smithi063.stdout:3790: done (5 left) 2024-08-17T17:10:53.747 INFO:tasks.rados.rados.0.smithi063.stdout:3791: done (4 left) 2024-08-17T17:10:53.747 INFO:tasks.rados.rados.0.smithi063.stdout:3792: done (3 left) 2024-08-17T17:10:53.747 INFO:tasks.rados.rados.0.smithi063.stdout:3793: done (2 left) 2024-08-17T17:10:53.747 INFO:tasks.rados.rados.0.smithi063.stdout:3794: done (1 left) 2024-08-17T17:10:53.747 INFO:tasks.rados.rados.0.smithi063.stdout:3795: done (0 left) 2024-08-17T17:10:53.747 INFO:tasks.rados.rados.0.smithi063.stdout:3796: snap_create 2024-08-17T17:10:53.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:10:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:10:53] "GET /metrics HTTP/1.1" 200 36338 "" "Prometheus/2.51.0" 2024-08-17T17:10:54.753 INFO:tasks.rados.rados.0.smithi063.stdout:3796: done (0 left) 2024-08-17T17:10:54.754 INFO:tasks.rados.rados.0.smithi063.stdout:3797: delete oid 32 current snap is 478 2024-08-17T17:10:54.758 INFO:tasks.rados.rados.0.smithi063.stdout:3797: done (0 left) 2024-08-17T17:10:54.758 INFO:tasks.rados.rados.0.smithi063.stdout:3798: write oid 40 current snap is 478 2024-08-17T17:10:54.759 INFO:tasks.rados.rados.0.smithi063.stdout:3798: seq_num 1429 ranges {417203=488832,1405239=611964} 2024-08-17T17:10:54.763 INFO:tasks.rados.rados.0.smithi063.stdout:3798: writing smithi06387500-40 from 417203 to 906035 tid 1 2024-08-17T17:10:54.769 INFO:tasks.rados.rados.0.smithi063.stdout:3798: writing smithi06387500-40 from 1405239 to 2017203 tid 2 2024-08-17T17:10:54.770 INFO:tasks.rados.rados.0.smithi063.stdout:3799: snap_create 2024-08-17T17:10:54.774 INFO:tasks.rados.rados.0.smithi063.stdout:3798: finishing write tid 1 to smithi06387500-40 2024-08-17T17:10:54.774 INFO:tasks.rados.rados.0.smithi063.stdout:3798: finishing write tid 2 to smithi06387500-40 2024-08-17T17:10:54.774 INFO:tasks.rados.rados.0.smithi063.stdout:3798: finishing write tid 3 to smithi06387500-40 2024-08-17T17:10:54.774 INFO:tasks.rados.rados.0.smithi063.stdout:3798: finishing write tid 4 to smithi06387500-40 2024-08-17T17:10:54.774 INFO:tasks.rados.rados.0.smithi063.stdout:3798: oid 40 updating version 0 to 1342 2024-08-17T17:10:54.774 INFO:tasks.rados.rados.0.smithi063.stdout:3798: oid 40 updating version 1342 to 1343 2024-08-17T17:10:54.774 INFO:tasks.rados.rados.0.smithi063.stdout:3798: oid 40 updating version 1343 to 1344 2024-08-17T17:10:54.774 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 1344 (ObjNum 1429 snap 478 seq_num 1429) dirty exists 2024-08-17T17:10:54.774 INFO:tasks.rados.rados.0.smithi063.stdout:3798: left oid 40 (ObjNum 1429 snap 478 seq_num 1429) 2024-08-17T17:10:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:54 smithi063 ceph-mon[114756]: osdmap e1026: 8 total, 8 up, 8 in 2024-08-17T17:10:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:54 smithi063 ceph-mon[114756]: pgmap v996: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 236 MiB data, 880 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-08-17T17:10:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:54 smithi063 ceph-mon[109843]: osdmap e1026: 8 total, 8 up, 8 in 2024-08-17T17:10:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:54 smithi063 ceph-mon[109843]: pgmap v996: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 236 MiB data, 880 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-08-17T17:10:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:54 smithi162 ceph-mon[104668]: osdmap e1026: 8 total, 8 up, 8 in 2024-08-17T17:10:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:54 smithi162 ceph-mon[104668]: pgmap v996: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 236 MiB data, 880 MiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-08-17T17:10:55.763 INFO:tasks.rados.rados.0.smithi063.stdout:3798: done (1 left) 2024-08-17T17:10:55.763 INFO:tasks.rados.rados.0.smithi063.stdout:3799: done (0 left) 2024-08-17T17:10:55.763 INFO:tasks.rados.rados.0.smithi063.stdout:3800: delete oid 17 current snap is 479 2024-08-17T17:10:55.767 INFO:tasks.rados.rados.0.smithi063.stdout:3800: done (0 left) 2024-08-17T17:10:55.767 INFO:tasks.rados.rados.0.smithi063.stdout:3801: delete oid 17 current snap is 479 2024-08-17T17:10:55.769 INFO:tasks.rados.rados.0.smithi063.stdout:3801: done (0 left) 2024-08-17T17:10:55.769 INFO:tasks.rados.rados.0.smithi063.stdout:3802: read oid 32 snap -1 2024-08-17T17:10:55.769 INFO:tasks.rados.rados.0.smithi063.stdout:3802: expect deleted 2024-08-17T17:10:55.769 INFO:tasks.rados.rados.0.smithi063.stdout:3803: rollback oid 27 current snap is 479 2024-08-17T17:10:55.769 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 27 to 477 2024-08-17T17:10:55.769 INFO:tasks.rados.rados.0.smithi063.stdout:3804: write oid 28 current snap is 479 2024-08-17T17:10:55.769 INFO:tasks.rados.rados.0.smithi063.stdout:3804: seq_num 1432 ranges {467624=430932,1697038=370586} 2024-08-17T17:10:55.774 INFO:tasks.rados.rados.0.smithi063.stdout:3804: writing smithi06387500-28 from 467624 to 898556 tid 1 2024-08-17T17:10:55.782 INFO:tasks.rados.rados.0.smithi063.stdout:3804: writing smithi06387500-28 from 1697038 to 2067624 tid 2 2024-08-17T17:10:55.782 INFO:tasks.rados.rados.0.smithi063.stdout:3805: read oid 38 snap -1 2024-08-17T17:10:55.783 INFO:tasks.rados.rados.0.smithi063.stdout:3805: expect (ObjNum 1277 snap 434 seq_num 1277) 2024-08-17T17:10:55.783 INFO:tasks.rados.rados.0.smithi063.stdout:3806: snap_remove snap 469 2024-08-17T17:10:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:55 smithi063 ceph-mon[114756]: osdmap e1027: 8 total, 8 up, 8 in 2024-08-17T17:10:56.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:55 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:10:56.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:55 smithi063 ceph-mon[109843]: osdmap e1027: 8 total, 8 up, 8 in 2024-08-17T17:10:56.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:55 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:10:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:55 smithi162 ceph-mon[104668]: osdmap e1027: 8 total, 8 up, 8 in 2024-08-17T17:10:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:55 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:10:56.772 INFO:tasks.rados.rados.0.smithi063.stdout:3806: done (4 left) 2024-08-17T17:10:56.772 INFO:tasks.rados.rados.0.smithi063.stdout:3807: read oid 34 snap -1 2024-08-17T17:10:56.773 INFO:tasks.rados.rados.0.smithi063.stdout:3807: expect deleted 2024-08-17T17:10:56.773 INFO:tasks.rados.rados.0.smithi063.stdout:3808: read oid 44 snap 459 2024-08-17T17:10:56.773 INFO:tasks.rados.rados.0.smithi063.stdout:3808: expect (ObjNum 1338 snap 451 seq_num 1338) 2024-08-17T17:10:56.773 INFO:tasks.rados.rados.0.smithi063.stdout:3809: write oid 17 current snap is 479 2024-08-17T17:10:56.773 INFO:tasks.rados.rados.0.smithi063.stdout:3809: seq_num 1433 ranges {484431=506181,1639167=445264} 2024-08-17T17:10:56.778 INFO:tasks.rados.rados.0.smithi063.stdout:3809: writing smithi06387500-17 from 484431 to 990612 tid 1 2024-08-17T17:10:56.786 INFO:tasks.rados.rados.0.smithi063.stdout:3809: writing smithi06387500-17 from 1639167 to 2084431 tid 2 2024-08-17T17:10:56.786 INFO:tasks.rados.rados.0.smithi063.stdout:3810: write oid 41 current snap is 479 2024-08-17T17:10:56.786 INFO:tasks.rados.rados.0.smithi063.stdout:3810: seq_num 1434 ranges {501238=581430,1697649=403589} 2024-08-17T17:10:56.791 INFO:tasks.rados.rados.0.smithi063.stdout:3810: writing smithi06387500-41 from 501238 to 1082668 tid 1 2024-08-17T17:10:56.797 INFO:tasks.rados.rados.0.smithi063.stdout:3810: writing smithi06387500-41 from 1697649 to 2101238 tid 2 2024-08-17T17:10:56.797 INFO:tasks.rados.rados.0.smithi063.stdout:3811: delete oid 14 current snap is 479 2024-08-17T17:10:56.797 INFO:tasks.rados.rados.0.smithi063.stdout:3803: finishing rollback tid 0 to smithi06387500-27 2024-08-17T17:10:56.798 INFO:tasks.rados.rados.0.smithi063.stdout:3803: finishing rollback tid 1 to smithi06387500-27 2024-08-17T17:10:56.798 INFO:tasks.rados.rados.0.smithi063.stdout:3803: finishing rollback tid 2 to smithi06387500-27 2024-08-17T17:10:56.798 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 27 v 363 (ObjNum 1403 snap 470 seq_num 1403) dirty exists 2024-08-17T17:10:56.798 INFO:tasks.rados.rados.0.smithi063.stdout:3804: finishing write tid 1 to smithi06387500-28 2024-08-17T17:10:56.798 INFO:tasks.rados.rados.0.smithi063.stdout:3805: expect (ObjNum 1277 snap 434 seq_num 1277) 2024-08-17T17:10:56.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:56 smithi162 ceph-mon[104668]: osdmap e1028: 8 total, 8 up, 8 in 2024-08-17T17:10:56.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:56 smithi162 ceph-mon[104668]: pgmap v999: 89 pgs: 89 active+clean; 223 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-08-17T17:10:56.982 INFO:tasks.rados.rados.0.smithi063.stdout:3804: finishing write tid 2 to smithi06387500-28 2024-08-17T17:10:56.983 INFO:tasks.rados.rados.0.smithi063.stdout:3804: finishing write tid 3 to smithi06387500-28 2024-08-17T17:10:56.983 INFO:tasks.rados.rados.0.smithi063.stdout:3804: finishing write tid 4 to smithi06387500-28 2024-08-17T17:10:56.983 INFO:tasks.rados.rados.0.smithi063.stdout:3804: oid 28 updating version 0 to 984 2024-08-17T17:10:56.983 INFO:tasks.rados.rados.0.smithi063.stdout:3804: oid 28 updating version 984 to 986 2024-08-17T17:10:56.983 INFO:tasks.rados.rados.0.smithi063.stdout:3804: oid 28 version 986 is already newer than 985 2024-08-17T17:10:56.983 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 28 v 986 (ObjNum 1432 snap 479 seq_num 1432) dirty exists 2024-08-17T17:10:56.983 INFO:tasks.rados.rados.0.smithi063.stdout:3804: left oid 28 (ObjNum 1432 snap 479 seq_num 1432) 2024-08-17T17:10:56.983 INFO:tasks.rados.rados.0.smithi063.stdout:3809: finishing write tid 1 to smithi06387500-17 2024-08-17T17:10:56.983 INFO:tasks.rados.rados.0.smithi063.stdout:3808: expect (ObjNum 1338 snap 451 seq_num 1338) 2024-08-17T17:10:57.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:56 smithi063 ceph-mon[109843]: osdmap e1028: 8 total, 8 up, 8 in 2024-08-17T17:10:57.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:56 smithi063 ceph-mon[109843]: pgmap v999: 89 pgs: 89 active+clean; 223 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-08-17T17:10:57.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:56 smithi063 ceph-mon[114756]: osdmap e1028: 8 total, 8 up, 8 in 2024-08-17T17:10:57.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:56 smithi063 ceph-mon[114756]: pgmap v999: 89 pgs: 89 active+clean; 223 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 2.3 MiB/s wr, 13 op/s 2024-08-17T17:10:57.126 INFO:tasks.rados.rados.0.smithi063.stdout:3809: finishing write tid 2 to smithi06387500-17 2024-08-17T17:10:57.126 INFO:tasks.rados.rados.0.smithi063.stdout:3809: finishing write tid 3 to smithi06387500-17 2024-08-17T17:10:57.126 INFO:tasks.rados.rados.0.smithi063.stdout:3809: finishing write tid 4 to smithi06387500-17 2024-08-17T17:10:57.126 INFO:tasks.rados.rados.0.smithi063.stdout:3809: oid 17 updating version 0 to 1348 2024-08-17T17:10:57.126 INFO:tasks.rados.rados.0.smithi063.stdout:3809: oid 17 updating version 1348 to 1350 2024-08-17T17:10:57.127 INFO:tasks.rados.rados.0.smithi063.stdout:3809: oid 17 version 1350 is already newer than 1349 2024-08-17T17:10:57.127 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 1350 (ObjNum 1433 snap 479 seq_num 1433) dirty exists 2024-08-17T17:10:57.127 INFO:tasks.rados.rados.0.smithi063.stdout:3809: left oid 17 (ObjNum 1433 snap 479 seq_num 1433) 2024-08-17T17:10:57.127 INFO:tasks.rados.rados.0.smithi063.stdout:3810: finishing write tid 1 to smithi06387500-41 2024-08-17T17:10:57.127 INFO:tasks.rados.rados.0.smithi063.stdout:3802: done (8 left) 2024-08-17T17:10:57.127 INFO:tasks.rados.rados.0.smithi063.stdout:3803: done (7 left) 2024-08-17T17:10:57.127 INFO:tasks.rados.rados.0.smithi063.stdout:3804: done (6 left) 2024-08-17T17:10:57.127 INFO:tasks.rados.rados.0.smithi063.stdout:3805: done (5 left) 2024-08-17T17:10:57.127 INFO:tasks.rados.rados.0.smithi063.stdout:3807: done (4 left) 2024-08-17T17:10:57.127 INFO:tasks.rados.rados.0.smithi063.stdout:3808: done (3 left) 2024-08-17T17:10:57.127 INFO:tasks.rados.rados.0.smithi063.stdout:3809: done (2 left) 2024-08-17T17:10:57.128 INFO:tasks.rados.rados.0.smithi063.stdout:3811: done (1 left) 2024-08-17T17:10:57.128 INFO:tasks.rados.rados.0.smithi063.stdout:3812: delete oid 17 current snap is 479 2024-08-17T17:10:57.128 INFO:tasks.rados.rados.0.smithi063.stdout:3810: finishing write tid 2 to smithi06387500-41 2024-08-17T17:10:57.128 INFO:tasks.rados.rados.0.smithi063.stdout:3810: finishing write tid 3 to smithi06387500-41 2024-08-17T17:10:57.128 INFO:tasks.rados.rados.0.smithi063.stdout:3810: finishing write tid 4 to smithi06387500-41 2024-08-17T17:10:57.128 INFO:tasks.rados.rados.0.smithi063.stdout:3810: oid 41 updating version 0 to 446 2024-08-17T17:10:57.128 INFO:tasks.rados.rados.0.smithi063.stdout:3810: oid 41 updating version 446 to 448 2024-08-17T17:10:57.128 INFO:tasks.rados.rados.0.smithi063.stdout:3810: oid 41 version 448 is already newer than 447 2024-08-17T17:10:57.128 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 448 (ObjNum 1434 snap 479 seq_num 1434) dirty exists 2024-08-17T17:10:57.128 INFO:tasks.rados.rados.0.smithi063.stdout:3810: left oid 41 (ObjNum 1434 snap 479 seq_num 1434) 2024-08-17T17:10:57.129 INFO:tasks.rados.rados.0.smithi063.stdout:3810: done (1 left) 2024-08-17T17:10:57.129 INFO:tasks.rados.rados.0.smithi063.stdout:3812: done (0 left) 2024-08-17T17:10:57.129 INFO:tasks.rados.rados.0.smithi063.stdout:3813: snap_create 2024-08-17T17:10:57.685 INFO:tasks.rados.rados.0.smithi063.stdout:3813: done (0 left) 2024-08-17T17:10:57.685 INFO:tasks.rados.rados.0.smithi063.stdout:3814: snap_remove snap 479 2024-08-17T17:10:58.014 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:57 smithi063 ceph-mon[114756]: osdmap e1029: 8 total, 8 up, 8 in 2024-08-17T17:10:58.014 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:57 smithi063 ceph-mon[114756]: osdmap e1030: 8 total, 8 up, 8 in 2024-08-17T17:10:58.014 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:57 smithi063 ceph-mon[109843]: osdmap e1029: 8 total, 8 up, 8 in 2024-08-17T17:10:58.014 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:57 smithi063 ceph-mon[109843]: osdmap e1030: 8 total, 8 up, 8 in 2024-08-17T17:10:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:57 smithi162 ceph-mon[104668]: osdmap e1029: 8 total, 8 up, 8 in 2024-08-17T17:10:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:57 smithi162 ceph-mon[104668]: osdmap e1030: 8 total, 8 up, 8 in 2024-08-17T17:10:58.696 INFO:tasks.rados.rados.0.smithi063.stdout:3814: done (0 left) 2024-08-17T17:10:58.696 INFO:tasks.rados.rados.0.smithi063.stdout:3815: rollback oid 34 current snap is 480 2024-08-17T17:10:58.696 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 34 to 478 2024-08-17T17:10:58.696 INFO:tasks.rados.rados.0.smithi063.stdout:3816: delete oid 9 current snap is 480 2024-08-17T17:10:58.698 INFO:tasks.rados.rados.0.smithi063.stdout:3815: finishing rollback tid 1 to smithi06387500-34 2024-08-17T17:10:58.698 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-08-17T17:10:58.700 INFO:tasks.rados.rados.0.smithi063.stdout:3815: done (1 left) 2024-08-17T17:10:58.700 INFO:tasks.rados.rados.0.smithi063.stdout:3816: done (0 left) 2024-08-17T17:10:58.700 INFO:tasks.rados.rados.0.smithi063.stdout:3817: write oid 15 current snap is 480 2024-08-17T17:10:58.700 INFO:tasks.rados.rados.0.smithi063.stdout:3817: seq_num 1438 ranges {568466=598779,1931577=236889} 2024-08-17T17:10:58.706 INFO:tasks.rados.rados.0.smithi063.stdout:3817: writing smithi06387500-15 from 568466 to 1167245 tid 1 2024-08-17T17:10:58.712 INFO:tasks.rados.rados.0.smithi063.stdout:3817: writing smithi06387500-15 from 1931577 to 2168466 tid 2 2024-08-17T17:10:58.713 INFO:tasks.rados.rados.0.smithi063.stdout:3818: delete oid 24 current snap is 480 2024-08-17T17:10:58.713 INFO:tasks.rados.rados.0.smithi063.stdout:3817: finishing write tid 1 to smithi06387500-15 2024-08-17T17:10:58.714 INFO:tasks.rados.rados.0.smithi063.stdout:3817: finishing write tid 2 to smithi06387500-15 2024-08-17T17:10:58.716 INFO:tasks.rados.rados.0.smithi063.stdout:3817: finishing write tid 3 to smithi06387500-15 2024-08-17T17:10:58.716 INFO:tasks.rados.rados.0.smithi063.stdout:3817: finishing write tid 4 to smithi06387500-15 2024-08-17T17:10:58.716 INFO:tasks.rados.rados.0.smithi063.stdout:3817: oid 15 updating version 0 to 1458 2024-08-17T17:10:58.716 INFO:tasks.rados.rados.0.smithi063.stdout:3817: oid 15 updating version 1458 to 1460 2024-08-17T17:10:58.716 INFO:tasks.rados.rados.0.smithi063.stdout:3817: oid 15 version 1460 is already newer than 1459 2024-08-17T17:10:58.717 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 1460 (ObjNum 1438 snap 480 seq_num 1438) dirty exists 2024-08-17T17:10:58.717 INFO:tasks.rados.rados.0.smithi063.stdout:3817: left oid 15 (ObjNum 1438 snap 480 seq_num 1438) 2024-08-17T17:10:58.717 INFO:tasks.rados.rados.0.smithi063.stdout:3817: done (1 left) 2024-08-17T17:10:58.717 INFO:tasks.rados.rados.0.smithi063.stdout:3818: done (0 left) 2024-08-17T17:10:58.717 INFO:tasks.rados.rados.0.smithi063.stdout:3819: delete oid 38 current snap is 480 2024-08-17T17:10:58.719 INFO:tasks.rados.rados.0.smithi063.stdout:3819: done (0 left) 2024-08-17T17:10:58.719 INFO:tasks.rados.rados.0.smithi063.stdout:3820: read oid 3 snap -1 2024-08-17T17:10:58.719 INFO:tasks.rados.rados.0.smithi063.stdout:3820: expect deleted 2024-08-17T17:10:58.719 INFO:tasks.rados.rados.0.smithi063.stdout:3821: read oid 8 snap 475 2024-08-17T17:10:58.719 INFO:tasks.rados.rados.0.smithi063.stdout:3821: expect (ObjNum 1391 snap 466 seq_num 1391) 2024-08-17T17:10:58.719 INFO:tasks.rados.rados.0.smithi063.stdout:3822: read oid 33 snap -1 2024-08-17T17:10:58.719 INFO:tasks.rados.rados.0.smithi063.stdout:3822: expect (ObjNum 1311 snap 443 seq_num 1311) 2024-08-17T17:10:58.719 INFO:tasks.rados.rados.0.smithi063.stdout:3823: write oid 19 current snap is 480 2024-08-17T17:10:58.719 INFO:tasks.rados.rados.0.smithi063.stdout:3823: seq_num 1441 ranges {618887=424526,1590670=604158} 2024-08-17T17:10:58.725 INFO:tasks.rados.rados.0.smithi063.stdout:3823: writing smithi06387500-19 from 618887 to 1043413 tid 1 2024-08-17T17:10:58.731 INFO:tasks.rados.rados.0.smithi063.stdout:3823: writing smithi06387500-19 from 1590670 to 2194828 tid 2 2024-08-17T17:10:58.731 INFO:tasks.rados.rados.0.smithi063.stdout:3824: snap_create 2024-08-17T17:10:58.731 INFO:tasks.rados.rados.0.smithi063.stdout:3821: expect (ObjNum 1391 snap 466 seq_num 1391) 2024-08-17T17:10:58.842 INFO:tasks.rados.rados.0.smithi063.stdout:3822: expect (ObjNum 1311 snap 443 seq_num 1311) 2024-08-17T17:10:58.909 INFO:tasks.rados.rados.0.smithi063.stdout:3823: finishing write tid 1 to smithi06387500-19 2024-08-17T17:10:58.910 INFO:tasks.rados.rados.0.smithi063.stdout:3823: finishing write tid 2 to smithi06387500-19 2024-08-17T17:10:58.910 INFO:tasks.rados.rados.0.smithi063.stdout:3823: finishing write tid 3 to smithi06387500-19 2024-08-17T17:10:58.910 INFO:tasks.rados.rados.0.smithi063.stdout:3823: finishing write tid 4 to smithi06387500-19 2024-08-17T17:10:58.910 INFO:tasks.rados.rados.0.smithi063.stdout:3823: oid 19 updating version 0 to 2402 2024-08-17T17:10:58.910 INFO:tasks.rados.rados.0.smithi063.stdout:3823: oid 19 updating version 2402 to 2403 2024-08-17T17:10:58.910 INFO:tasks.rados.rados.0.smithi063.stdout:3823: oid 19 version 2403 is already newer than 2401 2024-08-17T17:10:58.910 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 19 v 2403 (ObjNum 1441 snap 480 seq_num 1441) dirty exists 2024-08-17T17:10:58.910 INFO:tasks.rados.rados.0.smithi063.stdout:3823: left oid 19 (ObjNum 1441 snap 480 seq_num 1441) 2024-08-17T17:10:59.698 INFO:tasks.rados.rados.0.smithi063.stdout:3820: done (4 left) 2024-08-17T17:10:59.698 INFO:tasks.rados.rados.0.smithi063.stdout:3821: done (3 left) 2024-08-17T17:10:59.698 INFO:tasks.rados.rados.0.smithi063.stdout:3822: done (2 left) 2024-08-17T17:10:59.698 INFO:tasks.rados.rados.0.smithi063.stdout:3823: done (1 left) 2024-08-17T17:10:59.699 INFO:tasks.rados.rados.0.smithi063.stdout:3824: done (0 left) 2024-08-17T17:10:59.699 INFO:tasks.rados.rados.0.smithi063.stdout:3825: write oid 25 current snap is 481 2024-08-17T17:10:59.699 INFO:tasks.rados.rados.0.smithi063.stdout:3825: seq_num 1442 ranges {635694=499775,1649152=464169} 2024-08-17T17:10:59.704 INFO:tasks.rados.rados.0.smithi063.stdout:3825: writing smithi06387500-25 from 635694 to 1135469 tid 1 2024-08-17T17:10:59.710 INFO:tasks.rados.rados.0.smithi063.stdout:3825: writing smithi06387500-25 from 1649152 to 2113321 tid 2 2024-08-17T17:10:59.710 INFO:tasks.rados.rados.0.smithi063.stdout:3826: read oid 43 snap -1 2024-08-17T17:10:59.710 INFO:tasks.rados.rados.0.smithi063.stdout:3826: expect (ObjNum 1387 snap 465 seq_num 1387) 2024-08-17T17:10:59.711 INFO:tasks.rados.rados.0.smithi063.stdout:3827: snap_remove snap 478 2024-08-17T17:11:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:59 smithi063 ceph-mon[114756]: pgmap v1002: 89 pgs: 89 active+clean; 194 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 3.6 MiB/s wr, 24 op/s 2024-08-17T17:11:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:59 smithi063 ceph-mon[114756]: osdmap e1031: 8 total, 8 up, 8 in 2024-08-17T17:11:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:59 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:11:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:59 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:11:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:59 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:11:00.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:10:59 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:11:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:59 smithi063 ceph-mon[109843]: pgmap v1002: 89 pgs: 89 active+clean; 194 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 3.6 MiB/s wr, 24 op/s 2024-08-17T17:11:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:59 smithi063 ceph-mon[109843]: osdmap e1031: 8 total, 8 up, 8 in 2024-08-17T17:11:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:59 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:11:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:59 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:11:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:59 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:11:00.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:10:59 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:11:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:59 smithi162 ceph-mon[104668]: pgmap v1002: 89 pgs: 89 active+clean; 194 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 3.6 MiB/s wr, 24 op/s 2024-08-17T17:11:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:59 smithi162 ceph-mon[104668]: osdmap e1031: 8 total, 8 up, 8 in 2024-08-17T17:11:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:59 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:11:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:59 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:11:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:59 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:11:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:10:59 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:11:00.715 INFO:tasks.rados.rados.0.smithi063.stdout:3827: done (2 left) 2024-08-17T17:11:00.715 INFO:tasks.rados.rados.0.smithi063.stdout:3828: read oid 15 snap -1 2024-08-17T17:11:00.715 INFO:tasks.rados.rados.0.smithi063.stdout:3828: expect (ObjNum 1438 snap 480 seq_num 1438) 2024-08-17T17:11:00.715 INFO:tasks.rados.rados.0.smithi063.stdout:3829: read oid 11 snap -1 2024-08-17T17:11:00.715 INFO:tasks.rados.rados.0.smithi063.stdout:3829: expect (ObjNum 1423 snap 476 seq_num 1423) 2024-08-17T17:11:00.715 INFO:tasks.rados.rados.0.smithi063.stdout:3830: delete oid 1 current snap is 481 2024-08-17T17:11:00.715 INFO:tasks.rados.rados.0.smithi063.stdout:3825: finishing write tid 1 to smithi06387500-25 2024-08-17T17:11:00.715 INFO:tasks.rados.rados.0.smithi063.stdout:3825: finishing write tid 2 to smithi06387500-25 2024-08-17T17:11:00.715 INFO:tasks.rados.rados.0.smithi063.stdout:3825: finishing write tid 3 to smithi06387500-25 2024-08-17T17:11:00.715 INFO:tasks.rados.rados.0.smithi063.stdout:3825: finishing write tid 4 to smithi06387500-25 2024-08-17T17:11:00.715 INFO:tasks.rados.rados.0.smithi063.stdout:3825: oid 25 updating version 0 to 3769 2024-08-17T17:11:00.716 INFO:tasks.rados.rados.0.smithi063.stdout:3825: oid 25 updating version 3769 to 3770 2024-08-17T17:11:00.716 INFO:tasks.rados.rados.0.smithi063.stdout:3825: oid 25 updating version 3770 to 3771 2024-08-17T17:11:00.716 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 3771 (ObjNum 1442 snap 481 seq_num 1442) dirty exists 2024-08-17T17:11:00.716 INFO:tasks.rados.rados.0.smithi063.stdout:3825: left oid 25 (ObjNum 1442 snap 481 seq_num 1442) 2024-08-17T17:11:00.716 INFO:tasks.rados.rados.0.smithi063.stdout:3826: expect (ObjNum 1387 snap 465 seq_num 1387) 2024-08-17T17:11:00.830 INFO:tasks.rados.rados.0.smithi063.stdout:3828: expect (ObjNum 1438 snap 480 seq_num 1438) 2024-08-17T17:11:00.989 INFO:tasks.rados.rados.0.smithi063.stdout:3829: expect (ObjNum 1423 snap 476 seq_num 1423) 2024-08-17T17:11:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:00 smithi063 ceph-mon[109843]: osdmap e1032: 8 total, 8 up, 8 in 2024-08-17T17:11:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:00 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:00 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:11:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:00 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:11:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:00 smithi063 ceph-mon[109843]: pgmap v1005: 89 pgs: 89 active+clean; 194 MiB data, 810 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-08-17T17:11:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:00 smithi063 ceph-mon[114756]: osdmap e1032: 8 total, 8 up, 8 in 2024-08-17T17:11:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:00 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:00 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:11:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:00 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:11:01.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:00 smithi063 ceph-mon[114756]: pgmap v1005: 89 pgs: 89 active+clean; 194 MiB data, 810 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-08-17T17:11:01.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:00 smithi162 ceph-mon[104668]: osdmap e1032: 8 total, 8 up, 8 in 2024-08-17T17:11:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:00 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:11:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:00 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:11:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:00 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:11:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:00 smithi162 ceph-mon[104668]: pgmap v1005: 89 pgs: 89 active+clean; 194 MiB data, 810 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-08-17T17:11:01.234 INFO:tasks.rados.rados.0.smithi063.stdout:3825: done (4 left) 2024-08-17T17:11:01.234 INFO:tasks.rados.rados.0.smithi063.stdout:3826: done (3 left) 2024-08-17T17:11:01.234 INFO:tasks.rados.rados.0.smithi063.stdout:3828: done (2 left) 2024-08-17T17:11:01.234 INFO:tasks.rados.rados.0.smithi063.stdout:3829: done (1 left) 2024-08-17T17:11:01.234 INFO:tasks.rados.rados.0.smithi063.stdout:3830: done (0 left) 2024-08-17T17:11:01.235 INFO:tasks.rados.rados.0.smithi063.stdout:3831: write oid 38 current snap is 481 2024-08-17T17:11:01.235 INFO:tasks.rados.rados.0.smithi063.stdout:3831: seq_num 1444 ranges {669308=650273,1766116=467838} 2024-08-17T17:11:01.241 INFO:tasks.rados.rados.0.smithi063.stdout:3831: writing smithi06387500-38 from 669308 to 1319581 tid 1 2024-08-17T17:11:01.246 INFO:tasks.rados.rados.0.smithi063.stdout:3831: writing smithi06387500-38 from 1766116 to 2233954 tid 2 2024-08-17T17:11:01.246 INFO:tasks.rados.rados.0.smithi063.stdout:3832: snap_remove snap 463 2024-08-17T17:11:01.719 INFO:tasks.rados.rados.0.smithi063.stdout:3831: finishing write tid 1 to smithi06387500-38 2024-08-17T17:11:01.719 INFO:tasks.rados.rados.0.smithi063.stdout:3831: finishing write tid 2 to smithi06387500-38 2024-08-17T17:11:01.719 INFO:tasks.rados.rados.0.smithi063.stdout:3831: finishing write tid 3 to smithi06387500-38 2024-08-17T17:11:01.719 INFO:tasks.rados.rados.0.smithi063.stdout:3831: finishing write tid 4 to smithi06387500-38 2024-08-17T17:11:01.719 INFO:tasks.rados.rados.0.smithi063.stdout:3831: oid 38 updating version 0 to 752 2024-08-17T17:11:01.720 INFO:tasks.rados.rados.0.smithi063.stdout:3831: oid 38 updating version 752 to 753 2024-08-17T17:11:01.720 INFO:tasks.rados.rados.0.smithi063.stdout:3831: oid 38 version 753 is already newer than 751 2024-08-17T17:11:01.720 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 38 v 753 (ObjNum 1444 snap 481 seq_num 1444) dirty exists 2024-08-17T17:11:01.720 INFO:tasks.rados.rados.0.smithi063.stdout:3831: left oid 38 (ObjNum 1444 snap 481 seq_num 1444) 2024-08-17T17:11:01.720 INFO:tasks.rados.rados.0.smithi063.stdout:3831: done (1 left) 2024-08-17T17:11:01.721 INFO:tasks.rados.rados.0.smithi063.stdout:3832: done (0 left) 2024-08-17T17:11:01.721 INFO:tasks.rados.rados.0.smithi063.stdout:3833: read oid 31 snap 475 2024-08-17T17:11:01.721 INFO:tasks.rados.rados.0.smithi063.stdout:3833: expect deleted 2024-08-17T17:11:01.721 INFO:tasks.rados.rados.0.smithi063.stdout:3834: snap_create 2024-08-17T17:11:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:01 smithi063 ceph-mon[114756]: osdmap e1033: 8 total, 8 up, 8 in 2024-08-17T17:11:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:01 smithi063 ceph-mon[109843]: osdmap e1033: 8 total, 8 up, 8 in 2024-08-17T17:11:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:01 smithi162 ceph-mon[104668]: osdmap e1033: 8 total, 8 up, 8 in 2024-08-17T17:11:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:02.688 INFO:tasks.rados.rados.0.smithi063.stdout:3833: done (1 left) 2024-08-17T17:11:02.688 INFO:tasks.rados.rados.0.smithi063.stdout:3834: done (0 left) 2024-08-17T17:11:02.688 INFO:tasks.rados.rados.0.smithi063.stdout:3835: read oid 18 snap -1 2024-08-17T17:11:02.688 INFO:tasks.rados.rados.0.smithi063.stdout:3835: expect (ObjNum 1379 snap 463 seq_num 1379) 2024-08-17T17:11:02.688 INFO:tasks.rados.rados.0.smithi063.stdout:3836: snap_create 2024-08-17T17:11:02.696 INFO:tasks.rados.rados.0.smithi063.stdout:3835: expect (ObjNum 1379 snap 463 seq_num 1379) 2024-08-17T17:11:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:02 smithi063 ceph-mon[114756]: osdmap e1034: 8 total, 8 up, 8 in 2024-08-17T17:11:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:02 smithi063 ceph-mon[114756]: pgmap v1008: 89 pgs: 89 active+clean; 186 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 887 KiB/s wr, 11 op/s 2024-08-17T17:11:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:02 smithi063 ceph-mon[114756]: osdmap e1035: 8 total, 8 up, 8 in 2024-08-17T17:11:03.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:02 smithi063 ceph-mon[109843]: osdmap e1034: 8 total, 8 up, 8 in 2024-08-17T17:11:03.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:02 smithi063 ceph-mon[109843]: pgmap v1008: 89 pgs: 89 active+clean; 186 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 887 KiB/s wr, 11 op/s 2024-08-17T17:11:03.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:02 smithi063 ceph-mon[109843]: osdmap e1035: 8 total, 8 up, 8 in 2024-08-17T17:11:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:02 smithi162 ceph-mon[104668]: osdmap e1034: 8 total, 8 up, 8 in 2024-08-17T17:11:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:02 smithi162 ceph-mon[104668]: pgmap v1008: 89 pgs: 89 active+clean; 186 MiB data, 806 MiB used, 714 GiB / 715 GiB avail; 6.3 MiB/s rd, 887 KiB/s wr, 11 op/s 2024-08-17T17:11:03.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:02 smithi162 ceph-mon[104668]: osdmap e1035: 8 total, 8 up, 8 in 2024-08-17T17:11:03.688 INFO:tasks.rados.rados.0.smithi063.stdout:3835: done (1 left) 2024-08-17T17:11:03.688 INFO:tasks.rados.rados.0.smithi063.stdout:3836: done (0 left) 2024-08-17T17:11:03.689 INFO:tasks.rados.rados.0.smithi063.stdout:3837: write oid 20 current snap is 483 2024-08-17T17:11:03.689 INFO:tasks.rados.rados.0.smithi063.stdout:3837: seq_num 1445 ranges {686115=441875,1824598=461517} 2024-08-17T17:11:03.694 INFO:tasks.rados.rados.0.smithi063.stdout:3837: writing smithi06387500-20 from 686115 to 1127990 tid 1 2024-08-17T17:11:03.701 INFO:tasks.rados.rados.0.smithi063.stdout:3837: writing smithi06387500-20 from 1824598 to 2286115 tid 2 2024-08-17T17:11:03.701 INFO:tasks.rados.rados.0.smithi063.stdout:3838: snap_create 2024-08-17T17:11:03.702 INFO:tasks.rados.rados.0.smithi063.stdout:3837: finishing write tid 1 to smithi06387500-20 2024-08-17T17:11:03.705 INFO:tasks.rados.rados.0.smithi063.stdout:3837: finishing write tid 2 to smithi06387500-20 2024-08-17T17:11:03.705 INFO:tasks.rados.rados.0.smithi063.stdout:3837: finishing write tid 3 to smithi06387500-20 2024-08-17T17:11:03.705 INFO:tasks.rados.rados.0.smithi063.stdout:3837: finishing write tid 4 to smithi06387500-20 2024-08-17T17:11:03.705 INFO:tasks.rados.rados.0.smithi063.stdout:3837: oid 20 updating version 0 to 990 2024-08-17T17:11:03.706 INFO:tasks.rados.rados.0.smithi063.stdout:3837: oid 20 updating version 990 to 991 2024-08-17T17:11:03.706 INFO:tasks.rados.rados.0.smithi063.stdout:3837: oid 20 updating version 991 to 992 2024-08-17T17:11:03.706 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 992 (ObjNum 1445 snap 483 seq_num 1445) dirty exists 2024-08-17T17:11:03.706 INFO:tasks.rados.rados.0.smithi063.stdout:3837: left oid 20 (ObjNum 1445 snap 483 seq_num 1445) 2024-08-17T17:11:03.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:11:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:11:03] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.51.0" 2024-08-17T17:11:04.702 INFO:tasks.rados.rados.0.smithi063.stdout:3837: done (1 left) 2024-08-17T17:11:04.702 INFO:tasks.rados.rados.0.smithi063.stdout:3838: done (0 left) 2024-08-17T17:11:04.702 INFO:tasks.rados.rados.0.smithi063.stdout:3839: write oid 43 current snap is 484 2024-08-17T17:11:04.702 INFO:tasks.rados.rados.0.smithi063.stdout:3839: seq_num 1446 ranges {702922=517124,1883080=419842} 2024-08-17T17:11:04.708 INFO:tasks.rados.rados.0.smithi063.stdout:3839: writing smithi06387500-43 from 702922 to 1220046 tid 1 2024-08-17T17:11:04.715 INFO:tasks.rados.rados.0.smithi063.stdout:3839: writing smithi06387500-43 from 1883080 to 2302922 tid 2 2024-08-17T17:11:04.715 INFO:tasks.rados.rados.0.smithi063.stdout:3840: rollback oid 16 current snap is 484 2024-08-17T17:11:04.715 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 16 to 472 2024-08-17T17:11:04.715 INFO:tasks.rados.rados.0.smithi063.stdout:3841: rollback oid 31 current snap is 484 2024-08-17T17:11:04.716 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 31 to 473 2024-08-17T17:11:04.716 INFO:tasks.rados.rados.0.smithi063.stdout:3842: read oid 4 snap -1 2024-08-17T17:11:04.716 INFO:tasks.rados.rados.0.smithi063.stdout:3842: expect deleted 2024-08-17T17:11:04.716 INFO:tasks.rados.rados.0.smithi063.stdout:3843: write oid 29 current snap is 484 2024-08-17T17:11:04.716 INFO:tasks.rados.rados.0.smithi063.stdout:3843: seq_num 1447 ranges {719729=592373,1941562=378167} 2024-08-17T17:11:04.723 INFO:tasks.rados.rados.0.smithi063.stdout:3843: writing smithi06387500-29 from 719729 to 1312102 tid 1 2024-08-17T17:11:04.729 INFO:tasks.rados.rados.0.smithi063.stdout:3843: writing smithi06387500-29 from 1941562 to 2319729 tid 2 2024-08-17T17:11:04.729 INFO:tasks.rados.rados.0.smithi063.stdout:3844: write oid 44 current snap is 484 2024-08-17T17:11:04.729 INFO:tasks.rados.rados.0.smithi063.stdout:3844: seq_num 1448 ranges {736536=667622,2000044=336492} 2024-08-17T17:11:04.736 INFO:tasks.rados.rados.0.smithi063.stdout:3844: writing smithi06387500-44 from 736536 to 1404158 tid 1 2024-08-17T17:11:04.742 INFO:tasks.rados.rados.0.smithi063.stdout:3844: writing smithi06387500-44 from 2000044 to 2336536 tid 2 2024-08-17T17:11:04.742 INFO:tasks.rados.rados.0.smithi063.stdout:3845: read oid 33 snap -1 2024-08-17T17:11:04.742 INFO:tasks.rados.rados.0.smithi063.stdout:3845: expect (ObjNum 1311 snap 443 seq_num 1311) 2024-08-17T17:11:04.742 INFO:tasks.rados.rados.0.smithi063.stdout:3846: snap_create 2024-08-17T17:11:04.742 INFO:tasks.rados.rados.0.smithi063.stdout:3839: finishing write tid 1 to smithi06387500-43 2024-08-17T17:11:04.742 INFO:tasks.rados.rados.0.smithi063.stdout:3839: finishing write tid 2 to smithi06387500-43 2024-08-17T17:11:04.743 INFO:tasks.rados.rados.0.smithi063.stdout:3839: finishing write tid 3 to smithi06387500-43 2024-08-17T17:11:04.743 INFO:tasks.rados.rados.0.smithi063.stdout:3841: finishing rollback tid 1 to smithi06387500-31 2024-08-17T17:11:04.743 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 31 v 0 (ObjNum 145 snap 0 seq_num 1807394272) dirty dne 2024-08-17T17:11:04.743 INFO:tasks.rados.rados.0.smithi063.stdout:3839: finishing write tid 4 to smithi06387500-43 2024-08-17T17:11:04.743 INFO:tasks.rados.rados.0.smithi063.stdout:3839: oid 43 updating version 0 to 1897 2024-08-17T17:11:04.743 INFO:tasks.rados.rados.0.smithi063.stdout:3839: oid 43 version 1897 is already newer than 1895 2024-08-17T17:11:04.743 INFO:tasks.rados.rados.0.smithi063.stdout:3839: oid 43 version 1897 is already newer than 1896 2024-08-17T17:11:04.743 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 1897 (ObjNum 1446 snap 484 seq_num 1446) dirty exists 2024-08-17T17:11:04.743 INFO:tasks.rados.rados.0.smithi063.stdout:3839: left oid 43 (ObjNum 1446 snap 484 seq_num 1446) 2024-08-17T17:11:04.743 INFO:tasks.rados.rados.0.smithi063.stdout:3840: finishing rollback tid 0 to smithi06387500-16 2024-08-17T17:11:04.743 INFO:tasks.rados.rados.0.smithi063.stdout:3840: finishing rollback tid 1 to smithi06387500-16 2024-08-17T17:11:04.744 INFO:tasks.rados.rados.0.smithi063.stdout:3840: finishing rollback tid 2 to smithi06387500-16 2024-08-17T17:11:04.744 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 1476 (ObjNum 1369 snap 461 seq_num 1369) dirty exists 2024-08-17T17:11:04.744 INFO:tasks.rados.rados.0.smithi063.stdout:3843: finishing write tid 1 to smithi06387500-29 2024-08-17T17:11:04.744 INFO:tasks.rados.rados.0.smithi063.stdout:3843: finishing write tid 2 to smithi06387500-29 2024-08-17T17:11:04.744 INFO:tasks.rados.rados.0.smithi063.stdout:3843: finishing write tid 3 to smithi06387500-29 2024-08-17T17:11:04.744 INFO:tasks.rados.rados.0.smithi063.stdout:3843: finishing write tid 4 to smithi06387500-29 2024-08-17T17:11:04.744 INFO:tasks.rados.rados.0.smithi063.stdout:3843: oid 29 updating version 0 to 831 2024-08-17T17:11:04.744 INFO:tasks.rados.rados.0.smithi063.stdout:3843: oid 29 version 831 is already newer than 830 2024-08-17T17:11:04.744 INFO:tasks.rados.rados.0.smithi063.stdout:3843: oid 29 updating version 831 to 832 2024-08-17T17:11:04.744 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 832 (ObjNum 1447 snap 484 seq_num 1447) dirty exists 2024-08-17T17:11:04.745 INFO:tasks.rados.rados.0.smithi063.stdout:3843: left oid 29 (ObjNum 1447 snap 484 seq_num 1447) 2024-08-17T17:11:04.745 INFO:tasks.rados.rados.0.smithi063.stdout:3844: finishing write tid 1 to smithi06387500-44 2024-08-17T17:11:04.745 INFO:tasks.rados.rados.0.smithi063.stdout:3844: finishing write tid 2 to smithi06387500-44 2024-08-17T17:11:04.745 INFO:tasks.rados.rados.0.smithi063.stdout:3845: expect (ObjNum 1311 snap 443 seq_num 1311) 2024-08-17T17:11:04.811 INFO:tasks.rados.rados.0.smithi063.stdout:3844: finishing write tid 3 to smithi06387500-44 2024-08-17T17:11:04.812 INFO:tasks.rados.rados.0.smithi063.stdout:3844: finishing write tid 4 to smithi06387500-44 2024-08-17T17:11:04.812 INFO:tasks.rados.rados.0.smithi063.stdout:3844: oid 44 updating version 0 to 552 2024-08-17T17:11:04.812 INFO:tasks.rados.rados.0.smithi063.stdout:3844: oid 44 updating version 552 to 553 2024-08-17T17:11:04.812 INFO:tasks.rados.rados.0.smithi063.stdout:3844: oid 44 updating version 553 to 554 2024-08-17T17:11:04.812 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 554 (ObjNum 1448 snap 484 seq_num 1448) dirty exists 2024-08-17T17:11:04.812 INFO:tasks.rados.rados.0.smithi063.stdout:3844: left oid 44 (ObjNum 1448 snap 484 seq_num 1448) 2024-08-17T17:11:05.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:04 smithi063 ceph-mon[114756]: osdmap e1036: 8 total, 8 up, 8 in 2024-08-17T17:11:05.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:04 smithi063 ceph-mon[109843]: osdmap e1036: 8 total, 8 up, 8 in 2024-08-17T17:11:05.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:04 smithi162 ceph-mon[104668]: osdmap e1036: 8 total, 8 up, 8 in 2024-08-17T17:11:05.702 INFO:tasks.rados.rados.0.smithi063.stdout:3839: done (7 left) 2024-08-17T17:11:05.702 INFO:tasks.rados.rados.0.smithi063.stdout:3840: done (6 left) 2024-08-17T17:11:05.702 INFO:tasks.rados.rados.0.smithi063.stdout:3841: done (5 left) 2024-08-17T17:11:05.702 INFO:tasks.rados.rados.0.smithi063.stdout:3842: done (4 left) 2024-08-17T17:11:05.702 INFO:tasks.rados.rados.0.smithi063.stdout:3843: done (3 left) 2024-08-17T17:11:05.702 INFO:tasks.rados.rados.0.smithi063.stdout:3844: done (2 left) 2024-08-17T17:11:05.702 INFO:tasks.rados.rados.0.smithi063.stdout:3845: done (1 left) 2024-08-17T17:11:05.703 INFO:tasks.rados.rados.0.smithi063.stdout:3846: done (0 left) 2024-08-17T17:11:05.703 INFO:tasks.rados.rados.0.smithi063.stdout:3847: write oid 44 current snap is 485 2024-08-17T17:11:05.703 INFO:tasks.rados.rados.0.smithi063.stdout:3847: seq_num 1449 ranges {753343=742871,1942173=411170} 2024-08-17T17:11:05.710 INFO:tasks.rados.rados.0.smithi063.stdout:3847: writing smithi06387500-44 from 753343 to 1496214 tid 1 2024-08-17T17:11:05.715 INFO:tasks.rados.rados.0.smithi063.stdout:3847: writing smithi06387500-44 from 1942173 to 2353343 tid 2 2024-08-17T17:11:05.715 INFO:tasks.rados.rados.0.smithi063.stdout:3848: read oid 10 snap 481 2024-08-17T17:11:05.715 INFO:tasks.rados.rados.0.smithi063.stdout:3848: expect (ObjNum 1359 snap 455 seq_num 1359) 2024-08-17T17:11:05.715 INFO:tasks.rados.rados.0.smithi063.stdout:3849: write oid 41 current snap is 485 2024-08-17T17:11:05.715 INFO:tasks.rados.rados.0.smithi063.stdout:3849: seq_num 1450 ranges {770150=418120,1600655=478845} 2024-08-17T17:11:05.722 INFO:tasks.rados.rados.0.smithi063.stdout:3849: writing smithi06387500-41 from 770150 to 1188270 tid 1 2024-08-17T17:11:05.727 INFO:tasks.rados.rados.0.smithi063.stdout:3849: writing smithi06387500-41 from 1600655 to 2079500 tid 2 2024-08-17T17:11:05.727 INFO:tasks.rados.rados.0.smithi063.stdout:3850: delete oid 37 current snap is 485 2024-08-17T17:11:05.727 INFO:tasks.rados.rados.0.smithi063.stdout:3847: finishing write tid 1 to smithi06387500-44 2024-08-17T17:11:05.727 INFO:tasks.rados.rados.0.smithi063.stdout:3847: finishing write tid 2 to smithi06387500-44 2024-08-17T17:11:05.727 INFO:tasks.rados.rados.0.smithi063.stdout:3847: finishing write tid 3 to smithi06387500-44 2024-08-17T17:11:05.727 INFO:tasks.rados.rados.0.smithi063.stdout:3847: finishing write tid 4 to smithi06387500-44 2024-08-17T17:11:05.728 INFO:tasks.rados.rados.0.smithi063.stdout:3847: oid 44 updating version 0 to 558 2024-08-17T17:11:05.728 INFO:tasks.rados.rados.0.smithi063.stdout:3847: oid 44 version 558 is already newer than 557 2024-08-17T17:11:05.728 INFO:tasks.rados.rados.0.smithi063.stdout:3847: oid 44 version 558 is already newer than 556 2024-08-17T17:11:05.728 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 558 (ObjNum 1449 snap 485 seq_num 1449) dirty exists 2024-08-17T17:11:05.728 INFO:tasks.rados.rados.0.smithi063.stdout:3847: left oid 44 (ObjNum 1449 snap 485 seq_num 1449) 2024-08-17T17:11:05.728 INFO:tasks.rados.rados.0.smithi063.stdout:3848: expect (ObjNum 1359 snap 455 seq_num 1359) 2024-08-17T17:11:05.822 INFO:tasks.rados.rados.0.smithi063.stdout:3849: finishing write tid 1 to smithi06387500-41 2024-08-17T17:11:05.822 INFO:tasks.rados.rados.0.smithi063.stdout:3849: finishing write tid 2 to smithi06387500-41 2024-08-17T17:11:05.823 INFO:tasks.rados.rados.0.smithi063.stdout:3849: finishing write tid 3 to smithi06387500-41 2024-08-17T17:11:05.823 INFO:tasks.rados.rados.0.smithi063.stdout:3849: finishing write tid 4 to smithi06387500-41 2024-08-17T17:11:05.823 INFO:tasks.rados.rados.0.smithi063.stdout:3849: oid 41 updating version 0 to 454 2024-08-17T17:11:05.823 INFO:tasks.rados.rados.0.smithi063.stdout:3849: oid 41 version 454 is already newer than 452 2024-08-17T17:11:05.823 INFO:tasks.rados.rados.0.smithi063.stdout:3849: oid 41 version 454 is already newer than 453 2024-08-17T17:11:05.823 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 454 (ObjNum 1450 snap 485 seq_num 1450) dirty exists 2024-08-17T17:11:05.823 INFO:tasks.rados.rados.0.smithi063.stdout:3849: left oid 41 (ObjNum 1450 snap 485 seq_num 1450) 2024-08-17T17:11:05.823 INFO:tasks.rados.rados.0.smithi063.stdout:3847: done (3 left) 2024-08-17T17:11:05.823 INFO:tasks.rados.rados.0.smithi063.stdout:3848: done (2 left) 2024-08-17T17:11:05.823 INFO:tasks.rados.rados.0.smithi063.stdout:3849: done (1 left) 2024-08-17T17:11:05.823 INFO:tasks.rados.rados.0.smithi063.stdout:3850: done (0 left) 2024-08-17T17:11:05.824 INFO:tasks.rados.rados.0.smithi063.stdout:3851: delete oid 18 current snap is 485 2024-08-17T17:11:05.824 INFO:tasks.rados.rados.0.smithi063.stdout:3851: done (0 left) 2024-08-17T17:11:05.824 INFO:tasks.rados.rados.0.smithi063.stdout:3852: delete oid 47 current snap is 485 2024-08-17T17:11:05.826 INFO:tasks.rados.rados.0.smithi063.stdout:3852: done (0 left) 2024-08-17T17:11:05.826 INFO:tasks.rados.rados.0.smithi063.stdout:3853: snap_remove snap 472 2024-08-17T17:11:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:05 smithi063 ceph-mon[114756]: pgmap v1011: 89 pgs: 89 active+clean; 186 MiB data, 802 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 887 KiB/s wr, 16 op/s 2024-08-17T17:11:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:05 smithi063 ceph-mon[114756]: osdmap e1037: 8 total, 8 up, 8 in 2024-08-17T17:11:06.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:05 smithi063 ceph-mon[109843]: pgmap v1011: 89 pgs: 89 active+clean; 186 MiB data, 802 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 887 KiB/s wr, 16 op/s 2024-08-17T17:11:06.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:05 smithi063 ceph-mon[109843]: osdmap e1037: 8 total, 8 up, 8 in 2024-08-17T17:11:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:05 smithi162 ceph-mon[104668]: pgmap v1011: 89 pgs: 89 active+clean; 186 MiB data, 802 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 887 KiB/s wr, 16 op/s 2024-08-17T17:11:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:05 smithi162 ceph-mon[104668]: osdmap e1037: 8 total, 8 up, 8 in 2024-08-17T17:11:06.717 INFO:tasks.rados.rados.0.smithi063.stdout:3853: done (0 left) 2024-08-17T17:11:06.718 INFO:tasks.rados.rados.0.smithi063.stdout:3854: write oid 30 current snap is 485 2024-08-17T17:11:06.718 INFO:tasks.rados.rados.0.smithi063.stdout:3854: seq_num 1454 ranges {437378=435469,1434583=769830} 2024-08-17T17:11:06.723 INFO:tasks.rados.rados.0.smithi063.stdout:3854: writing smithi06387500-30 from 437378 to 872847 tid 1 2024-08-17T17:11:06.731 INFO:tasks.rados.rados.0.smithi063.stdout:3854: writing smithi06387500-30 from 1434583 to 2204413 tid 2 2024-08-17T17:11:06.731 INFO:tasks.rados.rados.0.smithi063.stdout:3855: snap_remove snap 477 2024-08-17T17:11:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:06 smithi063 ceph-mon[109843]: osdmap e1038: 8 total, 8 up, 8 in 2024-08-17T17:11:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:06 smithi063 ceph-mon[109843]: pgmap v1014: 89 pgs: 89 active+clean; 190 MiB data, 802 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 731 KiB/s wr, 17 op/s 2024-08-17T17:11:07.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:06 smithi063 ceph-mon[114756]: osdmap e1038: 8 total, 8 up, 8 in 2024-08-17T17:11:07.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:06 smithi063 ceph-mon[114756]: pgmap v1014: 89 pgs: 89 active+clean; 190 MiB data, 802 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 731 KiB/s wr, 17 op/s 2024-08-17T17:11:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:06 smithi162 ceph-mon[104668]: osdmap e1038: 8 total, 8 up, 8 in 2024-08-17T17:11:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:06 smithi162 ceph-mon[104668]: pgmap v1014: 89 pgs: 89 active+clean; 190 MiB data, 802 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 731 KiB/s wr, 17 op/s 2024-08-17T17:11:07.689 INFO:tasks.rados.rados.0.smithi063.stdout:3855: done (1 left) 2024-08-17T17:11:07.689 INFO:tasks.rados.rados.0.smithi063.stdout:3856: rollback oid 43 current snap is 485 2024-08-17T17:11:07.689 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 43 to 484 2024-08-17T17:11:07.689 INFO:tasks.rados.rados.0.smithi063.stdout:3854: finishing write tid 1 to smithi06387500-30 2024-08-17T17:11:07.689 INFO:tasks.rados.rados.0.smithi063.stdout:3854: finishing write tid 2 to smithi06387500-30 2024-08-17T17:11:07.689 INFO:tasks.rados.rados.0.smithi063.stdout:3854: finishing write tid 3 to smithi06387500-30 2024-08-17T17:11:07.689 INFO:tasks.rados.rados.0.smithi063.stdout:3854: finishing write tid 4 to smithi06387500-30 2024-08-17T17:11:07.690 INFO:tasks.rados.rados.0.smithi063.stdout:3854: oid 30 updating version 0 to 1905 2024-08-17T17:11:07.690 INFO:tasks.rados.rados.0.smithi063.stdout:3854: oid 30 updating version 1905 to 1906 2024-08-17T17:11:07.690 INFO:tasks.rados.rados.0.smithi063.stdout:3854: oid 30 updating version 1906 to 1907 2024-08-17T17:11:07.690 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1907 (ObjNum 1454 snap 485 seq_num 1454) dirty exists 2024-08-17T17:11:07.690 INFO:tasks.rados.rados.0.smithi063.stdout:3854: left oid 30 (ObjNum 1454 snap 485 seq_num 1454) 2024-08-17T17:11:07.690 INFO:tasks.rados.rados.0.smithi063.stdout:3854: done (1 left) 2024-08-17T17:11:07.690 INFO:tasks.rados.rados.0.smithi063.stdout:3857: snap_create 2024-08-17T17:11:07.697 INFO:tasks.rados.rados.0.smithi063.stdout:3856: finishing rollback tid 0 to smithi06387500-43 2024-08-17T17:11:07.700 INFO:tasks.rados.rados.0.smithi063.stdout:3856: finishing rollback tid 1 to smithi06387500-43 2024-08-17T17:11:07.700 INFO:tasks.rados.rados.0.smithi063.stdout:3856: finishing rollback tid 2 to smithi06387500-43 2024-08-17T17:11:07.700 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 1911 (ObjNum 1446 snap 484 seq_num 1446) dirty exists 2024-08-17T17:11:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:07 smithi063 ceph-mon[109843]: osdmap e1039: 8 total, 8 up, 8 in 2024-08-17T17:11:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:07 smithi063 ceph-mon[109843]: osdmap e1040: 8 total, 8 up, 8 in 2024-08-17T17:11:08.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:07 smithi063 ceph-mon[114756]: osdmap e1039: 8 total, 8 up, 8 in 2024-08-17T17:11:08.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:07 smithi063 ceph-mon[114756]: osdmap e1040: 8 total, 8 up, 8 in 2024-08-17T17:11:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:07 smithi162 ceph-mon[104668]: osdmap e1039: 8 total, 8 up, 8 in 2024-08-17T17:11:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:07 smithi162 ceph-mon[104668]: osdmap e1040: 8 total, 8 up, 8 in 2024-08-17T17:11:08.691 INFO:tasks.rados.rados.0.smithi063.stdout:3856: done (1 left) 2024-08-17T17:11:08.693 INFO:tasks.rados.rados.0.smithi063.stdout:3857: done (0 left) 2024-08-17T17:11:08.693 INFO:tasks.rados.rados.0.smithi063.stdout:3858: snap_create 2024-08-17T17:11:09.694 INFO:tasks.rados.rados.0.smithi063.stdout:3858: done (0 left) 2024-08-17T17:11:09.694 INFO:tasks.rados.rados.0.smithi063.stdout:3859: write oid 3 current snap is 487 2024-08-17T17:11:09.694 INFO:tasks.rados.rados.0.smithi063.stdout:3859: seq_num 1455 ranges {454185=510718,1493065=629841} 2024-08-17T17:11:09.699 INFO:tasks.rados.rados.0.smithi063.stdout:3859: writing smithi06387500-3 from 454185 to 964903 tid 1 2024-08-17T17:11:09.706 INFO:tasks.rados.rados.0.smithi063.stdout:3859: writing smithi06387500-3 from 1493065 to 2122906 tid 2 2024-08-17T17:11:09.706 INFO:tasks.rados.rados.0.smithi063.stdout:3860: snap_remove snap 470 2024-08-17T17:11:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:09 smithi063 ceph-mon[114756]: pgmap v1017: 89 pgs: 89 active+clean; 195 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 21 op/s 2024-08-17T17:11:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:09 smithi063 ceph-mon[114756]: osdmap e1041: 8 total, 8 up, 8 in 2024-08-17T17:11:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:09 smithi063 ceph-mon[109843]: pgmap v1017: 89 pgs: 89 active+clean; 195 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 21 op/s 2024-08-17T17:11:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:09 smithi063 ceph-mon[109843]: osdmap e1041: 8 total, 8 up, 8 in 2024-08-17T17:11:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:09 smithi162 ceph-mon[104668]: pgmap v1017: 89 pgs: 89 active+clean; 195 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 21 op/s 2024-08-17T17:11:10.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:09 smithi162 ceph-mon[104668]: osdmap e1041: 8 total, 8 up, 8 in 2024-08-17T17:11:10.706 INFO:tasks.rados.rados.0.smithi063.stdout:3860: done (1 left) 2024-08-17T17:11:10.706 INFO:tasks.rados.rados.0.smithi063.stdout:3861: snap_remove snap 484 2024-08-17T17:11:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:10 smithi063 ceph-mon[114756]: osdmap e1042: 8 total, 8 up, 8 in 2024-08-17T17:11:11.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:10 smithi063 ceph-mon[109843]: osdmap e1042: 8 total, 8 up, 8 in 2024-08-17T17:11:11.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:10 smithi162 ceph-mon[104668]: osdmap e1042: 8 total, 8 up, 8 in 2024-08-17T17:11:11.711 INFO:tasks.rados.rados.0.smithi063.stdout:3861: done (1 left) 2024-08-17T17:11:11.711 INFO:tasks.rados.rados.0.smithi063.stdout:3862: delete oid 45 current snap is 487 2024-08-17T17:11:11.711 INFO:tasks.rados.rados.0.smithi063.stdout:3859: finishing write tid 1 to smithi06387500-3 2024-08-17T17:11:11.711 INFO:tasks.rados.rados.0.smithi063.stdout:3859: finishing write tid 2 to smithi06387500-3 2024-08-17T17:11:11.711 INFO:tasks.rados.rados.0.smithi063.stdout:3859: finishing write tid 3 to smithi06387500-3 2024-08-17T17:11:11.711 INFO:tasks.rados.rados.0.smithi063.stdout:3859: finishing write tid 4 to smithi06387500-3 2024-08-17T17:11:11.711 INFO:tasks.rados.rados.0.smithi063.stdout:3859: oid 3 updating version 0 to 1362 2024-08-17T17:11:11.712 INFO:tasks.rados.rados.0.smithi063.stdout:3859: oid 3 version 1362 is already newer than 1361 2024-08-17T17:11:11.712 INFO:tasks.rados.rados.0.smithi063.stdout:3859: oid 3 version 1362 is already newer than 1360 2024-08-17T17:11:11.712 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 1362 (ObjNum 1455 snap 487 seq_num 1455) dirty exists 2024-08-17T17:11:11.712 INFO:tasks.rados.rados.0.smithi063.stdout:3859: left oid 3 (ObjNum 1455 snap 487 seq_num 1455) 2024-08-17T17:11:11.717 INFO:tasks.rados.rados.0.smithi063.stdout:3859: done (1 left) 2024-08-17T17:11:11.718 INFO:tasks.rados.rados.0.smithi063.stdout:3862: done (0 left) 2024-08-17T17:11:11.718 INFO:tasks.rados.rados.0.smithi063.stdout:3863: rollback oid 39 current snap is 487 2024-08-17T17:11:11.718 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 39 to 485 2024-08-17T17:11:11.718 INFO:tasks.rados.rados.0.smithi063.stdout:3864: write oid 50 current snap is 487 2024-08-17T17:11:11.718 INFO:tasks.rados.rados.0.smithi063.stdout:3864: seq_num 1457 ranges {487799=661216,1893676=594123} 2024-08-17T17:11:11.724 INFO:tasks.rados.rados.0.smithi063.stdout:3864: writing smithi06387500-50 from 487799 to 1149015 tid 1 2024-08-17T17:11:11.732 INFO:tasks.rados.rados.0.smithi063.stdout:3864: writing smithi06387500-50 from 1893676 to 2487799 tid 2 2024-08-17T17:11:11.732 INFO:tasks.rados.rados.0.smithi063.stdout:3865: write oid 37 current snap is 487 2024-08-17T17:11:11.732 INFO:tasks.rados.rados.0.smithi063.stdout:3863: finishing rollback tid 0 to smithi06387500-39 2024-08-17T17:11:11.732 INFO:tasks.rados.rados.0.smithi063.stdout:3865: seq_num 1458 ranges {504606=736465,1952158=552448} 2024-08-17T17:11:11.738 INFO:tasks.rados.rados.0.smithi063.stdout:3865: writing smithi06387500-37 from 504606 to 1241071 tid 1 2024-08-17T17:11:11.746 INFO:tasks.rados.rados.0.smithi063.stdout:3865: writing smithi06387500-37 from 1952158 to 2504606 tid 2 2024-08-17T17:11:11.746 INFO:tasks.rados.rados.0.smithi063.stdout:3866: snap_remove snap 474 2024-08-17T17:11:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:11 smithi063 ceph-mon[114756]: pgmap v1020: 89 pgs: 89 active+clean; 195 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-08-17T17:11:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:11 smithi063 ceph-mon[114756]: osdmap e1043: 8 total, 8 up, 8 in 2024-08-17T17:11:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:11 smithi063 ceph-mon[109843]: pgmap v1020: 89 pgs: 89 active+clean; 195 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-08-17T17:11:12.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:11 smithi063 ceph-mon[109843]: osdmap e1043: 8 total, 8 up, 8 in 2024-08-17T17:11:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:11 smithi162 ceph-mon[104668]: pgmap v1020: 89 pgs: 89 active+clean; 195 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.2 MiB/s wr, 9 op/s 2024-08-17T17:11:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:11 smithi162 ceph-mon[104668]: osdmap e1043: 8 total, 8 up, 8 in 2024-08-17T17:11:12.719 INFO:tasks.rados.rados.0.smithi063.stdout:3863: finishing rollback tid 1 to smithi06387500-39 2024-08-17T17:11:12.719 INFO:tasks.rados.rados.0.smithi063.stdout:3863: finishing rollback tid 2 to smithi06387500-39 2024-08-17T17:11:12.719 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 39 v 982 (ObjNum 1411 snap 473 seq_num 1411) dirty exists 2024-08-17T17:11:12.719 INFO:tasks.rados.rados.0.smithi063.stdout:3864: finishing write tid 1 to smithi06387500-50 2024-08-17T17:11:12.719 INFO:tasks.rados.rados.0.smithi063.stdout:3864: finishing write tid 2 to smithi06387500-50 2024-08-17T17:11:12.719 INFO:tasks.rados.rados.0.smithi063.stdout:3864: finishing write tid 3 to smithi06387500-50 2024-08-17T17:11:12.719 INFO:tasks.rados.rados.0.smithi063.stdout:3864: finishing write tid 4 to smithi06387500-50 2024-08-17T17:11:12.719 INFO:tasks.rados.rados.0.smithi063.stdout:3864: oid 50 updating version 0 to 2424 2024-08-17T17:11:12.719 INFO:tasks.rados.rados.0.smithi063.stdout:3864: oid 50 updating version 2424 to 2426 2024-08-17T17:11:12.720 INFO:tasks.rados.rados.0.smithi063.stdout:3864: oid 50 version 2426 is already newer than 2425 2024-08-17T17:11:12.720 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 50 v 2426 (ObjNum 1457 snap 487 seq_num 1457) dirty exists 2024-08-17T17:11:12.720 INFO:tasks.rados.rados.0.smithi063.stdout:3864: left oid 50 (ObjNum 1457 snap 487 seq_num 1457) 2024-08-17T17:11:12.720 INFO:tasks.rados.rados.0.smithi063.stdout:3865: finishing write tid 1 to smithi06387500-37 2024-08-17T17:11:12.720 INFO:tasks.rados.rados.0.smithi063.stdout:3865: finishing write tid 2 to smithi06387500-37 2024-08-17T17:11:12.720 INFO:tasks.rados.rados.0.smithi063.stdout:3865: finishing write tid 3 to smithi06387500-37 2024-08-17T17:11:12.720 INFO:tasks.rados.rados.0.smithi063.stdout:3865: finishing write tid 4 to smithi06387500-37 2024-08-17T17:11:12.720 INFO:tasks.rados.rados.0.smithi063.stdout:3865: oid 37 updating version 0 to 3799 2024-08-17T17:11:12.720 INFO:tasks.rados.rados.0.smithi063.stdout:3865: oid 37 updating version 3799 to 3800 2024-08-17T17:11:12.720 INFO:tasks.rados.rados.0.smithi063.stdout:3865: oid 37 version 3800 is already newer than 3798 2024-08-17T17:11:12.720 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 37 v 3800 (ObjNum 1458 snap 487 seq_num 1458) dirty exists 2024-08-17T17:11:12.720 INFO:tasks.rados.rados.0.smithi063.stdout:3865: left oid 37 (ObjNum 1458 snap 487 seq_num 1458) 2024-08-17T17:11:12.721 INFO:tasks.rados.rados.0.smithi063.stdout:3863: done (3 left) 2024-08-17T17:11:12.721 INFO:tasks.rados.rados.0.smithi063.stdout:3864: done (2 left) 2024-08-17T17:11:12.721 INFO:tasks.rados.rados.0.smithi063.stdout:3865: done (1 left) 2024-08-17T17:11:12.721 INFO:tasks.rados.rados.0.smithi063.stdout:3866: done (0 left) 2024-08-17T17:11:12.721 INFO:tasks.rados.rados.0.smithi063.stdout:3867: snap_create 2024-08-17T17:11:13.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:12 smithi063 ceph-mon[109843]: osdmap e1044: 8 total, 8 up, 8 in 2024-08-17T17:11:13.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:12 smithi063 ceph-mon[109843]: pgmap v1023: 89 pgs: 89 active+clean; 196 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 594 KiB/s wr, 4 op/s 2024-08-17T17:11:13.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:12 smithi063 ceph-mon[114756]: osdmap e1044: 8 total, 8 up, 8 in 2024-08-17T17:11:13.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:12 smithi063 ceph-mon[114756]: pgmap v1023: 89 pgs: 89 active+clean; 196 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 594 KiB/s wr, 4 op/s 2024-08-17T17:11:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:12 smithi162 ceph-mon[104668]: osdmap e1044: 8 total, 8 up, 8 in 2024-08-17T17:11:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:12 smithi162 ceph-mon[104668]: pgmap v1023: 89 pgs: 89 active+clean; 196 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 594 KiB/s wr, 4 op/s 2024-08-17T17:11:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:11:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:11:13] "GET /metrics HTTP/1.1" 200 36335 "" "Prometheus/2.51.0" 2024-08-17T17:11:13.718 INFO:tasks.rados.rados.0.smithi063.stdout:3867: done (0 left) 2024-08-17T17:11:13.718 INFO:tasks.rados.rados.0.smithi063.stdout:3868: snap_create 2024-08-17T17:11:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:13 smithi063 ceph-mon[114756]: osdmap e1045: 8 total, 8 up, 8 in 2024-08-17T17:11:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:13 smithi063 ceph-mon[109843]: osdmap e1045: 8 total, 8 up, 8 in 2024-08-17T17:11:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:13 smithi162 ceph-mon[104668]: osdmap e1045: 8 total, 8 up, 8 in 2024-08-17T17:11:14.729 INFO:tasks.rados.rados.0.smithi063.stdout:3868: done (0 left) 2024-08-17T17:11:14.729 INFO:tasks.rados.rados.0.smithi063.stdout:3869: snap_create 2024-08-17T17:11:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:14 smithi063 ceph-mon[114756]: osdmap e1046: 8 total, 8 up, 8 in 2024-08-17T17:11:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:14 smithi063 ceph-mon[114756]: pgmap v1026: 89 pgs: 89 active+clean; 196 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 594 KiB/s wr, 4 op/s 2024-08-17T17:11:15.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:14 smithi063 ceph-mon[109843]: osdmap e1046: 8 total, 8 up, 8 in 2024-08-17T17:11:15.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:14 smithi063 ceph-mon[109843]: pgmap v1026: 89 pgs: 89 active+clean; 196 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 594 KiB/s wr, 4 op/s 2024-08-17T17:11:15.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:14 smithi162 ceph-mon[104668]: osdmap e1046: 8 total, 8 up, 8 in 2024-08-17T17:11:15.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:14 smithi162 ceph-mon[104668]: pgmap v1026: 89 pgs: 89 active+clean; 196 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 594 KiB/s wr, 4 op/s 2024-08-17T17:11:15.730 INFO:tasks.rados.rados.0.smithi063.stdout:3869: done (0 left) 2024-08-17T17:11:15.730 INFO:tasks.rados.rados.0.smithi063.stdout:3870: write oid 5 current snap is 490 2024-08-17T17:11:15.730 INFO:tasks.rados.rados.0.smithi063.stdout:3870: seq_num 1459 ranges {521413=411714,1610640=637179} 2024-08-17T17:11:15.735 INFO:tasks.rados.rados.0.smithi063.stdout:3870: writing smithi06387500-5 from 521413 to 933127 tid 1 2024-08-17T17:11:15.742 INFO:tasks.rados.rados.0.smithi063.stdout:3870: writing smithi06387500-5 from 1610640 to 2247819 tid 2 2024-08-17T17:11:15.743 INFO:tasks.rados.rados.0.smithi063.stdout:3871: write oid 16 current snap is 490 2024-08-17T17:11:15.743 INFO:tasks.rados.rados.0.smithi063.stdout:3870: finishing write tid 1 to smithi06387500-5 2024-08-17T17:11:15.743 INFO:tasks.rados.rados.0.smithi063.stdout:3871: seq_num 1460 ranges {538220=603316,1785475=752745} 2024-08-17T17:11:15.749 INFO:tasks.rados.rados.0.smithi063.stdout:3871: writing smithi06387500-16 from 538220 to 1141536 tid 1 2024-08-17T17:11:15.757 INFO:tasks.rados.rados.0.smithi063.stdout:3871: writing smithi06387500-16 from 1785475 to 2538220 tid 2 2024-08-17T17:11:15.757 INFO:tasks.rados.rados.0.smithi063.stdout:3872: write oid 46 current snap is 490 2024-08-17T17:11:15.757 INFO:tasks.rados.rados.0.smithi063.stdout:3870: finishing write tid 2 to smithi06387500-5 2024-08-17T17:11:15.757 INFO:tasks.rados.rados.0.smithi063.stdout:3872: seq_num 1461 ranges {555027=678565,1727604=640848} 2024-08-17T17:11:15.764 INFO:tasks.rados.rados.0.smithi063.stdout:3872: writing smithi06387500-46 from 555027 to 1233592 tid 1 2024-08-17T17:11:15.771 INFO:tasks.rados.rados.0.smithi063.stdout:3872: writing smithi06387500-46 from 1727604 to 2368452 tid 2 2024-08-17T17:11:15.771 INFO:tasks.rados.rados.0.smithi063.stdout:3873: read oid 38 snap -1 2024-08-17T17:11:15.771 INFO:tasks.rados.rados.0.smithi063.stdout:3873: expect (ObjNum 1444 snap 481 seq_num 1444) 2024-08-17T17:11:15.771 INFO:tasks.rados.rados.0.smithi063.stdout:3874: delete oid 48 current snap is 490 2024-08-17T17:11:15.771 INFO:tasks.rados.rados.0.smithi063.stdout:3870: finishing write tid 3 to smithi06387500-5 2024-08-17T17:11:15.771 INFO:tasks.rados.rados.0.smithi063.stdout:3870: finishing write tid 4 to smithi06387500-5 2024-08-17T17:11:15.771 INFO:tasks.rados.rados.0.smithi063.stdout:3870: oid 5 updating version 0 to 2082 2024-08-17T17:11:15.772 INFO:tasks.rados.rados.0.smithi063.stdout:3870: oid 5 updating version 2082 to 2083 2024-08-17T17:11:15.772 INFO:tasks.rados.rados.0.smithi063.stdout:3870: oid 5 updating version 2083 to 2084 2024-08-17T17:11:15.772 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 2084 (ObjNum 1459 snap 490 seq_num 1459) dirty exists 2024-08-17T17:11:15.772 INFO:tasks.rados.rados.0.smithi063.stdout:3870: left oid 5 (ObjNum 1459 snap 490 seq_num 1459) 2024-08-17T17:11:15.772 INFO:tasks.rados.rados.0.smithi063.stdout:3871: finishing write tid 1 to smithi06387500-16 2024-08-17T17:11:15.772 INFO:tasks.rados.rados.0.smithi063.stdout:3871: finishing write tid 2 to smithi06387500-16 2024-08-17T17:11:15.772 INFO:tasks.rados.rados.0.smithi063.stdout:3871: finishing write tid 3 to smithi06387500-16 2024-08-17T17:11:15.772 INFO:tasks.rados.rados.0.smithi063.stdout:3871: finishing write tid 4 to smithi06387500-16 2024-08-17T17:11:15.772 INFO:tasks.rados.rados.0.smithi063.stdout:3871: oid 16 updating version 0 to 1498 2024-08-17T17:11:15.772 INFO:tasks.rados.rados.0.smithi063.stdout:3871: oid 16 updating version 1498 to 1499 2024-08-17T17:11:15.772 INFO:tasks.rados.rados.0.smithi063.stdout:3871: oid 16 updating version 1499 to 1500 2024-08-17T17:11:15.772 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 16 v 1500 (ObjNum 1460 snap 490 seq_num 1460) dirty exists 2024-08-17T17:11:15.772 INFO:tasks.rados.rados.0.smithi063.stdout:3871: left oid 16 (ObjNum 1460 snap 490 seq_num 1460) 2024-08-17T17:11:15.773 INFO:tasks.rados.rados.0.smithi063.stdout:3872: finishing write tid 1 to smithi06387500-46 2024-08-17T17:11:15.775 INFO:tasks.rados.rados.0.smithi063.stdout:3873: expect (ObjNum 1444 snap 481 seq_num 1444) 2024-08-17T17:11:15.850 INFO:tasks.rados.rados.0.smithi063.stdout:3872: finishing write tid 2 to smithi06387500-46 2024-08-17T17:11:15.850 INFO:tasks.rados.rados.0.smithi063.stdout:3872: finishing write tid 3 to smithi06387500-46 2024-08-17T17:11:15.850 INFO:tasks.rados.rados.0.smithi063.stdout:3872: finishing write tid 4 to smithi06387500-46 2024-08-17T17:11:15.850 INFO:tasks.rados.rados.0.smithi063.stdout:3872: oid 46 updating version 0 to 1501 2024-08-17T17:11:15.850 INFO:tasks.rados.rados.0.smithi063.stdout:3872: oid 46 updating version 1501 to 1503 2024-08-17T17:11:15.851 INFO:tasks.rados.rados.0.smithi063.stdout:3872: oid 46 version 1503 is already newer than 1502 2024-08-17T17:11:15.851 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 46 v 1503 (ObjNum 1461 snap 490 seq_num 1461) dirty exists 2024-08-17T17:11:15.851 INFO:tasks.rados.rados.0.smithi063.stdout:3872: left oid 46 (ObjNum 1461 snap 490 seq_num 1461) 2024-08-17T17:11:15.851 INFO:tasks.rados.rados.0.smithi063.stdout:3870: done (4 left) 2024-08-17T17:11:15.851 INFO:tasks.rados.rados.0.smithi063.stdout:3871: done (3 left) 2024-08-17T17:11:15.851 INFO:tasks.rados.rados.0.smithi063.stdout:3872: done (2 left) 2024-08-17T17:11:15.851 INFO:tasks.rados.rados.0.smithi063.stdout:3873: done (1 left) 2024-08-17T17:11:15.851 INFO:tasks.rados.rados.0.smithi063.stdout:3874: done (0 left) 2024-08-17T17:11:15.851 INFO:tasks.rados.rados.0.smithi063.stdout:3875: read oid 42 snap -1 2024-08-17T17:11:15.851 INFO:tasks.rados.rados.0.smithi063.stdout:3875: expect (ObjNum 1304 snap 443 seq_num 1304) 2024-08-17T17:11:15.851 INFO:tasks.rados.rados.0.smithi063.stdout:3876: write oid 48 current snap is 490 2024-08-17T17:11:15.851 INFO:tasks.rados.rados.0.smithi063.stdout:3876: seq_num 1463 ranges {588641=429063,1444568=644517,2492662=95979} 2024-08-17T17:11:15.855 INFO:tasks.rados.rados.0.smithi063.stdout:3876: writing smithi06387500-48 from 588641 to 1017704 tid 1 2024-08-17T17:11:15.862 INFO:tasks.rados.rados.0.smithi063.stdout:3876: writing smithi06387500-48 from 1444568 to 2089085 tid 2 2024-08-17T17:11:15.865 INFO:tasks.rados.rados.0.smithi063.stdout:3876: writing smithi06387500-48 from 2492662 to 2588641 tid 3 2024-08-17T17:11:15.865 INFO:tasks.rados.rados.0.smithi063.stdout:3877: delete oid 3 current snap is 490 2024-08-17T17:11:15.865 INFO:tasks.rados.rados.0.smithi063.stdout:3875: expect (ObjNum 1304 snap 443 seq_num 1304) 2024-08-17T17:11:15.999 INFO:tasks.rados.rados.0.smithi063.stdout:3876: finishing write tid 1 to smithi06387500-48 2024-08-17T17:11:15.999 INFO:tasks.rados.rados.0.smithi063.stdout:3876: finishing write tid 2 to smithi06387500-48 2024-08-17T17:11:15.999 INFO:tasks.rados.rados.0.smithi063.stdout:3876: finishing write tid 3 to smithi06387500-48 2024-08-17T17:11:15.999 INFO:tasks.rados.rados.0.smithi063.stdout:3876: finishing write tid 4 to smithi06387500-48 2024-08-17T17:11:16.000 INFO:tasks.rados.rados.0.smithi063.stdout:3876: finishing write tid 5 to smithi06387500-48 2024-08-17T17:11:16.000 INFO:tasks.rados.rados.0.smithi063.stdout:3876: oid 48 updating version 0 to 3809 2024-08-17T17:11:16.000 INFO:tasks.rados.rados.0.smithi063.stdout:3876: oid 48 version 3809 is already newer than 3807 2024-08-17T17:11:16.000 INFO:tasks.rados.rados.0.smithi063.stdout:3876: oid 48 version 3809 is already newer than 3806 2024-08-17T17:11:16.000 INFO:tasks.rados.rados.0.smithi063.stdout:3876: oid 48 version 3809 is already newer than 3808 2024-08-17T17:11:16.000 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 48 v 3809 (ObjNum 1463 snap 490 seq_num 1463) dirty exists 2024-08-17T17:11:16.000 INFO:tasks.rados.rados.0.smithi063.stdout:3876: left oid 48 (ObjNum 1463 snap 490 seq_num 1463) 2024-08-17T17:11:16.000 INFO:tasks.rados.rados.0.smithi063.stdout:3875: done (2 left) 2024-08-17T17:11:16.000 INFO:tasks.rados.rados.0.smithi063.stdout:3876: done (1 left) 2024-08-17T17:11:16.000 INFO:tasks.rados.rados.0.smithi063.stdout:3877: done (0 left) 2024-08-17T17:11:16.000 INFO:tasks.rados.rados.0.smithi063.stdout:3878: write oid 47 current snap is 490 2024-08-17T17:11:16.000 INFO:tasks.rados.rados.0.smithi063.stdout:3878: seq_num 1465 ranges {622255=579561,1845179=531833} 2024-08-17T17:11:16.005 INFO:tasks.rados.rados.0.smithi063.stdout:3878: writing smithi06387500-47 from 622255 to 1201816 tid 1 2024-08-17T17:11:16.012 INFO:tasks.rados.rados.0.smithi063.stdout:3878: writing smithi06387500-47 from 1845179 to 2377012 tid 2 2024-08-17T17:11:16.012 INFO:tasks.rados.rados.0.smithi063.stdout:3879: read oid 49 snap -1 2024-08-17T17:11:16.012 INFO:tasks.rados.rados.0.smithi063.stdout:3879: expect (ObjNum 1101 snap 366 seq_num 1101) 2024-08-17T17:11:16.012 INFO:tasks.rados.rados.0.smithi063.stdout:3880: write oid 23 current snap is 490 2024-08-17T17:11:16.012 INFO:tasks.rados.rados.0.smithi063.stdout:3878: finishing write tid 1 to smithi06387500-47 2024-08-17T17:11:16.012 INFO:tasks.rados.rados.0.smithi063.stdout:3880: seq_num 1466 ranges {639062=654810,1903661=735401} 2024-08-17T17:11:16.019 INFO:tasks.rados.rados.0.smithi063.stdout:3880: writing smithi06387500-23 from 639062 to 1293872 tid 1 2024-08-17T17:11:16.027 INFO:tasks.rados.rados.0.smithi063.stdout:3880: writing smithi06387500-23 from 1903661 to 2639062 tid 2 2024-08-17T17:11:16.027 INFO:tasks.rados.rados.0.smithi063.stdout:3881: write oid 44 current snap is 490 2024-08-17T17:11:16.027 INFO:tasks.rados.rados.0.smithi063.stdout:3878: finishing write tid 2 to smithi06387500-47 2024-08-17T17:11:16.027 INFO:tasks.rados.rados.0.smithi063.stdout:3878: finishing write tid 3 to smithi06387500-47 2024-08-17T17:11:16.027 INFO:tasks.rados.rados.0.smithi063.stdout:3878: finishing write tid 4 to smithi06387500-47 2024-08-17T17:11:16.027 INFO:tasks.rados.rados.0.smithi063.stdout:3878: oid 47 updating version 0 to 891 2024-08-17T17:11:16.027 INFO:tasks.rados.rados.0.smithi063.stdout:3878: oid 47 version 891 is already newer than 889 2024-08-17T17:11:16.027 INFO:tasks.rados.rados.0.smithi063.stdout:3878: oid 47 version 891 is already newer than 890 2024-08-17T17:11:16.027 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 891 (ObjNum 1465 snap 490 seq_num 1465) dirty exists 2024-08-17T17:11:16.027 INFO:tasks.rados.rados.0.smithi063.stdout:3878: left oid 47 (ObjNum 1465 snap 490 seq_num 1465) 2024-08-17T17:11:16.027 INFO:tasks.rados.rados.0.smithi063.stdout:3879: expect (ObjNum 1101 snap 366 seq_num 1101) 2024-08-17T17:11:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:15 smithi063 ceph-mon[109843]: osdmap e1047: 8 total, 8 up, 8 in 2024-08-17T17:11:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:15 smithi063 ceph-mon[114756]: osdmap e1047: 8 total, 8 up, 8 in 2024-08-17T17:11:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:15 smithi162 ceph-mon[104668]: osdmap e1047: 8 total, 8 up, 8 in 2024-08-17T17:11:16.167 INFO:tasks.rados.rados.0.smithi063.stdout:3880: finishing write tid 1 to smithi06387500-23 2024-08-17T17:11:16.167 INFO:tasks.rados.rados.0.smithi063.stdout:3880: finishing write tid 2 to smithi06387500-23 2024-08-17T17:11:16.167 INFO:tasks.rados.rados.0.smithi063.stdout:3880: finishing write tid 3 to smithi06387500-23 2024-08-17T17:11:16.167 INFO:tasks.rados.rados.0.smithi063.stdout:3880: finishing write tid 4 to smithi06387500-23 2024-08-17T17:11:16.168 INFO:tasks.rados.rados.0.smithi063.stdout:3880: oid 23 updating version 0 to 2087 2024-08-17T17:11:16.168 INFO:tasks.rados.rados.0.smithi063.stdout:3880: oid 23 updating version 2087 to 2088 2024-08-17T17:11:16.168 INFO:tasks.rados.rados.0.smithi063.stdout:3880: oid 23 version 2088 is already newer than 2086 2024-08-17T17:11:16.168 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 23 v 2088 (ObjNum 1466 snap 490 seq_num 1466) dirty exists 2024-08-17T17:11:16.168 INFO:tasks.rados.rados.0.smithi063.stdout:3880: left oid 23 (ObjNum 1466 snap 490 seq_num 1466) 2024-08-17T17:11:16.168 INFO:tasks.rados.rados.0.smithi063.stdout:3881: seq_num 1467 ranges {655869=730059,1962143=535502} 2024-08-17T17:11:16.174 INFO:tasks.rados.rados.0.smithi063.stdout:3881: writing smithi06387500-44 from 655869 to 1385928 tid 1 2024-08-17T17:11:16.181 INFO:tasks.rados.rados.0.smithi063.stdout:3881: writing smithi06387500-44 from 1962143 to 2497645 tid 2 2024-08-17T17:11:16.182 INFO:tasks.rados.rados.0.smithi063.stdout:3878: done (3 left) 2024-08-17T17:11:16.182 INFO:tasks.rados.rados.0.smithi063.stdout:3879: done (2 left) 2024-08-17T17:11:16.182 INFO:tasks.rados.rados.0.smithi063.stdout:3880: done (1 left) 2024-08-17T17:11:16.182 INFO:tasks.rados.rados.0.smithi063.stdout:3882: snap_create 2024-08-17T17:11:16.182 INFO:tasks.rados.rados.0.smithi063.stdout:3881: finishing write tid 1 to smithi06387500-44 2024-08-17T17:11:16.184 INFO:tasks.rados.rados.0.smithi063.stdout:3881: finishing write tid 2 to smithi06387500-44 2024-08-17T17:11:16.184 INFO:tasks.rados.rados.0.smithi063.stdout:3881: finishing write tid 3 to smithi06387500-44 2024-08-17T17:11:16.184 INFO:tasks.rados.rados.0.smithi063.stdout:3881: finishing write tid 4 to smithi06387500-44 2024-08-17T17:11:16.184 INFO:tasks.rados.rados.0.smithi063.stdout:3881: oid 44 updating version 0 to 571 2024-08-17T17:11:16.184 INFO:tasks.rados.rados.0.smithi063.stdout:3881: oid 44 updating version 571 to 572 2024-08-17T17:11:16.184 INFO:tasks.rados.rados.0.smithi063.stdout:3881: oid 44 version 572 is already newer than 570 2024-08-17T17:11:16.184 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 572 (ObjNum 1467 snap 490 seq_num 1467) dirty exists 2024-08-17T17:11:16.185 INFO:tasks.rados.rados.0.smithi063.stdout:3881: left oid 44 (ObjNum 1467 snap 490 seq_num 1467) 2024-08-17T17:11:16.741 INFO:tasks.rados.rados.0.smithi063.stdout:3881: done (1 left) 2024-08-17T17:11:16.741 INFO:tasks.rados.rados.0.smithi063.stdout:3882: done (0 left) 2024-08-17T17:11:16.741 INFO:tasks.rados.rados.0.smithi063.stdout:3883: read oid 50 snap -1 2024-08-17T17:11:16.741 INFO:tasks.rados.rados.0.smithi063.stdout:3883: expect (ObjNum 1457 snap 487 seq_num 1457) 2024-08-17T17:11:16.741 INFO:tasks.rados.rados.0.smithi063.stdout:3884: write oid 13 current snap is 491 2024-08-17T17:11:16.741 INFO:tasks.rados.rados.0.smithi063.stdout:3884: seq_num 1468 ranges {672676=521661,1736978=795513} 2024-08-17T17:11:16.747 INFO:tasks.rados.rados.0.smithi063.stdout:3884: writing smithi06387500-13 from 672676 to 1194337 tid 1 2024-08-17T17:11:16.755 INFO:tasks.rados.rados.0.smithi063.stdout:3884: writing smithi06387500-13 from 1736978 to 2532491 tid 2 2024-08-17T17:11:16.755 INFO:tasks.rados.rados.0.smithi063.stdout:3885: read oid 30 snap 489 2024-08-17T17:11:16.756 INFO:tasks.rados.rados.0.smithi063.stdout:3885: expect (ObjNum 1454 snap 485 seq_num 1454) 2024-08-17T17:11:16.756 INFO:tasks.rados.rados.0.smithi063.stdout:3886: read oid 5 snap -1 2024-08-17T17:11:16.756 INFO:tasks.rados.rados.0.smithi063.stdout:3886: expect (ObjNum 1459 snap 490 seq_num 1459) 2024-08-17T17:11:16.756 INFO:tasks.rados.rados.0.smithi063.stdout:3887: delete oid 12 current snap is 491 2024-08-17T17:11:16.756 INFO:tasks.rados.rados.0.smithi063.stdout:3883: expect (ObjNum 1457 snap 487 seq_num 1457) 2024-08-17T17:11:16.893 INFO:tasks.rados.rados.0.smithi063.stdout:3884: finishing write tid 1 to smithi06387500-13 2024-08-17T17:11:16.893 INFO:tasks.rados.rados.0.smithi063.stdout:3885: expect (ObjNum 1454 snap 485 seq_num 1454) 2024-08-17T17:11:17.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:16 smithi063 ceph-mon[109843]: osdmap e1048: 8 total, 8 up, 8 in 2024-08-17T17:11:17.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:16 smithi063 ceph-mon[109843]: pgmap v1029: 89 pgs: 89 active+clean; 198 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 307 KiB/s wr, 1 op/s 2024-08-17T17:11:17.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:17.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:16 smithi063 ceph-mon[114756]: osdmap e1048: 8 total, 8 up, 8 in 2024-08-17T17:11:17.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:16 smithi063 ceph-mon[114756]: pgmap v1029: 89 pgs: 89 active+clean; 198 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 307 KiB/s wr, 1 op/s 2024-08-17T17:11:17.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:16 smithi162 ceph-mon[104668]: osdmap e1048: 8 total, 8 up, 8 in 2024-08-17T17:11:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:16 smithi162 ceph-mon[104668]: pgmap v1029: 89 pgs: 89 active+clean; 198 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 307 KiB/s wr, 1 op/s 2024-08-17T17:11:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:17.119 INFO:tasks.rados.rados.0.smithi063.stdout:3884: finishing write tid 2 to smithi06387500-13 2024-08-17T17:11:17.119 INFO:tasks.rados.rados.0.smithi063.stdout:3884: finishing write tid 3 to smithi06387500-13 2024-08-17T17:11:17.120 INFO:tasks.rados.rados.0.smithi063.stdout:3884: finishing write tid 4 to smithi06387500-13 2024-08-17T17:11:17.120 INFO:tasks.rados.rados.0.smithi063.stdout:3884: oid 13 updating version 0 to 2091 2024-08-17T17:11:17.120 INFO:tasks.rados.rados.0.smithi063.stdout:3884: oid 13 updating version 2091 to 2092 2024-08-17T17:11:17.120 INFO:tasks.rados.rados.0.smithi063.stdout:3884: oid 13 version 2092 is already newer than 2090 2024-08-17T17:11:17.120 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 2092 (ObjNum 1468 snap 491 seq_num 1468) dirty exists 2024-08-17T17:11:17.120 INFO:tasks.rados.rados.0.smithi063.stdout:3884: left oid 13 (ObjNum 1468 snap 491 seq_num 1468) 2024-08-17T17:11:17.120 INFO:tasks.rados.rados.0.smithi063.stdout:3886: expect (ObjNum 1459 snap 490 seq_num 1459) 2024-08-17T17:11:17.246 INFO:tasks.rados.rados.0.smithi063.stdout:3883: done (4 left) 2024-08-17T17:11:17.247 INFO:tasks.rados.rados.0.smithi063.stdout:3884: done (3 left) 2024-08-17T17:11:17.247 INFO:tasks.rados.rados.0.smithi063.stdout:3885: done (2 left) 2024-08-17T17:11:17.247 INFO:tasks.rados.rados.0.smithi063.stdout:3886: done (1 left) 2024-08-17T17:11:17.247 INFO:tasks.rados.rados.0.smithi063.stdout:3887: done (0 left) 2024-08-17T17:11:17.247 INFO:tasks.rados.rados.0.smithi063.stdout:3888: snap_create 2024-08-17T17:11:17.693 INFO:tasks.rados.rados.0.smithi063.stdout:3888: done (0 left) 2024-08-17T17:11:17.693 INFO:tasks.rados.rados.0.smithi063.stdout:3889: read oid 3 snap -1 2024-08-17T17:11:17.693 INFO:tasks.rados.rados.0.smithi063.stdout:3889: expect deleted 2024-08-17T17:11:17.693 INFO:tasks.rados.rados.0.smithi063.stdout:3890: delete oid 34 current snap is 492 2024-08-17T17:11:17.699 INFO:tasks.rados.rados.0.smithi063.stdout:3889: done (1 left) 2024-08-17T17:11:17.699 INFO:tasks.rados.rados.0.smithi063.stdout:3890: done (0 left) 2024-08-17T17:11:17.699 INFO:tasks.rados.rados.0.smithi063.stdout:3891: rollback oid 14 current snap is 492 2024-08-17T17:11:17.700 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 14 to 490 2024-08-17T17:11:17.700 INFO:tasks.rados.rados.0.smithi063.stdout:3892: write oid 26 current snap is 492 2024-08-17T17:11:17.700 INFO:tasks.rados.rados.0.smithi063.stdout:3892: seq_num 1471 ranges {723097=747408,2196071=527026} 2024-08-17T17:11:17.707 INFO:tasks.rados.rados.0.smithi063.stdout:3892: writing smithi06387500-26 from 723097 to 1470505 tid 1 2024-08-17T17:11:17.715 INFO:tasks.rados.rados.0.smithi063.stdout:3892: writing smithi06387500-26 from 2196071 to 2723097 tid 2 2024-08-17T17:11:17.715 INFO:tasks.rados.rados.0.smithi063.stdout:3893: read oid 38 snap -1 2024-08-17T17:11:17.715 INFO:tasks.rados.rados.0.smithi063.stdout:3893: expect (ObjNum 1444 snap 481 seq_num 1444) 2024-08-17T17:11:17.715 INFO:tasks.rados.rados.0.smithi063.stdout:3894: delete oid 21 current snap is 492 2024-08-17T17:11:17.715 INFO:tasks.rados.rados.0.smithi063.stdout:3891: finishing rollback tid 1 to smithi06387500-14 2024-08-17T17:11:17.715 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 14 v 3767 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-08-17T17:11:17.715 INFO:tasks.rados.rados.0.smithi063.stdout:3892: finishing write tid 1 to smithi06387500-26 2024-08-17T17:11:17.718 INFO:tasks.rados.rados.0.smithi063.stdout:3891: done (3 left) 2024-08-17T17:11:17.718 INFO:tasks.rados.rados.0.smithi063.stdout:3894: done (2 left) 2024-08-17T17:11:17.718 INFO:tasks.rados.rados.0.smithi063.stdout:3895: write oid 44 current snap is 492 2024-08-17T17:11:17.719 INFO:tasks.rados.rados.0.smithi063.stdout:3895: seq_num 1473 ranges {756711=497906,1913035=546509} 2024-08-17T17:11:17.725 INFO:tasks.rados.rados.0.smithi063.stdout:3895: writing smithi06387500-44 from 756711 to 1254617 tid 1 2024-08-17T17:11:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:3895: writing smithi06387500-44 from 1913035 to 2459544 tid 2 2024-08-17T17:11:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:3896: delete oid 47 current snap is 492 2024-08-17T17:11:17.732 INFO:tasks.rados.rados.0.smithi063.stdout:3892: finishing write tid 2 to smithi06387500-26 2024-08-17T17:11:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:3892: finishing write tid 3 to smithi06387500-26 2024-08-17T17:11:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:3892: finishing write tid 4 to smithi06387500-26 2024-08-17T17:11:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:3892: oid 26 updating version 0 to 1372 2024-08-17T17:11:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:3892: oid 26 version 1372 is already newer than 1370 2024-08-17T17:11:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:3892: oid 26 version 1372 is already newer than 1371 2024-08-17T17:11:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 1372 (ObjNum 1471 snap 492 seq_num 1471) dirty exists 2024-08-17T17:11:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:3892: left oid 26 (ObjNum 1471 snap 492 seq_num 1471) 2024-08-17T17:11:17.733 INFO:tasks.rados.rados.0.smithi063.stdout:3893: expect (ObjNum 1444 snap 481 seq_num 1444) 2024-08-17T17:11:17.807 INFO:tasks.rados.rados.0.smithi063.stdout:3895: finishing write tid 1 to smithi06387500-44 2024-08-17T17:11:17.808 INFO:tasks.rados.rados.0.smithi063.stdout:3895: finishing write tid 2 to smithi06387500-44 2024-08-17T17:11:17.808 INFO:tasks.rados.rados.0.smithi063.stdout:3895: finishing write tid 3 to smithi06387500-44 2024-08-17T17:11:17.808 INFO:tasks.rados.rados.0.smithi063.stdout:3895: finishing write tid 4 to smithi06387500-44 2024-08-17T17:11:17.808 INFO:tasks.rados.rados.0.smithi063.stdout:3895: oid 44 updating version 0 to 574 2024-08-17T17:11:17.808 INFO:tasks.rados.rados.0.smithi063.stdout:3895: oid 44 updating version 574 to 575 2024-08-17T17:11:17.808 INFO:tasks.rados.rados.0.smithi063.stdout:3895: oid 44 updating version 575 to 576 2024-08-17T17:11:17.808 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 576 (ObjNum 1473 snap 492 seq_num 1473) dirty exists 2024-08-17T17:11:17.808 INFO:tasks.rados.rados.0.smithi063.stdout:3895: left oid 44 (ObjNum 1473 snap 492 seq_num 1473) 2024-08-17T17:11:17.808 INFO:tasks.rados.rados.0.smithi063.stdout:3892: done (3 left) 2024-08-17T17:11:17.808 INFO:tasks.rados.rados.0.smithi063.stdout:3893: done (2 left) 2024-08-17T17:11:17.808 INFO:tasks.rados.rados.0.smithi063.stdout:3895: done (1 left) 2024-08-17T17:11:17.808 INFO:tasks.rados.rados.0.smithi063.stdout:3896: done (0 left) 2024-08-17T17:11:17.809 INFO:tasks.rados.rados.0.smithi063.stdout:3897: write oid 13 current snap is 492 2024-08-17T17:11:17.809 INFO:tasks.rados.rados.0.smithi063.stdout:3897: seq_num 1475 ranges {790325=764757,2029999=550178} 2024-08-17T17:11:17.815 INFO:tasks.rados.rados.0.smithi063.stdout:3897: writing smithi06387500-13 from 790325 to 1555082 tid 1 2024-08-17T17:11:17.822 INFO:tasks.rados.rados.0.smithi063.stdout:3897: writing smithi06387500-13 from 2029999 to 2580177 tid 2 2024-08-17T17:11:17.822 INFO:tasks.rados.rados.0.smithi063.stdout:3898: read oid 25 snap -1 2024-08-17T17:11:17.822 INFO:tasks.rados.rados.0.smithi063.stdout:3898: expect (ObjNum 1442 snap 481 seq_num 1442) 2024-08-17T17:11:17.822 INFO:tasks.rados.rados.0.smithi063.stdout:3899: write oid 33 current snap is 492 2024-08-17T17:11:17.822 INFO:tasks.rados.rados.0.smithi063.stdout:3897: finishing write tid 1 to smithi06387500-13 2024-08-17T17:11:17.822 INFO:tasks.rados.rados.0.smithi063.stdout:3899: seq_num 1476 ranges {407132=440006,1288481=693836,2616455=190677} 2024-08-17T17:11:17.826 INFO:tasks.rados.rados.0.smithi063.stdout:3899: writing smithi06387500-33 from 407132 to 847138 tid 1 2024-08-17T17:11:17.832 INFO:tasks.rados.rados.0.smithi063.stdout:3899: writing smithi06387500-33 from 1288481 to 1982317 tid 2 2024-08-17T17:11:17.837 INFO:tasks.rados.rados.0.smithi063.stdout:3899: writing smithi06387500-33 from 2616455 to 2807132 tid 3 2024-08-17T17:11:17.837 INFO:tasks.rados.rados.0.smithi063.stdout:3900: delete oid 31 current snap is 492 2024-08-17T17:11:17.837 INFO:tasks.rados.rados.0.smithi063.stdout:3897: finishing write tid 2 to smithi06387500-13 2024-08-17T17:11:17.837 INFO:tasks.rados.rados.0.smithi063.stdout:3897: finishing write tid 3 to smithi06387500-13 2024-08-17T17:11:17.838 INFO:tasks.rados.rados.0.smithi063.stdout:3897: finishing write tid 4 to smithi06387500-13 2024-08-17T17:11:17.838 INFO:tasks.rados.rados.0.smithi063.stdout:3897: oid 13 updating version 0 to 2096 2024-08-17T17:11:17.838 INFO:tasks.rados.rados.0.smithi063.stdout:3897: oid 13 version 2096 is already newer than 2094 2024-08-17T17:11:17.838 INFO:tasks.rados.rados.0.smithi063.stdout:3897: oid 13 version 2096 is already newer than 2095 2024-08-17T17:11:17.838 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 13 v 2096 (ObjNum 1475 snap 492 seq_num 1475) dirty exists 2024-08-17T17:11:17.838 INFO:tasks.rados.rados.0.smithi063.stdout:3897: left oid 13 (ObjNum 1475 snap 492 seq_num 1475) 2024-08-17T17:11:17.838 INFO:tasks.rados.rados.0.smithi063.stdout:3898: expect (ObjNum 1442 snap 481 seq_num 1442) 2024-08-17T17:11:17.945 INFO:tasks.rados.rados.0.smithi063.stdout:3899: finishing write tid 1 to smithi06387500-33 2024-08-17T17:11:17.945 INFO:tasks.rados.rados.0.smithi063.stdout:3899: finishing write tid 2 to smithi06387500-33 2024-08-17T17:11:17.946 INFO:tasks.rados.rados.0.smithi063.stdout:3899: finishing write tid 3 to smithi06387500-33 2024-08-17T17:11:17.946 INFO:tasks.rados.rados.0.smithi063.stdout:3899: finishing write tid 4 to smithi06387500-33 2024-08-17T17:11:17.946 INFO:tasks.rados.rados.0.smithi063.stdout:3899: finishing write tid 5 to smithi06387500-33 2024-08-17T17:11:17.946 INFO:tasks.rados.rados.0.smithi063.stdout:3899: oid 33 updating version 0 to 1297 2024-08-17T17:11:17.946 INFO:tasks.rados.rados.0.smithi063.stdout:3899: oid 33 updating version 1297 to 1299 2024-08-17T17:11:17.946 INFO:tasks.rados.rados.0.smithi063.stdout:3899: oid 33 updating version 1299 to 1300 2024-08-17T17:11:17.946 INFO:tasks.rados.rados.0.smithi063.stdout:3899: oid 33 version 1300 is already newer than 1298 2024-08-17T17:11:17.946 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 1300 (ObjNum 1476 snap 492 seq_num 1476) dirty exists 2024-08-17T17:11:17.946 INFO:tasks.rados.rados.0.smithi063.stdout:3899: left oid 33 (ObjNum 1476 snap 492 seq_num 1476) 2024-08-17T17:11:17.946 INFO:tasks.rados.rados.0.smithi063.stdout:3897: done (3 left) 2024-08-17T17:11:17.947 INFO:tasks.rados.rados.0.smithi063.stdout:3898: done (2 left) 2024-08-17T17:11:17.947 INFO:tasks.rados.rados.0.smithi063.stdout:3899: done (1 left) 2024-08-17T17:11:17.947 INFO:tasks.rados.rados.0.smithi063.stdout:3900: done (0 left) 2024-08-17T17:11:17.947 INFO:tasks.rados.rados.0.smithi063.stdout:3901: snap_create 2024-08-17T17:11:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:17 smithi063 ceph-mon[114756]: osdmap e1049: 8 total, 8 up, 8 in 2024-08-17T17:11:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:17 smithi063 ceph-mon[114756]: osdmap e1050: 8 total, 8 up, 8 in 2024-08-17T17:11:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:17 smithi063 ceph-mon[109843]: osdmap e1049: 8 total, 8 up, 8 in 2024-08-17T17:11:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:17 smithi063 ceph-mon[109843]: osdmap e1050: 8 total, 8 up, 8 in 2024-08-17T17:11:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:17 smithi162 ceph-mon[104668]: osdmap e1049: 8 total, 8 up, 8 in 2024-08-17T17:11:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:17 smithi162 ceph-mon[104668]: osdmap e1050: 8 total, 8 up, 8 in 2024-08-17T17:11:18.752 INFO:tasks.rados.rados.0.smithi063.stdout:3901: done (0 left) 2024-08-17T17:11:18.752 INFO:tasks.rados.rados.0.smithi063.stdout:3902: read oid 41 snap 487 2024-08-17T17:11:18.752 INFO:tasks.rados.rados.0.smithi063.stdout:3902: expect (ObjNum 1450 snap 485 seq_num 1450) 2024-08-17T17:11:18.752 INFO:tasks.rados.rados.0.smithi063.stdout:3903: write oid 25 current snap is 493 2024-08-17T17:11:18.753 INFO:tasks.rados.rados.0.smithi063.stdout:3903: seq_num 1478 ranges {440746=590504,1689092=697505} 2024-08-17T17:11:18.757 INFO:tasks.rados.rados.0.smithi063.stdout:3903: writing smithi06387500-25 from 440746 to 1031250 tid 1 2024-08-17T17:11:18.765 INFO:tasks.rados.rados.0.smithi063.stdout:3903: writing smithi06387500-25 from 1689092 to 2386597 tid 2 2024-08-17T17:11:18.765 INFO:tasks.rados.rados.0.smithi063.stdout:3904: snap_remove snap 483 2024-08-17T17:11:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:18 smithi063 ceph-mon[114756]: pgmap v1032: 89 pgs: 89 active+clean; 205 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2024-08-17T17:11:19.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:18 smithi063 ceph-mon[109843]: pgmap v1032: 89 pgs: 89 active+clean; 205 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2024-08-17T17:11:19.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:18 smithi162 ceph-mon[104668]: pgmap v1032: 89 pgs: 89 active+clean; 205 MiB data, 837 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.8 MiB/s wr, 15 op/s 2024-08-17T17:11:19.768 INFO:tasks.rados.rados.0.smithi063.stdout:3904: done (2 left) 2024-08-17T17:11:19.768 INFO:tasks.rados.rados.0.smithi063.stdout:3905: write oid 2 current snap is 493 2024-08-17T17:11:19.768 INFO:tasks.rados.rados.0.smithi063.stdout:3905: seq_num 1479 ranges {457553=665753,1747574=557516} 2024-08-17T17:11:19.774 INFO:tasks.rados.rados.0.smithi063.stdout:3905: writing smithi06387500-2 from 457553 to 1123306 tid 1 2024-08-17T17:11:19.781 INFO:tasks.rados.rados.0.smithi063.stdout:3905: writing smithi06387500-2 from 1747574 to 2305090 tid 2 2024-08-17T17:11:19.781 INFO:tasks.rados.rados.0.smithi063.stdout:3906: read oid 39 snap -1 2024-08-17T17:11:19.781 INFO:tasks.rados.rados.0.smithi063.stdout:3906: expect (ObjNum 1411 snap 473 seq_num 1411) 2024-08-17T17:11:19.781 INFO:tasks.rados.rados.0.smithi063.stdout:3907: snap_remove snap 459 2024-08-17T17:11:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:19 smithi063 ceph-mon[114756]: osdmap e1051: 8 total, 8 up, 8 in 2024-08-17T17:11:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:19 smithi063 ceph-mon[109843]: osdmap e1051: 8 total, 8 up, 8 in 2024-08-17T17:11:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:19 smithi162 ceph-mon[104668]: osdmap e1051: 8 total, 8 up, 8 in 2024-08-17T17:11:20.771 INFO:tasks.rados.rados.0.smithi063.stdout:3907: done (4 left) 2024-08-17T17:11:20.771 INFO:tasks.rados.rados.0.smithi063.stdout:3902: expect (ObjNum 1450 snap 485 seq_num 1450) 2024-08-17T17:11:20.887 INFO:tasks.rados.rados.0.smithi063.stdout:3903: finishing write tid 1 to smithi06387500-25 2024-08-17T17:11:20.887 INFO:tasks.rados.rados.0.smithi063.stdout:3903: finishing write tid 2 to smithi06387500-25 2024-08-17T17:11:20.887 INFO:tasks.rados.rados.0.smithi063.stdout:3903: finishing write tid 3 to smithi06387500-25 2024-08-17T17:11:20.887 INFO:tasks.rados.rados.0.smithi063.stdout:3903: finishing write tid 4 to smithi06387500-25 2024-08-17T17:11:20.887 INFO:tasks.rados.rados.0.smithi063.stdout:3903: oid 25 updating version 0 to 3813 2024-08-17T17:11:20.887 INFO:tasks.rados.rados.0.smithi063.stdout:3903: oid 25 updating version 3813 to 3814 2024-08-17T17:11:20.887 INFO:tasks.rados.rados.0.smithi063.stdout:3903: oid 25 version 3814 is already newer than 3812 2024-08-17T17:11:20.887 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 25 v 3814 (ObjNum 1478 snap 493 seq_num 1478) dirty exists 2024-08-17T17:11:20.887 INFO:tasks.rados.rados.0.smithi063.stdout:3903: left oid 25 (ObjNum 1478 snap 493 seq_num 1478) 2024-08-17T17:11:20.888 INFO:tasks.rados.rados.0.smithi063.stdout:3905: finishing write tid 1 to smithi06387500-2 2024-08-17T17:11:20.888 INFO:tasks.rados.rados.0.smithi063.stdout:3905: finishing write tid 2 to smithi06387500-2 2024-08-17T17:11:20.888 INFO:tasks.rados.rados.0.smithi063.stdout:3905: finishing write tid 3 to smithi06387500-2 2024-08-17T17:11:20.888 INFO:tasks.rados.rados.0.smithi063.stdout:3905: finishing write tid 4 to smithi06387500-2 2024-08-17T17:11:20.888 INFO:tasks.rados.rados.0.smithi063.stdout:3905: oid 2 updating version 0 to 2105 2024-08-17T17:11:20.888 INFO:tasks.rados.rados.0.smithi063.stdout:3905: oid 2 version 2105 is already newer than 2104 2024-08-17T17:11:20.888 INFO:tasks.rados.rados.0.smithi063.stdout:3905: oid 2 version 2105 is already newer than 2103 2024-08-17T17:11:20.888 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 2105 (ObjNum 1479 snap 493 seq_num 1479) dirty exists 2024-08-17T17:11:20.888 INFO:tasks.rados.rados.0.smithi063.stdout:3905: left oid 2 (ObjNum 1479 snap 493 seq_num 1479) 2024-08-17T17:11:20.888 INFO:tasks.rados.rados.0.smithi063.stdout:3908: read oid 3 snap 487 2024-08-17T17:11:20.888 INFO:tasks.rados.rados.0.smithi063.stdout:3908: expect (ObjNum 1455 snap 487 seq_num 1455) 2024-08-17T17:11:20.888 INFO:tasks.rados.rados.0.smithi063.stdout:3902: done (4 left) 2024-08-17T17:11:20.889 INFO:tasks.rados.rados.0.smithi063.stdout:3903: done (3 left) 2024-08-17T17:11:20.889 INFO:tasks.rados.rados.0.smithi063.stdout:3905: done (2 left) 2024-08-17T17:11:20.889 INFO:tasks.rados.rados.0.smithi063.stdout:3909: read oid 25 snap -1 2024-08-17T17:11:20.889 INFO:tasks.rados.rados.0.smithi063.stdout:3909: expect (ObjNum 1478 snap 493 seq_num 1478) 2024-08-17T17:11:20.889 INFO:tasks.rados.rados.0.smithi063.stdout:3910: write oid 32 current snap is 493 2024-08-17T17:11:20.889 INFO:tasks.rados.rados.0.smithi063.stdout:3910: seq_num 1480 ranges {474360=741002,1806056=701174} 2024-08-17T17:11:20.893 INFO:tasks.rados.rados.0.smithi063.stdout:3910: writing smithi06387500-32 from 474360 to 1215362 tid 1 2024-08-17T17:11:20.900 INFO:tasks.rados.rados.0.smithi063.stdout:3910: writing smithi06387500-32 from 1806056 to 2507230 tid 2 2024-08-17T17:11:20.900 INFO:tasks.rados.rados.0.smithi063.stdout:3911: delete oid 38 current snap is 493 2024-08-17T17:11:20.900 INFO:tasks.rados.rados.0.smithi063.stdout:3906: expect (ObjNum 1411 snap 473 seq_num 1411) 2024-08-17T17:11:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:20 smithi063 ceph-mon[114756]: osdmap e1052: 8 total, 8 up, 8 in 2024-08-17T17:11:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:20 smithi063 ceph-mon[114756]: pgmap v1035: 89 pgs: 89 active+clean; 207 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-08-17T17:11:21.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:20 smithi063 ceph-mon[109843]: osdmap e1052: 8 total, 8 up, 8 in 2024-08-17T17:11:21.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:20 smithi063 ceph-mon[109843]: pgmap v1035: 89 pgs: 89 active+clean; 207 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-08-17T17:11:21.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:20 smithi162 ceph-mon[104668]: osdmap e1052: 8 total, 8 up, 8 in 2024-08-17T17:11:21.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:20 smithi162 ceph-mon[104668]: pgmap v1035: 89 pgs: 89 active+clean; 207 MiB data, 842 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 1.7 MiB/s wr, 15 op/s 2024-08-17T17:11:21.126 INFO:tasks.rados.rados.0.smithi063.stdout:3910: finishing write tid 1 to smithi06387500-32 2024-08-17T17:11:21.126 INFO:tasks.rados.rados.0.smithi063.stdout:3908: expect (ObjNum 1455 snap 487 seq_num 1455) 2024-08-17T17:11:21.207 INFO:tasks.rados.rados.0.smithi063.stdout:3909: expect (ObjNum 1478 snap 493 seq_num 1478) 2024-08-17T17:11:21.359 INFO:tasks.rados.rados.0.smithi063.stdout:3910: finishing write tid 2 to smithi06387500-32 2024-08-17T17:11:21.359 INFO:tasks.rados.rados.0.smithi063.stdout:3910: finishing write tid 3 to smithi06387500-32 2024-08-17T17:11:21.359 INFO:tasks.rados.rados.0.smithi063.stdout:3910: finishing write tid 4 to smithi06387500-32 2024-08-17T17:11:21.359 INFO:tasks.rados.rados.0.smithi063.stdout:3910: oid 32 updating version 0 to 902 2024-08-17T17:11:21.359 INFO:tasks.rados.rados.0.smithi063.stdout:3910: oid 32 version 902 is already newer than 900 2024-08-17T17:11:21.359 INFO:tasks.rados.rados.0.smithi063.stdout:3910: oid 32 version 902 is already newer than 901 2024-08-17T17:11:21.359 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 32 v 902 (ObjNum 1480 snap 493 seq_num 1480) dirty exists 2024-08-17T17:11:21.359 INFO:tasks.rados.rados.0.smithi063.stdout:3910: left oid 32 (ObjNum 1480 snap 493 seq_num 1480) 2024-08-17T17:11:21.359 INFO:tasks.rados.rados.0.smithi063.stdout:3906: done (4 left) 2024-08-17T17:11:21.359 INFO:tasks.rados.rados.0.smithi063.stdout:3908: done (3 left) 2024-08-17T17:11:21.360 INFO:tasks.rados.rados.0.smithi063.stdout:3909: done (2 left) 2024-08-17T17:11:21.360 INFO:tasks.rados.rados.0.smithi063.stdout:3910: done (1 left) 2024-08-17T17:11:21.360 INFO:tasks.rados.rados.0.smithi063.stdout:3911: done (0 left) 2024-08-17T17:11:21.360 INFO:tasks.rados.rados.0.smithi063.stdout:3912: rollback oid 35 current snap is 493 2024-08-17T17:11:21.360 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 35 to 490 2024-08-17T17:11:21.360 INFO:tasks.rados.rados.0.smithi063.stdout:3913: read oid 23 snap -1 2024-08-17T17:11:21.360 INFO:tasks.rados.rados.0.smithi063.stdout:3913: expect (ObjNum 1466 snap 490 seq_num 1466) 2024-08-17T17:11:21.360 INFO:tasks.rados.rados.0.smithi063.stdout:3914: snap_remove snap 488 2024-08-17T17:11:21.772 INFO:tasks.rados.rados.0.smithi063.stdout:3912: finishing rollback tid 0 to smithi06387500-35 2024-08-17T17:11:21.772 INFO:tasks.rados.rados.0.smithi063.stdout:3912: finishing rollback tid 1 to smithi06387500-35 2024-08-17T17:11:21.772 INFO:tasks.rados.rados.0.smithi063.stdout:3912: finishing rollback tid 2 to smithi06387500-35 2024-08-17T17:11:21.772 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 35 v 3828 (ObjNum 1424 snap 476 seq_num 1424) dirty exists 2024-08-17T17:11:21.772 INFO:tasks.rados.rados.0.smithi063.stdout:3913: expect (ObjNum 1466 snap 490 seq_num 1466) 2024-08-17T17:11:21.912 INFO:tasks.rados.rados.0.smithi063.stdout:3912: done (2 left) 2024-08-17T17:11:21.913 INFO:tasks.rados.rados.0.smithi063.stdout:3913: done (1 left) 2024-08-17T17:11:21.913 INFO:tasks.rados.rados.0.smithi063.stdout:3914: done (0 left) 2024-08-17T17:11:21.913 INFO:tasks.rados.rados.0.smithi063.stdout:3915: snap_remove snap 473 2024-08-17T17:11:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:21 smithi063 ceph-mon[114756]: osdmap e1053: 8 total, 8 up, 8 in 2024-08-17T17:11:22.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:21 smithi063 ceph-mon[109843]: osdmap e1053: 8 total, 8 up, 8 in 2024-08-17T17:11:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:21 smithi162 ceph-mon[104668]: osdmap e1053: 8 total, 8 up, 8 in 2024-08-17T17:11:22.697 INFO:tasks.rados.rados.0.smithi063.stdout:3915: done (0 left) 2024-08-17T17:11:22.697 INFO:tasks.rados.rados.0.smithi063.stdout:3916: snap_remove snap 475 2024-08-17T17:11:23.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:22 smithi162 ceph-mon[104668]: osdmap e1054: 8 total, 8 up, 8 in 2024-08-17T17:11:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:22 smithi162 ceph-mon[104668]: pgmap v1038: 89 pgs: 89 active+clean; 198 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.4 MiB/s wr, 31 op/s 2024-08-17T17:11:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:22 smithi162 ceph-mon[104668]: osdmap e1055: 8 total, 8 up, 8 in 2024-08-17T17:11:23.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:22 smithi063 ceph-mon[114756]: osdmap e1054: 8 total, 8 up, 8 in 2024-08-17T17:11:23.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:22 smithi063 ceph-mon[114756]: pgmap v1038: 89 pgs: 89 active+clean; 198 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.4 MiB/s wr, 31 op/s 2024-08-17T17:11:23.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:22 smithi063 ceph-mon[114756]: osdmap e1055: 8 total, 8 up, 8 in 2024-08-17T17:11:23.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:22 smithi063 ceph-mon[109843]: osdmap e1054: 8 total, 8 up, 8 in 2024-08-17T17:11:23.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:22 smithi063 ceph-mon[109843]: pgmap v1038: 89 pgs: 89 active+clean; 198 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.4 MiB/s wr, 31 op/s 2024-08-17T17:11:23.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:22 smithi063 ceph-mon[109843]: osdmap e1055: 8 total, 8 up, 8 in 2024-08-17T17:11:23.684 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:11:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:11:23] "GET /metrics HTTP/1.1" 200 36333 "" "Prometheus/2.51.0" 2024-08-17T17:11:23.697 INFO:tasks.rados.rados.0.smithi063.stdout:3916: done (0 left) 2024-08-17T17:11:23.697 INFO:tasks.rados.rados.0.smithi063.stdout:3917: rollback oid 21 current snap is 493 2024-08-17T17:11:23.697 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 21 to 489 2024-08-17T17:11:23.697 INFO:tasks.rados.rados.0.smithi063.stdout:3918: snap_remove snap 482 2024-08-17T17:11:24.702 INFO:tasks.rados.rados.0.smithi063.stdout:3918: done (1 left) 2024-08-17T17:11:24.702 INFO:tasks.rados.rados.0.smithi063.stdout:3919: read oid 50 snap 481 2024-08-17T17:11:24.702 INFO:tasks.rados.rados.0.smithi063.stdout:3919: expect (ObjNum 1088 snap 363 seq_num 1088) 2024-08-17T17:11:24.702 INFO:tasks.rados.rados.0.smithi063.stdout:3920: snap_create 2024-08-17T17:11:24.702 INFO:tasks.rados.rados.0.smithi063.stdout:3917: finishing rollback tid 1 to smithi06387500-21 2024-08-17T17:11:24.702 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 0 (ObjNum 33 snap 0 seq_num 1786890032) dirty dne 2024-08-17T17:11:24.711 INFO:tasks.rados.rados.0.smithi063.stdout:3919: expect (ObjNum 1088 snap 363 seq_num 1088) 2024-08-17T17:11:25.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:24 smithi063 ceph-mon[109843]: osdmap e1056: 8 total, 8 up, 8 in 2024-08-17T17:11:25.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:24 smithi063 ceph-mon[114756]: osdmap e1056: 8 total, 8 up, 8 in 2024-08-17T17:11:25.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:24 smithi162 ceph-mon[104668]: osdmap e1056: 8 total, 8 up, 8 in 2024-08-17T17:11:25.712 INFO:tasks.rados.rados.0.smithi063.stdout:3917: done (2 left) 2024-08-17T17:11:25.712 INFO:tasks.rados.rados.0.smithi063.stdout:3919: done (1 left) 2024-08-17T17:11:25.712 INFO:tasks.rados.rados.0.smithi063.stdout:3920: done (0 left) 2024-08-17T17:11:25.712 INFO:tasks.rados.rados.0.smithi063.stdout:3921: snap_remove snap 487 2024-08-17T17:11:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:25 smithi063 ceph-mon[114756]: pgmap v1041: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 196 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.8 MiB/s wr, 32 op/s 2024-08-17T17:11:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:25 smithi063 ceph-mon[114756]: osdmap e1057: 8 total, 8 up, 8 in 2024-08-17T17:11:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:25 smithi063 ceph-mon[109843]: pgmap v1041: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 196 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.8 MiB/s wr, 32 op/s 2024-08-17T17:11:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:25 smithi063 ceph-mon[109843]: osdmap e1057: 8 total, 8 up, 8 in 2024-08-17T17:11:26.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:25 smithi162 ceph-mon[104668]: pgmap v1041: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 196 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 2.8 MiB/s wr, 32 op/s 2024-08-17T17:11:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:25 smithi162 ceph-mon[104668]: osdmap e1057: 8 total, 8 up, 8 in 2024-08-17T17:11:26.718 INFO:tasks.rados.rados.0.smithi063.stdout:3921: done (0 left) 2024-08-17T17:11:26.718 INFO:tasks.rados.rados.0.smithi063.stdout:3922: write oid 9 current snap is 494 2024-08-17T17:11:26.718 INFO:tasks.rados.rados.0.smithi063.stdout:3922: seq_num 1482 ranges {507974=491500,1406667=704843,2548287=359687} 2024-08-17T17:11:26.725 INFO:tasks.rados.rados.0.smithi063.stdout:3922: writing smithi06387500-9 from 507974 to 999474 tid 1 2024-08-17T17:11:26.730 INFO:tasks.rados.rados.0.smithi063.stdout:3922: writing smithi06387500-9 from 1406667 to 2111510 tid 2 2024-08-17T17:11:26.735 INFO:tasks.rados.rados.0.smithi063.stdout:3922: writing smithi06387500-9 from 2548287 to 2907974 tid 3 2024-08-17T17:11:26.735 INFO:tasks.rados.rados.0.smithi063.stdout:3923: rollback oid 5 current snap is 494 2024-08-17T17:11:26.735 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 5 to 486 2024-08-17T17:11:26.736 INFO:tasks.rados.rados.0.smithi063.stdout:3922: finishing write tid 1 to smithi06387500-9 2024-08-17T17:11:26.736 INFO:tasks.rados.rados.0.smithi063.stdout:3922: finishing write tid 2 to smithi06387500-9 2024-08-17T17:11:26.736 INFO:tasks.rados.rados.0.smithi063.stdout:3924: delete oid 39 current snap is 494 2024-08-17T17:11:26.739 INFO:tasks.rados.rados.0.smithi063.stdout:3924: done (2 left) 2024-08-17T17:11:26.739 INFO:tasks.rados.rados.0.smithi063.stdout:3925: snap_remove snap 481 2024-08-17T17:11:27.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:26 smithi063 ceph-mon[114756]: osdmap e1058: 8 total, 8 up, 8 in 2024-08-17T17:11:27.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:26 smithi063 ceph-mon[114756]: pgmap v1044: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 172 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 676 KiB/s wr, 5 op/s 2024-08-17T17:11:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:26 smithi063 ceph-mon[109843]: osdmap e1058: 8 total, 8 up, 8 in 2024-08-17T17:11:27.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:26 smithi063 ceph-mon[109843]: pgmap v1044: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 172 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 676 KiB/s wr, 5 op/s 2024-08-17T17:11:27.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:26 smithi162 ceph-mon[104668]: osdmap e1058: 8 total, 8 up, 8 in 2024-08-17T17:11:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:26 smithi162 ceph-mon[104668]: pgmap v1044: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 172 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 676 KiB/s wr, 5 op/s 2024-08-17T17:11:27.694 INFO:tasks.rados.rados.0.smithi063.stdout:3925: done (2 left) 2024-08-17T17:11:27.695 INFO:tasks.rados.rados.0.smithi063.stdout:3926: snap_create 2024-08-17T17:11:27.695 INFO:tasks.rados.rados.0.smithi063.stdout:3923: finishing rollback tid 0 to smithi06387500-5 2024-08-17T17:11:27.695 INFO:tasks.rados.rados.0.smithi063.stdout:3923: finishing rollback tid 1 to smithi06387500-5 2024-08-17T17:11:27.695 INFO:tasks.rados.rados.0.smithi063.stdout:3923: finishing rollback tid 2 to smithi06387500-5 2024-08-17T17:11:27.695 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 5 v 2149 (ObjNum 1293 snap 441 seq_num 1293) dirty exists 2024-08-17T17:11:27.695 INFO:tasks.rados.rados.0.smithi063.stdout:3922: finishing write tid 3 to smithi06387500-9 2024-08-17T17:11:27.695 INFO:tasks.rados.rados.0.smithi063.stdout:3922: finishing write tid 4 to smithi06387500-9 2024-08-17T17:11:27.695 INFO:tasks.rados.rados.0.smithi063.stdout:3922: finishing write tid 5 to smithi06387500-9 2024-08-17T17:11:27.695 INFO:tasks.rados.rados.0.smithi063.stdout:3922: oid 9 updating version 0 to 1960 2024-08-17T17:11:27.695 INFO:tasks.rados.rados.0.smithi063.stdout:3922: oid 9 updating version 1960 to 1963 2024-08-17T17:11:27.695 INFO:tasks.rados.rados.0.smithi063.stdout:3922: oid 9 version 1963 is already newer than 1962 2024-08-17T17:11:27.695 INFO:tasks.rados.rados.0.smithi063.stdout:3922: oid 9 version 1963 is already newer than 1961 2024-08-17T17:11:27.696 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 9 v 1963 (ObjNum 1482 snap 494 seq_num 1482) dirty exists 2024-08-17T17:11:27.696 INFO:tasks.rados.rados.0.smithi063.stdout:3922: left oid 9 (ObjNum 1482 snap 494 seq_num 1482) 2024-08-17T17:11:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:27 smithi063 ceph-mon[114756]: osdmap e1059: 8 total, 8 up, 8 in 2024-08-17T17:11:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:27 smithi063 ceph-mon[114756]: osdmap e1060: 8 total, 8 up, 8 in 2024-08-17T17:11:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:27 smithi063 ceph-mon[109843]: osdmap e1059: 8 total, 8 up, 8 in 2024-08-17T17:11:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:27 smithi063 ceph-mon[109843]: osdmap e1060: 8 total, 8 up, 8 in 2024-08-17T17:11:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:27 smithi162 ceph-mon[104668]: osdmap e1059: 8 total, 8 up, 8 in 2024-08-17T17:11:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:27 smithi162 ceph-mon[104668]: osdmap e1060: 8 total, 8 up, 8 in 2024-08-17T17:11:28.696 INFO:tasks.rados.rados.0.smithi063.stdout:3922: done (2 left) 2024-08-17T17:11:28.696 INFO:tasks.rados.rados.0.smithi063.stdout:3923: done (1 left) 2024-08-17T17:11:28.696 INFO:tasks.rados.rados.0.smithi063.stdout:3926: done (0 left) 2024-08-17T17:11:28.696 INFO:tasks.rados.rados.0.smithi063.stdout:3927: write oid 33 current snap is 495 2024-08-17T17:11:28.696 INFO:tasks.rados.rados.0.smithi063.stdout:3927: seq_num 1484 ranges {541588=758351,2039984=708512} 2024-08-17T17:11:28.703 INFO:tasks.rados.rados.0.smithi063.stdout:3927: writing smithi06387500-33 from 541588 to 1299939 tid 1 2024-08-17T17:11:28.712 INFO:tasks.rados.rados.0.smithi063.stdout:3927: writing smithi06387500-33 from 2039984 to 2748496 tid 2 2024-08-17T17:11:28.712 INFO:tasks.rados.rados.0.smithi063.stdout:3928: snap_create 2024-08-17T17:11:28.712 INFO:tasks.rados.rados.0.smithi063.stdout:3927: finishing write tid 1 to smithi06387500-33 2024-08-17T17:11:28.715 INFO:tasks.rados.rados.0.smithi063.stdout:3927: finishing write tid 2 to smithi06387500-33 2024-08-17T17:11:28.715 INFO:tasks.rados.rados.0.smithi063.stdout:3927: finishing write tid 3 to smithi06387500-33 2024-08-17T17:11:28.716 INFO:tasks.rados.rados.0.smithi063.stdout:3927: finishing write tid 4 to smithi06387500-33 2024-08-17T17:11:28.716 INFO:tasks.rados.rados.0.smithi063.stdout:3927: oid 33 updating version 0 to 1321 2024-08-17T17:11:28.716 INFO:tasks.rados.rados.0.smithi063.stdout:3927: oid 33 updating version 1321 to 1322 2024-08-17T17:11:28.716 INFO:tasks.rados.rados.0.smithi063.stdout:3927: oid 33 version 1322 is already newer than 1320 2024-08-17T17:11:28.716 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 1322 (ObjNum 1484 snap 495 seq_num 1484) dirty exists 2024-08-17T17:11:28.716 INFO:tasks.rados.rados.0.smithi063.stdout:3927: left oid 33 (ObjNum 1484 snap 495 seq_num 1484) 2024-08-17T17:11:29.709 INFO:tasks.rados.rados.0.smithi063.stdout:3927: done (1 left) 2024-08-17T17:11:29.709 INFO:tasks.rados.rados.0.smithi063.stdout:3928: done (0 left) 2024-08-17T17:11:29.710 INFO:tasks.rados.rados.0.smithi063.stdout:3929: read oid 13 snap -1 2024-08-17T17:11:29.710 INFO:tasks.rados.rados.0.smithi063.stdout:3929: expect (ObjNum 1475 snap 492 seq_num 1475) 2024-08-17T17:11:29.710 INFO:tasks.rados.rados.0.smithi063.stdout:3930: write oid 45 current snap is 496 2024-08-17T17:11:29.710 INFO:tasks.rados.rados.0.smithi063.stdout:3930: seq_num 1485 ranges {558395=433600,1698466=568523,2863262=95133} 2024-08-17T17:11:29.714 INFO:tasks.rados.rados.0.smithi063.stdout:3930: writing smithi06387500-45 from 558395 to 991995 tid 1 2024-08-17T17:11:29.721 INFO:tasks.rados.rados.0.smithi063.stdout:3930: writing smithi06387500-45 from 1698466 to 2266989 tid 2 2024-08-17T17:11:29.725 INFO:tasks.rados.rados.0.smithi063.stdout:3930: writing smithi06387500-45 from 2863262 to 2958395 tid 3 2024-08-17T17:11:29.725 INFO:tasks.rados.rados.0.smithi063.stdout:3931: snap_remove snap 486 2024-08-17T17:11:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:29 smithi063 ceph-mon[109843]: pgmap v1047: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 171 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 380 KiB/s wr, 4 op/s 2024-08-17T17:11:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:29 smithi063 ceph-mon[109843]: osdmap e1061: 8 total, 8 up, 8 in 2024-08-17T17:11:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:29 smithi063 ceph-mon[114756]: pgmap v1047: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 171 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 380 KiB/s wr, 4 op/s 2024-08-17T17:11:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:29 smithi063 ceph-mon[114756]: osdmap e1061: 8 total, 8 up, 8 in 2024-08-17T17:11:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:29 smithi162 ceph-mon[104668]: pgmap v1047: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 171 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 380 KiB/s wr, 4 op/s 2024-08-17T17:11:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:29 smithi162 ceph-mon[104668]: osdmap e1061: 8 total, 8 up, 8 in 2024-08-17T17:11:30.717 INFO:tasks.rados.rados.0.smithi063.stdout:3931: done (2 left) 2024-08-17T17:11:30.718 INFO:tasks.rados.rados.0.smithi063.stdout:3932: write oid 22 current snap is 496 2024-08-17T17:11:30.718 INFO:tasks.rados.rados.0.smithi063.stdout:3932: seq_num 1486 ranges {575202=508849,1640595=712181} 2024-08-17T17:11:30.724 INFO:tasks.rados.rados.0.smithi063.stdout:3932: writing smithi06387500-22 from 575202 to 1084051 tid 1 2024-08-17T17:11:30.732 INFO:tasks.rados.rados.0.smithi063.stdout:3932: writing smithi06387500-22 from 1640595 to 2352776 tid 2 2024-08-17T17:11:30.732 INFO:tasks.rados.rados.0.smithi063.stdout:3933: rollback oid 30 current snap is 496 2024-08-17T17:11:30.732 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 30 to 480 2024-08-17T17:11:30.732 INFO:tasks.rados.rados.0.smithi063.stdout:3929: expect (ObjNum 1475 snap 492 seq_num 1475) 2024-08-17T17:11:30.986 INFO:tasks.rados.rados.0.smithi063.stdout:3930: finishing write tid 1 to smithi06387500-45 2024-08-17T17:11:30.986 INFO:tasks.rados.rados.0.smithi063.stdout:3930: finishing write tid 2 to smithi06387500-45 2024-08-17T17:11:30.986 INFO:tasks.rados.rados.0.smithi063.stdout:3929: done (3 left) 2024-08-17T17:11:30.986 INFO:tasks.rados.rados.0.smithi063.stdout:3934: snap_create 2024-08-17T17:11:30.986 INFO:tasks.rados.rados.0.smithi063.stdout:3930: finishing write tid 3 to smithi06387500-45 2024-08-17T17:11:30.986 INFO:tasks.rados.rados.0.smithi063.stdout:3930: finishing write tid 4 to smithi06387500-45 2024-08-17T17:11:30.986 INFO:tasks.rados.rados.0.smithi063.stdout:3930: finishing write tid 5 to smithi06387500-45 2024-08-17T17:11:30.986 INFO:tasks.rados.rados.0.smithi063.stdout:3930: oid 45 updating version 0 to 1012 2024-08-17T17:11:30.986 INFO:tasks.rados.rados.0.smithi063.stdout:3930: oid 45 updating version 1012 to 1013 2024-08-17T17:11:30.986 INFO:tasks.rados.rados.0.smithi063.stdout:3930: oid 45 updating version 1013 to 1014 2024-08-17T17:11:30.987 INFO:tasks.rados.rados.0.smithi063.stdout:3930: oid 45 version 1014 is already newer than 1011 2024-08-17T17:11:30.987 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 45 v 1014 (ObjNum 1485 snap 496 seq_num 1485) dirty exists 2024-08-17T17:11:30.987 INFO:tasks.rados.rados.0.smithi063.stdout:3930: left oid 45 (ObjNum 1485 snap 496 seq_num 1485) 2024-08-17T17:11:30.987 INFO:tasks.rados.rados.0.smithi063.stdout:3932: finishing write tid 1 to smithi06387500-22 2024-08-17T17:11:30.987 INFO:tasks.rados.rados.0.smithi063.stdout:3932: finishing write tid 2 to smithi06387500-22 2024-08-17T17:11:30.987 INFO:tasks.rados.rados.0.smithi063.stdout:3932: finishing write tid 3 to smithi06387500-22 2024-08-17T17:11:30.987 INFO:tasks.rados.rados.0.smithi063.stdout:3933: finishing rollback tid 0 to smithi06387500-30 2024-08-17T17:11:30.987 INFO:tasks.rados.rados.0.smithi063.stdout:3932: finishing write tid 4 to smithi06387500-22 2024-08-17T17:11:30.987 INFO:tasks.rados.rados.0.smithi063.stdout:3932: oid 22 updating version 0 to 1015 2024-08-17T17:11:30.987 INFO:tasks.rados.rados.0.smithi063.stdout:3932: oid 22 updating version 1015 to 1016 2024-08-17T17:11:30.987 INFO:tasks.rados.rados.0.smithi063.stdout:3932: oid 22 version 1016 is already newer than 1014 2024-08-17T17:11:30.987 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 22 v 1016 (ObjNum 1486 snap 496 seq_num 1486) dirty exists 2024-08-17T17:11:30.987 INFO:tasks.rados.rados.0.smithi063.stdout:3932: left oid 22 (ObjNum 1486 snap 496 seq_num 1486) 2024-08-17T17:11:30.988 INFO:tasks.rados.rados.0.smithi063.stdout:3933: finishing rollback tid 1 to smithi06387500-30 2024-08-17T17:11:30.988 INFO:tasks.rados.rados.0.smithi063.stdout:3933: finishing rollback tid 2 to smithi06387500-30 2024-08-17T17:11:30.988 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1971 (ObjNum 1418 snap 474 seq_num 1418) dirty exists 2024-08-17T17:11:30.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:30 smithi162 ceph-mon[104668]: osdmap e1062: 8 total, 8 up, 8 in 2024-08-17T17:11:30.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:30 smithi162 ceph-mon[104668]: pgmap v1050: 89 pgs: 89 active+clean; 171 MiB data, 763 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 380 KiB/s wr, 2 op/s 2024-08-17T17:11:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:30 smithi063 ceph-mon[114756]: osdmap e1062: 8 total, 8 up, 8 in 2024-08-17T17:11:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:30 smithi063 ceph-mon[114756]: pgmap v1050: 89 pgs: 89 active+clean; 171 MiB data, 763 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 380 KiB/s wr, 2 op/s 2024-08-17T17:11:31.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:30 smithi063 ceph-mon[109843]: osdmap e1062: 8 total, 8 up, 8 in 2024-08-17T17:11:31.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:30 smithi063 ceph-mon[109843]: pgmap v1050: 89 pgs: 89 active+clean; 171 MiB data, 763 MiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 380 KiB/s wr, 2 op/s 2024-08-17T17:11:31.718 INFO:tasks.rados.rados.0.smithi063.stdout:3930: done (3 left) 2024-08-17T17:11:31.718 INFO:tasks.rados.rados.0.smithi063.stdout:3932: done (2 left) 2024-08-17T17:11:31.718 INFO:tasks.rados.rados.0.smithi063.stdout:3933: done (1 left) 2024-08-17T17:11:31.718 INFO:tasks.rados.rados.0.smithi063.stdout:3934: done (0 left) 2024-08-17T17:11:31.718 INFO:tasks.rados.rados.0.smithi063.stdout:3935: write oid 20 current snap is 497 2024-08-17T17:11:31.719 INFO:tasks.rados.rados.0.smithi063.stdout:3935: seq_num 1487 ranges {592009=584098,1699077=572192,2801755=190254} 2024-08-17T17:11:31.724 INFO:tasks.rados.rados.0.smithi063.stdout:3935: writing smithi06387500-20 from 592009 to 1176107 tid 1 2024-08-17T17:11:31.731 INFO:tasks.rados.rados.0.smithi063.stdout:3935: writing smithi06387500-20 from 1699077 to 2271269 tid 2 2024-08-17T17:11:31.735 INFO:tasks.rados.rados.0.smithi063.stdout:3935: writing smithi06387500-20 from 2801755 to 2992009 tid 3 2024-08-17T17:11:31.735 INFO:tasks.rados.rados.0.smithi063.stdout:3936: write oid 34 current snap is 497 2024-08-17T17:11:31.735 INFO:tasks.rados.rados.0.smithi063.stdout:3935: finishing write tid 1 to smithi06387500-20 2024-08-17T17:11:31.735 INFO:tasks.rados.rados.0.smithi063.stdout:3936: seq_num 1488 ranges {608816=659347,1757559=715850} 2024-08-17T17:11:31.742 INFO:tasks.rados.rados.0.smithi063.stdout:3936: writing smithi06387500-34 from 608816 to 1268163 tid 1 2024-08-17T17:11:31.749 INFO:tasks.rados.rados.0.smithi063.stdout:3936: writing smithi06387500-34 from 1757559 to 2473409 tid 2 2024-08-17T17:11:31.750 INFO:tasks.rados.rados.0.smithi063.stdout:3937: read oid 46 snap -1 2024-08-17T17:11:31.750 INFO:tasks.rados.rados.0.smithi063.stdout:3937: expect (ObjNum 1461 snap 490 seq_num 1461) 2024-08-17T17:11:31.750 INFO:tasks.rados.rados.0.smithi063.stdout:3938: snap_create 2024-08-17T17:11:31.750 INFO:tasks.rados.rados.0.smithi063.stdout:3935: finishing write tid 2 to smithi06387500-20 2024-08-17T17:11:31.750 INFO:tasks.rados.rados.0.smithi063.stdout:3935: finishing write tid 3 to smithi06387500-20 2024-08-17T17:11:31.750 INFO:tasks.rados.rados.0.smithi063.stdout:3935: finishing write tid 4 to smithi06387500-20 2024-08-17T17:11:31.750 INFO:tasks.rados.rados.0.smithi063.stdout:3935: finishing write tid 5 to smithi06387500-20 2024-08-17T17:11:31.750 INFO:tasks.rados.rados.0.smithi063.stdout:3935: oid 20 updating version 0 to 1021 2024-08-17T17:11:31.750 INFO:tasks.rados.rados.0.smithi063.stdout:3935: oid 20 version 1021 is already newer than 1020 2024-08-17T17:11:31.750 INFO:tasks.rados.rados.0.smithi063.stdout:3935: oid 20 version 1021 is already newer than 1019 2024-08-17T17:11:31.750 INFO:tasks.rados.rados.0.smithi063.stdout:3935: oid 20 version 1021 is already newer than 1018 2024-08-17T17:11:31.751 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 20 v 1021 (ObjNum 1487 snap 497 seq_num 1487) dirty exists 2024-08-17T17:11:31.751 INFO:tasks.rados.rados.0.smithi063.stdout:3935: left oid 20 (ObjNum 1487 snap 497 seq_num 1487) 2024-08-17T17:11:31.751 INFO:tasks.rados.rados.0.smithi063.stdout:3936: finishing write tid 1 to smithi06387500-34 2024-08-17T17:11:31.754 INFO:tasks.rados.rados.0.smithi063.stdout:3937: expect (ObjNum 1461 snap 490 seq_num 1461) 2024-08-17T17:11:31.837 INFO:tasks.rados.rados.0.smithi063.stdout:3936: finishing write tid 2 to smithi06387500-34 2024-08-17T17:11:31.837 INFO:tasks.rados.rados.0.smithi063.stdout:3936: finishing write tid 3 to smithi06387500-34 2024-08-17T17:11:31.837 INFO:tasks.rados.rados.0.smithi063.stdout:3936: finishing write tid 4 to smithi06387500-34 2024-08-17T17:11:31.837 INFO:tasks.rados.rados.0.smithi063.stdout:3936: oid 34 updating version 0 to 2457 2024-08-17T17:11:31.837 INFO:tasks.rados.rados.0.smithi063.stdout:3936: oid 34 updating version 2457 to 2458 2024-08-17T17:11:31.837 INFO:tasks.rados.rados.0.smithi063.stdout:3936: oid 34 version 2458 is already newer than 2456 2024-08-17T17:11:31.838 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 34 v 2458 (ObjNum 1488 snap 497 seq_num 1488) dirty exists 2024-08-17T17:11:31.838 INFO:tasks.rados.rados.0.smithi063.stdout:3936: left oid 34 (ObjNum 1488 snap 497 seq_num 1488) 2024-08-17T17:11:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:31 smithi063 ceph-mon[114756]: osdmap e1063: 8 total, 8 up, 8 in 2024-08-17T17:11:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:32.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:31 smithi063 ceph-mon[109843]: osdmap e1063: 8 total, 8 up, 8 in 2024-08-17T17:11:32.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:31 smithi162 ceph-mon[104668]: osdmap e1063: 8 total, 8 up, 8 in 2024-08-17T17:11:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:32.696 INFO:tasks.rados.rados.0.smithi063.stdout:3935: done (3 left) 2024-08-17T17:11:32.696 INFO:tasks.rados.rados.0.smithi063.stdout:3936: done (2 left) 2024-08-17T17:11:32.696 INFO:tasks.rados.rados.0.smithi063.stdout:3937: done (1 left) 2024-08-17T17:11:32.696 INFO:tasks.rados.rados.0.smithi063.stdout:3938: done (0 left) 2024-08-17T17:11:32.696 INFO:tasks.rados.rados.0.smithi063.stdout:3939: snap_create 2024-08-17T17:11:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:32 smithi063 ceph-mon[109843]: osdmap e1064: 8 total, 8 up, 8 in 2024-08-17T17:11:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:32 smithi063 ceph-mon[109843]: pgmap v1053: 89 pgs: 89 active+clean; 176 MiB data, 790 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 268 KiB/s wr, 3 op/s 2024-08-17T17:11:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:32 smithi063 ceph-mon[109843]: osdmap e1065: 8 total, 8 up, 8 in 2024-08-17T17:11:33.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:32 smithi063 ceph-mon[114756]: osdmap e1064: 8 total, 8 up, 8 in 2024-08-17T17:11:33.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:32 smithi063 ceph-mon[114756]: pgmap v1053: 89 pgs: 89 active+clean; 176 MiB data, 790 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 268 KiB/s wr, 3 op/s 2024-08-17T17:11:33.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:32 smithi063 ceph-mon[114756]: osdmap e1065: 8 total, 8 up, 8 in 2024-08-17T17:11:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:32 smithi162 ceph-mon[104668]: osdmap e1064: 8 total, 8 up, 8 in 2024-08-17T17:11:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:32 smithi162 ceph-mon[104668]: pgmap v1053: 89 pgs: 89 active+clean; 176 MiB data, 790 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 268 KiB/s wr, 3 op/s 2024-08-17T17:11:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:32 smithi162 ceph-mon[104668]: osdmap e1065: 8 total, 8 up, 8 in 2024-08-17T17:11:33.699 INFO:tasks.rados.rados.0.smithi063.stdout:3939: done (0 left) 2024-08-17T17:11:33.699 INFO:tasks.rados.rados.0.smithi063.stdout:3940: snap_remove snap 498 2024-08-17T17:11:33.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:11:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:11:33] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.51.0" 2024-08-17T17:11:34.704 INFO:tasks.rados.rados.0.smithi063.stdout:3940: done (0 left) 2024-08-17T17:11:34.704 INFO:tasks.rados.rados.0.smithi063.stdout:3941: write oid 6 current snap is 499 2024-08-17T17:11:34.704 INFO:tasks.rados.rados.0.smithi063.stdout:3941: seq_num 1489 ranges {625623=734596,1816041=459508,2740248=285375} 2024-08-17T17:11:34.711 INFO:tasks.rados.rados.0.smithi063.stdout:3941: writing smithi06387500-6 from 625623 to 1360219 tid 1 2024-08-17T17:11:34.716 INFO:tasks.rados.rados.0.smithi063.stdout:3941: writing smithi06387500-6 from 1816041 to 2275549 tid 2 2024-08-17T17:11:34.721 INFO:tasks.rados.rados.0.smithi063.stdout:3941: writing smithi06387500-6 from 2740248 to 3025623 tid 3 2024-08-17T17:11:34.721 INFO:tasks.rados.rados.0.smithi063.stdout:3941: finishing write tid 1 to smithi06387500-6 2024-08-17T17:11:34.721 INFO:tasks.rados.rados.0.smithi063.stdout:3941: finishing write tid 2 to smithi06387500-6 2024-08-17T17:11:34.721 INFO:tasks.rados.rados.0.smithi063.stdout:3942: write oid 2 current snap is 499 2024-08-17T17:11:34.721 INFO:tasks.rados.rados.0.smithi063.stdout:3942: seq_num 1490 ranges {642430=409845,1758170=719519} 2024-08-17T17:11:34.727 INFO:tasks.rados.rados.0.smithi063.stdout:3942: writing smithi06387500-2 from 642430 to 1052275 tid 1 2024-08-17T17:11:34.735 INFO:tasks.rados.rados.0.smithi063.stdout:3942: writing smithi06387500-2 from 1758170 to 2477689 tid 2 2024-08-17T17:11:34.735 INFO:tasks.rados.rados.0.smithi063.stdout:3943: snap_remove snap 497 2024-08-17T17:11:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:34 smithi063 ceph-mon[109843]: osdmap e1066: 8 total, 8 up, 8 in 2024-08-17T17:11:35.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:34 smithi063 ceph-mon[114756]: osdmap e1066: 8 total, 8 up, 8 in 2024-08-17T17:11:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:34 smithi162 ceph-mon[104668]: osdmap e1066: 8 total, 8 up, 8 in 2024-08-17T17:11:35.717 INFO:tasks.rados.rados.0.smithi063.stdout:3943: done (2 left) 2024-08-17T17:11:35.717 INFO:tasks.rados.rados.0.smithi063.stdout:3944: snap_remove snap 494 2024-08-17T17:11:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:35 smithi063 ceph-mon[114756]: pgmap v1056: 89 pgs: 89 active+clean; 178 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 626 KiB/s wr, 5 op/s 2024-08-17T17:11:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:35 smithi063 ceph-mon[114756]: osdmap e1067: 8 total, 8 up, 8 in 2024-08-17T17:11:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:35 smithi063 ceph-mon[109843]: pgmap v1056: 89 pgs: 89 active+clean; 178 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 626 KiB/s wr, 5 op/s 2024-08-17T17:11:36.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:35 smithi063 ceph-mon[109843]: osdmap e1067: 8 total, 8 up, 8 in 2024-08-17T17:11:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:35 smithi162 ceph-mon[104668]: pgmap v1056: 89 pgs: 89 active+clean; 178 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 626 KiB/s wr, 5 op/s 2024-08-17T17:11:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:35 smithi162 ceph-mon[104668]: osdmap e1067: 8 total, 8 up, 8 in 2024-08-17T17:11:36.720 INFO:tasks.rados.rados.0.smithi063.stdout:3944: done (2 left) 2024-08-17T17:11:36.720 INFO:tasks.rados.rados.0.smithi063.stdout:3945: snap_remove snap 476 2024-08-17T17:11:37.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:36 smithi063 ceph-mon[114756]: osdmap e1068: 8 total, 8 up, 8 in 2024-08-17T17:11:37.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:36 smithi063 ceph-mon[114756]: pgmap v1059: 89 pgs: 89 active+clean; 184 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-08-17T17:11:37.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:36 smithi063 ceph-mon[109843]: osdmap e1068: 8 total, 8 up, 8 in 2024-08-17T17:11:37.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:36 smithi063 ceph-mon[109843]: pgmap v1059: 89 pgs: 89 active+clean; 184 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-08-17T17:11:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:36 smithi162 ceph-mon[104668]: osdmap e1068: 8 total, 8 up, 8 in 2024-08-17T17:11:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:36 smithi162 ceph-mon[104668]: pgmap v1059: 89 pgs: 89 active+clean; 184 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-08-17T17:11:37.699 INFO:tasks.rados.rados.0.smithi063.stdout:3945: done (2 left) 2024-08-17T17:11:37.699 INFO:tasks.rados.rados.0.smithi063.stdout:3946: delete oid 16 current snap is 499 2024-08-17T17:11:37.699 INFO:tasks.rados.rados.0.smithi063.stdout:3941: finishing write tid 3 to smithi06387500-6 2024-08-17T17:11:37.699 INFO:tasks.rados.rados.0.smithi063.stdout:3941: finishing write tid 4 to smithi06387500-6 2024-08-17T17:11:37.700 INFO:tasks.rados.rados.0.smithi063.stdout:3941: finishing write tid 5 to smithi06387500-6 2024-08-17T17:11:37.700 INFO:tasks.rados.rados.0.smithi063.stdout:3941: oid 6 updating version 0 to 1328 2024-08-17T17:11:37.700 INFO:tasks.rados.rados.0.smithi063.stdout:3941: oid 6 version 1328 is already newer than 1327 2024-08-17T17:11:37.700 INFO:tasks.rados.rados.0.smithi063.stdout:3941: oid 6 updating version 1328 to 1329 2024-08-17T17:11:37.700 INFO:tasks.rados.rados.0.smithi063.stdout:3941: oid 6 version 1329 is already newer than 1326 2024-08-17T17:11:37.700 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 6 v 1329 (ObjNum 1489 snap 499 seq_num 1489) dirty exists 2024-08-17T17:11:37.700 INFO:tasks.rados.rados.0.smithi063.stdout:3941: left oid 6 (ObjNum 1489 snap 499 seq_num 1489) 2024-08-17T17:11:37.700 INFO:tasks.rados.rados.0.smithi063.stdout:3942: finishing write tid 1 to smithi06387500-2 2024-08-17T17:11:37.700 INFO:tasks.rados.rados.0.smithi063.stdout:3942: finishing write tid 2 to smithi06387500-2 2024-08-17T17:11:37.700 INFO:tasks.rados.rados.0.smithi063.stdout:3942: finishing write tid 3 to smithi06387500-2 2024-08-17T17:11:37.700 INFO:tasks.rados.rados.0.smithi063.stdout:3942: finishing write tid 4 to smithi06387500-2 2024-08-17T17:11:37.700 INFO:tasks.rados.rados.0.smithi063.stdout:3942: oid 2 updating version 0 to 2165 2024-08-17T17:11:37.701 INFO:tasks.rados.rados.0.smithi063.stdout:3942: oid 2 version 2165 is already newer than 2164 2024-08-17T17:11:37.701 INFO:tasks.rados.rados.0.smithi063.stdout:3942: oid 2 version 2165 is already newer than 2163 2024-08-17T17:11:37.701 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 2165 (ObjNum 1490 snap 499 seq_num 1490) dirty exists 2024-08-17T17:11:37.701 INFO:tasks.rados.rados.0.smithi063.stdout:3942: left oid 2 (ObjNum 1490 snap 499 seq_num 1490) 2024-08-17T17:11:37.709 INFO:tasks.rados.rados.0.smithi063.stdout:3941: done (2 left) 2024-08-17T17:11:37.709 INFO:tasks.rados.rados.0.smithi063.stdout:3942: done (1 left) 2024-08-17T17:11:37.710 INFO:tasks.rados.rados.0.smithi063.stdout:3946: done (0 left) 2024-08-17T17:11:37.710 INFO:tasks.rados.rados.0.smithi063.stdout:3947: write oid 40 current snap is 499 2024-08-17T17:11:37.710 INFO:tasks.rados.rados.0.smithi063.stdout:3947: seq_num 1492 ranges {676044=676696,1991487=723188} 2024-08-17T17:11:37.717 INFO:tasks.rados.rados.0.smithi063.stdout:3947: writing smithi06387500-40 from 676044 to 1352740 tid 1 2024-08-17T17:11:37.726 INFO:tasks.rados.rados.0.smithi063.stdout:3947: writing smithi06387500-40 from 1991487 to 2714675 tid 2 2024-08-17T17:11:37.726 INFO:tasks.rados.rados.0.smithi063.stdout:3948: read oid 20 snap -1 2024-08-17T17:11:37.726 INFO:tasks.rados.rados.0.smithi063.stdout:3948: expect (ObjNum 1487 snap 497 seq_num 1487) 2024-08-17T17:11:37.726 INFO:tasks.rados.rados.0.smithi063.stdout:3949: read oid 23 snap -1 2024-08-17T17:11:37.726 INFO:tasks.rados.rados.0.smithi063.stdout:3949: expect (ObjNum 1466 snap 490 seq_num 1466) 2024-08-17T17:11:37.726 INFO:tasks.rados.rados.0.smithi063.stdout:3950: read oid 8 snap -1 2024-08-17T17:11:37.726 INFO:tasks.rados.rados.0.smithi063.stdout:3950: expect (ObjNum 1391 snap 466 seq_num 1391) 2024-08-17T17:11:37.726 INFO:tasks.rados.rados.0.smithi063.stdout:3951: read oid 24 snap -1 2024-08-17T17:11:37.726 INFO:tasks.rados.rados.0.smithi063.stdout:3951: expect deleted 2024-08-17T17:11:37.726 INFO:tasks.rados.rados.0.smithi063.stdout:3952: snap_remove snap 496 2024-08-17T17:11:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:37 smithi063 ceph-mon[114756]: osdmap e1069: 8 total, 8 up, 8 in 2024-08-17T17:11:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:37 smithi063 ceph-mon[114756]: osdmap e1070: 8 total, 8 up, 8 in 2024-08-17T17:11:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:37 smithi063 ceph-mon[109843]: osdmap e1069: 8 total, 8 up, 8 in 2024-08-17T17:11:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:37 smithi063 ceph-mon[109843]: osdmap e1070: 8 total, 8 up, 8 in 2024-08-17T17:11:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:37 smithi162 ceph-mon[104668]: osdmap e1069: 8 total, 8 up, 8 in 2024-08-17T17:11:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:37 smithi162 ceph-mon[104668]: osdmap e1070: 8 total, 8 up, 8 in 2024-08-17T17:11:38.725 INFO:tasks.rados.rados.0.smithi063.stdout:3952: done (5 left) 2024-08-17T17:11:38.726 INFO:tasks.rados.rados.0.smithi063.stdout:3953: read oid 2 snap -1 2024-08-17T17:11:38.726 INFO:tasks.rados.rados.0.smithi063.stdout:3953: expect (ObjNum 1490 snap 499 seq_num 1490) 2024-08-17T17:11:38.726 INFO:tasks.rados.rados.0.smithi063.stdout:3954: delete oid 12 current snap is 499 2024-08-17T17:11:38.726 INFO:tasks.rados.rados.0.smithi063.stdout:3947: finishing write tid 1 to smithi06387500-40 2024-08-17T17:11:38.726 INFO:tasks.rados.rados.0.smithi063.stdout:3947: finishing write tid 2 to smithi06387500-40 2024-08-17T17:11:38.726 INFO:tasks.rados.rados.0.smithi063.stdout:3947: finishing write tid 3 to smithi06387500-40 2024-08-17T17:11:38.726 INFO:tasks.rados.rados.0.smithi063.stdout:3947: finishing write tid 4 to smithi06387500-40 2024-08-17T17:11:38.726 INFO:tasks.rados.rados.0.smithi063.stdout:3947: oid 40 updating version 0 to 1391 2024-08-17T17:11:38.726 INFO:tasks.rados.rados.0.smithi063.stdout:3947: oid 40 updating version 1391 to 1392 2024-08-17T17:11:38.727 INFO:tasks.rados.rados.0.smithi063.stdout:3947: oid 40 version 1392 is already newer than 1390 2024-08-17T17:11:38.727 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 40 v 1392 (ObjNum 1492 snap 499 seq_num 1492) dirty exists 2024-08-17T17:11:38.727 INFO:tasks.rados.rados.0.smithi063.stdout:3947: left oid 40 (ObjNum 1492 snap 499 seq_num 1492) 2024-08-17T17:11:38.727 INFO:tasks.rados.rados.0.smithi063.stdout:3950: expect (ObjNum 1391 snap 466 seq_num 1391) 2024-08-17T17:11:38.837 INFO:tasks.rados.rados.0.smithi063.stdout:3948: expect (ObjNum 1487 snap 497 seq_num 1487) 2024-08-17T17:11:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:38 smithi063 ceph-mon[114756]: pgmap v1062: 89 pgs: 89 active+clean; 186 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2024-08-17T17:11:39.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:38 smithi063 ceph-mon[109843]: pgmap v1062: 89 pgs: 89 active+clean; 186 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2024-08-17T17:11:39.041 INFO:tasks.rados.rados.0.smithi063.stdout:3949: expect (ObjNum 1466 snap 490 seq_num 1466) 2024-08-17T17:11:39.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:38 smithi162 ceph-mon[104668]: pgmap v1062: 89 pgs: 89 active+clean; 186 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.6 MiB/s wr, 8 op/s 2024-08-17T17:11:39.182 INFO:tasks.rados.rados.0.smithi063.stdout:3953: expect (ObjNum 1490 snap 499 seq_num 1490) 2024-08-17T17:11:39.331 INFO:tasks.rados.rados.0.smithi063.stdout:3947: done (6 left) 2024-08-17T17:11:39.331 INFO:tasks.rados.rados.0.smithi063.stdout:3948: done (5 left) 2024-08-17T17:11:39.331 INFO:tasks.rados.rados.0.smithi063.stdout:3949: done (4 left) 2024-08-17T17:11:39.331 INFO:tasks.rados.rados.0.smithi063.stdout:3950: done (3 left) 2024-08-17T17:11:39.331 INFO:tasks.rados.rados.0.smithi063.stdout:3951: done (2 left) 2024-08-17T17:11:39.332 INFO:tasks.rados.rados.0.smithi063.stdout:3953: done (1 left) 2024-08-17T17:11:39.332 INFO:tasks.rados.rados.0.smithi063.stdout:3954: done (0 left) 2024-08-17T17:11:39.332 INFO:tasks.rados.rados.0.smithi063.stdout:3955: rollback oid 43 current snap is 499 2024-08-17T17:11:39.332 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 43 to 495 2024-08-17T17:11:39.332 INFO:tasks.rados.rados.0.smithi063.stdout:3956: delete oid 7 current snap is 499 2024-08-17T17:11:39.335 INFO:tasks.rados.rados.0.smithi063.stdout:3955: finishing rollback tid 0 to smithi06387500-43 2024-08-17T17:11:39.335 INFO:tasks.rados.rados.0.smithi063.stdout:3955: finishing rollback tid 1 to smithi06387500-43 2024-08-17T17:11:39.336 INFO:tasks.rados.rados.0.smithi063.stdout:3955: finishing rollback tid 2 to smithi06387500-43 2024-08-17T17:11:39.336 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 1983 (ObjNum 1446 snap 484 seq_num 1446) dirty exists 2024-08-17T17:11:39.337 INFO:tasks.rados.rados.0.smithi063.stdout:3955: done (1 left) 2024-08-17T17:11:39.338 INFO:tasks.rados.rados.0.smithi063.stdout:3956: done (0 left) 2024-08-17T17:11:39.338 INFO:tasks.rados.rados.0.smithi063.stdout:3957: read oid 44 snap -1 2024-08-17T17:11:39.338 INFO:tasks.rados.rados.0.smithi063.stdout:3957: expect (ObjNum 1473 snap 492 seq_num 1473) 2024-08-17T17:11:39.338 INFO:tasks.rados.rados.0.smithi063.stdout:3958: write oid 30 current snap is 499 2024-08-17T17:11:39.338 INFO:tasks.rados.rados.0.smithi063.stdout:3958: seq_num 1495 ranges {726465=502443,1650580=470515,2788433=338032} 2024-08-17T17:11:39.345 INFO:tasks.rados.rados.0.smithi063.stdout:3958: writing smithi06387500-30 from 726465 to 1228908 tid 1 2024-08-17T17:11:39.349 INFO:tasks.rados.rados.0.smithi063.stdout:3958: writing smithi06387500-30 from 1650580 to 2121095 tid 2 2024-08-17T17:11:39.355 INFO:tasks.rados.rados.0.smithi063.stdout:3958: writing smithi06387500-30 from 2788433 to 3126465 tid 3 2024-08-17T17:11:39.355 INFO:tasks.rados.rados.0.smithi063.stdout:3959: read oid 18 snap -1 2024-08-17T17:11:39.355 INFO:tasks.rados.rados.0.smithi063.stdout:3959: expect deleted 2024-08-17T17:11:39.355 INFO:tasks.rados.rados.0.smithi063.stdout:3960: snap_create 2024-08-17T17:11:39.355 INFO:tasks.rados.rados.0.smithi063.stdout:3957: expect (ObjNum 1473 snap 492 seq_num 1473) 2024-08-17T17:11:39.553 INFO:tasks.rados.rados.0.smithi063.stdout:3958: finishing write tid 1 to smithi06387500-30 2024-08-17T17:11:39.553 INFO:tasks.rados.rados.0.smithi063.stdout:3958: finishing write tid 2 to smithi06387500-30 2024-08-17T17:11:39.553 INFO:tasks.rados.rados.0.smithi063.stdout:3958: finishing write tid 3 to smithi06387500-30 2024-08-17T17:11:39.553 INFO:tasks.rados.rados.0.smithi063.stdout:3958: finishing write tid 4 to smithi06387500-30 2024-08-17T17:11:39.553 INFO:tasks.rados.rados.0.smithi063.stdout:3958: finishing write tid 5 to smithi06387500-30 2024-08-17T17:11:39.553 INFO:tasks.rados.rados.0.smithi063.stdout:3958: oid 30 updating version 0 to 1986 2024-08-17T17:11:39.553 INFO:tasks.rados.rados.0.smithi063.stdout:3958: oid 30 updating version 1986 to 1988 2024-08-17T17:11:39.553 INFO:tasks.rados.rados.0.smithi063.stdout:3958: oid 30 version 1988 is already newer than 1985 2024-08-17T17:11:39.554 INFO:tasks.rados.rados.0.smithi063.stdout:3958: oid 30 version 1988 is already newer than 1987 2024-08-17T17:11:39.554 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 1988 (ObjNum 1495 snap 499 seq_num 1495) dirty exists 2024-08-17T17:11:39.554 INFO:tasks.rados.rados.0.smithi063.stdout:3958: left oid 30 (ObjNum 1495 snap 499 seq_num 1495) 2024-08-17T17:11:39.727 INFO:tasks.rados.rados.0.smithi063.stdout:3957: done (3 left) 2024-08-17T17:11:39.728 INFO:tasks.rados.rados.0.smithi063.stdout:3958: done (2 left) 2024-08-17T17:11:39.728 INFO:tasks.rados.rados.0.smithi063.stdout:3959: done (1 left) 2024-08-17T17:11:39.728 INFO:tasks.rados.rados.0.smithi063.stdout:3960: done (0 left) 2024-08-17T17:11:39.728 INFO:tasks.rados.rados.0.smithi063.stdout:3961: write oid 43 current snap is 500 2024-08-17T17:11:39.728 INFO:tasks.rados.rados.0.smithi063.stdout:3961: seq_num 1496 ranges {743272=577692,2109062=730526} 2024-08-17T17:11:39.734 INFO:tasks.rados.rados.0.smithi063.stdout:3961: writing smithi06387500-43 from 743272 to 1320964 tid 1 2024-08-17T17:11:39.743 INFO:tasks.rados.rados.0.smithi063.stdout:3961: writing smithi06387500-43 from 2109062 to 2839588 tid 2 2024-08-17T17:11:39.743 INFO:tasks.rados.rados.0.smithi063.stdout:3962: write oid 18 current snap is 500 2024-08-17T17:11:39.743 INFO:tasks.rados.rados.0.smithi063.stdout:3961: finishing write tid 1 to smithi06387500-43 2024-08-17T17:11:39.743 INFO:tasks.rados.rados.0.smithi063.stdout:3962: seq_num 1497 ranges {760079=652941,2167544=474184} 2024-08-17T17:11:39.751 INFO:tasks.rados.rados.0.smithi063.stdout:3962: writing smithi06387500-18 from 760079 to 1413020 tid 1 2024-08-17T17:11:39.757 INFO:tasks.rados.rados.0.smithi063.stdout:3962: writing smithi06387500-18 from 2167544 to 2641728 tid 2 2024-08-17T17:11:39.758 INFO:tasks.rados.rados.0.smithi063.stdout:3963: write oid 3 current snap is 500 2024-08-17T17:11:39.758 INFO:tasks.rados.rados.0.smithi063.stdout:3961: finishing write tid 2 to smithi06387500-43 2024-08-17T17:11:39.758 INFO:tasks.rados.rados.0.smithi063.stdout:3963: seq_num 1498 ranges {776886=444543,1826026=734195,2987055=189831} 2024-08-17T17:11:39.764 INFO:tasks.rados.rados.0.smithi063.stdout:3963: writing smithi06387500-3 from 776886 to 1221429 tid 1 2024-08-17T17:11:39.771 INFO:tasks.rados.rados.0.smithi063.stdout:3963: writing smithi06387500-3 from 1826026 to 2560221 tid 2 2024-08-17T17:11:39.775 INFO:tasks.rados.rados.0.smithi063.stdout:3963: writing smithi06387500-3 from 2987055 to 3176886 tid 3 2024-08-17T17:11:39.775 INFO:tasks.rados.rados.0.smithi063.stdout:3964: read oid 8 snap 489 2024-08-17T17:11:39.775 INFO:tasks.rados.rados.0.smithi063.stdout:3964: expect (ObjNum 1391 snap 466 seq_num 1391) 2024-08-17T17:11:39.775 INFO:tasks.rados.rados.0.smithi063.stdout:3965: read oid 9 snap 493 2024-08-17T17:11:39.775 INFO:tasks.rados.rados.0.smithi063.stdout:3965: expect deleted 2024-08-17T17:11:39.775 INFO:tasks.rados.rados.0.smithi063.stdout:3966: snap_remove snap 490 2024-08-17T17:11:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:39 smithi063 ceph-mon[114756]: osdmap e1071: 8 total, 8 up, 8 in 2024-08-17T17:11:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:39 smithi063 ceph-mon[109843]: osdmap e1071: 8 total, 8 up, 8 in 2024-08-17T17:11:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:39 smithi162 ceph-mon[104668]: osdmap e1071: 8 total, 8 up, 8 in 2024-08-17T17:11:40.746 INFO:tasks.rados.rados.0.smithi063.stdout:3961: finishing write tid 3 to smithi06387500-43 2024-08-17T17:11:40.746 INFO:tasks.rados.rados.0.smithi063.stdout:3961: finishing write tid 4 to smithi06387500-43 2024-08-17T17:11:40.746 INFO:tasks.rados.rados.0.smithi063.stdout:3961: oid 43 updating version 0 to 1992 2024-08-17T17:11:40.746 INFO:tasks.rados.rados.0.smithi063.stdout:3961: oid 43 version 1992 is already newer than 1991 2024-08-17T17:11:40.747 INFO:tasks.rados.rados.0.smithi063.stdout:3961: oid 43 version 1992 is already newer than 1990 2024-08-17T17:11:40.747 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 43 v 1992 (ObjNum 1496 snap 500 seq_num 1496) dirty exists 2024-08-17T17:11:40.747 INFO:tasks.rados.rados.0.smithi063.stdout:3961: left oid 43 (ObjNum 1496 snap 500 seq_num 1496) 2024-08-17T17:11:40.747 INFO:tasks.rados.rados.0.smithi063.stdout:3962: finishing write tid 1 to smithi06387500-18 2024-08-17T17:11:40.747 INFO:tasks.rados.rados.0.smithi063.stdout:3962: finishing write tid 2 to smithi06387500-18 2024-08-17T17:11:40.747 INFO:tasks.rados.rados.0.smithi063.stdout:3962: finishing write tid 3 to smithi06387500-18 2024-08-17T17:11:40.747 INFO:tasks.rados.rados.0.smithi063.stdout:3962: finishing write tid 4 to smithi06387500-18 2024-08-17T17:11:40.747 INFO:tasks.rados.rados.0.smithi063.stdout:3962: oid 18 updating version 0 to 1397 2024-08-17T17:11:40.747 INFO:tasks.rados.rados.0.smithi063.stdout:3962: oid 18 version 1397 is already newer than 1396 2024-08-17T17:11:40.747 INFO:tasks.rados.rados.0.smithi063.stdout:3962: oid 18 version 1397 is already newer than 1395 2024-08-17T17:11:40.747 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 18 v 1397 (ObjNum 1497 snap 500 seq_num 1497) dirty exists 2024-08-17T17:11:40.747 INFO:tasks.rados.rados.0.smithi063.stdout:3962: left oid 18 (ObjNum 1497 snap 500 seq_num 1497) 2024-08-17T17:11:40.747 INFO:tasks.rados.rados.0.smithi063.stdout:3963: finishing write tid 1 to smithi06387500-3 2024-08-17T17:11:40.748 INFO:tasks.rados.rados.0.smithi063.stdout:3963: finishing write tid 2 to smithi06387500-3 2024-08-17T17:11:40.748 INFO:tasks.rados.rados.0.smithi063.stdout:3963: finishing write tid 3 to smithi06387500-3 2024-08-17T17:11:40.748 INFO:tasks.rados.rados.0.smithi063.stdout:3963: finishing write tid 4 to smithi06387500-3 2024-08-17T17:11:40.748 INFO:tasks.rados.rados.0.smithi063.stdout:3963: finishing write tid 5 to smithi06387500-3 2024-08-17T17:11:40.748 INFO:tasks.rados.rados.0.smithi063.stdout:3963: oid 3 updating version 0 to 1405 2024-08-17T17:11:40.748 INFO:tasks.rados.rados.0.smithi063.stdout:3963: oid 3 updating version 1405 to 1408 2024-08-17T17:11:40.748 INFO:tasks.rados.rados.0.smithi063.stdout:3963: oid 3 version 1408 is already newer than 1407 2024-08-17T17:11:40.748 INFO:tasks.rados.rados.0.smithi063.stdout:3963: oid 3 version 1408 is already newer than 1406 2024-08-17T17:11:40.748 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 3 v 1408 (ObjNum 1498 snap 500 seq_num 1498) dirty exists 2024-08-17T17:11:40.748 INFO:tasks.rados.rados.0.smithi063.stdout:3963: left oid 3 (ObjNum 1498 snap 500 seq_num 1498) 2024-08-17T17:11:40.748 INFO:tasks.rados.rados.0.smithi063.stdout:3964: expect (ObjNum 1391 snap 466 seq_num 1391) 2024-08-17T17:11:40.858 INFO:tasks.rados.rados.0.smithi063.stdout:3961: done (5 left) 2024-08-17T17:11:40.858 INFO:tasks.rados.rados.0.smithi063.stdout:3962: done (4 left) 2024-08-17T17:11:40.858 INFO:tasks.rados.rados.0.smithi063.stdout:3963: done (3 left) 2024-08-17T17:11:40.858 INFO:tasks.rados.rados.0.smithi063.stdout:3964: done (2 left) 2024-08-17T17:11:40.858 INFO:tasks.rados.rados.0.smithi063.stdout:3965: done (1 left) 2024-08-17T17:11:40.859 INFO:tasks.rados.rados.0.smithi063.stdout:3966: done (0 left) 2024-08-17T17:11:40.859 INFO:tasks.rados.rados.0.smithi063.stdout:3967: snap_create 2024-08-17T17:11:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:40 smithi063 ceph-mon[114756]: osdmap e1072: 8 total, 8 up, 8 in 2024-08-17T17:11:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:40 smithi063 ceph-mon[114756]: pgmap v1065: 89 pgs: 89 active+clean; 186 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 637 KiB/s wr, 2 op/s 2024-08-17T17:11:41.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:40 smithi063 ceph-mon[109843]: osdmap e1072: 8 total, 8 up, 8 in 2024-08-17T17:11:41.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:40 smithi063 ceph-mon[109843]: pgmap v1065: 89 pgs: 89 active+clean; 186 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 637 KiB/s wr, 2 op/s 2024-08-17T17:11:41.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:40 smithi162 ceph-mon[104668]: osdmap e1072: 8 total, 8 up, 8 in 2024-08-17T17:11:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:40 smithi162 ceph-mon[104668]: pgmap v1065: 89 pgs: 89 active+clean; 186 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 637 KiB/s wr, 2 op/s 2024-08-17T17:11:41.751 INFO:tasks.rados.rados.0.smithi063.stdout:3967: done (0 left) 2024-08-17T17:11:41.751 INFO:tasks.rados.rados.0.smithi063.stdout:3968: write oid 44 current snap is 501 2024-08-17T17:11:41.751 INFO:tasks.rados.rados.0.smithi063.stdout:3968: seq_num 1499 ranges {793693=519792,1884508=477853,2898125=295568} 2024-08-17T17:11:41.758 INFO:tasks.rados.rados.0.smithi063.stdout:3968: writing smithi06387500-44 from 793693 to 1313485 tid 1 2024-08-17T17:11:41.764 INFO:tasks.rados.rados.0.smithi063.stdout:3968: writing smithi06387500-44 from 1884508 to 2362361 tid 2 2024-08-17T17:11:41.769 INFO:tasks.rados.rados.0.smithi063.stdout:3968: writing smithi06387500-44 from 2898125 to 3193693 tid 3 2024-08-17T17:11:41.769 INFO:tasks.rados.rados.0.smithi063.stdout:3969: rollback oid 33 current snap is 501 2024-08-17T17:11:41.769 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 33 to 495 2024-08-17T17:11:41.769 INFO:tasks.rados.rados.0.smithi063.stdout:3968: finishing write tid 1 to smithi06387500-44 2024-08-17T17:11:41.769 INFO:tasks.rados.rados.0.smithi063.stdout:3968: finishing write tid 2 to smithi06387500-44 2024-08-17T17:11:41.769 INFO:tasks.rados.rados.0.smithi063.stdout:3970: snap_remove snap 499 2024-08-17T17:11:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:41 smithi063 ceph-mon[114756]: osdmap e1073: 8 total, 8 up, 8 in 2024-08-17T17:11:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:41 smithi063 ceph-mon[109843]: osdmap e1073: 8 total, 8 up, 8 in 2024-08-17T17:11:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:41 smithi162 ceph-mon[104668]: osdmap e1073: 8 total, 8 up, 8 in 2024-08-17T17:11:42.699 INFO:tasks.rados.rados.0.smithi063.stdout:3970: done (2 left) 2024-08-17T17:11:42.699 INFO:tasks.rados.rados.0.smithi063.stdout:3971: read oid 5 snap -1 2024-08-17T17:11:42.699 INFO:tasks.rados.rados.0.smithi063.stdout:3971: expect (ObjNum 1293 snap 441 seq_num 1293) 2024-08-17T17:11:42.699 INFO:tasks.rados.rados.0.smithi063.stdout:3972: snap_create 2024-08-17T17:11:42.699 INFO:tasks.rados.rados.0.smithi063.stdout:3968: finishing write tid 3 to smithi06387500-44 2024-08-17T17:11:42.700 INFO:tasks.rados.rados.0.smithi063.stdout:3968: finishing write tid 4 to smithi06387500-44 2024-08-17T17:11:42.700 INFO:tasks.rados.rados.0.smithi063.stdout:3968: finishing write tid 5 to smithi06387500-44 2024-08-17T17:11:42.700 INFO:tasks.rados.rados.0.smithi063.stdout:3968: oid 44 updating version 0 to 601 2024-08-17T17:11:42.700 INFO:tasks.rados.rados.0.smithi063.stdout:3968: oid 44 updating version 601 to 603 2024-08-17T17:11:42.700 INFO:tasks.rados.rados.0.smithi063.stdout:3968: oid 44 version 603 is already newer than 602 2024-08-17T17:11:42.700 INFO:tasks.rados.rados.0.smithi063.stdout:3968: oid 44 version 603 is already newer than 600 2024-08-17T17:11:42.700 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 44 v 603 (ObjNum 1499 snap 501 seq_num 1499) dirty exists 2024-08-17T17:11:42.700 INFO:tasks.rados.rados.0.smithi063.stdout:3968: left oid 44 (ObjNum 1499 snap 501 seq_num 1499) 2024-08-17T17:11:42.700 INFO:tasks.rados.rados.0.smithi063.stdout:3969: finishing rollback tid 0 to smithi06387500-33 2024-08-17T17:11:42.700 INFO:tasks.rados.rados.0.smithi063.stdout:3969: finishing rollback tid 1 to smithi06387500-33 2024-08-17T17:11:42.700 INFO:tasks.rados.rados.0.smithi063.stdout:3969: finishing rollback tid 2 to smithi06387500-33 2024-08-17T17:11:42.700 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 33 v 1349 (ObjNum 1484 snap 495 seq_num 1484) dirty exists 2024-08-17T17:11:42.716 INFO:tasks.rados.rados.0.smithi063.stdout:3971: expect (ObjNum 1293 snap 441 seq_num 1293) 2024-08-17T17:11:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:42 smithi063 ceph-mon[114756]: osdmap e1074: 8 total, 8 up, 8 in 2024-08-17T17:11:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:42 smithi063 ceph-mon[114756]: pgmap v1068: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 174 MiB data, 790 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-08-17T17:11:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:42 smithi063 ceph-mon[114756]: osdmap e1075: 8 total, 8 up, 8 in 2024-08-17T17:11:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:42 smithi063 ceph-mon[109843]: osdmap e1074: 8 total, 8 up, 8 in 2024-08-17T17:11:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:42 smithi063 ceph-mon[109843]: pgmap v1068: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 174 MiB data, 790 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-08-17T17:11:43.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:42 smithi063 ceph-mon[109843]: osdmap e1075: 8 total, 8 up, 8 in 2024-08-17T17:11:43.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:42 smithi162 ceph-mon[104668]: osdmap e1074: 8 total, 8 up, 8 in 2024-08-17T17:11:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:42 smithi162 ceph-mon[104668]: pgmap v1068: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 174 MiB data, 790 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-08-17T17:11:43.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:42 smithi162 ceph-mon[104668]: osdmap e1075: 8 total, 8 up, 8 in 2024-08-17T17:11:43.700 INFO:tasks.rados.rados.0.smithi063.stdout:3968: done (3 left) 2024-08-17T17:11:43.700 INFO:tasks.rados.rados.0.smithi063.stdout:3969: done (2 left) 2024-08-17T17:11:43.700 INFO:tasks.rados.rados.0.smithi063.stdout:3971: done (1 left) 2024-08-17T17:11:43.700 INFO:tasks.rados.rados.0.smithi063.stdout:3972: done (0 left) 2024-08-17T17:11:43.700 INFO:tasks.rados.rados.0.smithi063.stdout:3973: write oid 29 current snap is 502 2024-08-17T17:11:43.701 INFO:tasks.rados.rados.0.smithi063.stdout:3973: seq_num 1500 ranges {410500=595041,1542990=621511,2925548=284952} 2024-08-17T17:11:43.705 INFO:tasks.rados.rados.0.smithi063.stdout:3973: writing smithi06387500-29 from 410500 to 1005541 tid 1 2024-08-17T17:11:43.712 INFO:tasks.rados.rados.0.smithi063.stdout:3973: writing smithi06387500-29 from 1542990 to 2164501 tid 2 2024-08-17T17:11:43.718 INFO:tasks.rados.rados.0.smithi063.stdout:3973: writing smithi06387500-29 from 2925548 to 3210500 tid 3 2024-08-17T17:11:43.718 INFO:tasks.rados.rados.0.smithi063.stdout:3974: snap_create 2024-08-17T17:11:43.718 INFO:tasks.rados.rados.0.smithi063.stdout:3973: finishing write tid 1 to smithi06387500-29 2024-08-17T17:11:43.719 INFO:tasks.rados.rados.0.smithi063.stdout:3973: finishing write tid 2 to smithi06387500-29 2024-08-17T17:11:43.722 INFO:tasks.rados.rados.0.smithi063.stdout:3973: finishing write tid 3 to smithi06387500-29 2024-08-17T17:11:43.722 INFO:tasks.rados.rados.0.smithi063.stdout:3973: finishing write tid 4 to smithi06387500-29 2024-08-17T17:11:43.722 INFO:tasks.rados.rados.0.smithi063.stdout:3973: finishing write tid 5 to smithi06387500-29 2024-08-17T17:11:43.722 INFO:tasks.rados.rados.0.smithi063.stdout:3973: oid 29 updating version 0 to 861 2024-08-17T17:11:43.722 INFO:tasks.rados.rados.0.smithi063.stdout:3973: oid 29 updating version 861 to 862 2024-08-17T17:11:43.723 INFO:tasks.rados.rados.0.smithi063.stdout:3973: oid 29 version 862 is already newer than 860 2024-08-17T17:11:43.723 INFO:tasks.rados.rados.0.smithi063.stdout:3973: oid 29 version 862 is already newer than 859 2024-08-17T17:11:43.723 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 29 v 862 (ObjNum 1500 snap 502 seq_num 1500) dirty exists 2024-08-17T17:11:43.723 INFO:tasks.rados.rados.0.smithi063.stdout:3973: left oid 29 (ObjNum 1500 snap 502 seq_num 1500) 2024-08-17T17:11:43.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:11:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:11:43] "GET /metrics HTTP/1.1" 200 36334 "" "Prometheus/2.51.0" 2024-08-17T17:11:44.709 INFO:tasks.rados.rados.0.smithi063.stdout:3973: done (1 left) 2024-08-17T17:11:44.709 INFO:tasks.rados.rados.0.smithi063.stdout:3974: done (0 left) 2024-08-17T17:11:44.710 INFO:tasks.rados.rados.0.smithi063.stdout:3975: snap_create 2024-08-17T17:11:45.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:44 smithi063 ceph-mon[109843]: osdmap e1076: 8 total, 8 up, 8 in 2024-08-17T17:11:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:44 smithi063 ceph-mon[114756]: osdmap e1076: 8 total, 8 up, 8 in 2024-08-17T17:11:45.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:44 smithi162 ceph-mon[104668]: osdmap e1076: 8 total, 8 up, 8 in 2024-08-17T17:11:45.710 INFO:tasks.rados.rados.0.smithi063.stdout:3975: done (0 left) 2024-08-17T17:11:45.710 INFO:tasks.rados.rados.0.smithi063.stdout:3976: read oid 8 snap -1 2024-08-17T17:11:45.710 INFO:tasks.rados.rados.0.smithi063.stdout:3976: expect (ObjNum 1391 snap 466 seq_num 1391) 2024-08-17T17:11:45.710 INFO:tasks.rados.rados.0.smithi063.stdout:3977: read oid 38 snap -1 2024-08-17T17:11:45.711 INFO:tasks.rados.rados.0.smithi063.stdout:3977: expect deleted 2024-08-17T17:11:45.711 INFO:tasks.rados.rados.0.smithi063.stdout:3978: rollback oid 26 current snap is 504 2024-08-17T17:11:45.711 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 26 to 500 2024-08-17T17:11:45.711 INFO:tasks.rados.rados.0.smithi063.stdout:3979: snap_create 2024-08-17T17:11:45.717 INFO:tasks.rados.rados.0.smithi063.stdout:3978: finishing rollback tid 0 to smithi06387500-26 2024-08-17T17:11:45.717 INFO:tasks.rados.rados.0.smithi063.stdout:3978: finishing rollback tid 1 to smithi06387500-26 2024-08-17T17:11:45.717 INFO:tasks.rados.rados.0.smithi063.stdout:3978: finishing rollback tid 2 to smithi06387500-26 2024-08-17T17:11:45.717 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 26 v 1414 (ObjNum 1471 snap 492 seq_num 1471) dirty exists 2024-08-17T17:11:45.719 INFO:tasks.rados.rados.0.smithi063.stdout:3976: expect (ObjNum 1391 snap 466 seq_num 1391) 2024-08-17T17:11:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:45 smithi063 ceph-mon[109843]: pgmap v1071: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 174 MiB data, 789 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.9 MiB/s wr, 22 op/s 2024-08-17T17:11:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:45 smithi063 ceph-mon[109843]: osdmap e1077: 8 total, 8 up, 8 in 2024-08-17T17:11:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:45 smithi063 ceph-mon[114756]: pgmap v1071: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 174 MiB data, 789 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.9 MiB/s wr, 22 op/s 2024-08-17T17:11:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:45 smithi063 ceph-mon[114756]: osdmap e1077: 8 total, 8 up, 8 in 2024-08-17T17:11:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:45 smithi162 ceph-mon[104668]: pgmap v1071: 89 pgs: 1 active+clean+snaptrim, 88 active+clean; 174 MiB data, 789 MiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 1.9 MiB/s wr, 22 op/s 2024-08-17T17:11:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:45 smithi162 ceph-mon[104668]: osdmap e1077: 8 total, 8 up, 8 in 2024-08-17T17:11:46.724 INFO:tasks.rados.rados.0.smithi063.stdout:3976: done (3 left) 2024-08-17T17:11:46.724 INFO:tasks.rados.rados.0.smithi063.stdout:3977: done (2 left) 2024-08-17T17:11:46.725 INFO:tasks.rados.rados.0.smithi063.stdout:3978: done (1 left) 2024-08-17T17:11:46.725 INFO:tasks.rados.rados.0.smithi063.stdout:3979: done (0 left) 2024-08-17T17:11:46.725 INFO:tasks.rados.rados.0.smithi063.stdout:3980: read oid 35 snap -1 2024-08-17T17:11:46.725 INFO:tasks.rados.rados.0.smithi063.stdout:3980: expect (ObjNum 1424 snap 476 seq_num 1424) 2024-08-17T17:11:46.725 INFO:tasks.rados.rados.0.smithi063.stdout:3981: read oid 28 snap -1 2024-08-17T17:11:46.725 INFO:tasks.rados.rados.0.smithi063.stdout:3981: expect (ObjNum 1432 snap 479 seq_num 1432) 2024-08-17T17:11:46.725 INFO:tasks.rados.rados.0.smithi063.stdout:3982: read oid 8 snap 504 2024-08-17T17:11:46.725 INFO:tasks.rados.rados.0.smithi063.stdout:3982: expect (ObjNum 1391 snap 466 seq_num 1391) 2024-08-17T17:11:46.725 INFO:tasks.rados.rados.0.smithi063.stdout:3983: write oid 36 current snap is 505 2024-08-17T17:11:46.725 INFO:tasks.rados.rados.0.smithi063.stdout:3983: seq_num 1501 ranges {427307=670290,1601472=481522,2552971=483056} 2024-08-17T17:11:46.730 INFO:tasks.rados.rados.0.smithi063.stdout:3983: writing smithi06387500-36 from 427307 to 1097597 tid 1 2024-08-17T17:11:46.736 INFO:tasks.rados.rados.0.smithi063.stdout:3983: writing smithi06387500-36 from 1601472 to 2082994 tid 2 2024-08-17T17:11:46.741 INFO:tasks.rados.rados.0.smithi063.stdout:3983: writing smithi06387500-36 from 2552971 to 3036027 tid 3 2024-08-17T17:11:46.742 INFO:tasks.rados.rados.0.smithi063.stdout:3984: write oid 21 current snap is 505 2024-08-17T17:11:46.742 INFO:tasks.rados.rados.0.smithi063.stdout:3984: seq_num 1502 ranges {444114=745539,1943601=625180} 2024-08-17T17:11:46.748 INFO:tasks.rados.rados.0.smithi063.stdout:3984: writing smithi06387500-21 from 444114 to 1189653 tid 1 2024-08-17T17:11:46.756 INFO:tasks.rados.rados.0.smithi063.stdout:3984: writing smithi06387500-21 from 1943601 to 2568781 tid 2 2024-08-17T17:11:46.756 INFO:tasks.rados.rados.0.smithi063.stdout:3985: rollback oid 41 current snap is 505 2024-08-17T17:11:46.756 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 41 to 495 2024-08-17T17:11:46.756 INFO:tasks.rados.rados.0.smithi063.stdout:3982: expect (ObjNum 1391 snap 466 seq_num 1391) 2024-08-17T17:11:46.867 INFO:tasks.rados.rados.0.smithi063.stdout:3981: expect (ObjNum 1432 snap 479 seq_num 1432) 2024-08-17T17:11:46.937 INFO:tasks.rados.rados.0.smithi063.stdout:3980: expect (ObjNum 1424 snap 476 seq_num 1424) 2024-08-17T17:11:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:46 smithi063 ceph-mon[109843]: osdmap e1078: 8 total, 8 up, 8 in 2024-08-17T17:11:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:46 smithi063 ceph-mon[109843]: pgmap v1074: 89 pgs: 89 active+clean; 176 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 367 KiB/s wr, 6 op/s 2024-08-17T17:11:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:47.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:46 smithi063 ceph-mon[114756]: osdmap e1078: 8 total, 8 up, 8 in 2024-08-17T17:11:47.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:46 smithi063 ceph-mon[114756]: pgmap v1074: 89 pgs: 89 active+clean; 176 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 367 KiB/s wr, 6 op/s 2024-08-17T17:11:47.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:47.069 INFO:tasks.rados.rados.0.smithi063.stdout:3983: finishing write tid 1 to smithi06387500-36 2024-08-17T17:11:47.070 INFO:tasks.rados.rados.0.smithi063.stdout:3983: finishing write tid 2 to smithi06387500-36 2024-08-17T17:11:47.070 INFO:tasks.rados.rados.0.smithi063.stdout:3983: finishing write tid 3 to smithi06387500-36 2024-08-17T17:11:47.070 INFO:tasks.rados.rados.0.smithi063.stdout:3983: finishing write tid 4 to smithi06387500-36 2024-08-17T17:11:47.070 INFO:tasks.rados.rados.0.smithi063.stdout:3983: finishing write tid 5 to smithi06387500-36 2024-08-17T17:11:47.070 INFO:tasks.rados.rados.0.smithi063.stdout:3983: oid 36 updating version 0 to 3874 2024-08-17T17:11:47.070 INFO:tasks.rados.rados.0.smithi063.stdout:3983: oid 36 updating version 3874 to 3875 2024-08-17T17:11:47.070 INFO:tasks.rados.rados.0.smithi063.stdout:3983: oid 36 updating version 3875 to 3877 2024-08-17T17:11:47.070 INFO:tasks.rados.rados.0.smithi063.stdout:3983: oid 36 version 3877 is already newer than 3876 2024-08-17T17:11:47.071 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 36 v 3877 (ObjNum 1501 snap 505 seq_num 1501) dirty exists 2024-08-17T17:11:47.071 INFO:tasks.rados.rados.0.smithi063.stdout:3983: left oid 36 (ObjNum 1501 snap 505 seq_num 1501) 2024-08-17T17:11:47.071 INFO:tasks.rados.rados.0.smithi063.stdout:3984: finishing write tid 1 to smithi06387500-21 2024-08-17T17:11:47.071 INFO:tasks.rados.rados.0.smithi063.stdout:3980: done (5 left) 2024-08-17T17:11:47.071 INFO:tasks.rados.rados.0.smithi063.stdout:3981: done (4 left) 2024-08-17T17:11:47.071 INFO:tasks.rados.rados.0.smithi063.stdout:3982: done (3 left) 2024-08-17T17:11:47.071 INFO:tasks.rados.rados.0.smithi063.stdout:3983: done (2 left) 2024-08-17T17:11:47.071 INFO:tasks.rados.rados.0.smithi063.stdout:3986: write oid 47 current snap is 505 2024-08-17T17:11:47.071 INFO:tasks.rados.rados.0.smithi063.stdout:3985: finishing rollback tid 0 to smithi06387500-41 2024-08-17T17:11:47.071 INFO:tasks.rados.rados.0.smithi063.stdout:3985: finishing rollback tid 1 to smithi06387500-41 2024-08-17T17:11:47.071 INFO:tasks.rados.rados.0.smithi063.stdout:3985: finishing rollback tid 2 to smithi06387500-41 2024-08-17T17:11:47.071 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 41 v 468 (ObjNum 1450 snap 485 seq_num 1450) dirty exists 2024-08-17T17:11:47.071 INFO:tasks.rados.rados.0.smithi063.stdout:3984: finishing write tid 2 to smithi06387500-21 2024-08-17T17:11:47.071 INFO:tasks.rados.rados.0.smithi063.stdout:3986: seq_num 1503 ranges {460921=420788,1602083=485191,2491464=621953} 2024-08-17T17:11:47.074 INFO:tasks.rados.rados.0.smithi063.stdout:3986: writing smithi06387500-47 from 460921 to 881709 tid 1 2024-08-17T17:11:47.080 INFO:tasks.rados.rados.0.smithi063.stdout:3986: writing smithi06387500-47 from 1602083 to 2087274 tid 2 2024-08-17T17:11:47.086 INFO:tasks.rados.rados.0.smithi063.stdout:3986: writing smithi06387500-47 from 2491464 to 3113417 tid 3 2024-08-17T17:11:47.086 INFO:tasks.rados.rados.0.smithi063.stdout:3985: done (2 left) 2024-08-17T17:11:47.087 INFO:tasks.rados.rados.0.smithi063.stdout:3987: rollback oid 2 current snap is 505 2024-08-17T17:11:47.087 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 2 to 502 2024-08-17T17:11:47.087 INFO:tasks.rados.rados.0.smithi063.stdout:3984: finishing write tid 3 to smithi06387500-21 2024-08-17T17:11:47.087 INFO:tasks.rados.rados.0.smithi063.stdout:3984: finishing write tid 4 to smithi06387500-21 2024-08-17T17:11:47.087 INFO:tasks.rados.rados.0.smithi063.stdout:3984: oid 21 updating version 0 to 2464 2024-08-17T17:11:47.087 INFO:tasks.rados.rados.0.smithi063.stdout:3984: oid 21 version 2464 is already newer than 2463 2024-08-17T17:11:47.087 INFO:tasks.rados.rados.0.smithi063.stdout:3984: oid 21 updating version 2464 to 2465 2024-08-17T17:11:47.087 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 21 v 2465 (ObjNum 1502 snap 505 seq_num 1502) dirty exists 2024-08-17T17:11:47.087 INFO:tasks.rados.rados.0.smithi063.stdout:3984: left oid 21 (ObjNum 1502 snap 505 seq_num 1502) 2024-08-17T17:11:47.087 INFO:tasks.rados.rados.0.smithi063.stdout:3986: finishing write tid 1 to smithi06387500-47 2024-08-17T17:11:47.087 INFO:tasks.rados.rados.0.smithi063.stdout:3986: finishing write tid 2 to smithi06387500-47 2024-08-17T17:11:47.087 INFO:tasks.rados.rados.0.smithi063.stdout:3984: done (2 left) 2024-08-17T17:11:47.088 INFO:tasks.rados.rados.0.smithi063.stdout:3988: rollback oid 17 current snap is 505 2024-08-17T17:11:47.088 INFO:tasks.rados.rados.0.smithi063.stdout:rollback oid 17 to 501 2024-08-17T17:11:47.088 INFO:tasks.rados.rados.0.smithi063.stdout:3989: delete oid 25 current snap is 505 2024-08-17T17:11:47.089 INFO:tasks.rados.rados.0.smithi063.stdout:3986: finishing write tid 3 to smithi06387500-47 2024-08-17T17:11:47.090 INFO:tasks.rados.rados.0.smithi063.stdout:3986: finishing write tid 4 to smithi06387500-47 2024-08-17T17:11:47.090 INFO:tasks.rados.rados.0.smithi063.stdout:3986: finishing write tid 5 to smithi06387500-47 2024-08-17T17:11:47.090 INFO:tasks.rados.rados.0.smithi063.stdout:3986: oid 47 updating version 0 to 917 2024-08-17T17:11:47.090 INFO:tasks.rados.rados.0.smithi063.stdout:3986: oid 47 version 917 is already newer than 916 2024-08-17T17:11:47.090 INFO:tasks.rados.rados.0.smithi063.stdout:3986: oid 47 updating version 917 to 918 2024-08-17T17:11:47.090 INFO:tasks.rados.rados.0.smithi063.stdout:3986: oid 47 version 918 is already newer than 915 2024-08-17T17:11:47.090 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 47 v 918 (ObjNum 1503 snap 505 seq_num 1503) dirty exists 2024-08-17T17:11:47.090 INFO:tasks.rados.rados.0.smithi063.stdout:3986: left oid 47 (ObjNum 1503 snap 505 seq_num 1503) 2024-08-17T17:11:47.091 INFO:tasks.rados.rados.0.smithi063.stdout:3986: done (3 left) 2024-08-17T17:11:47.091 INFO:tasks.rados.rados.0.smithi063.stdout:3989: done (2 left) 2024-08-17T17:11:47.091 INFO:tasks.rados.rados.0.smithi063.stdout:3990: delete oid 38 current snap is 505 2024-08-17T17:11:47.091 INFO:tasks.rados.rados.0.smithi063.stdout:3988: finishing rollback tid 1 to smithi06387500-17 2024-08-17T17:11:47.091 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 17 v 0 (ObjNum 49 snap 0 seq_num 1114222583) dirty dne 2024-08-17T17:11:47.092 INFO:tasks.rados.rados.0.smithi063.stdout:3987: finishing rollback tid 0 to smithi06387500-2 2024-08-17T17:11:47.092 INFO:tasks.rados.rados.0.smithi063.stdout:3988: done (2 left) 2024-08-17T17:11:47.092 INFO:tasks.rados.rados.0.smithi063.stdout:3990: done (1 left) 2024-08-17T17:11:47.092 INFO:tasks.rados.rados.0.smithi063.stdout:3991: read oid 4 snap -1 2024-08-17T17:11:47.093 INFO:tasks.rados.rados.0.smithi063.stdout:3991: expect deleted 2024-08-17T17:11:47.093 INFO:tasks.rados.rados.0.smithi063.stdout:3992: read oid 20 snap -1 2024-08-17T17:11:47.093 INFO:tasks.rados.rados.0.smithi063.stdout:3992: expect (ObjNum 1487 snap 497 seq_num 1487) 2024-08-17T17:11:47.093 INFO:tasks.rados.rados.0.smithi063.stdout:3993: snap_create 2024-08-17T17:11:47.093 INFO:tasks.rados.rados.0.smithi063.stdout:3987: finishing rollback tid 1 to smithi06387500-2 2024-08-17T17:11:47.093 INFO:tasks.rados.rados.0.smithi063.stdout:3987: finishing rollback tid 2 to smithi06387500-2 2024-08-17T17:11:47.093 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 2 v 2187 (ObjNum 1490 snap 499 seq_num 1490) dirty exists 2024-08-17T17:11:47.093 INFO:tasks.rados.rados.0.smithi063.stdout:3992: expect (ObjNum 1487 snap 497 seq_num 1487) 2024-08-17T17:11:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:46 smithi162 ceph-mon[104668]: osdmap e1078: 8 total, 8 up, 8 in 2024-08-17T17:11:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:46 smithi162 ceph-mon[104668]: pgmap v1074: 89 pgs: 89 active+clean; 176 MiB data, 797 MiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 367 KiB/s wr, 6 op/s 2024-08-17T17:11:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:11:47.700 INFO:tasks.rados.rados.0.smithi063.stdout:3987: done (3 left) 2024-08-17T17:11:47.700 INFO:tasks.rados.rados.0.smithi063.stdout:3991: done (2 left) 2024-08-17T17:11:47.700 INFO:tasks.rados.rados.0.smithi063.stdout:3992: done (1 left) 2024-08-17T17:11:47.700 INFO:tasks.rados.rados.0.smithi063.stdout:3993: done (0 left) 2024-08-17T17:11:47.700 INFO:tasks.rados.rados.0.smithi063.stdout:3994: write oid 15 current snap is 506 2024-08-17T17:11:47.700 INFO:tasks.rados.rados.0.smithi063.stdout:3994: seq_num 1506 ranges {511342=762888,1777529=632518,2973733=337609} 2024-08-17T17:11:47.707 INFO:tasks.rados.rados.0.smithi063.stdout:3994: writing smithi06387500-15 from 511342 to 1274230 tid 1 2024-08-17T17:11:47.714 INFO:tasks.rados.rados.0.smithi063.stdout:3994: writing smithi06387500-15 from 1777529 to 2410047 tid 2 2024-08-17T17:11:47.719 INFO:tasks.rados.rados.0.smithi063.stdout:3994: writing smithi06387500-15 from 2973733 to 3311342 tid 3 2024-08-17T17:11:47.719 INFO:tasks.rados.rados.0.smithi063.stdout:3995: read oid 41 snap -1 2024-08-17T17:11:47.719 INFO:tasks.rados.rados.0.smithi063.stdout:3995: expect (ObjNum 1450 snap 485 seq_num 1450) 2024-08-17T17:11:47.719 INFO:tasks.rados.rados.0.smithi063.stdout:3996: snap_remove snap 502 2024-08-17T17:11:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:47 smithi063 ceph-mon[114756]: osdmap e1079: 8 total, 8 up, 8 in 2024-08-17T17:11:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:47 smithi063 ceph-mon[114756]: osdmap e1080: 8 total, 8 up, 8 in 2024-08-17T17:11:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:47 smithi063 ceph-mon[109843]: osdmap e1079: 8 total, 8 up, 8 in 2024-08-17T17:11:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:47 smithi063 ceph-mon[109843]: osdmap e1080: 8 total, 8 up, 8 in 2024-08-17T17:11:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:47 smithi162 ceph-mon[104668]: osdmap e1079: 8 total, 8 up, 8 in 2024-08-17T17:11:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:47 smithi162 ceph-mon[104668]: osdmap e1080: 8 total, 8 up, 8 in 2024-08-17T17:11:48.732 INFO:tasks.rados.rados.0.smithi063.stdout:3996: done (2 left) 2024-08-17T17:11:48.732 INFO:tasks.rados.rados.0.smithi063.stdout:3997: write oid 30 current snap is 506 2024-08-17T17:11:48.733 INFO:tasks.rados.rados.0.smithi063.stdout:3997: seq_num 1507 ranges {528149=438137,1436011=492529,2601156=667041} 2024-08-17T17:11:48.737 INFO:tasks.rados.rados.0.smithi063.stdout:3997: writing smithi06387500-30 from 528149 to 966286 tid 1 2024-08-17T17:11:48.744 INFO:tasks.rados.rados.0.smithi063.stdout:3997: writing smithi06387500-30 from 1436011 to 1928540 tid 2 2024-08-17T17:11:48.752 INFO:tasks.rados.rados.0.smithi063.stdout:3997: writing smithi06387500-30 from 2601156 to 3268197 tid 3 2024-08-17T17:11:48.752 INFO:tasks.rados.rados.0.smithi063.stdout:3998: write oid 12 current snap is 506 2024-08-17T17:11:48.752 INFO:tasks.rados.rados.0.smithi063.stdout:3994: finishing write tid 1 to smithi06387500-15 2024-08-17T17:11:48.752 INFO:tasks.rados.rados.0.smithi063.stdout:3994: finishing write tid 2 to smithi06387500-15 2024-08-17T17:11:48.753 INFO:tasks.rados.rados.0.smithi063.stdout:3994: finishing write tid 3 to smithi06387500-15 2024-08-17T17:11:48.753 INFO:tasks.rados.rados.0.smithi063.stdout:3994: finishing write tid 4 to smithi06387500-15 2024-08-17T17:11:48.753 INFO:tasks.rados.rados.0.smithi063.stdout:3994: finishing write tid 5 to smithi06387500-15 2024-08-17T17:11:48.753 INFO:tasks.rados.rados.0.smithi063.stdout:3994: oid 15 updating version 0 to 1545 2024-08-17T17:11:48.753 INFO:tasks.rados.rados.0.smithi063.stdout:3994: oid 15 updating version 1545 to 1548 2024-08-17T17:11:48.753 INFO:tasks.rados.rados.0.smithi063.stdout:3994: oid 15 version 1548 is already newer than 1547 2024-08-17T17:11:48.753 INFO:tasks.rados.rados.0.smithi063.stdout:3994: oid 15 version 1548 is already newer than 1546 2024-08-17T17:11:48.753 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 15 v 1548 (ObjNum 1506 snap 506 seq_num 1506) dirty exists 2024-08-17T17:11:48.753 INFO:tasks.rados.rados.0.smithi063.stdout:3994: left oid 15 (ObjNum 1506 snap 506 seq_num 1506) 2024-08-17T17:11:48.753 INFO:tasks.rados.rados.0.smithi063.stdout:3998: seq_num 1508 ranges {544956=513386,1494493=636187,2628579=594666} 2024-08-17T17:11:48.757 INFO:tasks.rados.rados.0.smithi063.stdout:3998: writing smithi06387500-12 from 544956 to 1058342 tid 1 2024-08-17T17:11:48.763 INFO:tasks.rados.rados.0.smithi063.stdout:3998: writing smithi06387500-12 from 1494493 to 2130680 tid 2 2024-08-17T17:11:48.770 INFO:tasks.rados.rados.0.smithi063.stdout:3998: writing smithi06387500-12 from 2628579 to 3223245 tid 3 2024-08-17T17:11:48.770 INFO:tasks.rados.rados.0.smithi063.stdout:3994: done (3 left) 2024-08-17T17:11:48.770 INFO:tasks.rados.rados.0.smithi063.stdout:3999: snap_create 2024-08-17T17:11:48.770 INFO:tasks.rados.rados.0.smithi063.stdout:3995: expect (ObjNum 1450 snap 485 seq_num 1450) 2024-08-17T17:11:48.882 INFO:tasks.rados.rados.0.smithi063.stdout:3997: finishing write tid 1 to smithi06387500-30 2024-08-17T17:11:48.882 INFO:tasks.rados.rados.0.smithi063.stdout:3997: finishing write tid 2 to smithi06387500-30 2024-08-17T17:11:48.882 INFO:tasks.rados.rados.0.smithi063.stdout:3997: finishing write tid 3 to smithi06387500-30 2024-08-17T17:11:48.882 INFO:tasks.rados.rados.0.smithi063.stdout:3997: finishing write tid 4 to smithi06387500-30 2024-08-17T17:11:48.882 INFO:tasks.rados.rados.0.smithi063.stdout:3997: finishing write tid 5 to smithi06387500-30 2024-08-17T17:11:48.882 INFO:tasks.rados.rados.0.smithi063.stdout:3997: oid 30 updating version 0 to 2000 2024-08-17T17:11:48.882 INFO:tasks.rados.rados.0.smithi063.stdout:3997: oid 30 updating version 2000 to 2001 2024-08-17T17:11:48.882 INFO:tasks.rados.rados.0.smithi063.stdout:3997: oid 30 updating version 2001 to 2002 2024-08-17T17:11:48.883 INFO:tasks.rados.rados.0.smithi063.stdout:3997: oid 30 updating version 2002 to 2003 2024-08-17T17:11:48.883 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 30 v 2003 (ObjNum 1507 snap 506 seq_num 1507) dirty exists 2024-08-17T17:11:48.883 INFO:tasks.rados.rados.0.smithi063.stdout:3997: left oid 30 (ObjNum 1507 snap 506 seq_num 1507) 2024-08-17T17:11:48.883 INFO:tasks.rados.rados.0.smithi063.stdout:3998: finishing write tid 1 to smithi06387500-12 2024-08-17T17:11:48.883 INFO:tasks.rados.rados.0.smithi063.stdout:3998: finishing write tid 2 to smithi06387500-12 2024-08-17T17:11:48.883 INFO:tasks.rados.rados.0.smithi063.stdout:3998: finishing write tid 3 to smithi06387500-12 2024-08-17T17:11:48.883 INFO:tasks.rados.rados.0.smithi063.stdout:3998: finishing write tid 4 to smithi06387500-12 2024-08-17T17:11:48.883 INFO:tasks.rados.rados.0.smithi063.stdout:3998: finishing write tid 5 to smithi06387500-12 2024-08-17T17:11:48.883 INFO:tasks.rados.rados.0.smithi063.stdout:3998: oid 12 updating version 0 to 865 2024-08-17T17:11:48.883 INFO:tasks.rados.rados.0.smithi063.stdout:3998: oid 12 updating version 865 to 866 2024-08-17T17:11:48.884 INFO:tasks.rados.rados.0.smithi063.stdout:3998: oid 12 version 866 is already newer than 864 2024-08-17T17:11:48.884 INFO:tasks.rados.rados.0.smithi063.stdout:3998: oid 12 version 866 is already newer than 863 2024-08-17T17:11:48.884 INFO:tasks.rados.rados.0.smithi063.stdout:update_object_version oid 12 v 866 (ObjNum 1508 snap 506 seq_num 1508) dirty exists 2024-08-17T17:11:48.884 INFO:tasks.rados.rados.0.smithi063.stdout:3998: left oid 12 (ObjNum 1508 snap 506 seq_num 1508) 2024-08-17T17:11:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:48 smithi063 ceph-mon[114756]: pgmap v1077: 89 pgs: 89 active+clean; 177 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 766 KiB/s wr, 12 op/s 2024-08-17T17:11:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:48 smithi063 ceph-mon[109843]: pgmap v1077: 89 pgs: 89 active+clean; 177 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 766 KiB/s wr, 12 op/s 2024-08-17T17:11:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:48 smithi162 ceph-mon[104668]: pgmap v1077: 89 pgs: 89 active+clean; 177 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 766 KiB/s wr, 12 op/s 2024-08-17T17:11:49.733 INFO:tasks.rados.rados.0.smithi063.stdout:3995: done (3 left) 2024-08-17T17:11:49.733 INFO:tasks.rados.rados.0.smithi063.stdout:3997: done (2 left) 2024-08-17T17:11:49.733 INFO:tasks.rados.rados.0.smithi063.stdout:3998: done (1 left) 2024-08-17T17:11:49.733 INFO:tasks.rados.rados.0.smithi063.stdout:3999: done (0 left) 2024-08-17T17:11:49.735 INFO:tasks.rados.rados.0.smithi063.stderr:0 errors. 2024-08-17T17:11:49.735 INFO:tasks.rados.rados.0.smithi063.stderr: 2024-08-17T17:11:49.739 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:11:49.962 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:11:50.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:49 smithi063 ceph-mon[109843]: osdmap e1081: 8 total, 8 up, 8 in 2024-08-17T17:11:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:49 smithi063 ceph-mon[114756]: osdmap e1081: 8 total, 8 up, 8 in 2024-08-17T17:11:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:49 smithi162 ceph-mon[104668]: osdmap e1081: 8 total, 8 up, 8 in 2024-08-17T17:11:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:50 smithi063 ceph-mon[114756]: osdmap e1082: 8 total, 8 up, 8 in 2024-08-17T17:11:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:50 smithi063 ceph-mon[114756]: pgmap v1080: 89 pgs: 89 active+clean; 180 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 773 KiB/s wr, 12 op/s 2024-08-17T17:11:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:50 smithi063 ceph-mon[109843]: osdmap e1082: 8 total, 8 up, 8 in 2024-08-17T17:11:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:50 smithi063 ceph-mon[109843]: pgmap v1080: 89 pgs: 89 active+clean; 180 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 773 KiB/s wr, 12 op/s 2024-08-17T17:11:51.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:50 smithi162 ceph-mon[104668]: osdmap e1082: 8 total, 8 up, 8 in 2024-08-17T17:11:51.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:50 smithi162 ceph-mon[104668]: pgmap v1080: 89 pgs: 89 active+clean; 180 MiB data, 809 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 773 KiB/s wr, 12 op/s 2024-08-17T17:11:51.845 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:11:51.845 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":1082,"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:46:54.156006+0000","modified":"2024-08-17T17:11:49.726349+0000","last_up_change":"2024-08-17T17:07:44.658208+0000","last_in_change":"2024-08-17T16:51:30.856523+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:49:54.431240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:52:22.131465+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T16:53:35.328125+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T16:53:36.292966+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"unique_pool_0","create_time":"2024-08-17T16:54:47.092633+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1082","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":641,"snap_epoch":1082,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.5,"score_stable":2.5,"optimal_score":1,"raw_score_acting":2.5,"raw_score_stable":2.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":726,"up_from":739,"up_thru":837,"down_at":727,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6802","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6803","nonce":4285380350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6804","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6805","nonce":4285380350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6808","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6809","nonce":4285380350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6806","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6807","nonce":4285380350}]},"public_addr":"172.21.15.63:6803/4285380350","cluster_addr":"172.21.15.63:6805/4285380350","heartbeat_back_addr":"172.21.15.63:6809/4285380350","heartbeat_front_addr":"172.21.15.63:6807/4285380350","state":["exists","up"]},{"osd":1,"uuid":"ae346e99-4706-4871-990a-f8e737572245","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":758,"up_from":771,"up_thru":837,"down_at":759,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6810","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6811","nonce":3170762701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6812","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6813","nonce":3170762701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6816","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6817","nonce":3170762701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6814","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6815","nonce":3170762701}]},"public_addr":"172.21.15.63:6811/3170762701","cluster_addr":"172.21.15.63:6813/3170762701","heartbeat_back_addr":"172.21.15.63:6817/3170762701","heartbeat_front_addr":"172.21.15.63:6815/3170762701","state":["exists","up"]},{"osd":2,"uuid":"2d85d75f-1280-443f-857d-d322bd60c451","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":791,"up_from":804,"up_thru":837,"down_at":792,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6818","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6819","nonce":3903658295}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6820","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6821","nonce":3903658295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6824","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6825","nonce":3903658295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6822","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6823","nonce":3903658295}]},"public_addr":"172.21.15.63:6819/3903658295","cluster_addr":"172.21.15.63:6821/3903658295","heartbeat_back_addr":"172.21.15.63:6825/3903658295","heartbeat_front_addr":"172.21.15.63:6823/3903658295","state":["exists","up"]},{"osd":3,"uuid":"771f43d2-e98b-43e0-921b-88e56fadda42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":824,"up_from":837,"up_thru":837,"down_at":825,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6826","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6827","nonce":1764356824}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6828","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6829","nonce":1764356824}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6832","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6833","nonce":1764356824}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6830","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6831","nonce":1764356824}]},"public_addr":"172.21.15.63:6827/1764356824","cluster_addr":"172.21.15.63:6829/1764356824","heartbeat_back_addr":"172.21.15.63:6833/1764356824","heartbeat_front_addr":"172.21.15.63:6831/1764356824","state":["exists","up"]},{"osd":4,"uuid":"73c3c545-e975-4a6f-a1cb-e4575ee7a9b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":837,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6800","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6801","nonce":1563885208}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6803","nonce":1563885208}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6807","nonce":1563885208}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6805","nonce":1563885208}]},"public_addr":"172.21.15.162:6801/1563885208","cluster_addr":"172.21.15.162:6803/1563885208","heartbeat_back_addr":"172.21.15.162:6807/1563885208","heartbeat_front_addr":"172.21.15.162:6805/1563885208","state":["exists","up"]},{"osd":5,"uuid":"fae3ac52-3194-4eb4-9510-e3899768353a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":837,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6809","nonce":1826502179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6810","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6811","nonce":1826502179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6814","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6815","nonce":1826502179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6812","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6813","nonce":1826502179}]},"public_addr":"172.21.15.162:6809/1826502179","cluster_addr":"172.21.15.162:6811/1826502179","heartbeat_back_addr":"172.21.15.162:6815/1826502179","heartbeat_front_addr":"172.21.15.162:6813/1826502179","state":["exists","up"]},{"osd":6,"uuid":"a8255b37-1be5-4cd4-9b86-889f6bf52902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":825,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6816","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6817","nonce":1675904899}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6818","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6819","nonce":1675904899}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6822","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6823","nonce":1675904899}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6820","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6821","nonce":1675904899}]},"public_addr":"172.21.15.162:6817/1675904899","cluster_addr":"172.21.15.162:6819/1675904899","heartbeat_back_addr":"172.21.15.162:6823/1675904899","heartbeat_front_addr":"172.21.15.162:6821/1675904899","state":["exists","up"]},{"osd":7,"uuid":"3d0bcd6d-171d-4d90-a033-ed17714a9d96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":837,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6824","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6825","nonce":2938230961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6826","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6827","nonce":2938230961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6830","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6831","nonce":2938230961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6828","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6829","nonce":2938230961}]},"public_addr":"172.21.15.162:6825/2938230961","cluster_addr":"172.21.15.162:6827/2938230961","heartbeat_back_addr":"172.21.15.162:6831/2938230961","heartbeat_front_addr":"172.21.15.162:6829/2938230961","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:02.667962+0000","dead_epoch":726},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:26.166590+0000","dead_epoch":758},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:50.153794+0000","dead_epoch":791},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:13.556156+0000","dead_epoch":824},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:34.827805+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:58.789701+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:22.650398+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:46.166424+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.63:6801/4052036587":"2024-08-18T16:59:45.829424+0000","172.21.15.63:6800/4052036587":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/753392608":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/701508812":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/2384509117":"2024-08-18T16:59:45.829424+0000","172.21.15.63:6800/987519051":"2024-08-18T16:56:38.413263+0000","172.21.15.162:0/810332452":"2024-08-18T16:57:08.874093+0000","172.21.15.162:0/1341643866":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/3746443558":"2024-08-18T16:56:38.413263+0000","172.21.15.63:0/3589494813":"2024-08-18T16:56:38.413263+0000","172.21.15.63:6800/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:0/2696979598":"2024-08-18T16:56:38.413263+0000","172.21.15.162:0/319999013":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/84364348":"2024-08-18T16:47:16.384724+0000","172.21.15.162:0/4017736715":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/2404156937":"2024-08-18T16:59:45.829424+0000","172.21.15.63:6801/987519051":"2024-08-18T16:56:38.413263+0000","172.21.15.63:0/955244129":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/664643590":"2024-08-18T16:59:45.829424+0000","172.21.15.63:6800/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.63:6801/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.63:6800/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.162:6834/3390906977":"2024-08-18T16:57:08.874093+0000","172.21.15.63:6801/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.162:6835/3390906977":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/1986138219":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/4067771896":"2024-08-18T16:56:38.413263+0000","172.21.15.63:6801/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:0/1256300214":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/1792393387":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3226753153":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3004218255":"2024-08-18T16:47:16.384724+0000","172.21.15.162:0/2317499213":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/2601197458":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/3540337558":"2024-08-18T16:47:16.384724+0000","172.21.15.162:0/2288037041":"2024-08-18T16:57:08.874093+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":634,"length":1},{"begin":637,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:11:52.148 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:51 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/591143361' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:11:52.148 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:51 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/591143361' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:11:52.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:51 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/591143361' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:11:52.423 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:11:52.638 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:11:53.463 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:11:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:11:53] "GET /metrics HTTP/1.1" 200 36337 "" "Prometheus/2.51.0" 2024-08-17T17:11:53.764 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:53 smithi063 ceph-mon[114756]: pgmap v1081: 89 pgs: 89 active+clean; 191 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-08-17T17:11:53.764 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:53 smithi063 ceph-mon[114756]: osdmap e1083: 8 total, 8 up, 8 in 2024-08-17T17:11:53.764 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:53 smithi063 ceph-mon[109843]: pgmap v1081: 89 pgs: 89 active+clean; 191 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-08-17T17:11:53.764 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:53 smithi063 ceph-mon[109843]: osdmap e1083: 8 total, 8 up, 8 in 2024-08-17T17:11:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:53 smithi162 ceph-mon[104668]: pgmap v1081: 89 pgs: 89 active+clean; 191 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-08-17T17:11:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:53 smithi162 ceph-mon[104668]: osdmap e1083: 8 total, 8 up, 8 in 2024-08-17T17:11:54.569 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:11:54.570 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:11:55.144 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1083,"stamp":"2024-08-17T17:11:54.020287+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":199766208,"num_objects":120,"num_object_clones":50,"num_object_copies":360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":120,"num_whiteouts":5,"num_read":7271,"num_read_kb":7361170,"num_write":5945,"num_write_kb":1353695,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":59,"num_bytes_recovered":51994489,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":22563,"ondisk_log_size":22563,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":850596,"kb_used_data":438004,"kb_used_omap":199,"kb_used_meta":412280,"kb_avail":749110620,"statfs":{"total":767960285184,"available":767089274880,"internally_reserved":0,"allocated":448516096,"data_stored":518542910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":422175484},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":13804351,"num_objects":5,"num_object_clones":4,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":-1,"num_read":27,"num_read_kb":14488,"num_write":27,"num_write_kb":7793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001372"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1242,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.013137+0000","last_change":"2024-08-17T17:07:45.684223+0000","last_active":"2024-08-17T17:11:53.013137+0000","last_peered":"2024-08-17T17:11:53.013137+0000","last_clean":"2024-08-17T17:11:53.013137+0000","last_became_active":"2024-08-17T17:07:45.683948+0000","last_became_peered":"2024-08-17T17:07:45.683948+0000","last_unstale":"2024-08-17T17:11:53.013137+0000","last_undegraded":"2024-08-17T17:11:53.013137+0000","last_fullsized":"2024-08-17T17:11:53.013137+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:07.677423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1005,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.736950+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:11:48.736950+0000","last_peered":"2024-08-17T17:11:48.736950+0000","last_clean":"2024-08-17T17:11:48.736950+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:11:48.736950+0000","last_undegraded":"2024-08-17T17:11:48.736950+0000","last_fullsized":"2024-08-17T17:11:48.736950+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1017,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.731681+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:11:48.731681+0000","last_peered":"2024-08-17T17:11:48.731681+0000","last_clean":"2024-08-17T17:11:48.731681+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:11:48.731681+0000","last_undegraded":"2024-08-17T17:11:48.731681+0000","last_fullsized":"2024-08-17T17:11:48.731681+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1362,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.847196+0000","last_change":"2024-08-17T17:07:45.683455+0000","last_active":"2024-08-17T17:11:49.847196+0000","last_peered":"2024-08-17T17:11:49.847196+0000","last_clean":"2024-08-17T17:11:49.847196+0000","last_became_active":"2024-08-17T17:07:45.683169+0000","last_became_peered":"2024-08-17T17:07:45.683169+0000","last_unstale":"2024-08-17T17:11:49.847196+0000","last_undegraded":"2024-08-17T17:11:49.847196+0000","last_fullsized":"2024-08-17T17:11:49.847196+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:18:19.562247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1321,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.847584+0000","last_change":"2024-08-17T17:07:45.684222+0000","last_active":"2024-08-17T17:11:49.847584+0000","last_peered":"2024-08-17T17:11:49.847584+0000","last_clean":"2024-08-17T17:11:49.847584+0000","last_became_active":"2024-08-17T17:07:45.683913+0000","last_became_peered":"2024-08-17T17:07:45.683913+0000","last_unstale":"2024-08-17T17:11:49.847584+0000","last_undegraded":"2024-08-17T17:11:49.847584+0000","last_fullsized":"2024-08-17T17:11:49.847584+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:59:33.944853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":988,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.738113+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:11:48.738113+0000","last_peered":"2024-08-17T17:11:48.738113+0000","last_clean":"2024-08-17T17:11:48.738113+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:11:48.738113+0000","last_undegraded":"2024-08-17T17:11:48.738113+0000","last_fullsized":"2024-08-17T17:11:48.738113+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1242,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011384+0000","last_change":"2024-08-17T17:07:45.687030+0000","last_active":"2024-08-17T17:11:53.011384+0000","last_peered":"2024-08-17T17:11:53.011384+0000","last_clean":"2024-08-17T17:11:53.011384+0000","last_became_active":"2024-08-17T17:07:45.686610+0000","last_became_peered":"2024-08-17T17:07:45.686610+0000","last_unstale":"2024-08-17T17:11:53.011384+0000","last_undegraded":"2024-08-17T17:11:53.011384+0000","last_fullsized":"2024-08-17T17:11:53.011384+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:02.425845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1027,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.729228+0000","last_change":"2024-08-17T17:06:07.632931+0000","last_active":"2024-08-17T17:11:48.729228+0000","last_peered":"2024-08-17T17:11:48.729228+0000","last_clean":"2024-08-17T17:11:48.729228+0000","last_became_active":"2024-08-17T17:06:07.632525+0000","last_became_peered":"2024-08-17T17:06:07.632525+0000","last_unstale":"2024-08-17T17:11:48.729228+0000","last_undegraded":"2024-08-17T17:11:48.729228+0000","last_fullsized":"2024-08-17T17:11:48.729228+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:01.740674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.9","version":"1081'470","reported_seq":3634,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.740967+0000","last_change":"2024-08-17T17:11:48.740967+0000","last_active":"2024-08-17T17:11:48.740967+0000","last_peered":"2024-08-17T17:11:48.740967+0000","last_clean":"2024-08-17T17:11:48.740967+0000","last_became_active":"2024-08-17T16:54:48.157754+0000","last_became_peered":"2024-08-17T16:54:48.157754+0000","last_unstale":"2024-08-17T17:11:48.740967+0000","last_undegraded":"2024-08-17T17:11:48.740967+0000","last_fullsized":"2024-08-17T17:11:48.740967+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":470,"log_dups_size":0,"ondisk_log_size":470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:10:21.029020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002984113,"stat_sum":{"num_bytes":3267115,"num_objects":3,"num_object_clones":2,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":192,"num_read_kb":207582,"num_write":131,"num_write_kb":30727,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"27a","length":"1"},{"start":"27d","length":"1"}]},{"pgid":"3.f","version":"72'1","reported_seq":1005,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.737501+0000","last_change":"2024-08-17T17:06:07.638468+0000","last_active":"2024-08-17T17:11:48.737501+0000","last_peered":"2024-08-17T17:11:48.737501+0000","last_clean":"2024-08-17T17:11:48.737501+0000","last_became_active":"2024-08-17T17:06:07.636046+0000","last_became_peered":"2024-08-17T17:06:07.636046+0000","last_unstale":"2024-08-17T17:11:48.737501+0000","last_undegraded":"2024-08-17T17:11:48.737501+0000","last_fullsized":"2024-08-17T17:11:48.737501+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:37:12.228090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1003,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.731348+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:11:48.731348+0000","last_peered":"2024-08-17T17:11:48.731348+0000","last_clean":"2024-08-17T17:11:48.731348+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:11:48.731348+0000","last_undegraded":"2024-08-17T17:11:48.731348+0000","last_fullsized":"2024-08-17T17:11:48.731348+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"1073'1035","reported_seq":4660,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.730057+0000","last_change":"2024-08-17T17:11:48.759925+0000","last_active":"2024-08-17T17:11:49.730057+0000","last_peered":"2024-08-17T17:11:49.730057+0000","last_clean":"2024-08-17T17:11:49.730057+0000","last_became_active":"2024-08-17T17:07:12.649612+0000","last_became_peered":"2024-08-17T17:07:12.649612+0000","last_unstale":"2024-08-17T17:11:49.730057+0000","last_undegraded":"2024-08-17T17:11:49.730057+0000","last_fullsized":"2024-08-17T17:11:49.730057+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":1035,"log_dups_size":0,"ondisk_log_size":1035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:59:49.390776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058424499999999995,"stat_sum":{"num_bytes":13164251,"num_objects":6,"num_object_clones":3,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":277,"num_read_kb":258426,"num_write":273,"num_write_kb":62469,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":8905339,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"27a","length":"1"},{"start":"27d","length":"1"}]},{"pgid":"3.e","version":"72'2","reported_seq":1006,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.737460+0000","last_change":"2024-08-17T17:06:39.638433+0000","last_active":"2024-08-17T17:11:48.737460+0000","last_peered":"2024-08-17T17:11:48.737460+0000","last_clean":"2024-08-17T17:11:48.737460+0000","last_became_active":"2024-08-17T17:06:39.638081+0000","last_became_peered":"2024-08-17T17:06:39.638081+0000","last_unstale":"2024-08-17T17:11:48.737460+0000","last_undegraded":"2024-08-17T17:11:48.737460+0000","last_fullsized":"2024-08-17T17:11:48.737460+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:13.558012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1037,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.735032+0000","last_change":"2024-08-17T17:07:45.693574+0000","last_active":"2024-08-17T17:11:48.735032+0000","last_peered":"2024-08-17T17:11:48.735032+0000","last_clean":"2024-08-17T17:11:48.735032+0000","last_became_active":"2024-08-17T17:07:45.693086+0000","last_became_peered":"2024-08-17T17:07:45.693086+0000","last_unstale":"2024-08-17T17:11:48.735032+0000","last_undegraded":"2024-08-17T17:11:48.735032+0000","last_fullsized":"2024-08-17T17:11:48.735032+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:31.888894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"1081'866","reported_seq":4505,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.730408+0000","last_change":"2024-08-17T17:11:48.761620+0000","last_active":"2024-08-17T17:11:49.730408+0000","last_peered":"2024-08-17T17:11:49.730408+0000","last_clean":"2024-08-17T17:11:49.730408+0000","last_became_active":"2024-08-17T17:07:12.649279+0000","last_became_peered":"2024-08-17T17:07:12.649279+0000","last_unstale":"2024-08-17T17:11:49.730408+0000","last_undegraded":"2024-08-17T17:11:49.730408+0000","last_fullsized":"2024-08-17T17:11:49.730408+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":866,"log_dups_size":0,"ondisk_log_size":866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:20:49.932063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018726319999999999,"stat_sum":{"num_bytes":11553913,"num_objects":4,"num_object_clones":2,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":283,"num_read_kb":286757,"num_write":234,"num_write_kb":56597,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":1520568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"27a","length":"1"},{"start":"27d","length":"1"}]},{"pgid":"3.d","version":"72'1","reported_seq":990,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.737275+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:11:48.737275+0000","last_peered":"2024-08-17T17:11:48.737275+0000","last_clean":"2024-08-17T17:11:48.737275+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:11:48.737275+0000","last_undegraded":"2024-08-17T17:11:48.737275+0000","last_fullsized":"2024-08-17T17:11:48.737275+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1019,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.729476+0000","last_change":"2024-08-17T17:06:39.629368+0000","last_active":"2024-08-17T17:11:48.729476+0000","last_peered":"2024-08-17T17:11:48.729476+0000","last_clean":"2024-08-17T17:11:48.729476+0000","last_became_active":"2024-08-17T17:06:39.628832+0000","last_became_peered":"2024-08-17T17:06:39.628832+0000","last_unstale":"2024-08-17T17:11:48.729476+0000","last_undegraded":"2024-08-17T17:11:48.729476+0000","last_fullsized":"2024-08-17T17:11:48.729476+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:54.063884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"1081'1550","reported_seq":5146,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.730294+0000","last_change":"2024-08-17T17:11:48.763493+0000","last_active":"2024-08-17T17:11:49.730294+0000","last_peered":"2024-08-17T17:11:49.730294+0000","last_clean":"2024-08-17T17:11:49.730294+0000","last_became_active":"2024-08-17T17:07:45.685028+0000","last_became_peered":"2024-08-17T17:07:45.685028+0000","last_unstale":"2024-08-17T17:11:49.730294+0000","last_undegraded":"2024-08-17T17:11:49.730294+0000","last_fullsized":"2024-08-17T17:11:49.730294+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":1550,"log_dups_size":0,"ondisk_log_size":1550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:23:03.542679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0033147189999999998,"stat_sum":{"num_bytes":11385038,"num_objects":7,"num_object_clones":4,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":1,"num_read":523,"num_read_kb":573604,"num_write":403,"num_write_kb":102192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":5490077,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"27a","length":"1"},{"start":"27d","length":"1"}]},{"pgid":"3.c","version":"0'0","reported_seq":1019,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.731593+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:11:48.731593+0000","last_peered":"2024-08-17T17:11:48.731593+0000","last_clean":"2024-08-17T17:11:48.731593+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:11:48.731593+0000","last_undegraded":"2024-08-17T17:11:48.731593+0000","last_fullsized":"2024-08-17T17:11:48.731593+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1334,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.847170+0000","last_change":"2024-08-17T17:07:12.651379+0000","last_active":"2024-08-17T17:11:49.847170+0000","last_peered":"2024-08-17T17:11:49.847170+0000","last_clean":"2024-08-17T17:11:49.847170+0000","last_became_active":"2024-08-17T17:07:12.651069+0000","last_became_peered":"2024-08-17T17:07:12.651069+0000","last_unstale":"2024-08-17T17:11:49.847170+0000","last_undegraded":"2024-08-17T17:11:49.847170+0000","last_fullsized":"2024-08-17T17:11:49.847170+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:25.384305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"1070'385","reported_seq":4020,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.730236+0000","last_change":"2024-08-17T17:11:48.760073+0000","last_active":"2024-08-17T17:11:49.730236+0000","last_peered":"2024-08-17T17:11:49.730236+0000","last_clean":"2024-08-17T17:11:49.730236+0000","last_became_active":"2024-08-17T17:07:12.645705+0000","last_became_peered":"2024-08-17T17:07:12.645705+0000","last_unstale":"2024-08-17T17:11:49.730236+0000","last_undegraded":"2024-08-17T17:11:49.730236+0000","last_fullsized":"2024-08-17T17:11:49.730236+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":385,"log_dups_size":0,"ondisk_log_size":385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:09:27.730890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050570599999999997,"stat_sum":{"num_bytes":3580221,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":89,"num_read_kb":88034,"num_write":110,"num_write_kb":28457,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":875814,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"27a","length":"1"},{"start":"27d","length":"1"}]},{"pgid":"3.b","version":"72'2","reported_seq":1255,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011227+0000","last_change":"2024-08-17T17:07:45.691437+0000","last_active":"2024-08-17T17:11:53.011227+0000","last_peered":"2024-08-17T17:11:53.011227+0000","last_clean":"2024-08-17T17:11:53.011227+0000","last_became_active":"2024-08-17T17:07:45.690567+0000","last_became_peered":"2024-08-17T17:07:45.690567+0000","last_unstale":"2024-08-17T17:11:53.011227+0000","last_undegraded":"2024-08-17T17:11:53.011227+0000","last_fullsized":"2024-08-17T17:11:53.011227+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:43:07.698912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1022,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.734559+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:11:48.734559+0000","last_peered":"2024-08-17T17:11:48.734559+0000","last_clean":"2024-08-17T17:11:48.734559+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:11:48.734559+0000","last_undegraded":"2024-08-17T17:11:48.734559+0000","last_fullsized":"2024-08-17T17:11:48.734559+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"1075'1351","reported_seq":4997,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:50.539256+0000","last_change":"2024-08-17T17:11:48.743032+0000","last_active":"2024-08-17T17:11:50.539256+0000","last_peered":"2024-08-17T17:11:50.539256+0000","last_clean":"2024-08-17T17:11:50.539256+0000","last_became_active":"2024-08-17T17:06:39.630939+0000","last_became_peered":"2024-08-17T17:06:39.630939+0000","last_unstale":"2024-08-17T17:11:50.539256+0000","last_undegraded":"2024-08-17T17:11:50.539256+0000","last_fullsized":"2024-08-17T17:11:50.539256+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":1351,"log_dups_size":0,"ondisk_log_size":1351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:50:17.366192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00093716199999999996,"stat_sum":{"num_bytes":13896495,"num_objects":7,"num_object_clones":4,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":393,"num_read_kb":430285,"num_write":353,"num_write_kb":83171,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":1125942,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"27a","length":"1"},{"start":"27d","length":"1"}]},{"pgid":"3.a","version":"0'0","reported_seq":1020,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.729625+0000","last_change":"2024-08-17T17:06:39.629619+0000","last_active":"2024-08-17T17:11:48.729625+0000","last_peered":"2024-08-17T17:11:48.729625+0000","last_clean":"2024-08-17T17:11:48.729625+0000","last_became_active":"2024-08-17T17:06:39.629295+0000","last_became_peered":"2024-08-17T17:06:39.629295+0000","last_unstale":"2024-08-17T17:11:48.729625+0000","last_undegraded":"2024-08-17T17:11:48.729625+0000","last_fullsized":"2024-08-17T17:11:48.729625+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:15.874423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1037,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.734824+0000","last_change":"2024-08-17T17:07:12.649178+0000","last_active":"2024-08-17T17:11:48.734824+0000","last_peered":"2024-08-17T17:11:48.734824+0000","last_clean":"2024-08-17T17:11:48.734824+0000","last_became_active":"2024-08-17T17:07:12.648844+0000","last_became_peered":"2024-08-17T17:07:12.648844+0000","last_unstale":"2024-08-17T17:11:48.734824+0000","last_undegraded":"2024-08-17T17:11:48.734824+0000","last_fullsized":"2024-08-17T17:11:48.734824+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:12:06.307736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"1081'1416","reported_seq":4491,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.735214+0000","last_change":"2024-08-17T17:11:48.735214+0000","last_active":"2024-08-17T17:11:48.735214+0000","last_peered":"2024-08-17T17:11:48.735214+0000","last_clean":"2024-08-17T17:11:48.735214+0000","last_became_active":"2024-08-17T16:54:48.157755+0000","last_became_peered":"2024-08-17T16:54:48.157755+0000","last_unstale":"2024-08-17T17:11:48.735214+0000","last_undegraded":"2024-08-17T17:11:48.735214+0000","last_fullsized":"2024-08-17T17:11:48.735214+0000","mapping_epoch":75,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":76,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":1416,"log_dups_size":0,"ondisk_log_size":1416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:56:34.523763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0032462290000000002,"stat_sum":{"num_bytes":9101576,"num_objects":5,"num_object_clones":3,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":396,"num_read_kb":411537,"num_write":351,"num_write_kb":85486,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"27a","length":"1"},{"start":"27d","length":"1"}]},{"pgid":"3.9","version":"0'0","reported_seq":1023,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.735026+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:11:48.735026+0000","last_peered":"2024-08-17T17:11:48.735026+0000","last_clean":"2024-08-17T17:11:48.735026+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:11:48.735026+0000","last_undegraded":"2024-08-17T17:11:48.735026+0000","last_fullsized":"2024-08-17T17:11:48.735026+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1022,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.734930+0000","last_change":"2024-08-17T17:06:39.627678+0000","last_active":"2024-08-17T17:11:48.734930+0000","last_peered":"2024-08-17T17:11:48.734930+0000","last_clean":"2024-08-17T17:11:48.734930+0000","last_became_active":"2024-08-17T17:06:39.627226+0000","last_became_peered":"2024-08-17T17:06:39.627226+0000","last_unstale":"2024-08-17T17:11:48.734930+0000","last_undegraded":"2024-08-17T17:11:48.734930+0000","last_fullsized":"2024-08-17T17:11:48.734930+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:01.868244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"1075'605","reported_seq":3798,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.735272+0000","last_change":"2024-08-17T17:11:48.735272+0000","last_active":"2024-08-17T17:11:48.735272+0000","last_peered":"2024-08-17T17:11:48.735272+0000","last_clean":"2024-08-17T17:11:48.735272+0000","last_became_active":"2024-08-17T17:06:07.638235+0000","last_became_peered":"2024-08-17T17:06:07.638235+0000","last_unstale":"2024-08-17T17:11:48.735272+0000","last_undegraded":"2024-08-17T17:11:48.735272+0000","last_fullsized":"2024-08-17T17:11:48.735272+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":605,"log_dups_size":0,"ondisk_log_size":605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:11.893057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049059699999999995,"stat_sum":{"num_bytes":7645443,"num_objects":5,"num_object_clones":4,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":163,"num_read_kb":166859,"num_write":168,"num_write_kb":42924,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":957496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"27a","length":"1"},{"start":"27d","length":"1"}]},{"pgid":"3.8","version":"0'0","reported_seq":1244,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011506+0000","last_change":"2024-08-17T17:07:45.680458+0000","last_active":"2024-08-17T17:11:53.011506+0000","last_peered":"2024-08-17T17:11:53.011506+0000","last_clean":"2024-08-17T17:11:53.011506+0000","last_became_active":"2024-08-17T17:07:45.680221+0000","last_became_peered":"2024-08-17T17:07:45.680221+0000","last_unstale":"2024-08-17T17:11:53.011506+0000","last_undegraded":"2024-08-17T17:11:53.011506+0000","last_fullsized":"2024-08-17T17:11:53.011506+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:55.083962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1265,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.730628+0000","last_change":"2024-08-17T17:07:45.682866+0000","last_active":"2024-08-17T17:11:49.730628+0000","last_peered":"2024-08-17T17:11:49.730628+0000","last_clean":"2024-08-17T17:11:49.730628+0000","last_became_active":"2024-08-17T17:07:45.682554+0000","last_became_peered":"2024-08-17T17:07:45.682554+0000","last_unstale":"2024-08-17T17:11:49.730628+0000","last_undegraded":"2024-08-17T17:11:49.730628+0000","last_fullsized":"2024-08-17T17:11:49.730628+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:25.975914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.4","version":"1073'1024","reported_seq":4115,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.737518+0000","last_change":"2024-08-17T17:11:48.737518+0000","last_active":"2024-08-17T17:11:48.737518+0000","last_peered":"2024-08-17T17:11:48.737518+0000","last_clean":"2024-08-17T17:11:48.737518+0000","last_became_active":"2024-08-17T17:07:12.648961+0000","last_became_peered":"2024-08-17T17:07:12.648961+0000","last_unstale":"2024-08-17T17:11:48.737518+0000","last_undegraded":"2024-08-17T17:11:48.737518+0000","last_fullsized":"2024-08-17T17:11:48.737518+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":1024,"log_dups_size":0,"ondisk_log_size":1024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:57.869738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050211899999999998,"stat_sum":{"num_bytes":9043189,"num_objects":5,"num_object_clones":3,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":1,"num_read":250,"num_read_kb":259288,"num_write":251,"num_write_kb":58217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":2511693,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"27a","length":"1"},{"start":"27d","length":"1"}]},{"pgid":"1.0","version":"369'149","reported_seq":1335,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.736830+0000","last_change":"2024-08-17T17:06:07.636484+0000","last_active":"2024-08-17T17:11:48.736830+0000","last_peered":"2024-08-17T17:11:48.736830+0000","last_clean":"2024-08-17T17:11:48.736830+0000","last_became_active":"2024-08-17T17:06:07.635998+0000","last_became_peered":"2024-08-17T17:06:07.635998+0000","last_unstale":"2024-08-17T17:11:48.736830+0000","last_undegraded":"2024-08-17T17:11:48.736830+0000","last_fullsized":"2024-08-17T17:11:48.736830+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:16.096587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1033,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.731686+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:11:48.731686+0000","last_peered":"2024-08-17T17:11:48.731686+0000","last_clean":"2024-08-17T17:11:48.731686+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:11:48.731686+0000","last_undegraded":"2024-08-17T17:11:48.731686+0000","last_fullsized":"2024-08-17T17:11:48.731686+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1224,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011400+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:11:53.011400+0000","last_peered":"2024-08-17T17:11:53.011400+0000","last_clean":"2024-08-17T17:11:53.011400+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:11:53.011400+0000","last_undegraded":"2024-08-17T17:11:53.011400+0000","last_fullsized":"2024-08-17T17:11:53.011400+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1004,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.729653+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:11:48.729653+0000","last_peered":"2024-08-17T17:11:48.729653+0000","last_clean":"2024-08-17T17:11:48.729653+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:11:48.729653+0000","last_undegraded":"2024-08-17T17:11:48.729653+0000","last_fullsized":"2024-08-17T17:11:48.729653+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.7","version":"1081'2003","reported_seq":5081,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.756338+0000","last_change":"2024-08-17T17:11:48.732455+0000","last_active":"2024-08-17T17:11:48.756338+0000","last_peered":"2024-08-17T17:11:48.756338+0000","last_clean":"2024-08-17T17:11:48.756338+0000","last_became_active":"2024-08-17T17:06:39.633904+0000","last_became_peered":"2024-08-17T17:06:39.633904+0000","last_unstale":"2024-08-17T17:11:48.756338+0000","last_undegraded":"2024-08-17T17:11:48.756338+0000","last_fullsized":"2024-08-17T17:11:48.756338+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":2003,"log_dups_size":0,"ondisk_log_size":2003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:31.308574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028459900000000002,"stat_sum":{"num_bytes":17713882,"num_objects":8,"num_object_clones":5,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":635,"num_read_kb":655914,"num_write":486,"num_write_kb":109428,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":6285540,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"27a","length":"1"},{"start":"27d","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":1032,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.736828+0000","last_change":"2024-08-17T17:06:39.638376+0000","last_active":"2024-08-17T17:11:48.736828+0000","last_peered":"2024-08-17T17:11:48.736828+0000","last_clean":"2024-08-17T17:11:48.736828+0000","last_became_active":"2024-08-17T17:06:39.638050+0000","last_became_peered":"2024-08-17T17:06:39.638050+0000","last_unstale":"2024-08-17T17:11:48.736828+0000","last_undegraded":"2024-08-17T17:11:48.736828+0000","last_fullsized":"2024-08-17T17:11:48.736828+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:05.782492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1362,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.847215+0000","last_change":"2024-08-17T17:07:45.683556+0000","last_active":"2024-08-17T17:11:49.847215+0000","last_peered":"2024-08-17T17:11:49.847215+0000","last_clean":"2024-08-17T17:11:49.847215+0000","last_became_active":"2024-08-17T17:07:45.683232+0000","last_became_peered":"2024-08-17T17:07:45.683232+0000","last_unstale":"2024-08-17T17:11:49.847215+0000","last_undegraded":"2024-08-17T17:11:49.847215+0000","last_fullsized":"2024-08-17T17:11:49.847215+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:16.112417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1334,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.847483+0000","last_change":"2024-08-17T17:07:12.651145+0000","last_active":"2024-08-17T17:11:49.847483+0000","last_peered":"2024-08-17T17:11:49.847483+0000","last_clean":"2024-08-17T17:11:49.847483+0000","last_became_active":"2024-08-17T17:07:12.650839+0000","last_became_peered":"2024-08-17T17:07:12.650839+0000","last_unstale":"2024-08-17T17:11:49.847483+0000","last_undegraded":"2024-08-17T17:11:49.847483+0000","last_fullsized":"2024-08-17T17:11:49.847483+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:59.795181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"1079'796","reported_seq":4427,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.730715+0000","last_change":"2024-08-17T17:11:48.760295+0000","last_active":"2024-08-17T17:11:49.730715+0000","last_peered":"2024-08-17T17:11:49.730715+0000","last_clean":"2024-08-17T17:11:49.730715+0000","last_became_active":"2024-08-17T17:07:12.647225+0000","last_became_peered":"2024-08-17T17:07:12.647225+0000","last_unstale":"2024-08-17T17:11:49.730715+0000","last_undegraded":"2024-08-17T17:11:49.730715+0000","last_fullsized":"2024-08-17T17:11:49.730715+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":796,"log_dups_size":0,"ondisk_log_size":796,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:08.422134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00058246899999999996,"stat_sum":{"num_bytes":6185669,"num_objects":3,"num_object_clones":1,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":1,"num_read":270,"num_read_kb":274689,"num_write":209,"num_write_kb":45858,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":5630554,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"27a","length":"1"},{"start":"27d","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":1305,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.730064+0000","last_change":"2024-08-17T17:07:45.681107+0000","last_active":"2024-08-17T17:11:49.730064+0000","last_peered":"2024-08-17T17:11:49.730064+0000","last_clean":"2024-08-17T17:11:49.730064+0000","last_became_active":"2024-08-17T17:07:45.680770+0000","last_became_peered":"2024-08-17T17:07:45.680770+0000","last_unstale":"2024-08-17T17:11:49.730064+0000","last_undegraded":"2024-08-17T17:11:49.730064+0000","last_fullsized":"2024-08-17T17:11:49.730064+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:53.820920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1295,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:50.538731+0000","last_change":"2024-08-17T17:07:12.649485+0000","last_active":"2024-08-17T17:11:50.538731+0000","last_peered":"2024-08-17T17:11:50.538731+0000","last_clean":"2024-08-17T17:11:50.538731+0000","last_became_active":"2024-08-17T17:07:12.649161+0000","last_became_peered":"2024-08-17T17:07:12.649161+0000","last_unstale":"2024-08-17T17:11:50.538731+0000","last_undegraded":"2024-08-17T17:11:50.538731+0000","last_fullsized":"2024-08-17T17:11:50.538731+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:18.370628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1310,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:50.539418+0000","last_change":"2024-08-17T17:06:39.627820+0000","last_active":"2024-08-17T17:11:50.539418+0000","last_peered":"2024-08-17T17:11:50.539418+0000","last_clean":"2024-08-17T17:11:50.539418+0000","last_became_active":"2024-08-17T17:06:39.627464+0000","last_became_peered":"2024-08-17T17:06:39.627464+0000","last_unstale":"2024-08-17T17:11:50.539418+0000","last_undegraded":"2024-08-17T17:11:50.539418+0000","last_fullsized":"2024-08-17T17:11:50.539418+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:01.500528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"1079'2465","reported_seq":6161,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.847479+0000","last_change":"2024-08-17T17:11:48.732808+0000","last_active":"2024-08-17T17:11:49.847479+0000","last_peered":"2024-08-17T17:11:49.847479+0000","last_clean":"2024-08-17T17:11:49.847479+0000","last_became_active":"2024-08-17T17:06:39.636694+0000","last_became_peered":"2024-08-17T17:06:39.636694+0000","last_unstale":"2024-08-17T17:11:49.847479+0000","last_undegraded":"2024-08-17T17:11:49.847479+0000","last_fullsized":"2024-08-17T17:11:49.847479+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":2465,"log_dups_size":0,"ondisk_log_size":2465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:05:52.687270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00092596699999999998,"stat_sum":{"num_bytes":15929500,"num_objects":6,"num_object_clones":1,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":893,"num_read_kb":1025540,"num_write":647,"num_write_kb":158368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":5006879,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"27a","length":"1"},{"start":"27d","length":"1"}]},{"pgid":"2.2","version":"66'2","reported_seq":1034,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.731814+0000","last_change":"2024-08-17T17:06:39.628745+0000","last_active":"2024-08-17T17:11:48.731814+0000","last_peered":"2024-08-17T17:11:48.731814+0000","last_clean":"2024-08-17T17:11:48.731814+0000","last_became_active":"2024-08-17T17:06:39.628144+0000","last_became_peered":"2024-08-17T17:06:39.628144+0000","last_unstale":"2024-08-17T17:11:48.731814+0000","last_undegraded":"2024-08-17T17:11:48.731814+0000","last_fullsized":"2024-08-17T17:11:48.731814+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:14.740936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1023,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.734601+0000","last_change":"2024-08-17T17:06:07.637581+0000","last_active":"2024-08-17T17:11:48.734601+0000","last_peered":"2024-08-17T17:11:48.734601+0000","last_clean":"2024-08-17T17:11:48.734601+0000","last_became_active":"2024-08-17T17:06:07.637282+0000","last_became_peered":"2024-08-17T17:06:07.637282+0000","last_unstale":"2024-08-17T17:11:48.734601+0000","last_undegraded":"2024-08-17T17:11:48.734601+0000","last_fullsized":"2024-08-17T17:11:48.734601+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.938734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1306,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:50.538909+0000","last_change":"2024-08-17T17:07:45.687502+0000","last_active":"2024-08-17T17:11:50.538909+0000","last_peered":"2024-08-17T17:11:50.538909+0000","last_clean":"2024-08-17T17:11:50.538909+0000","last_became_active":"2024-08-17T17:07:45.687197+0000","last_became_peered":"2024-08-17T17:07:45.687197+0000","last_unstale":"2024-08-17T17:11:50.538909+0000","last_undegraded":"2024-08-17T17:11:50.538909+0000","last_fullsized":"2024-08-17T17:11:50.538909+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:23.577838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"1073'1399","reported_seq":5086,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.847299+0000","last_change":"2024-08-17T17:11:48.733094+0000","last_active":"2024-08-17T17:11:49.847299+0000","last_peered":"2024-08-17T17:11:49.847299+0000","last_clean":"2024-08-17T17:11:49.847299+0000","last_became_active":"2024-08-17T17:06:07.629977+0000","last_became_peered":"2024-08-17T17:06:07.629977+0000","last_unstale":"2024-08-17T17:11:49.847299+0000","last_undegraded":"2024-08-17T17:11:49.847299+0000","last_fullsized":"2024-08-17T17:11:49.847299+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":1399,"log_dups_size":0,"ondisk_log_size":1399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:50:00.000153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054069299999999997,"stat_sum":{"num_bytes":12956101,"num_objects":5,"num_object_clones":2,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":375,"num_read_kb":374314,"num_write":303,"num_write_kb":69895,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":1608517,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"27a","length":"1"},{"start":"27d","length":"1"}]},{"pgid":"3.5","version":"0'0","reported_seq":1034,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.731879+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:11:48.731879+0000","last_peered":"2024-08-17T17:11:48.731879+0000","last_clean":"2024-08-17T17:11:48.731879+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:11:48.731879+0000","last_undegraded":"2024-08-17T17:11:48.731879+0000","last_fullsized":"2024-08-17T17:11:48.731879+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1324,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:50.539035+0000","last_change":"2024-08-17T17:06:39.633975+0000","last_active":"2024-08-17T17:11:50.539035+0000","last_peered":"2024-08-17T17:11:50.539035+0000","last_clean":"2024-08-17T17:11:50.539035+0000","last_became_active":"2024-08-17T17:06:39.631526+0000","last_became_peered":"2024-08-17T17:06:39.631526+0000","last_unstale":"2024-08-17T17:11:50.539035+0000","last_undegraded":"2024-08-17T17:11:50.539035+0000","last_fullsized":"2024-08-17T17:11:50.539035+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:58.542237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1284,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:50.538550+0000","last_change":"2024-08-17T17:06:39.627577+0000","last_active":"2024-08-17T17:11:50.538550+0000","last_peered":"2024-08-17T17:11:50.538550+0000","last_clean":"2024-08-17T17:11:50.538550+0000","last_became_active":"2024-08-17T17:06:39.626554+0000","last_became_peered":"2024-08-17T17:06:39.626554+0000","last_unstale":"2024-08-17T17:11:50.538550+0000","last_undegraded":"2024-08-17T17:11:50.538550+0000","last_fullsized":"2024-08-17T17:11:50.538550+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:04:48.533057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.2","version":"1081'2189","reported_seq":5208,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.733106+0000","last_change":"2024-08-17T17:11:48.733106+0000","last_active":"2024-08-17T17:11:48.733106+0000","last_peered":"2024-08-17T17:11:48.733106+0000","last_clean":"2024-08-17T17:11:48.733106+0000","last_became_active":"2024-08-17T17:06:07.628522+0000","last_became_peered":"2024-08-17T17:06:07.628522+0000","last_unstale":"2024-08-17T17:11:48.733106+0000","last_undegraded":"2024-08-17T17:11:48.733106+0000","last_fullsized":"2024-08-17T17:11:48.733106+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":2189,"log_dups_size":0,"ondisk_log_size":2189,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:02:53.791824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0038532739999999998,"stat_sum":{"num_bytes":25671526,"num_objects":13,"num_object_clones":8,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":1,"num_read":623,"num_read_kb":650858,"num_write":572,"num_write_kb":141040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"27a","length":"1"},{"start":"27d","length":"1"}]},{"pgid":"3.4","version":"0'0","reported_seq":1294,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:50.539030+0000","last_change":"2024-08-17T17:07:12.645660+0000","last_active":"2024-08-17T17:11:50.539030+0000","last_peered":"2024-08-17T17:11:50.539030+0000","last_clean":"2024-08-17T17:11:50.539030+0000","last_became_active":"2024-08-17T17:07:12.645368+0000","last_became_peered":"2024-08-17T17:07:12.645368+0000","last_unstale":"2024-08-17T17:11:50.539030+0000","last_undegraded":"2024-08-17T17:11:50.539030+0000","last_fullsized":"2024-08-17T17:11:50.539030+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:41:00.157045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1017,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.737298+0000","last_change":"2024-08-17T17:06:07.636239+0000","last_active":"2024-08-17T17:11:48.737298+0000","last_peered":"2024-08-17T17:11:48.737298+0000","last_clean":"2024-08-17T17:11:48.737298+0000","last_became_active":"2024-08-17T17:06:07.635881+0000","last_became_peered":"2024-08-17T17:06:07.635881+0000","last_unstale":"2024-08-17T17:11:48.737298+0000","last_undegraded":"2024-08-17T17:11:48.737298+0000","last_fullsized":"2024-08-17T17:11:48.737298+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:36.001648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1311,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.847279+0000","last_change":"2024-08-17T17:06:07.634369+0000","last_active":"2024-08-17T17:11:49.847279+0000","last_peered":"2024-08-17T17:11:49.847279+0000","last_clean":"2024-08-17T17:11:49.847279+0000","last_became_active":"2024-08-17T17:06:07.633769+0000","last_became_peered":"2024-08-17T17:06:07.633769+0000","last_unstale":"2024-08-17T17:11:49.847279+0000","last_undegraded":"2024-08-17T17:11:49.847279+0000","last_fullsized":"2024-08-17T17:11:49.847279+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:20.957493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"1081'3883","reported_seq":6766,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.740612+0000","last_change":"2024-08-17T17:11:48.740612+0000","last_active":"2024-08-17T17:11:48.740612+0000","last_peered":"2024-08-17T17:11:48.740612+0000","last_clean":"2024-08-17T17:11:48.740612+0000","last_became_active":"2024-08-17T17:07:45.693193+0000","last_became_peered":"2024-08-17T17:07:45.693193+0000","last_unstale":"2024-08-17T17:11:48.740612+0000","last_undegraded":"2024-08-17T17:11:48.740612+0000","last_fullsized":"2024-08-17T17:11:48.740612+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":3883,"log_dups_size":0,"ondisk_log_size":3883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:26.336579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0062511930000000004,"stat_sum":{"num_bytes":26125152,"num_objects":12,"num_object_clones":6,"num_object_copies":36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12,"num_whiteouts":1,"num_read":1165,"num_read_kb":1301352,"num_write":967,"num_write_kb":227548,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":7144388,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"27a","length":"1"},{"start":"27d","length":"1"}]},{"pgid":"3.7","version":"72'2","reported_seq":1255,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011605+0000","last_change":"2024-08-17T17:07:45.688526+0000","last_active":"2024-08-17T17:11:53.011605+0000","last_peered":"2024-08-17T17:11:53.011605+0000","last_clean":"2024-08-17T17:11:53.011605+0000","last_became_active":"2024-08-17T17:07:45.688091+0000","last_became_peered":"2024-08-17T17:07:45.688091+0000","last_unstale":"2024-08-17T17:11:53.011605+0000","last_undegraded":"2024-08-17T17:11:53.011605+0000","last_fullsized":"2024-08-17T17:11:53.011605+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:02.385683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1294,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:50.539149+0000","last_change":"2024-08-17T17:06:39.629005+0000","last_active":"2024-08-17T17:11:50.539149+0000","last_peered":"2024-08-17T17:11:50.539149+0000","last_clean":"2024-08-17T17:11:50.539149+0000","last_became_active":"2024-08-17T17:06:39.628679+0000","last_became_peered":"2024-08-17T17:06:39.628679+0000","last_unstale":"2024-08-17T17:11:50.539149+0000","last_undegraded":"2024-08-17T17:11:50.539149+0000","last_fullsized":"2024-08-17T17:11:50.539149+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:09.401059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1253,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011303+0000","last_change":"2024-08-17T17:07:45.695858+0000","last_active":"2024-08-17T17:11:53.011303+0000","last_peered":"2024-08-17T17:11:53.011303+0000","last_clean":"2024-08-17T17:11:53.011303+0000","last_became_active":"2024-08-17T17:07:45.694111+0000","last_became_peered":"2024-08-17T17:07:45.694111+0000","last_unstale":"2024-08-17T17:11:53.011303+0000","last_undegraded":"2024-08-17T17:11:53.011303+0000","last_fullsized":"2024-08-17T17:11:53.011303+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:40:38.772090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"1079'918","reported_seq":4493,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011117+0000","last_change":"2024-08-17T17:11:48.728969+0000","last_active":"2024-08-17T17:11:53.011117+0000","last_peered":"2024-08-17T17:11:53.011117+0000","last_clean":"2024-08-17T17:11:53.011117+0000","last_became_active":"2024-08-17T17:07:45.685102+0000","last_became_peered":"2024-08-17T17:07:45.685102+0000","last_unstale":"2024-08-17T17:11:53.011117+0000","last_undegraded":"2024-08-17T17:11:53.011117+0000","last_fullsized":"2024-08-17T17:11:53.011117+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":75,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:54:47.142762+0000","last_clean_scrub_stamp":"2024-08-17T16:54:47.142762+0000","objects_scrubbed":0,"log_size":918,"log_dups_size":0,"ondisk_log_size":918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:17:03.260696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00090147700000000003,"stat_sum":{"num_bytes":11951196,"num_objects":4,"num_object_clones":2,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":352,"num_read_kb":395297,"num_write":208,"num_write_kb":48869,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":3160578,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"27d","length":"1"}]},{"pgid":"3.6","version":"72'1","reported_seq":1325,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.847596+0000","last_change":"2024-08-17T17:06:39.636866+0000","last_active":"2024-08-17T17:11:49.847596+0000","last_peered":"2024-08-17T17:11:49.847596+0000","last_clean":"2024-08-17T17:11:49.847596+0000","last_became_active":"2024-08-17T17:06:39.636470+0000","last_became_peered":"2024-08-17T17:06:39.636470+0000","last_unstale":"2024-08-17T17:11:49.847596+0000","last_undegraded":"2024-08-17T17:11:49.847596+0000","last_fullsized":"2024-08-17T17:11:49.847596+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:14.028712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1031,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.729788+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:11:48.729788+0000","last_peered":"2024-08-17T17:11:48.729788+0000","last_clean":"2024-08-17T17:11:48.729788+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:11:48.729788+0000","last_undegraded":"2024-08-17T17:11:48.729788+0000","last_fullsized":"2024-08-17T17:11:48.729788+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1007,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.734649+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:11:48.734649+0000","last_peered":"2024-08-17T17:11:48.734649+0000","last_clean":"2024-08-17T17:11:48.734649+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:11:48.734649+0000","last_undegraded":"2024-08-17T17:11:48.734649+0000","last_fullsized":"2024-08-17T17:11:48.734649+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1021,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.734453+0000","last_change":"2024-08-17T17:06:39.627594+0000","last_active":"2024-08-17T17:11:48.734453+0000","last_peered":"2024-08-17T17:11:48.734453+0000","last_clean":"2024-08-17T17:11:48.734453+0000","last_became_active":"2024-08-17T17:06:39.627105+0000","last_became_peered":"2024-08-17T17:06:39.627105+0000","last_unstale":"2024-08-17T17:11:48.734453+0000","last_undegraded":"2024-08-17T17:11:48.734453+0000","last_fullsized":"2024-08-17T17:11:48.734453+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:30.105070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1030,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.734705+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:11:48.734705+0000","last_peered":"2024-08-17T17:11:48.734705+0000","last_clean":"2024-08-17T17:11:48.734705+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:11:48.734705+0000","last_undegraded":"2024-08-17T17:11:48.734705+0000","last_fullsized":"2024-08-17T17:11:48.734705+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1295,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:50.538431+0000","last_change":"2024-08-17T17:07:12.645440+0000","last_active":"2024-08-17T17:11:50.538431+0000","last_peered":"2024-08-17T17:11:50.538431+0000","last_clean":"2024-08-17T17:11:50.538431+0000","last_became_active":"2024-08-17T17:07:12.645147+0000","last_became_peered":"2024-08-17T17:07:12.645147+0000","last_unstale":"2024-08-17T17:11:50.538431+0000","last_undegraded":"2024-08-17T17:11:50.538431+0000","last_fullsized":"2024-08-17T17:11:50.538431+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:25.588528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1006,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.737179+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:11:48.737179+0000","last_peered":"2024-08-17T17:11:48.737179+0000","last_clean":"2024-08-17T17:11:48.737179+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:11:48.737179+0000","last_undegraded":"2024-08-17T17:11:48.737179+0000","last_fullsized":"2024-08-17T17:11:48.737179+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1018,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.731951+0000","last_change":"2024-08-17T17:06:39.625029+0000","last_active":"2024-08-17T17:11:48.731951+0000","last_peered":"2024-08-17T17:11:48.731951+0000","last_clean":"2024-08-17T17:11:48.731951+0000","last_became_active":"2024-08-17T17:06:39.624688+0000","last_became_peered":"2024-08-17T17:06:39.624688+0000","last_unstale":"2024-08-17T17:11:48.731951+0000","last_undegraded":"2024-08-17T17:11:48.731951+0000","last_fullsized":"2024-08-17T17:11:48.731951+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:54.849668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1223,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011093+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:11:53.011093+0000","last_peered":"2024-08-17T17:11:53.011093+0000","last_clean":"2024-08-17T17:11:53.011093+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:11:53.011093+0000","last_undegraded":"2024-08-17T17:11:53.011093+0000","last_fullsized":"2024-08-17T17:11:53.011093+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1362,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.847648+0000","last_change":"2024-08-17T17:07:45.690099+0000","last_active":"2024-08-17T17:11:49.847648+0000","last_peered":"2024-08-17T17:11:49.847648+0000","last_clean":"2024-08-17T17:11:49.847648+0000","last_became_active":"2024-08-17T17:07:45.689783+0000","last_became_peered":"2024-08-17T17:07:45.689783+0000","last_unstale":"2024-08-17T17:11:49.847648+0000","last_undegraded":"2024-08-17T17:11:49.847648+0000","last_fullsized":"2024-08-17T17:11:49.847648+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:38.311630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1242,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011203+0000","last_change":"2024-08-17T17:07:45.678950+0000","last_active":"2024-08-17T17:11:53.011203+0000","last_peered":"2024-08-17T17:11:53.011203+0000","last_clean":"2024-08-17T17:11:53.011203+0000","last_became_active":"2024-08-17T17:07:45.678472+0000","last_became_peered":"2024-08-17T17:07:45.678472+0000","last_unstale":"2024-08-17T17:11:53.011203+0000","last_undegraded":"2024-08-17T17:11:53.011203+0000","last_fullsized":"2024-08-17T17:11:53.011203+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:48:02.385927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1019,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.729720+0000","last_change":"2024-08-17T17:06:39.628542+0000","last_active":"2024-08-17T17:11:48.729720+0000","last_peered":"2024-08-17T17:11:48.729720+0000","last_clean":"2024-08-17T17:11:48.729720+0000","last_became_active":"2024-08-17T17:06:39.628253+0000","last_became_peered":"2024-08-17T17:06:39.628253+0000","last_unstale":"2024-08-17T17:11:48.729720+0000","last_undegraded":"2024-08-17T17:11:48.729720+0000","last_fullsized":"2024-08-17T17:11:48.729720+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:22.349326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1258,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011404+0000","last_change":"2024-08-17T17:07:45.696068+0000","last_active":"2024-08-17T17:11:53.011404+0000","last_peered":"2024-08-17T17:11:53.011404+0000","last_clean":"2024-08-17T17:11:53.011404+0000","last_became_active":"2024-08-17T17:07:45.694617+0000","last_became_peered":"2024-08-17T17:07:45.694617+0000","last_unstale":"2024-08-17T17:11:53.011404+0000","last_undegraded":"2024-08-17T17:11:53.011404+0000","last_fullsized":"2024-08-17T17:11:53.011404+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:08.646380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1322,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.847487+0000","last_change":"2024-08-17T17:07:45.684186+0000","last_active":"2024-08-17T17:11:49.847487+0000","last_peered":"2024-08-17T17:11:49.847487+0000","last_clean":"2024-08-17T17:11:49.847487+0000","last_became_active":"2024-08-17T17:07:45.683863+0000","last_became_peered":"2024-08-17T17:07:45.683863+0000","last_unstale":"2024-08-17T17:11:49.847487+0000","last_undegraded":"2024-08-17T17:11:49.847487+0000","last_fullsized":"2024-08-17T17:11:49.847487+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:52.106168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1294,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:50.538423+0000","last_change":"2024-08-17T17:07:45.688645+0000","last_active":"2024-08-17T17:11:50.538423+0000","last_peered":"2024-08-17T17:11:50.538423+0000","last_clean":"2024-08-17T17:11:50.538423+0000","last_became_active":"2024-08-17T17:07:45.688383+0000","last_became_peered":"2024-08-17T17:07:45.688383+0000","last_unstale":"2024-08-17T17:11:50.538423+0000","last_undegraded":"2024-08-17T17:11:50.538423+0000","last_fullsized":"2024-08-17T17:11:50.538423+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:33:12.209565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1242,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011313+0000","last_change":"2024-08-17T17:07:45.689782+0000","last_active":"2024-08-17T17:11:53.011313+0000","last_peered":"2024-08-17T17:11:53.011313+0000","last_clean":"2024-08-17T17:11:53.011313+0000","last_became_active":"2024-08-17T17:07:45.689312+0000","last_became_peered":"2024-08-17T17:07:45.689312+0000","last_unstale":"2024-08-17T17:11:53.011313+0000","last_undegraded":"2024-08-17T17:11:53.011313+0000","last_fullsized":"2024-08-17T17:11:53.011313+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:07.560555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1038,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.734506+0000","last_change":"2024-08-17T17:07:12.649175+0000","last_active":"2024-08-17T17:11:48.734506+0000","last_peered":"2024-08-17T17:11:48.734506+0000","last_clean":"2024-08-17T17:11:48.734506+0000","last_became_active":"2024-08-17T17:07:12.648822+0000","last_became_peered":"2024-08-17T17:07:12.648822+0000","last_unstale":"2024-08-17T17:11:48.734506+0000","last_undegraded":"2024-08-17T17:11:48.734506+0000","last_fullsized":"2024-08-17T17:11:48.734506+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:03.924431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1281,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.730221+0000","last_change":"2024-08-17T17:07:45.685278+0000","last_active":"2024-08-17T17:11:49.730221+0000","last_peered":"2024-08-17T17:11:49.730221+0000","last_clean":"2024-08-17T17:11:49.730221+0000","last_became_active":"2024-08-17T17:07:45.684963+0000","last_became_peered":"2024-08-17T17:07:45.684963+0000","last_unstale":"2024-08-17T17:11:49.730221+0000","last_undegraded":"2024-08-17T17:11:49.730221+0000","last_fullsized":"2024-08-17T17:11:49.730221+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:55:42.495791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":1313,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.847441+0000","last_change":"2024-08-17T17:06:07.634103+0000","last_active":"2024-08-17T17:11:49.847441+0000","last_peered":"2024-08-17T17:11:49.847441+0000","last_clean":"2024-08-17T17:11:49.847441+0000","last_became_active":"2024-08-17T17:06:07.633624+0000","last_became_peered":"2024-08-17T17:06:07.633624+0000","last_unstale":"2024-08-17T17:11:49.847441+0000","last_undegraded":"2024-08-17T17:11:49.847441+0000","last_fullsized":"2024-08-17T17:11:49.847441+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:58.573558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1036,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.734509+0000","last_change":"2024-08-17T17:07:45.684606+0000","last_active":"2024-08-17T17:11:48.734509+0000","last_peered":"2024-08-17T17:11:48.734509+0000","last_clean":"2024-08-17T17:11:48.734509+0000","last_became_active":"2024-08-17T17:07:45.684124+0000","last_became_peered":"2024-08-17T17:07:45.684124+0000","last_unstale":"2024-08-17T17:11:48.734509+0000","last_undegraded":"2024-08-17T17:11:48.734509+0000","last_fullsized":"2024-08-17T17:11:48.734509+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:16.110507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1020,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.731573+0000","last_change":"2024-08-17T17:06:39.625245+0000","last_active":"2024-08-17T17:11:48.731573+0000","last_peered":"2024-08-17T17:11:48.731573+0000","last_clean":"2024-08-17T17:11:48.731573+0000","last_became_active":"2024-08-17T17:06:39.624657+0000","last_became_peered":"2024-08-17T17:06:39.624657+0000","last_unstale":"2024-08-17T17:11:48.731573+0000","last_undegraded":"2024-08-17T17:11:48.731573+0000","last_fullsized":"2024-08-17T17:11:48.731573+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:43.097460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1036,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.734376+0000","last_change":"2024-08-17T17:07:45.684769+0000","last_active":"2024-08-17T17:11:48.734376+0000","last_peered":"2024-08-17T17:11:48.734376+0000","last_clean":"2024-08-17T17:11:48.734376+0000","last_became_active":"2024-08-17T17:07:45.684459+0000","last_became_peered":"2024-08-17T17:07:45.684459+0000","last_unstale":"2024-08-17T17:11:48.734376+0000","last_undegraded":"2024-08-17T17:11:48.734376+0000","last_fullsized":"2024-08-17T17:11:48.734376+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:30.731978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1005,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.731793+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:11:48.731793+0000","last_peered":"2024-08-17T17:11:48.731793+0000","last_clean":"2024-08-17T17:11:48.731793+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:11:48.731793+0000","last_undegraded":"2024-08-17T17:11:48.731793+0000","last_fullsized":"2024-08-17T17:11:48.731793+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1248,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011785+0000","last_change":"2024-08-17T17:07:45.692127+0000","last_active":"2024-08-17T17:11:53.011785+0000","last_peered":"2024-08-17T17:11:53.011785+0000","last_clean":"2024-08-17T17:11:53.011785+0000","last_became_active":"2024-08-17T17:07:45.690277+0000","last_became_peered":"2024-08-17T17:07:45.690277+0000","last_unstale":"2024-08-17T17:11:53.011785+0000","last_undegraded":"2024-08-17T17:11:53.011785+0000","last_fullsized":"2024-08-17T17:11:53.011785+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:04.555409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1237,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011374+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:11:53.011374+0000","last_peered":"2024-08-17T17:11:53.011374+0000","last_clean":"2024-08-17T17:11:53.011374+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:11:53.011374+0000","last_undegraded":"2024-08-17T17:11:53.011374+0000","last_fullsized":"2024-08-17T17:11:53.011374+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1021,"reported_epoch":1081,"state":"active+clean","last_fresh":"2024-08-17T17:11:48.734255+0000","last_change":"2024-08-17T17:06:39.627689+0000","last_active":"2024-08-17T17:11:48.734255+0000","last_peered":"2024-08-17T17:11:48.734255+0000","last_clean":"2024-08-17T17:11:48.734255+0000","last_became_active":"2024-08-17T17:06:39.627375+0000","last_became_peered":"2024-08-17T17:06:39.627375+0000","last_unstale":"2024-08-17T17:11:48.734255+0000","last_undegraded":"2024-08-17T17:11:48.734255+0000","last_fullsized":"2024-08-17T17:11:48.734255+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:51.954076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":1328,"reported_epoch":1082,"state":"active+clean","last_fresh":"2024-08-17T17:11:49.847670+0000","last_change":"2024-08-17T17:07:12.651933+0000","last_active":"2024-08-17T17:11:49.847670+0000","last_peered":"2024-08-17T17:11:49.847670+0000","last_clean":"2024-08-17T17:11:49.847670+0000","last_became_active":"2024-08-17T17:07:12.651592+0000","last_became_peered":"2024-08-17T17:07:12.651592+0000","last_unstale":"2024-08-17T17:11:49.847670+0000","last_undegraded":"2024-08-17T17:11:49.847670+0000","last_fullsized":"2024-08-17T17:11:49.847670+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:31.224038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":199170267,"num_objects":94,"num_object_clones":50,"num_object_copies":282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94,"num_whiteouts":5,"num_read":6879,"num_read_kb":7360336,"num_write":5666,"num_write_kb":1351246,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":53,"num_bytes_recovered":50223385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":364986368,"data_stored":459280263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":22355,"ondisk_log_size":22355,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201524,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":83844,"kb_used_data":38772,"kb_used_omap":1,"kb_used_meta":45054,"kb_avail":93661308,"statfs":{"total":95995035648,"available":95909179392,"internally_reserved":0,"allocated":39702528,"data_stored":43517759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":46135755},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463161,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":98180,"kb_used_data":40824,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93646972,"statfs":{"total":95995035648,"available":95894499328,"internally_reserved":0,"allocated":41803776,"data_stored":49887364,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":58718665},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626803,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":74956,"kb_used_data":38656,"kb_used_omap":42,"kb_used_meta":36245,"kb_avail":93670196,"statfs":{"total":95995035648,"available":95918280704,"internally_reserved":0,"allocated":39583744,"data_stored":41485109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":37114940},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785280,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":93316,"kb_used_data":52728,"kb_used_omap":48,"kb_used_meta":40527,"kb_avail":93651836,"statfs":{"total":95995035648,"available":95899480064,"internally_reserved":0,"allocated":53993472,"data_stored":60466196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":41500334},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724797,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":152744,"kb_used_data":73308,"kb_used_omap":1,"kb_used_meta":79422,"kb_avail":93592408,"statfs":{"total":95995035648,"available":95838625792,"internally_reserved":0,"allocated":75067392,"data_stored":90957255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":81328582},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986434,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":154128,"kb_used_data":68928,"kb_used_omap":1,"kb_used_meta":85182,"kb_avail":93591024,"statfs":{"total":95995035648,"available":95837208576,"internally_reserved":0,"allocated":70582272,"data_stored":83764455,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":87226822},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706041,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":77832,"kb_used_data":41660,"kb_used_omap":35,"kb_used_meta":36124,"kb_avail":93667320,"statfs":{"total":95995035648,"available":95915335680,"internally_reserved":0,"allocated":42659840,"data_stored":46373450,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":36991031},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831814,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":115596,"kb_used_data":83128,"kb_used_omap":65,"kb_used_meta":32382,"kb_avail":93629556,"statfs":{"total":95995035648,"available":95876665344,"internally_reserved":0,"allocated":85123072,"data_stored":102091322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":33159355},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":74317824,"data_stored":94317030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":43786240,"data_stored":53282857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":32448512,"data_stored":39189514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":29364224,"data_stored":34293387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":60354560,"data_stored":76576245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":64851968,"data_stored":83773276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":30986240,"data_stored":42108843,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28876800,"data_stored":35739111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:11:55.149 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-08-17T17:11:55.149 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-08-17T17:11:55.361 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:11:55.723 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:55 smithi063 ceph-mon[109843]: pgmap v1083: 89 pgs: 89 active+clean; 191 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-08-17T17:11:55.723 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:55 smithi063 ceph-mon[109843]: from='client.34455 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:11:55.723 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:55 smithi063 ceph-mon[114756]: pgmap v1083: 89 pgs: 89 active+clean; 191 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-08-17T17:11:55.724 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:55 smithi063 ceph-mon[114756]: from='client.34455 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:11:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:55 smithi162 ceph-mon[104668]: pgmap v1083: 89 pgs: 89 active+clean; 191 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 1.3 MiB/s wr, 8 op/s 2024-08-17T17:11:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:55 smithi162 ceph-mon[104668]: from='client.34455 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:11:57.699 INFO:teuthology.orchestra.run.smithi063.stderr:pool 'unique_pool_0' removed 2024-08-17T17:11:58.000 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:57 smithi063 ceph-mon[114756]: pgmap v1084: 89 pgs: 89 active+clean; 191 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 864 KiB/s wr, 6 op/s 2024-08-17T17:11:58.000 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:57 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2360039630' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:11:58.000 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:57 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2360039630' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:11:58.001 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:57 smithi063 ceph-mon[109843]: pgmap v1084: 89 pgs: 89 active+clean; 191 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 864 KiB/s wr, 6 op/s 2024-08-17T17:11:58.001 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:57 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2360039630' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:11:58.001 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:57 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2360039630' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:11:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:57 smithi162 ceph-mon[104668]: pgmap v1084: 89 pgs: 89 active+clean; 191 MiB data, 831 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 864 KiB/s wr, 6 op/s 2024-08-17T17:11:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:57 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2360039630' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:11:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:57 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2360039630' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:11:58.362 INFO:teuthology.task.sequential:In sequential, running task print... 2024-08-17T17:11:58.362 INFO:teuthology.task.print:**** done rados/snaps-few-objects 5-workload 2024-08-17T17:11:58.363 DEBUG:teuthology.parallel:result is None 2024-08-17T17:11:58.363 INFO:teuthology.run_tasks:Running task print... 2024-08-17T17:11:58.371 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-08-17T17:11:58.371 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-08-17T17:11:58.380 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-08-17T17:11:58.380 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-08-17T17:11:58.380 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph mgr dump --format=json 2024-08-17T17:11:58.601 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:11:58.996 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:58 smithi063 ceph-mon[114756]: osdmap e1084: 8 total, 8 up, 8 in 2024-08-17T17:11:58.996 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:58 smithi063 ceph-mon[114756]: pgmap v1086: 73 pgs: 73 active+clean; 582 KiB data, 831 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:11:58.996 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:58 smithi063 ceph-mon[109843]: osdmap e1084: 8 total, 8 up, 8 in 2024-08-17T17:11:58.996 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:58 smithi063 ceph-mon[109843]: pgmap v1086: 73 pgs: 73 active+clean; 582 KiB data, 831 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:11:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:58 smithi162 ceph-mon[104668]: osdmap e1084: 8 total, 8 up, 8 in 2024-08-17T17:11:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:58 smithi162 ceph-mon[104668]: pgmap v1086: 73 pgs: 73 active+clean; 582 KiB data, 831 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:12:00.085 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:11:59 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:00.085 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:11:59 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:00.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:11:59 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:12:00.425 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:12:01.277 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:00 smithi063 ceph-mon[109843]: pgmap v1087: 73 pgs: 73 active+clean; 582 KiB data, 831 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:12:01.277 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:00 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1188135138' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-08-17T17:12:01.277 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:00 smithi063 ceph-mon[114756]: pgmap v1087: 73 pgs: 73 active+clean; 582 KiB data, 831 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:12:01.277 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:00 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1188135138' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-08-17T17:12:01.308 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:00 smithi162 ceph-mon[104668]: pgmap v1087: 73 pgs: 73 active+clean; 582 KiB data, 831 MiB used, 714 GiB / 715 GiB avail 2024-08-17T17:12:01.308 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:00 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1188135138' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-08-17T17:12:01.651 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":40,"flags":0,"active_gid":25070,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6800","nonce":897111313},{"type":"v1","addr":"172.21.15.63:6801","nonce":897111313}]},"active_addr":"172.21.15.63:6801/897111313","active_change":"2024-08-17T16:59:45.829632+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":24886,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:1.26.1","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.63:8443/","prometheus":"http://172.21.15.63:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":367,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":2414850288}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":1508148579}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":3222828584}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.63:0","nonce":709635564}]}]} 2024-08-17T17:12:01.655 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-08-17T17:12:01.655 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T17:12:01.656 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:12:01.879 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:01.895 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:02.178 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:02.196 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:03.238 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:02 smithi063 ceph-mon[114756]: pgmap v1088: 73 pgs: 73 active+clean; 582 KiB data, 570 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:03.238 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:02 smithi063 ceph-mon[109843]: pgmap v1088: 73 pgs: 73 active+clean; 582 KiB data, 570 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:03.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:02 smithi162 ceph-mon[104668]: pgmap v1088: 73 pgs: 73 active+clean; 582 KiB data, 570 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:03.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:12:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:12:03] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.51.0" 2024-08-17T17:12:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:12:04.431 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":1084,"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:46:54.156006+0000","modified":"2024-08-17T17:11:57.696744+0000","last_up_change":"2024-08-17T17:07:44.658208+0000","last_in_change":"2024-08-17T16:51:30.856523+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:49:54.431240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:52:22.131465+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T16:53:35.328125+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T16:53:36.292966+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":726,"up_from":739,"up_thru":837,"down_at":727,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6802","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6803","nonce":4285380350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6804","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6805","nonce":4285380350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6808","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6809","nonce":4285380350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6806","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6807","nonce":4285380350}]},"public_addr":"172.21.15.63:6803/4285380350","cluster_addr":"172.21.15.63:6805/4285380350","heartbeat_back_addr":"172.21.15.63:6809/4285380350","heartbeat_front_addr":"172.21.15.63:6807/4285380350","state":["exists","up"]},{"osd":1,"uuid":"ae346e99-4706-4871-990a-f8e737572245","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":758,"up_from":771,"up_thru":837,"down_at":759,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6810","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6811","nonce":3170762701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6812","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6813","nonce":3170762701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6816","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6817","nonce":3170762701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6814","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6815","nonce":3170762701}]},"public_addr":"172.21.15.63:6811/3170762701","cluster_addr":"172.21.15.63:6813/3170762701","heartbeat_back_addr":"172.21.15.63:6817/3170762701","heartbeat_front_addr":"172.21.15.63:6815/3170762701","state":["exists","up"]},{"osd":2,"uuid":"2d85d75f-1280-443f-857d-d322bd60c451","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":791,"up_from":804,"up_thru":837,"down_at":792,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6818","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6819","nonce":3903658295}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6820","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6821","nonce":3903658295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6824","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6825","nonce":3903658295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6822","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6823","nonce":3903658295}]},"public_addr":"172.21.15.63:6819/3903658295","cluster_addr":"172.21.15.63:6821/3903658295","heartbeat_back_addr":"172.21.15.63:6825/3903658295","heartbeat_front_addr":"172.21.15.63:6823/3903658295","state":["exists","up"]},{"osd":3,"uuid":"771f43d2-e98b-43e0-921b-88e56fadda42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":824,"up_from":837,"up_thru":837,"down_at":825,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6826","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6827","nonce":1764356824}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6828","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6829","nonce":1764356824}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6832","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6833","nonce":1764356824}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6830","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6831","nonce":1764356824}]},"public_addr":"172.21.15.63:6827/1764356824","cluster_addr":"172.21.15.63:6829/1764356824","heartbeat_back_addr":"172.21.15.63:6833/1764356824","heartbeat_front_addr":"172.21.15.63:6831/1764356824","state":["exists","up"]},{"osd":4,"uuid":"73c3c545-e975-4a6f-a1cb-e4575ee7a9b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":837,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6800","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6801","nonce":1563885208}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6803","nonce":1563885208}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6807","nonce":1563885208}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6805","nonce":1563885208}]},"public_addr":"172.21.15.162:6801/1563885208","cluster_addr":"172.21.15.162:6803/1563885208","heartbeat_back_addr":"172.21.15.162:6807/1563885208","heartbeat_front_addr":"172.21.15.162:6805/1563885208","state":["exists","up"]},{"osd":5,"uuid":"fae3ac52-3194-4eb4-9510-e3899768353a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":837,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6809","nonce":1826502179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6810","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6811","nonce":1826502179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6814","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6815","nonce":1826502179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6812","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6813","nonce":1826502179}]},"public_addr":"172.21.15.162:6809/1826502179","cluster_addr":"172.21.15.162:6811/1826502179","heartbeat_back_addr":"172.21.15.162:6815/1826502179","heartbeat_front_addr":"172.21.15.162:6813/1826502179","state":["exists","up"]},{"osd":6,"uuid":"a8255b37-1be5-4cd4-9b86-889f6bf52902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":825,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6816","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6817","nonce":1675904899}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6818","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6819","nonce":1675904899}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6822","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6823","nonce":1675904899}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6820","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6821","nonce":1675904899}]},"public_addr":"172.21.15.162:6817/1675904899","cluster_addr":"172.21.15.162:6819/1675904899","heartbeat_back_addr":"172.21.15.162:6823/1675904899","heartbeat_front_addr":"172.21.15.162:6821/1675904899","state":["exists","up"]},{"osd":7,"uuid":"3d0bcd6d-171d-4d90-a033-ed17714a9d96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":837,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6824","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6825","nonce":2938230961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6826","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6827","nonce":2938230961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6830","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6831","nonce":2938230961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6828","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6829","nonce":2938230961}]},"public_addr":"172.21.15.162:6825/2938230961","cluster_addr":"172.21.15.162:6827/2938230961","heartbeat_back_addr":"172.21.15.162:6831/2938230961","heartbeat_front_addr":"172.21.15.162:6829/2938230961","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:02.667962+0000","dead_epoch":726},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:26.166590+0000","dead_epoch":758},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:50.153794+0000","dead_epoch":791},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:13.556156+0000","dead_epoch":824},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:34.827805+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:58.789701+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:22.650398+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:46.166424+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.63:6801/4052036587":"2024-08-18T16:59:45.829424+0000","172.21.15.63:6800/4052036587":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/753392608":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/701508812":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/2384509117":"2024-08-18T16:59:45.829424+0000","172.21.15.63:6800/987519051":"2024-08-18T16:56:38.413263+0000","172.21.15.162:0/810332452":"2024-08-18T16:57:08.874093+0000","172.21.15.162:0/1341643866":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/3746443558":"2024-08-18T16:56:38.413263+0000","172.21.15.63:0/3589494813":"2024-08-18T16:56:38.413263+0000","172.21.15.63:6800/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:0/2696979598":"2024-08-18T16:56:38.413263+0000","172.21.15.162:0/319999013":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/84364348":"2024-08-18T16:47:16.384724+0000","172.21.15.162:0/4017736715":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/2404156937":"2024-08-18T16:59:45.829424+0000","172.21.15.63:6801/987519051":"2024-08-18T16:56:38.413263+0000","172.21.15.63:0/955244129":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/664643590":"2024-08-18T16:59:45.829424+0000","172.21.15.63:6800/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.63:6801/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.63:6800/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.162:6834/3390906977":"2024-08-18T16:57:08.874093+0000","172.21.15.63:6801/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.162:6835/3390906977":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/1986138219":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/4067771896":"2024-08-18T16:56:38.413263+0000","172.21.15.63:6801/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:0/1256300214":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/1792393387":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3226753153":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3004218255":"2024-08-18T16:47:16.384724+0000","172.21.15.162:0/2317499213":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/2601197458":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/3540337558":"2024-08-18T16:47:16.384724+0000","172.21.15.162:0/2288037041":"2024-08-18T16:57:08.874093+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":5,"snaps":[{"begin":637,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:12:05.265 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:05 smithi063 ceph-mon[114756]: pgmap v1089: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:05.265 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:12:05.266 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:05 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:12:05.266 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:05 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/579164243' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:12:05.266 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:05 smithi063 ceph-mon[109843]: pgmap v1089: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:05.266 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:12:05.266 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:05 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:12:05.266 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:05 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/579164243' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:12:05.287 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T17:12:05.287 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:12:05.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:05 smithi162 ceph-mon[104668]: pgmap v1089: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:05.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:12:05.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:05 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:12:05.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:05 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/579164243' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:12:06.032 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:07.126 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:07 smithi063 ceph-mon[109843]: pgmap v1090: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:07.126 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:07 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:12:07.126 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:07 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:12:07.126 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:07 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:07.126 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:07 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:07.126 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:07 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:12:07.127 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:07 smithi063 ceph-mon[114756]: pgmap v1090: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:07.127 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:07 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:12:07.127 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:07 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:12:07.127 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:07 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:07.127 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:07 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:07.127 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:07 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:12:07.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:07 smithi162 ceph-mon[104668]: pgmap v1090: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:07.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:07 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:12:07.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:07 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:12:07.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:07 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:12:07.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:07 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:12:07.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:07 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:12:07.794 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:12:07.795 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":1084,"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:46:54.156006+0000","modified":"2024-08-17T17:11:57.696744+0000","last_up_change":"2024-08-17T17:07:44.658208+0000","last_in_change":"2024-08-17T16:51:30.856523+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:49:54.431240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:52:22.131465+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T16:53:35.328125+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T16:53:36.292966+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":726,"up_from":739,"up_thru":837,"down_at":727,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6802","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6803","nonce":4285380350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6804","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6805","nonce":4285380350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6808","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6809","nonce":4285380350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6806","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6807","nonce":4285380350}]},"public_addr":"172.21.15.63:6803/4285380350","cluster_addr":"172.21.15.63:6805/4285380350","heartbeat_back_addr":"172.21.15.63:6809/4285380350","heartbeat_front_addr":"172.21.15.63:6807/4285380350","state":["exists","up"]},{"osd":1,"uuid":"ae346e99-4706-4871-990a-f8e737572245","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":758,"up_from":771,"up_thru":837,"down_at":759,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6810","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6811","nonce":3170762701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6812","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6813","nonce":3170762701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6816","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6817","nonce":3170762701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6814","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6815","nonce":3170762701}]},"public_addr":"172.21.15.63:6811/3170762701","cluster_addr":"172.21.15.63:6813/3170762701","heartbeat_back_addr":"172.21.15.63:6817/3170762701","heartbeat_front_addr":"172.21.15.63:6815/3170762701","state":["exists","up"]},{"osd":2,"uuid":"2d85d75f-1280-443f-857d-d322bd60c451","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":791,"up_from":804,"up_thru":837,"down_at":792,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6818","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6819","nonce":3903658295}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6820","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6821","nonce":3903658295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6824","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6825","nonce":3903658295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6822","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6823","nonce":3903658295}]},"public_addr":"172.21.15.63:6819/3903658295","cluster_addr":"172.21.15.63:6821/3903658295","heartbeat_back_addr":"172.21.15.63:6825/3903658295","heartbeat_front_addr":"172.21.15.63:6823/3903658295","state":["exists","up"]},{"osd":3,"uuid":"771f43d2-e98b-43e0-921b-88e56fadda42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":824,"up_from":837,"up_thru":837,"down_at":825,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6826","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6827","nonce":1764356824}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6828","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6829","nonce":1764356824}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6832","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6833","nonce":1764356824}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6830","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6831","nonce":1764356824}]},"public_addr":"172.21.15.63:6827/1764356824","cluster_addr":"172.21.15.63:6829/1764356824","heartbeat_back_addr":"172.21.15.63:6833/1764356824","heartbeat_front_addr":"172.21.15.63:6831/1764356824","state":["exists","up"]},{"osd":4,"uuid":"73c3c545-e975-4a6f-a1cb-e4575ee7a9b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":837,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6800","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6801","nonce":1563885208}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6803","nonce":1563885208}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6807","nonce":1563885208}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6805","nonce":1563885208}]},"public_addr":"172.21.15.162:6801/1563885208","cluster_addr":"172.21.15.162:6803/1563885208","heartbeat_back_addr":"172.21.15.162:6807/1563885208","heartbeat_front_addr":"172.21.15.162:6805/1563885208","state":["exists","up"]},{"osd":5,"uuid":"fae3ac52-3194-4eb4-9510-e3899768353a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":837,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6809","nonce":1826502179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6810","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6811","nonce":1826502179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6814","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6815","nonce":1826502179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6812","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6813","nonce":1826502179}]},"public_addr":"172.21.15.162:6809/1826502179","cluster_addr":"172.21.15.162:6811/1826502179","heartbeat_back_addr":"172.21.15.162:6815/1826502179","heartbeat_front_addr":"172.21.15.162:6813/1826502179","state":["exists","up"]},{"osd":6,"uuid":"a8255b37-1be5-4cd4-9b86-889f6bf52902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":825,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6816","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6817","nonce":1675904899}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6818","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6819","nonce":1675904899}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6822","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6823","nonce":1675904899}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6820","nonce":1675904899},{"type":"v1","addr":"172.21.15.162:6821","nonce":1675904899}]},"public_addr":"172.21.15.162:6817/1675904899","cluster_addr":"172.21.15.162:6819/1675904899","heartbeat_back_addr":"172.21.15.162:6823/1675904899","heartbeat_front_addr":"172.21.15.162:6821/1675904899","state":["exists","up"]},{"osd":7,"uuid":"3d0bcd6d-171d-4d90-a033-ed17714a9d96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":837,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6824","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6825","nonce":2938230961}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6826","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6827","nonce":2938230961}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6830","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6831","nonce":2938230961}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6828","nonce":2938230961},{"type":"v1","addr":"172.21.15.162:6829","nonce":2938230961}]},"public_addr":"172.21.15.162:6825/2938230961","cluster_addr":"172.21.15.162:6827/2938230961","heartbeat_back_addr":"172.21.15.162:6831/2938230961","heartbeat_front_addr":"172.21.15.162:6829/2938230961","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:02.667962+0000","dead_epoch":726},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:26.166590+0000","dead_epoch":758},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:50.153794+0000","dead_epoch":791},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:13.556156+0000","dead_epoch":824},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:34.827805+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:58.789701+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:22.650398+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:46.166424+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.63:6801/4052036587":"2024-08-18T16:59:45.829424+0000","172.21.15.63:6800/4052036587":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/753392608":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/2384509117":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/664643590":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/3746443558":"2024-08-18T16:56:38.413263+0000","172.21.15.63:6800/987519051":"2024-08-18T16:56:38.413263+0000","172.21.15.162:0/810332452":"2024-08-18T16:57:08.874093+0000","172.21.15.162:0/1341643866":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/2696979598":"2024-08-18T16:56:38.413263+0000","172.21.15.162:0/319999013":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/2404156937":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/84364348":"2024-08-18T16:47:16.384724+0000","172.21.15.162:0/4017736715":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/3589494813":"2024-08-18T16:56:38.413263+0000","172.21.15.63:6800/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6801/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.63:6800/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.162:6834/3390906977":"2024-08-18T16:57:08.874093+0000","172.21.15.63:6801/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.162:6835/3390906977":"2024-08-18T16:57:08.874093+0000","172.21.15.63:6800/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.63:6801/987519051":"2024-08-18T16:56:38.413263+0000","172.21.15.63:0/955244129":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/1986138219":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/4067771896":"2024-08-18T16:56:38.413263+0000","172.21.15.63:6801/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:0/3226753153":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3004218255":"2024-08-18T16:47:16.384724+0000","172.21.15.162:0/2317499213":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/2601197458":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/3540337558":"2024-08-18T16:47:16.384724+0000","172.21.15.162:0/2288037041":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/1792393387":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/1256300214":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/701508812":"2024-08-18T16:52:20.398501+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":5,"snaps":[{"begin":637,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:12:08.307 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:08 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/843918967' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:12:08.307 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:08 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/843918967' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:12:08.332 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T17:12:08.332 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T17:12:08.333 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T17:12:08.333 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T17:12:08.333 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T17:12:08.334 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T17:12:08.334 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T17:12:08.335 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T17:12:08.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:08 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/843918967' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:12:08.706 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:08.764 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:08.783 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:08.784 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:08.784 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:08.784 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:08.967 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:09.144 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:09 smithi063 ceph-mon[109843]: pgmap v1091: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:09.145 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:09 smithi063 ceph-mon[114756]: pgmap v1091: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:09.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:09 smithi162 ceph-mon[104668]: pgmap v1091: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:11.046 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:11.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:11 smithi162 ceph-mon[104668]: pgmap v1092: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:11.479 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:11 smithi063 ceph-mon[114756]: pgmap v1092: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:11.479 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:11 smithi063 ceph-mon[109843]: pgmap v1092: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:13.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:13 smithi162 ceph-mon[104668]: pgmap v1093: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:13.423 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:13 smithi063 ceph-mon[109843]: pgmap v1093: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:13.423 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:12:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:12:13] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.51.0" 2024-08-17T17:12:13.423 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:13 smithi063 ceph-mon[114756]: pgmap v1093: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:15.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:15 smithi162 ceph-mon[104668]: pgmap v1094: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:15.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:15 smithi063 ceph-mon[114756]: pgmap v1094: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:15.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:15 smithi063 ceph-mon[109843]: pgmap v1094: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:17.267 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:17 smithi063 ceph-mon[114756]: pgmap v1095: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:17.267 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:17 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:17.268 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:17 smithi063 ceph-mon[109843]: pgmap v1095: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:17.268 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:17 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:17.290 INFO:teuthology.orchestra.run.smithi063.stdout:133143986439 2024-08-17T17:12:17.291 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T17:12:17.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:17 smithi162 ceph-mon[104668]: pgmap v1095: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:17.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:17 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:19.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:19 smithi162 ceph-mon[104668]: pgmap v1096: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:19.391 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:19 smithi063 ceph-mon[114756]: pgmap v1096: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:19.391 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:19 smithi063 ceph-mon[109843]: pgmap v1096: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:20.711 INFO:teuthology.orchestra.run.smithi063.stdout:3311419785286 2024-08-17T17:12:20.712 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T17:12:21.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:21 smithi063 ceph-mon[114756]: pgmap v1097: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:21.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:21 smithi063 ceph-mon[109843]: pgmap v1097: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:21.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:21 smithi162 ceph-mon[104668]: pgmap v1097: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:21.377 INFO:teuthology.orchestra.run.smithi063.stdout:167503724803 2024-08-17T17:12:21.377 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T17:12:22.094 INFO:teuthology.orchestra.run.smithi063.stdout:3594887626808 2024-08-17T17:12:22.095 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T17:12:22.468 INFO:teuthology.orchestra.run.smithi063.stdout:3173980831820 2024-08-17T17:12:22.468 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T17:12:23.073 INFO:teuthology.orchestra.run.smithi063.stdout:3453153706048 2024-08-17T17:12:23.073 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T17:12:23.301 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:12:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:12:23] "GET /metrics HTTP/1.1" 200 33976 "" "Prometheus/2.51.0" 2024-08-17T17:12:23.301 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:23 smithi063 ceph-mon[114756]: pgmap v1098: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:23.302 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:23 smithi063 ceph-mon[109843]: pgmap v1098: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:23.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:23 smithi162 ceph-mon[104668]: pgmap v1098: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:23.893 INFO:teuthology.orchestra.run.smithi063.stdout:201863463168 2024-08-17T17:12:23.893 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T17:12:24.131 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:24.544 INFO:teuthology.orchestra.run.smithi063.stdout:236223201531 2024-08-17T17:12:24.544 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T17:12:24.578 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:24.616 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:24.624 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:25.109 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:25 smithi063 ceph-mon[114756]: pgmap v1099: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:25.110 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:25 smithi063 ceph-mon[109843]: pgmap v1099: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:25.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:25 smithi162 ceph-mon[104668]: pgmap v1099: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:25.911 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:25.918 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:25.919 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:25.955 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:27.188 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:27 smithi063 ceph-mon[114756]: pgmap v1100: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:27.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:27 smithi063 ceph-mon[109843]: pgmap v1100: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:27.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:27 smithi162 ceph-mon[104668]: pgmap v1100: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:29.260 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:29 smithi063 ceph-mon[114756]: pgmap v1101: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:29.260 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:29 smithi063 ceph-mon[109843]: pgmap v1101: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:29.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:29 smithi162 ceph-mon[104668]: pgmap v1101: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:31.398 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:31 smithi162 ceph-mon[104668]: pgmap v1102: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:31.519 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:31 smithi063 ceph-mon[114756]: pgmap v1102: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:31.519 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:31 smithi063 ceph-mon[109843]: pgmap v1102: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:32.302 INFO:teuthology.orchestra.run.smithi063.stdout:3594887626811 2024-08-17T17:12:32.382 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:32 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:32.382 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:32 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3359892648' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:12:32.382 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:32 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:32.382 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:32 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3359892648' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:12:32.394 INFO:teuthology.orchestra.run.smithi063.stdout:3453153706050 2024-08-17T17:12:32.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:32 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:32.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:32 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3359892648' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:12:33.088 INFO:teuthology.orchestra.run.smithi063.stdout:167503724806 2024-08-17T17:12:33.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:12:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:12:33] "GET /metrics HTTP/1.1" 200 33980 "" "Prometheus/2.51.0" 2024-08-17T17:12:33.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:33 smithi063 ceph-mon[114756]: pgmap v1103: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:33.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:33 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4028256929' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:12:33.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:33 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2788907023' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:12:33.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:33 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4231730596' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:12:33.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:33 smithi063 ceph-mon[109843]: pgmap v1103: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:33.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:33 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4028256929' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:12:33.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:33 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2788907023' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:12:33.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:33 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4231730596' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:12:33.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:33 smithi162 ceph-mon[104668]: pgmap v1103: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:33.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:33 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4028256929' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:12:33.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:33 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2788907023' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:12:33.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:33 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4231730596' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:12:34.310 INFO:teuthology.orchestra.run.smithi063.stdout:3173980831823 2024-08-17T17:12:34.588 INFO:teuthology.orchestra.run.smithi063.stdout:3311419785288 2024-08-17T17:12:35.478 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:35 smithi063 ceph-mon[109843]: pgmap v1104: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:35.478 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:35 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3989310669' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:12:35.479 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:35 smithi063 ceph-mon[114756]: pgmap v1104: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:35.479 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:35 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3989310669' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:12:35.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:35 smithi162 ceph-mon[104668]: pgmap v1104: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:35.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:35 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3989310669' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:12:35.857 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3453153706048 got 3453153706050 for osd.2 2024-08-17T17:12:35.857 DEBUG:teuthology.parallel:result is None 2024-08-17T17:12:36.211 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3594887626808 got 3594887626811 for osd.3 2024-08-17T17:12:36.212 DEBUG:teuthology.parallel:result is None 2024-08-17T17:12:36.331 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724803 got 167503724806 for osd.5 2024-08-17T17:12:36.331 DEBUG:teuthology.parallel:result is None 2024-08-17T17:12:36.874 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3311419785286 got 3311419785288 for osd.1 2024-08-17T17:12:36.875 DEBUG:teuthology.parallel:result is None 2024-08-17T17:12:37.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:37 smithi063 ceph-mon[109843]: pgmap v1105: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:37.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:37 smithi063 ceph-mon[114756]: pgmap v1105: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:37.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:37 smithi162 ceph-mon[104668]: pgmap v1105: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:37.598 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3173980831820 got 3173980831823 for osd.0 2024-08-17T17:12:37.598 DEBUG:teuthology.parallel:result is None 2024-08-17T17:12:37.895 INFO:teuthology.orchestra.run.smithi063.stdout:201863463170 2024-08-17T17:12:37.957 INFO:teuthology.orchestra.run.smithi063.stdout:236223201533 2024-08-17T17:12:38.166 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:38 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3301542322' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:12:38.166 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:38 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1610920896' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:12:38.167 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:38 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3301542322' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:12:38.167 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:38 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1610920896' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:12:38.488 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463168 got 201863463170 for osd.6 2024-08-17T17:12:38.488 DEBUG:teuthology.parallel:result is None 2024-08-17T17:12:38.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:38 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3301542322' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:12:38.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:38 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1610920896' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:12:39.408 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:39 smithi063 ceph-mon[109843]: pgmap v1106: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:39.408 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:39 smithi063 ceph-mon[114756]: pgmap v1106: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:39.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:39 smithi162 ceph-mon[104668]: pgmap v1106: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:40.107 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201531 got 236223201533 for osd.7 2024-08-17T17:12:40.107 DEBUG:teuthology.parallel:result is None 2024-08-17T17:12:40.459 INFO:teuthology.orchestra.run.smithi063.stdout:133143986444 2024-08-17T17:12:41.021 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986439 got 133143986444 for osd.4 2024-08-17T17:12:41.021 DEBUG:teuthology.parallel:result is None 2024-08-17T17:12:41.022 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T17:12:41.022 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:12:41.238 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:41.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:41 smithi063 ceph-mon[114756]: pgmap v1107: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:41.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:41 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2607356698' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:12:41.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:41 smithi063 ceph-mon[109843]: pgmap v1107: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:41.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:41 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2607356698' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:12:41.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:41 smithi162 ceph-mon[104668]: pgmap v1107: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:41.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:41 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2607356698' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:12:42.979 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:12:42.980 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:12:43.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:43 smithi063 ceph-mon[114756]: pgmap v1108: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:43.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:43 smithi063 ceph-mon[109843]: pgmap v1108: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:43.537 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:12:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:12:43] "GET /metrics HTTP/1.1" 200 33980 "" "Prometheus/2.51.0" 2024-08-17T17:12:43.562 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1108,"stamp":"2024-08-17T17:12:42.032643+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":512404,"kb_used_data":81784,"kb_used_omap":199,"kb_used_meta":430328,"kb_avail":749448812,"statfs":{"total":767960285184,"available":767435583488,"internally_reserved":0,"allocated":83746816,"data_stored":59387167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":440656636},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002807"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1245,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074580+0000","last_change":"2024-08-17T17:07:45.684223+0000","last_active":"2024-08-17T17:11:59.074580+0000","last_peered":"2024-08-17T17:11:59.074580+0000","last_clean":"2024-08-17T17:11:59.074580+0000","last_became_active":"2024-08-17T17:07:45.683948+0000","last_became_peered":"2024-08-17T17:07:45.683948+0000","last_unstale":"2024-08-17T17:11:59.074580+0000","last_undegraded":"2024-08-17T17:11:59.074580+0000","last_fullsized":"2024-08-17T17:11:59.074580+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:07.677423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1008,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075622+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:11:59.075622+0000","last_peered":"2024-08-17T17:11:59.075622+0000","last_clean":"2024-08-17T17:11:59.075622+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:11:59.075622+0000","last_undegraded":"2024-08-17T17:11:59.075622+0000","last_fullsized":"2024-08-17T17:11:59.075622+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1019,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011091+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:11:53.011091+0000","last_peered":"2024-08-17T17:11:53.011091+0000","last_clean":"2024-08-17T17:11:53.011091+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:11:53.011091+0000","last_undegraded":"2024-08-17T17:11:53.011091+0000","last_fullsized":"2024-08-17T17:11:53.011091+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1366,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648138+0000","last_change":"2024-08-17T17:07:45.683455+0000","last_active":"2024-08-17T17:11:58.648138+0000","last_peered":"2024-08-17T17:11:58.648138+0000","last_clean":"2024-08-17T17:11:58.648138+0000","last_became_active":"2024-08-17T17:07:45.683169+0000","last_became_peered":"2024-08-17T17:07:45.683169+0000","last_unstale":"2024-08-17T17:11:58.648138+0000","last_undegraded":"2024-08-17T17:11:58.648138+0000","last_fullsized":"2024-08-17T17:11:58.648138+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:18:19.562247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1325,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648396+0000","last_change":"2024-08-17T17:07:45.684222+0000","last_active":"2024-08-17T17:11:58.648396+0000","last_peered":"2024-08-17T17:11:58.648396+0000","last_clean":"2024-08-17T17:11:58.648396+0000","last_became_active":"2024-08-17T17:07:45.683913+0000","last_became_peered":"2024-08-17T17:07:45.683913+0000","last_unstale":"2024-08-17T17:11:58.648396+0000","last_undegraded":"2024-08-17T17:11:58.648396+0000","last_fullsized":"2024-08-17T17:11:58.648396+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:59:33.944853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":991,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075457+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:11:59.075457+0000","last_peered":"2024-08-17T17:11:59.075457+0000","last_clean":"2024-08-17T17:11:59.075457+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:11:59.075457+0000","last_undegraded":"2024-08-17T17:11:59.075457+0000","last_fullsized":"2024-08-17T17:11:59.075457+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1245,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075397+0000","last_change":"2024-08-17T17:07:45.687030+0000","last_active":"2024-08-17T17:11:59.075397+0000","last_peered":"2024-08-17T17:11:59.075397+0000","last_clean":"2024-08-17T17:11:59.075397+0000","last_became_active":"2024-08-17T17:07:45.686610+0000","last_became_peered":"2024-08-17T17:07:45.686610+0000","last_unstale":"2024-08-17T17:11:59.075397+0000","last_undegraded":"2024-08-17T17:11:59.075397+0000","last_fullsized":"2024-08-17T17:11:59.075397+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:02.425845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1030,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702965+0000","last_change":"2024-08-17T17:06:07.632931+0000","last_active":"2024-08-17T17:11:57.702965+0000","last_peered":"2024-08-17T17:11:57.702965+0000","last_clean":"2024-08-17T17:11:57.702965+0000","last_became_active":"2024-08-17T17:06:07.632525+0000","last_became_peered":"2024-08-17T17:06:07.632525+0000","last_unstale":"2024-08-17T17:11:57.702965+0000","last_undegraded":"2024-08-17T17:11:57.702965+0000","last_fullsized":"2024-08-17T17:11:57.702965+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:01.740674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1007,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699675+0000","last_change":"2024-08-17T17:06:07.638468+0000","last_active":"2024-08-17T17:11:52.699675+0000","last_peered":"2024-08-17T17:11:52.699675+0000","last_clean":"2024-08-17T17:11:52.699675+0000","last_became_active":"2024-08-17T17:06:07.636046+0000","last_became_peered":"2024-08-17T17:06:07.636046+0000","last_unstale":"2024-08-17T17:11:52.699675+0000","last_undegraded":"2024-08-17T17:11:52.699675+0000","last_fullsized":"2024-08-17T17:11:52.699675+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:37:12.228090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1005,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.010559+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:11:53.010559+0000","last_peered":"2024-08-17T17:11:53.010559+0000","last_clean":"2024-08-17T17:11:53.010559+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:11:53.010559+0000","last_undegraded":"2024-08-17T17:11:53.010559+0000","last_fullsized":"2024-08-17T17:11:53.010559+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1008,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699604+0000","last_change":"2024-08-17T17:06:39.638433+0000","last_active":"2024-08-17T17:11:52.699604+0000","last_peered":"2024-08-17T17:11:52.699604+0000","last_clean":"2024-08-17T17:11:52.699604+0000","last_became_active":"2024-08-17T17:06:39.638081+0000","last_became_peered":"2024-08-17T17:06:39.638081+0000","last_unstale":"2024-08-17T17:11:52.699604+0000","last_undegraded":"2024-08-17T17:11:52.699604+0000","last_fullsized":"2024-08-17T17:11:52.699604+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:13.558012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800667+0000","last_change":"2024-08-17T17:07:45.693574+0000","last_active":"2024-08-17T17:11:52.800667+0000","last_peered":"2024-08-17T17:11:52.800667+0000","last_clean":"2024-08-17T17:11:52.800667+0000","last_became_active":"2024-08-17T17:07:45.693086+0000","last_became_peered":"2024-08-17T17:07:45.693086+0000","last_unstale":"2024-08-17T17:11:52.800667+0000","last_undegraded":"2024-08-17T17:11:52.800667+0000","last_fullsized":"2024-08-17T17:11:52.800667+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:31.888894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":992,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699566+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:11:52.699566+0000","last_peered":"2024-08-17T17:11:52.699566+0000","last_clean":"2024-08-17T17:11:52.699566+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:11:52.699566+0000","last_undegraded":"2024-08-17T17:11:52.699566+0000","last_fullsized":"2024-08-17T17:11:52.699566+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1021,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699849+0000","last_change":"2024-08-17T17:06:39.629368+0000","last_active":"2024-08-17T17:11:52.699849+0000","last_peered":"2024-08-17T17:11:52.699849+0000","last_clean":"2024-08-17T17:11:52.699849+0000","last_became_active":"2024-08-17T17:06:39.628832+0000","last_became_peered":"2024-08-17T17:06:39.628832+0000","last_unstale":"2024-08-17T17:11:52.699849+0000","last_undegraded":"2024-08-17T17:11:52.699849+0000","last_fullsized":"2024-08-17T17:11:52.699849+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:54.063884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1021,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011147+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:11:53.011147+0000","last_peered":"2024-08-17T17:11:53.011147+0000","last_clean":"2024-08-17T17:11:53.011147+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:11:53.011147+0000","last_undegraded":"2024-08-17T17:11:53.011147+0000","last_fullsized":"2024-08-17T17:11:53.011147+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1338,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.649075+0000","last_change":"2024-08-17T17:07:12.651379+0000","last_active":"2024-08-17T17:11:58.649075+0000","last_peered":"2024-08-17T17:11:58.649075+0000","last_clean":"2024-08-17T17:11:58.649075+0000","last_became_active":"2024-08-17T17:07:12.651069+0000","last_became_peered":"2024-08-17T17:07:12.651069+0000","last_unstale":"2024-08-17T17:11:58.649075+0000","last_undegraded":"2024-08-17T17:11:58.649075+0000","last_fullsized":"2024-08-17T17:11:58.649075+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:25.384305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1258,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075731+0000","last_change":"2024-08-17T17:07:45.691437+0000","last_active":"2024-08-17T17:11:59.075731+0000","last_peered":"2024-08-17T17:11:59.075731+0000","last_clean":"2024-08-17T17:11:59.075731+0000","last_became_active":"2024-08-17T17:07:45.690567+0000","last_became_peered":"2024-08-17T17:07:45.690567+0000","last_unstale":"2024-08-17T17:11:59.075731+0000","last_undegraded":"2024-08-17T17:11:59.075731+0000","last_fullsized":"2024-08-17T17:11:59.075731+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:43:07.698912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1024,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800708+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:11:52.800708+0000","last_peered":"2024-08-17T17:11:52.800708+0000","last_clean":"2024-08-17T17:11:52.800708+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:11:52.800708+0000","last_undegraded":"2024-08-17T17:11:52.800708+0000","last_fullsized":"2024-08-17T17:11:52.800708+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1022,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.700185+0000","last_change":"2024-08-17T17:06:39.629619+0000","last_active":"2024-08-17T17:11:52.700185+0000","last_peered":"2024-08-17T17:11:52.700185+0000","last_clean":"2024-08-17T17:11:52.700185+0000","last_became_active":"2024-08-17T17:06:39.629295+0000","last_became_peered":"2024-08-17T17:06:39.629295+0000","last_unstale":"2024-08-17T17:11:52.700185+0000","last_undegraded":"2024-08-17T17:11:52.700185+0000","last_fullsized":"2024-08-17T17:11:52.700185+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:15.874423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.801064+0000","last_change":"2024-08-17T17:07:12.649178+0000","last_active":"2024-08-17T17:11:52.801064+0000","last_peered":"2024-08-17T17:11:52.801064+0000","last_clean":"2024-08-17T17:11:52.801064+0000","last_became_active":"2024-08-17T17:07:12.648844+0000","last_became_peered":"2024-08-17T17:07:12.648844+0000","last_unstale":"2024-08-17T17:11:52.801064+0000","last_undegraded":"2024-08-17T17:11:52.801064+0000","last_fullsized":"2024-08-17T17:11:52.801064+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:12:06.307736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1025,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800742+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:11:52.800742+0000","last_peered":"2024-08-17T17:11:52.800742+0000","last_clean":"2024-08-17T17:11:52.800742+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:11:52.800742+0000","last_undegraded":"2024-08-17T17:11:52.800742+0000","last_fullsized":"2024-08-17T17:11:52.800742+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1024,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.801216+0000","last_change":"2024-08-17T17:06:39.627678+0000","last_active":"2024-08-17T17:11:52.801216+0000","last_peered":"2024-08-17T17:11:52.801216+0000","last_clean":"2024-08-17T17:11:52.801216+0000","last_became_active":"2024-08-17T17:06:39.627226+0000","last_became_peered":"2024-08-17T17:06:39.627226+0000","last_unstale":"2024-08-17T17:11:52.801216+0000","last_undegraded":"2024-08-17T17:11:52.801216+0000","last_fullsized":"2024-08-17T17:11:52.801216+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:01.868244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1247,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075390+0000","last_change":"2024-08-17T17:07:45.680458+0000","last_active":"2024-08-17T17:11:59.075390+0000","last_peered":"2024-08-17T17:11:59.075390+0000","last_clean":"2024-08-17T17:11:59.075390+0000","last_became_active":"2024-08-17T17:07:45.680221+0000","last_became_peered":"2024-08-17T17:07:45.680221+0000","last_unstale":"2024-08-17T17:11:59.075390+0000","last_undegraded":"2024-08-17T17:11:59.075390+0000","last_fullsized":"2024-08-17T17:11:59.075390+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:55.083962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1269,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075358+0000","last_change":"2024-08-17T17:07:45.682866+0000","last_active":"2024-08-17T17:11:59.075358+0000","last_peered":"2024-08-17T17:11:59.075358+0000","last_clean":"2024-08-17T17:11:59.075358+0000","last_became_active":"2024-08-17T17:07:45.682554+0000","last_became_peered":"2024-08-17T17:07:45.682554+0000","last_unstale":"2024-08-17T17:11:59.075358+0000","last_undegraded":"2024-08-17T17:11:59.075358+0000","last_fullsized":"2024-08-17T17:11:59.075358+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:25.975914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1338,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074716+0000","last_change":"2024-08-17T17:06:07.636484+0000","last_active":"2024-08-17T17:11:59.074716+0000","last_peered":"2024-08-17T17:11:59.074716+0000","last_clean":"2024-08-17T17:11:59.074716+0000","last_became_active":"2024-08-17T17:06:07.635998+0000","last_became_peered":"2024-08-17T17:06:07.635998+0000","last_unstale":"2024-08-17T17:11:59.074716+0000","last_undegraded":"2024-08-17T17:11:59.074716+0000","last_fullsized":"2024-08-17T17:11:59.074716+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:16.096587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1035,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011021+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:11:53.011021+0000","last_peered":"2024-08-17T17:11:53.011021+0000","last_clean":"2024-08-17T17:11:53.011021+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:11:53.011021+0000","last_undegraded":"2024-08-17T17:11:53.011021+0000","last_fullsized":"2024-08-17T17:11:53.011021+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1226,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074625+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:11:59.074625+0000","last_peered":"2024-08-17T17:11:59.074625+0000","last_clean":"2024-08-17T17:11:59.074625+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:11:59.074625+0000","last_undegraded":"2024-08-17T17:11:59.074625+0000","last_fullsized":"2024-08-17T17:11:59.074625+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1007,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703217+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:11:57.703217+0000","last_peered":"2024-08-17T17:11:57.703217+0000","last_clean":"2024-08-17T17:11:57.703217+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:11:57.703217+0000","last_undegraded":"2024-08-17T17:11:57.703217+0000","last_fullsized":"2024-08-17T17:11:57.703217+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1035,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074623+0000","last_change":"2024-08-17T17:06:39.638376+0000","last_active":"2024-08-17T17:11:59.074623+0000","last_peered":"2024-08-17T17:11:59.074623+0000","last_clean":"2024-08-17T17:11:59.074623+0000","last_became_active":"2024-08-17T17:06:39.638050+0000","last_became_peered":"2024-08-17T17:06:39.638050+0000","last_unstale":"2024-08-17T17:11:59.074623+0000","last_undegraded":"2024-08-17T17:11:59.074623+0000","last_fullsized":"2024-08-17T17:11:59.074623+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:05.782492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1367,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648019+0000","last_change":"2024-08-17T17:07:45.683556+0000","last_active":"2024-08-17T17:11:58.648019+0000","last_peered":"2024-08-17T17:11:58.648019+0000","last_clean":"2024-08-17T17:11:58.648019+0000","last_became_active":"2024-08-17T17:07:45.683232+0000","last_became_peered":"2024-08-17T17:07:45.683232+0000","last_unstale":"2024-08-17T17:11:58.648019+0000","last_undegraded":"2024-08-17T17:11:58.648019+0000","last_fullsized":"2024-08-17T17:11:58.648019+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:16.112417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1339,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648339+0000","last_change":"2024-08-17T17:07:12.651145+0000","last_active":"2024-08-17T17:11:58.648339+0000","last_peered":"2024-08-17T17:11:58.648339+0000","last_clean":"2024-08-17T17:11:58.648339+0000","last_became_active":"2024-08-17T17:07:12.650839+0000","last_became_peered":"2024-08-17T17:07:12.650839+0000","last_unstale":"2024-08-17T17:11:58.648339+0000","last_undegraded":"2024-08-17T17:11:58.648339+0000","last_fullsized":"2024-08-17T17:11:58.648339+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:59.795181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1309,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074165+0000","last_change":"2024-08-17T17:07:45.681107+0000","last_active":"2024-08-17T17:11:59.074165+0000","last_peered":"2024-08-17T17:11:59.074165+0000","last_clean":"2024-08-17T17:11:59.074165+0000","last_became_active":"2024-08-17T17:07:45.680770+0000","last_became_peered":"2024-08-17T17:07:45.680770+0000","last_unstale":"2024-08-17T17:11:59.074165+0000","last_undegraded":"2024-08-17T17:11:59.074165+0000","last_fullsized":"2024-08-17T17:11:59.074165+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:53.820920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1300,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.704078+0000","last_change":"2024-08-17T17:07:12.649485+0000","last_active":"2024-08-17T17:11:57.704078+0000","last_peered":"2024-08-17T17:11:57.704078+0000","last_clean":"2024-08-17T17:11:57.704078+0000","last_became_active":"2024-08-17T17:07:12.649161+0000","last_became_peered":"2024-08-17T17:07:12.649161+0000","last_unstale":"2024-08-17T17:11:57.704078+0000","last_undegraded":"2024-08-17T17:11:57.704078+0000","last_fullsized":"2024-08-17T17:11:57.704078+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:18.370628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1314,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703394+0000","last_change":"2024-08-17T17:06:39.627820+0000","last_active":"2024-08-17T17:11:57.703394+0000","last_peered":"2024-08-17T17:11:57.703394+0000","last_clean":"2024-08-17T17:11:57.703394+0000","last_became_active":"2024-08-17T17:06:39.627464+0000","last_became_peered":"2024-08-17T17:06:39.627464+0000","last_unstale":"2024-08-17T17:11:57.703394+0000","last_undegraded":"2024-08-17T17:11:57.703394+0000","last_fullsized":"2024-08-17T17:11:57.703394+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:01.500528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1036,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.010866+0000","last_change":"2024-08-17T17:06:39.628745+0000","last_active":"2024-08-17T17:11:53.010866+0000","last_peered":"2024-08-17T17:11:53.010866+0000","last_clean":"2024-08-17T17:11:53.010866+0000","last_became_active":"2024-08-17T17:06:39.628144+0000","last_became_peered":"2024-08-17T17:06:39.628144+0000","last_unstale":"2024-08-17T17:11:53.010866+0000","last_undegraded":"2024-08-17T17:11:53.010866+0000","last_fullsized":"2024-08-17T17:11:53.010866+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:14.740936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1026,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075115+0000","last_change":"2024-08-17T17:06:07.637581+0000","last_active":"2024-08-17T17:11:59.075115+0000","last_peered":"2024-08-17T17:11:59.075115+0000","last_clean":"2024-08-17T17:11:59.075115+0000","last_became_active":"2024-08-17T17:06:07.637282+0000","last_became_peered":"2024-08-17T17:06:07.637282+0000","last_unstale":"2024-08-17T17:11:59.075115+0000","last_undegraded":"2024-08-17T17:11:59.075115+0000","last_fullsized":"2024-08-17T17:11:59.075115+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.938734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1310,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702867+0000","last_change":"2024-08-17T17:07:45.687502+0000","last_active":"2024-08-17T17:11:57.702867+0000","last_peered":"2024-08-17T17:11:57.702867+0000","last_clean":"2024-08-17T17:11:57.702867+0000","last_became_active":"2024-08-17T17:07:45.687197+0000","last_became_peered":"2024-08-17T17:07:45.687197+0000","last_unstale":"2024-08-17T17:11:57.702867+0000","last_undegraded":"2024-08-17T17:11:57.702867+0000","last_fullsized":"2024-08-17T17:11:57.702867+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:23.577838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1037,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.981002+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:11:58.981002+0000","last_peered":"2024-08-17T17:11:58.981002+0000","last_clean":"2024-08-17T17:11:58.981002+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:11:58.981002+0000","last_undegraded":"2024-08-17T17:11:58.981002+0000","last_fullsized":"2024-08-17T17:11:58.981002+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1328,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703305+0000","last_change":"2024-08-17T17:06:39.633975+0000","last_active":"2024-08-17T17:11:57.703305+0000","last_peered":"2024-08-17T17:11:57.703305+0000","last_clean":"2024-08-17T17:11:57.703305+0000","last_became_active":"2024-08-17T17:06:39.631526+0000","last_became_peered":"2024-08-17T17:06:39.631526+0000","last_unstale":"2024-08-17T17:11:57.703305+0000","last_undegraded":"2024-08-17T17:11:57.703305+0000","last_fullsized":"2024-08-17T17:11:57.703305+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:58.542237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1288,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703023+0000","last_change":"2024-08-17T17:06:39.627577+0000","last_active":"2024-08-17T17:11:57.703023+0000","last_peered":"2024-08-17T17:11:57.703023+0000","last_clean":"2024-08-17T17:11:57.703023+0000","last_became_active":"2024-08-17T17:06:39.626554+0000","last_became_peered":"2024-08-17T17:06:39.626554+0000","last_unstale":"2024-08-17T17:11:57.703023+0000","last_undegraded":"2024-08-17T17:11:57.703023+0000","last_fullsized":"2024-08-17T17:11:57.703023+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:04:48.533057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1298,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702987+0000","last_change":"2024-08-17T17:07:12.645660+0000","last_active":"2024-08-17T17:11:57.702987+0000","last_peered":"2024-08-17T17:11:57.702987+0000","last_clean":"2024-08-17T17:11:57.702987+0000","last_became_active":"2024-08-17T17:07:12.645368+0000","last_became_peered":"2024-08-17T17:07:12.645368+0000","last_unstale":"2024-08-17T17:11:57.702987+0000","last_undegraded":"2024-08-17T17:11:57.702987+0000","last_fullsized":"2024-08-17T17:11:57.702987+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:41:00.157045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1019,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699659+0000","last_change":"2024-08-17T17:06:07.636239+0000","last_active":"2024-08-17T17:11:52.699659+0000","last_peered":"2024-08-17T17:11:52.699659+0000","last_clean":"2024-08-17T17:11:52.699659+0000","last_became_active":"2024-08-17T17:06:07.635881+0000","last_became_peered":"2024-08-17T17:06:07.635881+0000","last_unstale":"2024-08-17T17:11:52.699659+0000","last_undegraded":"2024-08-17T17:11:52.699659+0000","last_fullsized":"2024-08-17T17:11:52.699659+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:36.001648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1316,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648899+0000","last_change":"2024-08-17T17:06:07.634369+0000","last_active":"2024-08-17T17:11:58.648899+0000","last_peered":"2024-08-17T17:11:58.648899+0000","last_clean":"2024-08-17T17:11:58.648899+0000","last_became_active":"2024-08-17T17:06:07.633769+0000","last_became_peered":"2024-08-17T17:06:07.633769+0000","last_unstale":"2024-08-17T17:11:58.648899+0000","last_undegraded":"2024-08-17T17:11:58.648899+0000","last_fullsized":"2024-08-17T17:11:58.648899+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:20.957493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1258,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075605+0000","last_change":"2024-08-17T17:07:45.688526+0000","last_active":"2024-08-17T17:11:59.075605+0000","last_peered":"2024-08-17T17:11:59.075605+0000","last_clean":"2024-08-17T17:11:59.075605+0000","last_became_active":"2024-08-17T17:07:45.688091+0000","last_became_peered":"2024-08-17T17:07:45.688091+0000","last_unstale":"2024-08-17T17:11:59.075605+0000","last_undegraded":"2024-08-17T17:11:59.075605+0000","last_fullsized":"2024-08-17T17:11:59.075605+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:02.385683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1298,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703411+0000","last_change":"2024-08-17T17:06:39.629005+0000","last_active":"2024-08-17T17:11:57.703411+0000","last_peered":"2024-08-17T17:11:57.703411+0000","last_clean":"2024-08-17T17:11:57.703411+0000","last_became_active":"2024-08-17T17:06:39.628679+0000","last_became_peered":"2024-08-17T17:06:39.628679+0000","last_unstale":"2024-08-17T17:11:57.703411+0000","last_undegraded":"2024-08-17T17:11:57.703411+0000","last_fullsized":"2024-08-17T17:11:57.703411+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:09.401059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1255,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075180+0000","last_change":"2024-08-17T17:07:45.695858+0000","last_active":"2024-08-17T17:11:59.075180+0000","last_peered":"2024-08-17T17:11:59.075180+0000","last_clean":"2024-08-17T17:11:59.075180+0000","last_became_active":"2024-08-17T17:07:45.694111+0000","last_became_peered":"2024-08-17T17:07:45.694111+0000","last_unstale":"2024-08-17T17:11:59.075180+0000","last_undegraded":"2024-08-17T17:11:59.075180+0000","last_fullsized":"2024-08-17T17:11:59.075180+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:40:38.772090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":1330,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648523+0000","last_change":"2024-08-17T17:06:39.636866+0000","last_active":"2024-08-17T17:11:58.648523+0000","last_peered":"2024-08-17T17:11:58.648523+0000","last_clean":"2024-08-17T17:11:58.648523+0000","last_became_active":"2024-08-17T17:06:39.636470+0000","last_became_peered":"2024-08-17T17:06:39.636470+0000","last_unstale":"2024-08-17T17:11:58.648523+0000","last_undegraded":"2024-08-17T17:11:58.648523+0000","last_fullsized":"2024-08-17T17:11:58.648523+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:14.028712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1034,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:56.929467+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:11:56.929467+0000","last_peered":"2024-08-17T17:11:56.929467+0000","last_clean":"2024-08-17T17:11:56.929467+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:11:56.929467+0000","last_undegraded":"2024-08-17T17:11:56.929467+0000","last_fullsized":"2024-08-17T17:11:56.929467+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1010,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075411+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:11:59.075411+0000","last_peered":"2024-08-17T17:11:59.075411+0000","last_clean":"2024-08-17T17:11:59.075411+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:11:59.075411+0000","last_undegraded":"2024-08-17T17:11:59.075411+0000","last_fullsized":"2024-08-17T17:11:59.075411+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1024,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074961+0000","last_change":"2024-08-17T17:06:39.627594+0000","last_active":"2024-08-17T17:11:59.074961+0000","last_peered":"2024-08-17T17:11:59.074961+0000","last_clean":"2024-08-17T17:11:59.074961+0000","last_became_active":"2024-08-17T17:06:39.627105+0000","last_became_peered":"2024-08-17T17:06:39.627105+0000","last_unstale":"2024-08-17T17:11:59.074961+0000","last_undegraded":"2024-08-17T17:11:59.074961+0000","last_fullsized":"2024-08-17T17:11:59.074961+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:30.105070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1033,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075812+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:11:59.075812+0000","last_peered":"2024-08-17T17:11:59.075812+0000","last_clean":"2024-08-17T17:11:59.075812+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:11:59.075812+0000","last_undegraded":"2024-08-17T17:11:59.075812+0000","last_fullsized":"2024-08-17T17:11:59.075812+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1299,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703585+0000","last_change":"2024-08-17T17:07:12.645440+0000","last_active":"2024-08-17T17:11:57.703585+0000","last_peered":"2024-08-17T17:11:57.703585+0000","last_clean":"2024-08-17T17:11:57.703585+0000","last_became_active":"2024-08-17T17:07:12.645147+0000","last_became_peered":"2024-08-17T17:07:12.645147+0000","last_unstale":"2024-08-17T17:11:57.703585+0000","last_undegraded":"2024-08-17T17:11:57.703585+0000","last_fullsized":"2024-08-17T17:11:57.703585+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:25.588528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1008,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699386+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:11:52.699386+0000","last_peered":"2024-08-17T17:11:52.699386+0000","last_clean":"2024-08-17T17:11:52.699386+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:11:52.699386+0000","last_undegraded":"2024-08-17T17:11:52.699386+0000","last_fullsized":"2024-08-17T17:11:52.699386+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1020,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011326+0000","last_change":"2024-08-17T17:06:39.625029+0000","last_active":"2024-08-17T17:11:53.011326+0000","last_peered":"2024-08-17T17:11:53.011326+0000","last_clean":"2024-08-17T17:11:53.011326+0000","last_became_active":"2024-08-17T17:06:39.624688+0000","last_became_peered":"2024-08-17T17:06:39.624688+0000","last_unstale":"2024-08-17T17:11:53.011326+0000","last_undegraded":"2024-08-17T17:11:53.011326+0000","last_fullsized":"2024-08-17T17:11:53.011326+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:54.849668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1226,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075761+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:11:59.075761+0000","last_peered":"2024-08-17T17:11:59.075761+0000","last_clean":"2024-08-17T17:11:59.075761+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:11:59.075761+0000","last_undegraded":"2024-08-17T17:11:59.075761+0000","last_fullsized":"2024-08-17T17:11:59.075761+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1366,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648869+0000","last_change":"2024-08-17T17:07:45.690099+0000","last_active":"2024-08-17T17:11:58.648869+0000","last_peered":"2024-08-17T17:11:58.648869+0000","last_clean":"2024-08-17T17:11:58.648869+0000","last_became_active":"2024-08-17T17:07:45.689783+0000","last_became_peered":"2024-08-17T17:07:45.689783+0000","last_unstale":"2024-08-17T17:11:58.648869+0000","last_undegraded":"2024-08-17T17:11:58.648869+0000","last_fullsized":"2024-08-17T17:11:58.648869+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:38.311630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1245,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075127+0000","last_change":"2024-08-17T17:07:45.678950+0000","last_active":"2024-08-17T17:11:59.075127+0000","last_peered":"2024-08-17T17:11:59.075127+0000","last_clean":"2024-08-17T17:11:59.075127+0000","last_became_active":"2024-08-17T17:07:45.678472+0000","last_became_peered":"2024-08-17T17:07:45.678472+0000","last_unstale":"2024-08-17T17:11:59.075127+0000","last_undegraded":"2024-08-17T17:11:59.075127+0000","last_fullsized":"2024-08-17T17:11:59.075127+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:48:02.385927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1022,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703548+0000","last_change":"2024-08-17T17:06:39.628542+0000","last_active":"2024-08-17T17:11:57.703548+0000","last_peered":"2024-08-17T17:11:57.703548+0000","last_clean":"2024-08-17T17:11:57.703548+0000","last_became_active":"2024-08-17T17:06:39.628253+0000","last_became_peered":"2024-08-17T17:06:39.628253+0000","last_unstale":"2024-08-17T17:11:57.703548+0000","last_undegraded":"2024-08-17T17:11:57.703548+0000","last_fullsized":"2024-08-17T17:11:57.703548+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:22.349326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1261,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075295+0000","last_change":"2024-08-17T17:07:45.696068+0000","last_active":"2024-08-17T17:11:59.075295+0000","last_peered":"2024-08-17T17:11:59.075295+0000","last_clean":"2024-08-17T17:11:59.075295+0000","last_became_active":"2024-08-17T17:07:45.694617+0000","last_became_peered":"2024-08-17T17:07:45.694617+0000","last_unstale":"2024-08-17T17:11:59.075295+0000","last_undegraded":"2024-08-17T17:11:59.075295+0000","last_fullsized":"2024-08-17T17:11:59.075295+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:08.646380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1327,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648379+0000","last_change":"2024-08-17T17:07:45.684186+0000","last_active":"2024-08-17T17:11:58.648379+0000","last_peered":"2024-08-17T17:11:58.648379+0000","last_clean":"2024-08-17T17:11:58.648379+0000","last_became_active":"2024-08-17T17:07:45.683863+0000","last_became_peered":"2024-08-17T17:07:45.683863+0000","last_unstale":"2024-08-17T17:11:58.648379+0000","last_undegraded":"2024-08-17T17:11:58.648379+0000","last_fullsized":"2024-08-17T17:11:58.648379+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:52.106168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1298,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702758+0000","last_change":"2024-08-17T17:07:45.688645+0000","last_active":"2024-08-17T17:11:57.702758+0000","last_peered":"2024-08-17T17:11:57.702758+0000","last_clean":"2024-08-17T17:11:57.702758+0000","last_became_active":"2024-08-17T17:07:45.688383+0000","last_became_peered":"2024-08-17T17:07:45.688383+0000","last_unstale":"2024-08-17T17:11:57.702758+0000","last_undegraded":"2024-08-17T17:11:57.702758+0000","last_fullsized":"2024-08-17T17:11:57.702758+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:33:12.209565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1244,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074819+0000","last_change":"2024-08-17T17:07:45.689782+0000","last_active":"2024-08-17T17:11:59.074819+0000","last_peered":"2024-08-17T17:11:59.074819+0000","last_clean":"2024-08-17T17:11:59.074819+0000","last_became_active":"2024-08-17T17:07:45.689312+0000","last_became_peered":"2024-08-17T17:07:45.689312+0000","last_unstale":"2024-08-17T17:11:59.074819+0000","last_undegraded":"2024-08-17T17:11:59.074819+0000","last_fullsized":"2024-08-17T17:11:59.074819+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:07.560555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1040,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800592+0000","last_change":"2024-08-17T17:07:12.649175+0000","last_active":"2024-08-17T17:11:52.800592+0000","last_peered":"2024-08-17T17:11:52.800592+0000","last_clean":"2024-08-17T17:11:52.800592+0000","last_became_active":"2024-08-17T17:07:12.648822+0000","last_became_peered":"2024-08-17T17:07:12.648822+0000","last_unstale":"2024-08-17T17:11:52.800592+0000","last_undegraded":"2024-08-17T17:11:52.800592+0000","last_fullsized":"2024-08-17T17:11:52.800592+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:03.924431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1286,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074322+0000","last_change":"2024-08-17T17:07:45.685278+0000","last_active":"2024-08-17T17:11:59.074322+0000","last_peered":"2024-08-17T17:11:59.074322+0000","last_clean":"2024-08-17T17:11:59.074322+0000","last_became_active":"2024-08-17T17:07:45.684963+0000","last_became_peered":"2024-08-17T17:07:45.684963+0000","last_unstale":"2024-08-17T17:11:59.074322+0000","last_undegraded":"2024-08-17T17:11:59.074322+0000","last_fullsized":"2024-08-17T17:11:59.074322+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:55:42.495791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.649176+0000","last_change":"2024-08-17T17:06:07.634103+0000","last_active":"2024-08-17T17:11:58.649176+0000","last_peered":"2024-08-17T17:11:58.649176+0000","last_clean":"2024-08-17T17:11:58.649176+0000","last_became_active":"2024-08-17T17:06:07.633624+0000","last_became_peered":"2024-08-17T17:06:07.633624+0000","last_unstale":"2024-08-17T17:11:58.649176+0000","last_undegraded":"2024-08-17T17:11:58.649176+0000","last_fullsized":"2024-08-17T17:11:58.649176+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:58.573558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800611+0000","last_change":"2024-08-17T17:07:45.684606+0000","last_active":"2024-08-17T17:11:52.800611+0000","last_peered":"2024-08-17T17:11:52.800611+0000","last_clean":"2024-08-17T17:11:52.800611+0000","last_became_active":"2024-08-17T17:07:45.684124+0000","last_became_peered":"2024-08-17T17:07:45.684124+0000","last_unstale":"2024-08-17T17:11:52.800611+0000","last_undegraded":"2024-08-17T17:11:52.800611+0000","last_fullsized":"2024-08-17T17:11:52.800611+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:16.110507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1022,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011023+0000","last_change":"2024-08-17T17:06:39.625245+0000","last_active":"2024-08-17T17:11:53.011023+0000","last_peered":"2024-08-17T17:11:53.011023+0000","last_clean":"2024-08-17T17:11:53.011023+0000","last_became_active":"2024-08-17T17:06:39.624657+0000","last_became_peered":"2024-08-17T17:06:39.624657+0000","last_unstale":"2024-08-17T17:11:53.011023+0000","last_undegraded":"2024-08-17T17:11:53.011023+0000","last_fullsized":"2024-08-17T17:11:53.011023+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:43.097460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800484+0000","last_change":"2024-08-17T17:07:45.684769+0000","last_active":"2024-08-17T17:11:52.800484+0000","last_peered":"2024-08-17T17:11:52.800484+0000","last_clean":"2024-08-17T17:11:52.800484+0000","last_became_active":"2024-08-17T17:07:45.684459+0000","last_became_peered":"2024-08-17T17:07:45.684459+0000","last_unstale":"2024-08-17T17:11:52.800484+0000","last_undegraded":"2024-08-17T17:11:52.800484+0000","last_fullsized":"2024-08-17T17:11:52.800484+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:30.731978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1007,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.010451+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:11:53.010451+0000","last_peered":"2024-08-17T17:11:53.010451+0000","last_clean":"2024-08-17T17:11:53.010451+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:11:53.010451+0000","last_undegraded":"2024-08-17T17:11:53.010451+0000","last_fullsized":"2024-08-17T17:11:53.010451+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1251,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.076011+0000","last_change":"2024-08-17T17:07:45.692127+0000","last_active":"2024-08-17T17:11:59.076011+0000","last_peered":"2024-08-17T17:11:59.076011+0000","last_clean":"2024-08-17T17:11:59.076011+0000","last_became_active":"2024-08-17T17:07:45.690277+0000","last_became_peered":"2024-08-17T17:07:45.690277+0000","last_unstale":"2024-08-17T17:11:59.076011+0000","last_undegraded":"2024-08-17T17:11:59.076011+0000","last_fullsized":"2024-08-17T17:11:59.076011+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:04.555409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1239,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075238+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:11:59.075238+0000","last_peered":"2024-08-17T17:11:59.075238+0000","last_clean":"2024-08-17T17:11:59.075238+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:11:59.075238+0000","last_undegraded":"2024-08-17T17:11:59.075238+0000","last_fullsized":"2024-08-17T17:11:59.075238+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1023,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800297+0000","last_change":"2024-08-17T17:06:39.627689+0000","last_active":"2024-08-17T17:11:52.800297+0000","last_peered":"2024-08-17T17:11:52.800297+0000","last_clean":"2024-08-17T17:11:52.800297+0000","last_became_active":"2024-08-17T17:06:39.627375+0000","last_became_peered":"2024-08-17T17:06:39.627375+0000","last_unstale":"2024-08-17T17:11:52.800297+0000","last_undegraded":"2024-08-17T17:11:52.800297+0000","last_fullsized":"2024-08-17T17:11:52.800297+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:51.954076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":1333,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648938+0000","last_change":"2024-08-17T17:07:12.651933+0000","last_active":"2024-08-17T17:11:58.648938+0000","last_peered":"2024-08-17T17:11:58.648938+0000","last_clean":"2024-08-17T17:11:58.648938+0000","last_became_active":"2024-08-17T17:07:12.651592+0000","last_became_peered":"2024-08-17T17:07:12.651592+0000","last_unstale":"2024-08-17T17:11:58.648938+0000","last_undegraded":"2024-08-17T17:11:58.648938+0000","last_fullsized":"2024-08-17T17:11:58.648938+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:31.224038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201535,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":55668,"kb_used_data":10600,"kb_used_omap":1,"kb_used_meta":45054,"kb_avail":93689484,"statfs":{"total":95995035648,"available":95938031616,"internally_reserved":0,"allocated":10854400,"data_stored":7795252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":46135755},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463172,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67944,"kb_used_data":10592,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93677208,"statfs":{"total":95995035648,"available":95925460992,"internally_reserved":0,"allocated":10846208,"data_stored":7795125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":58718665},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626813,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":46296,"kb_used_data":9996,"kb_used_omap":42,"kb_used_meta":36245,"kb_avail":93698856,"statfs":{"total":95995035648,"available":95947628544,"internally_reserved":0,"allocated":10235904,"data_stored":7200014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":37114940},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785290,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":50584,"kb_used_data":9996,"kb_used_omap":48,"kb_used_meta":40527,"kb_avail":93694568,"statfs":{"total":95995035648,"available":95943237632,"internally_reserved":0,"allocated":10235904,"data_stored":7199943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":41500334},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724808,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":107488,"kb_used_data":10004,"kb_used_omap":1,"kb_used_meta":97470,"kb_avail":93637664,"statfs":{"total":95995035648,"available":95884967936,"internally_reserved":0,"allocated":10244096,"data_stored":7200583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":99809734},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986445,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":95212,"kb_used_data":10016,"kb_used_omap":1,"kb_used_meta":85182,"kb_avail":93649940,"statfs":{"total":95995035648,"available":95897538560,"internally_reserved":0,"allocated":10256384,"data_stored":7204814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":87226822},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706052,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":46172,"kb_used_data":10000,"kb_used_omap":35,"kb_used_meta":36124,"kb_avail":93698980,"statfs":{"total":95995035648,"available":95947755520,"internally_reserved":0,"allocated":10240000,"data_stored":7200540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":36991031},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831825,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":43040,"kb_used_data":10580,"kb_used_omap":65,"kb_used_meta":32382,"kb_avail":93702112,"statfs":{"total":95995035648,"available":95950962688,"internally_reserved":0,"allocated":10833920,"data_stored":7790896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":33159355},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:12:43.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:12:43.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:43 smithi162 ceph-mon[104668]: pgmap v1108: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:43.792 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:44.138 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:44 smithi063 ceph-mon[109843]: from='client.34581 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:12:44.493 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:44 smithi063 ceph-mon[114756]: from='client.34581 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:12:44.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:44 smithi162 ceph-mon[104668]: from='client.34581 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:12:45.405 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:45 smithi063 ceph-mon[114756]: pgmap v1109: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:45.406 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:45 smithi063 ceph-mon[109843]: pgmap v1109: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:45.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:45 smithi162 ceph-mon[104668]: pgmap v1109: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:45.618 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:12:45.619 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:12:46.292 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1109,"stamp":"2024-08-17T17:12:44.033171+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":512404,"kb_used_data":81784,"kb_used_omap":199,"kb_used_meta":430328,"kb_avail":749448812,"statfs":{"total":767960285184,"available":767435583488,"internally_reserved":0,"allocated":83746816,"data_stored":59387167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":440656636},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002932"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1245,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074580+0000","last_change":"2024-08-17T17:07:45.684223+0000","last_active":"2024-08-17T17:11:59.074580+0000","last_peered":"2024-08-17T17:11:59.074580+0000","last_clean":"2024-08-17T17:11:59.074580+0000","last_became_active":"2024-08-17T17:07:45.683948+0000","last_became_peered":"2024-08-17T17:07:45.683948+0000","last_unstale":"2024-08-17T17:11:59.074580+0000","last_undegraded":"2024-08-17T17:11:59.074580+0000","last_fullsized":"2024-08-17T17:11:59.074580+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:07.677423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1008,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075622+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:11:59.075622+0000","last_peered":"2024-08-17T17:11:59.075622+0000","last_clean":"2024-08-17T17:11:59.075622+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:11:59.075622+0000","last_undegraded":"2024-08-17T17:11:59.075622+0000","last_fullsized":"2024-08-17T17:11:59.075622+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1019,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011091+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:11:53.011091+0000","last_peered":"2024-08-17T17:11:53.011091+0000","last_clean":"2024-08-17T17:11:53.011091+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:11:53.011091+0000","last_undegraded":"2024-08-17T17:11:53.011091+0000","last_fullsized":"2024-08-17T17:11:53.011091+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1366,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648138+0000","last_change":"2024-08-17T17:07:45.683455+0000","last_active":"2024-08-17T17:11:58.648138+0000","last_peered":"2024-08-17T17:11:58.648138+0000","last_clean":"2024-08-17T17:11:58.648138+0000","last_became_active":"2024-08-17T17:07:45.683169+0000","last_became_peered":"2024-08-17T17:07:45.683169+0000","last_unstale":"2024-08-17T17:11:58.648138+0000","last_undegraded":"2024-08-17T17:11:58.648138+0000","last_fullsized":"2024-08-17T17:11:58.648138+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:18:19.562247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1325,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648396+0000","last_change":"2024-08-17T17:07:45.684222+0000","last_active":"2024-08-17T17:11:58.648396+0000","last_peered":"2024-08-17T17:11:58.648396+0000","last_clean":"2024-08-17T17:11:58.648396+0000","last_became_active":"2024-08-17T17:07:45.683913+0000","last_became_peered":"2024-08-17T17:07:45.683913+0000","last_unstale":"2024-08-17T17:11:58.648396+0000","last_undegraded":"2024-08-17T17:11:58.648396+0000","last_fullsized":"2024-08-17T17:11:58.648396+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:59:33.944853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":991,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075457+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:11:59.075457+0000","last_peered":"2024-08-17T17:11:59.075457+0000","last_clean":"2024-08-17T17:11:59.075457+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:11:59.075457+0000","last_undegraded":"2024-08-17T17:11:59.075457+0000","last_fullsized":"2024-08-17T17:11:59.075457+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1245,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075397+0000","last_change":"2024-08-17T17:07:45.687030+0000","last_active":"2024-08-17T17:11:59.075397+0000","last_peered":"2024-08-17T17:11:59.075397+0000","last_clean":"2024-08-17T17:11:59.075397+0000","last_became_active":"2024-08-17T17:07:45.686610+0000","last_became_peered":"2024-08-17T17:07:45.686610+0000","last_unstale":"2024-08-17T17:11:59.075397+0000","last_undegraded":"2024-08-17T17:11:59.075397+0000","last_fullsized":"2024-08-17T17:11:59.075397+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:02.425845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1030,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702965+0000","last_change":"2024-08-17T17:06:07.632931+0000","last_active":"2024-08-17T17:11:57.702965+0000","last_peered":"2024-08-17T17:11:57.702965+0000","last_clean":"2024-08-17T17:11:57.702965+0000","last_became_active":"2024-08-17T17:06:07.632525+0000","last_became_peered":"2024-08-17T17:06:07.632525+0000","last_unstale":"2024-08-17T17:11:57.702965+0000","last_undegraded":"2024-08-17T17:11:57.702965+0000","last_fullsized":"2024-08-17T17:11:57.702965+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:01.740674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1007,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699675+0000","last_change":"2024-08-17T17:06:07.638468+0000","last_active":"2024-08-17T17:11:52.699675+0000","last_peered":"2024-08-17T17:11:52.699675+0000","last_clean":"2024-08-17T17:11:52.699675+0000","last_became_active":"2024-08-17T17:06:07.636046+0000","last_became_peered":"2024-08-17T17:06:07.636046+0000","last_unstale":"2024-08-17T17:11:52.699675+0000","last_undegraded":"2024-08-17T17:11:52.699675+0000","last_fullsized":"2024-08-17T17:11:52.699675+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:37:12.228090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1005,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.010559+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:11:53.010559+0000","last_peered":"2024-08-17T17:11:53.010559+0000","last_clean":"2024-08-17T17:11:53.010559+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:11:53.010559+0000","last_undegraded":"2024-08-17T17:11:53.010559+0000","last_fullsized":"2024-08-17T17:11:53.010559+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1008,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699604+0000","last_change":"2024-08-17T17:06:39.638433+0000","last_active":"2024-08-17T17:11:52.699604+0000","last_peered":"2024-08-17T17:11:52.699604+0000","last_clean":"2024-08-17T17:11:52.699604+0000","last_became_active":"2024-08-17T17:06:39.638081+0000","last_became_peered":"2024-08-17T17:06:39.638081+0000","last_unstale":"2024-08-17T17:11:52.699604+0000","last_undegraded":"2024-08-17T17:11:52.699604+0000","last_fullsized":"2024-08-17T17:11:52.699604+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:13.558012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800667+0000","last_change":"2024-08-17T17:07:45.693574+0000","last_active":"2024-08-17T17:11:52.800667+0000","last_peered":"2024-08-17T17:11:52.800667+0000","last_clean":"2024-08-17T17:11:52.800667+0000","last_became_active":"2024-08-17T17:07:45.693086+0000","last_became_peered":"2024-08-17T17:07:45.693086+0000","last_unstale":"2024-08-17T17:11:52.800667+0000","last_undegraded":"2024-08-17T17:11:52.800667+0000","last_fullsized":"2024-08-17T17:11:52.800667+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:31.888894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":992,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699566+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:11:52.699566+0000","last_peered":"2024-08-17T17:11:52.699566+0000","last_clean":"2024-08-17T17:11:52.699566+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:11:52.699566+0000","last_undegraded":"2024-08-17T17:11:52.699566+0000","last_fullsized":"2024-08-17T17:11:52.699566+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1021,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699849+0000","last_change":"2024-08-17T17:06:39.629368+0000","last_active":"2024-08-17T17:11:52.699849+0000","last_peered":"2024-08-17T17:11:52.699849+0000","last_clean":"2024-08-17T17:11:52.699849+0000","last_became_active":"2024-08-17T17:06:39.628832+0000","last_became_peered":"2024-08-17T17:06:39.628832+0000","last_unstale":"2024-08-17T17:11:52.699849+0000","last_undegraded":"2024-08-17T17:11:52.699849+0000","last_fullsized":"2024-08-17T17:11:52.699849+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:54.063884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1021,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011147+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:11:53.011147+0000","last_peered":"2024-08-17T17:11:53.011147+0000","last_clean":"2024-08-17T17:11:53.011147+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:11:53.011147+0000","last_undegraded":"2024-08-17T17:11:53.011147+0000","last_fullsized":"2024-08-17T17:11:53.011147+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1338,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.649075+0000","last_change":"2024-08-17T17:07:12.651379+0000","last_active":"2024-08-17T17:11:58.649075+0000","last_peered":"2024-08-17T17:11:58.649075+0000","last_clean":"2024-08-17T17:11:58.649075+0000","last_became_active":"2024-08-17T17:07:12.651069+0000","last_became_peered":"2024-08-17T17:07:12.651069+0000","last_unstale":"2024-08-17T17:11:58.649075+0000","last_undegraded":"2024-08-17T17:11:58.649075+0000","last_fullsized":"2024-08-17T17:11:58.649075+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:25.384305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1258,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075731+0000","last_change":"2024-08-17T17:07:45.691437+0000","last_active":"2024-08-17T17:11:59.075731+0000","last_peered":"2024-08-17T17:11:59.075731+0000","last_clean":"2024-08-17T17:11:59.075731+0000","last_became_active":"2024-08-17T17:07:45.690567+0000","last_became_peered":"2024-08-17T17:07:45.690567+0000","last_unstale":"2024-08-17T17:11:59.075731+0000","last_undegraded":"2024-08-17T17:11:59.075731+0000","last_fullsized":"2024-08-17T17:11:59.075731+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:43:07.698912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1024,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800708+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:11:52.800708+0000","last_peered":"2024-08-17T17:11:52.800708+0000","last_clean":"2024-08-17T17:11:52.800708+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:11:52.800708+0000","last_undegraded":"2024-08-17T17:11:52.800708+0000","last_fullsized":"2024-08-17T17:11:52.800708+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1022,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.700185+0000","last_change":"2024-08-17T17:06:39.629619+0000","last_active":"2024-08-17T17:11:52.700185+0000","last_peered":"2024-08-17T17:11:52.700185+0000","last_clean":"2024-08-17T17:11:52.700185+0000","last_became_active":"2024-08-17T17:06:39.629295+0000","last_became_peered":"2024-08-17T17:06:39.629295+0000","last_unstale":"2024-08-17T17:11:52.700185+0000","last_undegraded":"2024-08-17T17:11:52.700185+0000","last_fullsized":"2024-08-17T17:11:52.700185+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:15.874423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.801064+0000","last_change":"2024-08-17T17:07:12.649178+0000","last_active":"2024-08-17T17:11:52.801064+0000","last_peered":"2024-08-17T17:11:52.801064+0000","last_clean":"2024-08-17T17:11:52.801064+0000","last_became_active":"2024-08-17T17:07:12.648844+0000","last_became_peered":"2024-08-17T17:07:12.648844+0000","last_unstale":"2024-08-17T17:11:52.801064+0000","last_undegraded":"2024-08-17T17:11:52.801064+0000","last_fullsized":"2024-08-17T17:11:52.801064+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:12:06.307736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1025,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800742+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:11:52.800742+0000","last_peered":"2024-08-17T17:11:52.800742+0000","last_clean":"2024-08-17T17:11:52.800742+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:11:52.800742+0000","last_undegraded":"2024-08-17T17:11:52.800742+0000","last_fullsized":"2024-08-17T17:11:52.800742+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1024,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.801216+0000","last_change":"2024-08-17T17:06:39.627678+0000","last_active":"2024-08-17T17:11:52.801216+0000","last_peered":"2024-08-17T17:11:52.801216+0000","last_clean":"2024-08-17T17:11:52.801216+0000","last_became_active":"2024-08-17T17:06:39.627226+0000","last_became_peered":"2024-08-17T17:06:39.627226+0000","last_unstale":"2024-08-17T17:11:52.801216+0000","last_undegraded":"2024-08-17T17:11:52.801216+0000","last_fullsized":"2024-08-17T17:11:52.801216+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:01.868244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1247,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075390+0000","last_change":"2024-08-17T17:07:45.680458+0000","last_active":"2024-08-17T17:11:59.075390+0000","last_peered":"2024-08-17T17:11:59.075390+0000","last_clean":"2024-08-17T17:11:59.075390+0000","last_became_active":"2024-08-17T17:07:45.680221+0000","last_became_peered":"2024-08-17T17:07:45.680221+0000","last_unstale":"2024-08-17T17:11:59.075390+0000","last_undegraded":"2024-08-17T17:11:59.075390+0000","last_fullsized":"2024-08-17T17:11:59.075390+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:55.083962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1269,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075358+0000","last_change":"2024-08-17T17:07:45.682866+0000","last_active":"2024-08-17T17:11:59.075358+0000","last_peered":"2024-08-17T17:11:59.075358+0000","last_clean":"2024-08-17T17:11:59.075358+0000","last_became_active":"2024-08-17T17:07:45.682554+0000","last_became_peered":"2024-08-17T17:07:45.682554+0000","last_unstale":"2024-08-17T17:11:59.075358+0000","last_undegraded":"2024-08-17T17:11:59.075358+0000","last_fullsized":"2024-08-17T17:11:59.075358+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:25.975914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1338,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074716+0000","last_change":"2024-08-17T17:06:07.636484+0000","last_active":"2024-08-17T17:11:59.074716+0000","last_peered":"2024-08-17T17:11:59.074716+0000","last_clean":"2024-08-17T17:11:59.074716+0000","last_became_active":"2024-08-17T17:06:07.635998+0000","last_became_peered":"2024-08-17T17:06:07.635998+0000","last_unstale":"2024-08-17T17:11:59.074716+0000","last_undegraded":"2024-08-17T17:11:59.074716+0000","last_fullsized":"2024-08-17T17:11:59.074716+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:16.096587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1035,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011021+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:11:53.011021+0000","last_peered":"2024-08-17T17:11:53.011021+0000","last_clean":"2024-08-17T17:11:53.011021+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:11:53.011021+0000","last_undegraded":"2024-08-17T17:11:53.011021+0000","last_fullsized":"2024-08-17T17:11:53.011021+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1226,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074625+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:11:59.074625+0000","last_peered":"2024-08-17T17:11:59.074625+0000","last_clean":"2024-08-17T17:11:59.074625+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:11:59.074625+0000","last_undegraded":"2024-08-17T17:11:59.074625+0000","last_fullsized":"2024-08-17T17:11:59.074625+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1007,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703217+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:11:57.703217+0000","last_peered":"2024-08-17T17:11:57.703217+0000","last_clean":"2024-08-17T17:11:57.703217+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:11:57.703217+0000","last_undegraded":"2024-08-17T17:11:57.703217+0000","last_fullsized":"2024-08-17T17:11:57.703217+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1035,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074623+0000","last_change":"2024-08-17T17:06:39.638376+0000","last_active":"2024-08-17T17:11:59.074623+0000","last_peered":"2024-08-17T17:11:59.074623+0000","last_clean":"2024-08-17T17:11:59.074623+0000","last_became_active":"2024-08-17T17:06:39.638050+0000","last_became_peered":"2024-08-17T17:06:39.638050+0000","last_unstale":"2024-08-17T17:11:59.074623+0000","last_undegraded":"2024-08-17T17:11:59.074623+0000","last_fullsized":"2024-08-17T17:11:59.074623+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:05.782492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1367,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648019+0000","last_change":"2024-08-17T17:07:45.683556+0000","last_active":"2024-08-17T17:11:58.648019+0000","last_peered":"2024-08-17T17:11:58.648019+0000","last_clean":"2024-08-17T17:11:58.648019+0000","last_became_active":"2024-08-17T17:07:45.683232+0000","last_became_peered":"2024-08-17T17:07:45.683232+0000","last_unstale":"2024-08-17T17:11:58.648019+0000","last_undegraded":"2024-08-17T17:11:58.648019+0000","last_fullsized":"2024-08-17T17:11:58.648019+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:16.112417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1339,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648339+0000","last_change":"2024-08-17T17:07:12.651145+0000","last_active":"2024-08-17T17:11:58.648339+0000","last_peered":"2024-08-17T17:11:58.648339+0000","last_clean":"2024-08-17T17:11:58.648339+0000","last_became_active":"2024-08-17T17:07:12.650839+0000","last_became_peered":"2024-08-17T17:07:12.650839+0000","last_unstale":"2024-08-17T17:11:58.648339+0000","last_undegraded":"2024-08-17T17:11:58.648339+0000","last_fullsized":"2024-08-17T17:11:58.648339+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:59.795181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1309,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074165+0000","last_change":"2024-08-17T17:07:45.681107+0000","last_active":"2024-08-17T17:11:59.074165+0000","last_peered":"2024-08-17T17:11:59.074165+0000","last_clean":"2024-08-17T17:11:59.074165+0000","last_became_active":"2024-08-17T17:07:45.680770+0000","last_became_peered":"2024-08-17T17:07:45.680770+0000","last_unstale":"2024-08-17T17:11:59.074165+0000","last_undegraded":"2024-08-17T17:11:59.074165+0000","last_fullsized":"2024-08-17T17:11:59.074165+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:53.820920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1300,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.704078+0000","last_change":"2024-08-17T17:07:12.649485+0000","last_active":"2024-08-17T17:11:57.704078+0000","last_peered":"2024-08-17T17:11:57.704078+0000","last_clean":"2024-08-17T17:11:57.704078+0000","last_became_active":"2024-08-17T17:07:12.649161+0000","last_became_peered":"2024-08-17T17:07:12.649161+0000","last_unstale":"2024-08-17T17:11:57.704078+0000","last_undegraded":"2024-08-17T17:11:57.704078+0000","last_fullsized":"2024-08-17T17:11:57.704078+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:18.370628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1314,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703394+0000","last_change":"2024-08-17T17:06:39.627820+0000","last_active":"2024-08-17T17:11:57.703394+0000","last_peered":"2024-08-17T17:11:57.703394+0000","last_clean":"2024-08-17T17:11:57.703394+0000","last_became_active":"2024-08-17T17:06:39.627464+0000","last_became_peered":"2024-08-17T17:06:39.627464+0000","last_unstale":"2024-08-17T17:11:57.703394+0000","last_undegraded":"2024-08-17T17:11:57.703394+0000","last_fullsized":"2024-08-17T17:11:57.703394+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:01.500528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1036,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.010866+0000","last_change":"2024-08-17T17:06:39.628745+0000","last_active":"2024-08-17T17:11:53.010866+0000","last_peered":"2024-08-17T17:11:53.010866+0000","last_clean":"2024-08-17T17:11:53.010866+0000","last_became_active":"2024-08-17T17:06:39.628144+0000","last_became_peered":"2024-08-17T17:06:39.628144+0000","last_unstale":"2024-08-17T17:11:53.010866+0000","last_undegraded":"2024-08-17T17:11:53.010866+0000","last_fullsized":"2024-08-17T17:11:53.010866+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:14.740936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1026,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075115+0000","last_change":"2024-08-17T17:06:07.637581+0000","last_active":"2024-08-17T17:11:59.075115+0000","last_peered":"2024-08-17T17:11:59.075115+0000","last_clean":"2024-08-17T17:11:59.075115+0000","last_became_active":"2024-08-17T17:06:07.637282+0000","last_became_peered":"2024-08-17T17:06:07.637282+0000","last_unstale":"2024-08-17T17:11:59.075115+0000","last_undegraded":"2024-08-17T17:11:59.075115+0000","last_fullsized":"2024-08-17T17:11:59.075115+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.938734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1310,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702867+0000","last_change":"2024-08-17T17:07:45.687502+0000","last_active":"2024-08-17T17:11:57.702867+0000","last_peered":"2024-08-17T17:11:57.702867+0000","last_clean":"2024-08-17T17:11:57.702867+0000","last_became_active":"2024-08-17T17:07:45.687197+0000","last_became_peered":"2024-08-17T17:07:45.687197+0000","last_unstale":"2024-08-17T17:11:57.702867+0000","last_undegraded":"2024-08-17T17:11:57.702867+0000","last_fullsized":"2024-08-17T17:11:57.702867+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:23.577838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1037,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.981002+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:11:58.981002+0000","last_peered":"2024-08-17T17:11:58.981002+0000","last_clean":"2024-08-17T17:11:58.981002+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:11:58.981002+0000","last_undegraded":"2024-08-17T17:11:58.981002+0000","last_fullsized":"2024-08-17T17:11:58.981002+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1328,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703305+0000","last_change":"2024-08-17T17:06:39.633975+0000","last_active":"2024-08-17T17:11:57.703305+0000","last_peered":"2024-08-17T17:11:57.703305+0000","last_clean":"2024-08-17T17:11:57.703305+0000","last_became_active":"2024-08-17T17:06:39.631526+0000","last_became_peered":"2024-08-17T17:06:39.631526+0000","last_unstale":"2024-08-17T17:11:57.703305+0000","last_undegraded":"2024-08-17T17:11:57.703305+0000","last_fullsized":"2024-08-17T17:11:57.703305+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:58.542237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1288,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703023+0000","last_change":"2024-08-17T17:06:39.627577+0000","last_active":"2024-08-17T17:11:57.703023+0000","last_peered":"2024-08-17T17:11:57.703023+0000","last_clean":"2024-08-17T17:11:57.703023+0000","last_became_active":"2024-08-17T17:06:39.626554+0000","last_became_peered":"2024-08-17T17:06:39.626554+0000","last_unstale":"2024-08-17T17:11:57.703023+0000","last_undegraded":"2024-08-17T17:11:57.703023+0000","last_fullsized":"2024-08-17T17:11:57.703023+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:04:48.533057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1298,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702987+0000","last_change":"2024-08-17T17:07:12.645660+0000","last_active":"2024-08-17T17:11:57.702987+0000","last_peered":"2024-08-17T17:11:57.702987+0000","last_clean":"2024-08-17T17:11:57.702987+0000","last_became_active":"2024-08-17T17:07:12.645368+0000","last_became_peered":"2024-08-17T17:07:12.645368+0000","last_unstale":"2024-08-17T17:11:57.702987+0000","last_undegraded":"2024-08-17T17:11:57.702987+0000","last_fullsized":"2024-08-17T17:11:57.702987+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:41:00.157045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1019,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699659+0000","last_change":"2024-08-17T17:06:07.636239+0000","last_active":"2024-08-17T17:11:52.699659+0000","last_peered":"2024-08-17T17:11:52.699659+0000","last_clean":"2024-08-17T17:11:52.699659+0000","last_became_active":"2024-08-17T17:06:07.635881+0000","last_became_peered":"2024-08-17T17:06:07.635881+0000","last_unstale":"2024-08-17T17:11:52.699659+0000","last_undegraded":"2024-08-17T17:11:52.699659+0000","last_fullsized":"2024-08-17T17:11:52.699659+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:36.001648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1316,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648899+0000","last_change":"2024-08-17T17:06:07.634369+0000","last_active":"2024-08-17T17:11:58.648899+0000","last_peered":"2024-08-17T17:11:58.648899+0000","last_clean":"2024-08-17T17:11:58.648899+0000","last_became_active":"2024-08-17T17:06:07.633769+0000","last_became_peered":"2024-08-17T17:06:07.633769+0000","last_unstale":"2024-08-17T17:11:58.648899+0000","last_undegraded":"2024-08-17T17:11:58.648899+0000","last_fullsized":"2024-08-17T17:11:58.648899+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:20.957493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1258,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075605+0000","last_change":"2024-08-17T17:07:45.688526+0000","last_active":"2024-08-17T17:11:59.075605+0000","last_peered":"2024-08-17T17:11:59.075605+0000","last_clean":"2024-08-17T17:11:59.075605+0000","last_became_active":"2024-08-17T17:07:45.688091+0000","last_became_peered":"2024-08-17T17:07:45.688091+0000","last_unstale":"2024-08-17T17:11:59.075605+0000","last_undegraded":"2024-08-17T17:11:59.075605+0000","last_fullsized":"2024-08-17T17:11:59.075605+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:02.385683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1298,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703411+0000","last_change":"2024-08-17T17:06:39.629005+0000","last_active":"2024-08-17T17:11:57.703411+0000","last_peered":"2024-08-17T17:11:57.703411+0000","last_clean":"2024-08-17T17:11:57.703411+0000","last_became_active":"2024-08-17T17:06:39.628679+0000","last_became_peered":"2024-08-17T17:06:39.628679+0000","last_unstale":"2024-08-17T17:11:57.703411+0000","last_undegraded":"2024-08-17T17:11:57.703411+0000","last_fullsized":"2024-08-17T17:11:57.703411+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:09.401059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1255,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075180+0000","last_change":"2024-08-17T17:07:45.695858+0000","last_active":"2024-08-17T17:11:59.075180+0000","last_peered":"2024-08-17T17:11:59.075180+0000","last_clean":"2024-08-17T17:11:59.075180+0000","last_became_active":"2024-08-17T17:07:45.694111+0000","last_became_peered":"2024-08-17T17:07:45.694111+0000","last_unstale":"2024-08-17T17:11:59.075180+0000","last_undegraded":"2024-08-17T17:11:59.075180+0000","last_fullsized":"2024-08-17T17:11:59.075180+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:40:38.772090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":1330,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648523+0000","last_change":"2024-08-17T17:06:39.636866+0000","last_active":"2024-08-17T17:11:58.648523+0000","last_peered":"2024-08-17T17:11:58.648523+0000","last_clean":"2024-08-17T17:11:58.648523+0000","last_became_active":"2024-08-17T17:06:39.636470+0000","last_became_peered":"2024-08-17T17:06:39.636470+0000","last_unstale":"2024-08-17T17:11:58.648523+0000","last_undegraded":"2024-08-17T17:11:58.648523+0000","last_fullsized":"2024-08-17T17:11:58.648523+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:14.028712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1034,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:56.929467+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:11:56.929467+0000","last_peered":"2024-08-17T17:11:56.929467+0000","last_clean":"2024-08-17T17:11:56.929467+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:11:56.929467+0000","last_undegraded":"2024-08-17T17:11:56.929467+0000","last_fullsized":"2024-08-17T17:11:56.929467+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1010,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075411+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:11:59.075411+0000","last_peered":"2024-08-17T17:11:59.075411+0000","last_clean":"2024-08-17T17:11:59.075411+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:11:59.075411+0000","last_undegraded":"2024-08-17T17:11:59.075411+0000","last_fullsized":"2024-08-17T17:11:59.075411+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1024,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074961+0000","last_change":"2024-08-17T17:06:39.627594+0000","last_active":"2024-08-17T17:11:59.074961+0000","last_peered":"2024-08-17T17:11:59.074961+0000","last_clean":"2024-08-17T17:11:59.074961+0000","last_became_active":"2024-08-17T17:06:39.627105+0000","last_became_peered":"2024-08-17T17:06:39.627105+0000","last_unstale":"2024-08-17T17:11:59.074961+0000","last_undegraded":"2024-08-17T17:11:59.074961+0000","last_fullsized":"2024-08-17T17:11:59.074961+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:30.105070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1033,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075812+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:11:59.075812+0000","last_peered":"2024-08-17T17:11:59.075812+0000","last_clean":"2024-08-17T17:11:59.075812+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:11:59.075812+0000","last_undegraded":"2024-08-17T17:11:59.075812+0000","last_fullsized":"2024-08-17T17:11:59.075812+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1299,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703585+0000","last_change":"2024-08-17T17:07:12.645440+0000","last_active":"2024-08-17T17:11:57.703585+0000","last_peered":"2024-08-17T17:11:57.703585+0000","last_clean":"2024-08-17T17:11:57.703585+0000","last_became_active":"2024-08-17T17:07:12.645147+0000","last_became_peered":"2024-08-17T17:07:12.645147+0000","last_unstale":"2024-08-17T17:11:57.703585+0000","last_undegraded":"2024-08-17T17:11:57.703585+0000","last_fullsized":"2024-08-17T17:11:57.703585+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:25.588528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1008,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699386+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:11:52.699386+0000","last_peered":"2024-08-17T17:11:52.699386+0000","last_clean":"2024-08-17T17:11:52.699386+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:11:52.699386+0000","last_undegraded":"2024-08-17T17:11:52.699386+0000","last_fullsized":"2024-08-17T17:11:52.699386+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1020,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011326+0000","last_change":"2024-08-17T17:06:39.625029+0000","last_active":"2024-08-17T17:11:53.011326+0000","last_peered":"2024-08-17T17:11:53.011326+0000","last_clean":"2024-08-17T17:11:53.011326+0000","last_became_active":"2024-08-17T17:06:39.624688+0000","last_became_peered":"2024-08-17T17:06:39.624688+0000","last_unstale":"2024-08-17T17:11:53.011326+0000","last_undegraded":"2024-08-17T17:11:53.011326+0000","last_fullsized":"2024-08-17T17:11:53.011326+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:54.849668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1226,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075761+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:11:59.075761+0000","last_peered":"2024-08-17T17:11:59.075761+0000","last_clean":"2024-08-17T17:11:59.075761+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:11:59.075761+0000","last_undegraded":"2024-08-17T17:11:59.075761+0000","last_fullsized":"2024-08-17T17:11:59.075761+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1366,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648869+0000","last_change":"2024-08-17T17:07:45.690099+0000","last_active":"2024-08-17T17:11:58.648869+0000","last_peered":"2024-08-17T17:11:58.648869+0000","last_clean":"2024-08-17T17:11:58.648869+0000","last_became_active":"2024-08-17T17:07:45.689783+0000","last_became_peered":"2024-08-17T17:07:45.689783+0000","last_unstale":"2024-08-17T17:11:58.648869+0000","last_undegraded":"2024-08-17T17:11:58.648869+0000","last_fullsized":"2024-08-17T17:11:58.648869+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:38.311630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1245,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075127+0000","last_change":"2024-08-17T17:07:45.678950+0000","last_active":"2024-08-17T17:11:59.075127+0000","last_peered":"2024-08-17T17:11:59.075127+0000","last_clean":"2024-08-17T17:11:59.075127+0000","last_became_active":"2024-08-17T17:07:45.678472+0000","last_became_peered":"2024-08-17T17:07:45.678472+0000","last_unstale":"2024-08-17T17:11:59.075127+0000","last_undegraded":"2024-08-17T17:11:59.075127+0000","last_fullsized":"2024-08-17T17:11:59.075127+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:48:02.385927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1022,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703548+0000","last_change":"2024-08-17T17:06:39.628542+0000","last_active":"2024-08-17T17:11:57.703548+0000","last_peered":"2024-08-17T17:11:57.703548+0000","last_clean":"2024-08-17T17:11:57.703548+0000","last_became_active":"2024-08-17T17:06:39.628253+0000","last_became_peered":"2024-08-17T17:06:39.628253+0000","last_unstale":"2024-08-17T17:11:57.703548+0000","last_undegraded":"2024-08-17T17:11:57.703548+0000","last_fullsized":"2024-08-17T17:11:57.703548+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:22.349326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1261,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075295+0000","last_change":"2024-08-17T17:07:45.696068+0000","last_active":"2024-08-17T17:11:59.075295+0000","last_peered":"2024-08-17T17:11:59.075295+0000","last_clean":"2024-08-17T17:11:59.075295+0000","last_became_active":"2024-08-17T17:07:45.694617+0000","last_became_peered":"2024-08-17T17:07:45.694617+0000","last_unstale":"2024-08-17T17:11:59.075295+0000","last_undegraded":"2024-08-17T17:11:59.075295+0000","last_fullsized":"2024-08-17T17:11:59.075295+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:08.646380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1327,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648379+0000","last_change":"2024-08-17T17:07:45.684186+0000","last_active":"2024-08-17T17:11:58.648379+0000","last_peered":"2024-08-17T17:11:58.648379+0000","last_clean":"2024-08-17T17:11:58.648379+0000","last_became_active":"2024-08-17T17:07:45.683863+0000","last_became_peered":"2024-08-17T17:07:45.683863+0000","last_unstale":"2024-08-17T17:11:58.648379+0000","last_undegraded":"2024-08-17T17:11:58.648379+0000","last_fullsized":"2024-08-17T17:11:58.648379+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:52.106168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1298,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702758+0000","last_change":"2024-08-17T17:07:45.688645+0000","last_active":"2024-08-17T17:11:57.702758+0000","last_peered":"2024-08-17T17:11:57.702758+0000","last_clean":"2024-08-17T17:11:57.702758+0000","last_became_active":"2024-08-17T17:07:45.688383+0000","last_became_peered":"2024-08-17T17:07:45.688383+0000","last_unstale":"2024-08-17T17:11:57.702758+0000","last_undegraded":"2024-08-17T17:11:57.702758+0000","last_fullsized":"2024-08-17T17:11:57.702758+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:33:12.209565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1245,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074861+0000","last_change":"2024-08-17T17:07:45.689782+0000","last_active":"2024-08-17T17:11:59.074861+0000","last_peered":"2024-08-17T17:11:59.074861+0000","last_clean":"2024-08-17T17:11:59.074861+0000","last_became_active":"2024-08-17T17:07:45.689312+0000","last_became_peered":"2024-08-17T17:07:45.689312+0000","last_unstale":"2024-08-17T17:11:59.074861+0000","last_undegraded":"2024-08-17T17:11:59.074861+0000","last_fullsized":"2024-08-17T17:11:59.074861+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:07.560555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1040,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800592+0000","last_change":"2024-08-17T17:07:12.649175+0000","last_active":"2024-08-17T17:11:52.800592+0000","last_peered":"2024-08-17T17:11:52.800592+0000","last_clean":"2024-08-17T17:11:52.800592+0000","last_became_active":"2024-08-17T17:07:12.648822+0000","last_became_peered":"2024-08-17T17:07:12.648822+0000","last_unstale":"2024-08-17T17:11:52.800592+0000","last_undegraded":"2024-08-17T17:11:52.800592+0000","last_fullsized":"2024-08-17T17:11:52.800592+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:03.924431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1286,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074322+0000","last_change":"2024-08-17T17:07:45.685278+0000","last_active":"2024-08-17T17:11:59.074322+0000","last_peered":"2024-08-17T17:11:59.074322+0000","last_clean":"2024-08-17T17:11:59.074322+0000","last_became_active":"2024-08-17T17:07:45.684963+0000","last_became_peered":"2024-08-17T17:07:45.684963+0000","last_unstale":"2024-08-17T17:11:59.074322+0000","last_undegraded":"2024-08-17T17:11:59.074322+0000","last_fullsized":"2024-08-17T17:11:59.074322+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:55:42.495791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.649176+0000","last_change":"2024-08-17T17:06:07.634103+0000","last_active":"2024-08-17T17:11:58.649176+0000","last_peered":"2024-08-17T17:11:58.649176+0000","last_clean":"2024-08-17T17:11:58.649176+0000","last_became_active":"2024-08-17T17:06:07.633624+0000","last_became_peered":"2024-08-17T17:06:07.633624+0000","last_unstale":"2024-08-17T17:11:58.649176+0000","last_undegraded":"2024-08-17T17:11:58.649176+0000","last_fullsized":"2024-08-17T17:11:58.649176+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:58.573558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800611+0000","last_change":"2024-08-17T17:07:45.684606+0000","last_active":"2024-08-17T17:11:52.800611+0000","last_peered":"2024-08-17T17:11:52.800611+0000","last_clean":"2024-08-17T17:11:52.800611+0000","last_became_active":"2024-08-17T17:07:45.684124+0000","last_became_peered":"2024-08-17T17:07:45.684124+0000","last_unstale":"2024-08-17T17:11:52.800611+0000","last_undegraded":"2024-08-17T17:11:52.800611+0000","last_fullsized":"2024-08-17T17:11:52.800611+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:16.110507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1022,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011023+0000","last_change":"2024-08-17T17:06:39.625245+0000","last_active":"2024-08-17T17:11:53.011023+0000","last_peered":"2024-08-17T17:11:53.011023+0000","last_clean":"2024-08-17T17:11:53.011023+0000","last_became_active":"2024-08-17T17:06:39.624657+0000","last_became_peered":"2024-08-17T17:06:39.624657+0000","last_unstale":"2024-08-17T17:11:53.011023+0000","last_undegraded":"2024-08-17T17:11:53.011023+0000","last_fullsized":"2024-08-17T17:11:53.011023+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:43.097460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800484+0000","last_change":"2024-08-17T17:07:45.684769+0000","last_active":"2024-08-17T17:11:52.800484+0000","last_peered":"2024-08-17T17:11:52.800484+0000","last_clean":"2024-08-17T17:11:52.800484+0000","last_became_active":"2024-08-17T17:07:45.684459+0000","last_became_peered":"2024-08-17T17:07:45.684459+0000","last_unstale":"2024-08-17T17:11:52.800484+0000","last_undegraded":"2024-08-17T17:11:52.800484+0000","last_fullsized":"2024-08-17T17:11:52.800484+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:30.731978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1007,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.010451+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:11:53.010451+0000","last_peered":"2024-08-17T17:11:53.010451+0000","last_clean":"2024-08-17T17:11:53.010451+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:11:53.010451+0000","last_undegraded":"2024-08-17T17:11:53.010451+0000","last_fullsized":"2024-08-17T17:11:53.010451+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1251,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.076011+0000","last_change":"2024-08-17T17:07:45.692127+0000","last_active":"2024-08-17T17:11:59.076011+0000","last_peered":"2024-08-17T17:11:59.076011+0000","last_clean":"2024-08-17T17:11:59.076011+0000","last_became_active":"2024-08-17T17:07:45.690277+0000","last_became_peered":"2024-08-17T17:07:45.690277+0000","last_unstale":"2024-08-17T17:11:59.076011+0000","last_undegraded":"2024-08-17T17:11:59.076011+0000","last_fullsized":"2024-08-17T17:11:59.076011+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:04.555409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1239,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075238+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:11:59.075238+0000","last_peered":"2024-08-17T17:11:59.075238+0000","last_clean":"2024-08-17T17:11:59.075238+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:11:59.075238+0000","last_undegraded":"2024-08-17T17:11:59.075238+0000","last_fullsized":"2024-08-17T17:11:59.075238+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1023,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800297+0000","last_change":"2024-08-17T17:06:39.627689+0000","last_active":"2024-08-17T17:11:52.800297+0000","last_peered":"2024-08-17T17:11:52.800297+0000","last_clean":"2024-08-17T17:11:52.800297+0000","last_became_active":"2024-08-17T17:06:39.627375+0000","last_became_peered":"2024-08-17T17:06:39.627375+0000","last_unstale":"2024-08-17T17:11:52.800297+0000","last_undegraded":"2024-08-17T17:11:52.800297+0000","last_fullsized":"2024-08-17T17:11:52.800297+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:51.954076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":1333,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648938+0000","last_change":"2024-08-17T17:07:12.651933+0000","last_active":"2024-08-17T17:11:58.648938+0000","last_peered":"2024-08-17T17:11:58.648938+0000","last_clean":"2024-08-17T17:11:58.648938+0000","last_became_active":"2024-08-17T17:07:12.651592+0000","last_became_peered":"2024-08-17T17:07:12.651592+0000","last_unstale":"2024-08-17T17:11:58.648938+0000","last_undegraded":"2024-08-17T17:11:58.648938+0000","last_fullsized":"2024-08-17T17:11:58.648938+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:31.224038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201535,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":55668,"kb_used_data":10600,"kb_used_omap":1,"kb_used_meta":45054,"kb_avail":93689484,"statfs":{"total":95995035648,"available":95938031616,"internally_reserved":0,"allocated":10854400,"data_stored":7795252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":46135755},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463172,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67944,"kb_used_data":10592,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93677208,"statfs":{"total":95995035648,"available":95925460992,"internally_reserved":0,"allocated":10846208,"data_stored":7795125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":58718665},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626814,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":46296,"kb_used_data":9996,"kb_used_omap":42,"kb_used_meta":36245,"kb_avail":93698856,"statfs":{"total":95995035648,"available":95947628544,"internally_reserved":0,"allocated":10235904,"data_stored":7200014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":37114940},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785291,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":50584,"kb_used_data":9996,"kb_used_omap":48,"kb_used_meta":40527,"kb_avail":93694568,"statfs":{"total":95995035648,"available":95943237632,"internally_reserved":0,"allocated":10235904,"data_stored":7199943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":41500334},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724808,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":107488,"kb_used_data":10004,"kb_used_omap":1,"kb_used_meta":97470,"kb_avail":93637664,"statfs":{"total":95995035648,"available":95884967936,"internally_reserved":0,"allocated":10244096,"data_stored":7200583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":99809734},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986445,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":95212,"kb_used_data":10016,"kb_used_omap":1,"kb_used_meta":85182,"kb_avail":93649940,"statfs":{"total":95995035648,"available":95897538560,"internally_reserved":0,"allocated":10256384,"data_stored":7204814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":87226822},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706052,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":46172,"kb_used_data":10000,"kb_used_omap":35,"kb_used_meta":36124,"kb_avail":93698980,"statfs":{"total":95995035648,"available":95947755520,"internally_reserved":0,"allocated":10240000,"data_stored":7200540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":36991031},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831825,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":43040,"kb_used_data":10580,"kb_used_omap":65,"kb_used_meta":32382,"kb_avail":93702112,"statfs":{"total":95995035648,"available":95950962688,"internally_reserved":0,"allocated":10833920,"data_stored":7790896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":33159355},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:12:46.295 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T17:12:46.295 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-08-17T17:12:46.295 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-08-17T17:12:46.296 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph health --format=json 2024-08-17T17:12:46.514 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:46.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:46 smithi063 ceph-mon[114756]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:12:46.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:46 smithi063 ceph-mon[109843]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:12:46.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:46 smithi162 ceph-mon[104668]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:12:47.275 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:47 smithi063 ceph-mon[109843]: pgmap v1110: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:47.275 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:47.275 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:47 smithi063 ceph-mon[114756]: pgmap v1110: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:47.276 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:47.598 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:47 smithi162 ceph-mon[104668]: pgmap v1110: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:47.598 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:12:48.275 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:12:48.275 INFO:teuthology.orchestra.run.smithi063.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-08-17T17:12:48.914 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-08-17T17:12:48.915 INFO:teuthology.run_tasks:Running task print... 2024-08-17T17:12:48.924 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2024-08-17T17:12:48.924 INFO:teuthology.run_tasks:Running task parallel... 2024-08-17T17:12:48.933 INFO:teuthology.task.parallel:starting parallel... 2024-08-17T17:12:48.933 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-08-17T17:12:48.933 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2024-08-17T17:12:48.936 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-08-17T17:12:48.936 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'cd8bdbeb9fbdab5733aa3aaadc042c965473e453'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '91f88e4d132ac08dc53355fa3d9717a53236a3a9'}} 2024-08-17T17:12:48.936 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-08-17T17:12:48.936 INFO:tasks.thrashosds:Beginning thrashosds... 2024-08-17T17:12:48.937 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-08-17T17:12:48.937 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:12:49.152 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:49.405 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:49 smithi063 ceph-mon[109843]: pgmap v1111: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:49.405 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:49 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2631600175' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-08-17T17:12:49.405 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:49 smithi063 ceph-mon[114756]: pgmap v1111: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:49.405 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:49 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2631600175' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-08-17T17:12:49.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:49 smithi162 ceph-mon[104668]: pgmap v1111: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:49 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2631600175' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-08-17T17:12:50.979 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:12:50.980 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:12:51.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:51 smithi063 ceph-mon[109843]: pgmap v1112: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:51.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:51 smithi063 ceph-mon[114756]: pgmap v1112: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:51.575 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1112,"stamp":"2024-08-17T17:12:50.034682+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":512404,"kb_used_data":81784,"kb_used_omap":199,"kb_used_meta":430328,"kb_avail":749448812,"statfs":{"total":767960285184,"available":767435583488,"internally_reserved":0,"allocated":83746816,"data_stored":59387167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":440656636},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002955"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1245,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074580+0000","last_change":"2024-08-17T17:07:45.684223+0000","last_active":"2024-08-17T17:11:59.074580+0000","last_peered":"2024-08-17T17:11:59.074580+0000","last_clean":"2024-08-17T17:11:59.074580+0000","last_became_active":"2024-08-17T17:07:45.683948+0000","last_became_peered":"2024-08-17T17:07:45.683948+0000","last_unstale":"2024-08-17T17:11:59.074580+0000","last_undegraded":"2024-08-17T17:11:59.074580+0000","last_fullsized":"2024-08-17T17:11:59.074580+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:07.677423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1008,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075622+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:11:59.075622+0000","last_peered":"2024-08-17T17:11:59.075622+0000","last_clean":"2024-08-17T17:11:59.075622+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:11:59.075622+0000","last_undegraded":"2024-08-17T17:11:59.075622+0000","last_fullsized":"2024-08-17T17:11:59.075622+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1019,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011091+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:11:53.011091+0000","last_peered":"2024-08-17T17:11:53.011091+0000","last_clean":"2024-08-17T17:11:53.011091+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:11:53.011091+0000","last_undegraded":"2024-08-17T17:11:53.011091+0000","last_fullsized":"2024-08-17T17:11:53.011091+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1366,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648138+0000","last_change":"2024-08-17T17:07:45.683455+0000","last_active":"2024-08-17T17:11:58.648138+0000","last_peered":"2024-08-17T17:11:58.648138+0000","last_clean":"2024-08-17T17:11:58.648138+0000","last_became_active":"2024-08-17T17:07:45.683169+0000","last_became_peered":"2024-08-17T17:07:45.683169+0000","last_unstale":"2024-08-17T17:11:58.648138+0000","last_undegraded":"2024-08-17T17:11:58.648138+0000","last_fullsized":"2024-08-17T17:11:58.648138+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:18:19.562247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1325,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648396+0000","last_change":"2024-08-17T17:07:45.684222+0000","last_active":"2024-08-17T17:11:58.648396+0000","last_peered":"2024-08-17T17:11:58.648396+0000","last_clean":"2024-08-17T17:11:58.648396+0000","last_became_active":"2024-08-17T17:07:45.683913+0000","last_became_peered":"2024-08-17T17:07:45.683913+0000","last_unstale":"2024-08-17T17:11:58.648396+0000","last_undegraded":"2024-08-17T17:11:58.648396+0000","last_fullsized":"2024-08-17T17:11:58.648396+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:59:33.944853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":991,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075457+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:11:59.075457+0000","last_peered":"2024-08-17T17:11:59.075457+0000","last_clean":"2024-08-17T17:11:59.075457+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:11:59.075457+0000","last_undegraded":"2024-08-17T17:11:59.075457+0000","last_fullsized":"2024-08-17T17:11:59.075457+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1245,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075397+0000","last_change":"2024-08-17T17:07:45.687030+0000","last_active":"2024-08-17T17:11:59.075397+0000","last_peered":"2024-08-17T17:11:59.075397+0000","last_clean":"2024-08-17T17:11:59.075397+0000","last_became_active":"2024-08-17T17:07:45.686610+0000","last_became_peered":"2024-08-17T17:07:45.686610+0000","last_unstale":"2024-08-17T17:11:59.075397+0000","last_undegraded":"2024-08-17T17:11:59.075397+0000","last_fullsized":"2024-08-17T17:11:59.075397+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:02.425845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1030,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702965+0000","last_change":"2024-08-17T17:06:07.632931+0000","last_active":"2024-08-17T17:11:57.702965+0000","last_peered":"2024-08-17T17:11:57.702965+0000","last_clean":"2024-08-17T17:11:57.702965+0000","last_became_active":"2024-08-17T17:06:07.632525+0000","last_became_peered":"2024-08-17T17:06:07.632525+0000","last_unstale":"2024-08-17T17:11:57.702965+0000","last_undegraded":"2024-08-17T17:11:57.702965+0000","last_fullsized":"2024-08-17T17:11:57.702965+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:01.740674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1007,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699675+0000","last_change":"2024-08-17T17:06:07.638468+0000","last_active":"2024-08-17T17:11:52.699675+0000","last_peered":"2024-08-17T17:11:52.699675+0000","last_clean":"2024-08-17T17:11:52.699675+0000","last_became_active":"2024-08-17T17:06:07.636046+0000","last_became_peered":"2024-08-17T17:06:07.636046+0000","last_unstale":"2024-08-17T17:11:52.699675+0000","last_undegraded":"2024-08-17T17:11:52.699675+0000","last_fullsized":"2024-08-17T17:11:52.699675+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:37:12.228090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1005,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.010559+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:11:53.010559+0000","last_peered":"2024-08-17T17:11:53.010559+0000","last_clean":"2024-08-17T17:11:53.010559+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:11:53.010559+0000","last_undegraded":"2024-08-17T17:11:53.010559+0000","last_fullsized":"2024-08-17T17:11:53.010559+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1008,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699604+0000","last_change":"2024-08-17T17:06:39.638433+0000","last_active":"2024-08-17T17:11:52.699604+0000","last_peered":"2024-08-17T17:11:52.699604+0000","last_clean":"2024-08-17T17:11:52.699604+0000","last_became_active":"2024-08-17T17:06:39.638081+0000","last_became_peered":"2024-08-17T17:06:39.638081+0000","last_unstale":"2024-08-17T17:11:52.699604+0000","last_undegraded":"2024-08-17T17:11:52.699604+0000","last_fullsized":"2024-08-17T17:11:52.699604+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:13.558012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800667+0000","last_change":"2024-08-17T17:07:45.693574+0000","last_active":"2024-08-17T17:11:52.800667+0000","last_peered":"2024-08-17T17:11:52.800667+0000","last_clean":"2024-08-17T17:11:52.800667+0000","last_became_active":"2024-08-17T17:07:45.693086+0000","last_became_peered":"2024-08-17T17:07:45.693086+0000","last_unstale":"2024-08-17T17:11:52.800667+0000","last_undegraded":"2024-08-17T17:11:52.800667+0000","last_fullsized":"2024-08-17T17:11:52.800667+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:31.888894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":992,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699566+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:11:52.699566+0000","last_peered":"2024-08-17T17:11:52.699566+0000","last_clean":"2024-08-17T17:11:52.699566+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:11:52.699566+0000","last_undegraded":"2024-08-17T17:11:52.699566+0000","last_fullsized":"2024-08-17T17:11:52.699566+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1021,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699849+0000","last_change":"2024-08-17T17:06:39.629368+0000","last_active":"2024-08-17T17:11:52.699849+0000","last_peered":"2024-08-17T17:11:52.699849+0000","last_clean":"2024-08-17T17:11:52.699849+0000","last_became_active":"2024-08-17T17:06:39.628832+0000","last_became_peered":"2024-08-17T17:06:39.628832+0000","last_unstale":"2024-08-17T17:11:52.699849+0000","last_undegraded":"2024-08-17T17:11:52.699849+0000","last_fullsized":"2024-08-17T17:11:52.699849+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:54.063884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1021,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011147+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:11:53.011147+0000","last_peered":"2024-08-17T17:11:53.011147+0000","last_clean":"2024-08-17T17:11:53.011147+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:11:53.011147+0000","last_undegraded":"2024-08-17T17:11:53.011147+0000","last_fullsized":"2024-08-17T17:11:53.011147+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1338,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.649075+0000","last_change":"2024-08-17T17:07:12.651379+0000","last_active":"2024-08-17T17:11:58.649075+0000","last_peered":"2024-08-17T17:11:58.649075+0000","last_clean":"2024-08-17T17:11:58.649075+0000","last_became_active":"2024-08-17T17:07:12.651069+0000","last_became_peered":"2024-08-17T17:07:12.651069+0000","last_unstale":"2024-08-17T17:11:58.649075+0000","last_undegraded":"2024-08-17T17:11:58.649075+0000","last_fullsized":"2024-08-17T17:11:58.649075+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:25.384305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1258,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075731+0000","last_change":"2024-08-17T17:07:45.691437+0000","last_active":"2024-08-17T17:11:59.075731+0000","last_peered":"2024-08-17T17:11:59.075731+0000","last_clean":"2024-08-17T17:11:59.075731+0000","last_became_active":"2024-08-17T17:07:45.690567+0000","last_became_peered":"2024-08-17T17:07:45.690567+0000","last_unstale":"2024-08-17T17:11:59.075731+0000","last_undegraded":"2024-08-17T17:11:59.075731+0000","last_fullsized":"2024-08-17T17:11:59.075731+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:43:07.698912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1024,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800708+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:11:52.800708+0000","last_peered":"2024-08-17T17:11:52.800708+0000","last_clean":"2024-08-17T17:11:52.800708+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:11:52.800708+0000","last_undegraded":"2024-08-17T17:11:52.800708+0000","last_fullsized":"2024-08-17T17:11:52.800708+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1022,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.700185+0000","last_change":"2024-08-17T17:06:39.629619+0000","last_active":"2024-08-17T17:11:52.700185+0000","last_peered":"2024-08-17T17:11:52.700185+0000","last_clean":"2024-08-17T17:11:52.700185+0000","last_became_active":"2024-08-17T17:06:39.629295+0000","last_became_peered":"2024-08-17T17:06:39.629295+0000","last_unstale":"2024-08-17T17:11:52.700185+0000","last_undegraded":"2024-08-17T17:11:52.700185+0000","last_fullsized":"2024-08-17T17:11:52.700185+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:15.874423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.801064+0000","last_change":"2024-08-17T17:07:12.649178+0000","last_active":"2024-08-17T17:11:52.801064+0000","last_peered":"2024-08-17T17:11:52.801064+0000","last_clean":"2024-08-17T17:11:52.801064+0000","last_became_active":"2024-08-17T17:07:12.648844+0000","last_became_peered":"2024-08-17T17:07:12.648844+0000","last_unstale":"2024-08-17T17:11:52.801064+0000","last_undegraded":"2024-08-17T17:11:52.801064+0000","last_fullsized":"2024-08-17T17:11:52.801064+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:12:06.307736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1025,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800742+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:11:52.800742+0000","last_peered":"2024-08-17T17:11:52.800742+0000","last_clean":"2024-08-17T17:11:52.800742+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:11:52.800742+0000","last_undegraded":"2024-08-17T17:11:52.800742+0000","last_fullsized":"2024-08-17T17:11:52.800742+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1024,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.801216+0000","last_change":"2024-08-17T17:06:39.627678+0000","last_active":"2024-08-17T17:11:52.801216+0000","last_peered":"2024-08-17T17:11:52.801216+0000","last_clean":"2024-08-17T17:11:52.801216+0000","last_became_active":"2024-08-17T17:06:39.627226+0000","last_became_peered":"2024-08-17T17:06:39.627226+0000","last_unstale":"2024-08-17T17:11:52.801216+0000","last_undegraded":"2024-08-17T17:11:52.801216+0000","last_fullsized":"2024-08-17T17:11:52.801216+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:01.868244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1247,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075390+0000","last_change":"2024-08-17T17:07:45.680458+0000","last_active":"2024-08-17T17:11:59.075390+0000","last_peered":"2024-08-17T17:11:59.075390+0000","last_clean":"2024-08-17T17:11:59.075390+0000","last_became_active":"2024-08-17T17:07:45.680221+0000","last_became_peered":"2024-08-17T17:07:45.680221+0000","last_unstale":"2024-08-17T17:11:59.075390+0000","last_undegraded":"2024-08-17T17:11:59.075390+0000","last_fullsized":"2024-08-17T17:11:59.075390+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:55.083962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1269,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075358+0000","last_change":"2024-08-17T17:07:45.682866+0000","last_active":"2024-08-17T17:11:59.075358+0000","last_peered":"2024-08-17T17:11:59.075358+0000","last_clean":"2024-08-17T17:11:59.075358+0000","last_became_active":"2024-08-17T17:07:45.682554+0000","last_became_peered":"2024-08-17T17:07:45.682554+0000","last_unstale":"2024-08-17T17:11:59.075358+0000","last_undegraded":"2024-08-17T17:11:59.075358+0000","last_fullsized":"2024-08-17T17:11:59.075358+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:25.975914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1338,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074716+0000","last_change":"2024-08-17T17:06:07.636484+0000","last_active":"2024-08-17T17:11:59.074716+0000","last_peered":"2024-08-17T17:11:59.074716+0000","last_clean":"2024-08-17T17:11:59.074716+0000","last_became_active":"2024-08-17T17:06:07.635998+0000","last_became_peered":"2024-08-17T17:06:07.635998+0000","last_unstale":"2024-08-17T17:11:59.074716+0000","last_undegraded":"2024-08-17T17:11:59.074716+0000","last_fullsized":"2024-08-17T17:11:59.074716+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:16.096587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1035,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011021+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:11:53.011021+0000","last_peered":"2024-08-17T17:11:53.011021+0000","last_clean":"2024-08-17T17:11:53.011021+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:11:53.011021+0000","last_undegraded":"2024-08-17T17:11:53.011021+0000","last_fullsized":"2024-08-17T17:11:53.011021+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1226,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074625+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:11:59.074625+0000","last_peered":"2024-08-17T17:11:59.074625+0000","last_clean":"2024-08-17T17:11:59.074625+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:11:59.074625+0000","last_undegraded":"2024-08-17T17:11:59.074625+0000","last_fullsized":"2024-08-17T17:11:59.074625+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1007,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703217+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:11:57.703217+0000","last_peered":"2024-08-17T17:11:57.703217+0000","last_clean":"2024-08-17T17:11:57.703217+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:11:57.703217+0000","last_undegraded":"2024-08-17T17:11:57.703217+0000","last_fullsized":"2024-08-17T17:11:57.703217+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1035,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074623+0000","last_change":"2024-08-17T17:06:39.638376+0000","last_active":"2024-08-17T17:11:59.074623+0000","last_peered":"2024-08-17T17:11:59.074623+0000","last_clean":"2024-08-17T17:11:59.074623+0000","last_became_active":"2024-08-17T17:06:39.638050+0000","last_became_peered":"2024-08-17T17:06:39.638050+0000","last_unstale":"2024-08-17T17:11:59.074623+0000","last_undegraded":"2024-08-17T17:11:59.074623+0000","last_fullsized":"2024-08-17T17:11:59.074623+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:05.782492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1367,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648019+0000","last_change":"2024-08-17T17:07:45.683556+0000","last_active":"2024-08-17T17:11:58.648019+0000","last_peered":"2024-08-17T17:11:58.648019+0000","last_clean":"2024-08-17T17:11:58.648019+0000","last_became_active":"2024-08-17T17:07:45.683232+0000","last_became_peered":"2024-08-17T17:07:45.683232+0000","last_unstale":"2024-08-17T17:11:58.648019+0000","last_undegraded":"2024-08-17T17:11:58.648019+0000","last_fullsized":"2024-08-17T17:11:58.648019+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:16.112417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1339,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648339+0000","last_change":"2024-08-17T17:07:12.651145+0000","last_active":"2024-08-17T17:11:58.648339+0000","last_peered":"2024-08-17T17:11:58.648339+0000","last_clean":"2024-08-17T17:11:58.648339+0000","last_became_active":"2024-08-17T17:07:12.650839+0000","last_became_peered":"2024-08-17T17:07:12.650839+0000","last_unstale":"2024-08-17T17:11:58.648339+0000","last_undegraded":"2024-08-17T17:11:58.648339+0000","last_fullsized":"2024-08-17T17:11:58.648339+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:59.795181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1309,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074165+0000","last_change":"2024-08-17T17:07:45.681107+0000","last_active":"2024-08-17T17:11:59.074165+0000","last_peered":"2024-08-17T17:11:59.074165+0000","last_clean":"2024-08-17T17:11:59.074165+0000","last_became_active":"2024-08-17T17:07:45.680770+0000","last_became_peered":"2024-08-17T17:07:45.680770+0000","last_unstale":"2024-08-17T17:11:59.074165+0000","last_undegraded":"2024-08-17T17:11:59.074165+0000","last_fullsized":"2024-08-17T17:11:59.074165+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:53.820920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1300,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.704078+0000","last_change":"2024-08-17T17:07:12.649485+0000","last_active":"2024-08-17T17:11:57.704078+0000","last_peered":"2024-08-17T17:11:57.704078+0000","last_clean":"2024-08-17T17:11:57.704078+0000","last_became_active":"2024-08-17T17:07:12.649161+0000","last_became_peered":"2024-08-17T17:07:12.649161+0000","last_unstale":"2024-08-17T17:11:57.704078+0000","last_undegraded":"2024-08-17T17:11:57.704078+0000","last_fullsized":"2024-08-17T17:11:57.704078+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:18.370628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1314,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703394+0000","last_change":"2024-08-17T17:06:39.627820+0000","last_active":"2024-08-17T17:11:57.703394+0000","last_peered":"2024-08-17T17:11:57.703394+0000","last_clean":"2024-08-17T17:11:57.703394+0000","last_became_active":"2024-08-17T17:06:39.627464+0000","last_became_peered":"2024-08-17T17:06:39.627464+0000","last_unstale":"2024-08-17T17:11:57.703394+0000","last_undegraded":"2024-08-17T17:11:57.703394+0000","last_fullsized":"2024-08-17T17:11:57.703394+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:01.500528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1036,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.010866+0000","last_change":"2024-08-17T17:06:39.628745+0000","last_active":"2024-08-17T17:11:53.010866+0000","last_peered":"2024-08-17T17:11:53.010866+0000","last_clean":"2024-08-17T17:11:53.010866+0000","last_became_active":"2024-08-17T17:06:39.628144+0000","last_became_peered":"2024-08-17T17:06:39.628144+0000","last_unstale":"2024-08-17T17:11:53.010866+0000","last_undegraded":"2024-08-17T17:11:53.010866+0000","last_fullsized":"2024-08-17T17:11:53.010866+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:14.740936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1026,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075115+0000","last_change":"2024-08-17T17:06:07.637581+0000","last_active":"2024-08-17T17:11:59.075115+0000","last_peered":"2024-08-17T17:11:59.075115+0000","last_clean":"2024-08-17T17:11:59.075115+0000","last_became_active":"2024-08-17T17:06:07.637282+0000","last_became_peered":"2024-08-17T17:06:07.637282+0000","last_unstale":"2024-08-17T17:11:59.075115+0000","last_undegraded":"2024-08-17T17:11:59.075115+0000","last_fullsized":"2024-08-17T17:11:59.075115+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.938734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1310,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702867+0000","last_change":"2024-08-17T17:07:45.687502+0000","last_active":"2024-08-17T17:11:57.702867+0000","last_peered":"2024-08-17T17:11:57.702867+0000","last_clean":"2024-08-17T17:11:57.702867+0000","last_became_active":"2024-08-17T17:07:45.687197+0000","last_became_peered":"2024-08-17T17:07:45.687197+0000","last_unstale":"2024-08-17T17:11:57.702867+0000","last_undegraded":"2024-08-17T17:11:57.702867+0000","last_fullsized":"2024-08-17T17:11:57.702867+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:23.577838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1037,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.981002+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:11:58.981002+0000","last_peered":"2024-08-17T17:11:58.981002+0000","last_clean":"2024-08-17T17:11:58.981002+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:11:58.981002+0000","last_undegraded":"2024-08-17T17:11:58.981002+0000","last_fullsized":"2024-08-17T17:11:58.981002+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1328,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703305+0000","last_change":"2024-08-17T17:06:39.633975+0000","last_active":"2024-08-17T17:11:57.703305+0000","last_peered":"2024-08-17T17:11:57.703305+0000","last_clean":"2024-08-17T17:11:57.703305+0000","last_became_active":"2024-08-17T17:06:39.631526+0000","last_became_peered":"2024-08-17T17:06:39.631526+0000","last_unstale":"2024-08-17T17:11:57.703305+0000","last_undegraded":"2024-08-17T17:11:57.703305+0000","last_fullsized":"2024-08-17T17:11:57.703305+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:58.542237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1288,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703023+0000","last_change":"2024-08-17T17:06:39.627577+0000","last_active":"2024-08-17T17:11:57.703023+0000","last_peered":"2024-08-17T17:11:57.703023+0000","last_clean":"2024-08-17T17:11:57.703023+0000","last_became_active":"2024-08-17T17:06:39.626554+0000","last_became_peered":"2024-08-17T17:06:39.626554+0000","last_unstale":"2024-08-17T17:11:57.703023+0000","last_undegraded":"2024-08-17T17:11:57.703023+0000","last_fullsized":"2024-08-17T17:11:57.703023+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:04:48.533057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1298,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702987+0000","last_change":"2024-08-17T17:07:12.645660+0000","last_active":"2024-08-17T17:11:57.702987+0000","last_peered":"2024-08-17T17:11:57.702987+0000","last_clean":"2024-08-17T17:11:57.702987+0000","last_became_active":"2024-08-17T17:07:12.645368+0000","last_became_peered":"2024-08-17T17:07:12.645368+0000","last_unstale":"2024-08-17T17:11:57.702987+0000","last_undegraded":"2024-08-17T17:11:57.702987+0000","last_fullsized":"2024-08-17T17:11:57.702987+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:41:00.157045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1019,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699659+0000","last_change":"2024-08-17T17:06:07.636239+0000","last_active":"2024-08-17T17:11:52.699659+0000","last_peered":"2024-08-17T17:11:52.699659+0000","last_clean":"2024-08-17T17:11:52.699659+0000","last_became_active":"2024-08-17T17:06:07.635881+0000","last_became_peered":"2024-08-17T17:06:07.635881+0000","last_unstale":"2024-08-17T17:11:52.699659+0000","last_undegraded":"2024-08-17T17:11:52.699659+0000","last_fullsized":"2024-08-17T17:11:52.699659+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:36.001648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1316,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648899+0000","last_change":"2024-08-17T17:06:07.634369+0000","last_active":"2024-08-17T17:11:58.648899+0000","last_peered":"2024-08-17T17:11:58.648899+0000","last_clean":"2024-08-17T17:11:58.648899+0000","last_became_active":"2024-08-17T17:06:07.633769+0000","last_became_peered":"2024-08-17T17:06:07.633769+0000","last_unstale":"2024-08-17T17:11:58.648899+0000","last_undegraded":"2024-08-17T17:11:58.648899+0000","last_fullsized":"2024-08-17T17:11:58.648899+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:20.957493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1258,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075605+0000","last_change":"2024-08-17T17:07:45.688526+0000","last_active":"2024-08-17T17:11:59.075605+0000","last_peered":"2024-08-17T17:11:59.075605+0000","last_clean":"2024-08-17T17:11:59.075605+0000","last_became_active":"2024-08-17T17:07:45.688091+0000","last_became_peered":"2024-08-17T17:07:45.688091+0000","last_unstale":"2024-08-17T17:11:59.075605+0000","last_undegraded":"2024-08-17T17:11:59.075605+0000","last_fullsized":"2024-08-17T17:11:59.075605+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:02.385683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1298,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703411+0000","last_change":"2024-08-17T17:06:39.629005+0000","last_active":"2024-08-17T17:11:57.703411+0000","last_peered":"2024-08-17T17:11:57.703411+0000","last_clean":"2024-08-17T17:11:57.703411+0000","last_became_active":"2024-08-17T17:06:39.628679+0000","last_became_peered":"2024-08-17T17:06:39.628679+0000","last_unstale":"2024-08-17T17:11:57.703411+0000","last_undegraded":"2024-08-17T17:11:57.703411+0000","last_fullsized":"2024-08-17T17:11:57.703411+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:09.401059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1255,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075180+0000","last_change":"2024-08-17T17:07:45.695858+0000","last_active":"2024-08-17T17:11:59.075180+0000","last_peered":"2024-08-17T17:11:59.075180+0000","last_clean":"2024-08-17T17:11:59.075180+0000","last_became_active":"2024-08-17T17:07:45.694111+0000","last_became_peered":"2024-08-17T17:07:45.694111+0000","last_unstale":"2024-08-17T17:11:59.075180+0000","last_undegraded":"2024-08-17T17:11:59.075180+0000","last_fullsized":"2024-08-17T17:11:59.075180+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:40:38.772090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":1330,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648523+0000","last_change":"2024-08-17T17:06:39.636866+0000","last_active":"2024-08-17T17:11:58.648523+0000","last_peered":"2024-08-17T17:11:58.648523+0000","last_clean":"2024-08-17T17:11:58.648523+0000","last_became_active":"2024-08-17T17:06:39.636470+0000","last_became_peered":"2024-08-17T17:06:39.636470+0000","last_unstale":"2024-08-17T17:11:58.648523+0000","last_undegraded":"2024-08-17T17:11:58.648523+0000","last_fullsized":"2024-08-17T17:11:58.648523+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:14.028712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1034,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:56.929467+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:11:56.929467+0000","last_peered":"2024-08-17T17:11:56.929467+0000","last_clean":"2024-08-17T17:11:56.929467+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:11:56.929467+0000","last_undegraded":"2024-08-17T17:11:56.929467+0000","last_fullsized":"2024-08-17T17:11:56.929467+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1010,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075411+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:11:59.075411+0000","last_peered":"2024-08-17T17:11:59.075411+0000","last_clean":"2024-08-17T17:11:59.075411+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:11:59.075411+0000","last_undegraded":"2024-08-17T17:11:59.075411+0000","last_fullsized":"2024-08-17T17:11:59.075411+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1024,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074961+0000","last_change":"2024-08-17T17:06:39.627594+0000","last_active":"2024-08-17T17:11:59.074961+0000","last_peered":"2024-08-17T17:11:59.074961+0000","last_clean":"2024-08-17T17:11:59.074961+0000","last_became_active":"2024-08-17T17:06:39.627105+0000","last_became_peered":"2024-08-17T17:06:39.627105+0000","last_unstale":"2024-08-17T17:11:59.074961+0000","last_undegraded":"2024-08-17T17:11:59.074961+0000","last_fullsized":"2024-08-17T17:11:59.074961+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:30.105070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1033,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075812+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:11:59.075812+0000","last_peered":"2024-08-17T17:11:59.075812+0000","last_clean":"2024-08-17T17:11:59.075812+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:11:59.075812+0000","last_undegraded":"2024-08-17T17:11:59.075812+0000","last_fullsized":"2024-08-17T17:11:59.075812+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1299,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703585+0000","last_change":"2024-08-17T17:07:12.645440+0000","last_active":"2024-08-17T17:11:57.703585+0000","last_peered":"2024-08-17T17:11:57.703585+0000","last_clean":"2024-08-17T17:11:57.703585+0000","last_became_active":"2024-08-17T17:07:12.645147+0000","last_became_peered":"2024-08-17T17:07:12.645147+0000","last_unstale":"2024-08-17T17:11:57.703585+0000","last_undegraded":"2024-08-17T17:11:57.703585+0000","last_fullsized":"2024-08-17T17:11:57.703585+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:25.588528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1008,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699386+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:11:52.699386+0000","last_peered":"2024-08-17T17:11:52.699386+0000","last_clean":"2024-08-17T17:11:52.699386+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:11:52.699386+0000","last_undegraded":"2024-08-17T17:11:52.699386+0000","last_fullsized":"2024-08-17T17:11:52.699386+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1020,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011326+0000","last_change":"2024-08-17T17:06:39.625029+0000","last_active":"2024-08-17T17:11:53.011326+0000","last_peered":"2024-08-17T17:11:53.011326+0000","last_clean":"2024-08-17T17:11:53.011326+0000","last_became_active":"2024-08-17T17:06:39.624688+0000","last_became_peered":"2024-08-17T17:06:39.624688+0000","last_unstale":"2024-08-17T17:11:53.011326+0000","last_undegraded":"2024-08-17T17:11:53.011326+0000","last_fullsized":"2024-08-17T17:11:53.011326+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:54.849668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1226,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075761+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:11:59.075761+0000","last_peered":"2024-08-17T17:11:59.075761+0000","last_clean":"2024-08-17T17:11:59.075761+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:11:59.075761+0000","last_undegraded":"2024-08-17T17:11:59.075761+0000","last_fullsized":"2024-08-17T17:11:59.075761+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1366,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648869+0000","last_change":"2024-08-17T17:07:45.690099+0000","last_active":"2024-08-17T17:11:58.648869+0000","last_peered":"2024-08-17T17:11:58.648869+0000","last_clean":"2024-08-17T17:11:58.648869+0000","last_became_active":"2024-08-17T17:07:45.689783+0000","last_became_peered":"2024-08-17T17:07:45.689783+0000","last_unstale":"2024-08-17T17:11:58.648869+0000","last_undegraded":"2024-08-17T17:11:58.648869+0000","last_fullsized":"2024-08-17T17:11:58.648869+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:38.311630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1245,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075127+0000","last_change":"2024-08-17T17:07:45.678950+0000","last_active":"2024-08-17T17:11:59.075127+0000","last_peered":"2024-08-17T17:11:59.075127+0000","last_clean":"2024-08-17T17:11:59.075127+0000","last_became_active":"2024-08-17T17:07:45.678472+0000","last_became_peered":"2024-08-17T17:07:45.678472+0000","last_unstale":"2024-08-17T17:11:59.075127+0000","last_undegraded":"2024-08-17T17:11:59.075127+0000","last_fullsized":"2024-08-17T17:11:59.075127+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:48:02.385927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1022,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703548+0000","last_change":"2024-08-17T17:06:39.628542+0000","last_active":"2024-08-17T17:11:57.703548+0000","last_peered":"2024-08-17T17:11:57.703548+0000","last_clean":"2024-08-17T17:11:57.703548+0000","last_became_active":"2024-08-17T17:06:39.628253+0000","last_became_peered":"2024-08-17T17:06:39.628253+0000","last_unstale":"2024-08-17T17:11:57.703548+0000","last_undegraded":"2024-08-17T17:11:57.703548+0000","last_fullsized":"2024-08-17T17:11:57.703548+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:22.349326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1261,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075295+0000","last_change":"2024-08-17T17:07:45.696068+0000","last_active":"2024-08-17T17:11:59.075295+0000","last_peered":"2024-08-17T17:11:59.075295+0000","last_clean":"2024-08-17T17:11:59.075295+0000","last_became_active":"2024-08-17T17:07:45.694617+0000","last_became_peered":"2024-08-17T17:07:45.694617+0000","last_unstale":"2024-08-17T17:11:59.075295+0000","last_undegraded":"2024-08-17T17:11:59.075295+0000","last_fullsized":"2024-08-17T17:11:59.075295+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:08.646380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1327,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648379+0000","last_change":"2024-08-17T17:07:45.684186+0000","last_active":"2024-08-17T17:11:58.648379+0000","last_peered":"2024-08-17T17:11:58.648379+0000","last_clean":"2024-08-17T17:11:58.648379+0000","last_became_active":"2024-08-17T17:07:45.683863+0000","last_became_peered":"2024-08-17T17:07:45.683863+0000","last_unstale":"2024-08-17T17:11:58.648379+0000","last_undegraded":"2024-08-17T17:11:58.648379+0000","last_fullsized":"2024-08-17T17:11:58.648379+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:52.106168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1298,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702758+0000","last_change":"2024-08-17T17:07:45.688645+0000","last_active":"2024-08-17T17:11:57.702758+0000","last_peered":"2024-08-17T17:11:57.702758+0000","last_clean":"2024-08-17T17:11:57.702758+0000","last_became_active":"2024-08-17T17:07:45.688383+0000","last_became_peered":"2024-08-17T17:07:45.688383+0000","last_unstale":"2024-08-17T17:11:57.702758+0000","last_undegraded":"2024-08-17T17:11:57.702758+0000","last_fullsized":"2024-08-17T17:11:57.702758+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:33:12.209565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1245,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074861+0000","last_change":"2024-08-17T17:07:45.689782+0000","last_active":"2024-08-17T17:11:59.074861+0000","last_peered":"2024-08-17T17:11:59.074861+0000","last_clean":"2024-08-17T17:11:59.074861+0000","last_became_active":"2024-08-17T17:07:45.689312+0000","last_became_peered":"2024-08-17T17:07:45.689312+0000","last_unstale":"2024-08-17T17:11:59.074861+0000","last_undegraded":"2024-08-17T17:11:59.074861+0000","last_fullsized":"2024-08-17T17:11:59.074861+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:07.560555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1040,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800592+0000","last_change":"2024-08-17T17:07:12.649175+0000","last_active":"2024-08-17T17:11:52.800592+0000","last_peered":"2024-08-17T17:11:52.800592+0000","last_clean":"2024-08-17T17:11:52.800592+0000","last_became_active":"2024-08-17T17:07:12.648822+0000","last_became_peered":"2024-08-17T17:07:12.648822+0000","last_unstale":"2024-08-17T17:11:52.800592+0000","last_undegraded":"2024-08-17T17:11:52.800592+0000","last_fullsized":"2024-08-17T17:11:52.800592+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:03.924431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1286,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074322+0000","last_change":"2024-08-17T17:07:45.685278+0000","last_active":"2024-08-17T17:11:59.074322+0000","last_peered":"2024-08-17T17:11:59.074322+0000","last_clean":"2024-08-17T17:11:59.074322+0000","last_became_active":"2024-08-17T17:07:45.684963+0000","last_became_peered":"2024-08-17T17:07:45.684963+0000","last_unstale":"2024-08-17T17:11:59.074322+0000","last_undegraded":"2024-08-17T17:11:59.074322+0000","last_fullsized":"2024-08-17T17:11:59.074322+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:55:42.495791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.649176+0000","last_change":"2024-08-17T17:06:07.634103+0000","last_active":"2024-08-17T17:11:58.649176+0000","last_peered":"2024-08-17T17:11:58.649176+0000","last_clean":"2024-08-17T17:11:58.649176+0000","last_became_active":"2024-08-17T17:06:07.633624+0000","last_became_peered":"2024-08-17T17:06:07.633624+0000","last_unstale":"2024-08-17T17:11:58.649176+0000","last_undegraded":"2024-08-17T17:11:58.649176+0000","last_fullsized":"2024-08-17T17:11:58.649176+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:58.573558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800611+0000","last_change":"2024-08-17T17:07:45.684606+0000","last_active":"2024-08-17T17:11:52.800611+0000","last_peered":"2024-08-17T17:11:52.800611+0000","last_clean":"2024-08-17T17:11:52.800611+0000","last_became_active":"2024-08-17T17:07:45.684124+0000","last_became_peered":"2024-08-17T17:07:45.684124+0000","last_unstale":"2024-08-17T17:11:52.800611+0000","last_undegraded":"2024-08-17T17:11:52.800611+0000","last_fullsized":"2024-08-17T17:11:52.800611+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:16.110507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1022,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011023+0000","last_change":"2024-08-17T17:06:39.625245+0000","last_active":"2024-08-17T17:11:53.011023+0000","last_peered":"2024-08-17T17:11:53.011023+0000","last_clean":"2024-08-17T17:11:53.011023+0000","last_became_active":"2024-08-17T17:06:39.624657+0000","last_became_peered":"2024-08-17T17:06:39.624657+0000","last_unstale":"2024-08-17T17:11:53.011023+0000","last_undegraded":"2024-08-17T17:11:53.011023+0000","last_fullsized":"2024-08-17T17:11:53.011023+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:43.097460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800484+0000","last_change":"2024-08-17T17:07:45.684769+0000","last_active":"2024-08-17T17:11:52.800484+0000","last_peered":"2024-08-17T17:11:52.800484+0000","last_clean":"2024-08-17T17:11:52.800484+0000","last_became_active":"2024-08-17T17:07:45.684459+0000","last_became_peered":"2024-08-17T17:07:45.684459+0000","last_unstale":"2024-08-17T17:11:52.800484+0000","last_undegraded":"2024-08-17T17:11:52.800484+0000","last_fullsized":"2024-08-17T17:11:52.800484+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:30.731978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1007,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.010451+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:11:53.010451+0000","last_peered":"2024-08-17T17:11:53.010451+0000","last_clean":"2024-08-17T17:11:53.010451+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:11:53.010451+0000","last_undegraded":"2024-08-17T17:11:53.010451+0000","last_fullsized":"2024-08-17T17:11:53.010451+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1251,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.076011+0000","last_change":"2024-08-17T17:07:45.692127+0000","last_active":"2024-08-17T17:11:59.076011+0000","last_peered":"2024-08-17T17:11:59.076011+0000","last_clean":"2024-08-17T17:11:59.076011+0000","last_became_active":"2024-08-17T17:07:45.690277+0000","last_became_peered":"2024-08-17T17:07:45.690277+0000","last_unstale":"2024-08-17T17:11:59.076011+0000","last_undegraded":"2024-08-17T17:11:59.076011+0000","last_fullsized":"2024-08-17T17:11:59.076011+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:04.555409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1239,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075238+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:11:59.075238+0000","last_peered":"2024-08-17T17:11:59.075238+0000","last_clean":"2024-08-17T17:11:59.075238+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:11:59.075238+0000","last_undegraded":"2024-08-17T17:11:59.075238+0000","last_fullsized":"2024-08-17T17:11:59.075238+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1023,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800297+0000","last_change":"2024-08-17T17:06:39.627689+0000","last_active":"2024-08-17T17:11:52.800297+0000","last_peered":"2024-08-17T17:11:52.800297+0000","last_clean":"2024-08-17T17:11:52.800297+0000","last_became_active":"2024-08-17T17:06:39.627375+0000","last_became_peered":"2024-08-17T17:06:39.627375+0000","last_unstale":"2024-08-17T17:11:52.800297+0000","last_undegraded":"2024-08-17T17:11:52.800297+0000","last_fullsized":"2024-08-17T17:11:52.800297+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:51.954076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":1333,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648938+0000","last_change":"2024-08-17T17:07:12.651933+0000","last_active":"2024-08-17T17:11:58.648938+0000","last_peered":"2024-08-17T17:11:58.648938+0000","last_clean":"2024-08-17T17:11:58.648938+0000","last_became_active":"2024-08-17T17:07:12.651592+0000","last_became_peered":"2024-08-17T17:07:12.651592+0000","last_unstale":"2024-08-17T17:11:58.648938+0000","last_undegraded":"2024-08-17T17:11:58.648938+0000","last_fullsized":"2024-08-17T17:11:58.648938+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:31.224038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201536,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":55668,"kb_used_data":10600,"kb_used_omap":1,"kb_used_meta":45054,"kb_avail":93689484,"statfs":{"total":95995035648,"available":95938031616,"internally_reserved":0,"allocated":10854400,"data_stored":7795252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":46135755},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463173,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67944,"kb_used_data":10592,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93677208,"statfs":{"total":95995035648,"available":95925460992,"internally_reserved":0,"allocated":10846208,"data_stored":7795125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":58718665},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626815,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":46296,"kb_used_data":9996,"kb_used_omap":42,"kb_used_meta":36245,"kb_avail":93698856,"statfs":{"total":95995035648,"available":95947628544,"internally_reserved":0,"allocated":10235904,"data_stored":7200014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":37114940},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785292,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":50584,"kb_used_data":9996,"kb_used_omap":48,"kb_used_meta":40527,"kb_avail":93694568,"statfs":{"total":95995035648,"available":95943237632,"internally_reserved":0,"allocated":10235904,"data_stored":7199943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":41500334},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724809,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":107488,"kb_used_data":10004,"kb_used_omap":1,"kb_used_meta":97470,"kb_avail":93637664,"statfs":{"total":95995035648,"available":95884967936,"internally_reserved":0,"allocated":10244096,"data_stored":7200583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":99809734},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986446,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":95212,"kb_used_data":10016,"kb_used_omap":1,"kb_used_meta":85182,"kb_avail":93649940,"statfs":{"total":95995035648,"available":95897538560,"internally_reserved":0,"allocated":10256384,"data_stored":7204814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":87226822},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706053,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":46172,"kb_used_data":10000,"kb_used_omap":35,"kb_used_meta":36124,"kb_avail":93698980,"statfs":{"total":95995035648,"available":95947755520,"internally_reserved":0,"allocated":10240000,"data_stored":7200540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":36991031},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831826,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":43040,"kb_used_data":10580,"kb_used_omap":65,"kb_used_meta":32382,"kb_avail":93702112,"statfs":{"total":95995035648,"available":95950962688,"internally_reserved":0,"allocated":10833920,"data_stored":7790896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":33159355},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:12:51.579 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:12:51.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:51 smithi162 ceph-mon[104668]: pgmap v1112: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:51.789 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:52.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:52 smithi063 ceph-mon[114756]: from='client.44528 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:12:52.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:52 smithi063 ceph-mon[109843]: from='client.44528 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:12:52.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:52 smithi162 ceph-mon[104668]: from='client.44528 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:12:53.215 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:53 smithi063 ceph-mon[109843]: pgmap v1113: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:53.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:53 smithi063 ceph-mon[114756]: pgmap v1113: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:53.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:12:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:12:53] "GET /metrics HTTP/1.1" 200 33974 "" "Prometheus/2.51.0" 2024-08-17T17:12:53.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:53 smithi162 ceph-mon[104668]: pgmap v1113: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:53.616 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:12:53.617 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:12:54.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:54 smithi063 ceph-mon[114756]: from='client.44534 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:12:54.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:54 smithi063 ceph-mon[109843]: from='client.44534 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:12:54.357 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1113,"stamp":"2024-08-17T17:12:52.035144+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":512404,"kb_used_data":81784,"kb_used_omap":199,"kb_used_meta":430328,"kb_avail":749448812,"statfs":{"total":767960285184,"available":767435583488,"internally_reserved":0,"allocated":83746816,"data_stored":59387167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":440656636},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002897"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1245,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074580+0000","last_change":"2024-08-17T17:07:45.684223+0000","last_active":"2024-08-17T17:11:59.074580+0000","last_peered":"2024-08-17T17:11:59.074580+0000","last_clean":"2024-08-17T17:11:59.074580+0000","last_became_active":"2024-08-17T17:07:45.683948+0000","last_became_peered":"2024-08-17T17:07:45.683948+0000","last_unstale":"2024-08-17T17:11:59.074580+0000","last_undegraded":"2024-08-17T17:11:59.074580+0000","last_fullsized":"2024-08-17T17:11:59.074580+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:07.677423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1008,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075622+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:11:59.075622+0000","last_peered":"2024-08-17T17:11:59.075622+0000","last_clean":"2024-08-17T17:11:59.075622+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:11:59.075622+0000","last_undegraded":"2024-08-17T17:11:59.075622+0000","last_fullsized":"2024-08-17T17:11:59.075622+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1019,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011091+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:11:53.011091+0000","last_peered":"2024-08-17T17:11:53.011091+0000","last_clean":"2024-08-17T17:11:53.011091+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:11:53.011091+0000","last_undegraded":"2024-08-17T17:11:53.011091+0000","last_fullsized":"2024-08-17T17:11:53.011091+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1366,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648138+0000","last_change":"2024-08-17T17:07:45.683455+0000","last_active":"2024-08-17T17:11:58.648138+0000","last_peered":"2024-08-17T17:11:58.648138+0000","last_clean":"2024-08-17T17:11:58.648138+0000","last_became_active":"2024-08-17T17:07:45.683169+0000","last_became_peered":"2024-08-17T17:07:45.683169+0000","last_unstale":"2024-08-17T17:11:58.648138+0000","last_undegraded":"2024-08-17T17:11:58.648138+0000","last_fullsized":"2024-08-17T17:11:58.648138+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:18:19.562247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1325,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648396+0000","last_change":"2024-08-17T17:07:45.684222+0000","last_active":"2024-08-17T17:11:58.648396+0000","last_peered":"2024-08-17T17:11:58.648396+0000","last_clean":"2024-08-17T17:11:58.648396+0000","last_became_active":"2024-08-17T17:07:45.683913+0000","last_became_peered":"2024-08-17T17:07:45.683913+0000","last_unstale":"2024-08-17T17:11:58.648396+0000","last_undegraded":"2024-08-17T17:11:58.648396+0000","last_fullsized":"2024-08-17T17:11:58.648396+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:59:33.944853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":991,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075457+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:11:59.075457+0000","last_peered":"2024-08-17T17:11:59.075457+0000","last_clean":"2024-08-17T17:11:59.075457+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:11:59.075457+0000","last_undegraded":"2024-08-17T17:11:59.075457+0000","last_fullsized":"2024-08-17T17:11:59.075457+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1245,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075397+0000","last_change":"2024-08-17T17:07:45.687030+0000","last_active":"2024-08-17T17:11:59.075397+0000","last_peered":"2024-08-17T17:11:59.075397+0000","last_clean":"2024-08-17T17:11:59.075397+0000","last_became_active":"2024-08-17T17:07:45.686610+0000","last_became_peered":"2024-08-17T17:07:45.686610+0000","last_unstale":"2024-08-17T17:11:59.075397+0000","last_undegraded":"2024-08-17T17:11:59.075397+0000","last_fullsized":"2024-08-17T17:11:59.075397+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:02.425845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1030,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702965+0000","last_change":"2024-08-17T17:06:07.632931+0000","last_active":"2024-08-17T17:11:57.702965+0000","last_peered":"2024-08-17T17:11:57.702965+0000","last_clean":"2024-08-17T17:11:57.702965+0000","last_became_active":"2024-08-17T17:06:07.632525+0000","last_became_peered":"2024-08-17T17:06:07.632525+0000","last_unstale":"2024-08-17T17:11:57.702965+0000","last_undegraded":"2024-08-17T17:11:57.702965+0000","last_fullsized":"2024-08-17T17:11:57.702965+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:01.740674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1007,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699675+0000","last_change":"2024-08-17T17:06:07.638468+0000","last_active":"2024-08-17T17:11:52.699675+0000","last_peered":"2024-08-17T17:11:52.699675+0000","last_clean":"2024-08-17T17:11:52.699675+0000","last_became_active":"2024-08-17T17:06:07.636046+0000","last_became_peered":"2024-08-17T17:06:07.636046+0000","last_unstale":"2024-08-17T17:11:52.699675+0000","last_undegraded":"2024-08-17T17:11:52.699675+0000","last_fullsized":"2024-08-17T17:11:52.699675+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:37:12.228090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1005,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.010559+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:11:53.010559+0000","last_peered":"2024-08-17T17:11:53.010559+0000","last_clean":"2024-08-17T17:11:53.010559+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:11:53.010559+0000","last_undegraded":"2024-08-17T17:11:53.010559+0000","last_fullsized":"2024-08-17T17:11:53.010559+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1008,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699604+0000","last_change":"2024-08-17T17:06:39.638433+0000","last_active":"2024-08-17T17:11:52.699604+0000","last_peered":"2024-08-17T17:11:52.699604+0000","last_clean":"2024-08-17T17:11:52.699604+0000","last_became_active":"2024-08-17T17:06:39.638081+0000","last_became_peered":"2024-08-17T17:06:39.638081+0000","last_unstale":"2024-08-17T17:11:52.699604+0000","last_undegraded":"2024-08-17T17:11:52.699604+0000","last_fullsized":"2024-08-17T17:11:52.699604+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:13.558012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800667+0000","last_change":"2024-08-17T17:07:45.693574+0000","last_active":"2024-08-17T17:11:52.800667+0000","last_peered":"2024-08-17T17:11:52.800667+0000","last_clean":"2024-08-17T17:11:52.800667+0000","last_became_active":"2024-08-17T17:07:45.693086+0000","last_became_peered":"2024-08-17T17:07:45.693086+0000","last_unstale":"2024-08-17T17:11:52.800667+0000","last_undegraded":"2024-08-17T17:11:52.800667+0000","last_fullsized":"2024-08-17T17:11:52.800667+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:31.888894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":992,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699566+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:11:52.699566+0000","last_peered":"2024-08-17T17:11:52.699566+0000","last_clean":"2024-08-17T17:11:52.699566+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:11:52.699566+0000","last_undegraded":"2024-08-17T17:11:52.699566+0000","last_fullsized":"2024-08-17T17:11:52.699566+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1021,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699849+0000","last_change":"2024-08-17T17:06:39.629368+0000","last_active":"2024-08-17T17:11:52.699849+0000","last_peered":"2024-08-17T17:11:52.699849+0000","last_clean":"2024-08-17T17:11:52.699849+0000","last_became_active":"2024-08-17T17:06:39.628832+0000","last_became_peered":"2024-08-17T17:06:39.628832+0000","last_unstale":"2024-08-17T17:11:52.699849+0000","last_undegraded":"2024-08-17T17:11:52.699849+0000","last_fullsized":"2024-08-17T17:11:52.699849+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:54.063884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1021,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011147+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:11:53.011147+0000","last_peered":"2024-08-17T17:11:53.011147+0000","last_clean":"2024-08-17T17:11:53.011147+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:11:53.011147+0000","last_undegraded":"2024-08-17T17:11:53.011147+0000","last_fullsized":"2024-08-17T17:11:53.011147+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1339,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.649118+0000","last_change":"2024-08-17T17:07:12.651379+0000","last_active":"2024-08-17T17:11:58.649118+0000","last_peered":"2024-08-17T17:11:58.649118+0000","last_clean":"2024-08-17T17:11:58.649118+0000","last_became_active":"2024-08-17T17:07:12.651069+0000","last_became_peered":"2024-08-17T17:07:12.651069+0000","last_unstale":"2024-08-17T17:11:58.649118+0000","last_undegraded":"2024-08-17T17:11:58.649118+0000","last_fullsized":"2024-08-17T17:11:58.649118+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:25.384305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1258,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075731+0000","last_change":"2024-08-17T17:07:45.691437+0000","last_active":"2024-08-17T17:11:59.075731+0000","last_peered":"2024-08-17T17:11:59.075731+0000","last_clean":"2024-08-17T17:11:59.075731+0000","last_became_active":"2024-08-17T17:07:45.690567+0000","last_became_peered":"2024-08-17T17:07:45.690567+0000","last_unstale":"2024-08-17T17:11:59.075731+0000","last_undegraded":"2024-08-17T17:11:59.075731+0000","last_fullsized":"2024-08-17T17:11:59.075731+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:43:07.698912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1024,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800708+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:11:52.800708+0000","last_peered":"2024-08-17T17:11:52.800708+0000","last_clean":"2024-08-17T17:11:52.800708+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:11:52.800708+0000","last_undegraded":"2024-08-17T17:11:52.800708+0000","last_fullsized":"2024-08-17T17:11:52.800708+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1022,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.700185+0000","last_change":"2024-08-17T17:06:39.629619+0000","last_active":"2024-08-17T17:11:52.700185+0000","last_peered":"2024-08-17T17:11:52.700185+0000","last_clean":"2024-08-17T17:11:52.700185+0000","last_became_active":"2024-08-17T17:06:39.629295+0000","last_became_peered":"2024-08-17T17:06:39.629295+0000","last_unstale":"2024-08-17T17:11:52.700185+0000","last_undegraded":"2024-08-17T17:11:52.700185+0000","last_fullsized":"2024-08-17T17:11:52.700185+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:15.874423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1039,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.801064+0000","last_change":"2024-08-17T17:07:12.649178+0000","last_active":"2024-08-17T17:11:52.801064+0000","last_peered":"2024-08-17T17:11:52.801064+0000","last_clean":"2024-08-17T17:11:52.801064+0000","last_became_active":"2024-08-17T17:07:12.648844+0000","last_became_peered":"2024-08-17T17:07:12.648844+0000","last_unstale":"2024-08-17T17:11:52.801064+0000","last_undegraded":"2024-08-17T17:11:52.801064+0000","last_fullsized":"2024-08-17T17:11:52.801064+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:12:06.307736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1025,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800742+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:11:52.800742+0000","last_peered":"2024-08-17T17:11:52.800742+0000","last_clean":"2024-08-17T17:11:52.800742+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:11:52.800742+0000","last_undegraded":"2024-08-17T17:11:52.800742+0000","last_fullsized":"2024-08-17T17:11:52.800742+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1024,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.801216+0000","last_change":"2024-08-17T17:06:39.627678+0000","last_active":"2024-08-17T17:11:52.801216+0000","last_peered":"2024-08-17T17:11:52.801216+0000","last_clean":"2024-08-17T17:11:52.801216+0000","last_became_active":"2024-08-17T17:06:39.627226+0000","last_became_peered":"2024-08-17T17:06:39.627226+0000","last_unstale":"2024-08-17T17:11:52.801216+0000","last_undegraded":"2024-08-17T17:11:52.801216+0000","last_fullsized":"2024-08-17T17:11:52.801216+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:01.868244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1247,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075390+0000","last_change":"2024-08-17T17:07:45.680458+0000","last_active":"2024-08-17T17:11:59.075390+0000","last_peered":"2024-08-17T17:11:59.075390+0000","last_clean":"2024-08-17T17:11:59.075390+0000","last_became_active":"2024-08-17T17:07:45.680221+0000","last_became_peered":"2024-08-17T17:07:45.680221+0000","last_unstale":"2024-08-17T17:11:59.075390+0000","last_undegraded":"2024-08-17T17:11:59.075390+0000","last_fullsized":"2024-08-17T17:11:59.075390+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:55.083962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1269,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075358+0000","last_change":"2024-08-17T17:07:45.682866+0000","last_active":"2024-08-17T17:11:59.075358+0000","last_peered":"2024-08-17T17:11:59.075358+0000","last_clean":"2024-08-17T17:11:59.075358+0000","last_became_active":"2024-08-17T17:07:45.682554+0000","last_became_peered":"2024-08-17T17:07:45.682554+0000","last_unstale":"2024-08-17T17:11:59.075358+0000","last_undegraded":"2024-08-17T17:11:59.075358+0000","last_fullsized":"2024-08-17T17:11:59.075358+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:25.975914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1338,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074716+0000","last_change":"2024-08-17T17:06:07.636484+0000","last_active":"2024-08-17T17:11:59.074716+0000","last_peered":"2024-08-17T17:11:59.074716+0000","last_clean":"2024-08-17T17:11:59.074716+0000","last_became_active":"2024-08-17T17:06:07.635998+0000","last_became_peered":"2024-08-17T17:06:07.635998+0000","last_unstale":"2024-08-17T17:11:59.074716+0000","last_undegraded":"2024-08-17T17:11:59.074716+0000","last_fullsized":"2024-08-17T17:11:59.074716+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:16.096587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1035,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011021+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:11:53.011021+0000","last_peered":"2024-08-17T17:11:53.011021+0000","last_clean":"2024-08-17T17:11:53.011021+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:11:53.011021+0000","last_undegraded":"2024-08-17T17:11:53.011021+0000","last_fullsized":"2024-08-17T17:11:53.011021+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1226,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074625+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:11:59.074625+0000","last_peered":"2024-08-17T17:11:59.074625+0000","last_clean":"2024-08-17T17:11:59.074625+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:11:59.074625+0000","last_undegraded":"2024-08-17T17:11:59.074625+0000","last_fullsized":"2024-08-17T17:11:59.074625+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1007,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703217+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:11:57.703217+0000","last_peered":"2024-08-17T17:11:57.703217+0000","last_clean":"2024-08-17T17:11:57.703217+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:11:57.703217+0000","last_undegraded":"2024-08-17T17:11:57.703217+0000","last_fullsized":"2024-08-17T17:11:57.703217+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1035,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074623+0000","last_change":"2024-08-17T17:06:39.638376+0000","last_active":"2024-08-17T17:11:59.074623+0000","last_peered":"2024-08-17T17:11:59.074623+0000","last_clean":"2024-08-17T17:11:59.074623+0000","last_became_active":"2024-08-17T17:06:39.638050+0000","last_became_peered":"2024-08-17T17:06:39.638050+0000","last_unstale":"2024-08-17T17:11:59.074623+0000","last_undegraded":"2024-08-17T17:11:59.074623+0000","last_fullsized":"2024-08-17T17:11:59.074623+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:05.782492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1367,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648019+0000","last_change":"2024-08-17T17:07:45.683556+0000","last_active":"2024-08-17T17:11:58.648019+0000","last_peered":"2024-08-17T17:11:58.648019+0000","last_clean":"2024-08-17T17:11:58.648019+0000","last_became_active":"2024-08-17T17:07:45.683232+0000","last_became_peered":"2024-08-17T17:07:45.683232+0000","last_unstale":"2024-08-17T17:11:58.648019+0000","last_undegraded":"2024-08-17T17:11:58.648019+0000","last_fullsized":"2024-08-17T17:11:58.648019+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:16.112417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1339,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648339+0000","last_change":"2024-08-17T17:07:12.651145+0000","last_active":"2024-08-17T17:11:58.648339+0000","last_peered":"2024-08-17T17:11:58.648339+0000","last_clean":"2024-08-17T17:11:58.648339+0000","last_became_active":"2024-08-17T17:07:12.650839+0000","last_became_peered":"2024-08-17T17:07:12.650839+0000","last_unstale":"2024-08-17T17:11:58.648339+0000","last_undegraded":"2024-08-17T17:11:58.648339+0000","last_fullsized":"2024-08-17T17:11:58.648339+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:59.795181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1309,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074165+0000","last_change":"2024-08-17T17:07:45.681107+0000","last_active":"2024-08-17T17:11:59.074165+0000","last_peered":"2024-08-17T17:11:59.074165+0000","last_clean":"2024-08-17T17:11:59.074165+0000","last_became_active":"2024-08-17T17:07:45.680770+0000","last_became_peered":"2024-08-17T17:07:45.680770+0000","last_unstale":"2024-08-17T17:11:59.074165+0000","last_undegraded":"2024-08-17T17:11:59.074165+0000","last_fullsized":"2024-08-17T17:11:59.074165+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:53.820920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1300,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.704078+0000","last_change":"2024-08-17T17:07:12.649485+0000","last_active":"2024-08-17T17:11:57.704078+0000","last_peered":"2024-08-17T17:11:57.704078+0000","last_clean":"2024-08-17T17:11:57.704078+0000","last_became_active":"2024-08-17T17:07:12.649161+0000","last_became_peered":"2024-08-17T17:07:12.649161+0000","last_unstale":"2024-08-17T17:11:57.704078+0000","last_undegraded":"2024-08-17T17:11:57.704078+0000","last_fullsized":"2024-08-17T17:11:57.704078+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:18.370628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1314,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703394+0000","last_change":"2024-08-17T17:06:39.627820+0000","last_active":"2024-08-17T17:11:57.703394+0000","last_peered":"2024-08-17T17:11:57.703394+0000","last_clean":"2024-08-17T17:11:57.703394+0000","last_became_active":"2024-08-17T17:06:39.627464+0000","last_became_peered":"2024-08-17T17:06:39.627464+0000","last_unstale":"2024-08-17T17:11:57.703394+0000","last_undegraded":"2024-08-17T17:11:57.703394+0000","last_fullsized":"2024-08-17T17:11:57.703394+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:01.500528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1036,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.010866+0000","last_change":"2024-08-17T17:06:39.628745+0000","last_active":"2024-08-17T17:11:53.010866+0000","last_peered":"2024-08-17T17:11:53.010866+0000","last_clean":"2024-08-17T17:11:53.010866+0000","last_became_active":"2024-08-17T17:06:39.628144+0000","last_became_peered":"2024-08-17T17:06:39.628144+0000","last_unstale":"2024-08-17T17:11:53.010866+0000","last_undegraded":"2024-08-17T17:11:53.010866+0000","last_fullsized":"2024-08-17T17:11:53.010866+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:14.740936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1026,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075115+0000","last_change":"2024-08-17T17:06:07.637581+0000","last_active":"2024-08-17T17:11:59.075115+0000","last_peered":"2024-08-17T17:11:59.075115+0000","last_clean":"2024-08-17T17:11:59.075115+0000","last_became_active":"2024-08-17T17:06:07.637282+0000","last_became_peered":"2024-08-17T17:06:07.637282+0000","last_unstale":"2024-08-17T17:11:59.075115+0000","last_undegraded":"2024-08-17T17:11:59.075115+0000","last_fullsized":"2024-08-17T17:11:59.075115+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.938734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1310,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702867+0000","last_change":"2024-08-17T17:07:45.687502+0000","last_active":"2024-08-17T17:11:57.702867+0000","last_peered":"2024-08-17T17:11:57.702867+0000","last_clean":"2024-08-17T17:11:57.702867+0000","last_became_active":"2024-08-17T17:07:45.687197+0000","last_became_peered":"2024-08-17T17:07:45.687197+0000","last_unstale":"2024-08-17T17:11:57.702867+0000","last_undegraded":"2024-08-17T17:11:57.702867+0000","last_fullsized":"2024-08-17T17:11:57.702867+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:23.577838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1037,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.981002+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:11:58.981002+0000","last_peered":"2024-08-17T17:11:58.981002+0000","last_clean":"2024-08-17T17:11:58.981002+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:11:58.981002+0000","last_undegraded":"2024-08-17T17:11:58.981002+0000","last_fullsized":"2024-08-17T17:11:58.981002+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1328,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703305+0000","last_change":"2024-08-17T17:06:39.633975+0000","last_active":"2024-08-17T17:11:57.703305+0000","last_peered":"2024-08-17T17:11:57.703305+0000","last_clean":"2024-08-17T17:11:57.703305+0000","last_became_active":"2024-08-17T17:06:39.631526+0000","last_became_peered":"2024-08-17T17:06:39.631526+0000","last_unstale":"2024-08-17T17:11:57.703305+0000","last_undegraded":"2024-08-17T17:11:57.703305+0000","last_fullsized":"2024-08-17T17:11:57.703305+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:58.542237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1288,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703023+0000","last_change":"2024-08-17T17:06:39.627577+0000","last_active":"2024-08-17T17:11:57.703023+0000","last_peered":"2024-08-17T17:11:57.703023+0000","last_clean":"2024-08-17T17:11:57.703023+0000","last_became_active":"2024-08-17T17:06:39.626554+0000","last_became_peered":"2024-08-17T17:06:39.626554+0000","last_unstale":"2024-08-17T17:11:57.703023+0000","last_undegraded":"2024-08-17T17:11:57.703023+0000","last_fullsized":"2024-08-17T17:11:57.703023+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:04:48.533057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1298,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702987+0000","last_change":"2024-08-17T17:07:12.645660+0000","last_active":"2024-08-17T17:11:57.702987+0000","last_peered":"2024-08-17T17:11:57.702987+0000","last_clean":"2024-08-17T17:11:57.702987+0000","last_became_active":"2024-08-17T17:07:12.645368+0000","last_became_peered":"2024-08-17T17:07:12.645368+0000","last_unstale":"2024-08-17T17:11:57.702987+0000","last_undegraded":"2024-08-17T17:11:57.702987+0000","last_fullsized":"2024-08-17T17:11:57.702987+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:41:00.157045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1019,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.699659+0000","last_change":"2024-08-17T17:06:07.636239+0000","last_active":"2024-08-17T17:11:52.699659+0000","last_peered":"2024-08-17T17:11:52.699659+0000","last_clean":"2024-08-17T17:11:52.699659+0000","last_became_active":"2024-08-17T17:06:07.635881+0000","last_became_peered":"2024-08-17T17:06:07.635881+0000","last_unstale":"2024-08-17T17:11:52.699659+0000","last_undegraded":"2024-08-17T17:11:52.699659+0000","last_fullsized":"2024-08-17T17:11:52.699659+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:36.001648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1316,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648899+0000","last_change":"2024-08-17T17:06:07.634369+0000","last_active":"2024-08-17T17:11:58.648899+0000","last_peered":"2024-08-17T17:11:58.648899+0000","last_clean":"2024-08-17T17:11:58.648899+0000","last_became_active":"2024-08-17T17:06:07.633769+0000","last_became_peered":"2024-08-17T17:06:07.633769+0000","last_unstale":"2024-08-17T17:11:58.648899+0000","last_undegraded":"2024-08-17T17:11:58.648899+0000","last_fullsized":"2024-08-17T17:11:58.648899+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:20.957493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1258,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075605+0000","last_change":"2024-08-17T17:07:45.688526+0000","last_active":"2024-08-17T17:11:59.075605+0000","last_peered":"2024-08-17T17:11:59.075605+0000","last_clean":"2024-08-17T17:11:59.075605+0000","last_became_active":"2024-08-17T17:07:45.688091+0000","last_became_peered":"2024-08-17T17:07:45.688091+0000","last_unstale":"2024-08-17T17:11:59.075605+0000","last_undegraded":"2024-08-17T17:11:59.075605+0000","last_fullsized":"2024-08-17T17:11:59.075605+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:02.385683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1298,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703411+0000","last_change":"2024-08-17T17:06:39.629005+0000","last_active":"2024-08-17T17:11:57.703411+0000","last_peered":"2024-08-17T17:11:57.703411+0000","last_clean":"2024-08-17T17:11:57.703411+0000","last_became_active":"2024-08-17T17:06:39.628679+0000","last_became_peered":"2024-08-17T17:06:39.628679+0000","last_unstale":"2024-08-17T17:11:57.703411+0000","last_undegraded":"2024-08-17T17:11:57.703411+0000","last_fullsized":"2024-08-17T17:11:57.703411+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:09.401059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1255,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075180+0000","last_change":"2024-08-17T17:07:45.695858+0000","last_active":"2024-08-17T17:11:59.075180+0000","last_peered":"2024-08-17T17:11:59.075180+0000","last_clean":"2024-08-17T17:11:59.075180+0000","last_became_active":"2024-08-17T17:07:45.694111+0000","last_became_peered":"2024-08-17T17:07:45.694111+0000","last_unstale":"2024-08-17T17:11:59.075180+0000","last_undegraded":"2024-08-17T17:11:59.075180+0000","last_fullsized":"2024-08-17T17:11:59.075180+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:40:38.772090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":1330,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648523+0000","last_change":"2024-08-17T17:06:39.636866+0000","last_active":"2024-08-17T17:11:58.648523+0000","last_peered":"2024-08-17T17:11:58.648523+0000","last_clean":"2024-08-17T17:11:58.648523+0000","last_became_active":"2024-08-17T17:06:39.636470+0000","last_became_peered":"2024-08-17T17:06:39.636470+0000","last_unstale":"2024-08-17T17:11:58.648523+0000","last_undegraded":"2024-08-17T17:11:58.648523+0000","last_fullsized":"2024-08-17T17:11:58.648523+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:14.028712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1034,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:56.929467+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:11:56.929467+0000","last_peered":"2024-08-17T17:11:56.929467+0000","last_clean":"2024-08-17T17:11:56.929467+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:11:56.929467+0000","last_undegraded":"2024-08-17T17:11:56.929467+0000","last_fullsized":"2024-08-17T17:11:56.929467+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1010,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075411+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:11:59.075411+0000","last_peered":"2024-08-17T17:11:59.075411+0000","last_clean":"2024-08-17T17:11:59.075411+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:11:59.075411+0000","last_undegraded":"2024-08-17T17:11:59.075411+0000","last_fullsized":"2024-08-17T17:11:59.075411+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1024,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074961+0000","last_change":"2024-08-17T17:06:39.627594+0000","last_active":"2024-08-17T17:11:59.074961+0000","last_peered":"2024-08-17T17:11:59.074961+0000","last_clean":"2024-08-17T17:11:59.074961+0000","last_became_active":"2024-08-17T17:06:39.627105+0000","last_became_peered":"2024-08-17T17:06:39.627105+0000","last_unstale":"2024-08-17T17:11:59.074961+0000","last_undegraded":"2024-08-17T17:11:59.074961+0000","last_fullsized":"2024-08-17T17:11:59.074961+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:30.105070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1033,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075812+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:11:59.075812+0000","last_peered":"2024-08-17T17:11:59.075812+0000","last_clean":"2024-08-17T17:11:59.075812+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:11:59.075812+0000","last_undegraded":"2024-08-17T17:11:59.075812+0000","last_fullsized":"2024-08-17T17:11:59.075812+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1299,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703585+0000","last_change":"2024-08-17T17:07:12.645440+0000","last_active":"2024-08-17T17:11:57.703585+0000","last_peered":"2024-08-17T17:11:57.703585+0000","last_clean":"2024-08-17T17:11:57.703585+0000","last_became_active":"2024-08-17T17:07:12.645147+0000","last_became_peered":"2024-08-17T17:07:12.645147+0000","last_unstale":"2024-08-17T17:11:57.703585+0000","last_undegraded":"2024-08-17T17:11:57.703585+0000","last_fullsized":"2024-08-17T17:11:57.703585+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:25.588528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1009,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.076163+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:11:59.076163+0000","last_peered":"2024-08-17T17:11:59.076163+0000","last_clean":"2024-08-17T17:11:59.076163+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:11:59.076163+0000","last_undegraded":"2024-08-17T17:11:59.076163+0000","last_fullsized":"2024-08-17T17:11:59.076163+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1020,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011326+0000","last_change":"2024-08-17T17:06:39.625029+0000","last_active":"2024-08-17T17:11:53.011326+0000","last_peered":"2024-08-17T17:11:53.011326+0000","last_clean":"2024-08-17T17:11:53.011326+0000","last_became_active":"2024-08-17T17:06:39.624688+0000","last_became_peered":"2024-08-17T17:06:39.624688+0000","last_unstale":"2024-08-17T17:11:53.011326+0000","last_undegraded":"2024-08-17T17:11:53.011326+0000","last_fullsized":"2024-08-17T17:11:53.011326+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:54.849668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1226,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075761+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:11:59.075761+0000","last_peered":"2024-08-17T17:11:59.075761+0000","last_clean":"2024-08-17T17:11:59.075761+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:11:59.075761+0000","last_undegraded":"2024-08-17T17:11:59.075761+0000","last_fullsized":"2024-08-17T17:11:59.075761+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1366,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648869+0000","last_change":"2024-08-17T17:07:45.690099+0000","last_active":"2024-08-17T17:11:58.648869+0000","last_peered":"2024-08-17T17:11:58.648869+0000","last_clean":"2024-08-17T17:11:58.648869+0000","last_became_active":"2024-08-17T17:07:45.689783+0000","last_became_peered":"2024-08-17T17:07:45.689783+0000","last_unstale":"2024-08-17T17:11:58.648869+0000","last_undegraded":"2024-08-17T17:11:58.648869+0000","last_fullsized":"2024-08-17T17:11:58.648869+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:38.311630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1245,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075127+0000","last_change":"2024-08-17T17:07:45.678950+0000","last_active":"2024-08-17T17:11:59.075127+0000","last_peered":"2024-08-17T17:11:59.075127+0000","last_clean":"2024-08-17T17:11:59.075127+0000","last_became_active":"2024-08-17T17:07:45.678472+0000","last_became_peered":"2024-08-17T17:07:45.678472+0000","last_unstale":"2024-08-17T17:11:59.075127+0000","last_undegraded":"2024-08-17T17:11:59.075127+0000","last_fullsized":"2024-08-17T17:11:59.075127+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:48:02.385927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1022,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.703548+0000","last_change":"2024-08-17T17:06:39.628542+0000","last_active":"2024-08-17T17:11:57.703548+0000","last_peered":"2024-08-17T17:11:57.703548+0000","last_clean":"2024-08-17T17:11:57.703548+0000","last_became_active":"2024-08-17T17:06:39.628253+0000","last_became_peered":"2024-08-17T17:06:39.628253+0000","last_unstale":"2024-08-17T17:11:57.703548+0000","last_undegraded":"2024-08-17T17:11:57.703548+0000","last_fullsized":"2024-08-17T17:11:57.703548+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:22.349326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1261,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075295+0000","last_change":"2024-08-17T17:07:45.696068+0000","last_active":"2024-08-17T17:11:59.075295+0000","last_peered":"2024-08-17T17:11:59.075295+0000","last_clean":"2024-08-17T17:11:59.075295+0000","last_became_active":"2024-08-17T17:07:45.694617+0000","last_became_peered":"2024-08-17T17:07:45.694617+0000","last_unstale":"2024-08-17T17:11:59.075295+0000","last_undegraded":"2024-08-17T17:11:59.075295+0000","last_fullsized":"2024-08-17T17:11:59.075295+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:08.646380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1327,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648379+0000","last_change":"2024-08-17T17:07:45.684186+0000","last_active":"2024-08-17T17:11:58.648379+0000","last_peered":"2024-08-17T17:11:58.648379+0000","last_clean":"2024-08-17T17:11:58.648379+0000","last_became_active":"2024-08-17T17:07:45.683863+0000","last_became_peered":"2024-08-17T17:07:45.683863+0000","last_unstale":"2024-08-17T17:11:58.648379+0000","last_undegraded":"2024-08-17T17:11:58.648379+0000","last_fullsized":"2024-08-17T17:11:58.648379+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:52.106168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1298,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:57.702758+0000","last_change":"2024-08-17T17:07:45.688645+0000","last_active":"2024-08-17T17:11:57.702758+0000","last_peered":"2024-08-17T17:11:57.702758+0000","last_clean":"2024-08-17T17:11:57.702758+0000","last_became_active":"2024-08-17T17:07:45.688383+0000","last_became_peered":"2024-08-17T17:07:45.688383+0000","last_unstale":"2024-08-17T17:11:57.702758+0000","last_undegraded":"2024-08-17T17:11:57.702758+0000","last_fullsized":"2024-08-17T17:11:57.702758+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:33:12.209565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1245,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074861+0000","last_change":"2024-08-17T17:07:45.689782+0000","last_active":"2024-08-17T17:11:59.074861+0000","last_peered":"2024-08-17T17:11:59.074861+0000","last_clean":"2024-08-17T17:11:59.074861+0000","last_became_active":"2024-08-17T17:07:45.689312+0000","last_became_peered":"2024-08-17T17:07:45.689312+0000","last_unstale":"2024-08-17T17:11:59.074861+0000","last_undegraded":"2024-08-17T17:11:59.074861+0000","last_fullsized":"2024-08-17T17:11:59.074861+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:07.560555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1040,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800592+0000","last_change":"2024-08-17T17:07:12.649175+0000","last_active":"2024-08-17T17:11:52.800592+0000","last_peered":"2024-08-17T17:11:52.800592+0000","last_clean":"2024-08-17T17:11:52.800592+0000","last_became_active":"2024-08-17T17:07:12.648822+0000","last_became_peered":"2024-08-17T17:07:12.648822+0000","last_unstale":"2024-08-17T17:11:52.800592+0000","last_undegraded":"2024-08-17T17:11:52.800592+0000","last_fullsized":"2024-08-17T17:11:52.800592+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:03.924431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1286,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.074322+0000","last_change":"2024-08-17T17:07:45.685278+0000","last_active":"2024-08-17T17:11:59.074322+0000","last_peered":"2024-08-17T17:11:59.074322+0000","last_clean":"2024-08-17T17:11:59.074322+0000","last_became_active":"2024-08-17T17:07:45.684963+0000","last_became_peered":"2024-08-17T17:07:45.684963+0000","last_unstale":"2024-08-17T17:11:59.074322+0000","last_undegraded":"2024-08-17T17:11:59.074322+0000","last_fullsized":"2024-08-17T17:11:59.074322+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:55:42.495791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":1317,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.649176+0000","last_change":"2024-08-17T17:06:07.634103+0000","last_active":"2024-08-17T17:11:58.649176+0000","last_peered":"2024-08-17T17:11:58.649176+0000","last_clean":"2024-08-17T17:11:58.649176+0000","last_became_active":"2024-08-17T17:06:07.633624+0000","last_became_peered":"2024-08-17T17:06:07.633624+0000","last_unstale":"2024-08-17T17:11:58.649176+0000","last_undegraded":"2024-08-17T17:11:58.649176+0000","last_fullsized":"2024-08-17T17:11:58.649176+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:58.573558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800611+0000","last_change":"2024-08-17T17:07:45.684606+0000","last_active":"2024-08-17T17:11:52.800611+0000","last_peered":"2024-08-17T17:11:52.800611+0000","last_clean":"2024-08-17T17:11:52.800611+0000","last_became_active":"2024-08-17T17:07:45.684124+0000","last_became_peered":"2024-08-17T17:07:45.684124+0000","last_unstale":"2024-08-17T17:11:52.800611+0000","last_undegraded":"2024-08-17T17:11:52.800611+0000","last_fullsized":"2024-08-17T17:11:52.800611+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:16.110507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1022,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.011023+0000","last_change":"2024-08-17T17:06:39.625245+0000","last_active":"2024-08-17T17:11:53.011023+0000","last_peered":"2024-08-17T17:11:53.011023+0000","last_clean":"2024-08-17T17:11:53.011023+0000","last_became_active":"2024-08-17T17:06:39.624657+0000","last_became_peered":"2024-08-17T17:06:39.624657+0000","last_unstale":"2024-08-17T17:11:53.011023+0000","last_undegraded":"2024-08-17T17:11:53.011023+0000","last_fullsized":"2024-08-17T17:11:53.011023+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:43.097460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1038,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800484+0000","last_change":"2024-08-17T17:07:45.684769+0000","last_active":"2024-08-17T17:11:52.800484+0000","last_peered":"2024-08-17T17:11:52.800484+0000","last_clean":"2024-08-17T17:11:52.800484+0000","last_became_active":"2024-08-17T17:07:45.684459+0000","last_became_peered":"2024-08-17T17:07:45.684459+0000","last_unstale":"2024-08-17T17:11:52.800484+0000","last_undegraded":"2024-08-17T17:11:52.800484+0000","last_fullsized":"2024-08-17T17:11:52.800484+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:30.731978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1007,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:53.010451+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:11:53.010451+0000","last_peered":"2024-08-17T17:11:53.010451+0000","last_clean":"2024-08-17T17:11:53.010451+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:11:53.010451+0000","last_undegraded":"2024-08-17T17:11:53.010451+0000","last_fullsized":"2024-08-17T17:11:53.010451+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1251,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.076011+0000","last_change":"2024-08-17T17:07:45.692127+0000","last_active":"2024-08-17T17:11:59.076011+0000","last_peered":"2024-08-17T17:11:59.076011+0000","last_clean":"2024-08-17T17:11:59.076011+0000","last_became_active":"2024-08-17T17:07:45.690277+0000","last_became_peered":"2024-08-17T17:07:45.690277+0000","last_unstale":"2024-08-17T17:11:59.076011+0000","last_undegraded":"2024-08-17T17:11:59.076011+0000","last_fullsized":"2024-08-17T17:11:59.076011+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:04.555409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1239,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:59.075238+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:11:59.075238+0000","last_peered":"2024-08-17T17:11:59.075238+0000","last_clean":"2024-08-17T17:11:59.075238+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:11:59.075238+0000","last_undegraded":"2024-08-17T17:11:59.075238+0000","last_fullsized":"2024-08-17T17:11:59.075238+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1023,"reported_epoch":1083,"state":"active+clean","last_fresh":"2024-08-17T17:11:52.800297+0000","last_change":"2024-08-17T17:06:39.627689+0000","last_active":"2024-08-17T17:11:52.800297+0000","last_peered":"2024-08-17T17:11:52.800297+0000","last_clean":"2024-08-17T17:11:52.800297+0000","last_became_active":"2024-08-17T17:06:39.627375+0000","last_became_peered":"2024-08-17T17:06:39.627375+0000","last_unstale":"2024-08-17T17:11:52.800297+0000","last_undegraded":"2024-08-17T17:11:52.800297+0000","last_fullsized":"2024-08-17T17:11:52.800297+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:51.954076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":1333,"reported_epoch":1084,"state":"active+clean","last_fresh":"2024-08-17T17:11:58.648938+0000","last_change":"2024-08-17T17:07:12.651933+0000","last_active":"2024-08-17T17:11:58.648938+0000","last_peered":"2024-08-17T17:11:58.648938+0000","last_clean":"2024-08-17T17:11:58.648938+0000","last_became_active":"2024-08-17T17:07:12.651592+0000","last_became_peered":"2024-08-17T17:07:12.651592+0000","last_unstale":"2024-08-17T17:11:58.648938+0000","last_undegraded":"2024-08-17T17:11:58.648938+0000","last_fullsized":"2024-08-17T17:11:58.648938+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:31.224038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201537,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":55668,"kb_used_data":10600,"kb_used_omap":1,"kb_used_meta":45054,"kb_avail":93689484,"statfs":{"total":95995035648,"available":95938031616,"internally_reserved":0,"allocated":10854400,"data_stored":7795252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":46135755},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463174,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":67944,"kb_used_data":10592,"kb_used_omap":1,"kb_used_meta":57342,"kb_avail":93677208,"statfs":{"total":95995035648,"available":95925460992,"internally_reserved":0,"allocated":10846208,"data_stored":7795125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":58718665},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626815,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":46296,"kb_used_data":9996,"kb_used_omap":42,"kb_used_meta":36245,"kb_avail":93698856,"statfs":{"total":95995035648,"available":95947628544,"internally_reserved":0,"allocated":10235904,"data_stored":7200014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":37114940},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785292,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":50584,"kb_used_data":9996,"kb_used_omap":48,"kb_used_meta":40527,"kb_avail":93694568,"statfs":{"total":95995035648,"available":95943237632,"internally_reserved":0,"allocated":10235904,"data_stored":7199943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":41500334},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724810,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":107488,"kb_used_data":10004,"kb_used_omap":1,"kb_used_meta":97470,"kb_avail":93637664,"statfs":{"total":95995035648,"available":95884967936,"internally_reserved":0,"allocated":10244096,"data_stored":7200583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":99809734},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986447,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":95212,"kb_used_data":10016,"kb_used_omap":1,"kb_used_meta":85182,"kb_avail":93649940,"statfs":{"total":95995035648,"available":95897538560,"internally_reserved":0,"allocated":10256384,"data_stored":7204814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":87226822},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706054,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":46172,"kb_used_data":10000,"kb_used_omap":35,"kb_used_meta":36124,"kb_avail":93698980,"statfs":{"total":95995035648,"available":95947755520,"internally_reserved":0,"allocated":10240000,"data_stored":7200540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":36991031},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831827,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":43040,"kb_used_data":10580,"kb_used_omap":65,"kb_used_meta":32382,"kb_avail":93702112,"statfs":{"total":95995035648,"available":95950962688,"internally_reserved":0,"allocated":10833920,"data_stored":7790896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":33159355},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:12:54.361 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-08-17T17:12:54.361 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump 2024-08-17T17:12:54.572 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:54.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:54 smithi162 ceph-mon[104668]: from='client.44534 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:12:55.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:55 smithi063 ceph-mon[114756]: pgmap v1114: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:55.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:55 smithi063 ceph-mon[109843]: pgmap v1114: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:55.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:55 smithi162 ceph-mon[104668]: pgmap v1114: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:56.272 INFO:teuthology.orchestra.run.smithi063.stdout:epoch 1084 2024-08-17T17:12:56.272 INFO:teuthology.orchestra.run.smithi063.stdout:fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:12:56.272 INFO:teuthology.orchestra.run.smithi063.stdout:created 2024-08-17T16:46:54.156006+0000 2024-08-17T17:12:56.272 INFO:teuthology.orchestra.run.smithi063.stdout:modified 2024-08-17T17:11:57.696744+0000 2024-08-17T17:12:56.273 INFO:teuthology.orchestra.run.smithi063.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-08-17T17:12:56.273 INFO:teuthology.orchestra.run.smithi063.stdout:crush_version 22 2024-08-17T17:12:56.273 INFO:teuthology.orchestra.run.smithi063.stdout:full_ratio 0.95 2024-08-17T17:12:56.273 INFO:teuthology.orchestra.run.smithi063.stdout:backfillfull_ratio 0.9 2024-08-17T17:12:56.273 INFO:teuthology.orchestra.run.smithi063.stdout:nearfull_ratio 0.85 2024-08-17T17:12:56.273 INFO:teuthology.orchestra.run.smithi063.stdout:require_min_compat_client luminous 2024-08-17T17:12:56.273 INFO:teuthology.orchestra.run.smithi063.stdout:min_compat_client luminous 2024-08-17T17:12:56.273 INFO:teuthology.orchestra.run.smithi063.stdout:require_osd_release reef 2024-08-17T17:12:56.274 INFO:teuthology.orchestra.run.smithi063.stdout:stretch_mode_enabled false 2024-08-17T17:12:56.274 INFO:teuthology.orchestra.run.smithi063.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2024-08-17T17:12:56.274 INFO:teuthology.orchestra.run.smithi063.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 66 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2024-08-17T17:12:56.274 INFO:teuthology.orchestra.run.smithi063.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 71 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2024-08-17T17:12:56.274 INFO:teuthology.orchestra.run.smithi063.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 72 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2024-08-17T17:12:56.274 INFO:teuthology.orchestra.run.smithi063.stdout:max_osd 8 2024-08-17T17:12:56.274 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 up in weight 1 up_from 739 up_thru 837 down_at 727 last_clean_interval [8,726) [v2:172.21.15.63:6802/4285380350,v1:172.21.15.63:6803/4285380350] [v2:172.21.15.63:6804/4285380350,v1:172.21.15.63:6805/4285380350] exists,up c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2 2024-08-17T17:12:56.274 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1 up in weight 1 up_from 771 up_thru 837 down_at 759 last_clean_interval [13,758) [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701] [v2:172.21.15.63:6812/3170762701,v1:172.21.15.63:6813/3170762701] exists,up ae346e99-4706-4871-990a-f8e737572245 2024-08-17T17:12:56.274 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2 up in weight 1 up_from 804 up_thru 837 down_at 792 last_clean_interval [18,791) [v2:172.21.15.63:6818/3903658295,v1:172.21.15.63:6819/3903658295] [v2:172.21.15.63:6820/3903658295,v1:172.21.15.63:6821/3903658295] exists,up 2d85d75f-1280-443f-857d-d322bd60c451 2024-08-17T17:12:56.275 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 up in weight 1 up_from 837 up_thru 837 down_at 825 last_clean_interval [26,824) [v2:172.21.15.63:6826/1764356824,v1:172.21.15.63:6827/1764356824] [v2:172.21.15.63:6828/1764356824,v1:172.21.15.63:6829/1764356824] exists,up 771f43d2-e98b-43e0-921b-88e56fadda42 2024-08-17T17:12:56.275 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 up in weight 1 up_from 31 up_thru 837 down_at 0 last_clean_interval [0,0) [v2:172.21.15.162:6800/1563885208,v1:172.21.15.162:6801/1563885208] [v2:172.21.15.162:6802/1563885208,v1:172.21.15.162:6803/1563885208] exists,up 73c3c545-e975-4a6f-a1cb-e4575ee7a9b0 2024-08-17T17:12:56.275 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5 up in weight 1 up_from 39 up_thru 837 down_at 0 last_clean_interval [0,0) [v2:172.21.15.162:6808/1826502179,v1:172.21.15.162:6809/1826502179] [v2:172.21.15.162:6810/1826502179,v1:172.21.15.162:6811/1826502179] exists,up fae3ac52-3194-4eb4-9510-e3899768353a 2024-08-17T17:12:56.275 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6 up in weight 1 up_from 47 up_thru 825 down_at 0 last_clean_interval [0,0) [v2:172.21.15.162:6816/1675904899,v1:172.21.15.162:6817/1675904899] [v2:172.21.15.162:6818/1675904899,v1:172.21.15.162:6819/1675904899] exists,up a8255b37-1be5-4cd4-9b86-889f6bf52902 2024-08-17T17:12:56.275 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7 up in weight 1 up_from 55 up_thru 837 down_at 0 last_clean_interval [0,0) [v2:172.21.15.162:6824/2938230961,v1:172.21.15.162:6825/2938230961] [v2:172.21.15.162:6826/2938230961,v1:172.21.15.162:6827/2938230961] exists,up 3d0bcd6d-171d-4d90-a033-ed17714a9d96 2024-08-17T17:12:56.275 INFO:teuthology.orchestra.run.smithi063.stdout:pg_upmap_items 4.6 [1,5] 2024-08-17T17:12:56.275 INFO:teuthology.orchestra.run.smithi063.stdout:pg_upmap_items 4.b [1,2] 2024-08-17T17:12:56.275 INFO:teuthology.orchestra.run.smithi063.stdout:pg_upmap_items 4.f [1,2] 2024-08-17T17:12:56.275 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:6801/4052036587 expires 2024-08-18T16:59:45.829424+0000 2024-08-17T17:12:56.275 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:6800/4052036587 expires 2024-08-18T16:59:45.829424+0000 2024-08-17T17:12:56.275 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/753392608 expires 2024-08-18T16:59:45.829424+0000 2024-08-17T17:12:56.275 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/701508812 expires 2024-08-18T16:52:20.398501+0000 2024-08-17T17:12:56.275 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/2384509117 expires 2024-08-18T16:59:45.829424+0000 2024-08-17T17:12:56.276 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:6800/987519051 expires 2024-08-18T16:56:38.413263+0000 2024-08-17T17:12:56.276 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.162:0/810332452 expires 2024-08-18T16:57:08.874093+0000 2024-08-17T17:12:56.276 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.162:0/1341643866 expires 2024-08-18T16:57:08.874093+0000 2024-08-17T17:12:56.276 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/3746443558 expires 2024-08-18T16:56:38.413263+0000 2024-08-17T17:12:56.276 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/3589494813 expires 2024-08-18T16:56:38.413263+0000 2024-08-17T17:12:56.276 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:6800/3322978932 expires 2024-08-18T16:47:16.384724+0000 2024-08-17T17:12:56.276 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/2696979598 expires 2024-08-18T16:56:38.413263+0000 2024-08-17T17:12:56.276 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.162:0/319999013 expires 2024-08-18T16:57:08.874093+0000 2024-08-17T17:12:56.276 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/84364348 expires 2024-08-18T16:47:16.384724+0000 2024-08-17T17:12:56.276 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.162:0/4017736715 expires 2024-08-18T16:57:08.874093+0000 2024-08-17T17:12:56.277 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/2404156937 expires 2024-08-18T16:59:45.829424+0000 2024-08-17T17:12:56.277 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:6801/987519051 expires 2024-08-18T16:56:38.413263+0000 2024-08-17T17:12:56.277 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/955244129 expires 2024-08-18T16:52:20.398501+0000 2024-08-17T17:12:56.277 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/664643590 expires 2024-08-18T16:59:45.829424+0000 2024-08-17T17:12:56.277 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:6800/83870469 expires 2024-08-18T16:52:20.398501+0000 2024-08-17T17:12:56.277 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:6801/2553533530 expires 2024-08-18T16:47:39.919472+0000 2024-08-17T17:12:56.277 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:6800/2553533530 expires 2024-08-18T16:47:39.919472+0000 2024-08-17T17:12:56.277 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.162:6834/3390906977 expires 2024-08-18T16:57:08.874093+0000 2024-08-17T17:12:56.277 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:6801/83870469 expires 2024-08-18T16:52:20.398501+0000 2024-08-17T17:12:56.277 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.162:6835/3390906977 expires 2024-08-18T16:57:08.874093+0000 2024-08-17T17:12:56.277 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/1986138219 expires 2024-08-18T16:47:39.919472+0000 2024-08-17T17:12:56.278 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/4067771896 expires 2024-08-18T16:56:38.413263+0000 2024-08-17T17:12:56.278 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:6801/3322978932 expires 2024-08-18T16:47:16.384724+0000 2024-08-17T17:12:56.278 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/1256300214 expires 2024-08-18T16:52:20.398501+0000 2024-08-17T17:12:56.278 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/1792393387 expires 2024-08-18T16:47:39.919472+0000 2024-08-17T17:12:56.278 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/3226753153 expires 2024-08-18T16:47:39.919472+0000 2024-08-17T17:12:56.278 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/3004218255 expires 2024-08-18T16:47:16.384724+0000 2024-08-17T17:12:56.278 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.162:0/2317499213 expires 2024-08-18T16:57:08.874093+0000 2024-08-17T17:12:56.278 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/2601197458 expires 2024-08-18T16:52:20.398501+0000 2024-08-17T17:12:56.278 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.63:0/3540337558 expires 2024-08-18T16:47:16.384724+0000 2024-08-17T17:12:56.278 INFO:teuthology.orchestra.run.smithi063.stdout:blocklist 172.21.15.162:0/2288037041 expires 2024-08-18T16:57:08.874093+0000 2024-08-17T17:12:56.871 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 739 up_thru 837 down_at 727 last_clean_interval [8,726) [v2:172.21.15.63:6802/4285380350,v1:172.21.15.63:6803/4285380350] [v2:172.21.15.63:6804/4285380350,v1:172.21.15.63:6805/4285380350] exists,up c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2', 'osd.1 up in weight 1 up_from 771 up_thru 837 down_at 759 last_clean_interval [13,758) [v2:172.21.15.63:6810/3170762701,v1:172.21.15.63:6811/3170762701] [v2:172.21.15.63:6812/3170762701,v1:172.21.15.63:6813/3170762701] exists,up ae346e99-4706-4871-990a-f8e737572245', 'osd.2 up in weight 1 up_from 804 up_thru 837 down_at 792 last_clean_interval [18,791) [v2:172.21.15.63:6818/3903658295,v1:172.21.15.63:6819/3903658295] [v2:172.21.15.63:6820/3903658295,v1:172.21.15.63:6821/3903658295] exists,up 2d85d75f-1280-443f-857d-d322bd60c451', 'osd.3 up in weight 1 up_from 837 up_thru 837 down_at 825 last_clean_interval [26,824) [v2:172.21.15.63:6826/1764356824,v1:172.21.15.63:6827/1764356824] [v2:172.21.15.63:6828/1764356824,v1:172.21.15.63:6829/1764356824] exists,up 771f43d2-e98b-43e0-921b-88e56fadda42', 'osd.4 up in weight 1 up_from 31 up_thru 837 down_at 0 last_clean_interval [0,0) [v2:172.21.15.162:6800/1563885208,v1:172.21.15.162:6801/1563885208] [v2:172.21.15.162:6802/1563885208,v1:172.21.15.162:6803/1563885208] exists,up 73c3c545-e975-4a6f-a1cb-e4575ee7a9b0', 'osd.5 up in weight 1 up_from 39 up_thru 837 down_at 0 last_clean_interval [0,0) [v2:172.21.15.162:6808/1826502179,v1:172.21.15.162:6809/1826502179] [v2:172.21.15.162:6810/1826502179,v1:172.21.15.162:6811/1826502179] exists,up fae3ac52-3194-4eb4-9510-e3899768353a', 'osd.6 up in weight 1 up_from 47 up_thru 825 down_at 0 last_clean_interval [0,0) [v2:172.21.15.162:6816/1675904899,v1:172.21.15.162:6817/1675904899] [v2:172.21.15.162:6818/1675904899,v1:172.21.15.162:6819/1675904899] exists,up a8255b37-1be5-4cd4-9b86-889f6bf52902', 'osd.7 up in weight 1 up_from 55 up_thru 837 down_at 0 last_clean_interval [0,0) [v2:172.21.15.162:6824/2938230961,v1:172.21.15.162:6825/2938230961] [v2:172.21.15.162:6826/2938230961,v1:172.21.15.162:6827/2938230961] exists,up 3d0bcd6d-171d-4d90-a033-ed17714a9d96'] 2024-08-17T17:12:56.871 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon mon.a config show 2024-08-17T17:12:57.082 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:57.495 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:57 smithi063 ceph-mon[114756]: pgmap v1115: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:57.495 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:57 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3288828529' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-08-17T17:12:57.495 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:57 smithi063 ceph-mon[109843]: pgmap v1115: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:57.495 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:57 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3288828529' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-08-17T17:12:57.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:57 smithi162 ceph-mon[104668]: pgmap v1115: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:57.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:57 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3288828529' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-08-17T17:12:58.566 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:12:58.566 INFO:teuthology.orchestra.run.smithi063.stdout: "name": "mon.a", 2024-08-17T17:12:58.566 INFO:teuthology.orchestra.run.smithi063.stdout: "cluster": "ceph", 2024-08-17T17:12:58.566 INFO:teuthology.orchestra.run.smithi063.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2024-08-17T17:12:58.566 INFO:teuthology.orchestra.run.smithi063.stdout: "admin_socket_mode": "", 2024-08-17T17:12:58.566 INFO:teuthology.orchestra.run.smithi063.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2024-08-17T17:12:58.566 INFO:teuthology.orchestra.run.smithi063.stdout: "auth_client_required": "cephx, none", 2024-08-17T17:12:58.566 INFO:teuthology.orchestra.run.smithi063.stdout: "auth_cluster_required": "cephx", 2024-08-17T17:12:58.566 INFO:teuthology.orchestra.run.smithi063.stdout: "auth_debug": "true", 2024-08-17T17:12:58.567 INFO:teuthology.orchestra.run.smithi063.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2024-08-17T17:12:58.567 INFO:teuthology.orchestra.run.smithi063.stdout: "auth_mon_ticket_ttl": "660.000000", 2024-08-17T17:12:58.567 INFO:teuthology.orchestra.run.smithi063.stdout: "auth_service_required": "cephx", 2024-08-17T17:12:58.567 INFO:teuthology.orchestra.run.smithi063.stdout: "auth_service_ticket_ttl": "240.000000", 2024-08-17T17:12:58.567 INFO:teuthology.orchestra.run.smithi063.stdout: "auth_supported": "", 2024-08-17T17:12:58.567 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_aio": "true", 2024-08-17T17:12:58.567 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_aio_max_queue_depth": "1024", 2024-08-17T17:12:58.567 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_aio_poll_ms": "250", 2024-08-17T17:12:58.567 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_aio_reap_max": "16", 2024-08-17T17:12:58.567 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_async_discard_threads": "0", 2024-08-17T17:12:58.567 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_block_size": "4096", 2024-08-17T17:12:58.568 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_debug_aio": "false", 2024-08-17T17:12:58.568 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_debug_aio_log_age": "5.000000", 2024-08-17T17:12:58.568 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2024-08-17T17:12:58.568 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_debug_inflight_ios": "false", 2024-08-17T17:12:58.568 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_enable_discard": "false", 2024-08-17T17:12:58.568 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_flock_retry": "3", 2024-08-17T17:12:58.568 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_flock_retry_interval": "0.100000", 2024-08-17T17:12:58.568 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_inject_crash": "0", 2024-08-17T17:12:58.568 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_inject_crash_flush_delay": "2", 2024-08-17T17:12:58.568 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_ioring": "false", 2024-08-17T17:12:58.569 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_ioring_hipri": "false", 2024-08-17T17:12:58.569 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_ioring_sqthread_poll": "false", 2024-08-17T17:12:58.569 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_nvme_unbind_from_kernel": "false", 2024-08-17T17:12:58.569 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_read_buffer_alignment": "4096", 2024-08-17T17:12:58.569 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_read_preallocated_huge_buffers": "", 2024-08-17T17:12:58.569 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_stalled_read_warn_lifetime": "86400", 2024-08-17T17:12:58.569 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_stalled_read_warn_threshold": "1", 2024-08-17T17:12:58.569 INFO:teuthology.orchestra.run.smithi063.stdout: "bdev_type": "", 2024-08-17T17:12:58.569 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_alloc_size": "1048576", 2024-08-17T17:12:58.569 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_allocator": "hybrid", 2024-08-17T17:12:58.569 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_buffered_io": "true", 2024-08-17T17:12:58.569 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_check_for_zeros": "false", 2024-08-17T17:12:58.569 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_check_volume_selector_often": "false", 2024-08-17T17:12:58.570 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_check_volume_selector_on_umount": "false", 2024-08-17T17:12:58.570 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_compact_log_sync": "false", 2024-08-17T17:12:58.570 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2024-08-17T17:12:58.570 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2024-08-17T17:12:58.570 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_log_compact_min_size": "16777216", 2024-08-17T17:12:58.570 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_log_replay_check_allocations": "true", 2024-08-17T17:12:58.570 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_max_log_runway": "4194304", 2024-08-17T17:12:58.570 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_max_prefetch": "1048576", 2024-08-17T17:12:58.570 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_min_flush_size": "524288", 2024-08-17T17:12:58.571 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_min_log_runway": "1048576", 2024-08-17T17:12:58.571 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_replay_recovery": "false", 2024-08-17T17:12:58.571 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_replay_recovery_disable_compact": "false", 2024-08-17T17:12:58.571 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_shared_alloc_size": "65536", 2024-08-17T17:12:58.571 INFO:teuthology.orchestra.run.smithi063.stdout: "bluefs_sync_write": "false", 2024-08-17T17:12:58.571 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2024-08-17T17:12:58.571 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2024-08-17T17:12:58.571 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2024-08-17T17:12:58.571 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_allocation_from_file": "true", 2024-08-17T17:12:58.571 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_allocator": "hybrid", 2024-08-17T17:12:58.571 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_async_db_compaction": "true", 2024-08-17T17:12:58.571 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2024-08-17T17:12:58.571 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2024-08-17T17:12:58.572 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2024-08-17T17:12:58.572 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2024-08-17T17:12:58.572 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_bdev_label_multi": "true", 2024-08-17T17:12:58.572 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_bdev_label_multi_upgrade": "false", 2024-08-17T17:12:58.572 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_bdev_label_require_all": "true", 2024-08-17T17:12:58.572 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2024-08-17T17:12:58.572 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_bitmapallocator_span_size": "1024", 2024-08-17T17:12:58.572 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_blobid_prealloc": "10240", 2024-08-17T17:12:58.572 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_block_create": "true", 2024-08-17T17:12:58.572 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_block_db_create": "false", 2024-08-17T17:12:58.572 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_block_db_path": "", 2024-08-17T17:12:58.573 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_block_db_size": "0", 2024-08-17T17:12:58.573 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_block_path": "", 2024-08-17T17:12:58.573 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_block_preallocate_file": "false", 2024-08-17T17:12:58.573 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_block_size": "107374182400", 2024-08-17T17:12:58.573 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_block_wal_create": "false", 2024-08-17T17:12:58.573 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_block_wal_path": "", 2024-08-17T17:12:58.573 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_block_wal_size": "100663296", 2024-08-17T17:12:58.573 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_bluefs": "true", 2024-08-17T17:12:58.573 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2024-08-17T17:12:58.573 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_bluefs_env_mirror": "false", 2024-08-17T17:12:58.573 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_bluefs_max_free": "10737418240", 2024-08-17T17:12:58.574 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_btree2_alloc_weight_factor": "2.000000", 2024-08-17T17:12:58.574 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2024-08-17T17:12:58.574 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2024-08-17T17:12:58.574 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2024-08-17T17:12:58.574 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2024-08-17T17:12:58.574 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2024-08-17T17:12:58.574 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_autotune": "true", 2024-08-17T17:12:58.574 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_autotune_interval": "5.000000", 2024-08-17T17:12:58.574 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2024-08-17T17:12:58.574 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_kv_ratio": "0.450000", 2024-08-17T17:12:58.574 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_meta_ratio": "0.450000", 2024-08-17T17:12:58.574 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_size": "0", 2024-08-17T17:12:58.575 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_size_hdd": "1073741824", 2024-08-17T17:12:58.575 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_size_ssd": "3221225472", 2024-08-17T17:12:58.575 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_trim_interval": "0.050000", 2024-08-17T17:12:58.575 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2024-08-17T17:12:58.575 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cache_type": "2q", 2024-08-17T17:12:58.575 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2024-08-17T17:12:58.575 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_clone_cow": "true", 2024-08-17T17:12:58.575 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_compression_algorithm": "snappy", 2024-08-17T17:12:58.575 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_compression_max_blob_size": "0", 2024-08-17T17:12:58.575 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2024-08-17T17:12:58.575 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2024-08-17T17:12:58.575 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_compression_min_blob_size": "0", 2024-08-17T17:12:58.575 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2024-08-17T17:12:58.576 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2024-08-17T17:12:58.576 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_compression_mode": "none", 2024-08-17T17:12:58.576 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_compression_required_ratio": "0.875000", 2024-08-17T17:12:58.576 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_csum_type": "crc32c", 2024-08-17T17:12:58.576 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_enforce_min_alloc_size": "0", 2024-08-17T17:12:58.576 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_enforce_settings": "default", 2024-08-17T17:12:58.576 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_freelist": "false", 2024-08-17T17:12:58.576 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_fsck_abort": "false", 2024-08-17T17:12:58.576 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2024-08-17T17:12:58.576 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2024-08-17T17:12:58.577 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_inject_read_err": "false", 2024-08-17T17:12:58.577 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_legacy_omap": "false", 2024-08-17T17:12:58.577 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_no_reuse_blocks": "false", 2024-08-17T17:12:58.577 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_omit_block_device_write": "false", 2024-08-17T17:12:58.577 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_omit_kv_commit": "false", 2024-08-17T17:12:58.577 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2024-08-17T17:12:58.577 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_prefragment_max": "1048576", 2024-08-17T17:12:58.577 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_random_read_err": "0.000000", 2024-08-17T17:12:58.577 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2024-08-17T17:12:58.577 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_small_allocations": "0", 2024-08-17T17:12:58.578 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2024-08-17T17:12:58.578 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_default_buffered_read": "true", 2024-08-17T17:12:58.578 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_default_buffered_write": "false", 2024-08-17T17:12:58.578 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_deferred_batch_ops": "0", 2024-08-17T17:12:58.578 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2024-08-17T17:12:58.578 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2024-08-17T17:12:58.578 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_elastic_shared_blobs": "true", 2024-08-17T17:12:58.578 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2024-08-17T17:12:58.578 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_extent_map_shard_max_size": "1200", 2024-08-17T17:12:58.578 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_extent_map_shard_min_size": "150", 2024-08-17T17:12:58.579 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_extent_map_shard_target_size": "500", 2024-08-17T17:12:58.579 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2024-08-17T17:12:58.579 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_fail_eio": "false", 2024-08-17T17:12:58.579 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_freelist_blocks_per_key": "128", 2024-08-17T17:12:58.579 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2024-08-17T17:12:58.579 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2024-08-17T17:12:58.579 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2024-08-17T17:12:58.579 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_fsck_on_mkfs": "true", 2024-08-17T17:12:58.579 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2024-08-17T17:12:58.579 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_fsck_on_mount": "false", 2024-08-17T17:12:58.580 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_fsck_on_mount_deep": "false", 2024-08-17T17:12:58.580 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_fsck_on_umount": "false", 2024-08-17T17:12:58.580 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_fsck_on_umount_deep": "false", 2024-08-17T17:12:58.580 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2024-08-17T17:12:58.580 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_fsck_quick_fix_threads": "2", 2024-08-17T17:12:58.580 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2024-08-17T17:12:58.580 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2024-08-17T17:12:58.580 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_gc_enable_blob_threshold": "0", 2024-08-17T17:12:58.580 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_gc_enable_total_threshold": "0", 2024-08-17T17:12:58.580 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2024-08-17T17:12:58.580 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_ignore_data_csum": "false", 2024-08-17T17:12:58.580 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2024-08-17T17:12:58.581 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_kvbackend": "rocksdb", 2024-08-17T17:12:58.581 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_log_collection_list_age": "60.000000", 2024-08-17T17:12:58.581 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2024-08-17T17:12:58.581 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_log_op_age": "5.000000", 2024-08-17T17:12:58.581 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_max_alloc_size": "0", 2024-08-17T17:12:58.581 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_max_blob_size": "0", 2024-08-17T17:12:58.581 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_max_blob_size_hdd": "65536", 2024-08-17T17:12:58.581 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_max_blob_size_ssd": "65536", 2024-08-17T17:12:58.581 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_max_defer_interval": "3.000000", 2024-08-17T17:12:58.581 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_max_deferred_txc": "32", 2024-08-17T17:12:58.582 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_min_alloc_size": "0", 2024-08-17T17:12:58.582 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_min_alloc_size_hdd": "4096", 2024-08-17T17:12:58.582 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_min_alloc_size_ssd": "4096", 2024-08-17T17:12:58.582 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_nid_prealloc": "1024", 2024-08-17T17:12:58.582 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_prefer_deferred_size": "0", 2024-08-17T17:12:58.582 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2024-08-17T17:12:58.582 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2024-08-17T17:12:58.582 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_qfsck_on_mount": "true", 2024-08-17T17:12:58.583 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_retry_disk_reads": "3", 2024-08-17T17:12:58.583 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_rocksdb_cf": "true", 2024-08-17T17:12:58.583 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2024-08-17T17:12:58.583 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2024-08-17T17:12:58.584 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_rocksdb_options_annex": "", 2024-08-17T17:12:58.584 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_slow_ops_warn_lifetime": "86400", 2024-08-17T17:12:58.584 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_slow_ops_warn_threshold": "1", 2024-08-17T17:12:58.584 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_spdk_coremask": "0x1", 2024-08-17T17:12:58.584 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_spdk_io_sleep": "5", 2024-08-17T17:12:58.584 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_spdk_max_io_completion": "0", 2024-08-17T17:12:58.584 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_spdk_mem": "512", 2024-08-17T17:12:58.584 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_sync_submit_transaction": "false", 2024-08-17T17:12:58.584 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_throttle_bytes": "67108864", 2024-08-17T17:12:58.584 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_throttle_cost_per_io": "0", 2024-08-17T17:12:58.585 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2024-08-17T17:12:58.585 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2024-08-17T17:12:58.585 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2024-08-17T17:12:58.585 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_throttle_trace_rate": "0.000000", 2024-08-17T17:12:58.585 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_tracing": "false", 2024-08-17T17:12:58.585 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2024-08-17T17:12:58.585 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2024-08-17T17:12:58.585 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_volume_selection_reserved": "0", 2024-08-17T17:12:58.585 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2024-08-17T17:12:58.585 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2024-08-17T17:12:58.585 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_warn_on_legacy_statfs": "true", 2024-08-17T17:12:58.585 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2024-08-17T17:12:58.586 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2024-08-17T17:12:58.586 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2024-08-17T17:12:58.586 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_write_v2": "false", 2024-08-17T17:12:58.586 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_write_v2_random": "false", 2024-08-17T17:12:58.586 INFO:teuthology.orchestra.run.smithi063.stdout: "bluestore_zero_block_detection": "false", 2024-08-17T17:12:58.586 INFO:teuthology.orchestra.run.smithi063.stdout: "cephadm_path": "/usr/sbin/cephadm", 2024-08-17T17:12:58.586 INFO:teuthology.orchestra.run.smithi063.stdout: "cephfs_mirror_action_update_interval": "2", 2024-08-17T17:12:58.586 INFO:teuthology.orchestra.run.smithi063.stdout: "cephfs_mirror_directory_scan_interval": "10", 2024-08-17T17:12:58.586 INFO:teuthology.orchestra.run.smithi063.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2024-08-17T17:12:58.586 INFO:teuthology.orchestra.run.smithi063.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2024-08-17T17:12:58.587 INFO:teuthology.orchestra.run.smithi063.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2024-08-17T17:12:58.587 INFO:teuthology.orchestra.run.smithi063.stdout: "cephfs_mirror_mount_timeout": "10", 2024-08-17T17:12:58.587 INFO:teuthology.orchestra.run.smithi063.stdout: "cephfs_mirror_perf_stats_prio": "5", 2024-08-17T17:12:58.587 INFO:teuthology.orchestra.run.smithi063.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2024-08-17T17:12:58.587 INFO:teuthology.orchestra.run.smithi063.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2024-08-17T17:12:58.587 INFO:teuthology.orchestra.run.smithi063.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2024-08-17T17:12:58.587 INFO:teuthology.orchestra.run.smithi063.stdout: "cephsqlite_blocklist_dead_locker": "true", 2024-08-17T17:12:58.587 INFO:teuthology.orchestra.run.smithi063.stdout: "cephsqlite_lock_renewal_interval": "2000", 2024-08-17T17:12:58.587 INFO:teuthology.orchestra.run.smithi063.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2024-08-17T17:12:58.587 INFO:teuthology.orchestra.run.smithi063.stdout: "cephx_cluster_require_signatures": "false", 2024-08-17T17:12:58.587 INFO:teuthology.orchestra.run.smithi063.stdout: "cephx_cluster_require_version": "2", 2024-08-17T17:12:58.588 INFO:teuthology.orchestra.run.smithi063.stdout: "cephx_require_signatures": "false", 2024-08-17T17:12:58.588 INFO:teuthology.orchestra.run.smithi063.stdout: "cephx_require_version": "2", 2024-08-17T17:12:58.588 INFO:teuthology.orchestra.run.smithi063.stdout: "cephx_service_require_signatures": "false", 2024-08-17T17:12:58.588 INFO:teuthology.orchestra.run.smithi063.stdout: "cephx_service_require_version": "2", 2024-08-17T17:12:58.588 INFO:teuthology.orchestra.run.smithi063.stdout: "cephx_sign_messages": "true", 2024-08-17T17:12:58.588 INFO:teuthology.orchestra.run.smithi063.stdout: "chdir": "", 2024-08-17T17:12:58.588 INFO:teuthology.orchestra.run.smithi063.stdout: "client_acl_type": "", 2024-08-17T17:12:58.588 INFO:teuthology.orchestra.run.smithi063.stdout: "client_asio_thread_count": "2", 2024-08-17T17:12:58.588 INFO:teuthology.orchestra.run.smithi063.stdout: "client_cache_mid": "0.750000", 2024-08-17T17:12:58.589 INFO:teuthology.orchestra.run.smithi063.stdout: "client_cache_size": "16384", 2024-08-17T17:12:58.589 INFO:teuthology.orchestra.run.smithi063.stdout: "client_caps_release_delay": "5", 2024-08-17T17:12:58.589 INFO:teuthology.orchestra.run.smithi063.stdout: "client_check_pool_perm": "true", 2024-08-17T17:12:58.589 INFO:teuthology.orchestra.run.smithi063.stdout: "client_collect_and_send_global_metrics": "false", 2024-08-17T17:12:58.589 INFO:teuthology.orchestra.run.smithi063.stdout: "client_debug_force_sync_read": "false", 2024-08-17T17:12:58.589 INFO:teuthology.orchestra.run.smithi063.stdout: "client_debug_getattr_caps": "false", 2024-08-17T17:12:58.589 INFO:teuthology.orchestra.run.smithi063.stdout: "client_debug_inject_features": "", 2024-08-17T17:12:58.589 INFO:teuthology.orchestra.run.smithi063.stdout: "client_debug_inject_tick_delay": "0", 2024-08-17T17:12:58.590 INFO:teuthology.orchestra.run.smithi063.stdout: "client_die_on_failed_dentry_invalidate": "true", 2024-08-17T17:12:58.590 INFO:teuthology.orchestra.run.smithi063.stdout: "client_die_on_failed_remount": "false", 2024-08-17T17:12:58.590 INFO:teuthology.orchestra.run.smithi063.stdout: "client_dirsize_rbytes": "true", 2024-08-17T17:12:58.590 INFO:teuthology.orchestra.run.smithi063.stdout: "client_force_lazyio": "false", 2024-08-17T17:12:58.590 INFO:teuthology.orchestra.run.smithi063.stdout: "client_fs": "", 2024-08-17T17:12:58.590 INFO:teuthology.orchestra.run.smithi063.stdout: "client_inject_fixed_oldest_tid": "false", 2024-08-17T17:12:58.590 INFO:teuthology.orchestra.run.smithi063.stdout: "client_inject_release_failure": "false", 2024-08-17T17:12:58.590 INFO:teuthology.orchestra.run.smithi063.stdout: "client_max_inline_size": "4096", 2024-08-17T17:12:58.590 INFO:teuthology.orchestra.run.smithi063.stdout: "client_max_retries_on_remount_failure": "5", 2024-08-17T17:12:58.590 INFO:teuthology.orchestra.run.smithi063.stdout: "client_mds_namespace": "", 2024-08-17T17:12:58.590 INFO:teuthology.orchestra.run.smithi063.stdout: "client_metadata": "", 2024-08-17T17:12:58.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client_mount_gid": "-1", 2024-08-17T17:12:58.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client_mount_timeout": "300", 2024-08-17T17:12:58.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client_mount_uid": "-1", 2024-08-17T17:12:58.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client_mountpoint": "/", 2024-08-17T17:12:58.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client_notify_timeout": "10", 2024-08-17T17:12:58.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client_oc": "true", 2024-08-17T17:12:58.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client_oc_max_dirty": "104857600", 2024-08-17T17:12:58.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client_oc_max_dirty_age": "5.000000", 2024-08-17T17:12:58.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client_oc_max_objects": "1000", 2024-08-17T17:12:58.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client_oc_size": "209715200", 2024-08-17T17:12:58.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client_oc_target_dirty": "8388608", 2024-08-17T17:12:58.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client_permissions": "true", 2024-08-17T17:12:58.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client_quota": "true", 2024-08-17T17:12:58.592 INFO:teuthology.orchestra.run.smithi063.stdout: "client_quota_df": "true", 2024-08-17T17:12:58.592 INFO:teuthology.orchestra.run.smithi063.stdout: "client_readahead_max_bytes": "0", 2024-08-17T17:12:58.592 INFO:teuthology.orchestra.run.smithi063.stdout: "client_readahead_max_periods": "4", 2024-08-17T17:12:58.592 INFO:teuthology.orchestra.run.smithi063.stdout: "client_readahead_min": "131072", 2024-08-17T17:12:58.592 INFO:teuthology.orchestra.run.smithi063.stdout: "client_reconnect_stale": "false", 2024-08-17T17:12:58.592 INFO:teuthology.orchestra.run.smithi063.stdout: "client_shutdown_timeout": "30", 2024-08-17T17:12:58.592 INFO:teuthology.orchestra.run.smithi063.stdout: "client_snapdir": ".snap", 2024-08-17T17:12:58.592 INFO:teuthology.orchestra.run.smithi063.stdout: "client_tick_interval": "1", 2024-08-17T17:12:58.592 INFO:teuthology.orchestra.run.smithi063.stdout: "client_trace": "", 2024-08-17T17:12:58.592 INFO:teuthology.orchestra.run.smithi063.stdout: "client_try_dentry_invalidate": "false", 2024-08-17T17:12:58.592 INFO:teuthology.orchestra.run.smithi063.stdout: "client_use_faked_inos": "false", 2024-08-17T17:12:58.592 INFO:teuthology.orchestra.run.smithi063.stdout: "client_use_random_mds": "false", 2024-08-17T17:12:58.592 INFO:teuthology.orchestra.run.smithi063.stdout: "clog_to_graylog": "false", 2024-08-17T17:12:58.593 INFO:teuthology.orchestra.run.smithi063.stdout: "clog_to_graylog_host": "127.0.0.1", 2024-08-17T17:12:58.593 INFO:teuthology.orchestra.run.smithi063.stdout: "clog_to_graylog_port": "12201", 2024-08-17T17:12:58.593 INFO:teuthology.orchestra.run.smithi063.stdout: "clog_to_monitors": "default=true", 2024-08-17T17:12:58.593 INFO:teuthology.orchestra.run.smithi063.stdout: "clog_to_syslog": "false", 2024-08-17T17:12:58.593 INFO:teuthology.orchestra.run.smithi063.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2024-08-17T17:12:58.593 INFO:teuthology.orchestra.run.smithi063.stdout: "clog_to_syslog_level": "info", 2024-08-17T17:12:58.593 INFO:teuthology.orchestra.run.smithi063.stdout: "cluster_addr": "-", 2024-08-17T17:12:58.593 INFO:teuthology.orchestra.run.smithi063.stdout: "cluster_network": "", 2024-08-17T17:12:58.593 INFO:teuthology.orchestra.run.smithi063.stdout: "cluster_network_interface": "", 2024-08-17T17:12:58.593 INFO:teuthology.orchestra.run.smithi063.stdout: "compressor_zlib_isal": "false", 2024-08-17T17:12:58.593 INFO:teuthology.orchestra.run.smithi063.stdout: "compressor_zlib_level": "5", 2024-08-17T17:12:58.593 INFO:teuthology.orchestra.run.smithi063.stdout: "compressor_zlib_winsize": "-15", 2024-08-17T17:12:58.593 INFO:teuthology.orchestra.run.smithi063.stdout: "compressor_zstd_level": "1", 2024-08-17T17:12:58.593 INFO:teuthology.orchestra.run.smithi063.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d", 2024-08-17T17:12:58.594 INFO:teuthology.orchestra.run.smithi063.stdout: "crash_dir": "/var/lib/ceph/crash", 2024-08-17T17:12:58.594 INFO:teuthology.orchestra.run.smithi063.stdout: "crimson_alien_op_num_threads": "6", 2024-08-17T17:12:58.594 INFO:teuthology.orchestra.run.smithi063.stdout: "crimson_alien_thread_cpu_cores": "", 2024-08-17T17:12:58.594 INFO:teuthology.orchestra.run.smithi063.stdout: "crimson_osd_obc_lru_size": "512", 2024-08-17T17:12:58.594 INFO:teuthology.orchestra.run.smithi063.stdout: "crimson_osd_scheduler_concurrency": "0", 2024-08-17T17:12:58.594 INFO:teuthology.orchestra.run.smithi063.stdout: "crimson_osd_stat_interval": "0", 2024-08-17T17:12:58.594 INFO:teuthology.orchestra.run.smithi063.stdout: "crimson_seastar_cpu_cores": "", 2024-08-17T17:12:58.594 INFO:teuthology.orchestra.run.smithi063.stdout: "crimson_seastar_num_threads": "0", 2024-08-17T17:12:58.594 INFO:teuthology.orchestra.run.smithi063.stdout: "crush_location": "", 2024-08-17T17:12:58.594 INFO:teuthology.orchestra.run.smithi063.stdout: "crush_location_hook": "", 2024-08-17T17:12:58.594 INFO:teuthology.orchestra.run.smithi063.stdout: "crush_location_hook_timeout": "10", 2024-08-17T17:12:58.594 INFO:teuthology.orchestra.run.smithi063.stdout: "daemonize": "false", 2024-08-17T17:12:58.595 INFO:teuthology.orchestra.run.smithi063.stdout: "daos_pool": "tank", 2024-08-17T17:12:58.595 INFO:teuthology.orchestra.run.smithi063.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2024-08-17T17:12:58.595 INFO:teuthology.orchestra.run.smithi063.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2024-08-17T17:12:58.595 INFO:teuthology.orchestra.run.smithi063.stdout: "dbstore_db_name_prefix": "dbstore", 2024-08-17T17:12:58.595 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_alienstore": "0/5", 2024-08-17T17:12:58.595 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_allow_any_pool_priority": "false", 2024-08-17T17:12:58.595 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_asok": "1/5", 2024-08-17T17:12:58.595 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_asok_assert_abort": "false", 2024-08-17T17:12:58.595 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_asserts_on_shutdown": "true", 2024-08-17T17:12:58.595 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_auth": "1/5", 2024-08-17T17:12:58.595 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_bdev": "1/3", 2024-08-17T17:12:58.595 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_bluefs": "1/5", 2024-08-17T17:12:58.595 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_bluestore": "1/5", 2024-08-17T17:12:58.595 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_buffer": "0/1", 2024-08-17T17:12:58.596 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_ceph_dedup": "0/5", 2024-08-17T17:12:58.596 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_ceph_exporter": "1/5", 2024-08-17T17:12:58.596 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_cephfs_mirror": "0/5", 2024-08-17T17:12:58.596 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_cephsqlite": "0/5", 2024-08-17T17:12:58.596 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_client": "0/5", 2024-08-17T17:12:58.596 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_compressor": "1/5", 2024-08-17T17:12:58.596 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_context": "0/1", 2024-08-17T17:12:58.596 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_crimson_interrupt": "0/5", 2024-08-17T17:12:58.596 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_crush": "1/1", 2024-08-17T17:12:58.596 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_crypto": "1/5", 2024-08-17T17:12:58.596 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_cyanstore": "0/5", 2024-08-17T17:12:58.596 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_deliberately_leak_memory": "false", 2024-08-17T17:12:58.596 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_disable_randomized_ping": "false", 2024-08-17T17:12:58.597 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_dpdk": "1/5", 2024-08-17T17:12:58.597 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_eventtrace": "1/5", 2024-08-17T17:12:58.597 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_filer": "0/1", 2024-08-17T17:12:58.597 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_filestore": "1/3", 2024-08-17T17:12:58.597 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_finisher": "1/1", 2024-08-17T17:12:58.597 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_fuse": "1/5", 2024-08-17T17:12:58.597 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_heartbeat_testing_span": "0", 2024-08-17T17:12:58.597 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_heartbeatmap": "1/5", 2024-08-17T17:12:58.597 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_immutable_obj_cache": "0/5", 2024-08-17T17:12:58.597 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_javaclient": "1/5", 2024-08-17T17:12:58.597 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_journal": "1/3", 2024-08-17T17:12:58.597 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_journaler": "0/5", 2024-08-17T17:12:58.597 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_kstore": "1/5", 2024-08-17T17:12:58.598 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_lockdep": "0/1", 2024-08-17T17:12:58.598 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_mclock": "1/5", 2024-08-17T17:12:58.598 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_mds": "1/5", 2024-08-17T17:12:58.598 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_mds_balancer": "1/5", 2024-08-17T17:12:58.598 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_mds_locker": "1/5", 2024-08-17T17:12:58.598 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_mds_log": "1/5", 2024-08-17T17:12:58.598 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_mds_log_expire": "1/5", 2024-08-17T17:12:58.598 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_mds_migrator": "1/5", 2024-08-17T17:12:58.598 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_mds_quiesce": "3/5", 2024-08-17T17:12:58.598 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_memstore": "1/5", 2024-08-17T17:12:58.598 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_mgr": "2/5", 2024-08-17T17:12:58.598 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_mgrc": "1/5", 2024-08-17T17:12:58.599 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_mon": "20/20", 2024-08-17T17:12:58.599 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_monc": "0/10", 2024-08-17T17:12:58.599 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_ms": "1/1", 2024-08-17T17:12:58.599 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_none": "0/5", 2024-08-17T17:12:58.599 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_objclass": "0/5", 2024-08-17T17:12:58.599 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_objectcacher": "0/5", 2024-08-17T17:12:58.599 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_objecter": "0/1", 2024-08-17T17:12:58.599 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_optracker": "0/5", 2024-08-17T17:12:58.599 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_osd": "1/5", 2024-08-17T17:12:58.599 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_paxos": "20/20", 2024-08-17T17:12:58.599 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_perfcounter": "1/5", 2024-08-17T17:12:58.600 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_prioritycache": "1/5", 2024-08-17T17:12:58.600 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_rados": "0/5", 2024-08-17T17:12:58.600 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_rbd": "0/5", 2024-08-17T17:12:58.600 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_rbd_mirror": "0/5", 2024-08-17T17:12:58.600 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_rbd_pwl": "0/5", 2024-08-17T17:12:58.600 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_rbd_replay": "0/5", 2024-08-17T17:12:58.600 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_refs": "0/0", 2024-08-17T17:12:58.600 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_reserver": "1/1", 2024-08-17T17:12:58.600 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_rgw": "1/5", 2024-08-17T17:12:58.600 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_rgw_access": "1/5", 2024-08-17T17:12:58.600 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_rgw_datacache": "1/5", 2024-08-17T17:12:58.600 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_rgw_dbstore": "1/5", 2024-08-17T17:12:58.600 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_rgw_flight": "1/5", 2024-08-17T17:12:58.601 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_rgw_lifecycle": "1/5", 2024-08-17T17:12:58.601 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_rgw_notification": "1/5", 2024-08-17T17:12:58.601 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_rgw_sync": "1/5", 2024-08-17T17:12:58.601 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_rocksdb": "4/5", 2024-08-17T17:12:58.601 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_seastore": "0/5", 2024-08-17T17:12:58.601 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_seastore_backref": "0/5", 2024-08-17T17:12:58.601 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_seastore_cache": "0/5", 2024-08-17T17:12:58.601 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_seastore_cleaner": "0/5", 2024-08-17T17:12:58.601 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_seastore_device": "0/5", 2024-08-17T17:12:58.601 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_seastore_epm": "0/5", 2024-08-17T17:12:58.601 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_seastore_fixedkv_tree": "0/5", 2024-08-17T17:12:58.601 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_seastore_journal": "0/5", 2024-08-17T17:12:58.602 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_seastore_lba": "0/5", 2024-08-17T17:12:58.602 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_seastore_odata": "0/5", 2024-08-17T17:12:58.602 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_seastore_omap": "0/5", 2024-08-17T17:12:58.602 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_seastore_onode": "0/5", 2024-08-17T17:12:58.602 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_seastore_t": "0/5", 2024-08-17T17:12:58.602 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_seastore_tm": "0/5", 2024-08-17T17:12:58.602 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_striper": "0/1", 2024-08-17T17:12:58.602 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_test": "0/5", 2024-08-17T17:12:58.602 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_throttle": "1/1", 2024-08-17T17:12:58.602 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_timer": "0/1", 2024-08-17T17:12:58.602 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_tp": "0/5", 2024-08-17T17:12:58.603 INFO:teuthology.orchestra.run.smithi063.stdout: "debug_trace": "1/5", 2024-08-17T17:12:58.603 INFO:teuthology.orchestra.run.smithi063.stdout: "defer_client_eviction_on_laggy_osds": "false", 2024-08-17T17:12:58.603 INFO:teuthology.orchestra.run.smithi063.stdout: "device_failure_prediction_mode": "none", 2024-08-17T17:12:58.603 INFO:teuthology.orchestra.run.smithi063.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2024-08-17T17:12:58.603 INFO:teuthology.orchestra.run.smithi063.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2024-08-17T17:12:58.603 INFO:teuthology.orchestra.run.smithi063.stdout: "err_to_graylog": "false", 2024-08-17T17:12:58.603 INFO:teuthology.orchestra.run.smithi063.stdout: "err_to_journald": "false", 2024-08-17T17:12:58.603 INFO:teuthology.orchestra.run.smithi063.stdout: "err_to_stderr": "true", 2024-08-17T17:12:58.603 INFO:teuthology.orchestra.run.smithi063.stdout: "err_to_syslog": "false", 2024-08-17T17:12:58.603 INFO:teuthology.orchestra.run.smithi063.stdout: "event_tracing": "false", 2024-08-17T17:12:58.603 INFO:teuthology.orchestra.run.smithi063.stdout: "exporter_addr": "0.0.0.0", 2024-08-17T17:12:58.603 INFO:teuthology.orchestra.run.smithi063.stdout: "exporter_cert_file": "", 2024-08-17T17:12:58.603 INFO:teuthology.orchestra.run.smithi063.stdout: "exporter_http_port": "9926", 2024-08-17T17:12:58.604 INFO:teuthology.orchestra.run.smithi063.stdout: "exporter_key_file": "", 2024-08-17T17:12:58.604 INFO:teuthology.orchestra.run.smithi063.stdout: "exporter_prio_limit": "5", 2024-08-17T17:12:58.604 INFO:teuthology.orchestra.run.smithi063.stdout: "exporter_sock_dir": "/var/run/ceph/", 2024-08-17T17:12:58.604 INFO:teuthology.orchestra.run.smithi063.stdout: "exporter_sort_metrics": "true", 2024-08-17T17:12:58.604 INFO:teuthology.orchestra.run.smithi063.stdout: "exporter_stats_period": "5", 2024-08-17T17:12:58.604 INFO:teuthology.orchestra.run.smithi063.stdout: "fake_statfs_for_testing": "0", 2024-08-17T17:12:58.604 INFO:teuthology.orchestra.run.smithi063.stdout: "fatal_signal_handlers": "true", 2024-08-17T17:12:58.604 INFO:teuthology.orchestra.run.smithi063.stdout: "filer_max_purge_ops": "10", 2024-08-17T17:12:58.604 INFO:teuthology.orchestra.run.smithi063.stdout: "filer_max_truncate_ops": "128", 2024-08-17T17:12:58.604 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_apply_finisher_threads": "1", 2024-08-17T17:12:58.604 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_blackhole": "false", 2024-08-17T17:12:58.605 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_btrfs_clone_range": "true", 2024-08-17T17:12:58.605 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_btrfs_snap": "true", 2024-08-17T17:12:58.605 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_caller_concurrency": "10", 2024-08-17T17:12:58.605 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_collect_device_partition_information": "true", 2024-08-17T17:12:58.605 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_commit_timeout": "600.000000", 2024-08-17T17:12:58.606 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_debug_inject_read_err": "false", 2024-08-17T17:12:58.606 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_debug_omap_check": "false", 2024-08-17T17:12:58.606 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_debug_verify_split": "false", 2024-08-17T17:12:58.606 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_dump_file": "", 2024-08-17T17:12:58.606 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2024-08-17T17:12:58.606 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_expected_throughput_ops": "200.000000", 2024-08-17T17:12:58.606 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_fadvise": "true", 2024-08-17T17:12:58.606 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_fail_eio": "true", 2024-08-17T17:12:58.606 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_fd_cache_shards": "16", 2024-08-17T17:12:58.606 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_fd_cache_size": "128", 2024-08-17T17:12:58.606 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_fiemap": "false", 2024-08-17T17:12:58.606 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_fiemap_threshold": "4096", 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_fsync_flushes_journal_data": "false", 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_index_retry_probability": "0.000000", 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_inject_stall": "0", 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_journal_parallel": "false", 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_journal_trailing": "false", 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_journal_writeahead": "false", 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_kill_at": "0", 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_max_alloc_hint_size": "1048576", 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_max_inline_xattr_size": "0", 2024-08-17T17:12:58.607 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_max_inline_xattr_size_other": "512", 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_max_inline_xattrs": "0", 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_max_inline_xattrs_other": "2", 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_max_inline_xattrs_xfs": "10", 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_max_sync_interval": "5.000000", 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_max_xattr_value_size": "0", 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_max_xattr_value_size_other": "1024", 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2024-08-17T17:12:58.608 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_merge_threshold": "-10", 2024-08-17T17:12:58.609 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_min_sync_interval": "0.010000", 2024-08-17T17:12:58.609 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_odsync_write": "false", 2024-08-17T17:12:58.609 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_omap_backend": "rocksdb", 2024-08-17T17:12:58.609 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_omap_backend_path": "", 2024-08-17T17:12:58.609 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_omap_header_cache_size": "1024", 2024-08-17T17:12:58.609 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_ondisk_finisher_threads": "1", 2024-08-17T17:12:58.609 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_op_thread_suicide_timeout": "180", 2024-08-17T17:12:58.609 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_op_thread_timeout": "60", 2024-08-17T17:12:58.609 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_op_threads": "2", 2024-08-17T17:12:58.609 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_punch_hole": "false", 2024-08-17T17:12:58.609 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2024-08-17T17:12:58.609 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2024-08-17T17:12:58.610 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2024-08-17T17:12:58.610 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_queue_high_threshhold": "0.900000", 2024-08-17T17:12:58.610 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_queue_low_threshhold": "0.300000", 2024-08-17T17:12:58.610 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_queue_max_bytes": "104857600", 2024-08-17T17:12:58.610 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2024-08-17T17:12:58.610 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2024-08-17T17:12:58.610 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2024-08-17T17:12:58.610 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_queue_max_ops": "50", 2024-08-17T17:12:58.610 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2024-08-17T17:12:58.610 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_seek_data_hole": "false", 2024-08-17T17:12:58.610 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_sloppy_crc": "false", 2024-08-17T17:12:58.610 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_sloppy_crc_block_size": "65536", 2024-08-17T17:12:58.611 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_splice": "false", 2024-08-17T17:12:58.611 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_split_multiple": "2", 2024-08-17T17:12:58.611 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_split_rand_factor": "20", 2024-08-17T17:12:58.611 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_update_to": "1000", 2024-08-17T17:12:58.611 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2024-08-17T17:12:58.611 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2024-08-17T17:12:58.611 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2024-08-17T17:12:58.611 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2024-08-17T17:12:58.611 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2024-08-17T17:12:58.611 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2024-08-17T17:12:58.611 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_wbthrottle_enable": "true", 2024-08-17T17:12:58.611 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2024-08-17T17:12:58.611 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2024-08-17T17:12:58.612 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2024-08-17T17:12:58.612 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2024-08-17T17:12:58.612 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2024-08-17T17:12:58.612 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2024-08-17T17:12:58.612 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_xfs_extsize": "false", 2024-08-17T17:12:58.612 INFO:teuthology.orchestra.run.smithi063.stdout: "filestore_zfs_snap": "false", 2024-08-17T17:12:58.612 INFO:teuthology.orchestra.run.smithi063.stdout: "fio_dir": "/tmp/fio", 2024-08-17T17:12:58.612 INFO:teuthology.orchestra.run.smithi063.stdout: "fsid": "321fa59e-5cb8-11ef-bccf-c7b262605968", 2024-08-17T17:12:58.612 INFO:teuthology.orchestra.run.smithi063.stdout: "fuse_allow_other": "true", 2024-08-17T17:12:58.612 INFO:teuthology.orchestra.run.smithi063.stdout: "fuse_atomic_o_trunc": "true", 2024-08-17T17:12:58.612 INFO:teuthology.orchestra.run.smithi063.stdout: "fuse_big_writes": "true", 2024-08-17T17:12:58.612 INFO:teuthology.orchestra.run.smithi063.stdout: "fuse_debug": "false", 2024-08-17T17:12:58.612 INFO:teuthology.orchestra.run.smithi063.stdout: "fuse_default_permissions": "false", 2024-08-17T17:12:58.613 INFO:teuthology.orchestra.run.smithi063.stdout: "fuse_disable_pagecache": "false", 2024-08-17T17:12:58.613 INFO:teuthology.orchestra.run.smithi063.stdout: "fuse_max_write": "0", 2024-08-17T17:12:58.613 INFO:teuthology.orchestra.run.smithi063.stdout: "fuse_multithreaded": "true", 2024-08-17T17:12:58.613 INFO:teuthology.orchestra.run.smithi063.stdout: "fuse_require_active_mds": "true", 2024-08-17T17:12:58.613 INFO:teuthology.orchestra.run.smithi063.stdout: "fuse_set_user_groups": "true", 2024-08-17T17:12:58.613 INFO:teuthology.orchestra.run.smithi063.stdout: "fuse_splice_move": "true", 2024-08-17T17:12:58.613 INFO:teuthology.orchestra.run.smithi063.stdout: "fuse_splice_read": "true", 2024-08-17T17:12:58.613 INFO:teuthology.orchestra.run.smithi063.stdout: "fuse_splice_write": "true", 2024-08-17T17:12:58.613 INFO:teuthology.orchestra.run.smithi063.stdout: "fuse_syncfs_on_mksnap": "true", 2024-08-17T17:12:58.613 INFO:teuthology.orchestra.run.smithi063.stdout: "fuse_use_invalidate_cb": "true", 2024-08-17T17:12:58.613 INFO:teuthology.orchestra.run.smithi063.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2024-08-17T17:12:58.613 INFO:teuthology.orchestra.run.smithi063.stdout: "gss_target_name": "ceph", 2024-08-17T17:12:58.613 INFO:teuthology.orchestra.run.smithi063.stdout: "heartbeat_file": "", 2024-08-17T17:12:58.613 INFO:teuthology.orchestra.run.smithi063.stdout: "heartbeat_inject_failure": "0", 2024-08-17T17:12:58.614 INFO:teuthology.orchestra.run.smithi063.stdout: "heartbeat_interval": "5", 2024-08-17T17:12:58.614 INFO:teuthology.orchestra.run.smithi063.stdout: "host": "smithi063", 2024-08-17T17:12:58.614 INFO:teuthology.orchestra.run.smithi063.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2024-08-17T17:12:58.614 INFO:teuthology.orchestra.run.smithi063.stdout: "immutable_object_cache_max_inflight_ops": "128", 2024-08-17T17:12:58.614 INFO:teuthology.orchestra.run.smithi063.stdout: "immutable_object_cache_max_size": "1073741824", 2024-08-17T17:12:58.614 INFO:teuthology.orchestra.run.smithi063.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2024-08-17T17:12:58.614 INFO:teuthology.orchestra.run.smithi063.stdout: "immutable_object_cache_qos_bps_burst": "0", 2024-08-17T17:12:58.614 INFO:teuthology.orchestra.run.smithi063.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2024-08-17T17:12:58.614 INFO:teuthology.orchestra.run.smithi063.stdout: "immutable_object_cache_qos_bps_limit": "0", 2024-08-17T17:12:58.614 INFO:teuthology.orchestra.run.smithi063.stdout: "immutable_object_cache_qos_iops_burst": "0", 2024-08-17T17:12:58.614 INFO:teuthology.orchestra.run.smithi063.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2024-08-17T17:12:58.614 INFO:teuthology.orchestra.run.smithi063.stdout: "immutable_object_cache_qos_iops_limit": "0", 2024-08-17T17:12:58.615 INFO:teuthology.orchestra.run.smithi063.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2024-08-17T17:12:58.615 INFO:teuthology.orchestra.run.smithi063.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2024-08-17T17:12:58.615 INFO:teuthology.orchestra.run.smithi063.stdout: "immutable_object_cache_watermark": "0.900000", 2024-08-17T17:12:58.615 INFO:teuthology.orchestra.run.smithi063.stdout: "inject_early_sigterm": "false", 2024-08-17T17:12:58.615 INFO:teuthology.orchestra.run.smithi063.stdout: "jaeger_agent_port": "6799", 2024-08-17T17:12:58.615 INFO:teuthology.orchestra.run.smithi063.stdout: "jaeger_tracing_enable": "false", 2024-08-17T17:12:58.615 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_aio": "true", 2024-08-17T17:12:58.615 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_align_min_size": "65536", 2024-08-17T17:12:58.615 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_block_align": "true", 2024-08-17T17:12:58.615 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_block_size": "4096", 2024-08-17T17:12:58.615 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_dio": "true", 2024-08-17T17:12:58.615 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_discard": "false", 2024-08-17T17:12:58.615 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_force_aio": "false", 2024-08-17T17:12:58.616 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_ignore_corruption": "false", 2024-08-17T17:12:58.616 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_max_write_bytes": "10485760", 2024-08-17T17:12:58.616 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_max_write_entries": "100", 2024-08-17T17:12:58.616 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_replay_from": "0", 2024-08-17T17:12:58.616 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_throttle_high_multiple": "0.000000", 2024-08-17T17:12:58.616 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_throttle_high_threshhold": "0.900000", 2024-08-17T17:12:58.616 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_throttle_low_threshhold": "0.600000", 2024-08-17T17:12:58.616 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_throttle_max_multiple": "0.000000", 2024-08-17T17:12:58.616 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_write_header_frequency": "0", 2024-08-17T17:12:58.616 INFO:teuthology.orchestra.run.smithi063.stdout: "journal_zero_on_create": "false", 2024-08-17T17:12:58.616 INFO:teuthology.orchestra.run.smithi063.stdout: "journaler_prefetch_periods": "10", 2024-08-17T17:12:58.616 INFO:teuthology.orchestra.run.smithi063.stdout: "journaler_prezero_periods": "5", 2024-08-17T17:12:58.616 INFO:teuthology.orchestra.run.smithi063.stdout: "journaler_write_head_interval": "15", 2024-08-17T17:12:58.617 INFO:teuthology.orchestra.run.smithi063.stdout: "key": "", 2024-08-17T17:12:58.617 INFO:teuthology.orchestra.run.smithi063.stdout: "keyfile": "", 2024-08-17T17:12:58.617 INFO:teuthology.orchestra.run.smithi063.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2024-08-17T17:12:58.617 INFO:teuthology.orchestra.run.smithi063.stdout: "kstore_backend": "rocksdb", 2024-08-17T17:12:58.617 INFO:teuthology.orchestra.run.smithi063.stdout: "kstore_default_stripe_size": "65536", 2024-08-17T17:12:58.617 INFO:teuthology.orchestra.run.smithi063.stdout: "kstore_fsck_on_mount": "false", 2024-08-17T17:12:58.617 INFO:teuthology.orchestra.run.smithi063.stdout: "kstore_fsck_on_mount_deep": "true", 2024-08-17T17:12:58.617 INFO:teuthology.orchestra.run.smithi063.stdout: "kstore_max_bytes": "67108864", 2024-08-17T17:12:58.617 INFO:teuthology.orchestra.run.smithi063.stdout: "kstore_max_ops": "512", 2024-08-17T17:12:58.617 INFO:teuthology.orchestra.run.smithi063.stdout: "kstore_nid_prealloc": "1024", 2024-08-17T17:12:58.617 INFO:teuthology.orchestra.run.smithi063.stdout: "kstore_onode_map_size": "1024", 2024-08-17T17:12:58.617 INFO:teuthology.orchestra.run.smithi063.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2024-08-17T17:12:58.617 INFO:teuthology.orchestra.run.smithi063.stdout: "kstore_sync_submit_transaction": "false", 2024-08-17T17:12:58.618 INFO:teuthology.orchestra.run.smithi063.stdout: "kstore_sync_transaction": "false", 2024-08-17T17:12:58.618 INFO:teuthology.orchestra.run.smithi063.stdout: "librados_thread_count": "2", 2024-08-17T17:12:58.618 INFO:teuthology.orchestra.run.smithi063.stdout: "lockdep": "false", 2024-08-17T17:12:58.618 INFO:teuthology.orchestra.run.smithi063.stdout: "lockdep_force_backtrace": "false", 2024-08-17T17:12:58.618 INFO:teuthology.orchestra.run.smithi063.stdout: "log_coarse_timestamps": "true", 2024-08-17T17:12:58.618 INFO:teuthology.orchestra.run.smithi063.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2024-08-17T17:12:58.618 INFO:teuthology.orchestra.run.smithi063.stdout: "log_flush_on_exit": "false", 2024-08-17T17:12:58.618 INFO:teuthology.orchestra.run.smithi063.stdout: "log_graylog_host": "127.0.0.1", 2024-08-17T17:12:58.618 INFO:teuthology.orchestra.run.smithi063.stdout: "log_graylog_port": "12201", 2024-08-17T17:12:58.618 INFO:teuthology.orchestra.run.smithi063.stdout: "log_max_new": "1000", 2024-08-17T17:12:58.618 INFO:teuthology.orchestra.run.smithi063.stdout: "log_max_recent": "10000", 2024-08-17T17:12:58.618 INFO:teuthology.orchestra.run.smithi063.stdout: "log_stderr_prefix": "", 2024-08-17T17:12:58.619 INFO:teuthology.orchestra.run.smithi063.stdout: "log_stop_at_utilization": "0.970000", 2024-08-17T17:12:58.619 INFO:teuthology.orchestra.run.smithi063.stdout: "log_to_file": "true", 2024-08-17T17:12:58.619 INFO:teuthology.orchestra.run.smithi063.stdout: "log_to_graylog": "false", 2024-08-17T17:12:58.619 INFO:teuthology.orchestra.run.smithi063.stdout: "log_to_journald": "false", 2024-08-17T17:12:58.619 INFO:teuthology.orchestra.run.smithi063.stdout: "log_to_stderr": "false", 2024-08-17T17:12:58.619 INFO:teuthology.orchestra.run.smithi063.stdout: "log_to_syslog": "false", 2024-08-17T17:12:58.619 INFO:teuthology.orchestra.run.smithi063.stdout: "max_rotating_auth_attempts": "10", 2024-08-17T17:12:58.619 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2024-08-17T17:12:58.619 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_action_on_write_error": "1", 2024-08-17T17:12:58.619 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_alternate_name_max": "8192", 2024-08-17T17:12:58.619 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_asio_thread_count": "2", 2024-08-17T17:12:58.619 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_export_pin": "true", 2024-08-17T17:12:58.619 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_fragment_dirs": "true", 2024-08-17T17:12:58.619 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2024-08-17T17:12:58.620 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_fragment_interval": "5", 2024-08-17T17:12:58.620 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_fragment_size_max": "100000", 2024-08-17T17:12:58.620 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_idle_threshold": "0.000000", 2024-08-17T17:12:58.620 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_interval": "10", 2024-08-17T17:12:58.620 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_max": "-1", 2024-08-17T17:12:58.620 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_max_until": "-1", 2024-08-17T17:12:58.620 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_merge_size": "50", 2024-08-17T17:12:58.620 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_midchunk": "0.300000", 2024-08-17T17:12:58.620 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_min_rebalance": "0.100000", 2024-08-17T17:12:58.620 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_min_start": "0.200000", 2024-08-17T17:12:58.620 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_minchunk": "0.001000", 2024-08-17T17:12:58.620 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_mode": "0", 2024-08-17T17:12:58.620 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_need_max": "1.200000", 2024-08-17T17:12:58.621 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_need_min": "0.800000", 2024-08-17T17:12:58.621 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_overload_epochs": "2", 2024-08-17T17:12:58.621 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_replicate_threshold": "8000.000000", 2024-08-17T17:12:58.621 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_sample_interval": "3.000000", 2024-08-17T17:12:58.621 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_split_bits": "3", 2024-08-17T17:12:58.621 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_split_rd": "25000.000000", 2024-08-17T17:12:58.621 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_split_size": "10000", 2024-08-17T17:12:58.621 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_split_wr": "10000.000000", 2024-08-17T17:12:58.621 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_target_decay": "10.000000", 2024-08-17T17:12:58.621 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2024-08-17T17:12:58.621 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_beacon_grace": "15.000000", 2024-08-17T17:12:58.621 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_beacon_interval": "4.000000", 2024-08-17T17:12:58.621 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_beacon_mon_down_grace": "60", 2024-08-17T17:12:58.622 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_cache_memory_limit": "4294967296", 2024-08-17T17:12:58.622 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_cache_mid": "0.700000", 2024-08-17T17:12:58.622 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_cache_quiesce_decay_rate": "1.000000", 2024-08-17T17:12:58.622 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_cache_quiesce_delay": "0", 2024-08-17T17:12:58.622 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_cache_quiesce_sleep": "200", 2024-08-17T17:12:58.622 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_cache_quiesce_splitauth": "true", 2024-08-17T17:12:58.622 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_cache_quiesce_threshold": "524288", 2024-08-17T17:12:58.622 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_cache_release_free_interval": "10", 2024-08-17T17:12:58.622 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_cache_reservation": "0.050000", 2024-08-17T17:12:58.622 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_cache_trim_decay_rate": "1.000000", 2024-08-17T17:12:58.622 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_cache_trim_interval": "1", 2024-08-17T17:12:58.622 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_cache_trim_threshold": "262144", 2024-08-17T17:12:58.622 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2024-08-17T17:12:58.623 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2024-08-17T17:12:58.623 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_client_delegate_inos_pct": "50", 2024-08-17T17:12:58.623 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_client_prealloc_inos": "1000", 2024-08-17T17:12:58.623 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2024-08-17T17:12:58.623 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_connect_bootstrapping": "false", 2024-08-17T17:12:58.623 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_damage_table_max_entries": "10000", 2024-08-17T17:12:58.623 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2024-08-17T17:12:58.623 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_debug_auth_pins": "false", 2024-08-17T17:12:58.623 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_debug_frag": "false", 2024-08-17T17:12:58.623 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_debug_scatterstat": "false", 2024-08-17T17:12:58.623 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_debug_subtrees": "false", 2024-08-17T17:12:58.623 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_decay_halflife": "5.000000", 2024-08-17T17:12:58.623 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_default_dir_hash": "2", 2024-08-17T17:12:58.624 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_defer_session_stale": "true", 2024-08-17T17:12:58.624 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_deny_all_reconnect": "false", 2024-08-17T17:12:58.624 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_dir_keys_per_op": "16384", 2024-08-17T17:12:58.624 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_dir_max_commit_size": "10", 2024-08-17T17:12:58.624 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_dir_max_entries": "0", 2024-08-17T17:12:58.624 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_dir_prefetch": "true", 2024-08-17T17:12:58.624 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_dirstat_min_interval": "1.000000", 2024-08-17T17:12:58.624 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_dump_cache_after_rejoin": "false", 2024-08-17T17:12:58.624 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_dump_cache_on_map": "false", 2024-08-17T17:12:58.624 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_dump_cache_threshold_file": "0", 2024-08-17T17:12:58.624 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2024-08-17T17:12:58.624 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_early_reply": "true", 2024-08-17T17:12:58.624 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_enable_op_tracker": "true", 2024-08-17T17:12:58.625 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_enforce_unique_name": "true", 2024-08-17T17:12:58.625 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_export_ephemeral_distributed": "true", 2024-08-17T17:12:58.625 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2024-08-17T17:12:58.625 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_export_ephemeral_random": "true", 2024-08-17T17:12:58.625 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_export_ephemeral_random_max": "0.010000", 2024-08-17T17:12:58.625 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_extraordinary_events_dump_interval": "0", 2024-08-17T17:12:58.625 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_forward_all_requests_to_auth": "false", 2024-08-17T17:12:58.625 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_freeze_tree_timeout": "30.000000", 2024-08-17T17:12:58.625 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_fscrypt_last_block_max_size": "4096", 2024-08-17T17:12:58.625 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_go_bad_corrupt_dentry": "true", 2024-08-17T17:12:58.625 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2024-08-17T17:12:58.625 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_health_cache_threshold": "1.500000", 2024-08-17T17:12:58.625 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_health_summarize_threshold": "10", 2024-08-17T17:12:58.626 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_heartbeat_grace": "15.000000", 2024-08-17T17:12:58.626 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_heartbeat_reset_grace": "1000", 2024-08-17T17:12:58.626 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_inject_health_dummy": "false", 2024-08-17T17:12:58.626 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2024-08-17T17:12:58.626 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_inject_migrator_session_race": "false", 2024-08-17T17:12:58.626 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2024-08-17T17:12:58.626 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_inject_skip_replaying_inotable": "false", 2024-08-17T17:12:58.626 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2024-08-17T17:12:58.626 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_join_fs": "", 2024-08-17T17:12:58.626 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_journal_format": "1", 2024-08-17T17:12:58.626 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_kill_after_journal_logs_flushed": "false", 2024-08-17T17:12:58.626 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_kill_create_at": "0", 2024-08-17T17:12:58.626 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_kill_dirfrag_at": "0", 2024-08-17T17:12:58.627 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_kill_export_at": "0", 2024-08-17T17:12:58.627 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_kill_import_at": "0", 2024-08-17T17:12:58.627 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_kill_journal_at": "0", 2024-08-17T17:12:58.627 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_kill_journal_expire_at": "0", 2024-08-17T17:12:58.627 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_kill_journal_replay_at": "0", 2024-08-17T17:12:58.627 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_kill_link_at": "0", 2024-08-17T17:12:58.627 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_kill_mdstable_at": "0", 2024-08-17T17:12:58.627 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_kill_openc_at": "0", 2024-08-17T17:12:58.627 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_kill_rename_at": "0", 2024-08-17T17:12:58.627 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_kill_shutdown_at": "0", 2024-08-17T17:12:58.627 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_log_event_large_threshold": "524288", 2024-08-17T17:12:58.627 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_log_events_per_segment": "1024", 2024-08-17T17:12:58.627 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_log_major_segment_event_ratio": "12", 2024-08-17T17:12:58.628 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_log_max_events": "-1", 2024-08-17T17:12:58.628 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_log_max_segments": "128", 2024-08-17T17:12:58.628 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_log_pause": "false", 2024-08-17T17:12:58.628 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_log_segment_size": "0", 2024-08-17T17:12:58.628 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_log_skip_corrupt_events": "false", 2024-08-17T17:12:58.628 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_log_skip_unbounded_events": "false", 2024-08-17T17:12:58.628 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_log_trim_decay_rate": "1.000000", 2024-08-17T17:12:58.628 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_log_trim_threshold": "128", 2024-08-17T17:12:58.628 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_log_trim_upkeep_interval": "1000", 2024-08-17T17:12:58.628 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_log_warn_factor": "2.000000", 2024-08-17T17:12:58.628 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_max_caps_per_client": "1048576", 2024-08-17T17:12:58.628 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_max_completed_flushes": "100000", 2024-08-17T17:12:58.628 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_max_completed_requests": "100000", 2024-08-17T17:12:58.629 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_max_export_size": "20971520", 2024-08-17T17:12:58.629 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_max_file_recover": "32", 2024-08-17T17:12:58.629 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_max_purge_files": "64", 2024-08-17T17:12:58.629 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_max_purge_ops": "8192", 2024-08-17T17:12:58.629 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2024-08-17T17:12:58.629 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_max_scrub_ops_in_progress": "5", 2024-08-17T17:12:58.629 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_max_snaps_per_dir": "100", 2024-08-17T17:12:58.629 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_metrics_update_interval": "2", 2024-08-17T17:12:58.629 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_min_caps_per_client": "100", 2024-08-17T17:12:58.629 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_min_caps_working_set": "10000", 2024-08-17T17:12:58.629 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_mon_shutdown_timeout": "5.000000", 2024-08-17T17:12:58.629 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_numa_node": "-1", 2024-08-17T17:12:58.629 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_oft_prefetch_dirfrags": "false", 2024-08-17T17:12:58.630 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_op_complaint_time": "30.000000", 2024-08-17T17:12:58.630 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_op_history_duration": "600", 2024-08-17T17:12:58.630 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_op_history_size": "20", 2024-08-17T17:12:58.630 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_op_history_slow_op_size": "20", 2024-08-17T17:12:58.630 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_op_history_slow_op_threshold": "10", 2024-08-17T17:12:58.630 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_op_log_threshold": "5", 2024-08-17T17:12:58.630 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_ping_grace": "15", 2024-08-17T17:12:58.630 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_ping_interval": "5", 2024-08-17T17:12:58.630 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2024-08-17T17:12:58.630 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_recall_global_max_decay_threshold": "131072", 2024-08-17T17:12:58.630 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_recall_max_caps": "30000", 2024-08-17T17:12:58.630 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_recall_max_decay_rate": "1.500000", 2024-08-17T17:12:58.630 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_recall_max_decay_threshold": "131072", 2024-08-17T17:12:58.631 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_recall_warning_decay_rate": "60.000000", 2024-08-17T17:12:58.631 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_recall_warning_threshold": "262144", 2024-08-17T17:12:58.631 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_reconnect_timeout": "45.000000", 2024-08-17T17:12:58.631 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_replay_interval": "1.000000", 2024-08-17T17:12:58.631 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_replay_unsafe_with_closed_session": "false", 2024-08-17T17:12:58.631 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_request_load_average_decay_rate": "60.000000", 2024-08-17T17:12:58.631 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_root_ino_gid": "0", 2024-08-17T17:12:58.631 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_root_ino_uid": "0", 2024-08-17T17:12:58.631 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_scatter_nudge_interval": "5.000000", 2024-08-17T17:12:58.631 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_server_dispatch_client_request_delay": "0", 2024-08-17T17:12:58.631 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_server_dispatch_killpoint_random": "0.000000", 2024-08-17T17:12:58.631 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_session_blocklist_on_evict": "true", 2024-08-17T17:12:58.631 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_session_blocklist_on_timeout": "true", 2024-08-17T17:12:58.633 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2024-08-17T17:12:58.633 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_session_cache_liveness_magnitude": "10", 2024-08-17T17:12:58.633 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2024-08-17T17:12:58.633 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_session_cap_acquisition_throttle": "100000", 2024-08-17T17:12:58.633 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2024-08-17T17:12:58.633 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_session_metadata_threshold": "16777216", 2024-08-17T17:12:58.633 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_sessionmap_keys_per_op": "1024", 2024-08-17T17:12:58.633 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_shutdown_check": "0", 2024-08-17T17:12:58.634 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_skip_ino": "0", 2024-08-17T17:12:58.634 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_sleep_rank_change": "0.000000", 2024-08-17T17:12:58.634 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_snap_max_uid": "4294967294", 2024-08-17T17:12:58.634 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_snap_min_uid": "0", 2024-08-17T17:12:58.634 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_snap_rstat": "false", 2024-08-17T17:12:58.634 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_standby_replay_damaged": "false", 2024-08-17T17:12:58.634 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_symlink_recovery": "true", 2024-08-17T17:12:58.634 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_task_status_update_interval": "2.000000", 2024-08-17T17:12:58.634 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_thrash_exports": "0", 2024-08-17T17:12:58.634 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_thrash_fragments": "0", 2024-08-17T17:12:58.634 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_tick_interval": "5.000000", 2024-08-17T17:12:58.634 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_valgrind_exit": "false", 2024-08-17T17:12:58.635 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_verify_backtrace": "1", 2024-08-17T17:12:58.635 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_verify_scatter": "false", 2024-08-17T17:12:58.635 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_wipe_ino_prealloc": "false", 2024-08-17T17:12:58.635 INFO:teuthology.orchestra.run.smithi063.stdout: "mds_wipe_sessions": "false", 2024-08-17T17:12:58.635 INFO:teuthology.orchestra.run.smithi063.stdout: "mempool_debug": "false", 2024-08-17T17:12:58.635 INFO:teuthology.orchestra.run.smithi063.stdout: "memstore_debug_omit_block_device_write": "false", 2024-08-17T17:12:58.635 INFO:teuthology.orchestra.run.smithi063.stdout: "memstore_device_bytes": "1073741824", 2024-08-17T17:12:58.635 INFO:teuthology.orchestra.run.smithi063.stdout: "memstore_page_set": "false", 2024-08-17T17:12:58.635 INFO:teuthology.orchestra.run.smithi063.stdout: "memstore_page_size": "65536", 2024-08-17T17:12:58.635 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_client_bytes": "134217728", 2024-08-17T17:12:58.635 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_client_messages": "512", 2024-08-17T17:12:58.635 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2024-08-17T17:12:58.635 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_connect_retry_interval": "1.000000", 2024-08-17T17:12:58.636 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2024-08-17T17:12:58.636 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2024-08-17T17:12:58.636 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_disabled_modules": "", 2024-08-17T17:12:58.636 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_enable_op_tracker": "true", 2024-08-17T17:12:58.636 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_initial_modules": "restful iostat nfs", 2024-08-17T17:12:58.636 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_max_pg_creating": "1024", 2024-08-17T17:12:58.636 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_max_pg_num_change": "128", 2024-08-17T17:12:58.636 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_mds_bytes": "134217728", 2024-08-17T17:12:58.636 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_mds_messages": "128", 2024-08-17T17:12:58.636 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2024-08-17T17:12:58.637 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_mon_bytes": "134217728", 2024-08-17T17:12:58.637 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_mon_messages": "128", 2024-08-17T17:12:58.637 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_num_op_tracker_shard": "32", 2024-08-17T17:12:58.637 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_op_complaint_time": "30.000000", 2024-08-17T17:12:58.637 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_op_history_duration": "600", 2024-08-17T17:12:58.637 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_op_history_size": "20", 2024-08-17T17:12:58.637 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_op_history_slow_op_size": "20", 2024-08-17T17:12:58.637 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_op_history_slow_op_threshold": "10.000000", 2024-08-17T17:12:58.637 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_op_log_threshold": "5", 2024-08-17T17:12:58.637 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_osd_bytes": "536870912", 2024-08-17T17:12:58.637 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_osd_messages": "8192", 2024-08-17T17:12:58.637 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_pool": "true", 2024-08-17T17:12:58.637 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_service_beacon_grace": "60.000000", 2024-08-17T17:12:58.638 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_standby_modules": "true", 2024-08-17T17:12:58.638 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_stats_period": "5", 2024-08-17T17:12:58.638 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_stats_threshold": "5", 2024-08-17T17:12:58.638 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_tick_period": "2", 2024-08-17T17:12:58.638 INFO:teuthology.orchestra.run.smithi063.stdout: "mgr_ttl_cache_expire_seconds": "0", 2024-08-17T17:12:58.638 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_accept_timeout_factor": "2.000000", 2024-08-17T17:12:58.638 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_allow_pool_delete": "true", 2024-08-17T17:12:58.638 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_allow_pool_size_one": "false", 2024-08-17T17:12:58.638 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_auth_validate_all_caps": "true", 2024-08-17T17:12:58.638 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2024-08-17T17:12:58.638 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2024-08-17T17:12:58.638 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_client_bytes": "104857600", 2024-08-17T17:12:58.638 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_client_directed_command_retry": "2", 2024-08-17T17:12:58.639 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_client_hunt_interval": "3.000000", 2024-08-17T17:12:58.639 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2024-08-17T17:12:58.639 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2024-08-17T17:12:58.639 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2024-08-17T17:12:58.639 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_client_hunt_parallel": "3", 2024-08-17T17:12:58.639 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_client_log_interval": "1.000000", 2024-08-17T17:12:58.639 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_client_max_log_entries_per_message": "1000", 2024-08-17T17:12:58.639 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_client_ping_interval": "10.000000", 2024-08-17T17:12:58.639 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_client_ping_timeout": "30.000000", 2024-08-17T17:12:58.639 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_client_target_rank": "-1", 2024-08-17T17:12:58.639 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_clock_drift_allowed": "1.000000", 2024-08-17T17:12:58.639 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2024-08-17T17:12:58.639 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2024-08-17T17:12:58.640 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_cluster_log_level": "debug", 2024-08-17T17:12:58.640 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_cluster_log_to_file": "true", 2024-08-17T17:12:58.640 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_cluster_log_to_graylog": "false", 2024-08-17T17:12:58.640 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2024-08-17T17:12:58.640 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_cluster_log_to_graylog_port": "12201", 2024-08-17T17:12:58.640 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_cluster_log_to_journald": "true", 2024-08-17T17:12:58.640 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_cluster_log_to_stderr": "false", 2024-08-17T17:12:58.640 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_cluster_log_to_syslog": "default=false", 2024-08-17T17:12:58.640 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2024-08-17T17:12:58.640 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_compact_on_bootstrap": "false", 2024-08-17T17:12:58.640 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_compact_on_start": "false", 2024-08-17T17:12:58.640 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_compact_on_trim": "true", 2024-08-17T17:12:58.641 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_con_tracker_persist_interval": "10", 2024-08-17T17:12:58.641 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_con_tracker_score_halflife": "43200", 2024-08-17T17:12:58.641 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_config_key_max_entry_size": "65536", 2024-08-17T17:12:58.641 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_cpu_threads": "4", 2024-08-17T17:12:58.641 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_crush_min_required_version": "hammer", 2024-08-17T17:12:58.641 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_daemon_bytes": "419430400", 2024-08-17T17:12:58.641 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2024-08-17T17:12:58.641 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_data_avail_crit": "5", 2024-08-17T17:12:58.641 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_data_avail_warn": "5", 2024-08-17T17:12:58.641 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_data_size_warn": "16106127360", 2024-08-17T17:12:58.641 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_debug_block_osdmap_trim": "false", 2024-08-17T17:12:58.641 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_debug_deprecated_as_obsolete": "false", 2024-08-17T17:12:58.641 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_debug_dump_json": "false", 2024-08-17T17:12:58.642 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2024-08-17T17:12:58.642 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_debug_dump_transactions": "false", 2024-08-17T17:12:58.642 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_debug_extra_checks": "false", 2024-08-17T17:12:58.642 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_debug_no_initial_persistent_features": "false", 2024-08-17T17:12:58.642 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2024-08-17T17:12:58.642 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_debug_no_require_reef": "false", 2024-08-17T17:12:58.642 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_debug_no_require_squid": "false", 2024-08-17T17:12:58.642 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2024-08-17T17:12:58.642 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_delta_reset_interval": "10.000000", 2024-08-17T17:12:58.642 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_dns_srv_name": "ceph-mon", 2024-08-17T17:12:58.642 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_down_mkfs_grace": "60", 2024-08-17T17:12:58.642 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_down_uptime_grace": "60", 2024-08-17T17:12:58.643 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_election_default_strategy": "1", 2024-08-17T17:12:58.643 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_election_timeout": "5.000000", 2024-08-17T17:12:58.643 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2024-08-17T17:12:58.643 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_elector_ping_divisor": "2", 2024-08-17T17:12:58.643 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_elector_ping_timeout": "2.000000", 2024-08-17T17:12:58.643 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_enable_op_tracker": "true", 2024-08-17T17:12:58.643 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_fake_pool_delete": "false", 2024-08-17T17:12:58.643 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_force_quorum_join": "false", 2024-08-17T17:12:58.643 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_fsmap_prune_threshold": "300", 2024-08-17T17:12:58.643 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_globalid_prealloc": "10000", 2024-08-17T17:12:58.643 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_health_detail_to_clog": "true", 2024-08-17T17:12:58.643 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_health_log_update_period": "5", 2024-08-17T17:12:58.643 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_health_max_detail": "50", 2024-08-17T17:12:58.644 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_health_to_clog": "true", 2024-08-17T17:12:58.644 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_health_to_clog_interval": "600", 2024-08-17T17:12:58.644 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2024-08-17T17:12:58.644 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_host": "[v2:172.21.15.63:3300/0,v1:172.21.15.63:6789/0] [v2:172.21.15.162:3300/0,v1:172.21.15.162:6789/0] [v2:172.21.15.63:3301/0,v1:172.21.15.63:6790/0]", 2024-08-17T17:12:58.644 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_host_override": "", 2024-08-17T17:12:58.644 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_initial_members": "", 2024-08-17T17:12:58.644 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2024-08-17T17:12:58.644 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2024-08-17T17:12:58.644 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_inject_transaction_delay_max": "10.000000", 2024-08-17T17:12:58.644 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2024-08-17T17:12:58.644 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_keyvaluedb": "rocksdb", 2024-08-17T17:12:58.644 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_lease": "5.000000", 2024-08-17T17:12:58.645 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2024-08-17T17:12:58.645 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_lease_renew_interval_factor": "0.600000", 2024-08-17T17:12:58.645 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_log_full_interval": "50", 2024-08-17T17:12:58.645 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_log_max": "10000", 2024-08-17T17:12:58.645 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_log_max_summary": "50", 2024-08-17T17:12:58.645 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_max_log_entries_per_event": "4096", 2024-08-17T17:12:58.645 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_max_log_epochs": "500", 2024-08-17T17:12:58.645 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_max_mdsmap_epochs": "500", 2024-08-17T17:12:58.645 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_max_mgrmap_epochs": "500", 2024-08-17T17:12:58.645 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_max_nvmeof_epochs": "500", 2024-08-17T17:12:58.645 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_max_osd": "10000", 2024-08-17T17:12:58.645 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_max_pg_per_osd": "10000", 2024-08-17T17:12:58.645 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_max_pool_pg_num": "65536", 2024-08-17T17:12:58.646 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_max_snap_prune_per_epoch": "100", 2024-08-17T17:12:58.646 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_mds_blocklist_interval": "86400.000000", 2024-08-17T17:12:58.646 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_mds_force_trim_to": "0", 2024-08-17T17:12:58.646 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_mds_skip_sanity": "true", 2024-08-17T17:12:58.646 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_memory_autotune": "true", 2024-08-17T17:12:58.646 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_memory_target": "2147483648", 2024-08-17T17:12:58.646 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_mgr_beacon_grace": "30", 2024-08-17T17:12:58.646 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2024-08-17T17:12:58.646 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_mgr_digest_period": "5", 2024-08-17T17:12:58.646 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_mgr_inactive_grace": "60", 2024-08-17T17:12:58.646 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_mgr_mkfs_grace": "240", 2024-08-17T17:12:58.646 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2024-08-17T17:12:58.647 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_min_osdmap_epochs": "500", 2024-08-17T17:12:58.647 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_nvmeofgw_beacon_grace": "10", 2024-08-17T17:12:58.647 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_nvmeofgw_set_group_id_retry": "1000", 2024-08-17T17:12:58.647 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_op_complaint_time": "30", 2024-08-17T17:12:58.647 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_op_history_duration": "600", 2024-08-17T17:12:58.647 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_op_history_size": "20", 2024-08-17T17:12:58.647 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_op_history_slow_op_size": "20", 2024-08-17T17:12:58.647 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_op_history_slow_op_threshold": "10", 2024-08-17T17:12:58.647 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_op_log_threshold": "5", 2024-08-17T17:12:58.647 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_adjust_down_out_interval": "true", 2024-08-17T17:12:58.647 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2024-08-17T17:12:58.647 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2024-08-17T17:12:58.647 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_auto_mark_in": "false", 2024-08-17T17:12:58.648 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_auto_mark_new_in": "true", 2024-08-17T17:12:58.648 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2024-08-17T17:12:58.648 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2024-08-17T17:12:58.648 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_cache_size": "500", 2024-08-17T17:12:58.648 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_cache_size_min": "134217728", 2024-08-17T17:12:58.648 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_crush_smoke_test": "true", 2024-08-17T17:12:58.648 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_destroyed_out_interval": "600", 2024-08-17T17:12:58.648 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_down_out_interval": "600", 2024-08-17T17:12:58.648 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_down_out_subtree_limit": "rack", 2024-08-17T17:12:58.648 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2024-08-17T17:12:58.648 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_force_trim_to": "0", 2024-08-17T17:12:58.648 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_full_ratio": "0.950000", 2024-08-17T17:12:58.648 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2024-08-17T17:12:58.648 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_laggy_halflife": "3600", 2024-08-17T17:12:58.649 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_laggy_max_interval": "300", 2024-08-17T17:12:58.649 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_laggy_weight": "0.300000", 2024-08-17T17:12:58.649 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2024-08-17T17:12:58.649 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_max_initial_pgs": "1024", 2024-08-17T17:12:58.649 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_min_down_reporters": "2", 2024-08-17T17:12:58.649 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_min_in_ratio": "0.750000", 2024-08-17T17:12:58.649 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_min_up_ratio": "0.300000", 2024-08-17T17:12:58.649 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_nearfull_ratio": "0.850000", 2024-08-17T17:12:58.649 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_prime_pg_temp": "true", 2024-08-17T17:12:58.649 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2024-08-17T17:12:58.649 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2024-08-17T17:12:58.649 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_report_timeout": "900", 2024-08-17T17:12:58.649 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_reporter_subtree_level": "osd", 2024-08-17T17:12:58.651 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2024-08-17T17:12:58.651 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_warn_num_repaired": "10", 2024-08-17T17:12:58.651 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osd_warn_op_age": "32.000000", 2024-08-17T17:12:58.651 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osdmap_full_prune_enabled": "true", 2024-08-17T17:12:58.651 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osdmap_full_prune_interval": "10", 2024-08-17T17:12:58.651 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osdmap_full_prune_min": "10000", 2024-08-17T17:12:58.651 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_osdmap_full_prune_txsize": "100", 2024-08-17T17:12:58.651 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2024-08-17T17:12:58.651 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_pg_stuck_threshold": "60", 2024-08-17T17:12:58.652 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2024-08-17T17:12:58.652 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_pg_warn_min_objects": "10000", 2024-08-17T17:12:58.652 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_pg_warn_min_per_osd": "0", 2024-08-17T17:12:58.652 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_pg_warn_min_pool_objects": "1000", 2024-08-17T17:12:58.652 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_pool_quota_crit_threshold": "0", 2024-08-17T17:12:58.652 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_pool_quota_warn_threshold": "0", 2024-08-17T17:12:58.652 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_probe_timeout": "2.000000", 2024-08-17T17:12:58.652 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_reweight_max_change": "0.050000", 2024-08-17T17:12:58.652 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_reweight_max_osds": "4", 2024-08-17T17:12:58.652 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_reweight_min_bytes_per_osd": "10", 2024-08-17T17:12:58.652 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_reweight_min_pgs_per_osd": "4", 2024-08-17T17:12:58.652 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2024-08-17T17:12:58.652 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2024-08-17T17:12:58.653 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2024-08-17T17:12:58.653 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_scrub_interval": "86400", 2024-08-17T17:12:58.653 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_scrub_max_keys": "100", 2024-08-17T17:12:58.653 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_scrub_timeout": "300", 2024-08-17T17:12:58.653 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_session_timeout": "300", 2024-08-17T17:12:58.653 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_smart_report_timeout": "5", 2024-08-17T17:12:58.653 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_stat_smooth_intervals": "6", 2024-08-17T17:12:58.653 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2024-08-17T17:12:58.653 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_stretch_pool_min_size": "2", 2024-08-17T17:12:58.653 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_stretch_pool_size": "4", 2024-08-17T17:12:58.653 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2024-08-17T17:12:58.653 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_subscribe_interval": "86400.000000", 2024-08-17T17:12:58.653 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_sync_debug": "false", 2024-08-17T17:12:58.654 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_sync_max_payload_keys": "2000", 2024-08-17T17:12:58.654 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_sync_max_payload_size": "1048576", 2024-08-17T17:12:58.654 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_sync_provider_kill_at": "0", 2024-08-17T17:12:58.654 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_sync_requester_kill_at": "0", 2024-08-17T17:12:58.654 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_sync_timeout": "60.000000", 2024-08-17T17:12:58.654 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_target_pg_per_osd": "100", 2024-08-17T17:12:58.654 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_tick_interval": "5", 2024-08-17T17:12:58.654 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_timecheck_interval": "300.000000", 2024-08-17T17:12:58.654 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_timecheck_skew_interval": "30.000000", 2024-08-17T17:12:58.654 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_use_min_delay_socket": "false", 2024-08-17T17:12:58.654 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_older_version_delay": "604800", 2024-08-17T17:12:58.654 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2024-08-17T17:12:58.655 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2024-08-17T17:12:58.655 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2024-08-17T17:12:58.655 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_filestore_osds": "true", 2024-08-17T17:12:58.655 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2024-08-17T17:12:58.655 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2024-08-17T17:12:58.655 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2024-08-17T17:12:58.655 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_misplaced": "false", 2024-08-17T17:12:58.655 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2024-08-17T17:12:58.655 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_older_version": "true", 2024-08-17T17:12:58.655 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2024-08-17T17:12:58.655 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_pool_no_app": "true", 2024-08-17T17:12:58.655 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_pool_no_redundancy": "true", 2024-08-17T17:12:58.655 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2024-08-17T17:12:58.656 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2024-08-17T17:12:58.656 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2024-08-17T17:12:58.656 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_on_too_few_osds": "false", 2024-08-17T17:12:58.656 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2024-08-17T17:12:58.656 INFO:teuthology.orchestra.run.smithi063.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2024-08-17T17:12:58.656 INFO:teuthology.orchestra.run.smithi063.stdout: "monmap": "", 2024-08-17T17:12:58.656 INFO:teuthology.orchestra.run.smithi063.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-08-17T17:12:58.656 INFO:teuthology.orchestra.run.smithi063.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2024-08-17T17:12:58.656 INFO:teuthology.orchestra.run.smithi063.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2024-08-17T17:12:58.656 INFO:teuthology.orchestra.run.smithi063.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-08-17T17:12:58.656 INFO:teuthology.orchestra.run.smithi063.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2024-08-17T17:12:58.656 INFO:teuthology.orchestra.run.smithi063.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2024-08-17T17:12:58.656 INFO:teuthology.orchestra.run.smithi063.stdout: "motr_tracing_enabled": "false", 2024-08-17T17:12:58.657 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_op_threads": "3", 2024-08-17T17:12:58.657 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_buffer_size": "131072", 2024-08-17T17:12:58.657 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_cm": "false", 2024-08-17T17:12:58.657 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_device_name": "", 2024-08-17T17:12:58.657 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_dscp": "96", 2024-08-17T17:12:58.657 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_enable_hugepage": "false", 2024-08-17T17:12:58.657 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_gid_idx": "0", 2024-08-17T17:12:58.657 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_local_gid": "", 2024-08-17T17:12:58.657 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_polling_us": "1000", 2024-08-17T17:12:58.657 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_port_num": "1", 2024-08-17T17:12:58.657 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_receive_buffers": "32768", 2024-08-17T17:12:58.657 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_receive_queue_len": "4096", 2024-08-17T17:12:58.657 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_roce_ver": "1", 2024-08-17T17:12:58.657 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_send_buffers": "1024", 2024-08-17T17:12:58.658 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_sl": "3", 2024-08-17T17:12:58.658 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_support_srq": "true", 2024-08-17T17:12:58.658 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_rdma_type": "ib", 2024-08-17T17:12:58.658 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_async_reap_threshold": "5", 2024-08-17T17:12:58.658 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_bind_before_connect": "false", 2024-08-17T17:12:58.658 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_bind_ipv4": "true", 2024-08-17T17:12:58.658 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_bind_ipv6": "false", 2024-08-17T17:12:58.658 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_bind_msgr1": "true", 2024-08-17T17:12:58.658 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_bind_msgr2": "true", 2024-08-17T17:12:58.658 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_bind_port_max": "7568", 2024-08-17T17:12:58.658 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_bind_port_min": "6800", 2024-08-17T17:12:58.658 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_bind_prefer_ipv4": "false", 2024-08-17T17:12:58.658 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_bind_retry_count": "3", 2024-08-17T17:12:58.659 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_bind_retry_delay": "5", 2024-08-17T17:12:58.659 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_blackhole_client": "false", 2024-08-17T17:12:58.659 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_blackhole_mds": "false", 2024-08-17T17:12:58.659 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_blackhole_mgr": "false", 2024-08-17T17:12:58.659 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_blackhole_mon": "false", 2024-08-17T17:12:58.659 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_blackhole_osd": "false", 2024-08-17T17:12:58.659 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_client_mode": "crc secure", 2024-08-17T17:12:58.659 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_client_throttle_retry_time_interval": "5000", 2024-08-17T17:12:58.659 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_cluster_mode": "crc secure", 2024-08-17T17:12:58.659 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_cluster_type": "", 2024-08-17T17:12:58.659 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_compress_secure": "false", 2024-08-17T17:12:58.659 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_connection_idle_timeout": "900", 2024-08-17T17:12:58.659 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_connection_ready_timeout": "10", 2024-08-17T17:12:58.660 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_crc_data": "true", 2024-08-17T17:12:58.660 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_crc_header": "true", 2024-08-17T17:12:58.660 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_die_on_bad_msg": "false", 2024-08-17T17:12:58.660 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_die_on_bug": "true", 2024-08-17T17:12:58.660 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_die_on_old_message": "true", 2024-08-17T17:12:58.660 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_die_on_skipped_message": "false", 2024-08-17T17:12:58.660 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_die_on_unhandled_msg": "false", 2024-08-17T17:12:58.660 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dispatch_throttle_bytes": "104857600", 2024-08-17T17:12:58.660 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dpdk_coremask": "0xF", 2024-08-17T17:12:58.660 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dpdk_debug_allow_loopback": "false", 2024-08-17T17:12:58.660 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dpdk_devs_allowlist": "", 2024-08-17T17:12:58.660 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dpdk_enable_tso": "true", 2024-08-17T17:12:58.660 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2024-08-17T17:12:58.661 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dpdk_host_ipv4_addr": "", 2024-08-17T17:12:58.661 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dpdk_hugepages": "", 2024-08-17T17:12:58.661 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dpdk_hw_flow_control": "true", 2024-08-17T17:12:58.661 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2024-08-17T17:12:58.661 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dpdk_lro": "true", 2024-08-17T17:12:58.661 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dpdk_memory_channel": "4", 2024-08-17T17:12:58.661 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2024-08-17T17:12:58.661 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dpdk_pmd": "", 2024-08-17T17:12:58.661 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dpdk_port_id": "0", 2024-08-17T17:12:58.661 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2024-08-17T17:12:58.661 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dump_corrupt_message_level": "1", 2024-08-17T17:12:58.661 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_dump_on_send": "false", 2024-08-17T17:12:58.661 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_initial_backoff": "0.200000", 2024-08-17T17:12:58.662 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_inject_delay_max": "1.000000", 2024-08-17T17:12:58.662 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_inject_delay_probability": "0.000000", 2024-08-17T17:12:58.662 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_inject_delay_type": "", 2024-08-17T17:12:58.662 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_inject_internal_delays": "0.000000", 2024-08-17T17:12:58.662 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_inject_network_congestion": "0", 2024-08-17T17:12:58.662 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_inject_socket_failures": "0", 2024-08-17T17:12:58.662 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_learn_addr_from_peer": "true", 2024-08-17T17:12:58.662 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_max_accept_failures": "4", 2024-08-17T17:12:58.662 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_max_backoff": "15.000000", 2024-08-17T17:12:58.662 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_mon_client_mode": "secure crc", 2024-08-17T17:12:58.662 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_mon_cluster_mode": "secure crc", 2024-08-17T17:12:58.662 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_mon_service_mode": "secure crc", 2024-08-17T17:12:58.662 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_osd_compress_min_size": "1024", 2024-08-17T17:12:58.663 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_osd_compress_mode": "none", 2024-08-17T17:12:58.663 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_osd_compression_algorithm": "snappy", 2024-08-17T17:12:58.663 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2024-08-17T17:12:58.663 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_pq_min_cost": "65536", 2024-08-17T17:12:58.663 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_public_type": "", 2024-08-17T17:12:58.663 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_service_mode": "crc secure", 2024-08-17T17:12:58.663 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_tcp_listen_backlog": "512", 2024-08-17T17:12:58.663 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_tcp_nodelay": "true", 2024-08-17T17:12:58.663 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_tcp_prefetch_max_size": "4096", 2024-08-17T17:12:58.663 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_tcp_rcvbuf": "0", 2024-08-17T17:12:58.663 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_time_events_min_wait_interval": "1000", 2024-08-17T17:12:58.663 INFO:teuthology.orchestra.run.smithi063.stdout: "ms_type": "async+posix", 2024-08-17T17:12:58.664 INFO:teuthology.orchestra.run.smithi063.stdout: "no_config_file": "false", 2024-08-17T17:12:58.664 INFO:teuthology.orchestra.run.smithi063.stdout: "nvmeof_mon_client_disconnect_panic": "100", 2024-08-17T17:12:58.664 INFO:teuthology.orchestra.run.smithi063.stdout: "nvmeof_mon_client_tick_period": "2", 2024-08-17T17:12:58.664 INFO:teuthology.orchestra.run.smithi063.stdout: "objecter_completion_locks_per_session": "32", 2024-08-17T17:12:58.664 INFO:teuthology.orchestra.run.smithi063.stdout: "objecter_debug_inject_relock_delay": "false", 2024-08-17T17:12:58.664 INFO:teuthology.orchestra.run.smithi063.stdout: "objecter_inflight_op_bytes": "104857600", 2024-08-17T17:12:58.664 INFO:teuthology.orchestra.run.smithi063.stdout: "objecter_inflight_ops": "1024", 2024-08-17T17:12:58.664 INFO:teuthology.orchestra.run.smithi063.stdout: "objecter_inject_no_watch_ping": "false", 2024-08-17T17:12:58.664 INFO:teuthology.orchestra.run.smithi063.stdout: "objecter_retry_writes_after_first_reply": "false", 2024-08-17T17:12:58.664 INFO:teuthology.orchestra.run.smithi063.stdout: "objecter_tick_interval": "5.000000", 2024-08-17T17:12:58.664 INFO:teuthology.orchestra.run.smithi063.stdout: "objecter_timeout": "10.000000", 2024-08-17T17:12:58.664 INFO:teuthology.orchestra.run.smithi063.stdout: "objectstore_blackhole": "false", 2024-08-17T17:12:58.664 INFO:teuthology.orchestra.run.smithi063.stdout: "objectstore_debug_throw_on_failed_txc": "false", 2024-08-17T17:12:58.664 INFO:teuthology.orchestra.run.smithi063.stdout: "openssl_engine_opts": "", 2024-08-17T17:12:58.665 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_agent_delay_time": "5.000000", 2024-08-17T17:12:58.665 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_agent_hist_halflife": "1000", 2024-08-17T17:12:58.665 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_agent_max_low_ops": "2", 2024-08-17T17:12:58.665 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_agent_max_ops": "4", 2024-08-17T17:12:58.665 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_agent_min_evict_effort": "0.100000", 2024-08-17T17:12:58.665 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_agent_quantize_effort": "0.100000", 2024-08-17T17:12:58.665 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_agent_slop": "0.020000", 2024-08-17T17:12:58.665 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_aggregated_slow_ops_logging": "true", 2024-08-17T17:12:58.665 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_allow_recovery_below_min_size": "true", 2024-08-17T17:12:58.665 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_asio_thread_count": "2", 2024-08-17T17:12:58.665 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_async_recovery_min_cost": "100", 2024-08-17T17:12:58.665 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_auto_mark_unfound_lost": "false", 2024-08-17T17:12:58.665 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_backfill_retry_interval": "30.000000", 2024-08-17T17:12:58.666 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_backfill_scan_max": "512", 2024-08-17T17:12:58.666 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_backfill_scan_min": "64", 2024-08-17T17:12:58.666 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_backoff_on_degraded": "false", 2024-08-17T17:12:58.666 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_backoff_on_peering": "false", 2024-08-17T17:12:58.666 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_backoff_on_unfound": "true", 2024-08-17T17:12:58.666 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_beacon_report_interval": "300", 2024-08-17T17:12:58.666 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_bench_duration": "30", 2024-08-17T17:12:58.666 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_bench_large_size_max_throughput": "104857600", 2024-08-17T17:12:58.666 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_bench_max_block_size": "67108864", 2024-08-17T17:12:58.666 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_bench_small_size_max_iops": "100", 2024-08-17T17:12:58.666 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_blkin_trace_all": "false", 2024-08-17T17:12:58.666 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_blocked_scrub_grace_period": "120", 2024-08-17T17:12:58.666 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2024-08-17T17:12:58.667 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2024-08-17T17:12:58.667 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2024-08-17T17:12:58.667 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_check_for_log_corruption": "false", 2024-08-17T17:12:58.667 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_check_max_object_name_len_on_startup": "true", 2024-08-17T17:12:58.667 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-08-17T17:12:58.667 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2024-08-17T17:12:58.667 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-08-17T17:12:58.667 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_class_update_on_start": "true", 2024-08-17T17:12:58.667 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_client_message_cap": "256", 2024-08-17T17:12:58.667 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_client_message_size_cap": "524288000", 2024-08-17T17:12:58.667 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_client_op_priority": "63", 2024-08-17T17:12:58.667 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_client_watch_timeout": "30", 2024-08-17T17:12:58.668 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_command_max_records": "256", 2024-08-17T17:12:58.668 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_compact_on_start": "false", 2024-08-17T17:12:58.668 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_copyfrom_max_chunk": "8388608", 2024-08-17T17:12:58.668 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_crush_chooseleaf_type": "1", 2024-08-17T17:12:58.668 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_crush_initial_weight": "-1.000000", 2024-08-17T17:12:58.668 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_crush_update_on_start": "true", 2024-08-17T17:12:58.668 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_crush_update_weight_set": "true", 2024-08-17T17:12:58.668 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2024-08-17T17:12:58.668 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2024-08-17T17:12:58.668 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2024-08-17T17:12:58.668 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_drop_ping_duration": "0", 2024-08-17T17:12:58.668 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_drop_ping_probability": "0.000000", 2024-08-17T17:12:58.668 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_feed_pullee": "-1", 2024-08-17T17:12:58.669 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_inject_copyfrom_error": "false", 2024-08-17T17:12:58.669 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2024-08-17T17:12:58.669 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2024-08-17T17:12:58.669 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_misdirected_ops": "false", 2024-08-17T17:12:58.669 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_no_acting_change": "false", 2024-08-17T17:12:58.669 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_no_purge_strays": "false", 2024-08-17T17:12:58.669 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_op_order": "false", 2024-08-17T17:12:58.669 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_pg_log_writeout": "false", 2024-08-17T17:12:58.669 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_pretend_recovery_active": "false", 2024-08-17T17:12:58.669 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_random_push_read_error": "0.000000", 2024-08-17T17:12:58.669 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2024-08-17T17:12:58.669 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_shutdown": "false", 2024-08-17T17:12:58.669 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2024-08-17T17:12:58.670 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2024-08-17T17:12:58.670 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_trim_objects": "false", 2024-08-17T17:12:58.670 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_verify_cached_snaps": "false", 2024-08-17T17:12:58.670 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_verify_missing_on_start": "false", 2024-08-17T17:12:58.670 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_verify_snaps": "false", 2024-08-17T17:12:58.670 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_debug_verify_stray_on_activate": "false", 2024-08-17T17:12:58.670 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_deep_scrub_interval": "604800.000000", 2024-08-17T17:12:58.670 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_deep_scrub_keys": "1024", 2024-08-17T17:12:58.670 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2024-08-17T17:12:58.670 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2024-08-17T17:12:58.670 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2024-08-17T17:12:58.670 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_deep_scrub_stride": "524288", 2024-08-17T17:12:58.670 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2024-08-17T17:12:58.671 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_default_data_pool_replay_window": "45", 2024-08-17T17:12:58.671 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_default_notify_timeout": "30", 2024-08-17T17:12:58.671 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_delete_sleep": "0.000000", 2024-08-17T17:12:58.671 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_delete_sleep_hdd": "5.000000", 2024-08-17T17:12:58.671 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_delete_sleep_hybrid": "1.000000", 2024-08-17T17:12:58.671 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_delete_sleep_ssd": "1.000000", 2024-08-17T17:12:58.671 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_discard_disconnected_ops": "true", 2024-08-17T17:12:58.671 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_ec_partial_reads": "true", 2024-08-17T17:12:58.671 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_enable_op_tracker": "true", 2024-08-17T17:12:58.671 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2024-08-17T17:12:58.671 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_extblkdev_plugins": "vdo", 2024-08-17T17:12:58.671 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_failsafe_full_ratio": "0.970000", 2024-08-17T17:12:58.671 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_fast_fail_on_connection_refused": "true", 2024-08-17T17:12:58.672 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_fast_info": "true", 2024-08-17T17:12:58.672 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_fast_shutdown": "true", 2024-08-17T17:12:58.672 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_fast_shutdown_notify_mon": "true", 2024-08-17T17:12:58.672 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_fast_shutdown_timeout": "15", 2024-08-17T17:12:58.672 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_find_best_info_ignore_history_les": "false", 2024-08-17T17:12:58.672 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_force_auth_primary_missing_objects": "100", 2024-08-17T17:12:58.672 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2024-08-17T17:12:58.672 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_function_tracing": "false", 2024-08-17T17:12:58.672 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_heartbeat_grace": "20", 2024-08-17T17:12:58.672 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_heartbeat_interval": "6", 2024-08-17T17:12:58.672 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2024-08-17T17:12:58.672 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_heartbeat_min_peers": "10", 2024-08-17T17:12:58.673 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_heartbeat_min_size": "2000", 2024-08-17T17:12:58.673 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_heartbeat_stale": "600", 2024-08-17T17:12:58.673 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2024-08-17T17:12:58.673 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_hit_set_max_size": "100000", 2024-08-17T17:12:58.673 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_hit_set_min_size": "1000", 2024-08-17T17:12:58.673 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_hit_set_namespace": ".ceph-internal", 2024-08-17T17:12:58.673 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_ignore_stale_divergent_priors": "false", 2024-08-17T17:12:58.673 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2024-08-17T17:12:58.673 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_inject_failure_on_pg_removal": "false", 2024-08-17T17:12:58.673 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2024-08-17T17:12:58.673 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_journal_flush_on_shutdown": "true", 2024-08-17T17:12:58.673 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_journal_size": "5120", 2024-08-17T17:12:58.673 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_kill_backfill_at": "0", 2024-08-17T17:12:58.674 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_loop_before_reset_tphandle": "64", 2024-08-17T17:12:58.674 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_map_cache_size": "50", 2024-08-17T17:12:58.674 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_map_dedup": "true", 2024-08-17T17:12:58.674 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_map_message_max": "40", 2024-08-17T17:12:58.674 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_map_message_max_bytes": "10485760", 2024-08-17T17:12:58.674 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_map_share_max_epochs": "40", 2024-08-17T17:12:58.674 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_attr_name_len": "100", 2024-08-17T17:12:58.674 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_attr_size": "0", 2024-08-17T17:12:58.674 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_backfills": "1", 2024-08-17T17:12:58.674 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_markdown_count": "5", 2024-08-17T17:12:58.674 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_markdown_period": "600", 2024-08-17T17:12:58.675 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_object_name_len": "2048", 2024-08-17T17:12:58.675 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_object_namespace_len": "256", 2024-08-17T17:12:58.675 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_object_size": "134217728", 2024-08-17T17:12:58.675 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2024-08-17T17:12:58.675 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_omap_entries_per_request": "1024", 2024-08-17T17:12:58.675 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_pg_blocked_by": "16", 2024-08-17T17:12:58.675 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_pg_log_entries": "10000", 2024-08-17T17:12:58.675 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2024-08-17T17:12:58.675 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_pgls": "1024", 2024-08-17T17:12:58.675 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_push_cost": "8388608", 2024-08-17T17:12:58.675 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_push_objects": "10", 2024-08-17T17:12:58.675 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_scrubs": "3", 2024-08-17T17:12:58.675 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2024-08-17T17:12:58.676 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_trimming_pgs": "2", 2024-08-17T17:12:58.676 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_write_op_reply_len": "64", 2024-08-17T17:12:58.676 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_max_write_size": "90", 2024-08-17T17:12:58.676 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2024-08-17T17:12:58.676 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2024-08-17T17:12:58.676 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2024-08-17T17:12:58.676 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2024-08-17T17:12:58.676 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2024-08-17T17:12:58.676 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2024-08-17T17:12:58.676 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2024-08-17T17:12:58.676 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_override_recovery_settings": "false", 2024-08-17T17:12:58.677 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_profile": "balanced", 2024-08-17T17:12:58.677 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2024-08-17T17:12:58.677 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2024-08-17T17:12:58.677 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2024-08-17T17:12:58.677 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2024-08-17T17:12:58.677 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2024-08-17T17:12:58.677 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2024-08-17T17:12:58.677 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2024-08-17T17:12:58.677 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2024-08-17T17:12:58.677 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2024-08-17T17:12:58.677 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_scheduler_client_wgt": "1", 2024-08-17T17:12:58.677 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mclock_skip_benchmark": "false", 2024-08-17T17:12:58.677 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_memory_base": "805306368", 2024-08-17T17:12:58.678 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_memory_cache_min": "134217728", 2024-08-17T17:12:58.678 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_memory_cache_resize_interval": "1.000000", 2024-08-17T17:12:58.678 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_memory_expected_fragmentation": "0.150000", 2024-08-17T17:12:58.678 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_memory_target": "4294967296", 2024-08-17T17:12:58.678 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_memory_target_autotune": "false", 2024-08-17T17:12:58.678 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2024-08-17T17:12:58.678 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_min_pg_log_entries": "250", 2024-08-17T17:12:58.678 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_min_recovery_priority": "0", 2024-08-17T17:12:58.678 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mon_heartbeat_interval": "30", 2024-08-17T17:12:58.678 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2024-08-17T17:12:58.678 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mon_report_interval": "5", 2024-08-17T17:12:58.678 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mon_report_max_in_flight": "2", 2024-08-17T17:12:58.679 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_mon_shutdown_timeout": "5.000000", 2024-08-17T17:12:58.679 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_num_cache_shards": "32", 2024-08-17T17:12:58.679 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_num_op_tracker_shard": "32", 2024-08-17T17:12:58.679 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_numa_auto_affinity": "true", 2024-08-17T17:12:58.679 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_numa_node": "-1", 2024-08-17T17:12:58.679 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_numa_prefer_iface": "true", 2024-08-17T17:12:58.679 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_object_clean_region_max_num_intervals": "10", 2024-08-17T17:12:58.679 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_objecter_finishers": "1", 2024-08-17T17:12:58.679 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_objectstore": "bluestore", 2024-08-17T17:12:58.679 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_objectstore_fuse": "false", 2024-08-17T17:12:58.679 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_objectstore_ideal_list_max": "64", 2024-08-17T17:12:58.679 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_objectstore_tracing": "false", 2024-08-17T17:12:58.679 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_complaint_time": "30.000000", 2024-08-17T17:12:58.680 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_history_duration": "600", 2024-08-17T17:12:58.680 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_history_size": "20", 2024-08-17T17:12:58.680 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_history_slow_op_size": "20", 2024-08-17T17:12:58.680 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2024-08-17T17:12:58.680 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_log_threshold": "5", 2024-08-17T17:12:58.680 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_num_shards": "0", 2024-08-17T17:12:58.680 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_num_shards_hdd": "5", 2024-08-17T17:12:58.680 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_num_shards_ssd": "8", 2024-08-17T17:12:58.680 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_num_threads_per_shard": "0", 2024-08-17T17:12:58.680 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2024-08-17T17:12:58.680 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2024-08-17T17:12:58.680 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2024-08-17T17:12:58.680 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_pq_min_cost": "65536", 2024-08-17T17:12:58.680 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_queue": "mclock_scheduler", 2024-08-17T17:12:58.681 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_queue_cut_off": "high", 2024-08-17T17:12:58.681 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_thread_suicide_timeout": "150", 2024-08-17T17:12:58.681 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_op_thread_timeout": "15", 2024-08-17T17:12:58.681 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_open_classes_on_start": "true", 2024-08-17T17:12:58.681 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_os_flags": "0", 2024-08-17T17:12:58.681 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_peering_op_priority": "255", 2024-08-17T17:12:58.681 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pg_delete_cost": "1048576", 2024-08-17T17:12:58.681 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pg_delete_priority": "5", 2024-08-17T17:12:58.681 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2024-08-17T17:12:58.681 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2024-08-17T17:12:58.681 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pg_log_dups_tracked": "3000", 2024-08-17T17:12:58.681 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pg_log_trim_max": "10000", 2024-08-17T17:12:58.681 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pg_log_trim_min": "100", 2024-08-17T17:12:58.682 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2024-08-17T17:12:58.682 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pg_object_context_cache_count": "64", 2024-08-17T17:12:58.682 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pg_stat_report_interval_max_epochs": "500", 2024-08-17T17:12:58.682 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pg_stat_report_interval_max_seconds": "5", 2024-08-17T17:12:58.682 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2024-08-17T17:12:58.682 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_cache_min_evict_age": "0", 2024-08-17T17:12:58.682 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_cache_min_flush_age": "0", 2024-08-17T17:12:58.682 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2024-08-17T17:12:58.682 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2024-08-17T17:12:58.682 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2024-08-17T17:12:58.682 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_crimson": "false", 2024-08-17T17:12:58.682 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_crush_rule": "-1", 2024-08-17T17:12:58.682 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_ec_fast_read": "false", 2024-08-17T17:12:58.683 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2024-08-17T17:12:58.683 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_flag_bulk": "false", 2024-08-17T17:12:58.683 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_flag_hashpspool": "true", 2024-08-17T17:12:58.683 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_flag_nodelete": "false", 2024-08-17T17:12:58.683 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_flag_nopgchange": "false", 2024-08-17T17:12:58.683 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_flag_nosizechange": "false", 2024-08-17T17:12:58.683 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_flags": "0", 2024-08-17T17:12:58.683 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2024-08-17T17:12:58.683 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_min_size": "0", 2024-08-17T17:12:58.683 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2024-08-17T17:12:58.683 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_pg_num": "32", 2024-08-17T17:12:58.683 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_pgp_num": "0", 2024-08-17T17:12:58.683 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2024-08-17T17:12:58.684 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_read_ratio": "70", 2024-08-17T17:12:58.684 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_size": "3", 2024-08-17T17:12:58.684 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_default_type": "replicated", 2024-08-17T17:12:58.684 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2024-08-17T17:12:58.684 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_pool_use_gmt_hitset": "true", 2024-08-17T17:12:58.684 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_push_per_object_cost": "1000", 2024-08-17T17:12:58.684 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_read_ec_check_for_errors": "false", 2024-08-17T17:12:58.684 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recover_clone_overlap": "true", 2024-08-17T17:12:58.684 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recover_clone_overlap_limit": "10", 2024-08-17T17:12:58.684 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_cost": "20971520", 2024-08-17T17:12:58.684 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_delay_start": "0.000000", 2024-08-17T17:12:58.684 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_max_active": "0", 2024-08-17T17:12:58.684 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_max_active_hdd": "3", 2024-08-17T17:12:58.685 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_max_active_ssd": "10", 2024-08-17T17:12:58.685 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_max_chunk": "8388608", 2024-08-17T17:12:58.685 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2024-08-17T17:12:58.685 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_max_single_start": "1", 2024-08-17T17:12:58.685 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_op_priority": "3", 2024-08-17T17:12:58.685 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_op_warn_multiple": "16", 2024-08-17T17:12:58.685 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_priority": "5", 2024-08-17T17:12:58.685 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_retry_interval": "30.000000", 2024-08-17T17:12:58.685 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_sleep": "0.000000", 2024-08-17T17:12:58.685 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_sleep_hdd": "0.100000", 2024-08-17T17:12:58.685 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2024-08-17T17:12:58.685 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_recovery_sleep_ssd": "0.000000", 2024-08-17T17:12:58.685 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_repair_during_recovery": "false", 2024-08-17T17:12:58.686 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_requested_scrub_priority": "120", 2024-08-17T17:12:58.686 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2024-08-17T17:12:58.686 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_rollback_to_cluster_snap": "", 2024-08-17T17:12:58.686 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_auto_repair": "false", 2024-08-17T17:12:58.686 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_auto_repair_num_errors": "5", 2024-08-17T17:12:58.686 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_backoff_ratio": "0.660000", 2024-08-17T17:12:58.686 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_begin_hour": "0", 2024-08-17T17:12:58.686 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_begin_week_day": "0", 2024-08-17T17:12:58.686 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_chunk_max": "25", 2024-08-17T17:12:58.686 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_chunk_min": "5", 2024-08-17T17:12:58.686 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_cost": "52428800", 2024-08-17T17:12:58.686 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_disable_reservation_queuing": "false", 2024-08-17T17:12:58.686 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_during_recovery": "false", 2024-08-17T17:12:58.687 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_end_hour": "0", 2024-08-17T17:12:58.687 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_end_week_day": "0", 2024-08-17T17:12:58.687 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_event_cost": "4096", 2024-08-17T17:12:58.687 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_extended_sleep": "0.000000", 2024-08-17T17:12:58.687 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2024-08-17T17:12:58.687 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_invalid_stats": "true", 2024-08-17T17:12:58.687 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_load_threshold": "0.500000", 2024-08-17T17:12:58.687 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_max_interval": "604800.000000", 2024-08-17T17:12:58.687 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_max_preemptions": "5", 2024-08-17T17:12:58.687 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_min_interval": "86400.000000", 2024-08-17T17:12:58.687 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_priority": "5", 2024-08-17T17:12:58.687 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_scrub_sleep": "0.000000", 2024-08-17T17:12:58.687 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_shallow_scrub_chunk_max": "100", 2024-08-17T17:12:58.688 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_shallow_scrub_chunk_min": "50", 2024-08-17T17:12:58.688 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_shutdown_pgref_assert": "false", 2024-08-17T17:12:58.688 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_skip_check_past_interval_bounds": "false", 2024-08-17T17:12:58.688 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_skip_data_digest": "false", 2024-08-17T17:12:58.688 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_smart_report_timeout": "5", 2024-08-17T17:12:58.688 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_snap_trim_cost": "1048576", 2024-08-17T17:12:58.688 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_snap_trim_priority": "5", 2024-08-17T17:12:58.688 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_snap_trim_sleep": "0.000000", 2024-08-17T17:12:58.688 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2024-08-17T17:12:58.688 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2024-08-17T17:12:58.688 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2024-08-17T17:12:58.688 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_stats_update_period_not_scrubbing": "120", 2024-08-17T17:12:58.688 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_stats_update_period_scrubbing": "15", 2024-08-17T17:12:58.688 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2024-08-17T17:12:58.689 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_target_transaction_size": "30", 2024-08-17T17:12:58.689 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_tier_default_cache_hit_set_count": "4", 2024-08-17T17:12:58.689 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2024-08-17T17:12:58.689 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2024-08-17T17:12:58.689 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2024-08-17T17:12:58.689 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2024-08-17T17:12:58.689 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2024-08-17T17:12:58.689 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2024-08-17T17:12:58.689 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_tier_default_cache_mode": "writeback", 2024-08-17T17:12:58.689 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2024-08-17T17:12:58.689 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_tier_promote_max_objects_sec": "25", 2024-08-17T17:12:58.689 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_tracing": "false", 2024-08-17T17:12:58.689 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_use_stale_snap": "false", 2024-08-17T17:12:58.690 INFO:teuthology.orchestra.run.smithi063.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2024-08-17T17:12:58.690 INFO:teuthology.orchestra.run.smithi063.stdout: "osdc_blkin_trace_all": "false", 2024-08-17T17:12:58.690 INFO:teuthology.orchestra.run.smithi063.stdout: "paxos_kill_at": "0", 2024-08-17T17:12:58.690 INFO:teuthology.orchestra.run.smithi063.stdout: "paxos_max_join_drift": "10", 2024-08-17T17:12:58.690 INFO:teuthology.orchestra.run.smithi063.stdout: "paxos_min": "500", 2024-08-17T17:12:58.690 INFO:teuthology.orchestra.run.smithi063.stdout: "paxos_min_wait": "0.050000", 2024-08-17T17:12:58.690 INFO:teuthology.orchestra.run.smithi063.stdout: "paxos_propose_interval": "1.000000", 2024-08-17T17:12:58.690 INFO:teuthology.orchestra.run.smithi063.stdout: "paxos_service_trim_max": "500", 2024-08-17T17:12:58.690 INFO:teuthology.orchestra.run.smithi063.stdout: "paxos_service_trim_max_multiplier": "20", 2024-08-17T17:12:58.690 INFO:teuthology.orchestra.run.smithi063.stdout: "paxos_service_trim_min": "250", 2024-08-17T17:12:58.690 INFO:teuthology.orchestra.run.smithi063.stdout: "paxos_stash_full_interval": "25", 2024-08-17T17:12:58.690 INFO:teuthology.orchestra.run.smithi063.stdout: "paxos_trim_max": "500", 2024-08-17T17:12:58.690 INFO:teuthology.orchestra.run.smithi063.stdout: "paxos_trim_min": "250", 2024-08-17T17:12:58.691 INFO:teuthology.orchestra.run.smithi063.stdout: "perf": "true", 2024-08-17T17:12:58.691 INFO:teuthology.orchestra.run.smithi063.stdout: "pid_file": "", 2024-08-17T17:12:58.691 INFO:teuthology.orchestra.run.smithi063.stdout: "plugin_crypto_accelerator": "crypto_isal", 2024-08-17T17:12:58.691 INFO:teuthology.orchestra.run.smithi063.stdout: "plugin_dir": "/usr/lib64/ceph", 2024-08-17T17:12:58.691 INFO:teuthology.orchestra.run.smithi063.stdout: "public_addr": "-", 2024-08-17T17:12:58.691 INFO:teuthology.orchestra.run.smithi063.stdout: "public_addrv": "", 2024-08-17T17:12:58.691 INFO:teuthology.orchestra.run.smithi063.stdout: "public_bind_addr": "-", 2024-08-17T17:12:58.691 INFO:teuthology.orchestra.run.smithi063.stdout: "public_network": "172.21.0.0/20", 2024-08-17T17:12:58.691 INFO:teuthology.orchestra.run.smithi063.stdout: "public_network_interface": "", 2024-08-17T17:12:58.691 INFO:teuthology.orchestra.run.smithi063.stdout: "qat_compressor_busy_polling": "false", 2024-08-17T17:12:58.691 INFO:teuthology.orchestra.run.smithi063.stdout: "qat_compressor_enabled": "false", 2024-08-17T17:12:58.691 INFO:teuthology.orchestra.run.smithi063.stdout: "qat_compressor_session_max_number": "256", 2024-08-17T17:12:58.691 INFO:teuthology.orchestra.run.smithi063.stdout: "rados_mon_op_timeout": "0", 2024-08-17T17:12:58.692 INFO:teuthology.orchestra.run.smithi063.stdout: "rados_osd_op_timeout": "0", 2024-08-17T17:12:58.692 INFO:teuthology.orchestra.run.smithi063.stdout: "rados_tracing": "false", 2024-08-17T17:12:58.692 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_atime_update_interval": "60", 2024-08-17T17:12:58.692 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2024-08-17T17:12:58.692 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_balance_parent_reads": "false", 2024-08-17T17:12:58.692 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_balance_snap_reads": "false", 2024-08-17T17:12:58.692 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_blkin_trace_all": "false", 2024-08-17T17:12:58.692 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_blocklist_expire_seconds": "0", 2024-08-17T17:12:58.692 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_blocklist_on_break_lock": "true", 2024-08-17T17:12:58.692 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_cache": "true", 2024-08-17T17:12:58.692 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_cache_block_writes_upfront": "false", 2024-08-17T17:12:58.692 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_cache_max_dirty": "25165824", 2024-08-17T17:12:58.692 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_cache_max_dirty_age": "1.000000", 2024-08-17T17:12:58.693 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_cache_max_dirty_object": "0", 2024-08-17T17:12:58.693 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_cache_policy": "writearound", 2024-08-17T17:12:58.693 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_cache_size": "33554432", 2024-08-17T17:12:58.693 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_cache_target_dirty": "16777216", 2024-08-17T17:12:58.693 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_cache_writethrough_until_flush": "true", 2024-08-17T17:12:58.693 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_clone_copy_on_read": "false", 2024-08-17T17:12:58.693 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_compression_hint": "none", 2024-08-17T17:12:58.693 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_concurrent_management_ops": "10", 2024-08-17T17:12:58.693 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_config_pool_override_update_timestamp": "0", 2024-08-17T17:12:58.693 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_default_clone_format": "auto", 2024-08-17T17:12:58.693 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_default_data_pool": "", 2024-08-17T17:12:58.693 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_default_features": "61", 2024-08-17T17:12:58.693 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_default_format": "2", 2024-08-17T17:12:58.693 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_default_map_options": "", 2024-08-17T17:12:58.694 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_default_order": "22", 2024-08-17T17:12:58.694 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_default_pool": "rbd", 2024-08-17T17:12:58.694 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2024-08-17T17:12:58.694 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_default_stripe_count": "0", 2024-08-17T17:12:58.694 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_default_stripe_unit": "0", 2024-08-17T17:12:58.694 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_disable_zero_copy_writes": "true", 2024-08-17T17:12:58.694 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_discard_granularity_bytes": "65536", 2024-08-17T17:12:58.694 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_discard_on_zeroed_write_same": "true", 2024-08-17T17:12:58.694 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_enable_alloc_hint": "true", 2024-08-17T17:12:58.694 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2024-08-17T17:12:58.694 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_io_scheduler": "simple", 2024-08-17T17:12:58.694 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2024-08-17T17:12:58.694 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_journal_commit_age": "5.000000", 2024-08-17T17:12:58.695 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2024-08-17T17:12:58.695 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_journal_max_payload_bytes": "16384", 2024-08-17T17:12:58.695 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_journal_object_flush_age": "0.000000", 2024-08-17T17:12:58.695 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_journal_object_flush_bytes": "1048576", 2024-08-17T17:12:58.695 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_journal_object_flush_interval": "0", 2024-08-17T17:12:58.695 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2024-08-17T17:12:58.695 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2024-08-17T17:12:58.695 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_journal_order": "24", 2024-08-17T17:12:58.695 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_journal_pool": "", 2024-08-17T17:12:58.695 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_journal_splay_width": "4", 2024-08-17T17:12:58.695 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_localize_parent_reads": "false", 2024-08-17T17:12:58.695 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_localize_snap_reads": "false", 2024-08-17T17:12:58.695 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2024-08-17T17:12:58.696 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2024-08-17T17:12:58.696 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2024-08-17T17:12:58.696 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2024-08-17T17:12:58.696 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2024-08-17T17:12:58.696 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2024-08-17T17:12:58.696 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_image_policy_type": "simple", 2024-08-17T17:12:58.696 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2024-08-17T17:12:58.696 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_image_state_check_interval": "30", 2024-08-17T17:12:58.696 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2024-08-17T17:12:58.696 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2024-08-17T17:12:58.696 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2024-08-17T17:12:58.696 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2024-08-17T17:12:58.696 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2024-08-17T17:12:58.697 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_memory_autotune": "true", 2024-08-17T17:12:58.697 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_memory_base": "805306368", 2024-08-17T17:12:58.697 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2024-08-17T17:12:58.697 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_memory_cache_min": "134217728", 2024-08-17T17:12:58.697 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2024-08-17T17:12:58.697 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2024-08-17T17:12:58.697 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_memory_target": "4294967296", 2024-08-17T17:12:58.697 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_perf_stats_prio": "5", 2024-08-17T17:12:58.697 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2024-08-17T17:12:58.697 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2024-08-17T17:12:58.697 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirroring_delete_delay": "0", 2024-08-17T17:12:58.697 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2024-08-17T17:12:58.697 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirroring_replay_delay": "0", 2024-08-17T17:12:58.698 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2024-08-17T17:12:58.698 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2024-08-17T17:12:58.698 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_move_to_trash_on_remove": "false", 2024-08-17T17:12:58.698 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2024-08-17T17:12:58.698 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_mtime_update_interval": "60", 2024-08-17T17:12:58.698 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_non_blocking_aio": "true", 2024-08-17T17:12:58.698 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_op_thread_timeout": "60", 2024-08-17T17:12:58.698 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_op_threads": "1", 2024-08-17T17:12:58.698 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_parent_cache_enabled": "false", 2024-08-17T17:12:58.698 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_persistent_cache_mode": "disabled", 2024-08-17T17:12:58.698 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_persistent_cache_path": "/tmp", 2024-08-17T17:12:58.698 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_persistent_cache_size": "1073741824", 2024-08-17T17:12:58.698 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_plugins": "", 2024-08-17T17:12:58.699 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_bps_burst": "0", 2024-08-17T17:12:58.699 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_bps_burst_seconds": "1", 2024-08-17T17:12:58.699 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_bps_limit": "0", 2024-08-17T17:12:58.699 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_exclude_ops": "0", 2024-08-17T17:12:58.699 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_iops_burst": "0", 2024-08-17T17:12:58.699 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_iops_burst_seconds": "1", 2024-08-17T17:12:58.699 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_iops_limit": "0", 2024-08-17T17:12:58.699 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_read_bps_burst": "0", 2024-08-17T17:12:58.699 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2024-08-17T17:12:58.699 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_read_bps_limit": "0", 2024-08-17T17:12:58.699 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_read_iops_burst": "0", 2024-08-17T17:12:58.699 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2024-08-17T17:12:58.699 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_read_iops_limit": "0", 2024-08-17T17:12:58.700 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_schedule_tick_min": "50", 2024-08-17T17:12:58.700 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_write_bps_burst": "0", 2024-08-17T17:12:58.700 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2024-08-17T17:12:58.700 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_write_bps_limit": "0", 2024-08-17T17:12:58.700 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_write_iops_burst": "0", 2024-08-17T17:12:58.700 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2024-08-17T17:12:58.700 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_qos_write_iops_limit": "0", 2024-08-17T17:12:58.700 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_quiesce_notification_attempts": "10", 2024-08-17T17:12:58.700 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_read_from_replica_policy": "default", 2024-08-17T17:12:58.700 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2024-08-17T17:12:58.700 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_readahead_max_bytes": "524288", 2024-08-17T17:12:58.700 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_readahead_trigger_requests": "10", 2024-08-17T17:12:58.700 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_request_timed_out_seconds": "30", 2024-08-17T17:12:58.701 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_skip_partial_discard": "true", 2024-08-17T17:12:58.701 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2024-08-17T17:12:58.701 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_tracing": "false", 2024-08-17T17:12:58.701 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_validate_names": "true", 2024-08-17T17:12:58.701 INFO:teuthology.orchestra.run.smithi063.stdout: "rbd_validate_pool": "true", 2024-08-17T17:12:58.701 INFO:teuthology.orchestra.run.smithi063.stdout: "restapi_base_url": "", 2024-08-17T17:12:58.701 INFO:teuthology.orchestra.run.smithi063.stdout: "restapi_log_level": "", 2024-08-17T17:12:58.701 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_account_default_quota_max_objects": "-1", 2024-08-17T17:12:58.701 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_account_default_quota_max_size": "-1", 2024-08-17T17:12:58.701 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_acl_grants_max_num": "100", 2024-08-17T17:12:58.701 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_admin_entry": "admin", 2024-08-17T17:12:58.701 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2024-08-17T17:12:58.702 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_asio_assert_yielding": "false", 2024-08-17T17:12:58.702 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_backend_store": "rados", 2024-08-17T17:12:58.702 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_barbican_url": "", 2024-08-17T17:12:58.702 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_beast_enable_async": "true", 2024-08-17T17:12:58.702 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_bucket_counters_cache": "false", 2024-08-17T17:12:58.702 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_bucket_counters_cache_size": "10000", 2024-08-17T17:12:58.702 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2024-08-17T17:12:58.702 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_bucket_default_quota_max_size": "-1", 2024-08-17T17:12:58.702 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_bucket_index_max_aio": "128", 2024-08-17T17:12:58.702 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2024-08-17T17:12:58.702 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_bucket_quota_cache_size": "10000", 2024-08-17T17:12:58.702 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_bucket_quota_ttl": "600", 2024-08-17T17:12:58.702 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_bucket_sync_spawn_window": "20", 2024-08-17T17:12:58.703 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_cache_enabled": "true", 2024-08-17T17:12:58.703 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_cache_expiry_interval": "900", 2024-08-17T17:12:58.703 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_cache_lru_size": "25000", 2024-08-17T17:12:58.703 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_config_store": "rados", 2024-08-17T17:12:58.703 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_content_length_compat": "false", 2024-08-17T17:12:58.703 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_copy_obj_progress": "true", 2024-08-17T17:12:58.703 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2024-08-17T17:12:58.703 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_cors_rules_max_num": "100", 2024-08-17T17:12:58.703 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_cross_domain_policy": "", 2024-08-17T17:12:58.703 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_default_encryption_key": "", 2024-08-17T17:12:58.703 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_kmip_addr": "", 2024-08-17T17:12:58.703 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_kmip_ca_path": "", 2024-08-17T17:12:58.703 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_kmip_client_cert": "", 2024-08-17T17:12:58.704 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_kmip_client_key": "", 2024-08-17T17:12:58.704 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_kmip_kms_key_template": "", 2024-08-17T17:12:58.704 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_kmip_password": "", 2024-08-17T17:12:58.704 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2024-08-17T17:12:58.704 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_kmip_username": "", 2024-08-17T17:12:58.704 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_require_ssl": "true", 2024-08-17T17:12:58.704 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2024-08-17T17:12:58.704 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2024-08-17T17:12:58.704 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_sse_s3_backend": "vault", 2024-08-17T17:12:58.704 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2024-08-17T17:12:58.704 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2024-08-17T17:12:58.704 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2024-08-17T17:12:58.704 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2024-08-17T17:12:58.705 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2024-08-17T17:12:58.705 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2024-08-17T17:12:58.705 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2024-08-17T17:12:58.705 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2024-08-17T17:12:58.705 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2024-08-17T17:12:58.705 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2024-08-17T17:12:58.705 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2024-08-17T17:12:58.705 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_suppress_logs": "true", 2024-08-17T17:12:58.705 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_vault_addr": "", 2024-08-17T17:12:58.705 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_vault_auth": "token", 2024-08-17T17:12:58.705 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_vault_namespace": "", 2024-08-17T17:12:58.705 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_vault_prefix": "", 2024-08-17T17:12:58.705 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_vault_secret_engine": "transit", 2024-08-17T17:12:58.706 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_vault_ssl_cacert": "", 2024-08-17T17:12:58.706 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2024-08-17T17:12:58.706 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2024-08-17T17:12:58.706 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_vault_token_file": "", 2024-08-17T17:12:58.706 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_crypt_vault_verify_ssl": "true", 2024-08-17T17:12:58.706 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_curl_buffersize": "524288", 2024-08-17T17:12:58.706 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_curl_low_speed_limit": "1024", 2024-08-17T17:12:58.706 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_curl_low_speed_time": "300", 2024-08-17T17:12:58.706 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_curl_tcp_keepalive": "0", 2024-08-17T17:12:58.706 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_curl_wait_timeout_ms": "1000", 2024-08-17T17:12:58.706 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2024-08-17T17:12:58.706 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2024-08-17T17:12:58.707 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2024-08-17T17:12:58.707 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2024-08-17T17:12:58.707 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d3n_l1_fadvise": "4", 2024-08-17T17:12:58.707 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2024-08-17T17:12:58.707 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d3n_libaio_aio_num": "64", 2024-08-17T17:12:58.707 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d3n_libaio_aio_threads": "20", 2024-08-17T17:12:58.707 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d4n_address": "127.0.0.1:6379", 2024-08-17T17:12:58.707 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d4n_l1_datacache_address": "127.0.0.1:6379", 2024-08-17T17:12:58.707 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d4n_l1_datacache_persistent_path": "/tmp/rgw_d4n_datacache/", 2024-08-17T17:12:58.707 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d4n_l1_datacache_size": "1073741824", 2024-08-17T17:12:58.707 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d4n_l1_evict_cache_on_start": "true", 2024-08-17T17:12:58.707 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d4n_l1_fadvise": "4", 2024-08-17T17:12:58.707 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d4n_libaio_aio_num": "64", 2024-08-17T17:12:58.708 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_d4n_libaio_aio_threads": "20", 2024-08-17T17:12:58.708 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2024-08-17T17:12:58.708 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_data_log_changes_size": "1000", 2024-08-17T17:12:58.708 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_data_log_num_shards": "128", 2024-08-17T17:12:58.708 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_data_log_obj_prefix": "data_log", 2024-08-17T17:12:58.708 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_data_log_window": "30", 2024-08-17T17:12:58.708 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_data_notify_interval_msec": "0", 2024-08-17T17:12:58.708 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_data_sync_poll_interval": "20", 2024-08-17T17:12:58.708 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_data_sync_spawn_window": "20", 2024-08-17T17:12:58.708 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_debug_inject_latency_bi_unlink": "0", 2024-08-17T17:12:58.708 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2024-08-17T17:12:58.708 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_debug_inject_set_olh_err": "0", 2024-08-17T17:12:58.708 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_default_data_log_backing": "fifo", 2024-08-17T17:12:58.708 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_default_realm_info_oid": "default.realm", 2024-08-17T17:12:58.709 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_default_region_info_oid": "default.region", 2024-08-17T17:12:58.709 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_default_zone_info_oid": "default.zone", 2024-08-17T17:12:58.709 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2024-08-17T17:12:58.709 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_defer_to_bucket_acls": "", 2024-08-17T17:12:58.709 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_delete_multi_obj_max_num": "1000", 2024-08-17T17:12:58.709 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_disable_s3select": "false", 2024-08-17T17:12:58.709 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dmclock_admin_lim": "0.000000", 2024-08-17T17:12:58.709 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dmclock_admin_res": "100.000000", 2024-08-17T17:12:58.709 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2024-08-17T17:12:58.709 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dmclock_auth_lim": "0.000000", 2024-08-17T17:12:58.709 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dmclock_auth_res": "200.000000", 2024-08-17T17:12:58.709 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2024-08-17T17:12:58.709 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dmclock_data_lim": "0.000000", 2024-08-17T17:12:58.710 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dmclock_data_res": "500.000000", 2024-08-17T17:12:58.710 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dmclock_data_wgt": "500.000000", 2024-08-17T17:12:58.710 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2024-08-17T17:12:58.710 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dmclock_metadata_res": "500.000000", 2024-08-17T17:12:58.710 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2024-08-17T17:12:58.710 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dns_name": "", 2024-08-17T17:12:58.710 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dns_s3website_name": "", 2024-08-17T17:12:58.710 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dynamic_resharding": "true", 2024-08-17T17:12:58.710 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dynamic_resharding_may_reduce": "true", 2024-08-17T17:12:58.710 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_dynamic_resharding_reduction_wait": "120", 2024-08-17T17:12:58.710 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2024-08-17T17:12:58.710 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_enable_gc_threads": "true", 2024-08-17T17:12:58.711 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_enable_lc_threads": "true", 2024-08-17T17:12:58.711 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_enable_mdsearch": "true", 2024-08-17T17:12:58.711 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_enable_ops_log": "false", 2024-08-17T17:12:58.711 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_enable_quota_threads": "true", 2024-08-17T17:12:58.711 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_enable_static_website": "false", 2024-08-17T17:12:58.711 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_enable_usage_log": "false", 2024-08-17T17:12:58.711 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_enforce_swift_acls": "true", 2024-08-17T17:12:58.711 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_exit_timeout_secs": "120", 2024-08-17T17:12:58.711 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_expose_bucket": "false", 2024-08-17T17:12:58.711 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_extended_http_attrs": "", 2024-08-17T17:12:58.711 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_filter": "none", 2024-08-17T17:12:58.711 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2024-08-17T17:12:58.711 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_frontends": "beast port=7480", 2024-08-17T17:12:58.712 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_gc_max_concurrent_io": "10", 2024-08-17T17:12:58.712 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_gc_max_deferred": "50", 2024-08-17T17:12:58.712 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2024-08-17T17:12:58.712 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_gc_max_objs": "32", 2024-08-17T17:12:58.712 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_gc_max_queue_size": "134213632", 2024-08-17T17:12:58.712 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_gc_max_trim_chunk": "16", 2024-08-17T17:12:58.712 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_gc_obj_min_wait": "7200", 2024-08-17T17:12:58.712 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_gc_processor_max_time": "3600", 2024-08-17T17:12:58.712 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_gc_processor_period": "3600", 2024-08-17T17:12:58.712 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_get_obj_max_req_size": "4194304", 2024-08-17T17:12:58.712 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_get_obj_window_size": "16777216", 2024-08-17T17:12:58.712 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_healthcheck_disabling_path": "", 2024-08-17T17:12:58.713 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_ignore_get_invalid_range": "false", 2024-08-17T17:12:58.713 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_init_timeout": "300", 2024-08-17T17:12:58.713 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2024-08-17T17:12:58.713 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2024-08-17T17:12:58.713 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_kafka_connection_idle": "300", 2024-08-17T17:12:58.713 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_kafka_message_timeout": "5000", 2024-08-17T17:12:58.713 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_kafka_sleep_timeout": "10", 2024-08-17T17:12:58.713 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_accepted_admin_roles": "", 2024-08-17T17:12:58.713 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_accepted_reader_roles": "", 2024-08-17T17:12:58.713 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2024-08-17T17:12:58.713 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_admin_domain": "", 2024-08-17T17:12:58.713 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_admin_password": "", 2024-08-17T17:12:58.713 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_admin_password_path": "", 2024-08-17T17:12:58.714 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_admin_project": "", 2024-08-17T17:12:58.714 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_admin_tenant": "", 2024-08-17T17:12:58.714 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_admin_token": "", 2024-08-17T17:12:58.714 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_admin_token_path": "", 2024-08-17T17:12:58.714 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_admin_user": "", 2024-08-17T17:12:58.714 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_api_version": "2", 2024-08-17T17:12:58.714 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_barbican_domain": "", 2024-08-17T17:12:58.714 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_barbican_password": "", 2024-08-17T17:12:58.714 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_barbican_project": "", 2024-08-17T17:12:58.714 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_barbican_tenant": "", 2024-08-17T17:12:58.714 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_barbican_user": "", 2024-08-17T17:12:58.714 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2024-08-17T17:12:58.714 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_implicit_tenants": "false", 2024-08-17T17:12:58.715 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2024-08-17T17:12:58.715 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_service_token_enabled": "false", 2024-08-17T17:12:58.715 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_token_cache_size": "10000", 2024-08-17T17:12:58.715 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_url": "", 2024-08-17T17:12:58.715 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_keystone_verify_ssl": "true", 2024-08-17T17:12:58.715 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_lc_debug_interval": "-1", 2024-08-17T17:12:58.715 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_lc_lock_max_time": "90", 2024-08-17T17:12:58.715 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_lc_max_objs": "32", 2024-08-17T17:12:58.715 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_lc_max_rules": "1000", 2024-08-17T17:12:58.715 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_lc_max_worker": "3", 2024-08-17T17:12:58.715 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_lc_max_wp_worker": "3", 2024-08-17T17:12:58.715 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_lc_thread_delay": "0", 2024-08-17T17:12:58.715 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2024-08-17T17:12:58.716 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_ldap_dnattr": "uid", 2024-08-17T17:12:58.716 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2024-08-17T17:12:58.716 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_ldap_searchfilter": "", 2024-08-17T17:12:58.716 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2024-08-17T17:12:58.716 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_ldap_uri": "", 2024-08-17T17:12:58.716 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_lfuda_sync_frequency": "60", 2024-08-17T17:12:58.716 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2024-08-17T17:12:58.716 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_list_bucket_min_readahead": "1000", 2024-08-17T17:12:58.716 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_list_buckets_max_chunk": "1000", 2024-08-17T17:12:58.716 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_log_http_headers": "", 2024-08-17T17:12:58.716 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_log_nonexistent_bucket": "false", 2024-08-17T17:12:58.716 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2024-08-17T17:12:58.716 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_log_object_name_utc": "false", 2024-08-17T17:12:58.717 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_lua_max_memory_per_state": "128000", 2024-08-17T17:12:58.717 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_luarocks_location": "/tmp/rgw_luarocks/mon.a", 2024-08-17T17:12:58.717 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_max_attr_name_len": "0", 2024-08-17T17:12:58.717 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_max_attr_size": "0", 2024-08-17T17:12:58.717 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_max_attrs_num_in_req": "0", 2024-08-17T17:12:58.717 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_max_chunk_size": "4194304", 2024-08-17T17:12:58.717 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_max_concurrent_requests": "1024", 2024-08-17T17:12:58.717 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2024-08-17T17:12:58.717 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_max_dynamic_shards": "1999", 2024-08-17T17:12:58.717 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_max_listing_results": "1000", 2024-08-17T17:12:58.717 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_max_notify_retries": "10", 2024-08-17T17:12:58.717 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_max_objs_per_shard": "100000", 2024-08-17T17:12:58.717 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_max_put_param_size": "1048576", 2024-08-17T17:12:58.718 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_max_put_size": "5368709120", 2024-08-17T17:12:58.718 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_max_slo_entries": "1000", 2024-08-17T17:12:58.718 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_md_log_max_shards": "64", 2024-08-17T17:12:58.718 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_md_notify_interval_msec": "200", 2024-08-17T17:12:58.718 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_meta_sync_poll_interval": "20", 2024-08-17T17:12:58.718 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_meta_sync_spawn_window": "20", 2024-08-17T17:12:58.718 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_mime_types_file": "/etc/mime.types", 2024-08-17T17:12:58.718 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_mp_lock_max_time": "600", 2024-08-17T17:12:58.718 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_multi_obj_del_max_aio": "16", 2024-08-17T17:12:58.718 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_multipart_min_part_size": "5242880", 2024-08-17T17:12:58.718 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_multipart_part_upload_limit": "10000", 2024-08-17T17:12:58.718 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_nfs_fhcache_partitions": "3", 2024-08-17T17:12:58.718 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_nfs_fhcache_size": "2017", 2024-08-17T17:12:58.719 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_nfs_frontends": "rgw-nfs", 2024-08-17T17:12:58.719 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2024-08-17T17:12:58.719 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_nfs_lru_lanes": "5", 2024-08-17T17:12:58.719 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_nfs_max_gc": "300", 2024-08-17T17:12:58.719 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_nfs_namespace_expire_secs": "300", 2024-08-17T17:12:58.719 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_nfs_run_gc_threads": "false", 2024-08-17T17:12:58.719 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_nfs_run_lc_threads": "false", 2024-08-17T17:12:58.719 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_nfs_run_quota_threads": "false", 2024-08-17T17:12:58.719 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_nfs_run_sync_thread": "false", 2024-08-17T17:12:58.719 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_nfs_s3_fast_attrs": "false", 2024-08-17T17:12:58.719 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_nfs_write_completion_interval_s": "10", 2024-08-17T17:12:58.719 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_num_async_rados_threads": "32", 2024-08-17T17:12:58.719 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_num_control_oids": "8", 2024-08-17T17:12:58.720 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_numa_node": "-1", 2024-08-17T17:12:58.720 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_obj_stripe_size": "4194304", 2024-08-17T17:12:58.720 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_obj_tombstone_cache_size": "1000", 2024-08-17T17:12:58.720 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_objexp_chunk_size": "100", 2024-08-17T17:12:58.720 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_objexp_gc_interval": "600", 2024-08-17T17:12:58.720 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_objexp_hints_num_shards": "127", 2024-08-17T17:12:58.720 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_olh_pending_timeout_sec": "3600", 2024-08-17T17:12:58.720 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_op_thread_suicide_timeout": "0", 2024-08-17T17:12:58.720 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_op_thread_timeout": "600", 2024-08-17T17:12:58.720 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_op_tracing": "false", 2024-08-17T17:12:58.720 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_opa_token": "", 2024-08-17T17:12:58.720 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_opa_url": "", 2024-08-17T17:12:58.720 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_opa_verify_ssl": "true", 2024-08-17T17:12:58.721 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_ops_log_data_backlog": "5242880", 2024-08-17T17:12:58.721 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2024-08-17T17:12:58.721 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_ops_log_rados": "false", 2024-08-17T17:12:58.721 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_ops_log_socket_path": "", 2024-08-17T17:12:58.721 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_override_bucket_index_max_shards": "0", 2024-08-17T17:12:58.721 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2024-08-17T17:12:58.721 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2024-08-17T17:12:58.721 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_period_push_interval": "2.000000", 2024-08-17T17:12:58.721 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_period_push_interval_max": "30.000000", 2024-08-17T17:12:58.721 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_period_root_pool": ".rgw.root", 2024-08-17T17:12:58.721 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_policy_reject_invalid_principals": "true", 2024-08-17T17:12:58.721 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2024-08-17T17:12:58.721 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_posix_cache_lanes": "3", 2024-08-17T17:12:58.722 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_posix_cache_lmdb_count": "3", 2024-08-17T17:12:58.722 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_posix_cache_max_buckets": "100", 2024-08-17T17:12:58.722 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_posix_cache_partitions": "3", 2024-08-17T17:12:58.722 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2024-08-17T17:12:58.722 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_print_continue": "true", 2024-08-17T17:12:58.722 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_print_prohibited_content_length": "false", 2024-08-17T17:12:58.722 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_put_obj_max_window_size": "67108864", 2024-08-17T17:12:58.722 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_put_obj_min_window_size": "16777216", 2024-08-17T17:12:58.722 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2024-08-17T17:12:58.722 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_rados_pool_recovery_priority": "5", 2024-08-17T17:12:58.722 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_rados_tracing": "false", 2024-08-17T17:12:58.722 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_realm": "", 2024-08-17T17:12:58.722 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_realm_id": "", 2024-08-17T17:12:58.722 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_realm_root_pool": ".rgw.root", 2024-08-17T17:12:58.723 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_region": "", 2024-08-17T17:12:58.723 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_region_root_pool": ".rgw.root", 2024-08-17T17:12:58.723 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_relaxed_region_enforcement": "false", 2024-08-17T17:12:58.723 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_relaxed_s3_bucket_names": "false", 2024-08-17T17:12:58.723 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2024-08-17T17:12:58.723 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_request_uri": "", 2024-08-17T17:12:58.723 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_reshard_batch_size": "64", 2024-08-17T17:12:58.723 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_reshard_bucket_lock_duration": "360", 2024-08-17T17:12:58.723 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_reshard_debug_interval": "-1", 2024-08-17T17:12:58.723 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_reshard_max_aio": "128", 2024-08-17T17:12:58.723 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_reshard_num_logs": "16", 2024-08-17T17:12:58.723 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_reshard_thread_interval": "600", 2024-08-17T17:12:58.723 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_resolve_cname": "false", 2024-08-17T17:12:58.724 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_rest_getusage_op_compat": "false", 2024-08-17T17:12:58.724 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_run_sync_thread": "true", 2024-08-17T17:12:58.724 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_s3_auth_disable_signature_url": "false", 2024-08-17T17:12:58.724 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_s3_auth_order": "sts, external, local", 2024-08-17T17:12:58.724 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_s3_auth_use_keystone": "false", 2024-08-17T17:12:58.724 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_s3_auth_use_ldap": "false", 2024-08-17T17:12:58.724 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_s3_auth_use_rados": "true", 2024-08-17T17:12:58.724 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_s3_auth_use_sts": "false", 2024-08-17T17:12:58.724 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_s3_client_max_sig_ver": "-1", 2024-08-17T17:12:58.724 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_s3_success_create_obj_status": "0", 2024-08-17T17:12:58.724 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_safe_max_objects_per_shard": "102400", 2024-08-17T17:12:58.724 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_scheduler_type": "throttler", 2024-08-17T17:12:58.724 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_script_uri": "", 2024-08-17T17:12:58.724 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_service_provider_name": "", 2024-08-17T17:12:58.725 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_shard_warning_threshold": "90.000000", 2024-08-17T17:12:58.725 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sts_client_id": "", 2024-08-17T17:12:58.725 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sts_client_secret": "", 2024-08-17T17:12:58.725 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sts_entry": "sts", 2024-08-17T17:12:58.725 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sts_key": "sts", 2024-08-17T17:12:58.725 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sts_max_session_duration": "43200", 2024-08-17T17:12:58.725 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sts_min_session_duration": "900", 2024-08-17T17:12:58.725 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sts_token_introspection_url": "", 2024-08-17T17:12:58.725 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_swift_account_in_url": "false", 2024-08-17T17:12:58.725 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_swift_auth_entry": "auth", 2024-08-17T17:12:58.725 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_swift_auth_url": "", 2024-08-17T17:12:58.725 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_swift_custom_header": "", 2024-08-17T17:12:58.725 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_swift_enforce_content_length": "false", 2024-08-17T17:12:58.726 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_swift_need_stats": "true", 2024-08-17T17:12:58.726 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_swift_tenant_name": "", 2024-08-17T17:12:58.726 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_swift_token_expiration": "86400", 2024-08-17T17:12:58.726 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_swift_url": "", 2024-08-17T17:12:58.726 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_swift_url_prefix": "swift", 2024-08-17T17:12:58.726 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_swift_versioning_enabled": "false", 2024-08-17T17:12:58.726 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2024-08-17T17:12:58.726 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2024-08-17T17:12:58.726 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sync_lease_period": "120", 2024-08-17T17:12:58.726 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2024-08-17T17:12:58.726 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sync_log_trim_interval": "1200", 2024-08-17T17:12:58.726 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sync_log_trim_max_buckets": "16", 2024-08-17T17:12:58.726 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2024-08-17T17:12:58.727 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2024-08-17T17:12:58.727 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sync_obj_etag_verify": "false", 2024-08-17T17:12:58.727 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sync_trace_history_size": "4096", 2024-08-17T17:12:58.727 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sync_trace_per_node_log_size": "32", 2024-08-17T17:12:58.727 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2024-08-17T17:12:58.727 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_thread_pool_size": "512", 2024-08-17T17:12:58.727 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_topic_persistency_max_retries": "0", 2024-08-17T17:12:58.727 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_topic_persistency_sleep_duration": "0", 2024-08-17T17:12:58.727 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_topic_persistency_time_to_live": "0", 2024-08-17T17:12:58.727 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_topic_require_publish_policy": "false", 2024-08-17T17:12:58.727 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_torrent_comment": "", 2024-08-17T17:12:58.727 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_torrent_createby": "", 2024-08-17T17:12:58.727 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_torrent_encoding": "", 2024-08-17T17:12:58.728 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_torrent_flag": "false", 2024-08-17T17:12:58.728 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_torrent_max_size": "5368709120", 2024-08-17T17:12:58.728 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_torrent_origin": "", 2024-08-17T17:12:58.728 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_torrent_sha_unit": "524288", 2024-08-17T17:12:58.728 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_torrent_tracker": "", 2024-08-17T17:12:58.728 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_trust_forwarded_https": "false", 2024-08-17T17:12:58.728 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_usage_log_flush_threshold": "1024", 2024-08-17T17:12:58.728 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_usage_log_tick_interval": "30", 2024-08-17T17:12:58.728 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_usage_max_shards": "32", 2024-08-17T17:12:58.728 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_usage_max_user_shards": "1", 2024-08-17T17:12:58.728 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_use_opa_authz": "false", 2024-08-17T17:12:58.728 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_user_counters_cache": "false", 2024-08-17T17:12:58.728 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_user_counters_cache_size": "10000", 2024-08-17T17:12:58.728 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_user_default_quota_max_objects": "-1", 2024-08-17T17:12:58.729 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_user_default_quota_max_size": "-1", 2024-08-17T17:12:58.729 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_user_max_buckets": "1000", 2024-08-17T17:12:58.729 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_user_policies_max_num": "100", 2024-08-17T17:12:58.729 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2024-08-17T17:12:58.729 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_user_quota_sync_idle_users": "false", 2024-08-17T17:12:58.729 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_user_quota_sync_interval": "86400", 2024-08-17T17:12:58.729 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_user_quota_sync_wait_time": "86400", 2024-08-17T17:12:58.729 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_user_unique_email": "true", 2024-08-17T17:12:58.729 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_verify_ssl": "true", 2024-08-17T17:12:58.729 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_website_routing_rules_max_num": "50", 2024-08-17T17:12:58.729 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_zone": "", 2024-08-17T17:12:58.729 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_zone_id": "", 2024-08-17T17:12:58.729 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_zone_root_pool": ".rgw.root", 2024-08-17T17:12:58.730 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_zonegroup": "", 2024-08-17T17:12:58.730 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_zonegroup_id": "", 2024-08-17T17:12:58.730 INFO:teuthology.orchestra.run.smithi063.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2024-08-17T17:12:58.730 INFO:teuthology.orchestra.run.smithi063.stdout: "rgwlc_auto_session_clear": "true", 2024-08-17T17:12:58.730 INFO:teuthology.orchestra.run.smithi063.stdout: "rgwlc_skip_bucket_step": "false", 2024-08-17T17:12:58.730 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_block_size": "4096", 2024-08-17T17:12:58.730 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_bloom_bits_per_key": "20", 2024-08-17T17:12:58.730 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2024-08-17T17:12:58.730 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2024-08-17T17:12:58.730 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_cache_row_ratio": "0.000000", 2024-08-17T17:12:58.730 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_cache_shard_bits": "4", 2024-08-17T17:12:58.730 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_cache_size": "536870912", 2024-08-17T17:12:58.730 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_cache_type": "binned_lru", 2024-08-17T17:12:58.731 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_cf_compact_on_deletion": "true", 2024-08-17T17:12:58.731 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2024-08-17T17:12:58.731 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2024-08-17T17:12:58.731 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_collect_compaction_stats": "false", 2024-08-17T17:12:58.731 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_collect_extended_stats": "false", 2024-08-17T17:12:58.731 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_collect_memory_stats": "false", 2024-08-17T17:12:58.731 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_delete_range_threshold": "1048576", 2024-08-17T17:12:58.731 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_index_type": "binary_search", 2024-08-17T17:12:58.731 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_log_to_ceph_log": "true", 2024-08-17T17:12:58.731 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_metadata_block_size": "4096", 2024-08-17T17:12:58.731 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_partition_filters": "false", 2024-08-17T17:12:58.731 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_perf": "false", 2024-08-17T17:12:58.732 INFO:teuthology.orchestra.run.smithi063.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2024-08-17T17:12:58.732 INFO:teuthology.orchestra.run.smithi063.stdout: "rotating_keys_bootstrap_timeout": "30", 2024-08-17T17:12:58.732 INFO:teuthology.orchestra.run.smithi063.stdout: "rotating_keys_renewal_timeout": "10", 2024-08-17T17:12:58.732 INFO:teuthology.orchestra.run.smithi063.stdout: "run_dir": "/var/run/ceph", 2024-08-17T17:12:58.732 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_block_create": "true", 2024-08-17T17:12:58.732 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_cache_lru_size": "67108864", 2024-08-17T17:12:58.732 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_cbjournal_size": "5368709120", 2024-08-17T17:12:58.732 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_data_delta_based_overwrite": "0", 2024-08-17T17:12:58.732 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_default_max_object_size": "16777216", 2024-08-17T17:12:58.732 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_default_object_metadata_reservation": "16777216", 2024-08-17T17:12:58.732 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_device_size": "53687091200", 2024-08-17T17:12:58.732 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_disable_end_to_end_data_protection": "true", 2024-08-17T17:12:58.732 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_full_integrity_check": "false", 2024-08-17T17:12:58.733 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_journal_batch_capacity": "16", 2024-08-17T17:12:58.733 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_journal_batch_flush_size": "16777216", 2024-08-17T17:12:58.733 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2024-08-17T17:12:58.733 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_journal_iodepth_limit": "5", 2024-08-17T17:12:58.733 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_main_device_type": "SSD", 2024-08-17T17:12:58.733 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_max_concurrent_transactions": "8", 2024-08-17T17:12:58.733 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_max_data_allocation_size": "32768", 2024-08-17T17:12:58.733 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2024-08-17T17:12:58.733 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2024-08-17T17:12:58.733 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2024-08-17T17:12:58.733 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_obj_data_write_amplification": "1.250000", 2024-08-17T17:12:58.733 INFO:teuthology.orchestra.run.smithi063.stdout: "seastore_segment_size": "67108864", 2024-08-17T17:12:58.733 INFO:teuthology.orchestra.run.smithi063.stdout: "set_keepcaps": "false", 2024-08-17T17:12:58.734 INFO:teuthology.orchestra.run.smithi063.stdout: "setgroup": "ceph", 2024-08-17T17:12:58.734 INFO:teuthology.orchestra.run.smithi063.stdout: "setuser": "ceph", 2024-08-17T17:12:58.734 INFO:teuthology.orchestra.run.smithi063.stdout: "setuser_match_path": "", 2024-08-17T17:12:58.734 INFO:teuthology.orchestra.run.smithi063.stdout: "target_max_misplaced_ratio": "0.050000", 2024-08-17T17:12:58.734 INFO:teuthology.orchestra.run.smithi063.stdout: "thp": "false", 2024-08-17T17:12:58.734 INFO:teuthology.orchestra.run.smithi063.stdout: "threadpool_default_timeout": "60", 2024-08-17T17:12:58.734 INFO:teuthology.orchestra.run.smithi063.stdout: "threadpool_empty_queue_max_wait": "2", 2024-08-17T17:12:58.734 INFO:teuthology.orchestra.run.smithi063.stdout: "throttler_perf_counter": "true", 2024-08-17T17:12:58.734 INFO:teuthology.orchestra.run.smithi063.stdout: "tmp_dir": "/tmp", 2024-08-17T17:12:58.734 INFO:teuthology.orchestra.run.smithi063.stdout: "tmp_file_template": "/tmp/ceph-mon.a.XXXXXX" 2024-08-17T17:12:58.734 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:12:59.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:12:59 smithi063 ceph-mon[109843]: pgmap v1116: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:59.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:12:59 smithi063 ceph-mon[114756]: pgmap v1116: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:12:59.363 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2024-08-17T17:12:59.577 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:12:59.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:12:59 smithi162 ceph-mon[104668]: pgmap v1116: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:01.132 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a: {} 2024-08-17T17:13:01.133 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b: {} 2024-08-17T17:13:01.133 INFO:teuthology.orchestra.run.smithi063.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:13:01.133 INFO:teuthology.orchestra.run.smithi063.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:13:01.139 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c: {} 2024-08-17T17:13:01.139 INFO:teuthology.orchestra.run.smithi063.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:13:01.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:01 smithi063 ceph-mon[109843]: pgmap v1117: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:01.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:01 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1422299030' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:13:01.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:01 smithi063 ceph-mon[114756]: pgmap v1117: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:01.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:01 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1422299030' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:13:01.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:01 smithi162 ceph-mon[104668]: pgmap v1117: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:01 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1422299030' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:13:01.727 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2024-08-17T17:13:01.727 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-08-17T17:13:01.727 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:13:01.729 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:13:01.729 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-08-17T17:13:01.729 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_1 16 2024-08-17T17:13:01.730 INFO:tasks.thrashosds.thrasher:starting do_thrash 2024-08-17T17:13:01.730 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:13:01.730 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:13:01.731 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2024-08-17T17:13:01.731 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:01.731 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2024-08-17T17:13:01.731 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:13:01.732 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2024-08-17T17:13:01.732 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:13:01.732 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2024-08-17T17:13:01.732 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:13:01.764 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:01.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:01.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:01.866 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:01.900 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:01.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:01.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:01.985 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:02.002 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:02.040 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:02.041 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:02.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:02.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:02.143 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:02.165 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:02.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:02.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:02.267 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:02.294 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:02.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:02.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:02.396 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:02.421 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:02.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:02.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:02.442 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:02 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:02.442 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:02 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:02.522 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:02.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:02.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:02.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:02.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:02 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:02.648 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:02.670 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:02.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:02.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:02.772 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:02.831 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:02.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:02.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:02.933 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:02.956 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:02.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:02.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:03.057 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:03.083 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:03.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:03.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:03.110 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:03.185 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:03.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:03.212 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:03.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:03.313 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:03.337 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:03.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:03.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:03.411 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:03.411 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:03.439 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:03.462 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:03.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:03.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:03.515 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:03 smithi063 ceph-mon[114756]: pgmap v1118: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:03.515 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:03 smithi063 ceph-mon[109843]: pgmap v1118: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:03.515 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:13:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:13:03] "GET /metrics HTTP/1.1" 200 33975 "" "Prometheus/2.51.0" 2024-08-17T17:13:03.564 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:03.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:03.587 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:03.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:03.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:03 smithi162 ceph-mon[104668]: pgmap v1118: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:03.688 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:03.712 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:03.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:03.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:03.814 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:03.838 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:03.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:03.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:03.940 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:03.963 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:03.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:03.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:04.065 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:04.091 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:04.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:04.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:04.193 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:04.218 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:04.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:04.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:04.320 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:04.344 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:04.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:04.345 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:04.446 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:04.469 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:04.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:04.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:04.571 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:04.593 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:04.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:04.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:04.695 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:04.721 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:04.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:04.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:04.823 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:04.848 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:04.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:04.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:04.950 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:04.974 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:04.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:04.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:05.076 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:05.099 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:05.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:05.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:05.201 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:05.225 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:05.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:05.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:05.327 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:05.351 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:05.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:05.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:05.427 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:05 smithi063 ceph-mon[114756]: pgmap v1119: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:05.427 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:05 smithi063 ceph-mon[109843]: pgmap v1119: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:05.453 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:05.478 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:05.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:05.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:05.580 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:05.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:05 smithi162 ceph-mon[104668]: pgmap v1119: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:05.606 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:05.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:05.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:05.707 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:05.733 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:05.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:05.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:05.834 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:05.860 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:05.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:05.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:05.963 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:05.988 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:05.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:05.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:06.090 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:06.112 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:06.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:06.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:06.214 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:06.240 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:06.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:06.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:06.343 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:06.365 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:06.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:06.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:06.467 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:06.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:06.493 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:06.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:06.594 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:06.618 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:06.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:06.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:06.720 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:06.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:06.743 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:06.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:06.844 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:06.867 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:06.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:06.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:06.969 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:06.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:06.992 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:06.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:07.093 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:07.115 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:07.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:07.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:07.199 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:13:07.199 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:13:07.217 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:07.239 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:07.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:07.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:07.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:07 smithi063 ceph-mon[114756]: pgmap v1120: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:07.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:07 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3434389967' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:13:07.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:07 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:13:07.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:07 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2966279' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:13:07.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:07 smithi063 ceph-mon[109843]: pgmap v1120: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:07.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:07 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3434389967' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:13:07.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:07 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:13:07.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:07 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2966279' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:13:07.340 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:07.361 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:07.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:07.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:07.463 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:07.484 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:07.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:07.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:07.586 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:07.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:07 smithi162 ceph-mon[104668]: pgmap v1120: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:07.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:07 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3434389967' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:13:07.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:07 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:13:07.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:07 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2966279' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:13:07.607 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:07.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:07.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:07.709 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:07.731 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:07.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:07.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:07.833 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:07.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:07.857 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:07.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:07.958 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:07.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:07.980 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:07.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:08.081 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:08.105 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:08.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:08.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:08.207 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:08.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:08.234 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:08.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:08.238 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:13:08.238 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:13:08.255 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:13:08.256 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:13:08.272 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:13:08.272 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-08-17T17:13:08.288 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:13:08.288 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:13:08.306 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:13:08.306 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:13:08.322 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:13:08.323 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:13:08.335 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:08.339 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:13:08.339 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:13:08.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:08 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:13:08.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:08 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3434389967' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:13:08.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:08 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2966279' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:13:08.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:08 smithi162 ceph-mon[104668]: osdmap e1085: 8 total, 8 up, 8 in 2024-08-17T17:13:08.356 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:13:08.356 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:13:08.361 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:08.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:08.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:08.462 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:08.486 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:08.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:08.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:08.512 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:08 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:13:08.512 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:08 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3434389967' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:13:08.512 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:08 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2966279' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:13:08.512 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:08 smithi063 ceph-mon[109843]: osdmap e1085: 8 total, 8 up, 8 in 2024-08-17T17:13:08.512 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:08 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:13:08.512 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:08 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3434389967' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:13:08.512 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:08 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2966279' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:13:08.512 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:08 smithi063 ceph-mon[114756]: osdmap e1085: 8 total, 8 up, 8 in 2024-08-17T17:13:08.589 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:08.611 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:08.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:08.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:08.713 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:08.734 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:08.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:08.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:08.836 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:08.856 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:08.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:08.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:08.959 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:08.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:08.981 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:08.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:09.082 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:09.103 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:09.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:09.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:09.205 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:09.226 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:09.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:09.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:09.323 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:09 smithi063 ceph-mon[109843]: pgmap v1122: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:09.324 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:09 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4133868102' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:13:09.324 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:09 smithi063 ceph-mon[114756]: pgmap v1122: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:09.324 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:09 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4133868102' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:13:09.328 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:09.349 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:09.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:09.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:09.451 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:09.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:09.477 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:09.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:09.577 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:09.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:09 smithi162 ceph-mon[104668]: pgmap v1122: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:09.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:09 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4133868102' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:13:09.601 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:09.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:09.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:09.703 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:09.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:09.727 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:09.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:09.828 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:09.850 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:09.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:09.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:09.898 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:13:09.952 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:09.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:09.975 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:09.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:10.076 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:10.098 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:10.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:10.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:10.199 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:10.221 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:10.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:10.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:10.323 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:10.348 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:10.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:10.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:10.450 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:10.474 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:10.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:10.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:10.576 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:10.597 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:10.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:10.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:10.699 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:10.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:10.721 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:10.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:10.786 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:13:10.822 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:10.854 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:10.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:10.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:10.867 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:13:10.867 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:13:10.956 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:10.979 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:10.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:10.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:11.081 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:11.103 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:11.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:11.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:11.205 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:11.228 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:11.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:11.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:11.331 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:11.353 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:11.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:11.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:11.454 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:11.476 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:11.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:11.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:11.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:11 smithi063 ceph-mon[114756]: pgmap v1123: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:11.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:11 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:11.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:11 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:11.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:11 smithi063 ceph-mon[109843]: pgmap v1123: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:11.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:11 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:11.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:11 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:11.578 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:11.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:11 smithi162 ceph-mon[104668]: pgmap v1123: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:11.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:11 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:11.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:11 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:11.603 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:11.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:11.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:11.705 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:11.723 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:11.729 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:11.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:11.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:11.831 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:11.853 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:11.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:11.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:11.955 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:11.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:11.977 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:11.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:12.078 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:12.101 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:12.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:12.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:12.202 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:12.215 INFO:teuthology.orchestra.run.smithi063.stderr:pool 'unique_pool_1' created 2024-08-17T17:13:12.232 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:12.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:12.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:12.334 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:12.351 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:13:12.356 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:12.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:12.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:12.366 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:12 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2906307270' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-17T17:13:12.366 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:12 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2906307270' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-17T17:13:12.458 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:12.479 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:12.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:12.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:12.507 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:13:12.581 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:12.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:12 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2906307270' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-08-17T17:13:12.611 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:12.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:12.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:12.713 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:12.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:12.736 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:12.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:12.837 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:12.858 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:12.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:12.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:12.960 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:12.981 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:12.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:12.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:13.083 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:13.104 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:13.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:13.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:13.206 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:13.231 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:13.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:13.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:13.333 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:13.354 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:13.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:13.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:13.456 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:13.479 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:13.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:13.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:13:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:13:13] "GET /metrics HTTP/1.1" 200 33975 "" "Prometheus/2.51.0" 2024-08-17T17:13:13.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:13 smithi063 ceph-mon[114756]: pgmap v1124: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:13.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:13 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2906307270' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-08-17T17:13:13.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:13 smithi063 ceph-mon[114756]: osdmap e1086: 8 total, 8 up, 8 in 2024-08-17T17:13:13.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:13 smithi063 ceph-mon[109843]: pgmap v1124: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:13.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:13 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2906307270' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-08-17T17:13:13.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:13 smithi063 ceph-mon[109843]: osdmap e1086: 8 total, 8 up, 8 in 2024-08-17T17:13:13.581 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:13.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:13 smithi162 ceph-mon[104668]: pgmap v1124: 73 pgs: 73 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:13.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:13 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2906307270' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-08-17T17:13:13.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:13 smithi162 ceph-mon[104668]: osdmap e1086: 8 total, 8 up, 8 in 2024-08-17T17:13:13.602 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:13.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:13.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:13.704 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:13.725 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:13.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:13.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:13.827 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:13.851 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:13.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:13.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:13.953 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:13.976 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:13.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:13.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:14.078 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:14.100 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:14.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:14.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:14.201 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:14.223 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:14.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:14.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:14.325 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:14.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:14.347 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:14.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:14.448 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:14.462 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:14 smithi063 ceph-mon[109843]: osdmap e1087: 8 total, 8 up, 8 in 2024-08-17T17:13:14.463 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:14 smithi063 ceph-mon[114756]: osdmap e1087: 8 total, 8 up, 8 in 2024-08-17T17:13:14.473 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:14.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:14.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:14.485 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-08-17T17:13:14.575 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:14.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:14 smithi162 ceph-mon[104668]: osdmap e1087: 8 total, 8 up, 8 in 2024-08-17T17:13:14.597 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:14.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:14.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:14.699 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:14.724 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:14.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:14.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:14.821 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:14.825 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:14.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:14.849 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:14.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:14.876 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:14.951 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:14.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:14.976 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:14.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:15.077 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:15.099 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:15.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:15.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:15.201 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:15.222 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:15.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:15.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:15.238 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:13:15.324 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:15.346 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:15.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:15.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:15.418 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:15 smithi063 ceph-mon[109843]: pgmap v1127: 89 pgs: 2 creating+peering, 11 unknown, 76 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:15.418 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:15 smithi063 ceph-mon[109843]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:13:15.418 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:15 smithi063 ceph-mon[109843]: osdmap e1088: 8 total, 8 up, 8 in 2024-08-17T17:13:15.419 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:15 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3991969831' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:13:15.419 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:15 smithi063 ceph-mon[114756]: pgmap v1127: 89 pgs: 2 creating+peering, 11 unknown, 76 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:15.419 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:15 smithi063 ceph-mon[114756]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:13:15.419 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:15 smithi063 ceph-mon[114756]: osdmap e1088: 8 total, 8 up, 8 in 2024-08-17T17:13:15.419 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:15 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3991969831' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:13:15.447 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:15.470 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:15.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:15.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:15.572 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:15.594 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:15.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:15.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:15.598 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:15 smithi162 ceph-mon[104668]: pgmap v1127: 89 pgs: 2 creating+peering, 11 unknown, 76 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:15.598 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:15 smithi162 ceph-mon[104668]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:13:15.599 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:15 smithi162 ceph-mon[104668]: osdmap e1088: 8 total, 8 up, 8 in 2024-08-17T17:13:15.599 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:15 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3991969831' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:13:15.698 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:15.720 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:15.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:15.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:15.822 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:15.845 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:15.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:15.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:15.947 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:15.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:15.970 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:15.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:16.053 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:16.071 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:16.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:16.094 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:16.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:16.195 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:16.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:16.221 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:16.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:16.322 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:16.346 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:16.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:16.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:16.418 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:16 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:13:16.418 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:16 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3991969831' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:13:16.418 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:16 smithi063 ceph-mon[114756]: osdmap e1089: 8 total, 8 up, 8 in 2024-08-17T17:13:16.419 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:16 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:13:16.419 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:16 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3991969831' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:13:16.419 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:16 smithi063 ceph-mon[109843]: osdmap e1089: 8 total, 8 up, 8 in 2024-08-17T17:13:16.448 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:16.470 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:16.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:16.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:16.517 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:16.572 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:16.594 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:16.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:16.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:16.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:16 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:13:16.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:16 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3991969831' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:13:16.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:16 smithi162 ceph-mon[104668]: osdmap e1089: 8 total, 8 up, 8 in 2024-08-17T17:13:16.696 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:16.718 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:16.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:16.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:16.820 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:16.846 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:16.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:16.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:16.948 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:16.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:16.971 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:16.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:17.072 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:17.098 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:17.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:17.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:17.200 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:17.225 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:17.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:17.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:17.327 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:17.349 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:17.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:17.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:17.451 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:17.473 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:17.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:17.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:17.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:17 smithi063 ceph-mon[109843]: pgmap v1130: 89 pgs: 2 creating+peering, 8 unknown, 79 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:17.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:17 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:17.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:17 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:17.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:17 smithi063 ceph-mon[114756]: pgmap v1130: 89 pgs: 2 creating+peering, 8 unknown, 79 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:17.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:17 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:17.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:17 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:17.575 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:17.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:17 smithi162 ceph-mon[104668]: pgmap v1130: 89 pgs: 2 creating+peering, 8 unknown, 79 active+clean; 582 KiB data, 500 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:17.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:17 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:17.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:17 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:17.597 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:17.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:17.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:17.699 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:17.725 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:17.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:17.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:17.826 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:17.850 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:17.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:17.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:17.952 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:17.978 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:17.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:17.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:18.079 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:18.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:18.103 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:18.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:18.204 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:18.228 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:18.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:18.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:18.330 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:18.353 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:18.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:18.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:18.438 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:13:18.454 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:18.504 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:18.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:18.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:18.606 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:18.632 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:18.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:18.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:18.734 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:18.759 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:18.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:18.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:18.861 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:18.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:18.885 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:18.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:18.986 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:19.011 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:19.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:19.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:19.113 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:19.139 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:19.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:19.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:19.241 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:19.266 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:19.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:19.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:19.324 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:13:19.368 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:19.390 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:19.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:19.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:19.492 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:19.518 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:19.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:19.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:19.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:19 smithi063 ceph-mon[109843]: pgmap v1131: 89 pgs: 89 active+clean; 582 KiB data, 495 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:19.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:19 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1742730425' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:13:19.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:19 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:13:19.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:19 smithi063 ceph-mon[114756]: pgmap v1131: 89 pgs: 89 active+clean; 582 KiB data, 495 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:19.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:19 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1742730425' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:13:19.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:19 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:13:19.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:19 smithi162 ceph-mon[104668]: pgmap v1131: 89 pgs: 89 active+clean; 582 KiB data, 495 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:19.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:19 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1742730425' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:13:19.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:19 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:13:19.620 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:19.643 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:19.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:19.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:19.744 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:19.767 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:19.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:19.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:19.869 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:19.892 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:19.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:19.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:19.907 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:13:19.995 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:20.022 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:20.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:20.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:20.044 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:13:20.045 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:13:20.125 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:20.151 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:20.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:20.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:20.253 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:20.275 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:20.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:20.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:20.377 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:20.402 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:20.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:20.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:20.504 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:20.527 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:20.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:20.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:20.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:20 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:13:20.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:20 smithi162 ceph-mon[104668]: osdmap e1090: 8 total, 8 up, 8 in 2024-08-17T17:13:20.616 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:20 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:13:20.616 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:20 smithi063 ceph-mon[114756]: osdmap e1090: 8 total, 8 up, 8 in 2024-08-17T17:13:20.616 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:20 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:13:20.616 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:20 smithi063 ceph-mon[109843]: osdmap e1090: 8 total, 8 up, 8 in 2024-08-17T17:13:20.629 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:20.641 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:13:20.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:20.655 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:20.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:20.756 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:20.779 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:20.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:20.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:20.881 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:20.907 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:20.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:20.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:21.009 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:21.033 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:21.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:21.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:21.135 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:21.157 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:21.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:21.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:21.259 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:21.283 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:21.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:21.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:21.385 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:21.408 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:21.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:21.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:21.510 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:21.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:21 smithi063 ceph-mon[114756]: pgmap v1133: 89 pgs: 89 active+clean; 582 KiB data, 495 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:21.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:21 smithi063 ceph-mon[109843]: pgmap v1133: 89 pgs: 89 active+clean; 582 KiB data, 495 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:21.536 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:21.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:21.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:21.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:21 smithi162 ceph-mon[104668]: pgmap v1133: 89 pgs: 89 active+clean; 582 KiB data, 495 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:21.612 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:21.638 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:21.663 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:21.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:21.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:21.765 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:21.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:21.788 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:21.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:21.890 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:21.912 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:21.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:21.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:22.014 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:22.036 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:22.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:22.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:22.137 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:22.163 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:22.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:22.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:22.265 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:22.290 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:22.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:22.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:22.392 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:22.416 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:22.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:22.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:22.518 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:22.544 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:22.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:22.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:22.646 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:22.670 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:22.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:22.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:22.772 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:22.795 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:22.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:22.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:22.897 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:22.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:22.921 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:22.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:22.942 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:13:22.942 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:13:22.957 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:13:22.957 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:13:22.985 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:13:22.986 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:13:23.002 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:13:23.002 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:13:23.018 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:13:23.019 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:13:23.022 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:23.035 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:13:23.036 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:13:23.047 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:23.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:23.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:23.052 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:13:23.052 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:13:23.071 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:13:23.072 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:13:23.149 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:23.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:23.173 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:23.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:23.273 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:23.295 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:23.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:23.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:23.397 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:23.422 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:23.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:23.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:23.458 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:13:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:13:23] "GET /metrics HTTP/1.1" 200 36333 "" "Prometheus/2.51.0" 2024-08-17T17:13:23.458 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:23 smithi063 ceph-mon[109843]: pgmap v1134: 89 pgs: 89 active+clean; 582 KiB data, 492 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:23.458 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:23 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4157137732' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:13:23.459 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:23 smithi063 ceph-mon[114756]: pgmap v1134: 89 pgs: 89 active+clean; 582 KiB data, 492 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:23.459 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:23 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4157137732' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:13:23.525 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:23.551 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:23.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:23.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:23.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:23 smithi162 ceph-mon[104668]: pgmap v1134: 89 pgs: 89 active+clean; 582 KiB data, 492 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:23.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:23 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4157137732' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:13:23.653 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:23.679 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:23.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:23.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:23.780 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:23.805 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:23.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:23.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:23.907 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:23.933 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:23.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:23.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:24.035 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:24.060 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:24.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:24.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:24.162 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:24.186 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:24.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:24.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:24.288 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:24.314 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:24.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:24.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:24.415 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:24.438 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:24.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:24.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:24.540 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:24.562 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:24.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:24.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:24.664 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:24.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:24.688 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:24.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:24.789 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:24.815 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:24.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:24.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:24.916 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:24.942 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:24.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:24.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:25.044 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:25.066 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:25.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:25.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:25.168 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:25.190 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:25.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:25.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:25.291 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:25.313 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:25.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:25.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:25.346 INFO:teuthology.orchestra.run.smithi063.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-08-17T17:13:25.415 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:25.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:25.438 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:25.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:25.539 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:25.561 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:25.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:25.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:25.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:25 smithi162 ceph-mon[104668]: pgmap v1135: 89 pgs: 89 active+clean; 582 KiB data, 491 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:25.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:25 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3967248825' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:13:25.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:25 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:25.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:25 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:25.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:25 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:25.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:25 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:13:25.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:25 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:25.664 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:25.679 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:25 smithi063 ceph-mon[114756]: pgmap v1135: 89 pgs: 89 active+clean; 582 KiB data, 491 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:25.680 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:25 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3967248825' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:13:25.680 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:25 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:25.680 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:25 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:25.680 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:25 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:25.680 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:25 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:13:25.680 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:25 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:25.680 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:25 smithi063 ceph-mon[109843]: pgmap v1135: 89 pgs: 89 active+clean; 582 KiB data, 491 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:25.680 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:25 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3967248825' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:13:25.680 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:25 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:25.680 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:25 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:25.681 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:25 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:13:25.681 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:25 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:13:25.681 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:25 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:25.690 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:25.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:25.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:25.792 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:25.814 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:25.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:25.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:25.915 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:25.941 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:25.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:25.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:26.043 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:26.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:26.069 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:26.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:26.170 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:26.194 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:26.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:26.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:26.296 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:26.322 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:26.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:26.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:26.424 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:26.431 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:13:26.431 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:13:26.431 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:13:26.432 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:13:26.432 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.432 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:6649 5.5 5:a8ef91eb:::smithi06387500-42:head [sparse-read 0~0,omap-get-vals-by-keys,omap-get-keys,omap-get-vals,omap-get-header,getxattrs] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e739)", 2024-08-17T17:13:26.432 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:06:06.624737+0000", 2024-08-17T17:13:26.432 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 439.80479031900001, 2024-08-17T17:13:26.432 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0156872910000001, 2024-08-17T17:13:26.432 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.432 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.432 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:13:26.432 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:26.432 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:13:26.433 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:13:26.433 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 6649 2024-08-17T17:13:26.433 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.433 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.433 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.433 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.433 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624737+0000", 2024-08-17T17:13:26.433 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.433 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.433 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.433 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.433 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624737+0000", 2024-08-17T17:13:26.433 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.434 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.434 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.434 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.434 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624739+0000", 2024-08-17T17:13:26.434 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.939e-06 2024-08-17T17:13:26.434 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.434 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.434 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.434 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624746+0000", 2024-08-17T17:13:26.434 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0129999999999996e-06 2024-08-17T17:13:26.434 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.434 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.435 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.435 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624760+0000", 2024-08-17T17:13:26.435 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4147e-05 2024-08-17T17:13:26.435 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.435 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.435 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.435 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624772+0000", 2024-08-17T17:13:26.435 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1637999999999999e-05 2024-08-17T17:13:26.435 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.436 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.436 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.436 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.635409+0000", 2024-08-17T17:13:26.436 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.010636726000000001 2024-08-17T17:13:26.436 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.436 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.436 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:13:26.436 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.635421+0000", 2024-08-17T17:13:26.437 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2747e-05 2024-08-17T17:13:26.437 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.437 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.437 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.437 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.640293+0000", 2024-08-17T17:13:26.437 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0048716579999999 2024-08-17T17:13:26.437 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.438 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.438 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.438 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.640424+0000", 2024-08-17T17:13:26.438 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013142300000000001 2024-08-17T17:13:26.438 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.438 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.438 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.438 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.438 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.438 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:6650 5.5 5:a8ef91eb:::smithi06387500-42:head [sparse-read 0~0] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e739)", 2024-08-17T17:13:26.438 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:06:06.624795+0000", 2024-08-17T17:13:26.438 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 439.804732871, 2024-08-17T17:13:26.439 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0159742039999999, 2024-08-17T17:13:26.439 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.439 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.439 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:13:26.439 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:26.439 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:13:26.439 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:13:26.439 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 6650 2024-08-17T17:13:26.439 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.439 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.439 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.439 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.440 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624795+0000", 2024-08-17T17:13:26.440 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.440 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.440 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.440 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.440 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624795+0000", 2024-08-17T17:13:26.440 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.440 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.440 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.440 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.440 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624795+0000", 2024-08-17T17:13:26.440 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4000000000000002e-07 2024-08-17T17:13:26.440 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.440 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.441 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.441 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624799+0000", 2024-08-17T17:13:26.441 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8330000000000004e-06 2024-08-17T17:13:26.441 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.441 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.441 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.441 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624807+0000", 2024-08-17T17:13:26.441 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0059999999999992e-06 2024-08-17T17:13:26.441 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.441 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.441 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.441 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624813+0000", 2024-08-17T17:13:26.442 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4150000000000004e-06 2024-08-17T17:13:26.442 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.442 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.442 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.442 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.635452+0000", 2024-08-17T17:13:26.442 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.010638544999999999 2024-08-17T17:13:26.442 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.442 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.442 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:13:26.442 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.635460+0000", 2024-08-17T17:13:26.442 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.5110000000000003e-06 2024-08-17T17:13:26.442 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.443 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.443 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.443 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.640677+0000", 2024-08-17T17:13:26.443 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0052166950000001 2024-08-17T17:13:26.443 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.443 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.443 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.443 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.640769+0000", 2024-08-17T17:13:26.443 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.1658999999999998e-05 2024-08-17T17:13:26.443 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.443 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.443 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.443 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.444 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.444 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:6651 5.5 5:a8ef91eb:::smithi06387500-42:head [sparse-read 0~0] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e739)", 2024-08-17T17:13:26.444 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:06:06.624821+0000", 2024-08-17T17:13:26.444 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 439.80470616899999, 2024-08-17T17:13:26.444 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0160774420000001, 2024-08-17T17:13:26.444 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.444 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.444 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:13:26.444 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:26.444 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:13:26.444 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:13:26.444 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 6651 2024-08-17T17:13:26.445 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.445 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.445 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.445 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.445 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624821+0000", 2024-08-17T17:13:26.445 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.445 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.445 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.445 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.445 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624821+0000", 2024-08-17T17:13:26.445 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.445 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.446 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.446 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.446 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624822+0000", 2024-08-17T17:13:26.446 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.08e-07 2024-08-17T17:13:26.446 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.446 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.446 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.446 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624824+0000", 2024-08-17T17:13:26.447 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5119999999999998e-06 2024-08-17T17:13:26.447 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.447 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.447 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.447 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624830+0000", 2024-08-17T17:13:26.447 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9669999999999998e-06 2024-08-17T17:13:26.447 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.447 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.447 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.447 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.624835+0000", 2024-08-17T17:13:26.447 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6120000000000001e-06 2024-08-17T17:13:26.447 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.448 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.448 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.448 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.635514+0000", 2024-08-17T17:13:26.448 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.010679045999999999 2024-08-17T17:13:26.448 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.448 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.448 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:13:26.448 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:06.635524+0000", 2024-08-17T17:13:26.448 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0682e-05 2024-08-17T17:13:26.448 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.448 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.448 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.449 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.640800+0000", 2024-08-17T17:13:26.450 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0052751609999999 2024-08-17T17:13:26.450 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.450 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.450 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.450 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.640899+0000", 2024-08-17T17:13:26.450 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9054000000000004e-05 2024-08-17T17:13:26.450 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.450 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.450 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.450 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.451 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.451 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:6664 5.5 5:a0f2081d:::smithi06387500-34:1d2 [sparse-read 0~1106271,omap-get-vals-by-keys,omap-get-keys,omap-get-vals,omap-get-header,getxattrs] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e740)", 2024-08-17T17:13:26.451 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:06:07.645507+0000", 2024-08-17T17:13:26.451 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 438.78402082600002, 2024-08-17T17:13:26.451 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.019889266999999999, 2024-08-17T17:13:26.451 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.451 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.451 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:13:26.451 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:26.451 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:13:26.451 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:13:26.451 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 6664 2024-08-17T17:13:26.452 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.452 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.452 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.452 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.452 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645507+0000", 2024-08-17T17:13:26.452 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.452 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.452 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.452 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.452 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645507+0000", 2024-08-17T17:13:26.452 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.453 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.453 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.453 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.453 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645508+0000", 2024-08-17T17:13:26.453 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.517e-06 2024-08-17T17:13:26.453 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.453 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.453 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.453 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645512+0000", 2024-08-17T17:13:26.453 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.1919999999999998e-06 2024-08-17T17:13:26.453 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.454 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.454 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.454 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645524+0000", 2024-08-17T17:13:26.454 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1878e-05 2024-08-17T17:13:26.454 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.454 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.454 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.454 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645532+0000", 2024-08-17T17:13:26.454 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9310000000000008e-06 2024-08-17T17:13:26.454 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.454 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.454 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.455 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645673+0000", 2024-08-17T17:13:26.455 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014105899999999999 2024-08-17T17:13:26.455 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.455 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.455 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for missing object", 2024-08-17T17:13:26.455 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645886+0000", 2024-08-17T17:13:26.455 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000212869 2024-08-17T17:13:26.455 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.455 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.455 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.455 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.655912+0000", 2024-08-17T17:13:26.456 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.010025624 2024-08-17T17:13:26.456 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.456 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.456 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for missing object", 2024-08-17T17:13:26.456 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.656252+0000", 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00034029399999999999 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.663992+0000", 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0077400749999999999 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.457 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.664134+0000", 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000141796 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.665396+0000", 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001262032 2024-08-17T17:13:26.458 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:6665 5.5 5:a0f2081d:::smithi06387500-34:1d2 [sparse-read 0~1106271] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e740)", 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:06:07.645610+0000", 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 438.78391694700002, 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.019993318999999999, 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:26.459 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 6665 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645610+0000", 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645610+0000", 2024-08-17T17:13:26.460 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.461 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.461 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.461 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.461 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645611+0000", 2024-08-17T17:13:26.461 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0900000000000001e-07 2024-08-17T17:13:26.461 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.461 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.461 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.461 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645615+0000", 2024-08-17T17:13:26.461 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4510000000000001e-06 2024-08-17T17:13:26.461 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.461 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.461 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.461 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645622+0000", 2024-08-17T17:13:26.462 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8150000000000006e-06 2024-08-17T17:13:26.462 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.462 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.462 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.462 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645628+0000", 2024-08-17T17:13:26.462 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9039999999999997e-06 2024-08-17T17:13:26.462 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.462 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.462 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.462 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645942+0000", 2024-08-17T17:13:26.462 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000313887 2024-08-17T17:13:26.462 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.463 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.463 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for missing object", 2024-08-17T17:13:26.463 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645990+0000", 2024-08-17T17:13:26.463 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7957999999999997e-05 2024-08-17T17:13:26.463 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.463 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.463 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.463 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.656304+0000", 2024-08-17T17:13:26.463 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.010314009000000001 2024-08-17T17:13:26.463 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.463 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.463 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for missing object", 2024-08-17T17:13:26.463 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.656415+0000", 2024-08-17T17:13:26.464 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000111339 2024-08-17T17:13:26.464 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.464 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.464 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.464 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.665426+0000", 2024-08-17T17:13:26.464 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0090101150000000008 2024-08-17T17:13:26.464 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.464 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.464 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.464 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.665488+0000", 2024-08-17T17:13:26.464 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2080000000000002e-05 2024-08-17T17:13:26.464 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.464 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.465 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.465 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.665604+0000", 2024-08-17T17:13:26.465 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000116152 2024-08-17T17:13:26.465 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.465 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.465 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.465 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.465 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.465 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:6666 5.5 5:a0f2081d:::smithi06387500-34:1d2 [read 0~2017071,checksum] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e740)", 2024-08-17T17:13:26.465 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:06:07.645692+0000", 2024-08-17T17:13:26.465 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 438.78383517999998, 2024-08-17T17:13:26.465 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.021300360000000001, 2024-08-17T17:13:26.465 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.466 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.466 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:13:26.466 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:26.466 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:13:26.466 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:13:26.466 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 6666 2024-08-17T17:13:26.466 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.466 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.467 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.467 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.467 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645692+0000", 2024-08-17T17:13:26.467 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.467 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.467 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.467 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.467 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645692+0000", 2024-08-17T17:13:26.467 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.467 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.467 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.467 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.468 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645693+0000", 2024-08-17T17:13:26.468 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8699999999999995e-07 2024-08-17T17:13:26.468 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.468 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.468 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.468 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645696+0000", 2024-08-17T17:13:26.468 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0699999999999998e-06 2024-08-17T17:13:26.468 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.468 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.468 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.468 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645711+0000", 2024-08-17T17:13:26.469 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4832e-05 2024-08-17T17:13:26.469 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.469 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.469 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.469 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.645717+0000", 2024-08-17T17:13:26.469 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9020000000000004e-06 2024-08-17T17:13:26.469 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.469 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.469 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.469 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.646021+0000", 2024-08-17T17:13:26.469 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00030422700000000001 2024-08-17T17:13:26.469 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.470 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.470 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for missing object", 2024-08-17T17:13:26.470 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.646064+0000", 2024-08-17T17:13:26.470 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3238000000000002e-05 2024-08-17T17:13:26.470 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.470 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.470 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.470 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.656471+0000", 2024-08-17T17:13:26.470 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.010406955000000001 2024-08-17T17:13:26.470 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.470 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.471 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for missing object", 2024-08-17T17:13:26.471 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.656585+0000", 2024-08-17T17:13:26.471 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000114382 2024-08-17T17:13:26.471 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.471 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.471 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.471 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.665630+0000", 2024-08-17T17:13:26.471 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0090448360000000005 2024-08-17T17:13:26.471 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.471 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.471 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.471 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.665687+0000", 2024-08-17T17:13:26.471 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7071999999999998e-05 2024-08-17T17:13:26.472 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.472 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.472 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.472 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:07.666993+0000", 2024-08-17T17:13:26.472 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001305259 2024-08-17T17:13:26.472 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.472 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.472 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.472 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.472 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.472 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7059 5.5 5:a0f2081d:::smithi06387500-34:head [append 0~0] snapc 1e6=[1e6,1e5,1e4,1e3,1e2,1da,1d8,1d3,1cf,1c7,1ba] ondisk+write+known_if_redirected+supports_pool_eio e773)", 2024-08-17T17:13:26.473 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:06:40.617151+0000", 2024-08-17T17:13:26.473 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 405.81237646400001, 2024-08-17T17:13:26.473 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.014212897, 2024-08-17T17:13:26.473 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.473 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.473 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:13:26.473 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:26.473 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:13:26.473 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:13:26.473 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7059 2024-08-17T17:13:26.473 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.473 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.474 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.474 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.474 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617151+0000", 2024-08-17T17:13:26.474 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.474 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.474 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.474 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.474 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617151+0000", 2024-08-17T17:13:26.474 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.474 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.474 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.474 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.474 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617152+0000", 2024-08-17T17:13:26.475 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2200000000000004e-07 2024-08-17T17:13:26.475 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.475 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.475 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.475 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617155+0000", 2024-08-17T17:13:26.475 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3110000000000001e-06 2024-08-17T17:13:26.475 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.475 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.475 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.475 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617163+0000", 2024-08-17T17:13:26.475 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8690000000000004e-06 2024-08-17T17:13:26.475 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.476 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.476 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.476 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617169+0000", 2024-08-17T17:13:26.476 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8599999999999998e-06 2024-08-17T17:13:26.476 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.476 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.476 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.476 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617241+0000", 2024-08-17T17:13:26.476 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2308e-05 2024-08-17T17:13:26.476 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.476 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.476 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting_for_map not empty", 2024-08-17T17:13:26.477 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617246+0000", 2024-08-17T17:13:26.477 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5310000000000001e-06 2024-08-17T17:13:26.477 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.477 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.477 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.477 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.627301+0000", 2024-08-17T17:13:26.477 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.010054220000000001 2024-08-17T17:13:26.477 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.477 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.478 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.478 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.627369+0000", 2024-08-17T17:13:26.478 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8048999999999998e-05 2024-08-17T17:13:26.478 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.478 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.478 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:13:26.478 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.627554+0000", 2024-08-17T17:13:26.478 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00018478200000000001 2024-08-17T17:13:26.478 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.478 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.479 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:13:26.479 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.631237+0000", 2024-08-17T17:13:26.479 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.003683188 2024-08-17T17:13:26.479 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.479 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.479 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:13:26.479 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.631279+0000", 2024-08-17T17:13:26.479 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2401000000000003e-05 2024-08-17T17:13:26.479 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.479 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.479 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:13:26.479 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.631290+0000", 2024-08-17T17:13:26.479 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0749e-05 2024-08-17T17:13:26.480 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.480 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.480 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:13:26.480 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.631325+0000", 2024-08-17T17:13:26.480 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4956999999999998e-05 2024-08-17T17:13:26.480 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.480 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.480 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.480 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.631364+0000", 2024-08-17T17:13:26.480 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9050000000000001e-05 2024-08-17T17:13:26.480 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.480 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.480 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.481 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.481 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.481 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7060 5.5 5:a0f2081d:::smithi06387500-34:head [rollback 1e3] snapc 1e6=[1e6,1e5,1e4,1e3,1e2,1da,1d8,1d3,1cf,1c7,1ba] ondisk+write+known_if_redirected+supports_pool_eio e773)", 2024-08-17T17:13:26.481 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:06:40.617200+0000", 2024-08-17T17:13:26.481 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 405.81232732699999, 2024-08-17T17:13:26.481 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.014307886000000001, 2024-08-17T17:13:26.481 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.481 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.481 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:13:26.481 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:26.481 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:13:26.481 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:13:26.481 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7060 2024-08-17T17:13:26.482 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.482 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.482 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.482 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.482 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617200+0000", 2024-08-17T17:13:26.482 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.482 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.482 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.482 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.482 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617200+0000", 2024-08-17T17:13:26.482 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.482 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.483 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.483 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.483 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617201+0000", 2024-08-17T17:13:26.483 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9299999999999997e-07 2024-08-17T17:13:26.483 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.483 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.483 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.483 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617208+0000", 2024-08-17T17:13:26.483 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5599999999999996e-06 2024-08-17T17:13:26.483 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.483 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.483 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.483 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617219+0000", 2024-08-17T17:13:26.484 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0206e-05 2024-08-17T17:13:26.484 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.484 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.484 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.484 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617229+0000", 2024-08-17T17:13:26.484 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0424e-05 2024-08-17T17:13:26.484 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.484 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.484 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.484 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617302+0000", 2024-08-17T17:13:26.484 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3379000000000003e-05 2024-08-17T17:13:26.484 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.485 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.485 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting_for_map not empty", 2024-08-17T17:13:26.485 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617320+0000", 2024-08-17T17:13:26.485 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7963000000000001e-05 2024-08-17T17:13:26.485 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.485 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.485 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.485 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.628765+0000", 2024-08-17T17:13:26.485 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011444368 2024-08-17T17:13:26.485 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.485 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.485 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.486 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.628943+0000", 2024-08-17T17:13:26.486 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00017807500000000001 2024-08-17T17:13:26.486 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.486 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.486 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:13:26.486 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.629190+0000", 2024-08-17T17:13:26.486 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00024742200000000002 2024-08-17T17:13:26.486 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.486 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.486 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:13:26.486 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.631374+0000", 2024-08-17T17:13:26.486 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0021836920000000001 2024-08-17T17:13:26.487 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.487 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.487 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:13:26.487 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.631412+0000", 2024-08-17T17:13:26.487 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8237e-05 2024-08-17T17:13:26.487 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.487 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.487 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:13:26.488 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.631445+0000", 2024-08-17T17:13:26.488 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.252e-05 2024-08-17T17:13:26.488 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.488 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.488 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:13:26.488 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.631479+0000", 2024-08-17T17:13:26.488 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4511999999999998e-05 2024-08-17T17:13:26.488 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.488 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.488 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.488 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.631508+0000", 2024-08-17T17:13:26.489 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8835000000000001e-05 2024-08-17T17:13:26.489 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.489 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.489 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.489 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.489 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.489 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7061 5.5 5:a0f2081d:::smithi06387500-34:head [append 0~0] snapc 1e6=[1e6,1e5,1e4,1e3,1e2,1da,1d8,1d3,1cf,1c7,1ba] ondisk+write+known_if_redirected+supports_pool_eio e773)", 2024-08-17T17:13:26.489 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:06:40.617242+0000", 2024-08-17T17:13:26.489 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 405.81228519799998, 2024-08-17T17:13:26.489 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.01454042, 2024-08-17T17:13:26.489 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.489 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.490 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:13:26.490 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:26.490 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:13:26.490 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:13:26.490 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7061 2024-08-17T17:13:26.490 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.490 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.490 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.490 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.490 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617242+0000", 2024-08-17T17:13:26.491 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.491 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.491 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.491 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.491 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617242+0000", 2024-08-17T17:13:26.491 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.491 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.491 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.491 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.491 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617243+0000", 2024-08-17T17:13:26.491 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0200000000000002e-07 2024-08-17T17:13:26.491 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.491 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.492 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.492 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617246+0000", 2024-08-17T17:13:26.492 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.2310000000000001e-06 2024-08-17T17:13:26.492 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.492 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.492 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.492 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617254+0000", 2024-08-17T17:13:26.492 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6020000000000002e-06 2024-08-17T17:13:26.492 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.492 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.492 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.492 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617259+0000", 2024-08-17T17:13:26.492 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7660000000000004e-06 2024-08-17T17:13:26.493 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.493 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.493 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.493 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617474+0000", 2024-08-17T17:13:26.493 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00021453100000000001 2024-08-17T17:13:26.493 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.493 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.493 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting_for_map not empty", 2024-08-17T17:13:26.493 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.617481+0000", 2024-08-17T17:13:26.493 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2799999999999998e-06 2024-08-17T17:13:26.493 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.494 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.494 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.494 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.630405+0000", 2024-08-17T17:13:26.494 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.012923537000000001 2024-08-17T17:13:26.494 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.494 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.494 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.494 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.630584+0000", 2024-08-17T17:13:26.494 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00017897 2024-08-17T17:13:26.494 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.494 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.495 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:13:26.495 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.630743+0000", 2024-08-17T17:13:26.495 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000159247 2024-08-17T17:13:26.495 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.495 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.495 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:13:26.495 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.631515+0000", 2024-08-17T17:13:26.495 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00077242499999999998 2024-08-17T17:13:26.495 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.495 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.495 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:13:26.495 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.631678+0000", 2024-08-17T17:13:26.495 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016282699999999999 2024-08-17T17:13:26.495 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.496 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.496 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:13:26.496 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.631714+0000", 2024-08-17T17:13:26.496 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5923000000000001e-05 2024-08-17T17:13:26.496 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.496 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.496 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:13:26.496 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.631754+0000", 2024-08-17T17:13:26.496 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9360999999999998e-05 2024-08-17T17:13:26.496 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.496 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.496 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.496 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:40.631783+0000", 2024-08-17T17:13:26.497 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.9118000000000001e-05 2024-08-17T17:13:26.497 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.497 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.497 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.497 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.497 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.497 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7338 5.b 5:daca9f56:::smithi06387500-12:head [delete] snapc 1f0=[1f0,1ef,1ee,1ec,1eb,1ea,1e9,1e8,1e7,1da,1d8,1cf] ondisk+write+peerstat_old+known_if_redirected+supports_pool_eio e792)", 2024-08-17T17:13:26.497 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:06:59.651546+0000", 2024-08-17T17:13:26.497 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 386.77798105199997, 2024-08-17T17:13:26.498 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.020823459, 2024-08-17T17:13:26.498 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.498 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.498 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:13:26.498 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:26.498 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:13:26.498 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:13:26.498 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7338 2024-08-17T17:13:26.498 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.498 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.499 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.499 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.499 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651546+0000", 2024-08-17T17:13:26.499 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.499 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.499 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.499 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.499 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651546+0000", 2024-08-17T17:13:26.499 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.499 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.499 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.499 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.500 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651548+0000", 2024-08-17T17:13:26.500 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8619999999999999e-06 2024-08-17T17:13:26.500 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.500 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.500 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.500 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651554+0000", 2024-08-17T17:13:26.500 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2489999999999998e-06 2024-08-17T17:13:26.500 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.500 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.500 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.500 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651569+0000", 2024-08-17T17:13:26.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4355000000000001e-05 2024-08-17T17:13:26.501 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.501 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.501 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.501 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651580+0000", 2024-08-17T17:13:26.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0848e-05 2024-08-17T17:13:26.501 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.501 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.501 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.501 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651906+0000", 2024-08-17T17:13:26.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00032660600000000001 2024-08-17T17:13:26.501 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.501 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.501 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op must wait for map", 2024-08-17T17:13:26.502 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651916+0000", 2024-08-17T17:13:26.502 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.6209999999999999e-06 2024-08-17T17:13:26.502 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.502 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.502 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.502 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.654283+0000", 2024-08-17T17:13:26.502 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0023670750000000002 2024-08-17T17:13:26.502 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.502 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.502 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:13:26.502 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.654292+0000", 2024-08-17T17:13:26.502 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.1139999999999995e-06 2024-08-17T17:13:26.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.503 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.503 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.503 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:00.670836+0000", 2024-08-17T17:13:26.503 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0165439119999999 2024-08-17T17:13:26.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.503 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.503 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.503 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:00.670958+0000", 2024-08-17T17:13:26.503 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012153 2024-08-17T17:13:26.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.503 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.503 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 5", 2024-08-17T17:13:26.503 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:00.671102+0000", 2024-08-17T17:13:26.504 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014450699999999999 2024-08-17T17:13:26.504 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.504 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.504 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:13:26.504 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:00.671883+0000", 2024-08-17T17:13:26.504 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00078096000000000003 2024-08-17T17:13:26.504 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.504 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.504 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:13:26.504 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:00.672275+0000", 2024-08-17T17:13:26.504 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00039224699999999997 2024-08-17T17:13:26.504 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.505 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.505 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:13:26.505 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:00.672313+0000", 2024-08-17T17:13:26.505 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7296999999999999e-05 2024-08-17T17:13:26.505 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.505 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.505 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.505 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:00.672370+0000", 2024-08-17T17:13:26.505 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7275999999999998e-05 2024-08-17T17:13:26.505 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.505 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.505 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.505 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.506 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.506 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:8420 5.3 5:c6fe7521:::smithi06387500-10:head [delete] snapc 221=[221,220,21f,21e,21d,21b,219,216,215,213,211,210,209,201,1e9,1da] ondisk+write+peerstat_old+known_if_redirected+supports_pool_eio e887)", 2024-08-17T17:13:26.506 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:08:34.695713+0000", 2024-08-17T17:13:26.506 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 291.73381393599999, 2024-08-17T17:13:26.506 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.021222264000000001, 2024-08-17T17:13:26.506 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.506 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.506 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:13:26.507 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:26.507 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:13:26.507 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:13:26.507 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 8420 2024-08-17T17:13:26.507 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.508 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.508 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.508 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.508 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:08:34.695713+0000", 2024-08-17T17:13:26.508 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.509 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.509 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.509 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.509 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:08:34.695713+0000", 2024-08-17T17:13:26.509 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.510 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.510 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.510 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.510 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:08:34.695715+0000", 2024-08-17T17:13:26.510 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.268e-06 2024-08-17T17:13:26.510 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.511 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.511 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.511 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:08:34.695720+0000", 2024-08-17T17:13:26.511 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9289999999999997e-06 2024-08-17T17:13:26.511 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.511 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.512 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.512 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:08:34.695733+0000", 2024-08-17T17:13:26.512 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3064000000000001e-05 2024-08-17T17:13:26.512 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.512 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.512 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.513 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:08:34.695739+0000", 2024-08-17T17:13:26.513 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0179999999999998e-06 2024-08-17T17:13:26.513 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.513 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.513 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.513 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:08:34.695800+0000", 2024-08-17T17:13:26.514 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1527999999999998e-05 2024-08-17T17:13:26.514 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.514 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.514 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.514 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:08:34.695881+0000", 2024-08-17T17:13:26.514 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0686999999999995e-05 2024-08-17T17:13:26.515 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.515 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.515 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:13:26.515 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:08:34.696018+0000", 2024-08-17T17:13:26.515 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013696300000000001 2024-08-17T17:13:26.515 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.516 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.516 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:13:26.516 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:08:34.697422+0000", 2024-08-17T17:13:26.516 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0014035689999999999 2024-08-17T17:13:26.516 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.517 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:13:26.517 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:08:34.697466+0000", 2024-08-17T17:13:26.517 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4928000000000001e-05 2024-08-17T17:13:26.517 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.518 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:13:26.518 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:08:34.716807+0000", 2024-08-17T17:13:26.518 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.019340458000000001 2024-08-17T17:13:26.518 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.518 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.519 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:13:26.519 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:08:34.716866+0000", 2024-08-17T17:13:26.519 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9092999999999998e-05 2024-08-17T17:13:26.519 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.519 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.519 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.520 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:08:34.716936+0000", 2024-08-17T17:13:26.520 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9758999999999994e-05 2024-08-17T17:13:26.520 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.520 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.520 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.520 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.521 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.521 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:9415 5.5 5:a043dd6a:::smithi06387500-50:24c [read 0~2286016,checksum,omap-get-vals-by-keys,omap-get-keys,omap-get-vals,omap-get-header,getxattrs] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e996)", 2024-08-17T17:13:26.521 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:10:23.678166+0000", 2024-08-17T17:13:26.521 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 182.75136140500001, 2024-08-17T17:13:26.521 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011678802, 2024-08-17T17:13:26.521 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.522 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.522 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:13:26.522 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:26.522 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:13:26.522 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:13:26.522 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 9415 2024-08-17T17:13:26.523 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.523 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.523 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.523 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.523 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678166+0000", 2024-08-17T17:13:26.523 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.524 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.524 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.524 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:26.526 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678166+0000", 2024-08-17T17:13:26.526 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.526 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.526 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.526 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.526 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678168+0000", 2024-08-17T17:13:26.526 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2510000000000001e-06 2024-08-17T17:13:26.526 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.526 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.526 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.527 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678187+0000", 2024-08-17T17:13:26.527 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8878000000000001e-05 2024-08-17T17:13:26.527 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.527 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.527 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.527 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678208+0000", 2024-08-17T17:13:26.527 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0611999999999998e-05 2024-08-17T17:13:26.527 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.527 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.527 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.527 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678217+0000", 2024-08-17T17:13:26.527 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8680000000000005e-06 2024-08-17T17:13:26.527 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.528 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.528 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.528 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678307+0000", 2024-08-17T17:13:26.528 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0742999999999995e-05 2024-08-17T17:13:26.528 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.528 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.528 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op must wait for map", 2024-08-17T17:13:26.528 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678315+0000", 2024-08-17T17:13:26.528 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3440000000000003e-06 2024-08-17T17:13:26.528 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.528 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.528 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.685252+0000", 2024-08-17T17:13:26.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0069368590000000001 2024-08-17T17:13:26.529 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.529 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.685391+0000", 2024-08-17T17:13:26.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013987900000000001 2024-08-17T17:13:26.529 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.529 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.689845+0000", 2024-08-17T17:13:26.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044533680000000001 2024-08-17T17:13:26.530 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.530 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.530 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.530 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.530 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.530 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:9416 5.5 5:a043dd6a:::smithi06387500-50:24c [sparse-read 0~1364466] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e996)", 2024-08-17T17:13:26.530 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:10:23.678250+0000", 2024-08-17T17:13:26.530 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 182.75127772499999, 2024-08-17T17:13:26.530 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011825280000000001, 2024-08-17T17:13:26.530 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.530 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.530 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:13:26.531 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:26.531 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:13:26.531 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:13:26.531 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 9416 2024-08-17T17:13:26.531 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.531 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.531 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.531 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.531 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678250+0000", 2024-08-17T17:13:26.531 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.531 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.531 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.531 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.532 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678250+0000", 2024-08-17T17:13:26.532 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.532 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.532 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.532 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.532 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678250+0000", 2024-08-17T17:13:26.532 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2900000000000003e-07 2024-08-17T17:13:26.532 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.532 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.532 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.532 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678253+0000", 2024-08-17T17:13:26.533 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1580000000000001e-06 2024-08-17T17:13:26.533 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.533 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.533 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.533 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678260+0000", 2024-08-17T17:13:26.533 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3890000000000002e-06 2024-08-17T17:13:26.533 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.533 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.533 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.533 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678265+0000", 2024-08-17T17:13:26.533 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2839999999999997e-06 2024-08-17T17:13:26.533 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.533 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.534 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.534 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678383+0000", 2024-08-17T17:13:26.534 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000118231 2024-08-17T17:13:26.534 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.534 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.534 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting_for_map not empty", 2024-08-17T17:13:26.534 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678389+0000", 2024-08-17T17:13:26.534 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0889999999999996e-06 2024-08-17T17:13:26.534 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.534 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.534 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.534 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.689880+0000", 2024-08-17T17:13:26.535 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011490169 2024-08-17T17:13:26.535 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.535 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.535 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.535 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.689965+0000", 2024-08-17T17:13:26.535 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.5360999999999998e-05 2024-08-17T17:13:26.535 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.535 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.535 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.535 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.690075+0000", 2024-08-17T17:13:26.535 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010997000000000001 2024-08-17T17:13:26.535 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.536 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.536 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.536 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.536 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.536 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:9417 5.5 5:a043dd6a:::smithi06387500-50:24c [read 0~2286016,checksum] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e996)", 2024-08-17T17:13:26.536 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:10:23.678303+0000", 2024-08-17T17:13:26.537 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 182.751224149, 2024-08-17T17:13:26.537 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.012948909999999999, 2024-08-17T17:13:26.537 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.537 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.537 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:13:26.537 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:26.538 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:13:26.538 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:13:26.538 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 9417 2024-08-17T17:13:26.538 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.538 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.538 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.538 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.538 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678303+0000", 2024-08-17T17:13:26.538 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.538 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.538 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.539 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.539 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678303+0000", 2024-08-17T17:13:26.539 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.539 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.539 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.539 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.539 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678304+0000", 2024-08-17T17:13:26.539 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0200000000000002e-07 2024-08-17T17:13:26.539 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.539 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.539 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.539 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678309+0000", 2024-08-17T17:13:26.539 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3120000000000001e-06 2024-08-17T17:13:26.540 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.540 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.540 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.540 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678316+0000", 2024-08-17T17:13:26.540 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9280000000000004e-06 2024-08-17T17:13:26.540 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.540 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.540 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.540 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678321+0000", 2024-08-17T17:13:26.540 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3730000000000001e-06 2024-08-17T17:13:26.540 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.540 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.540 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.541 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678415+0000", 2024-08-17T17:13:26.541 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3492999999999994e-05 2024-08-17T17:13:26.541 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.541 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.541 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting_for_map not empty", 2024-08-17T17:13:26.541 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.678420+0000", 2024-08-17T17:13:26.541 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0950000000000003e-06 2024-08-17T17:13:26.541 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.541 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.541 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.541 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.690138+0000", 2024-08-17T17:13:26.541 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011718176 2024-08-17T17:13:26.542 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.542 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.542 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.542 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.690274+0000", 2024-08-17T17:13:26.542 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013536200000000001 2024-08-17T17:13:26.542 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.542 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.542 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.542 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:23.691252+0000", 2024-08-17T17:13:26.542 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00097866899999999994 2024-08-17T17:13:26.542 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.543 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.543 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.543 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.543 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.543 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.6.0:1192 5.2 e1002/739 5:472fe546:::smithi06387500-1:24c v 1002'1994, mlcod=1001'1989)", 2024-08-17T17:13:26.543 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:10:29.722344+0000", 2024-08-17T17:13:26.543 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 176.70718321999999, 2024-08-17T17:13:26.543 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.013391013, 2024-08-17T17:13:26.543 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.543 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.543 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:13:26.543 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.544 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.544 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.544 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:29.722344+0000", 2024-08-17T17:13:26.544 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.544 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.544 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.544 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.544 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:29.722344+0000", 2024-08-17T17:13:26.544 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.544 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.544 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.544 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:29.722345+0000", 2024-08-17T17:13:26.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6100000000000002e-07 2024-08-17T17:13:26.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.545 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.545 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:29.722358+0000", 2024-08-17T17:13:26.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3599e-05 2024-08-17T17:13:26.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.545 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.545 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:29.722367+0000", 2024-08-17T17:13:26.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6060000000000003e-06 2024-08-17T17:13:26.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.545 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.546 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:29.722373+0000", 2024-08-17T17:13:26.546 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7320000000000003e-06 2024-08-17T17:13:26.546 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.546 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.546 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:29.731305+0000", 2024-08-17T17:13:26.546 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.008932331 2024-08-17T17:13:26.546 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.546 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.546 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:29.731330+0000", 2024-08-17T17:13:26.547 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4814e-05 2024-08-17T17:13:26.547 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.547 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.547 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:13:26.547 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:29.735676+0000", 2024-08-17T17:13:26.547 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0043467619999999997 2024-08-17T17:13:26.547 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.547 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.547 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.547 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:29.735735+0000", 2024-08-17T17:13:26.547 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8807999999999997e-05 2024-08-17T17:13:26.547 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.548 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.548 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.548 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.548 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.548 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.6.0:1226 5.2 e1025/739 5:49ac69ae:::smithi06387500-5:241 v 1025'2056, mlcod=1023'2053)", 2024-08-17T17:13:26.548 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:10:52.685838+0000", 2024-08-17T17:13:26.548 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 153.74368923899999, 2024-08-17T17:13:26.548 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011889179, 2024-08-17T17:13:26.548 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.548 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.548 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:13:26.548 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.548 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.549 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.549 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:52.685838+0000", 2024-08-17T17:13:26.549 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.549 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.549 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.549 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.549 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:52.685838+0000", 2024-08-17T17:13:26.549 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.549 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.549 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.549 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.549 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:52.685838+0000", 2024-08-17T17:13:26.549 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8999999999999998e-07 2024-08-17T17:13:26.550 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.550 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.550 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.550 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:52.685844+0000", 2024-08-17T17:13:26.550 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3499999999999996e-06 2024-08-17T17:13:26.550 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.550 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.550 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.550 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:52.685855+0000", 2024-08-17T17:13:26.550 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1379999999999999e-05 2024-08-17T17:13:26.550 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:52.685865+0000", 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.5599999999999999e-06 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:52.688289+0000", 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0024238340000000001 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.551 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:52.688309+0000", 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0417000000000001e-05 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:52.697683+0000", 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0093739849999999996 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:10:52.697727+0000", 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4363000000000002e-05 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.552 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.6.0:1257 5.2 e1053/739 5:4544b372:::smithi06387500-13:254 v 1053'2107, mlcod=1052'2104)", 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:11:20.767465+0000", 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 125.662061948, 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.013650529, 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.553 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.767465+0000", 2024-08-17T17:13:26.554 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.554 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.554 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.554 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.554 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.767465+0000", 2024-08-17T17:13:26.554 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.554 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.554 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.554 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.554 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.767466+0000", 2024-08-17T17:13:26.554 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0100000000000001e-07 2024-08-17T17:13:26.554 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.555 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.555 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.555 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.767472+0000", 2024-08-17T17:13:26.555 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4320000000000004e-06 2024-08-17T17:13:26.555 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.555 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.555 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.555 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.767487+0000", 2024-08-17T17:13:26.555 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4786999999999999e-05 2024-08-17T17:13:26.555 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.556 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.556 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.556 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.767498+0000", 2024-08-17T17:13:26.556 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1362999999999999e-05 2024-08-17T17:13:26.556 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.556 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.556 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.556 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.767598+0000", 2024-08-17T17:13:26.556 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9536000000000004e-05 2024-08-17T17:13:26.556 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.556 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.557 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.557 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.767621+0000", 2024-08-17T17:13:26.557 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3198e-05 2024-08-17T17:13:26.557 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.557 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.557 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:13:26.557 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.781071+0000", 2024-08-17T17:13:26.557 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.013450080999999999 2024-08-17T17:13:26.558 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.558 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.558 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.558 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.781116+0000", 2024-08-17T17:13:26.558 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4931000000000001e-05 2024-08-17T17:13:26.558 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.558 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.558 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.558 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.559 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.559 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.2.0:838 5.8 e1053/804 5:11bf5233:::smithi06387500-20:255 v 1053'1002, mlcod=1043'998)", 2024-08-17T17:13:26.559 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:11:20.767985+0000", 2024-08-17T17:13:26.559 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 125.661542735, 2024-08-17T17:13:26.559 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011684002000000001, 2024-08-17T17:13:26.559 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.559 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.559 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:13:26.559 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.559 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.559 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.560 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.767985+0000", 2024-08-17T17:13:26.560 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.560 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.560 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.560 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.560 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.767985+0000", 2024-08-17T17:13:26.560 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.560 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.560 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.561 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.561 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.767985+0000", 2024-08-17T17:13:26.561 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8899999999999999e-07 2024-08-17T17:13:26.561 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.561 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.561 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.561 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.767992+0000", 2024-08-17T17:13:26.561 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0609999999999998e-06 2024-08-17T17:13:26.561 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.561 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.561 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.561 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.768001+0000", 2024-08-17T17:13:26.561 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6340000000000007e-06 2024-08-17T17:13:26.562 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.562 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.562 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.562 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.768008+0000", 2024-08-17T17:13:26.562 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5170000000000002e-06 2024-08-17T17:13:26.562 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.562 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.562 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.562 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.768085+0000", 2024-08-17T17:13:26.562 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6487000000000001e-05 2024-08-17T17:13:26.562 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.563 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.563 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.563 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.768106+0000", 2024-08-17T17:13:26.563 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1503000000000001e-05 2024-08-17T17:13:26.563 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.563 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.563 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:13:26.563 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.779618+0000", 2024-08-17T17:13:26.563 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011511187000000001 2024-08-17T17:13:26.563 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.563 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.563 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.563 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.779669+0000", 2024-08-17T17:13:26.564 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1023999999999998e-05 2024-08-17T17:13:26.564 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.564 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.564 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.564 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.564 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.564 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.6.0:1258 5.2 e1053/739 5:472fe546:::smithi06387500-1:25f v 1053'2109, mlcod=1052'2104)", 2024-08-17T17:13:26.564 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:11:20.768620+0000", 2024-08-17T17:13:26.564 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 125.66090733199999, 2024-08-17T17:13:26.565 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.014117572, 2024-08-17T17:13:26.565 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.565 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.565 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:13:26.565 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.565 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.565 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.565 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.768620+0000", 2024-08-17T17:13:26.565 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.565 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.565 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.565 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.566 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.768620+0000", 2024-08-17T17:13:26.566 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.566 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.566 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.566 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.566 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.768621+0000", 2024-08-17T17:13:26.566 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.8299999999999995e-07 2024-08-17T17:13:26.566 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.566 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.566 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.566 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.768629+0000", 2024-08-17T17:13:26.566 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0039999999999999e-06 2024-08-17T17:13:26.566 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.567 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.567 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.567 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.768644+0000", 2024-08-17T17:13:26.567 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4629e-05 2024-08-17T17:13:26.567 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.567 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.567 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.567 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.768652+0000", 2024-08-17T17:13:26.567 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2749999999999995e-06 2024-08-17T17:13:26.567 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.567 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.567 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.567 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.777948+0000", 2024-08-17T17:13:26.568 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0092964020000000005 2024-08-17T17:13:26.568 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.568 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.568 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.568 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.777987+0000", 2024-08-17T17:13:26.568 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8779999999999998e-05 2024-08-17T17:13:26.568 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.568 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.568 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:13:26.568 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.782467+0000", 2024-08-17T17:13:26.568 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044797420000000001 2024-08-17T17:13:26.569 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.569 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.569 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.569 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.782738+0000", 2024-08-17T17:13:26.569 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000270757 2024-08-17T17:13:26.569 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.569 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.569 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.569 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.569 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.569 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.5.0:1950 5.7 e1053/771 5:e6dd6ba3:::smithi06387500-9:255 v 1053'1939, mlcod=1052'1935)", 2024-08-17T17:13:26.569 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:11:20.771130+0000", 2024-08-17T17:13:26.570 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 125.65839752700001, 2024-08-17T17:13:26.570 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.013540531, 2024-08-17T17:13:26.570 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:26.570 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:26.570 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:13:26.570 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:26.570 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.570 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:26.570 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.771130+0000", 2024-08-17T17:13:26.570 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.570 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.571 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.571 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:26.571 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.771130+0000", 2024-08-17T17:13:26.571 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:26.571 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.571 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.571 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:26.571 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.771130+0000", 2024-08-17T17:13:26.571 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8900000000000001e-07 2024-08-17T17:13:26.571 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.571 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.572 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:26.572 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.771135+0000", 2024-08-17T17:13:26.572 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2009999999999998e-06 2024-08-17T17:13:26.572 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.572 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.572 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:26.572 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.771148+0000", 2024-08-17T17:13:26.572 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2816000000000001e-05 2024-08-17T17:13:26.572 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.572 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.572 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:26.572 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.771158+0000", 2024-08-17T17:13:26.572 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0222e-05 2024-08-17T17:13:26.573 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.573 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.573 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:26.573 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.775809+0000", 2024-08-17T17:13:26.573 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0046504809999999997 2024-08-17T17:13:26.573 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.573 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.573 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:26.573 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.775832+0000", 2024-08-17T17:13:26.573 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2827999999999999e-05 2024-08-17T17:13:26.574 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.574 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.574 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:13:26.574 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.784613+0000", 2024-08-17T17:13:26.574 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0087817510000000008 2024-08-17T17:13:26.574 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:26.574 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:26.574 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:13:26.574 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:11:20.784670+0000", 2024-08-17T17:13:26.574 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6943e-05 2024-08-17T17:13:26.575 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.575 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.575 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.575 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:26.575 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:26.575 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:13:26.576 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:26.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:26.677 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:26.699 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:26.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:26.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:26.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:26 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3967248825' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:13:26.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:26 smithi063 ceph-mon[114756]: osdmap e1091: 8 total, 8 up, 8 in 2024-08-17T17:13:26.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:26 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3967248825' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:13:26.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:26 smithi063 ceph-mon[109843]: osdmap e1091: 8 total, 8 up, 8 in 2024-08-17T17:13:26.801 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:26.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:26.828 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:26.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:26 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3967248825' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:13:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:26 smithi162 ceph-mon[104668]: osdmap e1091: 8 total, 8 up, 8 in 2024-08-17T17:13:26.859 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:26.929 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:26.956 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:26.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:26.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:27.003 DEBUG:teuthology.orchestra.run.smithi063:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-08-17T17:13:27.006 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:13:27.006 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:13:27.056 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:hints = 1 2024-08-17T17:13:27.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:13:27.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Object prefix: benchmark_data_smithi063_160268 2024-08-17T17:13:27.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:13:27.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:13:27.057 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:27.092 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:27.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:27.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:27.194 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:27.224 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:27.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:27.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:27.230 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:27.325 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:27.358 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:27.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:27.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:27.459 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:27.491 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:27.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:27.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:27.593 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:27.611 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:27 smithi063 ceph-mon[114756]: pgmap v1137: 89 pgs: 89 active+clean; 582 KiB data, 491 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:27.611 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:27 smithi063 ceph-mon[114756]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:13:27.611 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:27 smithi063 ceph-mon[109843]: pgmap v1137: 89 pgs: 89 active+clean; 582 KiB data, 491 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:27.611 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:27 smithi063 ceph-mon[109843]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:13:27.634 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:27.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:27.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:27.736 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:27.776 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:27.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:27.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:27.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:27 smithi162 ceph-mon[104668]: pgmap v1137: 89 pgs: 89 active+clean; 582 KiB data, 491 MiB used, 715 GiB / 715 GiB avail 2024-08-17T17:13:27.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:27 smithi162 ceph-mon[104668]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:13:27.878 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:27.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:27.908 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:27.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:28.009 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:28.040 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:28.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:28.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:28.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 1 16 3786 3770 235.612 235.625 0.00501021 0.00422579 2024-08-17T17:13:28.142 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:28.174 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:28.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:28.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:28.276 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:28.306 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:28.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:28.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:28.408 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:28.449 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:28.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:28.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:28.551 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:28.590 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:28.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:28.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:28.692 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:28.724 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:28.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:28.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:28.826 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:28.843 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:13:28.857 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:28.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:28.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:28.959 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:29.004 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:29.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:29.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:29.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 2 16 7409 7393 231.015 226.438 0.00389892 0.00432006 2024-08-17T17:13:29.107 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:29.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:29.154 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:29.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:29.255 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:29.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:29.288 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:29.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:29.390 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:29.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:29.432 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:29.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:29.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:29 smithi063 ceph-mon[114756]: pgmap v1138: 89 pgs: 89 active+clean; 20 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s wr, 35 op/s 2024-08-17T17:13:29.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:29 smithi063 ceph-mon[109843]: pgmap v1138: 89 pgs: 89 active+clean; 20 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s wr, 35 op/s 2024-08-17T17:13:29.533 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:29.564 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:29.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:29.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:29.610 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:13:29.665 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:29.696 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:29.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:29.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:29.798 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:29.840 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:29.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:29.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:29.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:29 smithi162 ceph-mon[104668]: pgmap v1138: 89 pgs: 89 active+clean; 20 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s wr, 35 op/s 2024-08-17T17:13:29.941 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:29.974 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:29.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:29.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:30.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 3 16 10534 10518 219.078 195.312 0.00441895 0.00455493 2024-08-17T17:13:30.076 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:30.121 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:30.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:30.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:30.227 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:30.255 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:30.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:30.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:30.357 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:30.389 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:30.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:30.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:30.490 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:30.518 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:30.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:30.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:30.620 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:30.664 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:30.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:30.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:30.765 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:30.795 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:30.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:30.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:30.897 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:30.919 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:30.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:30.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:30.973 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:30.973 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:31.022 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:31.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 4 16 13186 13170 205.743 165.75 0.00822266 0.00485214 2024-08-17T17:13:31.081 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:31.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:31.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:31.184 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:31.214 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:31.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:31.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:31.316 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:31.345 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:31.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:31.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:31.374 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:13:31.446 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:31.486 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:31.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:31.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:31.588 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:31.628 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:31.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:31.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:31.647 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:31 smithi162 ceph-mon[104668]: pgmap v1139: 89 pgs: 89 active+clean; 92 MiB data, 673 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s wr, 146 op/s 2024-08-17T17:13:31.647 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:31 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/833358475' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:13:31.647 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:31 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:13:31.647 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:31.647 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:31.730 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:31.765 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:31.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:31.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:31.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:31 smithi063 ceph-mon[114756]: pgmap v1139: 89 pgs: 89 active+clean; 92 MiB data, 673 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s wr, 146 op/s 2024-08-17T17:13:31.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:31 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/833358475' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:13:31.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:31 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:13:31.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:31.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:31 smithi063 ceph-mon[109843]: pgmap v1139: 89 pgs: 89 active+clean; 92 MiB data, 673 MiB used, 715 GiB / 715 GiB avail; 9.1 MiB/s wr, 146 op/s 2024-08-17T17:13:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:31 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/833358475' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:13:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:31 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:13:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:13:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:31.867 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:31.910 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:31.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:31.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:32.012 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:32.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:32.049 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:32.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:32.058 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 5 16 15937 15921 198.979 171.938 0.00564735 0.00502027 2024-08-17T17:13:32.150 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:32.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:32.189 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:32.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:32.290 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:32.330 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:32.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:32.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:32.405 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:32 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:13:32.405 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:32 smithi162 ceph-mon[104668]: osdmap e1092: 8 total, 8 up, 8 in 2024-08-17T17:13:32.405 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:32 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1220501051' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:13:32.405 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:32 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:13:32.406 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:13:32.432 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:32.470 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:32.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:32.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:32.573 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:32.606 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:32.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:32.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:32.708 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:32.740 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:32.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:32.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:32.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:32 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:13:32.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:32 smithi063 ceph-mon[114756]: osdmap e1092: 8 total, 8 up, 8 in 2024-08-17T17:13:32.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:32 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1220501051' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:13:32.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:32 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:13:32.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:32 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:13:32.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:32 smithi063 ceph-mon[109843]: osdmap e1092: 8 total, 8 up, 8 in 2024-08-17T17:13:32.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:32 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1220501051' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:13:32.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:32 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:13:32.842 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:32.877 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:32.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:32.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:32.979 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:33.017 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:33.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:33.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:33.058 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 6 16 18781 18765 195.438 177.75 0.00823328 0.00511053 2024-08-17T17:13:33.119 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:33.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:33.165 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:33.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:33.266 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:33.304 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:33.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:33.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:33.407 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:33.507 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:33.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:33.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:33.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:13:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:13:33] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.51.0" 2024-08-17T17:13:33.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:33 smithi063 ceph-mon[114756]: pgmap v1141: 89 pgs: 89 active+clean; 722 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-08-17T17:13:33.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:33 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:13:33.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:33 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:13:33.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:33 smithi063 ceph-mon[114756]: osdmap e1093: 8 total, 8 up, 8 in 2024-08-17T17:13:33.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:33 smithi063 ceph-mon[109843]: pgmap v1141: 89 pgs: 89 active+clean; 722 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-08-17T17:13:33.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:33 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:13:33.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:33 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:13:33.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:33 smithi063 ceph-mon[109843]: osdmap e1093: 8 total, 8 up, 8 in 2024-08-17T17:13:33.608 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:33.643 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:33.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:33.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:33.745 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:33.776 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:33.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:33.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:33.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:33 smithi162 ceph-mon[104668]: pgmap v1141: 89 pgs: 89 active+clean; 722 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2024-08-17T17:13:33.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:33 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:13:33.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:33 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:13:33.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:33 smithi162 ceph-mon[104668]: osdmap e1093: 8 total, 8 up, 8 in 2024-08-17T17:13:33.878 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:33.911 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:33.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:33.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:34.013 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:34.042 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:34.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:34.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:34.058 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 7 16 21516 21500 191.936 170.938 0.00227781 0.00520411 2024-08-17T17:13:34.143 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:34.182 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:34.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:34.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:34.284 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:34.317 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:34.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:34.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:34.419 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:34.458 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:34.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:34.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:34.560 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:34.593 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:34.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:34.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:34.695 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:34.725 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:34.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:34.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:34.827 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:34.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:34.861 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:34.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:34.962 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:34.992 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:34.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:34.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:35.058 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 8 16 24245 24229 189.262 170.562 0.00773984 0.00527812 2024-08-17T17:13:35.093 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:35.116 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:35.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:35.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:35.218 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:35.254 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:35.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:35.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:35.356 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:35.392 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:35.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:35.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:35.494 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:35.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:35.535 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:35.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:35.636 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:35.647 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:35 smithi063 ceph-mon[109843]: pgmap v1143: 89 pgs: 89 active+clean; 1013 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-08-17T17:13:35.647 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:35 smithi063 ceph-mon[114756]: pgmap v1143: 89 pgs: 89 active+clean; 1013 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-08-17T17:13:35.667 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:35.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:35.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:35.683 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:13:35.769 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:35.815 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:35.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:35.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:35.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:35 smithi162 ceph-mon[104668]: pgmap v1143: 89 pgs: 89 active+clean; 1013 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 127 MiB/s wr, 2.02k op/s 2024-08-17T17:13:35.916 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:35.967 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:35.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:35.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:36.059 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 9 16 26456 26440 183.587 138.188 0.00990197 0.00544019 2024-08-17T17:13:36.069 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:36.118 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:36.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:36.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:36.220 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:36.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:36.254 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:36.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:36.355 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:36.369 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:13:36.369 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:13:36.369 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.369 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:27106 6.1 6:82cd0ead:::benchmark_data_smithi063_160268_object27105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:13:36.369 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:36.359998+0000", 2024-08-17T17:13:36.369 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0072398180000000003, 2024-08-17T17:13:36.370 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0072728849999999998, 2024-08-17T17:13:36.370 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:36.370 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:36.370 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:13:36.370 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:36.370 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:13:36.370 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:13:36.370 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 27106 2024-08-17T17:13:36.370 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.370 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:36.370 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.370 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:36.371 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.359998+0000", 2024-08-17T17:13:36.371 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:36.371 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.371 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.371 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:36.371 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.359998+0000", 2024-08-17T17:13:36.371 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:36.371 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.371 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.371 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:36.371 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.360000+0000", 2024-08-17T17:13:36.371 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5880000000000001e-06 2024-08-17T17:13:36.372 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.372 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.372 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:36.372 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.360023+0000", 2024-08-17T17:13:36.372 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3323999999999999e-05 2024-08-17T17:13:36.372 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.372 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.372 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:36.372 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.360041+0000", 2024-08-17T17:13:36.372 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8077999999999999e-05 2024-08-17T17:13:36.373 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.373 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.373 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:36.373 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.360058+0000", 2024-08-17T17:13:36.373 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.732e-05 2024-08-17T17:13:36.373 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.373 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.373 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:36.373 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.360170+0000", 2024-08-17T17:13:36.373 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000111308 2024-08-17T17:13:36.374 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.374 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.374 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:36.375 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.360295+0000", 2024-08-17T17:13:36.375 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000124828 2024-08-17T17:13:36.375 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.375 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.375 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:13:36.375 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.360428+0000", 2024-08-17T17:13:36.375 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013327799999999999 2024-08-17T17:13:36.375 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.375 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.375 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:13:36.376 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.362021+0000", 2024-08-17T17:13:36.376 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0015931980000000001 2024-08-17T17:13:36.376 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.376 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.376 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:13:36.376 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.364983+0000", 2024-08-17T17:13:36.376 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0029617150000000002 2024-08-17T17:13:36.376 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:36.376 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:36.376 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:36.376 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.376 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.377 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:27111 6.1 6:8b0175f8:::benchmark_data_smithi063_160268_object27110:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:13:36.377 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:36.362853+0000", 2024-08-17T17:13:36.377 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0043851489999999996, 2024-08-17T17:13:36.377 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0045035489999999999, 2024-08-17T17:13:36.377 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:36.377 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:36.377 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:13:36.377 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:36.377 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:13:36.377 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:13:36.377 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 27111 2024-08-17T17:13:36.377 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.377 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:36.378 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.378 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:36.378 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.362853+0000", 2024-08-17T17:13:36.378 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:36.378 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.378 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.378 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:36.378 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.362853+0000", 2024-08-17T17:13:36.378 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:36.378 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.378 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.378 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:36.378 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.362854+0000", 2024-08-17T17:13:36.379 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5680000000000001e-06 2024-08-17T17:13:36.379 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.379 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.379 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:36.379 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.362881+0000", 2024-08-17T17:13:36.379 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6800000000000001e-05 2024-08-17T17:13:36.379 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.379 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.379 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:36.379 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.362895+0000", 2024-08-17T17:13:36.379 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3439e-05 2024-08-17T17:13:36.379 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.379 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.380 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:36.380 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.362902+0000", 2024-08-17T17:13:36.380 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6399999999999997e-06 2024-08-17T17:13:36.380 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.380 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.380 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:36.380 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.362979+0000", 2024-08-17T17:13:36.380 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6600999999999994e-05 2024-08-17T17:13:36.380 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.380 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.380 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:36.380 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.363155+0000", 2024-08-17T17:13:36.380 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000176584 2024-08-17T17:13:36.381 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.381 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.381 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:13:36.381 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.363287+0000", 2024-08-17T17:13:36.381 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000131142 2024-08-17T17:13:36.381 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.381 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.381 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:13:36.381 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.365240+0000", 2024-08-17T17:13:36.381 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001953074 2024-08-17T17:13:36.381 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.381 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.381 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:13:36.381 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.365501+0000", 2024-08-17T17:13:36.382 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00026175299999999999 2024-08-17T17:13:36.382 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:36.382 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:36.382 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:36.382 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.382 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.382 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:27114 6.1 6:8e4e6fde:::benchmark_data_smithi063_160268_object27113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:13:36.382 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:36.365192+0000", 2024-08-17T17:13:36.382 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.00204586, 2024-08-17T17:13:36.382 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0022411060000000001, 2024-08-17T17:13:36.382 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:36.382 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:36.382 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:13:36.383 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:13:36.383 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:13:36.383 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:13:36.383 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 27114 2024-08-17T17:13:36.383 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.383 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:36.383 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.383 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:36.383 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.365192+0000", 2024-08-17T17:13:36.383 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:36.383 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.383 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.383 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:36.383 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.365192+0000", 2024-08-17T17:13:36.384 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:36.384 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.384 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.384 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:36.384 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.365193+0000", 2024-08-17T17:13:36.384 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3859999999999999e-06 2024-08-17T17:13:36.384 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.384 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.384 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:36.384 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.365268+0000", 2024-08-17T17:13:36.384 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4382999999999996e-05 2024-08-17T17:13:36.384 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.384 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.385 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:36.385 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.365302+0000", 2024-08-17T17:13:36.385 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4248000000000001e-05 2024-08-17T17:13:36.385 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.385 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.385 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:36.385 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.365313+0000", 2024-08-17T17:13:36.385 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0753e-05 2024-08-17T17:13:36.385 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.385 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.385 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:36.385 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.365533+0000", 2024-08-17T17:13:36.385 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000219896 2024-08-17T17:13:36.386 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.386 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.386 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:36.386 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.365666+0000", 2024-08-17T17:13:36.386 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000133585 2024-08-17T17:13:36.386 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.386 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.386 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:13:36.386 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.365803+0000", 2024-08-17T17:13:36.386 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013649499999999999 2024-08-17T17:13:36.386 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:36.386 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:36.386 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:36.387 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.387 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.387 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:27116 6.e e1093/1086 6:7f64d692:::benchmark_data_smithi063_160268_object27115:head v 1093'1732, mlcod=1093'1729)", 2024-08-17T17:13:36.387 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:36.367004+0000", 2024-08-17T17:13:36.387 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.000234269, 2024-08-17T17:13:36.387 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00049473699999999998, 2024-08-17T17:13:36.387 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:36.387 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:36.387 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:13:36.387 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:36.387 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.387 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:36.387 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.367004+0000", 2024-08-17T17:13:36.388 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:36.388 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.388 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.388 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:36.388 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.367004+0000", 2024-08-17T17:13:36.388 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:36.388 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.388 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.388 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:36.388 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.367005+0000", 2024-08-17T17:13:36.388 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.353e-06 2024-08-17T17:13:36.388 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.388 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.388 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:36.389 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.367039+0000", 2024-08-17T17:13:36.389 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4387999999999997e-05 2024-08-17T17:13:36.389 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.389 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.389 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:36.389 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.367053+0000", 2024-08-17T17:13:36.389 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3181e-05 2024-08-17T17:13:36.389 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.389 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.389 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:36.389 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.367060+0000", 2024-08-17T17:13:36.389 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0169999999999999e-06 2024-08-17T17:13:36.389 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.390 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.390 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:13:36.390 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.367229+0000", 2024-08-17T17:13:36.390 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016966 2024-08-17T17:13:36.390 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.390 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.390 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:13:36.390 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.367249+0000", 2024-08-17T17:13:36.390 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9933e-05 2024-08-17T17:13:36.391 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:36.391 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:36.391 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:36.391 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.391 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.391 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.44609.0:27106 6.1 e1093/1086)", 2024-08-17T17:13:36.392 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:36.367095+0000", 2024-08-17T17:13:36.392 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.000143129, 2024-08-17T17:13:36.392 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000449994, 2024-08-17T17:13:36.392 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:13:36.392 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:13:36.392 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "queued for pg", 2024-08-17T17:13:36.392 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:13:36.392 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.392 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:13:36.392 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.367095+0000", 2024-08-17T17:13:36.392 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:36.392 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.392 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.392 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:13:36.393 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.367095+0000", 2024-08-17T17:13:36.393 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:13:36.393 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.393 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.393 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:13:36.393 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.367095+0000", 2024-08-17T17:13:36.393 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1799999999999995e-07 2024-08-17T17:13:36.393 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.393 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.393 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:13:36.393 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.367098+0000", 2024-08-17T17:13:36.393 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.9009999999999998e-06 2024-08-17T17:13:36.394 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.394 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.394 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:13:36.394 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.367106+0000", 2024-08-17T17:13:36.394 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3100000000000003e-06 2024-08-17T17:13:36.394 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:13:36.394 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:13:36.394 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:13:36.394 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:36.367111+0000", 2024-08-17T17:13:36.394 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.169e-06 2024-08-17T17:13:36.395 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:36.395 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:13:36.395 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:36.395 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:13:36.395 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-08-17T17:13:36.395 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 5 2024-08-17T17:13:36.395 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:13:36.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:36.410 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:36.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:36.511 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:36.542 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:36.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:36.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:36.643 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:36.684 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:36.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:36.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:36.786 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:36.825 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:36.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:36.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:36.927 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:36.973 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:36.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:36.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:37.058 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 10 16 28721 28705 179.384 141.562 0.00579825 0.00556888 2024-08-17T17:13:37.075 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:37.106 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:37.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:37.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:37.208 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:37.228 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:37.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:37.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:37.330 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:37.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:37.369 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:37.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:37.470 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:37.498 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:37.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:37.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:37.600 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:37.632 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:37.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:37.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:37.734 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:37.755 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:37.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:37.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:37.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:37 smithi063 ceph-mon[109843]: pgmap v1144: 89 pgs: 89 active+clean; 1.1 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s 2024-08-17T17:13:37.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:37 smithi063 ceph-mon[114756]: pgmap v1144: 89 pgs: 89 active+clean; 1.1 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s 2024-08-17T17:13:37.814 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:13:37.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:37 smithi162 ceph-mon[104668]: pgmap v1144: 89 pgs: 89 active+clean; 1.1 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 144 MiB/s wr, 2.30k op/s 2024-08-17T17:13:37.857 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:37.897 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:37.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:37.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:37.999 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:38.036 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:38.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:38.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:38.058 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 11 16 30502 30486 173.194 111.312 0.042501 0.00575379 2024-08-17T17:13:38.139 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:38.179 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:38.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:38.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:38.282 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:38.314 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:38.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:38.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:38.416 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:38.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:38.442 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:38.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:38.543 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:38.580 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:38.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:38.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:38.682 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:38.712 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:38.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:38.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:38.814 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:38.848 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:38.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:38.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:38.949 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:38.988 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:38.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:38.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:39.058 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 12 16 32476 32460 169.041 123.375 0.0119467 0.00590968 2024-08-17T17:13:39.090 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:39.128 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:39.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:39.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:39.230 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:39.272 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:39.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:39.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:39.374 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:39.409 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:39.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:39.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:39.411 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:39.511 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:39.533 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:39.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:39.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:39.635 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:39.671 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:39.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:39.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:39.773 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:39.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:39 smithi063 ceph-mon[114756]: pgmap v1145: 89 pgs: 89 active+clean; 1.6 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 197 MiB/s wr, 3.15k op/s 2024-08-17T17:13:39.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:39 smithi063 ceph-mon[109843]: pgmap v1145: 89 pgs: 89 active+clean; 1.6 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 197 MiB/s wr, 3.15k op/s 2024-08-17T17:13:39.811 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:39.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:39.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:39.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:39 smithi162 ceph-mon[104668]: pgmap v1145: 89 pgs: 89 active+clean; 1.6 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 197 MiB/s wr, 3.15k op/s 2024-08-17T17:13:39.914 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:39.951 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:39.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:39.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:40.053 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:40.058 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 13 16 34222 34206 164.431 109.125 0.00369601 0.00607035 2024-08-17T17:13:40.074 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:40.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:40.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:40.176 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:40.209 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:40.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:40.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:40.310 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:40.350 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:40.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:40.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:40.451 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:40.493 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:40.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:40.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:40.594 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:40.625 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:40.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:40.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:40.727 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:40.758 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:40.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:40.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:40.860 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:40.891 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:40.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:40.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:40.992 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:41.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:41.015 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:41.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:41.058 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 14 16 35904 35888 160.194 105.125 0.00660933 0.00623617 2024-08-17T17:13:41.116 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:41.155 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:41.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:41.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:41.256 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:41.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:41.279 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:41.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:41.382 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:41.413 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:41.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:41.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:41.515 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:41.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:41.545 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:41.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:41.646 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:41.679 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:41 smithi063 ceph-mon[114756]: pgmap v1146: 89 pgs: 89 active+clean; 1.8 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 179 MiB/s wr, 2.87k op/s 2024-08-17T17:13:41.679 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:41 smithi063 ceph-mon[109843]: pgmap v1146: 89 pgs: 89 active+clean; 1.8 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 179 MiB/s wr, 2.87k op/s 2024-08-17T17:13:41.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:41.687 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:41.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:41.721 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:13:41.788 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:41.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:41 smithi162 ceph-mon[104668]: pgmap v1146: 89 pgs: 89 active+clean; 1.8 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 179 MiB/s wr, 2.87k op/s 2024-08-17T17:13:41.864 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:41.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:41.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:41.965 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:42.018 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:42.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:42.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:42.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 15 16 37838 37822 157.558 120.875 0.00667125 0.00634102 2024-08-17T17:13:42.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:42.161 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:42.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:42.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:42.263 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:42.296 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:42.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:42.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:42.398 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:42.446 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:42.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:42.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:42.534 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:13:42.534 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:13:42.548 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:42.570 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:13:42.571 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:13:42.590 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:42.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:42.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:42.610 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:13:42.614 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:13:42.664 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:13:42.665 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:13:42.692 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:42.701 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:13:42.701 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:13:42.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:42.731 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:42.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:42.734 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:13:42.734 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:13:42.773 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:13:42.773 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:13:42.793 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:13:42.793 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:13:42.832 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:42.869 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:42.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:42.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:42.971 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:43.010 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:43.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:43.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:43.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 16 16 39674 39658 154.882 114.75 0.00540491 0.00644943 2024-08-17T17:13:43.112 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:43.140 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:43.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:43.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:43.242 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:43.280 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:43.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:43.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:43.382 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:43.412 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:43.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:43.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:43.413 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:13:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:13:43] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.51.0" 2024-08-17T17:13:43.514 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:43.560 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:43.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:43.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:43.662 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:43.688 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:43.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:43.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:43.769 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:43 smithi162 ceph-mon[104668]: pgmap v1147: 89 pgs: 89 active+clean; 2.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-08-17T17:13:43.769 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:43 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1826953473' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:13:43.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:43 smithi063 ceph-mon[114756]: pgmap v1147: 89 pgs: 89 active+clean; 2.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-08-17T17:13:43.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:43 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1826953473' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:13:43.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:43 smithi063 ceph-mon[109843]: pgmap v1147: 89 pgs: 89 active+clean; 2.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s 2024-08-17T17:13:43.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:43 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1826953473' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:13:43.790 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:43.823 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:43.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:43.824 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:43.925 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:43.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:43.963 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:43.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:44.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 17 16 41655 41639 153.054 123.812 0.0185027 0.00652772 2024-08-17T17:13:44.064 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:44.095 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:44.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:44.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:44.197 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:44.229 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:44.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:44.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:44.331 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:44.359 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:44.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:44.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:44.461 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:44.489 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:44.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:44.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:44.549 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:44.557 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:44.591 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:44.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:44.655 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:44.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:44.756 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:44.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:44.788 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:44.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:44.889 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:44.930 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:44.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:44.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:45.032 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:45.062 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 18 16 43225 43209 150.002 98.125 0.00613836 0.00666114 2024-08-17T17:13:45.063 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:45.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:45.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:45.165 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:45.205 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:45.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:45.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:45.306 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:45.344 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:45.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:45.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:45.420 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:13:45.446 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:45.471 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:45.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:45.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:45.573 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:45.615 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:45.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:45.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:45.684 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:45 smithi063 ceph-mon[114756]: pgmap v1148: 89 pgs: 89 active+clean; 2.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 142 MiB/s wr, 2.28k op/s 2024-08-17T17:13:45.684 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:45 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/119950493' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:13:45.684 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:45 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:13:45.684 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:45 smithi063 ceph-mon[109843]: pgmap v1148: 89 pgs: 89 active+clean; 2.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 142 MiB/s wr, 2.28k op/s 2024-08-17T17:13:45.684 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:45 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/119950493' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:13:45.685 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:45 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:13:45.718 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:45.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:45.754 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:45.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:45.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:45 smithi162 ceph-mon[104668]: pgmap v1148: 89 pgs: 89 active+clean; 2.3 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 142 MiB/s wr, 2.28k op/s 2024-08-17T17:13:45.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:45 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/119950493' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:13:45.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:45 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:13:45.855 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:45.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:45.898 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:45.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:45.994 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:13:46.000 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:46.061 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 19 16 44858 44842 147.47 102.062 0.0111948 0.00677512 2024-08-17T17:13:46.084 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:46.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:46.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:46.185 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:46.211 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:46.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:46.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:46.313 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:46.338 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:46.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:46.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:46.440 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:46.485 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:46.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:46.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:46.587 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:46.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:46.615 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:46.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:46.716 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:46.758 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:46.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:46.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:46 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:13:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:46 smithi063 ceph-mon[114756]: osdmap e1094: 8 total, 8 up, 8 in 2024-08-17T17:13:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:46 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:13:46.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:46 smithi063 ceph-mon[109843]: osdmap e1094: 8 total, 8 up, 8 in 2024-08-17T17:13:46.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:46.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:46 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:13:46.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:46 smithi162 ceph-mon[104668]: osdmap e1094: 8 total, 8 up, 8 in 2024-08-17T17:13:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:13:46.860 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:46.882 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:46.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:46.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:46.984 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:47.015 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:47.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:47.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:47.062 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:13:47.062024+0000 min lat: 0.0014202 max lat: 0.151093 avg lat: 0.0068829 2024-08-17T17:13:47.062 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:13:47.062 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 20 16 46483 46467 145.174 101.562 0.00459646 0.0068829 2024-08-17T17:13:47.118 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:47.139 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:47.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:47.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:47.240 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:47.262 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:47.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:47.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:47.364 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:47.395 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:47.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:47.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:47.496 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:47.531 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:47.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:47.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:47.633 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:47.662 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:47.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:47.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:47.756 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:47 smithi063 ceph-mon[109843]: pgmap v1150: 89 pgs: 89 active+clean; 2.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-08-17T17:13:47.756 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:47 smithi063 ceph-mon[114756]: pgmap v1150: 89 pgs: 89 active+clean; 2.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-08-17T17:13:47.764 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:47.806 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:47.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:47.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:47.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:47 smithi162 ceph-mon[104668]: pgmap v1150: 89 pgs: 89 active+clean; 2.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s 2024-08-17T17:13:47.908 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:47.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:47.943 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:47.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:48.038 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:13:48.043 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:48.063 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 21 16 48190 48174 143.33 106.688 0.0223645 0.0069717 2024-08-17T17:13:48.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:48.119 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:48.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:48.220 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:48.248 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:48.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:48.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:48.350 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:48.379 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:48.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:48.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:48.481 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:48.526 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:48.538 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:48.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:48.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:48.640 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:48.672 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:48.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:48.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:48.774 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:48.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:48.828 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:48.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:48.929 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:48.965 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:48.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:48.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:49.063 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 22 15 49851 49836 141.537 103.875 0.00987165 0.0070605 2024-08-17T17:13:49.068 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:49.098 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:49.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:49.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:49.200 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:49.228 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:49.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:49.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:49.330 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:49.383 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:49.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:49.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:49.485 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:49.514 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:49.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:49.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:49.616 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:49.631 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:49 smithi063 ceph-mon[114756]: pgmap v1151: 89 pgs: 89 active+clean; 2.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s 2024-08-17T17:13:49.632 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:49 smithi063 ceph-mon[109843]: pgmap v1151: 89 pgs: 89 active+clean; 2.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s 2024-08-17T17:13:49.643 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:49.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:49.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:49.730 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:13:49.730 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:13:49.746 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:49.790 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:49.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:49.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:49.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:49 smithi162 ceph-mon[104668]: pgmap v1151: 89 pgs: 89 active+clean; 2.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.85k op/s 2024-08-17T17:13:49.892 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:49.918 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:49.919 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:49.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:49.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:50.021 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:50.063 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 23 16 51444 51428 139.709 99.5 0.0146537 0.00715177 2024-08-17T17:13:50.064 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:50.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:50.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:50.166 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:50.196 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:50.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:50.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:50.298 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:50.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:50.338 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:50.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:50.439 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:50.470 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:13:50.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:50.482 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:50.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:50.583 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:50.613 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:50.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:50.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:50.715 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:50.723 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:50 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3795180699' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:13:50.724 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:50 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3795180699' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:13:50.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:50.755 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:50.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:50.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:50 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3795180699' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:13:50.856 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:50.894 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:50.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:50.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:50.996 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:51.038 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:51.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:51.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:51.063 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 24 15 53172 53157 138.39 108.062 0.0110679 0.00722051 2024-08-17T17:13:51.141 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:51.177 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:51.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:51.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:51.278 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:51.316 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:51.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:51.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:51.418 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:51.448 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:51.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:51.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:51.501 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:13:51.550 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:51.601 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:51.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:51.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:51.703 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:51.731 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:51.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:51.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:51.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:51 smithi063 ceph-mon[114756]: pgmap v1152: 89 pgs: 89 active+clean; 2.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2024-08-17T17:13:51.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:51 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:13:51.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:51 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:13:51.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:51 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3795180699' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:13:51.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:51 smithi063 ceph-mon[114756]: osdmap e1095: 8 total, 8 up, 8 in 2024-08-17T17:13:51.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:51 smithi063 ceph-mon[109843]: pgmap v1152: 89 pgs: 89 active+clean; 2.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2024-08-17T17:13:51.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:51 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:13:51.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:51 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:13:51.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:51 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3795180699' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:13:51.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:51 smithi063 ceph-mon[109843]: osdmap e1095: 8 total, 8 up, 8 in 2024-08-17T17:13:51.833 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:51.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:51 smithi162 ceph-mon[104668]: pgmap v1152: 89 pgs: 89 active+clean; 2.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2024-08-17T17:13:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:51 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:13:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:51 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:13:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:51 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3795180699' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:13:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:51 smithi162 ceph-mon[104668]: osdmap e1095: 8 total, 8 up, 8 in 2024-08-17T17:13:51.861 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:51.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:51.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:51.962 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:51.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:51.994 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:51.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:52.064 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 25 16 54735 54719 136.76 97.625 0.0136041 0.0073069 2024-08-17T17:13:52.095 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:52.137 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:52.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:52.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:52.239 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:52.274 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:52.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:52.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:52.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:52.428 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:52.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:52.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:52.530 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:52.577 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:52.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:52.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:52.680 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:52.718 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:52.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:52.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:52.820 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:52.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:52.858 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:52.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:52.960 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:52.989 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:52.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:52.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:53.064 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 26 16 56341 56325 135.36 100.375 0.00771331 0.00738225 2024-08-17T17:13:53.091 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:53.112 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:53.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:53.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:53.214 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:53.249 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:53.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:53.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:53.351 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:53.381 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:53.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:53.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:53.483 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:53.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:53.529 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:53.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:53.551 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:53 smithi063 ceph-mon[109843]: pgmap v1154: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-08-17T17:13:53.551 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:13:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:13:53] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.51.0" 2024-08-17T17:13:53.551 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:53 smithi063 ceph-mon[114756]: pgmap v1154: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-08-17T17:13:53.630 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:53.666 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:53.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:53.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:53.768 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:53.801 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:53.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:53.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:53.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:53 smithi162 ceph-mon[104668]: pgmap v1154: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-08-17T17:13:53.871 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:53.903 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:53.931 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:13:53.946 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:53.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:53.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:54.048 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:54.064 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 27 16 57852 57836 133.844 94.4375 0.00797836 0.0074661 2024-08-17T17:13:54.076 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:54.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:54.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:54.177 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:54.228 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:54.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:54.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:54.330 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:54.358 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:54.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:54.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:54.460 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:54.503 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:54.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:54.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:54.605 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:54.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:54.632 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:54.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:54.733 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:54.752 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:13:54.753 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:13:54.753 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:13:54.753 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:13:54.753 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:13:54.753 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:13:54.753 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:13:54.753 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:13:54.762 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:54.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:54.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:54.791 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:13:54.791 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:13:54.837 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:13:54.837 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:13:54.863 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:54.884 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:13:54.884 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:13:54.890 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:54.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:54.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:54.927 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:13:54.927 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:13:54.992 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:55.018 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:55.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:55.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:55.064 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 28 16 59193 59177 132.058 83.8125 0.0277951 0.00756642 2024-08-17T17:13:55.120 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:55.150 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:55.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:55.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:55.252 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:55.297 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:55.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:55.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:55.399 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:55.436 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:55.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:55.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:55.523 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:13:55.539 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:55.568 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:55.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:55.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:55.623 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:55 smithi063 ceph-mon[114756]: pgmap v1155: 89 pgs: 89 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-08-17T17:13:55.624 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:55 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2547076904' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:13:55.624 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:55 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/84854316' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:13:55.624 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:55 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:13:55.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:55 smithi063 ceph-mon[109843]: pgmap v1155: 89 pgs: 89 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-08-17T17:13:55.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:55 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2547076904' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:13:55.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:55 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/84854316' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:13:55.624 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:55 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:13:55.670 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:55.704 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:55.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:55.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:55.805 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:55.836 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:55.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:55.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:55.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:55 smithi162 ceph-mon[104668]: pgmap v1155: 89 pgs: 89 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-08-17T17:13:55.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:55 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2547076904' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:13:55.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:55 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/84854316' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:13:55.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:55 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:13:55.938 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:55.979 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:55.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:55.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:56.064 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 29 16 60739 60723 130.836 96.625 0.00838795 0.00763795 2024-08-17T17:13:56.080 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:56.122 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:56.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:56.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:56.224 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:56.262 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:13:56.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:56.267 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:56.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:56.285 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:13:56.368 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:56.415 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:56.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:56.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:56.517 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:56.565 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:56.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:56.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:56.669 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:56.728 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:56.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:56.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:56.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:56 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:13:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:56 smithi063 ceph-mon[114756]: osdmap e1096: 8 total, 8 up, 8 in 2024-08-17T17:13:56.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:56 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:13:56.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:56 smithi063 ceph-mon[109843]: osdmap e1096: 8 total, 8 up, 8 in 2024-08-17T17:13:56.830 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:56.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:56 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:13:56.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:56 smithi162 ceph-mon[104668]: osdmap e1096: 8 total, 8 up, 8 in 2024-08-17T17:13:56.858 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:56.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:56.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:56.960 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:56.992 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:56.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:56.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:57.064 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 30 16 62068 62052 129.243 83.0625 0.00324068 0.00773106 2024-08-17T17:13:57.094 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:13:57.127 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:57.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:57.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:13:57.229 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:57.259 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:57.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:57.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:57.360 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:13:57.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:57.400 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:57.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:13:57.502 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:57.542 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:57.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:57.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:57.645 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:57.680 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:57.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:57.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:57.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:57 smithi063 ceph-mon[114756]: pgmap v1157: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-08-17T17:13:57.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:57 smithi063 ceph-mon[109843]: pgmap v1157: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-08-17T17:13:57.782 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:57.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:57.814 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:57.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:57.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:57 smithi162 ceph-mon[104668]: pgmap v1157: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2024-08-17T17:13:57.915 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:13:57.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:57.954 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:57.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:13:58.055 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:58.066 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 31 16 63554 63538 128.07 92.875 0.00874244 0.00780276 2024-08-17T17:13:58.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:58.098 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:58.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:58.199 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:58.240 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:58.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:58.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:58.342 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:58.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:58.375 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:58.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:58.477 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:58.513 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:58.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:58.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:58.615 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:13:58.653 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:58.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:58.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:13:58.755 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:13:58.778 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:58.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:58.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:13:58.880 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:58.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:58.914 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:58.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:59.015 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:59.036 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:59.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:59.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:59.064 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 32 16 64982 64966 126.857 89.25 0.0113079 0.00787754 2024-08-17T17:13:59.139 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:59.179 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:59.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:59.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:59.281 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:59.322 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:59.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:59.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:59.345 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:13:58 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=sqlstore.transactions t=2024-08-17T17:13:58.975863448Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-08-17T17:13:59.424 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:13:59.462 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:59.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:59.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:13:59.564 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:13:59.601 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:13:59.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:59.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:13:59.703 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:59.734 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:59.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:59.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:59.820 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:13:59 smithi063 ceph-mon[114756]: pgmap v1158: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-17T17:13:59.821 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:13:59 smithi063 ceph-mon[109843]: pgmap v1158: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-17T17:13:59.836 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:13:59.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:13:59 smithi162 ceph-mon[104668]: pgmap v1158: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2024-08-17T17:13:59.865 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:13:59.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:13:59.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:13:59.967 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:00.004 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:00.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:00.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:00.064 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 33 16 66443 66427 125.78 91.3125 0.0158568 0.00794458 2024-08-17T17:14:00.106 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:00.146 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:00.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:00.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:00.248 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:00.287 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:00.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:00.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:00.388 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:00.415 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:00.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:00.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:00.517 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:00.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:00.548 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:00.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:00.649 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:00.678 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:00.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:00.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:00.780 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:00.826 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:00.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:00.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:00.928 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:00.957 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:00.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:00.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:01.059 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:01.064 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 34 16 67925 67909 124.804 92.625 0.00884813 0.00800766 2024-08-17T17:14:01.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:01.090 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:01.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:01.092 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:01.191 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:01.251 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:01.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:01.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:01.353 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:01.384 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:01.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:01.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:01.425 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:14:01.425 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:14:01.425 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:14:01.425 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:14:01.425 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.425 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7038 5.c 5:398b4d62:::smithi06387500-8:head [delete] snapc 1e5=[1e5,1e4,1e3,1e2,1da,1d8,1d3,1cf,1c7,1ba] ondisk+write+peerstat_old+known_if_redirected+supports_pool_eio e771)", 2024-08-17T17:14:01.426 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:06:38.638485+0000", 2024-08-17T17:14:01.426 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 442.78317381800002, 2024-08-17T17:14:01.426 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.004926953, 2024-08-17T17:14:01.426 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.426 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.426 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.426 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.426 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:14:01.426 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:14:01.426 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7038 2024-08-17T17:14:01.426 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.427 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.427 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.427 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.427 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:38.638485+0000", 2024-08-17T17:14:01.427 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.427 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.427 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.427 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.427 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:38.638485+0000", 2024-08-17T17:14:01.427 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.427 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.427 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.427 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.428 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:38.638486+0000", 2024-08-17T17:14:01.428 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5099999999999999e-06 2024-08-17T17:14:01.428 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.428 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.428 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.428 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:38.638491+0000", 2024-08-17T17:14:01.428 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.215e-06 2024-08-17T17:14:01.428 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.428 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.428 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.428 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:38.638504+0000", 2024-08-17T17:14:01.428 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2150000000000001e-05 2024-08-17T17:14:01.429 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.429 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.429 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.429 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:38.638516+0000", 2024-08-17T17:14:01.429 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2429000000000001e-05 2024-08-17T17:14:01.429 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.429 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.429 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.429 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:38.638590+0000", 2024-08-17T17:14:01.429 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3814999999999997e-05 2024-08-17T17:14:01.429 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.429 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.429 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:14:01.430 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:38.638610+0000", 2024-08-17T17:14:01.430 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0239000000000001e-05 2024-08-17T17:14:01.430 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.430 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.430 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.430 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:39.636956+0000", 2024-08-17T17:14:01.430 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.99834622900000003 2024-08-17T17:14:01.430 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.430 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.430 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.430 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:39.637140+0000", 2024-08-17T17:14:01.430 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000183881 2024-08-17T17:14:01.431 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.431 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.431 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:14:01.431 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:39.637415+0000", 2024-08-17T17:14:01.431 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000275161 2024-08-17T17:14:01.431 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.431 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.431 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.431 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:39.643130+0000", 2024-08-17T17:14:01.431 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0057150830000000001 2024-08-17T17:14:01.431 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.431 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.432 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.432 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:39.643219+0000", 2024-08-17T17:14:01.432 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8339999999999998e-05 2024-08-17T17:14:01.432 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.432 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.432 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.432 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:39.643281+0000", 2024-08-17T17:14:01.432 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2700999999999995e-05 2024-08-17T17:14:01.432 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.432 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.432 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.432 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:39.643350+0000", 2024-08-17T17:14:01.433 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9070000000000001e-05 2024-08-17T17:14:01.433 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.433 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.433 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.433 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:39.643412+0000", 2024-08-17T17:14:01.433 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1130000000000006e-05 2024-08-17T17:14:01.433 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.433 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.433 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.433 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.433 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.433 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:24681 6.4 6:24853b47:::benchmark_data_smithi063_160268_object24680:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:01.433 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:35.250083+0000", 2024-08-17T17:14:01.434 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 26.171575596, 2024-08-17T17:14:01.434 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.067493977999999996, 2024-08-17T17:14:01.434 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.434 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.434 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.434 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.434 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.434 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.434 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 24681 2024-08-17T17:14:01.434 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.434 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.434 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.435 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.435 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.250083+0000", 2024-08-17T17:14:01.435 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.435 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.435 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.435 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.435 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.250083+0000", 2024-08-17T17:14:01.435 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.435 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.435 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.435 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.435 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.250084+0000", 2024-08-17T17:14:01.436 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3740000000000001e-06 2024-08-17T17:14:01.436 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.436 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.436 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.436 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.250107+0000", 2024-08-17T17:14:01.436 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2798000000000001e-05 2024-08-17T17:14:01.436 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.436 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.436 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.436 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.250118+0000", 2024-08-17T17:14:01.436 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1408e-05 2024-08-17T17:14:01.436 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.436 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.436 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.437 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.250139+0000", 2024-08-17T17:14:01.437 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0896000000000001e-05 2024-08-17T17:14:01.437 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.437 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.437 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.437 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.250275+0000", 2024-08-17T17:14:01.437 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013614500000000001 2024-08-17T17:14:01.437 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.437 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.437 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.437 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.250395+0000", 2024-08-17T17:14:01.437 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000119116 2024-08-17T17:14:01.438 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.438 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.438 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:14:01.438 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.250520+0000", 2024-08-17T17:14:01.438 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012559800000000001 2024-08-17T17:14:01.438 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.438 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.438 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.438 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.253179+0000", 2024-08-17T17:14:01.438 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002658667 2024-08-17T17:14:01.438 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.438 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.438 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.439 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.253186+0000", 2024-08-17T17:14:01.439 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4159999999999998e-06 2024-08-17T17:14:01.439 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.439 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.439 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.439 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.317487+0000", 2024-08-17T17:14:01.439 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.064300333000000001 2024-08-17T17:14:01.439 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.439 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.440 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.440 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.317540+0000", 2024-08-17T17:14:01.440 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3536999999999999e-05 2024-08-17T17:14:01.440 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.440 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.440 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.440 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.317577+0000", 2024-08-17T17:14:01.440 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6690000000000003e-05 2024-08-17T17:14:01.440 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.440 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.440 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.440 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.440 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.441 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:24682 6.4 6:2343f912:::benchmark_data_smithi063_160268_object24681:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:01.441 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:35.250765+0000", 2024-08-17T17:14:01.441 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 26.170893519, 2024-08-17T17:14:01.442 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.068917831999999998, 2024-08-17T17:14:01.442 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.442 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.442 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.442 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.442 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.442 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.442 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 24682 2024-08-17T17:14:01.442 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.442 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.443 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.443 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.443 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.250765+0000", 2024-08-17T17:14:01.443 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.443 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.443 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.443 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.443 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.250765+0000", 2024-08-17T17:14:01.443 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.443 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.443 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.443 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.444 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.250766+0000", 2024-08-17T17:14:01.444 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0800000000000004e-07 2024-08-17T17:14:01.444 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.444 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.444 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.444 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.250787+0000", 2024-08-17T17:14:01.444 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1384999999999999e-05 2024-08-17T17:14:01.444 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.444 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.444 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.444 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.250797+0000", 2024-08-17T17:14:01.444 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7659999999999993e-06 2024-08-17T17:14:01.444 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.445 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.445 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.445 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.250810+0000", 2024-08-17T17:14:01.445 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3437e-05 2024-08-17T17:14:01.445 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.445 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.445 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.445 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.251868+0000", 2024-08-17T17:14:01.445 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0010572559999999999 2024-08-17T17:14:01.445 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.445 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.446 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.446 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.251999+0000", 2024-08-17T17:14:01.446 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013117000000000001 2024-08-17T17:14:01.446 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.446 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.446 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:14:01.446 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.252140+0000", 2024-08-17T17:14:01.446 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014158399999999999 2024-08-17T17:14:01.446 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.446 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.446 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.446 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.253258+0000", 2024-08-17T17:14:01.446 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001117299 2024-08-17T17:14:01.447 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.447 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.447 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.447 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.253418+0000", 2024-08-17T17:14:01.447 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016039400000000001 2024-08-17T17:14:01.447 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.447 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.447 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.447 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.319573+0000", 2024-08-17T17:14:01.447 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.066154909999999997 2024-08-17T17:14:01.447 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.447 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.447 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.448 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.319633+0000", 2024-08-17T17:14:01.448 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0461000000000003e-05 2024-08-17T17:14:01.448 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.448 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.448 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.448 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:35.319683+0000", 2024-08-17T17:14:01.448 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9361999999999999e-05 2024-08-17T17:14:01.448 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.448 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.448 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.448 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.448 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.448 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:29049 6.4 6:23c29215:::benchmark_data_smithi063_160268_object29048:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:01.449 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:37.191314+0000", 2024-08-17T17:14:01.449 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 24.230344583000001, 2024-08-17T17:14:01.449 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.062536360999999999, 2024-08-17T17:14:01.449 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.449 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.449 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.449 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.449 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.449 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.449 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 29049 2024-08-17T17:14:01.449 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.449 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.449 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.450 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.450 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.191314+0000", 2024-08-17T17:14:01.450 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.450 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.450 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.450 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.450 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.191314+0000", 2024-08-17T17:14:01.450 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.450 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.450 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.450 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.450 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.191315+0000", 2024-08-17T17:14:01.450 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4750000000000001e-06 2024-08-17T17:14:01.451 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.451 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.451 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.451 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.191338+0000", 2024-08-17T17:14:01.451 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2174000000000001e-05 2024-08-17T17:14:01.451 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.451 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.451 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.451 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.191351+0000", 2024-08-17T17:14:01.451 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3044000000000001e-05 2024-08-17T17:14:01.451 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.451 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.451 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.452 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.191358+0000", 2024-08-17T17:14:01.452 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5970000000000003e-06 2024-08-17T17:14:01.452 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.452 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.452 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.452 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.191483+0000", 2024-08-17T17:14:01.452 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000124871 2024-08-17T17:14:01.452 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.452 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.452 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.452 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.191605+0000", 2024-08-17T17:14:01.452 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012233200000000001 2024-08-17T17:14:01.452 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.453 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.453 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:14:01.453 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.191727+0000", 2024-08-17T17:14:01.453 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000121787 2024-08-17T17:14:01.453 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.453 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.453 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.453 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.193370+0000", 2024-08-17T17:14:01.453 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001642424 2024-08-17T17:14:01.453 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.453 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.453 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.453 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.193556+0000", 2024-08-17T17:14:01.454 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00018647999999999999 2024-08-17T17:14:01.454 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.454 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.454 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.454 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.253736+0000", 2024-08-17T17:14:01.454 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.060179604999999997 2024-08-17T17:14:01.454 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.454 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.454 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.454 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.253803+0000", 2024-08-17T17:14:01.454 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7713999999999996e-05 2024-08-17T17:14:01.454 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.454 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.455 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.455 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.253850+0000", 2024-08-17T17:14:01.455 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6857999999999998e-05 2024-08-17T17:14:01.455 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.455 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.455 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.455 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.455 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.455 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:29056 6.4 6:282fb59f:::benchmark_data_smithi063_160268_object29055:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:01.455 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:37.194853+0000", 2024-08-17T17:14:01.455 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 24.226805751000001, 2024-08-17T17:14:01.455 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.067002570999999997, 2024-08-17T17:14:01.455 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.456 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.456 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.456 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.456 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.456 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.456 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 29056 2024-08-17T17:14:01.456 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.456 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.456 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.456 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.456 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.194853+0000", 2024-08-17T17:14:01.456 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.456 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.457 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.457 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.457 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.194853+0000", 2024-08-17T17:14:01.457 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.457 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.457 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.457 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.457 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.194854+0000", 2024-08-17T17:14:01.457 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2810000000000001e-06 2024-08-17T17:14:01.457 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.457 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.457 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.457 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.194870+0000", 2024-08-17T17:14:01.458 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5917e-05 2024-08-17T17:14:01.458 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.458 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.458 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.458 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.194879+0000", 2024-08-17T17:14:01.458 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.5100000000000004e-06 2024-08-17T17:14:01.458 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.458 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.458 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.458 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.194885+0000", 2024-08-17T17:14:01.458 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6400000000000002e-06 2024-08-17T17:14:01.458 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.458 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.459 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.459 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.194959+0000", 2024-08-17T17:14:01.459 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4218e-05 2024-08-17T17:14:01.459 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.459 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.459 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.459 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.195063+0000", 2024-08-17T17:14:01.459 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000103803 2024-08-17T17:14:01.459 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.459 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.459 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:14:01.459 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.195172+0000", 2024-08-17T17:14:01.459 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010916000000000001 2024-08-17T17:14:01.460 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.460 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.460 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.460 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.196089+0000", 2024-08-17T17:14:01.460 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00091665799999999999 2024-08-17T17:14:01.460 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.460 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.460 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.460 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.198010+0000", 2024-08-17T17:14:01.460 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0019207530000000001 2024-08-17T17:14:01.460 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.460 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.460 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.460 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.261657+0000", 2024-08-17T17:14:01.461 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.063647330000000002 2024-08-17T17:14:01.461 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.461 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.461 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.461 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.261805+0000", 2024-08-17T17:14:01.461 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000147532 2024-08-17T17:14:01.461 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.461 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.461 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.461 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.261855+0000", 2024-08-17T17:14:01.461 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0769000000000001e-05 2024-08-17T17:14:01.461 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.461 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.462 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.462 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.462 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.462 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:29060 6.4 6:2608e75d:::benchmark_data_smithi063_160268_object29059:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:01.462 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:37.196470+0000", 2024-08-17T17:14:01.462 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 24.225188583000001, 2024-08-17T17:14:01.462 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.06565066, 2024-08-17T17:14:01.462 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.462 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.462 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.462 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.462 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.462 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.463 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 29060 2024-08-17T17:14:01.463 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.463 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.463 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.463 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.463 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.196470+0000", 2024-08-17T17:14:01.463 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.463 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.463 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.463 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.463 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.196470+0000", 2024-08-17T17:14:01.463 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.463 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.464 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.464 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.464 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.196471+0000", 2024-08-17T17:14:01.464 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.2099999999999995e-07 2024-08-17T17:14:01.464 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.464 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.464 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.464 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.196488+0000", 2024-08-17T17:14:01.464 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7399999999999999e-05 2024-08-17T17:14:01.464 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.464 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.464 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.464 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.196497+0000", 2024-08-17T17:14:01.465 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6659999999999997e-06 2024-08-17T17:14:01.465 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.465 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.465 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.465 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.196502+0000", 2024-08-17T17:14:01.465 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0209999999999999e-06 2024-08-17T17:14:01.465 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.465 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.465 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.465 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.196626+0000", 2024-08-17T17:14:01.466 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012365100000000001 2024-08-17T17:14:01.466 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.466 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.466 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.466 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.196754+0000", 2024-08-17T17:14:01.466 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012886899999999999 2024-08-17T17:14:01.466 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.466 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.466 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:14:01.466 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.196904+0000", 2024-08-17T17:14:01.466 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014985899999999999 2024-08-17T17:14:01.466 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.466 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.467 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.467 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.198101+0000", 2024-08-17T17:14:01.467 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001197205 2024-08-17T17:14:01.467 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.467 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.467 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.467 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.198327+0000", 2024-08-17T17:14:01.467 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00022546099999999999 2024-08-17T17:14:01.467 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.467 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.467 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.467 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.261897+0000", 2024-08-17T17:14:01.467 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.063569867000000002 2024-08-17T17:14:01.468 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.468 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.468 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.468 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.262072+0000", 2024-08-17T17:14:01.468 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000175391 2024-08-17T17:14:01.468 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.468 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.468 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.468 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.262121+0000", 2024-08-17T17:14:01.468 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8349000000000001e-05 2024-08-17T17:14:01.468 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.468 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.468 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.468 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.469 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.469 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:29975 6.1 6:82e1ea0d:::benchmark_data_smithi063_160268_object29974:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:01.469 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:37.728080+0000", 2024-08-17T17:14:01.469 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 23.693578169999999, 2024-08-17T17:14:01.469 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.075472543000000003, 2024-08-17T17:14:01.469 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.469 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.469 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.469 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.469 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.469 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.469 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 29975 2024-08-17T17:14:01.470 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.470 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.470 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.470 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.470 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728080+0000", 2024-08-17T17:14:01.470 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.470 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.470 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.470 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.470 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728080+0000", 2024-08-17T17:14:01.470 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.470 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.470 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.471 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.471 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728082+0000", 2024-08-17T17:14:01.471 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.35e-06 2024-08-17T17:14:01.471 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.471 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.471 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.471 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728106+0000", 2024-08-17T17:14:01.471 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.438e-05 2024-08-17T17:14:01.471 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.471 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.471 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.471 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728119+0000", 2024-08-17T17:14:01.472 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.256e-05 2024-08-17T17:14:01.472 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.472 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.472 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.472 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728126+0000", 2024-08-17T17:14:01.472 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5650000000000004e-06 2024-08-17T17:14:01.472 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.472 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.472 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.472 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728194+0000", 2024-08-17T17:14:01.472 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8263999999999996e-05 2024-08-17T17:14:01.472 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.472 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.473 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.473 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728342+0000", 2024-08-17T17:14:01.473 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014800699999999999 2024-08-17T17:14:01.473 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.473 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.473 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:14:01.473 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728473+0000", 2024-08-17T17:14:01.473 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013097799999999999 2024-08-17T17:14:01.473 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.473 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.473 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.473 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.761767+0000", 2024-08-17T17:14:01.473 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.033293973999999997 2024-08-17T17:14:01.473 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.474 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.474 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.474 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.762011+0000", 2024-08-17T17:14:01.474 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000243315 2024-08-17T17:14:01.474 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.474 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.474 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.474 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.803446+0000", 2024-08-17T17:14:01.474 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.04143579 2024-08-17T17:14:01.474 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.474 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.474 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.475 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.803504+0000", 2024-08-17T17:14:01.475 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7759e-05 2024-08-17T17:14:01.475 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.475 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.475 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.475 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.803553+0000", 2024-08-17T17:14:01.475 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8600999999999998e-05 2024-08-17T17:14:01.475 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.475 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.475 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.475 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.475 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.476 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:30838 6.4 6:28e86723:::benchmark_data_smithi063_160268_object30837:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:01.476 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:38.205562+0000", 2024-08-17T17:14:01.476 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 23.216096207, 2024-08-17T17:14:01.476 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.063943714999999998, 2024-08-17T17:14:01.476 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.476 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.476 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.476 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.476 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.476 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.476 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 30838 2024-08-17T17:14:01.476 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.476 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.476 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.477 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.477 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:38.205562+0000", 2024-08-17T17:14:01.477 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.477 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.477 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.477 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.477 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:38.205562+0000", 2024-08-17T17:14:01.477 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.477 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.477 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.477 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.477 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:38.205563+0000", 2024-08-17T17:14:01.477 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4900000000000005e-07 2024-08-17T17:14:01.478 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.478 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.478 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.478 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:38.205581+0000", 2024-08-17T17:14:01.478 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7541e-05 2024-08-17T17:14:01.478 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.478 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.478 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.478 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:38.205589+0000", 2024-08-17T17:14:01.478 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9680000000000006e-06 2024-08-17T17:14:01.478 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.478 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.479 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.479 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:38.205593+0000", 2024-08-17T17:14:01.479 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8520000000000003e-06 2024-08-17T17:14:01.479 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.479 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.479 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.479 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:38.205633+0000", 2024-08-17T17:14:01.479 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.998e-05 2024-08-17T17:14:01.479 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.479 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.479 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.479 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:38.205732+0000", 2024-08-17T17:14:01.479 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9020999999999999e-05 2024-08-17T17:14:01.479 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.480 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.480 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:14:01.480 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:38.205852+0000", 2024-08-17T17:14:01.480 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011903700000000001 2024-08-17T17:14:01.480 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.480 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.480 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.480 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:38.206852+0000", 2024-08-17T17:14:01.480 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0010002990000000001 2024-08-17T17:14:01.480 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.480 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.480 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.480 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:38.207482+0000", 2024-08-17T17:14:01.481 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00062996299999999999 2024-08-17T17:14:01.481 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.481 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.481 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.481 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:38.269388+0000", 2024-08-17T17:14:01.481 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.061906434000000003 2024-08-17T17:14:01.481 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.481 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.481 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.481 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:38.269465+0000", 2024-08-17T17:14:01.481 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6577999999999998e-05 2024-08-17T17:14:01.481 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.481 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.482 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.482 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:38.269506+0000", 2024-08-17T17:14:01.482 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.1192999999999997e-05 2024-08-17T17:14:01.482 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.482 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.482 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.482 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.482 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.482 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:32643 6.4 6:28d4a114:::benchmark_data_smithi063_160268_object32642:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:01.482 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.131350+0000", 2024-08-17T17:14:01.482 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 22.290308394, 2024-08-17T17:14:01.482 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.086976158999999997, 2024-08-17T17:14:01.482 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.483 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.483 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.483 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.483 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.483 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.483 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 32643 2024-08-17T17:14:01.483 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.483 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.483 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.483 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.483 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131350+0000", 2024-08-17T17:14:01.483 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.483 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.483 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.484 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.484 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131350+0000", 2024-08-17T17:14:01.484 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.484 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.484 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.484 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.484 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131351+0000", 2024-08-17T17:14:01.484 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.195e-06 2024-08-17T17:14:01.484 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.484 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.484 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.484 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131377+0000", 2024-08-17T17:14:01.484 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5463e-05 2024-08-17T17:14:01.485 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.485 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.485 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.485 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131389+0000", 2024-08-17T17:14:01.485 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2147000000000001e-05 2024-08-17T17:14:01.485 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.485 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.485 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.485 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131396+0000", 2024-08-17T17:14:01.485 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:01.487 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1729999999999998e-06 2024-08-17T17:14:01.487 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.487 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.487 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.487 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131478+0000", 2024-08-17T17:14:01.487 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2399999999999997e-05 2024-08-17T17:14:01.487 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.487 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.487 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.488 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131625+0000", 2024-08-17T17:14:01.488 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014619000000000001 2024-08-17T17:14:01.488 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.488 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.488 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:14:01.488 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131770+0000", 2024-08-17T17:14:01.488 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014517800000000001 2024-08-17T17:14:01.488 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.488 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.488 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.488 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.133200+0000", 2024-08-17T17:14:01.488 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0014297069999999999 2024-08-17T17:14:01.488 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.489 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.489 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.489 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.135826+0000", 2024-08-17T17:14:01.489 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0026266179999999998 2024-08-17T17:14:01.489 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.489 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.489 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.489 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218155+0000", 2024-08-17T17:14:01.489 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.082328383000000005 2024-08-17T17:14:01.489 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.489 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.489 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.489 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218238+0000", 2024-08-17T17:14:01.490 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3920999999999998e-05 2024-08-17T17:14:01.490 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.490 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.490 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.490 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218326+0000", 2024-08-17T17:14:01.490 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7783999999999998e-05 2024-08-17T17:14:01.490 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.490 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.490 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.490 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.490 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.490 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:33167 6.4 6:22dca8bd:::benchmark_data_smithi063_160268_object33166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:01.491 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.448673+0000", 2024-08-17T17:14:01.491 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 21.972985802, 2024-08-17T17:14:01.491 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.094468580999999996, 2024-08-17T17:14:01.491 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.491 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.491 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.491 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.491 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.491 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.491 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 33167 2024-08-17T17:14:01.491 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.491 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.491 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.492 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.492 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448673+0000", 2024-08-17T17:14:01.492 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.492 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.492 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.492 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.492 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448673+0000", 2024-08-17T17:14:01.492 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.492 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.492 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.492 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.492 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448673+0000", 2024-08-17T17:14:01.492 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3000000000000001e-07 2024-08-17T17:14:01.493 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.493 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.493 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.493 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448693+0000", 2024-08-17T17:14:01.493 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9422000000000001e-05 2024-08-17T17:14:01.493 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.493 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.493 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.493 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448700+0000", 2024-08-17T17:14:01.493 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2779999999999996e-06 2024-08-17T17:14:01.493 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.493 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.494 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.494 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448706+0000", 2024-08-17T17:14:01.494 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6069999999999998e-06 2024-08-17T17:14:01.494 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.494 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.494 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.494 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.450351+0000", 2024-08-17T17:14:01.494 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001645526 2024-08-17T17:14:01.494 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.494 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.495 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.495 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.450469+0000", 2024-08-17T17:14:01.495 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000118239 2024-08-17T17:14:01.495 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.495 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.495 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:14:01.495 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.450598+0000", 2024-08-17T17:14:01.495 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012906300000000001 2024-08-17T17:14:01.495 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.495 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.495 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.495 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452120+0000", 2024-08-17T17:14:01.496 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0015215090000000001 2024-08-17T17:14:01.496 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.496 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.496 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.496 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452519+0000", 2024-08-17T17:14:01.496 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00039886099999999997 2024-08-17T17:14:01.496 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.496 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.496 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.496 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.543022+0000", 2024-08-17T17:14:01.496 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.090503086999999996 2024-08-17T17:14:01.496 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.496 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.497 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.497 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.543091+0000", 2024-08-17T17:14:01.497 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8806000000000005e-05 2024-08-17T17:14:01.497 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.497 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.497 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.497 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.543141+0000", 2024-08-17T17:14:01.497 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0652999999999998e-05 2024-08-17T17:14:01.497 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.497 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.497 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.497 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.497 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.498 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:33173 6.4 6:23580cad:::benchmark_data_smithi063_160268_object33172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:01.498 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.452560+0000", 2024-08-17T17:14:01.498 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 21.969098121999998, 2024-08-17T17:14:01.498 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.094171489999999997, 2024-08-17T17:14:01.498 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.498 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.498 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.498 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.498 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.498 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.498 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 33173 2024-08-17T17:14:01.498 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.499 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.499 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.499 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.499 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452560+0000", 2024-08-17T17:14:01.499 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.499 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.499 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.499 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.499 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452560+0000", 2024-08-17T17:14:01.499 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.499 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.499 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.500 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.500 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452562+0000", 2024-08-17T17:14:01.500 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5039999999999999e-06 2024-08-17T17:14:01.500 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.500 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.500 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.500 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452584+0000", 2024-08-17T17:14:01.500 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2578999999999998e-05 2024-08-17T17:14:01.500 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.500 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.500 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.500 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452595+0000", 2024-08-17T17:14:01.500 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0512e-05 2024-08-17T17:14:01.501 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.501 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.501 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.501 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452602+0000", 2024-08-17T17:14:01.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5710000000000003e-06 2024-08-17T17:14:01.501 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.501 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.501 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.501 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452692+0000", 2024-08-17T17:14:01.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0767999999999996e-05 2024-08-17T17:14:01.502 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.502 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.502 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.502 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452831+0000", 2024-08-17T17:14:01.502 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000138331 2024-08-17T17:14:01.502 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.502 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.502 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:14:01.502 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452965+0000", 2024-08-17T17:14:01.502 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013412600000000001 2024-08-17T17:14:01.502 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.503 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.503 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.503 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455446+0000", 2024-08-17T17:14:01.503 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0024813000000000001 2024-08-17T17:14:01.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.503 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.503 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.503 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539987+0000", 2024-08-17T17:14:01.503 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.084540709000000006 2024-08-17T17:14:01.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.503 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.503 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.504 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546616+0000", 2024-08-17T17:14:01.504 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0066297400000000003 2024-08-17T17:14:01.504 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.504 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.504 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.504 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546672+0000", 2024-08-17T17:14:01.504 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5600000000000003e-05 2024-08-17T17:14:01.504 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.504 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.504 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.504 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546732+0000", 2024-08-17T17:14:01.504 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9750000000000002e-05 2024-08-17T17:14:01.505 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.505 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.505 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.505 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.505 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.505 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:33174 6.4 6:2e7cd2fc:::benchmark_data_smithi063_160268_object33173:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:01.505 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.452620+0000", 2024-08-17T17:14:01.505 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 21.969038744999999, 2024-08-17T17:14:01.505 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.094311133000000005, 2024-08-17T17:14:01.505 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.505 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.505 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.506 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.506 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.506 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.506 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 33174 2024-08-17T17:14:01.506 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.506 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.506 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.506 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.506 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452620+0000", 2024-08-17T17:14:01.506 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.506 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.506 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.506 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.507 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452620+0000", 2024-08-17T17:14:01.507 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.507 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.507 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.507 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.507 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452620+0000", 2024-08-17T17:14:01.507 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8100000000000003e-07 2024-08-17T17:14:01.507 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.507 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.507 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.507 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452641+0000", 2024-08-17T17:14:01.507 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0205000000000001e-05 2024-08-17T17:14:01.508 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.508 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.508 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.508 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452648+0000", 2024-08-17T17:14:01.508 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0829999999999998e-06 2024-08-17T17:14:01.508 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.508 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.508 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.508 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452653+0000", 2024-08-17T17:14:01.508 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1420000000000001e-06 2024-08-17T17:14:01.508 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.508 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.509 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.509 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.454367+0000", 2024-08-17T17:14:01.509 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001713881 2024-08-17T17:14:01.509 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.509 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.509 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.509 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.454477+0000", 2024-08-17T17:14:01.509 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011024299999999999 2024-08-17T17:14:01.509 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.509 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.509 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:14:01.509 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.454554+0000", 2024-08-17T17:14:01.510 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7177999999999999e-05 2024-08-17T17:14:01.510 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.510 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.510 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.510 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455452+0000", 2024-08-17T17:14:01.510 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000898411 2024-08-17T17:14:01.510 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.510 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.510 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.510 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.540026+0000", 2024-08-17T17:14:01.510 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.084573537000000004 2024-08-17T17:14:01.510 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.511 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.511 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.511 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546819+0000", 2024-08-17T17:14:01.511 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.006792518 2024-08-17T17:14:01.511 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.511 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.511 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.511 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546874+0000", 2024-08-17T17:14:01.511 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5748000000000002e-05 2024-08-17T17:14:01.511 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.511 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.511 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.512 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546931+0000", 2024-08-17T17:14:01.512 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6606e-05 2024-08-17T17:14:01.512 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.512 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.512 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.512 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.512 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.512 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:33176 6.1 6:8a507c7a:::benchmark_data_smithi063_160268_object33175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:01.512 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.453381+0000", 2024-08-17T17:14:01.512 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 21.968277588999999, 2024-08-17T17:14:01.512 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.088080944999999994, 2024-08-17T17:14:01.512 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.512 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.513 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.513 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.513 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.513 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.513 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 33176 2024-08-17T17:14:01.513 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.513 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.513 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.513 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.513 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453381+0000", 2024-08-17T17:14:01.513 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.513 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.513 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.514 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.514 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453381+0000", 2024-08-17T17:14:01.514 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.514 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.514 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.514 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.514 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453382+0000", 2024-08-17T17:14:01.514 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.147e-06 2024-08-17T17:14:01.514 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.514 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.514 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.514 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453416+0000", 2024-08-17T17:14:01.515 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3822000000000002e-05 2024-08-17T17:14:01.515 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.515 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.515 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.515 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453427+0000", 2024-08-17T17:14:01.515 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1328000000000001e-05 2024-08-17T17:14:01.515 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.515 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.515 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.515 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453435+0000", 2024-08-17T17:14:01.515 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8869999999999992e-06 2024-08-17T17:14:01.515 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.515 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.516 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.516 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453492+0000", 2024-08-17T17:14:01.516 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7043000000000003e-05 2024-08-17T17:14:01.516 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.516 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.516 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.516 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453612+0000", 2024-08-17T17:14:01.516 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000119416 2024-08-17T17:14:01.516 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.516 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.516 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:14:01.516 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453744+0000", 2024-08-17T17:14:01.516 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000132211 2024-08-17T17:14:01.517 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.517 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.517 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455034+0000", 2024-08-17T17:14:01.517 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001289915 2024-08-17T17:14:01.517 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.517 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.517 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455499+0000", 2024-08-17T17:14:01.517 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00046549900000000002 2024-08-17T17:14:01.517 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.517 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.518 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541347+0000", 2024-08-17T17:14:01.518 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.085847657999999993 2024-08-17T17:14:01.518 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.518 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.518 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.518 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541412+0000", 2024-08-17T17:14:01.518 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5417000000000001e-05 2024-08-17T17:14:01.518 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.518 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.518 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.518 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541462+0000", 2024-08-17T17:14:01.518 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9601999999999997e-05 2024-08-17T17:14:01.518 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.519 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.519 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.519 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.519 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.519 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:33181 6.d e1093/1086 6:b650d8fa:::benchmark_data_smithi063_160268_object33180:head v 1093'2025, mlcod=1093'2023)", 2024-08-17T17:14:01.519 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.455644+0000", 2024-08-17T17:14:01.519 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 21.966014234999999, 2024-08-17T17:14:01.519 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.087465092999999994, 2024-08-17T17:14:01.519 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.519 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.519 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.519 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.519 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.520 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.520 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455644+0000", 2024-08-17T17:14:01.520 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.520 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.520 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.520 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.520 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455644+0000", 2024-08-17T17:14:01.520 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.520 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.520 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.520 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.520 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455645+0000", 2024-08-17T17:14:01.520 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9599999999999999e-07 2024-08-17T17:14:01.521 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.521 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.521 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.521 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455708+0000", 2024-08-17T17:14:01.521 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2724000000000005e-05 2024-08-17T17:14:01.521 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.521 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.521 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.521 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455715+0000", 2024-08-17T17:14:01.521 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8050000000000007e-06 2024-08-17T17:14:01.521 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.521 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.521 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.521 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455720+0000", 2024-08-17T17:14:01.522 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8559999999999998e-06 2024-08-17T17:14:01.522 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.522 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.522 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.522 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455766+0000", 2024-08-17T17:14:01.522 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6258999999999999e-05 2024-08-17T17:14:01.522 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.522 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.522 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.522 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455785+0000", 2024-08-17T17:14:01.522 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8329e-05 2024-08-17T17:14:01.522 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.522 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.523 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.523 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.543016+0000", 2024-08-17T17:14:01.523 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.087231031000000001 2024-08-17T17:14:01.523 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.523 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.523 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.523 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.543109+0000", 2024-08-17T17:14:01.523 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3492999999999994e-05 2024-08-17T17:14:01.523 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.523 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.523 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.523 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.523 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.524 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.44609.0:33176 6.1 e1093/1086 ondisk, result = 0)", 2024-08-17T17:14:01.524 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.456246+0000", 2024-08-17T17:14:01.524 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 21.965412843999999, 2024-08-17T17:14:01.524 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.085228506999999995, 2024-08-17T17:14:01.524 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.524 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.524 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:14:01.524 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.524 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.524 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.524 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456246+0000", 2024-08-17T17:14:01.524 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.524 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.525 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.525 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.525 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456246+0000", 2024-08-17T17:14:01.525 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.525 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.525 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.525 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.525 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456246+0000", 2024-08-17T17:14:01.525 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9800000000000003e-07 2024-08-17T17:14:01.525 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.525 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.525 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.525 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456250+0000", 2024-08-17T17:14:01.526 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8519999999999997e-06 2024-08-17T17:14:01.526 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.526 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.526 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.526 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541269+0000", 2024-08-17T17:14:01.526 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.085018446999999997 2024-08-17T17:14:01.526 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.526 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.526 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.526 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541279+0000", 2024-08-17T17:14:01.526 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0559e-05 2024-08-17T17:14:01.526 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.526 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.527 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.527 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541333+0000", 2024-08-17T17:14:01.527 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3792999999999998e-05 2024-08-17T17:14:01.527 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.527 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.527 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.527 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541341+0000", 2024-08-17T17:14:01.527 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2709999999999992e-06 2024-08-17T17:14:01.527 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.527 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.527 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.527 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541474+0000", 2024-08-17T17:14:01.527 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013288699999999999 2024-08-17T17:14:01.528 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.528 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.528 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.528 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.528 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.528 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.44609.0:33173 6.4 e1093/1086 ondisk, result = 0)", 2024-08-17T17:14:01.528 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.456941+0000", 2024-08-17T17:14:01.528 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 21.964717403999998, 2024-08-17T17:14:01.528 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.083053689999999999, 2024-08-17T17:14:01.528 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.528 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.528 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:14:01.528 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.529 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456941+0000", 2024-08-17T17:14:01.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.529 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.529 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456941+0000", 2024-08-17T17:14:01.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.529 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.529 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456942+0000", 2024-08-17T17:14:01.530 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2099999999999997e-07 2024-08-17T17:14:01.530 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.530 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.530 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.530 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456945+0000", 2024-08-17T17:14:01.530 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0970000000000002e-06 2024-08-17T17:14:01.530 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.530 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.530 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.530 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539891+0000", 2024-08-17T17:14:01.530 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.082946110000000003 2024-08-17T17:14:01.530 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.530 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.531 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.531 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539902+0000", 2024-08-17T17:14:01.531 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1094000000000001e-05 2024-08-17T17:14:01.531 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.531 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.531 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.531 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539971+0000", 2024-08-17T17:14:01.531 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9148999999999998e-05 2024-08-17T17:14:01.531 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.531 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.531 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.531 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539980+0000", 2024-08-17T17:14:01.532 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0359999999999995e-06 2024-08-17T17:14:01.532 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.532 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.532 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.532 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539995+0000", 2024-08-17T17:14:01.532 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4683e-05 2024-08-17T17:14:01.532 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.532 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.532 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.532 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.532 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.534 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34219 6.4 6:220ade53:::benchmark_data_smithi063_160268_object34218:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:01.534 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.044051+0000", 2024-08-17T17:14:01.534 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 21.377607081000001, 2024-08-17T17:14:01.534 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.091717032000000004, 2024-08-17T17:14:01.534 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.534 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.534 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.534 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.534 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.535 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.535 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34219 2024-08-17T17:14:01.535 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.535 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.535 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.535 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.535 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044051+0000", 2024-08-17T17:14:01.535 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.535 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.535 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.535 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.535 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044051+0000", 2024-08-17T17:14:01.535 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.536 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.536 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.536 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.536 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044052+0000", 2024-08-17T17:14:01.536 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4900000000000005e-07 2024-08-17T17:14:01.536 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.536 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.536 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.536 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044074+0000", 2024-08-17T17:14:01.536 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1696999999999999e-05 2024-08-17T17:14:01.536 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.536 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.536 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.537 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044084+0000", 2024-08-17T17:14:01.537 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0482000000000001e-05 2024-08-17T17:14:01.537 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.537 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.537 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.537 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044091+0000", 2024-08-17T17:14:01.537 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9600000000000003e-06 2024-08-17T17:14:01.537 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.537 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.537 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.537 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044144+0000", 2024-08-17T17:14:01.537 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3177000000000003e-05 2024-08-17T17:14:01.537 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.538 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.538 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.538 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044238+0000", 2024-08-17T17:14:01.538 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3733000000000005e-05 2024-08-17T17:14:01.538 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.538 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.538 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:14:01.538 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044316+0000", 2024-08-17T17:14:01.538 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7700999999999993e-05 2024-08-17T17:14:01.538 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.538 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.538 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.538 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.046094+0000", 2024-08-17T17:14:01.539 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0017784140000000001 2024-08-17T17:14:01.539 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.539 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.539 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.539 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.046102+0000", 2024-08-17T17:14:01.539 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8790000000000003e-06 2024-08-17T17:14:01.539 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.539 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.539 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.539 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.135653+0000", 2024-08-17T17:14:01.539 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.089550600999999994 2024-08-17T17:14:01.539 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.539 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.540 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.540 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.135720+0000", 2024-08-17T17:14:01.540 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7187999999999994e-05 2024-08-17T17:14:01.540 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.540 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.540 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.540 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.135768+0000", 2024-08-17T17:14:01.540 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8451000000000001e-05 2024-08-17T17:14:01.540 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.540 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.540 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.540 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.541 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.541 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35822 6.4 6:2f33fed9:::benchmark_data_smithi063_160268_object35821:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:01.541 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.883928+0000", 2024-08-17T17:14:01.541 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 20.537730937999999, 2024-08-17T17:14:01.541 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14375861300000001, 2024-08-17T17:14:01.541 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.541 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.541 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.541 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.541 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.541 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.541 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35822 2024-08-17T17:14:01.541 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.542 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.542 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.542 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.542 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883928+0000", 2024-08-17T17:14:01.542 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.542 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.542 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.542 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.542 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883928+0000", 2024-08-17T17:14:01.542 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.542 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.542 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.543 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.543 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883929+0000", 2024-08-17T17:14:01.543 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3459999999999999e-06 2024-08-17T17:14:01.543 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.543 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.543 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.543 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883947+0000", 2024-08-17T17:14:01.543 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8067000000000002e-05 2024-08-17T17:14:01.543 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.543 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.543 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.543 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883957+0000", 2024-08-17T17:14:01.543 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0179e-05 2024-08-17T17:14:01.544 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.544 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.544 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.544 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883963+0000", 2024-08-17T17:14:01.544 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.57e-06 2024-08-17T17:14:01.544 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.544 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.544 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.544 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.884008+0000", 2024-08-17T17:14:01.544 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5207000000000003e-05 2024-08-17T17:14:01.544 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.544 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.544 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.884091+0000", 2024-08-17T17:14:01.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3162000000000001e-05 2024-08-17T17:14:01.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.545 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.545 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:14:01.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.884169+0000", 2024-08-17T17:14:01.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8159999999999997e-05 2024-08-17T17:14:01.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.545 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.545 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.885227+0000", 2024-08-17T17:14:01.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0010574099999999999 2024-08-17T17:14:01.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.546 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.546 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.885625+0000", 2024-08-17T17:14:01.546 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00039848699999999999 2024-08-17T17:14:01.546 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.546 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.546 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027581+0000", 2024-08-17T17:14:01.546 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.141956267 2024-08-17T17:14:01.546 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.546 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.546 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027642+0000", 2024-08-17T17:14:01.547 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0757000000000001e-05 2024-08-17T17:14:01.547 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.547 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.547 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.547 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027686+0000", 2024-08-17T17:14:01.547 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4001000000000001e-05 2024-08-17T17:14:01.547 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.547 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.547 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.547 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.547 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.547 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:61715 6.4 6:275f6f39:::benchmark_data_smithi063_160268_object61714:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-08-17T17:14:01.547 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:56.780221+0000", 2024-08-17T17:14:01.548 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 4.6414378420000002, 2024-08-17T17:14:01.548 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.065618484000000005, 2024-08-17T17:14:01.548 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.548 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.548 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.548 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.548 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.548 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.548 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 61715 2024-08-17T17:14:01.548 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.548 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.548 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.549 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.549 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.780221+0000", 2024-08-17T17:14:01.549 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.549 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.549 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.549 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.549 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.780221+0000", 2024-08-17T17:14:01.549 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.549 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.549 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.549 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.549 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.780222+0000", 2024-08-17T17:14:01.549 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.753e-06 2024-08-17T17:14:01.549 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.550 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.550 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.550 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.780249+0000", 2024-08-17T17:14:01.550 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6279e-05 2024-08-17T17:14:01.550 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.550 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.550 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.550 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.780261+0000", 2024-08-17T17:14:01.550 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2772000000000001e-05 2024-08-17T17:14:01.550 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.550 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.550 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.551 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.780270+0000", 2024-08-17T17:14:01.551 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.1689999999999992e-06 2024-08-17T17:14:01.551 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.551 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.551 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.551 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.780538+0000", 2024-08-17T17:14:01.551 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00026812500000000003 2024-08-17T17:14:01.551 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.551 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.551 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.551 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.780657+0000", 2024-08-17T17:14:01.551 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000119249 2024-08-17T17:14:01.552 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.552 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.552 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:14:01.552 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.780801+0000", 2024-08-17T17:14:01.552 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014420200000000001 2024-08-17T17:14:01.552 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.552 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.552 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.552 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.783527+0000", 2024-08-17T17:14:01.552 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002725824 2024-08-17T17:14:01.552 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.552 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.553 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.553 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.786928+0000", 2024-08-17T17:14:01.553 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0034006930000000002 2024-08-17T17:14:01.553 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.553 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.553 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.553 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.845728+0000", 2024-08-17T17:14:01.553 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.058800753999999997 2024-08-17T17:14:01.553 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.553 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.553 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.553 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.845794+0000", 2024-08-17T17:14:01.553 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5203000000000006e-05 2024-08-17T17:14:01.554 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.554 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.554 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.554 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.845839+0000", 2024-08-17T17:14:01.554 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5460999999999997e-05 2024-08-17T17:14:01.554 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.554 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.554 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.554 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.554 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.554 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:61725 6.4 6:21aaa24f:::benchmark_data_smithi063_160268_object61724:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-08-17T17:14:01.554 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:56.784405+0000", 2024-08-17T17:14:01.555 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 4.6372529739999999, 2024-08-17T17:14:01.555 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.063348900999999999, 2024-08-17T17:14:01.555 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:01.555 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:01.555 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:01.555 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:01.555 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:01.555 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:01.555 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 61725 2024-08-17T17:14:01.555 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.555 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:01.555 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.555 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:01.556 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.784405+0000", 2024-08-17T17:14:01.556 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.556 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.556 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.556 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:01.556 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.784405+0000", 2024-08-17T17:14:01.556 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:01.556 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.556 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.556 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:01.556 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.784407+0000", 2024-08-17T17:14:01.556 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.359e-06 2024-08-17T17:14:01.556 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.557 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.557 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:01.557 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.784430+0000", 2024-08-17T17:14:01.557 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3532000000000001e-05 2024-08-17T17:14:01.557 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.557 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.557 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:01.557 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.784443+0000", 2024-08-17T17:14:01.557 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2347999999999999e-05 2024-08-17T17:14:01.557 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.557 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.557 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:01.557 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.784450+0000", 2024-08-17T17:14:01.558 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0489999999999998e-06 2024-08-17T17:14:01.558 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.558 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.558 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:01.558 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.784558+0000", 2024-08-17T17:14:01.558 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000108701 2024-08-17T17:14:01.558 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.558 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.558 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:01.558 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.784693+0000", 2024-08-17T17:14:01.558 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013477000000000001 2024-08-17T17:14:01.559 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.559 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.559 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:14:01.559 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.784843+0000", 2024-08-17T17:14:01.559 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000149395 2024-08-17T17:14:01.559 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.559 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.559 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.559 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.787532+0000", 2024-08-17T17:14:01.559 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0026894689999999999 2024-08-17T17:14:01.559 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.559 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.559 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:01.560 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.787857+0000", 2024-08-17T17:14:01.560 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00032482799999999999 2024-08-17T17:14:01.560 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.560 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.560 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:01.560 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.847591+0000", 2024-08-17T17:14:01.560 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.059733709000000003 2024-08-17T17:14:01.560 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.560 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.560 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:01.560 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.847698+0000", 2024-08-17T17:14:01.560 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010762 2024-08-17T17:14:01.560 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:01.561 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:01.561 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:01.561 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.847754+0000", 2024-08-17T17:14:01.561 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6121e-05 2024-08-17T17:14:01.561 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.561 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.561 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.561 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:01.561 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:01.561 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:14:01.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:01.562 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:01.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:01.640 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:01 smithi063 ceph-mon[114756]: pgmap v1159: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-08-17T17:14:01.641 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:01.641 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:01.641 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:01 smithi063 ceph-mon[109843]: pgmap v1159: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-08-17T17:14:01.641 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:01.641 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:01.663 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:01.692 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:01.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:01.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:01.793 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:01.828 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:01.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:01.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:01.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:01 smithi162 ceph-mon[104668]: pgmap v1159: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-08-17T17:14:01.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:01.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:01.930 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:01.973 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:01.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:01.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:01.975 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:14:02.064 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 35 16 69252 69236 123.608 82.9375 0.0194503 0.00808499 2024-08-17T17:14:02.075 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:02.101 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:02.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:02.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:02.203 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:02.245 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:02.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:02.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:02.347 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:02.388 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:02.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:02.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:02.490 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:02.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:02.515 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:02.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:02.616 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:02.649 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:02.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:02.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:02.750 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:02.776 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:02.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:02.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:02.878 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:02.917 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:02.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:02.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:03.019 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:03.053 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:03.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:03.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:03.064 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 36 16 70594 70578 122.504 83.875 0.0104029 0.00815843 2024-08-17T17:14:03.155 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:03.181 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:03.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:03.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:03.283 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:03.313 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:03.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:03.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:03.415 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:03.439 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:03.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:03.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:03.541 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:03.550 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:14:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:14:03] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.51.0" 2024-08-17T17:14:03.571 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:03.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:03.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:03.673 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:03.710 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:03.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:03.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:03.813 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:03.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:03 smithi162 ceph-mon[104668]: pgmap v1160: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:14:03.851 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:03.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:03.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:03.854 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:03 smithi063 ceph-mon[109843]: pgmap v1160: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:14:03.854 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:03 smithi063 ceph-mon[114756]: pgmap v1160: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:14:03.953 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:03.979 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:03.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:03.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:04.065 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 37 16 71917 71901 121.428 82.6875 0.00830859 0.00822906 2024-08-17T17:14:04.080 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:04.112 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:04.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:04.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:04.214 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:04.262 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:04.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:04.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:04.364 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:04.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:04.404 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:04.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:04.505 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:04.542 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:04.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:04.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:04.644 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:04.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:04.690 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:04.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:04.792 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:04.823 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:04.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:04.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:04.925 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:04.975 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:04.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:04.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:05.065 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 38 16 73247 73231 120.42 83.125 0.00607394 0.00829921 2024-08-17T17:14:05.078 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:05.111 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:05.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:05.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:05.213 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:05.248 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:05.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:05.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:05.350 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:05.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:05.410 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:05.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:05.511 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:05.550 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:05.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:05.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:05.652 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:05.690 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:05.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:05.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:05.757 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:05 smithi063 ceph-mon[109843]: pgmap v1161: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2024-08-17T17:14:05.758 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:05 smithi063 ceph-mon[114756]: pgmap v1161: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2024-08-17T17:14:05.791 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:05.796 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:14:05.833 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:05.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:05.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:05.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:05 smithi162 ceph-mon[104668]: pgmap v1161: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.45k op/s 2024-08-17T17:14:05.936 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:05.962 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:05.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:05.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:06.064 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:06.065 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 39 16 74499 74483 119.338 78.25 0.0129328 0.00837352 2024-08-17T17:14:06.091 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:06.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:06.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:06.193 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:06.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:06.243 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:06.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:06.344 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:06.371 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:06.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:06.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:06.473 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:06.503 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:06.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:06.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:06.582 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:14:06.605 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:06.650 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:06.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:06.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:06.751 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:06.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:06.792 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:06.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:06.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:06 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1474560306' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:14:06.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:06 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:14:06.854 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:06.893 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:06.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:06.950 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:06.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:06.987 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:06 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1474560306' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:14:06.987 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:06 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:14:06.987 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:06 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1474560306' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:14:06.987 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:06 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:14:07.051 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:07.066 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:14:07.066144+0000 min lat: 0.0014202 max lat: 0.151093 avg lat: 0.00845941 2024-08-17T17:14:07.066 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:14:07.066 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 40 16 75641 75625 118.138 71.375 0.0088057 0.00845941 2024-08-17T17:14:07.091 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:07.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:07.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:07.192 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:07.229 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:07.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:07.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:07.331 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:07.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:07.378 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:07.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:07.384 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:07.480 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:07.509 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:07.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:07.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:07.580 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:14:07.611 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:07.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:07.642 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:07.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:07.743 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:07.783 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:07.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:07.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:07.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:07 smithi162 ceph-mon[104668]: pgmap v1162: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:14:07.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:07 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:14:07.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:07 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:14:07.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:07 smithi162 ceph-mon[104668]: osdmap e1097: 8 total, 8 up, 8 in 2024-08-17T17:14:07.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:07 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2758579412' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:14:07.885 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:07.902 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:07 smithi063 ceph-mon[114756]: pgmap v1162: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:14:07.903 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:07 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:14:07.903 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:07 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:14:07.903 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:07 smithi063 ceph-mon[114756]: osdmap e1097: 8 total, 8 up, 8 in 2024-08-17T17:14:07.903 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:07 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2758579412' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:14:07.903 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:07 smithi063 ceph-mon[109843]: pgmap v1162: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:14:07.903 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:07 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:14:07.903 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:07 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:14:07.903 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:07 smithi063 ceph-mon[109843]: osdmap e1097: 8 total, 8 up, 8 in 2024-08-17T17:14:07.903 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:07 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2758579412' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:14:07.913 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:07.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:07.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:08.016 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:08.058 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:08.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:08.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:08.066 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 41 16 76939 76923 117.235 81.125 0.0155698 0.00852532 2024-08-17T17:14:08.160 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:08.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:08.193 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:08.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:08.294 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:08.334 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:08.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:08.436 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:08.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:08.475 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:08.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:08.576 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:08.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:08.606 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:08.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:08.692 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:14:08.707 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:08.785 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:08.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:08.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:08.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:08 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2758579412' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:14:08.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:08 smithi162 ceph-mon[104668]: osdmap e1098: 8 total, 8 up, 8 in 2024-08-17T17:14:08.888 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:08.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:08 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2758579412' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:14:08.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:08 smithi063 ceph-mon[109843]: osdmap e1098: 8 total, 8 up, 8 in 2024-08-17T17:14:08.913 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:08 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2758579412' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:14:08.913 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:08 smithi063 ceph-mon[114756]: osdmap e1098: 8 total, 8 up, 8 in 2024-08-17T17:14:08.917 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:08.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:08.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:09.019 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:09.065 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:09.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:09.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:09.067 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 42 16 78224 78208 116.355 80.3125 0.0130966 0.00858904 2024-08-17T17:14:09.167 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:09.195 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:09.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:09.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:09.297 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:09.299 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:14:09.325 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:09.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:09.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:09.427 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:09.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:09.466 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:09.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:09.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:09.615 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:09.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:09.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:09.717 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:09.754 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:09.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:09.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:09.856 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:09.898 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:09.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:09.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:10.000 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:09 smithi063 ceph-mon[114756]: pgmap v1165: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-08-17T17:14:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:09 smithi063 ceph-mon[109843]: pgmap v1165: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-08-17T17:14:10.045 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:10.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:10.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:10.066 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 43 16 79498 79482 115.5 79.625 0.00835235 0.00865299 2024-08-17T17:14:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:09 smithi162 ceph-mon[104668]: pgmap v1165: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-08-17T17:14:10.147 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:10.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:10.198 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:10.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:10.299 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:10.329 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:10.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:10.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:10.431 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:10.469 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:10.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:10.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:10.571 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:10.609 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:10.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:10.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:10.710 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:10.748 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:10.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:10.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:10.850 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:10.882 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:10.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:10.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:10.984 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:11.037 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:11.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:11.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:11.067 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 44 16 80764 80748 114.671 79.125 0.0130054 0.00871534 2024-08-17T17:14:11.139 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:11.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:11.172 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:11.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:11.273 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:11.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:11.320 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:11.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:11.421 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:11.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:11.468 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:11.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:11.569 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:11.618 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:11.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:11.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:11.720 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:11.741 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:11 smithi063 ceph-mon[109843]: pgmap v1166: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T17:14:11.741 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:11 smithi063 ceph-mon[114756]: pgmap v1166: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T17:14:11.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:11.751 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:11.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:11.790 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:14:11.790 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:14:11.830 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:14:11.830 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:14:11.830 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:14:11.830 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:14:11.830 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:14:11.831 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:14:11.831 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:14:11.831 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:14:11.852 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:11.870 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:14:11.870 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:14:11.906 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:14:11.907 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:14:11.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:11.908 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:11.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:11.938 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:14:11.938 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:14:11.977 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:14:11.977 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:14:12.008 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:12.047 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:12.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:12.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:12.067 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 45 16 81880 81864 113.673 69.75 0.015561 0.00879158 2024-08-17T17:14:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:11 smithi162 ceph-mon[104668]: pgmap v1166: 89 pgs: 89 active+clean; 4.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T17:14:12.149 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:12.178 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:12.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:12.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:12.280 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:12.338 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:12.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:12.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:12.440 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:12.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:12.493 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:12.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:12.594 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:12.622 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:12.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:12.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:12.724 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:12.744 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:12.749 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:12.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:12.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:12.851 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:12.880 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:12.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:12.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:12.982 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:13.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:13.018 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:13.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:13.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:12 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4151872391' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:14:13.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:12 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4151872391' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:14:13.068 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 46 16 83140 83124 112.912 78.75 0.00449505 0.00885163 2024-08-17T17:14:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:12 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4151872391' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:14:13.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:13.160 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:13.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:13.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:13.210 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:14:13.216 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:13.262 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:13.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:13.291 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:13.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:13.392 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:13.418 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:13.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:13.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:13.491 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:14:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:14:13] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.51.0" 2024-08-17T17:14:13.520 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:13.544 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:13.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:13.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:13.646 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:13.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:13.692 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:13.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:13.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:13 smithi063 ceph-mon[114756]: pgmap v1167: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-08-17T17:14:13.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:13 smithi063 ceph-mon[109843]: pgmap v1167: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-08-17T17:14:13.793 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:13.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:13.843 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:13.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:13.944 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:13.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:13.991 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:13.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:14.068 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 47 16 84140 84124 111.84 62.5 0.0201513 0.00893551 2024-08-17T17:14:14.092 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:13 smithi162 ceph-mon[104668]: pgmap v1167: 89 pgs: 89 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.34k op/s 2024-08-17T17:14:14.119 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:14.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:14.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:14.221 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:14.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:14.262 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:14.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:14.364 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:14.393 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:14.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:14.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:14.495 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:14.535 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:14.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:14.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:14.636 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:14.666 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:14.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:14.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:14.705 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:14:14.768 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:14.795 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:14.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:14.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:14.897 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:14.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:14.939 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:14.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:15.040 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:15.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:15.063 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:15.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:15.068 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 48 16 85173 85157 110.855 64.5625 0.00692718 0.00901038 2024-08-17T17:14:15.164 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:15.197 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:15.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:15.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:15.298 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:15.335 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:15.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:15.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:15.437 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:15.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:15.476 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:15.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:15.577 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:15.617 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:15.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:15.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:15.719 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:15.769 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:15.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:15.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:15.871 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:15.914 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:15.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:15.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:15.973 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:15 smithi063 ceph-mon[109843]: pgmap v1168: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-08-17T17:14:15.974 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:15 smithi063 ceph-mon[114756]: pgmap v1168: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-08-17T17:14:16.016 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:16.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:16.060 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:16.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:16.068 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 49 16 86385 86369 110.138 75.75 0.0151064 0.00907476 2024-08-17T17:14:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:15 smithi162 ceph-mon[104668]: pgmap v1168: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-08-17T17:14:16.162 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:16.196 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:16.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:16.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:16.298 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:16.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:16.333 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:16.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:16.433 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:16.462 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:16.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:16.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:16.564 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:16.588 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:16.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:16.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:16.689 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:16.732 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:16.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:16.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:16.834 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:16.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:16.865 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:16.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:16.966 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:16.990 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:17.013 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:17.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:17.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:17.069 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 50 16 87503 87487 109.332 69.875 0.0186733 0.00914071 2024-08-17T17:14:17.115 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:17.140 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:17.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:17.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:17.242 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:17.268 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:17.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:17.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:17.273 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:17.371 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:17.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:17.399 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:17.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:17.427 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:17 smithi063 ceph-mon[109843]: pgmap v1169: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:14:17.427 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:17 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:17.427 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:17 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:17.428 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:17 smithi063 ceph-mon[114756]: pgmap v1169: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:14:17.428 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:17 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:17.428 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:17 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:17.501 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:17.549 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:17.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:17.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:17.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:17 smithi162 ceph-mon[104668]: pgmap v1169: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-08-17T17:14:17.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:17 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:17.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:17 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:17.651 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:17.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:17.706 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:17.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:17.807 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:17.851 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:17.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:17.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:17.953 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:17.985 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:17.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:17.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:18.069 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 51 16 88573 88557 108.5 66.875 0.0140243 0.00921177 2024-08-17T17:14:18.087 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:18.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:18.131 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:18.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:18.232 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:18.267 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:18.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:18.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:18.340 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:14:18.349 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:14:18.368 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:18.412 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:18.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:18.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:18.514 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:18.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:18 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/251229644' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:14:18.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:18 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:14:18.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:18 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/661512973' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:14:18.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:18 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:14:18.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:18 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/251229644' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:14:18.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:18 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:14:18.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:18 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/661512973' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:14:18.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:18 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:14:18.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:18.545 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:18.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:18.646 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:18.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:18.691 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:18.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:18.792 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:18.823 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:18.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:18.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:18.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:18 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/251229644' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:14:18.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:18 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:14:18.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:18 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/661512973' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:14:18.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:18 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:14:18.925 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:18.956 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:18.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:18.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:19.058 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:19.069 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 52 16 89756 89740 107.835 73.9375 0.0101581 0.00926847 2024-08-17T17:14:19.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:19.091 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:19.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:19.192 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:19.240 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:19.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:19.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:19.341 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:19.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:19.373 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:19.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:19.474 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:19.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:19.515 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:19.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:19.585 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:19 smithi063 ceph-mon[109843]: pgmap v1170: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:14:19.586 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:19 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:14:19.586 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:19 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:14:19.586 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:19 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:14:19.586 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:19 smithi063 ceph-mon[109843]: osdmap e1099: 8 total, 8 up, 8 in 2024-08-17T17:14:19.586 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:19 smithi063 ceph-mon[114756]: pgmap v1170: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:14:19.586 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:19 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:14:19.586 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:19 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:14:19.586 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:19 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:14:19.587 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:19 smithi063 ceph-mon[114756]: osdmap e1099: 8 total, 8 up, 8 in 2024-08-17T17:14:19.616 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:19.642 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:19.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:19.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:19.744 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:19.782 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:19.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:19 smithi162 ceph-mon[104668]: pgmap v1170: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:14:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:19 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:14:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:19 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:14:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:19 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:14:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:19 smithi162 ceph-mon[104668]: osdmap e1099: 8 total, 8 up, 8 in 2024-08-17T17:14:19.884 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:19.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:19.930 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:19.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:20.031 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:20.070 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 53 16 90915 90899 107.166 72.4375 0.0163745 0.00932396 2024-08-17T17:14:20.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:20.085 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:20.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:20.186 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:20.232 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:20.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:20.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:20.334 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:20.360 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:20.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:20.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:20.462 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:20.494 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:20.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:20.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:20.596 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:20.624 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:20.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:20.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:20.726 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:20.771 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:20.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:20.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:20.873 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:20.905 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:20.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:20.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:21.007 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:21.048 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:21.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:21.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:21.070 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 54 16 92007 91991 106.445 68.25 0.0110665 0.00938932 2024-08-17T17:14:21.151 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:21.194 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:21.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:21.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:21.296 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:21.323 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:21.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:21.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:21.425 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:21.455 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:21.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:21.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:21.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:21 smithi063 ceph-mon[114756]: pgmap v1172: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:14:21.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:21 smithi063 ceph-mon[109843]: pgmap v1172: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:14:21.557 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:21.600 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:21.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:21.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:21.703 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:21.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:21.739 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:21.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:21.840 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:21.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:21 smithi162 ceph-mon[104668]: pgmap v1172: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:14:21.879 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:21.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:21.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:21.981 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:22.024 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:22.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:22.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:22.070 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 55 16 93150 93134 105.809 71.4375 0.0102468 0.00944594 2024-08-17T17:14:22.125 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:22.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:22.174 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:22.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:22.275 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:22.297 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:22.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:22.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:22.398 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:22.432 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:22.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:22.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:22.534 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:22.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:22.580 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:22.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:22.680 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:22.708 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:22.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:22.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:22.810 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:22.836 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:22.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:22.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:22.938 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:22.962 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:22.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:22.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:23.064 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:23.072 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 56 16 94177 94161 105.062 64.1875 0.0232621 0.00951245 2024-08-17T17:14:23.106 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:23.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:23.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:23.209 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:23.239 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:23.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:23.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:23.301 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:14:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:14:23] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.51.0" 2024-08-17T17:14:23.303 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:14:23.341 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:23.371 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:23.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:23.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:23.473 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:23.499 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:23.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:23.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:23.534 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:14:23.534 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:14:23.534 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:14:23.534 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:14:23.534 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:14:23.534 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:14:23.534 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:14:23.534 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:14:23.534 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:14:23.535 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:14:23.535 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:14:23.535 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:14:23.535 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:14:23.535 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:14:23.535 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:14:23.535 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:14:23.600 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:23.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:23 smithi063 ceph-mon[109843]: pgmap v1173: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:14:23.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:23 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2064977094' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:14:23.626 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:23 smithi063 ceph-mon[114756]: pgmap v1173: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:14:23.626 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:23 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2064977094' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:14:23.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:23.638 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:23.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:23.739 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:23.781 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:23.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:23.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:23.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:23 smithi162 ceph-mon[104668]: pgmap v1173: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:14:23.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:23 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2064977094' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:14:23.883 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:23.917 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:14:23.917 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:14:23.917 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:14:23.917 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:14:23.917 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:14:23.926 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:23.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:23.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:24.028 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:24.067 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:24.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:24.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:24.072 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 57 16 95229 95213 104.372 65.75 0.00903427 0.00957581 2024-08-17T17:14:24.169 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:24.214 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:24.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:24.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:24.316 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:24.347 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:24.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:24.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:24.449 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:24.491 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:24.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:24.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:24.593 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:24.637 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:24.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:24.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:24.740 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:24.768 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:24.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:24.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:24.870 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:24.894 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:14:24.910 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:24.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:24.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:25.012 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:25.035 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:25.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:25.037 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:14:25.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:25.072 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 58 16 96342 96326 103.772 69.5625 0.0263202 0.0096308 2024-08-17T17:14:25.138 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:25.160 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:25.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:25.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:25.181 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:25.262 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:25.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:25.315 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:25.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:25.416 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:25.450 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:25.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:25.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:25.552 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:25.602 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:25.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:25.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:25.689 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:25 smithi063 ceph-mon[114756]: pgmap v1174: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-17T17:14:25.690 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:25 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:14:25.690 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:25 smithi063 ceph-mon[109843]: pgmap v1174: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-17T17:14:25.690 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:25 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:14:25.705 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:25.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:25.768 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:25.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:25.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:25 smithi162 ceph-mon[104668]: pgmap v1174: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-17T17:14:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:25 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:14:25.869 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:25.917 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:25.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:25.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:26.019 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:26.064 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:26.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:26.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:26.074 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 59 16 97210 97194 102.932 54.25 0.0152349 0.00971017 2024-08-17T17:14:26.166 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:26.198 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:26.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:26.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:26.300 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:26.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:26.331 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:26.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:26.432 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:26.462 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:26.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:26.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:26.564 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:26.604 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:26.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:26.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:26.642 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:14:26.706 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:26.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:26.738 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:26.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:26.839 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:26.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:26.881 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:26.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:26.982 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:27.027 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:27.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:27.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:27.073 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:14:27.072997+0000 min lat: 0.0014202 max lat: 0.151093 avg lat: 0.00976935 2024-08-17T17:14:27.073 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:14:27.073 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 60 16 98257 98241 102.307 65.4375 0.00921903 0.00976935 2024-08-17T17:14:27.128 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:27.154 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:27.171 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:27.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:27.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:27.273 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:27.311 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:27.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:27.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:27.413 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:27.454 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:27.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:27.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:27.460 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:27 smithi162 ceph-mon[104668]: pgmap v1175: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-17T17:14:27.556 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:27.585 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:27.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:27.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:27.591 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:27 smithi063 ceph-mon[114756]: pgmap v1175: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-17T17:14:27.591 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:27 smithi063 ceph-mon[109843]: pgmap v1175: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-17T17:14:27.686 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:27.710 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:27.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:27.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:27.738 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:27.812 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:27.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:27.882 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:27.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:27.983 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:28.036 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:28.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:28.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:28.073 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 61 16 99241 99225 101.638 61.5 0.0165945 0.0098331 2024-08-17T17:14:28.138 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:28.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:28.200 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:28.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:28.301 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:28.347 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:28.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:28.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:28.449 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:28.485 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:28.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:28.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:28.587 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:28.639 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:28.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:28.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:28.741 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:28.779 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:28.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:28.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:28.883 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:28.911 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:28.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:28.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:29.013 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:29.056 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:29.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:29.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:29.073 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 62 16 100278 100262 101.044 64.8125 0.0198422 0.00989147 2024-08-17T17:14:29.159 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:29.207 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:29.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:29.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:29.309 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:29.355 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:29.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:29.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:29.457 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:29.483 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:29.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:29.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:29.497 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:29 smithi063 ceph-mon[114756]: pgmap v1176: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-08-17T17:14:29.497 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:29 smithi063 ceph-mon[109843]: pgmap v1176: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-08-17T17:14:29.585 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:29.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:29.621 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:29.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:29.671 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:29 smithi162 ceph-mon[104668]: pgmap v1176: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-08-17T17:14:29.722 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:29.756 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:29.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:29.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:29.858 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:29.889 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:29.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:29.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:29.991 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:30.026 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:30.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:30.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:30.073 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 63 16 101268 101252 100.422 61.875 0.00978574 0.00995191 2024-08-17T17:14:30.128 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:30.172 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:30.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:30.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:30.274 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:30.321 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:30.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:30.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:30.397 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:14:30.422 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:30.468 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:30.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:30.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:30.571 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:30.609 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:30.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:30.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:30.710 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:30.737 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:30 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1937309089' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:14:30.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:30.749 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:30.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:30.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:30 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1937309089' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:14:30.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:30 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1937309089' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:14:30.850 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:30.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:30.884 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:30.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:30.985 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:31.024 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:31.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:31.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:31.073 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 64 16 102316 102300 99.8768 65.5 0.0152891 0.010007 2024-08-17T17:14:31.127 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:31.167 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:31.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:31.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:31.269 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:31.314 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:31.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:31.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:31.416 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:31.454 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:31.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:31.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:31.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:31 smithi063 ceph-mon[109843]: pgmap v1177: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-17T17:14:31.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:31 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1937309089' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:14:31.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:31 smithi063 ceph-mon[109843]: osdmap e1100: 8 total, 8 up, 8 in 2024-08-17T17:14:31.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:31.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:31.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:31.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:31 smithi063 ceph-mon[114756]: pgmap v1177: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-17T17:14:31.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:31 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1937309089' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:14:31.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:31 smithi063 ceph-mon[114756]: osdmap e1100: 8 total, 8 up, 8 in 2024-08-17T17:14:31.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:31.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:31.532 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:31.556 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:31.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:31.606 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:31.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:31.675 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:31.707 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:31.755 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:31.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:31.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:31.768 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:31 smithi162 ceph-mon[104668]: pgmap v1177: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-08-17T17:14:31.768 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:31 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1937309089' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:14:31.768 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:31 smithi162 ceph-mon[104668]: osdmap e1100: 8 total, 8 up, 8 in 2024-08-17T17:14:31.768 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:31.768 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:31.768 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:31.857 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:31.905 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:31.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:31.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:32.007 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:32.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:32.050 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:32.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:32.073 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 65 16 103319 103303 99.3047 62.6875 0.00798254 0.010065 2024-08-17T17:14:32.151 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:32.178 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:32.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:32.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:32.280 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:32.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:32.323 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:32.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:32.424 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:32.455 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:32.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:32.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:32.558 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:32.594 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:32.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:32.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:32.696 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:32.740 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:32.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:32.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:32.842 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:32.885 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:32.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:32.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:32.987 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:33.014 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:33.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:33.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:33.073 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 66 16 104274 104258 98.7044 59.6875 0.0155514 0.0101262 2024-08-17T17:14:33.116 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:33.143 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:33.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:33.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:33.245 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:33.282 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:33.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:33.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:33.385 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:33.426 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:33.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:33.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:33.431 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:14:33.431 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:14:33.431 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:14:33.431 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:14:33.431 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.431 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7523 5.6 5:652908de:::smithi06387500-38:head [sparse-read 0~1570594,omap-get-vals-by-keys,omap-get-keys,omap-get-vals,omap-get-header,getxattrs] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e804)", 2024-08-17T17:14:33.431 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:07:11.652193+0000", 2024-08-17T17:14:33.432 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 441.11519184799999, 2024-08-17T17:14:33.432 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.012645974, 2024-08-17T17:14:33.432 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.432 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.432 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:14:33.432 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.432 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:14:33.432 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:14:33.432 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7523 2024-08-17T17:14:33.432 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.432 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.432 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.432 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.433 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652193+0000", 2024-08-17T17:14:33.433 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.433 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.433 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.433 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.433 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652193+0000", 2024-08-17T17:14:33.433 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.433 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.433 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.433 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.433 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652194+0000", 2024-08-17T17:14:33.433 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5009999999999999e-06 2024-08-17T17:14:33.433 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.434 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.434 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.434 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652200+0000", 2024-08-17T17:14:33.434 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6289999999999998e-06 2024-08-17T17:14:33.434 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.434 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.434 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.434 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652211+0000", 2024-08-17T17:14:33.434 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1199000000000001e-05 2024-08-17T17:14:33.434 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.434 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.435 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.435 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652222+0000", 2024-08-17T17:14:33.435 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0645999999999999e-05 2024-08-17T17:14:33.435 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.435 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.435 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.435 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.667870+0000", 2024-08-17T17:14:33.435 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.015648599999999999 2024-08-17T17:14:33.435 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.435 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.435 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:14:33.435 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.667896+0000", 2024-08-17T17:14:33.435 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5554e-05 2024-08-17T17:14:33.436 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.436 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.436 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.436 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.647903+0000", 2024-08-17T17:14:33.436 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98000743199999996 2024-08-17T17:14:33.436 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.436 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.436 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for missing object", 2024-08-17T17:14:33.436 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.648370+0000", 2024-08-17T17:14:33.436 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00046631999999999999 2024-08-17T17:14:33.436 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.436 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.437 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.437 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.662257+0000", 2024-08-17T17:14:33.437 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.013887595000000001 2024-08-17T17:14:33.437 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.437 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.437 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.437 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.662406+0000", 2024-08-17T17:14:33.437 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014876200000000001 2024-08-17T17:14:33.437 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.437 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.437 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.437 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.664839+0000", 2024-08-17T17:14:33.437 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002432736 2024-08-17T17:14:33.438 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.438 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.438 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.438 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.438 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.438 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7524 5.6 5:652908de:::smithi06387500-38:head [read 0~2201981,checksum] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e804)", 2024-08-17T17:14:33.438 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:07:11.652237+0000", 2024-08-17T17:14:33.438 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 441.11514737900001, 2024-08-17T17:14:33.438 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.013499726, 2024-08-17T17:14:33.438 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.438 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.438 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:14:33.439 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.439 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:14:33.439 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:14:33.439 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7524 2024-08-17T17:14:33.439 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.439 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.439 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.439 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.439 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652237+0000", 2024-08-17T17:14:33.439 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.439 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.439 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.439 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.440 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652237+0000", 2024-08-17T17:14:33.440 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.440 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.440 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.440 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.440 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652238+0000", 2024-08-17T17:14:33.440 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3399999999999999e-07 2024-08-17T17:14:33.440 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.440 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.440 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.440 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652241+0000", 2024-08-17T17:14:33.440 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0259999999999999e-06 2024-08-17T17:14:33.441 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.441 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.441 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.441 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652248+0000", 2024-08-17T17:14:33.441 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2960000000000001e-06 2024-08-17T17:14:33.441 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.441 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.441 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.441 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652254+0000", 2024-08-17T17:14:33.441 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8529999999999997e-06 2024-08-17T17:14:33.441 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.441 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.442 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.442 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.667975+0000", 2024-08-17T17:14:33.442 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.015720919999999999 2024-08-17T17:14:33.442 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.442 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.442 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:14:33.442 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.667985+0000", 2024-08-17T17:14:33.442 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0604e-05 2024-08-17T17:14:33.442 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.442 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.442 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.442 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.648562+0000", 2024-08-17T17:14:33.443 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98057684099999998 2024-08-17T17:14:33.443 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.443 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.443 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for missing object", 2024-08-17T17:14:33.443 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.648612+0000", 2024-08-17T17:14:33.443 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0111999999999997e-05 2024-08-17T17:14:33.443 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.443 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.443 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.443 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.664926+0000", 2024-08-17T17:14:33.443 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.016313782999999998 2024-08-17T17:14:33.443 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.443 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.444 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.444 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.665043+0000", 2024-08-17T17:14:33.444 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000117049 2024-08-17T17:14:33.444 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.444 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.444 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.444 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.665737+0000", 2024-08-17T17:14:33.444 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00069370800000000002 2024-08-17T17:14:33.444 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.444 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.444 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.444 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.444 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.445 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7525 5.6 5:652908de:::smithi06387500-38:head [read 0~2201981,checksum] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e804)", 2024-08-17T17:14:33.445 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:07:11.652262+0000", 2024-08-17T17:14:33.445 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 441.115122918, 2024-08-17T17:14:33.445 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0149410139999999, 2024-08-17T17:14:33.445 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.445 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.445 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:14:33.445 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.445 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:14:33.445 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:14:33.445 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7525 2024-08-17T17:14:33.445 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.445 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.446 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.446 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.446 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652262+0000", 2024-08-17T17:14:33.446 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.446 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.446 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.446 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.446 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652262+0000", 2024-08-17T17:14:33.446 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.446 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.446 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.446 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.447 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652262+0000", 2024-08-17T17:14:33.447 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7399999999999998e-07 2024-08-17T17:14:33.447 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.447 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.447 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.447 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652265+0000", 2024-08-17T17:14:33.447 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5569999999999998e-06 2024-08-17T17:14:33.447 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.447 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.447 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.447 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652271+0000", 2024-08-17T17:14:33.447 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3219999999999998e-06 2024-08-17T17:14:33.447 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.448 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.448 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.448 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.652276+0000", 2024-08-17T17:14:33.448 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.451e-06 2024-08-17T17:14:33.448 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.448 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.448 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.448 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.668015+0000", 2024-08-17T17:14:33.448 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.015738491 2024-08-17T17:14:33.448 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.448 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.448 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:14:33.449 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:11.668024+0000", 2024-08-17T17:14:33.449 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.1619999999999997e-06 2024-08-17T17:14:33.449 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.449 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.449 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.449 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.648725+0000", 2024-08-17T17:14:33.449 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98070130799999999 2024-08-17T17:14:33.449 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.449 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.449 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for missing object", 2024-08-17T17:14:33.449 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.648774+0000", 2024-08-17T17:14:33.449 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8887999999999997e-05 2024-08-17T17:14:33.449 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.450 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.450 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.450 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.665781+0000", 2024-08-17T17:14:33.450 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.017006797000000001 2024-08-17T17:14:33.450 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.450 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.450 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.450 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.665901+0000", 2024-08-17T17:14:33.450 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000119436 2024-08-17T17:14:33.450 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.450 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.450 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.451 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.667203+0000", 2024-08-17T17:14:33.451 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0013021280000000001 2024-08-17T17:14:33.451 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.451 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.451 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.451 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.451 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.451 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7527 5.8 5:11bf5233:::smithi06387500-20:head [write 538121~670832] snapc 1f4=[1f4,1f2,1ef,1ee,1ea,1e9,1e8,1da,1cf] ondisk+write+known_if_redirected+supports_pool_eio e804)", 2024-08-17T17:14:33.451 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:07:12.468597+0000", 2024-08-17T17:14:33.451 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 440.29878736299997, 2024-08-17T17:14:33.451 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20438372899999999, 2024-08-17T17:14:33.451 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.451 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.452 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:33.452 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.452 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:14:33.452 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:14:33.452 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7527 2024-08-17T17:14:33.452 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.452 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.452 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.452 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.452 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.468597+0000", 2024-08-17T17:14:33.452 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.452 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.452 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.453 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.453 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.468597+0000", 2024-08-17T17:14:33.453 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.453 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.453 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.453 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.453 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.468600+0000", 2024-08-17T17:14:33.453 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4399999999999999e-06 2024-08-17T17:14:33.453 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.453 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.453 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.453 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.468991+0000", 2024-08-17T17:14:33.454 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00039158200000000002 2024-08-17T17:14:33.454 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.454 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.454 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.454 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.469012+0000", 2024-08-17T17:14:33.454 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1005e-05 2024-08-17T17:14:33.454 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.454 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.454 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.454 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.469026+0000", 2024-08-17T17:14:33.454 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3509999999999999e-05 2024-08-17T17:14:33.454 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.455 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.455 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.455 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.469114+0000", 2024-08-17T17:14:33.455 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8103000000000007e-05 2024-08-17T17:14:33.455 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.455 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.455 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:14:33.455 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.469124+0000", 2024-08-17T17:14:33.455 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.6889999999999999e-06 2024-08-17T17:14:33.455 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.455 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.455 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.455 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.652227+0000", 2024-08-17T17:14:33.456 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.18310315999999999 2024-08-17T17:14:33.456 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.456 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.456 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for missing object", 2024-08-17T17:14:33.456 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.652361+0000", 2024-08-17T17:14:33.456 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013436200000000001 2024-08-17T17:14:33.456 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.456 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.456 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.456 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.664314+0000", 2024-08-17T17:14:33.456 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011953116999999999 2024-08-17T17:14:33.456 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.457 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.457 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for rw locks", 2024-08-17T17:14:33.457 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.664393+0000", 2024-08-17T17:14:33.457 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9126000000000001e-05 2024-08-17T17:14:33.457 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.457 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.457 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.457 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.665372+0000", 2024-08-17T17:14:33.457 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00097915000000000003 2024-08-17T17:14:33.457 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.457 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.457 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.457 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.665447+0000", 2024-08-17T17:14:33.458 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4719000000000001e-05 2024-08-17T17:14:33.458 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.458 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.458 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:14:33.458 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.665673+0000", 2024-08-17T17:14:33.458 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00022545900000000001 2024-08-17T17:14:33.458 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.458 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.458 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.458 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.671160+0000", 2024-08-17T17:14:33.458 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0054872319999999999 2024-08-17T17:14:33.459 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.459 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.459 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:33.459 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.671363+0000", 2024-08-17T17:14:33.459 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000202802 2024-08-17T17:14:33.459 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.459 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.459 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.459 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.672850+0000", 2024-08-17T17:14:33.459 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001487665 2024-08-17T17:14:33.459 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.459 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.459 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:33.460 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.672892+0000", 2024-08-17T17:14:33.460 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.1560000000000002e-05 2024-08-17T17:14:33.460 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.460 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.460 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.460 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.672981+0000", 2024-08-17T17:14:33.460 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.9048000000000005e-05 2024-08-17T17:14:33.460 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.460 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.460 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.460 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.461 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.461 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7528 5.8 5:11bf5233:::smithi06387500-20:head [write 1961521~413039] snapc 1f4=[1f4,1f2,1ef,1ee,1ea,1e9,1e8,1da,1cf] ondisk+write+known_if_redirected+supports_pool_eio e804)", 2024-08-17T17:14:33.461 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:07:12.474938+0000", 2024-08-17T17:14:33.461 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 440.292446197, 2024-08-17T17:14:33.461 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19924224700000001, 2024-08-17T17:14:33.461 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.461 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.461 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:33.461 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.461 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:14:33.461 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:14:33.461 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7528 2024-08-17T17:14:33.461 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.462 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.462 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.462 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.462 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.474938+0000", 2024-08-17T17:14:33.462 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.462 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.462 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.462 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.462 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.474938+0000", 2024-08-17T17:14:33.462 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.462 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.466 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.466 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.466 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.474941+0000", 2024-08-17T17:14:33.466 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5620000000000002e-06 2024-08-17T17:14:33.466 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.467 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.467 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.467 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475183+0000", 2024-08-17T17:14:33.467 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000241926 2024-08-17T17:14:33.467 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.467 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.467 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.467 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475200+0000", 2024-08-17T17:14:33.467 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7455000000000001e-05 2024-08-17T17:14:33.467 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.467 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.467 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.468 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475210+0000", 2024-08-17T17:14:33.468 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3309999999999993e-06 2024-08-17T17:14:33.468 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.468 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.468 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.468 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475304+0000", 2024-08-17T17:14:33.468 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4055000000000007e-05 2024-08-17T17:14:33.468 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.468 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.468 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:14:33.468 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475319+0000", 2024-08-17T17:14:33.468 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5650000000000001e-05 2024-08-17T17:14:33.468 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.469 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.469 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.469 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.652394+0000", 2024-08-17T17:14:33.469 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17707472099999999 2024-08-17T17:14:33.469 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.469 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.469 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for missing object", 2024-08-17T17:14:33.469 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.652447+0000", 2024-08-17T17:14:33.469 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2935000000000001e-05 2024-08-17T17:14:33.469 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.469 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.469 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.470 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.664456+0000", 2024-08-17T17:14:33.470 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.012009093 2024-08-17T17:14:33.470 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.470 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.470 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for rw locks", 2024-08-17T17:14:33.470 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.664582+0000", 2024-08-17T17:14:33.470 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000126186 2024-08-17T17:14:33.470 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.470 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.470 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.470 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.668142+0000", 2024-08-17T17:14:33.470 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.003559619 2024-08-17T17:14:33.471 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.471 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.471 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.471 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.668279+0000", 2024-08-17T17:14:33.471 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013721 2024-08-17T17:14:33.471 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.471 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.471 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:14:33.471 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.668505+0000", 2024-08-17T17:14:33.471 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00022565899999999999 2024-08-17T17:14:33.471 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.472 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.472 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.472 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.673964+0000", 2024-08-17T17:14:33.472 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0054589740000000001 2024-08-17T17:14:33.472 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.472 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.472 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.472 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.674069+0000", 2024-08-17T17:14:33.472 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010506900000000001 2024-08-17T17:14:33.472 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.472 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.472 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:33.473 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.674085+0000", 2024-08-17T17:14:33.473 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6002e-05 2024-08-17T17:14:33.473 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.473 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.473 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:33.473 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.674115+0000", 2024-08-17T17:14:33.473 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.9967999999999998e-05 2024-08-17T17:14:33.473 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.473 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.473 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.473 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.674181+0000", 2024-08-17T17:14:33.473 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5832000000000003e-05 2024-08-17T17:14:33.474 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.474 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.474 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.474 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.474 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.474 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7529 5.8 5:11bf5233:::smithi06387500-20:head [setxattr _header (58) in=7b,truncate 2538121] snapc 1f4=[1f4,1f2,1ef,1ee,1ea,1e9,1e8,1da,1cf] ondisk+write+known_if_redirected+supports_pool_eio e804)", 2024-08-17T17:14:33.474 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:07:12.475233+0000", 2024-08-17T17:14:33.474 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 440.29215192300001, 2024-08-17T17:14:33.474 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.199137697, 2024-08-17T17:14:33.474 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.474 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.474 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:33.474 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.475 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:14:33.475 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:14:33.475 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7529 2024-08-17T17:14:33.475 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.475 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.475 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.475 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.475 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475233+0000", 2024-08-17T17:14:33.475 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.475 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.475 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.475 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.475 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475233+0000", 2024-08-17T17:14:33.476 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.476 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.476 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.476 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.476 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475233+0000", 2024-08-17T17:14:33.476 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6300000000000005e-07 2024-08-17T17:14:33.476 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.476 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.476 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.476 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475237+0000", 2024-08-17T17:14:33.476 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5920000000000001e-06 2024-08-17T17:14:33.476 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.476 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.477 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.477 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475246+0000", 2024-08-17T17:14:33.477 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8789999999999992e-06 2024-08-17T17:14:33.477 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.477 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.477 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.477 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475253+0000", 2024-08-17T17:14:33.477 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4749999999999996e-06 2024-08-17T17:14:33.477 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.477 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.477 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.477 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475383+0000", 2024-08-17T17:14:33.477 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012958300000000001 2024-08-17T17:14:33.478 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.478 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.478 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:14:33.478 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475395+0000", 2024-08-17T17:14:33.478 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.234e-05 2024-08-17T17:14:33.478 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.478 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.478 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.478 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.652711+0000", 2024-08-17T17:14:33.478 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.177316274 2024-08-17T17:14:33.478 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.478 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.478 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for missing object", 2024-08-17T17:14:33.479 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.652772+0000", 2024-08-17T17:14:33.479 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0236999999999999e-05 2024-08-17T17:14:33.479 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.479 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.479 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.479 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.664655+0000", 2024-08-17T17:14:33.479 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011883757999999999 2024-08-17T17:14:33.479 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.479 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.479 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for rw locks", 2024-08-17T17:14:33.479 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.664717+0000", 2024-08-17T17:14:33.479 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2020999999999995e-05 2024-08-17T17:14:33.479 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.480 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.480 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.480 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.670465+0000", 2024-08-17T17:14:33.480 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0057477220000000002 2024-08-17T17:14:33.480 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.480 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.480 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.480 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.670525+0000", 2024-08-17T17:14:33.480 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9471e-05 2024-08-17T17:14:33.480 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.480 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.480 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:14:33.481 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.670641+0000", 2024-08-17T17:14:33.481 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000116199 2024-08-17T17:14:33.481 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.481 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.481 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.481 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.674020+0000", 2024-08-17T17:14:33.481 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0033789660000000002 2024-08-17T17:14:33.481 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.481 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.481 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:33.481 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.674194+0000", 2024-08-17T17:14:33.481 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00017450699999999999 2024-08-17T17:14:33.481 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.482 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.482 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.482 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.674223+0000", 2024-08-17T17:14:33.482 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8524e-05 2024-08-17T17:14:33.482 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.482 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.482 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:33.482 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.674259+0000", 2024-08-17T17:14:33.482 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6287e-05 2024-08-17T17:14:33.482 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.482 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.482 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.482 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.674370+0000", 2024-08-17T17:14:33.483 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000111199 2024-08-17T17:14:33.483 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.483 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.483 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.483 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.483 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.483 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7530 5.8 5:11bf5233:::smithi06387500-20:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e804)", 2024-08-17T17:14:33.483 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:07:12.475280+0000", 2024-08-17T17:14:33.483 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 440.29210410399997, 2024-08-17T17:14:33.483 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19927872999999999, 2024-08-17T17:14:33.483 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.483 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.483 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:14:33.484 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.484 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:14:33.484 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:14:33.484 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7530 2024-08-17T17:14:33.484 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.484 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.484 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.484 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.484 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475280+0000", 2024-08-17T17:14:33.484 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.484 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.484 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.484 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.485 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475280+0000", 2024-08-17T17:14:33.485 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.485 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.485 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.485 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.485 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475281+0000", 2024-08-17T17:14:33.485 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9999999999999997e-07 2024-08-17T17:14:33.485 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.485 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.485 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.485 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475284+0000", 2024-08-17T17:14:33.485 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.258e-06 2024-08-17T17:14:33.486 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.486 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.486 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.486 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475295+0000", 2024-08-17T17:14:33.486 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0318e-05 2024-08-17T17:14:33.486 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.486 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.486 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.486 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475301+0000", 2024-08-17T17:14:33.486 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1419999999999998e-06 2024-08-17T17:14:33.486 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.486 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.486 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.487 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475433+0000", 2024-08-17T17:14:33.487 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013225699999999999 2024-08-17T17:14:33.487 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.487 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.487 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:14:33.487 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.475442+0000", 2024-08-17T17:14:33.487 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7199999999999995e-06 2024-08-17T17:14:33.487 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.487 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.487 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.487 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.652804+0000", 2024-08-17T17:14:33.487 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.177362661 2024-08-17T17:14:33.488 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.488 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.488 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for missing object", 2024-08-17T17:14:33.488 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.652851+0000", 2024-08-17T17:14:33.488 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6758000000000002e-05 2024-08-17T17:14:33.488 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.488 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.488 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.488 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.664758+0000", 2024-08-17T17:14:33.488 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011906841 2024-08-17T17:14:33.488 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.488 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.488 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for rw locks", 2024-08-17T17:14:33.489 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.664873+0000", 2024-08-17T17:14:33.489 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000115097 2024-08-17T17:14:33.489 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.489 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.489 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.489 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.671225+0000", 2024-08-17T17:14:33.489 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0063515630000000002 2024-08-17T17:14:33.489 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.489 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.489 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for rw locks", 2024-08-17T17:14:33.489 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.671312+0000", 2024-08-17T17:14:33.489 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6928999999999994e-05 2024-08-17T17:14:33.489 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.490 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.490 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.490 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.674399+0000", 2024-08-17T17:14:33.490 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.003087233 2024-08-17T17:14:33.490 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.490 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.490 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.490 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.674454+0000", 2024-08-17T17:14:33.490 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5024e-05 2024-08-17T17:14:33.490 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.490 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.490 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.490 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:12.674559+0000", 2024-08-17T17:14:33.491 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010532899999999999 2024-08-17T17:14:33.491 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.491 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.491 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.491 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.491 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.491 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:29967 6.f 6:fe897d43:::benchmark_data_smithi063_160268_object29966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:33.491 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:37.723619+0000", 2024-08-17T17:14:33.491 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 55.043765249000003, 2024-08-17T17:14:33.491 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.082286437000000004, 2024-08-17T17:14:33.491 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.491 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.492 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:33.492 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.492 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:33.492 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:33.492 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 29967 2024-08-17T17:14:33.492 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.492 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.492 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.492 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.492 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.723619+0000", 2024-08-17T17:14:33.492 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.492 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.493 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.493 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.493 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.723619+0000", 2024-08-17T17:14:33.493 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.493 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.493 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.493 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.493 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.723621+0000", 2024-08-17T17:14:33.493 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.282e-06 2024-08-17T17:14:33.493 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.493 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.493 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.493 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.723639+0000", 2024-08-17T17:14:33.494 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8019000000000001e-05 2024-08-17T17:14:33.494 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.494 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.494 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.494 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.723647+0000", 2024-08-17T17:14:33.494 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6090000000000001e-06 2024-08-17T17:14:33.494 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.494 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.494 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.494 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.723652+0000", 2024-08-17T17:14:33.494 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6500000000000004e-06 2024-08-17T17:14:33.495 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.495 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.495 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.495 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.723704+0000", 2024-08-17T17:14:33.495 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1756999999999999e-05 2024-08-17T17:14:33.495 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.495 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.495 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.495 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.723825+0000", 2024-08-17T17:14:33.495 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000121122 2024-08-17T17:14:33.495 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.495 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.495 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:14:33.496 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.724009+0000", 2024-08-17T17:14:33.496 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00018442600000000001 2024-08-17T17:14:33.496 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.496 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.496 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:33.496 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.725154+0000", 2024-08-17T17:14:33.496 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0011450060000000001 2024-08-17T17:14:33.496 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.496 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.496 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.496 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.726058+0000", 2024-08-17T17:14:33.496 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00090363800000000001 2024-08-17T17:14:33.497 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.497 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.497 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.497 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.805813+0000", 2024-08-17T17:14:33.497 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.079754964999999997 2024-08-17T17:14:33.497 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.497 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.497 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:33.497 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.805864+0000", 2024-08-17T17:14:33.497 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1202999999999997e-05 2024-08-17T17:14:33.497 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.497 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.497 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.498 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.805906+0000", 2024-08-17T17:14:33.498 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.176e-05 2024-08-17T17:14:33.498 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.498 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.498 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.498 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.498 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.498 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:32657 6.f 6:f6c91f4f:::benchmark_data_smithi063_160268_object32656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:33.498 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.134602+0000", 2024-08-17T17:14:33.498 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 53.632782057, 2024-08-17T17:14:33.498 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.084200863000000001, 2024-08-17T17:14:33.499 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.499 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.499 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:33.499 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.499 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:33.499 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:33.499 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 32657 2024-08-17T17:14:33.499 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.499 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.499 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.499 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.499 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.134602+0000", 2024-08-17T17:14:33.499 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.499 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.500 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.500 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.500 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.134602+0000", 2024-08-17T17:14:33.500 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.500 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.500 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.500 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.500 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.134604+0000", 2024-08-17T17:14:33.500 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6559999999999999e-06 2024-08-17T17:14:33.500 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.500 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.500 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.501 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.134626+0000", 2024-08-17T17:14:33.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2107999999999999e-05 2024-08-17T17:14:33.501 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.501 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.501 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.501 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.134639+0000", 2024-08-17T17:14:33.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2576e-05 2024-08-17T17:14:33.501 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.501 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.501 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.501 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.134646+0000", 2024-08-17T17:14:33.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9079999999999998e-06 2024-08-17T17:14:33.501 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.502 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.502 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.502 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.134763+0000", 2024-08-17T17:14:33.502 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000117389 2024-08-17T17:14:33.502 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.502 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.502 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.502 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.134953+0000", 2024-08-17T17:14:33.502 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00019016000000000001 2024-08-17T17:14:33.502 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.502 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.502 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:14:33.503 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.135094+0000", 2024-08-17T17:14:33.503 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014074700000000001 2024-08-17T17:14:33.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.503 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.503 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.503 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218617+0000", 2024-08-17T17:14:33.503 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.083522664999999996 2024-08-17T17:14:33.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.503 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.503 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.503 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218704+0000", 2024-08-17T17:14:33.503 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7127999999999997e-05 2024-08-17T17:14:33.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.504 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.504 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:33.504 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218715+0000", 2024-08-17T17:14:33.504 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1379000000000001e-05 2024-08-17T17:14:33.504 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.504 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.504 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:33.504 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218763+0000", 2024-08-17T17:14:33.504 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7917000000000002e-05 2024-08-17T17:14:33.504 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.504 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.504 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.504 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218803+0000", 2024-08-17T17:14:33.505 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.0229999999999999e-05 2024-08-17T17:14:33.505 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.505 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.505 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.505 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.505 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.505 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:32659 6.f 6:fca6138f:::benchmark_data_smithi063_160268_object32658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:33.505 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.134669+0000", 2024-08-17T17:14:33.505 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 53.632715841, 2024-08-17T17:14:33.505 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.088673052000000002, 2024-08-17T17:14:33.505 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.505 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.506 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:33.506 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.506 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:33.506 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:33.506 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 32659 2024-08-17T17:14:33.506 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.506 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.506 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.506 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.506 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.134669+0000", 2024-08-17T17:14:33.506 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.506 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.507 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.507 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.507 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.134669+0000", 2024-08-17T17:14:33.507 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.507 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.507 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.507 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.507 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.134669+0000", 2024-08-17T17:14:33.507 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2200000000000004e-07 2024-08-17T17:14:33.507 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.507 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.507 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.507 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.134689+0000", 2024-08-17T17:14:33.508 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9570000000000001e-05 2024-08-17T17:14:33.508 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.508 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.508 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.508 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.134696+0000", 2024-08-17T17:14:33.508 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.25e-06 2024-08-17T17:14:33.508 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.508 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.508 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.508 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.134702+0000", 2024-08-17T17:14:33.508 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7010000000000001e-06 2024-08-17T17:14:33.508 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.509 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.509 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.509 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.136731+0000", 2024-08-17T17:14:33.509 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0020294480000000001 2024-08-17T17:14:33.509 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.509 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.509 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.509 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.136839+0000", 2024-08-17T17:14:33.509 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000107761 2024-08-17T17:14:33.509 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.509 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.509 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:14:33.509 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.136944+0000", 2024-08-17T17:14:33.509 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000104629 2024-08-17T17:14:33.510 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.510 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.510 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.510 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218668+0000", 2024-08-17T17:14:33.510 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.081724622999999996 2024-08-17T17:14:33.510 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.510 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.510 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:33.510 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218810+0000", 2024-08-17T17:14:33.510 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000142225 2024-08-17T17:14:33.510 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.510 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.510 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.511 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.223251+0000", 2024-08-17T17:14:33.511 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044405540000000002 2024-08-17T17:14:33.511 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.511 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.511 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:33.511 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.223302+0000", 2024-08-17T17:14:33.511 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1038000000000002e-05 2024-08-17T17:14:33.511 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.511 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.511 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.511 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.223342+0000", 2024-08-17T17:14:33.511 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9731000000000003e-05 2024-08-17T17:14:33.512 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.512 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.512 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.512 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.512 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.512 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:32662 6.f 6:f1ddddd1:::benchmark_data_smithi063_160268_object32661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:33.512 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.136515+0000", 2024-08-17T17:14:33.512 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 53.630869245, 2024-08-17T17:14:33.512 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.086942939999999996, 2024-08-17T17:14:33.512 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.512 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.512 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:33.512 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.513 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:33.513 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:33.513 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 32662 2024-08-17T17:14:33.513 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.513 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.513 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.513 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.513 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.136515+0000", 2024-08-17T17:14:33.513 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.513 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.513 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.513 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.513 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.136515+0000", 2024-08-17T17:14:33.514 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.514 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.514 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.514 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.514 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.136517+0000", 2024-08-17T17:14:33.514 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4619999999999999e-06 2024-08-17T17:14:33.514 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.514 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.514 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.514 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.136580+0000", 2024-08-17T17:14:33.514 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3008000000000004e-05 2024-08-17T17:14:33.515 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.515 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.515 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.515 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.136631+0000", 2024-08-17T17:14:33.515 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1236000000000002e-05 2024-08-17T17:14:33.515 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.515 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.515 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.515 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.136638+0000", 2024-08-17T17:14:33.515 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9999999999999999e-06 2024-08-17T17:14:33.515 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.515 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.515 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.516 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.138025+0000", 2024-08-17T17:14:33.516 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0013868979999999999 2024-08-17T17:14:33.516 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.516 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.516 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.516 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.138149+0000", 2024-08-17T17:14:33.516 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000124101 2024-08-17T17:14:33.516 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.516 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.516 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:14:33.516 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.138234+0000", 2024-08-17T17:14:33.516 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4602000000000001e-05 2024-08-17T17:14:33.517 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.517 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:33.517 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218815+0000", 2024-08-17T17:14:33.517 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.080581069000000005 2024-08-17T17:14:33.517 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.517 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.517 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.223216+0000", 2024-08-17T17:14:33.517 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044017300000000004 2024-08-17T17:14:33.517 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.518 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.518 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.223381+0000", 2024-08-17T17:14:33.518 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016415000000000001 2024-08-17T17:14:33.518 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.518 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.518 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:33.518 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.223427+0000", 2024-08-17T17:14:33.518 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6020999999999999e-05 2024-08-17T17:14:33.518 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.518 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.518 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.518 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.223458+0000", 2024-08-17T17:14:33.519 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1662999999999998e-05 2024-08-17T17:14:33.519 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.519 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.519 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.519 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.519 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.519 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:33177 6.f 6:f69ade5d:::benchmark_data_smithi063_160268_object33176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:33.519 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.453545+0000", 2024-08-17T17:14:33.519 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 53.313840052000003, 2024-08-17T17:14:33.519 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.093480868999999994, 2024-08-17T17:14:33.519 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.520 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.520 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:33.520 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.520 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:33.520 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:33.520 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 33177 2024-08-17T17:14:33.520 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.520 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.520 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.520 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.520 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453545+0000", 2024-08-17T17:14:33.520 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.521 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.521 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.521 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.521 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453545+0000", 2024-08-17T17:14:33.521 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.521 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.521 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.521 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.521 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453546+0000", 2024-08-17T17:14:33.521 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4839999999999999e-06 2024-08-17T17:14:33.521 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.521 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.522 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.522 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453567+0000", 2024-08-17T17:14:33.522 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1412000000000001e-05 2024-08-17T17:14:33.522 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.522 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.522 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.522 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453580+0000", 2024-08-17T17:14:33.522 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2354000000000001e-05 2024-08-17T17:14:33.522 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.522 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.522 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.523 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453587+0000", 2024-08-17T17:14:33.523 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1620000000000002e-06 2024-08-17T17:14:33.523 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.523 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.523 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.523 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453640+0000", 2024-08-17T17:14:33.523 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3324e-05 2024-08-17T17:14:33.523 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.523 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.523 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.523 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453770+0000", 2024-08-17T17:14:33.523 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012940299999999999 2024-08-17T17:14:33.524 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.524 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.524 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:14:33.524 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453957+0000", 2024-08-17T17:14:33.524 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000187787 2024-08-17T17:14:33.524 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.524 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.524 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:33.524 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455085+0000", 2024-08-17T17:14:33.524 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001127236 2024-08-17T17:14:33.524 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.524 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.525 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.525 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456745+0000", 2024-08-17T17:14:33.525 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00166062 2024-08-17T17:14:33.525 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.525 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.525 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.525 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546928+0000", 2024-08-17T17:14:33.525 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.090182727000000004 2024-08-17T17:14:33.525 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.525 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.525 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:33.526 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546984+0000", 2024-08-17T17:14:33.526 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6025e-05 2024-08-17T17:14:33.526 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.526 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.526 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.526 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.547025+0000", 2024-08-17T17:14:33.526 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.1335000000000003e-05 2024-08-17T17:14:33.526 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.526 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.526 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.526 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.526 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.527 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:33183 6.f 6:f5dc325e:::benchmark_data_smithi063_160268_object33182:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:33.527 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.455575+0000", 2024-08-17T17:14:33.527 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 53.311809206, 2024-08-17T17:14:33.527 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.096834585000000001, 2024-08-17T17:14:33.527 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.527 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.527 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:33.527 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.527 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:33.527 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:33.527 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 33183 2024-08-17T17:14:33.527 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.528 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.528 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.528 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.528 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455575+0000", 2024-08-17T17:14:33.528 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.528 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.528 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.528 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.528 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455575+0000", 2024-08-17T17:14:33.528 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.528 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.528 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455577+0000", 2024-08-17T17:14:33.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.226e-06 2024-08-17T17:14:33.529 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.529 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455599+0000", 2024-08-17T17:14:33.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2396999999999999e-05 2024-08-17T17:14:33.529 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.529 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455615+0000", 2024-08-17T17:14:33.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5654e-05 2024-08-17T17:14:33.529 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.530 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.530 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.530 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455621+0000", 2024-08-17T17:14:33.530 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8650000000000003e-06 2024-08-17T17:14:33.530 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.530 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.530 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.530 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455683+0000", 2024-08-17T17:14:33.530 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1329999999999997e-05 2024-08-17T17:14:33.530 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.530 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.530 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.531 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455799+0000", 2024-08-17T17:14:33.531 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000116177 2024-08-17T17:14:33.531 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.531 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.531 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:14:33.531 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455877+0000", 2024-08-17T17:14:33.531 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7669999999999993e-05 2024-08-17T17:14:33.531 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.531 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.531 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.531 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541242+0000", 2024-08-17T17:14:33.531 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.085364946999999997 2024-08-17T17:14:33.532 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.532 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.532 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:33.532 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541556+0000", 2024-08-17T17:14:33.532 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00031472500000000002 2024-08-17T17:14:33.532 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.532 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.532 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.532 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.550798+0000", 2024-08-17T17:14:33.532 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0092420709999999993 2024-08-17T17:14:33.532 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.532 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.532 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:33.533 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.552297+0000", 2024-08-17T17:14:33.533 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001498561 2024-08-17T17:14:33.533 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.533 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.533 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.533 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.552410+0000", 2024-08-17T17:14:33.533 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011296199999999999 2024-08-17T17:14:33.533 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.533 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.533 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.533 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.533 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.533 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:33182 6.3 e1093/1086 6:cfa569e0:::benchmark_data_smithi063_160268_object33181:head v 1093'2058, mlcod=1093'2055)", 2024-08-17T17:14:33.534 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.456640+0000", 2024-08-17T17:14:33.534 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 53.310744909, 2024-08-17T17:14:33.534 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.085156834000000001, 2024-08-17T17:14:33.534 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.534 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.534 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:33.534 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.534 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.534 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.534 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456640+0000", 2024-08-17T17:14:33.534 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.534 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.535 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.535 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.535 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456640+0000", 2024-08-17T17:14:33.535 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.535 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.535 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.535 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.535 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456640+0000", 2024-08-17T17:14:33.535 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.75e-07 2024-08-17T17:14:33.535 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.535 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.535 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.536 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456703+0000", 2024-08-17T17:14:33.536 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3109999999999997e-05 2024-08-17T17:14:33.536 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.536 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.536 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.536 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456712+0000", 2024-08-17T17:14:33.536 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.738e-06 2024-08-17T17:14:33.536 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.536 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.536 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.536 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456718+0000", 2024-08-17T17:14:33.536 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8080000000000001e-06 2024-08-17T17:14:33.537 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.537 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.537 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.537 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456755+0000", 2024-08-17T17:14:33.537 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7154999999999999e-05 2024-08-17T17:14:33.537 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.537 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.537 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.537 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456767+0000", 2024-08-17T17:14:33.537 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.201e-05 2024-08-17T17:14:33.537 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.537 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.537 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:33.538 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541750+0000", 2024-08-17T17:14:33.538 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.084982590999999996 2024-08-17T17:14:33.538 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.538 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.538 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.538 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541796+0000", 2024-08-17T17:14:33.538 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6947000000000003e-05 2024-08-17T17:14:33.538 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.538 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.538 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.538 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.538 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.539 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.44609.0:33183 6.f e1093/1086 ondisk, result = 0)", 2024-08-17T17:14:33.539 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.457358+0000", 2024-08-17T17:14:33.539 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 53.310026942, 2024-08-17T17:14:33.539 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.083891389999999996, 2024-08-17T17:14:33.539 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.539 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.539 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:14:33.539 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.539 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.539 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.539 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.457358+0000", 2024-08-17T17:14:33.539 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.539 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.540 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.540 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.540 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.457358+0000", 2024-08-17T17:14:33.540 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.540 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.540 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.540 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.540 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.457358+0000", 2024-08-17T17:14:33.540 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.37e-07 2024-08-17T17:14:33.540 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.540 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.540 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.540 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.457361+0000", 2024-08-17T17:14:33.541 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1530000000000001e-06 2024-08-17T17:14:33.541 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.541 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.541 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.541 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541110+0000", 2024-08-17T17:14:33.541 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.083748323999999999 2024-08-17T17:14:33.541 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.541 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.541 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.541 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541123+0000", 2024-08-17T17:14:33.541 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3538e-05 2024-08-17T17:14:33.541 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.541 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.542 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.542 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541227+0000", 2024-08-17T17:14:33.542 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000104107 2024-08-17T17:14:33.542 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.542 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.542 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.542 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541235+0000", 2024-08-17T17:14:33.542 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.1219999999999995e-06 2024-08-17T17:14:33.542 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.542 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.542 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.542 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541249+0000", 2024-08-17T17:14:33.542 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3609e-05 2024-08-17T17:14:33.543 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.543 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.543 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.543 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.543 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.543 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34212 6.f 6:f4d9d5ce:::benchmark_data_smithi063_160268_object34211:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:33.543 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.042903+0000", 2024-08-17T17:14:33.543 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 52.724481584999999, 2024-08-17T17:14:33.543 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.090290822000000007, 2024-08-17T17:14:33.543 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.543 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.543 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:33.544 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.544 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:33.544 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:33.544 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34212 2024-08-17T17:14:33.544 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.544 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.544 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.544 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.544 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042903+0000", 2024-08-17T17:14:33.544 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.544 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.544 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.544 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042903+0000", 2024-08-17T17:14:33.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.545 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.545 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042904+0000", 2024-08-17T17:14:33.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.409e-06 2024-08-17T17:14:33.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.545 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.545 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042943+0000", 2024-08-17T17:14:33.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8939000000000001e-05 2024-08-17T17:14:33.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.546 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.546 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042978+0000", 2024-08-17T17:14:33.546 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4366999999999998e-05 2024-08-17T17:14:33.546 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.546 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.546 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042985+0000", 2024-08-17T17:14:33.546 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1829999999999997e-06 2024-08-17T17:14:33.546 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.546 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.546 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043397+0000", 2024-08-17T17:14:33.547 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00041226400000000002 2024-08-17T17:14:33.547 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.547 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.547 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.547 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043523+0000", 2024-08-17T17:14:33.547 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000126241 2024-08-17T17:14:33.547 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.547 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.547 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:14:33.547 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043684+0000", 2024-08-17T17:14:33.547 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000160357 2024-08-17T17:14:33.547 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.547 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.548 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.548 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.045624+0000", 2024-08-17T17:14:33.548 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0019398340000000001 2024-08-17T17:14:33.548 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.548 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.548 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:33.548 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.045634+0000", 2024-08-17T17:14:33.548 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0654e-05 2024-08-17T17:14:33.548 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.548 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.548 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.548 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.132934+0000", 2024-08-17T17:14:33.548 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.087299508999999997 2024-08-17T17:14:33.549 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.549 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.549 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:33.549 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.133085+0000", 2024-08-17T17:14:33.549 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015161600000000001 2024-08-17T17:14:33.549 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.549 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.549 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.549 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.133194+0000", 2024-08-17T17:14:33.549 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000108449 2024-08-17T17:14:33.549 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.549 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.549 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.549 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.550 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.550 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35809 6.f 6:f6d7b118:::benchmark_data_smithi063_160268_object35808:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:33.550 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.876618+0000", 2024-08-17T17:14:33.550 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 51.890766757999998, 2024-08-17T17:14:33.550 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14764386800000001, 2024-08-17T17:14:33.550 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.550 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.550 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:33.550 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.550 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:33.550 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:33.550 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35809 2024-08-17T17:14:33.551 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.551 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.551 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.551 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.551 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876618+0000", 2024-08-17T17:14:33.551 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.551 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.551 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.551 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.551 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876618+0000", 2024-08-17T17:14:33.551 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.551 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.551 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.552 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.552 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876620+0000", 2024-08-17T17:14:33.552 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.795e-06 2024-08-17T17:14:33.552 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.552 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.552 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.552 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876643+0000", 2024-08-17T17:14:33.552 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.296e-05 2024-08-17T17:14:33.552 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.552 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.552 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.552 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876654+0000", 2024-08-17T17:14:33.552 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.13e-05 2024-08-17T17:14:33.553 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.553 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.553 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.553 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876661+0000", 2024-08-17T17:14:33.553 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0269999999999998e-06 2024-08-17T17:14:33.553 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.553 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.553 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.553 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876712+0000", 2024-08-17T17:14:33.553 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1388999999999999e-05 2024-08-17T17:14:33.553 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.553 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.554 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.554 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876826+0000", 2024-08-17T17:14:33.554 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000113377 2024-08-17T17:14:33.554 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.554 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.554 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:14:33.554 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876960+0000", 2024-08-17T17:14:33.554 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000134103 2024-08-17T17:14:33.554 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.554 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.554 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.554 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.879644+0000", 2024-08-17T17:14:33.555 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0026843520000000001 2024-08-17T17:14:33.555 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.555 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.555 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:33.555 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.879678+0000", 2024-08-17T17:14:33.555 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3433999999999999e-05 2024-08-17T17:14:33.555 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.555 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.555 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.555 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.023992+0000", 2024-08-17T17:14:33.555 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14431400899999999 2024-08-17T17:14:33.555 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.555 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.556 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:33.556 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.024099+0000", 2024-08-17T17:14:33.556 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000107754 2024-08-17T17:14:33.556 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.556 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.556 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.556 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.024262+0000", 2024-08-17T17:14:33.556 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000162368 2024-08-17T17:14:33.556 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.556 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.556 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.556 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.556 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.557 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35811 6.f 6:f8be1079:::benchmark_data_smithi063_160268_object35810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:33.557 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.877384+0000", 2024-08-17T17:14:33.557 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 51.890000344000001, 2024-08-17T17:14:33.557 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.15051810500000001, 2024-08-17T17:14:33.557 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.557 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.557 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:33.557 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.557 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:33.557 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:33.557 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35811 2024-08-17T17:14:33.557 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.558 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.558 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.558 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.558 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877384+0000", 2024-08-17T17:14:33.558 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.558 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.558 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.558 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.558 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877384+0000", 2024-08-17T17:14:33.558 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.558 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.558 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.558 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.559 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877385+0000", 2024-08-17T17:14:33.559 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.1800000000000004e-07 2024-08-17T17:14:33.559 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.559 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.559 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.559 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877404+0000", 2024-08-17T17:14:33.559 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8766e-05 2024-08-17T17:14:33.559 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.559 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.559 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.559 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877416+0000", 2024-08-17T17:14:33.559 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1770000000000001e-05 2024-08-17T17:14:33.560 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.560 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.560 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.560 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877423+0000", 2024-08-17T17:14:33.560 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0049999999999998e-06 2024-08-17T17:14:33.560 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.560 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.560 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.560 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878297+0000", 2024-08-17T17:14:33.560 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00087477799999999999 2024-08-17T17:14:33.560 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.560 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.560 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.561 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878431+0000", 2024-08-17T17:14:33.561 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013333900000000001 2024-08-17T17:14:33.561 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.561 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.561 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:14:33.561 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878556+0000", 2024-08-17T17:14:33.561 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000124764 2024-08-17T17:14:33.561 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.561 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.561 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:33.561 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880196+0000", 2024-08-17T17:14:33.561 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0016405949999999999 2024-08-17T17:14:33.562 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.562 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.562 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.562 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.881084+0000", 2024-08-17T17:14:33.562 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00088833800000000002 2024-08-17T17:14:33.562 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.562 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.562 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.562 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027510+0000", 2024-08-17T17:14:33.562 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.146425627 2024-08-17T17:14:33.562 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.563 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.563 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:33.563 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027847+0000", 2024-08-17T17:14:33.563 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000336397 2024-08-17T17:14:33.563 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.563 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.563 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.563 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027902+0000", 2024-08-17T17:14:33.563 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5807999999999998e-05 2024-08-17T17:14:33.563 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.563 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.563 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.563 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.564 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.564 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35820 6.f 6:f1fc3e55:::benchmark_data_smithi063_160268_object35819:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:14:33.564 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.882503+0000", 2024-08-17T17:14:33.564 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 51.884881290000003, 2024-08-17T17:14:33.564 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14575781700000001, 2024-08-17T17:14:33.564 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.564 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.564 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:33.564 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:14:33.564 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:14:33.564 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:14:33.564 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35820 2024-08-17T17:14:33.564 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.564 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.565 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.565 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.565 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882503+0000", 2024-08-17T17:14:33.565 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.565 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.565 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.565 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.565 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882503+0000", 2024-08-17T17:14:33.565 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.565 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.565 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.565 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.565 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882505+0000", 2024-08-17T17:14:33.566 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.556e-06 2024-08-17T17:14:33.566 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.566 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.566 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.566 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882528+0000", 2024-08-17T17:14:33.566 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3295999999999999e-05 2024-08-17T17:14:33.566 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.566 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.566 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.566 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882540+0000", 2024-08-17T17:14:33.566 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2099000000000001e-05 2024-08-17T17:14:33.566 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.566 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.567 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.567 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882548+0000", 2024-08-17T17:14:33.567 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7049999999999999e-06 2024-08-17T17:14:33.567 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.567 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.567 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.567 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882591+0000", 2024-08-17T17:14:33.567 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2775999999999998e-05 2024-08-17T17:14:33.567 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.567 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.567 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.567 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882678+0000", 2024-08-17T17:14:33.568 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7777999999999999e-05 2024-08-17T17:14:33.568 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.568 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.568 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:14:33.568 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882761+0000", 2024-08-17T17:14:33.568 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2347000000000003e-05 2024-08-17T17:14:33.568 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.568 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.568 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.568 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.023259+0000", 2024-08-17T17:14:33.568 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14049784100000001 2024-08-17T17:14:33.568 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.568 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.569 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:14:33.569 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.023430+0000", 2024-08-17T17:14:33.569 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000171194 2024-08-17T17:14:33.569 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.569 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.569 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:14:33.569 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.028113+0000", 2024-08-17T17:14:33.569 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0046829660000000002 2024-08-17T17:14:33.569 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.569 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.569 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:33.569 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.028217+0000", 2024-08-17T17:14:33.569 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000104621 2024-08-17T17:14:33.570 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.570 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.570 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.570 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.028261+0000", 2024-08-17T17:14:33.570 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3637999999999998e-05 2024-08-17T17:14:33.570 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.570 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.570 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.570 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.570 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.570 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:35818 6.3 e1093/1086 6:ccea9449:::benchmark_data_smithi063_160268_object35817:head v 1093'2214, mlcod=1093'2211)", 2024-08-17T17:14:33.570 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.882944+0000", 2024-08-17T17:14:33.570 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 51.884440333999997, 2024-08-17T17:14:33.571 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.13971906100000001, 2024-08-17T17:14:33.571 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:14:33.571 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:14:33.571 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:14:33.571 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:14:33.571 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.571 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:14:33.571 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882944+0000", 2024-08-17T17:14:33.571 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.571 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.571 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.571 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:14:33.571 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882944+0000", 2024-08-17T17:14:33.572 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:14:33.572 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.572 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.572 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:14:33.572 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882945+0000", 2024-08-17T17:14:33.572 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.111e-06 2024-08-17T17:14:33.572 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.572 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.572 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:14:33.572 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882976+0000", 2024-08-17T17:14:33.572 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0332999999999999e-05 2024-08-17T17:14:33.572 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.572 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.573 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:14:33.573 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020295+0000", 2024-08-17T17:14:33.573 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.137318948 2024-08-17T17:14:33.573 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.573 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.573 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:14:33.573 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020308+0000", 2024-08-17T17:14:33.573 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3488e-05 2024-08-17T17:14:33.573 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.573 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.573 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:14:33.573 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020379+0000", 2024-08-17T17:14:33.573 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1173000000000004e-05 2024-08-17T17:14:33.574 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.574 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.574 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:14:33.574 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020825+0000", 2024-08-17T17:14:33.574 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00044561600000000002 2024-08-17T17:14:33.574 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.574 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.574 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:14:33.574 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.022624+0000", 2024-08-17T17:14:33.574 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001799599 2024-08-17T17:14:33.574 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:14:33.574 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:14:33.575 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:14:33.575 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.022663+0000", 2024-08-17T17:14:33.575 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8792999999999999e-05 2024-08-17T17:14:33.575 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.575 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.575 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.575 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:14:33.575 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:14:33.575 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:14:33.575 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:33.620 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:33.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:33.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:33.722 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:33.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:33.751 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:33.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:33.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:33 smithi063 ceph-mon[109843]: pgmap v1179: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:14:33.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:14:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:14:33] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.51.0" 2024-08-17T17:14:33.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:33 smithi063 ceph-mon[114756]: pgmap v1179: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:14:33.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:33 smithi162 ceph-mon[104668]: pgmap v1179: 89 pgs: 89 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:14:33.852 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:33.880 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:33.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:33.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:33.982 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:34.025 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:34.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:34.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:34.073 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 67 16 105321 105305 98.208 65.4375 0.0120593 0.0101761 2024-08-17T17:14:34.127 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:34.153 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:34.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:34.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:34.255 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:34.300 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:34.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:34.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:34.402 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:34.428 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:34.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:34.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:34.529 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:34.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:34.577 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:34.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:34.678 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:34.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:34.722 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:34.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:34.823 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:34.854 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:34.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:34.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:34.995 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:35.022 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:35.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:35.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:35.024 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:14:35.074 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 68 16 106266 106250 97.6314 59.0625 0.010162 0.0102365 2024-08-17T17:14:35.124 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:35.149 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:35.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:35.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:35.251 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:35.281 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:35.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:35.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:35.382 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:35.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:35.424 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:35.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:35.431 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:14:35.526 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:35.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:35 smithi063 ceph-mon[109843]: pgmap v1180: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:14:35.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:35 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3759880392' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:14:35.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:35 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:14:35.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:35 smithi063 ceph-mon[114756]: pgmap v1180: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:14:35.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:35 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3759880392' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:14:35.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:35 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:14:35.558 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:35.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:35.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:35.660 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:35.703 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:35.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:35.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:35.805 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:35.833 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:35.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:35.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:35.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:35 smithi162 ceph-mon[104668]: pgmap v1180: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:14:35.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:35 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3759880392' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:14:35.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:35 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:14:35.860 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:14:35.861 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:14:35.934 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:35.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:35.985 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:35.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:36.074 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 69 16 107243 107227 97.1009 61.0625 0.0108106 0.0102934 2024-08-17T17:14:36.086 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:36.149 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:36.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:36.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:36.251 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:36.308 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:36.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:36.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:36.409 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:36.439 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:36.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:36.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:36.541 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:36.586 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:36.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:36.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:36.605 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:36 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:14:36.605 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:36 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:14:36.605 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:36 smithi063 ceph-mon[114756]: osdmap e1101: 8 total, 8 up, 8 in 2024-08-17T17:14:36.607 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:36 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:14:36.607 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:36 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:14:36.607 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:36 smithi063 ceph-mon[109843]: osdmap e1101: 8 total, 8 up, 8 in 2024-08-17T17:14:36.688 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:36.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:36.727 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:36.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:36.827 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:36.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:36 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:14:36.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:36 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:14:36.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:36 smithi162 ceph-mon[104668]: osdmap e1101: 8 total, 8 up, 8 in 2024-08-17T17:14:36.877 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:36.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:36.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:36.979 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:37.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:37.023 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:37.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:37.075 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 70 16 108247 108231 96.6102 62.75 0.00588349 0.0103458 2024-08-17T17:14:37.124 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:37.150 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:37.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:37.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:37.155 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:37.252 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:37.259 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:37.278 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:37.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:37.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:37.379 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:37.406 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:37.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:37.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:37.509 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:37.566 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:37.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:37.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:37.592 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:37 smithi063 ceph-mon[114756]: pgmap v1182: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2024-08-17T17:14:37.592 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:37 smithi063 ceph-mon[109843]: pgmap v1182: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2024-08-17T17:14:37.672 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:37.711 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:37.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:37.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:37.813 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:37.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:37 smithi162 ceph-mon[104668]: pgmap v1182: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 54 MiB/s wr, 867 op/s 2024-08-17T17:14:37.857 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:37.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:37.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:37.959 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:37.985 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:37.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:37.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:38.075 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 71 16 109152 109136 96.0461 56.5625 0.00876703 0.0104062 2024-08-17T17:14:38.087 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:38.137 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:38.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:38.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:38.239 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:38.274 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:38.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:38.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:38.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:38.421 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:38.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:38.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:38.523 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:38.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:38.582 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:38.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:38.643 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:14:38.680 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:38 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3369459726' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:14:38.681 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:38 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3369459726' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:14:38.684 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:38.689 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:14:38.689 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:14:38.689 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:14:38.689 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:14:38.689 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:14:38.689 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:14:38.689 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:14:38.690 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:14:38.690 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:14:38.690 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:14:38.690 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:14:38.690 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:14:38.690 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:14:38.690 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:14:38.690 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:14:38.690 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:14:38.731 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:38.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:38.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:38.833 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:38.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:38 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3369459726' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:14:38.884 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:38.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:38.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:38.986 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:39.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:39.032 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:39.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:39.075 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 72 16 110084 110068 95.5212 58.25 0.0141641 0.0104636 2024-08-17T17:14:39.133 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:39.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:39.174 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:39.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:39.275 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:39.301 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:39.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:39.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:39.403 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:39.438 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:39.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:39.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:39.540 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:39.581 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:39.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:39.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:39.683 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:39.712 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:39.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:39.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:39.731 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:39 smithi063 ceph-mon[114756]: pgmap v1183: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:14:39.731 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:39 smithi063 ceph-mon[109843]: pgmap v1183: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:14:39.814 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:39.843 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:39.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:39.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:39.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:39 smithi162 ceph-mon[104668]: pgmap v1183: 89 pgs: 89 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-08-17T17:14:39.946 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:39.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:39.988 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:39.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:40.075 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 73 16 111095 111079 95.0783 63.1875 0.0116669 0.0105126 2024-08-17T17:14:40.089 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:40.118 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:40.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:40.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:40.220 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:40.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:40.262 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:40.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:40.364 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:40.390 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:40.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:40.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:40.492 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:40.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:40.518 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:40.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:40.619 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:40.658 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:40.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:40.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:40.760 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:40.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:40.798 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:40.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:40.899 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:40.950 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:40.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:40.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:41.053 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:41.075 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 74 16 112041 112025 94.5918 59.125 0.0287858 0.0105664 2024-08-17T17:14:41.076 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:41.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:41.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:41.178 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:41.206 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:41.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:41.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:41.308 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:41.326 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:14:41.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:41.348 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:41.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:41.449 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:41.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:41.495 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:41.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:41.595 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:14:41.596 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:14:41.598 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:41.625 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:41.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:41.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:41.707 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:41 smithi063 ceph-mon[109843]: pgmap v1184: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:14:41.708 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:41 smithi063 ceph-mon[114756]: pgmap v1184: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:14:41.727 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:41.783 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:41.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:41.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:41.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:41 smithi162 ceph-mon[104668]: pgmap v1184: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-08-17T17:14:41.885 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:41.930 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:41.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:41.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:42.033 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:42.060 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:42.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:42.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:42.075 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 75 16 112914 112898 94.0581 54.5625 0.0222403 0.0106265 2024-08-17T17:14:42.163 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:42.206 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:42.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:42.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:42.308 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:42.353 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:42.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:42.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:42.456 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:42.484 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:42.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:42.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:42.586 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:42.625 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:42.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:42.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:42.690 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:42 smithi063 ceph-mon[109843]: from='client.44681 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:14:42.691 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:42 smithi063 ceph-mon[114756]: from='client.44681 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:14:42.725 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1184,"stamp":"2024-08-17T17:14:40.069812+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7101224933,"num_objects":108373,"num_object_clones":0,"num_object_copies":325119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":108373,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":108626,"num_write_kb":6936657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108555,"ondisk_log_size":108555,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21549196,"kb_used_data":20764420,"kb_used_omap":199,"kb_used_meta":784312,"kb_avail":728412020,"statfs":{"total":767960285184,"available":745893908480,"internally_reserved":0,"allocated":21262766080,"data_stored":21246435363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":803136252},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,5,6,9,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":654442496,"num_objects":9986,"num_object_clones":0,"num_object_copies":29958,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9986,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9986,"num_write_kb":639104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.671106"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1279,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.432956+0000","last_change":"2024-08-17T17:07:45.684223+0000","last_active":"2024-08-17T17:14:35.432956+0000","last_peered":"2024-08-17T17:14:35.432956+0000","last_clean":"2024-08-17T17:14:35.432956+0000","last_became_active":"2024-08-17T17:07:45.683948+0000","last_became_peered":"2024-08-17T17:07:45.683948+0000","last_unstale":"2024-08-17T17:14:35.432956+0000","last_undegraded":"2024-08-17T17:14:35.432956+0000","last_fullsized":"2024-08-17T17:14:35.432956+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:07.677423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1025,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.402229+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:14:30.402229+0000","last_peered":"2024-08-17T17:14:30.402229+0000","last_clean":"2024-08-17T17:14:30.402229+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:14:30.402229+0000","last_undegraded":"2024-08-17T17:14:30.402229+0000","last_fullsized":"2024-08-17T17:14:30.402229+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1037,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397289+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:14:30.397289+0000","last_peered":"2024-08-17T17:14:30.397289+0000","last_clean":"2024-08-17T17:14:30.397289+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:14:30.397289+0000","last_undegraded":"2024-08-17T17:14:30.397289+0000","last_fullsized":"2024-08-17T17:14:30.397289+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1400,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.437281+0000","last_change":"2024-08-17T17:07:45.683455+0000","last_active":"2024-08-17T17:14:35.437281+0000","last_peered":"2024-08-17T17:14:35.437281+0000","last_clean":"2024-08-17T17:14:35.437281+0000","last_became_active":"2024-08-17T17:07:45.683169+0000","last_became_peered":"2024-08-17T17:07:45.683169+0000","last_unstale":"2024-08-17T17:14:35.437281+0000","last_undegraded":"2024-08-17T17:14:35.437281+0000","last_fullsized":"2024-08-17T17:14:35.437281+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:18:19.562247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1359,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436543+0000","last_change":"2024-08-17T17:07:45.684222+0000","last_active":"2024-08-17T17:14:35.436543+0000","last_peered":"2024-08-17T17:14:35.436543+0000","last_clean":"2024-08-17T17:14:35.436543+0000","last_became_active":"2024-08-17T17:07:45.683913+0000","last_became_peered":"2024-08-17T17:07:45.683913+0000","last_unstale":"2024-08-17T17:14:35.436543+0000","last_undegraded":"2024-08-17T17:14:35.436543+0000","last_fullsized":"2024-08-17T17:14:35.436543+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:59:33.944853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1009,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.429054+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:14:35.429054+0000","last_peered":"2024-08-17T17:14:35.429054+0000","last_clean":"2024-08-17T17:14:35.429054+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:14:35.429054+0000","last_undegraded":"2024-08-17T17:14:35.429054+0000","last_fullsized":"2024-08-17T17:14:35.429054+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1279,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.440960+0000","last_change":"2024-08-17T17:07:45.687030+0000","last_active":"2024-08-17T17:14:35.440960+0000","last_peered":"2024-08-17T17:14:35.440960+0000","last_clean":"2024-08-17T17:14:35.440960+0000","last_became_active":"2024-08-17T17:07:45.686610+0000","last_became_peered":"2024-08-17T17:07:45.686610+0000","last_unstale":"2024-08-17T17:14:35.440960+0000","last_undegraded":"2024-08-17T17:14:35.440960+0000","last_fullsized":"2024-08-17T17:14:35.440960+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:02.425845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1048,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:31.956714+0000","last_change":"2024-08-17T17:06:07.632931+0000","last_active":"2024-08-17T17:14:31.956714+0000","last_peered":"2024-08-17T17:14:31.956714+0000","last_clean":"2024-08-17T17:14:31.956714+0000","last_became_active":"2024-08-17T17:06:07.632525+0000","last_became_peered":"2024-08-17T17:06:07.632525+0000","last_unstale":"2024-08-17T17:14:31.956714+0000","last_undegraded":"2024-08-17T17:14:31.956714+0000","last_fullsized":"2024-08-17T17:14:31.956714+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:01.740674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"1101'6818","reported_seq":6842,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:36.954494+0000","last_change":"2024-08-17T17:13:13.231789+0000","last_active":"2024-08-17T17:14:36.954494+0000","last_peered":"2024-08-17T17:14:36.954494+0000","last_clean":"2024-08-17T17:14:36.954494+0000","last_became_active":"2024-08-17T17:13:13.231524+0000","last_became_peered":"2024-08-17T17:13:13.231524+0000","last_unstale":"2024-08-17T17:14:36.954494+0000","last_undegraded":"2024-08-17T17:14:36.954494+0000","last_fullsized":"2024-08-17T17:14:36.954494+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6818,"log_dups_size":0,"ondisk_log_size":6818,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:19.506111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":446824448,"num_objects":6818,"num_object_clones":0,"num_object_copies":20454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6818,"num_write_kb":436352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1025,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.399973+0000","last_change":"2024-08-17T17:06:07.638468+0000","last_active":"2024-08-17T17:14:30.399973+0000","last_peered":"2024-08-17T17:14:30.399973+0000","last_clean":"2024-08-17T17:14:30.399973+0000","last_became_active":"2024-08-17T17:06:07.636046+0000","last_became_peered":"2024-08-17T17:06:07.636046+0000","last_unstale":"2024-08-17T17:14:30.399973+0000","last_undegraded":"2024-08-17T17:14:30.399973+0000","last_fullsized":"2024-08-17T17:14:30.399973+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:37:12.228090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1023,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.396463+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:14:30.396463+0000","last_peered":"2024-08-17T17:14:30.396463+0000","last_clean":"2024-08-17T17:14:30.396463+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:14:30.396463+0000","last_undegraded":"2024-08-17T17:14:30.396463+0000","last_fullsized":"2024-08-17T17:14:30.396463+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1101'6887","reported_seq":6928,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:38.883281+0000","last_change":"2024-08-17T17:13:13.227759+0000","last_active":"2024-08-17T17:14:38.883281+0000","last_peered":"2024-08-17T17:14:38.883281+0000","last_clean":"2024-08-17T17:14:38.883281+0000","last_became_active":"2024-08-17T17:13:13.227456+0000","last_became_peered":"2024-08-17T17:13:13.227456+0000","last_unstale":"2024-08-17T17:14:38.883281+0000","last_undegraded":"2024-08-17T17:14:38.883281+0000","last_fullsized":"2024-08-17T17:14:38.883281+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6887,"log_dups_size":0,"ondisk_log_size":6887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:17.924287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":451346432,"num_objects":6887,"num_object_clones":0,"num_object_copies":20661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6887,"num_write_kb":440768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1026,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.399772+0000","last_change":"2024-08-17T17:06:39.638433+0000","last_active":"2024-08-17T17:14:30.399772+0000","last_peered":"2024-08-17T17:14:30.399772+0000","last_clean":"2024-08-17T17:14:30.399772+0000","last_became_active":"2024-08-17T17:06:39.638081+0000","last_became_peered":"2024-08-17T17:06:39.638081+0000","last_unstale":"2024-08-17T17:14:30.399772+0000","last_undegraded":"2024-08-17T17:14:30.399772+0000","last_fullsized":"2024-08-17T17:14:30.399772+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:13.558012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1057,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.403724+0000","last_change":"2024-08-17T17:07:45.693574+0000","last_active":"2024-08-17T17:14:30.403724+0000","last_peered":"2024-08-17T17:14:30.403724+0000","last_clean":"2024-08-17T17:14:30.403724+0000","last_became_active":"2024-08-17T17:07:45.693086+0000","last_became_peered":"2024-08-17T17:07:45.693086+0000","last_unstale":"2024-08-17T17:14:30.403724+0000","last_undegraded":"2024-08-17T17:14:30.403724+0000","last_fullsized":"2024-08-17T17:14:30.403724+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:31.888894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1101'6665","reported_seq":6689,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:36.957815+0000","last_change":"2024-08-17T17:13:13.229168+0000","last_active":"2024-08-17T17:14:36.957815+0000","last_peered":"2024-08-17T17:14:36.957815+0000","last_clean":"2024-08-17T17:14:36.957815+0000","last_became_active":"2024-08-17T17:13:13.228899+0000","last_became_peered":"2024-08-17T17:13:13.228899+0000","last_unstale":"2024-08-17T17:14:36.957815+0000","last_undegraded":"2024-08-17T17:14:36.957815+0000","last_fullsized":"2024-08-17T17:14:36.957815+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6665,"log_dups_size":0,"ondisk_log_size":6665,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:10:26.885228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436797440,"num_objects":6665,"num_object_clones":0,"num_object_copies":19995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6665,"num_write_kb":426560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1010,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.399784+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:14:30.399784+0000","last_peered":"2024-08-17T17:14:30.399784+0000","last_clean":"2024-08-17T17:14:30.399784+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:14:30.399784+0000","last_undegraded":"2024-08-17T17:14:30.399784+0000","last_fullsized":"2024-08-17T17:14:30.399784+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1039,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397755+0000","last_change":"2024-08-17T17:06:39.629368+0000","last_active":"2024-08-17T17:14:30.397755+0000","last_peered":"2024-08-17T17:14:30.397755+0000","last_clean":"2024-08-17T17:14:30.397755+0000","last_became_active":"2024-08-17T17:06:39.628832+0000","last_became_peered":"2024-08-17T17:06:39.628832+0000","last_unstale":"2024-08-17T17:14:30.397755+0000","last_undegraded":"2024-08-17T17:14:30.397755+0000","last_fullsized":"2024-08-17T17:14:30.397755+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:54.063884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1101'6712","reported_seq":6752,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.784116+0000","last_change":"2024-08-17T17:13:13.229994+0000","last_active":"2024-08-17T17:14:35.784116+0000","last_peered":"2024-08-17T17:14:35.784116+0000","last_clean":"2024-08-17T17:14:35.784116+0000","last_became_active":"2024-08-17T17:13:13.229715+0000","last_became_peered":"2024-08-17T17:13:13.229715+0000","last_unstale":"2024-08-17T17:14:35.784116+0000","last_undegraded":"2024-08-17T17:14:35.784116+0000","last_fullsized":"2024-08-17T17:14:35.784116+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6712,"log_dups_size":0,"ondisk_log_size":6712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:18:20.964356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439877632,"num_objects":6712,"num_object_clones":0,"num_object_copies":20136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6712,"num_write_kb":429568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1039,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397060+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:14:30.397060+0000","last_peered":"2024-08-17T17:14:30.397060+0000","last_clean":"2024-08-17T17:14:30.397060+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:14:30.397060+0000","last_undegraded":"2024-08-17T17:14:30.397060+0000","last_fullsized":"2024-08-17T17:14:30.397060+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1372,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436284+0000","last_change":"2024-08-17T17:07:12.651379+0000","last_active":"2024-08-17T17:14:35.436284+0000","last_peered":"2024-08-17T17:14:35.436284+0000","last_clean":"2024-08-17T17:14:35.436284+0000","last_became_active":"2024-08-17T17:07:12.651069+0000","last_became_peered":"2024-08-17T17:07:12.651069+0000","last_unstale":"2024-08-17T17:14:35.436284+0000","last_undegraded":"2024-08-17T17:14:35.436284+0000","last_fullsized":"2024-08-17T17:14:35.436284+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:25.384305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1101'6983","reported_seq":7007,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:36.963348+0000","last_change":"2024-08-17T17:13:13.229385+0000","last_active":"2024-08-17T17:14:36.963348+0000","last_peered":"2024-08-17T17:14:36.963348+0000","last_clean":"2024-08-17T17:14:36.963348+0000","last_became_active":"2024-08-17T17:13:13.229101+0000","last_became_peered":"2024-08-17T17:13:13.229101+0000","last_unstale":"2024-08-17T17:14:36.963348+0000","last_undegraded":"2024-08-17T17:14:36.963348+0000","last_fullsized":"2024-08-17T17:14:36.963348+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6983,"log_dups_size":0,"ondisk_log_size":6983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:26.176057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":457637888,"num_objects":6983,"num_object_clones":0,"num_object_copies":20949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6983,"num_write_kb":446912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1293,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.433171+0000","last_change":"2024-08-17T17:07:45.691437+0000","last_active":"2024-08-17T17:14:35.433171+0000","last_peered":"2024-08-17T17:14:35.433171+0000","last_clean":"2024-08-17T17:14:35.433171+0000","last_became_active":"2024-08-17T17:07:45.690567+0000","last_became_peered":"2024-08-17T17:07:45.690567+0000","last_unstale":"2024-08-17T17:14:35.433171+0000","last_undegraded":"2024-08-17T17:14:35.433171+0000","last_fullsized":"2024-08-17T17:14:35.433171+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:43:07.698912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1042,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.404045+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:14:30.404045+0000","last_peered":"2024-08-17T17:14:30.404045+0000","last_clean":"2024-08-17T17:14:30.404045+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:14:30.404045+0000","last_undegraded":"2024-08-17T17:14:30.404045+0000","last_fullsized":"2024-08-17T17:14:30.404045+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1101'6631","reported_seq":6672,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.733188+0000","last_change":"2024-08-17T17:13:13.227406+0000","last_active":"2024-08-17T17:14:35.733188+0000","last_peered":"2024-08-17T17:14:35.733188+0000","last_clean":"2024-08-17T17:14:35.733188+0000","last_became_active":"2024-08-17T17:13:13.227128+0000","last_became_peered":"2024-08-17T17:13:13.227128+0000","last_unstale":"2024-08-17T17:14:35.733188+0000","last_undegraded":"2024-08-17T17:14:35.733188+0000","last_fullsized":"2024-08-17T17:14:35.733188+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6631,"log_dups_size":0,"ondisk_log_size":6631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:08.671422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":434569216,"num_objects":6631,"num_object_clones":0,"num_object_copies":19893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6631,"num_write_kb":424384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1040,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397931+0000","last_change":"2024-08-17T17:06:39.629619+0000","last_active":"2024-08-17T17:14:30.397931+0000","last_peered":"2024-08-17T17:14:30.397931+0000","last_clean":"2024-08-17T17:14:30.397931+0000","last_became_active":"2024-08-17T17:06:39.629295+0000","last_became_peered":"2024-08-17T17:06:39.629295+0000","last_unstale":"2024-08-17T17:14:30.397931+0000","last_undegraded":"2024-08-17T17:14:30.397931+0000","last_fullsized":"2024-08-17T17:14:30.397931+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:15.874423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1057,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.404565+0000","last_change":"2024-08-17T17:07:12.649178+0000","last_active":"2024-08-17T17:14:30.404565+0000","last_peered":"2024-08-17T17:14:30.404565+0000","last_clean":"2024-08-17T17:14:30.404565+0000","last_became_active":"2024-08-17T17:07:12.648844+0000","last_became_peered":"2024-08-17T17:07:12.648844+0000","last_unstale":"2024-08-17T17:14:30.404565+0000","last_undegraded":"2024-08-17T17:14:30.404565+0000","last_fullsized":"2024-08-17T17:14:30.404565+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:12:06.307736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1101'6709","reported_seq":6750,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:38.865606+0000","last_change":"2024-08-17T17:13:13.225854+0000","last_active":"2024-08-17T17:14:38.865606+0000","last_peered":"2024-08-17T17:14:38.865606+0000","last_clean":"2024-08-17T17:14:38.865606+0000","last_became_active":"2024-08-17T17:13:13.224588+0000","last_became_peered":"2024-08-17T17:13:13.224588+0000","last_unstale":"2024-08-17T17:14:38.865606+0000","last_undegraded":"2024-08-17T17:14:38.865606+0000","last_fullsized":"2024-08-17T17:14:38.865606+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6709,"log_dups_size":0,"ondisk_log_size":6709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:42:46.041842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439681024,"num_objects":6709,"num_object_clones":0,"num_object_copies":20127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6709,"num_write_kb":429376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1043,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.403788+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:14:30.403788+0000","last_peered":"2024-08-17T17:14:30.403788+0000","last_clean":"2024-08-17T17:14:30.403788+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:14:30.403788+0000","last_undegraded":"2024-08-17T17:14:30.403788+0000","last_fullsized":"2024-08-17T17:14:30.403788+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1042,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.403827+0000","last_change":"2024-08-17T17:06:39.627678+0000","last_active":"2024-08-17T17:14:30.403827+0000","last_peered":"2024-08-17T17:14:30.403827+0000","last_clean":"2024-08-17T17:14:30.403827+0000","last_became_active":"2024-08-17T17:06:39.627226+0000","last_became_peered":"2024-08-17T17:06:39.627226+0000","last_unstale":"2024-08-17T17:14:30.403827+0000","last_undegraded":"2024-08-17T17:14:30.403827+0000","last_fullsized":"2024-08-17T17:14:30.403827+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:01.868244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1101'6640","reported_seq":6664,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:36.961194+0000","last_change":"2024-08-17T17:13:13.230277+0000","last_active":"2024-08-17T17:14:36.961194+0000","last_peered":"2024-08-17T17:14:36.961194+0000","last_clean":"2024-08-17T17:14:36.961194+0000","last_became_active":"2024-08-17T17:13:13.229963+0000","last_became_peered":"2024-08-17T17:13:13.229963+0000","last_unstale":"2024-08-17T17:14:36.961194+0000","last_undegraded":"2024-08-17T17:14:36.961194+0000","last_fullsized":"2024-08-17T17:14:36.961194+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6640,"log_dups_size":0,"ondisk_log_size":6640,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:52.522525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435159040,"num_objects":6640,"num_object_clones":0,"num_object_copies":19920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6640,"num_write_kb":424960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1282,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.432450+0000","last_change":"2024-08-17T17:07:45.680458+0000","last_active":"2024-08-17T17:14:35.432450+0000","last_peered":"2024-08-17T17:14:35.432450+0000","last_clean":"2024-08-17T17:14:35.432450+0000","last_became_active":"2024-08-17T17:07:45.680221+0000","last_became_peered":"2024-08-17T17:07:45.680221+0000","last_unstale":"2024-08-17T17:14:35.432450+0000","last_undegraded":"2024-08-17T17:14:35.432450+0000","last_fullsized":"2024-08-17T17:14:35.432450+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:55.083962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1304,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.449617+0000","last_change":"2024-08-17T17:07:45.682866+0000","last_active":"2024-08-17T17:14:35.449617+0000","last_peered":"2024-08-17T17:14:35.449617+0000","last_clean":"2024-08-17T17:14:35.449617+0000","last_became_active":"2024-08-17T17:07:45.682554+0000","last_became_peered":"2024-08-17T17:07:45.682554+0000","last_unstale":"2024-08-17T17:14:35.449617+0000","last_undegraded":"2024-08-17T17:14:35.449617+0000","last_fullsized":"2024-08-17T17:14:35.449617+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:25.975914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.7","version":"1101'6807","reported_seq":6831,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:36.941222+0000","last_change":"2024-08-17T17:13:13.226791+0000","last_active":"2024-08-17T17:14:36.941222+0000","last_peered":"2024-08-17T17:14:36.941222+0000","last_clean":"2024-08-17T17:14:36.941222+0000","last_became_active":"2024-08-17T17:13:13.226491+0000","last_became_peered":"2024-08-17T17:13:13.226491+0000","last_unstale":"2024-08-17T17:14:36.941222+0000","last_undegraded":"2024-08-17T17:14:36.941222+0000","last_fullsized":"2024-08-17T17:14:36.941222+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6807,"log_dups_size":0,"ondisk_log_size":6807,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:10:33.413791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":446103552,"num_objects":6807,"num_object_clones":0,"num_object_copies":20421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6807,"num_write_kb":435648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1356,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.399623+0000","last_change":"2024-08-17T17:06:07.636484+0000","last_active":"2024-08-17T17:14:30.399623+0000","last_peered":"2024-08-17T17:14:30.399623+0000","last_clean":"2024-08-17T17:14:30.399623+0000","last_became_active":"2024-08-17T17:06:07.635998+0000","last_became_peered":"2024-08-17T17:06:07.635998+0000","last_unstale":"2024-08-17T17:14:30.399623+0000","last_undegraded":"2024-08-17T17:14:30.399623+0000","last_fullsized":"2024-08-17T17:14:30.399623+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:16.096587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1053,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.396611+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:14:30.396611+0000","last_peered":"2024-08-17T17:14:30.396611+0000","last_clean":"2024-08-17T17:14:30.396611+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:14:30.396611+0000","last_undegraded":"2024-08-17T17:14:30.396611+0000","last_fullsized":"2024-08-17T17:14:30.396611+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1261,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.432241+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:14:35.432241+0000","last_peered":"2024-08-17T17:14:35.432241+0000","last_clean":"2024-08-17T17:14:35.432241+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:14:35.432241+0000","last_undegraded":"2024-08-17T17:14:35.432241+0000","last_fullsized":"2024-08-17T17:14:35.432241+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1025,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.424042+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:14:35.424042+0000","last_peered":"2024-08-17T17:14:35.424042+0000","last_clean":"2024-08-17T17:14:35.424042+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:14:35.424042+0000","last_undegraded":"2024-08-17T17:14:35.424042+0000","last_fullsized":"2024-08-17T17:14:35.424042+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1101'6747","reported_seq":6786,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:37.748182+0000","last_change":"2024-08-17T17:13:13.225585+0000","last_active":"2024-08-17T17:14:37.748182+0000","last_peered":"2024-08-17T17:14:37.748182+0000","last_clean":"2024-08-17T17:14:37.748182+0000","last_became_active":"2024-08-17T17:13:13.225331+0000","last_became_peered":"2024-08-17T17:13:13.225331+0000","last_unstale":"2024-08-17T17:14:37.748182+0000","last_undegraded":"2024-08-17T17:14:37.748182+0000","last_fullsized":"2024-08-17T17:14:37.748182+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6747,"log_dups_size":0,"ondisk_log_size":6747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:23:44.342905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":442171392,"num_objects":6747,"num_object_clones":0,"num_object_copies":20241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6747,"num_write_kb":431808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1053,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:31.976406+0000","last_change":"2024-08-17T17:06:39.638376+0000","last_active":"2024-08-17T17:14:31.976406+0000","last_peered":"2024-08-17T17:14:31.976406+0000","last_clean":"2024-08-17T17:14:31.976406+0000","last_became_active":"2024-08-17T17:06:39.638050+0000","last_became_peered":"2024-08-17T17:06:39.638050+0000","last_unstale":"2024-08-17T17:14:31.976406+0000","last_undegraded":"2024-08-17T17:14:31.976406+0000","last_fullsized":"2024-08-17T17:14:31.976406+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:05.782492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1401,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.440145+0000","last_change":"2024-08-17T17:07:45.683556+0000","last_active":"2024-08-17T17:14:35.440145+0000","last_peered":"2024-08-17T17:14:35.440145+0000","last_clean":"2024-08-17T17:14:35.440145+0000","last_became_active":"2024-08-17T17:07:45.683232+0000","last_became_peered":"2024-08-17T17:07:45.683232+0000","last_unstale":"2024-08-17T17:14:35.440145+0000","last_undegraded":"2024-08-17T17:14:35.440145+0000","last_fullsized":"2024-08-17T17:14:35.440145+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:16.112417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1372,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436365+0000","last_change":"2024-08-17T17:07:12.651145+0000","last_active":"2024-08-17T17:14:35.436365+0000","last_peered":"2024-08-17T17:14:35.436365+0000","last_clean":"2024-08-17T17:14:35.436365+0000","last_became_active":"2024-08-17T17:07:12.650839+0000","last_became_peered":"2024-08-17T17:07:12.650839+0000","last_unstale":"2024-08-17T17:14:35.436365+0000","last_undegraded":"2024-08-17T17:14:35.436365+0000","last_fullsized":"2024-08-17T17:14:35.436365+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:59.795181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"1101'6801","reported_seq":6825,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:36.925497+0000","last_change":"2024-08-17T17:13:13.228848+0000","last_active":"2024-08-17T17:14:36.925497+0000","last_peered":"2024-08-17T17:14:36.925497+0000","last_clean":"2024-08-17T17:14:36.925497+0000","last_became_active":"2024-08-17T17:13:13.228573+0000","last_became_peered":"2024-08-17T17:13:13.228573+0000","last_unstale":"2024-08-17T17:14:36.925497+0000","last_undegraded":"2024-08-17T17:14:36.925497+0000","last_fullsized":"2024-08-17T17:14:36.925497+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6801,"log_dups_size":0,"ondisk_log_size":6801,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:01.011828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":445710336,"num_objects":6801,"num_object_clones":0,"num_object_copies":20403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6801,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6801,"num_write_kb":435264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1344,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.454278+0000","last_change":"2024-08-17T17:07:45.681107+0000","last_active":"2024-08-17T17:14:35.454278+0000","last_peered":"2024-08-17T17:14:35.454278+0000","last_clean":"2024-08-17T17:14:35.454278+0000","last_became_active":"2024-08-17T17:07:45.680770+0000","last_became_peered":"2024-08-17T17:07:45.680770+0000","last_unstale":"2024-08-17T17:14:35.454278+0000","last_undegraded":"2024-08-17T17:14:35.454278+0000","last_fullsized":"2024-08-17T17:14:35.454278+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:53.820920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1335,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.433129+0000","last_change":"2024-08-17T17:07:12.649485+0000","last_active":"2024-08-17T17:14:35.433129+0000","last_peered":"2024-08-17T17:14:35.433129+0000","last_clean":"2024-08-17T17:14:35.433129+0000","last_became_active":"2024-08-17T17:07:12.649161+0000","last_became_peered":"2024-08-17T17:07:12.649161+0000","last_unstale":"2024-08-17T17:14:35.433129+0000","last_undegraded":"2024-08-17T17:14:35.433129+0000","last_fullsized":"2024-08-17T17:14:35.433129+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:18.370628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1349,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.433525+0000","last_change":"2024-08-17T17:06:39.627820+0000","last_active":"2024-08-17T17:14:35.433525+0000","last_peered":"2024-08-17T17:14:35.433525+0000","last_clean":"2024-08-17T17:14:35.433525+0000","last_became_active":"2024-08-17T17:06:39.627464+0000","last_became_peered":"2024-08-17T17:06:39.627464+0000","last_unstale":"2024-08-17T17:14:35.433525+0000","last_undegraded":"2024-08-17T17:14:35.433525+0000","last_fullsized":"2024-08-17T17:14:35.433525+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:01.500528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1101'6921","reported_seq":6962,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:38.886758+0000","last_change":"2024-08-17T17:13:13.224403+0000","last_active":"2024-08-17T17:14:38.886758+0000","last_peered":"2024-08-17T17:14:38.886758+0000","last_clean":"2024-08-17T17:14:38.886758+0000","last_became_active":"2024-08-17T17:13:13.224119+0000","last_became_peered":"2024-08-17T17:13:13.224119+0000","last_unstale":"2024-08-17T17:14:38.886758+0000","last_undegraded":"2024-08-17T17:14:38.886758+0000","last_fullsized":"2024-08-17T17:14:38.886758+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6921,"log_dups_size":0,"ondisk_log_size":6921,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:20:58.208742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":453574656,"num_objects":6921,"num_object_clones":0,"num_object_copies":20763,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6921,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6921,"num_write_kb":442944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1054,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.396793+0000","last_change":"2024-08-17T17:06:39.628745+0000","last_active":"2024-08-17T17:14:30.396793+0000","last_peered":"2024-08-17T17:14:30.396793+0000","last_clean":"2024-08-17T17:14:30.396793+0000","last_became_active":"2024-08-17T17:06:39.628144+0000","last_became_peered":"2024-08-17T17:06:39.628144+0000","last_unstale":"2024-08-17T17:14:30.396793+0000","last_undegraded":"2024-08-17T17:14:30.396793+0000","last_fullsized":"2024-08-17T17:14:30.396793+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:14.740936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1044,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.404425+0000","last_change":"2024-08-17T17:06:07.637581+0000","last_active":"2024-08-17T17:14:30.404425+0000","last_peered":"2024-08-17T17:14:30.404425+0000","last_clean":"2024-08-17T17:14:30.404425+0000","last_became_active":"2024-08-17T17:06:07.637282+0000","last_became_peered":"2024-08-17T17:06:07.637282+0000","last_unstale":"2024-08-17T17:14:30.404425+0000","last_undegraded":"2024-08-17T17:14:30.404425+0000","last_fullsized":"2024-08-17T17:14:30.404425+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.938734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1345,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.434072+0000","last_change":"2024-08-17T17:07:45.687502+0000","last_active":"2024-08-17T17:14:35.434072+0000","last_peered":"2024-08-17T17:14:35.434072+0000","last_clean":"2024-08-17T17:14:35.434072+0000","last_became_active":"2024-08-17T17:07:45.687197+0000","last_became_peered":"2024-08-17T17:07:45.687197+0000","last_unstale":"2024-08-17T17:14:35.434072+0000","last_undegraded":"2024-08-17T17:14:35.434072+0000","last_fullsized":"2024-08-17T17:14:35.434072+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:23.577838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1101'6696","reported_seq":6736,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.768348+0000","last_change":"2024-08-17T17:13:13.231784+0000","last_active":"2024-08-17T17:14:35.768348+0000","last_peered":"2024-08-17T17:14:35.768348+0000","last_clean":"2024-08-17T17:14:35.768348+0000","last_became_active":"2024-08-17T17:13:13.231500+0000","last_became_peered":"2024-08-17T17:13:13.231500+0000","last_unstale":"2024-08-17T17:14:35.768348+0000","last_undegraded":"2024-08-17T17:14:35.768348+0000","last_fullsized":"2024-08-17T17:14:35.768348+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6696,"log_dups_size":0,"ondisk_log_size":6696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:47.466790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":438829056,"num_objects":6696,"num_object_clones":0,"num_object_copies":20088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6696,"num_write_kb":428544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1055,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:31.965398+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:14:31.965398+0000","last_peered":"2024-08-17T17:14:31.965398+0000","last_clean":"2024-08-17T17:14:31.965398+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:14:31.965398+0000","last_undegraded":"2024-08-17T17:14:31.965398+0000","last_fullsized":"2024-08-17T17:14:31.965398+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1363,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.434239+0000","last_change":"2024-08-17T17:06:39.633975+0000","last_active":"2024-08-17T17:14:35.434239+0000","last_peered":"2024-08-17T17:14:35.434239+0000","last_clean":"2024-08-17T17:14:35.434239+0000","last_became_active":"2024-08-17T17:06:39.631526+0000","last_became_peered":"2024-08-17T17:06:39.631526+0000","last_unstale":"2024-08-17T17:14:35.434239+0000","last_undegraded":"2024-08-17T17:14:35.434239+0000","last_fullsized":"2024-08-17T17:14:35.434239+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:58.542237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1323,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.433190+0000","last_change":"2024-08-17T17:06:39.627577+0000","last_active":"2024-08-17T17:14:35.433190+0000","last_peered":"2024-08-17T17:14:35.433190+0000","last_clean":"2024-08-17T17:14:35.433190+0000","last_became_active":"2024-08-17T17:06:39.626554+0000","last_became_peered":"2024-08-17T17:06:39.626554+0000","last_unstale":"2024-08-17T17:14:35.433190+0000","last_undegraded":"2024-08-17T17:14:35.433190+0000","last_fullsized":"2024-08-17T17:14:35.433190+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:04:48.533057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"1101'6813","reported_seq":6854,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:37.761951+0000","last_change":"2024-08-17T17:13:13.229376+0000","last_active":"2024-08-17T17:14:37.761951+0000","last_peered":"2024-08-17T17:14:37.761951+0000","last_clean":"2024-08-17T17:14:37.761951+0000","last_became_active":"2024-08-17T17:13:13.229090+0000","last_became_peered":"2024-08-17T17:13:13.229090+0000","last_unstale":"2024-08-17T17:14:37.761951+0000","last_undegraded":"2024-08-17T17:14:37.761951+0000","last_fullsized":"2024-08-17T17:14:37.761951+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6813,"log_dups_size":0,"ondisk_log_size":6813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:01:53.813720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":446496768,"num_objects":6813,"num_object_clones":0,"num_object_copies":20439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6813,"num_write_kb":436032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1333,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.434154+0000","last_change":"2024-08-17T17:07:12.645660+0000","last_active":"2024-08-17T17:14:35.434154+0000","last_peered":"2024-08-17T17:14:35.434154+0000","last_clean":"2024-08-17T17:14:35.434154+0000","last_became_active":"2024-08-17T17:07:12.645368+0000","last_became_peered":"2024-08-17T17:07:12.645368+0000","last_unstale":"2024-08-17T17:14:35.434154+0000","last_undegraded":"2024-08-17T17:14:35.434154+0000","last_fullsized":"2024-08-17T17:14:35.434154+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:41:00.157045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1037,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.399849+0000","last_change":"2024-08-17T17:06:07.636239+0000","last_active":"2024-08-17T17:14:30.399849+0000","last_peered":"2024-08-17T17:14:30.399849+0000","last_clean":"2024-08-17T17:14:30.399849+0000","last_became_active":"2024-08-17T17:06:07.635881+0000","last_became_peered":"2024-08-17T17:06:07.635881+0000","last_unstale":"2024-08-17T17:14:30.399849+0000","last_undegraded":"2024-08-17T17:14:30.399849+0000","last_fullsized":"2024-08-17T17:14:30.399849+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:36.001648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1350,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436384+0000","last_change":"2024-08-17T17:06:07.634369+0000","last_active":"2024-08-17T17:14:35.436384+0000","last_peered":"2024-08-17T17:14:35.436384+0000","last_clean":"2024-08-17T17:14:35.436384+0000","last_became_active":"2024-08-17T17:06:07.633769+0000","last_became_peered":"2024-08-17T17:06:07.633769+0000","last_unstale":"2024-08-17T17:14:35.436384+0000","last_undegraded":"2024-08-17T17:14:35.436384+0000","last_fullsized":"2024-08-17T17:14:35.436384+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:20.957493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1101'6773","reported_seq":6796,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:36.949665+0000","last_change":"2024-08-17T17:13:13.227597+0000","last_active":"2024-08-17T17:14:36.949665+0000","last_peered":"2024-08-17T17:14:36.949665+0000","last_clean":"2024-08-17T17:14:36.949665+0000","last_became_active":"2024-08-17T17:13:13.227117+0000","last_became_peered":"2024-08-17T17:13:13.227117+0000","last_unstale":"2024-08-17T17:14:36.949665+0000","last_undegraded":"2024-08-17T17:14:36.949665+0000","last_fullsized":"2024-08-17T17:14:36.949665+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6773,"log_dups_size":0,"ondisk_log_size":6773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:25.337770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":443875328,"num_objects":6773,"num_object_clones":0,"num_object_copies":20319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6773,"num_write_kb":433472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1293,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.441171+0000","last_change":"2024-08-17T17:07:45.688526+0000","last_active":"2024-08-17T17:14:35.441171+0000","last_peered":"2024-08-17T17:14:35.441171+0000","last_clean":"2024-08-17T17:14:35.441171+0000","last_became_active":"2024-08-17T17:07:45.688091+0000","last_became_peered":"2024-08-17T17:07:45.688091+0000","last_unstale":"2024-08-17T17:14:35.441171+0000","last_undegraded":"2024-08-17T17:14:35.441171+0000","last_fullsized":"2024-08-17T17:14:35.441171+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:02.385683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1333,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436743+0000","last_change":"2024-08-17T17:06:39.629005+0000","last_active":"2024-08-17T17:14:35.436743+0000","last_peered":"2024-08-17T17:14:35.436743+0000","last_clean":"2024-08-17T17:14:35.436743+0000","last_became_active":"2024-08-17T17:06:39.628679+0000","last_became_peered":"2024-08-17T17:06:39.628679+0000","last_unstale":"2024-08-17T17:14:35.436743+0000","last_undegraded":"2024-08-17T17:14:35.436743+0000","last_fullsized":"2024-08-17T17:14:35.436743+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:09.401059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1290,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.432189+0000","last_change":"2024-08-17T17:07:45.695858+0000","last_active":"2024-08-17T17:14:35.432189+0000","last_peered":"2024-08-17T17:14:35.432189+0000","last_clean":"2024-08-17T17:14:35.432189+0000","last_became_active":"2024-08-17T17:07:45.694111+0000","last_became_peered":"2024-08-17T17:07:45.694111+0000","last_unstale":"2024-08-17T17:14:35.432189+0000","last_undegraded":"2024-08-17T17:14:35.432189+0000","last_fullsized":"2024-08-17T17:14:35.432189+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:40:38.772090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1101'6744","reported_seq":6768,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:36.954797+0000","last_change":"2024-08-17T17:13:13.229123+0000","last_active":"2024-08-17T17:14:36.954797+0000","last_peered":"2024-08-17T17:14:36.954797+0000","last_clean":"2024-08-17T17:14:36.954797+0000","last_became_active":"2024-08-17T17:13:13.228818+0000","last_became_peered":"2024-08-17T17:13:13.228818+0000","last_unstale":"2024-08-17T17:14:36.954797+0000","last_undegraded":"2024-08-17T17:14:36.954797+0000","last_fullsized":"2024-08-17T17:14:36.954797+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":6744,"log_dups_size":0,"ondisk_log_size":6744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:05:54.121915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":441974784,"num_objects":6744,"num_object_clones":0,"num_object_copies":20232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6744,"num_write_kb":431616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":1363,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436630+0000","last_change":"2024-08-17T17:06:39.636866+0000","last_active":"2024-08-17T17:14:35.436630+0000","last_peered":"2024-08-17T17:14:35.436630+0000","last_clean":"2024-08-17T17:14:35.436630+0000","last_became_active":"2024-08-17T17:06:39.636470+0000","last_became_peered":"2024-08-17T17:06:39.636470+0000","last_unstale":"2024-08-17T17:14:35.436630+0000","last_undegraded":"2024-08-17T17:14:35.436630+0000","last_fullsized":"2024-08-17T17:14:35.436630+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:14.028712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1052,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397528+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:14:30.397528+0000","last_peered":"2024-08-17T17:14:30.397528+0000","last_clean":"2024-08-17T17:14:30.397528+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:14:30.397528+0000","last_undegraded":"2024-08-17T17:14:30.397528+0000","last_fullsized":"2024-08-17T17:14:30.397528+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1028,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.403564+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:14:30.403564+0000","last_peered":"2024-08-17T17:14:30.403564+0000","last_clean":"2024-08-17T17:14:30.403564+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:14:30.403564+0000","last_undegraded":"2024-08-17T17:14:30.403564+0000","last_fullsized":"2024-08-17T17:14:30.403564+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1041,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.404310+0000","last_change":"2024-08-17T17:06:39.627594+0000","last_active":"2024-08-17T17:14:30.404310+0000","last_peered":"2024-08-17T17:14:30.404310+0000","last_clean":"2024-08-17T17:14:30.404310+0000","last_became_active":"2024-08-17T17:06:39.627105+0000","last_became_peered":"2024-08-17T17:06:39.627105+0000","last_unstale":"2024-08-17T17:14:30.404310+0000","last_undegraded":"2024-08-17T17:14:30.404310+0000","last_fullsized":"2024-08-17T17:14:30.404310+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:30.105070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1050,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.404241+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:14:30.404241+0000","last_peered":"2024-08-17T17:14:30.404241+0000","last_clean":"2024-08-17T17:14:30.404241+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:14:30.404241+0000","last_undegraded":"2024-08-17T17:14:30.404241+0000","last_fullsized":"2024-08-17T17:14:30.404241+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1334,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.432964+0000","last_change":"2024-08-17T17:07:12.645440+0000","last_active":"2024-08-17T17:14:35.432964+0000","last_peered":"2024-08-17T17:14:35.432964+0000","last_clean":"2024-08-17T17:14:35.432964+0000","last_became_active":"2024-08-17T17:07:12.645147+0000","last_became_peered":"2024-08-17T17:07:12.645147+0000","last_unstale":"2024-08-17T17:14:35.432964+0000","last_undegraded":"2024-08-17T17:14:35.432964+0000","last_fullsized":"2024-08-17T17:14:35.432964+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:25.588528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1026,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.399662+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:14:30.399662+0000","last_peered":"2024-08-17T17:14:30.399662+0000","last_clean":"2024-08-17T17:14:30.399662+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:14:30.399662+0000","last_undegraded":"2024-08-17T17:14:30.399662+0000","last_fullsized":"2024-08-17T17:14:30.399662+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1038,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397577+0000","last_change":"2024-08-17T17:06:39.625029+0000","last_active":"2024-08-17T17:14:30.397577+0000","last_peered":"2024-08-17T17:14:30.397577+0000","last_clean":"2024-08-17T17:14:30.397577+0000","last_became_active":"2024-08-17T17:06:39.624688+0000","last_became_peered":"2024-08-17T17:06:39.624688+0000","last_unstale":"2024-08-17T17:14:30.397577+0000","last_undegraded":"2024-08-17T17:14:30.397577+0000","last_fullsized":"2024-08-17T17:14:30.397577+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:54.849668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1260,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.432082+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:14:35.432082+0000","last_peered":"2024-08-17T17:14:35.432082+0000","last_clean":"2024-08-17T17:14:35.432082+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:14:35.432082+0000","last_undegraded":"2024-08-17T17:14:35.432082+0000","last_fullsized":"2024-08-17T17:14:35.432082+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1400,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436951+0000","last_change":"2024-08-17T17:07:45.690099+0000","last_active":"2024-08-17T17:14:35.436951+0000","last_peered":"2024-08-17T17:14:35.436951+0000","last_clean":"2024-08-17T17:14:35.436951+0000","last_became_active":"2024-08-17T17:07:45.689783+0000","last_became_peered":"2024-08-17T17:07:45.689783+0000","last_unstale":"2024-08-17T17:14:35.436951+0000","last_undegraded":"2024-08-17T17:14:35.436951+0000","last_fullsized":"2024-08-17T17:14:35.436951+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:38.311630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1279,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.432105+0000","last_change":"2024-08-17T17:07:45.678950+0000","last_active":"2024-08-17T17:14:35.432105+0000","last_peered":"2024-08-17T17:14:35.432105+0000","last_clean":"2024-08-17T17:14:35.432105+0000","last_became_active":"2024-08-17T17:07:45.678472+0000","last_became_peered":"2024-08-17T17:07:45.678472+0000","last_unstale":"2024-08-17T17:14:35.432105+0000","last_undegraded":"2024-08-17T17:14:35.432105+0000","last_fullsized":"2024-08-17T17:14:35.432105+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:48:02.385927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1040,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397398+0000","last_change":"2024-08-17T17:06:39.628542+0000","last_active":"2024-08-17T17:14:30.397398+0000","last_peered":"2024-08-17T17:14:30.397398+0000","last_clean":"2024-08-17T17:14:30.397398+0000","last_became_active":"2024-08-17T17:06:39.628253+0000","last_became_peered":"2024-08-17T17:06:39.628253+0000","last_unstale":"2024-08-17T17:14:30.397398+0000","last_undegraded":"2024-08-17T17:14:30.397398+0000","last_fullsized":"2024-08-17T17:14:30.397398+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:22.349326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1295,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.432356+0000","last_change":"2024-08-17T17:07:45.696068+0000","last_active":"2024-08-17T17:14:35.432356+0000","last_peered":"2024-08-17T17:14:35.432356+0000","last_clean":"2024-08-17T17:14:35.432356+0000","last_became_active":"2024-08-17T17:07:45.694617+0000","last_became_peered":"2024-08-17T17:07:45.694617+0000","last_unstale":"2024-08-17T17:14:35.432356+0000","last_undegraded":"2024-08-17T17:14:35.432356+0000","last_fullsized":"2024-08-17T17:14:35.432356+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:08.646380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1361,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436456+0000","last_change":"2024-08-17T17:07:45.684186+0000","last_active":"2024-08-17T17:14:35.436456+0000","last_peered":"2024-08-17T17:14:35.436456+0000","last_clean":"2024-08-17T17:14:35.436456+0000","last_became_active":"2024-08-17T17:07:45.683863+0000","last_became_peered":"2024-08-17T17:07:45.683863+0000","last_unstale":"2024-08-17T17:14:35.436456+0000","last_undegraded":"2024-08-17T17:14:35.436456+0000","last_fullsized":"2024-08-17T17:14:35.436456+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:52.106168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1333,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436934+0000","last_change":"2024-08-17T17:07:45.688645+0000","last_active":"2024-08-17T17:14:35.436934+0000","last_peered":"2024-08-17T17:14:35.436934+0000","last_clean":"2024-08-17T17:14:35.436934+0000","last_became_active":"2024-08-17T17:07:45.688383+0000","last_became_peered":"2024-08-17T17:07:45.688383+0000","last_unstale":"2024-08-17T17:14:35.436934+0000","last_undegraded":"2024-08-17T17:14:35.436934+0000","last_fullsized":"2024-08-17T17:14:35.436934+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:33:12.209565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1279,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.432806+0000","last_change":"2024-08-17T17:07:45.689782+0000","last_active":"2024-08-17T17:14:35.432806+0000","last_peered":"2024-08-17T17:14:35.432806+0000","last_clean":"2024-08-17T17:14:35.432806+0000","last_became_active":"2024-08-17T17:07:45.689312+0000","last_became_peered":"2024-08-17T17:07:45.689312+0000","last_unstale":"2024-08-17T17:14:35.432806+0000","last_undegraded":"2024-08-17T17:14:35.432806+0000","last_fullsized":"2024-08-17T17:14:35.432806+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:07.560555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1058,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.404712+0000","last_change":"2024-08-17T17:07:12.649175+0000","last_active":"2024-08-17T17:14:30.404712+0000","last_peered":"2024-08-17T17:14:30.404712+0000","last_clean":"2024-08-17T17:14:30.404712+0000","last_became_active":"2024-08-17T17:07:12.648822+0000","last_became_peered":"2024-08-17T17:07:12.648822+0000","last_unstale":"2024-08-17T17:14:30.404712+0000","last_undegraded":"2024-08-17T17:14:30.404712+0000","last_fullsized":"2024-08-17T17:14:30.404712+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:03.924431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1320,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.442140+0000","last_change":"2024-08-17T17:07:45.685278+0000","last_active":"2024-08-17T17:14:35.442140+0000","last_peered":"2024-08-17T17:14:35.442140+0000","last_clean":"2024-08-17T17:14:35.442140+0000","last_became_active":"2024-08-17T17:07:45.684963+0000","last_became_peered":"2024-08-17T17:07:45.684963+0000","last_unstale":"2024-08-17T17:14:35.442140+0000","last_undegraded":"2024-08-17T17:14:35.442140+0000","last_fullsized":"2024-08-17T17:14:35.442140+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:55:42.495791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":1351,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436691+0000","last_change":"2024-08-17T17:06:07.634103+0000","last_active":"2024-08-17T17:14:35.436691+0000","last_peered":"2024-08-17T17:14:35.436691+0000","last_clean":"2024-08-17T17:14:35.436691+0000","last_became_active":"2024-08-17T17:06:07.633624+0000","last_became_peered":"2024-08-17T17:06:07.633624+0000","last_unstale":"2024-08-17T17:14:35.436691+0000","last_undegraded":"2024-08-17T17:14:35.436691+0000","last_fullsized":"2024-08-17T17:14:35.436691+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:58.573558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1056,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.404286+0000","last_change":"2024-08-17T17:07:45.684606+0000","last_active":"2024-08-17T17:14:30.404286+0000","last_peered":"2024-08-17T17:14:30.404286+0000","last_clean":"2024-08-17T17:14:30.404286+0000","last_became_active":"2024-08-17T17:07:45.684124+0000","last_became_peered":"2024-08-17T17:07:45.684124+0000","last_unstale":"2024-08-17T17:14:30.404286+0000","last_undegraded":"2024-08-17T17:14:30.404286+0000","last_fullsized":"2024-08-17T17:14:30.404286+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:16.110507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1040,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.396574+0000","last_change":"2024-08-17T17:06:39.625245+0000","last_active":"2024-08-17T17:14:30.396574+0000","last_peered":"2024-08-17T17:14:30.396574+0000","last_clean":"2024-08-17T17:14:30.396574+0000","last_became_active":"2024-08-17T17:06:39.624657+0000","last_became_peered":"2024-08-17T17:06:39.624657+0000","last_unstale":"2024-08-17T17:14:30.396574+0000","last_undegraded":"2024-08-17T17:14:30.396574+0000","last_fullsized":"2024-08-17T17:14:30.396574+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:43.097460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1056,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.404528+0000","last_change":"2024-08-17T17:07:45.684769+0000","last_active":"2024-08-17T17:14:30.404528+0000","last_peered":"2024-08-17T17:14:30.404528+0000","last_clean":"2024-08-17T17:14:30.404528+0000","last_became_active":"2024-08-17T17:07:45.684459+0000","last_became_peered":"2024-08-17T17:07:45.684459+0000","last_unstale":"2024-08-17T17:14:30.404528+0000","last_undegraded":"2024-08-17T17:14:30.404528+0000","last_fullsized":"2024-08-17T17:14:30.404528+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:30.731978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1025,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397370+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:14:30.397370+0000","last_peered":"2024-08-17T17:14:30.397370+0000","last_clean":"2024-08-17T17:14:30.397370+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:14:30.397370+0000","last_undegraded":"2024-08-17T17:14:30.397370+0000","last_fullsized":"2024-08-17T17:14:30.397370+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1285,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.432469+0000","last_change":"2024-08-17T17:07:45.692127+0000","last_active":"2024-08-17T17:14:35.432469+0000","last_peered":"2024-08-17T17:14:35.432469+0000","last_clean":"2024-08-17T17:14:35.432469+0000","last_became_active":"2024-08-17T17:07:45.690277+0000","last_became_peered":"2024-08-17T17:07:45.690277+0000","last_unstale":"2024-08-17T17:14:35.432469+0000","last_undegraded":"2024-08-17T17:14:35.432469+0000","last_fullsized":"2024-08-17T17:14:35.432469+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:04.555409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1274,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.433999+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:14:35.433999+0000","last_peered":"2024-08-17T17:14:35.433999+0000","last_clean":"2024-08-17T17:14:35.433999+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:14:35.433999+0000","last_undegraded":"2024-08-17T17:14:35.433999+0000","last_fullsized":"2024-08-17T17:14:35.433999+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1041,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.403344+0000","last_change":"2024-08-17T17:06:39.627689+0000","last_active":"2024-08-17T17:14:30.403344+0000","last_peered":"2024-08-17T17:14:30.403344+0000","last_clean":"2024-08-17T17:14:30.403344+0000","last_became_active":"2024-08-17T17:06:39.627375+0000","last_became_peered":"2024-08-17T17:06:39.627375+0000","last_unstale":"2024-08-17T17:14:30.403344+0000","last_undegraded":"2024-08-17T17:14:30.403344+0000","last_fullsized":"2024-08-17T17:14:30.403344+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:51.954076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":1367,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.437049+0000","last_change":"2024-08-17T17:07:12.651933+0000","last_active":"2024-08-17T17:14:35.437049+0000","last_peered":"2024-08-17T17:14:35.437049+0000","last_clean":"2024-08-17T17:14:35.437049+0000","last_became_active":"2024-08-17T17:07:12.651592+0000","last_became_peered":"2024-08-17T17:07:12.651592+0000","last_unstale":"2024-08-17T17:14:35.437049+0000","last_undegraded":"2024-08-17T17:14:35.437049+0000","last_fullsized":"2024-08-17T17:14:35.437049+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:31.224038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":7100628992,"num_objects":108347,"num_object_clones":0,"num_object_copies":325041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":108347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":108347,"num_write_kb":6934208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21290876928,"data_stored":21290876928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108347,"ondisk_log_size":108347,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201558,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2697096,"kb_used_data":2593852,"kb_used_omap":1,"kb_used_meta":103230,"kb_avail":91048056,"statfs":{"total":95995035648,"available":93233209344,"internally_reserved":0,"allocated":2656104448,"data_stored":2654197713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":105707979},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463195,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2272064,"kb_used_data":2164724,"kb_used_omap":1,"kb_used_meta":107326,"kb_avail":91473088,"statfs":{"total":95995035648,"available":93668442112,"internally_reserved":0,"allocated":2216677376,"data_stored":2214778706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":109902281},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626837,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4487852,"kb_used_data":4375204,"kb_used_omap":42,"kb_used_meta":112597,"kb_avail":89257300,"statfs":{"total":95995035648,"available":91399475200,"internally_reserved":0,"allocated":4480208896,"data_stored":4478054143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":115299388},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785314,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2243308,"kb_used_data":2179108,"kb_used_omap":48,"kb_used_meta":64143,"kb_avail":91501844,"statfs":{"total":95995035648,"available":93697888256,"internally_reserved":0,"allocated":2231406592,"data_stored":2229251768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":65683118},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724831,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2265460,"kb_used_data":2135976,"kb_used_omap":1,"kb_used_meta":129470,"kb_avail":91479692,"statfs":{"total":95995035648,"available":93675204608,"internally_reserved":0,"allocated":2187239424,"data_stored":2185348324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":132577734},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986468,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2301952,"kb_used_data":2180660,"kb_used_omap":1,"kb_used_meta":121278,"kb_avail":91443200,"statfs":{"total":95995035648,"available":93637836800,"internally_reserved":0,"allocated":2232995840,"data_stored":2231096683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":124189126},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706075,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3512680,"kb_used_data":3430504,"kb_used_omap":35,"kb_used_meta":81948,"kb_avail":90232472,"statfs":{"total":95995035648,"available":92398051328,"internally_reserved":0,"allocated":3512836096,"data_stored":3510677773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":83914807},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831848,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1768784,"kb_used_data":1704392,"kb_used_omap":65,"kb_used_meta":64318,"kb_avail":91976368,"statfs":{"total":95995035648,"available":94183800832,"internally_reserved":0,"allocated":1745297408,"data_stored":1743030253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":65861819},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1750466560,"data_stored":1750466560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2236022784,"data_stored":2236022784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3513384960,"data_stored":3513384960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4488298496,"data_stored":4488298496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2233729024,"data_stored":2233729024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2199912448,"data_stored":2199912448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2210201600,"data_stored":2210201600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2658861056,"data_stored":2658861056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:14:42.731 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:14:42.731 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:42.759 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:42.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:42.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:42.796 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:14:42.796 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:14:42.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:42 smithi162 ceph-mon[104668]: from='client.44681 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:14:42.861 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:42.916 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:42.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:42.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:43.018 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:43.044 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:43.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:43.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:43.077 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 76 16 113834 113818 93.576 57.5 0.0137498 0.0106815 2024-08-17T17:14:43.145 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:43.187 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:43.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:43.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:43.291 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:43.351 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:43.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:43.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:43.453 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:43.508 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:43.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:43.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:43.609 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:43.636 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:43.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:43.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:43.738 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:43.763 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:14:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:14:43] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.51.0" 2024-08-17T17:14:43.763 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:43 smithi063 ceph-mon[114756]: pgmap v1185: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2024-08-17T17:14:43.767 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:43 smithi063 ceph-mon[109843]: pgmap v1185: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2024-08-17T17:14:43.769 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:43.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:43.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:43.777 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:43.820 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:14:43.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:43 smithi162 ceph-mon[104668]: pgmap v1185: 89 pgs: 89 active+clean; 6.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s 2024-08-17T17:14:43.871 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:43.897 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:43.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:43.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:43.998 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:44.070 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:44.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:44.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:44.077 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 77 16 114721 114705 93.0807 55.4375 0.0450686 0.0107368 2024-08-17T17:14:44.173 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:44.222 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:44.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:44.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:44.324 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:44.353 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:44.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:44.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:44.455 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:44.488 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:44.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:44.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:44.589 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:44.616 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:44.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:44.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:44.718 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:44.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:44.755 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:44.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:44.856 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:44.896 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:44.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:44.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:44.998 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:45.036 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:45.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:45.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:45.076 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 78 16 115559 115543 92.5589 52.375 0.00955245 0.0107981 2024-08-17T17:14:45.138 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:45.165 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:45.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:45.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:45.267 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:45.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:45.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:45.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:45.401 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:45.433 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:45.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:45.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:45.534 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:45.560 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:45.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:45.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:45.662 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:45.687 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:45.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:45.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:45.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:45 smithi063 ceph-mon[109843]: pgmap v1186: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s 2024-08-17T17:14:45.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:45 smithi063 ceph-mon[114756]: pgmap v1186: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s 2024-08-17T17:14:45.789 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:45.831 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:45.840 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:45.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:45 smithi162 ceph-mon[104668]: pgmap v1186: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 961 op/s 2024-08-17T17:14:45.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:45.855 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:45.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:45.956 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:45.980 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:45.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:45.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:46.077 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 79 16 116401 116385 92.0534 52.625 0.0189353 0.0108579 2024-08-17T17:14:46.082 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:46.107 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:46.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:46.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:46.210 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:46.236 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:46.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:46.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:46.338 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:46.389 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:46.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:46.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:46.491 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:46.541 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:46.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:46.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:46.643 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:46.690 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:46.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:46.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:46.754 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:46.754 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:46.792 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:46.818 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:46.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:46.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:46.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:14:46.920 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:46.958 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:46.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:46.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:47.060 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:47.077 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:14:47.077290+0000 min lat: 0.0014202 max lat: 0.151093 avg lat: 0.0109147 2024-08-17T17:14:47.077 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:14:47.077 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 80 16 117269 117253 91.5809 54.25 0.0125056 0.0109147 2024-08-17T17:14:47.090 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:47.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:47.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:47.192 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:47.235 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:47.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:47.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:47.337 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:47.368 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:47.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:47.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:47.470 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:47.511 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:47.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:47.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:47.573 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:14:47.613 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:47.640 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:47.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:47.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:47.741 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:47.770 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:47.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:47.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:47.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:47 smithi063 ceph-mon[114756]: pgmap v1187: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2024-08-17T17:14:47.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:47 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/933355315' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:14:47.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:47 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:14:47.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:47 smithi063 ceph-mon[109843]: pgmap v1187: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2024-08-17T17:14:47.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:47 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/933355315' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:14:47.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:47 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:14:47.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:47 smithi162 ceph-mon[104668]: pgmap v1187: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2024-08-17T17:14:47.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:47 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/933355315' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:14:47.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:47 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:14:47.873 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:47.911 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:47.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:47.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:48.013 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:48.044 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:48.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:48.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:48.077 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 81 16 118196 118180 91.1655 57.9375 0.0129903 0.0109641 2024-08-17T17:14:48.147 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:48.173 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:48.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:48.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:48.275 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:48.316 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:48.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:48.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:48.418 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:48.449 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:48.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:48.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:48.550 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:48.555 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:48.572 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:48.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:48.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:48.674 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:48.678 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:48 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:14:48.679 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:48 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:14:48.679 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:48 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:14:48.679 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:48 smithi063 ceph-mon[114756]: osdmap e1102: 8 total, 8 up, 8 in 2024-08-17T17:14:48.679 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:48 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:14:48.679 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:48 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:14:48.679 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:48 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:14:48.680 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:48 smithi063 ceph-mon[109843]: osdmap e1102: 8 total, 8 up, 8 in 2024-08-17T17:14:48.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:48.705 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:48.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:48.806 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:48.836 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:48.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:48.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:48.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:48 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:14:48.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:48 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:14:48.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:48 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:14:48.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:48 smithi162 ceph-mon[104668]: osdmap e1102: 8 total, 8 up, 8 in 2024-08-17T17:14:48.938 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:48.973 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:48.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:48.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:49.077 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:49.078 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 82 16 119141 119125 90.774 59.0625 0.0518408 0.0110101 2024-08-17T17:14:49.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:49.105 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:49.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:49.206 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:49.235 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:49.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:49.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:49.337 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:49.364 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:49.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:49.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:49.466 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:49.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:49.515 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:49.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:49.616 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:49.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:49.642 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:49.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:49.716 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:49 smithi063 ceph-mon[109843]: pgmap v1189: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2024-08-17T17:14:49.716 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:49 smithi063 ceph-mon[114756]: pgmap v1189: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2024-08-17T17:14:49.743 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:49.770 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:49.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:49.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:49.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:49 smithi162 ceph-mon[104668]: pgmap v1189: 89 pgs: 89 active+clean; 7.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2024-08-17T17:14:49.872 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:49.898 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:49.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:49.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:50.000 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:50.041 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:50.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:50.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:50.077 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 83 16 120028 120012 90.3483 55.4375 0.00723059 0.0110629 2024-08-17T17:14:50.143 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:50.172 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:50.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:50.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:50.275 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:50.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:50.322 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:50.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:50.423 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:50.454 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:50.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:50.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:50.555 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:50.595 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:50.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:50.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:50.697 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:50.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:50.725 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:50.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:50.826 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:50.857 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:50.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:50.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:50.878 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:14:50.879 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:14:50.959 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:50.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:50.995 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:50.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:51.077 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 84 16 120916 120900 89.9334 55.5 0.00794314 0.0111143 2024-08-17T17:14:51.096 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:51.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:51.124 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:51.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:51.225 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:51.266 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:51.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:51.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:51.369 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:51.408 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:51.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:51.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:51.510 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:51.554 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:51.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:51.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:51.656 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:51.697 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:51.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:51.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:51.798 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:51.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:51.826 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:51.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:51.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:51 smithi162 ceph-mon[104668]: pgmap v1190: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2024-08-17T17:14:51.917 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:51 smithi063 ceph-mon[114756]: pgmap v1190: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2024-08-17T17:14:51.917 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:51 smithi063 ceph-mon[109843]: pgmap v1190: 89 pgs: 89 active+clean; 7.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2024-08-17T17:14:51.928 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:51.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:51.991 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:51.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:52.077 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 85 16 121775 121759 89.507 53.6875 0.0606571 0.0111666 2024-08-17T17:14:52.093 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:52.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:52.132 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:52.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:52.233 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:52.274 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:52.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:52.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:52.376 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:52.407 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:52.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:52.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:52.509 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:52.537 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:52.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:52.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:52.639 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:52.678 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:52.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:52.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:52.780 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:52.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:52.813 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:52.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:52.825 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1190,"stamp":"2024-08-17T17:14:50.079433+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7698323429,"num_objects":117484,"num_object_clones":0,"num_object_copies":352452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":117484,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":117737,"num_write_kb":7519761,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117366,"ondisk_log_size":117366,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":23346996,"kb_used_data":22521908,"kb_used_omap":199,"kb_used_meta":824824,"kb_avail":726614220,"statfs":{"total":767960285184,"available":744052961280,"internally_reserved":0,"allocated":23062433792,"data_stored":23046204843,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":844620540},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":597098496,"num_objects":9111,"num_object_clones":0,"num_object_copies":27333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9111,"num_write_kb":583104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009620"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1282,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.594278+0000","last_change":"2024-08-17T17:07:45.684223+0000","last_active":"2024-08-17T17:14:47.594278+0000","last_peered":"2024-08-17T17:14:47.594278+0000","last_clean":"2024-08-17T17:14:47.594278+0000","last_became_active":"2024-08-17T17:07:45.683948+0000","last_became_peered":"2024-08-17T17:07:45.683948+0000","last_unstale":"2024-08-17T17:14:47.594278+0000","last_undegraded":"2024-08-17T17:14:47.594278+0000","last_fullsized":"2024-08-17T17:14:47.594278+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:07.677423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1026,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.429356+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:14:35.429356+0000","last_peered":"2024-08-17T17:14:35.429356+0000","last_clean":"2024-08-17T17:14:35.429356+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:14:35.429356+0000","last_undegraded":"2024-08-17T17:14:35.429356+0000","last_fullsized":"2024-08-17T17:14:35.429356+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1037,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397289+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:14:30.397289+0000","last_peered":"2024-08-17T17:14:30.397289+0000","last_clean":"2024-08-17T17:14:30.397289+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:14:30.397289+0000","last_undegraded":"2024-08-17T17:14:30.397289+0000","last_fullsized":"2024-08-17T17:14:30.397289+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1400,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.437281+0000","last_change":"2024-08-17T17:07:45.683455+0000","last_active":"2024-08-17T17:14:35.437281+0000","last_peered":"2024-08-17T17:14:35.437281+0000","last_clean":"2024-08-17T17:14:35.437281+0000","last_became_active":"2024-08-17T17:07:45.683169+0000","last_became_peered":"2024-08-17T17:07:45.683169+0000","last_unstale":"2024-08-17T17:14:35.437281+0000","last_undegraded":"2024-08-17T17:14:35.437281+0000","last_fullsized":"2024-08-17T17:14:35.437281+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:18:19.562247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1359,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436543+0000","last_change":"2024-08-17T17:07:45.684222+0000","last_active":"2024-08-17T17:14:35.436543+0000","last_peered":"2024-08-17T17:14:35.436543+0000","last_clean":"2024-08-17T17:14:35.436543+0000","last_became_active":"2024-08-17T17:07:45.683913+0000","last_became_peered":"2024-08-17T17:07:45.683913+0000","last_unstale":"2024-08-17T17:14:35.436543+0000","last_undegraded":"2024-08-17T17:14:35.436543+0000","last_fullsized":"2024-08-17T17:14:35.436543+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:59:33.944853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1009,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.429054+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:14:35.429054+0000","last_peered":"2024-08-17T17:14:35.429054+0000","last_clean":"2024-08-17T17:14:35.429054+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:14:35.429054+0000","last_undegraded":"2024-08-17T17:14:35.429054+0000","last_fullsized":"2024-08-17T17:14:35.429054+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1281,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.595716+0000","last_change":"2024-08-17T17:07:45.687030+0000","last_active":"2024-08-17T17:14:47.595716+0000","last_peered":"2024-08-17T17:14:47.595716+0000","last_clean":"2024-08-17T17:14:47.595716+0000","last_became_active":"2024-08-17T17:07:45.686610+0000","last_became_peered":"2024-08-17T17:07:45.686610+0000","last_unstale":"2024-08-17T17:14:47.595716+0000","last_undegraded":"2024-08-17T17:14:47.595716+0000","last_fullsized":"2024-08-17T17:14:47.595716+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:02.425845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1048,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:31.956714+0000","last_change":"2024-08-17T17:06:07.632931+0000","last_active":"2024-08-17T17:14:31.956714+0000","last_peered":"2024-08-17T17:14:31.956714+0000","last_clean":"2024-08-17T17:14:31.956714+0000","last_became_active":"2024-08-17T17:06:07.632525+0000","last_became_peered":"2024-08-17T17:06:07.632525+0000","last_unstale":"2024-08-17T17:14:31.956714+0000","last_undegraded":"2024-08-17T17:14:31.956714+0000","last_fullsized":"2024-08-17T17:14:31.956714+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:01.740674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.a","version":"1101'7376","reported_seq":7400,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:46.953132+0000","last_change":"2024-08-17T17:13:13.231789+0000","last_active":"2024-08-17T17:14:46.953132+0000","last_peered":"2024-08-17T17:14:46.953132+0000","last_clean":"2024-08-17T17:14:46.953132+0000","last_became_active":"2024-08-17T17:13:13.231524+0000","last_became_peered":"2024-08-17T17:13:13.231524+0000","last_unstale":"2024-08-17T17:14:46.953132+0000","last_undegraded":"2024-08-17T17:14:46.953132+0000","last_fullsized":"2024-08-17T17:14:46.953132+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7376,"log_dups_size":0,"ondisk_log_size":7376,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:19.506111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":483393536,"num_objects":7376,"num_object_clones":0,"num_object_copies":22128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7376,"num_write_kb":472064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1025,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.399973+0000","last_change":"2024-08-17T17:06:07.638468+0000","last_active":"2024-08-17T17:14:30.399973+0000","last_peered":"2024-08-17T17:14:30.399973+0000","last_clean":"2024-08-17T17:14:30.399973+0000","last_became_active":"2024-08-17T17:06:07.636046+0000","last_became_peered":"2024-08-17T17:06:07.636046+0000","last_unstale":"2024-08-17T17:14:30.399973+0000","last_undegraded":"2024-08-17T17:14:30.399973+0000","last_fullsized":"2024-08-17T17:14:30.399973+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:37:12.228090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1023,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.396463+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:14:30.396463+0000","last_peered":"2024-08-17T17:14:30.396463+0000","last_clean":"2024-08-17T17:14:30.396463+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:14:30.396463+0000","last_undegraded":"2024-08-17T17:14:30.396463+0000","last_fullsized":"2024-08-17T17:14:30.396463+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"1102'7484","reported_seq":7526,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:48.882960+0000","last_change":"2024-08-17T17:13:13.227759+0000","last_active":"2024-08-17T17:14:48.882960+0000","last_peered":"2024-08-17T17:14:48.882960+0000","last_clean":"2024-08-17T17:14:48.882960+0000","last_became_active":"2024-08-17T17:13:13.227456+0000","last_became_peered":"2024-08-17T17:13:13.227456+0000","last_unstale":"2024-08-17T17:14:48.882960+0000","last_undegraded":"2024-08-17T17:14:48.882960+0000","last_fullsized":"2024-08-17T17:14:48.882960+0000","mapping_epoch":1086,"log_start":"1091'100","ondisk_log_start":"1091'100","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7384,"log_dups_size":0,"ondisk_log_size":7384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:17.924287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":490471424,"num_objects":7484,"num_object_clones":0,"num_object_copies":22452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7484,"num_write_kb":478976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1026,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.399772+0000","last_change":"2024-08-17T17:06:39.638433+0000","last_active":"2024-08-17T17:14:30.399772+0000","last_peered":"2024-08-17T17:14:30.399772+0000","last_clean":"2024-08-17T17:14:30.399772+0000","last_became_active":"2024-08-17T17:06:39.638081+0000","last_became_peered":"2024-08-17T17:06:39.638081+0000","last_unstale":"2024-08-17T17:14:30.399772+0000","last_undegraded":"2024-08-17T17:14:30.399772+0000","last_fullsized":"2024-08-17T17:14:30.399772+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:13.558012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1057,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.403724+0000","last_change":"2024-08-17T17:07:45.693574+0000","last_active":"2024-08-17T17:14:30.403724+0000","last_peered":"2024-08-17T17:14:30.403724+0000","last_clean":"2024-08-17T17:14:30.403724+0000","last_became_active":"2024-08-17T17:07:45.693086+0000","last_became_peered":"2024-08-17T17:07:45.693086+0000","last_unstale":"2024-08-17T17:14:30.403724+0000","last_undegraded":"2024-08-17T17:14:30.403724+0000","last_fullsized":"2024-08-17T17:14:30.403724+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:31.888894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"1101'7206","reported_seq":7230,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:46.901253+0000","last_change":"2024-08-17T17:13:13.229168+0000","last_active":"2024-08-17T17:14:46.901253+0000","last_peered":"2024-08-17T17:14:46.901253+0000","last_clean":"2024-08-17T17:14:46.901253+0000","last_became_active":"2024-08-17T17:13:13.228899+0000","last_became_peered":"2024-08-17T17:13:13.228899+0000","last_unstale":"2024-08-17T17:14:46.901253+0000","last_undegraded":"2024-08-17T17:14:46.901253+0000","last_fullsized":"2024-08-17T17:14:46.901253+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7206,"log_dups_size":0,"ondisk_log_size":7206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:10:26.885228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":472252416,"num_objects":7206,"num_object_clones":0,"num_object_copies":21618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7206,"num_write_kb":461184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1010,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.399784+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:14:30.399784+0000","last_peered":"2024-08-17T17:14:30.399784+0000","last_clean":"2024-08-17T17:14:30.399784+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:14:30.399784+0000","last_undegraded":"2024-08-17T17:14:30.399784+0000","last_fullsized":"2024-08-17T17:14:30.399784+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1039,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397755+0000","last_change":"2024-08-17T17:06:39.629368+0000","last_active":"2024-08-17T17:14:30.397755+0000","last_peered":"2024-08-17T17:14:30.397755+0000","last_clean":"2024-08-17T17:14:30.397755+0000","last_became_active":"2024-08-17T17:06:39.628832+0000","last_became_peered":"2024-08-17T17:06:39.628832+0000","last_unstale":"2024-08-17T17:14:30.397755+0000","last_undegraded":"2024-08-17T17:14:30.397755+0000","last_fullsized":"2024-08-17T17:14:30.397755+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:54.063884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"1101'7319","reported_seq":7358,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:45.797317+0000","last_change":"2024-08-17T17:13:13.229994+0000","last_active":"2024-08-17T17:14:45.797317+0000","last_peered":"2024-08-17T17:14:45.797317+0000","last_clean":"2024-08-17T17:14:45.797317+0000","last_became_active":"2024-08-17T17:13:13.229715+0000","last_became_peered":"2024-08-17T17:13:13.229715+0000","last_unstale":"2024-08-17T17:14:45.797317+0000","last_undegraded":"2024-08-17T17:14:45.797317+0000","last_fullsized":"2024-08-17T17:14:45.797317+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7319,"log_dups_size":0,"ondisk_log_size":7319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:18:20.964356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":479657984,"num_objects":7319,"num_object_clones":0,"num_object_copies":21957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7319,"num_write_kb":468416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1039,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397060+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:14:30.397060+0000","last_peered":"2024-08-17T17:14:30.397060+0000","last_clean":"2024-08-17T17:14:30.397060+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:14:30.397060+0000","last_undegraded":"2024-08-17T17:14:30.397060+0000","last_fullsized":"2024-08-17T17:14:30.397060+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1372,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436284+0000","last_change":"2024-08-17T17:07:12.651379+0000","last_active":"2024-08-17T17:14:35.436284+0000","last_peered":"2024-08-17T17:14:35.436284+0000","last_clean":"2024-08-17T17:14:35.436284+0000","last_became_active":"2024-08-17T17:07:12.651069+0000","last_became_peered":"2024-08-17T17:07:12.651069+0000","last_unstale":"2024-08-17T17:14:35.436284+0000","last_undegraded":"2024-08-17T17:14:35.436284+0000","last_fullsized":"2024-08-17T17:14:35.436284+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:25.384305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"1101'7541","reported_seq":7565,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:46.927289+0000","last_change":"2024-08-17T17:13:13.229385+0000","last_active":"2024-08-17T17:14:46.927289+0000","last_peered":"2024-08-17T17:14:46.927289+0000","last_clean":"2024-08-17T17:14:46.927289+0000","last_became_active":"2024-08-17T17:13:13.229101+0000","last_became_peered":"2024-08-17T17:13:13.229101+0000","last_unstale":"2024-08-17T17:14:46.927289+0000","last_undegraded":"2024-08-17T17:14:46.927289+0000","last_fullsized":"2024-08-17T17:14:46.927289+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7541,"log_dups_size":0,"ondisk_log_size":7541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:26.176057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":494206976,"num_objects":7541,"num_object_clones":0,"num_object_copies":22623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7541,"num_write_kb":482624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1295,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.600409+0000","last_change":"2024-08-17T17:07:45.691437+0000","last_active":"2024-08-17T17:14:47.600409+0000","last_peered":"2024-08-17T17:14:47.600409+0000","last_clean":"2024-08-17T17:14:47.600409+0000","last_became_active":"2024-08-17T17:07:45.690567+0000","last_became_peered":"2024-08-17T17:07:45.690567+0000","last_unstale":"2024-08-17T17:14:47.600409+0000","last_undegraded":"2024-08-17T17:14:47.600409+0000","last_fullsized":"2024-08-17T17:14:47.600409+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:43:07.698912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1042,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.404045+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:14:30.404045+0000","last_peered":"2024-08-17T17:14:30.404045+0000","last_clean":"2024-08-17T17:14:30.404045+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:14:30.404045+0000","last_undegraded":"2024-08-17T17:14:30.404045+0000","last_fullsized":"2024-08-17T17:14:30.404045+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"1101'7206","reported_seq":7247,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:45.780723+0000","last_change":"2024-08-17T17:13:13.227406+0000","last_active":"2024-08-17T17:14:45.780723+0000","last_peered":"2024-08-17T17:14:45.780723+0000","last_clean":"2024-08-17T17:14:45.780723+0000","last_became_active":"2024-08-17T17:13:13.227128+0000","last_became_peered":"2024-08-17T17:13:13.227128+0000","last_unstale":"2024-08-17T17:14:45.780723+0000","last_undegraded":"2024-08-17T17:14:45.780723+0000","last_fullsized":"2024-08-17T17:14:45.780723+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7206,"log_dups_size":0,"ondisk_log_size":7206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:43:08.671422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":472252416,"num_objects":7206,"num_object_clones":0,"num_object_copies":21618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7206,"num_write_kb":461184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1040,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397931+0000","last_change":"2024-08-17T17:06:39.629619+0000","last_active":"2024-08-17T17:14:30.397931+0000","last_peered":"2024-08-17T17:14:30.397931+0000","last_clean":"2024-08-17T17:14:30.397931+0000","last_became_active":"2024-08-17T17:06:39.629295+0000","last_became_peered":"2024-08-17T17:06:39.629295+0000","last_unstale":"2024-08-17T17:14:30.397931+0000","last_undegraded":"2024-08-17T17:14:30.397931+0000","last_fullsized":"2024-08-17T17:14:30.397931+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:15.874423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1057,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.404565+0000","last_change":"2024-08-17T17:07:12.649178+0000","last_active":"2024-08-17T17:14:30.404565+0000","last_peered":"2024-08-17T17:14:30.404565+0000","last_clean":"2024-08-17T17:14:30.404565+0000","last_became_active":"2024-08-17T17:07:12.648844+0000","last_became_peered":"2024-08-17T17:07:12.648844+0000","last_unstale":"2024-08-17T17:14:30.404565+0000","last_undegraded":"2024-08-17T17:14:30.404565+0000","last_fullsized":"2024-08-17T17:14:30.404565+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:12:06.307736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"1102'7280","reported_seq":7323,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:48.910173+0000","last_change":"2024-08-17T17:13:13.225854+0000","last_active":"2024-08-17T17:14:48.910173+0000","last_peered":"2024-08-17T17:14:48.910173+0000","last_clean":"2024-08-17T17:14:48.910173+0000","last_became_active":"2024-08-17T17:13:13.224588+0000","last_became_peered":"2024-08-17T17:13:13.224588+0000","last_unstale":"2024-08-17T17:14:48.910173+0000","last_undegraded":"2024-08-17T17:14:48.910173+0000","last_fullsized":"2024-08-17T17:14:48.910173+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7280,"log_dups_size":0,"ondisk_log_size":7280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:42:46.041842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":477102080,"num_objects":7280,"num_object_clones":0,"num_object_copies":21840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7280,"num_write_kb":465920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1043,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.403788+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:14:30.403788+0000","last_peered":"2024-08-17T17:14:30.403788+0000","last_clean":"2024-08-17T17:14:30.403788+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:14:30.403788+0000","last_undegraded":"2024-08-17T17:14:30.403788+0000","last_fullsized":"2024-08-17T17:14:30.403788+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1042,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.403827+0000","last_change":"2024-08-17T17:06:39.627678+0000","last_active":"2024-08-17T17:14:30.403827+0000","last_peered":"2024-08-17T17:14:30.403827+0000","last_clean":"2024-08-17T17:14:30.403827+0000","last_became_active":"2024-08-17T17:06:39.627226+0000","last_became_peered":"2024-08-17T17:06:39.627226+0000","last_unstale":"2024-08-17T17:14:30.403827+0000","last_undegraded":"2024-08-17T17:14:30.403827+0000","last_fullsized":"2024-08-17T17:14:30.403827+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:01.868244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"1101'7200","reported_seq":7224,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:46.961337+0000","last_change":"2024-08-17T17:13:13.230277+0000","last_active":"2024-08-17T17:14:46.961337+0000","last_peered":"2024-08-17T17:14:46.961337+0000","last_clean":"2024-08-17T17:14:46.961337+0000","last_became_active":"2024-08-17T17:13:13.229963+0000","last_became_peered":"2024-08-17T17:13:13.229963+0000","last_unstale":"2024-08-17T17:14:46.961337+0000","last_undegraded":"2024-08-17T17:14:46.961337+0000","last_fullsized":"2024-08-17T17:14:46.961337+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7200,"log_dups_size":0,"ondisk_log_size":7200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:52.522525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":471859200,"num_objects":7200,"num_object_clones":0,"num_object_copies":21600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7200,"num_write_kb":460800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1284,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.597198+0000","last_change":"2024-08-17T17:07:45.680458+0000","last_active":"2024-08-17T17:14:47.597198+0000","last_peered":"2024-08-17T17:14:47.597198+0000","last_clean":"2024-08-17T17:14:47.597198+0000","last_became_active":"2024-08-17T17:07:45.680221+0000","last_became_peered":"2024-08-17T17:07:45.680221+0000","last_unstale":"2024-08-17T17:14:47.597198+0000","last_undegraded":"2024-08-17T17:14:47.597198+0000","last_fullsized":"2024-08-17T17:14:47.597198+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:55.083962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1304,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.449617+0000","last_change":"2024-08-17T17:07:45.682866+0000","last_active":"2024-08-17T17:14:35.449617+0000","last_peered":"2024-08-17T17:14:35.449617+0000","last_clean":"2024-08-17T17:14:35.449617+0000","last_became_active":"2024-08-17T17:07:45.682554+0000","last_became_peered":"2024-08-17T17:07:45.682554+0000","last_unstale":"2024-08-17T17:14:35.449617+0000","last_undegraded":"2024-08-17T17:14:35.449617+0000","last_fullsized":"2024-08-17T17:14:35.449617+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:25.975914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.7","version":"1101'7369","reported_seq":7393,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:46.966652+0000","last_change":"2024-08-17T17:13:13.226791+0000","last_active":"2024-08-17T17:14:46.966652+0000","last_peered":"2024-08-17T17:14:46.966652+0000","last_clean":"2024-08-17T17:14:46.966652+0000","last_became_active":"2024-08-17T17:13:13.226491+0000","last_became_peered":"2024-08-17T17:13:13.226491+0000","last_unstale":"2024-08-17T17:14:46.966652+0000","last_undegraded":"2024-08-17T17:14:46.966652+0000","last_fullsized":"2024-08-17T17:14:46.966652+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7369,"log_dups_size":0,"ondisk_log_size":7369,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:10:33.413791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":482934784,"num_objects":7369,"num_object_clones":0,"num_object_copies":22107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7369,"num_write_kb":471616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1356,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.399623+0000","last_change":"2024-08-17T17:06:07.636484+0000","last_active":"2024-08-17T17:14:30.399623+0000","last_peered":"2024-08-17T17:14:30.399623+0000","last_clean":"2024-08-17T17:14:30.399623+0000","last_became_active":"2024-08-17T17:06:07.635998+0000","last_became_peered":"2024-08-17T17:06:07.635998+0000","last_unstale":"2024-08-17T17:14:30.399623+0000","last_undegraded":"2024-08-17T17:14:30.399623+0000","last_fullsized":"2024-08-17T17:14:30.399623+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:16.096587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1053,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.396611+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:14:30.396611+0000","last_peered":"2024-08-17T17:14:30.396611+0000","last_clean":"2024-08-17T17:14:30.396611+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:14:30.396611+0000","last_undegraded":"2024-08-17T17:14:30.396611+0000","last_fullsized":"2024-08-17T17:14:30.396611+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1263,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.595154+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:14:47.595154+0000","last_peered":"2024-08-17T17:14:47.595154+0000","last_clean":"2024-08-17T17:14:47.595154+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:14:47.595154+0000","last_undegraded":"2024-08-17T17:14:47.595154+0000","last_fullsized":"2024-08-17T17:14:47.595154+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1025,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.424042+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:14:35.424042+0000","last_peered":"2024-08-17T17:14:35.424042+0000","last_clean":"2024-08-17T17:14:35.424042+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:14:35.424042+0000","last_undegraded":"2024-08-17T17:14:35.424042+0000","last_fullsized":"2024-08-17T17:14:35.424042+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"1102'7329","reported_seq":7372,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.808271+0000","last_change":"2024-08-17T17:13:13.225585+0000","last_active":"2024-08-17T17:14:47.808271+0000","last_peered":"2024-08-17T17:14:47.808271+0000","last_clean":"2024-08-17T17:14:47.808271+0000","last_became_active":"2024-08-17T17:13:13.225331+0000","last_became_peered":"2024-08-17T17:13:13.225331+0000","last_unstale":"2024-08-17T17:14:47.808271+0000","last_undegraded":"2024-08-17T17:14:47.808271+0000","last_fullsized":"2024-08-17T17:14:47.808271+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7329,"log_dups_size":0,"ondisk_log_size":7329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:23:44.342905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":480313344,"num_objects":7329,"num_object_clones":0,"num_object_copies":21987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7329,"num_write_kb":469056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1053,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:31.976406+0000","last_change":"2024-08-17T17:06:39.638376+0000","last_active":"2024-08-17T17:14:31.976406+0000","last_peered":"2024-08-17T17:14:31.976406+0000","last_clean":"2024-08-17T17:14:31.976406+0000","last_became_active":"2024-08-17T17:06:39.638050+0000","last_became_peered":"2024-08-17T17:06:39.638050+0000","last_unstale":"2024-08-17T17:14:31.976406+0000","last_undegraded":"2024-08-17T17:14:31.976406+0000","last_fullsized":"2024-08-17T17:14:31.976406+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:05.782492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1401,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.440145+0000","last_change":"2024-08-17T17:07:45.683556+0000","last_active":"2024-08-17T17:14:35.440145+0000","last_peered":"2024-08-17T17:14:35.440145+0000","last_clean":"2024-08-17T17:14:35.440145+0000","last_became_active":"2024-08-17T17:07:45.683232+0000","last_became_peered":"2024-08-17T17:07:45.683232+0000","last_unstale":"2024-08-17T17:14:35.440145+0000","last_undegraded":"2024-08-17T17:14:35.440145+0000","last_fullsized":"2024-08-17T17:14:35.440145+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:59:16.112417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1373,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436405+0000","last_change":"2024-08-17T17:07:12.651145+0000","last_active":"2024-08-17T17:14:35.436405+0000","last_peered":"2024-08-17T17:14:35.436405+0000","last_clean":"2024-08-17T17:14:35.436405+0000","last_became_active":"2024-08-17T17:07:12.650839+0000","last_became_peered":"2024-08-17T17:07:12.650839+0000","last_unstale":"2024-08-17T17:14:35.436405+0000","last_undegraded":"2024-08-17T17:14:35.436405+0000","last_fullsized":"2024-08-17T17:14:35.436405+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:59.795181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.5","version":"1101'7378","reported_seq":7402,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:46.965648+0000","last_change":"2024-08-17T17:13:13.228848+0000","last_active":"2024-08-17T17:14:46.965648+0000","last_peered":"2024-08-17T17:14:46.965648+0000","last_clean":"2024-08-17T17:14:46.965648+0000","last_became_active":"2024-08-17T17:13:13.228573+0000","last_became_peered":"2024-08-17T17:13:13.228573+0000","last_unstale":"2024-08-17T17:14:46.965648+0000","last_undegraded":"2024-08-17T17:14:46.965648+0000","last_fullsized":"2024-08-17T17:14:46.965648+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7378,"log_dups_size":0,"ondisk_log_size":7378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:01.011828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":483524608,"num_objects":7378,"num_object_clones":0,"num_object_copies":22134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7378,"num_write_kb":472192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1344,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.454278+0000","last_change":"2024-08-17T17:07:45.681107+0000","last_active":"2024-08-17T17:14:35.454278+0000","last_peered":"2024-08-17T17:14:35.454278+0000","last_clean":"2024-08-17T17:14:35.454278+0000","last_became_active":"2024-08-17T17:07:45.680770+0000","last_became_peered":"2024-08-17T17:07:45.680770+0000","last_unstale":"2024-08-17T17:14:35.454278+0000","last_undegraded":"2024-08-17T17:14:35.454278+0000","last_fullsized":"2024-08-17T17:14:35.454278+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:53.820920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1337,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.597419+0000","last_change":"2024-08-17T17:07:12.649485+0000","last_active":"2024-08-17T17:14:47.597419+0000","last_peered":"2024-08-17T17:14:47.597419+0000","last_clean":"2024-08-17T17:14:47.597419+0000","last_became_active":"2024-08-17T17:07:12.649161+0000","last_became_peered":"2024-08-17T17:07:12.649161+0000","last_unstale":"2024-08-17T17:14:47.597419+0000","last_undegraded":"2024-08-17T17:14:47.597419+0000","last_fullsized":"2024-08-17T17:14:47.597419+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:18.370628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1351,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.598221+0000","last_change":"2024-08-17T17:06:39.627820+0000","last_active":"2024-08-17T17:14:47.598221+0000","last_peered":"2024-08-17T17:14:47.598221+0000","last_clean":"2024-08-17T17:14:47.598221+0000","last_became_active":"2024-08-17T17:06:39.627464+0000","last_became_peered":"2024-08-17T17:06:39.627464+0000","last_unstale":"2024-08-17T17:14:47.598221+0000","last_undegraded":"2024-08-17T17:14:47.598221+0000","last_fullsized":"2024-08-17T17:14:47.598221+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:05:01.500528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"1102'7534","reported_seq":7577,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:48.903433+0000","last_change":"2024-08-17T17:13:13.224403+0000","last_active":"2024-08-17T17:14:48.903433+0000","last_peered":"2024-08-17T17:14:48.903433+0000","last_clean":"2024-08-17T17:14:48.903433+0000","last_became_active":"2024-08-17T17:13:13.224119+0000","last_became_peered":"2024-08-17T17:13:13.224119+0000","last_unstale":"2024-08-17T17:14:48.903433+0000","last_undegraded":"2024-08-17T17:14:48.903433+0000","last_fullsized":"2024-08-17T17:14:48.903433+0000","mapping_epoch":1086,"log_start":"1091'200","ondisk_log_start":"1091'200","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7334,"log_dups_size":0,"ondisk_log_size":7334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:20:58.208742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":493748224,"num_objects":7534,"num_object_clones":0,"num_object_copies":22602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7534,"num_write_kb":482176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1054,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.396793+0000","last_change":"2024-08-17T17:06:39.628745+0000","last_active":"2024-08-17T17:14:30.396793+0000","last_peered":"2024-08-17T17:14:30.396793+0000","last_clean":"2024-08-17T17:14:30.396793+0000","last_became_active":"2024-08-17T17:06:39.628144+0000","last_became_peered":"2024-08-17T17:06:39.628144+0000","last_unstale":"2024-08-17T17:14:30.396793+0000","last_undegraded":"2024-08-17T17:14:30.396793+0000","last_fullsized":"2024-08-17T17:14:30.396793+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:14.740936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1044,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.404425+0000","last_change":"2024-08-17T17:06:07.637581+0000","last_active":"2024-08-17T17:14:30.404425+0000","last_peered":"2024-08-17T17:14:30.404425+0000","last_clean":"2024-08-17T17:14:30.404425+0000","last_became_active":"2024-08-17T17:06:07.637282+0000","last_became_peered":"2024-08-17T17:06:07.637282+0000","last_unstale":"2024-08-17T17:14:30.404425+0000","last_undegraded":"2024-08-17T17:14:30.404425+0000","last_fullsized":"2024-08-17T17:14:30.404425+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:11:56.938734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1347,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.597818+0000","last_change":"2024-08-17T17:07:45.687502+0000","last_active":"2024-08-17T17:14:47.597818+0000","last_peered":"2024-08-17T17:14:47.597818+0000","last_clean":"2024-08-17T17:14:47.597818+0000","last_became_active":"2024-08-17T17:07:45.687197+0000","last_became_peered":"2024-08-17T17:07:45.687197+0000","last_unstale":"2024-08-17T17:14:47.597818+0000","last_undegraded":"2024-08-17T17:14:47.597818+0000","last_fullsized":"2024-08-17T17:14:47.597818+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:23.577838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"1101'7223","reported_seq":7262,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:45.725140+0000","last_change":"2024-08-17T17:13:13.231784+0000","last_active":"2024-08-17T17:14:45.725140+0000","last_peered":"2024-08-17T17:14:45.725140+0000","last_clean":"2024-08-17T17:14:45.725140+0000","last_became_active":"2024-08-17T17:13:13.231500+0000","last_became_peered":"2024-08-17T17:13:13.231500+0000","last_unstale":"2024-08-17T17:14:45.725140+0000","last_undegraded":"2024-08-17T17:14:45.725140+0000","last_fullsized":"2024-08-17T17:14:45.725140+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7223,"log_dups_size":0,"ondisk_log_size":7223,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:47.466790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":473366528,"num_objects":7223,"num_object_clones":0,"num_object_copies":21669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7223,"num_write_kb":462272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1055,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:31.965398+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:14:31.965398+0000","last_peered":"2024-08-17T17:14:31.965398+0000","last_clean":"2024-08-17T17:14:31.965398+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:14:31.965398+0000","last_undegraded":"2024-08-17T17:14:31.965398+0000","last_fullsized":"2024-08-17T17:14:31.965398+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1365,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.598022+0000","last_change":"2024-08-17T17:06:39.633975+0000","last_active":"2024-08-17T17:14:47.598022+0000","last_peered":"2024-08-17T17:14:47.598022+0000","last_clean":"2024-08-17T17:14:47.598022+0000","last_became_active":"2024-08-17T17:06:39.631526+0000","last_became_peered":"2024-08-17T17:06:39.631526+0000","last_unstale":"2024-08-17T17:14:47.598022+0000","last_undegraded":"2024-08-17T17:14:47.598022+0000","last_fullsized":"2024-08-17T17:14:47.598022+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:16:58.542237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1325,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.597756+0000","last_change":"2024-08-17T17:06:39.627577+0000","last_active":"2024-08-17T17:14:47.597756+0000","last_peered":"2024-08-17T17:14:47.597756+0000","last_clean":"2024-08-17T17:14:47.597756+0000","last_became_active":"2024-08-17T17:06:39.626554+0000","last_became_peered":"2024-08-17T17:06:39.626554+0000","last_unstale":"2024-08-17T17:14:47.597756+0000","last_undegraded":"2024-08-17T17:14:47.597756+0000","last_fullsized":"2024-08-17T17:14:47.597756+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:04:48.533057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.1","version":"1102'7404","reported_seq":7446,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.815400+0000","last_change":"2024-08-17T17:13:13.229376+0000","last_active":"2024-08-17T17:14:47.815400+0000","last_peered":"2024-08-17T17:14:47.815400+0000","last_clean":"2024-08-17T17:14:47.815400+0000","last_became_active":"2024-08-17T17:13:13.229090+0000","last_became_peered":"2024-08-17T17:13:13.229090+0000","last_unstale":"2024-08-17T17:14:47.815400+0000","last_undegraded":"2024-08-17T17:14:47.815400+0000","last_fullsized":"2024-08-17T17:14:47.815400+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7404,"log_dups_size":0,"ondisk_log_size":7404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:01:53.813720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":485228544,"num_objects":7404,"num_object_clones":0,"num_object_copies":22212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7404,"num_write_kb":473856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1335,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.597906+0000","last_change":"2024-08-17T17:07:12.645660+0000","last_active":"2024-08-17T17:14:47.597906+0000","last_peered":"2024-08-17T17:14:47.597906+0000","last_clean":"2024-08-17T17:14:47.597906+0000","last_became_active":"2024-08-17T17:07:12.645368+0000","last_became_peered":"2024-08-17T17:07:12.645368+0000","last_unstale":"2024-08-17T17:14:47.597906+0000","last_undegraded":"2024-08-17T17:14:47.597906+0000","last_fullsized":"2024-08-17T17:14:47.597906+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:41:00.157045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1037,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.399849+0000","last_change":"2024-08-17T17:06:07.636239+0000","last_active":"2024-08-17T17:14:30.399849+0000","last_peered":"2024-08-17T17:14:30.399849+0000","last_clean":"2024-08-17T17:14:30.399849+0000","last_became_active":"2024-08-17T17:06:07.635881+0000","last_became_peered":"2024-08-17T17:06:07.635881+0000","last_unstale":"2024-08-17T17:14:30.399849+0000","last_undegraded":"2024-08-17T17:14:30.399849+0000","last_fullsized":"2024-08-17T17:14:30.399849+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:47:36.001648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1350,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436384+0000","last_change":"2024-08-17T17:06:07.634369+0000","last_active":"2024-08-17T17:14:35.436384+0000","last_peered":"2024-08-17T17:14:35.436384+0000","last_clean":"2024-08-17T17:14:35.436384+0000","last_became_active":"2024-08-17T17:06:07.633769+0000","last_became_peered":"2024-08-17T17:06:07.633769+0000","last_unstale":"2024-08-17T17:14:35.436384+0000","last_undegraded":"2024-08-17T17:14:35.436384+0000","last_fullsized":"2024-08-17T17:14:35.436384+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:20.957493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"1101'7316","reported_seq":7340,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:46.981059+0000","last_change":"2024-08-17T17:13:13.227597+0000","last_active":"2024-08-17T17:14:46.981059+0000","last_peered":"2024-08-17T17:14:46.981059+0000","last_clean":"2024-08-17T17:14:46.981059+0000","last_became_active":"2024-08-17T17:13:13.227117+0000","last_became_peered":"2024-08-17T17:13:13.227117+0000","last_unstale":"2024-08-17T17:14:46.981059+0000","last_undegraded":"2024-08-17T17:14:46.981059+0000","last_fullsized":"2024-08-17T17:14:46.981059+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7316,"log_dups_size":0,"ondisk_log_size":7316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:25.337770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":479461376,"num_objects":7316,"num_object_clones":0,"num_object_copies":21948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7316,"num_write_kb":468224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1295,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.595627+0000","last_change":"2024-08-17T17:07:45.688526+0000","last_active":"2024-08-17T17:14:47.595627+0000","last_peered":"2024-08-17T17:14:47.595627+0000","last_clean":"2024-08-17T17:14:47.595627+0000","last_became_active":"2024-08-17T17:07:45.688091+0000","last_became_peered":"2024-08-17T17:07:45.688091+0000","last_unstale":"2024-08-17T17:14:47.595627+0000","last_undegraded":"2024-08-17T17:14:47.595627+0000","last_fullsized":"2024-08-17T17:14:47.595627+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:02.385683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1335,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.599452+0000","last_change":"2024-08-17T17:06:39.629005+0000","last_active":"2024-08-17T17:14:47.599452+0000","last_peered":"2024-08-17T17:14:47.599452+0000","last_clean":"2024-08-17T17:14:47.599452+0000","last_became_active":"2024-08-17T17:06:39.628679+0000","last_became_peered":"2024-08-17T17:06:39.628679+0000","last_unstale":"2024-08-17T17:14:47.599452+0000","last_undegraded":"2024-08-17T17:14:47.599452+0000","last_fullsized":"2024-08-17T17:14:47.599452+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:09.401059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1292,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.593245+0000","last_change":"2024-08-17T17:07:45.695858+0000","last_active":"2024-08-17T17:14:47.593245+0000","last_peered":"2024-08-17T17:14:47.593245+0000","last_clean":"2024-08-17T17:14:47.593245+0000","last_became_active":"2024-08-17T17:07:45.694111+0000","last_became_peered":"2024-08-17T17:07:45.694111+0000","last_unstale":"2024-08-17T17:14:47.593245+0000","last_undegraded":"2024-08-17T17:14:47.593245+0000","last_fullsized":"2024-08-17T17:14:47.593245+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:40:38.772090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"1101'7293","reported_seq":7317,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:46.977291+0000","last_change":"2024-08-17T17:13:13.229123+0000","last_active":"2024-08-17T17:14:46.977291+0000","last_peered":"2024-08-17T17:14:46.977291+0000","last_clean":"2024-08-17T17:14:46.977291+0000","last_became_active":"2024-08-17T17:13:13.228818+0000","last_became_peered":"2024-08-17T17:13:13.228818+0000","last_unstale":"2024-08-17T17:14:46.977291+0000","last_undegraded":"2024-08-17T17:14:46.977291+0000","last_fullsized":"2024-08-17T17:14:46.977291+0000","mapping_epoch":1086,"log_start":"0'0","ondisk_log_start":"0'0","created":1086,"last_epoch_clean":1087,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:13:12.207763+0000","last_clean_scrub_stamp":"2024-08-17T17:13:12.207763+0000","objects_scrubbed":0,"log_size":7293,"log_dups_size":0,"ondisk_log_size":7293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:05:54.121915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":477954048,"num_objects":7293,"num_object_clones":0,"num_object_copies":21879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7293,"num_write_kb":466752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":1364,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436680+0000","last_change":"2024-08-17T17:06:39.636866+0000","last_active":"2024-08-17T17:14:35.436680+0000","last_peered":"2024-08-17T17:14:35.436680+0000","last_clean":"2024-08-17T17:14:35.436680+0000","last_became_active":"2024-08-17T17:06:39.636470+0000","last_became_peered":"2024-08-17T17:06:39.636470+0000","last_unstale":"2024-08-17T17:14:35.436680+0000","last_undegraded":"2024-08-17T17:14:35.436680+0000","last_fullsized":"2024-08-17T17:14:35.436680+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:14.028712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1052,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397528+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:14:30.397528+0000","last_peered":"2024-08-17T17:14:30.397528+0000","last_clean":"2024-08-17T17:14:30.397528+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:14:30.397528+0000","last_undegraded":"2024-08-17T17:14:30.397528+0000","last_fullsized":"2024-08-17T17:14:30.397528+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1028,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.403564+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:14:30.403564+0000","last_peered":"2024-08-17T17:14:30.403564+0000","last_clean":"2024-08-17T17:14:30.403564+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:14:30.403564+0000","last_undegraded":"2024-08-17T17:14:30.403564+0000","last_fullsized":"2024-08-17T17:14:30.403564+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1042,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.435827+0000","last_change":"2024-08-17T17:06:39.627594+0000","last_active":"2024-08-17T17:14:35.435827+0000","last_peered":"2024-08-17T17:14:35.435827+0000","last_clean":"2024-08-17T17:14:35.435827+0000","last_became_active":"2024-08-17T17:06:39.627105+0000","last_became_peered":"2024-08-17T17:06:39.627105+0000","last_unstale":"2024-08-17T17:14:35.435827+0000","last_undegraded":"2024-08-17T17:14:35.435827+0000","last_fullsized":"2024-08-17T17:14:35.435827+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:30.105070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1051,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436059+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:14:35.436059+0000","last_peered":"2024-08-17T17:14:35.436059+0000","last_clean":"2024-08-17T17:14:35.436059+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:14:35.436059+0000","last_undegraded":"2024-08-17T17:14:35.436059+0000","last_fullsized":"2024-08-17T17:14:35.436059+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1336,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.597602+0000","last_change":"2024-08-17T17:07:12.645440+0000","last_active":"2024-08-17T17:14:47.597602+0000","last_peered":"2024-08-17T17:14:47.597602+0000","last_clean":"2024-08-17T17:14:47.597602+0000","last_became_active":"2024-08-17T17:07:12.645147+0000","last_became_peered":"2024-08-17T17:07:12.645147+0000","last_unstale":"2024-08-17T17:14:47.597602+0000","last_undegraded":"2024-08-17T17:14:47.597602+0000","last_fullsized":"2024-08-17T17:14:47.597602+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:51:25.588528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1026,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.399662+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:14:30.399662+0000","last_peered":"2024-08-17T17:14:30.399662+0000","last_clean":"2024-08-17T17:14:30.399662+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:14:30.399662+0000","last_undegraded":"2024-08-17T17:14:30.399662+0000","last_fullsized":"2024-08-17T17:14:30.399662+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1038,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397577+0000","last_change":"2024-08-17T17:06:39.625029+0000","last_active":"2024-08-17T17:14:30.397577+0000","last_peered":"2024-08-17T17:14:30.397577+0000","last_clean":"2024-08-17T17:14:30.397577+0000","last_became_active":"2024-08-17T17:06:39.624688+0000","last_became_peered":"2024-08-17T17:06:39.624688+0000","last_unstale":"2024-08-17T17:14:30.397577+0000","last_undegraded":"2024-08-17T17:14:30.397577+0000","last_fullsized":"2024-08-17T17:14:30.397577+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:54.849668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1263,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.590490+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:14:47.590490+0000","last_peered":"2024-08-17T17:14:47.590490+0000","last_clean":"2024-08-17T17:14:47.590490+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:14:47.590490+0000","last_undegraded":"2024-08-17T17:14:47.590490+0000","last_fullsized":"2024-08-17T17:14:47.590490+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1400,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436951+0000","last_change":"2024-08-17T17:07:45.690099+0000","last_active":"2024-08-17T17:14:35.436951+0000","last_peered":"2024-08-17T17:14:35.436951+0000","last_clean":"2024-08-17T17:14:35.436951+0000","last_became_active":"2024-08-17T17:07:45.689783+0000","last_became_peered":"2024-08-17T17:07:45.689783+0000","last_unstale":"2024-08-17T17:14:35.436951+0000","last_undegraded":"2024-08-17T17:14:35.436951+0000","last_fullsized":"2024-08-17T17:14:35.436951+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:30:38.311630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1282,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.593163+0000","last_change":"2024-08-17T17:07:45.678950+0000","last_active":"2024-08-17T17:14:47.593163+0000","last_peered":"2024-08-17T17:14:47.593163+0000","last_clean":"2024-08-17T17:14:47.593163+0000","last_became_active":"2024-08-17T17:07:45.678472+0000","last_became_peered":"2024-08-17T17:07:45.678472+0000","last_unstale":"2024-08-17T17:14:47.593163+0000","last_undegraded":"2024-08-17T17:14:47.593163+0000","last_fullsized":"2024-08-17T17:14:47.593163+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:48:02.385927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1040,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397398+0000","last_change":"2024-08-17T17:06:39.628542+0000","last_active":"2024-08-17T17:14:30.397398+0000","last_peered":"2024-08-17T17:14:30.397398+0000","last_clean":"2024-08-17T17:14:30.397398+0000","last_became_active":"2024-08-17T17:06:39.628253+0000","last_became_peered":"2024-08-17T17:06:39.628253+0000","last_unstale":"2024-08-17T17:14:30.397398+0000","last_undegraded":"2024-08-17T17:14:30.397398+0000","last_fullsized":"2024-08-17T17:14:30.397398+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:22.349326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1297,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.596997+0000","last_change":"2024-08-17T17:07:45.696068+0000","last_active":"2024-08-17T17:14:47.596997+0000","last_peered":"2024-08-17T17:14:47.596997+0000","last_clean":"2024-08-17T17:14:47.596997+0000","last_became_active":"2024-08-17T17:07:45.694617+0000","last_became_peered":"2024-08-17T17:07:45.694617+0000","last_unstale":"2024-08-17T17:14:47.596997+0000","last_undegraded":"2024-08-17T17:14:47.596997+0000","last_fullsized":"2024-08-17T17:14:47.596997+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:28:08.646380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1361,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436456+0000","last_change":"2024-08-17T17:07:45.684186+0000","last_active":"2024-08-17T17:14:35.436456+0000","last_peered":"2024-08-17T17:14:35.436456+0000","last_clean":"2024-08-17T17:14:35.436456+0000","last_became_active":"2024-08-17T17:07:45.683863+0000","last_became_peered":"2024-08-17T17:07:45.683863+0000","last_unstale":"2024-08-17T17:14:35.436456+0000","last_undegraded":"2024-08-17T17:14:35.436456+0000","last_fullsized":"2024-08-17T17:14:35.436456+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:57:52.106168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1335,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.598807+0000","last_change":"2024-08-17T17:07:45.688645+0000","last_active":"2024-08-17T17:14:47.598807+0000","last_peered":"2024-08-17T17:14:47.598807+0000","last_clean":"2024-08-17T17:14:47.598807+0000","last_became_active":"2024-08-17T17:07:45.688383+0000","last_became_peered":"2024-08-17T17:07:45.688383+0000","last_unstale":"2024-08-17T17:14:47.598807+0000","last_undegraded":"2024-08-17T17:14:47.598807+0000","last_fullsized":"2024-08-17T17:14:47.598807+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:33:12.209565+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1281,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.594434+0000","last_change":"2024-08-17T17:07:45.689782+0000","last_active":"2024-08-17T17:14:47.594434+0000","last_peered":"2024-08-17T17:14:47.594434+0000","last_clean":"2024-08-17T17:14:47.594434+0000","last_became_active":"2024-08-17T17:07:45.689312+0000","last_became_peered":"2024-08-17T17:07:45.689312+0000","last_unstale":"2024-08-17T17:14:47.594434+0000","last_undegraded":"2024-08-17T17:14:47.594434+0000","last_fullsized":"2024-08-17T17:14:47.594434+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:07.560555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1058,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.404712+0000","last_change":"2024-08-17T17:07:12.649175+0000","last_active":"2024-08-17T17:14:30.404712+0000","last_peered":"2024-08-17T17:14:30.404712+0000","last_clean":"2024-08-17T17:14:30.404712+0000","last_became_active":"2024-08-17T17:07:12.648822+0000","last_became_peered":"2024-08-17T17:07:12.648822+0000","last_unstale":"2024-08-17T17:14:30.404712+0000","last_undegraded":"2024-08-17T17:14:30.404712+0000","last_fullsized":"2024-08-17T17:14:30.404712+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:03.924431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1321,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.442181+0000","last_change":"2024-08-17T17:07:45.685278+0000","last_active":"2024-08-17T17:14:35.442181+0000","last_peered":"2024-08-17T17:14:35.442181+0000","last_clean":"2024-08-17T17:14:35.442181+0000","last_became_active":"2024-08-17T17:07:45.684963+0000","last_became_peered":"2024-08-17T17:07:45.684963+0000","last_unstale":"2024-08-17T17:14:35.442181+0000","last_undegraded":"2024-08-17T17:14:35.442181+0000","last_fullsized":"2024-08-17T17:14:35.442181+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:55:42.495791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":1351,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.436691+0000","last_change":"2024-08-17T17:06:07.634103+0000","last_active":"2024-08-17T17:14:35.436691+0000","last_peered":"2024-08-17T17:14:35.436691+0000","last_clean":"2024-08-17T17:14:35.436691+0000","last_became_active":"2024-08-17T17:06:07.633624+0000","last_became_peered":"2024-08-17T17:06:07.633624+0000","last_unstale":"2024-08-17T17:14:35.436691+0000","last_undegraded":"2024-08-17T17:14:35.436691+0000","last_fullsized":"2024-08-17T17:14:35.436691+0000","mapping_epoch":739,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":740,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:58.573558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1056,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.404286+0000","last_change":"2024-08-17T17:07:45.684606+0000","last_active":"2024-08-17T17:14:30.404286+0000","last_peered":"2024-08-17T17:14:30.404286+0000","last_clean":"2024-08-17T17:14:30.404286+0000","last_became_active":"2024-08-17T17:07:45.684124+0000","last_became_peered":"2024-08-17T17:07:45.684124+0000","last_unstale":"2024-08-17T17:14:30.404286+0000","last_undegraded":"2024-08-17T17:14:30.404286+0000","last_fullsized":"2024-08-17T17:14:30.404286+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:16.110507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1040,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.396574+0000","last_change":"2024-08-17T17:06:39.625245+0000","last_active":"2024-08-17T17:14:30.396574+0000","last_peered":"2024-08-17T17:14:30.396574+0000","last_clean":"2024-08-17T17:14:30.396574+0000","last_became_active":"2024-08-17T17:06:39.624657+0000","last_became_peered":"2024-08-17T17:06:39.624657+0000","last_unstale":"2024-08-17T17:14:30.396574+0000","last_undegraded":"2024-08-17T17:14:30.396574+0000","last_fullsized":"2024-08-17T17:14:30.396574+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:43.097460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1056,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.404528+0000","last_change":"2024-08-17T17:07:45.684769+0000","last_active":"2024-08-17T17:14:30.404528+0000","last_peered":"2024-08-17T17:14:30.404528+0000","last_clean":"2024-08-17T17:14:30.404528+0000","last_became_active":"2024-08-17T17:07:45.684459+0000","last_became_peered":"2024-08-17T17:07:45.684459+0000","last_unstale":"2024-08-17T17:14:30.404528+0000","last_undegraded":"2024-08-17T17:14:30.404528+0000","last_fullsized":"2024-08-17T17:14:30.404528+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:21:30.731978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1025,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.397370+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:14:30.397370+0000","last_peered":"2024-08-17T17:14:30.397370+0000","last_clean":"2024-08-17T17:14:30.397370+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:14:30.397370+0000","last_undegraded":"2024-08-17T17:14:30.397370+0000","last_fullsized":"2024-08-17T17:14:30.397370+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1287,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.590778+0000","last_change":"2024-08-17T17:07:45.692127+0000","last_active":"2024-08-17T17:14:47.590778+0000","last_peered":"2024-08-17T17:14:47.590778+0000","last_clean":"2024-08-17T17:14:47.590778+0000","last_became_active":"2024-08-17T17:07:45.690277+0000","last_became_peered":"2024-08-17T17:07:45.690277+0000","last_unstale":"2024-08-17T17:14:47.590778+0000","last_undegraded":"2024-08-17T17:14:47.590778+0000","last_fullsized":"2024-08-17T17:14:47.590778+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:52:04.555409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1276,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-08-17T17:14:47.601794+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:14:47.601794+0000","last_peered":"2024-08-17T17:14:47.601794+0000","last_clean":"2024-08-17T17:14:47.601794+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:14:47.601794+0000","last_undegraded":"2024-08-17T17:14:47.601794+0000","last_fullsized":"2024-08-17T17:14:47.601794+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1041,"reported_epoch":1100,"state":"active+clean","last_fresh":"2024-08-17T17:14:30.403344+0000","last_change":"2024-08-17T17:06:39.627689+0000","last_active":"2024-08-17T17:14:30.403344+0000","last_peered":"2024-08-17T17:14:30.403344+0000","last_clean":"2024-08-17T17:14:30.403344+0000","last_became_active":"2024-08-17T17:06:39.627375+0000","last_became_peered":"2024-08-17T17:06:39.627375+0000","last_unstale":"2024-08-17T17:14:30.403344+0000","last_undegraded":"2024-08-17T17:14:30.403344+0000","last_fullsized":"2024-08-17T17:14:30.403344+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:51.954076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":1367,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-08-17T17:14:35.437049+0000","last_change":"2024-08-17T17:07:12.651933+0000","last_active":"2024-08-17T17:14:35.437049+0000","last_peered":"2024-08-17T17:14:35.437049+0000","last_clean":"2024-08-17T17:14:35.437049+0000","last_became_active":"2024-08-17T17:07:12.651592+0000","last_became_peered":"2024-08-17T17:07:12.651592+0000","last_unstale":"2024-08-17T17:14:35.437049+0000","last_undegraded":"2024-08-17T17:14:35.437049+0000","last_fullsized":"2024-08-17T17:14:35.437049+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:31.224038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":7697727488,"num_objects":117458,"num_object_clones":0,"num_object_copies":352374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":117458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":117458,"num_write_kb":7517312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":23084138496,"data_stored":23084138496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117158,"ondisk_log_size":117158,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201560,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2937540,"kb_used_data":2816252,"kb_used_omap":1,"kb_used_meta":121278,"kb_avail":90807612,"statfs":{"total":95995035648,"available":92986994688,"internally_reserved":0,"allocated":2883842048,"data_stored":2881935313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":124189131},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463197,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2448508,"kb_used_data":2345268,"kb_used_omap":1,"kb_used_meta":103230,"kb_avail":91296644,"statfs":{"total":95995035648,"available":93487763456,"internally_reserved":0,"allocated":2401554432,"data_stored":2399655762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":105707977},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626839,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4838660,"kb_used_data":4738492,"kb_used_omap":42,"kb_used_meta":100117,"kb_avail":88906492,"statfs":{"total":95995035648,"available":91040247808,"internally_reserved":0,"allocated":4852215808,"data_stored":4850112263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":102519868},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785316,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2437484,"kb_used_data":2360804,"kb_used_omap":48,"kb_used_meta":76623,"kb_avail":91307668,"statfs":{"total":95995035648,"available":93499052032,"internally_reserved":0,"allocated":2417463296,"data_stored":2415308472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":78462638},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724833,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2441008,"kb_used_data":2319720,"kb_used_omap":1,"kb_used_meta":121278,"kb_avail":91304144,"statfs":{"total":95995035648,"available":93495443456,"internally_reserved":0,"allocated":2375393280,"data_stored":2373502180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":124189126},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986470,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2481148,"kb_used_data":2363956,"kb_used_omap":1,"kb_used_meta":117182,"kb_avail":91264004,"statfs":{"total":95995035648,"available":93454340096,"internally_reserved":0,"allocated":2420690944,"data_stored":2418791787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":119994822},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706077,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3837296,"kb_used_data":3724776,"kb_used_omap":35,"kb_used_meta":112476,"kb_avail":89907856,"statfs":{"total":95995035648,"available":92065644544,"internally_reserved":0,"allocated":3814170624,"data_stored":3812012301,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":115175479},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831850,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1925352,"kb_used_data":1852640,"kb_used_omap":65,"kb_used_meta":72638,"kb_avail":91819800,"statfs":{"total":95995035648,"available":94023475200,"internally_reserved":0,"allocated":1897103360,"data_stored":1894886765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":74381499},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1900216320,"data_stored":1900216320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2423914496,"data_stored":2423914496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3813867520,"data_stored":3813867520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4862312448,"data_stored":4862312448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2417950720,"data_stored":2417950720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2384658432,"data_stored":2384658432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2396061696,"data_stored":2396061696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2885156864,"data_stored":2885156864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:14:52.831 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:14:52.831 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:14:52.831 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:14:52.831 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.000000 2024-08-17T17:14:52.831 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 0 0 2024-08-17T17:14:52.914 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:52.973 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:52.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:52.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:53.002 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:52 smithi063 ceph-mon[109843]: from='client.34749 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:14:53.002 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:52 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:53.003 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:52 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:53.003 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:52 smithi063 ceph-mon[114756]: from='client.34749 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:14:53.003 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:52 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:53.003 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:52 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:53.075 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:53.082 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 86 16 122677 122661 89.1217 56.375 0.0191435 0.0112148 2024-08-17T17:14:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:52 smithi162 ceph-mon[104668]: from='client.34749 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:14:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:52 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:52 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:53.101 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:14:53.102 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:14:53.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:53.121 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:53.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:53.145 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:14:53.146 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:14:53.184 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:14:53.184 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:14:53.218 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:14:53.218 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:14:53.223 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:53.248 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:14:53.249 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:14:53.251 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:53.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:53.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:53.284 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:14:53.285 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:14:53.306 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:14:53.306 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:14:53.338 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:14:53.339 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:14:53.343 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:14:53.353 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:53.377 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:53.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:53.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:53.479 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:53.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:53.544 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:53.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:53.645 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:53.673 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:53.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:53.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:53.726 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:14:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:14:53] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.51.0" 2024-08-17T17:14:53.727 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:53 smithi063 ceph-mon[109843]: pgmap v1191: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2024-08-17T17:14:53.727 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:53 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/903140550' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:14:53.775 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:53.810 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:53.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:53.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:53.911 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:53.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:53.949 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:53.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:54.021 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:53 smithi063 ceph-mon[114756]: pgmap v1191: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2024-08-17T17:14:54.021 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:53 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/903140550' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:14:54.050 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:54.078 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 87 16 123551 123535 88.7252 54.625 0.0128884 0.0112654 2024-08-17T17:14:54.084 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:54.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:54.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:53 smithi162 ceph-mon[104668]: pgmap v1191: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2024-08-17T17:14:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:53 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/903140550' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:14:54.186 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:54.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:54.230 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:54.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:54.331 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:54.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:54.364 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:54.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:54.465 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:54.489 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:54.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:54.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:54.591 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:54.618 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:54.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:54.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:54.720 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:54.742 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:54 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:54.742 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:54 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:14:54.742 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:54 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:54.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:54.761 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:54.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:54.862 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:54.899 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:54.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:54.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:55.001 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:55.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:54 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:55.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:54 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:14:55.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:54 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:55.031 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:55.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:55.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:55.078 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 88 16 124403 124387 88.3221 53.25 0.0130464 0.0113166 2024-08-17T17:14:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:54 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:14:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:54 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:14:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:54 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:14:55.132 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:55.162 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:55.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:55.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:55.264 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:55.304 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:55.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:55.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:55.406 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:55.449 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:55.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:55.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:55.550 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:55.590 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:55.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:55.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:55.692 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:55.740 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:55.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:55.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:55.842 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:55.872 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:55.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:55.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:55.974 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:56.015 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:56.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:56.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:55 smithi063 ceph-mon[114756]: pgmap v1192: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2024-08-17T17:14:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:55 smithi063 ceph-mon[109843]: pgmap v1192: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2024-08-17T17:14:56.078 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 89 16 125256 125240 87.9287 53.3125 0.0268681 0.011367 2024-08-17T17:14:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:55 smithi162 ceph-mon[104668]: pgmap v1192: 89 pgs: 89 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2024-08-17T17:14:56.117 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:56.144 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:56.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:56.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:56.246 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:56.272 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:56.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:56.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:56.373 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:56.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:56.419 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:56.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:56.520 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:56.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:56.561 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:56.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:56.662 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:56.716 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:56.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:56.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:56.820 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:56.868 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:56.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:56.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:56.970 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:56.981 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:56 smithi063 ceph-mon[109843]: pgmap v1193: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2024-08-17T17:14:56.981 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:56 smithi063 ceph-mon[114756]: pgmap v1193: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2024-08-17T17:14:56.994 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:56.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:56.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:57.001 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:57.078 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Total time run: 90.0097 2024-08-17T17:14:57.078 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Total writes made: 126104 2024-08-17T17:14:57.078 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Write size: 65536 2024-08-17T17:14:57.078 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Object size: 65536 2024-08-17T17:14:57.078 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Bandwidth (MB/sec): 87.5628 2024-08-17T17:14:57.078 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev Bandwidth: 39.029 2024-08-17T17:14:57.078 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max bandwidth (MB/sec): 235.625 2024-08-17T17:14:57.079 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min bandwidth (MB/sec): 52.375 2024-08-17T17:14:57.079 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Average IOPS: 1401 2024-08-17T17:14:57.079 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev IOPS: 624.465 2024-08-17T17:14:57.079 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max IOPS: 3770 2024-08-17T17:14:57.079 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min IOPS: 838 2024-08-17T17:14:57.079 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Average Latency(s): 0.0114161 2024-08-17T17:14:57.079 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev Latency(s): 0.00813401 2024-08-17T17:14:57.079 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max latency(s): 0.151093 2024-08-17T17:14:57.079 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min latency(s): 0.0014202 2024-08-17T17:14:57.084 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:14:57.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:56 smithi162 ceph-mon[104668]: pgmap v1193: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s 2024-08-17T17:14:57.096 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:57.138 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:57.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:57.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:57.240 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:57.270 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:57.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:57.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:57.372 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:57.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:57.411 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:57.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:57.512 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:57.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:57.539 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:57.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:57.641 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:57.676 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:57.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:57.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:57.777 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:57.806 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:57.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:57.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:57.908 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:57.948 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:57.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:57.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:58.049 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:58.081 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:58.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:58.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:58.183 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:58.226 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:58.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:58.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:58.328 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:58.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:58.370 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:58.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:58.448 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:14:58.448 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:14:58.448 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:14:58.449 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:14:58.449 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:14:58.471 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:14:58.509 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:58.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:58.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:14:58.611 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:58.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:58.637 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:58.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:58.675 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:14:58.694 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:14:58.738 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:14:58.789 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:58.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:58.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:14:58.890 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:58.961 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:58.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:58.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:59.063 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:14:59.090 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:59.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:59.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:14:59.193 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:59.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:59.224 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:59.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:59.326 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:14:59.371 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:59.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:59.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:14:59.473 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:14:59.513 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:14:59.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:59.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:14:59.615 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:14:59.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:59.641 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:59.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:14:59.742 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:14:59.767 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:59.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:59.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:14:59.868 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:14:59.896 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:14:59.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:14:59.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:14:59.998 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:00.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:14:59 smithi063 ceph-mon[109843]: pgmap v1194: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 55 MiB/s wr, 893 op/s 2024-08-17T17:15:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:14:59 smithi063 ceph-mon[114756]: pgmap v1194: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 55 MiB/s wr, 893 op/s 2024-08-17T17:15:00.031 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:00.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:00.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:14:59 smithi162 ceph-mon[104668]: pgmap v1194: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 55 MiB/s wr, 893 op/s 2024-08-17T17:15:00.133 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:00.162 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:00.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:00.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:00.264 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:00.293 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:00.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:00.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:00.395 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:00.422 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:00.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:00.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:00.524 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:00.550 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:00.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:00.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:00.652 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:00.693 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:00.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:00.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:00.795 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:00.822 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:00.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:00.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:00.924 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:00.948 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:15:00.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:00.982 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:00.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:01.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:00 smithi063 ceph-mon[109843]: pgmap v1195: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 807 op/s 2024-08-17T17:15:01.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:00 smithi063 ceph-mon[114756]: pgmap v1195: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 807 op/s 2024-08-17T17:15:01.084 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:01.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:00 smithi162 ceph-mon[104668]: pgmap v1195: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 48 MiB/s wr, 807 op/s 2024-08-17T17:15:01.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:01.128 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:01.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:01.229 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:01.282 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:01.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:01.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:01.384 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:01.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:01.464 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:01.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:01.565 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:01.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:01.617 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:01.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:01.718 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:01.742 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:01.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:01.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:01.844 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:01.872 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:01.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:01.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:01.973 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:02.009 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:02.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:02.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:02.111 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:02.137 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:02.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:02.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:02.188 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:02.239 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:02.263 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:02.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:02.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:02.365 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:02.409 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:02.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:02.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:02.513 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:02.538 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:02.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:02.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:02.640 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:02.669 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:02.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:02.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:02.678 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:02 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:15:02.678 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:02 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:02.678 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:02 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:15:02.678 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:02 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:02.690 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:02.771 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:02.799 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:02.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:02.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:02.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:02 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:15:02.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:02 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:02.901 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:02.949 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:02.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:02.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:03.051 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:03.084 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:03.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:03.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:03.185 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:03.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:03.224 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:03.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:03.324 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:15:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:15:03] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.51.0" 2024-08-17T17:15:03.325 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:03.359 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:03.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:03.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:03.364 INFO:teuthology.orchestra.run.smithi063.stderr:set osd.0 primary-affinity to 0 (802) 2024-08-17T17:15:03.374 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:15:03.460 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:03.491 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:03.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:03.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:03.593 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:03.613 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:03.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:03.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:03.636 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:03 smithi162 ceph-mon[104668]: pgmap v1196: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 996 op/s 2024-08-17T17:15:03.636 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:03 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4089843901' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.0} : dispatch 2024-08-17T17:15:03.636 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:03 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2057366356' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:15:03.636 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:03 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:15:03.716 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:03.737 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:03.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:03.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:03.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:03 smithi063 ceph-mon[114756]: pgmap v1196: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 996 op/s 2024-08-17T17:15:03.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:03 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4089843901' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.0} : dispatch 2024-08-17T17:15:03.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:03 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2057366356' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:15:03.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:03 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:15:03.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:03 smithi063 ceph-mon[109843]: pgmap v1196: 89 pgs: 89 active+clean; 7.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 B/s rd, 45 MiB/s wr, 996 op/s 2024-08-17T17:15:03.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:03 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4089843901' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.0} : dispatch 2024-08-17T17:15:03.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:03 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2057366356' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:15:03.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:03 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:15:03.838 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:03.860 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:03.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:03.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:03.962 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:03.983 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:03.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:03.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:04.085 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:04.108 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:04.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:04.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:04.210 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:04.232 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:04.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:04.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:04.334 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:04.355 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:04.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:04.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:04.457 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:04.487 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:04.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:04.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:04.589 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:04.615 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:04.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:04.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:04.718 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:04.725 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:04 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:15:04.725 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:04 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4089843901' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2024-08-17T17:15:04.725 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:04 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:15:04.725 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:04 smithi063 ceph-mon[114756]: osdmap e1103: 8 total, 8 up, 8 in 2024-08-17T17:15:04.725 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:04 smithi063 ceph-mon[114756]: osdmap e1104: 8 total, 8 up, 8 in 2024-08-17T17:15:04.726 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:04 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:15:04.726 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:04 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4089843901' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2024-08-17T17:15:04.726 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:04 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:15:04.726 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:04 smithi063 ceph-mon[109843]: osdmap e1103: 8 total, 8 up, 8 in 2024-08-17T17:15:04.726 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:04 smithi063 ceph-mon[109843]: osdmap e1104: 8 total, 8 up, 8 in 2024-08-17T17:15:04.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:04.762 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:04.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:04.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:04 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:15:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:04 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4089843901' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2024-08-17T17:15:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:04 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:15:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:04 smithi162 ceph-mon[104668]: osdmap e1103: 8 total, 8 up, 8 in 2024-08-17T17:15:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:04 smithi162 ceph-mon[104668]: osdmap e1104: 8 total, 8 up, 8 in 2024-08-17T17:15:04.863 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:04.906 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:04.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:04.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:05.008 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:05.052 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:05.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:05.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:05.154 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:05.184 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:05.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:05.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:05.286 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:05.317 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:05.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:05.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:05.419 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:05.454 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:05.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:05.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:05.556 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:05.585 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:05.616 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:05 smithi063 ceph-mon[114756]: pgmap v1198: 89 pgs: 2 peering, 87 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 889 op/s 2024-08-17T17:15:05.616 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:05 smithi063 ceph-mon[114756]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:15:05.617 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:05 smithi063 ceph-mon[109843]: pgmap v1198: 89 pgs: 2 peering, 87 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 889 op/s 2024-08-17T17:15:05.617 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:05 smithi063 ceph-mon[109843]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:15:05.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:05.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:05.719 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:05.768 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:05.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:05.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:05.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:05 smithi162 ceph-mon[104668]: pgmap v1198: 89 pgs: 2 peering, 87 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 889 op/s 2024-08-17T17:15:05.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:05 smithi162 ceph-mon[104668]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:15:05.870 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:05.902 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:05.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:05.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:06.004 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:06.042 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:06.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:06.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:06.102 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:15:06.145 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:06.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:06.196 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:06.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:06.297 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:06.324 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:06.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:06.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:06.426 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:06.510 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:06.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:06.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:06.612 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:06.648 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:06.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:06.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:06.750 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:06.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:06.778 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:06.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:06.879 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:06.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:06.917 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:06.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:07.012 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:15:07.012 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:15:07.018 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:07.043 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:07.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:07.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:07.050 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:15:07.050 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:15:07.096 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:15:07.096 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:15:07.121 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:15:07.122 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:15:07.122 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:15:07.122 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:15:07.122 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.122 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7827 5.0 5:0954c38c:::smithi06387500-32:head [sparse-read 0~2368842,omap-get-vals-by-keys,omap-get-keys,omap-get-vals,omap-get-header,getxattrs] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e837)", 2024-08-17T17:15:07.122 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:07:44.966694+0000", 2024-08-17T17:15:07.122 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 442.10831079399998, 2024-08-17T17:15:07.122 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.72717941200000002, 2024-08-17T17:15:07.122 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.122 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.122 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:15:07.122 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:15:07.123 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:15:07.123 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:15:07.123 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7827 2024-08-17T17:15:07.123 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.123 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.123 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.123 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.123 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966694+0000", 2024-08-17T17:15:07.123 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.123 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.123 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.123 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.124 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966694+0000", 2024-08-17T17:15:07.124 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.124 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.124 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.124 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.124 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966696+0000", 2024-08-17T17:15:07.124 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.584e-06 2024-08-17T17:15:07.124 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.124 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.124 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.125 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966703+0000", 2024-08-17T17:15:07.125 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7190000000000001e-06 2024-08-17T17:15:07.125 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.125 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.125 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.125 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966717+0000", 2024-08-17T17:15:07.125 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3745e-05 2024-08-17T17:15:07.125 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.125 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.125 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.126 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966728+0000", 2024-08-17T17:15:07.126 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1031e-05 2024-08-17T17:15:07.126 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.126 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.126 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.126 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966797+0000", 2024-08-17T17:15:07.126 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8462999999999998e-05 2024-08-17T17:15:07.126 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.126 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.126 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:15:07.127 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966809+0000", 2024-08-17T17:15:07.127 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2887e-05 2024-08-17T17:15:07.127 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.127 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.127 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.127 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:45.686269+0000", 2024-08-17T17:15:07.127 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.71945948900000001 2024-08-17T17:15:07.127 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.127 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.127 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.127 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:45.686410+0000", 2024-08-17T17:15:07.127 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014064000000000001 2024-08-17T17:15:07.128 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.128 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.128 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.128 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:45.693873+0000", 2024-08-17T17:15:07.128 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0074638539999999998 2024-08-17T17:15:07.128 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.128 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.128 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.128 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.128 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.128 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7828 5.0 5:0954c38c:::smithi06387500-32:head [read 0~2823774,checksum] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e837)", 2024-08-17T17:15:07.128 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:07:44.966748+0000", 2024-08-17T17:15:07.128 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 442.10825632299998, 2024-08-17T17:15:07.129 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.72887504700000005, 2024-08-17T17:15:07.129 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.129 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.129 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:15:07.129 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:15:07.129 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:15:07.129 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:15:07.129 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7828 2024-08-17T17:15:07.129 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.129 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.129 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.129 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.130 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966748+0000", 2024-08-17T17:15:07.130 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.130 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.130 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.130 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.130 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966748+0000", 2024-08-17T17:15:07.130 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.130 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.130 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.130 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.130 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966749+0000", 2024-08-17T17:15:07.130 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0100000000000005e-07 2024-08-17T17:15:07.130 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.131 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.131 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.131 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966753+0000", 2024-08-17T17:15:07.131 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6279999999999998e-06 2024-08-17T17:15:07.131 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.131 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.131 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.131 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966760+0000", 2024-08-17T17:15:07.131 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7130000000000005e-06 2024-08-17T17:15:07.131 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.131 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.132 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.132 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966782+0000", 2024-08-17T17:15:07.132 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1455e-05 2024-08-17T17:15:07.132 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.132 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.132 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.132 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966833+0000", 2024-08-17T17:15:07.132 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1280999999999999e-05 2024-08-17T17:15:07.132 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.132 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.132 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:15:07.132 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966839+0000", 2024-08-17T17:15:07.132 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3779999999999998e-06 2024-08-17T17:15:07.133 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.133 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.133 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.133 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:45.694057+0000", 2024-08-17T17:15:07.133 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.72721760199999996 2024-08-17T17:15:07.133 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.133 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.133 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.133 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:45.694157+0000", 2024-08-17T17:15:07.133 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000100383 2024-08-17T17:15:07.133 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.133 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.133 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.134 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:45.695623+0000", 2024-08-17T17:15:07.134 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001466106 2024-08-17T17:15:07.134 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.134 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.134 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.134 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.134 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.134 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7829 5.0 5:0954c38c:::smithi06387500-32:head [sparse-read 0~2368842] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e837)", 2024-08-17T17:15:07.134 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:07:44.966800+0000", 2024-08-17T17:15:07.134 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 442.10820476499998, 2024-08-17T17:15:07.134 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.72972638700000003, 2024-08-17T17:15:07.134 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.135 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.135 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:15:07.135 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:15:07.135 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:15:07.135 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:15:07.135 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7829 2024-08-17T17:15:07.135 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.135 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.135 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.135 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.135 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966800+0000", 2024-08-17T17:15:07.135 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.135 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.136 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.136 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.136 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966800+0000", 2024-08-17T17:15:07.136 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.136 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.136 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.136 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.136 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966800+0000", 2024-08-17T17:15:07.136 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.82e-07 2024-08-17T17:15:07.136 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.136 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.136 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.136 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966803+0000", 2024-08-17T17:15:07.137 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.7860000000000001e-06 2024-08-17T17:15:07.137 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.137 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.137 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.137 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966811+0000", 2024-08-17T17:15:07.137 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5569999999999998e-06 2024-08-17T17:15:07.137 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.137 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.137 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.137 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966816+0000", 2024-08-17T17:15:07.137 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4739999999999996e-06 2024-08-17T17:15:07.137 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.137 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.138 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.138 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966857+0000", 2024-08-17T17:15:07.138 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.0240000000000001e-05 2024-08-17T17:15:07.138 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.138 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.138 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:15:07.138 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:44.966861+0000", 2024-08-17T17:15:07.138 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3019999999999996e-06 2024-08-17T17:15:07.138 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.139 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.139 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.139 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:45.696240+0000", 2024-08-17T17:15:07.139 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.72937943699999996 2024-08-17T17:15:07.139 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.139 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.139 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.139 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:45.696343+0000", 2024-08-17T17:15:07.139 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000102884 2024-08-17T17:15:07.139 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.139 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.139 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.139 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:45.696526+0000", 2024-08-17T17:15:07.140 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00018322499999999999 2024-08-17T17:15:07.140 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.140 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.140 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.140 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.140 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.140 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35798 6.6 6:6b142a6f:::benchmark_data_smithi063_160268_object35797:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:07.140 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.872995+0000", 2024-08-17T17:15:07.140 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 86.202010106000003, 2024-08-17T17:15:07.140 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14771484700000001, 2024-08-17T17:15:07.140 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.140 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.140 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:07.141 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:15:07.141 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:15:07.141 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:07.141 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35798 2024-08-17T17:15:07.141 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.141 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.141 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.141 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.141 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.872995+0000", 2024-08-17T17:15:07.141 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.141 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.141 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.141 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.142 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.872995+0000", 2024-08-17T17:15:07.142 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.142 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.142 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.142 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.142 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.872996+0000", 2024-08-17T17:15:07.142 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2839999999999999e-06 2024-08-17T17:15:07.142 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.142 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.142 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.142 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.873017+0000", 2024-08-17T17:15:07.142 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0613000000000001e-05 2024-08-17T17:15:07.142 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.143 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.143 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.143 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.873026+0000", 2024-08-17T17:15:07.143 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7520000000000008e-06 2024-08-17T17:15:07.143 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.143 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.143 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.143 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.873032+0000", 2024-08-17T17:15:07.143 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1040000000000003e-06 2024-08-17T17:15:07.143 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.143 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.143 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.143 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.873118+0000", 2024-08-17T17:15:07.143 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.5515999999999999e-05 2024-08-17T17:15:07.144 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.144 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.144 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.144 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.873236+0000", 2024-08-17T17:15:07.144 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000118021 2024-08-17T17:15:07.144 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.144 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.144 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:15:07.144 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.873332+0000", 2024-08-17T17:15:07.144 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.5645999999999999e-05 2024-08-17T17:15:07.144 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.144 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.145 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:15:07.145 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.875236+0000", 2024-08-17T17:15:07.145 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0019039860000000001 2024-08-17T17:15:07.145 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.145 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.145 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.145 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877264+0000", 2024-08-17T17:15:07.145 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0020288039999999999 2024-08-17T17:15:07.145 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.145 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.145 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.145 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020610+0000", 2024-08-17T17:15:07.145 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.143345943 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020661+0000", 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0300999999999998e-05 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020710+0000", 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8877e-05 2024-08-17T17:15:07.146 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.147 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.147 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.147 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.147 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.147 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35805 6.6 6:655e213e:::benchmark_data_smithi063_160268_object35804:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:07.147 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.875282+0000", 2024-08-17T17:15:07.147 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 86.199722922000007, 2024-08-17T17:15:07.147 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14554330900000001, 2024-08-17T17:15:07.147 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.147 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.148 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:07.148 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:15:07.148 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:15:07.148 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:07.148 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35805 2024-08-17T17:15:07.148 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.148 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.148 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.148 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.148 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.875282+0000", 2024-08-17T17:15:07.148 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.148 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.148 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.149 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.149 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.875282+0000", 2024-08-17T17:15:07.149 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.149 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.149 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.149 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.149 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.875283+0000", 2024-08-17T17:15:07.149 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2860000000000001e-06 2024-08-17T17:15:07.149 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.149 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.149 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.149 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.875304+0000", 2024-08-17T17:15:07.149 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1154000000000001e-05 2024-08-17T17:15:07.150 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.150 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.150 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.150 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.875316+0000", 2024-08-17T17:15:07.150 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2177e-05 2024-08-17T17:15:07.150 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.150 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.150 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.150 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.875323+0000", 2024-08-17T17:15:07.151 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8889999999999996e-06 2024-08-17T17:15:07.151 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.151 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.151 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.151 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.875726+0000", 2024-08-17T17:15:07.151 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00040291900000000001 2024-08-17T17:15:07.151 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.151 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.151 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.151 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.875858+0000", 2024-08-17T17:15:07.151 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013197799999999999 2024-08-17T17:15:07.152 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.152 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.152 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:15:07.152 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876049+0000", 2024-08-17T17:15:07.152 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000190576 2024-08-17T17:15:07.152 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.152 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.152 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:15:07.152 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877437+0000", 2024-08-17T17:15:07.152 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0013876990000000001 2024-08-17T17:15:07.152 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.152 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.152 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.153 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020571+0000", 2024-08-17T17:15:07.153 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14313480000000001 2024-08-17T17:15:07.153 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.153 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.153 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.153 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020746+0000", 2024-08-17T17:15:07.153 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000174286 2024-08-17T17:15:07.153 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.153 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.153 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:15:07.153 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020789+0000", 2024-08-17T17:15:07.154 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3309999999999997e-05 2024-08-17T17:15:07.154 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.154 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.154 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.154 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020825+0000", 2024-08-17T17:15:07.154 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6235000000000001e-05 2024-08-17T17:15:07.154 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.154 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.154 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.154 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.154 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.154 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:35809 6.f e1093/1086 6:f6d7b118:::benchmark_data_smithi063_160268_object35808:head v 1093'2277, mlcod=1093'2274)", 2024-08-17T17:15:07.155 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.877456+0000", 2024-08-17T17:15:07.155 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 86.197548550999997, 2024-08-17T17:15:07.155 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.146270647, 2024-08-17T17:15:07.155 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.155 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.155 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:07.155 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.155 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.155 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.155 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877456+0000", 2024-08-17T17:15:07.155 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.155 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.155 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.156 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.156 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877456+0000", 2024-08-17T17:15:07.156 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.156 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.156 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.156 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.156 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877457+0000", 2024-08-17T17:15:07.156 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4799999999999997e-07 2024-08-17T17:15:07.156 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.156 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.156 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.156 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877477+0000", 2024-08-17T17:15:07.157 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0411000000000002e-05 2024-08-17T17:15:07.157 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.157 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.157 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.157 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877488+0000", 2024-08-17T17:15:07.157 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0791000000000001e-05 2024-08-17T17:15:07.157 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.157 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.157 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.157 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877495+0000", 2024-08-17T17:15:07.157 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.37e-06 2024-08-17T17:15:07.157 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.157 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.158 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.158 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878171+0000", 2024-08-17T17:15:07.158 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00067635199999999996 2024-08-17T17:15:07.158 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.158 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.158 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.158 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878189+0000", 2024-08-17T17:15:07.158 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8011000000000001e-05 2024-08-17T17:15:07.158 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.158 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.158 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:15:07.158 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.023678+0000", 2024-08-17T17:15:07.158 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14548936700000001 2024-08-17T17:15:07.159 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.159 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.159 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.159 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.023727+0000", 2024-08-17T17:15:07.159 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8497e-05 2024-08-17T17:15:07.159 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.159 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.159 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.159 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.159 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.159 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.44609.0:35805 6.6 e1093/1086 ondisk, result = 0)", 2024-08-17T17:15:07.160 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.878209+0000", 2024-08-17T17:15:07.160 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 86.196795809999998, 2024-08-17T17:15:07.160 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.142370406, 2024-08-17T17:15:07.160 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.160 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.160 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:15:07.160 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.160 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.160 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.161 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878209+0000", 2024-08-17T17:15:07.161 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.161 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.161 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.161 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.161 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878209+0000", 2024-08-17T17:15:07.162 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.162 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.162 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.162 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.162 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878210+0000", 2024-08-17T17:15:07.162 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1299999999999999e-07 2024-08-17T17:15:07.162 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.162 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.162 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.163 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878214+0000", 2024-08-17T17:15:07.163 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9729999999999999e-06 2024-08-17T17:15:07.163 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.163 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.163 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.163 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878222+0000", 2024-08-17T17:15:07.163 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7760000000000003e-06 2024-08-17T17:15:07.163 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.164 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.164 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.164 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878229+0000", 2024-08-17T17:15:07.164 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6259999999999997e-06 2024-08-17T17:15:07.164 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.164 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.164 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.164 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020557+0000", 2024-08-17T17:15:07.164 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14232789900000001 2024-08-17T17:15:07.165 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.165 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.165 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.165 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020565+0000", 2024-08-17T17:15:07.165 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6190000000000002e-06 2024-08-17T17:15:07.165 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.165 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.165 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.165 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020579+0000", 2024-08-17T17:15:07.165 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4800000000000001e-05 2024-08-17T17:15:07.165 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.165 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.165 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.166 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.166 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.166 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:35808 6.8 e1093/1086 6:182f8ffd:::benchmark_data_smithi063_160268_object35807:head v 1093'2243, mlcod=1093'2240)", 2024-08-17T17:15:07.166 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.878246+0000", 2024-08-17T17:15:07.166 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 86.196758368000005, 2024-08-17T17:15:07.166 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.142191873, 2024-08-17T17:15:07.166 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.166 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.166 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:07.166 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.166 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.166 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.166 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878246+0000", 2024-08-17T17:15:07.167 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.167 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.167 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.167 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.167 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878246+0000", 2024-08-17T17:15:07.167 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.167 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.167 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.167 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.167 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878247+0000", 2024-08-17T17:15:07.167 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4799999999999999e-07 2024-08-17T17:15:07.167 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.167 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.168 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.168 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878279+0000", 2024-08-17T17:15:07.168 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.2465000000000002e-05 2024-08-17T17:15:07.168 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.168 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.168 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.168 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878287+0000", 2024-08-17T17:15:07.168 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3470000000000001e-06 2024-08-17T17:15:07.168 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.168 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.168 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.168 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878292+0000", 2024-08-17T17:15:07.168 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0540000000000002e-06 2024-08-17T17:15:07.169 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.169 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.169 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.169 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878455+0000", 2024-08-17T17:15:07.169 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016288500000000001 2024-08-17T17:15:07.169 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.169 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.169 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.169 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878471+0000", 2024-08-17T17:15:07.169 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6523000000000001e-05 2024-08-17T17:15:07.169 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.169 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.169 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:15:07.170 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020388+0000", 2024-08-17T17:15:07.170 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14191709899999999 2024-08-17T17:15:07.170 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.170 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.170 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.170 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020438+0000", 2024-08-17T17:15:07.170 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0151999999999997e-05 2024-08-17T17:15:07.170 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.170 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.170 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.170 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.170 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.171 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:35807 6.2 e1093/1086 6:425615ad:::benchmark_data_smithi063_160268_object35806:head v 1093'2246, mlcod=1093'2244)", 2024-08-17T17:15:07.171 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.878318+0000", 2024-08-17T17:15:07.171 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 86.196687048000001, 2024-08-17T17:15:07.171 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14543654, 2024-08-17T17:15:07.171 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.171 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.171 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:07.171 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.171 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.171 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.171 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878318+0000", 2024-08-17T17:15:07.171 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.172 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.172 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.172 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.172 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878318+0000", 2024-08-17T17:15:07.172 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.172 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.172 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.173 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.173 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878318+0000", 2024-08-17T17:15:07.173 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3799999999999998e-07 2024-08-17T17:15:07.173 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.173 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.173 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.173 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878343+0000", 2024-08-17T17:15:07.173 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5077999999999999e-05 2024-08-17T17:15:07.173 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.173 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.173 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.173 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878350+0000", 2024-08-17T17:15:07.174 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8900000000000001e-06 2024-08-17T17:15:07.174 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.174 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.174 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.174 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878355+0000", 2024-08-17T17:15:07.174 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6979999999999997e-06 2024-08-17T17:15:07.174 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.174 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.174 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.174 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020213+0000", 2024-08-17T17:15:07.174 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14185819799999999 2024-08-17T17:15:07.174 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.175 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.175 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.175 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020237+0000", 2024-08-17T17:15:07.175 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3762000000000001e-05 2024-08-17T17:15:07.175 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.175 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.175 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:15:07.175 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.023712+0000", 2024-08-17T17:15:07.175 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0034750699999999998 2024-08-17T17:15:07.175 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.176 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.176 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.176 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.023754+0000", 2024-08-17T17:15:07.176 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2406e-05 2024-08-17T17:15:07.176 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.176 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.176 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.176 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.176 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.176 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.44609.0:35798 6.6 e1093/1086 ondisk, result = 0)", 2024-08-17T17:15:07.176 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.878369+0000", 2024-08-17T17:15:07.176 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 86.196635912000005, 2024-08-17T17:15:07.176 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14234665699999999, 2024-08-17T17:15:07.177 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.177 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.177 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:15:07.177 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.177 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.177 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.177 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878369+0000", 2024-08-17T17:15:07.177 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.177 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.177 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.177 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.177 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878369+0000", 2024-08-17T17:15:07.177 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.178 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.178 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.178 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.178 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878369+0000", 2024-08-17T17:15:07.178 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4200000000000002e-07 2024-08-17T17:15:07.178 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.178 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.178 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.178 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878371+0000", 2024-08-17T17:15:07.178 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9300000000000002e-06 2024-08-17T17:15:07.178 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.179 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.179 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.179 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878376+0000", 2024-08-17T17:15:07.179 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3090000000000003e-06 2024-08-17T17:15:07.179 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.179 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.179 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.179 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878380+0000", 2024-08-17T17:15:07.179 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8909999999999997e-06 2024-08-17T17:15:07.179 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.179 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.179 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.179 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020598+0000", 2024-08-17T17:15:07.179 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14221821200000001 2024-08-17T17:15:07.180 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.180 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.180 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.180 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020606+0000", 2024-08-17T17:15:07.180 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3139999999999998e-06 2024-08-17T17:15:07.180 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.180 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.180 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.180 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020716+0000", 2024-08-17T17:15:07.180 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000109759 2024-08-17T17:15:07.180 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.180 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.180 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.181 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.181 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.181 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.44609.0:35805 6.6 e1093/1086 ondisk, result = 0)", 2024-08-17T17:15:07.181 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.878388+0000", 2024-08-17T17:15:07.181 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 86.196616405, 2024-08-17T17:15:07.181 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.142441597, 2024-08-17T17:15:07.181 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.181 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.181 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:15:07.181 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.181 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.181 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.182 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878388+0000", 2024-08-17T17:15:07.182 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.182 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.182 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.182 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.182 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878388+0000", 2024-08-17T17:15:07.182 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.182 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.182 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.182 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.182 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878389+0000", 2024-08-17T17:15:07.182 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8400000000000001e-07 2024-08-17T17:15:07.183 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.183 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.183 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.183 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878390+0000", 2024-08-17T17:15:07.183 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5519999999999999e-06 2024-08-17T17:15:07.183 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.183 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.184 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.184 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878395+0000", 2024-08-17T17:15:07.184 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9239999999999998e-06 2024-08-17T17:15:07.184 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.184 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.184 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.185 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878399+0000", 2024-08-17T17:15:07.185 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.619e-06 2024-08-17T17:15:07.185 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.185 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.186 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.186 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020735+0000", 2024-08-17T17:15:07.186 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14233647499999999 2024-08-17T17:15:07.186 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.186 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.186 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.186 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020741+0000", 2024-08-17T17:15:07.186 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0820000000000004e-06 2024-08-17T17:15:07.186 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.186 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.187 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.187 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020830+0000", 2024-08-17T17:15:07.187 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8760999999999999e-05 2024-08-17T17:15:07.187 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.187 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.187 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.187 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.187 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.187 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:35811 6.f e1093/1086 6:f8be1079:::benchmark_data_smithi063_160268_object35810:head v 1093'2278, mlcod=1093'2274)", 2024-08-17T17:15:07.187 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.879343+0000", 2024-08-17T17:15:07.188 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 86.195662010999996, 2024-08-17T17:15:07.188 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14616310800000001, 2024-08-17T17:15:07.188 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.188 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.188 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:07.188 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.188 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.188 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.188 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.879343+0000", 2024-08-17T17:15:07.188 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.188 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.189 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.189 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.189 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.879343+0000", 2024-08-17T17:15:07.189 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.189 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.189 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.189 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.189 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.879343+0000", 2024-08-17T17:15:07.189 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0100000000000001e-07 2024-08-17T17:15:07.189 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.189 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.189 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.189 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.879365+0000", 2024-08-17T17:15:07.190 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1807e-05 2024-08-17T17:15:07.190 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.190 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.190 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.190 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.879372+0000", 2024-08-17T17:15:07.190 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6190000000000002e-06 2024-08-17T17:15:07.190 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.190 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.190 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.190 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.879378+0000", 2024-08-17T17:15:07.190 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3000000000000001e-06 2024-08-17T17:15:07.190 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.191 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.191 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.191 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.879507+0000", 2024-08-17T17:15:07.191 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012970000000000001 2024-08-17T17:15:07.191 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.191 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.191 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.191 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.879525+0000", 2024-08-17T17:15:07.191 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7728999999999999e-05 2024-08-17T17:15:07.191 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.192 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.192 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:15:07.192 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.025402+0000", 2024-08-17T17:15:07.192 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14587727 2024-08-17T17:15:07.192 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.192 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.192 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.192 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.025506+0000", 2024-08-17T17:15:07.192 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010338199999999999 2024-08-17T17:15:07.192 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.193 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.193 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.193 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.193 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.193 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35814 6.c 6:3de58c66:::benchmark_data_smithi063_160268_object35813:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:07.193 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.880041+0000", 2024-08-17T17:15:07.193 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 86.194963947000005, 2024-08-17T17:15:07.193 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.146597431, 2024-08-17T17:15:07.193 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.193 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.193 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:07.193 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:15:07.194 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:15:07.194 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:07.194 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35814 2024-08-17T17:15:07.194 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.194 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.194 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.194 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.194 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880041+0000", 2024-08-17T17:15:07.194 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.194 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.194 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.194 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.195 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880041+0000", 2024-08-17T17:15:07.195 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.195 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.195 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.195 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.195 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880042+0000", 2024-08-17T17:15:07.195 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5e-06 2024-08-17T17:15:07.195 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.195 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.195 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.195 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880076+0000", 2024-08-17T17:15:07.195 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3794000000000002e-05 2024-08-17T17:15:07.195 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.196 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.196 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.196 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020381+0000", 2024-08-17T17:15:07.196 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14030504599999999 2024-08-17T17:15:07.196 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.196 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.196 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.196 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020393+0000", 2024-08-17T17:15:07.196 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1952e-05 2024-08-17T17:15:07.196 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.196 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.196 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.196 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020554+0000", 2024-08-17T17:15:07.197 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016133599999999999 2024-08-17T17:15:07.197 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.197 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.197 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.197 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.023217+0000", 2024-08-17T17:15:07.197 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0026628519999999998 2024-08-17T17:15:07.197 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.197 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.197 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 5,6", 2024-08-17T17:15:07.197 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.023366+0000", 2024-08-17T17:15:07.197 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014854199999999999 2024-08-17T17:15:07.197 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.197 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.198 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.198 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.025017+0000", 2024-08-17T17:15:07.198 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0016512059999999999 2024-08-17T17:15:07.198 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.198 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.198 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.198 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.025055+0000", 2024-08-17T17:15:07.198 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8229999999999998e-05 2024-08-17T17:15:07.198 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.198 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.198 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:15:07.199 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.026516+0000", 2024-08-17T17:15:07.199 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001460622 2024-08-17T17:15:07.199 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.199 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.199 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:15:07.199 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.026590+0000", 2024-08-17T17:15:07.199 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3703999999999998e-05 2024-08-17T17:15:07.199 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.199 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.199 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.199 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.026638+0000", 2024-08-17T17:15:07.199 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8646999999999997e-05 2024-08-17T17:15:07.199 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.200 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.200 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.200 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.200 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.200 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:35816 6.0 e1093/1086 6:0defa481:::benchmark_data_smithi063_160268_object35815:head v 1093'2267, mlcod=1093'2263)", 2024-08-17T17:15:07.200 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.880831+0000", 2024-08-17T17:15:07.200 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 86.194173472000003, 2024-08-17T17:15:07.200 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14580554100000001, 2024-08-17T17:15:07.200 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.200 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.200 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:07.200 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.200 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.201 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.201 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880831+0000", 2024-08-17T17:15:07.201 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.201 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.201 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.201 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.201 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880831+0000", 2024-08-17T17:15:07.201 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.201 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.201 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.201 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.201 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880832+0000", 2024-08-17T17:15:07.201 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1179999999999999e-06 2024-08-17T17:15:07.202 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.202 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.202 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.202 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880861+0000", 2024-08-17T17:15:07.202 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8578e-05 2024-08-17T17:15:07.202 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.202 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.202 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.202 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.021158+0000", 2024-08-17T17:15:07.202 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14029731200000001 2024-08-17T17:15:07.202 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.202 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.203 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.203 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.021166+0000", 2024-08-17T17:15:07.203 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9489999999999996e-06 2024-08-17T17:15:07.203 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.203 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.203 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.203 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.022240+0000", 2024-08-17T17:15:07.203 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001073601 2024-08-17T17:15:07.203 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.203 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.203 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.203 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.022258+0000", 2024-08-17T17:15:07.203 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8621999999999999e-05 2024-08-17T17:15:07.204 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.204 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.204 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:15:07.204 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.026602+0000", 2024-08-17T17:15:07.204 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0043434700000000003 2024-08-17T17:15:07.204 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.204 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.204 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.204 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.026637+0000", 2024-08-17T17:15:07.204 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4891000000000002e-05 2024-08-17T17:15:07.204 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.204 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.204 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.204 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.205 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.205 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:131734 6.0 6:01519f00:::benchmark_data_smithi063_160268_object5627:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:07.205 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:15:03.383813+0000", 2024-08-17T17:15:07.205 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 3.691191506, 2024-08-17T17:15:07.205 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0183176700000001, 2024-08-17T17:15:07.205 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.205 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.205 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:15:07.205 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:15:07.205 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:15:07.205 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:07.205 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 131734 2024-08-17T17:15:07.206 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.206 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.206 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.206 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.206 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.383813+0000", 2024-08-17T17:15:07.206 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.206 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.206 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.206 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.206 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.383813+0000", 2024-08-17T17:15:07.207 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.207 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.207 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.207 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.207 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.383814+0000", 2024-08-17T17:15:07.207 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6399999999999998e-07 2024-08-17T17:15:07.207 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.207 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.207 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.207 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.383815+0000", 2024-08-17T17:15:07.207 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8029999999999999e-06 2024-08-17T17:15:07.207 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.207 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.207 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.208 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.383821+0000", 2024-08-17T17:15:07.208 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8579999999999996e-06 2024-08-17T17:15:07.208 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.208 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.208 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.208 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.383826+0000", 2024-08-17T17:15:07.208 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8149999999999996e-06 2024-08-17T17:15:07.208 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.208 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.208 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.208 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.400839+0000", 2024-08-17T17:15:07.208 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.017012528999999998 2024-08-17T17:15:07.208 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.209 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.209 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:15:07.209 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.400848+0000", 2024-08-17T17:15:07.209 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.8500000000000006e-06 2024-08-17T17:15:07.209 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.209 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.209 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.209 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.385445+0000", 2024-08-17T17:15:07.209 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98459668499999997 2024-08-17T17:15:07.209 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.209 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.209 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for missing object", 2024-08-17T17:15:07.209 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.385576+0000", 2024-08-17T17:15:07.209 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013096399999999999 2024-08-17T17:15:07.210 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.210 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.210 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.210 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.402061+0000", 2024-08-17T17:15:07.210 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.016484792000000002 2024-08-17T17:15:07.210 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.210 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.210 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.210 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.402131+0000", 2024-08-17T17:15:07.211 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0010000000000002e-05 2024-08-17T17:15:07.211 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.211 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.211 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.211 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.211 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.211 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:131765 6.0 6:0e2795f3:::benchmark_data_smithi063_160268_object5658:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:07.211 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:15:03.423018+0000", 2024-08-17T17:15:07.211 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 3.6519869979999999, 2024-08-17T17:15:07.211 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.97858144400000002, 2024-08-17T17:15:07.211 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.211 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.211 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:07.212 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:15:07.212 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:15:07.212 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:07.212 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 131765 2024-08-17T17:15:07.212 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.212 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.212 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.212 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.212 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.423018+0000", 2024-08-17T17:15:07.212 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.212 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.212 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.213 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.213 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.423018+0000", 2024-08-17T17:15:07.213 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.213 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.213 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.213 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.213 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.423019+0000", 2024-08-17T17:15:07.213 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4619999999999999e-06 2024-08-17T17:15:07.213 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.213 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.214 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.214 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.423027+0000", 2024-08-17T17:15:07.214 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3989999999999998e-06 2024-08-17T17:15:07.214 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.214 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.214 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.214 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.423040+0000", 2024-08-17T17:15:07.214 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3576e-05 2024-08-17T17:15:07.214 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.214 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.214 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.215 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.423049+0000", 2024-08-17T17:15:07.215 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2979999999999992e-06 2024-08-17T17:15:07.215 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.215 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.215 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.215 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.425546+0000", 2024-08-17T17:15:07.215 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00249716 2024-08-17T17:15:07.215 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.215 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.215 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:15:07.215 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.425558+0000", 2024-08-17T17:15:07.215 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2160999999999999e-05 2024-08-17T17:15:07.215 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.216 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.216 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.216 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.385845+0000", 2024-08-17T17:15:07.216 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.960287169 2024-08-17T17:15:07.216 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.216 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.216 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.216 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.385995+0000", 2024-08-17T17:15:07.216 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014971299999999999 2024-08-17T17:15:07.216 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.216 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.216 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:15:07.216 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.386603+0000", 2024-08-17T17:15:07.217 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00060817900000000003 2024-08-17T17:15:07.217 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.217 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.217 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.217 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.401289+0000", 2024-08-17T17:15:07.217 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.014685831 2024-08-17T17:15:07.217 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.217 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.217 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:15:07.217 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.401387+0000", 2024-08-17T17:15:07.217 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.8006999999999999e-05 2024-08-17T17:15:07.217 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.217 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.217 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.218 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.401447+0000", 2024-08-17T17:15:07.218 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9939000000000003e-05 2024-08-17T17:15:07.218 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.218 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.218 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:15:07.218 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.401506+0000", 2024-08-17T17:15:07.218 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9481999999999997e-05 2024-08-17T17:15:07.218 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.218 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.218 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.218 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.401599+0000", 2024-08-17T17:15:07.219 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3067999999999997e-05 2024-08-17T17:15:07.219 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.219 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.219 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.219 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.219 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.219 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:131794 6.0 6:0d3b845c:::benchmark_data_smithi063_160268_object5687:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:07.219 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:15:03.453149+0000", 2024-08-17T17:15:07.219 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 3.6218560119999998, 2024-08-17T17:15:07.219 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.94870007599999995, 2024-08-17T17:15:07.219 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.219 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.219 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:07.219 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:15:07.220 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:15:07.220 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:07.220 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 131794 2024-08-17T17:15:07.220 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.220 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.220 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.220 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.220 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453149+0000", 2024-08-17T17:15:07.220 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.220 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.220 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.220 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.221 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:07.221 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453149+0000", 2024-08-17T17:15:07.222 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.222 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.222 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.222 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.222 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453150+0000", 2024-08-17T17:15:07.222 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.415e-06 2024-08-17T17:15:07.222 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.222 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.222 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.222 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453155+0000", 2024-08-17T17:15:07.222 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8980000000000004e-06 2024-08-17T17:15:07.222 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.223 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.223 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.223 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453169+0000", 2024-08-17T17:15:07.223 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4406000000000001e-05 2024-08-17T17:15:07.223 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.223 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.223 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.223 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453177+0000", 2024-08-17T17:15:07.223 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0949999999999998e-06 2024-08-17T17:15:07.223 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.223 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.223 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.224 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453283+0000", 2024-08-17T17:15:07.224 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010690499999999999 2024-08-17T17:15:07.224 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.224 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.224 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:15:07.224 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453299+0000", 2024-08-17T17:15:07.224 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5923999999999999e-05 2024-08-17T17:15:07.224 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.224 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.224 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.224 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.395376+0000", 2024-08-17T17:15:07.224 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.94207697800000001 2024-08-17T17:15:07.224 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.225 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.225 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.225 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.395516+0000", 2024-08-17T17:15:07.225 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013987900000000001 2024-08-17T17:15:07.225 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.225 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.225 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:15:07.225 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.395658+0000", 2024-08-17T17:15:07.225 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000141292 2024-08-17T17:15:07.225 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.225 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.225 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:15:07.225 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.401391+0000", 2024-08-17T17:15:07.226 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0057332679999999997 2024-08-17T17:15:07.226 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.226 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.226 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.226 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.401656+0000", 2024-08-17T17:15:07.226 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000264966 2024-08-17T17:15:07.226 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.226 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.226 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.226 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.401712+0000", 2024-08-17T17:15:07.226 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6554000000000001e-05 2024-08-17T17:15:07.226 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.227 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.227 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:15:07.227 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.401765+0000", 2024-08-17T17:15:07.227 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2719999999999997e-05 2024-08-17T17:15:07.227 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.227 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.227 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.227 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.401849+0000", 2024-08-17T17:15:07.227 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3776000000000006e-05 2024-08-17T17:15:07.227 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.227 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.228 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.228 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.228 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.228 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:131796 6.0 6:0c0d9b45:::benchmark_data_smithi063_160268_object5689:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:07.228 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:15:03.453330+0000", 2024-08-17T17:15:07.228 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 3.6216746049999999, 2024-08-17T17:15:07.228 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.96818890999999996, 2024-08-17T17:15:07.228 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.228 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.228 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:07.228 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:15:07.228 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:15:07.229 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:07.229 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 131796 2024-08-17T17:15:07.229 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.229 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.229 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.229 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.229 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453330+0000", 2024-08-17T17:15:07.229 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.229 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.229 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.229 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.229 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453330+0000", 2024-08-17T17:15:07.229 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.230 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.230 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.230 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.230 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453331+0000", 2024-08-17T17:15:07.230 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.54e-07 2024-08-17T17:15:07.230 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.230 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.230 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.230 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453334+0000", 2024-08-17T17:15:07.230 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4170000000000001e-06 2024-08-17T17:15:07.230 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.230 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.231 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.231 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453344+0000", 2024-08-17T17:15:07.231 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.6509999999999996e-06 2024-08-17T17:15:07.231 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.231 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.231 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.231 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453350+0000", 2024-08-17T17:15:07.231 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0480000000000004e-06 2024-08-17T17:15:07.231 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.231 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.231 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.231 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453404+0000", 2024-08-17T17:15:07.232 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3820000000000003e-05 2024-08-17T17:15:07.232 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.232 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.232 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:15:07.232 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.453409+0000", 2024-08-17T17:15:07.232 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2460000000000003e-06 2024-08-17T17:15:07.232 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.232 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.232 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.232 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.402282+0000", 2024-08-17T17:15:07.232 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.94887251800000005 2024-08-17T17:15:07.233 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.233 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.233 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.233 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.402433+0000", 2024-08-17T17:15:07.233 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015118099999999999 2024-08-17T17:15:07.233 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.233 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.233 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:15:07.233 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.402545+0000", 2024-08-17T17:15:07.233 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011191 2024-08-17T17:15:07.233 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.233 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.233 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.234 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.413284+0000", 2024-08-17T17:15:07.234 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.010739039000000001 2024-08-17T17:15:07.234 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.234 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.234 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:15:07.234 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.413297+0000", 2024-08-17T17:15:07.234 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3169999999999999e-05 2024-08-17T17:15:07.234 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.234 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.234 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.234 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.421318+0000", 2024-08-17T17:15:07.234 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0080215770000000002 2024-08-17T17:15:07.234 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.235 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.235 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:15:07.235 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.421426+0000", 2024-08-17T17:15:07.235 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000107959 2024-08-17T17:15:07.235 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.235 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.235 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.235 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.421519+0000", 2024-08-17T17:15:07.235 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.2620000000000004e-05 2024-08-17T17:15:07.235 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.235 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.235 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.235 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.236 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.236 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:131810 6.0 6:0b5c1b25:::benchmark_data_smithi063_160268_object5703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:07.236 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:15:03.467271+0000", 2024-08-17T17:15:07.236 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 3.607734126, 2024-08-17T17:15:07.236 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.96049123599999997, 2024-08-17T17:15:07.236 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.236 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.236 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:07.236 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:15:07.236 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:15:07.236 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:07.236 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 131810 2024-08-17T17:15:07.236 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.236 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.237 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.237 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.237 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:15:07.237 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:15:07.237 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.467271+0000", 2024-08-17T17:15:07.238 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.238 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.238 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.238 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.238 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.467271+0000", 2024-08-17T17:15:07.238 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.238 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.238 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.238 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.238 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.467272+0000", 2024-08-17T17:15:07.238 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.823e-06 2024-08-17T17:15:07.238 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.239 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.239 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.239 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.467278+0000", 2024-08-17T17:15:07.239 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9719999999999997e-06 2024-08-17T17:15:07.239 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.239 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.239 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.239 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.467297+0000", 2024-08-17T17:15:07.239 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9051000000000001e-05 2024-08-17T17:15:07.239 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.239 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.240 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.240 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.467306+0000", 2024-08-17T17:15:07.240 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3760000000000008e-06 2024-08-17T17:15:07.240 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.240 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.240 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.240 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.467375+0000", 2024-08-17T17:15:07.240 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9135999999999997e-05 2024-08-17T17:15:07.240 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.240 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.240 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:15:07.240 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.467387+0000", 2024-08-17T17:15:07.240 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1837e-05 2024-08-17T17:15:07.240 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.241 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.241 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.241 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.406309+0000", 2024-08-17T17:15:07.241 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.93892257300000004 2024-08-17T17:15:07.241 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.241 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.241 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.241 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.406437+0000", 2024-08-17T17:15:07.241 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012773999999999999 2024-08-17T17:15:07.241 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.241 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.241 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:15:07.241 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.406572+0000", 2024-08-17T17:15:07.241 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013511400000000001 2024-08-17T17:15:07.242 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.242 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.242 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:15:07.242 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.413302+0000", 2024-08-17T17:15:07.242 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0067295410000000003 2024-08-17T17:15:07.242 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.242 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.242 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.242 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.413334+0000", 2024-08-17T17:15:07.242 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1779999999999997e-05 2024-08-17T17:15:07.242 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.242 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.243 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.243 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.427677+0000", 2024-08-17T17:15:07.243 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.014343277 2024-08-17T17:15:07.243 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.243 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.243 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:15:07.243 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.427726+0000", 2024-08-17T17:15:07.243 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9021999999999999e-05 2024-08-17T17:15:07.243 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.243 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.243 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.243 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.427762+0000", 2024-08-17T17:15:07.243 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5994000000000001e-05 2024-08-17T17:15:07.243 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.243 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.244 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.244 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.244 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.244 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:131847 6.0 6:005d3ca3:::benchmark_data_smithi063_160268_object5740:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:07.244 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:15:03.530834+0000", 2024-08-17T17:15:07.244 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 3.5441709530000001, 2024-08-17T17:15:07.244 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.89729046599999995, 2024-08-17T17:15:07.244 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:15:07.244 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:15:07.244 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:07.244 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:15:07.244 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:15:07.244 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:07.244 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 131847 2024-08-17T17:15:07.244 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.245 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:15:07.245 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.245 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:15:07.245 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.530834+0000", 2024-08-17T17:15:07.245 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.245 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.245 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.245 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:15:07.245 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.530834+0000", 2024-08-17T17:15:07.245 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:15:07.245 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.245 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.245 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:15:07.246 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.530835+0000", 2024-08-17T17:15:07.246 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.5499999999999996e-07 2024-08-17T17:15:07.246 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.246 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.246 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:15:07.246 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.530839+0000", 2024-08-17T17:15:07.246 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2230000000000001e-06 2024-08-17T17:15:07.246 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.246 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.246 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:15:07.246 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.530849+0000", 2024-08-17T17:15:07.246 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.5619999999999993e-06 2024-08-17T17:15:07.246 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.247 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.247 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:15:07.247 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.530853+0000", 2024-08-17T17:15:07.247 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.729e-06 2024-08-17T17:15:07.247 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.247 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.247 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.247 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.530926+0000", 2024-08-17T17:15:07.247 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2237000000000007e-05 2024-08-17T17:15:07.247 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.247 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.247 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:15:07.248 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:03.530940+0000", 2024-08-17T17:15:07.248 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4351e-05 2024-08-17T17:15:07.248 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.248 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.248 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:15:07.248 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.413368+0000", 2024-08-17T17:15:07.248 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.88242812999999998 2024-08-17T17:15:07.248 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.248 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.248 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:15:07.249 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.413494+0000", 2024-08-17T17:15:07.249 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012549300000000001 2024-08-17T17:15:07.249 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.249 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.249 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:15:07.249 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.413617+0000", 2024-08-17T17:15:07.249 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012310899999999999 2024-08-17T17:15:07.249 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.249 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.249 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:15:07.249 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.421282+0000", 2024-08-17T17:15:07.249 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0076658059999999998 2024-08-17T17:15:07.249 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.250 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.250 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.250 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.427312+0000", 2024-08-17T17:15:07.250 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0060294989999999998 2024-08-17T17:15:07.250 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.250 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.250 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:07.250 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.427986+0000", 2024-08-17T17:15:07.250 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00067377500000000005 2024-08-17T17:15:07.250 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.250 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.251 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:15:07.251 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.428080+0000", 2024-08-17T17:15:07.251 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4171000000000004e-05 2024-08-17T17:15:07.251 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:15:07.251 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:15:07.251 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:15:07.251 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:04.428124+0000", 2024-08-17T17:15:07.251 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4425999999999998e-05 2024-08-17T17:15:07.251 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.251 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.252 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.253 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:15:07.253 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:15:07.253 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:15:07.253 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:15:07.254 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:15:07.254 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:15:07.254 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:15:07.257 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:15:07.257 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:15:07.288 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:07.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:07.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:07.295 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:15:07.295 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:15:07.391 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:07.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:07.417 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:07.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:07.518 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:07.558 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:07.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:07.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:07.661 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:07.702 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:07.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:07.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:07 smithi063 ceph-mon[114756]: pgmap v1200: 89 pgs: 2 peering, 87 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 6.0 MiB/s wr, 705 op/s 2024-08-17T17:15:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:07 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2124218924' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:07.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:07 smithi063 ceph-mon[109843]: pgmap v1200: 89 pgs: 2 peering, 87 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 6.0 MiB/s wr, 705 op/s 2024-08-17T17:15:07.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:07 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2124218924' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:07.804 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:07.833 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:07.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:07.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:07.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:07 smithi162 ceph-mon[104668]: pgmap v1200: 89 pgs: 2 peering, 87 active+clean; 7.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 6.0 MiB/s wr, 705 op/s 2024-08-17T17:15:07.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:07 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2124218924' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:07.936 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:07.970 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:07.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:07.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:08.071 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:08.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:08.105 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:08.105 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:08.206 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:08.241 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:08.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:08.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:08.343 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:08.384 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:08.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:08.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:08.486 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:08.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:08.537 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:08.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:08.638 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:08.680 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:08.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:08.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:08.781 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:08.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:08.816 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:08.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:08.845 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:08.875 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:15:08.918 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:08.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:08.961 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:08.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:09.062 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:09.096 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:09.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:09.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:09.198 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:09.199 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:15:09.237 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:09.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:09.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:09.339 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:09.384 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:09.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:09.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:09.487 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:09.525 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:09.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:09.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:09.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:09 smithi162 ceph-mon[104668]: pgmap v1201: 89 pgs: 2 peering, 87 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 1.9 MiB/s wr, 927 op/s 2024-08-17T17:15:09.627 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:09.663 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:09.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:09.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:09.765 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:09.765 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:15:09.766 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:15:09.766 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:15:09.766 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 1.000000 2024-08-17T17:15:09.766 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 2 1 2024-08-17T17:15:09.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:09 smithi063 ceph-mon[109843]: pgmap v1201: 89 pgs: 2 peering, 87 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 1.9 MiB/s wr, 927 op/s 2024-08-17T17:15:09.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:09 smithi063 ceph-mon[114756]: pgmap v1201: 89 pgs: 2 peering, 87 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 1.9 MiB/s wr, 927 op/s 2024-08-17T17:15:09.815 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:09.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:09.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:09.917 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:09.950 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:09.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:09.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:10.051 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:10.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:10.106 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:10.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:10.207 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:10.233 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:10.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:10.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:10.307 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:10.335 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:10.395 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:10.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:10.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:10.409 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:15:10.461 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:10 smithi063 ceph-mon[109843]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:15:10.461 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:10 smithi063 ceph-mon[114756]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:15:10.497 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:10.525 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:10.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:10.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:10.627 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:10.660 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:10.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:10.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:10.762 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:10.789 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:10 smithi162 ceph-mon[104668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:15:10.805 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:10.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:10.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:10.909 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:10.953 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:10.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:10.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:11.056 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:11.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:11.113 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:11.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:11.214 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:11.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:11.248 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:11.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:11.252 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:15:11.252 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:15:11.349 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:11.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:11.396 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:11.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:11.497 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:11.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:11.537 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:11.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:11.626 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:11.638 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:11.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:11.701 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:11.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:11.761 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:11 smithi063 ceph-mon[109843]: pgmap v1202: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:15:11.761 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:11 smithi063 ceph-mon[114756]: pgmap v1202: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:15:11.802 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:11.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:11.837 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:11.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:11 smithi162 ceph-mon[104668]: pgmap v1202: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:15:11.938 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:11.979 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:11.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:11.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:12.081 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:12.109 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:12.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:12.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:12.211 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:12.262 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:12.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:12.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:12.356 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:15:12.364 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:12.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:12.425 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:12.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:12.462 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:15:12.527 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:12.580 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:12.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:12.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:12.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:12 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1119385046' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:12.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:12 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:12.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:12 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1119385046' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:12.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:12 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:12.614 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:12 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1119385046' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:12.614 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:12 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:12.656 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:15:12.683 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:12.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:12.716 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:12.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:12.817 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:12.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:12.849 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:12.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:12.950 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:12.989 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:12.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:12.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:13.091 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:13.134 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:13.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:13.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:13.236 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:13.274 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:13.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:13.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:13.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:13.415 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:13.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:13.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:13.467 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:15:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:15:13] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.51.0" 2024-08-17T17:15:13.517 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:13.552 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:13.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:13.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:13.654 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:13.681 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:13.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:13.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:13.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:13 smithi063 ceph-mon[114756]: pgmap v1203: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:15:13.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:13 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:15:13.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:13 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:15:13.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:13 smithi063 ceph-mon[114756]: osdmap e1105: 8 total, 8 up, 8 in 2024-08-17T17:15:13.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:13 smithi063 ceph-mon[109843]: pgmap v1203: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:15:13.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:13 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:15:13.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:13 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:15:13.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:13 smithi063 ceph-mon[109843]: osdmap e1105: 8 total, 8 up, 8 in 2024-08-17T17:15:13.783 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:13.810 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:13.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:13.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:13.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:13 smithi162 ceph-mon[104668]: pgmap v1203: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:15:13.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:13 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:15:13.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:13 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:15:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:13 smithi162 ceph-mon[104668]: osdmap e1105: 8 total, 8 up, 8 in 2024-08-17T17:15:13.912 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:13.956 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:13.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:13.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:14.058 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:14.086 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:14.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:14.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:14.188 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:14.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:14.216 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:14.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:14.317 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:14.350 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:14.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:14.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:14.452 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:14.482 INFO:teuthology.orchestra.run.smithi063.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-08-17T17:15:14.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:14.490 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:14.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:14.539 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:15:14.539 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:15:14.591 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:14.638 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:14.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:14.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:14.739 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:14.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:14 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2672300948' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:15:14.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:14 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2672300948' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:15:14.791 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:14.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:14.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:14.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:14 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2672300948' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:15:14.893 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:14.923 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:14.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:14.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:15.025 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:15.050 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:15.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:15.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:15.151 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:15.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:15.183 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:15.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:15.240 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:15:15.284 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:15.324 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:15.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:15.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:15.426 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:15.466 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:15.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:15.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:15.478 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:15 smithi162 ceph-mon[104668]: pgmap v1205: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:15:15.544 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:15:15.567 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:15.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:15.601 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:15.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:15.676 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:15 smithi063 ceph-mon[109843]: pgmap v1205: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:15:15.677 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:15 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2672300948' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:15:15.677 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:15 smithi063 ceph-mon[109843]: osdmap e1106: 8 total, 8 up, 8 in 2024-08-17T17:15:15.677 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:15 smithi063 ceph-mon[114756]: pgmap v1205: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:15:15.677 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:15 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2672300948' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:15:15.677 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:15 smithi063 ceph-mon[114756]: osdmap e1106: 8 total, 8 up, 8 in 2024-08-17T17:15:15.702 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:15.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:15.750 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:15.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:15.819 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:15 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2672300948' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:15:15.819 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:15 smithi162 ceph-mon[104668]: osdmap e1106: 8 total, 8 up, 8 in 2024-08-17T17:15:15.851 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:15.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:15.899 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:15.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:16.000 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:16.004 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:15:16.005 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:15:16.038 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:16.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:16.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:16.040 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:15:16.040 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:15:16.058 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:15:16.076 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:15:16.076 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:15:16.118 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:15:16.118 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:15:16.140 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:16.153 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:15:16.153 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:15:16.165 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:16.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:16.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:16.190 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:15:16.190 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:15:16.224 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:15:16.224 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:15:16.264 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:15:16.264 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:15:16.266 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:16.325 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:16.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:16.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:16.427 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:16.462 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:16.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:16.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:16.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:16 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/506999221' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:16.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:16 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:15:16.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:16 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/506999221' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:16 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:15:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:16.564 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:16.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:16.615 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:16.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:16.716 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:16.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:16.755 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:16.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:16.787 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:16 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/506999221' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:16.787 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:16 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:15:16.787 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:16.855 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:16.889 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:16.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:16.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:16.991 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:17.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:17.036 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:17.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:17.119 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:17.138 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:17.203 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:17.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:17.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:17.306 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:17.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:17.343 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:17.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:17.430 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:15:17.430 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:15:17.431 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:15:17.431 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:15:17.431 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.431 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.14817.0:6645 5.e 5:753e6f85:::smithi06387500-44:head [rollback 1d2] snapc 1d7=[1d6,1d5,1d4,1d3,1d2,1d0,1cf,1ce,1cd,1ca,1c8,1c7,1c2,1ba] ondisk+write+known_if_redirected+supports_pool_eio e739)", 2024-08-17T17:15:17.431 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:06:06.617038+0000", 2024-08-17T17:15:17.432 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 550.80896029799999, 2024-08-17T17:15:17.432 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0349813919999999, 2024-08-17T17:15:17.432 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.432 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:15:17.432 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.432 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.14817", 2024-08-17T17:15:17.433 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:15:17.433 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 6645 2024-08-17T17:15:17.433 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.433 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.433 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.433 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.433 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:06:06.617038+0000", 2024-08-17T17:15:17.434 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.434 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.434 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.434 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.434 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:06:06.617038+0000", 2024-08-17T17:15:17.434 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.435 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.435 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.435 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.435 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:06:06.617036+0000", 2024-08-17T17:15:17.435 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:15:17.435 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.436 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.436 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.436 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:06:06.617045+0000", 2024-08-17T17:15:17.436 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8980000000000002e-06 2024-08-17T17:15:17.436 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.436 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.437 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.437 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:06:06.617082+0000", 2024-08-17T17:15:17.437 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7308000000000003e-05 2024-08-17T17:15:17.437 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.437 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.437 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.438 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:06:06.617092+0000", 2024-08-17T17:15:17.438 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9010000000000006e-06 2024-08-17T17:15:17.438 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.438 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.438 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.438 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:06:06.618585+0000", 2024-08-17T17:15:17.439 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001493173 2024-08-17T17:15:17.439 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.439 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.439 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:17.439 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:06:06.618604+0000", 2024-08-17T17:15:17.440 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.8952000000000001e-05 2024-08-17T17:15:17.440 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.440 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.440 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.440 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:06:07.639105+0000", 2024-08-17T17:15:17.440 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.020501501 2024-08-17T17:15:17.440 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.441 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.441 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for degraded object", 2024-08-17T17:15:17.441 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:06:07.639470+0000", 2024-08-17T17:15:17.441 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00036437299999999998 2024-08-17T17:15:17.441 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.442 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.442 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.442 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:06:07.645321+0000", 2024-08-17T17:15:17.442 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0058506720000000003 2024-08-17T17:15:17.442 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.442 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.443 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.443 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:06:07.645409+0000", 2024-08-17T17:15:17.443 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8636999999999999e-05 2024-08-17T17:15:17.443 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.443 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.443 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.444 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:06:07.652019+0000", 2024-08-17T17:15:17.444 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0066100949999999999 2024-08-17T17:15:17.444 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.444 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.444 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.444 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.444 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.445 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.14817.0:7815 5.1 5:878aa1f0:::smithi06387500-48:head [read 0~3210401,checksum,omap-get-vals-by-keys,omap-get-keys,omap-get-vals,omap-get-header,getxattrs] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e837)", 2024-08-17T17:15:17.445 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:07:44.673241+0000", 2024-08-17T17:15:17.445 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 452.75275686200001, 2024-08-17T17:15:17.445 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0285751620000001, 2024-08-17T17:15:17.445 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.445 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:15:17.446 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.446 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.14817", 2024-08-17T17:15:17.446 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:15:17.446 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 7815 2024-08-17T17:15:17.446 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.447 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.447 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.447 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.447 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673241+0000", 2024-08-17T17:15:17.447 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.447 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.448 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.448 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.448 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673241+0000", 2024-08-17T17:15:17.448 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.448 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.448 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.449 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.449 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673240+0000", 2024-08-17T17:15:17.449 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:15:17.449 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.449 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.450 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.450 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673248+0000", 2024-08-17T17:15:17.450 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4460000000000001e-06 2024-08-17T17:15:17.450 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.450 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.450 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.451 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673263+0000", 2024-08-17T17:15:17.451 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5413999999999998e-05 2024-08-17T17:15:17.451 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.451 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.451 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.451 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673274+0000", 2024-08-17T17:15:17.451 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0848e-05 2024-08-17T17:15:17.452 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.452 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.452 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.452 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.674179+0000", 2024-08-17T17:15:17.452 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00090424099999999996 2024-08-17T17:15:17.452 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.453 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.453 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:17.453 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.674191+0000", 2024-08-17T17:15:17.453 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2340999999999999e-05 2024-08-17T17:15:17.453 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.453 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.454 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.454 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:45.695014+0000", 2024-08-17T17:15:17.454 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.020823109 2024-08-17T17:15:17.454 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.454 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.455 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.455 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:45.695147+0000", 2024-08-17T17:15:17.455 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013272699999999999 2024-08-17T17:15:17.455 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.455 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.455 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.455 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:45.701816+0000", 2024-08-17T17:15:17.456 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0066697670000000001 2024-08-17T17:15:17.456 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.456 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.456 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.456 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.456 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.457 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.14817.0:7816 5.1 5:878aa1f0:::smithi06387500-48:head [sparse-read 0~1850545] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e837)", 2024-08-17T17:15:17.457 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:07:44.673293+0000", 2024-08-17T17:15:17.457 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 452.75270562700001, 2024-08-17T17:15:17.457 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0290699409999999, 2024-08-17T17:15:17.457 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.458 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:15:17.458 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.458 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.14817", 2024-08-17T17:15:17.458 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:15:17.458 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 7816 2024-08-17T17:15:17.458 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.459 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.459 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.459 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.459 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673293+0000", 2024-08-17T17:15:17.459 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.459 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.460 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.460 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.460 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673293+0000", 2024-08-17T17:15:17.460 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.460 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.460 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.461 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.461 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673292+0000", 2024-08-17T17:15:17.461 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:15:17.461 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.461 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.461 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.462 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673296+0000", 2024-08-17T17:15:17.462 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1670000000000001e-06 2024-08-17T17:15:17.462 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.462 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.462 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.463 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:17.465 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673305+0000", 2024-08-17T17:15:17.465 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4740000000000005e-06 2024-08-17T17:15:17.465 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.466 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.466 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.466 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673311+0000", 2024-08-17T17:15:17.466 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8700000000000003e-06 2024-08-17T17:15:17.466 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.466 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.467 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.467 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.674215+0000", 2024-08-17T17:15:17.467 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00090389799999999998 2024-08-17T17:15:17.467 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.467 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.467 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:17.468 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.674224+0000", 2024-08-17T17:15:17.468 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6249999999999996e-06 2024-08-17T17:15:17.468 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.468 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.468 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.469 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:45.702119+0000", 2024-08-17T17:15:17.469 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0278950039999999 2024-08-17T17:15:17.469 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.469 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.469 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.469 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:45.702205+0000", 2024-08-17T17:15:17.469 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6379999999999996e-05 2024-08-17T17:15:17.470 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.470 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.470 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.470 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:45.702362+0000", 2024-08-17T17:15:17.470 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000157121 2024-08-17T17:15:17.471 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.471 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.471 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.471 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.471 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.471 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.14817.0:7817 5.1 5:878aa1f0:::smithi06387500-48:head [sparse-read 0~1850545] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e837)", 2024-08-17T17:15:17.472 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:07:44.673317+0000", 2024-08-17T17:15:17.472 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 452.75268067399998, 2024-08-17T17:15:17.472 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.02929263, 2024-08-17T17:15:17.472 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.473 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:15:17.473 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.473 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.14817", 2024-08-17T17:15:17.473 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:15:17.473 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 7817 2024-08-17T17:15:17.473 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.474 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.474 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.474 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.474 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673317+0000", 2024-08-17T17:15:17.474 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.474 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.475 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.475 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.475 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673317+0000", 2024-08-17T17:15:17.475 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.475 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.475 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.476 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.476 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673317+0000", 2024-08-17T17:15:17.476 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:15:17.476 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.476 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.477 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.477 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673320+0000", 2024-08-17T17:15:17.477 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6730000000000001e-06 2024-08-17T17:15:17.477 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.478 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.478 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.478 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673326+0000", 2024-08-17T17:15:17.478 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4810000000000003e-06 2024-08-17T17:15:17.479 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.479 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.479 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.481 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.673331+0000", 2024-08-17T17:15:17.481 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1590000000000001e-06 2024-08-17T17:15:17.482 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.482 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.482 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.482 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.674247+0000", 2024-08-17T17:15:17.482 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00091547600000000003 2024-08-17T17:15:17.482 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.482 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.482 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:17.482 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:44.674255+0000", 2024-08-17T17:15:17.482 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4440000000000008e-06 2024-08-17T17:15:17.482 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.482 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.483 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.483 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:45.702399+0000", 2024-08-17T17:15:17.483 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0281436509999999 2024-08-17T17:15:17.483 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.483 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.483 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.483 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:45.702458+0000", 2024-08-17T17:15:17.483 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9382999999999997e-05 2024-08-17T17:15:17.483 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.483 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.483 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.483 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:07:45.702610+0000", 2024-08-17T17:15:17.484 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015189000000000001 2024-08-17T17:15:17.484 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.484 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.484 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.484 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.484 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.484 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:29353 6.2 6:4762f47f:::benchmark_data_smithi063_160268_object29352:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:17.484 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.379378+0000", 2024-08-17T17:15:17.484 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 100.04662008299999, 2024-08-17T17:15:17.484 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.102636264, 2024-08-17T17:15:17.484 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.484 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:17.484 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.485 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:17.485 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:17.485 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 29353 2024-08-17T17:15:17.485 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.485 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.485 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.485 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.485 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379378+0000", 2024-08-17T17:15:17.485 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.485 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.485 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.485 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.486 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379378+0000", 2024-08-17T17:15:17.486 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.486 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.486 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.486 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.486 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379377+0000", 2024-08-17T17:15:17.486 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:17.486 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.486 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.486 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.486 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379441+0000", 2024-08-17T17:15:17.486 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4055999999999998e-05 2024-08-17T17:15:17.486 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.487 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.487 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.487 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379454+0000", 2024-08-17T17:15:17.487 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2724e-05 2024-08-17T17:15:17.487 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.487 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.487 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.487 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379459+0000", 2024-08-17T17:15:17.487 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1760000000000001e-06 2024-08-17T17:15:17.487 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.487 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.487 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.487 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379496+0000", 2024-08-17T17:15:17.488 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6964e-05 2024-08-17T17:15:17.488 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.488 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.488 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.488 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379605+0000", 2024-08-17T17:15:17.488 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010905599999999999 2024-08-17T17:15:17.488 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.488 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.488 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:17.488 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379736+0000", 2024-08-17T17:15:17.488 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013126699999999999 2024-08-17T17:15:17.488 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.488 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.489 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:17.489 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.380909+0000", 2024-08-17T17:15:17.489 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001172511 2024-08-17T17:15:17.489 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.489 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.489 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.489 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.382749+0000", 2024-08-17T17:15:17.489 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001840291 2024-08-17T17:15:17.489 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.489 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.489 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.489 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481915+0000", 2024-08-17T17:15:17.489 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.099166016999999995 2024-08-17T17:15:17.490 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.490 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.490 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:17.490 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481972+0000", 2024-08-17T17:15:17.490 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.694e-05 2024-08-17T17:15:17.490 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.490 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.490 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.490 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.482014+0000", 2024-08-17T17:15:17.490 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2472000000000003e-05 2024-08-17T17:15:17.490 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.490 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.491 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.491 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.491 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.491 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:29370 6.e 6:79482805:::benchmark_data_smithi063_160268_object29369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:17.491 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.386359+0000", 2024-08-17T17:15:17.491 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 100.039639003, 2024-08-17T17:15:17.491 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.097030557000000003, 2024-08-17T17:15:17.491 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.491 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:17.491 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.491 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:17.492 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:17.492 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 29370 2024-08-17T17:15:17.492 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.492 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.492 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.492 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.492 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386359+0000", 2024-08-17T17:15:17.492 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.492 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.492 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.492 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.492 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386359+0000", 2024-08-17T17:15:17.492 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.493 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.493 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.493 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.493 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386356+0000", 2024-08-17T17:15:17.493 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999967 2024-08-17T17:15:17.493 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.493 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.493 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.493 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386386+0000", 2024-08-17T17:15:17.493 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9947999999999999e-05 2024-08-17T17:15:17.493 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.493 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.494 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.494 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386398+0000", 2024-08-17T17:15:17.494 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2135e-05 2024-08-17T17:15:17.494 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.494 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.494 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.494 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386405+0000", 2024-08-17T17:15:17.494 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1459999999999999e-06 2024-08-17T17:15:17.494 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.494 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.494 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.494 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386478+0000", 2024-08-17T17:15:17.495 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2360000000000005e-05 2024-08-17T17:15:17.495 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.495 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.495 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.495 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386591+0000", 2024-08-17T17:15:17.495 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011355 2024-08-17T17:15:17.495 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.495 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.495 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:15:17.495 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386720+0000", 2024-08-17T17:15:17.495 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012937800000000001 2024-08-17T17:15:17.495 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.495 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.496 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:17.496 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481898+0000", 2024-08-17T17:15:17.496 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.095177169000000006 2024-08-17T17:15:17.496 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.496 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.496 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.496 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483085+0000", 2024-08-17T17:15:17.496 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0011874190000000001 2024-08-17T17:15:17.496 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.496 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.496 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.496 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483111+0000", 2024-08-17T17:15:17.496 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5942999999999999e-05 2024-08-17T17:15:17.497 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.497 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.497 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:17.497 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483351+0000", 2024-08-17T17:15:17.497 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00024028299999999999 2024-08-17T17:15:17.497 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.497 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.497 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.497 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483390+0000", 2024-08-17T17:15:17.497 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8439999999999998e-05 2024-08-17T17:15:17.497 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.497 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.497 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.498 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.498 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.498 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:29371 6.2 6:4ffb2a7f:::benchmark_data_smithi063_160268_object29370:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:17.498 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.386614+0000", 2024-08-17T17:15:17.498 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 100.03938370199999, 2024-08-17T17:15:17.498 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.099154596999999997, 2024-08-17T17:15:17.498 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.498 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:17.498 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.498 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:17.498 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:17.498 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 29371 2024-08-17T17:15:17.498 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.499 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.499 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.499 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.499 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386614+0000", 2024-08-17T17:15:17.499 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.499 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.499 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.499 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.499 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386614+0000", 2024-08-17T17:15:17.499 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.499 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.499 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.499 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.500 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386614+0000", 2024-08-17T17:15:17.500 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:17.500 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.500 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.500 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.500 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386660+0000", 2024-08-17T17:15:17.500 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5952999999999999e-05 2024-08-17T17:15:17.500 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.500 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.500 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.500 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386673+0000", 2024-08-17T17:15:17.500 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3614999999999999e-05 2024-08-17T17:15:17.501 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.501 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.501 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.501 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386680+0000", 2024-08-17T17:15:17.501 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5150000000000003e-06 2024-08-17T17:15:17.501 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.501 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.501 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.501 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386726+0000", 2024-08-17T17:15:17.501 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6020000000000003e-05 2024-08-17T17:15:17.501 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.501 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.501 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.502 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386849+0000", 2024-08-17T17:15:17.502 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012303699999999999 2024-08-17T17:15:17.502 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.502 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.502 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:17.502 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386935+0000", 2024-08-17T17:15:17.502 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6405999999999999e-05 2024-08-17T17:15:17.502 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.502 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.502 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.502 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.482051+0000", 2024-08-17T17:15:17.502 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.095116368000000007 2024-08-17T17:15:17.502 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.503 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.503 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.503 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483433+0000", 2024-08-17T17:15:17.503 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013810280000000001 2024-08-17T17:15:17.503 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.503 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.503 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:17.503 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.485669+0000", 2024-08-17T17:15:17.503 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002236848 2024-08-17T17:15:17.503 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.503 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.503 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:17.503 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.485731+0000", 2024-08-17T17:15:17.504 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1408000000000006e-05 2024-08-17T17:15:17.504 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.504 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.504 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.504 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.485769+0000", 2024-08-17T17:15:17.504 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8269000000000002e-05 2024-08-17T17:15:17.504 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.504 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.504 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.504 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.504 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.504 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop_reply(client.44609.0:29353 6.2 e1093/1086 ondisk, result = 0)", 2024-08-17T17:15:17.504 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.387682+0000", 2024-08-17T17:15:17.505 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 100.038316053, 2024-08-17T17:15:17.505 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.094339362999999996, 2024-08-17T17:15:17.505 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.505 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:15:17.505 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.505 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.505 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.505 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387682+0000", 2024-08-17T17:15:17.505 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.505 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.505 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.505 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.505 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387682+0000", 2024-08-17T17:15:17.506 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.506 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.506 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.506 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.506 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387681+0000", 2024-08-17T17:15:17.506 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:15:17.506 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.506 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.506 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.506 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387685+0000", 2024-08-17T17:15:17.506 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8709999999999999e-06 2024-08-17T17:15:17.506 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.507 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.507 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.507 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387699+0000", 2024-08-17T17:15:17.507 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4141999999999999e-05 2024-08-17T17:15:17.507 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.507 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.507 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.507 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387704+0000", 2024-08-17T17:15:17.507 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2370000000000003e-06 2024-08-17T17:15:17.507 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.507 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.507 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.507 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481898+0000", 2024-08-17T17:15:17.508 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.094193830000000006 2024-08-17T17:15:17.508 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.508 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.508 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.508 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481909+0000", 2024-08-17T17:15:17.508 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0992000000000001e-05 2024-08-17T17:15:17.508 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.508 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.508 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.508 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.482021+0000", 2024-08-17T17:15:17.508 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011291899999999999 2024-08-17T17:15:17.508 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.508 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.509 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.509 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.509 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.509 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.44609.0:29362 6.6 e1093/1086 6:6804913c:::benchmark_data_smithi063_160268_object29361:head v 1093'1824, mlcod=1093'1822)", 2024-08-17T17:15:17.509 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.387711+0000", 2024-08-17T17:15:17.509 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 100.038287598, 2024-08-17T17:15:17.509 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.095442569000000005, 2024-08-17T17:15:17.509 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.509 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:17.509 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.509 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.509 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.509 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387711+0000", 2024-08-17T17:15:17.509 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.510 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.510 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.510 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.510 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387711+0000", 2024-08-17T17:15:17.510 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.510 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.510 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.510 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.510 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387710+0000", 2024-08-17T17:15:17.511 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967296 2024-08-17T17:15:17.511 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.511 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.511 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.511 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387734+0000", 2024-08-17T17:15:17.511 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3649e-05 2024-08-17T17:15:17.511 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.511 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.511 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.511 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387738+0000", 2024-08-17T17:15:17.511 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4499999999999997e-06 2024-08-17T17:15:17.511 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.511 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.511 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.512 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387741+0000", 2024-08-17T17:15:17.512 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.7580000000000001e-06 2024-08-17T17:15:17.512 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.512 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.512 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.512 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387771+0000", 2024-08-17T17:15:17.512 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9403e-05 2024-08-17T17:15:17.512 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.512 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.512 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.512 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387790+0000", 2024-08-17T17:15:17.512 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9113000000000001e-05 2024-08-17T17:15:17.513 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.513 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.513 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:17.513 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483113+0000", 2024-08-17T17:15:17.513 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.095323518999999995 2024-08-17T17:15:17.513 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.513 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.513 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.513 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483153+0000", 2024-08-17T17:15:17.513 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9910000000000002e-05 2024-08-17T17:15:17.513 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.513 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.513 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.514 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.514 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.514 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.44609.0:29372 6.6 e1093/1086 6:6679e72a:::benchmark_data_smithi063_160268_object29371:head v 1093'1825, mlcod=1093'1822)", 2024-08-17T17:15:17.514 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.387747+0000", 2024-08-17T17:15:17.514 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 100.038251558, 2024-08-17T17:15:17.514 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.098332927000000001, 2024-08-17T17:15:17.514 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.514 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:17.514 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.514 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.514 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.514 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387747+0000", 2024-08-17T17:15:17.514 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.515 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.515 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.515 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.515 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387747+0000", 2024-08-17T17:15:17.515 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.515 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.515 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.515 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.515 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387746+0000", 2024-08-17T17:15:17.515 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:15:17.515 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.515 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.515 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.516 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387876+0000", 2024-08-17T17:15:17.516 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012919299999999999 2024-08-17T17:15:17.516 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.516 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.516 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.516 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387882+0000", 2024-08-17T17:15:17.516 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5189999999999998e-06 2024-08-17T17:15:17.516 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.516 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.516 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.516 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387886+0000", 2024-08-17T17:15:17.516 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.783e-06 2024-08-17T17:15:17.516 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.517 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.517 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.517 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481903+0000", 2024-08-17T17:15:17.517 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.094016684000000003 2024-08-17T17:15:17.517 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.517 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.517 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.517 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481926+0000", 2024-08-17T17:15:17.517 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3591000000000002e-05 2024-08-17T17:15:17.517 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.517 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.517 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:17.518 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.486048+0000", 2024-08-17T17:15:17.518 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.004122021 2024-08-17T17:15:17.518 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.518 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.518 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.518 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.486080+0000", 2024-08-17T17:15:17.518 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1386000000000001e-05 2024-08-17T17:15:17.518 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.518 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.518 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.518 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.518 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.519 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:29375 6.2 6:40783cdb:::benchmark_data_smithi063_160268_object29374:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:17.519 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.388464+0000", 2024-08-17T17:15:17.519 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 100.037534455, 2024-08-17T17:15:17.519 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.098667015999999996, 2024-08-17T17:15:17.519 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.519 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:17.519 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.519 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:17.519 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:17.519 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 29375 2024-08-17T17:15:17.519 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.519 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.519 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.520 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.520 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388464+0000", 2024-08-17T17:15:17.520 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.520 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.520 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.520 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.520 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388464+0000", 2024-08-17T17:15:17.520 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.520 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.520 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.520 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.520 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388463+0000", 2024-08-17T17:15:17.520 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:17.521 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.521 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.521 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.521 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388497+0000", 2024-08-17T17:15:17.521 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4789999999999997e-05 2024-08-17T17:15:17.521 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.521 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.521 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.521 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481860+0000", 2024-08-17T17:15:17.521 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.09336274 2024-08-17T17:15:17.521 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.521 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.522 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.522 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481878+0000", 2024-08-17T17:15:17.522 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7937000000000001e-05 2024-08-17T17:15:17.522 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.522 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.522 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.522 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.482077+0000", 2024-08-17T17:15:17.522 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000198514 2024-08-17T17:15:17.522 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.522 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.522 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.522 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.482222+0000", 2024-08-17T17:15:17.522 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014563999999999999 2024-08-17T17:15:17.523 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.523 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.523 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:17.523 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.482364+0000", 2024-08-17T17:15:17.523 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014209100000000001 2024-08-17T17:15:17.523 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.523 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.523 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.523 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.485816+0000", 2024-08-17T17:15:17.523 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003451951 2024-08-17T17:15:17.523 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.523 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.523 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.524 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.485866+0000", 2024-08-17T17:15:17.524 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9781000000000003e-05 2024-08-17T17:15:17.524 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.524 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.524 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:17.524 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.487051+0000", 2024-08-17T17:15:17.524 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0011850599999999999 2024-08-17T17:15:17.524 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.524 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.524 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:17.524 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.487096+0000", 2024-08-17T17:15:17.524 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5127999999999999e-05 2024-08-17T17:15:17.524 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.525 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.525 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.525 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.487131+0000", 2024-08-17T17:15:17.525 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4514999999999997e-05 2024-08-17T17:15:17.525 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.525 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.525 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.525 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.525 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.525 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop_reply(client.44609.0:29370 6.e e1093/1086 ondisk, result = 0)", 2024-08-17T17:15:17.525 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.388487+0000", 2024-08-17T17:15:17.525 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 100.037511642, 2024-08-17T17:15:17.525 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.094600769000000001, 2024-08-17T17:15:17.526 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.526 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:15:17.526 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.526 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.526 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.526 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388487+0000", 2024-08-17T17:15:17.526 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.526 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.526 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.526 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.526 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388487+0000", 2024-08-17T17:15:17.527 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.527 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.527 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.527 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.527 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388486+0000", 2024-08-17T17:15:17.527 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:15:17.527 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.527 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.527 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.527 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388490+0000", 2024-08-17T17:15:17.527 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.155e-06 2024-08-17T17:15:17.527 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.527 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.528 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.528 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481855+0000", 2024-08-17T17:15:17.528 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.093364427999999999 2024-08-17T17:15:17.528 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.528 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.528 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.528 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481878+0000", 2024-08-17T17:15:17.528 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3110999999999999e-05 2024-08-17T17:15:17.528 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.528 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.528 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.528 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483072+0000", 2024-08-17T17:15:17.528 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001194842 2024-08-17T17:15:17.529 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.529 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.529 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.529 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483079+0000", 2024-08-17T17:15:17.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9439999999999999e-06 2024-08-17T17:15:17.529 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.529 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.529 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.529 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483087+0000", 2024-08-17T17:15:17.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.875e-06 2024-08-17T17:15:17.529 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.529 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.530 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.530 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.530 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.530 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:32651 6.2 6:49c20ffc:::benchmark_data_smithi063_160268_object32650:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:17.530 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:39.132323+0000", 2024-08-17T17:15:17.530 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 98.293675042000004, 2024-08-17T17:15:17.530 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.086169089000000004, 2024-08-17T17:15:17.530 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.530 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:17.530 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.530 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:17.530 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:17.530 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 32651 2024-08-17T17:15:17.531 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.531 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.531 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.531 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.531 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132323+0000", 2024-08-17T17:15:17.531 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.531 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.531 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.531 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.531 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132323+0000", 2024-08-17T17:15:17.531 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.531 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.531 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.531 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132322+0000", 2024-08-17T17:15:17.532 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:15:17.532 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.532 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.532 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132385+0000", 2024-08-17T17:15:17.532 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2624000000000002e-05 2024-08-17T17:15:17.532 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.532 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.532 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132395+0000", 2024-08-17T17:15:17.532 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.6830000000000003e-06 2024-08-17T17:15:17.532 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.533 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.533 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.533 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132401+0000", 2024-08-17T17:15:17.533 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5209999999999999e-06 2024-08-17T17:15:17.533 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.533 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.533 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.533 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132474+0000", 2024-08-17T17:15:17.533 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2299999999999996e-05 2024-08-17T17:15:17.533 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.533 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.533 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.533 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132576+0000", 2024-08-17T17:15:17.534 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000102206 2024-08-17T17:15:17.534 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.534 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.534 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:17.534 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132686+0000", 2024-08-17T17:15:17.534 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000110332 2024-08-17T17:15:17.534 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.534 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.534 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:17.534 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.134043+0000", 2024-08-17T17:15:17.534 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013573909999999999 2024-08-17T17:15:17.534 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.535 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.535 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.535 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.136328+0000", 2024-08-17T17:15:17.535 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0022848600000000001 2024-08-17T17:15:17.535 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.535 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.535 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.535 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.218411+0000", 2024-08-17T17:15:17.535 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.082082811000000006 2024-08-17T17:15:17.535 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.535 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.535 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:17.535 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.218465+0000", 2024-08-17T17:15:17.536 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3538999999999997e-05 2024-08-17T17:15:17.536 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.536 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.536 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.536 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.218492+0000", 2024-08-17T17:15:17.536 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.7515999999999999e-05 2024-08-17T17:15:17.536 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.536 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.536 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.536 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.536 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.536 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:33172 6.2 6:40fd6398:::benchmark_data_smithi063_160268_object33171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:17.537 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:39.451372+0000", 2024-08-17T17:15:17.537 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 97.974626513000004, 2024-08-17T17:15:17.537 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.089318386999999999, 2024-08-17T17:15:17.537 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.537 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:17.537 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.537 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:17.537 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:17.537 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 33172 2024-08-17T17:15:17.537 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.537 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.537 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.537 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.537 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451372+0000", 2024-08-17T17:15:17.538 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.538 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.538 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.538 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.538 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451372+0000", 2024-08-17T17:15:17.538 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.538 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.538 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.538 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.538 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451370+0000", 2024-08-17T17:15:17.538 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:17.538 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.539 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.539 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.539 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451403+0000", 2024-08-17T17:15:17.539 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2957000000000003e-05 2024-08-17T17:15:17.539 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.539 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.539 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.539 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451416+0000", 2024-08-17T17:15:17.539 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2887e-05 2024-08-17T17:15:17.539 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.539 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.539 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.539 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451423+0000", 2024-08-17T17:15:17.540 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8020000000000003e-06 2024-08-17T17:15:17.540 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.540 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.540 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.540 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451492+0000", 2024-08-17T17:15:17.540 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9031999999999999e-05 2024-08-17T17:15:17.540 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.540 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.540 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.540 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451622+0000", 2024-08-17T17:15:17.540 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012985699999999999 2024-08-17T17:15:17.540 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.541 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:17.541 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451757+0000", 2024-08-17T17:15:17.541 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013562200000000001 2024-08-17T17:15:17.541 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.541 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:17.541 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453044+0000", 2024-08-17T17:15:17.541 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012871099999999999 2024-08-17T17:15:17.541 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.541 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.542 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.455396+0000", 2024-08-17T17:15:17.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023519919999999998 2024-08-17T17:15:17.542 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.542 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.542 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.542 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.540621+0000", 2024-08-17T17:15:17.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.085224464999999999 2024-08-17T17:15:17.542 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.542 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.542 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:17.542 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.540660+0000", 2024-08-17T17:15:17.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9440000000000002e-05 2024-08-17T17:15:17.543 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.543 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.543 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.543 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.540690+0000", 2024-08-17T17:15:17.543 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9708000000000001e-05 2024-08-17T17:15:17.543 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.543 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.543 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.543 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.543 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.543 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:33179 6.2 6:4d001853:::benchmark_data_smithi063_160268_object33178:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:17.544 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:39.453823+0000", 2024-08-17T17:15:17.544 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 97.972174668999997, 2024-08-17T17:15:17.544 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.089360489000000001, 2024-08-17T17:15:17.544 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.544 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:17.544 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.544 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:17.544 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:17.544 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 33179 2024-08-17T17:15:17.544 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.544 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.544 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.544 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.545 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453823+0000", 2024-08-17T17:15:17.545 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.545 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.545 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.545 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.545 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453823+0000", 2024-08-17T17:15:17.545 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.545 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.545 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.545 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.545 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453822+0000", 2024-08-17T17:15:17.545 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:17.546 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.546 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.546 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.546 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453916+0000", 2024-08-17T17:15:17.546 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4184000000000005e-05 2024-08-17T17:15:17.546 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.546 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.546 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.546 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453931+0000", 2024-08-17T17:15:17.546 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4997999999999999e-05 2024-08-17T17:15:17.546 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.546 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.547 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.547 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453939+0000", 2024-08-17T17:15:17.547 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4250000000000001e-06 2024-08-17T17:15:17.547 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.547 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.547 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.547 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453983+0000", 2024-08-17T17:15:17.547 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3804000000000003e-05 2024-08-17T17:15:17.547 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.547 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.547 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.548 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.454105+0000", 2024-08-17T17:15:17.548 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012191799999999999 2024-08-17T17:15:17.548 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.548 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.548 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:17.548 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.454241+0000", 2024-08-17T17:15:17.548 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000136396 2024-08-17T17:15:17.548 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.548 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.548 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.548 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.455812+0000", 2024-08-17T17:15:17.548 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001571277 2024-08-17T17:15:17.548 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.549 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:17.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.456213+0000", 2024-08-17T17:15:17.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00040079599999999998 2024-08-17T17:15:17.549 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.549 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.543091+0000", 2024-08-17T17:15:17.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.086878450999999995 2024-08-17T17:15:17.549 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.549 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:17.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.543147+0000", 2024-08-17T17:15:17.550 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5695000000000001e-05 2024-08-17T17:15:17.550 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.550 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.550 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.550 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.543184+0000", 2024-08-17T17:15:17.550 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6835000000000002e-05 2024-08-17T17:15:17.550 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.550 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.550 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.550 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.551 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.551 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:34214 6.2 6:4d7d9756:::benchmark_data_smithi063_160268_object34213:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:17.551 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.043114+0000", 2024-08-17T17:15:17.551 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 97.382884071000007, 2024-08-17T17:15:17.552 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.090588551000000003, 2024-08-17T17:15:17.552 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.552 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:17.552 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.552 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:17.553 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:17.553 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 34214 2024-08-17T17:15:17.553 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.553 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.553 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.553 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.553 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043114+0000", 2024-08-17T17:15:17.553 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.553 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.554 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.554 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.554 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043114+0000", 2024-08-17T17:15:17.554 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.554 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.554 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.554 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.554 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043112+0000", 2024-08-17T17:15:17.554 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:17.554 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.554 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.554 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.555 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043158+0000", 2024-08-17T17:15:17.555 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5021000000000001e-05 2024-08-17T17:15:17.555 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.555 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.555 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.555 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043177+0000", 2024-08-17T17:15:17.555 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9921000000000001e-05 2024-08-17T17:15:17.555 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.555 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.555 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043186+0000", 2024-08-17T17:15:17.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7110000000000001e-06 2024-08-17T17:15:17.556 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.556 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.556 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043238+0000", 2024-08-17T17:15:17.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2034999999999999e-05 2024-08-17T17:15:17.556 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.556 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.556 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043357+0000", 2024-08-17T17:15:17.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000119085 2024-08-17T17:15:17.556 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.557 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:17.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043534+0000", 2024-08-17T17:15:17.557 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017626400000000001 2024-08-17T17:15:17.557 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.557 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:17.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045175+0000", 2024-08-17T17:15:17.557 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016417070000000001 2024-08-17T17:15:17.557 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.557 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.558 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045757+0000", 2024-08-17T17:15:17.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00058215199999999995 2024-08-17T17:15:17.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.558 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.558 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.558 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.133592+0000", 2024-08-17T17:15:17.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.087834236999999996 2024-08-17T17:15:17.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.558 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.558 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:17.558 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.133660+0000", 2024-08-17T17:15:17.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8575e-05 2024-08-17T17:15:17.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.559 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.559 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.133703+0000", 2024-08-17T17:15:17.559 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2447999999999997e-05 2024-08-17T17:15:17.559 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.559 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.559 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.559 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.559 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.559 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:34221 6.2 6:4bfc0617:::benchmark_data_smithi063_160268_object34220:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:17.559 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.045750+0000", 2024-08-17T17:15:17.559 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 97.380248043999998, 2024-08-17T17:15:17.560 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.088048524000000003, 2024-08-17T17:15:17.560 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.560 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:17.560 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.560 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:17.560 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:17.560 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 34221 2024-08-17T17:15:17.560 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.560 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.560 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.560 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.560 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045750+0000", 2024-08-17T17:15:17.560 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.561 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.561 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.561 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.561 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045750+0000", 2024-08-17T17:15:17.561 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.561 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.561 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.561 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.561 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045748+0000", 2024-08-17T17:15:17.561 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:17.561 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.561 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.561 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.562 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045817+0000", 2024-08-17T17:15:17.562 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8992999999999995e-05 2024-08-17T17:15:17.562 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.562 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.562 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.562 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045831+0000", 2024-08-17T17:15:17.562 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3056e-05 2024-08-17T17:15:17.562 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.562 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.562 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.562 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045836+0000", 2024-08-17T17:15:17.562 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.699e-06 2024-08-17T17:15:17.562 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045888+0000", 2024-08-17T17:15:17.563 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1365e-05 2024-08-17T17:15:17.563 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.046026+0000", 2024-08-17T17:15:17.563 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013860000000000001 2024-08-17T17:15:17.563 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:17.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.046139+0000", 2024-08-17T17:15:17.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000113035 2024-08-17T17:15:17.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.564 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:17.564 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.047132+0000", 2024-08-17T17:15:17.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00099325299999999993 2024-08-17T17:15:17.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.564 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.564 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.047732+0000", 2024-08-17T17:15:17.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00059973199999999996 2024-08-17T17:15:17.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.133743+0000", 2024-08-17T17:15:17.565 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.086011140999999999 2024-08-17T17:15:17.565 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:17.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.133774+0000", 2024-08-17T17:15:17.565 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.0753000000000002e-05 2024-08-17T17:15:17.565 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.566 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.133799+0000", 2024-08-17T17:15:17.566 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4552000000000001e-05 2024-08-17T17:15:17.566 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.566 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.566 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.566 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.566 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.566 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:35807 6.2 6:425615ad:::benchmark_data_smithi063_160268_object35806:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:17.566 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.876489+0000", 2024-08-17T17:15:17.566 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 96.549509233999999, 2024-08-17T17:15:17.567 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14906064199999999, 2024-08-17T17:15:17.567 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.567 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:17.567 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.567 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:17.567 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:17.567 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 35807 2024-08-17T17:15:17.567 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.567 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.567 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.567 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.567 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876489+0000", 2024-08-17T17:15:17.567 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.568 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.568 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.568 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.568 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876489+0000", 2024-08-17T17:15:17.568 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.568 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.568 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.568 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.568 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876488+0000", 2024-08-17T17:15:17.569 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:17.569 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.569 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.569 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.569 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876522+0000", 2024-08-17T17:15:17.569 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4264999999999998e-05 2024-08-17T17:15:17.569 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.569 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.569 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.569 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876533+0000", 2024-08-17T17:15:17.569 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1549000000000001e-05 2024-08-17T17:15:17.570 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.570 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.570 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.570 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876540+0000", 2024-08-17T17:15:17.570 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6810000000000001e-06 2024-08-17T17:15:17.570 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.570 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.570 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.570 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876589+0000", 2024-08-17T17:15:17.570 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9308999999999998e-05 2024-08-17T17:15:17.570 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.571 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.571 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.571 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876713+0000", 2024-08-17T17:15:17.571 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012383300000000001 2024-08-17T17:15:17.571 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.571 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.571 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:17.571 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876896+0000", 2024-08-17T17:15:17.571 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000182466 2024-08-17T17:15:17.571 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.571 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.572 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:17.572 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.878495+0000", 2024-08-17T17:15:17.572 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00159953 2024-08-17T17:15:17.572 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.572 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.572 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.572 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.881641+0000", 2024-08-17T17:15:17.572 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0031456629999999999 2024-08-17T17:15:17.572 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.573 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.573 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.573 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025446+0000", 2024-08-17T17:15:17.573 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14380467199999999 2024-08-17T17:15:17.573 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.573 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.573 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:17.573 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025503+0000", 2024-08-17T17:15:17.573 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7846000000000001e-05 2024-08-17T17:15:17.573 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.573 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.573 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.573 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025550+0000", 2024-08-17T17:15:17.574 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6156999999999999e-05 2024-08-17T17:15:17.574 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.574 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.574 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.574 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.574 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.574 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:35813 6.2 6:4d5ac260:::benchmark_data_smithi063_160268_object35812:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:17.574 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.880079+0000", 2024-08-17T17:15:17.574 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 96.545919229000006, 2024-08-17T17:15:17.575 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.145731308, 2024-08-17T17:15:17.575 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.575 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:17.575 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.575 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:17.575 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:17.575 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 35813 2024-08-17T17:15:17.575 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.575 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.575 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.575 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.575 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880079+0000", 2024-08-17T17:15:17.575 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.576 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.576 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.576 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.576 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880079+0000", 2024-08-17T17:15:17.576 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.576 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.576 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.576 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.576 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880078+0000", 2024-08-17T17:15:17.576 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:17.576 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.577 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.577 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.577 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880133+0000", 2024-08-17T17:15:17.577 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.469e-05 2024-08-17T17:15:17.577 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.577 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.577 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.577 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880146+0000", 2024-08-17T17:15:17.577 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.314e-05 2024-08-17T17:15:17.577 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.577 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.577 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.577 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880151+0000", 2024-08-17T17:15:17.578 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2120000000000002e-06 2024-08-17T17:15:17.578 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.578 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.578 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.578 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880180+0000", 2024-08-17T17:15:17.578 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9244999999999999e-05 2024-08-17T17:15:17.578 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.578 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.578 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.578 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880304+0000", 2024-08-17T17:15:17.578 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012414699999999999 2024-08-17T17:15:17.578 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.579 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.579 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:17.579 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880443+0000", 2024-08-17T17:15:17.579 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013859699999999999 2024-08-17T17:15:17.579 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.579 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.579 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:17.579 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.881778+0000", 2024-08-17T17:15:17.579 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013348290000000001 2024-08-17T17:15:17.579 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.579 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.579 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.579 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.882544+0000", 2024-08-17T17:15:17.580 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00076663499999999999 2024-08-17T17:15:17.580 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.580 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.580 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.580 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025711+0000", 2024-08-17T17:15:17.580 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14316620999999999 2024-08-17T17:15:17.580 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.580 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.580 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:17.580 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025763+0000", 2024-08-17T17:15:17.580 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2191999999999998e-05 2024-08-17T17:15:17.580 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.580 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.581 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.581 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025810+0000", 2024-08-17T17:15:17.581 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7521000000000001e-05 2024-08-17T17:15:17.581 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.581 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.581 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.581 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.581 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.581 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:61718 6.2 6:46f0d56c:::benchmark_data_smithi063_160268_object61717:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-08-17T17:15:17.581 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:56.780446+0000", 2024-08-17T17:15:17.581 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 80.645551948000005, 2024-08-17T17:15:17.581 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.070249414999999996, 2024-08-17T17:15:17.581 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:17.582 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:17.582 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:17.582 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:17.582 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:17.582 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 61718 2024-08-17T17:15:17.582 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.582 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:17.582 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.582 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:17.582 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:56.780446+0000", 2024-08-17T17:15:17.582 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.582 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.582 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.583 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:17.583 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:56.780446+0000", 2024-08-17T17:15:17.583 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:17.583 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.583 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.583 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:17.583 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:56.780445+0000", 2024-08-17T17:15:17.583 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:17.583 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.583 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.583 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:17.583 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:56.780501+0000", 2024-08-17T17:15:17.583 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.571e-05 2024-08-17T17:15:17.584 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.584 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.584 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:17.584 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:56.780513+0000", 2024-08-17T17:15:17.584 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2088e-05 2024-08-17T17:15:17.584 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.584 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.584 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:17.584 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:56.780519+0000", 2024-08-17T17:15:17.584 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9020000000000004e-06 2024-08-17T17:15:17.584 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.585 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.585 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:17.585 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:56.780562+0000", 2024-08-17T17:15:17.585 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3120000000000001e-05 2024-08-17T17:15:17.585 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.585 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.585 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:17.585 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:56.780676+0000", 2024-08-17T17:15:17.585 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000114194 2024-08-17T17:15:17.585 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.586 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.586 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:17.586 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:56.780801+0000", 2024-08-17T17:15:17.586 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000125112 2024-08-17T17:15:17.586 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.586 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.586 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:17.586 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:56.782948+0000", 2024-08-17T17:15:17.586 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002146635 2024-08-17T17:15:17.586 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.586 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.586 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.586 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:56.784931+0000", 2024-08-17T17:15:17.587 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001983435 2024-08-17T17:15:17.587 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.587 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.587 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:17.587 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:56.850609+0000", 2024-08-17T17:15:17.587 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.065677581999999998 2024-08-17T17:15:17.587 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.587 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.587 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:17.588 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:56.850660+0000", 2024-08-17T17:15:17.588 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1529000000000001e-05 2024-08-17T17:15:17.588 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:17.588 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:17.588 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:17.588 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:56.850696+0000", 2024-08-17T17:15:17.588 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5301e-05 2024-08-17T17:15:17.588 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.588 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.588 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.588 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:17.588 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:17.588 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:15:17.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:17.590 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:17.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:17.627 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:17 smithi063 ceph-mon[114756]: pgmap v1207: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:15:17.628 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:17 smithi063 ceph-mon[109843]: pgmap v1207: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:15:17.692 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:17.724 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:17.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:17.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:17.826 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:17.839 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:17 smithi162 ceph-mon[104668]: pgmap v1207: 89 pgs: 89 active+clean; 6.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 0 B/s, 0 objects/s recovering 2024-08-17T17:15:17.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:17.872 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:17.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:17.973 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:18.003 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:18.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:18.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:18.105 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:18.131 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:18.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:18.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:18.232 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:18.272 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:18.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:18.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:18.365 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:15:18.374 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:18.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:18.446 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:18.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:18.547 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:18.587 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:18.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:18.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:18.689 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:18.718 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:15:18.725 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:18.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:18.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:18.828 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:18.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:18.860 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:18.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:18.961 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:19.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:19.004 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:19.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:19.071 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:15:19.105 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:19.130 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:19.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:19.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:19.232 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:19.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:19.285 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:19.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:19.386 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:19.450 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:19.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:19.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:19.453 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:19.518 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:15:19.552 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:19.581 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:19.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:19.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:19.683 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:19.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:19.721 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:19.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:19.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:19 smithi063 ceph-mon[114756]: pgmap v1208: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-08-17T17:15:19.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:19 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1904425981' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:15:19.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:19 smithi063 ceph-mon[109843]: pgmap v1208: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-08-17T17:15:19.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:19 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1904425981' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:15:19.822 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:19 smithi162 ceph-mon[104668]: pgmap v1208: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-08-17T17:15:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:19 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1904425981' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:15:19.853 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:19.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:19.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:19.955 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:20.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:20.005 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:20.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:20.106 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:20.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:20.147 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:20.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:20.248 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:20.284 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:20.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:20.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:20.386 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:20.421 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:20.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:20.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:20.525 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:20.559 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:20.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:20.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:20.645 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:20 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:15:20.645 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:20 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1904425981' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:15:20.645 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:20 smithi162 ceph-mon[104668]: osdmap e1107: 8 total, 8 up, 8 in 2024-08-17T17:15:20.646 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:15:20.646 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:15:20.646 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.646 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:146160 6.d 6:b2339646:::benchmark_data_smithi063_160268_object20053:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-08-17T17:15:20.646 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:20.633249+0000", 2024-08-17T17:15:20.646 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0099117319999999995, 2024-08-17T17:15:20.647 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0099421470000000001, 2024-08-17T17:15:20.647 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:20.647 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:15:20.647 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:20.647 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:20.647 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:20.647 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 146160 2024-08-17T17:15:20.647 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.647 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:20.647 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.647 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:20.647 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.633249+0000", 2024-08-17T17:15:20.648 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:20.648 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.648 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.648 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:20.648 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.633249+0000", 2024-08-17T17:15:20.648 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:20.648 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.648 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.648 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:20.648 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.633248+0000", 2024-08-17T17:15:20.648 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:20.648 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.648 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.649 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:20.649 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.633254+0000", 2024-08-17T17:15:20.649 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1190000000000002e-06 2024-08-17T17:15:20.649 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.649 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.649 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:20.649 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.633265+0000", 2024-08-17T17:15:20.649 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0552e-05 2024-08-17T17:15:20.649 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.649 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.649 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:20.649 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.633271+0000", 2024-08-17T17:15:20.649 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6969999999999996e-06 2024-08-17T17:15:20.650 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.650 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.650 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:20.650 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.633354+0000", 2024-08-17T17:15:20.650 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.2299999999999995e-05 2024-08-17T17:15:20.650 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.650 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.650 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:20.650 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.633466+0000", 2024-08-17T17:15:20.650 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000112744 2024-08-17T17:15:20.650 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.650 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.650 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:15:20.651 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.633573+0000", 2024-08-17T17:15:20.651 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010689299999999999 2024-08-17T17:15:20.651 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:20.651 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:20.651 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:20.651 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.651 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.651 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:146164 6.d 6:b90554d7:::benchmark_data_smithi063_160268_object20057:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-08-17T17:15:20.651 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:20.633450+0000", 2024-08-17T17:15:20.651 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0097113410000000001, 2024-08-17T17:15:20.652 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0098248929999999995, 2024-08-17T17:15:20.652 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:20.652 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:15:20.652 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:20.652 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:20.652 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:20.652 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 146164 2024-08-17T17:15:20.652 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.652 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:20.652 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.652 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:20.652 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.633450+0000", 2024-08-17T17:15:20.653 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:20.653 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.653 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.653 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:20.653 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.633450+0000", 2024-08-17T17:15:20.653 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:20.653 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.653 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.653 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:20.654 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.633449+0000", 2024-08-17T17:15:20.654 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:20.654 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.654 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.654 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:20.654 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.633454+0000", 2024-08-17T17:15:20.654 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6210000000000003e-06 2024-08-17T17:15:20.654 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.654 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.654 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:20.654 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.633476+0000", 2024-08-17T17:15:20.654 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2821000000000001e-05 2024-08-17T17:15:20.655 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.655 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.655 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:20.655 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.633484+0000", 2024-08-17T17:15:20.655 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.8939999999999993e-06 2024-08-17T17:15:20.655 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.655 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.655 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:20.655 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.642885+0000", 2024-08-17T17:15:20.655 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0094007369999999993 2024-08-17T17:15:20.655 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.655 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.655 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:20.656 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.643001+0000", 2024-08-17T17:15:20.656 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000116187 2024-08-17T17:15:20.656 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.656 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.656 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:15:20.656 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.643111+0000", 2024-08-17T17:15:20.656 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010987700000000001 2024-08-17T17:15:20.656 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:20.656 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:20.656 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:20.656 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.656 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.657 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:146169 6.7 6:e0a45105:::benchmark_data_smithi063_160268_object20062:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-08-17T17:15:20.657 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:20.634790+0000", 2024-08-17T17:15:20.657 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0083709219999999994, 2024-08-17T17:15:20.657 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0085595700000000007, 2024-08-17T17:15:20.657 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:20.657 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:15:20.657 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:20.657 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:20.657 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:20.657 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 146169 2024-08-17T17:15:20.657 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.657 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:20.657 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.658 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:20.658 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.634790+0000", 2024-08-17T17:15:20.658 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:20.658 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.658 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.658 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:20.658 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.634790+0000", 2024-08-17T17:15:20.658 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:20.658 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.658 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.658 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:20.658 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.634789+0000", 2024-08-17T17:15:20.658 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:20.659 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.659 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.659 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:20.659 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.634797+0000", 2024-08-17T17:15:20.659 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1779999999999997e-06 2024-08-17T17:15:20.659 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.659 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.659 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:20.659 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.634806+0000", 2024-08-17T17:15:20.659 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2809999999999997e-06 2024-08-17T17:15:20.659 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.659 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.659 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:20.660 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.634812+0000", 2024-08-17T17:15:20.660 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0789999999999997e-06 2024-08-17T17:15:20.660 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.660 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.660 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:20.660 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.634953+0000", 2024-08-17T17:15:20.660 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000141457 2024-08-17T17:15:20.660 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.660 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.660 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:20.661 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.635055+0000", 2024-08-17T17:15:20.661 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010146599999999999 2024-08-17T17:15:20.661 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.661 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.661 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:15:20.661 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.635150+0000", 2024-08-17T17:15:20.661 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4963999999999994e-05 2024-08-17T17:15:20.661 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.661 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.661 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:20.661 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.639829+0000", 2024-08-17T17:15:20.661 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.004679356 2024-08-17T17:15:20.661 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:20.662 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:20.662 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:20.662 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.662 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.662 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.44609.0:146170 6.4 e1107/1086 6:245516cd:::benchmark_data_smithi063_160268_object20063:head v 1107'9147, mlcod=1107'9145)", 2024-08-17T17:15:20.662 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:20.640037+0000", 2024-08-17T17:15:20.662 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0031239639999999999, 2024-08-17T17:15:20.662 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0033823730000000002, 2024-08-17T17:15:20.662 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:20.662 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:15:20.662 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:20.662 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.663 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:20.663 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640037+0000", 2024-08-17T17:15:20.663 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:20.663 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.663 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.663 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:20.663 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640037+0000", 2024-08-17T17:15:20.663 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:20.663 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.663 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.663 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:20.664 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640036+0000", 2024-08-17T17:15:20.664 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:20.664 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.664 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.664 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:20.664 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640045+0000", 2024-08-17T17:15:20.664 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7970000000000006e-06 2024-08-17T17:15:20.664 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.664 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.664 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:20.664 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640060+0000", 2024-08-17T17:15:20.664 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5423000000000001e-05 2024-08-17T17:15:20.665 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.665 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.665 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:20.665 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640067+0000", 2024-08-17T17:15:20.665 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4000000000000003e-06 2024-08-17T17:15:20.665 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.665 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.665 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:20.665 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640171+0000", 2024-08-17T17:15:20.665 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000103232 2024-08-17T17:15:20.665 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.665 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.666 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:20.666 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640190+0000", 2024-08-17T17:15:20.666 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9154e-05 2024-08-17T17:15:20.666 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:20.666 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:20.666 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:20.666 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.666 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.666 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop_reply(client.44609.0:146151 6.d e1107/1086)", 2024-08-17T17:15:20.666 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:20.640081+0000", 2024-08-17T17:15:20.666 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0030799120000000002, 2024-08-17T17:15:20.666 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0033874909999999999, 2024-08-17T17:15:20.667 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:20.667 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "queued for pg", 2024-08-17T17:15:20.667 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:20.667 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.667 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:20.667 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640081+0000", 2024-08-17T17:15:20.667 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:20.667 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.667 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.667 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:20.667 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640081+0000", 2024-08-17T17:15:20.667 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:20.668 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.668 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.668 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:20.668 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640081+0000", 2024-08-17T17:15:20.668 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:15:20.668 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.668 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.668 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:20.668 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640084+0000", 2024-08-17T17:15:20.668 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6170000000000001e-06 2024-08-17T17:15:20.668 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.668 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.669 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:20.669 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640089+0000", 2024-08-17T17:15:20.669 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4700000000000001e-06 2024-08-17T17:15:20.669 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.669 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.669 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:20.669 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640093+0000", 2024-08-17T17:15:20.669 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8430000000000003e-06 2024-08-17T17:15:20.669 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:20.669 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:20.669 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:20.670 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.670 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.670 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop_reply(client.44609.0:146160 6.d e1107/1086)", 2024-08-17T17:15:20.670 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:20.640105+0000", 2024-08-17T17:15:20.670 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0030562039999999999, 2024-08-17T17:15:20.670 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0034028249999999999, 2024-08-17T17:15:20.670 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:20.670 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "queued for pg", 2024-08-17T17:15:20.670 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:20.670 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.670 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:20.670 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640105+0000", 2024-08-17T17:15:20.670 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:20.671 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.671 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.671 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:20.671 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640105+0000", 2024-08-17T17:15:20.671 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:20.671 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.671 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.671 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:20.671 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640105+0000", 2024-08-17T17:15:20.671 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:15:20.671 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.671 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.671 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:20.672 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640107+0000", 2024-08-17T17:15:20.672 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2139999999999999e-06 2024-08-17T17:15:20.672 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.672 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.672 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:20.672 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640115+0000", 2024-08-17T17:15:20.672 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9189999999999999e-06 2024-08-17T17:15:20.672 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.672 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.672 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:20.672 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.640119+0000", 2024-08-17T17:15:20.672 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8190000000000002e-06 2024-08-17T17:15:20.672 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:20.673 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:20.673 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:20.673 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.673 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.673 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:146151 6.d 6:be5d4c27:::benchmark_data_smithi063_160268_object20044:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-08-17T17:15:20.673 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:20.622567+0000", 2024-08-17T17:15:20.673 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.020594110999999998, 2024-08-17T17:15:20.673 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.020985132, 2024-08-17T17:15:20.673 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:20.673 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:15:20.673 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:20.673 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:20.673 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:20.674 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 146151 2024-08-17T17:15:20.674 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.674 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:20.674 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.674 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:20.674 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.622567+0000", 2024-08-17T17:15:20.674 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:20.674 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.674 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.675 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:20.675 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.622567+0000", 2024-08-17T17:15:20.675 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:20.675 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.675 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.675 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:20.675 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.622566+0000", 2024-08-17T17:15:20.675 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:20.675 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.676 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.676 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:20.676 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.622571+0000", 2024-08-17T17:15:20.676 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3499999999999996e-06 2024-08-17T17:15:20.676 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.676 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.676 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:20.676 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.622583+0000", 2024-08-17T17:15:20.676 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2194999999999999e-05 2024-08-17T17:15:20.676 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.677 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.677 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:20.677 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.622590+0000", 2024-08-17T17:15:20.677 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3810000000000004e-06 2024-08-17T17:15:20.677 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.677 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.677 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:20.677 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.626005+0000", 2024-08-17T17:15:20.677 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0034152739999999998 2024-08-17T17:15:20.677 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.677 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.677 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:20.678 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.626120+0000", 2024-08-17T17:15:20.678 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000114567 2024-08-17T17:15:20.678 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.678 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.678 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:15:20.678 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.626224+0000", 2024-08-17T17:15:20.678 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000104762 2024-08-17T17:15:20.678 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:20.678 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:20.678 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:20.678 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:20.632986+0000", 2024-08-17T17:15:20.678 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0067616079999999997 2024-08-17T17:15:20.678 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:20.679 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:20.679 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:20.679 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:20.679 INFO:teuthology.orchestra.run.smithi162.stdout: ], 2024-08-17T17:15:20.679 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 7 2024-08-17T17:15:20.679 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:15:20.680 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:20.710 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:15:20.711 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:15:20.711 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-08-17T17:15:20.711 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:15:20.711 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:15:20.711 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-08-17T17:15:20.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:20.717 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:20.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:20.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:20 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:15:20.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:20 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1904425981' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:15:20.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:20 smithi063 ceph-mon[114756]: osdmap e1107: 8 total, 8 up, 8 in 2024-08-17T17:15:20.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:20 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:15:20.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:20 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1904425981' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:15:20.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:20 smithi063 ceph-mon[109843]: osdmap e1107: 8 total, 8 up, 8 in 2024-08-17T17:15:20.818 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:20.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:20.859 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:20.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:20.960 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:21.013 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:21.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:21.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:21.114 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:21.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:21.157 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:21.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:21.258 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:21.301 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:21.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:21.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:21.403 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:21.434 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:21.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:21.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:21.478 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:15:21.536 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:21.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:21.573 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:21.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:21.674 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:21.712 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:21.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:21.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:21.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:21 smithi063 ceph-mon[109843]: pgmap v1210: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2024-08-17T17:15:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:21 smithi063 ceph-mon[114756]: pgmap v1210: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2024-08-17T17:15:21.815 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:21.816 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:15:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:21 smithi162 ceph-mon[104668]: pgmap v1210: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2024-08-17T17:15:21.849 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:21.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:21.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:21.952 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:21.979 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:21.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:21.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:22.081 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:22.113 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:22.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:22.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:22.215 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:22.242 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:22.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:22.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:22.344 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:22.387 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:22.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:22.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:22.489 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:22.520 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:22.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:22.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:22.622 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:22.654 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:22.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:22.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:22.756 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:22.799 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:22.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:22.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:22.901 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:22.928 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:22.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:22.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:23.030 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:23.071 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:23.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:23.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:23.173 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:23.219 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:23.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:23.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:23.270 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:15:23.321 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:23.346 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:23.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:23.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:23.448 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:23.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:23.496 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:23.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:23.597 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:23.628 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:23.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:23.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:23.658 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:23.681 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:23 smithi162 ceph-mon[104668]: pgmap v1211: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2024-08-17T17:15:23.682 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:15:23.682 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:15:23.682 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:15:23.682 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:15:23.682 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:15:23.731 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:23.747 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:23 smithi063 ceph-mon[109843]: pgmap v1211: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2024-08-17T17:15:23.747 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:23 smithi063 ceph-mon[114756]: pgmap v1211: 89 pgs: 89 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2024-08-17T17:15:23.748 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:15:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:15:23] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.51.0" 2024-08-17T17:15:23.789 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:23.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:23.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:23.891 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:23.918 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:23.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:23.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:24.021 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:24.058 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:24.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:24.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:24.159 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:24.188 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:24.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:24.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:24.290 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:24.318 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:24.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:24.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:24.370 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:15:24.419 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:24.445 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:24.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:24.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:24.462 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:15:24.463 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:15:24.473 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:24.510 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:15:24.511 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:15:24.545 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:15:24.545 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:15:24.548 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:24.583 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:24.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:24.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:24.585 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:15:24.585 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:15:24.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:24 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/914502130' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:24.618 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:15:24.618 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:15:24.657 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:15:24.657 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:15:24.661 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:15:24.684 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:24.693 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:15:24.693 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:15:24.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:24.731 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:24.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:24.732 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:15:24.732 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:15:24.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:24 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/914502130' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:24.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:24 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/914502130' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:24.834 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:24.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:24.877 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:24.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:24.978 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:25.015 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:25.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:25.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:25.117 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:25.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:25.159 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:25.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:25.260 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:25.286 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:25.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:25.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:25.388 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:25.423 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:25.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:25.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:25.524 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:25.561 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:25.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:25.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:25.662 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:25.697 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:25.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:25.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:25.799 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:25.834 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:25.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:25.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:25.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:25 smithi162 ceph-mon[104668]: pgmap v1212: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 904 op/s 2024-08-17T17:15:25.936 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:25.956 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:25 smithi063 ceph-mon[114756]: pgmap v1212: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 904 op/s 2024-08-17T17:15:25.956 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:25 smithi063 ceph-mon[109843]: pgmap v1212: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 904 op/s 2024-08-17T17:15:25.966 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:25.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:25.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:26.067 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:26.253 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:26.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:26.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:26.354 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:26.379 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:26.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:26.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:26.481 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:26.522 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:26.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:26.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:26.624 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:26.653 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:26.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:26.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:26.754 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:26.789 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:26.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:26.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:26.805 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:15:26.806 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:15:26.891 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:26.930 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:26.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:26.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:27.032 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:27.064 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:27.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:27.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:27.166 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:27.196 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:27.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:27.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:27.298 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:27.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:27.330 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:27.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:27.431 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:27.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:27.473 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:27.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:27.485 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:15:27.574 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:27.615 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:27.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:27.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:27.717 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:27.760 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:27.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:27.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:27.769 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:27 smithi063 ceph-mon[109843]: pgmap v1213: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2024-08-17T17:15:27.770 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:27 smithi063 ceph-mon[114756]: pgmap v1213: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2024-08-17T17:15:27.805 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:15:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:27 smithi162 ceph-mon[104668]: pgmap v1213: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2024-08-17T17:15:27.861 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:27.890 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:27.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:27.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:27.992 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:28.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:28.032 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:28.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:28.133 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:28.167 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:28.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:28.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:28.202 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:15:28.202 INFO:teuthology.orchestra.run.smithi063.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:15:28.202 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:15:28.270 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:28.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:28.310 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:28.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:28.411 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:28.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:28.447 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:28.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:28.548 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:28.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:28.578 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:28.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:28.679 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:28.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:28.718 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:28.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:28.819 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:28.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:28.855 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:28.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:28.956 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:28.984 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:28.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:28.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:29.014 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:15:29.086 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:29.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:29.128 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:29.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:29.230 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:29.281 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:29.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:29.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:29.383 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:29.439 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:29.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:29.440 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:29.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:29.541 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:29.570 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:29.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:29.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:29.672 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:29.705 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:29.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:29.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:29.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:29 smithi063 ceph-mon[114756]: pgmap v1214: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2024-08-17T17:15:29.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:29 smithi063 ceph-mon[109843]: pgmap v1214: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2024-08-17T17:15:29.806 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:29.836 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:29.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:29.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:29.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:29 smithi162 ceph-mon[104668]: pgmap v1214: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 887 op/s 2024-08-17T17:15:29.938 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:29.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:29.985 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:29.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:30.086 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:30.088 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:15:30.088 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:15:30.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:30.127 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:30.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:30.228 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:30.276 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:30.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:30.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:30.378 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:30.405 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:30.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:30.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:30.506 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:30.556 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:30.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:30.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:30.658 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:30.700 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:30.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:30.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:30.787 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:15:30.802 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:30.838 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:30.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:30.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:30.940 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:30.971 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:30.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:30.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:31.073 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:31.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:31.109 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:31.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:31.112 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:15:31.210 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:31.263 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:31.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:31.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:31.365 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:31.413 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:31.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:31.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:31.515 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:31.552 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:31.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:31.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:31.568 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:15:31.653 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:31.685 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:31.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:31.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:31.689 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:31 smithi063 ceph-mon[109843]: pgmap v1215: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-08-17T17:15:31.689 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:31 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2700330087' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:31.689 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:31 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:31.689 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:31.689 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:31 smithi063 ceph-mon[114756]: pgmap v1215: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-08-17T17:15:31.689 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:31 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2700330087' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:31.689 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:31 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:31.689 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:31.787 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:31.826 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:31.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:31.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:31.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:31 smithi162 ceph-mon[104668]: pgmap v1215: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-08-17T17:15:31.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:31 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2700330087' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:31.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:31 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:31.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:31.928 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:31.968 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:31.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:31.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:32.070 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:32.099 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:32.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:32.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:32.201 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:32.205 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:15:32.205 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:15:32.235 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:32.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:32.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:32.257 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:15:32.257 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:15:32.293 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:15:32.293 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:15:32.338 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:32.349 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:15:32.349 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:15:32.364 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:32.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:32.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:32.379 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:15:32.379 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:15:32.422 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:15:32.423 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:15:32.454 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:15:32.454 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:15:32.466 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:32.489 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:15:32.489 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:15:32.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:32.502 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:32.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:32.603 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:32.635 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:32.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:32.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:32.737 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:32.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:32.769 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:32.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:32.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:32 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:15:32.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:32 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:15:32.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:32 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:15:32.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:32 smithi162 ceph-mon[104668]: osdmap e1108: 8 total, 8 up, 8 in 2024-08-17T17:15:32.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:32 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2183971429' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:32.870 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:32.888 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:32 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:15:32.888 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:32 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:15:32.888 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:32 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:15:32.888 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:32 smithi063 ceph-mon[109843]: osdmap e1108: 8 total, 8 up, 8 in 2024-08-17T17:15:32.888 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:32 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2183971429' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:32.888 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:32 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:15:32.888 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:32 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:15:32.888 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:32 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:15:32.888 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:32 smithi063 ceph-mon[114756]: osdmap e1108: 8 total, 8 up, 8 in 2024-08-17T17:15:32.889 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:32 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2183971429' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:32.899 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:32.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:32.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:33.001 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:33.037 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:33.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:33.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:33.138 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:33.170 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:33.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:33.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:33.273 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:33.309 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:33.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:33.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:33.411 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:33.450 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:33.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:33.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:33.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:15:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:15:33] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.51.0" 2024-08-17T17:15:33.551 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:33.587 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:33.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:33.589 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:33.689 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:33.724 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:33.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:33.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:33.825 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:33.827 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:15:33.827 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:15:33.827 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:15:33.827 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.000000 2024-08-17T17:15:33.827 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 1 0 2024-08-17T17:15:33.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:33 smithi162 ceph-mon[104668]: pgmap v1217: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2024-08-17T17:15:33.868 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:33.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:33.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:33.875 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:33 smithi063 ceph-mon[109843]: pgmap v1217: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2024-08-17T17:15:33.876 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:33 smithi063 ceph-mon[114756]: pgmap v1217: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2024-08-17T17:15:33.969 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:34.009 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:34.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:34.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:34.111 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:34.156 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:34.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:34.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:34.230 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:34.258 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:34.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:34.315 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:34.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:34.416 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:34.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:34.453 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:34.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:34.554 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:34.601 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:34.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:34.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:34.618 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:15:34.704 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:34.728 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:34.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:34.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:34.830 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:34.877 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:34.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:34.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:34.979 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:35.012 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:35.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:35.013 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:35.113 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:35.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:35.166 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:35.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:35.245 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:15:35.268 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:35.297 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:35.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:35.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:35.398 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:35.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:35.427 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:35.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:35.528 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:35.591 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:35.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:35.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:35.599 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:15:35.599 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:15:35.599 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:15:35.599 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:15:35.599 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:15:35.619 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:35.693 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:35.720 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:35.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:35.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:35.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:35 smithi063 ceph-mon[114756]: pgmap v1218: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2024-08-17T17:15:35.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:35 smithi063 ceph-mon[109843]: pgmap v1218: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2024-08-17T17:15:35.822 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:35.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:35 smithi162 ceph-mon[104668]: pgmap v1218: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2024-08-17T17:15:35.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:35.866 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:35.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:35.967 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:36.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:36.015 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:36.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:36.116 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:36.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:36.145 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:36.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:36.246 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:36.252 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:15:36.285 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:36.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:36.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:36.386 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:36.439 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:36.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:36.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:36.541 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:36.558 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:36.571 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:15:36.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:36.584 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:36.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:36.685 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:36.711 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:36.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:36.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:36.813 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:36.854 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:36.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:36.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:36.956 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:36.983 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:36.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:37.085 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:37.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:37.118 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:37.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:37.219 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:37.257 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:37.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:37.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:37.359 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:37.387 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:37.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:37.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:37.489 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:37.529 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:37.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:37.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:37.603 INFO:teuthology.orchestra.run.smithi063.stderr:set osd.1 primary-affinity to 0 (802) 2024-08-17T17:15:37.631 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:37.668 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:37.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:37.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:37.771 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:37.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:37 smithi063 ceph-mon[114756]: pgmap v1219: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-08-17T17:15:37.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:37 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1094374354' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 0.0} : dispatch 2024-08-17T17:15:37.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:37 smithi063 ceph-mon[109843]: pgmap v1219: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-08-17T17:15:37.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:37 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1094374354' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 0.0} : dispatch 2024-08-17T17:15:37.792 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:37.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:37.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:37.814 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:37 smithi162 ceph-mon[104668]: pgmap v1219: 89 pgs: 89 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-08-17T17:15:37.814 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:37 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1094374354' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 0.0} : dispatch 2024-08-17T17:15:37.894 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:37.918 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:37.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:37.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:38.020 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:38.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:38.041 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:38.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:38.142 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:38.165 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:38.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:38.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:38.267 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:38.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:38.291 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:38.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:38.392 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:38.418 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:38.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:38.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:38.519 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:38.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:38.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:38.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:38.648 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:38.680 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:38.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:38.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:38.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:38 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1094374354' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2024-08-17T17:15:38.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:38 smithi063 ceph-mon[109843]: osdmap e1109: 8 total, 8 up, 8 in 2024-08-17T17:15:38.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:38 smithi063 ceph-mon[109843]: osdmap e1110: 8 total, 8 up, 8 in 2024-08-17T17:15:38.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:38 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1094374354' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2024-08-17T17:15:38.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:38 smithi063 ceph-mon[114756]: osdmap e1109: 8 total, 8 up, 8 in 2024-08-17T17:15:38.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:38 smithi063 ceph-mon[114756]: osdmap e1110: 8 total, 8 up, 8 in 2024-08-17T17:15:38.782 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:38.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:38.822 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:38.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:38.923 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:38.940 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:38 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1094374354' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2024-08-17T17:15:38.940 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:38 smithi162 ceph-mon[104668]: osdmap e1109: 8 total, 8 up, 8 in 2024-08-17T17:15:38.940 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:38 smithi162 ceph-mon[104668]: osdmap e1110: 8 total, 8 up, 8 in 2024-08-17T17:15:38.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:38.966 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:38.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:39.067 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:39.101 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:39.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:39.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:39.204 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:39.234 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:15:39.234 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:15:39.234 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:15:39.234 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:15:39.234 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.234 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.44609.0:97577 6.5 e1099/1086 6:abf7670f:::benchmark_data_smithi063_160268_object97576:head v 1099'6121, mlcod=1099'6119)", 2024-08-17T17:15:39.234 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:14:26.416286+0000", 2024-08-17T17:15:39.234 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 72.813020210000005, 2024-08-17T17:15:39.234 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.071123277999999998, 2024-08-17T17:15:39.235 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.235 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.235 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.235 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.235 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.235 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.416286+0000", 2024-08-17T17:15:39.235 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.235 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.235 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.235 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.235 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.416286+0000", 2024-08-17T17:15:39.235 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.236 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.236 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.236 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.236 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.416285+0000", 2024-08-17T17:15:39.236 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:39.236 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.236 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.236 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.236 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.416312+0000", 2024-08-17T17:15:39.236 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6488000000000001e-05 2024-08-17T17:15:39.236 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.236 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.237 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.237 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.416323+0000", 2024-08-17T17:15:39.237 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1325000000000001e-05 2024-08-17T17:15:39.237 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.237 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.237 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.237 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.416331+0000", 2024-08-17T17:15:39.237 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4100000000000002e-06 2024-08-17T17:15:39.237 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.237 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.237 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.237 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.416395+0000", 2024-08-17T17:15:39.238 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4593999999999999e-05 2024-08-17T17:15:39.238 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.238 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.238 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.238 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.416415+0000", 2024-08-17T17:15:39.238 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9899e-05 2024-08-17T17:15:39.238 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.238 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.238 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.239 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.487370+0000", 2024-08-17T17:15:39.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.070954498000000005 2024-08-17T17:15:39.239 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.239 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.239 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.239 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.487410+0000", 2024-08-17T17:15:39.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9993000000000001e-05 2024-08-17T17:15:39.239 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.239 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.239 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.239 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.239 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.239 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.44609.0:97579 6.3 e1099/1086 6:ca996faf:::benchmark_data_smithi063_160268_object97578:head v 1099'6084, mlcod=1099'6080)", 2024-08-17T17:15:39.240 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:14:26.417272+0000", 2024-08-17T17:15:39.240 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 72.812034307000005, 2024-08-17T17:15:39.240 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.070200103, 2024-08-17T17:15:39.240 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.240 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.240 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.240 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.240 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.240 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.417272+0000", 2024-08-17T17:15:39.241 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.241 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.241 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.241 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.241 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.417272+0000", 2024-08-17T17:15:39.241 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.241 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.241 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.241 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.241 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.417272+0000", 2024-08-17T17:15:39.241 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:39.241 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.242 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.242 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.242 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.417299+0000", 2024-08-17T17:15:39.242 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.7742999999999998e-05 2024-08-17T17:15:39.242 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.242 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.242 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.242 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.417310+0000", 2024-08-17T17:15:39.242 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0298e-05 2024-08-17T17:15:39.242 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.242 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.242 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.242 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.417316+0000", 2024-08-17T17:15:39.243 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3690000000000003e-06 2024-08-17T17:15:39.243 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.243 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.243 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.243 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.417354+0000", 2024-08-17T17:15:39.243 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8241000000000002e-05 2024-08-17T17:15:39.243 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.243 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.243 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.243 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.417374+0000", 2024-08-17T17:15:39.243 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0239999999999999e-05 2024-08-17T17:15:39.243 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.243 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.244 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.244 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.487439+0000", 2024-08-17T17:15:39.244 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.070064834000000006 2024-08-17T17:15:39.244 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.244 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.244 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.244 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.487472+0000", 2024-08-17T17:15:39.244 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3124000000000003e-05 2024-08-17T17:15:39.244 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.244 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.244 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.244 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.244 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.244 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.44609.0:97581 6.a e1099/1086 6:5bc0b130:::benchmark_data_smithi063_160268_object97580:head v 1099'6149, mlcod=1099'6147)", 2024-08-17T17:15:39.245 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:14:26.419006+0000", 2024-08-17T17:15:39.245 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 72.810300497, 2024-08-17T17:15:39.245 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.077766874, 2024-08-17T17:15:39.245 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.245 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.245 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.245 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.245 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.245 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419006+0000", 2024-08-17T17:15:39.245 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.245 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.245 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.245 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.246 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419006+0000", 2024-08-17T17:15:39.246 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.246 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.246 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.246 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.246 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419005+0000", 2024-08-17T17:15:39.246 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:39.246 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.246 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.246 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.246 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419037+0000", 2024-08-17T17:15:39.246 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1760000000000001e-05 2024-08-17T17:15:39.246 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.247 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.247 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.247 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419050+0000", 2024-08-17T17:15:39.247 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3517e-05 2024-08-17T17:15:39.247 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.247 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.247 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.247 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419058+0000", 2024-08-17T17:15:39.247 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6020000000000002e-06 2024-08-17T17:15:39.247 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.247 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.247 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.247 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419100+0000", 2024-08-17T17:15:39.248 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1868e-05 2024-08-17T17:15:39.248 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.248 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.248 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.248 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419121+0000", 2024-08-17T17:15:39.248 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.1081999999999999e-05 2024-08-17T17:15:39.248 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.248 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.248 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.248 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.496717+0000", 2024-08-17T17:15:39.249 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.077596214999999996 2024-08-17T17:15:39.249 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.249 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.249 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.249 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.496773+0000", 2024-08-17T17:15:39.249 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5755999999999999e-05 2024-08-17T17:15:39.249 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.249 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.249 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.249 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.249 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.249 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.44609.0:97583 6.a e1099/1086 6:5a623a4e:::benchmark_data_smithi063_160268_object97582:head v 1099'6150, mlcod=1099'6147)", 2024-08-17T17:15:39.250 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:14:26.421947+0000", 2024-08-17T17:15:39.250 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 72.807359672999993, 2024-08-17T17:15:39.250 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.074887432000000004, 2024-08-17T17:15:39.250 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.250 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.250 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.250 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.250 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.250 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.421947+0000", 2024-08-17T17:15:39.250 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.250 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.250 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.251 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.251 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.421947+0000", 2024-08-17T17:15:39.251 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.251 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.251 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.251 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.251 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.421946+0000", 2024-08-17T17:15:39.251 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:39.251 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.251 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.251 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.251 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.421973+0000", 2024-08-17T17:15:39.251 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6738999999999999e-05 2024-08-17T17:15:39.252 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.252 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.252 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.252 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.487339+0000", 2024-08-17T17:15:39.252 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.065366558000000005 2024-08-17T17:15:39.252 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.252 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.252 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.252 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.487357+0000", 2024-08-17T17:15:39.252 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7680999999999999e-05 2024-08-17T17:15:39.252 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.252 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.252 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.253 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.489609+0000", 2024-08-17T17:15:39.253 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0022519990000000002 2024-08-17T17:15:39.253 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.253 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.253 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.253 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.489634+0000", 2024-08-17T17:15:39.253 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4640000000000001e-05 2024-08-17T17:15:39.253 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.253 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.253 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.253 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.496779+0000", 2024-08-17T17:15:39.253 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0071450990000000002 2024-08-17T17:15:39.253 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.254 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.254 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.254 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.496834+0000", 2024-08-17T17:15:39.254 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5643000000000002e-05 2024-08-17T17:15:39.254 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.254 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.254 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.254 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.254 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.254 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.44609.0:97592 6.1 e1099/1086 6:8ad02475:::benchmark_data_smithi063_160268_object97591:head v 1099'6095, mlcod=1099'6093)", 2024-08-17T17:15:39.254 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:14:26.426800+0000", 2024-08-17T17:15:39.254 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 72.802506808000004, 2024-08-17T17:15:39.254 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.069814447000000002, 2024-08-17T17:15:39.255 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.255 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.255 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.255 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.255 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.255 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.426800+0000", 2024-08-17T17:15:39.255 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.255 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.255 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.255 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.255 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.426800+0000", 2024-08-17T17:15:39.255 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.255 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.256 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.256 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.256 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.426798+0000", 2024-08-17T17:15:39.256 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:39.256 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.256 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.256 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.256 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.426845+0000", 2024-08-17T17:15:39.256 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6711000000000001e-05 2024-08-17T17:15:39.256 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.256 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.256 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.256 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.487342+0000", 2024-08-17T17:15:39.257 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.060497278000000002 2024-08-17T17:15:39.257 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.257 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.257 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.257 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.487357+0000", 2024-08-17T17:15:39.257 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4917e-05 2024-08-17T17:15:39.257 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.257 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.257 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.257 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.487413+0000", 2024-08-17T17:15:39.257 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5788000000000002e-05 2024-08-17T17:15:39.257 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.257 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.258 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.258 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.487435+0000", 2024-08-17T17:15:39.258 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2243e-05 2024-08-17T17:15:39.258 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.258 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.258 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.258 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.496538+0000", 2024-08-17T17:15:39.258 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0091027810000000008 2024-08-17T17:15:39.258 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.258 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.258 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.258 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.496614+0000", 2024-08-17T17:15:39.259 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6216000000000003e-05 2024-08-17T17:15:39.259 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.259 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.259 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.259 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.259 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.259 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.44609.0:131715 6.5 e1102/1086 6:a9f57834:::benchmark_data_smithi063_160268_object5608:head v 1102'8273, mlcod=1102'8269)", 2024-08-17T17:15:39.259 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.357907+0000", 2024-08-17T17:15:39.259 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 35.871399179000001, 2024-08-17T17:15:39.260 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.026755992999999999, 2024-08-17T17:15:39.260 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.260 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.260 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.260 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.260 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.260 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.357907+0000", 2024-08-17T17:15:39.260 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.260 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.260 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.260 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.260 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.357907+0000", 2024-08-17T17:15:39.261 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.261 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.261 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.261 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.261 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.357907+0000", 2024-08-17T17:15:39.261 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:15:39.261 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.261 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.261 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.261 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.357911+0000", 2024-08-17T17:15:39.261 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0029999999999996e-06 2024-08-17T17:15:39.261 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.262 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.262 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.262 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.357917+0000", 2024-08-17T17:15:39.262 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0379999999999996e-06 2024-08-17T17:15:39.262 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.262 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.262 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.262 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.357921+0000", 2024-08-17T17:15:39.262 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1010000000000002e-06 2024-08-17T17:15:39.262 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.262 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.262 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.262 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.366557+0000", 2024-08-17T17:15:39.263 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0086358040000000004 2024-08-17T17:15:39.263 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.263 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.263 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.263 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.366579+0000", 2024-08-17T17:15:39.263 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2044000000000001e-05 2024-08-17T17:15:39.263 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.263 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.263 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.263 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.384512+0000", 2024-08-17T17:15:39.263 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.017933106000000001 2024-08-17T17:15:39.263 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.263 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.264 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.264 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.384663+0000", 2024-08-17T17:15:39.264 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000151261 2024-08-17T17:15:39.264 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.264 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.264 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.264 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.264 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.264 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160939 6.1 6:893849fe:::benchmark_data_smithi063_160268_object34832:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:15:39.264 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.614043+0000", 2024-08-17T17:15:39.264 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 1.615263825, 2024-08-17T17:15:39.264 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0308899330000001, 2024-08-17T17:15:39.264 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.265 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.265 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:39.265 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:39.265 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:39.265 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160939 2024-08-17T17:15:39.265 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.265 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.265 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.265 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:39.267 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614043+0000", 2024-08-17T17:15:39.267 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.267 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.267 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.267 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.267 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614043+0000", 2024-08-17T17:15:39.267 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.267 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.267 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.268 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.268 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614041+0000", 2024-08-17T17:15:39.268 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:39.268 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.268 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.268 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.268 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614047+0000", 2024-08-17T17:15:39.268 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9440000000000002e-06 2024-08-17T17:15:39.268 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.268 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.268 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.268 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614057+0000", 2024-08-17T17:15:39.268 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3409999999999992e-06 2024-08-17T17:15:39.268 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.269 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.269 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.269 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614064+0000", 2024-08-17T17:15:39.269 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7190000000000001e-06 2024-08-17T17:15:39.269 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.269 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.269 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.269 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614117+0000", 2024-08-17T17:15:39.269 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2797000000000003e-05 2024-08-17T17:15:39.270 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.270 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.270 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:39.270 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614128+0000", 2024-08-17T17:15:39.270 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0891e-05 2024-08-17T17:15:39.270 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.270 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.270 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.270 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.631386+0000", 2024-08-17T17:15:39.270 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.017258346 2024-08-17T17:15:39.270 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.270 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.270 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.271 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.633791+0000", 2024-08-17T17:15:39.271 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0024042109999999998 2024-08-17T17:15:39.271 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.271 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.271 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:15:39.271 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.633951+0000", 2024-08-17T17:15:39.271 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016093800000000001 2024-08-17T17:15:39.271 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.271 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.271 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:39.271 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644732+0000", 2024-08-17T17:15:39.271 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010780998 2024-08-17T17:15:39.272 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.272 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.272 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.272 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644760+0000", 2024-08-17T17:15:39.272 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.7195e-05 2024-08-17T17:15:39.272 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.272 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.272 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.272 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644837+0000", 2024-08-17T17:15:39.272 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7256999999999996e-05 2024-08-17T17:15:39.272 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.272 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.272 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.273 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644891+0000", 2024-08-17T17:15:39.273 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4284999999999999e-05 2024-08-17T17:15:39.273 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.273 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.273 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.273 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644933+0000", 2024-08-17T17:15:39.273 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1493999999999999e-05 2024-08-17T17:15:39.273 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.273 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.273 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.273 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.273 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.274 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160958 6.1 6:804ff99b:::benchmark_data_smithi063_160268_object34851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:15:39.274 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.649687+0000", 2024-08-17T17:15:39.274 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 1.5796199770000001, 2024-08-17T17:15:39.274 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99790868700000002, 2024-08-17T17:15:39.274 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.274 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.274 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:39.274 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:39.274 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:39.275 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160958 2024-08-17T17:15:39.275 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.275 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.275 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.275 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.275 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649687+0000", 2024-08-17T17:15:39.275 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.275 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.275 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.275 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.275 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649687+0000", 2024-08-17T17:15:39.275 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.275 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.276 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.276 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.276 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649685+0000", 2024-08-17T17:15:39.276 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:15:39.276 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.276 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.276 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.276 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649695+0000", 2024-08-17T17:15:39.276 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9210000000000004e-06 2024-08-17T17:15:39.276 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.276 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.276 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.276 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649708+0000", 2024-08-17T17:15:39.277 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2947e-05 2024-08-17T17:15:39.277 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.277 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.277 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.277 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649715+0000", 2024-08-17T17:15:39.277 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9519999999999994e-06 2024-08-17T17:15:39.277 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.277 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.277 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.277 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649779+0000", 2024-08-17T17:15:39.277 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3683000000000007e-05 2024-08-17T17:15:39.277 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.277 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.278 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:39.278 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649792+0000", 2024-08-17T17:15:39.278 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3294e-05 2024-08-17T17:15:39.278 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.278 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.278 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.278 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.637682+0000", 2024-08-17T17:15:39.278 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98788912399999995 2024-08-17T17:15:39.278 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.278 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.278 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.278 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.637791+0000", 2024-08-17T17:15:39.279 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000109728 2024-08-17T17:15:39.279 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.279 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.279 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:15:39.279 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.637918+0000", 2024-08-17T17:15:39.279 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000126842 2024-08-17T17:15:39.279 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.279 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.279 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:39.279 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644787+0000", 2024-08-17T17:15:39.280 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0068684000000000002 2024-08-17T17:15:39.280 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.280 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.280 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.280 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647458+0000", 2024-08-17T17:15:39.280 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026717220000000001 2024-08-17T17:15:39.280 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.280 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.280 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.280 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647543+0000", 2024-08-17T17:15:39.280 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4790999999999995e-05 2024-08-17T17:15:39.280 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.281 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.281 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.281 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647572+0000", 2024-08-17T17:15:39.281 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8538999999999999e-05 2024-08-17T17:15:39.281 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.281 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.281 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.281 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647595+0000", 2024-08-17T17:15:39.281 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3648000000000001e-05 2024-08-17T17:15:39.281 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.281 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.281 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.281 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.282 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.282 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160968 6.1 6:8a3118ae:::benchmark_data_smithi063_160268_object34861:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:15:39.282 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.664096+0000", 2024-08-17T17:15:39.282 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 1.565210124, 2024-08-17T17:15:39.282 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99085020099999999, 2024-08-17T17:15:39.282 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.282 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.282 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:39.282 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:39.282 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:39.282 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160968 2024-08-17T17:15:39.282 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.282 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.283 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.283 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.283 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664096+0000", 2024-08-17T17:15:39.283 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.283 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.283 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.283 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.283 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664096+0000", 2024-08-17T17:15:39.283 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.283 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.283 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.283 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.284 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664095+0000", 2024-08-17T17:15:39.284 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:39.284 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.284 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.284 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.284 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664100+0000", 2024-08-17T17:15:39.284 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0880000000000002e-06 2024-08-17T17:15:39.284 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.284 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.284 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.284 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664112+0000", 2024-08-17T17:15:39.284 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1231e-05 2024-08-17T17:15:39.284 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.285 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.285 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.285 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664117+0000", 2024-08-17T17:15:39.285 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4829999999999999e-06 2024-08-17T17:15:39.285 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.285 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.285 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.285 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664174+0000", 2024-08-17T17:15:39.285 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6443999999999997e-05 2024-08-17T17:15:39.285 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.285 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.285 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:39.286 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664186+0000", 2024-08-17T17:15:39.286 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2500999999999999e-05 2024-08-17T17:15:39.286 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.286 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.286 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.286 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.641442+0000", 2024-08-17T17:15:39.286 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97725559900000003 2024-08-17T17:15:39.286 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.286 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.286 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.286 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.641557+0000", 2024-08-17T17:15:39.286 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000115661 2024-08-17T17:15:39.286 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.287 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.287 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:15:39.287 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.641675+0000", 2024-08-17T17:15:39.287 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011779 2024-08-17T17:15:39.287 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.287 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.287 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:39.287 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647462+0000", 2024-08-17T17:15:39.287 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0057873380000000004 2024-08-17T17:15:39.287 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.287 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.287 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.288 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647507+0000", 2024-08-17T17:15:39.288 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4776e-05 2024-08-17T17:15:39.288 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.288 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.288 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.288 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654882+0000", 2024-08-17T17:15:39.288 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0073748959999999997 2024-08-17T17:15:39.288 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.288 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.288 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.288 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654915+0000", 2024-08-17T17:15:39.288 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3333000000000001e-05 2024-08-17T17:15:39.288 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.289 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.289 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.289 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654947+0000", 2024-08-17T17:15:39.289 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1192999999999998e-05 2024-08-17T17:15:39.289 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.289 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.289 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.289 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.289 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.289 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160969 6.1 6:85b88efb:::benchmark_data_smithi063_160268_object34862:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:15:39.289 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.664210+0000", 2024-08-17T17:15:39.290 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 1.5650965889999999, 2024-08-17T17:15:39.290 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99084335700000004, 2024-08-17T17:15:39.290 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.290 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.290 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:39.290 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:39.290 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:39.290 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160969 2024-08-17T17:15:39.290 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.290 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.291 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.291 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.291 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664210+0000", 2024-08-17T17:15:39.291 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.291 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.291 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.291 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.291 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664210+0000", 2024-08-17T17:15:39.291 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.291 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.291 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.291 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.291 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664209+0000", 2024-08-17T17:15:39.292 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:39.292 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.292 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.292 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.292 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664215+0000", 2024-08-17T17:15:39.292 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8619999999999999e-06 2024-08-17T17:15:39.292 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.292 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.292 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.292 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664227+0000", 2024-08-17T17:15:39.292 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.295e-05 2024-08-17T17:15:39.292 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.292 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.293 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.293 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664234+0000", 2024-08-17T17:15:39.293 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8399999999999997e-06 2024-08-17T17:15:39.293 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.293 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.293 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.293 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664286+0000", 2024-08-17T17:15:39.293 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1928000000000002e-05 2024-08-17T17:15:39.293 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.293 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.293 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:39.293 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664300+0000", 2024-08-17T17:15:39.294 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3497e-05 2024-08-17T17:15:39.294 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.294 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.294 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.294 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644983+0000", 2024-08-17T17:15:39.294 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98068294099999997 2024-08-17T17:15:39.294 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.294 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.294 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.294 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.645050+0000", 2024-08-17T17:15:39.294 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7012000000000001e-05 2024-08-17T17:15:39.295 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.295 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.295 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:15:39.295 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.645116+0000", 2024-08-17T17:15:39.295 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6268999999999999e-05 2024-08-17T17:15:39.295 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.295 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.295 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:39.295 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654779+0000", 2024-08-17T17:15:39.295 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0096628619999999995 2024-08-17T17:15:39.295 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.295 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.295 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.296 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654813+0000", 2024-08-17T17:15:39.296 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4489000000000001e-05 2024-08-17T17:15:39.296 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.296 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.296 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.296 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654995+0000", 2024-08-17T17:15:39.296 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000181478 2024-08-17T17:15:39.296 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.296 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.296 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.296 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655021+0000", 2024-08-17T17:15:39.296 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6390999999999999e-05 2024-08-17T17:15:39.297 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.297 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.297 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.297 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655053+0000", 2024-08-17T17:15:39.297 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2147999999999998e-05 2024-08-17T17:15:39.297 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.297 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.297 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.297 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.297 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.297 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160976 6.1 6:892f13c5:::benchmark_data_smithi063_160268_object34869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:15:39.297 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.670369+0000", 2024-08-17T17:15:39.297 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 1.55893766, 2024-08-17T17:15:39.298 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99685337699999998, 2024-08-17T17:15:39.298 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.298 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.298 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:39.298 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:39.298 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:39.298 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160976 2024-08-17T17:15:39.298 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.298 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.298 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.298 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.298 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670369+0000", 2024-08-17T17:15:39.299 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.299 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.299 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.299 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.299 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670369+0000", 2024-08-17T17:15:39.299 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.299 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.299 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.299 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.299 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670367+0000", 2024-08-17T17:15:39.299 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:39.299 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.300 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.300 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.300 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670374+0000", 2024-08-17T17:15:39.300 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.827e-06 2024-08-17T17:15:39.300 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.300 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.300 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.300 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670390+0000", 2024-08-17T17:15:39.300 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5566e-05 2024-08-17T17:15:39.300 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.300 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.300 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.300 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670398+0000", 2024-08-17T17:15:39.301 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3289999999999994e-06 2024-08-17T17:15:39.301 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.301 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.301 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.301 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670470+0000", 2024-08-17T17:15:39.301 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1913000000000001e-05 2024-08-17T17:15:39.301 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.301 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.301 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:39.301 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670482+0000", 2024-08-17T17:15:39.301 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2018000000000001e-05 2024-08-17T17:15:39.301 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.302 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.302 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.302 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647627+0000", 2024-08-17T17:15:39.302 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.977145139 2024-08-17T17:15:39.302 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.302 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.302 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.302 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647700+0000", 2024-08-17T17:15:39.302 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2495000000000003e-05 2024-08-17T17:15:39.302 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.302 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.302 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:15:39.303 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647765+0000", 2024-08-17T17:15:39.303 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5495999999999998e-05 2024-08-17T17:15:39.303 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.303 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.303 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.303 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654846+0000", 2024-08-17T17:15:39.303 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0070811249999999997 2024-08-17T17:15:39.303 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.303 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.303 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:39.303 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654853+0000", 2024-08-17T17:15:39.303 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2929999999999997e-06 2024-08-17T17:15:39.303 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.304 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.304 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.304 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667120+0000", 2024-08-17T17:15:39.304 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012267867 2024-08-17T17:15:39.304 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.304 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.304 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.304 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667180+0000", 2024-08-17T17:15:39.304 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9376999999999997e-05 2024-08-17T17:15:39.304 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.304 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.304 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.304 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667222+0000", 2024-08-17T17:15:39.305 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2451999999999999e-05 2024-08-17T17:15:39.305 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.305 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.305 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.305 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.305 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.305 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160983 6.1 6:82d2572f:::benchmark_data_smithi063_160268_object34876:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:15:39.305 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.676198+0000", 2024-08-17T17:15:39.305 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 1.553108559, 2024-08-17T17:15:39.305 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99119904999999997, 2024-08-17T17:15:39.305 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.305 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.305 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:39.306 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:39.306 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:39.306 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160983 2024-08-17T17:15:39.306 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.306 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.306 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.306 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.306 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676198+0000", 2024-08-17T17:15:39.306 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.306 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.306 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.306 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.306 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676198+0000", 2024-08-17T17:15:39.307 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.307 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.307 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.307 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.307 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676196+0000", 2024-08-17T17:15:39.307 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:15:39.307 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.307 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.307 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.307 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676205+0000", 2024-08-17T17:15:39.307 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3079999999999997e-06 2024-08-17T17:15:39.307 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.307 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.308 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.308 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676223+0000", 2024-08-17T17:15:39.308 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7217000000000001e-05 2024-08-17T17:15:39.308 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.308 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.308 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.308 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676230+0000", 2024-08-17T17:15:39.308 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3479999999999998e-06 2024-08-17T17:15:39.308 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.308 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.308 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.308 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676293+0000", 2024-08-17T17:15:39.309 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2714999999999998e-05 2024-08-17T17:15:39.309 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.309 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.309 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:39.309 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676305+0000", 2024-08-17T17:15:39.309 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2455e-05 2024-08-17T17:15:39.309 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.309 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.309 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.309 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.651451+0000", 2024-08-17T17:15:39.309 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97514584400000004 2024-08-17T17:15:39.309 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.309 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.310 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.310 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.651523+0000", 2024-08-17T17:15:39.310 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2497999999999996e-05 2024-08-17T17:15:39.310 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.310 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.310 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:15:39.310 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.651598+0000", 2024-08-17T17:15:39.310 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4241999999999998e-05 2024-08-17T17:15:39.310 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.310 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.310 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:39.310 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659221+0000", 2024-08-17T17:15:39.311 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0076229289999999996 2024-08-17T17:15:39.311 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.311 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.311 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.311 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667006+0000", 2024-08-17T17:15:39.311 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00778548 2024-08-17T17:15:39.311 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.311 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.311 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.311 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667303+0000", 2024-08-17T17:15:39.311 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000297103 2024-08-17T17:15:39.311 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.311 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.312 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.312 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667362+0000", 2024-08-17T17:15:39.312 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8705000000000001e-05 2024-08-17T17:15:39.312 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.312 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.312 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.312 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667397+0000", 2024-08-17T17:15:39.312 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5120999999999998e-05 2024-08-17T17:15:39.312 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.312 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.312 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.312 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.312 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.313 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160989 6.1 6:8559d498:::benchmark_data_smithi063_160268_object34882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:15:39.313 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.684830+0000", 2024-08-17T17:15:39.313 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 1.5444765949999999, 2024-08-17T17:15:39.313 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98270078400000005, 2024-08-17T17:15:39.313 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.313 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.313 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:39.313 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:39.313 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:39.313 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160989 2024-08-17T17:15:39.313 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.313 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.313 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.314 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.314 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684830+0000", 2024-08-17T17:15:39.314 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.314 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.314 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.314 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.314 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684830+0000", 2024-08-17T17:15:39.314 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.314 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.314 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.314 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.315 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684828+0000", 2024-08-17T17:15:39.315 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:39.315 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.315 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.315 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.315 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684835+0000", 2024-08-17T17:15:39.315 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9890000000000003e-06 2024-08-17T17:15:39.315 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.315 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.315 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.315 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684851+0000", 2024-08-17T17:15:39.315 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5457000000000001e-05 2024-08-17T17:15:39.315 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.316 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.316 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.316 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684863+0000", 2024-08-17T17:15:39.316 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2302e-05 2024-08-17T17:15:39.316 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.316 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.316 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.316 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684941+0000", 2024-08-17T17:15:39.316 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7541999999999997e-05 2024-08-17T17:15:39.316 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.316 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.316 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:39.316 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684956+0000", 2024-08-17T17:15:39.317 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5148e-05 2024-08-17T17:15:39.317 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.317 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.317 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.317 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655087+0000", 2024-08-17T17:15:39.317 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97013142799999996 2024-08-17T17:15:39.317 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.317 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.317 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.317 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655187+0000", 2024-08-17T17:15:39.317 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9229999999999997e-05 2024-08-17T17:15:39.317 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.317 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.318 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:15:39.318 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655293+0000", 2024-08-17T17:15:39.318 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000106576 2024-08-17T17:15:39.318 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.318 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.318 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:39.318 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667235+0000", 2024-08-17T17:15:39.318 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011942026 2024-08-17T17:15:39.318 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.318 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.318 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.318 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667269+0000", 2024-08-17T17:15:39.318 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.417e-05 2024-08-17T17:15:39.319 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.319 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.319 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.319 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667449+0000", 2024-08-17T17:15:39.319 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000179292 2024-08-17T17:15:39.319 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.319 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.319 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.319 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667498+0000", 2024-08-17T17:15:39.319 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9821999999999998e-05 2024-08-17T17:15:39.319 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.319 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.320 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.320 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667531+0000", 2024-08-17T17:15:39.320 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2326999999999997e-05 2024-08-17T17:15:39.320 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.320 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.320 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.320 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.320 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.320 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160996 6.1 6:8ccbb0f4:::benchmark_data_smithi063_160268_object34889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:15:39.320 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.691144+0000", 2024-08-17T17:15:39.320 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 1.5381621249999999, 2024-08-17T17:15:39.320 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98531105200000002, 2024-08-17T17:15:39.320 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.321 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.321 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:39.321 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:39.321 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:39.321 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160996 2024-08-17T17:15:39.321 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.321 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.321 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.321 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.321 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691144+0000", 2024-08-17T17:15:39.321 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.321 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.322 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.322 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.322 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691144+0000", 2024-08-17T17:15:39.322 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.322 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.322 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.322 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.322 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691143+0000", 2024-08-17T17:15:39.322 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:39.325 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.326 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.326 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.326 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691149+0000", 2024-08-17T17:15:39.326 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0800000000000002e-06 2024-08-17T17:15:39.326 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.326 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.326 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.326 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691162+0000", 2024-08-17T17:15:39.326 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.322e-05 2024-08-17T17:15:39.327 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.327 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.327 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.327 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691168+0000", 2024-08-17T17:15:39.327 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0260000000000004e-06 2024-08-17T17:15:39.327 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.327 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.327 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.327 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691211+0000", 2024-08-17T17:15:39.327 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3013999999999998e-05 2024-08-17T17:15:39.327 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.327 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.328 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:39.328 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691220+0000", 2024-08-17T17:15:39.328 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8540000000000003e-06 2024-08-17T17:15:39.328 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.328 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.328 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.328 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659248+0000", 2024-08-17T17:15:39.328 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.96802809099999998 2024-08-17T17:15:39.328 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.328 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.328 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.328 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659357+0000", 2024-08-17T17:15:39.329 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010854 2024-08-17T17:15:39.329 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.329 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.329 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:15:39.329 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659466+0000", 2024-08-17T17:15:39.329 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000109437 2024-08-17T17:15:39.329 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.329 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.329 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:39.329 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667552+0000", 2024-08-17T17:15:39.329 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0080855279999999998 2024-08-17T17:15:39.329 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.329 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.330 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.330 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676310+0000", 2024-08-17T17:15:39.330 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0087583250000000008 2024-08-17T17:15:39.330 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.330 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.330 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.330 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676357+0000", 2024-08-17T17:15:39.330 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6786000000000002e-05 2024-08-17T17:15:39.330 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.330 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.330 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.330 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676409+0000", 2024-08-17T17:15:39.330 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2104000000000001e-05 2024-08-17T17:15:39.331 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.331 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.331 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.331 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676455+0000", 2024-08-17T17:15:39.331 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6362000000000001e-05 2024-08-17T17:15:39.331 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.331 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.331 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.331 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.331 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.331 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:161006 6.1 6:8d7d2cbb:::benchmark_data_smithi063_160268_object34899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:15:39.331 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.709227+0000", 2024-08-17T17:15:39.332 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 1.520079374, 2024-08-17T17:15:39.332 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97342043, 2024-08-17T17:15:39.332 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.332 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.332 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:39.332 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:39.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:39.333 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:39.333 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:39.333 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 161006 2024-08-17T17:15:39.334 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.334 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.334 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.334 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.334 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709227+0000", 2024-08-17T17:15:39.334 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.334 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.334 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.334 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.334 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709227+0000", 2024-08-17T17:15:39.334 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.335 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.335 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.335 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.335 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709226+0000", 2024-08-17T17:15:39.335 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:39.335 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.335 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.335 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.335 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709232+0000", 2024-08-17T17:15:39.335 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6219999999999997e-06 2024-08-17T17:15:39.335 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.335 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.335 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.336 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709243+0000", 2024-08-17T17:15:39.336 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0839000000000001e-05 2024-08-17T17:15:39.336 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.336 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.336 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.336 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709248+0000", 2024-08-17T17:15:39.336 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.322e-06 2024-08-17T17:15:39.336 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.336 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.336 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.336 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709293+0000", 2024-08-17T17:15:39.336 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4672000000000002e-05 2024-08-17T17:15:39.337 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.337 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.337 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:39.337 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709302+0000", 2024-08-17T17:15:39.337 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2240000000000001e-06 2024-08-17T17:15:39.337 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.337 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.337 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.337 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.663451+0000", 2024-08-17T17:15:39.337 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95414951299999995 2024-08-17T17:15:39.337 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.337 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.338 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.338 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.663549+0000", 2024-08-17T17:15:39.338 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7268000000000005e-05 2024-08-17T17:15:39.338 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.338 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.338 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:15:39.338 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.663655+0000", 2024-08-17T17:15:39.338 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000106872 2024-08-17T17:15:39.338 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.338 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.338 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:39.338 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667559+0000", 2024-08-17T17:15:39.339 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0039036629999999999 2024-08-17T17:15:39.339 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.339 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.339 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.339 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.678421+0000", 2024-08-17T17:15:39.339 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010861585 2024-08-17T17:15:39.339 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.339 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.339 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.339 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.682563+0000", 2024-08-17T17:15:39.339 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0041419159999999998 2024-08-17T17:15:39.340 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.340 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.340 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.340 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.682611+0000", 2024-08-17T17:15:39.340 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8585000000000003e-05 2024-08-17T17:15:39.340 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.340 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.340 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.340 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.682648+0000", 2024-08-17T17:15:39.340 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6449999999999998e-05 2024-08-17T17:15:39.340 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.340 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.341 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.341 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.341 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.341 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:161016 6.1 6:83fef0fc:::benchmark_data_smithi063_160268_object34909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:15:39.341 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.731599+0000", 2024-08-17T17:15:39.341 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 1.497708008, 2024-08-17T17:15:39.341 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95788104200000002, 2024-08-17T17:15:39.341 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.341 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.341 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:39.341 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:39.341 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:39.341 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 161016 2024-08-17T17:15:39.342 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.342 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.342 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.342 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.342 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731599+0000", 2024-08-17T17:15:39.342 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.342 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.342 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.342 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.342 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731599+0000", 2024-08-17T17:15:39.342 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.342 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.343 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.343 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.343 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731597+0000", 2024-08-17T17:15:39.343 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:39.343 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.343 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.343 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.343 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731604+0000", 2024-08-17T17:15:39.343 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3790000000000003e-06 2024-08-17T17:15:39.343 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.343 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.343 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.344 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731616+0000", 2024-08-17T17:15:39.344 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.261e-05 2024-08-17T17:15:39.344 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.344 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.344 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.344 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731622+0000", 2024-08-17T17:15:39.344 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1809999999999998e-06 2024-08-17T17:15:39.344 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.344 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.344 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.344 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731664+0000", 2024-08-17T17:15:39.344 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1355e-05 2024-08-17T17:15:39.345 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.345 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.345 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:39.345 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731673+0000", 2024-08-17T17:15:39.345 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.704e-06 2024-08-17T17:15:39.345 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.345 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.345 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.345 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667670+0000", 2024-08-17T17:15:39.345 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.93599733900000004 2024-08-17T17:15:39.345 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.345 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.346 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.346 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667779+0000", 2024-08-17T17:15:39.346 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000109268 2024-08-17T17:15:39.346 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.346 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.346 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:15:39.346 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667926+0000", 2024-08-17T17:15:39.346 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014687899999999999 2024-08-17T17:15:39.346 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.346 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.346 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:39.346 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676467+0000", 2024-08-17T17:15:39.347 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0085410450000000006 2024-08-17T17:15:39.347 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.347 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.347 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.347 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.678606+0000", 2024-08-17T17:15:39.347 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021387590000000001 2024-08-17T17:15:39.347 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.347 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.347 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.347 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.689357+0000", 2024-08-17T17:15:39.347 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010751024 2024-08-17T17:15:39.347 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.347 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.348 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.348 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.689441+0000", 2024-08-17T17:15:39.348 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4072000000000004e-05 2024-08-17T17:15:39.348 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.348 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.348 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.348 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.689480+0000", 2024-08-17T17:15:39.348 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8667000000000001e-05 2024-08-17T17:15:39.348 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.348 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.348 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.348 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.348 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.349 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:161188 6.1 6:85507563:::benchmark_data_smithi063_160268_object35081:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1110)", 2024-08-17T17:15:39.349 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:38.822493+0000", 2024-08-17T17:15:39.349 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.40681383599999998, 2024-08-17T17:15:39.349 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.028701095999999999, 2024-08-17T17:15:39.349 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.349 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.349 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:39.349 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:39.349 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:39.349 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 161188 2024-08-17T17:15:39.349 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.350 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.350 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.350 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.350 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.822493+0000", 2024-08-17T17:15:39.350 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.350 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.350 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.350 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.350 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.822493+0000", 2024-08-17T17:15:39.350 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.350 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.350 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.350 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.351 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.822492+0000", 2024-08-17T17:15:39.351 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:15:39.351 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.351 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.351 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.351 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.822495+0000", 2024-08-17T17:15:39.351 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.213e-06 2024-08-17T17:15:39.351 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.351 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.351 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.351 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.822503+0000", 2024-08-17T17:15:39.351 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2300000000000002e-06 2024-08-17T17:15:39.351 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.352 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.352 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.352 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.822507+0000", 2024-08-17T17:15:39.352 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5109999999999997e-06 2024-08-17T17:15:39.352 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.352 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.352 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.352 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.826570+0000", 2024-08-17T17:15:39.352 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0040624220000000004 2024-08-17T17:15:39.352 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.352 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.352 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.352 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.826702+0000", 2024-08-17T17:15:39.353 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013274399999999999 2024-08-17T17:15:39.353 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.353 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.353 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:15:39.353 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.826806+0000", 2024-08-17T17:15:39.353 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000103311 2024-08-17T17:15:39.353 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.353 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.353 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:39.353 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.831635+0000", 2024-08-17T17:15:39.353 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0048292450000000002 2024-08-17T17:15:39.353 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.353 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.354 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.354 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.837046+0000", 2024-08-17T17:15:39.354 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0054114610000000002 2024-08-17T17:15:39.354 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.354 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.354 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.354 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.851102+0000", 2024-08-17T17:15:39.354 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.014055949999999999 2024-08-17T17:15:39.354 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.354 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.354 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.354 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.851153+0000", 2024-08-17T17:15:39.355 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1051999999999998e-05 2024-08-17T17:15:39.355 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.355 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.355 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.355 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.851194+0000", 2024-08-17T17:15:39.355 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0457999999999998e-05 2024-08-17T17:15:39.355 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.355 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.355 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.355 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.355 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.355 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:161196 6.1 6:8299af29:::benchmark_data_smithi063_160268_object35089:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1110)", 2024-08-17T17:15:39.356 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:38.833446+0000", 2024-08-17T17:15:39.356 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.39586069600000001, 2024-08-17T17:15:39.356 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.029917216999999999, 2024-08-17T17:15:39.356 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.356 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.356 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:39.356 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:39.356 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:39.356 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 161196 2024-08-17T17:15:39.356 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.356 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.356 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.356 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.357 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.833446+0000", 2024-08-17T17:15:39.357 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.357 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.357 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.357 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.357 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.833446+0000", 2024-08-17T17:15:39.357 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.357 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.357 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.357 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.357 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.833445+0000", 2024-08-17T17:15:39.357 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:39.357 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.358 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.358 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.358 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.833451+0000", 2024-08-17T17:15:39.358 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.669e-06 2024-08-17T17:15:39.358 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.358 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.358 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.358 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.833464+0000", 2024-08-17T17:15:39.358 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.256e-05 2024-08-17T17:15:39.358 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.358 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.358 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.359 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.833471+0000", 2024-08-17T17:15:39.359 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7800000000000003e-06 2024-08-17T17:15:39.359 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.359 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.359 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.359 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.833515+0000", 2024-08-17T17:15:39.359 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4870000000000002e-05 2024-08-17T17:15:39.359 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.359 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.359 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.359 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.833639+0000", 2024-08-17T17:15:39.359 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012369799999999999 2024-08-17T17:15:39.359 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.359 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.360 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:15:39.360 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.833752+0000", 2024-08-17T17:15:39.360 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011272099999999999 2024-08-17T17:15:39.360 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.360 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.360 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:39.360 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.837053+0000", 2024-08-17T17:15:39.360 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0033012699999999998 2024-08-17T17:15:39.360 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.360 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.360 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.360 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.858183+0000", 2024-08-17T17:15:39.360 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.021129742 2024-08-17T17:15:39.361 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.361 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.361 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.361 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.863280+0000", 2024-08-17T17:15:39.361 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0050968180000000004 2024-08-17T17:15:39.361 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.361 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.361 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.361 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.863326+0000", 2024-08-17T17:15:39.361 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6338000000000003e-05 2024-08-17T17:15:39.361 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.361 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.361 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.362 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.863363+0000", 2024-08-17T17:15:39.362 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7024999999999999e-05 2024-08-17T17:15:39.362 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.362 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.362 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.362 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.362 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.362 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:161424 6.1 6:83fd8ece:::benchmark_data_smithi063_160268_object35317:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1110)", 2024-08-17T17:15:39.362 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:39.095062+0000", 2024-08-17T17:15:39.362 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.13424414400000001, 2024-08-17T17:15:39.362 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.026121029, 2024-08-17T17:15:39.362 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.362 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.363 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:39.363 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:39.363 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:39.363 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 161424 2024-08-17T17:15:39.363 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.363 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.363 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.363 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.363 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.095062+0000", 2024-08-17T17:15:39.363 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.363 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.363 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.364 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.364 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.095062+0000", 2024-08-17T17:15:39.364 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.364 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.364 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.364 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.364 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.095061+0000", 2024-08-17T17:15:39.364 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:15:39.364 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.364 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.364 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.364 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.095069+0000", 2024-08-17T17:15:39.364 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.1820000000000006e-06 2024-08-17T17:15:39.365 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.365 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.365 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.365 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.095084+0000", 2024-08-17T17:15:39.365 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5557999999999999e-05 2024-08-17T17:15:39.365 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.365 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.365 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.365 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.095092+0000", 2024-08-17T17:15:39.365 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9759999999999995e-06 2024-08-17T17:15:39.365 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.365 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.365 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.366 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.095151+0000", 2024-08-17T17:15:39.366 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8403999999999999e-05 2024-08-17T17:15:39.366 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.366 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.366 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.366 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.095281+0000", 2024-08-17T17:15:39.366 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000130228 2024-08-17T17:15:39.366 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.366 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.366 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:15:39.366 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.095391+0000", 2024-08-17T17:15:39.366 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000110497 2024-08-17T17:15:39.366 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.367 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.367 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:39.367 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.098812+0000", 2024-08-17T17:15:39.367 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0034202009999999999 2024-08-17T17:15:39.367 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.367 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.367 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.367 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.101503+0000", 2024-08-17T17:15:39.367 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026909989999999999 2024-08-17T17:15:39.367 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.367 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.367 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.368 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.120940+0000", 2024-08-17T17:15:39.368 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.019436839000000001 2024-08-17T17:15:39.368 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.368 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.368 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.368 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.121061+0000", 2024-08-17T17:15:39.368 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012150300000000001 2024-08-17T17:15:39.368 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.368 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.368 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.368 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.121183+0000", 2024-08-17T17:15:39.368 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000122433 2024-08-17T17:15:39.368 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.368 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.369 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.369 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.369 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.369 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:161438 6.1 6:87c774ba:::benchmark_data_smithi063_160268_object35331:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1110)", 2024-08-17T17:15:39.369 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:39.114081+0000", 2024-08-17T17:15:39.369 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.11522539800000001, 2024-08-17T17:15:39.369 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.026444401999999999, 2024-08-17T17:15:39.369 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:39.369 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:39.369 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:39.369 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:39.369 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:39.370 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 161438 2024-08-17T17:15:39.370 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.370 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:39.370 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.370 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:39.370 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.114081+0000", 2024-08-17T17:15:39.370 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.370 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.370 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.370 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:39.370 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.114081+0000", 2024-08-17T17:15:39.370 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:39.370 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.371 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.371 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:39.371 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.114080+0000", 2024-08-17T17:15:39.371 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:39.371 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.371 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.371 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:39.371 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.114087+0000", 2024-08-17T17:15:39.371 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1269999999999997e-06 2024-08-17T17:15:39.371 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.371 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.371 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:39.371 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.114104+0000", 2024-08-17T17:15:39.372 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6864e-05 2024-08-17T17:15:39.372 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.372 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.372 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:39.372 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.114111+0000", 2024-08-17T17:15:39.372 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.216e-06 2024-08-17T17:15:39.372 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.372 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.372 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:39.372 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.126088+0000", 2024-08-17T17:15:39.372 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011977033999999999 2024-08-17T17:15:39.372 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.372 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.373 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:39.373 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.126196+0000", 2024-08-17T17:15:39.373 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010854 2024-08-17T17:15:39.373 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.373 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.373 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:15:39.373 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.126302+0000", 2024-08-17T17:15:39.373 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000105808 2024-08-17T17:15:39.373 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.373 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.373 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.373 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.139948+0000", 2024-08-17T17:15:39.373 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.013645533 2024-08-17T17:15:39.374 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.374 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.374 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:39.374 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.140075+0000", 2024-08-17T17:15:39.374 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012704400000000001 2024-08-17T17:15:39.374 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.374 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.374 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:39.374 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.140122+0000", 2024-08-17T17:15:39.374 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6919999999999998e-05 2024-08-17T17:15:39.374 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.374 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.374 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:39.375 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.140464+0000", 2024-08-17T17:15:39.375 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000342302 2024-08-17T17:15:39.375 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:39.375 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:39.375 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:39.375 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:39.140526+0000", 2024-08-17T17:15:39.375 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1513000000000005e-05 2024-08-17T17:15:39.375 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.375 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.375 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.375 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:39.375 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:39.375 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:15:39.433 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:39.458 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:39.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:39.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:39.560 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:39.604 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:39.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:39.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:39.694 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:39 smithi063 ceph-mon[114756]: pgmap v1221: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2024-08-17T17:15:39.694 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:39 smithi063 ceph-mon[109843]: pgmap v1221: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2024-08-17T17:15:39.706 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:39.738 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:39.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:39.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:39.840 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:39.868 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:39.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:39.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:39.964 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:15:39.965 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:15:39.969 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:39.995 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:39.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:39.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:40.002 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:15:40.002 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:15:40.056 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:15:40.056 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:15:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:39 smithi162 ceph-mon[104668]: pgmap v1221: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 871 op/s 2024-08-17T17:15:40.097 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:40.104 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:15:40.104 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:15:40.137 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:40.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:40.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:40.141 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:15:40.141 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:15:40.174 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:15:40.174 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:15:40.205 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:15:40.205 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:15:40.239 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:40.241 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:15:40.241 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:15:40.265 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:40.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:40.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:40.303 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:15:40.368 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:40.402 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:40.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:40.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:40.504 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:40.535 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:40.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:40.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:40.637 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:40.644 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:15:40.652 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:15:40.677 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:40.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:40.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:40.778 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:40.808 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:40.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:40.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:40.910 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:40.945 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:40 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3732963773' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:40.945 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:40 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4170194737' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:15:40.946 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:40 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3732963773' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:40.946 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:40 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4170194737' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:15:40.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:40.951 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:40.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:40.958 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:40 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3732963773' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:40.958 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:40 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4170194737' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:15:41.052 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:41.083 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:41.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:41.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:41.185 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:41.227 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:41.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:41.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:41.329 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:41.358 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:41.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:41.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:41.460 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:41.498 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:41.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:41.499 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:41.600 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:41.640 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:41.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:41.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:41.741 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:41.781 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:41.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:41.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:41.883 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:41.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:41.923 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:41.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:41.932 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:41 smithi162 ceph-mon[104668]: pgmap v1223: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2024-08-17T17:15:41.932 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:41 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:15:41.932 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:41 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4170194737' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:15:41.932 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:41 smithi162 ceph-mon[104668]: osdmap e1111: 8 total, 8 up, 8 in 2024-08-17T17:15:42.025 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:41 smithi063 ceph-mon[114756]: pgmap v1223: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2024-08-17T17:15:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:41 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:15:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:41 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4170194737' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:15:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:41 smithi063 ceph-mon[114756]: osdmap e1111: 8 total, 8 up, 8 in 2024-08-17T17:15:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:41 smithi063 ceph-mon[109843]: pgmap v1223: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 640 op/s 2024-08-17T17:15:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:41 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:15:42.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:41 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4170194737' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:15:42.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:41 smithi063 ceph-mon[109843]: osdmap e1111: 8 total, 8 up, 8 in 2024-08-17T17:15:42.066 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:42.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:42.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:42.168 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:42.199 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:42.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:42.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:42.301 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:42.332 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:42.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:42.434 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:42.472 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:42.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:42.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:42.574 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:42.609 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:42.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:42.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:42.623 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:15:42.623 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:15:42.711 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:42.745 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:42.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:42.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:42.846 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:42.888 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:42.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:42.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:42.982 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:15:42.989 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:43.020 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:43.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:43.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:43.122 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:43.148 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:43.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:43.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:43.249 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:43.279 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:43.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:43.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:43.291 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:15:43.379 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:43.381 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:43.414 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:15:43.415 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:15:43.415 INFO:tasks.thrashosds.thrasher:reset_purged_snaps_last 2024-08-17T17:15:43.415 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.0 reset_purged_snaps_last 2024-08-17T17:15:43.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:43.437 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:43.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:43.538 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:43.551 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:15:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:15:43] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.51.0" 2024-08-17T17:15:43.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:43.579 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:43.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:43.620 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:15:43.674 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:15:43.681 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:43.703 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:43.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:43.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:43.805 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:43.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:43.832 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:43.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:43.924 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:43 smithi162 ceph-mon[104668]: pgmap v1225: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:15:43.933 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:43.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:43.984 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:43.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:43 smithi063 ceph-mon[114756]: pgmap v1225: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:15:44.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:43 smithi063 ceph-mon[109843]: pgmap v1225: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:15:44.085 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:44.125 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:44.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:44.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:44.228 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:44.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:44.267 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:44.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:44.369 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:44.443 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:44.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:44.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:44.545 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:44.578 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:44.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:44.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:44.680 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:44.716 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:44.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:44.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:44.817 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:44.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:44.855 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:44.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:44.914 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:44.956 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:45.016 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:45.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:45.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:45.118 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:45.163 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:45.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:45.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:45.236 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:45.265 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:45.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:45.322 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:45.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:45.423 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:45.457 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:45.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:45.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:45.559 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:45.590 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:45.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:45.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:45.692 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:45.733 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:45.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:45.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:45.769 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:15:45.769 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:15:45.784 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:15:45.784 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:15:45.827 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:15:45.828 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:15:45.835 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:45.859 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:45 smithi063 ceph-mon[109843]: pgmap v1226: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2024-08-17T17:15:45.860 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:45 smithi063 ceph-mon[114756]: pgmap v1226: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2024-08-17T17:15:45.865 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:15:45.865 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:15:45.887 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:45.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:45.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:45.916 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:15:45.916 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:15:45.951 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:15:45.952 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:15:45.986 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:15:45.987 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:15:45.989 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:46.018 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:15:46.018 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:15:46.026 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:46.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:46.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:46.052 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:15:46.052 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:15:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:45 smithi162 ceph-mon[104668]: pgmap v1226: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s 2024-08-17T17:15:46.128 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:46.165 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:46.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:46.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:46.267 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:46.296 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:46.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:46.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:46.398 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:46.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:46.422 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:46.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:46.524 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:46.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:46.558 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:46.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:46.593 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:15:46.660 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:46.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:46.717 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:46.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:46.818 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:46 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/364598842' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:46.858 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:46.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:46.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:46.924 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:46 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/364598842' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:46.924 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:46.925 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:46 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/364598842' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:46.925 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:15:46.934 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:15:46.960 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:47.001 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:47.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:47.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:47.102 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:47.136 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:47.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:47.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:47.238 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:47.277 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:47.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:47.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:47.379 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:47.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:47.416 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:47.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:47.517 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:47.547 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:47.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:47.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:47.649 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:47.679 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:47.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:47.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:47.780 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:47.811 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:47.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:47.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:47.913 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:47.947 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:47.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:47.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:47 smithi063 ceph-mon[114756]: pgmap v1227: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-08-17T17:15:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:47 smithi063 ceph-mon[109843]: pgmap v1227: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-08-17T17:15:48.051 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:48.089 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:48.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:48.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:47 smithi162 ceph-mon[104668]: pgmap v1227: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 650 op/s 2024-08-17T17:15:48.191 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:48.232 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:48.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:48.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:48.334 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:48.371 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:48.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:48.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:48.473 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:48.517 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:48.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:48.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:48.619 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:48.649 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:48.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:48.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:48.750 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:48.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:48.794 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:48.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:48.876 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:15:48.876 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:15:48.876 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:15:48.876 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:15:48.876 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.876 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:32646 6.8 6:1265bc8f:::benchmark_data_smithi063_160268_object32645:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:48.876 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:39.131532+0000", 2024-08-17T17:15:48.876 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 129.73683385499999, 2024-08-17T17:15:48.876 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.096578093000000004, 2024-08-17T17:15:48.877 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:48.877 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:48.877 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:48.877 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:48.877 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:48.877 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 32646 2024-08-17T17:15:48.877 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.877 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:48.877 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.877 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:48.878 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.131532+0000", 2024-08-17T17:15:48.878 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.878 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.878 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.878 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:48.879 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.131532+0000", 2024-08-17T17:15:48.879 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.879 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.879 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.879 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:48.879 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.131530+0000", 2024-08-17T17:15:48.880 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:48.880 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.880 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.880 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:48.880 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.131574+0000", 2024-08-17T17:15:48.880 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3711000000000002e-05 2024-08-17T17:15:48.881 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.881 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.881 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:48.881 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.131594+0000", 2024-08-17T17:15:48.881 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9952000000000001e-05 2024-08-17T17:15:48.881 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.882 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.882 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:48.882 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.131601+0000", 2024-08-17T17:15:48.882 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3869999999999997e-06 2024-08-17T17:15:48.882 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.882 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.882 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.882 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.131685+0000", 2024-08-17T17:15:48.882 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3515000000000003e-05 2024-08-17T17:15:48.883 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.883 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.883 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:48.883 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.131842+0000", 2024-08-17T17:15:48.883 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000157608 2024-08-17T17:15:48.883 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.883 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.883 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:48.883 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.131980+0000", 2024-08-17T17:15:48.883 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013732100000000001 2024-08-17T17:15:48.883 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.884 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.884 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:48.884 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.133490+0000", 2024-08-17T17:15:48.884 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015100619999999999 2024-08-17T17:15:48.884 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.884 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.884 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.884 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.138089+0000", 2024-08-17T17:15:48.884 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0045993070000000004 2024-08-17T17:15:48.884 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.884 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.885 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.885 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.228026+0000", 2024-08-17T17:15:48.885 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.089937038999999996 2024-08-17T17:15:48.885 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.885 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.885 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:48.885 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.228075+0000", 2024-08-17T17:15:48.885 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9384e-05 2024-08-17T17:15:48.885 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.886 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.886 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:48.886 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.228110+0000", 2024-08-17T17:15:48.886 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4291000000000001e-05 2024-08-17T17:15:48.886 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.886 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:48.886 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.886 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.886 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.886 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:32653 6.8 6:1c5994f7:::benchmark_data_smithi063_160268_object32652:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:48.886 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:39.133703+0000", 2024-08-17T17:15:48.887 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 129.73466259700001, 2024-08-17T17:15:48.887 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.094543853999999997, 2024-08-17T17:15:48.887 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:48.887 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:48.887 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:48.887 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:48.887 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:48.887 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 32653 2024-08-17T17:15:48.887 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.887 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:48.887 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.887 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:48.888 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.133703+0000", 2024-08-17T17:15:48.888 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.888 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.888 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.888 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:48.888 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.133703+0000", 2024-08-17T17:15:48.888 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.888 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.888 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.888 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:48.888 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.133702+0000", 2024-08-17T17:15:48.889 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:48.889 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.889 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.889 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:48.889 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.133766+0000", 2024-08-17T17:15:48.889 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4382999999999997e-05 2024-08-17T17:15:48.889 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.889 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.889 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:48.889 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.133778+0000", 2024-08-17T17:15:48.889 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1799e-05 2024-08-17T17:15:48.889 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.889 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.890 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:48.890 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.133786+0000", 2024-08-17T17:15:48.890 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9189999999999999e-06 2024-08-17T17:15:48.890 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.890 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.890 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.890 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.133853+0000", 2024-08-17T17:15:48.890 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6569999999999995e-05 2024-08-17T17:15:48.890 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.890 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.890 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:48.890 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.133999+0000", 2024-08-17T17:15:48.891 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014630399999999999 2024-08-17T17:15:48.891 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.891 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.891 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:48.891 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.134136+0000", 2024-08-17T17:15:48.891 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013706100000000001 2024-08-17T17:15:48.891 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.891 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.891 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:48.891 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.136226+0000", 2024-08-17T17:15:48.891 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0020901890000000001 2024-08-17T17:15:48.891 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.891 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.892 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.892 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.138120+0000", 2024-08-17T17:15:48.892 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0018937719999999999 2024-08-17T17:15:48.892 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.892 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.892 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.892 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.228161+0000", 2024-08-17T17:15:48.892 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.090040775000000003 2024-08-17T17:15:48.893 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.893 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.893 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:48.893 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.228210+0000", 2024-08-17T17:15:48.893 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9407999999999999e-05 2024-08-17T17:15:48.893 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.893 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.893 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:48.893 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.228247+0000", 2024-08-17T17:15:48.893 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6588000000000003e-05 2024-08-17T17:15:48.894 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.894 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:48.894 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.894 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.894 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.894 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:32654 6.8 6:1b9f13ce:::benchmark_data_smithi063_160268_object32653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:48.894 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:39.133924+0000", 2024-08-17T17:15:48.894 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 129.73444180600001, 2024-08-17T17:15:48.895 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.094466710999999995, 2024-08-17T17:15:48.895 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:48.895 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:48.895 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:48.895 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:48.895 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:48.895 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 32654 2024-08-17T17:15:48.895 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.895 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:48.895 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.895 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:48.895 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.133924+0000", 2024-08-17T17:15:48.896 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.896 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.896 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.896 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:48.896 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.133924+0000", 2024-08-17T17:15:48.896 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.896 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.896 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.896 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:48.896 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.133923+0000", 2024-08-17T17:15:48.896 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:15:48.896 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.897 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.897 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:48.897 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.133960+0000", 2024-08-17T17:15:48.897 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6717000000000001e-05 2024-08-17T17:15:48.897 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.897 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.897 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:48.897 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.133969+0000", 2024-08-17T17:15:48.897 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.6420000000000002e-06 2024-08-17T17:15:48.897 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.897 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.897 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:48.897 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.133976+0000", 2024-08-17T17:15:48.898 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6830000000000002e-06 2024-08-17T17:15:48.898 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.898 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.898 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.898 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.135252+0000", 2024-08-17T17:15:48.898 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001275857 2024-08-17T17:15:48.898 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.898 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.898 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:48.898 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.135357+0000", 2024-08-17T17:15:48.898 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010476600000000001 2024-08-17T17:15:48.899 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.899 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.899 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:48.899 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.135496+0000", 2024-08-17T17:15:48.899 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013966299999999999 2024-08-17T17:15:48.899 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.899 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.899 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:48.899 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.136385+0000", 2024-08-17T17:15:48.899 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00088898299999999998 2024-08-17T17:15:48.900 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.900 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.900 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.900 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.138411+0000", 2024-08-17T17:15:48.900 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002025677 2024-08-17T17:15:48.900 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.900 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.900 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.900 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.228312+0000", 2024-08-17T17:15:48.900 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.089900519999999998 2024-08-17T17:15:48.900 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.900 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.900 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:48.901 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.228354+0000", 2024-08-17T17:15:48.901 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2917000000000003e-05 2024-08-17T17:15:48.901 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.901 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.901 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:48.901 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.228390+0000", 2024-08-17T17:15:48.901 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5924999999999999e-05 2024-08-17T17:15:48.901 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.901 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:48.901 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.901 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.901 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.902 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:32661 6.8 6:18d383d2:::benchmark_data_smithi063_160268_object32660:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:48.902 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:39.136556+0000", 2024-08-17T17:15:48.902 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 129.731809782, 2024-08-17T17:15:48.902 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.091950964999999996, 2024-08-17T17:15:48.902 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:48.902 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:48.902 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:48.902 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:48.902 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:48.902 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 32661 2024-08-17T17:15:48.902 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.903 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:48.903 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.903 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:48.903 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.136556+0000", 2024-08-17T17:15:48.903 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.903 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.903 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.903 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:48.903 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.136556+0000", 2024-08-17T17:15:48.904 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.904 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.904 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.904 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:48.904 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.136555+0000", 2024-08-17T17:15:48.904 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:15:48.904 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.904 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.904 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:48.904 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.136574+0000", 2024-08-17T17:15:48.904 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.8935000000000001e-05 2024-08-17T17:15:48.904 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.905 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.905 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:48.905 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.136579+0000", 2024-08-17T17:15:48.905 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3279999999999996e-06 2024-08-17T17:15:48.905 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.905 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.905 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:48.905 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.136583+0000", 2024-08-17T17:15:48.905 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2339999999999999e-06 2024-08-17T17:15:48.905 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.905 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.905 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.906 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.136619+0000", 2024-08-17T17:15:48.906 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6711999999999997e-05 2024-08-17T17:15:48.906 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.906 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.906 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:48.906 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.136731+0000", 2024-08-17T17:15:48.906 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000111458 2024-08-17T17:15:48.906 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.906 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.906 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:48.906 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.136874+0000", 2024-08-17T17:15:48.906 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000142817 2024-08-17T17:15:48.907 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.907 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.907 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:48.907 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.137695+0000", 2024-08-17T17:15:48.907 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00082136199999999996 2024-08-17T17:15:48.907 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.907 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.907 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.907 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.217895+0000", 2024-08-17T17:15:48.907 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.080199468999999995 2024-08-17T17:15:48.907 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.907 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.908 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.908 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.228431+0000", 2024-08-17T17:15:48.908 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010536772 2024-08-17T17:15:48.908 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.908 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.908 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:48.908 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.228475+0000", 2024-08-17T17:15:48.908 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3272000000000002e-05 2024-08-17T17:15:48.908 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.908 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.908 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:48.908 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.228507+0000", 2024-08-17T17:15:48.908 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2115e-05 2024-08-17T17:15:48.909 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.909 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:48.909 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.909 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.909 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.909 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:33169 6.5 6:ab3fb7ed:::benchmark_data_smithi063_160268_object33168:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:48.909 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:39.449432+0000", 2024-08-17T17:15:48.909 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 129.41893314199999, 2024-08-17T17:15:48.909 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.093660018999999997, 2024-08-17T17:15:48.909 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:48.909 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:48.909 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:48.909 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:48.910 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:48.910 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 33169 2024-08-17T17:15:48.910 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.910 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:48.910 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.910 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:48.910 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.449432+0000", 2024-08-17T17:15:48.910 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.910 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.910 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.910 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:48.910 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.449432+0000", 2024-08-17T17:15:48.911 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.911 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.911 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.911 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:48.911 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.449432+0000", 2024-08-17T17:15:48.911 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:48.911 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.911 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.911 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:48.911 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.449509+0000", 2024-08-17T17:15:48.911 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7477000000000004e-05 2024-08-17T17:15:48.911 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.912 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.912 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:48.912 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.449519+0000", 2024-08-17T17:15:48.912 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0366e-05 2024-08-17T17:15:48.912 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.912 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.912 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:48.912 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.449526+0000", 2024-08-17T17:15:48.912 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1019999999999999e-06 2024-08-17T17:15:48.912 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.912 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.912 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.913 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.450932+0000", 2024-08-17T17:15:48.913 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001405269 2024-08-17T17:15:48.913 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.913 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.913 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:48.913 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451045+0000", 2024-08-17T17:15:48.913 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011360400000000001 2024-08-17T17:15:48.913 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.913 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.913 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:15:48.914 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451200+0000", 2024-08-17T17:15:48.914 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015458599999999999 2024-08-17T17:15:48.914 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.914 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.914 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.914 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.455105+0000", 2024-08-17T17:15:48.914 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0039048529999999998 2024-08-17T17:15:48.914 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.914 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.914 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:48.915 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.455143+0000", 2024-08-17T17:15:48.915 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8108999999999997e-05 2024-08-17T17:15:48.915 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.915 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.915 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.915 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.543003+0000", 2024-08-17T17:15:48.915 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.087859703999999997 2024-08-17T17:15:48.915 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.915 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.915 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:48.915 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.543050+0000", 2024-08-17T17:15:48.915 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7342000000000002e-05 2024-08-17T17:15:48.915 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.916 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.916 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:48.916 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.543092+0000", 2024-08-17T17:15:48.916 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2413999999999997e-05 2024-08-17T17:15:48.916 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.916 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:48.916 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.916 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.916 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.916 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:33170 6.5 6:a21482cc:::benchmark_data_smithi063_160268_object33169:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:48.916 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:39.449969+0000", 2024-08-17T17:15:48.916 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 129.41839682, 2024-08-17T17:15:48.917 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.093278675000000005, 2024-08-17T17:15:48.917 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:48.917 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:48.917 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:48.917 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:48.917 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:48.917 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 33170 2024-08-17T17:15:48.917 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.917 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:48.917 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.917 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:48.917 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.449969+0000", 2024-08-17T17:15:48.917 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.918 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.918 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.918 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:48.918 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.449969+0000", 2024-08-17T17:15:48.918 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.918 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.918 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.918 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:48.918 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.449968+0000", 2024-08-17T17:15:48.918 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:48.919 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.919 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.919 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:48.919 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.450781+0000", 2024-08-17T17:15:48.919 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00081291100000000004 2024-08-17T17:15:48.919 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.919 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.919 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:48.919 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.450794+0000", 2024-08-17T17:15:48.919 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.309e-05 2024-08-17T17:15:48.919 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.920 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.920 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:48.920 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.450801+0000", 2024-08-17T17:15:48.920 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.5959999999999998e-06 2024-08-17T17:15:48.920 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.920 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.920 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.920 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.452315+0000", 2024-08-17T17:15:48.920 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015131120000000001 2024-08-17T17:15:48.920 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.920 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.920 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:48.920 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.452403+0000", 2024-08-17T17:15:48.921 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8256000000000003e-05 2024-08-17T17:15:48.921 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.921 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.921 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:15:48.921 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.452553+0000", 2024-08-17T17:15:48.921 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000150581 2024-08-17T17:15:48.921 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.921 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.921 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.922 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.455136+0000", 2024-08-17T17:15:48.922 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002582272 2024-08-17T17:15:48.922 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.922 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.922 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:48.922 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.455149+0000", 2024-08-17T17:15:48.922 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2883e-05 2024-08-17T17:15:48.922 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.922 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.922 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.923 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.543148+0000", 2024-08-17T17:15:48.923 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.087999730999999998 2024-08-17T17:15:48.923 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.923 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.923 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:48.923 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.543199+0000", 2024-08-17T17:15:48.923 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1122000000000003e-05 2024-08-17T17:15:48.923 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.923 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.923 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:48.924 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.543247+0000", 2024-08-17T17:15:48.924 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7966999999999997e-05 2024-08-17T17:15:48.924 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.924 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:48.924 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.924 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.924 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.925 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:35808 6.8 6:182f8ffd:::benchmark_data_smithi063_160268_object35807:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:48.925 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.876819+0000", 2024-08-17T17:15:48.925 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 127.991546489, 2024-08-17T17:15:48.925 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.144014062, 2024-08-17T17:15:48.925 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:48.925 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:48.925 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:48.925 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:48.925 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:48.925 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 35808 2024-08-17T17:15:48.925 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.925 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:48.926 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.926 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:48.926 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876819+0000", 2024-08-17T17:15:48.926 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.926 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.926 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.926 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:48.926 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876819+0000", 2024-08-17T17:15:48.926 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.926 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.926 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.927 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:48.927 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876818+0000", 2024-08-17T17:15:48.927 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:48.927 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.927 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.927 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:48.927 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876847+0000", 2024-08-17T17:15:48.927 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9553e-05 2024-08-17T17:15:48.927 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.927 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.927 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:48.927 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876870+0000", 2024-08-17T17:15:48.928 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2133999999999998e-05 2024-08-17T17:15:48.928 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.928 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.928 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:48.928 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876876+0000", 2024-08-17T17:15:48.928 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:48.930 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5599999999999999e-06 2024-08-17T17:15:48.930 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.930 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.930 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.930 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876932+0000", 2024-08-17T17:15:48.930 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6078000000000001e-05 2024-08-17T17:15:48.930 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.930 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.931 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:48.931 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.877050+0000", 2024-08-17T17:15:48.931 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000117439 2024-08-17T17:15:48.931 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.931 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.931 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:48.931 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.877169+0000", 2024-08-17T17:15:48.931 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000119262 2024-08-17T17:15:48.931 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.931 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.932 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:48.932 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.878556+0000", 2024-08-17T17:15:48.932 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013872610000000001 2024-08-17T17:15:48.932 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.932 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.932 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.932 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.881767+0000", 2024-08-17T17:15:48.932 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0032103890000000001 2024-08-17T17:15:48.932 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.932 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.933 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.933 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.020762+0000", 2024-08-17T17:15:48.933 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.13899573000000001 2024-08-17T17:15:48.933 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.933 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.933 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:48.933 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.020803+0000", 2024-08-17T17:15:48.933 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0271000000000001e-05 2024-08-17T17:15:48.933 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.933 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.934 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:48.934 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.020833+0000", 2024-08-17T17:15:48.934 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.0482000000000001e-05 2024-08-17T17:15:48.934 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.934 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:48.934 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.934 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.934 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.934 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:35812 6.8 6:1fd0ca99:::benchmark_data_smithi063_160268_object35811:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:48.934 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.879715+0000", 2024-08-17T17:15:48.934 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 127.98865091, 2024-08-17T17:15:48.935 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.147860146, 2024-08-17T17:15:48.935 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:48.935 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:48.935 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:48.935 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:48.935 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:48.936 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 35812 2024-08-17T17:15:48.936 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.936 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:48.936 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.936 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:48.936 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.879715+0000", 2024-08-17T17:15:48.936 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.936 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.936 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.936 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:48.936 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.879715+0000", 2024-08-17T17:15:48.936 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.937 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.937 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.937 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:48.937 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.879713+0000", 2024-08-17T17:15:48.937 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:15:48.937 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.937 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.937 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:48.937 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.879776+0000", 2024-08-17T17:15:48.937 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3243000000000004e-05 2024-08-17T17:15:48.937 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.937 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.937 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:48.938 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.879791+0000", 2024-08-17T17:15:48.938 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5216e-05 2024-08-17T17:15:48.938 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.938 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.938 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:48.938 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.879799+0000", 2024-08-17T17:15:48.938 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0339999999999997e-06 2024-08-17T17:15:48.938 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.938 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.943 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.943 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.879850+0000", 2024-08-17T17:15:48.943 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1171000000000002e-05 2024-08-17T17:15:48.943 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.944 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.944 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:48.944 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880005+0000", 2024-08-17T17:15:48.944 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000154816 2024-08-17T17:15:48.944 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.944 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.944 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:15:48.944 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880161+0000", 2024-08-17T17:15:48.944 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000155508 2024-08-17T17:15:48.944 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.944 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.945 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:48.945 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.881775+0000", 2024-08-17T17:15:48.945 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016141980000000001 2024-08-17T17:15:48.945 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.945 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.945 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.945 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.881823+0000", 2024-08-17T17:15:48.945 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7828999999999999e-05 2024-08-17T17:15:48.945 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.945 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.946 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.946 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.027484+0000", 2024-08-17T17:15:48.946 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14566167399999999 2024-08-17T17:15:48.946 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.946 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.946 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:48.947 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.027535+0000", 2024-08-17T17:15:48.947 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0946999999999999e-05 2024-08-17T17:15:48.947 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.947 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.947 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:48.947 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.027575+0000", 2024-08-17T17:15:48.947 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9304000000000002e-05 2024-08-17T17:15:48.948 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.948 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:48.948 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.948 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.948 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.948 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:35818 6.3 6:ccea9449:::benchmark_data_smithi063_160268_object35817:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:48.948 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.880685+0000", 2024-08-17T17:15:48.948 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 127.987680201, 2024-08-17T17:15:48.948 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14219040599999999, 2024-08-17T17:15:48.948 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:48.948 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:48.948 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:48.949 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:48.949 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:48.949 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 35818 2024-08-17T17:15:48.949 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.949 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:48.949 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.949 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:48.949 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880685+0000", 2024-08-17T17:15:48.949 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.949 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.949 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.949 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:48.949 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880685+0000", 2024-08-17T17:15:48.950 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.950 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.950 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.950 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:48.950 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880684+0000", 2024-08-17T17:15:48.950 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:48.950 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.950 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.950 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:48.950 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880772+0000", 2024-08-17T17:15:48.951 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7903000000000002e-05 2024-08-17T17:15:48.951 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.951 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.951 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:48.951 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880783+0000", 2024-08-17T17:15:48.951 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1126e-05 2024-08-17T17:15:48.951 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.951 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.951 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:48.951 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880790+0000", 2024-08-17T17:15:48.951 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.64e-06 2024-08-17T17:15:48.951 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.952 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.952 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.952 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.882214+0000", 2024-08-17T17:15:48.952 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001423983 2024-08-17T17:15:48.952 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.952 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.952 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:48.952 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.882336+0000", 2024-08-17T17:15:48.952 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012227599999999999 2024-08-17T17:15:48.952 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.953 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.953 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:15:48.953 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.882475+0000", 2024-08-17T17:15:48.953 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013835299999999999 2024-08-17T17:15:48.953 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.953 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.954 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.954 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.883876+0000", 2024-08-17T17:15:48.954 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001401673 2024-08-17T17:15:48.954 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.954 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.954 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:48.954 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.883883+0000", 2024-08-17T17:15:48.954 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3559999999999998e-06 2024-08-17T17:15:48.954 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.955 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.955 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.955 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.022798+0000", 2024-08-17T17:15:48.955 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.13891547300000001 2024-08-17T17:15:48.955 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.955 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.955 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:48.955 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.022835+0000", 2024-08-17T17:15:48.955 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6560000000000002e-05 2024-08-17T17:15:48.955 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.955 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.955 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:48.956 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.022876+0000", 2024-08-17T17:15:48.956 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0969e-05 2024-08-17T17:15:48.956 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.956 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:48.956 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.956 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.956 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.956 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:35824 6.5 6:a1ee89be:::benchmark_data_smithi063_160268_object35823:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:15:48.956 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.885901+0000", 2024-08-17T17:15:48.956 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 127.982464351, 2024-08-17T17:15:48.957 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.13968393500000001, 2024-08-17T17:15:48.957 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:48.957 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:48.957 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:48.957 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:48.957 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:48.957 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 35824 2024-08-17T17:15:48.957 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.957 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:48.958 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.958 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:48.958 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.885901+0000", 2024-08-17T17:15:48.958 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.958 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.958 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.958 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:48.958 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.885901+0000", 2024-08-17T17:15:48.958 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.958 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.959 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.959 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:48.959 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.885900+0000", 2024-08-17T17:15:48.959 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:48.959 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.959 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.959 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:48.959 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.885933+0000", 2024-08-17T17:15:48.959 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3626e-05 2024-08-17T17:15:48.959 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.959 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.959 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:48.960 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.885950+0000", 2024-08-17T17:15:48.960 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7008999999999999e-05 2024-08-17T17:15:48.960 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.960 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.960 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:48.960 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.885958+0000", 2024-08-17T17:15:48.960 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.8800000000000008e-06 2024-08-17T17:15:48.960 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.960 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.960 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.960 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.885996+0000", 2024-08-17T17:15:48.960 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8148999999999997e-05 2024-08-17T17:15:48.960 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.961 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.961 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:48.961 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.886084+0000", 2024-08-17T17:15:48.961 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7713999999999994e-05 2024-08-17T17:15:48.961 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.961 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.961 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:15:48.961 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.886173+0000", 2024-08-17T17:15:48.961 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9302000000000007e-05 2024-08-17T17:15:48.961 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.961 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.961 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:48.961 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.887257+0000", 2024-08-17T17:15:48.962 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001083557 2024-08-17T17:15:48.962 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.962 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.962 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.962 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.887756+0000", 2024-08-17T17:15:48.962 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00049890399999999999 2024-08-17T17:15:48.962 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.962 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.962 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.962 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025506+0000", 2024-08-17T17:15:48.962 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.13775047300000001 2024-08-17T17:15:48.963 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.963 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.963 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:48.963 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025550+0000", 2024-08-17T17:15:48.963 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4240000000000003e-05 2024-08-17T17:15:48.963 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.964 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.964 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:48.964 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025585+0000", 2024-08-17T17:15:48.964 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4610999999999998e-05 2024-08-17T17:15:48.964 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.964 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:48.964 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.964 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.964 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.965 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131754 6.9 6:932b4294:::benchmark_data_smithi063_160268_object5647:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:48.965 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.410180+0000", 2024-08-17T17:15:48.965 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 45.458185532999998, 2024-08-17T17:15:48.965 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99255267400000002, 2024-08-17T17:15:48.965 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:48.965 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:48.965 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:48.965 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:48.965 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:48.965 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131754 2024-08-17T17:15:48.966 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.966 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:48.966 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.966 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:48.966 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410180+0000", 2024-08-17T17:15:48.966 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.966 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.966 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.966 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:48.966 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410180+0000", 2024-08-17T17:15:48.966 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.966 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.967 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.967 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:48.967 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410179+0000", 2024-08-17T17:15:48.967 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:48.967 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.967 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.967 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:48.967 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410184+0000", 2024-08-17T17:15:48.968 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3889999999999998e-06 2024-08-17T17:15:48.968 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.968 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.968 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:48.968 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410192+0000", 2024-08-17T17:15:48.969 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0090000000000007e-06 2024-08-17T17:15:48.969 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.969 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.969 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:48.969 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410196+0000", 2024-08-17T17:15:48.969 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6249999999999998e-06 2024-08-17T17:15:48.970 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.970 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.970 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.970 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410283+0000", 2024-08-17T17:15:48.970 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7316000000000002e-05 2024-08-17T17:15:48.970 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.970 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.970 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:48.970 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410296+0000", 2024-08-17T17:15:48.970 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.224e-05 2024-08-17T17:15:48.970 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.971 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.971 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.971 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.384191+0000", 2024-08-17T17:15:48.971 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97389498100000005 2024-08-17T17:15:48.971 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.971 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.971 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:48.971 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.386158+0000", 2024-08-17T17:15:48.971 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0019675019999999999 2024-08-17T17:15:48.971 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.971 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.972 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:15:48.972 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.386237+0000", 2024-08-17T17:15:48.972 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9043999999999997e-05 2024-08-17T17:15:48.972 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.972 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.972 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.972 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.397623+0000", 2024-08-17T17:15:48.972 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011385637000000001 2024-08-17T17:15:48.972 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.972 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.972 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:48.973 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.399643+0000", 2024-08-17T17:15:48.973 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0020198669999999998 2024-08-17T17:15:48.973 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.973 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.973 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.973 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402647+0000", 2024-08-17T17:15:48.973 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0030044149999999999 2024-08-17T17:15:48.974 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.974 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.974 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:48.974 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402697+0000", 2024-08-17T17:15:48.974 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0158999999999999e-05 2024-08-17T17:15:48.975 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.975 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.975 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:48.975 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402733+0000", 2024-08-17T17:15:48.975 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5327000000000002e-05 2024-08-17T17:15:48.975 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.975 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:48.975 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.975 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.975 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.976 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131760 6.9 6:97a252f6:::benchmark_data_smithi063_160268_object5653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:48.976 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.411623+0000", 2024-08-17T17:15:48.976 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 45.456742110999997, 2024-08-17T17:15:48.976 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.001959477, 2024-08-17T17:15:48.976 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:48.976 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:48.976 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:48.976 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:48.976 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:48.976 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131760 2024-08-17T17:15:48.977 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.977 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:48.977 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.977 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:48.977 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411623+0000", 2024-08-17T17:15:48.977 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.978 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.978 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.978 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:48.978 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411623+0000", 2024-08-17T17:15:48.978 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.978 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.979 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.979 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:48.979 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411623+0000", 2024-08-17T17:15:48.979 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:15:48.979 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.979 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.980 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:48.980 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411637+0000", 2024-08-17T17:15:48.980 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4090000000000001e-05 2024-08-17T17:15:48.980 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.980 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.980 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:48.980 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411644+0000", 2024-08-17T17:15:48.980 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6580000000000002e-06 2024-08-17T17:15:48.981 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.981 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.981 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:48.981 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411671+0000", 2024-08-17T17:15:48.981 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6797000000000001e-05 2024-08-17T17:15:48.981 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.981 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.981 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.981 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411722+0000", 2024-08-17T17:15:48.981 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0883000000000001e-05 2024-08-17T17:15:48.982 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.982 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.982 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:48.982 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411732+0000", 2024-08-17T17:15:48.982 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8879999999999992e-06 2024-08-17T17:15:48.982 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.982 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.982 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.982 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.388963+0000", 2024-08-17T17:15:48.982 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.977230873 2024-08-17T17:15:48.983 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.983 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.983 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:48.983 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.389067+0000", 2024-08-17T17:15:48.983 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010451 2024-08-17T17:15:48.983 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.983 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.983 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:15:48.983 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.389138+0000", 2024-08-17T17:15:48.983 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0607999999999999e-05 2024-08-17T17:15:48.983 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.983 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.984 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:48.984 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402743+0000", 2024-08-17T17:15:48.984 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.013605446 2024-08-17T17:15:48.984 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.984 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.984 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.984 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402812+0000", 2024-08-17T17:15:48.984 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8089999999999994e-05 2024-08-17T17:15:48.984 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.984 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.984 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.984 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.413532+0000", 2024-08-17T17:15:48.984 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010720929000000001 2024-08-17T17:15:48.985 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.985 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.985 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:48.985 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.413561+0000", 2024-08-17T17:15:48.985 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8481999999999999e-05 2024-08-17T17:15:48.985 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.985 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.985 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:48.985 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.413583+0000", 2024-08-17T17:15:48.985 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.1897999999999999e-05 2024-08-17T17:15:48.985 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.985 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:48.985 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.986 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.986 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.986 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131761 6.9 6:976cb822:::benchmark_data_smithi063_160268_object5654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:48.986 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.417190+0000", 2024-08-17T17:15:48.986 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 45.45117596, 2024-08-17T17:15:48.986 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.000269098, 2024-08-17T17:15:48.986 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:48.986 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:48.987 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:48.987 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:48.987 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:48.987 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131761 2024-08-17T17:15:48.987 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.987 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:48.987 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.987 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:48.987 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417190+0000", 2024-08-17T17:15:48.987 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.987 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.987 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.987 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:48.988 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417190+0000", 2024-08-17T17:15:48.988 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.988 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.988 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.988 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:48.988 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417188+0000", 2024-08-17T17:15:48.988 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:48.988 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.988 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.988 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:48.988 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417194+0000", 2024-08-17T17:15:48.988 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7169999999999996e-06 2024-08-17T17:15:48.988 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.989 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.989 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:48.989 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417206+0000", 2024-08-17T17:15:48.989 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1406999999999999e-05 2024-08-17T17:15:48.989 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.989 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.989 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:48.989 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417212+0000", 2024-08-17T17:15:48.989 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3450000000000002e-06 2024-08-17T17:15:48.989 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.989 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.989 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.990 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417272+0000", 2024-08-17T17:15:48.990 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9973999999999999e-05 2024-08-17T17:15:48.990 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.990 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.990 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:48.990 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417281+0000", 2024-08-17T17:15:48.990 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3959999999999995e-06 2024-08-17T17:15:48.990 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.990 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.990 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.990 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.391578+0000", 2024-08-17T17:15:48.991 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97429638900000004 2024-08-17T17:15:48.991 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.991 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.991 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:48.991 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.391644+0000", 2024-08-17T17:15:48.991 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6518000000000002e-05 2024-08-17T17:15:48.991 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.991 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.991 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:15:48.991 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.391702+0000", 2024-08-17T17:15:48.991 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8319000000000002e-05 2024-08-17T17:15:48.991 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.992 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.992 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:48.992 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402748+0000", 2024-08-17T17:15:48.992 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011045589999999999 2024-08-17T17:15:48.992 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.992 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.992 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.992 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.413608+0000", 2024-08-17T17:15:48.992 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010860017 2024-08-17T17:15:48.992 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.992 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.992 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:48.992 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.417390+0000", 2024-08-17T17:15:48.993 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0037816780000000001 2024-08-17T17:15:48.993 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.993 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.993 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:48.993 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.417431+0000", 2024-08-17T17:15:48.993 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1054999999999999e-05 2024-08-17T17:15:48.993 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.993 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.993 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:48.993 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.417459+0000", 2024-08-17T17:15:48.993 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.7793999999999999e-05 2024-08-17T17:15:48.993 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.994 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:48.994 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:48.994 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.994 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.994 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131788 6.9 6:90417142:::benchmark_data_smithi063_160268_object5681:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:48.994 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.447657+0000", 2024-08-17T17:15:48.994 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 45.420708255000001, 2024-08-17T17:15:48.994 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97105383700000003, 2024-08-17T17:15:48.994 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:48.995 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:48.995 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:48.995 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:48.995 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:48.995 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131788 2024-08-17T17:15:48.995 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.995 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:48.995 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.995 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:48.995 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447657+0000", 2024-08-17T17:15:48.995 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.995 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.996 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.996 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:48.996 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447657+0000", 2024-08-17T17:15:48.996 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:48.996 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.996 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.996 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:48.996 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447657+0000", 2024-08-17T17:15:48.996 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:15:48.996 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.996 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.996 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:48.997 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447660+0000", 2024-08-17T17:15:48.997 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9720000000000001e-06 2024-08-17T17:15:48.997 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.997 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.997 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:48.997 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447667+0000", 2024-08-17T17:15:48.997 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2069999999999998e-06 2024-08-17T17:15:48.997 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.997 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.997 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:48.997 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447672+0000", 2024-08-17T17:15:48.998 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.865e-06 2024-08-17T17:15:48.998 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.998 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.998 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.998 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447742+0000", 2024-08-17T17:15:48.998 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0279999999999998e-05 2024-08-17T17:15:48.998 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.998 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.998 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:48.998 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447751+0000", 2024-08-17T17:15:48.998 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.0650000000000005e-06 2024-08-17T17:15:48.999 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.999 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.999 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:48.999 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.393494+0000", 2024-08-17T17:15:48.999 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.94574265899999999 2024-08-17T17:15:48.999 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.999 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:48.999 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:48.999 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.393554+0000", 2024-08-17T17:15:48.999 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9993e-05 2024-08-17T17:15:48.999 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:48.999 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.000 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:15:49.000 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.393609+0000", 2024-08-17T17:15:49.000 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5299e-05 2024-08-17T17:15:49.000 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.000 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.000 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:49.000 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402754+0000", 2024-08-17T17:15:49.000 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0091449779999999998 2024-08-17T17:15:49.000 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.000 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.000 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:49.001 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.417295+0000", 2024-08-17T17:15:49.001 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.014540993 2024-08-17T17:15:49.001 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.001 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.001 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:49.001 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418646+0000", 2024-08-17T17:15:49.001 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001351229 2024-08-17T17:15:49.001 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.001 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.001 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:49.001 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418679+0000", 2024-08-17T17:15:49.001 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3148999999999997e-05 2024-08-17T17:15:49.002 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.002 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.002 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:49.002 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418711+0000", 2024-08-17T17:15:49.002 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1727999999999998e-05 2024-08-17T17:15:49.002 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:49.002 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:49.002 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:49.002 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.002 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.002 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131792 6.9 6:925a7a40:::benchmark_data_smithi063_160268_object5685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:49.002 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.449563+0000", 2024-08-17T17:15:49.003 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 45.418802518, 2024-08-17T17:15:49.003 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.969275954, 2024-08-17T17:15:49.003 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:49.003 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:49.003 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:49.003 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:49.003 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:49.003 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131792 2024-08-17T17:15:49.003 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.003 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:49.003 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.003 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:49.003 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449563+0000", 2024-08-17T17:15:49.004 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:49.004 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.004 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.004 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:49.004 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449563+0000", 2024-08-17T17:15:49.004 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:49.004 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.004 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.004 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:49.004 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449562+0000", 2024-08-17T17:15:49.004 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:15:49.004 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.005 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.005 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:49.005 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449565+0000", 2024-08-17T17:15:49.005 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6599999999999999e-06 2024-08-17T17:15:49.005 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.005 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.005 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:49.005 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449571+0000", 2024-08-17T17:15:49.005 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8760000000000002e-06 2024-08-17T17:15:49.005 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.005 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.005 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:49.006 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449575+0000", 2024-08-17T17:15:49.006 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4599999999999996e-06 2024-08-17T17:15:49.006 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.006 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.006 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:49.006 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449662+0000", 2024-08-17T17:15:49.006 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6129000000000002e-05 2024-08-17T17:15:49.006 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.006 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.006 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:49.006 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449671+0000", 2024-08-17T17:15:49.006 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.5419999999999994e-06 2024-08-17T17:15:49.006 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.007 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.007 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:49.007 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.395313+0000", 2024-08-17T17:15:49.007 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.94564203300000005 2024-08-17T17:15:49.007 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.007 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.007 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:49.007 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.395380+0000", 2024-08-17T17:15:49.007 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6877000000000004e-05 2024-08-17T17:15:49.007 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.007 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.007 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:15:49.007 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.395451+0000", 2024-08-17T17:15:49.008 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0899e-05 2024-08-17T17:15:49.008 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.008 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.008 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:49.008 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402758+0000", 2024-08-17T17:15:49.008 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.007306692 2024-08-17T17:15:49.008 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.008 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.008 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:49.008 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.417323+0000", 2024-08-17T17:15:49.008 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.014565494999999999 2024-08-17T17:15:49.008 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.009 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.009 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:49.009 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418755+0000", 2024-08-17T17:15:49.009 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001432165 2024-08-17T17:15:49.009 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.009 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.009 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:49.009 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418797+0000", 2024-08-17T17:15:49.009 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1294999999999997e-05 2024-08-17T17:15:49.009 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.009 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.009 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:49.009 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418839+0000", 2024-08-17T17:15:49.010 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2318999999999999e-05 2024-08-17T17:15:49.010 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:49.010 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:49.010 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:49.010 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.010 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.010 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131802 6.9 6:9647ea40:::benchmark_data_smithi063_160268_object5695:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:49.010 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.459916+0000", 2024-08-17T17:15:49.010 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 45.408449015999999, 2024-08-17T17:15:49.010 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95901592300000005, 2024-08-17T17:15:49.010 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:49.010 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:49.011 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:49.011 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:49.011 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:49.011 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131802 2024-08-17T17:15:49.011 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.011 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:49.011 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.011 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:49.011 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.459916+0000", 2024-08-17T17:15:49.011 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:49.011 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.011 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.011 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:49.012 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.459916+0000", 2024-08-17T17:15:49.012 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:49.012 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.012 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.012 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:49.012 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.459915+0000", 2024-08-17T17:15:49.012 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:49.012 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.012 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.012 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:49.012 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.459920+0000", 2024-08-17T17:15:49.012 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1070000000000004e-06 2024-08-17T17:15:49.012 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.013 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.013 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:49.013 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.459936+0000", 2024-08-17T17:15:49.013 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5534000000000001e-05 2024-08-17T17:15:49.013 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.013 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.013 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:49.013 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.459943+0000", 2024-08-17T17:15:49.013 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0990000000000001e-06 2024-08-17T17:15:49.013 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.013 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.014 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:49.014 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.460001+0000", 2024-08-17T17:15:49.014 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8019000000000001e-05 2024-08-17T17:15:49.014 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.014 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.014 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:49.014 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.460011+0000", 2024-08-17T17:15:49.014 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9299999999999998e-06 2024-08-17T17:15:49.014 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.014 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.014 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:49.014 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.397658+0000", 2024-08-17T17:15:49.015 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.93764710799999995 2024-08-17T17:15:49.015 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.015 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.015 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:49.015 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.397738+0000", 2024-08-17T17:15:49.015 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9847999999999999e-05 2024-08-17T17:15:49.015 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.015 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.015 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:15:49.015 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.397800+0000", 2024-08-17T17:15:49.015 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2558000000000007e-05 2024-08-17T17:15:49.015 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.015 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.016 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:49.016 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.407472+0000", 2024-08-17T17:15:49.016 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0096714390000000004 2024-08-17T17:15:49.016 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.016 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.016 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:49.016 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.417362+0000", 2024-08-17T17:15:49.016 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0098902230000000001 2024-08-17T17:15:49.016 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.016 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.016 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:49.016 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418878+0000", 2024-08-17T17:15:49.017 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015162089999999999 2024-08-17T17:15:49.017 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.017 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.017 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:49.017 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418902+0000", 2024-08-17T17:15:49.017 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3910999999999999e-05 2024-08-17T17:15:49.017 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.017 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.017 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:49.017 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418932+0000", 2024-08-17T17:15:49.017 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.0127000000000001e-05 2024-08-17T17:15:49.017 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:49.018 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:49.018 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:49.018 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.018 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.018 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131816 6.9 6:9135a939:::benchmark_data_smithi063_160268_object5709:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:49.018 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.474321+0000", 2024-08-17T17:15:49.018 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 45.394043990999997, 2024-08-17T17:15:49.018 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95894761299999998, 2024-08-17T17:15:49.018 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:49.018 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:49.018 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:49.018 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:49.019 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:49.019 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131816 2024-08-17T17:15:49.019 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.019 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:49.019 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.019 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:49.019 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474321+0000", 2024-08-17T17:15:49.019 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:49.019 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.020 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.020 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:49.020 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474321+0000", 2024-08-17T17:15:49.020 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:49.020 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.020 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.020 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:49.020 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474320+0000", 2024-08-17T17:15:49.020 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:49.020 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.020 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.020 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:49.021 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474326+0000", 2024-08-17T17:15:49.021 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1720000000000004e-06 2024-08-17T17:15:49.021 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.021 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.021 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:49.021 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474339+0000", 2024-08-17T17:15:49.021 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.323e-05 2024-08-17T17:15:49.021 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.021 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.021 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:49.021 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474346+0000", 2024-08-17T17:15:49.021 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.798e-06 2024-08-17T17:15:49.022 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.022 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.022 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:49.022 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474405+0000", 2024-08-17T17:15:49.022 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8764999999999997e-05 2024-08-17T17:15:49.022 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.022 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.022 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:49.022 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474415+0000", 2024-08-17T17:15:49.022 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0265000000000001e-05 2024-08-17T17:15:49.022 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.023 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.023 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:49.023 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.399696+0000", 2024-08-17T17:15:49.023 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.925280926 2024-08-17T17:15:49.023 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.023 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.023 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:49.023 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.399767+0000", 2024-08-17T17:15:49.023 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1382000000000002e-05 2024-08-17T17:15:49.024 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.024 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.024 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:15:49.024 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.399830+0000", 2024-08-17T17:15:49.024 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2318999999999997e-05 2024-08-17T17:15:49.024 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.025 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.025 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:49.025 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.407480+0000", 2024-08-17T17:15:49.025 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0076499940000000002 2024-08-17T17:15:49.025 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.025 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.025 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:49.025 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.428141+0000", 2024-08-17T17:15:49.026 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.020661312000000001 2024-08-17T17:15:49.026 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.026 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.026 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:49.026 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433189+0000", 2024-08-17T17:15:49.026 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0050483239999999999 2024-08-17T17:15:49.026 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.026 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.026 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:49.026 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433234+0000", 2024-08-17T17:15:49.026 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4511000000000001e-05 2024-08-17T17:15:49.027 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.027 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.027 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:49.027 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433269+0000", 2024-08-17T17:15:49.027 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5154000000000003e-05 2024-08-17T17:15:49.027 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:49.027 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:49.027 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:49.027 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.027 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.027 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131821 6.9 6:9cc10ae1:::benchmark_data_smithi063_160268_object5714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:49.027 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.481045+0000", 2024-08-17T17:15:49.027 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 45.387320006000003, 2024-08-17T17:15:49.028 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95235498799999996, 2024-08-17T17:15:49.028 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:49.028 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:49.028 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:49.028 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:49.028 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:49.028 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131821 2024-08-17T17:15:49.028 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.028 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:49.028 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.028 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:49.028 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481045+0000", 2024-08-17T17:15:49.028 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:49.029 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.029 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.029 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:49.029 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481045+0000", 2024-08-17T17:15:49.029 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:49.029 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.029 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.029 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:49.029 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481044+0000", 2024-08-17T17:15:49.029 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:15:49.029 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.029 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.030 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:49.030 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481051+0000", 2024-08-17T17:15:49.030 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1489999999999997e-06 2024-08-17T17:15:49.030 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.030 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.030 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:49.030 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481063+0000", 2024-08-17T17:15:49.030 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1209000000000001e-05 2024-08-17T17:15:49.030 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.030 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.030 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:49.031 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481070+0000", 2024-08-17T17:15:49.031 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9689999999999997e-06 2024-08-17T17:15:49.031 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.031 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.031 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:49.031 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481120+0000", 2024-08-17T17:15:49.031 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0664000000000001e-05 2024-08-17T17:15:49.031 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.031 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.031 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:49.031 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481131+0000", 2024-08-17T17:15:49.032 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0788000000000001e-05 2024-08-17T17:15:49.032 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.032 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.032 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:49.032 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402839+0000", 2024-08-17T17:15:49.032 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.92170828100000002 2024-08-17T17:15:49.032 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.032 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.033 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:49.033 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402908+0000", 2024-08-17T17:15:49.033 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8615999999999995e-05 2024-08-17T17:15:49.033 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.033 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.033 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:15:49.033 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.403002+0000", 2024-08-17T17:15:49.033 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3941000000000001e-05 2024-08-17T17:15:49.033 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.033 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.033 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:49.033 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.412030+0000", 2024-08-17T17:15:49.033 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0090276850000000006 2024-08-17T17:15:49.034 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.034 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.034 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:49.034 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.432479+0000", 2024-08-17T17:15:49.034 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.020449833000000001 2024-08-17T17:15:49.034 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.034 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.034 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:49.034 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433318+0000", 2024-08-17T17:15:49.035 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00083902200000000003 2024-08-17T17:15:49.035 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.035 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.035 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:49.035 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433364+0000", 2024-08-17T17:15:49.035 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5089000000000001e-05 2024-08-17T17:15:49.035 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.035 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.035 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:49.035 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433400+0000", 2024-08-17T17:15:49.035 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6933e-05 2024-08-17T17:15:49.035 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:49.036 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:49.036 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:49.036 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.036 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.036 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131837 6.9 6:9b5d10c1:::benchmark_data_smithi063_160268_object5730:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:49.036 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.510534+0000", 2024-08-17T17:15:49.036 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 45.357831486000002, 2024-08-17T17:15:49.036 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.92298864199999997, 2024-08-17T17:15:49.036 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:49.037 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:49.037 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:49.037 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:49.037 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:49.037 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131837 2024-08-17T17:15:49.037 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.037 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:49.037 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.037 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:49.037 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510534+0000", 2024-08-17T17:15:49.037 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:49.038 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.038 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.038 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:49.038 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510534+0000", 2024-08-17T17:15:49.038 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:49.038 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.038 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.038 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:49.038 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510533+0000", 2024-08-17T17:15:49.038 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:15:49.038 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.038 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.039 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:49.039 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510540+0000", 2024-08-17T17:15:49.039 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3889999999999999e-06 2024-08-17T17:15:49.039 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.039 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.039 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:49.039 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510556+0000", 2024-08-17T17:15:49.039 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6371999999999998e-05 2024-08-17T17:15:49.039 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.039 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.039 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:49.039 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510564+0000", 2024-08-17T17:15:49.039 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.537e-06 2024-08-17T17:15:49.040 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.040 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.040 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:49.040 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510630+0000", 2024-08-17T17:15:49.040 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6227000000000001e-05 2024-08-17T17:15:49.040 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.040 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.040 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:49.040 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510638+0000", 2024-08-17T17:15:49.040 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.259e-06 2024-08-17T17:15:49.040 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.040 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.041 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:49.041 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.407509+0000", 2024-08-17T17:15:49.041 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.896870838 2024-08-17T17:15:49.041 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.041 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.041 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:49.041 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.407578+0000", 2024-08-17T17:15:49.041 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8706000000000003e-05 2024-08-17T17:15:49.041 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.041 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.041 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:15:49.041 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.407638+0000", 2024-08-17T17:15:49.041 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0476000000000002e-05 2024-08-17T17:15:49.042 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.042 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.042 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:49.042 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.412032+0000", 2024-08-17T17:15:49.042 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.004393816 2024-08-17T17:15:49.042 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.042 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.042 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:49.042 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.432524+0000", 2024-08-17T17:15:49.042 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.020491366 2024-08-17T17:15:49.042 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.043 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.043 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:49.043 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433446+0000", 2024-08-17T17:15:49.043 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00092251300000000002 2024-08-17T17:15:49.043 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.043 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.043 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:49.043 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433487+0000", 2024-08-17T17:15:49.043 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0766000000000002e-05 2024-08-17T17:15:49.043 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.043 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.044 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:49.044 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433523+0000", 2024-08-17T17:15:49.044 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5707000000000002e-05 2024-08-17T17:15:49.044 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:49.044 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:49.044 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:49.044 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.044 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.044 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131865 6.9 6:9e0f52b2:::benchmark_data_smithi063_160268_object5758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:15:49.045 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.606289+0000", 2024-08-17T17:15:49.045 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 45.262076585000003, 2024-08-17T17:15:49.045 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.82736346199999999, 2024-08-17T17:15:49.045 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:15:49.045 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:15:49.045 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:15:49.045 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:15:49.045 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:15:49.045 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131865 2024-08-17T17:15:49.045 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.045 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:15:49.045 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.046 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:15:49.046 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606289+0000", 2024-08-17T17:15:49.046 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:49.046 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.046 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.046 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:15:49.046 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606289+0000", 2024-08-17T17:15:49.046 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:15:49.046 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.046 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.046 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:15:49.046 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606287+0000", 2024-08-17T17:15:49.047 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999976 2024-08-17T17:15:49.047 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.047 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.047 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:15:49.047 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606297+0000", 2024-08-17T17:15:49.047 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0429e-05 2024-08-17T17:15:49.048 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.048 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.048 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:15:49.048 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606317+0000", 2024-08-17T17:15:49.048 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0228e-05 2024-08-17T17:15:49.048 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.048 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.048 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:15:49.048 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606328+0000", 2024-08-17T17:15:49.048 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0409e-05 2024-08-17T17:15:49.048 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.049 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.049 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:49.049 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606381+0000", 2024-08-17T17:15:49.049 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3000000000000001e-05 2024-08-17T17:15:49.049 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.049 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.049 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:15:49.049 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606390+0000", 2024-08-17T17:15:49.049 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.1400000000000006e-06 2024-08-17T17:15:49.049 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.049 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.049 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:15:49.050 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.410180+0000", 2024-08-17T17:15:49.050 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.80379011099999997 2024-08-17T17:15:49.050 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.050 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.050 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:15:49.050 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.410245+0000", 2024-08-17T17:15:49.050 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5073999999999994e-05 2024-08-17T17:15:49.050 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.050 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.050 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:15:49.050 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.410306+0000", 2024-08-17T17:15:49.050 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1377999999999994e-05 2024-08-17T17:15:49.051 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.051 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.051 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:15:49.051 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.412153+0000", 2024-08-17T17:15:49.051 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00184633 2024-08-17T17:15:49.051 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.051 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.051 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:49.051 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.432552+0000", 2024-08-17T17:15:49.051 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.020399245 2024-08-17T17:15:49.051 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.051 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.052 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:15:49.052 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433572+0000", 2024-08-17T17:15:49.052 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001020139 2024-08-17T17:15:49.052 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.052 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.052 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:15:49.052 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433616+0000", 2024-08-17T17:15:49.052 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3988e-05 2024-08-17T17:15:49.052 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:15:49.053 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:15:49.053 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:15:49.053 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433652+0000", 2024-08-17T17:15:49.053 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6353000000000002e-05 2024-08-17T17:15:49.053 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:49.053 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:49.053 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:49.053 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:15:49.053 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:15:49.054 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:15:49.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:49.056 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:49.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:49.157 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:49.185 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:49.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:49.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:49.286 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:49.326 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:49.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:49.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:49.428 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:49.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:49.471 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:49.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:49.573 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:49.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:49.607 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:49.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:49.708 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:49.753 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:49.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:49.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:49.855 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:49.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:49.884 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:49.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:49.985 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:50.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:50.018 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:50.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:49 smithi063 ceph-mon[114756]: pgmap v1228: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-08-17T17:15:50.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:49 smithi063 ceph-mon[109843]: pgmap v1228: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-08-17T17:15:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:49 smithi162 ceph-mon[104668]: pgmap v1228: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-08-17T17:15:50.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:50.159 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:50.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:50.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:50.201 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:15:50.262 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:50.289 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:50.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:50.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:50.391 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:50.414 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:50.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:50.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:50.516 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:50.547 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:50.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:50.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:50.648 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:50.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:50.683 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:50.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:50.741 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:15:50.783 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:50.821 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:50.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:50.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:50.922 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:50.947 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:50.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:50.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:51.049 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:51.086 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:51.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:51.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:51.189 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:51.218 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:51.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:51.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:51.320 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:51.376 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:51.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:51.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:51.478 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:51.505 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:51.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:51.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:51.607 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:51.657 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:51.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:51.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:51.759 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:51.794 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:51.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:51.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:51.896 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:51.935 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:51.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:51.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:51 smithi063 ceph-mon[114756]: pgmap v1229: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2024-08-17T17:15:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:51 smithi063 ceph-mon[109843]: pgmap v1229: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2024-08-17T17:15:52.037 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:52.062 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:52.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:52.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:52.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:51 smithi162 ceph-mon[104668]: pgmap v1229: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2024-08-17T17:15:52.164 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:52.198 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:52.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:52.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:52.300 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:52.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:52.332 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:52.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:52.433 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:52.471 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:52.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:52.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:52.573 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:52.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:52.614 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:52.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:52.656 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.1 reset_purged_snaps_last 2024-08-17T17:15:52.715 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:52.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:52.758 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:52.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:52.859 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:52.889 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:52.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:52.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:52.991 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:53.016 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:53.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:53.070 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:53.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:53.171 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:53.211 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:53.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:53.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:53.313 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:53.346 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:53.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:53.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:53.448 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:53.468 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:53.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:53.490 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:53.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:53.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:15:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:15:53] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.51.0" 2024-08-17T17:15:53.591 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:53.644 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:53.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:53.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:53.713 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:15:53.746 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:53.775 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:53.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:53.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:53.877 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:53.912 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:53 smithi063 ceph-mon[109843]: pgmap v1230: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:15:53.912 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:53 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1794753107' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:53.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:53 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:53.913 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:53 smithi063 ceph-mon[114756]: pgmap v1230: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:15:53.913 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:53 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1794753107' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:53.913 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:53 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:53.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:53.916 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:53.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:54.018 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:54.055 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:54.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:54.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:53 smithi162 ceph-mon[104668]: pgmap v1230: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:15:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:53 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1794753107' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:53 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:15:54.111 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:15:54.157 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:54.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:54.201 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:54.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:54.302 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:54.331 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:54.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:54.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:54.433 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:54.464 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:54.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:54.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:54.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:54.619 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:54.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:54.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:54.721 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:54.762 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:54.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:54.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:54.864 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:54.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:54.898 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:54.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:54.999 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:55.006 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:54 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:15:55.007 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:54 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:15:55.007 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:54 smithi063 ceph-mon[114756]: osdmap e1112: 8 total, 8 up, 8 in 2024-08-17T17:15:55.007 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:54 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:15:55.007 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:54 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:15:55.007 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:54 smithi063 ceph-mon[109843]: osdmap e1112: 8 total, 8 up, 8 in 2024-08-17T17:15:55.087 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:55.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:55.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:54 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:15:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:54 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:15:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:54 smithi162 ceph-mon[104668]: osdmap e1112: 8 total, 8 up, 8 in 2024-08-17T17:15:55.189 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:55.241 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:55.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:55.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:55.343 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:55.380 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:55.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:55.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:55.482 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:55.508 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:55.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:55.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:55.610 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:55.662 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:55.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:55.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:55.764 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:55.808 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:55.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:55.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:55.911 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:55.945 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:55.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:55.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:55 smithi063 ceph-mon[109843]: pgmap v1232: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-08-17T17:15:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:55 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:55 smithi063 ceph-mon[114756]: pgmap v1232: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-08-17T17:15:56.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:55 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:56.047 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:56.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:56.087 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:56.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:55 smithi162 ceph-mon[104668]: pgmap v1232: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-08-17T17:15:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:55 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:15:56.187 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:56.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:56.224 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:56.225 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:56.326 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:56.361 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:56.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:56.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:56.463 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:56.501 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:15:56.502 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:15:56.503 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:56.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:56.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:56.543 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:15:56.543 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:15:56.582 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:15:56.582 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:15:56.597 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:15:56.606 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:56.626 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:15:56.626 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:15:56.661 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:15:56.662 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:15:56.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:56.680 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:56.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:56.698 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:15:56.698 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:15:56.756 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:15:56.756 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:15:56.782 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:56.873 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:15:56.873 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:15:56.873 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:56 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2381052700' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:56.874 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:56 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2381052700' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:56.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:56.910 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:56.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:57.011 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:57.023 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:56 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2381052700' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:15:57.052 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:57.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:57.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:57.154 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:57.195 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:57.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:57.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:57.296 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:57.330 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:57.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:57.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:57.432 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:57.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:57.481 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:57.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:57.582 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:15:57.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:57.626 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:57.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:15:57.727 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:57.771 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:57.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:57.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:57.873 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:57.909 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:57.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:58.011 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:58.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:57 smithi063 ceph-mon[114756]: pgmap v1233: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-08-17T17:15:58.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:57 smithi063 ceph-mon[109843]: pgmap v1233: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-08-17T17:15:58.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:58.039 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:58.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:57 smithi162 ceph-mon[104668]: pgmap v1233: 89 pgs: 89 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-08-17T17:15:58.140 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:58.169 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:58.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:58.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:58.270 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:58.313 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:58.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:58.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:58.415 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:58.451 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:58.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:58.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:58.553 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:58.583 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:58.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:58.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:58.684 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:58.718 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:58.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:58.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:58.820 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:58.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:58.859 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:58.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:58.960 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:15:59.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:59.015 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:59.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:15:59.116 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:15:59.145 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:59.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:59.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:15:59.247 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:15:59.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:59.297 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:59.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:15:59.398 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:59.427 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:59.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:59.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:15:59.528 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:15:59.570 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:59.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:59.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:15:59.672 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:15:59.707 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:15:59.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:59.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:15:59.809 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:15:59.843 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:59.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:59.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:15:59.872 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:15:59 smithi063 ceph-mon[114756]: pgmap v1234: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2024-08-17T17:15:59.873 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:15:59 smithi063 ceph-mon[109843]: pgmap v1234: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2024-08-17T17:15:59.945 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:15:59.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:15:59.979 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:15:59.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:00.013 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:15:59 smithi162 ceph-mon[104668]: pgmap v1234: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2024-08-17T17:16:00.080 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:00.107 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:16:00.108 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:16:00.117 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:00.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:00.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:00.219 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:00.259 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:00.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:00.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:00.361 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:00.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:00.406 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:00.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:00.507 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:00.520 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:16:00.540 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:00.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:00.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:00.642 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:00.666 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:00.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:00.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:00.767 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:00.812 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:00.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:00.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:00.914 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:00.953 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:00.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:00.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:01.055 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:01.086 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:01.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:01.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:01.188 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:01.213 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:01.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:01.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:01.262 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.2 reset_purged_snaps_last 2024-08-17T17:16:01.315 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:01.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:01.394 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:01.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:01.495 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:01.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:01.543 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:01.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:01.644 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:01.686 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:01.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:01.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:01.729 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:16:01.745 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:01.788 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:01.813 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:01.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:01.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:01.854 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:01.915 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:01.943 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:01.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:01.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:01.948 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:01 smithi063 ceph-mon[109843]: pgmap v1235: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-08-17T17:16:01.948 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:01.949 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:01 smithi063 ceph-mon[114756]: pgmap v1235: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-08-17T17:16:01.949 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:02.045 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:02.070 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:02.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:02.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:01 smithi162 ceph-mon[104668]: pgmap v1235: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-08-17T17:16:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:02.172 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:02.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:02.225 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:02.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:02.326 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:02.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:02.366 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:02.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:02.467 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:02.516 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:02.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:02.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:02.618 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:02.640 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:02.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:02.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:02.741 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:02.777 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:02.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:02.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:02.879 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:02.917 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:02.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:02.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:03.019 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:03.058 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:03.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:03.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:03.160 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:03.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:03.194 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:03.195 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:03.296 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:03.342 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:03.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:03.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:03.375 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:16:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:16:03] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.51.0" 2024-08-17T17:16:03.445 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:03.495 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:03.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:03.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:03.596 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:03.630 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:03.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:03.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:03.732 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:03.765 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:03.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:03.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:03.867 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:03.894 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:03.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:03.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:03.910 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:03.913 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:03.996 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:04.016 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:03 smithi063 ceph-mon[114756]: pgmap v1236: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-08-17T17:16:04.016 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:03 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:04.016 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:03 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:04.017 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:03 smithi063 ceph-mon[109843]: pgmap v1236: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-08-17T17:16:04.017 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:03 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:04.017 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:03 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:04.021 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:04.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:04.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:04.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:03 smithi162 ceph-mon[104668]: pgmap v1236: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-08-17T17:16:04.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:03 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:04.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:03 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:04.122 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:04.165 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:04.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:04.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:04.267 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:04.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:04.307 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:04.307 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:04.408 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:04.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:04.457 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:04.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:04.558 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:04.598 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:04.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:04.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:04.700 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:04.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:04.741 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:04.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:04.842 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:04.891 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:04.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:04.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:04.993 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:05.023 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:05.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:05.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:05.124 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:05.159 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:05.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:05.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:05.261 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:05.304 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:05.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:05.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:05.405 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:05.441 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:05.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:05.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:05.543 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:05.574 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:05.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:05.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:05.676 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:05.710 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:05.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:05.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:05.812 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:05.846 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:05.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:05.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:05.948 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:05.984 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:05.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:05.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:06.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:05 smithi063 ceph-mon[109843]: pgmap v1237: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-08-17T17:16:06.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:05 smithi063 ceph-mon[114756]: pgmap v1237: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-08-17T17:16:06.086 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:05 smithi162 ceph-mon[104668]: pgmap v1237: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-08-17T17:16:06.115 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:06.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:06.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:06.217 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:06.249 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:06.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:06.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:06.351 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:06.375 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:06.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:06.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:06.476 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:06.480 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:16:06.480 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:16:06.517 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:16:06.517 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:16:06.518 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:06.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:06.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:06.558 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:16:06.558 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:16:06.592 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:16:06.592 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:16:06.620 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:06.625 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:16:06.625 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:16:06.665 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:16:06.665 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:16:06.671 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:06.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:06.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:06.704 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:16:06.704 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:16:06.742 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:16:06.742 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:16:06.773 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:06.828 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:06.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:06.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:06.931 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:06.958 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:06.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:06.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:07.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:06 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2856152029' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:16:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:06 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2856152029' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:16:07.060 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:06 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2856152029' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:16:07.105 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:07.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:07.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:07.207 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:07.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:07.251 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:07.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:07.352 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:07.380 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:07.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:07.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:07.482 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:07.525 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:07.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:07.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:07.627 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:07.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:07.657 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:07.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:07.758 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:07.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:07.790 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:07.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:07.791 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:16:07.891 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:07.920 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:07.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:07.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:08.022 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:08.062 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:08.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:08.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:08.086 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:07 smithi063 ceph-mon[109843]: pgmap v1238: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-08-17T17:16:08.086 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:07 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/82247160' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:16:08.086 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:07 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:16:08.086 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:07 smithi063 ceph-mon[114756]: pgmap v1238: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-08-17T17:16:08.086 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:07 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/82247160' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:16:08.086 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:07 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:16:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:07 smithi162 ceph-mon[104668]: pgmap v1238: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2024-08-17T17:16:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:07 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/82247160' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:16:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:07 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:16:08.164 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:08.210 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:08.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:08.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:08.312 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:08.347 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:08.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:08.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:08.449 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:08.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:08.487 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:08.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:08.589 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:08.616 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:08.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:08.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:08.718 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:08.760 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:08.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:08.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:08.862 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:08.899 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:08.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:08.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:09.000 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:09.030 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:09.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:09.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:09.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:08 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:16:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:08 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:16:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:08 smithi162 ceph-mon[104668]: osdmap e1113: 8 total, 8 up, 8 in 2024-08-17T17:16:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:08 smithi162 ceph-mon[104668]: pgmap v1240: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2024-08-17T17:16:09.132 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:09.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:09.178 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:09.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:09.205 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:08 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:16:09.205 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:08 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:16:09.205 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:08 smithi063 ceph-mon[114756]: osdmap e1113: 8 total, 8 up, 8 in 2024-08-17T17:16:09.205 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:08 smithi063 ceph-mon[114756]: pgmap v1240: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2024-08-17T17:16:09.206 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:08 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:16:09.206 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:08 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:16:09.206 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:08 smithi063 ceph-mon[109843]: osdmap e1113: 8 total, 8 up, 8 in 2024-08-17T17:16:09.206 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:08 smithi063 ceph-mon[109843]: pgmap v1240: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2024-08-17T17:16:09.279 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:09.322 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:09.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:09.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:09.423 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:09.477 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:09.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:09.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:09.579 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:09.607 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:09.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:09.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:09.709 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:09.736 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:09.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:09.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:09.786 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:16:09.786 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:16:09.786 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:16:09.786 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:16:09.786 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:16:09.838 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:09.878 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:09.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:09.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:09.979 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:10.024 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:10.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:10.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:10.126 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:10.154 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:10.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:10.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:10.256 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:10.301 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:10.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:10.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:10.402 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:10.441 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:10.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:10.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:10.521 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:16:10.542 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:10.568 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:10.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:10.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:10.577 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:16:10.669 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:10.694 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:10.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:10.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:10.796 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:10.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:10.827 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:10.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:10.928 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:10.975 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:10.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:10.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:11.077 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:11.119 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:11.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:11.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:11.221 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:11.256 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:11.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:11.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:11.358 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:11.385 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:11.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:11.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:11.486 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:11.496 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:11 smithi063 ceph-mon[114756]: pgmap v1241: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-08-17T17:16:11.496 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:11 smithi063 ceph-mon[109843]: pgmap v1241: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-08-17T17:16:11.510 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:11.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:11.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:11.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:11 smithi162 ceph-mon[104668]: pgmap v1241: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-08-17T17:16:11.612 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:11.639 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:11.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:11.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:11.741 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:11.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:11.785 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:11.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:11.886 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:11.935 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:11.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:11.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:12.037 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:12.063 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:12.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:12.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:12.165 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:12.207 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:12.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:12.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:12.309 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:12.347 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:12.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:12.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:12.449 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:12.475 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:12.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:12.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:12.516 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.3 reset_purged_snaps_last 2024-08-17T17:16:12.577 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:12.616 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:12.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:12.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:12.718 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:12.777 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:12.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:12.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:12.879 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:12.882 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:16:12.948 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:12.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:12.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:13.050 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:13.064 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:13.074 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:13.140 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:13.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:13.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:13.243 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:13.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:13 smithi063 ceph-mon[114756]: pgmap v1242: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-08-17T17:16:13.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:13 smithi063 ceph-mon[109843]: pgmap v1242: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-08-17T17:16:13.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:13.287 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:13.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:13.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:13 smithi162 ceph-mon[104668]: pgmap v1242: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 729 op/s 2024-08-17T17:16:13.388 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:13.420 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:13.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:13.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:13.522 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:13.548 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:13.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:13.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:13.592 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:16:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:16:13] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.51.0" 2024-08-17T17:16:13.649 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:13.680 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:13.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:13.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:13.770 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:13.782 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:13.808 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:13.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:13.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:13.910 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:13.958 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:13.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:13.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:14.060 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:14.088 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:14.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:14.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:14.190 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:14.231 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:14.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:14.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:14.333 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:14.377 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:14.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:14.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:14.478 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:14.530 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:14.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:14.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:14.631 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:14.655 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:14.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:14.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:14.757 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:14.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:14.798 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:14.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:14.899 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:14.926 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:14.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:14.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:15.028 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:15.070 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:15.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:15.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:15.172 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:15.216 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:15.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:15.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:15.318 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:15.361 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:15.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:15.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:15.463 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:15.493 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:15.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:15.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:15.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:15 smithi063 ceph-mon[114756]: pgmap v1243: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-08-17T17:16:15.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:15 smithi063 ceph-mon[109843]: pgmap v1243: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-08-17T17:16:15.595 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:15.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:15 smithi162 ceph-mon[104668]: pgmap v1243: 89 pgs: 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-08-17T17:16:15.625 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:15.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:15.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:15.727 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:15.762 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:15.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:15.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:15.864 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:15.892 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:15.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:15.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:15.994 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:16.028 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:16.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:16.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:16.130 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:16.167 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:16.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:16.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:16.268 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:16.296 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:16.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:16.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:16.397 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:16.445 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:16.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:16.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:16.547 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:16.574 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:16.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:16.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:16.676 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:16.726 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:16.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:16.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:16.828 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:16.858 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:16.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:16.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:16.960 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:16.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:17.000 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:17.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:17.101 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:17.126 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:17.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:17.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:17.228 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:17.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:17.278 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:17.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:17.379 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:17.428 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:17.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:17.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:17.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:17 smithi063 ceph-mon[109843]: pgmap v1244: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-08-17T17:16:17.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:17 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:17.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:17 smithi063 ceph-mon[114756]: pgmap v1244: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-08-17T17:16:17.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:17 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:17.530 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:17.566 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:17.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:17.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:17.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:17 smithi162 ceph-mon[104668]: pgmap v1244: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-08-17T17:16:17.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:17 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:17.668 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:17.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:17.699 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:17.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:17.800 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:17.827 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:17.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:17.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:17.929 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:17.974 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:17.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:17.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:18.076 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:18.107 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:18.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:18.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:18.209 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:18.235 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:18.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:18.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:18.337 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:18.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:18.370 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:18.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:18.471 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:18.501 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:18.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:18.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:18.603 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:18.629 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:18.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:18.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:18.731 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:18.757 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:18.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:18.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:18.859 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:18.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:18.896 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:18.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:18.997 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:19.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:19.030 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:19.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:19.130 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:19.157 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:19.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:19.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:19.259 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:19.281 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:19.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:19.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:19.383 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:19.448 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:19.448 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:19 smithi063 ceph-mon[109843]: pgmap v1245: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-08-17T17:16:19.449 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:19 smithi063 ceph-mon[114756]: pgmap v1245: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-08-17T17:16:19.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:19.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:19.550 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:19.589 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:19.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:19.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:19.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:19 smithi162 ceph-mon[104668]: pgmap v1245: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 806 op/s 2024-08-17T17:16:19.692 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:19.721 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:19.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:19.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:19.823 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:19.853 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:19.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:19.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:19.955 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:19.989 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:19.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:19.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:20.091 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:20.118 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:20.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:20.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:20.220 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:20.250 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:20.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:20.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:20.352 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:20.393 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:20.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:20.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:20.460 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:16:20.460 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:16:20.460 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:16:20.460 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:16:20.461 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.461 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.14817.0:7338 5.b 5:daca9f56:::smithi06387500-12:head [delete] snapc 1f0=[1f0,1ef,1ee,1ec,1eb,1ea,1e9,1e8,1e7,1da,1d8,1cf] ondisk+write+peerstat_old+known_if_redirected+supports_pool_eio e792)", 2024-08-17T17:16:20.461 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:06:59.651546+0000", 2024-08-17T17:16:20.461 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 560.80533794300004, 2024-08-17T17:16:20.461 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.020823459, 2024-08-17T17:16:20.461 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.461 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.461 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.461 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:16:20.461 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.14817", 2024-08-17T17:16:20.461 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/1078629904", 2024-08-17T17:16:20.462 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 7338 2024-08-17T17:16:20.462 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.462 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.462 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.462 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.462 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651546+0000", 2024-08-17T17:16:20.462 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.462 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.462 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.462 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.462 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651546+0000", 2024-08-17T17:16:20.462 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.462 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.463 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.463 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.463 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651548+0000", 2024-08-17T17:16:20.463 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8619999999999999e-06 2024-08-17T17:16:20.463 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.463 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.463 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.465 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651554+0000", 2024-08-17T17:16:20.466 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2489999999999998e-06 2024-08-17T17:16:20.466 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.466 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.466 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.466 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651569+0000", 2024-08-17T17:16:20.466 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4355000000000001e-05 2024-08-17T17:16:20.466 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.466 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.466 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.466 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651580+0000", 2024-08-17T17:16:20.466 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0848e-05 2024-08-17T17:16:20.467 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.467 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.467 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.467 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651906+0000", 2024-08-17T17:16:20.467 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00032660600000000001 2024-08-17T17:16:20.467 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.467 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.467 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op must wait for map", 2024-08-17T17:16:20.467 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.651916+0000", 2024-08-17T17:16:20.467 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.6209999999999999e-06 2024-08-17T17:16:20.467 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.467 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.467 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.468 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.654283+0000", 2024-08-17T17:16:20.468 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0023670750000000002 2024-08-17T17:16:20.468 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.468 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.468 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:16:20.468 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:06:59.654292+0000", 2024-08-17T17:16:20.468 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.1139999999999995e-06 2024-08-17T17:16:20.468 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.468 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.468 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.468 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:00.670836+0000", 2024-08-17T17:16:20.468 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0165439119999999 2024-08-17T17:16:20.469 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.469 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.469 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.469 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:00.670958+0000", 2024-08-17T17:16:20.469 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012153 2024-08-17T17:16:20.469 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.469 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.469 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 5", 2024-08-17T17:16:20.469 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:00.671102+0000", 2024-08-17T17:16:20.469 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014450699999999999 2024-08-17T17:16:20.469 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.469 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.469 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:16:20.470 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:00.671883+0000", 2024-08-17T17:16:20.470 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00078096000000000003 2024-08-17T17:16:20.470 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.470 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.470 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.470 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:00.672275+0000", 2024-08-17T17:16:20.470 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00039224699999999997 2024-08-17T17:16:20.470 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.470 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.470 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.470 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:00.672313+0000", 2024-08-17T17:16:20.470 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7296999999999999e-05 2024-08-17T17:16:20.470 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.471 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.471 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.471 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:07:00.672370+0000", 2024-08-17T17:16:20.471 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7275999999999998e-05 2024-08-17T17:16:20.471 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.471 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.471 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.471 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.471 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.471 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:29976 6.9 6:9696ac29:::benchmark_data_smithi063_160268_object29975:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:16:20.471 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:37.728584+0000", 2024-08-17T17:16:20.471 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 162.72830034, 2024-08-17T17:16:20.471 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.076889863000000003, 2024-08-17T17:16:20.472 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.472 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.472 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.472 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:16:20.472 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:16:20.472 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:16:20.472 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 29976 2024-08-17T17:16:20.472 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.472 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.472 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.472 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.472 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728584+0000", 2024-08-17T17:16:20.472 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.473 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.473 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.473 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.473 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728584+0000", 2024-08-17T17:16:20.473 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.473 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.473 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.473 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.473 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728585+0000", 2024-08-17T17:16:20.473 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3900000000000004e-07 2024-08-17T17:16:20.473 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.473 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.473 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.473 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728604+0000", 2024-08-17T17:16:20.474 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8746e-05 2024-08-17T17:16:20.474 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.474 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.474 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.474 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728613+0000", 2024-08-17T17:16:20.474 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.662e-06 2024-08-17T17:16:20.474 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.474 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.474 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.474 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728618+0000", 2024-08-17T17:16:20.474 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6650000000000002e-06 2024-08-17T17:16:20.474 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.474 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.475 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.475 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728682+0000", 2024-08-17T17:16:20.475 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4083000000000003e-05 2024-08-17T17:16:20.475 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.475 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.475 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.475 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728765+0000", 2024-08-17T17:16:20.475 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3230000000000001e-05 2024-08-17T17:16:20.475 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.475 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.475 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:16:20.475 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728845+0000", 2024-08-17T17:16:20.475 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9689000000000002e-05 2024-08-17T17:16:20.476 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.476 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.476 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:16:20.476 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729565+0000", 2024-08-17T17:16:20.476 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00072059299999999995 2024-08-17T17:16:20.476 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.476 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.476 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.476 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.730712+0000", 2024-08-17T17:16:20.476 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0011460470000000001 2024-08-17T17:16:20.476 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.476 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.476 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.477 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.805360+0000", 2024-08-17T17:16:20.477 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.074648649999999997 2024-08-17T17:16:20.477 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.477 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.477 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.477 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.805426+0000", 2024-08-17T17:16:20.477 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5411000000000001e-05 2024-08-17T17:16:20.477 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.477 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.477 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.477 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.805474+0000", 2024-08-17T17:16:20.478 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8248000000000003e-05 2024-08-17T17:16:20.478 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.478 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.478 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.478 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.478 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.478 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:29978 6.9 6:9795df24:::benchmark_data_smithi063_160268_object29977:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:16:20.478 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:37.729544+0000", 2024-08-17T17:16:20.479 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 162.727340497, 2024-08-17T17:16:20.479 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.080494046999999999, 2024-08-17T17:16:20.479 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.479 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.479 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.479 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:16:20.479 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:16:20.479 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:16:20.479 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 29978 2024-08-17T17:16:20.479 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.479 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.479 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.479 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.480 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729544+0000", 2024-08-17T17:16:20.480 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.480 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.480 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.480 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.480 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729544+0000", 2024-08-17T17:16:20.480 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.480 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.480 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.480 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.480 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729545+0000", 2024-08-17T17:16:20.480 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.9899999999999999e-07 2024-08-17T17:16:20.481 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.481 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.481 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.481 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729566+0000", 2024-08-17T17:16:20.481 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1455e-05 2024-08-17T17:16:20.481 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.481 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.481 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.481 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729577+0000", 2024-08-17T17:16:20.481 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0804e-05 2024-08-17T17:16:20.481 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.481 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.482 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.482 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729583+0000", 2024-08-17T17:16:20.482 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4980000000000003e-06 2024-08-17T17:16:20.482 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.482 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.482 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.482 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729622+0000", 2024-08-17T17:16:20.482 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8441e-05 2024-08-17T17:16:20.482 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.482 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.482 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.482 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729697+0000", 2024-08-17T17:16:20.482 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4948000000000001e-05 2024-08-17T17:16:20.483 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.483 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.483 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:16:20.483 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729780+0000", 2024-08-17T17:16:20.483 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2812999999999995e-05 2024-08-17T17:16:20.483 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.483 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.483 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:16:20.483 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.730439+0000", 2024-08-17T17:16:20.483 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00065916200000000003 2024-08-17T17:16:20.483 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.483 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.483 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.484 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.730790+0000", 2024-08-17T17:16:20.484 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00035080499999999999 2024-08-17T17:16:20.484 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.484 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.484 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.484 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.809939+0000", 2024-08-17T17:16:20.484 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.079149580999999997 2024-08-17T17:16:20.484 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.484 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.484 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.484 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.809997+0000", 2024-08-17T17:16:20.484 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7386000000000002e-05 2024-08-17T17:16:20.484 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.485 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.485 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.485 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.810038+0000", 2024-08-17T17:16:20.485 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.1254999999999997e-05 2024-08-17T17:16:20.485 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.485 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.485 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.485 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.485 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.485 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34196 6.9 6:90093b33:::benchmark_data_smithi063_160268_object34195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:16:20.485 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.037012+0000", 2024-08-17T17:16:20.485 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 160.41987267900001, 2024-08-17T17:16:20.485 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.091924617, 2024-08-17T17:16:20.486 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.486 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.486 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.486 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:16:20.486 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:16:20.486 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:16:20.486 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34196 2024-08-17T17:16:20.486 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.486 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.486 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.486 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.486 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037012+0000", 2024-08-17T17:16:20.487 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.487 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.487 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.487 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.487 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037012+0000", 2024-08-17T17:16:20.487 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.487 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.487 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.487 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.487 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037013+0000", 2024-08-17T17:16:20.487 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1540000000000001e-06 2024-08-17T17:16:20.487 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.487 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.488 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.488 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037332+0000", 2024-08-17T17:16:20.488 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00031923299999999998 2024-08-17T17:16:20.488 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.488 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.488 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.488 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037344+0000", 2024-08-17T17:16:20.488 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1973999999999999e-05 2024-08-17T17:16:20.488 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.488 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.488 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.488 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037351+0000", 2024-08-17T17:16:20.488 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9609999999999999e-06 2024-08-17T17:16:20.489 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.489 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.489 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.489 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037649+0000", 2024-08-17T17:16:20.489 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00029795199999999998 2024-08-17T17:16:20.489 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.489 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.489 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.489 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037761+0000", 2024-08-17T17:16:20.489 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000112284 2024-08-17T17:16:20.489 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.489 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.489 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:16:20.490 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037889+0000", 2024-08-17T17:16:20.490 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012809900000000001 2024-08-17T17:16:20.490 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.490 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.490 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:16:20.490 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.039236+0000", 2024-08-17T17:16:20.490 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0013463940000000001 2024-08-17T17:16:20.490 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.490 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.490 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.490 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.041695+0000", 2024-08-17T17:16:20.490 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002459124 2024-08-17T17:16:20.490 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.491 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.491 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.491 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128820+0000", 2024-08-17T17:16:20.491 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.087125381000000002 2024-08-17T17:16:20.491 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.491 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.491 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.491 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128885+0000", 2024-08-17T17:16:20.491 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4826000000000006e-05 2024-08-17T17:16:20.491 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.491 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.491 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.491 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128936+0000", 2024-08-17T17:16:20.491 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1235e-05 2024-08-17T17:16:20.492 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.492 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.492 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.492 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.492 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.492 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34197 6.0 6:0173ef19:::benchmark_data_smithi063_160268_object34196:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:16:20.492 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.037483+0000", 2024-08-17T17:16:20.492 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 160.41940169200001, 2024-08-17T17:16:20.492 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.091051282999999997, 2024-08-17T17:16:20.492 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.492 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.493 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.493 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:16:20.493 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:16:20.493 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:16:20.493 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34197 2024-08-17T17:16:20.493 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.493 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.494 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.494 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.494 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037483+0000", 2024-08-17T17:16:20.494 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.494 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.494 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.494 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.494 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037483+0000", 2024-08-17T17:16:20.494 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.494 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.494 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.494 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.495 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037483+0000", 2024-08-17T17:16:20.495 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3099999999999997e-07 2024-08-17T17:16:20.495 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.495 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.495 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.495 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037533+0000", 2024-08-17T17:16:20.495 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9410000000000003e-05 2024-08-17T17:16:20.495 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.495 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.495 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.495 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037543+0000", 2024-08-17T17:16:20.495 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0292000000000001e-05 2024-08-17T17:16:20.496 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.496 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.496 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.496 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037549+0000", 2024-08-17T17:16:20.496 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.528e-06 2024-08-17T17:16:20.496 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.496 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.496 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.496 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.038545+0000", 2024-08-17T17:16:20.496 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00099573000000000001 2024-08-17T17:16:20.496 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.496 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.496 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.497 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.038670+0000", 2024-08-17T17:16:20.497 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000124387 2024-08-17T17:16:20.497 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.497 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.497 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:16:20.497 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.038803+0000", 2024-08-17T17:16:20.497 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000133199 2024-08-17T17:16:20.497 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.497 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.497 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:16:20.497 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.040082+0000", 2024-08-17T17:16:20.497 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001278922 2024-08-17T17:16:20.497 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.498 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.498 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.498 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042805+0000", 2024-08-17T17:16:20.498 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002723516 2024-08-17T17:16:20.498 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.498 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.498 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.498 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128419+0000", 2024-08-17T17:16:20.498 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.085614129999999997 2024-08-17T17:16:20.498 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.498 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.498 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.499 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128488+0000", 2024-08-17T17:16:20.499 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8233999999999998e-05 2024-08-17T17:16:20.499 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.499 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.499 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.499 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128534+0000", 2024-08-17T17:16:20.499 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6304000000000003e-05 2024-08-17T17:16:20.499 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.499 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.499 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.499 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.499 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.499 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34204 6.9 6:90e2ac6d:::benchmark_data_smithi063_160268_object34203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:16:20.500 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.039213+0000", 2024-08-17T17:16:20.500 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 160.41767099500001, 2024-08-17T17:16:20.500 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.089974211999999998, 2024-08-17T17:16:20.500 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.500 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.500 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.500 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:16:20.500 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:16:20.500 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:16:20.500 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34204 2024-08-17T17:16:20.500 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.500 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.500 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.501 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.501 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.039213+0000", 2024-08-17T17:16:20.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.501 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.501 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.501 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.501 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.039213+0000", 2024-08-17T17:16:20.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.501 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.501 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.501 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.501 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.039214+0000", 2024-08-17T17:16:20.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.2600000000000001e-07 2024-08-17T17:16:20.502 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.502 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.502 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.502 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.039238+0000", 2024-08-17T17:16:20.502 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3824000000000001e-05 2024-08-17T17:16:20.502 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.502 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.502 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.502 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.039248+0000", 2024-08-17T17:16:20.502 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9669999999999996e-06 2024-08-17T17:16:20.502 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.502 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.503 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.503 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.039255+0000", 2024-08-17T17:16:20.503 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7059999999999998e-06 2024-08-17T17:16:20.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.503 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.503 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.503 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.040287+0000", 2024-08-17T17:16:20.503 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0010324589999999999 2024-08-17T17:16:20.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.503 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.503 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.503 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.040402+0000", 2024-08-17T17:16:20.503 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011492 2024-08-17T17:16:20.504 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.504 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.504 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:16:20.504 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.040529+0000", 2024-08-17T17:16:20.504 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012688900000000001 2024-08-17T17:16:20.504 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.504 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.504 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:16:20.504 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.041867+0000", 2024-08-17T17:16:20.504 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0013376550000000001 2024-08-17T17:16:20.504 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.504 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.504 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.505 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128983+0000", 2024-08-17T17:16:20.505 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.087116491000000004 2024-08-17T17:16:20.505 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.505 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.505 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.505 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129096+0000", 2024-08-17T17:16:20.505 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000112509 2024-08-17T17:16:20.505 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.505 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.505 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.505 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129149+0000", 2024-08-17T17:16:20.505 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3325000000000002e-05 2024-08-17T17:16:20.505 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.506 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.506 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.506 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129188+0000", 2024-08-17T17:16:20.506 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8541000000000003e-05 2024-08-17T17:16:20.506 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.506 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.506 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.506 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.506 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.506 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34210 6.9 6:9155233c:::benchmark_data_smithi063_160268_object34209:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:16:20.506 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.042864+0000", 2024-08-17T17:16:20.506 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 160.414020609, 2024-08-17T17:16:20.506 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.087962027999999998, 2024-08-17T17:16:20.507 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.507 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.507 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.507 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:16:20.507 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:16:20.507 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:16:20.507 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34210 2024-08-17T17:16:20.507 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.507 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.507 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.507 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.507 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042864+0000", 2024-08-17T17:16:20.507 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.508 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.508 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.508 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.508 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042864+0000", 2024-08-17T17:16:20.508 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.508 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.508 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.508 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.508 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042865+0000", 2024-08-17T17:16:20.508 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.181e-06 2024-08-17T17:16:20.508 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.508 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.508 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.509 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042888+0000", 2024-08-17T17:16:20.509 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2993000000000002e-05 2024-08-17T17:16:20.509 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.509 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.509 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.509 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042899+0000", 2024-08-17T17:16:20.509 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1326e-05 2024-08-17T17:16:20.509 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.509 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.509 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.509 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042906+0000", 2024-08-17T17:16:20.509 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9199999999999998e-06 2024-08-17T17:16:20.509 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.510 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.510 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.510 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043115+0000", 2024-08-17T17:16:20.510 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000209323 2024-08-17T17:16:20.510 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.510 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.510 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.510 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043259+0000", 2024-08-17T17:16:20.510 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014336500000000001 2024-08-17T17:16:20.510 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.510 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.510 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:16:20.510 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043411+0000", 2024-08-17T17:16:20.511 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000151767 2024-08-17T17:16:20.511 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.511 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.511 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.511 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129023+0000", 2024-08-17T17:16:20.511 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.085612478000000006 2024-08-17T17:16:20.511 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.511 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.511 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.511 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129233+0000", 2024-08-17T17:16:20.511 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00021032200000000001 2024-08-17T17:16:20.511 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.511 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.512 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:16:20.512 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.130743+0000", 2024-08-17T17:16:20.512 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001510061 2024-08-17T17:16:20.512 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.512 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.512 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.512 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.130785+0000", 2024-08-17T17:16:20.512 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.1313999999999998e-05 2024-08-17T17:16:20.512 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.512 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.512 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.512 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.130826+0000", 2024-08-17T17:16:20.512 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.0978e-05 2024-08-17T17:16:20.512 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.513 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.513 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.513 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.513 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.513 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34211 6.9 6:951c0f16:::benchmark_data_smithi063_160268_object34210:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:16:20.513 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.042932+0000", 2024-08-17T17:16:20.513 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 160.41395193400001, 2024-08-17T17:16:20.513 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.087980920000000004, 2024-08-17T17:16:20.513 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.513 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.513 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.513 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:16:20.513 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:16:20.514 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:16:20.514 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34211 2024-08-17T17:16:20.514 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.514 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.514 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.514 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.514 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042932+0000", 2024-08-17T17:16:20.514 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.514 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.514 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.514 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.514 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042932+0000", 2024-08-17T17:16:20.514 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.515 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.515 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.515 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.515 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042933+0000", 2024-08-17T17:16:20.515 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6899999999999996e-07 2024-08-17T17:16:20.515 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.515 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.515 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.515 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042956+0000", 2024-08-17T17:16:20.515 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2646e-05 2024-08-17T17:16:20.515 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.515 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.515 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.516 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042965+0000", 2024-08-17T17:16:20.516 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3109999999999995e-06 2024-08-17T17:16:20.516 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.516 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.516 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.516 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042972+0000", 2024-08-17T17:16:20.516 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4829999999999996e-06 2024-08-17T17:16:20.516 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.516 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.516 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.516 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044942+0000", 2024-08-17T17:16:20.516 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001970201 2024-08-17T17:16:20.516 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.517 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.517 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.045071+0000", 2024-08-17T17:16:20.517 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012918399999999999 2024-08-17T17:16:20.517 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.517 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:16:20.517 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.045203+0000", 2024-08-17T17:16:20.517 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000132173 2024-08-17T17:16:20.517 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.517 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.517 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129060+0000", 2024-08-17T17:16:20.517 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.083857318 2024-08-17T17:16:20.518 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.518 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.518 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.518 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129290+0000", 2024-08-17T17:16:20.518 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00022933 2024-08-17T17:16:20.518 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.518 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.518 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:16:20.518 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.130837+0000", 2024-08-17T17:16:20.518 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0015471160000000001 2024-08-17T17:16:20.518 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.518 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.519 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.519 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.130879+0000", 2024-08-17T17:16:20.519 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2208e-05 2024-08-17T17:16:20.519 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.519 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.519 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.519 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.130913+0000", 2024-08-17T17:16:20.519 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4180999999999997e-05 2024-08-17T17:16:20.519 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.519 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.519 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.519 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.520 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.520 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34213 6.0 6:0bdfe2aa:::benchmark_data_smithi063_160268_object34212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:16:20.520 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.043016+0000", 2024-08-17T17:16:20.520 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 160.41386861699999, 2024-08-17T17:16:20.520 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.093410909, 2024-08-17T17:16:20.520 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.520 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.520 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.520 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:16:20.520 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:16:20.520 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:16:20.520 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34213 2024-08-17T17:16:20.520 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.521 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.521 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.521 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.521 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043016+0000", 2024-08-17T17:16:20.521 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.521 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.521 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.521 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.521 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043016+0000", 2024-08-17T17:16:20.521 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.521 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.521 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.521 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.522 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043017+0000", 2024-08-17T17:16:20.522 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8999999999999995e-07 2024-08-17T17:16:20.522 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.522 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.522 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.522 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043038+0000", 2024-08-17T17:16:20.522 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1098999999999999e-05 2024-08-17T17:16:20.522 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.522 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.522 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.522 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043048+0000", 2024-08-17T17:16:20.522 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9729999999999992e-06 2024-08-17T17:16:20.523 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:20.524 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.524 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.524 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.524 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043053+0000", 2024-08-17T17:16:20.524 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6069999999999998e-06 2024-08-17T17:16:20.524 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.524 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.524 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.524 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043941+0000", 2024-08-17T17:16:20.524 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00088757500000000004 2024-08-17T17:16:20.525 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.525 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.525 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.525 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044053+0000", 2024-08-17T17:16:20.525 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011174499999999999 2024-08-17T17:16:20.525 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.525 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.525 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:16:20.525 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044174+0000", 2024-08-17T17:16:20.525 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000121351 2024-08-17T17:16:20.525 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.525 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.525 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:16:20.526 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128544+0000", 2024-08-17T17:16:20.526 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.084370497000000003 2024-08-17T17:16:20.526 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.526 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.526 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.526 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128581+0000", 2024-08-17T17:16:20.526 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6316000000000003e-05 2024-08-17T17:16:20.526 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.526 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.526 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.526 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.136316+0000", 2024-08-17T17:16:20.526 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0077348740000000001 2024-08-17T17:16:20.527 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.527 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.527 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.527 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.136379+0000", 2024-08-17T17:16:20.527 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3622999999999997e-05 2024-08-17T17:16:20.527 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.527 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.527 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.527 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.136427+0000", 2024-08-17T17:16:20.527 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7358999999999998e-05 2024-08-17T17:16:20.527 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.527 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.527 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.528 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.528 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.528 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34215 6.0 6:0b2fbf44:::benchmark_data_smithi063_160268_object34214:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:16:20.528 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.043389+0000", 2024-08-17T17:16:20.528 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 160.413495673, 2024-08-17T17:16:20.528 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.093601496000000006, 2024-08-17T17:16:20.528 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.528 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.528 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.528 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:16:20.528 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:16:20.528 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:16:20.528 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34215 2024-08-17T17:16:20.528 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.529 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.529 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043389+0000", 2024-08-17T17:16:20.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.529 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.529 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043389+0000", 2024-08-17T17:16:20.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.529 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.529 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.530 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043389+0000", 2024-08-17T17:16:20.530 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3499999999999995e-07 2024-08-17T17:16:20.530 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.530 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.530 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.530 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043439+0000", 2024-08-17T17:16:20.530 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9734999999999997e-05 2024-08-17T17:16:20.530 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.530 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.530 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.530 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043449+0000", 2024-08-17T17:16:20.530 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4169999999999998e-06 2024-08-17T17:16:20.531 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.531 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.531 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.531 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043456+0000", 2024-08-17T17:16:20.531 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0620000000000003e-06 2024-08-17T17:16:20.531 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.531 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.531 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.531 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.126561+0000", 2024-08-17T17:16:20.531 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.083105629 2024-08-17T17:16:20.531 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.531 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.531 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.531 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.126728+0000", 2024-08-17T17:16:20.532 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016643800000000001 2024-08-17T17:16:20.532 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.532 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.532 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:16:20.532 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.126894+0000", 2024-08-17T17:16:20.532 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016649300000000001 2024-08-17T17:16:20.532 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.532 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.532 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:16:20.532 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129010+0000", 2024-08-17T17:16:20.532 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0021160480000000001 2024-08-17T17:16:20.532 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.532 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.533 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.533 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.131666+0000", 2024-08-17T17:16:20.533 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0026555139999999999 2024-08-17T17:16:20.533 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.533 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.533 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.533 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.136889+0000", 2024-08-17T17:16:20.533 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0052235069999999996 2024-08-17T17:16:20.533 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.533 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.533 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.533 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.136954+0000", 2024-08-17T17:16:20.533 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4984e-05 2024-08-17T17:16:20.534 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.534 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.534 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.534 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.136990+0000", 2024-08-17T17:16:20.534 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5933999999999998e-05 2024-08-17T17:16:20.534 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.534 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.534 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.534 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.534 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.534 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:34217 6.d e1093/1086 6:b571891d:::benchmark_data_smithi063_160268_object34216:head v 1093'2085, mlcod=1093'2083)", 2024-08-17T17:16:20.534 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.044815+0000", 2024-08-17T17:16:20.535 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 160.41206978700001, 2024-08-17T17:16:20.535 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.081785244000000007, 2024-08-17T17:16:20.535 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.535 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.535 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.535 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.535 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.535 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.535 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044815+0000", 2024-08-17T17:16:20.535 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.535 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.535 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.535 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.536 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044815+0000", 2024-08-17T17:16:20.536 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.536 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.536 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.536 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.536 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044815+0000", 2024-08-17T17:16:20.536 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3399999999999998e-07 2024-08-17T17:16:20.536 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.536 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.536 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.536 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044845+0000", 2024-08-17T17:16:20.536 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.9427000000000001e-05 2024-08-17T17:16:20.536 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.537 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.537 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.537 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044855+0000", 2024-08-17T17:16:20.537 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.8619999999999998e-06 2024-08-17T17:16:20.537 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.537 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.537 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.537 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044861+0000", 2024-08-17T17:16:20.537 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7020000000000004e-06 2024-08-17T17:16:20.537 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.537 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.537 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.537 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044916+0000", 2024-08-17T17:16:20.537 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4268999999999998e-05 2024-08-17T17:16:20.538 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.538 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.538 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.538 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044989+0000", 2024-08-17T17:16:20.538 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3128999999999997e-05 2024-08-17T17:16:20.538 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.538 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.538 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.538 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.126543+0000", 2024-08-17T17:16:20.538 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.081553977999999999 2024-08-17T17:16:20.538 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.538 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.539 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.539 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.126600+0000", 2024-08-17T17:16:20.539 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7043000000000003e-05 2024-08-17T17:16:20.539 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.539 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.539 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.539 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.539 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.539 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.44609.0:34197 6.0 e1093/1086 ondisk, result = 0)", 2024-08-17T17:16:20.539 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.045637+0000", 2024-08-17T17:16:20.539 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 160.41124740800001, 2024-08-17T17:16:20.539 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.082908979999999993, 2024-08-17T17:16:20.539 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.540 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.540 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:16:20.540 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.540 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.540 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.540 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.045637+0000", 2024-08-17T17:16:20.540 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.540 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.540 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.540 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.540 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.045637+0000", 2024-08-17T17:16:20.540 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.540 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.541 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.541 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.541 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.045638+0000", 2024-08-17T17:16:20.541 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3600000000000003e-07 2024-08-17T17:16:20.541 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.541 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.541 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.541 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.045641+0000", 2024-08-17T17:16:20.541 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.529e-06 2024-08-17T17:16:20.541 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.541 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.541 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.541 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.126636+0000", 2024-08-17T17:16:20.542 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.080995196000000005 2024-08-17T17:16:20.542 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.542 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.542 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.542 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.126646+0000", 2024-08-17T17:16:20.542 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4029999999999996e-06 2024-08-17T17:16:20.542 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.542 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.542 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.542 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128405+0000", 2024-08-17T17:16:20.542 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001759273 2024-08-17T17:16:20.542 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.542 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.543 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.543 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128413+0000", 2024-08-17T17:16:20.543 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0479999999999999e-06 2024-08-17T17:16:20.543 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.543 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.543 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.543 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128546+0000", 2024-08-17T17:16:20.543 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013289500000000001 2024-08-17T17:16:20.543 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.543 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.543 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.543 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.544 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.544 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35804 6.0 6:00446a4a:::benchmark_data_smithi063_160268_object35803:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:16:20.544 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.874443+0000", 2024-08-17T17:16:20.544 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 159.58244107799999, 2024-08-17T17:16:20.544 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14757826700000001, 2024-08-17T17:16:20.544 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.544 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.544 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.544 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:16:20.544 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:16:20.544 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:16:20.544 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35804 2024-08-17T17:16:20.544 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.545 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.545 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.545 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.874443+0000", 2024-08-17T17:16:20.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.545 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.545 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.874443+0000", 2024-08-17T17:16:20.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.545 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.545 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.874444+0000", 2024-08-17T17:16:20.546 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.147e-06 2024-08-17T17:16:20.546 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.546 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.546 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.874465+0000", 2024-08-17T17:16:20.546 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0462000000000002e-05 2024-08-17T17:16:20.546 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.546 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.546 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.874476+0000", 2024-08-17T17:16:20.546 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1481000000000001e-05 2024-08-17T17:16:20.547 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.547 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.547 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.547 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.874483+0000", 2024-08-17T17:16:20.547 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0330000000000003e-06 2024-08-17T17:16:20.547 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.547 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.547 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.547 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.875878+0000", 2024-08-17T17:16:20.548 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0013948960000000001 2024-08-17T17:16:20.548 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.548 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.548 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.548 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876005+0000", 2024-08-17T17:16:20.548 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012692399999999999 2024-08-17T17:16:20.548 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.548 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.548 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:16:20.548 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876663+0000", 2024-08-17T17:16:20.549 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0006579 2024-08-17T17:16:20.549 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.549 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.549 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:16:20.549 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878132+0000", 2024-08-17T17:16:20.549 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0014690790000000001 2024-08-17T17:16:20.549 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.549 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.549 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.549 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882446+0000", 2024-08-17T17:16:20.549 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.004314249 2024-08-17T17:16:20.549 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.549 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.550 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.550 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.021883+0000", 2024-08-17T17:16:20.550 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.139436485 2024-08-17T17:16:20.550 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.550 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.550 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.550 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.021974+0000", 2024-08-17T17:16:20.550 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0816000000000006e-05 2024-08-17T17:16:20.550 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.550 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.550 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.550 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.022022+0000", 2024-08-17T17:16:20.551 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7794999999999999e-05 2024-08-17T17:16:20.551 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.551 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.551 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.551 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.551 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.551 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35816 6.0 6:0defa481:::benchmark_data_smithi063_160268_object35815:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:16:20.551 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.880228+0000", 2024-08-17T17:16:20.551 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 159.57665634700001, 2024-08-17T17:16:20.551 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.146786307, 2024-08-17T17:16:20.551 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.551 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.551 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.552 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:16:20.552 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:16:20.552 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:16:20.552 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35816 2024-08-17T17:16:20.552 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.552 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.552 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.552 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.552 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880228+0000", 2024-08-17T17:16:20.552 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.552 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.552 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.552 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.553 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880228+0000", 2024-08-17T17:16:20.553 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.553 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.553 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.553 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.553 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880229+0000", 2024-08-17T17:16:20.553 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3850000000000001e-06 2024-08-17T17:16:20.553 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.553 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.553 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.553 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880255+0000", 2024-08-17T17:16:20.553 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5371000000000002e-05 2024-08-17T17:16:20.553 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.554 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.554 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.554 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880267+0000", 2024-08-17T17:16:20.554 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2503999999999999e-05 2024-08-17T17:16:20.554 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.554 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.554 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.554 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880325+0000", 2024-08-17T17:16:20.554 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8196999999999999e-05 2024-08-17T17:16:20.554 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.554 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.554 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.555 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880405+0000", 2024-08-17T17:16:20.555 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9374000000000003e-05 2024-08-17T17:16:20.555 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.555 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.555 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.555 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880565+0000", 2024-08-17T17:16:20.555 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000160097 2024-08-17T17:16:20.555 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.555 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.555 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:16:20.555 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880722+0000", 2024-08-17T17:16:20.555 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015664300000000001 2024-08-17T17:16:20.555 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.556 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.556 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.556 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882511+0000", 2024-08-17T17:16:20.556 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0017894009999999999 2024-08-17T17:16:20.556 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.556 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.556 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:16:20.556 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882525+0000", 2024-08-17T17:16:20.556 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3806e-05 2024-08-17T17:16:20.556 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.556 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.556 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.556 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.026906+0000", 2024-08-17T17:16:20.557 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.144381238 2024-08-17T17:16:20.557 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.557 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.557 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.557 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.026970+0000", 2024-08-17T17:16:20.557 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4058000000000002e-05 2024-08-17T17:16:20.557 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.557 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.557 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.557 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027014+0000", 2024-08-17T17:16:20.557 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4233000000000002e-05 2024-08-17T17:16:20.557 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.557 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.558 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.558 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.558 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.558 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:61714 6.0 6:04c0c08a:::benchmark_data_smithi063_160268_object61713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1096)", 2024-08-17T17:16:20.558 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:56.779528+0000", 2024-08-17T17:16:20.558 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 143.67735587300001, 2024-08-17T17:16:20.558 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.066073092999999999, 2024-08-17T17:16:20.558 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.558 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.558 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.558 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:16:20.558 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:16:20.558 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:16:20.559 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 61714 2024-08-17T17:16:20.559 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.559 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.559 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.559 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.559 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.779528+0000", 2024-08-17T17:16:20.559 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.559 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.559 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.559 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.559 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.779528+0000", 2024-08-17T17:16:20.559 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.559 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.560 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.560 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.560 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.779530+0000", 2024-08-17T17:16:20.560 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9630000000000001e-06 2024-08-17T17:16:20.560 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.560 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.560 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.560 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.779552+0000", 2024-08-17T17:16:20.560 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2011e-05 2024-08-17T17:16:20.560 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.560 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.560 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.560 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.779565+0000", 2024-08-17T17:16:20.561 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2340999999999999e-05 2024-08-17T17:16:20.561 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.561 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.561 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.561 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.779572+0000", 2024-08-17T17:16:20.561 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1810000000000004e-06 2024-08-17T17:16:20.561 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.561 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.561 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.561 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.779996+0000", 2024-08-17T17:16:20.561 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00042421199999999998 2024-08-17T17:16:20.561 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.561 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.562 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.562 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.780147+0000", 2024-08-17T17:16:20.562 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015037199999999999 2024-08-17T17:16:20.562 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.562 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.562 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:16:20.562 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.780296+0000", 2024-08-17T17:16:20.562 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000149445 2024-08-17T17:16:20.562 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.562 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.562 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:16:20.562 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.782901+0000", 2024-08-17T17:16:20.563 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0026051580000000002 2024-08-17T17:16:20.563 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.563 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.563 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.563 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.788210+0000", 2024-08-17T17:16:20.563 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0053086469999999997 2024-08-17T17:16:20.563 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.563 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.563 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.563 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.845451+0000", 2024-08-17T17:16:20.563 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.057241171 2024-08-17T17:16:20.563 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.563 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.563 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.564 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.845556+0000", 2024-08-17T17:16:20.564 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000104919 2024-08-17T17:16:20.564 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.564 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.564 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.564 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:56.845602+0000", 2024-08-17T17:16:20.564 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5673000000000001e-05 2024-08-17T17:16:20.564 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.564 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.564 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.564 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.564 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.564 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:103979 6.0 6:0ed307be:::benchmark_data_smithi063_160268_object103978:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-08-17T17:16:20.565 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:14:32.735002+0000", 2024-08-17T17:16:20.565 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 107.721881991, 2024-08-17T17:16:20.565 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.072518333000000004, 2024-08-17T17:16:20.565 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.565 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.565 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.565 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:16:20.565 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:16:20.565 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:16:20.565 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 103979 2024-08-17T17:16:20.565 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.565 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.566 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.566 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.566 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735002+0000", 2024-08-17T17:16:20.566 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.566 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.566 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.566 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.566 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735002+0000", 2024-08-17T17:16:20.566 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.566 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.566 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.566 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.566 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735004+0000", 2024-08-17T17:16:20.567 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6920000000000001e-06 2024-08-17T17:16:20.567 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.567 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.567 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.567 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735030+0000", 2024-08-17T17:16:20.567 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5859000000000001e-05 2024-08-17T17:16:20.567 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.567 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.567 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.567 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735046+0000", 2024-08-17T17:16:20.567 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6011e-05 2024-08-17T17:16:20.567 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.567 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.568 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.568 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735055+0000", 2024-08-17T17:16:20.568 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.687e-06 2024-08-17T17:16:20.568 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.568 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.568 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.568 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735117+0000", 2024-08-17T17:16:20.568 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2593000000000002e-05 2024-08-17T17:16:20.568 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.568 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.568 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.568 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735265+0000", 2024-08-17T17:16:20.568 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014776899999999999 2024-08-17T17:16:20.569 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.570 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.570 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:16:20.570 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735410+0000", 2024-08-17T17:16:20.570 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014535700000000001 2024-08-17T17:16:20.570 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.570 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.570 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:16:20.570 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.738784+0000", 2024-08-17T17:16:20.570 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0033735420000000002 2024-08-17T17:16:20.570 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.570 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.570 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.571 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.743477+0000", 2024-08-17T17:16:20.571 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0046932880000000003 2024-08-17T17:16:20.571 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.571 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.571 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.571 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.807327+0000", 2024-08-17T17:16:20.571 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.063849631000000004 2024-08-17T17:16:20.571 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.571 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.571 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.571 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.807447+0000", 2024-08-17T17:16:20.571 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000120333 2024-08-17T17:16:20.571 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.572 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.572 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.572 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.807521+0000", 2024-08-17T17:16:20.572 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3571000000000004e-05 2024-08-17T17:16:20.572 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.572 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.572 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.572 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.572 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.572 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:103981 6.0 6:0058b2e1:::benchmark_data_smithi063_160268_object103980:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-08-17T17:16:20.572 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:14:32.740091+0000", 2024-08-17T17:16:20.572 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 107.716793628, 2024-08-17T17:16:20.572 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.067609118999999995, 2024-08-17T17:16:20.573 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.573 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.573 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.573 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:16:20.573 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:16:20.573 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:16:20.573 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 103981 2024-08-17T17:16:20.573 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.573 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.573 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.573 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.573 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740091+0000", 2024-08-17T17:16:20.573 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.574 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.574 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.574 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.574 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740091+0000", 2024-08-17T17:16:20.574 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.574 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.574 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.574 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.574 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740092+0000", 2024-08-17T17:16:20.574 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.491e-06 2024-08-17T17:16:20.574 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.574 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.575 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.575 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740111+0000", 2024-08-17T17:16:20.575 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9270999999999999e-05 2024-08-17T17:16:20.575 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.575 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.575 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.575 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740120+0000", 2024-08-17T17:16:20.575 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7129999999999994e-06 2024-08-17T17:16:20.575 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.575 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.575 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.575 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740125+0000", 2024-08-17T17:16:20.575 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0150000000000003e-06 2024-08-17T17:16:20.576 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.576 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.576 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.576 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740169+0000", 2024-08-17T17:16:20.576 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3511999999999999e-05 2024-08-17T17:16:20.576 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.576 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.576 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.576 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740267+0000", 2024-08-17T17:16:20.576 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7802999999999999e-05 2024-08-17T17:16:20.576 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.576 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.576 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:16:20.577 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740356+0000", 2024-08-17T17:16:20.577 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.9894000000000004e-05 2024-08-17T17:16:20.577 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.577 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.577 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:16:20.577 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.743518+0000", 2024-08-17T17:16:20.577 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0031620210000000001 2024-08-17T17:16:20.577 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.577 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.577 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.577 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.777842+0000", 2024-08-17T17:16:20.577 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.034323448999999999 2024-08-17T17:16:20.578 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.578 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.578 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:16:20.578 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.807590+0000", 2024-08-17T17:16:20.578 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.029748465000000002 2024-08-17T17:16:20.578 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.578 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.578 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.578 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.807660+0000", 2024-08-17T17:16:20.578 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9906000000000005e-05 2024-08-17T17:16:20.578 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.578 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.578 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.579 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.807700+0000", 2024-08-17T17:16:20.579 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9579000000000002e-05 2024-08-17T17:16:20.579 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.579 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.579 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.579 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.579 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.579 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:190044 6.d e1113/1086 6:b42545b8:::benchmark_data_smithi063_160268_object63937:head v 1113'11620, mlcod=1113'11617)", 2024-08-17T17:16:20.579 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:16:12.859023+0000", 2024-08-17T17:16:20.579 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 7.597860893, 2024-08-17T17:16:20.579 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.053627771999999997, 2024-08-17T17:16:20.579 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.579 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.580 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.580 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.580 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.580 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.580 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.859023+0000", 2024-08-17T17:16:20.580 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.580 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.580 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.580 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.580 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.859023+0000", 2024-08-17T17:16:20.580 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.580 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.580 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.581 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.581 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.859025+0000", 2024-08-17T17:16:20.581 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.967e-06 2024-08-17T17:16:20.581 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.581 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.581 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.581 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.859041+0000", 2024-08-17T17:16:20.581 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5651999999999999e-05 2024-08-17T17:16:20.581 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.581 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.581 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.581 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.859066+0000", 2024-08-17T17:16:20.581 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5211999999999998e-05 2024-08-17T17:16:20.581 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.582 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.582 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.582 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.859080+0000", 2024-08-17T17:16:20.582 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3726e-05 2024-08-17T17:16:20.582 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.582 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.582 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.582 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.859370+0000", 2024-08-17T17:16:20.582 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00028966000000000002 2024-08-17T17:16:20.582 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.582 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.582 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.582 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.859426+0000", 2024-08-17T17:16:20.583 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6580999999999999e-05 2024-08-17T17:16:20.583 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.583 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.583 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.583 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.912490+0000", 2024-08-17T17:16:20.583 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.053064213999999998 2024-08-17T17:16:20.583 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.583 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.583 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.583 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.912651+0000", 2024-08-17T17:16:20.583 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016076 2024-08-17T17:16:20.583 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.583 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.584 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.584 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.584 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.584 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:190046 6.d e1113/1086 6:b69d2ac0:::benchmark_data_smithi063_160268_object63939:head v 1113'11621, mlcod=1113'11617)", 2024-08-17T17:16:20.584 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:16:12.860304+0000", 2024-08-17T17:16:20.584 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 7.5965799780000003, 2024-08-17T17:16:20.584 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0524938, 2024-08-17T17:16:20.584 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.584 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.584 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.584 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.584 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.584 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.585 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.860304+0000", 2024-08-17T17:16:20.585 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.585 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.585 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.585 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.585 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.860304+0000", 2024-08-17T17:16:20.585 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.585 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.585 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.585 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.585 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.860310+0000", 2024-08-17T17:16:20.585 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.383e-06 2024-08-17T17:16:20.585 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.586 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.586 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.586 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.860319+0000", 2024-08-17T17:16:20.586 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4509999999999998e-06 2024-08-17T17:16:20.586 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.586 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.586 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.586 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.860338+0000", 2024-08-17T17:16:20.586 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8836999999999999e-05 2024-08-17T17:16:20.586 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.586 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.586 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.586 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.860347+0000", 2024-08-17T17:16:20.587 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4499999999999993e-06 2024-08-17T17:16:20.587 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.587 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.587 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.587 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.871495+0000", 2024-08-17T17:16:20.587 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011147822999999999 2024-08-17T17:16:20.587 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.587 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.587 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.587 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.871614+0000", 2024-08-17T17:16:20.587 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000119029 2024-08-17T17:16:20.587 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.587 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.588 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.588 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.912656+0000", 2024-08-17T17:16:20.588 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.041041187 2024-08-17T17:16:20.588 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.588 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.588 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.588 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:12.912798+0000", 2024-08-17T17:16:20.588 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014264 2024-08-17T17:16:20.588 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.588 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.588 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.588 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.588 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.588 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:191329 6.9 e1113/1103 6:90913db7:::benchmark_data_smithi063_160268_object65222:head v 1113'12031, mlcod=1113'12028)", 2024-08-17T17:16:20.589 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:16:14.640010+0000", 2024-08-17T17:16:20.589 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 5.8168744810000002, 2024-08-17T17:16:20.589 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.045972961, 2024-08-17T17:16:20.589 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:16:20.589 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:16:20.589 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:16:20.589 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:16:20.589 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.589 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:16:20.589 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:14.640010+0000", 2024-08-17T17:16:20.589 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.589 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.589 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.590 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:16:20.590 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:14.640010+0000", 2024-08-17T17:16:20.590 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:16:20.590 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.590 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.590 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:16:20.590 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:14.640010+0000", 2024-08-17T17:16:20.590 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3600000000000003e-07 2024-08-17T17:16:20.590 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.590 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.590 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:16:20.590 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:14.640016+0000", 2024-08-17T17:16:20.590 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9320000000000001e-06 2024-08-17T17:16:20.591 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.591 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.591 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:16:20.591 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:14.640029+0000", 2024-08-17T17:16:20.591 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2622e-05 2024-08-17T17:16:20.591 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.591 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.591 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:16:20.591 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:14.640035+0000", 2024-08-17T17:16:20.591 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3049999999999998e-06 2024-08-17T17:16:20.591 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.591 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.591 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:16:20.592 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:14.655788+0000", 2024-08-17T17:16:20.592 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.015753105999999999 2024-08-17T17:16:20.592 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.592 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.592 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:16:20.592 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:14.655843+0000", 2024-08-17T17:16:20.592 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4846000000000003e-05 2024-08-17T17:16:20.592 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.592 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.592 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:16:20.592 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:14.685789+0000", 2024-08-17T17:16:20.592 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.029946160999999999 2024-08-17T17:16:20.592 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:16:20.593 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:16:20.593 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:16:20.593 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:16:14.685983+0000", 2024-08-17T17:16:20.593 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000193353 2024-08-17T17:16:20.593 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.593 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.593 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.593 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:16:20.593 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:16:20.593 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:16:20.595 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:20.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:20.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:20.697 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:20.734 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:20.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:20.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:20.835 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:20.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:20.875 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:20.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:20.976 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:21.026 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:21.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:21.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:21.128 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:21.156 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:21.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:21.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:21.215 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:16:21.258 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:21.285 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:21.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:21.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:21.387 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:21.414 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:21.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:21.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:21.488 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:21 smithi063 ceph-mon[109843]: pgmap v1246: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2024-08-17T17:16:21.489 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:21 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3135624068' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:16:21.489 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:21 smithi063 ceph-mon[114756]: pgmap v1246: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2024-08-17T17:16:21.489 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:21 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3135624068' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:16:21.516 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:21.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:21.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:21.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:21.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:21 smithi162 ceph-mon[104668]: pgmap v1246: 89 pgs: 89 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2024-08-17T17:16:21.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:21 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3135624068' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:16:21.604 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:21.648 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:21.676 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:21.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:21.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:21.778 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:21.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:21.805 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:21.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:21.906 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:21.957 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:21.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:21.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:22.059 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:22.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:22.088 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:22.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:22.189 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:22.203 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:22 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:16:22.204 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:22 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:16:22.204 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:22 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3135624068' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:16:22.204 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:22 smithi063 ceph-mon[109843]: osdmap e1114: 8 total, 8 up, 8 in 2024-08-17T17:16:22.204 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:22 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:22.204 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:22 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:22.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:22.243 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:22.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:22.344 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:22.386 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:22.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:22.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:22.488 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:22.491 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:22 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:16:22.491 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:22 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:16:22.491 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:22 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3135624068' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:16:22.491 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:22 smithi063 ceph-mon[114756]: osdmap e1114: 8 total, 8 up, 8 in 2024-08-17T17:16:22.491 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:22 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:22.492 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:22 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:22.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:22.530 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:22.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:22.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:22 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:16:22.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:22 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:16:22.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:22 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3135624068' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:16:22.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:22 smithi162 ceph-mon[104668]: osdmap e1114: 8 total, 8 up, 8 in 2024-08-17T17:16:22.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:22 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:22.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:22 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:22.630 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:22.656 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:22.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:22.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:22.758 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:22.800 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:22.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:22.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:22.901 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:22.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:22.934 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:22.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:23.035 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:23.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:23.102 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:23.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:23.203 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:23.230 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:23.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:23.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:23.290 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:23 smithi063 ceph-mon[114756]: pgmap v1248: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2024-08-17T17:16:23.290 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:23 smithi063 ceph-mon[109843]: pgmap v1248: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2024-08-17T17:16:23.332 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:23.357 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:23.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:23.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:23.459 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:23.488 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:23.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:23.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:23.590 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:23.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:23 smithi162 ceph-mon[104668]: pgmap v1248: 89 pgs: 89 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 843 op/s 2024-08-17T17:16:23.600 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:16:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:16:23] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.51.0" 2024-08-17T17:16:23.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:23.635 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:23.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:23.640 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:16:23.737 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:23.764 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:23.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:23.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:23.866 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:23.890 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:23.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:23.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:23.992 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:24.022 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:24.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:24.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:24.124 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:24.161 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:24.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:24.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:24.263 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:24.300 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:24.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:24.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:24.402 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:24.428 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:24.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:24.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:24.525 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:24 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:16:24.525 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:24 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:16:24.525 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:24 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:24.525 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:24 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:16:24.525 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:24 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:16:24.525 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:24 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:24.530 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:24.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:24.576 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:24.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:24.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:24 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:16:24.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:24 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:16:24.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:24 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:24.677 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:24.701 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:24.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:24.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:24.803 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:24.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:24.830 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:24.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:24.931 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:24.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:24.963 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:24.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:25.065 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:25.073 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:16:25.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:25.113 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:25.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:25.214 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:25.242 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:25.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:25.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:25.344 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:25.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:25.371 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:25.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:25.473 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:25.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:25.503 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:25.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:25.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:25 smithi162 ceph-mon[104668]: pgmap v1249: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-08-17T17:16:25.603 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:25.608 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:25 smithi063 ceph-mon[114756]: pgmap v1249: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-08-17T17:16:25.608 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:25 smithi063 ceph-mon[109843]: pgmap v1249: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-08-17T17:16:25.650 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.4 reset_purged_snaps_last 2024-08-17T17:16:25.659 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:25.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:25.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:25.761 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:25.786 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:25.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:25.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:25.890 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:25.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:25.916 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:25.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:25.980 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:25.989 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:16:25.990 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:16:26.017 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:26.040 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:16:26.041 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:16:26.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:26.042 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:26.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:26.082 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:16:26.082 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:16:26.127 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:16:26.127 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:16:26.143 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:26.162 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:16:26.162 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:16:26.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:26.188 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:26.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:26.198 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:16:26.198 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:16:26.226 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:16:26.226 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:16:26.261 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:16:26.261 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:16:26.289 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:26.315 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:26.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:26.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:26.416 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:26.455 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:26.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:26.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:26.489 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:26 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3633425906' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:16:26.490 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:26 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3633425906' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:16:26.558 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:26.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:26 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3633425906' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:16:26.596 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:26.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:26.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:26.698 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:26.734 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:26.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:26.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:26.836 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:26.870 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:26.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:26.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:26.972 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:27.014 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:27.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:27.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:27.119 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:27.156 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:27.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:27.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:27.258 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:27.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:27.292 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:27.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:27.393 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:27.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:27.426 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:27.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:27.525 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:27 smithi063 ceph-mon[109843]: pgmap v1250: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-08-17T17:16:27.542 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:27 smithi063 ceph-mon[114756]: pgmap v1250: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-08-17T17:16:27.542 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:27.563 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:27.573 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:27.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:27 smithi162 ceph-mon[104668]: pgmap v1250: 89 pgs: 89 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-08-17T17:16:27.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:27.621 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:27.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:27.722 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:27.770 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:27.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:27.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:27.872 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:27.903 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:27.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:27.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:28.005 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:28.036 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:28.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:28.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:28.138 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:28.186 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:28.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:28.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:28.288 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:28.329 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:28.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:28.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:28.431 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:28.462 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:28.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:28.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:28.564 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:28.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:28.609 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:28.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:28.710 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:28.747 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:28.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:28.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:28.849 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:28.874 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:28.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:28.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:28.976 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:29.028 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:29.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:29.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:29.129 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:29.175 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:29.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:29.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:29.277 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:29.306 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:29.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:29.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:29.408 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:29.432 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:29.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:29.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:29.507 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:29 smithi063 ceph-mon[109843]: pgmap v1251: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-08-17T17:16:29.508 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:29 smithi063 ceph-mon[114756]: pgmap v1251: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-08-17T17:16:29.533 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:29.561 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:29.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:29.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:29.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:16:29.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:29 smithi162 ceph-mon[104668]: pgmap v1251: 89 pgs: 89 active+clean; 3.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-08-17T17:16:29.664 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:29.718 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:29.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:29.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:29.820 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:29.854 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:29.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:29.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:29.956 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:29.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:29.999 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:29.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:30.100 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:30.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:30.129 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:30.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:30.231 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:30.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:30.294 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:30.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:30.395 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:30.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:30.424 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:30.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:30.525 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:30.555 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:30.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:30.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:30.657 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:30.688 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:30.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:30.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:30.718 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:16:30.718 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:16:30.790 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:30.833 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:30.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:30.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:30.934 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:30.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:30.963 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:30.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:31.064 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:31.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:31.093 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:31.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:31.194 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:31.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:31.225 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:31.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:31.326 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:31.351 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:31.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:31.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:31.452 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:31.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:31.486 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:31.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:31.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:31 smithi063 ceph-mon[109843]: pgmap v1252: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-08-17T17:16:31.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:31 smithi063 ceph-mon[114756]: pgmap v1252: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-08-17T17:16:31.588 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:31.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:31 smithi162 ceph-mon[104668]: pgmap v1252: 89 pgs: 89 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-08-17T17:16:31.617 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:31.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:31.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:31.719 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:31.759 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:31.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:31.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:31.777 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:16:31.861 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:31.884 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:31.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:31.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:31.986 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:32.012 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:32.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:32.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:32.114 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:32.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:32.174 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:32.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:32.275 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:32.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:32.328 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:32.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:32.429 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:32.468 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:32.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:32.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:32.570 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:32.613 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:32.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:32.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:32.691 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:32 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:32.691 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:32 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:32.692 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:32 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:32.692 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:32 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:32.715 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:32.733 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:32 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:32.733 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:32 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:32.742 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:32.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:32.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:32.844 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:32.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:32.882 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:32.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:32.983 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:33.015 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:33.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:33.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:33.117 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:33.153 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:33.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:33.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:33.202 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.5 reset_purged_snaps_last 2024-08-17T17:16:33.237 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:33.240 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:33.255 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:33.284 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:33.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:33.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:33.384 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:16:33.386 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:33.420 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:33.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:33.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:33.448 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:33 smithi063 ceph-mon[109843]: pgmap v1253: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 944 op/s 2024-08-17T17:16:33.448 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:33 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2858485897' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:16:33.448 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:33 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:16:33.449 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:16:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:16:33] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.51.0" 2024-08-17T17:16:33.449 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:33 smithi063 ceph-mon[114756]: pgmap v1253: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 944 op/s 2024-08-17T17:16:33.449 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:33 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2858485897' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:16:33.449 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:33 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:16:33.522 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:33.575 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:33.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:33.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:33.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:33 smithi162 ceph-mon[104668]: pgmap v1253: 89 pgs: 89 active+clean; 2.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 944 op/s 2024-08-17T17:16:33.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:33 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2858485897' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:16:33.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:33 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:16:33.676 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:33.715 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:33.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:33.817 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:33.848 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:33.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:33.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:33.950 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:33.990 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:33.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:33.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:34.092 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:34.135 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:34.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:34.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:34.237 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:34.271 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:34.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:34.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:34.373 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:34.411 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:34.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:34.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:34.513 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:34.540 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:34.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:34.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:34.642 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:34.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:34.671 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:34.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:34.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:34 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:16:34.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:34 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:16:34.710 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:34 smithi063 ceph-mon[109843]: osdmap e1115: 8 total, 8 up, 8 in 2024-08-17T17:16:34.710 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:34 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:16:34.710 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:34 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:16:34.710 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:34 smithi063 ceph-mon[114756]: osdmap e1115: 8 total, 8 up, 8 in 2024-08-17T17:16:34.773 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:34.823 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:34.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:34.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:34.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:34 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:16:34.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:34 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:16:34.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:34 smithi162 ceph-mon[104668]: osdmap e1115: 8 total, 8 up, 8 in 2024-08-17T17:16:34.924 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:34.951 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:34.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:34.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:35.053 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:35.081 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:35.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:35.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:35.183 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:35.223 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:35.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:35.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:35.325 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:35.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:35.361 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:35.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:35.462 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:35.484 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:35.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:35.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:35.586 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:35.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:35.614 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:35.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:35.715 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:35.756 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:35.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:35.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:35.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:35 smithi063 ceph-mon[109843]: pgmap v1255: 89 pgs: 89 active+clean; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 846 op/s 2024-08-17T17:16:35.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:35 smithi063 ceph-mon[114756]: pgmap v1255: 89 pgs: 89 active+clean; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 846 op/s 2024-08-17T17:16:35.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:35 smithi162 ceph-mon[104668]: pgmap v1255: 89 pgs: 89 active+clean; 2.8 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 846 op/s 2024-08-17T17:16:35.858 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:35.941 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:35.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:35.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:36.043 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:36.066 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:36.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:36.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:36.168 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:36.201 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:36.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:36.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:36.303 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:36.329 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:36.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:36.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:36.430 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:36.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:36.463 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:36.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:36.564 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:36.602 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:36.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:36.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:36.607 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:36.704 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:36.732 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:36.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:36.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:36.834 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:36.862 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:36.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:36.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:36.964 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:37.002 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:37.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:37.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:37.104 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:37.139 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:37.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:37.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:37.240 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:37.281 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:37.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:37.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:37.383 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:37.407 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:37.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:37.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:37.438 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:37 smithi063 ceph-mon[109843]: pgmap v1256: 89 pgs: 89 active+clean; 2.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 847 op/s 2024-08-17T17:16:37.438 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:37 smithi063 ceph-mon[114756]: pgmap v1256: 89 pgs: 89 active+clean; 2.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 847 op/s 2024-08-17T17:16:37.449 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:16:37.449 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:16:37.496 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:16:37.497 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:16:37.509 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:37.539 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:16:37.539 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:16:37.547 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:16:37.547 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:16:37.547 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:16:37.547 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:16:37.548 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:16:37.558 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:37.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:37.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:37.578 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:16:37.578 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:16:37.615 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:16:37.615 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:16:37.650 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:16:37.651 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:16:37.660 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:37.689 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:16:37.690 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:16:37.692 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:37.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:37.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:37.719 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:16:37.720 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:16:37.794 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:37.832 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:37.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:37.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:37.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:37 smithi162 ceph-mon[104668]: pgmap v1256: 89 pgs: 89 active+clean; 2.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 847 op/s 2024-08-17T17:16:37.934 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:37.961 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:37.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:37.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:38.062 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:38.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:38.104 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:38.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:38.205 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:38.218 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:16:38.242 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:38.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:38.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:38.345 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:38.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:38.387 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:38.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:38.470 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:38 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1474878128' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:16:38.470 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:38 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1474878128' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:16:38.489 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:38.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:38.544 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:38.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:38.645 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:38.650 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:16:38.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:38.685 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:38.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:38.787 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:38.816 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:38.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:38.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:38.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:38 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1474878128' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:16:38.918 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:38.977 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:38.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:38.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:39.079 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:39.132 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:39.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:39.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:39.233 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:39.263 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:39.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:39.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:39.365 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:39.401 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:39.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:39.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:39.503 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:39.556 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:39.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:39.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:39.658 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:39.681 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:39.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:39.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:39.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:39 smithi063 ceph-mon[109843]: pgmap v1257: 89 pgs: 89 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2024-08-17T17:16:39.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:39 smithi063 ceph-mon[114756]: pgmap v1257: 89 pgs: 89 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2024-08-17T17:16:39.783 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:39.816 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:39.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:39.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:39.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:39 smithi162 ceph-mon[104668]: pgmap v1257: 89 pgs: 89 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2024-08-17T17:16:39.919 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:39.952 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:39.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:39.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:40.053 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:40.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:40.092 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:40.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:40.193 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:40.236 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:40.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:40.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:40.341 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:40.368 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:40.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:40.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:40.469 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:40.509 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:40.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:40.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:40.611 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:40.660 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:40.670 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:40.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:40.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:40.772 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:40.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:40.799 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:40.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:40.900 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:40.952 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:40.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:40.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:41.054 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:41.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:41.093 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:41.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:41.194 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:41.229 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:41.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:41.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:41.331 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:41.389 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:41.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:41.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:41.491 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:41.515 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:41.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:41.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:41.617 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:41.658 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:41.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:41.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:41.760 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:41.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:41 smithi063 ceph-mon[114756]: pgmap v1258: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-08-17T17:16:41.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:41 smithi063 ceph-mon[109843]: pgmap v1258: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-08-17T17:16:41.786 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:41.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:41.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:41.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:41 smithi162 ceph-mon[104668]: pgmap v1258: 89 pgs: 89 active+clean; 2.5 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-08-17T17:16:41.888 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:41.925 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:41.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:41.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:42.027 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:42.066 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:42.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:42.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:42.168 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:42.202 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:42.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:42.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:42.304 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:42.338 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:42.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:42.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:42.440 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:42.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:42.486 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:42.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:42.586 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:16:42.587 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:42.617 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:42.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:42.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:42.719 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:42.744 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:42.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:42.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:42.846 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:42.895 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:42.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:42.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:42.996 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:43.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:43.032 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:43.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:43.133 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:43.162 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:43.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:43.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:43.264 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:43.287 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:43.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:43.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:43.318 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:43.389 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:43.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:43.429 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:43.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:43.481 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:16:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:16:43] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.51.0" 2024-08-17T17:16:43.482 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:43 smithi063 ceph-mon[114756]: pgmap v1259: 89 pgs: 89 active+clean; 2.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-08-17T17:16:43.482 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:43 smithi063 ceph-mon[109843]: pgmap v1259: 89 pgs: 89 active+clean; 2.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-08-17T17:16:43.531 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:43.562 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:43.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:43.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:43.664 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:43.712 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:43.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:43.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:43.726 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:43 smithi162 ceph-mon[104668]: pgmap v1259: 89 pgs: 89 active+clean; 2.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 836 op/s 2024-08-17T17:16:43.759 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.6 reset_purged_snaps_last 2024-08-17T17:16:43.814 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:43.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:43.871 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:43.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:43.972 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:43.997 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:43.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:43.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:44.099 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:44.159 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:44.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:44.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:44.261 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:44.305 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:44.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:44.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:44.407 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:44.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:44.435 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:44.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:44.512 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:44.536 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:44.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:44.585 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:44.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:44.685 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:44.711 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:44.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:44.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:44.813 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:44.868 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:44.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:44.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:44.969 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:45.005 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:45.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:45.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:45.107 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:45.130 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:45.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:45.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:45.232 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:45.275 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:45.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:45.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:45.376 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:45.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:45.409 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:45.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:45.431 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:16:45.510 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:45.537 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:45.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:45.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:45.639 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:45.677 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:45.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:45.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:45.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:45 smithi063 ceph-mon[114756]: pgmap v1260: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 777 op/s 2024-08-17T17:16:45.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:45 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/338364050' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:16:45.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:45 smithi063 ceph-mon[109843]: pgmap v1260: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 777 op/s 2024-08-17T17:16:45.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:45 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/338364050' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:16:45.780 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:45.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:45.810 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:45.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:45.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:45 smithi162 ceph-mon[104668]: pgmap v1260: 89 pgs: 89 active+clean; 2.3 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 777 op/s 2024-08-17T17:16:45.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:45 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/338364050' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:16:45.911 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:45.963 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:45.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:45.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:46.064 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:46.100 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:46.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:46.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:46.202 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:46.228 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:46.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:46.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:46.330 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:46.356 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:46.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:46.458 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:46.501 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:46.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:46.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:46.603 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:46.638 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:46.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:46.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:46.740 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:46.768 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:46 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:16:46.768 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:46 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/338364050' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:16:46.768 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:46 smithi063 ceph-mon[114756]: osdmap e1116: 8 total, 8 up, 8 in 2024-08-17T17:16:46.769 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:46.769 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:46.769 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:46 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:16:46.769 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:46 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/338364050' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:16:46.769 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:46 smithi063 ceph-mon[109843]: osdmap e1116: 8 total, 8 up, 8 in 2024-08-17T17:16:46.769 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:46.769 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:46.772 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:46.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:46.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:46.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:46 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:16:46.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:46 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/338364050' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:16:46.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:46 smithi162 ceph-mon[104668]: osdmap e1116: 8 total, 8 up, 8 in 2024-08-17T17:16:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:16:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:16:46.874 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:46.908 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:46.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:46.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:47.010 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:47.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:47.055 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:47.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:47.156 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:47.182 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:47.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:47.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:47.284 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:47.316 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:47.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:47.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:47.418 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:47.465 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:47.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:47.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:47.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:47.583 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:47.629 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:47.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:47.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:47.696 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:47 smithi063 ceph-mon[114756]: pgmap v1262: 89 pgs: 89 active+clean; 2.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2024-08-17T17:16:47.696 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:47 smithi063 ceph-mon[109843]: pgmap v1262: 89 pgs: 89 active+clean; 2.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2024-08-17T17:16:47.731 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:47.782 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:47.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:47.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:47.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:47 smithi162 ceph-mon[104668]: pgmap v1262: 89 pgs: 89 active+clean; 2.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 831 op/s 2024-08-17T17:16:47.886 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:47.921 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:47.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:47.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:48.022 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:48.175 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:48.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:48.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:48.276 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:48.302 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:48.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:48.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:48.403 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:48.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:48.445 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:48.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:48.546 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:48.574 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:48.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:48.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:48.675 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:48.708 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:48.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:48.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:48.810 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:48.839 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:48.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:48.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:48.941 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:48.979 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:48.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:48.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:49.049 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:16:49.049 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:16:49.081 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:49.085 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:16:49.085 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:16:49.128 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:49.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:49.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:49.129 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:16:49.130 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:16:49.160 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:16:49.161 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:16:49.192 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:16:49.192 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:16:49.230 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:49.231 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:16:49.232 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:16:49.258 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:49.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:49.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:49.267 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:16:49.267 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:16:49.299 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:16:49.302 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:16:49.360 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:49.396 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:49.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:49.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:49.498 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:49.526 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:49.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:49.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:49.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:49 smithi063 ceph-mon[109843]: pgmap v1263: 89 pgs: 89 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-08-17T17:16:49.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:49 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3958463788' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:16:49.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:49 smithi063 ceph-mon[114756]: pgmap v1263: 89 pgs: 89 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-08-17T17:16:49.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:49 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3958463788' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:16:49.628 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:49.666 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:49.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:49.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:49.768 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:49.816 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:49.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:49.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:49.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:49 smithi162 ceph-mon[104668]: pgmap v1263: 89 pgs: 89 active+clean; 2.1 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 812 op/s 2024-08-17T17:16:49.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:49 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3958463788' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:16:49.919 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:49.945 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:49.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:49.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:50.047 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:50.075 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:50.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:50.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:50.177 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:50.211 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:50.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:50.212 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:50.313 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:50.341 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:50.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:50.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:50.443 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:50.448 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:16:50.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:50.484 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:50.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:50.585 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:50.610 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:50.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:50.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:50.676 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:16:50.676 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:16:50.711 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:50.759 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:50.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:50.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:50.860 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:50.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:50.900 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:50.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:51.000 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:51.028 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:51.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:51.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:51.130 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:51.167 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:51.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:51.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:51.269 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:51.296 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:51.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:51.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:51.397 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:51.426 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:51.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:51.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:51.527 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:51.563 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:51.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:51.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:51.664 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:51.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:51.695 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:51.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:51.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:51 smithi063 ceph-mon[114756]: pgmap v1264: 89 pgs: 89 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-08-17T17:16:51.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:51 smithi063 ceph-mon[109843]: pgmap v1264: 89 pgs: 89 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-08-17T17:16:51.796 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:51.833 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:51.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:51.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:51.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:51 smithi162 ceph-mon[104668]: pgmap v1264: 89 pgs: 89 active+clean; 2.0 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 809 op/s 2024-08-17T17:16:51.935 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:51.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:51.977 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:51.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:52.078 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:52.109 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:52.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:52.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:52.210 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:52.243 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:52.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:52.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:52.344 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:52.372 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:52.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:52.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:52.401 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:16:52.473 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:52.518 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:52.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:52.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:52.620 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:52.683 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:16:52.690 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:52.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:52.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:52.792 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:52.836 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:52.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:52.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:52.938 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:52.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:52.972 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:52.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:53.073 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:53.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:53.137 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:53.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:53.196 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:53.238 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:53.261 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:53.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:53.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:53.282 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:53.352 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:16:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:16:53] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.51.0" 2024-08-17T17:16:53.363 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:53.411 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:53.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:53.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:53.422 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:16:53.515 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:53.557 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:53.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:53.618 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:53 smithi063 ceph-mon[109843]: pgmap v1265: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2024-08-17T17:16:53.618 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:53 smithi063 ceph-mon[114756]: pgmap v1265: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2024-08-17T17:16:53.659 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:53.686 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:53.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:53.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:53.787 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:53.830 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:53.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:53.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:53.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:53 smithi162 ceph-mon[104668]: pgmap v1265: 89 pgs: 89 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 821 op/s 2024-08-17T17:16:53.932 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:53.980 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:53.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:53.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:54.082 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:54.127 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:54.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:54.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:54.229 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:54.279 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:54.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:54.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:54.381 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:54.409 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:54.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:54.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:54.512 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:54.561 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:54.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:54.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:54.662 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:54.702 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:54.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:54.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:54.804 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:54.841 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:54.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:54.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:54.942 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:54.983 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:54.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:54.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:55.085 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:55.113 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:55.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:55.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:55.215 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:55.269 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:55.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:55.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:55.370 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:55.415 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:55.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:55.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:55.517 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:55.548 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:55.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:55.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:55.649 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:55.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:55.686 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:55.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:55.733 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:55 smithi063 ceph-mon[109843]: pgmap v1266: 89 pgs: 89 active+clean; 1.8 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-08-17T17:16:55.734 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:55 smithi063 ceph-mon[114756]: pgmap v1266: 89 pgs: 89 active+clean; 1.8 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-08-17T17:16:55.787 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:55.829 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:55.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:55.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:55.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:55 smithi162 ceph-mon[104668]: pgmap v1266: 89 pgs: 89 active+clean; 1.8 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-08-17T17:16:55.931 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:55.971 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:55.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:55.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:56.073 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:56.097 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:56.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:56.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:56.198 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:56.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:56.240 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:56.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:56.341 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:56.380 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:56.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:56.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:56.482 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:56.514 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:56.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:56.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:56.616 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:56.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:56.643 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:56.643 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:56.744 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:56.773 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:56.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:56.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:56.874 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:56.916 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:56.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:56.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:57.018 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:57.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:57.062 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:57.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:57.163 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:57.201 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:57.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:57.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:57.303 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:57.344 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:57.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:57.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:57.446 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:57.485 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:57.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:57.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:57.587 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:57.616 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:57.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:57.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:57.718 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:57.730 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:57 smithi063 ceph-mon[109843]: pgmap v1267: 89 pgs: 89 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2024-08-17T17:16:57.730 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:57 smithi063 ceph-mon[114756]: pgmap v1267: 89 pgs: 89 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2024-08-17T17:16:57.751 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:57.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:57.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:57.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:57 smithi162 ceph-mon[104668]: pgmap v1267: 89 pgs: 89 active+clean; 1.8 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 788 op/s 2024-08-17T17:16:57.853 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:57.877 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:57.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:57.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:57.979 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:58.017 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:58.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:58.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:58.118 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:58.151 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:58.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:58.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:58.253 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:58.261 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.7 reset_purged_snaps_last 2024-08-17T17:16:58.299 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:58.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:58.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:58.401 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:58.431 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:58.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:58.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:58.533 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:16:58.558 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:58.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:58.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:16:58.659 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:58.687 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:58.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:58.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:58.789 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:58.833 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:58.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:58.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:16:58.935 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:16:58.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:58.977 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:58.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:16:59.078 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:16:59.130 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:59.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:59.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:16:59.232 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:16:59.255 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:59.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:59.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:16:59.356 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:59.378 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:59.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:59.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:59.396 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:16:59.397 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:16:59.480 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:16:59.492 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:16:59.526 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:59.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:59.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:16:59.628 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:16:59.654 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:59.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:59.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:16:59.756 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:16:59.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:59 smithi063 ceph-mon[114756]: pgmap v1268: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2024-08-17T17:16:59.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:16:59 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1804858583' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:16:59.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:59 smithi063 ceph-mon[109843]: pgmap v1268: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2024-08-17T17:16:59.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:16:59 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1804858583' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:16:59.783 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:16:59.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:59.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:16:59.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:59 smithi162 ceph-mon[104668]: pgmap v1268: 89 pgs: 89 active+clean; 1.6 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2024-08-17T17:16:59.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:16:59 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1804858583' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:16:59.885 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:16:59.935 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:16:59.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:16:59.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:00.037 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:00.066 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:00.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:00.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:00.102 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:17:00.102 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:17:00.144 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:17:00.144 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:17:00.168 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:00.187 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:17:00.187 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:17:00.207 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:00.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:00.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:00.237 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:17:00.238 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:17:00.269 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:17:00.269 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:17:00.303 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:17:00.303 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:17:00.310 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:00.332 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:17:00.333 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:17:00.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:00.344 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:00.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:00.368 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:17:00.369 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:17:00.445 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:00.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:00.492 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:00.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:00.593 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:00.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:00.625 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:00.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:00.726 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:00.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:00.774 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:00.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:00.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:00 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:00.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:00 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1804858583' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:17:00.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:00 smithi063 ceph-mon[109843]: osdmap e1117: 8 total, 8 up, 8 in 2024-08-17T17:17:00.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:00 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1895245061' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:00.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:00 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:00.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:00 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1804858583' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:17:00.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:00 smithi063 ceph-mon[114756]: osdmap e1117: 8 total, 8 up, 8 in 2024-08-17T17:17:00.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:00 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1895245061' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:00.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:00 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:00 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1804858583' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:17:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:00 smithi162 ceph-mon[104668]: osdmap e1117: 8 total, 8 up, 8 in 2024-08-17T17:17:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:00 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1895245061' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:00.875 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:00.900 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:00.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:00.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:01.002 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:01.034 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:01.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:01.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:01.136 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:01.168 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:17:01.178 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:01.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:01.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:01.280 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:01.307 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:01.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:01.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:01.408 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:01.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:01.455 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:01.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:01.556 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:01.606 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:01.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:01.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:01.708 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:01.740 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:01.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:01.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:01 smithi063 ceph-mon[114756]: pgmap v1270: 89 pgs: 89 active+clean; 1.5 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2024-08-17T17:17:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:01.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:01 smithi063 ceph-mon[109843]: pgmap v1270: 89 pgs: 89 active+clean; 1.5 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2024-08-17T17:17:01.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:01.809 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:01.842 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:01.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:01 smithi162 ceph-mon[104668]: pgmap v1270: 89 pgs: 89 active+clean; 1.5 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2024-08-17T17:17:01.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:01.909 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:01.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:01.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:02.010 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:02.045 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:02.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:02.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:02.146 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:02.193 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:02.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:02.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:02.295 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:02.326 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:02.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:02.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:02.428 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:02.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:02.461 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:02.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:02.562 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:02.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:17:02.590 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:02.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:02.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:02.691 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:02.714 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:02.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:02.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:02.816 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:02.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:02.856 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:02.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:02.956 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:02.986 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:02.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:02.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:03.088 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:03.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:03.119 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:03.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:03.220 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:03.262 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:03.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:03.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:03.364 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:03.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:03.488 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:03.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:03.516 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:03.554 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:03.589 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:03.614 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:03.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:03.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:03.678 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:03 smithi063 ceph-mon[109843]: pgmap v1271: 89 pgs: 89 active+clean; 1.4 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-08-17T17:17:03.679 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:17:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:17:03] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.51.0" 2024-08-17T17:17:03.679 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:03 smithi063 ceph-mon[114756]: pgmap v1271: 89 pgs: 89 active+clean; 1.4 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-08-17T17:17:03.716 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:03.763 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:03.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:03.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:03.841 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:17:03.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:03 smithi162 ceph-mon[104668]: pgmap v1271: 89 pgs: 89 active+clean; 1.4 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-08-17T17:17:03.865 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:03.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:03.946 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:03.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:04.047 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:04.075 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:04.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:04.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:04.177 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:04.207 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:04.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:04.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:04.309 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:04.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:04.359 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:04.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:04.460 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:04.489 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:04.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:04.590 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:04.628 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:04.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:04.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:04.730 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:04.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:04.753 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:04.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:04.854 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:04.881 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:04.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:04.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:04.982 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:05.011 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:05.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:05.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:05.112 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:05.153 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:05.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:05.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:05.255 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:05.281 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:05.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:05.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:05.382 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:05.429 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:05.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:05.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:05.531 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:05.560 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:05.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:05.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:05.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:05 smithi063 ceph-mon[109843]: pgmap v1272: 89 pgs: 89 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 826 op/s 2024-08-17T17:17:05.626 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:05 smithi063 ceph-mon[114756]: pgmap v1272: 89 pgs: 89 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 826 op/s 2024-08-17T17:17:05.662 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:05.700 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:05.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:05.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:05.802 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:05.844 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:05.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:05.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:05 smithi162 ceph-mon[104668]: pgmap v1272: 89 pgs: 89 active+clean; 1.3 GiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 826 op/s 2024-08-17T17:17:05.946 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:05.982 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:05.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:05.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:06.084 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:06.130 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:06.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:06.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:06.232 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:06.262 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:06.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:06.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:06.364 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:06.396 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:06.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:06.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:06.498 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:06.524 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:06.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:06.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:06.625 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:06.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:06.670 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:06.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:06.771 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:06.796 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:06.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:06.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:06.898 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:06.933 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:06.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:06.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:07.035 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:07.069 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:07.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:07.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:07.171 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:07.212 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:07.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:07.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:07.314 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:07.373 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:07.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:07.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:07.475 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:07.519 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:07.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:07.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:07.621 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:07.662 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:07.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:07.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:07.764 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:07 smithi063 ceph-mon[114756]: pgmap v1273: 89 pgs: 89 active+clean; 1.3 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-08-17T17:17:07.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:07 smithi063 ceph-mon[109843]: pgmap v1273: 89 pgs: 89 active+clean; 1.3 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-08-17T17:17:07.784 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:07.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:07.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:07.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:07 smithi162 ceph-mon[104668]: pgmap v1273: 89 pgs: 89 active+clean; 1.3 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2024-08-17T17:17:07.886 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:07.918 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:07.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:07.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:08.020 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:08.074 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:08.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:08.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:08.176 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:08.204 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:08.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:08.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:08.306 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:08.351 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:08.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:08.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:08.453 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:08.483 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:08.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:08.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:08.585 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:08.615 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:08.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:08.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:08.717 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:08.754 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:08.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:08.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:08.856 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:08.887 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:08.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:08.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:08.989 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:09.015 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:09.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:09.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:09.117 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:09.154 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:09.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:09.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:09.256 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:09.286 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:09.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:09.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:09.388 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:09.424 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:09.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:09.526 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:09.577 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:09.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:09.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:09.660 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:17:09.660 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:17:09.660 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:17:09.661 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:17:09.661 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:17:09.683 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:09.728 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:09.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:09.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:09.744 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:17:09.744 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:09 smithi063 ceph-mon[114756]: pgmap v1274: 89 pgs: 89 active+clean; 1.1 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-08-17T17:17:09.744 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:09 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1623636868' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:09.744 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:09 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:09.745 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:09 smithi063 ceph-mon[109843]: pgmap v1274: 89 pgs: 89 active+clean; 1.1 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-08-17T17:17:09.745 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:09 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1623636868' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:09.745 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:09 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:09.830 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:09.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:09 smithi162 ceph-mon[104668]: pgmap v1274: 89 pgs: 89 active+clean; 1.1 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-08-17T17:17:09.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:09 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1623636868' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:09.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:09 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:09.873 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:09.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:09.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:09.975 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:10.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:10.005 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:10.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:10.106 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:10.140 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:10.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:10.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:10.242 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:10.279 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:10.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:10.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:10.381 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:10.422 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:10.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:10.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:10.524 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:10.562 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:10.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:10.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:10.663 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:10.689 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:10.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:10.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:10.790 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:10.832 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:10.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:10.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:10.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:10 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:17:10.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:10 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:17:10.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:10 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:17:10.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:10 smithi162 ceph-mon[104668]: osdmap e1118: 8 total, 8 up, 8 in 2024-08-17T17:17:10.913 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:10.934 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:10.961 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:10.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:10.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:11.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:10 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:17:11.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:10 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:17:11.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:10 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:17:11.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:10 smithi063 ceph-mon[109843]: osdmap e1118: 8 total, 8 up, 8 in 2024-08-17T17:17:11.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:10 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:17:11.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:10 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:17:11.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:10 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:17:11.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:10 smithi063 ceph-mon[114756]: osdmap e1118: 8 total, 8 up, 8 in 2024-08-17T17:17:11.062 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:11.108 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:11.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:11.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:11.209 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:11.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:11.245 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:11.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:11.346 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:11.386 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:11.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:11.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:11.488 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:11.533 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:11.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:11.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:11.557 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:17:11.635 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:11.659 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:11.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:11.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:11.760 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:11.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:11 smithi063 ceph-mon[109843]: pgmap v1276: 89 pgs: 89 active+clean; 1.0 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2024-08-17T17:17:11.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:11 smithi063 ceph-mon[114756]: pgmap v1276: 89 pgs: 89 active+clean; 1.0 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2024-08-17T17:17:11.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:11.826 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:11.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:11 smithi162 ceph-mon[104668]: pgmap v1276: 89 pgs: 89 active+clean; 1.0 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 838 op/s 2024-08-17T17:17:11.927 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:11.983 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:11.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:11.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:12.084 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:12.112 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:12.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:12.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:12.214 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:12.242 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:12.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:12.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:12.344 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:12.373 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:12.404 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:12.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:12.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:12.506 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:12.558 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:12.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:12.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:12.660 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:12.688 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:17:12.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:12.689 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:12.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:12.790 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:12.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:12.836 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:12.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:12.937 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:12.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:12.992 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:12.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:13.093 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:13.096 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:17:13.096 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:17:13.096 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-08-17T17:17:13.097 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:17:13.097 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:17:13.097 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-08-17T17:17:13.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:13.126 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:13.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:13.227 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:13.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:13.269 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:13.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:13.320 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:17:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:17:13] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.51.0" 2024-08-17T17:17:13.370 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:13.413 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:13.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:13.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:13.444 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:17:13.515 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:13.543 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:13.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:13.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:13.639 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:13 smithi063 ceph-mon[114756]: pgmap v1277: 89 pgs: 89 active+clean; 910 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2024-08-17T17:17:13.639 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:13 smithi063 ceph-mon[109843]: pgmap v1277: 89 pgs: 89 active+clean; 910 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2024-08-17T17:17:13.644 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:13.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:13.690 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:13.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:13.791 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:13.820 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:13.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:13.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:13.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:13 smithi162 ceph-mon[104668]: pgmap v1277: 89 pgs: 89 active+clean; 910 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 842 op/s 2024-08-17T17:17:13.922 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:13.947 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:13.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:13.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:14.049 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:14.086 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:14.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:14.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:14.188 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:14.224 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:14.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:14.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:14.326 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:14.352 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:14.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:14.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:14.454 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:14.484 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:14.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:14.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:14.585 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:14.626 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:14.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:14.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:14.728 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:14.778 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:14.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:14.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:14.880 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:14.906 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:14.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:14.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:15.008 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:15.045 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:15.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:15.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:15.147 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:15.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:15.184 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:15.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:15.285 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:15.317 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:15.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:15.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:15.419 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:15.450 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:15.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:15.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:15.551 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:15.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:15.584 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:15.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:15.685 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:15.721 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:15 smithi063 ceph-mon[114756]: pgmap v1278: 89 pgs: 89 active+clean; 844 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2024-08-17T17:17:15.721 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:15 smithi063 ceph-mon[109843]: pgmap v1278: 89 pgs: 89 active+clean; 844 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2024-08-17T17:17:15.731 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:15.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:15.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:15.833 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:15.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:15 smithi162 ceph-mon[104668]: pgmap v1278: 89 pgs: 89 active+clean; 844 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2024-08-17T17:17:15.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:15.863 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:15.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:15.964 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:16.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:16.003 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:16.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:16.081 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:16.105 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:16.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:16.133 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:16.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:16.167 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:17:16.167 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:17:16.203 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:17:16.204 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:17:16.233 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:16.239 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:17:16.239 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:17:16.267 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:16.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:16.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:16.284 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:17:16.284 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:17:16.327 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:17:16.327 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:17:16.369 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:16.395 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:17:16.395 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:17:16.404 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:16.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:16.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:16.414 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:17:16.414 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:17:16.444 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:17:16.445 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:17:16.479 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:17:16.479 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:17:16.506 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:16.541 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:16.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:16.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:16.643 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:16.677 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:17:16.677 INFO:teuthology.orchestra.run.smithi162.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:17:16.677 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:17:16.679 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:16.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:16.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:16.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:16 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/672521229' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:16.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:16 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:16.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:16.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:16 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/672521229' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:16.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:16 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:16.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:16.781 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:16.823 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:16.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:16.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:16 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/672521229' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:16 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:16.925 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:16.953 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:16.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:16.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:17.055 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:17.092 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:17.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:17.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:17.194 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:17.225 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:17.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:17.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:17.327 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:17.365 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:17.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:17.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:17.467 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:17.503 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:17.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:17.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:17.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:17 smithi162 ceph-mon[104668]: pgmap v1279: 89 pgs: 89 active+clean; 780 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-08-17T17:17:17.604 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:17.633 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:17.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:17.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:17.734 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:17.762 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:17.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:17.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:17.863 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:17.897 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:17.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:17.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:17.999 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:18.026 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:18.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:18.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:17 smithi063 ceph-mon[114756]: pgmap v1279: 89 pgs: 89 active+clean; 780 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-08-17T17:17:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:17 smithi063 ceph-mon[109843]: pgmap v1279: 89 pgs: 89 active+clean; 780 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 837 op/s 2024-08-17T17:17:18.128 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:18.160 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:18.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:18.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:18.261 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:18.305 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:18.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:18.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:18.407 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:18.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:18.441 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:18.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:18.542 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:18.553 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:17:18.567 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:18.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:18.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:18.669 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:18.703 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:18.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:18.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:18.871 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:18.902 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:18.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:18.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:19.004 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:19.034 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:19.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:19.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:19.135 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:19.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:19.180 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:19.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:19.281 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:19.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:19.318 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:19.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:19.419 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:19.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:19.459 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:19.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:19.560 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:19.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:19.608 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:19.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:19.645 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:19.705 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:17:19.709 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:19.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:19.774 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:19.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:19.794 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:19 smithi063 ceph-mon[109843]: pgmap v1280: 89 pgs: 89 active+clean; 596 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2024-08-17T17:17:19.794 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:19 smithi063 ceph-mon[114756]: pgmap v1280: 89 pgs: 89 active+clean; 596 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2024-08-17T17:17:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:19 smithi162 ceph-mon[104668]: pgmap v1280: 89 pgs: 89 active+clean; 596 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2024-08-17T17:17:19.875 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:19.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:19.938 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:19.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:20.039 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:20.066 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:20.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:20.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:20.168 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:20.212 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:20.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:20.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:20.314 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:20.339 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:20.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:20.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:20.441 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:20.480 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:20.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:20.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:20.582 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:20.605 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:17:20.623 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:20.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:20.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:20.725 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:20.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:20.763 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:20.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:20.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:20 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2015377173' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:17:20.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:20 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:17:20.864 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:20.893 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:20.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:20.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:20.960 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:20 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2015377173' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:17:20.960 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:20 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:17:20.961 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:20 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2015377173' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:17:20.961 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:20 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:17:20.996 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:21.039 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:21.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:21.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:21.141 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:21.178 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:21.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:21.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:21.280 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:21.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:21.324 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:21.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:21.425 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:21.465 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:21.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:21.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:21.567 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:21.608 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:21.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:21.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:21.710 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:21.745 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:21.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:21.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:21 smithi063 ceph-mon[114756]: pgmap v1281: 89 pgs: 89 active+clean; 529 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2024-08-17T17:17:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:21 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:21 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:17:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:21 smithi063 ceph-mon[114756]: osdmap e1119: 8 total, 8 up, 8 in 2024-08-17T17:17:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:21 smithi063 ceph-mon[109843]: pgmap v1281: 89 pgs: 89 active+clean; 529 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2024-08-17T17:17:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:21 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:21 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:17:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:21 smithi063 ceph-mon[109843]: osdmap e1119: 8 total, 8 up, 8 in 2024-08-17T17:17:21.822 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:21.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:21 smithi162 ceph-mon[104668]: pgmap v1281: 89 pgs: 89 active+clean; 529 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2024-08-17T17:17:21.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:21 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:21.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:21 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:17:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:21 smithi162 ceph-mon[104668]: osdmap e1119: 8 total, 8 up, 8 in 2024-08-17T17:17:21.847 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:21.937 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:21.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:21.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:22.039 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:22.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:22.071 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:22.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:22.172 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:22.195 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:22.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:22.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:22.297 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:22.329 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:22.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:22.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:22.376 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:17:22.376 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:17:22.376 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:17:22.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd out 0 2024-08-17T17:17:22.431 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:22.472 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:22.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:22.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:22.573 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:22.627 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:22.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:22.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:22.728 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:22.758 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:22.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:22.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:22.860 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:22.905 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:22.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:22.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:23.007 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:23.032 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:23.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:23.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:23.133 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:23.169 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:23.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:23.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:23.271 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:23.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:23.307 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:23.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:23.408 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:23.415 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:17:23.447 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:23.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:23.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:23.452 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:17:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:17:23] "GET /metrics HTTP/1.1" 200 36404 "" "Prometheus/2.51.0" 2024-08-17T17:17:23.549 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:23.574 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:23.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:23.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:23.676 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:23.746 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:23 smithi063 ceph-mon[109843]: pgmap v1283: 89 pgs: 89 active+clean; 464 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2024-08-17T17:17:23.746 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:23 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:23.746 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:23 smithi063 ceph-mon[114756]: pgmap v1283: 89 pgs: 89 active+clean; 464 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2024-08-17T17:17:23.746 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:23 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:23.747 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:17:23.747 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:17:23.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:23.783 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:23.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:23.884 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:23.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:23.948 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:23.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:24.049 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:23 smithi162 ceph-mon[104668]: pgmap v1283: 89 pgs: 89 active+clean; 464 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 713 op/s 2024-08-17T17:17:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:23 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:17:24.110 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:24.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:24.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:24.212 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:24.254 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:24.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:24.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:24.355 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:24.398 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:24.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:24.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:24.500 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:24.531 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:24.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:24.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:24.632 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:24.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:24.682 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:24.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:24.693 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:17:24.783 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:24.819 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:24.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:24.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:24.922 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:24.958 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:24.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:24.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:24.970 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:25.026 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:17:25.026 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:17:25.060 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:25.072 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:17:25.072 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:17:25.072 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:25.107 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:17:25.108 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:17:25.121 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:25.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:25.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:25.148 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:17:25.148 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:17:25.179 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:17:25.179 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:17:25.217 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:17:25.217 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:17:25.224 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:25.245 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:25.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:25.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:25.253 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:17:25.253 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:17:25.285 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:17:25.285 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:17:25.347 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:25.401 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:25.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:25.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:25.503 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:25.530 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:25.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:25.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:25.632 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:25.663 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:25.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:25.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:25.764 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:25.811 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:25.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:25.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:25.900 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:25 smithi162 ceph-mon[104668]: pgmap v1284: 89 pgs: 89 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2024-08-17T17:17:25.900 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:25 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1920960365' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:25.909 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:25 smithi063 ceph-mon[114756]: pgmap v1284: 89 pgs: 89 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2024-08-17T17:17:25.909 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:25 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1920960365' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:25.909 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:25 smithi063 ceph-mon[109843]: pgmap v1284: 89 pgs: 89 active+clean; 332 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 818 op/s 2024-08-17T17:17:25.909 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:25 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1920960365' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:25.912 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:25.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:25.944 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:25.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:26.045 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:26.083 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:26.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:26.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:26.185 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:26.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:26.227 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:26.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:26.328 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:26.360 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:26.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:26.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:26.461 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:26.498 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:26.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:26.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:26.600 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:26.641 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:26.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:26.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:26.743 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:26.774 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:26.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:26.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:26.876 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:26.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:26.931 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:26.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:27.032 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:27.075 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:27.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:27.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:27.177 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:27.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:27.206 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:27.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:27.307 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:27.365 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:27.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:27.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:27.467 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:27.502 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:27.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:27.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:27.603 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:27.645 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:27.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:27.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:27.747 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:27.775 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:27.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:27.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:27.864 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:27 smithi063 ceph-mon[109843]: pgmap v1285: 89 pgs: 89 active+clean; 269 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2024-08-17T17:17:27.864 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:27 smithi063 ceph-mon[114756]: pgmap v1285: 89 pgs: 89 active+clean; 269 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2024-08-17T17:17:27.877 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:27.908 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:27.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:27.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:28.010 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:28.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.049 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:28.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:27 smithi162 ceph-mon[104668]: pgmap v1285: 89 pgs: 89 active+clean; 269 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2024-08-17T17:17:28.098 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Removed 126104 objects 2024-08-17T17:17:28.098 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Clean up completed and total clean up time :151.013 2024-08-17T17:17:28.150 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:28.172 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:28.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:28.274 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:28.296 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:28.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:28.339 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:17:28.399 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:28.419 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:28.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:28.521 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:28.544 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:28.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:28.645 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:28.668 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:28.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:28.770 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:28.791 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:28.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:28.893 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:28.916 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:28.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:28.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:29.018 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:29.040 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:29.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:29.142 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:29.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.166 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:29.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:29.266 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:29.288 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:29.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:29.390 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:29.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.413 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:29.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:29.514 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:29.535 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:29.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:29.565 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:29.637 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:29.658 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:29.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:29.760 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:29.783 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:29.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:29.823 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:29 smithi063 ceph-mon[109843]: pgmap v1286: 89 pgs: 89 active+clean; 85 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2024-08-17T17:17:29.823 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:29 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:29.823 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:29 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:29.823 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:29 smithi063 ceph-mon[114756]: pgmap v1286: 89 pgs: 89 active+clean; 85 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2024-08-17T17:17:29.824 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:29 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:29.824 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:29 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:29.885 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:29.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:29.909 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:29.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:30.010 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:30.031 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:30.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:29 smithi162 ceph-mon[104668]: pgmap v1286: 89 pgs: 89 active+clean; 85 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 817 op/s 2024-08-17T17:17:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:29 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:29 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:30.133 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:30.155 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:30.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:30.257 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:30.279 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:30.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:30.381 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:30.404 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:30.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:30.505 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:30.528 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:30.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:30.609 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:17:30.630 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:30.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.654 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:30.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:30.754 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:30.776 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:30.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:30.878 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:30.903 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:30.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:30.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:31.005 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:30 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3374017957' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:31.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:30 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3374017957' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:31.030 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:31.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:31.061 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-08-17T17:17:31.061 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-08-17T17:17:31.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:30 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3374017957' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:31.133 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:31.158 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:31.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:31.260 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:31.282 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:31.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:31.383 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:31.409 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:31.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:31.511 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:31.533 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:31.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.534 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:31.635 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:31.660 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:31.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:31.762 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:31.785 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:31.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:31.887 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:31.912 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:31.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:31.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:31.964 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:31 smithi063 ceph-mon[109843]: pgmap v1287: 89 pgs: 89 active+clean; 39 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-08-17T17:17:31.964 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:31 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:31.964 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:31 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3374017957' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:17:31.964 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:31 smithi063 ceph-mon[109843]: osdmap e1120: 8 total, 8 up, 8 in 2024-08-17T17:17:31.964 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:31.964 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:31.965 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:31 smithi063 ceph-mon[114756]: pgmap v1287: 89 pgs: 89 active+clean; 39 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-08-17T17:17:31.965 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:31 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:31.965 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:31 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3374017957' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:17:31.965 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:31 smithi063 ceph-mon[114756]: osdmap e1120: 8 total, 8 up, 8 in 2024-08-17T17:17:31.965 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:31.965 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:32.013 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:32.037 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:32.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:31 smithi162 ceph-mon[104668]: pgmap v1287: 89 pgs: 89 active+clean; 39 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 783 op/s 2024-08-17T17:17:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:31 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:31 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3374017957' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:17:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:31 smithi162 ceph-mon[104668]: osdmap e1120: 8 total, 8 up, 8 in 2024-08-17T17:17:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:32.138 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:32.164 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:32.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:32.266 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:32.288 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:32.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:32.389 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:32.413 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:32.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:32.515 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:32.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:32 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a[109820]: 2024-08-17T17:17:32.260+0000 7f66fd20a640 -1 mon.a@0(leader).osd e1120 definitely_dead 0 2024-08-17T17:17:32.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.539 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:32.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:32.618 INFO:teuthology.orchestra.run.smithi063.stderr:marked out osd.0. 2024-08-17T17:17:32.639 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:32.662 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:32.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:32.764 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:32.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.790 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:32.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:32.891 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:32.913 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:32.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:32.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:32.979 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:32 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/503127899' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T17:17:32.979 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:32 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T17:17:32.979 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:32 smithi063 ceph-mon[114756]: Client client.admin marked osd.0 out, while it was still marked up 2024-08-17T17:17:32.979 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:32 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/503127899' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T17:17:32.979 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:32 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T17:17:32.979 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:32 smithi063 ceph-mon[109843]: Client client.admin marked osd.0 out, while it was still marked up 2024-08-17T17:17:33.016 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:33.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.040 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:33.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:32 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/503127899' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T17:17:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:32 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["0"]} : dispatch 2024-08-17T17:17:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:32 smithi162 ceph-mon[104668]: Client client.admin marked osd.0 out, while it was still marked up 2024-08-17T17:17:33.141 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:33.164 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:33.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:33.265 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:33.291 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:33.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:33.393 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:33.419 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:33.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:33.420 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:17:33.420 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:17:33.421 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:17:33.421 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:17:33.421 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:17:33.521 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:33.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:33.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:33.614 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:17:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:17:33] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.51.0" 2024-08-17T17:17:33.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:33 smithi063 ceph-mon[109843]: pgmap v1289: 89 pgs: 89 active+clean; 25 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-08-17T17:17:33.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:33 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-08-17T17:17:33.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:33 smithi063 ceph-mon[109843]: osdmap e1121: 8 total, 8 up, 7 in 2024-08-17T17:17:33.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:33 smithi063 ceph-mon[109843]: osdmap e1122: 8 total, 8 up, 7 in 2024-08-17T17:17:33.615 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:33 smithi063 ceph-mon[114756]: pgmap v1289: 89 pgs: 89 active+clean; 25 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-08-17T17:17:33.615 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:33 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-08-17T17:17:33.615 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:33 smithi063 ceph-mon[114756]: osdmap e1121: 8 total, 8 up, 7 in 2024-08-17T17:17:33.615 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:33 smithi063 ceph-mon[114756]: osdmap e1122: 8 total, 8 up, 7 in 2024-08-17T17:17:33.622 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:33.648 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:33.671 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:33.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:33.773 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:33.802 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:33.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:33.904 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:33.925 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:33.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:33.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:33.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:33 smithi162 ceph-mon[104668]: pgmap v1289: 89 pgs: 89 active+clean; 25 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 701 op/s 2024-08-17T17:17:33.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:33 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2024-08-17T17:17:33.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:33 smithi162 ceph-mon[104668]: osdmap e1121: 8 total, 8 up, 7 in 2024-08-17T17:17:33.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:33 smithi162 ceph-mon[104668]: osdmap e1122: 8 total, 8 up, 7 in 2024-08-17T17:17:34.027 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:34.048 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:34.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:34.137 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:17:34.203 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:34.224 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:34.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:34.326 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:34.348 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:34.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:34.450 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:34.472 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:34.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:34.565 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:17:34.574 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:34.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.597 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:34.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:34.698 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:34.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.721 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:34.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:34.822 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:34.843 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:34.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:34.866 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:34.946 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:34.966 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:34.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:34.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:34.981 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:34 smithi063 ceph-mon[114756]: osdmap e1123: 8 total, 8 up, 7 in 2024-08-17T17:17:34.981 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:34 smithi063 ceph-mon[114756]: 6.a starting backfill to osd.2 from (0'0,0'0] MAX to 1119'15854 2024-08-17T17:17:34.981 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:34 smithi063 ceph-mon[114756]: 6.d starting backfill to osd.2 from (0'0,0'0] MAX to 1119'15520 2024-08-17T17:17:34.981 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:34 smithi063 ceph-mon[114756]: 6.0 starting backfill to osd.5 from (0'0,0'0] MAX to 1119'15692 2024-08-17T17:17:34.981 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:34 smithi063 ceph-mon[114756]: 6.9 starting backfill to osd.3 from (0'0,0'0] MAX to 1119'15898 2024-08-17T17:17:34.981 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:34 smithi063 ceph-mon[109843]: osdmap e1123: 8 total, 8 up, 7 in 2024-08-17T17:17:34.982 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:34 smithi063 ceph-mon[109843]: 6.a starting backfill to osd.2 from (0'0,0'0] MAX to 1119'15854 2024-08-17T17:17:34.982 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:34 smithi063 ceph-mon[109843]: 6.d starting backfill to osd.2 from (0'0,0'0] MAX to 1119'15520 2024-08-17T17:17:34.982 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:34 smithi063 ceph-mon[109843]: 6.0 starting backfill to osd.5 from (0'0,0'0] MAX to 1119'15692 2024-08-17T17:17:34.982 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:34 smithi063 ceph-mon[109843]: 6.9 starting backfill to osd.3 from (0'0,0'0] MAX to 1119'15898 2024-08-17T17:17:35.068 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:35.092 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:35.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:34 smithi162 ceph-mon[104668]: osdmap e1123: 8 total, 8 up, 7 in 2024-08-17T17:17:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:34 smithi162 ceph-mon[104668]: 6.a starting backfill to osd.2 from (0'0,0'0] MAX to 1119'15854 2024-08-17T17:17:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:34 smithi162 ceph-mon[104668]: 6.d starting backfill to osd.2 from (0'0,0'0] MAX to 1119'15520 2024-08-17T17:17:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:34 smithi162 ceph-mon[104668]: 6.0 starting backfill to osd.5 from (0'0,0'0] MAX to 1119'15692 2024-08-17T17:17:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:34 smithi162 ceph-mon[104668]: 6.9 starting backfill to osd.3 from (0'0,0'0] MAX to 1119'15898 2024-08-17T17:17:35.193 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:35.216 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:35.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:35.318 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:35.340 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:35.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:35.427 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:35.427 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:35.442 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:35.463 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:35.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:35.564 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:35.585 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:35.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:35.687 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:35.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.709 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:35.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:35.811 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:35.834 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:35.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:35.935 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:35.958 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:35.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:35.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:36.059 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:36.081 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:36.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:35 smithi162 ceph-mon[104668]: pgmap v1293: 89 pgs: 89 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 155 op/s 2024-08-17T17:17:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:35 smithi162 ceph-mon[104668]: osdmap e1124: 8 total, 8 up, 7 in 2024-08-17T17:17:36.183 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:36.204 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:36.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.205 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:36.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:35 smithi063 ceph-mon[114756]: pgmap v1293: 89 pgs: 89 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 155 op/s 2024-08-17T17:17:36.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:35 smithi063 ceph-mon[114756]: osdmap e1124: 8 total, 8 up, 7 in 2024-08-17T17:17:36.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:35 smithi063 ceph-mon[109843]: pgmap v1293: 89 pgs: 89 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 0 B/s wr, 155 op/s 2024-08-17T17:17:36.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:35 smithi063 ceph-mon[109843]: osdmap e1124: 8 total, 8 up, 7 in 2024-08-17T17:17:36.306 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:36.327 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:36.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:36.428 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:36.449 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:36.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:36.551 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:36.573 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:36.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:36.675 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:36.696 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:36.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:36.798 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:36.820 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:36.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:36.881 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:36 smithi063 ceph-mon[114756]: osdmap e1125: 8 total, 8 up, 7 in 2024-08-17T17:17:36.882 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:36 smithi063 ceph-mon[109843]: osdmap e1125: 8 total, 8 up, 7 in 2024-08-17T17:17:36.922 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:36.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:36.945 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:36.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:37.045 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:37.066 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:37.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:36 smithi162 ceph-mon[104668]: osdmap e1125: 8 total, 8 up, 7 in 2024-08-17T17:17:37.169 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:37.190 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:37.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:37.292 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:37.312 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:37.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:37.414 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:37.439 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:37.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:37.541 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:37.563 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:37.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:37.665 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:37.686 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:37.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:37.788 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:37.811 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:37.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:37.913 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:37.934 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:37.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:37.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:37 smithi063 ceph-mon[114756]: pgmap v1296: 89 pgs: 1 active+clean+remapped, 88 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:17:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:37 smithi063 ceph-mon[109843]: pgmap v1296: 89 pgs: 1 active+clean+remapped, 88 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:17:38.037 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:38.060 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:38.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:37 smithi162 ceph-mon[104668]: pgmap v1296: 89 pgs: 1 active+clean+remapped, 88 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:17:38.162 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:38.185 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:38.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:38.287 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:38.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.310 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:38.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:38.411 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:38.435 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:38.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:38.537 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:38.560 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:38.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:38.662 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:38.684 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:38.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:38.786 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:38.811 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:38.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:38.912 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:38.935 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:38.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:38.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:39.037 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:39.059 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:39.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:39.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:39.161 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:39.183 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:39.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:39.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:39.285 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:39.310 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:39.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:39.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:39.412 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:39.434 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:39.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:39.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:39.536 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:39.558 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:39.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:39.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:39.659 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:39.685 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:39.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:39.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:39.786 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:39.810 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:17:39.811 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:17:39.812 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:39.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:39.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:39.914 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:39.939 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:39.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:39.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:40.040 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:40.062 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:40.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:40.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:39 smithi162 ceph-mon[104668]: pgmap v1297: 89 pgs: 1 active+clean+remapped, 88 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 108 KiB/s, 2 objects/s recovering 2024-08-17T17:17:40.096 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:39 smithi063 ceph-mon[114756]: pgmap v1297: 89 pgs: 1 active+clean+remapped, 88 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 108 KiB/s, 2 objects/s recovering 2024-08-17T17:17:40.096 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:39 smithi063 ceph-mon[109843]: pgmap v1297: 89 pgs: 1 active+clean+remapped, 88 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 108 KiB/s, 2 objects/s recovering 2024-08-17T17:17:40.165 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:40.190 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:40.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:40.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:40.292 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:40.315 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:40.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:40.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:40.416 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:40.442 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:40.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:40.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:40.544 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:40.567 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:40.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:40.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:40.668 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:40.694 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:40.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:40.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:40.795 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:40.821 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:40.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:40.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:40.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:40.945 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:40.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:40.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:41.047 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:41.073 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:41.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:41.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:41.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:40 smithi162 ceph-mon[104668]: pgmap v1298: 89 pgs: 89 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 91 KiB/s, 1 objects/s recovering 2024-08-17T17:17:41.176 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:41.202 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:41.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:41.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:41.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:40 smithi063 ceph-mon[114756]: pgmap v1298: 89 pgs: 89 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 91 KiB/s, 1 objects/s recovering 2024-08-17T17:17:41.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:40 smithi063 ceph-mon[109843]: pgmap v1298: 89 pgs: 89 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 91 KiB/s, 1 objects/s recovering 2024-08-17T17:17:41.304 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:41.329 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:41.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:41.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:41.431 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:41.456 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:41.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:41.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:41.558 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:41.584 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:41.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:41.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:41.686 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:41.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:41.712 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:41.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:41.813 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:41.836 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:41.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:41.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:41.938 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:41.964 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:41.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:41.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:42.066 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:42.091 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:42.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:42.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:42.193 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:42.218 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:42.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:42.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:42.319 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:42.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:42.345 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:42.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:42.446 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:42.472 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:42.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:42.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:42.573 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:42.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:42.600 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:42.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:42.701 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:42.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:42.728 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:42.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:42.829 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:42.854 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:42.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:42.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:42.956 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:42.981 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:42.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:42.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:43.083 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:43.107 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:43.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:43.209 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:43.233 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:43.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:43.335 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:43.359 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:43.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:43.461 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:43.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.486 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:43.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:43.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:17:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:17:43] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.51.0" 2024-08-17T17:17:43.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:43 smithi063 ceph-mon[114756]: pgmap v1299: 89 pgs: 89 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 72 KiB/s, 1 objects/s recovering 2024-08-17T17:17:43.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:43 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:43.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:43 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:43.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:43 smithi063 ceph-mon[109843]: pgmap v1299: 89 pgs: 89 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 72 KiB/s, 1 objects/s recovering 2024-08-17T17:17:43.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:43 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:43.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:43 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:43.587 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:43.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:43 smithi162 ceph-mon[104668]: pgmap v1299: 89 pgs: 89 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 72 KiB/s, 1 objects/s recovering 2024-08-17T17:17:43.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:43 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:43.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:43 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:43.612 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:43.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:43.714 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:43.738 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:43.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:43.840 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:43.863 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:43.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:43.965 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:43.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:43.991 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:43.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:44.092 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:44.115 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:44.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:44.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:44.217 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:44.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:44.240 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:44.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:44.341 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:44.367 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:17:44.367 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:17:44.367 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:17:44.367 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:17:44.367 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:17:44.367 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:17:44.367 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:17:44.367 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:17:44.368 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:17:44.368 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:17:44.368 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:17:44.368 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:17:44.368 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:17:44.368 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:17:44.368 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:17:44.368 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:17:44.373 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:44.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:44.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:44.474 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:44.498 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:44.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:44.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:44.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:44 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4027133977' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:44.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:44 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:44.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:44 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:17:44.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:44 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:44.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:44 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3455540255' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:17:44.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:44 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4027133977' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:44.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:44 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:44.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:44 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:17:44.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:44 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:44.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:44 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3455540255' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:17:44.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:44 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4027133977' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:44.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:44 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:17:44.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:44 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:17:44.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:44 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:17:44.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:44 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3455540255' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:17:44.600 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:44.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:44.624 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:44.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:44.725 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:44.747 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:44.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:44.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:44.849 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:44.871 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:44.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:44.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:44.901 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:17:44.905 INFO:teuthology.orchestra.run.smithi063.stderr:pool 'unique_pool_1' removed 2024-08-17T17:17:44.972 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:45.009 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:45.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:45.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:45.110 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:45.134 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:45.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:45.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:45.237 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:45.259 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:45.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:45.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:45.360 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:45.383 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:45.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:45.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:45.484 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:45.506 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:45.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:45.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:45.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:45 smithi063 ceph-mon[109843]: pgmap v1300: 89 pgs: 89 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 62 KiB/s, 1 objects/s recovering 2024-08-17T17:17:45.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:45 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1304422822' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:17:45.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:45 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:17:45.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:45 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:45.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:45 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3455540255' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:17:45.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:45 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:17:45.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:45 smithi063 ceph-mon[109843]: osdmap e1126: 8 total, 8 up, 7 in 2024-08-17T17:17:45.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:45 smithi063 ceph-mon[114756]: pgmap v1300: 89 pgs: 89 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 62 KiB/s, 1 objects/s recovering 2024-08-17T17:17:45.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:45 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1304422822' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:17:45.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:45 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:17:45.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:45 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:45.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:45 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3455540255' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:17:45.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:45 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:17:45.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:45 smithi063 ceph-mon[114756]: osdmap e1126: 8 total, 8 up, 7 in 2024-08-17T17:17:45.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:45 smithi162 ceph-mon[104668]: pgmap v1300: 89 pgs: 89 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 62 KiB/s, 1 objects/s recovering 2024-08-17T17:17:45.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:45 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1304422822' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:17:45.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:45 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:17:45.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:45 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:17:45.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:45 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3455540255' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:17:45.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:45 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:17:45.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:45 smithi162 ceph-mon[104668]: osdmap e1126: 8 total, 8 up, 7 in 2024-08-17T17:17:45.608 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:45.652 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:45.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:45.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:45.753 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:45.766 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:17:45.766 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:17:45.775 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:45.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:45.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:45.876 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:45.897 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:45.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:45.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:45.999 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:46.020 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:46.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:46.122 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:46.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.145 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:46.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:46.246 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:46.268 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:46.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:46.369 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:46.392 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:46.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:46.445 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]} : dispatch 2024-08-17T17:17:46.446 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]} : dispatch 2024-08-17T17:17:46.446 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]} : dispatch 2024-08-17T17:17:46.446 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]} : dispatch 2024-08-17T17:17:46.446 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]} : dispatch 2024-08-17T17:17:46.446 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]} : dispatch 2024-08-17T17:17:46.446 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]} : dispatch 2024-08-17T17:17:46.446 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]} : dispatch 2024-08-17T17:17:46.466 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:17:46.466 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:17:46.466 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-08-17T17:17:46.466 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_2 16 2024-08-17T17:17:46.494 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:46.516 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:46.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:46.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]} : dispatch 2024-08-17T17:17:46.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]} : dispatch 2024-08-17T17:17:46.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]} : dispatch 2024-08-17T17:17:46.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]} : dispatch 2024-08-17T17:17:46.618 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:46.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.641 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:46.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:46.742 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:46.763 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:46.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:46.864 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:46.887 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:46.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:46.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:46.947 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:17:46.989 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:47.014 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:47.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:47.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:47.116 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:47.134 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:17:47.140 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:47.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:47.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:47.180 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:47.221 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:47.242 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:47.265 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:47.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:47.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:47.358 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:17:47.367 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:47.390 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:47.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:47.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:47.410 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:47 smithi063 ceph-mon[114756]: pgmap v1302: 73 pgs: 73 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:17:47.410 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-08-17T17:17:47.410 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-08-17T17:17:47.410 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:47 smithi063 ceph-mon[114756]: osdmap e1127: 8 total, 8 up, 7 in 2024-08-17T17:17:47.410 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:47.411 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:47 smithi063 ceph-mon[109843]: pgmap v1302: 73 pgs: 73 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:17:47.411 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-08-17T17:17:47.411 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-08-17T17:17:47.411 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:47 smithi063 ceph-mon[109843]: osdmap e1127: 8 total, 8 up, 7 in 2024-08-17T17:17:47.411 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:47.481 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:47 smithi162 ceph-mon[104668]: pgmap v1302: 73 pgs: 73 active+clean; 582 KiB data, 1.1 GiB used, 625 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-08-17T17:17:47.481 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-08-17T17:17:47.481 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [1, 5]}]': finished 2024-08-17T17:17:47.481 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:47 smithi162 ceph-mon[104668]: osdmap e1127: 8 total, 8 up, 7 in 2024-08-17T17:17:47.481 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:17:47.492 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:47.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:47.514 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:47.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:47.589 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:17:47.616 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:47.654 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:47.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:47.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:47.755 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:47.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:47.778 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:47.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:47.878 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:47.899 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:47.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:47.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:48.002 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:48.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:48.024 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:48.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:48.124 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:48.147 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:48.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:48.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:48.249 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:48.271 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:48.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:48.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:48.372 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:48.395 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:48.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:48.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:48.496 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:48.520 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:48.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:48.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:48.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:48 smithi162 ceph-mon[104668]: osdmap e1128: 8 total, 8 up, 7 in 2024-08-17T17:17:48.622 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:48.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:48.646 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:48.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:48.746 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:48.761 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:48 smithi063 ceph-mon[114756]: osdmap e1128: 8 total, 8 up, 7 in 2024-08-17T17:17:48.761 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:48 smithi063 ceph-mon[109843]: osdmap e1128: 8 total, 8 up, 7 in 2024-08-17T17:17:48.768 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:48.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:48.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:48.869 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:48.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:48.892 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:48.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:48.993 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:49.015 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:49.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:49.049 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:17:49.049 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:17:49.117 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:49.140 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:49.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:49.242 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:49.264 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:49.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:49.365 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:49.388 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:49.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:49.489 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:49.512 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:49.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:49.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:49 smithi063 ceph-mon[114756]: pgmap v1305: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:49.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:49 smithi063 ceph-mon[114756]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:17:49.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:49 smithi063 ceph-mon[109843]: pgmap v1305: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:49.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:49 smithi063 ceph-mon[109843]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:17:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:49 smithi162 ceph-mon[104668]: pgmap v1305: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:49 smithi162 ceph-mon[104668]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:17:49.613 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:49.635 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:49.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:49.689 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:17:49.736 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:49.737 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:49.758 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:49.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:49.859 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:49.881 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:49.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:49.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:49.897 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:17:49.983 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:50.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.007 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:50.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:50.108 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:50.131 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:50.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:50.233 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:50.255 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:50.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:50.357 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:50.379 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:50.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:50.481 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:50.503 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:50.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:50.604 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:50.626 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:50.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:50.728 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:50.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.750 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:50.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:50.851 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:50.874 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:50.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:50.975 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:50.997 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:50.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:50.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:51.099 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:51.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:51.120 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:51.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:51.161 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:17:51.161 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:17:51.173 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:17:51.173 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:17:51.190 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:17:51.190 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:17:51.206 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:17:51.206 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:17:51.221 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:51.222 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:17:51.222 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:17:51.240 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:17:51.240 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:17:51.243 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:51.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:51.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:51.256 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:17:51.256 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:17:51.272 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:17:51.272 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:17:51.300 INFO:teuthology.orchestra.run.smithi063.stderr:pool 'unique_pool_2' created 2024-08-17T17:17:51.312 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:51 smithi063 ceph-mon[114756]: pgmap v1306: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:51.312 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:51 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4149334200' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T17:17:51.312 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:51 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T17:17:51.312 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:51 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3964611583' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:51.312 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:51 smithi063 ceph-mon[109843]: pgmap v1306: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:51.312 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:51 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4149334200' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T17:17:51.313 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:51 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T17:17:51.313 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:51 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3964611583' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:51.344 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:51.366 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:51.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:51.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:51.468 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:51.490 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:51.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:51.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:51.592 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:51.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:51 smithi162 ceph-mon[104668]: pgmap v1306: 73 pgs: 1 peering, 72 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:51.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:51 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4149334200' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T17:17:51.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:51 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-08-17T17:17:51.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:51 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3964611583' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:51.614 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:51.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:51.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:51.716 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:51.720 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:17:51.720 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:17:51.720 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:17:51.720 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:17:51.721 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:17:51.738 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:51.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:51.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:51.840 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:51.862 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:51.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:51.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:51.964 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:51.986 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:51.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:51.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:52.088 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:52.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:52.111 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:52.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:52.212 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:52.233 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:52.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:52.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:52.326 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:17:52.335 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:52.359 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:52.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:52.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:52.461 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:52.483 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:52.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:52.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:52.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:52 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-08-17T17:17:52.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:52 smithi063 ceph-mon[114756]: osdmap e1129: 8 total, 8 up, 7 in 2024-08-17T17:17:52.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:52 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-08-17T17:17:52.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:52 smithi063 ceph-mon[109843]: osdmap e1129: 8 total, 8 up, 7 in 2024-08-17T17:17:52.530 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:17:52.584 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:52.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:52 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-08-17T17:17:52.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:52 smithi162 ceph-mon[104668]: osdmap e1129: 8 total, 8 up, 7 in 2024-08-17T17:17:52.606 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:52.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:52.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:52.707 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:52.730 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:52.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:52.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:52.832 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:52.853 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:52.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:52.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:52.948 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-08-17T17:17:52.954 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:53.015 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:53.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:53.116 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:53.139 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:53.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:53.240 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:53.262 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:53.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.263 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:53.303 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:17:53.364 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:53.385 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:53.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:53.487 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:53.508 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:53.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:53.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:53 smithi162 ceph-mon[104668]: pgmap v1308: 89 pgs: 6 creating+peering, 10 unknown, 1 peering, 72 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:53.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:53 smithi162 ceph-mon[104668]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:17:53.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:53 smithi162 ceph-mon[104668]: osdmap e1130: 8 total, 8 up, 7 in 2024-08-17T17:17:53.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:53 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4227408547' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:53.597 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:17:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:17:53] "GET /metrics HTTP/1.1" 200 34048 "" "Prometheus/2.51.0" 2024-08-17T17:17:53.597 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:53 smithi063 ceph-mon[114756]: pgmap v1308: 89 pgs: 6 creating+peering, 10 unknown, 1 peering, 72 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:53.597 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:53 smithi063 ceph-mon[114756]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:17:53.597 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:53 smithi063 ceph-mon[114756]: osdmap e1130: 8 total, 8 up, 7 in 2024-08-17T17:17:53.597 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:53 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4227408547' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:53.597 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:53 smithi063 ceph-mon[109843]: pgmap v1308: 89 pgs: 6 creating+peering, 10 unknown, 1 peering, 72 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:53.598 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:53 smithi063 ceph-mon[109843]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:17:53.598 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:53 smithi063 ceph-mon[109843]: osdmap e1130: 8 total, 8 up, 7 in 2024-08-17T17:17:53.598 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:53 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4227408547' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:17:53.610 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:53.633 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:53.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:53.734 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:53.755 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:53.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:53.857 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:53.879 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:53.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:53.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:53.980 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:54.003 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:54.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:54.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:54.105 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:54.126 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:54.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:54.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:54.198 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:54.228 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:54.250 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:54.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:54.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:54.336 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:54 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:17:54.336 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:54 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4227408547' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:17:54.336 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:54 smithi162 ceph-mon[104668]: osdmap e1131: 8 total, 8 up, 7 in 2024-08-17T17:17:54.352 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:54.362 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:17:54.362 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:17:54.362 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:17:54.362 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:17:54.362 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.362 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:29353 6.2 6:4762f47f:::benchmark_data_smithi063_160268_object29352:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:17:54.362 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.379378+0000", 2024-08-17T17:17:54.362 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 256.9807242, 2024-08-17T17:17:54.363 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.102636264, 2024-08-17T17:17:54.363 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.363 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:54.363 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:17:54.363 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:17:54.363 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:17:54.363 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 29353 2024-08-17T17:17:54.363 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.363 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.363 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.364 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.364 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379378+0000", 2024-08-17T17:17:54.364 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.364 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.364 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.364 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.364 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379378+0000", 2024-08-17T17:17:54.364 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.364 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.364 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.364 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.364 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379377+0000", 2024-08-17T17:17:54.364 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:17:54.365 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.365 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.365 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.365 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379441+0000", 2024-08-17T17:17:54.365 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4055999999999998e-05 2024-08-17T17:17:54.365 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.365 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.365 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.365 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379454+0000", 2024-08-17T17:17:54.365 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2724e-05 2024-08-17T17:17:54.365 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.366 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.366 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.366 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379459+0000", 2024-08-17T17:17:54.366 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1760000000000001e-06 2024-08-17T17:17:54.366 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.366 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.366 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.366 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379496+0000", 2024-08-17T17:17:54.366 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6964e-05 2024-08-17T17:17:54.366 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.366 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.366 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.367 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379605+0000", 2024-08-17T17:17:54.367 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010905599999999999 2024-08-17T17:17:54.367 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.367 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.367 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:17:54.367 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.379736+0000", 2024-08-17T17:17:54.367 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013126699999999999 2024-08-17T17:17:54.367 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.367 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.367 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:17:54.367 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.380909+0000", 2024-08-17T17:17:54.367 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001172511 2024-08-17T17:17:54.367 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.368 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.368 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.368 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.382749+0000", 2024-08-17T17:17:54.368 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001840291 2024-08-17T17:17:54.368 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.368 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.368 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.368 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481915+0000", 2024-08-17T17:17:54.368 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.099166016999999995 2024-08-17T17:17:54.368 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.368 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.368 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:17:54.368 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481972+0000", 2024-08-17T17:17:54.368 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.694e-05 2024-08-17T17:17:54.369 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.369 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.369 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.369 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.482014+0000", 2024-08-17T17:17:54.369 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2472000000000003e-05 2024-08-17T17:17:54.369 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.369 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.369 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.369 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.369 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.369 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:29370 6.e 6:79482805:::benchmark_data_smithi063_160268_object29369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:17:54.369 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.386359+0000", 2024-08-17T17:17:54.370 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 256.97374311999999, 2024-08-17T17:17:54.370 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.097030557000000003, 2024-08-17T17:17:54.370 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.370 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:54.370 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:17:54.370 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:17:54.370 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:17:54.370 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 29370 2024-08-17T17:17:54.370 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.371 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.371 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.371 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.371 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386359+0000", 2024-08-17T17:17:54.371 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.371 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.371 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.371 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.371 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386359+0000", 2024-08-17T17:17:54.371 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.371 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.371 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.371 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.372 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386356+0000", 2024-08-17T17:17:54.372 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999967 2024-08-17T17:17:54.372 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.372 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.372 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.372 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386386+0000", 2024-08-17T17:17:54.372 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9947999999999999e-05 2024-08-17T17:17:54.372 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.372 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.372 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.372 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386398+0000", 2024-08-17T17:17:54.372 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2135e-05 2024-08-17T17:17:54.373 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.373 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.373 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.373 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386405+0000", 2024-08-17T17:17:54.373 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1459999999999999e-06 2024-08-17T17:17:54.373 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.373 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.373 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.373 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386478+0000", 2024-08-17T17:17:54.373 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2360000000000005e-05 2024-08-17T17:17:54.373 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.373 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.373 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.373 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386591+0000", 2024-08-17T17:17:54.374 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011355 2024-08-17T17:17:54.374 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.374 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.374 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:17:54.374 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386720+0000", 2024-08-17T17:17:54.374 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012937800000000001 2024-08-17T17:17:54.374 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.374 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.374 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:17:54.374 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481898+0000", 2024-08-17T17:17:54.374 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.095177169000000006 2024-08-17T17:17:54.374 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.375 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.375 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.375 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483085+0000", 2024-08-17T17:17:54.375 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0011874190000000001 2024-08-17T17:17:54.375 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.375 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.375 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.375 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483111+0000", 2024-08-17T17:17:54.375 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5942999999999999e-05 2024-08-17T17:17:54.375 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.375 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.375 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:17:54.375 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483351+0000", 2024-08-17T17:17:54.376 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00024028299999999999 2024-08-17T17:17:54.376 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.376 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.376 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.376 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483390+0000", 2024-08-17T17:17:54.376 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8439999999999998e-05 2024-08-17T17:17:54.376 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.376 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.376 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.376 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.376 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.376 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:29371 6.2 6:4ffb2a7f:::benchmark_data_smithi063_160268_object29370:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:17:54.376 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.386614+0000", 2024-08-17T17:17:54.376 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 256.97348781900001, 2024-08-17T17:17:54.377 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.099154596999999997, 2024-08-17T17:17:54.377 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.377 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:54.377 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:17:54.377 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:17:54.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:54.379 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:17:54.379 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 29371 2024-08-17T17:17:54.379 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.379 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.379 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.379 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.379 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386614+0000", 2024-08-17T17:17:54.379 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.379 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.379 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.380 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.380 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386614+0000", 2024-08-17T17:17:54.380 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.380 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.380 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.380 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.380 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386614+0000", 2024-08-17T17:17:54.380 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:17:54.380 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.380 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.380 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.380 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386660+0000", 2024-08-17T17:17:54.380 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5952999999999999e-05 2024-08-17T17:17:54.381 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.381 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.381 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.381 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386673+0000", 2024-08-17T17:17:54.381 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3614999999999999e-05 2024-08-17T17:17:54.381 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.381 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.381 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.381 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386680+0000", 2024-08-17T17:17:54.381 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5150000000000003e-06 2024-08-17T17:17:54.381 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.381 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.381 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.382 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386726+0000", 2024-08-17T17:17:54.382 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6020000000000003e-05 2024-08-17T17:17:54.382 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.382 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.382 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.382 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386849+0000", 2024-08-17T17:17:54.382 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012303699999999999 2024-08-17T17:17:54.382 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.382 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.382 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:17:54.382 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.386935+0000", 2024-08-17T17:17:54.383 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6405999999999999e-05 2024-08-17T17:17:54.383 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.383 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.383 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.383 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.482051+0000", 2024-08-17T17:17:54.383 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.095116368000000007 2024-08-17T17:17:54.383 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.383 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.383 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.383 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483433+0000", 2024-08-17T17:17:54.383 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013810280000000001 2024-08-17T17:17:54.383 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.383 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.383 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:17:54.384 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.485669+0000", 2024-08-17T17:17:54.384 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002236848 2024-08-17T17:17:54.384 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.384 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.384 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:17:54.384 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.485731+0000", 2024-08-17T17:17:54.384 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1408000000000006e-05 2024-08-17T17:17:54.384 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.384 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.384 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.384 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.485769+0000", 2024-08-17T17:17:54.384 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8269000000000002e-05 2024-08-17T17:17:54.385 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.385 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.385 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.385 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.385 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.385 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop_reply(client.44609.0:29353 6.2 e1093/1086 ondisk, result = 0)", 2024-08-17T17:17:54.385 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.387682+0000", 2024-08-17T17:17:54.385 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 256.97242017000002, 2024-08-17T17:17:54.385 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.094339362999999996, 2024-08-17T17:17:54.385 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.385 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:17:54.385 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.386 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.386 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.386 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387682+0000", 2024-08-17T17:17:54.386 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.386 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.386 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.386 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.386 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387682+0000", 2024-08-17T17:17:54.386 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.386 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.386 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.386 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.386 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387681+0000", 2024-08-17T17:17:54.387 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:17:54.387 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.387 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.387 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.387 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387685+0000", 2024-08-17T17:17:54.387 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8709999999999999e-06 2024-08-17T17:17:54.387 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.387 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.387 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.387 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387699+0000", 2024-08-17T17:17:54.387 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4141999999999999e-05 2024-08-17T17:17:54.387 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.387 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.388 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.388 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387704+0000", 2024-08-17T17:17:54.388 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2370000000000003e-06 2024-08-17T17:17:54.388 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.388 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.388 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.388 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481898+0000", 2024-08-17T17:17:54.388 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.094193830000000006 2024-08-17T17:17:54.388 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.388 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.388 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.388 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481909+0000", 2024-08-17T17:17:54.388 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0992000000000001e-05 2024-08-17T17:17:54.388 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.388 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.389 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.389 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.482021+0000", 2024-08-17T17:17:54.389 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011291899999999999 2024-08-17T17:17:54.389 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.389 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.389 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.389 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.389 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.389 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.44609.0:29362 6.6 e1093/1086 6:6804913c:::benchmark_data_smithi063_160268_object29361:head v 1093'1824, mlcod=1093'1822)", 2024-08-17T17:17:54.389 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.387711+0000", 2024-08-17T17:17:54.389 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 256.97239171500001, 2024-08-17T17:17:54.389 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.095442569000000005, 2024-08-17T17:17:54.389 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.390 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:54.390 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.390 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.390 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.390 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387711+0000", 2024-08-17T17:17:54.390 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.390 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.390 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.390 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.390 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387711+0000", 2024-08-17T17:17:54.390 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.390 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.390 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.390 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.391 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387710+0000", 2024-08-17T17:17:54.391 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967296 2024-08-17T17:17:54.391 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.391 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.391 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.391 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387734+0000", 2024-08-17T17:17:54.391 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3649e-05 2024-08-17T17:17:54.391 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.391 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.391 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.391 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387738+0000", 2024-08-17T17:17:54.391 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4499999999999997e-06 2024-08-17T17:17:54.391 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.391 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.392 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.392 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387741+0000", 2024-08-17T17:17:54.392 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.7580000000000001e-06 2024-08-17T17:17:54.392 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.392 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.392 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.392 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387771+0000", 2024-08-17T17:17:54.392 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9403e-05 2024-08-17T17:17:54.392 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.392 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.392 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.392 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387790+0000", 2024-08-17T17:17:54.392 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9113000000000001e-05 2024-08-17T17:17:54.392 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.393 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.393 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:17:54.393 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483113+0000", 2024-08-17T17:17:54.393 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.095323518999999995 2024-08-17T17:17:54.393 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.393 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.393 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.393 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483153+0000", 2024-08-17T17:17:54.393 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9910000000000002e-05 2024-08-17T17:17:54.393 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.393 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.393 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.393 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.394 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.394 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.44609.0:29372 6.6 e1093/1086 6:6679e72a:::benchmark_data_smithi063_160268_object29371:head v 1093'1825, mlcod=1093'1822)", 2024-08-17T17:17:54.394 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.387747+0000", 2024-08-17T17:17:54.394 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 256.97235567500002, 2024-08-17T17:17:54.394 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.098332927000000001, 2024-08-17T17:17:54.394 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.394 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:54.394 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.394 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.394 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.394 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387747+0000", 2024-08-17T17:17:54.394 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.394 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.395 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.395 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.395 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387747+0000", 2024-08-17T17:17:54.395 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.395 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.395 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.395 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.395 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387746+0000", 2024-08-17T17:17:54.395 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:17:54.395 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.395 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.395 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.395 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387876+0000", 2024-08-17T17:17:54.395 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012919299999999999 2024-08-17T17:17:54.395 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.396 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.396 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.396 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387882+0000", 2024-08-17T17:17:54.396 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5189999999999998e-06 2024-08-17T17:17:54.396 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.396 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.396 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.396 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.387886+0000", 2024-08-17T17:17:54.396 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.783e-06 2024-08-17T17:17:54.396 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.396 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.396 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.396 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481903+0000", 2024-08-17T17:17:54.396 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.094016684000000003 2024-08-17T17:17:54.397 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.397 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.397 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.397 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481926+0000", 2024-08-17T17:17:54.397 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3591000000000002e-05 2024-08-17T17:17:54.397 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.397 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.397 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:17:54.397 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.486048+0000", 2024-08-17T17:17:54.397 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.004122021 2024-08-17T17:17:54.397 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.397 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.397 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.398 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.486080+0000", 2024-08-17T17:17:54.398 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1386000000000001e-05 2024-08-17T17:17:54.398 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.398 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.398 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.398 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.398 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.398 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:29375 6.2 6:40783cdb:::benchmark_data_smithi063_160268_object29374:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:17:54.398 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.388464+0000", 2024-08-17T17:17:54.398 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 256.97163857200002, 2024-08-17T17:17:54.398 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.098667015999999996, 2024-08-17T17:17:54.398 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.398 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:54.399 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:17:54.399 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:17:54.399 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:17:54.399 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 29375 2024-08-17T17:17:54.399 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.399 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.399 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.399 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.399 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388464+0000", 2024-08-17T17:17:54.399 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.399 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.399 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.399 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.399 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388464+0000", 2024-08-17T17:17:54.400 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.400 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.400 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.400 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.400 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388463+0000", 2024-08-17T17:17:54.400 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:17:54.400 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.400 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.400 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.400 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388497+0000", 2024-08-17T17:17:54.400 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4789999999999997e-05 2024-08-17T17:17:54.400 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.400 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.400 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.401 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481860+0000", 2024-08-17T17:17:54.401 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.09336274 2024-08-17T17:17:54.401 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.401 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.401 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.401 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481878+0000", 2024-08-17T17:17:54.401 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7937000000000001e-05 2024-08-17T17:17:54.401 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.401 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.401 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.401 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.482077+0000", 2024-08-17T17:17:54.401 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000198514 2024-08-17T17:17:54.401 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.402 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.402 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.402 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.482222+0000", 2024-08-17T17:17:54.402 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014563999999999999 2024-08-17T17:17:54.402 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.402 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.402 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:17:54.402 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.482364+0000", 2024-08-17T17:17:54.402 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014209100000000001 2024-08-17T17:17:54.402 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.402 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.402 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.402 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.485816+0000", 2024-08-17T17:17:54.403 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003451951 2024-08-17T17:17:54.403 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.403 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.403 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.403 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.485866+0000", 2024-08-17T17:17:54.403 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9781000000000003e-05 2024-08-17T17:17:54.403 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.403 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.403 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:17:54.403 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.487051+0000", 2024-08-17T17:17:54.403 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0011850599999999999 2024-08-17T17:17:54.403 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.403 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.403 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:17:54.404 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.487096+0000", 2024-08-17T17:17:54.404 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5127999999999999e-05 2024-08-17T17:17:54.404 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.404 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.404 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.404 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.487131+0000", 2024-08-17T17:17:54.404 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4514999999999997e-05 2024-08-17T17:17:54.404 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.404 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.404 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.404 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.404 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.405 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop_reply(client.44609.0:29370 6.e e1093/1086 ondisk, result = 0)", 2024-08-17T17:17:54.405 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:37.388487+0000", 2024-08-17T17:17:54.405 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 256.97161575899997, 2024-08-17T17:17:54.405 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.094600769000000001, 2024-08-17T17:17:54.405 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.405 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:17:54.405 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.405 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.405 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.405 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388487+0000", 2024-08-17T17:17:54.405 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.405 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.405 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.405 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.405 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388487+0000", 2024-08-17T17:17:54.406 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.406 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.406 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.406 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.406 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388486+0000", 2024-08-17T17:17:54.406 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:17:54.406 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.406 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.406 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.406 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.388490+0000", 2024-08-17T17:17:54.406 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.155e-06 2024-08-17T17:17:54.406 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.406 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.406 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.407 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481855+0000", 2024-08-17T17:17:54.407 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.093364427999999999 2024-08-17T17:17:54.407 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.407 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.407 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.407 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.481878+0000", 2024-08-17T17:17:54.407 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3110999999999999e-05 2024-08-17T17:17:54.407 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.407 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.407 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.407 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483072+0000", 2024-08-17T17:17:54.407 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001194842 2024-08-17T17:17:54.407 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.407 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.408 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.408 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483079+0000", 2024-08-17T17:17:54.408 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9439999999999999e-06 2024-08-17T17:17:54.408 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.408 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.408 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.408 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:37.483087+0000", 2024-08-17T17:17:54.408 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.875e-06 2024-08-17T17:17:54.408 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.408 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.408 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.408 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.408 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.408 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:32651 6.2 6:49c20ffc:::benchmark_data_smithi063_160268_object32650:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:17:54.409 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:39.132323+0000", 2024-08-17T17:17:54.409 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 255.22777915899999, 2024-08-17T17:17:54.409 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.086169089000000004, 2024-08-17T17:17:54.409 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.409 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:54.409 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:17:54.409 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:17:54.409 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:17:54.409 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 32651 2024-08-17T17:17:54.409 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.409 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.409 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.409 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.410 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132323+0000", 2024-08-17T17:17:54.410 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.410 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.410 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.410 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.410 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132323+0000", 2024-08-17T17:17:54.410 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.410 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.410 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.410 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.410 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132322+0000", 2024-08-17T17:17:54.410 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:17:54.410 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.411 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.411 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.411 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132385+0000", 2024-08-17T17:17:54.411 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2624000000000002e-05 2024-08-17T17:17:54.411 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.411 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.411 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.411 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132395+0000", 2024-08-17T17:17:54.411 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.6830000000000003e-06 2024-08-17T17:17:54.411 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.411 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.411 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.411 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132401+0000", 2024-08-17T17:17:54.411 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5209999999999999e-06 2024-08-17T17:17:54.412 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.412 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.412 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.412 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132474+0000", 2024-08-17T17:17:54.412 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2299999999999996e-05 2024-08-17T17:17:54.412 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.412 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.412 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.412 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132576+0000", 2024-08-17T17:17:54.412 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000102206 2024-08-17T17:17:54.412 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.412 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.412 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:17:54.412 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.132686+0000", 2024-08-17T17:17:54.413 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000110332 2024-08-17T17:17:54.413 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.413 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.413 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:17:54.413 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.134043+0000", 2024-08-17T17:17:54.413 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013573909999999999 2024-08-17T17:17:54.413 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.413 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.413 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.413 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.136328+0000", 2024-08-17T17:17:54.413 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0022848600000000001 2024-08-17T17:17:54.413 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.413 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.414 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.414 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.218411+0000", 2024-08-17T17:17:54.414 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.082082811000000006 2024-08-17T17:17:54.414 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.414 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.414 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:17:54.414 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.218465+0000", 2024-08-17T17:17:54.414 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3538999999999997e-05 2024-08-17T17:17:54.414 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.414 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.414 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.414 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.218492+0000", 2024-08-17T17:17:54.414 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.7515999999999999e-05 2024-08-17T17:17:54.415 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.415 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.415 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.415 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.415 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.415 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:33172 6.2 6:40fd6398:::benchmark_data_smithi063_160268_object33171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:17:54.415 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:39.451372+0000", 2024-08-17T17:17:54.415 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 254.90873063000001, 2024-08-17T17:17:54.415 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.089318386999999999, 2024-08-17T17:17:54.415 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.415 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:54.415 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:17:54.415 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:17:54.415 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:17:54.416 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 33172 2024-08-17T17:17:54.416 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.416 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.416 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.416 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.416 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451372+0000", 2024-08-17T17:17:54.416 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.416 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.416 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.416 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.416 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451372+0000", 2024-08-17T17:17:54.416 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.416 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.417 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.417 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.417 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451370+0000", 2024-08-17T17:17:54.417 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:17:54.417 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.417 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.417 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.417 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451403+0000", 2024-08-17T17:17:54.417 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2957000000000003e-05 2024-08-17T17:17:54.417 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.417 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.417 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.417 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451416+0000", 2024-08-17T17:17:54.417 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2887e-05 2024-08-17T17:17:54.418 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.418 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.418 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.418 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451423+0000", 2024-08-17T17:17:54.418 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8020000000000003e-06 2024-08-17T17:17:54.418 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.418 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.418 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.418 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451492+0000", 2024-08-17T17:17:54.418 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9031999999999999e-05 2024-08-17T17:17:54.418 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.418 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.419 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.419 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451622+0000", 2024-08-17T17:17:54.419 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012985699999999999 2024-08-17T17:17:54.419 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.419 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.419 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:17:54.419 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.451757+0000", 2024-08-17T17:17:54.419 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013562200000000001 2024-08-17T17:17:54.419 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.419 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.419 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:17:54.419 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453044+0000", 2024-08-17T17:17:54.419 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012871099999999999 2024-08-17T17:17:54.419 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.419 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.420 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.420 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.455396+0000", 2024-08-17T17:17:54.420 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023519919999999998 2024-08-17T17:17:54.420 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.420 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.420 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.420 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.540621+0000", 2024-08-17T17:17:54.420 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.085224464999999999 2024-08-17T17:17:54.420 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.420 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.420 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:17:54.420 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.540660+0000", 2024-08-17T17:17:54.420 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9440000000000002e-05 2024-08-17T17:17:54.421 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.421 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.421 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.421 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.540690+0000", 2024-08-17T17:17:54.421 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9708000000000001e-05 2024-08-17T17:17:54.421 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.421 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.421 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.421 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.421 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.421 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:33179 6.2 6:4d001853:::benchmark_data_smithi063_160268_object33178:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:17:54.421 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:39.453823+0000", 2024-08-17T17:17:54.422 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 254.906278786, 2024-08-17T17:17:54.422 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.089360489000000001, 2024-08-17T17:17:54.422 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.422 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:54.422 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:17:54.422 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:17:54.422 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:17:54.422 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 33179 2024-08-17T17:17:54.422 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.422 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.422 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.422 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.423 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453823+0000", 2024-08-17T17:17:54.423 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.423 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.423 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.423 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.423 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453823+0000", 2024-08-17T17:17:54.423 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.423 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.423 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.423 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.423 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453822+0000", 2024-08-17T17:17:54.423 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:17:54.423 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.424 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.424 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.424 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453916+0000", 2024-08-17T17:17:54.424 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4184000000000005e-05 2024-08-17T17:17:54.424 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.424 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.424 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.424 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453931+0000", 2024-08-17T17:17:54.424 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4997999999999999e-05 2024-08-17T17:17:54.424 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.424 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.424 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.425 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453939+0000", 2024-08-17T17:17:54.425 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4250000000000001e-06 2024-08-17T17:17:54.425 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.425 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.425 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.425 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.453983+0000", 2024-08-17T17:17:54.425 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3804000000000003e-05 2024-08-17T17:17:54.425 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.425 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.425 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.425 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.454105+0000", 2024-08-17T17:17:54.425 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012191799999999999 2024-08-17T17:17:54.425 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.426 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.426 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:17:54.426 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.454241+0000", 2024-08-17T17:17:54.426 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000136396 2024-08-17T17:17:54.426 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.426 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.426 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.426 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.455812+0000", 2024-08-17T17:17:54.426 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001571277 2024-08-17T17:17:54.426 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.426 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.426 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:17:54.427 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.456213+0000", 2024-08-17T17:17:54.427 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00040079599999999998 2024-08-17T17:17:54.427 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.427 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.427 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.427 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.543091+0000", 2024-08-17T17:17:54.427 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.086878450999999995 2024-08-17T17:17:54.427 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.427 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.427 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:17:54.427 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.543147+0000", 2024-08-17T17:17:54.427 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5695000000000001e-05 2024-08-17T17:17:54.427 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.427 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.427 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.428 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:39.543184+0000", 2024-08-17T17:17:54.428 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6835000000000002e-05 2024-08-17T17:17:54.428 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.428 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.428 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.428 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.428 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.428 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:34214 6.2 6:4d7d9756:::benchmark_data_smithi063_160268_object34213:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:17:54.428 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.043114+0000", 2024-08-17T17:17:54.428 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 254.31698818800001, 2024-08-17T17:17:54.428 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.090588551000000003, 2024-08-17T17:17:54.428 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.428 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:54.429 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:17:54.429 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:17:54.429 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:17:54.429 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 34214 2024-08-17T17:17:54.429 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.429 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.429 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.429 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.429 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043114+0000", 2024-08-17T17:17:54.429 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.429 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.429 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.429 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.430 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043114+0000", 2024-08-17T17:17:54.430 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.430 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.430 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.430 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.430 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043112+0000", 2024-08-17T17:17:54.430 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:17:54.430 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.430 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.430 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.430 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043158+0000", 2024-08-17T17:17:54.430 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5021000000000001e-05 2024-08-17T17:17:54.430 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.431 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.431 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.431 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043177+0000", 2024-08-17T17:17:54.431 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9921000000000001e-05 2024-08-17T17:17:54.431 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.431 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.431 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.431 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043186+0000", 2024-08-17T17:17:54.431 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7110000000000001e-06 2024-08-17T17:17:54.431 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.431 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.431 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.432 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043238+0000", 2024-08-17T17:17:54.432 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2034999999999999e-05 2024-08-17T17:17:54.432 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.432 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.432 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.432 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043357+0000", 2024-08-17T17:17:54.432 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000119085 2024-08-17T17:17:54.432 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.432 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.432 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:17:54.432 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.043534+0000", 2024-08-17T17:17:54.432 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017626400000000001 2024-08-17T17:17:54.432 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.433 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.433 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:17:54.433 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045175+0000", 2024-08-17T17:17:54.433 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016417070000000001 2024-08-17T17:17:54.433 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.433 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.433 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.433 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045757+0000", 2024-08-17T17:17:54.433 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00058215199999999995 2024-08-17T17:17:54.433 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.433 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.433 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.433 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.133592+0000", 2024-08-17T17:17:54.434 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.087834236999999996 2024-08-17T17:17:54.434 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.434 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.434 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:17:54.434 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.133660+0000", 2024-08-17T17:17:54.434 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8575e-05 2024-08-17T17:17:54.434 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.434 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.434 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.434 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.133703+0000", 2024-08-17T17:17:54.434 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2447999999999997e-05 2024-08-17T17:17:54.434 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.434 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.435 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.435 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.435 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.435 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:34221 6.2 6:4bfc0617:::benchmark_data_smithi063_160268_object34220:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:17:54.435 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.045750+0000", 2024-08-17T17:17:54.435 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 254.31435216099999, 2024-08-17T17:17:54.435 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.088048524000000003, 2024-08-17T17:17:54.435 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.435 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:54.435 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:17:54.435 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:17:54.435 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:17:54.435 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 34221 2024-08-17T17:17:54.435 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.436 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.436 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.436 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.436 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045750+0000", 2024-08-17T17:17:54.436 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.436 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.436 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.436 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.436 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:54.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:54.437 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045750+0000", 2024-08-17T17:17:54.438 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.438 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.438 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.438 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.438 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045748+0000", 2024-08-17T17:17:54.438 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:17:54.438 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.438 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.438 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.438 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045817+0000", 2024-08-17T17:17:54.438 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8992999999999995e-05 2024-08-17T17:17:54.438 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.439 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.439 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.439 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045831+0000", 2024-08-17T17:17:54.439 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3056e-05 2024-08-17T17:17:54.439 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.439 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.439 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.439 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045836+0000", 2024-08-17T17:17:54.439 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.699e-06 2024-08-17T17:17:54.439 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.439 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.439 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.439 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.045888+0000", 2024-08-17T17:17:54.440 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1365e-05 2024-08-17T17:17:54.440 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.440 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.440 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.440 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.046026+0000", 2024-08-17T17:17:54.440 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013860000000000001 2024-08-17T17:17:54.440 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.440 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.440 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:17:54.440 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.046139+0000", 2024-08-17T17:17:54.440 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000113035 2024-08-17T17:17:54.440 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.440 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.440 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:17:54.441 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.047132+0000", 2024-08-17T17:17:54.441 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00099325299999999993 2024-08-17T17:17:54.441 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.441 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.441 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.441 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.047732+0000", 2024-08-17T17:17:54.441 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00059973199999999996 2024-08-17T17:17:54.441 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.441 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.441 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.441 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.133743+0000", 2024-08-17T17:17:54.441 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.086011140999999999 2024-08-17T17:17:54.441 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.442 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.442 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:17:54.442 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.133774+0000", 2024-08-17T17:17:54.442 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.0753000000000002e-05 2024-08-17T17:17:54.442 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.442 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.442 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.442 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.133799+0000", 2024-08-17T17:17:54.442 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4552000000000001e-05 2024-08-17T17:17:54.442 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.442 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.442 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.443 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.443 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.443 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:35807 6.2 6:425615ad:::benchmark_data_smithi063_160268_object35806:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:17:54.443 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.876489+0000", 2024-08-17T17:17:54.443 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 253.483613351, 2024-08-17T17:17:54.443 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14906064199999999, 2024-08-17T17:17:54.443 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.443 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:54.443 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:17:54.443 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:17:54.443 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:17:54.443 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 35807 2024-08-17T17:17:54.443 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.444 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.444 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.444 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.444 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876489+0000", 2024-08-17T17:17:54.444 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.444 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.444 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.444 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.444 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876489+0000", 2024-08-17T17:17:54.444 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.444 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.444 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.444 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.445 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876488+0000", 2024-08-17T17:17:54.445 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:17:54.445 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.445 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.445 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.445 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876522+0000", 2024-08-17T17:17:54.445 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4264999999999998e-05 2024-08-17T17:17:54.445 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.445 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.445 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.446 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876533+0000", 2024-08-17T17:17:54.446 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1549000000000001e-05 2024-08-17T17:17:54.446 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.446 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.446 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.446 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876540+0000", 2024-08-17T17:17:54.446 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6810000000000001e-06 2024-08-17T17:17:54.446 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.446 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.446 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.446 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876589+0000", 2024-08-17T17:17:54.446 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9308999999999998e-05 2024-08-17T17:17:54.446 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.447 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.447 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.447 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876713+0000", 2024-08-17T17:17:54.447 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012383300000000001 2024-08-17T17:17:54.447 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.447 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.447 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:17:54.447 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876896+0000", 2024-08-17T17:17:54.447 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000182466 2024-08-17T17:17:54.447 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.447 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.447 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:17:54.448 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.878495+0000", 2024-08-17T17:17:54.448 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00159953 2024-08-17T17:17:54.448 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.448 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.448 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.448 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.881641+0000", 2024-08-17T17:17:54.448 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0031456629999999999 2024-08-17T17:17:54.448 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.448 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.448 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.448 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025446+0000", 2024-08-17T17:17:54.448 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14380467199999999 2024-08-17T17:17:54.448 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.448 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.449 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:17:54.449 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025503+0000", 2024-08-17T17:17:54.449 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7846000000000001e-05 2024-08-17T17:17:54.449 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.449 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.449 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.449 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025550+0000", 2024-08-17T17:17:54.449 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6156999999999999e-05 2024-08-17T17:17:54.449 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.449 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.449 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.449 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.449 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.450 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:35813 6.2 6:4d5ac260:::benchmark_data_smithi063_160268_object35812:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:17:54.450 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.880079+0000", 2024-08-17T17:17:54.450 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 253.480023346, 2024-08-17T17:17:54.450 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.145731308, 2024-08-17T17:17:54.450 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.450 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:54.450 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:17:54.450 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:17:54.450 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:17:54.450 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 35813 2024-08-17T17:17:54.450 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.450 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.450 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.451 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.451 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880079+0000", 2024-08-17T17:17:54.451 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.451 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.451 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.451 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.451 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880079+0000", 2024-08-17T17:17:54.451 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.451 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.451 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.451 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.451 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880078+0000", 2024-08-17T17:17:54.451 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:17:54.452 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.452 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.452 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.452 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880133+0000", 2024-08-17T17:17:54.452 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.469e-05 2024-08-17T17:17:54.452 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.452 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.452 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.452 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880146+0000", 2024-08-17T17:17:54.452 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.314e-05 2024-08-17T17:17:54.452 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.452 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.453 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.453 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880151+0000", 2024-08-17T17:17:54.453 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2120000000000002e-06 2024-08-17T17:17:54.453 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.453 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.453 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.453 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880180+0000", 2024-08-17T17:17:54.453 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9244999999999999e-05 2024-08-17T17:17:54.453 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.453 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.453 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.453 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880304+0000", 2024-08-17T17:17:54.454 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012414699999999999 2024-08-17T17:17:54.454 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.454 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.454 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:17:54.454 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880443+0000", 2024-08-17T17:17:54.454 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013859699999999999 2024-08-17T17:17:54.454 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.454 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.454 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:17:54.454 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.881778+0000", 2024-08-17T17:17:54.454 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013348290000000001 2024-08-17T17:17:54.454 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.454 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.455 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.455 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.882544+0000", 2024-08-17T17:17:54.455 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00076663499999999999 2024-08-17T17:17:54.455 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.455 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.455 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.455 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025711+0000", 2024-08-17T17:17:54.455 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14316620999999999 2024-08-17T17:17:54.455 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.455 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.455 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:17:54.455 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025763+0000", 2024-08-17T17:17:54.455 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2191999999999998e-05 2024-08-17T17:17:54.456 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.456 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.456 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.456 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025810+0000", 2024-08-17T17:17:54.456 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7521000000000001e-05 2024-08-17T17:17:54.456 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.456 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.456 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.456 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.456 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.456 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:233113 6.2 6:44a3a098:::benchmark_data_smithi063_160268_object107006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-08-17T17:17:54.456 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:17:04.728566+0000", 2024-08-17T17:17:54.457 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 49.631536678000003, 2024-08-17T17:17:54.457 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.088816580000000006, 2024-08-17T17:17:54.457 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.457 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:54.457 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:17:54.457 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:17:54.457 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:17:54.457 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 233113 2024-08-17T17:17:54.457 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.457 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.457 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.457 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.457 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.728566+0000", 2024-08-17T17:17:54.457 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.458 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.458 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.458 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.458 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.728566+0000", 2024-08-17T17:17:54.458 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.458 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.458 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.458 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.458 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.728564+0000", 2024-08-17T17:17:54.458 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:17:54.458 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.458 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.459 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.459 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.728571+0000", 2024-08-17T17:17:54.459 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.131e-06 2024-08-17T17:17:54.459 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.459 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.459 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.459 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.728586+0000", 2024-08-17T17:17:54.459 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4353e-05 2024-08-17T17:17:54.459 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.459 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.459 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.459 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.728594+0000", 2024-08-17T17:17:54.459 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3210000000000005e-06 2024-08-17T17:17:54.460 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.460 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.460 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.460 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.728641+0000", 2024-08-17T17:17:54.460 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7234000000000002e-05 2024-08-17T17:17:54.460 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.460 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.460 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.460 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.728760+0000", 2024-08-17T17:17:54.460 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011900699999999999 2024-08-17T17:17:54.460 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.460 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.460 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:17:54.461 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.728916+0000", 2024-08-17T17:17:54.461 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015592399999999999 2024-08-17T17:17:54.461 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.461 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.461 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:17:54.461 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.735843+0000", 2024-08-17T17:17:54.461 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0069273759999999998 2024-08-17T17:17:54.461 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.461 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.461 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.461 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.738017+0000", 2024-08-17T17:17:54.461 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021732790000000002 2024-08-17T17:17:54.461 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.461 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.462 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.462 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.817294+0000", 2024-08-17T17:17:54.462 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.079277480999999997 2024-08-17T17:17:54.462 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.462 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.462 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:17:54.462 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.817340+0000", 2024-08-17T17:17:54.462 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6218999999999999e-05 2024-08-17T17:17:54.462 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.462 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.462 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.462 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.817382+0000", 2024-08-17T17:17:54.463 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1739999999999997e-05 2024-08-17T17:17:54.463 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.463 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.463 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.463 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.463 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.463 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:233114 6.2 6:4ac63635:::benchmark_data_smithi063_160268_object107007:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-08-17T17:17:54.463 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:17:04.729305+0000", 2024-08-17T17:17:54.463 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 49.630797313000002, 2024-08-17T17:17:54.463 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.088326397000000001, 2024-08-17T17:17:54.463 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:17:54.463 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:17:54.463 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:17:54.464 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:17:54.464 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:17:54.464 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 233114 2024-08-17T17:17:54.464 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.464 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:17:54.464 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.464 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:17:54.464 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.729305+0000", 2024-08-17T17:17:54.464 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.464 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.464 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.464 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:17:54.465 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.729305+0000", 2024-08-17T17:17:54.465 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:17:54.465 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.465 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.465 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:17:54.465 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.729304+0000", 2024-08-17T17:17:54.465 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:17:54.465 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.465 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.465 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:17:54.465 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.729310+0000", 2024-08-17T17:17:54.465 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8420000000000001e-06 2024-08-17T17:17:54.465 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.466 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.466 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:17:54.466 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.729330+0000", 2024-08-17T17:17:54.466 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0516000000000001e-05 2024-08-17T17:17:54.466 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.466 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.466 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:17:54.466 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.729338+0000", 2024-08-17T17:17:54.466 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.8180000000000004e-06 2024-08-17T17:17:54.466 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.466 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.466 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:17:54.467 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.732229+0000", 2024-08-17T17:17:54.467 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0028907350000000002 2024-08-17T17:17:54.467 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.467 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.467 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:17:54.467 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.732340+0000", 2024-08-17T17:17:54.467 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000110992 2024-08-17T17:17:54.467 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.467 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.467 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:17:54.467 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.732452+0000", 2024-08-17T17:17:54.467 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000112297 2024-08-17T17:17:54.468 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.468 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.468 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:17:54.468 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.736014+0000", 2024-08-17T17:17:54.468 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003562005 2024-08-17T17:17:54.468 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.468 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.468 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.468 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.738188+0000", 2024-08-17T17:17:54.468 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021738880000000001 2024-08-17T17:17:54.468 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.468 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.468 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:17:54.469 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.817514+0000", 2024-08-17T17:17:54.469 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.079325941999999997 2024-08-17T17:17:54.469 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.469 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.469 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:17:54.469 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.817563+0000", 2024-08-17T17:17:54.469 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9061999999999999e-05 2024-08-17T17:17:54.469 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:17:54.469 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:17:54.469 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:17:54.469 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.817631+0000", 2024-08-17T17:17:54.469 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8319999999999996e-05 2024-08-17T17:17:54.469 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.470 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.470 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.470 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:17:54.470 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:17:54.470 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:17:54.537 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:54.559 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:54.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:54.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:54.628 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:54 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:17:54.628 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:54 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4227408547' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:17:54.628 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:54 smithi063 ceph-mon[114756]: osdmap e1131: 8 total, 8 up, 7 in 2024-08-17T17:17:54.628 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:54 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:17:54.628 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:54 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4227408547' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:17:54.628 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:54 smithi063 ceph-mon[109843]: osdmap e1131: 8 total, 8 up, 7 in 2024-08-17T17:17:54.661 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:54.682 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:54.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:54.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:54.783 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:54.806 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:54.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:54.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:54.908 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:54.929 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:54.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:54.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:55.031 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:55.052 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:55.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:55.154 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:55.175 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:55.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:55.223 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:17:55.277 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:55.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:55.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:55.400 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:55.423 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:55.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:55.428 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:17:55.525 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:55.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.547 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:55.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:55.550 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:55 smithi162 ceph-mon[104668]: pgmap v1311: 89 pgs: 10 creating+peering, 6 unknown, 73 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:55.550 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:55 smithi162 ceph-mon[104668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:17:55.588 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:55 smithi063 ceph-mon[114756]: pgmap v1311: 89 pgs: 10 creating+peering, 6 unknown, 73 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:55.588 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:55 smithi063 ceph-mon[114756]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:17:55.589 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:55 smithi063 ceph-mon[109843]: pgmap v1311: 89 pgs: 10 creating+peering, 6 unknown, 73 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:55.589 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:55 smithi063 ceph-mon[109843]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:17:55.599 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:17:55.649 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:55.672 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:55.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:55.773 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:55.795 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:55.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:55.895 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:55.897 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:55.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:55.922 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:55.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:56.022 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:56.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:56.046 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:56.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:56.071 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:17:56.147 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:56.170 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:56.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:56.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:56.272 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:56.294 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:56.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:56.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:56.311 INFO:teuthology.orchestra.run.smithi063.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-08-17T17:17:56.396 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:56.419 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:56.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:56.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:56.521 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:56.545 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:56.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:56.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:56.583 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:56 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3150930061' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:17:56.584 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:56 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:17:56.632 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:56 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3150930061' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:17:56.633 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:56 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:17:56.633 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:56 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3150930061' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:17:56.633 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:56 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:17:56.646 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:56.668 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:56.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:56.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:56.770 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:56.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:56.792 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:56.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:56.893 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:56.915 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:56.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:56.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:57.017 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:57.040 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:57.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:57.121 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:17:57.121 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:17:57.121 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 0 2024-08-17T17:17:57.121 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:17:57.142 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:57.164 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:57.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:57.266 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:57.289 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:57.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:57.391 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:57.412 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:57.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:57.514 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:57.536 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:57.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:57.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:57 smithi162 ceph-mon[104668]: pgmap v1312: 89 pgs: 10 creating+peering, 2 unknown, 77 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:57.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:57 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:17:57.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:57 smithi162 ceph-mon[104668]: osdmap e1132: 8 total, 8 up, 7 in 2024-08-17T17:17:57.638 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:57.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.662 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:57.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:57.763 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:57.773 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:57 smithi063 ceph-mon[114756]: pgmap v1312: 89 pgs: 10 creating+peering, 2 unknown, 77 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:57.773 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:57 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:17:57.774 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:57 smithi063 ceph-mon[114756]: osdmap e1132: 8 total, 8 up, 7 in 2024-08-17T17:17:57.774 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:57 smithi063 ceph-mon[109843]: pgmap v1312: 89 pgs: 10 creating+peering, 2 unknown, 77 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:57.774 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:57 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:17:57.774 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:57 smithi063 ceph-mon[109843]: osdmap e1132: 8 total, 8 up, 7 in 2024-08-17T17:17:57.786 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:57.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:57.888 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:17:57.912 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:57.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:57.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:17:57.931 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:17:58.014 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:58.036 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:58.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:58.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:58.138 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:58.161 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:58.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:58.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:58.174 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:17:58.238 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:17:58.239 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:17:58.242 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:17:58.257 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:17:58.258 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:17:58.263 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:58.281 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:17:58.281 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:17:58.285 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:58.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:58.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:58.297 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:17:58.298 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:17:58.314 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:58 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2491272386' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:58.314 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:58 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2491272386' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:58.316 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:17:58.317 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:17:58.333 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:17:58.333 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:17:58.350 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:17:58.350 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:17:58.367 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:17:58.367 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:17:58.376 DEBUG:teuthology.orchestra.run.smithi063:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-08-17T17:17:58.377 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-08-17T17:17:58.377 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:17:58.379 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:17:58.379 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:17:58.386 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:58.433 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:hints = 1 2024-08-17T17:17:58.433 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:17:58.433 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Object prefix: benchmark_data_smithi063_196002 2024-08-17T17:17:58.434 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:17:58.434 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:17:58.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:58.454 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:58.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:58.556 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:17:58.588 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:58.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:58.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:17:58.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:58 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2491272386' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:17:58.690 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:17:58.726 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:58.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:58.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:17:58.827 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:58.873 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:58.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:58.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:58.974 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:59.009 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:59.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:59.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:17:59.111 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:17:59.147 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:59.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:59.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:17:59.249 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:59.287 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:59.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:59.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:59.389 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:59.423 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:59.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:59.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:59.434 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 1 16 4000 3984 248.986 249 0.00438895 0.0040036 2024-08-17T17:17:59.524 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:17:59.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:59 smithi063 ceph-mon[114756]: pgmap v1314: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:59.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:59 smithi063 ceph-mon[114756]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:17:59.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:17:59 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:17:59.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:59 smithi063 ceph-mon[109843]: pgmap v1314: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:59.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:59 smithi063 ceph-mon[109843]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:17:59.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:17:59 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:17:59.563 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:59.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:59.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:17:59.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:59 smithi162 ceph-mon[104668]: pgmap v1314: 89 pgs: 89 active+clean; 582 KiB data, 1.2 GiB used, 625 GiB / 626 GiB avail 2024-08-17T17:17:59.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:59 smithi162 ceph-mon[104668]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:17:59.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:17:59 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:17:59.665 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:17:59.705 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:59.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:59.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:17:59.807 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:17:59.845 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:17:59.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:59.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:17:59.947 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:17:59.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:17:59.995 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:17:59.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:00.096 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:00.133 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:00.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:00.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:00.235 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:00.273 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:00.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:00.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:00.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:00.416 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:00.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:00.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:00.434 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:18:00.434 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:18:00.434 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 2 16 7670 7654 239.17 229.375 0.00394401 0.00417391 2024-08-17T17:18:00.519 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:00.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:00.560 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:00.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:00.661 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:00.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:00.699 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:00.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:00.800 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:00.835 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:00.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:00.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:00.937 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:00.976 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:00.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:00.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:01.078 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:01.118 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:01.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:01.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:01.135 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:18:01.198 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:01.220 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:01.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:01.255 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:01.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:01.356 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:01.375 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:01 smithi063 ceph-mon[114756]: pgmap v1315: 89 pgs: 89 active+clean; 72 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 9.1 MiB/s wr, 146 op/s 2024-08-17T17:18:01.376 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:01 smithi063 ceph-mon[109843]: pgmap v1315: 89 pgs: 89 active+clean; 72 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 9.1 MiB/s wr, 146 op/s 2024-08-17T17:18:01.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:01.399 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:01.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:01.434 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 3 16 10909 10893 226.92 202.438 0.00289279 0.00439883 2024-08-17T17:18:01.481 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:18:01.500 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:01.539 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:01.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:01.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:01 smithi162 ceph-mon[104668]: pgmap v1315: 89 pgs: 89 active+clean; 72 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 9.1 MiB/s wr, 146 op/s 2024-08-17T17:18:01.641 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:01.677 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:01.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:01.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:01.779 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:01.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:01.817 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:01.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:01.918 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:01.951 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:01.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:01.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:02.053 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:02.087 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:02.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:02.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:02.189 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:02.224 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:02.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:02.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:02.325 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:02.348 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:02.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:02.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:02.434 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 4 16 13721 13705 214.12 175.75 0.00190261 0.00452268 2024-08-17T17:18:02.450 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:02.450 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:18:02.482 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:02.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:02.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:02.573 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:18:02.584 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:02.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:02 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:18:02.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:02 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:02.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:02 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/449394133' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:18:02.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:02 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:18:02.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:02 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:18:02.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:02 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:02.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:02 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/449394133' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:18:02.614 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:02 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:18:02.614 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:02 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:18:02.614 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:02 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:02.614 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:02 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/449394133' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:18:02.614 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:02 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:18:02.617 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:02.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:02.618 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:02.718 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:02.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:02.760 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:02.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:02.861 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:02.895 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:02.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:02.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:02.996 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:03.029 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:03.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:03.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:03.131 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:03.166 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:03.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:03.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:03.268 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:03.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:03.311 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:03.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:03.412 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:03.434 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 5 16 16734 16718 208.948 188.312 0.00340809 0.00477799 2024-08-17T17:18:03.455 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:03.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:03.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:03.510 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:18:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:18:03] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.51.0" 2024-08-17T17:18:03.510 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:03 smithi063 ceph-mon[114756]: pgmap v1316: 89 pgs: 89 active+clean; 456 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s 2024-08-17T17:18:03.511 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:03 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:03.511 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:03 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:18:03.511 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:03 smithi063 ceph-mon[114756]: osdmap e1133: 8 total, 8 up, 7 in 2024-08-17T17:18:03.511 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:03 smithi063 ceph-mon[109843]: pgmap v1316: 89 pgs: 89 active+clean; 456 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s 2024-08-17T17:18:03.511 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:03 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:03.511 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:03 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:18:03.511 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:03 smithi063 ceph-mon[109843]: osdmap e1133: 8 total, 8 up, 7 in 2024-08-17T17:18:03.558 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:03.595 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:03.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:03.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:03.620 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:03 smithi162 ceph-mon[104668]: pgmap v1316: 89 pgs: 89 active+clean; 456 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 51 MiB/s wr, 823 op/s 2024-08-17T17:18:03.620 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:03 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:03.620 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:03 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:18:03.620 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:03 smithi162 ceph-mon[104668]: osdmap e1133: 8 total, 8 up, 7 in 2024-08-17T17:18:03.696 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:03.731 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:03.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:03.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:03.833 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:03.870 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:03.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:03.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:03.958 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:18:03.958 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:18:03.972 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:04.015 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:04.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:04.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:04.117 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:04.153 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:04.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:04.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:04.255 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:04.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:04.296 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:04.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:04.397 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:04.435 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 6 16 19579 19563 203.756 177.812 0.00498581 0.00490116 2024-08-17T17:18:04.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:04.436 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:04.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:04.537 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:04.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:04.572 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:04.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:04.674 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:04.715 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:04.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:04.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:04.801 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:18:04.818 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:04.819 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:04.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:04.887 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:04.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:04.988 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:05.026 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:05.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:05.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:05.128 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:05.143 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:18:05.144 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:18:05.169 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:05.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:05.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:05.184 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:18:05.271 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:05.307 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:05.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:05.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:05.409 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:05.435 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 7 16 22035 22019 196.575 153.5 0.00722301 0.00508114 2024-08-17T17:18:05.445 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:05.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:05.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:05.547 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:05.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:05.585 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:05.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:05.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:05 smithi162 ceph-mon[104668]: pgmap v1318: 89 pgs: 89 active+clean; 661 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-08-17T17:18:05.650 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:05 smithi063 ceph-mon[114756]: pgmap v1318: 89 pgs: 89 active+clean; 661 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-08-17T17:18:05.650 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:05 smithi063 ceph-mon[109843]: pgmap v1318: 89 pgs: 89 active+clean; 661 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-08-17T17:18:05.686 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:05.730 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:05.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:05.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:05.832 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:05.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:05.867 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:05.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:05.968 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:06.004 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:06.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:06.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:06.106 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:06.143 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:06.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:06.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:06.245 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:06.280 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:06.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:06.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:06.354 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1318,"stamp":"2024-08-17T17:18:04.148151+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":693245925,"num_objects":10595,"num_object_clones":0,"num_object_copies":31785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10595,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":10848,"num_write_kb":678865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":2362076,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":10777,"ondisk_log_size":10777,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3264460,"kb_used_data":2045584,"kb_used_omap":133,"kb_used_meta":1218682,"kb_avail":652951604,"statfs":{"total":671965249536,"available":668622442496,"internally_reserved":0,"allocated":2094678016,"data_stored":2080526048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":136639,"internal_metadata":1247930945},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":692649984,"num_objects":10569,"num_object_clones":0,"num_object_copies":31707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10569,"num_write_kb":676416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002684"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1343,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.948429+0000","last_change":"2024-08-17T17:07:45.684223+0000","last_active":"2024-08-17T17:17:56.948429+0000","last_peered":"2024-08-17T17:17:56.948429+0000","last_clean":"2024-08-17T17:17:56.948429+0000","last_became_active":"2024-08-17T17:07:45.683948+0000","last_became_peered":"2024-08-17T17:07:45.683948+0000","last_unstale":"2024-08-17T17:17:56.948429+0000","last_undegraded":"2024-08-17T17:17:56.948429+0000","last_fullsized":"2024-08-17T17:17:56.948429+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:07.677423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1058,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:54.071903+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:17:54.071903+0000","last_peered":"2024-08-17T17:17:54.071903+0000","last_clean":"2024-08-17T17:17:54.071903+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:17:54.071903+0000","last_undegraded":"2024-08-17T17:17:54.071903+0000","last_fullsized":"2024-08-17T17:17:54.071903+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1070,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758600+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:17:53.758600+0000","last_peered":"2024-08-17T17:17:53.758600+0000","last_clean":"2024-08-17T17:17:53.758600+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:17:53.758600+0000","last_undegraded":"2024-08-17T17:17:53.758600+0000","last_fullsized":"2024-08-17T17:17:53.758600+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":916,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:54.071761+0000","last_change":"2024-08-17T17:17:32.851216+0000","last_active":"2024-08-17T17:17:54.071761+0000","last_peered":"2024-08-17T17:17:54.071761+0000","last_clean":"2024-08-17T17:17:54.071761+0000","last_became_active":"2024-08-17T17:17:32.850887+0000","last_became_peered":"2024-08-17T17:17:32.850887+0000","last_unstale":"2024-08-17T17:17:54.071761+0000","last_undegraded":"2024-08-17T17:17:54.071761+0000","last_fullsized":"2024-08-17T17:17:54.071761+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:22.286376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":902,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.948344+0000","last_change":"2024-08-17T17:17:32.796446+0000","last_active":"2024-08-17T17:17:56.948344+0000","last_peered":"2024-08-17T17:17:56.948344+0000","last_clean":"2024-08-17T17:17:56.948344+0000","last_became_active":"2024-08-17T17:17:32.796118+0000","last_became_peered":"2024-08-17T17:17:32.796118+0000","last_unstale":"2024-08-17T17:17:56.948344+0000","last_undegraded":"2024-08-17T17:17:56.948344+0000","last_fullsized":"2024-08-17T17:17:56.948344+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:06.422265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1041,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:54.071827+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:17:54.071827+0000","last_peered":"2024-08-17T17:17:54.071827+0000","last_clean":"2024-08-17T17:17:54.071827+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:17:54.071827+0000","last_undegraded":"2024-08-17T17:17:54.071827+0000","last_fullsized":"2024-08-17T17:17:54.071827+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1343,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.948860+0000","last_change":"2024-08-17T17:07:45.687030+0000","last_active":"2024-08-17T17:17:56.948860+0000","last_peered":"2024-08-17T17:17:56.948860+0000","last_clean":"2024-08-17T17:17:56.948860+0000","last_became_active":"2024-08-17T17:07:45.686610+0000","last_became_peered":"2024-08-17T17:07:45.686610+0000","last_unstale":"2024-08-17T17:17:56.948860+0000","last_undegraded":"2024-08-17T17:17:56.948860+0000","last_fullsized":"2024-08-17T17:17:56.948860+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:02.425845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1099,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.307385+0000","last_change":"2024-08-17T17:17:32.803448+0000","last_active":"2024-08-17T17:17:53.307385+0000","last_peered":"2024-08-17T17:17:53.307385+0000","last_clean":"2024-08-17T17:17:53.307385+0000","last_became_active":"2024-08-17T17:17:32.796752+0000","last_became_peered":"2024-08-17T17:17:32.796752+0000","last_unstale":"2024-08-17T17:17:53.307385+0000","last_undegraded":"2024-08-17T17:17:53.307385+0000","last_fullsized":"2024-08-17T17:17:53.307385+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:34.406763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1132'781","reported_seq":791,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:18:02.157964+0000","last_change":"2024-08-17T17:17:52.927899+0000","last_active":"2024-08-17T17:18:02.157964+0000","last_peered":"2024-08-17T17:18:02.157964+0000","last_clean":"2024-08-17T17:18:02.157964+0000","last_became_active":"2024-08-17T17:17:52.927591+0000","last_became_peered":"2024-08-17T17:17:52.927591+0000","last_unstale":"2024-08-17T17:18:02.157964+0000","last_undegraded":"2024-08-17T17:18:02.157964+0000","last_fullsized":"2024-08-17T17:18:02.157964+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":781,"log_dups_size":0,"ondisk_log_size":781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:31:21.681712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51183616,"num_objects":781,"num_object_clones":0,"num_object_copies":2343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":781,"num_write_kb":49984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1070,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:54.072356+0000","last_change":"2024-08-17T17:17:32.857514+0000","last_active":"2024-08-17T17:17:54.072356+0000","last_peered":"2024-08-17T17:17:54.072356+0000","last_clean":"2024-08-17T17:17:54.072356+0000","last_became_active":"2024-08-17T17:17:32.851253+0000","last_became_peered":"2024-08-17T17:17:32.851253+0000","last_unstale":"2024-08-17T17:17:54.072356+0000","last_undegraded":"2024-08-17T17:17:54.072356+0000","last_fullsized":"2024-08-17T17:17:54.072356+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:55.376227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1056,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758399+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:17:53.758399+0000","last_peered":"2024-08-17T17:17:53.758399+0000","last_clean":"2024-08-17T17:17:53.758399+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:17:53.758399+0000","last_undegraded":"2024-08-17T17:17:53.758399+0000","last_fullsized":"2024-08-17T17:17:53.758399+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1132'288","reported_seq":304,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:59.562147+0000","last_change":"2024-08-17T17:17:52.311116+0000","last_active":"2024-08-17T17:17:59.562147+0000","last_peered":"2024-08-17T17:17:59.562147+0000","last_clean":"2024-08-17T17:17:59.562147+0000","last_became_active":"2024-08-17T17:17:52.310809+0000","last_became_peered":"2024-08-17T17:17:52.310809+0000","last_unstale":"2024-08-17T17:17:59.562147+0000","last_undegraded":"2024-08-17T17:17:59.562147+0000","last_fullsized":"2024-08-17T17:17:59.562147+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":288,"log_dups_size":0,"ondisk_log_size":288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:06.780468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18874368,"num_objects":288,"num_object_clones":0,"num_object_copies":864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":288,"num_write_kb":18432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1058,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:54.072322+0000","last_change":"2024-08-17T17:06:39.638433+0000","last_active":"2024-08-17T17:17:54.072322+0000","last_peered":"2024-08-17T17:17:54.072322+0000","last_clean":"2024-08-17T17:17:54.072322+0000","last_became_active":"2024-08-17T17:06:39.638081+0000","last_became_peered":"2024-08-17T17:06:39.638081+0000","last_unstale":"2024-08-17T17:17:54.072322+0000","last_undegraded":"2024-08-17T17:17:54.072322+0000","last_fullsized":"2024-08-17T17:17:54.072322+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:13.558012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1090,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.305230+0000","last_change":"2024-08-17T17:07:45.693574+0000","last_active":"2024-08-17T17:17:53.305230+0000","last_peered":"2024-08-17T17:17:53.305230+0000","last_clean":"2024-08-17T17:17:53.305230+0000","last_became_active":"2024-08-17T17:07:45.693086+0000","last_became_peered":"2024-08-17T17:07:45.693086+0000","last_unstale":"2024-08-17T17:17:53.305230+0000","last_undegraded":"2024-08-17T17:17:53.305230+0000","last_fullsized":"2024-08-17T17:17:53.305230+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:31.888894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1132'843","reported_seq":855,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:18:02.124956+0000","last_change":"2024-08-17T17:17:52.926855+0000","last_active":"2024-08-17T17:18:02.124956+0000","last_peered":"2024-08-17T17:18:02.124956+0000","last_clean":"2024-08-17T17:18:02.124956+0000","last_became_active":"2024-08-17T17:17:52.926494+0000","last_became_peered":"2024-08-17T17:17:52.926494+0000","last_unstale":"2024-08-17T17:18:02.124956+0000","last_undegraded":"2024-08-17T17:18:02.124956+0000","last_fullsized":"2024-08-17T17:18:02.124956+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":843,"log_dups_size":0,"ondisk_log_size":843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:20:18.521178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55246848,"num_objects":843,"num_object_clones":0,"num_object_copies":2529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":843,"num_write_kb":53952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1042,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:54.072172+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:17:54.072172+0000","last_peered":"2024-08-17T17:17:54.072172+0000","last_clean":"2024-08-17T17:17:54.072172+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:17:54.072172+0000","last_undegraded":"2024-08-17T17:17:54.072172+0000","last_fullsized":"2024-08-17T17:17:54.072172+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1072,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.307439+0000","last_change":"2024-08-17T17:06:39.629368+0000","last_active":"2024-08-17T17:17:53.307439+0000","last_peered":"2024-08-17T17:17:53.307439+0000","last_clean":"2024-08-17T17:17:53.307439+0000","last_became_active":"2024-08-17T17:06:39.628832+0000","last_became_peered":"2024-08-17T17:06:39.628832+0000","last_unstale":"2024-08-17T17:17:53.307439+0000","last_undegraded":"2024-08-17T17:17:53.307439+0000","last_fullsized":"2024-08-17T17:17:53.307439+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:54.063884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1132'620","reported_seq":637,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:18:01.166911+0000","last_change":"2024-08-17T17:17:52.311243+0000","last_active":"2024-08-17T17:18:01.166911+0000","last_peered":"2024-08-17T17:18:01.166911+0000","last_clean":"2024-08-17T17:18:01.166911+0000","last_became_active":"2024-08-17T17:17:52.310687+0000","last_became_peered":"2024-08-17T17:17:52.310687+0000","last_unstale":"2024-08-17T17:18:01.166911+0000","last_undegraded":"2024-08-17T17:18:01.166911+0000","last_fullsized":"2024-08-17T17:18:01.166911+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":620,"log_dups_size":0,"ondisk_log_size":620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:53.141395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40632320,"num_objects":620,"num_object_clones":0,"num_object_copies":1860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":620,"num_write_kb":39680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1071,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758253+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:17:53.758253+0000","last_peered":"2024-08-17T17:17:53.758253+0000","last_clean":"2024-08-17T17:17:53.758253+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:17:53.758253+0000","last_undegraded":"2024-08-17T17:17:53.758253+0000","last_fullsized":"2024-08-17T17:17:53.758253+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":849,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.309127+0000","last_change":"2024-08-17T17:17:32.795917+0000","last_active":"2024-08-17T17:17:56.309127+0000","last_peered":"2024-08-17T17:17:56.309127+0000","last_clean":"2024-08-17T17:17:56.309127+0000","last_became_active":"2024-08-17T17:17:32.795583+0000","last_became_peered":"2024-08-17T17:17:32.795583+0000","last_unstale":"2024-08-17T17:17:56.309127+0000","last_undegraded":"2024-08-17T17:17:56.309127+0000","last_fullsized":"2024-08-17T17:17:56.309127+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:49.883791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.f","version":"1132'300","reported_seq":317,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:59.566308+0000","last_change":"2024-08-17T17:17:52.311554+0000","last_active":"2024-08-17T17:17:59.566308+0000","last_peered":"2024-08-17T17:17:59.566308+0000","last_clean":"2024-08-17T17:17:59.566308+0000","last_became_active":"2024-08-17T17:17:52.311201+0000","last_became_peered":"2024-08-17T17:17:52.311201+0000","last_unstale":"2024-08-17T17:17:59.566308+0000","last_undegraded":"2024-08-17T17:17:59.566308+0000","last_fullsized":"2024-08-17T17:17:59.566308+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":300,"log_dups_size":0,"ondisk_log_size":300,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:23:05.713988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19660800,"num_objects":300,"num_object_clones":0,"num_object_copies":900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":300,"num_write_kb":19200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1367,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.948513+0000","last_change":"2024-08-17T17:17:32.855621+0000","last_active":"2024-08-17T17:17:56.948513+0000","last_peered":"2024-08-17T17:17:56.948513+0000","last_clean":"2024-08-17T17:17:56.948513+0000","last_became_active":"2024-08-17T17:17:32.852279+0000","last_became_peered":"2024-08-17T17:17:32.852279+0000","last_unstale":"2024-08-17T17:17:56.948513+0000","last_undegraded":"2024-08-17T17:17:56.948513+0000","last_fullsized":"2024-08-17T17:17:56.948513+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:10.524789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1074,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.305376+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:17:53.305376+0000","last_peered":"2024-08-17T17:17:53.305376+0000","last_clean":"2024-08-17T17:17:53.305376+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:17:53.305376+0000","last_undegraded":"2024-08-17T17:17:53.305376+0000","last_fullsized":"2024-08-17T17:17:53.305376+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"1132'636","reported_seq":653,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:18:01.150484+0000","last_change":"2024-08-17T17:17:52.312176+0000","last_active":"2024-08-17T17:18:01.150484+0000","last_peered":"2024-08-17T17:18:01.150484+0000","last_clean":"2024-08-17T17:18:01.150484+0000","last_became_active":"2024-08-17T17:17:52.311816+0000","last_became_peered":"2024-08-17T17:17:52.311816+0000","last_unstale":"2024-08-17T17:18:01.150484+0000","last_undegraded":"2024-08-17T17:18:01.150484+0000","last_fullsized":"2024-08-17T17:18:01.150484+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":636,"log_dups_size":0,"ondisk_log_size":636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:14:13.595455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41680896,"num_objects":636,"num_object_clones":0,"num_object_copies":1908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":636,"num_write_kb":40704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1073,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.307509+0000","last_change":"2024-08-17T17:06:39.629619+0000","last_active":"2024-08-17T17:17:53.307509+0000","last_peered":"2024-08-17T17:17:53.307509+0000","last_clean":"2024-08-17T17:17:53.307509+0000","last_became_active":"2024-08-17T17:06:39.629295+0000","last_became_peered":"2024-08-17T17:06:39.629295+0000","last_unstale":"2024-08-17T17:17:53.307509+0000","last_undegraded":"2024-08-17T17:17:53.307509+0000","last_fullsized":"2024-08-17T17:17:53.307509+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:15.874423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1089,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.305461+0000","last_change":"2024-08-17T17:07:12.649178+0000","last_active":"2024-08-17T17:17:53.305461+0000","last_peered":"2024-08-17T17:17:53.305461+0000","last_clean":"2024-08-17T17:17:53.305461+0000","last_became_active":"2024-08-17T17:07:12.648844+0000","last_became_peered":"2024-08-17T17:07:12.648844+0000","last_unstale":"2024-08-17T17:17:53.305461+0000","last_undegraded":"2024-08-17T17:17:53.305461+0000","last_fullsized":"2024-08-17T17:17:53.305461+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:12:06.307736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1132'802","reported_seq":814,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:18:02.052211+0000","last_change":"2024-08-17T17:17:52.928868+0000","last_active":"2024-08-17T17:18:02.052211+0000","last_peered":"2024-08-17T17:18:02.052211+0000","last_clean":"2024-08-17T17:18:02.052211+0000","last_became_active":"2024-08-17T17:17:52.928583+0000","last_became_peered":"2024-08-17T17:17:52.928583+0000","last_unstale":"2024-08-17T17:18:02.052211+0000","last_undegraded":"2024-08-17T17:18:02.052211+0000","last_fullsized":"2024-08-17T17:18:02.052211+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":802,"log_dups_size":0,"ondisk_log_size":802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:36:23.472554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52559872,"num_objects":802,"num_object_clones":0,"num_object_copies":2406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":802,"num_write_kb":51328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1076,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.305274+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:17:53.305274+0000","last_peered":"2024-08-17T17:17:53.305274+0000","last_clean":"2024-08-17T17:17:53.305274+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:17:53.305274+0000","last_undegraded":"2024-08-17T17:17:53.305274+0000","last_fullsized":"2024-08-17T17:17:53.305274+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1074,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.305515+0000","last_change":"2024-08-17T17:06:39.627678+0000","last_active":"2024-08-17T17:17:53.305515+0000","last_peered":"2024-08-17T17:17:53.305515+0000","last_clean":"2024-08-17T17:17:53.305515+0000","last_became_active":"2024-08-17T17:06:39.627226+0000","last_became_peered":"2024-08-17T17:06:39.627226+0000","last_unstale":"2024-08-17T17:17:53.305515+0000","last_undegraded":"2024-08-17T17:17:53.305515+0000","last_fullsized":"2024-08-17T17:17:53.305515+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:01.868244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"1132'808","reported_seq":820,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:18:02.049598+0000","last_change":"2024-08-17T17:17:52.309565+0000","last_active":"2024-08-17T17:18:02.049598+0000","last_peered":"2024-08-17T17:18:02.049598+0000","last_clean":"2024-08-17T17:18:02.049598+0000","last_became_active":"2024-08-17T17:17:52.309298+0000","last_became_peered":"2024-08-17T17:17:52.309298+0000","last_unstale":"2024-08-17T17:18:02.049598+0000","last_undegraded":"2024-08-17T17:18:02.049598+0000","last_fullsized":"2024-08-17T17:18:02.049598+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":808,"log_dups_size":0,"ondisk_log_size":808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:27:31.251993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52953088,"num_objects":808,"num_object_clones":0,"num_object_copies":2424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":808,"num_write_kb":51712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1345,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.948168+0000","last_change":"2024-08-17T17:07:45.680458+0000","last_active":"2024-08-17T17:17:56.948168+0000","last_peered":"2024-08-17T17:17:56.948168+0000","last_clean":"2024-08-17T17:17:56.948168+0000","last_became_active":"2024-08-17T17:07:45.680221+0000","last_became_peered":"2024-08-17T17:07:45.680221+0000","last_unstale":"2024-08-17T17:17:56.948168+0000","last_undegraded":"2024-08-17T17:17:56.948168+0000","last_fullsized":"2024-08-17T17:17:56.948168+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:55.083962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1367,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.309527+0000","last_change":"2024-08-17T17:07:45.682866+0000","last_active":"2024-08-17T17:17:56.309527+0000","last_peered":"2024-08-17T17:17:56.309527+0000","last_clean":"2024-08-17T17:17:56.309527+0000","last_became_active":"2024-08-17T17:07:45.682554+0000","last_became_peered":"2024-08-17T17:07:45.682554+0000","last_unstale":"2024-08-17T17:17:56.309527+0000","last_undegraded":"2024-08-17T17:17:56.309527+0000","last_fullsized":"2024-08-17T17:17:56.309527+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:25.975914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.6","version":"1132'828","reported_seq":840,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:18:02.170592+0000","last_change":"2024-08-17T17:17:52.312417+0000","last_active":"2024-08-17T17:18:02.170592+0000","last_peered":"2024-08-17T17:18:02.170592+0000","last_clean":"2024-08-17T17:18:02.170592+0000","last_became_active":"2024-08-17T17:17:52.312180+0000","last_became_peered":"2024-08-17T17:17:52.312180+0000","last_unstale":"2024-08-17T17:18:02.170592+0000","last_undegraded":"2024-08-17T17:18:02.170592+0000","last_fullsized":"2024-08-17T17:18:02.170592+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":828,"log_dups_size":0,"ondisk_log_size":828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:01.854411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54263808,"num_objects":828,"num_object_clones":0,"num_object_copies":2484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":828,"num_write_kb":52992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1412,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:57.010795+0000","last_change":"2024-08-17T17:17:35.800638+0000","last_active":"2024-08-17T17:17:57.010795+0000","last_peered":"2024-08-17T17:17:57.010795+0000","last_clean":"2024-08-17T17:17:57.010795+0000","last_became_active":"2024-08-17T17:17:35.800069+0000","last_became_peered":"2024-08-17T17:17:35.800069+0000","last_unstale":"2024-08-17T17:17:57.010795+0000","last_undegraded":"2024-08-17T17:17:57.010795+0000","last_fullsized":"2024-08-17T17:17:57.010795+0000","mapping_epoch":1124,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1125,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:25:14.529123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1086,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758499+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:17:53.758499+0000","last_peered":"2024-08-17T17:17:53.758499+0000","last_clean":"2024-08-17T17:17:53.758499+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:17:53.758499+0000","last_undegraded":"2024-08-17T17:17:53.758499+0000","last_fullsized":"2024-08-17T17:17:53.758499+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1324,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.948300+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:17:56.948300+0000","last_peered":"2024-08-17T17:17:56.948300+0000","last_clean":"2024-08-17T17:17:56.948300+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:17:56.948300+0000","last_undegraded":"2024-08-17T17:17:56.948300+0000","last_fullsized":"2024-08-17T17:17:56.948300+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1057,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.307691+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:17:53.307691+0000","last_peered":"2024-08-17T17:17:53.307691+0000","last_clean":"2024-08-17T17:17:53.307691+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:17:53.307691+0000","last_undegraded":"2024-08-17T17:17:53.307691+0000","last_fullsized":"2024-08-17T17:17:53.307691+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1132'832","reported_seq":844,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:18:02.160939+0000","last_change":"2024-08-17T17:17:52.929107+0000","last_active":"2024-08-17T17:18:02.160939+0000","last_peered":"2024-08-17T17:18:02.160939+0000","last_clean":"2024-08-17T17:18:02.160939+0000","last_became_active":"2024-08-17T17:17:52.928848+0000","last_became_peered":"2024-08-17T17:17:52.928848+0000","last_unstale":"2024-08-17T17:18:02.160939+0000","last_undegraded":"2024-08-17T17:18:02.160939+0000","last_fullsized":"2024-08-17T17:18:02.160939+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":832,"log_dups_size":0,"ondisk_log_size":832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:33.415593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54525952,"num_objects":832,"num_object_clones":0,"num_object_copies":2496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":832,"num_write_kb":53248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1094,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:54.071767+0000","last_change":"2024-08-17T17:17:32.851428+0000","last_active":"2024-08-17T17:17:54.071767+0000","last_peered":"2024-08-17T17:17:54.071767+0000","last_clean":"2024-08-17T17:17:54.071767+0000","last_became_active":"2024-08-17T17:17:32.850884+0000","last_became_peered":"2024-08-17T17:17:32.850884+0000","last_unstale":"2024-08-17T17:17:54.071767+0000","last_undegraded":"2024-08-17T17:17:54.071767+0000","last_fullsized":"2024-08-17T17:17:54.071767+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:37:44.208775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":916,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.305102+0000","last_change":"2024-08-17T17:17:32.797955+0000","last_active":"2024-08-17T17:17:53.305102+0000","last_peered":"2024-08-17T17:17:53.305102+0000","last_clean":"2024-08-17T17:17:53.305102+0000","last_became_active":"2024-08-17T17:17:32.797623+0000","last_became_peered":"2024-08-17T17:17:32.797623+0000","last_unstale":"2024-08-17T17:17:53.305102+0000","last_undegraded":"2024-08-17T17:17:53.305102+0000","last_fullsized":"2024-08-17T17:17:53.305102+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:08:27.794050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":822,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758678+0000","last_change":"2024-08-17T17:17:32.793469+0000","last_active":"2024-08-17T17:17:53.758678+0000","last_peered":"2024-08-17T17:17:53.758678+0000","last_clean":"2024-08-17T17:17:53.758678+0000","last_became_active":"2024-08-17T17:17:32.793114+0000","last_became_peered":"2024-08-17T17:17:32.793114+0000","last_unstale":"2024-08-17T17:17:53.758678+0000","last_undegraded":"2024-08-17T17:17:53.758678+0000","last_fullsized":"2024-08-17T17:17:53.758678+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:17.999164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"1132'848","reported_seq":860,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:18:02.132879+0000","last_change":"2024-08-17T17:17:52.928812+0000","last_active":"2024-08-17T17:18:02.132879+0000","last_peered":"2024-08-17T17:18:02.132879+0000","last_clean":"2024-08-17T17:18:02.132879+0000","last_became_active":"2024-08-17T17:17:52.928533+0000","last_became_peered":"2024-08-17T17:17:52.928533+0000","last_unstale":"2024-08-17T17:18:02.132879+0000","last_undegraded":"2024-08-17T17:18:02.132879+0000","last_fullsized":"2024-08-17T17:18:02.132879+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":848,"log_dups_size":0,"ondisk_log_size":848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:19:00.282088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55574528,"num_objects":848,"num_object_clones":0,"num_object_copies":2544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":848,"num_write_kb":54272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1415,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.308954+0000","last_change":"2024-08-17T17:17:32.851381+0000","last_active":"2024-08-17T17:17:56.308954+0000","last_peered":"2024-08-17T17:17:56.308954+0000","last_clean":"2024-08-17T17:17:56.308954+0000","last_became_active":"2024-08-17T17:17:32.850867+0000","last_became_peered":"2024-08-17T17:17:32.850867+0000","last_unstale":"2024-08-17T17:17:56.308954+0000","last_undegraded":"2024-08-17T17:17:56.308954+0000","last_fullsized":"2024-08-17T17:17:56.308954+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:51:02.516515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":800,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.309059+0000","last_change":"2024-08-17T17:15:38.610415+0000","last_active":"2024-08-17T17:17:56.309059+0000","last_peered":"2024-08-17T17:17:56.309059+0000","last_clean":"2024-08-17T17:17:56.309059+0000","last_became_active":"2024-08-17T17:15:38.610083+0000","last_became_peered":"2024-08-17T17:15:38.610083+0000","last_unstale":"2024-08-17T17:17:56.309059+0000","last_undegraded":"2024-08-17T17:17:56.309059+0000","last_fullsized":"2024-08-17T17:17:56.309059+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:12.007494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":742,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758698+0000","last_change":"2024-08-17T17:17:32.851823+0000","last_active":"2024-08-17T17:17:53.758698+0000","last_peered":"2024-08-17T17:17:53.758698+0000","last_clean":"2024-08-17T17:17:53.758698+0000","last_became_active":"2024-08-17T17:17:32.851404+0000","last_became_peered":"2024-08-17T17:17:32.851404+0000","last_unstale":"2024-08-17T17:17:53.758698+0000","last_undegraded":"2024-08-17T17:17:53.758698+0000","last_fullsized":"2024-08-17T17:17:53.758698+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:40:30.362384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"1132'756","reported_seq":768,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:18:02.057796+0000","last_change":"2024-08-17T17:17:52.926959+0000","last_active":"2024-08-17T17:18:02.057796+0000","last_peered":"2024-08-17T17:18:02.057796+0000","last_clean":"2024-08-17T17:18:02.057796+0000","last_became_active":"2024-08-17T17:17:52.926563+0000","last_became_peered":"2024-08-17T17:17:52.926563+0000","last_unstale":"2024-08-17T17:18:02.057796+0000","last_undegraded":"2024-08-17T17:18:02.057796+0000","last_fullsized":"2024-08-17T17:18:02.057796+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":756,"log_dups_size":0,"ondisk_log_size":756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:52:38.110516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49545216,"num_objects":756,"num_object_clones":0,"num_object_copies":2268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":756,"num_write_kb":48384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1087,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758505+0000","last_change":"2024-08-17T17:06:39.628745+0000","last_active":"2024-08-17T17:17:53.758505+0000","last_peered":"2024-08-17T17:17:53.758505+0000","last_clean":"2024-08-17T17:17:53.758505+0000","last_became_active":"2024-08-17T17:06:39.628144+0000","last_became_peered":"2024-08-17T17:06:39.628144+0000","last_unstale":"2024-08-17T17:17:53.758505+0000","last_undegraded":"2024-08-17T17:17:53.758505+0000","last_fullsized":"2024-08-17T17:17:53.758505+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:14.740936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1085,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.305431+0000","last_change":"2024-08-17T17:17:32.852814+0000","last_active":"2024-08-17T17:17:53.305431+0000","last_peered":"2024-08-17T17:17:53.305431+0000","last_clean":"2024-08-17T17:17:53.305431+0000","last_became_active":"2024-08-17T17:17:32.852487+0000","last_became_peered":"2024-08-17T17:17:32.852487+0000","last_unstale":"2024-08-17T17:17:53.305431+0000","last_undegraded":"2024-08-17T17:17:53.305431+0000","last_fullsized":"2024-08-17T17:17:53.305431+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:25:52.565337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":876,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.309424+0000","last_change":"2024-08-17T17:15:38.608764+0000","last_active":"2024-08-17T17:17:56.309424+0000","last_peered":"2024-08-17T17:17:56.309424+0000","last_clean":"2024-08-17T17:17:56.309424+0000","last_became_active":"2024-08-17T17:15:38.608353+0000","last_became_peered":"2024-08-17T17:15:38.608353+0000","last_unstale":"2024-08-17T17:17:56.309424+0000","last_undegraded":"2024-08-17T17:17:56.309424+0000","last_fullsized":"2024-08-17T17:17:56.309424+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:42.873441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.1","version":"1132'819","reported_seq":831,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:18:02.131988+0000","last_change":"2024-08-17T17:17:52.309484+0000","last_active":"2024-08-17T17:18:02.131988+0000","last_peered":"2024-08-17T17:18:02.131988+0000","last_clean":"2024-08-17T17:18:02.131988+0000","last_became_active":"2024-08-17T17:17:52.308991+0000","last_became_peered":"2024-08-17T17:17:52.308991+0000","last_unstale":"2024-08-17T17:18:02.131988+0000","last_undegraded":"2024-08-17T17:18:02.131988+0000","last_fullsized":"2024-08-17T17:18:02.131988+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":819,"log_dups_size":0,"ondisk_log_size":819,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:40.953569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53673984,"num_objects":819,"num_object_clones":0,"num_object_copies":2457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":819,"num_write_kb":52416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1087,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758673+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:17:53.758673+0000","last_peered":"2024-08-17T17:17:53.758673+0000","last_clean":"2024-08-17T17:17:53.758673+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:17:53.758673+0000","last_undegraded":"2024-08-17T17:17:53.758673+0000","last_fullsized":"2024-08-17T17:17:53.758673+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":751,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:54.071979+0000","last_change":"2024-08-17T17:17:32.851868+0000","last_active":"2024-08-17T17:17:54.071979+0000","last_peered":"2024-08-17T17:17:54.071979+0000","last_clean":"2024-08-17T17:17:54.071979+0000","last_became_active":"2024-08-17T17:17:32.851497+0000","last_became_peered":"2024-08-17T17:17:32.851497+0000","last_unstale":"2024-08-17T17:17:54.071979+0000","last_undegraded":"2024-08-17T17:17:54.071979+0000","last_fullsized":"2024-08-17T17:17:54.071979+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:13:07.286659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":702,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758411+0000","last_change":"2024-08-17T17:15:38.609480+0000","last_active":"2024-08-17T17:17:53.758411+0000","last_peered":"2024-08-17T17:17:53.758411+0000","last_clean":"2024-08-17T17:17:53.758411+0000","last_became_active":"2024-08-17T17:15:38.609120+0000","last_became_peered":"2024-08-17T17:15:38.609120+0000","last_unstale":"2024-08-17T17:17:53.758411+0000","last_undegraded":"2024-08-17T17:17:53.758411+0000","last_fullsized":"2024-08-17T17:17:53.758411+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:11:03.113425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.0","version":"1132'849","reported_seq":861,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:18:02.149537+0000","last_change":"2024-08-17T17:17:52.927073+0000","last_active":"2024-08-17T17:18:02.149537+0000","last_peered":"2024-08-17T17:18:02.149537+0000","last_clean":"2024-08-17T17:18:02.149537+0000","last_became_active":"2024-08-17T17:17:52.926643+0000","last_became_peered":"2024-08-17T17:17:52.926643+0000","last_unstale":"2024-08-17T17:18:02.149537+0000","last_undegraded":"2024-08-17T17:18:02.149537+0000","last_fullsized":"2024-08-17T17:18:02.149537+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":849,"log_dups_size":0,"ondisk_log_size":849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:18.661591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55640064,"num_objects":849,"num_object_clones":0,"num_object_copies":2547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":849,"num_write_kb":54336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":799,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.309013+0000","last_change":"2024-08-17T17:15:38.609837+0000","last_active":"2024-08-17T17:17:56.309013+0000","last_peered":"2024-08-17T17:17:56.309013+0000","last_clean":"2024-08-17T17:17:56.309013+0000","last_became_active":"2024-08-17T17:15:38.609474+0000","last_became_peered":"2024-08-17T17:15:38.609474+0000","last_unstale":"2024-08-17T17:17:56.309013+0000","last_undegraded":"2024-08-17T17:17:56.309013+0000","last_fullsized":"2024-08-17T17:17:56.309013+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:31:42.450582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1079,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:54.072232+0000","last_change":"2024-08-17T17:17:32.852884+0000","last_active":"2024-08-17T17:17:54.072232+0000","last_peered":"2024-08-17T17:17:54.072232+0000","last_clean":"2024-08-17T17:17:54.072232+0000","last_became_active":"2024-08-17T17:17:32.852580+0000","last_became_peered":"2024-08-17T17:17:32.852580+0000","last_unstale":"2024-08-17T17:17:54.072232+0000","last_undegraded":"2024-08-17T17:17:54.072232+0000","last_fullsized":"2024-08-17T17:17:54.072232+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:14:37.421512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":671,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758421+0000","last_change":"2024-08-17T17:17:32.852470+0000","last_active":"2024-08-17T17:17:53.758421+0000","last_peered":"2024-08-17T17:17:53.758421+0000","last_clean":"2024-08-17T17:17:53.758421+0000","last_became_active":"2024-08-17T17:17:32.852144+0000","last_became_peered":"2024-08-17T17:17:32.852144+0000","last_unstale":"2024-08-17T17:17:53.758421+0000","last_undegraded":"2024-08-17T17:17:53.758421+0000","last_fullsized":"2024-08-17T17:17:53.758421+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:07.441396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.3","version":"1132'283","reported_seq":300,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:59.568837+0000","last_change":"2024-08-17T17:17:52.310380+0000","last_active":"2024-08-17T17:17:59.568837+0000","last_peered":"2024-08-17T17:17:59.568837+0000","last_clean":"2024-08-17T17:17:59.568837+0000","last_became_active":"2024-08-17T17:17:52.310073+0000","last_became_peered":"2024-08-17T17:17:52.310073+0000","last_unstale":"2024-08-17T17:17:59.568837+0000","last_undegraded":"2024-08-17T17:17:59.568837+0000","last_fullsized":"2024-08-17T17:17:59.568837+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":283,"log_dups_size":0,"ondisk_log_size":283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:26:56.341767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18546688,"num_objects":283,"num_object_clones":0,"num_object_copies":849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":283,"num_write_kb":18112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1367,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.949014+0000","last_change":"2024-08-17T17:17:32.799674+0000","last_active":"2024-08-17T17:17:56.949014+0000","last_peered":"2024-08-17T17:17:56.949014+0000","last_clean":"2024-08-17T17:17:56.949014+0000","last_became_active":"2024-08-17T17:17:32.796108+0000","last_became_peered":"2024-08-17T17:17:32.796108+0000","last_unstale":"2024-08-17T17:17:56.949014+0000","last_undegraded":"2024-08-17T17:17:56.949014+0000","last_fullsized":"2024-08-17T17:17:56.949014+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:34:42.105102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":712,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.307720+0000","last_change":"2024-08-17T17:15:38.609543+0000","last_active":"2024-08-17T17:17:53.307720+0000","last_peered":"2024-08-17T17:17:53.307720+0000","last_clean":"2024-08-17T17:17:53.307720+0000","last_became_active":"2024-08-17T17:15:38.609238+0000","last_became_peered":"2024-08-17T17:15:38.609238+0000","last_unstale":"2024-08-17T17:17:53.307720+0000","last_undegraded":"2024-08-17T17:17:53.307720+0000","last_fullsized":"2024-08-17T17:17:53.307720+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:32.029842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1361,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.948146+0000","last_change":"2024-08-17T17:17:32.852691+0000","last_active":"2024-08-17T17:17:56.948146+0000","last_peered":"2024-08-17T17:17:56.948146+0000","last_clean":"2024-08-17T17:17:56.948146+0000","last_became_active":"2024-08-17T17:17:32.852019+0000","last_became_peered":"2024-08-17T17:17:32.852019+0000","last_unstale":"2024-08-17T17:17:56.948146+0000","last_undegraded":"2024-08-17T17:17:56.948146+0000","last_fullsized":"2024-08-17T17:17:56.948146+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:17:42.058647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1132'276","reported_seq":291,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:59.572302+0000","last_change":"2024-08-17T17:17:52.928818+0000","last_active":"2024-08-17T17:17:59.572302+0000","last_peered":"2024-08-17T17:17:59.572302+0000","last_clean":"2024-08-17T17:17:59.572302+0000","last_became_active":"2024-08-17T17:17:52.928511+0000","last_became_peered":"2024-08-17T17:17:52.928511+0000","last_unstale":"2024-08-17T17:17:59.572302+0000","last_undegraded":"2024-08-17T17:17:59.572302+0000","last_fullsized":"2024-08-17T17:17:59.572302+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":276,"log_dups_size":0,"ondisk_log_size":276,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:54:34.862989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18087936,"num_objects":276,"num_object_clones":0,"num_object_copies":828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":276,"num_write_kb":17664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":60,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.305441+0000","last_change":"2024-08-17T17:17:32.859323+0000","last_active":"2024-08-17T17:17:53.305441+0000","last_peered":"2024-08-17T17:17:53.305441+0000","last_clean":"2024-08-17T17:17:53.305441+0000","last_became_active":"2024-08-17T17:17:32.852458+0000","last_became_peered":"2024-08-17T17:17:32.852458+0000","last_unstale":"2024-08-17T17:17:53.305441+0000","last_undegraded":"2024-08-17T17:17:53.305441+0000","last_fullsized":"2024-08-17T17:17:53.305441+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:51:31.286488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1084,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.307809+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:17:53.307809+0000","last_peered":"2024-08-17T17:17:53.307809+0000","last_clean":"2024-08-17T17:17:53.307809+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:17:53.307809+0000","last_undegraded":"2024-08-17T17:17:53.307809+0000","last_fullsized":"2024-08-17T17:17:53.307809+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1060,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.305123+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:17:53.305123+0000","last_peered":"2024-08-17T17:17:53.305123+0000","last_clean":"2024-08-17T17:17:53.305123+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:17:53.305123+0000","last_undegraded":"2024-08-17T17:17:53.305123+0000","last_fullsized":"2024-08-17T17:17:53.305123+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1074,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.305324+0000","last_change":"2024-08-17T17:06:39.627594+0000","last_active":"2024-08-17T17:17:53.305324+0000","last_peered":"2024-08-17T17:17:53.305324+0000","last_clean":"2024-08-17T17:17:53.305324+0000","last_became_active":"2024-08-17T17:06:39.627105+0000","last_became_peered":"2024-08-17T17:06:39.627105+0000","last_unstale":"2024-08-17T17:17:53.305324+0000","last_undegraded":"2024-08-17T17:17:53.305324+0000","last_fullsized":"2024-08-17T17:17:53.305324+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:30.105070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1083,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.305469+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:17:53.305469+0000","last_peered":"2024-08-17T17:17:53.305469+0000","last_clean":"2024-08-17T17:17:53.305469+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:17:53.305469+0000","last_undegraded":"2024-08-17T17:17:53.305469+0000","last_fullsized":"2024-08-17T17:17:53.305469+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":15,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758348+0000","last_change":"2024-08-17T17:17:47.283984+0000","last_active":"2024-08-17T17:17:53.758348+0000","last_peered":"2024-08-17T17:17:53.758348+0000","last_clean":"2024-08-17T17:17:53.758348+0000","last_became_active":"2024-08-17T17:17:47.283270+0000","last_became_peered":"2024-08-17T17:17:47.283270+0000","last_unstale":"2024-08-17T17:17:53.758348+0000","last_undegraded":"2024-08-17T17:17:53.758348+0000","last_fullsized":"2024-08-17T17:17:53.758348+0000","mapping_epoch":1127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:16.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1059,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:54.071614+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:17:54.071614+0000","last_peered":"2024-08-17T17:17:54.071614+0000","last_clean":"2024-08-17T17:17:54.071614+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:17:54.071614+0000","last_undegraded":"2024-08-17T17:17:54.071614+0000","last_fullsized":"2024-08-17T17:17:54.071614+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1071,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758686+0000","last_change":"2024-08-17T17:06:39.625029+0000","last_active":"2024-08-17T17:17:53.758686+0000","last_peered":"2024-08-17T17:17:53.758686+0000","last_clean":"2024-08-17T17:17:53.758686+0000","last_became_active":"2024-08-17T17:06:39.624688+0000","last_became_peered":"2024-08-17T17:06:39.624688+0000","last_unstale":"2024-08-17T17:17:53.758686+0000","last_undegraded":"2024-08-17T17:17:53.758686+0000","last_fullsized":"2024-08-17T17:17:53.758686+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:54.849668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1324,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.947991+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:17:56.947991+0000","last_peered":"2024-08-17T17:17:56.947991+0000","last_clean":"2024-08-17T17:17:56.947991+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:17:56.947991+0000","last_undegraded":"2024-08-17T17:17:56.947991+0000","last_fullsized":"2024-08-17T17:17:56.947991+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":916,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758759+0000","last_change":"2024-08-17T17:17:32.796452+0000","last_active":"2024-08-17T17:17:53.758759+0000","last_peered":"2024-08-17T17:17:53.758759+0000","last_clean":"2024-08-17T17:17:53.758759+0000","last_became_active":"2024-08-17T17:17:32.795964+0000","last_became_peered":"2024-08-17T17:17:32.795964+0000","last_unstale":"2024-08-17T17:17:53.758759+0000","last_undegraded":"2024-08-17T17:17:53.758759+0000","last_fullsized":"2024-08-17T17:17:53.758759+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:15.604103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1343,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.948045+0000","last_change":"2024-08-17T17:07:45.678950+0000","last_active":"2024-08-17T17:17:56.948045+0000","last_peered":"2024-08-17T17:17:56.948045+0000","last_clean":"2024-08-17T17:17:56.948045+0000","last_became_active":"2024-08-17T17:07:45.678472+0000","last_became_peered":"2024-08-17T17:07:45.678472+0000","last_unstale":"2024-08-17T17:17:56.948045+0000","last_undegraded":"2024-08-17T17:17:56.948045+0000","last_fullsized":"2024-08-17T17:17:56.948045+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:48:02.385927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1072,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.307759+0000","last_change":"2024-08-17T17:06:39.628542+0000","last_active":"2024-08-17T17:17:53.307759+0000","last_peered":"2024-08-17T17:17:53.307759+0000","last_clean":"2024-08-17T17:17:53.307759+0000","last_became_active":"2024-08-17T17:06:39.628253+0000","last_became_peered":"2024-08-17T17:06:39.628253+0000","last_unstale":"2024-08-17T17:17:53.307759+0000","last_undegraded":"2024-08-17T17:17:53.307759+0000","last_fullsized":"2024-08-17T17:17:53.307759+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:22.349326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1366,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.947819+0000","last_change":"2024-08-17T17:17:32.852397+0000","last_active":"2024-08-17T17:17:56.947819+0000","last_peered":"2024-08-17T17:17:56.947819+0000","last_clean":"2024-08-17T17:17:56.947819+0000","last_became_active":"2024-08-17T17:17:32.851621+0000","last_became_peered":"2024-08-17T17:17:32.851621+0000","last_unstale":"2024-08-17T17:17:56.947819+0000","last_undegraded":"2024-08-17T17:17:56.947819+0000","last_fullsized":"2024-08-17T17:17:56.947819+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:37:18.214084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":876,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:54.072282+0000","last_change":"2024-08-17T17:17:32.851510+0000","last_active":"2024-08-17T17:17:54.072282+0000","last_peered":"2024-08-17T17:17:54.072282+0000","last_clean":"2024-08-17T17:17:54.072282+0000","last_became_active":"2024-08-17T17:17:32.851199+0000","last_became_peered":"2024-08-17T17:17:32.851199+0000","last_unstale":"2024-08-17T17:17:54.072282+0000","last_undegraded":"2024-08-17T17:17:54.072282+0000","last_fullsized":"2024-08-17T17:17:54.072282+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:07:24.289278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":863,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.948127+0000","last_change":"2024-08-17T17:15:38.610768+0000","last_active":"2024-08-17T17:17:56.948127+0000","last_peered":"2024-08-17T17:17:56.948127+0000","last_clean":"2024-08-17T17:17:56.948127+0000","last_became_active":"2024-08-17T17:15:38.610337+0000","last_became_peered":"2024-08-17T17:15:38.610337+0000","last_unstale":"2024-08-17T17:17:56.948127+0000","last_undegraded":"2024-08-17T17:17:56.948127+0000","last_fullsized":"2024-08-17T17:17:56.948127+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:15.632778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1343,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.948521+0000","last_change":"2024-08-17T17:07:45.689782+0000","last_active":"2024-08-17T17:17:56.948521+0000","last_peered":"2024-08-17T17:17:56.948521+0000","last_clean":"2024-08-17T17:17:56.948521+0000","last_became_active":"2024-08-17T17:07:45.689312+0000","last_became_peered":"2024-08-17T17:07:45.689312+0000","last_unstale":"2024-08-17T17:17:56.948521+0000","last_undegraded":"2024-08-17T17:17:56.948521+0000","last_fullsized":"2024-08-17T17:17:56.948521+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:07.560555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1090,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.305180+0000","last_change":"2024-08-17T17:07:12.649175+0000","last_active":"2024-08-17T17:17:53.305180+0000","last_peered":"2024-08-17T17:17:53.305180+0000","last_clean":"2024-08-17T17:17:53.305180+0000","last_became_active":"2024-08-17T17:07:12.648822+0000","last_became_peered":"2024-08-17T17:07:12.648822+0000","last_unstale":"2024-08-17T17:17:53.305180+0000","last_undegraded":"2024-08-17T17:17:53.305180+0000","last_fullsized":"2024-08-17T17:17:53.305180+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:03.924431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1391,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.309327+0000","last_change":"2024-08-17T17:17:47.316690+0000","last_active":"2024-08-17T17:17:56.309327+0000","last_peered":"2024-08-17T17:17:56.309327+0000","last_clean":"2024-08-17T17:17:56.309327+0000","last_became_active":"2024-08-17T17:17:47.316333+0000","last_became_peered":"2024-08-17T17:17:47.316333+0000","last_unstale":"2024-08-17T17:17:56.309327+0000","last_undegraded":"2024-08-17T17:17:56.309327+0000","last_fullsized":"2024-08-17T17:17:56.309327+0000","mapping_epoch":1127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:04.542237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":69,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.305237+0000","last_change":"2024-08-17T17:17:32.854358+0000","last_active":"2024-08-17T17:17:53.305237+0000","last_peered":"2024-08-17T17:17:53.305237+0000","last_clean":"2024-08-17T17:17:53.305237+0000","last_became_active":"2024-08-17T17:17:32.854073+0000","last_became_peered":"2024-08-17T17:17:32.854073+0000","last_unstale":"2024-08-17T17:17:53.305237+0000","last_undegraded":"2024-08-17T17:17:53.305237+0000","last_fullsized":"2024-08-17T17:17:53.305237+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:10:05.592164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1088,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.305322+0000","last_change":"2024-08-17T17:07:45.684606+0000","last_active":"2024-08-17T17:17:53.305322+0000","last_peered":"2024-08-17T17:17:53.305322+0000","last_clean":"2024-08-17T17:17:53.305322+0000","last_became_active":"2024-08-17T17:07:45.684124+0000","last_became_peered":"2024-08-17T17:07:45.684124+0000","last_unstale":"2024-08-17T17:17:53.305322+0000","last_undegraded":"2024-08-17T17:17:53.305322+0000","last_fullsized":"2024-08-17T17:17:53.305322+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:16.110507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1072,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758599+0000","last_change":"2024-08-17T17:06:39.625245+0000","last_active":"2024-08-17T17:17:53.758599+0000","last_peered":"2024-08-17T17:17:53.758599+0000","last_clean":"2024-08-17T17:17:53.758599+0000","last_became_active":"2024-08-17T17:06:39.624657+0000","last_became_peered":"2024-08-17T17:06:39.624657+0000","last_unstale":"2024-08-17T17:17:53.758599+0000","last_undegraded":"2024-08-17T17:17:53.758599+0000","last_fullsized":"2024-08-17T17:17:53.758599+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:43.097460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1097,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.305095+0000","last_change":"2024-08-17T17:17:32.796353+0000","last_active":"2024-08-17T17:17:53.305095+0000","last_peered":"2024-08-17T17:17:53.305095+0000","last_clean":"2024-08-17T17:17:53.305095+0000","last_became_active":"2024-08-17T17:17:32.796002+0000","last_became_peered":"2024-08-17T17:17:32.796002+0000","last_unstale":"2024-08-17T17:17:53.305095+0000","last_undegraded":"2024-08-17T17:17:53.305095+0000","last_fullsized":"2024-08-17T17:17:53.305095+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:13:24.683597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1057,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758626+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:17:53.758626+0000","last_peered":"2024-08-17T17:17:53.758626+0000","last_clean":"2024-08-17T17:17:53.758626+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:17:53.758626+0000","last_undegraded":"2024-08-17T17:17:53.758626+0000","last_fullsized":"2024-08-17T17:17:53.758626+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1356,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.947835+0000","last_change":"2024-08-17T17:17:32.791886+0000","last_active":"2024-08-17T17:17:56.947835+0000","last_peered":"2024-08-17T17:17:56.947835+0000","last_clean":"2024-08-17T17:17:56.947835+0000","last_became_active":"2024-08-17T17:17:32.791528+0000","last_became_peered":"2024-08-17T17:17:32.791528+0000","last_unstale":"2024-08-17T17:17:56.947835+0000","last_undegraded":"2024-08-17T17:17:56.947835+0000","last_fullsized":"2024-08-17T17:17:56.947835+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:33:04.318911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1338,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.948834+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:17:56.948834+0000","last_peered":"2024-08-17T17:17:56.948834+0000","last_clean":"2024-08-17T17:17:56.948834+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:17:56.948834+0000","last_undegraded":"2024-08-17T17:17:56.948834+0000","last_fullsized":"2024-08-17T17:17:56.948834+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1074,"reported_epoch":1132,"state":"active+clean","last_fresh":"2024-08-17T17:17:56.917956+0000","last_change":"2024-08-17T17:06:39.627689+0000","last_active":"2024-08-17T17:17:56.917956+0000","last_peered":"2024-08-17T17:17:56.917956+0000","last_clean":"2024-08-17T17:17:56.917956+0000","last_became_active":"2024-08-17T17:06:39.627375+0000","last_became_peered":"2024-08-17T17:06:39.627375+0000","last_unstale":"2024-08-17T17:17:56.917956+0000","last_undegraded":"2024-08-17T17:17:56.917956+0000","last_fullsized":"2024-08-17T17:17:56.917956+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:51.954076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":85,"reported_epoch":1131,"state":"active+clean","last_fresh":"2024-08-17T17:17:53.758753+0000","last_change":"2024-08-17T17:17:32.800443+0000","last_active":"2024-08-17T17:17:53.758753+0000","last_peered":"2024-08-17T17:17:53.758753+0000","last_clean":"2024-08-17T17:17:53.758753+0000","last_became_active":"2024-08-17T17:17:32.795103+0000","last_became_peered":"2024-08-17T17:17:32.795103+0000","last_unstale":"2024-08-17T17:17:53.758753+0000","last_undegraded":"2024-08-17T17:17:53.758753+0000","last_fullsized":"2024-08-17T17:17:53.758753+0000","mapping_epoch":1121,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1122,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:17:19.842778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":692649984,"num_objects":10569,"num_object_clones":0,"num_object_copies":31707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10569,"num_write_kb":676416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2205024256,"data_stored":2205024256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":10569,"ondisk_log_size":10569,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":604,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201599,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":497084,"kb_used_data":321652,"kb_used_omap":1,"kb_used_meta":175422,"kb_avail":93248068,"statfs":{"total":95995035648,"available":95486021632,"internally_reserved":0,"allocated":329371648,"data_stored":327338181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":179632587},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463236,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":407928,"kb_used_data":268592,"kb_used_omap":1,"kb_used_meta":139326,"kb_avail":93337224,"statfs":{"total":95995035648,"available":95577317376,"internally_reserved":0,"allocated":275038208,"data_stored":273009292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":142670281},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626877,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":426656,"kb_used_data":177944,"kb_used_omap":42,"kb_used_meta":248661,"kb_avail":93318496,"statfs":{"total":95995035648,"available":95558139904,"internally_reserved":0,"allocated":182214656,"data_stored":180204447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":254628924},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785355,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":620600,"kb_used_data":502252,"kb_used_omap":48,"kb_used_meta":118287,"kb_avail":93124552,"statfs":{"total":95995035648,"available":95359541248,"internally_reserved":0,"allocated":514306048,"data_stored":512284172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":121126574},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724872,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":416936,"kb_used_data":241504,"kb_used_omap":1,"kb_used_meta":175422,"kb_avail":93328216,"statfs":{"total":95995035648,"available":95568093184,"internally_reserved":0,"allocated":247300096,"data_stored":245282264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":179632582},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986509,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":490740,"kb_used_data":301356,"kb_used_omap":1,"kb_used_meta":189374,"kb_avail":93254412,"statfs":{"total":95995035648,"available":95492517888,"internally_reserved":0,"allocated":308588544,"data_stored":306562655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":193919430},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706116,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":404516,"kb_used_data":232284,"kb_used_omap":35,"kb_used_meta":172188,"kb_avail":93340636,"statfs":{"total":95995035648,"available":95580811264,"internally_reserved":0,"allocated":237858816,"data_stored":235845037,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":176320567},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":0,"seq":3173980831889,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":515702784,"data_stored":515702784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":250085376,"data_stored":250085376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":203816960,"data_stored":203816960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":325582848,"data_stored":325582848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":271253504,"data_stored":271253504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":311361536,"data_stored":311361536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327221248,"data_stored":327221248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:18:06.361 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T17:18:06.361 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set .mgr pg_num 11 2024-08-17T17:18:06.381 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:06.435 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 8 16 24582 24566 191.888 159.188 0.00930552 0.00520576 2024-08-17T17:18:06.446 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:06.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:06.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:06.512 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:06 smithi162 ceph-mon[104668]: from='client.54718 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:18:06.514 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:18:06.548 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:06.590 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:06.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:06.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:06.692 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:06.737 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:06.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:06.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:06.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:06 smithi063 ceph-mon[114756]: from='client.54718 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:18:06.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:06 smithi063 ceph-mon[109843]: from='client.54718 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:18:06.840 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:06.870 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:06.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:06.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:06.972 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:06.979 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:07.004 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:07.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:07.105 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:07.146 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:07.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:07.248 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:07.294 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:07.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:07.396 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:07.435 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 9 16 26922 26906 186.816 146.25 0.00694803 0.00534694 2024-08-17T17:18:07.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.436 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:07.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:07.514 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:18:07.515 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:18:07.537 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:07.572 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:07.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:07.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:07 smithi162 ceph-mon[104668]: pgmap v1319: 89 pgs: 89 active+clean; 899 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-08-17T17:18:07.643 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:07 smithi063 ceph-mon[114756]: pgmap v1319: 89 pgs: 89 active+clean; 899 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-08-17T17:18:07.643 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:07 smithi063 ceph-mon[109843]: pgmap v1319: 89 pgs: 89 active+clean; 899 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 91 MiB/s wr, 1.46k op/s 2024-08-17T17:18:07.675 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:07.706 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:07.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:07.808 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:07.830 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:07.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:07.932 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:07.956 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:07.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:07.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:08.058 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:08.094 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:08.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:08.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:08.198 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:08.226 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:08.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:08.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:08.328 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:08.364 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:08.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:08.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:08.436 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 10 16 28748 28732 179.546 114.125 0.00196317 0.00552799 2024-08-17T17:18:08.465 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:08.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:08.491 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:08.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:08.592 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:08.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:08.622 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:08.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:08.660 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:18:08.660 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:18:08.661 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:18:08.661 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:18:08.661 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:18:08.661 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:18:08.661 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:18:08.661 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:18:08.661 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:18:08.661 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:18:08.661 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:18:08.662 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:18:08.662 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:18:08.662 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:18:08.662 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:18:08.662 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:18:08.662 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:08 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2940173656' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:08.662 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:08 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2940173656' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:08.723 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:08.744 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:08.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:08.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:08.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:08 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2940173656' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:08.846 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:08.882 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:08.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:08.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:08.984 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:09.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.023 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:09.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:09.124 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:09.159 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:09.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:09.261 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:09.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.296 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:09.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:09.397 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:09.435 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:09.436 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 11 16 30575 30559 173.603 114.188 0.00427572 0.00575451 2024-08-17T17:18:09.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:09.537 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:09.574 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:09.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:09.677 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:09.715 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:09.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:09.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:09 smithi063 ceph-mon[109843]: pgmap v1320: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 152 MiB/s wr, 2.42k op/s 2024-08-17T17:18:09.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:09 smithi063 ceph-mon[114756]: pgmap v1320: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 152 MiB/s wr, 2.42k op/s 2024-08-17T17:18:09.817 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:09.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:09 smithi162 ceph-mon[104668]: pgmap v1320: 89 pgs: 89 active+clean; 1.5 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 152 MiB/s wr, 2.42k op/s 2024-08-17T17:18:09.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.855 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:09.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:09.956 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:09.981 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:09.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:09.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:10.082 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:10.120 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:10.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:10.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:10.221 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:10.255 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:10.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:10.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:10.356 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:10.395 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:10.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:10.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:10.436 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 12 16 32546 32530 169.401 123.188 0.00178366 0.0058924 2024-08-17T17:18:10.496 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:10.519 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:10.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:10.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:10.621 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:10.660 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:10.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:10.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:10.762 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:10.783 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:10.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:10.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:10.885 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:10.917 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:10.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:10.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:11.019 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:11.059 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:11.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:11.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:11.105 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:18:11.160 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:11.186 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:11.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:11.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:11.287 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:11.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:11.326 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:11.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:11.399 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:18:11.426 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:11.436 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 13 16 34011 33995 163.413 91.5625 0.00466818 0.00611439 2024-08-17T17:18:11.466 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:11.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:11.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:11.568 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:11.589 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:11.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:11.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:11.673 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:11 smithi162 ceph-mon[104668]: pgmap v1321: 89 pgs: 89 active+clean; 1.7 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 162 MiB/s wr, 2.59k op/s 2024-08-17T17:18:11.691 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:11.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:11.729 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:11.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:11.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:11 smithi063 ceph-mon[114756]: pgmap v1321: 89 pgs: 89 active+clean; 1.7 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 162 MiB/s wr, 2.59k op/s 2024-08-17T17:18:11.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:11 smithi063 ceph-mon[109843]: pgmap v1321: 89 pgs: 89 active+clean; 1.7 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 162 MiB/s wr, 2.59k op/s 2024-08-17T17:18:11.831 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:11.869 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:11.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:11.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:11.971 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:12.006 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:12.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:12.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:12.108 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:12.142 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:12.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:12.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:12.243 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:12.277 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:12.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:12.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:12.379 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:12.406 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:12.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:12.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:12.436 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 14 16 35747 35731 159.49 108.5 0.00900237 0.00626456 2024-08-17T17:18:12.508 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:12.541 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:12.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:12.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:12.643 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:12.681 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:12.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:12.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:12.783 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:12.817 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:12.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:12.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:12.919 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:12.957 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:12.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:12.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:12.995 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:13.059 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:13.095 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:13.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:13.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:13.197 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:13.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:13.228 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:13.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:13.329 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:13.365 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:13.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:13.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:13.383 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:18:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:18:13] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.51.0" 2024-08-17T17:18:13.436 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 15 15 37771 37756 157.295 126.562 0.0051175 0.00635198 2024-08-17T17:18:13.467 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:13.489 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:13.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:13.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:13.591 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:13.632 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:13.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:13.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:13.695 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:13 smithi063 ceph-mon[114756]: pgmap v1322: 89 pgs: 89 active+clean; 1.9 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-08-17T17:18:13.695 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:13 smithi063 ceph-mon[109843]: pgmap v1322: 89 pgs: 89 active+clean; 1.9 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-08-17T17:18:13.734 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:13.759 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:13.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:13.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:13.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:13 smithi162 ceph-mon[104668]: pgmap v1322: 89 pgs: 89 active+clean; 1.9 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 152 MiB/s wr, 2.43k op/s 2024-08-17T17:18:13.861 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:13.897 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:13.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:13.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:13.999 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:14.032 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:14.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:14.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:14.134 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:14.168 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:14.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:14.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:14.270 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:14.318 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:14.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:14.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:14.371 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:18:14.371 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:18:14.372 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:18:14.372 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:18:14.372 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:18:14.420 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:14.436 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 16 16 39613 39597 154.655 115.062 0.00521605 0.00646097 2024-08-17T17:18:14.453 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:14.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:14.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:14.555 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:14.588 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:14.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:14.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:14.690 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:14.736 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:14.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:14.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:14.837 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:14.868 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:14.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:14.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:14.970 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:14.998 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:14.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:14.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:15.064 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:18:15.114 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:15.149 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:15.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:15.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:15.251 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:15.282 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:15.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:15.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:15.384 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:15.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:15.429 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:15.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:15.436 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 17 16 41354 41338 151.958 108.812 0.00427607 0.00657617 2024-08-17T17:18:15.459 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:18:15.530 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:15.568 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:15.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:15.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:15.620 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:18:15.670 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:15.673 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:15 smithi063 ceph-mon[109843]: pgmap v1323: 89 pgs: 89 active+clean; 2.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-08-17T17:18:15.673 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:15 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/246568026' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:15.673 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:15 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:15.673 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:15 smithi063 ceph-mon[114756]: pgmap v1323: 89 pgs: 89 active+clean; 2.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-08-17T17:18:15.673 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:15 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/246568026' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:15.674 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:15 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:15.695 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:18:15.703 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:15.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:15.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:15.757 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:15 smithi162 ceph-mon[104668]: pgmap v1323: 89 pgs: 89 active+clean; 2.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s 2024-08-17T17:18:15.757 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:15 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/246568026' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:15.757 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:15 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:15.804 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:15.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:15.850 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:15.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:15.951 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:15.988 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:15.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:15.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:16.090 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:16.130 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:16.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:16.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:16.232 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:16.266 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:16.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:16.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:16.368 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:16.403 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:16.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:16.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:16.436 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 18 16 43117 43101 149.637 110.188 0.0100233 0.00667762 2024-08-17T17:18:16.505 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:16.545 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:16.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:16.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:16.647 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:16.683 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:16.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:16.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:16.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:16 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:16.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:16 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:18:16.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:16 smithi063 ceph-mon[114756]: osdmap e1134: 8 total, 8 up, 7 in 2024-08-17T17:18:16.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:16 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:18:16.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:16 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:16 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:18:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:16 smithi063 ceph-mon[109843]: osdmap e1134: 8 total, 8 up, 7 in 2024-08-17T17:18:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:16 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:18:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:16.786 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:16.825 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:16.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:16.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:16 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:16 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:18:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:16 smithi162 ceph-mon[104668]: osdmap e1134: 8 total, 8 up, 7 in 2024-08-17T17:18:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:16 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:18:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:16.927 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:16.962 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:16.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:17.064 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:17.098 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:17.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:17.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:17.200 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:17.233 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:17.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:17.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:17.335 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:17.359 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:17.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:17.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:17.412 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:17 smithi063 ceph-mon[109843]: pgmap v1325: 89 pgs: 89 active+clean; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s 2024-08-17T17:18:17.429 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:17.436 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 19 16 44756 44740 147.152 102.438 0.00320152 0.00678865 2024-08-17T17:18:17.461 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:17.491 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:17.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:17.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:17.592 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:17.635 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:17.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:17.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:17.719 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:17 smithi063 ceph-mon[114756]: pgmap v1325: 89 pgs: 89 active+clean; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s 2024-08-17T17:18:17.738 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:17.768 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:17.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:17.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:17.806 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:17 smithi162 ceph-mon[104668]: pgmap v1325: 89 pgs: 89 active+clean; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s 2024-08-17T17:18:17.870 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:17.912 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:17.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:17.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:18.014 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:18.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:18.054 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:18.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:18.155 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:18.202 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:18.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:18.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:18.304 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:18.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:18.346 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:18.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:18.419 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:18 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3606909788' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-08-17T17:18:18.430 INFO:teuthology.orchestra.run.smithi063.stderr:set pool 1 pg_num to 11 2024-08-17T17:18:18.436 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:18:18.436740+0000 min lat: 0.00126452 max lat: 0.180176 avg lat: 0.00687462 2024-08-17T17:18:18.436 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:18:18.436 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 20 16 46534 46518 145.35 111.125 0.0111153 0.00687462 2024-08-17T17:18:18.447 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:18.486 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:18.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:18.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:18.491 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:18:18.491 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:18:18.491 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:18:18.491 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:18:18.491 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.491 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.44609.0:97581 6.a e1099/1086 6:5bc0b130:::benchmark_data_smithi063_160268_object97580:head v 1099'6149, mlcod=1099'6147)", 2024-08-17T17:18:18.492 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:14:26.419006+0000", 2024-08-17T17:18:18.492 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 232.06877107, 2024-08-17T17:18:18.492 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.077766874, 2024-08-17T17:18:18.492 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.492 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.492 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.492 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.492 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.492 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419006+0000", 2024-08-17T17:18:18.492 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.492 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.493 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.493 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.493 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419006+0000", 2024-08-17T17:18:18.493 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.493 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.493 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.493 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.493 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419005+0000", 2024-08-17T17:18:18.493 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:18.493 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.493 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.493 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.493 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419037+0000", 2024-08-17T17:18:18.493 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1760000000000001e-05 2024-08-17T17:18:18.493 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.494 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.494 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.494 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419050+0000", 2024-08-17T17:18:18.494 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3517e-05 2024-08-17T17:18:18.494 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.494 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.494 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.494 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419058+0000", 2024-08-17T17:18:18.494 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6020000000000002e-06 2024-08-17T17:18:18.494 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.494 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.494 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.494 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419100+0000", 2024-08-17T17:18:18.495 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1868e-05 2024-08-17T17:18:18.495 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.495 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.495 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.495 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.419121+0000", 2024-08-17T17:18:18.495 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.1081999999999999e-05 2024-08-17T17:18:18.495 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.495 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.495 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.495 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.496717+0000", 2024-08-17T17:18:18.495 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.077596214999999996 2024-08-17T17:18:18.495 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.495 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.495 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.495 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.496773+0000", 2024-08-17T17:18:18.496 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5755999999999999e-05 2024-08-17T17:18:18.496 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.496 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.496 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.496 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.496 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.496 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.44609.0:97583 6.a e1099/1086 6:5a623a4e:::benchmark_data_smithi063_160268_object97582:head v 1099'6150, mlcod=1099'6147)", 2024-08-17T17:18:18.496 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:14:26.421947+0000", 2024-08-17T17:18:18.496 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 232.06583024599999, 2024-08-17T17:18:18.496 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.074887432000000004, 2024-08-17T17:18:18.496 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.496 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.497 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.497 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.497 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.497 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.421947+0000", 2024-08-17T17:18:18.497 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.497 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.497 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.497 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.497 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.421947+0000", 2024-08-17T17:18:18.497 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.497 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.497 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.497 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.498 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.421946+0000", 2024-08-17T17:18:18.498 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:18.498 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.498 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.498 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.498 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.421973+0000", 2024-08-17T17:18:18.498 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6738999999999999e-05 2024-08-17T17:18:18.498 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.498 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.498 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.498 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.487339+0000", 2024-08-17T17:18:18.498 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.065366558000000005 2024-08-17T17:18:18.499 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.499 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.499 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.499 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.487357+0000", 2024-08-17T17:18:18.499 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7680999999999999e-05 2024-08-17T17:18:18.499 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.499 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.499 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.499 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.489609+0000", 2024-08-17T17:18:18.499 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0022519990000000002 2024-08-17T17:18:18.499 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.499 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.499 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.499 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.489634+0000", 2024-08-17T17:18:18.500 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4640000000000001e-05 2024-08-17T17:18:18.500 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.500 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.500 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.500 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.496779+0000", 2024-08-17T17:18:18.500 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0071450990000000002 2024-08-17T17:18:18.500 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.500 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.500 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.500 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:14:26.496834+0000", 2024-08-17T17:18:18.500 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5643000000000002e-05 2024-08-17T17:18:18.500 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.500 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.500 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.501 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.501 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.501 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160939 6.1 6:893849fe:::benchmark_data_smithi063_160268_object34832:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:18:18.501 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.614043+0000", 2024-08-17T17:18:18.501 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.87373439800001, 2024-08-17T17:18:18.501 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0308899330000001, 2024-08-17T17:18:18.501 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.501 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.501 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.501 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:18.501 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:18.501 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160939 2024-08-17T17:18:18.501 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.501 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.502 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.502 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.502 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614043+0000", 2024-08-17T17:18:18.502 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.502 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.502 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.502 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.502 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614043+0000", 2024-08-17T17:18:18.502 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.502 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.502 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.502 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.502 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614041+0000", 2024-08-17T17:18:18.503 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:18:18.503 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.503 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.503 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.503 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614047+0000", 2024-08-17T17:18:18.503 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9440000000000002e-06 2024-08-17T17:18:18.503 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.503 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.503 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.503 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614057+0000", 2024-08-17T17:18:18.503 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3409999999999992e-06 2024-08-17T17:18:18.503 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.504 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.504 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.504 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614064+0000", 2024-08-17T17:18:18.504 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7190000000000001e-06 2024-08-17T17:18:18.504 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.504 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.504 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.504 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614117+0000", 2024-08-17T17:18:18.504 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2797000000000003e-05 2024-08-17T17:18:18.504 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.504 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.504 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:18.504 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614128+0000", 2024-08-17T17:18:18.505 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0891e-05 2024-08-17T17:18:18.505 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.505 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.505 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.505 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.631386+0000", 2024-08-17T17:18:18.505 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.017258346 2024-08-17T17:18:18.505 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.505 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.505 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.505 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.633791+0000", 2024-08-17T17:18:18.505 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0024042109999999998 2024-08-17T17:18:18.505 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.506 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.506 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:18:18.506 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.633951+0000", 2024-08-17T17:18:18.506 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016093800000000001 2024-08-17T17:18:18.506 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.506 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.506 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.507 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644732+0000", 2024-08-17T17:18:18.507 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010780998 2024-08-17T17:18:18.507 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.507 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.507 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.507 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644760+0000", 2024-08-17T17:18:18.507 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.7195e-05 2024-08-17T17:18:18.507 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.507 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.507 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.507 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644837+0000", 2024-08-17T17:18:18.507 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7256999999999996e-05 2024-08-17T17:18:18.507 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.508 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.508 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.508 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644891+0000", 2024-08-17T17:18:18.508 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4284999999999999e-05 2024-08-17T17:18:18.508 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.508 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.508 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.508 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644933+0000", 2024-08-17T17:18:18.508 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1493999999999999e-05 2024-08-17T17:18:18.508 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.508 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.508 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.508 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.508 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.509 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160958 6.1 6:804ff99b:::benchmark_data_smithi063_160268_object34851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:18:18.509 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.649687+0000", 2024-08-17T17:18:18.509 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.83809055, 2024-08-17T17:18:18.509 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99790868700000002, 2024-08-17T17:18:18.509 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.509 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.509 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.509 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:18.509 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:18.509 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160958 2024-08-17T17:18:18.509 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.509 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.509 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.510 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.510 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649687+0000", 2024-08-17T17:18:18.510 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.510 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.510 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.510 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.510 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649687+0000", 2024-08-17T17:18:18.510 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.510 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.510 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.510 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.510 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649685+0000", 2024-08-17T17:18:18.510 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:18:18.510 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.511 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.511 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.511 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649695+0000", 2024-08-17T17:18:18.511 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9210000000000004e-06 2024-08-17T17:18:18.511 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.511 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.511 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.511 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649708+0000", 2024-08-17T17:18:18.511 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2947e-05 2024-08-17T17:18:18.511 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.511 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.511 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.511 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649715+0000", 2024-08-17T17:18:18.511 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9519999999999994e-06 2024-08-17T17:18:18.512 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.512 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.512 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.512 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649779+0000", 2024-08-17T17:18:18.512 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3683000000000007e-05 2024-08-17T17:18:18.512 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.512 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.512 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:18.512 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649792+0000", 2024-08-17T17:18:18.512 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3294e-05 2024-08-17T17:18:18.512 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.512 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.512 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.512 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.637682+0000", 2024-08-17T17:18:18.513 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98788912399999995 2024-08-17T17:18:18.513 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.513 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.513 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.513 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.637791+0000", 2024-08-17T17:18:18.513 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000109728 2024-08-17T17:18:18.513 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.513 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.513 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:18:18.513 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.637918+0000", 2024-08-17T17:18:18.513 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000126842 2024-08-17T17:18:18.513 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.513 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.513 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.514 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644787+0000", 2024-08-17T17:18:18.514 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0068684000000000002 2024-08-17T17:18:18.514 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.514 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.514 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.514 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647458+0000", 2024-08-17T17:18:18.514 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026717220000000001 2024-08-17T17:18:18.514 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.514 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.514 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.514 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647543+0000", 2024-08-17T17:18:18.514 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4790999999999995e-05 2024-08-17T17:18:18.514 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.514 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.515 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.515 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647572+0000", 2024-08-17T17:18:18.515 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8538999999999999e-05 2024-08-17T17:18:18.515 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.515 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.515 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.515 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647595+0000", 2024-08-17T17:18:18.515 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3648000000000001e-05 2024-08-17T17:18:18.515 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.515 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.515 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.515 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.515 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.515 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160968 6.1 6:8a3118ae:::benchmark_data_smithi063_160268_object34861:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:18:18.516 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.664096+0000", 2024-08-17T17:18:18.516 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.82368069699999, 2024-08-17T17:18:18.516 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99085020099999999, 2024-08-17T17:18:18.516 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.516 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.516 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.516 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:18.516 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:18.516 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160968 2024-08-17T17:18:18.516 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.516 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.516 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.516 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.517 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664096+0000", 2024-08-17T17:18:18.517 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.517 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.517 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.517 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.517 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664096+0000", 2024-08-17T17:18:18.517 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.517 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.517 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.517 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.517 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664095+0000", 2024-08-17T17:18:18.517 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:18.517 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.518 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.518 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.518 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664100+0000", 2024-08-17T17:18:18.518 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0880000000000002e-06 2024-08-17T17:18:18.518 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.518 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.518 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.518 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664112+0000", 2024-08-17T17:18:18.518 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1231e-05 2024-08-17T17:18:18.518 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.518 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.518 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.518 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664117+0000", 2024-08-17T17:18:18.519 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4829999999999999e-06 2024-08-17T17:18:18.519 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.519 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.519 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.519 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664174+0000", 2024-08-17T17:18:18.519 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6443999999999997e-05 2024-08-17T17:18:18.519 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.519 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.519 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:18.519 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664186+0000", 2024-08-17T17:18:18.519 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2500999999999999e-05 2024-08-17T17:18:18.519 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.519 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.520 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.520 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.641442+0000", 2024-08-17T17:18:18.520 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97725559900000003 2024-08-17T17:18:18.520 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.520 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.520 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.520 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.641557+0000", 2024-08-17T17:18:18.520 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000115661 2024-08-17T17:18:18.520 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.520 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.520 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:18:18.520 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.641675+0000", 2024-08-17T17:18:18.520 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011779 2024-08-17T17:18:18.520 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.521 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.521 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.521 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647462+0000", 2024-08-17T17:18:18.521 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0057873380000000004 2024-08-17T17:18:18.521 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.521 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.521 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.521 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647507+0000", 2024-08-17T17:18:18.521 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4776e-05 2024-08-17T17:18:18.521 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.521 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.521 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.521 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654882+0000", 2024-08-17T17:18:18.521 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0073748959999999997 2024-08-17T17:18:18.522 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.522 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.522 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.522 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654915+0000", 2024-08-17T17:18:18.522 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3333000000000001e-05 2024-08-17T17:18:18.522 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.522 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.522 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.522 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654947+0000", 2024-08-17T17:18:18.522 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1192999999999998e-05 2024-08-17T17:18:18.523 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.523 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.523 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.523 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.523 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.523 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160969 6.1 6:85b88efb:::benchmark_data_smithi063_160268_object34862:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:18:18.523 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.664210+0000", 2024-08-17T17:18:18.523 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.82356716199999, 2024-08-17T17:18:18.523 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99084335700000004, 2024-08-17T17:18:18.523 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.523 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.523 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.524 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:18.524 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:18.524 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160969 2024-08-17T17:18:18.524 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.524 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.524 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.524 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.524 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664210+0000", 2024-08-17T17:18:18.524 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.524 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.524 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.524 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.524 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664210+0000", 2024-08-17T17:18:18.524 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.525 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.525 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.525 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.525 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664209+0000", 2024-08-17T17:18:18.525 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:18:18.525 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.525 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.525 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.525 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664215+0000", 2024-08-17T17:18:18.525 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8619999999999999e-06 2024-08-17T17:18:18.525 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.525 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.525 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.525 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664227+0000", 2024-08-17T17:18:18.526 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.295e-05 2024-08-17T17:18:18.526 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.526 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.526 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.526 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664234+0000", 2024-08-17T17:18:18.526 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8399999999999997e-06 2024-08-17T17:18:18.526 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.526 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.526 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.526 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664286+0000", 2024-08-17T17:18:18.526 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1928000000000002e-05 2024-08-17T17:18:18.526 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.526 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.527 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:18.527 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664300+0000", 2024-08-17T17:18:18.527 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3497e-05 2024-08-17T17:18:18.527 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.527 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.527 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.527 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644983+0000", 2024-08-17T17:18:18.527 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98068294099999997 2024-08-17T17:18:18.527 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.527 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.527 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.527 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.645050+0000", 2024-08-17T17:18:18.527 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7012000000000001e-05 2024-08-17T17:18:18.528 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.528 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.528 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:18:18.528 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.645116+0000", 2024-08-17T17:18:18.528 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6268999999999999e-05 2024-08-17T17:18:18.528 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.528 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.528 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.528 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654779+0000", 2024-08-17T17:18:18.528 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0096628619999999995 2024-08-17T17:18:18.529 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.529 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.529 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.529 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654813+0000", 2024-08-17T17:18:18.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4489000000000001e-05 2024-08-17T17:18:18.529 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.529 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.529 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.529 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654995+0000", 2024-08-17T17:18:18.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000181478 2024-08-17T17:18:18.529 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.529 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.530 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.530 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655021+0000", 2024-08-17T17:18:18.530 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6390999999999999e-05 2024-08-17T17:18:18.530 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.530 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.530 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.530 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655053+0000", 2024-08-17T17:18:18.530 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2147999999999998e-05 2024-08-17T17:18:18.530 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.530 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.531 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.531 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.531 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.531 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160976 6.1 6:892f13c5:::benchmark_data_smithi063_160268_object34869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:18:18.531 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.670369+0000", 2024-08-17T17:18:18.531 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.81740823300001, 2024-08-17T17:18:18.531 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99685337699999998, 2024-08-17T17:18:18.531 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.531 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.531 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.531 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:18.531 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:18.531 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160976 2024-08-17T17:18:18.531 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.532 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.532 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.532 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670369+0000", 2024-08-17T17:18:18.532 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.532 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.532 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.532 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670369+0000", 2024-08-17T17:18:18.532 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.532 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.532 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.532 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670367+0000", 2024-08-17T17:18:18.533 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:18:18.533 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.533 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.533 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.533 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670374+0000", 2024-08-17T17:18:18.533 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.827e-06 2024-08-17T17:18:18.533 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.533 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.533 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.533 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670390+0000", 2024-08-17T17:18:18.533 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5566e-05 2024-08-17T17:18:18.533 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.534 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.534 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.534 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670398+0000", 2024-08-17T17:18:18.534 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3289999999999994e-06 2024-08-17T17:18:18.534 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.534 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.534 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.534 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670470+0000", 2024-08-17T17:18:18.534 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1913000000000001e-05 2024-08-17T17:18:18.534 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.534 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.535 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:18.535 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670482+0000", 2024-08-17T17:18:18.535 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2018000000000001e-05 2024-08-17T17:18:18.535 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.535 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.535 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.535 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647627+0000", 2024-08-17T17:18:18.535 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.977145139 2024-08-17T17:18:18.535 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.535 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.535 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.535 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647700+0000", 2024-08-17T17:18:18.536 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2495000000000003e-05 2024-08-17T17:18:18.536 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.536 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.536 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:18:18.537 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647765+0000", 2024-08-17T17:18:18.537 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5495999999999998e-05 2024-08-17T17:18:18.537 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.537 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.537 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.537 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654846+0000", 2024-08-17T17:18:18.537 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0070811249999999997 2024-08-17T17:18:18.537 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.537 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.537 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.537 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654853+0000", 2024-08-17T17:18:18.538 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2929999999999997e-06 2024-08-17T17:18:18.538 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.538 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.538 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.538 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667120+0000", 2024-08-17T17:18:18.538 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012267867 2024-08-17T17:18:18.538 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.538 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.538 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.538 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667180+0000", 2024-08-17T17:18:18.539 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9376999999999997e-05 2024-08-17T17:18:18.539 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.539 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.539 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.539 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667222+0000", 2024-08-17T17:18:18.539 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2451999999999999e-05 2024-08-17T17:18:18.539 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.539 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.539 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.539 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.539 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.539 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160983 6.1 6:82d2572f:::benchmark_data_smithi063_160268_object34876:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:18:18.540 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.676198+0000", 2024-08-17T17:18:18.540 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.81157913199999, 2024-08-17T17:18:18.540 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99119904999999997, 2024-08-17T17:18:18.540 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.540 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.540 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.540 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:18.540 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:18.540 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160983 2024-08-17T17:18:18.540 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.540 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.540 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.540 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.541 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676198+0000", 2024-08-17T17:18:18.541 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.541 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.541 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.541 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676198+0000", 2024-08-17T17:18:18.541 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.541 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.541 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.541 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676196+0000", 2024-08-17T17:18:18.541 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:18:18.541 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.542 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.542 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676205+0000", 2024-08-17T17:18:18.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3079999999999997e-06 2024-08-17T17:18:18.542 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.542 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.542 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.542 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676223+0000", 2024-08-17T17:18:18.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7217000000000001e-05 2024-08-17T17:18:18.542 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.542 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.542 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.542 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676230+0000", 2024-08-17T17:18:18.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3479999999999998e-06 2024-08-17T17:18:18.543 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.543 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.543 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.543 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676293+0000", 2024-08-17T17:18:18.543 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2714999999999998e-05 2024-08-17T17:18:18.543 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.543 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.543 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:18.543 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676305+0000", 2024-08-17T17:18:18.543 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2455e-05 2024-08-17T17:18:18.543 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.543 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.543 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.543 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.651451+0000", 2024-08-17T17:18:18.544 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97514584400000004 2024-08-17T17:18:18.544 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.544 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.544 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.544 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.651523+0000", 2024-08-17T17:18:18.544 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2497999999999996e-05 2024-08-17T17:18:18.544 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.544 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.544 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:18:18.544 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.651598+0000", 2024-08-17T17:18:18.544 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4241999999999998e-05 2024-08-17T17:18:18.544 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.544 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.544 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.545 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659221+0000", 2024-08-17T17:18:18.545 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0076229289999999996 2024-08-17T17:18:18.545 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.545 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.545 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.545 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667006+0000", 2024-08-17T17:18:18.545 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00778548 2024-08-17T17:18:18.545 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.545 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.545 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.545 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667303+0000", 2024-08-17T17:18:18.545 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000297103 2024-08-17T17:18:18.545 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.546 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.546 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.546 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667362+0000", 2024-08-17T17:18:18.546 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8705000000000001e-05 2024-08-17T17:18:18.546 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.546 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.546 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.546 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667397+0000", 2024-08-17T17:18:18.546 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5120999999999998e-05 2024-08-17T17:18:18.546 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.546 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.547 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.547 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.547 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.547 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160989 6.1 6:8559d498:::benchmark_data_smithi063_160268_object34882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:18:18.547 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.684830+0000", 2024-08-17T17:18:18.547 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.802947168, 2024-08-17T17:18:18.547 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98270078400000005, 2024-08-17T17:18:18.547 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.547 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.547 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.547 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:18.547 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:18.547 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160989 2024-08-17T17:18:18.548 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.548 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.548 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.548 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.548 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684830+0000", 2024-08-17T17:18:18.548 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.548 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.548 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.548 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.548 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684830+0000", 2024-08-17T17:18:18.548 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.548 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.548 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684828+0000", 2024-08-17T17:18:18.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:18:18.549 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.549 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684835+0000", 2024-08-17T17:18:18.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9890000000000003e-06 2024-08-17T17:18:18.549 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.549 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684851+0000", 2024-08-17T17:18:18.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5457000000000001e-05 2024-08-17T17:18:18.550 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.550 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.550 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.550 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684863+0000", 2024-08-17T17:18:18.550 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2302e-05 2024-08-17T17:18:18.550 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.550 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.550 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.550 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684941+0000", 2024-08-17T17:18:18.550 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7541999999999997e-05 2024-08-17T17:18:18.550 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.551 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.551 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:18.551 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684956+0000", 2024-08-17T17:18:18.551 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5148e-05 2024-08-17T17:18:18.551 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.551 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.551 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.551 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655087+0000", 2024-08-17T17:18:18.551 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97013142799999996 2024-08-17T17:18:18.551 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.551 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.551 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.552 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655187+0000", 2024-08-17T17:18:18.552 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9229999999999997e-05 2024-08-17T17:18:18.552 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.552 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.552 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:18:18.552 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655293+0000", 2024-08-17T17:18:18.552 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000106576 2024-08-17T17:18:18.552 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.552 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.552 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.552 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667235+0000", 2024-08-17T17:18:18.552 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011942026 2024-08-17T17:18:18.552 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.553 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.553 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.553 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667269+0000", 2024-08-17T17:18:18.553 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.417e-05 2024-08-17T17:18:18.553 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.553 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.553 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.553 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667449+0000", 2024-08-17T17:18:18.553 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000179292 2024-08-17T17:18:18.553 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.553 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.553 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.553 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667498+0000", 2024-08-17T17:18:18.553 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9821999999999998e-05 2024-08-17T17:18:18.554 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.554 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.554 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.554 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667531+0000", 2024-08-17T17:18:18.554 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2326999999999997e-05 2024-08-17T17:18:18.554 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.554 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.554 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.554 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.554 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.554 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160996 6.1 6:8ccbb0f4:::benchmark_data_smithi063_160268_object34889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:18:18.554 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.691144+0000", 2024-08-17T17:18:18.554 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.796632698, 2024-08-17T17:18:18.555 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98531105200000002, 2024-08-17T17:18:18.555 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.555 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.555 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.555 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:18.555 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:18.555 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160996 2024-08-17T17:18:18.555 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.555 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.555 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.555 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.555 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691144+0000", 2024-08-17T17:18:18.555 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.555 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.556 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.556 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691144+0000", 2024-08-17T17:18:18.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.556 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.556 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.556 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691143+0000", 2024-08-17T17:18:18.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:18:18.556 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.556 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.556 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691149+0000", 2024-08-17T17:18:18.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0800000000000002e-06 2024-08-17T17:18:18.557 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.557 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691162+0000", 2024-08-17T17:18:18.557 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.322e-05 2024-08-17T17:18:18.557 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.557 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691168+0000", 2024-08-17T17:18:18.557 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0260000000000004e-06 2024-08-17T17:18:18.557 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.558 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.558 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.558 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691211+0000", 2024-08-17T17:18:18.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3013999999999998e-05 2024-08-17T17:18:18.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.558 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.558 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:18.558 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691220+0000", 2024-08-17T17:18:18.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8540000000000003e-06 2024-08-17T17:18:18.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.558 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.558 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659248+0000", 2024-08-17T17:18:18.559 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.96802809099999998 2024-08-17T17:18:18.559 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.559 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.559 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659357+0000", 2024-08-17T17:18:18.559 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010854 2024-08-17T17:18:18.559 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.559 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.559 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:18:18.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659466+0000", 2024-08-17T17:18:18.559 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000109437 2024-08-17T17:18:18.560 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.560 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.560 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.560 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667552+0000", 2024-08-17T17:18:18.560 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0080855279999999998 2024-08-17T17:18:18.560 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.560 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.560 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.560 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676310+0000", 2024-08-17T17:18:18.560 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0087583250000000008 2024-08-17T17:18:18.560 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.560 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.560 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.561 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676357+0000", 2024-08-17T17:18:18.561 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6786000000000002e-05 2024-08-17T17:18:18.561 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.561 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.561 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.561 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676409+0000", 2024-08-17T17:18:18.561 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2104000000000001e-05 2024-08-17T17:18:18.561 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.561 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.561 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.561 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676455+0000", 2024-08-17T17:18:18.561 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6362000000000001e-05 2024-08-17T17:18:18.562 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.562 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.562 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.562 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.562 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.562 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:161006 6.1 6:8d7d2cbb:::benchmark_data_smithi063_160268_object34899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:18:18.562 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.709227+0000", 2024-08-17T17:18:18.562 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.77854994699999, 2024-08-17T17:18:18.562 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97342043, 2024-08-17T17:18:18.562 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.562 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.562 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.563 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:18.563 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:18.563 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 161006 2024-08-17T17:18:18.563 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.563 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709227+0000", 2024-08-17T17:18:18.563 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.563 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.564 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709227+0000", 2024-08-17T17:18:18.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.564 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.564 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709226+0000", 2024-08-17T17:18:18.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:18.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.564 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.564 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709232+0000", 2024-08-17T17:18:18.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6219999999999997e-06 2024-08-17T17:18:18.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709243+0000", 2024-08-17T17:18:18.565 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0839000000000001e-05 2024-08-17T17:18:18.565 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709248+0000", 2024-08-17T17:18:18.565 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.322e-06 2024-08-17T17:18:18.565 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709293+0000", 2024-08-17T17:18:18.566 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4672000000000002e-05 2024-08-17T17:18:18.566 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.566 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.566 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:18.566 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709302+0000", 2024-08-17T17:18:18.566 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2240000000000001e-06 2024-08-17T17:18:18.566 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.566 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.566 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.566 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.663451+0000", 2024-08-17T17:18:18.566 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95414951299999995 2024-08-17T17:18:18.566 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.566 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.567 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.567 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.663549+0000", 2024-08-17T17:18:18.567 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7268000000000005e-05 2024-08-17T17:18:18.567 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.567 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.567 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:18:18.567 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.663655+0000", 2024-08-17T17:18:18.567 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000106872 2024-08-17T17:18:18.567 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.567 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.567 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.567 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667559+0000", 2024-08-17T17:18:18.568 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0039036629999999999 2024-08-17T17:18:18.568 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.568 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.568 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.568 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.678421+0000", 2024-08-17T17:18:18.568 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010861585 2024-08-17T17:18:18.568 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.568 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.568 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.568 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.682563+0000", 2024-08-17T17:18:18.568 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0041419159999999998 2024-08-17T17:18:18.568 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.568 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.569 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.569 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.682611+0000", 2024-08-17T17:18:18.569 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8585000000000003e-05 2024-08-17T17:18:18.569 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.569 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.569 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.569 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.682648+0000", 2024-08-17T17:18:18.569 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6449999999999998e-05 2024-08-17T17:18:18.569 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.569 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.569 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.569 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.570 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.570 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:161016 6.1 6:83fef0fc:::benchmark_data_smithi063_160268_object34909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:18:18.570 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.731599+0000", 2024-08-17T17:18:18.570 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.756178581, 2024-08-17T17:18:18.570 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95788104200000002, 2024-08-17T17:18:18.570 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.570 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.570 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.570 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:18.570 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:18.570 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 161016 2024-08-17T17:18:18.570 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.570 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.571 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.571 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.571 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731599+0000", 2024-08-17T17:18:18.571 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.571 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.571 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.571 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.571 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731599+0000", 2024-08-17T17:18:18.571 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.571 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.571 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.571 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.571 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731597+0000", 2024-08-17T17:18:18.571 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:18:18.572 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.572 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.572 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.572 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731604+0000", 2024-08-17T17:18:18.572 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3790000000000003e-06 2024-08-17T17:18:18.572 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.572 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.572 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.572 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731616+0000", 2024-08-17T17:18:18.572 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.261e-05 2024-08-17T17:18:18.572 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.572 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.572 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.573 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731622+0000", 2024-08-17T17:18:18.573 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1809999999999998e-06 2024-08-17T17:18:18.573 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.573 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.573 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.573 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731664+0000", 2024-08-17T17:18:18.573 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1355e-05 2024-08-17T17:18:18.573 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.573 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.573 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:18.573 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.731673+0000", 2024-08-17T17:18:18.573 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.704e-06 2024-08-17T17:18:18.573 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.574 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.574 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.574 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667670+0000", 2024-08-17T17:18:18.574 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.93599733900000004 2024-08-17T17:18:18.574 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.574 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.574 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.574 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667779+0000", 2024-08-17T17:18:18.574 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000109268 2024-08-17T17:18:18.574 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.574 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.574 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:18:18.574 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667926+0000", 2024-08-17T17:18:18.575 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014687899999999999 2024-08-17T17:18:18.575 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.575 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.575 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.575 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676467+0000", 2024-08-17T17:18:18.575 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0085410450000000006 2024-08-17T17:18:18.575 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.575 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.575 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.575 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.678606+0000", 2024-08-17T17:18:18.575 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021387590000000001 2024-08-17T17:18:18.575 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.576 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.576 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.576 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.689357+0000", 2024-08-17T17:18:18.576 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010751024 2024-08-17T17:18:18.576 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.576 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.576 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.576 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.689441+0000", 2024-08-17T17:18:18.576 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4072000000000004e-05 2024-08-17T17:18:18.576 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.576 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.576 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.576 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.689480+0000", 2024-08-17T17:18:18.576 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8667000000000001e-05 2024-08-17T17:18:18.577 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.577 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.577 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.577 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.577 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.577 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:30163 7.c 7:32acba98:::benchmark_data_smithi063_196002_object30162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:18:18.577 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:09.180499+0000", 2024-08-17T17:18:18.577 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 9.3072784469999998, 2024-08-17T17:18:18.577 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.091236245999999993, 2024-08-17T17:18:18.577 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.577 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.577 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.577 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:18:18.578 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:18:18.578 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 30163 2024-08-17T17:18:18.578 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.578 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.578 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.578 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.578 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.180499+0000", 2024-08-17T17:18:18.578 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.578 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.578 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.578 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.578 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.180499+0000", 2024-08-17T17:18:18.579 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.579 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.579 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.579 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.579 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.180497+0000", 2024-08-17T17:18:18.579 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:18:18.579 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.579 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.579 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.579 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.180540+0000", 2024-08-17T17:18:18.579 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3231e-05 2024-08-17T17:18:18.579 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.579 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.579 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.580 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.180558+0000", 2024-08-17T17:18:18.580 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7498000000000001e-05 2024-08-17T17:18:18.580 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.580 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.580 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.580 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.180565+0000", 2024-08-17T17:18:18.580 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8059999999999997e-06 2024-08-17T17:18:18.580 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.580 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.580 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.580 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.180600+0000", 2024-08-17T17:18:18.580 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4910000000000003e-05 2024-08-17T17:18:18.580 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.580 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.581 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.581 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.180710+0000", 2024-08-17T17:18:18.581 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000110562 2024-08-17T17:18:18.581 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.581 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.581 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:18:18.581 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.180833+0000", 2024-08-17T17:18:18.582 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012306799999999999 2024-08-17T17:18:18.582 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.582 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.582 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.582 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.182660+0000", 2024-08-17T17:18:18.582 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0018263839999999999 2024-08-17T17:18:18.582 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.582 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.583 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.583 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.183959+0000", 2024-08-17T17:18:18.583 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012989340000000001 2024-08-17T17:18:18.583 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.583 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.583 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.583 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.271584+0000", 2024-08-17T17:18:18.583 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.087625136000000006 2024-08-17T17:18:18.583 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.583 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.583 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.583 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.271668+0000", 2024-08-17T17:18:18.583 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4559000000000001e-05 2024-08-17T17:18:18.583 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.584 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.584 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.584 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.271735+0000", 2024-08-17T17:18:18.584 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6709000000000001e-05 2024-08-17T17:18:18.584 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.584 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.584 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.584 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.584 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.584 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:30176 7.c 7:3816dd24:::benchmark_data_smithi063_196002_object30175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:18:18.584 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:09.185631+0000", 2024-08-17T17:18:18.584 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 9.3021457430000005, 2024-08-17T17:18:18.584 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.089843302, 2024-08-17T17:18:18.584 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.585 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.585 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.585 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:18:18.585 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:18:18.585 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 30176 2024-08-17T17:18:18.585 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.585 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.585 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.585 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.585 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.185631+0000", 2024-08-17T17:18:18.585 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.585 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.585 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.585 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.586 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.185631+0000", 2024-08-17T17:18:18.586 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.586 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.586 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.586 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.586 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.185630+0000", 2024-08-17T17:18:18.586 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:18:18.586 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.586 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.586 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.586 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.185672+0000", 2024-08-17T17:18:18.586 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2509999999999998e-05 2024-08-17T17:18:18.587 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.587 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.587 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.587 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.185688+0000", 2024-08-17T17:18:18.587 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5546999999999999e-05 2024-08-17T17:18:18.587 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.587 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.587 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.587 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.185695+0000", 2024-08-17T17:18:18.587 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8310000000000003e-06 2024-08-17T17:18:18.587 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.587 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.587 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.588 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.185914+0000", 2024-08-17T17:18:18.588 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000219521 2024-08-17T17:18:18.588 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.588 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.588 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.588 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.186041+0000", 2024-08-17T17:18:18.588 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000127136 2024-08-17T17:18:18.588 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.588 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.588 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:18:18.588 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.186143+0000", 2024-08-17T17:18:18.588 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000101339 2024-08-17T17:18:18.588 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.588 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.589 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.589 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.187432+0000", 2024-08-17T17:18:18.589 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001289721 2024-08-17T17:18:18.589 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.589 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.589 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.589 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.187668+0000", 2024-08-17T17:18:18.589 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00023550900000000001 2024-08-17T17:18:18.589 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.589 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.589 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.589 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.275371+0000", 2024-08-17T17:18:18.589 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.087703015999999995 2024-08-17T17:18:18.590 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.590 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.590 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.590 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.275424+0000", 2024-08-17T17:18:18.590 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3115000000000002e-05 2024-08-17T17:18:18.590 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.590 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.590 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.590 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:09.275475+0000", 2024-08-17T17:18:18.590 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0581000000000002e-05 2024-08-17T17:18:18.590 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.590 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.590 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.591 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.591 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.591 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:32526 7.d 7:bb337b93:::benchmark_data_smithi063_196002_object32525:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:18:18.591 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:10.419552+0000", 2024-08-17T17:18:18.591 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 8.0682254990000004, 2024-08-17T17:18:18.591 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14201709200000001, 2024-08-17T17:18:18.591 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.591 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.591 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.591 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:18:18.591 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:18:18.591 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 32526 2024-08-17T17:18:18.592 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.592 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.592 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.592 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.592 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.419552+0000", 2024-08-17T17:18:18.592 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.592 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.592 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.592 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.592 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.419552+0000", 2024-08-17T17:18:18.592 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.592 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.592 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.592 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.593 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.419550+0000", 2024-08-17T17:18:18.593 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:18.593 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.593 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.593 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.593 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.419583+0000", 2024-08-17T17:18:18.593 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2833000000000002e-05 2024-08-17T17:18:18.593 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.593 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.593 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.593 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.419592+0000", 2024-08-17T17:18:18.594 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6160000000000002e-06 2024-08-17T17:18:18.594 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.594 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.594 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.594 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.419598+0000", 2024-08-17T17:18:18.594 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1299999999999998e-06 2024-08-17T17:18:18.594 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.594 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.594 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.594 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.419634+0000", 2024-08-17T17:18:18.594 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6365000000000002e-05 2024-08-17T17:18:18.594 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.595 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.595 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.595 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.419759+0000", 2024-08-17T17:18:18.595 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000124225 2024-08-17T17:18:18.595 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.595 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.595 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:18:18.595 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.419901+0000", 2024-08-17T17:18:18.595 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014281400000000001 2024-08-17T17:18:18.595 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.595 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.595 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.596 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.421900+0000", 2024-08-17T17:18:18.596 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0019983290000000001 2024-08-17T17:18:18.596 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.596 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.596 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.596 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.422989+0000", 2024-08-17T17:18:18.596 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0010892180000000001 2024-08-17T17:18:18.596 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.596 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.596 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.596 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.561500+0000", 2024-08-17T17:18:18.596 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.13851095399999999 2024-08-17T17:18:18.597 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.597 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.597 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.597 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.561544+0000", 2024-08-17T17:18:18.597 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3677000000000002e-05 2024-08-17T17:18:18.597 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.597 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.597 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.597 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.561569+0000", 2024-08-17T17:18:18.597 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5065000000000001e-05 2024-08-17T17:18:18.597 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.597 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.598 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.598 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.598 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.598 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:32530 7.c 7:3efd6e9c:::benchmark_data_smithi063_196002_object32529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:18:18.598 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:10.420037+0000", 2024-08-17T17:18:18.598 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 8.0677397899999992, 2024-08-17T17:18:18.598 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14441789599999999, 2024-08-17T17:18:18.598 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.598 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.598 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.598 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:18:18.599 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:18:18.599 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 32530 2024-08-17T17:18:18.599 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.599 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.599 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.599 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.599 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.420037+0000", 2024-08-17T17:18:18.599 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.599 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.599 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.599 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.600 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.420037+0000", 2024-08-17T17:18:18.600 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.600 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.600 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.600 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.600 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.420036+0000", 2024-08-17T17:18:18.600 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:18.600 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.600 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.600 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.600 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.420067+0000", 2024-08-17T17:18:18.600 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.0408000000000001e-05 2024-08-17T17:18:18.601 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.601 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.601 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.601 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.420079+0000", 2024-08-17T17:18:18.601 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2516e-05 2024-08-17T17:18:18.601 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.601 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.601 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.601 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.420086+0000", 2024-08-17T17:18:18.601 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3010000000000003e-06 2024-08-17T17:18:18.601 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.601 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.602 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.602 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.420152+0000", 2024-08-17T17:18:18.602 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6143999999999996e-05 2024-08-17T17:18:18.602 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.602 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.602 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.602 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.420262+0000", 2024-08-17T17:18:18.602 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000110388 2024-08-17T17:18:18.602 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.602 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.602 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:18:18.603 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.420387+0000", 2024-08-17T17:18:18.603 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012474599999999999 2024-08-17T17:18:18.603 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.603 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.603 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.603 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.421943+0000", 2024-08-17T17:18:18.603 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00155583 2024-08-17T17:18:18.603 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.603 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.603 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.603 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.423812+0000", 2024-08-17T17:18:18.603 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0018687070000000001 2024-08-17T17:18:18.604 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.604 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.604 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.604 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.564402+0000", 2024-08-17T17:18:18.604 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14059006299999999 2024-08-17T17:18:18.604 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.604 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.604 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.604 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.564434+0000", 2024-08-17T17:18:18.604 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2088000000000002e-05 2024-08-17T17:18:18.604 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.604 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.605 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.605 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.564455+0000", 2024-08-17T17:18:18.605 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.1582000000000001e-05 2024-08-17T17:18:18.605 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.605 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.605 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.605 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.605 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.605 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:32539 7.7 7:efac7d46:::benchmark_data_smithi063_196002_object32538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:18:18.605 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:10.425723+0000", 2024-08-17T17:18:18.605 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 8.0620536779999998, 2024-08-17T17:18:18.606 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14224500800000001, 2024-08-17T17:18:18.606 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.606 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.606 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.606 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:18:18.606 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:18:18.606 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 32539 2024-08-17T17:18:18.606 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.606 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.606 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.606 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.606 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.425723+0000", 2024-08-17T17:18:18.607 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.607 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.607 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.607 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.607 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.425723+0000", 2024-08-17T17:18:18.607 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.607 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.607 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.607 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.607 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.425721+0000", 2024-08-17T17:18:18.607 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:18:18.607 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.607 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.608 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.608 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.425782+0000", 2024-08-17T17:18:18.608 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0364e-05 2024-08-17T17:18:18.608 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.608 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.608 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.608 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.425799+0000", 2024-08-17T17:18:18.608 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7278999999999998e-05 2024-08-17T17:18:18.608 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.608 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.608 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.608 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.425807+0000", 2024-08-17T17:18:18.609 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9640000000000003e-06 2024-08-17T17:18:18.609 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.609 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.609 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.609 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.425887+0000", 2024-08-17T17:18:18.609 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0098999999999993e-05 2024-08-17T17:18:18.609 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.609 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.609 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.609 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.426159+0000", 2024-08-17T17:18:18.609 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000271339 2024-08-17T17:18:18.609 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.609 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.609 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:18:18.610 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.426304+0000", 2024-08-17T17:18:18.610 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014562200000000001 2024-08-17T17:18:18.610 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.610 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.610 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.610 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.427458+0000", 2024-08-17T17:18:18.610 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001153793 2024-08-17T17:18:18.610 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.610 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.610 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.610 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.427533+0000", 2024-08-17T17:18:18.610 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4605999999999997e-05 2024-08-17T17:18:18.610 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.610 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.611 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.611 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.567889+0000", 2024-08-17T17:18:18.611 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14035682999999999 2024-08-17T17:18:18.611 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.611 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.611 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.611 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.567934+0000", 2024-08-17T17:18:18.611 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4880999999999999e-05 2024-08-17T17:18:18.611 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.611 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.611 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.611 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.567968+0000", 2024-08-17T17:18:18.611 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4201e-05 2024-08-17T17:18:18.611 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.612 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.612 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.612 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.612 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.612 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:32952 7.c 7:3d9d4cb5:::benchmark_data_smithi063_196002_object32951:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:18:18.612 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:10.733209+0000", 2024-08-17T17:18:18.612 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 7.7545683219999999, 2024-08-17T17:18:18.612 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.096013609999999999, 2024-08-17T17:18:18.612 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.612 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.612 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.612 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:18:18.612 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:18:18.612 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 32952 2024-08-17T17:18:18.613 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.613 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.613 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.613 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.613 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.733209+0000", 2024-08-17T17:18:18.613 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.613 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.613 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.613 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.613 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.733209+0000", 2024-08-17T17:18:18.613 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.613 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.613 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.614 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.614 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.733208+0000", 2024-08-17T17:18:18.614 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:18.614 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.614 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.614 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.614 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.733234+0000", 2024-08-17T17:18:18.614 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6205999999999999e-05 2024-08-17T17:18:18.614 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.614 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.614 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.614 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.733244+0000", 2024-08-17T17:18:18.614 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7419999999999992e-06 2024-08-17T17:18:18.614 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.615 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.615 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.615 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.733249+0000", 2024-08-17T17:18:18.615 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8590000000000004e-06 2024-08-17T17:18:18.615 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.615 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.615 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.615 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.733286+0000", 2024-08-17T17:18:18.615 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.697e-05 2024-08-17T17:18:18.615 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.615 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.615 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.615 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.733358+0000", 2024-08-17T17:18:18.615 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2428000000000005e-05 2024-08-17T17:18:18.616 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.616 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.616 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:18:18.616 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.733435+0000", 2024-08-17T17:18:18.616 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7268999999999995e-05 2024-08-17T17:18:18.616 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.616 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.616 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.616 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.734761+0000", 2024-08-17T17:18:18.616 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013254250000000001 2024-08-17T17:18:18.616 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.616 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.616 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.616 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.734885+0000", 2024-08-17T17:18:18.616 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012376499999999999 2024-08-17T17:18:18.617 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.617 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.617 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.617 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.829146+0000", 2024-08-17T17:18:18.617 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.094261205000000001 2024-08-17T17:18:18.617 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.617 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.617 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.617 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.829189+0000", 2024-08-17T17:18:18.617 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3501000000000002e-05 2024-08-17T17:18:18.617 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.617 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.617 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.617 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.829222+0000", 2024-08-17T17:18:18.618 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3160000000000001e-05 2024-08-17T17:18:18.618 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.618 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.618 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.618 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.618 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.618 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:34081 7.d 7:beec9c09:::benchmark_data_smithi063_196002_object34080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:18:18.618 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:11.470195+0000", 2024-08-17T17:18:18.618 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 7.0175817179999997, 2024-08-17T17:18:18.618 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.16400772899999999, 2024-08-17T17:18:18.618 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.618 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.619 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.619 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:18:18.619 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:18:18.619 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 34081 2024-08-17T17:18:18.619 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.619 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.619 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.619 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.619 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.470195+0000", 2024-08-17T17:18:18.619 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.619 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.619 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.619 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.619 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.470195+0000", 2024-08-17T17:18:18.620 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.620 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.620 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.620 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.620 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.470194+0000", 2024-08-17T17:18:18.620 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:18:18.620 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.620 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.620 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.620 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.470224+0000", 2024-08-17T17:18:18.620 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.0278999999999999e-05 2024-08-17T17:18:18.620 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.620 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.620 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.621 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.470238+0000", 2024-08-17T17:18:18.621 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3771000000000001e-05 2024-08-17T17:18:18.621 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.621 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.621 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.621 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.470247+0000", 2024-08-17T17:18:18.621 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6899999999999998e-06 2024-08-17T17:18:18.621 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.621 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.621 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.621 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.470307+0000", 2024-08-17T17:18:18.621 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0451000000000001e-05 2024-08-17T17:18:18.621 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.621 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.622 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.622 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.470429+0000", 2024-08-17T17:18:18.622 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000121495 2024-08-17T17:18:18.622 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.622 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.622 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:18:18.622 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.470568+0000", 2024-08-17T17:18:18.622 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013876399999999999 2024-08-17T17:18:18.622 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.622 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.622 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.622 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471778+0000", 2024-08-17T17:18:18.622 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012102510000000001 2024-08-17T17:18:18.623 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.623 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.623 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.623 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.472433+0000", 2024-08-17T17:18:18.623 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00065568699999999996 2024-08-17T17:18:18.623 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.623 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.623 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.623 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.634107+0000", 2024-08-17T17:18:18.623 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.161673504 2024-08-17T17:18:18.623 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.623 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.623 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.623 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.634169+0000", 2024-08-17T17:18:18.624 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1604000000000002e-05 2024-08-17T17:18:18.624 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.624 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.624 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.624 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.634203+0000", 2024-08-17T17:18:18.624 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4564000000000003e-05 2024-08-17T17:18:18.624 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.624 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.624 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.624 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.624 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.624 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:37752 7.7 7:efd71e6a:::benchmark_data_smithi063_196002_object37751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:18:18.624 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:13.426903+0000", 2024-08-17T17:18:18.624 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 5.0608746050000004, 2024-08-17T17:18:18.625 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.074518111999999997, 2024-08-17T17:18:18.625 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:18.625 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:18.625 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:18.625 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:18:18.625 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:18:18.625 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 37752 2024-08-17T17:18:18.625 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.625 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:18.625 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.625 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:18.625 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:13.426903+0000", 2024-08-17T17:18:18.625 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.625 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.626 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.626 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:18.626 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:13.426903+0000", 2024-08-17T17:18:18.626 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:18.626 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.626 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.626 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:18.626 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:13.426902+0000", 2024-08-17T17:18:18.626 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:18.626 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.626 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.626 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:18.626 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:13.426958+0000", 2024-08-17T17:18:18.627 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6552000000000003e-05 2024-08-17T17:18:18.627 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.627 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.627 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:18.627 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:13.426969+0000", 2024-08-17T17:18:18.627 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1011e-05 2024-08-17T17:18:18.627 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.627 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.627 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:18.627 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:13.426976+0000", 2024-08-17T17:18:18.627 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7159999999999997e-06 2024-08-17T17:18:18.627 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.628 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.628 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:18.628 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:13.427005+0000", 2024-08-17T17:18:18.628 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9311000000000001e-05 2024-08-17T17:18:18.628 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.628 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.628 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:18.628 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:13.427079+0000", 2024-08-17T17:18:18.628 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3565999999999993e-05 2024-08-17T17:18:18.628 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.628 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.628 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:18:18.628 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:13.427172+0000", 2024-08-17T17:18:18.628 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3397e-05 2024-08-17T17:18:18.629 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.629 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.629 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:18.629 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:13.428352+0000", 2024-08-17T17:18:18.629 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001180068 2024-08-17T17:18:18.629 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.629 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.629 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.629 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:13.431118+0000", 2024-08-17T17:18:18.629 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002765539 2024-08-17T17:18:18.629 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.629 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.629 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:18.629 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:13.501282+0000", 2024-08-17T17:18:18.630 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.070164700999999996 2024-08-17T17:18:18.630 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.630 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.630 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:18.630 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:13.501328+0000", 2024-08-17T17:18:18.630 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5799e-05 2024-08-17T17:18:18.630 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:18.630 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:18.630 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:18.630 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:13.501421+0000", 2024-08-17T17:18:18.630 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2347000000000002e-05 2024-08-17T17:18:18.630 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.630 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.631 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.631 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:18.631 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:18.631 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:18:18.634 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:18.671 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:18.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:18.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:18.774 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:18.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:18 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3606909788' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-08-17T17:18:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:18 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3606909788' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-08-17T17:18:18.813 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:18.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:18.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:18.915 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:18.954 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:18.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:18.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:19.056 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:19.096 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:19.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:19.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:19.178 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:18:19.198 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:19.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:19.262 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:19.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:19.363 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:19.405 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:19.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:19.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:19.438 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 21 16 48257 48241 143.548 107.688 0.00685064 0.00696049 2024-08-17T17:18:19.507 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:19.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:19.543 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:19.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:19.643 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:19.678 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:19.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:19.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:19.719 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:19 smithi063 ceph-mon[109843]: pgmap v1326: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 117 MiB/s wr, 1.88k op/s 2024-08-17T17:18:19.719 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:19 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3606909788' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-08-17T17:18:19.720 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:19 smithi063 ceph-mon[109843]: osdmap e1135: 8 total, 8 up, 7 in 2024-08-17T17:18:19.720 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:19 smithi063 ceph-mon[114756]: pgmap v1326: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 117 MiB/s wr, 1.88k op/s 2024-08-17T17:18:19.720 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:19 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3606909788' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-08-17T17:18:19.720 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:19 smithi063 ceph-mon[114756]: osdmap e1135: 8 total, 8 up, 7 in 2024-08-17T17:18:19.779 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:19.811 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:19.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:19.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:19 smithi162 ceph-mon[104668]: pgmap v1326: 89 pgs: 89 active+clean; 2.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 117 MiB/s wr, 1.88k op/s 2024-08-17T17:18:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:19 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3606909788' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-08-17T17:18:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:19 smithi162 ceph-mon[104668]: osdmap e1135: 8 total, 8 up, 7 in 2024-08-17T17:18:19.913 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:19.955 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:19.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:19.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:20.056 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:20.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:20.097 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:20.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:20.198 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:20.230 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:20.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:20.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:20.332 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:20.367 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:20.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:20.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:20.438 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 22 16 49950 49934 141.832 105.812 0.00612016 0.00704501 2024-08-17T17:18:20.440 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:18:20.469 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:20.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:20.511 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:20.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:20.612 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:20.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:20.648 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:20.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:20.660 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:20 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-08-17T17:18:20.660 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:20 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-08-17T17:18:20.660 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:20 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-08-17T17:18:20.660 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:20 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-08-17T17:18:20.749 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:20.775 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:20.783 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:20.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:20.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:20.790 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:18:20.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:20 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-08-17T17:18:20.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:20 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-08-17T17:18:20.850 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:18:20.850 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:18:20.885 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:20.890 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:18:20.890 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:18:20.927 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:18:20.928 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:18:20.931 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:20.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:20.933 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:20.964 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:18:20.964 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:18:20.997 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:18:20.997 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:18:21.029 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:18:21.029 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:18:21.033 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:21.061 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:18:21.061 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:18:21.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:21.078 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:21.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:21.098 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:18:21.098 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:18:21.179 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:21.218 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:21.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:21.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:21.320 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:21.342 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:21.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:21.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:21.438 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 23 16 51561 51545 140.043 100.688 0.0045175 0.00713542 2024-08-17T17:18:21.443 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:21.483 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:21.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:21.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:21.585 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:21.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:21 smithi162 ceph-mon[104668]: pgmap v1328: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-08-17T17:18:21.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:21 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-08-17T17:18:21.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:21 smithi162 ceph-mon[104668]: osdmap e1136: 8 total, 8 up, 7 in 2024-08-17T17:18:21.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:21 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3597641260' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:21.623 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:21.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:21.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:21.725 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:21.764 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:21.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:21.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:21 smithi063 ceph-mon[114756]: pgmap v1328: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-08-17T17:18:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:21 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-08-17T17:18:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:21 smithi063 ceph-mon[114756]: osdmap e1136: 8 total, 8 up, 7 in 2024-08-17T17:18:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:21 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3597641260' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:21 smithi063 ceph-mon[109843]: pgmap v1328: 89 pgs: 89 active+clean; 2.8 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 106 MiB/s wr, 1.69k op/s 2024-08-17T17:18:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:21 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-08-17T17:18:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:21 smithi063 ceph-mon[109843]: osdmap e1136: 8 total, 8 up, 7 in 2024-08-17T17:18:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:21 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3597641260' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:21.866 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:21.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:21.901 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:21.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:22.002 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:22.041 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:22.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:22.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:22.143 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:22.182 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:22.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:22.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:22.284 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:22.312 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:22.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:22.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:22.413 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:22.435 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:22.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:22.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:22.438 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 24 16 53107 53091 138.234 96.625 0.00230412 0.00720583 2024-08-17T17:18:22.536 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:22.573 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:22.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:22.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:22.675 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:22.707 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:22.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:22.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:22.747 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:22 smithi063 ceph-mon[109843]: osdmap e1137: 8 total, 8 up, 7 in 2024-08-17T17:18:22.747 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:22 smithi063 ceph-mon[114756]: osdmap e1137: 8 total, 8 up, 7 in 2024-08-17T17:18:22.808 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:22.845 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:22.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:22.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:22 smithi162 ceph-mon[104668]: osdmap e1137: 8 total, 8 up, 7 in 2024-08-17T17:18:22.947 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:22.979 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:22.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:22.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:23.081 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:23.120 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:23.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:23.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:23.222 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:23.261 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:23.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:23.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:23.363 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:23.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:23.407 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:23.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:23.438 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 25 16 54858 54842 137.082 109.438 0.00843587 0.007289 2024-08-17T17:18:23.461 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:18:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:18:23] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.51.0" 2024-08-17T17:18:23.509 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:23.557 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:23.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:23.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:23.575 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:23 smithi162 ceph-mon[104668]: pgmap v1331: 99 pgs: 10 unknown, 89 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-08-17T17:18:23.659 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:23.696 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:23.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:23.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:23.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:23 smithi063 ceph-mon[114756]: pgmap v1331: 99 pgs: 10 unknown, 89 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-08-17T17:18:23.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:23 smithi063 ceph-mon[109843]: pgmap v1331: 99 pgs: 10 unknown, 89 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-08-17T17:18:23.798 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:23.839 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:23.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:23.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:23.894 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:18:23.894 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:18:23.941 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:23.981 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:23.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:23.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:24.083 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:24.120 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:24.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:24.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:24.222 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:24.264 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:24.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:24.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:24.366 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:24.400 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:24.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:24.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:24.438 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 26 16 56512 56496 135.785 103.375 0.00687206 0.00735927 2024-08-17T17:18:24.475 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:18:24.502 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:24.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:24.545 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:24.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:24.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:24 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-08-17T17:18:24.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:24 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-08-17T17:18:24.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:24 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3987856187' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:18:24.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:24 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:18:24.646 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:24.685 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:24.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:24.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:24.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:24 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-08-17T17:18:24.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:24 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-08-17T17:18:24.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:24 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3987856187' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:18:24.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:24 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:18:24.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:24 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-08-17T17:18:24.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:24 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-08-17T17:18:24.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:24 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3987856187' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:18:24.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:24 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:18:24.786 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:24.787 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:18:24.819 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:24.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:24.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:24.921 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:24.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:24.964 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:24.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:25.065 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:25.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:25.107 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:25.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:25.208 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:25.234 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:18:25.234 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:18:25.234 INFO:tasks.thrashosds.thrasher:Adding osd 0 2024-08-17T17:18:25.234 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd in 0 2024-08-17T17:18:25.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:25.244 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:25.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:25.345 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:25.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:25.390 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:25.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:25.391 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:18:25.438 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 27 16 58078 58062 134.381 97.875 0.0182074 0.00743606 2024-08-17T17:18:25.492 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:25.523 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:25.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:25.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:25.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:25 smithi063 ceph-mon[114756]: pgmap v1332: 99 pgs: 99 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-08-17T17:18:25.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:25 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:25.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:25 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-08-17T17:18:25.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:25 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:18:25.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:25 smithi063 ceph-mon[114756]: osdmap e1138: 8 total, 8 up, 7 in 2024-08-17T17:18:25.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:25 smithi063 ceph-mon[114756]: 1.0 scrub starts 2024-08-17T17:18:25.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:25 smithi063 ceph-mon[114756]: 1.0 scrub ok 2024-08-17T17:18:25.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:25 smithi063 ceph-mon[109843]: pgmap v1332: 99 pgs: 99 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-08-17T17:18:25.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:25 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:25.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:25 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-08-17T17:18:25.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:25 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:18:25.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:25 smithi063 ceph-mon[109843]: osdmap e1138: 8 total, 8 up, 7 in 2024-08-17T17:18:25.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:25 smithi063 ceph-mon[109843]: 1.0 scrub starts 2024-08-17T17:18:25.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:25 smithi063 ceph-mon[109843]: 1.0 scrub ok 2024-08-17T17:18:25.624 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:25.626 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:25.638 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:25.663 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:25.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:25.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:25.750 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:18:25.765 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:25.807 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:25.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:25.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:25 smithi162 ceph-mon[104668]: pgmap v1332: 99 pgs: 99 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-08-17T17:18:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:25 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:25 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-08-17T17:18:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:25 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:18:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:25 smithi162 ceph-mon[104668]: osdmap e1138: 8 total, 8 up, 7 in 2024-08-17T17:18:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:25 smithi162 ceph-mon[104668]: 1.0 scrub starts 2024-08-17T17:18:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:25 smithi162 ceph-mon[104668]: 1.0 scrub ok 2024-08-17T17:18:25.908 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:25.938 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:25.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:25.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:26.040 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:26.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:26.085 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:26.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:26.186 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:26.227 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:26.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:26.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:26.328 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:26.360 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:26.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:26.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:26.438 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 28 16 59588 59572 132.952 94.375 0.00566955 0.00751599 2024-08-17T17:18:26.462 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:26.499 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:26.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:26.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:26.601 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:26.639 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:26.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:26.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:26.741 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:26.759 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:26 smithi063 ceph-mon[114756]: osdmap e1139: 8 total, 8 up, 7 in 2024-08-17T17:18:26.759 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:26 smithi063 ceph-mon[114756]: osdmap e1140: 8 total, 8 up, 7 in 2024-08-17T17:18:26.759 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:26 smithi063 ceph-mon[109843]: osdmap e1139: 8 total, 8 up, 7 in 2024-08-17T17:18:26.759 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:26 smithi063 ceph-mon[109843]: osdmap e1140: 8 total, 8 up, 7 in 2024-08-17T17:18:26.772 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:26.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:26.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:26.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:26 smithi162 ceph-mon[104668]: osdmap e1139: 8 total, 8 up, 7 in 2024-08-17T17:18:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:26 smithi162 ceph-mon[104668]: osdmap e1140: 8 total, 8 up, 7 in 2024-08-17T17:18:26.874 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:26.916 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:26.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:26.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:27.018 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:27.054 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:27.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:27.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:27.156 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:27.198 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:27.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:27.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:27.299 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:27.341 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:27.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:27.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:27.438 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 29 16 61210 61194 131.862 101.375 0.0111541 0.0075782 2024-08-17T17:18:27.443 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:27.478 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:27.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:27.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:27.494 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:18:27.580 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:27.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:27.622 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:27.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:27.654 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:27 smithi162 ceph-mon[104668]: pgmap v1335: 99 pgs: 1 unknown, 98 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-17T17:18:27.654 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:27 smithi162 ceph-mon[104668]: osdmap e1141: 8 total, 8 up, 7 in 2024-08-17T17:18:27.710 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:18:27.710 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:18:27.723 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:27.766 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:27.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:27.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:27.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:27 smithi063 ceph-mon[114756]: pgmap v1335: 99 pgs: 1 unknown, 98 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-17T17:18:27.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:27 smithi063 ceph-mon[114756]: osdmap e1141: 8 total, 8 up, 7 in 2024-08-17T17:18:27.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:27 smithi063 ceph-mon[109843]: pgmap v1335: 99 pgs: 1 unknown, 98 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-17T17:18:27.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:27 smithi063 ceph-mon[109843]: osdmap e1141: 8 total, 8 up, 7 in 2024-08-17T17:18:27.868 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:27.910 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:27.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:27.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:28.012 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:28.042 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:28.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:28.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:28.144 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:28.186 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:28.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:28.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:28.287 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:28.325 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:28.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:28.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:28.427 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:28.439 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 30 16 62703 62687 130.577 93.3125 0.00251287 0.00764931 2024-08-17T17:18:28.454 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:28.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:28.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:28.556 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:28.609 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:28.609 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:28.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:28.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:28.711 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:28.743 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:28.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:28.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:28.844 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:28.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:28.881 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:28.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:28.883 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:18:28.983 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:29.028 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:29.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:29.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:29.130 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:29.171 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:29.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:29.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:29.273 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:29.276 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:18:29.315 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:29.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:29.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:29.416 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:29.439 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 31 16 64155 64139 129.292 90.75 0.0096451 0.00772914 2024-08-17T17:18:29.455 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:29.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:29.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:29.557 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:29.594 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:29.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:29.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:29.696 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:29.733 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:29 smithi063 ceph-mon[114756]: pgmap v1338: 99 pgs: 5 active+undersized+remapped, 1 unknown, 93 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 124 MiB/s wr, 1.97k op/s 2024-08-17T17:18:29.733 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:29 smithi063 ceph-mon[114756]: osdmap e1142: 8 total, 8 up, 7 in 2024-08-17T17:18:29.733 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:29 smithi063 ceph-mon[109843]: pgmap v1338: 99 pgs: 5 active+undersized+remapped, 1 unknown, 93 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 124 MiB/s wr, 1.97k op/s 2024-08-17T17:18:29.733 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:29 smithi063 ceph-mon[109843]: osdmap e1142: 8 total, 8 up, 7 in 2024-08-17T17:18:29.734 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:29 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a[109820]: 2024-08-17T17:18:29.670+0000 7f66fd20a640 -1 mon.a@0(leader).osd e1142 definitely_dead 0 2024-08-17T17:18:29.736 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:29.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:29.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:29.747 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:29 smithi162 ceph-mon[104668]: pgmap v1338: 99 pgs: 5 active+undersized+remapped, 1 unknown, 93 active+clean; 3.6 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 124 MiB/s wr, 1.97k op/s 2024-08-17T17:18:29.747 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:29 smithi162 ceph-mon[104668]: osdmap e1142: 8 total, 8 up, 7 in 2024-08-17T17:18:29.838 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:29.886 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:29.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:29.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:29.988 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:30.010 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:18:30.010 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:18:30.028 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:30.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:30.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:30.048 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:18:30.049 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:18:30.084 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:18:30.084 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:18:30.116 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:18:30.116 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:18:30.130 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:30.151 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:18:30.151 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:18:30.172 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:30.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:30.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:30.187 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:18:30.188 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:18:30.220 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:18:30.221 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:18:30.255 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:18:30.256 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:18:30.274 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:30.306 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:30.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:30.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:30.408 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:30.439 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 32 16 65605 65589 128.084 90.625 0.00480176 0.00780234 2024-08-17T17:18:30.446 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:30.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:30.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:30.524 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:30 smithi063 ceph-mon[109843]: 1.4 scrub starts 2024-08-17T17:18:30.524 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:30 smithi063 ceph-mon[109843]: 1.4 scrub ok 2024-08-17T17:18:30.524 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:30 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3961907320' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:18:30.524 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:30 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:18:30.525 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:30 smithi063 ceph-mon[114756]: 1.4 scrub starts 2024-08-17T17:18:30.525 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:30 smithi063 ceph-mon[114756]: 1.4 scrub ok 2024-08-17T17:18:30.525 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:30 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3961907320' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:18:30.525 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:30 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:18:30.525 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:30 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/155420045' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:30.527 INFO:teuthology.orchestra.run.smithi063.stderr:marked in osd.0. 2024-08-17T17:18:30.547 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:30.572 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:30.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:30.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:30.674 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:30.696 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:30.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:30.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:30.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:30 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/155420045' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:30.798 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:30.811 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:30 smithi162 ceph-mon[104668]: 1.4 scrub starts 2024-08-17T17:18:30.811 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:30 smithi162 ceph-mon[104668]: 1.4 scrub ok 2024-08-17T17:18:30.811 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:30 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3961907320' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:18:30.811 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:30 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["0"]} : dispatch 2024-08-17T17:18:30.811 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:30 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/155420045' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:30.822 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:30.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:30.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:30.924 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:30.945 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:30.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:30.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:31.047 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:31.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:31.070 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:31.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:31.171 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:31.194 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:31.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:31.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:31.296 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:31.317 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:31.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:31.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:31.419 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:31.439 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 33 16 65764 65748 124.504 9.9375 0.00774371 0.00780963 2024-08-17T17:18:31.442 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:31.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:31.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:31.545 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:31.568 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:31.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:31.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:31.670 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:31.692 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:31.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:31.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:31.748 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:31 smithi162 ceph-mon[104668]: pgmap v1340: 99 pgs: 5 active+undersized+remapped, 94 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-08-17T17:18:31.748 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:31 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-08-17T17:18:31.748 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:31 smithi162 ceph-mon[104668]: osdmap e1143: 8 total, 8 up, 8 in 2024-08-17T17:18:31.749 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:31.753 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:18:31.753 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:18:31.753 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:18:31.753 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:18:31.753 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.753 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:35808 6.8 6:182f8ffd:::benchmark_data_smithi063_160268_object35807:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:18:31.753 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.876819+0000", 2024-08-17T17:18:31.753 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 290.87430860900002, 2024-08-17T17:18:31.754 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.144014062, 2024-08-17T17:18:31.754 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.754 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.754 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.754 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.754 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.754 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 35808 2024-08-17T17:18:31.754 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.754 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.754 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.754 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.754 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876819+0000", 2024-08-17T17:18:31.755 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.755 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.755 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.755 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.755 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876819+0000", 2024-08-17T17:18:31.755 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.755 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.755 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.755 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.755 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876818+0000", 2024-08-17T17:18:31.755 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:31.755 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.756 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.756 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.756 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876847+0000", 2024-08-17T17:18:31.756 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9553e-05 2024-08-17T17:18:31.756 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.756 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.756 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.756 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876870+0000", 2024-08-17T17:18:31.756 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2133999999999998e-05 2024-08-17T17:18:31.756 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.756 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.756 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.756 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876876+0000", 2024-08-17T17:18:31.756 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5599999999999999e-06 2024-08-17T17:18:31.757 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.757 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.757 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.757 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.876932+0000", 2024-08-17T17:18:31.757 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6078000000000001e-05 2024-08-17T17:18:31.757 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.757 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.757 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.757 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.877050+0000", 2024-08-17T17:18:31.757 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000117439 2024-08-17T17:18:31.757 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.757 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.757 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:18:31.757 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.877169+0000", 2024-08-17T17:18:31.758 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000119262 2024-08-17T17:18:31.758 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.758 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.758 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.758 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.878556+0000", 2024-08-17T17:18:31.758 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013872610000000001 2024-08-17T17:18:31.758 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.758 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.758 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.758 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.881767+0000", 2024-08-17T17:18:31.758 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0032103890000000001 2024-08-17T17:18:31.758 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.759 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.759 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.759 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.020762+0000", 2024-08-17T17:18:31.759 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.13899573000000001 2024-08-17T17:18:31.759 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.759 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.759 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.759 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.020803+0000", 2024-08-17T17:18:31.759 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0271000000000001e-05 2024-08-17T17:18:31.759 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.759 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.759 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.759 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.020833+0000", 2024-08-17T17:18:31.760 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.0482000000000001e-05 2024-08-17T17:18:31.760 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.760 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.760 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.760 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.760 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.760 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:35812 6.8 6:1fd0ca99:::benchmark_data_smithi063_160268_object35811:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:18:31.760 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.879715+0000", 2024-08-17T17:18:31.760 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 290.87141302999999, 2024-08-17T17:18:31.760 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.147860146, 2024-08-17T17:18:31.760 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.760 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.760 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.761 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.761 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.761 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 35812 2024-08-17T17:18:31.761 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.761 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.761 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.761 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.761 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.879715+0000", 2024-08-17T17:18:31.761 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.761 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.761 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.761 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.761 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.879715+0000", 2024-08-17T17:18:31.761 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.762 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.762 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.762 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.762 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.879713+0000", 2024-08-17T17:18:31.762 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:18:31.762 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.762 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.762 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.762 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.879776+0000", 2024-08-17T17:18:31.762 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3243000000000004e-05 2024-08-17T17:18:31.762 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.762 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.762 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.763 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.879791+0000", 2024-08-17T17:18:31.763 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5216e-05 2024-08-17T17:18:31.763 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.763 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.763 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.763 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.879799+0000", 2024-08-17T17:18:31.763 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0339999999999997e-06 2024-08-17T17:18:31.763 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.763 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.763 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.763 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.879850+0000", 2024-08-17T17:18:31.763 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1171000000000002e-05 2024-08-17T17:18:31.763 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.764 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.764 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.764 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880005+0000", 2024-08-17T17:18:31.764 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000154816 2024-08-17T17:18:31.764 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.764 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.764 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:18:31.764 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880161+0000", 2024-08-17T17:18:31.764 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000155508 2024-08-17T17:18:31.764 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.764 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.764 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.764 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.881775+0000", 2024-08-17T17:18:31.764 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016141980000000001 2024-08-17T17:18:31.764 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.765 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.765 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.765 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.881823+0000", 2024-08-17T17:18:31.765 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7828999999999999e-05 2024-08-17T17:18:31.765 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.765 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.765 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.765 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.027484+0000", 2024-08-17T17:18:31.765 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14566167399999999 2024-08-17T17:18:31.765 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.765 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.765 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.765 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.027535+0000", 2024-08-17T17:18:31.766 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0946999999999999e-05 2024-08-17T17:18:31.766 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.766 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.766 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.766 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.027575+0000", 2024-08-17T17:18:31.766 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9304000000000002e-05 2024-08-17T17:18:31.766 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.766 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.766 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.766 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.766 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.766 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:35818 6.3 6:ccea9449:::benchmark_data_smithi063_160268_object35817:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:18:31.766 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.880685+0000", 2024-08-17T17:18:31.767 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 290.87044232099998, 2024-08-17T17:18:31.767 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14219040599999999, 2024-08-17T17:18:31.767 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.767 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.767 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.767 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.768 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.768 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 35818 2024-08-17T17:18:31.768 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.768 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.768 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.768 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.768 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880685+0000", 2024-08-17T17:18:31.768 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.768 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.768 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.769 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.769 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880685+0000", 2024-08-17T17:18:31.769 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.769 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.769 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.769 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.769 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880684+0000", 2024-08-17T17:18:31.769 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:31.769 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.769 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.769 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.769 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880772+0000", 2024-08-17T17:18:31.770 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7903000000000002e-05 2024-08-17T17:18:31.770 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.770 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.770 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.770 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880783+0000", 2024-08-17T17:18:31.770 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1126e-05 2024-08-17T17:18:31.770 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.770 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.770 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.770 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.880790+0000", 2024-08-17T17:18:31.770 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.64e-06 2024-08-17T17:18:31.770 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.770 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.771 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.771 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.882214+0000", 2024-08-17T17:18:31.771 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001423983 2024-08-17T17:18:31.771 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.771 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.771 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.771 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.882336+0000", 2024-08-17T17:18:31.771 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012227599999999999 2024-08-17T17:18:31.771 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.771 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.771 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:18:31.771 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.882475+0000", 2024-08-17T17:18:31.771 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013835299999999999 2024-08-17T17:18:31.772 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.772 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.772 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.772 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.883876+0000", 2024-08-17T17:18:31.772 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001401673 2024-08-17T17:18:31.772 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.772 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.772 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.772 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.883883+0000", 2024-08-17T17:18:31.772 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3559999999999998e-06 2024-08-17T17:18:31.772 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.772 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.772 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.773 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.022798+0000", 2024-08-17T17:18:31.773 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.13891547300000001 2024-08-17T17:18:31.773 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.773 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.773 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.773 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.022835+0000", 2024-08-17T17:18:31.773 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6560000000000002e-05 2024-08-17T17:18:31.773 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.773 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.773 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.773 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.022876+0000", 2024-08-17T17:18:31.773 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0969e-05 2024-08-17T17:18:31.774 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.774 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.774 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.774 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.774 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.774 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:35824 6.5 6:a1ee89be:::benchmark_data_smithi063_160268_object35823:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:18:31.774 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:13:40.885901+0000", 2024-08-17T17:18:31.774 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 290.86522647100003, 2024-08-17T17:18:31.774 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.13968393500000001, 2024-08-17T17:18:31.774 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.774 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.774 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.775 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.775 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.775 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 35824 2024-08-17T17:18:31.775 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.775 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.775 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.775 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.775 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.885901+0000", 2024-08-17T17:18:31.775 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.775 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.775 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.775 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.775 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.885901+0000", 2024-08-17T17:18:31.775 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.776 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.776 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.776 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.776 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.885900+0000", 2024-08-17T17:18:31.776 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:18:31.776 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.776 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.776 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.776 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.885933+0000", 2024-08-17T17:18:31.776 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3626e-05 2024-08-17T17:18:31.776 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.776 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.776 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.776 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.885950+0000", 2024-08-17T17:18:31.777 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7008999999999999e-05 2024-08-17T17:18:31.777 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.777 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.777 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.777 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.885958+0000", 2024-08-17T17:18:31.777 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.8800000000000008e-06 2024-08-17T17:18:31.777 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.777 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.777 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.777 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.885996+0000", 2024-08-17T17:18:31.777 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8148999999999997e-05 2024-08-17T17:18:31.777 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.777 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.777 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.778 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.886084+0000", 2024-08-17T17:18:31.778 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7713999999999994e-05 2024-08-17T17:18:31.778 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.778 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.778 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:18:31.778 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.886173+0000", 2024-08-17T17:18:31.778 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9302000000000007e-05 2024-08-17T17:18:31.778 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.778 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.778 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.778 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.887257+0000", 2024-08-17T17:18:31.778 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001083557 2024-08-17T17:18:31.778 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.778 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.779 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.779 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:40.887756+0000", 2024-08-17T17:18:31.779 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00049890399999999999 2024-08-17T17:18:31.779 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.779 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.779 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.779 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025506+0000", 2024-08-17T17:18:31.779 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.13775047300000001 2024-08-17T17:18:31.779 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.779 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.779 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.779 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025550+0000", 2024-08-17T17:18:31.779 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4240000000000003e-05 2024-08-17T17:18:31.780 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.780 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.780 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.780 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:13:41.025585+0000", 2024-08-17T17:18:31.780 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4610999999999998e-05 2024-08-17T17:18:31.780 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.780 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.780 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.780 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.780 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.780 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131754 6.9 6:932b4294:::benchmark_data_smithi063_160268_object5647:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:18:31.780 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.410180+0000", 2024-08-17T17:18:31.780 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 208.340947653, 2024-08-17T17:18:31.780 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99255267400000002, 2024-08-17T17:18:31.781 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.781 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.781 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.781 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.781 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.781 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131754 2024-08-17T17:18:31.781 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.781 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.781 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.781 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.781 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410180+0000", 2024-08-17T17:18:31.781 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.781 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.781 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.782 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.782 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410180+0000", 2024-08-17T17:18:31.782 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.782 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.782 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.782 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.782 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410179+0000", 2024-08-17T17:18:31.782 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:31.782 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.782 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.782 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.782 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410184+0000", 2024-08-17T17:18:31.782 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3889999999999998e-06 2024-08-17T17:18:31.783 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.783 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.783 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.783 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410192+0000", 2024-08-17T17:18:31.783 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0090000000000007e-06 2024-08-17T17:18:31.783 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.783 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.783 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.783 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410196+0000", 2024-08-17T17:18:31.783 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6249999999999998e-06 2024-08-17T17:18:31.783 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.783 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.784 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.784 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410283+0000", 2024-08-17T17:18:31.784 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7316000000000002e-05 2024-08-17T17:18:31.784 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.784 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.784 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:31.784 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.410296+0000", 2024-08-17T17:18:31.784 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.224e-05 2024-08-17T17:18:31.784 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.784 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.784 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.784 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.384191+0000", 2024-08-17T17:18:31.784 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97389498100000005 2024-08-17T17:18:31.785 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.785 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.785 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.785 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.386158+0000", 2024-08-17T17:18:31.785 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0019675019999999999 2024-08-17T17:18:31.785 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.785 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.785 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:18:31.785 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.386237+0000", 2024-08-17T17:18:31.785 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9043999999999997e-05 2024-08-17T17:18:31.785 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.785 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.785 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.786 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.397623+0000", 2024-08-17T17:18:31.786 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011385637000000001 2024-08-17T17:18:31.786 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.786 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.786 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.786 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.399643+0000", 2024-08-17T17:18:31.786 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0020198669999999998 2024-08-17T17:18:31.786 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.786 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.786 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.786 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402647+0000", 2024-08-17T17:18:31.786 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0030044149999999999 2024-08-17T17:18:31.786 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.787 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.787 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.787 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402697+0000", 2024-08-17T17:18:31.787 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0158999999999999e-05 2024-08-17T17:18:31.787 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.787 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.787 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.787 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402733+0000", 2024-08-17T17:18:31.787 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5327000000000002e-05 2024-08-17T17:18:31.787 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.787 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.787 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.787 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.787 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.788 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131760 6.9 6:97a252f6:::benchmark_data_smithi063_160268_object5653:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:18:31.788 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.411623+0000", 2024-08-17T17:18:31.788 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 208.33950423100001, 2024-08-17T17:18:31.788 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.001959477, 2024-08-17T17:18:31.788 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.788 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.788 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.788 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.788 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.788 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131760 2024-08-17T17:18:31.788 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.788 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.789 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.789 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.789 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411623+0000", 2024-08-17T17:18:31.789 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.789 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.789 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.789 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.789 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411623+0000", 2024-08-17T17:18:31.789 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.789 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.789 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.789 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.789 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411623+0000", 2024-08-17T17:18:31.789 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:18:31.790 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.790 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.790 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.790 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411637+0000", 2024-08-17T17:18:31.790 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4090000000000001e-05 2024-08-17T17:18:31.790 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.790 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.790 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.790 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411644+0000", 2024-08-17T17:18:31.790 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6580000000000002e-06 2024-08-17T17:18:31.790 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.790 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.790 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.791 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411671+0000", 2024-08-17T17:18:31.791 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6797000000000001e-05 2024-08-17T17:18:31.791 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.791 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.791 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.791 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411722+0000", 2024-08-17T17:18:31.791 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0883000000000001e-05 2024-08-17T17:18:31.791 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.791 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.791 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:31.791 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.411732+0000", 2024-08-17T17:18:31.791 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8879999999999992e-06 2024-08-17T17:18:31.791 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.792 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.792 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.792 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.388963+0000", 2024-08-17T17:18:31.792 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.977230873 2024-08-17T17:18:31.792 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.792 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.792 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.792 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.389067+0000", 2024-08-17T17:18:31.792 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010451 2024-08-17T17:18:31.792 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.792 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.792 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:18:31.792 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.389138+0000", 2024-08-17T17:18:31.793 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0607999999999999e-05 2024-08-17T17:18:31.793 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.793 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.793 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.793 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402743+0000", 2024-08-17T17:18:31.793 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.013605446 2024-08-17T17:18:31.793 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.793 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.793 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.793 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402812+0000", 2024-08-17T17:18:31.793 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8089999999999994e-05 2024-08-17T17:18:31.793 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.793 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.794 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.794 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.413532+0000", 2024-08-17T17:18:31.794 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010720929000000001 2024-08-17T17:18:31.794 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.794 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.794 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.794 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.413561+0000", 2024-08-17T17:18:31.794 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8481999999999999e-05 2024-08-17T17:18:31.794 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.794 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.794 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.794 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.413583+0000", 2024-08-17T17:18:31.794 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.1897999999999999e-05 2024-08-17T17:18:31.795 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.795 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.795 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.795 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.795 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.795 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131761 6.9 6:976cb822:::benchmark_data_smithi063_160268_object5654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:18:31.795 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.417190+0000", 2024-08-17T17:18:31.795 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 208.33393808, 2024-08-17T17:18:31.795 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.000269098, 2024-08-17T17:18:31.795 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.795 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.795 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.795 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.795 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.795 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131761 2024-08-17T17:18:31.796 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.796 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.796 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.796 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.796 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417190+0000", 2024-08-17T17:18:31.796 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.796 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.796 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.796 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.796 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417190+0000", 2024-08-17T17:18:31.796 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.796 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.796 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.796 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.796 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417188+0000", 2024-08-17T17:18:31.797 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:31.797 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.797 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.797 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.797 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417194+0000", 2024-08-17T17:18:31.797 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7169999999999996e-06 2024-08-17T17:18:31.797 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.797 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.797 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.797 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417206+0000", 2024-08-17T17:18:31.797 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1406999999999999e-05 2024-08-17T17:18:31.797 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.797 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.797 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.797 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417212+0000", 2024-08-17T17:18:31.798 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3450000000000002e-06 2024-08-17T17:18:31.798 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.798 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.798 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.798 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417272+0000", 2024-08-17T17:18:31.798 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9973999999999999e-05 2024-08-17T17:18:31.798 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.798 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.798 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:31.798 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.417281+0000", 2024-08-17T17:18:31.798 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3959999999999995e-06 2024-08-17T17:18:31.798 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.799 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.799 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.799 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.391578+0000", 2024-08-17T17:18:31.799 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97429638900000004 2024-08-17T17:18:31.799 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.799 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.799 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.799 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.391644+0000", 2024-08-17T17:18:31.799 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6518000000000002e-05 2024-08-17T17:18:31.799 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.799 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.799 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:18:31.799 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.391702+0000", 2024-08-17T17:18:31.799 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8319000000000002e-05 2024-08-17T17:18:31.800 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.800 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.800 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.800 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402748+0000", 2024-08-17T17:18:31.800 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011045589999999999 2024-08-17T17:18:31.800 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.800 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.800 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.800 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.413608+0000", 2024-08-17T17:18:31.800 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010860017 2024-08-17T17:18:31.800 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.800 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.800 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.801 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.417390+0000", 2024-08-17T17:18:31.801 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0037816780000000001 2024-08-17T17:18:31.801 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.801 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.801 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.801 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.417431+0000", 2024-08-17T17:18:31.801 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1054999999999999e-05 2024-08-17T17:18:31.801 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.801 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.801 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.801 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.417459+0000", 2024-08-17T17:18:31.801 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.7793999999999999e-05 2024-08-17T17:18:31.801 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.802 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.802 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.802 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.802 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.802 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131788 6.9 6:90417142:::benchmark_data_smithi063_160268_object5681:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:18:31.802 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.447657+0000", 2024-08-17T17:18:31.802 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 208.30347037499999, 2024-08-17T17:18:31.802 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97105383700000003, 2024-08-17T17:18:31.802 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.802 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.802 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.802 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.803 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.803 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131788 2024-08-17T17:18:31.803 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.803 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.803 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.803 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.803 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447657+0000", 2024-08-17T17:18:31.803 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.803 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.803 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.803 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.803 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447657+0000", 2024-08-17T17:18:31.803 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.803 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.803 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.804 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.804 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447657+0000", 2024-08-17T17:18:31.804 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:18:31.804 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.804 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.804 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.804 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447660+0000", 2024-08-17T17:18:31.804 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9720000000000001e-06 2024-08-17T17:18:31.804 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.804 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.804 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.804 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447667+0000", 2024-08-17T17:18:31.804 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2069999999999998e-06 2024-08-17T17:18:31.804 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.805 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.805 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.805 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447672+0000", 2024-08-17T17:18:31.805 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.865e-06 2024-08-17T17:18:31.805 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.805 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.805 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.805 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447742+0000", 2024-08-17T17:18:31.805 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0279999999999998e-05 2024-08-17T17:18:31.805 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.805 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.805 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:31.805 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.447751+0000", 2024-08-17T17:18:31.805 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.0650000000000005e-06 2024-08-17T17:18:31.806 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.806 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.806 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.806 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.393494+0000", 2024-08-17T17:18:31.806 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.94574265899999999 2024-08-17T17:18:31.806 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.806 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.806 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.806 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.393554+0000", 2024-08-17T17:18:31.806 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9993e-05 2024-08-17T17:18:31.806 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.806 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.807 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:18:31.807 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.393609+0000", 2024-08-17T17:18:31.807 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5299e-05 2024-08-17T17:18:31.807 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.807 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.807 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.807 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402754+0000", 2024-08-17T17:18:31.807 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0091449779999999998 2024-08-17T17:18:31.807 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.807 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.807 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.807 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.417295+0000", 2024-08-17T17:18:31.807 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.014540993 2024-08-17T17:18:31.807 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.808 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.808 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.808 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418646+0000", 2024-08-17T17:18:31.808 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001351229 2024-08-17T17:18:31.808 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.808 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.808 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.808 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418679+0000", 2024-08-17T17:18:31.808 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3148999999999997e-05 2024-08-17T17:18:31.808 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.808 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.808 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.808 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418711+0000", 2024-08-17T17:18:31.809 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1727999999999998e-05 2024-08-17T17:18:31.809 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.809 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.809 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.809 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.809 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.809 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131792 6.9 6:925a7a40:::benchmark_data_smithi063_160268_object5685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:18:31.809 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.449563+0000", 2024-08-17T17:18:31.809 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 208.301564638, 2024-08-17T17:18:31.809 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.969275954, 2024-08-17T17:18:31.809 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.809 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.809 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.810 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.810 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.810 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131792 2024-08-17T17:18:31.810 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.810 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.810 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.810 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.810 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449563+0000", 2024-08-17T17:18:31.810 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.810 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.810 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.810 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.810 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449563+0000", 2024-08-17T17:18:31.811 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.811 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.811 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.811 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.811 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449562+0000", 2024-08-17T17:18:31.811 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:18:31.811 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.811 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.811 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.811 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449565+0000", 2024-08-17T17:18:31.811 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6599999999999999e-06 2024-08-17T17:18:31.811 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.811 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.812 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.812 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449571+0000", 2024-08-17T17:18:31.812 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8760000000000002e-06 2024-08-17T17:18:31.812 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.812 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.812 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.812 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449575+0000", 2024-08-17T17:18:31.812 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4599999999999996e-06 2024-08-17T17:18:31.812 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.812 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.812 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.812 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449662+0000", 2024-08-17T17:18:31.813 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6129000000000002e-05 2024-08-17T17:18:31.813 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.813 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.813 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:31.813 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.449671+0000", 2024-08-17T17:18:31.813 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.5419999999999994e-06 2024-08-17T17:18:31.813 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.813 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.813 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.813 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.395313+0000", 2024-08-17T17:18:31.813 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.94564203300000005 2024-08-17T17:18:31.813 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.813 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.814 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.814 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.395380+0000", 2024-08-17T17:18:31.814 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6877000000000004e-05 2024-08-17T17:18:31.814 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.814 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.814 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:18:31.814 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.395451+0000", 2024-08-17T17:18:31.814 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0899e-05 2024-08-17T17:18:31.814 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.814 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.814 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.814 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402758+0000", 2024-08-17T17:18:31.814 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.007306692 2024-08-17T17:18:31.815 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.815 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.815 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.815 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.417323+0000", 2024-08-17T17:18:31.815 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.014565494999999999 2024-08-17T17:18:31.815 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.815 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.815 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.815 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418755+0000", 2024-08-17T17:18:31.815 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001432165 2024-08-17T17:18:31.815 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.815 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.815 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.815 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418797+0000", 2024-08-17T17:18:31.816 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1294999999999997e-05 2024-08-17T17:18:31.816 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.816 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.816 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.816 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418839+0000", 2024-08-17T17:18:31.816 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2318999999999999e-05 2024-08-17T17:18:31.816 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.816 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.816 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.816 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.816 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.816 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131802 6.9 6:9647ea40:::benchmark_data_smithi063_160268_object5695:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:18:31.816 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.459916+0000", 2024-08-17T17:18:31.816 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 208.29121113599999, 2024-08-17T17:18:31.817 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95901592300000005, 2024-08-17T17:18:31.817 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.817 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.817 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.817 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.817 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.817 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131802 2024-08-17T17:18:31.817 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.817 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.817 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.817 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.817 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.459916+0000", 2024-08-17T17:18:31.817 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.818 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.818 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.818 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.818 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.459916+0000", 2024-08-17T17:18:31.818 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.818 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.818 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.818 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.818 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.459915+0000", 2024-08-17T17:18:31.818 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:31.818 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.818 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.818 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.819 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.459920+0000", 2024-08-17T17:18:31.819 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1070000000000004e-06 2024-08-17T17:18:31.819 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.819 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.819 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.819 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.459936+0000", 2024-08-17T17:18:31.819 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5534000000000001e-05 2024-08-17T17:18:31.819 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.819 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.819 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.819 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.459943+0000", 2024-08-17T17:18:31.819 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0990000000000001e-06 2024-08-17T17:18:31.819 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.819 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.819 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.820 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.460001+0000", 2024-08-17T17:18:31.820 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8019000000000001e-05 2024-08-17T17:18:31.820 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.820 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.820 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:31.820 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.460011+0000", 2024-08-17T17:18:31.820 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9299999999999998e-06 2024-08-17T17:18:31.820 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.820 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.820 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.820 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.397658+0000", 2024-08-17T17:18:31.820 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.93764710799999995 2024-08-17T17:18:31.820 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.820 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.821 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.821 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.397738+0000", 2024-08-17T17:18:31.821 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9847999999999999e-05 2024-08-17T17:18:31.821 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.821 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.821 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:18:31.821 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.397800+0000", 2024-08-17T17:18:31.821 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2558000000000007e-05 2024-08-17T17:18:31.821 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.821 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.821 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.821 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.407472+0000", 2024-08-17T17:18:31.821 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0096714390000000004 2024-08-17T17:18:31.822 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.822 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.822 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.822 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.417362+0000", 2024-08-17T17:18:31.822 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0098902230000000001 2024-08-17T17:18:31.822 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.822 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.822 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.822 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418878+0000", 2024-08-17T17:18:31.822 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015162089999999999 2024-08-17T17:18:31.822 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.822 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.823 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.823 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418902+0000", 2024-08-17T17:18:31.823 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3910999999999999e-05 2024-08-17T17:18:31.823 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.823 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.823 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.823 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.418932+0000", 2024-08-17T17:18:31.823 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.0127000000000001e-05 2024-08-17T17:18:31.823 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.823 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.823 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.823 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.823 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.823 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131816 6.9 6:9135a939:::benchmark_data_smithi063_160268_object5709:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:18:31.824 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.474321+0000", 2024-08-17T17:18:31.824 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 208.27680611100001, 2024-08-17T17:18:31.824 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95894761299999998, 2024-08-17T17:18:31.824 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.824 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.824 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.824 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.824 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.824 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131816 2024-08-17T17:18:31.824 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.824 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.824 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.824 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.824 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474321+0000", 2024-08-17T17:18:31.824 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.825 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.825 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.825 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.825 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474321+0000", 2024-08-17T17:18:31.825 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:31.826 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:31 smithi063 ceph-mon[114756]: pgmap v1340: 99 pgs: 5 active+undersized+remapped, 94 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-08-17T17:18:31.826 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:31 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-08-17T17:18:31.826 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:31 smithi063 ceph-mon[114756]: osdmap e1143: 8 total, 8 up, 8 in 2024-08-17T17:18:31.826 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:31.827 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:31 smithi063 ceph-mon[109843]: pgmap v1340: 99 pgs: 5 active+undersized+remapped, 94 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-08-17T17:18:31.827 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:31 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2024-08-17T17:18:31.827 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:31 smithi063 ceph-mon[109843]: osdmap e1143: 8 total, 8 up, 8 in 2024-08-17T17:18:31.827 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:31.827 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.828 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.828 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.828 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.828 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474320+0000", 2024-08-17T17:18:31.828 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:18:31.828 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.828 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.828 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.828 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474326+0000", 2024-08-17T17:18:31.828 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1720000000000004e-06 2024-08-17T17:18:31.828 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.828 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.828 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.829 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474339+0000", 2024-08-17T17:18:31.829 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.323e-05 2024-08-17T17:18:31.829 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.829 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.829 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.829 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474346+0000", 2024-08-17T17:18:31.829 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.798e-06 2024-08-17T17:18:31.829 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.829 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.829 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.829 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474405+0000", 2024-08-17T17:18:31.829 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8764999999999997e-05 2024-08-17T17:18:31.829 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.829 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.830 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:31.830 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.474415+0000", 2024-08-17T17:18:31.830 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0265000000000001e-05 2024-08-17T17:18:31.830 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.830 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.830 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.830 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.399696+0000", 2024-08-17T17:18:31.830 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.925280926 2024-08-17T17:18:31.830 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.830 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.830 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.830 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.399767+0000", 2024-08-17T17:18:31.830 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1382000000000002e-05 2024-08-17T17:18:31.831 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.831 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.831 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:18:31.831 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.399830+0000", 2024-08-17T17:18:31.831 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2318999999999997e-05 2024-08-17T17:18:31.831 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.831 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.831 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.831 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.407480+0000", 2024-08-17T17:18:31.831 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0076499940000000002 2024-08-17T17:18:31.831 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.831 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.831 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.832 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.428141+0000", 2024-08-17T17:18:31.832 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.020661312000000001 2024-08-17T17:18:31.832 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.832 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.832 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.832 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433189+0000", 2024-08-17T17:18:31.832 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0050483239999999999 2024-08-17T17:18:31.832 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.832 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.832 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.832 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433234+0000", 2024-08-17T17:18:31.832 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4511000000000001e-05 2024-08-17T17:18:31.832 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.832 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.833 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.833 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433269+0000", 2024-08-17T17:18:31.833 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5154000000000003e-05 2024-08-17T17:18:31.833 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.833 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.833 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.833 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.833 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.833 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131821 6.9 6:9cc10ae1:::benchmark_data_smithi063_160268_object5714:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:18:31.833 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.481045+0000", 2024-08-17T17:18:31.833 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 208.27008212600001, 2024-08-17T17:18:31.833 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95235498799999996, 2024-08-17T17:18:31.833 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.834 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.834 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.834 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.834 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.834 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131821 2024-08-17T17:18:31.834 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.834 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.834 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.834 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.834 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481045+0000", 2024-08-17T17:18:31.834 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.834 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.835 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.835 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.835 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481045+0000", 2024-08-17T17:18:31.835 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.835 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.835 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.835 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.835 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481044+0000", 2024-08-17T17:18:31.835 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:31.835 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.835 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.835 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.835 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481051+0000", 2024-08-17T17:18:31.836 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1489999999999997e-06 2024-08-17T17:18:31.836 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.836 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.836 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.836 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481063+0000", 2024-08-17T17:18:31.836 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1209000000000001e-05 2024-08-17T17:18:31.836 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.836 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.836 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.836 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481070+0000", 2024-08-17T17:18:31.836 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9689999999999997e-06 2024-08-17T17:18:31.836 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.836 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.836 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.837 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481120+0000", 2024-08-17T17:18:31.837 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0664000000000001e-05 2024-08-17T17:18:31.837 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.837 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.837 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:31.837 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.481131+0000", 2024-08-17T17:18:31.837 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0788000000000001e-05 2024-08-17T17:18:31.837 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.837 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.837 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.837 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402839+0000", 2024-08-17T17:18:31.837 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.92170828100000002 2024-08-17T17:18:31.837 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.838 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.838 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.838 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.402908+0000", 2024-08-17T17:18:31.838 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8615999999999995e-05 2024-08-17T17:18:31.838 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.838 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.838 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:18:31.838 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.403002+0000", 2024-08-17T17:18:31.838 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3941000000000001e-05 2024-08-17T17:18:31.838 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.838 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.838 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.838 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.412030+0000", 2024-08-17T17:18:31.839 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0090276850000000006 2024-08-17T17:18:31.839 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.839 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.839 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.839 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.432479+0000", 2024-08-17T17:18:31.839 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.020449833000000001 2024-08-17T17:18:31.839 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.839 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.839 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.839 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433318+0000", 2024-08-17T17:18:31.839 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00083902200000000003 2024-08-17T17:18:31.839 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.839 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.839 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.840 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433364+0000", 2024-08-17T17:18:31.840 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5089000000000001e-05 2024-08-17T17:18:31.840 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.840 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.840 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.840 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433400+0000", 2024-08-17T17:18:31.840 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6933e-05 2024-08-17T17:18:31.840 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.840 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.840 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.840 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.840 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.840 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131837 6.9 6:9b5d10c1:::benchmark_data_smithi063_160268_object5730:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:18:31.841 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.510534+0000", 2024-08-17T17:18:31.841 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 208.240593606, 2024-08-17T17:18:31.841 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.92298864199999997, 2024-08-17T17:18:31.841 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.841 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.841 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.841 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.841 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.841 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131837 2024-08-17T17:18:31.841 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.841 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.841 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.841 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.841 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510534+0000", 2024-08-17T17:18:31.842 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.842 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.842 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.842 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.842 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510534+0000", 2024-08-17T17:18:31.842 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.842 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.842 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.842 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.842 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510533+0000", 2024-08-17T17:18:31.842 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:18:31.842 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.843 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.843 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.843 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510540+0000", 2024-08-17T17:18:31.843 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3889999999999999e-06 2024-08-17T17:18:31.843 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.843 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.843 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.843 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510556+0000", 2024-08-17T17:18:31.843 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6371999999999998e-05 2024-08-17T17:18:31.843 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.843 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.843 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.843 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510564+0000", 2024-08-17T17:18:31.843 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.537e-06 2024-08-17T17:18:31.844 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.844 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.844 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.844 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510630+0000", 2024-08-17T17:18:31.844 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6227000000000001e-05 2024-08-17T17:18:31.844 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.844 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.844 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:31.844 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.510638+0000", 2024-08-17T17:18:31.844 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.259e-06 2024-08-17T17:18:31.844 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.844 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.844 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.844 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.407509+0000", 2024-08-17T17:18:31.845 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.896870838 2024-08-17T17:18:31.845 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.845 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.845 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.845 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.407578+0000", 2024-08-17T17:18:31.845 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8706000000000003e-05 2024-08-17T17:18:31.845 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.845 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.845 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:18:31.845 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.407638+0000", 2024-08-17T17:18:31.845 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0476000000000002e-05 2024-08-17T17:18:31.845 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.845 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.846 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.846 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.412032+0000", 2024-08-17T17:18:31.846 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.004393816 2024-08-17T17:18:31.846 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.846 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.846 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.846 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.432524+0000", 2024-08-17T17:18:31.846 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.020491366 2024-08-17T17:18:31.846 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.846 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.846 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.846 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433446+0000", 2024-08-17T17:18:31.847 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00092251300000000002 2024-08-17T17:18:31.847 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.847 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.847 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.847 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433487+0000", 2024-08-17T17:18:31.847 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0766000000000002e-05 2024-08-17T17:18:31.847 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.847 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.847 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.847 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433523+0000", 2024-08-17T17:18:31.847 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5707000000000002e-05 2024-08-17T17:18:31.847 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.847 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.847 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.848 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.848 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.848 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:131865 6.9 6:9e0f52b2:::benchmark_data_smithi063_160268_object5758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1103)", 2024-08-17T17:18:31.848 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:03.606289+0000", 2024-08-17T17:18:31.848 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 208.14483870500001, 2024-08-17T17:18:31.848 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.82736346199999999, 2024-08-17T17:18:31.848 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.848 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.848 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.848 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.848 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.848 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 131865 2024-08-17T17:18:31.848 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.849 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.849 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.849 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.849 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606289+0000", 2024-08-17T17:18:31.849 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.849 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.849 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.849 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.849 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606289+0000", 2024-08-17T17:18:31.849 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.849 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.849 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.849 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.849 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606287+0000", 2024-08-17T17:18:31.850 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999976 2024-08-17T17:18:31.850 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.850 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.850 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.850 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606297+0000", 2024-08-17T17:18:31.850 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0429e-05 2024-08-17T17:18:31.850 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.850 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.850 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.850 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606317+0000", 2024-08-17T17:18:31.850 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0228e-05 2024-08-17T17:18:31.850 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.850 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.851 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.851 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606328+0000", 2024-08-17T17:18:31.851 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0409e-05 2024-08-17T17:18:31.851 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.851 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.851 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.851 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606381+0000", 2024-08-17T17:18:31.851 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3000000000000001e-05 2024-08-17T17:18:31.851 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.851 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.851 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:18:31.851 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:03.606390+0000", 2024-08-17T17:18:31.851 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.1400000000000006e-06 2024-08-17T17:18:31.852 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.852 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.852 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.852 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.410180+0000", 2024-08-17T17:18:31.852 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.80379011099999997 2024-08-17T17:18:31.852 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.852 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.852 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.852 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.410245+0000", 2024-08-17T17:18:31.852 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5073999999999994e-05 2024-08-17T17:18:31.852 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.852 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.852 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:18:31.852 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.410306+0000", 2024-08-17T17:18:31.853 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1377999999999994e-05 2024-08-17T17:18:31.853 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.853 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.853 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.853 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.412153+0000", 2024-08-17T17:18:31.853 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00184633 2024-08-17T17:18:31.853 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.853 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.853 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.853 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.432552+0000", 2024-08-17T17:18:31.853 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.020399245 2024-08-17T17:18:31.853 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.853 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.853 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.854 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433572+0000", 2024-08-17T17:18:31.854 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001020139 2024-08-17T17:18:31.854 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.854 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.854 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.854 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433616+0000", 2024-08-17T17:18:31.854 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3988e-05 2024-08-17T17:18:31.854 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.854 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.854 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.854 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:04.433652+0000", 2024-08-17T17:18:31.854 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6353000000000002e-05 2024-08-17T17:18:31.854 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.855 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.855 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.855 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.855 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.855 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:233086 6.8 6:15b9c686:::benchmark_data_smithi063_160268_object106979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-08-17T17:18:31.855 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:17:04.694218+0000", 2024-08-17T17:18:31.855 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 87.056909895999993, 2024-08-17T17:18:31.855 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.11770143399999999, 2024-08-17T17:18:31.855 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.855 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.855 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.855 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.855 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.856 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 233086 2024-08-17T17:18:31.856 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.856 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.856 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.856 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.856 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.694218+0000", 2024-08-17T17:18:31.856 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.856 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.856 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.856 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.856 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.694218+0000", 2024-08-17T17:18:31.856 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.856 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.857 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.857 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.857 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.694217+0000", 2024-08-17T17:18:31.857 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:31.857 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.857 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.857 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.857 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.694222+0000", 2024-08-17T17:18:31.857 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8840000000000002e-06 2024-08-17T17:18:31.857 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.857 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.857 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.857 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.694232+0000", 2024-08-17T17:18:31.858 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0087e-05 2024-08-17T17:18:31.858 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.858 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.858 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.858 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.694237+0000", 2024-08-17T17:18:31.858 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9130000000000002e-06 2024-08-17T17:18:31.858 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.858 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.858 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.858 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.695889+0000", 2024-08-17T17:18:31.858 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016518489999999999 2024-08-17T17:18:31.858 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.859 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.859 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.859 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.695996+0000", 2024-08-17T17:18:31.859 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010742399999999999 2024-08-17T17:18:31.859 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.859 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.859 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:18:31.859 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.696120+0000", 2024-08-17T17:18:31.859 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000124077 2024-08-17T17:18:31.859 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.859 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.859 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.859 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.702870+0000", 2024-08-17T17:18:31.859 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0067497470000000004 2024-08-17T17:18:31.859 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.860 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.860 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.860 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.731445+0000", 2024-08-17T17:18:31.860 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.028574762 2024-08-17T17:18:31.860 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.860 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.860 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.860 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.811834+0000", 2024-08-17T17:18:31.860 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.080389245999999998 2024-08-17T17:18:31.860 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.860 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.860 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.860 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.811884+0000", 2024-08-17T17:18:31.861 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0430999999999999e-05 2024-08-17T17:18:31.861 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.861 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.861 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.861 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.811919+0000", 2024-08-17T17:18:31.861 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4909000000000001e-05 2024-08-17T17:18:31.861 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.861 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.861 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.861 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.861 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.861 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:233087 6.8 6:187b46d6:::benchmark_data_smithi063_160268_object106980:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-08-17T17:18:31.861 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:17:04.694249+0000", 2024-08-17T17:18:31.862 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 87.056878271000002, 2024-08-17T17:18:31.862 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.12651457299999999, 2024-08-17T17:18:31.862 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.862 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.862 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.862 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.862 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.862 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 233087 2024-08-17T17:18:31.862 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.862 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.862 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.862 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.862 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.694249+0000", 2024-08-17T17:18:31.862 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.863 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.863 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.863 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.863 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.694249+0000", 2024-08-17T17:18:31.863 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.863 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.863 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.863 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.863 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.694249+0000", 2024-08-17T17:18:31.863 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:18:31.863 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.863 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.863 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.864 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.694252+0000", 2024-08-17T17:18:31.864 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8700000000000001e-06 2024-08-17T17:18:31.864 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.864 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.864 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.864 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.694261+0000", 2024-08-17T17:18:31.864 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.6919999999999997e-06 2024-08-17T17:18:31.864 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.864 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.864 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.864 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.694267+0000", 2024-08-17T17:18:31.864 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4419999999999997e-06 2024-08-17T17:18:31.864 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.864 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.865 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.865 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.699375+0000", 2024-08-17T17:18:31.865 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0051083980000000001 2024-08-17T17:18:31.865 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.865 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.865 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.865 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.699451+0000", 2024-08-17T17:18:31.865 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.5788e-05 2024-08-17T17:18:31.865 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.865 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.865 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:18:31.865 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.699524+0000", 2024-08-17T17:18:31.865 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3402e-05 2024-08-17T17:18:31.865 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.866 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.866 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.866 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.703033+0000", 2024-08-17T17:18:31.866 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0035086219999999999 2024-08-17T17:18:31.866 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.866 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.866 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.866 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.731472+0000", 2024-08-17T17:18:31.866 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.028439441999999999 2024-08-17T17:18:31.866 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.866 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.866 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.867 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.818367+0000", 2024-08-17T17:18:31.867 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.086894784000000003 2024-08-17T17:18:31.867 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.867 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.867 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.867 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.820726+0000", 2024-08-17T17:18:31.867 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023591440000000001 2024-08-17T17:18:31.867 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.867 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.867 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.867 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.820764+0000", 2024-08-17T17:18:31.867 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.752e-05 2024-08-17T17:18:31.867 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.868 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.868 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.868 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.868 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.868 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:233107 6.8 6:1a8699ea:::benchmark_data_smithi063_160268_object107000:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1117)", 2024-08-17T17:18:31.868 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:17:04.717591+0000", 2024-08-17T17:18:31.868 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 87.033537100999993, 2024-08-17T17:18:31.868 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.10376273900000001, 2024-08-17T17:18:31.868 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.868 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.868 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.868 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:18:31.868 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:31.868 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 233107 2024-08-17T17:18:31.869 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.869 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.869 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.869 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.869 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.717591+0000", 2024-08-17T17:18:31.869 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.869 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.869 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.869 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.869 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.717591+0000", 2024-08-17T17:18:31.869 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.869 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.869 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.869 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.869 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.717589+0000", 2024-08-17T17:18:31.870 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:18:31.870 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.870 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.870 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.870 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.717596+0000", 2024-08-17T17:18:31.870 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2659999999999998e-06 2024-08-17T17:18:31.870 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.870 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.870 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.870 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.717609+0000", 2024-08-17T17:18:31.870 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3086e-05 2024-08-17T17:18:31.870 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.870 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.871 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.871 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.717616+0000", 2024-08-17T17:18:31.871 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2710000000000003e-06 2024-08-17T17:18:31.871 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.871 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.871 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.871 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.717672+0000", 2024-08-17T17:18:31.871 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6402e-05 2024-08-17T17:18:31.871 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.871 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.871 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.871 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.717797+0000", 2024-08-17T17:18:31.872 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012459 2024-08-17T17:18:31.872 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.872 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.872 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:18:31.872 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.717928+0000", 2024-08-17T17:18:31.872 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013137199999999999 2024-08-17T17:18:31.872 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.872 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.872 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.872 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.722189+0000", 2024-08-17T17:18:31.872 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0042607280000000001 2024-08-17T17:18:31.872 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.872 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.873 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.873 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.739459+0000", 2024-08-17T17:18:31.873 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.017270258 2024-08-17T17:18:31.873 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.873 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.873 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.873 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.821281+0000", 2024-08-17T17:18:31.873 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.081821636000000003 2024-08-17T17:18:31.873 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.873 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.873 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.873 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.821321+0000", 2024-08-17T17:18:31.873 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0062999999999999e-05 2024-08-17T17:18:31.874 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.874 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.874 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.874 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:17:04.821353+0000", 2024-08-17T17:18:31.874 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2199000000000001e-05 2024-08-17T17:18:31.874 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.874 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.874 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.874 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.874 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.874 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:32948 7.5 7:aeea7c62:::benchmark_data_smithi063_196002_object32947:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:18:31.874 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:10.728479+0000", 2024-08-17T17:18:31.874 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 21.022648405000002, 2024-08-17T17:18:31.875 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.103503516, 2024-08-17T17:18:31.875 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.875 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.875 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.875 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:18:31.875 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:18:31.875 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 32948 2024-08-17T17:18:31.875 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.875 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.875 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.875 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.875 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.728479+0000", 2024-08-17T17:18:31.875 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.876 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.876 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.876 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.876 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.728479+0000", 2024-08-17T17:18:31.876 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.876 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.876 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.876 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.876 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.728479+0000", 2024-08-17T17:18:31.876 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:18:31.876 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.876 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.876 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.876 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.728695+0000", 2024-08-17T17:18:31.877 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000216457 2024-08-17T17:18:31.877 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.877 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.877 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.877 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.728700+0000", 2024-08-17T17:18:31.877 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1440000000000002e-06 2024-08-17T17:18:31.877 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.877 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.877 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.877 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.728704+0000", 2024-08-17T17:18:31.877 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.523e-06 2024-08-17T17:18:31.877 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.877 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.878 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.878 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.729697+0000", 2024-08-17T17:18:31.878 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00099301300000000006 2024-08-17T17:18:31.878 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.878 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.878 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.878 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.729777+0000", 2024-08-17T17:18:31.878 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0049999999999994e-05 2024-08-17T17:18:31.878 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.878 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.878 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:18:31.878 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.729854+0000", 2024-08-17T17:18:31.878 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7003999999999997e-05 2024-08-17T17:18:31.879 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.879 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.879 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.879 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.827279+0000", 2024-08-17T17:18:31.879 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.097424695000000006 2024-08-17T17:18:31.879 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.879 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.879 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.879 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.827425+0000", 2024-08-17T17:18:31.879 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000146546 2024-08-17T17:18:31.879 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.879 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.879 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.880 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.831851+0000", 2024-08-17T17:18:31.880 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0044258149999999996 2024-08-17T17:18:31.880 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.880 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.880 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.880 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.831946+0000", 2024-08-17T17:18:31.880 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.5421999999999995e-05 2024-08-17T17:18:31.880 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.880 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.880 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.880 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:10.831983+0000", 2024-08-17T17:18:31.880 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6427999999999997e-05 2024-08-17T17:18:31.880 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.880 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.881 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.881 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.881 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.881 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:34051 7.5 7:a40e9396:::benchmark_data_smithi063_196002_object34050:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:18:31.881 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:11.455425+0000", 2024-08-17T17:18:31.881 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 20.295702681000002, 2024-08-17T17:18:31.881 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.17788235399999999, 2024-08-17T17:18:31.881 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.881 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.881 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.881 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:18:31.881 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:18:31.881 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 34051 2024-08-17T17:18:31.882 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.882 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.882 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.882 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.882 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.455425+0000", 2024-08-17T17:18:31.882 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.882 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.882 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.882 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.882 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.455425+0000", 2024-08-17T17:18:31.882 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.882 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.882 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.883 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.883 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.455423+0000", 2024-08-17T17:18:31.883 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:18:31.883 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.883 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.883 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.883 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.455450+0000", 2024-08-17T17:18:31.883 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6877e-05 2024-08-17T17:18:31.883 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.883 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.883 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.883 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.455463+0000", 2024-08-17T17:18:31.883 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2413e-05 2024-08-17T17:18:31.883 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.884 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.884 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.884 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.455469+0000", 2024-08-17T17:18:31.884 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5529999999999998e-06 2024-08-17T17:18:31.884 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.884 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.884 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.884 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.455517+0000", 2024-08-17T17:18:31.884 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7809000000000003e-05 2024-08-17T17:18:31.884 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.884 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.884 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.884 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.455637+0000", 2024-08-17T17:18:31.885 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012058899999999999 2024-08-17T17:18:31.885 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.885 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.885 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:18:31.885 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.455768+0000", 2024-08-17T17:18:31.885 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013068300000000001 2024-08-17T17:18:31.885 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.885 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.885 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.885 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.457249+0000", 2024-08-17T17:18:31.885 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0014805560000000001 2024-08-17T17:18:31.885 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.885 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.885 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.886 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.459808+0000", 2024-08-17T17:18:31.886 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0025588859999999998 2024-08-17T17:18:31.886 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.886 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.886 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.886 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.633205+0000", 2024-08-17T17:18:31.886 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.17339705799999999 2024-08-17T17:18:31.886 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.886 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.886 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.886 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.633263+0000", 2024-08-17T17:18:31.886 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8276000000000002e-05 2024-08-17T17:18:31.887 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.887 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.887 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.887 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.633307+0000", 2024-08-17T17:18:31.887 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4342999999999999e-05 2024-08-17T17:18:31.887 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.887 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.887 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.887 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.887 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.887 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:34082 7.5 7:ac1a773f:::benchmark_data_smithi063_196002_object34081:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:18:31.887 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:11.471442+0000", 2024-08-17T17:18:31.887 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 20.279685519000001, 2024-08-17T17:18:31.888 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.162121139, 2024-08-17T17:18:31.888 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:18:31.888 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:31.888 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:18:31.888 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:18:31.888 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:18:31.888 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 34082 2024-08-17T17:18:31.888 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.888 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:18:31.888 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.888 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:18:31.888 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471442+0000", 2024-08-17T17:18:31.888 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.889 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.889 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.889 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:18:31.889 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471442+0000", 2024-08-17T17:18:31.889 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:18:31.889 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.889 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.889 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:18:31.889 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471441+0000", 2024-08-17T17:18:31.889 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:18:31.889 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.889 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.889 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:18:31.889 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471507+0000", 2024-08-17T17:18:31.890 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6419000000000002e-05 2024-08-17T17:18:31.890 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.890 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.890 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:18:31.890 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471521+0000", 2024-08-17T17:18:31.890 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3577000000000001e-05 2024-08-17T17:18:31.890 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.890 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.890 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:18:31.890 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471527+0000", 2024-08-17T17:18:31.890 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.353e-06 2024-08-17T17:18:31.890 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.890 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.891 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:18:31.891 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471573+0000", 2024-08-17T17:18:31.891 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5386999999999998e-05 2024-08-17T17:18:31.891 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.891 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.891 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:18:31.891 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471683+0000", 2024-08-17T17:18:31.891 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000110207 2024-08-17T17:18:31.891 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.891 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.891 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:18:31.891 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471816+0000", 2024-08-17T17:18:31.891 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013365100000000001 2024-08-17T17:18:31.892 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.892 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.892 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:18:31.892 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.473178+0000", 2024-08-17T17:18:31.892 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013619979999999999 2024-08-17T17:18:31.892 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.892 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.892 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.892 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.473250+0000", 2024-08-17T17:18:31.892 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1361999999999999e-05 2024-08-17T17:18:31.892 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.892 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.892 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:31.893 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.633487+0000", 2024-08-17T17:18:31.893 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.16023672 2024-08-17T17:18:31.893 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.893 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.893 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:18:31.893 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.633531+0000", 2024-08-17T17:18:31.893 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4214000000000001e-05 2024-08-17T17:18:31.893 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:18:31.893 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:18:31.893 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:18:31.893 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.633563+0000", 2024-08-17T17:18:31.893 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2475999999999999e-05 2024-08-17T17:18:31.893 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.893 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.894 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.894 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:18:31.894 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:18:31.894 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:18:31.915 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:31.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:31.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:32.017 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:32.038 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:32.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:32.140 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:32.161 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:32.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.162 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:32.263 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:32.284 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:32.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:32.385 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:32.406 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:32.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:32.439 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 34 16 65764 65748 120.842 0 - 0.00780963 2024-08-17T17:18:32.508 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:32.529 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:32.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:32.619 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:18:32.631 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:32.653 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:32.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:32.754 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:32.780 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:32.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:32.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:32 smithi162 ceph-mon[104668]: osdmap e1144: 8 total, 8 up, 8 in 2024-08-17T17:18:32.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:32 smithi162 ceph-mon[104668]: 1.a scrub starts 2024-08-17T17:18:32.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:32 smithi162 ceph-mon[104668]: 1.a scrub ok 2024-08-17T17:18:32.882 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:32.916 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:32.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:32.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:33.018 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:32 smithi063 ceph-mon[114756]: osdmap e1144: 8 total, 8 up, 8 in 2024-08-17T17:18:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:32 smithi063 ceph-mon[114756]: 1.a scrub starts 2024-08-17T17:18:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:32 smithi063 ceph-mon[114756]: 1.a scrub ok 2024-08-17T17:18:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:32 smithi063 ceph-mon[109843]: osdmap e1144: 8 total, 8 up, 8 in 2024-08-17T17:18:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:32 smithi063 ceph-mon[109843]: 1.a scrub starts 2024-08-17T17:18:33.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:32 smithi063 ceph-mon[109843]: 1.a scrub ok 2024-08-17T17:18:33.051 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:33.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:33.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:33.152 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:33.183 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:33.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:33.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:33.285 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:33.327 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:33.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:33.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:33.429 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:33.439 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 35 16 66172 66156 118.118 12.75 0.00498795 0.00845863 2024-08-17T17:18:33.467 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:33.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:33.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:33.471 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:18:33.549 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:18:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:18:33] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.51.0" 2024-08-17T17:18:33.550 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:33 smithi063 ceph-mon[114756]: pgmap v1343: 99 pgs: 5 remapped+peering, 3 peering, 2 active+undersized+remapped, 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 0 B/s, 1 objects/s recovering 2024-08-17T17:18:33.550 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:33 smithi063 ceph-mon[114756]: osdmap e1145: 8 total, 8 up, 8 in 2024-08-17T17:18:33.551 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:33 smithi063 ceph-mon[114756]: osdmap e1146: 8 total, 8 up, 8 in 2024-08-17T17:18:33.551 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:33 smithi063 ceph-mon[114756]: 1.8 scrub starts 2024-08-17T17:18:33.551 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:33 smithi063 ceph-mon[114756]: 1.8 scrub ok 2024-08-17T17:18:33.551 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:33 smithi063 ceph-mon[109843]: pgmap v1343: 99 pgs: 5 remapped+peering, 3 peering, 2 active+undersized+remapped, 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 0 B/s, 1 objects/s recovering 2024-08-17T17:18:33.551 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:33 smithi063 ceph-mon[109843]: osdmap e1145: 8 total, 8 up, 8 in 2024-08-17T17:18:33.551 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:33 smithi063 ceph-mon[109843]: osdmap e1146: 8 total, 8 up, 8 in 2024-08-17T17:18:33.551 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:33 smithi063 ceph-mon[109843]: 1.8 scrub starts 2024-08-17T17:18:33.551 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:33 smithi063 ceph-mon[109843]: 1.8 scrub ok 2024-08-17T17:18:33.569 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:33.592 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:33.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:33.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:33.654 INFO:tasks.thrashosds.thrasher:Added osd 0 2024-08-17T17:18:33.694 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:33.717 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:33.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:33.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:33.770 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:33.819 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:33.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:33 smithi162 ceph-mon[104668]: pgmap v1343: 99 pgs: 5 remapped+peering, 3 peering, 2 active+undersized+remapped, 89 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 0 B/s, 1 objects/s recovering 2024-08-17T17:18:33.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:33 smithi162 ceph-mon[104668]: osdmap e1145: 8 total, 8 up, 8 in 2024-08-17T17:18:33.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:33 smithi162 ceph-mon[104668]: osdmap e1146: 8 total, 8 up, 8 in 2024-08-17T17:18:33.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:33 smithi162 ceph-mon[104668]: 1.8 scrub starts 2024-08-17T17:18:33.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:33 smithi162 ceph-mon[104668]: 1.8 scrub ok 2024-08-17T17:18:33.847 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:33.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:33.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:33.949 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:33.989 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:33.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:33.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:34.091 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:34.118 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:34.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:34.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:34.220 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:34.251 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:34.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:34.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:34.353 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:34.386 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:34.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:34.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:34.439 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 36 16 66983 66967 116.245 50.6875 0.0268019 0.00859628 2024-08-17T17:18:34.488 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:34.525 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:34.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:34.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:34.626 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:34.660 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:34.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:34.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:34.762 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:34.791 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:34.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:34.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:34.820 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:18:34.892 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:34.931 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:34.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:34.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:35.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:34 smithi063 ceph-mon[114756]: osdmap e1147: 8 total, 8 up, 8 in 2024-08-17T17:18:35.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:34 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3118384092' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:35.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:34 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:34 smithi063 ceph-mon[109843]: osdmap e1147: 8 total, 8 up, 8 in 2024-08-17T17:18:35.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:34 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3118384092' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:35.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:34 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:35.033 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:35.075 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:35.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:35.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:34 smithi162 ceph-mon[104668]: osdmap e1147: 8 total, 8 up, 8 in 2024-08-17T17:18:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:34 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3118384092' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:34 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:18:35.177 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:35.209 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:35.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:35.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:35.311 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:35.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:35.348 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:35.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:35.380 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:35.440 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 37 16 67894 67878 114.64 56.9375 0.00568757 0.00871332 2024-08-17T17:18:35.448 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:35.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:35.605 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:35.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:35.706 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:35.748 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:35.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:35.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:35.849 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:35.878 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:35.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:35.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:35.980 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:36.007 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:36.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:36.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:36.047 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:35 smithi063 ceph-mon[109843]: pgmap v1347: 99 pgs: 10 remapped+peering, 3 peering, 86 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 0 B/s, 2 objects/s recovering 2024-08-17T17:18:36.047 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:35 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:18:36.047 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:35 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:18:36.047 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:35 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:18:36.047 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:35 smithi063 ceph-mon[109843]: osdmap e1148: 8 total, 8 up, 8 in 2024-08-17T17:18:36.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:35 smithi063 ceph-mon[114756]: pgmap v1347: 99 pgs: 10 remapped+peering, 3 peering, 86 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 0 B/s, 2 objects/s recovering 2024-08-17T17:18:36.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:35 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:18:36.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:35 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:18:36.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:35 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:18:36.048 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:35 smithi063 ceph-mon[114756]: osdmap e1148: 8 total, 8 up, 8 in 2024-08-17T17:18:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:35 smithi162 ceph-mon[104668]: pgmap v1347: 99 pgs: 10 remapped+peering, 3 peering, 86 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 0 B/s, 2 objects/s recovering 2024-08-17T17:18:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:35 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:18:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:35 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:18:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:35 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:18:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:35 smithi162 ceph-mon[104668]: osdmap e1148: 8 total, 8 up, 8 in 2024-08-17T17:18:36.109 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:36.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:36.155 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:36.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:36.256 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:36.274 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:18:36.274 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:18:36.274 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.274 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:68566 7.6 e1148/1144 7:67d1c17c:::benchmark_data_smithi063_196002_object68565:head v 1148'4290, mlcod=0'0)", 2024-08-17T17:18:36.274 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:36.217292+0000", 2024-08-17T17:18:36.275 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.013801300000000001, 2024-08-17T17:18:36.275 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.013832093, 2024-08-17T17:18:36.275 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:36.275 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:36.275 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:18:36.275 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:36.275 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.275 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:36.275 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.217292+0000", 2024-08-17T17:18:36.275 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:36.275 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.276 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.276 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:36.276 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.217292+0000", 2024-08-17T17:18:36.276 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:36.276 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.276 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.276 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:36.276 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.217294+0000", 2024-08-17T17:18:36.276 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1519999999999999e-06 2024-08-17T17:18:36.277 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.277 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.277 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:36.277 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.217332+0000", 2024-08-17T17:18:36.277 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8707000000000001e-05 2024-08-17T17:18:36.277 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.277 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.277 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:36.277 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.217350+0000", 2024-08-17T17:18:36.277 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8045000000000001e-05 2024-08-17T17:18:36.277 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.277 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.278 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:36.278 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.217360+0000", 2024-08-17T17:18:36.278 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.594e-06 2024-08-17T17:18:36.278 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.278 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.278 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:36.278 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.218580+0000", 2024-08-17T17:18:36.278 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0012201340000000001 2024-08-17T17:18:36.278 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.278 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.278 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:36.279 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.218611+0000", 2024-08-17T17:18:36.279 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1338999999999999e-05 2024-08-17T17:18:36.279 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:36.279 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:36.279 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:36.279 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.279 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.279 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:68565 7.7 e1148/1144 7:e3732572:::benchmark_data_smithi063_196002_object68564:head v 1148'4133, mlcod=0'0)", 2024-08-17T17:18:36.279 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:36.217754+0000", 2024-08-17T17:18:36.279 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.013339314999999999, 2024-08-17T17:18:36.279 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.013436829000000001, 2024-08-17T17:18:36.279 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:36.279 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:36.280 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:18:36.280 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:36.280 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.280 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:36.280 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.217754+0000", 2024-08-17T17:18:36.280 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:36.280 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.280 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.280 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:36.280 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.217754+0000", 2024-08-17T17:18:36.280 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:36.280 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.280 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.280 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:36.281 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.217755+0000", 2024-08-17T17:18:36.281 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4499999999999996e-07 2024-08-17T17:18:36.281 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.281 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.281 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:36.281 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.217787+0000", 2024-08-17T17:18:36.281 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1677999999999997e-05 2024-08-17T17:18:36.281 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.281 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.281 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:36.281 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.217804+0000", 2024-08-17T17:18:36.281 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6651e-05 2024-08-17T17:18:36.281 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.282 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.282 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:36.282 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.217811+0000", 2024-08-17T17:18:36.282 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6660000000000008e-06 2024-08-17T17:18:36.282 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.282 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.282 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:36.282 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.217897+0000", 2024-08-17T17:18:36.282 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.5737999999999998e-05 2024-08-17T17:18:36.282 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.282 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.282 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:36.283 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.217937+0000", 2024-08-17T17:18:36.283 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.0225000000000002e-05 2024-08-17T17:18:36.283 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:36.283 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:36.283 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:36.283 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.283 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.283 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:68567 7.6 e1148/1144 7:6ac75666:::benchmark_data_smithi063_196002_object68566:head v 1148'4291, mlcod=0'0)", 2024-08-17T17:18:36.283 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:36.220248+0000", 2024-08-17T17:18:36.283 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.010845426, 2024-08-17T17:18:36.283 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.010999858, 2024-08-17T17:18:36.283 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:36.283 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:36.284 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:18:36.284 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:36.284 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.284 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:36.284 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.220248+0000", 2024-08-17T17:18:36.284 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:36.284 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.284 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.284 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:36.285 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.220248+0000", 2024-08-17T17:18:36.285 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:36.285 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.285 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.285 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:36.285 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.220249+0000", 2024-08-17T17:18:36.285 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2399999999999997e-07 2024-08-17T17:18:36.285 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.285 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.285 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:36.285 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.220306+0000", 2024-08-17T17:18:36.285 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6490000000000003e-05 2024-08-17T17:18:36.286 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.286 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.286 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:36.286 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.220316+0000", 2024-08-17T17:18:36.286 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0379e-05 2024-08-17T17:18:36.286 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.286 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.286 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:36.286 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.220324+0000", 2024-08-17T17:18:36.286 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5480000000000004e-06 2024-08-17T17:18:36.286 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.286 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.287 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:36.287 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.224276+0000", 2024-08-17T17:18:36.287 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0039527030000000001 2024-08-17T17:18:36.287 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.287 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.287 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:36.287 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.224351+0000", 2024-08-17T17:18:36.287 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4642999999999997e-05 2024-08-17T17:18:36.287 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:36.287 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:36.287 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:36.287 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.288 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.288 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:68571 7.6 e1148/1144 7:63461586:::benchmark_data_smithi063_196002_object68570:head v 1148'4292, mlcod=0'0)", 2024-08-17T17:18:36.288 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:36.223000+0000", 2024-08-17T17:18:36.288 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0080934000000000006, 2024-08-17T17:18:36.288 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0083021129999999999, 2024-08-17T17:18:36.288 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:36.288 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:36.288 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:18:36.288 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:36.288 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.288 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:36.288 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.223000+0000", 2024-08-17T17:18:36.288 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:36.289 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.289 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.289 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:36.289 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.223000+0000", 2024-08-17T17:18:36.289 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:36.289 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.289 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.289 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:36.289 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.223002+0000", 2024-08-17T17:18:36.289 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2619999999999999e-06 2024-08-17T17:18:36.290 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.290 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.290 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:36.290 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.223143+0000", 2024-08-17T17:18:36.290 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000141101 2024-08-17T17:18:36.290 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.290 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.290 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:36.290 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.223161+0000", 2024-08-17T17:18:36.290 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8199999999999999e-05 2024-08-17T17:18:36.290 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.290 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.291 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:36.291 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.223170+0000", 2024-08-17T17:18:36.291 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.5969999999999992e-06 2024-08-17T17:18:36.291 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.291 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.291 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:36.291 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.229930+0000", 2024-08-17T17:18:36.291 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.006760416 2024-08-17T17:18:36.291 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.291 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.291 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:36.291 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.229990+0000", 2024-08-17T17:18:36.291 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0213000000000001e-05 2024-08-17T17:18:36.292 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:36.292 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:36.292 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:36.292 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.292 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.292 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:68570 7.e e1148/1144)", 2024-08-17T17:18:36.292 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:36.225210+0000", 2024-08-17T17:18:36.292 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0058840760000000002, 2024-08-17T17:18:36.292 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0061400860000000003, 2024-08-17T17:18:36.292 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:36.292 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:36.292 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "queued for pg", 2024-08-17T17:18:36.293 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:36.293 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.293 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:36.293 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.225210+0000", 2024-08-17T17:18:36.293 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:36.293 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.294 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.294 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:36.294 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.225210+0000", 2024-08-17T17:18:36.294 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:36.294 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.295 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.295 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:36.295 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.225211+0000", 2024-08-17T17:18:36.295 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.08e-06 2024-08-17T17:18:36.295 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.295 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.295 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:36.295 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.225257+0000", 2024-08-17T17:18:36.295 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5791000000000003e-05 2024-08-17T17:18:36.295 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.295 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.295 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:36.295 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.225273+0000", 2024-08-17T17:18:36.296 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6898e-05 2024-08-17T17:18:36.296 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.296 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.296 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:36.296 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.225283+0000", 2024-08-17T17:18:36.296 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0410000000000003e-06 2024-08-17T17:18:36.296 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:36.296 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:36.296 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:36.296 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.297 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.297 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:68572 7.6 e1148/1144)", 2024-08-17T17:18:36.297 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:36.225235+0000", 2024-08-17T17:18:36.297 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0058591770000000001, 2024-08-17T17:18:36.297 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0061617169999999997, 2024-08-17T17:18:36.297 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:36.297 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:36.297 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "queued for pg", 2024-08-17T17:18:36.298 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:36.298 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.298 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:36.298 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.225235+0000", 2024-08-17T17:18:36.298 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:36.298 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.298 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.298 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:36.298 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.225235+0000", 2024-08-17T17:18:36.298 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:36.298 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.298 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.299 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:36.299 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.225235+0000", 2024-08-17T17:18:36.299 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.41e-07 2024-08-17T17:18:36.299 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.299 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.299 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:36.300 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.225356+0000", 2024-08-17T17:18:36.300 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012083600000000001 2024-08-17T17:18:36.300 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.300 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.300 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:36.300 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.225364+0000", 2024-08-17T17:18:36.300 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4449999999999996e-06 2024-08-17T17:18:36.300 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.300 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.301 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:36.301 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.225369+0000", 2024-08-17T17:18:36.301 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2140000000000004e-06 2024-08-17T17:18:36.301 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:36.301 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:36.301 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:36.301 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.301 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.301 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "MOSDPGPush(7.5 1148/1144 [PushOp(7:ab058105:::benchmark_data_smithi063_196002_object1687:head, version: 1132'99, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:ab058105:::benchmark_data_smithi063_196002_object1687:head@1132'99, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:18:36.301 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:36.230892+0000", 2024-08-17T17:18:36.301 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.000201666, 2024-08-17T17:18:36.301 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00057904400000000002, 2024-08-17T17:18:36.302 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:36.302 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:36.302 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:18:36.302 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:36.302 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.302 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:36.302 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.230892+0000", 2024-08-17T17:18:36.302 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:36.302 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.302 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.302 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:36.303 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.230892+0000", 2024-08-17T17:18:36.303 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:36.303 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.303 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.303 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:36.303 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.230893+0000", 2024-08-17T17:18:36.303 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.364e-06 2024-08-17T17:18:36.303 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.303 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.303 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:36.303 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.231283+0000", 2024-08-17T17:18:36.303 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000389842 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.231316+0000", 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.2209000000000003e-05 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.231334+0000", 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8134999999999998e-05 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:36.304 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:36.231438+0000", 2024-08-17T17:18:36.305 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000103849 2024-08-17T17:18:36.305 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:36.305 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:36.305 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:36.305 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:36.305 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-08-17T17:18:36.305 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 7 2024-08-17T17:18:36.305 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:18:36.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:36.306 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:36.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:36.407 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:36.441 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 38 15 68760 68745 113.047 54.1875 0.0158891 0.00883924 2024-08-17T17:18:36.450 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:36.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:36.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:36.552 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:36.592 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:36.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:36.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:36.694 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:36.733 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:36.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:36.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:36.835 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:36.879 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:36.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:36.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:36.981 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:37.017 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:37.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:37.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:37.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:36 smithi063 ceph-mon[114756]: 1.1 scrub starts 2024-08-17T17:18:37.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:36 smithi063 ceph-mon[114756]: 1.1 scrub ok 2024-08-17T17:18:37.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:36 smithi063 ceph-mon[109843]: 1.1 scrub starts 2024-08-17T17:18:37.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:36 smithi063 ceph-mon[109843]: 1.1 scrub ok 2024-08-17T17:18:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:36 smithi162 ceph-mon[104668]: 1.1 scrub starts 2024-08-17T17:18:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:36 smithi162 ceph-mon[104668]: 1.1 scrub ok 2024-08-17T17:18:37.119 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:37.147 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:37.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:37.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:37.248 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:37.276 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:37.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:37.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:37.378 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:37.405 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:37.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:37.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:37.441 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 39 16 69670 69654 111.605 56.8125 0.0258937 0.00895312 2024-08-17T17:18:37.507 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:37.553 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:37.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:37.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:37.656 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:37.695 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:37.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:37.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:37.797 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:37.826 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:37.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:37.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:37.928 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:37.957 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:37.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:37.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:38.059 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:38.087 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:38.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:38.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:37 smithi162 ceph-mon[104668]: pgmap v1349: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 10 remapped+peering, 3 peering, 85 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 30 MiB/s wr, 471 op/s; 4154/198342 objects degraded (2.094%); 0 B/s, 1 objects/s recovering 2024-08-17T17:18:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:37 smithi162 ceph-mon[104668]: Health check failed: Degraded data redundancy: 4154/198342 objects degraded (2.094%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:38.138 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:37 smithi063 ceph-mon[114756]: pgmap v1349: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 10 remapped+peering, 3 peering, 85 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 30 MiB/s wr, 471 op/s; 4154/198342 objects degraded (2.094%); 0 B/s, 1 objects/s recovering 2024-08-17T17:18:38.138 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:37 smithi063 ceph-mon[114756]: Health check failed: Degraded data redundancy: 4154/198342 objects degraded (2.094%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:38.138 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:37 smithi063 ceph-mon[109843]: pgmap v1349: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 10 remapped+peering, 3 peering, 85 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 30 MiB/s wr, 471 op/s; 4154/198342 objects degraded (2.094%); 0 B/s, 1 objects/s recovering 2024-08-17T17:18:38.138 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:37 smithi063 ceph-mon[109843]: Health check failed: Degraded data redundancy: 4154/198342 objects degraded (2.094%), 1 pg degraded (PG_DEGRADED) 2024-08-17T17:18:38.189 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:38.219 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:38.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:38.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:38.320 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:38.349 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:38.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:38.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:38.441 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:18:38.441156+0000 min lat: 0.00126452 max lat: 2.52879 avg lat: 0.00906175 2024-08-17T17:18:38.441 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:18:38.441 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 40 16 70606 70590 110.278 58.5 0.0272203 0.00906175 2024-08-17T17:18:38.452 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:38.475 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:38.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:38.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:38.577 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:38.605 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:38.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:38.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:38.655 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:18:38.655 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:18:38.655 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_2 2024-08-17T17:18:38.656 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:18:38.707 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:38.751 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:38.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:38.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:38.822 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:18:38.853 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:38.879 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:38.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:38.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:38.981 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:38 smithi063 ceph-mon[114756]: pgmap v1350: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 37 MiB/s wr, 581 op/s; 28763/206193 objects degraded (13.950%); 4053/206193 objects misplaced (1.966%); 1.4 MiB/s, 23 objects/s recovering 2024-08-17T17:18:39.030 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:39.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:39.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:39.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:38 smithi063 ceph-mon[109843]: pgmap v1350: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 37 MiB/s wr, 581 op/s; 28763/206193 objects degraded (13.950%); 4053/206193 objects misplaced (1.966%); 1.4 MiB/s, 23 objects/s recovering 2024-08-17T17:18:39.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:38 smithi162 ceph-mon[104668]: pgmap v1350: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 37 MiB/s wr, 581 op/s; 28763/206193 objects degraded (13.950%); 4053/206193 objects misplaced (1.966%); 1.4 MiB/s, 23 objects/s recovering 2024-08-17T17:18:39.132 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:39.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:39.193 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:39.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:39.294 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:39.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:39.339 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:39.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:39.439 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:39.440 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:39.441 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 41 16 71407 71391 108.809 50.0625 0.0219383 0.00918327 2024-08-17T17:18:39.450 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:18:39.466 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:39.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:39.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:39.475 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:18:39.475 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:18:39.513 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:18:39.514 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:18:39.550 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:18:39.550 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:18:39.567 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:39.586 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:18:39.586 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:18:39.594 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:39.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:39.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:39.622 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:18:39.622 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:18:39.661 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:18:39.662 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:18:39.695 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:39.696 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:18:39.696 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:18:39.720 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:39.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:39.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:39.737 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:18:39.737 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:18:39.822 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:39.848 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:39.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:39.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:39.950 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:39.976 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:39.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:39.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:39 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3221792649' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:39 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3221792649' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:40.078 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:40.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:39 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3221792649' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:40.134 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:40.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:40.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:40.236 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:40.263 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:40.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:40.264 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:40.365 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:40.392 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:40.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:40.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:40.441 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 42 16 72242 72226 107.461 52.1875 0.0127621 0.00929989 2024-08-17T17:18:40.495 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:40.537 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:40.547 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:40.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:40.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:40.649 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:40.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:40.706 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:40.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:40.807 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:40.845 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:40.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:40.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:40.947 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:40.994 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:40.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:40.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:41.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:40 smithi063 ceph-mon[109843]: pgmap v1351: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 37 MiB/s wr, 592 op/s; 28763/209469 objects degraded (13.731%); 4053/209469 objects misplaced (1.935%); 1.0 MiB/s, 17 objects/s recovering 2024-08-17T17:18:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:40 smithi063 ceph-mon[114756]: pgmap v1351: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 37 MiB/s wr, 592 op/s; 28763/209469 objects degraded (13.731%); 4053/209469 objects misplaced (1.935%); 1.0 MiB/s, 17 objects/s recovering 2024-08-17T17:18:41.097 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:41.136 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:41.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:41.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:41.238 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:41.267 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:41.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:41.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:41.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:40 smithi162 ceph-mon[104668]: pgmap v1351: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 37 MiB/s wr, 592 op/s; 28763/209469 objects degraded (13.731%); 4053/209469 objects misplaced (1.935%); 1.0 MiB/s, 17 objects/s recovering 2024-08-17T17:18:41.369 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:41.411 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:41.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:41.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:41.441 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 43 16 73134 73118 106.258 55.75 0.0131148 0.00940561 2024-08-17T17:18:41.513 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:41.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:41.559 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:41.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:41.615 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:41.660 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:41.722 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:41.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:41.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:41.824 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:41.856 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:41.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:41.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:41.957 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:41.985 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:41.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:41.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:42.086 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:42.133 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:42.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:42.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:42.235 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:42.261 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:42.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:42.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:42.362 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:42.387 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:42.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:42.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:42.441 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 44 16 73948 73932 105 50.875 0.00881843 0.00951762 2024-08-17T17:18:42.489 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:42.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:42.517 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:42.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:42.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:42 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 28763/215979 objects degraded (13.317%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:42.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:42 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 28763/215979 objects degraded (13.317%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:42.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:42 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 28763/215979 objects degraded (13.317%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:42.619 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:42.654 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:42.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:42.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:42.755 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:42.796 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:42.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:42.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:42.897 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:42.937 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:42.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:42.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:43.038 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:43.077 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:43.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:43.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:43.107 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:18:43.179 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:43.207 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:43.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:43.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:43.309 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:43.339 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:43.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:43.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:43.441 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:43.443 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 45 16 74817 74801 103.872 54.3125 0.00943388 0.00962042 2024-08-17T17:18:43.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:43.484 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:43.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:43.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:43 smithi063 ceph-mon[109843]: pgmap v1352: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s; 28763/215979 objects degraded (13.317%); 4053/215979 objects misplaced (1.877%); 943 KiB/s, 15 objects/s recovering 2024-08-17T17:18:43.530 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:18:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:18:43] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.51.0" 2024-08-17T17:18:43.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:43 smithi063 ceph-mon[114756]: pgmap v1352: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s; 28763/215979 objects degraded (13.317%); 4053/215979 objects misplaced (1.877%); 943 KiB/s, 15 objects/s recovering 2024-08-17T17:18:43.585 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:43.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:43 smithi162 ceph-mon[104668]: pgmap v1352: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 49 MiB/s wr, 781 op/s; 28763/215979 objects degraded (13.317%); 4053/215979 objects misplaced (1.877%); 943 KiB/s, 15 objects/s recovering 2024-08-17T17:18:43.614 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:43.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:43.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:43.715 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:43.746 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:43.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:43.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:43.847 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:43.886 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:43.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:43.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:43.988 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:44.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:44.030 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:44.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:44.131 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:44.161 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:44.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:44.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:44.263 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:44.289 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:44.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:44.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:44.393 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:44.442 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 46 16 75597 75581 102.673 48.75 0.0173746 0.00973279 2024-08-17T17:18:44.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:44.443 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:44.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:44.544 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:44.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:44.590 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:44.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:44.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:44 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:44.610 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:44 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:44.610 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:44 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:18:44.691 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:44.723 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:44.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:44.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:44.825 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:44.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:44.870 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:44.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:44.971 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:45.011 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:45.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:45.113 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:45.140 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:45.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:45.242 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:45.265 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:45.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:45.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:45 smithi162 ceph-mon[104668]: pgmap v1353: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s; 28763/219258 objects degraded (13.118%); 3900/219258 objects misplaced (1.779%); 1.7 MiB/s, 28 objects/s recovering 2024-08-17T17:18:45.367 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:45.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.414 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:45.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:45.442 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 47 16 76284 76268 101.402 42.9375 0.00960287 0.0098542 2024-08-17T17:18:45.516 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:45.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:45 smithi063 ceph-mon[109843]: pgmap v1353: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s; 28763/219258 objects degraded (13.118%); 3900/219258 objects misplaced (1.779%); 1.7 MiB/s, 28 objects/s recovering 2024-08-17T17:18:45.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:45 smithi063 ceph-mon[114756]: pgmap v1353: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s; 28763/219258 objects degraded (13.118%); 3900/219258 objects misplaced (1.779%); 1.7 MiB/s, 28 objects/s recovering 2024-08-17T17:18:45.557 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:45.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:45.564 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:18:45.564 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:18:45.660 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:45.689 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:45.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:45.765 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:18:45.766 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:18:45.791 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:45.819 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:45.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:45.921 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:45.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:45.953 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:45.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:46.054 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:46.083 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:46.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:46.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:46.185 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:46.215 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:46.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:46.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:46.317 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:46.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:46.362 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:46.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:46.442 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 48 16 77023 77007 100.252 46.1875 0.0237398 0.00996373 2024-08-17T17:18:46.463 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:46.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:46.503 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:46.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:46.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:46 smithi162 ceph-mon[104668]: from='client.44981 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:18:46.604 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:46.645 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:46.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:46.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:46.747 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:46.778 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:46.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:46.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:46.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:46 smithi063 ceph-mon[109843]: from='client.44981 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:18:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:46 smithi063 ceph-mon[114756]: from='client.44981 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:18:46.880 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:46.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:46.911 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:46.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:47.012 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:47.054 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:47.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:47.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:47.156 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:47.195 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:47.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:47.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:47.297 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:47.323 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:47.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:47.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:47.425 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:47.442 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 49 16 77915 77899 99.3437 55.75 0.0148788 0.0100597 2024-08-17T17:18:47.462 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:47.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:47.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:47.564 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:47.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:47 smithi162 ceph-mon[104668]: pgmap v1354: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 44 MiB/s wr, 700 op/s; 28763/222312 objects degraded (12.938%); 3900/222312 objects misplaced (1.754%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:18:47.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:18:47.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:47.609 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:47.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:47.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:47.711 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:47.753 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:47.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:47.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:47.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:47 smithi063 ceph-mon[114756]: pgmap v1354: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 44 MiB/s wr, 700 op/s; 28763/222312 objects degraded (12.938%); 3900/222312 objects misplaced (1.754%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:18:47.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:18:47.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:47.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:47 smithi063 ceph-mon[109843]: pgmap v1354: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 44 MiB/s wr, 700 op/s; 28763/222312 objects degraded (12.938%); 3900/222312 objects misplaced (1.754%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:18:47.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:18:47.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:18:47.855 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:47.884 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:47.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:47.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:47.985 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:48.026 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:48.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:48.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:48.128 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:48.175 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:48.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:48.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:48.277 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:48.306 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:48.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:48.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:48.408 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:48.431 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:18:48.443 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 50 16 78770 78754 98.4252 53.4375 0.0682474 0.0101509 2024-08-17T17:18:48.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:48.468 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:48.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:48.569 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:48.596 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:48.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:48.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:48.609 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:48 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 28763/222312 objects degraded (12.938%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:48.609 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:48 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 28763/222312 objects degraded (12.938%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:48.698 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:48.744 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:48.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:48.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:48.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:48 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 28763/222312 objects degraded (12.938%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:48.846 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:48.874 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:48.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:48.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:48.976 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:49.005 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:49.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:49.107 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:49.129 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1353,"stamp":"2024-08-17T17:18:44.160889+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4788912585,"num_objects":73086,"num_object_clones":0,"num_object_copies":219258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":28763,"num_objects_misplaced":3900,"num_objects_unfound":0,"num_objects_dirty":73086,"num_whiteouts":0,"num_read":210,"num_read_kb":453,"num_write":73241,"num_write_kb":4677128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":309,"num_bytes_recovered":19752365,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74757,"ondisk_log_size":74757,"up":297,"acting":289,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":308,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15796856,"kb_used_data":14047224,"kb_used_omap":199,"kb_used_meta":1749368,"kb_avail":734164360,"statfs":{"total":767960285184,"available":751784304640,"internally_reserved":0,"allocated":14384357376,"data_stored":14367507199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":1791353596},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,2,4,6,4,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":502554324,"num_objects":7666,"num_object_clones":0,"num_object_copies":22998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":28763,"num_objects_misplaced":3900,"num_objects_unfound":0,"num_objects_dirty":7666,"num_whiteouts":0,"num_read":-71,"num_read_kb":-152,"num_write":7627,"num_write_kb":490185,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":285,"num_bytes_recovered":18093053,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003257"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1375,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.824810+0000","last_change":"2024-08-17T17:07:45.684223+0000","last_active":"2024-08-17T17:18:34.824810+0000","last_peered":"2024-08-17T17:18:34.824810+0000","last_clean":"2024-08-17T17:18:34.824810+0000","last_became_active":"2024-08-17T17:07:45.683948+0000","last_became_peered":"2024-08-17T17:07:45.683948+0000","last_unstale":"2024-08-17T17:18:34.824810+0000","last_undegraded":"2024-08-17T17:18:34.824810+0000","last_fullsized":"2024-08-17T17:18:34.824810+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:07.677423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1074,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.803232+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:18:33.803232+0000","last_peered":"2024-08-17T17:18:33.803232+0000","last_clean":"2024-08-17T17:18:33.803232+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:18:33.803232+0000","last_undegraded":"2024-08-17T17:18:33.803232+0000","last_fullsized":"2024-08-17T17:18:33.803232+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1086,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.815846+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:18:33.815846+0000","last_peered":"2024-08-17T17:18:33.815846+0000","last_clean":"2024-08-17T17:18:33.815846+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:18:33.815846+0000","last_undegraded":"2024-08-17T17:18:33.815846+0000","last_fullsized":"2024-08-17T17:18:33.815846+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":941,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.802346+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:18:33.802346+0000","last_peered":"2024-08-17T17:18:33.802346+0000","last_clean":"2024-08-17T17:18:33.802346+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:18:33.802346+0000","last_undegraded":"2024-08-17T17:18:33.802346+0000","last_fullsized":"2024-08-17T17:18:33.802346+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":941,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.826077+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:18:34.826077+0000","last_peered":"2024-08-17T17:18:34.826077+0000","last_clean":"2024-08-17T17:18:34.826077+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:18:34.826077+0000","last_undegraded":"2024-08-17T17:18:34.826077+0000","last_fullsized":"2024-08-17T17:18:34.826077+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1057,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.802033+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:18:33.802033+0000","last_peered":"2024-08-17T17:18:33.802033+0000","last_clean":"2024-08-17T17:18:33.802033+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:18:33.802033+0000","last_undegraded":"2024-08-17T17:18:33.802033+0000","last_fullsized":"2024-08-17T17:18:33.802033+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1375,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.827203+0000","last_change":"2024-08-17T17:07:45.687030+0000","last_active":"2024-08-17T17:18:34.827203+0000","last_peered":"2024-08-17T17:18:34.827203+0000","last_clean":"2024-08-17T17:18:34.827203+0000","last_became_active":"2024-08-17T17:07:45.686610+0000","last_became_peered":"2024-08-17T17:07:45.686610+0000","last_unstale":"2024-08-17T17:18:34.827203+0000","last_undegraded":"2024-08-17T17:18:34.827203+0000","last_fullsized":"2024-08-17T17:18:34.827203+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:02.425845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1134,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.802399+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:18:33.802399+0000","last_peered":"2024-08-17T17:18:33.802399+0000","last_clean":"2024-08-17T17:18:33.802399+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:18:33.802399+0000","last_undegraded":"2024-08-17T17:18:33.802399+0000","last_fullsized":"2024-08-17T17:18:33.802399+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1148'4613","reported_seq":4641,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:42.156417+0000","last_change":"2024-08-17T17:17:52.927899+0000","last_active":"2024-08-17T17:18:42.156417+0000","last_peered":"2024-08-17T17:18:42.156417+0000","last_clean":"2024-08-17T17:18:42.156417+0000","last_became_active":"2024-08-17T17:17:52.927591+0000","last_became_peered":"2024-08-17T17:17:52.927591+0000","last_unstale":"2024-08-17T17:18:42.156417+0000","last_undegraded":"2024-08-17T17:18:42.156417+0000","last_fullsized":"2024-08-17T17:18:42.156417+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4613,"log_dups_size":0,"ondisk_log_size":4613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:31:21.681712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302317568,"num_objects":4613,"num_object_clones":0,"num_object_copies":13839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4613,"num_write_kb":295232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1100,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.803622+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:18:33.803622+0000","last_peered":"2024-08-17T17:18:33.803622+0000","last_clean":"2024-08-17T17:18:33.803622+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:18:33.803622+0000","last_undegraded":"2024-08-17T17:18:33.803622+0000","last_fullsized":"2024-08-17T17:18:33.803622+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1072,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.814963+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:18:33.814963+0000","last_peered":"2024-08-17T17:18:33.814963+0000","last_clean":"2024-08-17T17:18:33.814963+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:18:33.814963+0000","last_undegraded":"2024-08-17T17:18:33.814963+0000","last_fullsized":"2024-08-17T17:18:33.814963+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1148'4463","reported_seq":4512,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:39.562206+0000","last_change":"2024-08-17T17:17:52.311116+0000","last_active":"2024-08-17T17:18:39.562206+0000","last_peered":"2024-08-17T17:18:39.562206+0000","last_clean":"2024-08-17T17:18:39.562206+0000","last_became_active":"2024-08-17T17:17:52.310809+0000","last_became_peered":"2024-08-17T17:17:52.310809+0000","last_unstale":"2024-08-17T17:18:39.562206+0000","last_undegraded":"2024-08-17T17:18:39.562206+0000","last_fullsized":"2024-08-17T17:18:39.562206+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4463,"log_dups_size":0,"ondisk_log_size":4463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:06.780468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292487168,"num_objects":4463,"num_object_clones":0,"num_object_copies":13389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4463,"num_write_kb":285632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1075,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.809401+0000","last_change":"2024-08-17T17:06:39.638433+0000","last_active":"2024-08-17T17:18:33.809401+0000","last_peered":"2024-08-17T17:18:33.809401+0000","last_clean":"2024-08-17T17:18:33.809401+0000","last_became_active":"2024-08-17T17:06:39.638081+0000","last_became_peered":"2024-08-17T17:06:39.638081+0000","last_unstale":"2024-08-17T17:18:33.809401+0000","last_undegraded":"2024-08-17T17:18:33.809401+0000","last_fullsized":"2024-08-17T17:18:33.809401+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:13.558012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1106,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.812912+0000","last_change":"2024-08-17T17:07:45.693574+0000","last_active":"2024-08-17T17:18:33.812912+0000","last_peered":"2024-08-17T17:18:33.812912+0000","last_clean":"2024-08-17T17:18:33.812912+0000","last_became_active":"2024-08-17T17:07:45.693086+0000","last_became_peered":"2024-08-17T17:07:45.693086+0000","last_unstale":"2024-08-17T17:18:33.812912+0000","last_undegraded":"2024-08-17T17:18:33.812912+0000","last_fullsized":"2024-08-17T17:18:33.812912+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:31.888894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1148'4620","reported_seq":4659,"reported_epoch":1148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:18:42.093569+0000","last_change":"2024-08-17T17:18:33.036233+0000","last_active":"2024-08-17T17:18:42.093569+0000","last_peered":"2024-08-17T17:18:42.093569+0000","last_clean":"2024-08-17T17:18:30.513360+0000","last_became_active":"2024-08-17T17:18:33.023231+0000","last_became_peered":"2024-08-17T17:18:33.023231+0000","last_unstale":"2024-08-17T17:18:42.093569+0000","last_undegraded":"2024-08-17T17:18:32.652845+0000","last_fullsized":"2024-08-17T17:18:32.534874+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4620,"log_dups_size":0,"ondisk_log_size":4620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:21:08.396275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302776320,"num_objects":4620,"num_object_clones":0,"num_object_copies":13860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4115,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4620,"num_write_kb":295680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,1],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"1,4","objects":505},{"shards":"1,4,7","objects":4115}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1059,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.806280+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:18:33.806280+0000","last_peered":"2024-08-17T17:18:33.806280+0000","last_clean":"2024-08-17T17:18:33.806280+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:18:33.806280+0000","last_undegraded":"2024-08-17T17:18:33.806280+0000","last_fullsized":"2024-08-17T17:18:33.806280+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1088,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.802555+0000","last_change":"2024-08-17T17:06:39.629368+0000","last_active":"2024-08-17T17:18:33.802555+0000","last_peered":"2024-08-17T17:18:33.802555+0000","last_clean":"2024-08-17T17:18:33.802555+0000","last_became_active":"2024-08-17T17:06:39.628832+0000","last_became_peered":"2024-08-17T17:06:39.628832+0000","last_unstale":"2024-08-17T17:18:33.802555+0000","last_undegraded":"2024-08-17T17:18:33.802555+0000","last_fullsized":"2024-08-17T17:18:33.802555+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:54.063884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1148'4560","reported_seq":4609,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:41.185044+0000","last_change":"2024-08-17T17:17:52.311243+0000","last_active":"2024-08-17T17:18:41.185044+0000","last_peered":"2024-08-17T17:18:41.185044+0000","last_clean":"2024-08-17T17:18:41.185044+0000","last_became_active":"2024-08-17T17:17:52.310687+0000","last_became_peered":"2024-08-17T17:17:52.310687+0000","last_unstale":"2024-08-17T17:18:41.185044+0000","last_undegraded":"2024-08-17T17:18:41.185044+0000","last_fullsized":"2024-08-17T17:18:41.185044+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4560,"log_dups_size":0,"ondisk_log_size":4560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:23:53.141395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298844160,"num_objects":4560,"num_object_clones":0,"num_object_copies":13680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4560,"num_write_kb":291840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1088,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.815701+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:18:33.815701+0000","last_peered":"2024-08-17T17:18:33.815701+0000","last_clean":"2024-08-17T17:18:33.815701+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:18:33.815701+0000","last_undegraded":"2024-08-17T17:18:33.815701+0000","last_fullsized":"2024-08-17T17:18:33.815701+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":888,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.821481+0000","last_change":"2024-08-17T17:18:31.539133+0000","last_active":"2024-08-17T17:18:34.821481+0000","last_peered":"2024-08-17T17:18:34.821481+0000","last_clean":"2024-08-17T17:18:34.821481+0000","last_became_active":"2024-08-17T17:18:31.538669+0000","last_became_peered":"2024-08-17T17:18:31.538669+0000","last_unstale":"2024-08-17T17:18:34.821481+0000","last_undegraded":"2024-08-17T17:18:34.821481+0000","last_fullsized":"2024-08-17T17:18:34.821481+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:34.785446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1455,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.801837+0000","last_change":"2024-08-17T17:18:28.522790+0000","last_active":"2024-08-17T17:18:33.801837+0000","last_peered":"2024-08-17T17:18:33.801837+0000","last_clean":"2024-08-17T17:18:33.801837+0000","last_became_active":"2024-08-17T17:18:28.521573+0000","last_became_peered":"2024-08-17T17:18:28.521573+0000","last_unstale":"2024-08-17T17:18:33.801837+0000","last_undegraded":"2024-08-17T17:18:33.801837+0000","last_fullsized":"2024-08-17T17:18:33.801837+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":76,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":214679,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.f","version":"1148'4519","reported_seq":4571,"reported_epoch":1148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:18:39.563649+0000","last_change":"2024-08-17T17:18:33.026499+0000","last_active":"2024-08-17T17:18:39.563649+0000","last_peered":"2024-08-17T17:18:39.563649+0000","last_clean":"2024-08-17T17:18:30.502708+0000","last_became_active":"2024-08-17T17:18:33.018495+0000","last_became_peered":"2024-08-17T17:18:33.018495+0000","last_unstale":"2024-08-17T17:18:39.563649+0000","last_undegraded":"2024-08-17T17:18:32.555520+0000","last_fullsized":"2024-08-17T17:18:32.533633+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4519,"log_dups_size":0,"ondisk_log_size":4519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296157184,"num_objects":4519,"num_object_clones":0,"num_object_copies":13557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4154,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4519,"num_write_kb":289216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":4154},{"shards":"3,7","objects":365}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1410,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.824426+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:18:34.824426+0000","last_peered":"2024-08-17T17:18:34.824426+0000","last_clean":"2024-08-17T17:18:34.824426+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:18:34.824426+0000","last_undegraded":"2024-08-17T17:18:34.824426+0000","last_fullsized":"2024-08-17T17:18:34.824426+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1091,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.808661+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:18:33.808661+0000","last_peered":"2024-08-17T17:18:33.808661+0000","last_clean":"2024-08-17T17:18:33.808661+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:18:33.808661+0000","last_undegraded":"2024-08-17T17:18:33.808661+0000","last_fullsized":"2024-08-17T17:18:33.808661+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1413,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.813495+0000","last_change":"2024-08-17T17:18:32.884334+0000","last_active":"2024-08-17T17:18:33.813495+0000","last_peered":"2024-08-17T17:18:33.813495+0000","last_clean":"2024-08-17T17:18:33.813495+0000","last_became_active":"2024-08-17T17:18:28.520186+0000","last_became_peered":"2024-08-17T17:18:28.520186+0000","last_unstale":"2024-08-17T17:18:33.813495+0000","last_undegraded":"2024-08-17T17:18:33.813495+0000","last_fullsized":"2024-08-17T17:18:33.813495+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:32.884246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:32.884246+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:49.286656+0000","scrub_duration":0.001842677,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.e","version":"1148'4644","reported_seq":4694,"reported_epoch":1148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:18:41.188893+0000","last_change":"2024-08-17T17:18:33.023744+0000","last_active":"2024-08-17T17:18:41.188893+0000","last_peered":"2024-08-17T17:18:41.188893+0000","last_clean":"2024-08-17T17:18:30.515740+0000","last_became_active":"2024-08-17T17:18:33.016776+0000","last_became_peered":"2024-08-17T17:18:33.016776+0000","last_unstale":"2024-08-17T17:18:41.188893+0000","last_undegraded":"2024-08-17T17:18:32.554862+0000","last_fullsized":"2024-08-17T17:18:32.535040+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4644,"log_dups_size":0,"ondisk_log_size":4644,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304349184,"num_objects":4644,"num_object_clones":0,"num_object_copies":13932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4181,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4644,"num_write_kb":297216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":4181},{"shards":"2,5","objects":463}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1089,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.802671+0000","last_change":"2024-08-17T17:06:39.629619+0000","last_active":"2024-08-17T17:18:33.802671+0000","last_peered":"2024-08-17T17:18:33.802671+0000","last_clean":"2024-08-17T17:18:33.802671+0000","last_became_active":"2024-08-17T17:06:39.629295+0000","last_became_peered":"2024-08-17T17:06:39.629295+0000","last_unstale":"2024-08-17T17:18:33.802671+0000","last_undegraded":"2024-08-17T17:18:33.802671+0000","last_fullsized":"2024-08-17T17:18:33.802671+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:15.874423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1106,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.806143+0000","last_change":"2024-08-17T17:07:12.649178+0000","last_active":"2024-08-17T17:18:33.806143+0000","last_peered":"2024-08-17T17:18:33.806143+0000","last_clean":"2024-08-17T17:18:33.806143+0000","last_became_active":"2024-08-17T17:07:12.648844+0000","last_became_peered":"2024-08-17T17:07:12.648844+0000","last_unstale":"2024-08-17T17:18:33.806143+0000","last_undegraded":"2024-08-17T17:18:33.806143+0000","last_fullsized":"2024-08-17T17:18:33.806143+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:12:06.307736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1148'4585","reported_seq":4621,"reported_epoch":1148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:18:42.063870+0000","last_change":"2024-08-17T17:18:33.033028+0000","last_active":"2024-08-17T17:18:42.063870+0000","last_peered":"2024-08-17T17:18:42.063870+0000","last_clean":"2024-08-17T17:18:30.508341+0000","last_became_active":"2024-08-17T17:18:33.019697+0000","last_became_peered":"2024-08-17T17:18:33.019697+0000","last_unstale":"2024-08-17T17:18:42.063870+0000","last_undegraded":"2024-08-17T17:18:32.562771+0000","last_fullsized":"2024-08-17T17:18:32.533408+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4585,"log_dups_size":0,"ondisk_log_size":4585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:58:13.699201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300482560,"num_objects":4585,"num_object_clones":0,"num_object_copies":13755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4070,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4585,"num_write_kb":293440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":4070},{"shards":"1,6","objects":515}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1092,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.812978+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:18:33.812978+0000","last_peered":"2024-08-17T17:18:33.812978+0000","last_clean":"2024-08-17T17:18:33.812978+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:18:33.812978+0000","last_undegraded":"2024-08-17T17:18:33.812978+0000","last_fullsized":"2024-08-17T17:18:33.812978+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1091,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.806724+0000","last_change":"2024-08-17T17:06:39.627678+0000","last_active":"2024-08-17T17:18:33.806724+0000","last_peered":"2024-08-17T17:18:33.806724+0000","last_clean":"2024-08-17T17:18:33.806724+0000","last_became_active":"2024-08-17T17:06:39.627226+0000","last_became_peered":"2024-08-17T17:06:39.627226+0000","last_unstale":"2024-08-17T17:18:33.806724+0000","last_undegraded":"2024-08-17T17:18:33.806724+0000","last_fullsized":"2024-08-17T17:18:33.806724+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:01.868244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1413,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.814967+0000","last_change":"2024-08-17T17:18:31.850829+0000","last_active":"2024-08-17T17:18:33.814967+0000","last_peered":"2024-08-17T17:18:33.814967+0000","last_clean":"2024-08-17T17:18:33.814967+0000","last_became_active":"2024-08-17T17:18:28.520946+0000","last_became_peered":"2024-08-17T17:18:28.520946+0000","last_unstale":"2024-08-17T17:18:33.814967+0000","last_undegraded":"2024-08-17T17:18:33.814967+0000","last_fullsized":"2024-08-17T17:18:33.814967+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:31.850745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:31.850745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:17.218084+0000","scrub_duration":0.001317447,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.c","version":"1148'4539","reported_seq":4567,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:42.007257+0000","last_change":"2024-08-17T17:17:52.309565+0000","last_active":"2024-08-17T17:18:42.007257+0000","last_peered":"2024-08-17T17:18:42.007257+0000","last_clean":"2024-08-17T17:18:42.007257+0000","last_became_active":"2024-08-17T17:17:52.309298+0000","last_became_peered":"2024-08-17T17:17:52.309298+0000","last_unstale":"2024-08-17T17:18:42.007257+0000","last_undegraded":"2024-08-17T17:18:42.007257+0000","last_fullsized":"2024-08-17T17:18:42.007257+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4539,"log_dups_size":0,"ondisk_log_size":4539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:27:31.251993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297467904,"num_objects":4539,"num_object_clones":0,"num_object_copies":13617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4539,"num_write_kb":290496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1378,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.822236+0000","last_change":"2024-08-17T17:07:45.680458+0000","last_active":"2024-08-17T17:18:34.822236+0000","last_peered":"2024-08-17T17:18:34.822236+0000","last_clean":"2024-08-17T17:18:34.822236+0000","last_became_active":"2024-08-17T17:07:45.680221+0000","last_became_peered":"2024-08-17T17:07:45.680221+0000","last_unstale":"2024-08-17T17:18:34.822236+0000","last_undegraded":"2024-08-17T17:18:34.822236+0000","last_fullsized":"2024-08-17T17:18:34.822236+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:55.083962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1400,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.819652+0000","last_change":"2024-08-17T17:07:45.682866+0000","last_active":"2024-08-17T17:18:34.819652+0000","last_peered":"2024-08-17T17:18:34.819652+0000","last_clean":"2024-08-17T17:18:34.819652+0000","last_became_active":"2024-08-17T17:07:45.682554+0000","last_became_peered":"2024-08-17T17:07:45.682554+0000","last_unstale":"2024-08-17T17:18:34.819652+0000","last_undegraded":"2024-08-17T17:18:34.819652+0000","last_fullsized":"2024-08-17T17:18:34.819652+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:25.975914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.6","version":"1148'4625","reported_seq":4659,"reported_epoch":1148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:18:42.100153+0000","last_change":"2024-08-17T17:18:33.028972+0000","last_active":"2024-08-17T17:18:42.100153+0000","last_peered":"2024-08-17T17:18:42.100153+0000","last_clean":"2024-08-17T17:18:30.519687+0000","last_became_active":"2024-08-17T17:18:33.019311+0000","last_became_peered":"2024-08-17T17:18:33.019311+0000","last_unstale":"2024-08-17T17:18:42.100153+0000","last_undegraded":"2024-08-17T17:18:32.564602+0000","last_fullsized":"2024-08-17T17:18:32.537765+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4625,"log_dups_size":0,"ondisk_log_size":4625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:25:54.623777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303104000,"num_objects":4625,"num_object_clones":0,"num_object_copies":13875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4122,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4625,"num_write_kb":296000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,3],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"3,5,7","objects":4122},{"shards":"3,7","objects":503}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1467,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.817747+0000","last_change":"2024-08-17T17:18:33.814375+0000","last_active":"2024-08-17T17:18:33.817747+0000","last_peered":"2024-08-17T17:18:33.817747+0000","last_clean":"2024-08-17T17:18:33.817747+0000","last_became_active":"2024-08-17T17:18:33.813208+0000","last_became_peered":"2024-08-17T17:18:33.813208+0000","last_unstale":"2024-08-17T17:18:33.817747+0000","last_undegraded":"2024-08-17T17:18:33.817747+0000","last_fullsized":"2024-08-17T17:18:33.817747+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:25.057675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:25.057675+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:20:45.235879+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1102,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.816542+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:18:33.816542+0000","last_peered":"2024-08-17T17:18:33.816542+0000","last_clean":"2024-08-17T17:18:33.816542+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:18:33.816542+0000","last_undegraded":"2024-08-17T17:18:33.816542+0000","last_fullsized":"2024-08-17T17:18:33.816542+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1357,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.822088+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:18:34.822088+0000","last_peered":"2024-08-17T17:18:34.822088+0000","last_clean":"2024-08-17T17:18:34.822088+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:18:34.822088+0000","last_undegraded":"2024-08-17T17:18:34.822088+0000","last_fullsized":"2024-08-17T17:18:34.822088+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1073,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.808041+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:18:33.808041+0000","last_peered":"2024-08-17T17:18:33.808041+0000","last_clean":"2024-08-17T17:18:33.808041+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:18:33.808041+0000","last_undegraded":"2024-08-17T17:18:33.808041+0000","last_fullsized":"2024-08-17T17:18:33.808041+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":660,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.804001+0000","last_change":"2024-08-17T17:18:28.524571+0000","last_active":"2024-08-17T17:18:33.804001+0000","last_peered":"2024-08-17T17:18:33.804001+0000","last_clean":"2024-08-17T17:18:33.804001+0000","last_became_active":"2024-08-17T17:18:28.524280+0000","last_became_peered":"2024-08-17T17:18:28.524280+0000","last_unstale":"2024-08-17T17:18:33.804001+0000","last_undegraded":"2024-08-17T17:18:33.804001+0000","last_fullsized":"2024-08-17T17:18:33.804001+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1148'4670","reported_seq":5275,"reported_epoch":1148,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:18:42.184487+0000","last_change":"2024-08-17T17:18:33.014213+0000","last_active":"2024-08-17T17:18:42.184487+0000","last_peered":"2024-08-17T17:18:42.184487+0000","last_clean":"2024-08-17T17:18:30.511218+0000","last_became_active":"2024-08-17T17:18:32.994278+0000","last_became_peered":"2024-08-17T17:18:32.994278+0000","last_unstale":"2024-08-17T17:18:42.184487+0000","last_undegraded":"2024-08-17T17:18:42.184487+0000","last_fullsized":"2024-08-17T17:18:32.535826+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4670,"log_dups_size":0,"ondisk_log_size":4670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:46.545290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306053120,"num_objects":4670,"num_object_clones":0,"num_object_copies":14010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3900,"num_objects_unfound":0,"num_objects_dirty":4670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4670,"num_write_kb":298880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":284,"num_bytes_recovered":18612224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"1,2,7","objects":3900},{"shards":"2,7","objects":770}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1120,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.801799+0000","last_change":"2024-08-17T17:18:31.539042+0000","last_active":"2024-08-17T17:18:33.801799+0000","last_peered":"2024-08-17T17:18:33.801799+0000","last_clean":"2024-08-17T17:18:33.801799+0000","last_became_active":"2024-08-17T17:18:31.537879+0000","last_became_peered":"2024-08-17T17:18:31.537879+0000","last_unstale":"2024-08-17T17:18:33.801799+0000","last_undegraded":"2024-08-17T17:18:33.801799+0000","last_fullsized":"2024-08-17T17:18:33.801799+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:36:27.859975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":941,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.812146+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:18:33.812146+0000","last_peered":"2024-08-17T17:18:33.812146+0000","last_clean":"2024-08-17T17:18:33.812146+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:18:33.812146+0000","last_undegraded":"2024-08-17T17:18:33.812146+0000","last_fullsized":"2024-08-17T17:18:33.812146+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":847,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.816681+0000","last_change":"2024-08-17T17:18:31.536646+0000","last_active":"2024-08-17T17:18:33.816681+0000","last_peered":"2024-08-17T17:18:33.816681+0000","last_clean":"2024-08-17T17:18:33.816681+0000","last_became_active":"2024-08-17T17:18:31.536326+0000","last_became_peered":"2024-08-17T17:18:31.536326+0000","last_unstale":"2024-08-17T17:18:33.816681+0000","last_undegraded":"2024-08-17T17:18:33.816681+0000","last_fullsized":"2024-08-17T17:18:33.816681+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:11:03.952344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1472,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.817762+0000","last_change":"2024-08-17T17:18:33.817028+0000","last_active":"2024-08-17T17:18:33.817762+0000","last_peered":"2024-08-17T17:18:33.817762+0000","last_clean":"2024-08-17T17:18:33.817762+0000","last_became_active":"2024-08-17T17:18:33.816660+0000","last_became_peered":"2024-08-17T17:18:33.816660+0000","last_unstale":"2024-08-17T17:18:33.817762+0000","last_undegraded":"2024-08-17T17:18:33.817762+0000","last_fullsized":"2024-08-17T17:18:33.817762+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":214680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.4","version":"1148'4646","reported_seq":4682,"reported_epoch":1148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:18:42.092239+0000","last_change":"2024-08-17T17:18:33.039455+0000","last_active":"2024-08-17T17:18:42.092239+0000","last_peered":"2024-08-17T17:18:42.092239+0000","last_clean":"2024-08-17T17:18:30.520652+0000","last_became_active":"2024-08-17T17:18:33.021672+0000","last_became_peered":"2024-08-17T17:18:33.021672+0000","last_unstale":"2024-08-17T17:18:42.092239+0000","last_undegraded":"2024-08-17T17:18:32.566380+0000","last_fullsized":"2024-08-17T17:18:32.534773+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4646,"log_dups_size":0,"ondisk_log_size":4646,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:43:13.654277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304480256,"num_objects":4646,"num_object_clones":0,"num_object_copies":13938,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4178,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4646,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4646,"num_write_kb":297344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,1],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"1,4","objects":468},{"shards":"1,4,6","objects":4178}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1454,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.820239+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:18:34.820239+0000","last_peered":"2024-08-17T17:18:34.820239+0000","last_clean":"2024-08-17T17:18:34.820239+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:18:34.820239+0000","last_undegraded":"2024-08-17T17:18:34.820239+0000","last_fullsized":"2024-08-17T17:18:34.820239+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":832,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.819382+0000","last_change":"2024-08-17T17:15:38.610415+0000","last_active":"2024-08-17T17:18:34.819382+0000","last_peered":"2024-08-17T17:18:34.819382+0000","last_clean":"2024-08-17T17:18:34.819382+0000","last_became_active":"2024-08-17T17:15:38.610083+0000","last_became_peered":"2024-08-17T17:15:38.610083+0000","last_unstale":"2024-08-17T17:18:34.819382+0000","last_undegraded":"2024-08-17T17:18:34.819382+0000","last_fullsized":"2024-08-17T17:18:34.819382+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:12.007494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":767,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.816094+0000","last_change":"2024-08-17T17:18:31.534212+0000","last_active":"2024-08-17T17:18:33.816094+0000","last_peered":"2024-08-17T17:18:33.816094+0000","last_clean":"2024-08-17T17:18:33.816094+0000","last_became_active":"2024-08-17T17:18:31.533792+0000","last_became_peered":"2024-08-17T17:18:31.533792+0000","last_unstale":"2024-08-17T17:18:33.816094+0000","last_undegraded":"2024-08-17T17:18:33.816094+0000","last_fullsized":"2024-08-17T17:18:33.816094+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:13:01.220425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1430,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:35.916405+0000","last_change":"2024-08-17T17:18:35.916482+0000","last_active":"2024-08-17T17:18:35.916405+0000","last_peered":"2024-08-17T17:18:35.916405+0000","last_clean":"2024-08-17T17:18:35.916405+0000","last_became_active":"2024-08-17T17:18:34.827747+0000","last_became_peered":"2024-08-17T17:18:34.827747+0000","last_unstale":"2024-08-17T17:18:35.916405+0000","last_undegraded":"2024-08-17T17:18:35.916405+0000","last_fullsized":"2024-08-17T17:18:35.916405+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:35.916381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:35.916381+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:28.492622+0000","scrub_duration":0.005340521,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"1148'4457","reported_seq":4491,"reported_epoch":1148,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:18:42.058903+0000","last_change":"2024-08-17T17:18:33.000265+0000","last_active":"2024-08-17T17:18:42.058903+0000","last_peered":"2024-08-17T17:18:42.058903+0000","last_clean":"2024-08-17T17:18:30.516571+0000","last_became_active":"2024-08-17T17:18:32.993078+0000","last_became_peered":"2024-08-17T17:18:32.993078+0000","last_unstale":"2024-08-17T17:18:42.058903+0000","last_undegraded":"2024-08-17T17:18:32.564751+0000","last_fullsized":"2024-08-17T17:18:32.533735+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4457,"log_dups_size":0,"ondisk_log_size":4457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:57.421996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292093952,"num_objects":4457,"num_object_clones":0,"num_object_copies":13371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3943,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4457,"num_write_kb":285248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,3],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":3943},{"shards":"3,6","objects":514}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1103,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.816445+0000","last_change":"2024-08-17T17:06:39.628745+0000","last_active":"2024-08-17T17:18:33.816445+0000","last_peered":"2024-08-17T17:18:33.816445+0000","last_clean":"2024-08-17T17:18:33.816445+0000","last_became_active":"2024-08-17T17:06:39.628144+0000","last_became_peered":"2024-08-17T17:06:39.628144+0000","last_unstale":"2024-08-17T17:18:33.816445+0000","last_undegraded":"2024-08-17T17:18:33.816445+0000","last_fullsized":"2024-08-17T17:18:33.816445+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:14.740936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1111,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.826631+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:18:34.826631+0000","last_peered":"2024-08-17T17:18:34.826631+0000","last_clean":"2024-08-17T17:18:34.826631+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:18:34.826631+0000","last_undegraded":"2024-08-17T17:18:34.826631+0000","last_fullsized":"2024-08-17T17:18:34.826631+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":908,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.819492+0000","last_change":"2024-08-17T17:15:38.608764+0000","last_active":"2024-08-17T17:18:34.819492+0000","last_peered":"2024-08-17T17:18:34.819492+0000","last_clean":"2024-08-17T17:18:34.819492+0000","last_became_active":"2024-08-17T17:15:38.608353+0000","last_became_peered":"2024-08-17T17:15:38.608353+0000","last_unstale":"2024-08-17T17:18:34.819492+0000","last_undegraded":"2024-08-17T17:18:34.819492+0000","last_fullsized":"2024-08-17T17:18:34.819492+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:42.873441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":660,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.802515+0000","last_change":"2024-08-17T17:18:28.527058+0000","last_active":"2024-08-17T17:18:33.802515+0000","last_peered":"2024-08-17T17:18:33.802515+0000","last_clean":"2024-08-17T17:18:33.802515+0000","last_became_active":"2024-08-17T17:18:28.526439+0000","last_became_peered":"2024-08-17T17:18:28.526439+0000","last_unstale":"2024-08-17T17:18:33.802515+0000","last_undegraded":"2024-08-17T17:18:33.802515+0000","last_fullsized":"2024-08-17T17:18:33.802515+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.1","version":"1148'4535","reported_seq":4563,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:42.118854+0000","last_change":"2024-08-17T17:17:52.309484+0000","last_active":"2024-08-17T17:18:42.118854+0000","last_peered":"2024-08-17T17:18:42.118854+0000","last_clean":"2024-08-17T17:18:42.118854+0000","last_became_active":"2024-08-17T17:17:52.308991+0000","last_became_peered":"2024-08-17T17:17:52.308991+0000","last_unstale":"2024-08-17T17:18:42.118854+0000","last_undegraded":"2024-08-17T17:18:42.118854+0000","last_fullsized":"2024-08-17T17:18:42.118854+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4535,"log_dups_size":0,"ondisk_log_size":4535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:40.953569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297205760,"num_objects":4535,"num_object_clones":0,"num_object_copies":13605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4535,"num_write_kb":290240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1104,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.817260+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:18:34.817260+0000","last_peered":"2024-08-17T17:18:34.817260+0000","last_clean":"2024-08-17T17:18:34.817260+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:18:34.817260+0000","last_undegraded":"2024-08-17T17:18:34.817260+0000","last_fullsized":"2024-08-17T17:18:34.817260+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":776,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.803080+0000","last_change":"2024-08-17T17:18:31.537256+0000","last_active":"2024-08-17T17:18:33.803080+0000","last_peered":"2024-08-17T17:18:33.803080+0000","last_clean":"2024-08-17T17:18:33.803080+0000","last_became_active":"2024-08-17T17:18:31.536194+0000","last_became_peered":"2024-08-17T17:18:31.536194+0000","last_unstale":"2024-08-17T17:18:33.803080+0000","last_undegraded":"2024-08-17T17:18:33.803080+0000","last_fullsized":"2024-08-17T17:18:33.803080+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:46.453717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":718,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.815108+0000","last_change":"2024-08-17T17:15:38.609480+0000","last_active":"2024-08-17T17:18:33.815108+0000","last_peered":"2024-08-17T17:18:33.815108+0000","last_clean":"2024-08-17T17:18:33.815108+0000","last_became_active":"2024-08-17T17:15:38.609120+0000","last_became_peered":"2024-08-17T17:15:38.609120+0000","last_unstale":"2024-08-17T17:18:33.815108+0000","last_undegraded":"2024-08-17T17:18:33.815108+0000","last_fullsized":"2024-08-17T17:18:33.815108+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:11:03.113425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":677,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.831943+0000","last_change":"2024-08-17T17:18:34.831943+0000","last_active":"2024-08-17T17:18:34.831943+0000","last_peered":"2024-08-17T17:18:34.831943+0000","last_clean":"2024-08-17T17:18:34.831943+0000","last_became_active":"2024-08-17T17:18:34.831498+0000","last_became_peered":"2024-08-17T17:18:34.831498+0000","last_unstale":"2024-08-17T17:18:34.831943+0000","last_undegraded":"2024-08-17T17:18:34.831943+0000","last_fullsized":"2024-08-17T17:18:34.831943+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.0","version":"1148'4639","reported_seq":4667,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:42.118518+0000","last_change":"2024-08-17T17:17:52.927073+0000","last_active":"2024-08-17T17:18:42.118518+0000","last_peered":"2024-08-17T17:18:42.118518+0000","last_clean":"2024-08-17T17:18:42.118518+0000","last_became_active":"2024-08-17T17:17:52.926643+0000","last_became_peered":"2024-08-17T17:17:52.926643+0000","last_unstale":"2024-08-17T17:18:42.118518+0000","last_undegraded":"2024-08-17T17:18:42.118518+0000","last_fullsized":"2024-08-17T17:18:42.118518+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4639,"log_dups_size":0,"ondisk_log_size":4639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:18.661591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304021504,"num_objects":4639,"num_object_clones":0,"num_object_copies":13917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4639,"num_write_kb":296896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":831,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.819318+0000","last_change":"2024-08-17T17:15:38.609837+0000","last_active":"2024-08-17T17:18:34.819318+0000","last_peered":"2024-08-17T17:18:34.819318+0000","last_clean":"2024-08-17T17:18:34.819318+0000","last_became_active":"2024-08-17T17:15:38.609474+0000","last_became_peered":"2024-08-17T17:15:38.609474+0000","last_unstale":"2024-08-17T17:18:34.819318+0000","last_undegraded":"2024-08-17T17:18:34.819318+0000","last_fullsized":"2024-08-17T17:18:34.819318+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:31:42.450582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1104,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.806351+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:18:33.806351+0000","last_peered":"2024-08-17T17:18:33.806351+0000","last_clean":"2024-08-17T17:18:33.806351+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:18:33.806351+0000","last_undegraded":"2024-08-17T17:18:33.806351+0000","last_fullsized":"2024-08-17T17:18:33.806351+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":696,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.816470+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:18:33.816470+0000","last_peered":"2024-08-17T17:18:33.816470+0000","last_clean":"2024-08-17T17:18:33.816470+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:18:33.816470+0000","last_undegraded":"2024-08-17T17:18:33.816470+0000","last_fullsized":"2024-08-17T17:18:33.816470+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1401,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.806302+0000","last_change":"2024-08-17T17:18:28.526894+0000","last_active":"2024-08-17T17:18:33.806302+0000","last_peered":"2024-08-17T17:18:33.806302+0000","last_clean":"2024-08-17T17:18:33.806302+0000","last_became_active":"2024-08-17T17:18:28.526484+0000","last_became_peered":"2024-08-17T17:18:28.526484+0000","last_unstale":"2024-08-17T17:18:33.806302+0000","last_undegraded":"2024-08-17T17:18:33.806302+0000","last_fullsized":"2024-08-17T17:18:33.806302+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.3","version":"1148'4434","reported_seq":4483,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:39.616063+0000","last_change":"2024-08-17T17:17:52.310380+0000","last_active":"2024-08-17T17:18:39.616063+0000","last_peered":"2024-08-17T17:18:39.616063+0000","last_clean":"2024-08-17T17:18:39.616063+0000","last_became_active":"2024-08-17T17:17:52.310073+0000","last_became_peered":"2024-08-17T17:17:52.310073+0000","last_unstale":"2024-08-17T17:18:39.616063+0000","last_undegraded":"2024-08-17T17:18:39.616063+0000","last_fullsized":"2024-08-17T17:18:39.616063+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4434,"log_dups_size":0,"ondisk_log_size":4434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:26:56.341767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290586624,"num_objects":4434,"num_object_clones":0,"num_object_copies":13302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4434,"num_write_kb":283776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1411,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.827435+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:18:34.827435+0000","last_peered":"2024-08-17T17:18:34.827435+0000","last_clean":"2024-08-17T17:18:34.827435+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:18:34.827435+0000","last_undegraded":"2024-08-17T17:18:34.827435+0000","last_fullsized":"2024-08-17T17:18:34.827435+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":728,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.803717+0000","last_change":"2024-08-17T17:15:38.609543+0000","last_active":"2024-08-17T17:18:33.803717+0000","last_peered":"2024-08-17T17:18:33.803717+0000","last_clean":"2024-08-17T17:18:33.803717+0000","last_became_active":"2024-08-17T17:15:38.609238+0000","last_became_peered":"2024-08-17T17:15:38.609238+0000","last_unstale":"2024-08-17T17:18:33.803717+0000","last_undegraded":"2024-08-17T17:18:33.803717+0000","last_fullsized":"2024-08-17T17:18:33.803717+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:32.029842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1400,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.822082+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:18:34.822082+0000","last_peered":"2024-08-17T17:18:34.822082+0000","last_clean":"2024-08-17T17:18:34.822082+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:18:34.822082+0000","last_undegraded":"2024-08-17T17:18:34.822082+0000","last_fullsized":"2024-08-17T17:18:34.822082+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1415,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.824675+0000","last_change":"2024-08-17T17:18:29.390118+0000","last_active":"2024-08-17T17:18:34.824675+0000","last_peered":"2024-08-17T17:18:34.824675+0000","last_clean":"2024-08-17T17:18:34.824675+0000","last_became_active":"2024-08-17T17:18:28.528874+0000","last_became_peered":"2024-08-17T17:18:28.528874+0000","last_unstale":"2024-08-17T17:18:34.824675+0000","last_undegraded":"2024-08-17T17:18:34.824675+0000","last_fullsized":"2024-08-17T17:18:34.824675+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:29.390003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:29.390003+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:27.511820+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1148'4510","reported_seq":4559,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:39.579019+0000","last_change":"2024-08-17T17:17:52.928818+0000","last_active":"2024-08-17T17:18:39.579019+0000","last_peered":"2024-08-17T17:18:39.579019+0000","last_clean":"2024-08-17T17:18:39.579019+0000","last_became_active":"2024-08-17T17:17:52.928511+0000","last_became_peered":"2024-08-17T17:17:52.928511+0000","last_unstale":"2024-08-17T17:18:39.579019+0000","last_undegraded":"2024-08-17T17:18:39.579019+0000","last_fullsized":"2024-08-17T17:18:39.579019+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":4510,"log_dups_size":0,"ondisk_log_size":4510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:54:34.862989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295567360,"num_objects":4510,"num_object_clones":0,"num_object_copies":13530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4510,"num_write_kb":288640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":89,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.806579+0000","last_change":"2024-08-17T17:18:31.553838+0000","last_active":"2024-08-17T17:18:33.806579+0000","last_peered":"2024-08-17T17:18:33.806579+0000","last_clean":"2024-08-17T17:18:33.806579+0000","last_became_active":"2024-08-17T17:18:31.537264+0000","last_became_peered":"2024-08-17T17:18:31.537264+0000","last_unstale":"2024-08-17T17:18:33.806579+0000","last_undegraded":"2024-08-17T17:18:33.806579+0000","last_fullsized":"2024-08-17T17:18:33.806579+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:26.972836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1101,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.804831+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:18:33.804831+0000","last_peered":"2024-08-17T17:18:33.804831+0000","last_clean":"2024-08-17T17:18:33.804831+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:18:33.804831+0000","last_undegraded":"2024-08-17T17:18:33.804831+0000","last_fullsized":"2024-08-17T17:18:33.804831+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1077,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.827886+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:18:34.827886+0000","last_peered":"2024-08-17T17:18:34.827886+0000","last_clean":"2024-08-17T17:18:34.827886+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:18:34.827886+0000","last_undegraded":"2024-08-17T17:18:34.827886+0000","last_fullsized":"2024-08-17T17:18:34.827886+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1090,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.808404+0000","last_change":"2024-08-17T17:06:39.627594+0000","last_active":"2024-08-17T17:18:33.808404+0000","last_peered":"2024-08-17T17:18:33.808404+0000","last_clean":"2024-08-17T17:18:33.808404+0000","last_became_active":"2024-08-17T17:06:39.627105+0000","last_became_peered":"2024-08-17T17:06:39.627105+0000","last_unstale":"2024-08-17T17:18:33.808404+0000","last_undegraded":"2024-08-17T17:18:33.808404+0000","last_fullsized":"2024-08-17T17:18:33.808404+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:30.105070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1099,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.808799+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:18:33.808799+0000","last_peered":"2024-08-17T17:18:33.808799+0000","last_clean":"2024-08-17T17:18:33.808799+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:18:33.808799+0000","last_undegraded":"2024-08-17T17:18:33.808799+0000","last_fullsized":"2024-08-17T17:18:33.808799+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":31,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.815037+0000","last_change":"2024-08-17T17:17:47.283984+0000","last_active":"2024-08-17T17:18:33.815037+0000","last_peered":"2024-08-17T17:18:33.815037+0000","last_clean":"2024-08-17T17:18:33.815037+0000","last_became_active":"2024-08-17T17:17:47.283270+0000","last_became_peered":"2024-08-17T17:17:47.283270+0000","last_unstale":"2024-08-17T17:18:33.815037+0000","last_undegraded":"2024-08-17T17:18:33.815037+0000","last_fullsized":"2024-08-17T17:18:33.815037+0000","mapping_epoch":1127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:16.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1075,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.806147+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:18:33.806147+0000","last_peered":"2024-08-17T17:18:33.806147+0000","last_clean":"2024-08-17T17:18:33.806147+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:18:33.806147+0000","last_undegraded":"2024-08-17T17:18:33.806147+0000","last_fullsized":"2024-08-17T17:18:33.806147+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1087,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.817239+0000","last_change":"2024-08-17T17:06:39.625029+0000","last_active":"2024-08-17T17:18:33.817239+0000","last_peered":"2024-08-17T17:18:33.817239+0000","last_clean":"2024-08-17T17:18:33.817239+0000","last_became_active":"2024-08-17T17:06:39.624688+0000","last_became_peered":"2024-08-17T17:06:39.624688+0000","last_unstale":"2024-08-17T17:18:33.817239+0000","last_undegraded":"2024-08-17T17:18:33.817239+0000","last_fullsized":"2024-08-17T17:18:33.817239+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:54.849668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1356,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.821485+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:18:34.821485+0000","last_peered":"2024-08-17T17:18:34.821485+0000","last_clean":"2024-08-17T17:18:34.821485+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:18:34.821485+0000","last_undegraded":"2024-08-17T17:18:34.821485+0000","last_fullsized":"2024-08-17T17:18:34.821485+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":941,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.816294+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:18:33.816294+0000","last_peered":"2024-08-17T17:18:33.816294+0000","last_clean":"2024-08-17T17:18:33.816294+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:18:33.816294+0000","last_undegraded":"2024-08-17T17:18:33.816294+0000","last_fullsized":"2024-08-17T17:18:33.816294+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1375,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.821998+0000","last_change":"2024-08-17T17:07:45.678950+0000","last_active":"2024-08-17T17:18:34.821998+0000","last_peered":"2024-08-17T17:18:34.821998+0000","last_clean":"2024-08-17T17:18:34.821998+0000","last_became_active":"2024-08-17T17:07:45.678472+0000","last_became_peered":"2024-08-17T17:07:45.678472+0000","last_unstale":"2024-08-17T17:18:34.821998+0000","last_undegraded":"2024-08-17T17:18:34.821998+0000","last_fullsized":"2024-08-17T17:18:34.821998+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:48:02.385927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1089,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.824007+0000","last_change":"2024-08-17T17:06:39.628542+0000","last_active":"2024-08-17T17:18:34.824007+0000","last_peered":"2024-08-17T17:18:34.824007+0000","last_clean":"2024-08-17T17:18:34.824007+0000","last_became_active":"2024-08-17T17:06:39.628253+0000","last_became_peered":"2024-08-17T17:06:39.628253+0000","last_unstale":"2024-08-17T17:18:34.824007+0000","last_undegraded":"2024-08-17T17:18:34.824007+0000","last_fullsized":"2024-08-17T17:18:34.824007+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:22.349326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1405,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.822157+0000","last_change":"2024-08-17T17:18:31.532183+0000","last_active":"2024-08-17T17:18:34.822157+0000","last_peered":"2024-08-17T17:18:34.822157+0000","last_clean":"2024-08-17T17:18:34.822157+0000","last_became_active":"2024-08-17T17:18:31.531094+0000","last_became_peered":"2024-08-17T17:18:31.531094+0000","last_unstale":"2024-08-17T17:18:34.822157+0000","last_undegraded":"2024-08-17T17:18:34.822157+0000","last_fullsized":"2024-08-17T17:18:34.822157+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:01.763127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":901,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.809325+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:18:33.809325+0000","last_peered":"2024-08-17T17:18:33.809325+0000","last_clean":"2024-08-17T17:18:33.809325+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:18:33.809325+0000","last_undegraded":"2024-08-17T17:18:33.809325+0000","last_fullsized":"2024-08-17T17:18:33.809325+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":896,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.821665+0000","last_change":"2024-08-17T17:15:38.610768+0000","last_active":"2024-08-17T17:18:34.821665+0000","last_peered":"2024-08-17T17:18:34.821665+0000","last_clean":"2024-08-17T17:18:34.821665+0000","last_became_active":"2024-08-17T17:15:38.610337+0000","last_became_peered":"2024-08-17T17:15:38.610337+0000","last_unstale":"2024-08-17T17:18:34.821665+0000","last_undegraded":"2024-08-17T17:18:34.821665+0000","last_fullsized":"2024-08-17T17:18:34.821665+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:15.632778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1375,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.824394+0000","last_change":"2024-08-17T17:07:45.689782+0000","last_active":"2024-08-17T17:18:34.824394+0000","last_peered":"2024-08-17T17:18:34.824394+0000","last_clean":"2024-08-17T17:18:34.824394+0000","last_became_active":"2024-08-17T17:07:45.689312+0000","last_became_peered":"2024-08-17T17:07:45.689312+0000","last_unstale":"2024-08-17T17:18:34.824394+0000","last_undegraded":"2024-08-17T17:18:34.824394+0000","last_fullsized":"2024-08-17T17:18:34.824394+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:07.560555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1107,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.806957+0000","last_change":"2024-08-17T17:07:12.649175+0000","last_active":"2024-08-17T17:18:33.806957+0000","last_peered":"2024-08-17T17:18:33.806957+0000","last_clean":"2024-08-17T17:18:33.806957+0000","last_became_active":"2024-08-17T17:07:12.648822+0000","last_became_peered":"2024-08-17T17:07:12.648822+0000","last_unstale":"2024-08-17T17:18:33.806957+0000","last_undegraded":"2024-08-17T17:18:33.806957+0000","last_fullsized":"2024-08-17T17:18:33.806957+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:03.924431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1423,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.819414+0000","last_change":"2024-08-17T17:17:47.316690+0000","last_active":"2024-08-17T17:18:34.819414+0000","last_peered":"2024-08-17T17:18:34.819414+0000","last_clean":"2024-08-17T17:18:34.819414+0000","last_became_active":"2024-08-17T17:17:47.316333+0000","last_became_peered":"2024-08-17T17:17:47.316333+0000","last_unstale":"2024-08-17T17:18:34.819414+0000","last_undegraded":"2024-08-17T17:18:34.819414+0000","last_fullsized":"2024-08-17T17:18:34.819414+0000","mapping_epoch":1127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:04.542237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":94,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.807339+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:18:33.807339+0000","last_peered":"2024-08-17T17:18:33.807339+0000","last_clean":"2024-08-17T17:18:33.807339+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:18:33.807339+0000","last_undegraded":"2024-08-17T17:18:33.807339+0000","last_fullsized":"2024-08-17T17:18:33.807339+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1105,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.826457+0000","last_change":"2024-08-17T17:07:45.684606+0000","last_active":"2024-08-17T17:18:34.826457+0000","last_peered":"2024-08-17T17:18:34.826457+0000","last_clean":"2024-08-17T17:18:34.826457+0000","last_became_active":"2024-08-17T17:07:45.684124+0000","last_became_peered":"2024-08-17T17:07:45.684124+0000","last_unstale":"2024-08-17T17:18:34.826457+0000","last_undegraded":"2024-08-17T17:18:34.826457+0000","last_fullsized":"2024-08-17T17:18:34.826457+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:16.110507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1089,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.816890+0000","last_change":"2024-08-17T17:06:39.625245+0000","last_active":"2024-08-17T17:18:34.816890+0000","last_peered":"2024-08-17T17:18:34.816890+0000","last_clean":"2024-08-17T17:18:34.816890+0000","last_became_active":"2024-08-17T17:06:39.624657+0000","last_became_peered":"2024-08-17T17:06:39.624657+0000","last_unstale":"2024-08-17T17:18:34.816890+0000","last_undegraded":"2024-08-17T17:18:34.816890+0000","last_fullsized":"2024-08-17T17:18:34.816890+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:43.097460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1123,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.826039+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:18:34.826039+0000","last_peered":"2024-08-17T17:18:34.826039+0000","last_clean":"2024-08-17T17:18:34.826039+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:18:34.826039+0000","last_undegraded":"2024-08-17T17:18:34.826039+0000","last_fullsized":"2024-08-17T17:18:34.826039+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1074,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.817295+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:18:34.817295+0000","last_peered":"2024-08-17T17:18:34.817295+0000","last_clean":"2024-08-17T17:18:34.817295+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:18:34.817295+0000","last_undegraded":"2024-08-17T17:18:34.817295+0000","last_fullsized":"2024-08-17T17:18:34.817295+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1395,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.821787+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:18:34.821787+0000","last_peered":"2024-08-17T17:18:34.821787+0000","last_clean":"2024-08-17T17:18:34.821787+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:18:34.821787+0000","last_undegraded":"2024-08-17T17:18:34.821787+0000","last_fullsized":"2024-08-17T17:18:34.821787+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1370,"reported_epoch":1148,"state":"active+clean","last_fresh":"2024-08-17T17:18:34.826126+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:18:34.826126+0000","last_peered":"2024-08-17T17:18:34.826126+0000","last_clean":"2024-08-17T17:18:34.826126+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:18:34.826126+0000","last_undegraded":"2024-08-17T17:18:34.826126+0000","last_fullsized":"2024-08-17T17:18:34.826126+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1090,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.804806+0000","last_change":"2024-08-17T17:06:39.627689+0000","last_active":"2024-08-17T17:18:33.804806+0000","last_peered":"2024-08-17T17:18:33.804806+0000","last_clean":"2024-08-17T17:18:33.804806+0000","last_became_active":"2024-08-17T17:06:39.627375+0000","last_became_peered":"2024-08-17T17:06:39.627375+0000","last_unstale":"2024-08-17T17:18:33.804806+0000","last_undegraded":"2024-08-17T17:18:33.804806+0000","last_fullsized":"2024-08-17T17:18:33.804806+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:51.954076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":116,"reported_epoch":1147,"state":"active+clean","last_fresh":"2024-08-17T17:18:33.816219+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:18:33.816219+0000","last_peered":"2024-08-17T17:18:33.816219+0000","last_clean":"2024-08-17T17:18:33.816219+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:18:33.816219+0000","last_undegraded":"2024-08-17T17:18:33.816219+0000","last_fullsized":"2024-08-17T17:18:33.816219+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4787994624,"num_objects":73059,"num_object_clones":0,"num_object_copies":219177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":28763,"num_objects_misplaced":3900,"num_objects_unfound":0,"num_objects_dirty":73059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":73059,"num_write_kb":4675776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":285,"num_bytes_recovered":18677760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14398259200,"data_stored":14398259200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73059,"ondisk_log_size":73059,"up":48,"acting":40,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":912388,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":210,"num_read_kb":453,"num_write":118,"num_write_kb":1319,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1073397,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201607,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1978580,"kb_used_data":1749004,"kb_used_omap":1,"kb_used_meta":229566,"kb_avail":91766572,"statfs":{"total":95995035648,"available":93968969728,"internally_reserved":0,"allocated":1790980096,"data_stored":1788863184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":235076043},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463244,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1898508,"kb_used_data":1723076,"kb_used_omap":1,"kb_used_meta":175422,"kb_avail":91846644,"statfs":{"total":95995035648,"available":94050963456,"internally_reserved":0,"allocated":1764429824,"data_stored":1762320977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":179632585},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626885,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3401088,"kb_used_data":3094072,"kb_used_omap":42,"kb_used_meta":306965,"kb_avail":90344064,"statfs":{"total":95995035648,"available":92512321536,"internally_reserved":0,"allocated":3168329728,"data_stored":3166228970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":314332220},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785363,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2516028,"kb_used_data":2308852,"kb_used_omap":48,"kb_used_meta":207119,"kb_avail":91229124,"statfs":{"total":95995035648,"available":93418622976,"internally_reserved":0,"allocated":2364264448,"data_stored":2362167171,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":212090542},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724880,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1669448,"kb_used_data":1453824,"kb_used_omap":1,"kb_used_meta":215614,"kb_avail":92075704,"statfs":{"total":95995035648,"available":94285520896,"internally_reserved":0,"allocated":1488715776,"data_stored":1486607395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":220789190},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986517,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1959952,"kb_used_data":1734472,"kb_used_omap":1,"kb_used_meta":225470,"kb_avail":91785200,"statfs":{"total":95995035648,"available":93988044800,"internally_reserved":0,"allocated":1776099328,"data_stored":1773986442,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":230881734},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706124,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2010948,"kb_used_data":1762364,"kb_used_omap":35,"kb_used_meta":248540,"kb_avail":91734204,"statfs":{"total":95995035648,"available":93935824896,"internally_reserved":0,"allocated":1804660736,"data_stored":1802556408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":254505015},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831897,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":362304,"kb_used_data":221560,"kb_used_omap":65,"kb_used_meta":140670,"kb_avail":93382848,"statfs":{"total":95995035648,"available":95624036352,"internally_reserved":0,"allocated":226877440,"data_stored":224776652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":144046267},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,1,4,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":247267328,"data_stored":247267328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2373124096,"data_stored":2373124096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1801256960,"data_stored":1801256960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3164471296,"data_stored":3164471296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1778253824,"data_stored":1778253824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1486422016,"data_stored":1486422016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1762852864,"data_stored":1762852864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1784610816,"data_stored":1784610816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:18:49.134 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T17:18:49.134 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_2 pg_num 26 2024-08-17T17:18:49.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.135 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:49.135 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:49.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:49.236 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:49.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.294 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:49.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:49.360 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:49 smithi063 ceph-mon[114756]: pgmap v1355: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s; 28763/231261 objects degraded (12.437%); 3764/231261 objects misplaced (1.628%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:18:49.395 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:49.424 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:49.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:49.443 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 51 16 79563 79547 97.4671 49.5625 0.0150024 0.0102534 2024-08-17T17:18:49.526 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:49.551 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:49.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:49.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:49 smithi162 ceph-mon[104668]: pgmap v1355: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s; 28763/231261 objects degraded (12.437%); 3764/231261 objects misplaced (1.628%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:18:49.653 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:49.671 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:49 smithi063 ceph-mon[109843]: pgmap v1355: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s; 28763/231261 objects degraded (12.437%); 3764/231261 objects misplaced (1.628%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:18:49.703 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:49.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:49.805 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:49.854 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:49.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:49.956 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:49.995 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:49.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:49.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:50.097 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:50.123 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:50.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:50.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:50.227 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:50.251 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:50.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:50.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:50.352 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:50.400 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:50.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:50.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:50.443 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 52 16 80379 80363 96.5727 51 0.0266815 0.0103481 2024-08-17T17:18:50.503 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:50.507 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:50.525 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:50.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:50.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:50.627 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:50.655 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:50.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:50.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:50.757 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:50.783 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:50 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:18:50.783 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:50 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:18:50.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:50 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:18:50.784 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:50 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:18:50.839 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:50.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:50.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:50.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:50 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:18:50.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:50 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:18:50.941 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:50.968 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:50.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:50.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:51.069 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:51.098 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:51.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:51.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:51.200 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:51.240 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:51.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:51.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:51.341 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:51.382 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:51.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:51.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:51.443 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 53 16 81139 81123 95.6468 47.5 0.00698576 0.0104482 2024-08-17T17:18:51.485 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:51.524 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:51.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:51.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:51.584 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:18:51.626 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:51.651 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:51.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:51.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:51.752 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:51.814 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:51.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:51.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:51.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:51 smithi162 ceph-mon[104668]: pgmap v1356: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s; 28763/234237 objects degraded (12.279%); 3764/234237 objects misplaced (1.607%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:18:51.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:51 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1881689919' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:18:51.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:51 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:18:51.916 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:51.942 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:51.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:51.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:51.985 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:51 smithi063 ceph-mon[109843]: pgmap v1356: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s; 28763/234237 objects degraded (12.279%); 3764/234237 objects misplaced (1.607%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:18:51.985 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:51 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1881689919' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:18:51.985 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:51 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:18:51.985 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:51 smithi063 ceph-mon[114756]: pgmap v1356: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 49 MiB/s wr, 778 op/s; 28763/234237 objects degraded (12.279%); 3764/234237 objects misplaced (1.607%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:18:51.985 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:51 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1881689919' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:18:51.985 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:51 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:18:52.044 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:52.070 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:52.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:52.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:52.172 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:52.202 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:52.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:52.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:52.304 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:52.344 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:52.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:52.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:52.444 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 54 16 81970 81954 94.8356 51.9375 0.00718662 0.0105364 2024-08-17T17:18:52.446 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:52.496 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:52.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:52.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:52.598 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:52.633 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:52.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:52.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:52.697 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:52 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:52.697 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:52 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:18:52.698 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:52 smithi063 ceph-mon[114756]: osdmap e1149: 8 total, 8 up, 8 in 2024-08-17T17:18:52.698 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:52 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:52.698 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:52 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:18:52.698 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:52 smithi063 ceph-mon[109843]: osdmap e1149: 8 total, 8 up, 8 in 2024-08-17T17:18:52.734 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:52.757 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:52.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:52.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:52.859 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:52.910 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:52.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:52.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:53.012 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:53.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:53.042 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:53.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:52 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:18:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:52 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:18:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:52 smithi162 ceph-mon[104668]: osdmap e1149: 8 total, 8 up, 8 in 2024-08-17T17:18:53.143 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:53.183 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:53.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:53.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:53.285 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:53.335 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:53.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:53.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:53.438 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:53.445 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 55 16 82793 82777 94.0453 51.4375 0.0168666 0.0106256 2024-08-17T17:18:53.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:53.482 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:53.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:53.488 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:18:53.488 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:18:53.525 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:18:53.526 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:18:53.526 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:18:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:18:53] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.51.0" 2024-08-17T17:18:53.558 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:18:53.558 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:18:53.583 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:53.594 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:18:53.594 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:18:53.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:53.625 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:53.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:53.628 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:18:53.628 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:18:53.659 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:18:53.659 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:18:53.691 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:18:53.691 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:18:53.723 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:53.725 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:18:53.725 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:18:53.726 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:53.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:53 smithi063 ceph-mon[114756]: pgmap v1358: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s; 28763/240330 objects degraded (11.968%); 3764/240330 objects misplaced (1.566%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:18:53.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:53 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 28763/240330 objects degraded (11.968%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:53.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:53 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2180360091' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:53.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:53 smithi063 ceph-mon[109843]: pgmap v1358: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s; 28763/240330 objects degraded (11.968%); 3764/240330 objects misplaced (1.566%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:18:53.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:53 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 28763/240330 objects degraded (11.968%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:53.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:53 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2180360091' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:53.789 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:53.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:53.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:53.891 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:53.922 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:53.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:53.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:54.023 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:54.068 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:54.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:53 smithi162 ceph-mon[104668]: pgmap v1358: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 811 op/s; 28763/240330 objects degraded (11.968%); 3764/240330 objects misplaced (1.566%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:18:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:53 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 28763/240330 objects degraded (11.968%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:53 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2180360091' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:18:54.170 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:54.217 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:54.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:54.319 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:54.346 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:54.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:54.431 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:18:54.446 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 56 16 83578 83562 93.2416 49.0625 0.019952 0.0107182 2024-08-17T17:18:54.448 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:54.475 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:54.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:54.576 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:54.603 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:54.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:54.705 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:54.732 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:54.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:54.834 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:54.863 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:54.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:54.965 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:54.990 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:54.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:54.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:55.092 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:55.144 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:55.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:55.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:55.245 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:55.290 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:55.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:55.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:55.392 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:55.419 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:55.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:55.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:55.446 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 57 16 84375 84359 92.4797 49.8125 0.00625249 0.0108062 2024-08-17T17:18:55.520 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:55.554 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:55.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:55.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:55.656 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:55.694 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:55.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:55.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:55.796 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:55.832 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:55.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:55.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:55.888 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:55 smithi063 ceph-mon[109843]: pgmap v1359: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s; 28763/243474 objects degraded (11.814%); 3617/243474 objects misplaced (1.486%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:18:55.888 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:55 smithi063 ceph-mon[114756]: pgmap v1359: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s; 28763/243474 objects degraded (11.814%); 3617/243474 objects misplaced (1.486%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:18:55.934 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:55.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:55.995 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:55.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:56.095 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:55 smithi162 ceph-mon[104668]: pgmap v1359: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 50 MiB/s wr, 806 op/s; 28763/243474 objects degraded (11.814%); 3617/243474 objects misplaced (1.486%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:18:56.138 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:56.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:56.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:56.240 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:56.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:56.279 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:56.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:56.380 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:56.410 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:56.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:56.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:56.446 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 58 16 85175 85159 91.7465 50 0.0241908 0.0108924 2024-08-17T17:18:56.512 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:56.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:56.565 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:56.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:56.666 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:56.693 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:56.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:56.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:56.727 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:18:56.795 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:56.847 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:18:56.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:56.853 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:56.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:56.953 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:18:57.023 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:57.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:57.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:18:57.125 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:57.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:57.198 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:57.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:57.299 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:57.323 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:57.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:57.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:57.424 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:57.446 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 59 16 85876 85860 90.9341 43.8125 0.0468123 0.0109891 2024-08-17T17:18:57.448 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:57.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:57.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:57.551 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:57.575 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:57.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:57.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:57.676 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:57.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:57.719 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:57.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:18:57.820 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:57.850 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:57.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:57.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:57.951 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:18:58.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:58.011 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:58.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:18:58.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:57 smithi063 ceph-mon[109843]: pgmap v1360: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s; 28763/246327 objects degraded (11.677%); 3617/246327 objects misplaced (1.468%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:18:58.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:57 smithi063 ceph-mon[114756]: pgmap v1360: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s; 28763/246327 objects degraded (11.677%); 3617/246327 objects misplaced (1.468%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:18:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:57 smithi162 ceph-mon[104668]: pgmap v1360: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 800 op/s; 28763/246327 objects degraded (11.677%); 3617/246327 objects misplaced (1.468%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:18:58.112 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:58.138 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:58.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:58.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:58.240 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:18:58.264 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:58.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:58.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:18:58.365 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:58.391 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:58.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:58.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:58.446 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:18:58.446831+0000 min lat: 0.00126452 max lat: 2.52879 avg lat: 0.011079 2024-08-17T17:18:58.447 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:18:58.447 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 60 16 86626 86610 90.1998 46.875 0.0155925 0.011079 2024-08-17T17:18:58.493 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:58.520 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:58.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:58.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:58.610 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:58 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/5995006' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:18:58.611 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:58 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:18:58.611 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:58 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 28763/246327 objects degraded (11.677%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:58.621 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:58.633 INFO:teuthology.orchestra.run.smithi063.stderr:set pool 7 pg_num to 26 2024-08-17T17:18:58.672 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:58.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:58.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:58.774 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:18:58.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:58.828 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:18:58.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:18:58.928 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:58.961 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:58.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:58.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:59.009 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:58 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/5995006' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:18:59.009 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:58 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:18:59.009 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:58 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 28763/246327 objects degraded (11.677%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:59.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:58 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/5995006' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:18:59.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:58 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:18:59.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:58 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 28763/246327 objects degraded (11.677%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:18:59.064 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:18:59.093 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:59.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:59.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:18:59.195 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:59.229 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:59.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:59.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:59.331 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:59.345 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:18:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=cleanup t=2024-08-17T17:18:59.008079545Z level=info msg="Completed cleanup jobs" duration=3.973028ms 2024-08-17T17:18:59.345 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:18:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=grafana.update.checker t=2024-08-17T17:18:59.139056342Z level=info msg="Update check succeeded" duration=62.37492ms 2024-08-17T17:18:59.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:18:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=plugins.update.checker t=2024-08-17T17:18:59.148206849Z level=info msg="Update check succeeded" duration=59.606247ms 2024-08-17T17:18:59.359 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:59.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:59.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:59.447 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 61 16 87419 87403 89.5336 49.5625 0.0131489 0.0111623 2024-08-17T17:18:59.461 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:59.491 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:59.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:59.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:59.592 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:18:59.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:59.622 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:59.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:18:59.723 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:18:59.752 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:18:59.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:18:59.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:18:59.854 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:18:59.887 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:59 smithi063 ceph-mon[114756]: pgmap v1361: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s; 28763/255351 objects degraded (11.264%); 3474/255351 objects misplaced (1.360%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:18:59.888 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:59 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:18:59.888 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:18:59 smithi063 ceph-mon[114756]: osdmap e1150: 8 total, 8 up, 8 in 2024-08-17T17:18:59.892 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:59 smithi063 ceph-mon[109843]: pgmap v1361: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s; 28763/255351 objects degraded (11.264%); 3474/255351 objects misplaced (1.360%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:18:59.892 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:59 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:18:59.892 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:18:59 smithi063 ceph-mon[109843]: osdmap e1150: 8 total, 8 up, 8 in 2024-08-17T17:18:59.893 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:18:59.893 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:18:59.893 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:18:59.893 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:18:59.893 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.893 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:29976 6.9 6:9696ac29:::benchmark_data_smithi063_160268_object29975:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:18:59.893 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:37.728584+0000", 2024-08-17T17:18:59.893 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 320.50158723499999, 2024-08-17T17:18:59.893 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.076889863000000003, 2024-08-17T17:18:59.893 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.893 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.894 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:59.894 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:18:59.894 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:18:59.894 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:59.894 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 29976 2024-08-17T17:18:59.894 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.894 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.894 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.894 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.894 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728584+0000", 2024-08-17T17:18:59.894 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.894 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.894 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.895 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.895 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728584+0000", 2024-08-17T17:18:59.895 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.895 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.895 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.895 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.895 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728585+0000", 2024-08-17T17:18:59.895 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3900000000000004e-07 2024-08-17T17:18:59.895 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.895 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.895 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.895 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728604+0000", 2024-08-17T17:18:59.895 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8746e-05 2024-08-17T17:18:59.896 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.896 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.896 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.896 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728613+0000", 2024-08-17T17:18:59.896 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.662e-06 2024-08-17T17:18:59.896 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.896 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.896 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.896 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728618+0000", 2024-08-17T17:18:59.896 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6650000000000002e-06 2024-08-17T17:18:59.896 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.896 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.897 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.897 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728682+0000", 2024-08-17T17:18:59.897 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4083000000000003e-05 2024-08-17T17:18:59.897 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.897 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.897 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.897 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728765+0000", 2024-08-17T17:18:59.897 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3230000000000001e-05 2024-08-17T17:18:59.897 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.897 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.897 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:18:59.897 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728845+0000", 2024-08-17T17:18:59.897 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9689000000000002e-05 2024-08-17T17:18:59.898 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.898 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.898 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:18:59.898 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729565+0000", 2024-08-17T17:18:59.898 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00072059299999999995 2024-08-17T17:18:59.898 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.898 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.898 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.898 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.730712+0000", 2024-08-17T17:18:59.898 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0011460470000000001 2024-08-17T17:18:59.898 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.898 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.898 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.899 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.805360+0000", 2024-08-17T17:18:59.899 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.074648649999999997 2024-08-17T17:18:59.899 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.899 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.899 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:18:59.899 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.805426+0000", 2024-08-17T17:18:59.899 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5411000000000001e-05 2024-08-17T17:18:59.899 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.899 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.899 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.899 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.805474+0000", 2024-08-17T17:18:59.899 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8248000000000003e-05 2024-08-17T17:18:59.899 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.900 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.900 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.900 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.900 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.900 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:29978 6.9 6:9795df24:::benchmark_data_smithi063_160268_object29977:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:18:59.900 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:37.729544+0000", 2024-08-17T17:18:59.900 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 320.50062739200001, 2024-08-17T17:18:59.900 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.080494046999999999, 2024-08-17T17:18:59.900 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.900 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.900 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:59.900 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:18:59.900 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:18:59.900 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:59.901 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 29978 2024-08-17T17:18:59.901 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.901 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.901 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.901 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.901 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729544+0000", 2024-08-17T17:18:59.901 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.901 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.901 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.901 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.901 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729544+0000", 2024-08-17T17:18:59.901 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.901 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.901 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.901 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.902 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729545+0000", 2024-08-17T17:18:59.902 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.9899999999999999e-07 2024-08-17T17:18:59.902 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.902 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.902 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.902 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729566+0000", 2024-08-17T17:18:59.902 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1455e-05 2024-08-17T17:18:59.902 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.902 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.902 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.902 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729577+0000", 2024-08-17T17:18:59.902 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0804e-05 2024-08-17T17:18:59.902 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.903 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.903 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.903 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729583+0000", 2024-08-17T17:18:59.903 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4980000000000003e-06 2024-08-17T17:18:59.903 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.903 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.903 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.903 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729622+0000", 2024-08-17T17:18:59.903 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8441e-05 2024-08-17T17:18:59.903 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.903 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.903 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.903 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729697+0000", 2024-08-17T17:18:59.903 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4948000000000001e-05 2024-08-17T17:18:59.903 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.904 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.904 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:18:59.904 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.729780+0000", 2024-08-17T17:18:59.904 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2812999999999995e-05 2024-08-17T17:18:59.904 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.904 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.904 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:18:59.904 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.730439+0000", 2024-08-17T17:18:59.904 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00065916200000000003 2024-08-17T17:18:59.904 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.904 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.904 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.904 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.730790+0000", 2024-08-17T17:18:59.904 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00035080499999999999 2024-08-17T17:18:59.904 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.905 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.905 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.905 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.809939+0000", 2024-08-17T17:18:59.905 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.079149580999999997 2024-08-17T17:18:59.905 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.905 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.905 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:18:59.905 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.809997+0000", 2024-08-17T17:18:59.905 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7386000000000002e-05 2024-08-17T17:18:59.905 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.905 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.905 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.905 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.810038+0000", 2024-08-17T17:18:59.905 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.1254999999999997e-05 2024-08-17T17:18:59.905 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.906 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.906 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.906 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.906 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.906 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34196 6.9 6:90093b33:::benchmark_data_smithi063_160268_object34195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:18:59.906 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.037012+0000", 2024-08-17T17:18:59.906 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 318.19315957399999, 2024-08-17T17:18:59.906 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.091924617, 2024-08-17T17:18:59.906 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.906 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.906 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:59.906 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:18:59.906 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:18:59.907 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:59.907 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34196 2024-08-17T17:18:59.907 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.907 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.907 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.907 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.907 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037012+0000", 2024-08-17T17:18:59.907 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.907 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.907 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.907 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.907 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037012+0000", 2024-08-17T17:18:59.907 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.908 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.908 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.908 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.908 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037013+0000", 2024-08-17T17:18:59.908 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1540000000000001e-06 2024-08-17T17:18:59.908 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.908 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.908 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.908 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037332+0000", 2024-08-17T17:18:59.908 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00031923299999999998 2024-08-17T17:18:59.908 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.908 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.908 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.909 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037344+0000", 2024-08-17T17:18:59.909 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1973999999999999e-05 2024-08-17T17:18:59.909 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.909 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.909 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.909 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037351+0000", 2024-08-17T17:18:59.909 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9609999999999999e-06 2024-08-17T17:18:59.909 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.909 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.909 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.909 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037649+0000", 2024-08-17T17:18:59.909 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00029795199999999998 2024-08-17T17:18:59.909 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.910 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.910 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.910 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037761+0000", 2024-08-17T17:18:59.910 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000112284 2024-08-17T17:18:59.910 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.910 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.910 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:18:59.910 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037889+0000", 2024-08-17T17:18:59.910 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012809900000000001 2024-08-17T17:18:59.910 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.910 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.910 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:18:59.910 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.039236+0000", 2024-08-17T17:18:59.911 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0013463940000000001 2024-08-17T17:18:59.911 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.911 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.911 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.911 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.041695+0000", 2024-08-17T17:18:59.911 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002459124 2024-08-17T17:18:59.911 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.911 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.911 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.911 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128820+0000", 2024-08-17T17:18:59.911 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.087125381000000002 2024-08-17T17:18:59.911 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.911 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.912 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:18:59.912 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128885+0000", 2024-08-17T17:18:59.912 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4826000000000006e-05 2024-08-17T17:18:59.912 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.912 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.912 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.912 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128936+0000", 2024-08-17T17:18:59.912 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1235e-05 2024-08-17T17:18:59.912 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.912 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.912 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.912 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.912 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.913 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34197 6.0 6:0173ef19:::benchmark_data_smithi063_160268_object34196:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:18:59.913 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.037483+0000", 2024-08-17T17:18:59.913 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 318.19268858700002, 2024-08-17T17:18:59.913 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.091051282999999997, 2024-08-17T17:18:59.913 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.913 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.913 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:59.913 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:18:59.913 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:18:59.913 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:59.913 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34197 2024-08-17T17:18:59.913 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.913 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.914 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.914 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.914 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037483+0000", 2024-08-17T17:18:59.914 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.914 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.914 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.914 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.914 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037483+0000", 2024-08-17T17:18:59.914 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.914 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.914 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.914 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.915 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037483+0000", 2024-08-17T17:18:59.915 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3099999999999997e-07 2024-08-17T17:18:59.915 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.915 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.915 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.915 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037533+0000", 2024-08-17T17:18:59.915 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9410000000000003e-05 2024-08-17T17:18:59.915 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.915 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.915 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.915 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037543+0000", 2024-08-17T17:18:59.915 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0292000000000001e-05 2024-08-17T17:18:59.915 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.915 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.916 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.916 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.037549+0000", 2024-08-17T17:18:59.916 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.528e-06 2024-08-17T17:18:59.916 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.916 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.916 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.916 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.038545+0000", 2024-08-17T17:18:59.916 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00099573000000000001 2024-08-17T17:18:59.916 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.916 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.916 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.916 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.038670+0000", 2024-08-17T17:18:59.916 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000124387 2024-08-17T17:18:59.917 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.917 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.917 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:18:59.917 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.038803+0000", 2024-08-17T17:18:59.917 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000133199 2024-08-17T17:18:59.917 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.917 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.917 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:18:59.917 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.040082+0000", 2024-08-17T17:18:59.917 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001278922 2024-08-17T17:18:59.917 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.917 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.917 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.917 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042805+0000", 2024-08-17T17:18:59.917 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002723516 2024-08-17T17:18:59.918 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.918 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.918 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.918 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128419+0000", 2024-08-17T17:18:59.918 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.085614129999999997 2024-08-17T17:18:59.918 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.918 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.918 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:18:59.918 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128488+0000", 2024-08-17T17:18:59.918 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8233999999999998e-05 2024-08-17T17:18:59.918 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.918 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.918 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.919 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128534+0000", 2024-08-17T17:18:59.919 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6304000000000003e-05 2024-08-17T17:18:59.919 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.919 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.919 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.919 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.919 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.919 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34204 6.9 6:90e2ac6d:::benchmark_data_smithi063_160268_object34203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:18:59.919 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.039213+0000", 2024-08-17T17:18:59.919 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 318.19095788999999, 2024-08-17T17:18:59.919 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.089974211999999998, 2024-08-17T17:18:59.919 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.919 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.920 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:59.920 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:18:59.920 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:18:59.920 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:59.920 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34204 2024-08-17T17:18:59.920 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.920 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.920 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.920 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.920 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.039213+0000", 2024-08-17T17:18:59.920 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.920 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.920 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.920 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.921 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.039213+0000", 2024-08-17T17:18:59.921 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.921 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.921 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.921 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.921 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.039214+0000", 2024-08-17T17:18:59.921 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.2600000000000001e-07 2024-08-17T17:18:59.921 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.921 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.921 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.921 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.039238+0000", 2024-08-17T17:18:59.921 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3824000000000001e-05 2024-08-17T17:18:59.921 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.922 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.922 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.922 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.039248+0000", 2024-08-17T17:18:59.922 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9669999999999996e-06 2024-08-17T17:18:59.922 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.922 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.922 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.922 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.039255+0000", 2024-08-17T17:18:59.922 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7059999999999998e-06 2024-08-17T17:18:59.922 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.922 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.922 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.922 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.040287+0000", 2024-08-17T17:18:59.923 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0010324589999999999 2024-08-17T17:18:59.923 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.923 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.923 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.923 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.040402+0000", 2024-08-17T17:18:59.923 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011492 2024-08-17T17:18:59.923 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.923 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.923 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:18:59.923 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.040529+0000", 2024-08-17T17:18:59.923 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012688900000000001 2024-08-17T17:18:59.923 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.923 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.923 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:18:59.924 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.041867+0000", 2024-08-17T17:18:59.924 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0013376550000000001 2024-08-17T17:18:59.924 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.924 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.924 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.924 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128983+0000", 2024-08-17T17:18:59.924 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.087116491000000004 2024-08-17T17:18:59.924 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.924 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.924 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.924 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129096+0000", 2024-08-17T17:18:59.924 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000112509 2024-08-17T17:18:59.924 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.925 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.925 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:18:59.925 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129149+0000", 2024-08-17T17:18:59.925 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3325000000000002e-05 2024-08-17T17:18:59.925 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.925 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.925 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.925 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129188+0000", 2024-08-17T17:18:59.925 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8541000000000003e-05 2024-08-17T17:18:59.925 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.925 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.925 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.925 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.925 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.925 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34210 6.9 6:9155233c:::benchmark_data_smithi063_160268_object34209:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:18:59.926 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.042864+0000", 2024-08-17T17:18:59.926 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 318.18730750399999, 2024-08-17T17:18:59.926 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.087962027999999998, 2024-08-17T17:18:59.926 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.926 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.926 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:59.926 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:18:59.926 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:18:59.926 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:59.926 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34210 2024-08-17T17:18:59.926 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.926 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.926 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.927 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.927 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042864+0000", 2024-08-17T17:18:59.927 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.927 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.927 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.927 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.927 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042864+0000", 2024-08-17T17:18:59.927 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.927 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.927 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.927 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.927 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042865+0000", 2024-08-17T17:18:59.927 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.181e-06 2024-08-17T17:18:59.927 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.928 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.928 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.928 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042888+0000", 2024-08-17T17:18:59.928 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2993000000000002e-05 2024-08-17T17:18:59.928 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.928 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.928 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.928 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042899+0000", 2024-08-17T17:18:59.928 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1326e-05 2024-08-17T17:18:59.928 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.928 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.928 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.928 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042906+0000", 2024-08-17T17:18:59.929 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9199999999999998e-06 2024-08-17T17:18:59.929 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.929 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.929 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.929 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043115+0000", 2024-08-17T17:18:59.929 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000209323 2024-08-17T17:18:59.929 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.929 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.929 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.929 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043259+0000", 2024-08-17T17:18:59.929 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014336500000000001 2024-08-17T17:18:59.929 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.929 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.930 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:18:59.930 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043411+0000", 2024-08-17T17:18:59.930 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000151767 2024-08-17T17:18:59.930 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.930 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.930 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.930 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129023+0000", 2024-08-17T17:18:59.930 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.085612478000000006 2024-08-17T17:18:59.930 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.930 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.930 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.930 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129233+0000", 2024-08-17T17:18:59.930 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00021032200000000001 2024-08-17T17:18:59.930 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.130743+0000", 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001510061 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.130785+0000", 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.1313999999999998e-05 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.931 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.130826+0000", 2024-08-17T17:18:59.933 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.0978e-05 2024-08-17T17:18:59.933 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.933 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.933 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.933 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.933 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.933 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34211 6.9 6:951c0f16:::benchmark_data_smithi063_160268_object34210:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:18:59.934 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.042932+0000", 2024-08-17T17:18:59.934 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 318.18723882900002, 2024-08-17T17:18:59.934 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.087980920000000004, 2024-08-17T17:18:59.934 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.934 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.934 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:59.934 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:18:59.934 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:18:59.934 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:59.934 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34211 2024-08-17T17:18:59.934 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.934 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.935 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.935 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.935 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042932+0000", 2024-08-17T17:18:59.935 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.935 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.935 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.935 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.935 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042932+0000", 2024-08-17T17:18:59.935 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.935 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.935 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.935 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.935 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042933+0000", 2024-08-17T17:18:59.935 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6899999999999996e-07 2024-08-17T17:18:59.935 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.936 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.936 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.936 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042956+0000", 2024-08-17T17:18:59.936 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2646e-05 2024-08-17T17:18:59.936 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.936 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.936 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.936 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042965+0000", 2024-08-17T17:18:59.936 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3109999999999995e-06 2024-08-17T17:18:59.936 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.936 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.936 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.936 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.042972+0000", 2024-08-17T17:18:59.937 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4829999999999996e-06 2024-08-17T17:18:59.937 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.937 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.937 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.937 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044942+0000", 2024-08-17T17:18:59.937 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001970201 2024-08-17T17:18:59.937 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.937 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.937 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.937 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.045071+0000", 2024-08-17T17:18:59.937 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012918399999999999 2024-08-17T17:18:59.937 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.938 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.938 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:18:59.938 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.045203+0000", 2024-08-17T17:18:59.938 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000132173 2024-08-17T17:18:59.938 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.938 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.938 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.938 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129060+0000", 2024-08-17T17:18:59.938 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.083857318 2024-08-17T17:18:59.938 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.938 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.938 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.938 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129290+0000", 2024-08-17T17:18:59.939 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00022933 2024-08-17T17:18:59.939 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.939 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.939 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:18:59.939 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.130837+0000", 2024-08-17T17:18:59.939 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0015471160000000001 2024-08-17T17:18:59.939 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.939 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.939 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:18:59.939 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.130879+0000", 2024-08-17T17:18:59.939 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2208e-05 2024-08-17T17:18:59.939 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.939 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.939 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.939 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.130913+0000", 2024-08-17T17:18:59.940 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4180999999999997e-05 2024-08-17T17:18:59.940 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.940 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.940 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.940 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.940 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.940 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34213 6.0 6:0bdfe2aa:::benchmark_data_smithi063_160268_object34212:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:18:59.940 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.043016+0000", 2024-08-17T17:18:59.940 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 318.187155512, 2024-08-17T17:18:59.940 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.093410909, 2024-08-17T17:18:59.940 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.940 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.940 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:59.940 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:18:59.941 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:18:59.941 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:59.941 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34213 2024-08-17T17:18:59.941 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.941 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.941 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.941 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.941 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043016+0000", 2024-08-17T17:18:59.941 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.941 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.941 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.941 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.941 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043016+0000", 2024-08-17T17:18:59.942 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.942 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.942 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.942 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.942 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043017+0000", 2024-08-17T17:18:59.942 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8999999999999995e-07 2024-08-17T17:18:59.942 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.942 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.942 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.942 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043038+0000", 2024-08-17T17:18:59.942 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1098999999999999e-05 2024-08-17T17:18:59.942 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.943 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.943 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.943 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043048+0000", 2024-08-17T17:18:59.943 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9729999999999992e-06 2024-08-17T17:18:59.943 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.943 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.943 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.943 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043053+0000", 2024-08-17T17:18:59.943 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6069999999999998e-06 2024-08-17T17:18:59.943 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.943 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.943 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.943 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043941+0000", 2024-08-17T17:18:59.943 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00088757500000000004 2024-08-17T17:18:59.943 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.944 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.944 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.944 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044053+0000", 2024-08-17T17:18:59.944 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011174499999999999 2024-08-17T17:18:59.944 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.944 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.944 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:18:59.944 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044174+0000", 2024-08-17T17:18:59.944 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000121351 2024-08-17T17:18:59.944 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.944 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.944 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:18:59.944 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128544+0000", 2024-08-17T17:18:59.944 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.084370497000000003 2024-08-17T17:18:59.944 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.945 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.945 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.945 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128581+0000", 2024-08-17T17:18:59.945 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6316000000000003e-05 2024-08-17T17:18:59.945 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.945 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.945 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.945 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.136316+0000", 2024-08-17T17:18:59.945 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0077348740000000001 2024-08-17T17:18:59.945 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.945 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.945 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:18:59.946 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.136379+0000", 2024-08-17T17:18:59.946 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3622999999999997e-05 2024-08-17T17:18:59.946 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.946 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.946 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.946 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.136427+0000", 2024-08-17T17:18:59.946 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7358999999999998e-05 2024-08-17T17:18:59.946 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.946 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.946 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.946 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.946 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.946 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34215 6.0 6:0b2fbf44:::benchmark_data_smithi063_160268_object34214:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:18:59.946 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.043389+0000", 2024-08-17T17:18:59.947 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 318.18678256800001, 2024-08-17T17:18:59.947 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.093601496000000006, 2024-08-17T17:18:59.947 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.947 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.947 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:59.947 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:18:59.947 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:18:59.947 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:59.947 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34215 2024-08-17T17:18:59.947 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.947 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.947 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.947 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.948 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043389+0000", 2024-08-17T17:18:59.948 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.948 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.948 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.948 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.948 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043389+0000", 2024-08-17T17:18:59.948 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.948 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.948 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.948 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.948 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043389+0000", 2024-08-17T17:18:59.948 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3499999999999995e-07 2024-08-17T17:18:59.948 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.949 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.949 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.949 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043439+0000", 2024-08-17T17:18:59.949 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9734999999999997e-05 2024-08-17T17:18:59.949 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.949 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.949 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.949 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043449+0000", 2024-08-17T17:18:59.949 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4169999999999998e-06 2024-08-17T17:18:59.949 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.949 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.949 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.950 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.043456+0000", 2024-08-17T17:18:59.950 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0620000000000003e-06 2024-08-17T17:18:59.950 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.950 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.950 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.950 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.126561+0000", 2024-08-17T17:18:59.950 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.083105629 2024-08-17T17:18:59.950 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.950 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.950 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.950 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.126728+0000", 2024-08-17T17:18:59.950 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016643800000000001 2024-08-17T17:18:59.951 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.951 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.951 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:18:59.951 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.126894+0000", 2024-08-17T17:18:59.951 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016649300000000001 2024-08-17T17:18:59.951 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.951 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.951 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:18:59.951 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.129010+0000", 2024-08-17T17:18:59.951 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0021160480000000001 2024-08-17T17:18:59.951 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.951 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.951 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.952 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.131666+0000", 2024-08-17T17:18:59.952 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0026555139999999999 2024-08-17T17:18:59.952 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.952 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.952 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.952 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.136889+0000", 2024-08-17T17:18:59.952 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0052235069999999996 2024-08-17T17:18:59.952 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.952 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.952 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:18:59.952 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.136954+0000", 2024-08-17T17:18:59.952 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4984e-05 2024-08-17T17:18:59.953 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.953 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.953 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.953 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.136990+0000", 2024-08-17T17:18:59.953 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5933999999999998e-05 2024-08-17T17:18:59.953 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.953 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.953 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.953 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.953 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.953 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:34217 6.d e1093/1086 6:b571891d:::benchmark_data_smithi063_160268_object34216:head v 1093'2085, mlcod=1093'2083)", 2024-08-17T17:18:59.953 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.044815+0000", 2024-08-17T17:18:59.953 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 318.18535668200002, 2024-08-17T17:18:59.953 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.081785244000000007, 2024-08-17T17:18:59.954 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.954 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.954 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:59.954 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.954 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.954 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.954 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044815+0000", 2024-08-17T17:18:59.954 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.954 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.954 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.954 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.954 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044815+0000", 2024-08-17T17:18:59.955 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.955 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.955 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.955 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.955 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044815+0000", 2024-08-17T17:18:59.955 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3399999999999998e-07 2024-08-17T17:18:59.955 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.955 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.955 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.955 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044845+0000", 2024-08-17T17:18:59.955 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.9427000000000001e-05 2024-08-17T17:18:59.955 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.955 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.955 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.956 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044855+0000", 2024-08-17T17:18:59.956 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.8619999999999998e-06 2024-08-17T17:18:59.956 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.956 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.956 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.956 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044861+0000", 2024-08-17T17:18:59.956 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7020000000000004e-06 2024-08-17T17:18:59.956 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.956 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.956 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.956 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044916+0000", 2024-08-17T17:18:59.956 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4268999999999998e-05 2024-08-17T17:18:59.956 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.956 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.957 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.957 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044989+0000", 2024-08-17T17:18:59.957 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3128999999999997e-05 2024-08-17T17:18:59.957 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.957 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.957 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:18:59.957 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.126543+0000", 2024-08-17T17:18:59.957 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.081553977999999999 2024-08-17T17:18:59.957 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.957 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.957 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.957 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.126600+0000", 2024-08-17T17:18:59.957 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7043000000000003e-05 2024-08-17T17:18:59.957 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.958 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.958 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.958 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.958 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.958 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.44609.0:34197 6.0 e1093/1086 ondisk, result = 0)", 2024-08-17T17:18:59.958 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.045637+0000", 2024-08-17T17:18:59.958 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 318.18453430300002, 2024-08-17T17:18:59.958 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.082908979999999993, 2024-08-17T17:18:59.958 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.958 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.958 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:18:59.958 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.959 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.959 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.959 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.045637+0000", 2024-08-17T17:18:59.959 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.959 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.959 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.959 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.959 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.045637+0000", 2024-08-17T17:18:59.959 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.959 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.959 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.959 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.959 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.045638+0000", 2024-08-17T17:18:59.959 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3600000000000003e-07 2024-08-17T17:18:59.959 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.960 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.960 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.960 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.045641+0000", 2024-08-17T17:18:59.960 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.529e-06 2024-08-17T17:18:59.960 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.960 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.960 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.960 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.126636+0000", 2024-08-17T17:18:59.960 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.080995196000000005 2024-08-17T17:18:59.960 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.960 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.960 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.960 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.126646+0000", 2024-08-17T17:18:59.960 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4029999999999996e-06 2024-08-17T17:18:59.960 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.961 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.961 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.961 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128405+0000", 2024-08-17T17:18:59.961 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001759273 2024-08-17T17:18:59.961 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.961 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.961 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.961 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128413+0000", 2024-08-17T17:18:59.961 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0479999999999999e-06 2024-08-17T17:18:59.961 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.961 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.961 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.961 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.128546+0000", 2024-08-17T17:18:59.962 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013289500000000001 2024-08-17T17:18:59.962 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.962 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.962 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.962 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.962 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.962 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35804 6.0 6:00446a4a:::benchmark_data_smithi063_160268_object35803:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:18:59.962 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.874443+0000", 2024-08-17T17:18:59.962 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 317.355727973, 2024-08-17T17:18:59.962 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14757826700000001, 2024-08-17T17:18:59.962 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.962 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.963 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:59.963 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:18:59.963 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:18:59.963 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:59.963 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35804 2024-08-17T17:18:59.963 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.963 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.963 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.963 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.963 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.874443+0000", 2024-08-17T17:18:59.963 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.963 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.963 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.963 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.963 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.874443+0000", 2024-08-17T17:18:59.964 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.964 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.964 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.964 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.964 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.874444+0000", 2024-08-17T17:18:59.964 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.147e-06 2024-08-17T17:18:59.964 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.964 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.964 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.964 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.874465+0000", 2024-08-17T17:18:59.964 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0462000000000002e-05 2024-08-17T17:18:59.964 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.964 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.964 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.964 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.874476+0000", 2024-08-17T17:18:59.965 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1481000000000001e-05 2024-08-17T17:18:59.965 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.965 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.965 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.965 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.874483+0000", 2024-08-17T17:18:59.965 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0330000000000003e-06 2024-08-17T17:18:59.965 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.965 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.965 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.965 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.875878+0000", 2024-08-17T17:18:59.965 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0013948960000000001 2024-08-17T17:18:59.965 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.965 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.965 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.966 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876005+0000", 2024-08-17T17:18:59.966 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012692399999999999 2024-08-17T17:18:59.966 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.966 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.966 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:18:59.966 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876663+0000", 2024-08-17T17:18:59.966 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0006579 2024-08-17T17:18:59.966 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.966 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.966 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:18:59.966 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878132+0000", 2024-08-17T17:18:59.966 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0014690790000000001 2024-08-17T17:18:59.966 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.967 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.967 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.967 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882446+0000", 2024-08-17T17:18:59.967 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.004314249 2024-08-17T17:18:59.967 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.967 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.967 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.967 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.021883+0000", 2024-08-17T17:18:59.967 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.139436485 2024-08-17T17:18:59.967 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.967 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.967 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:18:59.967 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.021974+0000", 2024-08-17T17:18:59.967 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0816000000000006e-05 2024-08-17T17:18:59.968 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.968 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.968 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.968 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.022022+0000", 2024-08-17T17:18:59.968 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7794999999999999e-05 2024-08-17T17:18:59.968 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.968 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.968 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.968 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.968 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.968 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35816 6.0 6:0defa481:::benchmark_data_smithi063_160268_object35815:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:18:59.968 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.880228+0000", 2024-08-17T17:18:59.968 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 317.34994324199999, 2024-08-17T17:18:59.968 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.146786307, 2024-08-17T17:18:59.968 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.969 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.969 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:59.969 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:18:59.969 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:18:59.969 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:59.969 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35816 2024-08-17T17:18:59.969 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.969 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.969 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.969 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.969 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880228+0000", 2024-08-17T17:18:59.969 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.969 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.970 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.970 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.970 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880228+0000", 2024-08-17T17:18:59.970 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.970 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.970 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.970 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.970 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880229+0000", 2024-08-17T17:18:59.970 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3850000000000001e-06 2024-08-17T17:18:59.970 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.970 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.970 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.970 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880255+0000", 2024-08-17T17:18:59.971 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5371000000000002e-05 2024-08-17T17:18:59.971 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.971 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.971 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.971 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880267+0000", 2024-08-17T17:18:59.971 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2503999999999999e-05 2024-08-17T17:18:59.971 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.971 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.971 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.971 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880325+0000", 2024-08-17T17:18:59.971 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8196999999999999e-05 2024-08-17T17:18:59.971 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.971 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.972 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.972 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880405+0000", 2024-08-17T17:18:59.972 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9374000000000003e-05 2024-08-17T17:18:59.972 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.972 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.972 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.972 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880565+0000", 2024-08-17T17:18:59.972 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000160097 2024-08-17T17:18:59.972 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.972 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.972 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:18:59.972 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880722+0000", 2024-08-17T17:18:59.973 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015664300000000001 2024-08-17T17:18:59.973 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.973 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.973 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.973 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882511+0000", 2024-08-17T17:18:59.973 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0017894009999999999 2024-08-17T17:18:59.973 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.973 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.973 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:18:59.973 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882525+0000", 2024-08-17T17:18:59.973 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3806e-05 2024-08-17T17:18:59.973 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.974 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.974 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.974 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.026906+0000", 2024-08-17T17:18:59.974 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.144381238 2024-08-17T17:18:59.974 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.974 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.974 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:18:59.974 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.026970+0000", 2024-08-17T17:18:59.974 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4058000000000002e-05 2024-08-17T17:18:59.974 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.974 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.974 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.974 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027014+0000", 2024-08-17T17:18:59.974 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4233000000000002e-05 2024-08-17T17:18:59.975 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.975 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.975 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.975 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.975 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.975 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:103979 6.0 6:0ed307be:::benchmark_data_smithi063_160268_object103978:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-08-17T17:18:59.975 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:14:32.735002+0000", 2024-08-17T17:18:59.975 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 265.49516888599999, 2024-08-17T17:18:59.975 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.072518333000000004, 2024-08-17T17:18:59.975 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.975 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.975 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:59.975 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:18:59.975 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:18:59.976 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:59.976 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 103979 2024-08-17T17:18:59.976 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.976 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.976 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.976 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.976 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735002+0000", 2024-08-17T17:18:59.976 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.976 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.976 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.976 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.976 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735002+0000", 2024-08-17T17:18:59.976 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.976 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.977 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.977 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.977 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735004+0000", 2024-08-17T17:18:59.977 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6920000000000001e-06 2024-08-17T17:18:59.977 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.977 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.977 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.977 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735030+0000", 2024-08-17T17:18:59.977 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5859000000000001e-05 2024-08-17T17:18:59.977 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.977 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.977 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.977 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735046+0000", 2024-08-17T17:18:59.977 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6011e-05 2024-08-17T17:18:59.977 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.978 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.978 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.978 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735055+0000", 2024-08-17T17:18:59.978 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.687e-06 2024-08-17T17:18:59.978 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.978 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.978 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.978 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735117+0000", 2024-08-17T17:18:59.978 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2593000000000002e-05 2024-08-17T17:18:59.978 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.978 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.978 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.978 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735265+0000", 2024-08-17T17:18:59.978 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014776899999999999 2024-08-17T17:18:59.979 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.979 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.979 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:18:59.979 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.735410+0000", 2024-08-17T17:18:59.979 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014535700000000001 2024-08-17T17:18:59.979 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.979 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.979 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:18:59.979 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.738784+0000", 2024-08-17T17:18:59.979 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0033735420000000002 2024-08-17T17:18:59.979 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.979 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.979 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.980 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.743477+0000", 2024-08-17T17:18:59.980 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0046932880000000003 2024-08-17T17:18:59.980 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.980 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.980 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.980 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.807327+0000", 2024-08-17T17:18:59.980 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.063849631000000004 2024-08-17T17:18:59.980 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.980 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.980 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:18:59.980 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.807447+0000", 2024-08-17T17:18:59.980 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000120333 2024-08-17T17:18:59.980 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.981 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.981 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.981 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.807521+0000", 2024-08-17T17:18:59.981 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3571000000000004e-05 2024-08-17T17:18:59.981 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.981 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.981 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.981 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.981 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.981 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:103981 6.0 6:0058b2e1:::benchmark_data_smithi063_160268_object103980:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1100)", 2024-08-17T17:18:59.981 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:14:32.740091+0000", 2024-08-17T17:18:59.981 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 265.49008052300002, 2024-08-17T17:18:59.981 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.067609118999999995, 2024-08-17T17:18:59.981 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.982 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.982 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:59.982 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:18:59.982 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:18:59.982 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:18:59.982 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 103981 2024-08-17T17:18:59.982 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.982 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.982 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.982 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.982 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740091+0000", 2024-08-17T17:18:59.982 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.982 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.983 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.983 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.983 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740091+0000", 2024-08-17T17:18:59.983 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.983 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.983 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.983 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.983 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740092+0000", 2024-08-17T17:18:59.983 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.491e-06 2024-08-17T17:18:59.983 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.983 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.983 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.983 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740111+0000", 2024-08-17T17:18:59.983 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9270999999999999e-05 2024-08-17T17:18:59.984 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.984 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.984 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.984 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740120+0000", 2024-08-17T17:18:59.984 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7129999999999994e-06 2024-08-17T17:18:59.984 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.984 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.984 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.984 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740125+0000", 2024-08-17T17:18:59.984 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0150000000000003e-06 2024-08-17T17:18:59.984 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.984 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.984 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.984 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740169+0000", 2024-08-17T17:18:59.984 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3511999999999999e-05 2024-08-17T17:18:59.985 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.985 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.985 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.985 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740267+0000", 2024-08-17T17:18:59.985 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7802999999999999e-05 2024-08-17T17:18:59.985 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.985 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.985 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:18:59.985 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.740356+0000", 2024-08-17T17:18:59.985 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.9894000000000004e-05 2024-08-17T17:18:59.985 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.985 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.985 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:18:59.985 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.743518+0000", 2024-08-17T17:18:59.986 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0031620210000000001 2024-08-17T17:18:59.986 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.986 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.986 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.986 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.777842+0000", 2024-08-17T17:18:59.986 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.034323448999999999 2024-08-17T17:18:59.986 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.986 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.986 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:18:59.986 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.807590+0000", 2024-08-17T17:18:59.986 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.029748465000000002 2024-08-17T17:18:59.986 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.986 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.986 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:18:59.987 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.807660+0000", 2024-08-17T17:18:59.987 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9906000000000005e-05 2024-08-17T17:18:59.987 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.987 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.987 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.987 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:32.807700+0000", 2024-08-17T17:18:59.987 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9579000000000002e-05 2024-08-17T17:18:59.987 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.987 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.987 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.987 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.987 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.987 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "MOSDPGPush(7.5 1148/1144 [PushOp(7:aced7b73:::benchmark_data_smithi063_196002_object2401:head, version: 1132'148, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:aced7b73:::benchmark_data_smithi063_196002_object2401:head@1132'148, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:18:59.988 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:37.677154+0000", 2024-08-17T17:18:59.988 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 20.55301721, 2024-08-17T17:18:59.988 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.067687675000000003, 2024-08-17T17:18:59.988 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.988 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.988 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:18:59.988 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.988 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.988 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.988 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:37.677154+0000", 2024-08-17T17:18:59.988 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.988 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.988 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.988 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.989 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:37.677154+0000", 2024-08-17T17:18:59.989 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.989 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.989 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.989 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.989 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:37.677154+0000", 2024-08-17T17:18:59.989 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3300000000000001e-07 2024-08-17T17:18:59.989 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.989 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.989 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.989 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:37.677202+0000", 2024-08-17T17:18:59.989 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7422000000000001e-05 2024-08-17T17:18:59.989 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.989 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.989 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.990 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:37.677218+0000", 2024-08-17T17:18:59.990 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6027000000000001e-05 2024-08-17T17:18:59.990 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.990 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.990 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.990 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:37.677231+0000", 2024-08-17T17:18:59.990 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3662000000000001e-05 2024-08-17T17:18:59.990 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.990 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.990 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.990 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:37.742884+0000", 2024-08-17T17:18:59.990 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.065652622999999993 2024-08-17T17:18:59.990 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.990 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.991 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.991 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:37.742902+0000", 2024-08-17T17:18:59.991 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7646e-05 2024-08-17T17:18:59.991 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.991 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.991 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.991 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:37.744842+0000", 2024-08-17T17:18:59.991 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001940062 2024-08-17T17:18:59.991 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.991 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.991 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.991 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.991 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.991 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "MOSDPGPush(7.5 1149/1144 [PushOp(7:aa8dfb9c:::benchmark_data_smithi063_196002_object9253:head, version: 1132'606, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:aa8dfb9c:::benchmark_data_smithi063_196002_object9253:head@1132'606, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:18:59.992 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:53.656740+0000", 2024-08-17T17:18:59.992 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 4.5734311099999996, 2024-08-17T17:18:59.992 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.152369581, 2024-08-17T17:18:59.992 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.992 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.992 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:18:59.992 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.992 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.992 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.992 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656740+0000", 2024-08-17T17:18:59.992 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.992 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.992 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.992 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.993 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656740+0000", 2024-08-17T17:18:59.993 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.993 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.993 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.993 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.993 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656741+0000", 2024-08-17T17:18:59.993 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0079999999999999e-06 2024-08-17T17:18:59.993 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.993 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.993 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.993 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656947+0000", 2024-08-17T17:18:59.993 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00020619200000000001 2024-08-17T17:18:59.993 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.993 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.993 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.994 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656975+0000", 2024-08-17T17:18:59.994 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.7746000000000002e-05 2024-08-17T17:18:59.994 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.994 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.994 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.994 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656998+0000", 2024-08-17T17:18:59.994 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3006e-05 2024-08-17T17:18:59.994 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.994 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.994 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.994 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.803287+0000", 2024-08-17T17:18:59.994 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14628891699999999 2024-08-17T17:18:59.994 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.994 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.995 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.995 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.803301+0000", 2024-08-17T17:18:59.995 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3920000000000001e-05 2024-08-17T17:18:59.995 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.995 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.995 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:18:59.995 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.809110+0000", 2024-08-17T17:18:59.995 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0058087920000000001 2024-08-17T17:18:59.995 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.995 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:18:59.995 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:18:59.995 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.995 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.996 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:83000 7.5 e1149/1144 7:a4b1338c:::benchmark_data_smithi063_196002_object82999:head v 1149'5278, mlcod=1132'605)", 2024-08-17T17:18:59.996 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:53.728634+0000", 2024-08-17T17:18:59.996 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 4.5015368890000005, 2024-08-17T17:18:59.996 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.080472642999999996, 2024-08-17T17:18:59.996 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:18:59.996 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:18:59.996 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:18:59.996 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:18:59.996 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.996 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:18:59.996 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.728634+0000", 2024-08-17T17:18:59.996 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.997 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.997 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.997 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:18:59.997 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.728634+0000", 2024-08-17T17:18:59.997 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:18:59.997 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.997 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.997 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:18:59.997 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.728635+0000", 2024-08-17T17:18:59.997 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.159e-06 2024-08-17T17:18:59.997 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.997 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.997 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:18:59.997 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.728679+0000", 2024-08-17T17:18:59.998 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3881000000000002e-05 2024-08-17T17:18:59.998 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.998 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.998 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:18:59.998 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.728694+0000", 2024-08-17T17:18:59.998 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.417e-05 2024-08-17T17:18:59.998 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.998 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.998 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:18:59.998 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.728703+0000", 2024-08-17T17:18:59.998 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.73e-06 2024-08-17T17:18:59.998 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.756776+0000", 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.028072901000000001 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.756796+0000", 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9544000000000001e-05 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.809068+0000", 2024-08-17T17:18:59.999 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.052271973999999999 2024-08-17T17:19:00.000 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.000 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.000 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:00.000 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.809107+0000", 2024-08-17T17:19:00.000 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9283999999999999e-05 2024-08-17T17:19:00.000 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:00.000 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:00.000 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:00.000 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.000 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.000 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:83001 7.d e1149/1144 7:beff0267:::benchmark_data_smithi063_196002_object83000:head v 1149'5169, mlcod=0'0)", 2024-08-17T17:19:00.000 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:53.729888+0000", 2024-08-17T17:19:00.000 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 4.500282736, 2024-08-17T17:19:00.001 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.079173338999999995, 2024-08-17T17:19:00.001 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:00.001 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:00.001 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:00.001 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:00.001 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.001 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:00.001 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.729888+0000", 2024-08-17T17:19:00.001 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:00.001 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.001 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.001 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:00.001 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.729888+0000", 2024-08-17T17:19:00.001 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:00.001 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.002 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.002 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:00.002 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.729889+0000", 2024-08-17T17:19:00.002 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9000000000000005e-07 2024-08-17T17:19:00.002 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.002 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.002 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:00.002 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.729945+0000", 2024-08-17T17:19:00.002 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5047000000000003e-05 2024-08-17T17:19:00.002 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.002 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.002 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:00.003 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.729956+0000", 2024-08-17T17:19:00.003 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1489e-05 2024-08-17T17:19:00.003 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.003 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.003 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:00.003 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.729963+0000", 2024-08-17T17:19:00.003 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9349999999999997e-06 2024-08-17T17:19:00.003 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.003 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.003 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:00.003 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.754853+0000", 2024-08-17T17:19:00.003 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.024889582 2024-08-17T17:19:00.003 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.003 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.003 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:00.004 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.754885+0000", 2024-08-17T17:19:00.004 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.2515999999999998e-05 2024-08-17T17:19:00.004 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.004 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.004 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:00.004 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.809004+0000", 2024-08-17T17:19:00.004 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.054119338000000003 2024-08-17T17:19:00.004 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.004 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.004 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:00.004 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.809062+0000", 2024-08-17T17:19:00.004 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7442000000000003e-05 2024-08-17T17:19:00.004 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:00.004 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:00.005 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:00.005 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.005 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.005 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:83005 7.5 e1149/1144 7:a67db851:::benchmark_data_smithi063_196002_object83004:head v 1149'5279, mlcod=1132'605)", 2024-08-17T17:19:00.005 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:53.730394+0000", 2024-08-17T17:19:00.005 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 4.4997773969999999, 2024-08-17T17:19:00.006 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.078745140000000005, 2024-08-17T17:19:00.006 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:00.006 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:00.006 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:00.006 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:00.006 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.006 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:00.006 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.730394+0000", 2024-08-17T17:19:00.006 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:00.006 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.006 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.006 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:00.007 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.730394+0000", 2024-08-17T17:19:00.007 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:00.007 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.007 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.007 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:00.007 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.730395+0000", 2024-08-17T17:19:00.007 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1699999999999997e-07 2024-08-17T17:19:00.007 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.007 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.007 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:00.007 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.730428+0000", 2024-08-17T17:19:00.007 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3015999999999997e-05 2024-08-17T17:19:00.007 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.008 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.008 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:00.008 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.730439+0000", 2024-08-17T17:19:00.008 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1134e-05 2024-08-17T17:19:00.008 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.008 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.008 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:00.008 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.730446+0000", 2024-08-17T17:19:00.008 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0720000000000002e-06 2024-08-17T17:19:00.008 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.008 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.008 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:00.009 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.763139+0000", 2024-08-17T17:19:00.009 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.032692802 2024-08-17T17:19:00.009 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.009 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.009 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:00.009 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.763164+0000", 2024-08-17T17:19:00.009 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5437000000000001e-05 2024-08-17T17:19:00.009 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.009 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.009 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:00.009 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.809113+0000", 2024-08-17T17:19:00.009 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.045948680999999998 2024-08-17T17:19:00.009 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:00.010 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:00.010 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:00.010 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.809139+0000", 2024-08-17T17:19:00.010 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6281000000000001e-05 2024-08-17T17:19:00.010 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:00.010 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:00.010 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:00.010 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:00.010 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:00.010 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:19:00.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:00.011 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:00.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:59 smithi162 ceph-mon[104668]: pgmap v1361: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 50 MiB/s wr, 802 op/s; 28763/255351 objects degraded (11.264%); 3474/255351 objects misplaced (1.360%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:19:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:59 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:19:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:18:59 smithi162 ceph-mon[104668]: osdmap e1150: 8 total, 8 up, 8 in 2024-08-17T17:19:00.112 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:00.159 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:00.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:00.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:00.260 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:00.286 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:00.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:00.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:00.388 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:00.414 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:00.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:00.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:00.447 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 62 16 88219 88203 88.8955 50 0.00647166 0.0112408 2024-08-17T17:19:00.516 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:00.567 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:00.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:00.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:00.669 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:00.740 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:00.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:00.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:00.842 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:00.863 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:00.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:00.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:00.965 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:00.995 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:00.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:00.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:00 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:19:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:00 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:19:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:00 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:19:01.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:00 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:19:01.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:00 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:19:01.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:00 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:19:01.097 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:01.120 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:01.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:01.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:01.159 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:19:01.222 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:01.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:01.244 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:01.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:01.345 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:01.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:01.371 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:01.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:01.447 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 63 16 88399 88383 87.6631 11.25 0.00363837 0.0112904 2024-08-17T17:19:01.472 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:01.493 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:01.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:01.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:01.595 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:01.618 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:01.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:01.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:01.720 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:01.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:01.758 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:01.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:01 smithi063 ceph-mon[114756]: pgmap v1363: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 28763/258276 objects degraded (11.137%); 3474/258276 objects misplaced (1.345%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:19:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:19:01.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:01 smithi063 ceph-mon[114756]: osdmap e1151: 8 total, 8 up, 8 in 2024-08-17T17:19:01.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:19:01.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:01 smithi063 ceph-mon[109843]: pgmap v1363: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 28763/258276 objects degraded (11.137%); 3474/258276 objects misplaced (1.345%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:19:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:19:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:01 smithi063 ceph-mon[109843]: osdmap e1151: 8 total, 8 up, 8 in 2024-08-17T17:19:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:19:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:01.859 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:01.882 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:01.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:01.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:01.984 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:02.006 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:02.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:02.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:01 smithi162 ceph-mon[104668]: pgmap v1363: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 28763/258276 objects degraded (11.137%); 3474/258276 objects misplaced (1.345%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:19:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:19:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:01 smithi162 ceph-mon[104668]: osdmap e1151: 8 total, 8 up, 8 in 2024-08-17T17:19:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:19:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:02.108 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:02.131 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:02.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:02.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:02.174 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:02.179 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:02.233 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:02.256 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:02.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:02.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:02.358 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:02.383 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:02.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:02.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:02.447 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 64 16 88451 88435 86.3443 3.25 0.00374 0.0113797 2024-08-17T17:19:02.484 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:02.508 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:02.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:02.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:02.609 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:02.632 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:02.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:02.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:02.651 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:19:02.734 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:02.761 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:02.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:02.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:02.863 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:02.893 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:02 smithi063 ceph-mon[114756]: osdmap e1152: 8 total, 8 up, 8 in 2024-08-17T17:19:02.893 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:02 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3097130585' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:19:02.893 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:02 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:19:02.894 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:02 smithi063 ceph-mon[109843]: osdmap e1152: 8 total, 8 up, 8 in 2024-08-17T17:19:02.894 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:02 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3097130585' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:19:02.894 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:02 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:19:02.908 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:02.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:02.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:03.010 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:03.038 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:03.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:03.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:02 smithi162 ceph-mon[104668]: osdmap e1152: 8 total, 8 up, 8 in 2024-08-17T17:19:03.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:02 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3097130585' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:19:03.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:02 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:19:03.140 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:03.184 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:03.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:03.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:03.286 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:03.311 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:03.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:03.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:03.413 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:03.448 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 65 16 89251 89235 85.7842 50 0.00559335 0.0116508 2024-08-17T17:19:03.461 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:03.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:03.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:03.550 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:19:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:19:03] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-08-17T17:19:03.563 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:03.592 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:03.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:03.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:03.694 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:03.737 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:03.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:03.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:03.839 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:03.864 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:03.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:03.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:03.967 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:03.993 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:03.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:03.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:03.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:03 smithi162 ceph-mon[104668]: pgmap v1366: 109 pgs: 1 active+recovery_wait+degraded, 2 peering, 9 unknown, 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s; 22620/261153 objects degraded (8.662%); 7629/261153 objects misplaced (2.921%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:19:03.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:03 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:19:03.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:03 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:19:03.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:03 smithi162 ceph-mon[104668]: osdmap e1153: 8 total, 8 up, 8 in 2024-08-17T17:19:03.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:03 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 22620/261153 objects degraded (8.662%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:03.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:03 smithi162 ceph-mon[104668]: 7.2 scrub starts 2024-08-17T17:19:04.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:03 smithi063 ceph-mon[114756]: pgmap v1366: 109 pgs: 1 active+recovery_wait+degraded, 2 peering, 9 unknown, 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s; 22620/261153 objects degraded (8.662%); 7629/261153 objects misplaced (2.921%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:19:04.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:03 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:19:04.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:03 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:19:04.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:03 smithi063 ceph-mon[114756]: osdmap e1153: 8 total, 8 up, 8 in 2024-08-17T17:19:04.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:03 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 22620/261153 objects degraded (8.662%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:04.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:03 smithi063 ceph-mon[114756]: 7.2 scrub starts 2024-08-17T17:19:04.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:03 smithi063 ceph-mon[109843]: pgmap v1366: 109 pgs: 1 active+recovery_wait+degraded, 2 peering, 9 unknown, 2 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 51 MiB/s wr, 823 op/s; 22620/261153 objects degraded (8.662%); 7629/261153 objects misplaced (2.921%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:19:04.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:03 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:19:04.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:03 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:19:04.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:03 smithi063 ceph-mon[109843]: osdmap e1153: 8 total, 8 up, 8 in 2024-08-17T17:19:04.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:03 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 22620/261153 objects degraded (8.662%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:04.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:03 smithi063 ceph-mon[109843]: 7.2 scrub starts 2024-08-17T17:19:04.096 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:04.134 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:04.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:04.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:04.237 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:04.304 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:04.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:04.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:04.405 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:04.447 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:04.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:04.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:04.448 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 66 16 90304 90288 85.4815 65.8125 0.0412452 0.0116915 2024-08-17T17:19:04.549 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:04.596 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:04.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:04.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:04.698 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:04.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:04.746 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:04.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:04.839 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:19:04.840 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:19:04.840 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2024-08-17T17:19:04.840 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:19:04.840 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:19:04.840 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 config set filestore_inject_stall 3 2024-08-17T17:19:04.847 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:04.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:04.883 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:04.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:04.983 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:05.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:05.018 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:05.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:05.119 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:05.143 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:19:05.156 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:05.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:05.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:05.258 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:05.295 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:05.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:05.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:05.397 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:05.442 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:05.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:05.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:05.448 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 67 16 91304 91288 85.1385 62.5 0.00545167 0.0117394 2024-08-17T17:19:05.544 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:05.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:05.590 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:05.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:05.691 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:05.727 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:05.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:05.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:05.829 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:05.863 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:05.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:05.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:05.928 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:05 smithi162 ceph-mon[104668]: pgmap v1368: 109 pgs: 5 active+recovery_wait+degraded, 6 peering, 3 unknown, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 35 MiB/s wr, 562 op/s; 18235/264711 objects degraded (6.889%); 4155/264711 objects misplaced (1.570%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:19:05.928 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:05 smithi162 ceph-mon[104668]: 7.12 scrub starts 2024-08-17T17:19:05.965 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:05.994 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:05.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:05.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:06.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:05 smithi063 ceph-mon[109843]: pgmap v1368: 109 pgs: 5 active+recovery_wait+degraded, 6 peering, 3 unknown, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 35 MiB/s wr, 562 op/s; 18235/264711 objects degraded (6.889%); 4155/264711 objects misplaced (1.570%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:19:06.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:05 smithi063 ceph-mon[109843]: 7.12 scrub starts 2024-08-17T17:19:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:05 smithi063 ceph-mon[114756]: pgmap v1368: 109 pgs: 5 active+recovery_wait+degraded, 6 peering, 3 unknown, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 35 MiB/s wr, 562 op/s; 18235/264711 objects degraded (6.889%); 4155/264711 objects misplaced (1.570%); 1.5 MiB/s, 23 objects/s recovering 2024-08-17T17:19:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:05 smithi063 ceph-mon[114756]: 7.12 scrub starts 2024-08-17T17:19:06.096 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:06.143 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:06.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:06.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:06.245 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:06.414 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:19:06.441 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:06.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:06.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:06.448 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 68 16 92339 92323 84.8377 64.6875 0.00883419 0.0117814 2024-08-17T17:19:06.543 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:06.570 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:06.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:06.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:06.672 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:06.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:06.713 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:06.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:06.814 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:06.844 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:06.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:06.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:06.946 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:06.974 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:06.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:06.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:07.076 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:07.120 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:07.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:07.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:07.222 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:07.271 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:19:07.271 INFO:teuthology.orchestra.run.smithi162.stdout: "success": "" 2024-08-17T17:19:07.271 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:19:07.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:07.275 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:07.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:07.323 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:19:07.324 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:19:07.364 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:19:07.365 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:19:07.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:07.413 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:19:07.413 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:19:07.420 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:07.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:07.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:07.448 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:19:07.449 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:19:07.449 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 69 16 93280 93264 84.4605 58.8125 0.0205845 0.0118324 2024-08-17T17:19:07.487 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:19:07.487 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:19:07.522 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:07.525 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:19:07.526 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:19:07.539 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:19:07.539 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:19:07.548 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:07.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:07.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:07.572 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:19:07.573 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:19:07.616 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:19:07.616 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:19:07.650 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:07.704 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:07.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:07.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:07.806 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:07.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:07.846 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:07.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:07.947 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:07.973 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:07.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:07.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:07 smithi063 ceph-mon[109843]: pgmap v1369: 109 pgs: 5 active+recovery_wait+degraded, 6 peering, 1 unknown, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 30 MiB/s wr, 479 op/s; 18235/266910 objects degraded (6.832%); 4155/266910 objects misplaced (1.557%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:19:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:07 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3367554665' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:07 smithi063 ceph-mon[114756]: pgmap v1369: 109 pgs: 5 active+recovery_wait+degraded, 6 peering, 1 unknown, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 30 MiB/s wr, 479 op/s; 18235/266910 objects degraded (6.832%); 4155/266910 objects misplaced (1.557%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:19:08.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:07 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3367554665' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:08.074 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:07 smithi162 ceph-mon[104668]: pgmap v1369: 109 pgs: 5 active+recovery_wait+degraded, 6 peering, 1 unknown, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 30 MiB/s wr, 479 op/s; 18235/266910 objects degraded (6.832%); 4155/266910 objects misplaced (1.557%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:19:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:07 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3367554665' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:08.101 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:08.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:08.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:08.203 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:08.230 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:08.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:08.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:08.332 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:08.361 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:08.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:08.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:08.448 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 70 16 94214 94198 84.0878 58.375 0.0135685 0.0118859 2024-08-17T17:19:08.463 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:08.510 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:08.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:08.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:08.612 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:08.650 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:08.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:08.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:08.752 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:08.779 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:08.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:08.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:08.831 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:08.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:08 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 18235/266910 objects degraded (6.832%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:08.881 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:08.907 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:08.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:08.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:08.949 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:08 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 18235/266910 objects degraded (6.832%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:08.949 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:08 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 18235/266910 objects degraded (6.832%), 7 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:09.010 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:09.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:09.055 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:09.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:09.156 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:09.184 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:09.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:09.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:09.286 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:09.288 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:19:09.313 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:09.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:09.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:09.416 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:09.443 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:09.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:09.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:09.448 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 71 16 95230 95214 83.7978 63.5 0.0118062 0.011926 2024-08-17T17:19:09.545 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:09.569 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:09.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:09.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:09.671 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:09.699 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:09.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:09.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:09.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:09 smithi063 ceph-mon[109843]: pgmap v1370: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s; 27952/277020 objects degraded (10.090%); 3944/277020 objects misplaced (1.424%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:19:09.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:09 smithi063 ceph-mon[109843]: 7.2 scrub starts 2024-08-17T17:19:09.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:09 smithi063 ceph-mon[114756]: pgmap v1370: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s; 27952/277020 objects degraded (10.090%); 3944/277020 objects misplaced (1.424%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:19:09.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:09 smithi063 ceph-mon[114756]: 7.2 scrub starts 2024-08-17T17:19:09.801 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:09.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:09.859 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:09.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:09.960 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:10.001 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:10.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:10.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:09 smithi162 ceph-mon[104668]: pgmap v1370: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s; 27952/277020 objects degraded (10.090%); 3944/277020 objects misplaced (1.424%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:19:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:09 smithi162 ceph-mon[104668]: 7.2 scrub starts 2024-08-17T17:19:10.102 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:10.143 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:10.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:10.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:10.245 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:10.287 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:10.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:10.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:10.389 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:10.435 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:10.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:10.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:10.448 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 72 16 96172 96156 83.4516 58.875 0.0141334 0.0119774 2024-08-17T17:19:10.536 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:10.564 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:10.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:10.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:10.666 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:10.707 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:10.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:10.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:10.809 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:10.837 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:10.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:10.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:10.847 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:19:10.939 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:10.980 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:10.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:10.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:11.082 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:11.107 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:11.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:11.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:11.209 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:11.238 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:11.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:11.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:11.339 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:11.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:11.379 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:11.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:11.449 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 73 16 97106 97090 83.108 58.375 0.00582426 0.0120249 2024-08-17T17:19:11.480 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:11.509 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:11.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:11.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:11.610 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:11.636 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:11.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:11.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:11.738 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:11.774 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:11.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:11.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:11.876 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:11.912 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:11.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:11.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:12.013 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:11 smithi063 ceph-mon[114756]: pgmap v1371: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s; 27952/280764 objects degraded (9.956%); 3944/280764 objects misplaced (1.405%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:19:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:11 smithi063 ceph-mon[114756]: 7.12 scrub starts 2024-08-17T17:19:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:11 smithi063 ceph-mon[109843]: pgmap v1371: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s; 27952/280764 objects degraded (9.956%); 3944/280764 objects misplaced (1.405%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:19:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:11 smithi063 ceph-mon[109843]: 7.12 scrub starts 2024-08-17T17:19:12.043 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:12.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:12.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:11 smithi162 ceph-mon[104668]: pgmap v1371: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 828 op/s; 27952/280764 objects degraded (9.956%); 3944/280764 objects misplaced (1.405%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:19:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:11 smithi162 ceph-mon[104668]: 7.12 scrub starts 2024-08-17T17:19:12.145 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:12.187 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:12.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:12.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:12.289 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:12.318 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:12.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:12.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:12.420 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:12.448 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:12.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:12.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:12.449 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 74 16 98008 97992 82.7467 56.375 0.00667945 0.0120794 2024-08-17T17:19:12.550 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:12.590 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:12.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:12.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:12.692 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:12.732 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:12.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:12.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:12.834 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:12.864 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:12.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:12.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:12.966 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:12.991 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:12.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:12.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:13.093 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:13.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:13.149 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:13.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:13.250 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:13.272 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:13.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:13.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:13.374 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:13.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:13.427 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:13.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:13.449 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 75 16 98952 98936 82.4301 59 0.00704092 0.0121247 2024-08-17T17:19:13.518 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:19:13.519 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:19:13.519 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_2 2024-08-17T17:19:13.519 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:19:13.528 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:19:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:19:13] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-08-17T17:19:13.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:13.590 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:13.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:13.669 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:13.691 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:13.713 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:13.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:13.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:13.814 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:13.825 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:13.827 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:13 smithi063 ceph-mon[114756]: pgmap v1372: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s; 27952/287469 objects degraded (9.723%); 3838/287469 objects misplaced (1.335%); 2.6 MiB/s, 42 objects/s recovering 2024-08-17T17:19:13.827 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:13 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 27952/287469 objects degraded (9.723%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:13.827 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:13 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:19:13.827 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:13 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:19:13.828 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:13 smithi063 ceph-mon[109843]: pgmap v1372: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s; 27952/287469 objects degraded (9.723%); 3838/287469 objects misplaced (1.335%); 2.6 MiB/s, 42 objects/s recovering 2024-08-17T17:19:13.828 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:13 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 27952/287469 objects degraded (9.723%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:13.828 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:13 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:19:13.828 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:13 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:19:13.839 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:13.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:13.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:13.941 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:13.967 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:13.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:13.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:14.069 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:13 smithi162 ceph-mon[104668]: pgmap v1372: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 55 MiB/s wr, 876 op/s; 27952/287469 objects degraded (9.723%); 3838/287469 objects misplaced (1.335%); 2.6 MiB/s, 42 objects/s recovering 2024-08-17T17:19:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:13 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 27952/287469 objects degraded (9.723%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:13 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:19:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:13 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:19:14.096 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:14.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:14.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:14.198 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:14.220 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:14.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:14.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:14.322 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:14.350 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:14.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:14.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:14.449 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 76 16 99735 99719 81.9894 48.9375 0.00891813 0.0121911 2024-08-17T17:19:14.452 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:14.480 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:14.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:14.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:14.582 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:14.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:14.621 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:14.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:14.722 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:14.759 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:14.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:14.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:14.861 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:14.888 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:14.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:14.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:14.989 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:14.995 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:14 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:19:14.995 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:14 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:19:14.995 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:14 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:19:14.995 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:14 smithi162 ceph-mon[104668]: 7.2 scrub starts 2024-08-17T17:19:15.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.013 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:15.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:15.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:14 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:19:15.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:14 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:19:15.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:14 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:19:15.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:14 smithi063 ceph-mon[114756]: 7.2 scrub starts 2024-08-17T17:19:15.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:14 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:19:15.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:14 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:19:15.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:14 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:19:15.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:14 smithi063 ceph-mon[109843]: 7.2 scrub starts 2024-08-17T17:19:15.114 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:15.144 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:15.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:15.246 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:15.288 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:15.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:15.390 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:15.416 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:15.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:15.449 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 77 16 100635 100619 81.6551 56.25 0.00345458 0.0122408 2024-08-17T17:19:15.518 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:15.542 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:15.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:15.644 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:15.697 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:15.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:15.799 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:15.839 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:15.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:15.937 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:15 smithi063 ceph-mon[114756]: pgmap v1373: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s; 27952/291105 objects degraded (9.602%); 3838/291105 objects misplaced (1.318%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:19:15.938 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:15 smithi063 ceph-mon[109843]: pgmap v1373: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s; 27952/291105 objects degraded (9.602%); 3838/291105 objects misplaced (1.318%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:19:15.941 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:15.969 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:15.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:15.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:16.071 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:15 smithi162 ceph-mon[104668]: pgmap v1373: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 48 MiB/s wr, 763 op/s; 27952/291105 objects degraded (9.602%); 3838/291105 objects misplaced (1.318%); 1.7 MiB/s, 27 objects/s recovering 2024-08-17T17:19:16.114 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:16.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:16.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:16.216 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:16.256 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:16.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:16.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:16.358 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:16.385 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:16.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:16.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:16.449 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 78 16 101614 101598 81.3926 61.1875 0.0147795 0.0122798 2024-08-17T17:19:16.487 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:16.513 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:16.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:16.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:16.615 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:16.652 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:16.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:16.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:16.754 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:16.801 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:16.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:16.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:16.903 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:16.931 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:16.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:16.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:17.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:17.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:16 smithi063 ceph-mon[109843]: 7.12 scrub starts 2024-08-17T17:19:17.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:17.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:16 smithi063 ceph-mon[114756]: 7.12 scrub starts 2024-08-17T17:19:17.033 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:17.079 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:17.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:17.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:16 smithi162 ceph-mon[104668]: 7.12 scrub starts 2024-08-17T17:19:17.181 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:17.210 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:17.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:17.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:17.311 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:17.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:17.361 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:17.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:17.449 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 79 16 102609 102593 81.1495 62.1875 0.0089249 0.0123172 2024-08-17T17:19:17.462 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:17.487 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:17.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:17.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:17.589 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:17.637 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:17.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:17.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:17.740 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:17.765 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:17.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:17.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:17.854 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:17 smithi063 ceph-mon[114756]: pgmap v1374: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s; 27952/294660 objects degraded (9.486%); 3838/294660 objects misplaced (1.303%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:19:17.855 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:17 smithi063 ceph-mon[109843]: pgmap v1374: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s; 27952/294660 objects degraded (9.486%); 3838/294660 objects misplaced (1.303%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:19:17.867 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:17.913 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:17.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:17.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:18.015 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:18.040 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:18.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:18.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:17 smithi162 ceph-mon[104668]: pgmap v1374: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 52 MiB/s wr, 831 op/s; 27952/294660 objects degraded (9.486%); 3838/294660 objects misplaced (1.303%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:19:18.142 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:18.169 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:18.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:18.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:18.271 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:18.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:18.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:18.300 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:18.401 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:18.428 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:18.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:18.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:18.449 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:19:18.449742+0000 min lat: 0.00126452 max lat: 2.52879 avg lat: 0.0123615 2024-08-17T17:19:18.449 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:19:18.450 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 80 16 103533 103517 80.8569 57.75 0.0430538 0.0123615 2024-08-17T17:19:18.530 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:18.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:18.591 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:18.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:18.691 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:18.710 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:18 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 27952/294660 objects degraded (9.486%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:18.710 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:18 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3723549696' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:19:18.734 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:19:18.745 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:18.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:18.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:18.755 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:19:18.755 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:19:18.848 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:18.891 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:18.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:18.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:18.984 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:18 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 27952/294660 objects degraded (9.486%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:18.985 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:18 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3723549696' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:19:18.993 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:19.019 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:19.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:19.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:18 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 27952/294660 objects degraded (9.486%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:19.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:18 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3723549696' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:19:19.121 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:19.149 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:19.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:19.251 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:19.275 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:19.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:19.377 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:19.404 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:19.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:19.421 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:19:19.421 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:19:19.450 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 81 16 104396 104380 80.5246 53.9375 0.00967994 0.0124108 2024-08-17T17:19:19.456 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:19:19.457 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:19:19.493 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:19:19.493 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:19:19.506 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:19.532 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:19.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:19.540 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:19:19.540 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:19:19.574 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:19:19.574 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:19:19.604 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:19:19.604 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:19:19.634 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:19.638 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:19:19.638 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:19:19.673 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:19:19.673 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:19:19.684 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:19.786 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:19.824 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:19.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:19.926 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:19.969 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:19.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:19.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:19 smithi063 ceph-mon[114756]: pgmap v1375: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 27952/304929 objects degraded (9.167%); 3716/304929 objects misplaced (1.219%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:19:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:19 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:19:20.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:19 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3723549696' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:19:20.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:19 smithi063 ceph-mon[114756]: osdmap e1154: 8 total, 8 up, 8 in 2024-08-17T17:19:20.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:19 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/460457436' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:20.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:19 smithi063 ceph-mon[114756]: 7.2 scrub starts 2024-08-17T17:19:20.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:19 smithi063 ceph-mon[109843]: pgmap v1375: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 27952/304929 objects degraded (9.167%); 3716/304929 objects misplaced (1.219%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:19:20.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:19 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:19:20.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:19 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3723549696' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:19:20.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:19 smithi063 ceph-mon[109843]: osdmap e1154: 8 total, 8 up, 8 in 2024-08-17T17:19:20.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:19 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/460457436' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:20.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:19 smithi063 ceph-mon[109843]: 7.2 scrub starts 2024-08-17T17:19:20.071 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:19 smithi162 ceph-mon[104668]: pgmap v1375: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 27952/304929 objects degraded (9.167%); 3716/304929 objects misplaced (1.219%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:19:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:19 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:19:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:19 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3723549696' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:19:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:19 smithi162 ceph-mon[104668]: osdmap e1154: 8 total, 8 up, 8 in 2024-08-17T17:19:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:19 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/460457436' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:19 smithi162 ceph-mon[104668]: 7.2 scrub starts 2024-08-17T17:19:20.098 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:20.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:20.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:20.200 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:20.228 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:20.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:20.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:20.330 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:20.351 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:20.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:20.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:20.450 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 82 16 105205 105189 80.1592 50.5625 0.0466001 0.0124692 2024-08-17T17:19:20.453 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:20.483 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:20.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:20.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:20.585 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:20.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:20.632 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:20.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:20.733 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:20.759 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:20.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:20.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:20.800 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:19:20.861 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:20.888 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:20.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:20.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:20.990 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:21.016 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:21.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:21.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:21.118 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:21.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:21.156 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:21.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:21.257 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:21.283 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:21.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:21.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:21.385 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:21.415 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:21.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:21.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:21.450 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 83 16 106108 106092 79.8733 56.4375 0.020253 0.0125135 2024-08-17T17:19:21.517 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:21.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:21.546 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:21.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:21.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:21.648 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:21.674 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:21.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:21.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:21.776 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:21.803 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:21.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:21.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:21.905 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:21.946 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:21.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:21.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:21.967 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:21 smithi063 ceph-mon[109843]: pgmap v1377: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 27952/308304 objects degraded (9.066%); 3716/308304 objects misplaced (1.205%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:19:21.967 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:21 smithi063 ceph-mon[109843]: 7.2 scrub ok 2024-08-17T17:19:21.967 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:21 smithi063 ceph-mon[109843]: 7.12 scrub starts 2024-08-17T17:19:21.968 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:21 smithi063 ceph-mon[114756]: pgmap v1377: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 27952/308304 objects degraded (9.066%); 3716/308304 objects misplaced (1.205%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:19:21.968 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:21 smithi063 ceph-mon[114756]: 7.2 scrub ok 2024-08-17T17:19:21.968 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:21 smithi063 ceph-mon[114756]: 7.12 scrub starts 2024-08-17T17:19:22.048 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:22.091 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:19:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:21 smithi162 ceph-mon[104668]: pgmap v1377: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 27952/308304 objects degraded (9.066%); 3716/308304 objects misplaced (1.205%); 1.4 MiB/s, 22 objects/s recovering 2024-08-17T17:19:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:21 smithi162 ceph-mon[104668]: 7.2 scrub ok 2024-08-17T17:19:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:21 smithi162 ceph-mon[104668]: 7.12 scrub starts 2024-08-17T17:19:22.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:22.114 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:22.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:22.215 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:22.268 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:22.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:22.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:22.370 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:22.395 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:22.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:22.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:22.450 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 84 16 106947 106931 79.5463 52.4375 0.0196912 0.012565 2024-08-17T17:19:22.497 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:22.555 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:22.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:22.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:22.657 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:22.680 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:22.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:22.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:22.782 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:22.808 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:22.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:22.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:22.819 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:22.909 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:22.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:22.971 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:22.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:23.072 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:23.102 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:23.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:23.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:23.204 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:23.247 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:23.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:23.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:23.349 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:23.391 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:23.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:23.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:23.450 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 85 16 107850 107834 79.2744 56.4375 0.0245524 0.0126069 2024-08-17T17:19:23.493 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:23.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:23.524 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:23.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:23.625 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:23.652 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:23.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:23.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:23.705 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:23.728 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:23 smithi063 ceph-mon[109843]: pgmap v1378: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s; 27952/315210 objects degraded (8.868%); 3623/315210 objects misplaced (1.149%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:19:23.728 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:23 smithi063 ceph-mon[109843]: 7.12 scrub ok 2024-08-17T17:19:23.728 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:23 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 27952/315210 objects degraded (8.868%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:23.728 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:19:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:19:23] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.51.0" 2024-08-17T17:19:23.755 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:23.794 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:23.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:23.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:23.896 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:23.925 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:23.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:23.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:24.014 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:19:24.027 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:23 smithi063 ceph-mon[114756]: pgmap v1378: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s; 27952/315210 objects degraded (8.868%); 3623/315210 objects misplaced (1.149%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:19:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:23 smithi063 ceph-mon[114756]: 7.12 scrub ok 2024-08-17T17:19:24.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:23 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 27952/315210 objects degraded (8.868%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:24.057 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:24.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:24.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:23 smithi162 ceph-mon[104668]: pgmap v1378: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s; 27952/315210 objects degraded (8.868%); 3623/315210 objects misplaced (1.149%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:19:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:23 smithi162 ceph-mon[104668]: 7.12 scrub ok 2024-08-17T17:19:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:23 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 27952/315210 objects degraded (8.868%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:24.158 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:24.182 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:24.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:24.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:24.283 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:24.307 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:24.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:24.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:24.409 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:24.437 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:24.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:24.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:24.451 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 86 16 108784 108768 79.0311 58.375 0.00540694 0.0126478 2024-08-17T17:19:24.539 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:24.570 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:24.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:24.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:24.671 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:24.700 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:24.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:24.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:24.802 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:24.826 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:24.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:24.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:24.928 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:24.977 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:24.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:24.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:25.079 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:25.105 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:25.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:25.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:25.207 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:25.233 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:25.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:25.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:25.335 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:25.366 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:25.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:25.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:25.451 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 87 16 109678 109662 78.7649 55.875 0.00770299 0.01269 2024-08-17T17:19:25.468 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:25.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:25.496 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:25.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:25.597 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:25.621 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:25.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:25.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:25.723 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:25.764 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:25.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:25.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:25.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:25 smithi063 ceph-mon[114756]: pgmap v1379: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 27952/318468 objects degraded (8.777%); 3623/318468 objects misplaced (1.138%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:19:25.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:25 smithi063 ceph-mon[109843]: pgmap v1379: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 27952/318468 objects degraded (8.777%); 3623/318468 objects misplaced (1.138%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:19:25.866 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:25.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:25.916 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:25.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:26.017 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:26.070 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:26.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:26.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:26.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:25 smithi162 ceph-mon[104668]: pgmap v1379: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 1 active+clean+scrubbing, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 57 MiB/s wr, 911 op/s; 27952/318468 objects degraded (8.777%); 3623/318468 objects misplaced (1.138%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:19:26.161 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:19:26.161 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:19:26.161 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:19:26.161 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:19:26.162 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.162 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:29975 6.1 6:82e1ea0d:::benchmark_data_smithi063_160268_object29974:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:19:26.162 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:37.728080+0000", 2024-08-17T17:19:26.162 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 348.42798516699997, 2024-08-17T17:19:26.162 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.075472543000000003, 2024-08-17T17:19:26.162 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.162 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.162 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.162 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:26.162 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:19:26.162 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:19:26.163 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 29975 2024-08-17T17:19:26.163 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.163 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.163 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.163 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.163 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728080+0000", 2024-08-17T17:19:26.163 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.163 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.163 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.163 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.163 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728080+0000", 2024-08-17T17:19:26.163 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.163 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.164 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.164 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.164 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728082+0000", 2024-08-17T17:19:26.164 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.35e-06 2024-08-17T17:19:26.164 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.164 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.164 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.164 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728106+0000", 2024-08-17T17:19:26.164 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.438e-05 2024-08-17T17:19:26.164 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.164 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.164 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.165 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728119+0000", 2024-08-17T17:19:26.165 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.256e-05 2024-08-17T17:19:26.165 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.165 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.165 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.165 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728126+0000", 2024-08-17T17:19:26.165 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5650000000000004e-06 2024-08-17T17:19:26.165 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.165 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.165 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.165 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728194+0000", 2024-08-17T17:19:26.165 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8263999999999996e-05 2024-08-17T17:19:26.165 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.166 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.166 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.166 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728342+0000", 2024-08-17T17:19:26.166 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014800699999999999 2024-08-17T17:19:26.166 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.166 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.166 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:19:26.166 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728473+0000", 2024-08-17T17:19:26.166 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013097799999999999 2024-08-17T17:19:26.166 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.166 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.166 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.166 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.761767+0000", 2024-08-17T17:19:26.167 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.033293973999999997 2024-08-17T17:19:26.167 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.167 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.167 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:26.167 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.762011+0000", 2024-08-17T17:19:26.167 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000243315 2024-08-17T17:19:26.167 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.167 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.167 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.167 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.803446+0000", 2024-08-17T17:19:26.167 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.04143579 2024-08-17T17:19:26.167 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.168 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.168 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.168 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.803504+0000", 2024-08-17T17:19:26.168 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7759e-05 2024-08-17T17:19:26.168 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.168 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.168 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.168 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.803553+0000", 2024-08-17T17:19:26.168 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8600999999999998e-05 2024-08-17T17:19:26.168 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.168 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.168 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.168 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.169 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.169 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:32643 6.4 6:28d4a114:::benchmark_data_smithi063_160268_object32642:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:19:26.169 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.131350+0000", 2024-08-17T17:19:26.169 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 347.02471539099997, 2024-08-17T17:19:26.169 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.086976158999999997, 2024-08-17T17:19:26.169 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.169 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.169 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.169 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:26.169 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:19:26.169 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:19:26.169 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 32643 2024-08-17T17:19:26.170 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.170 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.170 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.170 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.170 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131350+0000", 2024-08-17T17:19:26.170 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.170 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.170 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.170 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.170 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131350+0000", 2024-08-17T17:19:26.170 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.170 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.170 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.171 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.171 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131351+0000", 2024-08-17T17:19:26.171 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.195e-06 2024-08-17T17:19:26.171 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.171 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.171 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.171 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131377+0000", 2024-08-17T17:19:26.171 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5463e-05 2024-08-17T17:19:26.171 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.171 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.171 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.171 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131389+0000", 2024-08-17T17:19:26.172 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2147000000000001e-05 2024-08-17T17:19:26.172 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.172 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.172 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.172 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131396+0000", 2024-08-17T17:19:26.172 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1729999999999998e-06 2024-08-17T17:19:26.172 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.172 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.172 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.172 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131478+0000", 2024-08-17T17:19:26.172 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2399999999999997e-05 2024-08-17T17:19:26.172 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.172 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.172 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.173 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131625+0000", 2024-08-17T17:19:26.173 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014619000000000001 2024-08-17T17:19:26.173 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.173 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.173 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:19:26.173 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131770+0000", 2024-08-17T17:19:26.173 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014517800000000001 2024-08-17T17:19:26.173 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.173 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.173 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:26.173 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.133200+0000", 2024-08-17T17:19:26.173 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0014297069999999999 2024-08-17T17:19:26.173 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.174 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.174 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.174 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.135826+0000", 2024-08-17T17:19:26.174 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0026266179999999998 2024-08-17T17:19:26.174 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.174 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.174 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.174 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218155+0000", 2024-08-17T17:19:26.174 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.082328383000000005 2024-08-17T17:19:26.174 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.174 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.174 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.175 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218238+0000", 2024-08-17T17:19:26.175 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3920999999999998e-05 2024-08-17T17:19:26.175 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.175 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.175 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.175 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218326+0000", 2024-08-17T17:19:26.175 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7783999999999998e-05 2024-08-17T17:19:26.175 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.175 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.175 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.175 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.175 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.175 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:33167 6.4 6:22dca8bd:::benchmark_data_smithi063_160268_object33166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:19:26.175 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.448673+0000", 2024-08-17T17:19:26.176 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 346.70739279899999, 2024-08-17T17:19:26.176 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.094468580999999996, 2024-08-17T17:19:26.176 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.176 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.176 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.176 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:26.176 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:19:26.176 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:19:26.176 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 33167 2024-08-17T17:19:26.176 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.176 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.176 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.176 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.177 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448673+0000", 2024-08-17T17:19:26.177 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.177 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.177 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.177 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.177 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448673+0000", 2024-08-17T17:19:26.177 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.177 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.177 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.177 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.177 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448673+0000", 2024-08-17T17:19:26.177 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3000000000000001e-07 2024-08-17T17:19:26.177 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.178 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.178 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.179 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448693+0000", 2024-08-17T17:19:26.179 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9422000000000001e-05 2024-08-17T17:19:26.179 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.179 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.179 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.179 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448700+0000", 2024-08-17T17:19:26.180 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2779999999999996e-06 2024-08-17T17:19:26.180 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.180 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.180 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.180 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448706+0000", 2024-08-17T17:19:26.180 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6069999999999998e-06 2024-08-17T17:19:26.180 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.180 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.180 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.180 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.450351+0000", 2024-08-17T17:19:26.180 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001645526 2024-08-17T17:19:26.181 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.181 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.181 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.181 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.450469+0000", 2024-08-17T17:19:26.181 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000118239 2024-08-17T17:19:26.181 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.181 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.181 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:19:26.181 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.450598+0000", 2024-08-17T17:19:26.181 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012906300000000001 2024-08-17T17:19:26.182 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.182 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.182 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:26.182 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452120+0000", 2024-08-17T17:19:26.182 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0015215090000000001 2024-08-17T17:19:26.182 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.182 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.182 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.182 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452519+0000", 2024-08-17T17:19:26.182 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00039886099999999997 2024-08-17T17:19:26.182 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.182 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.183 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.183 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.543022+0000", 2024-08-17T17:19:26.183 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.090503086999999996 2024-08-17T17:19:26.183 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.183 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.183 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.183 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.543091+0000", 2024-08-17T17:19:26.183 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8806000000000005e-05 2024-08-17T17:19:26.183 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.183 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.183 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.183 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.543141+0000", 2024-08-17T17:19:26.183 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0652999999999998e-05 2024-08-17T17:19:26.183 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.184 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.184 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.184 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.184 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.184 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:33173 6.4 6:23580cad:::benchmark_data_smithi063_160268_object33172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:19:26.184 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.452560+0000", 2024-08-17T17:19:26.184 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 346.703505119, 2024-08-17T17:19:26.184 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.094171489999999997, 2024-08-17T17:19:26.184 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.185 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.185 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.185 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:26.185 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:19:26.185 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:19:26.185 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 33173 2024-08-17T17:19:26.185 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.185 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.185 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.186 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.186 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452560+0000", 2024-08-17T17:19:26.186 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.186 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.186 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.186 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.186 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452560+0000", 2024-08-17T17:19:26.186 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.186 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.186 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.186 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.186 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452562+0000", 2024-08-17T17:19:26.187 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5039999999999999e-06 2024-08-17T17:19:26.187 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.187 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.187 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.187 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452584+0000", 2024-08-17T17:19:26.187 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2578999999999998e-05 2024-08-17T17:19:26.187 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.187 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.187 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.187 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452595+0000", 2024-08-17T17:19:26.187 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0512e-05 2024-08-17T17:19:26.187 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.187 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.188 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.188 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452602+0000", 2024-08-17T17:19:26.188 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5710000000000003e-06 2024-08-17T17:19:26.188 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.188 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.188 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.188 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452692+0000", 2024-08-17T17:19:26.188 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0767999999999996e-05 2024-08-17T17:19:26.188 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.188 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.188 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.188 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452831+0000", 2024-08-17T17:19:26.189 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000138331 2024-08-17T17:19:26.189 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.189 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.189 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:19:26.189 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452965+0000", 2024-08-17T17:19:26.189 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013412600000000001 2024-08-17T17:19:26.189 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.189 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.189 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:26.189 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455446+0000", 2024-08-17T17:19:26.189 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0024813000000000001 2024-08-17T17:19:26.189 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.190 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.190 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.190 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539987+0000", 2024-08-17T17:19:26.190 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.084540709000000006 2024-08-17T17:19:26.190 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.190 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.190 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.190 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546616+0000", 2024-08-17T17:19:26.190 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0066297400000000003 2024-08-17T17:19:26.190 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.190 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.190 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.190 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546672+0000", 2024-08-17T17:19:26.191 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5600000000000003e-05 2024-08-17T17:19:26.191 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.191 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.191 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.191 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546732+0000", 2024-08-17T17:19:26.191 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9750000000000002e-05 2024-08-17T17:19:26.191 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.191 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.191 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.191 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.191 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.191 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:33174 6.4 6:2e7cd2fc:::benchmark_data_smithi063_160268_object33173:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:19:26.191 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.452620+0000", 2024-08-17T17:19:26.192 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 346.70344574199999, 2024-08-17T17:19:26.192 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.094311133000000005, 2024-08-17T17:19:26.192 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.192 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.192 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.192 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:26.192 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:19:26.192 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:19:26.192 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 33174 2024-08-17T17:19:26.192 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.192 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.192 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.192 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.192 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452620+0000", 2024-08-17T17:19:26.193 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.193 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.193 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.193 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.193 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452620+0000", 2024-08-17T17:19:26.193 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.193 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:26.194 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.194 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.195 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.195 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452620+0000", 2024-08-17T17:19:26.195 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8100000000000003e-07 2024-08-17T17:19:26.195 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.195 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.195 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.195 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452641+0000", 2024-08-17T17:19:26.195 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0205000000000001e-05 2024-08-17T17:19:26.196 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.196 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.196 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.196 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452648+0000", 2024-08-17T17:19:26.196 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0829999999999998e-06 2024-08-17T17:19:26.196 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.196 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.196 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.196 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452653+0000", 2024-08-17T17:19:26.196 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1420000000000001e-06 2024-08-17T17:19:26.196 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.196 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.197 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.197 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.454367+0000", 2024-08-17T17:19:26.197 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001713881 2024-08-17T17:19:26.197 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.197 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.197 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.197 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.454477+0000", 2024-08-17T17:19:26.197 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011024299999999999 2024-08-17T17:19:26.197 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.197 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.197 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:19:26.197 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.454554+0000", 2024-08-17T17:19:26.198 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7177999999999999e-05 2024-08-17T17:19:26.198 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.198 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.198 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:26.198 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455452+0000", 2024-08-17T17:19:26.198 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000898411 2024-08-17T17:19:26.198 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.198 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.198 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.198 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.540026+0000", 2024-08-17T17:19:26.199 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.084573537000000004 2024-08-17T17:19:26.199 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.199 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.199 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.199 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546819+0000", 2024-08-17T17:19:26.199 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.006792518 2024-08-17T17:19:26.199 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.199 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.199 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.200 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546874+0000", 2024-08-17T17:19:26.200 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5748000000000002e-05 2024-08-17T17:19:26.201 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.201 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.201 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.201 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546931+0000", 2024-08-17T17:19:26.201 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6606e-05 2024-08-17T17:19:26.201 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.201 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.201 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.201 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.201 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.201 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:33176 6.1 6:8a507c7a:::benchmark_data_smithi063_160268_object33175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:19:26.201 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.453381+0000", 2024-08-17T17:19:26.202 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 346.70268458599998, 2024-08-17T17:19:26.202 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.088080944999999994, 2024-08-17T17:19:26.202 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.202 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.202 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.202 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:26.202 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:19:26.202 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:19:26.202 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 33176 2024-08-17T17:19:26.202 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.202 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.202 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.202 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.203 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453381+0000", 2024-08-17T17:19:26.203 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.203 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.203 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.203 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.203 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453381+0000", 2024-08-17T17:19:26.203 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.203 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.203 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.203 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.203 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453382+0000", 2024-08-17T17:19:26.203 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.147e-06 2024-08-17T17:19:26.204 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.204 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.204 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.204 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453416+0000", 2024-08-17T17:19:26.204 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3822000000000002e-05 2024-08-17T17:19:26.204 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.204 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.204 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.204 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453427+0000", 2024-08-17T17:19:26.204 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1328000000000001e-05 2024-08-17T17:19:26.204 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.204 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.204 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.205 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453435+0000", 2024-08-17T17:19:26.205 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8869999999999992e-06 2024-08-17T17:19:26.205 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.205 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.205 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.205 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453492+0000", 2024-08-17T17:19:26.205 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7043000000000003e-05 2024-08-17T17:19:26.205 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.205 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.205 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.205 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453612+0000", 2024-08-17T17:19:26.205 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000119416 2024-08-17T17:19:26.205 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.206 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.206 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:19:26.206 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453744+0000", 2024-08-17T17:19:26.206 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000132211 2024-08-17T17:19:26.206 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.206 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.206 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:26.206 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455034+0000", 2024-08-17T17:19:26.206 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001289915 2024-08-17T17:19:26.206 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.206 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.206 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.207 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455499+0000", 2024-08-17T17:19:26.207 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00046549900000000002 2024-08-17T17:19:26.207 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.207 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.207 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.207 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541347+0000", 2024-08-17T17:19:26.207 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.085847657999999993 2024-08-17T17:19:26.207 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.207 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.207 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.207 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541412+0000", 2024-08-17T17:19:26.207 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5417000000000001e-05 2024-08-17T17:19:26.207 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.207 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.208 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.208 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541462+0000", 2024-08-17T17:19:26.208 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9601999999999997e-05 2024-08-17T17:19:26.208 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.208 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.208 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.208 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.208 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.208 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:33181 6.d e1093/1086 6:b650d8fa:::benchmark_data_smithi063_160268_object33180:head v 1093'2025, mlcod=1093'2023)", 2024-08-17T17:19:26.208 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.455644+0000", 2024-08-17T17:19:26.208 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 346.700421232, 2024-08-17T17:19:26.208 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.087465092999999994, 2024-08-17T17:19:26.208 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.209 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.209 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.209 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.209 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.209 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.209 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455644+0000", 2024-08-17T17:19:26.209 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.209 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.209 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.209 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.209 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455644+0000", 2024-08-17T17:19:26.209 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.210 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.210 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.210 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.210 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455645+0000", 2024-08-17T17:19:26.210 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9599999999999999e-07 2024-08-17T17:19:26.210 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.210 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.210 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.210 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455708+0000", 2024-08-17T17:19:26.210 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2724000000000005e-05 2024-08-17T17:19:26.210 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.210 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.210 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.211 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455715+0000", 2024-08-17T17:19:26.211 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8050000000000007e-06 2024-08-17T17:19:26.211 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.211 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.211 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.211 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455720+0000", 2024-08-17T17:19:26.211 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8559999999999998e-06 2024-08-17T17:19:26.211 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.211 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.211 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.211 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455766+0000", 2024-08-17T17:19:26.211 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6258999999999999e-05 2024-08-17T17:19:26.211 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.212 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.212 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.212 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455785+0000", 2024-08-17T17:19:26.212 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8329e-05 2024-08-17T17:19:26.212 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.212 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.212 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.212 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.543016+0000", 2024-08-17T17:19:26.212 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.087231031000000001 2024-08-17T17:19:26.212 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.212 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.212 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.213 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.543109+0000", 2024-08-17T17:19:26.213 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3492999999999994e-05 2024-08-17T17:19:26.213 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.213 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.213 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.213 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.213 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.213 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.44609.0:33176 6.1 e1093/1086 ondisk, result = 0)", 2024-08-17T17:19:26.213 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.456246+0000", 2024-08-17T17:19:26.213 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 346.69981984100002, 2024-08-17T17:19:26.213 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.085228506999999995, 2024-08-17T17:19:26.213 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.213 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.214 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:19:26.214 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.214 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.214 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.214 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456246+0000", 2024-08-17T17:19:26.214 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.214 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.214 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.214 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.214 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456246+0000", 2024-08-17T17:19:26.214 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.214 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.214 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.215 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.215 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456246+0000", 2024-08-17T17:19:26.215 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9800000000000003e-07 2024-08-17T17:19:26.215 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.215 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.215 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.215 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456250+0000", 2024-08-17T17:19:26.215 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8519999999999997e-06 2024-08-17T17:19:26.215 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.215 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.215 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.215 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541269+0000", 2024-08-17T17:19:26.215 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.085018446999999997 2024-08-17T17:19:26.216 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.216 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.216 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.216 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541279+0000", 2024-08-17T17:19:26.216 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0559e-05 2024-08-17T17:19:26.216 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.216 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.216 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.216 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541333+0000", 2024-08-17T17:19:26.216 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3792999999999998e-05 2024-08-17T17:19:26.216 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.216 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.216 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.217 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541341+0000", 2024-08-17T17:19:26.217 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2709999999999992e-06 2024-08-17T17:19:26.217 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.217 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.217 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.217 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541474+0000", 2024-08-17T17:19:26.217 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013288699999999999 2024-08-17T17:19:26.217 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.217 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.217 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.217 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.217 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.217 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.44609.0:33173 6.4 e1093/1086 ondisk, result = 0)", 2024-08-17T17:19:26.217 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.456941+0000", 2024-08-17T17:19:26.218 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 346.69912440100001, 2024-08-17T17:19:26.218 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.083053689999999999, 2024-08-17T17:19:26.218 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.218 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.218 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:19:26.218 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.218 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.218 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.218 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456941+0000", 2024-08-17T17:19:26.218 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.218 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.218 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.219 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.219 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456941+0000", 2024-08-17T17:19:26.219 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.219 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.219 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.219 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.219 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456942+0000", 2024-08-17T17:19:26.219 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2099999999999997e-07 2024-08-17T17:19:26.219 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.219 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.219 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.219 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456945+0000", 2024-08-17T17:19:26.219 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0970000000000002e-06 2024-08-17T17:19:26.219 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.220 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.220 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.220 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539891+0000", 2024-08-17T17:19:26.220 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.082946110000000003 2024-08-17T17:19:26.220 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.220 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.220 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.220 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539902+0000", 2024-08-17T17:19:26.220 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1094000000000001e-05 2024-08-17T17:19:26.220 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.220 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.220 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.220 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539971+0000", 2024-08-17T17:19:26.221 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9148999999999998e-05 2024-08-17T17:19:26.221 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.221 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.221 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.221 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539980+0000", 2024-08-17T17:19:26.221 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0359999999999995e-06 2024-08-17T17:19:26.221 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.221 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.221 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.221 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539995+0000", 2024-08-17T17:19:26.221 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4683e-05 2024-08-17T17:19:26.221 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.221 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.222 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.222 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.222 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.222 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34219 6.4 6:220ade53:::benchmark_data_smithi063_160268_object34218:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:19:26.222 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.044051+0000", 2024-08-17T17:19:26.222 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 346.11201407800002, 2024-08-17T17:19:26.222 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.091717032000000004, 2024-08-17T17:19:26.222 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.222 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.222 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.222 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:26.222 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:19:26.222 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:19:26.223 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34219 2024-08-17T17:19:26.223 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.223 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.223 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.223 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.223 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044051+0000", 2024-08-17T17:19:26.223 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.223 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.223 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.223 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.223 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044051+0000", 2024-08-17T17:19:26.223 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.223 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.223 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.224 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.224 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044052+0000", 2024-08-17T17:19:26.224 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4900000000000005e-07 2024-08-17T17:19:26.224 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.224 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.224 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.224 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044074+0000", 2024-08-17T17:19:26.224 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1696999999999999e-05 2024-08-17T17:19:26.224 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.224 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.224 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.224 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044084+0000", 2024-08-17T17:19:26.224 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0482000000000001e-05 2024-08-17T17:19:26.225 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.225 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.225 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.225 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044091+0000", 2024-08-17T17:19:26.225 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9600000000000003e-06 2024-08-17T17:19:26.225 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.225 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.225 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.225 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044144+0000", 2024-08-17T17:19:26.225 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3177000000000003e-05 2024-08-17T17:19:26.225 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.225 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.225 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.226 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044238+0000", 2024-08-17T17:19:26.226 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3733000000000005e-05 2024-08-17T17:19:26.226 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.226 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.226 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:19:26.226 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044316+0000", 2024-08-17T17:19:26.227 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7700999999999993e-05 2024-08-17T17:19:26.227 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.227 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.227 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.227 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.046094+0000", 2024-08-17T17:19:26.227 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0017784140000000001 2024-08-17T17:19:26.228 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.228 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.228 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:26.228 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.046102+0000", 2024-08-17T17:19:26.228 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8790000000000003e-06 2024-08-17T17:19:26.228 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.228 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.228 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.228 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.135653+0000", 2024-08-17T17:19:26.228 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.089550600999999994 2024-08-17T17:19:26.228 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.228 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.228 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.229 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.135720+0000", 2024-08-17T17:19:26.229 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7187999999999994e-05 2024-08-17T17:19:26.229 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.229 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.229 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.229 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.135768+0000", 2024-08-17T17:19:26.229 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8451000000000001e-05 2024-08-17T17:19:26.229 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.229 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.229 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.229 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.229 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.229 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35822 6.4 6:2f33fed9:::benchmark_data_smithi063_160268_object35821:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:19:26.230 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.883928+0000", 2024-08-17T17:19:26.230 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 345.27213793499999, 2024-08-17T17:19:26.230 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14375861300000001, 2024-08-17T17:19:26.230 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.230 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.230 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.230 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:26.230 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:19:26.230 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:19:26.230 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35822 2024-08-17T17:19:26.230 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.230 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.231 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.231 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.231 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883928+0000", 2024-08-17T17:19:26.231 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.231 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.231 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.231 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.231 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883928+0000", 2024-08-17T17:19:26.231 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.231 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.231 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.231 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.231 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883929+0000", 2024-08-17T17:19:26.232 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3459999999999999e-06 2024-08-17T17:19:26.232 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.232 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.232 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.232 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883947+0000", 2024-08-17T17:19:26.232 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8067000000000002e-05 2024-08-17T17:19:26.232 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.232 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.232 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.232 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883957+0000", 2024-08-17T17:19:26.232 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0179e-05 2024-08-17T17:19:26.232 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.233 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.233 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.233 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883963+0000", 2024-08-17T17:19:26.233 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.57e-06 2024-08-17T17:19:26.233 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.233 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.233 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.233 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.884008+0000", 2024-08-17T17:19:26.233 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5207000000000003e-05 2024-08-17T17:19:26.233 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.233 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.233 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.233 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.884091+0000", 2024-08-17T17:19:26.234 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3162000000000001e-05 2024-08-17T17:19:26.234 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.234 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.234 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:19:26.234 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.884169+0000", 2024-08-17T17:19:26.234 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8159999999999997e-05 2024-08-17T17:19:26.234 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.234 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.234 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:26.234 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.885227+0000", 2024-08-17T17:19:26.234 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0010574099999999999 2024-08-17T17:19:26.234 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.234 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.235 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.235 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.885625+0000", 2024-08-17T17:19:26.235 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00039848699999999999 2024-08-17T17:19:26.235 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.235 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.235 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.235 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027581+0000", 2024-08-17T17:19:26.235 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.141956267 2024-08-17T17:19:26.235 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.235 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.235 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.235 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027642+0000", 2024-08-17T17:19:26.235 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0757000000000001e-05 2024-08-17T17:19:26.236 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.236 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.236 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.236 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027686+0000", 2024-08-17T17:19:26.236 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4001000000000001e-05 2024-08-17T17:19:26.236 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.236 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.236 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.236 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.236 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.236 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:124810 6.4 6:2af1567b:::benchmark_data_smithi063_160268_object124809:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-08-17T17:19:26.236 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:14:55.542514+0000", 2024-08-17T17:19:26.236 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 270.61355157600002, 2024-08-17T17:19:26.237 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.094516375999999999, 2024-08-17T17:19:26.237 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.237 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.237 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.237 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:26.237 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:19:26.237 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:19:26.237 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 124810 2024-08-17T17:19:26.237 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.237 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.237 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.237 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.237 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.542514+0000", 2024-08-17T17:19:26.237 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.238 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.238 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.238 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.238 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.542514+0000", 2024-08-17T17:19:26.238 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.238 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.238 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.238 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.238 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.542515+0000", 2024-08-17T17:19:26.238 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3680000000000001e-06 2024-08-17T17:19:26.238 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.238 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.238 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.239 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.542536+0000", 2024-08-17T17:19:26.239 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0990000000000001e-05 2024-08-17T17:19:26.239 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.239 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.239 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.239 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.542554+0000", 2024-08-17T17:19:26.239 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7271000000000001e-05 2024-08-17T17:19:26.239 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.239 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.239 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.239 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.542561+0000", 2024-08-17T17:19:26.239 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0180000000000004e-06 2024-08-17T17:19:26.239 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.240 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.240 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.240 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.551700+0000", 2024-08-17T17:19:26.240 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0091395299999999999 2024-08-17T17:19:26.240 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.240 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.240 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.240 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.552029+0000", 2024-08-17T17:19:26.240 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00032915800000000002 2024-08-17T17:19:26.240 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.240 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.240 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:19:26.240 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.552331+0000", 2024-08-17T17:19:26.241 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00030189300000000002 2024-08-17T17:19:26.241 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.241 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.241 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.241 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.555864+0000", 2024-08-17T17:19:26.241 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0035326200000000002 2024-08-17T17:19:26.241 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.241 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.241 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:26.241 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.556270+0000", 2024-08-17T17:19:26.241 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00040584399999999999 2024-08-17T17:19:26.241 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.241 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.242 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.242 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.636877+0000", 2024-08-17T17:19:26.242 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.080607636999999996 2024-08-17T17:19:26.242 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.242 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.242 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.242 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.636986+0000", 2024-08-17T17:19:26.242 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010857 2024-08-17T17:19:26.242 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.242 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.242 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.243 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.637030+0000", 2024-08-17T17:19:26.243 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4477000000000001e-05 2024-08-17T17:19:26.243 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.243 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.243 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.243 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.243 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.243 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:124848 6.4 6:2a0bafff:::benchmark_data_smithi063_160268_object124847:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-08-17T17:19:26.243 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:14:55.583601+0000", 2024-08-17T17:19:26.243 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 270.57246459200002, 2024-08-17T17:19:26.243 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.073727231000000004, 2024-08-17T17:19:26.243 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.243 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.244 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.244 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:26.244 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:19:26.244 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:19:26.244 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 124848 2024-08-17T17:19:26.244 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.244 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.244 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.244 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.244 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583601+0000", 2024-08-17T17:19:26.244 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.244 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.244 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.245 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.245 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583601+0000", 2024-08-17T17:19:26.245 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.245 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.245 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.245 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.245 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583602+0000", 2024-08-17T17:19:26.245 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0470000000000001e-06 2024-08-17T17:19:26.245 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.245 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.245 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.245 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583622+0000", 2024-08-17T17:19:26.245 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0463e-05 2024-08-17T17:19:26.246 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.246 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.246 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.246 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583631+0000", 2024-08-17T17:19:26.246 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0289999999999994e-06 2024-08-17T17:19:26.246 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.246 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.246 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.246 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583637+0000", 2024-08-17T17:19:26.246 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7710000000000003e-06 2024-08-17T17:19:26.246 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.246 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.246 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.247 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583696+0000", 2024-08-17T17:19:26.247 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8801000000000002e-05 2024-08-17T17:19:26.247 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.247 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.247 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.247 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583875+0000", 2024-08-17T17:19:26.247 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000178596 2024-08-17T17:19:26.247 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.247 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.247 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:19:26.247 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.584044+0000", 2024-08-17T17:19:26.247 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016988699999999999 2024-08-17T17:19:26.247 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.248 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.248 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:26.248 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.588047+0000", 2024-08-17T17:19:26.248 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0040021049999999997 2024-08-17T17:19:26.248 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.248 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.248 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.248 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.592375+0000", 2024-08-17T17:19:26.248 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0043281129999999998 2024-08-17T17:19:26.248 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.248 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.248 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.248 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.656456+0000", 2024-08-17T17:19:26.249 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.064081263999999999 2024-08-17T17:19:26.249 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.249 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.249 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.249 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.657279+0000", 2024-08-17T17:19:26.249 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00082331500000000003 2024-08-17T17:19:26.249 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.249 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.249 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.249 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.657328+0000", 2024-08-17T17:19:26.249 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.884e-05 2024-08-17T17:19:26.249 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.250 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.250 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.250 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.250 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.250 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:146039 6.4 6:270a08b7:::benchmark_data_smithi063_160268_object19932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-08-17T17:19:26.250 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:15:20.434951+0000", 2024-08-17T17:19:26.250 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 245.72111487500001, 2024-08-17T17:19:26.250 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.083836758999999997, 2024-08-17T17:19:26.250 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.250 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.250 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.250 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:26.251 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:19:26.251 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:19:26.251 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 146039 2024-08-17T17:19:26.251 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.251 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.251 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.251 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.251 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.434951+0000", 2024-08-17T17:19:26.251 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.251 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.251 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.251 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.251 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.434951+0000", 2024-08-17T17:19:26.252 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.252 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.252 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.252 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.252 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.434951+0000", 2024-08-17T17:19:26.252 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7000000000000003e-07 2024-08-17T17:19:26.252 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.252 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.252 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.252 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.434955+0000", 2024-08-17T17:19:26.252 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5109999999999999e-06 2024-08-17T17:19:26.252 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.252 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.253 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.253 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.434963+0000", 2024-08-17T17:19:26.253 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2549999999999997e-06 2024-08-17T17:19:26.253 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.253 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.253 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.253 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.434968+0000", 2024-08-17T17:19:26.253 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1309999999999997e-06 2024-08-17T17:19:26.253 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.253 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.253 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.253 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.435011+0000", 2024-08-17T17:19:26.254 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3130999999999998e-05 2024-08-17T17:19:26.254 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.254 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.254 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.254 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.435085+0000", 2024-08-17T17:19:26.254 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3689000000000005e-05 2024-08-17T17:19:26.254 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.254 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.254 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:19:26.254 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.435154+0000", 2024-08-17T17:19:26.254 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9207000000000003e-05 2024-08-17T17:19:26.254 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.254 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.254 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:26.255 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.438866+0000", 2024-08-17T17:19:26.255 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0037120999999999999 2024-08-17T17:19:26.255 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.255 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.255 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.255 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.439494+0000", 2024-08-17T17:19:26.255 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00062716699999999998 2024-08-17T17:19:26.255 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.255 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.255 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:26.255 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.518685+0000", 2024-08-17T17:19:26.255 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.079191119000000004 2024-08-17T17:19:26.256 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.256 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.256 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.256 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.518744+0000", 2024-08-17T17:19:26.256 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8971999999999997e-05 2024-08-17T17:19:26.256 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.256 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.256 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.256 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.518787+0000", 2024-08-17T17:19:26.256 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3606999999999998e-05 2024-08-17T17:19:26.256 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.256 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.256 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.257 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.257 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.257 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:34039 7.9 e1133/1129 7:9f55a860:::benchmark_data_smithi063_196002_object34038:head v 1133'2100, mlcod=1133'2095)", 2024-08-17T17:19:26.257 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:11.452652+0000", 2024-08-17T17:19:26.257 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 74.703413671999996, 2024-08-17T17:19:26.257 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17720034600000001, 2024-08-17T17:19:26.257 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.257 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.257 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.257 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.257 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.257 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.258 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.452652+0000", 2024-08-17T17:19:26.258 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.258 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.258 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.258 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.258 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.452652+0000", 2024-08-17T17:19:26.258 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.258 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.258 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.258 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.258 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.452652+0000", 2024-08-17T17:19:26.259 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5600000000000001e-07 2024-08-17T17:19:26.259 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.259 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.259 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.259 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.452692+0000", 2024-08-17T17:19:26.259 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.0157999999999997e-05 2024-08-17T17:19:26.259 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.259 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.259 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.259 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.452698+0000", 2024-08-17T17:19:26.259 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.066e-06 2024-08-17T17:19:26.259 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.260 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.260 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.260 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.452702+0000", 2024-08-17T17:19:26.260 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7110000000000001e-06 2024-08-17T17:19:26.260 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.260 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.260 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.260 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.453996+0000", 2024-08-17T17:19:26.260 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0012937179999999999 2024-08-17T17:19:26.260 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.260 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.260 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.260 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.454013+0000", 2024-08-17T17:19:26.261 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7680999999999999e-05 2024-08-17T17:19:26.261 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.261 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.261 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.261 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.455127+0000", 2024-08-17T17:19:26.261 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0011138630000000001 2024-08-17T17:19:26.261 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.261 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.261 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.261 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629852+0000", 2024-08-17T17:19:26.261 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17472479299999999 2024-08-17T17:19:26.261 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.262 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.262 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.262 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.262 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.262 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:34051 7.5 e1133/1129 7:a40e9396:::benchmark_data_smithi063_196002_object34050:head v 1133'2196, mlcod=1133'2194)", 2024-08-17T17:19:26.262 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:11.456432+0000", 2024-08-17T17:19:26.262 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 74.699633059000007, 2024-08-17T17:19:26.262 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17630179500000001, 2024-08-17T17:19:26.262 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.262 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.262 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.262 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.263 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.263 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.263 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456432+0000", 2024-08-17T17:19:26.263 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.263 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.263 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.263 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.263 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456432+0000", 2024-08-17T17:19:26.263 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.263 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.263 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.263 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.263 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456434+0000", 2024-08-17T17:19:26.264 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2300000000000001e-06 2024-08-17T17:19:26.264 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.264 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.264 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.264 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456485+0000", 2024-08-17T17:19:26.264 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1539000000000002e-05 2024-08-17T17:19:26.264 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.264 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.264 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.264 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629879+0000", 2024-08-17T17:19:26.264 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.173393513 2024-08-17T17:19:26.264 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.264 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.264 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.265 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629886+0000", 2024-08-17T17:19:26.265 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9929999999999998e-06 2024-08-17T17:19:26.265 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.265 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.265 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.265 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629949+0000", 2024-08-17T17:19:26.265 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3737000000000003e-05 2024-08-17T17:19:26.265 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.265 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.265 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.265 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629973+0000", 2024-08-17T17:19:26.265 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3646e-05 2024-08-17T17:19:26.266 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.266 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.266 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.266 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.632699+0000", 2024-08-17T17:19:26.266 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0027261220000000001 2024-08-17T17:19:26.266 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.266 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.266 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.266 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.632734+0000", 2024-08-17T17:19:26.266 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5015000000000003e-05 2024-08-17T17:19:26.266 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.266 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.266 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.267 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.267 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.267 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:34053 7.0 e1133/1129 7:0b9bda54:::benchmark_data_smithi063_196002_object34052:head v 1133'2211, mlcod=1133'2209)", 2024-08-17T17:19:26.267 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:11.456547+0000", 2024-08-17T17:19:26.267 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 74.699518443000002, 2024-08-17T17:19:26.267 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.174798225, 2024-08-17T17:19:26.267 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.267 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.267 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.267 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.267 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.267 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.267 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456547+0000", 2024-08-17T17:19:26.268 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.268 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.268 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.268 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.268 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456547+0000", 2024-08-17T17:19:26.268 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.268 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.268 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.268 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.268 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456548+0000", 2024-08-17T17:19:26.268 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.68e-07 2024-08-17T17:19:26.268 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.269 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.269 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.269 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456667+0000", 2024-08-17T17:19:26.269 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011935200000000001 2024-08-17T17:19:26.269 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.269 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.269 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.269 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629846+0000", 2024-08-17T17:19:26.269 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17317944800000001 2024-08-17T17:19:26.269 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.270 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.270 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.270 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629865+0000", 2024-08-17T17:19:26.270 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8340000000000001e-05 2024-08-17T17:19:26.270 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.270 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.270 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.270 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629941+0000", 2024-08-17T17:19:26.270 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6447999999999997e-05 2024-08-17T17:19:26.270 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.270 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.271 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.271 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629973+0000", 2024-08-17T17:19:26.271 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1538000000000002e-05 2024-08-17T17:19:26.271 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.271 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.271 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.271 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.631305+0000", 2024-08-17T17:19:26.271 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0013325819999999999 2024-08-17T17:19:26.271 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.271 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.271 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.271 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.631345+0000", 2024-08-17T17:19:26.271 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9848999999999997e-05 2024-08-17T17:19:26.271 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.272 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.272 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.272 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.272 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.272 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:34055 7.2 e1133/1129 7:4c133eb8:::benchmark_data_smithi063_196002_object34054:head v 1133'2149, mlcod=1133'2147)", 2024-08-17T17:19:26.272 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:11.456861+0000", 2024-08-17T17:19:26.272 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 74.699204167999994, 2024-08-17T17:19:26.272 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17531513200000001, 2024-08-17T17:19:26.273 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.273 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.273 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.273 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.273 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.273 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.274 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456861+0000", 2024-08-17T17:19:26.274 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.274 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.274 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.274 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.274 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456861+0000", 2024-08-17T17:19:26.274 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.274 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.274 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.274 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.274 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456862+0000", 2024-08-17T17:19:26.274 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0100000000000005e-07 2024-08-17T17:19:26.275 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.275 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.275 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.275 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456886+0000", 2024-08-17T17:19:26.275 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4442000000000001e-05 2024-08-17T17:19:26.275 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.275 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.275 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.275 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629875+0000", 2024-08-17T17:19:26.275 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17298824099999999 2024-08-17T17:19:26.275 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.275 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.275 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.275 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629886+0000", 2024-08-17T17:19:26.276 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1065e-05 2024-08-17T17:19:26.276 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.276 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.276 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.276 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.630078+0000", 2024-08-17T17:19:26.276 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00019224199999999999 2024-08-17T17:19:26.276 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.276 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.276 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.276 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.630100+0000", 2024-08-17T17:19:26.276 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1676e-05 2024-08-17T17:19:26.276 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.277 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.277 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.277 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.632137+0000", 2024-08-17T17:19:26.277 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0020373840000000002 2024-08-17T17:19:26.277 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.277 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.277 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.277 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.632176+0000", 2024-08-17T17:19:26.277 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9480999999999997e-05 2024-08-17T17:19:26.277 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.277 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.277 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.277 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.277 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.278 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88374 7.d e1150/1144 7:b7ece4c7:::benchmark_data_smithi063_196002_object88373:head v 1150'5490, mlcod=0'0)", 2024-08-17T17:19:26.278 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.649423+0000", 2024-08-17T17:19:26.278 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 25.506642030999998, 2024-08-17T17:19:26.278 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.28106146399999998, 2024-08-17T17:19:26.278 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.278 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.278 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:26.278 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.278 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.278 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.278 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649423+0000", 2024-08-17T17:19:26.278 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.278 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.279 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.279 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.279 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649423+0000", 2024-08-17T17:19:26.279 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.279 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.279 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.279 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.279 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649425+0000", 2024-08-17T17:19:26.279 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.186e-06 2024-08-17T17:19:26.279 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.279 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.279 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.279 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649464+0000", 2024-08-17T17:19:26.280 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9591000000000001e-05 2024-08-17T17:19:26.280 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.280 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.280 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.280 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649477+0000", 2024-08-17T17:19:26.280 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.293e-05 2024-08-17T17:19:26.280 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.280 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.280 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.280 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649486+0000", 2024-08-17T17:19:26.280 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4249999999999998e-06 2024-08-17T17:19:26.280 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.280 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.280 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.281 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649719+0000", 2024-08-17T17:19:26.281 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00023391199999999999 2024-08-17T17:19:26.281 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.281 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.281 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.281 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649782+0000", 2024-08-17T17:19:26.281 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2582000000000005e-05 2024-08-17T17:19:26.281 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.281 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.281 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:26.281 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.930315+0000", 2024-08-17T17:19:26.281 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.28053328 2024-08-17T17:19:26.281 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.281 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.282 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.282 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.930485+0000", 2024-08-17T17:19:26.282 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016955799999999999 2024-08-17T17:19:26.282 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.282 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.282 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.282 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.282 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.282 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88375 7.2 e1150/1129 7:44faffa9:::benchmark_data_smithi063_196002_object88374:head v 1150'5570, mlcod=1150'5567)", 2024-08-17T17:19:26.282 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.652123+0000", 2024-08-17T17:19:26.282 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 25.503942185, 2024-08-17T17:19:26.282 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.29175277300000002, 2024-08-17T17:19:26.282 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:26.283 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:26.283 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:19:26.283 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:26.283 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.283 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:26.283 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652123+0000", 2024-08-17T17:19:26.283 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.283 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.283 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.283 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:26.283 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652123+0000", 2024-08-17T17:19:26.283 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:26.283 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.283 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.283 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:26.284 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652124+0000", 2024-08-17T17:19:26.284 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3099999999999999e-07 2024-08-17T17:19:26.284 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.284 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.284 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:26.284 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652147+0000", 2024-08-17T17:19:26.284 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3261999999999999e-05 2024-08-17T17:19:26.284 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.284 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.284 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:26.284 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652166+0000", 2024-08-17T17:19:26.284 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9517e-05 2024-08-17T17:19:26.284 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.284 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.285 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:26.285 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652175+0000", 2024-08-17T17:19:26.285 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6100000000000006e-06 2024-08-17T17:19:26.285 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.285 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.285 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:26.285 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652554+0000", 2024-08-17T17:19:26.285 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00037904699999999998 2024-08-17T17:19:26.285 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.285 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.285 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:26.285 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652575+0000", 2024-08-17T17:19:26.285 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.069e-05 2024-08-17T17:19:26.285 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:26.286 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:26.286 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:26.286 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.943876+0000", 2024-08-17T17:19:26.286 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.29130141599999998 2024-08-17T17:19:26.286 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.286 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.286 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.286 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:26.286 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:26.286 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:19:26.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:26.287 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:26.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:26.388 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:26.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:26.439 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:26.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:26.451 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 88 15 110438 110423 78.4104 47.5625 0.0121779 0.0127467 2024-08-17T17:19:26.540 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:26.564 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:26.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:26.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:26.666 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:26.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:26.786 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:26.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:26.809 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:19:26.811 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:19:26.887 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:26.926 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:26.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:26.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:27.027 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:27.064 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:27.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:27.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:27.166 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:27.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:27.212 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:27.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:27.313 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:27.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:27.353 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:27.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:27.451 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 89 16 111219 111203 78.0771 48.75 0.0436927 0.0128012 2024-08-17T17:19:27.453 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:27.483 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:27.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:27.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:27.585 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:27.611 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:27.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:27.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:27.713 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:27.717 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:27.764 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:19:27.774 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:27.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:27.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:27.877 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:27.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:27.942 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:27.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:27 smithi063 ceph-mon[109843]: pgmap v1380: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 15 MiB/s wr, 236 op/s; 27952/321708 objects degraded (8.689%); 3623/321708 objects misplaced (1.126%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:19:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:27 smithi063 ceph-mon[109843]: from='client.35067 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:19:28.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:27 smithi063 ceph-mon[114756]: pgmap v1380: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 15 MiB/s wr, 236 op/s; 27952/321708 objects degraded (8.689%); 3623/321708 objects misplaced (1.126%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:19:28.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:27 smithi063 ceph-mon[114756]: from='client.35067 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:19:28.043 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:27 smithi162 ceph-mon[104668]: pgmap v1380: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 15 MiB/s wr, 236 op/s; 27952/321708 objects degraded (8.689%); 3623/321708 objects misplaced (1.126%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:19:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:27 smithi162 ceph-mon[104668]: from='client.35067 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:19:28.105 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:28.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:28.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:28.200 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1380,"stamp":"2024-08-17T17:19:26.176464+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7026966985,"num_objects":107236,"num_object_clones":0,"num_object_copies":321708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27952,"num_objects_misplaced":3623,"num_objects_unfound":0,"num_objects_dirty":107236,"num_whiteouts":0,"num_read":210,"num_read_kb":453,"num_write":100743,"num_write_kb":6437256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1397,"num_bytes_recovered":91055533,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161873,"ondisk_log_size":161873,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":341,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22703192,"kb_used_data":20618008,"kb_used_omap":199,"kb_used_meta":2084920,"kb_avail":727258024,"statfs":{"total":767960285184,"available":744712216576,"internally_reserved":0,"allocated":21112840192,"data_stored":21095715727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":2134958844},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":590872576,"num_objects":9016,"num_object_clones":0,"num_object_copies":27048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-215,"num_objects_unfound":0,"num_objects_dirty":9016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2368,"num_write_kb":151552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":215,"num_bytes_recovered":14090240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004518"},"pg_stats":[{"pgid":"7.17","version":"1154'5911","reported_seq":5960,"reported_epoch":1154,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:22.029939+0000","last_change":"2024-08-17T17:19:01.756567+0000","last_active":"2024-08-17T17:19:22.029939+0000","last_peered":"2024-08-17T17:19:22.029939+0000","last_clean":"2024-08-17T17:18:30.516571+0000","last_became_active":"2024-08-17T17:19:01.750697+0000","last_became_peered":"2024-08-17T17:19:01.750697+0000","last_unstale":"2024-08-17T17:19:22.029939+0000","last_undegraded":"2024-08-17T17:19:01.715352+0000","last_fullsized":"2024-08-17T17:19:22.029939+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":5911,"log_dups_size":0,"ondisk_log_size":5911,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211746816,"num_objects":3231,"num_object_clones":0,"num_object_copies":9693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1981,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3231,"num_write_kb":206784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"0,3,6","objects":1250},{"shards":"1,3,6","objects":1981}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1388,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.750611+0000","last_change":"2024-08-17T17:07:45.684223+0000","last_active":"2024-08-17T17:19:18.750611+0000","last_peered":"2024-08-17T17:19:18.750611+0000","last_clean":"2024-08-17T17:19:18.750611+0000","last_became_active":"2024-08-17T17:07:45.683948+0000","last_became_peered":"2024-08-17T17:07:45.683948+0000","last_unstale":"2024-08-17T17:19:18.750611+0000","last_undegraded":"2024-08-17T17:19:18.750611+0000","last_fullsized":"2024-08-17T17:19:18.750611+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:07.677423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1081,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.753106+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:19:18.753106+0000","last_peered":"2024-08-17T17:19:18.753106+0000","last_clean":"2024-08-17T17:19:18.753106+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:19:18.753106+0000","last_undegraded":"2024-08-17T17:19:18.753106+0000","last_fullsized":"2024-08-17T17:19:18.753106+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.16","version":"1154'6126","reported_seq":6174,"reported_epoch":1154,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:22.167531+0000","last_change":"2024-08-17T17:19:01.787015+0000","last_active":"2024-08-17T17:19:22.167531+0000","last_peered":"2024-08-17T17:19:22.167531+0000","last_clean":"2024-08-17T17:18:30.519687+0000","last_became_active":"2024-08-17T17:19:01.765205+0000","last_became_peered":"2024-08-17T17:19:01.765205+0000","last_unstale":"2024-08-17T17:19:22.167531+0000","last_undegraded":"2024-08-17T17:19:01.759821+0000","last_fullsized":"2024-08-17T17:19:22.167531+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6126,"log_dups_size":0,"ondisk_log_size":6126,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220200960,"num_objects":3360,"num_object_clones":0,"num_object_copies":10080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2111,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3360,"num_write_kb":215040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"0,3,7","objects":1249},{"shards":"3,5,7","objects":2111}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1092,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.659913+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:19:02.659913+0000","last_peered":"2024-08-17T17:19:02.659913+0000","last_clean":"2024-08-17T17:19:02.659913+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:19:02.659913+0000","last_undegraded":"2024-08-17T17:19:02.659913+0000","last_fullsized":"2024-08-17T17:19:02.659913+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":947,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.649006+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:19:02.649006+0000","last_peered":"2024-08-17T17:19:02.649006+0000","last_clean":"2024-08-17T17:19:02.649006+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:19:02.649006+0000","last_undegraded":"2024-08-17T17:19:02.649006+0000","last_fullsized":"2024-08-17T17:19:02.649006+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.15","version":"1154'6219","reported_seq":7898,"reported_epoch":1154,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:22.151534+0000","last_change":"2024-08-17T17:19:01.776021+0000","last_active":"2024-08-17T17:19:22.151534+0000","last_peered":"2024-08-17T17:19:22.151534+0000","last_clean":"2024-08-17T17:18:30.511218+0000","last_became_active":"2024-08-17T17:19:01.771169+0000","last_became_peered":"2024-08-17T17:19:01.771169+0000","last_unstale":"2024-08-17T17:19:22.151534+0000","last_undegraded":"2024-08-17T17:19:01.734234+0000","last_fullsized":"2024-08-17T17:19:22.151534+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6219,"log_dups_size":0,"ondisk_log_size":6219,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222855168,"num_objects":3401,"num_object_clones":0,"num_object_copies":10203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1719,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3401,"num_write_kb":217632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":407,"num_bytes_recovered":26673152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"0,2,7","objects":1682},{"shards":"1,2,7","objects":1719}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":953,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.750588+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:19:18.750588+0000","last_peered":"2024-08-17T17:19:18.750588+0000","last_clean":"2024-08-17T17:19:18.750588+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:19:18.750588+0000","last_undegraded":"2024-08-17T17:19:18.750588+0000","last_fullsized":"2024-08-17T17:19:18.750588+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1064,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:17.218096+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:19:17.218096+0000","last_peered":"2024-08-17T17:19:17.218096+0000","last_clean":"2024-08-17T17:19:17.218096+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:19:17.218096+0000","last_undegraded":"2024-08-17T17:19:17.218096+0000","last_fullsized":"2024-08-17T17:19:17.218096+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.14","version":"1154'6141","reported_seq":6191,"reported_epoch":1154,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:22.140844+0000","last_change":"2024-08-17T17:19:01.769109+0000","last_active":"2024-08-17T17:19:22.140844+0000","last_peered":"2024-08-17T17:19:22.140844+0000","last_clean":"2024-08-17T17:18:30.520652+0000","last_became_active":"2024-08-17T17:19:01.765052+0000","last_became_peered":"2024-08-17T17:19:01.765052+0000","last_unstale":"2024-08-17T17:19:22.140844+0000","last_undegraded":"2024-08-17T17:19:01.729866+0000","last_fullsized":"2024-08-17T17:19:22.140844+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6141,"log_dups_size":0,"ondisk_log_size":6141,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219906048,"num_objects":3356,"num_object_clones":0,"num_object_copies":10068,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2111,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3356,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3356,"num_write_kb":214752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"0,1,4","objects":1245},{"shards":"1,4,6","objects":2111}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1387,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.759603+0000","last_change":"2024-08-17T17:07:45.687030+0000","last_active":"2024-08-17T17:19:18.759603+0000","last_peered":"2024-08-17T17:19:18.759603+0000","last_clean":"2024-08-17T17:19:18.759603+0000","last_became_active":"2024-08-17T17:07:45.686610+0000","last_became_peered":"2024-08-17T17:07:45.686610+0000","last_unstale":"2024-08-17T17:19:18.759603+0000","last_undegraded":"2024-08-17T17:19:18.759603+0000","last_fullsized":"2024-08-17T17:19:18.759603+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:02.425845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1141,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:12.113695+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:19:12.113695+0000","last_peered":"2024-08-17T17:19:12.113695+0000","last_clean":"2024-08-17T17:19:12.113695+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:19:12.113695+0000","last_undegraded":"2024-08-17T17:19:12.113695+0000","last_fullsized":"2024-08-17T17:19:12.113695+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1154'6704","reported_seq":6738,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:22.173780+0000","last_change":"2024-08-17T17:17:52.927899+0000","last_active":"2024-08-17T17:19:22.173780+0000","last_peered":"2024-08-17T17:19:22.173780+0000","last_clean":"2024-08-17T17:19:22.173780+0000","last_became_active":"2024-08-17T17:17:52.927591+0000","last_became_peered":"2024-08-17T17:17:52.927591+0000","last_unstale":"2024-08-17T17:19:22.173780+0000","last_undegraded":"2024-08-17T17:19:22.173780+0000","last_fullsized":"2024-08-17T17:19:22.173780+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6704,"log_dups_size":0,"ondisk_log_size":6704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:31:21.681712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439353344,"num_objects":6704,"num_object_clones":0,"num_object_copies":20112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6704,"num_write_kb":429056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1106,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.650468+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:19:02.650468+0000","last_peered":"2024-08-17T17:19:02.650468+0000","last_clean":"2024-08-17T17:19:02.650468+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:19:02.650468+0000","last_undegraded":"2024-08-17T17:19:02.650468+0000","last_fullsized":"2024-08-17T17:19:02.650468+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1078,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.659288+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:19:02.659288+0000","last_peered":"2024-08-17T17:19:02.659288+0000","last_clean":"2024-08-17T17:19:02.659288+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:19:02.659288+0000","last_undegraded":"2024-08-17T17:19:02.659288+0000","last_fullsized":"2024-08-17T17:19:02.659288+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1154'6768","reported_seq":6831,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:24.664667+0000","last_change":"2024-08-17T17:17:52.311116+0000","last_active":"2024-08-17T17:19:24.664667+0000","last_peered":"2024-08-17T17:19:24.664667+0000","last_clean":"2024-08-17T17:19:24.664667+0000","last_became_active":"2024-08-17T17:17:52.310809+0000","last_became_peered":"2024-08-17T17:17:52.310809+0000","last_unstale":"2024-08-17T17:19:24.664667+0000","last_undegraded":"2024-08-17T17:19:24.664667+0000","last_fullsized":"2024-08-17T17:19:24.664667+0000","mapping_epoch":1129,"log_start":"1133'1100","ondisk_log_start":"1133'1100","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":5668,"log_dups_size":0,"ondisk_log_size":5668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:16:12.113051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":443547648,"num_objects":6768,"num_object_clones":0,"num_object_copies":20304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6768,"num_write_kb":433152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1081,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.655157+0000","last_change":"2024-08-17T17:06:39.638433+0000","last_active":"2024-08-17T17:19:02.655157+0000","last_peered":"2024-08-17T17:19:02.655157+0000","last_clean":"2024-08-17T17:19:02.655157+0000","last_became_active":"2024-08-17T17:06:39.638081+0000","last_became_peered":"2024-08-17T17:06:39.638081+0000","last_unstale":"2024-08-17T17:19:02.655157+0000","last_undegraded":"2024-08-17T17:19:02.655157+0000","last_fullsized":"2024-08-17T17:19:02.655157+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:13.558012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1112,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.664343+0000","last_change":"2024-08-17T17:07:45.693574+0000","last_active":"2024-08-17T17:19:02.664343+0000","last_peered":"2024-08-17T17:19:02.664343+0000","last_clean":"2024-08-17T17:19:02.664343+0000","last_became_active":"2024-08-17T17:07:45.693086+0000","last_became_peered":"2024-08-17T17:07:45.693086+0000","last_unstale":"2024-08-17T17:19:02.664343+0000","last_undegraded":"2024-08-17T17:19:02.664343+0000","last_fullsized":"2024-08-17T17:19:02.664343+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:31.888894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1154'6091","reported_seq":6145,"reported_epoch":1154,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:22.054342+0000","last_change":"2024-08-17T17:19:02.700822+0000","last_active":"2024-08-17T17:19:22.054342+0000","last_peered":"2024-08-17T17:19:22.054342+0000","last_clean":"2024-08-17T17:18:30.513360+0000","last_became_active":"2024-08-17T17:19:02.673093+0000","last_became_peered":"2024-08-17T17:19:02.673093+0000","last_unstale":"2024-08-17T17:19:22.054342+0000","last_undegraded":"2024-08-17T17:19:02.662660+0000","last_fullsized":"2024-08-17T17:19:22.054342+0000","mapping_epoch":1152,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6091,"log_dups_size":0,"ondisk_log_size":6091,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218234880,"num_objects":3330,"num_object_clones":0,"num_object_copies":9990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2026,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3330,"num_write_kb":213120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"0,1,4","objects":1304},{"shards":"1,4,7","objects":2026}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1065,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.656270+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:19:02.656270+0000","last_peered":"2024-08-17T17:19:02.656270+0000","last_clean":"2024-08-17T17:19:02.656270+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:19:02.656270+0000","last_undegraded":"2024-08-17T17:19:02.656270+0000","last_fullsized":"2024-08-17T17:19:02.656270+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1094,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.649978+0000","last_change":"2024-08-17T17:06:39.629368+0000","last_active":"2024-08-17T17:19:02.649978+0000","last_peered":"2024-08-17T17:19:02.649978+0000","last_clean":"2024-08-17T17:19:02.649978+0000","last_became_active":"2024-08-17T17:06:39.628832+0000","last_became_peered":"2024-08-17T17:06:39.628832+0000","last_unstale":"2024-08-17T17:19:02.649978+0000","last_undegraded":"2024-08-17T17:19:02.649978+0000","last_fullsized":"2024-08-17T17:19:02.649978+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:54.063884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1154'6034","reported_seq":6108,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:21.248197+0000","last_change":"2024-08-17T17:19:18.430412+0000","last_active":"2024-08-17T17:19:21.248197+0000","last_peered":"2024-08-17T17:19:21.248197+0000","last_clean":"2024-08-17T17:19:21.248197+0000","last_became_active":"2024-08-17T17:19:01.657134+0000","last_became_peered":"2024-08-17T17:19:01.657134+0000","last_unstale":"2024-08-17T17:19:21.248197+0000","last_undegraded":"2024-08-17T17:19:21.248197+0000","last_fullsized":"2024-08-17T17:19:21.248197+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6034,"log_dups_size":0,"ondisk_log_size":6034,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:19:23.371885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214630400,"num_objects":3275,"num_object_clones":0,"num_object_copies":9825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3275,"num_write_kb":209600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1094,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.659804+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:19:02.659804+0000","last_peered":"2024-08-17T17:19:02.659804+0000","last_clean":"2024-08-17T17:19:02.659804+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:19:02.659804+0000","last_undegraded":"2024-08-17T17:19:02.659804+0000","last_fullsized":"2024-08-17T17:19:02.659804+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":900,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.753381+0000","last_change":"2024-08-17T17:18:31.539133+0000","last_active":"2024-08-17T17:19:18.753381+0000","last_peered":"2024-08-17T17:19:18.753381+0000","last_clean":"2024-08-17T17:19:18.753381+0000","last_became_active":"2024-08-17T17:18:31.538669+0000","last_became_peered":"2024-08-17T17:18:31.538669+0000","last_unstale":"2024-08-17T17:19:18.753381+0000","last_undegraded":"2024-08-17T17:19:18.753381+0000","last_fullsized":"2024-08-17T17:19:18.753381+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:34.785446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1461,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.649382+0000","last_change":"2024-08-17T17:18:28.522790+0000","last_active":"2024-08-17T17:19:02.649382+0000","last_peered":"2024-08-17T17:19:02.649382+0000","last_clean":"2024-08-17T17:19:02.649382+0000","last_became_active":"2024-08-17T17:18:28.521573+0000","last_became_peered":"2024-08-17T17:18:28.521573+0000","last_unstale":"2024-08-17T17:19:02.649382+0000","last_undegraded":"2024-08-17T17:19:02.649382+0000","last_fullsized":"2024-08-17T17:19:02.649382+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":76,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":214679,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.f","version":"1154'6849","reported_seq":6913,"reported_epoch":1154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:19:24.652728+0000","last_change":"2024-08-17T17:18:33.026499+0000","last_active":"2024-08-17T17:19:24.652728+0000","last_peered":"2024-08-17T17:19:24.652728+0000","last_clean":"2024-08-17T17:18:30.502708+0000","last_became_active":"2024-08-17T17:18:33.018495+0000","last_became_peered":"2024-08-17T17:18:33.018495+0000","last_unstale":"2024-08-17T17:19:24.652728+0000","last_undegraded":"2024-08-17T17:18:32.555520+0000","last_fullsized":"2024-08-17T17:18:32.533633+0000","mapping_epoch":1144,"log_start":"1133'1100","ondisk_log_start":"1133'1100","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":5749,"log_dups_size":0,"ondisk_log_size":5749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":448856064,"num_objects":6849,"num_object_clones":0,"num_object_copies":20547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4154,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6849,"num_write_kb":438336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":4154},{"shards":"3,7","objects":2695}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1423,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.748269+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:19:18.748269+0000","last_peered":"2024-08-17T17:19:18.748269+0000","last_clean":"2024-08-17T17:19:18.748269+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:19:18.748269+0000","last_undegraded":"2024-08-17T17:19:18.748269+0000","last_fullsized":"2024-08-17T17:19:18.748269+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1097,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.660166+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:19:02.660166+0000","last_peered":"2024-08-17T17:19:02.660166+0000","last_clean":"2024-08-17T17:19:02.660166+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:19:02.660166+0000","last_undegraded":"2024-08-17T17:19:02.660166+0000","last_fullsized":"2024-08-17T17:19:02.660166+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1419,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.659166+0000","last_change":"2024-08-17T17:18:32.884334+0000","last_active":"2024-08-17T17:19:02.659166+0000","last_peered":"2024-08-17T17:19:02.659166+0000","last_clean":"2024-08-17T17:19:02.659166+0000","last_became_active":"2024-08-17T17:18:28.520186+0000","last_became_peered":"2024-08-17T17:18:28.520186+0000","last_unstale":"2024-08-17T17:19:02.659166+0000","last_undegraded":"2024-08-17T17:19:02.659166+0000","last_fullsized":"2024-08-17T17:19:02.659166+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:32.884246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:32.884246+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:49.286656+0000","scrub_duration":0.001842677,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.e","version":"1154'6747","reported_seq":7927,"reported_epoch":1154,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:19:21.280328+0000","last_change":"2024-08-17T17:19:00.937351+0000","last_active":"2024-08-17T17:19:21.280328+0000","last_peered":"2024-08-17T17:19:21.280328+0000","last_clean":"2024-08-17T17:18:30.515740+0000","last_became_active":"2024-08-17T17:18:33.016776+0000","last_became_peered":"2024-08-17T17:18:33.016776+0000","last_unstale":"2024-08-17T17:19:21.280328+0000","last_undegraded":"2024-08-17T17:19:21.280328+0000","last_fullsized":"2024-08-17T17:18:32.535040+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6747,"log_dups_size":0,"ondisk_log_size":6747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":442171392,"num_objects":6747,"num_object_clones":0,"num_object_copies":20241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3623,"num_objects_unfound":0,"num_objects_dirty":6747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6747,"num_write_kb":431808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":558,"num_bytes_recovered":36569088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":3623},{"shards":"2,5","objects":3124}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1095,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.650159+0000","last_change":"2024-08-17T17:06:39.629619+0000","last_active":"2024-08-17T17:19:02.650159+0000","last_peered":"2024-08-17T17:19:02.650159+0000","last_clean":"2024-08-17T17:19:02.650159+0000","last_became_active":"2024-08-17T17:06:39.629295+0000","last_became_peered":"2024-08-17T17:06:39.629295+0000","last_unstale":"2024-08-17T17:19:02.650159+0000","last_undegraded":"2024-08-17T17:19:02.650159+0000","last_fullsized":"2024-08-17T17:19:02.650159+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:15.874423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1112,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.651657+0000","last_change":"2024-08-17T17:07:12.649178+0000","last_active":"2024-08-17T17:19:02.651657+0000","last_peered":"2024-08-17T17:19:02.651657+0000","last_clean":"2024-08-17T17:19:02.651657+0000","last_became_active":"2024-08-17T17:07:12.648844+0000","last_became_peered":"2024-08-17T17:07:12.648844+0000","last_unstale":"2024-08-17T17:19:02.651657+0000","last_undegraded":"2024-08-17T17:19:02.651657+0000","last_fullsized":"2024-08-17T17:19:02.651657+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:12:06.307736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1154'6636","reported_seq":6678,"reported_epoch":1154,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:19:22.015997+0000","last_change":"2024-08-17T17:18:33.033028+0000","last_active":"2024-08-17T17:19:22.015997+0000","last_peered":"2024-08-17T17:19:22.015997+0000","last_clean":"2024-08-17T17:18:30.508341+0000","last_became_active":"2024-08-17T17:18:33.019697+0000","last_became_peered":"2024-08-17T17:18:33.019697+0000","last_unstale":"2024-08-17T17:19:22.015997+0000","last_undegraded":"2024-08-17T17:18:32.562771+0000","last_fullsized":"2024-08-17T17:18:32.533408+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6636,"log_dups_size":0,"ondisk_log_size":6636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:58:13.699201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":434896896,"num_objects":6636,"num_object_clones":0,"num_object_copies":19908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4070,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6636,"num_write_kb":424704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":4070},{"shards":"1,6","objects":2566}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1098,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.664413+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:19:02.664413+0000","last_peered":"2024-08-17T17:19:02.664413+0000","last_clean":"2024-08-17T17:19:02.664413+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:19:02.664413+0000","last_undegraded":"2024-08-17T17:19:02.664413+0000","last_fullsized":"2024-08-17T17:19:02.664413+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1097,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.652121+0000","last_change":"2024-08-17T17:06:39.627678+0000","last_active":"2024-08-17T17:19:02.652121+0000","last_peered":"2024-08-17T17:19:02.652121+0000","last_clean":"2024-08-17T17:19:02.652121+0000","last_became_active":"2024-08-17T17:06:39.627226+0000","last_became_peered":"2024-08-17T17:06:39.627226+0000","last_unstale":"2024-08-17T17:19:02.652121+0000","last_undegraded":"2024-08-17T17:19:02.652121+0000","last_fullsized":"2024-08-17T17:19:02.652121+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:01.868244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1419,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.659325+0000","last_change":"2024-08-17T17:18:31.850829+0000","last_active":"2024-08-17T17:19:02.659325+0000","last_peered":"2024-08-17T17:19:02.659325+0000","last_clean":"2024-08-17T17:19:02.659325+0000","last_became_active":"2024-08-17T17:18:28.520946+0000","last_became_peered":"2024-08-17T17:18:28.520946+0000","last_unstale":"2024-08-17T17:19:02.659325+0000","last_undegraded":"2024-08-17T17:19:02.659325+0000","last_fullsized":"2024-08-17T17:19:02.659325+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:31.850745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:31.850745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:17.218084+0000","scrub_duration":0.001317447,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.c","version":"1154'6598","reported_seq":6632,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:22.076709+0000","last_change":"2024-08-17T17:17:52.309565+0000","last_active":"2024-08-17T17:19:22.076709+0000","last_peered":"2024-08-17T17:19:22.076709+0000","last_clean":"2024-08-17T17:19:22.076709+0000","last_became_active":"2024-08-17T17:17:52.309298+0000","last_became_peered":"2024-08-17T17:17:52.309298+0000","last_unstale":"2024-08-17T17:19:22.076709+0000","last_undegraded":"2024-08-17T17:19:22.076709+0000","last_fullsized":"2024-08-17T17:19:22.076709+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6598,"log_dups_size":0,"ondisk_log_size":6598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:27:31.251993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432406528,"num_objects":6598,"num_object_clones":0,"num_object_copies":19794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6598,"num_write_kb":422272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1390,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.748073+0000","last_change":"2024-08-17T17:07:45.680458+0000","last_active":"2024-08-17T17:19:18.748073+0000","last_peered":"2024-08-17T17:19:18.748073+0000","last_clean":"2024-08-17T17:19:18.748073+0000","last_became_active":"2024-08-17T17:07:45.680221+0000","last_became_peered":"2024-08-17T17:07:45.680221+0000","last_unstale":"2024-08-17T17:19:18.748073+0000","last_undegraded":"2024-08-17T17:19:18.748073+0000","last_fullsized":"2024-08-17T17:19:18.748073+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:55.083962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1412,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.755262+0000","last_change":"2024-08-17T17:07:45.682866+0000","last_active":"2024-08-17T17:19:18.755262+0000","last_peered":"2024-08-17T17:19:18.755262+0000","last_clean":"2024-08-17T17:19:18.755262+0000","last_became_active":"2024-08-17T17:07:45.682554+0000","last_became_peered":"2024-08-17T17:07:45.682554+0000","last_unstale":"2024-08-17T17:19:18.755262+0000","last_undegraded":"2024-08-17T17:19:18.755262+0000","last_fullsized":"2024-08-17T17:19:18.755262+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:25.975914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.6","version":"1154'6148","reported_seq":6197,"reported_epoch":1154,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:22.093888+0000","last_change":"2024-08-17T17:19:02.694376+0000","last_active":"2024-08-17T17:19:22.093888+0000","last_peered":"2024-08-17T17:19:22.093888+0000","last_clean":"2024-08-17T17:18:30.519687+0000","last_became_active":"2024-08-17T17:19:02.689996+0000","last_became_peered":"2024-08-17T17:19:02.689996+0000","last_unstale":"2024-08-17T17:19:22.093888+0000","last_undegraded":"2024-08-17T17:19:02.656489+0000","last_fullsized":"2024-08-17T17:19:22.093888+0000","mapping_epoch":1152,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6148,"log_dups_size":0,"ondisk_log_size":6148,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221642752,"num_objects":3382,"num_object_clones":0,"num_object_copies":10146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2011,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3382,"num_write_kb":216448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"0,3,7","objects":1371},{"shards":"3,5,7","objects":2011}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1473,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.649230+0000","last_change":"2024-08-17T17:18:33.814375+0000","last_active":"2024-08-17T17:19:02.649230+0000","last_peered":"2024-08-17T17:19:02.649230+0000","last_clean":"2024-08-17T17:19:02.649230+0000","last_became_active":"2024-08-17T17:18:33.813208+0000","last_became_peered":"2024-08-17T17:18:33.813208+0000","last_unstale":"2024-08-17T17:19:02.649230+0000","last_undegraded":"2024-08-17T17:19:02.649230+0000","last_fullsized":"2024-08-17T17:19:02.649230+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:25.057675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:25.057675+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:20:45.235879+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1108,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.659690+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:19:02.659690+0000","last_peered":"2024-08-17T17:19:02.659690+0000","last_clean":"2024-08-17T17:19:02.659690+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:19:02.659690+0000","last_undegraded":"2024-08-17T17:19:02.659690+0000","last_fullsized":"2024-08-17T17:19:02.659690+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1369,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.755925+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:19:18.755925+0000","last_peered":"2024-08-17T17:19:18.755925+0000","last_clean":"2024-08-17T17:19:18.755925+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:19:18.755925+0000","last_undegraded":"2024-08-17T17:19:18.755925+0000","last_fullsized":"2024-08-17T17:19:18.755925+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1080,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.659735+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:19:02.659735+0000","last_peered":"2024-08-17T17:19:02.659735+0000","last_clean":"2024-08-17T17:19:02.659735+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:19:02.659735+0000","last_undegraded":"2024-08-17T17:19:02.659735+0000","last_fullsized":"2024-08-17T17:19:02.659735+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":666,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.649966+0000","last_change":"2024-08-17T17:18:28.524571+0000","last_active":"2024-08-17T17:19:02.649966+0000","last_peered":"2024-08-17T17:19:02.649966+0000","last_clean":"2024-08-17T17:19:02.649966+0000","last_became_active":"2024-08-17T17:18:28.524280+0000","last_became_peered":"2024-08-17T17:18:28.524280+0000","last_unstale":"2024-08-17T17:19:02.649966+0000","last_undegraded":"2024-08-17T17:19:02.649966+0000","last_fullsized":"2024-08-17T17:19:02.649966+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1154'6207","reported_seq":7887,"reported_epoch":1154,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:22.130361+0000","last_change":"2024-08-17T17:19:02.692013+0000","last_active":"2024-08-17T17:19:22.130361+0000","last_peered":"2024-08-17T17:19:22.130361+0000","last_clean":"2024-08-17T17:18:30.511218+0000","last_became_active":"2024-08-17T17:19:02.665577+0000","last_became_peered":"2024-08-17T17:19:02.665577+0000","last_unstale":"2024-08-17T17:19:22.130361+0000","last_undegraded":"2024-08-17T17:19:02.654944+0000","last_fullsized":"2024-08-17T17:19:22.130361+0000","mapping_epoch":1152,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6207,"log_dups_size":0,"ondisk_log_size":6207,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222068736,"num_objects":3388,"num_object_clones":0,"num_object_copies":10164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1651,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3388,"num_write_kb":216864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":407,"num_bytes_recovered":26673152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"0,2,7","objects":1737},{"shards":"1,2,7","objects":1651}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1126,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.649091+0000","last_change":"2024-08-17T17:18:31.539042+0000","last_active":"2024-08-17T17:19:02.649091+0000","last_peered":"2024-08-17T17:19:02.649091+0000","last_clean":"2024-08-17T17:19:02.649091+0000","last_became_active":"2024-08-17T17:18:31.537879+0000","last_became_peered":"2024-08-17T17:18:31.537879+0000","last_unstale":"2024-08-17T17:19:02.649091+0000","last_undegraded":"2024-08-17T17:19:02.649091+0000","last_fullsized":"2024-08-17T17:19:02.649091+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:36:27.859975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":948,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:17.170221+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:19:17.170221+0000","last_peered":"2024-08-17T17:19:17.170221+0000","last_clean":"2024-08-17T17:19:17.170221+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:19:17.170221+0000","last_undegraded":"2024-08-17T17:19:17.170221+0000","last_fullsized":"2024-08-17T17:19:17.170221+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":853,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.662361+0000","last_change":"2024-08-17T17:18:31.536646+0000","last_active":"2024-08-17T17:19:02.662361+0000","last_peered":"2024-08-17T17:19:02.662361+0000","last_clean":"2024-08-17T17:19:02.662361+0000","last_became_active":"2024-08-17T17:18:31.536326+0000","last_became_peered":"2024-08-17T17:18:31.536326+0000","last_unstale":"2024-08-17T17:19:02.662361+0000","last_undegraded":"2024-08-17T17:19:02.662361+0000","last_fullsized":"2024-08-17T17:19:02.662361+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:11:03.952344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1478,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.649007+0000","last_change":"2024-08-17T17:18:33.817028+0000","last_active":"2024-08-17T17:19:02.649007+0000","last_peered":"2024-08-17T17:19:02.649007+0000","last_clean":"2024-08-17T17:19:02.649007+0000","last_became_active":"2024-08-17T17:18:33.816660+0000","last_became_peered":"2024-08-17T17:18:33.816660+0000","last_unstale":"2024-08-17T17:19:02.649007+0000","last_undegraded":"2024-08-17T17:19:02.649007+0000","last_fullsized":"2024-08-17T17:19:02.649007+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":214680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.4","version":"1154'6163","reported_seq":6214,"reported_epoch":1154,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:22.076251+0000","last_change":"2024-08-17T17:19:02.697004+0000","last_active":"2024-08-17T17:19:22.076251+0000","last_peered":"2024-08-17T17:19:22.076251+0000","last_clean":"2024-08-17T17:18:30.520652+0000","last_became_active":"2024-08-17T17:19:02.667978+0000","last_became_peered":"2024-08-17T17:19:02.667978+0000","last_unstale":"2024-08-17T17:19:22.076251+0000","last_undegraded":"2024-08-17T17:19:02.661396+0000","last_fullsized":"2024-08-17T17:19:22.076251+0000","mapping_epoch":1152,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6163,"log_dups_size":0,"ondisk_log_size":6163,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221347840,"num_objects":3377,"num_object_clones":0,"num_object_copies":10131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2067,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3377,"num_write_kb":216160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"0,1,4","objects":1310},{"shards":"1,4,6","objects":2067}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1466,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.753034+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:19:18.753034+0000","last_peered":"2024-08-17T17:19:18.753034+0000","last_clean":"2024-08-17T17:19:18.753034+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:19:18.753034+0000","last_undegraded":"2024-08-17T17:19:18.753034+0000","last_fullsized":"2024-08-17T17:19:18.753034+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":844,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.754578+0000","last_change":"2024-08-17T17:15:38.610415+0000","last_active":"2024-08-17T17:19:18.754578+0000","last_peered":"2024-08-17T17:19:18.754578+0000","last_clean":"2024-08-17T17:19:18.754578+0000","last_became_active":"2024-08-17T17:15:38.610083+0000","last_became_peered":"2024-08-17T17:15:38.610083+0000","last_unstale":"2024-08-17T17:19:18.754578+0000","last_undegraded":"2024-08-17T17:19:18.754578+0000","last_fullsized":"2024-08-17T17:19:18.754578+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:12.007494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":773,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.660031+0000","last_change":"2024-08-17T17:18:31.534212+0000","last_active":"2024-08-17T17:19:02.660031+0000","last_peered":"2024-08-17T17:19:02.660031+0000","last_clean":"2024-08-17T17:19:02.660031+0000","last_became_active":"2024-08-17T17:18:31.533792+0000","last_became_peered":"2024-08-17T17:18:31.533792+0000","last_unstale":"2024-08-17T17:19:02.660031+0000","last_undegraded":"2024-08-17T17:19:02.660031+0000","last_fullsized":"2024-08-17T17:19:02.660031+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:13:01.220425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1436,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.659176+0000","last_change":"2024-08-17T17:18:35.916482+0000","last_active":"2024-08-17T17:19:02.659176+0000","last_peered":"2024-08-17T17:19:02.659176+0000","last_clean":"2024-08-17T17:19:02.659176+0000","last_became_active":"2024-08-17T17:18:34.827747+0000","last_became_peered":"2024-08-17T17:18:34.827747+0000","last_unstale":"2024-08-17T17:19:02.659176+0000","last_undegraded":"2024-08-17T17:19:02.659176+0000","last_fullsized":"2024-08-17T17:19:02.659176+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:35.916381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:35.916381+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:28.492622+0000","scrub_duration":0.005340521,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"1154'5930","reported_seq":5980,"reported_epoch":1154,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:22.044387+0000","last_change":"2024-08-17T17:19:02.695693+0000","last_active":"2024-08-17T17:19:22.044387+0000","last_peered":"2024-08-17T17:19:22.044387+0000","last_clean":"2024-08-17T17:18:30.516571+0000","last_became_active":"2024-08-17T17:19:02.666519+0000","last_became_peered":"2024-08-17T17:19:02.666519+0000","last_unstale":"2024-08-17T17:19:22.044387+0000","last_undegraded":"2024-08-17T17:19:02.656623+0000","last_fullsized":"2024-08-17T17:19:22.044387+0000","mapping_epoch":1152,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":5930,"log_dups_size":0,"ondisk_log_size":5930,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212992000,"num_objects":3250,"num_object_clones":0,"num_object_copies":9750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1962,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3250,"num_write_kb":208000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"0,3,6","objects":1288},{"shards":"1,3,6","objects":1962}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1109,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.659605+0000","last_change":"2024-08-17T17:06:39.628745+0000","last_active":"2024-08-17T17:19:02.659605+0000","last_peered":"2024-08-17T17:19:02.659605+0000","last_clean":"2024-08-17T17:19:02.659605+0000","last_became_active":"2024-08-17T17:06:39.628144+0000","last_became_peered":"2024-08-17T17:06:39.628144+0000","last_unstale":"2024-08-17T17:19:02.659605+0000","last_undegraded":"2024-08-17T17:19:02.659605+0000","last_fullsized":"2024-08-17T17:19:02.659605+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:14.740936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1117,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.653325+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:19:02.653325+0000","last_peered":"2024-08-17T17:19:02.653325+0000","last_clean":"2024-08-17T17:19:02.653325+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:19:02.653325+0000","last_undegraded":"2024-08-17T17:19:02.653325+0000","last_fullsized":"2024-08-17T17:19:02.653325+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":920,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.753301+0000","last_change":"2024-08-17T17:15:38.608764+0000","last_active":"2024-08-17T17:19:18.753301+0000","last_peered":"2024-08-17T17:19:18.753301+0000","last_clean":"2024-08-17T17:19:18.753301+0000","last_became_active":"2024-08-17T17:15:38.608353+0000","last_became_peered":"2024-08-17T17:15:38.608353+0000","last_unstale":"2024-08-17T17:19:18.753301+0000","last_undegraded":"2024-08-17T17:19:18.753301+0000","last_fullsized":"2024-08-17T17:19:18.753301+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:42.873441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":666,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.650982+0000","last_change":"2024-08-17T17:18:28.527058+0000","last_active":"2024-08-17T17:19:02.650982+0000","last_peered":"2024-08-17T17:19:02.650982+0000","last_clean":"2024-08-17T17:19:02.650982+0000","last_became_active":"2024-08-17T17:18:28.526439+0000","last_became_peered":"2024-08-17T17:18:28.526439+0000","last_unstale":"2024-08-17T17:19:02.650982+0000","last_undegraded":"2024-08-17T17:19:02.650982+0000","last_fullsized":"2024-08-17T17:19:02.650982+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.1","version":"1154'5989","reported_seq":6033,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:22.027998+0000","last_change":"2024-08-17T17:19:01.686339+0000","last_active":"2024-08-17T17:19:22.027998+0000","last_peered":"2024-08-17T17:19:22.027998+0000","last_clean":"2024-08-17T17:19:22.027998+0000","last_became_active":"2024-08-17T17:19:01.682436+0000","last_became_peered":"2024-08-17T17:19:01.682436+0000","last_unstale":"2024-08-17T17:19:22.027998+0000","last_undegraded":"2024-08-17T17:19:22.027998+0000","last_fullsized":"2024-08-17T17:19:22.027998+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":5989,"log_dups_size":0,"ondisk_log_size":5989,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214892544,"num_objects":3279,"num_object_clones":0,"num_object_copies":9837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3279,"num_write_kb":209856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1110,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.659989+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:19:02.659989+0000","last_peered":"2024-08-17T17:19:02.659989+0000","last_clean":"2024-08-17T17:19:02.659989+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:19:02.659989+0000","last_undegraded":"2024-08-17T17:19:02.659989+0000","last_fullsized":"2024-08-17T17:19:02.659989+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":782,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.649374+0000","last_change":"2024-08-17T17:18:31.537256+0000","last_active":"2024-08-17T17:19:02.649374+0000","last_peered":"2024-08-17T17:19:02.649374+0000","last_clean":"2024-08-17T17:19:02.649374+0000","last_became_active":"2024-08-17T17:18:31.536194+0000","last_became_peered":"2024-08-17T17:18:31.536194+0000","last_unstale":"2024-08-17T17:19:02.649374+0000","last_undegraded":"2024-08-17T17:19:02.649374+0000","last_fullsized":"2024-08-17T17:19:02.649374+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:46.453717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":724,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.659425+0000","last_change":"2024-08-17T17:15:38.609480+0000","last_active":"2024-08-17T17:19:02.659425+0000","last_peered":"2024-08-17T17:19:02.659425+0000","last_clean":"2024-08-17T17:19:02.659425+0000","last_became_active":"2024-08-17T17:15:38.609120+0000","last_became_peered":"2024-08-17T17:15:38.609120+0000","last_unstale":"2024-08-17T17:19:02.659425+0000","last_undegraded":"2024-08-17T17:19:02.659425+0000","last_fullsized":"2024-08-17T17:19:02.659425+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:11:03.113425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":683,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.650650+0000","last_change":"2024-08-17T17:18:34.831943+0000","last_active":"2024-08-17T17:19:02.650650+0000","last_peered":"2024-08-17T17:19:02.650650+0000","last_clean":"2024-08-17T17:19:02.650650+0000","last_became_active":"2024-08-17T17:18:34.831498+0000","last_became_peered":"2024-08-17T17:18:34.831498+0000","last_unstale":"2024-08-17T17:19:02.650650+0000","last_undegraded":"2024-08-17T17:19:02.650650+0000","last_fullsized":"2024-08-17T17:19:02.650650+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.0","version":"1154'6180","reported_seq":6224,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:22.201219+0000","last_change":"2024-08-17T17:19:01.664407+0000","last_active":"2024-08-17T17:19:22.201219+0000","last_peered":"2024-08-17T17:19:22.201219+0000","last_clean":"2024-08-17T17:19:22.201219+0000","last_became_active":"2024-08-17T17:19:01.659646+0000","last_became_peered":"2024-08-17T17:19:01.659646+0000","last_unstale":"2024-08-17T17:19:22.201219+0000","last_undegraded":"2024-08-17T17:19:22.201219+0000","last_fullsized":"2024-08-17T17:19:22.201219+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6180,"log_dups_size":0,"ondisk_log_size":6180,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221741056,"num_objects":3383,"num_object_clones":0,"num_object_copies":10149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3383,"num_write_kb":216544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":843,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.753380+0000","last_change":"2024-08-17T17:15:38.609837+0000","last_active":"2024-08-17T17:19:18.753380+0000","last_peered":"2024-08-17T17:19:18.753380+0000","last_clean":"2024-08-17T17:19:18.753380+0000","last_became_active":"2024-08-17T17:15:38.609474+0000","last_became_peered":"2024-08-17T17:15:38.609474+0000","last_unstale":"2024-08-17T17:19:18.753380+0000","last_undegraded":"2024-08-17T17:19:18.753380+0000","last_fullsized":"2024-08-17T17:19:18.753380+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:31:42.450582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1110,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.656349+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:19:02.656349+0000","last_peered":"2024-08-17T17:19:02.656349+0000","last_clean":"2024-08-17T17:19:02.656349+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:19:02.656349+0000","last_undegraded":"2024-08-17T17:19:02.656349+0000","last_fullsized":"2024-08-17T17:19:02.656349+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":703,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:17.215921+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:19:17.215921+0000","last_peered":"2024-08-17T17:19:17.215921+0000","last_clean":"2024-08-17T17:19:17.215921+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:19:17.215921+0000","last_undegraded":"2024-08-17T17:19:17.215921+0000","last_fullsized":"2024-08-17T17:19:17.215921+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1407,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.652074+0000","last_change":"2024-08-17T17:18:28.526894+0000","last_active":"2024-08-17T17:19:02.652074+0000","last_peered":"2024-08-17T17:19:02.652074+0000","last_clean":"2024-08-17T17:19:02.652074+0000","last_became_active":"2024-08-17T17:18:28.526484+0000","last_became_peered":"2024-08-17T17:18:28.526484+0000","last_unstale":"2024-08-17T17:19:02.652074+0000","last_undegraded":"2024-08-17T17:19:02.652074+0000","last_fullsized":"2024-08-17T17:19:02.652074+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.3","version":"1154'6094","reported_seq":6169,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:24.659220+0000","last_change":"2024-08-17T17:19:23.570508+0000","last_active":"2024-08-17T17:19:24.659220+0000","last_peered":"2024-08-17T17:19:24.659220+0000","last_clean":"2024-08-17T17:19:24.659220+0000","last_became_active":"2024-08-17T17:19:01.681349+0000","last_became_peered":"2024-08-17T17:19:01.681349+0000","last_unstale":"2024-08-17T17:19:24.659220+0000","last_undegraded":"2024-08-17T17:19:24.659220+0000","last_fullsized":"2024-08-17T17:19:24.659220+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6094,"log_dups_size":0,"ondisk_log_size":6094,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:19:28.549966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221020160,"num_objects":3372,"num_object_clones":0,"num_object_copies":10116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3372,"num_write_kb":215840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1423,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.759882+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:19:18.759882+0000","last_peered":"2024-08-17T17:19:18.759882+0000","last_clean":"2024-08-17T17:19:18.759882+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:19:18.759882+0000","last_undegraded":"2024-08-17T17:19:18.759882+0000","last_fullsized":"2024-08-17T17:19:18.759882+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":734,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.650840+0000","last_change":"2024-08-17T17:15:38.609543+0000","last_active":"2024-08-17T17:19:02.650840+0000","last_peered":"2024-08-17T17:19:02.650840+0000","last_clean":"2024-08-17T17:19:02.650840+0000","last_became_active":"2024-08-17T17:15:38.609238+0000","last_became_peered":"2024-08-17T17:15:38.609238+0000","last_unstale":"2024-08-17T17:19:02.650840+0000","last_undegraded":"2024-08-17T17:19:02.650840+0000","last_fullsized":"2024-08-17T17:19:02.650840+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:32.029842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1412,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.747876+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:19:18.747876+0000","last_peered":"2024-08-17T17:19:18.747876+0000","last_clean":"2024-08-17T17:19:18.747876+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:19:18.747876+0000","last_undegraded":"2024-08-17T17:19:18.747876+0000","last_fullsized":"2024-08-17T17:19:18.747876+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1427,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.750405+0000","last_change":"2024-08-17T17:18:29.390118+0000","last_active":"2024-08-17T17:19:18.750405+0000","last_peered":"2024-08-17T17:19:18.750405+0000","last_clean":"2024-08-17T17:19:18.750405+0000","last_became_active":"2024-08-17T17:18:28.528874+0000","last_became_peered":"2024-08-17T17:18:28.528874+0000","last_unstale":"2024-08-17T17:19:18.750405+0000","last_undegraded":"2024-08-17T17:19:18.750405+0000","last_fullsized":"2024-08-17T17:19:18.750405+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:29.390003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:29.390003+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:27.511820+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1154'6180","reported_seq":6299,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:24.633864+0000","last_change":"2024-08-17T17:19:20.831069+0000","last_active":"2024-08-17T17:19:24.633864+0000","last_peered":"2024-08-17T17:19:24.633864+0000","last_clean":"2024-08-17T17:19:24.633864+0000","last_became_active":"2024-08-17T17:19:01.696058+0000","last_became_peered":"2024-08-17T17:19:01.696058+0000","last_unstale":"2024-08-17T17:19:24.633864+0000","last_undegraded":"2024-08-17T17:19:24.633864+0000","last_fullsized":"2024-08-17T17:19:24.633864+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":0,"last_scrub":"1154'6072","last_scrub_stamp":"2024-08-17T17:19:20.830703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:19:20.830703+0000","objects_scrubbed":3288,"log_size":6180,"log_dups_size":0,"ondisk_log_size":6180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:50:22.381897+0000","scrub_duration":1266,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223674368,"num_objects":3413,"num_object_clones":0,"num_object_copies":10239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":125,"num_write_kb":8000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":95,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.652029+0000","last_change":"2024-08-17T17:18:31.553838+0000","last_active":"2024-08-17T17:19:02.652029+0000","last_peered":"2024-08-17T17:19:02.652029+0000","last_clean":"2024-08-17T17:19:02.652029+0000","last_became_active":"2024-08-17T17:18:31.537264+0000","last_became_peered":"2024-08-17T17:18:31.537264+0000","last_unstale":"2024-08-17T17:19:02.652029+0000","last_undegraded":"2024-08-17T17:19:02.652029+0000","last_fullsized":"2024-08-17T17:19:02.652029+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:26.972836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1107,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.653442+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:19:02.653442+0000","last_peered":"2024-08-17T17:19:02.653442+0000","last_clean":"2024-08-17T17:19:02.653442+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:19:02.653442+0000","last_undegraded":"2024-08-17T17:19:02.653442+0000","last_fullsized":"2024-08-17T17:19:02.653442+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1083,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.664161+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:19:02.664161+0000","last_peered":"2024-08-17T17:19:02.664161+0000","last_clean":"2024-08-17T17:19:02.664161+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:19:02.664161+0000","last_undegraded":"2024-08-17T17:19:02.664161+0000","last_fullsized":"2024-08-17T17:19:02.664161+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.10","version":"1154'6176","reported_seq":6220,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:22.191846+0000","last_change":"2024-08-17T17:19:01.710174+0000","last_active":"2024-08-17T17:19:22.191846+0000","last_peered":"2024-08-17T17:19:22.191846+0000","last_clean":"2024-08-17T17:19:22.191846+0000","last_became_active":"2024-08-17T17:19:01.708023+0000","last_became_peered":"2024-08-17T17:19:01.708023+0000","last_unstale":"2024-08-17T17:19:22.191846+0000","last_undegraded":"2024-08-17T17:19:22.191846+0000","last_fullsized":"2024-08-17T17:19:22.191846+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6176,"log_dups_size":0,"ondisk_log_size":6176,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221478912,"num_objects":3380,"num_object_clones":0,"num_object_copies":10140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3380,"num_write_kb":216288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1096,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.653203+0000","last_change":"2024-08-17T17:06:39.627594+0000","last_active":"2024-08-17T17:19:02.653203+0000","last_peered":"2024-08-17T17:19:02.653203+0000","last_clean":"2024-08-17T17:19:02.653203+0000","last_became_active":"2024-08-17T17:06:39.627105+0000","last_became_peered":"2024-08-17T17:06:39.627105+0000","last_unstale":"2024-08-17T17:19:02.653203+0000","last_undegraded":"2024-08-17T17:19:02.653203+0000","last_fullsized":"2024-08-17T17:19:02.653203+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:30.105070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1106,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.737134+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:19:18.737134+0000","last_peered":"2024-08-17T17:19:18.737134+0000","last_clean":"2024-08-17T17:19:18.737134+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:19:18.737134+0000","last_undegraded":"2024-08-17T17:19:18.737134+0000","last_fullsized":"2024-08-17T17:19:18.737134+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.11","version":"1154'5984","reported_seq":6028,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:22.068234+0000","last_change":"2024-08-17T17:19:01.705286+0000","last_active":"2024-08-17T17:19:22.068234+0000","last_peered":"2024-08-17T17:19:22.068234+0000","last_clean":"2024-08-17T17:19:22.068234+0000","last_became_active":"2024-08-17T17:19:01.702922+0000","last_became_peered":"2024-08-17T17:19:01.702922+0000","last_unstale":"2024-08-17T17:19:22.068234+0000","last_undegraded":"2024-08-17T17:19:22.068234+0000","last_fullsized":"2024-08-17T17:19:22.068234+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":5984,"log_dups_size":0,"ondisk_log_size":5984,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214564864,"num_objects":3274,"num_object_clones":0,"num_object_copies":9822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3274,"num_write_kb":209536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":37,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.659411+0000","last_change":"2024-08-17T17:17:47.283984+0000","last_active":"2024-08-17T17:19:02.659411+0000","last_peered":"2024-08-17T17:19:02.659411+0000","last_clean":"2024-08-17T17:19:02.659411+0000","last_became_active":"2024-08-17T17:17:47.283270+0000","last_became_peered":"2024-08-17T17:17:47.283270+0000","last_unstale":"2024-08-17T17:19:02.659411+0000","last_undegraded":"2024-08-17T17:19:02.659411+0000","last_fullsized":"2024-08-17T17:19:02.659411+0000","mapping_epoch":1127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:16.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1081,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.656131+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:19:02.656131+0000","last_peered":"2024-08-17T17:19:02.656131+0000","last_clean":"2024-08-17T17:19:02.656131+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:19:02.656131+0000","last_undegraded":"2024-08-17T17:19:02.656131+0000","last_fullsized":"2024-08-17T17:19:02.656131+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.12","version":"1154'6239","reported_seq":6357,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:24.704596+0000","last_change":"2024-08-17T17:19:22.720455+0000","last_active":"2024-08-17T17:19:24.704596+0000","last_peered":"2024-08-17T17:19:24.704596+0000","last_clean":"2024-08-17T17:19:24.704596+0000","last_became_active":"2024-08-17T17:19:01.711724+0000","last_became_peered":"2024-08-17T17:19:01.711724+0000","last_unstale":"2024-08-17T17:19:24.704596+0000","last_undegraded":"2024-08-17T17:19:24.704596+0000","last_fullsized":"2024-08-17T17:19:24.704596+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":5,"last_scrub":"1154'6171","last_scrub_stamp":"2024-08-17T17:19:22.720072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:19:22.720072+0000","objects_scrubbed":3360,"log_size":6239,"log_dups_size":0,"ondisk_log_size":6239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:15:12.696656+0000","scrub_duration":1168,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225181696,"num_objects":3436,"num_object_clones":0,"num_object_copies":10308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":76,"num_write_kb":4864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1093,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.662371+0000","last_change":"2024-08-17T17:06:39.625029+0000","last_active":"2024-08-17T17:19:02.662371+0000","last_peered":"2024-08-17T17:19:02.662371+0000","last_clean":"2024-08-17T17:19:02.662371+0000","last_became_active":"2024-08-17T17:06:39.624688+0000","last_became_peered":"2024-08-17T17:06:39.624688+0000","last_unstale":"2024-08-17T17:19:02.662371+0000","last_undegraded":"2024-08-17T17:19:02.662371+0000","last_fullsized":"2024-08-17T17:19:02.662371+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:54.849668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1369,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.748276+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:19:18.748276+0000","last_peered":"2024-08-17T17:19:18.748276+0000","last_clean":"2024-08-17T17:19:18.748276+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:19:18.748276+0000","last_undegraded":"2024-08-17T17:19:18.748276+0000","last_fullsized":"2024-08-17T17:19:18.748276+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.13","version":"1154'6116","reported_seq":6190,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:24.645620+0000","last_change":"2024-08-17T17:19:20.593812+0000","last_active":"2024-08-17T17:19:24.645620+0000","last_peered":"2024-08-17T17:19:24.645620+0000","last_clean":"2024-08-17T17:19:24.645620+0000","last_became_active":"2024-08-17T17:19:01.708389+0000","last_became_peered":"2024-08-17T17:19:01.708389+0000","last_unstale":"2024-08-17T17:19:24.645620+0000","last_undegraded":"2024-08-17T17:19:24.645620+0000","last_fullsized":"2024-08-17T17:19:24.645620+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6116,"log_dups_size":0,"ondisk_log_size":6116,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:19:25.586871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222461952,"num_objects":3395,"num_object_clones":0,"num_object_copies":10185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3395,"num_write_kb":217248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":947,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.660208+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:19:02.660208+0000","last_peered":"2024-08-17T17:19:02.660208+0000","last_clean":"2024-08-17T17:19:02.660208+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:19:02.660208+0000","last_undegraded":"2024-08-17T17:19:02.660208+0000","last_fullsized":"2024-08-17T17:19:02.660208+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1388,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.747883+0000","last_change":"2024-08-17T17:07:45.678950+0000","last_active":"2024-08-17T17:19:18.747883+0000","last_peered":"2024-08-17T17:19:18.747883+0000","last_clean":"2024-08-17T17:19:18.747883+0000","last_became_active":"2024-08-17T17:07:45.678472+0000","last_became_peered":"2024-08-17T17:07:45.678472+0000","last_unstale":"2024-08-17T17:19:18.747883+0000","last_undegraded":"2024-08-17T17:19:18.747883+0000","last_fullsized":"2024-08-17T17:19:18.747883+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:48:02.385927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1095,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.653369+0000","last_change":"2024-08-17T17:06:39.628542+0000","last_active":"2024-08-17T17:19:02.653369+0000","last_peered":"2024-08-17T17:19:02.653369+0000","last_clean":"2024-08-17T17:19:02.653369+0000","last_became_active":"2024-08-17T17:06:39.628253+0000","last_became_peered":"2024-08-17T17:06:39.628253+0000","last_unstale":"2024-08-17T17:19:02.653369+0000","last_undegraded":"2024-08-17T17:19:02.653369+0000","last_fullsized":"2024-08-17T17:19:02.653369+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:22.349326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1417,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.747989+0000","last_change":"2024-08-17T17:18:31.532183+0000","last_active":"2024-08-17T17:19:18.747989+0000","last_peered":"2024-08-17T17:19:18.747989+0000","last_clean":"2024-08-17T17:19:18.747989+0000","last_became_active":"2024-08-17T17:18:31.531094+0000","last_became_peered":"2024-08-17T17:18:31.531094+0000","last_unstale":"2024-08-17T17:19:18.747989+0000","last_undegraded":"2024-08-17T17:19:18.747989+0000","last_fullsized":"2024-08-17T17:19:18.747989+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:01.763127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":908,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:07.212346+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:19:07.212346+0000","last_peered":"2024-08-17T17:19:07.212346+0000","last_clean":"2024-08-17T17:19:07.212346+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:19:07.212346+0000","last_undegraded":"2024-08-17T17:19:07.212346+0000","last_fullsized":"2024-08-17T17:19:07.212346+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":908,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.748431+0000","last_change":"2024-08-17T17:15:38.610768+0000","last_active":"2024-08-17T17:19:18.748431+0000","last_peered":"2024-08-17T17:19:18.748431+0000","last_clean":"2024-08-17T17:19:18.748431+0000","last_became_active":"2024-08-17T17:15:38.610337+0000","last_became_peered":"2024-08-17T17:15:38.610337+0000","last_unstale":"2024-08-17T17:19:18.748431+0000","last_undegraded":"2024-08-17T17:19:18.748431+0000","last_fullsized":"2024-08-17T17:19:18.748431+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:15.632778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1387,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.748437+0000","last_change":"2024-08-17T17:07:45.689782+0000","last_active":"2024-08-17T17:19:18.748437+0000","last_peered":"2024-08-17T17:19:18.748437+0000","last_clean":"2024-08-17T17:19:18.748437+0000","last_became_active":"2024-08-17T17:07:45.689312+0000","last_became_peered":"2024-08-17T17:07:45.689312+0000","last_unstale":"2024-08-17T17:19:18.748437+0000","last_undegraded":"2024-08-17T17:19:18.748437+0000","last_fullsized":"2024-08-17T17:19:18.748437+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:07.560555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1113,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.651827+0000","last_change":"2024-08-17T17:07:12.649175+0000","last_active":"2024-08-17T17:19:02.651827+0000","last_peered":"2024-08-17T17:19:02.651827+0000","last_clean":"2024-08-17T17:19:02.651827+0000","last_became_active":"2024-08-17T17:07:12.648822+0000","last_became_peered":"2024-08-17T17:07:12.648822+0000","last_unstale":"2024-08-17T17:19:02.651827+0000","last_undegraded":"2024-08-17T17:19:02.651827+0000","last_fullsized":"2024-08-17T17:19:02.651827+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:03.924431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1435,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.753178+0000","last_change":"2024-08-17T17:17:47.316690+0000","last_active":"2024-08-17T17:19:18.753178+0000","last_peered":"2024-08-17T17:19:18.753178+0000","last_clean":"2024-08-17T17:19:18.753178+0000","last_became_active":"2024-08-17T17:17:47.316333+0000","last_became_peered":"2024-08-17T17:17:47.316333+0000","last_unstale":"2024-08-17T17:19:18.753178+0000","last_undegraded":"2024-08-17T17:19:18.753178+0000","last_fullsized":"2024-08-17T17:19:18.753178+0000","mapping_epoch":1127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:04.542237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":100,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.653020+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:19:02.653020+0000","last_peered":"2024-08-17T17:19:02.653020+0000","last_clean":"2024-08-17T17:19:02.653020+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:19:02.653020+0000","last_undegraded":"2024-08-17T17:19:02.653020+0000","last_fullsized":"2024-08-17T17:19:02.653020+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.18","version":"1154'6041","reported_seq":6115,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:21.261575+0000","last_change":"2024-08-17T17:19:19.469146+0000","last_active":"2024-08-17T17:19:21.261575+0000","last_peered":"2024-08-17T17:19:21.261575+0000","last_clean":"2024-08-17T17:19:21.261575+0000","last_became_active":"2024-08-17T17:19:01.702866+0000","last_became_peered":"2024-08-17T17:19:01.702866+0000","last_unstale":"2024-08-17T17:19:21.261575+0000","last_undegraded":"2024-08-17T17:19:21.261575+0000","last_fullsized":"2024-08-17T17:19:21.261575+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6041,"log_dups_size":0,"ondisk_log_size":6041,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:19:24.377259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215089152,"num_objects":3282,"num_object_clones":0,"num_object_copies":9846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3282,"num_write_kb":210048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1111,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.653139+0000","last_change":"2024-08-17T17:07:45.684606+0000","last_active":"2024-08-17T17:19:02.653139+0000","last_peered":"2024-08-17T17:19:02.653139+0000","last_clean":"2024-08-17T17:19:02.653139+0000","last_became_active":"2024-08-17T17:07:45.684124+0000","last_became_peered":"2024-08-17T17:07:45.684124+0000","last_unstale":"2024-08-17T17:19:02.653139+0000","last_undegraded":"2024-08-17T17:19:02.653139+0000","last_fullsized":"2024-08-17T17:19:02.653139+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:16.110507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1095,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.659768+0000","last_change":"2024-08-17T17:06:39.625245+0000","last_active":"2024-08-17T17:19:02.659768+0000","last_peered":"2024-08-17T17:19:02.659768+0000","last_clean":"2024-08-17T17:19:02.659768+0000","last_became_active":"2024-08-17T17:06:39.624657+0000","last_became_peered":"2024-08-17T17:06:39.624657+0000","last_unstale":"2024-08-17T17:19:02.659768+0000","last_undegraded":"2024-08-17T17:19:02.659768+0000","last_fullsized":"2024-08-17T17:19:02.659768+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:43.097460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.19","version":"1154'6104","reported_seq":6157,"reported_epoch":1154,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:22.133532+0000","last_change":"2024-08-17T17:19:01.771867+0000","last_active":"2024-08-17T17:19:22.133532+0000","last_peered":"2024-08-17T17:19:22.133532+0000","last_clean":"2024-08-17T17:18:30.513360+0000","last_became_active":"2024-08-17T17:19:01.766350+0000","last_became_peered":"2024-08-17T17:19:01.766350+0000","last_unstale":"2024-08-17T17:19:22.133532+0000","last_undegraded":"2024-08-17T17:19:01.712084+0000","last_fullsized":"2024-08-17T17:19:22.133532+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6104,"log_dups_size":0,"ondisk_log_size":6104,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219086848,"num_objects":3343,"num_object_clones":0,"num_object_copies":10029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2089,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3343,"num_write_kb":213952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"0,1,4","objects":1254},{"shards":"1,4,7","objects":2089}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1129,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.651687+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:19:02.651687+0000","last_peered":"2024-08-17T17:19:02.651687+0000","last_clean":"2024-08-17T17:19:02.651687+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:19:02.651687+0000","last_undegraded":"2024-08-17T17:19:02.651687+0000","last_fullsized":"2024-08-17T17:19:02.651687+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1080,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.659939+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:19:02.659939+0000","last_peered":"2024-08-17T17:19:02.659939+0000","last_clean":"2024-08-17T17:19:02.659939+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:19:02.659939+0000","last_undegraded":"2024-08-17T17:19:02.659939+0000","last_fullsized":"2024-08-17T17:19:02.659939+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1407,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.748573+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:19:18.748573+0000","last_peered":"2024-08-17T17:19:18.748573+0000","last_clean":"2024-08-17T17:19:18.748573+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:19:18.748573+0000","last_undegraded":"2024-08-17T17:19:18.748573+0000","last_fullsized":"2024-08-17T17:19:18.748573+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1382,"reported_epoch":1154,"state":"active+clean","last_fresh":"2024-08-17T17:19:18.750680+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:19:18.750680+0000","last_peered":"2024-08-17T17:19:18.750680+0000","last_clean":"2024-08-17T17:19:18.750680+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:19:18.750680+0000","last_undegraded":"2024-08-17T17:19:18.750680+0000","last_fullsized":"2024-08-17T17:19:18.750680+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1096,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:02.652720+0000","last_change":"2024-08-17T17:06:39.627689+0000","last_active":"2024-08-17T17:19:02.652720+0000","last_peered":"2024-08-17T17:19:02.652720+0000","last_clean":"2024-08-17T17:19:02.652720+0000","last_became_active":"2024-08-17T17:06:39.627375+0000","last_became_peered":"2024-08-17T17:06:39.627375+0000","last_unstale":"2024-08-17T17:19:02.652720+0000","last_undegraded":"2024-08-17T17:19:02.652720+0000","last_fullsized":"2024-08-17T17:19:02.652720+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:51.954076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":123,"reported_epoch":1153,"state":"active+clean","last_fresh":"2024-08-17T17:19:17.216019+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:19:17.216019+0000","last_peered":"2024-08-17T17:19:17.216019+0000","last_clean":"2024-08-17T17:19:17.216019+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:19:17.216019+0000","last_undegraded":"2024-08-17T17:19:17.216019+0000","last_fullsized":"2024-08-17T17:19:17.216019+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":7026049024,"num_objects":107209,"num_object_clones":0,"num_object_copies":321627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27952,"num_objects_misplaced":3623,"num_objects_unfound":0,"num_objects_dirty":107209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":100561,"num_write_kb":6435904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1373,"num_bytes_recovered":89980928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21129134080,"data_stored":21129134080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":160175,"ondisk_log_size":160175,"up":78,"acting":75,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":912388,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":210,"num_read_kb":453,"num_write":118,"num_write_kb":1319,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1073397,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201615,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2678632,"kb_used_data":2404768,"kb_used_omap":1,"kb_used_meta":273854,"kb_avail":91066520,"statfs":{"total":95995035648,"available":93252116480,"internally_reserved":0,"allocated":2462482432,"data_stored":2460331234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":280426955},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463252,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2618464,"kb_used_data":2380696,"kb_used_omap":1,"kb_used_meta":237758,"kb_avail":91126688,"statfs":{"total":95995035648,"available":93313728512,"internally_reserved":0,"allocated":2437832704,"data_stored":2435689571,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":243464649},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626894,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4782484,"kb_used_data":4439372,"kb_used_omap":42,"kb_used_meta":343061,"kb_avail":88962668,"statfs":{"total":95995035648,"available":91097772032,"internally_reserved":0,"allocated":4545916928,"data_stored":4543781884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":351294524},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785371,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3340752,"kb_used_data":3101640,"kb_used_omap":48,"kb_used_meta":239055,"kb_avail":90404400,"statfs":{"total":95995035648,"available":92574105600,"internally_reserved":0,"allocated":3176079360,"data_stored":3173947797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":244793006},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724888,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2243484,"kb_used_data":1991764,"kb_used_omap":1,"kb_used_meta":251710,"kb_avail":91501668,"statfs":{"total":95995035648,"available":93697708032,"internally_reserved":0,"allocated":2039566336,"data_stored":2037423669,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":257751494},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986525,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2635172,"kb_used_data":2383452,"kb_used_omap":1,"kb_used_meta":251710,"kb_avail":91109980,"statfs":{"total":95995035648,"available":93296619520,"internally_reserved":0,"allocated":2440654848,"data_stored":2438507676,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":257751494},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706132,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2832600,"kb_used_data":2535440,"kb_used_omap":35,"kb_used_meta":297116,"kb_avail":90912552,"statfs":{"total":95995035648,"available":93094453248,"internally_reserved":0,"allocated":2596290560,"data_stored":2594151946,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":304246839},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831905,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1571604,"kb_used_data":1380876,"kb_used_omap":65,"kb_used_meta":190654,"kb_avail":92173548,"statfs":{"total":95995035648,"available":94385713152,"internally_reserved":0,"allocated":1414017024,"data_stored":1411881950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":195229883},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1410859008,"data_stored":1410859008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3188916224,"data_stored":3188916224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2610036736,"data_stored":2610036736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4543217664,"data_stored":4543217664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2445869056,"data_stored":2445869056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2033254400,"data_stored":2033254400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2434400256,"data_stored":2434400256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2462580736,"data_stored":2462580736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:19:28.205 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_2 pgp_num 26 2024-08-17T17:19:28.207 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:28.232 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:28.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:28.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:28.334 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:28.362 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:28.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:28.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:28.451 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 90 2 112090 112088 77.8241 55.3125 0.0397546 0.0128432 2024-08-17T17:19:28.464 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:28.486 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:28.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:28.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:28.588 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:28.609 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:28.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:28.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:28.711 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:28.732 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:28.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:28.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:28.834 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:28.856 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:28.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:28.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:28.958 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:28.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:28.980 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:28.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:28.999 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:28 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 27952/321708 objects degraded (8.689%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:28.999 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:28 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 27952/321708 objects degraded (8.689%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:29.045 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:29.082 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:29.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:28 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 27952/321708 objects degraded (8.689%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:29.107 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:29.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:29.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:29.209 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:29.232 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:29.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:29.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:29.334 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:29.360 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:29.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:29.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:29.451 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Total time run: 90.0202 2024-08-17T17:19:29.451 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Total writes made: 112090 2024-08-17T17:19:29.451 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Write size: 65536 2024-08-17T17:19:29.451 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Object size: 65536 2024-08-17T17:19:29.451 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Bandwidth (MB/sec): 77.8228 2024-08-17T17:19:29.452 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev Bandwidth: 46.5243 2024-08-17T17:19:29.452 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max bandwidth (MB/sec): 249 2024-08-17T17:19:29.452 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min bandwidth (MB/sec): 0 2024-08-17T17:19:29.452 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Average IOPS: 1245 2024-08-17T17:19:29.452 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev IOPS: 744.388 2024-08-17T17:19:29.452 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max IOPS: 3984 2024-08-17T17:19:29.452 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min IOPS: 0 2024-08-17T17:19:29.452 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Average Latency(s): 0.0128436 2024-08-17T17:19:29.452 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev Latency(s): 0.0372619 2024-08-17T17:19:29.452 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max latency(s): 2.52879 2024-08-17T17:19:29.452 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min latency(s): 0.00126452 2024-08-17T17:19:29.462 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:29.465 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:19:29.499 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:29.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:29.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:29.601 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:29.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:29.629 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:29.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:29.730 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:29.760 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:29.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:29.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:29.862 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:29.888 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:29.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:29.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:29.989 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:30.018 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:30.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:30.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:29 smithi063 ceph-mon[114756]: pgmap v1381: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 13 MiB/s wr, 212 op/s; 27952/331236 objects degraded (8.439%); 3525/331236 objects misplaced (1.064%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:19:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:29 smithi063 ceph-mon[109843]: pgmap v1381: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 13 MiB/s wr, 212 op/s; 27952/331236 objects degraded (8.439%); 3525/331236 objects misplaced (1.064%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:19:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:29 smithi162 ceph-mon[104668]: pgmap v1381: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 13 MiB/s wr, 212 op/s; 27952/331236 objects degraded (8.439%); 3525/331236 objects misplaced (1.064%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:19:30.120 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:30.146 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:30.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:30.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:30.248 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:30.295 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:30.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:30.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:30.397 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:30.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:30.436 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:30.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:30.537 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:30.584 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:30.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:30.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:30.686 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:30.730 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:30.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:30.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:30.832 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:30.858 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:30.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:30.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:30.960 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:30.987 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:30.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:30.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:31.089 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:31.128 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:31.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:31.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:31.230 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:31.258 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:31.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:31.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:31.359 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:31.399 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:31.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:31.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:31.501 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:31.525 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:31.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:31.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:31.627 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:31.657 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:31.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:31.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:31.759 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:31.805 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:31.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:31.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:31.907 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:31.934 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:31.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:31.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:31 smithi063 ceph-mon[114756]: pgmap v1382: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 16 MiB/s wr, 256 op/s; 27952/333372 objects degraded (8.385%); 3525/333372 objects misplaced (1.057%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:19:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:31 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2827548847' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:19:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:31 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:19:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:31 smithi063 ceph-mon[109843]: pgmap v1382: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 16 MiB/s wr, 256 op/s; 27952/333372 objects degraded (8.385%); 3525/333372 objects misplaced (1.057%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:19:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:31 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2827548847' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:19:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:31 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:19:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:32.037 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:32.066 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:32.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:32.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:31 smithi162 ceph-mon[104668]: pgmap v1382: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 16 MiB/s wr, 256 op/s; 27952/333372 objects degraded (8.385%); 3525/333372 objects misplaced (1.057%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:19:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:31 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2827548847' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:19:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:31 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:19:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:32.168 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:32.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:32.209 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:32.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:32.311 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:32.354 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:32.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:32.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:32.456 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:32.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:32.517 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:32.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:32.618 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:32.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:32.645 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:32.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:32.746 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:32.775 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:32.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:32.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:32.877 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:32.920 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:32.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:32.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:33.022 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:33.027 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:32 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:19:33.028 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:32 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:19:33.028 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:32 smithi063 ceph-mon[109843]: osdmap e1155: 8 total, 8 up, 8 in 2024-08-17T17:19:33.028 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:32 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:19:33.028 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:32 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:19:33.028 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:32 smithi063 ceph-mon[114756]: osdmap e1155: 8 total, 8 up, 8 in 2024-08-17T17:19:33.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:33.066 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:33.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:32 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:19:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:32 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:19:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:32 smithi162 ceph-mon[104668]: osdmap e1155: 8 total, 8 up, 8 in 2024-08-17T17:19:33.167 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:33.209 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:33.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:33.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:33.310 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:33.328 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:19:33.328 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:19:33.328 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.328 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:115415 7.e 7:77b77560:::benchmark_data_smithi063_196002_object3322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1155)", 2024-08-17T17:19:33.329 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:33.288269+0000", 2024-08-17T17:19:33.329 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.034546258000000003, 2024-08-17T17:19:33.329 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.034578696999999999, 2024-08-17T17:19:33.329 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:33.329 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:33.329 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:19:33.329 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:33.329 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:19:33.329 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:19:33.329 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 115415 2024-08-17T17:19:33.329 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.329 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:33.330 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.330 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:33.330 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.288269+0000", 2024-08-17T17:19:33.330 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:33.330 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.330 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.330 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:33.330 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.288269+0000", 2024-08-17T17:19:33.330 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:33.330 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.330 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.330 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:33.331 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.288271+0000", 2024-08-17T17:19:33.331 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.7049999999999999e-06 2024-08-17T17:19:33.331 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.331 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.331 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:33.331 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.288281+0000", 2024-08-17T17:19:33.331 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.5070000000000006e-06 2024-08-17T17:19:33.331 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.331 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.331 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:33.331 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.288307+0000", 2024-08-17T17:19:33.331 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5843999999999999e-05 2024-08-17T17:19:33.332 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.332 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.332 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:33.332 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.288318+0000", 2024-08-17T17:19:33.332 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1279e-05 2024-08-17T17:19:33.332 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.332 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.332 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:33.332 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.296236+0000", 2024-08-17T17:19:33.332 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0079180420000000001 2024-08-17T17:19:33.332 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.332 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.332 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:33.333 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.296614+0000", 2024-08-17T17:19:33.333 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00037766500000000001 2024-08-17T17:19:33.333 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.333 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.333 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:19:33.333 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.297027+0000", 2024-08-17T17:19:33.333 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00041313500000000002 2024-08-17T17:19:33.333 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.333 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.333 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:33.333 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.312376+0000", 2024-08-17T17:19:33.333 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.015349154 2024-08-17T17:19:33.333 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.334 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.334 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:33.334 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.313827+0000", 2024-08-17T17:19:33.334 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0014513709999999999 2024-08-17T17:19:33.334 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:33.334 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:33.334 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:33.334 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.334 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.334 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.34965.0:115415 7.e e1155/1144)", 2024-08-17T17:19:33.334 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:33.322329+0000", 2024-08-17T17:19:33.334 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.00048602500000000001, 2024-08-17T17:19:33.335 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00060263500000000002, 2024-08-17T17:19:33.335 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:33.335 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:33.335 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "queued for pg", 2024-08-17T17:19:33.335 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:33.335 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.335 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:33.335 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.322329+0000", 2024-08-17T17:19:33.335 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:33.335 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.335 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.335 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:33.335 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.322329+0000", 2024-08-17T17:19:33.336 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:33.336 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.336 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.336 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:33.336 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.322331+0000", 2024-08-17T17:19:33.336 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9319999999999999e-06 2024-08-17T17:19:33.336 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.336 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.336 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:33.336 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.322350+0000", 2024-08-17T17:19:33.336 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8814e-05 2024-08-17T17:19:33.336 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.336 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.337 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:33.337 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.322371+0000", 2024-08-17T17:19:33.337 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1702999999999999e-05 2024-08-17T17:19:33.337 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:33.337 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:33.337 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:33.337 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:33.322383+0000", 2024-08-17T17:19:33.337 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2007999999999999e-05 2024-08-17T17:19:33.337 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:33.337 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:33.337 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:33.338 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:33.338 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-08-17T17:19:33.338 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 2 2024-08-17T17:19:33.338 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:19:33.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:33.351 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:33.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:33.445 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:19:33.446 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:19:33.452 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:33.455 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:19:33.455 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:19:33.499 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:19:33.499 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:19:33.500 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:33.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:33.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:33.541 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:19:33.541 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:19:33.541 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:19:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:19:33] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.51.0" 2024-08-17T17:19:33.572 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:19:33.572 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:19:33.603 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:33.606 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:19:33.606 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:19:33.640 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:19:33.641 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:19:33.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:33.649 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:33.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:33.675 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:19:33.676 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:19:33.750 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:33.777 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:33.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:33.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:33.879 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:33.898 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:33 smithi063 ceph-mon[114756]: pgmap v1384: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 870 KiB/s wr, 127 op/s; 27506/332163 objects degraded (8.281%); 3326/332163 objects misplaced (1.001%); 4.6 MiB/s, 74 objects/s recovering 2024-08-17T17:19:33.898 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:33 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 27506/332163 objects degraded (8.281%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:33.898 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:33 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1345760512' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:33.898 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:33 smithi063 ceph-mon[109843]: pgmap v1384: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 870 KiB/s wr, 127 op/s; 27506/332163 objects degraded (8.281%); 3326/332163 objects misplaced (1.001%); 4.6 MiB/s, 74 objects/s recovering 2024-08-17T17:19:33.898 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:33 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 27506/332163 objects degraded (8.281%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:33.899 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:33 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1345760512' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:33.910 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:33.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:33.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:34.011 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:34.056 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:34.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:34.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:34.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:33 smithi162 ceph-mon[104668]: pgmap v1384: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 870 KiB/s wr, 127 op/s; 27506/332163 objects degraded (8.281%); 3326/332163 objects misplaced (1.001%); 4.6 MiB/s, 74 objects/s recovering 2024-08-17T17:19:34.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:33 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 27506/332163 objects degraded (8.281%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:19:34.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:33 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1345760512' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:34.158 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:34.185 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:34.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:34.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:34.288 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:34.320 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:34.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:34.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:34.422 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:34.460 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:34.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:34.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:34.561 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:34.589 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:34.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:34.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:34.691 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:34.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:34.721 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:34.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:34.822 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:34.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:34.861 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:34.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:34.962 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:35.001 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:35.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:35.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:35.103 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:35.137 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:35.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:35.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:35.239 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:35.278 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:35.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:35.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:35.380 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:35.402 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:35.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:35.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:35.504 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:35.547 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:35.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:35.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:35.649 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:35.687 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:35.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:35.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:35.789 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:35.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:35.818 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:35.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:35.919 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:35.947 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:35.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:35.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:36.049 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:36.083 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:36.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:36.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:35 smithi162 ceph-mon[104668]: pgmap v1385: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 102 op/s; 27364/331230 objects degraded (8.261%); 3326/331230 objects misplaced (1.004%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:19:36.185 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:36.214 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:36.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:36.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:36.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:35 smithi063 ceph-mon[114756]: pgmap v1385: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 102 op/s; 27364/331230 objects degraded (8.261%); 3326/331230 objects misplaced (1.004%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:19:36.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:35 smithi063 ceph-mon[109843]: pgmap v1385: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 102 op/s; 27364/331230 objects degraded (8.261%); 3326/331230 objects misplaced (1.004%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:19:36.296 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:19:36.316 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:36.361 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:36.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:36.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:36.463 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:36.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:36.501 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:36.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:36.602 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:36.628 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:36.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:36.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:36.730 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:36.769 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:36.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:36.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:36.871 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:36.893 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:36.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:36.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:36.995 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:37.036 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:37.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:37.138 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:37.164 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:37.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:37.265 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:37.294 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:37.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:37.396 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:37.416 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:37.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:37.518 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:37.553 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:37.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:37.655 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:37.699 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:37.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:37.801 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:37.823 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:37.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:37.925 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:37.951 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:37.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:37.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:38.052 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:38.076 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:38.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:38.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:37 smithi162 ceph-mon[104668]: pgmap v1386: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 767 op/s; 27364/327957 objects degraded (8.344%); 3326/327957 objects misplaced (1.014%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:19:38.178 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:38.222 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:38.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:38.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:38.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:37 smithi063 ceph-mon[109843]: pgmap v1386: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 767 op/s; 27364/327957 objects degraded (8.344%); 3326/327957 objects misplaced (1.014%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:19:38.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:37 smithi063 ceph-mon[114756]: pgmap v1386: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 30 MiB/s wr, 767 op/s; 27364/327957 objects degraded (8.344%); 3326/327957 objects misplaced (1.014%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:19:38.324 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:38.359 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:38.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:38.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:38.461 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:38.500 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:38.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:38.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:38.602 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:38.632 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:38.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:38.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:38.734 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:38.764 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:38.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:38.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:38.866 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:38.875 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:19:38.911 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:38.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:38.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:39.013 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:39.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:38 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 27364/327957 objects degraded (8.344%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:19:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:38 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 27364/327957 objects degraded (8.344%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:19:39.039 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:39.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:39.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:39.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:38 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 27364/327957 objects degraded (8.344%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:19:39.141 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:39.176 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:39.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:39.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:39.278 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:39.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:39.308 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:39.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:39.409 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:39.455 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:39.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:39.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:39.557 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:39.591 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:39.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:39.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:39.693 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:39.727 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:39.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:39.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:39.832 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:39.858 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:39.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:39.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:39.960 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:39.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:39.993 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:39.993 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:40.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:39 smithi063 ceph-mon[114756]: pgmap v1387: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 11 MiB/s wr, 768 op/s; 26295/318399 objects degraded (8.259%); 3203/318399 objects misplaced (1.006%); 12 MiB/s, 197 objects/s recovering 2024-08-17T17:19:40.032 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:39 smithi063 ceph-mon[109843]: pgmap v1387: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 11 MiB/s wr, 768 op/s; 26295/318399 objects degraded (8.259%); 3203/318399 objects misplaced (1.006%); 12 MiB/s, 197 objects/s recovering 2024-08-17T17:19:40.045 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:40.094 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:40.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:39 smithi162 ceph-mon[104668]: pgmap v1387: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 102 B/s rd, 11 MiB/s wr, 768 op/s; 26295/318399 objects degraded (8.259%); 3203/318399 objects misplaced (1.006%); 12 MiB/s, 197 objects/s recovering 2024-08-17T17:19:40.119 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:40.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:40.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:40.124 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:40.221 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:40.249 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:40.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:40.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:40.350 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:40.376 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:40.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:40.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:40.478 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:40.528 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:40.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:40.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:40.630 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:40.656 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:40.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:40.658 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:40.759 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:40.786 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:40.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:40.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:40.887 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:40.913 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:40.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:40.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:41.014 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:41.040 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:41.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:41.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:41.142 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:41.184 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:41.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:41.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:41.285 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:41.289 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:19:41.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:41.313 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:41.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:41.414 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:41.443 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:41.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:41.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:41.545 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:41.575 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:41.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:41.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:41.676 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:41.722 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:41.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:41.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:41.824 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:41.869 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:41.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:41.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:41.887 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:41 smithi063 ceph-mon[109843]: pgmap v1388: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 5.9 MiB/s wr, 785 op/s; 26295/315441 objects degraded (8.336%); 3203/315441 objects misplaced (1.015%); 12 MiB/s, 197 objects/s recovering 2024-08-17T17:19:41.889 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:41 smithi063 ceph-mon[114756]: pgmap v1388: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 5.9 MiB/s wr, 785 op/s; 26295/315441 objects degraded (8.336%); 3203/315441 objects misplaced (1.015%); 12 MiB/s, 197 objects/s recovering 2024-08-17T17:19:41.971 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:42.026 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:42.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:42.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:42.085 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:19:42.169 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:42.170 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:41 smithi162 ceph-mon[104668]: pgmap v1388: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 102 B/s rd, 5.9 MiB/s wr, 785 op/s; 26295/315441 objects degraded (8.336%); 3203/315441 objects misplaced (1.015%); 12 MiB/s, 197 objects/s recovering 2024-08-17T17:19:42.213 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:42.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:42.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:42.315 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:42.340 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:42.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:42.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:42.442 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:42.504 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:42.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:42.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:42.574 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:42.607 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:42.632 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:42.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:42.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:42.734 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:42.764 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:42.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:42.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:42.866 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:42.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:42.895 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:42.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:42.998 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:43.037 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:43.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:43.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:43.139 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:43.162 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:43.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:43.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:43.264 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:43.300 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:43.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:43.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:43.402 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:43.427 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:43.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:43.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:43.529 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:43.530 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:19:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:19:43] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.51.0" 2024-08-17T17:19:43.555 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:43.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:43.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:43.635 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:19:43.635 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:19:43.657 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:43.696 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:43.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:43.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:43.797 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:43.832 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:43.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:43.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:43.933 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:43.972 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:43.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:43.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:43 smithi063 ceph-mon[114756]: pgmap v1389: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 98 B/s rd, 4.2 MiB/s wr, 907 op/s; 25544/309447 objects degraded (8.255%); 3092/309447 objects misplaced (0.999%); 16 MiB/s, 253 objects/s recovering 2024-08-17T17:19:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:43 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 25544/309447 objects degraded (8.255%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:19:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:43 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/190328674' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:19:44.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:43 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:19:44.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:43 smithi063 ceph-mon[109843]: pgmap v1389: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 98 B/s rd, 4.2 MiB/s wr, 907 op/s; 25544/309447 objects degraded (8.255%); 3092/309447 objects misplaced (0.999%); 16 MiB/s, 253 objects/s recovering 2024-08-17T17:19:44.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:43 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 25544/309447 objects degraded (8.255%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:19:44.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:43 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/190328674' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:19:44.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:43 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:19:44.074 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:44.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:43 smithi162 ceph-mon[104668]: pgmap v1389: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 98 B/s rd, 4.2 MiB/s wr, 907 op/s; 25544/309447 objects degraded (8.255%); 3092/309447 objects misplaced (0.999%); 16 MiB/s, 253 objects/s recovering 2024-08-17T17:19:44.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:43 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 25544/309447 objects degraded (8.255%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:19:44.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:43 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/190328674' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:19:44.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:43 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:19:44.111 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:44.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:44.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:44.213 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:44.242 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:44.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:44.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:44.344 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:44.379 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:44.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:44.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:44.481 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:44.503 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:44.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:44.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:44.605 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:44.633 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:44.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:44.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:44.735 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:44.764 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:44.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:44.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:44.865 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:44.892 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:44.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:44.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:44.994 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:45.015 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:45.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:45.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:44 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2024-08-17T17:19:45.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:44 smithi162 ceph-mon[104668]: osdmap e1156: 8 total, 8 up, 8 in 2024-08-17T17:19:45.117 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:45.154 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:45.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:45.256 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:45.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:44 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2024-08-17T17:19:45.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:44 smithi063 ceph-mon[109843]: osdmap e1156: 8 total, 8 up, 8 in 2024-08-17T17:19:45.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:44 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "26"}]': finished 2024-08-17T17:19:45.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:44 smithi063 ceph-mon[114756]: osdmap e1156: 8 total, 8 up, 8 in 2024-08-17T17:19:45.283 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:45.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:45.385 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:45.411 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:45.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:45.513 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:45.542 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:45.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:45.644 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:45.680 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:45.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:45.730 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:45.782 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:45.806 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:45.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.807 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:45.907 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:45.932 INFO:teuthology.orchestra.run.smithi063.stderr:set pool 7 pgp_num to 26 2024-08-17T17:19:45.940 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:45.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:45.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:46.042 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:46.078 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:46.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:46.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:45 smithi162 ceph-mon[104668]: pgmap v1391: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 25278/306210 objects degraded (8.255%); 3092/306210 objects misplaced (1.010%); 14 MiB/s, 231 objects/s recovering 2024-08-17T17:19:46.180 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:46.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:46.216 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:46.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:46.218 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:45 smithi063 ceph-mon[114756]: pgmap v1391: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 25278/306210 objects degraded (8.255%); 3092/306210 objects misplaced (1.010%); 14 MiB/s, 231 objects/s recovering 2024-08-17T17:19:46.218 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:45 smithi063 ceph-mon[109843]: pgmap v1391: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 833 op/s; 25278/306210 objects degraded (8.255%); 3092/306210 objects misplaced (1.010%); 14 MiB/s, 231 objects/s recovering 2024-08-17T17:19:46.317 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:46.342 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:46.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:46.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:46.444 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:46.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:46.472 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:46.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:46.573 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:46.603 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:46.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:46.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:46.705 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:46.735 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:46.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:46.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:46.837 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:46.864 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:46.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:46.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:46.966 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:47.003 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:47.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:47.105 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:47.133 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:47.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:47.235 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:47.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.272 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:47.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:47.373 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:47.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.411 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:47.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:47.512 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:47.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:47 smithi063 ceph-mon[109843]: pgmap v1392: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 25278/302970 objects degraded (8.343%); 3092/302970 objects misplaced (1.021%); 14 MiB/s, 231 objects/s recovering 2024-08-17T17:19:47.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:19:47.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:47.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:47 smithi063 ceph-mon[114756]: pgmap v1392: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 25278/302970 objects degraded (8.343%); 3092/302970 objects misplaced (1.021%); 14 MiB/s, 231 objects/s recovering 2024-08-17T17:19:47.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:19:47.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:47.538 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:47.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:47.639 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:47.666 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:47.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:47.767 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:47.801 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:47.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:47.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:47 smithi162 ceph-mon[104668]: pgmap v1392: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 25278/302970 objects degraded (8.343%); 3092/302970 objects misplaced (1.021%); 14 MiB/s, 231 objects/s recovering 2024-08-17T17:19:47.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:19:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:19:47.903 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:47.965 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:47.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:47.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:48.067 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:48.106 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:48.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:48.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:48.208 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:48.244 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:48.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:48.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:48.346 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:48.377 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:48.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:48.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:48.478 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:48.508 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:48.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:48.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:48.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:48 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 25278/302970 objects degraded (8.343%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:19:48.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:48 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 25278/302970 objects degraded (8.343%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:19:48.610 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:48.651 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:48.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:48.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:48.667 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:19:48.752 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:48.779 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:48.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:48.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:48.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:48 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 25278/302970 objects degraded (8.343%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:19:48.882 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:48.910 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:48.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:48.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:49.012 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:49.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:49.058 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:49.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:49.159 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:49.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:49.199 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:49.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:49.300 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:49.324 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:49.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:49.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:49.426 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:49.464 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:49.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:49.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:49.565 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:49.596 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:49.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:49.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:49.698 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:49.725 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:49.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:49.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:49.774 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:49 smithi063 ceph-mon[109843]: pgmap v1393: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 24107/293790 objects degraded (8.206%); 2978/293790 objects misplaced (1.014%); 15 MiB/s, 241 objects/s recovering 2024-08-17T17:19:49.774 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:49 smithi063 ceph-mon[114756]: pgmap v1393: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 24107/293790 objects degraded (8.206%); 2978/293790 objects misplaced (1.014%); 15 MiB/s, 241 objects/s recovering 2024-08-17T17:19:49.827 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:49.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:49 smithi162 ceph-mon[104668]: pgmap v1393: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 24107/293790 objects degraded (8.206%); 2978/293790 objects misplaced (1.014%); 15 MiB/s, 241 objects/s recovering 2024-08-17T17:19:49.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:49.871 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:49.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:49.972 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:49.998 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:49.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:49.999 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:50.100 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:50.155 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:50.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:50.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:50.257 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:50.288 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:50.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:50.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:50.389 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:50.436 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:50.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:50.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:50.539 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:50.564 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:50.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:50.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:50.666 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:50.709 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:50.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:50.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:50.810 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:50.834 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:50.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:50.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:50.935 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:50.961 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:50.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:50.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:50.992 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:19:50.992 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:19:50.992 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:19:50.992 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:19:50.992 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:19:50.992 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:19:50.992 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:19:50.992 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:19:50.993 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:19:50.993 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:19:50.993 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:19:50.993 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:19:50.993 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:19:50.993 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:19:50.993 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:19:50.993 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:19:51.064 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:51.086 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:51.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:51.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:51.188 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:51.214 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:51.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:51.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:51.315 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:51.346 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:51.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:51.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:51.378 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:51.394 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:19:51.448 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:51.490 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:51.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:51.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:51.594 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:51.620 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:51.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:51.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:51.722 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:51.748 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:51.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:51.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:51.752 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:51 smithi063 ceph-mon[109843]: pgmap v1394: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 24107/290718 objects degraded (8.292%); 2978/290718 objects misplaced (1.024%); 15 MiB/s, 241 objects/s recovering 2024-08-17T17:19:51.752 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:51 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3623637391' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:51.752 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:51 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1807351901' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:19:51.752 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:51 smithi063 ceph-mon[114756]: pgmap v1394: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 24107/290718 objects degraded (8.292%); 2978/290718 objects misplaced (1.024%); 15 MiB/s, 241 objects/s recovering 2024-08-17T17:19:51.752 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:51 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3623637391' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:51.752 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:51 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1807351901' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:19:51.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:51 smithi162 ceph-mon[104668]: pgmap v1394: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 24107/290718 objects degraded (8.292%); 2978/290718 objects misplaced (1.024%); 15 MiB/s, 241 objects/s recovering 2024-08-17T17:19:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:51 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3623637391' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:19:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:51 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1807351901' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:19:51.849 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:51.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:51.893 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:51.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:51.994 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:52.037 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:52.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:52.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:52.139 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:52.180 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:52.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:52.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:52.283 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:52.320 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:52.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:52.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:52.422 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:52.439 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:52 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:19:52.440 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:52 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1807351901' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:19:52.440 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:52 smithi063 ceph-mon[114756]: osdmap e1157: 8 total, 8 up, 8 in 2024-08-17T17:19:52.440 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:52 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:19:52.440 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:52 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1807351901' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:19:52.440 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:52 smithi063 ceph-mon[109843]: osdmap e1157: 8 total, 8 up, 8 in 2024-08-17T17:19:52.463 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:52.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:52.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:52.565 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:52.592 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:52.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:52.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:52.693 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:52.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:52.726 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:52.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:52.827 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:52.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:52 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:19:52.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:52 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1807351901' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:19:52.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:52 smithi162 ceph-mon[104668]: osdmap e1157: 8 total, 8 up, 8 in 2024-08-17T17:19:52.853 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:52.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:52.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:52.955 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:52.965 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:19:52.965 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:19:52.965 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-08-17T17:19:52.966 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:19:52.997 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:52.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:52.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:53.099 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:53.126 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:53.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:53.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:53.230 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:53.278 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:53.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:53.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:53.380 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:53.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:53.424 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:53.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:53.525 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:53.567 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:53.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:53.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:53.584 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:53 smithi063 ceph-mon[109843]: pgmap v1396: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 23344/284595 objects degraded (8.203%); 2858/284595 objects misplaced (1.004%); 16 MiB/s, 258 objects/s recovering 2024-08-17T17:19:53.584 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:53 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 23344/284595 objects degraded (8.203%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:19:53.584 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:19:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:19:53] "GET /metrics HTTP/1.1" 200 36437 "" "Prometheus/2.51.0" 2024-08-17T17:19:53.585 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:53 smithi063 ceph-mon[114756]: pgmap v1396: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 23344/284595 objects degraded (8.203%); 2858/284595 objects misplaced (1.004%); 16 MiB/s, 258 objects/s recovering 2024-08-17T17:19:53.585 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:53 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 23344/284595 objects degraded (8.203%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:19:53.669 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:53.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:53.704 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:53.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:53.805 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:53.833 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:53.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:53.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:53.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:53 smithi162 ceph-mon[104668]: pgmap v1396: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 23344/284595 objects degraded (8.203%); 2858/284595 objects misplaced (1.004%); 16 MiB/s, 258 objects/s recovering 2024-08-17T17:19:53.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:53 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 23344/284595 objects degraded (8.203%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:19:53.935 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:53.961 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:53.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:53.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:53.989 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:54.063 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:54.129 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:54.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:54.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:54.227 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:19:54.231 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:54.254 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:54.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:54.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:54.356 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:54.381 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:54.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:54.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:54.482 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:54.543 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:54.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:54.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:54.645 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:54.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:54.685 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:54.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:54.786 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:54.822 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:54.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:54.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:54.908 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:19:54.915 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:19:54.915 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:19:54.915 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:19:54.915 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:19:54.915 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.915 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35809 6.f 6:f6d7b118:::benchmark_data_smithi063_160268_object35808:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:19:54.915 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.876618+0000", 2024-08-17T17:19:54.915 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 374.03254142200001, 2024-08-17T17:19:54.916 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14764386800000001, 2024-08-17T17:19:54.916 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:54.916 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:54.916 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:54.916 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:54.916 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:19:54.916 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:19:54.916 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35809 2024-08-17T17:19:54.916 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.916 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:54.916 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.916 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:54.916 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876618+0000", 2024-08-17T17:19:54.916 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.917 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.917 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.917 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:54.917 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876618+0000", 2024-08-17T17:19:54.917 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.917 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.917 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.917 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:54.917 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876620+0000", 2024-08-17T17:19:54.917 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.795e-06 2024-08-17T17:19:54.917 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.917 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.917 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:54.917 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876643+0000", 2024-08-17T17:19:54.918 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.296e-05 2024-08-17T17:19:54.918 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.918 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.918 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:54.918 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876654+0000", 2024-08-17T17:19:54.918 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.13e-05 2024-08-17T17:19:54.918 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.918 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.918 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:54.918 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876661+0000", 2024-08-17T17:19:54.918 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0269999999999998e-06 2024-08-17T17:19:54.918 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.918 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.919 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.919 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876712+0000", 2024-08-17T17:19:54.919 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1388999999999999e-05 2024-08-17T17:19:54.919 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.919 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.919 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:54.919 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876826+0000", 2024-08-17T17:19:54.919 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000113377 2024-08-17T17:19:54.919 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.919 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.919 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:19:54.919 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876960+0000", 2024-08-17T17:19:54.919 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000134103 2024-08-17T17:19:54.919 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.920 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.920 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.920 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.879644+0000", 2024-08-17T17:19:54.920 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0026843520000000001 2024-08-17T17:19:54.920 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.920 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.920 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:54.920 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.879678+0000", 2024-08-17T17:19:54.920 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3433999999999999e-05 2024-08-17T17:19:54.920 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.920 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.920 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.921 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.023992+0000", 2024-08-17T17:19:54.921 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14431400899999999 2024-08-17T17:19:54.921 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.921 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.921 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:54.921 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.024099+0000", 2024-08-17T17:19:54.921 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000107754 2024-08-17T17:19:54.921 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.921 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.921 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:54.921 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.024262+0000", 2024-08-17T17:19:54.921 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000162368 2024-08-17T17:19:54.921 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.921 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:54.921 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.922 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.922 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.922 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35811 6.f 6:f8be1079:::benchmark_data_smithi063_160268_object35810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:19:54.922 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.877384+0000", 2024-08-17T17:19:54.922 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 374.03177500800001, 2024-08-17T17:19:54.922 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.15051810500000001, 2024-08-17T17:19:54.922 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:54.922 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:54.922 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:54.922 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:54.922 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:19:54.922 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:19:54.922 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35811 2024-08-17T17:19:54.923 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.923 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:54.923 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.923 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:54.923 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877384+0000", 2024-08-17T17:19:54.923 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.923 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.923 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.923 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:54.923 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877384+0000", 2024-08-17T17:19:54.923 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.923 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.923 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.924 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:54.924 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877385+0000", 2024-08-17T17:19:54.924 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.1800000000000004e-07 2024-08-17T17:19:54.924 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.924 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.924 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:54.924 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877404+0000", 2024-08-17T17:19:54.924 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8766e-05 2024-08-17T17:19:54.924 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.924 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.924 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:54.924 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877416+0000", 2024-08-17T17:19:54.925 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1770000000000001e-05 2024-08-17T17:19:54.925 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.925 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.925 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:54.925 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877423+0000", 2024-08-17T17:19:54.925 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0049999999999998e-06 2024-08-17T17:19:54.925 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.925 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.925 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.925 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878297+0000", 2024-08-17T17:19:54.925 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00087477799999999999 2024-08-17T17:19:54.925 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.926 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.926 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:54.926 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878431+0000", 2024-08-17T17:19:54.926 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013333900000000001 2024-08-17T17:19:54.926 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.926 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.926 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:19:54.926 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878556+0000", 2024-08-17T17:19:54.926 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000124764 2024-08-17T17:19:54.926 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.926 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.926 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:54.927 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880196+0000", 2024-08-17T17:19:54.927 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0016405949999999999 2024-08-17T17:19:54.927 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.927 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.927 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.927 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.881084+0000", 2024-08-17T17:19:54.927 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00088833800000000002 2024-08-17T17:19:54.927 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.927 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.927 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.927 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027510+0000", 2024-08-17T17:19:54.927 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.146425627 2024-08-17T17:19:54.927 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.928 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.928 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:54.928 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027847+0000", 2024-08-17T17:19:54.928 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000336397 2024-08-17T17:19:54.928 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.928 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.928 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:54.928 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027902+0000", 2024-08-17T17:19:54.928 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5807999999999998e-05 2024-08-17T17:19:54.928 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.928 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:54.928 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.928 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.929 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.929 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35820 6.f 6:f1fc3e55:::benchmark_data_smithi063_160268_object35819:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:19:54.929 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.882503+0000", 2024-08-17T17:19:54.929 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 374.02665595399998, 2024-08-17T17:19:54.929 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14575781700000001, 2024-08-17T17:19:54.929 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:54.929 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:54.929 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:54.929 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:54.929 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:19:54.929 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:19:54.929 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35820 2024-08-17T17:19:54.930 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.930 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:54.930 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.930 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:54.930 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882503+0000", 2024-08-17T17:19:54.930 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.930 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.930 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.930 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:54.930 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882503+0000", 2024-08-17T17:19:54.930 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.930 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.931 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.931 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:54.931 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882505+0000", 2024-08-17T17:19:54.931 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.556e-06 2024-08-17T17:19:54.931 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.931 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.931 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:54.931 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882528+0000", 2024-08-17T17:19:54.931 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3295999999999999e-05 2024-08-17T17:19:54.931 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.931 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.931 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:54.931 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882540+0000", 2024-08-17T17:19:54.932 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2099000000000001e-05 2024-08-17T17:19:54.932 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.932 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.932 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:54.932 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882548+0000", 2024-08-17T17:19:54.932 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7049999999999999e-06 2024-08-17T17:19:54.932 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.932 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.932 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.932 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882591+0000", 2024-08-17T17:19:54.932 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2775999999999998e-05 2024-08-17T17:19:54.932 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.932 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.932 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:54.933 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882678+0000", 2024-08-17T17:19:54.933 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7777999999999999e-05 2024-08-17T17:19:54.933 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.933 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.933 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:19:54.933 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882761+0000", 2024-08-17T17:19:54.933 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2347000000000003e-05 2024-08-17T17:19:54.933 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.933 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.933 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.933 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.023259+0000", 2024-08-17T17:19:54.933 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14049784100000001 2024-08-17T17:19:54.933 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.933 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.933 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:54.934 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.023430+0000", 2024-08-17T17:19:54.934 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000171194 2024-08-17T17:19:54.934 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.934 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.934 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.934 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.028113+0000", 2024-08-17T17:19:54.934 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0046829660000000002 2024-08-17T17:19:54.934 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.934 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.934 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:54.934 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.028217+0000", 2024-08-17T17:19:54.934 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000104621 2024-08-17T17:19:54.935 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.935 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.935 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:54.935 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.028261+0000", 2024-08-17T17:19:54.935 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3637999999999998e-05 2024-08-17T17:19:54.935 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.935 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:54.935 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.935 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.935 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.935 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:35818 6.3 e1093/1086 6:ccea9449:::benchmark_data_smithi063_160268_object35817:head v 1093'2214, mlcod=1093'2211)", 2024-08-17T17:19:54.935 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.882944+0000", 2024-08-17T17:19:54.935 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 374.026214998, 2024-08-17T17:19:54.936 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.13971906100000001, 2024-08-17T17:19:54.936 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:54.936 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:54.936 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:54.936 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:54.936 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.936 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:54.936 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882944+0000", 2024-08-17T17:19:54.936 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.936 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.936 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.936 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:54.936 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882944+0000", 2024-08-17T17:19:54.937 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.937 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.937 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.937 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:54.937 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882945+0000", 2024-08-17T17:19:54.937 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.111e-06 2024-08-17T17:19:54.937 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.937 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.937 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:54.937 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882976+0000", 2024-08-17T17:19:54.937 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0332999999999999e-05 2024-08-17T17:19:54.937 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.937 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.937 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:54.938 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020295+0000", 2024-08-17T17:19:54.938 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.137318948 2024-08-17T17:19:54.938 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.938 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.938 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:54.938 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020308+0000", 2024-08-17T17:19:54.938 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3488e-05 2024-08-17T17:19:54.938 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.938 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.938 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.938 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020379+0000", 2024-08-17T17:19:54.938 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1173000000000004e-05 2024-08-17T17:19:54.939 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.939 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.939 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:54.939 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020825+0000", 2024-08-17T17:19:54.939 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00044561600000000002 2024-08-17T17:19:54.939 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.939 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.939 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:54.939 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.022624+0000", 2024-08-17T17:19:54.939 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001799599 2024-08-17T17:19:54.939 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.939 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.939 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:54.939 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.022663+0000", 2024-08-17T17:19:54.940 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8792999999999999e-05 2024-08-17T17:19:54.940 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.940 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:54.940 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.940 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.940 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.940 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:28732 7.e 7:7ccf22b7:::benchmark_data_smithi063_196002_object28731:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:19:54.940 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:08.352987+0000", 2024-08-17T17:19:54.940 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 106.55617223, 2024-08-17T17:19:54.940 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.099912441000000005, 2024-08-17T17:19:54.940 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:54.940 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:54.940 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:54.941 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:54.941 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:19:54.941 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:19:54.941 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 28732 2024-08-17T17:19:54.941 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.941 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:54.941 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.941 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:54.941 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:08.352987+0000", 2024-08-17T17:19:54.941 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.941 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.941 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.941 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:54.942 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:08.352987+0000", 2024-08-17T17:19:54.942 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.942 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.942 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.942 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:54.942 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:08.352988+0000", 2024-08-17T17:19:54.942 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7199999999999998e-07 2024-08-17T17:19:54.942 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.942 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.942 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:54.942 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:08.353008+0000", 2024-08-17T17:19:54.942 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0213000000000001e-05 2024-08-17T17:19:54.942 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.943 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.943 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:54.943 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:08.353026+0000", 2024-08-17T17:19:54.943 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7841000000000001e-05 2024-08-17T17:19:54.943 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.943 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.943 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:54.943 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:08.353032+0000", 2024-08-17T17:19:54.943 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4370000000000004e-06 2024-08-17T17:19:54.943 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.943 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.943 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.943 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:08.353146+0000", 2024-08-17T17:19:54.944 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011349900000000001 2024-08-17T17:19:54.944 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.944 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.944 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:54.944 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:08.353270+0000", 2024-08-17T17:19:54.944 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:54.945 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012420899999999999 2024-08-17T17:19:54.945 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.945 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.945 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:19:54.946 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:08.353411+0000", 2024-08-17T17:19:54.946 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014109800000000001 2024-08-17T17:19:54.946 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.946 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.946 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.946 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:08.357074+0000", 2024-08-17T17:19:54.946 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0036633619999999999 2024-08-17T17:19:54.946 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.946 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.946 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:54.946 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:08.358548+0000", 2024-08-17T17:19:54.946 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0014734220000000001 2024-08-17T17:19:54.947 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.947 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.947 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.947 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:08.452797+0000", 2024-08-17T17:19:54.947 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.094248888000000003 2024-08-17T17:19:54.947 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.947 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.947 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:54.947 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:08.452856+0000", 2024-08-17T17:19:54.947 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8922000000000003e-05 2024-08-17T17:19:54.947 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.947 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.947 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:54.948 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:08.452899+0000", 2024-08-17T17:19:54.948 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3778e-05 2024-08-17T17:19:54.948 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.948 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:54.948 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.948 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.948 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.948 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:30164 7.e 7:7bec00c1:::benchmark_data_smithi063_196002_object30163:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:19:54.948 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:09.180746+0000", 2024-08-17T17:19:54.948 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 105.72841296599999, 2024-08-17T17:19:54.948 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.099371275999999994, 2024-08-17T17:19:54.948 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:54.948 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:54.948 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:54.949 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:54.949 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:19:54.949 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:19:54.949 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 30164 2024-08-17T17:19:54.949 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.949 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:54.949 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.949 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:54.949 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:09.180746+0000", 2024-08-17T17:19:54.949 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.949 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.949 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.950 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:54.950 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:09.180746+0000", 2024-08-17T17:19:54.950 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.950 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.950 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.950 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:54.950 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:09.180748+0000", 2024-08-17T17:19:54.950 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4929999999999999e-06 2024-08-17T17:19:54.950 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.950 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.950 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:54.950 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:09.180773+0000", 2024-08-17T17:19:54.951 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5667e-05 2024-08-17T17:19:54.951 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.951 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.951 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:54.951 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:09.180786+0000", 2024-08-17T17:19:54.951 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2597e-05 2024-08-17T17:19:54.951 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.951 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.951 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:54.951 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:09.180791+0000", 2024-08-17T17:19:54.951 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4399999999999996e-06 2024-08-17T17:19:54.951 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.951 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.952 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.952 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:09.180846+0000", 2024-08-17T17:19:54.952 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4409e-05 2024-08-17T17:19:54.952 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.952 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.952 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:54.952 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:09.181075+0000", 2024-08-17T17:19:54.952 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00022951300000000001 2024-08-17T17:19:54.952 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.952 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.952 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:19:54.952 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:09.181249+0000", 2024-08-17T17:19:54.952 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000173253 2024-08-17T17:19:54.953 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.953 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.953 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:54.953 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:09.182374+0000", 2024-08-17T17:19:54.953 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001125832 2024-08-17T17:19:54.953 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.953 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.953 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.953 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:09.182916+0000", 2024-08-17T17:19:54.953 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00054142700000000003 2024-08-17T17:19:54.953 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.953 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.954 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.954 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:09.280021+0000", 2024-08-17T17:19:54.954 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.097105075999999999 2024-08-17T17:19:54.954 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.954 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.954 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:54.954 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:09.280078+0000", 2024-08-17T17:19:54.954 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6814999999999997e-05 2024-08-17T17:19:54.954 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.954 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.954 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:54.954 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:09.280118+0000", 2024-08-17T17:19:54.955 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9753999999999999e-05 2024-08-17T17:19:54.955 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.955 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:54.955 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.955 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.955 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.955 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:32536 7.8 7:1e6dc57b:::benchmark_data_smithi063_196002_object32535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:19:54.955 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:10.423425+0000", 2024-08-17T17:19:54.955 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 104.485733823, 2024-08-17T17:19:54.955 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.140115024, 2024-08-17T17:19:54.955 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:54.955 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:54.956 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:54.956 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:54.956 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:19:54.956 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:19:54.956 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 32536 2024-08-17T17:19:54.956 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.956 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:54.956 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.956 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:54.956 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.423425+0000", 2024-08-17T17:19:54.956 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.956 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.956 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.957 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:54.957 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.423425+0000", 2024-08-17T17:19:54.957 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.957 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.957 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.957 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:54.957 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.423426+0000", 2024-08-17T17:19:54.957 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3e-07 2024-08-17T17:19:54.957 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.957 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.957 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:54.957 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.423464+0000", 2024-08-17T17:19:54.957 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7622e-05 2024-08-17T17:19:54.958 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.958 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.958 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:54.958 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.423472+0000", 2024-08-17T17:19:54.958 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7069999999999998e-06 2024-08-17T17:19:54.958 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.958 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.958 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:54.958 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.423477+0000", 2024-08-17T17:19:54.958 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8289999999999998e-06 2024-08-17T17:19:54.958 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.958 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.959 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.959 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.424138+0000", 2024-08-17T17:19:54.959 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00066053200000000002 2024-08-17T17:19:54.959 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.959 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.959 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:54.959 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.424247+0000", 2024-08-17T17:19:54.959 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010904300000000001 2024-08-17T17:19:54.959 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.959 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.959 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:19:54.959 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.424366+0000", 2024-08-17T17:19:54.959 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011934600000000001 2024-08-17T17:19:54.959 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.960 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.960 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:54.960 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425352+0000", 2024-08-17T17:19:54.960 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00098572599999999992 2024-08-17T17:19:54.960 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.960 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.960 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.960 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563349+0000", 2024-08-17T17:19:54.960 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.13799671599999999 2024-08-17T17:19:54.960 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.960 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.960 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.961 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563391+0000", 2024-08-17T17:19:54.961 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2231000000000003e-05 2024-08-17T17:19:54.961 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.961 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.961 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:54.961 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563496+0000", 2024-08-17T17:19:54.961 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010518699999999999 2024-08-17T17:19:54.961 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.961 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.961 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:54.961 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563540+0000", 2024-08-17T17:19:54.961 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4354999999999998e-05 2024-08-17T17:19:54.961 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.961 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:54.962 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.962 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.962 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.962 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:32540 7.8 7:1eb42e3f:::benchmark_data_smithi063_196002_object32539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:19:54.962 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:10.425834+0000", 2024-08-17T17:19:54.962 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 104.48332533599999, 2024-08-17T17:19:54.962 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.13808171999999999, 2024-08-17T17:19:54.962 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:54.962 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:54.962 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:54.962 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:54.962 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:19:54.963 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:19:54.963 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 32540 2024-08-17T17:19:54.963 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.963 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:54.963 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.963 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:54.963 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425834+0000", 2024-08-17T17:19:54.963 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.963 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.963 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.963 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:54.963 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425834+0000", 2024-08-17T17:19:54.963 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.963 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.964 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.964 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:54.964 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425835+0000", 2024-08-17T17:19:54.964 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.2399999999999996e-07 2024-08-17T17:19:54.964 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.964 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.964 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:54.964 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425854+0000", 2024-08-17T17:19:54.964 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9689000000000001e-05 2024-08-17T17:19:54.964 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.964 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.964 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:54.965 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425864+0000", 2024-08-17T17:19:54.965 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.2480000000000002e-06 2024-08-17T17:19:54.965 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.965 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.965 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:54.965 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425869+0000", 2024-08-17T17:19:54.965 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0719999999999999e-06 2024-08-17T17:19:54.965 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.965 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.965 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.965 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425996+0000", 2024-08-17T17:19:54.965 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012675499999999999 2024-08-17T17:19:54.965 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.966 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.966 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:54.966 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426112+0000", 2024-08-17T17:19:54.966 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000116519 2024-08-17T17:19:54.966 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.966 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.966 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:19:54.966 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426221+0000", 2024-08-17T17:19:54.966 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000108554 2024-08-17T17:19:54.966 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.966 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.966 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:54.967 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563360+0000", 2024-08-17T17:19:54.967 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.137139071 2024-08-17T17:19:54.967 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.967 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.967 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.967 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563579+0000", 2024-08-17T17:19:54.967 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00021886099999999999 2024-08-17T17:19:54.967 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.967 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.969 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.969 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563817+0000", 2024-08-17T17:19:54.969 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00023799699999999999 2024-08-17T17:19:54.969 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.969 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.970 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:54.970 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563876+0000", 2024-08-17T17:19:54.970 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9166999999999998e-05 2024-08-17T17:19:54.970 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.970 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.970 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:54.970 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563916+0000", 2024-08-17T17:19:54.970 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9863000000000001e-05 2024-08-17T17:19:54.970 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.970 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:54.970 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.970 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.971 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.971 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.34965.0:32536 7.8 e1133/1129 ondisk, result = 0)", 2024-08-17T17:19:54.971 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:10.426058+0000", 2024-08-17T17:19:54.971 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 104.48310118400001, 2024-08-17T17:19:54.971 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.13730266199999999, 2024-08-17T17:19:54.971 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:54.971 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:54.971 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:19:54.971 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:54.971 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.971 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:54.971 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426058+0000", 2024-08-17T17:19:54.971 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.971 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.972 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.972 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:54.972 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426058+0000", 2024-08-17T17:19:54.972 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.972 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.972 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.972 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:54.972 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426059+0000", 2024-08-17T17:19:54.972 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8699999999999995e-07 2024-08-17T17:19:54.972 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.972 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.972 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:54.972 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426066+0000", 2024-08-17T17:19:54.972 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2590000000000003e-06 2024-08-17T17:19:54.973 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.973 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.973 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:54.973 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426074+0000", 2024-08-17T17:19:54.973 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4309999999999994e-06 2024-08-17T17:19:54.973 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.973 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.973 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:54.973 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426080+0000", 2024-08-17T17:19:54.973 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9970000000000004e-06 2024-08-17T17:19:54.973 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.973 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.973 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.973 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563335+0000", 2024-08-17T17:19:54.974 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.13725435599999999 2024-08-17T17:19:54.974 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.974 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.974 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:54.974 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563343+0000", 2024-08-17T17:19:54.974 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0760000000000003e-06 2024-08-17T17:19:54.974 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.974 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.974 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:54.974 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563361+0000", 2024-08-17T17:19:54.974 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7955999999999999e-05 2024-08-17T17:19:54.974 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.975 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:54.975 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.975 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.975 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.975 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.34965.0:32536 7.8 e1133/1129 ondisk, result = 0)", 2024-08-17T17:19:54.975 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:10.426119+0000", 2024-08-17T17:19:54.975 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 104.483040669, 2024-08-17T17:19:54.975 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.137427676, 2024-08-17T17:19:54.975 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:54.975 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:54.975 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:19:54.975 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:54.975 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.976 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:54.976 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426119+0000", 2024-08-17T17:19:54.976 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.976 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.976 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.976 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:54.976 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426119+0000", 2024-08-17T17:19:54.976 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.976 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.976 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.976 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:54.976 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426119+0000", 2024-08-17T17:19:54.976 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5999999999999999e-07 2024-08-17T17:19:54.977 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.977 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.977 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:54.977 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426122+0000", 2024-08-17T17:19:54.977 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.9299999999999999e-06 2024-08-17T17:19:54.977 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.977 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.977 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:54.977 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426129+0000", 2024-08-17T17:19:54.977 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2690000000000002e-06 2024-08-17T17:19:54.977 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.977 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.977 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:54.977 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426134+0000", 2024-08-17T17:19:54.978 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1919999999999996e-06 2024-08-17T17:19:54.978 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.978 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.978 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.978 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563381+0000", 2024-08-17T17:19:54.978 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.137246487 2024-08-17T17:19:54.978 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.978 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.978 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:54.978 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563387+0000", 2024-08-17T17:19:54.978 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7420000000000003e-06 2024-08-17T17:19:54.978 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.978 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.979 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:54.979 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563546+0000", 2024-08-17T17:19:54.979 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015959599999999999 2024-08-17T17:19:54.979 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.979 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:54.979 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.979 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.979 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.979 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:35583 7.e 7:70113c18:::benchmark_data_smithi063_196002_object35582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:19:54.979 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:12.287590+0000", 2024-08-17T17:19:54.979 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 102.621569428, 2024-08-17T17:19:54.979 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.099221852999999999, 2024-08-17T17:19:54.979 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:54.979 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:54.980 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:54.980 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:54.980 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:19:54.980 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:19:54.980 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35583 2024-08-17T17:19:54.980 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.980 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:54.980 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.980 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:54.980 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:12.287590+0000", 2024-08-17T17:19:54.980 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.980 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.981 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.981 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:54.981 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:12.287590+0000", 2024-08-17T17:19:54.981 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.981 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.981 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.981 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:54.981 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:12.287591+0000", 2024-08-17T17:19:54.981 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1570000000000001e-06 2024-08-17T17:19:54.981 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.981 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.981 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:54.981 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:12.287638+0000", 2024-08-17T17:19:54.982 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7178999999999997e-05 2024-08-17T17:19:54.982 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.982 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.982 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:54.982 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:12.287648+0000", 2024-08-17T17:19:54.982 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9830000000000008e-06 2024-08-17T17:19:54.982 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.982 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.982 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:54.982 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:12.287655+0000", 2024-08-17T17:19:54.982 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4579999999999998e-06 2024-08-17T17:19:54.982 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.982 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.983 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.983 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:12.292647+0000", 2024-08-17T17:19:54.983 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0049920069999999997 2024-08-17T17:19:54.983 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.983 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.983 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:54.983 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:12.292756+0000", 2024-08-17T17:19:54.983 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010958 2024-08-17T17:19:54.983 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.983 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.983 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:19:54.983 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:12.292892+0000", 2024-08-17T17:19:54.983 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013582500000000001 2024-08-17T17:19:54.984 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.984 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.984 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:54.984 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:12.295987+0000", 2024-08-17T17:19:54.984 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.003094884 2024-08-17T17:19:54.984 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.984 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.984 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.984 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:12.297098+0000", 2024-08-17T17:19:54.984 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001111555 2024-08-17T17:19:54.984 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.984 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.984 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.985 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:12.386691+0000", 2024-08-17T17:19:54.985 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.089592771000000002 2024-08-17T17:19:54.985 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.985 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.985 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:54.985 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:12.386758+0000", 2024-08-17T17:19:54.985 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6680000000000005e-05 2024-08-17T17:19:54.985 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.985 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.985 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:54.985 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:12.386812+0000", 2024-08-17T17:19:54.985 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3773999999999997e-05 2024-08-17T17:19:54.986 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.986 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:54.986 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.986 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.986 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.986 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:65727 7.e 7:7466d76d:::benchmark_data_smithi063_196002_object65726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-08-17T17:19:54.986 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:30.516551+0000", 2024-08-17T17:19:54.986 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 84.392608679999995, 2024-08-17T17:19:54.986 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5070886789999998, 2024-08-17T17:19:54.986 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:54.986 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:54.986 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:19:54.986 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:54.987 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:19:54.987 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:19:54.987 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 65727 2024-08-17T17:19:54.987 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.987 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:54.987 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.987 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:54.987 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516551+0000", 2024-08-17T17:19:54.987 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.987 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.987 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.987 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:54.987 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516551+0000", 2024-08-17T17:19:54.987 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.988 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.988 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.988 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:54.988 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516587+0000", 2024-08-17T17:19:54.988 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6622000000000003e-05 2024-08-17T17:19:54.988 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.988 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.988 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:54.988 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516612+0000", 2024-08-17T17:19:54.988 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4496999999999999e-05 2024-08-17T17:19:54.988 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.988 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.988 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:54.988 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516627+0000", 2024-08-17T17:19:54.989 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5061e-05 2024-08-17T17:19:54.989 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.989 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.989 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:54.989 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516634+0000", 2024-08-17T17:19:54.989 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3980000000000002e-06 2024-08-17T17:19:54.989 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.989 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.989 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.989 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516708+0000", 2024-08-17T17:19:54.989 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3986000000000006e-05 2024-08-17T17:19:54.989 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.989 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.989 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:54.990 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516845+0000", 2024-08-17T17:19:54.990 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013681699999999999 2024-08-17T17:19:54.990 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.990 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.990 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:19:54.990 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516985+0000", 2024-08-17T17:19:54.990 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014038000000000001 2024-08-17T17:19:54.990 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.990 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.990 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:54.990 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.519528+0000", 2024-08-17T17:19:54.990 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002542515 2024-08-17T17:19:54.990 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.991 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.991 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.991 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.520307+0000", 2024-08-17T17:19:54.991 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00077905200000000002 2024-08-17T17:19:54.991 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.991 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.991 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.991 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.535671+0000", 2024-08-17T17:19:54.991 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.015363751 2024-08-17T17:19:54.991 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.991 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.992 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:19:54.992 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.535694+0000", 2024-08-17T17:19:54.992 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3139e-05 2024-08-17T17:19:54.993 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.993 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.993 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.993 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:31.526159+0000", 2024-08-17T17:19:54.993 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.99046498699999996 2024-08-17T17:19:54.993 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.993 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.993 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:19:54.993 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:31.526179+0000", 2024-08-17T17:19:54.993 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0432e-05 2024-08-17T17:19:54.993 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.993 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.993 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.993 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:33.023535+0000", 2024-08-17T17:19:54.994 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.497355684 2024-08-17T17:19:54.994 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.994 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.994 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:54.994 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:33.023639+0000", 2024-08-17T17:19:54.994 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010435800000000001 2024-08-17T17:19:54.994 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.994 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:54.994 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:54.994 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.994 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.994 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:88377 7.e 7:72a20e0d:::benchmark_data_smithi063_196002_object88376:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1150)", 2024-08-17T17:19:54.995 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.650876+0000", 2024-08-17T17:19:54.995 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 54.258282772999998, 2024-08-17T17:19:54.995 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.28977614600000001, 2024-08-17T17:19:54.995 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:54.995 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:54.995 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:54.995 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:54.995 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:19:54.995 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:19:54.995 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 88377 2024-08-17T17:19:54.995 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.995 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:54.995 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.996 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:54.996 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650876+0000", 2024-08-17T17:19:54.996 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.996 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.996 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.996 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:54.996 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650876+0000", 2024-08-17T17:19:54.996 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:54.996 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.996 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.996 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:54.996 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650879+0000", 2024-08-17T17:19:54.996 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.187e-06 2024-08-17T17:19:54.996 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.997 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.997 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:54.997 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650905+0000", 2024-08-17T17:19:54.997 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6446999999999999e-05 2024-08-17T17:19:54.997 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.997 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.997 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:54.997 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650936+0000", 2024-08-17T17:19:54.997 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1050999999999998e-05 2024-08-17T17:19:54.997 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.997 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.997 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:54.997 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650945+0000", 2024-08-17T17:19:54.998 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8400000000000001e-06 2024-08-17T17:19:54.998 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.998 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.998 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:54.998 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651123+0000", 2024-08-17T17:19:54.998 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00017799800000000001 2024-08-17T17:19:54.998 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.998 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.998 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:54.998 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651483+0000", 2024-08-17T17:19:54.998 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00035978699999999998 2024-08-17T17:19:54.998 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.998 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.999 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:19:54.999 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.671281+0000", 2024-08-17T17:19:54.999 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.019797911000000001 2024-08-17T17:19:54.999 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.999 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.999 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:54.999 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.705325+0000", 2024-08-17T17:19:54.999 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.034044498999999999 2024-08-17T17:19:54.999 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:54.999 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:54.999 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:54.999 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.760144+0000", 2024-08-17T17:19:55.000 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.054818791999999998 2024-08-17T17:19:55.000 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.000 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.000 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:55.000 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.937269+0000", 2024-08-17T17:19:55.000 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17712487399999999 2024-08-17T17:19:55.000 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.000 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.000 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:55.000 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.937348+0000", 2024-08-17T17:19:55.000 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8988999999999999e-05 2024-08-17T17:19:55.000 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.000 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.000 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:55.001 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.940653+0000", 2024-08-17T17:19:55.001 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0033047710000000002 2024-08-17T17:19:55.001 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.001 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:55.001 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.001 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.001 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.001 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88375 7.2 e1150/1129 7:44faffa9:::benchmark_data_smithi063_196002_object88374:head v 1150'5570, mlcod=1150'5567)", 2024-08-17T17:19:55.001 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.651624+0000", 2024-08-17T17:19:55.001 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 54.257535095000001, 2024-08-17T17:19:55.001 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.16577204300000001, 2024-08-17T17:19:55.001 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:55.001 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:55.001 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:19:55.002 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:55.002 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.002 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:55.002 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651624+0000", 2024-08-17T17:19:55.002 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:55.002 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.002 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.002 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:55.002 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651624+0000", 2024-08-17T17:19:55.002 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:55.002 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.002 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.002 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:55.002 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651625+0000", 2024-08-17T17:19:55.003 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3099999999999997e-07 2024-08-17T17:19:55.003 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.003 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.003 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:55.003 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651676+0000", 2024-08-17T17:19:55.003 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1603e-05 2024-08-17T17:19:55.003 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.003 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.003 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:55.003 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651698+0000", 2024-08-17T17:19:55.003 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1809000000000001e-05 2024-08-17T17:19:55.003 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.003 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.004 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:55.004 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651717+0000", 2024-08-17T17:19:55.004 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8516999999999999e-05 2024-08-17T17:19:55.004 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.004 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.004 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:55.004 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651792+0000", 2024-08-17T17:19:55.004 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5512000000000005e-05 2024-08-17T17:19:55.004 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.004 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.004 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:55.004 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651821+0000", 2024-08-17T17:19:55.004 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.9067000000000001e-05 2024-08-17T17:19:55.005 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.005 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.005 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:55.005 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.817396+0000", 2024-08-17T17:19:55.005 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.16557490399999999 2024-08-17T17:19:55.005 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.005 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:55.005 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.005 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.005 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.005 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:88379 7.e 7:77a5a758:::benchmark_data_smithi063_196002_object88378:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1150)", 2024-08-17T17:19:55.005 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.696360+0000", 2024-08-17T17:19:55.005 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 54.21279964, 2024-08-17T17:19:55.006 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.24823995800000001, 2024-08-17T17:19:55.006 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:55.006 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:55.006 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:55.006 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:55.006 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:19:55.006 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:19:55.006 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 88379 2024-08-17T17:19:55.006 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.006 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:55.006 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.006 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:55.006 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.696360+0000", 2024-08-17T17:19:55.007 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:55.007 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.007 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.007 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:55.007 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.696360+0000", 2024-08-17T17:19:55.007 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:55.007 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.007 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.007 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:55.007 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.696362+0000", 2024-08-17T17:19:55.007 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2850000000000001e-06 2024-08-17T17:19:55.007 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.007 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.008 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:55.008 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.696400+0000", 2024-08-17T17:19:55.008 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7744999999999999e-05 2024-08-17T17:19:55.008 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.008 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.008 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:55.008 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.696427+0000", 2024-08-17T17:19:55.008 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.7237e-05 2024-08-17T17:19:55.008 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.008 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.008 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:55.008 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.696440+0000", 2024-08-17T17:19:55.008 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3352e-05 2024-08-17T17:19:55.008 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.009 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.009 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:55.009 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.701817+0000", 2024-08-17T17:19:55.009 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0053767579999999997 2024-08-17T17:19:55.009 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.009 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.009 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:55.009 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.702035+0000", 2024-08-17T17:19:55.009 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00021799100000000001 2024-08-17T17:19:55.009 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.009 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.009 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:19:55.009 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.702242+0000", 2024-08-17T17:19:55.010 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00020687 2024-08-17T17:19:55.010 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.010 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.010 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:55.010 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.723533+0000", 2024-08-17T17:19:55.010 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.02129166 2024-08-17T17:19:55.010 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.010 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.010 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:55.010 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.760280+0000", 2024-08-17T17:19:55.010 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.036746264000000001 2024-08-17T17:19:55.010 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.010 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.011 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:55.011 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.940876+0000", 2024-08-17T17:19:55.011 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.18059586 2024-08-17T17:19:55.011 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.011 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.011 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:55.011 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.941254+0000", 2024-08-17T17:19:55.011 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00037889599999999999 2024-08-17T17:19:55.011 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.011 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.011 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:55.011 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.944600+0000", 2024-08-17T17:19:55.011 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0033450400000000001 2024-08-17T17:19:55.011 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.012 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:55.012 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.012 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.012 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.012 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:105109 7.e 7:7473e8cc:::benchmark_data_smithi063_196002_object105108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1154)", 2024-08-17T17:19:55.012 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:20.282406+0000", 2024-08-17T17:19:55.012 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 34.626752725000003, 2024-08-17T17:19:55.012 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.111354245, 2024-08-17T17:19:55.012 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:55.012 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:55.012 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:55.012 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:55.013 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:19:55.013 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:19:55.013 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 105109 2024-08-17T17:19:55.013 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.013 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:55.013 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.013 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:55.013 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.282406+0000", 2024-08-17T17:19:55.013 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:55.013 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.013 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.013 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:55.013 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.282406+0000", 2024-08-17T17:19:55.014 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:55.014 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.014 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.014 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:55.014 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.282408+0000", 2024-08-17T17:19:55.014 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2899999999999999e-06 2024-08-17T17:19:55.014 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.014 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.014 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:55.014 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.282430+0000", 2024-08-17T17:19:55.014 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2354999999999999e-05 2024-08-17T17:19:55.014 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.015 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.015 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:55.015 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.282442+0000", 2024-08-17T17:19:55.015 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2130000000000001e-05 2024-08-17T17:19:55.015 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.015 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.015 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:55.015 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.282449+0000", 2024-08-17T17:19:55.015 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8449999999999997e-06 2024-08-17T17:19:55.015 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.015 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.015 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:55.015 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.298085+0000", 2024-08-17T17:19:55.015 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.015635491000000001 2024-08-17T17:19:55.016 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.016 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.016 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:55.016 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.298196+0000", 2024-08-17T17:19:55.016 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011118 2024-08-17T17:19:55.016 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.016 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.016 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:19:55.016 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.298295+0000", 2024-08-17T17:19:55.016 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9461000000000002e-05 2024-08-17T17:19:55.016 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.016 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.016 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:55.016 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.390964+0000", 2024-08-17T17:19:55.017 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.092668833000000006 2024-08-17T17:19:55.017 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.017 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.017 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:55.017 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.391036+0000", 2024-08-17T17:19:55.017 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1662999999999995e-05 2024-08-17T17:19:55.017 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.017 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.017 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:55.017 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.391100+0000", 2024-08-17T17:19:55.017 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4561999999999996e-05 2024-08-17T17:19:55.017 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.017 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.018 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:55.018 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.391177+0000", 2024-08-17T17:19:55.018 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6452000000000006e-05 2024-08-17T17:19:55.018 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.018 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.018 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:55.018 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.393761+0000", 2024-08-17T17:19:55.018 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0025839830000000002 2024-08-17T17:19:55.018 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.018 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:55.018 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.018 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.018 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.019 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:105126 7.e 7:7fc1a2a3:::benchmark_data_smithi063_196002_object105125:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1154)", 2024-08-17T17:19:55.019 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:20.297848+0000", 2024-08-17T17:19:55.019 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 34.611311213, 2024-08-17T17:19:55.019 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14215101799999999, 2024-08-17T17:19:55.019 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:55.019 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:55.019 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:55.019 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:55.019 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:19:55.019 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:19:55.019 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 105126 2024-08-17T17:19:55.019 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.020 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:55.020 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.020 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:55.020 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.297848+0000", 2024-08-17T17:19:55.020 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:55.020 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.020 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.020 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:55.020 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.297848+0000", 2024-08-17T17:19:55.020 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:55.020 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.020 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.020 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:55.020 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.297850+0000", 2024-08-17T17:19:55.021 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6840000000000001e-06 2024-08-17T17:19:55.021 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.021 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.021 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:55.021 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.297876+0000", 2024-08-17T17:19:55.021 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5969000000000002e-05 2024-08-17T17:19:55.021 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.021 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.021 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:55.021 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.297890+0000", 2024-08-17T17:19:55.021 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4431e-05 2024-08-17T17:19:55.021 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.021 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.022 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:55.022 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.297899+0000", 2024-08-17T17:19:55.022 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6000000000000007e-06 2024-08-17T17:19:55.022 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.022 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.022 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:55.022 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.301414+0000", 2024-08-17T17:19:55.022 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0035151570000000001 2024-08-17T17:19:55.022 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.022 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.022 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:55.022 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.374944+0000", 2024-08-17T17:19:55.022 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.073530288999999999 2024-08-17T17:19:55.023 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.023 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.023 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:19:55.023 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.375129+0000", 2024-08-17T17:19:55.023 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00018519599999999999 2024-08-17T17:19:55.023 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.023 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.023 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:55.023 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.393824+0000", 2024-08-17T17:19:55.023 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.018695169000000001 2024-08-17T17:19:55.023 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.023 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.023 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:55.023 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.393874+0000", 2024-08-17T17:19:55.023 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9783000000000001e-05 2024-08-17T17:19:55.024 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.024 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.024 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:55.024 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.437062+0000", 2024-08-17T17:19:55.024 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.043187495999999999 2024-08-17T17:19:55.024 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.024 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.024 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:55.024 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.437125+0000", 2024-08-17T17:19:55.024 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3131000000000003e-05 2024-08-17T17:19:55.024 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.024 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.024 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:55.025 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.439999+0000", 2024-08-17T17:19:55.025 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0028741130000000002 2024-08-17T17:19:55.025 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.025 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:55.025 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.025 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.025 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.025 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:110267 7.e 7:7b24cbfb:::benchmark_data_smithi063_196002_object110266:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1154)", 2024-08-17T17:19:55.025 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:26.196331+0000", 2024-08-17T17:19:55.025 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 28.712828258999998, 2024-08-17T17:19:55.025 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.100960129, 2024-08-17T17:19:55.025 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:55.026 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:55.026 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:55.026 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:55.026 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:19:55.026 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:19:55.026 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 110267 2024-08-17T17:19:55.026 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.026 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:55.026 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.026 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:55.026 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.196331+0000", 2024-08-17T17:19:55.026 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:55.027 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.027 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.027 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:55.027 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.196331+0000", 2024-08-17T17:19:55.027 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:55.027 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.027 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.027 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:55.027 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.196331+0000", 2024-08-17T17:19:55.027 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4299999999999998e-07 2024-08-17T17:19:55.027 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.027 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.027 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:55.028 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.196377+0000", 2024-08-17T17:19:55.028 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5262999999999997e-05 2024-08-17T17:19:55.028 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.028 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.028 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:55.028 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.196384+0000", 2024-08-17T17:19:55.028 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9639999999999997e-06 2024-08-17T17:19:55.028 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.028 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.028 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:55.028 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.196388+0000", 2024-08-17T17:19:55.028 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7829999999999998e-06 2024-08-17T17:19:55.028 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.028 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.029 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:55.029 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.216406+0000", 2024-08-17T17:19:55.029 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.020017284999999999 2024-08-17T17:19:55.029 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.029 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.029 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:55.029 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.216567+0000", 2024-08-17T17:19:55.029 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000161356 2024-08-17T17:19:55.029 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.029 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.029 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:19:55.029 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.216728+0000", 2024-08-17T17:19:55.030 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016093999999999999 2024-08-17T17:19:55.030 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.030 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.030 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:55.030 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.229891+0000", 2024-08-17T17:19:55.030 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.013163341 2024-08-17T17:19:55.030 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.030 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.030 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:55.030 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.230098+0000", 2024-08-17T17:19:55.030 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000206224 2024-08-17T17:19:55.030 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.030 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.031 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:55.031 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.294730+0000", 2024-08-17T17:19:55.031 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.064632043 2024-08-17T17:19:55.031 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.031 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.031 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:55.031 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.294793+0000", 2024-08-17T17:19:55.031 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3225000000000005e-05 2024-08-17T17:19:55.031 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.031 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.031 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:55.031 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.297291+0000", 2024-08-17T17:19:55.031 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0024982619999999998 2024-08-17T17:19:55.031 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.032 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:55.032 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.032 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.032 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.032 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:110268 7.e 7:74a91cbd:::benchmark_data_smithi063_196002_object110267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1154)", 2024-08-17T17:19:55.032 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:26.198710+0000", 2024-08-17T17:19:55.032 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 28.710449463, 2024-08-17T17:19:55.032 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.101461755, 2024-08-17T17:19:55.032 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:55.032 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:55.032 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:55.032 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:55.032 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:19:55.032 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 110268 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.198710+0000", 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.198710+0000", 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.033 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:55.034 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.198712+0000", 2024-08-17T17:19:55.034 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.75e-06 2024-08-17T17:19:55.034 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.034 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.034 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:55.034 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.198769+0000", 2024-08-17T17:19:55.034 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7562000000000002e-05 2024-08-17T17:19:55.034 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.034 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.034 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:55.034 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.198782+0000", 2024-08-17T17:19:55.034 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2863e-05 2024-08-17T17:19:55.034 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.035 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.035 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:55.035 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.198790+0000", 2024-08-17T17:19:55.035 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6450000000000005e-06 2024-08-17T17:19:55.035 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.035 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.035 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:55.035 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.220420+0000", 2024-08-17T17:19:55.036 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.021630226999999998 2024-08-17T17:19:55.036 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.036 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.036 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:55.036 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.220590+0000", 2024-08-17T17:19:55.036 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000170373 2024-08-17T17:19:55.036 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.036 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.036 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:19:55.037 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.220779+0000", 2024-08-17T17:19:55.037 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00018921899999999999 2024-08-17T17:19:55.037 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.037 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.037 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:55.037 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.230381+0000", 2024-08-17T17:19:55.037 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0096016339999999995 2024-08-17T17:19:55.037 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.037 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.037 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:55.037 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.242460+0000", 2024-08-17T17:19:55.037 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.012078907 2024-08-17T17:19:55.037 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.037 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.037 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:55.038 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.297360+0000", 2024-08-17T17:19:55.038 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.054899719999999999 2024-08-17T17:19:55.038 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.038 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.038 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:55.038 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.297440+0000", 2024-08-17T17:19:55.038 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0400000000000003e-05 2024-08-17T17:19:55.038 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.038 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.038 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:55.038 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.300172+0000", 2024-08-17T17:19:55.038 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0027314549999999998 2024-08-17T17:19:55.038 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.039 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:55.039 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.039 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.039 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.039 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:110269 7.e 7:751937d9:::benchmark_data_smithi063_196002_object110268:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1154)", 2024-08-17T17:19:55.039 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:26.200167+0000", 2024-08-17T17:19:55.039 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 28.708991848, 2024-08-17T17:19:55.039 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.110361028, 2024-08-17T17:19:55.039 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:19:55.039 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:19:55.039 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:19:55.039 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:19:55.039 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:19:55.040 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:19:55.040 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 110269 2024-08-17T17:19:55.040 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.040 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:19:55.040 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.040 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:19:55.040 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.200167+0000", 2024-08-17T17:19:55.040 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:55.040 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.040 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.040 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:19:55.040 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.200167+0000", 2024-08-17T17:19:55.040 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:19:55.040 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.040 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.041 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:19:55.041 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.200169+0000", 2024-08-17T17:19:55.041 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4279999999999999e-06 2024-08-17T17:19:55.041 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.041 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.041 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:19:55.041 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.200193+0000", 2024-08-17T17:19:55.041 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4131e-05 2024-08-17T17:19:55.041 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.041 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.041 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:19:55.041 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.200203+0000", 2024-08-17T17:19:55.041 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0556e-05 2024-08-17T17:19:55.041 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.042 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.042 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:19:55.042 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.200211+0000", 2024-08-17T17:19:55.042 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0530000000000001e-06 2024-08-17T17:19:55.042 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.042 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.042 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:19:55.042 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.230954+0000", 2024-08-17T17:19:55.042 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.030743232999999998 2024-08-17T17:19:55.042 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.042 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.042 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:19:55.042 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.231631+0000", 2024-08-17T17:19:55.042 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00067678599999999997 2024-08-17T17:19:55.043 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.043 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.043 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:19:55.043 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.232263+0000", 2024-08-17T17:19:55.043 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00063260200000000001 2024-08-17T17:19:55.043 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.043 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.043 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:55.043 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.266042+0000", 2024-08-17T17:19:55.043 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.033778375999999999 2024-08-17T17:19:55.043 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.043 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.043 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:19:55.044 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.288072+0000", 2024-08-17T17:19:55.044 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.022030861999999998 2024-08-17T17:19:55.044 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.044 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.044 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:19:55.044 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.307830+0000", 2024-08-17T17:19:55.044 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.019757820999999998 2024-08-17T17:19:55.044 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.044 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.044 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:19:55.044 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.307895+0000", 2024-08-17T17:19:55.044 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5036000000000005e-05 2024-08-17T17:19:55.044 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:19:55.044 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:19:55.044 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:19:55.045 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:26.310528+0000", 2024-08-17T17:19:55.045 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002633144 2024-08-17T17:19:55.045 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.045 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:55.045 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.045 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:19:55.045 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:19:55.045 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:19:55.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:55.050 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:55.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:55.151 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:55.175 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:55.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:55.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:55.277 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:55.303 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:55.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:55.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:55.405 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:55.430 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:55.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:55.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:55.532 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:55.558 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:55.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:55.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:55.660 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:55.672 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:55 smithi063 ceph-mon[114756]: pgmap v1397: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 22792/281385 objects degraded (8.100%); 2858/281385 objects misplaced (1.016%); 17 MiB/s, 271 objects/s recovering 2024-08-17T17:19:55.672 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:55 smithi063 ceph-mon[109843]: pgmap v1397: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 22792/281385 objects degraded (8.100%); 2858/281385 objects misplaced (1.016%); 17 MiB/s, 271 objects/s recovering 2024-08-17T17:19:55.712 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:55.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:55.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:55.814 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:55.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:55 smithi162 ceph-mon[104668]: pgmap v1397: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 827 op/s; 22792/281385 objects degraded (8.100%); 2858/281385 objects misplaced (1.016%); 17 MiB/s, 271 objects/s recovering 2024-08-17T17:19:55.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:55.852 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:55.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:55.953 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:55.988 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:55.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:55.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:56.090 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:56.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:56.117 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:56.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:56.217 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:56.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:56.267 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:56.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:56.368 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:56.407 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:56.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:56.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:56.508 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:56.535 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:56.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:56.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:56.636 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:56.679 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:56.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:56.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:56.780 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:56.806 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:56.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:56.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:56.908 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:56.934 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:56.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:56.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:57.035 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:57.056 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:57.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:57.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:57.158 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:57.193 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:57.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:57.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:57.295 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:57.333 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:57.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:57.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:57.436 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:19:57.516 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:57.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:57.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:19:57.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:57 smithi063 ceph-mon[109843]: pgmap v1398: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 22792/278520 objects degraded (8.183%); 2858/278520 objects misplaced (1.026%); 17 MiB/s, 271 objects/s recovering 2024-08-17T17:19:57.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:57 smithi063 ceph-mon[114756]: pgmap v1398: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 22792/278520 objects degraded (8.183%); 2858/278520 objects misplaced (1.026%); 17 MiB/s, 271 objects/s recovering 2024-08-17T17:19:57.618 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:57.640 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:57.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:57.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:57.741 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:57.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:57.782 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:57.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:57.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:57 smithi162 ceph-mon[104668]: pgmap v1398: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 22792/278520 objects degraded (8.183%); 2858/278520 objects misplaced (1.026%); 17 MiB/s, 271 objects/s recovering 2024-08-17T17:19:57.883 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:57.912 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:57.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:57.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:58.014 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:58.049 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:58.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:58.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:58.150 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:58.175 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:58.176 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:58.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:58.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:58.277 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:58.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:58.330 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:58.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:58.431 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:58.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:58.469 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:58.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:58.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:58 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 22792/278520 objects degraded (8.183%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:19:58.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:58 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 22792/278520 objects degraded (8.183%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:19:58.570 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:19:58.613 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:58.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:58.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:19:58.715 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:58.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:58.760 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:58.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:58.782 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:19:58.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:58 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 22792/278520 objects degraded (8.183%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:19:58.862 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:58.931 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:58.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:58.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:59.033 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:19:59.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:59.097 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:59.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:19:59.198 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:59.227 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:59.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:59.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:19:59.329 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:19:59.356 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:59.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:59.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:19:59.458 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:19:59.501 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:59.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:59.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:19:59.603 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:19:59.613 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:19:59 smithi063 ceph-mon[109843]: pgmap v1399: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 21623/270141 objects degraded (8.004%); 2738/270141 objects misplaced (1.014%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:19:59.613 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:19:59 smithi063 ceph-mon[114756]: pgmap v1399: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 21623/270141 objects degraded (8.004%); 2738/270141 objects misplaced (1.014%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:19:59.623 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:19:59.650 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:19:59.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:59.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:19:59.752 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:19:59.778 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:59.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:59.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:19:59.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:19:59 smithi162 ceph-mon[104668]: pgmap v1399: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 21623/270141 objects degraded (8.004%); 2738/270141 objects misplaced (1.014%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:19:59.880 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:19:59.881 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:19:59.883 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:19:59.908 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:19:59.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:19:59.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:00.010 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:00.052 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:00.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:00.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:00.154 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:00.191 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:00.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:00.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:00.293 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:00.329 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:00.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:00.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:00.432 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:00.468 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:00.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:00.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:00.570 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:00.611 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:00.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:00.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:00.713 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:00.741 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:00.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:00.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:00.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 21623/270141 objects degraded (8.004%), 12 pgs degraded, 3 pgs undersized 2024-08-17T17:20:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-08-17T17:20:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: [WRN] PG_DEGRADED: Degraded data redundancy: 21623/270141 objects degraded (8.004%), 12 pgs degraded, 3 pgs undersized 2024-08-17T17:20:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: pg 7.4 is active+recovery_wait+degraded, acting [4,0,1] 2024-08-17T17:20:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: pg 7.5 is active+recovery_wait+degraded, acting [7,2,0] 2024-08-17T17:20:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: pg 7.6 is active+recovery_wait+degraded, acting [7,0,3] 2024-08-17T17:20:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: pg 7.7 is active+recovery_wait+degraded, acting [6,0,3] 2024-08-17T17:20:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: pg 7.9 is active+recovery_wait+degraded, acting [4,0,1] 2024-08-17T17:20:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: pg 7.d is stuck undersized for 85s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2024-08-17T17:20:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: pg 7.e is stuck undersized for 85s, current state active+recovering+undersized+remapped, last acting [2,5] 2024-08-17T17:20:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: pg 7.f is stuck undersized for 85s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2024-08-17T17:20:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: pg 7.14 is active+recovery_wait+degraded, acting [4,0,1] 2024-08-17T17:20:00.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: pg 7.15 is active+recovery_wait+degraded, acting [7,2,0] 2024-08-17T17:20:00.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: pg 7.16 is active+recovery_wait+degraded, acting [7,0,3] 2024-08-17T17:20:00.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: pg 7.17 is active+recovery_wait+degraded, acting [6,0,3] 2024-08-17T17:20:00.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[109843]: pg 7.19 is active+recovery_wait+degraded, acting [4,0,1] 2024-08-17T17:20:00.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 21623/270141 objects degraded (8.004%), 12 pgs degraded, 3 pgs undersized 2024-08-17T17:20:00.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-08-17T17:20:00.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: [WRN] PG_DEGRADED: Degraded data redundancy: 21623/270141 objects degraded (8.004%), 12 pgs degraded, 3 pgs undersized 2024-08-17T17:20:00.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: pg 7.4 is active+recovery_wait+degraded, acting [4,0,1] 2024-08-17T17:20:00.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: pg 7.5 is active+recovery_wait+degraded, acting [7,2,0] 2024-08-17T17:20:00.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: pg 7.6 is active+recovery_wait+degraded, acting [7,0,3] 2024-08-17T17:20:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: pg 7.7 is active+recovery_wait+degraded, acting [6,0,3] 2024-08-17T17:20:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: pg 7.9 is active+recovery_wait+degraded, acting [4,0,1] 2024-08-17T17:20:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: pg 7.d is stuck undersized for 85s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2024-08-17T17:20:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: pg 7.e is stuck undersized for 85s, current state active+recovering+undersized+remapped, last acting [2,5] 2024-08-17T17:20:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: pg 7.f is stuck undersized for 85s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2024-08-17T17:20:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: pg 7.14 is active+recovery_wait+degraded, acting [4,0,1] 2024-08-17T17:20:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: pg 7.15 is active+recovery_wait+degraded, acting [7,2,0] 2024-08-17T17:20:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: pg 7.16 is active+recovery_wait+degraded, acting [7,0,3] 2024-08-17T17:20:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: pg 7.17 is active+recovery_wait+degraded, acting [6,0,3] 2024-08-17T17:20:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:00 smithi063 ceph-mon[114756]: pg 7.19 is active+recovery_wait+degraded, acting [4,0,1] 2024-08-17T17:20:00.843 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 21623/270141 objects degraded (8.004%), 12 pgs degraded, 3 pgs undersized 2024-08-17T17:20:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-08-17T17:20:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: [WRN] PG_DEGRADED: Degraded data redundancy: 21623/270141 objects degraded (8.004%), 12 pgs degraded, 3 pgs undersized 2024-08-17T17:20:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: pg 7.4 is active+recovery_wait+degraded, acting [4,0,1] 2024-08-17T17:20:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: pg 7.5 is active+recovery_wait+degraded, acting [7,2,0] 2024-08-17T17:20:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: pg 7.6 is active+recovery_wait+degraded, acting [7,0,3] 2024-08-17T17:20:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: pg 7.7 is active+recovery_wait+degraded, acting [6,0,3] 2024-08-17T17:20:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: pg 7.9 is active+recovery_wait+degraded, acting [4,0,1] 2024-08-17T17:20:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: pg 7.d is stuck undersized for 85s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,1] 2024-08-17T17:20:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: pg 7.e is stuck undersized for 85s, current state active+recovering+undersized+remapped, last acting [2,5] 2024-08-17T17:20:00.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: pg 7.f is stuck undersized for 85s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,7] 2024-08-17T17:20:00.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: pg 7.14 is active+recovery_wait+degraded, acting [4,0,1] 2024-08-17T17:20:00.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: pg 7.15 is active+recovery_wait+degraded, acting [7,2,0] 2024-08-17T17:20:00.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: pg 7.16 is active+recovery_wait+degraded, acting [7,0,3] 2024-08-17T17:20:00.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: pg 7.17 is active+recovery_wait+degraded, acting [6,0,3] 2024-08-17T17:20:00.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:00 smithi162 ceph-mon[104668]: pg 7.19 is active+recovery_wait+degraded, acting [4,0,1] 2024-08-17T17:20:00.880 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:00.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:00.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:00.982 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:01.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.024 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:01.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:01.125 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:01.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.157 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:01.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:01.258 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:01.285 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:01.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:01.386 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:01.412 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:01.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:01.514 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:01.551 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:01.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:01.653 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:01.682 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:01.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:01 smithi063 ceph-mon[114756]: from='client.54805 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:20:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:01 smithi063 ceph-mon[114756]: pgmap v1400: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 21623/267201 objects degraded (8.092%); 2738/267201 objects misplaced (1.025%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:01 smithi063 ceph-mon[109843]: from='client.54805 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:20:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:01 smithi063 ceph-mon[109843]: pgmap v1400: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 21623/267201 objects degraded (8.092%); 2738/267201 objects misplaced (1.025%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:01.783 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:01.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.813 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:01.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:01.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:01 smithi162 ceph-mon[104668]: from='client.54805 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:20:01.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:01 smithi162 ceph-mon[104668]: pgmap v1400: 109 pgs: 1 active+recovering+undersized+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 21623/267201 objects degraded (8.092%); 2738/267201 objects misplaced (1.025%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:01.914 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:01.962 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:01.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:01.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:02.064 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:02.103 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:02.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:02.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:02.205 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:02.243 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:02.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:02.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:02.345 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:02.372 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:02.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:02.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:02.474 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:02.502 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:02.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:02.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:02.603 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:02.646 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:02.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:02.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:02.747 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:02.773 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:02.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:02.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:02.875 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:02.919 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:02.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:02.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:03.021 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:03.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:03.053 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:03.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:03.154 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:03.180 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:03.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:03.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:03.283 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:03.309 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:20:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:20:03] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.51.0" 2024-08-17T17:20:03.321 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:03.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:03.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:03.423 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:03.464 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:03.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:03.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:03.566 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:03.589 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:03.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:03.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:03.691 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:03.696 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:03 smithi063 ceph-mon[114756]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 20946/261342 objects degraded (8.015%); 2552/261342 objects misplaced (0.976%); 20 MiB/s, 321 objects/s recovering 2024-08-17T17:20:03.696 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:03 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 20946/261342 objects degraded (8.015%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:03.696 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:03 smithi063 ceph-mon[109843]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 20946/261342 objects degraded (8.015%); 2552/261342 objects misplaced (0.976%); 20 MiB/s, 321 objects/s recovering 2024-08-17T17:20:03.697 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:03 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 20946/261342 objects degraded (8.015%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:03.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:03.734 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:03.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:03.835 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:03.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:03 smithi162 ceph-mon[104668]: pgmap v1401: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 20946/261342 objects degraded (8.015%); 2552/261342 objects misplaced (0.976%); 20 MiB/s, 321 objects/s recovering 2024-08-17T17:20:03.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:03 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 20946/261342 objects degraded (8.015%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:03.861 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:03.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:03.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:03.963 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:04.012 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:04.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:04.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:04.113 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:04.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:04.159 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:04.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:04.260 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:04.301 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:04.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:04.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:04.350 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1399,"stamp":"2024-08-17T17:19:58.192029+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5900403169,"num_objects":90047,"num_object_clones":0,"num_object_copies":270141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21623,"num_objects_misplaced":2738,"num_objects_unfound":0,"num_objects_dirty":90047,"num_whiteouts":0,"num_read":211,"num_read_kb":454,"num_write":127696,"num_write_kb":6749641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8611,"num_bytes_recovered":563832237,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":183926,"ondisk_log_size":183926,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":341,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20412104,"kb_used_data":18180868,"kb_used_omap":199,"kb_used_meta":2230776,"kb_avail":729549112,"statfs":{"total":767960285184,"available":747058290688,"internally_reserved":0,"allocated":18617208832,"data_stored":18599981679,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":2284315388},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[11,1,7,10,4,8,4],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-516620288,"num_objects":-7883,"num_object_clones":0,"num_object_copies":-23649,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2484,"num_objects_misplaced":-240,"num_objects_unfound":0,"num_objects_dirty":-7883,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7883,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2724,"num_bytes_recovered":178520064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005692"},"pg_stats":[{"pgid":"7.17","version":"1157'6759","reported_seq":8199,"reported_epoch":1157,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:57.075063+0000","last_change":"2024-08-17T17:19:01.756567+0000","last_active":"2024-08-17T17:19:57.075063+0000","last_peered":"2024-08-17T17:19:57.075063+0000","last_clean":"2024-08-17T17:18:30.516571+0000","last_became_active":"2024-08-17T17:19:01.750697+0000","last_became_peered":"2024-08-17T17:19:01.750697+0000","last_unstale":"2024-08-17T17:19:57.075063+0000","last_undegraded":"2024-08-17T17:19:01.715352+0000","last_fullsized":"2024-08-17T17:19:57.075063+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6759,"log_dups_size":0,"ondisk_log_size":6759,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176357376,"num_objects":2691,"num_object_clones":0,"num_object_copies":8073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1287,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4079,"num_write_kb":216640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":694,"num_bytes_recovered":45481984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"0,3,6","objects":1404},{"shards":"1,3,6","objects":1287}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1394,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.414176+0000","last_change":"2024-08-17T17:07:45.684223+0000","last_active":"2024-08-17T17:19:51.414176+0000","last_peered":"2024-08-17T17:19:51.414176+0000","last_clean":"2024-08-17T17:19:51.414176+0000","last_became_active":"2024-08-17T17:07:45.683948+0000","last_became_peered":"2024-08-17T17:07:45.683948+0000","last_unstale":"2024-08-17T17:19:51.414176+0000","last_undegraded":"2024-08-17T17:19:51.414176+0000","last_fullsized":"2024-08-17T17:19:51.414176+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:07.677423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1084,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.816573+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:19:43.816573+0000","last_peered":"2024-08-17T17:19:43.816573+0000","last_clean":"2024-08-17T17:19:43.816573+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:19:43.816573+0000","last_undegraded":"2024-08-17T17:19:43.816573+0000","last_fullsized":"2024-08-17T17:19:43.816573+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.16","version":"1157'7010","reported_seq":8485,"reported_epoch":1157,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:57.215137+0000","last_change":"2024-08-17T17:19:01.787015+0000","last_active":"2024-08-17T17:19:57.215137+0000","last_peered":"2024-08-17T17:19:57.215137+0000","last_clean":"2024-08-17T17:18:30.519687+0000","last_became_active":"2024-08-17T17:19:01.765205+0000","last_became_peered":"2024-08-17T17:19:01.765205+0000","last_unstale":"2024-08-17T17:19:57.215137+0000","last_undegraded":"2024-08-17T17:19:01.759821+0000","last_fullsized":"2024-08-17T17:19:57.215137+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7010,"log_dups_size":0,"ondisk_log_size":7010,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184811520,"num_objects":2820,"num_object_clones":0,"num_object_copies":8460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1399,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4244,"num_write_kb":226048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":712,"num_bytes_recovered":46661632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"0,3,7","objects":1421},{"shards":"3,5,7","objects":1399}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1095,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.823580+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:19:43.823580+0000","last_peered":"2024-08-17T17:19:43.823580+0000","last_clean":"2024-08-17T17:19:43.823580+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:19:43.823580+0000","last_undegraded":"2024-08-17T17:19:43.823580+0000","last_fullsized":"2024-08-17T17:19:43.823580+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":951,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:47.233785+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:19:47.233785+0000","last_peered":"2024-08-17T17:19:47.233785+0000","last_clean":"2024-08-17T17:19:47.233785+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:19:47.233785+0000","last_undegraded":"2024-08-17T17:19:47.233785+0000","last_fullsized":"2024-08-17T17:19:47.233785+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.15","version":"1157'7116","reported_seq":9472,"reported_epoch":1157,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:57.079033+0000","last_change":"2024-08-17T17:19:01.776021+0000","last_active":"2024-08-17T17:19:57.079033+0000","last_peered":"2024-08-17T17:19:57.079033+0000","last_clean":"2024-08-17T17:18:30.511218+0000","last_became_active":"2024-08-17T17:19:01.771169+0000","last_became_peered":"2024-08-17T17:19:01.771169+0000","last_unstale":"2024-08-17T17:19:57.079033+0000","last_undegraded":"2024-08-17T17:19:01.734234+0000","last_fullsized":"2024-08-17T17:19:57.079033+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7116,"log_dups_size":0,"ondisk_log_size":7116,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185303040,"num_objects":2828,"num_object_clones":0,"num_object_copies":8484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1382,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4298,"num_write_kb":228000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":744,"num_bytes_recovered":48758784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"0,2,7","objects":1446},{"shards":"1,2,7","objects":1382}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":960,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.413728+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:19:51.413728+0000","last_peered":"2024-08-17T17:19:51.413728+0000","last_clean":"2024-08-17T17:19:51.413728+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:19:51.413728+0000","last_undegraded":"2024-08-17T17:19:51.413728+0000","last_fullsized":"2024-08-17T17:19:51.413728+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1067,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.816480+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:19:43.816480+0000","last_peered":"2024-08-17T17:19:43.816480+0000","last_clean":"2024-08-17T17:19:43.816480+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:19:43.816480+0000","last_undegraded":"2024-08-17T17:19:43.816480+0000","last_fullsized":"2024-08-17T17:19:43.816480+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.14","version":"1157'7026","reported_seq":8549,"reported_epoch":1157,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:56.920540+0000","last_change":"2024-08-17T17:19:01.769109+0000","last_active":"2024-08-17T17:19:56.920540+0000","last_peered":"2024-08-17T17:19:56.920540+0000","last_clean":"2024-08-17T17:18:30.520652+0000","last_became_active":"2024-08-17T17:19:01.765052+0000","last_became_peered":"2024-08-17T17:19:01.765052+0000","last_unstale":"2024-08-17T17:19:56.920540+0000","last_undegraded":"2024-08-17T17:19:01.729866+0000","last_fullsized":"2024-08-17T17:19:56.920540+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7026,"log_dups_size":0,"ondisk_log_size":7026,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181567488,"num_objects":2771,"num_object_clones":0,"num_object_copies":8313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1376,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4241,"num_write_kb":224352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":735,"num_bytes_recovered":48168960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"0,1,4","objects":1395},{"shards":"1,4,6","objects":1376}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1394,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.432934+0000","last_change":"2024-08-17T17:07:45.687030+0000","last_active":"2024-08-17T17:19:51.432934+0000","last_peered":"2024-08-17T17:19:51.432934+0000","last_clean":"2024-08-17T17:19:51.432934+0000","last_became_active":"2024-08-17T17:07:45.686610+0000","last_became_peered":"2024-08-17T17:07:45.686610+0000","last_unstale":"2024-08-17T17:19:51.432934+0000","last_undegraded":"2024-08-17T17:19:51.432934+0000","last_fullsized":"2024-08-17T17:19:51.432934+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:02.425845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1144,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.813394+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:19:43.813394+0000","last_peered":"2024-08-17T17:19:43.813394+0000","last_clean":"2024-08-17T17:19:43.813394+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:19:43.813394+0000","last_undegraded":"2024-08-17T17:19:43.813394+0000","last_fullsized":"2024-08-17T17:19:43.813394+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1157'8383","reported_seq":8420,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:57.185960+0000","last_change":"2024-08-17T17:17:52.927899+0000","last_active":"2024-08-17T17:19:57.185960+0000","last_peered":"2024-08-17T17:19:57.185960+0000","last_clean":"2024-08-17T17:19:57.185960+0000","last_became_active":"2024-08-17T17:17:52.927591+0000","last_became_peered":"2024-08-17T17:17:52.927591+0000","last_unstale":"2024-08-17T17:19:57.185960+0000","last_undegraded":"2024-08-17T17:19:57.185960+0000","last_fullsized":"2024-08-17T17:19:57.185960+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":8383,"log_dups_size":0,"ondisk_log_size":8383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:31:21.681712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372703232,"num_objects":5687,"num_object_clones":0,"num_object_copies":17061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8383,"num_write_kb":450240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1109,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.817168+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:19:43.817168+0000","last_peered":"2024-08-17T17:19:43.817168+0000","last_clean":"2024-08-17T17:19:43.817168+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:19:43.817168+0000","last_undegraded":"2024-08-17T17:19:43.817168+0000","last_fullsized":"2024-08-17T17:19:43.817168+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1082,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.823318+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:19:43.823318+0000","last_peered":"2024-08-17T17:19:43.823318+0000","last_clean":"2024-08-17T17:19:43.823318+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:19:43.823318+0000","last_undegraded":"2024-08-17T17:19:43.823318+0000","last_fullsized":"2024-08-17T17:19:43.823318+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1157'8271","reported_seq":8341,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:54.691718+0000","last_change":"2024-08-17T17:17:52.311116+0000","last_active":"2024-08-17T17:19:54.691718+0000","last_peered":"2024-08-17T17:19:54.691718+0000","last_clean":"2024-08-17T17:19:54.691718+0000","last_became_active":"2024-08-17T17:17:52.310809+0000","last_became_peered":"2024-08-17T17:17:52.310809+0000","last_unstale":"2024-08-17T17:19:54.691718+0000","last_undegraded":"2024-08-17T17:19:54.691718+0000","last_fullsized":"2024-08-17T17:19:54.691718+0000","mapping_epoch":1129,"log_start":"1134'2600","ondisk_log_start":"1134'2600","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":5671,"log_dups_size":0,"ondisk_log_size":5671,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:00:48.009463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368902144,"num_objects":5629,"num_object_clones":0,"num_object_copies":16887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5629,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8271,"num_write_kb":444800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1084,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.820061+0000","last_change":"2024-08-17T17:06:39.638433+0000","last_active":"2024-08-17T17:19:43.820061+0000","last_peered":"2024-08-17T17:19:43.820061+0000","last_clean":"2024-08-17T17:19:43.820061+0000","last_became_active":"2024-08-17T17:06:39.638081+0000","last_became_peered":"2024-08-17T17:06:39.638081+0000","last_unstale":"2024-08-17T17:19:43.820061+0000","last_undegraded":"2024-08-17T17:19:43.820061+0000","last_fullsized":"2024-08-17T17:19:43.820061+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:13.558012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1116,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:47.185080+0000","last_change":"2024-08-17T17:07:45.693574+0000","last_active":"2024-08-17T17:19:47.185080+0000","last_peered":"2024-08-17T17:19:47.185080+0000","last_clean":"2024-08-17T17:19:47.185080+0000","last_became_active":"2024-08-17T17:07:45.693086+0000","last_became_peered":"2024-08-17T17:07:45.693086+0000","last_unstale":"2024-08-17T17:19:47.185080+0000","last_undegraded":"2024-08-17T17:19:47.185080+0000","last_fullsized":"2024-08-17T17:19:47.185080+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:31.888894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1157'6947","reported_seq":8376,"reported_epoch":1157,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:57.181268+0000","last_change":"2024-08-17T17:19:02.700822+0000","last_active":"2024-08-17T17:19:57.181268+0000","last_peered":"2024-08-17T17:19:57.181268+0000","last_clean":"2024-08-17T17:18:30.513360+0000","last_became_active":"2024-08-17T17:19:02.673093+0000","last_became_peered":"2024-08-17T17:19:02.673093+0000","last_unstale":"2024-08-17T17:19:57.181268+0000","last_undegraded":"2024-08-17T17:19:02.662660+0000","last_fullsized":"2024-08-17T17:19:57.181268+0000","mapping_epoch":1152,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6947,"log_dups_size":0,"ondisk_log_size":6947,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184549376,"num_objects":2816,"num_object_clones":0,"num_object_copies":8448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1340,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4186,"num_write_kb":224064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":686,"num_bytes_recovered":44990464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"0,1,4","objects":1476},{"shards":"1,4,7","objects":1340}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1068,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.818797+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:19:43.818797+0000","last_peered":"2024-08-17T17:19:43.818797+0000","last_clean":"2024-08-17T17:19:43.818797+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:19:43.818797+0000","last_undegraded":"2024-08-17T17:19:43.818797+0000","last_fullsized":"2024-08-17T17:19:43.818797+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1098,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:47.136289+0000","last_change":"2024-08-17T17:06:39.629368+0000","last_active":"2024-08-17T17:19:47.136289+0000","last_peered":"2024-08-17T17:19:47.136289+0000","last_clean":"2024-08-17T17:19:47.136289+0000","last_became_active":"2024-08-17T17:06:39.628832+0000","last_became_peered":"2024-08-17T17:06:39.628832+0000","last_unstale":"2024-08-17T17:19:47.136289+0000","last_undegraded":"2024-08-17T17:19:47.136289+0000","last_fullsized":"2024-08-17T17:19:47.136289+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:54.063884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1157'6930","reported_seq":7017,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:56.307535+0000","last_change":"2024-08-17T17:19:51.419831+0000","last_active":"2024-08-17T17:19:56.307535+0000","last_peered":"2024-08-17T17:19:56.307535+0000","last_clean":"2024-08-17T17:19:56.307535+0000","last_became_active":"2024-08-17T17:19:01.657134+0000","last_became_peered":"2024-08-17T17:19:01.657134+0000","last_unstale":"2024-08-17T17:19:56.307535+0000","last_undegraded":"2024-08-17T17:19:56.307535+0000","last_fullsized":"2024-08-17T17:19:56.307535+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6930,"log_dups_size":0,"ondisk_log_size":6930,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:19:56.374088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179634176,"num_objects":2741,"num_object_clones":0,"num_object_copies":8223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4171,"num_write_kb":221184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1097,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.823707+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:19:43.823707+0000","last_peered":"2024-08-17T17:19:43.823707+0000","last_clean":"2024-08-17T17:19:43.823707+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:19:43.823707+0000","last_undegraded":"2024-08-17T17:19:43.823707+0000","last_fullsized":"2024-08-17T17:19:43.823707+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":907,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.419384+0000","last_change":"2024-08-17T17:18:31.539133+0000","last_active":"2024-08-17T17:19:51.419384+0000","last_peered":"2024-08-17T17:19:51.419384+0000","last_clean":"2024-08-17T17:19:51.419384+0000","last_became_active":"2024-08-17T17:18:31.538669+0000","last_became_peered":"2024-08-17T17:18:31.538669+0000","last_unstale":"2024-08-17T17:19:51.419384+0000","last_undegraded":"2024-08-17T17:19:51.419384+0000","last_fullsized":"2024-08-17T17:19:51.419384+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:34.785446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1464,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.816323+0000","last_change":"2024-08-17T17:18:28.522790+0000","last_active":"2024-08-17T17:19:43.816323+0000","last_peered":"2024-08-17T17:19:43.816323+0000","last_clean":"2024-08-17T17:19:43.816323+0000","last_became_active":"2024-08-17T17:18:28.521573+0000","last_became_peered":"2024-08-17T17:18:28.521573+0000","last_unstale":"2024-08-17T17:19:43.816323+0000","last_undegraded":"2024-08-17T17:19:43.816323+0000","last_fullsized":"2024-08-17T17:19:43.816323+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":76,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":214679,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.f","version":"1157'8323","reported_seq":8393,"reported_epoch":1157,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:19:54.727864+0000","last_change":"2024-08-17T17:18:33.026499+0000","last_active":"2024-08-17T17:19:54.727864+0000","last_peered":"2024-08-17T17:19:54.727864+0000","last_clean":"2024-08-17T17:18:30.502708+0000","last_became_active":"2024-08-17T17:18:33.018495+0000","last_became_peered":"2024-08-17T17:18:33.018495+0000","last_unstale":"2024-08-17T17:19:54.727864+0000","last_undegraded":"2024-08-17T17:18:32.555520+0000","last_fullsized":"2024-08-17T17:18:32.533633+0000","mapping_epoch":1144,"log_start":"1134'2600","ondisk_log_start":"1134'2600","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":5723,"log_dups_size":0,"ondisk_log_size":5723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376111104,"num_objects":5739,"num_object_clones":0,"num_object_copies":17217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4154,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8323,"num_write_kb":449984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":2862},{"shards":"3,7","objects":1585}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1429,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.415336+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:19:51.415336+0000","last_peered":"2024-08-17T17:19:51.415336+0000","last_clean":"2024-08-17T17:19:51.415336+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:19:51.415336+0000","last_undegraded":"2024-08-17T17:19:51.415336+0000","last_fullsized":"2024-08-17T17:19:51.415336+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1100,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.830051+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:19:43.830051+0000","last_peered":"2024-08-17T17:19:43.830051+0000","last_clean":"2024-08-17T17:19:43.830051+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:19:43.830051+0000","last_undegraded":"2024-08-17T17:19:43.830051+0000","last_fullsized":"2024-08-17T17:19:43.830051+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1422,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.823151+0000","last_change":"2024-08-17T17:18:32.884334+0000","last_active":"2024-08-17T17:19:43.823151+0000","last_peered":"2024-08-17T17:19:43.823151+0000","last_clean":"2024-08-17T17:19:43.823151+0000","last_became_active":"2024-08-17T17:18:28.520186+0000","last_became_peered":"2024-08-17T17:18:28.520186+0000","last_unstale":"2024-08-17T17:19:43.823151+0000","last_undegraded":"2024-08-17T17:19:43.823151+0000","last_fullsized":"2024-08-17T17:19:43.823151+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:32.884246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:32.884246+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:49.286656+0000","scrub_duration":0.001842677,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.e","version":"1157'8535","reported_seq":11491,"reported_epoch":1157,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:19:56.328383+0000","last_change":"2024-08-17T17:19:00.937351+0000","last_active":"2024-08-17T17:19:56.328383+0000","last_peered":"2024-08-17T17:19:56.328383+0000","last_clean":"2024-08-17T17:18:30.515740+0000","last_became_active":"2024-08-17T17:18:33.016776+0000","last_became_peered":"2024-08-17T17:18:33.016776+0000","last_unstale":"2024-08-17T17:19:56.328383+0000","last_undegraded":"2024-08-17T17:19:56.328383+0000","last_fullsized":"2024-08-17T17:18:32.535040+0000","mapping_epoch":1144,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":8535,"log_dups_size":0,"ondisk_log_size":8535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376373248,"num_objects":5743,"num_object_clones":0,"num_object_copies":17229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2738,"num_objects_unfound":0,"num_objects_dirty":5743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8535,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1443,"num_bytes_recovered":94568448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":2738},{"shards":"2,5","objects":3005}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1099,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.813617+0000","last_change":"2024-08-17T17:06:39.629619+0000","last_active":"2024-08-17T17:19:43.813617+0000","last_peered":"2024-08-17T17:19:43.813617+0000","last_clean":"2024-08-17T17:19:43.813617+0000","last_became_active":"2024-08-17T17:06:39.629295+0000","last_became_peered":"2024-08-17T17:06:39.629295+0000","last_unstale":"2024-08-17T17:19:43.813617+0000","last_undegraded":"2024-08-17T17:19:43.813617+0000","last_fullsized":"2024-08-17T17:19:43.813617+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:15.874423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1115,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.827837+0000","last_change":"2024-08-17T17:07:12.649178+0000","last_active":"2024-08-17T17:19:43.827837+0000","last_peered":"2024-08-17T17:19:43.827837+0000","last_clean":"2024-08-17T17:19:43.827837+0000","last_became_active":"2024-08-17T17:07:12.648844+0000","last_became_peered":"2024-08-17T17:07:12.648844+0000","last_unstale":"2024-08-17T17:19:43.827837+0000","last_undegraded":"2024-08-17T17:19:43.827837+0000","last_fullsized":"2024-08-17T17:19:43.827837+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:12:06.307736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1157'8373","reported_seq":8419,"reported_epoch":1157,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:19:57.077962+0000","last_change":"2024-08-17T17:18:33.033028+0000","last_active":"2024-08-17T17:19:57.077962+0000","last_peered":"2024-08-17T17:19:57.077962+0000","last_clean":"2024-08-17T17:18:30.508341+0000","last_became_active":"2024-08-17T17:18:33.019697+0000","last_became_peered":"2024-08-17T17:18:33.019697+0000","last_unstale":"2024-08-17T17:19:57.077962+0000","last_undegraded":"2024-08-17T17:18:32.562771+0000","last_fullsized":"2024-08-17T17:18:32.533408+0000","mapping_epoch":1144,"log_start":"1133'1000","ondisk_log_start":"1133'1000","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7373,"log_dups_size":0,"ondisk_log_size":7373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:58:13.699201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369491992,"num_objects":5639,"num_object_clones":0,"num_object_copies":16917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4070,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5639,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8373,"num_write_kb":448321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":2703},{"shards":"1,6","objects":1569}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1102,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.834675+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:19:43.834675+0000","last_peered":"2024-08-17T17:19:43.834675+0000","last_clean":"2024-08-17T17:19:43.834675+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:19:43.834675+0000","last_undegraded":"2024-08-17T17:19:43.834675+0000","last_fullsized":"2024-08-17T17:19:43.834675+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1100,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.828239+0000","last_change":"2024-08-17T17:06:39.627678+0000","last_active":"2024-08-17T17:19:43.828239+0000","last_peered":"2024-08-17T17:19:43.828239+0000","last_clean":"2024-08-17T17:19:43.828239+0000","last_became_active":"2024-08-17T17:06:39.627226+0000","last_became_peered":"2024-08-17T17:06:39.627226+0000","last_unstale":"2024-08-17T17:19:43.828239+0000","last_undegraded":"2024-08-17T17:19:43.828239+0000","last_fullsized":"2024-08-17T17:19:43.828239+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:01.868244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1422,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.823291+0000","last_change":"2024-08-17T17:18:31.850829+0000","last_active":"2024-08-17T17:19:43.823291+0000","last_peered":"2024-08-17T17:19:43.823291+0000","last_clean":"2024-08-17T17:19:43.823291+0000","last_became_active":"2024-08-17T17:18:28.520946+0000","last_became_peered":"2024-08-17T17:18:28.520946+0000","last_unstale":"2024-08-17T17:19:43.823291+0000","last_undegraded":"2024-08-17T17:19:43.823291+0000","last_fullsized":"2024-08-17T17:19:43.823291+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:31.850745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:31.850745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:17.218084+0000","scrub_duration":0.001317447,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.c","version":"1157'8312","reported_seq":8349,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:57.136418+0000","last_change":"2024-08-17T17:17:52.309565+0000","last_active":"2024-08-17T17:19:57.136418+0000","last_peered":"2024-08-17T17:19:57.136418+0000","last_clean":"2024-08-17T17:19:57.136418+0000","last_became_active":"2024-08-17T17:17:52.309298+0000","last_became_peered":"2024-08-17T17:17:52.309298+0000","last_unstale":"2024-08-17T17:19:57.136418+0000","last_undegraded":"2024-08-17T17:19:57.136418+0000","last_fullsized":"2024-08-17T17:19:57.136418+0000","mapping_epoch":1129,"log_start":"1133'900","ondisk_log_start":"1133'900","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7412,"log_dups_size":0,"ondisk_log_size":7412,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:27:31.251993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361496576,"num_objects":5516,"num_object_clones":0,"num_object_copies":16548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8312,"num_write_kb":442496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1396,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.415023+0000","last_change":"2024-08-17T17:07:45.680458+0000","last_active":"2024-08-17T17:19:51.415023+0000","last_peered":"2024-08-17T17:19:51.415023+0000","last_clean":"2024-08-17T17:19:51.415023+0000","last_became_active":"2024-08-17T17:07:45.680221+0000","last_became_peered":"2024-08-17T17:07:45.680221+0000","last_unstale":"2024-08-17T17:19:51.415023+0000","last_undegraded":"2024-08-17T17:19:51.415023+0000","last_fullsized":"2024-08-17T17:19:51.415023+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:55.083962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1418,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.420510+0000","last_change":"2024-08-17T17:07:45.682866+0000","last_active":"2024-08-17T17:19:51.420510+0000","last_peered":"2024-08-17T17:19:51.420510+0000","last_clean":"2024-08-17T17:19:51.420510+0000","last_became_active":"2024-08-17T17:07:45.682554+0000","last_became_peered":"2024-08-17T17:07:45.682554+0000","last_unstale":"2024-08-17T17:19:51.420510+0000","last_undegraded":"2024-08-17T17:19:51.420510+0000","last_fullsized":"2024-08-17T17:19:51.420510+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:25.975914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.6","version":"1157'7070","reported_seq":8572,"reported_epoch":1157,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:57.215660+0000","last_change":"2024-08-17T17:19:02.694376+0000","last_active":"2024-08-17T17:19:57.215660+0000","last_peered":"2024-08-17T17:19:57.215660+0000","last_clean":"2024-08-17T17:18:30.519687+0000","last_became_active":"2024-08-17T17:19:02.689996+0000","last_became_peered":"2024-08-17T17:19:02.689996+0000","last_unstale":"2024-08-17T17:19:57.215660+0000","last_undegraded":"2024-08-17T17:19:02.656489+0000","last_fullsized":"2024-08-17T17:19:57.215660+0000","mapping_epoch":1152,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7070,"log_dups_size":0,"ondisk_log_size":7070,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187039744,"num_objects":2854,"num_object_clones":0,"num_object_copies":8562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1286,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4304,"num_write_kb":229056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":725,"num_bytes_recovered":47513600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"0,3,7","objects":1568},{"shards":"3,5,7","objects":1286}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1476,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.816815+0000","last_change":"2024-08-17T17:18:33.814375+0000","last_active":"2024-08-17T17:19:43.816815+0000","last_peered":"2024-08-17T17:19:43.816815+0000","last_clean":"2024-08-17T17:19:43.816815+0000","last_became_active":"2024-08-17T17:18:33.813208+0000","last_became_peered":"2024-08-17T17:18:33.813208+0000","last_unstale":"2024-08-17T17:19:43.816815+0000","last_undegraded":"2024-08-17T17:19:43.816815+0000","last_fullsized":"2024-08-17T17:19:43.816815+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:25.057675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:25.057675+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:20:45.235879+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1112,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.825818+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:19:43.825818+0000","last_peered":"2024-08-17T17:19:43.825818+0000","last_clean":"2024-08-17T17:19:43.825818+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:19:43.825818+0000","last_undegraded":"2024-08-17T17:19:43.825818+0000","last_fullsized":"2024-08-17T17:19:43.825818+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1375,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.415723+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:19:51.415723+0000","last_peered":"2024-08-17T17:19:51.415723+0000","last_clean":"2024-08-17T17:19:51.415723+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:19:51.415723+0000","last_undegraded":"2024-08-17T17:19:51.415723+0000","last_fullsized":"2024-08-17T17:19:51.415723+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1083,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.814341+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:19:43.814341+0000","last_peered":"2024-08-17T17:19:43.814341+0000","last_clean":"2024-08-17T17:19:43.814341+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:19:43.814341+0000","last_undegraded":"2024-08-17T17:19:43.814341+0000","last_fullsized":"2024-08-17T17:19:43.814341+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":669,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.813530+0000","last_change":"2024-08-17T17:18:28.524571+0000","last_active":"2024-08-17T17:19:43.813530+0000","last_peered":"2024-08-17T17:19:43.813530+0000","last_clean":"2024-08-17T17:19:43.813530+0000","last_became_active":"2024-08-17T17:18:28.524280+0000","last_became_peered":"2024-08-17T17:18:28.524280+0000","last_unstale":"2024-08-17T17:19:43.813530+0000","last_undegraded":"2024-08-17T17:19:43.813530+0000","last_fullsized":"2024-08-17T17:19:43.813530+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1157'7099","reported_seq":9383,"reported_epoch":1157,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:57.181602+0000","last_change":"2024-08-17T17:19:02.692013+0000","last_active":"2024-08-17T17:19:57.181602+0000","last_peered":"2024-08-17T17:19:57.181602+0000","last_clean":"2024-08-17T17:18:30.511218+0000","last_became_active":"2024-08-17T17:19:02.665577+0000","last_became_peered":"2024-08-17T17:19:02.665577+0000","last_unstale":"2024-08-17T17:19:57.181602+0000","last_undegraded":"2024-08-17T17:19:02.654944+0000","last_fullsized":"2024-08-17T17:19:57.181602+0000","mapping_epoch":1152,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7099,"log_dups_size":0,"ondisk_log_size":7099,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186679296,"num_objects":2848,"num_object_clones":0,"num_object_copies":8544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1351,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4280,"num_write_kb":228128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":707,"num_bytes_recovered":46333952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"0,2,7","objects":1497},{"shards":"1,2,7","objects":1351}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1129,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.816744+0000","last_change":"2024-08-17T17:18:31.539042+0000","last_active":"2024-08-17T17:19:43.816744+0000","last_peered":"2024-08-17T17:19:43.816744+0000","last_clean":"2024-08-17T17:19:43.816744+0000","last_became_active":"2024-08-17T17:18:31.537879+0000","last_became_peered":"2024-08-17T17:18:31.537879+0000","last_unstale":"2024-08-17T17:19:43.816744+0000","last_undegraded":"2024-08-17T17:19:43.816744+0000","last_fullsized":"2024-08-17T17:19:43.816744+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:36:27.859975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":951,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.834491+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:19:43.834491+0000","last_peered":"2024-08-17T17:19:43.834491+0000","last_clean":"2024-08-17T17:19:43.834491+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:19:43.834491+0000","last_undegraded":"2024-08-17T17:19:43.834491+0000","last_fullsized":"2024-08-17T17:19:43.834491+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":857,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.824036+0000","last_change":"2024-08-17T17:18:31.536646+0000","last_active":"2024-08-17T17:19:43.824036+0000","last_peered":"2024-08-17T17:19:43.824036+0000","last_clean":"2024-08-17T17:19:43.824036+0000","last_became_active":"2024-08-17T17:18:31.536326+0000","last_became_peered":"2024-08-17T17:18:31.536326+0000","last_unstale":"2024-08-17T17:19:43.824036+0000","last_undegraded":"2024-08-17T17:19:43.824036+0000","last_fullsized":"2024-08-17T17:19:43.824036+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:11:03.952344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1481,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.816204+0000","last_change":"2024-08-17T17:18:33.817028+0000","last_active":"2024-08-17T17:19:43.816204+0000","last_peered":"2024-08-17T17:19:43.816204+0000","last_clean":"2024-08-17T17:19:43.816204+0000","last_became_active":"2024-08-17T17:18:33.816660+0000","last_became_peered":"2024-08-17T17:18:33.816660+0000","last_unstale":"2024-08-17T17:19:43.816204+0000","last_undegraded":"2024-08-17T17:19:43.816204+0000","last_fullsized":"2024-08-17T17:19:43.816204+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":214680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.4","version":"1157'7076","reported_seq":8599,"reported_epoch":1157,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:57.109356+0000","last_change":"2024-08-17T17:19:02.697004+0000","last_active":"2024-08-17T17:19:57.109356+0000","last_peered":"2024-08-17T17:19:57.109356+0000","last_clean":"2024-08-17T17:18:30.520652+0000","last_became_active":"2024-08-17T17:19:02.667978+0000","last_became_peered":"2024-08-17T17:19:02.667978+0000","last_unstale":"2024-08-17T17:19:57.109356+0000","last_undegraded":"2024-08-17T17:19:02.661396+0000","last_fullsized":"2024-08-17T17:19:57.109356+0000","mapping_epoch":1152,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7076,"log_dups_size":0,"ondisk_log_size":7076,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184975360,"num_objects":2822,"num_object_clones":0,"num_object_copies":8466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1333,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4290,"num_write_kb":227616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":734,"num_bytes_recovered":48103424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"0,1,4","objects":1489},{"shards":"1,4,6","objects":1333}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1473,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.418402+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:19:51.418402+0000","last_peered":"2024-08-17T17:19:51.418402+0000","last_clean":"2024-08-17T17:19:51.418402+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:19:51.418402+0000","last_undegraded":"2024-08-17T17:19:51.418402+0000","last_fullsized":"2024-08-17T17:19:51.418402+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":851,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.419917+0000","last_change":"2024-08-17T17:15:38.610415+0000","last_active":"2024-08-17T17:19:51.419917+0000","last_peered":"2024-08-17T17:19:51.419917+0000","last_clean":"2024-08-17T17:19:51.419917+0000","last_became_active":"2024-08-17T17:15:38.610083+0000","last_became_peered":"2024-08-17T17:15:38.610083+0000","last_unstale":"2024-08-17T17:19:51.419917+0000","last_undegraded":"2024-08-17T17:19:51.419917+0000","last_fullsized":"2024-08-17T17:19:51.419917+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:12.007494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":776,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.823935+0000","last_change":"2024-08-17T17:18:31.534212+0000","last_active":"2024-08-17T17:19:43.823935+0000","last_peered":"2024-08-17T17:19:43.823935+0000","last_clean":"2024-08-17T17:19:43.823935+0000","last_became_active":"2024-08-17T17:18:31.533792+0000","last_became_peered":"2024-08-17T17:18:31.533792+0000","last_unstale":"2024-08-17T17:19:43.823935+0000","last_undegraded":"2024-08-17T17:19:43.823935+0000","last_fullsized":"2024-08-17T17:19:43.823935+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:13:01.220425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1439,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.823159+0000","last_change":"2024-08-17T17:18:35.916482+0000","last_active":"2024-08-17T17:19:43.823159+0000","last_peered":"2024-08-17T17:19:43.823159+0000","last_clean":"2024-08-17T17:19:43.823159+0000","last_became_active":"2024-08-17T17:18:34.827747+0000","last_became_peered":"2024-08-17T17:18:34.827747+0000","last_unstale":"2024-08-17T17:19:43.823159+0000","last_undegraded":"2024-08-17T17:19:43.823159+0000","last_fullsized":"2024-08-17T17:19:43.823159+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:35.916381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:35.916381+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:28.492622+0000","scrub_duration":0.005340521,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"1157'6745","reported_seq":8130,"reported_epoch":1157,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:57.075050+0000","last_change":"2024-08-17T17:19:02.695693+0000","last_active":"2024-08-17T17:19:57.075050+0000","last_peered":"2024-08-17T17:19:57.075050+0000","last_clean":"2024-08-17T17:18:30.516571+0000","last_became_active":"2024-08-17T17:19:02.666519+0000","last_became_peered":"2024-08-17T17:19:02.666519+0000","last_unstale":"2024-08-17T17:19:57.075050+0000","last_undegraded":"2024-08-17T17:19:02.656623+0000","last_fullsized":"2024-08-17T17:19:57.075050+0000","mapping_epoch":1152,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6745,"log_dups_size":0,"ondisk_log_size":6745,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179109888,"num_objects":2733,"num_object_clones":0,"num_object_copies":8199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1296,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4065,"num_write_kb":217536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":666,"num_bytes_recovered":43646976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"0,3,6","objects":1437},{"shards":"1,3,6","objects":1296}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1113,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.823674+0000","last_change":"2024-08-17T17:06:39.628745+0000","last_active":"2024-08-17T17:19:43.823674+0000","last_peered":"2024-08-17T17:19:43.823674+0000","last_clean":"2024-08-17T17:19:43.823674+0000","last_became_active":"2024-08-17T17:06:39.628144+0000","last_became_peered":"2024-08-17T17:06:39.628144+0000","last_unstale":"2024-08-17T17:19:43.823674+0000","last_undegraded":"2024-08-17T17:19:43.823674+0000","last_fullsized":"2024-08-17T17:19:43.823674+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:14.740936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1120,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.828061+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:19:43.828061+0000","last_peered":"2024-08-17T17:19:43.828061+0000","last_clean":"2024-08-17T17:19:43.828061+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:19:43.828061+0000","last_undegraded":"2024-08-17T17:19:43.828061+0000","last_fullsized":"2024-08-17T17:19:43.828061+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":927,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.418866+0000","last_change":"2024-08-17T17:15:38.608764+0000","last_active":"2024-08-17T17:19:51.418866+0000","last_peered":"2024-08-17T17:19:51.418866+0000","last_clean":"2024-08-17T17:19:51.418866+0000","last_became_active":"2024-08-17T17:15:38.608353+0000","last_became_peered":"2024-08-17T17:15:38.608353+0000","last_unstale":"2024-08-17T17:19:51.418866+0000","last_undegraded":"2024-08-17T17:19:51.418866+0000","last_fullsized":"2024-08-17T17:19:51.418866+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:42.873441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":669,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.815029+0000","last_change":"2024-08-17T17:18:28.527058+0000","last_active":"2024-08-17T17:19:43.815029+0000","last_peered":"2024-08-17T17:19:43.815029+0000","last_clean":"2024-08-17T17:19:43.815029+0000","last_became_active":"2024-08-17T17:18:28.526439+0000","last_became_peered":"2024-08-17T17:18:28.526439+0000","last_unstale":"2024-08-17T17:19:43.815029+0000","last_undegraded":"2024-08-17T17:19:43.815029+0000","last_fullsized":"2024-08-17T17:19:43.815029+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.1","version":"1157'6869","reported_seq":6917,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:57.147081+0000","last_change":"2024-08-17T17:19:01.686339+0000","last_active":"2024-08-17T17:19:57.147081+0000","last_peered":"2024-08-17T17:19:57.147081+0000","last_clean":"2024-08-17T17:19:57.147081+0000","last_became_active":"2024-08-17T17:19:01.682436+0000","last_became_peered":"2024-08-17T17:19:01.682436+0000","last_unstale":"2024-08-17T17:19:57.147081+0000","last_undegraded":"2024-08-17T17:19:57.147081+0000","last_fullsized":"2024-08-17T17:19:57.147081+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6869,"log_dups_size":0,"ondisk_log_size":6869,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178978816,"num_objects":2731,"num_object_clones":0,"num_object_copies":8193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4159,"num_write_kb":220480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1113,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.823804+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:19:43.823804+0000","last_peered":"2024-08-17T17:19:43.823804+0000","last_clean":"2024-08-17T17:19:43.823804+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:19:43.823804+0000","last_undegraded":"2024-08-17T17:19:43.823804+0000","last_fullsized":"2024-08-17T17:19:43.823804+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":786,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.418145+0000","last_change":"2024-08-17T17:18:31.537256+0000","last_active":"2024-08-17T17:19:51.418145+0000","last_peered":"2024-08-17T17:19:51.418145+0000","last_clean":"2024-08-17T17:19:51.418145+0000","last_became_active":"2024-08-17T17:18:31.536194+0000","last_became_peered":"2024-08-17T17:18:31.536194+0000","last_unstale":"2024-08-17T17:19:51.418145+0000","last_undegraded":"2024-08-17T17:19:51.418145+0000","last_fullsized":"2024-08-17T17:19:51.418145+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:46.453717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":728,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.404350+0000","last_change":"2024-08-17T17:15:38.609480+0000","last_active":"2024-08-17T17:19:51.404350+0000","last_peered":"2024-08-17T17:19:51.404350+0000","last_clean":"2024-08-17T17:19:51.404350+0000","last_became_active":"2024-08-17T17:15:38.609120+0000","last_became_peered":"2024-08-17T17:15:38.609120+0000","last_unstale":"2024-08-17T17:19:51.404350+0000","last_undegraded":"2024-08-17T17:19:51.404350+0000","last_fullsized":"2024-08-17T17:19:51.404350+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:11:03.113425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":686,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.813572+0000","last_change":"2024-08-17T17:18:34.831943+0000","last_active":"2024-08-17T17:19:43.813572+0000","last_peered":"2024-08-17T17:19:43.813572+0000","last_clean":"2024-08-17T17:19:43.813572+0000","last_became_active":"2024-08-17T17:18:34.831498+0000","last_became_peered":"2024-08-17T17:18:34.831498+0000","last_unstale":"2024-08-17T17:19:43.813572+0000","last_undegraded":"2024-08-17T17:19:43.813572+0000","last_fullsized":"2024-08-17T17:19:43.813572+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.0","version":"1157'7073","reported_seq":7121,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:57.159816+0000","last_change":"2024-08-17T17:19:01.664407+0000","last_active":"2024-08-17T17:19:57.159816+0000","last_peered":"2024-08-17T17:19:57.159816+0000","last_clean":"2024-08-17T17:19:57.159816+0000","last_became_active":"2024-08-17T17:19:01.659646+0000","last_became_peered":"2024-08-17T17:19:01.659646+0000","last_unstale":"2024-08-17T17:19:57.159816+0000","last_undegraded":"2024-08-17T17:19:57.159816+0000","last_fullsized":"2024-08-17T17:19:57.159816+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7073,"log_dups_size":0,"ondisk_log_size":7073,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183926784,"num_objects":2806,"num_object_clones":0,"num_object_copies":8418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4276,"num_write_kb":226656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":850,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.418690+0000","last_change":"2024-08-17T17:15:38.609837+0000","last_active":"2024-08-17T17:19:51.418690+0000","last_peered":"2024-08-17T17:19:51.418690+0000","last_clean":"2024-08-17T17:19:51.418690+0000","last_became_active":"2024-08-17T17:15:38.609474+0000","last_became_peered":"2024-08-17T17:15:38.609474+0000","last_unstale":"2024-08-17T17:19:51.418690+0000","last_undegraded":"2024-08-17T17:19:51.418690+0000","last_fullsized":"2024-08-17T17:19:51.418690+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:31:42.450582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1114,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.818877+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:19:43.818877+0000","last_peered":"2024-08-17T17:19:43.818877+0000","last_clean":"2024-08-17T17:19:43.818877+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:19:43.818877+0000","last_undegraded":"2024-08-17T17:19:43.818877+0000","last_fullsized":"2024-08-17T17:19:43.818877+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":706,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.825742+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:19:43.825742+0000","last_peered":"2024-08-17T17:19:43.825742+0000","last_clean":"2024-08-17T17:19:43.825742+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:19:43.825742+0000","last_undegraded":"2024-08-17T17:19:43.825742+0000","last_fullsized":"2024-08-17T17:19:43.825742+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1410,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.827976+0000","last_change":"2024-08-17T17:18:28.526894+0000","last_active":"2024-08-17T17:19:43.827976+0000","last_peered":"2024-08-17T17:19:43.827976+0000","last_clean":"2024-08-17T17:19:43.827976+0000","last_became_active":"2024-08-17T17:18:28.526484+0000","last_became_peered":"2024-08-17T17:18:28.526484+0000","last_unstale":"2024-08-17T17:19:43.827976+0000","last_undegraded":"2024-08-17T17:19:43.827976+0000","last_fullsized":"2024-08-17T17:19:43.827976+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.3","version":"1157'6856","reported_seq":6943,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:54.629204+0000","last_change":"2024-08-17T17:19:51.547347+0000","last_active":"2024-08-17T17:19:54.629204+0000","last_peered":"2024-08-17T17:19:54.629204+0000","last_clean":"2024-08-17T17:19:54.629204+0000","last_became_active":"2024-08-17T17:19:01.681349+0000","last_became_peered":"2024-08-17T17:19:01.681349+0000","last_unstale":"2024-08-17T17:19:54.629204+0000","last_undegraded":"2024-08-17T17:19:54.629204+0000","last_fullsized":"2024-08-17T17:19:54.629204+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6856,"log_dups_size":0,"ondisk_log_size":6856,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:19:56.546853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183402496,"num_objects":2798,"num_object_clones":0,"num_object_copies":8394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4134,"num_write_kb":221856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1429,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.433130+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:19:51.433130+0000","last_peered":"2024-08-17T17:19:51.433130+0000","last_clean":"2024-08-17T17:19:51.433130+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:19:51.433130+0000","last_undegraded":"2024-08-17T17:19:51.433130+0000","last_fullsized":"2024-08-17T17:19:51.433130+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":737,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.813763+0000","last_change":"2024-08-17T17:15:38.609543+0000","last_active":"2024-08-17T17:19:43.813763+0000","last_peered":"2024-08-17T17:19:43.813763+0000","last_clean":"2024-08-17T17:19:43.813763+0000","last_became_active":"2024-08-17T17:15:38.609238+0000","last_became_peered":"2024-08-17T17:15:38.609238+0000","last_unstale":"2024-08-17T17:19:43.813763+0000","last_undegraded":"2024-08-17T17:19:43.813763+0000","last_fullsized":"2024-08-17T17:19:43.813763+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:32.029842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1419,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.414851+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:19:51.414851+0000","last_peered":"2024-08-17T17:19:51.414851+0000","last_clean":"2024-08-17T17:19:51.414851+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:19:51.414851+0000","last_undegraded":"2024-08-17T17:19:51.414851+0000","last_fullsized":"2024-08-17T17:19:51.414851+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1433,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.414030+0000","last_change":"2024-08-17T17:18:29.390118+0000","last_active":"2024-08-17T17:19:51.414030+0000","last_peered":"2024-08-17T17:19:51.414030+0000","last_clean":"2024-08-17T17:19:51.414030+0000","last_became_active":"2024-08-17T17:18:28.528874+0000","last_became_peered":"2024-08-17T17:18:28.528874+0000","last_unstale":"2024-08-17T17:19:51.414030+0000","last_undegraded":"2024-08-17T17:19:51.414030+0000","last_fullsized":"2024-08-17T17:19:51.414030+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:29.390003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:29.390003+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:27.511820+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1157'6922","reported_seq":7048,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:54.710807+0000","last_change":"2024-08-17T17:19:20.831069+0000","last_active":"2024-08-17T17:19:54.710807+0000","last_peered":"2024-08-17T17:19:54.710807+0000","last_clean":"2024-08-17T17:19:54.710807+0000","last_became_active":"2024-08-17T17:19:01.696058+0000","last_became_peered":"2024-08-17T17:19:01.696058+0000","last_unstale":"2024-08-17T17:19:54.710807+0000","last_undegraded":"2024-08-17T17:19:54.710807+0000","last_fullsized":"2024-08-17T17:19:54.710807+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":0,"last_scrub":"1154'6072","last_scrub_stamp":"2024-08-17T17:19:20.830703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:19:20.830703+0000","objects_scrubbed":3288,"log_size":6922,"log_dups_size":0,"ondisk_log_size":6922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:05:03.872989+0000","scrub_duration":1266,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189071360,"num_objects":2885,"num_object_clones":0,"num_object_copies":8655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":867,"num_write_kb":14848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":98,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.828161+0000","last_change":"2024-08-17T17:18:31.553838+0000","last_active":"2024-08-17T17:19:43.828161+0000","last_peered":"2024-08-17T17:19:43.828161+0000","last_clean":"2024-08-17T17:19:43.828161+0000","last_became_active":"2024-08-17T17:18:31.537264+0000","last_became_peered":"2024-08-17T17:18:31.537264+0000","last_unstale":"2024-08-17T17:19:43.828161+0000","last_undegraded":"2024-08-17T17:19:43.828161+0000","last_fullsized":"2024-08-17T17:19:43.828161+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:26.972836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1110,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.815212+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:19:43.815212+0000","last_peered":"2024-08-17T17:19:43.815212+0000","last_clean":"2024-08-17T17:19:43.815212+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:19:43.815212+0000","last_undegraded":"2024-08-17T17:19:43.815212+0000","last_fullsized":"2024-08-17T17:19:43.815212+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1086,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.834552+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:19:43.834552+0000","last_peered":"2024-08-17T17:19:43.834552+0000","last_clean":"2024-08-17T17:19:43.834552+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:19:43.834552+0000","last_undegraded":"2024-08-17T17:19:43.834552+0000","last_fullsized":"2024-08-17T17:19:43.834552+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.10","version":"1157'7072","reported_seq":7119,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:57.193800+0000","last_change":"2024-08-17T17:19:01.710174+0000","last_active":"2024-08-17T17:19:57.193800+0000","last_peered":"2024-08-17T17:19:57.193800+0000","last_clean":"2024-08-17T17:19:57.193800+0000","last_became_active":"2024-08-17T17:19:01.708023+0000","last_became_peered":"2024-08-17T17:19:01.708023+0000","last_unstale":"2024-08-17T17:19:57.193800+0000","last_undegraded":"2024-08-17T17:19:57.193800+0000","last_fullsized":"2024-08-17T17:19:57.193800+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7072,"log_dups_size":0,"ondisk_log_size":7072,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185040896,"num_objects":2824,"num_object_clones":0,"num_object_copies":8472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4276,"num_write_kb":227168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1100,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.827912+0000","last_change":"2024-08-17T17:06:39.627594+0000","last_active":"2024-08-17T17:19:43.827912+0000","last_peered":"2024-08-17T17:19:43.827912+0000","last_clean":"2024-08-17T17:19:43.827912+0000","last_became_active":"2024-08-17T17:06:39.627105+0000","last_became_peered":"2024-08-17T17:06:39.627105+0000","last_unstale":"2024-08-17T17:19:43.827912+0000","last_undegraded":"2024-08-17T17:19:43.827912+0000","last_fullsized":"2024-08-17T17:19:43.827912+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:30.105070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1109,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.830228+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:19:43.830228+0000","last_peered":"2024-08-17T17:19:43.830228+0000","last_clean":"2024-08-17T17:19:43.830228+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:19:43.830228+0000","last_undegraded":"2024-08-17T17:19:43.830228+0000","last_fullsized":"2024-08-17T17:19:43.830228+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.11","version":"1157'6835","reported_seq":6882,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:57.180391+0000","last_change":"2024-08-17T17:19:01.705286+0000","last_active":"2024-08-17T17:19:57.180391+0000","last_peered":"2024-08-17T17:19:57.180391+0000","last_clean":"2024-08-17T17:19:57.180391+0000","last_became_active":"2024-08-17T17:19:01.702922+0000","last_became_peered":"2024-08-17T17:19:01.702922+0000","last_unstale":"2024-08-17T17:19:57.180391+0000","last_undegraded":"2024-08-17T17:19:57.180391+0000","last_fullsized":"2024-08-17T17:19:57.180391+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6835,"log_dups_size":0,"ondisk_log_size":6835,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178978816,"num_objects":2731,"num_object_clones":0,"num_object_copies":8193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4125,"num_write_kb":219392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":40,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.823371+0000","last_change":"2024-08-17T17:17:47.283984+0000","last_active":"2024-08-17T17:19:43.823371+0000","last_peered":"2024-08-17T17:19:43.823371+0000","last_clean":"2024-08-17T17:19:43.823371+0000","last_became_active":"2024-08-17T17:17:47.283270+0000","last_became_peered":"2024-08-17T17:17:47.283270+0000","last_unstale":"2024-08-17T17:19:43.823371+0000","last_undegraded":"2024-08-17T17:19:43.823371+0000","last_fullsized":"2024-08-17T17:19:43.823371+0000","mapping_epoch":1127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:16.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1084,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.818669+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:19:43.818669+0000","last_peered":"2024-08-17T17:19:43.818669+0000","last_clean":"2024-08-17T17:19:43.818669+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:19:43.818669+0000","last_undegraded":"2024-08-17T17:19:43.818669+0000","last_fullsized":"2024-08-17T17:19:43.818669+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.12","version":"1157'6982","reported_seq":7107,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:54.734551+0000","last_change":"2024-08-17T17:19:22.720455+0000","last_active":"2024-08-17T17:19:54.734551+0000","last_peered":"2024-08-17T17:19:54.734551+0000","last_clean":"2024-08-17T17:19:54.734551+0000","last_became_active":"2024-08-17T17:19:01.711724+0000","last_became_peered":"2024-08-17T17:19:01.711724+0000","last_unstale":"2024-08-17T17:19:54.734551+0000","last_undegraded":"2024-08-17T17:19:54.734551+0000","last_fullsized":"2024-08-17T17:19:54.734551+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":5,"last_scrub":"1154'6171","last_scrub_stamp":"2024-08-17T17:19:22.720072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:19:22.720072+0000","objects_scrubbed":3360,"log_size":6982,"log_dups_size":0,"ondisk_log_size":6982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:20:40.745083+0000","scrub_duration":1168,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188284928,"num_objects":2873,"num_object_clones":0,"num_object_copies":8619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":819,"num_write_kb":10624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1096,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.824109+0000","last_change":"2024-08-17T17:06:39.625029+0000","last_active":"2024-08-17T17:19:43.824109+0000","last_peered":"2024-08-17T17:19:43.824109+0000","last_clean":"2024-08-17T17:19:43.824109+0000","last_became_active":"2024-08-17T17:06:39.624688+0000","last_became_peered":"2024-08-17T17:06:39.624688+0000","last_unstale":"2024-08-17T17:19:43.824109+0000","last_undegraded":"2024-08-17T17:19:43.824109+0000","last_fullsized":"2024-08-17T17:19:43.824109+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:54.849668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1375,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.413612+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:19:51.413612+0000","last_peered":"2024-08-17T17:19:51.413612+0000","last_clean":"2024-08-17T17:19:51.413612+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:19:51.413612+0000","last_undegraded":"2024-08-17T17:19:51.413612+0000","last_fullsized":"2024-08-17T17:19:51.413612+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.13","version":"1157'6846","reported_seq":6933,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:54.620605+0000","last_change":"2024-08-17T17:19:50.617467+0000","last_active":"2024-08-17T17:19:54.620605+0000","last_peered":"2024-08-17T17:19:54.620605+0000","last_clean":"2024-08-17T17:19:54.620605+0000","last_became_active":"2024-08-17T17:19:01.708389+0000","last_became_peered":"2024-08-17T17:19:01.708389+0000","last_unstale":"2024-08-17T17:19:54.620605+0000","last_undegraded":"2024-08-17T17:19:54.620605+0000","last_fullsized":"2024-08-17T17:19:54.620605+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6846,"log_dups_size":0,"ondisk_log_size":6846,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:19:55.593012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189038592,"num_objects":2885,"num_object_clones":0,"num_object_copies":8655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4125,"num_write_kb":224288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":951,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.404589+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:19:51.404589+0000","last_peered":"2024-08-17T17:19:51.404589+0000","last_clean":"2024-08-17T17:19:51.404589+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:19:51.404589+0000","last_undegraded":"2024-08-17T17:19:51.404589+0000","last_fullsized":"2024-08-17T17:19:51.404589+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1394,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.416739+0000","last_change":"2024-08-17T17:07:45.678950+0000","last_active":"2024-08-17T17:19:51.416739+0000","last_peered":"2024-08-17T17:19:51.416739+0000","last_clean":"2024-08-17T17:19:51.416739+0000","last_became_active":"2024-08-17T17:07:45.678472+0000","last_became_peered":"2024-08-17T17:07:45.678472+0000","last_unstale":"2024-08-17T17:19:51.416739+0000","last_undegraded":"2024-08-17T17:19:51.416739+0000","last_fullsized":"2024-08-17T17:19:51.416739+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:48:02.385927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1098,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.815145+0000","last_change":"2024-08-17T17:06:39.628542+0000","last_active":"2024-08-17T17:19:43.815145+0000","last_peered":"2024-08-17T17:19:43.815145+0000","last_clean":"2024-08-17T17:19:43.815145+0000","last_became_active":"2024-08-17T17:06:39.628253+0000","last_became_peered":"2024-08-17T17:06:39.628253+0000","last_unstale":"2024-08-17T17:19:43.815145+0000","last_undegraded":"2024-08-17T17:19:43.815145+0000","last_fullsized":"2024-08-17T17:19:43.815145+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:22.349326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1424,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.414938+0000","last_change":"2024-08-17T17:18:31.532183+0000","last_active":"2024-08-17T17:19:51.414938+0000","last_peered":"2024-08-17T17:19:51.414938+0000","last_clean":"2024-08-17T17:19:51.414938+0000","last_became_active":"2024-08-17T17:18:31.531094+0000","last_became_peered":"2024-08-17T17:18:31.531094+0000","last_unstale":"2024-08-17T17:19:51.414938+0000","last_undegraded":"2024-08-17T17:19:51.414938+0000","last_fullsized":"2024-08-17T17:19:51.414938+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:01.763127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":911,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.819986+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:19:43.819986+0000","last_peered":"2024-08-17T17:19:43.819986+0000","last_clean":"2024-08-17T17:19:43.819986+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:19:43.819986+0000","last_undegraded":"2024-08-17T17:19:43.819986+0000","last_fullsized":"2024-08-17T17:19:43.819986+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":914,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.413825+0000","last_change":"2024-08-17T17:15:38.610768+0000","last_active":"2024-08-17T17:19:51.413825+0000","last_peered":"2024-08-17T17:19:51.413825+0000","last_clean":"2024-08-17T17:19:51.413825+0000","last_became_active":"2024-08-17T17:15:38.610337+0000","last_became_peered":"2024-08-17T17:15:38.610337+0000","last_unstale":"2024-08-17T17:19:51.413825+0000","last_undegraded":"2024-08-17T17:19:51.413825+0000","last_fullsized":"2024-08-17T17:19:51.413825+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:15.632778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1393,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.415098+0000","last_change":"2024-08-17T17:07:45.689782+0000","last_active":"2024-08-17T17:19:51.415098+0000","last_peered":"2024-08-17T17:19:51.415098+0000","last_clean":"2024-08-17T17:19:51.415098+0000","last_became_active":"2024-08-17T17:07:45.689312+0000","last_became_peered":"2024-08-17T17:07:45.689312+0000","last_unstale":"2024-08-17T17:19:51.415098+0000","last_undegraded":"2024-08-17T17:19:51.415098+0000","last_fullsized":"2024-08-17T17:19:51.415098+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:07.560555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1116,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.827766+0000","last_change":"2024-08-17T17:07:12.649175+0000","last_active":"2024-08-17T17:19:43.827766+0000","last_peered":"2024-08-17T17:19:43.827766+0000","last_clean":"2024-08-17T17:19:43.827766+0000","last_became_active":"2024-08-17T17:07:12.648822+0000","last_became_peered":"2024-08-17T17:07:12.648822+0000","last_unstale":"2024-08-17T17:19:43.827766+0000","last_undegraded":"2024-08-17T17:19:43.827766+0000","last_fullsized":"2024-08-17T17:19:43.827766+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:03.924431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1442,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.418783+0000","last_change":"2024-08-17T17:17:47.316690+0000","last_active":"2024-08-17T17:19:51.418783+0000","last_peered":"2024-08-17T17:19:51.418783+0000","last_clean":"2024-08-17T17:19:51.418783+0000","last_became_active":"2024-08-17T17:17:47.316333+0000","last_became_peered":"2024-08-17T17:17:47.316333+0000","last_unstale":"2024-08-17T17:19:51.418783+0000","last_undegraded":"2024-08-17T17:19:51.418783+0000","last_fullsized":"2024-08-17T17:19:51.418783+0000","mapping_epoch":1127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:04.542237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":103,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.827680+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:19:43.827680+0000","last_peered":"2024-08-17T17:19:43.827680+0000","last_clean":"2024-08-17T17:19:43.827680+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:19:43.827680+0000","last_undegraded":"2024-08-17T17:19:43.827680+0000","last_fullsized":"2024-08-17T17:19:43.827680+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.18","version":"1157'6876","reported_seq":6964,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:56.312508+0000","last_change":"2024-08-17T17:19:55.471555+0000","last_active":"2024-08-17T17:19:56.312508+0000","last_peered":"2024-08-17T17:19:56.312508+0000","last_clean":"2024-08-17T17:19:56.312508+0000","last_became_active":"2024-08-17T17:19:01.702866+0000","last_became_peered":"2024-08-17T17:19:01.702866+0000","last_unstale":"2024-08-17T17:19:56.312508+0000","last_undegraded":"2024-08-17T17:19:56.312508+0000","last_fullsized":"2024-08-17T17:19:56.312508+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":6876,"log_dups_size":0,"ondisk_log_size":6876,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:20:00.427773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185401344,"num_objects":2829,"num_object_clones":0,"num_object_copies":8487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4117,"num_write_kb":222272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1114,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.827830+0000","last_change":"2024-08-17T17:07:45.684606+0000","last_active":"2024-08-17T17:19:43.827830+0000","last_peered":"2024-08-17T17:19:43.827830+0000","last_clean":"2024-08-17T17:19:43.827830+0000","last_became_active":"2024-08-17T17:07:45.684124+0000","last_became_peered":"2024-08-17T17:07:45.684124+0000","last_unstale":"2024-08-17T17:19:43.827830+0000","last_undegraded":"2024-08-17T17:19:43.827830+0000","last_fullsized":"2024-08-17T17:19:43.827830+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:16.110507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1098,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.823665+0000","last_change":"2024-08-17T17:06:39.625245+0000","last_active":"2024-08-17T17:19:43.823665+0000","last_peered":"2024-08-17T17:19:43.823665+0000","last_clean":"2024-08-17T17:19:43.823665+0000","last_became_active":"2024-08-17T17:06:39.624657+0000","last_became_peered":"2024-08-17T17:06:39.624657+0000","last_unstale":"2024-08-17T17:19:43.823665+0000","last_undegraded":"2024-08-17T17:19:43.823665+0000","last_fullsized":"2024-08-17T17:19:43.823665+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:43.097460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.19","version":"1157'7022","reported_seq":8558,"reported_epoch":1157,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:19:57.100992+0000","last_change":"2024-08-17T17:19:01.771867+0000","last_active":"2024-08-17T17:19:57.100992+0000","last_peered":"2024-08-17T17:19:57.100992+0000","last_clean":"2024-08-17T17:18:30.513360+0000","last_became_active":"2024-08-17T17:19:01.766350+0000","last_became_peered":"2024-08-17T17:19:01.766350+0000","last_unstale":"2024-08-17T17:19:57.100992+0000","last_undegraded":"2024-08-17T17:19:01.712084+0000","last_fullsized":"2024-08-17T17:19:57.100992+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7022,"log_dups_size":0,"ondisk_log_size":7022,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182255616,"num_objects":2781,"num_object_clones":0,"num_object_copies":8343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1349,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4261,"num_write_kb":225344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":741,"num_bytes_recovered":48529408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"1,4,7","objects":1349}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1132,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.827505+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:19:43.827505+0000","last_peered":"2024-08-17T17:19:43.827505+0000","last_clean":"2024-08-17T17:19:43.827505+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:19:43.827505+0000","last_undegraded":"2024-08-17T17:19:43.827505+0000","last_fullsized":"2024-08-17T17:19:43.827505+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1083,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.823776+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:19:43.823776+0000","last_peered":"2024-08-17T17:19:43.823776+0000","last_clean":"2024-08-17T17:19:43.823776+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:19:43.823776+0000","last_undegraded":"2024-08-17T17:19:43.823776+0000","last_fullsized":"2024-08-17T17:19:43.823776+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1414,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.413971+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:19:51.413971+0000","last_peered":"2024-08-17T17:19:51.413971+0000","last_clean":"2024-08-17T17:19:51.413971+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:19:51.413971+0000","last_undegraded":"2024-08-17T17:19:51.413971+0000","last_fullsized":"2024-08-17T17:19:51.413971+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1389,"reported_epoch":1157,"state":"active+clean","last_fresh":"2024-08-17T17:19:51.413799+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:19:51.413799+0000","last_peered":"2024-08-17T17:19:51.413799+0000","last_clean":"2024-08-17T17:19:51.413799+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:19:51.413799+0000","last_undegraded":"2024-08-17T17:19:51.413799+0000","last_fullsized":"2024-08-17T17:19:51.413799+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1099,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.828131+0000","last_change":"2024-08-17T17:06:39.627689+0000","last_active":"2024-08-17T17:19:43.828131+0000","last_peered":"2024-08-17T17:19:43.828131+0000","last_clean":"2024-08-17T17:19:43.828131+0000","last_became_active":"2024-08-17T17:06:39.627375+0000","last_became_peered":"2024-08-17T17:06:39.627375+0000","last_unstale":"2024-08-17T17:19:43.828131+0000","last_undegraded":"2024-08-17T17:19:43.828131+0000","last_fullsized":"2024-08-17T17:19:43.828131+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:51.954076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":126,"reported_epoch":1156,"state":"active+clean","last_fresh":"2024-08-17T17:19:43.824055+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:19:43.824055+0000","last_peered":"2024-08-17T17:19:43.824055+0000","last_clean":"2024-08-17T17:19:43.824055+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:19:43.824055+0000","last_undegraded":"2024-08-17T17:19:43.824055+0000","last_fullsized":"2024-08-17T17:19:43.824055+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":5899485208,"num_objects":90020,"num_object_clones":0,"num_object_copies":270060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21623,"num_objects_misplaced":2738,"num_objects_unfound":0,"num_objects_dirty":90020,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":127514,"num_write_kb":6748289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8587,"num_bytes_recovered":562757632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18493485056,"data_stored":18493472840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":182228,"ondisk_log_size":182228,"up":78,"acting":75,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":912388,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":210,"num_read_kb":453,"num_write":118,"num_write_kb":1319,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1073397,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201622,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2361292,"kb_used_data":2069380,"kb_used_omap":1,"kb_used_meta":291902,"kb_avail":91383860,"statfs":{"total":95995035648,"available":93577072640,"internally_reserved":0,"allocated":2119045120,"data_stored":2116882613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":298908107},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,1,3,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463259,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2307976,"kb_used_data":2052160,"kb_used_omap":1,"kb_used_meta":255806,"kb_avail":91437176,"statfs":{"total":95995035648,"available":93631668224,"internally_reserved":0,"allocated":2101411840,"data_stored":2099253326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":261945801},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626900,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4166776,"kb_used_data":3809584,"kb_used_omap":42,"kb_used_meta":356949,"kb_avail":89578376,"statfs":{"total":95995035648,"available":91728257024,"internally_reserved":0,"allocated":3901014016,"data_stored":3898867663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":365515836},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,4,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785377,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3030204,"kb_used_data":2773040,"kb_used_omap":48,"kb_used_meta":257103,"kb_avail":90714948,"statfs":{"total":95995035648,"available":92892106752,"internally_reserved":0,"allocated":2839592960,"data_stored":2837446016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":263274158},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724895,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1973632,"kb_used_data":1726008,"kb_used_omap":1,"kb_used_meta":247614,"kb_avail":91771520,"statfs":{"total":95995035648,"available":93974036480,"internally_reserved":0,"allocated":1767432192,"data_stored":1765278216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":253557190},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986532,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2342984,"kb_used_data":2055168,"kb_used_omap":1,"kb_used_meta":287806,"kb_avail":91402168,"statfs":{"total":95995035648,"available":93595820032,"internally_reserved":0,"allocated":2104492032,"data_stored":2102333551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":294713798},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706139,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2476284,"kb_used_data":2191604,"kb_used_omap":35,"kb_used_meta":284636,"kb_avail":91268868,"statfs":{"total":95995035648,"available":93459320832,"internally_reserved":0,"allocated":2244202496,"data_stored":2242052573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":291467319},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,0,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831912,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1752956,"kb_used_data":1503924,"kb_used_omap":65,"kb_used_meta":248958,"kb_avail":91992196,"statfs":{"total":95995035648,"available":94200008704,"internally_reserved":0,"allocated":1540018176,"data_stored":1537867721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":254933179},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1532497920,"data_stored":1532493848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2818117632,"data_stored":2818113560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2220818432,"data_stored":2220818432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3887595520,"data_stored":3887595520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2091712512,"data_stored":2091712512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1746862080,"data_stored":1746862080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2090995712,"data_stored":2090991640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2104885248,"data_stored":2104885248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:20:04.362 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set .mgr pgp_num 11 2024-08-17T17:20:04.402 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:04.427 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:04.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:04.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:04.455 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:20:04.528 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:04.554 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:04.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:04.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:04.656 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:04.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:04.724 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:04.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:04.739 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:04 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1383550791' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:04.739 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:04 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:04.740 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:04 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1383550791' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:04.740 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:04 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:04.823 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:20:04.823 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:20:04.823 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:20:04.823 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:20:04.823 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:20:04.823 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:20:04.825 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:04.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:04 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1383550791' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:04.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:04 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:04.849 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:04.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:04.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:04.856 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:20:04.856 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:20:04.886 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:20:04.887 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:20:04.919 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:20:04.919 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:20:04.950 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:20:04.950 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:20:04.951 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:04.983 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:20:04.983 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:20:04.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:04.992 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:04.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:05.093 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:05.119 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:05.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:05.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:05.221 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:05.247 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:05.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:05.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:05.348 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:05.387 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:05.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:05.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:05.488 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:05.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:05.517 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:05.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:05.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:05 smithi063 ceph-mon[114756]: pgmap v1402: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 20455/258393 objects degraded (7.916%); 2552/258393 objects misplaced (0.988%); 17 MiB/s, 266 objects/s recovering 2024-08-17T17:20:05.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:05 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:05.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:05 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:05.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:05 smithi063 ceph-mon[114756]: osdmap e1158: 8 total, 8 up, 8 in 2024-08-17T17:20:05.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:05 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4262296825' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:05.535 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:05 smithi063 ceph-mon[109843]: pgmap v1402: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 20455/258393 objects degraded (7.916%); 2552/258393 objects misplaced (0.988%); 17 MiB/s, 266 objects/s recovering 2024-08-17T17:20:05.535 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:05 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:05.535 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:05 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:05.535 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:05 smithi063 ceph-mon[109843]: osdmap e1158: 8 total, 8 up, 8 in 2024-08-17T17:20:05.535 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:05 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4262296825' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:05.536 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:05.618 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:05.652 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:05.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:05.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:05.754 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:05.783 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:05.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:05.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:05 smithi162 ceph-mon[104668]: pgmap v1402: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 20455/258393 objects degraded (7.916%); 2552/258393 objects misplaced (0.988%); 17 MiB/s, 266 objects/s recovering 2024-08-17T17:20:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:05 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:05 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:05 smithi162 ceph-mon[104668]: osdmap e1158: 8 total, 8 up, 8 in 2024-08-17T17:20:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:05 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4262296825' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:05.886 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:05.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:05.927 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:05.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:06.028 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:06.097 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:06.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:06.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:06.099 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:06.198 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:06.239 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:06.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:06.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:06.341 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:06.388 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:06.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:06.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:06.490 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:06.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:06.527 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:06.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:06.628 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:06.655 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:06.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:06.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:06.756 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:06.781 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:06.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:06.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:06.883 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:06.921 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:06.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:06.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:07.023 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:07.050 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:07.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:07.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:07.152 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:07.177 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:07.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:07.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:07.280 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:07.311 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:20:07.317 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:07.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:07.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:07.418 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:07.459 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:07.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:07.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:07.561 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:07.606 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:07.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:07.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:07.615 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:07 smithi063 ceph-mon[109843]: pgmap v1404: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 20455/255345 objects degraded (8.011%); 2552/255345 objects misplaced (0.999%); 17 MiB/s, 264 objects/s recovering 2024-08-17T17:20:07.616 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:07 smithi063 ceph-mon[114756]: pgmap v1404: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 20455/255345 objects degraded (8.011%); 2552/255345 objects misplaced (0.999%); 17 MiB/s, 264 objects/s recovering 2024-08-17T17:20:07.708 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:07.744 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:07.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:07.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:07.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:07 smithi162 ceph-mon[104668]: pgmap v1404: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 20455/255345 objects degraded (8.011%); 2552/255345 objects misplaced (0.999%); 17 MiB/s, 264 objects/s recovering 2024-08-17T17:20:07.847 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:07.883 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:07.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:07.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:07.985 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:08.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:08.022 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:08.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:08.065 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:20:08.124 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:08.190 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:08.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:08.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:08.291 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:08.347 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:08.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:08.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:08.448 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:08.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:08.503 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:08.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:08.604 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:08.688 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:08.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:08.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:08.728 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:08 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 20455/255345 objects degraded (8.011%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:08.728 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:08 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 20455/255345 objects degraded (8.011%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:08.791 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:08.835 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:08.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:08.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:08.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:08 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 20455/255345 objects degraded (8.011%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:08.940 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:08.978 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:08.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:08.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:09.080 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:09.109 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:09.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:09.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:09.211 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:09.257 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:09.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:09.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:09.359 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:09.405 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:09.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:09.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:09.507 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:09.531 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:09.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:09.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:09.631 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:09 smithi063 ceph-mon[114756]: pgmap v1405: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 19368/246120 objects degraded (7.869%); 2289/246120 objects misplaced (0.930%); 17 MiB/s, 270 objects/s recovering 2024-08-17T17:20:09.631 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:09 smithi063 ceph-mon[109843]: pgmap v1405: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 19368/246120 objects degraded (7.869%); 2289/246120 objects misplaced (0.930%); 17 MiB/s, 270 objects/s recovering 2024-08-17T17:20:09.633 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:09.658 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:09.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:09.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:09.696 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:20:09.696 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:20:09.760 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:09.782 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:09.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:09.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:09.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:09 smithi162 ceph-mon[104668]: pgmap v1405: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 800 op/s; 19368/246120 objects degraded (7.869%); 2289/246120 objects misplaced (0.930%); 17 MiB/s, 270 objects/s recovering 2024-08-17T17:20:09.884 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:09.934 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:09.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:09.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:10.036 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:10.071 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:10.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:10.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:10.172 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:10.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:10.217 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:10.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:10.318 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:10.362 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:10.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:10.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:10.431 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:10.435 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:10.464 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:10.484 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:20:10.488 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:10.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:10.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:10.506 INFO:teuthology.orchestra.run.smithi063.stderr:set pool 1 pgp_num to 11 2024-08-17T17:20:10.589 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:10.634 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:10.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:10.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:10.736 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:10.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:10 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1893081920' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-08-17T17:20:10.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:10 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-08-17T17:20:10.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:10 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1893081920' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-08-17T17:20:10.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:10 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-08-17T17:20:10.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:10.790 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:10.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:10.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:10 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1893081920' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-08-17T17:20:10.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:10 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-08-17T17:20:10.891 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:10.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:10.922 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:10.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:11.023 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:11.048 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:11.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:11.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:11.150 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:11.190 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:11.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:11.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:11.292 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:11.317 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:11.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:11.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:11.419 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:11.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:11.460 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:11.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:11.560 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:11.587 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:11.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:11.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:11.689 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:11.715 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:11.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:11.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:11.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:11 smithi063 ceph-mon[114756]: pgmap v1406: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 19368/243150 objects degraded (7.965%); 2289/243150 objects misplaced (0.941%); 17 MiB/s, 270 objects/s recovering 2024-08-17T17:20:11.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:11 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-08-17T17:20:11.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:11 smithi063 ceph-mon[114756]: osdmap e1159: 8 total, 8 up, 8 in 2024-08-17T17:20:11.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:11 smithi063 ceph-mon[109843]: pgmap v1406: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 19368/243150 objects degraded (7.965%); 2289/243150 objects misplaced (0.941%); 17 MiB/s, 270 objects/s recovering 2024-08-17T17:20:11.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:11 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-08-17T17:20:11.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:11 smithi063 ceph-mon[109843]: osdmap e1159: 8 total, 8 up, 8 in 2024-08-17T17:20:11.816 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:11.843 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:11.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:11.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:11 smithi162 ceph-mon[104668]: pgmap v1406: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 19368/243150 objects degraded (7.965%); 2289/243150 objects misplaced (0.941%); 17 MiB/s, 270 objects/s recovering 2024-08-17T17:20:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:11 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-08-17T17:20:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:11 smithi162 ceph-mon[104668]: osdmap e1159: 8 total, 8 up, 8 in 2024-08-17T17:20:11.946 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:11.975 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:11.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:11.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:12.077 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:12.114 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:12.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:12.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:12.215 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:12.242 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:12.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:12.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:12.344 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:12.380 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:12.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:12.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:12.482 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:12.520 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:12.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:12.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:12.622 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:12.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:12.647 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:12.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:12.748 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:12.790 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:12.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:12.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:13.075 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:13.103 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:13.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:13.205 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:13.232 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:13.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.233 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:13.334 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:13.360 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:13.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:13.462 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:13.505 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:13.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:13.607 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:13.646 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:13.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:13.700 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:20:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:20:13] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.51.0" 2024-08-17T17:20:13.701 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:13 smithi063 ceph-mon[114756]: pgmap v1408: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 18796/237324 objects degraded (7.920%); 2032/237324 objects misplaced (0.856%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:13.701 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:13 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 18796/237324 objects degraded (7.920%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:13.701 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:13 smithi063 ceph-mon[109843]: pgmap v1408: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 18796/237324 objects degraded (7.920%); 2032/237324 objects misplaced (0.856%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:13.701 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:13 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 18796/237324 objects degraded (7.920%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:13.748 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:13.822 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:13.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:13.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:13 smithi162 ceph-mon[104668]: pgmap v1408: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 877 op/s; 18796/237324 objects degraded (7.920%); 2032/237324 objects misplaced (0.856%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:13 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 18796/237324 objects degraded (7.920%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:13.924 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:13.962 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:13.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:13.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:14.064 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:14.092 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:14.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:14.194 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:14.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.232 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:14.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:14.333 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:14.350 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:14.359 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:14.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:14.461 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:14.488 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:14.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:14.590 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:14.617 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:14 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:20:14.618 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:14 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:20:14.649 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:14.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:14.750 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:14.794 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:14.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:14.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:14 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:20:14.897 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:14.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:14.936 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:14.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:15.037 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:15.083 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:15.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:15.185 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:15.210 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:15.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:15.264 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:20:15.265 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:20:15.310 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:20:15.311 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:20:15.312 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:15.341 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:15.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:15.352 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:20:15.352 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:20:15.400 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:20:15.401 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:20:15.443 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:15.447 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:20:15.447 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:20:15.488 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:20:15.489 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:20:15.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.506 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:15.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:15.534 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:20:15.535 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:20:15.574 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:20:15.575 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:20:15.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:15 smithi162 ceph-mon[104668]: pgmap v1409: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 18312/234453 objects degraded (7.811%); 2032/234453 objects misplaced (0.867%); 17 MiB/s, 273 objects/s recovering 2024-08-17T17:20:15.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:15 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:15.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:15 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:15.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:15 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3143617917' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:15.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:15 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:15.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:15 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:15.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:15 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/819532032' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:15.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:15 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:15.606 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:15.631 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:15.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:15.733 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:15.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.796 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:15.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:15.845 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[114756]: pgmap v1409: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 18312/234453 objects degraded (7.811%); 2032/234453 objects misplaced (0.867%); 17 MiB/s, 273 objects/s recovering 2024-08-17T17:20:15.845 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:15.845 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:15.846 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3143617917' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:15.846 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:15.846 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:15.846 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/819532032' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:15.846 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:15.846 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[109843]: pgmap v1409: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 18312/234453 objects degraded (7.811%); 2032/234453 objects misplaced (0.867%); 17 MiB/s, 273 objects/s recovering 2024-08-17T17:20:15.847 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:15.847 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:15.847 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3143617917' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:15.847 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:15.847 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:15.847 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/819532032' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:15.847 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:15 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:20:15.897 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:15.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:15.931 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:15.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:16.032 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:16.065 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:16.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:16.168 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:16.207 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:16.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:16.304 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:20:16.309 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:16.355 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:16.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:16.457 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:16.482 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:16.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:16.584 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:16.624 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:16.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:16.726 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:16.738 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:16 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:16.738 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:16 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:20:16.738 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:16 smithi063 ceph-mon[109843]: osdmap e1160: 8 total, 8 up, 8 in 2024-08-17T17:20:16.739 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:16.739 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:16 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:16.739 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:16 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:20:16.739 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:16 smithi063 ceph-mon[114756]: osdmap e1160: 8 total, 8 up, 8 in 2024-08-17T17:20:16.739 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:16.759 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:16.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:16 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:16 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:20:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:16 smithi162 ceph-mon[104668]: osdmap e1160: 8 total, 8 up, 8 in 2024-08-17T17:20:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:16.861 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:16.901 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:16.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:16.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:17.002 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:17.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.044 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:17.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:17.145 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:17.173 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:17.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.174 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:17.275 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:17.305 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:17.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:17.407 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:17.447 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:17.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:17.549 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:17.603 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:17.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:17.605 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:17 smithi162 ceph-mon[104668]: pgmap v1410: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 18312/231516 objects degraded (7.910%); 2032/231516 objects misplaced (0.878%); 17 MiB/s, 266 objects/s recovering 2024-08-17T17:20:17.705 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:17.731 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:17.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:17.833 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:17.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.868 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:17.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:17.969 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:17.975 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:20:17.976 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:20:17.976 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_2 2024-08-17T17:20:17.976 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:20:17.998 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:17.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:17.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:18.004 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:17 smithi063 ceph-mon[109843]: pgmap v1410: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 18312/231516 objects degraded (7.910%); 2032/231516 objects misplaced (0.878%); 17 MiB/s, 266 objects/s recovering 2024-08-17T17:20:18.005 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:17 smithi063 ceph-mon[114756]: pgmap v1410: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 18312/231516 objects degraded (7.910%); 2032/231516 objects misplaced (0.878%); 17 MiB/s, 266 objects/s recovering 2024-08-17T17:20:18.100 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:18.184 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:18.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:18.285 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:18.316 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:18.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:18.418 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:18.446 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:18.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:18.547 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:18.586 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:18.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:18.688 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:18.716 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:18.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:18.818 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:18.842 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:18.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:18.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:18 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 18312/231516 objects degraded (7.910%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:18.944 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:18.986 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:18.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:18.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:18 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 18312/231516 objects degraded (7.910%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:19.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:18 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 18312/231516 objects degraded (7.910%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:19.088 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:19.125 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:19.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:19.147 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:20:19.227 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:19.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.278 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:19.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:19.379 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:19.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.432 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:19.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:19.533 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:19.566 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:19.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:19.574 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:20:19.668 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:19.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.717 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:19.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:19.818 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:19.839 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:19.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:19 smithi162 ceph-mon[104668]: pgmap v1412: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 17168/222840 objects degraded (7.704%); 1773/222840 objects misplaced (0.796%); 21 MiB/s, 339 objects/s recovering 2024-08-17T17:20:19.941 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:19.975 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:19.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:19.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:19 smithi063 ceph-mon[114756]: pgmap v1412: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 17168/222840 objects degraded (7.704%); 1773/222840 objects misplaced (0.796%); 21 MiB/s, 339 objects/s recovering 2024-08-17T17:20:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:19 smithi063 ceph-mon[109843]: pgmap v1412: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 17168/222840 objects degraded (7.704%); 1773/222840 objects misplaced (0.796%); 21 MiB/s, 339 objects/s recovering 2024-08-17T17:20:20.077 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:20.119 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:20.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:20.221 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:20.268 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:20.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:20.371 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:20.421 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:20.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:20.523 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:20.570 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:20.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:20.671 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:20.694 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:20.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:20.795 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:20.832 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:20.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:20.934 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:20.972 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:20.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:20.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:20 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:20 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:21.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:20 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:21.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:20 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:21.074 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:21.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:20 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:21.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:20 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:21.102 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:21.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:21.204 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:21.228 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:21.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:21.330 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:21.354 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:21.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:21.456 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:21.483 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:21.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:21.585 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:21.624 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:21.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:21.725 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:21.755 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:21.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:21.857 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:21.885 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:21.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:21.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:21.960 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:20:21.961 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:20:21.961 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:20:21.961 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:20:21.961 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:20:21.961 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:21 smithi063 ceph-mon[114756]: pgmap v1413: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 17168/220176 objects degraded (7.797%); 1773/220176 objects misplaced (0.805%); 17 MiB/s, 279 objects/s recovering 2024-08-17T17:20:21.961 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:21 smithi063 ceph-mon[109843]: pgmap v1413: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 17168/220176 objects degraded (7.797%); 1773/220176 objects misplaced (0.805%); 17 MiB/s, 279 objects/s recovering 2024-08-17T17:20:21.987 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:22.025 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:22.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:21 smithi162 ceph-mon[104668]: pgmap v1413: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 17168/220176 objects degraded (7.797%); 1773/220176 objects misplaced (0.805%); 17 MiB/s, 279 objects/s recovering 2024-08-17T17:20:22.127 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:22.154 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:22.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:22.256 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:22.277 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:22.286 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:22.287 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:22.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.330 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:22.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:22.431 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:22.469 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:22.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.470 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:22.571 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:22.614 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:22.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:22.716 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:22.742 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:22.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:22.844 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:22.890 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:22.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:22.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:22.992 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:23.020 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:23.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:23.122 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:23.124 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:20:23.215 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:23.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:23.317 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:23.340 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:20:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:20:23] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-08-17T17:20:23.342 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:23.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:23.443 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:23.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.468 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:23.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:23.569 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:23.593 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:23.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:23.695 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:23.722 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:23.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:23.824 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:23.850 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:23.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:23.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:23.951 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:24.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.007 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:24.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:24.007 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:23 smithi063 ceph-mon[114756]: pgmap v1414: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 16552/214530 objects degraded (7.715%); 1534/214530 objects misplaced (0.715%); 17 MiB/s, 273 objects/s recovering 2024-08-17T17:20:24.007 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:23 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 16552/214530 objects degraded (7.715%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:24.008 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:23 smithi063 ceph-mon[109843]: pgmap v1414: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 16552/214530 objects degraded (7.715%); 1534/214530 objects misplaced (0.715%); 17 MiB/s, 273 objects/s recovering 2024-08-17T17:20:24.008 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:23 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 16552/214530 objects degraded (7.715%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:23 smithi162 ceph-mon[104668]: pgmap v1414: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 16552/214530 objects degraded (7.715%); 1534/214530 objects misplaced (0.715%); 17 MiB/s, 273 objects/s recovering 2024-08-17T17:20:24.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:23 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 16552/214530 objects degraded (7.715%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:24.108 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:24.166 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:24.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:24.268 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:24.297 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:24.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:24.399 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:24.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.452 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:24.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:24.553 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:24.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.598 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:24.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:24.699 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:24.724 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:24.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:24.826 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:24.852 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:24.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:24.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:24.954 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:25.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.001 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:25.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:25.102 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:25.128 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:25.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:25.230 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:25.255 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:25.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:25.357 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:25.396 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:25.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:25.498 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:25.568 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:25.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:25.670 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:25.698 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:25.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:25.800 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:25.826 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:25.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:25.928 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:25.953 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:25.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:25.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:25.960 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:25 smithi063 ceph-mon[109843]: pgmap v1415: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 16077/211740 objects degraded (7.593%); 1534/211740 objects misplaced (0.724%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:25.960 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:25 smithi063 ceph-mon[114756]: pgmap v1415: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 16077/211740 objects degraded (7.593%); 1534/211740 objects misplaced (0.724%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:26.055 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:26.095 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:26.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:25 smithi162 ceph-mon[104668]: pgmap v1415: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 16077/211740 objects degraded (7.593%); 1534/211740 objects misplaced (0.724%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:26.197 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:26.225 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:26.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:26.326 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:26.359 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:26.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:26.461 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:26.501 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:26.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:26.603 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:26.643 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:26.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:26.745 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:26.767 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:26.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:26.869 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:26.910 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:26.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:26.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:27.012 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:27.056 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:27.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:27.093 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:20:27.093 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:20:27.093 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:20:27.093 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:20:27.093 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:20:27.093 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:20:27.094 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:20:27.094 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:20:27.094 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:20:27.094 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:20:27.094 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:20:27.094 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:20:27.099 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:20:27.100 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:20:27.137 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:20:27.137 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:20:27.158 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:27.181 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:27.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:27.283 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:27.320 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:27.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.321 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:27.422 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:27.458 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:27.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:27.560 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:27.595 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:27.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:27.698 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:27.735 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:27.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:27.837 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:27.875 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:27.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:27.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:27.977 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:28.018 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:28.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:27 smithi063 ceph-mon[114756]: pgmap v1416: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 16077/209025 objects degraded (7.691%); 1534/209025 objects misplaced (0.734%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:27 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1715303106' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:27 smithi063 ceph-mon[109843]: pgmap v1416: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 16077/209025 objects degraded (7.691%); 1534/209025 objects misplaced (0.734%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:28.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:27 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1715303106' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:27 smithi162 ceph-mon[104668]: pgmap v1416: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 16077/209025 objects degraded (7.691%); 1534/209025 objects misplaced (0.734%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:27 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1715303106' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:28.120 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:28.149 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:28.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.150 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:28.251 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:28.276 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:28.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:28.378 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:28.416 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:28.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:28.518 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:28.555 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:28.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:28.657 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:28.697 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:28.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:28.799 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:28.838 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:28.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:28.940 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:28.965 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:28.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:28.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:29.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:28 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 16077/209025 objects degraded (7.691%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:29.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:28 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 16077/209025 objects degraded (7.691%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:29.067 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:29.094 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:29.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:29.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:28 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 16077/209025 objects degraded (7.691%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:29.196 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:29.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.225 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:29.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:29.326 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:29.369 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:29.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:29.470 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:29.510 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:29.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:29.612 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:29.639 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:29.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:29.740 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:29.766 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:29.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:29.867 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:29.893 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:29.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:29.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:29.995 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:30.020 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:30.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:30.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:29 smithi063 ceph-mon[109843]: pgmap v1417: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 14904/200580 objects degraded (7.430%); 1323/200580 objects misplaced (0.660%); 22 MiB/s, 345 objects/s recovering 2024-08-17T17:20:30.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:29 smithi063 ceph-mon[114756]: pgmap v1417: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 14904/200580 objects degraded (7.430%); 1323/200580 objects misplaced (0.660%); 22 MiB/s, 345 objects/s recovering 2024-08-17T17:20:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:29 smithi162 ceph-mon[104668]: pgmap v1417: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 14904/200580 objects degraded (7.430%); 1323/200580 objects misplaced (0.660%); 22 MiB/s, 345 objects/s recovering 2024-08-17T17:20:30.122 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:30.149 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:30.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:30.251 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:30.288 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:30.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:30.390 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:30.429 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:30.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:30.531 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:30.558 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:30.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:30.660 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:30.688 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:30.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:30.791 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:30.859 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:30.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:30.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:30.961 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:31.001 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:31.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:31.103 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:31.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.148 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:31.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:31.249 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:31.273 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:31.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:31.375 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:31.398 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:31.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:31.499 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:31.554 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:31.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:31.602 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:31.656 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:31.681 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:31.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:31.750 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:31 smithi063 ceph-mon[109843]: pgmap v1418: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 14904/197640 objects degraded (7.541%); 1323/197640 objects misplaced (0.669%); 14 MiB/s, 225 objects/s recovering 2024-08-17T17:20:31.750 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:31.750 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:31 smithi063 ceph-mon[114756]: pgmap v1418: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 14904/197640 objects degraded (7.541%); 1323/197640 objects misplaced (0.669%); 14 MiB/s, 225 objects/s recovering 2024-08-17T17:20:31.750 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:31.755 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:20:31.757 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:20:31.783 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:31.806 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:31.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:31.907 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:31.959 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:31.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:31.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:32.060 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:32.090 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:32.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:31 smithi162 ceph-mon[104668]: pgmap v1418: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 14904/197640 objects degraded (7.541%); 1323/197640 objects misplaced (0.669%); 14 MiB/s, 225 objects/s recovering 2024-08-17T17:20:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:32.192 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:32.240 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:32.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:32.262 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:20:32.342 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:32.367 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:32.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:32.469 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:32.494 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:32.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:32.595 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:32.645 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:32.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:32.742 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:20:32.747 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:32.772 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:32.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:32.873 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:32.898 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:32.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:32.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:33.000 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:32 smithi063 ceph-mon[109843]: from='client.35127 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:20:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:32 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3999499647' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:33.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:32 smithi063 ceph-mon[114756]: from='client.35127 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:20:33.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:32 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3999499647' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:33.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.042 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:33.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:32 smithi162 ceph-mon[104668]: from='client.35127 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:20:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:32 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3999499647' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:33.143 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:33.163 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1418,"stamp":"2024-08-17T17:20:30.214692+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4316594657,"num_objects":65880,"num_object_clones":0,"num_object_copies":197640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14904,"num_objects_misplaced":1323,"num_objects_unfound":0,"num_objects_dirty":65880,"num_whiteouts":0,"num_read":211,"num_read_kb":454,"num_write":151863,"num_write_kb":6749641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16745,"num_bytes_recovered":1096902061,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":199493,"ondisk_log_size":199493,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":341,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16616932,"kb_used_data":14249700,"kb_used_omap":199,"kb_used_meta":2366904,"kb_avail":733344284,"statfs":{"total":767960285184,"available":750944546816,"internally_reserved":0,"allocated":14591692800,"data_stored":14574375415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":2423710460},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,3,3,9,8,3,7,4],"upper_bound":256},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-550502400,"num_objects":-8400,"num_object_clones":0,"num_object_copies":-25200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2264,"num_objects_misplaced":-450,"num_objects_unfound":0,"num_objects_dirty":-8400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8400,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2714,"num_bytes_recovered":177864704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.015321"},"pg_stats":[{"pgid":"7.17","version":"1160'7484","reported_seq":10377,"reported_epoch":1160,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:20:27.111595+0000","last_change":"2024-08-17T17:19:01.756567+0000","last_active":"2024-08-17T17:20:27.111595+0000","last_peered":"2024-08-17T17:20:27.111595+0000","last_clean":"2024-08-17T17:18:30.516571+0000","last_became_active":"2024-08-17T17:19:01.750697+0000","last_became_peered":"2024-08-17T17:19:01.750697+0000","last_unstale":"2024-08-17T17:20:27.111595+0000","last_undegraded":"2024-08-17T17:19:01.715352+0000","last_fullsized":"2024-08-17T17:20:27.111595+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7484,"log_dups_size":0,"ondisk_log_size":7484,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128843776,"num_objects":1966,"num_object_clones":0,"num_object_copies":5898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":562,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4804,"num_write_kb":216640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1419,"num_bytes_recovered":92995584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"0,3,6","objects":1404},{"shards":"1,3,6","objects":562}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1400,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.304079+0000","last_change":"2024-08-17T17:07:45.684223+0000","last_active":"2024-08-17T17:20:16.304079+0000","last_peered":"2024-08-17T17:20:16.304079+0000","last_clean":"2024-08-17T17:20:16.304079+0000","last_became_active":"2024-08-17T17:07:45.683948+0000","last_became_peered":"2024-08-17T17:07:45.683948+0000","last_unstale":"2024-08-17T17:20:16.304079+0000","last_undegraded":"2024-08-17T17:20:16.304079+0000","last_fullsized":"2024-08-17T17:20:16.304079+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:07.677423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1087,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.500744+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:20:10.500744+0000","last_peered":"2024-08-17T17:20:10.500744+0000","last_clean":"2024-08-17T17:20:10.500744+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:20:10.500744+0000","last_undegraded":"2024-08-17T17:20:10.500744+0000","last_fullsized":"2024-08-17T17:20:10.500744+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.16","version":"1160'7759","reported_seq":10735,"reported_epoch":1160,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:20:27.219628+0000","last_change":"2024-08-17T17:19:01.787015+0000","last_active":"2024-08-17T17:20:27.219628+0000","last_peered":"2024-08-17T17:20:27.219628+0000","last_clean":"2024-08-17T17:18:30.519687+0000","last_became_active":"2024-08-17T17:19:01.765205+0000","last_became_peered":"2024-08-17T17:19:01.765205+0000","last_unstale":"2024-08-17T17:20:27.219628+0000","last_undegraded":"2024-08-17T17:19:01.759821+0000","last_fullsized":"2024-08-17T17:20:27.219628+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7759,"log_dups_size":0,"ondisk_log_size":7759,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135725056,"num_objects":2071,"num_object_clones":0,"num_object_copies":6213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":650,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4993,"num_write_kb":226048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1461,"num_bytes_recovered":95748096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"0,3,7","objects":1421},{"shards":"3,5,7","objects":650}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1099,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:12.256423+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:20:12.256423+0000","last_peered":"2024-08-17T17:20:12.256423+0000","last_clean":"2024-08-17T17:20:12.256423+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:20:12.256423+0000","last_undegraded":"2024-08-17T17:20:12.256423+0000","last_fullsized":"2024-08-17T17:20:12.256423+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":954,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.500532+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:20:10.500532+0000","last_peered":"2024-08-17T17:20:10.500532+0000","last_clean":"2024-08-17T17:20:10.500532+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:20:10.500532+0000","last_undegraded":"2024-08-17T17:20:10.500532+0000","last_fullsized":"2024-08-17T17:20:10.500532+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.15","version":"1160'7856","reported_seq":11696,"reported_epoch":1160,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:20:27.246891+0000","last_change":"2024-08-17T17:19:01.776021+0000","last_active":"2024-08-17T17:20:27.246891+0000","last_peered":"2024-08-17T17:20:27.246891+0000","last_clean":"2024-08-17T17:18:30.511218+0000","last_became_active":"2024-08-17T17:19:01.771169+0000","last_became_peered":"2024-08-17T17:19:01.771169+0000","last_unstale":"2024-08-17T17:20:27.246891+0000","last_undegraded":"2024-08-17T17:19:01.734234+0000","last_fullsized":"2024-08-17T17:20:27.246891+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7856,"log_dups_size":0,"ondisk_log_size":7856,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136806400,"num_objects":2088,"num_object_clones":0,"num_object_copies":6264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":641,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5038,"num_write_kb":228000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1485,"num_bytes_recovered":97320960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"0,2,7","objects":1447},{"shards":"1,2,7","objects":641}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":966,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.304713+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:20:16.304713+0000","last_peered":"2024-08-17T17:20:16.304713+0000","last_clean":"2024-08-17T17:20:16.304713+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:20:16.304713+0000","last_undegraded":"2024-08-17T17:20:16.304713+0000","last_fullsized":"2024-08-17T17:20:16.304713+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1070,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.500372+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:20:10.500372+0000","last_peered":"2024-08-17T17:20:10.500372+0000","last_clean":"2024-08-17T17:20:10.500372+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:20:10.500372+0000","last_undegraded":"2024-08-17T17:20:10.500372+0000","last_fullsized":"2024-08-17T17:20:10.500372+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.14","version":"1160'7791","reported_seq":10847,"reported_epoch":1160,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:20:27.180294+0000","last_change":"2024-08-17T17:19:01.769109+0000","last_active":"2024-08-17T17:20:27.180294+0000","last_peered":"2024-08-17T17:20:27.180294+0000","last_clean":"2024-08-17T17:18:30.520652+0000","last_became_active":"2024-08-17T17:19:01.765052+0000","last_became_peered":"2024-08-17T17:19:01.765052+0000","last_unstale":"2024-08-17T17:20:27.180294+0000","last_undegraded":"2024-08-17T17:19:01.729866+0000","last_fullsized":"2024-08-17T17:20:27.180294+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7791,"log_dups_size":0,"ondisk_log_size":7791,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131432448,"num_objects":2006,"num_object_clones":0,"num_object_copies":6018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":611,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5006,"num_write_kb":224352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1500,"num_bytes_recovered":98304000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"0,1,4","objects":1395},{"shards":"1,4,6","objects":611}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1400,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.312781+0000","last_change":"2024-08-17T17:07:45.687030+0000","last_active":"2024-08-17T17:20:16.312781+0000","last_peered":"2024-08-17T17:20:16.312781+0000","last_clean":"2024-08-17T17:20:16.312781+0000","last_became_active":"2024-08-17T17:07:45.686610+0000","last_became_peered":"2024-08-17T17:07:45.686610+0000","last_unstale":"2024-08-17T17:20:16.312781+0000","last_undegraded":"2024-08-17T17:20:16.312781+0000","last_fullsized":"2024-08-17T17:20:16.312781+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:02.425845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1147,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.492926+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:20:10.492926+0000","last_peered":"2024-08-17T17:20:10.492926+0000","last_clean":"2024-08-17T17:20:10.492926+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:20:10.492926+0000","last_undegraded":"2024-08-17T17:20:10.492926+0000","last_fullsized":"2024-08-17T17:20:10.492926+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"1160'9833","reported_seq":9874,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:27.232664+0000","last_change":"2024-08-17T17:17:52.927899+0000","last_active":"2024-08-17T17:20:27.232664+0000","last_peered":"2024-08-17T17:20:27.232664+0000","last_clean":"2024-08-17T17:20:27.232664+0000","last_became_active":"2024-08-17T17:17:52.927591+0000","last_became_peered":"2024-08-17T17:17:52.927591+0000","last_unstale":"2024-08-17T17:20:27.232664+0000","last_undegraded":"2024-08-17T17:20:27.232664+0000","last_fullsized":"2024-08-17T17:20:27.232664+0000","mapping_epoch":1129,"log_start":"1133'1000","ondisk_log_start":"1133'1000","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":8833,"log_dups_size":0,"ondisk_log_size":8833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:31:21.681712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277676032,"num_objects":4237,"num_object_clones":0,"num_object_copies":12711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9833,"num_write_kb":450240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1112,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.500979+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:20:10.500979+0000","last_peered":"2024-08-17T17:20:10.500979+0000","last_clean":"2024-08-17T17:20:10.500979+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:20:10.500979+0000","last_undegraded":"2024-08-17T17:20:10.500979+0000","last_fullsized":"2024-08-17T17:20:10.500979+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1085,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.516088+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:20:10.516088+0000","last_peered":"2024-08-17T17:20:10.516088+0000","last_clean":"2024-08-17T17:20:10.516088+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:20:10.516088+0000","last_undegraded":"2024-08-17T17:20:10.516088+0000","last_fullsized":"2024-08-17T17:20:10.516088+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"1160'9930","reported_seq":10007,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:29.808100+0000","last_change":"2024-08-17T17:17:52.311116+0000","last_active":"2024-08-17T17:20:29.808100+0000","last_peered":"2024-08-17T17:20:29.808100+0000","last_clean":"2024-08-17T17:20:29.808100+0000","last_became_active":"2024-08-17T17:17:52.310809+0000","last_became_peered":"2024-08-17T17:17:52.310809+0000","last_unstale":"2024-08-17T17:20:29.808100+0000","last_undegraded":"2024-08-17T17:20:29.808100+0000","last_fullsized":"2024-08-17T17:20:29.808100+0000","mapping_epoch":1129,"log_start":"1147'4200","ondisk_log_start":"1147'4200","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":5730,"log_dups_size":0,"ondisk_log_size":5730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:00:48.009463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260177920,"num_objects":3970,"num_object_clones":0,"num_object_copies":11910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9930,"num_write_kb":444800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1087,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.501572+0000","last_change":"2024-08-17T17:06:39.638433+0000","last_active":"2024-08-17T17:20:10.501572+0000","last_peered":"2024-08-17T17:20:10.501572+0000","last_clean":"2024-08-17T17:20:10.501572+0000","last_became_active":"2024-08-17T17:06:39.638081+0000","last_became_peered":"2024-08-17T17:06:39.638081+0000","last_unstale":"2024-08-17T17:20:10.501572+0000","last_undegraded":"2024-08-17T17:20:10.501572+0000","last_fullsized":"2024-08-17T17:20:10.501572+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:13.558012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1119,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.511987+0000","last_change":"2024-08-17T17:07:45.693574+0000","last_active":"2024-08-17T17:20:10.511987+0000","last_peered":"2024-08-17T17:20:10.511987+0000","last_clean":"2024-08-17T17:20:10.511987+0000","last_became_active":"2024-08-17T17:07:45.693086+0000","last_became_peered":"2024-08-17T17:07:45.693086+0000","last_unstale":"2024-08-17T17:20:10.511987+0000","last_undegraded":"2024-08-17T17:20:10.511987+0000","last_fullsized":"2024-08-17T17:20:10.511987+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:27:31.888894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"1160'7659","reported_seq":10514,"reported_epoch":1160,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:20:27.174730+0000","last_change":"2024-08-17T17:19:02.700822+0000","last_active":"2024-08-17T17:20:27.174730+0000","last_peered":"2024-08-17T17:20:27.174730+0000","last_clean":"2024-08-17T17:18:30.513360+0000","last_became_active":"2024-08-17T17:19:02.673093+0000","last_became_peered":"2024-08-17T17:19:02.673093+0000","last_unstale":"2024-08-17T17:20:27.174730+0000","last_undegraded":"2024-08-17T17:19:02.662660+0000","last_fullsized":"2024-08-17T17:20:27.174730+0000","mapping_epoch":1152,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7659,"log_dups_size":0,"ondisk_log_size":7659,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137887744,"num_objects":2104,"num_object_clones":0,"num_object_copies":6312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4898,"num_write_kb":224064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1397,"num_bytes_recovered":91586560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"0,1,4","objects":1475},{"shards":"1,4,7","objects":629}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1071,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.502793+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:20:10.502793+0000","last_peered":"2024-08-17T17:20:10.502793+0000","last_clean":"2024-08-17T17:20:10.502793+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:20:10.502793+0000","last_undegraded":"2024-08-17T17:20:10.502793+0000","last_fullsized":"2024-08-17T17:20:10.502793+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1101,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.492901+0000","last_change":"2024-08-17T17:06:39.629368+0000","last_active":"2024-08-17T17:20:10.492901+0000","last_peered":"2024-08-17T17:20:10.492901+0000","last_clean":"2024-08-17T17:20:10.492901+0000","last_became_active":"2024-08-17T17:06:39.628832+0000","last_became_peered":"2024-08-17T17:06:39.628832+0000","last_unstale":"2024-08-17T17:20:10.492901+0000","last_undegraded":"2024-08-17T17:20:10.492901+0000","last_fullsized":"2024-08-17T17:20:10.492901+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:02:54.063884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"1160'7679","reported_seq":7779,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:26.373885+0000","last_change":"2024-08-17T17:20:24.708888+0000","last_active":"2024-08-17T17:20:26.373885+0000","last_peered":"2024-08-17T17:20:26.373885+0000","last_clean":"2024-08-17T17:20:26.373885+0000","last_became_active":"2024-08-17T17:19:01.657134+0000","last_became_peered":"2024-08-17T17:19:01.657134+0000","last_unstale":"2024-08-17T17:20:26.373885+0000","last_undegraded":"2024-08-17T17:20:26.373885+0000","last_fullsized":"2024-08-17T17:20:26.373885+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7679,"log_dups_size":0,"ondisk_log_size":7679,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:20:29.641252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130547712,"num_objects":1992,"num_object_clones":0,"num_object_copies":5976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4920,"num_write_kb":221184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1100,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.516333+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:20:10.516333+0000","last_peered":"2024-08-17T17:20:10.516333+0000","last_clean":"2024-08-17T17:20:10.516333+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:20:10.516333+0000","last_undegraded":"2024-08-17T17:20:10.516333+0000","last_fullsized":"2024-08-17T17:20:10.516333+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":913,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.304209+0000","last_change":"2024-08-17T17:18:31.539133+0000","last_active":"2024-08-17T17:20:16.304209+0000","last_peered":"2024-08-17T17:20:16.304209+0000","last_clean":"2024-08-17T17:20:16.304209+0000","last_became_active":"2024-08-17T17:18:31.538669+0000","last_became_peered":"2024-08-17T17:18:31.538669+0000","last_unstale":"2024-08-17T17:20:16.304209+0000","last_undegraded":"2024-08-17T17:20:16.304209+0000","last_fullsized":"2024-08-17T17:20:16.304209+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:34.785446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1467,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.500204+0000","last_change":"2024-08-17T17:18:28.522790+0000","last_active":"2024-08-17T17:20:10.500204+0000","last_peered":"2024-08-17T17:20:10.500204+0000","last_clean":"2024-08-17T17:20:10.500204+0000","last_became_active":"2024-08-17T17:18:28.521573+0000","last_became_peered":"2024-08-17T17:18:28.521573+0000","last_unstale":"2024-08-17T17:20:10.500204+0000","last_undegraded":"2024-08-17T17:20:10.500204+0000","last_fullsized":"2024-08-17T17:20:10.500204+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":76,"num_write":19,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":214679,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.f","version":"1160'10043","reported_seq":10119,"reported_epoch":1160,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:20:29.847126+0000","last_change":"2024-08-17T17:18:33.026499+0000","last_active":"2024-08-17T17:20:29.847126+0000","last_peered":"2024-08-17T17:20:29.847126+0000","last_clean":"2024-08-17T17:18:30.502708+0000","last_became_active":"2024-08-17T17:18:33.018495+0000","last_became_peered":"2024-08-17T17:18:33.018495+0000","last_unstale":"2024-08-17T17:20:29.847126+0000","last_undegraded":"2024-08-17T17:18:32.555520+0000","last_fullsized":"2024-08-17T17:18:32.533633+0000","mapping_epoch":1144,"log_start":"1148'4300","ondisk_log_start":"1148'4300","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":5743,"log_dups_size":0,"ondisk_log_size":5743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263389184,"num_objects":4019,"num_object_clones":0,"num_object_copies":12057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4154,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10043,"num_write_kb":449984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,7],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":1142},{"shards":"3,7","objects":-135}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1435,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.306108+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:20:16.306108+0000","last_peered":"2024-08-17T17:20:16.306108+0000","last_clean":"2024-08-17T17:20:16.306108+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:20:16.306108+0000","last_undegraded":"2024-08-17T17:20:16.306108+0000","last_fullsized":"2024-08-17T17:20:16.306108+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1103,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.512603+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:20:10.512603+0000","last_peered":"2024-08-17T17:20:10.512603+0000","last_clean":"2024-08-17T17:20:10.512603+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:20:10.512603+0000","last_undegraded":"2024-08-17T17:20:10.512603+0000","last_fullsized":"2024-08-17T17:20:10.512603+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1425,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.515944+0000","last_change":"2024-08-17T17:18:32.884334+0000","last_active":"2024-08-17T17:20:10.515944+0000","last_peered":"2024-08-17T17:20:10.515944+0000","last_clean":"2024-08-17T17:20:10.515944+0000","last_became_active":"2024-08-17T17:18:28.520186+0000","last_became_peered":"2024-08-17T17:18:28.520186+0000","last_unstale":"2024-08-17T17:20:10.515944+0000","last_undegraded":"2024-08-17T17:20:10.515944+0000","last_fullsized":"2024-08-17T17:20:10.515944+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:32.884246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:32.884246+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:49.286656+0000","scrub_duration":0.001842677,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.e","version":"1160'9995","reported_seq":14628,"reported_epoch":1160,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:20:26.396289+0000","last_change":"2024-08-17T17:19:58.835950+0000","last_active":"2024-08-17T17:20:26.396289+0000","last_peered":"2024-08-17T17:20:26.396289+0000","last_clean":"2024-08-17T17:18:30.515740+0000","last_became_active":"2024-08-17T17:18:33.016776+0000","last_became_peered":"2024-08-17T17:18:33.016776+0000","last_unstale":"2024-08-17T17:20:26.396289+0000","last_undegraded":"2024-08-17T17:19:58.834387+0000","last_fullsized":"2024-08-17T17:18:32.535040+0000","mapping_epoch":1144,"log_start":"1133'1400","ondisk_log_start":"1133'1400","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":8595,"log_dups_size":0,"ondisk_log_size":8595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280690688,"num_objects":4283,"num_object_clones":0,"num_object_copies":12849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":580,"num_objects_misplaced":1323,"num_objects_unfound":0,"num_objects_dirty":4283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9995,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2278,"num_bytes_recovered":149291008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":1323},{"shards":"2,5","objects":2380}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1102,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.493014+0000","last_change":"2024-08-17T17:06:39.629619+0000","last_active":"2024-08-17T17:20:10.493014+0000","last_peered":"2024-08-17T17:20:10.493014+0000","last_clean":"2024-08-17T17:20:10.493014+0000","last_became_active":"2024-08-17T17:06:39.629295+0000","last_became_peered":"2024-08-17T17:06:39.629295+0000","last_unstale":"2024-08-17T17:20:10.493014+0000","last_undegraded":"2024-08-17T17:20:10.493014+0000","last_fullsized":"2024-08-17T17:20:10.493014+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:30:15.874423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1118,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.511082+0000","last_change":"2024-08-17T17:07:12.649178+0000","last_active":"2024-08-17T17:20:10.511082+0000","last_peered":"2024-08-17T17:20:10.511082+0000","last_clean":"2024-08-17T17:20:10.511082+0000","last_became_active":"2024-08-17T17:07:12.648844+0000","last_became_peered":"2024-08-17T17:07:12.648844+0000","last_unstale":"2024-08-17T17:20:10.511082+0000","last_undegraded":"2024-08-17T17:20:10.511082+0000","last_fullsized":"2024-08-17T17:20:10.511082+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:12:06.307736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"1160'9826","reported_seq":9876,"reported_epoch":1160,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:20:27.094436+0000","last_change":"2024-08-17T17:18:33.033028+0000","last_active":"2024-08-17T17:20:27.094436+0000","last_peered":"2024-08-17T17:20:27.094436+0000","last_clean":"2024-08-17T17:18:30.508341+0000","last_became_active":"2024-08-17T17:18:33.019697+0000","last_became_peered":"2024-08-17T17:18:33.019697+0000","last_unstale":"2024-08-17T17:20:27.094436+0000","last_undegraded":"2024-08-17T17:18:32.562771+0000","last_fullsized":"2024-08-17T17:18:32.533408+0000","mapping_epoch":1144,"log_start":"1133'2500","ondisk_log_start":"1133'2500","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7326,"log_dups_size":0,"ondisk_log_size":7326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:58:13.699201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274268184,"num_objects":4186,"num_object_clones":0,"num_object_copies":12558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4070,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4186,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9826,"num_write_kb":448321,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":1250},{"shards":"1,6","objects":116}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1105,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.512065+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:20:10.512065+0000","last_peered":"2024-08-17T17:20:10.512065+0000","last_clean":"2024-08-17T17:20:10.512065+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:20:10.512065+0000","last_undegraded":"2024-08-17T17:20:10.512065+0000","last_fullsized":"2024-08-17T17:20:10.512065+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1103,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.511770+0000","last_change":"2024-08-17T17:06:39.627678+0000","last_active":"2024-08-17T17:20:10.511770+0000","last_peered":"2024-08-17T17:20:10.511770+0000","last_clean":"2024-08-17T17:20:10.511770+0000","last_became_active":"2024-08-17T17:06:39.627226+0000","last_became_peered":"2024-08-17T17:06:39.627226+0000","last_unstale":"2024-08-17T17:20:10.511770+0000","last_undegraded":"2024-08-17T17:20:10.511770+0000","last_fullsized":"2024-08-17T17:20:10.511770+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:01.868244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1425,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.516055+0000","last_change":"2024-08-17T17:18:31.850829+0000","last_active":"2024-08-17T17:20:10.516055+0000","last_peered":"2024-08-17T17:20:10.516055+0000","last_clean":"2024-08-17T17:20:10.516055+0000","last_became_active":"2024-08-17T17:18:28.520946+0000","last_became_peered":"2024-08-17T17:18:28.520946+0000","last_unstale":"2024-08-17T17:20:10.516055+0000","last_undegraded":"2024-08-17T17:20:10.516055+0000","last_fullsized":"2024-08-17T17:20:10.516055+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:31.850745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:31.850745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:17.218084+0000","scrub_duration":0.001317447,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.c","version":"1160'9697","reported_seq":9738,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:27.131747+0000","last_change":"2024-08-17T17:17:52.309565+0000","last_active":"2024-08-17T17:20:27.131747+0000","last_peered":"2024-08-17T17:20:27.131747+0000","last_clean":"2024-08-17T17:20:27.131747+0000","last_became_active":"2024-08-17T17:17:52.309298+0000","last_became_peered":"2024-08-17T17:17:52.309298+0000","last_unstale":"2024-08-17T17:20:27.131747+0000","last_undegraded":"2024-08-17T17:20:27.131747+0000","last_fullsized":"2024-08-17T17:20:27.131747+0000","mapping_epoch":1129,"log_start":"1133'2300","ondisk_log_start":"1133'2300","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7397,"log_dups_size":0,"ondisk_log_size":7397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:27:31.251993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270729216,"num_objects":4131,"num_object_clones":0,"num_object_copies":12393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9697,"num_write_kb":442496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1402,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.312623+0000","last_change":"2024-08-17T17:07:45.680458+0000","last_active":"2024-08-17T17:20:16.312623+0000","last_peered":"2024-08-17T17:20:16.312623+0000","last_clean":"2024-08-17T17:20:16.312623+0000","last_became_active":"2024-08-17T17:07:45.680221+0000","last_became_peered":"2024-08-17T17:07:45.680221+0000","last_unstale":"2024-08-17T17:20:16.312623+0000","last_undegraded":"2024-08-17T17:20:16.312623+0000","last_fullsized":"2024-08-17T17:20:16.312623+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:55.083962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1424,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.303754+0000","last_change":"2024-08-17T17:07:45.682866+0000","last_active":"2024-08-17T17:20:16.303754+0000","last_peered":"2024-08-17T17:20:16.303754+0000","last_clean":"2024-08-17T17:20:16.303754+0000","last_became_active":"2024-08-17T17:07:45.682554+0000","last_became_peered":"2024-08-17T17:07:45.682554+0000","last_unstale":"2024-08-17T17:20:16.303754+0000","last_undegraded":"2024-08-17T17:20:16.303754+0000","last_fullsized":"2024-08-17T17:20:16.303754+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:25.975914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.6","version":"1160'7782","reported_seq":10714,"reported_epoch":1160,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:20:27.258121+0000","last_change":"2024-08-17T17:19:02.694376+0000","last_active":"2024-08-17T17:20:27.258121+0000","last_peered":"2024-08-17T17:20:27.258121+0000","last_clean":"2024-08-17T17:18:30.519687+0000","last_became_active":"2024-08-17T17:19:02.689996+0000","last_became_peered":"2024-08-17T17:19:02.689996+0000","last_unstale":"2024-08-17T17:20:27.258121+0000","last_undegraded":"2024-08-17T17:19:02.656489+0000","last_fullsized":"2024-08-17T17:20:27.258121+0000","mapping_epoch":1152,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7782,"log_dups_size":0,"ondisk_log_size":7782,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140378112,"num_objects":2142,"num_object_clones":0,"num_object_copies":6426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":573,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5016,"num_write_kb":229056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1438,"num_bytes_recovered":94240768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":["7","3","5"],"object_location_counts":[{"shards":"0,3,7","objects":1569},{"shards":"3,5,7","objects":573}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1480,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.500204+0000","last_change":"2024-08-17T17:18:33.814375+0000","last_active":"2024-08-17T17:20:10.500204+0000","last_peered":"2024-08-17T17:20:10.500204+0000","last_clean":"2024-08-17T17:20:10.500204+0000","last_became_active":"2024-08-17T17:18:33.813208+0000","last_became_peered":"2024-08-17T17:18:33.813208+0000","last_unstale":"2024-08-17T17:20:10.500204+0000","last_undegraded":"2024-08-17T17:20:10.500204+0000","last_fullsized":"2024-08-17T17:20:10.500204+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:25.057675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:25.057675+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:20:45.235879+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1115,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.516254+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:20:10.516254+0000","last_peered":"2024-08-17T17:20:10.516254+0000","last_clean":"2024-08-17T17:20:10.516254+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:20:10.516254+0000","last_undegraded":"2024-08-17T17:20:10.516254+0000","last_fullsized":"2024-08-17T17:20:10.516254+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1382,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.314069+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:20:16.314069+0000","last_peered":"2024-08-17T17:20:16.314069+0000","last_clean":"2024-08-17T17:20:16.314069+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:20:16.314069+0000","last_undegraded":"2024-08-17T17:20:16.314069+0000","last_fullsized":"2024-08-17T17:20:16.314069+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1086,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.496548+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:20:10.496548+0000","last_peered":"2024-08-17T17:20:10.496548+0000","last_clean":"2024-08-17T17:20:10.496548+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:20:10.496548+0000","last_undegraded":"2024-08-17T17:20:10.496548+0000","last_fullsized":"2024-08-17T17:20:10.496548+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":672,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.493050+0000","last_change":"2024-08-17T17:18:28.524571+0000","last_active":"2024-08-17T17:20:10.493050+0000","last_peered":"2024-08-17T17:20:10.493050+0000","last_clean":"2024-08-17T17:20:10.493050+0000","last_became_active":"2024-08-17T17:18:28.524280+0000","last_became_peered":"2024-08-17T17:18:28.524280+0000","last_unstale":"2024-08-17T17:20:10.493050+0000","last_undegraded":"2024-08-17T17:20:10.493050+0000","last_fullsized":"2024-08-17T17:20:10.493050+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"1160'7871","reported_seq":11702,"reported_epoch":1160,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:20:27.213687+0000","last_change":"2024-08-17T17:19:02.692013+0000","last_active":"2024-08-17T17:20:27.213687+0000","last_peered":"2024-08-17T17:20:27.213687+0000","last_clean":"2024-08-17T17:18:30.511218+0000","last_became_active":"2024-08-17T17:19:02.665577+0000","last_became_peered":"2024-08-17T17:19:02.665577+0000","last_unstale":"2024-08-17T17:20:27.213687+0000","last_undegraded":"2024-08-17T17:19:02.654944+0000","last_fullsized":"2024-08-17T17:20:27.213687+0000","mapping_epoch":1152,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7871,"log_dups_size":0,"ondisk_log_size":7871,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136085504,"num_objects":2076,"num_object_clones":0,"num_object_copies":6228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":579,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5052,"num_write_kb":228128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1479,"num_bytes_recovered":96927744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":["7","1","2"],"object_location_counts":[{"shards":"0,2,7","objects":1497},{"shards":"1,2,7","objects":579}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1132,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.500150+0000","last_change":"2024-08-17T17:18:31.539042+0000","last_active":"2024-08-17T17:20:10.500150+0000","last_peered":"2024-08-17T17:20:10.500150+0000","last_clean":"2024-08-17T17:20:10.500150+0000","last_became_active":"2024-08-17T17:18:31.537879+0000","last_became_peered":"2024-08-17T17:18:31.537879+0000","last_unstale":"2024-08-17T17:20:10.500150+0000","last_undegraded":"2024-08-17T17:20:10.500150+0000","last_fullsized":"2024-08-17T17:20:10.500150+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:36:27.859975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":954,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.511784+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:20:10.511784+0000","last_peered":"2024-08-17T17:20:10.511784+0000","last_clean":"2024-08-17T17:20:10.511784+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:20:10.511784+0000","last_undegraded":"2024-08-17T17:20:10.511784+0000","last_fullsized":"2024-08-17T17:20:10.511784+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":860,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.518453+0000","last_change":"2024-08-17T17:18:31.536646+0000","last_active":"2024-08-17T17:20:10.518453+0000","last_peered":"2024-08-17T17:20:10.518453+0000","last_clean":"2024-08-17T17:20:10.518453+0000","last_became_active":"2024-08-17T17:18:31.536326+0000","last_became_peered":"2024-08-17T17:18:31.536326+0000","last_unstale":"2024-08-17T17:20:10.518453+0000","last_undegraded":"2024-08-17T17:20:10.518453+0000","last_fullsized":"2024-08-17T17:20:10.518453+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:11:03.952344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1485,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.306651+0000","last_change":"2024-08-17T17:18:33.817028+0000","last_active":"2024-08-17T17:20:16.306651+0000","last_peered":"2024-08-17T17:20:16.306651+0000","last_clean":"2024-08-17T17:20:16.306651+0000","last_became_active":"2024-08-17T17:18:33.816660+0000","last_became_peered":"2024-08-17T17:18:33.816660+0000","last_unstale":"2024-08-17T17:20:16.306651+0000","last_undegraded":"2024-08-17T17:20:16.306651+0000","last_fullsized":"2024-08-17T17:20:16.306651+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":214680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.4","version":"1160'7793","reported_seq":10753,"reported_epoch":1160,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:20:27.184281+0000","last_change":"2024-08-17T17:19:02.697004+0000","last_active":"2024-08-17T17:20:27.184281+0000","last_peered":"2024-08-17T17:20:27.184281+0000","last_clean":"2024-08-17T17:18:30.520652+0000","last_became_active":"2024-08-17T17:19:02.667978+0000","last_became_peered":"2024-08-17T17:19:02.667978+0000","last_unstale":"2024-08-17T17:20:27.184281+0000","last_undegraded":"2024-08-17T17:19:02.661396+0000","last_fullsized":"2024-08-17T17:20:27.184281+0000","mapping_epoch":1152,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7793,"log_dups_size":0,"ondisk_log_size":7793,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137986048,"num_objects":2105,"num_object_clones":0,"num_object_copies":6315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":616,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5007,"num_write_kb":227616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1451,"num_bytes_recovered":95092736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1","6"],"object_location_counts":[{"shards":"0,1,4","objects":1489},{"shards":"1,4,6","objects":616}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1479,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.303317+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:20:16.303317+0000","last_peered":"2024-08-17T17:20:16.303317+0000","last_clean":"2024-08-17T17:20:16.303317+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:20:16.303317+0000","last_undegraded":"2024-08-17T17:20:16.303317+0000","last_fullsized":"2024-08-17T17:20:16.303317+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":857,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.304700+0000","last_change":"2024-08-17T17:15:38.610415+0000","last_active":"2024-08-17T17:20:16.304700+0000","last_peered":"2024-08-17T17:20:16.304700+0000","last_clean":"2024-08-17T17:20:16.304700+0000","last_became_active":"2024-08-17T17:15:38.610083+0000","last_became_peered":"2024-08-17T17:15:38.610083+0000","last_unstale":"2024-08-17T17:20:16.304700+0000","last_undegraded":"2024-08-17T17:20:16.304700+0000","last_fullsized":"2024-08-17T17:20:16.304700+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:47:12.007494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":780,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.516357+0000","last_change":"2024-08-17T17:18:31.534212+0000","last_active":"2024-08-17T17:20:10.516357+0000","last_peered":"2024-08-17T17:20:10.516357+0000","last_clean":"2024-08-17T17:20:10.516357+0000","last_became_active":"2024-08-17T17:18:31.533792+0000","last_became_peered":"2024-08-17T17:18:31.533792+0000","last_unstale":"2024-08-17T17:20:10.516357+0000","last_undegraded":"2024-08-17T17:20:10.516357+0000","last_fullsized":"2024-08-17T17:20:10.516357+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:13:01.220425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1442,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.516017+0000","last_change":"2024-08-17T17:18:35.916482+0000","last_active":"2024-08-17T17:20:10.516017+0000","last_peered":"2024-08-17T17:20:10.516017+0000","last_clean":"2024-08-17T17:20:10.516017+0000","last_became_active":"2024-08-17T17:18:34.827747+0000","last_became_peered":"2024-08-17T17:18:34.827747+0000","last_unstale":"2024-08-17T17:20:10.516017+0000","last_undegraded":"2024-08-17T17:20:10.516017+0000","last_fullsized":"2024-08-17T17:20:10.516017+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:35.916381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:35.916381+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:28.492622+0000","scrub_duration":0.005340521,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"1160'7412","reported_seq":10135,"reported_epoch":1160,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:20:27.062788+0000","last_change":"2024-08-17T17:19:02.695693+0000","last_active":"2024-08-17T17:20:27.062788+0000","last_peered":"2024-08-17T17:20:27.062788+0000","last_clean":"2024-08-17T17:18:30.516571+0000","last_became_active":"2024-08-17T17:19:02.666519+0000","last_became_peered":"2024-08-17T17:19:02.666519+0000","last_unstale":"2024-08-17T17:20:27.062788+0000","last_undegraded":"2024-08-17T17:19:02.656623+0000","last_fullsized":"2024-08-17T17:20:27.062788+0000","mapping_epoch":1152,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7412,"log_dups_size":0,"ondisk_log_size":7412,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135397376,"num_objects":2066,"num_object_clones":0,"num_object_copies":6198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":629,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4732,"num_write_kb":217536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1333,"num_bytes_recovered":87359488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"0,3,6","objects":1437},{"shards":"1,3,6","objects":629}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1116,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.516270+0000","last_change":"2024-08-17T17:06:39.628745+0000","last_active":"2024-08-17T17:20:10.516270+0000","last_peered":"2024-08-17T17:20:10.516270+0000","last_clean":"2024-08-17T17:20:10.516270+0000","last_became_active":"2024-08-17T17:06:39.628144+0000","last_became_peered":"2024-08-17T17:06:39.628144+0000","last_unstale":"2024-08-17T17:20:10.516270+0000","last_undegraded":"2024-08-17T17:20:10.516270+0000","last_fullsized":"2024-08-17T17:20:10.516270+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:14.740936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1123,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.511262+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:20:10.511262+0000","last_peered":"2024-08-17T17:20:10.511262+0000","last_clean":"2024-08-17T17:20:10.511262+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:20:10.511262+0000","last_undegraded":"2024-08-17T17:20:10.511262+0000","last_fullsized":"2024-08-17T17:20:10.511262+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":933,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.303589+0000","last_change":"2024-08-17T17:15:38.608764+0000","last_active":"2024-08-17T17:20:16.303589+0000","last_peered":"2024-08-17T17:20:16.303589+0000","last_clean":"2024-08-17T17:20:16.303589+0000","last_became_active":"2024-08-17T17:15:38.608353+0000","last_became_peered":"2024-08-17T17:15:38.608353+0000","last_unstale":"2024-08-17T17:20:16.303589+0000","last_undegraded":"2024-08-17T17:20:16.303589+0000","last_fullsized":"2024-08-17T17:20:16.303589+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:42.873441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":672,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.493240+0000","last_change":"2024-08-17T17:18:28.527058+0000","last_active":"2024-08-17T17:20:10.493240+0000","last_peered":"2024-08-17T17:20:10.493240+0000","last_clean":"2024-08-17T17:20:10.493240+0000","last_became_active":"2024-08-17T17:18:28.526439+0000","last_became_peered":"2024-08-17T17:18:28.526439+0000","last_unstale":"2024-08-17T17:20:10.493240+0000","last_undegraded":"2024-08-17T17:20:10.493240+0000","last_fullsized":"2024-08-17T17:20:10.493240+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":19,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.1","version":"1160'7595","reported_seq":7646,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:27.092425+0000","last_change":"2024-08-17T17:19:01.686339+0000","last_active":"2024-08-17T17:20:27.092425+0000","last_peered":"2024-08-17T17:20:27.092425+0000","last_clean":"2024-08-17T17:20:27.092425+0000","last_became_active":"2024-08-17T17:19:01.682436+0000","last_became_peered":"2024-08-17T17:19:01.682436+0000","last_unstale":"2024-08-17T17:20:27.092425+0000","last_undegraded":"2024-08-17T17:20:27.092425+0000","last_fullsized":"2024-08-17T17:20:27.092425+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7595,"log_dups_size":0,"ondisk_log_size":7595,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131399680,"num_objects":2005,"num_object_clones":0,"num_object_copies":6015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4885,"num_write_kb":220480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1116,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.516456+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:20:10.516456+0000","last_peered":"2024-08-17T17:20:10.516456+0000","last_clean":"2024-08-17T17:20:10.516456+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:20:10.516456+0000","last_undegraded":"2024-08-17T17:20:10.516456+0000","last_fullsized":"2024-08-17T17:20:10.516456+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":789,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.502532+0000","last_change":"2024-08-17T17:18:31.537256+0000","last_active":"2024-08-17T17:20:10.502532+0000","last_peered":"2024-08-17T17:20:10.502532+0000","last_clean":"2024-08-17T17:20:10.502532+0000","last_became_active":"2024-08-17T17:18:31.536194+0000","last_became_peered":"2024-08-17T17:18:31.536194+0000","last_unstale":"2024-08-17T17:20:10.502532+0000","last_undegraded":"2024-08-17T17:20:10.502532+0000","last_fullsized":"2024-08-17T17:20:10.502532+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:46.453717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":731,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.516135+0000","last_change":"2024-08-17T17:15:38.609480+0000","last_active":"2024-08-17T17:20:10.516135+0000","last_peered":"2024-08-17T17:20:10.516135+0000","last_clean":"2024-08-17T17:20:10.516135+0000","last_became_active":"2024-08-17T17:15:38.609120+0000","last_became_peered":"2024-08-17T17:15:38.609120+0000","last_unstale":"2024-08-17T17:20:10.516135+0000","last_undegraded":"2024-08-17T17:20:10.516135+0000","last_fullsized":"2024-08-17T17:20:10.516135+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:11:03.113425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":689,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.493166+0000","last_change":"2024-08-17T17:18:34.831943+0000","last_active":"2024-08-17T17:20:10.493166+0000","last_peered":"2024-08-17T17:20:10.493166+0000","last_clean":"2024-08-17T17:20:10.493166+0000","last_became_active":"2024-08-17T17:18:34.831498+0000","last_became_peered":"2024-08-17T17:18:34.831498+0000","last_unstale":"2024-08-17T17:20:10.493166+0000","last_undegraded":"2024-08-17T17:20:10.493166+0000","last_fullsized":"2024-08-17T17:20:10.493166+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":34,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.0","version":"1160'7819","reported_seq":7870,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:27.196807+0000","last_change":"2024-08-17T17:19:01.664407+0000","last_active":"2024-08-17T17:20:27.196807+0000","last_peered":"2024-08-17T17:20:27.196807+0000","last_clean":"2024-08-17T17:20:27.196807+0000","last_became_active":"2024-08-17T17:19:01.659646+0000","last_became_peered":"2024-08-17T17:19:01.659646+0000","last_unstale":"2024-08-17T17:20:27.196807+0000","last_undegraded":"2024-08-17T17:20:27.196807+0000","last_fullsized":"2024-08-17T17:20:27.196807+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7819,"log_dups_size":0,"ondisk_log_size":7819,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135036928,"num_objects":2060,"num_object_clones":0,"num_object_copies":6180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5022,"num_write_kb":226656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":856,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.303425+0000","last_change":"2024-08-17T17:15:38.609837+0000","last_active":"2024-08-17T17:20:16.303425+0000","last_peered":"2024-08-17T17:20:16.303425+0000","last_clean":"2024-08-17T17:20:16.303425+0000","last_became_active":"2024-08-17T17:15:38.609474+0000","last_became_peered":"2024-08-17T17:15:38.609474+0000","last_unstale":"2024-08-17T17:20:16.303425+0000","last_undegraded":"2024-08-17T17:20:16.303425+0000","last_fullsized":"2024-08-17T17:20:16.303425+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:31:42.450582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1117,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.502786+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:20:10.502786+0000","last_peered":"2024-08-17T17:20:10.502786+0000","last_clean":"2024-08-17T17:20:10.502786+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:20:10.502786+0000","last_undegraded":"2024-08-17T17:20:10.502786+0000","last_fullsized":"2024-08-17T17:20:10.502786+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":709,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.516159+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:20:10.516159+0000","last_peered":"2024-08-17T17:20:10.516159+0000","last_clean":"2024-08-17T17:20:10.516159+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:20:10.516159+0000","last_undegraded":"2024-08-17T17:20:10.516159+0000","last_fullsized":"2024-08-17T17:20:10.516159+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1413,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.511236+0000","last_change":"2024-08-17T17:18:28.526894+0000","last_active":"2024-08-17T17:20:10.511236+0000","last_peered":"2024-08-17T17:20:10.511236+0000","last_clean":"2024-08-17T17:20:10.511236+0000","last_became_active":"2024-08-17T17:18:28.526484+0000","last_became_peered":"2024-08-17T17:18:28.526484+0000","last_unstale":"2024-08-17T17:20:10.511236+0000","last_undegraded":"2024-08-17T17:20:10.511236+0000","last_fullsized":"2024-08-17T17:20:10.511236+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T16:49:54.582155+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.3","version":"1160'7683","reported_seq":7783,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:29.711791+0000","last_change":"2024-08-17T17:20:25.559729+0000","last_active":"2024-08-17T17:20:29.711791+0000","last_peered":"2024-08-17T17:20:29.711791+0000","last_clean":"2024-08-17T17:20:29.711791+0000","last_became_active":"2024-08-17T17:19:01.681349+0000","last_became_peered":"2024-08-17T17:19:01.681349+0000","last_unstale":"2024-08-17T17:20:29.711791+0000","last_undegraded":"2024-08-17T17:20:29.711791+0000","last_fullsized":"2024-08-17T17:20:29.711791+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7683,"log_dups_size":0,"ondisk_log_size":7683,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:20:30.536968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129204224,"num_objects":1971,"num_object_clones":0,"num_object_copies":5913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4961,"num_write_kb":221856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1435,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.313031+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:20:16.313031+0000","last_peered":"2024-08-17T17:20:16.313031+0000","last_clean":"2024-08-17T17:20:16.313031+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:20:16.313031+0000","last_undegraded":"2024-08-17T17:20:16.313031+0000","last_fullsized":"2024-08-17T17:20:16.313031+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":741,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.493237+0000","last_change":"2024-08-17T17:15:38.609543+0000","last_active":"2024-08-17T17:20:10.493237+0000","last_peered":"2024-08-17T17:20:10.493237+0000","last_clean":"2024-08-17T17:20:10.493237+0000","last_became_active":"2024-08-17T17:15:38.609238+0000","last_became_peered":"2024-08-17T17:15:38.609238+0000","last_unstale":"2024-08-17T17:20:10.493237+0000","last_undegraded":"2024-08-17T17:20:10.493237+0000","last_fullsized":"2024-08-17T17:20:10.493237+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:32.029842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1425,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.312447+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:20:16.312447+0000","last_peered":"2024-08-17T17:20:16.312447+0000","last_clean":"2024-08-17T17:20:16.312447+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:20:16.312447+0000","last_undegraded":"2024-08-17T17:20:16.312447+0000","last_fullsized":"2024-08-17T17:20:16.312447+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1441,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.303834+0000","last_change":"2024-08-17T17:18:29.390118+0000","last_active":"2024-08-17T17:20:16.303834+0000","last_peered":"2024-08-17T17:20:16.303834+0000","last_clean":"2024-08-17T17:20:16.303834+0000","last_became_active":"2024-08-17T17:18:28.528874+0000","last_became_peered":"2024-08-17T17:18:28.528874+0000","last_unstale":"2024-08-17T17:20:16.303834+0000","last_undegraded":"2024-08-17T17:20:16.303834+0000","last_fullsized":"2024-08-17T17:20:16.303834+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:29.390003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:29.390003+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:35.908604+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"1160'7784","reported_seq":7916,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:29.825162+0000","last_change":"2024-08-17T17:19:20.831069+0000","last_active":"2024-08-17T17:20:29.825162+0000","last_peered":"2024-08-17T17:20:29.825162+0000","last_clean":"2024-08-17T17:20:29.825162+0000","last_became_active":"2024-08-17T17:19:01.696058+0000","last_became_peered":"2024-08-17T17:19:01.696058+0000","last_unstale":"2024-08-17T17:20:29.825162+0000","last_undegraded":"2024-08-17T17:20:29.825162+0000","last_fullsized":"2024-08-17T17:20:29.825162+0000","mapping_epoch":1129,"log_start":"0'0","ondisk_log_start":"0'0","created":1129,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":0,"last_scrub":"1154'6072","last_scrub_stamp":"2024-08-17T17:19:20.830703+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:19:20.830703+0000","objects_scrubbed":3288,"log_size":7784,"log_dups_size":0,"ondisk_log_size":7784,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:05:03.872989+0000","scrub_duration":1266,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132579328,"num_objects":2023,"num_object_clones":0,"num_object_copies":6069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1729,"num_write_kb":14848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":102,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.511689+0000","last_change":"2024-08-17T17:18:31.553838+0000","last_active":"2024-08-17T17:20:10.511689+0000","last_peered":"2024-08-17T17:20:10.511689+0000","last_clean":"2024-08-17T17:20:10.511689+0000","last_became_active":"2024-08-17T17:18:31.537264+0000","last_became_peered":"2024-08-17T17:18:31.537264+0000","last_unstale":"2024-08-17T17:20:10.511689+0000","last_undegraded":"2024-08-17T17:20:10.511689+0000","last_fullsized":"2024-08-17T17:20:10.511689+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:26.972836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1113,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.494327+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:20:10.494327+0000","last_peered":"2024-08-17T17:20:10.494327+0000","last_clean":"2024-08-17T17:20:10.494327+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:20:10.494327+0000","last_undegraded":"2024-08-17T17:20:10.494327+0000","last_fullsized":"2024-08-17T17:20:10.494327+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1089,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.511804+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:20:10.511804+0000","last_peered":"2024-08-17T17:20:10.511804+0000","last_clean":"2024-08-17T17:20:10.511804+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:20:10.511804+0000","last_undegraded":"2024-08-17T17:20:10.511804+0000","last_fullsized":"2024-08-17T17:20:10.511804+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.10","version":"1160'7786","reported_seq":7836,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:27.158490+0000","last_change":"2024-08-17T17:19:01.710174+0000","last_active":"2024-08-17T17:20:27.158490+0000","last_peered":"2024-08-17T17:20:27.158490+0000","last_clean":"2024-08-17T17:20:27.158490+0000","last_became_active":"2024-08-17T17:19:01.708023+0000","last_became_peered":"2024-08-17T17:19:01.708023+0000","last_unstale":"2024-08-17T17:20:27.158490+0000","last_undegraded":"2024-08-17T17:20:27.158490+0000","last_fullsized":"2024-08-17T17:20:27.158490+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7786,"log_dups_size":0,"ondisk_log_size":7786,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138248192,"num_objects":2110,"num_object_clones":0,"num_object_copies":6330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4990,"num_write_kb":227168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1103,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.511148+0000","last_change":"2024-08-17T17:06:39.627594+0000","last_active":"2024-08-17T17:20:10.511148+0000","last_peered":"2024-08-17T17:20:10.511148+0000","last_clean":"2024-08-17T17:20:10.511148+0000","last_became_active":"2024-08-17T17:06:39.627105+0000","last_became_peered":"2024-08-17T17:06:39.627105+0000","last_unstale":"2024-08-17T17:20:10.511148+0000","last_undegraded":"2024-08-17T17:20:10.511148+0000","last_fullsized":"2024-08-17T17:20:10.511148+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:21:30.105070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1112,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.512802+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:20:10.512802+0000","last_peered":"2024-08-17T17:20:10.512802+0000","last_clean":"2024-08-17T17:20:10.512802+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:20:10.512802+0000","last_undegraded":"2024-08-17T17:20:10.512802+0000","last_fullsized":"2024-08-17T17:20:10.512802+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.11","version":"1160'7536","reported_seq":7586,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:27.081595+0000","last_change":"2024-08-17T17:19:01.705286+0000","last_active":"2024-08-17T17:20:27.081595+0000","last_peered":"2024-08-17T17:20:27.081595+0000","last_clean":"2024-08-17T17:20:27.081595+0000","last_became_active":"2024-08-17T17:19:01.702922+0000","last_became_peered":"2024-08-17T17:19:01.702922+0000","last_unstale":"2024-08-17T17:20:27.081595+0000","last_undegraded":"2024-08-17T17:20:27.081595+0000","last_fullsized":"2024-08-17T17:20:27.081595+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7536,"log_dups_size":0,"ondisk_log_size":7536,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133038080,"num_objects":2030,"num_object_clones":0,"num_object_copies":6090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4826,"num_write_kb":219392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":44,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.516106+0000","last_change":"2024-08-17T17:17:47.283984+0000","last_active":"2024-08-17T17:20:10.516106+0000","last_peered":"2024-08-17T17:20:10.516106+0000","last_clean":"2024-08-17T17:20:10.516106+0000","last_became_active":"2024-08-17T17:17:47.283270+0000","last_became_peered":"2024-08-17T17:17:47.283270+0000","last_unstale":"2024-08-17T17:20:10.516106+0000","last_undegraded":"2024-08-17T17:20:10.516106+0000","last_fullsized":"2024-08-17T17:20:10.516106+0000","mapping_epoch":1127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:31:16.726944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1088,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.502741+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:20:10.502741+0000","last_peered":"2024-08-17T17:20:10.502741+0000","last_clean":"2024-08-17T17:20:10.502741+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:20:10.502741+0000","last_undegraded":"2024-08-17T17:20:10.502741+0000","last_fullsized":"2024-08-17T17:20:10.502741+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.12","version":"1160'7830","reported_seq":7961,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:29.811180+0000","last_change":"2024-08-17T17:19:22.720455+0000","last_active":"2024-08-17T17:20:29.811180+0000","last_peered":"2024-08-17T17:20:29.811180+0000","last_clean":"2024-08-17T17:20:29.811180+0000","last_became_active":"2024-08-17T17:19:01.711724+0000","last_became_peered":"2024-08-17T17:19:01.711724+0000","last_unstale":"2024-08-17T17:20:29.811180+0000","last_undegraded":"2024-08-17T17:20:29.811180+0000","last_fullsized":"2024-08-17T17:20:29.811180+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":5,"last_scrub":"1154'6171","last_scrub_stamp":"2024-08-17T17:19:22.720072+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:19:22.720072+0000","objects_scrubbed":3360,"log_size":7830,"log_dups_size":0,"ondisk_log_size":7830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:20:40.745083+0000","scrub_duration":1168,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132710400,"num_objects":2025,"num_object_clones":0,"num_object_copies":6075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1667,"num_write_kb":10624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1100,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:12.256474+0000","last_change":"2024-08-17T17:06:39.625029+0000","last_active":"2024-08-17T17:20:12.256474+0000","last_peered":"2024-08-17T17:20:12.256474+0000","last_clean":"2024-08-17T17:20:12.256474+0000","last_became_active":"2024-08-17T17:06:39.624688+0000","last_became_peered":"2024-08-17T17:06:39.624688+0000","last_unstale":"2024-08-17T17:20:12.256474+0000","last_undegraded":"2024-08-17T17:20:12.256474+0000","last_fullsized":"2024-08-17T17:20:12.256474+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:54.849668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1381,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.305225+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:20:16.305225+0000","last_peered":"2024-08-17T17:20:16.305225+0000","last_clean":"2024-08-17T17:20:16.305225+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:20:16.305225+0000","last_undegraded":"2024-08-17T17:20:16.305225+0000","last_fullsized":"2024-08-17T17:20:16.305225+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.13","version":"1160'7668","reported_seq":7768,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:29.838468+0000","last_change":"2024-08-17T17:20:28.600792+0000","last_active":"2024-08-17T17:20:29.838468+0000","last_peered":"2024-08-17T17:20:29.838468+0000","last_clean":"2024-08-17T17:20:29.838468+0000","last_became_active":"2024-08-17T17:19:01.708389+0000","last_became_peered":"2024-08-17T17:19:01.708389+0000","last_unstale":"2024-08-17T17:20:29.838468+0000","last_undegraded":"2024-08-17T17:20:29.838468+0000","last_fullsized":"2024-08-17T17:20:29.838468+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7668,"log_dups_size":0,"ondisk_log_size":7668,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:20:33.534204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135168000,"num_objects":2063,"num_object_clones":0,"num_object_copies":6189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4947,"num_write_kb":224288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":954,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.516450+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:20:10.516450+0000","last_peered":"2024-08-17T17:20:10.516450+0000","last_clean":"2024-08-17T17:20:10.516450+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:20:10.516450+0000","last_undegraded":"2024-08-17T17:20:10.516450+0000","last_fullsized":"2024-08-17T17:20:10.516450+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1400,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.312360+0000","last_change":"2024-08-17T17:07:45.678950+0000","last_active":"2024-08-17T17:20:16.312360+0000","last_peered":"2024-08-17T17:20:16.312360+0000","last_clean":"2024-08-17T17:20:16.312360+0000","last_became_active":"2024-08-17T17:07:45.678472+0000","last_became_peered":"2024-08-17T17:07:45.678472+0000","last_unstale":"2024-08-17T17:20:16.312360+0000","last_undegraded":"2024-08-17T17:20:16.312360+0000","last_fullsized":"2024-08-17T17:20:16.312360+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:48:02.385927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1101,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.494249+0000","last_change":"2024-08-17T17:06:39.628542+0000","last_active":"2024-08-17T17:20:10.494249+0000","last_peered":"2024-08-17T17:20:10.494249+0000","last_clean":"2024-08-17T17:20:10.494249+0000","last_became_active":"2024-08-17T17:06:39.628253+0000","last_became_peered":"2024-08-17T17:06:39.628253+0000","last_unstale":"2024-08-17T17:20:10.494249+0000","last_undegraded":"2024-08-17T17:20:10.494249+0000","last_fullsized":"2024-08-17T17:20:10.494249+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:29:22.349326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1430,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.312526+0000","last_change":"2024-08-17T17:18:31.532183+0000","last_active":"2024-08-17T17:20:16.312526+0000","last_peered":"2024-08-17T17:20:16.312526+0000","last_clean":"2024-08-17T17:20:16.312526+0000","last_became_active":"2024-08-17T17:18:31.531094+0000","last_became_peered":"2024-08-17T17:18:31.531094+0000","last_unstale":"2024-08-17T17:20:16.312526+0000","last_undegraded":"2024-08-17T17:20:16.312526+0000","last_fullsized":"2024-08-17T17:20:16.312526+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:01.763127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":914,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.501526+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:20:10.501526+0000","last_peered":"2024-08-17T17:20:10.501526+0000","last_clean":"2024-08-17T17:20:10.501526+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:20:10.501526+0000","last_undegraded":"2024-08-17T17:20:10.501526+0000","last_fullsized":"2024-08-17T17:20:10.501526+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":921,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.305465+0000","last_change":"2024-08-17T17:15:38.610768+0000","last_active":"2024-08-17T17:20:16.305465+0000","last_peered":"2024-08-17T17:20:16.305465+0000","last_clean":"2024-08-17T17:20:16.305465+0000","last_became_active":"2024-08-17T17:15:38.610337+0000","last_became_peered":"2024-08-17T17:15:38.610337+0000","last_unstale":"2024-08-17T17:20:16.305465+0000","last_undegraded":"2024-08-17T17:20:16.305465+0000","last_fullsized":"2024-08-17T17:20:16.305465+0000","mapping_epoch":1109,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:50:15.632778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1400,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.303397+0000","last_change":"2024-08-17T17:07:45.689782+0000","last_active":"2024-08-17T17:20:16.303397+0000","last_peered":"2024-08-17T17:20:16.303397+0000","last_clean":"2024-08-17T17:20:16.303397+0000","last_became_active":"2024-08-17T17:07:45.689312+0000","last_became_peered":"2024-08-17T17:07:45.689312+0000","last_unstale":"2024-08-17T17:20:16.303397+0000","last_undegraded":"2024-08-17T17:20:16.303397+0000","last_fullsized":"2024-08-17T17:20:16.303397+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:07.560555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1119,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.510664+0000","last_change":"2024-08-17T17:07:12.649175+0000","last_active":"2024-08-17T17:20:10.510664+0000","last_peered":"2024-08-17T17:20:10.510664+0000","last_clean":"2024-08-17T17:20:10.510664+0000","last_became_active":"2024-08-17T17:07:12.648822+0000","last_became_peered":"2024-08-17T17:07:12.648822+0000","last_unstale":"2024-08-17T17:20:10.510664+0000","last_undegraded":"2024-08-17T17:20:10.510664+0000","last_fullsized":"2024-08-17T17:20:10.510664+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:12:03.924431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1448,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.303510+0000","last_change":"2024-08-17T17:17:47.316690+0000","last_active":"2024-08-17T17:20:16.303510+0000","last_peered":"2024-08-17T17:20:16.303510+0000","last_clean":"2024-08-17T17:20:16.303510+0000","last_became_active":"2024-08-17T17:17:47.316333+0000","last_became_peered":"2024-08-17T17:17:47.316333+0000","last_unstale":"2024-08-17T17:20:16.303510+0000","last_undegraded":"2024-08-17T17:20:16.303510+0000","last_fullsized":"2024-08-17T17:20:16.303510+0000","mapping_epoch":1127,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:04.542237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":107,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.510952+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:20:10.510952+0000","last_peered":"2024-08-17T17:20:10.510952+0000","last_clean":"2024-08-17T17:20:10.510952+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:20:10.510952+0000","last_undegraded":"2024-08-17T17:20:10.510952+0000","last_fullsized":"2024-08-17T17:20:10.510952+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.18","version":"1160'7623","reported_seq":7722,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:26.353653+0000","last_change":"2024-08-17T17:20:23.647805+0000","last_active":"2024-08-17T17:20:26.353653+0000","last_peered":"2024-08-17T17:20:26.353653+0000","last_clean":"2024-08-17T17:20:26.353653+0000","last_became_active":"2024-08-17T17:19:01.702866+0000","last_became_peered":"2024-08-17T17:19:01.702866+0000","last_unstale":"2024-08-17T17:20:26.353653+0000","last_undegraded":"2024-08-17T17:20:26.353653+0000","last_fullsized":"2024-08-17T17:20:26.353653+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1152,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7623,"log_dups_size":0,"ondisk_log_size":7623,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:20:28.616263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136445952,"num_objects":2082,"num_object_clones":0,"num_object_copies":6246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4864,"num_write_kb":222272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1117,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.511084+0000","last_change":"2024-08-17T17:07:45.684606+0000","last_active":"2024-08-17T17:20:10.511084+0000","last_peered":"2024-08-17T17:20:10.511084+0000","last_clean":"2024-08-17T17:20:10.511084+0000","last_became_active":"2024-08-17T17:07:45.684124+0000","last_became_peered":"2024-08-17T17:07:45.684124+0000","last_unstale":"2024-08-17T17:20:10.511084+0000","last_undegraded":"2024-08-17T17:20:10.511084+0000","last_fullsized":"2024-08-17T17:20:10.511084+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:41:16.110507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1101,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.516284+0000","last_change":"2024-08-17T17:06:39.625245+0000","last_active":"2024-08-17T17:20:10.516284+0000","last_peered":"2024-08-17T17:20:10.516284+0000","last_clean":"2024-08-17T17:20:10.516284+0000","last_became_active":"2024-08-17T17:06:39.624657+0000","last_became_peered":"2024-08-17T17:06:39.624657+0000","last_unstale":"2024-08-17T17:20:10.516284+0000","last_undegraded":"2024-08-17T17:20:10.516284+0000","last_fullsized":"2024-08-17T17:20:10.516284+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:43.097460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.19","version":"1160'7761","reported_seq":10778,"reported_epoch":1160,"state":"active+recovery_wait+degraded","last_fresh":"2024-08-17T17:20:27.187062+0000","last_change":"2024-08-17T17:19:01.771867+0000","last_active":"2024-08-17T17:20:27.187062+0000","last_peered":"2024-08-17T17:20:27.187062+0000","last_clean":"2024-08-17T17:18:30.513360+0000","last_became_active":"2024-08-17T17:19:01.766350+0000","last_became_peered":"2024-08-17T17:19:01.766350+0000","last_unstale":"2024-08-17T17:20:27.187062+0000","last_undegraded":"2024-08-17T17:19:01.712084+0000","last_fullsized":"2024-08-17T17:20:27.187062+0000","mapping_epoch":1151,"log_start":"0'0","ondisk_log_start":"0'0","created":1151,"last_epoch_clean":1130,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:17:51.286638+0000","last_clean_scrub_stamp":"2024-08-17T17:17:51.286638+0000","objects_scrubbed":0,"log_size":7761,"log_dups_size":0,"ondisk_log_size":7761,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133824512,"num_objects":2042,"num_object_clones":0,"num_object_copies":6126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":610,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5000,"num_write_kb":225344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1480,"num_bytes_recovered":96960512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"0,1,4","objects":1432},{"shards":"1,4,7","objects":610}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1135,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.510520+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:20:10.510520+0000","last_peered":"2024-08-17T17:20:10.510520+0000","last_clean":"2024-08-17T17:20:10.510520+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:20:10.510520+0000","last_undegraded":"2024-08-17T17:20:10.510520+0000","last_fullsized":"2024-08-17T17:20:10.510520+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1086,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.516413+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:20:10.516413+0000","last_peered":"2024-08-17T17:20:10.516413+0000","last_clean":"2024-08-17T17:20:10.516413+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:20:10.516413+0000","last_undegraded":"2024-08-17T17:20:10.516413+0000","last_fullsized":"2024-08-17T17:20:10.516413+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1420,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.305783+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:20:16.305783+0000","last_peered":"2024-08-17T17:20:16.305783+0000","last_clean":"2024-08-17T17:20:16.305783+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:20:16.305783+0000","last_undegraded":"2024-08-17T17:20:16.305783+0000","last_fullsized":"2024-08-17T17:20:16.305783+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1395,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.304802+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:20:16.304802+0000","last_peered":"2024-08-17T17:20:16.304802+0000","last_clean":"2024-08-17T17:20:16.304802+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:20:16.304802+0000","last_undegraded":"2024-08-17T17:20:16.304802+0000","last_fullsized":"2024-08-17T17:20:16.304802+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1103,"reported_epoch":1160,"state":"active+clean","last_fresh":"2024-08-17T17:20:16.292754+0000","last_change":"2024-08-17T17:06:39.627689+0000","last_active":"2024-08-17T17:20:16.292754+0000","last_peered":"2024-08-17T17:20:16.292754+0000","last_clean":"2024-08-17T17:20:16.292754+0000","last_became_active":"2024-08-17T17:06:39.627375+0000","last_became_peered":"2024-08-17T17:06:39.627375+0000","last_unstale":"2024-08-17T17:20:16.292754+0000","last_undegraded":"2024-08-17T17:20:16.292754+0000","last_fullsized":"2024-08-17T17:20:16.292754+0000","mapping_epoch":771,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":772,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:07:51.954076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":129,"reported_epoch":1159,"state":"active+clean","last_fresh":"2024-08-17T17:20:10.516453+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:20:10.516453+0000","last_peered":"2024-08-17T17:20:10.516453+0000","last_clean":"2024-08-17T17:20:10.516453+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:20:10.516453+0000","last_undegraded":"2024-08-17T17:20:10.516453+0000","last_fullsized":"2024-08-17T17:20:10.516453+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":4315676696,"num_objects":65853,"num_object_clones":0,"num_object_copies":197559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14904,"num_objects_misplaced":1323,"num_objects_unfound":0,"num_objects_dirty":65853,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":151681,"num_write_kb":6748289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16721,"num_bytes_recovered":1095827456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14463283200,"data_stored":14463270984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":197795,"ondisk_log_size":197795,"up":78,"acting":75,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":912388,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":210,"num_read_kb":453,"num_write":118,"num_write_kb":1319,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1073397,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201628,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1890288,"kb_used_data":1602472,"kb_used_omap":1,"kb_used_meta":287806,"kb_avail":91854864,"statfs":{"total":95995035648,"available":94059380736,"internally_reserved":0,"allocated":1640931328,"data_stored":1638757542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":294713803},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,0,1,2,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463265,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1848684,"kb_used_data":1596964,"kb_used_omap":1,"kb_used_meta":251710,"kb_avail":91896468,"statfs":{"total":95995035648,"available":94101983232,"internally_reserved":0,"allocated":1635291136,"data_stored":1633121343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":257751497},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626907,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3227932,"kb_used_data":2848724,"kb_used_omap":42,"kb_used_meta":379157,"kb_avail":90517220,"statfs":{"total":95995035648,"available":92689633280,"internally_reserved":0,"allocated":2917093376,"data_stored":2914935744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":388256828},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785384,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2372828,"kb_used_data":2123924,"kb_used_omap":48,"kb_used_meta":248783,"kb_avail":91372324,"statfs":{"total":95995035648,"available":93565259776,"internally_reserved":0,"allocated":2174898176,"data_stored":2172739953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":254754478},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724901,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1608932,"kb_used_data":1353116,"kb_used_omap":1,"kb_used_meta":255806,"kb_avail":92136220,"statfs":{"total":95995035648,"available":94347489280,"internally_reserved":0,"allocated":1385590784,"data_stored":1383425529,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":261945798},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986538,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1878252,"kb_used_data":1594532,"kb_used_omap":1,"kb_used_meta":283710,"kb_avail":91866900,"statfs":{"total":95995035648,"available":94071705600,"internally_reserved":0,"allocated":1632800768,"data_stored":1630631008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":290519494},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706145,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1949984,"kb_used_data":1629208,"kb_used_omap":35,"kb_used_meta":320732,"kb_avail":91795168,"statfs":{"total":95995035648,"available":93998252032,"internally_reserved":0,"allocated":1668308992,"data_stored":1666147790,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":328429623},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,2,5,1,4,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831918,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1840032,"kb_used_data":1500760,"kb_used_omap":65,"kb_used_meta":339198,"kb_avail":91905120,"statfs":{"total":95995035648,"available":94110842880,"internally_reserved":0,"allocated":1536778240,"data_stored":1534616506,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":347338939},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,2,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1529679872,"data_stored":1529675800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2153451520,"data_stored":2153447448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1645936640,"data_stored":1645936640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2898132992,"data_stored":2898132992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1616314368,"data_stored":1616314368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1368195072,"data_stored":1368195072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1625362432,"data_stored":1625358360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1626210304,"data_stored":1626210304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:20:33.202 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T17:20:33.203 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_2 pg_num 36 2024-08-17T17:20:33.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.204 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:33.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:33.304 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:33.355 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:33.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:33.457 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:33.481 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:33.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:33.509 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:20:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:20:33] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.51.0" 2024-08-17T17:20:33.582 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:33.609 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:33.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:33.710 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:33.737 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:33.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:33.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:33 smithi063 ceph-mon[114756]: pgmap v1419: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 14312/191739 objects degraded (7.464%); 1049/191739 objects misplaced (0.547%); 19 MiB/s, 297 objects/s recovering 2024-08-17T17:20:33.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:33 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:20:33.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:33 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3999499647' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:33.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:33 smithi063 ceph-mon[114756]: osdmap e1161: 8 total, 8 up, 8 in 2024-08-17T17:20:33.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:33 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 14312/191739 objects degraded (7.464%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:33.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:33 smithi063 ceph-mon[109843]: pgmap v1419: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 14312/191739 objects degraded (7.464%); 1049/191739 objects misplaced (0.547%); 19 MiB/s, 297 objects/s recovering 2024-08-17T17:20:33.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:33 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:20:33.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:33 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3999499647' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:33.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:33 smithi063 ceph-mon[109843]: osdmap e1161: 8 total, 8 up, 8 in 2024-08-17T17:20:33.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:33 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 14312/191739 objects degraded (7.464%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:33.839 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:33.876 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:33.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:33.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:33.978 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:33.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:33 smithi162 ceph-mon[104668]: pgmap v1419: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 14312/191739 objects degraded (7.464%); 1049/191739 objects misplaced (0.547%); 19 MiB/s, 297 objects/s recovering 2024-08-17T17:20:33.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:33 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:20:33.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:33 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3999499647' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:20:33.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:33 smithi162 ceph-mon[104668]: osdmap e1161: 8 total, 8 up, 8 in 2024-08-17T17:20:33.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:33 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 14312/191739 objects degraded (7.464%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:34.013 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:34.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:34.115 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:34.152 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:34.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:34.253 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:34.292 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:34.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:34.393 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:34.456 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:34.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:34.558 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:34.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.610 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:34.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:34.711 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:34.759 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:34.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:34.861 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:34.905 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:34.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:34.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:35.007 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:35.050 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:35.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:35.152 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:35.179 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:35.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:35.280 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:35.306 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:35.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:35.338 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:20:35.338 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:20:35.407 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:35.432 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:35.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:35.534 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:35.564 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:35.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:35.666 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:35.693 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:35.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:35.795 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:35.821 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:35.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:35.854 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:20:35.924 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:35.984 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:35.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:35.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:35.993 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:35 smithi063 ceph-mon[109843]: pgmap v1421: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 13808/188715 objects degraded (7.317%); 1049/188715 objects misplaced (0.556%); 17 MiB/s, 275 objects/s recovering 2024-08-17T17:20:35.994 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:35 smithi063 ceph-mon[114756]: pgmap v1421: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 13808/188715 objects degraded (7.317%); 1049/188715 objects misplaced (0.556%); 17 MiB/s, 275 objects/s recovering 2024-08-17T17:20:36.086 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:35 smithi162 ceph-mon[104668]: pgmap v1421: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 13808/188715 objects degraded (7.317%); 1049/188715 objects misplaced (0.556%); 17 MiB/s, 275 objects/s recovering 2024-08-17T17:20:36.126 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:36.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:36.230 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:36.268 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:36.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:36.370 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:36.409 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:36.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:36.442 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:36.456 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:36.456 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:20:36.511 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:36.542 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:36.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:36.643 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:36.677 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:36.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:36.755 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:36.779 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:36.838 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:36.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:36.940 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:36.968 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:36.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:36.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:37.070 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:37.095 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:37.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:37.197 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:37.231 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:37.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:37.333 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:37.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.363 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:37.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:37.464 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:37.493 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:37.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:37.595 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:37.621 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:37.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:37.723 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:37.747 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:37.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:37.848 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:37.887 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:37.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:37.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:37.989 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:37 smithi063 ceph-mon[109843]: pgmap v1422: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 13808/185787 objects degraded (7.432%); 1049/185787 objects misplaced (0.565%); 17 MiB/s, 275 objects/s recovering 2024-08-17T17:20:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:37 smithi063 ceph-mon[114756]: pgmap v1422: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 13808/185787 objects degraded (7.432%); 1049/185787 objects misplaced (0.565%); 17 MiB/s, 275 objects/s recovering 2024-08-17T17:20:38.032 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:38.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:37 smithi162 ceph-mon[104668]: pgmap v1422: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 13808/185787 objects degraded (7.432%); 1049/185787 objects misplaced (0.565%); 17 MiB/s, 275 objects/s recovering 2024-08-17T17:20:38.134 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:38.162 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:38.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:38.263 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:38.288 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:38.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:38.390 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:38.416 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:38.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:38.518 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:38.560 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:38.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:38.662 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:38.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.703 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:38.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:38.804 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:38.831 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:38.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:38.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:38 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 13808/185787 objects degraded (7.432%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:38.933 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:38.960 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:38.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:38.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:39.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:38 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 13808/185787 objects degraded (7.432%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:38 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 13808/185787 objects degraded (7.432%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:39.062 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:39.096 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:39.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:39.193 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:20:39.193 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:20:39.198 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:39.243 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:39.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:39.345 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:39.371 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:39.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:39.473 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:39.500 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:39.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:39.601 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:39.613 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:39.629 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:39.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:39.731 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:39.759 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:39.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:39.861 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:39.886 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:39.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:39.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:39.974 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:20:39.988 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:39 smithi063 ceph-mon[114756]: pgmap v1423: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 12658/177276 objects degraded (7.140%); 839/177276 objects misplaced (0.473%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:39 smithi063 ceph-mon[109843]: pgmap v1423: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 12658/177276 objects degraded (7.140%); 839/177276 objects misplaced (0.473%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:40.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.033 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:40.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:39 smithi162 ceph-mon[104668]: pgmap v1423: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 776 op/s; 12658/177276 objects degraded (7.140%); 839/177276 objects misplaced (0.473%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:40.134 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:40.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.186 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:40.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:40.260 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:40.286 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:40.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.337 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:40.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:40.437 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:40.483 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:40.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:40.585 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:40.613 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:40.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:40.714 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:40.741 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:40.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:40.843 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:40.868 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:40.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:40.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:40.970 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:41.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.010 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:41.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:41.111 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:41.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.153 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:41.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:41.254 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:41.282 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:41.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:41.383 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:41.408 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:41.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:41.510 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:41.549 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:41.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:41.651 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:41.691 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:41.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:41.793 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:41.798 INFO:teuthology.orchestra.run.smithi063.stderr:set pool 7 pg_num to 36 2024-08-17T17:20:41.822 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:41.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:41.924 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:41.951 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:41.952 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:41 smithi162 ceph-mon[104668]: pgmap v1424: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 12658/174357 objects degraded (7.260%); 839/174357 objects misplaced (0.481%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:41.952 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:41 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1771979114' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:20:41.952 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:41 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:20:41.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:41.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:42.054 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:42.094 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:42.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:42.196 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:42.239 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:42.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:42.251 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:20:42.252 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:20:42.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:41 smithi063 ceph-mon[109843]: pgmap v1424: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 12658/174357 objects degraded (7.260%); 839/174357 objects misplaced (0.481%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:42.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:41 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1771979114' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:20:42.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:41 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:20:42.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:41 smithi063 ceph-mon[114756]: pgmap v1424: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 12658/174357 objects degraded (7.260%); 839/174357 objects misplaced (0.481%); 17 MiB/s, 272 objects/s recovering 2024-08-17T17:20:42.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:41 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1771979114' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:20:42.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:41 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"} : dispatch 2024-08-17T17:20:42.342 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:42.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.380 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:42.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:42.481 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:42.510 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:42.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:42.612 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:42.639 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:42.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:42.741 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:42.765 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:42.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:42.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:42 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"}]': finished 2024-08-17T17:20:42.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:42 smithi162 ceph-mon[104668]: osdmap e1162: 8 total, 8 up, 8 in 2024-08-17T17:20:42.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:42 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T17:20:42.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:42 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T17:20:42.867 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:42.931 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:42.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:42.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:43.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:42 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"}]': finished 2024-08-17T17:20:43.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:42 smithi063 ceph-mon[114756]: osdmap e1162: 8 total, 8 up, 8 in 2024-08-17T17:20:43.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:42 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T17:20:43.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:42 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T17:20:43.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:42 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"}]': finished 2024-08-17T17:20:43.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:42 smithi063 ceph-mon[109843]: osdmap e1162: 8 total, 8 up, 8 in 2024-08-17T17:20:43.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:42 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T17:20:43.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:42 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"} : dispatch 2024-08-17T17:20:43.033 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:43.059 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:20:43.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.072 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:43.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:43.173 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:43.198 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:43.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:43.282 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:43.300 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:43.324 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:43.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:43.401 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:20:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:20:43] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.51.0" 2024-08-17T17:20:43.425 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:43.448 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:43.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:43.550 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:43.573 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:43.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:43.674 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:43.696 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:43.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:43.798 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:43.821 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:43.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:43.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:43.954 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:43.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:43.955 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:44.055 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:44.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.078 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:44.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:44.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:43 smithi162 ceph-mon[104668]: pgmap v1426: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 12048/168513 objects degraded (7.150%); 577/168513 objects misplaced (0.342%); 15 MiB/s, 235 objects/s recovering 2024-08-17T17:20:44.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:43 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"}]': finished 2024-08-17T17:20:44.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:43 smithi162 ceph-mon[104668]: osdmap e1163: 8 total, 8 up, 8 in 2024-08-17T17:20:44.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:43 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 12048/168513 objects degraded (7.150%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:44.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:43 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2839646945' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:44.121 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:20:44.121 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:20:44.121 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:20:44.122 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:20:44.122 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:20:44.122 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:20:44.122 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:20:44.122 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:20:44.122 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:20:44.122 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:20:44.123 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:20:44.123 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:20:44.123 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:20:44.123 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:20:44.123 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:20:44.123 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:20:44.123 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:43 smithi063 ceph-mon[109843]: pgmap v1426: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 12048/168513 objects degraded (7.150%); 577/168513 objects misplaced (0.342%); 15 MiB/s, 235 objects/s recovering 2024-08-17T17:20:44.124 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:43 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"}]': finished 2024-08-17T17:20:44.124 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:43 smithi063 ceph-mon[109843]: osdmap e1163: 8 total, 8 up, 8 in 2024-08-17T17:20:44.124 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:43 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 12048/168513 objects degraded (7.150%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:44.124 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:43 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2839646945' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:44.124 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:43 smithi063 ceph-mon[114756]: pgmap v1426: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 12048/168513 objects degraded (7.150%); 577/168513 objects misplaced (0.342%); 15 MiB/s, 235 objects/s recovering 2024-08-17T17:20:44.124 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:43 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"}]': finished 2024-08-17T17:20:44.124 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:43 smithi063 ceph-mon[114756]: osdmap e1163: 8 total, 8 up, 8 in 2024-08-17T17:20:44.124 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:43 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 12048/168513 objects degraded (7.150%), 13 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:44.124 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:43 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2839646945' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:44.180 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:44.203 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:44.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:44.305 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:44.326 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:44.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:44.428 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:44.450 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:44.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:44.552 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:44.578 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:44.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:44.680 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:44.705 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:44.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:44.807 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:44.831 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:44.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:44.917 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:44 smithi162 ceph-mon[104668]: osdmap e1164: 8 total, 8 up, 8 in 2024-08-17T17:20:44.917 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:44 smithi162 ceph-mon[104668]: 7.a deep-scrub starts 2024-08-17T17:20:44.934 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:44.963 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:44.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:44.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:45.065 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:45.080 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:44 smithi063 ceph-mon[109843]: osdmap e1164: 8 total, 8 up, 8 in 2024-08-17T17:20:45.080 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:44 smithi063 ceph-mon[109843]: 7.a deep-scrub starts 2024-08-17T17:20:45.080 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:44 smithi063 ceph-mon[114756]: osdmap e1164: 8 total, 8 up, 8 in 2024-08-17T17:20:45.080 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:44 smithi063 ceph-mon[114756]: 7.a deep-scrub starts 2024-08-17T17:20:45.091 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:45.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:45.184 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:20:45.184 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:20:45.184 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:20:45.184 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:20:45.184 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.184 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:34052 7.9 7:9b99e1b1:::benchmark_data_smithi063_196002_object34051:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:20:45.185 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:11.455587+0000", 2024-08-17T17:20:45.185 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 153.725388501, 2024-08-17T17:20:45.185 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.17968889699999999, 2024-08-17T17:20:45.185 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.185 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.185 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.185 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.185 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.185 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 34052 2024-08-17T17:20:45.185 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.185 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.185 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.186 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.186 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.455587+0000", 2024-08-17T17:20:45.186 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.186 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.186 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.186 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.186 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.455587+0000", 2024-08-17T17:20:45.186 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.186 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.186 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.186 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.186 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.455586+0000", 2024-08-17T17:20:45.187 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:20:45.187 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.187 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.187 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.187 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.455620+0000", 2024-08-17T17:20:45.187 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3942999999999997e-05 2024-08-17T17:20:45.187 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.187 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.187 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.187 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.455631+0000", 2024-08-17T17:20:45.187 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1877e-05 2024-08-17T17:20:45.187 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.187 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.188 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.188 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.455639+0000", 2024-08-17T17:20:45.188 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1450000000000002e-06 2024-08-17T17:20:45.188 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.188 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.188 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.188 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.457749+0000", 2024-08-17T17:20:45.188 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021101570000000001 2024-08-17T17:20:45.188 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.188 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.188 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.188 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.457875+0000", 2024-08-17T17:20:45.188 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012626900000000001 2024-08-17T17:20:45.188 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.189 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.189 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:20:45.189 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.458003+0000", 2024-08-17T17:20:45.189 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000128484 2024-08-17T17:20:45.189 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.189 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.189 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.189 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.460057+0000", 2024-08-17T17:20:45.189 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002053844 2024-08-17T17:20:45.189 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.189 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.189 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.190 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.462016+0000", 2024-08-17T17:20:45.190 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001958474 2024-08-17T17:20:45.190 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.190 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.190 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.190 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.635203+0000", 2024-08-17T17:20:45.190 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.17318721000000001 2024-08-17T17:20:45.190 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.190 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.190 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.190 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.635243+0000", 2024-08-17T17:20:45.190 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0497000000000002e-05 2024-08-17T17:20:45.191 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.191 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.191 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.191 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.635276+0000", 2024-08-17T17:20:45.191 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2598000000000002e-05 2024-08-17T17:20:45.191 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.191 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.191 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.191 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.191 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.191 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:34054 7.9 7:982e3efb:::benchmark_data_smithi063_196002_object34053:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:20:45.191 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:11.456233+0000", 2024-08-17T17:20:45.191 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 153.72474225100001, 2024-08-17T17:20:45.192 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.179724351, 2024-08-17T17:20:45.192 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.192 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.192 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.192 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.192 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.192 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 34054 2024-08-17T17:20:45.192 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.192 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.192 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.192 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.192 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.456233+0000", 2024-08-17T17:20:45.192 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.193 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.193 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.193 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.193 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.456233+0000", 2024-08-17T17:20:45.193 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.193 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.193 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.193 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.193 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.456233+0000", 2024-08-17T17:20:45.193 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:20:45.193 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.193 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.193 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.194 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.456262+0000", 2024-08-17T17:20:45.194 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9451999999999999e-05 2024-08-17T17:20:45.194 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.194 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.194 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.194 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.456273+0000", 2024-08-17T17:20:45.194 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0594999999999999e-05 2024-08-17T17:20:45.194 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.194 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.194 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.194 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.456280+0000", 2024-08-17T17:20:45.194 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.939e-06 2024-08-17T17:20:45.194 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.195 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.195 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.195 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.460110+0000", 2024-08-17T17:20:45.195 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0038299839999999998 2024-08-17T17:20:45.195 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.195 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.195 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.195 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.460241+0000", 2024-08-17T17:20:45.195 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013128 2024-08-17T17:20:45.195 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.195 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.195 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:20:45.195 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.460367+0000", 2024-08-17T17:20:45.196 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012641900000000001 2024-08-17T17:20:45.196 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.196 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.196 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.196 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.463048+0000", 2024-08-17T17:20:45.196 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026806909999999998 2024-08-17T17:20:45.196 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.196 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.196 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.196 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.463810+0000", 2024-08-17T17:20:45.196 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00076169900000000001 2024-08-17T17:20:45.196 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.196 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.197 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.197 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.635881+0000", 2024-08-17T17:20:45.197 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.17207149699999999 2024-08-17T17:20:45.197 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.197 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.197 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.197 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.635925+0000", 2024-08-17T17:20:45.197 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4042000000000003e-05 2024-08-17T17:20:45.197 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.197 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.197 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.197 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.635958+0000", 2024-08-17T17:20:45.197 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2611999999999999e-05 2024-08-17T17:20:45.197 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.198 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.198 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.198 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.198 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.198 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:34061 7.4 7:20e5371f:::benchmark_data_smithi063_196002_object34060:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:20:45.198 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:11.457894+0000", 2024-08-17T17:20:45.198 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 153.723081251, 2024-08-17T17:20:45.198 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.17540006299999999, 2024-08-17T17:20:45.198 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.198 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.198 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.198 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.199 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:45.200 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.200 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 34061 2024-08-17T17:20:45.200 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.200 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.200 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.200 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.200 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.457894+0000", 2024-08-17T17:20:45.200 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.200 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.201 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.201 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.201 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.457894+0000", 2024-08-17T17:20:45.201 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.201 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.201 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.201 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.201 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.457894+0000", 2024-08-17T17:20:45.201 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:20:45.201 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.201 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.201 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.201 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.457923+0000", 2024-08-17T17:20:45.202 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.976e-05 2024-08-17T17:20:45.202 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.202 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.202 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.202 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.457933+0000", 2024-08-17T17:20:45.202 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4900000000000006e-06 2024-08-17T17:20:45.202 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.202 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.202 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.202 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.457939+0000", 2024-08-17T17:20:45.202 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8309999999999997e-06 2024-08-17T17:20:45.203 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.203 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.203 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.203 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.458010+0000", 2024-08-17T17:20:45.203 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0912000000000001e-05 2024-08-17T17:20:45.203 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.203 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.203 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.203 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.458133+0000", 2024-08-17T17:20:45.203 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012318999999999999 2024-08-17T17:20:45.203 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.203 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.203 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:20:45.203 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.458267+0000", 2024-08-17T17:20:45.204 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013418499999999999 2024-08-17T17:20:45.204 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.204 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.204 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.204 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.461344+0000", 2024-08-17T17:20:45.204 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0030775360000000001 2024-08-17T17:20:45.204 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.204 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.204 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.204 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.463385+0000", 2024-08-17T17:20:45.204 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0020405029999999999 2024-08-17T17:20:45.204 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.205 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.205 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.205 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.633206+0000", 2024-08-17T17:20:45.205 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.169820835 2024-08-17T17:20:45.205 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.205 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.205 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.205 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.633253+0000", 2024-08-17T17:20:45.205 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7414999999999999e-05 2024-08-17T17:20:45.206 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.206 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.206 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.206 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.633295+0000", 2024-08-17T17:20:45.206 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1315e-05 2024-08-17T17:20:45.206 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.206 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.206 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.206 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.206 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.206 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:34070 7.9 7:9a4be4ae:::benchmark_data_smithi063_196002_object34069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:20:45.206 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:11.465050+0000", 2024-08-17T17:20:45.206 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 153.715925656, 2024-08-17T17:20:45.207 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.17100792400000001, 2024-08-17T17:20:45.207 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.207 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.207 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.207 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.207 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.207 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 34070 2024-08-17T17:20:45.207 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.207 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.207 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.207 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.207 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.465050+0000", 2024-08-17T17:20:45.207 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.207 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.208 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.208 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.208 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.465050+0000", 2024-08-17T17:20:45.208 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.208 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.208 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.208 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.208 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.465049+0000", 2024-08-17T17:20:45.208 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:20:45.208 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.208 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.208 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.208 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.465089+0000", 2024-08-17T17:20:45.208 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0528000000000002e-05 2024-08-17T17:20:45.208 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.209 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.209 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.209 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.465100+0000", 2024-08-17T17:20:45.209 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0677e-05 2024-08-17T17:20:45.209 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.209 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.209 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.209 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.465113+0000", 2024-08-17T17:20:45.209 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3001e-05 2024-08-17T17:20:45.209 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.209 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.209 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.209 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.465161+0000", 2024-08-17T17:20:45.209 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8498000000000002e-05 2024-08-17T17:20:45.210 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.210 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.210 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.210 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.465297+0000", 2024-08-17T17:20:45.210 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013569400000000001 2024-08-17T17:20:45.210 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.210 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.210 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:20:45.210 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.465441+0000", 2024-08-17T17:20:45.210 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014416200000000001 2024-08-17T17:20:45.210 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.210 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.210 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.211 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.468339+0000", 2024-08-17T17:20:45.211 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0028976449999999999 2024-08-17T17:20:45.211 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.211 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.211 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.211 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.469750+0000", 2024-08-17T17:20:45.211 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0014113249999999999 2024-08-17T17:20:45.211 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.211 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.211 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.211 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.635989+0000", 2024-08-17T17:20:45.211 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.16623886700000001 2024-08-17T17:20:45.212 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.212 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.212 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.212 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.636028+0000", 2024-08-17T17:20:45.212 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9310000000000001e-05 2024-08-17T17:20:45.212 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.212 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.212 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.212 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.636058+0000", 2024-08-17T17:20:45.212 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9748e-05 2024-08-17T17:20:45.212 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.212 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.212 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.213 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.213 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.213 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:34075 7.4 7:2faa5755:::benchmark_data_smithi063_196002_object34074:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:20:45.213 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:11.466795+0000", 2024-08-17T17:20:45.213 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 153.714180677, 2024-08-17T17:20:45.213 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.170088922, 2024-08-17T17:20:45.213 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.213 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.213 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.213 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.213 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.213 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 34075 2024-08-17T17:20:45.213 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.214 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.214 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.214 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.214 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.466795+0000", 2024-08-17T17:20:45.214 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.214 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.214 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.214 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.214 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.466795+0000", 2024-08-17T17:20:45.214 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.214 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.214 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.214 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.215 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.466794+0000", 2024-08-17T17:20:45.215 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:20:45.215 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.215 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.215 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.215 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.466836+0000", 2024-08-17T17:20:45.215 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.286e-05 2024-08-17T17:20:45.215 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.215 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.215 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.215 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.466847+0000", 2024-08-17T17:20:45.215 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0909e-05 2024-08-17T17:20:45.215 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.216 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.216 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.216 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.466854+0000", 2024-08-17T17:20:45.216 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9580000000000001e-06 2024-08-17T17:20:45.216 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.216 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.216 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.216 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.466909+0000", 2024-08-17T17:20:45.216 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4543000000000003e-05 2024-08-17T17:20:45.216 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.216 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.216 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.216 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.467034+0000", 2024-08-17T17:20:45.216 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000125146 2024-08-17T17:20:45.217 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.217 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.217 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:20:45.217 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.467228+0000", 2024-08-17T17:20:45.217 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00019413699999999999 2024-08-17T17:20:45.217 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.217 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.217 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.217 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.469478+0000", 2024-08-17T17:20:45.217 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0022495380000000001 2024-08-17T17:20:45.217 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.217 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.217 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.218 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.469716+0000", 2024-08-17T17:20:45.218 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00023819600000000001 2024-08-17T17:20:45.218 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.218 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.218 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.218 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.636669+0000", 2024-08-17T17:20:45.218 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.16695337399999999 2024-08-17T17:20:45.218 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.218 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.218 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.218 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.636750+0000", 2024-08-17T17:20:45.218 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.1007000000000006e-05 2024-08-17T17:20:45.218 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.219 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.219 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.219 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.636884+0000", 2024-08-17T17:20:45.219 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000133638 2024-08-17T17:20:45.219 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.219 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.219 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.219 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.219 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.219 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:34077 7.9 7:95f53d41:::benchmark_data_smithi063_196002_object34076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:20:45.219 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:11.468190+0000", 2024-08-17T17:20:45.219 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 153.712785424, 2024-08-17T17:20:45.220 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.167964851, 2024-08-17T17:20:45.220 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.220 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.220 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.220 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.220 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.220 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 34077 2024-08-17T17:20:45.220 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.220 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.220 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.220 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.220 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.468190+0000", 2024-08-17T17:20:45.220 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.220 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.221 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.221 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.221 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.468190+0000", 2024-08-17T17:20:45.221 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.221 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.221 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.221 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.221 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.468189+0000", 2024-08-17T17:20:45.221 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:20:45.221 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.221 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.221 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.221 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.468260+0000", 2024-08-17T17:20:45.222 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1378999999999995e-05 2024-08-17T17:20:45.222 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.222 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.222 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.222 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.468271+0000", 2024-08-17T17:20:45.222 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0295999999999999e-05 2024-08-17T17:20:45.222 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.222 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.222 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.222 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.468277+0000", 2024-08-17T17:20:45.222 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6100000000000002e-06 2024-08-17T17:20:45.222 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.223 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.223 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.223 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.468361+0000", 2024-08-17T17:20:45.223 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3469999999999999e-05 2024-08-17T17:20:45.223 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.223 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.223 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.223 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.468491+0000", 2024-08-17T17:20:45.223 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000130455 2024-08-17T17:20:45.223 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.223 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.223 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:20:45.223 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.468682+0000", 2024-08-17T17:20:45.224 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00019101400000000001 2024-08-17T17:20:45.224 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.224 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.224 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.224 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.469874+0000", 2024-08-17T17:20:45.224 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001191908 2024-08-17T17:20:45.224 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.224 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.224 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.224 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.470520+0000", 2024-08-17T17:20:45.224 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00064543800000000002 2024-08-17T17:20:45.224 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.224 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.224 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.224 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.636087+0000", 2024-08-17T17:20:45.225 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.16556689299999999 2024-08-17T17:20:45.225 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.225 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.225 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.225 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.636126+0000", 2024-08-17T17:20:45.225 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9221000000000003e-05 2024-08-17T17:20:45.225 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.225 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.225 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.225 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.636155+0000", 2024-08-17T17:20:45.225 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9346999999999999e-05 2024-08-17T17:20:45.225 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.225 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.225 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.226 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.226 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.226 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:34083 7.4 7:23300d26:::benchmark_data_smithi063_196002_object34082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:20:45.226 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:11.471704+0000", 2024-08-17T17:20:45.226 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 153.70927175599999, 2024-08-17T17:20:45.226 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.165769903, 2024-08-17T17:20:45.226 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.226 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.226 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.226 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.226 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.226 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 34083 2024-08-17T17:20:45.227 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.227 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.227 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.227 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.227 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471704+0000", 2024-08-17T17:20:45.227 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.227 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.227 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.227 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.227 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471704+0000", 2024-08-17T17:20:45.227 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.227 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.227 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.228 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.228 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471703+0000", 2024-08-17T17:20:45.228 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:20:45.228 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.228 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.228 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.228 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471762+0000", 2024-08-17T17:20:45.228 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9077999999999999e-05 2024-08-17T17:20:45.228 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.228 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.228 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.228 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471772+0000", 2024-08-17T17:20:45.228 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.978e-06 2024-08-17T17:20:45.228 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.229 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.229 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.229 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471778+0000", 2024-08-17T17:20:45.229 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7150000000000003e-06 2024-08-17T17:20:45.229 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.229 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.229 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.229 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471825+0000", 2024-08-17T17:20:45.229 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7543999999999997e-05 2024-08-17T17:20:45.229 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.229 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.229 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.229 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.471963+0000", 2024-08-17T17:20:45.230 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000137446 2024-08-17T17:20:45.230 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.230 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.230 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:20:45.230 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.472059+0000", 2024-08-17T17:20:45.230 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.5950000000000001e-05 2024-08-17T17:20:45.230 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.230 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.230 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.230 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.473230+0000", 2024-08-17T17:20:45.230 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001171402 2024-08-17T17:20:45.230 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.230 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.231 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.231 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.473621+0000", 2024-08-17T17:20:45.231 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00039081700000000001 2024-08-17T17:20:45.231 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.231 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.231 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.231 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.636943+0000", 2024-08-17T17:20:45.231 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.163322257 2024-08-17T17:20:45.231 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.231 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.231 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.231 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.637430+0000", 2024-08-17T17:20:45.231 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00048677099999999999 2024-08-17T17:20:45.232 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.232 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.232 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.232 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.637474+0000", 2024-08-17T17:20:45.232 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3862000000000001e-05 2024-08-17T17:20:45.232 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.232 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.232 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.232 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.232 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.232 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65730 7.9 7:90546b68:::benchmark_data_smithi063_196002_object65729:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-08-17T17:20:45.232 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.520085+0000", 2024-08-17T17:20:45.233 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 134.660891085, 2024-08-17T17:20:45.233 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.528465084, 2024-08-17T17:20:45.233 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.233 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:20:45.233 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.233 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.233 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.233 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65730 2024-08-17T17:20:45.233 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.233 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.233 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.233 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.233 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520085+0000", 2024-08-17T17:20:45.234 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.234 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.234 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.234 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.234 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520085+0000", 2024-08-17T17:20:45.234 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.234 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.234 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.234 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.234 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520083+0000", 2024-08-17T17:20:45.234 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:20:45.234 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.235 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.235 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.235 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520144+0000", 2024-08-17T17:20:45.235 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0470000000000002e-05 2024-08-17T17:20:45.235 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.235 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.235 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.235 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520162+0000", 2024-08-17T17:20:45.235 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.8495000000000002e-05 2024-08-17T17:20:45.235 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.235 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.235 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.235 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520170+0000", 2024-08-17T17:20:45.235 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.5240000000000003e-06 2024-08-17T17:20:45.236 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.236 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.236 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.236 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520241+0000", 2024-08-17T17:20:45.236 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0772000000000006e-05 2024-08-17T17:20:45.236 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.236 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.236 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.236 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520372+0000", 2024-08-17T17:20:45.236 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000131113 2024-08-17T17:20:45.236 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.236 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.236 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:20:45.236 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520505+0000", 2024-08-17T17:20:45.237 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000133363 2024-08-17T17:20:45.237 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.237 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.237 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.237 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535294+0000", 2024-08-17T17:20:45.237 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.014789412 2024-08-17T17:20:45.237 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.237 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.237 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.237 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535306+0000", 2024-08-17T17:20:45.237 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2003e-05 2024-08-17T17:20:45.237 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.237 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.238 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.238 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.535092+0000", 2024-08-17T17:20:45.238 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99978559499999997 2024-08-17T17:20:45.238 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.238 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.238 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.238 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.535104+0000", 2024-08-17T17:20:45.238 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1783999999999999e-05 2024-08-17T17:20:45.238 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.238 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.238 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.238 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.034715+0000", 2024-08-17T17:20:45.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.499611566 2024-08-17T17:20:45.239 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.239 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.239 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for degraded object", 2024-08-17T17:20:45.239 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.036087+0000", 2024-08-17T17:20:45.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013718249999999999 2024-08-17T17:20:45.239 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.239 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.239 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.239 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.048433+0000", 2024-08-17T17:20:45.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012345626 2024-08-17T17:20:45.239 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.239 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.239 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.240 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.048550+0000", 2024-08-17T17:20:45.240 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011689200000000001 2024-08-17T17:20:45.240 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.240 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.240 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.240 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.240 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.240 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65762 7.4 7:25ea59d9:::benchmark_data_smithi063_196002_object65761:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:20:45.240 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.559197+0000", 2024-08-17T17:20:45.240 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 134.62177885, 2024-08-17T17:20:45.240 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5033075609999997, 2024-08-17T17:20:45.240 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.240 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.240 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.241 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.241 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.241 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65762 2024-08-17T17:20:45.241 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.241 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.241 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.241 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.241 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559197+0000", 2024-08-17T17:20:45.241 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.241 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.241 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.241 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.241 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559197+0000", 2024-08-17T17:20:45.242 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.242 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.242 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.242 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.242 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559195+0000", 2024-08-17T17:20:45.242 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:20:45.242 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.242 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.242 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.242 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559275+0000", 2024-08-17T17:20:45.242 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9611999999999996e-05 2024-08-17T17:20:45.242 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.242 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.243 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.243 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559293+0000", 2024-08-17T17:20:45.243 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7776000000000001e-05 2024-08-17T17:20:45.243 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.243 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.243 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.243 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559302+0000", 2024-08-17T17:20:45.243 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2720000000000003e-06 2024-08-17T17:20:45.243 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.243 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.243 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.244 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559362+0000", 2024-08-17T17:20:45.244 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0421000000000003e-05 2024-08-17T17:20:45.245 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.245 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.245 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.245 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559373+0000", 2024-08-17T17:20:45.245 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0739e-05 2024-08-17T17:20:45.245 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.245 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.245 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.245 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.533131+0000", 2024-08-17T17:20:45.245 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.973758291 2024-08-17T17:20:45.245 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.245 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.246 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.246 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.533145+0000", 2024-08-17T17:20:45.246 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3608000000000001e-05 2024-08-17T17:20:45.246 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.246 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.246 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.246 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.035648+0000", 2024-08-17T17:20:45.246 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5025030340000001 2024-08-17T17:20:45.246 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.246 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.246 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.246 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.035818+0000", 2024-08-17T17:20:45.247 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016974000000000001 2024-08-17T17:20:45.247 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.247 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.247 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:20:45.247 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.035978+0000", 2024-08-17T17:20:45.247 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015974199999999999 2024-08-17T17:20:45.247 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.247 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.247 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.247 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.048070+0000", 2024-08-17T17:20:45.247 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012092233000000001 2024-08-17T17:20:45.247 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.247 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.248 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.248 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.048159+0000", 2024-08-17T17:20:45.248 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8843000000000003e-05 2024-08-17T17:20:45.248 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.248 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.248 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.248 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.059346+0000", 2024-08-17T17:20:45.248 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011186928 2024-08-17T17:20:45.248 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.248 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.248 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.248 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.059395+0000", 2024-08-17T17:20:45.248 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9570000000000001e-05 2024-08-17T17:20:45.248 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.249 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.249 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.249 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.062504+0000", 2024-08-17T17:20:45.249 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0031092400000000001 2024-08-17T17:20:45.249 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.249 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.249 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.249 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.249 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.249 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65764 7.9 7:95797802:::benchmark_data_smithi063_196002_object65763:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:20:45.249 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.559319+0000", 2024-08-17T17:20:45.249 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 134.62165718, 2024-08-17T17:20:45.249 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4950287260000001, 2024-08-17T17:20:45.250 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.250 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.250 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.250 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.250 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.250 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65764 2024-08-17T17:20:45.250 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.250 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.250 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.250 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.250 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559319+0000", 2024-08-17T17:20:45.250 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.251 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.251 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.251 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.251 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559319+0000", 2024-08-17T17:20:45.251 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.251 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.251 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.251 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.251 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559318+0000", 2024-08-17T17:20:45.251 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:20:45.251 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.251 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.251 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.252 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559380+0000", 2024-08-17T17:20:45.252 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1808000000000002e-05 2024-08-17T17:20:45.252 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.252 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.252 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.252 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559386+0000", 2024-08-17T17:20:45.252 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3300000000000004e-06 2024-08-17T17:20:45.252 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.252 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.252 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.252 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559392+0000", 2024-08-17T17:20:45.252 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.789e-06 2024-08-17T17:20:45.253 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.253 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.253 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.253 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559433+0000", 2024-08-17T17:20:45.253 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1162999999999999e-05 2024-08-17T17:20:45.253 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.253 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.253 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.253 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559440+0000", 2024-08-17T17:20:45.253 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4680000000000003e-06 2024-08-17T17:20:45.253 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.253 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.253 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.254 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.535130+0000", 2024-08-17T17:20:45.254 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97568918999999998 2024-08-17T17:20:45.254 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.254 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.254 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.254 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.535138+0000", 2024-08-17T17:20:45.254 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.2700000000000004e-06 2024-08-17T17:20:45.254 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.254 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.254 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.254 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.039509+0000", 2024-08-17T17:20:45.254 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.504371227 2024-08-17T17:20:45.255 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.255 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.255 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.255 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.039651+0000", 2024-08-17T17:20:45.255 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014202400000000001 2024-08-17T17:20:45.255 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.255 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.255 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:20:45.255 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.039809+0000", 2024-08-17T17:20:45.255 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015809 2024-08-17T17:20:45.255 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.255 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.255 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.255 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.048393+0000", 2024-08-17T17:20:45.256 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.008583528 2024-08-17T17:20:45.256 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.256 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.256 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.256 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.048682+0000", 2024-08-17T17:20:45.256 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00028978599999999999 2024-08-17T17:20:45.256 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.256 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.256 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.256 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.052659+0000", 2024-08-17T17:20:45.256 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0039768340000000003 2024-08-17T17:20:45.256 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.256 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.257 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.257 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.052700+0000", 2024-08-17T17:20:45.257 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0844999999999999e-05 2024-08-17T17:20:45.257 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.257 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.257 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.257 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.054347+0000", 2024-08-17T17:20:45.257 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016471050000000001 2024-08-17T17:20:45.257 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.257 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.257 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.257 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.257 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.258 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88383 7.19 7:9fffae81:::benchmark_data_smithi063_196002_object88382:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:20:45.258 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:00.933708+0000", 2024-08-17T17:20:45.258 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 104.247267948, 2024-08-17T17:20:45.258 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.845462465, 2024-08-17T17:20:45.258 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.258 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.258 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.258 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.258 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.258 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88383 2024-08-17T17:20:45.258 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.258 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.258 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.259 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.259 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933708+0000", 2024-08-17T17:20:45.259 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.259 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.259 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.259 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.259 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933708+0000", 2024-08-17T17:20:45.259 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.259 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.259 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.259 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.259 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933706+0000", 2024-08-17T17:20:45.259 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:20:45.260 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.260 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.260 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.260 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933753+0000", 2024-08-17T17:20:45.260 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6516e-05 2024-08-17T17:20:45.260 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.260 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.260 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.260 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933766+0000", 2024-08-17T17:20:45.260 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2879e-05 2024-08-17T17:20:45.260 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.260 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.261 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.261 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933774+0000", 2024-08-17T17:20:45.261 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9829999999999996e-06 2024-08-17T17:20:45.261 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.261 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.261 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.261 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933823+0000", 2024-08-17T17:20:45.261 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8936e-05 2024-08-17T17:20:45.261 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.261 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.261 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.261 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933830+0000", 2024-08-17T17:20:45.261 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6229999999999997e-06 2024-08-17T17:20:45.262 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.262 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.262 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.262 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.773096+0000", 2024-08-17T17:20:45.262 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.83926587799999997 2024-08-17T17:20:45.262 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.262 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.262 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.262 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.773204+0000", 2024-08-17T17:20:45.262 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010809199999999999 2024-08-17T17:20:45.262 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.262 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.262 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:20:45.263 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.773306+0000", 2024-08-17T17:20:45.263 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010179399999999999 2024-08-17T17:20:45.263 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.263 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.263 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.263 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.775896+0000", 2024-08-17T17:20:45.263 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0025904669999999999 2024-08-17T17:20:45.263 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.263 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.263 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.263 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.775925+0000", 2024-08-17T17:20:45.263 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8167e-05 2024-08-17T17:20:45.263 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.264 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.264 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.264 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.777385+0000", 2024-08-17T17:20:45.264 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0014603400000000001 2024-08-17T17:20:45.264 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.264 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.264 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.264 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.777464+0000", 2024-08-17T17:20:45.264 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9040000000000002e-05 2024-08-17T17:20:45.264 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.264 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.264 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.264 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.779170+0000", 2024-08-17T17:20:45.265 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0017062290000000001 2024-08-17T17:20:45.265 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.265 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.265 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.265 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.265 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.265 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88386 7.4 7:22787498:::benchmark_data_smithi063_196002_object88385:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:20:45.265 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:00.979852+0000", 2024-08-17T17:20:45.265 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 104.201123899, 2024-08-17T17:20:45.265 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7198389029999999, 2024-08-17T17:20:45.265 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.265 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.266 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.266 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.266 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.266 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88386 2024-08-17T17:20:45.266 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.266 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.266 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.266 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.266 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979852+0000", 2024-08-17T17:20:45.267 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.267 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.267 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.267 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.267 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979852+0000", 2024-08-17T17:20:45.267 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.267 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.267 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.267 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.267 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979850+0000", 2024-08-17T17:20:45.267 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:20:45.267 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.268 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.268 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.268 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979920+0000", 2024-08-17T17:20:45.268 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9574999999999997e-05 2024-08-17T17:20:45.268 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.268 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.268 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.268 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979935+0000", 2024-08-17T17:20:45.268 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5150999999999999e-05 2024-08-17T17:20:45.268 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.268 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.268 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.268 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979951+0000", 2024-08-17T17:20:45.269 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6509999999999999e-05 2024-08-17T17:20:45.269 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.269 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.269 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.269 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.980004+0000", 2024-08-17T17:20:45.269 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2992999999999999e-05 2024-08-17T17:20:45.269 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.269 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.269 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.269 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.980013+0000", 2024-08-17T17:20:45.269 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3270000000000001e-06 2024-08-17T17:20:45.269 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.269 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.270 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.270 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.653569+0000", 2024-08-17T17:20:45.270 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.67355651100000002 2024-08-17T17:20:45.270 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.270 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.270 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.270 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.653581+0000", 2024-08-17T17:20:45.270 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2017e-05 2024-08-17T17:20:45.270 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.270 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.270 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.270 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.673365+0000", 2024-08-17T17:20:45.270 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.01978422 2024-08-17T17:20:45.271 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.271 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.271 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.271 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.673553+0000", 2024-08-17T17:20:45.271 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00018712399999999999 2024-08-17T17:20:45.271 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.271 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.271 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:20:45.271 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.673745+0000", 2024-08-17T17:20:45.271 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00019201600000000001 2024-08-17T17:20:45.271 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.271 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.271 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.272 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.698545+0000", 2024-08-17T17:20:45.272 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.024800866000000001 2024-08-17T17:20:45.272 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.272 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.272 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.272 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.698579+0000", 2024-08-17T17:20:45.272 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3624000000000002e-05 2024-08-17T17:20:45.272 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.272 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.272 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.272 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.698674+0000", 2024-08-17T17:20:45.272 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4993000000000003e-05 2024-08-17T17:20:45.272 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.272 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.273 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.273 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.698734+0000", 2024-08-17T17:20:45.273 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9583000000000002e-05 2024-08-17T17:20:45.273 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.273 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.273 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.273 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.699691+0000", 2024-08-17T17:20:45.273 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00095704399999999999 2024-08-17T17:20:45.273 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.273 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.273 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.273 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.273 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.274 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88387 7.19 7:9eee2576:::benchmark_data_smithi063_196002_object88386:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:20:45.274 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:00.979965+0000", 2024-08-17T17:20:45.274 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 104.20101056599999, 2024-08-17T17:20:45.274 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.805482531, 2024-08-17T17:20:45.274 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.274 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.274 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.274 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.274 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.274 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88387 2024-08-17T17:20:45.274 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.275 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.275 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.275 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.275 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979965+0000", 2024-08-17T17:20:45.275 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.275 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.275 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.275 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.275 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979965+0000", 2024-08-17T17:20:45.275 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.275 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.275 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.275 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.275 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979965+0000", 2024-08-17T17:20:45.276 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:20:45.276 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.276 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.276 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.276 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979994+0000", 2024-08-17T17:20:45.276 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8992999999999998e-05 2024-08-17T17:20:45.276 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.276 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.276 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.276 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.980003+0000", 2024-08-17T17:20:45.276 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3100000000000006e-06 2024-08-17T17:20:45.276 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.277 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.277 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.277 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.980009+0000", 2024-08-17T17:20:45.277 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3269999999999997e-06 2024-08-17T17:20:45.277 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.277 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.277 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.277 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.980048+0000", 2024-08-17T17:20:45.277 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8420000000000001e-05 2024-08-17T17:20:45.277 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.277 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.277 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.277 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.980054+0000", 2024-08-17T17:20:45.278 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8090000000000004e-06 2024-08-17T17:20:45.278 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.278 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.278 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.278 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.774383+0000", 2024-08-17T17:20:45.278 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.79432830300000001 2024-08-17T17:20:45.278 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.278 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.278 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.278 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.774533+0000", 2024-08-17T17:20:45.278 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015014999999999999 2024-08-17T17:20:45.278 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.279 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.279 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:20:45.279 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.774714+0000", 2024-08-17T17:20:45.279 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00018148099999999999 2024-08-17T17:20:45.279 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.279 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.279 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.279 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.776025+0000", 2024-08-17T17:20:45.279 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013108149999999999 2024-08-17T17:20:45.279 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.279 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.279 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.279 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.776180+0000", 2024-08-17T17:20:45.279 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000155198 2024-08-17T17:20:45.280 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.280 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.280 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.280 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.783976+0000", 2024-08-17T17:20:45.280 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0077961020000000001 2024-08-17T17:20:45.280 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.280 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.280 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.280 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.784011+0000", 2024-08-17T17:20:45.280 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4273000000000002e-05 2024-08-17T17:20:45.280 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.280 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.280 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.281 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.785448+0000", 2024-08-17T17:20:45.281 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001436959 2024-08-17T17:20:45.281 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.281 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.281 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.281 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.281 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.281 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88393 7.9 7:9286a306:::benchmark_data_smithi063_196002_object88392:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:20:45.281 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.004627+0000", 2024-08-17T17:20:45.281 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 104.176348832, 2024-08-17T17:20:45.281 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.701585377, 2024-08-17T17:20:45.281 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.281 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.282 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.282 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.282 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.282 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88393 2024-08-17T17:20:45.282 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.282 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.282 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.282 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.282 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004627+0000", 2024-08-17T17:20:45.282 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.282 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.282 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.283 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.283 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004627+0000", 2024-08-17T17:20:45.283 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.283 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.283 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.283 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.283 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004625+0000", 2024-08-17T17:20:45.283 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:20:45.283 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.283 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.283 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.283 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004663+0000", 2024-08-17T17:20:45.283 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8390999999999999e-05 2024-08-17T17:20:45.283 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.284 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.284 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.284 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004678+0000", 2024-08-17T17:20:45.284 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4637999999999999e-05 2024-08-17T17:20:45.284 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.284 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.284 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.284 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004686+0000", 2024-08-17T17:20:45.284 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4169999999999992e-06 2024-08-17T17:20:45.284 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.284 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.284 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.284 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004734+0000", 2024-08-17T17:20:45.284 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7255000000000001e-05 2024-08-17T17:20:45.284 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.285 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.285 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.285 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004741+0000", 2024-08-17T17:20:45.285 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6179999999999997e-06 2024-08-17T17:20:45.285 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.285 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.285 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.285 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.655714+0000", 2024-08-17T17:20:45.285 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.65097307000000004 2024-08-17T17:20:45.285 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.285 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.285 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.285 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.655724+0000", 2024-08-17T17:20:45.285 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8490000000000001e-06 2024-08-17T17:20:45.286 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.286 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.286 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.286 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.678130+0000", 2024-08-17T17:20:45.286 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0224053420000001 2024-08-17T17:20:45.286 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.286 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.286 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.286 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.678331+0000", 2024-08-17T17:20:45.286 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00020100399999999999 2024-08-17T17:20:45.286 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.286 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.286 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:20:45.287 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.678525+0000", 2024-08-17T17:20:45.287 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000194809 2024-08-17T17:20:45.287 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.287 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.287 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.287 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.702033+0000", 2024-08-17T17:20:45.287 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.023507920000000002 2024-08-17T17:20:45.287 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.287 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.287 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.287 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.702074+0000", 2024-08-17T17:20:45.287 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0812999999999997e-05 2024-08-17T17:20:45.287 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.288 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.288 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.288 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.704880+0000", 2024-08-17T17:20:45.288 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0028057170000000001 2024-08-17T17:20:45.288 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.288 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.288 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.288 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.704938+0000", 2024-08-17T17:20:45.288 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7982999999999997e-05 2024-08-17T17:20:45.288 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.288 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.288 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.288 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.706212+0000", 2024-08-17T17:20:45.289 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001274358 2024-08-17T17:20:45.289 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.289 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.289 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.289 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.289 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.289 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88394 7.1 7:80428bd7:::benchmark_data_smithi063_196002_object88393:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:20:45.289 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.006598+0000", 2024-08-17T17:20:45.289 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 104.174377647, 2024-08-17T17:20:45.289 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.69599852200000001, 2024-08-17T17:20:45.289 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.289 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.290 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.290 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.290 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.290 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88394 2024-08-17T17:20:45.290 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.290 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.290 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.290 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.290 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006598+0000", 2024-08-17T17:20:45.290 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.290 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.290 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.290 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.291 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006598+0000", 2024-08-17T17:20:45.291 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.291 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.291 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.291 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.291 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006597+0000", 2024-08-17T17:20:45.291 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:20:45.291 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.291 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.291 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.291 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006649+0000", 2024-08-17T17:20:45.291 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2046000000000003e-05 2024-08-17T17:20:45.291 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.292 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.292 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.292 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006661+0000", 2024-08-17T17:20:45.292 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2234e-05 2024-08-17T17:20:45.292 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.292 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.292 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.292 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006669+0000", 2024-08-17T17:20:45.292 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4610000000000002e-06 2024-08-17T17:20:45.292 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.292 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.292 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.292 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006720+0000", 2024-08-17T17:20:45.293 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1153999999999998e-05 2024-08-17T17:20:45.293 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.293 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.293 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.293 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006728+0000", 2024-08-17T17:20:45.293 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.1100000000000003e-06 2024-08-17T17:20:45.293 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.293 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.293 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.293 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.684581+0000", 2024-08-17T17:20:45.293 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.67785242599999995 2024-08-17T17:20:45.293 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.293 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.294 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.294 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.684731+0000", 2024-08-17T17:20:45.294 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015084699999999999 2024-08-17T17:20:45.294 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.294 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.294 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:20:45.294 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.684900+0000", 2024-08-17T17:20:45.294 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016867 2024-08-17T17:20:45.294 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.294 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.294 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.294 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.686413+0000", 2024-08-17T17:20:45.295 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001513295 2024-08-17T17:20:45.295 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.295 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.295 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.295 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.686609+0000", 2024-08-17T17:20:45.295 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00019517699999999999 2024-08-17T17:20:45.295 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.295 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.295 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.295 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.702497+0000", 2024-08-17T17:20:45.295 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.015888475999999999 2024-08-17T17:20:45.295 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.295 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.296 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.296 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.702555+0000", 2024-08-17T17:20:45.296 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8285000000000002e-05 2024-08-17T17:20:45.296 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.296 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.296 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.296 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.702597+0000", 2024-08-17T17:20:45.296 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1230000000000003e-05 2024-08-17T17:20:45.296 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.296 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.296 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.296 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.297 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.297 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88399 7.9 7:95c6c148:::benchmark_data_smithi063_196002_object88398:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:20:45.297 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.034388+0000", 2024-08-17T17:20:45.297 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 104.146587439, 2024-08-17T17:20:45.297 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.687116794, 2024-08-17T17:20:45.297 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.297 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.297 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.297 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.297 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.297 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88399 2024-08-17T17:20:45.297 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.298 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.298 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.298 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.298 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034388+0000", 2024-08-17T17:20:45.298 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.298 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.298 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.298 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.298 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034388+0000", 2024-08-17T17:20:45.298 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.298 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.298 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.298 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.299 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034387+0000", 2024-08-17T17:20:45.299 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:20:45.299 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.299 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.299 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.299 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034418+0000", 2024-08-17T17:20:45.299 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1087999999999998e-05 2024-08-17T17:20:45.299 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.299 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.299 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.299 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034434+0000", 2024-08-17T17:20:45.299 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6064000000000001e-05 2024-08-17T17:20:45.299 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.300 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.300 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.300 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034444+0000", 2024-08-17T17:20:45.300 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0078000000000001e-05 2024-08-17T17:20:45.300 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.300 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.300 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.300 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034503+0000", 2024-08-17T17:20:45.300 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9336000000000002e-05 2024-08-17T17:20:45.300 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.300 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.300 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.300 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034513+0000", 2024-08-17T17:20:45.301 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2369999999999998e-06 2024-08-17T17:20:45.301 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.301 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.301 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.301 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.655746+0000", 2024-08-17T17:20:45.301 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.62123385900000005 2024-08-17T17:20:45.301 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.301 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.301 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.301 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.655753+0000", 2024-08-17T17:20:45.301 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4960000000000001e-06 2024-08-17T17:20:45.301 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.301 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.302 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.302 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.702100+0000", 2024-08-17T17:20:45.302 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0463472949999999 2024-08-17T17:20:45.302 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.302 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.302 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.302 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.702256+0000", 2024-08-17T17:20:45.302 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000155857 2024-08-17T17:20:45.302 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.302 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.302 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:20:45.303 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.702420+0000", 2024-08-17T17:20:45.303 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016427799999999999 2024-08-17T17:20:45.303 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.303 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.303 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.303 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.709813+0000", 2024-08-17T17:20:45.303 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0073923210000000003 2024-08-17T17:20:45.303 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.303 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.303 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.303 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.711143+0000", 2024-08-17T17:20:45.303 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013302520000000001 2024-08-17T17:20:45.304 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.304 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.304 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.304 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.720255+0000", 2024-08-17T17:20:45.304 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0091118799999999993 2024-08-17T17:20:45.304 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.304 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.304 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.304 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.720292+0000", 2024-08-17T17:20:45.304 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.731e-05 2024-08-17T17:20:45.304 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.304 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.304 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.305 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.721505+0000", 2024-08-17T17:20:45.305 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012129370000000001 2024-08-17T17:20:45.305 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.305 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.305 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.305 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.305 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.305 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88400 7.9 7:900aeefa:::benchmark_data_smithi063_196002_object88399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1152)", 2024-08-17T17:20:45.305 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.699818+0000", 2024-08-17T17:20:45.305 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 103.481157783, 2024-08-17T17:20:45.305 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0249885809999999, 2024-08-17T17:20:45.305 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.306 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.306 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.306 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.306 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.306 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88400 2024-08-17T17:20:45.306 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.306 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.306 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.306 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.306 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699818+0000", 2024-08-17T17:20:45.306 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.306 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.306 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.307 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.307 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699818+0000", 2024-08-17T17:20:45.307 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.307 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.307 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.307 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.307 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699816+0000", 2024-08-17T17:20:45.307 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:20:45.307 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.307 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.307 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.307 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699911+0000", 2024-08-17T17:20:45.307 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4989999999999997e-05 2024-08-17T17:20:45.308 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.308 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.308 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.308 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699931+0000", 2024-08-17T17:20:45.308 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9667999999999999e-05 2024-08-17T17:20:45.308 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.308 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.308 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.308 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699939+0000", 2024-08-17T17:20:45.308 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.5399999999999996e-06 2024-08-17T17:20:45.308 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.308 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.309 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.309 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699986+0000", 2024-08-17T17:20:45.309 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6833999999999999e-05 2024-08-17T17:20:45.309 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.309 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.309 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.309 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699996+0000", 2024-08-17T17:20:45.309 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7270000000000002e-06 2024-08-17T17:20:45.309 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.309 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.309 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.309 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.703779+0000", 2024-08-17T17:20:45.309 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.003783026 2024-08-17T17:20:45.310 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.310 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.310 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.310 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.703883+0000", 2024-08-17T17:20:45.310 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000104282 2024-08-17T17:20:45.310 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.310 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.310 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:20:45.310 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.703973+0000", 2024-08-17T17:20:45.310 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9666999999999994e-05 2024-08-17T17:20:45.310 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.310 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.311 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.311 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.709818+0000", 2024-08-17T17:20:45.311 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0058452620000000004 2024-08-17T17:20:45.311 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.311 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.311 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.311 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.711179+0000", 2024-08-17T17:20:45.311 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001361077 2024-08-17T17:20:45.311 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.311 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.311 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.311 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.723783+0000", 2024-08-17T17:20:45.311 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012604148000000001 2024-08-17T17:20:45.312 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.312 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.312 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.312 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.723819+0000", 2024-08-17T17:20:45.312 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5394e-05 2024-08-17T17:20:45.312 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.312 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.312 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.312 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.724806+0000", 2024-08-17T17:20:45.312 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0009877009999999999 2024-08-17T17:20:45.312 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.312 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.312 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.313 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.313 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.313 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88408 7.9 7:92f3bf61:::benchmark_data_smithi063_196002_object88407:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1152)", 2024-08-17T17:20:45.313 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.763022+0000", 2024-08-17T17:20:45.313 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 103.41795367899999, 2024-08-17T17:20:45.313 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.962521195, 2024-08-17T17:20:45.313 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.313 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.313 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.313 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.313 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.313 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88408 2024-08-17T17:20:45.313 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.314 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.314 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.314 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.314 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.763022+0000", 2024-08-17T17:20:45.314 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.314 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.314 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.314 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.314 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.763022+0000", 2024-08-17T17:20:45.314 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.314 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.314 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.315 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.315 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.763020+0000", 2024-08-17T17:20:45.315 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:20:45.315 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.315 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.315 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.315 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.763061+0000", 2024-08-17T17:20:45.315 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1201000000000001e-05 2024-08-17T17:20:45.315 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.315 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.315 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.315 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.763076+0000", 2024-08-17T17:20:45.315 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4712000000000001e-05 2024-08-17T17:20:45.315 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.316 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.316 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.316 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.763086+0000", 2024-08-17T17:20:45.316 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2909999999999996e-06 2024-08-17T17:20:45.316 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.316 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.316 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.316 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.774363+0000", 2024-08-17T17:20:45.316 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011277852999999999 2024-08-17T17:20:45.316 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.316 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.316 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.316 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.774373+0000", 2024-08-17T17:20:45.317 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8430000000000005e-06 2024-08-17T17:20:45.317 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.317 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.317 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.317 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.706262+0000", 2024-08-17T17:20:45.317 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.93188885399999999 2024-08-17T17:20:45.317 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.317 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.317 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.317 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.706517+0000", 2024-08-17T17:20:45.317 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00025510600000000001 2024-08-17T17:20:45.317 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.317 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.318 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:20:45.318 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.706677+0000", 2024-08-17T17:20:45.318 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000159925 2024-08-17T17:20:45.318 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.319 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.319 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.319 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.711150+0000", 2024-08-17T17:20:45.319 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0044729679999999999 2024-08-17T17:20:45.319 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.320 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.320 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.320 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.712847+0000", 2024-08-17T17:20:45.320 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016967130000000001 2024-08-17T17:20:45.320 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.320 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.320 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.320 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.724849+0000", 2024-08-17T17:20:45.320 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.01200242 2024-08-17T17:20:45.320 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.320 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.320 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.320 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.724892+0000", 2024-08-17T17:20:45.321 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2956e-05 2024-08-17T17:20:45.321 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.321 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.321 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.321 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.725543+0000", 2024-08-17T17:20:45.321 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00065106599999999995 2024-08-17T17:20:45.321 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.321 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.321 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.321 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.321 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.321 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88450 7.9 7:966ebe59:::benchmark_data_smithi063_196002_object88449:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1152)", 2024-08-17T17:20:45.321 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.840584+0000", 2024-08-17T17:20:45.322 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 103.340391545, 2024-08-17T17:20:45.322 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.88908479399999996, 2024-08-17T17:20:45.322 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.322 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.322 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.322 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.322 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.322 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88450 2024-08-17T17:20:45.322 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.322 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.322 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.322 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.322 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840584+0000", 2024-08-17T17:20:45.323 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.323 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.323 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.323 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.323 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840584+0000", 2024-08-17T17:20:45.323 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.323 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.323 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.323 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.323 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840582+0000", 2024-08-17T17:20:45.323 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:20:45.323 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.323 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.324 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.324 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840637+0000", 2024-08-17T17:20:45.324 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4929000000000002e-05 2024-08-17T17:20:45.324 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.324 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.324 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.324 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840651+0000", 2024-08-17T17:20:45.324 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3851e-05 2024-08-17T17:20:45.324 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.324 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.324 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.324 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840656+0000", 2024-08-17T17:20:45.324 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2870000000000004e-06 2024-08-17T17:20:45.325 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.325 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.325 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.325 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840714+0000", 2024-08-17T17:20:45.325 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7904e-05 2024-08-17T17:20:45.325 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.325 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.325 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.325 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840726+0000", 2024-08-17T17:20:45.325 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2075999999999999e-05 2024-08-17T17:20:45.325 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.325 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.326 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.326 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.708243+0000", 2024-08-17T17:20:45.326 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.86751665700000002 2024-08-17T17:20:45.326 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.326 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.326 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.326 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.708434+0000", 2024-08-17T17:20:45.326 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00019137000000000001 2024-08-17T17:20:45.326 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.326 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.326 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:20:45.326 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.708553+0000", 2024-08-17T17:20:45.326 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000118205 2024-08-17T17:20:45.327 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.327 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.327 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.327 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.711189+0000", 2024-08-17T17:20:45.327 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00263649 2024-08-17T17:20:45.327 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.327 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.327 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.327 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.713490+0000", 2024-08-17T17:20:45.327 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023009509999999999 2024-08-17T17:20:45.327 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.327 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.327 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.327 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.728149+0000", 2024-08-17T17:20:45.328 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.014658775000000001 2024-08-17T17:20:45.328 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.328 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.328 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.328 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.728204+0000", 2024-08-17T17:20:45.328 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5546e-05 2024-08-17T17:20:45.328 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.328 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.328 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.328 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.729669+0000", 2024-08-17T17:20:45.328 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001464541 2024-08-17T17:20:45.328 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.329 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.329 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.329 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.329 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.329 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88451 7.9 7:95f9a7c9:::benchmark_data_smithi063_196002_object88450:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1152)", 2024-08-17T17:20:45.329 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.844136+0000", 2024-08-17T17:20:45.329 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 103.336839669, 2024-08-17T17:20:45.329 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.88693776499999999, 2024-08-17T17:20:45.329 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:45.329 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:20:45.329 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:20:45.329 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:20:45.329 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:20:45.330 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88451 2024-08-17T17:20:45.330 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.330 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:45.330 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.330 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:45.330 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844136+0000", 2024-08-17T17:20:45.330 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.330 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.330 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.330 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:45.330 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844136+0000", 2024-08-17T17:20:45.330 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:45.331 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.331 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.331 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:45.331 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844135+0000", 2024-08-17T17:20:45.331 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:20:45.331 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.331 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.331 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:45.331 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844251+0000", 2024-08-17T17:20:45.331 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011606400000000001 2024-08-17T17:20:45.331 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.331 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.331 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:45.332 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844264+0000", 2024-08-17T17:20:45.332 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2659e-05 2024-08-17T17:20:45.332 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.332 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.332 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:45.332 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844271+0000", 2024-08-17T17:20:45.332 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.861e-06 2024-08-17T17:20:45.332 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.332 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.332 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.332 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844318+0000", 2024-08-17T17:20:45.332 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7549999999999997e-05 2024-08-17T17:20:45.332 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.333 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.333 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:20:45.333 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844326+0000", 2024-08-17T17:20:45.333 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.8669999999999994e-06 2024-08-17T17:20:45.333 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.333 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.333 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:45.333 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.709851+0000", 2024-08-17T17:20:45.333 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.86552479400000004 2024-08-17T17:20:45.333 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.333 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.333 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:45.333 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.709949+0000", 2024-08-17T17:20:45.334 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8265999999999997e-05 2024-08-17T17:20:45.334 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.334 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.334 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:20:45.334 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.710047+0000", 2024-08-17T17:20:45.334 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8220000000000005e-05 2024-08-17T17:20:45.334 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.334 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.334 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:20:45.334 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.711235+0000", 2024-08-17T17:20:45.334 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001187926 2024-08-17T17:20:45.334 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.335 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.335 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.335 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.713526+0000", 2024-08-17T17:20:45.335 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002291104 2024-08-17T17:20:45.335 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.335 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.335 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:20:45.335 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.729734+0000", 2024-08-17T17:20:45.335 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.016208024000000001 2024-08-17T17:20:45.335 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.335 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.335 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:20:45.335 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.729798+0000", 2024-08-17T17:20:45.335 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3497999999999994e-05 2024-08-17T17:20:45.336 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:45.336 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:45.336 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:20:45.336 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.731074+0000", 2024-08-17T17:20:45.336 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012758509999999999 2024-08-17T17:20:45.336 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.336 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.336 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.336 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:45.336 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:45.336 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:20:45.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.337 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:45.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:45.438 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:45.464 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:45.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:45.566 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:45.594 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:45.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:45.696 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:45.723 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:45.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:45.825 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:45.864 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:45.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:45.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:45.966 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:46.012 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:46.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:45 smithi063 ceph-mon[109843]: pgmap v1429: 119 pgs: 10 unknown, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 11529/165432 objects degraded (6.969%); 577/165432 objects misplaced (0.349%); 14 MiB/s, 231 objects/s recovering 2024-08-17T17:20:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:45 smithi063 ceph-mon[109843]: osdmap e1165: 8 total, 8 up, 8 in 2024-08-17T17:20:46.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:45 smithi063 ceph-mon[114756]: pgmap v1429: 119 pgs: 10 unknown, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 11529/165432 objects degraded (6.969%); 577/165432 objects misplaced (0.349%); 14 MiB/s, 231 objects/s recovering 2024-08-17T17:20:46.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:45 smithi063 ceph-mon[114756]: osdmap e1165: 8 total, 8 up, 8 in 2024-08-17T17:20:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:45 smithi162 ceph-mon[104668]: pgmap v1429: 119 pgs: 10 unknown, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 11529/165432 objects degraded (6.969%); 577/165432 objects misplaced (0.349%); 14 MiB/s, 231 objects/s recovering 2024-08-17T17:20:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:45 smithi162 ceph-mon[104668]: osdmap e1165: 8 total, 8 up, 8 in 2024-08-17T17:20:46.114 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:46.159 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:46.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:46.241 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:20:46.261 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:46.314 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:46.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:46.416 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:46.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.447 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:46.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:46.528 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:46.548 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:46.576 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:46.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:46.678 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:46.724 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:46.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:46.826 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:46.881 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:46.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:46.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:46.964 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:46 smithi162 ceph-mon[104668]: 7.10 scrub starts 2024-08-17T17:20:46.964 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:46 smithi162 ceph-mon[104668]: 7.10 scrub ok 2024-08-17T17:20:46.964 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:46.964 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:46.965 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:46 smithi162 ceph-mon[104668]: 7.2 deep-scrub starts 2024-08-17T17:20:46.965 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:46 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1016932101' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:20:46.965 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:46.965 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:46.983 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:47.043 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:47.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:47.145 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:47.191 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:47.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[109843]: 7.10 scrub starts 2024-08-17T17:20:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[109843]: 7.10 scrub ok 2024-08-17T17:20:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[109843]: 7.2 deep-scrub starts 2024-08-17T17:20:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1016932101' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:20:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:47.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[114756]: 7.10 scrub starts 2024-08-17T17:20:47.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[114756]: 7.10 scrub ok 2024-08-17T17:20:47.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:47.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:20:47.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[114756]: 7.2 deep-scrub starts 2024-08-17T17:20:47.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1016932101' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:20:47.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:47.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:47.293 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:47.320 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:47.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:47.353 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:20:47.422 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:47.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.467 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:47.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:47.484 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:20:47.568 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:47.593 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:47.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:47.682 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:20:47.682 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:20:47.682 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [1, 2, 3, 4, 5, 6, 7, 0] 2024-08-17T17:20:47.682 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd out 1 2024-08-17T17:20:47.695 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:47.719 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:47.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:47.821 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:47.844 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:47.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:47.945 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:47.992 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:47.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:47.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[109843]: pgmap v1431: 119 pgs: 1 active+clean+scrubbing+deep, 6 unknown, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 11523/163569 objects degraded (7.045%); 577/163569 objects misplaced (0.353%); 17 MiB/s, 277 objects/s recovering 2024-08-17T17:20:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[109843]: 7.0 scrub starts 2024-08-17T17:20:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[109843]: 7.0 scrub ok 2024-08-17T17:20:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1016932101' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:20:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[109843]: osdmap e1166: 8 total, 8 up, 8 in 2024-08-17T17:20:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:20:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:20:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[109843]: 7.22 scrub starts 2024-08-17T17:20:48.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[114756]: pgmap v1431: 119 pgs: 1 active+clean+scrubbing+deep, 6 unknown, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 11523/163569 objects degraded (7.045%); 577/163569 objects misplaced (0.353%); 17 MiB/s, 277 objects/s recovering 2024-08-17T17:20:48.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[114756]: 7.0 scrub starts 2024-08-17T17:20:48.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[114756]: 7.0 scrub ok 2024-08-17T17:20:48.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:48.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1016932101' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:20:48.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[114756]: osdmap e1166: 8 total, 8 up, 8 in 2024-08-17T17:20:48.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:20:48.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:20:48.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:48.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:47 smithi063 ceph-mon[114756]: 7.22 scrub starts 2024-08-17T17:20:48.094 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:48.121 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:48.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:48.169 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:47 smithi162 ceph-mon[104668]: pgmap v1431: 119 pgs: 1 active+clean+scrubbing+deep, 6 unknown, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 698 op/s; 11523/163569 objects degraded (7.045%); 577/163569 objects misplaced (0.353%); 17 MiB/s, 277 objects/s recovering 2024-08-17T17:20:48.170 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:47 smithi162 ceph-mon[104668]: 7.0 scrub starts 2024-08-17T17:20:48.170 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:47 smithi162 ceph-mon[104668]: 7.0 scrub ok 2024-08-17T17:20:48.170 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:47 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:20:48.170 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:47 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1016932101' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:20:48.170 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:47 smithi162 ceph-mon[104668]: osdmap e1166: 8 total, 8 up, 8 in 2024-08-17T17:20:48.170 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:20:48.170 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:20:48.170 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:20:48.170 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:47 smithi162 ceph-mon[104668]: 7.22 scrub starts 2024-08-17T17:20:48.223 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:48.259 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:48.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:48.361 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:48.418 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:20:48.418 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:20:48.418 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.419 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(osd.3.0:99209 7.a e1166/1163 7:541071a4:::benchmark_data_smithi063_196002_object87650:head v 1166'11494, mlcod=1166'11472)", 2024-08-17T17:20:48.419 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:20:48.416830+0000", 2024-08-17T17:20:48.419 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.000389925, 2024-08-17T17:20:48.419 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00041677700000000001, 2024-08-17T17:20:48.419 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:48.419 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:20:48.419 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:48.419 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.419 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:48.419 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:48.416830+0000", 2024-08-17T17:20:48.419 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:48.419 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:48.419 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.419 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:48.420 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:48.416830+0000", 2024-08-17T17:20:48.420 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:48.420 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:48.420 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.420 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:48.420 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:48.416829+0000", 2024-08-17T17:20:48.420 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:20:48.420 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:48.420 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.420 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:48.420 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:48.416851+0000", 2024-08-17T17:20:48.420 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2084999999999999e-05 2024-08-17T17:20:48.420 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:48.420 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.421 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:48.421 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:48.416869+0000", 2024-08-17T17:20:48.421 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.8085e-05 2024-08-17T17:20:48.421 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:48.421 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.421 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:48.421 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:48.416874+0000", 2024-08-17T17:20:48.421 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9030000000000003e-06 2024-08-17T17:20:48.421 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:48.421 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.421 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:20:48.421 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:48.416919+0000", 2024-08-17T17:20:48.421 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4991999999999999e-05 2024-08-17T17:20:48.421 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:48.422 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.422 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:20:48.422 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:48.416933+0000", 2024-08-17T17:20:48.422 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4182000000000001e-05 2024-08-17T17:20:48.422 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:48.422 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:48.422 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:48.422 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:48.422 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.422 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(osd.3.0:99210 7.a e1166/1163)", 2024-08-17T17:20:48.422 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:20:48.416883+0000", 2024-08-17T17:20:48.422 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.00033641500000000001, 2024-08-17T17:20:48.422 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000454618, 2024-08-17T17:20:48.423 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:20:48.423 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "queued for pg", 2024-08-17T17:20:48.423 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:20:48.423 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.423 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:20:48.423 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:48.416883+0000", 2024-08-17T17:20:48.423 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:48.423 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:48.423 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.423 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:20:48.423 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:48.416883+0000", 2024-08-17T17:20:48.423 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:20:48.423 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:48.423 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.424 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:20:48.424 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:48.416883+0000", 2024-08-17T17:20:48.424 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:20:48.424 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:48.424 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.424 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:20:48.424 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:48.416886+0000", 2024-08-17T17:20:48.424 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6909999999999999e-06 2024-08-17T17:20:48.424 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:48.424 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.424 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:20:48.424 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:48.416892+0000", 2024-08-17T17:20:48.424 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9519999999999999e-06 2024-08-17T17:20:48.424 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:20:48.425 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:20:48.425 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:20:48.425 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:48.416897+0000", 2024-08-17T17:20:48.425 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1400000000000002e-06 2024-08-17T17:20:48.425 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:48.425 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:20:48.425 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:48.425 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:20:48.425 INFO:teuthology.orchestra.run.smithi162.stdout: ], 2024-08-17T17:20:48.425 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 2 2024-08-17T17:20:48.425 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:20:48.426 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:48.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.426 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:48.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:48.455 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:48.527 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:48.567 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:48.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:48.669 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:48.712 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:48.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:48.814 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:48.854 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:48.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:48.956 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:48.997 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:48.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:48.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:48 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 11523/163569 objects degraded (7.045%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:48 smithi162 ceph-mon[104668]: pgmap v1433: 119 pgs: 1 active+clean+scrubbing+deep, 12 active+recovery_wait+degraded, 1 active+recovering, 105 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 10405/158193 objects degraded (6.577%); 50/158193 objects misplaced (0.032%); 19 MiB/s, 304 objects/s recovering 2024-08-17T17:20:49.099 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:49.137 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:49.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:49.234 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:20:49.239 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:49.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:48 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 11523/163569 objects degraded (7.045%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:49.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:48 smithi063 ceph-mon[109843]: pgmap v1433: 119 pgs: 1 active+clean+scrubbing+deep, 12 active+recovery_wait+degraded, 1 active+recovering, 105 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 10405/158193 objects degraded (6.577%); 50/158193 objects misplaced (0.032%); 19 MiB/s, 304 objects/s recovering 2024-08-17T17:20:49.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:48 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 11523/163569 objects degraded (7.045%), 14 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:20:49.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:48 smithi063 ceph-mon[114756]: pgmap v1433: 119 pgs: 1 active+clean+scrubbing+deep, 12 active+recovery_wait+degraded, 1 active+recovering, 105 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 10405/158193 objects degraded (6.577%); 50/158193 objects misplaced (0.032%); 19 MiB/s, 304 objects/s recovering 2024-08-17T17:20:49.306 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:49.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:49.408 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:49.442 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:49.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:49.502 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:49.544 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:49.567 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:49.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:49.669 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:49.695 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:49.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:49.797 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:49.818 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:49.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:49.919 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:49.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:49.961 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:49.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:50.062 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:50.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.101 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:50.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:50.202 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:50.227 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:20:50.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.230 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:50.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:50.332 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:50.376 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:50.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:50.477 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:50.504 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:50.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:50.605 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:50.639 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:50.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:50.741 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:50.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.787 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:50.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:50.888 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:50.935 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:50.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:50.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:51.037 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:51.066 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:51.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:51.168 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:51.197 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:51.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:51.298 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:51.330 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:20:51.330 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:20:51.330 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:20:51.330 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:20:51.330 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:20:51.331 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:51 smithi162 ceph-mon[104668]: pgmap v1434: 119 pgs: 2 active+clean+scrubbing+deep, 12 active+recovery_wait+degraded, 1 active+recovering, 104 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 10159/155247 objects degraded (6.544%); 50/155247 objects misplaced (0.032%); 18 MiB/s, 296 objects/s recovering 2024-08-17T17:20:51.332 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:51.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:51.434 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:51.459 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:51.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:51.547 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:51 smithi063 ceph-mon[114756]: pgmap v1434: 119 pgs: 2 active+clean+scrubbing+deep, 12 active+recovery_wait+degraded, 1 active+recovering, 104 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 10159/155247 objects degraded (6.544%); 50/155247 objects misplaced (0.032%); 18 MiB/s, 296 objects/s recovering 2024-08-17T17:20:51.547 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:51 smithi063 ceph-mon[109843]: pgmap v1434: 119 pgs: 2 active+clean+scrubbing+deep, 12 active+recovery_wait+degraded, 1 active+recovering, 104 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 10159/155247 objects degraded (6.544%); 50/155247 objects misplaced (0.032%); 18 MiB/s, 296 objects/s recovering 2024-08-17T17:20:51.561 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:51.585 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:51.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:51.687 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:51.714 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:51.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:51.816 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:51.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:51.877 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:51.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:51.978 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:51.990 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:20:52.016 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:52.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:52.118 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:52.171 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:52.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:52.249 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:52.273 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:52.313 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:52.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:52.415 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:52.440 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:52.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:52.541 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:52.581 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:52.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:52.683 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:52.718 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:52.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:52.819 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:52.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.846 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:52.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:52.947 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:52.976 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:52.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:52.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:53.078 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:53.101 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:53.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:53.203 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:53.231 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:53.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:53.333 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:53.367 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:53.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:53.469 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:53.492 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:53.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:53.520 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:53 smithi162 ceph-mon[104668]: pgmap v1435: 119 pgs: 2 active+clean+scrubbing+deep, 11 active+recovery_wait+degraded, 1 active+recovering, 105 active+clean; 3.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 8859/149682 objects degraded (5.919%); 22 MiB/s, 378 objects/s recovering 2024-08-17T17:20:53.520 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:53 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 8859/149682 objects degraded (5.919%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:20:53.594 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:53.597 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:20:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:20:53] "GET /metrics HTTP/1.1" 200 36436 "" "Prometheus/2.51.0" 2024-08-17T17:20:53.598 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:53 smithi063 ceph-mon[114756]: pgmap v1435: 119 pgs: 2 active+clean+scrubbing+deep, 11 active+recovery_wait+degraded, 1 active+recovering, 105 active+clean; 3.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 8859/149682 objects degraded (5.919%); 22 MiB/s, 378 objects/s recovering 2024-08-17T17:20:53.598 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:53 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 8859/149682 objects degraded (5.919%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:20:53.598 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:53 smithi063 ceph-mon[109843]: pgmap v1435: 119 pgs: 2 active+clean+scrubbing+deep, 11 active+recovery_wait+degraded, 1 active+recovering, 105 active+clean; 3.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 8859/149682 objects degraded (5.919%); 22 MiB/s, 378 objects/s recovering 2024-08-17T17:20:53.598 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:53 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 8859/149682 objects degraded (5.919%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:20:53.640 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:53.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:53.742 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:53.766 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:53.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:53.868 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:53.893 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:53.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:53.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:53.994 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:54.030 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:54.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:54.088 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:20:54.089 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:20:54.132 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:54.159 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:54.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:54.261 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:54.271 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:20:54.271 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:20:54.272 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:20:54.272 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:20:54.272 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:20:54.272 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:20:54.272 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:20:54.272 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:20:54.272 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:20:54.272 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:20:54.272 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:20:54.272 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:20:54.273 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:20:54.273 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:20:54.273 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:20:54.273 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:20:54.286 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:54.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:54.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:54 smithi162 ceph-mon[104668]: 7.2 deep-scrub ok 2024-08-17T17:20:54.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:54 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1530517569' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:54.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:54 smithi162 ceph-mon[104668]: 7.22 scrub starts 2024-08-17T17:20:54.388 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:54.413 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:54.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:54.517 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:54.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:54 smithi063 ceph-mon[114756]: 7.2 deep-scrub ok 2024-08-17T17:20:54.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:54 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1530517569' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:54.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:54 smithi063 ceph-mon[114756]: 7.22 scrub starts 2024-08-17T17:20:54.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:54 smithi063 ceph-mon[109843]: 7.2 deep-scrub ok 2024-08-17T17:20:54.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:54 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1530517569' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:20:54.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:54 smithi063 ceph-mon[109843]: 7.22 scrub starts 2024-08-17T17:20:54.542 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:54.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:54.644 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:54.668 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:54.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:54.770 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:54.797 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:54.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:54.811 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:20:54.844 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:54.899 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:54.928 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:54.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:54.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:55.030 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:55.082 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:55.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:55.116 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:55.184 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:55.207 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:55.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:55.309 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:55.310 INFO:teuthology.orchestra.run.smithi063.stderr:marked out osd.1. 2024-08-17T17:20:55.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.343 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:55.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:55.444 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:55.464 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:55.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:55.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:55.592 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:55.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:55.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:55 smithi162 ceph-mon[104668]: 7.1e scrub starts 2024-08-17T17:20:55.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:55 smithi162 ceph-mon[104668]: 7.a deep-scrub ok 2024-08-17T17:20:55.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:55 smithi162 ceph-mon[104668]: pgmap v1436: 119 pgs: 2 active+clean+scrubbing+deep, 11 active+recovery_wait+degraded, 1 active+recovering, 105 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 8556/147111 objects degraded (5.816%); 21 MiB/s, 354 objects/s recovering 2024-08-17T17:20:55.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:55 smithi162 ceph-mon[104668]: 7.1a scrub starts 2024-08-17T17:20:55.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:55 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2054579988' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T17:20:55.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:55 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T17:20:55.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:55 smithi162 ceph-mon[104668]: Client client.admin marked osd.1 out, while it was still marked up 2024-08-17T17:20:55.693 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:55.715 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:55.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:55.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:55 smithi063 ceph-mon[114756]: 7.1e scrub starts 2024-08-17T17:20:55.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:55 smithi063 ceph-mon[114756]: 7.a deep-scrub ok 2024-08-17T17:20:55.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:55 smithi063 ceph-mon[114756]: pgmap v1436: 119 pgs: 2 active+clean+scrubbing+deep, 11 active+recovery_wait+degraded, 1 active+recovering, 105 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 8556/147111 objects degraded (5.816%); 21 MiB/s, 354 objects/s recovering 2024-08-17T17:20:55.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:55 smithi063 ceph-mon[114756]: 7.1a scrub starts 2024-08-17T17:20:55.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:55 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2054579988' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T17:20:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:55 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T17:20:55.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:55 smithi063 ceph-mon[114756]: Client client.admin marked osd.1 out, while it was still marked up 2024-08-17T17:20:55.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:55 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a[109820]: 2024-08-17T17:20:55.283+0000 7f66fd20a640 -1 mon.a@0(leader).osd e1166 definitely_dead 0 2024-08-17T17:20:55.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:55 smithi063 ceph-mon[109843]: 7.1e scrub starts 2024-08-17T17:20:55.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:55 smithi063 ceph-mon[109843]: 7.a deep-scrub ok 2024-08-17T17:20:55.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:55 smithi063 ceph-mon[109843]: pgmap v1436: 119 pgs: 2 active+clean+scrubbing+deep, 11 active+recovery_wait+degraded, 1 active+recovering, 105 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 8556/147111 objects degraded (5.816%); 21 MiB/s, 354 objects/s recovering 2024-08-17T17:20:55.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:55 smithi063 ceph-mon[109843]: 7.1a scrub starts 2024-08-17T17:20:55.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:55 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2054579988' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T17:20:55.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:55 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T17:20:55.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:55 smithi063 ceph-mon[109843]: Client client.admin marked osd.1 out, while it was still marked up 2024-08-17T17:20:55.817 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:55.842 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:55.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:55.943 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:55.967 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:55.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:55.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:56.069 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:56.093 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:56.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:56.195 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:56.219 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:56.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:56.321 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:56.362 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:56.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:56.456 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:56 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-08-17T17:20:56.456 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:56 smithi063 ceph-mon[109843]: osdmap e1167: 8 total, 8 up, 7 in 2024-08-17T17:20:56.456 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:56 smithi063 ceph-mon[109843]: 7.23 scrub starts 2024-08-17T17:20:56.456 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:56 smithi063 ceph-mon[109843]: osdmap e1168: 8 total, 8 up, 7 in 2024-08-17T17:20:56.457 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:56 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-08-17T17:20:56.457 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:56 smithi063 ceph-mon[114756]: osdmap e1167: 8 total, 8 up, 7 in 2024-08-17T17:20:56.457 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:56 smithi063 ceph-mon[114756]: 7.23 scrub starts 2024-08-17T17:20:56.457 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:56 smithi063 ceph-mon[114756]: osdmap e1168: 8 total, 8 up, 7 in 2024-08-17T17:20:56.463 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:56.486 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:56.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:56.588 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:56.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:56 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-08-17T17:20:56.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:56 smithi162 ceph-mon[104668]: osdmap e1167: 8 total, 8 up, 7 in 2024-08-17T17:20:56.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:56 smithi162 ceph-mon[104668]: 7.23 scrub starts 2024-08-17T17:20:56.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:56 smithi162 ceph-mon[104668]: osdmap e1168: 8 total, 8 up, 7 in 2024-08-17T17:20:56.611 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:56.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:56.712 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:56.737 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:56.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:56.839 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:56.863 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:56.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:56.964 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:56.988 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:56.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:56.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:57.089 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:57.112 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:57.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:57.214 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:57.238 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:57.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:57.340 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:57.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.387 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:57.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:57.415 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:20:57.432 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:20:57.433 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:20:57.433 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:57 smithi162 ceph-mon[104668]: pgmap v1438: 119 pgs: 10 active+recovery_wait+degraded, 2 active+recovering, 107 active+clean; 2.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 6487/143907 objects degraded (4.508%); 12 MiB/s, 219 objects/s recovering 2024-08-17T17:20:57.433 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:57 smithi162 ceph-mon[104668]: 7.13 scrub starts 2024-08-17T17:20:57.433 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:57 smithi162 ceph-mon[104668]: osdmap e1169: 8 total, 8 up, 7 in 2024-08-17T17:20:57.488 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:57.515 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:57.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:57.617 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:57.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.657 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:57.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:57.737 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:57 smithi063 ceph-mon[114756]: pgmap v1438: 119 pgs: 10 active+recovery_wait+degraded, 2 active+recovering, 107 active+clean; 2.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 6487/143907 objects degraded (4.508%); 12 MiB/s, 219 objects/s recovering 2024-08-17T17:20:57.737 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:57 smithi063 ceph-mon[114756]: 7.13 scrub starts 2024-08-17T17:20:57.737 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:57 smithi063 ceph-mon[114756]: osdmap e1169: 8 total, 8 up, 7 in 2024-08-17T17:20:57.737 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:57 smithi063 ceph-mon[109843]: pgmap v1438: 119 pgs: 10 active+recovery_wait+degraded, 2 active+recovering, 107 active+clean; 2.9 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 6487/143907 objects degraded (4.508%); 12 MiB/s, 219 objects/s recovering 2024-08-17T17:20:57.737 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:57 smithi063 ceph-mon[109843]: 7.13 scrub starts 2024-08-17T17:20:57.737 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:57 smithi063 ceph-mon[109843]: osdmap e1169: 8 total, 8 up, 7 in 2024-08-17T17:20:57.759 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:57.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.789 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:57.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:57.840 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:20:57.890 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:20:57.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:57.938 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:57.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:20:58.039 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:20:58.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.077 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:58.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:20:58.178 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:58.210 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:58.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:58.312 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:20:58.347 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:58 smithi063 ceph-mon[114756]: 7.12 starting backfill to osd.7 from (0'0,0'0] MAX to 1166'8413 2024-08-17T17:20:58.347 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:58 smithi063 ceph-mon[114756]: 7.b starting backfill to osd.2 from (0'0,0'0] MAX to 1166'10906 2024-08-17T17:20:58.348 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:58 smithi063 ceph-mon[114756]: 7.1b starting backfill to osd.2 from (0'0,0'0] MAX to 1166'10875 2024-08-17T17:20:58.348 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:58 smithi063 ceph-mon[114756]: 7.3 scrub starts 2024-08-17T17:20:58.348 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:58 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 6487/143907 objects degraded (4.508%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:20:58.348 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:58 smithi063 ceph-mon[109843]: 7.12 starting backfill to osd.7 from (0'0,0'0] MAX to 1166'8413 2024-08-17T17:20:58.348 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:58 smithi063 ceph-mon[109843]: 7.b starting backfill to osd.2 from (0'0,0'0] MAX to 1166'10906 2024-08-17T17:20:58.348 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:58 smithi063 ceph-mon[109843]: 7.1b starting backfill to osd.2 from (0'0,0'0] MAX to 1166'10875 2024-08-17T17:20:58.348 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:58 smithi063 ceph-mon[109843]: 7.3 scrub starts 2024-08-17T17:20:58.348 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:58 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 6487/143907 objects degraded (4.508%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:20:58.348 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:58 smithi063 ceph-mon[109843]: osdmap e1170: 8 total, 8 up, 7 in 2024-08-17T17:20:58.358 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:58.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:20:58.461 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:58.491 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:58.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:58.593 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:58.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:58 smithi162 ceph-mon[104668]: 7.12 starting backfill to osd.7 from (0'0,0'0] MAX to 1166'8413 2024-08-17T17:20:58.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:58 smithi162 ceph-mon[104668]: 7.b starting backfill to osd.2 from (0'0,0'0] MAX to 1166'10906 2024-08-17T17:20:58.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:58 smithi162 ceph-mon[104668]: 7.1b starting backfill to osd.2 from (0'0,0'0] MAX to 1166'10875 2024-08-17T17:20:58.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:58 smithi162 ceph-mon[104668]: 7.3 scrub starts 2024-08-17T17:20:58.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:58 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 6487/143907 objects degraded (4.508%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:20:58.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:58 smithi162 ceph-mon[104668]: osdmap e1170: 8 total, 8 up, 7 in 2024-08-17T17:20:58.623 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:58.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:58.683 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:58 smithi063 ceph-mon[114756]: osdmap e1170: 8 total, 8 up, 7 in 2024-08-17T17:20:58.724 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:58.754 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:58.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:58.856 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:58.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:58.906 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:58.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:58.953 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:20:59.007 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:59.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.047 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:59.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:59.148 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:20:59.187 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:59.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:20:59.289 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:59.314 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:20:59.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.340 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:59.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:59.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:59 smithi162 ceph-mon[104668]: pgmap v1442: 119 pgs: 1 active+recovery_wait, 12 remapped+peering, 1 active+recovery_wait+degraded, 2 active+recovering, 103 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 1833/137895 objects degraded (1.329%); 13 MiB/s, 279 objects/s recovering 2024-08-17T17:20:59.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:59 smithi162 ceph-mon[104668]: osdmap e1171: 8 total, 8 up, 7 in 2024-08-17T17:20:59.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:59 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/697480154' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:59.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:20:59 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:59.441 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:20:59.470 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:59.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:20:59.571 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:20:59.608 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:20:59.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:20:59.706 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:59 smithi063 ceph-mon[109843]: pgmap v1442: 119 pgs: 1 active+recovery_wait, 12 remapped+peering, 1 active+recovery_wait+degraded, 2 active+recovering, 103 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 1833/137895 objects degraded (1.329%); 13 MiB/s, 279 objects/s recovering 2024-08-17T17:20:59.706 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:59 smithi063 ceph-mon[109843]: osdmap e1171: 8 total, 8 up, 7 in 2024-08-17T17:20:59.706 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:59 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/697480154' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:59.707 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:20:59 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:59.707 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:59 smithi063 ceph-mon[114756]: pgmap v1442: 119 pgs: 1 active+recovery_wait, 12 remapped+peering, 1 active+recovery_wait+degraded, 2 active+recovering, 103 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 1833/137895 objects degraded (1.329%); 13 MiB/s, 279 objects/s recovering 2024-08-17T17:20:59.707 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:59 smithi063 ceph-mon[114756]: osdmap e1171: 8 total, 8 up, 7 in 2024-08-17T17:20:59.707 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:59 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/697480154' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:59.707 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:20:59 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:20:59.711 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:20:59.747 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:59.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:20:59.849 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:20:59.881 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:20:59.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:20:59.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:20:59.909 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:20:59.983 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:00.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.021 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:00.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:00.122 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:00.157 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:00.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:00.259 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:00.297 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:00.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:00.398 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:00.436 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:00.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:00.538 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:00.568 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:00.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:00.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:00 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:00.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:00 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:00.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:00 smithi162 ceph-mon[104668]: osdmap e1172: 8 total, 8 up, 7 in 2024-08-17T17:21:00.626 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:00 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:00.626 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:00 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:00.626 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:00 smithi063 ceph-mon[114756]: osdmap e1172: 8 total, 8 up, 7 in 2024-08-17T17:21:00.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:00 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:00.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:00 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:00.626 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:00 smithi063 ceph-mon[109843]: osdmap e1172: 8 total, 8 up, 7 in 2024-08-17T17:21:00.671 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:00.710 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:00.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:00.812 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:00.851 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:00.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:00.953 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:00.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:00.991 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:00.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:01.091 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:01.128 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:01.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:01.153 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0] out_osds: [1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:21:01.153 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:21:01.154 INFO:tasks.thrashosds.thrasher:Adding osd 1 2024-08-17T17:21:01.154 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd in 1 2024-08-17T17:21:01.229 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:01.280 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:01.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:01.381 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:01.413 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:01 smithi063 ceph-mon[114756]: pgmap v1445: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 12 remapped+peering, 2 active+recovering, 100 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 4701/135624 objects degraded (3.466%); 1363/135624 objects misplaced (1.005%); 9.5 MiB/s, 446 objects/s recovering 2024-08-17T17:21:01.414 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:01 smithi063 ceph-mon[109843]: pgmap v1445: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 12 remapped+peering, 2 active+recovering, 100 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 4701/135624 objects degraded (3.466%); 1363/135624 objects misplaced (1.005%); 9.5 MiB/s, 446 objects/s recovering 2024-08-17T17:21:01.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.434 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:01.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:01.499 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:21:01.500 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:21:01.535 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:01.540 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:21:01.541 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:21:01.559 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:01.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.573 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:01.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:01.578 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:21:01.578 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:21:01.613 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:21:01.613 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:21:01.658 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:01 smithi162 ceph-mon[104668]: pgmap v1445: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 12 remapped+peering, 2 active+recovering, 100 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 4701/135624 objects degraded (3.466%); 1363/135624 objects misplaced (1.005%); 9.5 MiB/s, 446 objects/s recovering 2024-08-17T17:21:01.674 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:01.684 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:21:01.684 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:21:01.708 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:01.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:01.718 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:21:01.718 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:21:01.756 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:21:01.757 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:21:01.788 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:21:01.788 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:21:01.810 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:01.836 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:01.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:01.937 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:01.970 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:01.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:01.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:02.072 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:02.113 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:02.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:02.214 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:02.250 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:02.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:02.351 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:02.395 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:02.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:02.496 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:02.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.520 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:02.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:02.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:02 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:21:02.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:02 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:02.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:02 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/140033742' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:02.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:02 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:21:02.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:02 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:02.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:02 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/140033742' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:02.621 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:02.662 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:02.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:02.688 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:21:02.688 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:21:02.688 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:02 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:21:02.688 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:02 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:02.688 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:02 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/140033742' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:02.764 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:02.798 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:02.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:02.900 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:02.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:02.936 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:02.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:03.037 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:03.067 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:03.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:03.168 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:03.178 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:21:03.201 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:03.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:03.303 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:03.340 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:03.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:03.392 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:21:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:21:03] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.51.0" 2024-08-17T17:21:03.441 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:03.481 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:03.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:03.583 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:03.615 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:03.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:03.654 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:03 smithi063 ceph-mon[114756]: pgmap v1446: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 5 remapped+peering, 1 active+recovering, 102 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 649 op/s; 29390/128304 objects degraded (22.907%); 1363/128304 objects misplaced (1.062%); 0 B/s, 270 objects/s recovering 2024-08-17T17:21:03.654 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:03 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 29390/128304 objects degraded (22.907%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:03.654 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:03 smithi063 ceph-mon[109843]: pgmap v1446: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 5 remapped+peering, 1 active+recovering, 102 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 649 op/s; 29390/128304 objects degraded (22.907%); 1363/128304 objects misplaced (1.062%); 0 B/s, 270 objects/s recovering 2024-08-17T17:21:03.654 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:03 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 29390/128304 objects degraded (22.907%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:03.718 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:03.745 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:03.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:03.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:03 smithi162 ceph-mon[104668]: pgmap v1446: 119 pgs: 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 5 remapped+peering, 1 active+recovering, 102 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 649 op/s; 29390/128304 objects degraded (22.907%); 1363/128304 objects misplaced (1.062%); 0 B/s, 270 objects/s recovering 2024-08-17T17:21:03.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:03 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 29390/128304 objects degraded (22.907%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:03.847 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:03.891 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:03.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:03.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:03.993 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:04.014 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:04.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:04.116 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:04.151 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:04.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:04.192 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:04.253 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:04.285 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:04.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:04.387 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:04.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.428 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:04.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:04.529 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:04.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.561 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:04.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:04.662 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:04.713 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:04.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:04.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:04 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a[109820]: 2024-08-17T17:21:04.565+0000 7f66fd20a640 -1 mon.a@0(leader).osd e1172 definitely_dead 0 2024-08-17T17:21:04.815 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:04.837 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:04.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:04.938 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:04.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:04.982 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:04.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:05.083 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:05.107 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:05.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:05.209 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:05.246 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:05.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:05.348 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:05.386 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:05.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:05.422 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:05 smithi063 ceph-mon[114756]: pgmap v1447: 119 pgs: 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 1 active+recovering, 102 active+clean; 2.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 34686/124491 objects degraded (27.862%); 4360/124491 objects misplaced (3.502%); 3.1 MiB/s, 260 objects/s recovering 2024-08-17T17:21:05.422 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:05 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1947783224' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T17:21:05.422 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:05 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T17:21:05.422 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:05 smithi063 ceph-mon[109843]: pgmap v1447: 119 pgs: 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 1 active+recovering, 102 active+clean; 2.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 34686/124491 objects degraded (27.862%); 4360/124491 objects misplaced (3.502%); 3.1 MiB/s, 260 objects/s recovering 2024-08-17T17:21:05.422 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:05 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1947783224' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T17:21:05.422 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:05 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T17:21:05.427 INFO:teuthology.orchestra.run.smithi063.stderr:marked in osd.1. 2024-08-17T17:21:05.488 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:05.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.512 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:05.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:05.613 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:05.635 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:05.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:05.656 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:21:05.737 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:05.758 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:05.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:05.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:05 smithi162 ceph-mon[104668]: pgmap v1447: 119 pgs: 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 1 active+recovering, 102 active+clean; 2.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 704 op/s; 34686/124491 objects degraded (27.862%); 4360/124491 objects misplaced (3.502%); 3.1 MiB/s, 260 objects/s recovering 2024-08-17T17:21:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:05 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1947783224' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T17:21:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:05 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T17:21:05.860 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:05.882 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:05.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:05.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:05.983 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:06.008 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:06.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:06.110 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:06.133 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:06.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:06.235 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:06.258 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:06.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:06.360 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:06.386 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:06.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:06.488 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:06.512 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:06.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:06.614 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:06.638 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:06.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:06.740 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:06.761 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:06.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:06.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:06 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-08-17T17:21:06.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:06 smithi063 ceph-mon[114756]: osdmap e1173: 8 total, 8 up, 8 in 2024-08-17T17:21:06.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:06 smithi063 ceph-mon[114756]: 7.1a scrub starts 2024-08-17T17:21:06.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:06 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-08-17T17:21:06.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:06 smithi063 ceph-mon[109843]: osdmap e1173: 8 total, 8 up, 8 in 2024-08-17T17:21:06.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:06 smithi063 ceph-mon[109843]: 7.1a scrub starts 2024-08-17T17:21:06.788 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:06 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-08-17T17:21:06.788 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:06 smithi162 ceph-mon[104668]: osdmap e1173: 8 total, 8 up, 8 in 2024-08-17T17:21:06.788 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:06 smithi162 ceph-mon[104668]: 7.1a scrub starts 2024-08-17T17:21:06.863 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:06.884 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:06.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:06.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:06.986 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:07.007 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:07.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:07.109 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:07.130 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:07.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:07.231 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:07.252 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:07.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:07.354 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:07.375 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:07.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:07.476 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:07.518 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:07.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:07.620 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:07.647 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:07.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:07.749 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:07.773 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:07.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:07.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:07 smithi063 ceph-mon[109843]: pgmap v1449: 119 pgs: 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 1 active+recovering, 102 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 779 op/s; 34686/120759 objects degraded (28.723%); 4229/120759 objects misplaced (3.502%); 2.7 MiB/s, 296 objects/s recovering 2024-08-17T17:21:07.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:07 smithi063 ceph-mon[109843]: osdmap e1174: 8 total, 8 up, 8 in 2024-08-17T17:21:07.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:07 smithi063 ceph-mon[109843]: 7.23 scrub starts 2024-08-17T17:21:07.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:07 smithi063 ceph-mon[114756]: pgmap v1449: 119 pgs: 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 1 active+recovering, 102 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 779 op/s; 34686/120759 objects degraded (28.723%); 4229/120759 objects misplaced (3.502%); 2.7 MiB/s, 296 objects/s recovering 2024-08-17T17:21:07.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:07 smithi063 ceph-mon[114756]: osdmap e1174: 8 total, 8 up, 8 in 2024-08-17T17:21:07.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:07 smithi063 ceph-mon[114756]: 7.23 scrub starts 2024-08-17T17:21:07.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:07 smithi162 ceph-mon[104668]: pgmap v1449: 119 pgs: 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 1 active+recovering, 102 active+clean; 2.5 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 779 op/s; 34686/120759 objects degraded (28.723%); 4229/120759 objects misplaced (3.502%); 2.7 MiB/s, 296 objects/s recovering 2024-08-17T17:21:07.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:07 smithi162 ceph-mon[104668]: osdmap e1174: 8 total, 8 up, 8 in 2024-08-17T17:21:07.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:07 smithi162 ceph-mon[104668]: 7.23 scrub starts 2024-08-17T17:21:07.875 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:07.897 INFO:tasks.thrashosds.thrasher:Added osd 1 2024-08-17T17:21:07.900 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:07.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:07.900 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:08.001 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:08.029 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:08.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:08.130 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:08.158 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:08.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:08.259 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:08.286 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:08.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:08.388 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:08.415 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:08.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:08.457 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:08.517 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:08.543 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:08.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:08.610 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:21:08.645 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:08.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.691 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:08.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:08.770 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:08 smithi063 ceph-mon[114756]: osdmap e1175: 8 total, 8 up, 8 in 2024-08-17T17:21:08.770 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:08 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 34686/120759 objects degraded (28.723%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:08.770 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:08 smithi063 ceph-mon[114756]: osdmap e1176: 8 total, 8 up, 8 in 2024-08-17T17:21:08.770 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:08 smithi063 ceph-mon[109843]: osdmap e1175: 8 total, 8 up, 8 in 2024-08-17T17:21:08.770 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:08 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 34686/120759 objects degraded (28.723%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:08.770 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:08 smithi063 ceph-mon[109843]: osdmap e1176: 8 total, 8 up, 8 in 2024-08-17T17:21:08.793 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:08.828 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:08.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:08.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:08 smithi162 ceph-mon[104668]: osdmap e1175: 8 total, 8 up, 8 in 2024-08-17T17:21:08.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:08 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 34686/120759 objects degraded (28.723%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:08.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:08 smithi162 ceph-mon[104668]: osdmap e1176: 8 total, 8 up, 8 in 2024-08-17T17:21:08.904 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:21:08.930 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:08.963 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:08.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:08.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:09.065 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:09.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.103 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:09.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:09.204 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:09.237 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:09.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:09.339 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:09.372 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:09.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:09.474 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:09.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.498 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:09.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:09.599 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:09.622 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:09.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:09.724 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:09.749 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:09.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:09.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:09 smithi063 ceph-mon[109843]: pgmap v1453: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+undersized+remapped, 2 peering, 1 activating, 1 remapped+peering, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 101 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 9813/113391 objects degraded (8.654%); 1232/113391 objects misplaced (1.087%); 3.5 MiB/s, 265 objects/s recovering 2024-08-17T17:21:09.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:09 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3682727298' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:21:09.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:09 smithi063 ceph-mon[109843]: osdmap e1177: 8 total, 8 up, 8 in 2024-08-17T17:21:09.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:09 smithi063 ceph-mon[114756]: pgmap v1453: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+undersized+remapped, 2 peering, 1 activating, 1 remapped+peering, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 101 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 9813/113391 objects degraded (8.654%); 1232/113391 objects misplaced (1.087%); 3.5 MiB/s, 265 objects/s recovering 2024-08-17T17:21:09.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:09 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3682727298' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:21:09.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:09 smithi063 ceph-mon[114756]: osdmap e1177: 8 total, 8 up, 8 in 2024-08-17T17:21:09.831 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:09 smithi162 ceph-mon[104668]: pgmap v1453: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+undersized+remapped, 2 peering, 1 activating, 1 remapped+peering, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 101 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 9813/113391 objects degraded (8.654%); 1232/113391 objects misplaced (1.087%); 3.5 MiB/s, 265 objects/s recovering 2024-08-17T17:21:09.831 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:09 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3682727298' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:21:09.831 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:09 smithi162 ceph-mon[104668]: osdmap e1177: 8 total, 8 up, 8 in 2024-08-17T17:21:09.852 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:09.875 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:09.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:09.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:09.897 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:21:09.977 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:10.005 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:10.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:10.107 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:10.137 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:10.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:10.239 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:10.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.284 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:10.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:10.385 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:10.424 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:10.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:10.526 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:10.556 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:10.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:10.658 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:10.684 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:10.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:10.699 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:10 smithi162 ceph-mon[104668]: 7.e scrub starts 2024-08-17T17:21:10.700 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:10 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:10.700 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:10 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3682727298' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:21:10.700 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:10 smithi162 ceph-mon[104668]: osdmap e1178: 8 total, 8 up, 8 in 2024-08-17T17:21:10.757 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:10 smithi063 ceph-mon[114756]: 7.e scrub starts 2024-08-17T17:21:10.757 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:10 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:10.757 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:10 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3682727298' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:21:10.758 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:10 smithi063 ceph-mon[114756]: osdmap e1178: 8 total, 8 up, 8 in 2024-08-17T17:21:10.758 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:10 smithi063 ceph-mon[109843]: 7.e scrub starts 2024-08-17T17:21:10.758 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:10 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:10.758 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:10 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3682727298' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:21:10.758 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:10 smithi063 ceph-mon[109843]: osdmap e1178: 8 total, 8 up, 8 in 2024-08-17T17:21:10.786 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:10.829 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:10.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:10.931 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:10.983 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:10.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:10.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:11.083 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:21:11.083 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:21:11.083 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:21:11.084 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:21:11.084 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.084 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160939 6.1 6:893849fe:::benchmark_data_smithi063_160268_object34832:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:21:11.084 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.614043+0000", 2024-08-17T17:21:11.084 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 333.46368166500002, 2024-08-17T17:21:11.084 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0308899330000001, 2024-08-17T17:21:11.084 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.084 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.084 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.084 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:21:11.084 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:21:11.084 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160939 2024-08-17T17:21:11.084 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.085 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.085 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.085 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.085 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614043+0000", 2024-08-17T17:21:11.085 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.085 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.085 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.085 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.085 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614043+0000", 2024-08-17T17:21:11.085 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.085 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.085 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.085 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.086 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614041+0000", 2024-08-17T17:21:11.086 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:21:11.086 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.086 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.086 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.086 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614047+0000", 2024-08-17T17:21:11.086 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9440000000000002e-06 2024-08-17T17:21:11.086 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.086 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.086 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.086 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614057+0000", 2024-08-17T17:21:11.087 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3409999999999992e-06 2024-08-17T17:21:11.087 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.087 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.087 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.087 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614064+0000", 2024-08-17T17:21:11.087 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7190000000000001e-06 2024-08-17T17:21:11.087 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.087 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.087 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.087 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614117+0000", 2024-08-17T17:21:11.087 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2797000000000003e-05 2024-08-17T17:21:11.087 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.088 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.088 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.088 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614128+0000", 2024-08-17T17:21:11.088 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0891e-05 2024-08-17T17:21:11.088 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.088 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.088 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.088 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.631386+0000", 2024-08-17T17:21:11.088 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.017258346 2024-08-17T17:21:11.088 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.088 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.088 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.088 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.633791+0000", 2024-08-17T17:21:11.089 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0024042109999999998 2024-08-17T17:21:11.089 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.089 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.089 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:21:11.089 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.633951+0000", 2024-08-17T17:21:11.089 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016093800000000001 2024-08-17T17:21:11.089 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.089 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.089 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.089 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644732+0000", 2024-08-17T17:21:11.089 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010780998 2024-08-17T17:21:11.089 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.090 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.090 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.090 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644760+0000", 2024-08-17T17:21:11.090 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.7195e-05 2024-08-17T17:21:11.090 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.090 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.090 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.090 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644837+0000", 2024-08-17T17:21:11.090 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7256999999999996e-05 2024-08-17T17:21:11.090 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.090 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.090 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.091 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644891+0000", 2024-08-17T17:21:11.091 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4284999999999999e-05 2024-08-17T17:21:11.091 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.091 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.091 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.091 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644933+0000", 2024-08-17T17:21:11.091 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1493999999999999e-05 2024-08-17T17:21:11.091 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.091 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.091 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.091 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.091 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.092 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160958 6.1 6:804ff99b:::benchmark_data_smithi063_160268_object34851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:21:11.092 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.649687+0000", 2024-08-17T17:21:11.092 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 333.42803781700002, 2024-08-17T17:21:11.092 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99790868700000002, 2024-08-17T17:21:11.092 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.092 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.092 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.092 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:21:11.092 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:21:11.092 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160958 2024-08-17T17:21:11.092 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.092 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.093 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.093 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.093 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649687+0000", 2024-08-17T17:21:11.093 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.093 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.093 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.093 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.093 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649687+0000", 2024-08-17T17:21:11.093 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.093 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.093 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.093 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.093 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649685+0000", 2024-08-17T17:21:11.094 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:21:11.094 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.094 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.094 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.094 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649695+0000", 2024-08-17T17:21:11.094 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9210000000000004e-06 2024-08-17T17:21:11.094 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.094 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.094 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.094 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649708+0000", 2024-08-17T17:21:11.094 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2947e-05 2024-08-17T17:21:11.095 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.095 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.095 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.095 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649715+0000", 2024-08-17T17:21:11.095 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9519999999999994e-06 2024-08-17T17:21:11.095 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.095 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.095 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.095 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649779+0000", 2024-08-17T17:21:11.095 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3683000000000007e-05 2024-08-17T17:21:11.095 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.095 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.095 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.096 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649792+0000", 2024-08-17T17:21:11.096 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3294e-05 2024-08-17T17:21:11.096 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.096 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.096 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.096 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.637682+0000", 2024-08-17T17:21:11.096 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98788912399999995 2024-08-17T17:21:11.096 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.096 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.096 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.096 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.637791+0000", 2024-08-17T17:21:11.096 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000109728 2024-08-17T17:21:11.096 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.097 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.097 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:21:11.097 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.637918+0000", 2024-08-17T17:21:11.097 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000126842 2024-08-17T17:21:11.097 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.097 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.097 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.097 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644787+0000", 2024-08-17T17:21:11.097 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0068684000000000002 2024-08-17T17:21:11.097 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.097 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.097 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.097 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647458+0000", 2024-08-17T17:21:11.098 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026717220000000001 2024-08-17T17:21:11.098 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.098 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.098 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.098 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647543+0000", 2024-08-17T17:21:11.098 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4790999999999995e-05 2024-08-17T17:21:11.098 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.098 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.098 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.098 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647572+0000", 2024-08-17T17:21:11.098 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8538999999999999e-05 2024-08-17T17:21:11.098 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.099 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.099 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.099 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647595+0000", 2024-08-17T17:21:11.099 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3648000000000001e-05 2024-08-17T17:21:11.099 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.099 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.099 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.099 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.099 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.099 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160968 6.1 6:8a3118ae:::benchmark_data_smithi063_160268_object34861:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:21:11.099 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.664096+0000", 2024-08-17T17:21:11.099 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 333.413627964, 2024-08-17T17:21:11.100 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99085020099999999, 2024-08-17T17:21:11.100 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.100 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.100 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.100 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:21:11.100 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:21:11.100 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160968 2024-08-17T17:21:11.100 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.100 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.100 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.100 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.100 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664096+0000", 2024-08-17T17:21:11.101 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.101 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.101 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.101 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.101 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664096+0000", 2024-08-17T17:21:11.101 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.101 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.101 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.101 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.101 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664095+0000", 2024-08-17T17:21:11.101 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:21:11.101 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.101 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.102 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.102 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664100+0000", 2024-08-17T17:21:11.102 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0880000000000002e-06 2024-08-17T17:21:11.102 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.102 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.102 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.102 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664112+0000", 2024-08-17T17:21:11.102 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1231e-05 2024-08-17T17:21:11.102 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.102 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.102 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.102 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664117+0000", 2024-08-17T17:21:11.102 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4829999999999999e-06 2024-08-17T17:21:11.103 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.103 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.103 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.103 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664174+0000", 2024-08-17T17:21:11.103 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6443999999999997e-05 2024-08-17T17:21:11.103 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.103 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.103 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.103 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664186+0000", 2024-08-17T17:21:11.103 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2500999999999999e-05 2024-08-17T17:21:11.103 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.103 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.104 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.104 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.641442+0000", 2024-08-17T17:21:11.104 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97725559900000003 2024-08-17T17:21:11.104 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.104 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.104 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.104 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.641557+0000", 2024-08-17T17:21:11.104 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000115661 2024-08-17T17:21:11.104 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.104 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.104 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:21:11.104 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.641675+0000", 2024-08-17T17:21:11.104 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011779 2024-08-17T17:21:11.105 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.105 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.105 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.105 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647462+0000", 2024-08-17T17:21:11.105 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0057873380000000004 2024-08-17T17:21:11.105 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.105 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.105 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.105 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647507+0000", 2024-08-17T17:21:11.105 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4776e-05 2024-08-17T17:21:11.105 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.105 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.105 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.106 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654882+0000", 2024-08-17T17:21:11.106 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0073748959999999997 2024-08-17T17:21:11.106 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.106 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.106 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.106 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654915+0000", 2024-08-17T17:21:11.106 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3333000000000001e-05 2024-08-17T17:21:11.106 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.106 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.106 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.106 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654947+0000", 2024-08-17T17:21:11.106 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1192999999999998e-05 2024-08-17T17:21:11.107 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.107 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.107 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.107 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.107 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.107 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160969 6.1 6:85b88efb:::benchmark_data_smithi063_160268_object34862:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:21:11.107 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.664210+0000", 2024-08-17T17:21:11.107 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 333.41351442899997, 2024-08-17T17:21:11.107 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99084335700000004, 2024-08-17T17:21:11.107 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.107 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.107 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.107 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:21:11.108 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:21:11.108 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160969 2024-08-17T17:21:11.108 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.108 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.108 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.108 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.108 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664210+0000", 2024-08-17T17:21:11.108 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.108 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.108 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.108 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.108 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664210+0000", 2024-08-17T17:21:11.109 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.109 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.109 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.109 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.109 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664209+0000", 2024-08-17T17:21:11.109 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:21:11.109 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.109 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.109 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.109 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664215+0000", 2024-08-17T17:21:11.109 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8619999999999999e-06 2024-08-17T17:21:11.109 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.109 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.110 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.110 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664227+0000", 2024-08-17T17:21:11.110 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.295e-05 2024-08-17T17:21:11.110 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.110 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.110 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.110 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664234+0000", 2024-08-17T17:21:11.110 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8399999999999997e-06 2024-08-17T17:21:11.110 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.110 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.110 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.110 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664286+0000", 2024-08-17T17:21:11.111 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1928000000000002e-05 2024-08-17T17:21:11.111 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.111 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.111 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.111 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664300+0000", 2024-08-17T17:21:11.111 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3497e-05 2024-08-17T17:21:11.111 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.111 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.111 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.111 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644983+0000", 2024-08-17T17:21:11.111 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98068294099999997 2024-08-17T17:21:11.111 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.111 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.112 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.112 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.645050+0000", 2024-08-17T17:21:11.112 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7012000000000001e-05 2024-08-17T17:21:11.112 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.112 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.112 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:21:11.112 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.645116+0000", 2024-08-17T17:21:11.112 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6268999999999999e-05 2024-08-17T17:21:11.112 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.112 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.112 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.112 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654779+0000", 2024-08-17T17:21:11.112 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0096628619999999995 2024-08-17T17:21:11.113 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.113 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.113 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.113 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654813+0000", 2024-08-17T17:21:11.113 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4489000000000001e-05 2024-08-17T17:21:11.113 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.113 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.113 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.113 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654995+0000", 2024-08-17T17:21:11.113 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000181478 2024-08-17T17:21:11.113 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.113 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.113 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.114 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655021+0000", 2024-08-17T17:21:11.114 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6390999999999999e-05 2024-08-17T17:21:11.114 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.114 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:11.115 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.115 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.115 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655053+0000", 2024-08-17T17:21:11.115 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2147999999999998e-05 2024-08-17T17:21:11.115 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.116 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.116 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.116 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.116 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.116 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160976 6.1 6:892f13c5:::benchmark_data_smithi063_160268_object34869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:21:11.116 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.670369+0000", 2024-08-17T17:21:11.116 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 333.40735549999999, 2024-08-17T17:21:11.116 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99685337699999998, 2024-08-17T17:21:11.116 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.116 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.116 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.116 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:21:11.116 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:21:11.116 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160976 2024-08-17T17:21:11.117 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.117 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.117 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.117 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.117 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670369+0000", 2024-08-17T17:21:11.117 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.117 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.117 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.117 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.117 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670369+0000", 2024-08-17T17:21:11.117 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.117 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.118 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.118 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.118 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670367+0000", 2024-08-17T17:21:11.118 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:21:11.118 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.118 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.118 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.118 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670374+0000", 2024-08-17T17:21:11.118 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.827e-06 2024-08-17T17:21:11.118 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.118 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.119 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.119 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670390+0000", 2024-08-17T17:21:11.119 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5566e-05 2024-08-17T17:21:11.119 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.119 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.119 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.119 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670398+0000", 2024-08-17T17:21:11.119 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3289999999999994e-06 2024-08-17T17:21:11.119 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.119 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.119 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.119 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670470+0000", 2024-08-17T17:21:11.120 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1913000000000001e-05 2024-08-17T17:21:11.120 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.120 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.120 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.120 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670482+0000", 2024-08-17T17:21:11.120 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2018000000000001e-05 2024-08-17T17:21:11.120 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.120 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.120 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.120 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647627+0000", 2024-08-17T17:21:11.120 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.977145139 2024-08-17T17:21:11.120 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.121 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.121 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.121 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647700+0000", 2024-08-17T17:21:11.121 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2495000000000003e-05 2024-08-17T17:21:11.121 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.121 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.121 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:21:11.121 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647765+0000", 2024-08-17T17:21:11.121 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5495999999999998e-05 2024-08-17T17:21:11.121 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.121 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.121 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.122 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654846+0000", 2024-08-17T17:21:11.122 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0070811249999999997 2024-08-17T17:21:11.122 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.122 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.122 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.122 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654853+0000", 2024-08-17T17:21:11.122 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2929999999999997e-06 2024-08-17T17:21:11.122 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.122 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.122 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.122 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667120+0000", 2024-08-17T17:21:11.122 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012267867 2024-08-17T17:21:11.122 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.123 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.123 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.123 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667180+0000", 2024-08-17T17:21:11.123 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9376999999999997e-05 2024-08-17T17:21:11.123 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.123 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.123 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.123 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667222+0000", 2024-08-17T17:21:11.123 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2451999999999999e-05 2024-08-17T17:21:11.123 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.123 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.123 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.123 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.124 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.124 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160983 6.1 6:82d2572f:::benchmark_data_smithi063_160268_object34876:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:21:11.124 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.676198+0000", 2024-08-17T17:21:11.124 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 333.40152639899998, 2024-08-17T17:21:11.124 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99119904999999997, 2024-08-17T17:21:11.124 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.124 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.124 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.124 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:21:11.124 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:21:11.124 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160983 2024-08-17T17:21:11.124 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.124 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.125 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.125 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.125 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676198+0000", 2024-08-17T17:21:11.125 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.125 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.125 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.125 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.125 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676198+0000", 2024-08-17T17:21:11.125 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.125 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.125 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.125 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.125 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676196+0000", 2024-08-17T17:21:11.126 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:21:11.126 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.126 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.126 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.126 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676205+0000", 2024-08-17T17:21:11.126 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3079999999999997e-06 2024-08-17T17:21:11.126 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.126 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.126 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.126 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676223+0000", 2024-08-17T17:21:11.126 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7217000000000001e-05 2024-08-17T17:21:11.126 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.126 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.127 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.127 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676230+0000", 2024-08-17T17:21:11.127 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3479999999999998e-06 2024-08-17T17:21:11.127 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.127 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.127 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.127 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676293+0000", 2024-08-17T17:21:11.127 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2714999999999998e-05 2024-08-17T17:21:11.127 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.127 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.127 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.127 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676305+0000", 2024-08-17T17:21:11.127 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2455e-05 2024-08-17T17:21:11.128 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.128 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.128 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.128 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.651451+0000", 2024-08-17T17:21:11.128 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97514584400000004 2024-08-17T17:21:11.128 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.128 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.128 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.128 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.651523+0000", 2024-08-17T17:21:11.128 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2497999999999996e-05 2024-08-17T17:21:11.128 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.128 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.129 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:21:11.129 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.651598+0000", 2024-08-17T17:21:11.129 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4241999999999998e-05 2024-08-17T17:21:11.129 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.129 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.129 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.129 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659221+0000", 2024-08-17T17:21:11.129 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0076229289999999996 2024-08-17T17:21:11.129 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.129 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.129 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.129 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667006+0000", 2024-08-17T17:21:11.129 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00778548 2024-08-17T17:21:11.130 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.130 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.130 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.130 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667303+0000", 2024-08-17T17:21:11.130 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000297103 2024-08-17T17:21:11.130 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.130 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.130 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.130 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667362+0000", 2024-08-17T17:21:11.130 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8705000000000001e-05 2024-08-17T17:21:11.130 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.130 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.131 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.131 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667397+0000", 2024-08-17T17:21:11.131 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5120999999999998e-05 2024-08-17T17:21:11.131 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.131 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.131 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.131 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.131 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.131 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160989 6.1 6:8559d498:::benchmark_data_smithi063_160268_object34882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:21:11.131 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.684830+0000", 2024-08-17T17:21:11.131 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 333.39289443500002, 2024-08-17T17:21:11.131 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98270078400000005, 2024-08-17T17:21:11.132 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.132 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.132 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.132 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:21:11.132 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:21:11.132 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160989 2024-08-17T17:21:11.132 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.132 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.132 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.132 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.132 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684830+0000", 2024-08-17T17:21:11.132 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.133 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.133 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.133 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.133 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684830+0000", 2024-08-17T17:21:11.133 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.133 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.133 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.133 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.133 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684828+0000", 2024-08-17T17:21:11.133 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:21:11.133 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.133 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.134 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.134 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684835+0000", 2024-08-17T17:21:11.134 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9890000000000003e-06 2024-08-17T17:21:11.134 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.134 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.134 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.134 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684851+0000", 2024-08-17T17:21:11.134 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5457000000000001e-05 2024-08-17T17:21:11.134 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.134 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.134 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.134 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684863+0000", 2024-08-17T17:21:11.134 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2302e-05 2024-08-17T17:21:11.135 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.135 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.135 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.135 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684941+0000", 2024-08-17T17:21:11.135 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7541999999999997e-05 2024-08-17T17:21:11.135 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.135 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.135 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.135 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684956+0000", 2024-08-17T17:21:11.135 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5148e-05 2024-08-17T17:21:11.135 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.135 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.136 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.136 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655087+0000", 2024-08-17T17:21:11.136 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97013142799999996 2024-08-17T17:21:11.136 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.136 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.136 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.136 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655187+0000", 2024-08-17T17:21:11.136 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9229999999999997e-05 2024-08-17T17:21:11.136 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.136 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.136 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:21:11.136 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655293+0000", 2024-08-17T17:21:11.136 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000106576 2024-08-17T17:21:11.137 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.137 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.137 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.137 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667235+0000", 2024-08-17T17:21:11.137 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011942026 2024-08-17T17:21:11.137 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.137 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.137 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.137 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667269+0000", 2024-08-17T17:21:11.137 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.417e-05 2024-08-17T17:21:11.137 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.137 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.138 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.138 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667449+0000", 2024-08-17T17:21:11.138 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000179292 2024-08-17T17:21:11.138 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.138 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.138 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.138 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667498+0000", 2024-08-17T17:21:11.138 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9821999999999998e-05 2024-08-17T17:21:11.138 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.138 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.138 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.138 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667531+0000", 2024-08-17T17:21:11.139 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2326999999999997e-05 2024-08-17T17:21:11.139 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.139 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.139 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.139 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.139 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.139 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160996 6.1 6:8ccbb0f4:::benchmark_data_smithi063_160268_object34889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:21:11.139 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.691144+0000", 2024-08-17T17:21:11.139 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 333.38657996500001, 2024-08-17T17:21:11.139 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98531105200000002, 2024-08-17T17:21:11.139 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.139 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.140 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.140 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:21:11.140 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:21:11.140 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160996 2024-08-17T17:21:11.140 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.140 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.140 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.140 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.140 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691144+0000", 2024-08-17T17:21:11.140 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.140 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.140 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.140 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.141 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691144+0000", 2024-08-17T17:21:11.141 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.141 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.141 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.141 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.141 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691143+0000", 2024-08-17T17:21:11.141 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:21:11.141 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.141 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.141 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.141 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691149+0000", 2024-08-17T17:21:11.141 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0800000000000002e-06 2024-08-17T17:21:11.142 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.142 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.142 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.142 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691162+0000", 2024-08-17T17:21:11.142 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.322e-05 2024-08-17T17:21:11.142 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.142 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.142 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.142 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691168+0000", 2024-08-17T17:21:11.142 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0260000000000004e-06 2024-08-17T17:21:11.142 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.142 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.142 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.142 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691211+0000", 2024-08-17T17:21:11.143 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3013999999999998e-05 2024-08-17T17:21:11.143 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.143 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.143 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.143 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691220+0000", 2024-08-17T17:21:11.143 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8540000000000003e-06 2024-08-17T17:21:11.143 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.143 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.143 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.143 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659248+0000", 2024-08-17T17:21:11.143 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.96802809099999998 2024-08-17T17:21:11.143 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.143 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.143 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.144 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659357+0000", 2024-08-17T17:21:11.144 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010854 2024-08-17T17:21:11.144 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.144 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.144 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:21:11.144 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659466+0000", 2024-08-17T17:21:11.144 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000109437 2024-08-17T17:21:11.144 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.144 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.144 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.144 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667552+0000", 2024-08-17T17:21:11.144 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0080855279999999998 2024-08-17T17:21:11.145 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.145 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.145 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.145 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676310+0000", 2024-08-17T17:21:11.145 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0087583250000000008 2024-08-17T17:21:11.145 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.145 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.145 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.145 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676357+0000", 2024-08-17T17:21:11.145 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6786000000000002e-05 2024-08-17T17:21:11.145 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.145 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.145 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.145 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676409+0000", 2024-08-17T17:21:11.146 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2104000000000001e-05 2024-08-17T17:21:11.146 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.146 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.146 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.146 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676455+0000", 2024-08-17T17:21:11.146 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6362000000000001e-05 2024-08-17T17:21:11.146 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.146 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.146 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.146 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.146 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.146 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:161006 6.1 6:8d7d2cbb:::benchmark_data_smithi063_160268_object34899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:21:11.147 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.709227+0000", 2024-08-17T17:21:11.147 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 333.368497214, 2024-08-17T17:21:11.147 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97342043, 2024-08-17T17:21:11.147 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.147 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.147 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.147 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:21:11.147 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:21:11.147 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 161006 2024-08-17T17:21:11.147 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.147 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.147 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.147 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.148 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709227+0000", 2024-08-17T17:21:11.148 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.148 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.148 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.148 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.148 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709227+0000", 2024-08-17T17:21:11.148 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.148 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.148 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.148 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.148 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709226+0000", 2024-08-17T17:21:11.148 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:21:11.149 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.149 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.149 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.149 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709232+0000", 2024-08-17T17:21:11.149 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6219999999999997e-06 2024-08-17T17:21:11.149 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.149 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.149 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.149 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709243+0000", 2024-08-17T17:21:11.149 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0839000000000001e-05 2024-08-17T17:21:11.149 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.149 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.149 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.150 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709248+0000", 2024-08-17T17:21:11.150 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.322e-06 2024-08-17T17:21:11.150 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.150 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.150 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.150 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709293+0000", 2024-08-17T17:21:11.150 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4672000000000002e-05 2024-08-17T17:21:11.150 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.150 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.150 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.150 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709302+0000", 2024-08-17T17:21:11.150 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2240000000000001e-06 2024-08-17T17:21:11.150 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.151 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.151 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.151 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.663451+0000", 2024-08-17T17:21:11.151 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95414951299999995 2024-08-17T17:21:11.151 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.151 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.151 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.151 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.663549+0000", 2024-08-17T17:21:11.151 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7268000000000005e-05 2024-08-17T17:21:11.151 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.151 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.152 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:21:11.152 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.663655+0000", 2024-08-17T17:21:11.152 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000106872 2024-08-17T17:21:11.152 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.152 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.152 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.152 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667559+0000", 2024-08-17T17:21:11.152 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0039036629999999999 2024-08-17T17:21:11.152 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.152 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.152 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.152 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.678421+0000", 2024-08-17T17:21:11.152 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010861585 2024-08-17T17:21:11.153 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.153 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.153 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.153 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.682563+0000", 2024-08-17T17:21:11.153 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0041419159999999998 2024-08-17T17:21:11.153 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.153 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.153 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.153 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.682611+0000", 2024-08-17T17:21:11.153 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8585000000000003e-05 2024-08-17T17:21:11.153 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.153 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.153 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.154 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.682648+0000", 2024-08-17T17:21:11.154 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6449999999999998e-05 2024-08-17T17:21:11.154 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.154 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.154 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.154 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.154 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.154 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65735 7.7 7:e7f48e21:::benchmark_data_smithi063_196002_object65734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-08-17T17:21:11.154 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.525390+0000", 2024-08-17T17:21:11.154 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.55233457200001, 2024-08-17T17:21:11.154 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4912375070000001, 2024-08-17T17:21:11.154 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.155 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.155 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.155 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:21:11.155 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:21:11.155 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65735 2024-08-17T17:21:11.155 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.155 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.155 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.155 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.155 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525390+0000", 2024-08-17T17:21:11.155 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.155 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.155 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.156 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.156 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525390+0000", 2024-08-17T17:21:11.156 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.156 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.156 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.156 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.156 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525388+0000", 2024-08-17T17:21:11.156 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:21:11.156 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.156 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.156 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.156 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525427+0000", 2024-08-17T17:21:11.156 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8878000000000003e-05 2024-08-17T17:21:11.157 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.157 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.157 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.157 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525439+0000", 2024-08-17T17:21:11.157 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2507000000000001e-05 2024-08-17T17:21:11.157 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.157 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.157 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.157 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525447+0000", 2024-08-17T17:21:11.157 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7530000000000001e-06 2024-08-17T17:21:11.157 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.157 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.158 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.158 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.533687+0000", 2024-08-17T17:21:11.158 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0082400100000000007 2024-08-17T17:21:11.158 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.158 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.158 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.158 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.533697+0000", 2024-08-17T17:21:11.158 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0455000000000001e-05 2024-08-17T17:21:11.158 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.158 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.158 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.158 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523807+0000", 2024-08-17T17:21:11.159 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99010963600000002 2024-08-17T17:21:11.159 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.159 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.159 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.159 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523817+0000", 2024-08-17T17:21:11.159 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7189999999999996e-06 2024-08-17T17:21:11.159 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.159 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.159 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.159 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:32.998106+0000", 2024-08-17T17:21:11.159 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4742891280000001 2024-08-17T17:21:11.159 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.159 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.160 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.160 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:32.998241+0000", 2024-08-17T17:21:11.160 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013504000000000001 2024-08-17T17:21:11.160 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.160 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.160 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:21:11.160 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:32.998370+0000", 2024-08-17T17:21:11.160 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012903799999999999 2024-08-17T17:21:11.160 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.160 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.160 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.161 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.003851+0000", 2024-08-17T17:21:11.162 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0054806999999999998 2024-08-17T17:21:11.162 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.162 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.162 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.162 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.005429+0000", 2024-08-17T17:21:11.162 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015783819999999999 2024-08-17T17:21:11.162 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.162 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.162 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.162 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.014199+0000", 2024-08-17T17:21:11.162 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0087699130000000007 2024-08-17T17:21:11.163 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.163 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.163 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.163 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.014257+0000", 2024-08-17T17:21:11.163 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8193999999999999e-05 2024-08-17T17:21:11.163 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.163 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.163 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.163 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.016627+0000", 2024-08-17T17:21:11.163 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023701730000000002 2024-08-17T17:21:11.163 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.163 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.164 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.164 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.164 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.164 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65738 7.d 7:b7ed0697:::benchmark_data_smithi063_196002_object65737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:21:11.164 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.535426+0000", 2024-08-17T17:21:11.164 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.54229848, 2024-08-17T17:21:11.164 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5234001109999999, 2024-08-17T17:21:11.164 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.164 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.164 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.164 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:21:11.164 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:21:11.165 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65738 2024-08-17T17:21:11.165 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.165 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.165 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.165 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.165 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535426+0000", 2024-08-17T17:21:11.165 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.165 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.165 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.165 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.165 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535426+0000", 2024-08-17T17:21:11.165 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.165 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.166 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.166 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.166 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535424+0000", 2024-08-17T17:21:11.166 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:21:11.166 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.166 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.166 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.166 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535475+0000", 2024-08-17T17:21:11.166 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0512e-05 2024-08-17T17:21:11.166 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.166 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.166 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.167 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535486+0000", 2024-08-17T17:21:11.167 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0689999999999999e-05 2024-08-17T17:21:11.167 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.167 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.167 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.167 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535492+0000", 2024-08-17T17:21:11.167 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7379999999999997e-06 2024-08-17T17:21:11.167 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.167 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.167 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.167 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537456+0000", 2024-08-17T17:21:11.167 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0019633559999999999 2024-08-17T17:21:11.168 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.168 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.168 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.168 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537467+0000", 2024-08-17T17:21:11.168 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0784999999999999e-05 2024-08-17T17:21:11.168 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.168 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.168 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.168 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523461+0000", 2024-08-17T17:21:11.168 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.985994009 2024-08-17T17:21:11.168 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.168 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.169 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.169 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523472+0000", 2024-08-17T17:21:11.169 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1806e-05 2024-08-17T17:21:11.169 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.169 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.169 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.169 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.030195+0000", 2024-08-17T17:21:11.169 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.506722414 2024-08-17T17:21:11.169 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.169 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.169 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.169 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.030370+0000", 2024-08-17T17:21:11.169 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017517499999999999 2024-08-17T17:21:11.170 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.170 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.170 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:21:11.170 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.030534+0000", 2024-08-17T17:21:11.170 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016449999999999999 2024-08-17T17:21:11.170 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.170 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.170 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.170 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.038628+0000", 2024-08-17T17:21:11.170 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0080940379999999996 2024-08-17T17:21:11.170 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.170 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.170 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.171 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.038638+0000", 2024-08-17T17:21:11.171 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8770000000000005e-06 2024-08-17T17:21:11.171 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.171 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.171 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.171 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.056336+0000", 2024-08-17T17:21:11.171 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.017698034000000001 2024-08-17T17:21:11.171 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.171 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.171 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.171 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.056374+0000", 2024-08-17T17:21:11.171 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7753000000000003e-05 2024-08-17T17:21:11.171 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.172 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.172 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.172 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.058826+0000", 2024-08-17T17:21:11.172 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0024519149999999998 2024-08-17T17:21:11.172 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.172 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.172 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.172 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.172 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.172 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65740 7.7 7:e4a48506:::benchmark_data_smithi063_196002_object65739:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:21:11.172 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.537667+0000", 2024-08-17T17:21:11.172 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.54005696799999, 2024-08-17T17:21:11.173 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4812937370000001, 2024-08-17T17:21:11.173 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.173 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.173 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.173 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:21:11.173 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:21:11.173 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65740 2024-08-17T17:21:11.173 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.173 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.173 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.173 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.173 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537667+0000", 2024-08-17T17:21:11.174 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.174 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.174 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.174 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.174 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537667+0000", 2024-08-17T17:21:11.174 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.174 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.174 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.174 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.174 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537666+0000", 2024-08-17T17:21:11.174 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:21:11.174 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.174 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.175 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.175 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537699+0000", 2024-08-17T17:21:11.175 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2543000000000003e-05 2024-08-17T17:21:11.175 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.175 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.175 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.175 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537708+0000", 2024-08-17T17:21:11.175 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9700000000000005e-06 2024-08-17T17:21:11.175 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.175 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.175 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.175 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537715+0000", 2024-08-17T17:21:11.176 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1860000000000003e-06 2024-08-17T17:21:11.176 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.176 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.176 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.176 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537786+0000", 2024-08-17T17:21:11.176 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0732e-05 2024-08-17T17:21:11.176 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.176 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.176 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.176 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537795+0000", 2024-08-17T17:21:11.176 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7370000000000001e-06 2024-08-17T17:21:11.176 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.176 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.177 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.177 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524063+0000", 2024-08-17T17:21:11.177 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98626732800000005 2024-08-17T17:21:11.177 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.177 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.177 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.177 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524072+0000", 2024-08-17T17:21:11.177 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.1949999999999992e-06 2024-08-17T17:21:11.177 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.177 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.177 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.178 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.001760+0000", 2024-08-17T17:21:11.178 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4776884450000001 2024-08-17T17:21:11.178 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.178 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.178 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.178 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.001916+0000", 2024-08-17T17:21:11.178 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015600400000000001 2024-08-17T17:21:11.178 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.178 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.178 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:21:11.178 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.002089+0000", 2024-08-17T17:21:11.178 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017285900000000001 2024-08-17T17:21:11.179 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.179 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.179 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.179 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.006924+0000", 2024-08-17T17:21:11.179 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0048349899999999999 2024-08-17T17:21:11.179 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.179 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.179 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.179 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.006974+0000", 2024-08-17T17:21:11.179 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9648999999999998e-05 2024-08-17T17:21:11.179 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.179 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.179 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.180 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.016708+0000", 2024-08-17T17:21:11.180 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0097340340000000008 2024-08-17T17:21:11.180 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.180 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.180 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.180 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.016776+0000", 2024-08-17T17:21:11.180 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8320999999999999e-05 2024-08-17T17:21:11.180 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.180 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.180 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.180 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.018961+0000", 2024-08-17T17:21:11.180 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021849840000000001 2024-08-17T17:21:11.180 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.181 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.181 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.181 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.181 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.181 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65748 7.d 7:bfbcc155:::benchmark_data_smithi063_196002_object65747:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:21:11.181 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.545206+0000", 2024-08-17T17:21:11.181 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.53251873299999, 2024-08-17T17:21:11.181 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5163881620000002, 2024-08-17T17:21:11.181 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.181 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.181 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.181 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:21:11.182 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:21:11.182 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65748 2024-08-17T17:21:11.182 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.182 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.182 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.182 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.182 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545206+0000", 2024-08-17T17:21:11.182 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.182 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.182 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.182 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.182 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545206+0000", 2024-08-17T17:21:11.182 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.183 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.183 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.183 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.183 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545205+0000", 2024-08-17T17:21:11.183 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:21:11.183 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.183 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.183 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.183 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545338+0000", 2024-08-17T17:21:11.183 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000133375 2024-08-17T17:21:11.183 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.183 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.183 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.184 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545351+0000", 2024-08-17T17:21:11.184 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2901000000000001e-05 2024-08-17T17:21:11.184 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.184 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.184 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.184 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545358+0000", 2024-08-17T17:21:11.184 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9009999999999997e-06 2024-08-17T17:21:11.184 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.184 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.184 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.184 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545409+0000", 2024-08-17T17:21:11.184 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1038999999999997e-05 2024-08-17T17:21:11.184 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.185 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.185 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.185 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545418+0000", 2024-08-17T17:21:11.185 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.0990000000000005e-06 2024-08-17T17:21:11.185 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.185 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.185 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.185 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523596+0000", 2024-08-17T17:21:11.185 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97817841900000002 2024-08-17T17:21:11.185 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.185 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.185 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.185 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523605+0000", 2024-08-17T17:21:11.185 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.1630000000000002e-06 2024-08-17T17:21:11.186 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.186 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.186 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.186 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.035987+0000", 2024-08-17T17:21:11.186 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.512381864 2024-08-17T17:21:11.186 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.186 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.186 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.186 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.036136+0000", 2024-08-17T17:21:11.186 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014822899999999999 2024-08-17T17:21:11.186 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.186 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.187 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:21:11.187 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.036293+0000", 2024-08-17T17:21:11.187 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000157305 2024-08-17T17:21:11.187 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.187 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.187 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.187 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.041715+0000", 2024-08-17T17:21:11.187 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.005422105 2024-08-17T17:21:11.187 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.187 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.187 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.187 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.041772+0000", 2024-08-17T17:21:11.187 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7457999999999998e-05 2024-08-17T17:21:11.188 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.188 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.188 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.188 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.058975+0000", 2024-08-17T17:21:11.188 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.017202696999999999 2024-08-17T17:21:11.188 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.188 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.188 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.188 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.059136+0000", 2024-08-17T17:21:11.188 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016129800000000001 2024-08-17T17:21:11.188 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.188 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.189 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.189 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.061594+0000", 2024-08-17T17:21:11.189 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002457416 2024-08-17T17:21:11.189 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.189 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.189 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.189 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.189 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.189 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65753 7.7 7:e08ff533:::benchmark_data_smithi063_196002_object65752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:21:11.189 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.548721+0000", 2024-08-17T17:21:11.189 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.52900342999999, 2024-08-17T17:21:11.189 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.472712064, 2024-08-17T17:21:11.190 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.190 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.190 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.190 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:21:11.190 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:21:11.190 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65753 2024-08-17T17:21:11.190 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.190 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.190 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.190 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.190 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548721+0000", 2024-08-17T17:21:11.190 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.190 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.191 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.191 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.191 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548721+0000", 2024-08-17T17:21:11.191 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.191 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.191 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.191 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.191 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548720+0000", 2024-08-17T17:21:11.191 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:21:11.191 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.191 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.191 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.191 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548841+0000", 2024-08-17T17:21:11.192 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012082399999999999 2024-08-17T17:21:11.192 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.192 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.192 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.192 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548851+0000", 2024-08-17T17:21:11.192 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9660000000000008e-06 2024-08-17T17:21:11.192 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.192 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.192 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.192 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548862+0000", 2024-08-17T17:21:11.192 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0838e-05 2024-08-17T17:21:11.192 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.193 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.193 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.193 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548911+0000", 2024-08-17T17:21:11.193 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9004e-05 2024-08-17T17:21:11.193 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.193 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.193 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.193 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548921+0000", 2024-08-17T17:21:11.193 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9180000000000006e-06 2024-08-17T17:21:11.193 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.193 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.193 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.194 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524131+0000", 2024-08-17T17:21:11.194 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.975210566 2024-08-17T17:21:11.194 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.194 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.194 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.194 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524140+0000", 2024-08-17T17:21:11.194 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.242e-06 2024-08-17T17:21:11.194 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.194 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.194 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.194 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.003888+0000", 2024-08-17T17:21:11.195 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4797479820000001 2024-08-17T17:21:11.195 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.195 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.195 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.195 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.004030+0000", 2024-08-17T17:21:11.195 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000142495 2024-08-17T17:21:11.195 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.195 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.195 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:21:11.195 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.004221+0000", 2024-08-17T17:21:11.195 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00019079500000000001 2024-08-17T17:21:11.195 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.195 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.196 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.196 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.006933+0000", 2024-08-17T17:21:11.196 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00271244 2024-08-17T17:21:11.196 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.196 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.196 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.196 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.008656+0000", 2024-08-17T17:21:11.196 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001723154 2024-08-17T17:21:11.196 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.196 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.196 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.196 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.019021+0000", 2024-08-17T17:21:11.196 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010364310999999999 2024-08-17T17:21:11.197 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.197 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.197 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.197 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.019082+0000", 2024-08-17T17:21:11.197 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0998000000000001e-05 2024-08-17T17:21:11.197 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.197 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.197 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.197 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.021433+0000", 2024-08-17T17:21:11.197 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023512530000000002 2024-08-17T17:21:11.197 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.198 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.198 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.198 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.198 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.198 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65763 7.7 7:e13a5ff4:::benchmark_data_smithi063_196002_object65762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:21:11.198 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.559250+0000", 2024-08-17T17:21:11.198 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 160.518474228, 2024-08-17T17:21:11.198 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4649766419999999, 2024-08-17T17:21:11.198 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.198 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.198 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.199 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:21:11.199 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:21:11.199 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65763 2024-08-17T17:21:11.199 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.199 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.199 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.199 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.199 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559250+0000", 2024-08-17T17:21:11.199 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.199 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.199 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.199 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.200 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559250+0000", 2024-08-17T17:21:11.200 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.200 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.200 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.200 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.200 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559249+0000", 2024-08-17T17:21:11.200 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:21:11.200 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.200 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.200 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.200 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559294+0000", 2024-08-17T17:21:11.200 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5008e-05 2024-08-17T17:21:11.201 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.201 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.201 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.201 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559304+0000", 2024-08-17T17:21:11.201 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0543999999999999e-05 2024-08-17T17:21:11.201 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.201 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.201 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.201 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559311+0000", 2024-08-17T17:21:11.201 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9999999999999999e-06 2024-08-17T17:21:11.201 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.201 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.202 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.202 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559352+0000", 2024-08-17T17:21:11.202 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0099999999999999e-05 2024-08-17T17:21:11.202 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.202 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.202 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.202 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559360+0000", 2024-08-17T17:21:11.202 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4090000000000003e-06 2024-08-17T17:21:11.202 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.202 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.202 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.202 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524193+0000", 2024-08-17T17:21:11.203 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.96483273400000003 2024-08-17T17:21:11.203 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.203 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.203 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.203 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524201+0000", 2024-08-17T17:21:11.203 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.5280000000000003e-06 2024-08-17T17:21:11.203 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.203 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.203 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.203 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.005532+0000", 2024-08-17T17:21:11.203 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4813310959999999 2024-08-17T17:21:11.203 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.203 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.203 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.204 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.005618+0000", 2024-08-17T17:21:11.204 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6101999999999997e-05 2024-08-17T17:21:11.204 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.204 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.204 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:21:11.204 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.005742+0000", 2024-08-17T17:21:11.204 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012364099999999999 2024-08-17T17:21:11.204 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.204 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.204 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.204 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.007064+0000", 2024-08-17T17:21:11.204 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013215550000000001 2024-08-17T17:21:11.205 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.205 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.205 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.205 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.008743+0000", 2024-08-17T17:21:11.205 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016792140000000001 2024-08-17T17:21:11.205 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.205 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.205 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.205 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.021506+0000", 2024-08-17T17:21:11.205 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012763480000000001 2024-08-17T17:21:11.205 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.205 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.205 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.205 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.021565+0000", 2024-08-17T17:21:11.206 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8486000000000002e-05 2024-08-17T17:21:11.206 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.206 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.206 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.206 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.024227+0000", 2024-08-17T17:21:11.206 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026619759999999999 2024-08-17T17:21:11.206 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.206 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.206 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.206 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.206 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.206 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88363 7.7 7:e6097857:::benchmark_data_smithi063_196002_object88362:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:21:11.207 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:00.712141+0000", 2024-08-17T17:21:11.207 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 130.36558335500001, 2024-08-17T17:21:11.207 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9602415930000001, 2024-08-17T17:21:11.207 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.207 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:21:11.207 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.207 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:21:11.207 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:21:11.207 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88363 2024-08-17T17:21:11.207 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.207 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.207 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.207 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.208 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712141+0000", 2024-08-17T17:21:11.208 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.208 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.208 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.208 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.208 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712141+0000", 2024-08-17T17:21:11.208 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.208 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.208 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.208 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.208 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712138+0000", 2024-08-17T17:21:11.208 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999971 2024-08-17T17:21:11.208 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.209 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.209 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.209 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712181+0000", 2024-08-17T17:21:11.209 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3390999999999998e-05 2024-08-17T17:21:11.209 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.209 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.209 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.209 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712208+0000", 2024-08-17T17:21:11.209 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6259e-05 2024-08-17T17:21:11.209 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.209 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.209 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.210 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712219+0000", 2024-08-17T17:21:11.210 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1686e-05 2024-08-17T17:21:11.210 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.210 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.210 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.210 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.771097+0000", 2024-08-17T17:21:11.210 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.058877691000000003 2024-08-17T17:21:11.210 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.210 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.210 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.210 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.771106+0000", 2024-08-17T17:21:11.210 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6079999999999996e-06 2024-08-17T17:21:11.210 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.211 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.211 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.211 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.650544+0000", 2024-08-17T17:21:11.211 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.879438045 2024-08-17T17:21:11.211 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.211 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.211 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.211 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.650553+0000", 2024-08-17T17:21:11.211 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.6520000000000001e-06 2024-08-17T17:21:11.211 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.211 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.211 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.212 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.672283+0000", 2024-08-17T17:21:11.212 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0217297430000001 2024-08-17T17:21:11.212 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.212 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.212 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.212 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.672383+0000", 2024-08-17T17:21:11.212 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9489999999999998e-05 2024-08-17T17:21:11.212 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.212 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.212 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.212 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.212 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.212 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:170142 7.d 7:b454d72d:::benchmark_data_smithi063_196002_object58049:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1164)", 2024-08-17T17:21:11.213 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:20:43.923974+0000", 2024-08-17T17:21:11.213 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 27.153750466999998, 2024-08-17T17:21:11.213 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98031507699999998, 2024-08-17T17:21:11.213 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.213 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.213 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.213 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:21:11.213 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:21:11.213 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 170142 2024-08-17T17:21:11.213 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.213 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.213 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.214 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.214 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923974+0000", 2024-08-17T17:21:11.214 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.214 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.214 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.214 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.214 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923974+0000", 2024-08-17T17:21:11.214 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.214 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.214 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.214 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.214 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923973+0000", 2024-08-17T17:21:11.215 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:21:11.215 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.215 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.215 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.215 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923978+0000", 2024-08-17T17:21:11.215 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9819999999999999e-06 2024-08-17T17:21:11.215 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.215 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.215 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.215 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923989+0000", 2024-08-17T17:21:11.215 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1197e-05 2024-08-17T17:21:11.215 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.215 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.216 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.216 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923994+0000", 2024-08-17T17:21:11.216 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9599999999999999e-06 2024-08-17T17:21:11.216 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.216 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.216 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.216 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.924041+0000", 2024-08-17T17:21:11.216 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7633999999999998e-05 2024-08-17T17:21:11.216 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.216 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.216 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.216 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.924050+0000", 2024-08-17T17:21:11.216 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0940000000000008e-06 2024-08-17T17:21:11.217 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.217 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.217 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.217 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.863783+0000", 2024-08-17T17:21:11.217 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.93973320699999996 2024-08-17T17:21:11.217 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.217 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.217 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for degraded object", 2024-08-17T17:21:11.217 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.864178+0000", 2024-08-17T17:21:11.217 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00039516399999999999 2024-08-17T17:21:11.217 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.217 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.217 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.217 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.885932+0000", 2024-08-17T17:21:11.218 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.021753828999999999 2024-08-17T17:21:11.218 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.218 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.218 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.218 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.886015+0000", 2024-08-17T17:21:11.218 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3139999999999993e-05 2024-08-17T17:21:11.218 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.218 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.218 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:21:11.218 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.886349+0000", 2024-08-17T17:21:11.218 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000334555 2024-08-17T17:21:11.218 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.218 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.219 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.219 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.891760+0000", 2024-08-17T17:21:11.219 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0054106279999999998 2024-08-17T17:21:11.219 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.219 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.219 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.219 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.900574+0000", 2024-08-17T17:21:11.219 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0088143300000000004 2024-08-17T17:21:11.219 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.219 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.219 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.219 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.903791+0000", 2024-08-17T17:21:11.219 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0032164749999999999 2024-08-17T17:21:11.220 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.220 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.220 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.220 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.903846+0000", 2024-08-17T17:21:11.220 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5198000000000002e-05 2024-08-17T17:21:11.220 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.220 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.220 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.220 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.904289+0000", 2024-08-17T17:21:11.220 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00044295599999999999 2024-08-17T17:21:11.220 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.220 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.220 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.221 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.221 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.221 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186372 7.d 7:b527266b:::benchmark_data_smithi063_196002_object74279:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1172)", 2024-08-17T17:21:11.221 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.416634+0000", 2024-08-17T17:21:11.221 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 5.6610906669999999, 2024-08-17T17:21:11.221 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.260521416, 2024-08-17T17:21:11.221 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.221 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:21:11.221 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.221 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:21:11.221 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:21:11.221 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186372 2024-08-17T17:21:11.221 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.221 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.222 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.222 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.222 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416634+0000", 2024-08-17T17:21:11.222 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.222 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.222 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.222 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.222 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416634+0000", 2024-08-17T17:21:11.222 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.222 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.222 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.222 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.223 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416632+0000", 2024-08-17T17:21:11.223 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:21:11.223 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.223 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.223 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.223 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416640+0000", 2024-08-17T17:21:11.223 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4399999999999999e-06 2024-08-17T17:21:11.223 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.223 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.223 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.223 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416653+0000", 2024-08-17T17:21:11.223 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3383999999999999e-05 2024-08-17T17:21:11.224 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.224 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.224 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.224 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416661+0000", 2024-08-17T17:21:11.224 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3100000000000001e-06 2024-08-17T17:21:11.224 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.224 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.224 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.224 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416714+0000", 2024-08-17T17:21:11.224 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3134999999999999e-05 2024-08-17T17:21:11.224 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.224 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.224 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.224 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416894+0000", 2024-08-17T17:21:11.225 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017996599999999999 2024-08-17T17:21:11.225 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.225 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.225 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:21:11.225 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.417361+0000", 2024-08-17T17:21:11.225 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00046639499999999998 2024-08-17T17:21:11.225 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.225 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.225 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.225 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.432229+0000", 2024-08-17T17:21:11.225 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.014867811 2024-08-17T17:21:11.225 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.225 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.226 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.226 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.432244+0000", 2024-08-17T17:21:11.226 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5260999999999998e-05 2024-08-17T17:21:11.226 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.226 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.226 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.226 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.623931+0000", 2024-08-17T17:21:11.226 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1916872569999999 2024-08-17T17:21:11.226 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.226 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.226 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:21:11.226 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.624058+0000", 2024-08-17T17:21:11.226 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000127033 2024-08-17T17:21:11.227 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.227 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.227 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.227 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.677057+0000", 2024-08-17T17:21:11.227 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.052998521999999999 2024-08-17T17:21:11.227 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.227 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.227 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.227 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.677155+0000", 2024-08-17T17:21:11.227 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8443999999999995e-05 2024-08-17T17:21:11.227 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.227 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.227 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.228 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.228 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.228 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186392 7.d 7:b05cf35d:::benchmark_data_smithi063_196002_object74299:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1173)", 2024-08-17T17:21:11.228 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.464967+0000", 2024-08-17T17:21:11.228 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 5.6127575360000002, 2024-08-17T17:21:11.228 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2400478340000001, 2024-08-17T17:21:11.228 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.228 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.228 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.228 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:21:11.228 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:21:11.228 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186392 2024-08-17T17:21:11.228 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464967+0000", 2024-08-17T17:21:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464967+0000", 2024-08-17T17:21:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.229 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.230 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464966+0000", 2024-08-17T17:21:11.230 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:21:11.230 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.230 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.230 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.230 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464971+0000", 2024-08-17T17:21:11.230 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5700000000000003e-06 2024-08-17T17:21:11.230 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.230 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.230 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.230 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464979+0000", 2024-08-17T17:21:11.231 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.5019999999999992e-06 2024-08-17T17:21:11.231 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.231 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.231 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.231 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464985+0000", 2024-08-17T17:21:11.231 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6919999999999999e-06 2024-08-17T17:21:11.231 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.231 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.231 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.231 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.472207+0000", 2024-08-17T17:21:11.231 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0072221009999999999 2024-08-17T17:21:11.231 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.231 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.232 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.232 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.472220+0000", 2024-08-17T17:21:11.232 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.314e-05 2024-08-17T17:21:11.232 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.232 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.232 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.232 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.677176+0000", 2024-08-17T17:21:11.232 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.204956286 2024-08-17T17:21:11.232 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.232 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.232 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.232 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.677275+0000", 2024-08-17T17:21:11.232 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9032999999999999e-05 2024-08-17T17:21:11.233 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.233 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.233 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:21:11.233 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.677671+0000", 2024-08-17T17:21:11.233 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00039520700000000001 2024-08-17T17:21:11.233 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.233 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.233 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.233 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.682047+0000", 2024-08-17T17:21:11.233 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0043766289999999999 2024-08-17T17:21:11.233 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.233 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.234 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.234 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.686155+0000", 2024-08-17T17:21:11.234 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0041077830000000003 2024-08-17T17:21:11.234 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.234 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.234 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.234 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.704798+0000", 2024-08-17T17:21:11.234 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.018642549000000001 2024-08-17T17:21:11.234 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.234 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.234 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.234 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.704836+0000", 2024-08-17T17:21:11.235 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8770999999999999e-05 2024-08-17T17:21:11.235 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.235 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.235 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.235 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.705015+0000", 2024-08-17T17:21:11.235 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017840600000000001 2024-08-17T17:21:11.235 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.235 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.235 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.235 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.235 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.235 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186408 7.1d 7:bfe64f7d:::benchmark_data_smithi063_196002_object74315:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1173)", 2024-08-17T17:21:11.235 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.493415+0000", 2024-08-17T17:21:11.236 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 5.5843094310000003, 2024-08-17T17:21:11.236 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1831635359999999, 2024-08-17T17:21:11.236 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:21:11.236 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:11.236 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:21:11.236 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:21:11.236 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:21:11.236 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186408 2024-08-17T17:21:11.236 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.236 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:21:11.236 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.236 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:21:11.237 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493415+0000", 2024-08-17T17:21:11.238 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.238 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.238 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.238 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:21:11.238 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493415+0000", 2024-08-17T17:21:11.238 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:21:11.238 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.238 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.238 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:21:11.238 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493414+0000", 2024-08-17T17:21:11.238 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:21:11.238 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.239 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.239 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:21:11.239 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493419+0000", 2024-08-17T17:21:11.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4320000000000001e-06 2024-08-17T17:21:11.239 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.239 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.239 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:21:11.239 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493429+0000", 2024-08-17T17:21:11.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0558e-05 2024-08-17T17:21:11.239 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.239 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.239 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:21:11.240 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493436+0000", 2024-08-17T17:21:11.240 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4459999999999998e-06 2024-08-17T17:21:11.240 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.240 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.240 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.240 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493483+0000", 2024-08-17T17:21:11.240 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7105999999999999e-05 2024-08-17T17:21:11.240 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.240 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.240 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:21:11.240 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493491+0000", 2024-08-17T17:21:11.240 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4919999999999993e-06 2024-08-17T17:21:11.240 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.240 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.241 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:21:11.241 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.540092+0000", 2024-08-17T17:21:11.241 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.046600636 2024-08-17T17:21:11.241 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.241 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.241 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:21:11.241 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.540231+0000", 2024-08-17T17:21:11.241 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013884100000000001 2024-08-17T17:21:11.241 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.241 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.241 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:21:11.241 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.540568+0000", 2024-08-17T17:21:11.242 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00033741400000000002 2024-08-17T17:21:11.242 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.242 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.242 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.242 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.546007+0000", 2024-08-17T17:21:11.242 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.005438539 2024-08-17T17:21:11.242 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.242 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.242 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:21:11.242 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.546046+0000", 2024-08-17T17:21:11.242 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9446000000000001e-05 2024-08-17T17:21:11.242 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.242 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.243 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:21:11.243 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676197+0000", 2024-08-17T17:21:11.243 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.13015059400000001 2024-08-17T17:21:11.243 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.243 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.243 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:21:11.243 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676334+0000", 2024-08-17T17:21:11.243 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000137718 2024-08-17T17:21:11.243 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:21:11.243 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:21:11.243 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:21:11.243 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676579+0000", 2024-08-17T17:21:11.243 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00024415 2024-08-17T17:21:11.244 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.244 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.244 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.244 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:21:11.244 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:21:11.244 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:21:11.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.245 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:11.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:11.346 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:11.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.386 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:11.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:11.487 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:11.515 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:11.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:11.617 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:11.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.644 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:11.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:11.745 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:11.772 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:11.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:11.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:11 smithi063 ceph-mon[114756]: pgmap v1456: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+undersized+remapped, 2 peering, 1 activating, 1 remapped+peering, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 102 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 5884/111336 objects degraded (5.285%); 3.7 MiB/s, 505 objects/s recovering 2024-08-17T17:21:11.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:11 smithi063 ceph-mon[114756]: 7.1a scrub starts 2024-08-17T17:21:11.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:11 smithi063 ceph-mon[109843]: pgmap v1456: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+undersized+remapped, 2 peering, 1 activating, 1 remapped+peering, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 102 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 5884/111336 objects degraded (5.285%); 3.7 MiB/s, 505 objects/s recovering 2024-08-17T17:21:11.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:11 smithi063 ceph-mon[109843]: 7.1a scrub starts 2024-08-17T17:21:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:11 smithi162 ceph-mon[104668]: pgmap v1456: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+undersized+remapped, 2 peering, 1 activating, 1 remapped+peering, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 102 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 823 op/s; 5884/111336 objects degraded (5.285%); 3.7 MiB/s, 505 objects/s recovering 2024-08-17T17:21:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:11 smithi162 ceph-mon[104668]: 7.1a scrub starts 2024-08-17T17:21:11.874 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:11.902 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:11.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:11.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:12.004 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:12.030 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:21:12.040 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:12.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:12.141 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:12.170 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:12.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:12.272 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:12.307 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:12.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:12.352 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:21:12.357 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:21:12.358 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:21:12.402 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:21:12.402 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:21:12.408 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:12.435 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:12.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:12.438 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:21:12.438 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:21:12.477 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:21:12.478 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:21:12.513 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:21:12.514 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:21:12.536 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:12.548 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:21:12.548 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:21:12.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.581 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:12.581 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:12.582 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:21:12.582 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:21:12.616 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:21:12.617 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:21:12.681 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:12.720 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:12.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:12.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:12 smithi063 ceph-mon[109843]: 7.1e scrub starts 2024-08-17T17:21:12.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:12 smithi063 ceph-mon[109843]: 7.1a scrub ok 2024-08-17T17:21:12.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:12 smithi063 ceph-mon[109843]: 7.1e scrub ok 2024-08-17T17:21:12.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:12 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4228276824' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:12.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:12 smithi063 ceph-mon[114756]: 7.1e scrub starts 2024-08-17T17:21:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:12 smithi063 ceph-mon[114756]: 7.1a scrub ok 2024-08-17T17:21:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:12 smithi063 ceph-mon[114756]: 7.1e scrub ok 2024-08-17T17:21:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:12 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4228276824' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:12.822 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:12.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:12 smithi162 ceph-mon[104668]: 7.1e scrub starts 2024-08-17T17:21:12.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:12 smithi162 ceph-mon[104668]: 7.1a scrub ok 2024-08-17T17:21:12.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:12 smithi162 ceph-mon[104668]: 7.1e scrub ok 2024-08-17T17:21:12.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:12 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4228276824' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:12.861 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:12.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:12.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:12.897 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:21:12.897 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:21:12.962 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:13.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.012 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:13.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:13.114 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:13.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.162 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:13.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:13.263 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:13.288 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:13.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:13.369 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:13.391 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:13.418 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:13.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:13.516 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:21:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:21:13] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.51.0" 2024-08-17T17:21:13.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:13 smithi063 ceph-mon[114756]: pgmap v1457: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+undersized+remapped, 2 peering, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 104 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 5884/104508 objects degraded (5.630%) 2024-08-17T17:21:13.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:13 smithi063 ceph-mon[114756]: 7.1b scrub starts 2024-08-17T17:21:13.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:13 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 5884/104508 objects degraded (5.630%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:13.517 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:13 smithi063 ceph-mon[109843]: pgmap v1457: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+undersized+remapped, 2 peering, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 104 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 5884/104508 objects degraded (5.630%) 2024-08-17T17:21:13.517 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:13 smithi063 ceph-mon[109843]: 7.1b scrub starts 2024-08-17T17:21:13.517 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:13 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 5884/104508 objects degraded (5.630%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:13.520 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:13.559 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:13.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:13.586 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:21:13.661 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:13.694 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:13.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:13.713 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:13 smithi162 ceph-mon[104668]: pgmap v1457: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+undersized+remapped, 2 peering, 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 104 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 5884/104508 objects degraded (5.630%) 2024-08-17T17:21:13.713 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:13 smithi162 ceph-mon[104668]: 7.1b scrub starts 2024-08-17T17:21:13.713 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:13 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 5884/104508 objects degraded (5.630%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:13.796 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:13.820 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:13.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:13.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:13.976 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:13.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:13.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:14.078 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:14.120 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:14.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:14.222 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:14.395 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:14.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:14.497 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:14.527 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:14.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:14.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:14 smithi162 ceph-mon[104668]: 7.b scrub starts 2024-08-17T17:21:14.628 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:14.656 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:14.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:14.722 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:14 smithi063 ceph-mon[114756]: 7.b scrub starts 2024-08-17T17:21:14.722 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:14 smithi063 ceph-mon[109843]: 7.b scrub starts 2024-08-17T17:21:14.758 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:14.781 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:14.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.803 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:14.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:14.904 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:14.935 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:21:14.935 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:21:14.951 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:14.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:14.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:15.053 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:15.078 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:15.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:15.180 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:15.224 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:15.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:15.301 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:21:15.327 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:15.354 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:15.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:15.456 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:15.481 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:15.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:15.583 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:15.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:15 smithi162 ceph-mon[104668]: 7.e scrub starts 2024-08-17T17:21:15.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:15 smithi162 ceph-mon[104668]: pgmap v1458: 119 pgs: 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 109 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 5884/100662 objects degraded (5.845%) 2024-08-17T17:21:15.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:15 smithi162 ceph-mon[104668]: 7.e scrub ok 2024-08-17T17:21:15.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:15 smithi162 ceph-mon[104668]: 7.1f scrub starts 2024-08-17T17:21:15.614 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:15.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:15.664 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:21:15.717 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:15.753 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:15.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:15.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:15 smithi063 ceph-mon[114756]: 7.e scrub starts 2024-08-17T17:21:15.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:15 smithi063 ceph-mon[114756]: pgmap v1458: 119 pgs: 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 109 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 5884/100662 objects degraded (5.845%) 2024-08-17T17:21:15.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:15 smithi063 ceph-mon[114756]: 7.e scrub ok 2024-08-17T17:21:15.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:15 smithi063 ceph-mon[114756]: 7.1f scrub starts 2024-08-17T17:21:15.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:15 smithi063 ceph-mon[109843]: 7.e scrub starts 2024-08-17T17:21:15.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:15 smithi063 ceph-mon[109843]: pgmap v1458: 119 pgs: 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 109 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 5884/100662 objects degraded (5.845%) 2024-08-17T17:21:15.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:15 smithi063 ceph-mon[109843]: 7.e scrub ok 2024-08-17T17:21:15.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:15 smithi063 ceph-mon[109843]: 7.1f scrub starts 2024-08-17T17:21:15.854 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:15.893 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:15.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:15.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:15.981 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:21:15.996 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:16.036 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:16.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:16.137 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:16.181 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:16.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:16.282 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:16.314 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:16.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:16.416 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:16.440 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:16.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:16.542 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:16.571 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:16.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:16.673 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:16.702 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:16.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:16.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:16 smithi063 ceph-mon[114756]: 7.1f scrub ok 2024-08-17T17:21:16.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:16 smithi063 ceph-mon[114756]: 7.20 scrub starts 2024-08-17T17:21:16.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:16 smithi063 ceph-mon[114756]: 7.20 scrub ok 2024-08-17T17:21:16.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:16 smithi063 ceph-mon[114756]: 1.9 scrub starts 2024-08-17T17:21:16.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:16 smithi063 ceph-mon[114756]: 1.9 scrub ok 2024-08-17T17:21:16.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:16 smithi063 ceph-mon[109843]: 7.1f scrub ok 2024-08-17T17:21:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:16 smithi063 ceph-mon[109843]: 7.20 scrub starts 2024-08-17T17:21:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:16 smithi063 ceph-mon[109843]: 7.20 scrub ok 2024-08-17T17:21:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:16 smithi063 ceph-mon[109843]: 1.9 scrub starts 2024-08-17T17:21:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:16 smithi063 ceph-mon[109843]: 1.9 scrub ok 2024-08-17T17:21:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:16.804 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:16.833 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:16.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:16 smithi162 ceph-mon[104668]: 7.1f scrub ok 2024-08-17T17:21:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:16 smithi162 ceph-mon[104668]: 7.20 scrub starts 2024-08-17T17:21:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:16 smithi162 ceph-mon[104668]: 7.20 scrub ok 2024-08-17T17:21:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:16 smithi162 ceph-mon[104668]: 1.9 scrub starts 2024-08-17T17:21:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:16 smithi162 ceph-mon[104668]: 1.9 scrub ok 2024-08-17T17:21:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:16.935 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:16.986 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:16.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:16.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:17.087 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:17.132 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:17.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:17.234 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:17.278 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:17.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:17.380 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:17.406 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:17.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:17.507 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:17.514 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:21:17.546 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:17.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:17.648 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:17.678 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:17.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:17.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:17 smithi063 ceph-mon[114756]: pgmap v1459: 119 pgs: 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 5884/96459 objects degraded (6.100%) 2024-08-17T17:21:17.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:17 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3435480975' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:21:17.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:17 smithi063 ceph-mon[109843]: pgmap v1459: 119 pgs: 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 5884/96459 objects degraded (6.100%) 2024-08-17T17:21:17.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:17 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3435480975' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:21:17.780 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:17.815 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:17.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:17.829 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:17 smithi162 ceph-mon[104668]: pgmap v1459: 119 pgs: 1 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 5884/96459 objects degraded (6.100%) 2024-08-17T17:21:17.829 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:17 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3435480975' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:21:17.845 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:21:17.845 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:21:17.916 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:17.947 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:17.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:17.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:18.049 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:18.080 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:18.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:18.182 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:18.223 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:18.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:18.325 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:18.354 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:18.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:18.456 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:18.480 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:18.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:18.531 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:21:18.582 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:18.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.613 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:18.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:18.714 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:18.754 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:18.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:18.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:18 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3435480975' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:21:18.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:18 smithi063 ceph-mon[109843]: osdmap e1179: 8 total, 8 up, 8 in 2024-08-17T17:21:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:18 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 5884/96459 objects degraded (6.100%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:18 smithi063 ceph-mon[109843]: 1.2 scrub starts 2024-08-17T17:21:18.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:18 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3435480975' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:21:18.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:18 smithi063 ceph-mon[114756]: osdmap e1179: 8 total, 8 up, 8 in 2024-08-17T17:21:18.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:18 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 5884/96459 objects degraded (6.100%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:18.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:18 smithi063 ceph-mon[114756]: 1.2 scrub starts 2024-08-17T17:21:18.836 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:21:18.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:18 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3435480975' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:21:18.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:18 smithi162 ceph-mon[104668]: osdmap e1179: 8 total, 8 up, 8 in 2024-08-17T17:21:18.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:18 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 5884/96459 objects degraded (6.100%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:18.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:18 smithi162 ceph-mon[104668]: 1.2 scrub starts 2024-08-17T17:21:18.856 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:18.898 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:18.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:18.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:19.000 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:19.030 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:19.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:19.132 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:19.159 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:19.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:19.261 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:19.297 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:19.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:19.399 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:19.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.428 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:19.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:19.529 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:19.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.573 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:19.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:19.674 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:19.674 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:19 smithi162 ceph-mon[104668]: pgmap v1461: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3849/83289 objects degraded (4.621%) 2024-08-17T17:21:19.707 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:19.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:19.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:19 smithi063 ceph-mon[114756]: pgmap v1461: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3849/83289 objects degraded (4.621%) 2024-08-17T17:21:19.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:19 smithi063 ceph-mon[109843]: pgmap v1461: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.7 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 3849/83289 objects degraded (4.621%) 2024-08-17T17:21:19.810 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:19.836 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:19.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:19.938 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:19.984 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:19.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:19.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:20.086 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:20.113 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:20.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:20.215 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:20.255 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:20.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:20.357 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:20.379 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:20.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:20.481 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:20.513 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:20.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:20.615 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:20.643 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:20.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:20.745 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:20.772 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:20.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:20.873 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:20.902 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:20.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:20.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:20.994 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:21:20.995 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:21:21.005 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:21.042 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:21.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:21.144 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:21.205 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:21.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:21.307 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:21.334 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:21.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:21.436 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:21.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.463 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:21.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:21.564 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:21.590 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:21.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:21.675 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:21 smithi162 ceph-mon[104668]: pgmap v1462: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 3849/78852 objects degraded (4.881%) 2024-08-17T17:21:21.675 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:21 smithi162 ceph-mon[104668]: 7.16 scrub starts 2024-08-17T17:21:21.675 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:21 smithi162 ceph-mon[104668]: 7.16 scrub ok 2024-08-17T17:21:21.685 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:21 smithi063 ceph-mon[109843]: pgmap v1462: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 3849/78852 objects degraded (4.881%) 2024-08-17T17:21:21.685 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:21 smithi063 ceph-mon[109843]: 7.16 scrub starts 2024-08-17T17:21:21.685 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:21 smithi063 ceph-mon[109843]: 7.16 scrub ok 2024-08-17T17:21:21.685 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:21 smithi063 ceph-mon[114756]: pgmap v1462: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 3849/78852 objects degraded (4.881%) 2024-08-17T17:21:21.686 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:21 smithi063 ceph-mon[114756]: 7.16 scrub starts 2024-08-17T17:21:21.686 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:21 smithi063 ceph-mon[114756]: 7.16 scrub ok 2024-08-17T17:21:21.692 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:21.708 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:21:21.722 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:21.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:21.824 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:21.878 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:21.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:21.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:21.980 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:22.006 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:22.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:22.108 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:22.146 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:21:22.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.156 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:22.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:22.257 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:22.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.318 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:22.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:22.390 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:22.419 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:22.442 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:22.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.443 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:22.483 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:22.544 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:22.610 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:22.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:22.711 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:22.733 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:22.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:22.834 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:22.861 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:22.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:22.963 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:22.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:22.990 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:22.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:23.091 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:23.113 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:23.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:23.214 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:23.240 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:23.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:23.342 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:23.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.398 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:23.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:23.499 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:23.524 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:23.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:23.552 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:21:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:21:23] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.51.0" 2024-08-17T17:21:23.569 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:21:23.626 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:23.673 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:23.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:23.774 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:23.814 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:23.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:23 smithi162 ceph-mon[104668]: pgmap v1463: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 3849/70185 objects degraded (5.484%) 2024-08-17T17:21:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:23 smithi162 ceph-mon[104668]: 7.1b scrub starts 2024-08-17T17:21:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:23 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/559010901' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:23 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 3849/70185 objects degraded (5.484%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:23.895 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:23 smithi063 ceph-mon[114756]: pgmap v1463: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 3849/70185 objects degraded (5.484%) 2024-08-17T17:21:23.895 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:23 smithi063 ceph-mon[114756]: 7.1b scrub starts 2024-08-17T17:21:23.895 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:23 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/559010901' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:23.895 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:23 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 3849/70185 objects degraded (5.484%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:23.895 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:23 smithi063 ceph-mon[109843]: pgmap v1463: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.4 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 3849/70185 objects degraded (5.484%) 2024-08-17T17:21:23.895 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:23 smithi063 ceph-mon[109843]: 7.1b scrub starts 2024-08-17T17:21:23.896 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:23 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/559010901' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:23.896 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:23 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 3849/70185 objects degraded (5.484%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:23.915 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:23.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:23.965 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:23.965 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:24.066 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:24.115 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:24.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:24.216 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:24.242 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:24.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:24.344 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:24.382 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:24.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:24.483 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:24.525 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:24.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:24.626 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:24.667 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:24.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:24.685 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:24 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:21:24.685 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:24 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/559010901' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:24.685 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:24 smithi063 ceph-mon[114756]: osdmap e1180: 8 total, 8 up, 8 in 2024-08-17T17:21:24.685 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:24 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:21:24.685 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:24 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/559010901' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:24.685 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:24 smithi063 ceph-mon[109843]: osdmap e1180: 8 total, 8 up, 8 in 2024-08-17T17:21:24.769 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:24.820 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:24.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:24.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:24 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:21:24.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:24 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/559010901' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:24.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:24 smithi162 ceph-mon[104668]: osdmap e1180: 8 total, 8 up, 8 in 2024-08-17T17:21:24.921 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:24.946 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:24.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:24.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:25.047 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:25.074 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:25.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:25.176 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:25.201 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:25.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:25.303 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:25.347 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:25.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:25.449 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:25.490 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:25.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:25.592 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:25.634 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:25.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:25.736 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:25.764 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:25.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:25.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:25 smithi162 ceph-mon[104668]: pgmap v1465: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+clean+scrubbing, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.3 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 3849/65841 objects degraded (5.846%) 2024-08-17T17:21:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:25 smithi162 ceph-mon[104668]: 7.15 scrub starts 2024-08-17T17:21:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:25 smithi162 ceph-mon[104668]: 7.15 scrub ok 2024-08-17T17:21:25.852 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:25 smithi063 ceph-mon[109843]: pgmap v1465: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+clean+scrubbing, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.3 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 3849/65841 objects degraded (5.846%) 2024-08-17T17:21:25.852 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:25 smithi063 ceph-mon[109843]: 7.15 scrub starts 2024-08-17T17:21:25.852 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:25 smithi063 ceph-mon[109843]: 7.15 scrub ok 2024-08-17T17:21:25.852 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:25 smithi063 ceph-mon[114756]: pgmap v1465: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+clean+scrubbing, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.3 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 3849/65841 objects degraded (5.846%) 2024-08-17T17:21:25.852 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:25 smithi063 ceph-mon[114756]: 7.15 scrub starts 2024-08-17T17:21:25.852 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:25 smithi063 ceph-mon[114756]: 7.15 scrub ok 2024-08-17T17:21:25.866 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:25.907 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:25.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:25.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:26.009 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:26.048 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:26.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:26.150 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:26.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.192 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:26.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:26.293 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:26.323 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:26.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:26.424 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:26.464 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:26.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:26.565 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:26.606 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:26.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:26.643 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:26 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/84756897' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:26.644 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:26 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/84756897' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:26.708 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:26.756 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:26.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:26.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:26 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/84756897' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:26.858 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:26.896 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:26.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:26.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:26.953 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:21:26.953 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:21:26.953 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:21:26.954 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:21:26.954 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:21:26.954 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:21:26.954 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:21:26.954 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:21:26.954 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:21:26.955 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:21:26.955 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:21:26.955 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:21:26.955 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:21:26.955 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:21:26.955 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:21:26.955 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:21:26.998 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:27.025 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:27.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:27.127 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:27.162 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:27.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.174 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:27.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:27.275 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:27.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.315 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:27.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:27.416 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:27.443 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:27.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.444 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:27.545 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:27.573 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:27.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:27.675 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:27.738 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:27.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:27.839 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:27 smithi063 ceph-mon[109843]: pgmap v1466: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 91 op/s; 3849/61899 objects degraded (6.218%) 2024-08-17T17:21:27.839 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:27 smithi063 ceph-mon[109843]: 7.b scrub starts 2024-08-17T17:21:27.839 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:27 smithi063 ceph-mon[114756]: pgmap v1466: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 91 op/s; 3849/61899 objects degraded (6.218%) 2024-08-17T17:21:27.839 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:27 smithi063 ceph-mon[114756]: 7.b scrub starts 2024-08-17T17:21:27.841 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:27.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:27 smithi162 ceph-mon[104668]: pgmap v1466: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 1.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 91 op/s; 3849/61899 objects degraded (6.218%) 2024-08-17T17:21:27.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:27 smithi162 ceph-mon[104668]: 7.b scrub starts 2024-08-17T17:21:27.867 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:27.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:27.968 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:27.995 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:27.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:27.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:28.097 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:28.124 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:28.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:28.226 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:28.279 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:28.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:28.381 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:28.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.420 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:28.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:28.432 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:21:28.522 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:28.577 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:28.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:28.679 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:28.703 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:28.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:28.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:28 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 3849/61899 objects degraded (6.218%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:28.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:28 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 3849/61899 objects degraded (6.218%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:28.805 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:28.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:28 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 3849/61899 objects degraded (6.218%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:28.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.855 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:28.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:28.956 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:28.996 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:28.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:28.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:29.097 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:29.141 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:29.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:29.243 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:29.268 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:29.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:29.370 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:29.418 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:29.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:29.520 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:29.548 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:29.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:29.650 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:29.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.692 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:29.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:29.720 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:21:29.742 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:29 smithi063 ceph-mon[114756]: pgmap v1467: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3849/50214 objects degraded (7.665%) 2024-08-17T17:21:29.743 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:29 smithi063 ceph-mon[109843]: pgmap v1467: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3849/50214 objects degraded (7.665%) 2024-08-17T17:21:29.793 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:29.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.834 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:29.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:29.935 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:29.967 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:29.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:29.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:30.069 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:29 smithi162 ceph-mon[104668]: pgmap v1467: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 1.0 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3849/50214 objects degraded (7.665%) 2024-08-17T17:21:30.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.100 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:30.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:30.201 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:30.236 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:30.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:30.338 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:30.394 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:30.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:30.450 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:21:30.450 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:21:30.450 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.450 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:211762 7.d e1180/1173 7:b3fa7423:::benchmark_data_smithi063_196002_object99669:head v 1180'11975, mlcod=1158'8977)", 2024-08-17T17:21:30.450 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:21:30.440766+0000", 2024-08-17T17:21:30.450 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.006793543, 2024-08-17T17:21:30.450 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0067998279999999999, 2024-08-17T17:21:30.450 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:21:30.451 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:21:30.451 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:21:30.451 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:21:30.451 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.451 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:21:30.451 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.440766+0000", 2024-08-17T17:21:30.451 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:21:30.451 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.451 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.451 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:21:30.451 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.440766+0000", 2024-08-17T17:21:30.451 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:21:30.452 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.452 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.452 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:21:30.452 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.440767+0000", 2024-08-17T17:21:30.452 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5499999999999998e-07 2024-08-17T17:21:30.452 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.452 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.452 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:21:30.452 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.440773+0000", 2024-08-17T17:21:30.453 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4840000000000001e-06 2024-08-17T17:21:30.453 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.453 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.453 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:21:30.453 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.440787+0000", 2024-08-17T17:21:30.453 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4192999999999999e-05 2024-08-17T17:21:30.453 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.453 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.453 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:21:30.453 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.440794+0000", 2024-08-17T17:21:30.453 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7669999999999998e-06 2024-08-17T17:21:30.453 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.453 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.454 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:21:30.454 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.442049+0000", 2024-08-17T17:21:30.454 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0012548780000000001 2024-08-17T17:21:30.454 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.454 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.454 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:21:30.454 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.442072+0000", 2024-08-17T17:21:30.454 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2614000000000001e-05 2024-08-17T17:21:30.454 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.454 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.454 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:21:30.455 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.447542+0000", 2024-08-17T17:21:30.455 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0054702270000000002 2024-08-17T17:21:30.455 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.455 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.455 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:21:30.455 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.447566+0000", 2024-08-17T17:21:30.455 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4009999999999999e-05 2024-08-17T17:21:30.455 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:21:30.455 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:21:30.455 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:21:30.455 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.455 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.455 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:211763 7.1f e1180/1163 7:fd305992:::benchmark_data_smithi063_196002_object99670:head v 1180'11989, mlcod=1180'11987)", 2024-08-17T17:21:30.456 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:21:30.444744+0000", 2024-08-17T17:21:30.456 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0028159779999999998, 2024-08-17T17:21:30.456 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0029238129999999999, 2024-08-17T17:21:30.456 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:21:30.456 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:21:30.456 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:21:30.456 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:21:30.456 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.456 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:21:30.456 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.444744+0000", 2024-08-17T17:21:30.456 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:21:30.456 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.456 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.457 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:21:30.457 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.444744+0000", 2024-08-17T17:21:30.457 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:21:30.457 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.457 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.457 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:21:30.457 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.444744+0000", 2024-08-17T17:21:30.457 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0699999999999996e-07 2024-08-17T17:21:30.457 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.457 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.457 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:21:30.457 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.444751+0000", 2024-08-17T17:21:30.457 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6059999999999999e-06 2024-08-17T17:21:30.457 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.458 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.458 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:21:30.458 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.444767+0000", 2024-08-17T17:21:30.458 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5617999999999999e-05 2024-08-17T17:21:30.458 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.458 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.458 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:21:30.458 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.444774+0000", 2024-08-17T17:21:30.458 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7000000000000008e-06 2024-08-17T17:21:30.458 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.458 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.458 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:21:30.459 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.444849+0000", 2024-08-17T17:21:30.459 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4733000000000004e-05 2024-08-17T17:21:30.459 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.459 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.459 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:21:30.459 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.444870+0000", 2024-08-17T17:21:30.459 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1041999999999999e-05 2024-08-17T17:21:30.459 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:21:30.459 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:21:30.459 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:21:30.459 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.459 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.459 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:211764 7.17 e1180/1151)", 2024-08-17T17:21:30.459 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:21:30.447643+0000", 2024-08-17T17:21:30.460 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 4294967295.9999161, 2024-08-17T17:21:30.460 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5261999999999999e-05, 2024-08-17T17:21:30.460 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:21:30.460 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:21:30.460 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "queued for pg", 2024-08-17T17:21:30.460 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:21:30.460 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.460 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:21:30.460 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.447643+0000", 2024-08-17T17:21:30.460 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:21:30.460 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.460 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.460 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:21:30.461 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.447643+0000", 2024-08-17T17:21:30.461 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:21:30.461 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.461 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.461 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:21:30.461 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.447644+0000", 2024-08-17T17:21:30.461 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3500000000000005e-07 2024-08-17T17:21:30.461 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.461 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.461 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:21:30.461 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.447651+0000", 2024-08-17T17:21:30.461 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6590000000000001e-06 2024-08-17T17:21:30.461 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.461 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.462 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:21:30.462 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.447662+0000", 2024-08-17T17:21:30.462 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0691e-05 2024-08-17T17:21:30.462 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.462 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.462 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:21:30.462 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.447669+0000", 2024-08-17T17:21:30.462 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2080000000000003e-06 2024-08-17T17:21:30.462 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:21:30.462 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:21:30.462 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:21:30.462 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.462 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.463 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:211767 7.17 e1180/1151)", 2024-08-17T17:21:30.463 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:21:30.447705+0000", 2024-08-17T17:21:30.463 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 4294967295.999855, 2024-08-17T17:21:30.463 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3276999999999998e-05, 2024-08-17T17:21:30.463 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:21:30.463 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:21:30.463 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "queued for pg", 2024-08-17T17:21:30.463 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:21:30.463 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.463 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:21:30.463 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.447705+0000", 2024-08-17T17:21:30.463 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:21:30.463 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.463 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.464 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:21:30.464 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.447705+0000", 2024-08-17T17:21:30.464 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:21:30.464 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.464 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.464 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:21:30.464 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.447705+0000", 2024-08-17T17:21:30.464 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0899999999999997e-07 2024-08-17T17:21:30.464 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.464 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.464 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:21:30.464 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.447709+0000", 2024-08-17T17:21:30.464 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8970000000000001e-06 2024-08-17T17:21:30.464 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.465 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.465 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:21:30.465 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.447718+0000", 2024-08-17T17:21:30.465 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0410000000000003e-06 2024-08-17T17:21:30.465 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:21:30.465 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:21:30.465 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:21:30.465 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:30.447723+0000", 2024-08-17T17:21:30.465 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7400000000000004e-06 2024-08-17T17:21:30.465 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:21:30.465 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:21:30.465 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:21:30.465 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:21:30.465 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-08-17T17:21:30.466 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 4 2024-08-17T17:21:30.466 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:21:30.496 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:30.525 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:30.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:30.626 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:30.671 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:30.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:30.773 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:30.801 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:30.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:30.903 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:30.944 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:30.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:30.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:30.995 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:30 smithi063 ceph-mon[109843]: 7.1b deep-scrub starts 2024-08-17T17:21:30.996 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:30 smithi063 ceph-mon[114756]: 7.1b deep-scrub starts 2024-08-17T17:21:31.045 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:31.093 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:31.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:31.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:30 smithi162 ceph-mon[104668]: 7.1b deep-scrub starts 2024-08-17T17:21:31.196 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:31.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.251 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:31.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:31.352 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:31.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.380 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:31.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:31.481 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:31.510 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:31.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:31.613 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:31.644 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:31.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.645 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:31.733 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:21:31.746 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:31.812 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:31.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:31.914 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:31.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:31.940 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:31.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:31 smithi063 ceph-mon[109843]: pgmap v1468: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 955 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 3849/45870 objects degraded (8.391%) 2024-08-17T17:21:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:31 smithi063 ceph-mon[109843]: 7.5 deep-scrub starts 2024-08-17T17:21:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:21:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:32.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:31 smithi063 ceph-mon[114756]: pgmap v1468: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 955 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 3849/45870 objects degraded (8.391%) 2024-08-17T17:21:32.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:31 smithi063 ceph-mon[114756]: 7.5 deep-scrub starts 2024-08-17T17:21:32.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:21:32.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:32.040 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:32.086 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:32.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:31 smithi162 ceph-mon[104668]: pgmap v1468: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 955 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 3849/45870 objects degraded (8.391%) 2024-08-17T17:21:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:31 smithi162 ceph-mon[104668]: 7.5 deep-scrub starts 2024-08-17T17:21:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:21:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:32.188 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:32.236 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:32.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:32.338 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:32.345 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:32.350 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:32.364 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:32.405 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:32.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:32.508 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:32.536 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:32.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:32.638 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:32.677 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:32.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:32.693 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:32 smithi063 ceph-mon[109843]: 7.b deep-scrub starts 2024-08-17T17:21:32.693 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:32 smithi063 ceph-mon[114756]: 7.b deep-scrub starts 2024-08-17T17:21:32.778 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:32.808 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:32.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:32.911 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:32.954 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:32.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:32.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:33.055 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:33.084 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:33.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:32 smithi162 ceph-mon[104668]: 7.b deep-scrub starts 2024-08-17T17:21:33.186 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:33.214 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:33.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:33.316 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:33.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.357 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:33.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:33.458 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:33.487 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:33.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:33.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:21:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:21:33] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.51.0" 2024-08-17T17:21:33.589 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:33.617 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:33.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:33.633 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:21:33.720 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:33.758 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:33.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:33.860 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:33.893 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:33.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:33.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:33.995 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:33.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:33 smithi162 ceph-mon[104668]: pgmap v1469: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 786 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 3849/37788 objects degraded (10.186%) 2024-08-17T17:21:33.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:33 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 3849/37788 objects degraded (10.186%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:33.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:33 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3298632258' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:21:33.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:33 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:21:34.024 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:33 smithi063 ceph-mon[114756]: pgmap v1469: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 786 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 3849/37788 objects degraded (10.186%) 2024-08-17T17:21:34.024 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:33 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 3849/37788 objects degraded (10.186%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:34.024 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:33 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3298632258' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:21:34.025 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:33 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:21:34.025 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:33 smithi063 ceph-mon[109843]: pgmap v1469: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 786 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 3849/37788 objects degraded (10.186%) 2024-08-17T17:21:34.025 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:33 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 3849/37788 objects degraded (10.186%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:34.025 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:33 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3298632258' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:21:34.025 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:33 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:21:34.033 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:34.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:34.135 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:34.167 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:34.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:34.269 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:34.296 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:34.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:34.398 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:34.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.435 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:34.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:34.537 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:34.576 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:34.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:34.678 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:34.711 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:34.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:34.813 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:34.844 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:34.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:34.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:34 smithi162 ceph-mon[104668]: 7.1b deep-scrub starts 2024-08-17T17:21:34.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:34 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:21:34.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:34 smithi162 ceph-mon[104668]: osdmap e1181: 8 total, 8 up, 8 in 2024-08-17T17:21:34.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:34 smithi162 ceph-mon[104668]: 7.5 deep-scrub ok 2024-08-17T17:21:34.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:34 smithi162 ceph-mon[104668]: 7.6 scrub starts 2024-08-17T17:21:34.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:34 smithi162 ceph-mon[104668]: 7.6 scrub ok 2024-08-17T17:21:34.945 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:34.987 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:34.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:34.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:35.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:34 smithi063 ceph-mon[114756]: 7.1b deep-scrub starts 2024-08-17T17:21:35.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:34 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:21:35.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:34 smithi063 ceph-mon[114756]: osdmap e1181: 8 total, 8 up, 8 in 2024-08-17T17:21:35.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:34 smithi063 ceph-mon[114756]: 7.5 deep-scrub ok 2024-08-17T17:21:35.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:34 smithi063 ceph-mon[114756]: 7.6 scrub starts 2024-08-17T17:21:35.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:34 smithi063 ceph-mon[114756]: 7.6 scrub ok 2024-08-17T17:21:35.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:34 smithi063 ceph-mon[109843]: 7.1b deep-scrub starts 2024-08-17T17:21:35.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:34 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:21:35.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:34 smithi063 ceph-mon[109843]: osdmap e1181: 8 total, 8 up, 8 in 2024-08-17T17:21:35.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:34 smithi063 ceph-mon[109843]: 7.5 deep-scrub ok 2024-08-17T17:21:35.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:34 smithi063 ceph-mon[109843]: 7.6 scrub starts 2024-08-17T17:21:35.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:34 smithi063 ceph-mon[109843]: 7.6 scrub ok 2024-08-17T17:21:35.089 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:35.092 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:21:35.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.117 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:35.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:35.218 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:35.249 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:35.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:35.351 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:35.400 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:35.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:35.501 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:35.540 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:35.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:35.641 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:35.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.675 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:35.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:35.776 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:35.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.819 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:35.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:35.900 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:35 smithi063 ceph-mon[109843]: pgmap v1471: 119 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 702 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 3849/33720 objects degraded (11.415%) 2024-08-17T17:21:35.900 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:35 smithi063 ceph-mon[109843]: 1.2 deep-scrub starts 2024-08-17T17:21:35.901 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:35 smithi063 ceph-mon[114756]: pgmap v1471: 119 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 702 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 3849/33720 objects degraded (11.415%) 2024-08-17T17:21:35.901 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:35 smithi063 ceph-mon[114756]: 1.2 deep-scrub starts 2024-08-17T17:21:35.920 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:35.958 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:35.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:35.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:36.060 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:35 smithi162 ceph-mon[104668]: pgmap v1471: 119 pgs: 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 702 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 3849/33720 objects degraded (11.415%) 2024-08-17T17:21:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:35 smithi162 ceph-mon[104668]: 1.2 deep-scrub starts 2024-08-17T17:21:36.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.097 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:36.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:36.198 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:36.225 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:36.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:36.326 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:36.354 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:36.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:36.456 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:36.497 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:36.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:36.599 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:36.629 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:36.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:36.668 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:21:36.668 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:21:36.668 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:21:36.668 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:21:36.668 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:21:36.731 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:36.773 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:36.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:36.875 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:36.925 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:36.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:36.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:37.027 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:37.057 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:37.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:37.159 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:37.189 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:37.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:37.291 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:37.319 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:37.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:37.420 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:37.473 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:37.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:37.575 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:37.603 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:37.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:37.672 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:37 smithi063 ceph-mon[109843]: pgmap v1472: 119 pgs: 1 active+recovery_wait+degraded, 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 622 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 3849/29898 objects degraded (12.874%) 2024-08-17T17:21:37.672 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:37 smithi063 ceph-mon[109843]: 7.b scrub starts 2024-08-17T17:21:37.705 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:37.734 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:37.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:37.836 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:37.867 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:37.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:37.968 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:37.993 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:37.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:37.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:37 smithi063 ceph-mon[114756]: pgmap v1472: 119 pgs: 1 active+recovery_wait+degraded, 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 622 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 3849/29898 objects degraded (12.874%) 2024-08-17T17:21:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:37 smithi063 ceph-mon[114756]: 7.b scrub starts 2024-08-17T17:21:38.095 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:37 smithi162 ceph-mon[104668]: pgmap v1472: 119 pgs: 1 active+recovery_wait+degraded, 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 622 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 3849/29898 objects degraded (12.874%) 2024-08-17T17:21:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:37 smithi162 ceph-mon[104668]: 7.b scrub starts 2024-08-17T17:21:38.102 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:21:38.126 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:38.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:38.228 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:38.256 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:38.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:38.358 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:38.418 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:38.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:38.520 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:38.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:38.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:38.648 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:38.679 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:38.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:38.781 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:38.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.843 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:38.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:38.944 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:38.964 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:38 smithi063 ceph-mon[114756]: 7.1b deep-scrub starts 2024-08-17T17:21:38.964 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:38 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 3849/29898 objects degraded (12.874%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:38.964 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:38 smithi063 ceph-mon[114756]: 1.2 scrub starts 2024-08-17T17:21:38.964 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:38 smithi063 ceph-mon[109843]: 7.1b deep-scrub starts 2024-08-17T17:21:38.964 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:38 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 3849/29898 objects degraded (12.874%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:38.964 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:38 smithi063 ceph-mon[109843]: 1.2 scrub starts 2024-08-17T17:21:38.994 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:38.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:38.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:39.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:38 smithi162 ceph-mon[104668]: 7.1b deep-scrub starts 2024-08-17T17:21:39.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:38 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 3849/29898 objects degraded (12.874%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:38 smithi162 ceph-mon[104668]: 1.2 scrub starts 2024-08-17T17:21:39.096 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:39.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.122 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:39.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:39.223 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:39.247 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:39.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:39.349 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:39.395 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:39.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:39.496 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:39.534 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:39.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:39.638 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:39.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.689 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:39.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:39.696 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:39.698 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:39.730 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:21:39.789 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:39.807 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:21:39.807 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:21:39.846 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:39.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:39.850 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:39 smithi063 ceph-mon[109843]: pgmap v1473: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+clean+scrubbing+deep, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 372 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 3849/17910 objects degraded (21.491%) 2024-08-17T17:21:39.850 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:39 smithi063 ceph-mon[109843]: 1.2 deep-scrub starts 2024-08-17T17:21:39.851 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:39 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1186659670' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:39.851 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:39 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:39.851 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:39 smithi063 ceph-mon[114756]: pgmap v1473: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+clean+scrubbing+deep, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 372 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 3849/17910 objects degraded (21.491%) 2024-08-17T17:21:39.851 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:39 smithi063 ceph-mon[114756]: 1.2 deep-scrub starts 2024-08-17T17:21:39.851 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:39 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1186659670' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:39.851 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:39 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:39.851 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:21:39.852 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:21:39.891 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:21:39.891 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:21:39.932 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:21:39.932 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:21:39.949 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:39.968 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:21:39.969 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:21:39.975 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:39.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:39.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:40.003 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:21:40.003 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:21:40.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:21:40.038 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:21:40.073 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:21:40.074 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:21:40.076 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:39 smithi162 ceph-mon[104668]: pgmap v1473: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+clean+scrubbing+deep, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 372 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 3849/17910 objects degraded (21.491%) 2024-08-17T17:21:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:39 smithi162 ceph-mon[104668]: 1.2 deep-scrub starts 2024-08-17T17:21:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:39 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1186659670' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:40.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:39 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:21:40.120 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:40.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:40.222 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:40.251 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:40.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:40.352 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:40.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.394 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:40.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:40.495 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:40.539 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:40.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:40.641 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:40.669 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:40.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:40.771 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:40.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:40 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:40.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:40 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:21:40.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:40 smithi063 ceph-mon[109843]: osdmap e1182: 8 total, 8 up, 8 in 2024-08-17T17:21:40.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:40 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3095156179' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:40.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:40 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:40.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:40 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:21:40.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:40 smithi063 ceph-mon[114756]: osdmap e1182: 8 total, 8 up, 8 in 2024-08-17T17:21:40.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:40 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3095156179' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:40.799 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:40.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:40.900 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:40.939 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:40.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:40.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:41.040 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:41.067 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:41.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:41.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:40 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:40 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:21:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:40 smithi162 ceph-mon[104668]: osdmap e1182: 8 total, 8 up, 8 in 2024-08-17T17:21:41.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:40 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3095156179' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:41.169 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:41.202 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:41.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:41.304 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:41.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.346 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:41.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:41.448 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:41.487 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:41.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:41.589 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:41.628 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:41.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:41.730 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:41.757 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:41.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:41.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:41 smithi063 ceph-mon[114756]: pgmap v1475: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 289 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 3849/13923 objects degraded (27.645%) 2024-08-17T17:21:41.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:41 smithi063 ceph-mon[109843]: pgmap v1475: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 289 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 3849/13923 objects degraded (27.645%) 2024-08-17T17:21:41.859 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:41.886 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:41.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:41.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:41.988 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:42.030 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:42.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:42.091 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Removed 112090 objects 2024-08-17T17:21:42.092 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Clean up completed and total clean up time :132.628 2024-08-17T17:21:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:41 smithi162 ceph-mon[104668]: pgmap v1475: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing+deep, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 289 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 3849/13923 objects degraded (27.645%) 2024-08-17T17:21:42.132 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:42.153 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:42.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:42.255 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:42.278 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:42.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:42.380 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:42.401 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:42.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:42.503 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:42.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.525 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:42.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:42.626 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:42.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.648 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:42.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:42.749 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:42.769 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:42.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:42.871 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:42.892 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:42.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:42.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:42.994 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:43.016 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:43.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:43.118 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:43.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.142 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:43.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:43.243 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:43.266 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:43.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:43.367 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:43.390 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:43.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:43.492 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:43.519 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:43.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:43.600 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:21:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:21:43] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.51.0" 2024-08-17T17:21:43.621 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:43.642 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:43.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:43.744 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:43.767 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:43.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:43.869 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:43.891 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:43.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:43.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:43.942 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:43 smithi063 ceph-mon[114756]: pgmap v1476: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+clean+scrubbing+deep, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 137 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3849/6603 objects degraded (58.292%) 2024-08-17T17:21:43.942 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:43 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 3849/6603 objects degraded (58.292%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:43.942 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:43 smithi063 ceph-mon[109843]: pgmap v1476: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+clean+scrubbing+deep, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 137 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3849/6603 objects degraded (58.292%) 2024-08-17T17:21:43.942 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:43 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 3849/6603 objects degraded (58.292%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:43.993 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:44.016 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:44.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:44.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:43 smithi162 ceph-mon[104668]: pgmap v1476: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+clean+scrubbing+deep, 7 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 137 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 3849/6603 objects degraded (58.292%) 2024-08-17T17:21:44.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:43 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 3849/6603 objects degraded (58.292%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:44.117 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:44.139 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:44.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:44.191 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:21:44.192 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-08-17T17:21:44.192 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-08-17T17:21:44.241 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:44.262 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:44.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:44.364 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:44.385 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:44.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:44.487 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:44.508 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:44.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:44.610 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:44.632 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:44.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:44.733 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:44.756 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:44.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:44.858 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:44.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:44.880 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:44.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:44.981 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:45.115 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:44 smithi162 ceph-mon[104668]: osdmap e1183: 8 total, 8 up, 8 in 2024-08-17T17:21:45.115 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:21:45.116 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:44 smithi063 ceph-mon[114756]: osdmap e1183: 8 total, 8 up, 8 in 2024-08-17T17:21:45.116 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:44 smithi063 ceph-mon[109843]: osdmap e1183: 8 total, 8 up, 8 in 2024-08-17T17:21:45.138 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:45.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:45.241 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:45.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.264 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:45.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:45.365 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:45.388 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:45.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:45.490 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:45.512 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:45.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:45.614 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:45.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.636 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:45.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:45.666 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:21:45.666 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:21:45.737 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:45.743 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:21:45.761 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:45.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:45.863 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:45.886 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:45 smithi063 ceph-mon[109843]: pgmap v1478: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 56 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3849/2745 objects degraded (140.219%) 2024-08-17T17:21:45.887 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:45 smithi063 ceph-mon[109843]: osdmap e1184: 8 total, 8 up, 8 in 2024-08-17T17:21:45.887 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:45 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1240633911' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:21:45.887 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:45 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:21:45.887 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:45 smithi063 ceph-mon[114756]: pgmap v1478: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 56 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3849/2745 objects degraded (140.219%) 2024-08-17T17:21:45.887 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:45 smithi063 ceph-mon[114756]: osdmap e1184: 8 total, 8 up, 8 in 2024-08-17T17:21:45.887 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:45 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1240633911' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:21:45.887 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:45 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:21:45.888 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:45.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:45.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:45.990 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:46.012 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:46.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:46.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:45 smithi162 ceph-mon[104668]: pgmap v1478: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 56 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 3849/2745 objects degraded (140.219%) 2024-08-17T17:21:46.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:45 smithi162 ceph-mon[104668]: osdmap e1184: 8 total, 8 up, 8 in 2024-08-17T17:21:46.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:45 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1240633911' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:21:46.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:45 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:21:46.115 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:46.139 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:46.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:46.240 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:46.264 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:46.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:46.366 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:46.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.390 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:46.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:46.491 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:46.513 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:46.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:46.615 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:46.637 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:46.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:46.650 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:46.650 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:46.677 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:21:46.739 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:46.783 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:46.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:46.886 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:46.909 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:46.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:46.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:47.011 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:46 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:21:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:46 smithi063 ceph-mon[109843]: osdmap e1185: 8 total, 8 up, 8 in 2024-08-17T17:21:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:21:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:46 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:21:47.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:46 smithi063 ceph-mon[114756]: osdmap e1185: 8 total, 8 up, 8 in 2024-08-17T17:21:47.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:21:47.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:47.032 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:47.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:47.060 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:21:47.074 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:46 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:21:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:46 smithi162 ceph-mon[104668]: osdmap e1185: 8 total, 8 up, 8 in 2024-08-17T17:21:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:21:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:21:47.134 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:47.157 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:47.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:47.259 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:47.282 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:47.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:47.384 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:47.407 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:47.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:47.510 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:47.535 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:47.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:47.637 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:47.663 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:47.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:47.765 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:47.786 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:47.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:47.888 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:47.909 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:47.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:47.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:48.011 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:47 smithi063 ceph-mon[114756]: pgmap v1481: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 25 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 3849/1260 objects degraded (305.476%); 0 B/s, 55 objects/s recovering 2024-08-17T17:21:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:21:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:47 smithi063 ceph-mon[109843]: pgmap v1481: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 25 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 3849/1260 objects degraded (305.476%); 0 B/s, 55 objects/s recovering 2024-08-17T17:21:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:21:48.032 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:48.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:47 smithi162 ceph-mon[104668]: pgmap v1481: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 25 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 764 op/s; 3849/1260 objects degraded (305.476%); 0 B/s, 55 objects/s recovering 2024-08-17T17:21:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:21:48.134 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:48.155 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:48.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:48.257 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:48.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.279 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:48.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:48.380 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:48.403 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:48.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:48.504 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:48.527 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:48.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:48.629 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:48.651 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:48.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:48.752 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:48.778 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:48.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:48.880 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:48.905 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:48.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:48.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:49.007 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:48 smithi063 ceph-mon[114756]: osdmap e1186: 8 total, 8 up, 8 in 2024-08-17T17:21:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:48 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 3849/1260 objects degraded (305.476%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:48 smithi063 ceph-mon[109843]: osdmap e1186: 8 total, 8 up, 8 in 2024-08-17T17:21:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:48 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 3849/1260 objects degraded (305.476%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:49.030 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:49.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:48 smithi162 ceph-mon[104668]: osdmap e1186: 8 total, 8 up, 8 in 2024-08-17T17:21:49.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:48 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 3849/1260 objects degraded (305.476%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:21:49.132 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:49.154 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:49.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:49.257 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:49.279 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:49.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:49.380 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:49.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.402 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:49.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:49.503 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:49.524 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:49.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:49.626 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:49.648 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:49.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:49.749 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:49.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.774 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:49.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:49.875 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:49.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:49.899 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:49.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:50.000 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:50.022 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:50.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:49 smithi162 ceph-mon[104668]: pgmap v1483: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 21 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 1764/1065 objects degraded (165.634%); 0 B/s, 553 objects/s recovering 2024-08-17T17:21:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:49 smithi162 ceph-mon[104668]: 7.f scrub starts 2024-08-17T17:21:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:49 smithi162 ceph-mon[104668]: osdmap e1187: 8 total, 8 up, 8 in 2024-08-17T17:21:50.124 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:50.147 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:50.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:50.249 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:50.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.272 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:50.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:49 smithi063 ceph-mon[114756]: pgmap v1483: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 21 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 1764/1065 objects degraded (165.634%); 0 B/s, 553 objects/s recovering 2024-08-17T17:21:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:49 smithi063 ceph-mon[114756]: 7.f scrub starts 2024-08-17T17:21:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:49 smithi063 ceph-mon[114756]: osdmap e1187: 8 total, 8 up, 8 in 2024-08-17T17:21:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:49 smithi063 ceph-mon[109843]: pgmap v1483: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 21 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 147 op/s; 1764/1065 objects degraded (165.634%); 0 B/s, 553 objects/s recovering 2024-08-17T17:21:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:49 smithi063 ceph-mon[109843]: 7.f scrub starts 2024-08-17T17:21:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:49 smithi063 ceph-mon[109843]: osdmap e1187: 8 total, 8 up, 8 in 2024-08-17T17:21:50.373 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:50.394 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:50.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:50.496 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:50.520 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:50.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:50.621 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:50.644 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:50.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:50.746 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:50.767 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:50.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:50.869 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:50.892 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:50.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:50.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:50.994 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:51.017 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:51.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:50 smithi063 ceph-mon[114756]: osdmap e1188: 8 total, 8 up, 8 in 2024-08-17T17:21:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:50 smithi063 ceph-mon[109843]: osdmap e1188: 8 total, 8 up, 8 in 2024-08-17T17:21:51.075 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:50 smithi162 ceph-mon[104668]: osdmap e1188: 8 total, 8 up, 8 in 2024-08-17T17:21:51.119 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:51.140 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:51.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:51.242 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:51.263 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:51.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:51.365 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:51.389 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:51.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:51.482 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:51.491 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:51.512 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:51.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:51.614 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:51.636 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:51.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:51.738 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:51.759 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:51.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:51.861 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:51.883 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:51.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:51.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:51.985 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:52.007 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:52.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:52.085 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:51 smithi063 ceph-mon[109843]: pgmap v1486: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 21 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 19 op/s; 992/1065 objects degraded (93.146%); 0 B/s, 725 objects/s recovering 2024-08-17T17:21:52.085 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:51 smithi063 ceph-mon[109843]: 7.22 scrub starts 2024-08-17T17:21:52.085 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:51 smithi063 ceph-mon[109843]: osdmap e1189: 8 total, 8 up, 8 in 2024-08-17T17:21:52.086 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:51 smithi063 ceph-mon[114756]: pgmap v1486: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 21 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 19 op/s; 992/1065 objects degraded (93.146%); 0 B/s, 725 objects/s recovering 2024-08-17T17:21:52.086 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:51 smithi063 ceph-mon[114756]: 7.22 scrub starts 2024-08-17T17:21:52.086 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:51 smithi063 ceph-mon[114756]: osdmap e1189: 8 total, 8 up, 8 in 2024-08-17T17:21:52.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:51 smithi162 ceph-mon[104668]: pgmap v1486: 119 pgs: 1 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 21 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 19 op/s; 992/1065 objects degraded (93.146%); 0 B/s, 725 objects/s recovering 2024-08-17T17:21:52.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:51 smithi162 ceph-mon[104668]: 7.22 scrub starts 2024-08-17T17:21:52.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:51 smithi162 ceph-mon[104668]: osdmap e1189: 8 total, 8 up, 8 in 2024-08-17T17:21:52.109 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:52.130 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:52.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:52.232 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:52.254 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:52.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:52.356 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:52.379 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:52.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:52.480 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:52.503 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:52.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:52.604 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:52.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.627 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:52.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:52.728 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:52.750 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:52.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:52.852 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:52.873 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:52.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:52.975 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:52.998 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:52.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:52.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:52 smithi162 ceph-mon[104668]: 7.23 scrub starts 2024-08-17T17:21:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:52 smithi162 ceph-mon[104668]: osdmap e1190: 8 total, 8 up, 8 in 2024-08-17T17:21:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:52 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:21:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:52 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:21:53.100 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:53.103 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:52 smithi063 ceph-mon[114756]: 7.23 scrub starts 2024-08-17T17:21:53.103 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:52 smithi063 ceph-mon[114756]: osdmap e1190: 8 total, 8 up, 8 in 2024-08-17T17:21:53.103 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:52 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:21:53.103 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:52 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:21:53.103 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:52 smithi063 ceph-mon[109843]: 7.23 scrub starts 2024-08-17T17:21:53.103 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:52 smithi063 ceph-mon[109843]: osdmap e1190: 8 total, 8 up, 8 in 2024-08-17T17:21:53.104 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:52 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:21:53.104 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:52 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:21:53.110 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:53.123 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:53.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:53.225 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:53.247 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:53.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:53.349 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:53.371 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:53.372 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:21:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:21:53] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.51.0" 2024-08-17T17:21:53.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:53.473 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:53.494 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:53.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:53.596 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:53.619 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:53.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:53.720 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:53.743 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:53.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:53.807 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:21:53.844 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:53.867 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:53.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:53.969 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:53.992 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:53.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:53.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:53 smithi063 ceph-mon[114756]: pgmap v1489: 119 pgs: 1 peering, 118 active+clean; 21 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 441 objects/s recovering 2024-08-17T17:21:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:53 smithi063 ceph-mon[114756]: 7.13 scrub starts 2024-08-17T17:21:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:53 smithi063 ceph-mon[114756]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 992/1065 objects degraded (93.146%), 4 pgs degraded) 2024-08-17T17:21:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:53 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2861855935' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:53 smithi063 ceph-mon[109843]: pgmap v1489: 119 pgs: 1 peering, 118 active+clean; 21 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 441 objects/s recovering 2024-08-17T17:21:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:53 smithi063 ceph-mon[109843]: 7.13 scrub starts 2024-08-17T17:21:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:53 smithi063 ceph-mon[109843]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 992/1065 objects degraded (93.146%), 4 pgs degraded) 2024-08-17T17:21:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:53 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2861855935' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:54.093 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:53 smithi162 ceph-mon[104668]: pgmap v1489: 119 pgs: 1 peering, 118 active+clean; 21 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 441 objects/s recovering 2024-08-17T17:21:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:53 smithi162 ceph-mon[104668]: 7.13 scrub starts 2024-08-17T17:21:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:53 smithi162 ceph-mon[104668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 992/1065 objects degraded (93.146%), 4 pgs degraded) 2024-08-17T17:21:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:53 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2861855935' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:21:54.117 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:54.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:54.219 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:54.243 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:54.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:54.345 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:54.367 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:54.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:54.469 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:54.493 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:54.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:54.594 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:54.617 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:54.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:54.719 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:54.742 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:54.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:54.815 INFO:teuthology.orchestra.run.smithi063.stderr:pool 'unique_pool_2' removed 2024-08-17T17:21:54.845 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:54.882 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:54.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:54.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:54.984 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:55.006 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:55.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:54 smithi162 ceph-mon[104668]: 7.3 scrub starts 2024-08-17T17:21:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:54 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:54 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2861855935' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:54 smithi162 ceph-mon[104668]: osdmap e1191: 8 total, 8 up, 8 in 2024-08-17T17:21:55.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:54 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3653309278' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:21:55.108 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:55.136 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:55.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:55.238 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:55.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.270 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:55.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:55.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:54 smithi063 ceph-mon[109843]: 7.3 scrub starts 2024-08-17T17:21:55.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:54 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:55.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:54 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2861855935' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:55.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:54 smithi063 ceph-mon[109843]: osdmap e1191: 8 total, 8 up, 8 in 2024-08-17T17:21:55.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:54 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3653309278' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:21:55.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:54 smithi063 ceph-mon[114756]: 7.3 scrub starts 2024-08-17T17:21:55.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:54 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:21:55.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:54 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2861855935' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:21:55.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:54 smithi063 ceph-mon[114756]: osdmap e1191: 8 total, 8 up, 8 in 2024-08-17T17:21:55.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:54 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3653309278' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:21:55.371 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:55.394 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:55.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:55.496 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:55.519 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:55.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:55.621 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:55.642 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:55.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:55.744 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:55.767 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:55.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:55.869 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:55.890 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:55.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:55.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:55.992 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:56.013 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:56.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:55 smithi063 ceph-mon[109843]: pgmap v1491: 119 pgs: 1 peering, 118 active+clean; 21 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 394 objects/s recovering 2024-08-17T17:21:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:55 smithi063 ceph-mon[109843]: 7.f scrub starts 2024-08-17T17:21:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:55 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3653309278' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:21:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:55 smithi063 ceph-mon[109843]: osdmap e1192: 8 total, 8 up, 8 in 2024-08-17T17:21:56.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:55 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4197868481' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:56.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:55 smithi063 ceph-mon[114756]: pgmap v1491: 119 pgs: 1 peering, 118 active+clean; 21 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 394 objects/s recovering 2024-08-17T17:21:56.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:55 smithi063 ceph-mon[114756]: 7.f scrub starts 2024-08-17T17:21:56.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:55 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3653309278' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:21:56.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:55 smithi063 ceph-mon[114756]: osdmap e1192: 8 total, 8 up, 8 in 2024-08-17T17:21:56.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:55 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4197868481' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:55 smithi162 ceph-mon[104668]: pgmap v1491: 119 pgs: 1 peering, 118 active+clean; 21 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 394 objects/s recovering 2024-08-17T17:21:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:55 smithi162 ceph-mon[104668]: 7.f scrub starts 2024-08-17T17:21:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:55 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3653309278' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:21:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:55 smithi162 ceph-mon[104668]: osdmap e1192: 8 total, 8 up, 8 in 2024-08-17T17:21:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:55 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4197868481' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:21:56.114 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:56.135 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:56.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:56.237 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:56.259 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:56.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:56.360 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:56.383 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:56.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:56.484 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:56.506 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:56.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:56.608 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:56.629 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:56.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:56.731 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:56.754 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:56.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:56.855 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:56.878 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:56.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:56.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:56.979 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:57.001 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:57.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:57.103 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:57.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.126 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:57.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:57.227 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:57.251 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:57.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:57.332 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:21:57.332 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:21:57.333 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:21:57.333 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:21:57.333 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:21:57.333 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:21:57.333 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:21:57.333 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:21:57.333 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:21:57.334 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:21:57.334 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:21:57.334 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:21:57.334 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:21:57.334 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:21:57.334 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:21:57.334 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:21:57.353 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:57.374 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:57.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:57.476 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:57.503 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:57.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:57.605 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:57.630 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:57.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:57.732 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:57.755 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:57.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:57.856 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:57.883 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:57.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:57.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:57.920 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:57 smithi063 ceph-mon[114756]: pgmap v1493: 83 pgs: 83 active+clean; 844 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:21:57.920 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:57 smithi063 ceph-mon[109843]: pgmap v1493: 83 pgs: 83 active+clean; 844 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:21:57.984 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:58.008 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:58.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:57 smithi162 ceph-mon[104668]: pgmap v1493: 83 pgs: 83 active+clean; 844 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:21:58.110 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:58.132 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:58.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:58.233 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:21:58.260 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:58.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:21:58.361 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:58.387 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:58.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:58.489 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:21:58.515 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:58.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:21:58.597 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:21:58.597 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:21:58.598 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-08-17T17:21:58.598 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_3 16 2024-08-17T17:21:58.617 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:58.664 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:58.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:58.765 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:58.788 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:58.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:58.824 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:21:58.890 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:21:58.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:58.913 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:58.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:21:59.015 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:59.037 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:59.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:59.139 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:59.161 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:59.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:59.263 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:21:59.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.285 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:59.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:21:59.386 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:21:59.412 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:59.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:21:59.487 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:21:59.487 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:21:59.487 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 0 2024-08-17T17:21:59.487 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:21:59.514 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:59.539 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:59.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:59.641 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:21:59.664 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:21:59.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:21:59.698 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:59.701 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:21:59.766 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:21:59.788 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:59.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:21:59.807 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:21:59.890 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:21:59.914 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:21:59.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:21:59.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:21:59.943 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:22:00.017 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:00.042 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:00.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:59 smithi162 ceph-mon[104668]: pgmap v1494: 83 pgs: 83 active+clean; 844 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:59 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/126714068' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:22:00.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:21:59 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:22:00.144 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:00.169 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:00.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:00.201 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:59 smithi063 ceph-mon[114756]: pgmap v1494: 83 pgs: 83 active+clean; 844 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:00.201 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:59 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/126714068' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:22:00.201 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:21:59 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:22:00.202 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:59 smithi063 ceph-mon[109843]: pgmap v1494: 83 pgs: 83 active+clean; 844 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:00.202 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:59 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/126714068' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:22:00.202 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:21:59 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:22:00.225 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:22:00.271 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:00.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.295 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:00.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:00.396 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:00.422 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:00.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:00.523 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:00.548 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:00.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:00.650 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:00.672 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:00.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:00.774 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:00.800 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:00.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:00.902 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:00.924 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:00.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:00.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:01.026 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:01.048 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:01.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:01.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:00 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:22:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:00 smithi162 ceph-mon[104668]: osdmap e1193: 8 total, 8 up, 8 in 2024-08-17T17:22:01.149 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:01.175 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:01.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:01.277 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:01.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:00 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:22:01.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:00 smithi063 ceph-mon[114756]: osdmap e1193: 8 total, 8 up, 8 in 2024-08-17T17:22:01.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:00 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:22:01.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:00 smithi063 ceph-mon[109843]: osdmap e1193: 8 total, 8 up, 8 in 2024-08-17T17:22:01.302 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:01.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:01.375 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:22:01.403 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:01.439 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:01.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:01.541 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:01.564 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:01.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:01.665 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:01.687 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:01.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:01.789 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:01.811 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:01.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:01.912 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:01.938 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:01.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:01.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:01 smithi063 ceph-mon[114756]: pgmap v1496: 83 pgs: 83 active+clean; 844 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:01 smithi063 ceph-mon[109843]: pgmap v1496: 83 pgs: 83 active+clean; 844 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:02.040 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:02.065 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:02.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:01 smithi162 ceph-mon[104668]: pgmap v1496: 83 pgs: 83 active+clean; 844 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:02.167 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:02.193 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:02.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:02.294 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:02.317 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:02.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:02.418 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:02.444 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:02.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:02.462 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:02.546 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:02.570 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:02.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:02.672 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:02.694 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:02.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:02.796 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:02.819 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:02.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:02.921 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:02.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:02.948 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:02.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:03.050 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:03.072 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:03.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:03.174 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:03.197 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:03.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:03.298 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:03.323 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:03.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:03.424 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:03.447 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:03.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:03.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:22:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:22:03] "GET /metrics HTTP/1.1" 200 34055 "" "Prometheus/2.51.0" 2024-08-17T17:22:03.548 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:03.571 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:03.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:03.672 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:03.699 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:03.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:03.801 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:03.827 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:03.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:03.928 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:03.951 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:03.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:03.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:04.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:03 smithi063 ceph-mon[114756]: pgmap v1497: 83 pgs: 83 active+clean; 844 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:04.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:03 smithi063 ceph-mon[109843]: pgmap v1497: 83 pgs: 83 active+clean; 844 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:04.054 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:04.075 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:04.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:04.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:03 smithi162 ceph-mon[104668]: pgmap v1497: 83 pgs: 83 active+clean; 844 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:04.177 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:04.200 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:04.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:04.302 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:04.326 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:04.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:04.428 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:04.451 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:04.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:04.553 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:04.575 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:04.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:04.677 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:04.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.700 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:04.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:04.801 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:04.823 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:04.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:04.925 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:04.947 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:04.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:04.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:05.049 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:05.071 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:05.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:05.173 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:05.195 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:05.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:05.297 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:05.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.320 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:05.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:05.421 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:05.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.442 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:05.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:05.543 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:05.564 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:05.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:05.665 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:05.686 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:05.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:05.750 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:05.751 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:05.788 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:05.809 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:05.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:05.824 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:22:05.911 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:05.933 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:05.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:05.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:06.035 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:06.056 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:06.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:06.078 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:05 smithi063 ceph-mon[114756]: pgmap v1498: 83 pgs: 83 active+clean; 844 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:06.078 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:05 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/382906274' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:22:06.079 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:05 smithi063 ceph-mon[109843]: pgmap v1498: 83 pgs: 83 active+clean; 844 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:06.079 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:05 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/382906274' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:22:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:05 smithi162 ceph-mon[104668]: pgmap v1498: 83 pgs: 83 active+clean; 844 KiB data, 3.5 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:05 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/382906274' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:22:06.158 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:06.179 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:06.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:06.280 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:06.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.302 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:06.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:06.403 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:06.427 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:06.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:06.529 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:06.551 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:06.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:06.653 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:06.670 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:22:06.670 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:22:06.674 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:06.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:06.687 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:22:06.687 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:22:06.703 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:22:06.704 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:22:06.720 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:22:06.720 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:22:06.736 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:22:06.736 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:22:06.753 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:22:06.753 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:22:06.770 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:22:06.770 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:22:06.776 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:06.786 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:22:06.787 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:22:06.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.799 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:06.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:06.900 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:06.923 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:06.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:06.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:07.024 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:07.047 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:07.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:06 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:06 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/382906274' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:22:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:06 smithi162 ceph-mon[104668]: osdmap e1194: 8 total, 8 up, 8 in 2024-08-17T17:22:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:06 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/90787316' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:07.096 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:06 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:07.096 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:06 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/382906274' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:22:07.096 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:06 smithi063 ceph-mon[109843]: osdmap e1194: 8 total, 8 up, 8 in 2024-08-17T17:22:07.096 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:06 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/90787316' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:07.097 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:06 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:07.097 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:06 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/382906274' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:22:07.097 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:06 smithi063 ceph-mon[114756]: osdmap e1194: 8 total, 8 up, 8 in 2024-08-17T17:22:07.097 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:06 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/90787316' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:07.148 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:07.170 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:07.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:07.272 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:07.295 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:07.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:07.396 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:07.419 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:07.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:07.521 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:07.544 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:07.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:07.645 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:07.668 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:07.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:07.770 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:07.796 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:07.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:07.898 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:07.924 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:07.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:07.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:08.025 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:08.050 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:08.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:07 smithi162 ceph-mon[104668]: 1.7 scrub starts 2024-08-17T17:22:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:07 smithi162 ceph-mon[104668]: 1.7 scrub ok 2024-08-17T17:22:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:07 smithi162 ceph-mon[104668]: pgmap v1500: 83 pgs: 83 active+clean; 844 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:07 smithi162 ceph-mon[104668]: 1.5 scrub starts 2024-08-17T17:22:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:07 smithi162 ceph-mon[104668]: 1.5 scrub ok 2024-08-17T17:22:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:07 smithi162 ceph-mon[104668]: 1.2 scrub starts 2024-08-17T17:22:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:07 smithi162 ceph-mon[104668]: 1.2 scrub ok 2024-08-17T17:22:08.152 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:08.171 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:07 smithi063 ceph-mon[114756]: 1.7 scrub starts 2024-08-17T17:22:08.171 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:07 smithi063 ceph-mon[114756]: 1.7 scrub ok 2024-08-17T17:22:08.171 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:07 smithi063 ceph-mon[114756]: pgmap v1500: 83 pgs: 83 active+clean; 844 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:08.171 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:07 smithi063 ceph-mon[114756]: 1.5 scrub starts 2024-08-17T17:22:08.171 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:07 smithi063 ceph-mon[114756]: 1.5 scrub ok 2024-08-17T17:22:08.172 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:07 smithi063 ceph-mon[114756]: 1.2 scrub starts 2024-08-17T17:22:08.172 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:07 smithi063 ceph-mon[114756]: 1.2 scrub ok 2024-08-17T17:22:08.172 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:07 smithi063 ceph-mon[109843]: 1.7 scrub starts 2024-08-17T17:22:08.172 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:07 smithi063 ceph-mon[109843]: 1.7 scrub ok 2024-08-17T17:22:08.172 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:07 smithi063 ceph-mon[109843]: pgmap v1500: 83 pgs: 83 active+clean; 844 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:08.172 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:07 smithi063 ceph-mon[109843]: 1.5 scrub starts 2024-08-17T17:22:08.172 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:07 smithi063 ceph-mon[109843]: 1.5 scrub ok 2024-08-17T17:22:08.172 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:07 smithi063 ceph-mon[109843]: 1.2 scrub starts 2024-08-17T17:22:08.172 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:07 smithi063 ceph-mon[109843]: 1.2 scrub ok 2024-08-17T17:22:08.177 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:08.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:08.279 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:08.303 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:08.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:08.405 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:08.428 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:08.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:08.530 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:08.552 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:08.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:08.653 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:08.675 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:08.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:08.777 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:08.799 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:08.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:08.901 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:08.924 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:08.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:08.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:09.025 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:09.051 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:09.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:09.153 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:09.175 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:09.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:09.277 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:09.302 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:09.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:09.405 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:09.429 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:09.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:09.530 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:09.553 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:09.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:09.655 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:09.680 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:09.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:09.782 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:09.807 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:09.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:09.844 INFO:teuthology.orchestra.run.smithi063.stderr:pool 'unique_pool_3' created 2024-08-17T17:22:09.909 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:09.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:09.935 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:09.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:10.036 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:10.049 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:22:10.059 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:10.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:09 smithi162 ceph-mon[104668]: 1.6 scrub starts 2024-08-17T17:22:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:09 smithi162 ceph-mon[104668]: 1.6 scrub ok 2024-08-17T17:22:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:09 smithi162 ceph-mon[104668]: pgmap v1501: 83 pgs: 83 active+clean; 687 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:09 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2876897329' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T17:22:10.161 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:10.187 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:10.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:10.277 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:09 smithi063 ceph-mon[114756]: 1.6 scrub starts 2024-08-17T17:22:10.277 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:09 smithi063 ceph-mon[114756]: 1.6 scrub ok 2024-08-17T17:22:10.277 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:09 smithi063 ceph-mon[114756]: pgmap v1501: 83 pgs: 83 active+clean; 687 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:10.277 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:09 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2876897329' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T17:22:10.277 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:09 smithi063 ceph-mon[109843]: 1.6 scrub starts 2024-08-17T17:22:10.277 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:09 smithi063 ceph-mon[109843]: 1.6 scrub ok 2024-08-17T17:22:10.277 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:09 smithi063 ceph-mon[109843]: pgmap v1501: 83 pgs: 83 active+clean; 687 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:10.278 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:09 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2876897329' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-08-17T17:22:10.289 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:10.314 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:10.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:10.416 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:10.440 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:10.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:10.542 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:10.569 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:10.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:10.640 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-08-17T17:22:10.671 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:10.705 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:10.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:10.806 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:10.829 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:10.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:10.906 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:10 smithi063 ceph-mon[109843]: 1.3 scrub starts 2024-08-17T17:22:10.906 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:10 smithi063 ceph-mon[109843]: 1.3 scrub ok 2024-08-17T17:22:10.906 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:10 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2876897329' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-08-17T17:22:10.906 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:10 smithi063 ceph-mon[109843]: osdmap e1195: 8 total, 8 up, 8 in 2024-08-17T17:22:10.906 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:10 smithi063 ceph-mon[114756]: 1.3 scrub starts 2024-08-17T17:22:10.906 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:10 smithi063 ceph-mon[114756]: 1.3 scrub ok 2024-08-17T17:22:10.907 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:10 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2876897329' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-08-17T17:22:10.907 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:10 smithi063 ceph-mon[114756]: osdmap e1195: 8 total, 8 up, 8 in 2024-08-17T17:22:10.931 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:10.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:10.956 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:10.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:11.057 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:11.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.084 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:11.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:11.185 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:11.210 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:11.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:11.312 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:11.334 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:11.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:11.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:10 smithi162 ceph-mon[104668]: 1.3 scrub starts 2024-08-17T17:22:11.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:10 smithi162 ceph-mon[104668]: 1.3 scrub ok 2024-08-17T17:22:11.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:10 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2876897329' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-08-17T17:22:11.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:10 smithi162 ceph-mon[104668]: osdmap e1195: 8 total, 8 up, 8 in 2024-08-17T17:22:11.436 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:11.461 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:11.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:11.563 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:11.586 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:11.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:11.688 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:11.714 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:11.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:11.816 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:11.840 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:11.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:11.942 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:11.964 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:11.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:11.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:12.066 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:12.091 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:12.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:12.115 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:11 smithi063 ceph-mon[109843]: pgmap v1503: 99 pgs: 5 creating+peering, 11 unknown, 83 active+clean; 687 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:12.115 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:11 smithi063 ceph-mon[109843]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:22:12.115 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:11 smithi063 ceph-mon[109843]: osdmap e1196: 8 total, 8 up, 8 in 2024-08-17T17:22:12.115 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:11 smithi063 ceph-mon[114756]: pgmap v1503: 99 pgs: 5 creating+peering, 11 unknown, 83 active+clean; 687 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:12.115 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:11 smithi063 ceph-mon[114756]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:22:12.115 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:11 smithi063 ceph-mon[114756]: osdmap e1196: 8 total, 8 up, 8 in 2024-08-17T17:22:12.193 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:12.218 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:12.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:12.319 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:12.341 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:12.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:12.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:11 smithi162 ceph-mon[104668]: pgmap v1503: 99 pgs: 5 creating+peering, 11 unknown, 83 active+clean; 687 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:12.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:11 smithi162 ceph-mon[104668]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:22:12.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:11 smithi162 ceph-mon[104668]: osdmap e1196: 8 total, 8 up, 8 in 2024-08-17T17:22:12.443 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:12.472 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:12.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:12.573 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:12.598 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:12.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:12.700 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:12.722 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:12.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:12.824 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:12.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.847 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:12.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:12.948 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:12.970 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:12.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:12.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:13.072 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:13.077 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:13.077 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:13.098 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:13.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:13.188 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:12 smithi063 ceph-mon[114756]: osdmap e1197: 8 total, 8 up, 8 in 2024-08-17T17:22:13.189 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:12 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:13.189 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:12 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:13.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:12 smithi063 ceph-mon[109843]: osdmap e1197: 8 total, 8 up, 8 in 2024-08-17T17:22:13.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:12 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:13.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:12 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:13.199 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:13.223 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:13.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:13.305 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:22:13.305 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:22:13.324 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:13.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:12 smithi162 ceph-mon[104668]: osdmap e1197: 8 total, 8 up, 8 in 2024-08-17T17:22:13.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:12 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:13.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:12 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:13.350 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:13.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:13.452 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:13.477 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:13.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:22:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:22:13] "GET /metrics HTTP/1.1" 200 34055 "" "Prometheus/2.51.0" 2024-08-17T17:22:13.578 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:13.603 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:13.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:13.705 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:13.727 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:13.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:13.829 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:13.855 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:13.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:13.870 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:22:13.957 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:13.981 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:13.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:13.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:14.018 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:22:14.083 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:14.123 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:14.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:14.134 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:22:14.186 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:13 smithi063 ceph-mon[114756]: pgmap v1506: 99 pgs: 5 creating+peering, 4 unknown, 90 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:14.186 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:13 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/953807971' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:22:14.187 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:13 smithi063 ceph-mon[109843]: pgmap v1506: 99 pgs: 5 creating+peering, 4 unknown, 90 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:14.187 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:13 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/953807971' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:22:14.225 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:14.250 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:14.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.252 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:14.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:13 smithi162 ceph-mon[104668]: pgmap v1506: 99 pgs: 5 creating+peering, 4 unknown, 90 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:14.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:13 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/953807971' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:22:14.352 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:14.375 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:14.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:14.476 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:14.502 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:14.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:14.604 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:14.627 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:14.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:14.668 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:22:14.730 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:14.754 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:14.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:14.857 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:14.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:14.882 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:14.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:14.983 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:15.006 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:15.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:15.008 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:14 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/953807971' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:22:15.008 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:14 smithi063 ceph-mon[114756]: osdmap e1198: 8 total, 8 up, 8 in 2024-08-17T17:22:15.008 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:14 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:22:15.008 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:14 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:22:15.009 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:14 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:15.009 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:14 smithi063 ceph-mon[114756]: pgmap v1508: 99 pgs: 5 creating+peering, 94 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:15.009 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:14 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/953807971' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:22:15.009 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:14 smithi063 ceph-mon[109843]: osdmap e1198: 8 total, 8 up, 8 in 2024-08-17T17:22:15.009 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:14 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:22:15.009 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:14 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:22:15.009 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:14 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:15.009 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:14 smithi063 ceph-mon[109843]: pgmap v1508: 99 pgs: 5 creating+peering, 94 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:15.108 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:15.133 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:15.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:15.176 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:14 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/953807971' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:22:15.176 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:14 smithi162 ceph-mon[104668]: osdmap e1198: 8 total, 8 up, 8 in 2024-08-17T17:22:15.176 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:14 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:22:15.176 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:14 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:22:15.176 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:14 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:15.176 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:14 smithi162 ceph-mon[104668]: pgmap v1508: 99 pgs: 5 creating+peering, 94 active+clean; 582 KiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:15.235 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:15.258 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:15.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:15.359 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:15.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.383 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:15.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:15.484 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:15.506 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:15.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:15.608 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:15.630 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:15.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:15.732 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:15.757 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:15.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:15.858 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:15.881 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:15.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:15.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:15.983 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:16.005 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:16.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:16.096 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:16.099 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:16.107 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:16.132 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:16.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:16.234 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:16.259 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:16.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:16.361 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:16.385 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:16.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:16.487 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:16.512 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:16.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:16.614 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:16.635 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:16.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:16.737 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:16.759 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:16.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:16.861 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:16.887 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:16.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:16.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:16.989 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:17.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.012 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:17.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:17.113 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:17.135 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:17.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:17.236 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:17.261 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:17.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:17.353 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:17 smithi063 ceph-mon[114756]: pgmap v1509: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:17.353 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:17 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:17.353 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:17 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:17.353 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:17 smithi063 ceph-mon[109843]: pgmap v1509: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:17.353 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:17 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:17.353 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:17 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:17.363 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:17.386 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:17.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:17.488 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:17.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.513 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:17.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:17.614 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:17.636 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:17.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:17.738 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:17.760 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:17.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:17.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:17 smithi162 ceph-mon[104668]: pgmap v1509: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:17.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:17 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:17.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:17 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:17.861 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:17.887 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:17.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:17.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:17.904 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:17.989 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:18.015 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:18.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:18.118 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:18.143 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:18.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:18.245 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:18.268 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:18.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:18.370 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:18.392 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:18.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:18.494 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:18.516 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:18.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:18.618 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:18.644 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:18.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:18.746 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:18.771 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:18.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:18.873 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:18.902 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:18.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:18.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:19.004 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:19.028 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:19.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:19.130 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:19.152 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:19.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:19.254 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:19.279 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:19.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:19.381 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:19.405 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:19.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:19.506 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:19.529 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:19.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:19 smithi063 ceph-mon[114756]: pgmap v1510: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:19.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:19 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1691018689' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:19.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:19 smithi063 ceph-mon[109843]: pgmap v1510: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:19.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:19 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1691018689' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:19.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:19.631 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:19.655 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:19.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:19.756 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:19.781 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:19.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:19 smithi162 ceph-mon[104668]: pgmap v1510: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:19 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1691018689' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:19.883 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:19.905 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:19.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:19.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:20.007 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:20.030 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:20.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:20.078 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:22:20.078 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:22:20.078 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:22:20.078 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:22:20.079 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:22:20.079 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:22:20.079 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:22:20.079 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:22:20.079 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:22:20.079 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:22:20.079 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:22:20.079 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:22:20.080 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:22:20.080 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:22:20.080 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:22:20.080 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:22:20.131 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:20.157 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:20.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:20.259 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:20.282 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:20.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:20.384 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:20.410 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:20.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:20.512 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:20.534 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:20.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:20.636 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:20.661 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:20.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:20.762 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:20.786 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:20.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:20.890 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:20.913 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:20.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:20.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:21.014 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:21.040 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:21.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:21.142 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:21.167 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:21.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:21.268 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:21.292 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:21.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.293 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:21.393 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:21.419 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:21.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:21.429 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:21 smithi063 ceph-mon[114756]: pgmap v1511: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:21.430 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:21 smithi063 ceph-mon[109843]: pgmap v1511: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:21.521 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:21.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:21.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:21.647 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:21.672 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:21.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:21.774 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:21.796 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:21.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:21.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:21 smithi162 ceph-mon[104668]: pgmap v1511: 99 pgs: 99 active+clean; 582 KiB data, 3.3 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:21.897 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:21.919 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:21.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:21.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:22.021 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:22.046 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:22.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:22.148 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:22.171 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:22.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:22.273 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:22.295 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:22.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:22.363 INFO:teuthology.orchestra.run.smithi063.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-08-17T17:22:22.397 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:22.419 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:22.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:22.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:22 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/540170619' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:22:22.517 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:22 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/540170619' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:22:22.521 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:22.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:22.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:22.648 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:22.670 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:22.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:22.771 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:22.791 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:22:22.797 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:22.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:22.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:22 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/540170619' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:22:22.899 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:22.925 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:22.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:22.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:23.026 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:23.050 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:23.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:23.142 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:22:23.142 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:22:23.142 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:22:23.142 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:22:23.142 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.142 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:29975 6.1 6:82e1ea0d:::benchmark_data_smithi063_160268_object29974:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:22:23.142 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:37.728080+0000", 2024-08-17T17:22:23.143 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 525.40897081799994, 2024-08-17T17:22:23.143 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.075472543000000003, 2024-08-17T17:22:23.143 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.143 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.143 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.143 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:23.143 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:22:23.143 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:22:23.143 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 29975 2024-08-17T17:22:23.143 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.143 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.143 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.143 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.144 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728080+0000", 2024-08-17T17:22:23.144 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.144 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.144 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.144 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.144 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728080+0000", 2024-08-17T17:22:23.144 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.144 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.144 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.144 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.144 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728082+0000", 2024-08-17T17:22:23.144 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.35e-06 2024-08-17T17:22:23.145 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.145 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.145 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.145 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728106+0000", 2024-08-17T17:22:23.145 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.438e-05 2024-08-17T17:22:23.145 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.145 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.145 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.145 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728119+0000", 2024-08-17T17:22:23.145 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.256e-05 2024-08-17T17:22:23.145 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.145 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.145 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.145 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728126+0000", 2024-08-17T17:22:23.146 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5650000000000004e-06 2024-08-17T17:22:23.146 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.146 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.146 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.146 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728194+0000", 2024-08-17T17:22:23.146 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8263999999999996e-05 2024-08-17T17:22:23.146 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.146 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.146 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.146 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728342+0000", 2024-08-17T17:22:23.146 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014800699999999999 2024-08-17T17:22:23.146 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.147 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.147 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:22:23.147 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.728473+0000", 2024-08-17T17:22:23.147 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013097799999999999 2024-08-17T17:22:23.147 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.147 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.147 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.147 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.761767+0000", 2024-08-17T17:22:23.147 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.033293973999999997 2024-08-17T17:22:23.147 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.147 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.147 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:23.147 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.762011+0000", 2024-08-17T17:22:23.147 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000243315 2024-08-17T17:22:23.148 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.148 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.148 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.148 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.803446+0000", 2024-08-17T17:22:23.148 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.04143579 2024-08-17T17:22:23.148 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.148 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.148 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.148 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.803504+0000", 2024-08-17T17:22:23.148 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7759e-05 2024-08-17T17:22:23.148 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.148 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.148 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.148 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:37.803553+0000", 2024-08-17T17:22:23.149 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8600999999999998e-05 2024-08-17T17:22:23.149 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.149 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.149 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.149 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.149 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.149 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:32643 6.4 6:28d4a114:::benchmark_data_smithi063_160268_object32642:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:22:23.149 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.131350+0000", 2024-08-17T17:22:23.149 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 524.00570104200006, 2024-08-17T17:22:23.149 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.086976158999999997, 2024-08-17T17:22:23.149 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.149 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.150 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.150 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:23.150 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:22:23.150 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:22:23.150 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 32643 2024-08-17T17:22:23.150 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.150 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.150 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.150 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.150 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131350+0000", 2024-08-17T17:22:23.150 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.150 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.150 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.151 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.151 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131350+0000", 2024-08-17T17:22:23.151 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.151 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.151 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.151 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.151 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131351+0000", 2024-08-17T17:22:23.151 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.195e-06 2024-08-17T17:22:23.151 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.151 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.151 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.151 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131377+0000", 2024-08-17T17:22:23.151 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5463e-05 2024-08-17T17:22:23.151 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.151 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.152 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.152 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131389+0000", 2024-08-17T17:22:23.152 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2147000000000001e-05 2024-08-17T17:22:23.152 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.152 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.152 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.152 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131396+0000", 2024-08-17T17:22:23.152 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1729999999999998e-06 2024-08-17T17:22:23.152 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.152 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.152 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.152 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131478+0000", 2024-08-17T17:22:23.152 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2399999999999997e-05 2024-08-17T17:22:23.152 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.153 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.153 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.153 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131625+0000", 2024-08-17T17:22:23.153 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014619000000000001 2024-08-17T17:22:23.153 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.153 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.153 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:22:23.153 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.131770+0000", 2024-08-17T17:22:23.153 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014517800000000001 2024-08-17T17:22:23.153 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.153 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.153 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:23.153 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.133200+0000", 2024-08-17T17:22:23.153 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0014297069999999999 2024-08-17T17:22:23.153 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.154 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.154 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.154 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.135826+0000", 2024-08-17T17:22:23.154 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0026266179999999998 2024-08-17T17:22:23.154 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.154 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.154 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.154 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218155+0000", 2024-08-17T17:22:23.154 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.082328383000000005 2024-08-17T17:22:23.154 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.154 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.154 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.154 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218238+0000", 2024-08-17T17:22:23.155 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3920999999999998e-05 2024-08-17T17:22:23.155 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.155 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.155 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.155 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.218326+0000", 2024-08-17T17:22:23.155 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7783999999999998e-05 2024-08-17T17:22:23.155 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.155 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.155 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.155 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.155 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.155 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:33167 6.4 6:22dca8bd:::benchmark_data_smithi063_160268_object33166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:22:23.155 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.448673+0000", 2024-08-17T17:22:23.155 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 523.68837844999996, 2024-08-17T17:22:23.155 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.094468580999999996, 2024-08-17T17:22:23.156 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.156 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.156 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.156 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:23.156 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:22:23.156 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:22:23.156 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 33167 2024-08-17T17:22:23.156 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.156 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.156 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.156 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.157 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448673+0000", 2024-08-17T17:22:23.157 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.157 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.157 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.157 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.157 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448673+0000", 2024-08-17T17:22:23.157 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.157 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.157 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.157 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.157 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448673+0000", 2024-08-17T17:22:23.157 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3000000000000001e-07 2024-08-17T17:22:23.157 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.158 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.158 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.158 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448693+0000", 2024-08-17T17:22:23.158 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9422000000000001e-05 2024-08-17T17:22:23.158 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.158 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.158 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.158 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448700+0000", 2024-08-17T17:22:23.158 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2779999999999996e-06 2024-08-17T17:22:23.158 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.158 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.158 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.158 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.448706+0000", 2024-08-17T17:22:23.159 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6069999999999998e-06 2024-08-17T17:22:23.159 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.159 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.159 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.159 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.450351+0000", 2024-08-17T17:22:23.159 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001645526 2024-08-17T17:22:23.159 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.159 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.159 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.159 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.450469+0000", 2024-08-17T17:22:23.159 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000118239 2024-08-17T17:22:23.159 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.159 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.159 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:22:23.160 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.450598+0000", 2024-08-17T17:22:23.160 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012906300000000001 2024-08-17T17:22:23.160 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.160 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.160 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:23.160 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452120+0000", 2024-08-17T17:22:23.160 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0015215090000000001 2024-08-17T17:22:23.160 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.160 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.160 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.160 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452519+0000", 2024-08-17T17:22:23.160 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00039886099999999997 2024-08-17T17:22:23.160 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.160 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.160 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.161 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.543022+0000", 2024-08-17T17:22:23.161 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.090503086999999996 2024-08-17T17:22:23.161 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.161 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.161 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.161 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.543091+0000", 2024-08-17T17:22:23.161 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8806000000000005e-05 2024-08-17T17:22:23.161 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.161 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.161 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.161 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.543141+0000", 2024-08-17T17:22:23.161 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0652999999999998e-05 2024-08-17T17:22:23.161 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.161 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.162 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.162 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.162 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.162 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:33173 6.4 6:23580cad:::benchmark_data_smithi063_160268_object33172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:22:23.162 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.452560+0000", 2024-08-17T17:22:23.162 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 523.68449077000002, 2024-08-17T17:22:23.162 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.094171489999999997, 2024-08-17T17:22:23.162 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.163 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.163 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.163 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:23.163 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:22:23.163 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:22:23.163 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 33173 2024-08-17T17:22:23.163 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.163 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.163 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.163 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.163 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452560+0000", 2024-08-17T17:22:23.163 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.163 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.164 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.164 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.164 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452560+0000", 2024-08-17T17:22:23.164 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.164 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.164 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.164 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.164 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452562+0000", 2024-08-17T17:22:23.164 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5039999999999999e-06 2024-08-17T17:22:23.164 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.164 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.164 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.164 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452584+0000", 2024-08-17T17:22:23.165 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2578999999999998e-05 2024-08-17T17:22:23.165 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.165 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.165 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.165 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452595+0000", 2024-08-17T17:22:23.165 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0512e-05 2024-08-17T17:22:23.165 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.165 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.165 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.165 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452602+0000", 2024-08-17T17:22:23.166 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5710000000000003e-06 2024-08-17T17:22:23.166 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.166 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.166 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.166 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452692+0000", 2024-08-17T17:22:23.166 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0767999999999996e-05 2024-08-17T17:22:23.166 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.166 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.166 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.166 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452831+0000", 2024-08-17T17:22:23.166 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000138331 2024-08-17T17:22:23.166 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.167 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.167 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:22:23.167 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452965+0000", 2024-08-17T17:22:23.167 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013412600000000001 2024-08-17T17:22:23.167 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.167 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.167 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:23.167 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455446+0000", 2024-08-17T17:22:23.167 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0024813000000000001 2024-08-17T17:22:23.167 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.167 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.167 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.167 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539987+0000", 2024-08-17T17:22:23.167 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.084540709000000006 2024-08-17T17:22:23.168 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.168 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.168 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.168 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546616+0000", 2024-08-17T17:22:23.168 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0066297400000000003 2024-08-17T17:22:23.168 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.168 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.168 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.168 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546672+0000", 2024-08-17T17:22:23.168 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5600000000000003e-05 2024-08-17T17:22:23.168 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.168 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.168 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.168 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546732+0000", 2024-08-17T17:22:23.169 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9750000000000002e-05 2024-08-17T17:22:23.169 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.169 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.169 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.169 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.169 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.169 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:33174 6.4 6:2e7cd2fc:::benchmark_data_smithi063_160268_object33173:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:22:23.169 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.452620+0000", 2024-08-17T17:22:23.169 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 523.68443139299995, 2024-08-17T17:22:23.169 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.094311133000000005, 2024-08-17T17:22:23.169 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.169 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.169 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.169 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:23.169 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:22:23.170 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:22:23.170 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 33174 2024-08-17T17:22:23.170 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.170 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.170 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.170 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.170 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452620+0000", 2024-08-17T17:22:23.170 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.170 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.170 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.170 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.170 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452620+0000", 2024-08-17T17:22:23.170 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.171 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.171 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.171 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.171 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452620+0000", 2024-08-17T17:22:23.171 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8100000000000003e-07 2024-08-17T17:22:23.171 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.171 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.171 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.171 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452641+0000", 2024-08-17T17:22:23.171 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0205000000000001e-05 2024-08-17T17:22:23.172 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.172 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.172 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.172 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452648+0000", 2024-08-17T17:22:23.172 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0829999999999998e-06 2024-08-17T17:22:23.172 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.172 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.172 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.172 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.452653+0000", 2024-08-17T17:22:23.172 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1420000000000001e-06 2024-08-17T17:22:23.172 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.172 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.173 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.173 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.454367+0000", 2024-08-17T17:22:23.173 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001713881 2024-08-17T17:22:23.173 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.173 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.173 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.173 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.454477+0000", 2024-08-17T17:22:23.173 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011024299999999999 2024-08-17T17:22:23.173 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.173 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.173 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:22:23.173 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.454554+0000", 2024-08-17T17:22:23.173 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7177999999999999e-05 2024-08-17T17:22:23.174 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.174 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.174 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:23.174 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455452+0000", 2024-08-17T17:22:23.174 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000898411 2024-08-17T17:22:23.174 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.174 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.174 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.174 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.540026+0000", 2024-08-17T17:22:23.174 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.084573537000000004 2024-08-17T17:22:23.174 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.174 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.174 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.175 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546819+0000", 2024-08-17T17:22:23.175 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.006792518 2024-08-17T17:22:23.175 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.175 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.175 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.175 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546874+0000", 2024-08-17T17:22:23.175 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5748000000000002e-05 2024-08-17T17:22:23.175 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.175 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.175 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.175 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.546931+0000", 2024-08-17T17:22:23.175 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6606e-05 2024-08-17T17:22:23.175 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.175 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.175 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.176 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.176 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.176 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:33176 6.1 6:8a507c7a:::benchmark_data_smithi063_160268_object33175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:22:23.176 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.453381+0000", 2024-08-17T17:22:23.176 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 523.683670237, 2024-08-17T17:22:23.176 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.088080944999999994, 2024-08-17T17:22:23.176 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.176 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.176 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.176 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:23.176 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:22:23.176 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:22:23.176 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 33176 2024-08-17T17:22:23.176 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.176 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.177 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.177 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.177 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453381+0000", 2024-08-17T17:22:23.177 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.177 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.177 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.177 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.177 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453381+0000", 2024-08-17T17:22:23.177 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.177 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.177 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.177 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.177 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453382+0000", 2024-08-17T17:22:23.177 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.147e-06 2024-08-17T17:22:23.177 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.178 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.178 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.178 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453416+0000", 2024-08-17T17:22:23.178 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3822000000000002e-05 2024-08-17T17:22:23.178 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.178 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.178 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.178 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453427+0000", 2024-08-17T17:22:23.178 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1328000000000001e-05 2024-08-17T17:22:23.178 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.178 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.178 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.178 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453435+0000", 2024-08-17T17:22:23.178 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8869999999999992e-06 2024-08-17T17:22:23.178 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.179 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.179 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.179 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453492+0000", 2024-08-17T17:22:23.179 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7043000000000003e-05 2024-08-17T17:22:23.179 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.179 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.179 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.179 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453612+0000", 2024-08-17T17:22:23.179 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000119416 2024-08-17T17:22:23.179 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.179 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.180 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:22:23.180 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.453744+0000", 2024-08-17T17:22:23.180 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000132211 2024-08-17T17:22:23.180 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.180 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.180 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:23.180 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455034+0000", 2024-08-17T17:22:23.180 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001289915 2024-08-17T17:22:23.180 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.180 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.180 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.180 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455499+0000", 2024-08-17T17:22:23.180 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00046549900000000002 2024-08-17T17:22:23.180 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.180 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.181 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.181 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541347+0000", 2024-08-17T17:22:23.181 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.085847657999999993 2024-08-17T17:22:23.181 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.181 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.181 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.181 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541412+0000", 2024-08-17T17:22:23.181 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5417000000000001e-05 2024-08-17T17:22:23.181 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.181 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.181 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.181 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541462+0000", 2024-08-17T17:22:23.181 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9601999999999997e-05 2024-08-17T17:22:23.181 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.182 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.182 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.182 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.182 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.182 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:33181 6.d e1093/1086 6:b650d8fa:::benchmark_data_smithi063_160268_object33180:head v 1093'2025, mlcod=1093'2023)", 2024-08-17T17:22:23.182 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.455644+0000", 2024-08-17T17:22:23.182 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 523.68140688300002, 2024-08-17T17:22:23.182 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.087465092999999994, 2024-08-17T17:22:23.182 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.182 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.183 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.183 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.183 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.183 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.183 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455644+0000", 2024-08-17T17:22:23.183 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.183 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.183 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.183 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.183 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455644+0000", 2024-08-17T17:22:23.183 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.183 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.183 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.183 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.183 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455645+0000", 2024-08-17T17:22:23.184 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9599999999999999e-07 2024-08-17T17:22:23.184 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.184 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.184 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.184 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455708+0000", 2024-08-17T17:22:23.184 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2724000000000005e-05 2024-08-17T17:22:23.184 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.184 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.184 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.184 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455715+0000", 2024-08-17T17:22:23.184 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8050000000000007e-06 2024-08-17T17:22:23.184 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.184 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.184 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.184 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455720+0000", 2024-08-17T17:22:23.185 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8559999999999998e-06 2024-08-17T17:22:23.185 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.185 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.185 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:23.186 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.186 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455766+0000", 2024-08-17T17:22:23.186 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6258999999999999e-05 2024-08-17T17:22:23.186 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.186 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.186 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.186 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.455785+0000", 2024-08-17T17:22:23.187 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8329e-05 2024-08-17T17:22:23.187 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.187 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.187 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.187 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.543016+0000", 2024-08-17T17:22:23.187 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.087231031000000001 2024-08-17T17:22:23.187 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.187 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.187 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.187 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.543109+0000", 2024-08-17T17:22:23.187 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3492999999999994e-05 2024-08-17T17:22:23.187 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.187 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.188 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.188 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.188 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.188 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.44609.0:33176 6.1 e1093/1086 ondisk, result = 0)", 2024-08-17T17:22:23.188 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.456246+0000", 2024-08-17T17:22:23.188 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 523.68080549199999, 2024-08-17T17:22:23.188 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.085228506999999995, 2024-08-17T17:22:23.188 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.188 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.188 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:22:23.188 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.188 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.188 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.188 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456246+0000", 2024-08-17T17:22:23.189 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.189 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.189 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.189 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.189 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456246+0000", 2024-08-17T17:22:23.189 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.189 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.189 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.189 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.189 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456246+0000", 2024-08-17T17:22:23.189 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9800000000000003e-07 2024-08-17T17:22:23.189 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.189 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.189 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.190 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456250+0000", 2024-08-17T17:22:23.190 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8519999999999997e-06 2024-08-17T17:22:23.190 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.190 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.190 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.190 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541269+0000", 2024-08-17T17:22:23.190 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.085018446999999997 2024-08-17T17:22:23.190 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.190 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.190 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.190 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541279+0000", 2024-08-17T17:22:23.190 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0559e-05 2024-08-17T17:22:23.190 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.191 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.191 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.191 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541333+0000", 2024-08-17T17:22:23.191 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3792999999999998e-05 2024-08-17T17:22:23.191 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.191 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.191 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.191 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541341+0000", 2024-08-17T17:22:23.191 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2709999999999992e-06 2024-08-17T17:22:23.191 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.191 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.191 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.191 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.541474+0000", 2024-08-17T17:22:23.191 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013288699999999999 2024-08-17T17:22:23.191 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.192 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.192 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.192 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.192 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.192 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.44609.0:33173 6.4 e1093/1086 ondisk, result = 0)", 2024-08-17T17:22:23.192 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:39.456941+0000", 2024-08-17T17:22:23.192 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 523.68011005200003, 2024-08-17T17:22:23.192 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.083053689999999999, 2024-08-17T17:22:23.192 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.192 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.192 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:22:23.193 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.193 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.193 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.193 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456941+0000", 2024-08-17T17:22:23.193 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.193 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.193 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.193 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.193 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456941+0000", 2024-08-17T17:22:23.193 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.193 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.193 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.194 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.194 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456942+0000", 2024-08-17T17:22:23.194 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2099999999999997e-07 2024-08-17T17:22:23.194 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.194 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.194 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.194 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.456945+0000", 2024-08-17T17:22:23.194 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0970000000000002e-06 2024-08-17T17:22:23.194 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.194 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.194 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.194 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539891+0000", 2024-08-17T17:22:23.194 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.082946110000000003 2024-08-17T17:22:23.194 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.195 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.195 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.195 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539902+0000", 2024-08-17T17:22:23.195 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1094000000000001e-05 2024-08-17T17:22:23.195 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.195 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.195 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.195 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539971+0000", 2024-08-17T17:22:23.195 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9148999999999998e-05 2024-08-17T17:22:23.195 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.195 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.195 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.195 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539980+0000", 2024-08-17T17:22:23.196 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0359999999999995e-06 2024-08-17T17:22:23.196 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.196 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.196 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.196 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:39.539995+0000", 2024-08-17T17:22:23.196 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4683e-05 2024-08-17T17:22:23.196 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.196 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.196 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.196 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.196 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.196 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:34219 6.4 6:220ade53:::benchmark_data_smithi063_160268_object34218:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:22:23.196 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.044051+0000", 2024-08-17T17:22:23.197 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 523.09299972899998, 2024-08-17T17:22:23.197 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.091717032000000004, 2024-08-17T17:22:23.197 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.197 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.197 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.197 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:23.197 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:22:23.197 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:22:23.197 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 34219 2024-08-17T17:22:23.197 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.197 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.198 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.198 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.198 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044051+0000", 2024-08-17T17:22:23.198 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.198 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.198 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.198 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.198 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044051+0000", 2024-08-17T17:22:23.198 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.198 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.198 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.198 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.198 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044052+0000", 2024-08-17T17:22:23.199 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4900000000000005e-07 2024-08-17T17:22:23.199 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.199 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.199 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.199 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044074+0000", 2024-08-17T17:22:23.199 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1696999999999999e-05 2024-08-17T17:22:23.199 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.199 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.199 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.199 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044084+0000", 2024-08-17T17:22:23.199 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0482000000000001e-05 2024-08-17T17:22:23.199 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.199 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.200 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.200 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044091+0000", 2024-08-17T17:22:23.200 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9600000000000003e-06 2024-08-17T17:22:23.200 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.200 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.200 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.200 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044144+0000", 2024-08-17T17:22:23.200 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3177000000000003e-05 2024-08-17T17:22:23.200 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.200 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.200 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.200 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044238+0000", 2024-08-17T17:22:23.200 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3733000000000005e-05 2024-08-17T17:22:23.201 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.201 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.201 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:22:23.201 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.044316+0000", 2024-08-17T17:22:23.201 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7700999999999993e-05 2024-08-17T17:22:23.201 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.201 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.201 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.201 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.046094+0000", 2024-08-17T17:22:23.201 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0017784140000000001 2024-08-17T17:22:23.201 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.201 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.201 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:23.202 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.046102+0000", 2024-08-17T17:22:23.202 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8790000000000003e-06 2024-08-17T17:22:23.202 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.202 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.202 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.202 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.135653+0000", 2024-08-17T17:22:23.202 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.089550600999999994 2024-08-17T17:22:23.202 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.202 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.202 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.202 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.135720+0000", 2024-08-17T17:22:23.202 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7187999999999994e-05 2024-08-17T17:22:23.202 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.203 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.203 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.203 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.135768+0000", 2024-08-17T17:22:23.203 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8451000000000001e-05 2024-08-17T17:22:23.203 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.203 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.203 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.203 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.203 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.203 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35822 6.4 6:2f33fed9:::benchmark_data_smithi063_160268_object35821:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:22:23.203 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.883928+0000", 2024-08-17T17:22:23.203 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 522.25312358600002, 2024-08-17T17:22:23.204 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14375861300000001, 2024-08-17T17:22:23.204 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.204 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.204 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.204 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:23.204 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:22:23.204 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:22:23.204 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35822 2024-08-17T17:22:23.204 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.204 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.204 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.204 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.205 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883928+0000", 2024-08-17T17:22:23.205 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.205 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.205 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.205 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.205 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883928+0000", 2024-08-17T17:22:23.205 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.205 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.205 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.205 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.206 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883929+0000", 2024-08-17T17:22:23.206 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3459999999999999e-06 2024-08-17T17:22:23.206 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.206 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.206 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.206 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883947+0000", 2024-08-17T17:22:23.206 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8067000000000002e-05 2024-08-17T17:22:23.206 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.206 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.206 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.206 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883957+0000", 2024-08-17T17:22:23.206 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0179e-05 2024-08-17T17:22:23.206 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.207 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.207 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.207 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.883963+0000", 2024-08-17T17:22:23.207 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.57e-06 2024-08-17T17:22:23.207 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.207 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.207 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.207 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.884008+0000", 2024-08-17T17:22:23.207 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5207000000000003e-05 2024-08-17T17:22:23.207 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.207 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.207 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.207 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.884091+0000", 2024-08-17T17:22:23.207 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3162000000000001e-05 2024-08-17T17:22:23.208 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.208 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.208 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:22:23.208 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.884169+0000", 2024-08-17T17:22:23.208 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8159999999999997e-05 2024-08-17T17:22:23.208 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.208 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.208 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:23.208 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.885227+0000", 2024-08-17T17:22:23.208 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0010574099999999999 2024-08-17T17:22:23.208 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.208 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.209 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.209 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.885625+0000", 2024-08-17T17:22:23.209 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00039848699999999999 2024-08-17T17:22:23.209 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.209 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.209 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.209 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027581+0000", 2024-08-17T17:22:23.209 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.141956267 2024-08-17T17:22:23.209 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.210 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.210 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.210 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027642+0000", 2024-08-17T17:22:23.210 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0757000000000001e-05 2024-08-17T17:22:23.210 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.210 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.210 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.210 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027686+0000", 2024-08-17T17:22:23.210 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4001000000000001e-05 2024-08-17T17:22:23.210 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.210 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.210 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.210 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.211 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.211 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:124810 6.4 6:2af1567b:::benchmark_data_smithi063_160268_object124809:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-08-17T17:22:23.211 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:14:55.542514+0000", 2024-08-17T17:22:23.211 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 447.59453722699999, 2024-08-17T17:22:23.211 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.094516375999999999, 2024-08-17T17:22:23.211 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.211 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.211 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.211 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:23.211 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:22:23.211 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:22:23.211 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 124810 2024-08-17T17:22:23.211 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.211 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.211 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.212 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.212 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.542514+0000", 2024-08-17T17:22:23.212 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.212 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.212 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.212 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.212 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.542514+0000", 2024-08-17T17:22:23.212 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.212 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.212 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.212 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.212 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.542515+0000", 2024-08-17T17:22:23.212 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3680000000000001e-06 2024-08-17T17:22:23.212 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.212 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.213 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.213 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.542536+0000", 2024-08-17T17:22:23.213 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0990000000000001e-05 2024-08-17T17:22:23.213 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.213 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.213 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.213 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.542554+0000", 2024-08-17T17:22:23.213 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7271000000000001e-05 2024-08-17T17:22:23.213 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.213 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.213 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.213 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.542561+0000", 2024-08-17T17:22:23.213 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0180000000000004e-06 2024-08-17T17:22:23.213 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.213 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.214 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.214 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.551700+0000", 2024-08-17T17:22:23.214 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0091395299999999999 2024-08-17T17:22:23.214 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.214 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.214 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.214 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.552029+0000", 2024-08-17T17:22:23.214 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00032915800000000002 2024-08-17T17:22:23.214 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.214 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.215 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:22:23.215 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.552331+0000", 2024-08-17T17:22:23.215 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00030189300000000002 2024-08-17T17:22:23.215 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.215 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.215 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.215 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.555864+0000", 2024-08-17T17:22:23.215 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0035326200000000002 2024-08-17T17:22:23.215 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.215 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.215 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:23.215 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.556270+0000", 2024-08-17T17:22:23.215 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00040584399999999999 2024-08-17T17:22:23.215 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.216 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.216 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.216 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.636877+0000", 2024-08-17T17:22:23.216 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.080607636999999996 2024-08-17T17:22:23.216 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.216 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.216 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.216 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.636986+0000", 2024-08-17T17:22:23.216 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010857 2024-08-17T17:22:23.216 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.216 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.216 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.216 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.637030+0000", 2024-08-17T17:22:23.217 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4477000000000001e-05 2024-08-17T17:22:23.217 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.217 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.217 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.217 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.217 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.217 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:124848 6.4 6:2a0bafff:::benchmark_data_smithi063_160268_object124847:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1102)", 2024-08-17T17:22:23.217 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:14:55.583601+0000", 2024-08-17T17:22:23.217 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 447.55345024299999, 2024-08-17T17:22:23.217 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.073727231000000004, 2024-08-17T17:22:23.217 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.217 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.217 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.218 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:23.218 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:22:23.218 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:22:23.218 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 124848 2024-08-17T17:22:23.218 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.218 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.218 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.218 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.218 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583601+0000", 2024-08-17T17:22:23.218 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.218 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.218 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.219 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.219 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583601+0000", 2024-08-17T17:22:23.219 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.219 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.219 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.219 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.219 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583602+0000", 2024-08-17T17:22:23.219 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0470000000000001e-06 2024-08-17T17:22:23.219 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.219 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.219 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.219 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583622+0000", 2024-08-17T17:22:23.219 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0463e-05 2024-08-17T17:22:23.219 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.220 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.220 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.220 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583631+0000", 2024-08-17T17:22:23.220 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0289999999999994e-06 2024-08-17T17:22:23.220 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.220 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.220 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.220 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583637+0000", 2024-08-17T17:22:23.220 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7710000000000003e-06 2024-08-17T17:22:23.220 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.220 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.220 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.221 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583696+0000", 2024-08-17T17:22:23.221 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8801000000000002e-05 2024-08-17T17:22:23.221 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.221 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.221 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.221 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.583875+0000", 2024-08-17T17:22:23.221 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000178596 2024-08-17T17:22:23.221 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.221 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.221 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:22:23.221 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.584044+0000", 2024-08-17T17:22:23.221 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016988699999999999 2024-08-17T17:22:23.222 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.222 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.222 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:23.222 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.588047+0000", 2024-08-17T17:22:23.222 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0040021049999999997 2024-08-17T17:22:23.222 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.222 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.222 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.222 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.592375+0000", 2024-08-17T17:22:23.222 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0043281129999999998 2024-08-17T17:22:23.222 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.222 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.222 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.223 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.656456+0000", 2024-08-17T17:22:23.223 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.064081263999999999 2024-08-17T17:22:23.223 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.223 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.223 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.223 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.657279+0000", 2024-08-17T17:22:23.223 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00082331500000000003 2024-08-17T17:22:23.223 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.223 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.223 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.223 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:14:55.657328+0000", 2024-08-17T17:22:23.223 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.884e-05 2024-08-17T17:22:23.223 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.223 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.223 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.224 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.224 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.224 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:146039 6.4 6:270a08b7:::benchmark_data_smithi063_160268_object19932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1107)", 2024-08-17T17:22:23.224 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:15:20.434951+0000", 2024-08-17T17:22:23.224 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 422.70210052599998, 2024-08-17T17:22:23.224 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.083836758999999997, 2024-08-17T17:22:23.224 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.224 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.224 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.225 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:23.225 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:22:23.225 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:22:23.225 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 146039 2024-08-17T17:22:23.225 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.225 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.225 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.225 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.225 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.434951+0000", 2024-08-17T17:22:23.225 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.225 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.226 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.226 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.226 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.434951+0000", 2024-08-17T17:22:23.226 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.226 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.226 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.226 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.226 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.434951+0000", 2024-08-17T17:22:23.226 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7000000000000003e-07 2024-08-17T17:22:23.226 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.226 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.226 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.226 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.434955+0000", 2024-08-17T17:22:23.226 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5109999999999999e-06 2024-08-17T17:22:23.227 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.227 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.227 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.227 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.434963+0000", 2024-08-17T17:22:23.227 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2549999999999997e-06 2024-08-17T17:22:23.227 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.227 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.227 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.227 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.434968+0000", 2024-08-17T17:22:23.227 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1309999999999997e-06 2024-08-17T17:22:23.227 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.227 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.227 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.227 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.435011+0000", 2024-08-17T17:22:23.228 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3130999999999998e-05 2024-08-17T17:22:23.228 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.228 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.228 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.228 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.435085+0000", 2024-08-17T17:22:23.228 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3689000000000005e-05 2024-08-17T17:22:23.228 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.228 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.228 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:22:23.228 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.435154+0000", 2024-08-17T17:22:23.228 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9207000000000003e-05 2024-08-17T17:22:23.228 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.228 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.229 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:23.229 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.438866+0000", 2024-08-17T17:22:23.229 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0037120999999999999 2024-08-17T17:22:23.229 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.229 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.229 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.229 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.439494+0000", 2024-08-17T17:22:23.229 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00062716699999999998 2024-08-17T17:22:23.229 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.229 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.229 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:23.229 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.518685+0000", 2024-08-17T17:22:23.229 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.079191119000000004 2024-08-17T17:22:23.230 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.230 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.230 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.230 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.518744+0000", 2024-08-17T17:22:23.230 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8971999999999997e-05 2024-08-17T17:22:23.230 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.230 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.230 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.230 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:15:20.518787+0000", 2024-08-17T17:22:23.230 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3606999999999998e-05 2024-08-17T17:22:23.230 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.230 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.230 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.231 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.231 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.231 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:34039 7.9 e1133/1129 7:9f55a860:::benchmark_data_smithi063_196002_object34038:head v 1133'2100, mlcod=1133'2095)", 2024-08-17T17:22:23.231 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:11.452652+0000", 2024-08-17T17:22:23.231 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 251.68439932300001, 2024-08-17T17:22:23.231 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17720034600000001, 2024-08-17T17:22:23.231 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.231 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.231 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.231 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.231 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.231 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.231 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.452652+0000", 2024-08-17T17:22:23.232 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.232 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.232 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.232 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.232 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.452652+0000", 2024-08-17T17:22:23.232 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.232 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.232 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.232 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.232 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.452652+0000", 2024-08-17T17:22:23.232 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5600000000000001e-07 2024-08-17T17:22:23.232 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.232 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.232 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.232 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.452692+0000", 2024-08-17T17:22:23.233 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.0157999999999997e-05 2024-08-17T17:22:23.233 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.233 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.233 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.233 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.452698+0000", 2024-08-17T17:22:23.233 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.066e-06 2024-08-17T17:22:23.233 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.233 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.233 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.233 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.452702+0000", 2024-08-17T17:22:23.234 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7110000000000001e-06 2024-08-17T17:22:23.234 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.234 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.234 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.234 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.453996+0000", 2024-08-17T17:22:23.234 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0012937179999999999 2024-08-17T17:22:23.235 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.235 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.235 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.235 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.454013+0000", 2024-08-17T17:22:23.235 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7680999999999999e-05 2024-08-17T17:22:23.235 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.235 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.236 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.236 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.455127+0000", 2024-08-17T17:22:23.236 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0011138630000000001 2024-08-17T17:22:23.236 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.236 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.236 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.236 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629852+0000", 2024-08-17T17:22:23.236 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17472479299999999 2024-08-17T17:22:23.236 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.236 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.236 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.237 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.237 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.237 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:34051 7.5 e1133/1129 7:a40e9396:::benchmark_data_smithi063_196002_object34050:head v 1133'2196, mlcod=1133'2194)", 2024-08-17T17:22:23.237 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:11.456432+0000", 2024-08-17T17:22:23.237 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 251.68061871, 2024-08-17T17:22:23.237 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17630179500000001, 2024-08-17T17:22:23.237 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.237 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.237 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.237 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.237 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.237 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.238 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456432+0000", 2024-08-17T17:22:23.238 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.238 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.238 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.238 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.238 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456432+0000", 2024-08-17T17:22:23.238 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.238 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.238 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.238 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.238 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456434+0000", 2024-08-17T17:22:23.238 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2300000000000001e-06 2024-08-17T17:22:23.238 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.239 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.239 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.239 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456485+0000", 2024-08-17T17:22:23.239 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1539000000000002e-05 2024-08-17T17:22:23.239 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.239 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.239 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.239 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629879+0000", 2024-08-17T17:22:23.239 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.173393513 2024-08-17T17:22:23.239 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.239 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.239 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.239 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629886+0000", 2024-08-17T17:22:23.239 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9929999999999998e-06 2024-08-17T17:22:23.239 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.240 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.240 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.240 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629949+0000", 2024-08-17T17:22:23.240 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3737000000000003e-05 2024-08-17T17:22:23.240 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.240 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.240 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.240 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629973+0000", 2024-08-17T17:22:23.240 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3646e-05 2024-08-17T17:22:23.240 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.240 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.240 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.240 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.632699+0000", 2024-08-17T17:22:23.240 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0027261220000000001 2024-08-17T17:22:23.241 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.241 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.241 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.241 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.632734+0000", 2024-08-17T17:22:23.241 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5015000000000003e-05 2024-08-17T17:22:23.241 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.241 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.241 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.241 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.241 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.241 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:34053 7.0 e1133/1129 7:0b9bda54:::benchmark_data_smithi063_196002_object34052:head v 1133'2211, mlcod=1133'2209)", 2024-08-17T17:22:23.241 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:11.456547+0000", 2024-08-17T17:22:23.241 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 251.68050409400001, 2024-08-17T17:22:23.241 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.174798225, 2024-08-17T17:22:23.242 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.242 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.242 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.242 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.242 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.242 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.242 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456547+0000", 2024-08-17T17:22:23.242 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.242 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.242 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.242 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.242 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456547+0000", 2024-08-17T17:22:23.243 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.243 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.243 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.243 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.243 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456548+0000", 2024-08-17T17:22:23.243 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.68e-07 2024-08-17T17:22:23.243 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.243 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.243 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.243 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456667+0000", 2024-08-17T17:22:23.243 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011935200000000001 2024-08-17T17:22:23.243 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.243 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.243 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.244 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629846+0000", 2024-08-17T17:22:23.244 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17317944800000001 2024-08-17T17:22:23.244 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.244 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.244 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.244 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629865+0000", 2024-08-17T17:22:23.244 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8340000000000001e-05 2024-08-17T17:22:23.244 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.244 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.244 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.244 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629941+0000", 2024-08-17T17:22:23.244 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6447999999999997e-05 2024-08-17T17:22:23.245 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.245 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.245 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.245 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629973+0000", 2024-08-17T17:22:23.245 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1538000000000002e-05 2024-08-17T17:22:23.245 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.245 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.245 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.245 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.631305+0000", 2024-08-17T17:22:23.245 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0013325819999999999 2024-08-17T17:22:23.245 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.245 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.245 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.245 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.631345+0000", 2024-08-17T17:22:23.246 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9848999999999997e-05 2024-08-17T17:22:23.246 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.246 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.246 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.246 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.246 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.246 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:34055 7.2 e1133/1129 7:4c133eb8:::benchmark_data_smithi063_196002_object34054:head v 1133'2149, mlcod=1133'2147)", 2024-08-17T17:22:23.246 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:11.456861+0000", 2024-08-17T17:22:23.246 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 251.68018981899999, 2024-08-17T17:22:23.246 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17531513200000001, 2024-08-17T17:22:23.246 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.247 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.247 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.247 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.247 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.247 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.247 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456861+0000", 2024-08-17T17:22:23.247 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.247 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.247 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.247 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.247 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456861+0000", 2024-08-17T17:22:23.247 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.248 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.249 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.249 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.249 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456862+0000", 2024-08-17T17:22:23.249 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0100000000000005e-07 2024-08-17T17:22:23.249 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.249 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.249 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.249 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.456886+0000", 2024-08-17T17:22:23.250 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4442000000000001e-05 2024-08-17T17:22:23.250 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.250 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.250 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.250 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629875+0000", 2024-08-17T17:22:23.250 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17298824099999999 2024-08-17T17:22:23.250 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.250 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.250 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.250 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.629886+0000", 2024-08-17T17:22:23.250 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1065e-05 2024-08-17T17:22:23.250 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.251 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.251 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.251 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.630078+0000", 2024-08-17T17:22:23.251 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00019224199999999999 2024-08-17T17:22:23.251 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.251 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.251 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.251 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.630100+0000", 2024-08-17T17:22:23.251 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1676e-05 2024-08-17T17:22:23.251 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.251 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.251 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.251 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.632137+0000", 2024-08-17T17:22:23.251 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0020373840000000002 2024-08-17T17:22:23.252 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.252 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.252 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.252 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:11.632176+0000", 2024-08-17T17:22:23.252 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9480999999999997e-05 2024-08-17T17:22:23.252 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.252 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.252 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.252 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.252 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.252 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88374 7.d e1150/1144 7:b7ece4c7:::benchmark_data_smithi063_196002_object88373:head v 1150'5490, mlcod=0'0)", 2024-08-17T17:22:23.252 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.649423+0000", 2024-08-17T17:22:23.252 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 202.48762768200001, 2024-08-17T17:22:23.252 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.28106146399999998, 2024-08-17T17:22:23.253 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.253 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.253 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:23.253 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.253 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.253 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.253 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649423+0000", 2024-08-17T17:22:23.253 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.253 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.253 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.253 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.253 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649423+0000", 2024-08-17T17:22:23.253 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.254 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.254 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.254 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.254 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649425+0000", 2024-08-17T17:22:23.254 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.186e-06 2024-08-17T17:22:23.254 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.254 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.254 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.254 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649464+0000", 2024-08-17T17:22:23.254 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9591000000000001e-05 2024-08-17T17:22:23.254 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.254 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.255 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.255 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649477+0000", 2024-08-17T17:22:23.255 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.293e-05 2024-08-17T17:22:23.255 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.255 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.255 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.255 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649486+0000", 2024-08-17T17:22:23.255 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4249999999999998e-06 2024-08-17T17:22:23.255 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.255 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.255 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.255 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649719+0000", 2024-08-17T17:22:23.255 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00023391199999999999 2024-08-17T17:22:23.255 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.255 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.256 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.256 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.649782+0000", 2024-08-17T17:22:23.256 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2582000000000005e-05 2024-08-17T17:22:23.256 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.256 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.256 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:23.256 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.930315+0000", 2024-08-17T17:22:23.256 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.28053328 2024-08-17T17:22:23.256 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.256 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.256 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.256 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.930485+0000", 2024-08-17T17:22:23.256 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016955799999999999 2024-08-17T17:22:23.257 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.257 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.257 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.257 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.257 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.257 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88375 7.2 e1150/1129 7:44faffa9:::benchmark_data_smithi063_196002_object88374:head v 1150'5570, mlcod=1150'5567)", 2024-08-17T17:22:23.257 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.652123+0000", 2024-08-17T17:22:23.257 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 202.484927836, 2024-08-17T17:22:23.257 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.29175277300000002, 2024-08-17T17:22:23.257 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:23.257 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:23.258 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:22:23.258 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:23.258 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.258 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:23.258 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652123+0000", 2024-08-17T17:22:23.258 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.258 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.258 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.258 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:23.258 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652123+0000", 2024-08-17T17:22:23.258 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:23.258 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.259 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.259 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:23.259 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652124+0000", 2024-08-17T17:22:23.259 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3099999999999999e-07 2024-08-17T17:22:23.259 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.259 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.259 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:23.259 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652147+0000", 2024-08-17T17:22:23.259 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3261999999999999e-05 2024-08-17T17:22:23.259 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.259 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.259 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:23.260 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652166+0000", 2024-08-17T17:22:23.260 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9517e-05 2024-08-17T17:22:23.260 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.260 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.260 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:23.260 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652175+0000", 2024-08-17T17:22:23.260 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6100000000000006e-06 2024-08-17T17:22:23.260 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.260 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.260 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:23.260 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652554+0000", 2024-08-17T17:22:23.260 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00037904699999999998 2024-08-17T17:22:23.260 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.261 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.261 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:23.261 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.652575+0000", 2024-08-17T17:22:23.261 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.069e-05 2024-08-17T17:22:23.261 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:23.261 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:23.261 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:23.261 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.943876+0000", 2024-08-17T17:22:23.261 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.29130141599999998 2024-08-17T17:22:23.261 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.261 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.261 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.261 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:23.261 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:23.262 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:22:23.287 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:23.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:23.389 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:23.414 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:23.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:23.516 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:23.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.543 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:23.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:23.644 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:23.667 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:23.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:23.769 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:23.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:22:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:22:23] "GET /metrics HTTP/1.1" 200 36320 "" "Prometheus/2.51.0" 2024-08-17T17:22:23.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:23 smithi063 ceph-mon[114756]: pgmap v1512: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:23.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:23 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/540170619' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:22:23.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:23 smithi063 ceph-mon[114756]: osdmap e1199: 8 total, 8 up, 8 in 2024-08-17T17:22:23.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:23 smithi063 ceph-mon[109843]: pgmap v1512: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:23.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:23 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/540170619' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:22:23.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:23 smithi063 ceph-mon[109843]: osdmap e1199: 8 total, 8 up, 8 in 2024-08-17T17:22:23.794 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:23.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:23 smithi162 ceph-mon[104668]: pgmap v1512: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:23 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/540170619' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:22:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:23 smithi162 ceph-mon[104668]: osdmap e1199: 8 total, 8 up, 8 in 2024-08-17T17:22:23.896 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:23.918 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:23.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:23.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:24.020 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:24.046 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:24.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:24.148 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:24.172 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:24.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:24.274 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:24.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:24.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:24.372 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:22:24.400 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:24.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.424 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:24.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:24.482 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:22:24.526 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:24.549 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:24.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:24.651 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:24.677 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:24.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:24.686 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:24 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2277686436' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:24.686 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:24 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:24.686 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:24 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1755595907' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:22:24.686 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:24 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2277686436' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:24.686 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:24 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:24.686 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:24 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1755595907' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:22:24.779 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:24.802 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:24.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:24.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:24 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2277686436' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:24.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:24 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:24.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:24 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1755595907' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:22:24.903 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:24.929 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:24.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:24.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:25.031 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:25.053 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:25.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:25.155 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:25.180 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:25.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:25.198 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:22:25.281 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:25.304 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:25.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:25.406 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:25.431 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:25.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:25.533 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:25.554 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:25.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:25.656 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:25.678 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:25.679 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:25 smithi063 ceph-mon[114756]: pgmap v1514: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:25.679 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:25 smithi063 ceph-mon[114756]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:22:25.679 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:25 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:22:25.679 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:25 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:25.679 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:25 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1755595907' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:22:25.679 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:25 smithi063 ceph-mon[114756]: osdmap e1200: 8 total, 8 up, 8 in 2024-08-17T17:22:25.679 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:25 smithi063 ceph-mon[109843]: pgmap v1514: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:25.679 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:25 smithi063 ceph-mon[109843]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:22:25.679 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:25 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:22:25.679 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:25 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:25.679 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:25 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1755595907' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:22:25.680 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:25 smithi063 ceph-mon[109843]: osdmap e1200: 8 total, 8 up, 8 in 2024-08-17T17:22:25.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:25.781 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:25.806 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:25.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:25.841 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:22:25.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:25 smithi162 ceph-mon[104668]: pgmap v1514: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:25 smithi162 ceph-mon[104668]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:22:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:25 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:22:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:25 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:25 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1755595907' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:22:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:25 smithi162 ceph-mon[104668]: osdmap e1200: 8 total, 8 up, 8 in 2024-08-17T17:22:25.908 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:25.933 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:25.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:25.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:26.035 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:26.058 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:26.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:26.159 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:26.181 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:26.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:26.283 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:26.305 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:26.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:26.407 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:26.431 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:26.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:26.446 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:26.533 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:26.555 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:26.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:26.657 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:26.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.681 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:26.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:26.782 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:26.804 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:26.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:26.906 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:26.931 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:26.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:26.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:27.033 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:27.056 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:27.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:27.084 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:22:27.159 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:27.176 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:27.177 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:27.183 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:27.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:27.285 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:27.307 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:27.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:27.409 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:27.432 DEBUG:teuthology.orchestra.run.smithi063:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-08-17T17:22:27.435 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:27.436 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:22:27.436 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:22:27.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:27.492 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:hints = 1 2024-08-17T17:22:27.492 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:22:27.492 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Object prefix: benchmark_data_smithi063_230025 2024-08-17T17:22:27.493 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:22:27.493 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:22:27.539 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:27.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.584 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:27.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:27.657 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:27 smithi063 ceph-mon[109843]: pgmap v1516: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:27.658 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:27 smithi063 ceph-mon[114756]: pgmap v1516: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:27.684 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:27.722 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:27.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:27.824 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:27.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:27 smithi162 ceph-mon[104668]: pgmap v1516: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:27.862 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:27.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:27.964 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:27.997 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:27.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:27.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:28.099 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:28.136 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:28.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:28.238 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:28.270 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:28.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:28.372 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:28.413 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:28.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:28.493 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 1 16 3808 3792 236.989 237 0.00179149 0.00419896 2024-08-17T17:22:28.515 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:28.548 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:28.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:28.650 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:28.687 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:28.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:28.789 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:28.820 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:28.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:28.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:28.959 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:28.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:28.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:29.061 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:29.090 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:29.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:29.192 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:29.233 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:29.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:29.334 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:29.360 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:29.390 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:29.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:29.491 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:29.495 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 2 16 7402 7386 230.796 224.625 0.00551573 0.00432566 2024-08-17T17:22:29.531 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:29.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.532 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:29.633 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:29.670 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:29.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:29.771 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:29.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:29 smithi063 ceph-mon[109843]: pgmap v1517: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:29.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:29 smithi063 ceph-mon[114756]: pgmap v1517: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:29.807 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:29.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:29.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:29 smithi162 ceph-mon[104668]: pgmap v1517: 99 pgs: 99 active+clean; 582 KiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:22:29.909 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:29.929 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:29.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:29.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:30.031 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:30.070 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:30.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:30.172 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:30.203 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:30.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:30.305 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:30.344 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:30.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.345 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:30.446 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:30.479 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:30.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:30.493 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 3 15 10389 10374 216.108 186.75 0.00331433 0.00462076 2024-08-17T17:22:30.581 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:30.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.614 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:30.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:30.715 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:30.751 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:30.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:30.852 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:30.892 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:30.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:30.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:30.994 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:31.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.026 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:31.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:31.127 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:31.156 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:22:31.157 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:22:31.157 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.157 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45218.0:12434 8.1 8:840a876e:::benchmark_data_smithi063_230025_object12433:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1200)", 2024-08-17T17:22:31.157 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:22:31.152997+0000", 2024-08-17T17:22:31.157 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.001699248, 2024-08-17T17:22:31.157 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001734424, 2024-08-17T17:22:31.157 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:31.157 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:31.157 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:22:31.157 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:31.157 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45218", 2024-08-17T17:22:31.157 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:22:31.158 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 12434 2024-08-17T17:22:31.158 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.158 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:31.158 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.158 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:31.158 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.152997+0000", 2024-08-17T17:22:31.158 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:31.158 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.158 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.158 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:31.158 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.152997+0000", 2024-08-17T17:22:31.158 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:31.159 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.159 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.159 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:31.159 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.152998+0000", 2024-08-17T17:22:31.159 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2300000000000001e-06 2024-08-17T17:22:31.159 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.159 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.159 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:31.159 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.153022+0000", 2024-08-17T17:22:31.159 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3519e-05 2024-08-17T17:22:31.159 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.160 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.160 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:31.160 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.153037+0000", 2024-08-17T17:22:31.160 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5019e-05 2024-08-17T17:22:31.160 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.160 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.160 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:31.160 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.153047+0000", 2024-08-17T17:22:31.160 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0311e-05 2024-08-17T17:22:31.160 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.160 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.160 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:31.161 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.153154+0000", 2024-08-17T17:22:31.161 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000106746 2024-08-17T17:22:31.161 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.161 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.161 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:31.161 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.153303+0000", 2024-08-17T17:22:31.161 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014928399999999999 2024-08-17T17:22:31.161 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.161 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.161 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:22:31.161 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.153440+0000", 2024-08-17T17:22:31.161 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013655500000000001 2024-08-17T17:22:31.162 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:31.162 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:31.162 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:31.162 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.162 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.162 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45218.0:12438 8.a 8:57bbf20c:::benchmark_data_smithi063_230025_object12437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1200)", 2024-08-17T17:22:31.162 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:22:31.153085+0000", 2024-08-17T17:22:31.162 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.001611777, 2024-08-17T17:22:31.162 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0017227639999999999, 2024-08-17T17:22:31.162 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:31.162 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:31.162 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:22:31.163 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:31.163 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45218", 2024-08-17T17:22:31.163 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:22:31.163 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 12438 2024-08-17T17:22:31.163 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.163 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:31.163 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.163 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:31.163 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.153085+0000", 2024-08-17T17:22:31.163 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:31.163 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.163 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.163 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:31.164 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.153085+0000", 2024-08-17T17:22:31.164 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:31.164 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.164 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.164 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:31.164 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.153085+0000", 2024-08-17T17:22:31.164 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0100000000000004e-07 2024-08-17T17:22:31.164 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.164 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.164 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:31.164 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.153124+0000", 2024-08-17T17:22:31.164 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9079999999999999e-05 2024-08-17T17:22:31.165 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.165 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.165 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:31.165 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.153135+0000", 2024-08-17T17:22:31.165 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0186999999999999e-05 2024-08-17T17:22:31.165 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.165 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.165 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:31.165 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.153141+0000", 2024-08-17T17:22:31.165 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3810000000000004e-06 2024-08-17T17:22:31.165 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.165 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.165 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:31.166 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.153205+0000", 2024-08-17T17:22:31.166 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4468999999999995e-05 2024-08-17T17:22:31.166 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.166 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.166 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:31.166 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.153351+0000", 2024-08-17T17:22:31.166 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014547 2024-08-17T17:22:31.166 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:31.166 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:31.166 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,5", 2024-08-17T17:22:31.166 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:22:31.153484+0000", 2024-08-17T17:22:31.166 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013356800000000001 2024-08-17T17:22:31.166 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:31.167 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:31.167 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:31.167 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:31.167 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-08-17T17:22:31.167 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 2 2024-08-17T17:22:31.167 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:22:31.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.168 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:31.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:31.269 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:31.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:31.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:31.401 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:31.435 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:31.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:31.493 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 4 16 13489 13473 210.499 193.688 0.00582302 0.00474449 2024-08-17T17:22:31.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:31 smithi063 ceph-mon[114756]: pgmap v1518: 99 pgs: 99 active+clean; 177 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s wr, 352 op/s 2024-08-17T17:22:31.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:31.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:31.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:31 smithi063 ceph-mon[109843]: pgmap v1518: 99 pgs: 99 active+clean; 177 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s wr, 352 op/s 2024-08-17T17:22:31.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:31.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:31.536 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:31.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.583 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:31.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:31.684 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:31.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.716 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:31.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:31.817 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:31.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:31 smithi162 ceph-mon[104668]: pgmap v1518: 99 pgs: 99 active+clean; 177 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s wr, 352 op/s 2024-08-17T17:22:31.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:31.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:31.855 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:31.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:31.956 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:31.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:31.994 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:31.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:32.094 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:32.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.134 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:32.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:32.235 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:32.263 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:32.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:32.365 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:32.398 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:32.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:32.493 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 5 16 16553 16537 206.695 191.5 0.00245752 0.00483224 2024-08-17T17:22:32.500 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:32.532 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:32.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:32.634 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:32.665 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:32.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:32.768 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:32.807 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:32.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:32.910 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:32.946 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:32.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:32.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:33.048 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:33.088 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:33.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:33.190 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:33.223 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:33.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:33.325 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:33.368 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:33.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:33.394 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:22:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:22:33] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.51.0" 2024-08-17T17:22:33.449 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:22:33.449 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:22:33.450 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:22:33.450 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:22:33.470 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:33.478 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:22:33.478 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:22:33.493 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 6 16 19324 19308 201.108 173.188 0.0138202 0.00496356 2024-08-17T17:22:33.509 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:33.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:33.515 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:22:33.515 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:22:33.548 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:22:33.549 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:22:33.584 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:22:33.585 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:22:33.611 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:33.624 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:22:33.624 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:22:33.641 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:33.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:33.647 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:33 smithi162 ceph-mon[104668]: pgmap v1519: 99 pgs: 99 active+clean; 577 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s 2024-08-17T17:22:33.647 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:33 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2648380900' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:33.663 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:22:33.663 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:22:33.664 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:33 smithi063 ceph-mon[109843]: pgmap v1519: 99 pgs: 99 active+clean; 577 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s 2024-08-17T17:22:33.664 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:33 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2648380900' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:33.665 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:33 smithi063 ceph-mon[114756]: pgmap v1519: 99 pgs: 99 active+clean; 577 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s 2024-08-17T17:22:33.665 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:33 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2648380900' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:33.742 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:33.777 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:33.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:33.879 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:33.913 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:33.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:33.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:34.014 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:34.051 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:34.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:34.153 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:34.184 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:34.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:34.286 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:34.325 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:34.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:34.427 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:34.461 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:34.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:34.493 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 7 16 22008 21992 196.34 167.75 0.00462409 0.00508753 2024-08-17T17:22:34.563 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:34.601 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:34.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:34.702 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:34.742 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:34.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:34.843 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:34.877 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:34.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:34.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:34.979 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:35.011 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:35.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:35.113 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:35.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.143 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:35.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:35.244 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:35.275 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:35.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:35.377 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:35.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.417 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:35.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:35.493 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 8 16 24510 24494 191.343 156.375 0.00346751 0.00522109 2024-08-17T17:22:35.519 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:35.551 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:35.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:35.652 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:35.686 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:35.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:35.742 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:35 smithi063 ceph-mon[109843]: pgmap v1520: 99 pgs: 99 active+clean; 827 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-08-17T17:22:35.742 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:35 smithi063 ceph-mon[114756]: pgmap v1520: 99 pgs: 99 active+clean; 827 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-08-17T17:22:35.786 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:22:35.788 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:35.828 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:35.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:35.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:35 smithi162 ceph-mon[104668]: pgmap v1520: 99 pgs: 99 active+clean; 827 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-08-17T17:22:35.930 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:35.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:35.977 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:35.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:36.078 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:36.109 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:36.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:36.211 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:36.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.260 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:36.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:36.361 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:36.393 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:36.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:36.412 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:22:36.494 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 9 16 26836 26820 186.234 145.375 0.00591509 0.00536362 2024-08-17T17:22:36.495 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:36.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.530 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:36.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:36.631 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:36.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.673 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:36.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:36.774 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:36.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:36 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2668793165' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:22:36.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:36 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:22:36.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:36 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2668793165' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:22:36.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:36 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:22:36.804 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:36.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:36.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:36 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2668793165' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:22:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:36 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:22:36.906 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:36.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:36.942 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:36.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:37.043 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:37.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.081 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:37.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:37.181 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:37.214 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:37.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:37.315 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:37.346 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:37.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:37.439 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:22:37.452 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:37.482 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:37.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:37.494 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 10 16 29152 29136 182.084 144.75 0.00451996 0.00548571 2024-08-17T17:22:37.583 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:37.617 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:37.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:37.716 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:37 smithi063 ceph-mon[114756]: pgmap v1521: 99 pgs: 99 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:22:37.716 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:37 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:22:37.716 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:37 smithi063 ceph-mon[114756]: osdmap e1201: 8 total, 8 up, 8 in 2024-08-17T17:22:37.716 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:37 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2468123382' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:22:37.716 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:37 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:22:37.717 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:37 smithi063 ceph-mon[109843]: pgmap v1521: 99 pgs: 99 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:22:37.717 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:37 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:22:37.717 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:37 smithi063 ceph-mon[109843]: osdmap e1201: 8 total, 8 up, 8 in 2024-08-17T17:22:37.717 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:37 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2468123382' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:22:37.717 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:37 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:22:37.718 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:37.749 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:37.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:37.761 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:22:37.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:37 smithi162 ceph-mon[104668]: pgmap v1521: 99 pgs: 99 active+clean; 1.1 GiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:22:37.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:37 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:22:37.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:37 smithi162 ceph-mon[104668]: osdmap e1201: 8 total, 8 up, 8 in 2024-08-17T17:22:37.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:37 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2468123382' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:22:37.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:37 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:22:37.851 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:37.900 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:37.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:37.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:38.001 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:38.024 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:38.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:38.125 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:38.160 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:38.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:38.261 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:38.299 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:38.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:38.401 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:38.433 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:38.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:38.494 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 11 16 31312 31296 177.803 135 0.00613108 0.00561783 2024-08-17T17:22:38.534 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:38.539 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:22:38.560 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:38.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:38.661 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:38.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.695 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:38.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:38.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:38 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:38.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:38 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:22:38.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:38 smithi063 ceph-mon[109843]: osdmap e1202: 8 total, 8 up, 8 in 2024-08-17T17:22:38.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:38 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:38.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:38 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:22:38.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:38 smithi063 ceph-mon[114756]: osdmap e1202: 8 total, 8 up, 8 in 2024-08-17T17:22:38.796 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:38.827 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:38.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:38.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:38 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:38.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:38 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:22:38.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:38 smithi162 ceph-mon[104668]: osdmap e1202: 8 total, 8 up, 8 in 2024-08-17T17:22:38.929 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:38.967 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:38.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:38.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:39.069 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:39.106 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:39.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:39.208 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:39.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.246 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:39.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:39.347 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:39.379 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:39.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:39.480 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:39.494 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 12 16 33276 33260 173.214 122.75 0.00669203 0.005768 2024-08-17T17:22:39.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.512 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:39.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:39.613 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:39.644 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:39.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:39.746 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:39.768 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:39.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:39.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:39 smithi063 ceph-mon[114756]: pgmap v1524: 99 pgs: 99 active+clean; 1.6 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 186 MiB/s wr, 2.97k op/s 2024-08-17T17:22:39.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:39 smithi063 ceph-mon[109843]: pgmap v1524: 99 pgs: 99 active+clean; 1.6 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 186 MiB/s wr, 2.97k op/s 2024-08-17T17:22:39.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:39 smithi162 ceph-mon[104668]: pgmap v1524: 99 pgs: 99 active+clean; 1.6 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 186 MiB/s wr, 2.97k op/s 2024-08-17T17:22:39.870 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:39.894 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:39.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:39.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:39.995 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:40.033 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:40.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:40.135 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:40.179 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:40.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:40.281 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:40.322 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:40.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:40.424 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:40.468 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:40.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:40.496 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 13 16 35025 35009 168.274 109.312 0.0110476 0.00593595 2024-08-17T17:22:40.514 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:40.570 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:40.600 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:40.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:40.702 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:40.731 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:40.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:40.833 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:40.839 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:40.877 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:40.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:40.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:40.979 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:41.025 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:41.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:41.127 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:41.159 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:41.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:41.261 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:41.292 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:41.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:41.393 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:41.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.433 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:41.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:41.496 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 14 16 37104 37088 165.535 129.938 0.0141975 0.00603331 2024-08-17T17:22:41.534 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:41.574 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:41.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:41.639 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:41 smithi063 ceph-mon[114756]: pgmap v1525: 99 pgs: 99 active+clean; 1.8 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 162 MiB/s wr, 2.59k op/s 2024-08-17T17:22:41.640 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:41 smithi063 ceph-mon[109843]: pgmap v1525: 99 pgs: 99 active+clean; 1.8 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 162 MiB/s wr, 2.59k op/s 2024-08-17T17:22:41.676 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:41.709 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:41.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:41.811 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:41.841 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:41.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:41.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:41 smithi162 ceph-mon[104668]: pgmap v1525: 99 pgs: 99 active+clean; 1.8 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 162 MiB/s wr, 2.59k op/s 2024-08-17T17:22:41.944 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:41.971 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:41.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:41.985 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:41.985 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:42.086 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:42.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.129 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:42.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:42.230 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:42.265 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:42.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:42.367 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:42.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.399 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:42.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:42.458 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:22:42.496 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 15 16 39149 39133 163.02 127.812 0.00729262 0.00612772 2024-08-17T17:22:42.500 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:42.528 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:42.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:42.630 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:42.657 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:42.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:42.759 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:42.789 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:42.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:42.891 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:42.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:42.929 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:42.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:43.030 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:43.060 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:43.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:43.161 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:43.193 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:43.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:43.295 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:43.326 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:43.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:43.428 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:43.461 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:43.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:43.496 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 16 16 41059 41043 160.292 119.375 0.00397227 0.00623322 2024-08-17T17:22:43.513 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:22:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:22:43] "GET /metrics HTTP/1.1" 200 36377 "" "Prometheus/2.51.0" 2024-08-17T17:22:43.513 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:43 smithi063 ceph-mon[109843]: pgmap v1526: 99 pgs: 99 active+clean; 2.1 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 166 MiB/s wr, 2.65k op/s 2024-08-17T17:22:43.514 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:43 smithi063 ceph-mon[114756]: pgmap v1526: 99 pgs: 99 active+clean; 2.1 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 166 MiB/s wr, 2.65k op/s 2024-08-17T17:22:43.562 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:43.598 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:43.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:43.700 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:43.736 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:43.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:43.838 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:43.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:43 smithi162 ceph-mon[104668]: pgmap v1526: 99 pgs: 99 active+clean; 2.1 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 166 MiB/s wr, 2.65k op/s 2024-08-17T17:22:43.869 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:43.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:43.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:43.970 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:44.003 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:44.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:44.105 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:44.142 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:44.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:44.243 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:44.277 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:44.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:44.379 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:44.421 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:44.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:44.496 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 17 16 43045 43029 158.164 124.125 0.0106574 0.00631691 2024-08-17T17:22:44.524 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:44.561 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:44.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:44.663 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:44.712 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:44.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:44.736 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:22:44.736 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:22:44.814 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:44.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.857 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:44.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:44.958 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:44.990 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:44.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:44.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:45.092 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:45.112 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:45.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:45.214 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:45.253 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:45.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:45.355 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:45.377 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:45.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:45.479 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:45.496 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 18 16 44602 44586 154.783 97.3125 0.0068045 0.00645582 2024-08-17T17:22:45.519 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:45.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:45.621 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:45.661 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:45.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:45.763 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:45.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:45 smithi063 ceph-mon[114756]: pgmap v1527: 99 pgs: 99 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-08-17T17:22:45.783 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:45 smithi063 ceph-mon[109843]: pgmap v1527: 99 pgs: 99 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-08-17T17:22:45.802 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:45.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:45.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:45 smithi162 ceph-mon[104668]: pgmap v1527: 99 pgs: 99 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 150 MiB/s wr, 2.40k op/s 2024-08-17T17:22:45.904 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:45.935 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:45.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:45.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:46.037 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:46.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.082 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:46.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:46.182 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:46.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.229 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:46.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:46.330 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:46.366 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:22:46.366 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:22:46.371 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:46.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:46.640 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:22:46.640 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:22:46.640 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:22:46.641 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:22:46.641 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:22:46.641 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:22:46.642 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 19 16 46433 46417 152.659 114.438 0.00490805 0.00654458 2024-08-17T17:22:46.642 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:22:46.642 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:22:46.643 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:22:46.643 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:22:46.643 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2363961534' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:46.643 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:46.643 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]} : dispatch 2024-08-17T17:22:46.643 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-08-17T17:22:46.643 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]} : dispatch 2024-08-17T17:22:46.644 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]} : dispatch 2024-08-17T17:22:46.644 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-08-17T17:22:46.644 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]} : dispatch 2024-08-17T17:22:46.644 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:46.644 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2363961534' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:46.644 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:46.644 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]} : dispatch 2024-08-17T17:22:46.644 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-08-17T17:22:46.645 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]} : dispatch 2024-08-17T17:22:46.645 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]} : dispatch 2024-08-17T17:22:46.645 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-08-17T17:22:46.645 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]} : dispatch 2024-08-17T17:22:46.645 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:46.645 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:22:46.645 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:22:46.646 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:22:46.646 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:22:46.740 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:46.769 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:46.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:46.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:46 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2363961534' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:22:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:22:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]} : dispatch 2024-08-17T17:22:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-08-17T17:22:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]} : dispatch 2024-08-17T17:22:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]} : dispatch 2024-08-17T17:22:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-08-17T17:22:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]} : dispatch 2024-08-17T17:22:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:22:46.872 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:46.910 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:46.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:46.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:47.012 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:47.048 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:47.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:47.150 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:47.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.185 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:47.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:47.286 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:47.321 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:47.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:47.423 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:47.455 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:47.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:47.496 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:22:47.496799+0000 min lat: 0.00109066 max lat: 0.153833 avg lat: 0.00664499 2024-08-17T17:22:47.497 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:22:47.497 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 20 16 48143 48127 150.37 106.875 0.00689954 0.00664499 2024-08-17T17:22:47.556 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:47.587 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:47.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:47.689 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:47.712 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:47.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:47.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:47 smithi063 ceph-mon[114756]: pgmap v1528: 99 pgs: 99 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-08-17T17:22:47.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-08-17T17:22:47.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-08-17T17:22:47.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-08-17T17:22:47.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:47 smithi063 ceph-mon[114756]: osdmap e1203: 8 total, 8 up, 8 in 2024-08-17T17:22:47.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:47 smithi063 ceph-mon[109843]: pgmap v1528: 99 pgs: 99 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-08-17T17:22:47.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-08-17T17:22:47.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-08-17T17:22:47.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-08-17T17:22:47.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:47 smithi063 ceph-mon[109843]: osdmap e1203: 8 total, 8 up, 8 in 2024-08-17T17:22:47.814 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:47.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:47 smithi162 ceph-mon[104668]: pgmap v1528: 99 pgs: 99 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2024-08-17T17:22:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2024-08-17T17:22:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-08-17T17:22:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-08-17T17:22:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:47 smithi162 ceph-mon[104668]: osdmap e1203: 8 total, 8 up, 8 in 2024-08-17T17:22:47.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.852 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:47.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:47.953 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:47.994 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:47.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:47.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:48.096 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:48.134 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:48.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:48.235 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:48.270 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:48.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:48.371 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:48.401 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:48.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:48.497 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 21 16 49766 49750 148.04 101.438 0.0578443 0.0067467 2024-08-17T17:22:48.503 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:48.541 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:48.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:48.642 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:48.666 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:48.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:48.768 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:48.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.805 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:48.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:48.906 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:48.940 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:48.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:48.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:49.042 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:49.064 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:49.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:49.166 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:49.195 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:49.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:49.297 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:49.331 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:49.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:49.433 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:49.454 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:49.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:49.497 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 22 16 51057 51041 144.978 80.6875 0.00165668 0.00685479 2024-08-17T17:22:49.556 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:49.595 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:49.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:49.698 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:49.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.731 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:49.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:49.756 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:49 smithi063 ceph-mon[109843]: pgmap v1530: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-08-17T17:22:49.756 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:49 smithi063 ceph-mon[109843]: osdmap e1204: 8 total, 8 up, 8 in 2024-08-17T17:22:49.757 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:49 smithi063 ceph-mon[114756]: pgmap v1530: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-08-17T17:22:49.757 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:49 smithi063 ceph-mon[114756]: osdmap e1204: 8 total, 8 up, 8 in 2024-08-17T17:22:49.833 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:49 smithi162 ceph-mon[104668]: pgmap v1530: 99 pgs: 99 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 123 MiB/s wr, 1.97k op/s 2024-08-17T17:22:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:49 smithi162 ceph-mon[104668]: osdmap e1204: 8 total, 8 up, 8 in 2024-08-17T17:22:49.854 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:49.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:49.956 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:49.987 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:49.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:49.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:50.089 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:50.120 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:50.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:50.222 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:50.262 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:50.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:50.363 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:50.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.399 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:50.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:50.497 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 23 16 52534 52518 142.688 92.3125 0.00678507 0.00700347 2024-08-17T17:22:50.500 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:50.537 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:50.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:50.639 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:50.681 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:50.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:50.783 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:50.823 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:50.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:50.925 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:50.967 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:50.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:50.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:51.069 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:51.113 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:51.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:51.215 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:51.246 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:51.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:51.349 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:51.381 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:51.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.382 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:51.483 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:51.497 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 24 16 54165 54149 140.99 101.938 0.0148673 0.0070875 2024-08-17T17:22:51.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.515 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:51.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:51.618 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:51.650 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:51.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:51.752 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:51.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:51 smithi063 ceph-mon[114756]: pgmap v1532: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-08-17T17:22:51.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:51 smithi063 ceph-mon[109843]: pgmap v1532: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-08-17T17:22:51.782 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:51.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:51 smithi162 ceph-mon[104668]: pgmap v1532: 99 pgs: 99 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2024-08-17T17:22:51.885 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:51.923 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:51.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:51.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:52.025 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:52.043 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:52.072 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:22:52.083 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:52.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:52.186 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:52.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.222 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:52.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:52.323 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:52.378 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:22:52.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.379 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:52.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:52.481 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:52.499 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 25 16 55728 55712 139.247 97.6875 0.00910687 0.00717624 2024-08-17T17:22:52.522 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:52.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:52.625 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:52.665 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:52.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:52.733 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:52 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1929884945' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:22:52.733 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:52 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:22:52.734 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:52 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1929884945' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:22:52.734 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:52 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:22:52.767 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:52.798 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:52.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:52.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:52 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1929884945' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:22:52.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:52 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:22:52.899 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:52.927 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:52.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:52.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:53.029 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:53.068 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:53.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:53.162 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:22:53.162 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:22:53.170 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:53.200 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:53.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:53.302 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:53.331 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:53.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:53.392 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:22:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:22:53] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.51.0" 2024-08-17T17:22:53.433 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:53.474 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:53.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:53.499 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 26 16 57326 57310 137.732 99.875 0.00342703 0.00725493 2024-08-17T17:22:53.576 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:53.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.630 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:53.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:53.651 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:53.731 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:53.766 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:53.767 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:53 smithi063 ceph-mon[114756]: pgmap v1533: 99 pgs: 99 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-08-17T17:22:53.767 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:53 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:22:53.767 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:53 smithi063 ceph-mon[114756]: osdmap e1205: 8 total, 8 up, 8 in 2024-08-17T17:22:53.768 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:53 smithi063 ceph-mon[109843]: pgmap v1533: 99 pgs: 99 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-08-17T17:22:53.768 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:53 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:22:53.768 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:53 smithi063 ceph-mon[109843]: osdmap e1205: 8 total, 8 up, 8 in 2024-08-17T17:22:53.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:53.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:53 smithi162 ceph-mon[104668]: pgmap v1533: 99 pgs: 99 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 119 MiB/s wr, 1.91k op/s 2024-08-17T17:22:53.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:53 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:22:53.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:53 smithi162 ceph-mon[104668]: osdmap e1205: 8 total, 8 up, 8 in 2024-08-17T17:22:53.869 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:53.892 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:22:53.903 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:53.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:53.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:54.005 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:54.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.036 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:54.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:54.137 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:54.171 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:54.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:54.261 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:54.272 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:54.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:54.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:54.401 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:54.433 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:54.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:54.499 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 27 16 58811 58795 136.068 92.8125 0.0158297 0.0073435 2024-08-17T17:22:54.535 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:54.582 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:54.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:54.684 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:54.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.715 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:54.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:54.816 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:54.857 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:54.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:54.959 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:54.997 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:54.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:54.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:55.099 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:55.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.129 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:55.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:55.231 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:55.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.271 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:55.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:55.372 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:55.407 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:55.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:55.499 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 28 16 60315 60299 134.566 94 0.0099014 0.00742598 2024-08-17T17:22:55.509 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:55.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.548 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:55.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:55.649 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:55.654 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:55 smithi063 ceph-mon[114756]: pgmap v1535: 99 pgs: 99 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-17T17:22:55.656 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:55 smithi063 ceph-mon[109843]: pgmap v1535: 99 pgs: 99 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-17T17:22:55.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.679 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:55.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:55.780 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:55.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.828 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:55.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:55.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:55 smithi162 ceph-mon[104668]: pgmap v1535: 99 pgs: 99 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-08-17T17:22:55.929 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:55.954 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:55.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:55.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:56.056 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:56.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.097 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:56.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:56.198 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:56.225 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:56.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:56.327 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:56.356 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:56.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:56.412 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:22:56.457 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:56.496 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:56.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:56.499 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 29 16 61513 61497 132.507 74.875 0.0120059 0.00754175 2024-08-17T17:22:56.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:56 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/396850997' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:56.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:56 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/396850997' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:56.598 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:56.635 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:56.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:56.737 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:56.765 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:56.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:56.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:56 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/396850997' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:22:56.867 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:56.909 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:56.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:56.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:57.011 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:57.050 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:57.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:57.152 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:57.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.202 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:57.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:57.303 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:57.348 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:57.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:57.450 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:57.491 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:57.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:57.499 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 30 16 62916 62900 131.013 87.6875 0.00495701 0.00762743 2024-08-17T17:22:57.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:57 smithi063 ceph-mon[114756]: pgmap v1536: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-08-17T17:22:57.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:57 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:57.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:57 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/396850997' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:57.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:57 smithi063 ceph-mon[114756]: osdmap e1206: 8 total, 8 up, 8 in 2024-08-17T17:22:57.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:57 smithi063 ceph-mon[109843]: pgmap v1536: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-08-17T17:22:57.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:57 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:57.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:57 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/396850997' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:57.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:57 smithi063 ceph-mon[109843]: osdmap e1206: 8 total, 8 up, 8 in 2024-08-17T17:22:57.593 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:57.629 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:57.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:57.731 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:57.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.780 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:57.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:57 smithi162 ceph-mon[104668]: pgmap v1536: 99 pgs: 99 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-08-17T17:22:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:57 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:22:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:57 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/396850997' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:22:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:57 smithi162 ceph-mon[104668]: osdmap e1206: 8 total, 8 up, 8 in 2024-08-17T17:22:57.881 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:57.922 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:57.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:57.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:58.024 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:22:58.056 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:58.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:22:58.158 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:58.190 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:58.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:58.292 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:58.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.334 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:58.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:22:58.358 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:22:58.435 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:58.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.497 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:58.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:58.500 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 31 16 64376 64360 129.729 91.25 0.00905521 0.0077034 2024-08-17T17:22:58.598 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:22:58.624 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:58.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:22:58.726 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:58.765 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:58.866 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:58.905 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:58.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:58.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:59.007 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:22:59.045 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:59.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:22:59.147 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:22:59.180 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:59.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:22:59.211 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:22:58 smithi063 ceph-mon[114756]: pgmap v1538: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:22:59.211 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:22:58 smithi063 ceph-mon[109843]: pgmap v1538: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:22:59.254 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:22:59.281 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:59.293 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:22:59.294 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:22:59.294 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:22:59.294 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:22:59.294 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.294 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35809 6.f 6:f6d7b118:::benchmark_data_smithi063_160268_object35808:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:22:59.294 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.876618+0000", 2024-08-17T17:22:59.294 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 558.41277103799996, 2024-08-17T17:22:59.294 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14764386800000001, 2024-08-17T17:22:59.294 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.294 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.295 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:59.295 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:59.295 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:22:59.295 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:22:59.295 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35809 2024-08-17T17:22:59.295 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.295 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.295 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.295 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.295 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876618+0000", 2024-08-17T17:22:59.295 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.295 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.296 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.296 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.296 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876618+0000", 2024-08-17T17:22:59.296 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.296 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.296 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.296 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.296 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876620+0000", 2024-08-17T17:22:59.296 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.795e-06 2024-08-17T17:22:59.296 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.296 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.296 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.296 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876643+0000", 2024-08-17T17:22:59.297 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.296e-05 2024-08-17T17:22:59.297 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.297 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.297 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.297 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876654+0000", 2024-08-17T17:22:59.297 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.13e-05 2024-08-17T17:22:59.297 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.297 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.297 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.297 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876661+0000", 2024-08-17T17:22:59.297 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0269999999999998e-06 2024-08-17T17:22:59.297 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.298 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.298 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.298 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876712+0000", 2024-08-17T17:22:59.298 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1388999999999999e-05 2024-08-17T17:22:59.298 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.298 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.298 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.298 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876826+0000", 2024-08-17T17:22:59.298 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000113377 2024-08-17T17:22:59.298 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.298 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.298 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:22:59.299 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.876960+0000", 2024-08-17T17:22:59.299 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000134103 2024-08-17T17:22:59.299 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.299 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.299 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.299 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.879644+0000", 2024-08-17T17:22:59.299 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0026843520000000001 2024-08-17T17:22:59.299 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.299 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.299 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:59.299 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.879678+0000", 2024-08-17T17:22:59.299 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3433999999999999e-05 2024-08-17T17:22:59.299 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.300 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.300 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.300 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.023992+0000", 2024-08-17T17:22:59.300 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14431400899999999 2024-08-17T17:22:59.300 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.300 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.300 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:59.300 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.024099+0000", 2024-08-17T17:22:59.301 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000107754 2024-08-17T17:22:59.301 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.301 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.301 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.301 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.024262+0000", 2024-08-17T17:22:59.301 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000162368 2024-08-17T17:22:59.301 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.301 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.301 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.301 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.301 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.301 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35811 6.f 6:f8be1079:::benchmark_data_smithi063_160268_object35810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:22:59.302 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.877384+0000", 2024-08-17T17:22:59.302 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 558.41200462400002, 2024-08-17T17:22:59.302 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.15051810500000001, 2024-08-17T17:22:59.302 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.302 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.302 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:59.302 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:59.302 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:22:59.302 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:22:59.302 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35811 2024-08-17T17:22:59.302 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.302 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.303 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.303 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.303 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877384+0000", 2024-08-17T17:22:59.303 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.303 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.303 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.303 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.303 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877384+0000", 2024-08-17T17:22:59.303 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.303 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.303 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.303 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.303 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877385+0000", 2024-08-17T17:22:59.304 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.1800000000000004e-07 2024-08-17T17:22:59.304 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.304 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.304 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.304 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877404+0000", 2024-08-17T17:22:59.304 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8766e-05 2024-08-17T17:22:59.304 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.304 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.304 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.304 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877416+0000", 2024-08-17T17:22:59.304 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1770000000000001e-05 2024-08-17T17:22:59.304 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.304 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.305 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.305 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.877423+0000", 2024-08-17T17:22:59.305 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0049999999999998e-06 2024-08-17T17:22:59.305 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.305 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.305 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.305 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878297+0000", 2024-08-17T17:22:59.305 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00087477799999999999 2024-08-17T17:22:59.305 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.305 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.305 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.305 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878431+0000", 2024-08-17T17:22:59.305 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013333900000000001 2024-08-17T17:22:59.306 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.306 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.306 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:22:59.306 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.878556+0000", 2024-08-17T17:22:59.306 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000124764 2024-08-17T17:22:59.306 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.306 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.306 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:59.306 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.880196+0000", 2024-08-17T17:22:59.306 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0016405949999999999 2024-08-17T17:22:59.306 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.306 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.307 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.307 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.881084+0000", 2024-08-17T17:22:59.307 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00088833800000000002 2024-08-17T17:22:59.307 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.307 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.307 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.307 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027510+0000", 2024-08-17T17:22:59.307 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.146425627 2024-08-17T17:22:59.307 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.307 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.307 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:59.307 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027847+0000", 2024-08-17T17:22:59.307 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000336397 2024-08-17T17:22:59.308 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.308 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.308 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.308 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.027902+0000", 2024-08-17T17:22:59.308 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5807999999999998e-05 2024-08-17T17:22:59.308 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.308 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.308 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.308 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.308 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.308 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.44609.0:35820 6.f 6:f1fc3e55:::benchmark_data_smithi063_160268_object35819:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1093)", 2024-08-17T17:22:59.308 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.882503+0000", 2024-08-17T17:22:59.308 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 558.40688556999999, 2024-08-17T17:22:59.309 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14575781700000001, 2024-08-17T17:22:59.309 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.309 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.309 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:59.309 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:59.310 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.44609", 2024-08-17T17:22:59.310 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:22:59.310 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 35820 2024-08-17T17:22:59.310 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.310 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.310 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.310 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.310 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882503+0000", 2024-08-17T17:22:59.310 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.310 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.310 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.311 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.311 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882503+0000", 2024-08-17T17:22:59.311 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.311 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.311 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.311 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.311 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882505+0000", 2024-08-17T17:22:59.311 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.556e-06 2024-08-17T17:22:59.311 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.311 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.311 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.311 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882528+0000", 2024-08-17T17:22:59.311 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3295999999999999e-05 2024-08-17T17:22:59.312 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.312 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.312 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.312 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882540+0000", 2024-08-17T17:22:59.312 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2099000000000001e-05 2024-08-17T17:22:59.312 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.312 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.312 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.312 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882548+0000", 2024-08-17T17:22:59.312 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7049999999999999e-06 2024-08-17T17:22:59.312 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.312 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.312 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.312 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882591+0000", 2024-08-17T17:22:59.313 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2775999999999998e-05 2024-08-17T17:22:59.313 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.313 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.313 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.313 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882678+0000", 2024-08-17T17:22:59.313 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7777999999999999e-05 2024-08-17T17:22:59.313 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.313 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.313 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,4", 2024-08-17T17:22:59.313 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882761+0000", 2024-08-17T17:22:59.313 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2347000000000003e-05 2024-08-17T17:22:59.313 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.313 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.314 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.314 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.023259+0000", 2024-08-17T17:22:59.314 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14049784100000001 2024-08-17T17:22:59.314 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.314 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.314 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:59.314 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.023430+0000", 2024-08-17T17:22:59.314 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000171194 2024-08-17T17:22:59.314 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.314 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.314 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.314 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.028113+0000", 2024-08-17T17:22:59.315 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0046829660000000002 2024-08-17T17:22:59.315 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.315 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.315 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:59.315 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.028217+0000", 2024-08-17T17:22:59.315 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000104621 2024-08-17T17:22:59.315 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.315 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.315 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.315 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.028261+0000", 2024-08-17T17:22:59.315 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3637999999999998e-05 2024-08-17T17:22:59.315 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.315 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.316 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.316 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.316 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.316 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.44609.0:35818 6.3 e1093/1086 6:ccea9449:::benchmark_data_smithi063_160268_object35817:head v 1093'2214, mlcod=1093'2211)", 2024-08-17T17:22:59.316 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:13:40.882944+0000", 2024-08-17T17:22:59.316 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 558.40644461399995, 2024-08-17T17:22:59.316 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.13971906100000001, 2024-08-17T17:22:59.316 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.316 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.316 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:59.316 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.316 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.317 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.317 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882944+0000", 2024-08-17T17:22:59.317 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.317 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.317 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.317 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.317 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882944+0000", 2024-08-17T17:22:59.317 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.317 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.317 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.317 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.317 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882945+0000", 2024-08-17T17:22:59.317 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.111e-06 2024-08-17T17:22:59.318 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.318 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.318 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.318 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:40.882976+0000", 2024-08-17T17:22:59.318 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0332999999999999e-05 2024-08-17T17:22:59.318 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.318 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.318 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.318 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020295+0000", 2024-08-17T17:22:59.318 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.137318948 2024-08-17T17:22:59.318 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.318 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.318 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.319 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020308+0000", 2024-08-17T17:22:59.319 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3488e-05 2024-08-17T17:22:59.319 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.319 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.319 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.319 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020379+0000", 2024-08-17T17:22:59.319 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1173000000000004e-05 2024-08-17T17:22:59.319 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.319 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.319 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.319 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.020825+0000", 2024-08-17T17:22:59.319 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00044561600000000002 2024-08-17T17:22:59.319 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.319 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.320 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:59.320 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.022624+0000", 2024-08-17T17:22:59.320 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001799599 2024-08-17T17:22:59.320 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.320 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.320 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.320 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:13:41.022663+0000", 2024-08-17T17:22:59.320 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8792999999999999e-05 2024-08-17T17:22:59.320 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.320 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.320 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.320 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.320 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.321 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:32536 7.8 7:1e6dc57b:::benchmark_data_smithi063_196002_object32535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:22:59.321 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:10.423425+0000", 2024-08-17T17:22:59.321 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 288.86596343899998, 2024-08-17T17:22:59.321 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.140115024, 2024-08-17T17:22:59.321 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.321 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.321 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:59.321 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:59.321 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:22:59.321 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:22:59.321 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 32536 2024-08-17T17:22:59.321 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.322 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.322 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.322 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.322 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.423425+0000", 2024-08-17T17:22:59.322 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.322 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.322 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.322 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.322 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.423425+0000", 2024-08-17T17:22:59.322 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.322 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.322 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.323 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.323 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.423426+0000", 2024-08-17T17:22:59.323 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3e-07 2024-08-17T17:22:59.323 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.323 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.323 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.323 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.423464+0000", 2024-08-17T17:22:59.323 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7622e-05 2024-08-17T17:22:59.323 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.323 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.323 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.323 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.423472+0000", 2024-08-17T17:22:59.323 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7069999999999998e-06 2024-08-17T17:22:59.324 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.324 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.324 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.324 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.423477+0000", 2024-08-17T17:22:59.324 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8289999999999998e-06 2024-08-17T17:22:59.324 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.324 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.324 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.324 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.424138+0000", 2024-08-17T17:22:59.324 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00066053200000000002 2024-08-17T17:22:59.324 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.324 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.324 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.325 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.424247+0000", 2024-08-17T17:22:59.325 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010904300000000001 2024-08-17T17:22:59.325 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.325 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.325 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:22:59.325 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.424366+0000", 2024-08-17T17:22:59.325 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011934600000000001 2024-08-17T17:22:59.325 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.325 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.325 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:59.325 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425352+0000", 2024-08-17T17:22:59.325 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00098572599999999992 2024-08-17T17:22:59.325 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.326 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.326 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.326 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563349+0000", 2024-08-17T17:22:59.326 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.13799671599999999 2024-08-17T17:22:59.326 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.326 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.326 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.326 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563391+0000", 2024-08-17T17:22:59.326 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2231000000000003e-05 2024-08-17T17:22:59.326 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.326 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.326 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:59.327 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563496+0000", 2024-08-17T17:22:59.327 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010518699999999999 2024-08-17T17:22:59.327 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.327 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.327 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.327 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563540+0000", 2024-08-17T17:22:59.327 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4354999999999998e-05 2024-08-17T17:22:59.327 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.327 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.327 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.327 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.327 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.327 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:32540 7.8 7:1eb42e3f:::benchmark_data_smithi063_196002_object32539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:22:59.327 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:10.425834+0000", 2024-08-17T17:22:59.328 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 288.86355495200002, 2024-08-17T17:22:59.328 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.13808171999999999, 2024-08-17T17:22:59.328 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.328 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.328 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:59.328 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:59.328 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:22:59.328 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:22:59.328 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 32540 2024-08-17T17:22:59.328 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.328 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.328 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.329 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.329 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425834+0000", 2024-08-17T17:22:59.329 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.329 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.329 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.329 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.329 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425834+0000", 2024-08-17T17:22:59.329 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.329 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.329 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.329 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.329 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425835+0000", 2024-08-17T17:22:59.329 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.2399999999999996e-07 2024-08-17T17:22:59.330 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.330 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.330 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.330 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425854+0000", 2024-08-17T17:22:59.330 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9689000000000001e-05 2024-08-17T17:22:59.330 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.330 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.330 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.330 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425864+0000", 2024-08-17T17:22:59.330 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.2480000000000002e-06 2024-08-17T17:22:59.330 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.330 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.331 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.331 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425869+0000", 2024-08-17T17:22:59.331 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0719999999999999e-06 2024-08-17T17:22:59.331 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.331 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.331 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.331 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.425996+0000", 2024-08-17T17:22:59.331 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012675499999999999 2024-08-17T17:22:59.331 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.331 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.331 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.331 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426112+0000", 2024-08-17T17:22:59.332 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000116519 2024-08-17T17:22:59.332 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.332 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.332 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:22:59.332 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426221+0000", 2024-08-17T17:22:59.332 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000108554 2024-08-17T17:22:59.332 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.332 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.332 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:59.332 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563360+0000", 2024-08-17T17:22:59.332 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.137139071 2024-08-17T17:22:59.332 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.332 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.333 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.333 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563579+0000", 2024-08-17T17:22:59.333 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00021886099999999999 2024-08-17T17:22:59.333 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.333 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.333 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.333 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563817+0000", 2024-08-17T17:22:59.333 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00023799699999999999 2024-08-17T17:22:59.333 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.333 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.333 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:59.333 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563876+0000", 2024-08-17T17:22:59.333 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9166999999999998e-05 2024-08-17T17:22:59.334 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.334 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.334 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.334 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563916+0000", 2024-08-17T17:22:59.334 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9863000000000001e-05 2024-08-17T17:22:59.334 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.334 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.334 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.334 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.334 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.334 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.34965.0:32536 7.8 e1133/1129 ondisk, result = 0)", 2024-08-17T17:22:59.334 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:10.426058+0000", 2024-08-17T17:22:59.335 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 288.86333079999997, 2024-08-17T17:22:59.335 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.13730266199999999, 2024-08-17T17:22:59.335 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.335 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.335 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:22:59.335 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.335 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.335 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.335 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426058+0000", 2024-08-17T17:22:59.335 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.335 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.335 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.335 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.336 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426058+0000", 2024-08-17T17:22:59.336 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.336 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.336 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.336 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.336 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426059+0000", 2024-08-17T17:22:59.336 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8699999999999995e-07 2024-08-17T17:22:59.336 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.336 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.336 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.336 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426066+0000", 2024-08-17T17:22:59.336 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2590000000000003e-06 2024-08-17T17:22:59.336 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.337 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.337 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.337 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426074+0000", 2024-08-17T17:22:59.337 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4309999999999994e-06 2024-08-17T17:22:59.337 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.337 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.337 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.337 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426080+0000", 2024-08-17T17:22:59.337 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9970000000000004e-06 2024-08-17T17:22:59.337 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.337 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.338 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.338 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563335+0000", 2024-08-17T17:22:59.338 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.13725435599999999 2024-08-17T17:22:59.338 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.338 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.338 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.338 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563343+0000", 2024-08-17T17:22:59.338 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0760000000000003e-06 2024-08-17T17:22:59.338 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.338 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.338 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.338 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563361+0000", 2024-08-17T17:22:59.339 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7955999999999999e-05 2024-08-17T17:22:59.339 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.339 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.339 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.339 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.339 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.339 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.34965.0:32536 7.8 e1133/1129 ondisk, result = 0)", 2024-08-17T17:22:59.339 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:10.426119+0000", 2024-08-17T17:22:59.339 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 288.863270285, 2024-08-17T17:22:59.339 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.137427676, 2024-08-17T17:22:59.339 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.339 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.339 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:22:59.340 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.340 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.340 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.340 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426119+0000", 2024-08-17T17:22:59.340 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.340 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.340 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.340 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.341 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426119+0000", 2024-08-17T17:22:59.341 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.341 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.341 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.341 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.341 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426119+0000", 2024-08-17T17:22:59.342 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5999999999999999e-07 2024-08-17T17:22:59.342 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.342 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.342 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.342 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426122+0000", 2024-08-17T17:22:59.342 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.9299999999999999e-06 2024-08-17T17:22:59.342 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.342 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.342 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.342 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426129+0000", 2024-08-17T17:22:59.342 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2690000000000002e-06 2024-08-17T17:22:59.343 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.343 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.343 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.343 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.426134+0000", 2024-08-17T17:22:59.343 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1919999999999996e-06 2024-08-17T17:22:59.343 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.343 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.343 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.343 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563381+0000", 2024-08-17T17:22:59.343 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.137246487 2024-08-17T17:22:59.343 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.343 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.343 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.343 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563387+0000", 2024-08-17T17:22:59.344 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7420000000000003e-06 2024-08-17T17:22:59.344 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.344 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.344 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.344 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:10.563546+0000", 2024-08-17T17:22:59.344 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015959599999999999 2024-08-17T17:22:59.344 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.344 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.344 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.344 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.344 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.344 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:65727 7.e 7:7466d76d:::benchmark_data_smithi063_196002_object65726:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-08-17T17:22:59.344 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:30.516551+0000", 2024-08-17T17:22:59.345 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 268.77283829599997, 2024-08-17T17:22:59.345 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5070886789999998, 2024-08-17T17:22:59.345 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.345 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.345 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:22:59.345 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:59.345 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:22:59.345 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:22:59.345 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 65727 2024-08-17T17:22:59.345 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.345 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.345 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.346 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.346 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516551+0000", 2024-08-17T17:22:59.346 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.346 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.346 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.346 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.346 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516551+0000", 2024-08-17T17:22:59.346 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.346 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.346 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.346 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.346 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516587+0000", 2024-08-17T17:22:59.347 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6622000000000003e-05 2024-08-17T17:22:59.347 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.347 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.347 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.347 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516612+0000", 2024-08-17T17:22:59.347 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4496999999999999e-05 2024-08-17T17:22:59.347 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.347 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.347 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.347 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516627+0000", 2024-08-17T17:22:59.347 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5061e-05 2024-08-17T17:22:59.347 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516634+0000", 2024-08-17T17:22:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3980000000000002e-06 2024-08-17T17:22:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516708+0000", 2024-08-17T17:22:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3986000000000006e-05 2024-08-17T17:22:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516845+0000", 2024-08-17T17:22:59.348 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013681699999999999 2024-08-17T17:22:59.349 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.349 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.349 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:22:59.349 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.516985+0000", 2024-08-17T17:22:59.349 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014038000000000001 2024-08-17T17:22:59.349 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.349 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.349 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:59.349 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.519528+0000", 2024-08-17T17:22:59.349 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002542515 2024-08-17T17:22:59.349 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.349 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.349 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.350 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.520307+0000", 2024-08-17T17:22:59.350 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00077905200000000002 2024-08-17T17:22:59.350 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.350 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.350 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.350 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.535671+0000", 2024-08-17T17:22:59.350 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.015363751 2024-08-17T17:22:59.350 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.350 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.350 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:22:59.350 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:30.535694+0000", 2024-08-17T17:22:59.350 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3139e-05 2024-08-17T17:22:59.350 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.351 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.351 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.351 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:31.526159+0000", 2024-08-17T17:22:59.351 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.99046498699999996 2024-08-17T17:22:59.351 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.351 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.351 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:22:59.351 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:31.526179+0000", 2024-08-17T17:22:59.351 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0432e-05 2024-08-17T17:22:59.351 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.351 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.352 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.352 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:33.023535+0000", 2024-08-17T17:22:59.352 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.497355684 2024-08-17T17:22:59.352 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.352 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.352 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.352 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:33.023639+0000", 2024-08-17T17:22:59.352 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010435800000000001 2024-08-17T17:22:59.352 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.352 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.352 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.352 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.352 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.353 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:88377 7.e 7:72a20e0d:::benchmark_data_smithi063_196002_object88376:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1150)", 2024-08-17T17:22:59.353 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.650876+0000", 2024-08-17T17:22:59.353 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 238.638512389, 2024-08-17T17:22:59.353 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.28977614600000001, 2024-08-17T17:22:59.353 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.353 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.353 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:59.353 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:59.353 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:22:59.353 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:22:59.353 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 88377 2024-08-17T17:22:59.353 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.354 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.354 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.354 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.354 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650876+0000", 2024-08-17T17:22:59.354 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.354 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.354 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.354 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.354 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650876+0000", 2024-08-17T17:22:59.354 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.354 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.354 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.355 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.355 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650879+0000", 2024-08-17T17:22:59.355 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.187e-06 2024-08-17T17:22:59.355 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.355 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.355 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.355 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650905+0000", 2024-08-17T17:22:59.355 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6446999999999999e-05 2024-08-17T17:22:59.355 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.355 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.355 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.355 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650936+0000", 2024-08-17T17:22:59.355 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1050999999999998e-05 2024-08-17T17:22:59.356 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.356 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.356 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.356 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650945+0000", 2024-08-17T17:22:59.356 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8400000000000001e-06 2024-08-17T17:22:59.356 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.356 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.356 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.356 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651123+0000", 2024-08-17T17:22:59.356 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00017799800000000001 2024-08-17T17:22:59.356 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.356 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.357 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.357 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651483+0000", 2024-08-17T17:22:59.357 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00035978699999999998 2024-08-17T17:22:59.357 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.357 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.357 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:22:59.357 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.671281+0000", 2024-08-17T17:22:59.357 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.019797911000000001 2024-08-17T17:22:59.357 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.357 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.357 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:59.357 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.705325+0000", 2024-08-17T17:22:59.358 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.034044498999999999 2024-08-17T17:22:59.358 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.358 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.358 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.358 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.760144+0000", 2024-08-17T17:22:59.358 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.054818791999999998 2024-08-17T17:22:59.358 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.358 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.358 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.358 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.937269+0000", 2024-08-17T17:22:59.358 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17712487399999999 2024-08-17T17:22:59.358 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.359 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.359 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:59.359 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.937348+0000", 2024-08-17T17:22:59.359 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8988999999999999e-05 2024-08-17T17:22:59.359 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.359 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.359 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.359 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.940653+0000", 2024-08-17T17:22:59.359 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0033047710000000002 2024-08-17T17:22:59.359 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.359 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.359 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.359 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.360 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.360 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88375 7.2 e1150/1129 7:44faffa9:::benchmark_data_smithi063_196002_object88374:head v 1150'5570, mlcod=1150'5567)", 2024-08-17T17:22:59.360 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.651624+0000", 2024-08-17T17:22:59.360 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 238.63776471099999, 2024-08-17T17:22:59.360 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.16577204300000001, 2024-08-17T17:22:59.360 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.360 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.360 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:22:59.360 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.360 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.360 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.361 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651624+0000", 2024-08-17T17:22:59.361 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.361 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.361 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.361 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.362 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651624+0000", 2024-08-17T17:22:59.362 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.362 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.363 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.363 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.363 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651625+0000", 2024-08-17T17:22:59.363 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3099999999999997e-07 2024-08-17T17:22:59.363 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.363 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.363 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.363 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651676+0000", 2024-08-17T17:22:59.363 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1603e-05 2024-08-17T17:22:59.363 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.363 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.363 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.364 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651698+0000", 2024-08-17T17:22:59.364 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1809000000000001e-05 2024-08-17T17:22:59.364 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.364 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.364 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.364 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651717+0000", 2024-08-17T17:22:59.364 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8516999999999999e-05 2024-08-17T17:22:59.364 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.364 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.364 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.364 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651792+0000", 2024-08-17T17:22:59.365 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5512000000000005e-05 2024-08-17T17:22:59.365 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.365 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.365 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.365 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651821+0000", 2024-08-17T17:22:59.365 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.9067000000000001e-05 2024-08-17T17:22:59.365 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.365 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.365 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.365 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.817396+0000", 2024-08-17T17:22:59.365 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.16557490399999999 2024-08-17T17:22:59.365 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.365 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.366 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.366 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.366 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.366 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:88379 7.e 7:77a5a758:::benchmark_data_smithi063_196002_object88378:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1150)", 2024-08-17T17:22:59.366 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.696360+0000", 2024-08-17T17:22:59.366 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 238.59302925599999, 2024-08-17T17:22:59.366 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.24823995800000001, 2024-08-17T17:22:59.366 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.366 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.366 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:59.366 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:59.366 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:22:59.367 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:22:59.367 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 88379 2024-08-17T17:22:59.367 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.367 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.367 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.367 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.367 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.696360+0000", 2024-08-17T17:22:59.367 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.367 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.367 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.367 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.367 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.696360+0000", 2024-08-17T17:22:59.368 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.368 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.368 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.368 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.368 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.696362+0000", 2024-08-17T17:22:59.368 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2850000000000001e-06 2024-08-17T17:22:59.368 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.368 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.368 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.368 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.696400+0000", 2024-08-17T17:22:59.368 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7744999999999999e-05 2024-08-17T17:22:59.368 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.368 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.369 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.369 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.696427+0000", 2024-08-17T17:22:59.369 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.7237e-05 2024-08-17T17:22:59.369 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.369 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.369 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.369 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.696440+0000", 2024-08-17T17:22:59.369 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3352e-05 2024-08-17T17:22:59.369 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.369 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.369 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.369 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.701817+0000", 2024-08-17T17:22:59.370 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0053767579999999997 2024-08-17T17:22:59.370 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.370 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.370 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.370 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.702035+0000", 2024-08-17T17:22:59.370 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00021799100000000001 2024-08-17T17:22:59.370 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.370 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.370 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:22:59.370 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.702242+0000", 2024-08-17T17:22:59.370 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00020687 2024-08-17T17:22:59.370 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.370 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.371 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:59.371 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.723533+0000", 2024-08-17T17:22:59.371 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.02129166 2024-08-17T17:22:59.371 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.371 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.371 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.371 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.760280+0000", 2024-08-17T17:22:59.371 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.036746264000000001 2024-08-17T17:22:59.371 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.371 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.371 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.371 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.940876+0000", 2024-08-17T17:22:59.371 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.18059586 2024-08-17T17:22:59.372 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.372 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.372 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:59.372 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.941254+0000", 2024-08-17T17:22:59.372 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00037889599999999999 2024-08-17T17:22:59.372 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.372 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.372 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.372 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.944600+0000", 2024-08-17T17:22:59.372 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0033450400000000001 2024-08-17T17:22:59.372 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.372 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.372 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.373 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.373 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.373 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:105126 7.e 7:7fc1a2a3:::benchmark_data_smithi063_196002_object105125:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1154)", 2024-08-17T17:22:59.373 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:20.297848+0000", 2024-08-17T17:22:59.373 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:22:58 smithi162 ceph-mon[104668]: pgmap v1538: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2024-08-17T17:22:59.374 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 218.991540829, 2024-08-17T17:22:59.374 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14215101799999999, 2024-08-17T17:22:59.374 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.374 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.374 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:59.374 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:59.374 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:22:59.374 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:22:59.374 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 105126 2024-08-17T17:22:59.375 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.375 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.375 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.375 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.375 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.297848+0000", 2024-08-17T17:22:59.375 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.375 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.375 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.375 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.375 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.297848+0000", 2024-08-17T17:22:59.375 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.375 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.376 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.376 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.376 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.297850+0000", 2024-08-17T17:22:59.376 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6840000000000001e-06 2024-08-17T17:22:59.376 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.376 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.376 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.376 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.297876+0000", 2024-08-17T17:22:59.376 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5969000000000002e-05 2024-08-17T17:22:59.376 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.376 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.376 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.377 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.297890+0000", 2024-08-17T17:22:59.377 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4431e-05 2024-08-17T17:22:59.377 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.377 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.377 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.377 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.297899+0000", 2024-08-17T17:22:59.377 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6000000000000007e-06 2024-08-17T17:22:59.377 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.377 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.377 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.377 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.301414+0000", 2024-08-17T17:22:59.378 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0035151570000000001 2024-08-17T17:22:59.378 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.378 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.378 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.378 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.374944+0000", 2024-08-17T17:22:59.378 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.073530288999999999 2024-08-17T17:22:59.378 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.378 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.378 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:22:59.378 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.375129+0000", 2024-08-17T17:22:59.378 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00018519599999999999 2024-08-17T17:22:59.378 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.379 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.379 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.379 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.393824+0000", 2024-08-17T17:22:59.379 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.018695169000000001 2024-08-17T17:22:59.379 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.379 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.379 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.379 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.393874+0000", 2024-08-17T17:22:59.379 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9783000000000001e-05 2024-08-17T17:22:59.379 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.379 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.379 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:59.379 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.437062+0000", 2024-08-17T17:22:59.380 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.043187495999999999 2024-08-17T17:22:59.380 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.380 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.380 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:59.380 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.437125+0000", 2024-08-17T17:22:59.380 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3131000000000003e-05 2024-08-17T17:22:59.380 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.380 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.380 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.380 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.439999+0000", 2024-08-17T17:22:59.380 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0028741130000000002 2024-08-17T17:22:59.380 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.381 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.381 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.381 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.381 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.381 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "MOSDPGPushReply(7.e 1157/1144 [PushReplyOp(7:75d76b99:::benchmark_data_smithi063_196002_object23402:head)])", 2024-08-17T17:22:59.381 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:57.557290+0000", 2024-08-17T17:22:59.381 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 181.73209885200001, 2024-08-17T17:22:59.381 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17745955299999999, 2024-08-17T17:22:59.381 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.381 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.381 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:22:59.381 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.381 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.382 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.382 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.557290+0000", 2024-08-17T17:22:59.382 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.382 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.382 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.382 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.382 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.557290+0000", 2024-08-17T17:22:59.382 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.382 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.382 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.382 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.382 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.557290+0000", 2024-08-17T17:22:59.383 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9299999999999999e-07 2024-08-17T17:22:59.383 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.383 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.383 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.383 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.557295+0000", 2024-08-17T17:22:59.383 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.211e-06 2024-08-17T17:22:59.383 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.383 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.383 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.383 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.557303+0000", 2024-08-17T17:22:59.383 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2679999999999994e-06 2024-08-17T17:22:59.383 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.383 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.384 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.384 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.557309+0000", 2024-08-17T17:22:59.384 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.981e-06 2024-08-17T17:22:59.384 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.384 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.384 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.384 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.660581+0000", 2024-08-17T17:22:59.384 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.10327185899999999 2024-08-17T17:22:59.384 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.384 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.384 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.384 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.734750+0000", 2024-08-17T17:22:59.385 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.074168840999999999 2024-08-17T17:22:59.385 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.385 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.385 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.385 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.385 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.385 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:135016 7.e 7:7ca61f7f:::benchmark_data_smithi063_196002_object22923:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1157)", 2024-08-17T17:22:59.385 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:57.560559+0000", 2024-08-17T17:22:59.385 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 181.72882967800001, 2024-08-17T17:22:59.385 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.18443891400000001, 2024-08-17T17:22:59.385 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.385 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.386 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:59.386 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:59.386 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:22:59.386 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:22:59.386 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 135016 2024-08-17T17:22:59.386 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.386 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.386 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.386 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.386 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.560559+0000", 2024-08-17T17:22:59.386 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.386 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.386 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.387 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.387 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.560559+0000", 2024-08-17T17:22:59.387 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.387 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.387 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.387 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.387 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.560560+0000", 2024-08-17T17:22:59.387 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8100000000000002e-07 2024-08-17T17:22:59.387 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.387 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.387 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.387 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.560564+0000", 2024-08-17T17:22:59.388 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.0219999999999998e-06 2024-08-17T17:22:59.388 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.388 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.388 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.388 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.560574+0000", 2024-08-17T17:22:59.388 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.6719999999999999e-06 2024-08-17T17:22:59.388 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.388 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.388 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.388 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.560581+0000", 2024-08-17T17:22:59.388 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.126e-06 2024-08-17T17:22:59.388 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.388 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.389 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.389 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.650358+0000", 2024-08-17T17:22:59.389 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.089776863999999998 2024-08-17T17:22:59.389 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.389 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.389 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.389 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.650446+0000", 2024-08-17T17:22:59.389 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7967000000000007e-05 2024-08-17T17:22:59.389 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.389 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.389 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:22:59.389 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.650575+0000", 2024-08-17T17:22:59.389 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012945 2024-08-17T17:22:59.390 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.390 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.390 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.390 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.734812+0000", 2024-08-17T17:22:59.390 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.084236488999999998 2024-08-17T17:22:59.390 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.390 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.390 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.390 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.734908+0000", 2024-08-17T17:22:59.390 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.6001000000000005e-05 2024-08-17T17:22:59.390 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.390 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.391 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:59.391 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.741988+0000", 2024-08-17T17:22:59.391 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0070800669999999998 2024-08-17T17:22:59.391 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.391 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.391 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:59.391 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.742026+0000", 2024-08-17T17:22:59.391 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8296000000000001e-05 2024-08-17T17:22:59.391 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.391 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.391 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.391 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:57.744998+0000", 2024-08-17T17:22:59.392 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0029721790000000001 2024-08-17T17:22:59.392 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.392 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.392 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.392 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.392 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.392 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:153369 7.e 7:7fdbd7af:::benchmark_data_smithi063_196002_object41276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1160)", 2024-08-17T17:22:59.392 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:20:21.093402+0000", 2024-08-17T17:22:59.392 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 158.195987084, 2024-08-17T17:22:59.392 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14373694300000001, 2024-08-17T17:22:59.392 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.392 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.393 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:59.393 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:59.393 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:22:59.393 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:22:59.393 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 153369 2024-08-17T17:22:59.393 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.393 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.393 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.393 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.393 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.093402+0000", 2024-08-17T17:22:59.393 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.393 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.393 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.394 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.394 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.093402+0000", 2024-08-17T17:22:59.394 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.394 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.394 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.394 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.394 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.093403+0000", 2024-08-17T17:22:59.394 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4139999999999999e-06 2024-08-17T17:22:59.394 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.394 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.394 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.394 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.093408+0000", 2024-08-17T17:22:59.394 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9799999999999998e-06 2024-08-17T17:22:59.395 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.395 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.395 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.395 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.093419+0000", 2024-08-17T17:22:59.395 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1342000000000001e-05 2024-08-17T17:22:59.395 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.395 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.395 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.395 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.093426+0000", 2024-08-17T17:22:59.395 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6030000000000001e-06 2024-08-17T17:22:59.395 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.395 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.395 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.395 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.098041+0000", 2024-08-17T17:22:59.396 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0046150610000000002 2024-08-17T17:22:59.396 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.396 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.396 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for rw locks", 2024-08-17T17:22:59.396 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.098151+0000", 2024-08-17T17:22:59.396 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000109707 2024-08-17T17:22:59.396 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.396 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.396 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.396 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.211049+0000", 2024-08-17T17:22:59.396 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.112898024 2024-08-17T17:22:59.396 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.396 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.396 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.397 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.211159+0000", 2024-08-17T17:22:59.397 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000110544 2024-08-17T17:22:59.397 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.397 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.397 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:22:59.397 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.211357+0000", 2024-08-17T17:22:59.397 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00019790399999999999 2024-08-17T17:22:59.397 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.397 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.397 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.397 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.220113+0000", 2024-08-17T17:22:59.397 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0087555029999999995 2024-08-17T17:22:59.397 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.397 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.398 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:59.398 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.223434+0000", 2024-08-17T17:22:59.398 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0033210829999999998 2024-08-17T17:22:59.398 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.398 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.398 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.398 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.235458+0000", 2024-08-17T17:22:59.398 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.012024014 2024-08-17T17:22:59.398 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.398 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.398 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:59.398 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.235525+0000", 2024-08-17T17:22:59.399 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6958000000000005e-05 2024-08-17T17:22:59.399 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.399 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.399 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.399 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:21.237139+0000", 2024-08-17T17:22:59.399 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001613806 2024-08-17T17:22:59.399 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.399 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.399 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.399 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.399 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.399 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:170155 7.e 7:73cb9dd4:::benchmark_data_smithi063_196002_object58062:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1164)", 2024-08-17T17:22:59.399 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:20:43.949943+0000", 2024-08-17T17:22:59.400 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 135.33944615300001, 2024-08-17T17:22:59.400 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.92340106200000005, 2024-08-17T17:22:59.400 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.400 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.400 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:59.400 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:59.400 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:22:59.400 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:22:59.400 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 170155 2024-08-17T17:22:59.401 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.401 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.401 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.401 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.401 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.949943+0000", 2024-08-17T17:22:59.401 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.401 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.401 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.401 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.401 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.949943+0000", 2024-08-17T17:22:59.401 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.401 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.401 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.401 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.402 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.949944+0000", 2024-08-17T17:22:59.402 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6840000000000001e-06 2024-08-17T17:22:59.402 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.402 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.402 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.402 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.949951+0000", 2024-08-17T17:22:59.402 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1999999999999999e-06 2024-08-17T17:22:59.402 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.402 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.402 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.402 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.949963+0000", 2024-08-17T17:22:59.402 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2639999999999999e-05 2024-08-17T17:22:59.403 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.403 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.403 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.403 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.949971+0000", 2024-08-17T17:22:59.403 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8900000000000007e-06 2024-08-17T17:22:59.403 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.403 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.403 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.403 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.950036+0000", 2024-08-17T17:22:59.403 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4511000000000006e-05 2024-08-17T17:22:59.403 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.403 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.403 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:22:59.404 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.950050+0000", 2024-08-17T17:22:59.404 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4107999999999999e-05 2024-08-17T17:22:59.404 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.404 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.404 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.404 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.864638+0000", 2024-08-17T17:22:59.404 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.91458820399999996 2024-08-17T17:22:59.404 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.404 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.404 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.404 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.864749+0000", 2024-08-17T17:22:59.404 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011121899999999999 2024-08-17T17:22:59.404 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.406 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:59.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:59.406 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.406 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:22:59.407 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.865144+0000", 2024-08-17T17:22:59.407 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00039510700000000001 2024-08-17T17:22:59.407 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.407 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.407 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.407 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.872672+0000", 2024-08-17T17:22:59.407 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0075282550000000002 2024-08-17T17:22:59.407 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.407 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.407 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.407 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.872783+0000", 2024-08-17T17:22:59.407 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000110784 2024-08-17T17:22:59.407 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.408 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.408 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:59.408 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.872802+0000", 2024-08-17T17:22:59.408 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9185999999999999e-05 2024-08-17T17:22:59.408 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.408 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.408 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:59.408 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.872867+0000", 2024-08-17T17:22:59.408 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4293000000000003e-05 2024-08-17T17:22:59.408 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.408 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.408 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.408 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.873344+0000", 2024-08-17T17:22:59.409 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00047698100000000002 2024-08-17T17:22:59.409 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.409 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.409 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.409 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.409 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.409 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:170172 7.e 7:734a2b0d:::benchmark_data_smithi063_196002_object58079:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1164)", 2024-08-17T17:22:59.409 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:20:43.980154+0000", 2024-08-17T17:22:59.409 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 135.309234825, 2024-08-17T17:22:59.409 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.89372299499999996, 2024-08-17T17:22:59.409 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.409 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.409 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:59.410 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:59.410 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:22:59.410 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:22:59.410 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 170172 2024-08-17T17:22:59.410 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.410 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.410 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.410 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.410 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.980154+0000", 2024-08-17T17:22:59.410 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.410 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.410 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.410 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.411 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.980154+0000", 2024-08-17T17:22:59.411 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.411 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.411 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.411 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.411 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.980156+0000", 2024-08-17T17:22:59.411 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6619999999999999e-06 2024-08-17T17:22:59.411 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.411 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.411 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.411 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.980161+0000", 2024-08-17T17:22:59.411 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7350000000000001e-06 2024-08-17T17:22:59.411 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.412 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.412 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.412 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.980175+0000", 2024-08-17T17:22:59.412 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.345e-05 2024-08-17T17:22:59.412 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.412 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.412 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.412 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.980183+0000", 2024-08-17T17:22:59.412 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2279999999999998e-06 2024-08-17T17:22:59.412 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.412 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.412 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.412 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.980262+0000", 2024-08-17T17:22:59.413 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8551000000000001e-05 2024-08-17T17:22:59.413 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.413 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.413 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:22:59.413 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.980276+0000", 2024-08-17T17:22:59.413 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4731e-05 2024-08-17T17:22:59.413 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.413 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.413 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.413 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.867444+0000", 2024-08-17T17:22:59.413 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.88716756100000005 2024-08-17T17:22:59.413 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.414 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.414 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.414 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.867572+0000", 2024-08-17T17:22:59.414 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012812899999999999 2024-08-17T17:22:59.414 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.414 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.414 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:22:59.414 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.867925+0000", 2024-08-17T17:22:59.414 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000352713 2024-08-17T17:22:59.414 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.414 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.414 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.415 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.873398+0000", 2024-08-17T17:22:59.415 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.005473299 2024-08-17T17:22:59.415 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.415 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.415 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:59.415 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.873419+0000", 2024-08-17T17:22:59.415 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1305999999999999e-05 2024-08-17T17:22:59.415 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.415 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.415 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.415 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.873466+0000", 2024-08-17T17:22:59.415 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6173e-05 2024-08-17T17:22:59.415 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.415 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.416 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:59.416 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.873601+0000", 2024-08-17T17:22:59.416 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000135019 2024-08-17T17:22:59.416 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.416 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.416 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.416 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.873877+0000", 2024-08-17T17:22:59.416 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000276438 2024-08-17T17:22:59.416 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.416 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.416 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.416 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.416 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.417 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:170179 7.e 7:75e01b51:::benchmark_data_smithi063_196002_object58086:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1164)", 2024-08-17T17:22:59.417 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:20:43.993550+0000", 2024-08-17T17:22:59.417 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 135.29583850500001, 2024-08-17T17:22:59.417 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.89224257399999995, 2024-08-17T17:22:59.417 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.417 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.417 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:59.417 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:59.417 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:22:59.417 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:22:59.417 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 170179 2024-08-17T17:22:59.417 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.418 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.418 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.418 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.418 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.993550+0000", 2024-08-17T17:22:59.418 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.418 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.418 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.418 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.418 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.993550+0000", 2024-08-17T17:22:59.418 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.418 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.418 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.419 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.419 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.993552+0000", 2024-08-17T17:22:59.419 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.624e-06 2024-08-17T17:22:59.419 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.419 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.419 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.419 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.993557+0000", 2024-08-17T17:22:59.419 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9099999999999996e-06 2024-08-17T17:22:59.419 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.419 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.419 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.419 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.993568+0000", 2024-08-17T17:22:59.419 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1535000000000001e-05 2024-08-17T17:22:59.419 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.420 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.420 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.420 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.993575+0000", 2024-08-17T17:22:59.420 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.725e-06 2024-08-17T17:22:59.420 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.420 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.420 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.420 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.993630+0000", 2024-08-17T17:22:59.420 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5250999999999997e-05 2024-08-17T17:22:59.420 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.420 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.420 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:22:59.421 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:43.993645+0000", 2024-08-17T17:22:59.421 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4603999999999999e-05 2024-08-17T17:22:59.421 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.421 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.421 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.421 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.869873+0000", 2024-08-17T17:22:59.421 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.87622776099999999 2024-08-17T17:22:59.421 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.421 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.421 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for degraded object", 2024-08-17T17:22:59.421 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.872587+0000", 2024-08-17T17:22:59.421 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002714265 2024-08-17T17:22:59.421 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.422 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.422 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.422 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.878206+0000", 2024-08-17T17:22:59.422 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.005618507 2024-08-17T17:22:59.422 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.422 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.422 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.422 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.878292+0000", 2024-08-17T17:22:59.422 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6706000000000006e-05 2024-08-17T17:22:59.422 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.422 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.422 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:22:59.422 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.878746+0000", 2024-08-17T17:22:59.423 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00045335500000000001 2024-08-17T17:22:59.423 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.423 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.423 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.423 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.885255+0000", 2024-08-17T17:22:59.423 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0065090790000000001 2024-08-17T17:22:59.423 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.423 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.423 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:59.423 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.885273+0000", 2024-08-17T17:22:59.423 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8301e-05 2024-08-17T17:22:59.423 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.424 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.424 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.424 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.885415+0000", 2024-08-17T17:22:59.424 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014174399999999999 2024-08-17T17:22:59.424 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.424 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.424 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:59.424 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.885524+0000", 2024-08-17T17:22:59.424 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000109069 2024-08-17T17:22:59.424 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.424 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.424 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.424 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.885793+0000", 2024-08-17T17:22:59.425 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00026913799999999998 2024-08-17T17:22:59.425 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.425 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.425 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.425 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.425 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.425 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.34965.0:170192 7.e 7:729a1559:::benchmark_data_smithi063_196002_object58099:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1164)", 2024-08-17T17:22:59.425 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:20:44.011791+0000", 2024-08-17T17:22:59.425 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 135.27759763899999, 2024-08-17T17:22:59.425 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.88215049599999995, 2024-08-17T17:22:59.425 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:22:59.425 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:22:59.425 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:22:59.426 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:22:59.426 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.34965", 2024-08-17T17:22:59.426 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:22:59.426 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 170192 2024-08-17T17:22:59.426 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.426 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:22:59.426 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.426 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:22:59.426 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.011791+0000", 2024-08-17T17:22:59.426 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.426 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.426 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.426 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:22:59.427 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.011791+0000", 2024-08-17T17:22:59.427 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:22:59.427 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.427 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.427 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:22:59.427 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.011793+0000", 2024-08-17T17:22:59.427 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5519999999999999e-06 2024-08-17T17:22:59.427 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.427 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.427 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:22:59.427 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.011798+0000", 2024-08-17T17:22:59.427 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7470000000000002e-06 2024-08-17T17:22:59.428 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.428 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.428 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:22:59.428 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.011811+0000", 2024-08-17T17:22:59.428 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2830000000000001e-05 2024-08-17T17:22:59.428 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.428 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.428 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:22:59.428 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.011819+0000", 2024-08-17T17:22:59.428 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6199999999999999e-06 2024-08-17T17:22:59.428 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.428 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.429 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.429 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.011876+0000", 2024-08-17T17:22:59.429 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7401000000000002e-05 2024-08-17T17:22:59.429 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.429 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.429 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:22:59.429 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.011886+0000", 2024-08-17T17:22:59.429 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.6229999999999992e-06 2024-08-17T17:22:59.429 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.429 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.429 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.429 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.873927+0000", 2024-08-17T17:22:59.430 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.86204063200000003 2024-08-17T17:22:59.430 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.430 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.430 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for degraded object", 2024-08-17T17:22:59.430 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.874650+0000", 2024-08-17T17:22:59.430 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00072346599999999997 2024-08-17T17:22:59.430 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.430 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.430 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:22:59.430 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.881631+0000", 2024-08-17T17:22:59.430 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0069806069999999998 2024-08-17T17:22:59.430 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.430 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.430 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:22:59.431 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.881711+0000", 2024-08-17T17:22:59.431 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0760999999999995e-05 2024-08-17T17:22:59.431 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.431 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.431 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:22:59.431 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.882120+0000", 2024-08-17T17:22:59.431 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000408778 2024-08-17T17:22:59.431 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.431 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.431 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:22:59.431 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.885278+0000", 2024-08-17T17:22:59.431 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.003158241 2024-08-17T17:22:59.431 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.432 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.432 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.432 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.885832+0000", 2024-08-17T17:22:59.432 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00055332999999999999 2024-08-17T17:22:59.432 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.432 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.432 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:22:59.432 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.893398+0000", 2024-08-17T17:22:59.432 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0075666270000000002 2024-08-17T17:22:59.432 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.432 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.432 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:22:59.433 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.893550+0000", 2024-08-17T17:22:59.433 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015172300000000001 2024-08-17T17:22:59.433 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:22:59.433 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:22:59.433 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:22:59.433 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:20:44.893942+0000", 2024-08-17T17:22:59.433 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00039155800000000002 2024-08-17T17:22:59.433 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.433 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.433 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.433 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:22:59.433 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:22:59.433 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:22:59.500 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 32 16 65715 65699 128.289 83.6875 0.00861461 0.00778929 2024-08-17T17:22:59.507 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:22:59.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.551 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:59.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:22:59.652 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:22:59.677 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:22:59.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.679 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:22:59.780 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:22:59.821 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:59.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:22:59.923 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:22:59.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:22:59.968 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:22:59.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:00.068 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:00.100 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:00.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:00.202 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:00.239 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:00.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:00.341 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:00.389 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:00.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:00.491 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:00.500 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 33 16 67105 67089 127.034 86.875 0.0231142 0.00786635 2024-08-17T17:23:00.526 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:00.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:00.596 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:23:00.597 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:23:00.627 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:00.664 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:00.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:00.766 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:00.801 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:23:00.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.803 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:00.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:00.903 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:00.935 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:00.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:00.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:01.037 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:01.084 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:01.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:01.186 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:01.222 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:01.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:01.324 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:01.364 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:01.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:01.467 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:01.500 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 34 16 68354 68338 125.594 78.0625 0.00849443 0.0079573 2024-08-17T17:23:01.508 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:01.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:01.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:01 smithi162 ceph-mon[104668]: from='client.45251 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:23:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:01 smithi162 ceph-mon[104668]: pgmap v1539: 99 pgs: 99 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-08-17T17:23:01.610 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:01.644 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:01.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:01.746 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:01.764 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:01 smithi063 ceph-mon[114756]: from='client.45251 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:23:01.765 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:01 smithi063 ceph-mon[114756]: pgmap v1539: 99 pgs: 99 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-08-17T17:23:01.765 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:01 smithi063 ceph-mon[109843]: from='client.45251 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:23:01.765 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:01 smithi063 ceph-mon[109843]: pgmap v1539: 99 pgs: 99 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-08-17T17:23:01.797 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:01.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:01.805 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1538,"stamp":"2024-08-17T17:22:58.262446+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4028569573,"num_objects":61488,"num_object_clones":0,"num_object_copies":184464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":61488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61526,"num_write_kb":3933601,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1246,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":63160,"ondisk_log_size":63160,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":306,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14896904,"kb_used_data":11731324,"kb_used_omap":199,"kb_used_meta":3165304,"kb_avail":735064312,"statfs":{"total":767960285184,"available":752705855488,"internally_reserved":0,"allocated":12012875776,"data_stored":11993875307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":3241272060},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,4,7,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":831389696,"num_objects":12686,"num_object_clones":0,"num_object_copies":38058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12686,"num_write_kb":811904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002050"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1505,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.387787+0000","last_change":"2024-08-17T17:21:06.463828+0000","last_active":"2024-08-17T17:22:52.387787+0000","last_peered":"2024-08-17T17:22:52.387787+0000","last_clean":"2024-08-17T17:22:52.387787+0000","last_became_active":"2024-08-17T17:21:06.463464+0000","last_became_peered":"2024-08-17T17:21:06.463464+0000","last_unstale":"2024-08-17T17:22:52.387787+0000","last_undegraded":"2024-08-17T17:22:52.387787+0000","last_fullsized":"2024-08-17T17:22:52.387787+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:46.465006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1134,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.374581+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:22:52.374581+0000","last_peered":"2024-08-17T17:22:52.374581+0000","last_clean":"2024-08-17T17:22:52.374581+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:22:52.374581+0000","last_undegraded":"2024-08-17T17:22:52.374581+0000","last_fullsized":"2024-08-17T17:22:52.374581+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1146,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.378972+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:22:52.378972+0000","last_peered":"2024-08-17T17:22:52.378972+0000","last_clean":"2024-08-17T17:22:52.378972+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:22:52.378972+0000","last_undegraded":"2024-08-17T17:22:52.378972+0000","last_fullsized":"2024-08-17T17:22:52.378972+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1001,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.374488+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:22:52.374488+0000","last_peered":"2024-08-17T17:22:52.374488+0000","last_clean":"2024-08-17T17:22:52.374488+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:22:52.374488+0000","last_undegraded":"2024-08-17T17:22:52.374488+0000","last_fullsized":"2024-08-17T17:22:52.374488+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1057,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.388168+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:22:52.388168+0000","last_peered":"2024-08-17T17:22:52.388168+0000","last_clean":"2024-08-17T17:22:52.388168+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:22:52.388168+0000","last_undegraded":"2024-08-17T17:22:52.388168+0000","last_fullsized":"2024-08-17T17:22:52.388168+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1117,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.374450+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:22:52.374450+0000","last_peered":"2024-08-17T17:22:52.374450+0000","last_clean":"2024-08-17T17:22:52.374450+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:22:52.374450+0000","last_undegraded":"2024-08-17T17:22:52.374450+0000","last_fullsized":"2024-08-17T17:22:52.374450+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1505,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.387878+0000","last_change":"2024-08-17T17:21:06.442598+0000","last_active":"2024-08-17T17:22:52.387878+0000","last_peered":"2024-08-17T17:22:52.387878+0000","last_clean":"2024-08-17T17:22:52.387878+0000","last_became_active":"2024-08-17T17:21:06.442264+0000","last_became_peered":"2024-08-17T17:21:06.442264+0000","last_unstale":"2024-08-17T17:22:52.387878+0000","last_undegraded":"2024-08-17T17:22:52.387878+0000","last_fullsized":"2024-08-17T17:22:52.387878+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:35.003620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1194,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.371542+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:22:52.371542+0000","last_peered":"2024-08-17T17:22:52.371542+0000","last_clean":"2024-08-17T17:22:52.371542+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:22:52.371542+0000","last_undegraded":"2024-08-17T17:22:52.371542+0000","last_fullsized":"2024-08-17T17:22:52.371542+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"1206'3840","reported_seq":3860,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.308244+0000","last_change":"2024-08-17T17:22:10.861776+0000","last_active":"2024-08-17T17:22:57.308244+0000","last_peered":"2024-08-17T17:22:57.308244+0000","last_clean":"2024-08-17T17:22:57.308244+0000","last_became_active":"2024-08-17T17:22:10.861509+0000","last_became_peered":"2024-08-17T17:22:10.861509+0000","last_unstale":"2024-08-17T17:22:57.308244+0000","last_undegraded":"2024-08-17T17:22:57.308244+0000","last_fullsized":"2024-08-17T17:22:57.308244+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3840,"log_dups_size":0,"ondisk_log_size":3840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:16:43.850909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251658240,"num_objects":3840,"num_object_clones":0,"num_object_copies":11520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3840,"num_write_kb":245760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1160,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.420989+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:22:56.420989+0000","last_peered":"2024-08-17T17:22:56.420989+0000","last_clean":"2024-08-17T17:22:56.420989+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:22:56.420989+0000","last_undegraded":"2024-08-17T17:22:56.420989+0000","last_fullsized":"2024-08-17T17:22:56.420989+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1132,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.379573+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:22:52.379573+0000","last_peered":"2024-08-17T17:22:52.379573+0000","last_clean":"2024-08-17T17:22:52.379573+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:22:52.379573+0000","last_undegraded":"2024-08-17T17:22:52.379573+0000","last_fullsized":"2024-08-17T17:22:52.379573+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"1206'3899","reported_seq":3919,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.205549+0000","last_change":"2024-08-17T17:22:10.860210+0000","last_active":"2024-08-17T17:22:57.205549+0000","last_peered":"2024-08-17T17:22:57.205549+0000","last_clean":"2024-08-17T17:22:57.205549+0000","last_became_active":"2024-08-17T17:22:10.859854+0000","last_became_peered":"2024-08-17T17:22:10.859854+0000","last_unstale":"2024-08-17T17:22:57.205549+0000","last_undegraded":"2024-08-17T17:22:57.205549+0000","last_fullsized":"2024-08-17T17:22:57.205549+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3899,"log_dups_size":0,"ondisk_log_size":3899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:31.313667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255524864,"num_objects":3899,"num_object_clones":0,"num_object_copies":11697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3899,"num_write_kb":249536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1161,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421039+0000","last_change":"2024-08-17T17:21:06.540689+0000","last_active":"2024-08-17T17:22:56.421039+0000","last_peered":"2024-08-17T17:22:56.421039+0000","last_clean":"2024-08-17T17:22:56.421039+0000","last_became_active":"2024-08-17T17:21:06.461614+0000","last_became_peered":"2024-08-17T17:21:06.461614+0000","last_unstale":"2024-08-17T17:22:56.421039+0000","last_undegraded":"2024-08-17T17:22:56.421039+0000","last_fullsized":"2024-08-17T17:22:56.421039+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:11.696898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1184,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.373296+0000","last_change":"2024-08-17T17:21:06.470389+0000","last_active":"2024-08-17T17:22:52.373296+0000","last_peered":"2024-08-17T17:22:52.373296+0000","last_clean":"2024-08-17T17:22:52.373296+0000","last_became_active":"2024-08-17T17:21:06.469966+0000","last_became_peered":"2024-08-17T17:21:06.469966+0000","last_unstale":"2024-08-17T17:22:52.373296+0000","last_undegraded":"2024-08-17T17:22:52.373296+0000","last_fullsized":"2024-08-17T17:22:52.373296+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:56.140616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"1206'3898","reported_seq":3918,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.201780+0000","last_change":"2024-08-17T17:22:10.943999+0000","last_active":"2024-08-17T17:22:57.201780+0000","last_peered":"2024-08-17T17:22:57.201780+0000","last_clean":"2024-08-17T17:22:57.201780+0000","last_became_active":"2024-08-17T17:22:10.943661+0000","last_became_peered":"2024-08-17T17:22:10.943661+0000","last_unstale":"2024-08-17T17:22:57.201780+0000","last_undegraded":"2024-08-17T17:22:57.201780+0000","last_fullsized":"2024-08-17T17:22:57.201780+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3898,"log_dups_size":0,"ondisk_log_size":3898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:21:49.050541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255459328,"num_objects":3898,"num_object_clones":0,"num_object_copies":11694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3898,"num_write_kb":249472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1119,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.374803+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:22:52.374803+0000","last_peered":"2024-08-17T17:22:52.374803+0000","last_clean":"2024-08-17T17:22:52.374803+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:22:52.374803+0000","last_undegraded":"2024-08-17T17:22:52.374803+0000","last_fullsized":"2024-08-17T17:22:52.374803+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1166,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.371792+0000","last_change":"2024-08-17T17:21:06.446798+0000","last_active":"2024-08-17T17:22:52.371792+0000","last_peered":"2024-08-17T17:22:52.371792+0000","last_clean":"2024-08-17T17:22:52.371792+0000","last_became_active":"2024-08-17T17:21:06.446205+0000","last_became_peered":"2024-08-17T17:21:06.446205+0000","last_unstale":"2024-08-17T17:22:52.371792+0000","last_undegraded":"2024-08-17T17:22:52.371792+0000","last_fullsized":"2024-08-17T17:22:52.371792+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:28.584048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"1205'3683","reported_seq":3714,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:55.065567+0000","last_change":"2024-08-17T17:22:10.900862+0000","last_active":"2024-08-17T17:22:55.065567+0000","last_peered":"2024-08-17T17:22:55.065567+0000","last_clean":"2024-08-17T17:22:55.065567+0000","last_became_active":"2024-08-17T17:22:10.900552+0000","last_became_peered":"2024-08-17T17:22:10.900552+0000","last_unstale":"2024-08-17T17:22:55.065567+0000","last_undegraded":"2024-08-17T17:22:55.065567+0000","last_fullsized":"2024-08-17T17:22:55.065567+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3683,"log_dups_size":0,"ondisk_log_size":3683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:07.195144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241369088,"num_objects":3683,"num_object_clones":0,"num_object_copies":11049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3683,"num_write_kb":235712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1148,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.379517+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:22:52.379517+0000","last_peered":"2024-08-17T17:22:52.379517+0000","last_clean":"2024-08-17T17:22:52.379517+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:22:52.379517+0000","last_undegraded":"2024-08-17T17:22:52.379517+0000","last_fullsized":"2024-08-17T17:22:52.379517+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":929,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.373255+0000","last_change":"2024-08-17T17:21:06.460905+0000","last_active":"2024-08-17T17:22:52.373255+0000","last_peered":"2024-08-17T17:22:52.373255+0000","last_clean":"2024-08-17T17:22:52.373255+0000","last_became_active":"2024-08-17T17:21:06.460592+0000","last_became_peered":"2024-08-17T17:21:06.460592+0000","last_unstale":"2024-08-17T17:22:52.373255+0000","last_undegraded":"2024-08-17T17:22:52.373255+0000","last_fullsized":"2024-08-17T17:22:52.373255+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:10.146916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.0","version":"1205'3664","reported_seq":3695,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:55.070377+0000","last_change":"2024-08-17T17:22:10.900154+0000","last_active":"2024-08-17T17:22:55.070377+0000","last_peered":"2024-08-17T17:22:55.070377+0000","last_clean":"2024-08-17T17:22:55.070377+0000","last_became_active":"2024-08-17T17:22:10.899766+0000","last_became_peered":"2024-08-17T17:22:10.899766+0000","last_unstale":"2024-08-17T17:22:55.070377+0000","last_undegraded":"2024-08-17T17:22:55.070377+0000","last_fullsized":"2024-08-17T17:22:55.070377+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3664,"log_dups_size":0,"ondisk_log_size":3664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:07.705078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240123904,"num_objects":3664,"num_object_clones":0,"num_object_copies":10992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3664,"num_write_kb":234496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1527,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.374314+0000","last_change":"2024-08-17T17:21:16.100398+0000","last_active":"2024-08-17T17:22:52.374314+0000","last_peered":"2024-08-17T17:22:52.374314+0000","last_clean":"2024-08-17T17:22:52.374314+0000","last_became_active":"2024-08-17T17:18:28.521573+0000","last_became_peered":"2024-08-17T17:18:28.521573+0000","last_unstale":"2024-08-17T17:22:52.374314+0000","last_undegraded":"2024-08-17T17:22:52.374314+0000","last_fullsized":"2024-08-17T17:22:52.374314+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:21:16.100258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:21:16.100258+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:57.628353+0000","scrub_duration":0.011045358999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1526,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.387589+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:22:52.387589+0000","last_peered":"2024-08-17T17:22:52.387589+0000","last_clean":"2024-08-17T17:22:52.387589+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:22:52.387589+0000","last_undegraded":"2024-08-17T17:22:52.387589+0000","last_fullsized":"2024-08-17T17:22:52.387589+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1151,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.418617+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:22:56.418617+0000","last_peered":"2024-08-17T17:22:56.418617+0000","last_clean":"2024-08-17T17:22:56.418617+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:22:56.418617+0000","last_undegraded":"2024-08-17T17:22:56.418617+0000","last_fullsized":"2024-08-17T17:22:56.418617+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"1206'3805","reported_seq":3838,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.495388+0000","last_change":"2024-08-17T17:22:10.944783+0000","last_active":"2024-08-17T17:22:56.495388+0000","last_peered":"2024-08-17T17:22:56.495388+0000","last_clean":"2024-08-17T17:22:56.495388+0000","last_became_active":"2024-08-17T17:22:10.944487+0000","last_became_peered":"2024-08-17T17:22:10.944487+0000","last_unstale":"2024-08-17T17:22:56.495388+0000","last_undegraded":"2024-08-17T17:22:56.495388+0000","last_fullsized":"2024-08-17T17:22:56.495388+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3805,"log_dups_size":0,"ondisk_log_size":3805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.515134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249364480,"num_objects":3805,"num_object_clones":0,"num_object_copies":11415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3805,"num_write_kb":243520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1473,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.379494+0000","last_change":"2024-08-17T17:18:32.884334+0000","last_active":"2024-08-17T17:22:52.379494+0000","last_peered":"2024-08-17T17:22:52.379494+0000","last_clean":"2024-08-17T17:22:52.379494+0000","last_became_active":"2024-08-17T17:18:28.520186+0000","last_became_peered":"2024-08-17T17:18:28.520186+0000","last_unstale":"2024-08-17T17:22:52.379494+0000","last_undegraded":"2024-08-17T17:22:52.379494+0000","last_fullsized":"2024-08-17T17:22:52.379494+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:32.884246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:32.884246+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:49.286656+0000","scrub_duration":0.001842677,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1167,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.371933+0000","last_change":"2024-08-17T17:21:06.446841+0000","last_active":"2024-08-17T17:22:52.371933+0000","last_peered":"2024-08-17T17:22:52.371933+0000","last_clean":"2024-08-17T17:22:52.371933+0000","last_became_active":"2024-08-17T17:21:06.446271+0000","last_became_peered":"2024-08-17T17:21:06.446271+0000","last_unstale":"2024-08-17T17:22:52.371933+0000","last_undegraded":"2024-08-17T17:22:52.371933+0000","last_fullsized":"2024-08-17T17:22:52.371933+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:20.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1184,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417678+0000","last_change":"2024-08-17T17:21:06.480911+0000","last_active":"2024-08-17T17:22:56.417678+0000","last_peered":"2024-08-17T17:22:56.417678+0000","last_clean":"2024-08-17T17:22:56.417678+0000","last_became_active":"2024-08-17T17:21:06.480551+0000","last_became_peered":"2024-08-17T17:21:06.480551+0000","last_unstale":"2024-08-17T17:22:56.417678+0000","last_undegraded":"2024-08-17T17:22:56.417678+0000","last_fullsized":"2024-08-17T17:22:56.417678+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:26.003704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"1206'3972","reported_seq":3992,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.214595+0000","last_change":"2024-08-17T17:22:10.944020+0000","last_active":"2024-08-17T17:22:57.214595+0000","last_peered":"2024-08-17T17:22:57.214595+0000","last_clean":"2024-08-17T17:22:57.214595+0000","last_became_active":"2024-08-17T17:22:10.943694+0000","last_became_peered":"2024-08-17T17:22:10.943694+0000","last_unstale":"2024-08-17T17:22:57.214595+0000","last_undegraded":"2024-08-17T17:22:57.214595+0000","last_fullsized":"2024-08-17T17:22:57.214595+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3972,"log_dups_size":0,"ondisk_log_size":3972,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:34.525383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260308992,"num_objects":3972,"num_object_clones":0,"num_object_copies":11916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3972,"num_write_kb":254208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1152,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.373374+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:22:52.373374+0000","last_peered":"2024-08-17T17:22:52.373374+0000","last_clean":"2024-08-17T17:22:52.373374+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:22:52.373374+0000","last_undegraded":"2024-08-17T17:22:52.373374+0000","last_fullsized":"2024-08-17T17:22:52.373374+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1168,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.373558+0000","last_change":"2024-08-17T17:21:06.451521+0000","last_active":"2024-08-17T17:22:52.373558+0000","last_peered":"2024-08-17T17:22:52.373558+0000","last_clean":"2024-08-17T17:22:52.373558+0000","last_became_active":"2024-08-17T17:21:06.451197+0000","last_became_peered":"2024-08-17T17:21:06.451197+0000","last_unstale":"2024-08-17T17:22:52.373558+0000","last_undegraded":"2024-08-17T17:22:52.373558+0000","last_fullsized":"2024-08-17T17:22:52.373558+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:04.000692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"1206'3999","reported_seq":4019,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.302959+0000","last_change":"2024-08-17T17:22:10.900282+0000","last_active":"2024-08-17T17:22:57.302959+0000","last_peered":"2024-08-17T17:22:57.302959+0000","last_clean":"2024-08-17T17:22:57.302959+0000","last_became_active":"2024-08-17T17:22:10.899947+0000","last_became_peered":"2024-08-17T17:22:10.899947+0000","last_unstale":"2024-08-17T17:22:57.302959+0000","last_undegraded":"2024-08-17T17:22:57.302959+0000","last_fullsized":"2024-08-17T17:22:57.302959+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3999,"log_dups_size":0,"ondisk_log_size":3999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:10.148524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262078464,"num_objects":3999,"num_object_clones":0,"num_object_copies":11997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3999,"num_write_kb":255936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1473,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.378890+0000","last_change":"2024-08-17T17:18:31.850829+0000","last_active":"2024-08-17T17:22:52.378890+0000","last_peered":"2024-08-17T17:22:52.378890+0000","last_clean":"2024-08-17T17:22:52.378890+0000","last_became_active":"2024-08-17T17:18:28.520946+0000","last_became_peered":"2024-08-17T17:18:28.520946+0000","last_unstale":"2024-08-17T17:22:52.378890+0000","last_undegraded":"2024-08-17T17:22:52.378890+0000","last_fullsized":"2024-08-17T17:22:52.378890+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:31.850745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:31.850745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:17.218084+0000","scrub_duration":0.001317447,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1508,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.386786+0000","last_change":"2024-08-17T17:21:06.441118+0000","last_active":"2024-08-17T17:22:52.386786+0000","last_peered":"2024-08-17T17:22:52.386786+0000","last_clean":"2024-08-17T17:22:52.386786+0000","last_became_active":"2024-08-17T17:21:06.440800+0000","last_became_peered":"2024-08-17T17:21:06.440800+0000","last_unstale":"2024-08-17T17:22:52.386786+0000","last_undegraded":"2024-08-17T17:22:52.386786+0000","last_fullsized":"2024-08-17T17:22:52.386786+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:08.091528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":16,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421723+0000","last_change":"2024-08-17T17:22:48.374766+0000","last_active":"2024-08-17T17:22:56.421723+0000","last_peered":"2024-08-17T17:22:56.421723+0000","last_clean":"2024-08-17T17:22:56.421723+0000","last_became_active":"2024-08-17T17:22:48.373556+0000","last_became_peered":"2024-08-17T17:22:48.373556+0000","last_unstale":"2024-08-17T17:22:56.421723+0000","last_undegraded":"2024-08-17T17:22:56.421723+0000","last_fullsized":"2024-08-17T17:22:56.421723+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:33.362080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.9","version":"1206'3908","reported_seq":3928,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.204425+0000","last_change":"2024-08-17T17:22:10.944057+0000","last_active":"2024-08-17T17:22:57.204425+0000","last_peered":"2024-08-17T17:22:57.204425+0000","last_clean":"2024-08-17T17:22:57.204425+0000","last_became_active":"2024-08-17T17:22:10.943681+0000","last_became_peered":"2024-08-17T17:22:10.943681+0000","last_unstale":"2024-08-17T17:22:57.204425+0000","last_undegraded":"2024-08-17T17:22:57.204425+0000","last_fullsized":"2024-08-17T17:22:57.204425+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3908,"log_dups_size":0,"ondisk_log_size":3908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:24.524487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256114688,"num_objects":3908,"num_object_clones":0,"num_object_copies":11724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3908,"num_write_kb":250112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1527,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.374392+0000","last_change":"2024-08-17T17:18:33.814375+0000","last_active":"2024-08-17T17:22:52.374392+0000","last_peered":"2024-08-17T17:22:52.374392+0000","last_clean":"2024-08-17T17:22:52.374392+0000","last_became_active":"2024-08-17T17:18:33.813208+0000","last_became_peered":"2024-08-17T17:18:33.813208+0000","last_unstale":"2024-08-17T17:22:52.374392+0000","last_undegraded":"2024-08-17T17:22:52.374392+0000","last_fullsized":"2024-08-17T17:22:52.374392+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:25.057675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:25.057675+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:20:45.235879+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1162,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.379078+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:22:52.379078+0000","last_peered":"2024-08-17T17:22:52.379078+0000","last_clean":"2024-08-17T17:22:52.379078+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:22:52.379078+0000","last_undegraded":"2024-08-17T17:22:52.379078+0000","last_fullsized":"2024-08-17T17:22:52.379078+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1473,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.387143+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:22:52.387143+0000","last_peered":"2024-08-17T17:22:52.387143+0000","last_clean":"2024-08-17T17:22:52.387143+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:22:52.387143+0000","last_undegraded":"2024-08-17T17:22:52.387143+0000","last_fullsized":"2024-08-17T17:22:52.387143+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1134,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415795+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:22:56.415795+0000","last_peered":"2024-08-17T17:22:56.415795+0000","last_clean":"2024-08-17T17:22:56.415795+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:22:56.415795+0000","last_undegraded":"2024-08-17T17:22:56.415795+0000","last_fullsized":"2024-08-17T17:22:56.415795+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"1206'3870","reported_seq":3903,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.468057+0000","last_change":"2024-08-17T17:22:10.900900+0000","last_active":"2024-08-17T17:22:56.468057+0000","last_peered":"2024-08-17T17:22:56.468057+0000","last_clean":"2024-08-17T17:22:56.468057+0000","last_became_active":"2024-08-17T17:22:10.899812+0000","last_became_peered":"2024-08-17T17:22:10.899812+0000","last_unstale":"2024-08-17T17:22:56.468057+0000","last_undegraded":"2024-08-17T17:22:56.468057+0000","last_fullsized":"2024-08-17T17:22:56.468057+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3870,"log_dups_size":0,"ondisk_log_size":3870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:25:56.258127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253624320,"num_objects":3870,"num_object_clones":0,"num_object_copies":11610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3870,"num_write_kb":247680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":766,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.371397+0000","last_change":"2024-08-17T17:22:09.230320+0000","last_active":"2024-08-17T17:22:52.371397+0000","last_peered":"2024-08-17T17:22:52.371397+0000","last_clean":"2024-08-17T17:22:52.371397+0000","last_became_active":"2024-08-17T17:21:08.910463+0000","last_became_peered":"2024-08-17T17:21:08.910463+0000","last_unstale":"2024-08-17T17:22:52.371397+0000","last_undegraded":"2024-08-17T17:22:52.371397+0000","last_fullsized":"2024-08-17T17:22:52.371397+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:09.230214+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.230214+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:58.582858+0000","scrub_duration":0.001729597,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1198,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.374347+0000","last_change":"2024-08-17T17:21:06.430639+0000","last_active":"2024-08-17T17:22:52.374347+0000","last_peered":"2024-08-17T17:22:52.374347+0000","last_clean":"2024-08-17T17:22:52.374347+0000","last_became_active":"2024-08-17T17:21:06.430297+0000","last_became_peered":"2024-08-17T17:21:06.430297+0000","last_unstale":"2024-08-17T17:22:52.374347+0000","last_undegraded":"2024-08-17T17:22:52.374347+0000","last_fullsized":"2024-08-17T17:22:52.374347+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:57.794620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1001,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.373038+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:22:52.373038+0000","last_peered":"2024-08-17T17:22:52.373038+0000","last_clean":"2024-08-17T17:22:52.373038+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:22:52.373038+0000","last_undegraded":"2024-08-17T17:22:52.373038+0000","last_fullsized":"2024-08-17T17:22:52.373038+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.378961+0000","last_change":"2024-08-17T17:22:48.370769+0000","last_active":"2024-08-17T17:22:52.378961+0000","last_peered":"2024-08-17T17:22:52.378961+0000","last_clean":"2024-08-17T17:22:52.378961+0000","last_became_active":"2024-08-17T17:22:48.370531+0000","last_became_peered":"2024-08-17T17:22:48.370531+0000","last_unstale":"2024-08-17T17:22:52.378961+0000","last_undegraded":"2024-08-17T17:22:52.378961+0000","last_fullsized":"2024-08-17T17:22:52.378961+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:02.943823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.b","version":"1205'3743","reported_seq":3774,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:55.094679+0000","last_change":"2024-08-17T17:22:10.900740+0000","last_active":"2024-08-17T17:22:55.094679+0000","last_peered":"2024-08-17T17:22:55.094679+0000","last_clean":"2024-08-17T17:22:55.094679+0000","last_became_active":"2024-08-17T17:22:10.900452+0000","last_became_peered":"2024-08-17T17:22:10.900452+0000","last_unstale":"2024-08-17T17:22:55.094679+0000","last_undegraded":"2024-08-17T17:22:55.094679+0000","last_fullsized":"2024-08-17T17:22:55.094679+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3743,"log_dups_size":0,"ondisk_log_size":3743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:38:40.285223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245301248,"num_objects":3743,"num_object_clones":0,"num_object_copies":11229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3743,"num_write_kb":239552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1548,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.374465+0000","last_change":"2024-08-17T17:22:07.078517+0000","last_active":"2024-08-17T17:22:52.374465+0000","last_peered":"2024-08-17T17:22:52.374465+0000","last_clean":"2024-08-17T17:22:52.374465+0000","last_became_active":"2024-08-17T17:18:33.816660+0000","last_became_peered":"2024-08-17T17:18:33.816660+0000","last_unstale":"2024-08-17T17:22:52.374465+0000","last_undegraded":"2024-08-17T17:22:52.374465+0000","last_fullsized":"2024-08-17T17:22:52.374465+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:07.078434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:07.078434+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:56.205556+0000","scrub_duration":0.0013001919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1572,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.422379+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:22:56.422379+0000","last_peered":"2024-08-17T17:22:56.422379+0000","last_clean":"2024-08-17T17:22:56.422379+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:22:56.422379+0000","last_undegraded":"2024-08-17T17:22:56.422379+0000","last_fullsized":"2024-08-17T17:22:56.422379+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":964,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421250+0000","last_change":"2024-08-17T17:21:06.428684+0000","last_active":"2024-08-17T17:22:56.421250+0000","last_peered":"2024-08-17T17:22:56.421250+0000","last_clean":"2024-08-17T17:22:56.421250+0000","last_became_active":"2024-08-17T17:21:06.428178+0000","last_became_peered":"2024-08-17T17:21:06.428178+0000","last_unstale":"2024-08-17T17:22:56.421250+0000","last_undegraded":"2024-08-17T17:22:56.421250+0000","last_fullsized":"2024-08-17T17:22:56.421250+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:14.836119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":845,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.378997+0000","last_change":"2024-08-17T17:21:06.443565+0000","last_active":"2024-08-17T17:22:52.378997+0000","last_peered":"2024-08-17T17:22:52.378997+0000","last_clean":"2024-08-17T17:22:52.378997+0000","last_became_active":"2024-08-17T17:21:06.443215+0000","last_became_peered":"2024-08-17T17:21:06.443215+0000","last_unstale":"2024-08-17T17:22:52.378997+0000","last_undegraded":"2024-08-17T17:22:52.378997+0000","last_fullsized":"2024-08-17T17:22:52.378997+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:47.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"1206'3931","reported_seq":3951,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.302736+0000","last_change":"2024-08-17T17:22:10.859676+0000","last_active":"2024-08-17T17:22:57.302736+0000","last_peered":"2024-08-17T17:22:57.302736+0000","last_clean":"2024-08-17T17:22:57.302736+0000","last_became_active":"2024-08-17T17:22:10.859289+0000","last_became_peered":"2024-08-17T17:22:10.859289+0000","last_unstale":"2024-08-17T17:22:57.302736+0000","last_undegraded":"2024-08-17T17:22:57.302736+0000","last_fullsized":"2024-08-17T17:22:57.302736+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3931,"log_dups_size":0,"ondisk_log_size":3931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:43.302993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257622016,"num_objects":3931,"num_object_clones":0,"num_object_copies":11793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3931,"num_write_kb":251584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1490,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.378703+0000","last_change":"2024-08-17T17:18:35.916482+0000","last_active":"2024-08-17T17:22:52.378703+0000","last_peered":"2024-08-17T17:22:52.378703+0000","last_clean":"2024-08-17T17:22:52.378703+0000","last_became_active":"2024-08-17T17:18:34.827747+0000","last_became_peered":"2024-08-17T17:18:34.827747+0000","last_unstale":"2024-08-17T17:22:52.378703+0000","last_undegraded":"2024-08-17T17:22:52.378703+0000","last_fullsized":"2024-08-17T17:22:52.378703+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:35.916381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:35.916381+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:28.492622+0000","scrub_duration":0.005340521,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1189,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.379321+0000","last_change":"2024-08-17T17:21:06.456978+0000","last_active":"2024-08-17T17:22:52.379321+0000","last_peered":"2024-08-17T17:22:52.379321+0000","last_clean":"2024-08-17T17:22:52.379321+0000","last_became_active":"2024-08-17T17:21:06.452265+0000","last_became_peered":"2024-08-17T17:21:06.452265+0000","last_unstale":"2024-08-17T17:22:52.379321+0000","last_undegraded":"2024-08-17T17:22:52.379321+0000","last_fullsized":"2024-08-17T17:22:52.379321+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:52.893746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1171,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.373710+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:22:52.373710+0000","last_peered":"2024-08-17T17:22:52.373710+0000","last_clean":"2024-08-17T17:22:52.373710+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:22:52.373710+0000","last_undegraded":"2024-08-17T17:22:52.373710+0000","last_fullsized":"2024-08-17T17:22:52.373710+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1040,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421603+0000","last_change":"2024-08-17T17:21:06.435754+0000","last_active":"2024-08-17T17:22:56.421603+0000","last_peered":"2024-08-17T17:22:56.421603+0000","last_clean":"2024-08-17T17:22:56.421603+0000","last_became_active":"2024-08-17T17:21:06.435442+0000","last_became_peered":"2024-08-17T17:21:06.435442+0000","last_unstale":"2024-08-17T17:22:56.421603+0000","last_undegraded":"2024-08-17T17:22:56.421603+0000","last_fullsized":"2024-08-17T17:22:56.421603+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:34.849042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"1206'3815","reported_seq":3848,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421553+0000","last_change":"2024-08-17T17:22:10.944986+0000","last_active":"2024-08-17T17:22:56.421553+0000","last_peered":"2024-08-17T17:22:56.421553+0000","last_clean":"2024-08-17T17:22:56.421553+0000","last_became_active":"2024-08-17T17:22:10.944267+0000","last_became_peered":"2024-08-17T17:22:10.944267+0000","last_unstale":"2024-08-17T17:22:56.421553+0000","last_undegraded":"2024-08-17T17:22:56.421553+0000","last_fullsized":"2024-08-17T17:22:56.421553+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3815,"log_dups_size":0,"ondisk_log_size":3815,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:52.209246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250019840,"num_objects":3815,"num_object_clones":0,"num_object_copies":11445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3815,"num_write_kb":244160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":732,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.371353+0000","last_change":"2024-08-17T17:22:06.152683+0000","last_active":"2024-08-17T17:22:52.371353+0000","last_peered":"2024-08-17T17:22:52.371353+0000","last_clean":"2024-08-17T17:22:52.371353+0000","last_became_active":"2024-08-17T17:18:28.526439+0000","last_became_peered":"2024-08-17T17:18:28.526439+0000","last_unstale":"2024-08-17T17:22:52.371353+0000","last_undegraded":"2024-08-17T17:22:52.371353+0000","last_fullsized":"2024-08-17T17:22:52.371353+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:06.152592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:06.152592+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:30.807010+0000","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1164,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.379216+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:22:52.379216+0000","last_peered":"2024-08-17T17:22:52.379216+0000","last_clean":"2024-08-17T17:22:52.379216+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:22:52.379216+0000","last_undegraded":"2024-08-17T17:22:52.379216+0000","last_fullsized":"2024-08-17T17:22:52.379216+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":854,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.376373+0000","last_change":"2024-08-17T17:21:06.434991+0000","last_active":"2024-08-17T17:22:52.376373+0000","last_peered":"2024-08-17T17:22:52.376373+0000","last_clean":"2024-08-17T17:22:52.376373+0000","last_became_active":"2024-08-17T17:21:06.434630+0000","last_became_peered":"2024-08-17T17:21:06.434630+0000","last_unstale":"2024-08-17T17:22:52.376373+0000","last_undegraded":"2024-08-17T17:22:52.376373+0000","last_fullsized":"2024-08-17T17:22:52.376373+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:24.104609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":796,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.379181+0000","last_change":"2024-08-17T17:21:06.462532+0000","last_active":"2024-08-17T17:22:52.379181+0000","last_peered":"2024-08-17T17:22:52.379181+0000","last_clean":"2024-08-17T17:22:52.379181+0000","last_became_active":"2024-08-17T17:21:06.462183+0000","last_became_peered":"2024-08-17T17:21:06.462183+0000","last_unstale":"2024-08-17T17:22:52.379181+0000","last_undegraded":"2024-08-17T17:22:52.379181+0000","last_fullsized":"2024-08-17T17:22:52.379181+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:50.033322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.f","version":"1205'3760","reported_seq":3791,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:55.082937+0000","last_change":"2024-08-17T17:22:10.900592+0000","last_active":"2024-08-17T17:22:55.082937+0000","last_peered":"2024-08-17T17:22:55.082937+0000","last_clean":"2024-08-17T17:22:55.082937+0000","last_became_active":"2024-08-17T17:22:10.900279+0000","last_became_peered":"2024-08-17T17:22:10.900279+0000","last_unstale":"2024-08-17T17:22:55.082937+0000","last_undegraded":"2024-08-17T17:22:55.082937+0000","last_fullsized":"2024-08-17T17:22:55.082937+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3760,"log_dups_size":0,"ondisk_log_size":3760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:52:25.130107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246415360,"num_objects":3760,"num_object_clones":0,"num_object_copies":11280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3760,"num_write_kb":240640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":749,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.371809+0000","last_change":"2024-08-17T17:22:08.218836+0000","last_active":"2024-08-17T17:22:52.371809+0000","last_peered":"2024-08-17T17:22:52.371809+0000","last_clean":"2024-08-17T17:22:52.371809+0000","last_became_active":"2024-08-17T17:18:34.831498+0000","last_became_peered":"2024-08-17T17:18:34.831498+0000","last_unstale":"2024-08-17T17:22:52.371809+0000","last_undegraded":"2024-08-17T17:22:52.371809+0000","last_fullsized":"2024-08-17T17:22:52.371809+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:08.218709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:08.218709+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:19.097715+0000","scrub_duration":0.002239024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":963,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421419+0000","last_change":"2024-08-17T17:21:06.433720+0000","last_active":"2024-08-17T17:22:56.421419+0000","last_peered":"2024-08-17T17:22:56.421419+0000","last_clean":"2024-08-17T17:22:56.421419+0000","last_became_active":"2024-08-17T17:21:06.433292+0000","last_became_peered":"2024-08-17T17:21:06.433292+0000","last_unstale":"2024-08-17T17:22:56.421419+0000","last_undegraded":"2024-08-17T17:22:56.421419+0000","last_fullsized":"2024-08-17T17:22:56.421419+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:32.246594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1164,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.374803+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:22:52.374803+0000","last_peered":"2024-08-17T17:22:52.374803+0000","last_clean":"2024-08-17T17:22:52.374803+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:22:52.374803+0000","last_undegraded":"2024-08-17T17:22:52.374803+0000","last_fullsized":"2024-08-17T17:22:52.374803+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":756,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.379004+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:22:52.379004+0000","last_peered":"2024-08-17T17:22:52.379004+0000","last_clean":"2024-08-17T17:22:52.379004+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:22:52.379004+0000","last_undegraded":"2024-08-17T17:22:52.379004+0000","last_fullsized":"2024-08-17T17:22:52.379004+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"1205'3738","reported_seq":3769,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:55.080295+0000","last_change":"2024-08-17T17:22:10.944050+0000","last_active":"2024-08-17T17:22:55.080295+0000","last_peered":"2024-08-17T17:22:55.080295+0000","last_clean":"2024-08-17T17:22:55.080295+0000","last_became_active":"2024-08-17T17:22:10.943712+0000","last_became_peered":"2024-08-17T17:22:10.943712+0000","last_unstale":"2024-08-17T17:22:55.080295+0000","last_undegraded":"2024-08-17T17:22:55.080295+0000","last_fullsized":"2024-08-17T17:22:55.080295+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3738,"log_dups_size":0,"ondisk_log_size":3738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:44.623031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244973568,"num_objects":3738,"num_object_clones":0,"num_object_copies":11214,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3738,"num_write_kb":239232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1473,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.373716+0000","last_change":"2024-08-17T17:22:06.995885+0000","last_active":"2024-08-17T17:22:52.373716+0000","last_peered":"2024-08-17T17:22:52.373716+0000","last_clean":"2024-08-17T17:22:52.373716+0000","last_became_active":"2024-08-17T17:18:28.526484+0000","last_became_peered":"2024-08-17T17:18:28.526484+0000","last_unstale":"2024-08-17T17:22:52.373716+0000","last_undegraded":"2024-08-17T17:22:52.373716+0000","last_fullsized":"2024-08-17T17:22:52.373716+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:06.995792+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:06.995792+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:06.341398+0000","scrub_duration":0.0016139520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1527,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.388488+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:22:52.388488+0000","last_peered":"2024-08-17T17:22:52.388488+0000","last_clean":"2024-08-17T17:22:52.388488+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:22:52.388488+0000","last_undegraded":"2024-08-17T17:22:52.388488+0000","last_fullsized":"2024-08-17T17:22:52.388488+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":806,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.372005+0000","last_change":"2024-08-17T17:21:06.452395+0000","last_active":"2024-08-17T17:22:52.372005+0000","last_peered":"2024-08-17T17:22:52.372005+0000","last_clean":"2024-08-17T17:22:52.372005+0000","last_became_active":"2024-08-17T17:21:06.451867+0000","last_became_peered":"2024-08-17T17:21:06.451867+0000","last_unstale":"2024-08-17T17:22:52.372005+0000","last_undegraded":"2024-08-17T17:22:52.372005+0000","last_fullsized":"2024-08-17T17:22:52.372005+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:50.597092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1516,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.386325+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:22:52.386325+0000","last_peered":"2024-08-17T17:22:52.386325+0000","last_clean":"2024-08-17T17:22:52.386325+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:22:52.386325+0000","last_undegraded":"2024-08-17T17:22:52.386325+0000","last_fullsized":"2024-08-17T17:22:52.386325+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"1206'3937","reported_seq":3957,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.281388+0000","last_change":"2024-08-17T17:22:10.859993+0000","last_active":"2024-08-17T17:22:57.281388+0000","last_peered":"2024-08-17T17:22:57.281388+0000","last_clean":"2024-08-17T17:22:57.281388+0000","last_became_active":"2024-08-17T17:22:10.859296+0000","last_became_peered":"2024-08-17T17:22:10.859296+0000","last_unstale":"2024-08-17T17:22:57.281388+0000","last_undegraded":"2024-08-17T17:22:57.281388+0000","last_fullsized":"2024-08-17T17:22:57.281388+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":3937,"log_dups_size":0,"ondisk_log_size":3937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:42.095858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258015232,"num_objects":3937,"num_object_clones":0,"num_object_copies":11811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3937,"num_write_kb":251968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1532,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.387636+0000","last_change":"2024-08-17T17:18:29.390118+0000","last_active":"2024-08-17T17:22:52.387636+0000","last_peered":"2024-08-17T17:22:52.387636+0000","last_clean":"2024-08-17T17:22:52.387636+0000","last_became_active":"2024-08-17T17:18:28.528874+0000","last_became_peered":"2024-08-17T17:18:28.528874+0000","last_unstale":"2024-08-17T17:22:52.387636+0000","last_undegraded":"2024-08-17T17:22:52.387636+0000","last_fullsized":"2024-08-17T17:22:52.387636+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:29.390003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:29.390003+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:35.908604+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":175,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.373507+0000","last_change":"2024-08-17T17:21:06.519965+0000","last_active":"2024-08-17T17:22:52.373507+0000","last_peered":"2024-08-17T17:22:52.373507+0000","last_clean":"2024-08-17T17:22:52.373507+0000","last_became_active":"2024-08-17T17:21:06.459952+0000","last_became_peered":"2024-08-17T17:21:06.459952+0000","last_unstale":"2024-08-17T17:22:52.373507+0000","last_undegraded":"2024-08-17T17:22:52.373507+0000","last_fullsized":"2024-08-17T17:22:52.373507+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:27.828098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1161,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.371485+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:22:52.371485+0000","last_peered":"2024-08-17T17:22:52.371485+0000","last_clean":"2024-08-17T17:22:52.371485+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:22:52.371485+0000","last_undegraded":"2024-08-17T17:22:52.371485+0000","last_fullsized":"2024-08-17T17:22:52.371485+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1137,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.373117+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:22:52.373117+0000","last_peered":"2024-08-17T17:22:52.373117+0000","last_clean":"2024-08-17T17:22:52.373117+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:22:52.373117+0000","last_undegraded":"2024-08-17T17:22:52.373117+0000","last_fullsized":"2024-08-17T17:22:52.373117+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1168,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.373515+0000","last_change":"2024-08-17T17:21:06.458000+0000","last_active":"2024-08-17T17:22:52.373515+0000","last_peered":"2024-08-17T17:22:52.373515+0000","last_clean":"2024-08-17T17:22:52.373515+0000","last_became_active":"2024-08-17T17:21:06.457411+0000","last_became_peered":"2024-08-17T17:21:06.457411+0000","last_unstale":"2024-08-17T17:22:52.373515+0000","last_undegraded":"2024-08-17T17:22:52.373515+0000","last_fullsized":"2024-08-17T17:22:52.373515+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:52.740639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1159,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.373122+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:22:52.373122+0000","last_peered":"2024-08-17T17:22:52.373122+0000","last_clean":"2024-08-17T17:22:52.373122+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:22:52.373122+0000","last_undegraded":"2024-08-17T17:22:52.373122+0000","last_fullsized":"2024-08-17T17:22:52.373122+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":828,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.371864+0000","last_change":"2024-08-17T17:21:06.454587+0000","last_active":"2024-08-17T17:22:52.371864+0000","last_peered":"2024-08-17T17:22:52.371864+0000","last_clean":"2024-08-17T17:22:52.371864+0000","last_became_active":"2024-08-17T17:21:06.454229+0000","last_became_peered":"2024-08-17T17:21:06.454229+0000","last_unstale":"2024-08-17T17:22:52.371864+0000","last_undegraded":"2024-08-17T17:22:52.371864+0000","last_fullsized":"2024-08-17T17:22:52.371864+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:38.340263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1135,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.374714+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:22:52.374714+0000","last_peered":"2024-08-17T17:22:52.374714+0000","last_clean":"2024-08-17T17:22:52.374714+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:22:52.374714+0000","last_undegraded":"2024-08-17T17:22:52.374714+0000","last_fullsized":"2024-08-17T17:22:52.374714+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1165,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.379045+0000","last_change":"2024-08-17T17:21:06.442796+0000","last_active":"2024-08-17T17:22:52.379045+0000","last_peered":"2024-08-17T17:22:52.379045+0000","last_clean":"2024-08-17T17:22:52.379045+0000","last_became_active":"2024-08-17T17:21:06.442466+0000","last_became_peered":"2024-08-17T17:21:06.442466+0000","last_unstale":"2024-08-17T17:22:52.379045+0000","last_undegraded":"2024-08-17T17:22:52.379045+0000","last_fullsized":"2024-08-17T17:22:52.379045+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:36.366620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1472,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.386722+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:22:52.386722+0000","last_peered":"2024-08-17T17:22:52.386722+0000","last_clean":"2024-08-17T17:22:52.386722+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:22:52.386722+0000","last_undegraded":"2024-08-17T17:22:52.386722+0000","last_fullsized":"2024-08-17T17:22:52.386722+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1001,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.379130+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:22:52.379130+0000","last_peered":"2024-08-17T17:22:52.379130+0000","last_clean":"2024-08-17T17:22:52.379130+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:22:52.379130+0000","last_undegraded":"2024-08-17T17:22:52.379130+0000","last_fullsized":"2024-08-17T17:22:52.379130+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1505,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.386236+0000","last_change":"2024-08-17T17:21:06.468788+0000","last_active":"2024-08-17T17:22:52.386236+0000","last_peered":"2024-08-17T17:22:52.386236+0000","last_clean":"2024-08-17T17:22:52.386236+0000","last_became_active":"2024-08-17T17:21:06.467141+0000","last_became_peered":"2024-08-17T17:21:06.467141+0000","last_unstale":"2024-08-17T17:22:52.386236+0000","last_undegraded":"2024-08-17T17:22:52.386236+0000","last_fullsized":"2024-08-17T17:22:52.386236+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:51.301359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1167,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.371420+0000","last_change":"2024-08-17T17:21:06.435998+0000","last_active":"2024-08-17T17:22:52.371420+0000","last_peered":"2024-08-17T17:22:52.371420+0000","last_clean":"2024-08-17T17:22:52.371420+0000","last_became_active":"2024-08-17T17:21:06.435579+0000","last_became_peered":"2024-08-17T17:21:06.435579+0000","last_unstale":"2024-08-17T17:22:52.371420+0000","last_undegraded":"2024-08-17T17:22:52.371420+0000","last_fullsized":"2024-08-17T17:22:52.371420+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:01.743074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1535,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.386641+0000","last_change":"2024-08-17T17:21:06.473627+0000","last_active":"2024-08-17T17:22:52.386641+0000","last_peered":"2024-08-17T17:22:52.386641+0000","last_clean":"2024-08-17T17:22:52.386641+0000","last_became_active":"2024-08-17T17:21:06.473062+0000","last_became_peered":"2024-08-17T17:21:06.473062+0000","last_unstale":"2024-08-17T17:22:52.386641+0000","last_undegraded":"2024-08-17T17:22:52.386641+0000","last_fullsized":"2024-08-17T17:22:52.386641+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:21.413163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":961,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.374802+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:22:52.374802+0000","last_peered":"2024-08-17T17:22:52.374802+0000","last_clean":"2024-08-17T17:22:52.374802+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:22:52.374802+0000","last_undegraded":"2024-08-17T17:22:52.374802+0000","last_fullsized":"2024-08-17T17:22:52.374802+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1026,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.386955+0000","last_change":"2024-08-17T17:21:06.471265+0000","last_active":"2024-08-17T17:22:52.386955+0000","last_peered":"2024-08-17T17:22:52.386955+0000","last_clean":"2024-08-17T17:22:52.386955+0000","last_became_active":"2024-08-17T17:21:06.470943+0000","last_became_peered":"2024-08-17T17:21:06.470943+0000","last_unstale":"2024-08-17T17:22:52.386955+0000","last_undegraded":"2024-08-17T17:22:52.386955+0000","last_fullsized":"2024-08-17T17:22:52.386955+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:41.070024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1512,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.387700+0000","last_change":"2024-08-17T17:22:48.377351+0000","last_active":"2024-08-17T17:22:52.387700+0000","last_peered":"2024-08-17T17:22:52.387700+0000","last_clean":"2024-08-17T17:22:52.387700+0000","last_became_active":"2024-08-17T17:22:48.377059+0000","last_became_peered":"2024-08-17T17:22:48.377059+0000","last_unstale":"2024-08-17T17:22:52.387700+0000","last_undegraded":"2024-08-17T17:22:52.387700+0000","last_fullsized":"2024-08-17T17:22:52.387700+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:15.250902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1184,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.374062+0000","last_change":"2024-08-17T17:21:06.459139+0000","last_active":"2024-08-17T17:22:52.374062+0000","last_peered":"2024-08-17T17:22:52.374062+0000","last_clean":"2024-08-17T17:22:52.374062+0000","last_became_active":"2024-08-17T17:21:06.457850+0000","last_became_peered":"2024-08-17T17:21:06.457850+0000","last_unstale":"2024-08-17T17:22:52.374062+0000","last_undegraded":"2024-08-17T17:22:52.374062+0000","last_fullsized":"2024-08-17T17:22:52.374062+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:13.888576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1555,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421513+0000","last_change":"2024-08-17T17:21:06.446321+0000","last_active":"2024-08-17T17:22:56.421513+0000","last_peered":"2024-08-17T17:22:56.421513+0000","last_clean":"2024-08-17T17:22:56.421513+0000","last_became_active":"2024-08-17T17:21:06.446019+0000","last_became_peered":"2024-08-17T17:21:06.446019+0000","last_unstale":"2024-08-17T17:22:56.421513+0000","last_undegraded":"2024-08-17T17:22:56.421513+0000","last_fullsized":"2024-08-17T17:22:56.421513+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:26.750070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":154,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.373332+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:22:52.373332+0000","last_peered":"2024-08-17T17:22:52.373332+0000","last_clean":"2024-08-17T17:22:52.373332+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:22:52.373332+0000","last_undegraded":"2024-08-17T17:22:52.373332+0000","last_fullsized":"2024-08-17T17:22:52.373332+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1182,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.373449+0000","last_change":"2024-08-17T17:21:06.465804+0000","last_active":"2024-08-17T17:22:52.373449+0000","last_peered":"2024-08-17T17:22:52.373449+0000","last_clean":"2024-08-17T17:22:52.373449+0000","last_became_active":"2024-08-17T17:21:06.465461+0000","last_became_peered":"2024-08-17T17:21:06.465461+0000","last_unstale":"2024-08-17T17:22:52.373449+0000","last_undegraded":"2024-08-17T17:22:52.373449+0000","last_fullsized":"2024-08-17T17:22:52.373449+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:06.443667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1174,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.379375+0000","last_change":"2024-08-17T17:21:06.481634+0000","last_active":"2024-08-17T17:22:52.379375+0000","last_peered":"2024-08-17T17:22:52.379375+0000","last_clean":"2024-08-17T17:22:52.379375+0000","last_became_active":"2024-08-17T17:21:06.469081+0000","last_became_peered":"2024-08-17T17:21:06.469081+0000","last_unstale":"2024-08-17T17:22:52.379375+0000","last_undegraded":"2024-08-17T17:22:52.379375+0000","last_fullsized":"2024-08-17T17:22:52.379375+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:53:26.954738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1182,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.373985+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:22:52.373985+0000","last_peered":"2024-08-17T17:22:52.373985+0000","last_clean":"2024-08-17T17:22:52.373985+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:22:52.373985+0000","last_undegraded":"2024-08-17T17:22:52.373985+0000","last_fullsized":"2024-08-17T17:22:52.373985+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1133,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.379026+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:22:52.379026+0000","last_peered":"2024-08-17T17:22:52.379026+0000","last_clean":"2024-08-17T17:22:52.379026+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:22:52.379026+0000","last_undegraded":"2024-08-17T17:22:52.379026+0000","last_fullsized":"2024-08-17T17:22:52.379026+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1511,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.387044+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:22:52.387044+0000","last_peered":"2024-08-17T17:22:52.387044+0000","last_clean":"2024-08-17T17:22:52.387044+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:22:52.387044+0000","last_undegraded":"2024-08-17T17:22:52.387044+0000","last_fullsized":"2024-08-17T17:22:52.387044+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1486,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.388260+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:22:52.388260+0000","last_peered":"2024-08-17T17:22:52.388260+0000","last_clean":"2024-08-17T17:22:52.388260+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:22:52.388260+0000","last_undegraded":"2024-08-17T17:22:52.388260+0000","last_fullsized":"2024-08-17T17:22:52.388260+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1168,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.372612+0000","last_change":"2024-08-17T17:21:06.483469+0000","last_active":"2024-08-17T17:22:52.372612+0000","last_peered":"2024-08-17T17:22:52.372612+0000","last_clean":"2024-08-17T17:22:52.372612+0000","last_became_active":"2024-08-17T17:21:06.483162+0000","last_became_peered":"2024-08-17T17:21:06.483162+0000","last_unstale":"2024-08-17T17:22:52.372612+0000","last_undegraded":"2024-08-17T17:22:52.372612+0000","last_fullsized":"2024-08-17T17:22:52.372612+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:24.898353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":176,"reported_epoch":1205,"state":"active+clean","last_fresh":"2024-08-17T17:22:52.379133+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:22:52.379133+0000","last_peered":"2024-08-17T17:22:52.379133+0000","last_clean":"2024-08-17T17:22:52.379133+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:22:52.379133+0000","last_undegraded":"2024-08-17T17:22:52.379133+0000","last_fullsized":"2024-08-17T17:22:52.379133+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4027973632,"num_objects":61462,"num_object_clones":0,"num_object_copies":184386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":61462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61462,"num_write_kb":3933568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12025004032,"data_stored":12025004032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":61462,"ondisk_log_size":61462,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201658,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1615216,"kb_used_data":1237160,"kb_used_omap":1,"kb_used_meta":378046,"kb_avail":92129936,"statfs":{"total":95995035648,"available":94341054464,"internally_reserved":0,"allocated":1266851840,"data_stored":1264467652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":387119563},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463295,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1596216,"kb_used_data":1276396,"kb_used_omap":1,"kb_used_meta":319806,"kb_avail":92148936,"statfs":{"total":95995035648,"available":94360510464,"internally_reserved":0,"allocated":1307029504,"data_stored":1304649353,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":327481801},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626936,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1982108,"kb_used_data":1454356,"kb_used_omap":42,"kb_used_meta":527701,"kb_avail":91763044,"statfs":{"total":95995035648,"available":93965357056,"internally_reserved":0,"allocated":1489260544,"data_stored":1486892510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":540365884},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785413,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1934620,"kb_used_data":1573328,"kb_used_omap":48,"kb_used_meta":361231,"kb_avail":91810532,"statfs":{"total":95995035648,"available":94013984768,"internally_reserved":0,"allocated":1611087872,"data_stored":1608723319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":369901230},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724931,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2328436,"kb_used_data":1994664,"kb_used_omap":1,"kb_used_meta":333758,"kb_avail":91416716,"statfs":{"total":95995035648,"available":93610717184,"internally_reserved":0,"allocated":2042535936,"data_stored":2040156251,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":341768646},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986568,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1615992,"kb_used_data":1251888,"kb_used_omap":1,"kb_used_meta":364094,"kb_avail":92129160,"statfs":{"total":95995035648,"available":94340259840,"internally_reserved":0,"allocated":1281933312,"data_stored":1279549122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":372832710},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706175,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1871648,"kb_used_data":1474520,"kb_used_omap":35,"kb_used_meta":397084,"kb_avail":91873504,"statfs":{"total":95995035648,"available":94078468096,"internally_reserved":0,"allocated":1509908480,"data_stored":1507536876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":406614071},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831948,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1952668,"kb_used_data":1469012,"kb_used_omap":65,"kb_used_meta":483582,"kb_avail":91792484,"statfs":{"total":95995035648,"available":93995503616,"internally_reserved":0,"allocated":1504268288,"data_stored":1501900224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":495188155},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1517682688,"data_stored":1517682688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1631322112,"data_stored":1631322112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1506803712,"data_stored":1506803712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1464270848,"data_stored":1464270848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1280114688,"data_stored":1280114688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2057961472,"data_stored":2057961472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1281949696,"data_stored":1281949696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1284898816,"data_stored":1284898816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:23:01.810 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:23:01.899 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:01.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:01.968 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:01.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:02.069 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:02.104 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:02.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:02.206 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:02.242 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:02.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:02.344 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:02.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.381 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:02.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:02.482 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:02.500 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 35 16 69767 69751 124.529 88.3125 0.0116271 0.00802498 2024-08-17T17:23:02.516 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:02.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:02.523 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:02 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:02.523 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:02 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:02.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:02 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:02.618 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:02.658 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:02.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:02.760 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:02.797 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:02.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:02.899 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:02.926 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:02.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:02.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:02.976 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:03.028 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:03.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.075 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:03.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:03.176 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:03.213 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:03.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:03.235 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:03.314 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:03.343 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:03.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:03.445 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:03.445 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:03 smithi063 ceph-mon[114756]: pgmap v1540: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-08-17T17:23:03.446 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:03 smithi063 ceph-mon[109843]: pgmap v1540: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-08-17T17:23:03.446 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:23:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:23:03] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.51.0" 2024-08-17T17:23:03.473 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:03.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:03.500 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 36 16 71080 71064 123.349 82.0625 0.0112763 0.00810199 2024-08-17T17:23:03.575 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:03.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:03 smithi162 ceph-mon[104668]: pgmap v1540: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-08-17T17:23:03.623 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:03.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:03.653 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:23:03.654 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:23:03.688 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:23:03.688 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:23:03.723 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:23:03.723 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:23:03.725 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:03.781 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:23:03.781 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:23:03.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.783 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:03.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:03.819 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:23:03.819 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:23:03.858 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:23:03.858 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:23:03.884 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:03.892 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:23:03.892 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:23:03.911 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:03.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:03.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:03.927 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:23:03.927 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:23:04.013 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:04.038 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:04.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:04.140 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:04.172 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:04.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:04.274 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:04.316 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:04.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:04.418 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:04.453 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:04.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:04.501 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 37 16 72379 72363 122.209 81.1875 0.0110102 0.0081778 2024-08-17T17:23:04.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:04 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1085337968' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:04.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:04 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1085337968' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:04.558 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:04.598 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:04.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:04.700 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:04.729 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:04.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:04.831 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:04.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:04 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1085337968' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:04.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:04.866 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:04.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:04.967 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:05.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.003 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:05.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:05.104 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:05.133 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:05.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:05.235 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:05.270 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:05.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:05.372 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:05.407 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:05.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:05.502 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 38 16 73816 73800 121.357 89.8125 0.0125553 0.00823429 2024-08-17T17:23:05.509 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:05.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.564 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:05.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:05.665 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:05.696 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:05.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:05.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:05 smithi063 ceph-mon[114756]: pgmap v1541: 99 pgs: 99 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-08-17T17:23:05.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:05 smithi063 ceph-mon[109843]: pgmap v1541: 99 pgs: 99 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-08-17T17:23:05.798 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:05.835 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:05.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:05.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:05 smithi162 ceph-mon[104668]: pgmap v1541: 99 pgs: 99 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-08-17T17:23:05.852 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:05.937 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:05.981 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:05.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:05.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:06.084 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:06.117 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:06.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:06.219 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:06.252 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:06.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:06.354 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:06.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.385 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:06.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:06.486 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:06.501 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 39 16 75102 75086 120.306 80.375 0.0151053 0.008307 2024-08-17T17:23:06.517 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:06.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:06.619 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:06.652 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:06.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:06.753 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:06.785 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:06.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:06.887 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:06.913 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:06.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:06.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:07.014 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:07.045 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:07.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:07.147 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:07.190 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:07.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:07.292 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:07.329 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:07.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:07.431 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:07.462 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:07.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:07.501 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:23:07.501182+0000 min lat: 0.00109066 max lat: 0.153833 avg lat: 0.00838478 2024-08-17T17:23:07.501 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:23:07.501 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 40 16 76315 76299 119.193 75.8125 0.00994836 0.00838478 2024-08-17T17:23:07.564 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:07.601 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:07.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:07.703 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:07.725 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:07.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:07.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:07 smithi063 ceph-mon[109843]: pgmap v1542: 99 pgs: 99 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T17:23:07.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:07 smithi063 ceph-mon[114756]: pgmap v1542: 99 pgs: 99 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T17:23:07.826 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:07.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:07 smithi162 ceph-mon[104668]: pgmap v1542: 99 pgs: 99 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-08-17T17:23:07.862 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:07.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:07.964 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:07.988 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:07.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:07.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:08.090 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:08.134 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:08.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:08.237 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:08.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.281 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:08.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:08.382 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:08.412 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:08.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:08.501 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 41 16 77548 77532 118.166 77.0625 0.0160235 0.0084582 2024-08-17T17:23:08.514 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:08.545 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:08.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:08.647 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:08.685 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:08.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:08.787 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:08.831 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:08.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:08.933 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:08.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:08.971 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:08.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:09.072 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:09.098 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:09.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:09.200 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:09.233 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:09.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:09.334 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:09.384 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:09.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:09.388 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:09 smithi063 ceph-mon[109843]: pgmap v1543: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-08-17T17:23:09.388 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:09 smithi063 ceph-mon[114756]: pgmap v1543: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-08-17T17:23:09.486 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:09.501 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 42 16 78712 78696 117.084 72.75 0.010173 0.00853547 2024-08-17T17:23:09.518 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:09.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:09.620 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:09.672 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:09.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:09.768 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:23:09.768 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:23:09.775 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:09.825 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:09.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:09.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:09 smithi162 ceph-mon[104668]: pgmap v1543: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-08-17T17:23:09.927 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:09.969 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:09.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:09.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:10.071 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:10.100 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:10.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:10.202 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:10.233 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:10.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:10.335 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:10.372 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:10.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:10.385 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:23:10.388 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:10 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3957097123' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:23:10.388 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:10 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:23:10.388 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:10 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3957097123' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:23:10.388 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:10 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:23:10.465 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:23:10.474 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:10.501 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 43 16 79886 79870 116.068 73.375 0.00686571 0.00861054 2024-08-17T17:23:10.506 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:10.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:10.608 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:10.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.668 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:10.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:10.769 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:10.782 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:23:10.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.794 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:10.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:10.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:10 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3957097123' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:23:10.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:10 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:23:10.896 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:10.940 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:10.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:10.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:11.042 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:11.086 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:11.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:11.188 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:11.217 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:11.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:11.318 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:11.349 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:11.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:11.450 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:11.472 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:11.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:11.501 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 44 16 80927 80911 114.909 65.0625 0.0112959 0.00869785 2024-08-17T17:23:11.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:11 smithi063 ceph-mon[109843]: pgmap v1544: 99 pgs: 99 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-17T17:23:11.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:11 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:11.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:11 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:23:11.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:11 smithi063 ceph-mon[109843]: osdmap e1207: 8 total, 8 up, 8 in 2024-08-17T17:23:11.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:11 smithi063 ceph-mon[114756]: pgmap v1544: 99 pgs: 99 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-17T17:23:11.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:11 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:11.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:11 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:23:11.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:11 smithi063 ceph-mon[114756]: osdmap e1207: 8 total, 8 up, 8 in 2024-08-17T17:23:11.573 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:11.607 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:11.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:11.709 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:11.746 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:11.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:11 smithi162 ceph-mon[104668]: pgmap v1544: 99 pgs: 99 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-17T17:23:11.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:11 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:11.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:11 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:23:11.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:11 smithi162 ceph-mon[104668]: osdmap e1207: 8 total, 8 up, 8 in 2024-08-17T17:23:11.848 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:11.893 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:11.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:11.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:11.994 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:12.024 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:12.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:12.126 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:12.159 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:12.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:12.261 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:12.295 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:12.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:12.397 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:12.422 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:12.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:12.502 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 45 16 82088 82072 113.968 72.5625 0.00849695 0.00876784 2024-08-17T17:23:12.524 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:12.557 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:12.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:12.659 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:12.693 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:12.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:12.795 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:12.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.838 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:12.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:12.940 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:12.978 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:12.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:12.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:13.080 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:13.114 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:13.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:13.216 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:13.253 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:13.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:13.355 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:13.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.384 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:13.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:13.485 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:13.501 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 46 16 83304 83288 113.142 76 0.0127327 0.0088331 2024-08-17T17:23:13.516 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:13.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:13.618 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:13.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.658 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:13.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:13.758 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:13.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:23:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:23:13] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.51.0" 2024-08-17T17:23:13.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:13 smithi063 ceph-mon[114756]: pgmap v1546: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-17T17:23:13.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:13 smithi063 ceph-mon[109843]: pgmap v1546: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-17T17:23:13.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.793 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:13.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:13.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:13 smithi162 ceph-mon[104668]: pgmap v1546: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-08-17T17:23:13.894 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:13.936 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:13.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:13.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:14.038 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:14.074 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:14.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:14.178 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:14.208 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:14.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:14.310 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:14.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.361 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:14.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:14.462 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:14.502 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 47 16 84442 84426 112.245 71.125 0.0180745 0.00890354 2024-08-17T17:23:14.507 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:14.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:14.598 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:14 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:23:14.598 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:14 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:23:14.609 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:14.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.661 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:14.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:14.763 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:14.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.809 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:14.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:14.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:14 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:23:14.910 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:14.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:14.951 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:14.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:15.051 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:15.061 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:15.078 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:15.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:15.180 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:15.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.234 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:15.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:15.335 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:15.366 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:15.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:15.468 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:15.503 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:15.503 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 48 16 85625 85609 111.447 73.9375 0.00893671 0.00896753 2024-08-17T17:23:15.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:15.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:15 smithi063 ceph-mon[109843]: pgmap v1547: 99 pgs: 99 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-08-17T17:23:15.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:15 smithi063 ceph-mon[114756]: pgmap v1547: 99 pgs: 99 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-08-17T17:23:15.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:15 smithi162 ceph-mon[104668]: pgmap v1547: 99 pgs: 99 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-08-17T17:23:15.605 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:15.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.658 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:15.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:15.759 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:15.787 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:15.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:15.889 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:15.917 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:15.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:15.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:16.019 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:16.052 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:23:16.057 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:23:16.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.059 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:16.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:16.160 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:16.188 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:16.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:16.290 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:16.324 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:16.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:16.426 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:16.469 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:16.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:16.503 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 49 16 86744 86728 110.6 69.9375 0.00908167 0.00903578 2024-08-17T17:23:16.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:16.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:16.571 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:16.607 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:16.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:16.709 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:16.745 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:16.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:16.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:16.847 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:16.896 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:16.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:16.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:16.957 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:23:16.999 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:17.032 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:17.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:17.075 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:17.134 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:17.237 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:17.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:17.339 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:17.377 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:17.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:17.479 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:17.503 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 50 16 87914 87898 109.851 73.125 0.0170824 0.00909832 2024-08-17T17:23:17.515 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:17.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:17.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:17 smithi063 ceph-mon[109843]: from='client.45263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:23:17.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:17 smithi063 ceph-mon[109843]: pgmap v1548: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:23:17.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:17 smithi063 ceph-mon[114756]: from='client.45263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:23:17.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:17 smithi063 ceph-mon[114756]: pgmap v1548: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:23:17.617 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1547,"stamp":"2024-08-17T17:23:14.271304+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5305014245,"num_objects":80965,"num_object_clones":0,"num_object_copies":242895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":81003,"num_write_kb":5180129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1246,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82637,"ondisk_log_size":82637,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18792604,"kb_used_data":15629900,"kb_used_omap":199,"kb_used_meta":3162424,"kb_avail":731168612,"statfs":{"total":767960285184,"available":748716658688,"internally_reserved":0,"allocated":16005017600,"data_stored":15985968795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":3238322940},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":824705024,"num_objects":12584,"num_object_clones":0,"num_object_copies":37752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12584,"num_write_kb":805376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006067"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1507,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.424095+0000","last_change":"2024-08-17T17:21:06.463828+0000","last_active":"2024-08-17T17:22:56.424095+0000","last_peered":"2024-08-17T17:22:56.424095+0000","last_clean":"2024-08-17T17:22:56.424095+0000","last_became_active":"2024-08-17T17:21:06.463464+0000","last_became_peered":"2024-08-17T17:21:06.463464+0000","last_unstale":"2024-08-17T17:22:56.424095+0000","last_undegraded":"2024-08-17T17:22:56.424095+0000","last_fullsized":"2024-08-17T17:22:56.424095+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:46.465006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1135,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.420493+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:22:56.420493+0000","last_peered":"2024-08-17T17:22:56.420493+0000","last_clean":"2024-08-17T17:22:56.420493+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:22:56.420493+0000","last_undegraded":"2024-08-17T17:22:56.420493+0000","last_fullsized":"2024-08-17T17:22:56.420493+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1147,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415489+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:22:56.415489+0000","last_peered":"2024-08-17T17:22:56.415489+0000","last_clean":"2024-08-17T17:22:56.415489+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:22:56.415489+0000","last_undegraded":"2024-08-17T17:22:56.415489+0000","last_fullsized":"2024-08-17T17:22:56.415489+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1002,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.420230+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:22:56.420230+0000","last_peered":"2024-08-17T17:22:56.420230+0000","last_clean":"2024-08-17T17:22:56.420230+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:22:56.420230+0000","last_undegraded":"2024-08-17T17:22:56.420230+0000","last_fullsized":"2024-08-17T17:22:56.420230+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1059,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.423608+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:22:56.423608+0000","last_peered":"2024-08-17T17:22:56.423608+0000","last_clean":"2024-08-17T17:22:56.423608+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:22:56.423608+0000","last_undegraded":"2024-08-17T17:22:56.423608+0000","last_fullsized":"2024-08-17T17:22:56.423608+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1118,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.420113+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:22:56.420113+0000","last_peered":"2024-08-17T17:22:56.420113+0000","last_clean":"2024-08-17T17:22:56.420113+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:22:56.420113+0000","last_undegraded":"2024-08-17T17:22:56.420113+0000","last_fullsized":"2024-08-17T17:22:56.420113+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1507,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.425235+0000","last_change":"2024-08-17T17:21:06.442598+0000","last_active":"2024-08-17T17:22:56.425235+0000","last_peered":"2024-08-17T17:22:56.425235+0000","last_clean":"2024-08-17T17:22:56.425235+0000","last_became_active":"2024-08-17T17:21:06.442264+0000","last_became_peered":"2024-08-17T17:21:06.442264+0000","last_unstale":"2024-08-17T17:22:56.425235+0000","last_undegraded":"2024-08-17T17:22:56.425235+0000","last_fullsized":"2024-08-17T17:22:56.425235+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:35.003620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1195,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414239+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:22:56.414239+0000","last_peered":"2024-08-17T17:22:56.414239+0000","last_clean":"2024-08-17T17:22:56.414239+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:22:56.414239+0000","last_undegraded":"2024-08-17T17:22:56.414239+0000","last_fullsized":"2024-08-17T17:22:56.414239+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"1207'5032","reported_seq":5053,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:12.335682+0000","last_change":"2024-08-17T17:22:10.861776+0000","last_active":"2024-08-17T17:23:12.335682+0000","last_peered":"2024-08-17T17:23:12.335682+0000","last_clean":"2024-08-17T17:23:12.335682+0000","last_became_active":"2024-08-17T17:22:10.861509+0000","last_became_peered":"2024-08-17T17:22:10.861509+0000","last_unstale":"2024-08-17T17:23:12.335682+0000","last_undegraded":"2024-08-17T17:23:12.335682+0000","last_fullsized":"2024-08-17T17:23:12.335682+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5032,"log_dups_size":0,"ondisk_log_size":5032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:16:43.850909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329777152,"num_objects":5032,"num_object_clones":0,"num_object_copies":15096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5032,"num_write_kb":322048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1161,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.310806+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:22:57.310806+0000","last_peered":"2024-08-17T17:22:57.310806+0000","last_clean":"2024-08-17T17:22:57.310806+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:22:57.310806+0000","last_undegraded":"2024-08-17T17:22:57.310806+0000","last_fullsized":"2024-08-17T17:22:57.310806+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1133,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415219+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:22:56.415219+0000","last_peered":"2024-08-17T17:22:56.415219+0000","last_clean":"2024-08-17T17:22:56.415219+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:22:56.415219+0000","last_undegraded":"2024-08-17T17:22:56.415219+0000","last_fullsized":"2024-08-17T17:22:56.415219+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"1207'5131","reported_seq":5152,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:12.239250+0000","last_change":"2024-08-17T17:22:10.860210+0000","last_active":"2024-08-17T17:23:12.239250+0000","last_peered":"2024-08-17T17:23:12.239250+0000","last_clean":"2024-08-17T17:23:12.239250+0000","last_became_active":"2024-08-17T17:22:10.859854+0000","last_became_peered":"2024-08-17T17:22:10.859854+0000","last_unstale":"2024-08-17T17:23:12.239250+0000","last_undegraded":"2024-08-17T17:23:12.239250+0000","last_fullsized":"2024-08-17T17:23:12.239250+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5131,"log_dups_size":0,"ondisk_log_size":5131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:31.313667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336265216,"num_objects":5131,"num_object_clones":0,"num_object_copies":15393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5131,"num_write_kb":328384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1162,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.310784+0000","last_change":"2024-08-17T17:21:06.540689+0000","last_active":"2024-08-17T17:22:57.310784+0000","last_peered":"2024-08-17T17:22:57.310784+0000","last_clean":"2024-08-17T17:22:57.310784+0000","last_became_active":"2024-08-17T17:21:06.461614+0000","last_became_peered":"2024-08-17T17:21:06.461614+0000","last_unstale":"2024-08-17T17:22:57.310784+0000","last_undegraded":"2024-08-17T17:22:57.310784+0000","last_fullsized":"2024-08-17T17:22:57.310784+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:11.696898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1185,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417386+0000","last_change":"2024-08-17T17:21:06.470389+0000","last_active":"2024-08-17T17:22:56.417386+0000","last_peered":"2024-08-17T17:22:56.417386+0000","last_clean":"2024-08-17T17:22:56.417386+0000","last_became_active":"2024-08-17T17:21:06.469966+0000","last_became_peered":"2024-08-17T17:21:06.469966+0000","last_unstale":"2024-08-17T17:22:56.417386+0000","last_undegraded":"2024-08-17T17:22:56.417386+0000","last_fullsized":"2024-08-17T17:22:56.417386+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:56.140616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"1207'5055","reported_seq":5076,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:12.235734+0000","last_change":"2024-08-17T17:22:10.943999+0000","last_active":"2024-08-17T17:23:12.235734+0000","last_peered":"2024-08-17T17:23:12.235734+0000","last_clean":"2024-08-17T17:23:12.235734+0000","last_became_active":"2024-08-17T17:22:10.943661+0000","last_became_peered":"2024-08-17T17:22:10.943661+0000","last_unstale":"2024-08-17T17:23:12.235734+0000","last_undegraded":"2024-08-17T17:23:12.235734+0000","last_fullsized":"2024-08-17T17:23:12.235734+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5055,"log_dups_size":0,"ondisk_log_size":5055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:21:49.050541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331284480,"num_objects":5055,"num_object_clones":0,"num_object_copies":15165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5055,"num_write_kb":323520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1120,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421145+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:22:56.421145+0000","last_peered":"2024-08-17T17:22:56.421145+0000","last_clean":"2024-08-17T17:22:56.421145+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:22:56.421145+0000","last_undegraded":"2024-08-17T17:22:56.421145+0000","last_fullsized":"2024-08-17T17:22:56.421145+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1167,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414538+0000","last_change":"2024-08-17T17:21:06.446798+0000","last_active":"2024-08-17T17:22:56.414538+0000","last_peered":"2024-08-17T17:22:56.414538+0000","last_clean":"2024-08-17T17:22:56.414538+0000","last_became_active":"2024-08-17T17:21:06.446205+0000","last_became_peered":"2024-08-17T17:21:06.446205+0000","last_unstale":"2024-08-17T17:22:56.414538+0000","last_undegraded":"2024-08-17T17:22:56.414538+0000","last_fullsized":"2024-08-17T17:22:56.414538+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:28.584048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"1206'4917","reported_seq":4950,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.123657+0000","last_change":"2024-08-17T17:22:10.900862+0000","last_active":"2024-08-17T17:23:10.123657+0000","last_peered":"2024-08-17T17:23:10.123657+0000","last_clean":"2024-08-17T17:23:10.123657+0000","last_became_active":"2024-08-17T17:22:10.900552+0000","last_became_peered":"2024-08-17T17:22:10.900552+0000","last_unstale":"2024-08-17T17:23:10.123657+0000","last_undegraded":"2024-08-17T17:23:10.123657+0000","last_fullsized":"2024-08-17T17:23:10.123657+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":4917,"log_dups_size":0,"ondisk_log_size":4917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:07.195144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322240512,"num_objects":4917,"num_object_clones":0,"num_object_copies":14751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4917,"num_write_kb":314688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1149,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414971+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:22:56.414971+0000","last_peered":"2024-08-17T17:22:56.414971+0000","last_clean":"2024-08-17T17:22:56.414971+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:22:56.414971+0000","last_undegraded":"2024-08-17T17:22:56.414971+0000","last_fullsized":"2024-08-17T17:22:56.414971+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":931,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.392418+0000","last_change":"2024-08-17T17:21:06.460905+0000","last_active":"2024-08-17T17:23:10.392418+0000","last_peered":"2024-08-17T17:23:10.392418+0000","last_clean":"2024-08-17T17:23:10.392418+0000","last_became_active":"2024-08-17T17:21:06.460592+0000","last_became_peered":"2024-08-17T17:21:06.460592+0000","last_unstale":"2024-08-17T17:23:10.392418+0000","last_undegraded":"2024-08-17T17:23:10.392418+0000","last_fullsized":"2024-08-17T17:23:10.392418+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:10.146916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.0","version":"1206'4883","reported_seq":4916,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.124640+0000","last_change":"2024-08-17T17:22:10.900154+0000","last_active":"2024-08-17T17:23:10.124640+0000","last_peered":"2024-08-17T17:23:10.124640+0000","last_clean":"2024-08-17T17:23:10.124640+0000","last_became_active":"2024-08-17T17:22:10.899766+0000","last_became_peered":"2024-08-17T17:22:10.899766+0000","last_unstale":"2024-08-17T17:23:10.124640+0000","last_undegraded":"2024-08-17T17:23:10.124640+0000","last_fullsized":"2024-08-17T17:23:10.124640+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":4883,"log_dups_size":0,"ondisk_log_size":4883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:07.705078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320012288,"num_objects":4883,"num_object_clones":0,"num_object_copies":14649,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4883,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4883,"num_write_kb":312512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1528,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.419942+0000","last_change":"2024-08-17T17:21:16.100398+0000","last_active":"2024-08-17T17:22:56.419942+0000","last_peered":"2024-08-17T17:22:56.419942+0000","last_clean":"2024-08-17T17:22:56.419942+0000","last_became_active":"2024-08-17T17:18:28.521573+0000","last_became_peered":"2024-08-17T17:18:28.521573+0000","last_unstale":"2024-08-17T17:22:56.419942+0000","last_undegraded":"2024-08-17T17:22:56.419942+0000","last_fullsized":"2024-08-17T17:22:56.419942+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:21:16.100258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:21:16.100258+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:57.628353+0000","scrub_duration":0.011045358999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1528,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.422531+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:22:56.422531+0000","last_peered":"2024-08-17T17:22:56.422531+0000","last_clean":"2024-08-17T17:22:56.422531+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:22:56.422531+0000","last_undegraded":"2024-08-17T17:22:56.422531+0000","last_fullsized":"2024-08-17T17:22:56.422531+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1152,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.249708+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:22:57.249708+0000","last_peered":"2024-08-17T17:22:57.249708+0000","last_clean":"2024-08-17T17:22:57.249708+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:22:57.249708+0000","last_undegraded":"2024-08-17T17:22:57.249708+0000","last_fullsized":"2024-08-17T17:22:57.249708+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"1207'4970","reported_seq":5004,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:11.551598+0000","last_change":"2024-08-17T17:22:10.944783+0000","last_active":"2024-08-17T17:23:11.551598+0000","last_peered":"2024-08-17T17:23:11.551598+0000","last_clean":"2024-08-17T17:23:11.551598+0000","last_became_active":"2024-08-17T17:22:10.944487+0000","last_became_peered":"2024-08-17T17:22:10.944487+0000","last_unstale":"2024-08-17T17:23:11.551598+0000","last_undegraded":"2024-08-17T17:23:11.551598+0000","last_fullsized":"2024-08-17T17:23:11.551598+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":4970,"log_dups_size":0,"ondisk_log_size":4970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.515134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325713920,"num_objects":4970,"num_object_clones":0,"num_object_copies":14910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4970,"num_write_kb":318080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1474,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414963+0000","last_change":"2024-08-17T17:18:32.884334+0000","last_active":"2024-08-17T17:22:56.414963+0000","last_peered":"2024-08-17T17:22:56.414963+0000","last_clean":"2024-08-17T17:22:56.414963+0000","last_became_active":"2024-08-17T17:18:28.520186+0000","last_became_peered":"2024-08-17T17:18:28.520186+0000","last_unstale":"2024-08-17T17:22:56.414963+0000","last_undegraded":"2024-08-17T17:22:56.414963+0000","last_fullsized":"2024-08-17T17:22:56.414963+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:32.884246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:32.884246+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:49.286656+0000","scrub_duration":0.001842677,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1168,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414687+0000","last_change":"2024-08-17T17:21:06.446841+0000","last_active":"2024-08-17T17:22:56.414687+0000","last_peered":"2024-08-17T17:22:56.414687+0000","last_clean":"2024-08-17T17:22:56.414687+0000","last_became_active":"2024-08-17T17:21:06.446271+0000","last_became_peered":"2024-08-17T17:21:06.446271+0000","last_unstale":"2024-08-17T17:22:56.414687+0000","last_undegraded":"2024-08-17T17:22:56.414687+0000","last_fullsized":"2024-08-17T17:22:56.414687+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:20.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1185,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.249739+0000","last_change":"2024-08-17T17:21:06.480911+0000","last_active":"2024-08-17T17:22:57.249739+0000","last_peered":"2024-08-17T17:22:57.249739+0000","last_clean":"2024-08-17T17:22:57.249739+0000","last_became_active":"2024-08-17T17:21:06.480551+0000","last_became_peered":"2024-08-17T17:21:06.480551+0000","last_unstale":"2024-08-17T17:22:57.249739+0000","last_undegraded":"2024-08-17T17:22:57.249739+0000","last_fullsized":"2024-08-17T17:22:57.249739+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:26.003704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"1207'5237","reported_seq":5258,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:12.229421+0000","last_change":"2024-08-17T17:22:10.944020+0000","last_active":"2024-08-17T17:23:12.229421+0000","last_peered":"2024-08-17T17:23:12.229421+0000","last_clean":"2024-08-17T17:23:12.229421+0000","last_became_active":"2024-08-17T17:22:10.943694+0000","last_became_peered":"2024-08-17T17:22:10.943694+0000","last_unstale":"2024-08-17T17:23:12.229421+0000","last_undegraded":"2024-08-17T17:23:12.229421+0000","last_fullsized":"2024-08-17T17:23:12.229421+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5237,"log_dups_size":0,"ondisk_log_size":5237,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:34.525383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343212032,"num_objects":5237,"num_object_clones":0,"num_object_copies":15711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5237,"num_write_kb":335168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1153,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417473+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:22:56.417473+0000","last_peered":"2024-08-17T17:22:56.417473+0000","last_clean":"2024-08-17T17:22:56.417473+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:22:56.417473+0000","last_undegraded":"2024-08-17T17:22:56.417473+0000","last_fullsized":"2024-08-17T17:22:56.417473+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1170,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:23:02.252360+0000","last_change":"2024-08-17T17:21:06.451521+0000","last_active":"2024-08-17T17:23:02.252360+0000","last_peered":"2024-08-17T17:23:02.252360+0000","last_clean":"2024-08-17T17:23:02.252360+0000","last_became_active":"2024-08-17T17:21:06.451197+0000","last_became_peered":"2024-08-17T17:21:06.451197+0000","last_unstale":"2024-08-17T17:23:02.252360+0000","last_undegraded":"2024-08-17T17:23:02.252360+0000","last_fullsized":"2024-08-17T17:23:02.252360+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:04.000692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"1207'5203","reported_seq":5224,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:12.315805+0000","last_change":"2024-08-17T17:22:10.900282+0000","last_active":"2024-08-17T17:23:12.315805+0000","last_peered":"2024-08-17T17:23:12.315805+0000","last_clean":"2024-08-17T17:23:12.315805+0000","last_became_active":"2024-08-17T17:22:10.899947+0000","last_became_peered":"2024-08-17T17:22:10.899947+0000","last_unstale":"2024-08-17T17:23:12.315805+0000","last_undegraded":"2024-08-17T17:23:12.315805+0000","last_fullsized":"2024-08-17T17:23:12.315805+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5203,"log_dups_size":0,"ondisk_log_size":5203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:10.148524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340983808,"num_objects":5203,"num_object_clones":0,"num_object_copies":15609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5203,"num_write_kb":332992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1474,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415316+0000","last_change":"2024-08-17T17:18:31.850829+0000","last_active":"2024-08-17T17:22:56.415316+0000","last_peered":"2024-08-17T17:22:56.415316+0000","last_clean":"2024-08-17T17:22:56.415316+0000","last_became_active":"2024-08-17T17:18:28.520946+0000","last_became_peered":"2024-08-17T17:18:28.520946+0000","last_unstale":"2024-08-17T17:22:56.415316+0000","last_undegraded":"2024-08-17T17:22:56.415316+0000","last_fullsized":"2024-08-17T17:22:56.415316+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:31.850745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:31.850745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:17.218084+0000","scrub_duration":0.001317447,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1510,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.426270+0000","last_change":"2024-08-17T17:21:06.441118+0000","last_active":"2024-08-17T17:22:56.426270+0000","last_peered":"2024-08-17T17:22:56.426270+0000","last_clean":"2024-08-17T17:22:56.426270+0000","last_became_active":"2024-08-17T17:21:06.440800+0000","last_became_peered":"2024-08-17T17:21:06.440800+0000","last_unstale":"2024-08-17T17:22:56.426270+0000","last_undegraded":"2024-08-17T17:22:56.426270+0000","last_fullsized":"2024-08-17T17:22:56.426270+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:08.091528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":18,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.396377+0000","last_change":"2024-08-17T17:22:48.374766+0000","last_active":"2024-08-17T17:23:10.396377+0000","last_peered":"2024-08-17T17:23:10.396377+0000","last_clean":"2024-08-17T17:23:10.396377+0000","last_became_active":"2024-08-17T17:22:48.373556+0000","last_became_peered":"2024-08-17T17:22:48.373556+0000","last_unstale":"2024-08-17T17:23:10.396377+0000","last_undegraded":"2024-08-17T17:23:10.396377+0000","last_fullsized":"2024-08-17T17:23:10.396377+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:33.362080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.9","version":"1207'5117","reported_seq":5138,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:12.210582+0000","last_change":"2024-08-17T17:22:10.944057+0000","last_active":"2024-08-17T17:23:12.210582+0000","last_peered":"2024-08-17T17:23:12.210582+0000","last_clean":"2024-08-17T17:23:12.210582+0000","last_became_active":"2024-08-17T17:22:10.943681+0000","last_became_peered":"2024-08-17T17:22:10.943681+0000","last_unstale":"2024-08-17T17:23:12.210582+0000","last_undegraded":"2024-08-17T17:23:12.210582+0000","last_fullsized":"2024-08-17T17:23:12.210582+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5117,"log_dups_size":0,"ondisk_log_size":5117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:24.524487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335347712,"num_objects":5117,"num_object_clones":0,"num_object_copies":15351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5117,"num_write_kb":327488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1528,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421612+0000","last_change":"2024-08-17T17:18:33.814375+0000","last_active":"2024-08-17T17:22:56.421612+0000","last_peered":"2024-08-17T17:22:56.421612+0000","last_clean":"2024-08-17T17:22:56.421612+0000","last_became_active":"2024-08-17T17:18:33.813208+0000","last_became_peered":"2024-08-17T17:18:33.813208+0000","last_unstale":"2024-08-17T17:22:56.421612+0000","last_undegraded":"2024-08-17T17:22:56.421612+0000","last_fullsized":"2024-08-17T17:22:56.421612+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:25.057675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:25.057675+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:20:45.235879+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1163,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414765+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:22:56.414765+0000","last_peered":"2024-08-17T17:22:56.414765+0000","last_clean":"2024-08-17T17:22:56.414765+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:22:56.414765+0000","last_undegraded":"2024-08-17T17:22:56.414765+0000","last_fullsized":"2024-08-17T17:22:56.414765+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1475,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.420983+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:22:56.420983+0000","last_peered":"2024-08-17T17:22:56.420983+0000","last_clean":"2024-08-17T17:22:56.420983+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:22:56.420983+0000","last_undegraded":"2024-08-17T17:22:56.420983+0000","last_fullsized":"2024-08-17T17:22:56.420983+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1135,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.218577+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:22:57.218577+0000","last_peered":"2024-08-17T17:22:57.218577+0000","last_clean":"2024-08-17T17:22:57.218577+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:22:57.218577+0000","last_undegraded":"2024-08-17T17:22:57.218577+0000","last_fullsized":"2024-08-17T17:22:57.218577+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"1207'5107","reported_seq":5142,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:11.520426+0000","last_change":"2024-08-17T17:22:10.900900+0000","last_active":"2024-08-17T17:23:11.520426+0000","last_peered":"2024-08-17T17:23:11.520426+0000","last_clean":"2024-08-17T17:23:11.520426+0000","last_became_active":"2024-08-17T17:22:10.899812+0000","last_became_peered":"2024-08-17T17:22:10.899812+0000","last_unstale":"2024-08-17T17:23:11.520426+0000","last_undegraded":"2024-08-17T17:23:11.520426+0000","last_fullsized":"2024-08-17T17:23:11.520426+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5107,"log_dups_size":0,"ondisk_log_size":5107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:25:56.258127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334692352,"num_objects":5107,"num_object_clones":0,"num_object_copies":15321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5107,"num_write_kb":326848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":767,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414472+0000","last_change":"2024-08-17T17:22:09.230320+0000","last_active":"2024-08-17T17:22:56.414472+0000","last_peered":"2024-08-17T17:22:56.414472+0000","last_clean":"2024-08-17T17:22:56.414472+0000","last_became_active":"2024-08-17T17:21:08.910463+0000","last_became_peered":"2024-08-17T17:21:08.910463+0000","last_unstale":"2024-08-17T17:22:56.414472+0000","last_undegraded":"2024-08-17T17:22:56.414472+0000","last_fullsized":"2024-08-17T17:22:56.414472+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:09.230214+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.230214+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:58.582858+0000","scrub_duration":0.001729597,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1199,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421543+0000","last_change":"2024-08-17T17:21:06.430639+0000","last_active":"2024-08-17T17:22:56.421543+0000","last_peered":"2024-08-17T17:22:56.421543+0000","last_clean":"2024-08-17T17:22:56.421543+0000","last_became_active":"2024-08-17T17:21:06.430297+0000","last_became_peered":"2024-08-17T17:21:06.430297+0000","last_unstale":"2024-08-17T17:22:56.421543+0000","last_undegraded":"2024-08-17T17:22:56.421543+0000","last_fullsized":"2024-08-17T17:22:56.421543+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:57.794620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1002,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417581+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:22:56.417581+0000","last_peered":"2024-08-17T17:22:56.417581+0000","last_clean":"2024-08-17T17:22:56.417581+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:22:56.417581+0000","last_undegraded":"2024-08-17T17:22:56.417581+0000","last_fullsized":"2024-08-17T17:22:56.417581+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414874+0000","last_change":"2024-08-17T17:22:48.370769+0000","last_active":"2024-08-17T17:22:56.414874+0000","last_peered":"2024-08-17T17:22:56.414874+0000","last_clean":"2024-08-17T17:22:56.414874+0000","last_became_active":"2024-08-17T17:22:48.370531+0000","last_became_peered":"2024-08-17T17:22:48.370531+0000","last_unstale":"2024-08-17T17:22:56.414874+0000","last_undegraded":"2024-08-17T17:22:56.414874+0000","last_fullsized":"2024-08-17T17:22:56.414874+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:02.943823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.b","version":"1206'5011","reported_seq":5044,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.110588+0000","last_change":"2024-08-17T17:22:10.900740+0000","last_active":"2024-08-17T17:23:10.110588+0000","last_peered":"2024-08-17T17:23:10.110588+0000","last_clean":"2024-08-17T17:23:10.110588+0000","last_became_active":"2024-08-17T17:22:10.900452+0000","last_became_peered":"2024-08-17T17:22:10.900452+0000","last_unstale":"2024-08-17T17:23:10.110588+0000","last_undegraded":"2024-08-17T17:23:10.110588+0000","last_fullsized":"2024-08-17T17:23:10.110588+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5011,"log_dups_size":0,"ondisk_log_size":5011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:38:40.285223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328400896,"num_objects":5011,"num_object_clones":0,"num_object_copies":15033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5011,"num_write_kb":320704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1549,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.420147+0000","last_change":"2024-08-17T17:22:07.078517+0000","last_active":"2024-08-17T17:22:56.420147+0000","last_peered":"2024-08-17T17:22:56.420147+0000","last_clean":"2024-08-17T17:22:56.420147+0000","last_became_active":"2024-08-17T17:18:33.816660+0000","last_became_peered":"2024-08-17T17:18:33.816660+0000","last_unstale":"2024-08-17T17:22:56.420147+0000","last_undegraded":"2024-08-17T17:22:56.420147+0000","last_fullsized":"2024-08-17T17:22:56.420147+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:07.078434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:07.078434+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:56.205556+0000","scrub_duration":0.0013001919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1574,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.396569+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:23:10.396569+0000","last_peered":"2024-08-17T17:23:10.396569+0000","last_clean":"2024-08-17T17:23:10.396569+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:23:10.396569+0000","last_undegraded":"2024-08-17T17:23:10.396569+0000","last_fullsized":"2024-08-17T17:23:10.396569+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":966,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.396064+0000","last_change":"2024-08-17T17:21:06.428684+0000","last_active":"2024-08-17T17:23:10.396064+0000","last_peered":"2024-08-17T17:23:10.396064+0000","last_clean":"2024-08-17T17:23:10.396064+0000","last_became_active":"2024-08-17T17:21:06.428178+0000","last_became_peered":"2024-08-17T17:21:06.428178+0000","last_unstale":"2024-08-17T17:23:10.396064+0000","last_undegraded":"2024-08-17T17:23:10.396064+0000","last_fullsized":"2024-08-17T17:23:10.396064+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:14.836119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":846,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415453+0000","last_change":"2024-08-17T17:21:06.443565+0000","last_active":"2024-08-17T17:22:56.415453+0000","last_peered":"2024-08-17T17:22:56.415453+0000","last_clean":"2024-08-17T17:22:56.415453+0000","last_became_active":"2024-08-17T17:21:06.443215+0000","last_became_peered":"2024-08-17T17:21:06.443215+0000","last_unstale":"2024-08-17T17:22:56.415453+0000","last_undegraded":"2024-08-17T17:22:56.415453+0000","last_fullsized":"2024-08-17T17:22:56.415453+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:47.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"1207'5118","reported_seq":5139,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:12.322900+0000","last_change":"2024-08-17T17:22:10.859676+0000","last_active":"2024-08-17T17:23:12.322900+0000","last_peered":"2024-08-17T17:23:12.322900+0000","last_clean":"2024-08-17T17:23:12.322900+0000","last_became_active":"2024-08-17T17:22:10.859289+0000","last_became_peered":"2024-08-17T17:22:10.859289+0000","last_unstale":"2024-08-17T17:23:12.322900+0000","last_undegraded":"2024-08-17T17:23:12.322900+0000","last_fullsized":"2024-08-17T17:23:12.322900+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5118,"log_dups_size":0,"ondisk_log_size":5118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:43.302993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335413248,"num_objects":5118,"num_object_clones":0,"num_object_copies":15354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5118,"num_write_kb":327552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1491,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415178+0000","last_change":"2024-08-17T17:18:35.916482+0000","last_active":"2024-08-17T17:22:56.415178+0000","last_peered":"2024-08-17T17:22:56.415178+0000","last_clean":"2024-08-17T17:22:56.415178+0000","last_became_active":"2024-08-17T17:18:34.827747+0000","last_became_peered":"2024-08-17T17:18:34.827747+0000","last_unstale":"2024-08-17T17:22:56.415178+0000","last_undegraded":"2024-08-17T17:22:56.415178+0000","last_fullsized":"2024-08-17T17:22:56.415178+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:35.916381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:35.916381+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:28.492622+0000","scrub_duration":0.005340521,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1190,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.416332+0000","last_change":"2024-08-17T17:21:06.456978+0000","last_active":"2024-08-17T17:22:56.416332+0000","last_peered":"2024-08-17T17:22:56.416332+0000","last_clean":"2024-08-17T17:22:56.416332+0000","last_became_active":"2024-08-17T17:21:06.452265+0000","last_became_peered":"2024-08-17T17:21:06.452265+0000","last_unstale":"2024-08-17T17:22:56.416332+0000","last_undegraded":"2024-08-17T17:22:56.416332+0000","last_fullsized":"2024-08-17T17:22:56.416332+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:52.893746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1172,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417922+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:22:56.417922+0000","last_peered":"2024-08-17T17:22:56.417922+0000","last_clean":"2024-08-17T17:22:56.417922+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:22:56.417922+0000","last_undegraded":"2024-08-17T17:22:56.417922+0000","last_fullsized":"2024-08-17T17:22:56.417922+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1042,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.396805+0000","last_change":"2024-08-17T17:21:06.435754+0000","last_active":"2024-08-17T17:23:10.396805+0000","last_peered":"2024-08-17T17:23:10.396805+0000","last_clean":"2024-08-17T17:23:10.396805+0000","last_became_active":"2024-08-17T17:21:06.435442+0000","last_became_peered":"2024-08-17T17:21:06.435442+0000","last_unstale":"2024-08-17T17:23:10.396805+0000","last_undegraded":"2024-08-17T17:23:10.396805+0000","last_fullsized":"2024-08-17T17:23:10.396805+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:34.849042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"1207'5033","reported_seq":5068,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:11.549461+0000","last_change":"2024-08-17T17:22:10.944986+0000","last_active":"2024-08-17T17:23:11.549461+0000","last_peered":"2024-08-17T17:23:11.549461+0000","last_clean":"2024-08-17T17:23:11.549461+0000","last_became_active":"2024-08-17T17:22:10.944267+0000","last_became_peered":"2024-08-17T17:22:10.944267+0000","last_unstale":"2024-08-17T17:23:11.549461+0000","last_undegraded":"2024-08-17T17:23:11.549461+0000","last_fullsized":"2024-08-17T17:23:11.549461+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5033,"log_dups_size":0,"ondisk_log_size":5033,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:52.209246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329842688,"num_objects":5033,"num_object_clones":0,"num_object_copies":15099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5033,"num_write_kb":322112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":733,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415788+0000","last_change":"2024-08-17T17:22:06.152683+0000","last_active":"2024-08-17T17:22:56.415788+0000","last_peered":"2024-08-17T17:22:56.415788+0000","last_clean":"2024-08-17T17:22:56.415788+0000","last_became_active":"2024-08-17T17:18:28.526439+0000","last_became_peered":"2024-08-17T17:18:28.526439+0000","last_unstale":"2024-08-17T17:22:56.415788+0000","last_undegraded":"2024-08-17T17:22:56.415788+0000","last_fullsized":"2024-08-17T17:22:56.415788+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:06.152592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:06.152592+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:30.807010+0000","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1165,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415646+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:22:56.415646+0000","last_peered":"2024-08-17T17:22:56.415646+0000","last_clean":"2024-08-17T17:22:56.415646+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:22:56.415646+0000","last_undegraded":"2024-08-17T17:22:56.415646+0000","last_fullsized":"2024-08-17T17:22:56.415646+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":855,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.420776+0000","last_change":"2024-08-17T17:21:06.434991+0000","last_active":"2024-08-17T17:22:56.420776+0000","last_peered":"2024-08-17T17:22:56.420776+0000","last_clean":"2024-08-17T17:22:56.420776+0000","last_became_active":"2024-08-17T17:21:06.434630+0000","last_became_peered":"2024-08-17T17:21:06.434630+0000","last_unstale":"2024-08-17T17:22:56.420776+0000","last_undegraded":"2024-08-17T17:22:56.420776+0000","last_fullsized":"2024-08-17T17:22:56.420776+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:24.104609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":797,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.416080+0000","last_change":"2024-08-17T17:21:06.462532+0000","last_active":"2024-08-17T17:22:56.416080+0000","last_peered":"2024-08-17T17:22:56.416080+0000","last_clean":"2024-08-17T17:22:56.416080+0000","last_became_active":"2024-08-17T17:21:06.462183+0000","last_became_peered":"2024-08-17T17:21:06.462183+0000","last_unstale":"2024-08-17T17:22:56.416080+0000","last_undegraded":"2024-08-17T17:22:56.416080+0000","last_fullsized":"2024-08-17T17:22:56.416080+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:50.033322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.f","version":"1206'4982","reported_seq":5014,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.107915+0000","last_change":"2024-08-17T17:22:10.900592+0000","last_active":"2024-08-17T17:23:10.107915+0000","last_peered":"2024-08-17T17:23:10.107915+0000","last_clean":"2024-08-17T17:23:10.107915+0000","last_became_active":"2024-08-17T17:22:10.900279+0000","last_became_peered":"2024-08-17T17:22:10.900279+0000","last_unstale":"2024-08-17T17:23:10.107915+0000","last_undegraded":"2024-08-17T17:23:10.107915+0000","last_fullsized":"2024-08-17T17:23:10.107915+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":4982,"log_dups_size":0,"ondisk_log_size":4982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:52:25.130107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326500352,"num_objects":4982,"num_object_clones":0,"num_object_copies":14946,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4982,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4982,"num_write_kb":318848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":750,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415656+0000","last_change":"2024-08-17T17:22:08.218836+0000","last_active":"2024-08-17T17:22:56.415656+0000","last_peered":"2024-08-17T17:22:56.415656+0000","last_clean":"2024-08-17T17:22:56.415656+0000","last_became_active":"2024-08-17T17:18:34.831498+0000","last_became_peered":"2024-08-17T17:18:34.831498+0000","last_unstale":"2024-08-17T17:22:56.415656+0000","last_undegraded":"2024-08-17T17:22:56.415656+0000","last_fullsized":"2024-08-17T17:22:56.415656+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:08.218709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:08.218709+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:19.097715+0000","scrub_duration":0.002239024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":965,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.396705+0000","last_change":"2024-08-17T17:21:06.433720+0000","last_active":"2024-08-17T17:23:10.396705+0000","last_peered":"2024-08-17T17:23:10.396705+0000","last_clean":"2024-08-17T17:23:10.396705+0000","last_became_active":"2024-08-17T17:21:06.433292+0000","last_became_peered":"2024-08-17T17:21:06.433292+0000","last_unstale":"2024-08-17T17:23:10.396705+0000","last_undegraded":"2024-08-17T17:23:10.396705+0000","last_fullsized":"2024-08-17T17:23:10.396705+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:32.246594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1165,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421209+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:22:56.421209+0000","last_peered":"2024-08-17T17:22:56.421209+0000","last_clean":"2024-08-17T17:22:56.421209+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:22:56.421209+0000","last_undegraded":"2024-08-17T17:22:56.421209+0000","last_fullsized":"2024-08-17T17:22:56.421209+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":757,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414683+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:22:56.414683+0000","last_peered":"2024-08-17T17:22:56.414683+0000","last_clean":"2024-08-17T17:22:56.414683+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:22:56.414683+0000","last_undegraded":"2024-08-17T17:22:56.414683+0000","last_fullsized":"2024-08-17T17:22:56.414683+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"1206'5023","reported_seq":5056,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.099960+0000","last_change":"2024-08-17T17:22:10.944050+0000","last_active":"2024-08-17T17:23:10.099960+0000","last_peered":"2024-08-17T17:23:10.099960+0000","last_clean":"2024-08-17T17:23:10.099960+0000","last_became_active":"2024-08-17T17:22:10.943712+0000","last_became_peered":"2024-08-17T17:22:10.943712+0000","last_unstale":"2024-08-17T17:23:10.099960+0000","last_undegraded":"2024-08-17T17:23:10.099960+0000","last_fullsized":"2024-08-17T17:23:10.099960+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5023,"log_dups_size":0,"ondisk_log_size":5023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:44.623031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329187328,"num_objects":5023,"num_object_clones":0,"num_object_copies":15069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5023,"num_write_kb":321472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1474,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417831+0000","last_change":"2024-08-17T17:22:06.995885+0000","last_active":"2024-08-17T17:22:56.417831+0000","last_peered":"2024-08-17T17:22:56.417831+0000","last_clean":"2024-08-17T17:22:56.417831+0000","last_became_active":"2024-08-17T17:18:28.526484+0000","last_became_peered":"2024-08-17T17:18:28.526484+0000","last_unstale":"2024-08-17T17:22:56.417831+0000","last_undegraded":"2024-08-17T17:22:56.417831+0000","last_fullsized":"2024-08-17T17:22:56.417831+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:06.995792+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:06.995792+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:06.341398+0000","scrub_duration":0.0016139520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1529,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.425469+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:22:56.425469+0000","last_peered":"2024-08-17T17:22:56.425469+0000","last_clean":"2024-08-17T17:22:56.425469+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:22:56.425469+0000","last_undegraded":"2024-08-17T17:22:56.425469+0000","last_fullsized":"2024-08-17T17:22:56.425469+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":807,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415837+0000","last_change":"2024-08-17T17:21:06.452395+0000","last_active":"2024-08-17T17:22:56.415837+0000","last_peered":"2024-08-17T17:22:56.415837+0000","last_clean":"2024-08-17T17:22:56.415837+0000","last_became_active":"2024-08-17T17:21:06.451867+0000","last_became_peered":"2024-08-17T17:21:06.451867+0000","last_unstale":"2024-08-17T17:22:56.415837+0000","last_undegraded":"2024-08-17T17:22:56.415837+0000","last_fullsized":"2024-08-17T17:22:56.415837+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:50.597092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1518,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.426102+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:22:56.426102+0000","last_peered":"2024-08-17T17:22:56.426102+0000","last_clean":"2024-08-17T17:22:56.426102+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:22:56.426102+0000","last_undegraded":"2024-08-17T17:22:56.426102+0000","last_fullsized":"2024-08-17T17:22:56.426102+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"1207'5120","reported_seq":5141,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:12.315406+0000","last_change":"2024-08-17T17:22:10.859993+0000","last_active":"2024-08-17T17:23:12.315406+0000","last_peered":"2024-08-17T17:23:12.315406+0000","last_clean":"2024-08-17T17:23:12.315406+0000","last_became_active":"2024-08-17T17:22:10.859296+0000","last_became_peered":"2024-08-17T17:22:10.859296+0000","last_unstale":"2024-08-17T17:23:12.315406+0000","last_undegraded":"2024-08-17T17:23:12.315406+0000","last_fullsized":"2024-08-17T17:23:12.315406+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5120,"log_dups_size":0,"ondisk_log_size":5120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:42.095858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335544320,"num_objects":5120,"num_object_clones":0,"num_object_copies":15360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5120,"num_write_kb":327680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1534,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.423909+0000","last_change":"2024-08-17T17:18:29.390118+0000","last_active":"2024-08-17T17:22:56.423909+0000","last_peered":"2024-08-17T17:22:56.423909+0000","last_clean":"2024-08-17T17:22:56.423909+0000","last_became_active":"2024-08-17T17:18:28.528874+0000","last_became_peered":"2024-08-17T17:18:28.528874+0000","last_unstale":"2024-08-17T17:22:56.423909+0000","last_undegraded":"2024-08-17T17:22:56.423909+0000","last_fullsized":"2024-08-17T17:22:56.423909+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:29.390003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:29.390003+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:35.908604+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":176,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417576+0000","last_change":"2024-08-17T17:21:06.519965+0000","last_active":"2024-08-17T17:22:56.417576+0000","last_peered":"2024-08-17T17:22:56.417576+0000","last_clean":"2024-08-17T17:22:56.417576+0000","last_became_active":"2024-08-17T17:21:06.459952+0000","last_became_peered":"2024-08-17T17:21:06.459952+0000","last_unstale":"2024-08-17T17:22:56.417576+0000","last_undegraded":"2024-08-17T17:22:56.417576+0000","last_fullsized":"2024-08-17T17:22:56.417576+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:27.828098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1162,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415920+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:22:56.415920+0000","last_peered":"2024-08-17T17:22:56.415920+0000","last_clean":"2024-08-17T17:22:56.415920+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:22:56.415920+0000","last_undegraded":"2024-08-17T17:22:56.415920+0000","last_fullsized":"2024-08-17T17:22:56.415920+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1138,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417186+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:22:56.417186+0000","last_peered":"2024-08-17T17:22:56.417186+0000","last_clean":"2024-08-17T17:22:56.417186+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:22:56.417186+0000","last_undegraded":"2024-08-17T17:22:56.417186+0000","last_fullsized":"2024-08-17T17:22:56.417186+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1169,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417637+0000","last_change":"2024-08-17T17:21:06.458000+0000","last_active":"2024-08-17T17:22:56.417637+0000","last_peered":"2024-08-17T17:22:56.417637+0000","last_clean":"2024-08-17T17:22:56.417637+0000","last_became_active":"2024-08-17T17:21:06.457411+0000","last_became_peered":"2024-08-17T17:21:06.457411+0000","last_unstale":"2024-08-17T17:22:56.417637+0000","last_undegraded":"2024-08-17T17:22:56.417637+0000","last_fullsized":"2024-08-17T17:22:56.417637+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:52.740639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1160,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.418917+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:22:56.418917+0000","last_peered":"2024-08-17T17:22:56.418917+0000","last_clean":"2024-08-17T17:22:56.418917+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:22:56.418917+0000","last_undegraded":"2024-08-17T17:22:56.418917+0000","last_fullsized":"2024-08-17T17:22:56.418917+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":829,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414612+0000","last_change":"2024-08-17T17:21:06.454587+0000","last_active":"2024-08-17T17:22:56.414612+0000","last_peered":"2024-08-17T17:22:56.414612+0000","last_clean":"2024-08-17T17:22:56.414612+0000","last_became_active":"2024-08-17T17:21:06.454229+0000","last_became_peered":"2024-08-17T17:21:06.454229+0000","last_unstale":"2024-08-17T17:22:56.414612+0000","last_undegraded":"2024-08-17T17:22:56.414612+0000","last_fullsized":"2024-08-17T17:22:56.414612+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:38.340263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1136,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421025+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:22:56.421025+0000","last_peered":"2024-08-17T17:22:56.421025+0000","last_clean":"2024-08-17T17:22:56.421025+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:22:56.421025+0000","last_undegraded":"2024-08-17T17:22:56.421025+0000","last_fullsized":"2024-08-17T17:22:56.421025+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1166,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415505+0000","last_change":"2024-08-17T17:21:06.442796+0000","last_active":"2024-08-17T17:22:56.415505+0000","last_peered":"2024-08-17T17:22:56.415505+0000","last_clean":"2024-08-17T17:22:56.415505+0000","last_became_active":"2024-08-17T17:21:06.442466+0000","last_became_peered":"2024-08-17T17:21:06.442466+0000","last_unstale":"2024-08-17T17:22:56.415505+0000","last_undegraded":"2024-08-17T17:22:56.415505+0000","last_fullsized":"2024-08-17T17:22:56.415505+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:36.366620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1474,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421194+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:22:56.421194+0000","last_peered":"2024-08-17T17:22:56.421194+0000","last_clean":"2024-08-17T17:22:56.421194+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:22:56.421194+0000","last_undegraded":"2024-08-17T17:22:56.421194+0000","last_fullsized":"2024-08-17T17:22:56.421194+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1002,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415285+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:22:56.415285+0000","last_peered":"2024-08-17T17:22:56.415285+0000","last_clean":"2024-08-17T17:22:56.415285+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:22:56.415285+0000","last_undegraded":"2024-08-17T17:22:56.415285+0000","last_fullsized":"2024-08-17T17:22:56.415285+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1507,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.426027+0000","last_change":"2024-08-17T17:21:06.468788+0000","last_active":"2024-08-17T17:22:56.426027+0000","last_peered":"2024-08-17T17:22:56.426027+0000","last_clean":"2024-08-17T17:22:56.426027+0000","last_became_active":"2024-08-17T17:21:06.467141+0000","last_became_peered":"2024-08-17T17:21:06.467141+0000","last_unstale":"2024-08-17T17:22:56.426027+0000","last_undegraded":"2024-08-17T17:22:56.426027+0000","last_fullsized":"2024-08-17T17:22:56.426027+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:51.301359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1168,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415873+0000","last_change":"2024-08-17T17:21:06.435998+0000","last_active":"2024-08-17T17:22:56.415873+0000","last_peered":"2024-08-17T17:22:56.415873+0000","last_clean":"2024-08-17T17:22:56.415873+0000","last_became_active":"2024-08-17T17:21:06.435579+0000","last_became_peered":"2024-08-17T17:21:06.435579+0000","last_unstale":"2024-08-17T17:22:56.415873+0000","last_undegraded":"2024-08-17T17:22:56.415873+0000","last_fullsized":"2024-08-17T17:22:56.415873+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:01.743074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1537,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.426174+0000","last_change":"2024-08-17T17:21:06.473627+0000","last_active":"2024-08-17T17:22:56.426174+0000","last_peered":"2024-08-17T17:22:56.426174+0000","last_clean":"2024-08-17T17:22:56.426174+0000","last_became_active":"2024-08-17T17:21:06.473062+0000","last_became_peered":"2024-08-17T17:21:06.473062+0000","last_unstale":"2024-08-17T17:22:56.426174+0000","last_undegraded":"2024-08-17T17:22:56.426174+0000","last_fullsized":"2024-08-17T17:22:56.426174+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:21.413163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":963,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:23:07.315445+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:23:07.315445+0000","last_peered":"2024-08-17T17:23:07.315445+0000","last_clean":"2024-08-17T17:23:07.315445+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:23:07.315445+0000","last_undegraded":"2024-08-17T17:23:07.315445+0000","last_fullsized":"2024-08-17T17:23:07.315445+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1028,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421990+0000","last_change":"2024-08-17T17:21:06.471265+0000","last_active":"2024-08-17T17:22:56.421990+0000","last_peered":"2024-08-17T17:22:56.421990+0000","last_clean":"2024-08-17T17:22:56.421990+0000","last_became_active":"2024-08-17T17:21:06.470943+0000","last_became_peered":"2024-08-17T17:21:06.470943+0000","last_unstale":"2024-08-17T17:22:56.421990+0000","last_undegraded":"2024-08-17T17:22:56.421990+0000","last_fullsized":"2024-08-17T17:22:56.421990+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:41.070024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1514,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.422955+0000","last_change":"2024-08-17T17:22:48.377351+0000","last_active":"2024-08-17T17:22:56.422955+0000","last_peered":"2024-08-17T17:22:56.422955+0000","last_clean":"2024-08-17T17:22:56.422955+0000","last_became_active":"2024-08-17T17:22:48.377059+0000","last_became_peered":"2024-08-17T17:22:48.377059+0000","last_unstale":"2024-08-17T17:22:56.422955+0000","last_undegraded":"2024-08-17T17:22:56.422955+0000","last_fullsized":"2024-08-17T17:22:56.422955+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:15.250902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1185,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417912+0000","last_change":"2024-08-17T17:21:06.459139+0000","last_active":"2024-08-17T17:22:56.417912+0000","last_peered":"2024-08-17T17:22:56.417912+0000","last_clean":"2024-08-17T17:22:56.417912+0000","last_became_active":"2024-08-17T17:21:06.457850+0000","last_became_peered":"2024-08-17T17:21:06.457850+0000","last_unstale":"2024-08-17T17:22:56.417912+0000","last_undegraded":"2024-08-17T17:22:56.417912+0000","last_fullsized":"2024-08-17T17:22:56.417912+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:13.888576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1557,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.396718+0000","last_change":"2024-08-17T17:21:06.446321+0000","last_active":"2024-08-17T17:23:10.396718+0000","last_peered":"2024-08-17T17:23:10.396718+0000","last_clean":"2024-08-17T17:23:10.396718+0000","last_became_active":"2024-08-17T17:21:06.446019+0000","last_became_peered":"2024-08-17T17:21:06.446019+0000","last_unstale":"2024-08-17T17:23:10.396718+0000","last_undegraded":"2024-08-17T17:23:10.396718+0000","last_fullsized":"2024-08-17T17:23:10.396718+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:26.750070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":155,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417421+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:22:56.417421+0000","last_peered":"2024-08-17T17:22:56.417421+0000","last_clean":"2024-08-17T17:22:56.417421+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:22:56.417421+0000","last_undegraded":"2024-08-17T17:22:56.417421+0000","last_fullsized":"2024-08-17T17:22:56.417421+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1183,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417563+0000","last_change":"2024-08-17T17:21:06.465804+0000","last_active":"2024-08-17T17:22:56.417563+0000","last_peered":"2024-08-17T17:22:56.417563+0000","last_clean":"2024-08-17T17:22:56.417563+0000","last_became_active":"2024-08-17T17:21:06.465461+0000","last_became_peered":"2024-08-17T17:21:06.465461+0000","last_unstale":"2024-08-17T17:22:56.417563+0000","last_undegraded":"2024-08-17T17:22:56.417563+0000","last_fullsized":"2024-08-17T17:22:56.417563+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:06.443667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1175,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414946+0000","last_change":"2024-08-17T17:21:06.481634+0000","last_active":"2024-08-17T17:22:56.414946+0000","last_peered":"2024-08-17T17:22:56.414946+0000","last_clean":"2024-08-17T17:22:56.414946+0000","last_became_active":"2024-08-17T17:21:06.469081+0000","last_became_peered":"2024-08-17T17:21:06.469081+0000","last_unstale":"2024-08-17T17:22:56.414946+0000","last_undegraded":"2024-08-17T17:22:56.414946+0000","last_fullsized":"2024-08-17T17:22:56.414946+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:53:26.954738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1183,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417419+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:22:56.417419+0000","last_peered":"2024-08-17T17:22:56.417419+0000","last_clean":"2024-08-17T17:22:56.417419+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:22:56.417419+0000","last_undegraded":"2024-08-17T17:22:56.417419+0000","last_fullsized":"2024-08-17T17:22:56.417419+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1134,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415572+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:22:56.415572+0000","last_peered":"2024-08-17T17:22:56.415572+0000","last_clean":"2024-08-17T17:22:56.415572+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:22:56.415572+0000","last_undegraded":"2024-08-17T17:22:56.415572+0000","last_fullsized":"2024-08-17T17:22:56.415572+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1513,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.422197+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:22:56.422197+0000","last_peered":"2024-08-17T17:22:56.422197+0000","last_clean":"2024-08-17T17:22:56.422197+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:22:56.422197+0000","last_undegraded":"2024-08-17T17:22:56.422197+0000","last_fullsized":"2024-08-17T17:22:56.422197+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1488,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.423687+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:22:56.423687+0000","last_peered":"2024-08-17T17:22:56.423687+0000","last_clean":"2024-08-17T17:22:56.423687+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:22:56.423687+0000","last_undegraded":"2024-08-17T17:22:56.423687+0000","last_fullsized":"2024-08-17T17:22:56.423687+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1169,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417153+0000","last_change":"2024-08-17T17:21:06.483469+0000","last_active":"2024-08-17T17:22:56.417153+0000","last_peered":"2024-08-17T17:22:56.417153+0000","last_clean":"2024-08-17T17:22:56.417153+0000","last_became_active":"2024-08-17T17:21:06.483162+0000","last_became_peered":"2024-08-17T17:21:06.483162+0000","last_unstale":"2024-08-17T17:22:56.417153+0000","last_undegraded":"2024-08-17T17:22:56.417153+0000","last_fullsized":"2024-08-17T17:22:56.417153+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:24.898353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":177,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415191+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:22:56.415191+0000","last_peered":"2024-08-17T17:22:56.415191+0000","last_clean":"2024-08-17T17:22:56.415191+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:22:56.415191+0000","last_undegraded":"2024-08-17T17:22:56.415191+0000","last_fullsized":"2024-08-17T17:22:56.415191+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5304418304,"num_objects":80939,"num_object_clones":0,"num_object_copies":242817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":80939,"num_write_kb":5180096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16028794880,"data_stored":16028794880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":80939,"ondisk_log_size":80939,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201661,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2012236,"kb_used_data":1625984,"kb_used_omap":1,"kb_used_meta":386238,"kb_avail":91732916,"statfs":{"total":95995035648,"available":93934505984,"internally_reserved":0,"allocated":1665007616,"data_stored":1662615372,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":395508171},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463298,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1965508,"kb_used_data":1641592,"kb_used_omap":1,"kb_used_meta":323902,"kb_avail":91779644,"statfs":{"total":95995035648,"available":93982355456,"internally_reserved":0,"allocated":1680990208,"data_stored":1678606029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":331676105},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626939,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2429032,"kb_used_data":1909600,"kb_used_omap":42,"kb_used_meta":519381,"kb_avail":91316120,"statfs":{"total":95995035648,"available":93507706880,"internally_reserved":0,"allocated":1955430400,"data_stored":1953058338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":531846204},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785417,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2664180,"kb_used_data":2311208,"kb_used_omap":48,"kb_used_meta":352911,"kb_avail":91080972,"statfs":{"total":95995035648,"available":93266915328,"internally_reserved":0,"allocated":2366676992,"data_stored":2364304383,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":361381550},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724934,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2957312,"kb_used_data":2619444,"kb_used_omap":1,"kb_used_meta":337854,"kb_avail":90787840,"statfs":{"total":95995035648,"available":92966748160,"internally_reserved":0,"allocated":2682310656,"data_stored":2679926943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":345962950},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986571,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1999364,"kb_used_data":1643452,"kb_used_omap":1,"kb_used_meta":355902,"kb_avail":91745788,"statfs":{"total":95995035648,"available":93947686912,"internally_reserved":0,"allocated":1682894848,"data_stored":1680506630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":364444102},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706178,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2341944,"kb_used_data":1935088,"kb_used_omap":35,"kb_used_meta":406812,"kb_avail":91403208,"statfs":{"total":95995035648,"available":93596884992,"internally_reserved":0,"allocated":1981530112,"data_stored":1979150452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":416575543},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831951,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2423028,"kb_used_data":1943532,"kb_used_omap":65,"kb_used_meta":479422,"kb_avail":91322124,"statfs":{"total":95995035648,"available":93513854976,"internally_reserved":0,"allocated":1990176768,"data_stored":1987800648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":490928315},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1995702272,"data_stored":1995702272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2377973760,"data_stored":2377973760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1982201856,"data_stored":1982201856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1951268864,"data_stored":1951268864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1679360000,"data_stored":1679360000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2688352256,"data_stored":2688352256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1675231232,"data_stored":1675231232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1678704640,"data_stored":1678704640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:23:17.622 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:23:17.622 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:23:17.622 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_3 2024-08-17T17:23:17.623 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:23:17.623 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:17.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.666 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:17.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:17.767 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:17.793 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:17.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:17.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:17 smithi162 ceph-mon[104668]: from='client.45263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:23:17.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:17 smithi162 ceph-mon[104668]: pgmap v1548: 99 pgs: 99 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 76 MiB/s wr, 1.22k op/s 2024-08-17T17:23:17.895 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:17.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:17.948 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:17.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:18.049 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:18.090 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:18.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:18.192 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:18.223 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:18.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:18.325 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:18.361 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:18.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:18.415 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:18 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2104301659' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:18.415 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:18 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2104301659' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:18.463 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:18.489 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:18.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:18.503 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 51 16 88995 88979 109.021 67.5625 0.0067797 0.00916518 2024-08-17T17:23:18.590 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:18.630 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:23:18.630 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:23:18.630 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:23:18.631 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:23:18.631 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:23:18.631 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:23:18.631 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:23:18.631 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:23:18.631 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:23:18.631 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:23:18.631 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:23:18.631 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:23:18.631 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:23:18.631 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:23:18.632 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:23:18.632 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:23:18.640 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:18.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:18.742 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:18.788 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:18.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:18.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:18 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2104301659' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:18.891 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:18.921 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:18.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:18.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:19.023 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:19.068 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:19.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:19.170 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:19.211 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:19.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:19.313 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:19.348 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:19.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:19.450 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:19.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.501 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:19.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:19.504 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 52 16 90118 90102 108.273 70.1875 0.00736948 0.0092303 2024-08-17T17:23:19.602 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:19.646 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:19.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:19.747 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:19.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:19 smithi063 ceph-mon[109843]: pgmap v1549: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:23:19.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:19 smithi063 ceph-mon[114756]: pgmap v1549: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:23:19.782 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:19 smithi162 ceph-mon[104668]: pgmap v1549: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-08-17T17:23:19.884 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:19.907 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:19.920 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:19.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:19.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:19.928 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:20.022 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:20.071 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:20.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:20.172 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:20.205 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:20.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:20.307 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:20.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.353 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:20.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:20.454 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:20.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.497 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:20.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:20.504 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 53 16 91184 91168 107.487 66.625 0.00484906 0.0092987 2024-08-17T17:23:20.598 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:20.654 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:20.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:20.756 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:20.803 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:20.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:20.905 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:20.934 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:20.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:20.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:20 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:20 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:21.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:20 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:21.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:20 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:21.035 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:21.060 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:21.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:21.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:20 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:21.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:20 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:21.162 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:21.191 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:21.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:21.251 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:23:21.251 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:23:21.251 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:23:21.251 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:23:21.251 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:23:21.293 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:21.337 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:21.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:21.387 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:23:21.439 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:21.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.463 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:21.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:21.504 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 54 16 92279 92263 106.764 68.4375 0.00781102 0.00936041 2024-08-17T17:23:21.564 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:21.590 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:21.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:21.691 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:21.694 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:21 smithi063 ceph-mon[109843]: pgmap v1550: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T17:23:21.725 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:21.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:21.826 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:21.858 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:21.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:21.960 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:21.992 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:21.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:21.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:21 smithi063 ceph-mon[114756]: pgmap v1550: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T17:23:22.094 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:21 smithi162 ceph-mon[104668]: pgmap v1550: 99 pgs: 99 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T17:23:22.122 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:22.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:22.224 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:22.270 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:22.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:22.372 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:22.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.411 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:22.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:22.504 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 55 16 93419 93403 106.118 71.25 0.0106173 0.00941865 2024-08-17T17:23:22.512 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:22.554 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:22.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:22.656 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:22.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.697 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:22.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:22.798 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:22.833 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:22.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:22.935 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:22.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:22.986 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:22.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:23.087 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:23.131 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:23.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:23.233 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:23.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.268 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:23.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:23.369 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:23.391 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:23:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:23:23] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.51.0" 2024-08-17T17:23:23.406 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:23.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:23.504 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 56 16 94576 94560 105.515 72.3125 0.0103278 0.00947166 2024-08-17T17:23:23.508 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:23.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.552 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:23.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:23.653 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:23.686 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:23.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.688 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:23.788 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:23.824 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:23.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:23.927 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:23.973 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:23.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:23.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:23 smithi063 ceph-mon[109843]: pgmap v1551: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s 2024-08-17T17:23:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:23 smithi063 ceph-mon[114756]: pgmap v1551: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s 2024-08-17T17:23:24.076 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:23 smithi162 ceph-mon[104668]: pgmap v1551: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s 2024-08-17T17:23:24.118 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:24.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:24.220 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:24.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.263 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:24.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:24.364 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:24.409 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:24.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:24.504 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 57 15 95581 95566 104.767 62.875 0.00897396 0.0095387 2024-08-17T17:23:24.512 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:24.553 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:24.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:24.654 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:24.695 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:24.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:24.797 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:24.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.843 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:24.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:24.944 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:24.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:24.987 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:24.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:25.088 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:25.133 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:25.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:25.235 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:25.280 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:25.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:25.381 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:25.441 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:25.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:25.504 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 58 16 96532 96516 103.984 59.375 0.0127266 0.00961168 2024-08-17T17:23:25.543 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:25.573 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:25.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:25.675 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:25.697 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:25.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:25.799 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:25.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.855 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:25.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:25.876 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:25 smithi063 ceph-mon[109843]: pgmap v1552: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T17:23:25.877 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:25 smithi063 ceph-mon[114756]: pgmap v1552: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T17:23:25.878 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:23:25.881 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:23:25.956 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:25.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:25.993 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:25.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:26.094 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:26.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:25 smithi162 ceph-mon[104668]: pgmap v1552: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T17:23:26.135 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:26.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:26.237 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:26.275 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:26.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:26.377 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:26.403 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:26.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:26.504 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 59 16 97460 97444 103.205 58 0.0109418 0.00968444 2024-08-17T17:23:26.505 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:26.534 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:26.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:26.636 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:26.655 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:23:26.681 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:26.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:26.734 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:23:26.783 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:26.843 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:26.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:26.945 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:26.980 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:26.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:26.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:27.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:26 smithi063 ceph-mon[109843]: from='client.35355 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:23:27.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:26 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3929588038' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:27.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:26 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:27.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:26 smithi063 ceph-mon[114756]: from='client.35355 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:23:27.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:26 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3929588038' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:27.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:26 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:27.082 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:26 smithi162 ceph-mon[104668]: from='client.35355 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:23:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:26 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3929588038' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:26 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:27.118 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:27.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:27.220 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:27.249 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:27.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:27.351 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:27.392 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:27.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:27.494 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:27.504 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:23:27.504714+0000 min lat: 0.00109066 max lat: 0.153833 avg lat: 0.00974411 2024-08-17T17:23:27.504 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:23:27.504 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 60 16 98507 98491 102.575 65.4375 0.0128456 0.00974411 2024-08-17T17:23:27.529 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:27.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:27.624 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1552,"stamp":"2024-08-17T17:23:24.278987+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6047406053,"num_objects":92293,"num_object_clones":0,"num_object_copies":276879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":92293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":92331,"num_write_kb":5905121,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1246,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93965,"ondisk_log_size":93965,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21071064,"kb_used_data":17823512,"kb_used_omap":199,"kb_used_meta":3247288,"kb_avail":728890152,"statfs":{"total":767960285184,"available":746383515648,"internally_reserved":0,"allocated":18251276288,"data_stored":18232223455,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":3325223676},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,7,9,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":832831488,"num_objects":12708,"num_object_clones":0,"num_object_copies":38124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12708,"num_write_kb":813312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.008804"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1509,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.387505+0000","last_change":"2024-08-17T17:21:06.463828+0000","last_active":"2024-08-17T17:23:10.387505+0000","last_peered":"2024-08-17T17:23:10.387505+0000","last_clean":"2024-08-17T17:23:10.387505+0000","last_became_active":"2024-08-17T17:21:06.463464+0000","last_became_peered":"2024-08-17T17:21:06.463464+0000","last_unstale":"2024-08-17T17:23:10.387505+0000","last_undegraded":"2024-08-17T17:23:10.387505+0000","last_fullsized":"2024-08-17T17:23:10.387505+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:46.465006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1135,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.420493+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:22:56.420493+0000","last_peered":"2024-08-17T17:22:56.420493+0000","last_clean":"2024-08-17T17:22:56.420493+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:22:56.420493+0000","last_undegraded":"2024-08-17T17:22:56.420493+0000","last_fullsized":"2024-08-17T17:22:56.420493+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1147,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415489+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:22:56.415489+0000","last_peered":"2024-08-17T17:22:56.415489+0000","last_clean":"2024-08-17T17:22:56.415489+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:22:56.415489+0000","last_undegraded":"2024-08-17T17:22:56.415489+0000","last_fullsized":"2024-08-17T17:22:56.415489+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1002,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.420230+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:22:56.420230+0000","last_peered":"2024-08-17T17:22:56.420230+0000","last_clean":"2024-08-17T17:22:56.420230+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:22:56.420230+0000","last_undegraded":"2024-08-17T17:22:56.420230+0000","last_fullsized":"2024-08-17T17:22:56.420230+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1061,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.386658+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:23:10.386658+0000","last_peered":"2024-08-17T17:23:10.386658+0000","last_clean":"2024-08-17T17:23:10.386658+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:23:10.386658+0000","last_undegraded":"2024-08-17T17:23:10.386658+0000","last_fullsized":"2024-08-17T17:23:10.386658+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1118,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.420113+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:22:56.420113+0000","last_peered":"2024-08-17T17:22:56.420113+0000","last_clean":"2024-08-17T17:22:56.420113+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:22:56.420113+0000","last_undegraded":"2024-08-17T17:22:56.420113+0000","last_fullsized":"2024-08-17T17:22:56.420113+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1509,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.388879+0000","last_change":"2024-08-17T17:21:06.442598+0000","last_active":"2024-08-17T17:23:10.388879+0000","last_peered":"2024-08-17T17:23:10.388879+0000","last_clean":"2024-08-17T17:23:10.388879+0000","last_became_active":"2024-08-17T17:21:06.442264+0000","last_became_peered":"2024-08-17T17:21:06.442264+0000","last_unstale":"2024-08-17T17:23:10.388879+0000","last_undegraded":"2024-08-17T17:23:10.388879+0000","last_fullsized":"2024-08-17T17:23:10.388879+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:35.003620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1195,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414239+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:22:56.414239+0000","last_peered":"2024-08-17T17:22:56.414239+0000","last_clean":"2024-08-17T17:22:56.414239+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:22:56.414239+0000","last_undegraded":"2024-08-17T17:22:56.414239+0000","last_fullsized":"2024-08-17T17:22:56.414239+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"1207'5762","reported_seq":5783,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:22.299392+0000","last_change":"2024-08-17T17:22:10.861776+0000","last_active":"2024-08-17T17:23:22.299392+0000","last_peered":"2024-08-17T17:23:22.299392+0000","last_clean":"2024-08-17T17:23:22.299392+0000","last_became_active":"2024-08-17T17:22:10.861509+0000","last_became_peered":"2024-08-17T17:22:10.861509+0000","last_unstale":"2024-08-17T17:23:22.299392+0000","last_undegraded":"2024-08-17T17:23:22.299392+0000","last_fullsized":"2024-08-17T17:23:22.299392+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5762,"log_dups_size":0,"ondisk_log_size":5762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:16:43.850909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":377618432,"num_objects":5762,"num_object_clones":0,"num_object_copies":17286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5762,"num_write_kb":368768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1161,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.310806+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:22:57.310806+0000","last_peered":"2024-08-17T17:22:57.310806+0000","last_clean":"2024-08-17T17:22:57.310806+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:22:57.310806+0000","last_undegraded":"2024-08-17T17:22:57.310806+0000","last_fullsized":"2024-08-17T17:22:57.310806+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1133,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415219+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:22:56.415219+0000","last_peered":"2024-08-17T17:22:56.415219+0000","last_clean":"2024-08-17T17:22:56.415219+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:22:56.415219+0000","last_undegraded":"2024-08-17T17:22:56.415219+0000","last_fullsized":"2024-08-17T17:22:56.415219+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"1207'5892","reported_seq":5913,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:22.215386+0000","last_change":"2024-08-17T17:22:10.860210+0000","last_active":"2024-08-17T17:23:22.215386+0000","last_peered":"2024-08-17T17:23:22.215386+0000","last_clean":"2024-08-17T17:23:22.215386+0000","last_became_active":"2024-08-17T17:22:10.859854+0000","last_became_peered":"2024-08-17T17:22:10.859854+0000","last_unstale":"2024-08-17T17:23:22.215386+0000","last_undegraded":"2024-08-17T17:23:22.215386+0000","last_fullsized":"2024-08-17T17:23:22.215386+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5892,"log_dups_size":0,"ondisk_log_size":5892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:31.313667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386138112,"num_objects":5892,"num_object_clones":0,"num_object_copies":17676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5892,"num_write_kb":377088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1162,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.310784+0000","last_change":"2024-08-17T17:21:06.540689+0000","last_active":"2024-08-17T17:22:57.310784+0000","last_peered":"2024-08-17T17:22:57.310784+0000","last_clean":"2024-08-17T17:22:57.310784+0000","last_became_active":"2024-08-17T17:21:06.461614+0000","last_became_peered":"2024-08-17T17:21:06.461614+0000","last_unstale":"2024-08-17T17:22:57.310784+0000","last_undegraded":"2024-08-17T17:22:57.310784+0000","last_fullsized":"2024-08-17T17:22:57.310784+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:11.696898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1185,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417386+0000","last_change":"2024-08-17T17:21:06.470389+0000","last_active":"2024-08-17T17:22:56.417386+0000","last_peered":"2024-08-17T17:22:56.417386+0000","last_clean":"2024-08-17T17:22:56.417386+0000","last_became_active":"2024-08-17T17:21:06.469966+0000","last_became_peered":"2024-08-17T17:21:06.469966+0000","last_unstale":"2024-08-17T17:22:56.417386+0000","last_undegraded":"2024-08-17T17:22:56.417386+0000","last_fullsized":"2024-08-17T17:22:56.417386+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:56.140616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"1207'5777","reported_seq":5798,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:22.224730+0000","last_change":"2024-08-17T17:22:10.943999+0000","last_active":"2024-08-17T17:23:22.224730+0000","last_peered":"2024-08-17T17:23:22.224730+0000","last_clean":"2024-08-17T17:23:22.224730+0000","last_became_active":"2024-08-17T17:22:10.943661+0000","last_became_peered":"2024-08-17T17:22:10.943661+0000","last_unstale":"2024-08-17T17:23:22.224730+0000","last_undegraded":"2024-08-17T17:23:22.224730+0000","last_fullsized":"2024-08-17T17:23:22.224730+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5777,"log_dups_size":0,"ondisk_log_size":5777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:21:49.050541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378601472,"num_objects":5777,"num_object_clones":0,"num_object_copies":17331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5777,"num_write_kb":369728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1120,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421145+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:22:56.421145+0000","last_peered":"2024-08-17T17:22:56.421145+0000","last_clean":"2024-08-17T17:22:56.421145+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:22:56.421145+0000","last_undegraded":"2024-08-17T17:22:56.421145+0000","last_fullsized":"2024-08-17T17:22:56.421145+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1167,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414538+0000","last_change":"2024-08-17T17:21:06.446798+0000","last_active":"2024-08-17T17:22:56.414538+0000","last_peered":"2024-08-17T17:22:56.414538+0000","last_clean":"2024-08-17T17:22:56.414538+0000","last_became_active":"2024-08-17T17:21:06.446205+0000","last_became_peered":"2024-08-17T17:21:06.446205+0000","last_unstale":"2024-08-17T17:22:56.414538+0000","last_undegraded":"2024-08-17T17:22:56.414538+0000","last_fullsized":"2024-08-17T17:22:56.414538+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:28.584048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"1207'5625","reported_seq":5660,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:20.145496+0000","last_change":"2024-08-17T17:22:10.900862+0000","last_active":"2024-08-17T17:23:20.145496+0000","last_peered":"2024-08-17T17:23:20.145496+0000","last_clean":"2024-08-17T17:23:20.145496+0000","last_became_active":"2024-08-17T17:22:10.900552+0000","last_became_peered":"2024-08-17T17:22:10.900552+0000","last_unstale":"2024-08-17T17:23:20.145496+0000","last_undegraded":"2024-08-17T17:23:20.145496+0000","last_fullsized":"2024-08-17T17:23:20.145496+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5625,"log_dups_size":0,"ondisk_log_size":5625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:07.195144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368640000,"num_objects":5625,"num_object_clones":0,"num_object_copies":16875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5625,"num_write_kb":360000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1149,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414971+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:22:56.414971+0000","last_peered":"2024-08-17T17:22:56.414971+0000","last_clean":"2024-08-17T17:22:56.414971+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:22:56.414971+0000","last_undegraded":"2024-08-17T17:22:56.414971+0000","last_fullsized":"2024-08-17T17:22:56.414971+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":931,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.392418+0000","last_change":"2024-08-17T17:21:06.460905+0000","last_active":"2024-08-17T17:23:10.392418+0000","last_peered":"2024-08-17T17:23:10.392418+0000","last_clean":"2024-08-17T17:23:10.392418+0000","last_became_active":"2024-08-17T17:21:06.460592+0000","last_became_peered":"2024-08-17T17:21:06.460592+0000","last_unstale":"2024-08-17T17:23:10.392418+0000","last_undegraded":"2024-08-17T17:23:10.392418+0000","last_fullsized":"2024-08-17T17:23:10.392418+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:10.146916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.0","version":"1207'5538","reported_seq":5573,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:20.131632+0000","last_change":"2024-08-17T17:22:10.900154+0000","last_active":"2024-08-17T17:23:20.131632+0000","last_peered":"2024-08-17T17:23:20.131632+0000","last_clean":"2024-08-17T17:23:20.131632+0000","last_became_active":"2024-08-17T17:22:10.899766+0000","last_became_peered":"2024-08-17T17:22:10.899766+0000","last_unstale":"2024-08-17T17:23:20.131632+0000","last_undegraded":"2024-08-17T17:23:20.131632+0000","last_fullsized":"2024-08-17T17:23:20.131632+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5538,"log_dups_size":0,"ondisk_log_size":5538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:22:07.705078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362938368,"num_objects":5538,"num_object_clones":0,"num_object_copies":16614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5538,"num_write_kb":354432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1528,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.419942+0000","last_change":"2024-08-17T17:21:16.100398+0000","last_active":"2024-08-17T17:22:56.419942+0000","last_peered":"2024-08-17T17:22:56.419942+0000","last_clean":"2024-08-17T17:22:56.419942+0000","last_became_active":"2024-08-17T17:18:28.521573+0000","last_became_peered":"2024-08-17T17:18:28.521573+0000","last_unstale":"2024-08-17T17:22:56.419942+0000","last_undegraded":"2024-08-17T17:22:56.419942+0000","last_fullsized":"2024-08-17T17:22:56.419942+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:21:16.100258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:21:16.100258+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:57.628353+0000","scrub_duration":0.011045358999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1530,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.387648+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:23:10.387648+0000","last_peered":"2024-08-17T17:23:10.387648+0000","last_clean":"2024-08-17T17:23:10.387648+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:23:10.387648+0000","last_undegraded":"2024-08-17T17:23:10.387648+0000","last_fullsized":"2024-08-17T17:23:10.387648+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1152,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.249708+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:22:57.249708+0000","last_peered":"2024-08-17T17:22:57.249708+0000","last_clean":"2024-08-17T17:22:57.249708+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:22:57.249708+0000","last_undegraded":"2024-08-17T17:22:57.249708+0000","last_fullsized":"2024-08-17T17:22:57.249708+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"1207'5690","reported_seq":5724,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:21.544077+0000","last_change":"2024-08-17T17:22:10.944783+0000","last_active":"2024-08-17T17:23:21.544077+0000","last_peered":"2024-08-17T17:23:21.544077+0000","last_clean":"2024-08-17T17:23:21.544077+0000","last_became_active":"2024-08-17T17:22:10.944487+0000","last_became_peered":"2024-08-17T17:22:10.944487+0000","last_unstale":"2024-08-17T17:23:21.544077+0000","last_undegraded":"2024-08-17T17:23:21.544077+0000","last_fullsized":"2024-08-17T17:23:21.544077+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5690,"log_dups_size":0,"ondisk_log_size":5690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:07:28.515134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372899840,"num_objects":5690,"num_object_clones":0,"num_object_copies":17070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5690,"num_write_kb":364160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1474,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414963+0000","last_change":"2024-08-17T17:18:32.884334+0000","last_active":"2024-08-17T17:22:56.414963+0000","last_peered":"2024-08-17T17:22:56.414963+0000","last_clean":"2024-08-17T17:22:56.414963+0000","last_became_active":"2024-08-17T17:18:28.520186+0000","last_became_peered":"2024-08-17T17:18:28.520186+0000","last_unstale":"2024-08-17T17:22:56.414963+0000","last_undegraded":"2024-08-17T17:22:56.414963+0000","last_fullsized":"2024-08-17T17:22:56.414963+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:32.884246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:32.884246+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:49.286656+0000","scrub_duration":0.001842677,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1168,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414687+0000","last_change":"2024-08-17T17:21:06.446841+0000","last_active":"2024-08-17T17:22:56.414687+0000","last_peered":"2024-08-17T17:22:56.414687+0000","last_clean":"2024-08-17T17:22:56.414687+0000","last_became_active":"2024-08-17T17:21:06.446271+0000","last_became_peered":"2024-08-17T17:21:06.446271+0000","last_unstale":"2024-08-17T17:22:56.414687+0000","last_undegraded":"2024-08-17T17:22:56.414687+0000","last_fullsized":"2024-08-17T17:22:56.414687+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:20.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1185,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.249739+0000","last_change":"2024-08-17T17:21:06.480911+0000","last_active":"2024-08-17T17:22:57.249739+0000","last_peered":"2024-08-17T17:22:57.249739+0000","last_clean":"2024-08-17T17:22:57.249739+0000","last_became_active":"2024-08-17T17:21:06.480551+0000","last_became_peered":"2024-08-17T17:21:06.480551+0000","last_unstale":"2024-08-17T17:22:57.249739+0000","last_undegraded":"2024-08-17T17:22:57.249739+0000","last_fullsized":"2024-08-17T17:22:57.249739+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:26.003704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"1207'5946","reported_seq":5967,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:22.247342+0000","last_change":"2024-08-17T17:22:10.944020+0000","last_active":"2024-08-17T17:23:22.247342+0000","last_peered":"2024-08-17T17:23:22.247342+0000","last_clean":"2024-08-17T17:23:22.247342+0000","last_became_active":"2024-08-17T17:22:10.943694+0000","last_became_peered":"2024-08-17T17:22:10.943694+0000","last_unstale":"2024-08-17T17:23:22.247342+0000","last_undegraded":"2024-08-17T17:23:22.247342+0000","last_fullsized":"2024-08-17T17:23:22.247342+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5946,"log_dups_size":0,"ondisk_log_size":5946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:34.525383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389677056,"num_objects":5946,"num_object_clones":0,"num_object_copies":17838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5946,"num_write_kb":380544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1153,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417473+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:22:56.417473+0000","last_peered":"2024-08-17T17:22:56.417473+0000","last_clean":"2024-08-17T17:22:56.417473+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:22:56.417473+0000","last_undegraded":"2024-08-17T17:22:56.417473+0000","last_fullsized":"2024-08-17T17:22:56.417473+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1170,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:23:02.252360+0000","last_change":"2024-08-17T17:21:06.451521+0000","last_active":"2024-08-17T17:23:02.252360+0000","last_peered":"2024-08-17T17:23:02.252360+0000","last_clean":"2024-08-17T17:23:02.252360+0000","last_became_active":"2024-08-17T17:21:06.451197+0000","last_became_peered":"2024-08-17T17:21:06.451197+0000","last_unstale":"2024-08-17T17:23:02.252360+0000","last_undegraded":"2024-08-17T17:23:02.252360+0000","last_fullsized":"2024-08-17T17:23:02.252360+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:04.000692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"1207'5886","reported_seq":5907,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:22.326790+0000","last_change":"2024-08-17T17:22:10.900282+0000","last_active":"2024-08-17T17:23:22.326790+0000","last_peered":"2024-08-17T17:23:22.326790+0000","last_clean":"2024-08-17T17:23:22.326790+0000","last_became_active":"2024-08-17T17:22:10.899947+0000","last_became_peered":"2024-08-17T17:22:10.899947+0000","last_unstale":"2024-08-17T17:23:22.326790+0000","last_undegraded":"2024-08-17T17:23:22.326790+0000","last_fullsized":"2024-08-17T17:23:22.326790+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5886,"log_dups_size":0,"ondisk_log_size":5886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:10.148524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385744896,"num_objects":5886,"num_object_clones":0,"num_object_copies":17658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5886,"num_write_kb":376704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1474,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415316+0000","last_change":"2024-08-17T17:18:31.850829+0000","last_active":"2024-08-17T17:22:56.415316+0000","last_peered":"2024-08-17T17:22:56.415316+0000","last_clean":"2024-08-17T17:22:56.415316+0000","last_became_active":"2024-08-17T17:18:28.520946+0000","last_became_peered":"2024-08-17T17:18:28.520946+0000","last_unstale":"2024-08-17T17:22:56.415316+0000","last_undegraded":"2024-08-17T17:22:56.415316+0000","last_fullsized":"2024-08-17T17:22:56.415316+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:31.850745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:31.850745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:17.218084+0000","scrub_duration":0.001317447,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1512,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.385310+0000","last_change":"2024-08-17T17:21:06.441118+0000","last_active":"2024-08-17T17:23:10.385310+0000","last_peered":"2024-08-17T17:23:10.385310+0000","last_clean":"2024-08-17T17:23:10.385310+0000","last_became_active":"2024-08-17T17:21:06.440800+0000","last_became_peered":"2024-08-17T17:21:06.440800+0000","last_unstale":"2024-08-17T17:23:10.385310+0000","last_undegraded":"2024-08-17T17:23:10.385310+0000","last_fullsized":"2024-08-17T17:23:10.385310+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:08.091528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":18,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.396377+0000","last_change":"2024-08-17T17:22:48.374766+0000","last_active":"2024-08-17T17:23:10.396377+0000","last_peered":"2024-08-17T17:23:10.396377+0000","last_clean":"2024-08-17T17:23:10.396377+0000","last_became_active":"2024-08-17T17:22:48.373556+0000","last_became_peered":"2024-08-17T17:22:48.373556+0000","last_unstale":"2024-08-17T17:23:10.396377+0000","last_undegraded":"2024-08-17T17:23:10.396377+0000","last_fullsized":"2024-08-17T17:23:10.396377+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:33.362080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.9","version":"1207'5821","reported_seq":5842,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:22.234636+0000","last_change":"2024-08-17T17:22:10.944057+0000","last_active":"2024-08-17T17:23:22.234636+0000","last_peered":"2024-08-17T17:23:22.234636+0000","last_clean":"2024-08-17T17:23:22.234636+0000","last_became_active":"2024-08-17T17:22:10.943681+0000","last_became_peered":"2024-08-17T17:22:10.943681+0000","last_unstale":"2024-08-17T17:23:22.234636+0000","last_undegraded":"2024-08-17T17:23:22.234636+0000","last_fullsized":"2024-08-17T17:23:22.234636+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5821,"log_dups_size":0,"ondisk_log_size":5821,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:24.524487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381485056,"num_objects":5821,"num_object_clones":0,"num_object_copies":17463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5821,"num_write_kb":372544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1528,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421612+0000","last_change":"2024-08-17T17:18:33.814375+0000","last_active":"2024-08-17T17:22:56.421612+0000","last_peered":"2024-08-17T17:22:56.421612+0000","last_clean":"2024-08-17T17:22:56.421612+0000","last_became_active":"2024-08-17T17:18:33.813208+0000","last_became_peered":"2024-08-17T17:18:33.813208+0000","last_unstale":"2024-08-17T17:22:56.421612+0000","last_undegraded":"2024-08-17T17:22:56.421612+0000","last_fullsized":"2024-08-17T17:22:56.421612+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:25.057675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:25.057675+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:20:45.235879+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1163,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414765+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:22:56.414765+0000","last_peered":"2024-08-17T17:22:56.414765+0000","last_clean":"2024-08-17T17:22:56.414765+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:22:56.414765+0000","last_undegraded":"2024-08-17T17:22:56.414765+0000","last_fullsized":"2024-08-17T17:22:56.414765+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1477,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.384736+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:23:10.384736+0000","last_peered":"2024-08-17T17:23:10.384736+0000","last_clean":"2024-08-17T17:23:10.384736+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:23:10.384736+0000","last_undegraded":"2024-08-17T17:23:10.384736+0000","last_fullsized":"2024-08-17T17:23:10.384736+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1135,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:57.218577+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:22:57.218577+0000","last_peered":"2024-08-17T17:22:57.218577+0000","last_clean":"2024-08-17T17:22:57.218577+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:22:57.218577+0000","last_undegraded":"2024-08-17T17:22:57.218577+0000","last_fullsized":"2024-08-17T17:22:57.218577+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"1207'5842","reported_seq":5877,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:21.544405+0000","last_change":"2024-08-17T17:22:10.900900+0000","last_active":"2024-08-17T17:23:21.544405+0000","last_peered":"2024-08-17T17:23:21.544405+0000","last_clean":"2024-08-17T17:23:21.544405+0000","last_became_active":"2024-08-17T17:22:10.899812+0000","last_became_peered":"2024-08-17T17:22:10.899812+0000","last_unstale":"2024-08-17T17:23:21.544405+0000","last_undegraded":"2024-08-17T17:23:21.544405+0000","last_fullsized":"2024-08-17T17:23:21.544405+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5842,"log_dups_size":0,"ondisk_log_size":5842,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:25:56.258127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382861312,"num_objects":5842,"num_object_clones":0,"num_object_copies":17526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5842,"num_write_kb":373888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":767,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414472+0000","last_change":"2024-08-17T17:22:09.230320+0000","last_active":"2024-08-17T17:22:56.414472+0000","last_peered":"2024-08-17T17:22:56.414472+0000","last_clean":"2024-08-17T17:22:56.414472+0000","last_became_active":"2024-08-17T17:21:08.910463+0000","last_became_peered":"2024-08-17T17:21:08.910463+0000","last_unstale":"2024-08-17T17:22:56.414472+0000","last_undegraded":"2024-08-17T17:22:56.414472+0000","last_fullsized":"2024-08-17T17:22:56.414472+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:09.230214+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.230214+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:58.582858+0000","scrub_duration":0.001729597,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1199,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421543+0000","last_change":"2024-08-17T17:21:06.430639+0000","last_active":"2024-08-17T17:22:56.421543+0000","last_peered":"2024-08-17T17:22:56.421543+0000","last_clean":"2024-08-17T17:22:56.421543+0000","last_became_active":"2024-08-17T17:21:06.430297+0000","last_became_peered":"2024-08-17T17:21:06.430297+0000","last_unstale":"2024-08-17T17:22:56.421543+0000","last_undegraded":"2024-08-17T17:22:56.421543+0000","last_fullsized":"2024-08-17T17:22:56.421543+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:57.794620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1002,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417581+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:22:56.417581+0000","last_peered":"2024-08-17T17:22:56.417581+0000","last_clean":"2024-08-17T17:22:56.417581+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:22:56.417581+0000","last_undegraded":"2024-08-17T17:22:56.417581+0000","last_fullsized":"2024-08-17T17:22:56.417581+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414874+0000","last_change":"2024-08-17T17:22:48.370769+0000","last_active":"2024-08-17T17:22:56.414874+0000","last_peered":"2024-08-17T17:22:56.414874+0000","last_clean":"2024-08-17T17:22:56.414874+0000","last_became_active":"2024-08-17T17:22:48.370531+0000","last_became_peered":"2024-08-17T17:22:48.370531+0000","last_unstale":"2024-08-17T17:22:56.414874+0000","last_undegraded":"2024-08-17T17:22:56.414874+0000","last_fullsized":"2024-08-17T17:22:56.414874+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:02.943823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.b","version":"1207'5688","reported_seq":5723,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:20.118854+0000","last_change":"2024-08-17T17:22:10.900740+0000","last_active":"2024-08-17T17:23:20.118854+0000","last_peered":"2024-08-17T17:23:20.118854+0000","last_clean":"2024-08-17T17:23:20.118854+0000","last_became_active":"2024-08-17T17:22:10.900452+0000","last_became_peered":"2024-08-17T17:22:10.900452+0000","last_unstale":"2024-08-17T17:23:20.118854+0000","last_undegraded":"2024-08-17T17:23:20.118854+0000","last_fullsized":"2024-08-17T17:23:20.118854+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5688,"log_dups_size":0,"ondisk_log_size":5688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:38:40.285223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372768768,"num_objects":5688,"num_object_clones":0,"num_object_copies":17064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5688,"num_write_kb":364032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1549,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.420147+0000","last_change":"2024-08-17T17:22:07.078517+0000","last_active":"2024-08-17T17:22:56.420147+0000","last_peered":"2024-08-17T17:22:56.420147+0000","last_clean":"2024-08-17T17:22:56.420147+0000","last_became_active":"2024-08-17T17:18:33.816660+0000","last_became_peered":"2024-08-17T17:18:33.816660+0000","last_unstale":"2024-08-17T17:22:56.420147+0000","last_undegraded":"2024-08-17T17:22:56.420147+0000","last_fullsized":"2024-08-17T17:22:56.420147+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:07.078434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:07.078434+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:56.205556+0000","scrub_duration":0.0013001919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1574,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.396569+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:23:10.396569+0000","last_peered":"2024-08-17T17:23:10.396569+0000","last_clean":"2024-08-17T17:23:10.396569+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:23:10.396569+0000","last_undegraded":"2024-08-17T17:23:10.396569+0000","last_fullsized":"2024-08-17T17:23:10.396569+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":966,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.396064+0000","last_change":"2024-08-17T17:21:06.428684+0000","last_active":"2024-08-17T17:23:10.396064+0000","last_peered":"2024-08-17T17:23:10.396064+0000","last_clean":"2024-08-17T17:23:10.396064+0000","last_became_active":"2024-08-17T17:21:06.428178+0000","last_became_peered":"2024-08-17T17:21:06.428178+0000","last_unstale":"2024-08-17T17:23:10.396064+0000","last_undegraded":"2024-08-17T17:23:10.396064+0000","last_fullsized":"2024-08-17T17:23:10.396064+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:14.836119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":846,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415453+0000","last_change":"2024-08-17T17:21:06.443565+0000","last_active":"2024-08-17T17:22:56.415453+0000","last_peered":"2024-08-17T17:22:56.415453+0000","last_clean":"2024-08-17T17:22:56.415453+0000","last_became_active":"2024-08-17T17:21:06.443215+0000","last_became_peered":"2024-08-17T17:21:06.443215+0000","last_unstale":"2024-08-17T17:22:56.415453+0000","last_undegraded":"2024-08-17T17:22:56.415453+0000","last_fullsized":"2024-08-17T17:22:56.415453+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:47.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"1207'5849","reported_seq":5868,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:22.323145+0000","last_change":"2024-08-17T17:22:10.859676+0000","last_active":"2024-08-17T17:23:22.323145+0000","last_peered":"2024-08-17T17:23:22.323145+0000","last_clean":"2024-08-17T17:23:22.323145+0000","last_became_active":"2024-08-17T17:22:10.859289+0000","last_became_peered":"2024-08-17T17:22:10.859289+0000","last_unstale":"2024-08-17T17:23:22.323145+0000","last_undegraded":"2024-08-17T17:23:22.323145+0000","last_fullsized":"2024-08-17T17:23:22.323145+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5849,"log_dups_size":0,"ondisk_log_size":5849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:43.302993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383320064,"num_objects":5849,"num_object_clones":0,"num_object_copies":17547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5849,"num_write_kb":374336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1491,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415178+0000","last_change":"2024-08-17T17:18:35.916482+0000","last_active":"2024-08-17T17:22:56.415178+0000","last_peered":"2024-08-17T17:22:56.415178+0000","last_clean":"2024-08-17T17:22:56.415178+0000","last_became_active":"2024-08-17T17:18:34.827747+0000","last_became_peered":"2024-08-17T17:18:34.827747+0000","last_unstale":"2024-08-17T17:22:56.415178+0000","last_undegraded":"2024-08-17T17:22:56.415178+0000","last_fullsized":"2024-08-17T17:22:56.415178+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:35.916381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:35.916381+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:28.492622+0000","scrub_duration":0.005340521,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1190,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.416332+0000","last_change":"2024-08-17T17:21:06.456978+0000","last_active":"2024-08-17T17:22:56.416332+0000","last_peered":"2024-08-17T17:22:56.416332+0000","last_clean":"2024-08-17T17:22:56.416332+0000","last_became_active":"2024-08-17T17:21:06.452265+0000","last_became_peered":"2024-08-17T17:21:06.452265+0000","last_unstale":"2024-08-17T17:22:56.416332+0000","last_undegraded":"2024-08-17T17:22:56.416332+0000","last_fullsized":"2024-08-17T17:22:56.416332+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:52.893746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1172,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417922+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:22:56.417922+0000","last_peered":"2024-08-17T17:22:56.417922+0000","last_clean":"2024-08-17T17:22:56.417922+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:22:56.417922+0000","last_undegraded":"2024-08-17T17:22:56.417922+0000","last_fullsized":"2024-08-17T17:22:56.417922+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1042,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.396805+0000","last_change":"2024-08-17T17:21:06.435754+0000","last_active":"2024-08-17T17:23:10.396805+0000","last_peered":"2024-08-17T17:23:10.396805+0000","last_clean":"2024-08-17T17:23:10.396805+0000","last_became_active":"2024-08-17T17:21:06.435442+0000","last_became_peered":"2024-08-17T17:21:06.435442+0000","last_unstale":"2024-08-17T17:23:10.396805+0000","last_undegraded":"2024-08-17T17:23:10.396805+0000","last_fullsized":"2024-08-17T17:23:10.396805+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:34.849042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"1207'5749","reported_seq":5784,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:21.527779+0000","last_change":"2024-08-17T17:22:10.944986+0000","last_active":"2024-08-17T17:23:21.527779+0000","last_peered":"2024-08-17T17:23:21.527779+0000","last_clean":"2024-08-17T17:23:21.527779+0000","last_became_active":"2024-08-17T17:22:10.944267+0000","last_became_peered":"2024-08-17T17:22:10.944267+0000","last_unstale":"2024-08-17T17:23:21.527779+0000","last_undegraded":"2024-08-17T17:23:21.527779+0000","last_fullsized":"2024-08-17T17:23:21.527779+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5749,"log_dups_size":0,"ondisk_log_size":5749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:39:52.209246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376766464,"num_objects":5749,"num_object_clones":0,"num_object_copies":17247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5749,"num_write_kb":367936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":733,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415788+0000","last_change":"2024-08-17T17:22:06.152683+0000","last_active":"2024-08-17T17:22:56.415788+0000","last_peered":"2024-08-17T17:22:56.415788+0000","last_clean":"2024-08-17T17:22:56.415788+0000","last_became_active":"2024-08-17T17:18:28.526439+0000","last_became_peered":"2024-08-17T17:18:28.526439+0000","last_unstale":"2024-08-17T17:22:56.415788+0000","last_undegraded":"2024-08-17T17:22:56.415788+0000","last_fullsized":"2024-08-17T17:22:56.415788+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:06.152592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:06.152592+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:30.807010+0000","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1165,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415646+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:22:56.415646+0000","last_peered":"2024-08-17T17:22:56.415646+0000","last_clean":"2024-08-17T17:22:56.415646+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:22:56.415646+0000","last_undegraded":"2024-08-17T17:22:56.415646+0000","last_fullsized":"2024-08-17T17:22:56.415646+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":855,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.420776+0000","last_change":"2024-08-17T17:21:06.434991+0000","last_active":"2024-08-17T17:22:56.420776+0000","last_peered":"2024-08-17T17:22:56.420776+0000","last_clean":"2024-08-17T17:22:56.420776+0000","last_became_active":"2024-08-17T17:21:06.434630+0000","last_became_peered":"2024-08-17T17:21:06.434630+0000","last_unstale":"2024-08-17T17:22:56.420776+0000","last_undegraded":"2024-08-17T17:22:56.420776+0000","last_fullsized":"2024-08-17T17:22:56.420776+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:24.104609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":797,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.416080+0000","last_change":"2024-08-17T17:21:06.462532+0000","last_active":"2024-08-17T17:22:56.416080+0000","last_peered":"2024-08-17T17:22:56.416080+0000","last_clean":"2024-08-17T17:22:56.416080+0000","last_became_active":"2024-08-17T17:21:06.462183+0000","last_became_peered":"2024-08-17T17:21:06.462183+0000","last_unstale":"2024-08-17T17:22:56.416080+0000","last_undegraded":"2024-08-17T17:22:56.416080+0000","last_fullsized":"2024-08-17T17:22:56.416080+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:50.033322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.f","version":"1207'5678","reported_seq":5713,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:20.113705+0000","last_change":"2024-08-17T17:22:10.900592+0000","last_active":"2024-08-17T17:23:20.113705+0000","last_peered":"2024-08-17T17:23:20.113705+0000","last_clean":"2024-08-17T17:23:20.113705+0000","last_became_active":"2024-08-17T17:22:10.900279+0000","last_became_peered":"2024-08-17T17:22:10.900279+0000","last_unstale":"2024-08-17T17:23:20.113705+0000","last_undegraded":"2024-08-17T17:23:20.113705+0000","last_fullsized":"2024-08-17T17:23:20.113705+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5678,"log_dups_size":0,"ondisk_log_size":5678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:52:25.130107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372113408,"num_objects":5678,"num_object_clones":0,"num_object_copies":17034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5678,"num_write_kb":363392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":750,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415656+0000","last_change":"2024-08-17T17:22:08.218836+0000","last_active":"2024-08-17T17:22:56.415656+0000","last_peered":"2024-08-17T17:22:56.415656+0000","last_clean":"2024-08-17T17:22:56.415656+0000","last_became_active":"2024-08-17T17:18:34.831498+0000","last_became_peered":"2024-08-17T17:18:34.831498+0000","last_unstale":"2024-08-17T17:22:56.415656+0000","last_undegraded":"2024-08-17T17:22:56.415656+0000","last_fullsized":"2024-08-17T17:22:56.415656+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:08.218709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:08.218709+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:19.097715+0000","scrub_duration":0.002239024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":965,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.396705+0000","last_change":"2024-08-17T17:21:06.433720+0000","last_active":"2024-08-17T17:23:10.396705+0000","last_peered":"2024-08-17T17:23:10.396705+0000","last_clean":"2024-08-17T17:23:10.396705+0000","last_became_active":"2024-08-17T17:21:06.433292+0000","last_became_peered":"2024-08-17T17:21:06.433292+0000","last_unstale":"2024-08-17T17:23:10.396705+0000","last_undegraded":"2024-08-17T17:23:10.396705+0000","last_fullsized":"2024-08-17T17:23:10.396705+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:32.246594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1165,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421209+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:22:56.421209+0000","last_peered":"2024-08-17T17:22:56.421209+0000","last_clean":"2024-08-17T17:22:56.421209+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:22:56.421209+0000","last_undegraded":"2024-08-17T17:22:56.421209+0000","last_fullsized":"2024-08-17T17:22:56.421209+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":757,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414683+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:22:56.414683+0000","last_peered":"2024-08-17T17:22:56.414683+0000","last_clean":"2024-08-17T17:22:56.414683+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:22:56.414683+0000","last_undegraded":"2024-08-17T17:22:56.414683+0000","last_fullsized":"2024-08-17T17:22:56.414683+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"1207'5737","reported_seq":5772,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:20.131329+0000","last_change":"2024-08-17T17:22:10.944050+0000","last_active":"2024-08-17T17:23:20.131329+0000","last_peered":"2024-08-17T17:23:20.131329+0000","last_clean":"2024-08-17T17:23:20.131329+0000","last_became_active":"2024-08-17T17:22:10.943712+0000","last_became_peered":"2024-08-17T17:22:10.943712+0000","last_unstale":"2024-08-17T17:23:20.131329+0000","last_undegraded":"2024-08-17T17:23:20.131329+0000","last_fullsized":"2024-08-17T17:23:20.131329+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5737,"log_dups_size":0,"ondisk_log_size":5737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:01:44.623031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375980032,"num_objects":5737,"num_object_clones":0,"num_object_copies":17211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5737,"num_write_kb":367168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1474,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417831+0000","last_change":"2024-08-17T17:22:06.995885+0000","last_active":"2024-08-17T17:22:56.417831+0000","last_peered":"2024-08-17T17:22:56.417831+0000","last_clean":"2024-08-17T17:22:56.417831+0000","last_became_active":"2024-08-17T17:18:28.526484+0000","last_became_peered":"2024-08-17T17:18:28.526484+0000","last_unstale":"2024-08-17T17:22:56.417831+0000","last_undegraded":"2024-08-17T17:22:56.417831+0000","last_fullsized":"2024-08-17T17:22:56.417831+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:06.995792+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:06.995792+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:06.341398+0000","scrub_duration":0.0016139520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1531,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.389208+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:23:10.389208+0000","last_peered":"2024-08-17T17:23:10.389208+0000","last_clean":"2024-08-17T17:23:10.389208+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:23:10.389208+0000","last_undegraded":"2024-08-17T17:23:10.389208+0000","last_fullsized":"2024-08-17T17:23:10.389208+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":807,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415837+0000","last_change":"2024-08-17T17:21:06.452395+0000","last_active":"2024-08-17T17:22:56.415837+0000","last_peered":"2024-08-17T17:22:56.415837+0000","last_clean":"2024-08-17T17:22:56.415837+0000","last_became_active":"2024-08-17T17:21:06.451867+0000","last_became_peered":"2024-08-17T17:21:06.451867+0000","last_unstale":"2024-08-17T17:22:56.415837+0000","last_undegraded":"2024-08-17T17:22:56.415837+0000","last_fullsized":"2024-08-17T17:22:56.415837+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:50.597092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1520,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.385140+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:23:10.385140+0000","last_peered":"2024-08-17T17:23:10.385140+0000","last_clean":"2024-08-17T17:23:10.385140+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:23:10.385140+0000","last_undegraded":"2024-08-17T17:23:10.385140+0000","last_fullsized":"2024-08-17T17:23:10.385140+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"1207'5787","reported_seq":5808,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:22.311937+0000","last_change":"2024-08-17T17:22:10.859993+0000","last_active":"2024-08-17T17:23:22.311937+0000","last_peered":"2024-08-17T17:23:22.311937+0000","last_clean":"2024-08-17T17:23:22.311937+0000","last_became_active":"2024-08-17T17:22:10.859296+0000","last_became_peered":"2024-08-17T17:22:10.859296+0000","last_unstale":"2024-08-17T17:23:22.311937+0000","last_undegraded":"2024-08-17T17:23:22.311937+0000","last_fullsized":"2024-08-17T17:23:22.311937+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":5787,"log_dups_size":0,"ondisk_log_size":5787,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:42.095858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379256832,"num_objects":5787,"num_object_clones":0,"num_object_copies":17361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5787,"num_write_kb":370368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1536,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.387354+0000","last_change":"2024-08-17T17:18:29.390118+0000","last_active":"2024-08-17T17:23:10.387354+0000","last_peered":"2024-08-17T17:23:10.387354+0000","last_clean":"2024-08-17T17:23:10.387354+0000","last_became_active":"2024-08-17T17:18:28.528874+0000","last_became_peered":"2024-08-17T17:18:28.528874+0000","last_unstale":"2024-08-17T17:23:10.387354+0000","last_undegraded":"2024-08-17T17:23:10.387354+0000","last_fullsized":"2024-08-17T17:23:10.387354+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:29.390003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:29.390003+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:35.908604+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":176,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417576+0000","last_change":"2024-08-17T17:21:06.519965+0000","last_active":"2024-08-17T17:22:56.417576+0000","last_peered":"2024-08-17T17:22:56.417576+0000","last_clean":"2024-08-17T17:22:56.417576+0000","last_became_active":"2024-08-17T17:21:06.459952+0000","last_became_peered":"2024-08-17T17:21:06.459952+0000","last_unstale":"2024-08-17T17:22:56.417576+0000","last_undegraded":"2024-08-17T17:22:56.417576+0000","last_fullsized":"2024-08-17T17:22:56.417576+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:27.828098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1162,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415920+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:22:56.415920+0000","last_peered":"2024-08-17T17:22:56.415920+0000","last_clean":"2024-08-17T17:22:56.415920+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:22:56.415920+0000","last_undegraded":"2024-08-17T17:22:56.415920+0000","last_fullsized":"2024-08-17T17:22:56.415920+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1138,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417186+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:22:56.417186+0000","last_peered":"2024-08-17T17:22:56.417186+0000","last_clean":"2024-08-17T17:22:56.417186+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:22:56.417186+0000","last_undegraded":"2024-08-17T17:22:56.417186+0000","last_fullsized":"2024-08-17T17:22:56.417186+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1169,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417637+0000","last_change":"2024-08-17T17:21:06.458000+0000","last_active":"2024-08-17T17:22:56.417637+0000","last_peered":"2024-08-17T17:22:56.417637+0000","last_clean":"2024-08-17T17:22:56.417637+0000","last_became_active":"2024-08-17T17:21:06.457411+0000","last_became_peered":"2024-08-17T17:21:06.457411+0000","last_unstale":"2024-08-17T17:22:56.417637+0000","last_undegraded":"2024-08-17T17:22:56.417637+0000","last_fullsized":"2024-08-17T17:22:56.417637+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:52.740639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1160,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.418917+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:22:56.418917+0000","last_peered":"2024-08-17T17:22:56.418917+0000","last_clean":"2024-08-17T17:22:56.418917+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:22:56.418917+0000","last_undegraded":"2024-08-17T17:22:56.418917+0000","last_fullsized":"2024-08-17T17:22:56.418917+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":830,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.385351+0000","last_change":"2024-08-17T17:21:06.454587+0000","last_active":"2024-08-17T17:23:10.385351+0000","last_peered":"2024-08-17T17:23:10.385351+0000","last_clean":"2024-08-17T17:23:10.385351+0000","last_became_active":"2024-08-17T17:21:06.454229+0000","last_became_peered":"2024-08-17T17:21:06.454229+0000","last_unstale":"2024-08-17T17:23:10.385351+0000","last_undegraded":"2024-08-17T17:23:10.385351+0000","last_fullsized":"2024-08-17T17:23:10.385351+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:38.340263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1136,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.421025+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:22:56.421025+0000","last_peered":"2024-08-17T17:22:56.421025+0000","last_clean":"2024-08-17T17:22:56.421025+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:22:56.421025+0000","last_undegraded":"2024-08-17T17:22:56.421025+0000","last_fullsized":"2024-08-17T17:22:56.421025+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1166,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415505+0000","last_change":"2024-08-17T17:21:06.442796+0000","last_active":"2024-08-17T17:22:56.415505+0000","last_peered":"2024-08-17T17:22:56.415505+0000","last_clean":"2024-08-17T17:22:56.415505+0000","last_became_active":"2024-08-17T17:21:06.442466+0000","last_became_peered":"2024-08-17T17:21:06.442466+0000","last_unstale":"2024-08-17T17:22:56.415505+0000","last_undegraded":"2024-08-17T17:22:56.415505+0000","last_fullsized":"2024-08-17T17:22:56.415505+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:36.366620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1476,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.385177+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:23:10.385177+0000","last_peered":"2024-08-17T17:23:10.385177+0000","last_clean":"2024-08-17T17:23:10.385177+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:23:10.385177+0000","last_undegraded":"2024-08-17T17:23:10.385177+0000","last_fullsized":"2024-08-17T17:23:10.385177+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1002,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415285+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:22:56.415285+0000","last_peered":"2024-08-17T17:22:56.415285+0000","last_clean":"2024-08-17T17:22:56.415285+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:22:56.415285+0000","last_undegraded":"2024-08-17T17:22:56.415285+0000","last_fullsized":"2024-08-17T17:22:56.415285+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1509,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.385054+0000","last_change":"2024-08-17T17:21:06.468788+0000","last_active":"2024-08-17T17:23:10.385054+0000","last_peered":"2024-08-17T17:23:10.385054+0000","last_clean":"2024-08-17T17:23:10.385054+0000","last_became_active":"2024-08-17T17:21:06.467141+0000","last_became_peered":"2024-08-17T17:21:06.467141+0000","last_unstale":"2024-08-17T17:23:10.385054+0000","last_undegraded":"2024-08-17T17:23:10.385054+0000","last_fullsized":"2024-08-17T17:23:10.385054+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:51.301359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1168,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415873+0000","last_change":"2024-08-17T17:21:06.435998+0000","last_active":"2024-08-17T17:22:56.415873+0000","last_peered":"2024-08-17T17:22:56.415873+0000","last_clean":"2024-08-17T17:22:56.415873+0000","last_became_active":"2024-08-17T17:21:06.435579+0000","last_became_peered":"2024-08-17T17:21:06.435579+0000","last_unstale":"2024-08-17T17:22:56.415873+0000","last_undegraded":"2024-08-17T17:22:56.415873+0000","last_fullsized":"2024-08-17T17:22:56.415873+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:01.743074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1539,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.385223+0000","last_change":"2024-08-17T17:21:06.473627+0000","last_active":"2024-08-17T17:23:10.385223+0000","last_peered":"2024-08-17T17:23:10.385223+0000","last_clean":"2024-08-17T17:23:10.385223+0000","last_became_active":"2024-08-17T17:21:06.473062+0000","last_became_peered":"2024-08-17T17:21:06.473062+0000","last_unstale":"2024-08-17T17:23:10.385223+0000","last_undegraded":"2024-08-17T17:23:10.385223+0000","last_fullsized":"2024-08-17T17:23:10.385223+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:21.413163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":963,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:23:07.315445+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:23:07.315445+0000","last_peered":"2024-08-17T17:23:07.315445+0000","last_clean":"2024-08-17T17:23:07.315445+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:23:07.315445+0000","last_undegraded":"2024-08-17T17:23:07.315445+0000","last_fullsized":"2024-08-17T17:23:07.315445+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1030,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.385366+0000","last_change":"2024-08-17T17:21:06.471265+0000","last_active":"2024-08-17T17:23:10.385366+0000","last_peered":"2024-08-17T17:23:10.385366+0000","last_clean":"2024-08-17T17:23:10.385366+0000","last_became_active":"2024-08-17T17:21:06.470943+0000","last_became_peered":"2024-08-17T17:21:06.470943+0000","last_unstale":"2024-08-17T17:23:10.385366+0000","last_undegraded":"2024-08-17T17:23:10.385366+0000","last_fullsized":"2024-08-17T17:23:10.385366+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:41.070024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1516,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.386844+0000","last_change":"2024-08-17T17:22:48.377351+0000","last_active":"2024-08-17T17:23:10.386844+0000","last_peered":"2024-08-17T17:23:10.386844+0000","last_clean":"2024-08-17T17:23:10.386844+0000","last_became_active":"2024-08-17T17:22:48.377059+0000","last_became_peered":"2024-08-17T17:22:48.377059+0000","last_unstale":"2024-08-17T17:23:10.386844+0000","last_undegraded":"2024-08-17T17:23:10.386844+0000","last_fullsized":"2024-08-17T17:23:10.386844+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:15.250902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1185,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417912+0000","last_change":"2024-08-17T17:21:06.459139+0000","last_active":"2024-08-17T17:22:56.417912+0000","last_peered":"2024-08-17T17:22:56.417912+0000","last_clean":"2024-08-17T17:22:56.417912+0000","last_became_active":"2024-08-17T17:21:06.457850+0000","last_became_peered":"2024-08-17T17:21:06.457850+0000","last_unstale":"2024-08-17T17:22:56.417912+0000","last_undegraded":"2024-08-17T17:22:56.417912+0000","last_fullsized":"2024-08-17T17:22:56.417912+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:13.888576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1557,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.396718+0000","last_change":"2024-08-17T17:21:06.446321+0000","last_active":"2024-08-17T17:23:10.396718+0000","last_peered":"2024-08-17T17:23:10.396718+0000","last_clean":"2024-08-17T17:23:10.396718+0000","last_became_active":"2024-08-17T17:21:06.446019+0000","last_became_peered":"2024-08-17T17:21:06.446019+0000","last_unstale":"2024-08-17T17:23:10.396718+0000","last_undegraded":"2024-08-17T17:23:10.396718+0000","last_fullsized":"2024-08-17T17:23:10.396718+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:26.750070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":155,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417421+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:22:56.417421+0000","last_peered":"2024-08-17T17:22:56.417421+0000","last_clean":"2024-08-17T17:22:56.417421+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:22:56.417421+0000","last_undegraded":"2024-08-17T17:22:56.417421+0000","last_fullsized":"2024-08-17T17:22:56.417421+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1183,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417563+0000","last_change":"2024-08-17T17:21:06.465804+0000","last_active":"2024-08-17T17:22:56.417563+0000","last_peered":"2024-08-17T17:22:56.417563+0000","last_clean":"2024-08-17T17:22:56.417563+0000","last_became_active":"2024-08-17T17:21:06.465461+0000","last_became_peered":"2024-08-17T17:21:06.465461+0000","last_unstale":"2024-08-17T17:22:56.417563+0000","last_undegraded":"2024-08-17T17:22:56.417563+0000","last_fullsized":"2024-08-17T17:22:56.417563+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:06.443667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1175,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.414946+0000","last_change":"2024-08-17T17:21:06.481634+0000","last_active":"2024-08-17T17:22:56.414946+0000","last_peered":"2024-08-17T17:22:56.414946+0000","last_clean":"2024-08-17T17:22:56.414946+0000","last_became_active":"2024-08-17T17:21:06.469081+0000","last_became_peered":"2024-08-17T17:21:06.469081+0000","last_unstale":"2024-08-17T17:22:56.414946+0000","last_undegraded":"2024-08-17T17:22:56.414946+0000","last_fullsized":"2024-08-17T17:22:56.414946+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:53:26.954738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1183,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417419+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:22:56.417419+0000","last_peered":"2024-08-17T17:22:56.417419+0000","last_clean":"2024-08-17T17:22:56.417419+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:22:56.417419+0000","last_undegraded":"2024-08-17T17:22:56.417419+0000","last_fullsized":"2024-08-17T17:22:56.417419+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1134,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415572+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:22:56.415572+0000","last_peered":"2024-08-17T17:22:56.415572+0000","last_clean":"2024-08-17T17:22:56.415572+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:22:56.415572+0000","last_undegraded":"2024-08-17T17:22:56.415572+0000","last_fullsized":"2024-08-17T17:22:56.415572+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1515,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.385499+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:23:10.385499+0000","last_peered":"2024-08-17T17:23:10.385499+0000","last_clean":"2024-08-17T17:23:10.385499+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:23:10.385499+0000","last_undegraded":"2024-08-17T17:23:10.385499+0000","last_fullsized":"2024-08-17T17:23:10.385499+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1490,"reported_epoch":1207,"state":"active+clean","last_fresh":"2024-08-17T17:23:10.386746+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:23:10.386746+0000","last_peered":"2024-08-17T17:23:10.386746+0000","last_clean":"2024-08-17T17:23:10.386746+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:23:10.386746+0000","last_undegraded":"2024-08-17T17:23:10.386746+0000","last_fullsized":"2024-08-17T17:23:10.386746+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1169,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.417153+0000","last_change":"2024-08-17T17:21:06.483469+0000","last_active":"2024-08-17T17:22:56.417153+0000","last_peered":"2024-08-17T17:22:56.417153+0000","last_clean":"2024-08-17T17:22:56.417153+0000","last_became_active":"2024-08-17T17:21:06.483162+0000","last_became_peered":"2024-08-17T17:21:06.483162+0000","last_unstale":"2024-08-17T17:22:56.417153+0000","last_undegraded":"2024-08-17T17:22:56.417153+0000","last_fullsized":"2024-08-17T17:22:56.417153+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:24.898353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":177,"reported_epoch":1206,"state":"active+clean","last_fresh":"2024-08-17T17:22:56.415191+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:22:56.415191+0000","last_peered":"2024-08-17T17:22:56.415191+0000","last_clean":"2024-08-17T17:22:56.415191+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:22:56.415191+0000","last_undegraded":"2024-08-17T17:22:56.415191+0000","last_fullsized":"2024-08-17T17:22:56.415191+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":6046810112,"num_objects":92267,"num_object_clones":0,"num_object_copies":276801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":92267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":92267,"num_write_kb":5905088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18253938688,"data_stored":18253938688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92267,"ondisk_log_size":92267,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201663,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2257160,"kb_used_data":1875008,"kb_used_omap":1,"kb_used_meta":382142,"kb_avail":91487992,"statfs":{"total":95995035648,"available":93683703808,"internally_reserved":0,"allocated":1920008192,"data_stored":1917615948,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":391313867},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463300,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2189952,"kb_used_data":1870136,"kb_used_omap":1,"kb_used_meta":319806,"kb_avail":91555200,"statfs":{"total":95995035648,"available":93752524800,"internally_reserved":0,"allocated":1915019264,"data_stored":1912635085,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":327481801},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626941,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2717044,"kb_used_data":2179564,"kb_used_omap":42,"kb_used_meta":537429,"kb_avail":91028108,"statfs":{"total":95995035648,"available":93212782592,"internally_reserved":0,"allocated":2231873536,"data_stored":2229497446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":550327356},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785419,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2979056,"kb_used_data":2626088,"kb_used_omap":48,"kb_used_meta":352911,"kb_avail":90766096,"statfs":{"total":95995035648,"available":92944482304,"internally_reserved":0,"allocated":2689114112,"data_stored":2686741503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":361381550},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724936,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3331196,"kb_used_data":2975284,"kb_used_omap":1,"kb_used_meta":355902,"kb_avail":90413956,"statfs":{"total":95995035648,"available":92583890944,"internally_reserved":0,"allocated":3046690816,"data_stored":3044307103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":364444102},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986573,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2241284,"kb_used_data":1871420,"kb_used_omap":1,"kb_used_meta":369854,"kb_avail":91503868,"statfs":{"total":95995035648,"available":93699960832,"internally_reserved":0,"allocated":1916334080,"data_stored":1913945862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":378730950},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706180,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2625400,"kb_used_data":2206064,"kb_used_omap":35,"kb_used_meta":419292,"kb_avail":91119752,"statfs":{"total":95995035648,"available":93306626048,"internally_reserved":0,"allocated":2259009536,"data_stored":2256629876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":429355063},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831953,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2729972,"kb_used_data":2219948,"kb_used_omap":65,"kb_used_meta":509950,"kb_avail":91015180,"statfs":{"total":95995035648,"available":93199544320,"internally_reserved":0,"allocated":2273226752,"data_stored":2270850632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":522188987},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2274230272,"data_stored":2274230272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2696806400,"data_stored":2696806400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2260795392,"data_stored":2260795392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2226061312,"data_stored":2226061312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1914503168,"data_stored":1914503168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3057844224,"data_stored":3057844224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1910046720,"data_stored":1910046720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1913651200,"data_stored":1913651200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:23:27.629 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_3 pgp_num 16 2024-08-17T17:23:27.631 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:27.705 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:27.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:27.806 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:27.835 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:27.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:27.936 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:27 smithi063 ceph-mon[109843]: pgmap v1553: 99 pgs: 99 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-08-17T17:23:27.936 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:27 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:23:27.936 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:27 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:23:27.936 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:27 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:27.936 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:27 smithi063 ceph-mon[109843]: osdmap e1208: 8 total, 8 up, 8 in 2024-08-17T17:23:27.936 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:27.937 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:27 smithi063 ceph-mon[114756]: pgmap v1553: 99 pgs: 99 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-08-17T17:23:27.937 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:27 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:23:27.937 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:27 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:23:27.937 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:27 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:27.937 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:27 smithi063 ceph-mon[114756]: osdmap e1208: 8 total, 8 up, 8 in 2024-08-17T17:23:27.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:27.986 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:27.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:28.087 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:27 smithi162 ceph-mon[104668]: pgmap v1553: 99 pgs: 99 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-08-17T17:23:28.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:27 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:23:28.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:27 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:23:28.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:27 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:28.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:27 smithi162 ceph-mon[104668]: osdmap e1208: 8 total, 8 up, 8 in 2024-08-17T17:23:28.116 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:28.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:28.217 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:28.250 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:28.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:28.328 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:28.351 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:28.377 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:28.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:28.479 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:28.506 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 61 16 99572 99556 101.982 66.5625 0.0132044 0.00980005 2024-08-17T17:23:28.531 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:28.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:28.633 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:28.665 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:28.698 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:28.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:28.800 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:28.852 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:28.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:28.953 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:28.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:28.991 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:28.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:29.092 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:29.118 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:29.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:29.220 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:29.248 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:29.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:29.350 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:29.374 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:29.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:29.476 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:29.506 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 62 16 100615 100599 101.389 65.1875 0.0187085 0.00985762 2024-08-17T17:23:29.508 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:29.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:29.610 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:29.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.653 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:29.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:29.754 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:29.783 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:29.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:29.885 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:29.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:29.921 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:29.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:30.022 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:29 smithi063 ceph-mon[114756]: pgmap v1555: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T17:23:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:29 smithi063 ceph-mon[109843]: pgmap v1555: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T17:23:30.044 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:30.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:29 smithi162 ceph-mon[104668]: pgmap v1555: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T17:23:30.146 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:30.186 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:30.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:30.288 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:30.314 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:23:30.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.319 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:30.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:30.335 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:30.420 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:30.452 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:30.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:30.506 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 63 16 101614 101598 100.77 62.4375 0.0116185 0.0099177 2024-08-17T17:23:30.554 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:30.582 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:30.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:30.683 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:30.718 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:30.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:30.820 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:30.855 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:30.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:30.956 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:30.991 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:30.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:30.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:31.093 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:31.136 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:31.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:31.238 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:31.267 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:31.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:31.369 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:31.401 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:31.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:31.503 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:31.507 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 64 16 102698 102682 100.254 67.75 0.0253349 0.00996924 2024-08-17T17:23:31.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.534 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:31.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:31.635 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:31.667 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:31.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:31.769 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:31.810 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:31.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:31.911 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:31.943 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:31.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:31.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:31.997 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:31 smithi063 ceph-mon[114756]: pgmap v1556: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T17:23:31.997 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:31.998 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:31.998 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:31 smithi063 ceph-mon[109843]: pgmap v1556: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T17:23:31.998 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:31.998 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:32.045 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:32.085 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:32.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:31 smithi162 ceph-mon[104668]: pgmap v1556: 99 pgs: 99 active+clean; 6.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T17:23:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:32.186 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:32.218 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:32.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:32.319 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:32.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.354 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:32.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:32.455 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:32.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.488 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:32.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:32.507 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 65 16 103769 103753 99.7416 66.9375 0.0130712 0.0100208 2024-08-17T17:23:32.588 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:32.619 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:32.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:32.721 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:32.768 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:32.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:32.870 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:32.907 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:32.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:32.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:33.009 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:33.039 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:33.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:33.141 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:33.176 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:33.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:33.278 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:33.316 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:33.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:33.418 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:33.456 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:33.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:33.506 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 66 16 104824 104808 99.2294 65.9375 0.0144689 0.0100727 2024-08-17T17:23:33.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:23:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:23:33] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.51.0" 2024-08-17T17:23:33.558 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:33.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.608 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:33.608 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:33.709 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:33.755 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:33.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:33.810 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:33 smithi063 ceph-mon[114756]: pgmap v1557: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:23:33.810 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:33 smithi063 ceph-mon[109843]: pgmap v1557: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:23:33.857 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:33.898 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:33.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:33.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:34.000 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:34.009 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:23:34.009 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:23:34.049 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:23:34.049 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:23:34.052 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:34.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:34.092 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:23:34.092 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:23:34.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:33 smithi162 ceph-mon[104668]: pgmap v1557: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:23:34.141 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:23:34.141 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:23:34.154 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:34.164 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:34.203 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:23:34.204 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:23:34.207 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:34.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:34.245 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:23:34.245 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:23:34.286 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:23:34.286 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:23:34.309 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:34.322 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:23:34.322 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:23:34.338 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:34.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:34.440 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:34.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.472 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:34.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:34.507 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 67 16 105746 105730 98.6084 57.625 0.0134462 0.0101356 2024-08-17T17:23:34.573 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:34.609 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:34.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:34.711 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:34.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.757 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:34.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:34.784 INFO:teuthology.orchestra.run.smithi063.stderr:set pool 8 pgp_num to 16 2024-08-17T17:23:34.858 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:34.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:34.891 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:34.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:34.992 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:34 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1817082979' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:34 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1963989019' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:23:35.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:34 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:23:35.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:34 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1817082979' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:35.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:34 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1963989019' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:23:35.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:34 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:23:35.032 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:35.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:34 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1817082979' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:34 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1963989019' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:23:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:34 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:23:35.133 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:35.176 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:35.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:35.278 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:35.319 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:35.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:35.421 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:35.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.462 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:35.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:35.507 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 68 16 106757 106741 98.0875 63.1875 0.0170936 0.0101898 2024-08-17T17:23:35.562 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:35.592 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:35.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:35.694 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:35.728 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:35.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:35.830 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:35.875 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:35.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:35.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:35.977 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:36.014 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:36.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:35 smithi063 ceph-mon[109843]: pgmap v1558: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:23:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:35 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2024-08-17T17:23:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:35 smithi063 ceph-mon[109843]: osdmap e1209: 8 total, 8 up, 8 in 2024-08-17T17:23:36.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:35 smithi063 ceph-mon[114756]: pgmap v1558: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:23:36.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:35 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2024-08-17T17:23:36.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:35 smithi063 ceph-mon[114756]: osdmap e1209: 8 total, 8 up, 8 in 2024-08-17T17:23:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:35 smithi162 ceph-mon[104668]: pgmap v1558: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:23:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:35 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2024-08-17T17:23:36.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:35 smithi162 ceph-mon[104668]: osdmap e1209: 8 total, 8 up, 8 in 2024-08-17T17:23:36.116 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:36.161 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:36.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:36.263 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:36.290 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:36.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:36.391 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:36.426 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:36.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:36.507 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 69 16 107771 107755 97.5844 63.375 0.0176258 0.0102424 2024-08-17T17:23:36.527 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:36.564 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:36.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:36.666 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:36.694 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:36.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:36.795 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:36.829 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:36.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:36.931 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:36.964 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:36.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:36.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:37.065 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:37.113 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:37.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:37.216 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:37.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.259 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:37.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:37.360 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:37.397 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:37.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:37.499 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:37.507 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 70 16 108792 108776 97.1013 63.8125 0.00678811 0.0102929 2024-08-17T17:23:37.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.552 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:37.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:37.562 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:23:37.653 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:37.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.697 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:37.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:37.798 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:37.828 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:37.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:37.930 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:37.989 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:37.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:37.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:37 smithi063 ceph-mon[109843]: pgmap v1560: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2024-08-17T17:23:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:37 smithi063 ceph-mon[114756]: pgmap v1560: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2024-08-17T17:23:38.092 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:37 smithi162 ceph-mon[104668]: pgmap v1560: 99 pgs: 99 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2024-08-17T17:23:38.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.134 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:38.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:38.235 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:38.282 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:38.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:38.384 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:38.421 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:38.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.423 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:38.509 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 71 16 109735 109719 96.5623 58.9375 0.0140814 0.0103505 2024-08-17T17:23:38.523 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:38.550 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:38.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:38.652 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:38.698 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:38.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:38.708 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:23:38.708 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:23:38.800 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:38.835 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:38.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:38.935 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:38 smithi063 ceph-mon[109843]: pgmap v1561: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:23:38.937 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:38.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:38.993 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:38.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:39.094 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:39.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.130 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:39.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:39.231 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:39.265 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:39.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:39.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:38 smithi063 ceph-mon[114756]: pgmap v1561: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:23:39.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:38 smithi162 ceph-mon[104668]: pgmap v1561: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:23:39.367 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:39.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.396 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:39.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:39.496 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:39.510 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 72 16 110740 110724 96.0923 62.8125 0.0155205 0.010402 2024-08-17T17:23:39.542 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:39.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:39.644 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:39.684 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:39.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:39.786 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:39.821 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:23:39.845 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:39.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:39.948 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:39.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:39.988 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:39.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:40.089 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:40.150 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:40.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:40.155 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:40.252 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:40.292 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:40.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:40.394 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:40.432 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:40.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:40.510 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 73 16 111711 111695 95.6073 60.6875 0.00870647 0.0104542 2024-08-17T17:23:40.534 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:40.556 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:40.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:40.658 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:40.694 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:40.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:40.796 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:40.828 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:40.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:40.930 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:40.964 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:40.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:40.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:41.066 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:41.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.099 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:41.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:41.200 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:41.243 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:41.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:41.345 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:41.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.384 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:41.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:41.485 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:41.510 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 74 16 112449 112433 94.9386 46.125 0.018066 0.010528 2024-08-17T17:23:41.511 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:41.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:41.608 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:41 smithi063 ceph-mon[114756]: pgmap v1562: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:23:41.609 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:41 smithi063 ceph-mon[109843]: pgmap v1562: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:23:41.613 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:41.636 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:41.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:41.738 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:41.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.781 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:41.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:41.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:41 smithi162 ceph-mon[104668]: pgmap v1562: 99 pgs: 99 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:23:41.882 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:41.906 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:41.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:41.925 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:41.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:42.026 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:42.071 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:42.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:42.088 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:23:42.088 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:23:42.089 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-08-17T17:23:42.089 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:23:42.089 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:23:42.089 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-08-17T17:23:42.173 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:42.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.220 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:42.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:42.321 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:42.359 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:23:42.390 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:42.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:42.492 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:42.510 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 75 16 113171 113155 94.2737 45.125 0.132226 0.0106017 2024-08-17T17:23:42.526 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:42.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:42.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:42 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3171779850' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:42.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:42 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:42.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:42 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3171779850' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:42.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:42 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:42.628 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:42.659 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:42.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:42.761 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:42.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.783 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:42.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:42.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:42 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3171779850' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:42.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:42 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:23:42.884 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:42.917 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:42.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:42.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:43.019 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:43.058 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:43.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:43.160 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:43.207 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:43.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:43.309 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:43.347 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:43.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:43.449 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:43.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.470 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:43.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:43.511 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 76 16 114096 114080 93.794 57.8125 0.0170879 0.010656 2024-08-17T17:23:43.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:23:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:23:43] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.51.0" 2024-08-17T17:23:43.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:43 smithi063 ceph-mon[114756]: pgmap v1563: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-17T17:23:43.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:43 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:43.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:43 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:23:43.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:43 smithi063 ceph-mon[114756]: osdmap e1210: 8 total, 8 up, 8 in 2024-08-17T17:23:43.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:43 smithi063 ceph-mon[109843]: pgmap v1563: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-17T17:23:43.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:43 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:43.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:43 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:23:43.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:43 smithi063 ceph-mon[109843]: osdmap e1210: 8 total, 8 up, 8 in 2024-08-17T17:23:43.572 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:43.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:43 smithi162 ceph-mon[104668]: pgmap v1563: 99 pgs: 99 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2024-08-17T17:23:43.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:43 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:43.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:43 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:23:43.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:43 smithi162 ceph-mon[104668]: osdmap e1210: 8 total, 8 up, 8 in 2024-08-17T17:23:43.596 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:43.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:43.698 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:43.741 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:43.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:43.843 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:43.872 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:43.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:43.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:43.973 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:44.000 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:44.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:44.101 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:44.145 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:44.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:44.247 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:44.279 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:44.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:44.381 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:44.418 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:44.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:44.511 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 77 16 114950 114934 93.269 53.375 0.015621 0.0107161 2024-08-17T17:23:44.520 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:44.560 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:44.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:44.661 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:44.704 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:44.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:44.805 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:44.857 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:44.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:44.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:44.959 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:44.999 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:45.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:45.101 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:45.136 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:45.238 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:45.276 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:45.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:45.293 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:23:45.378 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:45.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.407 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:45.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:45.493 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:45 smithi162 ceph-mon[104668]: pgmap v1565: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:23:45.493 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:45.493 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:45 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:45.508 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:45.514 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 78 16 115864 115848 92.8047 57.125 0.0182572 0.0107696 2024-08-17T17:23:45.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.577 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:45.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:45.608 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:45 smithi063 ceph-mon[114756]: pgmap v1565: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:23:45.608 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:45.608 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:45 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:45.609 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:45 smithi063 ceph-mon[109843]: pgmap v1565: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:23:45.609 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:45.609 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:45 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:45.626 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:23:45.626 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:23:45.678 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:45.727 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:45.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:45.741 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:45.818 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:23:45.819 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:23:45.829 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:45.868 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:23:45.868 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:23:45.875 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:45.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:45.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:45.910 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:23:45.911 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:23:45.954 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:23:45.954 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:23:45.977 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:45.989 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:23:45.989 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:23:46.002 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:46.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:46.029 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:23:46.029 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:23:46.066 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:23:46.066 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:23:46.104 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:46.118 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:23:46.118 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:23:46.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.143 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:46.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:46.244 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:46.285 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:46.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:46.311 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:23:46.387 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:46.423 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:46.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:46.512 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 79 16 116652 116636 92.2535 49.25 0.0127346 0.0108346 2024-08-17T17:23:46.525 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:46.575 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:46.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:46.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:46 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3896045375' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:46.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:23:46.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:23:46.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:46.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:46.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:46.678 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:46.683 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:46.728 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:46.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:46.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:46 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3896045375' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:46.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:23:46.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:23:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:46 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3896045375' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:23:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:23:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:23:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:23:46.835 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:46.869 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:46.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:46.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:46.971 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:47.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.003 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:47.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:47.104 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:47.130 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:47.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:47.232 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:47.274 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:47.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:47.376 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:47.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.422 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:47.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:47.512 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:23:47.512024+0000 min lat: 0.00109066 max lat: 0.153833 avg lat: 0.0108806 2024-08-17T17:23:47.512 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:23:47.512 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 80 16 117600 117584 91.8409 59.25 0.00843514 0.0108806 2024-08-17T17:23:47.522 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:47.556 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:47.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:47.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:47 smithi162 ceph-mon[104668]: pgmap v1566: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2024-08-17T17:23:47.659 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:47.690 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:47.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:47.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:47 smithi063 ceph-mon[114756]: pgmap v1566: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2024-08-17T17:23:47.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:47 smithi063 ceph-mon[109843]: pgmap v1566: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2024-08-17T17:23:47.792 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:47.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:47.929 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:47.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:48.024 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:48.030 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:48.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.092 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:48.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:48.193 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:48.235 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:48.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:48.337 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:48.368 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:48.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:48.469 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:48.507 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:48.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:48.512 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 81 16 118575 118559 91.4594 60.9375 0.0157729 0.0109287 2024-08-17T17:23:48.609 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:48.647 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:48.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:48.749 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:48.780 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:48.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:48.881 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:48.921 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:48.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:48.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:49.023 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:49.048 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:49.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:49.150 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:49.188 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:49.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:49.290 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:49.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.328 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:49.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:49.353 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:23:49.354 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:23:49.354 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:23:49.354 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:23:49.354 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:23:49.429 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:49.465 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:49.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:49.512 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 82 16 119513 119497 91.0589 58.625 0.024813 0.0109773 2024-08-17T17:23:49.568 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:49.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:49 smithi162 ceph-mon[104668]: pgmap v1567: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2024-08-17T17:23:49.596 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:49.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:49.688 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:49 smithi063 ceph-mon[109843]: pgmap v1567: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2024-08-17T17:23:49.690 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:49 smithi063 ceph-mon[114756]: pgmap v1567: 99 pgs: 99 active+clean; 7.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2024-08-17T17:23:49.698 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:49.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.734 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:49.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:49.834 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:49.869 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:49.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:49.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:49.970 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:50.007 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:50.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:50.109 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:50.150 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:50.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:50.192 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:23:50.251 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:50.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.288 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:50.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:50.389 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:50.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.444 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:50.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:50.512 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 83 16 120458 120442 90.6734 59.0625 0.021988 0.0110231 2024-08-17T17:23:50.543 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:50.545 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:50.585 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:50.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:50.685 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:23:50.687 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:50.722 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:50.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:50.824 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:50.854 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:50.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:50.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:50.956 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:51.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.007 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:51.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:51.076 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:23:51.076 INFO:teuthology.orchestra.run.smithi063.stdout: "success": "" 2024-08-17T17:23:51.077 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:23:51.108 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:51.157 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:51.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:51.259 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:51.298 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:51.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:51.400 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:51.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.443 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:51.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:51.512 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 84 16 121342 121326 90.2517 55.25 0.0353778 0.0110747 2024-08-17T17:23:51.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:51 smithi063 ceph-mon[114756]: pgmap v1568: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2024-08-17T17:23:51.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:51 smithi063 ceph-mon[109843]: pgmap v1568: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2024-08-17T17:23:51.544 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:51.574 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:51.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:51.676 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:51.707 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:51.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:51.809 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:51.824 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:51 smithi162 ceph-mon[104668]: pgmap v1568: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 55 MiB/s wr, 884 op/s 2024-08-17T17:23:51.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.847 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:51.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:51.948 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:51.986 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:51.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:51.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:52.088 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:52.128 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:52.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:52.230 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:52.276 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:52.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:52.331 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:52.378 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:52.436 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:52.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:52.470 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:23:52.471 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:23:52.471 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:23:52.471 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:23:52.471 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.471 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:34054 7.9 7:982e3efb:::benchmark_data_smithi063_196002_object34053:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1133)", 2024-08-17T17:23:52.471 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:11.456233+0000", 2024-08-17T17:23:52.471 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 341.01078277300002, 2024-08-17T17:23:52.471 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.179724351, 2024-08-17T17:23:52.471 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.471 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.471 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.472 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.472 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.472 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 34054 2024-08-17T17:23:52.472 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.472 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.472 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.472 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.472 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.456233+0000", 2024-08-17T17:23:52.472 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.472 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.472 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.472 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.472 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.456233+0000", 2024-08-17T17:23:52.472 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.473 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.473 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.473 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.473 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.456233+0000", 2024-08-17T17:23:52.473 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:23:52.473 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.473 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.473 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.473 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.456262+0000", 2024-08-17T17:23:52.473 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9451999999999999e-05 2024-08-17T17:23:52.473 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.473 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.473 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.473 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.456273+0000", 2024-08-17T17:23:52.474 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0594999999999999e-05 2024-08-17T17:23:52.474 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.474 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.474 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.474 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.456280+0000", 2024-08-17T17:23:52.474 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.939e-06 2024-08-17T17:23:52.474 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.474 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.474 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.474 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.460110+0000", 2024-08-17T17:23:52.474 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0038299839999999998 2024-08-17T17:23:52.474 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.474 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.474 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.475 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.460241+0000", 2024-08-17T17:23:52.475 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013128 2024-08-17T17:23:52.475 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.475 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.475 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:23:52.475 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.460367+0000", 2024-08-17T17:23:52.475 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012641900000000001 2024-08-17T17:23:52.475 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.475 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.475 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.475 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.463048+0000", 2024-08-17T17:23:52.475 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026806909999999998 2024-08-17T17:23:52.475 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.475 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.476 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.476 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.463810+0000", 2024-08-17T17:23:52.476 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00076169900000000001 2024-08-17T17:23:52.476 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.476 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.476 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.476 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.635881+0000", 2024-08-17T17:23:52.476 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.17207149699999999 2024-08-17T17:23:52.476 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.476 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.476 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.476 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.635925+0000", 2024-08-17T17:23:52.476 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4042000000000003e-05 2024-08-17T17:23:52.476 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.476 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.477 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.477 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:11.635958+0000", 2024-08-17T17:23:52.477 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2611999999999999e-05 2024-08-17T17:23:52.477 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.477 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.477 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.477 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.477 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.477 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65730 7.9 7:90546b68:::benchmark_data_smithi063_196002_object65729:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-08-17T17:23:52.477 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.520085+0000", 2024-08-17T17:23:52.477 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 321.94693160700001, 2024-08-17T17:23:52.477 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.528465084, 2024-08-17T17:23:52.478 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.478 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:23:52.478 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.478 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.478 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.478 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65730 2024-08-17T17:23:52.478 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.478 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.478 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.478 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.478 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520085+0000", 2024-08-17T17:23:52.478 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.478 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.479 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.479 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.479 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520085+0000", 2024-08-17T17:23:52.479 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.479 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.479 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.479 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.479 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520083+0000", 2024-08-17T17:23:52.479 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:52.479 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.479 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.479 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.479 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520144+0000", 2024-08-17T17:23:52.480 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0470000000000002e-05 2024-08-17T17:23:52.480 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.480 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.480 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.480 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520162+0000", 2024-08-17T17:23:52.480 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.8495000000000002e-05 2024-08-17T17:23:52.480 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.480 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.480 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.480 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520170+0000", 2024-08-17T17:23:52.481 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.5240000000000003e-06 2024-08-17T17:23:52.481 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.481 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.481 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.481 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520241+0000", 2024-08-17T17:23:52.481 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0772000000000006e-05 2024-08-17T17:23:52.481 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.481 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.481 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.481 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520372+0000", 2024-08-17T17:23:52.481 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000131113 2024-08-17T17:23:52.481 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.481 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.481 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:23:52.481 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520505+0000", 2024-08-17T17:23:52.482 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000133363 2024-08-17T17:23:52.482 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.482 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.482 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.482 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535294+0000", 2024-08-17T17:23:52.482 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.014789412 2024-08-17T17:23:52.482 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.482 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.482 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.482 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535306+0000", 2024-08-17T17:23:52.482 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2003e-05 2024-08-17T17:23:52.482 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.482 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.483 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.483 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.535092+0000", 2024-08-17T17:23:52.483 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99978559499999997 2024-08-17T17:23:52.483 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.483 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.483 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.483 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.535104+0000", 2024-08-17T17:23:52.483 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1783999999999999e-05 2024-08-17T17:23:52.483 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.483 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.483 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.483 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.034715+0000", 2024-08-17T17:23:52.483 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.499611566 2024-08-17T17:23:52.483 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.483 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.484 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for degraded object", 2024-08-17T17:23:52.484 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.036087+0000", 2024-08-17T17:23:52.484 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013718249999999999 2024-08-17T17:23:52.484 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.484 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.484 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.484 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.048433+0000", 2024-08-17T17:23:52.484 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012345626 2024-08-17T17:23:52.484 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.484 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.484 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.484 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.048550+0000", 2024-08-17T17:23:52.484 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011689200000000001 2024-08-17T17:23:52.484 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.484 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.485 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.485 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.485 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.485 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65762 7.4 7:25ea59d9:::benchmark_data_smithi063_196002_object65761:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:23:52.486 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.559197+0000", 2024-08-17T17:23:52.486 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 321.90781937200001, 2024-08-17T17:23:52.486 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5033075609999997, 2024-08-17T17:23:52.486 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.486 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.486 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.486 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.486 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.486 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65762 2024-08-17T17:23:52.486 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.486 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.487 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.487 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.487 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559197+0000", 2024-08-17T17:23:52.487 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.487 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.487 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.487 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.487 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559197+0000", 2024-08-17T17:23:52.487 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.487 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.487 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.487 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.487 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559195+0000", 2024-08-17T17:23:52.488 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:52.488 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.488 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.488 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.488 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559275+0000", 2024-08-17T17:23:52.488 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9611999999999996e-05 2024-08-17T17:23:52.488 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.488 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.488 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.488 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559293+0000", 2024-08-17T17:23:52.488 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7776000000000001e-05 2024-08-17T17:23:52.488 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.488 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.488 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.489 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559302+0000", 2024-08-17T17:23:52.489 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2720000000000003e-06 2024-08-17T17:23:52.489 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.489 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.489 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.489 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559362+0000", 2024-08-17T17:23:52.489 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0421000000000003e-05 2024-08-17T17:23:52.489 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.489 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.489 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.489 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559373+0000", 2024-08-17T17:23:52.489 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0739e-05 2024-08-17T17:23:52.489 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.489 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.490 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.490 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.533131+0000", 2024-08-17T17:23:52.490 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.973758291 2024-08-17T17:23:52.490 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.490 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.490 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.490 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.533145+0000", 2024-08-17T17:23:52.490 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3608000000000001e-05 2024-08-17T17:23:52.490 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.490 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.490 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.490 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.035648+0000", 2024-08-17T17:23:52.490 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5025030340000001 2024-08-17T17:23:52.491 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.491 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.491 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.491 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.035818+0000", 2024-08-17T17:23:52.491 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016974000000000001 2024-08-17T17:23:52.491 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.491 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.491 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.491 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.035978+0000", 2024-08-17T17:23:52.491 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015974199999999999 2024-08-17T17:23:52.491 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.491 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.491 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.491 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.048070+0000", 2024-08-17T17:23:52.491 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012092233000000001 2024-08-17T17:23:52.492 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.492 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.492 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.492 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.048159+0000", 2024-08-17T17:23:52.492 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8843000000000003e-05 2024-08-17T17:23:52.492 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.492 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.492 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.492 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.059346+0000", 2024-08-17T17:23:52.492 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011186928 2024-08-17T17:23:52.492 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.492 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.492 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.492 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.059395+0000", 2024-08-17T17:23:52.493 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9570000000000001e-05 2024-08-17T17:23:52.493 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.493 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.493 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.493 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.062504+0000", 2024-08-17T17:23:52.493 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0031092400000000001 2024-08-17T17:23:52.493 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.493 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.493 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.493 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.493 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.493 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65764 7.9 7:95797802:::benchmark_data_smithi063_196002_object65763:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:23:52.493 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.559319+0000", 2024-08-17T17:23:52.493 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 321.90769770200001, 2024-08-17T17:23:52.493 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4950287260000001, 2024-08-17T17:23:52.494 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.494 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.494 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.494 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.494 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.494 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65764 2024-08-17T17:23:52.494 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.494 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.494 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.494 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.494 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559319+0000", 2024-08-17T17:23:52.494 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.494 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.495 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.495 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.495 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559319+0000", 2024-08-17T17:23:52.495 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.495 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.495 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.495 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.495 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559318+0000", 2024-08-17T17:23:52.495 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:23:52.495 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.495 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.495 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.495 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559380+0000", 2024-08-17T17:23:52.495 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1808000000000002e-05 2024-08-17T17:23:52.496 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.496 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.496 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.496 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559386+0000", 2024-08-17T17:23:52.496 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3300000000000004e-06 2024-08-17T17:23:52.496 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.496 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.496 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.496 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559392+0000", 2024-08-17T17:23:52.496 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.789e-06 2024-08-17T17:23:52.496 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.496 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.496 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.496 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559433+0000", 2024-08-17T17:23:52.497 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1162999999999999e-05 2024-08-17T17:23:52.497 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.497 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.497 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.497 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559440+0000", 2024-08-17T17:23:52.497 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4680000000000003e-06 2024-08-17T17:23:52.497 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.497 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.497 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.497 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.535130+0000", 2024-08-17T17:23:52.497 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97568918999999998 2024-08-17T17:23:52.497 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.497 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.498 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.498 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.535138+0000", 2024-08-17T17:23:52.498 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.2700000000000004e-06 2024-08-17T17:23:52.498 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.498 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.498 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.498 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.039509+0000", 2024-08-17T17:23:52.498 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.504371227 2024-08-17T17:23:52.498 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.498 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.498 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.499 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.039651+0000", 2024-08-17T17:23:52.499 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014202400000000001 2024-08-17T17:23:52.499 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.499 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.499 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.499 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.039809+0000", 2024-08-17T17:23:52.499 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015809 2024-08-17T17:23:52.499 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.499 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.499 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.499 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.048393+0000", 2024-08-17T17:23:52.499 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.008583528 2024-08-17T17:23:52.499 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.499 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.500 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.500 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.048682+0000", 2024-08-17T17:23:52.500 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00028978599999999999 2024-08-17T17:23:52.500 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.500 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.500 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.500 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.052659+0000", 2024-08-17T17:23:52.500 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0039768340000000003 2024-08-17T17:23:52.500 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.500 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.500 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.500 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.052700+0000", 2024-08-17T17:23:52.500 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0844999999999999e-05 2024-08-17T17:23:52.500 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.500 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.501 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.501 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.054347+0000", 2024-08-17T17:23:52.501 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016471050000000001 2024-08-17T17:23:52.501 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.501 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.501 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.501 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.501 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.501 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88383 7.19 7:9fffae81:::benchmark_data_smithi063_196002_object88382:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:23:52.501 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:00.933708+0000", 2024-08-17T17:23:52.501 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 291.53330847000001, 2024-08-17T17:23:52.501 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.845462465, 2024-08-17T17:23:52.501 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.501 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.502 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.502 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.502 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.502 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88383 2024-08-17T17:23:52.502 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.502 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.502 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.502 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.502 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933708+0000", 2024-08-17T17:23:52.502 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.502 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.502 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.502 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.502 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933708+0000", 2024-08-17T17:23:52.503 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.503 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.503 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.503 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.503 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933706+0000", 2024-08-17T17:23:52.503 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:52.503 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.503 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.503 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.503 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933753+0000", 2024-08-17T17:23:52.503 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6516e-05 2024-08-17T17:23:52.503 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.503 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.503 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.503 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933766+0000", 2024-08-17T17:23:52.504 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2879e-05 2024-08-17T17:23:52.504 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.504 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.504 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.504 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933774+0000", 2024-08-17T17:23:52.504 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9829999999999996e-06 2024-08-17T17:23:52.504 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.504 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.504 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.504 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933823+0000", 2024-08-17T17:23:52.504 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8936e-05 2024-08-17T17:23:52.504 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.504 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.504 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.504 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.933830+0000", 2024-08-17T17:23:52.505 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6229999999999997e-06 2024-08-17T17:23:52.505 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.505 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.505 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.505 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.773096+0000", 2024-08-17T17:23:52.505 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.83926587799999997 2024-08-17T17:23:52.505 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.505 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.505 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.505 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.773204+0000", 2024-08-17T17:23:52.505 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010809199999999999 2024-08-17T17:23:52.505 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.506 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.506 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.506 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.773306+0000", 2024-08-17T17:23:52.506 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010179399999999999 2024-08-17T17:23:52.506 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.506 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.506 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.506 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.775896+0000", 2024-08-17T17:23:52.506 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0025904669999999999 2024-08-17T17:23:52.506 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.506 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.506 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.507 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.775925+0000", 2024-08-17T17:23:52.507 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8167e-05 2024-08-17T17:23:52.507 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.507 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.507 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.507 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.777385+0000", 2024-08-17T17:23:52.507 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0014603400000000001 2024-08-17T17:23:52.507 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.507 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.507 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.507 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.777464+0000", 2024-08-17T17:23:52.507 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9040000000000002e-05 2024-08-17T17:23:52.507 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.508 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.508 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.508 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.779170+0000", 2024-08-17T17:23:52.508 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0017062290000000001 2024-08-17T17:23:52.508 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.508 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.508 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.508 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.508 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.508 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88386 7.4 7:22787498:::benchmark_data_smithi063_196002_object88385:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:23:52.508 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:00.979852+0000", 2024-08-17T17:23:52.508 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 291.48716442099999, 2024-08-17T17:23:52.508 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7198389029999999, 2024-08-17T17:23:52.509 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.509 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.509 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.509 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.509 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.509 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88386 2024-08-17T17:23:52.509 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.509 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.509 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.509 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.509 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979852+0000", 2024-08-17T17:23:52.509 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.510 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.510 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.510 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.510 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979852+0000", 2024-08-17T17:23:52.510 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.510 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.510 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.510 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.510 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979850+0000", 2024-08-17T17:23:52.510 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:52.510 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.510 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.511 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.511 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979920+0000", 2024-08-17T17:23:52.511 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9574999999999997e-05 2024-08-17T17:23:52.511 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.511 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.511 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.511 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979935+0000", 2024-08-17T17:23:52.511 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5150999999999999e-05 2024-08-17T17:23:52.511 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.511 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.511 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.511 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979951+0000", 2024-08-17T17:23:52.511 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6509999999999999e-05 2024-08-17T17:23:52.512 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.512 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.512 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.512 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.980004+0000", 2024-08-17T17:23:52.512 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2992999999999999e-05 2024-08-17T17:23:52.512 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.512 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.512 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.512 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.980013+0000", 2024-08-17T17:23:52.513 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3270000000000001e-06 2024-08-17T17:23:52.513 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.513 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.513 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.513 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.653569+0000", 2024-08-17T17:23:52.513 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.67355651100000002 2024-08-17T17:23:52.513 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.513 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.513 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.513 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.653581+0000", 2024-08-17T17:23:52.513 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2017e-05 2024-08-17T17:23:52.513 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.514 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.514 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.514 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.673365+0000", 2024-08-17T17:23:52.514 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.01978422 2024-08-17T17:23:52.514 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.514 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.514 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.514 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.673553+0000", 2024-08-17T17:23:52.514 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00018712399999999999 2024-08-17T17:23:52.514 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.514 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.515 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.515 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.673745+0000", 2024-08-17T17:23:52.515 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00019201600000000001 2024-08-17T17:23:52.515 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.515 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.515 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.515 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.698545+0000", 2024-08-17T17:23:52.515 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.024800866000000001 2024-08-17T17:23:52.515 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.515 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.515 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.515 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.698579+0000", 2024-08-17T17:23:52.515 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3624000000000002e-05 2024-08-17T17:23:52.515 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.516 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.516 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.516 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.698674+0000", 2024-08-17T17:23:52.516 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4993000000000003e-05 2024-08-17T17:23:52.516 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.516 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.516 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.516 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.698734+0000", 2024-08-17T17:23:52.516 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9583000000000002e-05 2024-08-17T17:23:52.516 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.516 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.516 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.516 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.699691+0000", 2024-08-17T17:23:52.516 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00095704399999999999 2024-08-17T17:23:52.517 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.517 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.517 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.517 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.517 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.517 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88387 7.19 7:9eee2576:::benchmark_data_smithi063_196002_object88386:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:23:52.517 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:00.979965+0000", 2024-08-17T17:23:52.517 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 291.48705108799999, 2024-08-17T17:23:52.517 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.805482531, 2024-08-17T17:23:52.517 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.517 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.517 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.517 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.517 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.517 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88387 2024-08-17T17:23:52.518 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.518 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.518 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.518 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.518 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979965+0000", 2024-08-17T17:23:52.518 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.518 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.518 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.518 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.518 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979965+0000", 2024-08-17T17:23:52.518 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.518 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.519 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.519 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.519 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979965+0000", 2024-08-17T17:23:52.519 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:23:52.519 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.519 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.519 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.519 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.979994+0000", 2024-08-17T17:23:52.519 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8992999999999998e-05 2024-08-17T17:23:52.519 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.519 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.519 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.519 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.980003+0000", 2024-08-17T17:23:52.520 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3100000000000006e-06 2024-08-17T17:23:52.520 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.520 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.520 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.520 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.980009+0000", 2024-08-17T17:23:52.520 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3269999999999997e-06 2024-08-17T17:23:52.520 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.520 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.520 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.520 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.980048+0000", 2024-08-17T17:23:52.520 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8420000000000001e-05 2024-08-17T17:23:52.520 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.520 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.520 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.521 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.980054+0000", 2024-08-17T17:23:52.521 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8090000000000004e-06 2024-08-17T17:23:52.521 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.521 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.521 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.521 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.774383+0000", 2024-08-17T17:23:52.521 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.79432830300000001 2024-08-17T17:23:52.521 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.521 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.521 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.521 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.774533+0000", 2024-08-17T17:23:52.521 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015014999999999999 2024-08-17T17:23:52.521 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.521 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.521 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.522 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.774714+0000", 2024-08-17T17:23:52.522 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00018148099999999999 2024-08-17T17:23:52.522 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.522 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.522 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.522 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.776025+0000", 2024-08-17T17:23:52.522 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013108149999999999 2024-08-17T17:23:52.522 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.522 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.522 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.522 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.776180+0000", 2024-08-17T17:23:52.522 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000155198 2024-08-17T17:23:52.522 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.522 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.523 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.523 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.783976+0000", 2024-08-17T17:23:52.523 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0077961020000000001 2024-08-17T17:23:52.523 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.523 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.523 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.523 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.784011+0000", 2024-08-17T17:23:52.523 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4273000000000002e-05 2024-08-17T17:23:52.523 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.523 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.523 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.523 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.785448+0000", 2024-08-17T17:23:52.523 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001436959 2024-08-17T17:23:52.523 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.523 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88393 7.9 7:9286a306:::benchmark_data_smithi063_196002_object88392:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.004627+0000", 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 291.46238935399998, 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.701585377, 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88393 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.524 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.525 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.525 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004627+0000", 2024-08-17T17:23:52.525 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.525 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.525 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.525 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.525 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004627+0000", 2024-08-17T17:23:52.525 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.525 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.525 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.525 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.525 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004625+0000", 2024-08-17T17:23:52.525 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:23:52.526 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.526 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.526 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.526 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004663+0000", 2024-08-17T17:23:52.526 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8390999999999999e-05 2024-08-17T17:23:52.526 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.526 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.526 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.526 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004678+0000", 2024-08-17T17:23:52.526 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4637999999999999e-05 2024-08-17T17:23:52.526 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.526 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.526 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.527 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004686+0000", 2024-08-17T17:23:52.527 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4169999999999992e-06 2024-08-17T17:23:52.527 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.527 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.527 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.527 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004734+0000", 2024-08-17T17:23:52.527 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7255000000000001e-05 2024-08-17T17:23:52.527 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.527 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.528 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 85 16 122251 122235 89.8583 56.8125 0.0150277 0.0111227 2024-08-17T17:23:52.528 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.528 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.004741+0000", 2024-08-17T17:23:52.528 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6179999999999997e-06 2024-08-17T17:23:52.528 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.529 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.529 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.529 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.655714+0000", 2024-08-17T17:23:52.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.65097307000000004 2024-08-17T17:23:52.529 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.529 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.529 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.529 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.655724+0000", 2024-08-17T17:23:52.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8490000000000001e-06 2024-08-17T17:23:52.529 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.529 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.529 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.529 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.678130+0000", 2024-08-17T17:23:52.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0224053420000001 2024-08-17T17:23:52.530 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.530 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.530 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.530 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.678331+0000", 2024-08-17T17:23:52.530 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00020100399999999999 2024-08-17T17:23:52.530 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.530 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.530 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.530 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.678525+0000", 2024-08-17T17:23:52.530 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000194809 2024-08-17T17:23:52.530 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.530 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.530 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.530 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.702033+0000", 2024-08-17T17:23:52.531 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.023507920000000002 2024-08-17T17:23:52.531 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.531 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.531 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.531 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.702074+0000", 2024-08-17T17:23:52.531 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0812999999999997e-05 2024-08-17T17:23:52.531 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.531 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.531 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.531 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.704880+0000", 2024-08-17T17:23:52.531 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0028057170000000001 2024-08-17T17:23:52.531 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.531 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.532 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.704938+0000", 2024-08-17T17:23:52.532 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7982999999999997e-05 2024-08-17T17:23:52.532 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.532 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.532 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.706212+0000", 2024-08-17T17:23:52.532 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001274358 2024-08-17T17:23:52.532 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.532 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.532 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.532 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.533 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.533 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88394 7.1 7:80428bd7:::benchmark_data_smithi063_196002_object88393:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:23:52.533 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.006598+0000", 2024-08-17T17:23:52.533 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 291.46041816899998, 2024-08-17T17:23:52.533 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.69599852200000001, 2024-08-17T17:23:52.533 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.533 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.533 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.533 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.533 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.533 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88394 2024-08-17T17:23:52.533 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.533 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.533 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.534 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.534 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006598+0000", 2024-08-17T17:23:52.534 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.534 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.534 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.534 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.534 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006598+0000", 2024-08-17T17:23:52.534 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.534 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.534 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.534 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.534 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006597+0000", 2024-08-17T17:23:52.534 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:23:52.535 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.535 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.535 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.535 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006649+0000", 2024-08-17T17:23:52.535 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2046000000000003e-05 2024-08-17T17:23:52.535 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.535 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.535 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.535 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006661+0000", 2024-08-17T17:23:52.535 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2234e-05 2024-08-17T17:23:52.535 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.535 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.535 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.535 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006669+0000", 2024-08-17T17:23:52.536 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4610000000000002e-06 2024-08-17T17:23:52.536 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.536 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.536 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.536 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006720+0000", 2024-08-17T17:23:52.536 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1153999999999998e-05 2024-08-17T17:23:52.536 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.536 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.536 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.536 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.006728+0000", 2024-08-17T17:23:52.536 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.1100000000000003e-06 2024-08-17T17:23:52.536 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.536 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.537 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.537 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.684581+0000", 2024-08-17T17:23:52.537 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.67785242599999995 2024-08-17T17:23:52.537 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.537 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.537 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.537 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.684731+0000", 2024-08-17T17:23:52.537 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015084699999999999 2024-08-17T17:23:52.537 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.537 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.537 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,6", 2024-08-17T17:23:52.537 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.684900+0000", 2024-08-17T17:23:52.537 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016867 2024-08-17T17:23:52.537 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.538 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.538 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.538 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.686413+0000", 2024-08-17T17:23:52.538 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001513295 2024-08-17T17:23:52.538 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.538 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.538 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.538 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.686609+0000", 2024-08-17T17:23:52.538 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00019517699999999999 2024-08-17T17:23:52.538 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.538 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.538 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.539 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.702497+0000", 2024-08-17T17:23:52.539 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.015888475999999999 2024-08-17T17:23:52.539 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.539 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.539 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.539 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.702555+0000", 2024-08-17T17:23:52.539 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8285000000000002e-05 2024-08-17T17:23:52.539 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.539 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.539 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.539 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.702597+0000", 2024-08-17T17:23:52.539 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1230000000000003e-05 2024-08-17T17:23:52.539 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.539 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.540 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.540 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.540 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.540 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88399 7.9 7:95c6c148:::benchmark_data_smithi063_196002_object88398:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:23:52.540 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.034388+0000", 2024-08-17T17:23:52.540 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 291.43262796099998, 2024-08-17T17:23:52.540 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.687116794, 2024-08-17T17:23:52.540 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.540 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.540 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.540 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.540 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.540 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88399 2024-08-17T17:23:52.540 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.541 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.541 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.541 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034388+0000", 2024-08-17T17:23:52.541 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.541 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.541 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.541 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034388+0000", 2024-08-17T17:23:52.541 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.541 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.541 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.541 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034387+0000", 2024-08-17T17:23:52.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:52.542 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.542 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.542 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.542 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034418+0000", 2024-08-17T17:23:52.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1087999999999998e-05 2024-08-17T17:23:52.542 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.542 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.542 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.542 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034434+0000", 2024-08-17T17:23:52.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6064000000000001e-05 2024-08-17T17:23:52.542 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.542 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.542 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.543 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034444+0000", 2024-08-17T17:23:52.543 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0078000000000001e-05 2024-08-17T17:23:52.543 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.543 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.543 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.543 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034503+0000", 2024-08-17T17:23:52.543 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9336000000000002e-05 2024-08-17T17:23:52.543 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.543 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.543 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.543 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.034513+0000", 2024-08-17T17:23:52.543 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2369999999999998e-06 2024-08-17T17:23:52.543 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.543 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.543 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.544 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.655746+0000", 2024-08-17T17:23:52.544 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.62123385900000005 2024-08-17T17:23:52.544 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.544 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.544 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.544 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.655753+0000", 2024-08-17T17:23:52.544 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4960000000000001e-06 2024-08-17T17:23:52.544 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.544 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.544 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.544 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.702100+0000", 2024-08-17T17:23:52.544 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0463472949999999 2024-08-17T17:23:52.544 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.544 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.545 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.545 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.702256+0000", 2024-08-17T17:23:52.545 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000155857 2024-08-17T17:23:52.545 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.545 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.545 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.545 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.702420+0000", 2024-08-17T17:23:52.545 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016427799999999999 2024-08-17T17:23:52.545 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.545 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.546 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.546 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.709813+0000", 2024-08-17T17:23:52.546 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0073923210000000003 2024-08-17T17:23:52.546 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.546 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.546 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.546 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.711143+0000", 2024-08-17T17:23:52.546 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013302520000000001 2024-08-17T17:23:52.546 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.546 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.546 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.546 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.720255+0000", 2024-08-17T17:23:52.546 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0091118799999999993 2024-08-17T17:23:52.546 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.547 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.547 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.547 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.720292+0000", 2024-08-17T17:23:52.547 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.731e-05 2024-08-17T17:23:52.547 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.547 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.547 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.547 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.721505+0000", 2024-08-17T17:23:52.547 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012129370000000001 2024-08-17T17:23:52.547 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.547 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.547 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.548 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.548 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.548 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88400 7.9 7:900aeefa:::benchmark_data_smithi063_196002_object88399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1152)", 2024-08-17T17:23:52.548 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.699818+0000", 2024-08-17T17:23:52.548 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 290.76719830500002, 2024-08-17T17:23:52.548 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0249885809999999, 2024-08-17T17:23:52.548 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.548 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.548 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.548 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.548 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.548 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88400 2024-08-17T17:23:52.548 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.548 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.548 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699818+0000", 2024-08-17T17:23:52.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.549 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.549 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699818+0000", 2024-08-17T17:23:52.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.549 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.549 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699816+0000", 2024-08-17T17:23:52.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:23:52.549 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.550 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.550 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.550 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699911+0000", 2024-08-17T17:23:52.550 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4989999999999997e-05 2024-08-17T17:23:52.550 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.550 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.550 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.550 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699931+0000", 2024-08-17T17:23:52.550 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9667999999999999e-05 2024-08-17T17:23:52.550 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.550 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.550 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.551 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699939+0000", 2024-08-17T17:23:52.551 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.5399999999999996e-06 2024-08-17T17:23:52.551 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.551 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.551 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.551 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699986+0000", 2024-08-17T17:23:52.551 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6833999999999999e-05 2024-08-17T17:23:52.551 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.551 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.551 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.551 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.699996+0000", 2024-08-17T17:23:52.551 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7270000000000002e-06 2024-08-17T17:23:52.551 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.551 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.552 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.552 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.703779+0000", 2024-08-17T17:23:52.552 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.003783026 2024-08-17T17:23:52.552 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.552 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.552 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.552 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.703883+0000", 2024-08-17T17:23:52.552 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000104282 2024-08-17T17:23:52.552 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.552 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.552 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.552 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.703973+0000", 2024-08-17T17:23:52.552 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9666999999999994e-05 2024-08-17T17:23:52.552 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.709818+0000", 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0058452620000000004 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.711179+0000", 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001361077 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.723783+0000", 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012604148000000001 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.553 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.554 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.554 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.723819+0000", 2024-08-17T17:23:52.554 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5394e-05 2024-08-17T17:23:52.554 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.554 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.554 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.554 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.724806+0000", 2024-08-17T17:23:52.554 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0009877009999999999 2024-08-17T17:23:52.554 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.554 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.554 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.554 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.554 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.554 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88408 7.9 7:92f3bf61:::benchmark_data_smithi063_196002_object88407:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1152)", 2024-08-17T17:23:52.554 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.763022+0000", 2024-08-17T17:23:52.555 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 290.703994201, 2024-08-17T17:23:52.555 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.962521195, 2024-08-17T17:23:52.555 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.555 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.555 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.555 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.555 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.555 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88408 2024-08-17T17:23:52.555 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.555 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.555 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.555 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.555 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.763022+0000", 2024-08-17T17:23:52.555 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.555 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.556 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.556 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.763022+0000", 2024-08-17T17:23:52.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.556 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.556 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.556 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.763020+0000", 2024-08-17T17:23:52.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:23:52.556 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.556 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.556 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.763061+0000", 2024-08-17T17:23:52.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1201000000000001e-05 2024-08-17T17:23:52.557 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.557 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.763076+0000", 2024-08-17T17:23:52.557 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4712000000000001e-05 2024-08-17T17:23:52.557 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.557 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.763086+0000", 2024-08-17T17:23:52.557 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2909999999999996e-06 2024-08-17T17:23:52.557 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.557 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.774363+0000", 2024-08-17T17:23:52.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011277852999999999 2024-08-17T17:23:52.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.558 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.558 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.558 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.774373+0000", 2024-08-17T17:23:52.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8430000000000005e-06 2024-08-17T17:23:52.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.558 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.558 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.558 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.706262+0000", 2024-08-17T17:23:52.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.93188885399999999 2024-08-17T17:23:52.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.558 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.558 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.706517+0000", 2024-08-17T17:23:52.559 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00025510600000000001 2024-08-17T17:23:52.559 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.559 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.559 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.706677+0000", 2024-08-17T17:23:52.559 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000159925 2024-08-17T17:23:52.559 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.559 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.559 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.711150+0000", 2024-08-17T17:23:52.559 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0044729679999999999 2024-08-17T17:23:52.559 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.559 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.560 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.560 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.712847+0000", 2024-08-17T17:23:52.560 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016967130000000001 2024-08-17T17:23:52.560 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.560 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.560 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.560 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.724849+0000", 2024-08-17T17:23:52.560 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.01200242 2024-08-17T17:23:52.560 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.560 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.560 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.560 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.724892+0000", 2024-08-17T17:23:52.561 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2956e-05 2024-08-17T17:23:52.561 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.561 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.561 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.561 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.725543+0000", 2024-08-17T17:23:52.561 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00065106599999999995 2024-08-17T17:23:52.561 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.561 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.561 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.561 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.561 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.561 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88450 7.9 7:966ebe59:::benchmark_data_smithi063_196002_object88449:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1152)", 2024-08-17T17:23:52.561 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.840584+0000", 2024-08-17T17:23:52.561 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 290.626432067, 2024-08-17T17:23:52.562 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.88908479399999996, 2024-08-17T17:23:52.562 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.562 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.562 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.562 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.562 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.562 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88450 2024-08-17T17:23:52.562 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.562 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.562 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.562 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.562 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840584+0000", 2024-08-17T17:23:52.562 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.562 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840584+0000", 2024-08-17T17:23:52.563 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.563 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840582+0000", 2024-08-17T17:23:52.563 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:23:52.563 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840637+0000", 2024-08-17T17:23:52.563 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4929000000000002e-05 2024-08-17T17:23:52.563 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.564 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.564 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840651+0000", 2024-08-17T17:23:52.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3851e-05 2024-08-17T17:23:52.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.564 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.564 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840656+0000", 2024-08-17T17:23:52.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2870000000000004e-06 2024-08-17T17:23:52.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.564 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.564 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840714+0000", 2024-08-17T17:23:52.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7904e-05 2024-08-17T17:23:52.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.840726+0000", 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2075999999999999e-05 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.708243+0000", 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.86751665700000002 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.708434+0000", 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00019137000000000001 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.566 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.566 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.708553+0000", 2024-08-17T17:23:52.566 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000118205 2024-08-17T17:23:52.566 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.566 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.566 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.566 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.711189+0000", 2024-08-17T17:23:52.566 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00263649 2024-08-17T17:23:52.566 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.566 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.566 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.566 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.713490+0000", 2024-08-17T17:23:52.566 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023009509999999999 2024-08-17T17:23:52.566 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.567 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.567 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.567 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.728149+0000", 2024-08-17T17:23:52.567 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.014658775000000001 2024-08-17T17:23:52.567 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.567 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.567 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.567 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.728204+0000", 2024-08-17T17:23:52.567 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5546e-05 2024-08-17T17:23:52.567 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.567 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.567 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.567 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.729669+0000", 2024-08-17T17:23:52.567 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001464541 2024-08-17T17:23:52.567 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.568 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.568 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.568 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.568 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.568 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88451 7.9 7:95f9a7c9:::benchmark_data_smithi063_196002_object88450:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1152)", 2024-08-17T17:23:52.568 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.844136+0000", 2024-08-17T17:23:52.568 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 290.62288019099998, 2024-08-17T17:23:52.568 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.88693776499999999, 2024-08-17T17:23:52.568 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.568 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.568 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.568 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.568 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.568 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88451 2024-08-17T17:23:52.568 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.569 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.569 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.569 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:52.570 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.570 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844136+0000", 2024-08-17T17:23:52.570 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.570 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.570 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.570 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.570 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844136+0000", 2024-08-17T17:23:52.570 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.571 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.571 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.571 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.571 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844135+0000", 2024-08-17T17:23:52.571 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:23:52.571 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.571 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.571 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.571 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844251+0000", 2024-08-17T17:23:52.571 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011606400000000001 2024-08-17T17:23:52.571 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.571 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.571 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.571 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844264+0000", 2024-08-17T17:23:52.571 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2659e-05 2024-08-17T17:23:52.572 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.572 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.572 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.572 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844271+0000", 2024-08-17T17:23:52.572 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.861e-06 2024-08-17T17:23:52.572 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.572 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.572 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.572 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844318+0000", 2024-08-17T17:23:52.572 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7549999999999997e-05 2024-08-17T17:23:52.572 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.572 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.572 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.572 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.844326+0000", 2024-08-17T17:23:52.572 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.8669999999999994e-06 2024-08-17T17:23:52.573 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.573 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.573 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.573 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.709851+0000", 2024-08-17T17:23:52.573 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.86552479400000004 2024-08-17T17:23:52.573 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.573 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.573 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.573 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.709949+0000", 2024-08-17T17:23:52.573 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8265999999999997e-05 2024-08-17T17:23:52.573 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.573 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.573 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.573 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.710047+0000", 2024-08-17T17:23:52.574 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8220000000000005e-05 2024-08-17T17:23:52.574 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.574 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.574 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.574 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.711235+0000", 2024-08-17T17:23:52.574 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001187926 2024-08-17T17:23:52.574 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.574 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.574 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.574 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.713526+0000", 2024-08-17T17:23:52.574 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002291104 2024-08-17T17:23:52.574 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.574 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.574 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.574 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.729734+0000", 2024-08-17T17:23:52.575 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.016208024000000001 2024-08-17T17:23:52.575 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.575 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.575 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.575 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.729798+0000", 2024-08-17T17:23:52.575 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3497999999999994e-05 2024-08-17T17:23:52.575 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.575 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.575 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.575 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.731074+0000", 2024-08-17T17:23:52.575 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012758509999999999 2024-08-17T17:23:52.575 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.575 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.575 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.576 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.576 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.576 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186366 7.19 7:9c5202aa:::benchmark_data_smithi063_196002_object74273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1172)", 2024-08-17T17:23:52.576 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.412881+0000", 2024-08-17T17:23:52.576 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 167.054135446, 2024-08-17T17:23:52.576 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.232035218, 2024-08-17T17:23:52.576 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.576 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:23:52.576 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.576 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.576 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.576 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186366 2024-08-17T17:23:52.576 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.576 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.576 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.577 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.577 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.412881+0000", 2024-08-17T17:23:52.577 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.577 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.577 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.577 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.577 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.412881+0000", 2024-08-17T17:23:52.577 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.577 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.577 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.577 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.577 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.412879+0000", 2024-08-17T17:23:52.577 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:52.578 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.578 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.578 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.578 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.412889+0000", 2024-08-17T17:23:52.578 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.6339999999999996e-06 2024-08-17T17:23:52.578 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.578 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.578 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.578 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.412906+0000", 2024-08-17T17:23:52.578 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7068e-05 2024-08-17T17:23:52.578 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.578 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.579 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.579 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.412914+0000", 2024-08-17T17:23:52.579 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.5780000000000001e-06 2024-08-17T17:23:52.579 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.579 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.579 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.579 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.412984+0000", 2024-08-17T17:23:52.579 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0393000000000002e-05 2024-08-17T17:23:52.579 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.579 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.579 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.579 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.413159+0000", 2024-08-17T17:23:52.579 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000175305 2024-08-17T17:23:52.579 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.580 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.580 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:23:52.580 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.413743+0000", 2024-08-17T17:23:52.580 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000583532 2024-08-17T17:23:52.580 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.580 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.580 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.580 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.417086+0000", 2024-08-17T17:23:52.580 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003343688 2024-08-17T17:23:52.580 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.581 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.581 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.581 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.420322+0000", 2024-08-17T17:23:52.581 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0032359400000000001 2024-08-17T17:23:52.581 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.581 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.581 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.581 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.438793+0000", 2024-08-17T17:23:52.581 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.018470284999999999 2024-08-17T17:23:52.581 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.581 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.581 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.581 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.438808+0000", 2024-08-17T17:23:52.582 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5463000000000001e-05 2024-08-17T17:23:52.582 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.582 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.582 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.582 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.637733+0000", 2024-08-17T17:23:52.582 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1989244699999999 2024-08-17T17:23:52.582 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.582 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.582 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:23:52.582 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.637886+0000", 2024-08-17T17:23:52.582 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000153361 2024-08-17T17:23:52.582 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.582 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.582 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.583 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.644771+0000", 2024-08-17T17:23:52.583 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0068851529999999998 2024-08-17T17:23:52.583 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.583 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.583 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.583 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.644916+0000", 2024-08-17T17:23:52.583 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014488399999999999 2024-08-17T17:23:52.583 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.583 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.583 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.583 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.583 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.584 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186378 7.4 7:22ed332f:::benchmark_data_smithi063_196002_object74285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1173)", 2024-08-17T17:23:52.584 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.439687+0000", 2024-08-17T17:23:52.584 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 167.02732947300001, 2024-08-17T17:23:52.584 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2047585249999999, 2024-08-17T17:23:52.584 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.584 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.584 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.584 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.584 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.584 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186378 2024-08-17T17:23:52.584 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.584 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.584 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.585 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.585 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.439687+0000", 2024-08-17T17:23:52.585 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.585 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.585 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.585 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.585 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.439687+0000", 2024-08-17T17:23:52.585 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.585 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.585 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.585 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.585 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.439685+0000", 2024-08-17T17:23:52.585 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:52.586 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.586 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.586 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.586 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.439692+0000", 2024-08-17T17:23:52.586 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.348e-06 2024-08-17T17:23:52.586 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.586 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.586 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.586 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.439704+0000", 2024-08-17T17:23:52.586 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2833000000000001e-05 2024-08-17T17:23:52.586 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.586 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.587 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.587 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.439712+0000", 2024-08-17T17:23:52.587 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2300000000000002e-06 2024-08-17T17:23:52.587 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.587 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.587 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.587 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.440295+0000", 2024-08-17T17:23:52.587 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00058309499999999995 2024-08-17T17:23:52.587 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.587 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.587 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.587 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.440304+0000", 2024-08-17T17:23:52.587 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7849999999999997e-06 2024-08-17T17:23:52.587 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.588 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.588 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.588 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.554694+0000", 2024-08-17T17:23:52.588 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.114390843 2024-08-17T17:23:52.588 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.588 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.588 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.588 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.554842+0000", 2024-08-17T17:23:52.588 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000147916 2024-08-17T17:23:52.588 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.588 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.588 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.588 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.555372+0000", 2024-08-17T17:23:52.589 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00052959999999999997 2024-08-17T17:23:52.589 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.589 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.589 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.589 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.643709+0000", 2024-08-17T17:23:52.589 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.088336820999999996 2024-08-17T17:23:52.589 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.589 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.589 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.589 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.643755+0000", 2024-08-17T17:23:52.589 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6411e-05 2024-08-17T17:23:52.589 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.589 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.589 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.589 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.644258+0000", 2024-08-17T17:23:52.590 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00050251899999999999 2024-08-17T17:23:52.590 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.590 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.590 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.590 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.644313+0000", 2024-08-17T17:23:52.590 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5111000000000001e-05 2024-08-17T17:23:52.590 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.590 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.590 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.590 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.644445+0000", 2024-08-17T17:23:52.590 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013250299999999999 2024-08-17T17:23:52.590 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.591 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.591 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.591 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.591 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.591 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186380 7.19 7:9ff80d14:::benchmark_data_smithi063_196002_object74287:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1173)", 2024-08-17T17:23:52.591 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.440508+0000", 2024-08-17T17:23:52.591 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 167.026508667, 2024-08-17T17:23:52.591 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.205991123, 2024-08-17T17:23:52.591 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.591 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.591 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.591 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.591 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.591 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186380 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.440508+0000", 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.440508+0000", 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.440506+0000", 2024-08-17T17:23:52.592 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:23:52.593 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.593 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.593 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.593 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.440512+0000", 2024-08-17T17:23:52.593 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8309999999999997e-06 2024-08-17T17:23:52.593 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.593 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.593 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.593 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.440523+0000", 2024-08-17T17:23:52.593 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.097e-05 2024-08-17T17:23:52.593 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.593 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.593 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.593 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.440530+0000", 2024-08-17T17:23:52.593 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9829999999999999e-06 2024-08-17T17:23:52.594 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.594 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.594 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.594 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.440708+0000", 2024-08-17T17:23:52.594 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017752499999999999 2024-08-17T17:23:52.594 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.594 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.594 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.594 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.440718+0000", 2024-08-17T17:23:52.594 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0237999999999999e-05 2024-08-17T17:23:52.594 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.594 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.594 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.595 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.638200+0000", 2024-08-17T17:23:52.595 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1974824930000001 2024-08-17T17:23:52.595 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.595 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.595 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.595 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.638323+0000", 2024-08-17T17:23:52.595 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000122748 2024-08-17T17:23:52.595 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.595 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.595 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.595 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.638926+0000", 2024-08-17T17:23:52.595 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00060236999999999997 2024-08-17T17:23:52.595 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.596 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.596 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.596 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.644385+0000", 2024-08-17T17:23:52.596 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0054593990000000002 2024-08-17T17:23:52.596 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.596 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.596 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.596 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.644444+0000", 2024-08-17T17:23:52.596 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8566000000000001e-05 2024-08-17T17:23:52.596 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.596 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.596 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.596 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.646305+0000", 2024-08-17T17:23:52.596 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001861021 2024-08-17T17:23:52.597 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.597 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.597 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.597 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.646385+0000", 2024-08-17T17:23:52.597 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0047000000000001e-05 2024-08-17T17:23:52.597 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.597 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.597 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.597 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.646499+0000", 2024-08-17T17:23:52.597 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000114047 2024-08-17T17:23:52.597 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.597 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.597 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.598 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.598 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.598 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186396 7.14 7:2d40915f:::benchmark_data_smithi063_196002_object74303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1173)", 2024-08-17T17:23:52.598 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.472645+0000", 2024-08-17T17:23:52.598 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 166.99437137199999, 2024-08-17T17:23:52.598 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.17712423, 2024-08-17T17:23:52.598 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.598 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.598 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.598 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.598 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.598 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186396 2024-08-17T17:23:52.598 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.599 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.599 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.599 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.599 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.472645+0000", 2024-08-17T17:23:52.599 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.599 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.599 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.599 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.599 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.472645+0000", 2024-08-17T17:23:52.599 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.599 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.599 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.600 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.600 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.472643+0000", 2024-08-17T17:23:52.600 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:52.600 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.600 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.600 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.600 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.472650+0000", 2024-08-17T17:23:52.600 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7340000000000002e-06 2024-08-17T17:23:52.600 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.600 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.600 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.600 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.472663+0000", 2024-08-17T17:23:52.600 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3149000000000001e-05 2024-08-17T17:23:52.600 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.601 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.601 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.601 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.472671+0000", 2024-08-17T17:23:52.601 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9079999999999995e-06 2024-08-17T17:23:52.601 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.601 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.601 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.601 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.472717+0000", 2024-08-17T17:23:52.601 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5540000000000001e-05 2024-08-17T17:23:52.601 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.601 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.601 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.601 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.472724+0000", 2024-08-17T17:23:52.601 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7139999999999993e-06 2024-08-17T17:23:52.602 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.602 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.602 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.602 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.640902+0000", 2024-08-17T17:23:52.602 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.168177193 2024-08-17T17:23:52.602 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.602 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.602 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.602 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.641026+0000", 2024-08-17T17:23:52.602 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012450300000000001 2024-08-17T17:23:52.602 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.602 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.602 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.602 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.641471+0000", 2024-08-17T17:23:52.602 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00044470600000000001 2024-08-17T17:23:52.603 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.603 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.603 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.603 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.643764+0000", 2024-08-17T17:23:52.603 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0022927910000000002 2024-08-17T17:23:52.603 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.603 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.603 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.603 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.648186+0000", 2024-08-17T17:23:52.603 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0044226320000000001 2024-08-17T17:23:52.603 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.603 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.603 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.603 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649571+0000", 2024-08-17T17:23:52.604 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001384662 2024-08-17T17:23:52.604 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.604 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.604 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.604 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649646+0000", 2024-08-17T17:23:52.604 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.5181999999999999e-05 2024-08-17T17:23:52.604 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.604 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.604 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.604 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649769+0000", 2024-08-17T17:23:52.604 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012306900000000001 2024-08-17T17:23:52.604 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.605 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.605 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.605 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.605 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.605 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186404 7.9 7:91cb9959:::benchmark_data_smithi063_196002_object74311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1173)", 2024-08-17T17:23:52.605 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.484901+0000", 2024-08-17T17:23:52.605 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 166.98211473399999, 2024-08-17T17:23:52.605 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.159781985, 2024-08-17T17:23:52.605 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.605 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.605 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.605 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.605 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.605 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186404 2024-08-17T17:23:52.605 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.606 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.606 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.606 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.606 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.484901+0000", 2024-08-17T17:23:52.606 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.606 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.606 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.606 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.606 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.484901+0000", 2024-08-17T17:23:52.606 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.606 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.606 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.606 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.606 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.484900+0000", 2024-08-17T17:23:52.607 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:23:52.607 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.607 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.607 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.607 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.484908+0000", 2024-08-17T17:23:52.607 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.123e-06 2024-08-17T17:23:52.607 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.607 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.607 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.607 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.484924+0000", 2024-08-17T17:23:52.607 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6334e-05 2024-08-17T17:23:52.607 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.607 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.607 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.608 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.484933+0000", 2024-08-17T17:23:52.608 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6610000000000006e-06 2024-08-17T17:23:52.608 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.608 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.608 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.608 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.484987+0000", 2024-08-17T17:23:52.608 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4347e-05 2024-08-17T17:23:52.608 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.608 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.608 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.608 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.484999+0000", 2024-08-17T17:23:52.608 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.24e-05 2024-08-17T17:23:52.608 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.638189+0000", 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.153189341 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.638323+0000", 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000133728 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.638875+0000", 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00055289100000000002 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.609 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.610 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.610 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.644376+0000", 2024-08-17T17:23:52.610 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.005500291 2024-08-17T17:23:52.610 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.610 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.610 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.610 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.644391+0000", 2024-08-17T17:23:52.610 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5440000000000001e-05 2024-08-17T17:23:52.610 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.610 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.610 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.610 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.644480+0000", 2024-08-17T17:23:52.610 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8497000000000003e-05 2024-08-17T17:23:52.611 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.611 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.611 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.611 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.644564+0000", 2024-08-17T17:23:52.611 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4368000000000003e-05 2024-08-17T17:23:52.611 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.611 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.611 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.611 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.644683+0000", 2024-08-17T17:23:52.611 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011940800000000001 2024-08-17T17:23:52.611 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.611 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.611 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.611 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.612 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.612 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186407 7.14 7:2d53723f:::benchmark_data_smithi063_196002_object74314:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1173)", 2024-08-17T17:23:52.612 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.493416+0000", 2024-08-17T17:23:52.612 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 166.973599737, 2024-08-17T17:23:52.612 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.156653259, 2024-08-17T17:23:52.612 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:52.612 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:23:52.612 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:52.612 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:23:52.612 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:23:52.612 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186407 2024-08-17T17:23:52.612 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.612 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:52.612 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.613 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:52.613 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493416+0000", 2024-08-17T17:23:52.613 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.613 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.613 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.613 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:52.613 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493416+0000", 2024-08-17T17:23:52.613 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:52.613 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.613 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.613 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:52.613 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493415+0000", 2024-08-17T17:23:52.613 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:23:52.614 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.614 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.614 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:52.614 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493421+0000", 2024-08-17T17:23:52.614 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4919999999999998e-06 2024-08-17T17:23:52.614 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.614 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.614 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:52.614 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493435+0000", 2024-08-17T17:23:52.614 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4062e-05 2024-08-17T17:23:52.614 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.614 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.614 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:52.615 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493442+0000", 2024-08-17T17:23:52.615 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1570000000000003e-06 2024-08-17T17:23:52.615 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.615 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.615 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.615 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493485+0000", 2024-08-17T17:23:52.615 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2061000000000003e-05 2024-08-17T17:23:52.615 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.615 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.615 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:23:52.615 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493492+0000", 2024-08-17T17:23:52.615 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3950000000000003e-06 2024-08-17T17:23:52.615 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.616 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.616 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:52.616 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.643960+0000", 2024-08-17T17:23:52.616 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1504677189999999 2024-08-17T17:23:52.616 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.616 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.616 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:52.616 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.644086+0000", 2024-08-17T17:23:52.616 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012680999999999999 2024-08-17T17:23:52.616 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.616 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.616 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:23:52.616 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.644417+0000", 2024-08-17T17:23:52.616 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00033075699999999999 2024-08-17T17:23:52.616 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.617 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.617 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:52.617 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.646621+0000", 2024-08-17T17:23:52.617 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002203553 2024-08-17T17:23:52.617 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.617 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.617 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.617 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.648221+0000", 2024-08-17T17:23:52.617 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016006219999999999 2024-08-17T17:23:52.617 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.617 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.617 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:52.617 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649907+0000", 2024-08-17T17:23:52.618 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001686019 2024-08-17T17:23:52.618 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.618 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.618 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:23:52.618 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649951+0000", 2024-08-17T17:23:52.618 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3532999999999998e-05 2024-08-17T17:23:52.618 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:52.618 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:52.618 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:23:52.618 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650070+0000", 2024-08-17T17:23:52.618 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000118768 2024-08-17T17:23:52.618 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.618 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.618 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.619 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:52.619 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:52.619 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:23:52.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.665 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:52.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:52.766 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:52.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.807 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:52.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:52.908 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:52.951 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:52.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:52.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:53.053 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:53.092 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:53.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:53.194 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:53.236 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:53.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:53.338 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:53.382 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:53.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:53.401 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:23:53.419 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:23:53.484 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:53.512 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 86 16 123147 123131 89.4646 56 0.0253165 0.0111723 2024-08-17T17:23:53.522 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:53.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:53.624 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:53.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.673 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:53.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:53.741 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:53.772 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:23:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:23:53] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.51.0" 2024-08-17T17:23:53.773 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:53 smithi063 ceph-mon[114756]: pgmap v1569: 99 pgs: 99 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2024-08-17T17:23:53.773 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:53 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/738747901' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:53.774 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:53 smithi063 ceph-mon[109843]: pgmap v1569: 99 pgs: 99 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2024-08-17T17:23:53.774 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:53 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/738747901' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:53.774 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:53.819 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:53.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:53.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:53 smithi162 ceph-mon[104668]: pgmap v1569: 99 pgs: 99 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2024-08-17T17:23:53.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:53 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/738747901' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:23:53.921 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:53.955 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:53.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:53.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:54.057 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:54.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.089 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:54.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:54.190 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:54.225 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:54.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:54.327 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:54.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.367 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:54.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:54.468 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:54.510 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:54.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:54.512 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 87 16 124016 124000 89.0606 54.3125 0.00992995 0.011223 2024-08-17T17:23:54.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:54 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:54.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:54 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/738747901' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:54.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:54 smithi063 ceph-mon[109843]: osdmap e1211: 8 total, 8 up, 8 in 2024-08-17T17:23:54.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:54 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:54.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:54 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/738747901' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:54.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:54 smithi063 ceph-mon[114756]: osdmap e1211: 8 total, 8 up, 8 in 2024-08-17T17:23:54.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:54 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:23:54.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:54 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/738747901' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:23:54.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:54 smithi162 ceph-mon[104668]: osdmap e1211: 8 total, 8 up, 8 in 2024-08-17T17:23:54.612 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:54.654 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:54.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:54.757 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:54.796 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:54.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:54.898 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:54.927 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:54.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:54.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:55.029 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:55.086 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:55.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:55.188 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:55.225 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:55.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:55.327 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:55.368 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:55.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:55.470 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:55.512 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 88 16 124846 124830 88.638 51.875 0.018687 0.0112756 2024-08-17T17:23:55.522 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:55.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:55.608 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:55 smithi063 ceph-mon[114756]: pgmap v1571: 99 pgs: 99 active+clean; 7.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2024-08-17T17:23:55.608 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:55 smithi063 ceph-mon[109843]: pgmap v1571: 99 pgs: 99 active+clean; 7.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2024-08-17T17:23:55.624 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:55.669 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:55.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:55.713 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:23:55.714 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:23:55.735 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:55 smithi162 ceph-mon[104668]: pgmap v1571: 99 pgs: 99 active+clean; 7.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2024-08-17T17:23:55.755 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:23:55.755 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:23:55.771 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:55.775 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:23:55.775 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:23:55.775 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.775 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:125028 8.4 8:221bfe66:::benchmark_data_smithi063_230025_object125027:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1211)", 2024-08-17T17:23:55.775 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:23:55.752339+0000", 2024-08-17T17:23:55.775 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.021301892999999999, 2024-08-17T17:23:55.775 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.021336765000000001, 2024-08-17T17:23:55.775 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:55.775 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:23:55.776 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:55.776 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:23:55.776 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:23:55.776 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 125028 2024-08-17T17:23:55.776 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.776 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:55.776 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.776 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:55.776 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.752339+0000", 2024-08-17T17:23:55.776 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:55.776 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.776 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.776 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:55.776 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.752339+0000", 2024-08-17T17:23:55.777 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:55.777 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.777 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.777 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:55.777 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.752338+0000", 2024-08-17T17:23:55.777 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:55.777 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.777 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.777 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:55.777 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.752375+0000", 2024-08-17T17:23:55.777 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6727000000000003e-05 2024-08-17T17:23:55.777 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.777 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.777 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:55.778 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.752393+0000", 2024-08-17T17:23:55.778 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7881999999999999e-05 2024-08-17T17:23:55.778 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.778 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.778 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:55.778 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.752401+0000", 2024-08-17T17:23:55.778 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.5310000000000001e-06 2024-08-17T17:23:55.778 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.778 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.778 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:55.778 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.752822+0000", 2024-08-17T17:23:55.778 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00042089900000000002 2024-08-17T17:23:55.778 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.779 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.779 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:55.779 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.753004+0000", 2024-08-17T17:23:55.779 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00018206200000000001 2024-08-17T17:23:55.779 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.779 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.779 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:23:55.779 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.753185+0000", 2024-08-17T17:23:55.779 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00018070200000000001 2024-08-17T17:23:55.779 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.779 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.779 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:55.779 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.764714+0000", 2024-08-17T17:23:55.780 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011529022 2024-08-17T17:23:55.780 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.780 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.780 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:55.780 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.764722+0000", 2024-08-17T17:23:55.780 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3469999999999999e-06 2024-08-17T17:23:55.780 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:55.780 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:55.780 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:55.780 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.780 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.780 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:125035 8.4 8:280a9a20:::benchmark_data_smithi063_230025_object125034:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1211)", 2024-08-17T17:23:55.780 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:23:55.757877+0000", 2024-08-17T17:23:55.781 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.015764123000000001, 2024-08-17T17:23:55.781 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.015886125000000001, 2024-08-17T17:23:55.781 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:55.781 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:23:55.781 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:55.781 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:23:55.781 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:23:55.781 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 125035 2024-08-17T17:23:55.781 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.781 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:55.781 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.781 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:55.781 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.757877+0000", 2024-08-17T17:23:55.781 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:55.782 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.782 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.782 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:55.782 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.757877+0000", 2024-08-17T17:23:55.782 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:55.782 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.782 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.782 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:55.782 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.757876+0000", 2024-08-17T17:23:55.782 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:55.782 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.782 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.782 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:55.783 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.757932+0000", 2024-08-17T17:23:55.783 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5680000000000002e-05 2024-08-17T17:23:55.783 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.783 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.783 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:55.783 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.757943+0000", 2024-08-17T17:23:55.783 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1068e-05 2024-08-17T17:23:55.783 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.783 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.783 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:55.783 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.757950+0000", 2024-08-17T17:23:55.783 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0890000000000002e-06 2024-08-17T17:23:55.783 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.783 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.784 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:55.784 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.760224+0000", 2024-08-17T17:23:55.784 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0022740989999999999 2024-08-17T17:23:55.784 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.784 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.784 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:55.784 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.760378+0000", 2024-08-17T17:23:55.784 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015436399999999999 2024-08-17T17:23:55.784 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.784 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.784 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:23:55.784 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.760542+0000", 2024-08-17T17:23:55.784 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016395299999999999 2024-08-17T17:23:55.785 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.785 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.785 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:55.785 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765198+0000", 2024-08-17T17:23:55.785 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0046562940000000001 2024-08-17T17:23:55.785 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.785 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.785 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:55.785 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.770221+0000", 2024-08-17T17:23:55.785 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0050225959999999998 2024-08-17T17:23:55.785 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:55.785 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:55.785 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:55.786 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.786 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.786 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:125042 8.4 8:26b15edc:::benchmark_data_smithi063_230025_object125041:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1211)", 2024-08-17T17:23:55.786 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:23:55.765701+0000", 2024-08-17T17:23:55.786 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0079407000000000002, 2024-08-17T17:23:55.786 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.008141611, 2024-08-17T17:23:55.786 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:55.786 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:23:55.786 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:55.786 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:23:55.786 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:23:55.786 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 125042 2024-08-17T17:23:55.786 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.787 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:55.787 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.787 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:55.787 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765701+0000", 2024-08-17T17:23:55.787 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:55.787 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.787 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.787 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:55.787 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765701+0000", 2024-08-17T17:23:55.787 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:55.787 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.787 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.787 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:55.788 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765699+0000", 2024-08-17T17:23:55.788 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:23:55.788 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.788 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.788 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:55.788 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765733+0000", 2024-08-17T17:23:55.788 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3806999999999997e-05 2024-08-17T17:23:55.788 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.788 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.788 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:55.788 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765750+0000", 2024-08-17T17:23:55.788 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6850999999999998e-05 2024-08-17T17:23:55.788 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.789 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.789 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:55.789 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765791+0000", 2024-08-17T17:23:55.789 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1140999999999998e-05 2024-08-17T17:23:55.789 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.789 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.789 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:55.789 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765864+0000", 2024-08-17T17:23:55.789 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3545000000000001e-05 2024-08-17T17:23:55.790 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.790 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.790 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:55.790 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765999+0000", 2024-08-17T17:23:55.790 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013466800000000001 2024-08-17T17:23:55.790 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.790 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.790 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:23:55.790 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.766140+0000", 2024-08-17T17:23:55.790 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014052800000000001 2024-08-17T17:23:55.790 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.790 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.790 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:55.791 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.770566+0000", 2024-08-17T17:23:55.791 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0044266790000000002 2024-08-17T17:23:55.791 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.791 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.791 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:55.791 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.771734+0000", 2024-08-17T17:23:55.791 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0011673040000000001 2024-08-17T17:23:55.791 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:55.791 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:55.791 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:55.791 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.791 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.791 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:125043 8.3 8:cd547680:::benchmark_data_smithi063_230025_object125042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1211)", 2024-08-17T17:23:55.791 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:23:55.765825+0000", 2024-08-17T17:23:55.791 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.007816379, 2024-08-17T17:23:55.792 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0081025349999999993, 2024-08-17T17:23:55.792 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:55.792 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:23:55.792 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:23:55.792 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:23:55.792 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:23:55.792 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 125043 2024-08-17T17:23:55.792 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.792 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:55.792 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.792 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:55.792 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765825+0000", 2024-08-17T17:23:55.792 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:55.793 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.793 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.793 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:55.793 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765825+0000", 2024-08-17T17:23:55.793 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:55.793 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.793 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.793 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:55.793 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765824+0000", 2024-08-17T17:23:55.793 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:23:55.793 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.793 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.793 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:55.794 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765854+0000", 2024-08-17T17:23:55.794 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9607999999999998e-05 2024-08-17T17:23:55.794 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.794 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.794 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:55.794 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765872+0000", 2024-08-17T17:23:55.794 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7813e-05 2024-08-17T17:23:55.794 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.794 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.794 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:55.794 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765878+0000", 2024-08-17T17:23:55.794 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7179999999999999e-06 2024-08-17T17:23:55.794 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.795 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.795 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:55.795 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.765933+0000", 2024-08-17T17:23:55.795 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4809999999999999e-05 2024-08-17T17:23:55.795 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.795 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.795 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:55.795 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.766070+0000", 2024-08-17T17:23:55.795 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013683099999999999 2024-08-17T17:23:55.795 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.795 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.795 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:23:55.795 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.766207+0000", 2024-08-17T17:23:55.795 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000136852 2024-08-17T17:23:55.796 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.796 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.796 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:23:55.796 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.771416+0000", 2024-08-17T17:23:55.796 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0052094159999999997 2024-08-17T17:23:55.796 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.796 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.796 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:23:55.796 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.772540+0000", 2024-08-17T17:23:55.796 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0011233230000000001 2024-08-17T17:23:55.796 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:55.796 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:55.796 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:55.796 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.797 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.797 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.45218.0:125046 8.2 e1211/1195 8:49592ad3:::benchmark_data_smithi063_230025_object125045:head v 1211'7976, mlcod=1211'7974)", 2024-08-17T17:23:55.797 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:23:55.771555+0000", 2024-08-17T17:23:55.797 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0020867680000000001, 2024-08-17T17:23:55.797 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0024465120000000001, 2024-08-17T17:23:55.797 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:23:55.797 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:23:55.797 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:23:55.797 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.797 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:23:55.797 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.771555+0000", 2024-08-17T17:23:55.797 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:55.797 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.798 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.798 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:23:55.798 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.771555+0000", 2024-08-17T17:23:55.798 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:23:55.798 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.798 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.798 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:23:55.798 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.771554+0000", 2024-08-17T17:23:55.798 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:23:55.798 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.798 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.798 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:23:55.799 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.771579+0000", 2024-08-17T17:23:55.799 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4935000000000001e-05 2024-08-17T17:23:55.799 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.799 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.799 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:23:55.799 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.771592+0000", 2024-08-17T17:23:55.799 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3669000000000001e-05 2024-08-17T17:23:55.799 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.799 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.799 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:23:55.799 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.771599+0000", 2024-08-17T17:23:55.799 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6590000000000001e-06 2024-08-17T17:23:55.799 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.799 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.799 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:23:55.800 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.771652+0000", 2024-08-17T17:23:55.800 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2636999999999998e-05 2024-08-17T17:23:55.800 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:23:55.800 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:23:55.800 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:23:55.800 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:23:55.771672+0000", 2024-08-17T17:23:55.800 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0611e-05 2024-08-17T17:23:55.800 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:55.800 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:23:55.800 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:55.800 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:23:55.800 INFO:teuthology.orchestra.run.smithi162.stdout: ], 2024-08-17T17:23:55.800 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 5 2024-08-17T17:23:55.800 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:23:55.801 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:23:55.801 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:23:55.820 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:55.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:55.828 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:23:55.828 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:23:55.870 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:23:55.870 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:23:55.911 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:23:55.911 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:23:55.921 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:55.943 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:23:55.943 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:23:55.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:55.967 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:55.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:55.978 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:23:55.978 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:23:56.068 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:56.104 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:56.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:56.205 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:56.253 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:56.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:56.356 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:56.400 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:56.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:56.502 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:56.512 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 89 16 125659 125643 88.213 50.8125 0.0219017 0.0113311 2024-08-17T17:23:56.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.541 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:56.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:56.596 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:23:56.643 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:56.679 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:56.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:56.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:56 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2108024658' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:56.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:56 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2108024658' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:56.781 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:23:56.819 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:56.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:23:56.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:56 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2108024658' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:23:56.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:56.952 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:56.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:56.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:56.956 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:23:57.024 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:23:57.025 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:23:57.025 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-08-17T17:23:57.025 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:23:57.054 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:57.068 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:23:57.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.111 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:57.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:57.212 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:57.248 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:57.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:57.350 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:57.385 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:57.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:57.452 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:57.487 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:57.498 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:23:57.513 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Total time run: 90.0101 2024-08-17T17:23:57.513 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Total writes made: 126487 2024-08-17T17:23:57.513 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Write size: 65536 2024-08-17T17:23:57.513 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Object size: 65536 2024-08-17T17:23:57.513 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Bandwidth (MB/sec): 87.8284 2024-08-17T17:23:57.513 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev Bandwidth: 40.3089 2024-08-17T17:23:57.513 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max bandwidth (MB/sec): 237 2024-08-17T17:23:57.513 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min bandwidth (MB/sec): 45.125 2024-08-17T17:23:57.513 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Average IOPS: 1405 2024-08-17T17:23:57.514 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev IOPS: 644.942 2024-08-17T17:23:57.514 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max IOPS: 3792 2024-08-17T17:23:57.514 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min IOPS: 722 2024-08-17T17:23:57.514 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Average Latency(s): 0.0113811 2024-08-17T17:23:57.514 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev Latency(s): 0.00892834 2024-08-17T17:23:57.514 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max latency(s): 0.153833 2024-08-17T17:23:57.514 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min latency(s): 0.00109066 2024-08-17T17:23:57.518 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:23:57.531 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:57.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:57.633 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:57.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.678 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:57.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:57.755 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:57 smithi063 ceph-mon[114756]: pgmap v1572: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2024-08-17T17:23:57.756 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:57 smithi063 ceph-mon[109843]: pgmap v1572: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2024-08-17T17:23:57.779 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:57.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.806 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:57.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:57 smithi162 ceph-mon[104668]: pgmap v1572: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 910 op/s 2024-08-17T17:23:57.908 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:57.953 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:57.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:57.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:58.054 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:58.095 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:58.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:58.196 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:58.227 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:58.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:58.329 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:58.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.370 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:58.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:58.471 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:23:58.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.509 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:58.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:23:58.610 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:58.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.645 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:58.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:58.746 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:23:58.781 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:58.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.782 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:23:58.883 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:58.909 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:58.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:58.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:58.916 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:23:58.938 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:23:58.938 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:23:59.012 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:59.064 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:59.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:59.166 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:23:59.201 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:59.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:23:59.303 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:23:59.343 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:59.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:23:59.444 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:23:59.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.485 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:59.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:23:59.586 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:59.610 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:59.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:59.711 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:23:59.743 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:23:59.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:23:59.763 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:23:59 smithi162 ceph-mon[104668]: pgmap v1573: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 56 MiB/s wr, 899 op/s 2024-08-17T17:23:59.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:23:59 smithi063 ceph-mon[109843]: pgmap v1573: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 56 MiB/s wr, 899 op/s 2024-08-17T17:23:59.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:23:59 smithi063 ceph-mon[114756]: pgmap v1573: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 56 MiB/s wr, 899 op/s 2024-08-17T17:23:59.845 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:23:59.875 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:23:59.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:23:59.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:23:59.977 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:00.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.010 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:00.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:00.084 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:24:00.111 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:00.155 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:00.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:00.257 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:00.310 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:00.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:00.412 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:00.429 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:24:00.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.456 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:00.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:00.557 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:00.601 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:00.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:00.703 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:00.743 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:00.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:00.845 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:00.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:00.893 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:00.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:00.994 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:01.017 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:01.024 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:01.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:01.126 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:01.150 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:01.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:01.252 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:01.296 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:01.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:01.398 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:01.446 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:01.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:01.459 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:24:01.485 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:24:01.488 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:24:01.528 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:01 smithi063 ceph-mon[114756]: pgmap v1574: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 51 MiB/s wr, 893 op/s 2024-08-17T17:24:01.528 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:01.528 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:01 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3314548850' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:01.528 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:01 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:01.528 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:01 smithi063 ceph-mon[109843]: pgmap v1574: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 51 MiB/s wr, 893 op/s 2024-08-17T17:24:01.528 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:01.528 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:01 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3314548850' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:01.528 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:01 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:01.548 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:01.587 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:01.682 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:01 smithi162 ceph-mon[104668]: pgmap v1574: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 51 MiB/s wr, 893 op/s 2024-08-17T17:24:01.683 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:01.683 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:01 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3314548850' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:01.683 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:01 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:01.689 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:01.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.735 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:01.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:01.836 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:01.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:01.877 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:01.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:01.978 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:02.018 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:02.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:02.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:02.155 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:02.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:02.257 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:02.294 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:02.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:02.368 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:24:02.368 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:24:02.396 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:02.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.436 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:02.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:02.537 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:02.566 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:02.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:02.668 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:02.702 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:02.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:02.719 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:02 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:02.719 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:02 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:24:02.719 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:02 smithi162 ceph-mon[104668]: osdmap e1212: 8 total, 8 up, 8 in 2024-08-17T17:24:02.719 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:02 smithi162 ceph-mon[104668]: from='client.35391 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:02.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:02 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:02.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:02 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:24:02.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:02 smithi063 ceph-mon[114756]: osdmap e1212: 8 total, 8 up, 8 in 2024-08-17T17:24:02.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:02 smithi063 ceph-mon[114756]: from='client.35391 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:02.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:02 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:02.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:02 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:24:02.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:02 smithi063 ceph-mon[109843]: osdmap e1212: 8 total, 8 up, 8 in 2024-08-17T17:24:02.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:02 smithi063 ceph-mon[109843]: from='client.35391 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:02.804 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:02.845 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:02.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:02.946 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:02.983 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:02.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:02.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:03.085 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:03.101 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:24:03.122 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:03.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:03.225 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:03.264 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:03.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:03.366 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:03.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.413 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:03.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:03.485 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:24:03.514 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:03.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:03 smithi063 ceph-mon[109843]: pgmap v1576: 99 pgs: 99 active+clean; 7.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 115 B/s rd, 35 MiB/s wr, 865 op/s 2024-08-17T17:24:03.530 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:24:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:24:03] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.51.0" 2024-08-17T17:24:03.531 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:03 smithi063 ceph-mon[114756]: pgmap v1576: 99 pgs: 99 active+clean; 7.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 115 B/s rd, 35 MiB/s wr, 865 op/s 2024-08-17T17:24:03.547 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:03.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:03.649 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:03.683 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:03.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:03.768 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:03 smithi162 ceph-mon[104668]: pgmap v1576: 99 pgs: 99 active+clean; 7.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 115 B/s rd, 35 MiB/s wr, 865 op/s 2024-08-17T17:24:03.785 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:03.827 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:03.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:03.845 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1575,"stamp":"2024-08-17T17:24:01.455590+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8224511997,"num_objects":125514,"num_object_clones":0,"num_object_copies":376542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":125514,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":127198,"num_write_kb":8083874,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1246,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123832,"ondisk_log_size":123832,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":27574296,"kb_used_data":24186584,"kb_used_omap":199,"kb_used_meta":3387448,"kb_avail":722386920,"statfs":{"total":767960285184,"available":739724206080,"internally_reserved":0,"allocated":24767062016,"data_stored":24747883007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":3468747516},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,1,3,5,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":340131864,"num_objects":5191,"num_object_clones":0,"num_object_copies":15573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5191,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6837,"num_write_kb":384833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.163854"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":1517,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.412947+0000","last_change":"2024-08-17T17:21:06.463828+0000","last_active":"2024-08-17T17:23:53.412947+0000","last_peered":"2024-08-17T17:23:53.412947+0000","last_clean":"2024-08-17T17:23:53.412947+0000","last_became_active":"2024-08-17T17:21:06.463464+0000","last_became_peered":"2024-08-17T17:21:06.463464+0000","last_unstale":"2024-08-17T17:23:53.412947+0000","last_undegraded":"2024-08-17T17:23:53.412947+0000","last_fullsized":"2024-08-17T17:23:53.412947+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:46.465006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1139,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.473573+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:23:42.473573+0000","last_peered":"2024-08-17T17:23:42.473573+0000","last_clean":"2024-08-17T17:23:42.473573+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:23:42.473573+0000","last_undegraded":"2024-08-17T17:23:42.473573+0000","last_fullsized":"2024-08-17T17:23:42.473573+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1151,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.486987+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:23:42.486987+0000","last_peered":"2024-08-17T17:23:42.486987+0000","last_clean":"2024-08-17T17:23:42.486987+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:23:42.486987+0000","last_undegraded":"2024-08-17T17:23:42.486987+0000","last_fullsized":"2024-08-17T17:23:42.486987+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1006,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.473511+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:23:42.473511+0000","last_peered":"2024-08-17T17:23:42.473511+0000","last_clean":"2024-08-17T17:23:42.473511+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:23:42.473511+0000","last_undegraded":"2024-08-17T17:23:42.473511+0000","last_fullsized":"2024-08-17T17:23:42.473511+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1069,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.412484+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:23:53.412484+0000","last_peered":"2024-08-17T17:23:53.412484+0000","last_clean":"2024-08-17T17:23:53.412484+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:23:53.412484+0000","last_undegraded":"2024-08-17T17:23:53.412484+0000","last_fullsized":"2024-08-17T17:23:53.412484+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1123,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.419413+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:23:53.419413+0000","last_peered":"2024-08-17T17:23:53.419413+0000","last_clean":"2024-08-17T17:23:53.419413+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:23:53.419413+0000","last_undegraded":"2024-08-17T17:23:53.419413+0000","last_fullsized":"2024-08-17T17:23:53.419413+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1517,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.418909+0000","last_change":"2024-08-17T17:21:06.442598+0000","last_active":"2024-08-17T17:23:53.418909+0000","last_peered":"2024-08-17T17:23:53.418909+0000","last_clean":"2024-08-17T17:23:53.418909+0000","last_became_active":"2024-08-17T17:21:06.442264+0000","last_became_peered":"2024-08-17T17:21:06.442264+0000","last_unstale":"2024-08-17T17:23:53.418909+0000","last_undegraded":"2024-08-17T17:23:53.418909+0000","last_fullsized":"2024-08-17T17:23:53.418909+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:35.003620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1200,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:52.270973+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:23:52.270973+0000","last_peered":"2024-08-17T17:23:52.270973+0000","last_clean":"2024-08-17T17:23:52.270973+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:23:52.270973+0000","last_undegraded":"2024-08-17T17:23:52.270973+0000","last_fullsized":"2024-08-17T17:23:52.270973+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"1211'7873","reported_seq":7898,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.556729+0000","last_change":"2024-08-17T17:22:10.861776+0000","last_active":"2024-08-17T17:23:57.556729+0000","last_peered":"2024-08-17T17:23:57.556729+0000","last_clean":"2024-08-17T17:23:57.556729+0000","last_became_active":"2024-08-17T17:22:10.861509+0000","last_became_peered":"2024-08-17T17:22:10.861509+0000","last_unstale":"2024-08-17T17:23:57.556729+0000","last_undegraded":"2024-08-17T17:23:57.556729+0000","last_fullsized":"2024-08-17T17:23:57.556729+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7873,"log_dups_size":0,"ondisk_log_size":7873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:16:43.850909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":515440640,"num_objects":7865,"num_object_clones":0,"num_object_copies":23595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7873,"num_write_kb":503616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1165,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.486576+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:23:42.486576+0000","last_peered":"2024-08-17T17:23:42.486576+0000","last_clean":"2024-08-17T17:23:42.486576+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:23:42.486576+0000","last_undegraded":"2024-08-17T17:23:42.486576+0000","last_fullsized":"2024-08-17T17:23:42.486576+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1137,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.365303+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:23:42.365303+0000","last_peered":"2024-08-17T17:23:42.365303+0000","last_clean":"2024-08-17T17:23:42.365303+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:23:42.365303+0000","last_undegraded":"2024-08-17T17:23:42.365303+0000","last_fullsized":"2024-08-17T17:23:42.365303+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"1211'7959","reported_seq":7984,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.228078+0000","last_change":"2024-08-17T17:22:10.860210+0000","last_active":"2024-08-17T17:23:57.228078+0000","last_peered":"2024-08-17T17:23:57.228078+0000","last_clean":"2024-08-17T17:23:57.228078+0000","last_became_active":"2024-08-17T17:22:10.859854+0000","last_became_peered":"2024-08-17T17:22:10.859854+0000","last_unstale":"2024-08-17T17:23:57.228078+0000","last_undegraded":"2024-08-17T17:23:57.228078+0000","last_fullsized":"2024-08-17T17:23:57.228078+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7959,"log_dups_size":0,"ondisk_log_size":7959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:31.313667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":521601024,"num_objects":7959,"num_object_clones":0,"num_object_copies":23877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7959,"num_write_kb":509376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1166,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.486252+0000","last_change":"2024-08-17T17:21:06.540689+0000","last_active":"2024-08-17T17:23:42.486252+0000","last_peered":"2024-08-17T17:23:42.486252+0000","last_clean":"2024-08-17T17:23:42.486252+0000","last_became_active":"2024-08-17T17:21:06.461614+0000","last_became_peered":"2024-08-17T17:21:06.461614+0000","last_unstale":"2024-08-17T17:23:42.486252+0000","last_undegraded":"2024-08-17T17:23:42.486252+0000","last_fullsized":"2024-08-17T17:23:42.486252+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:11.696898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1189,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.492940+0000","last_change":"2024-08-17T17:21:06.470389+0000","last_active":"2024-08-17T17:23:42.492940+0000","last_peered":"2024-08-17T17:23:42.492940+0000","last_clean":"2024-08-17T17:23:42.492940+0000","last_became_active":"2024-08-17T17:21:06.469966+0000","last_became_peered":"2024-08-17T17:21:06.469966+0000","last_unstale":"2024-08-17T17:23:42.492940+0000","last_undegraded":"2024-08-17T17:23:42.492940+0000","last_fullsized":"2024-08-17T17:23:42.492940+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:56.140616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"1211'7869","reported_seq":7894,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.194608+0000","last_change":"2024-08-17T17:22:10.943999+0000","last_active":"2024-08-17T17:23:57.194608+0000","last_peered":"2024-08-17T17:23:57.194608+0000","last_clean":"2024-08-17T17:23:57.194608+0000","last_became_active":"2024-08-17T17:22:10.943661+0000","last_became_peered":"2024-08-17T17:22:10.943661+0000","last_unstale":"2024-08-17T17:23:57.194608+0000","last_undegraded":"2024-08-17T17:23:57.194608+0000","last_fullsized":"2024-08-17T17:23:57.194608+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7869,"log_dups_size":0,"ondisk_log_size":7869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:21:49.050541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":515702784,"num_objects":7869,"num_object_clones":0,"num_object_copies":23607,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7869,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7869,"num_write_kb":503616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1124,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.486213+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:23:42.486213+0000","last_peered":"2024-08-17T17:23:42.486213+0000","last_clean":"2024-08-17T17:23:42.486213+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:23:42.486213+0000","last_undegraded":"2024-08-17T17:23:42.486213+0000","last_fullsized":"2024-08-17T17:23:42.486213+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1171,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.354140+0000","last_change":"2024-08-17T17:21:06.446798+0000","last_active":"2024-08-17T17:23:42.354140+0000","last_peered":"2024-08-17T17:23:42.354140+0000","last_clean":"2024-08-17T17:23:42.354140+0000","last_became_active":"2024-08-17T17:21:06.446205+0000","last_became_peered":"2024-08-17T17:21:06.446205+0000","last_unstale":"2024-08-17T17:23:42.354140+0000","last_undegraded":"2024-08-17T17:23:42.354140+0000","last_fullsized":"2024-08-17T17:23:42.354140+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:28.584048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"1211'7955","reported_seq":7998,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:24:00.268259+0000","last_change":"2024-08-17T17:22:10.900862+0000","last_active":"2024-08-17T17:24:00.268259+0000","last_peered":"2024-08-17T17:24:00.268259+0000","last_clean":"2024-08-17T17:24:00.268259+0000","last_became_active":"2024-08-17T17:22:10.900552+0000","last_became_peered":"2024-08-17T17:22:10.900552+0000","last_unstale":"2024-08-17T17:24:00.268259+0000","last_undegraded":"2024-08-17T17:24:00.268259+0000","last_fullsized":"2024-08-17T17:24:00.268259+0000","mapping_epoch":1195,"log_start":"1200'800","ondisk_log_start":"1200'800","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7155,"log_dups_size":0,"ondisk_log_size":7155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:06.689517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":500367360,"num_objects":7635,"num_object_clones":0,"num_object_copies":22905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7955,"num_write_kb":498880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1153,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.487118+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:23:42.487118+0000","last_peered":"2024-08-17T17:23:42.487118+0000","last_clean":"2024-08-17T17:23:42.487118+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:23:42.487118+0000","last_undegraded":"2024-08-17T17:23:42.487118+0000","last_fullsized":"2024-08-17T17:23:42.487118+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":935,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.493263+0000","last_change":"2024-08-17T17:21:06.460905+0000","last_active":"2024-08-17T17:23:42.493263+0000","last_peered":"2024-08-17T17:23:42.493263+0000","last_clean":"2024-08-17T17:23:42.493263+0000","last_became_active":"2024-08-17T17:21:06.460592+0000","last_became_peered":"2024-08-17T17:21:06.460592+0000","last_unstale":"2024-08-17T17:23:42.493263+0000","last_undegraded":"2024-08-17T17:23:42.493263+0000","last_fullsized":"2024-08-17T17:23:42.493263+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:10.146916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.0","version":"1211'7877","reported_seq":7921,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:24:00.249038+0000","last_change":"2024-08-17T17:22:10.900154+0000","last_active":"2024-08-17T17:24:00.249038+0000","last_peered":"2024-08-17T17:24:00.249038+0000","last_clean":"2024-08-17T17:24:00.249038+0000","last_became_active":"2024-08-17T17:22:10.899766+0000","last_became_peered":"2024-08-17T17:22:10.899766+0000","last_unstale":"2024-08-17T17:24:00.249038+0000","last_undegraded":"2024-08-17T17:24:00.249038+0000","last_fullsized":"2024-08-17T17:24:00.249038+0000","mapping_epoch":1195,"log_start":"1200'700","ondisk_log_start":"1200'700","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7177,"log_dups_size":0,"ondisk_log_size":7177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:02:14.160943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":494469120,"num_objects":7545,"num_object_clones":0,"num_object_copies":22635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7877,"num_write_kb":493504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1532,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.473358+0000","last_change":"2024-08-17T17:21:16.100398+0000","last_active":"2024-08-17T17:23:42.473358+0000","last_peered":"2024-08-17T17:23:42.473358+0000","last_clean":"2024-08-17T17:23:42.473358+0000","last_became_active":"2024-08-17T17:18:28.521573+0000","last_became_peered":"2024-08-17T17:18:28.521573+0000","last_unstale":"2024-08-17T17:23:42.473358+0000","last_undegraded":"2024-08-17T17:23:42.473358+0000","last_fullsized":"2024-08-17T17:23:42.473358+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:21:16.100258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:21:16.100258+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:57.628353+0000","scrub_duration":0.011045358999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1539,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.415542+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:23:53.415542+0000","last_peered":"2024-08-17T17:23:53.415542+0000","last_clean":"2024-08-17T17:23:53.415542+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:23:53.415542+0000","last_undegraded":"2024-08-17T17:23:53.415542+0000","last_fullsized":"2024-08-17T17:23:53.415542+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1156,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.493447+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:23:42.493447+0000","last_peered":"2024-08-17T17:23:42.493447+0000","last_clean":"2024-08-17T17:23:42.493447+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:23:42.493447+0000","last_undegraded":"2024-08-17T17:23:42.493447+0000","last_fullsized":"2024-08-17T17:23:42.493447+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"1211'7699","reported_seq":7743,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.587034+0000","last_change":"2024-08-17T17:22:10.944783+0000","last_active":"2024-08-17T17:23:56.587034+0000","last_peered":"2024-08-17T17:23:56.587034+0000","last_clean":"2024-08-17T17:23:56.587034+0000","last_became_active":"2024-08-17T17:22:10.944487+0000","last_became_peered":"2024-08-17T17:22:10.944487+0000","last_unstale":"2024-08-17T17:23:56.587034+0000","last_undegraded":"2024-08-17T17:23:56.587034+0000","last_fullsized":"2024-08-17T17:23:56.587034+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7699,"log_dups_size":0,"ondisk_log_size":7699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:57:11.921301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":504561664,"num_objects":7699,"num_object_clones":0,"num_object_copies":23097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7699,"num_write_kb":492736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1478,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.365098+0000","last_change":"2024-08-17T17:18:32.884334+0000","last_active":"2024-08-17T17:23:42.365098+0000","last_peered":"2024-08-17T17:23:42.365098+0000","last_clean":"2024-08-17T17:23:42.365098+0000","last_became_active":"2024-08-17T17:18:28.520186+0000","last_became_peered":"2024-08-17T17:18:28.520186+0000","last_unstale":"2024-08-17T17:23:42.365098+0000","last_undegraded":"2024-08-17T17:23:42.365098+0000","last_fullsized":"2024-08-17T17:23:42.365098+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:32.884246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:32.884246+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:49.286656+0000","scrub_duration":0.001842677,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1172,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.354267+0000","last_change":"2024-08-17T17:21:06.446841+0000","last_active":"2024-08-17T17:23:42.354267+0000","last_peered":"2024-08-17T17:23:42.354267+0000","last_clean":"2024-08-17T17:23:42.354267+0000","last_became_active":"2024-08-17T17:21:06.446271+0000","last_became_peered":"2024-08-17T17:21:06.446271+0000","last_unstale":"2024-08-17T17:23:42.354267+0000","last_undegraded":"2024-08-17T17:23:42.354267+0000","last_fullsized":"2024-08-17T17:23:42.354267+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:20.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1189,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.493700+0000","last_change":"2024-08-17T17:21:06.480911+0000","last_active":"2024-08-17T17:23:42.493700+0000","last_peered":"2024-08-17T17:23:42.493700+0000","last_clean":"2024-08-17T17:23:42.493700+0000","last_became_active":"2024-08-17T17:21:06.480551+0000","last_became_peered":"2024-08-17T17:21:06.480551+0000","last_unstale":"2024-08-17T17:23:42.493700+0000","last_undegraded":"2024-08-17T17:23:42.493700+0000","last_fullsized":"2024-08-17T17:23:42.493700+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:26.003704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"1211'8062","reported_seq":8087,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.232458+0000","last_change":"2024-08-17T17:22:10.944020+0000","last_active":"2024-08-17T17:23:57.232458+0000","last_peered":"2024-08-17T17:23:57.232458+0000","last_clean":"2024-08-17T17:23:57.232458+0000","last_became_active":"2024-08-17T17:22:10.943694+0000","last_became_peered":"2024-08-17T17:22:10.943694+0000","last_unstale":"2024-08-17T17:23:57.232458+0000","last_undegraded":"2024-08-17T17:23:57.232458+0000","last_fullsized":"2024-08-17T17:23:57.232458+0000","mapping_epoch":1195,"log_start":"1200'700","ondisk_log_start":"1200'700","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7362,"log_dups_size":0,"ondisk_log_size":7362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:34.525383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":528351232,"num_objects":8062,"num_object_clones":0,"num_object_copies":24186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8062,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8062,"num_write_kb":515968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1157,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.493015+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:23:42.493015+0000","last_peered":"2024-08-17T17:23:42.493015+0000","last_clean":"2024-08-17T17:23:42.493015+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:23:42.493015+0000","last_undegraded":"2024-08-17T17:23:42.493015+0000","last_fullsized":"2024-08-17T17:23:42.493015+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1174,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.493844+0000","last_change":"2024-08-17T17:21:06.451521+0000","last_active":"2024-08-17T17:23:42.493844+0000","last_peered":"2024-08-17T17:23:42.493844+0000","last_clean":"2024-08-17T17:23:42.493844+0000","last_became_active":"2024-08-17T17:21:06.451197+0000","last_became_peered":"2024-08-17T17:21:06.451197+0000","last_unstale":"2024-08-17T17:23:42.493844+0000","last_undegraded":"2024-08-17T17:23:42.493844+0000","last_fullsized":"2024-08-17T17:23:42.493844+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:04.000692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"1211'7993","reported_seq":8018,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.528877+0000","last_change":"2024-08-17T17:22:10.900282+0000","last_active":"2024-08-17T17:23:57.528877+0000","last_peered":"2024-08-17T17:23:57.528877+0000","last_clean":"2024-08-17T17:23:57.528877+0000","last_became_active":"2024-08-17T17:22:10.899947+0000","last_became_peered":"2024-08-17T17:22:10.899947+0000","last_unstale":"2024-08-17T17:23:57.528877+0000","last_undegraded":"2024-08-17T17:23:57.528877+0000","last_fullsized":"2024-08-17T17:23:57.528877+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7993,"log_dups_size":0,"ondisk_log_size":7993,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:10.148524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":523567104,"num_objects":7989,"num_object_clones":0,"num_object_copies":23967,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7989,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7993,"num_write_kb":511424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1478,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.486579+0000","last_change":"2024-08-17T17:18:31.850829+0000","last_active":"2024-08-17T17:23:42.486579+0000","last_peered":"2024-08-17T17:23:42.486579+0000","last_clean":"2024-08-17T17:23:42.486579+0000","last_became_active":"2024-08-17T17:18:28.520946+0000","last_became_peered":"2024-08-17T17:18:28.520946+0000","last_unstale":"2024-08-17T17:23:42.486579+0000","last_undegraded":"2024-08-17T17:23:42.486579+0000","last_fullsized":"2024-08-17T17:23:42.486579+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:31.850745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:31.850745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:17.218084+0000","scrub_duration":0.001317447,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1520,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.414270+0000","last_change":"2024-08-17T17:21:06.441118+0000","last_active":"2024-08-17T17:23:53.414270+0000","last_peered":"2024-08-17T17:23:53.414270+0000","last_clean":"2024-08-17T17:23:53.414270+0000","last_became_active":"2024-08-17T17:21:06.440800+0000","last_became_peered":"2024-08-17T17:21:06.440800+0000","last_unstale":"2024-08-17T17:23:53.414270+0000","last_undegraded":"2024-08-17T17:23:53.414270+0000","last_fullsized":"2024-08-17T17:23:53.414270+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:08.091528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":26,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.420977+0000","last_change":"2024-08-17T17:22:48.374766+0000","last_active":"2024-08-17T17:23:53.420977+0000","last_peered":"2024-08-17T17:23:53.420977+0000","last_clean":"2024-08-17T17:23:53.420977+0000","last_became_active":"2024-08-17T17:22:48.373556+0000","last_became_peered":"2024-08-17T17:22:48.373556+0000","last_unstale":"2024-08-17T17:23:53.420977+0000","last_undegraded":"2024-08-17T17:23:53.420977+0000","last_fullsized":"2024-08-17T17:23:53.420977+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:33.362080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.9","version":"1211'7944","reported_seq":7969,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.237356+0000","last_change":"2024-08-17T17:22:10.944057+0000","last_active":"2024-08-17T17:23:57.237356+0000","last_peered":"2024-08-17T17:23:57.237356+0000","last_clean":"2024-08-17T17:23:57.237356+0000","last_became_active":"2024-08-17T17:22:10.943681+0000","last_became_peered":"2024-08-17T17:22:10.943681+0000","last_unstale":"2024-08-17T17:23:57.237356+0000","last_undegraded":"2024-08-17T17:23:57.237356+0000","last_fullsized":"2024-08-17T17:23:57.237356+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7944,"log_dups_size":0,"ondisk_log_size":7944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:24.524487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":520617984,"num_objects":7944,"num_object_clones":0,"num_object_copies":23832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7944,"num_write_kb":508416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1532,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.473729+0000","last_change":"2024-08-17T17:18:33.814375+0000","last_active":"2024-08-17T17:23:42.473729+0000","last_peered":"2024-08-17T17:23:42.473729+0000","last_clean":"2024-08-17T17:23:42.473729+0000","last_became_active":"2024-08-17T17:18:33.813208+0000","last_became_peered":"2024-08-17T17:18:33.813208+0000","last_unstale":"2024-08-17T17:23:42.473729+0000","last_undegraded":"2024-08-17T17:23:42.473729+0000","last_fullsized":"2024-08-17T17:23:42.473729+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:25.057675+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:25.057675+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:20:45.235879+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1168,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.510744+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:23:42.510744+0000","last_peered":"2024-08-17T17:23:42.510744+0000","last_clean":"2024-08-17T17:23:42.510744+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:23:42.510744+0000","last_undegraded":"2024-08-17T17:23:42.510744+0000","last_fullsized":"2024-08-17T17:23:42.510744+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1485,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.412673+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:23:53.412673+0000","last_peered":"2024-08-17T17:23:53.412673+0000","last_clean":"2024-08-17T17:23:53.412673+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:23:53.412673+0000","last_undegraded":"2024-08-17T17:23:53.412673+0000","last_fullsized":"2024-08-17T17:23:53.412673+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1139,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.354584+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:23:42.354584+0000","last_peered":"2024-08-17T17:23:42.354584+0000","last_clean":"2024-08-17T17:23:42.354584+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:23:42.354584+0000","last_undegraded":"2024-08-17T17:23:42.354584+0000","last_fullsized":"2024-08-17T17:23:42.354584+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"1211'7951","reported_seq":7993,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.592761+0000","last_change":"2024-08-17T17:22:10.900900+0000","last_active":"2024-08-17T17:23:56.592761+0000","last_peered":"2024-08-17T17:23:56.592761+0000","last_clean":"2024-08-17T17:23:56.592761+0000","last_became_active":"2024-08-17T17:22:10.899812+0000","last_became_peered":"2024-08-17T17:22:10.899812+0000","last_unstale":"2024-08-17T17:23:56.592761+0000","last_undegraded":"2024-08-17T17:23:56.592761+0000","last_fullsized":"2024-08-17T17:23:56.592761+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7951,"log_dups_size":0,"ondisk_log_size":7951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:49:09.639747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":521076736,"num_objects":7951,"num_object_clones":0,"num_object_copies":23853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7951,"num_write_kb":508864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":771,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.354425+0000","last_change":"2024-08-17T17:22:09.230320+0000","last_active":"2024-08-17T17:23:42.354425+0000","last_peered":"2024-08-17T17:23:42.354425+0000","last_clean":"2024-08-17T17:23:42.354425+0000","last_became_active":"2024-08-17T17:21:08.910463+0000","last_became_peered":"2024-08-17T17:21:08.910463+0000","last_unstale":"2024-08-17T17:23:42.354425+0000","last_undegraded":"2024-08-17T17:23:42.354425+0000","last_fullsized":"2024-08-17T17:23:42.354425+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1177,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:09.230214+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.230214+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:47:58.582858+0000","scrub_duration":0.001729597,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1203,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.473651+0000","last_change":"2024-08-17T17:21:06.430639+0000","last_active":"2024-08-17T17:23:42.473651+0000","last_peered":"2024-08-17T17:23:42.473651+0000","last_clean":"2024-08-17T17:23:42.473651+0000","last_became_active":"2024-08-17T17:21:06.430297+0000","last_became_peered":"2024-08-17T17:21:06.430297+0000","last_unstale":"2024-08-17T17:23:42.473651+0000","last_undegraded":"2024-08-17T17:23:42.473651+0000","last_fullsized":"2024-08-17T17:23:42.473651+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:57.794620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1007,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.492667+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:23:42.492667+0000","last_peered":"2024-08-17T17:23:42.492667+0000","last_clean":"2024-08-17T17:23:42.492667+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:23:42.492667+0000","last_undegraded":"2024-08-17T17:23:42.492667+0000","last_fullsized":"2024-08-17T17:23:42.492667+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":18,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.486760+0000","last_change":"2024-08-17T17:22:48.370769+0000","last_active":"2024-08-17T17:23:42.486760+0000","last_peered":"2024-08-17T17:23:42.486760+0000","last_clean":"2024-08-17T17:23:42.486760+0000","last_became_active":"2024-08-17T17:22:48.370531+0000","last_became_peered":"2024-08-17T17:22:48.370531+0000","last_unstale":"2024-08-17T17:23:42.486760+0000","last_undegraded":"2024-08-17T17:23:42.486760+0000","last_fullsized":"2024-08-17T17:23:42.486760+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:02.943823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.b","version":"1211'8048","reported_seq":8092,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:24:00.226327+0000","last_change":"2024-08-17T17:22:10.900740+0000","last_active":"2024-08-17T17:24:00.226327+0000","last_peered":"2024-08-17T17:24:00.226327+0000","last_clean":"2024-08-17T17:24:00.226327+0000","last_became_active":"2024-08-17T17:22:10.900452+0000","last_became_peered":"2024-08-17T17:22:10.900452+0000","last_unstale":"2024-08-17T17:24:00.226327+0000","last_undegraded":"2024-08-17T17:24:00.226327+0000","last_fullsized":"2024-08-17T17:24:00.226327+0000","mapping_epoch":1195,"log_start":"1200'900","ondisk_log_start":"1200'900","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7148,"log_dups_size":0,"ondisk_log_size":7148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:49:25.950741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":507379712,"num_objects":7742,"num_object_clones":0,"num_object_copies":23226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8048,"num_write_kb":505280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1553,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.473429+0000","last_change":"2024-08-17T17:22:07.078517+0000","last_active":"2024-08-17T17:23:42.473429+0000","last_peered":"2024-08-17T17:23:42.473429+0000","last_clean":"2024-08-17T17:23:42.473429+0000","last_became_active":"2024-08-17T17:18:33.816660+0000","last_became_peered":"2024-08-17T17:18:33.816660+0000","last_unstale":"2024-08-17T17:23:42.473429+0000","last_undegraded":"2024-08-17T17:23:42.473429+0000","last_fullsized":"2024-08-17T17:23:42.473429+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1147,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:07.078434+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:07.078434+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:43:56.205556+0000","scrub_duration":0.0013001919999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1583,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.420018+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:23:53.420018+0000","last_peered":"2024-08-17T17:23:53.420018+0000","last_clean":"2024-08-17T17:23:53.420018+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:23:53.420018+0000","last_undegraded":"2024-08-17T17:23:53.420018+0000","last_fullsized":"2024-08-17T17:23:53.420018+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":975,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.419150+0000","last_change":"2024-08-17T17:21:06.428684+0000","last_active":"2024-08-17T17:23:53.419150+0000","last_peered":"2024-08-17T17:23:53.419150+0000","last_clean":"2024-08-17T17:23:53.419150+0000","last_became_active":"2024-08-17T17:21:06.428178+0000","last_became_peered":"2024-08-17T17:21:06.428178+0000","last_unstale":"2024-08-17T17:23:53.419150+0000","last_undegraded":"2024-08-17T17:23:53.419150+0000","last_fullsized":"2024-08-17T17:23:53.419150+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:14.836119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":850,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.486490+0000","last_change":"2024-08-17T17:21:06.443565+0000","last_active":"2024-08-17T17:23:42.486490+0000","last_peered":"2024-08-17T17:23:42.486490+0000","last_clean":"2024-08-17T17:23:42.486490+0000","last_became_active":"2024-08-17T17:21:06.443215+0000","last_became_peered":"2024-08-17T17:21:06.443215+0000","last_unstale":"2024-08-17T17:23:42.486490+0000","last_undegraded":"2024-08-17T17:23:42.486490+0000","last_fullsized":"2024-08-17T17:23:42.486490+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:47.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"1211'7945","reported_seq":7970,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.539598+0000","last_change":"2024-08-17T17:22:10.859676+0000","last_active":"2024-08-17T17:23:57.539598+0000","last_peered":"2024-08-17T17:23:57.539598+0000","last_clean":"2024-08-17T17:23:57.539598+0000","last_became_active":"2024-08-17T17:22:10.859289+0000","last_became_peered":"2024-08-17T17:22:10.859289+0000","last_unstale":"2024-08-17T17:23:57.539598+0000","last_undegraded":"2024-08-17T17:23:57.539598+0000","last_fullsized":"2024-08-17T17:23:57.539598+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7945,"log_dups_size":0,"ondisk_log_size":7945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:43.302993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":520290304,"num_objects":7939,"num_object_clones":0,"num_object_copies":23817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7945,"num_write_kb":508288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1495,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.365152+0000","last_change":"2024-08-17T17:18:35.916482+0000","last_active":"2024-08-17T17:23:42.365152+0000","last_peered":"2024-08-17T17:23:42.365152+0000","last_clean":"2024-08-17T17:23:42.365152+0000","last_became_active":"2024-08-17T17:18:34.827747+0000","last_became_peered":"2024-08-17T17:18:34.827747+0000","last_unstale":"2024-08-17T17:23:42.365152+0000","last_undegraded":"2024-08-17T17:23:42.365152+0000","last_fullsized":"2024-08-17T17:23:42.365152+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:35.916381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:35.916381+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:28.492622+0000","scrub_duration":0.005340521,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1195,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.505828+0000","last_change":"2024-08-17T17:21:06.456978+0000","last_active":"2024-08-17T17:23:42.505828+0000","last_peered":"2024-08-17T17:23:42.505828+0000","last_clean":"2024-08-17T17:23:42.505828+0000","last_became_active":"2024-08-17T17:21:06.452265+0000","last_became_peered":"2024-08-17T17:21:06.452265+0000","last_unstale":"2024-08-17T17:23:42.505828+0000","last_undegraded":"2024-08-17T17:23:42.505828+0000","last_fullsized":"2024-08-17T17:23:42.505828+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:52.893746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1176,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.494149+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:23:42.494149+0000","last_peered":"2024-08-17T17:23:42.494149+0000","last_clean":"2024-08-17T17:23:42.494149+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:23:42.494149+0000","last_undegraded":"2024-08-17T17:23:42.494149+0000","last_fullsized":"2024-08-17T17:23:42.494149+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1050,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.423054+0000","last_change":"2024-08-17T17:21:06.435754+0000","last_active":"2024-08-17T17:23:53.423054+0000","last_peered":"2024-08-17T17:23:53.423054+0000","last_clean":"2024-08-17T17:23:53.423054+0000","last_became_active":"2024-08-17T17:21:06.435442+0000","last_became_peered":"2024-08-17T17:21:06.435442+0000","last_unstale":"2024-08-17T17:23:53.423054+0000","last_undegraded":"2024-08-17T17:23:53.423054+0000","last_fullsized":"2024-08-17T17:23:53.423054+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:34.849042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"1211'7830","reported_seq":7874,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:56.614587+0000","last_change":"2024-08-17T17:22:10.944986+0000","last_active":"2024-08-17T17:23:56.614587+0000","last_peered":"2024-08-17T17:23:56.614587+0000","last_clean":"2024-08-17T17:23:56.614587+0000","last_became_active":"2024-08-17T17:22:10.944267+0000","last_became_peered":"2024-08-17T17:22:10.944267+0000","last_unstale":"2024-08-17T17:23:56.614587+0000","last_undegraded":"2024-08-17T17:23:56.614587+0000","last_fullsized":"2024-08-17T17:23:56.614587+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7830,"log_dups_size":0,"ondisk_log_size":7830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:05:11.441347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":513146880,"num_objects":7830,"num_object_clones":0,"num_object_copies":23490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7830,"num_write_kb":501120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":737,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.470813+0000","last_change":"2024-08-17T17:22:06.152683+0000","last_active":"2024-08-17T17:23:42.470813+0000","last_peered":"2024-08-17T17:23:42.470813+0000","last_clean":"2024-08-17T17:23:42.470813+0000","last_became_active":"2024-08-17T17:18:28.526439+0000","last_became_peered":"2024-08-17T17:18:28.526439+0000","last_unstale":"2024-08-17T17:23:42.470813+0000","last_undegraded":"2024-08-17T17:23:42.470813+0000","last_fullsized":"2024-08-17T17:23:42.470813+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:06.152592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:06.152592+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:30.807010+0000","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1169,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.486900+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:23:42.486900+0000","last_peered":"2024-08-17T17:23:42.486900+0000","last_clean":"2024-08-17T17:23:42.486900+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:23:42.486900+0000","last_undegraded":"2024-08-17T17:23:42.486900+0000","last_fullsized":"2024-08-17T17:23:42.486900+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":859,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.473687+0000","last_change":"2024-08-17T17:21:06.434991+0000","last_active":"2024-08-17T17:23:42.473687+0000","last_peered":"2024-08-17T17:23:42.473687+0000","last_clean":"2024-08-17T17:23:42.473687+0000","last_became_active":"2024-08-17T17:21:06.434630+0000","last_became_peered":"2024-08-17T17:21:06.434630+0000","last_unstale":"2024-08-17T17:23:42.473687+0000","last_undegraded":"2024-08-17T17:23:42.473687+0000","last_fullsized":"2024-08-17T17:23:42.473687+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:24.104609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":801,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.365278+0000","last_change":"2024-08-17T17:21:06.462532+0000","last_active":"2024-08-17T17:23:42.365278+0000","last_peered":"2024-08-17T17:23:42.365278+0000","last_clean":"2024-08-17T17:23:42.365278+0000","last_became_active":"2024-08-17T17:21:06.462183+0000","last_became_peered":"2024-08-17T17:21:06.462183+0000","last_unstale":"2024-08-17T17:23:42.365278+0000","last_undegraded":"2024-08-17T17:23:42.365278+0000","last_fullsized":"2024-08-17T17:23:42.365278+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:50.033322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.f","version":"1211'8058","reported_seq":8102,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:24:00.280853+0000","last_change":"2024-08-17T17:22:10.900592+0000","last_active":"2024-08-17T17:24:00.280853+0000","last_peered":"2024-08-17T17:24:00.280853+0000","last_clean":"2024-08-17T17:24:00.280853+0000","last_became_active":"2024-08-17T17:22:10.900279+0000","last_became_peered":"2024-08-17T17:22:10.900279+0000","last_unstale":"2024-08-17T17:24:00.280853+0000","last_undegraded":"2024-08-17T17:24:00.280853+0000","last_fullsized":"2024-08-17T17:24:00.280853+0000","mapping_epoch":1195,"log_start":"1200'900","ondisk_log_start":"1200'900","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7158,"log_dups_size":0,"ondisk_log_size":7158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:49:23.819372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":504889344,"num_objects":7704,"num_object_clones":0,"num_object_copies":23112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8058,"num_write_kb":504384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":754,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.471280+0000","last_change":"2024-08-17T17:22:08.218836+0000","last_active":"2024-08-17T17:23:42.471280+0000","last_peered":"2024-08-17T17:23:42.471280+0000","last_clean":"2024-08-17T17:23:42.471280+0000","last_became_active":"2024-08-17T17:18:34.831498+0000","last_became_peered":"2024-08-17T17:18:34.831498+0000","last_unstale":"2024-08-17T17:23:42.471280+0000","last_undegraded":"2024-08-17T17:23:42.471280+0000","last_fullsized":"2024-08-17T17:23:42.471280+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1148,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:08.218709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:08.218709+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:28:19.097715+0000","scrub_duration":0.002239024,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":973,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.422956+0000","last_change":"2024-08-17T17:21:06.433720+0000","last_active":"2024-08-17T17:23:53.422956+0000","last_peered":"2024-08-17T17:23:53.422956+0000","last_clean":"2024-08-17T17:23:53.422956+0000","last_became_active":"2024-08-17T17:21:06.433292+0000","last_became_peered":"2024-08-17T17:21:06.433292+0000","last_unstale":"2024-08-17T17:23:53.422956+0000","last_undegraded":"2024-08-17T17:23:53.422956+0000","last_fullsized":"2024-08-17T17:23:53.422956+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:32.246594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1170,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:47.348363+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:23:47.348363+0000","last_peered":"2024-08-17T17:23:47.348363+0000","last_clean":"2024-08-17T17:23:47.348363+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:23:47.348363+0000","last_undegraded":"2024-08-17T17:23:47.348363+0000","last_fullsized":"2024-08-17T17:23:47.348363+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":762,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.486416+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:23:42.486416+0000","last_peered":"2024-08-17T17:23:42.486416+0000","last_clean":"2024-08-17T17:23:42.486416+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:23:42.486416+0000","last_undegraded":"2024-08-17T17:23:42.486416+0000","last_fullsized":"2024-08-17T17:23:42.486416+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"1211'8195","reported_seq":8239,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:24:00.227588+0000","last_change":"2024-08-17T17:22:10.944050+0000","last_active":"2024-08-17T17:24:00.227588+0000","last_peered":"2024-08-17T17:24:00.227588+0000","last_clean":"2024-08-17T17:24:00.227588+0000","last_became_active":"2024-08-17T17:22:10.943712+0000","last_became_peered":"2024-08-17T17:22:10.943712+0000","last_unstale":"2024-08-17T17:24:00.227588+0000","last_undegraded":"2024-08-17T17:24:00.227588+0000","last_fullsized":"2024-08-17T17:24:00.227588+0000","mapping_epoch":1195,"log_start":"1200'1000","ondisk_log_start":"1200'1000","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7195,"log_dups_size":0,"ondisk_log_size":7195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:13.015652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":516882432,"num_objects":7887,"num_object_clones":0,"num_object_copies":23661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8195,"num_write_kb":514624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1478,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.493846+0000","last_change":"2024-08-17T17:22:06.995885+0000","last_active":"2024-08-17T17:23:42.493846+0000","last_peered":"2024-08-17T17:23:42.493846+0000","last_clean":"2024-08-17T17:23:42.493846+0000","last_became_active":"2024-08-17T17:18:28.526484+0000","last_became_peered":"2024-08-17T17:18:28.526484+0000","last_unstale":"2024-08-17T17:23:42.493846+0000","last_undegraded":"2024-08-17T17:23:42.493846+0000","last_fullsized":"2024-08-17T17:23:42.493846+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:06.995792+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:06.995792+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:06.341398+0000","scrub_duration":0.0016139520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1539,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.419235+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:23:53.419235+0000","last_peered":"2024-08-17T17:23:53.419235+0000","last_clean":"2024-08-17T17:23:53.419235+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:23:53.419235+0000","last_undegraded":"2024-08-17T17:23:53.419235+0000","last_fullsized":"2024-08-17T17:23:53.419235+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":811,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.471506+0000","last_change":"2024-08-17T17:21:06.452395+0000","last_active":"2024-08-17T17:23:42.471506+0000","last_peered":"2024-08-17T17:23:42.471506+0000","last_clean":"2024-08-17T17:23:42.471506+0000","last_became_active":"2024-08-17T17:21:06.451867+0000","last_became_peered":"2024-08-17T17:21:06.451867+0000","last_unstale":"2024-08-17T17:23:42.471506+0000","last_undegraded":"2024-08-17T17:23:42.471506+0000","last_fullsized":"2024-08-17T17:23:42.471506+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:50.597092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1529,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.414187+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:23:53.414187+0000","last_peered":"2024-08-17T17:23:53.414187+0000","last_clean":"2024-08-17T17:23:53.414187+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:23:53.414187+0000","last_undegraded":"2024-08-17T17:23:53.414187+0000","last_fullsized":"2024-08-17T17:23:53.414187+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"1211'7876","reported_seq":7902,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:57.545500+0000","last_change":"2024-08-17T17:22:10.859993+0000","last_active":"2024-08-17T17:23:57.545500+0000","last_peered":"2024-08-17T17:23:57.545500+0000","last_clean":"2024-08-17T17:23:57.545500+0000","last_became_active":"2024-08-17T17:22:10.859296+0000","last_became_peered":"2024-08-17T17:22:10.859296+0000","last_unstale":"2024-08-17T17:23:57.545500+0000","last_undegraded":"2024-08-17T17:23:57.545500+0000","last_fullsized":"2024-08-17T17:23:57.545500+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7876,"log_dups_size":0,"ondisk_log_size":7876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:42.095858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":515571736,"num_objects":7868,"num_object_clones":0,"num_object_copies":23604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7868,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7876,"num_write_kb":503745,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1544,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.412784+0000","last_change":"2024-08-17T17:18:29.390118+0000","last_active":"2024-08-17T17:23:53.412784+0000","last_peered":"2024-08-17T17:23:53.412784+0000","last_clean":"2024-08-17T17:23:53.412784+0000","last_became_active":"2024-08-17T17:18:28.528874+0000","last_became_peered":"2024-08-17T17:18:28.528874+0000","last_unstale":"2024-08-17T17:23:53.412784+0000","last_undegraded":"2024-08-17T17:23:53.412784+0000","last_fullsized":"2024-08-17T17:23:53.412784+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:29.390003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:29.390003+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:26:35.908604+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":180,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.493776+0000","last_change":"2024-08-17T17:21:06.519965+0000","last_active":"2024-08-17T17:23:42.493776+0000","last_peered":"2024-08-17T17:23:42.493776+0000","last_clean":"2024-08-17T17:23:42.493776+0000","last_became_active":"2024-08-17T17:21:06.459952+0000","last_became_peered":"2024-08-17T17:21:06.459952+0000","last_unstale":"2024-08-17T17:23:42.493776+0000","last_undegraded":"2024-08-17T17:23:42.493776+0000","last_fullsized":"2024-08-17T17:23:42.493776+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:27.828098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1166,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.471017+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:23:42.471017+0000","last_peered":"2024-08-17T17:23:42.471017+0000","last_clean":"2024-08-17T17:23:42.471017+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:23:42.471017+0000","last_undegraded":"2024-08-17T17:23:42.471017+0000","last_fullsized":"2024-08-17T17:23:42.471017+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1142,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.492752+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:23:42.492752+0000","last_peered":"2024-08-17T17:23:42.492752+0000","last_clean":"2024-08-17T17:23:42.492752+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:23:42.492752+0000","last_undegraded":"2024-08-17T17:23:42.492752+0000","last_fullsized":"2024-08-17T17:23:42.492752+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1173,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.493588+0000","last_change":"2024-08-17T17:21:06.458000+0000","last_active":"2024-08-17T17:23:42.493588+0000","last_peered":"2024-08-17T17:23:42.493588+0000","last_clean":"2024-08-17T17:23:42.493588+0000","last_became_active":"2024-08-17T17:21:06.457411+0000","last_became_peered":"2024-08-17T17:21:06.457411+0000","last_unstale":"2024-08-17T17:23:42.493588+0000","last_undegraded":"2024-08-17T17:23:42.493588+0000","last_fullsized":"2024-08-17T17:23:42.493588+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:52.740639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1164,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.493628+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:23:42.493628+0000","last_peered":"2024-08-17T17:23:42.493628+0000","last_clean":"2024-08-17T17:23:42.493628+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:23:42.493628+0000","last_undegraded":"2024-08-17T17:23:42.493628+0000","last_fullsized":"2024-08-17T17:23:42.493628+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":834,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.354202+0000","last_change":"2024-08-17T17:21:06.454587+0000","last_active":"2024-08-17T17:23:42.354202+0000","last_peered":"2024-08-17T17:23:42.354202+0000","last_clean":"2024-08-17T17:23:42.354202+0000","last_became_active":"2024-08-17T17:21:06.454229+0000","last_became_peered":"2024-08-17T17:21:06.454229+0000","last_unstale":"2024-08-17T17:23:42.354202+0000","last_undegraded":"2024-08-17T17:23:42.354202+0000","last_fullsized":"2024-08-17T17:23:42.354202+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:38.340263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1140,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.486673+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:23:42.486673+0000","last_peered":"2024-08-17T17:23:42.486673+0000","last_clean":"2024-08-17T17:23:42.486673+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:23:42.486673+0000","last_undegraded":"2024-08-17T17:23:42.486673+0000","last_fullsized":"2024-08-17T17:23:42.486673+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1170,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.486387+0000","last_change":"2024-08-17T17:21:06.442796+0000","last_active":"2024-08-17T17:23:42.486387+0000","last_peered":"2024-08-17T17:23:42.486387+0000","last_clean":"2024-08-17T17:23:42.486387+0000","last_became_active":"2024-08-17T17:21:06.442466+0000","last_became_peered":"2024-08-17T17:21:06.442466+0000","last_unstale":"2024-08-17T17:23:42.486387+0000","last_undegraded":"2024-08-17T17:23:42.486387+0000","last_fullsized":"2024-08-17T17:23:42.486387+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:36.366620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1484,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.412045+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:23:53.412045+0000","last_peered":"2024-08-17T17:23:53.412045+0000","last_clean":"2024-08-17T17:23:53.412045+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:23:53.412045+0000","last_undegraded":"2024-08-17T17:23:53.412045+0000","last_fullsized":"2024-08-17T17:23:53.412045+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1006,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.486718+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:23:42.486718+0000","last_peered":"2024-08-17T17:23:42.486718+0000","last_clean":"2024-08-17T17:23:42.486718+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:23:42.486718+0000","last_undegraded":"2024-08-17T17:23:42.486718+0000","last_fullsized":"2024-08-17T17:23:42.486718+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1517,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.414130+0000","last_change":"2024-08-17T17:21:06.468788+0000","last_active":"2024-08-17T17:23:53.414130+0000","last_peered":"2024-08-17T17:23:53.414130+0000","last_clean":"2024-08-17T17:23:53.414130+0000","last_became_active":"2024-08-17T17:21:06.467141+0000","last_became_peered":"2024-08-17T17:21:06.467141+0000","last_unstale":"2024-08-17T17:23:53.414130+0000","last_undegraded":"2024-08-17T17:23:53.414130+0000","last_fullsized":"2024-08-17T17:23:53.414130+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:51.301359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1172,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.471399+0000","last_change":"2024-08-17T17:21:06.435998+0000","last_active":"2024-08-17T17:23:42.471399+0000","last_peered":"2024-08-17T17:23:42.471399+0000","last_clean":"2024-08-17T17:23:42.471399+0000","last_became_active":"2024-08-17T17:21:06.435579+0000","last_became_peered":"2024-08-17T17:21:06.435579+0000","last_unstale":"2024-08-17T17:23:42.471399+0000","last_undegraded":"2024-08-17T17:23:42.471399+0000","last_fullsized":"2024-08-17T17:23:42.471399+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:01.743074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1547,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.414218+0000","last_change":"2024-08-17T17:21:06.473627+0000","last_active":"2024-08-17T17:23:53.414218+0000","last_peered":"2024-08-17T17:23:53.414218+0000","last_clean":"2024-08-17T17:23:53.414218+0000","last_became_active":"2024-08-17T17:21:06.473062+0000","last_became_peered":"2024-08-17T17:21:06.473062+0000","last_unstale":"2024-08-17T17:23:53.414218+0000","last_undegraded":"2024-08-17T17:23:53.414218+0000","last_fullsized":"2024-08-17T17:23:53.414218+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:21.413163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":967,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.486167+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:23:42.486167+0000","last_peered":"2024-08-17T17:23:42.486167+0000","last_clean":"2024-08-17T17:23:42.486167+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:23:42.486167+0000","last_undegraded":"2024-08-17T17:23:42.486167+0000","last_fullsized":"2024-08-17T17:23:42.486167+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1038,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.412250+0000","last_change":"2024-08-17T17:21:06.471265+0000","last_active":"2024-08-17T17:23:53.412250+0000","last_peered":"2024-08-17T17:23:53.412250+0000","last_clean":"2024-08-17T17:23:53.412250+0000","last_became_active":"2024-08-17T17:21:06.470943+0000","last_became_peered":"2024-08-17T17:21:06.470943+0000","last_unstale":"2024-08-17T17:23:53.412250+0000","last_undegraded":"2024-08-17T17:23:53.412250+0000","last_fullsized":"2024-08-17T17:23:53.412250+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:41.070024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1524,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.412106+0000","last_change":"2024-08-17T17:22:48.377351+0000","last_active":"2024-08-17T17:23:53.412106+0000","last_peered":"2024-08-17T17:23:53.412106+0000","last_clean":"2024-08-17T17:23:53.412106+0000","last_became_active":"2024-08-17T17:22:48.377059+0000","last_became_peered":"2024-08-17T17:22:48.377059+0000","last_unstale":"2024-08-17T17:23:53.412106+0000","last_undegraded":"2024-08-17T17:23:53.412106+0000","last_fullsized":"2024-08-17T17:23:53.412106+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:15.250902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1190,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.493343+0000","last_change":"2024-08-17T17:21:06.459139+0000","last_active":"2024-08-17T17:23:42.493343+0000","last_peered":"2024-08-17T17:23:42.493343+0000","last_clean":"2024-08-17T17:23:42.493343+0000","last_became_active":"2024-08-17T17:21:06.457850+0000","last_became_peered":"2024-08-17T17:21:06.457850+0000","last_unstale":"2024-08-17T17:23:42.493343+0000","last_undegraded":"2024-08-17T17:23:42.493343+0000","last_fullsized":"2024-08-17T17:23:42.493343+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:13.888576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1565,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.423028+0000","last_change":"2024-08-17T17:21:06.446321+0000","last_active":"2024-08-17T17:23:53.423028+0000","last_peered":"2024-08-17T17:23:53.423028+0000","last_clean":"2024-08-17T17:23:53.423028+0000","last_became_active":"2024-08-17T17:21:06.446019+0000","last_became_peered":"2024-08-17T17:21:06.446019+0000","last_unstale":"2024-08-17T17:23:53.423028+0000","last_undegraded":"2024-08-17T17:23:53.423028+0000","last_fullsized":"2024-08-17T17:23:53.423028+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:26.750070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":159,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.493368+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:23:42.493368+0000","last_peered":"2024-08-17T17:23:42.493368+0000","last_clean":"2024-08-17T17:23:42.493368+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:23:42.493368+0000","last_undegraded":"2024-08-17T17:23:42.493368+0000","last_fullsized":"2024-08-17T17:23:42.493368+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1188,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.493510+0000","last_change":"2024-08-17T17:21:06.465804+0000","last_active":"2024-08-17T17:23:42.493510+0000","last_peered":"2024-08-17T17:23:42.493510+0000","last_clean":"2024-08-17T17:23:42.493510+0000","last_became_active":"2024-08-17T17:21:06.465461+0000","last_became_peered":"2024-08-17T17:21:06.465461+0000","last_unstale":"2024-08-17T17:23:42.493510+0000","last_undegraded":"2024-08-17T17:23:42.493510+0000","last_fullsized":"2024-08-17T17:23:42.493510+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:06.443667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1180,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.487038+0000","last_change":"2024-08-17T17:21:06.481634+0000","last_active":"2024-08-17T17:23:42.487038+0000","last_peered":"2024-08-17T17:23:42.487038+0000","last_clean":"2024-08-17T17:23:42.487038+0000","last_became_active":"2024-08-17T17:21:06.469081+0000","last_became_peered":"2024-08-17T17:21:06.469081+0000","last_unstale":"2024-08-17T17:23:42.487038+0000","last_undegraded":"2024-08-17T17:23:42.487038+0000","last_fullsized":"2024-08-17T17:23:42.487038+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:53:26.954738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1188,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.493192+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:23:42.493192+0000","last_peered":"2024-08-17T17:23:42.493192+0000","last_clean":"2024-08-17T17:23:42.493192+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:23:42.493192+0000","last_undegraded":"2024-08-17T17:23:42.493192+0000","last_fullsized":"2024-08-17T17:23:42.493192+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1139,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.523681+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:23:42.523681+0000","last_peered":"2024-08-17T17:23:42.523681+0000","last_clean":"2024-08-17T17:23:42.523681+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:23:42.523681+0000","last_undegraded":"2024-08-17T17:23:42.523681+0000","last_fullsized":"2024-08-17T17:23:42.523681+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1523,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.412377+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:23:53.412377+0000","last_peered":"2024-08-17T17:23:53.412377+0000","last_clean":"2024-08-17T17:23:53.412377+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:23:53.412377+0000","last_undegraded":"2024-08-17T17:23:53.412377+0000","last_fullsized":"2024-08-17T17:23:53.412377+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1499,"reported_epoch":1211,"state":"active+clean","last_fresh":"2024-08-17T17:23:53.412566+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:23:53.412566+0000","last_peered":"2024-08-17T17:23:53.412566+0000","last_clean":"2024-08-17T17:23:53.412566+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:23:53.412566+0000","last_undegraded":"2024-08-17T17:23:53.412566+0000","last_fullsized":"2024-08-17T17:23:53.412566+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1173,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.492566+0000","last_change":"2024-08-17T17:21:06.483469+0000","last_active":"2024-08-17T17:23:42.492566+0000","last_peered":"2024-08-17T17:23:42.492566+0000","last_clean":"2024-08-17T17:23:42.492566+0000","last_became_active":"2024-08-17T17:21:06.483162+0000","last_became_peered":"2024-08-17T17:21:06.483162+0000","last_unstale":"2024-08-17T17:23:42.492566+0000","last_undegraded":"2024-08-17T17:23:42.492566+0000","last_fullsized":"2024-08-17T17:23:42.492566+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:24.898353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":182,"reported_epoch":1210,"state":"active+clean","last_fresh":"2024-08-17T17:23:42.486639+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:23:42.486639+0000","last_peered":"2024-08-17T17:23:42.486639+0000","last_clean":"2024-08-17T17:23:42.486639+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:23:42.486639+0000","last_undegraded":"2024-08-17T17:23:42.486639+0000","last_fullsized":"2024-08-17T17:23:42.486639+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":8223916056,"num_objects":125488,"num_object_clones":0,"num_object_copies":376464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":125488,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":127134,"num_write_kb":8083841,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":24735584256,"data_stored":24735580184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122134,"ondisk_log_size":122134,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201670,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2925752,"kb_used_data":2533744,"kb_used_omap":1,"kb_used_meta":391998,"kb_avail":90819400,"statfs":{"total":95995035648,"available":92999065600,"internally_reserved":0,"allocated":2594553856,"data_stored":2592145840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":401406411},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463307,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2881392,"kb_used_data":2535336,"kb_used_omap":1,"kb_used_meta":346046,"kb_avail":90863760,"statfs":{"total":95995035648,"available":93044490240,"internally_reserved":0,"allocated":2596184064,"data_stored":2593784113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":354351561},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626949,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3532388,"kb_used_data":2976860,"kb_used_omap":42,"kb_used_meta":555477,"kb_avail":90212764,"statfs":{"total":95995035648,"available":92377870336,"internally_reserved":0,"allocated":3048304640,"data_stored":3045912778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":568808508},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785426,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3902688,"kb_used_data":3513624,"kb_used_omap":48,"kb_used_meta":389007,"kb_avail":89842464,"statfs":{"total":95995035648,"available":91998683136,"internally_reserved":0,"allocated":3597950976,"data_stored":3595562595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":398343854},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724943,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4437100,"kb_used_data":4049188,"kb_used_omap":1,"kb_used_meta":387902,"kb_avail":89308052,"statfs":{"total":95995035648,"available":91451445248,"internally_reserved":0,"allocated":4146368512,"data_stored":4143969027,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":397212102},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,0,2,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986580,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2918964,"kb_used_data":2540908,"kb_used_omap":1,"kb_used_meta":378046,"kb_avail":90826188,"statfs":{"total":95995035648,"available":93006016512,"internally_reserved":0,"allocated":2601889792,"data_stored":2599485802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":387119558},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706187,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3441768,"kb_used_data":3008544,"kb_used_omap":35,"kb_used_meta":433180,"kb_avail":90303384,"statfs":{"total":95995035648,"available":92470665216,"internally_reserved":0,"allocated":3080749056,"data_stored":3078353624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":443576375},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831960,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3534244,"kb_used_data":3028380,"kb_used_omap":65,"kb_used_meta":505790,"kb_avail":90210908,"statfs":{"total":95995035648,"available":92375969792,"internally_reserved":0,"allocated":3101061120,"data_stored":3098669228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":517929147},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3100573696,"data_stored":3100573696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3582001152,"data_stored":3582001152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3078094848,"data_stored":3078094848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3032678400,"data_stored":3032678400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2597191680,"data_stored":2597191680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4154003456,"data_stored":4153999384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2591227904,"data_stored":2591227904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2599813120,"data_stored":2599813120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:24:03.850 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T17:24:03.850 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set .mgr pg_num 21 2024-08-17T17:24:03.930 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:03.998 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:03.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:03.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:04.099 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:04.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.149 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:04.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:04.250 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:04.293 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:04.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:04.395 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:04.432 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:04.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:04.534 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:04.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.585 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:04.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:04.686 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:04.722 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:04.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:04.824 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:04.859 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:04.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:04.961 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:04.996 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:04.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:04.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:05.098 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:05.125 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:05.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:05.227 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:05.229 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:05.291 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:05.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:05.393 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:05.429 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:05.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:05.530 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:05.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.580 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:05.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:05.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:05 smithi162 ceph-mon[104668]: pgmap v1577: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 32 MiB/s wr, 882 op/s 2024-08-17T17:24:05.633 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:24:05.633 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:24:05.681 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:05.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.715 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:05.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:05.728 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:05 smithi063 ceph-mon[114756]: pgmap v1577: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 32 MiB/s wr, 882 op/s 2024-08-17T17:24:05.729 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:05 smithi063 ceph-mon[109843]: pgmap v1577: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 32 MiB/s wr, 882 op/s 2024-08-17T17:24:05.816 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:05.854 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:05.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:05.956 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:05.984 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:05.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:05.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:06.086 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:06.120 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:06.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:06.222 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:06.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.260 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:06.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:06.356 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:24:06.356 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:24:06.356 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:24:06.356 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:24:06.356 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:24:06.356 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:24:06.357 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:24:06.357 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:24:06.357 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:24:06.357 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:24:06.357 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:24:06.357 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:24:06.357 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:24:06.357 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:24:06.357 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:24:06.357 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:24:06.361 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:06.397 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:06.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:06.500 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:24:06.500 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:06.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:06 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2350357842' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:06.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:06 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2350357842' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:06.541 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:06.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:06.592 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:24:06.643 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:06.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.698 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:06.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:06.799 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:06.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:06 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2350357842' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:06.846 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:06.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:06.852 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:24:06.948 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:06.995 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:06.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:06.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:07.097 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:07.140 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:07.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:07.242 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:07.277 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:07.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:07.380 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:07.407 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:07.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:07.509 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:07.540 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:07.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:07.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:07 smithi162 ceph-mon[104668]: pgmap v1578: 99 pgs: 99 active+clean; 7.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 23 MiB/s wr, 885 op/s 2024-08-17T17:24:07.642 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:07.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.674 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:07.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:07.775 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:07.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:07 smithi063 ceph-mon[109843]: pgmap v1578: 99 pgs: 99 active+clean; 7.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 23 MiB/s wr, 885 op/s 2024-08-17T17:24:07.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:07 smithi063 ceph-mon[114756]: pgmap v1578: 99 pgs: 99 active+clean; 7.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 23 MiB/s wr, 885 op/s 2024-08-17T17:24:07.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.807 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:07.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:07.908 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:07.945 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:07.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:07.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:08.047 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:08.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.084 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:08.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:08.185 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:08.219 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:08.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:08.267 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:08.321 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:08.378 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:08.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:08.480 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:08.519 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:08.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:08.621 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:08.662 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:08.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:08.764 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:08.801 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:08.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:08.885 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:24:08.885 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:24:08.885 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:24:08.885 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:24:08.885 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:24:08.904 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:08.931 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:08.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:08.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:09.033 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:09.035 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:24:09.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.073 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:09.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:09.174 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:09.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.208 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:09.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:09.309 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:09.351 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:09.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:09.453 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:09.491 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:09.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:09.525 INFO:teuthology.orchestra.run.smithi063.stderr:set pool 1 pg_num to 21 2024-08-17T17:24:09.593 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:09.630 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:09.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:09.732 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:09.768 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:09.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:09.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:09 smithi063 ceph-mon[109843]: pgmap v1579: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 4.7 MiB/s wr, 901 op/s 2024-08-17T17:24:09.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:09 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2078611483' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-08-17T17:24:09.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:09 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-08-17T17:24:09.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:09 smithi063 ceph-mon[114756]: pgmap v1579: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 4.7 MiB/s wr, 901 op/s 2024-08-17T17:24:09.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:09 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2078611483' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-08-17T17:24:09.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:09 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-08-17T17:24:09.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:09 smithi162 ceph-mon[104668]: pgmap v1579: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 4.7 MiB/s wr, 901 op/s 2024-08-17T17:24:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:09 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2078611483' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-08-17T17:24:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:09 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-08-17T17:24:09.870 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:09.880 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:24:09.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:09.895 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:09.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:09.997 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:10.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.059 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:10.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:10.160 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:10.222 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:10.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:10.266 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:24:10.324 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:10.358 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:10.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:10.459 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:10.481 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:10.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:10.583 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:10.620 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:10.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:10.721 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:10.742 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:10.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.743 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:10.844 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:10.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:10 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-08-17T17:24:10.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:10 smithi162 ceph-mon[104668]: osdmap e1213: 8 total, 8 up, 8 in 2024-08-17T17:24:10.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:10 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-08-17T17:24:10.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:10 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-08-17T17:24:10.881 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:10.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:10.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:10.983 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:11.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.021 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:11.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:10 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-08-17T17:24:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:10 smithi063 ceph-mon[114756]: osdmap e1213: 8 total, 8 up, 8 in 2024-08-17T17:24:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:10 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-08-17T17:24:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:10 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-08-17T17:24:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:10 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-08-17T17:24:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:10 smithi063 ceph-mon[109843]: osdmap e1213: 8 total, 8 up, 8 in 2024-08-17T17:24:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:10 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-08-17T17:24:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:10 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-08-17T17:24:11.122 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:11.158 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:11.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:11.260 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:11.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.309 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:11.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:11.410 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:11.457 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:11.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:11.561 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:11.601 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:11.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:11.703 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:11.734 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:11.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:11.754 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:11 smithi063 ceph-mon[109843]: pgmap v1581: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 1.3 MiB/s wr, 862 op/s 2024-08-17T17:24:11.754 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:11 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-08-17T17:24:11.754 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:11 smithi063 ceph-mon[109843]: osdmap e1214: 8 total, 8 up, 8 in 2024-08-17T17:24:11.754 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:11 smithi063 ceph-mon[114756]: pgmap v1581: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 1.3 MiB/s wr, 862 op/s 2024-08-17T17:24:11.754 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:11 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-08-17T17:24:11.755 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:11 smithi063 ceph-mon[114756]: osdmap e1214: 8 total, 8 up, 8 in 2024-08-17T17:24:11.836 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:11 smithi162 ceph-mon[104668]: pgmap v1581: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 1.3 MiB/s wr, 862 op/s 2024-08-17T17:24:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:11 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-08-17T17:24:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:11 smithi162 ceph-mon[104668]: osdmap e1214: 8 total, 8 up, 8 in 2024-08-17T17:24:11.874 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:11.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:11.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:11.976 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:12.016 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:12.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:12.118 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:12.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.169 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:12.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:12.270 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:12.301 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:12.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:12.403 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:12.437 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:12.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:12.441 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:12.539 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:12.583 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:12.584 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:24:12.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:12.604 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:12 smithi063 ceph-mon[114756]: osdmap e1215: 8 total, 8 up, 8 in 2024-08-17T17:24:12.604 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:12 smithi063 ceph-mon[114756]: 1.3 scrub starts 2024-08-17T17:24:12.604 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:12 smithi063 ceph-mon[114756]: 1.3 scrub ok 2024-08-17T17:24:12.604 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:12 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/553996237' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:12.604 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:12 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:12.604 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:12 smithi063 ceph-mon[114756]: 1.2 scrub starts 2024-08-17T17:24:12.604 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:12 smithi063 ceph-mon[109843]: osdmap e1215: 8 total, 8 up, 8 in 2024-08-17T17:24:12.605 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:12 smithi063 ceph-mon[109843]: 1.3 scrub starts 2024-08-17T17:24:12.605 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:12 smithi063 ceph-mon[109843]: 1.3 scrub ok 2024-08-17T17:24:12.605 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:12 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/553996237' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:12.605 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:12 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:12.605 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:12 smithi063 ceph-mon[109843]: 1.2 scrub starts 2024-08-17T17:24:12.686 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:12.716 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:12.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:12.818 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:12.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:12 smithi162 ceph-mon[104668]: osdmap e1215: 8 total, 8 up, 8 in 2024-08-17T17:24:12.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:12 smithi162 ceph-mon[104668]: 1.3 scrub starts 2024-08-17T17:24:12.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:12 smithi162 ceph-mon[104668]: 1.3 scrub ok 2024-08-17T17:24:12.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:12 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/553996237' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:12.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:12 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:12.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:12 smithi162 ceph-mon[104668]: 1.2 scrub starts 2024-08-17T17:24:12.851 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:12.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:12.953 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:12.994 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:12.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:12.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:13.095 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:13.141 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:13.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:13.247 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:13.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.280 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:13.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:13.381 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:13.425 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:13.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:13.526 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:24:13.526 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:24:13.526 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:24:13.526 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:24:13.526 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.526 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160939 6.1 6:893849fe:::benchmark_data_smithi063_160268_object34832:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:24:13.526 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.614043+0000", 2024-08-17T17:24:13.526 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 515.90796442099997, 2024-08-17T17:24:13.526 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0308899330000001, 2024-08-17T17:24:13.527 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.527 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.527 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.527 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:24:13.527 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:24:13.527 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160939 2024-08-17T17:24:13.527 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.527 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.527 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.527 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.527 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614043+0000", 2024-08-17T17:24:13.527 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.528 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.528 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.528 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.528 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614043+0000", 2024-08-17T17:24:13.528 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.528 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.528 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.528 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.528 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614041+0000", 2024-08-17T17:24:13.528 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:13.528 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.528 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.528 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.529 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614047+0000", 2024-08-17T17:24:13.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9440000000000002e-06 2024-08-17T17:24:13.529 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.529 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.529 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.529 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614057+0000", 2024-08-17T17:24:13.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3409999999999992e-06 2024-08-17T17:24:13.529 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.529 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.529 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.529 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614064+0000", 2024-08-17T17:24:13.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7190000000000001e-06 2024-08-17T17:24:13.529 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.530 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.530 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.530 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614117+0000", 2024-08-17T17:24:13.530 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2797000000000003e-05 2024-08-17T17:24:13.530 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.530 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.530 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.530 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.614128+0000", 2024-08-17T17:24:13.530 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0891e-05 2024-08-17T17:24:13.530 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.530 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.530 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.530 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.631386+0000", 2024-08-17T17:24:13.531 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.017258346 2024-08-17T17:24:13.531 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.531 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.531 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.531 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.633791+0000", 2024-08-17T17:24:13.531 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0024042109999999998 2024-08-17T17:24:13.531 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.531 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.531 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:24:13.531 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.633951+0000", 2024-08-17T17:24:13.531 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016093800000000001 2024-08-17T17:24:13.531 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.532 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.532 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644732+0000", 2024-08-17T17:24:13.532 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010780998 2024-08-17T17:24:13.532 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.532 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.532 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644760+0000", 2024-08-17T17:24:13.532 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.7195e-05 2024-08-17T17:24:13.532 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.532 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.532 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644837+0000", 2024-08-17T17:24:13.533 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7256999999999996e-05 2024-08-17T17:24:13.533 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.533 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.533 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.533 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644891+0000", 2024-08-17T17:24:13.533 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4284999999999999e-05 2024-08-17T17:24:13.533 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.533 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.533 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.533 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644933+0000", 2024-08-17T17:24:13.533 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1493999999999999e-05 2024-08-17T17:24:13.533 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.533 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.534 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.534 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.534 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.534 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160958 6.1 6:804ff99b:::benchmark_data_smithi063_160268_object34851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:24:13.534 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.649687+0000", 2024-08-17T17:24:13.534 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 515.87232057300002, 2024-08-17T17:24:13.534 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99790868700000002, 2024-08-17T17:24:13.534 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.534 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.534 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.534 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:24:13.534 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:24:13.534 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160958 2024-08-17T17:24:13.535 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.535 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.535 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.535 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.535 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649687+0000", 2024-08-17T17:24:13.535 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.535 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.535 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.535 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.535 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649687+0000", 2024-08-17T17:24:13.535 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.535 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.536 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.536 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.536 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649685+0000", 2024-08-17T17:24:13.536 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:24:13.536 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.536 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.536 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.536 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649695+0000", 2024-08-17T17:24:13.536 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9210000000000004e-06 2024-08-17T17:24:13.536 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.536 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.536 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.536 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649708+0000", 2024-08-17T17:24:13.537 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2947e-05 2024-08-17T17:24:13.537 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.537 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.537 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.537 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649715+0000", 2024-08-17T17:24:13.537 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9519999999999994e-06 2024-08-17T17:24:13.537 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.537 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.537 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.538 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649779+0000", 2024-08-17T17:24:13.538 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3683000000000007e-05 2024-08-17T17:24:13.538 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.538 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.538 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.538 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.649792+0000", 2024-08-17T17:24:13.538 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3294e-05 2024-08-17T17:24:13.538 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.538 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.539 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.539 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.637682+0000", 2024-08-17T17:24:13.539 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98788912399999995 2024-08-17T17:24:13.539 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.539 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.539 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.539 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.637791+0000", 2024-08-17T17:24:13.539 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000109728 2024-08-17T17:24:13.539 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.540 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.540 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:24:13.540 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.637918+0000", 2024-08-17T17:24:13.540 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000126842 2024-08-17T17:24:13.540 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.540 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.540 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.540 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644787+0000", 2024-08-17T17:24:13.540 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0068684000000000002 2024-08-17T17:24:13.540 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.540 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.540 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.540 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647458+0000", 2024-08-17T17:24:13.540 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026717220000000001 2024-08-17T17:24:13.541 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.541 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.541 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647543+0000", 2024-08-17T17:24:13.541 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4790999999999995e-05 2024-08-17T17:24:13.541 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.541 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.541 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647572+0000", 2024-08-17T17:24:13.541 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8538999999999999e-05 2024-08-17T17:24:13.541 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.542 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.542 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647595+0000", 2024-08-17T17:24:13.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3648000000000001e-05 2024-08-17T17:24:13.542 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.542 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.542 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.543 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.543 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.543 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160968 6.1 6:8a3118ae:::benchmark_data_smithi063_160268_object34861:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:24:13.543 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.664096+0000", 2024-08-17T17:24:13.543 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 515.85791071999995, 2024-08-17T17:24:13.543 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99085020099999999, 2024-08-17T17:24:13.543 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.543 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.543 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.543 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:24:13.543 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:24:13.543 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160968 2024-08-17T17:24:13.543 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.544 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.544 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.544 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.544 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664096+0000", 2024-08-17T17:24:13.544 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.544 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.544 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.544 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.544 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664096+0000", 2024-08-17T17:24:13.544 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.544 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.544 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.545 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.545 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664095+0000", 2024-08-17T17:24:13.545 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:24:13.545 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.545 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.545 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.545 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664100+0000", 2024-08-17T17:24:13.545 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0880000000000002e-06 2024-08-17T17:24:13.545 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.545 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.545 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.545 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664112+0000", 2024-08-17T17:24:13.545 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1231e-05 2024-08-17T17:24:13.545 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.546 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.546 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.546 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664117+0000", 2024-08-17T17:24:13.546 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4829999999999999e-06 2024-08-17T17:24:13.546 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.546 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.546 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.546 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664174+0000", 2024-08-17T17:24:13.547 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6443999999999997e-05 2024-08-17T17:24:13.547 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.548 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.548 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.548 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664186+0000", 2024-08-17T17:24:13.548 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2500999999999999e-05 2024-08-17T17:24:13.548 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.548 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.548 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.548 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.641442+0000", 2024-08-17T17:24:13.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97725559900000003 2024-08-17T17:24:13.549 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.549 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.641557+0000", 2024-08-17T17:24:13.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000115661 2024-08-17T17:24:13.549 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.549 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:24:13.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.641675+0000", 2024-08-17T17:24:13.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011779 2024-08-17T17:24:13.550 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.550 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.550 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.550 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647462+0000", 2024-08-17T17:24:13.550 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0057873380000000004 2024-08-17T17:24:13.550 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.550 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.550 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.550 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647507+0000", 2024-08-17T17:24:13.550 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4776e-05 2024-08-17T17:24:13.550 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.551 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.551 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.551 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654882+0000", 2024-08-17T17:24:13.551 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0073748959999999997 2024-08-17T17:24:13.551 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.551 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.551 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.551 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654915+0000", 2024-08-17T17:24:13.551 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3333000000000001e-05 2024-08-17T17:24:13.551 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.551 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.551 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.551 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654947+0000", 2024-08-17T17:24:13.552 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1192999999999998e-05 2024-08-17T17:24:13.552 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.553 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.553 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.553 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.553 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.553 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160969 6.1 6:85b88efb:::benchmark_data_smithi063_160268_object34862:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:24:13.554 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.664210+0000", 2024-08-17T17:24:13.554 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 515.85779718499998, 2024-08-17T17:24:13.554 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99084335700000004, 2024-08-17T17:24:13.554 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.554 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.554 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.554 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:24:13.554 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:24:13.554 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160969 2024-08-17T17:24:13.554 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.554 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.555 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.555 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.555 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664210+0000", 2024-08-17T17:24:13.555 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.555 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.555 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.555 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.555 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664210+0000", 2024-08-17T17:24:13.555 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.555 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.555 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.556 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664209+0000", 2024-08-17T17:24:13.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:13.556 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.556 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.556 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664215+0000", 2024-08-17T17:24:13.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8619999999999999e-06 2024-08-17T17:24:13.556 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.556 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664227+0000", 2024-08-17T17:24:13.557 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.295e-05 2024-08-17T17:24:13.557 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.557 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664234+0000", 2024-08-17T17:24:13.557 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8399999999999997e-06 2024-08-17T17:24:13.557 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.557 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664286+0000", 2024-08-17T17:24:13.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1928000000000002e-05 2024-08-17T17:24:13.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.558 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.558 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.558 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.664300+0000", 2024-08-17T17:24:13.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3497e-05 2024-08-17T17:24:13.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.558 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.558 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.558 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.644983+0000", 2024-08-17T17:24:13.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98068294099999997 2024-08-17T17:24:13.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.559 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.559 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.645050+0000", 2024-08-17T17:24:13.559 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7012000000000001e-05 2024-08-17T17:24:13.559 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.559 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.559 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:24:13.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.645116+0000", 2024-08-17T17:24:13.559 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6268999999999999e-05 2024-08-17T17:24:13.559 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.559 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.559 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654779+0000", 2024-08-17T17:24:13.559 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0096628619999999995 2024-08-17T17:24:13.560 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.560 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.560 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.560 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654813+0000", 2024-08-17T17:24:13.560 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4489000000000001e-05 2024-08-17T17:24:13.560 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.560 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.560 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.560 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654995+0000", 2024-08-17T17:24:13.560 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000181478 2024-08-17T17:24:13.560 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.560 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.560 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.561 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655021+0000", 2024-08-17T17:24:13.561 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6390999999999999e-05 2024-08-17T17:24:13.561 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.561 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.561 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.561 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655053+0000", 2024-08-17T17:24:13.561 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2147999999999998e-05 2024-08-17T17:24:13.561 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.561 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.561 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.561 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.562 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.562 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160976 6.1 6:892f13c5:::benchmark_data_smithi063_160268_object34869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:24:13.562 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.670369+0000", 2024-08-17T17:24:13.562 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 515.851638256, 2024-08-17T17:24:13.562 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99685337699999998, 2024-08-17T17:24:13.562 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.562 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.562 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.562 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:24:13.562 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:24:13.562 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160976 2024-08-17T17:24:13.562 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.563 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670369+0000", 2024-08-17T17:24:13.563 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.563 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670369+0000", 2024-08-17T17:24:13.563 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.563 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670367+0000", 2024-08-17T17:24:13.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:13.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.564 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.564 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670374+0000", 2024-08-17T17:24:13.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.827e-06 2024-08-17T17:24:13.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.564 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.564 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670390+0000", 2024-08-17T17:24:13.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5566e-05 2024-08-17T17:24:13.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670398+0000", 2024-08-17T17:24:13.565 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3289999999999994e-06 2024-08-17T17:24:13.565 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670470+0000", 2024-08-17T17:24:13.565 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1913000000000001e-05 2024-08-17T17:24:13.565 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.670482+0000", 2024-08-17T17:24:13.566 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2018000000000001e-05 2024-08-17T17:24:13.566 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.566 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.566 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.566 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647627+0000", 2024-08-17T17:24:13.566 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.977145139 2024-08-17T17:24:13.566 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.566 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.566 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.566 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647700+0000", 2024-08-17T17:24:13.566 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2495000000000003e-05 2024-08-17T17:24:13.567 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.567 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.567 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:24:13.567 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.647765+0000", 2024-08-17T17:24:13.567 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5495999999999998e-05 2024-08-17T17:24:13.567 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.567 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.567 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.567 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654846+0000", 2024-08-17T17:24:13.567 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0070811249999999997 2024-08-17T17:24:13.567 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.567 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.567 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.568 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.654853+0000", 2024-08-17T17:24:13.568 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2929999999999997e-06 2024-08-17T17:24:13.568 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.568 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.568 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.568 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667120+0000", 2024-08-17T17:24:13.568 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012267867 2024-08-17T17:24:13.568 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.568 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.568 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.568 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667180+0000", 2024-08-17T17:24:13.568 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9376999999999997e-05 2024-08-17T17:24:13.569 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.569 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.569 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.569 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667222+0000", 2024-08-17T17:24:13.569 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2451999999999999e-05 2024-08-17T17:24:13.569 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.569 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.569 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.569 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.569 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.569 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160983 6.1 6:82d2572f:::benchmark_data_smithi063_160268_object34876:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:24:13.569 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.676198+0000", 2024-08-17T17:24:13.570 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 515.84580915499998, 2024-08-17T17:24:13.570 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99119904999999997, 2024-08-17T17:24:13.570 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.570 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.570 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.570 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:24:13.570 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:24:13.570 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160983 2024-08-17T17:24:13.570 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.570 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.570 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.570 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.571 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676198+0000", 2024-08-17T17:24:13.571 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.571 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.571 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.571 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.571 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676198+0000", 2024-08-17T17:24:13.571 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.571 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.571 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.571 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.571 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676196+0000", 2024-08-17T17:24:13.571 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:24:13.572 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.572 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.572 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.572 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676205+0000", 2024-08-17T17:24:13.572 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3079999999999997e-06 2024-08-17T17:24:13.572 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.572 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.572 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.572 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676223+0000", 2024-08-17T17:24:13.572 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7217000000000001e-05 2024-08-17T17:24:13.572 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.572 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.572 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.573 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676230+0000", 2024-08-17T17:24:13.573 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3479999999999998e-06 2024-08-17T17:24:13.573 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.573 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.573 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.573 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676293+0000", 2024-08-17T17:24:13.573 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2714999999999998e-05 2024-08-17T17:24:13.573 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.573 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.573 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.573 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.676305+0000", 2024-08-17T17:24:13.573 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2455e-05 2024-08-17T17:24:13.574 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.574 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.574 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.574 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.651451+0000", 2024-08-17T17:24:13.574 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97514584400000004 2024-08-17T17:24:13.574 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.574 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.574 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.574 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.651523+0000", 2024-08-17T17:24:13.574 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2497999999999996e-05 2024-08-17T17:24:13.574 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.574 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.575 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:24:13.575 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.651598+0000", 2024-08-17T17:24:13.575 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4241999999999998e-05 2024-08-17T17:24:13.575 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.575 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.575 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.575 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659221+0000", 2024-08-17T17:24:13.575 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0076229289999999996 2024-08-17T17:24:13.575 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.575 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.576 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.576 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667006+0000", 2024-08-17T17:24:13.576 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00778548 2024-08-17T17:24:13.576 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.576 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.576 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.576 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667303+0000", 2024-08-17T17:24:13.576 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000297103 2024-08-17T17:24:13.576 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.576 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.576 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.577 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667362+0000", 2024-08-17T17:24:13.577 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:24:13] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.51.0" 2024-08-17T17:24:13.578 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8705000000000001e-05 2024-08-17T17:24:13.578 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.578 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.578 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.578 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667397+0000", 2024-08-17T17:24:13.578 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5120999999999998e-05 2024-08-17T17:24:13.578 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.579 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.579 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.579 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.579 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.579 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160989 6.1 6:8559d498:::benchmark_data_smithi063_160268_object34882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:24:13.579 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.684830+0000", 2024-08-17T17:24:13.579 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 515.83717719100002, 2024-08-17T17:24:13.579 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98270078400000005, 2024-08-17T17:24:13.579 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.579 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.579 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.579 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:24:13.579 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:24:13.580 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160989 2024-08-17T17:24:13.580 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.580 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.580 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.580 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.580 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684830+0000", 2024-08-17T17:24:13.580 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.580 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.580 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.580 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.580 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684830+0000", 2024-08-17T17:24:13.580 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.580 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.580 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.581 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.581 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684828+0000", 2024-08-17T17:24:13.581 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:13.581 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.581 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.581 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.581 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684835+0000", 2024-08-17T17:24:13.581 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9890000000000003e-06 2024-08-17T17:24:13.581 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.581 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.581 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.581 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684851+0000", 2024-08-17T17:24:13.581 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5457000000000001e-05 2024-08-17T17:24:13.581 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.582 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.582 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.582 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684863+0000", 2024-08-17T17:24:13.582 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2302e-05 2024-08-17T17:24:13.582 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.582 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.582 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.582 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684941+0000", 2024-08-17T17:24:13.582 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7541999999999997e-05 2024-08-17T17:24:13.582 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.582 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.582 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.582 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.684956+0000", 2024-08-17T17:24:13.583 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5148e-05 2024-08-17T17:24:13.583 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.583 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.583 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.583 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655087+0000", 2024-08-17T17:24:13.583 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97013142799999996 2024-08-17T17:24:13.583 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.583 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.583 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.583 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655187+0000", 2024-08-17T17:24:13.583 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9229999999999997e-05 2024-08-17T17:24:13.583 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.583 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.584 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:24:13.584 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.655293+0000", 2024-08-17T17:24:13.584 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000106576 2024-08-17T17:24:13.584 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.584 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.584 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.584 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667235+0000", 2024-08-17T17:24:13.584 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011942026 2024-08-17T17:24:13.584 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.584 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.584 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.584 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667269+0000", 2024-08-17T17:24:13.584 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.417e-05 2024-08-17T17:24:13.585 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.585 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.585 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.585 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667449+0000", 2024-08-17T17:24:13.585 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000179292 2024-08-17T17:24:13.585 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.585 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.585 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.585 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667498+0000", 2024-08-17T17:24:13.585 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9821999999999998e-05 2024-08-17T17:24:13.585 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.585 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.586 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.586 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667531+0000", 2024-08-17T17:24:13.586 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2326999999999997e-05 2024-08-17T17:24:13.586 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.586 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.586 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.586 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.586 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.586 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:160996 6.1 6:8ccbb0f4:::benchmark_data_smithi063_160268_object34889:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:24:13.586 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.691144+0000", 2024-08-17T17:24:13.586 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 515.83086272100002, 2024-08-17T17:24:13.587 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98531105200000002, 2024-08-17T17:24:13.587 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.587 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.587 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.587 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:24:13.587 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:24:13.587 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 160996 2024-08-17T17:24:13.587 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.587 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.587 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.587 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.587 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691144+0000", 2024-08-17T17:24:13.587 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.588 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.588 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.588 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.588 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691144+0000", 2024-08-17T17:24:13.588 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.588 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.588 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.588 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.588 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691143+0000", 2024-08-17T17:24:13.588 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:13.588 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.588 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.589 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.589 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691149+0000", 2024-08-17T17:24:13.589 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0800000000000002e-06 2024-08-17T17:24:13.589 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.589 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.589 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.589 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691162+0000", 2024-08-17T17:24:13.589 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.322e-05 2024-08-17T17:24:13.589 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.589 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.589 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.589 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691168+0000", 2024-08-17T17:24:13.590 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0260000000000004e-06 2024-08-17T17:24:13.590 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.590 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.590 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.590 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691211+0000", 2024-08-17T17:24:13.590 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3013999999999998e-05 2024-08-17T17:24:13.590 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.590 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.590 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.590 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.691220+0000", 2024-08-17T17:24:13.590 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8540000000000003e-06 2024-08-17T17:24:13.590 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.590 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.591 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.591 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659248+0000", 2024-08-17T17:24:13.591 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.96802809099999998 2024-08-17T17:24:13.591 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.591 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.591 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.591 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659357+0000", 2024-08-17T17:24:13.591 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010854 2024-08-17T17:24:13.591 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.591 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.591 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:24:13.591 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.659466+0000", 2024-08-17T17:24:13.592 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000109437 2024-08-17T17:24:13.592 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.592 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.592 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.592 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667552+0000", 2024-08-17T17:24:13.592 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0080855279999999998 2024-08-17T17:24:13.592 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.592 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.592 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.592 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676310+0000", 2024-08-17T17:24:13.592 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0087583250000000008 2024-08-17T17:24:13.592 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.592 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.593 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.593 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676357+0000", 2024-08-17T17:24:13.593 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6786000000000002e-05 2024-08-17T17:24:13.593 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.593 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.593 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.593 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676409+0000", 2024-08-17T17:24:13.593 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2104000000000001e-05 2024-08-17T17:24:13.593 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.593 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.593 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.593 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.676455+0000", 2024-08-17T17:24:13.594 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6362000000000001e-05 2024-08-17T17:24:13.594 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.594 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.594 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.594 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.594 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.594 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.44609.0:161006 6.1 6:8d7d2cbb:::benchmark_data_smithi063_160268_object34899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1109)", 2024-08-17T17:24:13.594 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:15:37.709227+0000", 2024-08-17T17:24:13.594 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 515.81277996999995, 2024-08-17T17:24:13.594 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97342043, 2024-08-17T17:24:13.594 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.595 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.595 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.595 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.44609", 2024-08-17T17:24:13.595 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/821121425", 2024-08-17T17:24:13.595 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 161006 2024-08-17T17:24:13.595 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.595 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.595 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.595 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.595 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709227+0000", 2024-08-17T17:24:13.596 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.596 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.596 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.596 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.596 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709227+0000", 2024-08-17T17:24:13.596 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.596 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.596 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.596 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.596 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709226+0000", 2024-08-17T17:24:13.597 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:24:13.597 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.597 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.597 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.597 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709232+0000", 2024-08-17T17:24:13.597 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6219999999999997e-06 2024-08-17T17:24:13.597 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.597 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.597 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.597 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709243+0000", 2024-08-17T17:24:13.597 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0839000000000001e-05 2024-08-17T17:24:13.597 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.597 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.598 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.598 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709248+0000", 2024-08-17T17:24:13.598 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.322e-06 2024-08-17T17:24:13.598 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.598 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.598 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.598 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709293+0000", 2024-08-17T17:24:13.598 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4672000000000002e-05 2024-08-17T17:24:13.598 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.598 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.598 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.598 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:37.709302+0000", 2024-08-17T17:24:13.599 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2240000000000001e-06 2024-08-17T17:24:13.599 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.599 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.599 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.599 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.663451+0000", 2024-08-17T17:24:13.599 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95414951299999995 2024-08-17T17:24:13.599 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.599 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.599 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.599 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.663549+0000", 2024-08-17T17:24:13.599 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7268000000000005e-05 2024-08-17T17:24:13.599 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.600 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.600 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:24:13.600 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.663655+0000", 2024-08-17T17:24:13.600 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000106872 2024-08-17T17:24:13.600 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.600 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.600 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.600 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.667559+0000", 2024-08-17T17:24:13.600 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0039036629999999999 2024-08-17T17:24:13.600 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.600 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.600 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.600 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.678421+0000", 2024-08-17T17:24:13.600 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010861585 2024-08-17T17:24:13.601 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.601 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.601 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.601 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.682563+0000", 2024-08-17T17:24:13.601 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0041419159999999998 2024-08-17T17:24:13.601 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.601 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.601 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.601 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.682611+0000", 2024-08-17T17:24:13.601 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8585000000000003e-05 2024-08-17T17:24:13.601 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.601 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.601 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.602 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:15:38.682648+0000", 2024-08-17T17:24:13.602 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6449999999999998e-05 2024-08-17T17:24:13.602 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.602 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.602 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.602 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.602 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.602 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65735 7.7 7:e7f48e21:::benchmark_data_smithi063_196002_object65734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-08-17T17:24:13.602 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.525390+0000", 2024-08-17T17:24:13.602 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 342.99661732800001, 2024-08-17T17:24:13.602 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4912375070000001, 2024-08-17T17:24:13.602 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.602 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.603 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.603 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:24:13.603 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:24:13.603 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65735 2024-08-17T17:24:13.603 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.603 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.603 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.603 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.603 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525390+0000", 2024-08-17T17:24:13.603 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.603 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.603 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.603 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.604 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525390+0000", 2024-08-17T17:24:13.604 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.604 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.604 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.604 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.604 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525388+0000", 2024-08-17T17:24:13.604 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:24:13.604 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.604 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.604 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.604 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525427+0000", 2024-08-17T17:24:13.604 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8878000000000003e-05 2024-08-17T17:24:13.605 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.605 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.605 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.605 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525439+0000", 2024-08-17T17:24:13.605 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2507000000000001e-05 2024-08-17T17:24:13.605 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.605 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.605 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.605 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525447+0000", 2024-08-17T17:24:13.605 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7530000000000001e-06 2024-08-17T17:24:13.605 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.605 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.606 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.606 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.533687+0000", 2024-08-17T17:24:13.606 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0082400100000000007 2024-08-17T17:24:13.606 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.606 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.606 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.606 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.533697+0000", 2024-08-17T17:24:13.606 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0455000000000001e-05 2024-08-17T17:24:13.606 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.606 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.606 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.607 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523807+0000", 2024-08-17T17:24:13.607 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99010963600000002 2024-08-17T17:24:13.607 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.607 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.607 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.607 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523817+0000", 2024-08-17T17:24:13.607 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7189999999999996e-06 2024-08-17T17:24:13.607 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.607 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.607 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.607 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:32.998106+0000", 2024-08-17T17:24:13.607 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4742891280000001 2024-08-17T17:24:13.608 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.608 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.608 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.608 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:32.998241+0000", 2024-08-17T17:24:13.608 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013504000000000001 2024-08-17T17:24:13.608 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.608 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.608 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:24:13.608 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:32.998370+0000", 2024-08-17T17:24:13.608 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012903799999999999 2024-08-17T17:24:13.608 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.608 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.609 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.609 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.003851+0000", 2024-08-17T17:24:13.609 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0054806999999999998 2024-08-17T17:24:13.609 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.609 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.609 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.609 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.005429+0000", 2024-08-17T17:24:13.609 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015783819999999999 2024-08-17T17:24:13.609 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.609 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.609 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.609 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.014199+0000", 2024-08-17T17:24:13.610 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0087699130000000007 2024-08-17T17:24:13.610 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.610 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.610 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.610 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.014257+0000", 2024-08-17T17:24:13.610 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8193999999999999e-05 2024-08-17T17:24:13.610 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.610 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.610 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.610 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.016627+0000", 2024-08-17T17:24:13.610 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023701730000000002 2024-08-17T17:24:13.611 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.611 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.611 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.611 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.611 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.611 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65738 7.d 7:b7ed0697:::benchmark_data_smithi063_196002_object65737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:24:13.611 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.535426+0000", 2024-08-17T17:24:13.611 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 342.98658123600001, 2024-08-17T17:24:13.611 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5234001109999999, 2024-08-17T17:24:13.611 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.611 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.611 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.611 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:24:13.612 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:24:13.612 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65738 2024-08-17T17:24:13.612 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.612 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.612 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.612 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.612 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535426+0000", 2024-08-17T17:24:13.612 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.612 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.612 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.612 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.612 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535426+0000", 2024-08-17T17:24:13.612 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.612 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.613 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.613 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.613 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535424+0000", 2024-08-17T17:24:13.613 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:13.613 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.613 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.613 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.613 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535475+0000", 2024-08-17T17:24:13.613 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0512e-05 2024-08-17T17:24:13.613 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.613 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.613 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.613 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535486+0000", 2024-08-17T17:24:13.613 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0689999999999999e-05 2024-08-17T17:24:13.614 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.614 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.614 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.614 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535492+0000", 2024-08-17T17:24:13.614 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7379999999999997e-06 2024-08-17T17:24:13.614 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.614 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.614 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.614 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537456+0000", 2024-08-17T17:24:13.614 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0019633559999999999 2024-08-17T17:24:13.614 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.614 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.615 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.615 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537467+0000", 2024-08-17T17:24:13.615 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0784999999999999e-05 2024-08-17T17:24:13.615 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.615 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.615 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.615 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523461+0000", 2024-08-17T17:24:13.615 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.985994009 2024-08-17T17:24:13.615 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.615 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.615 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.615 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523472+0000", 2024-08-17T17:24:13.615 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1806e-05 2024-08-17T17:24:13.616 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.616 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.616 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.616 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.030195+0000", 2024-08-17T17:24:13.616 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.506722414 2024-08-17T17:24:13.616 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.616 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.616 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.616 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.030370+0000", 2024-08-17T17:24:13.616 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017517499999999999 2024-08-17T17:24:13.616 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.616 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.616 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:24:13.616 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.030534+0000", 2024-08-17T17:24:13.617 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016449999999999999 2024-08-17T17:24:13.617 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.617 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.617 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.617 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.038628+0000", 2024-08-17T17:24:13.617 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0080940379999999996 2024-08-17T17:24:13.617 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.617 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.617 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.617 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.038638+0000", 2024-08-17T17:24:13.617 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8770000000000005e-06 2024-08-17T17:24:13.617 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.618 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.618 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.618 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.056336+0000", 2024-08-17T17:24:13.618 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.017698034000000001 2024-08-17T17:24:13.618 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.618 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.618 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.618 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.056374+0000", 2024-08-17T17:24:13.618 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7753000000000003e-05 2024-08-17T17:24:13.618 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.618 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.618 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.619 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.058826+0000", 2024-08-17T17:24:13.619 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0024519149999999998 2024-08-17T17:24:13.619 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.619 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.619 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.619 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.619 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.619 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65740 7.7 7:e4a48506:::benchmark_data_smithi063_196002_object65739:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:24:13.619 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.537667+0000", 2024-08-17T17:24:13.619 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 342.98433972399999, 2024-08-17T17:24:13.619 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4812937370000001, 2024-08-17T17:24:13.619 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.619 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.619 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.620 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:24:13.620 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:24:13.620 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65740 2024-08-17T17:24:13.620 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.620 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.620 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.620 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.620 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537667+0000", 2024-08-17T17:24:13.620 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.620 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.620 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.620 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.621 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537667+0000", 2024-08-17T17:24:13.621 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.621 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.621 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.621 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.621 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537666+0000", 2024-08-17T17:24:13.621 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:13.621 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.621 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.621 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.621 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537699+0000", 2024-08-17T17:24:13.621 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2543000000000003e-05 2024-08-17T17:24:13.622 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.622 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.622 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.622 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537708+0000", 2024-08-17T17:24:13.622 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9700000000000005e-06 2024-08-17T17:24:13.622 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.622 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.622 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.622 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537715+0000", 2024-08-17T17:24:13.622 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1860000000000003e-06 2024-08-17T17:24:13.622 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.622 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.623 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.623 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537786+0000", 2024-08-17T17:24:13.623 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0732e-05 2024-08-17T17:24:13.623 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.623 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.623 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.623 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537795+0000", 2024-08-17T17:24:13.623 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7370000000000001e-06 2024-08-17T17:24:13.623 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.623 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.623 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.623 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524063+0000", 2024-08-17T17:24:13.623 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98626732800000005 2024-08-17T17:24:13.624 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.624 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.624 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.624 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524072+0000", 2024-08-17T17:24:13.624 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.1949999999999992e-06 2024-08-17T17:24:13.624 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.624 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.624 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.624 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.001760+0000", 2024-08-17T17:24:13.624 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4776884450000001 2024-08-17T17:24:13.624 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.624 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.624 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.624 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.001916+0000", 2024-08-17T17:24:13.625 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015600400000000001 2024-08-17T17:24:13.625 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.625 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.625 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:24:13.625 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.002089+0000", 2024-08-17T17:24:13.625 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017285900000000001 2024-08-17T17:24:13.625 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.625 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.625 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.625 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.006924+0000", 2024-08-17T17:24:13.625 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0048349899999999999 2024-08-17T17:24:13.625 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.625 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.625 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.626 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.006974+0000", 2024-08-17T17:24:13.626 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9648999999999998e-05 2024-08-17T17:24:13.626 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.626 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.626 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.626 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.016708+0000", 2024-08-17T17:24:13.626 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0097340340000000008 2024-08-17T17:24:13.626 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.626 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.626 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.626 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.016776+0000", 2024-08-17T17:24:13.626 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8320999999999999e-05 2024-08-17T17:24:13.627 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.627 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.627 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.627 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.018961+0000", 2024-08-17T17:24:13.627 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021849840000000001 2024-08-17T17:24:13.627 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.627 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.627 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.627 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.627 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.627 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65748 7.d 7:bfbcc155:::benchmark_data_smithi063_196002_object65747:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:24:13.627 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.545206+0000", 2024-08-17T17:24:13.627 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 342.97680148900002, 2024-08-17T17:24:13.628 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5163881620000002, 2024-08-17T17:24:13.628 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.628 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.628 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.628 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:24:13.628 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:24:13.628 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65748 2024-08-17T17:24:13.628 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.628 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.628 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.628 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.628 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545206+0000", 2024-08-17T17:24:13.628 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.629 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.629 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.629 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.629 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545206+0000", 2024-08-17T17:24:13.629 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.629 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.629 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.629 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.629 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545205+0000", 2024-08-17T17:24:13.629 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:24:13.629 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.629 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.629 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.630 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545338+0000", 2024-08-17T17:24:13.630 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000133375 2024-08-17T17:24:13.630 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.630 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.630 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.630 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545351+0000", 2024-08-17T17:24:13.630 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2901000000000001e-05 2024-08-17T17:24:13.631 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.631 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.631 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.631 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545358+0000", 2024-08-17T17:24:13.631 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9009999999999997e-06 2024-08-17T17:24:13.631 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.631 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.631 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.631 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545409+0000", 2024-08-17T17:24:13.631 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1038999999999997e-05 2024-08-17T17:24:13.631 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.631 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.632 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.632 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545418+0000", 2024-08-17T17:24:13.632 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.0990000000000005e-06 2024-08-17T17:24:13.632 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.632 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.632 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.632 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523596+0000", 2024-08-17T17:24:13.632 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97817841900000002 2024-08-17T17:24:13.632 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.632 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.632 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.632 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523605+0000", 2024-08-17T17:24:13.633 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.1630000000000002e-06 2024-08-17T17:24:13.633 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.633 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.633 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.633 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.035987+0000", 2024-08-17T17:24:13.633 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.512381864 2024-08-17T17:24:13.633 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.633 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.633 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.633 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.036136+0000", 2024-08-17T17:24:13.633 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014822899999999999 2024-08-17T17:24:13.633 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.633 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.634 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:24:13.634 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.036293+0000", 2024-08-17T17:24:13.634 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000157305 2024-08-17T17:24:13.634 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.634 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.634 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.634 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.041715+0000", 2024-08-17T17:24:13.634 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.005422105 2024-08-17T17:24:13.634 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.634 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.634 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.635 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.041772+0000", 2024-08-17T17:24:13.635 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7457999999999998e-05 2024-08-17T17:24:13.635 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.635 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.635 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.635 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.058975+0000", 2024-08-17T17:24:13.635 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.017202696999999999 2024-08-17T17:24:13.635 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.635 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.635 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.635 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.059136+0000", 2024-08-17T17:24:13.635 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016129800000000001 2024-08-17T17:24:13.636 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.636 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.636 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.636 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.061594+0000", 2024-08-17T17:24:13.636 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002457416 2024-08-17T17:24:13.636 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.636 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.636 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.636 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.636 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.636 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65753 7.7 7:e08ff533:::benchmark_data_smithi063_196002_object65752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:24:13.636 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.548721+0000", 2024-08-17T17:24:13.636 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 342.973286186, 2024-08-17T17:24:13.637 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.472712064, 2024-08-17T17:24:13.637 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.637 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.637 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.637 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:24:13.637 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:24:13.637 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65753 2024-08-17T17:24:13.637 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.637 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.637 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.637 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.638 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548721+0000", 2024-08-17T17:24:13.638 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.638 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.638 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.638 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.638 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548721+0000", 2024-08-17T17:24:13.638 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.639 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.639 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.639 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.639 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548720+0000", 2024-08-17T17:24:13.639 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:24:13.639 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.639 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.639 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.639 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:13.641 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548841+0000", 2024-08-17T17:24:13.641 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012082399999999999 2024-08-17T17:24:13.641 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.641 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.641 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.641 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548851+0000", 2024-08-17T17:24:13.641 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9660000000000008e-06 2024-08-17T17:24:13.641 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.641 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.641 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.641 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548862+0000", 2024-08-17T17:24:13.642 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0838e-05 2024-08-17T17:24:13.642 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.642 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.642 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.642 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548911+0000", 2024-08-17T17:24:13.642 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9004e-05 2024-08-17T17:24:13.642 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.642 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.642 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.642 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548921+0000", 2024-08-17T17:24:13.642 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9180000000000006e-06 2024-08-17T17:24:13.643 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.643 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.643 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.643 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524131+0000", 2024-08-17T17:24:13.643 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.975210566 2024-08-17T17:24:13.643 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.643 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.643 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.643 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524140+0000", 2024-08-17T17:24:13.643 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.242e-06 2024-08-17T17:24:13.643 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.644 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.644 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.644 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.003888+0000", 2024-08-17T17:24:13.644 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4797479820000001 2024-08-17T17:24:13.644 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.644 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.644 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.644 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.004030+0000", 2024-08-17T17:24:13.644 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000142495 2024-08-17T17:24:13.644 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.644 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.644 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:24:13.645 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.004221+0000", 2024-08-17T17:24:13.645 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00019079500000000001 2024-08-17T17:24:13.645 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.645 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.645 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.645 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.006933+0000", 2024-08-17T17:24:13.645 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00271244 2024-08-17T17:24:13.645 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.645 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.645 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.645 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.008656+0000", 2024-08-17T17:24:13.645 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001723154 2024-08-17T17:24:13.645 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.646 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.646 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.646 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.019021+0000", 2024-08-17T17:24:13.646 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010364310999999999 2024-08-17T17:24:13.646 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.646 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.646 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.646 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.019082+0000", 2024-08-17T17:24:13.646 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0998000000000001e-05 2024-08-17T17:24:13.646 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.646 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.646 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.647 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.021433+0000", 2024-08-17T17:24:13.647 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023512530000000002 2024-08-17T17:24:13.647 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.647 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.647 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.647 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.647 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.647 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65763 7.7 7:e13a5ff4:::benchmark_data_smithi063_196002_object65762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:24:13.647 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.559250+0000", 2024-08-17T17:24:13.647 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 342.96275698400001, 2024-08-17T17:24:13.647 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4649766419999999, 2024-08-17T17:24:13.647 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.648 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.648 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.648 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:24:13.648 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:24:13.648 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65763 2024-08-17T17:24:13.648 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.648 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.648 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.648 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.648 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559250+0000", 2024-08-17T17:24:13.648 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.649 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.649 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.649 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.649 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559250+0000", 2024-08-17T17:24:13.649 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.649 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.649 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.649 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.649 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559249+0000", 2024-08-17T17:24:13.649 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:13.649 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.649 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.650 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.650 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559294+0000", 2024-08-17T17:24:13.650 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5008e-05 2024-08-17T17:24:13.650 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.650 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.650 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.650 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559304+0000", 2024-08-17T17:24:13.650 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0543999999999999e-05 2024-08-17T17:24:13.650 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.650 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.650 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.651 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559311+0000", 2024-08-17T17:24:13.651 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9999999999999999e-06 2024-08-17T17:24:13.651 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.651 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.651 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.651 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559352+0000", 2024-08-17T17:24:13.651 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0099999999999999e-05 2024-08-17T17:24:13.651 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.651 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.651 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.651 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559360+0000", 2024-08-17T17:24:13.651 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4090000000000003e-06 2024-08-17T17:24:13.651 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.652 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.652 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.652 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524193+0000", 2024-08-17T17:24:13.652 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.96483273400000003 2024-08-17T17:24:13.652 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.652 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.652 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.652 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524201+0000", 2024-08-17T17:24:13.652 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.5280000000000003e-06 2024-08-17T17:24:13.652 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.652 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.652 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.652 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.005532+0000", 2024-08-17T17:24:13.653 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4813310959999999 2024-08-17T17:24:13.653 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.653 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.653 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.653 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.005618+0000", 2024-08-17T17:24:13.653 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6101999999999997e-05 2024-08-17T17:24:13.653 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.653 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.653 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:24:13.653 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.005742+0000", 2024-08-17T17:24:13.653 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012364099999999999 2024-08-17T17:24:13.653 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.653 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.654 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.654 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.007064+0000", 2024-08-17T17:24:13.654 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013215550000000001 2024-08-17T17:24:13.654 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.654 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.654 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.654 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.008743+0000", 2024-08-17T17:24:13.654 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016792140000000001 2024-08-17T17:24:13.654 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.654 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.654 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.655 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.021506+0000", 2024-08-17T17:24:13.655 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012763480000000001 2024-08-17T17:24:13.655 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.655 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.655 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.655 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.021565+0000", 2024-08-17T17:24:13.655 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8486000000000002e-05 2024-08-17T17:24:13.655 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.655 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.655 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.655 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.024227+0000", 2024-08-17T17:24:13.656 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026619759999999999 2024-08-17T17:24:13.656 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.656 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.656 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.656 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.656 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.656 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88363 7.7 7:e6097857:::benchmark_data_smithi063_196002_object88362:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:24:13.656 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:00.712141+0000", 2024-08-17T17:24:13.656 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 312.80986611100002, 2024-08-17T17:24:13.656 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9602415930000001, 2024-08-17T17:24:13.656 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.656 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:24:13.657 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.657 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:24:13.657 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:24:13.657 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88363 2024-08-17T17:24:13.657 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.657 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.657 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.657 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.657 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712141+0000", 2024-08-17T17:24:13.657 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.657 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.657 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.657 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.658 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712141+0000", 2024-08-17T17:24:13.658 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.658 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.658 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.658 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.658 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712138+0000", 2024-08-17T17:24:13.658 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999971 2024-08-17T17:24:13.658 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.658 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.658 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.658 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712181+0000", 2024-08-17T17:24:13.658 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3390999999999998e-05 2024-08-17T17:24:13.658 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.659 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.659 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.659 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712208+0000", 2024-08-17T17:24:13.659 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6259e-05 2024-08-17T17:24:13.659 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.659 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.659 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.659 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712219+0000", 2024-08-17T17:24:13.659 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1686e-05 2024-08-17T17:24:13.659 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.659 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.659 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.660 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.771097+0000", 2024-08-17T17:24:13.660 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.058877691000000003 2024-08-17T17:24:13.660 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.660 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.660 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.660 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.771106+0000", 2024-08-17T17:24:13.660 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6079999999999996e-06 2024-08-17T17:24:13.660 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.660 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.660 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.660 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.650544+0000", 2024-08-17T17:24:13.660 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.879438045 2024-08-17T17:24:13.660 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.660 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.660 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.661 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.650553+0000", 2024-08-17T17:24:13.661 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.6520000000000001e-06 2024-08-17T17:24:13.661 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.661 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.661 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.661 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.672283+0000", 2024-08-17T17:24:13.661 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0217297430000001 2024-08-17T17:24:13.661 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.661 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.661 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.661 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.672383+0000", 2024-08-17T17:24:13.661 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9489999999999998e-05 2024-08-17T17:24:13.662 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.662 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.662 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.662 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.662 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.662 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:170142 7.d 7:b454d72d:::benchmark_data_smithi063_196002_object58049:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1164)", 2024-08-17T17:24:13.662 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:20:43.923974+0000", 2024-08-17T17:24:13.662 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 209.59803322299999, 2024-08-17T17:24:13.662 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98031507699999998, 2024-08-17T17:24:13.662 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.662 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.663 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.663 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:24:13.663 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:24:13.663 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 170142 2024-08-17T17:24:13.663 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.663 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.663 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.663 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.663 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923974+0000", 2024-08-17T17:24:13.663 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.663 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.663 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.663 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.664 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923974+0000", 2024-08-17T17:24:13.664 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.664 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.664 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.664 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.664 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923973+0000", 2024-08-17T17:24:13.664 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:13.664 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.664 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.664 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.664 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923978+0000", 2024-08-17T17:24:13.664 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9819999999999999e-06 2024-08-17T17:24:13.664 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.665 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.665 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.665 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923989+0000", 2024-08-17T17:24:13.665 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1197e-05 2024-08-17T17:24:13.665 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.665 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.665 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.665 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923994+0000", 2024-08-17T17:24:13.665 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9599999999999999e-06 2024-08-17T17:24:13.665 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.665 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.665 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.666 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.924041+0000", 2024-08-17T17:24:13.666 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7633999999999998e-05 2024-08-17T17:24:13.666 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.666 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.666 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.666 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.924050+0000", 2024-08-17T17:24:13.666 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0940000000000008e-06 2024-08-17T17:24:13.666 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.666 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.666 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.666 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.863783+0000", 2024-08-17T17:24:13.666 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.93973320699999996 2024-08-17T17:24:13.667 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.667 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.667 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for degraded object", 2024-08-17T17:24:13.667 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.864178+0000", 2024-08-17T17:24:13.667 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00039516399999999999 2024-08-17T17:24:13.667 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.667 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.667 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.667 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.885932+0000", 2024-08-17T17:24:13.667 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.021753828999999999 2024-08-17T17:24:13.667 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.667 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.668 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.668 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.886015+0000", 2024-08-17T17:24:13.668 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3139999999999993e-05 2024-08-17T17:24:13.668 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.668 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.668 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:24:13.668 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.886349+0000", 2024-08-17T17:24:13.668 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000334555 2024-08-17T17:24:13.668 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.668 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.668 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.668 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.891760+0000", 2024-08-17T17:24:13.668 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0054106279999999998 2024-08-17T17:24:13.669 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.669 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.669 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.669 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.900574+0000", 2024-08-17T17:24:13.669 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0088143300000000004 2024-08-17T17:24:13.669 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.669 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.669 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.669 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.903791+0000", 2024-08-17T17:24:13.669 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0032164749999999999 2024-08-17T17:24:13.669 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.669 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.669 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.670 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.903846+0000", 2024-08-17T17:24:13.670 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5198000000000002e-05 2024-08-17T17:24:13.670 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.670 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.670 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.670 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.904289+0000", 2024-08-17T17:24:13.670 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00044295599999999999 2024-08-17T17:24:13.670 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.670 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.670 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.670 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.671 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.671 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186372 7.d 7:b527266b:::benchmark_data_smithi063_196002_object74279:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1172)", 2024-08-17T17:24:13.671 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.416634+0000", 2024-08-17T17:24:13.671 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 188.105373423, 2024-08-17T17:24:13.671 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.260521416, 2024-08-17T17:24:13.671 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.671 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:24:13.671 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.671 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:24:13.671 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:24:13.671 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186372 2024-08-17T17:24:13.672 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.672 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.672 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.672 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.672 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416634+0000", 2024-08-17T17:24:13.672 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.672 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.672 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.672 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.672 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416634+0000", 2024-08-17T17:24:13.672 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.672 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.673 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.673 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.673 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416632+0000", 2024-08-17T17:24:13.673 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:24:13.673 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.673 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.673 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.673 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416640+0000", 2024-08-17T17:24:13.673 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4399999999999999e-06 2024-08-17T17:24:13.673 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.673 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.674 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.674 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416653+0000", 2024-08-17T17:24:13.674 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3383999999999999e-05 2024-08-17T17:24:13.674 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.674 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.674 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.674 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416661+0000", 2024-08-17T17:24:13.674 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3100000000000001e-06 2024-08-17T17:24:13.674 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.674 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.674 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.675 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416714+0000", 2024-08-17T17:24:13.675 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3134999999999999e-05 2024-08-17T17:24:13.675 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.675 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.675 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.675 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416894+0000", 2024-08-17T17:24:13.675 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017996599999999999 2024-08-17T17:24:13.675 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.675 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.675 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:24:13.675 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.417361+0000", 2024-08-17T17:24:13.675 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00046639499999999998 2024-08-17T17:24:13.676 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.676 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.676 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.676 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.432229+0000", 2024-08-17T17:24:13.676 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.014867811 2024-08-17T17:24:13.676 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.676 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.676 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.676 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.432244+0000", 2024-08-17T17:24:13.676 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5260999999999998e-05 2024-08-17T17:24:13.676 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.676 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.676 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.677 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.623931+0000", 2024-08-17T17:24:13.677 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1916872569999999 2024-08-17T17:24:13.677 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.677 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.677 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:24:13.677 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.624058+0000", 2024-08-17T17:24:13.677 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000127033 2024-08-17T17:24:13.677 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.677 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.677 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.677 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.677057+0000", 2024-08-17T17:24:13.677 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.052998521999999999 2024-08-17T17:24:13.677 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.678 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.678 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.678 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.677155+0000", 2024-08-17T17:24:13.678 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8443999999999995e-05 2024-08-17T17:24:13.678 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.678 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.678 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.678 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.678 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.678 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186392 7.d 7:b05cf35d:::benchmark_data_smithi063_196002_object74299:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1173)", 2024-08-17T17:24:13.678 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.464967+0000", 2024-08-17T17:24:13.679 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 188.05704029200001, 2024-08-17T17:24:13.679 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2400478340000001, 2024-08-17T17:24:13.679 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.679 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.679 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.679 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:24:13.679 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:24:13.679 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186392 2024-08-17T17:24:13.679 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.679 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.679 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.679 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.679 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464967+0000", 2024-08-17T17:24:13.680 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.680 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.680 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.680 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.680 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464967+0000", 2024-08-17T17:24:13.680 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.680 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.680 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.680 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.680 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464966+0000", 2024-08-17T17:24:13.680 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:24:13.680 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.681 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.681 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.681 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464971+0000", 2024-08-17T17:24:13.681 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5700000000000003e-06 2024-08-17T17:24:13.681 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.681 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.681 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.681 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464979+0000", 2024-08-17T17:24:13.681 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.5019999999999992e-06 2024-08-17T17:24:13.681 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.681 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.681 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.681 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464985+0000", 2024-08-17T17:24:13.681 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6919999999999999e-06 2024-08-17T17:24:13.682 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.682 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.682 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.682 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.472207+0000", 2024-08-17T17:24:13.682 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0072221009999999999 2024-08-17T17:24:13.682 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.682 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.682 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.682 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.472220+0000", 2024-08-17T17:24:13.682 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.314e-05 2024-08-17T17:24:13.682 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.683 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.683 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.683 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.677176+0000", 2024-08-17T17:24:13.683 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.204956286 2024-08-17T17:24:13.683 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.683 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.683 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.683 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.677275+0000", 2024-08-17T17:24:13.683 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9032999999999999e-05 2024-08-17T17:24:13.683 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.683 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.683 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:24:13.684 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.677671+0000", 2024-08-17T17:24:13.684 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00039520700000000001 2024-08-17T17:24:13.684 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.684 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.684 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.684 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.682047+0000", 2024-08-17T17:24:13.684 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0043766289999999999 2024-08-17T17:24:13.684 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.684 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.684 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.684 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.686155+0000", 2024-08-17T17:24:13.684 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0041077830000000003 2024-08-17T17:24:13.685 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.685 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.685 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.685 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.704798+0000", 2024-08-17T17:24:13.685 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.018642549000000001 2024-08-17T17:24:13.685 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.685 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.685 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.685 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.704836+0000", 2024-08-17T17:24:13.685 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8770999999999999e-05 2024-08-17T17:24:13.685 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.685 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.686 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.686 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.705015+0000", 2024-08-17T17:24:13.686 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017840600000000001 2024-08-17T17:24:13.686 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.686 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.686 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.686 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.686 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.686 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186408 7.1d 7:bfe64f7d:::benchmark_data_smithi063_196002_object74315:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1173)", 2024-08-17T17:24:13.686 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.493415+0000", 2024-08-17T17:24:13.686 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 188.02859218699999, 2024-08-17T17:24:13.687 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1831635359999999, 2024-08-17T17:24:13.687 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:24:13.687 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:13.687 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:24:13.687 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:24:13.687 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:24:13.687 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186408 2024-08-17T17:24:13.687 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.687 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:24:13.687 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.687 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:24:13.687 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493415+0000", 2024-08-17T17:24:13.687 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.688 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.688 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.688 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:24:13.688 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493415+0000", 2024-08-17T17:24:13.688 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:24:13.688 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.688 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.688 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:24:13.688 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493414+0000", 2024-08-17T17:24:13.688 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:24:13.688 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.688 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.688 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:24:13.689 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493419+0000", 2024-08-17T17:24:13.689 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4320000000000001e-06 2024-08-17T17:24:13.689 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.689 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.689 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:24:13.689 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493429+0000", 2024-08-17T17:24:13.689 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0558e-05 2024-08-17T17:24:13.689 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.689 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.689 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:24:13.689 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493436+0000", 2024-08-17T17:24:13.689 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4459999999999998e-06 2024-08-17T17:24:13.690 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.690 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.690 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.690 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493483+0000", 2024-08-17T17:24:13.690 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7105999999999999e-05 2024-08-17T17:24:13.690 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.690 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.690 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:24:13.690 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493491+0000", 2024-08-17T17:24:13.690 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4919999999999993e-06 2024-08-17T17:24:13.690 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.690 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.691 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:24:13.691 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.540092+0000", 2024-08-17T17:24:13.691 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.046600636 2024-08-17T17:24:13.691 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.691 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.691 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:24:13.691 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.540231+0000", 2024-08-17T17:24:13.691 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013884100000000001 2024-08-17T17:24:13.691 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.691 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.691 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:24:13.691 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.540568+0000", 2024-08-17T17:24:13.692 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00033741400000000002 2024-08-17T17:24:13.692 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.692 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.692 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.692 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.546007+0000", 2024-08-17T17:24:13.692 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.005438539 2024-08-17T17:24:13.692 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.692 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.692 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:24:13.692 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.546046+0000", 2024-08-17T17:24:13.692 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9446000000000001e-05 2024-08-17T17:24:13.692 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.693 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.693 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:24:13.693 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676197+0000", 2024-08-17T17:24:13.693 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.13015059400000001 2024-08-17T17:24:13.693 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.693 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.693 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:24:13.693 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676334+0000", 2024-08-17T17:24:13.693 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000137718 2024-08-17T17:24:13.693 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:24:13.693 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:24:13.693 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:24:13.693 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676579+0000", 2024-08-17T17:24:13.694 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00024415 2024-08-17T17:24:13.694 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.694 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.694 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.694 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:24:13.694 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:24:13.694 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:24:13.735 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:13.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:13.837 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:13.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:13 smithi162 ceph-mon[104668]: 1.4 scrub starts 2024-08-17T17:24:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:13 smithi162 ceph-mon[104668]: 1.4 scrub ok 2024-08-17T17:24:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:13 smithi162 ceph-mon[104668]: pgmap v1584: 109 pgs: 5 unknown, 104 active+clean; 7.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:24:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:13 smithi162 ceph-mon[104668]: 1.2 scrub ok 2024-08-17T17:24:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:13 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:24:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:13 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:24:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:13 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:13 smithi162 ceph-mon[104668]: osdmap e1216: 8 total, 8 up, 8 in 2024-08-17T17:24:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:13 smithi162 ceph-mon[104668]: 1.6 scrub starts 2024-08-17T17:24:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:13 smithi162 ceph-mon[104668]: 1.6 scrub ok 2024-08-17T17:24:13.868 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:13.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:13.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:13.970 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:14.000 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:14.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[109843]: 1.4 scrub starts 2024-08-17T17:24:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[109843]: 1.4 scrub ok 2024-08-17T17:24:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[109843]: pgmap v1584: 109 pgs: 5 unknown, 104 active+clean; 7.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:24:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[109843]: 1.2 scrub ok 2024-08-17T17:24:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:24:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:24:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[109843]: osdmap e1216: 8 total, 8 up, 8 in 2024-08-17T17:24:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[109843]: 1.6 scrub starts 2024-08-17T17:24:14.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[109843]: 1.6 scrub ok 2024-08-17T17:24:14.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[114756]: 1.4 scrub starts 2024-08-17T17:24:14.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[114756]: 1.4 scrub ok 2024-08-17T17:24:14.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[114756]: pgmap v1584: 109 pgs: 5 unknown, 104 active+clean; 7.0 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:24:14.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[114756]: 1.2 scrub ok 2024-08-17T17:24:14.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:24:14.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:24:14.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:14.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[114756]: osdmap e1216: 8 total, 8 up, 8 in 2024-08-17T17:24:14.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[114756]: 1.6 scrub starts 2024-08-17T17:24:14.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:13 smithi063 ceph-mon[114756]: 1.6 scrub ok 2024-08-17T17:24:14.102 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:14.141 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:14.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:14.243 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:14.278 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:14.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:14.381 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:14.414 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:14.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:14.498 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:24:14.515 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:14.587 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:14.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:14.689 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:14.734 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:14.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:14.835 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:14.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:14 smithi162 ceph-mon[104668]: 1.14 scrub starts 2024-08-17T17:24:14.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:14 smithi162 ceph-mon[104668]: 1.14 scrub ok 2024-08-17T17:24:14.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:14 smithi162 ceph-mon[104668]: 1.5 scrub starts 2024-08-17T17:24:14.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:14 smithi162 ceph-mon[104668]: 1.5 scrub ok 2024-08-17T17:24:14.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:14 smithi162 ceph-mon[104668]: 1.0 scrub starts 2024-08-17T17:24:14.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:14 smithi162 ceph-mon[104668]: 1.0 scrub ok 2024-08-17T17:24:14.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:14 smithi162 ceph-mon[104668]: 1.1 scrub starts 2024-08-17T17:24:14.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:14 smithi162 ceph-mon[104668]: 1.1 scrub ok 2024-08-17T17:24:14.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:14 smithi162 ceph-mon[104668]: 1.12 scrub starts 2024-08-17T17:24:14.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:14 smithi162 ceph-mon[104668]: 1.12 scrub ok 2024-08-17T17:24:14.847 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:24:14.876 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:14.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:14.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:14.950 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[109843]: 1.14 scrub starts 2024-08-17T17:24:14.950 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[109843]: 1.14 scrub ok 2024-08-17T17:24:14.950 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[109843]: 1.5 scrub starts 2024-08-17T17:24:14.950 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[109843]: 1.5 scrub ok 2024-08-17T17:24:14.950 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[109843]: 1.0 scrub starts 2024-08-17T17:24:14.950 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[109843]: 1.0 scrub ok 2024-08-17T17:24:14.950 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[109843]: 1.1 scrub starts 2024-08-17T17:24:14.950 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[109843]: 1.1 scrub ok 2024-08-17T17:24:14.950 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[109843]: 1.12 scrub starts 2024-08-17T17:24:14.950 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[109843]: 1.12 scrub ok 2024-08-17T17:24:14.951 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[114756]: 1.14 scrub starts 2024-08-17T17:24:14.951 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[114756]: 1.14 scrub ok 2024-08-17T17:24:14.951 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[114756]: 1.5 scrub starts 2024-08-17T17:24:14.951 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[114756]: 1.5 scrub ok 2024-08-17T17:24:14.951 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[114756]: 1.0 scrub starts 2024-08-17T17:24:14.951 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[114756]: 1.0 scrub ok 2024-08-17T17:24:14.951 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[114756]: 1.1 scrub starts 2024-08-17T17:24:14.951 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[114756]: 1.1 scrub ok 2024-08-17T17:24:14.951 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[114756]: 1.12 scrub starts 2024-08-17T17:24:14.951 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:14 smithi063 ceph-mon[114756]: 1.12 scrub ok 2024-08-17T17:24:14.978 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:15.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.026 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:15.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:15.127 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:15.159 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:15.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:15.263 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:15.291 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:15.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:15.394 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:15.437 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:15.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:15.492 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:24:15.539 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:15.600 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:15 smithi063 ceph-mon[109843]: 1.c scrub starts 2024-08-17T17:24:15.600 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:15 smithi063 ceph-mon[109843]: 1.c scrub ok 2024-08-17T17:24:15.600 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:15 smithi063 ceph-mon[109843]: pgmap v1586: 109 pgs: 2 unknown, 107 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 904 op/s 2024-08-17T17:24:15.600 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:15 smithi063 ceph-mon[109843]: 1.d scrub starts 2024-08-17T17:24:15.600 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:15 smithi063 ceph-mon[109843]: 1.10 scrub starts 2024-08-17T17:24:15.600 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:15 smithi063 ceph-mon[109843]: 1.10 scrub ok 2024-08-17T17:24:15.600 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:15 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/398315890' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:15.611 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:15.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:15.629 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:24:15.629 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:24:15.667 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:24:15.668 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:24:15.713 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:15.720 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:24:15.720 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:24:15.759 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:24:15.760 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:24:15.762 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:15.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:15.792 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:24:15.792 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:24:15.826 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:24:15.827 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:24:15.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:15 smithi162 ceph-mon[104668]: 1.c scrub starts 2024-08-17T17:24:15.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:15 smithi162 ceph-mon[104668]: 1.c scrub ok 2024-08-17T17:24:15.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:15 smithi162 ceph-mon[104668]: pgmap v1586: 109 pgs: 2 unknown, 107 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 904 op/s 2024-08-17T17:24:15.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:15 smithi162 ceph-mon[104668]: 1.d scrub starts 2024-08-17T17:24:15.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:15 smithi162 ceph-mon[104668]: 1.10 scrub starts 2024-08-17T17:24:15.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:15 smithi162 ceph-mon[104668]: 1.10 scrub ok 2024-08-17T17:24:15.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:15 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/398315890' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:15.857 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:24:15.857 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:24:15.857 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:15 smithi063 ceph-mon[114756]: 1.c scrub starts 2024-08-17T17:24:15.857 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:15 smithi063 ceph-mon[114756]: 1.c scrub ok 2024-08-17T17:24:15.857 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:15 smithi063 ceph-mon[114756]: pgmap v1586: 109 pgs: 2 unknown, 107 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 904 op/s 2024-08-17T17:24:15.858 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:15 smithi063 ceph-mon[114756]: 1.d scrub starts 2024-08-17T17:24:15.858 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:15 smithi063 ceph-mon[114756]: 1.10 scrub starts 2024-08-17T17:24:15.858 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:15 smithi063 ceph-mon[114756]: 1.10 scrub ok 2024-08-17T17:24:15.858 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:15 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/398315890' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:15.864 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:15.894 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:24:15.895 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:24:15.895 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:15.918 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:15.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:15.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:16.020 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:16.058 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:16.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:16.160 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:16.198 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:16.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:16.300 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:16.339 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:16.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:16.441 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:16.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.477 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:16.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:16.578 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:16.606 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:16.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:16.708 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:16.736 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:16.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:16.838 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:16.881 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:16.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:16.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:16.920 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:16 smithi063 ceph-mon[109843]: 1.d scrub ok 2024-08-17T17:24:16.920 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:16 smithi063 ceph-mon[109843]: 1.7 scrub starts 2024-08-17T17:24:16.921 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:16 smithi063 ceph-mon[109843]: 1.7 scrub ok 2024-08-17T17:24:16.921 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:24:16.921 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:16 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:24:16.921 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:16 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:24:16.921 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:16.921 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:16 smithi063 ceph-mon[114756]: 1.d scrub ok 2024-08-17T17:24:16.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:16 smithi063 ceph-mon[114756]: 1.7 scrub starts 2024-08-17T17:24:16.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:16 smithi063 ceph-mon[114756]: 1.7 scrub ok 2024-08-17T17:24:16.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:24:16.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:16 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:24:16.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:16 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:24:16.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:16.983 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:17.016 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:17.017 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:16 smithi162 ceph-mon[104668]: 1.d scrub ok 2024-08-17T17:24:17.017 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:16 smithi162 ceph-mon[104668]: 1.7 scrub starts 2024-08-17T17:24:17.017 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:16 smithi162 ceph-mon[104668]: 1.7 scrub ok 2024-08-17T17:24:17.017 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:24:17.017 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:16 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-08-17T17:24:17.017 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:16 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:24:17.017 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:17.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:17.119 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:17.157 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:17.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:17.259 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:17.286 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:17.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:17.384 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:24:17.384 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:24:17.384 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-08-17T17:24:17.384 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:24:17.387 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:17.415 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:17.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:17.517 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:17.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.554 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:17.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:17.655 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:17.685 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:17.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:17.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:17 smithi063 ceph-mon[109843]: pgmap v1587: 109 pgs: 109 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 958 op/s 2024-08-17T17:24:17.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:17 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-08-17T17:24:17.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:17 smithi063 ceph-mon[109843]: osdmap e1217: 8 total, 8 up, 8 in 2024-08-17T17:24:17.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:17 smithi063 ceph-mon[114756]: pgmap v1587: 109 pgs: 109 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 958 op/s 2024-08-17T17:24:17.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:17 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-08-17T17:24:17.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:17 smithi063 ceph-mon[114756]: osdmap e1217: 8 total, 8 up, 8 in 2024-08-17T17:24:17.786 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:17.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.824 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:17.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:17.925 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:17.955 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:17.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:17.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:17.967 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:17 smithi162 ceph-mon[104668]: pgmap v1587: 109 pgs: 109 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 958 op/s 2024-08-17T17:24:17.967 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:17 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-08-17T17:24:17.967 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:17 smithi162 ceph-mon[104668]: osdmap e1217: 8 total, 8 up, 8 in 2024-08-17T17:24:18.057 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:18.082 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:18.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:18.184 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:18.251 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:18.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:18.255 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:18.353 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:18.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.392 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:18.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:18.435 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:24:18.435 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:24:18.494 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:18.532 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:18.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:18.634 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:18.659 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:18.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:18.748 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:18 smithi063 ceph-mon[114756]: osdmap e1218: 8 total, 8 up, 8 in 2024-08-17T17:24:18.748 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:18 smithi063 ceph-mon[114756]: osdmap e1219: 8 total, 8 up, 8 in 2024-08-17T17:24:18.749 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:18 smithi063 ceph-mon[109843]: osdmap e1218: 8 total, 8 up, 8 in 2024-08-17T17:24:18.749 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:18 smithi063 ceph-mon[109843]: osdmap e1219: 8 total, 8 up, 8 in 2024-08-17T17:24:18.762 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:18.793 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:18.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:18.895 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:18.923 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:18.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:18.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:19.025 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:19.054 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:19.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:19.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:18 smithi162 ceph-mon[104668]: osdmap e1218: 8 total, 8 up, 8 in 2024-08-17T17:24:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:18 smithi162 ceph-mon[104668]: osdmap e1219: 8 total, 8 up, 8 in 2024-08-17T17:24:19.153 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:24:19.156 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:19.201 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:19.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:19.303 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:19.350 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:19.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:19.452 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:19.454 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:24:19.479 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:19.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:19.513 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:24:19.583 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:19.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.620 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:19.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:19.630 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:24:19.721 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:19.763 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:19.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:19.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:19 smithi063 ceph-mon[109843]: pgmap v1590: 109 pgs: 2 peering, 6 unknown, 101 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 956 op/s 2024-08-17T17:24:19.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:19 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1436161646' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:24:19.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:19 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:19.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:19 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1436161646' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:24:19.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:19 smithi063 ceph-mon[114756]: pgmap v1590: 109 pgs: 2 peering, 6 unknown, 101 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 956 op/s 2024-08-17T17:24:19.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:19 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1436161646' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:24:19.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:19 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:19.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:19 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1436161646' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:24:19.865 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:19.914 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:19.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:19.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:20.001 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:19 smithi162 ceph-mon[104668]: pgmap v1590: 109 pgs: 2 peering, 6 unknown, 101 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 956 op/s 2024-08-17T17:24:20.001 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:19 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1436161646' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:24:20.001 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:19 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:20.002 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:19 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1436161646' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:24:20.016 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:20.061 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:20.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:20.163 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:20.195 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:20.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:20.297 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:20.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.333 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:20.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:20.434 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:20.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.466 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:20.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:20.567 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:20.612 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:20.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:20.714 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:20.751 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:20.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:20.840 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:20.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:20 smithi162 ceph-mon[104668]: osdmap e1220: 8 total, 8 up, 8 in 2024-08-17T17:24:20.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:20 smithi162 ceph-mon[104668]: osdmap e1221: 8 total, 8 up, 8 in 2024-08-17T17:24:20.853 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:20.900 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:20.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:20.947 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:20 smithi063 ceph-mon[109843]: osdmap e1220: 8 total, 8 up, 8 in 2024-08-17T17:24:20.947 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:20 smithi063 ceph-mon[109843]: osdmap e1221: 8 total, 8 up, 8 in 2024-08-17T17:24:20.947 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:20 smithi063 ceph-mon[114756]: osdmap e1220: 8 total, 8 up, 8 in 2024-08-17T17:24:20.948 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:20 smithi063 ceph-mon[114756]: osdmap e1221: 8 total, 8 up, 8 in 2024-08-17T17:24:21.002 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:21.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.030 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:21.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:21.131 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:21.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.177 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:21.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:21.187 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:24:21.191 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:24:21.279 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:21.313 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:21.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:21.416 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:21.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.450 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:21.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:21.551 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:21.581 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:21.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:21.683 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:21.705 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:21.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:21.746 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:21 smithi162 ceph-mon[104668]: pgmap v1593: 109 pgs: 2 peering, 2 remapped+peering, 5 unknown, 100 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 12 KiB/s rd, 37 KiB/s wr, 816 op/s; 26 KiB/s, 0 objects/s recovering 2024-08-17T17:24:21.746 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:21 smithi162 ceph-mon[104668]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:24:21.754 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:24:21.754 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:24:21.808 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:21.846 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:21.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:21.947 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:21.987 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:21.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:21.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:21 smithi063 ceph-mon[114756]: pgmap v1593: 109 pgs: 2 peering, 2 remapped+peering, 5 unknown, 100 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 12 KiB/s rd, 37 KiB/s wr, 816 op/s; 26 KiB/s, 0 objects/s recovering 2024-08-17T17:24:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:21 smithi063 ceph-mon[114756]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:24:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:21 smithi063 ceph-mon[109843]: pgmap v1593: 109 pgs: 2 peering, 2 remapped+peering, 5 unknown, 100 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 12 KiB/s rd, 37 KiB/s wr, 816 op/s; 26 KiB/s, 0 objects/s recovering 2024-08-17T17:24:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:21 smithi063 ceph-mon[109843]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-08-17T17:24:22.088 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:22.120 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:22.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:22.221 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:22.248 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:22.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:22.350 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:22.380 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:22.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:22.482 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:24:22.482 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:22.522 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:22.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:22.624 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:22.661 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:22.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:22.763 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:22.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.810 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:22.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:22.824 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1594,"stamp":"2024-08-17T17:24:20.634079+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7062922177,"num_objects":107789,"num_object_clones":0,"num_object_copies":323367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":107789,"num_whiteouts":0,"num_read":25,"num_read_kb":51,"num_write":145291,"num_write_kb":8095349,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":436266,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119478,"ondisk_log_size":119478,"up":327,"acting":326,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":329,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":24347212,"kb_used_data":20918844,"kb_used_omap":199,"kb_used_meta":3427768,"kb_avail":725614004,"statfs":{"total":767960285184,"available":743028740096,"internally_reserved":0,"allocated":21420896256,"data_stored":21401505397,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":3510035196},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,4,5,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-307655740,"num_objects":-4695,"num_object_clones":0,"num_object_copies":-14085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4695,"num_whiteouts":0,"num_read":24,"num_read_kb":50,"num_write":4709,"num_write_kb":147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":435020,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.026014"},"pg_stats":[{"pgid":"1.11","version":"369'149","reported_seq":1511,"reported_epoch":1217,"state":"peering","last_fresh":"2024-08-17T17:24:16.614521+0000","last_change":"2024-08-17T17:24:16.614704+0000","last_active":"2024-08-17T17:24:16.614521+0000","last_peered":"2024-08-17T17:24:12.584423+0000","last_clean":"2024-08-17T17:24:12.584423+0000","last_became_active":"2024-08-17T17:24:11.584108+0000","last_became_peered":"2024-08-17T17:24:11.584108+0000","last_unstale":"2024-08-17T17:24:16.614521+0000","last_undegraded":"2024-08-17T17:24:16.614521+0000","last_fullsized":"2024-08-17T17:24:16.614521+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:35.916381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:35.916381+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.005340521,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,2,3],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1536,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.643238+0000","last_change":"2024-08-17T17:21:06.463828+0000","last_active":"2024-08-17T17:24:19.643238+0000","last_peered":"2024-08-17T17:24:19.643238+0000","last_clean":"2024-08-17T17:24:19.643238+0000","last_became_active":"2024-08-17T17:21:06.463464+0000","last_became_peered":"2024-08-17T17:21:06.463464+0000","last_unstale":"2024-08-17T17:24:19.643238+0000","last_undegraded":"2024-08-17T17:24:19.643238+0000","last_fullsized":"2024-08-17T17:24:19.643238+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:46.465006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1146,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.585894+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:24:12.585894+0000","last_peered":"2024-08-17T17:24:12.585894+0000","last_clean":"2024-08-17T17:24:12.585894+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:24:12.585894+0000","last_undegraded":"2024-08-17T17:24:12.585894+0000","last_fullsized":"2024-08-17T17:24:12.585894+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"0'0","reported_seq":1,"reported_epoch":1217,"state":"unknown","last_fresh":"2024-08-17T17:24:16.608636+0000","last_change":"2024-08-17T17:24:17.585050+0000","last_active":"0.000000","last_peered":"0.000000","last_clean":"0.000000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:24:16.608636+0000","last_undegraded":"2024-08-17T17:24:16.608636+0000","last_fullsized":"2024-08-17T17:24:16.608636+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:15.468075+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:15.468075+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:58:27.086348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1157,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.584328+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:24:12.584328+0000","last_peered":"2024-08-17T17:24:12.584328+0000","last_clean":"2024-08-17T17:24:12.584328+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:24:12.584328+0000","last_undegraded":"2024-08-17T17:24:12.584328+0000","last_fullsized":"2024-08-17T17:24:12.584328+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1012,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.585722+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:24:12.585722+0000","last_peered":"2024-08-17T17:24:12.585722+0000","last_clean":"2024-08-17T17:24:12.585722+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:24:12.585722+0000","last_undegraded":"2024-08-17T17:24:12.585722+0000","last_fullsized":"2024-08-17T17:24:12.585722+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":713,"reported_epoch":1220,"state":"peering","last_fresh":"2024-08-17T17:24:18.632879+0000","last_change":"2024-08-17T17:24:20.339376+0000","last_active":"2024-08-17T17:24:20.339376+0000","last_peered":"2024-08-17T17:24:18.632879+0000","last_clean":"2024-08-17T17:20:59.909154+0000","last_became_active":"2024-08-17T17:24:18.632328+0000","last_became_peered":"2024-08-17T17:24:18.632328+0000","last_unstale":"2024-08-17T17:24:18.632879+0000","last_undegraded":"2024-08-17T17:24:18.632879+0000","last_fullsized":"2024-08-17T17:24:18.621167+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:09.230214+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.230214+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":25,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1087,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.639158+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:24:19.639158+0000","last_peered":"2024-08-17T17:24:19.639158+0000","last_clean":"2024-08-17T17:24:19.639158+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:24:19.639158+0000","last_undegraded":"2024-08-17T17:24:19.639158+0000","last_fullsized":"2024-08-17T17:24:19.639158+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1129,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.585615+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:24:12.585615+0000","last_peered":"2024-08-17T17:24:12.585615+0000","last_clean":"2024-08-17T17:24:12.585615+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:24:12.585615+0000","last_undegraded":"2024-08-17T17:24:12.585615+0000","last_fullsized":"2024-08-17T17:24:12.585615+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"0'0","reported_seq":1,"reported_epoch":1217,"state":"unknown","last_fresh":"2024-08-17T17:24:16.614992+0000","last_change":"2024-08-17T17:24:17.585142+0000","last_active":"0.000000","last_peered":"0.000000","last_clean":"0.000000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:24:16.614992+0000","last_undegraded":"2024-08-17T17:24:16.614992+0000","last_fullsized":"2024-08-17T17:24:16.614992+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:14.491020+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:14.491020+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:57:46.576323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1535,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.639688+0000","last_change":"2024-08-17T17:21:06.442598+0000","last_active":"2024-08-17T17:24:19.639688+0000","last_peered":"2024-08-17T17:24:19.639688+0000","last_clean":"2024-08-17T17:24:19.639688+0000","last_became_active":"2024-08-17T17:21:06.442264+0000","last_became_peered":"2024-08-17T17:21:06.442264+0000","last_unstale":"2024-08-17T17:24:19.639688+0000","last_undegraded":"2024-08-17T17:24:19.639688+0000","last_fullsized":"2024-08-17T17:24:19.639688+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:35.003620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1206,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.582085+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:24:12.582085+0000","last_peered":"2024-08-17T17:24:12.582085+0000","last_clean":"2024-08-17T17:24:12.582085+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:24:12.582085+0000","last_undegraded":"2024-08-17T17:24:12.582085+0000","last_fullsized":"2024-08-17T17:24:12.582085+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"0'0","reported_seq":1,"reported_epoch":1217,"state":"unknown","last_fresh":"2024-08-17T17:24:16.609476+0000","last_change":"2024-08-17T17:24:17.362733+0000","last_active":"0.000000","last_peered":"0.000000","last_clean":"0.000000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:24:16.609476+0000","last_undegraded":"2024-08-17T17:24:16.609476+0000","last_fullsized":"2024-08-17T17:24:16.609476+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:15.318697+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:15.318697+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:10.559838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"1217'8956","reported_seq":8988,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:17.575865+0000","last_change":"2024-08-17T17:22:10.861776+0000","last_active":"2024-08-17T17:24:17.575865+0000","last_peered":"2024-08-17T17:24:17.575865+0000","last_clean":"2024-08-17T17:24:17.575865+0000","last_became_active":"2024-08-17T17:22:10.861509+0000","last_became_peered":"2024-08-17T17:22:10.861509+0000","last_unstale":"2024-08-17T17:24:17.575865+0000","last_undegraded":"2024-08-17T17:24:17.575865+0000","last_fullsized":"2024-08-17T17:24:17.575865+0000","mapping_epoch":1195,"log_start":"1200'1198","ondisk_log_start":"1200'1198","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7758,"log_dups_size":0,"ondisk_log_size":7758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:16:43.850909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":444465152,"num_objects":6782,"num_object_clones":0,"num_object_copies":20346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8956,"num_write_kb":503616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1171,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.585793+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:24:12.585793+0000","last_peered":"2024-08-17T17:24:12.585793+0000","last_clean":"2024-08-17T17:24:12.585793+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:24:12.585793+0000","last_undegraded":"2024-08-17T17:24:12.585793+0000","last_fullsized":"2024-08-17T17:24:12.585793+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1144,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.584252+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:24:12.584252+0000","last_peered":"2024-08-17T17:24:12.584252+0000","last_clean":"2024-08-17T17:24:12.584252+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:24:12.584252+0000","last_undegraded":"2024-08-17T17:24:12.584252+0000","last_fullsized":"2024-08-17T17:24:12.584252+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1586,"reported_epoch":1219,"state":"peering","last_fresh":"2024-08-17T17:24:18.676735+0000","last_change":"2024-08-17T17:24:19.642910+0000","last_active":"2024-08-17T17:24:19.642910+0000","last_peered":"2024-08-17T17:24:18.676735+0000","last_clean":"2024-08-17T17:24:15.331254+0000","last_became_active":"2024-08-17T17:24:18.647898+0000","last_became_peered":"2024-08-17T17:24:18.647898+0000","last_unstale":"2024-08-17T17:24:18.676735+0000","last_undegraded":"2024-08-17T17:24:18.676735+0000","last_fullsized":"2024-08-17T17:24:18.629284+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:14.056604+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:14.056604+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,5,6,7],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.5","version":"1217'9130","reported_seq":9162,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:17.309770+0000","last_change":"2024-08-17T17:22:10.860210+0000","last_active":"2024-08-17T17:24:17.309770+0000","last_peered":"2024-08-17T17:24:17.309770+0000","last_clean":"2024-08-17T17:24:17.309770+0000","last_became_active":"2024-08-17T17:22:10.859854+0000","last_became_peered":"2024-08-17T17:22:10.859854+0000","last_unstale":"2024-08-17T17:24:17.309770+0000","last_undegraded":"2024-08-17T17:24:17.309770+0000","last_fullsized":"2024-08-17T17:24:17.309770+0000","mapping_epoch":1195,"log_start":"1202'2151","ondisk_log_start":"1202'2151","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":6979,"log_dups_size":0,"ondisk_log_size":6979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:14:31.313667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":446300160,"num_objects":6810,"num_object_clones":0,"num_object_copies":20430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9130,"num_write_kb":510080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1172,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.586026+0000","last_change":"2024-08-17T17:21:06.540689+0000","last_active":"2024-08-17T17:24:12.586026+0000","last_peered":"2024-08-17T17:24:12.586026+0000","last_clean":"2024-08-17T17:24:12.586026+0000","last_became_active":"2024-08-17T17:21:06.461614+0000","last_became_peered":"2024-08-17T17:21:06.461614+0000","last_unstale":"2024-08-17T17:24:12.586026+0000","last_undegraded":"2024-08-17T17:24:12.586026+0000","last_fullsized":"2024-08-17T17:24:12.586026+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:11.696898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1196,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.587621+0000","last_change":"2024-08-17T17:21:06.470389+0000","last_active":"2024-08-17T17:24:12.587621+0000","last_peered":"2024-08-17T17:24:12.587621+0000","last_clean":"2024-08-17T17:24:12.587621+0000","last_became_active":"2024-08-17T17:21:06.469966+0000","last_became_peered":"2024-08-17T17:21:06.469966+0000","last_unstale":"2024-08-17T17:24:12.587621+0000","last_undegraded":"2024-08-17T17:24:12.587621+0000","last_fullsized":"2024-08-17T17:24:12.587621+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:56.140616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":753,"reported_epoch":1217,"state":"peering","last_fresh":"2024-08-17T17:24:16.607393+0000","last_change":"2024-08-17T17:24:16.607644+0000","last_active":"2024-08-17T17:24:16.607393+0000","last_peered":"2024-08-17T17:24:12.583154+0000","last_clean":"2024-08-17T17:24:12.583154+0000","last_became_active":"2024-08-17T17:24:11.580746+0000","last_became_peered":"2024-08-17T17:24:11.580746+0000","last_unstale":"2024-08-17T17:24:16.607393+0000","last_undegraded":"2024-08-17T17:24:16.607393+0000","last_fullsized":"2024-08-17T17:24:16.607393+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:06.152592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:06.152592+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4,5,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.6","version":"1217'8996","reported_seq":9027,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:17.321508+0000","last_change":"2024-08-17T17:22:10.943999+0000","last_active":"2024-08-17T17:24:17.321508+0000","last_peered":"2024-08-17T17:24:17.321508+0000","last_clean":"2024-08-17T17:24:17.321508+0000","last_became_active":"2024-08-17T17:22:10.943661+0000","last_became_peered":"2024-08-17T17:22:10.943661+0000","last_unstale":"2024-08-17T17:24:17.321508+0000","last_undegraded":"2024-08-17T17:24:17.321508+0000","last_fullsized":"2024-08-17T17:24:17.321508+0000","mapping_epoch":1195,"log_start":"1202'1938","ondisk_log_start":"1202'1938","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7058,"log_dups_size":0,"ondisk_log_size":7058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:21:49.050541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":443678720,"num_objects":6770,"num_object_clones":0,"num_object_copies":20310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8996,"num_write_kb":504512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1130,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.586004+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:24:12.586004+0000","last_peered":"2024-08-17T17:24:12.586004+0000","last_clean":"2024-08-17T17:24:12.586004+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:24:12.586004+0000","last_undegraded":"2024-08-17T17:24:12.586004+0000","last_fullsized":"2024-08-17T17:24:12.586004+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1178,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.581812+0000","last_change":"2024-08-17T17:21:06.446798+0000","last_active":"2024-08-17T17:24:12.581812+0000","last_peered":"2024-08-17T17:24:12.581812+0000","last_clean":"2024-08-17T17:24:12.581812+0000","last_became_active":"2024-08-17T17:21:06.446205+0000","last_became_peered":"2024-08-17T17:21:06.446205+0000","last_unstale":"2024-08-17T17:24:12.581812+0000","last_undegraded":"2024-08-17T17:24:12.581812+0000","last_fullsized":"2024-08-17T17:24:12.581812+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:28.584048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"0'0","reported_seq":1,"reported_epoch":1217,"state":"unknown","last_fresh":"2024-08-17T17:24:16.609309+0000","last_change":"2024-08-17T17:24:17.291195+0000","last_active":"0.000000","last_peered":"0.000000","last_clean":"0.000000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:24:16.609309+0000","last_undegraded":"2024-08-17T17:24:16.609309+0000","last_fullsized":"2024-08-17T17:24:16.609309+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:08.218709+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:08.218709+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:41:21.299781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.7","version":"1220'9064","reported_seq":9125,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:20.339541+0000","last_change":"2024-08-17T17:22:10.900862+0000","last_active":"2024-08-17T17:24:20.339541+0000","last_peered":"2024-08-17T17:24:20.339541+0000","last_clean":"2024-08-17T17:24:20.339541+0000","last_became_active":"2024-08-17T17:22:10.900552+0000","last_became_peered":"2024-08-17T17:22:10.900552+0000","last_unstale":"2024-08-17T17:24:20.339541+0000","last_undegraded":"2024-08-17T17:24:20.339541+0000","last_fullsized":"2024-08-17T17:24:20.339541+0000","mapping_epoch":1195,"log_start":"1202'2811","ondisk_log_start":"1202'2811","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":6253,"log_dups_size":0,"ondisk_log_size":6253,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:06.689517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":427687936,"num_objects":6526,"num_object_clones":0,"num_object_copies":19578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9064,"num_write_kb":498880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1159,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.584336+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:24:12.584336+0000","last_peered":"2024-08-17T17:24:12.584336+0000","last_clean":"2024-08-17T17:24:12.584336+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:24:12.584336+0000","last_undegraded":"2024-08-17T17:24:12.584336+0000","last_fullsized":"2024-08-17T17:24:12.584336+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":941,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.587683+0000","last_change":"2024-08-17T17:21:06.460905+0000","last_active":"2024-08-17T17:24:12.587683+0000","last_peered":"2024-08-17T17:24:12.587683+0000","last_clean":"2024-08-17T17:24:12.587683+0000","last_became_active":"2024-08-17T17:21:06.460592+0000","last_became_peered":"2024-08-17T17:21:06.460592+0000","last_unstale":"2024-08-17T17:24:12.587683+0000","last_undegraded":"2024-08-17T17:24:12.587683+0000","last_fullsized":"2024-08-17T17:24:12.587683+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:10.146916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.0","version":"1220'8955","reported_seq":9017,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:20.319047+0000","last_change":"2024-08-17T17:22:10.900154+0000","last_active":"2024-08-17T17:24:20.319047+0000","last_peered":"2024-08-17T17:24:20.319047+0000","last_clean":"2024-08-17T17:24:20.319047+0000","last_became_active":"2024-08-17T17:22:10.899766+0000","last_became_peered":"2024-08-17T17:22:10.899766+0000","last_unstale":"2024-08-17T17:24:20.319047+0000","last_undegraded":"2024-08-17T17:24:20.319047+0000","last_fullsized":"2024-08-17T17:24:20.319047+0000","mapping_epoch":1195,"log_start":"1202'2619","ondisk_log_start":"1202'2619","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":6336,"log_dups_size":0,"ondisk_log_size":6336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:02:14.160943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":423821312,"num_objects":6467,"num_object_clones":0,"num_object_copies":19401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8955,"num_write_kb":493504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1538,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.585488+0000","last_change":"2024-08-17T17:21:16.100398+0000","last_active":"2024-08-17T17:24:12.585488+0000","last_peered":"2024-08-17T17:24:12.585488+0000","last_clean":"2024-08-17T17:24:12.585488+0000","last_became_active":"2024-08-17T17:18:28.521573+0000","last_became_peered":"2024-08-17T17:18:28.521573+0000","last_unstale":"2024-08-17T17:24:12.585488+0000","last_undegraded":"2024-08-17T17:24:12.585488+0000","last_fullsized":"2024-08-17T17:24:12.585488+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:21:16.100258+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:21:16.100258+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:57.628353+0000","scrub_duration":0.011045358999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1557,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.640080+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:24:19.640080+0000","last_peered":"2024-08-17T17:24:19.640080+0000","last_clean":"2024-08-17T17:24:19.640080+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:24:19.640080+0000","last_undegraded":"2024-08-17T17:24:19.640080+0000","last_fullsized":"2024-08-17T17:24:19.640080+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1162,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.587748+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:24:12.587748+0000","last_peered":"2024-08-17T17:24:12.587748+0000","last_clean":"2024-08-17T17:24:12.587748+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:24:12.587748+0000","last_undegraded":"2024-08-17T17:24:12.587748+0000","last_fullsized":"2024-08-17T17:24:12.587748+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"1217'8787","reported_seq":8844,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:16.667787+0000","last_change":"2024-08-17T17:22:10.944783+0000","last_active":"2024-08-17T17:24:16.667787+0000","last_peered":"2024-08-17T17:24:16.667787+0000","last_clean":"2024-08-17T17:24:16.667787+0000","last_became_active":"2024-08-17T17:22:10.944487+0000","last_became_peered":"2024-08-17T17:22:10.944487+0000","last_unstale":"2024-08-17T17:24:16.667787+0000","last_undegraded":"2024-08-17T17:24:16.667787+0000","last_fullsized":"2024-08-17T17:24:16.667787+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":8787,"log_dups_size":0,"ondisk_log_size":8787,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:57:11.921301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":438894592,"num_objects":6697,"num_object_clones":0,"num_object_copies":20091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8787,"num_write_kb":495488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1484,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.584093+0000","last_change":"2024-08-17T17:18:32.884334+0000","last_active":"2024-08-17T17:24:12.584093+0000","last_peered":"2024-08-17T17:24:12.584093+0000","last_clean":"2024-08-17T17:24:12.584093+0000","last_became_active":"2024-08-17T17:18:28.520186+0000","last_became_peered":"2024-08-17T17:18:28.520186+0000","last_unstale":"2024-08-17T17:24:12.584093+0000","last_undegraded":"2024-08-17T17:24:12.584093+0000","last_fullsized":"2024-08-17T17:24:12.584093+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:32.884246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:32.884246+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:49.286656+0000","scrub_duration":0.001842677,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1179,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.581969+0000","last_change":"2024-08-17T17:21:06.446841+0000","last_active":"2024-08-17T17:24:12.581969+0000","last_peered":"2024-08-17T17:24:12.581969+0000","last_clean":"2024-08-17T17:24:12.581969+0000","last_became_active":"2024-08-17T17:21:06.446271+0000","last_became_peered":"2024-08-17T17:21:06.446271+0000","last_unstale":"2024-08-17T17:24:12.581969+0000","last_undegraded":"2024-08-17T17:24:12.581969+0000","last_fullsized":"2024-08-17T17:24:12.581969+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:20.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1195,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.588044+0000","last_change":"2024-08-17T17:21:06.480911+0000","last_active":"2024-08-17T17:24:12.588044+0000","last_peered":"2024-08-17T17:24:12.588044+0000","last_clean":"2024-08-17T17:24:12.588044+0000","last_became_active":"2024-08-17T17:21:06.480551+0000","last_became_peered":"2024-08-17T17:21:06.480551+0000","last_unstale":"2024-08-17T17:24:12.588044+0000","last_undegraded":"2024-08-17T17:24:12.588044+0000","last_fullsized":"2024-08-17T17:24:12.588044+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:26.003704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":709,"reported_epoch":1218,"state":"remapped+peering","last_fresh":"2024-08-17T17:24:17.622861+0000","last_change":"2024-08-17T17:21:06.422617+0000","last_active":"2024-08-17T17:21:05.427225+0000","last_peered":"2024-08-17T17:20:59.909154+0000","last_clean":"2024-08-17T17:20:59.909154+0000","last_became_active":"2024-08-17T17:20:58.917665+0000","last_became_peered":"2024-08-17T17:20:58.917665+0000","last_unstale":"2024-08-17T17:24:17.622861+0000","last_undegraded":"2024-08-17T17:24:17.622861+0000","last_fullsized":"2024-08-17T17:24:17.622861+0000","mapping_epoch":1218,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:22:09.230214+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.230214+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":25,"num_write":7,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"1217'9195","reported_seq":9227,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:17.277270+0000","last_change":"2024-08-17T17:22:10.944020+0000","last_active":"2024-08-17T17:24:17.277270+0000","last_peered":"2024-08-17T17:24:17.277270+0000","last_clean":"2024-08-17T17:24:17.277270+0000","last_became_active":"2024-08-17T17:22:10.943694+0000","last_became_peered":"2024-08-17T17:22:10.943694+0000","last_unstale":"2024-08-17T17:24:17.277270+0000","last_undegraded":"2024-08-17T17:24:17.277270+0000","last_fullsized":"2024-08-17T17:24:17.277270+0000","mapping_epoch":1195,"log_start":"1202'2622","ondisk_log_start":"1202'2622","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":6573,"log_dups_size":0,"ondisk_log_size":6573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:34.525383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":455933952,"num_objects":6957,"num_object_clones":0,"num_object_copies":20871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6957,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9195,"num_write_kb":516864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1164,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.587676+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:24:12.587676+0000","last_peered":"2024-08-17T17:24:12.587676+0000","last_clean":"2024-08-17T17:24:12.587676+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:24:12.587676+0000","last_undegraded":"2024-08-17T17:24:12.587676+0000","last_fullsized":"2024-08-17T17:24:12.587676+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1180,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.588380+0000","last_change":"2024-08-17T17:21:06.451521+0000","last_active":"2024-08-17T17:24:12.588380+0000","last_peered":"2024-08-17T17:24:12.588380+0000","last_clean":"2024-08-17T17:24:12.588380+0000","last_became_active":"2024-08-17T17:21:06.451197+0000","last_became_peered":"2024-08-17T17:21:06.451197+0000","last_unstale":"2024-08-17T17:24:12.588380+0000","last_undegraded":"2024-08-17T17:24:12.588380+0000","last_fullsized":"2024-08-17T17:24:12.588380+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:04.000692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"1217'9130","reported_seq":9160,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:17.584516+0000","last_change":"2024-08-17T17:22:10.900282+0000","last_active":"2024-08-17T17:24:17.584516+0000","last_peered":"2024-08-17T17:24:17.584516+0000","last_clean":"2024-08-17T17:24:17.584516+0000","last_became_active":"2024-08-17T17:22:10.899947+0000","last_became_peered":"2024-08-17T17:22:10.899947+0000","last_unstale":"2024-08-17T17:24:17.584516+0000","last_undegraded":"2024-08-17T17:24:17.584516+0000","last_fullsized":"2024-08-17T17:24:17.584516+0000","mapping_epoch":1195,"log_start":"1200'1373","ondisk_log_start":"1200'1373","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7757,"log_dups_size":0,"ondisk_log_size":7757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:10.148524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":449052672,"num_objects":6852,"num_object_clones":0,"num_object_copies":20556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9130,"num_write_kb":511424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1484,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.584073+0000","last_change":"2024-08-17T17:18:31.850829+0000","last_active":"2024-08-17T17:24:12.584073+0000","last_peered":"2024-08-17T17:24:12.584073+0000","last_clean":"2024-08-17T17:24:12.584073+0000","last_became_active":"2024-08-17T17:18:28.520946+0000","last_became_peered":"2024-08-17T17:18:28.520946+0000","last_unstale":"2024-08-17T17:24:12.584073+0000","last_undegraded":"2024-08-17T17:24:12.584073+0000","last_fullsized":"2024-08-17T17:24:12.584073+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1142,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:18:31.850745+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:18:31.850745+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:04:17.218084+0000","scrub_duration":0.001317447,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1538,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.641647+0000","last_change":"2024-08-17T17:21:06.441118+0000","last_active":"2024-08-17T17:24:19.641647+0000","last_peered":"2024-08-17T17:24:19.641647+0000","last_clean":"2024-08-17T17:24:19.641647+0000","last_became_active":"2024-08-17T17:21:06.440800+0000","last_became_peered":"2024-08-17T17:21:06.440800+0000","last_unstale":"2024-08-17T17:24:19.641647+0000","last_undegraded":"2024-08-17T17:24:19.641647+0000","last_fullsized":"2024-08-17T17:24:19.641647+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:08.091528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":38,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:16.607421+0000","last_change":"2024-08-17T17:22:48.374766+0000","last_active":"2024-08-17T17:24:16.607421+0000","last_peered":"2024-08-17T17:24:16.607421+0000","last_clean":"2024-08-17T17:24:16.607421+0000","last_became_active":"2024-08-17T17:22:48.373556+0000","last_became_peered":"2024-08-17T17:22:48.373556+0000","last_unstale":"2024-08-17T17:24:16.607421+0000","last_undegraded":"2024-08-17T17:24:16.607421+0000","last_fullsized":"2024-08-17T17:24:16.607421+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:33.362080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.9","version":"1217'9023","reported_seq":9054,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:17.265669+0000","last_change":"2024-08-17T17:22:10.944057+0000","last_active":"2024-08-17T17:24:17.265669+0000","last_peered":"2024-08-17T17:24:17.265669+0000","last_clean":"2024-08-17T17:24:17.265669+0000","last_became_active":"2024-08-17T17:22:10.943681+0000","last_became_peered":"2024-08-17T17:22:10.943681+0000","last_unstale":"2024-08-17T17:24:17.265669+0000","last_undegraded":"2024-08-17T17:24:17.265669+0000","last_fullsized":"2024-08-17T17:24:17.265669+0000","mapping_epoch":1195,"log_start":"1202'1975","ondisk_log_start":"1202'1975","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7048,"log_dups_size":0,"ondisk_log_size":7048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:24.524487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":450953216,"num_objects":6881,"num_object_clones":0,"num_object_copies":20643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9023,"num_write_kb":508928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1560,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.542832+0000","last_change":"2024-08-17T17:24:13.542832+0000","last_active":"2024-08-17T17:24:13.542832+0000","last_peered":"2024-08-17T17:24:13.542832+0000","last_clean":"2024-08-17T17:24:13.542832+0000","last_became_active":"2024-08-17T17:24:11.570282+0000","last_became_peered":"2024-08-17T17:24:11.570282+0000","last_unstale":"2024-08-17T17:24:13.542832+0000","last_undegraded":"2024-08-17T17:24:13.542832+0000","last_fullsized":"2024-08-17T17:24:13.542832+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:49.414162+0000","scrub_duration":0.017597167,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1174,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.583952+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:24:12.583952+0000","last_peered":"2024-08-17T17:24:12.583952+0000","last_clean":"2024-08-17T17:24:12.583952+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:24:12.583952+0000","last_undegraded":"2024-08-17T17:24:12.583952+0000","last_fullsized":"2024-08-17T17:24:12.583952+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1503,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.637340+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:24:19.637340+0000","last_peered":"2024-08-17T17:24:19.637340+0000","last_clean":"2024-08-17T17:24:19.637340+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:24:19.637340+0000","last_undegraded":"2024-08-17T17:24:19.637340+0000","last_fullsized":"2024-08-17T17:24:19.637340+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1145,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.582963+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:24:12.582963+0000","last_peered":"2024-08-17T17:24:12.582963+0000","last_clean":"2024-08-17T17:24:12.582963+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:24:12.582963+0000","last_undegraded":"2024-08-17T17:24:12.582963+0000","last_fullsized":"2024-08-17T17:24:12.582963+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"1217'9076","reported_seq":9132,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:16.660936+0000","last_change":"2024-08-17T17:22:10.900900+0000","last_active":"2024-08-17T17:24:16.660936+0000","last_peered":"2024-08-17T17:24:16.660936+0000","last_clean":"2024-08-17T17:24:16.660936+0000","last_became_active":"2024-08-17T17:22:10.899812+0000","last_became_peered":"2024-08-17T17:22:10.899812+0000","last_unstale":"2024-08-17T17:24:16.660936+0000","last_undegraded":"2024-08-17T17:24:16.660936+0000","last_fullsized":"2024-08-17T17:24:16.660936+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":9076,"log_dups_size":0,"ondisk_log_size":9076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:49:09.639747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":453246976,"num_objects":6916,"num_object_clones":0,"num_object_copies":20748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6916,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9076,"num_write_kb":511744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":798,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.582286+0000","last_change":"2024-08-17T17:24:11.936322+0000","last_active":"2024-08-17T17:24:12.582286+0000","last_peered":"2024-08-17T17:24:12.582286+0000","last_clean":"2024-08-17T17:24:12.582286+0000","last_became_active":"2024-08-17T17:24:11.574846+0000","last_became_peered":"2024-08-17T17:24:11.574846+0000","last_unstale":"2024-08-17T17:24:12.582286+0000","last_undegraded":"2024-08-17T17:24:12.582286+0000","last_fullsized":"2024-08-17T17:24:12.582286+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:11.936236+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:11.936236+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:31.674851+0000","scrub_duration":0.0035797139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1209,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.585587+0000","last_change":"2024-08-17T17:21:06.430639+0000","last_active":"2024-08-17T17:24:12.585587+0000","last_peered":"2024-08-17T17:24:12.585587+0000","last_clean":"2024-08-17T17:24:12.585587+0000","last_became_active":"2024-08-17T17:21:06.430297+0000","last_became_peered":"2024-08-17T17:21:06.430297+0000","last_unstale":"2024-08-17T17:24:12.585587+0000","last_undegraded":"2024-08-17T17:24:12.585587+0000","last_fullsized":"2024-08-17T17:24:12.585587+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:57.794620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1013,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.587530+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:24:12.587530+0000","last_peered":"2024-08-17T17:24:12.587530+0000","last_clean":"2024-08-17T17:24:12.587530+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:24:12.587530+0000","last_undegraded":"2024-08-17T17:24:12.587530+0000","last_fullsized":"2024-08-17T17:24:12.587530+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":24,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.584333+0000","last_change":"2024-08-17T17:22:48.370769+0000","last_active":"2024-08-17T17:24:12.584333+0000","last_peered":"2024-08-17T17:24:12.584333+0000","last_clean":"2024-08-17T17:24:12.584333+0000","last_became_active":"2024-08-17T17:22:48.370531+0000","last_became_peered":"2024-08-17T17:22:48.370531+0000","last_unstale":"2024-08-17T17:24:12.584333+0000","last_undegraded":"2024-08-17T17:24:12.584333+0000","last_fullsized":"2024-08-17T17:24:12.584333+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:02.943823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.b","version":"1220'9246","reported_seq":9308,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:20.299897+0000","last_change":"2024-08-17T17:22:10.900740+0000","last_active":"2024-08-17T17:24:20.299897+0000","last_peered":"2024-08-17T17:24:20.299897+0000","last_clean":"2024-08-17T17:24:20.299897+0000","last_became_active":"2024-08-17T17:22:10.900452+0000","last_became_peered":"2024-08-17T17:22:10.900452+0000","last_unstale":"2024-08-17T17:24:20.299897+0000","last_undegraded":"2024-08-17T17:24:20.299897+0000","last_fullsized":"2024-08-17T17:24:20.299897+0000","mapping_epoch":1195,"log_start":"1202'2903","ondisk_log_start":"1202'2903","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":6343,"log_dups_size":0,"ondisk_log_size":6343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:49:25.950741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428867584,"num_objects":6544,"num_object_clones":0,"num_object_copies":19632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9246,"num_write_kb":505280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1580,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.585629+0000","last_change":"2024-08-17T17:24:12.519946+0000","last_active":"2024-08-17T17:24:12.585629+0000","last_peered":"2024-08-17T17:24:12.585629+0000","last_clean":"2024-08-17T17:24:12.585629+0000","last_became_active":"2024-08-17T17:24:11.569747+0000","last_became_peered":"2024-08-17T17:24:11.569747+0000","last_unstale":"2024-08-17T17:24:12.585629+0000","last_undegraded":"2024-08-17T17:24:12.585629+0000","last_fullsized":"2024-08-17T17:24:12.585629+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.519838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.519838+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:56.003993+0000","scrub_duration":0.0068224740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1595,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:16.609160+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:24:16.609160+0000","last_peered":"2024-08-17T17:24:16.609160+0000","last_clean":"2024-08-17T17:24:16.609160+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:24:16.609160+0000","last_undegraded":"2024-08-17T17:24:16.609160+0000","last_fullsized":"2024-08-17T17:24:16.609160+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":987,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:16.607367+0000","last_change":"2024-08-17T17:21:06.428684+0000","last_active":"2024-08-17T17:24:16.607367+0000","last_peered":"2024-08-17T17:24:16.607367+0000","last_clean":"2024-08-17T17:24:16.607367+0000","last_became_active":"2024-08-17T17:21:06.428178+0000","last_became_peered":"2024-08-17T17:21:06.428178+0000","last_unstale":"2024-08-17T17:24:16.607367+0000","last_undegraded":"2024-08-17T17:24:16.607367+0000","last_fullsized":"2024-08-17T17:24:16.607367+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:14.836119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":856,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.584039+0000","last_change":"2024-08-17T17:21:06.443565+0000","last_active":"2024-08-17T17:24:12.584039+0000","last_peered":"2024-08-17T17:24:12.584039+0000","last_clean":"2024-08-17T17:24:12.584039+0000","last_became_active":"2024-08-17T17:21:06.443215+0000","last_became_peered":"2024-08-17T17:21:06.443215+0000","last_unstale":"2024-08-17T17:24:12.584039+0000","last_undegraded":"2024-08-17T17:24:12.584039+0000","last_fullsized":"2024-08-17T17:24:12.584039+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:47.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.8","version":"1217'9122","reported_seq":9153,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:17.567058+0000","last_change":"2024-08-17T17:22:10.859676+0000","last_active":"2024-08-17T17:24:17.567058+0000","last_peered":"2024-08-17T17:24:17.567058+0000","last_clean":"2024-08-17T17:24:17.567058+0000","last_became_active":"2024-08-17T17:22:10.859289+0000","last_became_peered":"2024-08-17T17:22:10.859289+0000","last_unstale":"2024-08-17T17:24:17.567058+0000","last_undegraded":"2024-08-17T17:24:17.567058+0000","last_fullsized":"2024-08-17T17:24:17.567058+0000","mapping_epoch":1195,"log_start":"1200'1376","ondisk_log_start":"1200'1376","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7746,"log_dups_size":0,"ondisk_log_size":7746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:43.302993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":443154432,"num_objects":6762,"num_object_clones":0,"num_object_copies":20286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9122,"num_write_kb":508288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1522,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.795227+0000","last_change":"2024-08-17T17:24:13.795227+0000","last_active":"2024-08-17T17:24:13.795227+0000","last_peered":"2024-08-17T17:24:13.795227+0000","last_clean":"2024-08-17T17:24:13.795227+0000","last_became_active":"2024-08-17T17:24:11.578352+0000","last_became_peered":"2024-08-17T17:24:11.578352+0000","last_unstale":"2024-08-17T17:24:13.795227+0000","last_undegraded":"2024-08-17T17:24:13.795227+0000","last_fullsized":"2024-08-17T17:24:13.795227+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:36:39.440808+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1201,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.584223+0000","last_change":"2024-08-17T17:21:06.456978+0000","last_active":"2024-08-17T17:24:12.584223+0000","last_peered":"2024-08-17T17:24:12.584223+0000","last_clean":"2024-08-17T17:24:12.584223+0000","last_became_active":"2024-08-17T17:21:06.452265+0000","last_became_peered":"2024-08-17T17:21:06.452265+0000","last_unstale":"2024-08-17T17:24:12.584223+0000","last_undegraded":"2024-08-17T17:24:12.584223+0000","last_fullsized":"2024-08-17T17:24:12.584223+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:52.893746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1182,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.587985+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:24:12.587985+0000","last_peered":"2024-08-17T17:24:12.587985+0000","last_clean":"2024-08-17T17:24:12.587985+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:24:12.587985+0000","last_undegraded":"2024-08-17T17:24:12.587985+0000","last_fullsized":"2024-08-17T17:24:12.587985+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1063,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:16.608356+0000","last_change":"2024-08-17T17:21:06.435754+0000","last_active":"2024-08-17T17:24:16.608356+0000","last_peered":"2024-08-17T17:24:16.608356+0000","last_clean":"2024-08-17T17:24:16.608356+0000","last_became_active":"2024-08-17T17:21:06.435442+0000","last_became_peered":"2024-08-17T17:21:06.435442+0000","last_unstale":"2024-08-17T17:24:16.608356+0000","last_undegraded":"2024-08-17T17:24:16.608356+0000","last_fullsized":"2024-08-17T17:24:16.608356+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:34.849042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.e","version":"1217'8956","reported_seq":9012,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:16.635405+0000","last_change":"2024-08-17T17:22:10.944986+0000","last_active":"2024-08-17T17:24:16.635405+0000","last_peered":"2024-08-17T17:24:16.635405+0000","last_clean":"2024-08-17T17:24:16.635405+0000","last_became_active":"2024-08-17T17:22:10.944267+0000","last_became_peered":"2024-08-17T17:22:10.944267+0000","last_unstale":"2024-08-17T17:24:16.635405+0000","last_undegraded":"2024-08-17T17:24:16.635405+0000","last_fullsized":"2024-08-17T17:24:16.635405+0000","mapping_epoch":1195,"log_start":"0'0","ondisk_log_start":"0'0","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":8956,"log_dups_size":0,"ondisk_log_size":8956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:05:11.441347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":444858368,"num_objects":6788,"num_object_clones":0,"num_object_copies":20364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8956,"num_write_kb":503808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":764,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:15.948372+0000","last_change":"2024-08-17T17:24:15.948372+0000","last_active":"2024-08-17T17:24:15.948372+0000","last_peered":"2024-08-17T17:24:15.948372+0000","last_clean":"2024-08-17T17:24:15.948372+0000","last_became_active":"2024-08-17T17:24:11.581114+0000","last_became_peered":"2024-08-17T17:24:11.581114+0000","last_unstale":"2024-08-17T17:24:15.948372+0000","last_undegraded":"2024-08-17T17:24:15.948372+0000","last_fullsized":"2024-08-17T17:24:15.948372+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:15.948179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:15.948179+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:53.160009+0000","scrub_duration":0.014325694999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1175,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.584531+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:24:12.584531+0000","last_peered":"2024-08-17T17:24:12.584531+0000","last_clean":"2024-08-17T17:24:12.584531+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:24:12.584531+0000","last_undegraded":"2024-08-17T17:24:12.584531+0000","last_fullsized":"2024-08-17T17:24:12.584531+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":866,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.585871+0000","last_change":"2024-08-17T17:21:06.434991+0000","last_active":"2024-08-17T17:24:12.585871+0000","last_peered":"2024-08-17T17:24:12.585871+0000","last_clean":"2024-08-17T17:24:12.585871+0000","last_became_active":"2024-08-17T17:21:06.434630+0000","last_became_peered":"2024-08-17T17:21:06.434630+0000","last_unstale":"2024-08-17T17:24:12.585871+0000","last_undegraded":"2024-08-17T17:24:12.585871+0000","last_fullsized":"2024-08-17T17:24:12.585871+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:24.104609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":808,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.584148+0000","last_change":"2024-08-17T17:21:06.462532+0000","last_active":"2024-08-17T17:24:12.584148+0000","last_peered":"2024-08-17T17:24:12.584148+0000","last_clean":"2024-08-17T17:24:12.584148+0000","last_became_active":"2024-08-17T17:21:06.462183+0000","last_became_peered":"2024-08-17T17:21:06.462183+0000","last_unstale":"2024-08-17T17:24:12.584148+0000","last_undegraded":"2024-08-17T17:24:12.584148+0000","last_fullsized":"2024-08-17T17:24:12.584148+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:50.033322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.f","version":"1220'9224","reported_seq":9286,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:20.323135+0000","last_change":"2024-08-17T17:22:10.900592+0000","last_active":"2024-08-17T17:24:20.323135+0000","last_peered":"2024-08-17T17:24:20.323135+0000","last_clean":"2024-08-17T17:24:20.323135+0000","last_became_active":"2024-08-17T17:22:10.900279+0000","last_became_peered":"2024-08-17T17:22:10.900279+0000","last_unstale":"2024-08-17T17:24:20.323135+0000","last_undegraded":"2024-08-17T17:24:20.323135+0000","last_fullsized":"2024-08-17T17:24:20.323135+0000","mapping_epoch":1195,"log_start":"1202'2961","ondisk_log_start":"1202'2961","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":6263,"log_dups_size":0,"ondisk_log_size":6263,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:49:23.819372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428474368,"num_objects":6538,"num_object_clones":0,"num_object_copies":19614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9224,"num_write_kb":504384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":781,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.943970+0000","last_change":"2024-08-17T17:24:12.943970+0000","last_active":"2024-08-17T17:24:12.943970+0000","last_peered":"2024-08-17T17:24:12.943970+0000","last_clean":"2024-08-17T17:24:12.943970+0000","last_became_active":"2024-08-17T17:24:11.581141+0000","last_became_peered":"2024-08-17T17:24:11.581141+0000","last_unstale":"2024-08-17T17:24:12.943970+0000","last_undegraded":"2024-08-17T17:24:12.943970+0000","last_fullsized":"2024-08-17T17:24:12.943970+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.943804+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.943804+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:02:02.291520+0000","scrub_duration":0.0091260200000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":986,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:16.608170+0000","last_change":"2024-08-17T17:21:06.433720+0000","last_active":"2024-08-17T17:24:16.608170+0000","last_peered":"2024-08-17T17:24:16.608170+0000","last_clean":"2024-08-17T17:24:16.608170+0000","last_became_active":"2024-08-17T17:21:06.433292+0000","last_became_peered":"2024-08-17T17:21:06.433292+0000","last_unstale":"2024-08-17T17:24:16.608170+0000","last_undegraded":"2024-08-17T17:24:16.608170+0000","last_fullsized":"2024-08-17T17:24:16.608170+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:32.246594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1176,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.586034+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:24:12.586034+0000","last_peered":"2024-08-17T17:24:12.586034+0000","last_clean":"2024-08-17T17:24:12.586034+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:24:12.586034+0000","last_undegraded":"2024-08-17T17:24:12.586034+0000","last_fullsized":"2024-08-17T17:24:12.586034+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":768,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.583872+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:24:12.583872+0000","last_peered":"2024-08-17T17:24:12.583872+0000","last_clean":"2024-08-17T17:24:12.583872+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:24:12.583872+0000","last_undegraded":"2024-08-17T17:24:12.583872+0000","last_fullsized":"2024-08-17T17:24:12.583872+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"1220'9328","reported_seq":9390,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:20.324173+0000","last_change":"2024-08-17T17:22:10.944050+0000","last_active":"2024-08-17T17:24:20.324173+0000","last_peered":"2024-08-17T17:24:20.324173+0000","last_clean":"2024-08-17T17:24:20.324173+0000","last_became_active":"2024-08-17T17:22:10.943712+0000","last_became_peered":"2024-08-17T17:22:10.943712+0000","last_unstale":"2024-08-17T17:24:20.324173+0000","last_undegraded":"2024-08-17T17:24:20.324173+0000","last_fullsized":"2024-08-17T17:24:20.324173+0000","mapping_epoch":1195,"log_start":"1202'2980","ondisk_log_start":"1202'2980","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":6348,"log_dups_size":0,"ondisk_log_size":6348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:13.015652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":442630144,"num_objects":6754,"num_object_clones":0,"num_object_copies":20262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9328,"num_write_kb":514624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1505,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:13.211079+0000","last_change":"2024-08-17T17:24:13.211079+0000","last_active":"2024-08-17T17:24:13.211079+0000","last_peered":"2024-08-17T17:24:13.211079+0000","last_clean":"2024-08-17T17:24:13.211079+0000","last_became_active":"2024-08-17T17:24:11.572039+0000","last_became_peered":"2024-08-17T17:24:11.572039+0000","last_unstale":"2024-08-17T17:24:13.211079+0000","last_undegraded":"2024-08-17T17:24:13.211079+0000","last_fullsized":"2024-08-17T17:24:13.211079+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.210956+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.210956+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:18:46.803867+0000","scrub_duration":0.0033494829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1557,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.640000+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:24:19.640000+0000","last_peered":"2024-08-17T17:24:19.640000+0000","last_clean":"2024-08-17T17:24:19.640000+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:24:19.640000+0000","last_undegraded":"2024-08-17T17:24:19.640000+0000","last_fullsized":"2024-08-17T17:24:19.640000+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":817,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.583317+0000","last_change":"2024-08-17T17:21:06.452395+0000","last_active":"2024-08-17T17:24:12.583317+0000","last_peered":"2024-08-17T17:24:12.583317+0000","last_clean":"2024-08-17T17:24:12.583317+0000","last_became_active":"2024-08-17T17:21:06.451867+0000","last_became_peered":"2024-08-17T17:21:06.451867+0000","last_unstale":"2024-08-17T17:24:12.583317+0000","last_undegraded":"2024-08-17T17:24:12.583317+0000","last_fullsized":"2024-08-17T17:24:12.583317+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:50.597092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1547,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.641466+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:24:19.641466+0000","last_peered":"2024-08-17T17:24:19.641466+0000","last_clean":"2024-08-17T17:24:19.641466+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:24:19.641466+0000","last_undegraded":"2024-08-17T17:24:19.641466+0000","last_fullsized":"2024-08-17T17:24:19.641466+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"1217'9025","reported_seq":9057,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:17.555203+0000","last_change":"2024-08-17T17:22:10.859993+0000","last_active":"2024-08-17T17:24:17.555203+0000","last_peered":"2024-08-17T17:24:17.555203+0000","last_clean":"2024-08-17T17:24:17.555203+0000","last_became_active":"2024-08-17T17:22:10.859296+0000","last_became_peered":"2024-08-17T17:22:10.859296+0000","last_unstale":"2024-08-17T17:24:17.555203+0000","last_undegraded":"2024-08-17T17:24:17.555203+0000","last_fullsized":"2024-08-17T17:24:17.555203+0000","mapping_epoch":1195,"log_start":"1200'1271","ondisk_log_start":"1200'1271","created":1195,"last_epoch_clean":1196,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:22:09.836447+0000","last_clean_scrub_stamp":"2024-08-17T17:22:09.836447+0000","objects_scrubbed":0,"log_size":7754,"log_dups_size":0,"ondisk_log_size":7754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:42.095858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":440270872,"num_objects":6719,"num_object_clones":0,"num_object_copies":20157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6719,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9025,"num_write_kb":503745,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1579,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.642643+0000","last_change":"2024-08-17T17:24:12.033217+0000","last_active":"2024-08-17T17:24:19.642643+0000","last_peered":"2024-08-17T17:24:19.642643+0000","last_clean":"2024-08-17T17:24:19.642643+0000","last_became_active":"2024-08-17T17:24:11.572877+0000","last_became_peered":"2024-08-17T17:24:11.572877+0000","last_unstale":"2024-08-17T17:24:19.642643+0000","last_undegraded":"2024-08-17T17:24:19.642643+0000","last_fullsized":"2024-08-17T17:24:19.642643+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.033101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.033101+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:34:14.662387+0000","scrub_duration":20,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":186,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.588378+0000","last_change":"2024-08-17T17:21:06.519965+0000","last_active":"2024-08-17T17:24:12.588378+0000","last_peered":"2024-08-17T17:24:12.588378+0000","last_clean":"2024-08-17T17:24:12.588378+0000","last_became_active":"2024-08-17T17:21:06.459952+0000","last_became_peered":"2024-08-17T17:21:06.459952+0000","last_unstale":"2024-08-17T17:24:12.588378+0000","last_undegraded":"2024-08-17T17:24:12.588378+0000","last_fullsized":"2024-08-17T17:24:12.588378+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:27.828098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1172,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.583225+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:24:12.583225+0000","last_peered":"2024-08-17T17:24:12.583225+0000","last_clean":"2024-08-17T17:24:12.583225+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:24:12.583225+0000","last_undegraded":"2024-08-17T17:24:12.583225+0000","last_fullsized":"2024-08-17T17:24:12.583225+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1148,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.587522+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:24:12.587522+0000","last_peered":"2024-08-17T17:24:12.587522+0000","last_clean":"2024-08-17T17:24:12.587522+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:24:12.587522+0000","last_undegraded":"2024-08-17T17:24:12.587522+0000","last_fullsized":"2024-08-17T17:24:12.587522+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1180,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.587900+0000","last_change":"2024-08-17T17:21:06.458000+0000","last_active":"2024-08-17T17:24:12.587900+0000","last_peered":"2024-08-17T17:24:12.587900+0000","last_clean":"2024-08-17T17:24:12.587900+0000","last_became_active":"2024-08-17T17:21:06.457411+0000","last_became_peered":"2024-08-17T17:21:06.457411+0000","last_unstale":"2024-08-17T17:24:12.587900+0000","last_undegraded":"2024-08-17T17:24:12.587900+0000","last_fullsized":"2024-08-17T17:24:12.587900+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:52.740639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1171,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.587833+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:24:12.587833+0000","last_peered":"2024-08-17T17:24:12.587833+0000","last_clean":"2024-08-17T17:24:12.587833+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:24:12.587833+0000","last_undegraded":"2024-08-17T17:24:12.587833+0000","last_fullsized":"2024-08-17T17:24:12.587833+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":840,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.581893+0000","last_change":"2024-08-17T17:21:06.454587+0000","last_active":"2024-08-17T17:24:12.581893+0000","last_peered":"2024-08-17T17:24:12.581893+0000","last_clean":"2024-08-17T17:24:12.581893+0000","last_became_active":"2024-08-17T17:21:06.454229+0000","last_became_peered":"2024-08-17T17:21:06.454229+0000","last_unstale":"2024-08-17T17:24:12.581893+0000","last_undegraded":"2024-08-17T17:24:12.581893+0000","last_fullsized":"2024-08-17T17:24:12.581893+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:38.340263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1146,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.585910+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:24:12.585910+0000","last_peered":"2024-08-17T17:24:12.585910+0000","last_clean":"2024-08-17T17:24:12.585910+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:24:12.585910+0000","last_undegraded":"2024-08-17T17:24:12.585910+0000","last_fullsized":"2024-08-17T17:24:12.585910+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"0'0","reported_seq":1,"reported_epoch":1217,"state":"unknown","last_fresh":"2024-08-17T17:24:16.613364+0000","last_change":"2024-08-17T17:24:16.668364+0000","last_active":"0.000000","last_peered":"0.000000","last_clean":"0.000000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:24:16.613364+0000","last_undegraded":"2024-08-17T17:24:16.613364+0000","last_fullsized":"2024-08-17T17:24:16.613364+0000","mapping_epoch":1217,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1176,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.583948+0000","last_change":"2024-08-17T17:21:06.442796+0000","last_active":"2024-08-17T17:24:12.583948+0000","last_peered":"2024-08-17T17:24:12.583948+0000","last_clean":"2024-08-17T17:24:12.583948+0000","last_became_active":"2024-08-17T17:21:06.442466+0000","last_became_peered":"2024-08-17T17:21:06.442466+0000","last_unstale":"2024-08-17T17:24:12.583948+0000","last_undegraded":"2024-08-17T17:24:12.583948+0000","last_fullsized":"2024-08-17T17:24:12.583948+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:36.366620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1503,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.637340+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:24:19.637340+0000","last_peered":"2024-08-17T17:24:19.637340+0000","last_clean":"2024-08-17T17:24:19.637340+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:24:19.637340+0000","last_undegraded":"2024-08-17T17:24:19.637340+0000","last_fullsized":"2024-08-17T17:24:19.637340+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1012,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.584266+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:24:12.584266+0000","last_peered":"2024-08-17T17:24:12.584266+0000","last_clean":"2024-08-17T17:24:12.584266+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:24:12.584266+0000","last_undegraded":"2024-08-17T17:24:12.584266+0000","last_fullsized":"2024-08-17T17:24:12.584266+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1536,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.641372+0000","last_change":"2024-08-17T17:21:06.468788+0000","last_active":"2024-08-17T17:24:19.641372+0000","last_peered":"2024-08-17T17:24:19.641372+0000","last_clean":"2024-08-17T17:24:19.641372+0000","last_became_active":"2024-08-17T17:21:06.467141+0000","last_became_peered":"2024-08-17T17:21:06.467141+0000","last_unstale":"2024-08-17T17:24:19.641372+0000","last_undegraded":"2024-08-17T17:24:19.641372+0000","last_fullsized":"2024-08-17T17:24:19.641372+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:51.301359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1178,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.583220+0000","last_change":"2024-08-17T17:21:06.435998+0000","last_active":"2024-08-17T17:24:12.583220+0000","last_peered":"2024-08-17T17:24:12.583220+0000","last_clean":"2024-08-17T17:24:12.583220+0000","last_became_active":"2024-08-17T17:21:06.435579+0000","last_became_peered":"2024-08-17T17:21:06.435579+0000","last_unstale":"2024-08-17T17:24:12.583220+0000","last_undegraded":"2024-08-17T17:24:12.583220+0000","last_fullsized":"2024-08-17T17:24:12.583220+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:01.743074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1565,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.641552+0000","last_change":"2024-08-17T17:21:06.473627+0000","last_active":"2024-08-17T17:24:19.641552+0000","last_peered":"2024-08-17T17:24:19.641552+0000","last_clean":"2024-08-17T17:24:19.641552+0000","last_became_active":"2024-08-17T17:21:06.473062+0000","last_became_peered":"2024-08-17T17:21:06.473062+0000","last_unstale":"2024-08-17T17:24:19.641552+0000","last_undegraded":"2024-08-17T17:24:19.641552+0000","last_fullsized":"2024-08-17T17:24:19.641552+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:21.413163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":973,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.585981+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:24:12.585981+0000","last_peered":"2024-08-17T17:24:12.585981+0000","last_clean":"2024-08-17T17:24:12.585981+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:24:12.585981+0000","last_undegraded":"2024-08-17T17:24:12.585981+0000","last_fullsized":"2024-08-17T17:24:12.585981+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1056,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.637680+0000","last_change":"2024-08-17T17:21:06.471265+0000","last_active":"2024-08-17T17:24:19.637680+0000","last_peered":"2024-08-17T17:24:19.637680+0000","last_clean":"2024-08-17T17:24:19.637680+0000","last_became_active":"2024-08-17T17:21:06.470943+0000","last_became_peered":"2024-08-17T17:21:06.470943+0000","last_unstale":"2024-08-17T17:24:19.637680+0000","last_undegraded":"2024-08-17T17:24:19.637680+0000","last_fullsized":"2024-08-17T17:24:19.637680+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:41.070024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1542,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.638163+0000","last_change":"2024-08-17T17:22:48.377351+0000","last_active":"2024-08-17T17:24:19.638163+0000","last_peered":"2024-08-17T17:24:19.638163+0000","last_clean":"2024-08-17T17:24:19.638163+0000","last_became_active":"2024-08-17T17:22:48.377059+0000","last_became_peered":"2024-08-17T17:22:48.377059+0000","last_unstale":"2024-08-17T17:24:19.638163+0000","last_undegraded":"2024-08-17T17:24:19.638163+0000","last_fullsized":"2024-08-17T17:24:19.638163+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:15.250902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1196,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.589706+0000","last_change":"2024-08-17T17:21:06.459139+0000","last_active":"2024-08-17T17:24:12.589706+0000","last_peered":"2024-08-17T17:24:12.589706+0000","last_clean":"2024-08-17T17:24:12.589706+0000","last_became_active":"2024-08-17T17:21:06.457850+0000","last_became_peered":"2024-08-17T17:21:06.457850+0000","last_unstale":"2024-08-17T17:24:12.589706+0000","last_undegraded":"2024-08-17T17:24:12.589706+0000","last_fullsized":"2024-08-17T17:24:12.589706+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:13.888576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1577,"reported_epoch":1217,"state":"active+clean","last_fresh":"2024-08-17T17:24:16.608271+0000","last_change":"2024-08-17T17:21:06.446321+0000","last_active":"2024-08-17T17:24:16.608271+0000","last_peered":"2024-08-17T17:24:16.608271+0000","last_clean":"2024-08-17T17:24:16.608271+0000","last_became_active":"2024-08-17T17:21:06.446019+0000","last_became_peered":"2024-08-17T17:21:06.446019+0000","last_unstale":"2024-08-17T17:24:16.608271+0000","last_undegraded":"2024-08-17T17:24:16.608271+0000","last_fullsized":"2024-08-17T17:24:16.608271+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:26.750070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":165,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.587718+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:24:12.587718+0000","last_peered":"2024-08-17T17:24:12.587718+0000","last_clean":"2024-08-17T17:24:12.587718+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:24:12.587718+0000","last_undegraded":"2024-08-17T17:24:12.587718+0000","last_fullsized":"2024-08-17T17:24:12.587718+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1194,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.587823+0000","last_change":"2024-08-17T17:21:06.465804+0000","last_active":"2024-08-17T17:24:12.587823+0000","last_peered":"2024-08-17T17:24:12.587823+0000","last_clean":"2024-08-17T17:24:12.587823+0000","last_became_active":"2024-08-17T17:21:06.465461+0000","last_became_peered":"2024-08-17T17:21:06.465461+0000","last_unstale":"2024-08-17T17:24:12.587823+0000","last_undegraded":"2024-08-17T17:24:12.587823+0000","last_fullsized":"2024-08-17T17:24:12.587823+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:06.443667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1186,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.584379+0000","last_change":"2024-08-17T17:21:06.481634+0000","last_active":"2024-08-17T17:24:12.584379+0000","last_peered":"2024-08-17T17:24:12.584379+0000","last_clean":"2024-08-17T17:24:12.584379+0000","last_became_active":"2024-08-17T17:21:06.469081+0000","last_became_peered":"2024-08-17T17:21:06.469081+0000","last_unstale":"2024-08-17T17:24:12.584379+0000","last_undegraded":"2024-08-17T17:24:12.584379+0000","last_fullsized":"2024-08-17T17:24:12.584379+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:53:26.954738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1194,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.589617+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:24:12.589617+0000","last_peered":"2024-08-17T17:24:12.589617+0000","last_clean":"2024-08-17T17:24:12.589617+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:24:12.589617+0000","last_undegraded":"2024-08-17T17:24:12.589617+0000","last_fullsized":"2024-08-17T17:24:12.589617+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1145,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.584411+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:24:12.584411+0000","last_peered":"2024-08-17T17:24:12.584411+0000","last_clean":"2024-08-17T17:24:12.584411+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:24:12.584411+0000","last_undegraded":"2024-08-17T17:24:12.584411+0000","last_fullsized":"2024-08-17T17:24:12.584411+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1541,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.637844+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:24:19.637844+0000","last_peered":"2024-08-17T17:24:19.637844+0000","last_clean":"2024-08-17T17:24:19.637844+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:24:19.637844+0000","last_undegraded":"2024-08-17T17:24:19.637844+0000","last_fullsized":"2024-08-17T17:24:19.637844+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1517,"reported_epoch":1220,"state":"active+clean","last_fresh":"2024-08-17T17:24:19.639251+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:24:19.639251+0000","last_peered":"2024-08-17T17:24:19.639251+0000","last_clean":"2024-08-17T17:24:19.639251+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:24:19.639251+0000","last_undegraded":"2024-08-17T17:24:19.639251+0000","last_fullsized":"2024-08-17T17:24:19.639251+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1179,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.587511+0000","last_change":"2024-08-17T17:21:06.483469+0000","last_active":"2024-08-17T17:24:12.587511+0000","last_peered":"2024-08-17T17:24:12.587511+0000","last_clean":"2024-08-17T17:24:12.587511+0000","last_became_active":"2024-08-17T17:21:06.483162+0000","last_became_peered":"2024-08-17T17:21:06.483162+0000","last_unstale":"2024-08-17T17:24:12.587511+0000","last_undegraded":"2024-08-17T17:24:12.587511+0000","last_fullsized":"2024-08-17T17:24:12.587511+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:24.898353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":188,"reported_epoch":1216,"state":"active+clean","last_fresh":"2024-08-17T17:24:12.584169+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:24:12.584169+0000","last_peered":"2024-08-17T17:24:12.584169+0000","last_clean":"2024-08-17T17:24:12.584169+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:24:12.584169+0000","last_undegraded":"2024-08-17T17:24:12.584169+0000","last_fullsized":"2024-08-17T17:24:12.584169+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":7062290456,"num_objects":107763,"num_object_clones":0,"num_object_copies":323289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":107763,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":145213,"num_write_kb":8095169,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21272670208,"data_stored":21272657992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117035,"ondisk_log_size":117035,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":626148,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":24,"num_read_kb":50,"num_write":14,"num_write_kb":147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":435020,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2384,"ondisk_log_size":2384,"up":63,"acting":62,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201674,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2598904,"kb_used_data":2202800,"kb_used_omap":1,"kb_used_meta":396094,"kb_avail":91146248,"statfs":{"total":95995035648,"available":93333757952,"internally_reserved":0,"allocated":2255667200,"data_stored":2253235760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":405600715},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463311,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2527868,"kb_used_data":2194096,"kb_used_omap":1,"kb_used_meta":333758,"kb_avail":91217284,"statfs":{"total":95995035648,"available":93406498816,"internally_reserved":0,"allocated":2246754304,"data_stored":2244331343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":341768649},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626953,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3105308,"kb_used_data":2541456,"kb_used_omap":42,"kb_used_meta":563797,"kb_avail":90639844,"statfs":{"total":95995035648,"available":92815200256,"internally_reserved":0,"allocated":2602450944,"data_stored":2600017032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":577328188},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785430,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3410876,"kb_used_data":3009200,"kb_used_omap":48,"kb_used_meta":401487,"kb_avail":90334276,"statfs":{"total":95995035648,"available":92502298624,"internally_reserved":0,"allocated":3081420800,"data_stored":3079001921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":411123374},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724947,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3901944,"kb_used_data":3513968,"kb_used_omap":1,"kb_used_meta":387902,"kb_avail":89843208,"statfs":{"total":95995035648,"available":91999444992,"internally_reserved":0,"allocated":3598303232,"data_stored":3595876665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":397212102},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986584,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2587392,"kb_used_data":2187060,"kb_used_omap":1,"kb_used_meta":400190,"kb_avail":91157760,"statfs":{"total":95995035648,"available":93345546240,"internally_reserved":0,"allocated":2239549440,"data_stored":2237122440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":409795014},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706191,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3048360,"kb_used_data":2623456,"kb_used_omap":35,"kb_used_meta":424860,"kb_avail":90696792,"statfs":{"total":95995035648,"available":92873515008,"internally_reserved":0,"allocated":2686418944,"data_stored":2684000088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":435056695},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,2,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831964,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3166560,"kb_used_data":2646808,"kb_used_omap":65,"kb_used_meta":519678,"kb_avail":90578592,"statfs":{"total":95995035648,"available":92752478208,"internally_reserved":0,"allocated":2710331392,"data_stored":2707920148,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":532150459},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2698969088,"data_stored":2698969088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3062366208,"data_stored":3062366208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2672693248,"data_stored":2672689176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2584543232,"data_stored":2584543232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2230255616,"data_stored":2230255616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3563720704,"data_stored":3563716632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2230583296,"data_stored":2230583296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2229538816,"data_stored":2229534744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:24:22.829 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T17:24:22.829 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set .mgr pg_num 31 2024-08-17T17:24:22.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:22 smithi162 ceph-mon[104668]: 1.13 scrub starts 2024-08-17T17:24:22.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:22 smithi162 ceph-mon[104668]: from='client.35418 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:22.857 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:24:22.911 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:22.946 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:22 smithi063 ceph-mon[114756]: 1.13 scrub starts 2024-08-17T17:24:22.946 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:22 smithi063 ceph-mon[114756]: from='client.35418 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:22.946 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:22 smithi063 ceph-mon[109843]: 1.13 scrub starts 2024-08-17T17:24:22.947 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:22 smithi063 ceph-mon[109843]: from='client.35418 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:24:22.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:22.953 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:22.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:23.054 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:23.083 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:23.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:23.185 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:23.224 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:23.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:23.326 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:23.359 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:23.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:23.461 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:23.490 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:23.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:23.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:24:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:24:23] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.51.0" 2024-08-17T17:24:23.592 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:23.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.620 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:23.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:23.721 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:23.750 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:23.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:23.852 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:23.880 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:23.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:23.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:23.982 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:24.019 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:24.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:23 smithi063 ceph-mon[114756]: pgmap v1595: 109 pgs: 4 peering, 1 remapped+peering, 3 unknown, 101 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 585 op/s; 107 KiB/s, 0 objects/s recovering 2024-08-17T17:24:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:23 smithi063 ceph-mon[109843]: pgmap v1595: 109 pgs: 4 peering, 1 remapped+peering, 3 unknown, 101 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 585 op/s; 107 KiB/s, 0 objects/s recovering 2024-08-17T17:24:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:23 smithi162 ceph-mon[104668]: pgmap v1595: 109 pgs: 4 peering, 1 remapped+peering, 3 unknown, 101 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 585 op/s; 107 KiB/s, 0 objects/s recovering 2024-08-17T17:24:24.121 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:24.177 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:24.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:24.279 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:24.289 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:24.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.344 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:24.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:24.445 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:24.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.494 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:24.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:24.595 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:24.628 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:24.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:24.730 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:24.757 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:24:24.757 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:24:24.766 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:24.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:24.868 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:24.895 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:24.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:24.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:24.996 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:25.009 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:24 smithi063 ceph-mon[109843]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:24:25.009 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:24:25.009 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:24 smithi063 ceph-mon[114756]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:24:25.010 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:24:25.037 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:25.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:25.049 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:24:25.049 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:24:25.083 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:24:25.083 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:24:25.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:24 smithi162 ceph-mon[104668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:24:25.132 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:24:25.132 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:24:25.140 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:25.165 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:25.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:25.172 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:24:25.173 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:24:25.210 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:24:25.211 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:24:25.212 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:24:25.252 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:24:25.253 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:24:25.267 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:25.288 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:24:25.288 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:24:25.317 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:25.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:25.419 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:25.454 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:24:25.465 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:25.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:25.567 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:25.617 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:25.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:25.718 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:25.758 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:25.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:25.860 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:25.896 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:25.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:25.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:25.999 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:26.028 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:26.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:25 smithi063 ceph-mon[114756]: pgmap v1596: 109 pgs: 2 peering, 107 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 27 KiB/s rd, 79 KiB/s wr, 814 op/s; 114 KiB/s, 0 objects/s recovering 2024-08-17T17:24:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:25 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3634235602' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:25 smithi063 ceph-mon[109843]: pgmap v1596: 109 pgs: 2 peering, 107 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 27 KiB/s rd, 79 KiB/s wr, 814 op/s; 114 KiB/s, 0 objects/s recovering 2024-08-17T17:24:26.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:25 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3634235602' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:26.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:25 smithi162 ceph-mon[104668]: pgmap v1596: 109 pgs: 2 peering, 107 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 27 KiB/s rd, 79 KiB/s wr, 814 op/s; 114 KiB/s, 0 objects/s recovering 2024-08-17T17:24:26.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:25 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3634235602' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:26.130 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:26.151 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:26.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:26.254 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:26.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.284 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:26.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:26.385 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:26.418 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:26.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:26.520 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:26.558 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:26.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:26.660 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:26.693 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:26.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:26.795 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:26.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:26.856 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:26.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:26.957 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:27.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.007 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:27.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:27.108 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:27.138 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:27.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:27.240 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:27.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.280 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:27.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:27.381 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:27.408 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:27.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:27.510 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:27.538 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:27.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:27.640 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:27.668 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:27.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:27.770 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:27.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.804 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:27.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:27.890 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:27 smithi063 ceph-mon[109843]: 1.13 scrub starts 2024-08-17T17:24:27.890 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:27 smithi063 ceph-mon[109843]: pgmap v1597: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 15 KiB/s rd, 43 KiB/s wr, 822 op/s; 76 KiB/s, 0 objects/s recovering 2024-08-17T17:24:27.895 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:27 smithi063 ceph-mon[114756]: 1.13 scrub starts 2024-08-17T17:24:27.895 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:27 smithi063 ceph-mon[114756]: pgmap v1597: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 15 KiB/s rd, 43 KiB/s wr, 822 op/s; 76 KiB/s, 0 objects/s recovering 2024-08-17T17:24:27.906 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:27.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:27.937 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:27.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:28.038 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:28.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.074 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:28.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:28.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:27 smithi162 ceph-mon[104668]: 1.13 scrub starts 2024-08-17T17:24:28.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:27 smithi162 ceph-mon[104668]: pgmap v1597: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 15 KiB/s rd, 43 KiB/s wr, 822 op/s; 76 KiB/s, 0 objects/s recovering 2024-08-17T17:24:28.176 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:28.182 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:28.190 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:28.204 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:28.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:28.306 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:28.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.352 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:28.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:28.452 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:28.489 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:28.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:28.591 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:28.635 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:28.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:28.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:28.737 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:28.778 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:29.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:29.174 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:29.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.220 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:29.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:29.321 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:29.348 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:29.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:29.449 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:29.486 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:29.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:29.588 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:29.629 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:29.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:29.731 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:29.762 INFO:teuthology.orchestra.run.smithi063.stderr:set pool 1 pg_num to 31 2024-08-17T17:24:29.768 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:29.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:29.870 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:29.926 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:29.927 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:24:29.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:29.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:30.018 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:29 smithi063 ceph-mon[109843]: pgmap v1598: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 13 KiB/s rd, 38 KiB/s wr, 1.09k op/s; 67 KiB/s, 0 objects/s recovering 2024-08-17T17:24:30.019 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:29 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3248020432' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-08-17T17:24:30.019 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:29 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-08-17T17:24:30.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:29 smithi063 ceph-mon[114756]: pgmap v1598: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 13 KiB/s rd, 38 KiB/s wr, 1.09k op/s; 67 KiB/s, 0 objects/s recovering 2024-08-17T17:24:30.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:29 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3248020432' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-08-17T17:24:30.019 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:29 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-08-17T17:24:30.030 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:30.055 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:30.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:29 smithi162 ceph-mon[104668]: pgmap v1598: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 13 KiB/s rd, 38 KiB/s wr, 1.09k op/s; 67 KiB/s, 0 objects/s recovering 2024-08-17T17:24:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:29 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3248020432' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-08-17T17:24:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:29 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-08-17T17:24:30.157 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:30.184 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:30.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:30.286 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:30.317 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:30.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:30.418 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:30.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.450 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:30.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:30.552 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:30.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.592 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:30.592 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:30.692 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:30.724 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:30.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:30.827 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:30.871 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:30.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:30.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:30.973 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:31.010 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:31.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:31.065 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:30 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-08-17T17:24:31.066 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:30 smithi063 ceph-mon[109843]: osdmap e1222: 8 total, 8 up, 8 in 2024-08-17T17:24:31.066 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:30 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-08-17T17:24:31.066 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:30 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-08-17T17:24:31.066 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:30 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-08-17T17:24:31.066 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:30 smithi063 ceph-mon[114756]: osdmap e1222: 8 total, 8 up, 8 in 2024-08-17T17:24:31.066 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:30 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-08-17T17:24:31.066 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:30 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-08-17T17:24:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:30 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-08-17T17:24:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:30 smithi162 ceph-mon[104668]: osdmap e1222: 8 total, 8 up, 8 in 2024-08-17T17:24:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:30 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-08-17T17:24:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:30 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-08-17T17:24:31.113 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:31.145 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:31.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:31.247 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:31.280 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:31.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:31.382 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:31.417 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:31.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:31.519 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:31.550 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:31.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:31.652 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:31.690 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:31.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:31.792 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:31.822 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:31.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:31.923 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:31.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:31.956 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:31.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:32.057 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:32.092 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:32.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:31 smithi162 ceph-mon[104668]: 1.13 scrub starts 2024-08-17T17:24:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:31 smithi162 ceph-mon[104668]: pgmap v1600: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 6.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 12 KiB/s rd, 34 KiB/s wr, 823 op/s; 27 KiB/s, 0 objects/s recovering 2024-08-17T17:24:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-08-17T17:24:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:31 smithi162 ceph-mon[104668]: osdmap e1223: 8 total, 8 up, 8 in 2024-08-17T17:24:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:24:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:32.196 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:32.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:31 smithi063 ceph-mon[114756]: 1.13 scrub starts 2024-08-17T17:24:32.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:31 smithi063 ceph-mon[114756]: pgmap v1600: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 6.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 12 KiB/s rd, 34 KiB/s wr, 823 op/s; 27 KiB/s, 0 objects/s recovering 2024-08-17T17:24:32.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-08-17T17:24:32.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:31 smithi063 ceph-mon[114756]: osdmap e1223: 8 total, 8 up, 8 in 2024-08-17T17:24:32.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:24:32.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:32.216 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:31 smithi063 ceph-mon[109843]: 1.13 scrub starts 2024-08-17T17:24:32.216 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:31 smithi063 ceph-mon[109843]: pgmap v1600: 109 pgs: 1 active+clean+scrubbing, 108 active+clean; 6.1 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 12 KiB/s rd, 34 KiB/s wr, 823 op/s; 27 KiB/s, 0 objects/s recovering 2024-08-17T17:24:32.216 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-08-17T17:24:32.216 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:31 smithi063 ceph-mon[109843]: osdmap e1223: 8 total, 8 up, 8 in 2024-08-17T17:24:32.216 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:24:32.216 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:32.230 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:32.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:32.332 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:32.359 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:32.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:32.461 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:32.493 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:32.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:32.595 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:32.630 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:32.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:32.732 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:32.770 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:32.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:32.872 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:32.906 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:32.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:32.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:33.008 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:33.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.034 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:33.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:32 smithi162 ceph-mon[104668]: 1.13 scrub starts 2024-08-17T17:24:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:32 smithi162 ceph-mon[104668]: osdmap e1224: 8 total, 8 up, 8 in 2024-08-17T17:24:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:32 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/143648574' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:32 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:33.132 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:24:33.133 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:32 smithi063 ceph-mon[114756]: 1.13 scrub starts 2024-08-17T17:24:33.133 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:32 smithi063 ceph-mon[114756]: osdmap e1224: 8 total, 8 up, 8 in 2024-08-17T17:24:33.133 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:32 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/143648574' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:33.133 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:32 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:33.133 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:32 smithi063 ceph-mon[109843]: 1.13 scrub starts 2024-08-17T17:24:33.133 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:32 smithi063 ceph-mon[109843]: osdmap e1224: 8 total, 8 up, 8 in 2024-08-17T17:24:33.133 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:32 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/143648574' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:33.133 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:32 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:24:33.136 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:33.173 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:33.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:33.275 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:33.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.307 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:33.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:33.408 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:33.430 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:24:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:24:33] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-08-17T17:24:33.438 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:33.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:33.540 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:33.578 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:33.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:33.680 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:33.716 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:33.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:33.818 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:33.850 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:33.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:33.952 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:33.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:33.991 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:33.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:34.091 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:33 smithi063 ceph-mon[114756]: 1.c scrub starts 2024-08-17T17:24:34.091 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:33 smithi063 ceph-mon[114756]: pgmap v1603: 119 pgs: 8 unknown, 1 active+clean+scrubbing, 110 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2024-08-17T17:24:34.091 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:33 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:34.091 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:33 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:34.092 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:33 smithi063 ceph-mon[114756]: osdmap e1225: 8 total, 8 up, 8 in 2024-08-17T17:24:34.092 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:33 smithi063 ceph-mon[109843]: 1.c scrub starts 2024-08-17T17:24:34.092 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:33 smithi063 ceph-mon[109843]: pgmap v1603: 119 pgs: 8 unknown, 1 active+clean+scrubbing, 110 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2024-08-17T17:24:34.092 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:33 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:34.092 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:33 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:34.092 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:33 smithi063 ceph-mon[109843]: osdmap e1225: 8 total, 8 up, 8 in 2024-08-17T17:24:34.092 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:34.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:33 smithi162 ceph-mon[104668]: 1.c scrub starts 2024-08-17T17:24:34.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:33 smithi162 ceph-mon[104668]: pgmap v1603: 119 pgs: 8 unknown, 1 active+clean+scrubbing, 110 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 947 op/s 2024-08-17T17:24:34.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:33 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:34.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:33 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:24:34.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:33 smithi162 ceph-mon[104668]: osdmap e1225: 8 total, 8 up, 8 in 2024-08-17T17:24:34.131 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:34.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:34.235 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:34.256 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:34.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:34.358 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:34.395 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:34.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:34.497 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:34.527 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:34.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:34.629 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:34.663 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:34.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:34.765 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:34.804 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:34.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:34.806 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:34.906 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:34.950 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:34.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:34.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:35.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:34 smithi063 ceph-mon[114756]: 1.1c scrub starts 2024-08-17T17:24:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:34 smithi063 ceph-mon[109843]: 1.1c scrub starts 2024-08-17T17:24:35.054 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:35.086 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:35.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:34 smithi162 ceph-mon[104668]: 1.1c scrub starts 2024-08-17T17:24:35.189 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:35.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.240 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:35.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:35.341 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:35.377 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:35.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:35.479 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:35.488 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:24:35.488 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:24:35.505 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:35.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:35.607 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:35.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.651 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:35.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:35.752 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:35.764 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:24:35.764 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:24:35.765 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:35.765 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45218.0:161789 8.9 e1225/1195 8:9d8f4725:::benchmark_data_smithi063_230025_object35299:head v 1225'10174, mlcod=1225'10172)", 2024-08-17T17:24:35.765 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:24:35.755033+0000", 2024-08-17T17:24:35.765 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0052585109999999996, 2024-08-17T17:24:35.765 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0052930399999999997, 2024-08-17T17:24:35.765 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:35.765 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:35.765 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:24:35.765 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:35.765 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:35.765 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:35.765 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:35.755033+0000", 2024-08-17T17:24:35.765 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:35.766 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:35.766 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:35.766 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:35.766 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:35.755033+0000", 2024-08-17T17:24:35.766 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:35.766 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:35.766 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:35.766 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:35.766 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:35.755035+0000", 2024-08-17T17:24:35.766 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.265e-06 2024-08-17T17:24:35.766 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:35.767 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:35.767 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:35.767 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:35.755048+0000", 2024-08-17T17:24:35.767 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3857e-05 2024-08-17T17:24:35.767 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:35.767 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:35.767 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:35.767 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:35.755071+0000", 2024-08-17T17:24:35.767 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2444999999999999e-05 2024-08-17T17:24:35.767 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:35.767 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:35.768 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:35.768 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:35.755080+0000", 2024-08-17T17:24:35.768 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.1779999999999992e-06 2024-08-17T17:24:35.768 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:35.768 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:35.768 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:35.768 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:35.755301+0000", 2024-08-17T17:24:35.768 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000220903 2024-08-17T17:24:35.768 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:35.768 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:35.769 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:35.769 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:35.755361+0000", 2024-08-17T17:24:35.769 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9806000000000003e-05 2024-08-17T17:24:35.769 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:35.769 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:35.770 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:35.770 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:35.770 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-08-17T17:24:35.770 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 1 2024-08-17T17:24:35.770 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:24:35.806 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:35.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:35.889 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:24:35.908 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:35.934 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:35.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:35.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:35 smithi063 ceph-mon[109843]: pgmap v1605: 119 pgs: 1 unknown, 1 active+clean+scrubbing, 117 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:24:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:35 smithi063 ceph-mon[114756]: pgmap v1605: 119 pgs: 1 unknown, 1 active+clean+scrubbing, 117 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:24:36.036 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:36.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.081 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:36.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:35 smithi162 ceph-mon[104668]: pgmap v1605: 119 pgs: 1 unknown, 1 active+clean+scrubbing, 117 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:24:36.182 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:36.219 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:36.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:36.321 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:36.352 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:36.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:36.454 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:36.497 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:36.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:36.599 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:36.633 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:36.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:36.735 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:36.774 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:36.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:36.876 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:36.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:36.930 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:36.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:37.031 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:37.074 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:37.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:37.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:36 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-08-17T17:24:37.099 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:36 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-08-17T17:24:37.176 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:37.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.204 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:37.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:37.225 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:36 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-08-17T17:24:37.225 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:36 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-08-17T17:24:37.225 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:36 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-08-17T17:24:37.225 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:36 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-08-17T17:24:37.305 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:37.341 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:37.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:37.443 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:37.475 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:37.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:37.577 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:37.616 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:37.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:37.718 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:37.751 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:37.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:37.853 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:37.897 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:37.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:37.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:37.999 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:37 smithi063 ceph-mon[114756]: pgmap v1606: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-08-17T17:24:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:37 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-08-17T17:24:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:37 smithi063 ceph-mon[114756]: osdmap e1226: 8 total, 8 up, 8 in 2024-08-17T17:24:38.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:37 smithi063 ceph-mon[109843]: pgmap v1606: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-08-17T17:24:38.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:37 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-08-17T17:24:38.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:37 smithi063 ceph-mon[109843]: osdmap e1226: 8 total, 8 up, 8 in 2024-08-17T17:24:38.032 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:38.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:38.134 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:38.185 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:38.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:38.229 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:38.260 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:24:38.275 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:38.287 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:38.337 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:38.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:38.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:37 smithi162 ceph-mon[104668]: pgmap v1606: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-08-17T17:24:38.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:37 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-08-17T17:24:38.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:37 smithi162 ceph-mon[104668]: osdmap e1226: 8 total, 8 up, 8 in 2024-08-17T17:24:38.439 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:38.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.466 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:38.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:38.567 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:38.593 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:38.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:38.695 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:38.721 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:38.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:38.823 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:38.851 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:38.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:38.892 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:24:38.893 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:24:38.929 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:24:38.930 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:24:38.953 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:38.971 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:24:38.971 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:24:38.984 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:38.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:38.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:39.012 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:24:39.012 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:24:39.062 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:24:39.063 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:24:39.063 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:38 smithi063 ceph-mon[114756]: 1.13 scrub starts 2024-08-17T17:24:39.063 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:38 smithi063 ceph-mon[114756]: osdmap e1227: 8 total, 8 up, 8 in 2024-08-17T17:24:39.063 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:38 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1155624353' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:39.063 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:38 smithi063 ceph-mon[114756]: osdmap e1228: 8 total, 8 up, 8 in 2024-08-17T17:24:39.063 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:38 smithi063 ceph-mon[109843]: 1.13 scrub starts 2024-08-17T17:24:39.064 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:38 smithi063 ceph-mon[109843]: osdmap e1227: 8 total, 8 up, 8 in 2024-08-17T17:24:39.064 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:38 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1155624353' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:39.064 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:38 smithi063 ceph-mon[109843]: osdmap e1228: 8 total, 8 up, 8 in 2024-08-17T17:24:39.085 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:39.106 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:24:39.107 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:24:39.113 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:39.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:39.140 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:24:39.140 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:24:39.180 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:24:39.180 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:24:39.215 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:39.244 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:39.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:39.346 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:39.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:38 smithi162 ceph-mon[104668]: 1.13 scrub starts 2024-08-17T17:24:39.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:38 smithi162 ceph-mon[104668]: osdmap e1227: 8 total, 8 up, 8 in 2024-08-17T17:24:39.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:38 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1155624353' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:39.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:38 smithi162 ceph-mon[104668]: osdmap e1228: 8 total, 8 up, 8 in 2024-08-17T17:24:39.385 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:39.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:39.486 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:39.525 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:39.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:39.627 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:39.670 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:39.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:39.772 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:39.805 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:39.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:39.907 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:39.934 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:39.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:39.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:40.026 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:39 smithi063 ceph-mon[114756]: 1.c scrub starts 2024-08-17T17:24:40.026 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:39 smithi063 ceph-mon[114756]: pgmap v1609: 119 pgs: 2 peering, 4 unknown, 113 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-08-17T17:24:40.026 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:39 smithi063 ceph-mon[114756]: osdmap e1229: 8 total, 8 up, 8 in 2024-08-17T17:24:40.026 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:39 smithi063 ceph-mon[109843]: 1.c scrub starts 2024-08-17T17:24:40.026 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:39 smithi063 ceph-mon[109843]: pgmap v1609: 119 pgs: 2 peering, 4 unknown, 113 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-08-17T17:24:40.026 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:39 smithi063 ceph-mon[109843]: osdmap e1229: 8 total, 8 up, 8 in 2024-08-17T17:24:40.036 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:40.065 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:40.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:40.167 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:40.192 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:40.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:40.293 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:40.320 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:40.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:40.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:39 smithi162 ceph-mon[104668]: 1.c scrub starts 2024-08-17T17:24:40.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:39 smithi162 ceph-mon[104668]: pgmap v1609: 119 pgs: 2 peering, 4 unknown, 113 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-08-17T17:24:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:39 smithi162 ceph-mon[104668]: osdmap e1229: 8 total, 8 up, 8 in 2024-08-17T17:24:40.422 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:40.451 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:40.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:40.553 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:40.596 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:40.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:40.698 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:40.733 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:40.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:40.835 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:40.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:40.876 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:40.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:40.977 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:41.022 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:41.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:41.124 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:41.166 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:41.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:41.268 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:41.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:41.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:41.400 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:41.437 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:41.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:41.539 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:41.573 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:41.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:41.675 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:41.703 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:41.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:41.806 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:41.838 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:41.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:41.939 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:41.973 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:41.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:41.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:42.064 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:24:42.076 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:42.167 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:42.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:42.269 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:42.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:41 smithi063 ceph-mon[109843]: pgmap v1612: 119 pgs: 2 peering, 1 remapped+peering, 4 unknown, 112 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-08-17T17:24:42.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:41 smithi063 ceph-mon[109843]: osdmap e1230: 8 total, 8 up, 8 in 2024-08-17T17:24:42.282 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:41 smithi063 ceph-mon[109843]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:24:42.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:41 smithi063 ceph-mon[114756]: pgmap v1612: 119 pgs: 2 peering, 1 remapped+peering, 4 unknown, 112 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-08-17T17:24:42.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:41 smithi063 ceph-mon[114756]: osdmap e1230: 8 total, 8 up, 8 in 2024-08-17T17:24:42.283 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:41 smithi063 ceph-mon[114756]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:24:42.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.324 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:42.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:42.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:41 smithi162 ceph-mon[104668]: pgmap v1612: 119 pgs: 2 peering, 1 remapped+peering, 4 unknown, 112 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 835 op/s 2024-08-17T17:24:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:41 smithi162 ceph-mon[104668]: osdmap e1230: 8 total, 8 up, 8 in 2024-08-17T17:24:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:41 smithi162 ceph-mon[104668]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:24:42.425 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:42.454 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:42.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:42.498 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:42.556 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:42.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.603 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:42.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:42.704 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:42.733 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:42.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.734 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:42.835 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:42.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:42.885 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:42.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:42.986 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:43.027 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:43.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:42 smithi063 ceph-mon[109843]: 1.1e scrub starts 2024-08-17T17:24:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:42 smithi063 ceph-mon[114756]: 1.1e scrub starts 2024-08-17T17:24:43.129 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:43.150 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:43.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:43.252 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:43.281 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:43.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:43.326 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:24:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:24:43] "GET /metrics HTTP/1.1" 200 36430 "" "Prometheus/2.51.0" 2024-08-17T17:24:43.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:42 smithi162 ceph-mon[104668]: 1.1e scrub starts 2024-08-17T17:24:43.383 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:43.416 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:43.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:43.518 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:43.539 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:43.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:43.642 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:43.664 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:43.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:43.765 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:43.802 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:43.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:43.904 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:43.931 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:43.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:43.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:44.034 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:44.072 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:44.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:44.174 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:44.202 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:44.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:44.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:43 smithi063 ceph-mon[114756]: 1.1d scrub starts 2024-08-17T17:24:44.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:43 smithi063 ceph-mon[114756]: pgmap v1614: 119 pgs: 6 peering, 3 unknown, 110 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 21 KiB/s rd, 61 KiB/s wr, 646 op/s; 50 KiB/s, 0 objects/s recovering 2024-08-17T17:24:44.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:43 smithi063 ceph-mon[114756]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:24:44.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:43 smithi063 ceph-mon[109843]: 1.1d scrub starts 2024-08-17T17:24:44.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:43 smithi063 ceph-mon[109843]: pgmap v1614: 119 pgs: 6 peering, 3 unknown, 110 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 21 KiB/s rd, 61 KiB/s wr, 646 op/s; 50 KiB/s, 0 objects/s recovering 2024-08-17T17:24:44.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:43 smithi063 ceph-mon[109843]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:24:44.304 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:44.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:43 smithi162 ceph-mon[104668]: 1.1d scrub starts 2024-08-17T17:24:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:43 smithi162 ceph-mon[104668]: pgmap v1614: 119 pgs: 6 peering, 3 unknown, 110 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 21 KiB/s rd, 61 KiB/s wr, 646 op/s; 50 KiB/s, 0 objects/s recovering 2024-08-17T17:24:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:43 smithi162 ceph-mon[104668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:24:44.346 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:44.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:44.448 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:44.475 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:44.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:44.577 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:44.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.604 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:44.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:44.671 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:44.706 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:44.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.750 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:44.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:44.851 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:44.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:44.879 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:44.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:44.892 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:24:44.923 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:44 smithi063 ceph-mon[109843]: 1.16 scrub starts 2024-08-17T17:24:44.923 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:44 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/547676106' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:24:44.924 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:44 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:24:44.924 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:44 smithi063 ceph-mon[114756]: 1.16 scrub starts 2024-08-17T17:24:44.924 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:44 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/547676106' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:24:44.924 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:44 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:24:44.980 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.015 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:45.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:45.117 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:45.144 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:45.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:45.246 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:45.277 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:45.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:45.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:44 smithi162 ceph-mon[104668]: 1.16 scrub starts 2024-08-17T17:24:45.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:44 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/547676106' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:24:45.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:44 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:24:45.379 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:45.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.418 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:45.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:45.519 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:45.557 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:45.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:45.659 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:45.700 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:45.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:45.801 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:45.849 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:45.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:45.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:45.914 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:24:45.950 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:46.000 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:46.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:46.102 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:46.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.133 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:46.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:46.147 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:45 smithi063 ceph-mon[109843]: 1.19 scrub starts 2024-08-17T17:24:46.147 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:45 smithi063 ceph-mon[109843]: pgmap v1615: 119 pgs: 4 peering, 115 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 22 KiB/s rd, 64 KiB/s wr, 865 op/s; 93 KiB/s, 0 objects/s recovering 2024-08-17T17:24:46.147 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:45 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:24:46.148 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:45 smithi063 ceph-mon[109843]: osdmap e1231: 8 total, 8 up, 8 in 2024-08-17T17:24:46.148 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:45 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/379736055' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:46.148 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:45 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:46.148 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:45 smithi063 ceph-mon[114756]: 1.19 scrub starts 2024-08-17T17:24:46.148 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:45 smithi063 ceph-mon[114756]: pgmap v1615: 119 pgs: 4 peering, 115 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 22 KiB/s rd, 64 KiB/s wr, 865 op/s; 93 KiB/s, 0 objects/s recovering 2024-08-17T17:24:46.148 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:45 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:24:46.148 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:45 smithi063 ceph-mon[114756]: osdmap e1231: 8 total, 8 up, 8 in 2024-08-17T17:24:46.148 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:45 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/379736055' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:46.148 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:45 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:46.234 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:46.271 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:46.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:46.316 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:24:46.316 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:24:46.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:45 smithi162 ceph-mon[104668]: 1.19 scrub starts 2024-08-17T17:24:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:45 smithi162 ceph-mon[104668]: pgmap v1615: 119 pgs: 4 peering, 115 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 22 KiB/s rd, 64 KiB/s wr, 865 op/s; 93 KiB/s, 0 objects/s recovering 2024-08-17T17:24:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:45 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:24:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:45 smithi162 ceph-mon[104668]: osdmap e1231: 8 total, 8 up, 8 in 2024-08-17T17:24:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:45 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/379736055' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:46.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:45 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:24:46.373 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:46.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.412 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:46.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:46.513 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:46.546 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:46.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:46.648 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:46.695 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:46.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:46.796 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:46.833 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:46.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:46.934 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:46.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:46.967 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:46.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:47.068 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:47.111 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:47.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:47.153 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:46 smithi162 ceph-mon[104668]: 1.13 scrub starts 2024-08-17T17:24:47.153 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:46 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:47.154 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:46 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:24:47.154 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:46 smithi162 ceph-mon[104668]: osdmap e1232: 8 total, 8 up, 8 in 2024-08-17T17:24:47.154 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:24:47.154 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:47.213 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:47.256 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:47.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:47.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:46 smithi063 ceph-mon[109843]: 1.13 scrub starts 2024-08-17T17:24:47.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:46 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:46 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:24:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:46 smithi063 ceph-mon[109843]: osdmap e1232: 8 total, 8 up, 8 in 2024-08-17T17:24:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:24:47.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:47.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:46 smithi063 ceph-mon[114756]: 1.13 scrub starts 2024-08-17T17:24:47.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:46 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:24:47.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:46 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:24:47.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:46 smithi063 ceph-mon[114756]: osdmap e1232: 8 total, 8 up, 8 in 2024-08-17T17:24:47.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:24:47.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:24:47.358 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:47.389 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:47.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:47.491 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:47.524 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:47.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:47.626 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:47.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.663 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:47.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:47.764 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:47.804 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:47.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:47.906 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:47.937 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:47.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:47.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:48.039 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:48.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.074 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:48.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:48.144 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:47 smithi162 ceph-mon[104668]: 1.c scrub starts 2024-08-17T17:24:48.144 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:47 smithi162 ceph-mon[104668]: 1.c scrub ok 2024-08-17T17:24:48.145 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:47 smithi162 ceph-mon[104668]: pgmap v1618: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 22 KiB/s rd, 64 KiB/s wr, 1.10k op/s; 93 KiB/s, 0 objects/s recovering 2024-08-17T17:24:48.145 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:47 smithi162 ceph-mon[104668]: 1.1e scrub starts 2024-08-17T17:24:48.145 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:47 smithi162 ceph-mon[104668]: 1.1e scrub ok 2024-08-17T17:24:48.175 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:48.215 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:48.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:48.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:47 smithi063 ceph-mon[109843]: 1.c scrub starts 2024-08-17T17:24:48.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:47 smithi063 ceph-mon[109843]: 1.c scrub ok 2024-08-17T17:24:48.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:47 smithi063 ceph-mon[109843]: pgmap v1618: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 22 KiB/s rd, 64 KiB/s wr, 1.10k op/s; 93 KiB/s, 0 objects/s recovering 2024-08-17T17:24:48.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:47 smithi063 ceph-mon[109843]: 1.1e scrub starts 2024-08-17T17:24:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:47 smithi063 ceph-mon[109843]: 1.1e scrub ok 2024-08-17T17:24:48.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:47 smithi063 ceph-mon[114756]: 1.c scrub starts 2024-08-17T17:24:48.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:47 smithi063 ceph-mon[114756]: 1.c scrub ok 2024-08-17T17:24:48.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:47 smithi063 ceph-mon[114756]: pgmap v1618: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 22 KiB/s rd, 64 KiB/s wr, 1.10k op/s; 93 KiB/s, 0 objects/s recovering 2024-08-17T17:24:48.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:47 smithi063 ceph-mon[114756]: 1.1e scrub starts 2024-08-17T17:24:48.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:47 smithi063 ceph-mon[114756]: 1.1e scrub ok 2024-08-17T17:24:48.316 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:48.337 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:48.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:48.439 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:48.472 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:48.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:48.574 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:48.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.607 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:48.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:48.708 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:48.743 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:48.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:48.845 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:48.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:48.893 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:48.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:48.994 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:49.033 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:49.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:49.134 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:49.188 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:49.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:49.290 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:49.325 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:49.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:49.427 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:49.461 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:49.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:49.563 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:49.602 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:49.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:49.704 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:49.744 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:49.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:49.846 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:49.878 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:49.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:49.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:49.980 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:50.007 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:50.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:50.056 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:24:50.109 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:50.138 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:50.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:50.170 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:49 smithi162 ceph-mon[104668]: 1.1d scrub starts 2024-08-17T17:24:50.170 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:49 smithi162 ceph-mon[104668]: 1.1d scrub ok 2024-08-17T17:24:50.171 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:49 smithi162 ceph-mon[104668]: 1.f scrub starts 2024-08-17T17:24:50.171 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:49 smithi162 ceph-mon[104668]: 1.f scrub ok 2024-08-17T17:24:50.171 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:49 smithi162 ceph-mon[104668]: pgmap v1619: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 10 KiB/s rd, 30 KiB/s wr, 1.10k op/s; 56 KiB/s, 0 objects/s recovering 2024-08-17T17:24:50.171 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:49 smithi162 ceph-mon[104668]: 1.10 scrub starts 2024-08-17T17:24:50.171 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:49 smithi162 ceph-mon[104668]: 1.10 scrub ok 2024-08-17T17:24:50.171 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:49 smithi162 ceph-mon[104668]: 1.16 scrub starts 2024-08-17T17:24:50.171 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:49 smithi162 ceph-mon[104668]: 1.16 scrub ok 2024-08-17T17:24:50.171 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:49 smithi162 ceph-mon[104668]: 1.5 scrub starts 2024-08-17T17:24:50.171 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:49 smithi162 ceph-mon[104668]: 1.5 scrub ok 2024-08-17T17:24:50.240 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[109843]: 1.1d scrub starts 2024-08-17T17:24:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[109843]: 1.1d scrub ok 2024-08-17T17:24:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[109843]: 1.f scrub starts 2024-08-17T17:24:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[109843]: 1.f scrub ok 2024-08-17T17:24:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[109843]: pgmap v1619: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 10 KiB/s rd, 30 KiB/s wr, 1.10k op/s; 56 KiB/s, 0 objects/s recovering 2024-08-17T17:24:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[109843]: 1.10 scrub starts 2024-08-17T17:24:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[109843]: 1.10 scrub ok 2024-08-17T17:24:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[109843]: 1.16 scrub starts 2024-08-17T17:24:50.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[109843]: 1.16 scrub ok 2024-08-17T17:24:50.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[109843]: 1.5 scrub starts 2024-08-17T17:24:50.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[109843]: 1.5 scrub ok 2024-08-17T17:24:50.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[114756]: 1.1d scrub starts 2024-08-17T17:24:50.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[114756]: 1.1d scrub ok 2024-08-17T17:24:50.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[114756]: 1.f scrub starts 2024-08-17T17:24:50.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[114756]: 1.f scrub ok 2024-08-17T17:24:50.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[114756]: pgmap v1619: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 10 KiB/s rd, 30 KiB/s wr, 1.10k op/s; 56 KiB/s, 0 objects/s recovering 2024-08-17T17:24:50.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[114756]: 1.10 scrub starts 2024-08-17T17:24:50.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[114756]: 1.10 scrub ok 2024-08-17T17:24:50.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[114756]: 1.16 scrub starts 2024-08-17T17:24:50.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[114756]: 1.16 scrub ok 2024-08-17T17:24:50.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[114756]: 1.5 scrub starts 2024-08-17T17:24:50.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:49 smithi063 ceph-mon[114756]: 1.5 scrub ok 2024-08-17T17:24:50.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.284 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:50.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:50.385 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:50.432 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:50.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:50.533 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:50.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.563 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:50.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:50.664 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:50.686 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:50.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:50.788 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:50.816 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:50.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:50.823 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:24:50.917 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:50.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:50.940 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:50.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:51.041 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:51.074 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:51.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:51.096 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:50 smithi063 ceph-mon[114756]: 1.11 scrub starts 2024-08-17T17:24:51.097 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:50 smithi063 ceph-mon[114756]: 1.11 scrub ok 2024-08-17T17:24:51.097 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:50 smithi063 ceph-mon[114756]: 1.19 scrub starts 2024-08-17T17:24:51.097 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:50 smithi063 ceph-mon[114756]: 1.19 scrub ok 2024-08-17T17:24:51.097 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:50 smithi063 ceph-mon[114756]: 1.d scrub starts 2024-08-17T17:24:51.097 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:50 smithi063 ceph-mon[114756]: 1.d scrub ok 2024-08-17T17:24:51.097 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:50 smithi063 ceph-mon[109843]: 1.11 scrub starts 2024-08-17T17:24:51.097 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:50 smithi063 ceph-mon[109843]: 1.11 scrub ok 2024-08-17T17:24:51.097 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:50 smithi063 ceph-mon[109843]: 1.19 scrub starts 2024-08-17T17:24:51.097 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:50 smithi063 ceph-mon[109843]: 1.19 scrub ok 2024-08-17T17:24:51.098 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:50 smithi063 ceph-mon[109843]: 1.d scrub starts 2024-08-17T17:24:51.098 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:50 smithi063 ceph-mon[109843]: 1.d scrub ok 2024-08-17T17:24:51.143 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:24:51.176 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:51.199 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:51.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:51.273 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:50 smithi162 ceph-mon[104668]: 1.11 scrub starts 2024-08-17T17:24:51.273 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:50 smithi162 ceph-mon[104668]: 1.11 scrub ok 2024-08-17T17:24:51.274 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:50 smithi162 ceph-mon[104668]: 1.19 scrub starts 2024-08-17T17:24:51.274 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:50 smithi162 ceph-mon[104668]: 1.19 scrub ok 2024-08-17T17:24:51.274 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:50 smithi162 ceph-mon[104668]: 1.d scrub starts 2024-08-17T17:24:51.274 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:50 smithi162 ceph-mon[104668]: 1.d scrub ok 2024-08-17T17:24:51.301 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:51.377 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:51.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:51.480 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:51.494 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:24:51.494 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:24:51.521 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:51.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:51.531 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:24:51.531 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:24:51.566 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:24:51.566 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:24:51.603 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:24:51.604 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:24:51.623 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:51.636 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:24:51.637 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:24:51.650 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:51.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:51.669 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:24:51.670 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:24:51.707 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:24:51.707 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:24:51.752 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:24:51.752 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:24:51.752 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:51.795 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:51.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:51.897 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:51.946 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:51.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:51.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:51 smithi063 ceph-mon[114756]: pgmap v1620: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 4.7 KiB/s rd, 14 KiB/s wr, 860 op/s; 42 KiB/s, 0 objects/s recovering 2024-08-17T17:24:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:51 smithi063 ceph-mon[114756]: 1.12 scrub starts 2024-08-17T17:24:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:51 smithi063 ceph-mon[114756]: 1.12 scrub ok 2024-08-17T17:24:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:51 smithi063 ceph-mon[114756]: 1.13 scrub starts 2024-08-17T17:24:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:51 smithi063 ceph-mon[114756]: 1.13 scrub ok 2024-08-17T17:24:52.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:51 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1277995029' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:52.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:51 smithi063 ceph-mon[109843]: pgmap v1620: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 4.7 KiB/s rd, 14 KiB/s wr, 860 op/s; 42 KiB/s, 0 objects/s recovering 2024-08-17T17:24:52.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:51 smithi063 ceph-mon[109843]: 1.12 scrub starts 2024-08-17T17:24:52.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:51 smithi063 ceph-mon[109843]: 1.12 scrub ok 2024-08-17T17:24:52.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:51 smithi063 ceph-mon[109843]: 1.13 scrub starts 2024-08-17T17:24:52.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:51 smithi063 ceph-mon[109843]: 1.13 scrub ok 2024-08-17T17:24:52.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:51 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1277995029' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:52.048 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:52.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.072 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:52.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:52.173 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:52.212 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:52.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:52.273 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:51 smithi162 ceph-mon[104668]: pgmap v1620: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 4.7 KiB/s rd, 14 KiB/s wr, 860 op/s; 42 KiB/s, 0 objects/s recovering 2024-08-17T17:24:52.273 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:51 smithi162 ceph-mon[104668]: 1.12 scrub starts 2024-08-17T17:24:52.273 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:51 smithi162 ceph-mon[104668]: 1.12 scrub ok 2024-08-17T17:24:52.273 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:51 smithi162 ceph-mon[104668]: 1.13 scrub starts 2024-08-17T17:24:52.273 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:51 smithi162 ceph-mon[104668]: 1.13 scrub ok 2024-08-17T17:24:52.273 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:51 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1277995029' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:24:52.314 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:52.349 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:52.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:52.451 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:52.493 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:52.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:52.594 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:52.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.634 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:52.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:52.734 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:52.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.770 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:52.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:52.871 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:52.902 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:52.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:52.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:53.004 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:53.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.039 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:53.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:53.140 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:53.183 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:53.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:53.278 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:52 smithi063 ceph-mon[114756]: 1.e scrub starts 2024-08-17T17:24:53.278 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:52 smithi063 ceph-mon[114756]: 1.e scrub ok 2024-08-17T17:24:53.278 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:52 smithi063 ceph-mon[114756]: pgmap v1621: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2024-08-17T17:24:53.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:52 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:24:53.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:52 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:24:53.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:52 smithi063 ceph-mon[109843]: 1.e scrub starts 2024-08-17T17:24:53.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:52 smithi063 ceph-mon[109843]: 1.e scrub ok 2024-08-17T17:24:53.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:52 smithi063 ceph-mon[109843]: pgmap v1621: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2024-08-17T17:24:53.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:52 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:24:53.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:52 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:24:53.284 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:53.329 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:53.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:53.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:52 smithi162 ceph-mon[104668]: 1.e scrub starts 2024-08-17T17:24:53.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:52 smithi162 ceph-mon[104668]: 1.e scrub ok 2024-08-17T17:24:53.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:52 smithi162 ceph-mon[104668]: pgmap v1621: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2024-08-17T17:24:53.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:52 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:24:53.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:52 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:24:53.431 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:53.473 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:53.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.474 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:53.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:24:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:24:53] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.51.0" 2024-08-17T17:24:53.575 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:53.613 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:53.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:53.715 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:53.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.740 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:53.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:53.841 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:53.875 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:53.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:53.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:53.977 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:54.008 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:54.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:54.011 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:54.022 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:54.110 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:54.147 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:54.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:54.199 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:53 smithi063 ceph-mon[109843]: 1.a scrub starts 2024-08-17T17:24:54.199 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:53 smithi063 ceph-mon[109843]: 1.a scrub ok 2024-08-17T17:24:54.199 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:53 smithi063 ceph-mon[109843]: 1.1a scrub starts 2024-08-17T17:24:54.199 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:53 smithi063 ceph-mon[109843]: 1.1a scrub ok 2024-08-17T17:24:54.199 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:53 smithi063 ceph-mon[114756]: 1.a scrub starts 2024-08-17T17:24:54.199 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:53 smithi063 ceph-mon[114756]: 1.a scrub ok 2024-08-17T17:24:54.199 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:53 smithi063 ceph-mon[114756]: 1.1a scrub starts 2024-08-17T17:24:54.200 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:53 smithi063 ceph-mon[114756]: 1.1a scrub ok 2024-08-17T17:24:54.248 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:54.278 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:54.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:54.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:53 smithi162 ceph-mon[104668]: 1.a scrub starts 2024-08-17T17:24:54.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:53 smithi162 ceph-mon[104668]: 1.a scrub ok 2024-08-17T17:24:54.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:53 smithi162 ceph-mon[104668]: 1.1a scrub starts 2024-08-17T17:24:54.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:53 smithi162 ceph-mon[104668]: 1.1a scrub ok 2024-08-17T17:24:54.380 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:54.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.414 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:54.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:54.515 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:54.534 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:24:54.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.563 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:54.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:54.664 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:54.688 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:54.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:54.790 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:54.815 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:54.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:54.917 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:54.938 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:54.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:54.939 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:54 smithi063 ceph-mon[114756]: 1.6 scrub starts 2024-08-17T17:24:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:54 smithi063 ceph-mon[114756]: 1.6 scrub ok 2024-08-17T17:24:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:54 smithi063 ceph-mon[114756]: pgmap v1622: 119 pgs: 119 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 956 op/s 2024-08-17T17:24:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:54 smithi063 ceph-mon[114756]: 1.9 scrub starts 2024-08-17T17:24:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:54 smithi063 ceph-mon[114756]: 1.9 scrub ok 2024-08-17T17:24:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:54 smithi063 ceph-mon[109843]: 1.6 scrub starts 2024-08-17T17:24:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:54 smithi063 ceph-mon[109843]: 1.6 scrub ok 2024-08-17T17:24:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:54 smithi063 ceph-mon[109843]: pgmap v1622: 119 pgs: 119 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 956 op/s 2024-08-17T17:24:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:54 smithi063 ceph-mon[109843]: 1.9 scrub starts 2024-08-17T17:24:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:54 smithi063 ceph-mon[109843]: 1.9 scrub ok 2024-08-17T17:24:55.040 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:55.062 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:55.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:55.164 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:55.193 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:55.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:55.294 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:55.337 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:55.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:55.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:54 smithi162 ceph-mon[104668]: 1.6 scrub starts 2024-08-17T17:24:55.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:54 smithi162 ceph-mon[104668]: 1.6 scrub ok 2024-08-17T17:24:55.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:54 smithi162 ceph-mon[104668]: pgmap v1622: 119 pgs: 119 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 956 op/s 2024-08-17T17:24:55.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:54 smithi162 ceph-mon[104668]: 1.9 scrub starts 2024-08-17T17:24:55.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:54 smithi162 ceph-mon[104668]: 1.9 scrub ok 2024-08-17T17:24:55.439 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:55.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.478 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:55.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:55.579 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:55.612 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:55.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:55.653 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:24:55.715 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:55.740 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:55.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:55.842 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:55.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:55.898 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:55.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:55.999 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:56.012 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:56 smithi063 ceph-mon[109843]: 1.7 scrub starts 2024-08-17T17:24:56.012 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:56 smithi063 ceph-mon[109843]: 1.7 scrub ok 2024-08-17T17:24:56.042 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:56.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:56.144 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:56.182 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:56.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:56.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:56 smithi063 ceph-mon[114756]: 1.7 scrub starts 2024-08-17T17:24:56.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:56 smithi063 ceph-mon[114756]: 1.7 scrub ok 2024-08-17T17:24:56.285 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:56.327 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:56.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:56.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:56 smithi162 ceph-mon[104668]: 1.7 scrub starts 2024-08-17T17:24:56.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:56 smithi162 ceph-mon[104668]: 1.7 scrub ok 2024-08-17T17:24:56.429 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:56.468 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:56.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:56.570 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:56.623 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:56.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:56.725 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:56.749 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:56.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:56.851 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:56.896 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:56.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:56.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:56.998 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:57.049 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:57.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:57.151 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:57.191 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:57.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:57.274 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:57 smithi063 ceph-mon[109843]: pgmap v1623: 119 pgs: 119 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-08-17T17:24:57.274 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:57 smithi063 ceph-mon[114756]: pgmap v1623: 119 pgs: 119 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-08-17T17:24:57.294 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:57.333 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:24:57.333 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:24:57.334 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:24:57.334 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:24:57.334 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.334 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "MOSDPGPush(7.5 1149/1144 [PushOp(7:aa8dfb9c:::benchmark_data_smithi063_196002_object9253:head, version: 1132'606, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:aa8dfb9c:::benchmark_data_smithi063_196002_object9253:head@1132'606, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:24:57.334 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:53.656740+0000", 2024-08-17T17:24:57.334 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 363.67210955199999, 2024-08-17T17:24:57.334 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.152369581, 2024-08-17T17:24:57.334 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.334 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.334 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:24:57.334 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.334 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.335 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.335 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656740+0000", 2024-08-17T17:24:57.335 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.335 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.335 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.335 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.335 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656740+0000", 2024-08-17T17:24:57.335 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.335 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.335 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.335 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.335 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656741+0000", 2024-08-17T17:24:57.335 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0079999999999999e-06 2024-08-17T17:24:57.336 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.336 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.336 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.336 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656947+0000", 2024-08-17T17:24:57.336 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00020619200000000001 2024-08-17T17:24:57.336 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.336 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.336 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.336 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656975+0000", 2024-08-17T17:24:57.336 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.7746000000000002e-05 2024-08-17T17:24:57.336 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.336 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.336 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.336 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656998+0000", 2024-08-17T17:24:57.337 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3006e-05 2024-08-17T17:24:57.337 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.337 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.337 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.337 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.803287+0000", 2024-08-17T17:24:57.337 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14628891699999999 2024-08-17T17:24:57.337 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.337 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.337 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.337 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.803301+0000", 2024-08-17T17:24:57.337 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3920000000000001e-05 2024-08-17T17:24:57.337 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.337 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.338 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.338 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.809110+0000", 2024-08-17T17:24:57.338 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0058087920000000001 2024-08-17T17:24:57.338 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.338 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.338 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.338 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.338 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.338 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88367 7.d e1150/1144 7:bbf555f3:::benchmark_data_smithi063_196002_object88366:head v 1150'5489, mlcod=0'0)", 2024-08-17T17:24:57.338 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.645168+0000", 2024-08-17T17:24:57.338 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 356.68368205000002, 2024-08-17T17:24:57.339 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.210459275, 2024-08-17T17:24:57.339 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.339 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.339 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.339 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.339 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.339 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.339 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.645168+0000", 2024-08-17T17:24:57.339 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.339 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.339 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.339 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.340 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.645168+0000", 2024-08-17T17:24:57.340 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.340 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.340 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.340 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.340 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.645169+0000", 2024-08-17T17:24:57.340 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1969999999999999e-06 2024-08-17T17:24:57.340 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.340 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.340 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.340 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646316+0000", 2024-08-17T17:24:57.340 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0011472229999999999 2024-08-17T17:24:57.341 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.341 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.341 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.341 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646328+0000", 2024-08-17T17:24:57.341 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1657999999999999e-05 2024-08-17T17:24:57.341 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.341 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.341 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.341 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646333+0000", 2024-08-17T17:24:57.341 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.733e-06 2024-08-17T17:24:57.341 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.342 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.342 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.342 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646392+0000", 2024-08-17T17:24:57.342 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8805999999999999e-05 2024-08-17T17:24:57.342 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.342 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.342 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.342 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646423+0000", 2024-08-17T17:24:57.342 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1179000000000001e-05 2024-08-17T17:24:57.342 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.342 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.342 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.342 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.855586+0000", 2024-08-17T17:24:57.343 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20916273699999999 2024-08-17T17:24:57.343 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.343 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.343 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.343 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.855627+0000", 2024-08-17T17:24:57.343 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.0741999999999997e-05 2024-08-17T17:24:57.343 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.343 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.343 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.343 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.343 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.343 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88364 7.6 e1150/1144 7:671e019e:::benchmark_data_smithi063_196002_object88363:head v 1150'5532, mlcod=0'0)", 2024-08-17T17:24:57.343 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.646551+0000", 2024-08-17T17:24:57.344 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 356.68229837199999, 2024-08-17T17:24:57.344 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.22748069200000001, 2024-08-17T17:24:57.344 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.344 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.344 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:24:57.344 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.344 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.344 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.344 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646551+0000", 2024-08-17T17:24:57.344 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.344 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.344 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.344 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.345 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646551+0000", 2024-08-17T17:24:57.345 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.345 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.345 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.345 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.345 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646552+0000", 2024-08-17T17:24:57.345 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.29e-07 2024-08-17T17:24:57.345 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.345 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.345 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.345 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646636+0000", 2024-08-17T17:24:57.345 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4449000000000004e-05 2024-08-17T17:24:57.345 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.345 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.346 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.346 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646645+0000", 2024-08-17T17:24:57.346 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7099999999999996e-06 2024-08-17T17:24:57.346 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.346 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.346 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.346 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646650+0000", 2024-08-17T17:24:57.346 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1780000000000002e-06 2024-08-17T17:24:57.346 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.346 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.346 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.346 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646720+0000", 2024-08-17T17:24:57.346 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9924000000000003e-05 2024-08-17T17:24:57.347 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.347 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.347 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.347 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646752+0000", 2024-08-17T17:24:57.347 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1919999999999999e-05 2024-08-17T17:24:57.347 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.347 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.347 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.347 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.874032+0000", 2024-08-17T17:24:57.347 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.227280282 2024-08-17T17:24:57.347 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.347 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.347 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.348 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.348 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.348 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "MOSDPGPush(7.5 1150/1144 [PushOp(7:a927a2c1:::benchmark_data_smithi063_196002_object12871:head, version: 1132'815, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:a927a2c1:::benchmark_data_smithi063_196002_object12871:head@1132'815, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:24:57.348 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.650543+0000", 2024-08-17T17:24:57.348 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 356.67830715600002, 2024-08-17T17:24:57.348 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20508452899999999, 2024-08-17T17:24:57.348 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.348 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.348 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:24:57.348 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.348 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.348 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.348 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650543+0000", 2024-08-17T17:24:57.348 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.349 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.349 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.349 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.349 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650543+0000", 2024-08-17T17:24:57.349 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.349 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.349 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.349 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.349 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650544+0000", 2024-08-17T17:24:57.349 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3659999999999999e-06 2024-08-17T17:24:57.349 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.349 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.349 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.350 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650585+0000", 2024-08-17T17:24:57.350 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.1542999999999998e-05 2024-08-17T17:24:57.350 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.350 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.350 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.350 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650612+0000", 2024-08-17T17:24:57.350 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6242999999999999e-05 2024-08-17T17:24:57.350 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.350 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.350 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.350 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650629+0000", 2024-08-17T17:24:57.350 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.747e-05 2024-08-17T17:24:57.351 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.351 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.351 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.351 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.855605+0000", 2024-08-17T17:24:57.351 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20497623200000001 2024-08-17T17:24:57.351 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.351 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.351 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.351 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.855627+0000", 2024-08-17T17:24:57.351 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1675000000000002e-05 2024-08-17T17:24:57.351 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.351 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.351 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.352 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.352 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.352 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88374 7.d e1150/1144 7:b7ece4c7:::benchmark_data_smithi063_196002_object88373:head v 1150'5490, mlcod=0'0)", 2024-08-17T17:24:57.352 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.651831+0000", 2024-08-17T17:24:57.352 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 356.67701871600002, 2024-08-17T17:24:57.352 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20382080899999999, 2024-08-17T17:24:57.352 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.352 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.352 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.352 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.352 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.352 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.352 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651831+0000", 2024-08-17T17:24:57.352 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.353 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.353 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.353 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.353 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651831+0000", 2024-08-17T17:24:57.353 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.353 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.353 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.353 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.353 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651832+0000", 2024-08-17T17:24:57.353 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.06e-06 2024-08-17T17:24:57.353 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.353 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.353 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.354 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651872+0000", 2024-08-17T17:24:57.354 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9501e-05 2024-08-17T17:24:57.354 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.354 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.354 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.354 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651889+0000", 2024-08-17T17:24:57.354 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7815000000000001e-05 2024-08-17T17:24:57.354 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.354 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.354 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.354 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651898+0000", 2024-08-17T17:24:57.354 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.5040000000000002e-06 2024-08-17T17:24:57.354 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.355 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.355 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.355 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.653677+0000", 2024-08-17T17:24:57.355 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001778899 2024-08-17T17:24:57.355 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.355 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.355 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.355 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.653704+0000", 2024-08-17T17:24:57.355 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.7161999999999998e-05 2024-08-17T17:24:57.355 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.355 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.355 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.355 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.855630+0000", 2024-08-17T17:24:57.356 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20192644600000001 2024-08-17T17:24:57.356 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.356 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.356 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.356 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.855652+0000", 2024-08-17T17:24:57.356 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1421999999999999e-05 2024-08-17T17:24:57.356 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.356 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.356 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.356 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.356 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.356 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88377 7.e e1150/1144 7:72a20e0d:::benchmark_data_smithi063_196002_object88376:head v 1150'5626, mlcod=0'0)", 2024-08-17T17:24:57.356 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.682989+0000", 2024-08-17T17:24:57.357 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 356.64586071999997, 2024-08-17T17:24:57.357 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.25261817399999997, 2024-08-17T17:24:57.357 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.357 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.357 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.357 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.357 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.357 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.357 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.682989+0000", 2024-08-17T17:24:57.357 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.357 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.357 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.357 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.357 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.682989+0000", 2024-08-17T17:24:57.358 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.358 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.358 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.358 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.358 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.682989+0000", 2024-08-17T17:24:57.358 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3599999999999999e-07 2024-08-17T17:24:57.358 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.358 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.358 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.358 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.683026+0000", 2024-08-17T17:24:57.358 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7171e-05 2024-08-17T17:24:57.358 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.358 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.359 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.359 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.683049+0000", 2024-08-17T17:24:57.359 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2577000000000001e-05 2024-08-17T17:24:57.359 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.359 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.359 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.359 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.683062+0000", 2024-08-17T17:24:57.359 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3134e-05 2024-08-17T17:24:57.359 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.359 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.359 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.359 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.875331+0000", 2024-08-17T17:24:57.359 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19226912900000001 2024-08-17T17:24:57.359 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.360 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.360 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.360 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.875369+0000", 2024-08-17T17:24:57.360 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7978000000000001e-05 2024-08-17T17:24:57.360 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.360 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.360 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.360 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.935541+0000", 2024-08-17T17:24:57.360 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.060171988000000003 2024-08-17T17:24:57.360 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.360 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.360 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.360 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.935607+0000", 2024-08-17T17:24:57.360 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5860999999999998e-05 2024-08-17T17:24:57.360 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.361 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.361 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.361 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.361 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.361 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88379 7.e e1151/1144 7:77a5a758:::benchmark_data_smithi063_196002_object88378:head v 1151'5627, mlcod=0'0)", 2024-08-17T17:24:57.361 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.707652+0000", 2024-08-17T17:24:57.361 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 356.62119794500001, 2024-08-17T17:24:57.361 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.22874829799999999, 2024-08-17T17:24:57.361 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.361 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.362 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:57 smithi162 ceph-mon[104668]: pgmap v1623: 119 pgs: 119 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-08-17T17:24:57.362 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.362 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.362 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.362 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.363 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.707652+0000", 2024-08-17T17:24:57.363 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.363 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.365 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.365 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.365 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.707652+0000", 2024-08-17T17:24:57.365 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.365 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.365 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.366 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.366 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.707654+0000", 2024-08-17T17:24:57.366 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0449999999999999e-06 2024-08-17T17:24:57.366 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.366 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.366 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.366 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.707692+0000", 2024-08-17T17:24:57.366 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8145999999999997e-05 2024-08-17T17:24:57.366 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.366 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.366 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.366 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.707715+0000", 2024-08-17T17:24:57.367 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3397e-05 2024-08-17T17:24:57.367 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.367 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.367 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.367 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.707728+0000", 2024-08-17T17:24:57.367 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2893e-05 2024-08-17T17:24:57.367 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.367 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.367 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.367 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.887668+0000", 2024-08-17T17:24:57.367 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.179940088 2024-08-17T17:24:57.367 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.367 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.368 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.368 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.887705+0000", 2024-08-17T17:24:57.368 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6947999999999999e-05 2024-08-17T17:24:57.368 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.368 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.368 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.368 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.936360+0000", 2024-08-17T17:24:57.368 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.048654898000000002 2024-08-17T17:24:57.368 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.368 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.368 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.368 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.936400+0000", 2024-08-17T17:24:57.368 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9882999999999997e-05 2024-08-17T17:24:57.368 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.369 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.369 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.369 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.369 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.369 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "MOSDPGPush(7.e 1154/1144 [PushOp(7:7d25040c:::benchmark_data_smithi063_196002_object8959:head, version: 1132'552, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:7d25040c:::benchmark_data_smithi063_196002_object8959:head@1132'552, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:24:57.369 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:20.740336+0000", 2024-08-17T17:24:57.369 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 336.58851330300001, 2024-08-17T17:24:57.369 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.151667202, 2024-08-17T17:24:57.369 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.369 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.369 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:24:57.369 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.369 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.369 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.369 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.740336+0000", 2024-08-17T17:24:57.370 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.370 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.370 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.370 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.370 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.740336+0000", 2024-08-17T17:24:57.370 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.370 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.370 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.370 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.370 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.740337+0000", 2024-08-17T17:24:57.370 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.1800000000000001e-07 2024-08-17T17:24:57.370 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.370 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.371 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.371 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.740368+0000", 2024-08-17T17:24:57.371 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1130999999999997e-05 2024-08-17T17:24:57.371 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.371 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.371 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.371 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.740384+0000", 2024-08-17T17:24:57.371 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5610000000000001e-05 2024-08-17T17:24:57.371 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.371 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.371 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.371 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.740397+0000", 2024-08-17T17:24:57.372 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3380000000000001e-05 2024-08-17T17:24:57.372 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.372 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.372 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.372 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.882222+0000", 2024-08-17T17:24:57.372 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14182467000000001 2024-08-17T17:24:57.372 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.372 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.372 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.372 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.882251+0000", 2024-08-17T17:24:57.372 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.9852000000000001e-05 2024-08-17T17:24:57.372 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.372 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.372 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.373 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.892004+0000", 2024-08-17T17:24:57.373 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0097521410000000006 2024-08-17T17:24:57.373 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.373 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.373 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.373 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.373 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.373 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:134362 7.17 e1157/1151 7:e8dc9a7c:::benchmark_data_smithi063_196002_object22269:head v 1157'6750, mlcod=1133'1341)", 2024-08-17T17:24:57.373 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:56.694759+0000", 2024-08-17T17:24:57.373 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 300.63409032499999, 2024-08-17T17:24:57.373 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.106734392, 2024-08-17T17:24:57.373 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.373 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.373 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.374 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.374 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.374 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.374 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.694759+0000", 2024-08-17T17:24:57.374 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.374 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.374 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.374 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.374 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.694759+0000", 2024-08-17T17:24:57.374 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.374 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.374 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.374 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.375 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.694760+0000", 2024-08-17T17:24:57.375 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3e-07 2024-08-17T17:24:57.375 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.375 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.375 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.375 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.694766+0000", 2024-08-17T17:24:57.375 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3080000000000004e-06 2024-08-17T17:24:57.375 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.375 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.375 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.375 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.694777+0000", 2024-08-17T17:24:57.375 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0247e-05 2024-08-17T17:24:57.375 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.375 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.376 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.376 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.694783+0000", 2024-08-17T17:24:57.376 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2500000000000003e-06 2024-08-17T17:24:57.376 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.376 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.376 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.376 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.694849+0000", 2024-08-17T17:24:57.376 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5810999999999997e-05 2024-08-17T17:24:57.376 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.376 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.376 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.376 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.694871+0000", 2024-08-17T17:24:57.376 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2459e-05 2024-08-17T17:24:57.376 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.377 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.377 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.377 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.801456+0000", 2024-08-17T17:24:57.377 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.106584465 2024-08-17T17:24:57.377 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.377 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.377 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.377 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.801494+0000", 2024-08-17T17:24:57.377 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8121999999999999e-05 2024-08-17T17:24:57.377 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.377 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.377 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.377 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.377 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.378 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:134370 7.6 e1157/1152 7:608408ef:::benchmark_data_smithi063_196002_object22277:head v 1157'7059, mlcod=1133'1411)", 2024-08-17T17:24:57.378 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:56.694985+0000", 2024-08-17T17:24:57.378 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 300.63386502999998, 2024-08-17T17:24:57.378 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.10359499799999999, 2024-08-17T17:24:57.378 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.378 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.378 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.378 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.378 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.378 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.378 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.694985+0000", 2024-08-17T17:24:57.378 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.379 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.379 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.379 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.379 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.694985+0000", 2024-08-17T17:24:57.379 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.379 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.379 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.379 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.379 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.694985+0000", 2024-08-17T17:24:57.379 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0200000000000002e-07 2024-08-17T17:24:57.379 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.379 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.379 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.380 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.694992+0000", 2024-08-17T17:24:57.380 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1289999999999998e-06 2024-08-17T17:24:57.380 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.380 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.380 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.380 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.695002+0000", 2024-08-17T17:24:57.380 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7669999999999998e-06 2024-08-17T17:24:57.380 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.380 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.380 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.380 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.695008+0000", 2024-08-17T17:24:57.380 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8000000000000004e-06 2024-08-17T17:24:57.380 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.380 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.381 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.381 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.695075+0000", 2024-08-17T17:24:57.381 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7547999999999998e-05 2024-08-17T17:24:57.381 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.381 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.381 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.381 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.695096+0000", 2024-08-17T17:24:57.381 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0859000000000001e-05 2024-08-17T17:24:57.381 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.381 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.381 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.381 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.798527+0000", 2024-08-17T17:24:57.381 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.10343079299999999 2024-08-17T17:24:57.381 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.382 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.382 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.382 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.798580+0000", 2024-08-17T17:24:57.382 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2500000000000002e-05 2024-08-17T17:24:57.382 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.382 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.382 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.382 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.382 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.382 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:134375 7.17 e1157/1151 7:e88b3ee8:::benchmark_data_smithi063_196002_object22282:head v 1157'6751, mlcod=1133'1341)", 2024-08-17T17:24:57.382 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:56.697662+0000", 2024-08-17T17:24:57.382 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 300.63118769200003, 2024-08-17T17:24:57.382 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.10388185799999999, 2024-08-17T17:24:57.383 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.383 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.383 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.383 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.383 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.383 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.383 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.697662+0000", 2024-08-17T17:24:57.383 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.383 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.383 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.383 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.383 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.697662+0000", 2024-08-17T17:24:57.383 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.384 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.384 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.384 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.384 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.697663+0000", 2024-08-17T17:24:57.384 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1459999999999999e-06 2024-08-17T17:24:57.384 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.384 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.384 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.384 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.697670+0000", 2024-08-17T17:24:57.384 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.323e-06 2024-08-17T17:24:57.384 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.384 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.384 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.384 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.787400+0000", 2024-08-17T17:24:57.384 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.089729444000000005 2024-08-17T17:24:57.385 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.385 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.385 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.385 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.787415+0000", 2024-08-17T17:24:57.385 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5546e-05 2024-08-17T17:24:57.385 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.385 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.385 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.385 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.790223+0000", 2024-08-17T17:24:57.385 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0028072169999999999 2024-08-17T17:24:57.385 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.385 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.385 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.385 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.790244+0000", 2024-08-17T17:24:57.386 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0981000000000001e-05 2024-08-17T17:24:57.386 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.386 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.386 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.386 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.801521+0000", 2024-08-17T17:24:57.386 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011277343 2024-08-17T17:24:57.386 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.386 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.386 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.386 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.801544+0000", 2024-08-17T17:24:57.386 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2858e-05 2024-08-17T17:24:57.386 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.386 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.386 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.387 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.387 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.387 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:134382 7.4 e1157/1152 7:2641aebd:::benchmark_data_smithi063_196002_object22289:head v 1157'7069, mlcod=1133'1454)", 2024-08-17T17:24:57.387 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:56.697940+0000", 2024-08-17T17:24:57.387 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 300.63090946800003, 2024-08-17T17:24:57.387 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.102782122, 2024-08-17T17:24:57.387 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.387 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.387 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.387 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.387 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.387 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.387 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.697940+0000", 2024-08-17T17:24:57.387 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.388 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.388 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.388 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.388 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.697940+0000", 2024-08-17T17:24:57.388 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.388 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.388 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.388 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.388 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.697951+0000", 2024-08-17T17:24:57.388 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1096e-05 2024-08-17T17:24:57.388 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.388 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.388 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.388 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.697957+0000", 2024-08-17T17:24:57.389 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6550000000000001e-06 2024-08-17T17:24:57.389 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.389 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.389 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.389 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.789937+0000", 2024-08-17T17:24:57.389 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.091979869000000006 2024-08-17T17:24:57.389 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.389 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.389 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.389 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.789948+0000", 2024-08-17T17:24:57.389 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1238e-05 2024-08-17T17:24:57.389 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.389 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.389 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.390 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.791297+0000", 2024-08-17T17:24:57.390 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001348906 2024-08-17T17:24:57.390 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.390 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.390 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.390 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.791325+0000", 2024-08-17T17:24:57.390 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8073999999999999e-05 2024-08-17T17:24:57.390 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.390 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.390 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.390 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.800676+0000", 2024-08-17T17:24:57.390 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0093512339999999999 2024-08-17T17:24:57.390 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.390 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.391 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.391 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:56.800722+0000", 2024-08-17T17:24:57.391 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6050000000000001e-05 2024-08-17T17:24:57.391 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.391 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.391 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.391 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.391 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.391 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.7.0:82783 7.5 e1180/1152 7:a27210bc:::benchmark_data_smithi063_196002_object109261:head v 1180'9710, mlcod=1180'9693)", 2024-08-17T17:24:57.391 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:21:32.185748+0000", 2024-08-17T17:24:57.391 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 205.14310131900001, 2024-08-17T17:24:57.391 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.107565682, 2024-08-17T17:24:57.391 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.391 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.391 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.392 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.392 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.392 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.392 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.185748+0000", 2024-08-17T17:24:57.392 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.392 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.392 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.392 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.393 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:57.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:57.394 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.185748+0000", 2024-08-17T17:24:57.394 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.394 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.394 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.394 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.394 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.185749+0000", 2024-08-17T17:24:57.394 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7700000000000004e-07 2024-08-17T17:24:57.394 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.394 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.394 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.394 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.185759+0000", 2024-08-17T17:24:57.394 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.6199999999999994e-06 2024-08-17T17:24:57.395 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.395 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.395 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.395 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.185774+0000", 2024-08-17T17:24:57.395 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5131e-05 2024-08-17T17:24:57.395 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.395 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.395 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.395 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.185781+0000", 2024-08-17T17:24:57.395 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5020000000000003e-06 2024-08-17T17:24:57.395 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.395 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.395 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.395 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.241151+0000", 2024-08-17T17:24:57.396 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.055370115999999997 2024-08-17T17:24:57.396 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.396 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.396 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.396 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.241171+0000", 2024-08-17T17:24:57.396 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9585e-05 2024-08-17T17:24:57.396 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.396 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.396 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.396 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.292583+0000", 2024-08-17T17:24:57.396 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.051412226999999998 2024-08-17T17:24:57.396 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.396 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.396 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.397 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.293314+0000", 2024-08-17T17:24:57.397 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00073082400000000004 2024-08-17T17:24:57.397 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.397 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.397 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.397 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.397 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.397 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.7.0:82784 7.5 e1180/1152 7:a27b286a:::benchmark_data_smithi063_196002_object109295:head v 1180'9711, mlcod=1180'9693)", 2024-08-17T17:24:57.397 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:21:32.187266+0000", 2024-08-17T17:24:57.397 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 205.14158385100001, 2024-08-17T17:24:57.397 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.10628850400000001, 2024-08-17T17:24:57.397 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.397 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.398 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.398 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.398 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.398 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.398 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.187266+0000", 2024-08-17T17:24:57.398 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.398 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.398 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.398 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.398 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.187266+0000", 2024-08-17T17:24:57.398 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.398 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.398 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.398 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.399 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.187266+0000", 2024-08-17T17:24:57.399 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1099999999999995e-07 2024-08-17T17:24:57.399 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.399 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.399 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.399 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.187276+0000", 2024-08-17T17:24:57.399 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.5230000000000001e-06 2024-08-17T17:24:57.399 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.399 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.399 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.399 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.187293+0000", 2024-08-17T17:24:57.399 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6583999999999999e-05 2024-08-17T17:24:57.399 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.400 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.400 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.400 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.187300+0000", 2024-08-17T17:24:57.400 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6729999999999992e-06 2024-08-17T17:24:57.400 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.400 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.400 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.400 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.248643+0000", 2024-08-17T17:24:57.400 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.061342332999999999 2024-08-17T17:24:57.400 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.400 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.400 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.400 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.248694+0000", 2024-08-17T17:24:57.401 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1767000000000001e-05 2024-08-17T17:24:57.401 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.401 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.401 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.401 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.293349+0000", 2024-08-17T17:24:57.401 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.044654204000000003 2024-08-17T17:24:57.401 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.401 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.401 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.401 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.293554+0000", 2024-08-17T17:24:57.401 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00020580899999999999 2024-08-17T17:24:57.401 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.401 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.402 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.402 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.402 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.402 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.7.0:82785 7.5 e1180/1152 7:a27c8beb:::benchmark_data_smithi063_196002_object111430:head v 1180'9712, mlcod=1180'9693)", 2024-08-17T17:24:57.402 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:21:32.189186+0000", 2024-08-17T17:24:57.402 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 205.13966404199999, 2024-08-17T17:24:57.402 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.104570231, 2024-08-17T17:24:57.402 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.402 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.402 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.402 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.402 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.403 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.403 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.189186+0000", 2024-08-17T17:24:57.403 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.403 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.403 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.403 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.403 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.189186+0000", 2024-08-17T17:24:57.403 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.403 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.403 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.403 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.403 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.189186+0000", 2024-08-17T17:24:57.403 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3900000000000004e-07 2024-08-17T17:24:57.403 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.403 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.404 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.404 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.189196+0000", 2024-08-17T17:24:57.404 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3170000000000008e-06 2024-08-17T17:24:57.404 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.404 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.404 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.404 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.189207+0000", 2024-08-17T17:24:57.404 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1683e-05 2024-08-17T17:24:57.404 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.404 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.404 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.404 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.189214+0000", 2024-08-17T17:24:57.404 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.64e-06 2024-08-17T17:24:57.404 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.404 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.405 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.405 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.252385+0000", 2024-08-17T17:24:57.405 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.063171299 2024-08-17T17:24:57.405 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.405 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.405 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.405 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.252404+0000", 2024-08-17T17:24:57.405 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8708000000000001e-05 2024-08-17T17:24:57.405 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.405 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.405 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.405 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.293557+0000", 2024-08-17T17:24:57.405 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.041153062999999997 2024-08-17T17:24:57.406 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.406 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.406 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.406 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.293756+0000", 2024-08-17T17:24:57.406 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000198882 2024-08-17T17:24:57.406 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.406 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.406 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.406 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.406 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.406 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45218.0:138112 8.2 e1213/1195 8:4cfb624c:::benchmark_data_smithi063_230025_object11622:head v 1213'8819, mlcod=1213'8817)", 2024-08-17T17:24:57.406 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:24:10.386314+0000", 2024-08-17T17:24:57.407 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 46.942535898999999, 2024-08-17T17:24:57.407 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.123093868, 2024-08-17T17:24:57.407 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.407 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.407 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.407 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.407 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.407 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.407 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386314+0000", 2024-08-17T17:24:57.407 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.407 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.407 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.407 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.407 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386314+0000", 2024-08-17T17:24:57.408 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.408 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.408 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.408 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.408 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386315+0000", 2024-08-17T17:24:57.408 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0449999999999999e-06 2024-08-17T17:24:57.408 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.408 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.408 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.408 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386325+0000", 2024-08-17T17:24:57.408 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0037999999999999e-05 2024-08-17T17:24:57.408 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.408 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.408 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.409 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386337+0000", 2024-08-17T17:24:57.409 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1958e-05 2024-08-17T17:24:57.409 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.409 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.409 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.409 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386345+0000", 2024-08-17T17:24:57.409 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.1489999999999994e-06 2024-08-17T17:24:57.409 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.409 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.409 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.409 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386423+0000", 2024-08-17T17:24:57.409 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7682999999999995e-05 2024-08-17T17:24:57.410 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.410 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.410 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.410 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386445+0000", 2024-08-17T17:24:57.410 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2558e-05 2024-08-17T17:24:57.410 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.410 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.410 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.410 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390898+0000", 2024-08-17T17:24:57.410 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044525980000000003 2024-08-17T17:24:57.410 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.410 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.410 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.410 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509408+0000", 2024-08-17T17:24:57.411 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.11850983900000001 2024-08-17T17:24:57.411 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.411 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.411 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.411 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.411 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.411 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45218.0:138114 8.6 e1213/1195 8:6a96a6b7:::benchmark_data_smithi063_230025_object11624:head v 1213'8609, mlcod=1213'8607)", 2024-08-17T17:24:57.411 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:24:10.389955+0000", 2024-08-17T17:24:57.411 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 46.938895013, 2024-08-17T17:24:57.411 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.12685882800000001, 2024-08-17T17:24:57.411 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.411 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.411 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.411 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.411 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.412 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.412 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.389955+0000", 2024-08-17T17:24:57.412 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.412 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.412 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.412 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.412 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.389955+0000", 2024-08-17T17:24:57.412 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.412 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.412 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.412 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.412 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.389956+0000", 2024-08-17T17:24:57.412 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8700000000000004e-07 2024-08-17T17:24:57.412 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.412 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.413 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.413 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.389960+0000", 2024-08-17T17:24:57.413 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9490000000000004e-06 2024-08-17T17:24:57.413 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.413 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.413 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.413 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.389974+0000", 2024-08-17T17:24:57.413 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3152999999999999e-05 2024-08-17T17:24:57.413 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.413 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.413 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.413 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.389982+0000", 2024-08-17T17:24:57.413 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9889999999999992e-06 2024-08-17T17:24:57.413 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.413 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.414 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.414 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390058+0000", 2024-08-17T17:24:57.414 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6427000000000005e-05 2024-08-17T17:24:57.414 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.414 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.414 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.414 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390078+0000", 2024-08-17T17:24:57.414 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9629000000000001e-05 2024-08-17T17:24:57.414 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.414 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.414 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.414 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.516764+0000", 2024-08-17T17:24:57.414 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.126685933 2024-08-17T17:24:57.415 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.415 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.415 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.415 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.516813+0000", 2024-08-17T17:24:57.415 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9861000000000003e-05 2024-08-17T17:24:57.415 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.415 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.415 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.415 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.415 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.415 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45218.0:138103 8.c e1213/1195 8:30f0a028:::benchmark_data_smithi063_230025_object11613:head v 1213'8745, mlcod=1213'8743)", 2024-08-17T17:24:57.415 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:24:10.390810+0000", 2024-08-17T17:24:57.415 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 46.938039648999997, 2024-08-17T17:24:57.415 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14214605299999999, 2024-08-17T17:24:57.415 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.416 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.416 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.416 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.416 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.416 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.416 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390810+0000", 2024-08-17T17:24:57.416 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.416 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.416 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.416 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.416 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390810+0000", 2024-08-17T17:24:57.416 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.416 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.416 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.416 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390811+0000", 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7999999999999995e-07 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390817+0000", 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0100000000000001e-06 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509388+0000", 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.118571464 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.417 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509396+0000", 2024-08-17T17:24:57.418 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9980000000000003e-06 2024-08-17T17:24:57.418 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.418 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.418 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.418 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509570+0000", 2024-08-17T17:24:57.418 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000174279 2024-08-17T17:24:57.418 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.418 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.418 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.418 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509590+0000", 2024-08-17T17:24:57.418 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9083999999999999e-05 2024-08-17T17:24:57.418 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.419 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.419 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.419 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.532822+0000", 2024-08-17T17:24:57.419 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.023232017000000001 2024-08-17T17:24:57.419 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.419 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.419 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.419 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.532956+0000", 2024-08-17T17:24:57.419 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013452099999999999 2024-08-17T17:24:57.419 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.419 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.419 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.419 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.419 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.420 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45218.0:138118 8.2 e1213/1195 8:48763a67:::benchmark_data_smithi063_230025_object11628:head v 1213'8820, mlcod=1213'8817)", 2024-08-17T17:24:57.420 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:24:10.393238+0000", 2024-08-17T17:24:57.420 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 46.935611807000001, 2024-08-17T17:24:57.420 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.123133243, 2024-08-17T17:24:57.420 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.420 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.420 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.420 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.420 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.420 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.420 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.393238+0000", 2024-08-17T17:24:57.420 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.420 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.420 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.421 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.421 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.393238+0000", 2024-08-17T17:24:57.421 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.421 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.421 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.421 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.421 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.393239+0000", 2024-08-17T17:24:57.421 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0839999999999999e-06 2024-08-17T17:24:57.421 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.421 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.421 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.421 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.393250+0000", 2024-08-17T17:24:57.421 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.063e-05 2024-08-17T17:24:57.422 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.422 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.422 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.422 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509130+0000", 2024-08-17T17:24:57.422 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.11588066599999999 2024-08-17T17:24:57.422 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.422 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.422 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.422 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509164+0000", 2024-08-17T17:24:57.422 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3503000000000001e-05 2024-08-17T17:24:57.422 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.423 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.423 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.423 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509437+0000", 2024-08-17T17:24:57.423 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000272902 2024-08-17T17:24:57.423 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.423 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.423 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.423 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509462+0000", 2024-08-17T17:24:57.423 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5718999999999999e-05 2024-08-17T17:24:57.423 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.423 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.423 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.423 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.516304+0000", 2024-08-17T17:24:57.423 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0068414399999999998 2024-08-17T17:24:57.424 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.424 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.424 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.424 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.516371+0000", 2024-08-17T17:24:57.424 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7298999999999994e-05 2024-08-17T17:24:57.424 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.424 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.424 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.424 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.424 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.424 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45218.0:138121 8.9 e1213/1195 8:980b75d7:::benchmark_data_smithi063_230025_object11631:head v 1213'8648, mlcod=1213'8646)", 2024-08-17T17:24:57.424 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:24:10.395445+0000", 2024-08-17T17:24:57.424 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 46.933404670000002, 2024-08-17T17:24:57.424 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.129388477, 2024-08-17T17:24:57.424 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:24:57.425 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:24:57.425 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:24:57.425 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:24:57.425 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.425 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:24:57.425 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.395445+0000", 2024-08-17T17:24:57.425 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.425 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.425 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.425 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:24:57.425 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.395445+0000", 2024-08-17T17:24:57.425 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:24:57.425 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.425 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.425 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:24:57.426 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.395446+0000", 2024-08-17T17:24:57.426 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.1500000000000003e-07 2024-08-17T17:24:57.426 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.426 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.426 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:24:57.426 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.395454+0000", 2024-08-17T17:24:57.426 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7700000000000001e-06 2024-08-17T17:24:57.426 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.426 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.426 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:24:57.426 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509168+0000", 2024-08-17T17:24:57.426 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.11371437500000001 2024-08-17T17:24:57.426 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.427 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.427 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:24:57.427 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509196+0000", 2024-08-17T17:24:57.427 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8255999999999998e-05 2024-08-17T17:24:57.427 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.427 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.427 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:24:57.427 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509487+0000", 2024-08-17T17:24:57.427 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00029032400000000001 2024-08-17T17:24:57.427 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.427 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.427 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:24:57.427 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509507+0000", 2024-08-17T17:24:57.427 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9964000000000001e-05 2024-08-17T17:24:57.427 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.428 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.428 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:24:57.428 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.524782+0000", 2024-08-17T17:24:57.428 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.015275476 2024-08-17T17:24:57.428 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:24:57.428 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:24:57.428 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:24:57.428 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.524833+0000", 2024-08-17T17:24:57.428 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1397000000000003e-05 2024-08-17T17:24:57.428 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.428 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.428 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.428 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:24:57.428 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:24:57.428 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:24:57.494 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:57.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.532 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:57.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:57.633 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:24:57.670 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:57.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:24:57.772 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:57.798 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:57.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:57.900 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:57.940 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:57.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:57.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:58.042 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:58.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.078 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:58.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:58.179 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:24:58.220 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:58.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.221 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:24:58.322 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:24:58.356 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:58.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:24:58.458 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:58.492 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:58.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:58.593 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:24:58.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:58 smithi162 ceph-mon[104668]: 1.b scrub starts 2024-08-17T17:24:58.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:58 smithi162 ceph-mon[104668]: 1.b scrub ok 2024-08-17T17:24:58.630 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:58.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:24:58.731 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:58.761 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:58.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:58.778 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:58 smithi063 ceph-mon[114756]: 1.b scrub starts 2024-08-17T17:24:58.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:58 smithi063 ceph-mon[114756]: 1.b scrub ok 2024-08-17T17:24:58.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:58 smithi063 ceph-mon[109843]: 1.b scrub starts 2024-08-17T17:24:58.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:58 smithi063 ceph-mon[109843]: 1.b scrub ok 2024-08-17T17:24:58.863 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:58.916 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:58.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:58.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:59.018 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:24:59.045 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:59.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:24:59.147 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:59.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.199 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:59.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:59.302 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:59.334 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:59.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:59.435 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:24:59.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.475 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:59.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:24:59.576 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:24:59.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.629 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:24:59.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:24:59.730 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:59.743 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:59 smithi063 ceph-mon[114756]: pgmap v1624: 119 pgs: 119 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 991 op/s 2024-08-17T17:24:59.744 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:59 smithi063 ceph-mon[114756]: 1.8 scrub starts 2024-08-17T17:24:59.744 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:24:59 smithi063 ceph-mon[114756]: 1.8 scrub ok 2024-08-17T17:24:59.744 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:59 smithi063 ceph-mon[109843]: pgmap v1624: 119 pgs: 119 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 991 op/s 2024-08-17T17:24:59.744 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:59 smithi063 ceph-mon[109843]: 1.8 scrub starts 2024-08-17T17:24:59.744 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:24:59 smithi063 ceph-mon[109843]: 1.8 scrub ok 2024-08-17T17:24:59.764 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:59.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:24:59.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:59 smithi162 ceph-mon[104668]: pgmap v1624: 119 pgs: 119 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 991 op/s 2024-08-17T17:24:59.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:59 smithi162 ceph-mon[104668]: 1.8 scrub starts 2024-08-17T17:24:59.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:24:59 smithi162 ceph-mon[104668]: 1.8 scrub ok 2024-08-17T17:24:59.866 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:24:59.898 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:24:59.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:24:59.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:00.000 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:00.033 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:00.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:00.135 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:00.166 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:25:00.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.181 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:00.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:00.283 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:00.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.313 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:00.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:00.351 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:00.414 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:00.441 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:00.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:00.543 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:00.569 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:00.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:00.671 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:00.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.697 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:00.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:00.798 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:00.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.847 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:00.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:00.948 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:00.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:00.985 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:00.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:01.086 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:01.112 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:01.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:01.214 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:01.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.242 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:01.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:01.343 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:01.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.376 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:01.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:01.415 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:25:01.478 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:01.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.525 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:01.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:01.626 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:01.671 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:01.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:01.773 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:01.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:01 smithi063 ceph-mon[109843]: pgmap v1625: 119 pgs: 119 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-08-17T17:25:01.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:01 smithi063 ceph-mon[109843]: 1.1b scrub starts 2024-08-17T17:25:01.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:01 smithi063 ceph-mon[109843]: 1.1b scrub ok 2024-08-17T17:25:01.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:01 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2775503887' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:01.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:01 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:01.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:01 smithi063 ceph-mon[114756]: pgmap v1625: 119 pgs: 119 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-08-17T17:25:01.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:01 smithi063 ceph-mon[114756]: 1.1b scrub starts 2024-08-17T17:25:01.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:01 smithi063 ceph-mon[114756]: 1.1b scrub ok 2024-08-17T17:25:01.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:01 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2775503887' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:01.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:01 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:01.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:01.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.822 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:01.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:01.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:01 smithi162 ceph-mon[104668]: pgmap v1625: 119 pgs: 119 active+clean; 4.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 726 op/s 2024-08-17T17:25:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:01 smithi162 ceph-mon[104668]: 1.1b scrub starts 2024-08-17T17:25:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:01 smithi162 ceph-mon[104668]: 1.1b scrub ok 2024-08-17T17:25:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:01 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2775503887' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:01 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:01.923 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:01.948 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:01.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:01.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:02.049 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:02.077 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:02.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:02.179 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:02.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.212 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:02.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:02.313 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:02.347 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:02.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:02.411 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:25:02.449 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:02.484 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:02.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:02.586 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:02.620 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:02.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:02.722 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:02.764 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:02.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:02.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:02 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:25:02.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:02 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:25:02.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:02 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:02.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:02 smithi063 ceph-mon[114756]: osdmap e1233: 8 total, 8 up, 8 in 2024-08-17T17:25:02.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:02 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2781309825' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:25:02.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:02 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:25:02.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:02 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:25:02.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:02 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:02.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:02 smithi063 ceph-mon[109843]: osdmap e1233: 8 total, 8 up, 8 in 2024-08-17T17:25:02.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:02 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2781309825' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:25:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:02 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:25:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:02 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:25:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:02 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:02 smithi162 ceph-mon[104668]: osdmap e1233: 8 total, 8 up, 8 in 2024-08-17T17:25:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:02 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2781309825' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:25:02.866 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:02.899 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:02.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:02.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:03.001 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:03.030 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:03.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:03.132 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:03.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.176 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:03.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:03.277 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:03.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:03.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:03.401 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:03.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.441 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:03.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:03.542 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:03.563 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:03 smithi063 ceph-mon[114756]: pgmap v1627: 119 pgs: 119 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2024-08-17T17:25:03.563 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:03 smithi063 ceph-mon[114756]: 1.1c scrub starts 2024-08-17T17:25:03.563 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:03 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2781309825' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:25:03.563 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:03 smithi063 ceph-mon[114756]: osdmap e1234: 8 total, 8 up, 8 in 2024-08-17T17:25:03.563 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:03 smithi063 ceph-mon[114756]: 1.1c scrub ok 2024-08-17T17:25:03.563 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:03 smithi063 ceph-mon[114756]: 1.18 scrub starts 2024-08-17T17:25:03.563 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:03 smithi063 ceph-mon[114756]: 1.18 scrub ok 2024-08-17T17:25:03.564 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:25:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:25:03] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.51.0" 2024-08-17T17:25:03.564 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:03 smithi063 ceph-mon[109843]: pgmap v1627: 119 pgs: 119 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2024-08-17T17:25:03.564 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:03 smithi063 ceph-mon[109843]: 1.1c scrub starts 2024-08-17T17:25:03.564 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:03 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2781309825' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:25:03.564 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:03 smithi063 ceph-mon[109843]: osdmap e1234: 8 total, 8 up, 8 in 2024-08-17T17:25:03.564 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:03 smithi063 ceph-mon[109843]: 1.1c scrub ok 2024-08-17T17:25:03.564 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:03 smithi063 ceph-mon[109843]: 1.18 scrub starts 2024-08-17T17:25:03.564 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:03 smithi063 ceph-mon[109843]: 1.18 scrub ok 2024-08-17T17:25:03.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.601 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:03.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:03.702 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:03.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.733 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:03.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:03.746 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:03 smithi162 ceph-mon[104668]: pgmap v1627: 119 pgs: 119 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 884 op/s 2024-08-17T17:25:03.746 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:03 smithi162 ceph-mon[104668]: 1.1c scrub starts 2024-08-17T17:25:03.746 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:03 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2781309825' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:25:03.746 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:03 smithi162 ceph-mon[104668]: osdmap e1234: 8 total, 8 up, 8 in 2024-08-17T17:25:03.746 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:03 smithi162 ceph-mon[104668]: 1.1c scrub ok 2024-08-17T17:25:03.746 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:03 smithi162 ceph-mon[104668]: 1.18 scrub starts 2024-08-17T17:25:03.746 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:03 smithi162 ceph-mon[104668]: 1.18 scrub ok 2024-08-17T17:25:03.834 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:03.877 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:03.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:03.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:03.979 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:04.007 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:04.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:04.109 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:04.143 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:04.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:04.245 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:04.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.277 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:04.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:04.378 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:04.418 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:04.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:04.520 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:04.563 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:04.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:04.647 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:04 smithi063 ceph-mon[114756]: 1.17 deep-scrub starts 2024-08-17T17:25:04.647 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:04 smithi063 ceph-mon[114756]: 1.17 deep-scrub ok 2024-08-17T17:25:04.647 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:04 smithi063 ceph-mon[109843]: 1.17 deep-scrub starts 2024-08-17T17:25:04.647 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:04 smithi063 ceph-mon[109843]: 1.17 deep-scrub ok 2024-08-17T17:25:04.665 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:04.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.694 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:04.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:04.795 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:04.833 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:04.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:04.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:04 smithi162 ceph-mon[104668]: 1.17 deep-scrub starts 2024-08-17T17:25:04.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:04 smithi162 ceph-mon[104668]: 1.17 deep-scrub ok 2024-08-17T17:25:04.935 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:04.970 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:04.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:04.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:05.073 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:05.105 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:05.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:05.207 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:05.249 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:05.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:05.305 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:25:05.351 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:05.400 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:05.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:05.478 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:25:05.478 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:25:05.478 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:25:05.478 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:25:05.479 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:25:05.479 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:25:05.479 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:25:05.479 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:25:05.479 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:25:05.479 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:25:05.480 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:25:05.480 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:25:05.480 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:25:05.480 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:25:05.480 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:25:05.480 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:25:05.501 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:05.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:05 smithi063 ceph-mon[114756]: pgmap v1629: 119 pgs: 119 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 973 op/s 2024-08-17T17:25:05.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:05 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2178908105' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:05.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:05 smithi063 ceph-mon[109843]: pgmap v1629: 119 pgs: 119 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 973 op/s 2024-08-17T17:25:05.533 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:05 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2178908105' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:05.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.551 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:05.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:05.652 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:05.696 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:05.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:05.798 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:05.835 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:05.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:05.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:05 smithi162 ceph-mon[104668]: pgmap v1629: 119 pgs: 119 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 973 op/s 2024-08-17T17:25:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:05 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2178908105' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:05.937 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:05.981 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:05.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:05.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:06.083 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:06.125 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:06.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:06.227 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:06.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.305 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:06.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:06.406 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:06.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.451 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:06.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:06.552 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:06.580 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:06.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:06.682 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:06.712 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:06.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:06.813 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:06.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.850 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:06.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:06.952 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:06.983 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:06.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:06.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:07.085 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:07.111 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:07.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:07.213 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:07.258 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:07.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:07.360 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:07.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.397 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:07.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:07.498 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:07.529 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:07.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:07 smithi063 ceph-mon[114756]: pgmap v1630: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2024-08-17T17:25:07.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:07 smithi063 ceph-mon[109843]: pgmap v1630: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2024-08-17T17:25:07.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:07.631 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:07.667 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:07.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:07.768 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:07.806 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:07.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:07.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:07 smithi162 ceph-mon[104668]: pgmap v1630: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2024-08-17T17:25:07.891 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:07.908 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:07.916 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:07.940 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:07.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:07.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:08.042 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:08.069 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:08.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:08.171 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:08.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.211 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:08.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:08.265 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:25:08.312 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:08.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.353 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:08.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:08.454 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:08.501 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:08.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:08.586 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:08 smithi063 ceph-mon[114756]: 1.15 deep-scrub starts 2024-08-17T17:25:08.587 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:08 smithi063 ceph-mon[114756]: 1.15 deep-scrub ok 2024-08-17T17:25:08.587 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:08 smithi063 ceph-mon[109843]: 1.15 deep-scrub starts 2024-08-17T17:25:08.587 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:08 smithi063 ceph-mon[109843]: 1.15 deep-scrub ok 2024-08-17T17:25:08.603 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:08.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.653 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:08.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:08.715 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:25:08.754 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:08.785 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:08.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:08.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:08 smithi162 ceph-mon[104668]: 1.15 deep-scrub starts 2024-08-17T17:25:08.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:08 smithi162 ceph-mon[104668]: 1.15 deep-scrub ok 2024-08-17T17:25:08.887 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:08.918 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:08.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:08.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:09.020 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:09.071 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:09.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:09.173 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:09.205 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:09.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:09.307 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:09.338 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:09.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:09.440 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:09.477 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:09.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:09.579 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:09.607 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:09.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:09.709 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:09.743 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:09.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:09.762 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:09 smithi063 ceph-mon[109843]: pgmap v1631: 119 pgs: 119 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s 2024-08-17T17:25:09.762 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:09 smithi063 ceph-mon[114756]: pgmap v1631: 119 pgs: 119 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s 2024-08-17T17:25:09.844 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:09.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:09 smithi162 ceph-mon[104668]: pgmap v1631: 119 pgs: 119 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.19k op/s 2024-08-17T17:25:09.895 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:09.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:09.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:09.997 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:10.026 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:10.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:10.129 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:10.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.167 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:10.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:10.268 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:10.315 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:10.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:10.417 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:10.459 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:10.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:10.561 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:10.600 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:10.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:10.702 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:10.752 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:10.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:10.854 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:10.908 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:10.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:10.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:11.010 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:11.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.042 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:11.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:11.144 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:11.182 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:11.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:11.285 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:11.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.330 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:11.330 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:11.431 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:11.477 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:11.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:11.579 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:11.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.629 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:11.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:11.730 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:11.775 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:11 smithi063 ceph-mon[109843]: pgmap v1632: 119 pgs: 119 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-08-17T17:25:11.775 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:11 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:25:11.775 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:11 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:25:11.775 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:11 smithi063 ceph-mon[114756]: pgmap v1632: 119 pgs: 119 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-08-17T17:25:11.775 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:11 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:25:11.775 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:11 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:25:11.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.779 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:11.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:11.784 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:11.785 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:11.844 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:25:11.844 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:25:11.881 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:11.925 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:11.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:11.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:12.027 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:12.055 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:12.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:11 smithi162 ceph-mon[104668]: pgmap v1632: 119 pgs: 119 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-08-17T17:25:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:11 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:25:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:11 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:25:12.157 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:12.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.196 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:12.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:12.297 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:12.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.331 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:12.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:12.432 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:12.460 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:12.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:12.547 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:25:12.562 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:12.639 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:12.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:12.741 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:12.776 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:12.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:12.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:12 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:25:12.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:12 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:25:12.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:12 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:25:12.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:12 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:25:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:12 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:25:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:12 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:25:12.878 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:12.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:12.922 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:12.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:13.023 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:13.061 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:13.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:12 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:25:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:12 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:25:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:12 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:25:13.163 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:13.206 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:13.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:13.309 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:13.342 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:13.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:13.444 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:13.475 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:13.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:25:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:25:13] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.51.0" 2024-08-17T17:25:13.577 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:13.619 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:13.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:13.721 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:13.748 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:13.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:13.850 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:13.873 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:13.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:13.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:13.975 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:14.013 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:14.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:13 smithi063 ceph-mon[114756]: pgmap v1633: 119 pgs: 119 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-08-17T17:25:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:13 smithi063 ceph-mon[109843]: pgmap v1633: 119 pgs: 119 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-08-17T17:25:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:13 smithi162 ceph-mon[104668]: pgmap v1633: 119 pgs: 119 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 959 op/s 2024-08-17T17:25:14.115 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:14.155 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:14.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:14.257 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:14.291 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:14.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:14.393 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:14.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.436 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:14.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:14.537 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:14.570 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:14.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:14.672 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:14.702 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:14.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.703 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:14.750 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:25:14.803 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:14.840 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:14.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:14.941 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:14.964 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:14.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:14.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:15.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:14 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2894003239' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:25:15.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:14 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:25:15.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:14 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2894003239' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:25:15.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:14 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:25:15.067 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:15.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:14 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2894003239' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:25:15.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:14 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:25:15.101 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:15.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:15.203 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:15.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.230 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:15.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:15.331 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:15.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.374 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:15.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:15.475 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:15.501 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:15.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:15.603 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:15.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.652 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:15.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:15.753 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:15.783 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:15.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:15.885 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:15.918 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:15.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:15.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:16.020 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:16.021 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:15 smithi063 ceph-mon[114756]: pgmap v1634: 119 pgs: 119 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2024-08-17T17:25:16.022 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:15 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:25:16.022 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:15 smithi063 ceph-mon[114756]: osdmap e1235: 8 total, 8 up, 8 in 2024-08-17T17:25:16.022 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:15 smithi063 ceph-mon[109843]: pgmap v1634: 119 pgs: 119 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2024-08-17T17:25:16.022 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:15 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:25:16.022 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:15 smithi063 ceph-mon[109843]: osdmap e1235: 8 total, 8 up, 8 in 2024-08-17T17:25:16.055 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:16.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:15 smithi162 ceph-mon[104668]: pgmap v1634: 119 pgs: 119 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 911 op/s 2024-08-17T17:25:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:15 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:25:16.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:15 smithi162 ceph-mon[104668]: osdmap e1235: 8 total, 8 up, 8 in 2024-08-17T17:25:16.157 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:16.198 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:16.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:16.300 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:16.345 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:16.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:16.447 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:16.496 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:16.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:16.598 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:16.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.644 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:16.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:16.745 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:16.790 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:16.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:16.892 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:16.929 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:16.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:16.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:17.031 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:17.058 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:25:17.059 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:17.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:17.162 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:17.215 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:17.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:17.316 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:17.351 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:17.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:17.453 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:17.480 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:17.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:17.583 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:17.600 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:17 smithi063 ceph-mon[114756]: pgmap v1636: 119 pgs: 119 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-08-17T17:25:17.600 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:17 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:25:17.601 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:17 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:17.601 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:17 smithi063 ceph-mon[109843]: pgmap v1636: 119 pgs: 119 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-08-17T17:25:17.601 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:17 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:25:17.601 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:17 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:17.610 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:17.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:17.712 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:17.764 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:17.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:17.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:17 smithi162 ceph-mon[104668]: pgmap v1636: 119 pgs: 119 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 963 op/s 2024-08-17T17:25:17.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:17 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:25:17.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:17 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:17.866 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:17.909 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:17.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:17.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:18.011 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:18.038 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:18.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:18.140 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:18.177 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:18.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:18.280 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:18.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.319 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:18.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:18.421 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:18.451 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:18.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:18.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:18 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3950062924' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:18.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:18 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3950062924' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:18.553 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:18.554 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:25:18.554 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:25:18.554 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:25:18.554 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:25:18.554 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:25:18.555 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:25:18.555 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:25:18.555 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:25:18.555 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:25:18.555 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:25:18.556 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:25:18.556 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:25:18.556 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:25:18.556 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:25:18.556 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:25:18.556 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:25:18.590 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:18.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:18.692 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:18.729 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:18.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:18.830 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:18.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:18 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3950062924' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:18.863 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:18.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:18.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:18.965 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:19.001 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:19.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.003 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:19.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:19.104 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:19.145 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:19.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:19.247 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:19.274 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:19.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:19.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:19.415 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:19.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:19.518 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:19.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:19 smithi063 ceph-mon[114756]: pgmap v1637: 119 pgs: 119 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2024-08-17T17:25:19.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:19 smithi063 ceph-mon[109843]: pgmap v1637: 119 pgs: 119 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2024-08-17T17:25:19.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.555 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:19.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:19.656 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.686 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:19.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:19.787 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:19.825 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:19.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:19 smithi162 ceph-mon[104668]: pgmap v1637: 119 pgs: 119 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2024-08-17T17:25:19.927 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:19.968 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:19.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:19.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:20.070 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:20.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.110 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:20.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:20.212 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:20.246 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:20.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:20.348 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:20.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.382 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:20.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:20.483 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:20.520 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:20.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:20.621 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:20.655 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:20.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:20.757 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:20.795 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:20.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:20.897 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:20.930 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:20.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:20.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:20.991 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:21.031 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:21.061 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:21.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:21.163 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:21.192 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:21.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:21.293 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:21.320 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:21.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:21.422 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:21.449 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:21.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:21.551 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:21.578 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:21.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:21.680 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:21.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.712 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:21.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:21.747 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:21 smithi063 ceph-mon[114756]: pgmap v1638: 119 pgs: 119 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 974 op/s 2024-08-17T17:25:21.747 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:21 smithi063 ceph-mon[109843]: pgmap v1638: 119 pgs: 119 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 974 op/s 2024-08-17T17:25:21.814 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:21 smithi162 ceph-mon[104668]: pgmap v1638: 119 pgs: 119 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 974 op/s 2024-08-17T17:25:21.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:21.869 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:21.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:21.970 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:22.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.002 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:22.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:22.103 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:22.143 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:22.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:22.180 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:25:22.246 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:22.272 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:22.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:22.374 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:22.396 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:25:22.401 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:22.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:22.503 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:22.533 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:22.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:22.635 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:22.677 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:22.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:22.779 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:22.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:22 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/593242758' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:22.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:22 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:22.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:22 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/593242758' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:22.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:22 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:22.814 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:22.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:22 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/593242758' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:22.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:22 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:25:22.918 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:22.953 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:22.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:22.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:23.055 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:23.102 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:23.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:23.204 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:23.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.239 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:23.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:23.340 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:23.365 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:23.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:23.467 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:23.509 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:23.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:23.611 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:23.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.654 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:23.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:23.681 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:25:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:25:23] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.51.0" 2024-08-17T17:25:23.681 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:23 smithi063 ceph-mon[114756]: pgmap v1639: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 978 op/s 2024-08-17T17:25:23.682 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:23 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:23.682 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:23 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:25:23.682 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:23 smithi063 ceph-mon[114756]: osdmap e1236: 8 total, 8 up, 8 in 2024-08-17T17:25:23.682 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:23 smithi063 ceph-mon[109843]: pgmap v1639: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 978 op/s 2024-08-17T17:25:23.682 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:23 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:23.682 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:23 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:25:23.682 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:23 smithi063 ceph-mon[109843]: osdmap e1236: 8 total, 8 up, 8 in 2024-08-17T17:25:23.755 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:23.795 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:23.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:23 smithi162 ceph-mon[104668]: pgmap v1639: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 978 op/s 2024-08-17T17:25:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:23 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:23 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:25:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:23 smithi162 ceph-mon[104668]: osdmap e1236: 8 total, 8 up, 8 in 2024-08-17T17:25:23.897 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:23.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:23.934 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:23.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:24.035 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:24.066 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:24.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:24.170 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:24.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.221 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:24.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:24.322 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:24.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.365 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:24.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:24.466 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:24.503 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:24.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:24.604 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:24.665 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:24.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:24.767 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:24.808 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:24.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:24.910 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:24.940 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:24.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:24.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:25.042 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.079 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:25.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:25.180 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:25.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.215 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:25.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:25.317 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:25.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.355 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:25.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:25.457 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:25.468 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:25 smithi063 ceph-mon[109843]: pgmap v1641: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:25:25.468 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:25 smithi063 ceph-mon[114756]: pgmap v1641: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:25:25.507 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:25.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:25.609 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:25.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.662 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:25.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:25.710 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:25:25.710 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:25:25.710 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:25:25.710 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:25:25.710 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:25:25.763 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:25.807 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:25.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:25.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:25 smithi162 ceph-mon[104668]: pgmap v1641: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-08-17T17:25:25.909 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:25.939 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:25.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:25.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:26.041 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:26.081 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:26.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:26.111 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:26.183 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:26.221 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:26.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:26.323 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:26.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.363 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:26.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:26.390 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:25:26.464 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:26.503 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:26.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:26.605 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:26.640 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:26.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:26.659 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:25:26.665 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:26 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2945503923' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:25:26.665 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:26 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2945503923' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:25:26.707 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:25:26.741 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:26.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.793 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:26.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:26.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:26 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2945503923' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:25:26.894 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:26.964 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:26.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:26.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:27.066 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:27.096 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:27.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:27.199 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:27.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.249 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:27.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:27.350 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:27.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.386 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:27.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:27.487 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:27.526 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:27.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:27.628 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:27.657 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:27 smithi063 ceph-mon[109843]: pgmap v1642: 119 pgs: 119 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2024-08-17T17:25:27.658 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:27 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2945503923' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:25:27.658 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:27 smithi063 ceph-mon[109843]: osdmap e1237: 8 total, 8 up, 8 in 2024-08-17T17:25:27.658 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:27 smithi063 ceph-mon[114756]: pgmap v1642: 119 pgs: 119 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2024-08-17T17:25:27.658 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:27 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2945503923' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:25:27.659 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:27 smithi063 ceph-mon[114756]: osdmap e1237: 8 total, 8 up, 8 in 2024-08-17T17:25:27.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.662 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:27.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:27.698 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:25:27.762 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:27.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.819 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:27.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:27 smithi162 ceph-mon[104668]: pgmap v1642: 119 pgs: 119 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2024-08-17T17:25:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:27 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2945503923' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:25:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:27 smithi162 ceph-mon[104668]: osdmap e1237: 8 total, 8 up, 8 in 2024-08-17T17:25:27.921 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:27.947 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:27.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:27.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:28.049 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:28.082 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:28.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:28.184 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:28.211 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:28.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:28.313 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:28.349 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:28.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:28.451 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:28.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.490 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:28.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:28.592 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:28.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.632 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:28.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:28.733 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:28.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.765 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:28.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:28.866 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:28.906 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:28.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:28.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:29.008 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:29.040 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:29.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:29.071 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:29.141 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:29.166 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:29.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:29.268 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:29.304 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:29.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:29.406 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:29.430 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:29.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:29.532 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:29.564 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:29.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:29.641 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:29 smithi063 ceph-mon[109843]: pgmap v1644: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-08-17T17:25:29.641 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:29 smithi063 ceph-mon[114756]: pgmap v1644: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-08-17T17:25:29.665 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:29.710 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:29.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:29.812 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:29.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:29 smithi162 ceph-mon[104668]: pgmap v1644: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s 2024-08-17T17:25:29.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.861 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:29.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:29.962 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:29.998 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:29.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:29.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:30.100 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:30.104 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:30.147 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:30.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:30.249 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:30.273 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:30.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:30.375 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:30.396 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:30.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:30.498 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:30.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.532 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:30.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:30.633 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:30.664 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:30.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:30.766 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:30.802 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:30.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:30.903 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:30.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:30.932 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:30.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:30.999 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:25:30.999 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:25:31.034 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:31.040 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:25:31.041 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:25:31.064 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:31.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:31.076 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:25:31.076 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:25:31.121 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:25:31.121 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:25:31.160 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:25:31.160 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:25:31.166 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:31.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:25:31.196 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:25:31.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.203 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:31.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:31.229 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:25:31.229 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:25:31.279 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:25:31.279 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:25:31.304 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:31.330 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:31.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:31.432 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:31.458 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:31.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:31.561 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:31.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.595 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:31.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:31.696 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:31.731 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:31.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:31.751 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:31 smithi063 ceph-mon[114756]: pgmap v1645: 119 pgs: 119 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-08-17T17:25:31.751 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:31 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1241713135' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:31.751 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:25:31.752 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:31.752 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:31 smithi063 ceph-mon[109843]: pgmap v1645: 119 pgs: 119 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-08-17T17:25:31.752 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:31 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1241713135' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:31.752 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:25:31.752 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:31.833 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:31.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:31 smithi162 ceph-mon[104668]: pgmap v1645: 119 pgs: 119 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-08-17T17:25:31.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:31 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1241713135' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:31.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:25:31.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:31.865 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:31.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:31.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:31.967 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:32.011 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:32.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:32.112 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:32.141 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:32.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:32.242 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:32.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.283 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:32.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:32.343 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:32.384 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:32.419 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:32.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:32.521 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:32.557 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:32.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:32.659 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:32.701 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:32.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:32.803 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:32.829 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:32.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:32.931 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:32.983 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:32.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:32.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:33.084 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:33.109 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:33.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:33.211 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:33.244 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:33.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:33.346 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:33.371 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:33.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:33.422 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:33 smithi063 ceph-mon[109843]: pgmap v1646: 119 pgs: 119 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-08-17T17:25:33.423 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:25:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:25:33] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.51.0" 2024-08-17T17:25:33.472 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:33.503 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:33.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:33.605 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:33.646 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:33.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:33.748 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:33.779 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:33.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:33 smithi063 ceph-mon[114756]: pgmap v1646: 119 pgs: 119 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-08-17T17:25:33.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:33.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:33 smithi162 ceph-mon[104668]: pgmap v1646: 119 pgs: 119 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-08-17T17:25:33.882 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:33.917 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:33.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:33.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:34.019 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:34.052 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:34.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:34.154 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:34.189 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:34.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:34.290 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:34.319 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:34.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:34.420 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:34.457 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:34.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:34.559 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:34.598 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:34.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:34.700 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:34.753 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:34.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:34.855 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:34.898 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:34.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:34.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:35.000 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:35.028 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:35.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:35.130 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:35.160 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:35.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:35.262 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:35.286 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:35.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:35.388 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:35.424 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:35.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:35.446 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:25:35.526 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:35.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:35 smithi063 ceph-mon[109843]: pgmap v1647: 119 pgs: 119 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2024-08-17T17:25:35.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:35 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3701613362' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:35.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:35 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:35.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:35 smithi063 ceph-mon[114756]: pgmap v1647: 119 pgs: 119 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2024-08-17T17:25:35.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:35 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3701613362' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:35.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:35 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:35.561 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:35.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:35.663 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:35.707 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:35.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:35.809 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:35.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:35 smithi162 ceph-mon[104668]: pgmap v1647: 119 pgs: 119 active+clean; 2.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2024-08-17T17:25:35.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:35 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3701613362' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:35.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:35 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:25:35.847 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:35.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:35.949 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:35.981 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:35.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:35.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:36.082 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:36.110 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:36.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:36.211 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:36.236 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:36.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:36.338 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:36.386 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:36.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:36.487 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:36.528 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:36.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:36.630 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:36.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.665 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:36.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:36.766 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:36.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:36 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:36.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:36 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:36.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:36 smithi063 ceph-mon[114756]: osdmap e1238: 8 total, 8 up, 8 in 2024-08-17T17:25:36.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:36 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:36.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:36 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:36.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:36 smithi063 ceph-mon[109843]: osdmap e1238: 8 total, 8 up, 8 in 2024-08-17T17:25:36.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.799 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:36.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:36.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:36 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:36 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:25:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:36 smithi162 ceph-mon[104668]: osdmap e1238: 8 total, 8 up, 8 in 2024-08-17T17:25:36.900 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:36.941 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:36.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:36.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:37.042 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:37.066 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:37.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:37.168 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:37.203 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:37.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:37.305 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:37.347 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:37.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:37.449 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:37.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.476 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:37.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:37.577 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:37.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.602 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:37.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:37.703 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:37.739 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:37.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:37.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:37 smithi063 ceph-mon[114756]: pgmap v1649: 119 pgs: 119 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2024-08-17T17:25:37.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:37 smithi063 ceph-mon[109843]: pgmap v1649: 119 pgs: 119 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2024-08-17T17:25:37.841 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:37.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:37 smithi162 ceph-mon[104668]: pgmap v1649: 119 pgs: 119 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 926 op/s 2024-08-17T17:25:37.880 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:37.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:37.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:37.982 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:38.004 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:38.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:38.107 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:38.147 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:38.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:38.244 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:25:38.244 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:25:38.249 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:38.288 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:38.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:38.390 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:38.426 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:38.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:38.530 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:38.562 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:38.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:38.664 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:38.696 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:38.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:38.797 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:38.828 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:38.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:38.929 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:38.971 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:38.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:38.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:39.005 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:25:39.072 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:39.096 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:39.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:39.198 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:39.221 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:39.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:39.322 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:39.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.352 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:39.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:39.453 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:39.456 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:25:39.480 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:39.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:39.524 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:25:39.582 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:39.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.629 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:39.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:39.730 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:39.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.766 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:39.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:39.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:39 smithi063 ceph-mon[109843]: pgmap v1650: 119 pgs: 119 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2024-08-17T17:25:39.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:39 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2077764193' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:25:39.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:39 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:25:39.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:39 smithi063 ceph-mon[114756]: pgmap v1650: 119 pgs: 119 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2024-08-17T17:25:39.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:39 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2077764193' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:25:39.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:39 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:25:39.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:39 smithi162 ceph-mon[104668]: pgmap v1650: 119 pgs: 119 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2024-08-17T17:25:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:39 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2077764193' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:25:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:39 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:25:39.867 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:39.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:39.900 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:39.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:40.001 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:40.030 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:40.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:40.044 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:25:40.132 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:40.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.192 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:40.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:40.195 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:25:40.290 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:40.293 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:40.317 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:40.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:40.363 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:40.419 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:40.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.471 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:40.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:40.572 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:40.605 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:40.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:40.707 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:40.713 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:40 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:25:40.713 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:40 smithi063 ceph-mon[114756]: osdmap e1239: 8 total, 8 up, 8 in 2024-08-17T17:25:40.714 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:40 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:25:40.714 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:40 smithi063 ceph-mon[109843]: osdmap e1239: 8 total, 8 up, 8 in 2024-08-17T17:25:40.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.753 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:40.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:40.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:40 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:25:40.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:40 smithi162 ceph-mon[104668]: osdmap e1239: 8 total, 8 up, 8 in 2024-08-17T17:25:40.854 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:40.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:40.897 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:40.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:40.998 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:41.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.038 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:41.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:41.139 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:41.185 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:41.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:41.287 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:41.326 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:41.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:41.428 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:41.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.456 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:41.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:41.557 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:41.593 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:41.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:41.695 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:41.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.728 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:41.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:41.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:41 smithi063 ceph-mon[114756]: pgmap v1652: 119 pgs: 119 active+clean; 2.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-08-17T17:25:41.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:41 smithi063 ceph-mon[109843]: pgmap v1652: 119 pgs: 119 active+clean; 2.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-08-17T17:25:41.829 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:41.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:41 smithi162 ceph-mon[104668]: pgmap v1652: 119 pgs: 119 active+clean; 2.1 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-08-17T17:25:41.860 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:41.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:41.962 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:41.998 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:41.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:41.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:42.100 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:42.135 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:42.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:42.237 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:42.268 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:42.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:42.369 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:42.397 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:42.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:42.499 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:42.542 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:42.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:42.643 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:42.676 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:42.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:42.778 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:42.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.822 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:42.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:42.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:42.964 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:42.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:42.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:43.066 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:43.108 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:43.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:43.210 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:43.250 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:43.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:43.296 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:43.298 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:43.353 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:43.383 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:43.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:43.484 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:43.528 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:43 smithi063 ceph-mon[109843]: pgmap v1653: 119 pgs: 119 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2024-08-17T17:25:43.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:25:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:25:43] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.51.0" 2024-08-17T17:25:43.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:43 smithi063 ceph-mon[114756]: pgmap v1653: 119 pgs: 119 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2024-08-17T17:25:43.531 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:43.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:43.633 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:43.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.672 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:43.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:43.773 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:43.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.801 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:43.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:43.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:43 smithi162 ceph-mon[104668]: pgmap v1653: 119 pgs: 119 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2024-08-17T17:25:43.902 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:43.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:43.947 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:43.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:44.048 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:44.069 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:44.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.071 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:44.172 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:44.203 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:44.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:44.305 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:44.340 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:44.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:44.443 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:44.479 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:44.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:44.581 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:44.626 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:44.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:44.728 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:44.757 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:44.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:44.858 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:44.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:44.902 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:44.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:45.003 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:45.050 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:45.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:45.152 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:45.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.195 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:45.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:45.296 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:45.325 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:45.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:45.427 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:45.454 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:45.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:45.480 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:25:45.480 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:25:45.506 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:25:45.556 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:45.589 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:45.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:45.691 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:45.735 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:45.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:45.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:45 smithi063 ceph-mon[114756]: pgmap v1654: 119 pgs: 119 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:25:45.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:45 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2777920496' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:25:45.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:45 smithi063 ceph-mon[109843]: pgmap v1654: 119 pgs: 119 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:25:45.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:45 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2777920496' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:25:45.837 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:45.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:45 smithi162 ceph-mon[104668]: pgmap v1654: 119 pgs: 119 active+clean; 1.8 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:25:45.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:45 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2777920496' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:25:45.874 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:45.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:45.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:45.975 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:46.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.006 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:46.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:46.107 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:46.146 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:46.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:46.248 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:46.282 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:46.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.283 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:46.384 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:46.409 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:46.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:46.511 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:46.555 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:46.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:46.657 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:46.692 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:46.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:46.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:46 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:46 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2777920496' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:25:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:46 smithi063 ceph-mon[114756]: osdmap e1240: 8 total, 8 up, 8 in 2024-08-17T17:25:46.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:46 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:46 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2777920496' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:25:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:46 smithi063 ceph-mon[109843]: osdmap e1240: 8 total, 8 up, 8 in 2024-08-17T17:25:46.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:46.794 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:46.823 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:46.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:46.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:46 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:25:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:46 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2777920496' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:25:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:46 smithi162 ceph-mon[104668]: osdmap e1240: 8 total, 8 up, 8 in 2024-08-17T17:25:46.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:25:46.925 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:46.959 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:46.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:46.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:47.062 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:47.094 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:47.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:47.196 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:47.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.239 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:47.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:47.340 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:47.384 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:47.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:47.486 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:47.512 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:47.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:47.614 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:47.642 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:47.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:47.744 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:47.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:47 smithi063 ceph-mon[114756]: pgmap v1656: 119 pgs: 119 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2024-08-17T17:25:47.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:47 smithi063 ceph-mon[109843]: pgmap v1656: 119 pgs: 119 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2024-08-17T17:25:47.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.782 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:47.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:47.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:47 smithi162 ceph-mon[104668]: pgmap v1656: 119 pgs: 119 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 782 op/s 2024-08-17T17:25:47.883 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:47.919 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:47.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:47.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:48.021 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:48.058 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:48.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:48.160 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:48.188 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:48.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:48.290 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:48.319 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:25:48.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.340 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:48.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:48.441 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:48.501 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:48.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:48.602 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:48.627 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:48.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:48.729 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:48.763 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:48.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:48.865 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:48.903 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:48.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:48.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:49.005 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:49.039 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:49.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:49.141 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:49.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.184 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:49.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:49.262 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:25:49.262 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:25:49.285 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:49.300 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:25:49.300 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:25:49.335 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:25:49.335 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:25:49.337 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:49.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:49.366 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:25:49.366 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:25:49.405 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:25:49.405 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:25:49.437 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:25:49.437 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:25:49.439 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:49.469 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:25:49.470 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:25:49.495 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:49.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:49.507 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:25:49.507 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:25:49.597 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:49.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.632 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:49.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:49.733 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:49.762 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:49.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:49.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:49 smithi063 ceph-mon[114756]: pgmap v1657: 119 pgs: 119 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:25:49.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:49 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3697929436' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:49.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:49 smithi063 ceph-mon[109843]: pgmap v1657: 119 pgs: 119 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:25:49.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:49 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3697929436' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:49.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:49 smithi162 ceph-mon[104668]: pgmap v1657: 119 pgs: 119 active+clean; 1.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:25:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:49 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3697929436' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:25:49.864 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:49.894 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:49.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:49.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:49.996 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:50.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.030 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:50.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:50.131 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:50.162 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:50.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:50.264 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:50.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.292 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:50.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:50.393 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:50.435 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:50.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:50.538 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:50.575 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:50.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:50.677 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:50.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.708 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:50.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:50.715 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:50.811 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:50.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.856 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:50.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:50.957 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:50.983 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:50.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:50.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:50.985 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:25:51.085 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:51.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.123 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:51.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:51.224 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:51.274 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:51.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:51.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:51.415 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:51.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:51.518 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:51.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:51 smithi063 ceph-mon[109843]: pgmap v1658: 119 pgs: 119 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 967 op/s 2024-08-17T17:25:51.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:51 smithi063 ceph-mon[114756]: pgmap v1658: 119 pgs: 119 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 967 op/s 2024-08-17T17:25:51.547 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:51.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:51.648 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:51.682 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:51.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:51.784 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:51.823 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:51.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:51.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:51 smithi162 ceph-mon[104668]: pgmap v1658: 119 pgs: 119 active+clean; 1.5 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 967 op/s 2024-08-17T17:25:51.925 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:51.967 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:51.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:51.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:52.068 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:52.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.113 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:52.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:52.214 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:52.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.253 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:52.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:52.354 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:52.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.382 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:52.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:52.483 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:52.516 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:52.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:52.618 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:52.649 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:52.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:52.712 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:25:52.751 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:52.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:52 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3256389485' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:25:52.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:52 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3256389485' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:25:52.799 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:52.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:52.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:52 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3256389485' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:25:52.902 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:52.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:52.931 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:52.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:53.032 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:53.052 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:53.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:53.154 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:53.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.190 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:53.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:53.291 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:53.319 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:53.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:53.421 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:53.447 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:53.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:53.539 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:25:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:25:53] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.51.0" 2024-08-17T17:25:53.549 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:53.576 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:53.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:53.678 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:53.704 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:53.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:53.806 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:53.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:53 smithi162 ceph-mon[104668]: pgmap v1659: 119 pgs: 119 active+clean; 1.3 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2024-08-17T17:25:53.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:53 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3256389485' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:25:53.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:53 smithi162 ceph-mon[104668]: osdmap e1241: 8 total, 8 up, 8 in 2024-08-17T17:25:53.847 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:53.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:53.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:53.949 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:54.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.010 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:54.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:54.014 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:53 smithi063 ceph-mon[109843]: pgmap v1659: 119 pgs: 119 active+clean; 1.3 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2024-08-17T17:25:54.014 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:53 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3256389485' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:25:54.014 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:53 smithi063 ceph-mon[109843]: osdmap e1241: 8 total, 8 up, 8 in 2024-08-17T17:25:54.014 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:53 smithi063 ceph-mon[114756]: pgmap v1659: 119 pgs: 119 active+clean; 1.3 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 975 op/s 2024-08-17T17:25:54.014 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:53 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3256389485' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:25:54.014 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:53 smithi063 ceph-mon[114756]: osdmap e1241: 8 total, 8 up, 8 in 2024-08-17T17:25:54.111 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:54.132 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:54.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:54.233 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:54.256 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:54.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:54.358 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:54.380 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:54.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:54.482 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:54.503 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:54.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:54.605 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:54.628 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:54.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:54.730 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:54.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.773 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:54.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:54.829 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:25:54.875 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:54.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:54.910 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:54.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:55.012 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:55.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.038 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:55.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:55.139 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:55.180 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:55.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:55.282 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:55.318 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:55.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:55.421 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:55.447 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:55.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:55.549 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:55.575 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:55.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:55.677 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:55.714 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:55.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:55.816 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:55.822 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:55 smithi063 ceph-mon[109843]: pgmap v1661: 119 pgs: 119 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:25:55.822 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:55 smithi063 ceph-mon[114756]: pgmap v1661: 119 pgs: 119 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:25:55.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:55 smithi162 ceph-mon[104668]: pgmap v1661: 119 pgs: 119 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:25:55.846 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:55.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:55.949 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:55.984 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:55.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:55.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:56.077 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:25:56.086 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:56.171 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:56.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:56.273 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:56.332 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:56.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:56.434 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:56.476 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:56.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:56.508 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:56.561 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:25:56.561 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:25:56.561 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:25:56.561 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:25:56.561 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:25:56.572 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:56.578 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:56.611 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:56.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:56.713 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:56.751 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:56.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:56.853 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:56.906 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:56.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:56.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:57.007 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:57.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.048 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:57.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:57.149 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:57.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.184 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:57.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:57.285 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:57.313 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:57.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:57.368 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:25:57.414 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:57.458 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:57.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:57.560 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:25:57.598 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:57 smithi063 ceph-mon[114756]: pgmap v1662: 119 pgs: 119 active+clean; 1.1 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 966 op/s 2024-08-17T17:25:57.598 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:57 smithi063 ceph-mon[109843]: pgmap v1662: 119 pgs: 119 active+clean; 1.1 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 966 op/s 2024-08-17T17:25:57.610 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:57.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:25:57.712 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:57.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.742 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:57.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:57.843 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:57.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:57 smithi162 ceph-mon[104668]: pgmap v1662: 119 pgs: 119 active+clean; 1.1 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 966 op/s 2024-08-17T17:25:57.887 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:57.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:57.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:57.989 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:58.032 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:58.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:58.134 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:58.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.184 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:58.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:58.285 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:58.319 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:58.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:58.421 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:25:58.487 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:58.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:25:58.589 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:58.626 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:58.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:25:58.728 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:58.770 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:58.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:58.872 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:58.911 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:58.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:58.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:59.013 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:59.055 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:59.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.056 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:59.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:59.071 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:25:59.157 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:59.186 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:59.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:59.288 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:25:59.315 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:59.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:25:59.416 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:25:59.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.450 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:59.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:25:59.551 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:25:59.581 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:59.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:25:59.683 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:25:59.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.720 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:59.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:25:59.821 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:25:59.843 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:25:59.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:25:59.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:25:59 smithi162 ceph-mon[104668]: pgmap v1663: 119 pgs: 119 active+clean; 935 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-08-17T17:25:59.930 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:25:59 smithi063 ceph-mon[114756]: pgmap v1663: 119 pgs: 119 active+clean; 935 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-08-17T17:25:59.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:25:59 smithi063 ceph-mon[109843]: pgmap v1663: 119 pgs: 119 active+clean; 935 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-08-17T17:25:59.945 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:25:59.982 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:25:59.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:25:59.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:00.084 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:00.118 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:00.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:00.220 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:00.247 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:00.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:00.350 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:00.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.393 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:00.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:00.494 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:00.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.518 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:00.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:00.619 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:00.653 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:00.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:00.755 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:00.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.790 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:00.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:00.891 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:00.934 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:00.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:00.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:01.081 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:01.111 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:01.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:01.213 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:01.252 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:01.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:01.354 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:01.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.386 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:01.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:01.487 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:01.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.545 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:01.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:01.646 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:01.676 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:01.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:01.746 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:01 smithi063 ceph-mon[109843]: pgmap v1664: 119 pgs: 119 active+clean; 935 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-08-17T17:26:01.746 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:01.746 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:01 smithi063 ceph-mon[114756]: pgmap v1664: 119 pgs: 119 active+clean; 935 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-08-17T17:26:01.747 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:01.778 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:01.820 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:01.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:01.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:01 smithi162 ceph-mon[104668]: pgmap v1664: 119 pgs: 119 active+clean; 935 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 906 op/s 2024-08-17T17:26:01.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:01.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:01.959 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:01.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:01.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:02.061 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:02.091 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:02.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:02.193 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:02.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.219 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:02.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:02.320 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:02.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.354 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:02.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:02.455 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:02.495 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:02.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:02.597 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:02.639 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:02.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:02.741 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:02.774 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:02.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:02.876 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:02.914 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:02.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:02.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:03.018 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:03.062 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:03.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.063 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:03.164 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:03.192 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:03.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:03.294 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:03.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.336 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:03.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:03.340 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:26:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:26:03] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.51.0" 2024-08-17T17:26:03.437 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:03.476 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:03.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:03.577 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:03.602 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:03.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:03.643 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:26:03.644 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:26:03.644 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:03 smithi063 ceph-mon[109843]: pgmap v1665: 119 pgs: 119 active+clean; 787 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-08-17T17:26:03.644 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:03 smithi063 ceph-mon[114756]: pgmap v1665: 119 pgs: 119 active+clean; 787 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-08-17T17:26:03.680 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:26:03.680 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:26:03.704 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:03.719 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:26:03.719 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:26:03.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.739 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:03.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:03.764 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:26:03.764 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:26:03.797 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:26:03.797 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:26:03.828 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:26:03.829 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:26:03.840 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:03.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:03 smithi162 ceph-mon[104668]: pgmap v1665: 119 pgs: 119 active+clean; 787 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-08-17T17:26:03.858 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:26:03.858 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:26:03.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:03.874 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:03.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:03.884 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:26:03.884 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:26:03.975 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:04.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.011 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:04.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:04.112 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:04.147 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:04.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:04.249 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:04.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.288 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:04.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:04.389 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:04.430 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:04.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:04.531 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:04.556 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:04.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:04.657 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:04.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.687 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:04.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:04.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:04 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3133597043' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:04.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:04 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3133597043' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:04.788 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:04.820 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:04.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:04.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:04 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3133597043' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:04.921 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:04.963 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:04.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:04.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:05.065 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:05.091 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:05.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:05.193 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:05.228 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:05.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:05.330 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:05.366 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:05.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:05.468 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:05.507 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:05.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:05.582 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:26:05.608 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:05.651 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:05.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:05.753 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:05.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.800 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:05.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:05.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:05 smithi162 ceph-mon[104668]: pgmap v1666: 119 pgs: 119 active+clean; 636 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-08-17T17:26:05.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:05 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2918276886' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:05.901 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:05.927 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:05.928 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:05 smithi063 ceph-mon[114756]: pgmap v1666: 119 pgs: 119 active+clean; 636 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-08-17T17:26:05.928 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:05 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2918276886' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:05.928 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:05 smithi063 ceph-mon[109843]: pgmap v1666: 119 pgs: 119 active+clean; 636 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-08-17T17:26:05.928 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:05 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2918276886' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:05.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:05.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:06.030 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:06.064 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:06.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:06.166 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:06.198 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:06.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:06.300 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:06.346 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:06.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:06.448 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:06.482 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:06.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:06.584 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:06.613 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:06.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:06.715 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:06.751 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:06.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:06.798 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:06 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:26:06.798 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:06 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:26:06.798 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:06 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2918276886' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:06.798 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:06 smithi063 ceph-mon[114756]: osdmap e1242: 8 total, 8 up, 8 in 2024-08-17T17:26:06.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:06 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:26:06.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:06 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:26:06.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:06 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2918276886' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:06.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:06 smithi063 ceph-mon[109843]: osdmap e1242: 8 total, 8 up, 8 in 2024-08-17T17:26:06.853 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:06.878 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:06.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:06.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:06.980 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:07.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.031 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:07.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:06 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:26:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:06 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:26:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:06 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2918276886' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:06 smithi162 ceph-mon[104668]: osdmap e1242: 8 total, 8 up, 8 in 2024-08-17T17:26:07.132 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:07.152 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:26:07.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.182 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:07.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:07.283 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:07.326 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:26:07.326 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:26:07.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.330 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:07.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:07.431 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:07.471 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:07.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:07.573 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:07.610 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:07.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:07.615 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:26:07.711 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:07.750 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:07.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:07 smithi063 ceph-mon[114756]: pgmap v1668: 119 pgs: 119 active+clean; 540 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 913 op/s 2024-08-17T17:26:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:07 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1187137794' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:26:07.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:07 smithi063 ceph-mon[109843]: pgmap v1668: 119 pgs: 119 active+clean; 540 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 913 op/s 2024-08-17T17:26:07.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:07 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1187137794' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:26:07.851 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:07.891 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:07.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:07.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:07.993 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:08.032 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:08.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:07 smithi162 ceph-mon[104668]: pgmap v1668: 119 pgs: 119 active+clean; 540 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 913 op/s 2024-08-17T17:26:08.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:07 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1187137794' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:26:08.134 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:08.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.173 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:08.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:08.274 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:08.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.311 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:08.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:08.350 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:26:08.412 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:08.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.457 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:08.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:08.558 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:08.587 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:08.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:08.689 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:08.719 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:08.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:08.821 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:08.847 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:08.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:08.949 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:08.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:08.985 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:08.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:09.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:08 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1187137794' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:26:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:08 smithi063 ceph-mon[114756]: osdmap e1243: 8 total, 8 up, 8 in 2024-08-17T17:26:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:08 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1187137794' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:26:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:08 smithi063 ceph-mon[109843]: osdmap e1243: 8 total, 8 up, 8 in 2024-08-17T17:26:09.087 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:09.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:08 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1187137794' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:26:09.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:08 smithi162 ceph-mon[104668]: osdmap e1243: 8 total, 8 up, 8 in 2024-08-17T17:26:09.138 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:09.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:09.240 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:09.270 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:09.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:09.372 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:09.415 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:09.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:09.517 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:09.542 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:09.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:09.644 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:09.665 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:09.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.666 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:09.767 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:09.802 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:09.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:09.903 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:09.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:09.945 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:09.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:09 smithi063 ceph-mon[109843]: pgmap v1670: 119 pgs: 119 active+clean; 333 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s 2024-08-17T17:26:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:09 smithi063 ceph-mon[114756]: pgmap v1670: 119 pgs: 119 active+clean; 333 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s 2024-08-17T17:26:10.046 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:10.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.076 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:10.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:09 smithi162 ceph-mon[104668]: pgmap v1670: 119 pgs: 119 active+clean; 333 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.20k op/s 2024-08-17T17:26:10.177 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:10.214 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:10.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:10.316 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:10.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.345 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:10.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:10.446 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:10.488 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:10.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:10.589 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:10.635 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:10.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:10.737 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:10.783 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:10.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:10.884 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:10.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:10.918 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:10.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:11.019 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:11.050 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:11.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:11.073 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:26:11.152 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:11.166 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:26:11.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.217 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:11.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:11.223 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:11.276 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:11.318 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:11.380 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:11.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:11.481 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:11.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.534 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:11.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:11.635 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:11.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.680 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:11.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:11.781 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:11.810 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:11.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:11.912 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:11.947 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:11.947 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:11 smithi063 ceph-mon[114756]: pgmap v1671: 119 pgs: 119 active+clean; 333 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-08-17T17:26:11.948 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:11 smithi063 ceph-mon[109843]: pgmap v1671: 119 pgs: 119 active+clean; 333 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-08-17T17:26:11.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:11.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:12.049 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:12.078 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:12.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:11 smithi162 ceph-mon[104668]: pgmap v1671: 119 pgs: 119 active+clean; 333 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-08-17T17:26:12.180 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:12.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.218 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:12.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:12.319 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:12.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.357 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:12.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:12.459 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:12.496 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:12.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:12.598 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:12.638 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:12.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:12.648 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Removed 126487 objects 2024-08-17T17:26:12.648 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Clean up completed and total clean up time :135.13 2024-08-17T17:26:12.740 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:12.762 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:12.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:12.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:12 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:26:12.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:12 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:26:12.864 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:12.885 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:12.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:12.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:12.987 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:13.008 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:13.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:13.073 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:12 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:26:13.110 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:13.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.132 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:13.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:13.185 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-08-17T17:26:13.185 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-08-17T17:26:13.233 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:13.255 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:13.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:13.357 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:13.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.379 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:13.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:13.480 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:13.503 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:13.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:26:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:26:13] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.51.0" 2024-08-17T17:26:13.605 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:13.637 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:13.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:13.738 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:13.760 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:13.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:13.861 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:13.885 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:13.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:13.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:13.920 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:13 smithi063 ceph-mon[114756]: pgmap v1672: 119 pgs: 119 active+clean; 169 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 934 op/s 2024-08-17T17:26:13.920 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:13 smithi063 ceph-mon[109843]: pgmap v1672: 119 pgs: 119 active+clean; 169 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 934 op/s 2024-08-17T17:26:13.987 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:14.010 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:14.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:14.034 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:13 smithi162 ceph-mon[104668]: pgmap v1672: 119 pgs: 119 active+clean; 169 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 934 op/s 2024-08-17T17:26:14.111 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:14.133 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:14.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:14.235 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:14.244 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:26:14.244 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:26:14.256 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:14.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:14.261 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:26:14.261 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:26:14.277 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:26:14.278 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:26:14.294 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:26:14.294 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:26:14.311 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:26:14.311 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:26:14.328 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:26:14.328 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:26:14.345 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:26:14.345 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:26:14.357 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:14.362 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:26:14.363 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:26:14.382 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:14.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:14.484 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:14.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.507 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:14.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:14.608 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:14.631 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:14.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:14.669 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:26:14.734 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:14.755 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:14.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:14.857 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:14.879 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:14.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:14.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:14.971 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:14 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3025908627' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:14.972 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:14 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3883861742' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:26:14.972 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:14 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3025908627' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:14.972 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:14 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3883861742' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:26:14.980 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:14.984 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:14 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3025908627' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:14.984 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:14 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3883861742' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:26:15.001 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:15.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:15.018 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:15.103 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:15.126 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:15.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:15.227 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:15.249 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:15.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:15.351 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:15.373 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:15.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:15.474 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:15.495 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:15.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:15.546 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:15.546 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:15.598 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:15.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.621 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:15.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:15.722 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:15.744 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:15.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:15.846 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:15.867 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:15.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:15.969 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:15.991 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:15.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:15.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:15 smithi063 ceph-mon[114756]: pgmap v1673: 119 pgs: 119 active+clean; 46 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:26:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:15 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3883861742' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:26:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:15 smithi063 ceph-mon[114756]: osdmap e1244: 8 total, 8 up, 8 in 2024-08-17T17:26:16.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:15 smithi063 ceph-mon[109843]: pgmap v1673: 119 pgs: 119 active+clean; 46 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:26:16.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:15 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3883861742' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:26:16.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:15 smithi063 ceph-mon[109843]: osdmap e1244: 8 total, 8 up, 8 in 2024-08-17T17:26:16.092 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:15 smithi162 ceph-mon[104668]: pgmap v1673: 119 pgs: 119 active+clean; 46 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:26:16.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:15 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3883861742' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:26:16.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:15 smithi162 ceph-mon[104668]: osdmap e1244: 8 total, 8 up, 8 in 2024-08-17T17:26:16.114 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:16.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:16.216 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:16.239 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:16.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:16.341 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:16.344 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:26:16.344 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:26:16.363 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:16.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:16.465 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:16.489 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:16.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:16.590 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:16.619 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:16.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:16.721 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:16.742 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:16.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:16.844 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:16.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.865 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:16.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:16.966 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:16.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:16.989 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:16.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:16.993 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:26:17.090 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:17.110 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:17.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:17.212 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:17.234 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:17.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:17.336 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:17.357 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:17.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:17.459 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:17.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.483 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:17.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:17.584 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:17.605 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:17.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:17.707 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:17.728 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:17.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:17.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:17 smithi063 ceph-mon[109843]: pgmap v1675: 119 pgs: 119 active+clean; 11 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2024-08-17T17:26:17.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:17 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:17.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:17 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:17.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:17 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:17.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:17 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:17.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:17 smithi063 ceph-mon[114756]: pgmap v1675: 119 pgs: 119 active+clean; 11 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2024-08-17T17:26:17.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:17 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:17.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:17 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:17.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:17 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:17.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:17 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:17.830 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:17.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:17 smithi162 ceph-mon[104668]: pgmap v1675: 119 pgs: 119 active+clean; 11 MiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2024-08-17T17:26:17.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:17 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:17.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:17 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:17.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:17 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:17.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:17 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:17.853 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:17.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:17.955 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:17.979 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:17.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:17.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:18.081 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:18.104 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:18.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:18.206 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:18.229 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:18.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:18.330 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:18.352 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:18.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:18.454 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:18.476 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:18.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:18.577 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:18.604 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:18.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:18.705 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:18.731 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:18.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:18.833 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:18.857 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:18.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:18.959 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:18.981 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:18.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:18.983 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:19.083 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:19.106 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:19.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:19.208 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:19.230 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:19.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.231 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:19.332 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:19.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.354 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:19.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:19.371 INFO:teuthology.orchestra.run.smithi063.stderr:pool 'unique_pool_3' removed 2024-08-17T17:26:19.456 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:19.477 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:19.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:19.579 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:19.602 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:19.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:19.704 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:19.726 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:19.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:19.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:19 smithi063 ceph-mon[114756]: pgmap v1676: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 531 op/s 2024-08-17T17:26:19.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:19 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1630544644' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:26:19.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:19 smithi063 ceph-mon[109843]: pgmap v1676: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 531 op/s 2024-08-17T17:26:19.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:19 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1630544644' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:26:19.827 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:19 smithi162 ceph-mon[104668]: pgmap v1676: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 531 op/s 2024-08-17T17:26:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:19 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1630544644' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:26:19.850 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:19.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:19.952 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:19.978 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:19.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:19.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:20.079 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:20.100 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:20.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:20.202 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:20.224 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:20.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:20.326 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:20.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.350 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:20.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:20.451 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:20.473 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:20.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:20.576 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:20.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.599 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:20.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:20.610 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:20 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1630544644' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:26:20.610 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:20 smithi063 ceph-mon[109843]: osdmap e1245: 8 total, 8 up, 8 in 2024-08-17T17:26:20.610 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:20 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1630544644' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:26:20.610 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:20 smithi063 ceph-mon[114756]: osdmap e1245: 8 total, 8 up, 8 in 2024-08-17T17:26:20.632 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:26:20.632 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:26:20.632 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2024-08-17T17:26:20.633 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_4 16 2024-08-17T17:26:20.699 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:20.721 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:20.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:20.823 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:20.844 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:20.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:20.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:20 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1630544644' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:26:20.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:20 smithi162 ceph-mon[104668]: osdmap e1245: 8 total, 8 up, 8 in 2024-08-17T17:26:20.947 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:20.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:20.969 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:20.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:21.070 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:21.094 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:21.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:21.196 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:21.218 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:21.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:21.320 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:21.346 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:21.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:21.448 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:21.458 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:26:21.458 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:26:21.459 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 0 2024-08-17T17:26:21.459 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:26:21.471 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:21.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:21.573 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:21.596 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:21.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:21.698 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:21.721 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:21.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:21 smithi063 ceph-mon[114756]: pgmap v1678: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:21 smithi063 ceph-mon[109843]: pgmap v1678: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:21.823 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:21.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.846 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:21.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:21.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:21 smithi162 ceph-mon[104668]: pgmap v1678: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:21.947 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:21.973 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:21.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:21.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:22.075 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:22.101 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:22.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:22.155 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:26:22.203 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:22.229 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:22.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:22.330 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:22.355 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:22.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:22.415 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:22.415 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:22.456 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:22.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.482 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:22.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:22.583 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:22.605 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:22.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:22.707 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:22.734 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:22.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:22.836 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:22.858 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:22.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:22.960 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:22.982 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:22.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:22.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:23.084 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:23.110 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:23.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:23.205 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:23.212 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:23.235 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:23.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:23.337 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:23.361 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:23.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:23.462 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:23.486 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:23.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:23.588 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:23.610 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:23.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:23.695 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:23 smithi063 ceph-mon[109843]: pgmap v1679: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:23.696 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:26:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:26:23] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.51.0" 2024-08-17T17:26:23.696 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:23 smithi063 ceph-mon[114756]: pgmap v1679: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:23.711 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:23.755 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:23.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:23 smithi162 ceph-mon[104668]: pgmap v1679: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:23.858 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:23.880 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:23.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:23.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:23.982 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:24.004 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:24.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:24.106 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:24.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.133 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:24.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:24.234 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:24.257 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:24.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:24.359 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:24.383 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:24.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:24.391 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:26:24.486 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:24.512 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:24.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:24.614 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:24.639 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:24.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:24.741 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:24.765 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:24.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:24.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:24 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/157640219' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:26:24.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:24 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:26:24.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:24 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/157640219' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:26:24.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:24 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:26:24.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:24 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/157640219' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:26:24.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:24 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:26:24.867 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:24.892 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:24.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:24.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:24.994 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:25.018 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:25.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:25.119 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:25.143 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:25.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:25.244 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:25.267 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:25.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:25.369 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:25.394 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:25.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:25.487 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:25 smithi063 ceph-mon[114756]: pgmap v1680: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:25.487 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:25 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:25.487 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:25 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:26:25.487 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:25 smithi063 ceph-mon[114756]: osdmap e1246: 8 total, 8 up, 8 in 2024-08-17T17:26:25.487 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:25 smithi063 ceph-mon[109843]: pgmap v1680: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:25.488 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:25 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:25.488 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:25 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:26:25.488 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:25 smithi063 ceph-mon[109843]: osdmap e1246: 8 total, 8 up, 8 in 2024-08-17T17:26:25.496 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:25.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.519 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:25.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:25.620 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:25.643 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:25.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:25.744 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:25.770 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:25.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:25 smithi162 ceph-mon[104668]: pgmap v1680: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:25 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:25 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:26:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:25 smithi162 ceph-mon[104668]: osdmap e1246: 8 total, 8 up, 8 in 2024-08-17T17:26:25.872 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:25.898 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:25.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:25.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:26.000 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:26.023 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:26.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:26.125 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:26.147 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:26.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:26.249 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:26.272 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:26.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:26.373 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:26.396 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:26.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:26.498 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:26.522 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:26.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:26.624 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:26.647 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:26.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:26.655 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:26:26.655 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:26:26.656 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:26:26.656 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:26:26.656 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:26:26.656 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:26:26.656 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:26:26.656 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:26:26.656 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:26:26.657 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:26:26.657 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:26:26.657 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:26:26.657 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:26:26.657 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:26:26.671 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:26:26.672 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:26:26.749 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:26.773 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:26.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:26.875 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:26.898 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:26.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:26.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:26.999 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:27.025 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:27.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:27.127 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:27.150 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:27.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:27.251 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:27.277 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:27.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:27.379 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:27.404 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:27.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:27.505 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:27.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:27 smithi063 ceph-mon[114756]: pgmap v1682: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:27.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:27 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/676510812' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:27.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:27 smithi063 ceph-mon[109843]: pgmap v1682: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:27.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:27 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/676510812' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:27.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.531 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:27.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:27.632 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:27.655 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:27.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:27.756 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:27.779 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:27.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:27.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:27 smithi162 ceph-mon[104668]: pgmap v1682: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:27.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:27 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/676510812' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:27.881 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:27.905 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:27.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:27.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:28.006 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:28.010 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:26:28.011 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:26:28.036 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:28.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:28.137 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:28.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.161 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:28.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:28.261 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:28.285 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:28.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:28.387 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:28.409 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:28.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:28.511 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:28.533 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:28.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:28.635 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:28.656 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:28.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:28.758 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:28.784 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:28.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:28.886 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:28.909 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:28.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:28.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:29.010 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:29.036 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:29.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:29.138 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:29.164 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:29.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:29.265 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:29.291 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:29.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:29.393 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:29.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.416 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:29.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:29.517 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:29.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:29 smithi063 ceph-mon[109843]: from='client.35544 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:29.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:29 smithi063 ceph-mon[109843]: pgmap v1683: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:29.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:29 smithi063 ceph-mon[114756]: from='client.35544 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:29.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:29 smithi063 ceph-mon[114756]: pgmap v1683: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:29.540 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:29.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:29.642 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:29.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.668 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:29.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:29.696 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1682,"stamp":"2024-08-17T17:26:26.357850+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1246,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4678,"ondisk_log_size":4678,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":321,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3806820,"kb_used_data":66372,"kb_used_omap":199,"kb_used_meta":3740152,"kb_avail":746154396,"statfs":{"total":767960285184,"available":764062101504,"internally_reserved":0,"allocated":67964928,"data_stored":47561238,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":3829916412},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":88,"apply_latency_ms":88,"commit_latency_ns":88000000,"apply_latency_ns":88000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-252976,"num_write_kb":-8095169,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002037"},"pg_stats":[{"pgid":"1.11","version":"369'149","reported_seq":1567,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284352+0000","last_change":"2024-08-17T17:24:49.721626+0000","last_active":"2024-08-17T17:26:15.284352+0000","last_peered":"2024-08-17T17:26:15.284352+0000","last_clean":"2024-08-17T17:26:15.284352+0000","last_became_active":"2024-08-17T17:24:20.649128+0000","last_became_peered":"2024-08-17T17:24:20.649128+0000","last_unstale":"2024-08-17T17:26:15.284352+0000","last_undegraded":"2024-08-17T17:26:15.284352+0000","last_fullsized":"2024-08-17T17:26:15.284352+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:03:15.343059+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1589,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711879+0000","last_change":"2024-08-17T17:21:06.463828+0000","last_active":"2024-08-17T17:26:24.711879+0000","last_peered":"2024-08-17T17:26:24.711879+0000","last_clean":"2024-08-17T17:26:24.711879+0000","last_became_active":"2024-08-17T17:21:06.463464+0000","last_became_peered":"2024-08-17T17:21:06.463464+0000","last_unstale":"2024-08-17T17:26:24.711879+0000","last_undegraded":"2024-08-17T17:26:24.711879+0000","last_fullsized":"2024-08-17T17:26:24.711879+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:46.465006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1175,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.850148+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:26:19.850148+0000","last_peered":"2024-08-17T17:26:19.850148+0000","last_clean":"2024-08-17T17:26:19.850148+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:26:19.850148+0000","last_undegraded":"2024-08-17T17:26:19.850148+0000","last_fullsized":"2024-08-17T17:26:19.850148+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1498,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284274+0000","last_change":"2024-08-17T17:24:48.765312+0000","last_active":"2024-08-17T17:26:15.284274+0000","last_peered":"2024-08-17T17:26:15.284274+0000","last_clean":"2024-08-17T17:26:15.284274+0000","last_became_active":"2024-08-17T17:24:20.642731+0000","last_became_peered":"2024-08-17T17:24:20.642731+0000","last_unstale":"2024-08-17T17:26:15.284274+0000","last_undegraded":"2024-08-17T17:26:15.284274+0000","last_fullsized":"2024-08-17T17:26:15.284274+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:08:36.138960+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1186,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284819+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:26:15.284819+0000","last_peered":"2024-08-17T17:26:15.284819+0000","last_clean":"2024-08-17T17:26:15.284819+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:26:15.284819+0000","last_undegraded":"2024-08-17T17:26:15.284819+0000","last_fullsized":"2024-08-17T17:26:15.284819+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1041,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.149520+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:26:15.149520+0000","last_peered":"2024-08-17T17:26:15.149520+0000","last_clean":"2024-08-17T17:26:15.149520+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:26:15.149520+0000","last_undegraded":"2024-08-17T17:26:15.149520+0000","last_fullsized":"2024-08-17T17:26:15.149520+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":802,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711697+0000","last_change":"2024-08-17T17:24:50.880046+0000","last_active":"2024-08-17T17:26:24.711697+0000","last_peered":"2024-08-17T17:26:24.711697+0000","last_clean":"2024-08-17T17:26:24.711697+0000","last_became_active":"2024-08-17T17:24:20.652165+0000","last_became_peered":"2024-08-17T17:24:20.652165+0000","last_unstale":"2024-08-17T17:26:24.711697+0000","last_undegraded":"2024-08-17T17:26:24.711697+0000","last_fullsized":"2024-08-17T17:26:24.711697+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:00.416810+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1140,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711989+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:26:24.711989+0000","last_peered":"2024-08-17T17:26:24.711989+0000","last_clean":"2024-08-17T17:26:24.711989+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:26:24.711989+0000","last_undegraded":"2024-08-17T17:26:24.711989+0000","last_fullsized":"2024-08-17T17:26:24.711989+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1158,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:17.416755+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:26:17.416755+0000","last_peered":"2024-08-17T17:26:17.416755+0000","last_clean":"2024-08-17T17:26:17.416755+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:26:17.416755+0000","last_undegraded":"2024-08-17T17:26:17.416755+0000","last_fullsized":"2024-08-17T17:26:17.416755+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1503,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284514+0000","last_change":"2024-08-17T17:24:50.727183+0000","last_active":"2024-08-17T17:26:15.284514+0000","last_peered":"2024-08-17T17:26:15.284514+0000","last_clean":"2024-08-17T17:26:15.284514+0000","last_became_active":"2024-08-17T17:24:20.646409+0000","last_became_peered":"2024-08-17T17:24:20.646409+0000","last_unstale":"2024-08-17T17:26:15.284514+0000","last_undegraded":"2024-08-17T17:26:15.284514+0000","last_fullsized":"2024-08-17T17:26:15.284514+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.086193+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1588,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711944+0000","last_change":"2024-08-17T17:21:06.442598+0000","last_active":"2024-08-17T17:26:24.711944+0000","last_peered":"2024-08-17T17:26:24.711944+0000","last_clean":"2024-08-17T17:26:24.711944+0000","last_became_active":"2024-08-17T17:21:06.442264+0000","last_became_peered":"2024-08-17T17:21:06.442264+0000","last_unstale":"2024-08-17T17:26:24.711944+0000","last_undegraded":"2024-08-17T17:26:24.711944+0000","last_fullsized":"2024-08-17T17:26:24.711944+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:35.003620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1235,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.673235+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:26:14.673235+0000","last_peered":"2024-08-17T17:26:14.673235+0000","last_clean":"2024-08-17T17:26:14.673235+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:26:14.673235+0000","last_undegraded":"2024-08-17T17:26:14.673235+0000","last_fullsized":"2024-08-17T17:26:14.673235+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1441,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.149754+0000","last_change":"2024-08-17T17:24:50.410621+0000","last_active":"2024-08-17T17:26:15.149754+0000","last_peered":"2024-08-17T17:26:15.149754+0000","last_clean":"2024-08-17T17:26:15.149754+0000","last_became_active":"2024-08-17T17:24:31.863062+0000","last_became_peered":"2024-08-17T17:24:31.863062+0000","last_unstale":"2024-08-17T17:26:15.149754+0000","last_undegraded":"2024-08-17T17:26:15.149754+0000","last_fullsized":"2024-08-17T17:26:15.149754+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.410491+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.410491+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:11.717684+0000","scrub_duration":0.0066315480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1200,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.149996+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:26:15.149996+0000","last_peered":"2024-08-17T17:26:15.149996+0000","last_clean":"2024-08-17T17:26:15.149996+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:26:15.149996+0000","last_undegraded":"2024-08-17T17:26:15.149996+0000","last_fullsized":"2024-08-17T17:26:15.149996+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1173,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284559+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:26:15.284559+0000","last_peered":"2024-08-17T17:26:15.284559+0000","last_clean":"2024-08-17T17:26:15.284559+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:26:15.284559+0000","last_undegraded":"2024-08-17T17:26:15.284559+0000","last_fullsized":"2024-08-17T17:26:15.284559+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1670,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711728+0000","last_change":"2024-08-17T17:24:45.985725+0000","last_active":"2024-08-17T17:26:24.711728+0000","last_peered":"2024-08-17T17:26:24.711728+0000","last_clean":"2024-08-17T17:26:24.711728+0000","last_became_active":"2024-08-17T17:24:31.795836+0000","last_became_peered":"2024-08-17T17:24:31.795836+0000","last_unstale":"2024-08-17T17:26:24.711728+0000","last_undegraded":"2024-08-17T17:26:24.711728+0000","last_fullsized":"2024-08-17T17:26:24.711728+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:45.985596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:45.985596+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:17.897808+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1201,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.150053+0000","last_change":"2024-08-17T17:21:06.540689+0000","last_active":"2024-08-17T17:26:15.150053+0000","last_peered":"2024-08-17T17:26:15.150053+0000","last_clean":"2024-08-17T17:26:15.150053+0000","last_became_active":"2024-08-17T17:21:06.461614+0000","last_became_peered":"2024-08-17T17:21:06.461614+0000","last_unstale":"2024-08-17T17:26:15.150053+0000","last_undegraded":"2024-08-17T17:26:15.150053+0000","last_fullsized":"2024-08-17T17:26:15.150053+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:11.696898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1225,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:17.348587+0000","last_change":"2024-08-17T17:21:06.470389+0000","last_active":"2024-08-17T17:26:17.348587+0000","last_peered":"2024-08-17T17:26:17.348587+0000","last_clean":"2024-08-17T17:26:17.348587+0000","last_became_active":"2024-08-17T17:21:06.469966+0000","last_became_peered":"2024-08-17T17:21:06.469966+0000","last_unstale":"2024-08-17T17:26:17.348587+0000","last_undegraded":"2024-08-17T17:26:17.348587+0000","last_fullsized":"2024-08-17T17:26:17.348587+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:56.140616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":809,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.673736+0000","last_change":"2024-08-17T17:24:48.192447+0000","last_active":"2024-08-17T17:26:14.673736+0000","last_peered":"2024-08-17T17:26:14.673736+0000","last_clean":"2024-08-17T17:26:14.673736+0000","last_became_active":"2024-08-17T17:24:20.644240+0000","last_became_peered":"2024-08-17T17:24:20.644240+0000","last_unstale":"2024-08-17T17:26:14.673736+0000","last_undegraded":"2024-08-17T17:26:14.673736+0000","last_fullsized":"2024-08-17T17:26:14.673736+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:56.795337+0000","scrub_duration":0.010920255,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1159,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.149999+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:26:15.149999+0000","last_peered":"2024-08-17T17:26:15.149999+0000","last_clean":"2024-08-17T17:26:15.149999+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:26:15.149999+0000","last_undegraded":"2024-08-17T17:26:15.149999+0000","last_fullsized":"2024-08-17T17:26:15.149999+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1207,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:17.436740+0000","last_change":"2024-08-17T17:21:06.446798+0000","last_active":"2024-08-17T17:26:17.436740+0000","last_peered":"2024-08-17T17:26:17.436740+0000","last_clean":"2024-08-17T17:26:17.436740+0000","last_became_active":"2024-08-17T17:21:06.446205+0000","last_became_peered":"2024-08-17T17:21:06.446205+0000","last_unstale":"2024-08-17T17:26:17.436740+0000","last_undegraded":"2024-08-17T17:26:17.436740+0000","last_fullsized":"2024-08-17T17:26:17.436740+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:28.584048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":717,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.676119+0000","last_change":"2024-08-17T17:24:52.102415+0000","last_active":"2024-08-17T17:26:14.676119+0000","last_peered":"2024-08-17T17:26:14.676119+0000","last_clean":"2024-08-17T17:26:14.676119+0000","last_became_active":"2024-08-17T17:24:31.795843+0000","last_became_peered":"2024-08-17T17:24:31.795843+0000","last_unstale":"2024-08-17T17:26:14.676119+0000","last_undegraded":"2024-08-17T17:26:14.676119+0000","last_fullsized":"2024-08-17T17:26:14.676119+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:32:08.434197+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1188,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284753+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:26:15.284753+0000","last_peered":"2024-08-17T17:26:15.284753+0000","last_clean":"2024-08-17T17:26:15.284753+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:26:15.284753+0000","last_undegraded":"2024-08-17T17:26:15.284753+0000","last_fullsized":"2024-08-17T17:26:15.284753+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":970,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.675881+0000","last_change":"2024-08-17T17:21:06.460905+0000","last_active":"2024-08-17T17:26:14.675881+0000","last_peered":"2024-08-17T17:26:14.675881+0000","last_clean":"2024-08-17T17:26:14.675881+0000","last_became_active":"2024-08-17T17:21:06.460592+0000","last_became_peered":"2024-08-17T17:21:06.460592+0000","last_unstale":"2024-08-17T17:26:14.675881+0000","last_undegraded":"2024-08-17T17:26:14.675881+0000","last_fullsized":"2024-08-17T17:26:14.675881+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:10.146916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1588,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.149448+0000","last_change":"2024-08-17T17:24:54.418765+0000","last_active":"2024-08-17T17:26:15.149448+0000","last_peered":"2024-08-17T17:26:15.149448+0000","last_clean":"2024-08-17T17:26:15.149448+0000","last_became_active":"2024-08-17T17:24:31.800263+0000","last_became_peered":"2024-08-17T17:24:31.800263+0000","last_unstale":"2024-08-17T17:26:15.149448+0000","last_undegraded":"2024-08-17T17:26:15.149448+0000","last_fullsized":"2024-08-17T17:26:15.149448+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:54.418644+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:54.418644+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:11.024294+0000","scrub_duration":0.013346135,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1610,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711654+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:26:24.711654+0000","last_peered":"2024-08-17T17:26:24.711654+0000","last_clean":"2024-08-17T17:26:24.711654+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:26:24.711654+0000","last_undegraded":"2024-08-17T17:26:24.711654+0000","last_fullsized":"2024-08-17T17:26:24.711654+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1191,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.676098+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:26:14.676098+0000","last_peered":"2024-08-17T17:26:14.676098+0000","last_clean":"2024-08-17T17:26:14.676098+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:26:14.676098+0000","last_undegraded":"2024-08-17T17:26:14.676098+0000","last_fullsized":"2024-08-17T17:26:14.676098+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1534,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284373+0000","last_change":"2024-08-17T17:24:58.712331+0000","last_active":"2024-08-17T17:26:15.284373+0000","last_peered":"2024-08-17T17:26:15.284373+0000","last_clean":"2024-08-17T17:26:15.284373+0000","last_became_active":"2024-08-17T17:24:31.802584+0000","last_became_peered":"2024-08-17T17:24:31.802584+0000","last_unstale":"2024-08-17T17:26:15.284373+0000","last_undegraded":"2024-08-17T17:26:15.284373+0000","last_fullsized":"2024-08-17T17:26:15.284373+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:59.616217+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1208,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.673396+0000","last_change":"2024-08-17T17:21:06.446841+0000","last_active":"2024-08-17T17:26:14.673396+0000","last_peered":"2024-08-17T17:26:14.673396+0000","last_clean":"2024-08-17T17:26:14.673396+0000","last_became_active":"2024-08-17T17:21:06.446271+0000","last_became_peered":"2024-08-17T17:21:06.446271+0000","last_unstale":"2024-08-17T17:26:14.673396+0000","last_undegraded":"2024-08-17T17:26:14.673396+0000","last_fullsized":"2024-08-17T17:26:14.673396+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:20.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1224,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.676320+0000","last_change":"2024-08-17T17:21:06.480911+0000","last_active":"2024-08-17T17:26:14.676320+0000","last_peered":"2024-08-17T17:26:14.676320+0000","last_clean":"2024-08-17T17:26:14.676320+0000","last_became_active":"2024-08-17T17:21:06.480551+0000","last_became_peered":"2024-08-17T17:21:06.480551+0000","last_unstale":"2024-08-17T17:26:14.676320+0000","last_undegraded":"2024-08-17T17:26:14.676320+0000","last_fullsized":"2024-08-17T17:26:14.676320+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:26.003704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":766,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284534+0000","last_change":"2024-08-17T17:24:57.682573+0000","last_active":"2024-08-17T17:26:15.284534+0000","last_peered":"2024-08-17T17:26:15.284534+0000","last_clean":"2024-08-17T17:26:15.284534+0000","last_became_active":"2024-08-17T17:24:31.869575+0000","last_became_peered":"2024-08-17T17:24:31.869575+0000","last_unstale":"2024-08-17T17:26:15.284534+0000","last_undegraded":"2024-08-17T17:26:15.284534+0000","last_fullsized":"2024-08-17T17:26:15.284534+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:57.682409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:57.682409+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:42.144615+0000","scrub_duration":0.0066879779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1193,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.675902+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:26:14.675902+0000","last_peered":"2024-08-17T17:26:14.675902+0000","last_clean":"2024-08-17T17:26:14.675902+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:26:14.675902+0000","last_undegraded":"2024-08-17T17:26:14.675902+0000","last_fullsized":"2024-08-17T17:26:14.675902+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1209,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.676299+0000","last_change":"2024-08-17T17:21:06.451521+0000","last_active":"2024-08-17T17:26:14.676299+0000","last_peered":"2024-08-17T17:26:14.676299+0000","last_clean":"2024-08-17T17:26:14.676299+0000","last_became_active":"2024-08-17T17:21:06.451197+0000","last_became_peered":"2024-08-17T17:21:06.451197+0000","last_unstale":"2024-08-17T17:26:14.676299+0000","last_undegraded":"2024-08-17T17:26:14.676299+0000","last_fullsized":"2024-08-17T17:26:14.676299+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:04.000692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1534,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284476+0000","last_change":"2024-08-17T17:24:52.716059+0000","last_active":"2024-08-17T17:26:15.284476+0000","last_peered":"2024-08-17T17:26:15.284476+0000","last_clean":"2024-08-17T17:26:15.284476+0000","last_became_active":"2024-08-17T17:24:31.798173+0000","last_became_peered":"2024-08-17T17:24:31.798173+0000","last_unstale":"2024-08-17T17:26:15.284476+0000","last_undegraded":"2024-08-17T17:26:15.284476+0000","last_fullsized":"2024-08-17T17:26:15.284476+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:04:56.094994+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1591,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711389+0000","last_change":"2024-08-17T17:21:06.441118+0000","last_active":"2024-08-17T17:26:24.711389+0000","last_peered":"2024-08-17T17:26:24.711389+0000","last_clean":"2024-08-17T17:26:24.711389+0000","last_became_active":"2024-08-17T17:21:06.440800+0000","last_became_peered":"2024-08-17T17:21:06.440800+0000","last_unstale":"2024-08-17T17:26:24.711389+0000","last_undegraded":"2024-08-17T17:26:24.711389+0000","last_fullsized":"2024-08-17T17:26:24.711389+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:08.091528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":95,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.901179+0000","last_change":"2024-08-17T17:22:48.374766+0000","last_active":"2024-08-17T17:26:19.901179+0000","last_peered":"2024-08-17T17:26:19.901179+0000","last_clean":"2024-08-17T17:26:19.901179+0000","last_became_active":"2024-08-17T17:22:48.373556+0000","last_became_peered":"2024-08-17T17:22:48.373556+0000","last_unstale":"2024-08-17T17:26:19.901179+0000","last_undegraded":"2024-08-17T17:26:19.901179+0000","last_fullsized":"2024-08-17T17:26:19.901179+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:33.362080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1589,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.149692+0000","last_change":"2024-08-17T17:24:13.542832+0000","last_active":"2024-08-17T17:26:15.149692+0000","last_peered":"2024-08-17T17:26:15.149692+0000","last_clean":"2024-08-17T17:26:15.149692+0000","last_became_active":"2024-08-17T17:24:11.570282+0000","last_became_peered":"2024-08-17T17:24:11.570282+0000","last_unstale":"2024-08-17T17:26:15.149692+0000","last_undegraded":"2024-08-17T17:26:15.149692+0000","last_fullsized":"2024-08-17T17:26:15.149692+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:49.414162+0000","scrub_duration":0.017597167,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1203,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.285067+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:26:15.285067+0000","last_peered":"2024-08-17T17:26:15.285067+0000","last_clean":"2024-08-17T17:26:15.285067+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:26:15.285067+0000","last_undegraded":"2024-08-17T17:26:15.285067+0000","last_fullsized":"2024-08-17T17:26:15.285067+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1556,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711543+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:26:24.711543+0000","last_peered":"2024-08-17T17:26:24.711543+0000","last_clean":"2024-08-17T17:26:24.711543+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:26:24.711543+0000","last_undegraded":"2024-08-17T17:26:24.711543+0000","last_fullsized":"2024-08-17T17:26:24.711543+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1174,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.673531+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:26:14.673531+0000","last_peered":"2024-08-17T17:26:14.673531+0000","last_clean":"2024-08-17T17:26:14.673531+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:26:14.673531+0000","last_undegraded":"2024-08-17T17:26:14.673531+0000","last_fullsized":"2024-08-17T17:26:14.673531+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":827,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.673361+0000","last_change":"2024-08-17T17:24:11.936322+0000","last_active":"2024-08-17T17:26:14.673361+0000","last_peered":"2024-08-17T17:26:14.673361+0000","last_clean":"2024-08-17T17:26:14.673361+0000","last_became_active":"2024-08-17T17:24:11.574846+0000","last_became_peered":"2024-08-17T17:24:11.574846+0000","last_unstale":"2024-08-17T17:26:14.673361+0000","last_undegraded":"2024-08-17T17:26:14.673361+0000","last_fullsized":"2024-08-17T17:26:14.673361+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:11.936236+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:11.936236+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:31.674851+0000","scrub_duration":0.0035797139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1238,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.149587+0000","last_change":"2024-08-17T17:21:06.430639+0000","last_active":"2024-08-17T17:26:15.149587+0000","last_peered":"2024-08-17T17:26:15.149587+0000","last_clean":"2024-08-17T17:26:15.149587+0000","last_became_active":"2024-08-17T17:21:06.430297+0000","last_became_peered":"2024-08-17T17:21:06.430297+0000","last_unstale":"2024-08-17T17:26:15.149587+0000","last_undegraded":"2024-08-17T17:26:15.149587+0000","last_fullsized":"2024-08-17T17:26:15.149587+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:57.794620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1042,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.675738+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:26:14.675738+0000","last_peered":"2024-08-17T17:26:14.675738+0000","last_clean":"2024-08-17T17:26:14.675738+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:26:14.675738+0000","last_undegraded":"2024-08-17T17:26:14.675738+0000","last_fullsized":"2024-08-17T17:26:14.675738+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":53,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284805+0000","last_change":"2024-08-17T17:22:48.370769+0000","last_active":"2024-08-17T17:26:15.284805+0000","last_peered":"2024-08-17T17:26:15.284805+0000","last_clean":"2024-08-17T17:26:15.284805+0000","last_became_active":"2024-08-17T17:22:48.370531+0000","last_became_peered":"2024-08-17T17:22:48.370531+0000","last_unstale":"2024-08-17T17:26:15.284805+0000","last_undegraded":"2024-08-17T17:26:15.284805+0000","last_fullsized":"2024-08-17T17:26:15.284805+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:02.943823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1609,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.149518+0000","last_change":"2024-08-17T17:24:12.519946+0000","last_active":"2024-08-17T17:26:15.149518+0000","last_peered":"2024-08-17T17:26:15.149518+0000","last_clean":"2024-08-17T17:26:15.149518+0000","last_became_active":"2024-08-17T17:24:11.569747+0000","last_became_peered":"2024-08-17T17:24:11.569747+0000","last_unstale":"2024-08-17T17:26:15.149518+0000","last_undegraded":"2024-08-17T17:26:15.149518+0000","last_fullsized":"2024-08-17T17:26:15.149518+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.519838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.519838+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:56.003993+0000","scrub_duration":0.0068224740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1652,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.901622+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:26:19.901622+0000","last_peered":"2024-08-17T17:26:19.901622+0000","last_clean":"2024-08-17T17:26:19.901622+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:26:19.901622+0000","last_undegraded":"2024-08-17T17:26:19.901622+0000","last_fullsized":"2024-08-17T17:26:19.901622+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1044,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.898215+0000","last_change":"2024-08-17T17:21:06.428684+0000","last_active":"2024-08-17T17:26:19.898215+0000","last_peered":"2024-08-17T17:26:19.898215+0000","last_clean":"2024-08-17T17:26:19.898215+0000","last_became_active":"2024-08-17T17:21:06.428178+0000","last_became_peered":"2024-08-17T17:21:06.428178+0000","last_unstale":"2024-08-17T17:26:19.898215+0000","last_undegraded":"2024-08-17T17:26:19.898215+0000","last_fullsized":"2024-08-17T17:26:19.898215+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:14.836119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":885,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284840+0000","last_change":"2024-08-17T17:21:06.443565+0000","last_active":"2024-08-17T17:26:15.284840+0000","last_peered":"2024-08-17T17:26:15.284840+0000","last_clean":"2024-08-17T17:26:15.284840+0000","last_became_active":"2024-08-17T17:21:06.443215+0000","last_became_peered":"2024-08-17T17:21:06.443215+0000","last_unstale":"2024-08-17T17:26:15.284840+0000","last_undegraded":"2024-08-17T17:26:15.284840+0000","last_fullsized":"2024-08-17T17:26:15.284840+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:47.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1551,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284429+0000","last_change":"2024-08-17T17:24:13.795227+0000","last_active":"2024-08-17T17:26:15.284429+0000","last_peered":"2024-08-17T17:26:15.284429+0000","last_clean":"2024-08-17T17:26:15.284429+0000","last_became_active":"2024-08-17T17:24:11.578352+0000","last_became_peered":"2024-08-17T17:24:11.578352+0000","last_unstale":"2024-08-17T17:26:15.284429+0000","last_undegraded":"2024-08-17T17:26:15.284429+0000","last_fullsized":"2024-08-17T17:26:15.284429+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:36:39.440808+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1230,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284644+0000","last_change":"2024-08-17T17:21:06.456978+0000","last_active":"2024-08-17T17:26:15.284644+0000","last_peered":"2024-08-17T17:26:15.284644+0000","last_clean":"2024-08-17T17:26:15.284644+0000","last_became_active":"2024-08-17T17:21:06.452265+0000","last_became_peered":"2024-08-17T17:21:06.452265+0000","last_unstale":"2024-08-17T17:26:15.284644+0000","last_undegraded":"2024-08-17T17:26:15.284644+0000","last_fullsized":"2024-08-17T17:26:15.284644+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:52.893746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1211,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.676408+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:26:14.676408+0000","last_peered":"2024-08-17T17:26:14.676408+0000","last_clean":"2024-08-17T17:26:14.676408+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:26:14.676408+0000","last_undegraded":"2024-08-17T17:26:14.676408+0000","last_fullsized":"2024-08-17T17:26:14.676408+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1120,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.898638+0000","last_change":"2024-08-17T17:21:06.435754+0000","last_active":"2024-08-17T17:26:19.898638+0000","last_peered":"2024-08-17T17:26:19.898638+0000","last_clean":"2024-08-17T17:26:19.898638+0000","last_became_active":"2024-08-17T17:21:06.435442+0000","last_became_peered":"2024-08-17T17:21:06.435442+0000","last_unstale":"2024-08-17T17:26:19.898638+0000","last_undegraded":"2024-08-17T17:26:19.898638+0000","last_fullsized":"2024-08-17T17:26:19.898638+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:34.849042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":814,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.673690+0000","last_change":"2024-08-17T17:24:55.041464+0000","last_active":"2024-08-17T17:26:14.673690+0000","last_peered":"2024-08-17T17:26:14.673690+0000","last_clean":"2024-08-17T17:26:14.673690+0000","last_became_active":"2024-08-17T17:24:31.800289+0000","last_became_peered":"2024-08-17T17:24:31.800289+0000","last_unstale":"2024-08-17T17:26:14.673690+0000","last_undegraded":"2024-08-17T17:26:14.673690+0000","last_fullsized":"2024-08-17T17:26:14.673690+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:55.041378+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:55.041378+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:43.974501+0000","scrub_duration":0.001623642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1204,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284879+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:26:15.284879+0000","last_peered":"2024-08-17T17:26:15.284879+0000","last_clean":"2024-08-17T17:26:15.284879+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:26:15.284879+0000","last_undegraded":"2024-08-17T17:26:15.284879+0000","last_fullsized":"2024-08-17T17:26:15.284879+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":895,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.149875+0000","last_change":"2024-08-17T17:21:06.434991+0000","last_active":"2024-08-17T17:26:15.149875+0000","last_peered":"2024-08-17T17:26:15.149875+0000","last_clean":"2024-08-17T17:26:15.149875+0000","last_became_active":"2024-08-17T17:21:06.434630+0000","last_became_peered":"2024-08-17T17:21:06.434630+0000","last_unstale":"2024-08-17T17:26:15.149875+0000","last_undegraded":"2024-08-17T17:26:15.149875+0000","last_fullsized":"2024-08-17T17:26:15.149875+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:24.104609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":837,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284560+0000","last_change":"2024-08-17T17:21:06.462532+0000","last_active":"2024-08-17T17:26:15.284560+0000","last_peered":"2024-08-17T17:26:15.284560+0000","last_clean":"2024-08-17T17:26:15.284560+0000","last_became_active":"2024-08-17T17:21:06.462183+0000","last_became_peered":"2024-08-17T17:21:06.462183+0000","last_unstale":"2024-08-17T17:26:15.284560+0000","last_undegraded":"2024-08-17T17:26:15.284560+0000","last_fullsized":"2024-08-17T17:26:15.284560+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:50.033322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":831,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.673683+0000","last_change":"2024-08-17T17:24:53.103270+0000","last_active":"2024-08-17T17:26:14.673683+0000","last_peered":"2024-08-17T17:26:14.673683+0000","last_clean":"2024-08-17T17:26:14.673683+0000","last_became_active":"2024-08-17T17:24:31.800823+0000","last_became_peered":"2024-08-17T17:24:31.800823+0000","last_unstale":"2024-08-17T17:26:14.673683+0000","last_undegraded":"2024-08-17T17:26:14.673683+0000","last_fullsized":"2024-08-17T17:26:14.673683+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.103133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.103133+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:52.444781+0000","scrub_duration":0.0074281759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1043,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.898496+0000","last_change":"2024-08-17T17:21:06.433720+0000","last_active":"2024-08-17T17:26:19.898496+0000","last_peered":"2024-08-17T17:26:19.898496+0000","last_clean":"2024-08-17T17:26:19.898496+0000","last_became_active":"2024-08-17T17:21:06.433292+0000","last_became_peered":"2024-08-17T17:21:06.433292+0000","last_unstale":"2024-08-17T17:26:19.898496+0000","last_undegraded":"2024-08-17T17:26:19.898496+0000","last_fullsized":"2024-08-17T17:26:19.898496+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:32.246594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1205,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.150058+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:26:15.150058+0000","last_peered":"2024-08-17T17:26:15.150058+0000","last_clean":"2024-08-17T17:26:15.150058+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:26:15.150058+0000","last_undegraded":"2024-08-17T17:26:15.150058+0000","last_fullsized":"2024-08-17T17:26:15.150058+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":797,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.285036+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:26:15.285036+0000","last_peered":"2024-08-17T17:26:15.285036+0000","last_clean":"2024-08-17T17:26:15.285036+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:26:15.285036+0000","last_undegraded":"2024-08-17T17:26:15.285036+0000","last_fullsized":"2024-08-17T17:26:15.285036+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1555,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.676385+0000","last_change":"2024-08-17T17:24:49.119578+0000","last_active":"2024-08-17T17:26:14.676385+0000","last_peered":"2024-08-17T17:26:14.676385+0000","last_clean":"2024-08-17T17:26:14.676385+0000","last_became_active":"2024-08-17T17:24:31.807987+0000","last_became_peered":"2024-08-17T17:24:31.807987+0000","last_unstale":"2024-08-17T17:26:14.676385+0000","last_undegraded":"2024-08-17T17:26:14.676385+0000","last_fullsized":"2024-08-17T17:26:14.676385+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:34.052786+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1610,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.712065+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:26:24.712065+0000","last_peered":"2024-08-17T17:26:24.712065+0000","last_clean":"2024-08-17T17:26:24.712065+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:26:24.712065+0000","last_undegraded":"2024-08-17T17:26:24.712065+0000","last_fullsized":"2024-08-17T17:26:24.712065+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":846,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.673742+0000","last_change":"2024-08-17T17:21:06.452395+0000","last_active":"2024-08-17T17:26:14.673742+0000","last_peered":"2024-08-17T17:26:14.673742+0000","last_clean":"2024-08-17T17:26:14.673742+0000","last_became_active":"2024-08-17T17:21:06.451867+0000","last_became_peered":"2024-08-17T17:21:06.451867+0000","last_unstale":"2024-08-17T17:26:14.673742+0000","last_undegraded":"2024-08-17T17:26:14.673742+0000","last_fullsized":"2024-08-17T17:26:14.673742+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:50.597092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1600,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711286+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:26:24.711286+0000","last_peered":"2024-08-17T17:26:24.711286+0000","last_clean":"2024-08-17T17:26:24.711286+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:26:24.711286+0000","last_undegraded":"2024-08-17T17:26:24.711286+0000","last_fullsized":"2024-08-17T17:26:24.711286+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1635,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711600+0000","last_change":"2024-08-17T17:24:12.033217+0000","last_active":"2024-08-17T17:26:24.711600+0000","last_peered":"2024-08-17T17:26:24.711600+0000","last_clean":"2024-08-17T17:26:24.711600+0000","last_became_active":"2024-08-17T17:24:11.572877+0000","last_became_peered":"2024-08-17T17:24:11.572877+0000","last_unstale":"2024-08-17T17:26:24.711600+0000","last_undegraded":"2024-08-17T17:26:24.711600+0000","last_fullsized":"2024-08-17T17:26:24.711600+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.033101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.033101+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.264475+0000","scrub_duration":20,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":215,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.676316+0000","last_change":"2024-08-17T17:21:06.519965+0000","last_active":"2024-08-17T17:26:14.676316+0000","last_peered":"2024-08-17T17:26:14.676316+0000","last_clean":"2024-08-17T17:26:14.676316+0000","last_became_active":"2024-08-17T17:21:06.459952+0000","last_became_peered":"2024-08-17T17:21:06.459952+0000","last_unstale":"2024-08-17T17:26:14.676316+0000","last_undegraded":"2024-08-17T17:26:14.676316+0000","last_fullsized":"2024-08-17T17:26:14.676316+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:27.828098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1201,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.673774+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:26:14.673774+0000","last_peered":"2024-08-17T17:26:14.673774+0000","last_clean":"2024-08-17T17:26:14.673774+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:26:14.673774+0000","last_undegraded":"2024-08-17T17:26:14.673774+0000","last_fullsized":"2024-08-17T17:26:14.673774+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1177,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.675740+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:26:14.675740+0000","last_peered":"2024-08-17T17:26:14.675740+0000","last_clean":"2024-08-17T17:26:14.675740+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:26:14.675740+0000","last_undegraded":"2024-08-17T17:26:14.675740+0000","last_fullsized":"2024-08-17T17:26:14.675740+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":700,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711776+0000","last_change":"2024-08-17T17:24:48.935651+0000","last_active":"2024-08-17T17:26:24.711776+0000","last_peered":"2024-08-17T17:26:24.711776+0000","last_clean":"2024-08-17T17:26:24.711776+0000","last_became_active":"2024-08-17T17:24:40.897753+0000","last_became_peered":"2024-08-17T17:24:40.897753+0000","last_unstale":"2024-08-17T17:26:24.711776+0000","last_undegraded":"2024-08-17T17:26:24.711776+0000","last_fullsized":"2024-08-17T17:26:24.711776+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:10:15.118428+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1208,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.676205+0000","last_change":"2024-08-17T17:21:06.458000+0000","last_active":"2024-08-17T17:26:14.676205+0000","last_peered":"2024-08-17T17:26:14.676205+0000","last_clean":"2024-08-17T17:26:14.676205+0000","last_became_active":"2024-08-17T17:21:06.457411+0000","last_became_peered":"2024-08-17T17:21:06.457411+0000","last_unstale":"2024-08-17T17:26:14.676205+0000","last_undegraded":"2024-08-17T17:26:14.676205+0000","last_fullsized":"2024-08-17T17:26:14.676205+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:52.740639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1200,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.900036+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:26:19.900036+0000","last_peered":"2024-08-17T17:26:19.900036+0000","last_clean":"2024-08-17T17:26:19.900036+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:26:19.900036+0000","last_undegraded":"2024-08-17T17:26:19.900036+0000","last_fullsized":"2024-08-17T17:26:19.900036+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":676,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284826+0000","last_change":"2024-08-17T17:25:03.663370+0000","last_active":"2024-08-17T17:26:15.284826+0000","last_peered":"2024-08-17T17:26:15.284826+0000","last_clean":"2024-08-17T17:26:15.284826+0000","last_became_active":"2024-08-17T17:24:40.886214+0000","last_became_peered":"2024-08-17T17:24:40.886214+0000","last_unstale":"2024-08-17T17:26:15.284826+0000","last_undegraded":"2024-08-17T17:26:15.284826+0000","last_fullsized":"2024-08-17T17:26:15.284826+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:11:39.272029+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":869,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.673327+0000","last_change":"2024-08-17T17:21:06.454587+0000","last_active":"2024-08-17T17:26:14.673327+0000","last_peered":"2024-08-17T17:26:14.673327+0000","last_clean":"2024-08-17T17:26:14.673327+0000","last_became_active":"2024-08-17T17:21:06.454229+0000","last_became_peered":"2024-08-17T17:21:06.454229+0000","last_unstale":"2024-08-17T17:26:14.673327+0000","last_undegraded":"2024-08-17T17:26:14.673327+0000","last_fullsized":"2024-08-17T17:26:14.673327+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:38.340263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1175,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.149942+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:26:15.149942+0000","last_peered":"2024-08-17T17:26:15.149942+0000","last_clean":"2024-08-17T17:26:15.149942+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:26:15.149942+0000","last_undegraded":"2024-08-17T17:26:15.149942+0000","last_fullsized":"2024-08-17T17:26:15.149942+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1635,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.898409+0000","last_change":"2024-08-17T17:24:20.649977+0000","last_active":"2024-08-17T17:26:19.898409+0000","last_peered":"2024-08-17T17:26:19.898409+0000","last_clean":"2024-08-17T17:26:19.898409+0000","last_became_active":"2024-08-17T17:24:20.649336+0000","last_became_peered":"2024-08-17T17:24:20.649336+0000","last_unstale":"2024-08-17T17:26:19.898409+0000","last_undegraded":"2024-08-17T17:26:19.898409+0000","last_fullsized":"2024-08-17T17:26:19.898409+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:57.187271+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1205,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284803+0000","last_change":"2024-08-17T17:21:06.442796+0000","last_active":"2024-08-17T17:26:15.284803+0000","last_peered":"2024-08-17T17:26:15.284803+0000","last_clean":"2024-08-17T17:26:15.284803+0000","last_became_active":"2024-08-17T17:21:06.442466+0000","last_became_peered":"2024-08-17T17:21:06.442466+0000","last_unstale":"2024-08-17T17:26:15.284803+0000","last_undegraded":"2024-08-17T17:26:15.284803+0000","last_fullsized":"2024-08-17T17:26:15.284803+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:36.366620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1556,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711417+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:26:24.711417+0000","last_peered":"2024-08-17T17:26:24.711417+0000","last_clean":"2024-08-17T17:26:24.711417+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:26:24.711417+0000","last_undegraded":"2024-08-17T17:26:24.711417+0000","last_fullsized":"2024-08-17T17:26:24.711417+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1423,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.150008+0000","last_change":"2024-08-17T17:25:07.472056+0000","last_active":"2024-08-17T17:26:15.150008+0000","last_peered":"2024-08-17T17:26:15.150008+0000","last_clean":"2024-08-17T17:26:15.150008+0000","last_became_active":"2024-08-17T17:24:40.883356+0000","last_became_peered":"2024-08-17T17:24:40.883356+0000","last_unstale":"2024-08-17T17:26:15.150008+0000","last_undegraded":"2024-08-17T17:26:15.150008+0000","last_fullsized":"2024-08-17T17:26:15.150008+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:43:27.883917+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1041,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284964+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:26:15.284964+0000","last_peered":"2024-08-17T17:26:15.284964+0000","last_clean":"2024-08-17T17:26:15.284964+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:26:15.284964+0000","last_undegraded":"2024-08-17T17:26:15.284964+0000","last_fullsized":"2024-08-17T17:26:15.284964+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1589,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711261+0000","last_change":"2024-08-17T17:21:06.468788+0000","last_active":"2024-08-17T17:26:24.711261+0000","last_peered":"2024-08-17T17:26:24.711261+0000","last_clean":"2024-08-17T17:26:24.711261+0000","last_became_active":"2024-08-17T17:21:06.467141+0000","last_became_peered":"2024-08-17T17:21:06.467141+0000","last_unstale":"2024-08-17T17:26:24.711261+0000","last_undegraded":"2024-08-17T17:26:24.711261+0000","last_fullsized":"2024-08-17T17:26:24.711261+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:51.301359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1423,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.675806+0000","last_change":"2024-08-17T17:24:53.130774+0000","last_active":"2024-08-17T17:26:14.675806+0000","last_peered":"2024-08-17T17:26:14.675806+0000","last_clean":"2024-08-17T17:26:14.675806+0000","last_became_active":"2024-08-17T17:24:40.883388+0000","last_became_peered":"2024-08-17T17:24:40.883388+0000","last_unstale":"2024-08-17T17:26:14.675806+0000","last_undegraded":"2024-08-17T17:26:14.675806+0000","last_fullsized":"2024-08-17T17:26:14.675806+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:36.772875+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1207,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.673734+0000","last_change":"2024-08-17T17:21:06.435998+0000","last_active":"2024-08-17T17:26:14.673734+0000","last_peered":"2024-08-17T17:26:14.673734+0000","last_clean":"2024-08-17T17:26:14.673734+0000","last_became_active":"2024-08-17T17:21:06.435579+0000","last_became_peered":"2024-08-17T17:21:06.435579+0000","last_unstale":"2024-08-17T17:26:14.673734+0000","last_undegraded":"2024-08-17T17:26:14.673734+0000","last_fullsized":"2024-08-17T17:26:14.673734+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:01.743074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1618,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711312+0000","last_change":"2024-08-17T17:21:06.473627+0000","last_active":"2024-08-17T17:26:24.711312+0000","last_peered":"2024-08-17T17:26:24.711312+0000","last_clean":"2024-08-17T17:26:24.711312+0000","last_became_active":"2024-08-17T17:21:06.473062+0000","last_became_peered":"2024-08-17T17:21:06.473062+0000","last_unstale":"2024-08-17T17:26:24.711312+0000","last_undegraded":"2024-08-17T17:26:24.711312+0000","last_fullsized":"2024-08-17T17:26:24.711312+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:21.413163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":783,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284430+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:26:15.284430+0000","last_peered":"2024-08-17T17:26:15.284430+0000","last_clean":"2024-08-17T17:26:15.284430+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:26:15.284430+0000","last_undegraded":"2024-08-17T17:26:15.284430+0000","last_fullsized":"2024-08-17T17:26:15.284430+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1002,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.150023+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:26:15.150023+0000","last_peered":"2024-08-17T17:26:15.150023+0000","last_clean":"2024-08-17T17:26:15.150023+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:26:15.150023+0000","last_undegraded":"2024-08-17T17:26:15.150023+0000","last_fullsized":"2024-08-17T17:26:15.150023+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1109,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711490+0000","last_change":"2024-08-17T17:21:06.471265+0000","last_active":"2024-08-17T17:26:24.711490+0000","last_peered":"2024-08-17T17:26:24.711490+0000","last_clean":"2024-08-17T17:26:24.711490+0000","last_became_active":"2024-08-17T17:21:06.470943+0000","last_became_peered":"2024-08-17T17:21:06.470943+0000","last_unstale":"2024-08-17T17:26:24.711490+0000","last_undegraded":"2024-08-17T17:26:24.711490+0000","last_fullsized":"2024-08-17T17:26:24.711490+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:41.070024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1555,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284533+0000","last_change":"2024-08-17T17:25:02.696608+0000","last_active":"2024-08-17T17:26:15.284533+0000","last_peered":"2024-08-17T17:26:15.284533+0000","last_clean":"2024-08-17T17:26:15.284533+0000","last_became_active":"2024-08-17T17:24:40.877715+0000","last_became_peered":"2024-08-17T17:24:40.877715+0000","last_unstale":"2024-08-17T17:26:15.284533+0000","last_undegraded":"2024-08-17T17:26:15.284533+0000","last_fullsized":"2024-08-17T17:26:15.284533+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:48:00.941285+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1595,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711149+0000","last_change":"2024-08-17T17:22:48.377351+0000","last_active":"2024-08-17T17:26:24.711149+0000","last_peered":"2024-08-17T17:26:24.711149+0000","last_clean":"2024-08-17T17:26:24.711149+0000","last_became_active":"2024-08-17T17:22:48.377059+0000","last_became_peered":"2024-08-17T17:22:48.377059+0000","last_unstale":"2024-08-17T17:26:24.711149+0000","last_undegraded":"2024-08-17T17:26:24.711149+0000","last_fullsized":"2024-08-17T17:26:24.711149+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:15.250902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1225,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.676118+0000","last_change":"2024-08-17T17:21:06.459139+0000","last_active":"2024-08-17T17:26:14.676118+0000","last_peered":"2024-08-17T17:26:14.676118+0000","last_clean":"2024-08-17T17:26:14.676118+0000","last_became_active":"2024-08-17T17:21:06.457850+0000","last_became_peered":"2024-08-17T17:21:06.457850+0000","last_unstale":"2024-08-17T17:26:14.676118+0000","last_undegraded":"2024-08-17T17:26:14.676118+0000","last_fullsized":"2024-08-17T17:26:14.676118+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:13.888576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1496,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711277+0000","last_change":"2024-08-17T17:24:49.911734+0000","last_active":"2024-08-17T17:26:24.711277+0000","last_peered":"2024-08-17T17:26:24.711277+0000","last_clean":"2024-08-17T17:26:24.711277+0000","last_became_active":"2024-08-17T17:24:40.899102+0000","last_became_peered":"2024-08-17T17:24:40.899102+0000","last_unstale":"2024-08-17T17:26:24.711277+0000","last_undegraded":"2024-08-17T17:26:24.711277+0000","last_fullsized":"2024-08-17T17:26:24.711277+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.911622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.911622+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:30:27.987033+0000","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1634,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.898569+0000","last_change":"2024-08-17T17:21:06.446321+0000","last_active":"2024-08-17T17:26:19.898569+0000","last_peered":"2024-08-17T17:26:19.898569+0000","last_clean":"2024-08-17T17:26:19.898569+0000","last_became_active":"2024-08-17T17:21:06.446019+0000","last_became_peered":"2024-08-17T17:21:06.446019+0000","last_unstale":"2024-08-17T17:26:19.898569+0000","last_undegraded":"2024-08-17T17:26:19.898569+0000","last_fullsized":"2024-08-17T17:26:19.898569+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:26.750070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":194,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.675992+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:26:14.675992+0000","last_peered":"2024-08-17T17:26:14.675992+0000","last_clean":"2024-08-17T17:26:14.675992+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:26:14.675992+0000","last_undegraded":"2024-08-17T17:26:14.675992+0000","last_fullsized":"2024-08-17T17:26:14.675992+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":700,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711964+0000","last_change":"2024-08-17T17:24:46.946320+0000","last_active":"2024-08-17T17:26:24.711964+0000","last_peered":"2024-08-17T17:26:24.711964+0000","last_clean":"2024-08-17T17:26:24.711964+0000","last_became_active":"2024-08-17T17:24:40.880380+0000","last_became_peered":"2024-08-17T17:24:40.880380+0000","last_unstale":"2024-08-17T17:26:24.711964+0000","last_undegraded":"2024-08-17T17:26:24.711964+0000","last_fullsized":"2024-08-17T17:26:24.711964+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:54:30.637162+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1223,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.676108+0000","last_change":"2024-08-17T17:21:06.465804+0000","last_active":"2024-08-17T17:26:14.676108+0000","last_peered":"2024-08-17T17:26:14.676108+0000","last_clean":"2024-08-17T17:26:14.676108+0000","last_became_active":"2024-08-17T17:21:06.465461+0000","last_became_peered":"2024-08-17T17:21:06.465461+0000","last_unstale":"2024-08-17T17:26:14.676108+0000","last_undegraded":"2024-08-17T17:26:14.676108+0000","last_fullsized":"2024-08-17T17:26:14.676108+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:06.443667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1215,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284710+0000","last_change":"2024-08-17T17:21:06.481634+0000","last_active":"2024-08-17T17:26:15.284710+0000","last_peered":"2024-08-17T17:26:15.284710+0000","last_clean":"2024-08-17T17:26:15.284710+0000","last_became_active":"2024-08-17T17:21:06.469081+0000","last_became_peered":"2024-08-17T17:21:06.469081+0000","last_unstale":"2024-08-17T17:26:15.284710+0000","last_undegraded":"2024-08-17T17:26:15.284710+0000","last_fullsized":"2024-08-17T17:26:15.284710+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:53:26.954738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1223,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.676011+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:26:14.676011+0000","last_peered":"2024-08-17T17:26:14.676011+0000","last_clean":"2024-08-17T17:26:14.676011+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:26:14.676011+0000","last_undegraded":"2024-08-17T17:26:14.676011+0000","last_fullsized":"2024-08-17T17:26:14.676011+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1174,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284869+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:26:15.284869+0000","last_peered":"2024-08-17T17:26:15.284869+0000","last_clean":"2024-08-17T17:26:15.284869+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:26:15.284869+0000","last_undegraded":"2024-08-17T17:26:15.284869+0000","last_fullsized":"2024-08-17T17:26:15.284869+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1614,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.149751+0000","last_change":"2024-08-17T17:25:02.417545+0000","last_active":"2024-08-17T17:26:15.149751+0000","last_peered":"2024-08-17T17:26:15.149751+0000","last_clean":"2024-08-17T17:26:15.149751+0000","last_became_active":"2024-08-17T17:24:40.883220+0000","last_became_peered":"2024-08-17T17:24:40.883220+0000","last_unstale":"2024-08-17T17:26:15.149751+0000","last_undegraded":"2024-08-17T17:26:15.149751+0000","last_fullsized":"2024-08-17T17:26:15.149751+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:28.257805+0000","scrub_duration":0.0084952199999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1594,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711587+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:26:24.711587+0000","last_peered":"2024-08-17T17:26:24.711587+0000","last_clean":"2024-08-17T17:26:24.711587+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:26:24.711587+0000","last_undegraded":"2024-08-17T17:26:24.711587+0000","last_fullsized":"2024-08-17T17:26:24.711587+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1570,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.712026+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:26:24.712026+0000","last_peered":"2024-08-17T17:26:24.712026+0000","last_clean":"2024-08-17T17:26:24.712026+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:26:24.712026+0000","last_undegraded":"2024-08-17T17:26:24.712026+0000","last_fullsized":"2024-08-17T17:26:24.712026+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1444,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.711725+0000","last_change":"2024-08-17T17:24:47.961645+0000","last_active":"2024-08-17T17:26:24.711725+0000","last_peered":"2024-08-17T17:26:24.711725+0000","last_clean":"2024-08-17T17:26:24.711725+0000","last_became_active":"2024-08-17T17:24:40.897539+0000","last_became_peered":"2024-08-17T17:24:40.897539+0000","last_unstale":"2024-08-17T17:26:24.711725+0000","last_undegraded":"2024-08-17T17:26:24.711725+0000","last_fullsized":"2024-08-17T17:26:24.711725+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:47.961445+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:47.961445+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:04:32.031483+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1208,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:14.675728+0000","last_change":"2024-08-17T17:21:06.483469+0000","last_active":"2024-08-17T17:26:14.675728+0000","last_peered":"2024-08-17T17:26:14.675728+0000","last_clean":"2024-08-17T17:26:14.675728+0000","last_became_active":"2024-08-17T17:21:06.483162+0000","last_became_peered":"2024-08-17T17:21:06.483162+0000","last_unstale":"2024-08-17T17:26:14.675728+0000","last_undegraded":"2024-08-17T17:26:14.675728+0000","last_fullsized":"2024-08-17T17:26:14.675728+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:24.898353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":217,"reported_epoch":1244,"state":"active+clean","last_fresh":"2024-08-17T17:26:15.284929+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:26:15.284929+0000","last_peered":"2024-08-17T17:26:15.284929+0000","last_clean":"2024-08-17T17:26:15.284929+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:26:15.284929+0000","last_undegraded":"2024-08-17T17:26:15.284929+0000","last_fullsized":"2024-08-17T17:26:15.284929+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201699,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":444652,"kb_used_data":8352,"kb_used_omap":1,"kb_used_meta":436286,"kb_avail":93300500,"statfs":{"total":95995035648,"available":95539712000,"internally_reserved":0,"allocated":8552448,"data_stored":5988344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":446757323},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463336,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":378272,"kb_used_data":8404,"kb_used_omap":1,"kb_used_meta":369854,"kb_avail":93366880,"statfs":{"total":95995035648,"available":95607685120,"internally_reserved":0,"allocated":8605696,"data_stored":6053209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":378730953},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626978,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":618076,"kb_used_data":8400,"kb_used_omap":42,"kb_used_meta":609621,"kb_avail":93127076,"statfs":{"total":95995035648,"available":95362125824,"internally_reserved":0,"allocated":8601600,"data_stored":6056361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":624251964},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785455,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":437392,"kb_used_data":8068,"kb_used_omap":48,"kb_used_meta":429263,"kb_avail":93307760,"statfs":{"total":95995035648,"available":95547146240,"internally_reserved":0,"allocated":8261632,"data_stored":5720715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":439565998},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724972,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":458588,"kb_used_data":8336,"kb_used_omap":1,"kb_used_meta":450238,"kb_avail":93286564,"statfs":{"total":95995035648,"available":95525441536,"internally_reserved":0,"allocated":8536064,"data_stored":5984043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":461044166},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":20,"apply_latency_ms":20,"commit_latency_ns":20000000,"apply_latency_ns":20000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986609,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":432100,"kb_used_data":8088,"kb_used_omap":1,"kb_used_meta":423998,"kb_avail":93313052,"statfs":{"total":95995035648,"available":95552565248,"internally_reserved":0,"allocated":8282112,"data_stored":5725586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":434174406},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706216,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":469336,"kb_used_data":8332,"kb_used_omap":35,"kb_used_meta":460956,"kb_avail":93275816,"statfs":{"total":95995035648,"available":95514435584,"internally_reserved":0,"allocated":8531968,"data_stored":5984000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":472018999},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831989,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":568404,"kb_used_data":8392,"kb_used_omap":65,"kb_used_meta":559934,"kb_avail":93176748,"statfs":{"total":95995035648,"available":95412989952,"internally_reserved":0,"allocated":8593408,"data_stored":6048980,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":573372603},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:26:29.702 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:26:29.769 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:29.793 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:29.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:29.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:29 smithi162 ceph-mon[104668]: from='client.35544 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:29.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:29 smithi162 ceph-mon[104668]: pgmap v1683: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:29.895 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:29.918 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:29.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:29.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:30.020 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:30.042 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:30.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:30.143 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:30.169 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:30.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:30.271 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:30.292 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:26:30.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.297 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:30.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:30.398 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:30.422 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:30.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:30.524 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:30.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.547 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:30.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:30.620 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:30.648 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:30.673 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:30.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:30.775 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:30.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.798 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:30.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:30.899 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:30.923 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:30.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:30.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:31.025 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:31.047 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:31.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:31.149 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:31.171 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:31.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:31.273 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:31.300 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:31.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:31.401 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:31.424 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:31.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:31.525 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:31.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:31 smithi063 ceph-mon[114756]: pgmap v1684: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:31.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:31.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:31.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:31 smithi063 ceph-mon[109843]: pgmap v1684: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:31.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:31.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:31.551 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:31.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:31.653 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:31.738 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:31.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:31.840 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:31.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:31 smithi162 ceph-mon[104668]: pgmap v1684: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:31.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:31.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:31.864 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:31.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:31.966 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:31.992 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:31.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:31.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:32.093 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:32.116 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:32.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:32.217 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:32.239 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:32.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:32.341 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:32.366 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:32.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:32.468 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:32.492 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:32.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:32.521 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:32.594 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:32.618 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:32.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:32.640 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:26:32.719 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:32.728 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:26:32.728 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:26:32.745 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:32.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.746 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:32.847 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:32.871 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:32.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:32.973 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:32.996 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:32.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:32.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:33.098 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:33.123 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:33.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:33.225 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:33.247 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:33.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:33.349 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:33.375 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:33.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:33.408 INFO:teuthology.orchestra.run.smithi063.stderr:pool 'unique_pool_4' created 2024-08-17T17:26:33.449 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:26:33.476 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:33.516 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:33.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:33.612 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:33.618 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:33.644 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:33.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:33.647 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:33 smithi162 ceph-mon[104668]: pgmap v1685: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:33.647 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:33 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1145763906' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T17:26:33.647 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:33 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T17:26:33.712 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:26:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:26:33] "GET /metrics HTTP/1.1" 200 34040 "" "Prometheus/2.51.0" 2024-08-17T17:26:33.713 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:33 smithi063 ceph-mon[109843]: pgmap v1685: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:33.713 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:33 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1145763906' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T17:26:33.713 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:33 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T17:26:33.713 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:33 smithi063 ceph-mon[114756]: pgmap v1685: 103 pgs: 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:33.713 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:33 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1145763906' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T17:26:33.714 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:33 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-08-17T17:26:33.746 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:33.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.772 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:33.772 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:33.873 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:33.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:33.897 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:33.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:33.998 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:34.021 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:34.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:34.041 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2024-08-17T17:26:34.123 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:34.145 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:34.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:34.247 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:34.272 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:34.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:34.374 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:34.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.397 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:34.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:34.498 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:34.522 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:34.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:34.623 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:34.646 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:34.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:34.661 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:34 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-08-17T17:26:34.661 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:34 smithi162 ceph-mon[104668]: osdmap e1247: 8 total, 8 up, 8 in 2024-08-17T17:26:34.748 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:34.770 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:34.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:34.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:34 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-08-17T17:26:34.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:34 smithi063 ceph-mon[114756]: osdmap e1247: 8 total, 8 up, 8 in 2024-08-17T17:26:34.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:34 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-08-17T17:26:34.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:34 smithi063 ceph-mon[109843]: osdmap e1247: 8 total, 8 up, 8 in 2024-08-17T17:26:34.872 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:34.894 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:34.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:34.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:34.999 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:35.022 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:35.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:35.124 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:35.147 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:35.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:35.249 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:35.270 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:35.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:35.372 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:35.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.395 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:35.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:35.472 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:35 smithi063 ceph-mon[114756]: pgmap v1687: 119 pgs: 16 unknown, 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:35.473 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:35 smithi063 ceph-mon[114756]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:26:35.473 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:35 smithi063 ceph-mon[114756]: osdmap e1248: 8 total, 8 up, 8 in 2024-08-17T17:26:35.473 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:35 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:35.473 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:35 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:35.473 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:35 smithi063 ceph-mon[109843]: pgmap v1687: 119 pgs: 16 unknown, 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:35.473 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:35 smithi063 ceph-mon[109843]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:26:35.473 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:35 smithi063 ceph-mon[109843]: osdmap e1248: 8 total, 8 up, 8 in 2024-08-17T17:26:35.473 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:35 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:35.473 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:35 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:35.496 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:35.518 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:35.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:35.620 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:35.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.642 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:35.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:35.743 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:35.764 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:35.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:35.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:35 smithi162 ceph-mon[104668]: pgmap v1687: 119 pgs: 16 unknown, 103 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:35.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:35 smithi162 ceph-mon[104668]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:26:35.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:35 smithi162 ceph-mon[104668]: osdmap e1248: 8 total, 8 up, 8 in 2024-08-17T17:26:35.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:35 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:35.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:35 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:35.866 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:35.888 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:35.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:35.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:35.990 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:36.012 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:36.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:36.114 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:36.135 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:36.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:36.237 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:36.260 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:36.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:36.362 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:36.384 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:36.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:36.486 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:36.508 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:36.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:36.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:36 smithi063 ceph-mon[114756]: osdmap e1249: 8 total, 8 up, 8 in 2024-08-17T17:26:36.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:36 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:26:36.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:36 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:26:36.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:36 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:36.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:36 smithi063 ceph-mon[109843]: osdmap e1249: 8 total, 8 up, 8 in 2024-08-17T17:26:36.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:36 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:26:36.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:36 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:26:36.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:36 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:36.611 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:36.634 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:36.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:36.736 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:36.759 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:36.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:36 smithi162 ceph-mon[104668]: osdmap e1249: 8 total, 8 up, 8 in 2024-08-17T17:26:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:36 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:26:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:36 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:26:36.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:36 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:26:36.860 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:36.886 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:36.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:36.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:36.988 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:37.011 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:37.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:37.113 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:37.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.136 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:37.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:37.237 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:37.259 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:37.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:37.361 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:37.384 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:37.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:37.486 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:37.508 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:37.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:37.610 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:37.632 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:37.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:37.692 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:37 smithi063 ceph-mon[109843]: pgmap v1690: 119 pgs: 12 unknown, 107 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:37.692 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:37 smithi063 ceph-mon[114756]: pgmap v1690: 119 pgs: 12 unknown, 107 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:37.713 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:37.713 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:37.714 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:37.734 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:37.758 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:37.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:37.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:37 smithi162 ceph-mon[104668]: pgmap v1690: 119 pgs: 12 unknown, 107 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:37.864 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:37.888 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:37.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:37.889 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:37.990 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:38.014 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:38.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:38.105 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:26:38.107 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:26:38.117 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:38.142 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:38.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:38.244 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:38.269 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:38.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:38.370 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:38.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.400 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:38.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:38.501 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:38.523 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:38.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:38.625 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:38.648 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:38.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:38.750 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:38.772 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:38.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:38.874 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:38.900 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:38.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:38.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:39.002 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:39.028 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:39.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:39.130 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:39.153 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:39.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:39.255 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:39.281 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:39.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:39.382 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:39.404 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:39.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:39.506 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:39.526 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:39 smithi063 ceph-mon[114756]: from='client.45494 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:39.526 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:39 smithi063 ceph-mon[114756]: pgmap v1691: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:39.527 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:39 smithi063 ceph-mon[109843]: from='client.45494 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:39.527 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:39 smithi063 ceph-mon[109843]: pgmap v1691: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:39.530 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:39.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:39.632 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:39.654 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:39.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:39.756 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:39.782 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:39.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:39 smithi162 ceph-mon[104668]: from='client.45494 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:26:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:39 smithi162 ceph-mon[104668]: pgmap v1691: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:39.885 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:39.907 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:39.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:39.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:40.009 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:40.035 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:40.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:40.137 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:40.162 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:40.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:40.264 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:40.289 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:40.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:40.392 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:40.418 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:40.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:40.520 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:40.545 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:40.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:40.648 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:40.673 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:40.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:40.775 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:40.801 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:40.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:40.903 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:40.927 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:40.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:40.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:41.029 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:41.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.055 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:41.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:41.156 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:41.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.181 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:41.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:41.283 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:41.305 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:41.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:41.407 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:41.430 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:41.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:41.472 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1690,"stamp":"2024-08-17T17:26:36.360232+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1246,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4678,"ondisk_log_size":4678,"up":321,"acting":321,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":317,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3806928,"kb_used_data":66480,"kb_used_omap":199,"kb_used_meta":3740152,"kb_avail":746154288,"statfs":{"total":767960285184,"available":764061990912,"internally_reserved":0,"allocated":68075520,"data_stored":47633409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":3829916412},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001265"},"pg_stats":[{"pgid":"1.11","version":"369'149","reported_seq":1569,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:25.173100+0000","last_change":"2024-08-17T17:24:49.721626+0000","last_active":"2024-08-17T17:26:25.173100+0000","last_peered":"2024-08-17T17:26:25.173100+0000","last_clean":"2024-08-17T17:26:25.173100+0000","last_became_active":"2024-08-17T17:24:20.649128+0000","last_became_peered":"2024-08-17T17:24:20.649128+0000","last_unstale":"2024-08-17T17:26:25.173100+0000","last_undegraded":"2024-08-17T17:26:25.173100+0000","last_fullsized":"2024-08-17T17:26:25.173100+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:03:15.343059+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1593,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.405991+0000","last_change":"2024-08-17T17:21:06.463828+0000","last_active":"2024-08-17T17:26:34.405991+0000","last_peered":"2024-08-17T17:26:34.405991+0000","last_clean":"2024-08-17T17:26:34.405991+0000","last_became_active":"2024-08-17T17:21:06.463464+0000","last_became_peered":"2024-08-17T17:21:06.463464+0000","last_unstale":"2024-08-17T17:26:34.405991+0000","last_undegraded":"2024-08-17T17:26:34.405991+0000","last_fullsized":"2024-08-17T17:26:34.405991+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:46.465006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1176,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:22.417414+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:26:22.417414+0000","last_peered":"2024-08-17T17:26:22.417414+0000","last_clean":"2024-08-17T17:26:22.417414+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:26:22.417414+0000","last_undegraded":"2024-08-17T17:26:22.417414+0000","last_fullsized":"2024-08-17T17:26:22.417414+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1499,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.898614+0000","last_change":"2024-08-17T17:24:48.765312+0000","last_active":"2024-08-17T17:26:19.898614+0000","last_peered":"2024-08-17T17:26:19.898614+0000","last_clean":"2024-08-17T17:26:19.898614+0000","last_became_active":"2024-08-17T17:24:20.642731+0000","last_became_peered":"2024-08-17T17:24:20.642731+0000","last_unstale":"2024-08-17T17:26:19.898614+0000","last_undegraded":"2024-08-17T17:26:19.898614+0000","last_fullsized":"2024-08-17T17:26:19.898614+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:08:36.138960+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1187,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.904141+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:26:19.904141+0000","last_peered":"2024-08-17T17:26:19.904141+0000","last_clean":"2024-08-17T17:26:19.904141+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:26:19.904141+0000","last_undegraded":"2024-08-17T17:26:19.904141+0000","last_fullsized":"2024-08-17T17:26:19.904141+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1042,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.850232+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:26:19.850232+0000","last_peered":"2024-08-17T17:26:19.850232+0000","last_clean":"2024-08-17T17:26:19.850232+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:26:19.850232+0000","last_undegraded":"2024-08-17T17:26:19.850232+0000","last_fullsized":"2024-08-17T17:26:19.850232+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":806,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.405158+0000","last_change":"2024-08-17T17:24:50.880046+0000","last_active":"2024-08-17T17:26:34.405158+0000","last_peered":"2024-08-17T17:26:34.405158+0000","last_clean":"2024-08-17T17:26:34.405158+0000","last_became_active":"2024-08-17T17:24:20.652165+0000","last_became_peered":"2024-08-17T17:24:20.652165+0000","last_unstale":"2024-08-17T17:26:34.405158+0000","last_undegraded":"2024-08-17T17:26:34.405158+0000","last_fullsized":"2024-08-17T17:26:34.405158+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:00.416810+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1144,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.405891+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:26:34.405891+0000","last_peered":"2024-08-17T17:26:34.405891+0000","last_clean":"2024-08-17T17:26:34.405891+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:26:34.405891+0000","last_undegraded":"2024-08-17T17:26:34.405891+0000","last_fullsized":"2024-08-17T17:26:34.405891+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1159,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.850071+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:26:19.850071+0000","last_peered":"2024-08-17T17:26:19.850071+0000","last_clean":"2024-08-17T17:26:19.850071+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:26:19.850071+0000","last_undegraded":"2024-08-17T17:26:19.850071+0000","last_fullsized":"2024-08-17T17:26:19.850071+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1504,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.902358+0000","last_change":"2024-08-17T17:24:50.727183+0000","last_active":"2024-08-17T17:26:19.902358+0000","last_peered":"2024-08-17T17:26:19.902358+0000","last_clean":"2024-08-17T17:26:19.902358+0000","last_became_active":"2024-08-17T17:24:20.646409+0000","last_became_peered":"2024-08-17T17:24:20.646409+0000","last_unstale":"2024-08-17T17:26:19.902358+0000","last_undegraded":"2024-08-17T17:26:19.902358+0000","last_fullsized":"2024-08-17T17:26:19.902358+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.086193+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1592,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.405303+0000","last_change":"2024-08-17T17:21:06.442598+0000","last_active":"2024-08-17T17:26:34.405303+0000","last_peered":"2024-08-17T17:26:34.405303+0000","last_clean":"2024-08-17T17:26:34.405303+0000","last_became_active":"2024-08-17T17:21:06.442264+0000","last_became_peered":"2024-08-17T17:21:06.442264+0000","last_unstale":"2024-08-17T17:26:34.405303+0000","last_undegraded":"2024-08-17T17:26:34.405303+0000","last_fullsized":"2024-08-17T17:26:34.405303+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:35.003620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1236,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.375020+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:26:19.375020+0000","last_peered":"2024-08-17T17:26:19.375020+0000","last_clean":"2024-08-17T17:26:19.375020+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:26:19.375020+0000","last_undegraded":"2024-08-17T17:26:19.375020+0000","last_fullsized":"2024-08-17T17:26:19.375020+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:26:33.391667+0000","last_change":"2024-08-17T17:26:33.391667+0000","last_active":"2024-08-17T17:26:33.391667+0000","last_peered":"2024-08-17T17:26:33.391667+0000","last_clean":"2024-08-17T17:26:33.391667+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:26:33.391667+0000","last_undegraded":"2024-08-17T17:26:33.391667+0000","last_fullsized":"2024-08-17T17:26:33.391667+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1443,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:25.076696+0000","last_change":"2024-08-17T17:24:50.410621+0000","last_active":"2024-08-17T17:26:25.076696+0000","last_peered":"2024-08-17T17:26:25.076696+0000","last_clean":"2024-08-17T17:26:25.076696+0000","last_became_active":"2024-08-17T17:24:31.863062+0000","last_became_peered":"2024-08-17T17:24:31.863062+0000","last_unstale":"2024-08-17T17:26:25.076696+0000","last_undegraded":"2024-08-17T17:26:25.076696+0000","last_fullsized":"2024-08-17T17:26:25.076696+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.410491+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.410491+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:11.717684+0000","scrub_duration":0.0066315480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1201,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.850488+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:26:19.850488+0000","last_peered":"2024-08-17T17:26:19.850488+0000","last_clean":"2024-08-17T17:26:19.850488+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:26:19.850488+0000","last_undegraded":"2024-08-17T17:26:19.850488+0000","last_fullsized":"2024-08-17T17:26:19.850488+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1174,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.906526+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:26:19.906526+0000","last_peered":"2024-08-17T17:26:19.906526+0000","last_clean":"2024-08-17T17:26:19.906526+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:26:19.906526+0000","last_undegraded":"2024-08-17T17:26:19.906526+0000","last_fullsized":"2024-08-17T17:26:19.906526+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:26:33.391667+0000","last_change":"2024-08-17T17:26:33.391667+0000","last_active":"2024-08-17T17:26:33.391667+0000","last_peered":"2024-08-17T17:26:33.391667+0000","last_clean":"2024-08-17T17:26:33.391667+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:26:33.391667+0000","last_undegraded":"2024-08-17T17:26:33.391667+0000","last_fullsized":"2024-08-17T17:26:33.391667+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1675,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.405829+0000","last_change":"2024-08-17T17:24:45.985725+0000","last_active":"2024-08-17T17:26:34.405829+0000","last_peered":"2024-08-17T17:26:34.405829+0000","last_clean":"2024-08-17T17:26:34.405829+0000","last_became_active":"2024-08-17T17:24:31.795836+0000","last_became_peered":"2024-08-17T17:24:31.795836+0000","last_unstale":"2024-08-17T17:26:34.405829+0000","last_undegraded":"2024-08-17T17:26:34.405829+0000","last_fullsized":"2024-08-17T17:26:34.405829+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:45.985596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:45.985596+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:17.897808+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1202,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.850787+0000","last_change":"2024-08-17T17:21:06.540689+0000","last_active":"2024-08-17T17:26:19.850787+0000","last_peered":"2024-08-17T17:26:19.850787+0000","last_clean":"2024-08-17T17:26:19.850787+0000","last_became_active":"2024-08-17T17:21:06.461614+0000","last_became_peered":"2024-08-17T17:21:06.461614+0000","last_unstale":"2024-08-17T17:26:19.850787+0000","last_undegraded":"2024-08-17T17:26:19.850787+0000","last_fullsized":"2024-08-17T17:26:19.850787+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:11.696898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1226,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.898874+0000","last_change":"2024-08-17T17:21:06.470389+0000","last_active":"2024-08-17T17:26:19.898874+0000","last_peered":"2024-08-17T17:26:19.898874+0000","last_clean":"2024-08-17T17:26:19.898874+0000","last_became_active":"2024-08-17T17:21:06.469966+0000","last_became_peered":"2024-08-17T17:21:06.469966+0000","last_unstale":"2024-08-17T17:26:19.898874+0000","last_undegraded":"2024-08-17T17:26:19.898874+0000","last_fullsized":"2024-08-17T17:26:19.898874+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:56.140616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:26:33.391667+0000","last_change":"2024-08-17T17:26:33.391667+0000","last_active":"2024-08-17T17:26:33.391667+0000","last_peered":"2024-08-17T17:26:33.391667+0000","last_clean":"2024-08-17T17:26:33.391667+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:26:33.391667+0000","last_undegraded":"2024-08-17T17:26:33.391667+0000","last_fullsized":"2024-08-17T17:26:33.391667+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":811,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.390627+0000","last_change":"2024-08-17T17:24:48.192447+0000","last_active":"2024-08-17T17:26:24.390627+0000","last_peered":"2024-08-17T17:26:24.390627+0000","last_clean":"2024-08-17T17:26:24.390627+0000","last_became_active":"2024-08-17T17:24:20.644240+0000","last_became_peered":"2024-08-17T17:24:20.644240+0000","last_unstale":"2024-08-17T17:26:24.390627+0000","last_undegraded":"2024-08-17T17:26:24.390627+0000","last_fullsized":"2024-08-17T17:26:24.390627+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:56.795337+0000","scrub_duration":0.010920255,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1160,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.851107+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:26:19.851107+0000","last_peered":"2024-08-17T17:26:19.851107+0000","last_clean":"2024-08-17T17:26:19.851107+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:26:19.851107+0000","last_undegraded":"2024-08-17T17:26:19.851107+0000","last_fullsized":"2024-08-17T17:26:19.851107+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1208,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.375083+0000","last_change":"2024-08-17T17:21:06.446798+0000","last_active":"2024-08-17T17:26:19.375083+0000","last_peered":"2024-08-17T17:26:19.375083+0000","last_clean":"2024-08-17T17:26:19.375083+0000","last_became_active":"2024-08-17T17:21:06.446205+0000","last_became_peered":"2024-08-17T17:21:06.446205+0000","last_unstale":"2024-08-17T17:26:19.375083+0000","last_undegraded":"2024-08-17T17:26:19.375083+0000","last_fullsized":"2024-08-17T17:26:19.375083+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:28.584048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:26:33.391667+0000","last_change":"2024-08-17T17:26:33.391667+0000","last_active":"2024-08-17T17:26:33.391667+0000","last_peered":"2024-08-17T17:26:33.391667+0000","last_clean":"2024-08-17T17:26:33.391667+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:26:33.391667+0000","last_undegraded":"2024-08-17T17:26:33.391667+0000","last_fullsized":"2024-08-17T17:26:33.391667+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":719,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.685654+0000","last_change":"2024-08-17T17:24:52.102415+0000","last_active":"2024-08-17T17:26:24.685654+0000","last_peered":"2024-08-17T17:26:24.685654+0000","last_clean":"2024-08-17T17:26:24.685654+0000","last_became_active":"2024-08-17T17:24:31.795843+0000","last_became_peered":"2024-08-17T17:24:31.795843+0000","last_unstale":"2024-08-17T17:26:24.685654+0000","last_undegraded":"2024-08-17T17:26:24.685654+0000","last_fullsized":"2024-08-17T17:26:24.685654+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:32:08.434197+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1189,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.905491+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:26:19.905491+0000","last_peered":"2024-08-17T17:26:19.905491+0000","last_clean":"2024-08-17T17:26:19.905491+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:26:19.905491+0000","last_undegraded":"2024-08-17T17:26:19.905491+0000","last_fullsized":"2024-08-17T17:26:19.905491+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":971,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.898877+0000","last_change":"2024-08-17T17:21:06.460905+0000","last_active":"2024-08-17T17:26:19.898877+0000","last_peered":"2024-08-17T17:26:19.898877+0000","last_clean":"2024-08-17T17:26:19.898877+0000","last_became_active":"2024-08-17T17:21:06.460592+0000","last_became_peered":"2024-08-17T17:21:06.460592+0000","last_unstale":"2024-08-17T17:26:19.898877+0000","last_undegraded":"2024-08-17T17:26:19.898877+0000","last_fullsized":"2024-08-17T17:26:19.898877+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:10.146916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":11,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.410885+0000","last_change":"2024-08-17T17:26:34.410513+0000","last_active":"2024-08-17T17:26:34.410885+0000","last_peered":"2024-08-17T17:26:34.410885+0000","last_clean":"2024-08-17T17:26:34.410885+0000","last_became_active":"2024-08-17T17:26:34.410199+0000","last_became_peered":"2024-08-17T17:26:34.410199+0000","last_unstale":"2024-08-17T17:26:34.410885+0000","last_undegraded":"2024-08-17T17:26:34.410885+0000","last_fullsized":"2024-08-17T17:26:34.410885+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:27:24.621284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1589,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.849956+0000","last_change":"2024-08-17T17:24:54.418765+0000","last_active":"2024-08-17T17:26:19.849956+0000","last_peered":"2024-08-17T17:26:19.849956+0000","last_clean":"2024-08-17T17:26:19.849956+0000","last_became_active":"2024-08-17T17:24:31.800263+0000","last_became_peered":"2024-08-17T17:24:31.800263+0000","last_unstale":"2024-08-17T17:26:19.849956+0000","last_undegraded":"2024-08-17T17:26:19.849956+0000","last_fullsized":"2024-08-17T17:26:19.849956+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:54.418644+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:54.418644+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:11.024294+0000","scrub_duration":0.013346135,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1614,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.405315+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:26:34.405315+0000","last_peered":"2024-08-17T17:26:34.405315+0000","last_clean":"2024-08-17T17:26:34.405315+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:26:34.405315+0000","last_undegraded":"2024-08-17T17:26:34.405315+0000","last_fullsized":"2024-08-17T17:26:34.405315+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1192,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.899884+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:26:19.899884+0000","last_peered":"2024-08-17T17:26:19.899884+0000","last_clean":"2024-08-17T17:26:19.899884+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:26:19.899884+0000","last_undegraded":"2024-08-17T17:26:19.899884+0000","last_fullsized":"2024-08-17T17:26:19.899884+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:26:33.391667+0000","last_change":"2024-08-17T17:26:33.391667+0000","last_active":"2024-08-17T17:26:33.391667+0000","last_peered":"2024-08-17T17:26:33.391667+0000","last_clean":"2024-08-17T17:26:33.391667+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:26:33.391667+0000","last_undegraded":"2024-08-17T17:26:33.391667+0000","last_fullsized":"2024-08-17T17:26:33.391667+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1535,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.906334+0000","last_change":"2024-08-17T17:24:58.712331+0000","last_active":"2024-08-17T17:26:19.906334+0000","last_peered":"2024-08-17T17:26:19.906334+0000","last_clean":"2024-08-17T17:26:19.906334+0000","last_became_active":"2024-08-17T17:24:31.802584+0000","last_became_peered":"2024-08-17T17:24:31.802584+0000","last_unstale":"2024-08-17T17:26:19.906334+0000","last_undegraded":"2024-08-17T17:26:19.906334+0000","last_fullsized":"2024-08-17T17:26:19.906334+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:59.616217+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1209,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.375168+0000","last_change":"2024-08-17T17:21:06.446841+0000","last_active":"2024-08-17T17:26:19.375168+0000","last_peered":"2024-08-17T17:26:19.375168+0000","last_clean":"2024-08-17T17:26:19.375168+0000","last_became_active":"2024-08-17T17:21:06.446271+0000","last_became_peered":"2024-08-17T17:21:06.446271+0000","last_unstale":"2024-08-17T17:26:19.375168+0000","last_undegraded":"2024-08-17T17:26:19.375168+0000","last_fullsized":"2024-08-17T17:26:19.375168+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:20.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1225,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.899705+0000","last_change":"2024-08-17T17:21:06.480911+0000","last_active":"2024-08-17T17:26:19.899705+0000","last_peered":"2024-08-17T17:26:19.899705+0000","last_clean":"2024-08-17T17:26:19.899705+0000","last_became_active":"2024-08-17T17:21:06.480551+0000","last_became_peered":"2024-08-17T17:21:06.480551+0000","last_unstale":"2024-08-17T17:26:19.899705+0000","last_undegraded":"2024-08-17T17:26:19.899705+0000","last_fullsized":"2024-08-17T17:26:19.899705+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:26.003704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:26:33.391667+0000","last_change":"2024-08-17T17:26:33.391667+0000","last_active":"2024-08-17T17:26:33.391667+0000","last_peered":"2024-08-17T17:26:33.391667+0000","last_clean":"2024-08-17T17:26:33.391667+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:26:33.391667+0000","last_undegraded":"2024-08-17T17:26:33.391667+0000","last_fullsized":"2024-08-17T17:26:33.391667+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":768,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:25.173210+0000","last_change":"2024-08-17T17:24:57.682573+0000","last_active":"2024-08-17T17:26:25.173210+0000","last_peered":"2024-08-17T17:26:25.173210+0000","last_clean":"2024-08-17T17:26:25.173210+0000","last_became_active":"2024-08-17T17:24:31.869575+0000","last_became_peered":"2024-08-17T17:24:31.869575+0000","last_unstale":"2024-08-17T17:26:25.173210+0000","last_undegraded":"2024-08-17T17:26:25.173210+0000","last_fullsized":"2024-08-17T17:26:25.173210+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:57.682409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:57.682409+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:42.144615+0000","scrub_duration":0.0066879779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1194,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.898954+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:26:19.898954+0000","last_peered":"2024-08-17T17:26:19.898954+0000","last_clean":"2024-08-17T17:26:19.898954+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:26:19.898954+0000","last_undegraded":"2024-08-17T17:26:19.898954+0000","last_fullsized":"2024-08-17T17:26:19.898954+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1210,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.899475+0000","last_change":"2024-08-17T17:21:06.451521+0000","last_active":"2024-08-17T17:26:19.899475+0000","last_peered":"2024-08-17T17:26:19.899475+0000","last_clean":"2024-08-17T17:26:19.899475+0000","last_became_active":"2024-08-17T17:21:06.451197+0000","last_became_peered":"2024-08-17T17:21:06.451197+0000","last_unstale":"2024-08-17T17:26:19.899475+0000","last_undegraded":"2024-08-17T17:26:19.899475+0000","last_fullsized":"2024-08-17T17:26:19.899475+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:04.000692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":11,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:35.024019+0000","last_change":"2024-08-17T17:26:35.023654+0000","last_active":"2024-08-17T17:26:35.024019+0000","last_peered":"2024-08-17T17:26:35.024019+0000","last_clean":"2024-08-17T17:26:35.024019+0000","last_became_active":"2024-08-17T17:26:35.022968+0000","last_became_peered":"2024-08-17T17:26:35.022968+0000","last_unstale":"2024-08-17T17:26:35.024019+0000","last_undegraded":"2024-08-17T17:26:35.024019+0000","last_fullsized":"2024-08-17T17:26:35.024019+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:01.733564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1535,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.902274+0000","last_change":"2024-08-17T17:24:52.716059+0000","last_active":"2024-08-17T17:26:19.902274+0000","last_peered":"2024-08-17T17:26:19.902274+0000","last_clean":"2024-08-17T17:26:19.902274+0000","last_became_active":"2024-08-17T17:24:31.798173+0000","last_became_peered":"2024-08-17T17:24:31.798173+0000","last_unstale":"2024-08-17T17:26:19.902274+0000","last_undegraded":"2024-08-17T17:26:19.902274+0000","last_fullsized":"2024-08-17T17:26:19.902274+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:04:56.094994+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1595,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.404797+0000","last_change":"2024-08-17T17:21:06.441118+0000","last_active":"2024-08-17T17:26:34.404797+0000","last_peered":"2024-08-17T17:26:34.404797+0000","last_clean":"2024-08-17T17:26:34.404797+0000","last_became_active":"2024-08-17T17:21:06.440800+0000","last_became_peered":"2024-08-17T17:21:06.440800+0000","last_unstale":"2024-08-17T17:26:34.404797+0000","last_undegraded":"2024-08-17T17:26:34.404797+0000","last_fullsized":"2024-08-17T17:26:34.404797+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:08.091528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":97,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.389074+0000","last_change":"2024-08-17T17:22:48.374766+0000","last_active":"2024-08-17T17:26:24.389074+0000","last_peered":"2024-08-17T17:26:24.389074+0000","last_clean":"2024-08-17T17:26:24.389074+0000","last_became_active":"2024-08-17T17:22:48.373556+0000","last_became_peered":"2024-08-17T17:22:48.373556+0000","last_unstale":"2024-08-17T17:26:24.389074+0000","last_undegraded":"2024-08-17T17:26:24.389074+0000","last_fullsized":"2024-08-17T17:26:24.389074+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:33.362080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":11,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.520150+0000","last_change":"2024-08-17T17:26:34.519719+0000","last_active":"2024-08-17T17:26:34.520150+0000","last_peered":"2024-08-17T17:26:34.520150+0000","last_clean":"2024-08-17T17:26:34.520150+0000","last_became_active":"2024-08-17T17:26:34.519368+0000","last_became_peered":"2024-08-17T17:26:34.519368+0000","last_unstale":"2024-08-17T17:26:34.520150+0000","last_undegraded":"2024-08-17T17:26:34.520150+0000","last_fullsized":"2024-08-17T17:26:34.520150+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:59.095875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1590,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.849827+0000","last_change":"2024-08-17T17:24:13.542832+0000","last_active":"2024-08-17T17:26:19.849827+0000","last_peered":"2024-08-17T17:26:19.849827+0000","last_clean":"2024-08-17T17:26:19.849827+0000","last_became_active":"2024-08-17T17:24:11.570282+0000","last_became_peered":"2024-08-17T17:24:11.570282+0000","last_unstale":"2024-08-17T17:26:19.849827+0000","last_undegraded":"2024-08-17T17:26:19.849827+0000","last_fullsized":"2024-08-17T17:26:19.849827+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:49.414162+0000","scrub_duration":0.017597167,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1204,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.904620+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:26:19.904620+0000","last_peered":"2024-08-17T17:26:19.904620+0000","last_clean":"2024-08-17T17:26:19.904620+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:26:19.904620+0000","last_undegraded":"2024-08-17T17:26:19.904620+0000","last_fullsized":"2024-08-17T17:26:19.904620+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1561,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.405080+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:26:34.405080+0000","last_peered":"2024-08-17T17:26:34.405080+0000","last_clean":"2024-08-17T17:26:34.405080+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:26:34.405080+0000","last_undegraded":"2024-08-17T17:26:34.405080+0000","last_fullsized":"2024-08-17T17:26:34.405080+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1175,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.375456+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:26:19.375456+0000","last_peered":"2024-08-17T17:26:19.375456+0000","last_clean":"2024-08-17T17:26:19.375456+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:26:19.375456+0000","last_undegraded":"2024-08-17T17:26:19.375456+0000","last_fullsized":"2024-08-17T17:26:19.375456+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:26:33.391667+0000","last_change":"2024-08-17T17:26:33.391667+0000","last_active":"2024-08-17T17:26:33.391667+0000","last_peered":"2024-08-17T17:26:33.391667+0000","last_clean":"2024-08-17T17:26:33.391667+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:26:33.391667+0000","last_undegraded":"2024-08-17T17:26:33.391667+0000","last_fullsized":"2024-08-17T17:26:33.391667+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":828,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.375131+0000","last_change":"2024-08-17T17:24:11.936322+0000","last_active":"2024-08-17T17:26:19.375131+0000","last_peered":"2024-08-17T17:26:19.375131+0000","last_clean":"2024-08-17T17:26:19.375131+0000","last_became_active":"2024-08-17T17:24:11.574846+0000","last_became_peered":"2024-08-17T17:24:11.574846+0000","last_unstale":"2024-08-17T17:26:19.375131+0000","last_undegraded":"2024-08-17T17:26:19.375131+0000","last_fullsized":"2024-08-17T17:26:19.375131+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:11.936236+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:11.936236+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:31.674851+0000","scrub_duration":0.0035797139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1240,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:25.076523+0000","last_change":"2024-08-17T17:21:06.430639+0000","last_active":"2024-08-17T17:26:25.076523+0000","last_peered":"2024-08-17T17:26:25.076523+0000","last_clean":"2024-08-17T17:26:25.076523+0000","last_became_active":"2024-08-17T17:21:06.430297+0000","last_became_peered":"2024-08-17T17:21:06.430297+0000","last_unstale":"2024-08-17T17:26:25.076523+0000","last_undegraded":"2024-08-17T17:26:25.076523+0000","last_fullsized":"2024-08-17T17:26:25.076523+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:57.794620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1043,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.898633+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:26:19.898633+0000","last_peered":"2024-08-17T17:26:19.898633+0000","last_clean":"2024-08-17T17:26:19.898633+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:26:19.898633+0000","last_undegraded":"2024-08-17T17:26:19.898633+0000","last_fullsized":"2024-08-17T17:26:19.898633+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":54,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.900144+0000","last_change":"2024-08-17T17:22:48.370769+0000","last_active":"2024-08-17T17:26:19.900144+0000","last_peered":"2024-08-17T17:26:19.900144+0000","last_clean":"2024-08-17T17:26:19.900144+0000","last_became_active":"2024-08-17T17:22:48.370531+0000","last_became_peered":"2024-08-17T17:22:48.370531+0000","last_unstale":"2024-08-17T17:26:19.900144+0000","last_undegraded":"2024-08-17T17:26:19.900144+0000","last_fullsized":"2024-08-17T17:26:19.900144+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:02.943823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:26:33.391667+0000","last_change":"2024-08-17T17:26:33.391667+0000","last_active":"2024-08-17T17:26:33.391667+0000","last_peered":"2024-08-17T17:26:33.391667+0000","last_clean":"2024-08-17T17:26:33.391667+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:26:33.391667+0000","last_undegraded":"2024-08-17T17:26:33.391667+0000","last_fullsized":"2024-08-17T17:26:33.391667+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1610,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.849993+0000","last_change":"2024-08-17T17:24:12.519946+0000","last_active":"2024-08-17T17:26:19.849993+0000","last_peered":"2024-08-17T17:26:19.849993+0000","last_clean":"2024-08-17T17:26:19.849993+0000","last_became_active":"2024-08-17T17:24:11.569747+0000","last_became_peered":"2024-08-17T17:24:11.569747+0000","last_unstale":"2024-08-17T17:26:19.849993+0000","last_undegraded":"2024-08-17T17:26:19.849993+0000","last_fullsized":"2024-08-17T17:26:19.849993+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.519838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.519838+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:56.003993+0000","scrub_duration":0.0068224740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1654,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.389083+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:26:24.389083+0000","last_peered":"2024-08-17T17:26:24.389083+0000","last_clean":"2024-08-17T17:26:24.389083+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:26:24.389083+0000","last_undegraded":"2024-08-17T17:26:24.389083+0000","last_fullsized":"2024-08-17T17:26:24.389083+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1046,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.388795+0000","last_change":"2024-08-17T17:21:06.428684+0000","last_active":"2024-08-17T17:26:24.388795+0000","last_peered":"2024-08-17T17:26:24.388795+0000","last_clean":"2024-08-17T17:26:24.388795+0000","last_became_active":"2024-08-17T17:21:06.428178+0000","last_became_peered":"2024-08-17T17:21:06.428178+0000","last_unstale":"2024-08-17T17:26:24.388795+0000","last_undegraded":"2024-08-17T17:26:24.388795+0000","last_fullsized":"2024-08-17T17:26:24.388795+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:14.836119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":887,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:25.173468+0000","last_change":"2024-08-17T17:21:06.443565+0000","last_active":"2024-08-17T17:26:25.173468+0000","last_peered":"2024-08-17T17:26:25.173468+0000","last_clean":"2024-08-17T17:26:25.173468+0000","last_became_active":"2024-08-17T17:21:06.443215+0000","last_became_peered":"2024-08-17T17:21:06.443215+0000","last_unstale":"2024-08-17T17:26:25.173468+0000","last_undegraded":"2024-08-17T17:26:25.173468+0000","last_fullsized":"2024-08-17T17:26:25.173468+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:47.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:26:33.391667+0000","last_change":"2024-08-17T17:26:33.391667+0000","last_active":"2024-08-17T17:26:33.391667+0000","last_peered":"2024-08-17T17:26:33.391667+0000","last_clean":"2024-08-17T17:26:33.391667+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:26:33.391667+0000","last_undegraded":"2024-08-17T17:26:33.391667+0000","last_fullsized":"2024-08-17T17:26:33.391667+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1552,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.899359+0000","last_change":"2024-08-17T17:24:13.795227+0000","last_active":"2024-08-17T17:26:19.899359+0000","last_peered":"2024-08-17T17:26:19.899359+0000","last_clean":"2024-08-17T17:26:19.899359+0000","last_became_active":"2024-08-17T17:24:11.578352+0000","last_became_peered":"2024-08-17T17:24:11.578352+0000","last_unstale":"2024-08-17T17:26:19.899359+0000","last_undegraded":"2024-08-17T17:26:19.899359+0000","last_fullsized":"2024-08-17T17:26:19.899359+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:36:39.440808+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1231,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.902540+0000","last_change":"2024-08-17T17:21:06.456978+0000","last_active":"2024-08-17T17:26:19.902540+0000","last_peered":"2024-08-17T17:26:19.902540+0000","last_clean":"2024-08-17T17:26:19.902540+0000","last_became_active":"2024-08-17T17:21:06.452265+0000","last_became_peered":"2024-08-17T17:21:06.452265+0000","last_unstale":"2024-08-17T17:26:19.902540+0000","last_undegraded":"2024-08-17T17:26:19.902540+0000","last_fullsized":"2024-08-17T17:26:19.902540+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:52.893746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1212,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.899369+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:26:19.899369+0000","last_peered":"2024-08-17T17:26:19.899369+0000","last_clean":"2024-08-17T17:26:19.899369+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:26:19.899369+0000","last_undegraded":"2024-08-17T17:26:19.899369+0000","last_fullsized":"2024-08-17T17:26:19.899369+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1122,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.389250+0000","last_change":"2024-08-17T17:21:06.435754+0000","last_active":"2024-08-17T17:26:24.389250+0000","last_peered":"2024-08-17T17:26:24.389250+0000","last_clean":"2024-08-17T17:26:24.389250+0000","last_became_active":"2024-08-17T17:21:06.435442+0000","last_became_peered":"2024-08-17T17:21:06.435442+0000","last_unstale":"2024-08-17T17:26:24.389250+0000","last_undegraded":"2024-08-17T17:26:24.389250+0000","last_fullsized":"2024-08-17T17:26:24.389250+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:34.849042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:26:33.391667+0000","last_change":"2024-08-17T17:26:33.391667+0000","last_active":"2024-08-17T17:26:33.391667+0000","last_peered":"2024-08-17T17:26:33.391667+0000","last_clean":"2024-08-17T17:26:33.391667+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:26:33.391667+0000","last_undegraded":"2024-08-17T17:26:33.391667+0000","last_fullsized":"2024-08-17T17:26:33.391667+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":815,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.375591+0000","last_change":"2024-08-17T17:24:55.041464+0000","last_active":"2024-08-17T17:26:19.375591+0000","last_peered":"2024-08-17T17:26:19.375591+0000","last_clean":"2024-08-17T17:26:19.375591+0000","last_became_active":"2024-08-17T17:24:31.800289+0000","last_became_peered":"2024-08-17T17:24:31.800289+0000","last_unstale":"2024-08-17T17:26:19.375591+0000","last_undegraded":"2024-08-17T17:26:19.375591+0000","last_fullsized":"2024-08-17T17:26:19.375591+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:55.041378+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:55.041378+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:43.974501+0000","scrub_duration":0.001623642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1205,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.904297+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:26:19.904297+0000","last_peered":"2024-08-17T17:26:19.904297+0000","last_clean":"2024-08-17T17:26:19.904297+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:26:19.904297+0000","last_undegraded":"2024-08-17T17:26:19.904297+0000","last_fullsized":"2024-08-17T17:26:19.904297+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":896,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.850496+0000","last_change":"2024-08-17T17:21:06.434991+0000","last_active":"2024-08-17T17:26:19.850496+0000","last_peered":"2024-08-17T17:26:19.850496+0000","last_clean":"2024-08-17T17:26:19.850496+0000","last_became_active":"2024-08-17T17:21:06.434630+0000","last_became_peered":"2024-08-17T17:21:06.434630+0000","last_unstale":"2024-08-17T17:26:19.850496+0000","last_undegraded":"2024-08-17T17:26:19.850496+0000","last_fullsized":"2024-08-17T17:26:19.850496+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:24.104609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":838,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.902423+0000","last_change":"2024-08-17T17:21:06.462532+0000","last_active":"2024-08-17T17:26:19.902423+0000","last_peered":"2024-08-17T17:26:19.902423+0000","last_clean":"2024-08-17T17:26:19.902423+0000","last_became_active":"2024-08-17T17:21:06.462183+0000","last_became_peered":"2024-08-17T17:21:06.462183+0000","last_unstale":"2024-08-17T17:26:19.902423+0000","last_undegraded":"2024-08-17T17:26:19.902423+0000","last_fullsized":"2024-08-17T17:26:19.902423+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:50.033322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:26:33.391667+0000","last_change":"2024-08-17T17:26:33.391667+0000","last_active":"2024-08-17T17:26:33.391667+0000","last_peered":"2024-08-17T17:26:33.391667+0000","last_clean":"2024-08-17T17:26:33.391667+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:26:33.391667+0000","last_undegraded":"2024-08-17T17:26:33.391667+0000","last_fullsized":"2024-08-17T17:26:33.391667+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":832,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.375507+0000","last_change":"2024-08-17T17:24:53.103270+0000","last_active":"2024-08-17T17:26:19.375507+0000","last_peered":"2024-08-17T17:26:19.375507+0000","last_clean":"2024-08-17T17:26:19.375507+0000","last_became_active":"2024-08-17T17:24:31.800823+0000","last_became_peered":"2024-08-17T17:24:31.800823+0000","last_unstale":"2024-08-17T17:26:19.375507+0000","last_undegraded":"2024-08-17T17:26:19.375507+0000","last_fullsized":"2024-08-17T17:26:19.375507+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.103133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.103133+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:52.444781+0000","scrub_duration":0.0074281759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1045,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.389060+0000","last_change":"2024-08-17T17:21:06.433720+0000","last_active":"2024-08-17T17:26:24.389060+0000","last_peered":"2024-08-17T17:26:24.389060+0000","last_clean":"2024-08-17T17:26:24.389060+0000","last_became_active":"2024-08-17T17:21:06.433292+0000","last_became_peered":"2024-08-17T17:21:06.433292+0000","last_unstale":"2024-08-17T17:26:24.389060+0000","last_undegraded":"2024-08-17T17:26:24.389060+0000","last_fullsized":"2024-08-17T17:26:24.389060+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:32.246594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1206,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.851178+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:26:19.851178+0000","last_peered":"2024-08-17T17:26:19.851178+0000","last_clean":"2024-08-17T17:26:19.851178+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:26:19.851178+0000","last_undegraded":"2024-08-17T17:26:19.851178+0000","last_fullsized":"2024-08-17T17:26:19.851178+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":798,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.904549+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:26:19.904549+0000","last_peered":"2024-08-17T17:26:19.904549+0000","last_clean":"2024-08-17T17:26:19.904549+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:26:19.904549+0000","last_undegraded":"2024-08-17T17:26:19.904549+0000","last_fullsized":"2024-08-17T17:26:19.904549+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2024-08-17T17:26:33.391667+0000","last_change":"2024-08-17T17:26:33.391667+0000","last_active":"2024-08-17T17:26:33.391667+0000","last_peered":"2024-08-17T17:26:33.391667+0000","last_clean":"2024-08-17T17:26:33.391667+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-08-17T17:26:33.391667+0000","last_undegraded":"2024-08-17T17:26:33.391667+0000","last_fullsized":"2024-08-17T17:26:33.391667+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1556,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.899873+0000","last_change":"2024-08-17T17:24:49.119578+0000","last_active":"2024-08-17T17:26:19.899873+0000","last_peered":"2024-08-17T17:26:19.899873+0000","last_clean":"2024-08-17T17:26:19.899873+0000","last_became_active":"2024-08-17T17:24:31.807987+0000","last_became_peered":"2024-08-17T17:24:31.807987+0000","last_unstale":"2024-08-17T17:26:19.899873+0000","last_undegraded":"2024-08-17T17:26:19.899873+0000","last_fullsized":"2024-08-17T17:26:19.899873+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:34.052786+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1614,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.405530+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:26:34.405530+0000","last_peered":"2024-08-17T17:26:34.405530+0000","last_clean":"2024-08-17T17:26:34.405530+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:26:34.405530+0000","last_undegraded":"2024-08-17T17:26:34.405530+0000","last_fullsized":"2024-08-17T17:26:34.405530+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":848,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.390702+0000","last_change":"2024-08-17T17:21:06.452395+0000","last_active":"2024-08-17T17:26:24.390702+0000","last_peered":"2024-08-17T17:26:24.390702+0000","last_clean":"2024-08-17T17:26:24.390702+0000","last_became_active":"2024-08-17T17:21:06.451867+0000","last_became_peered":"2024-08-17T17:21:06.451867+0000","last_unstale":"2024-08-17T17:26:24.390702+0000","last_undegraded":"2024-08-17T17:26:24.390702+0000","last_fullsized":"2024-08-17T17:26:24.390702+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:50.597092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1604,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.404595+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:26:34.404595+0000","last_peered":"2024-08-17T17:26:34.404595+0000","last_clean":"2024-08-17T17:26:34.404595+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:26:34.404595+0000","last_undegraded":"2024-08-17T17:26:34.404595+0000","last_fullsized":"2024-08-17T17:26:34.404595+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":11,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.412250+0000","last_change":"2024-08-17T17:26:34.412017+0000","last_active":"2024-08-17T17:26:34.412250+0000","last_peered":"2024-08-17T17:26:34.412250+0000","last_clean":"2024-08-17T17:26:34.412250+0000","last_became_active":"2024-08-17T17:26:34.411768+0000","last_became_peered":"2024-08-17T17:26:34.411768+0000","last_unstale":"2024-08-17T17:26:34.412250+0000","last_undegraded":"2024-08-17T17:26:34.412250+0000","last_fullsized":"2024-08-17T17:26:34.412250+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:49:50.671390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1639,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.405730+0000","last_change":"2024-08-17T17:24:12.033217+0000","last_active":"2024-08-17T17:26:34.405730+0000","last_peered":"2024-08-17T17:26:34.405730+0000","last_clean":"2024-08-17T17:26:34.405730+0000","last_became_active":"2024-08-17T17:24:11.572877+0000","last_became_peered":"2024-08-17T17:24:11.572877+0000","last_unstale":"2024-08-17T17:26:34.405730+0000","last_undegraded":"2024-08-17T17:26:34.405730+0000","last_fullsized":"2024-08-17T17:26:34.405730+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.033101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.033101+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.264475+0000","scrub_duration":20,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":217,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.685692+0000","last_change":"2024-08-17T17:21:06.519965+0000","last_active":"2024-08-17T17:26:24.685692+0000","last_peered":"2024-08-17T17:26:24.685692+0000","last_clean":"2024-08-17T17:26:24.685692+0000","last_became_active":"2024-08-17T17:21:06.459952+0000","last_became_peered":"2024-08-17T17:21:06.459952+0000","last_unstale":"2024-08-17T17:26:24.685692+0000","last_undegraded":"2024-08-17T17:26:24.685692+0000","last_fullsized":"2024-08-17T17:26:24.685692+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:27.828098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1202,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.375835+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:26:19.375835+0000","last_peered":"2024-08-17T17:26:19.375835+0000","last_clean":"2024-08-17T17:26:19.375835+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:26:19.375835+0000","last_undegraded":"2024-08-17T17:26:19.375835+0000","last_fullsized":"2024-08-17T17:26:19.375835+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1178,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.898729+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:26:19.898729+0000","last_peered":"2024-08-17T17:26:19.898729+0000","last_clean":"2024-08-17T17:26:19.898729+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:26:19.898729+0000","last_undegraded":"2024-08-17T17:26:19.898729+0000","last_fullsized":"2024-08-17T17:26:19.898729+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":704,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.405517+0000","last_change":"2024-08-17T17:24:48.935651+0000","last_active":"2024-08-17T17:26:34.405517+0000","last_peered":"2024-08-17T17:26:34.405517+0000","last_clean":"2024-08-17T17:26:34.405517+0000","last_became_active":"2024-08-17T17:24:40.897753+0000","last_became_peered":"2024-08-17T17:24:40.897753+0000","last_unstale":"2024-08-17T17:26:34.405517+0000","last_undegraded":"2024-08-17T17:26:34.405517+0000","last_fullsized":"2024-08-17T17:26:34.405517+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:10:15.118428+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1210,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.685835+0000","last_change":"2024-08-17T17:21:06.458000+0000","last_active":"2024-08-17T17:26:24.685835+0000","last_peered":"2024-08-17T17:26:24.685835+0000","last_clean":"2024-08-17T17:26:24.685835+0000","last_became_active":"2024-08-17T17:21:06.457411+0000","last_became_peered":"2024-08-17T17:21:06.457411+0000","last_unstale":"2024-08-17T17:26:24.685835+0000","last_undegraded":"2024-08-17T17:26:24.685835+0000","last_fullsized":"2024-08-17T17:26:24.685835+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:52.740639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1201,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:22.349237+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:26:22.349237+0000","last_peered":"2024-08-17T17:26:22.349237+0000","last_clean":"2024-08-17T17:26:22.349237+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:26:22.349237+0000","last_undegraded":"2024-08-17T17:26:22.349237+0000","last_fullsized":"2024-08-17T17:26:22.349237+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":677,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.899696+0000","last_change":"2024-08-17T17:25:03.663370+0000","last_active":"2024-08-17T17:26:19.899696+0000","last_peered":"2024-08-17T17:26:19.899696+0000","last_clean":"2024-08-17T17:26:19.899696+0000","last_became_active":"2024-08-17T17:24:40.886214+0000","last_became_peered":"2024-08-17T17:24:40.886214+0000","last_unstale":"2024-08-17T17:26:19.899696+0000","last_undegraded":"2024-08-17T17:26:19.899696+0000","last_fullsized":"2024-08-17T17:26:19.899696+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:11:39.272029+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":870,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.375130+0000","last_change":"2024-08-17T17:21:06.454587+0000","last_active":"2024-08-17T17:26:19.375130+0000","last_peered":"2024-08-17T17:26:19.375130+0000","last_clean":"2024-08-17T17:26:19.375130+0000","last_became_active":"2024-08-17T17:21:06.454229+0000","last_became_peered":"2024-08-17T17:21:06.454229+0000","last_unstale":"2024-08-17T17:26:19.375130+0000","last_undegraded":"2024-08-17T17:26:19.375130+0000","last_fullsized":"2024-08-17T17:26:19.375130+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:38.340263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1176,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.851012+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:26:19.851012+0000","last_peered":"2024-08-17T17:26:19.851012+0000","last_clean":"2024-08-17T17:26:19.851012+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:26:19.851012+0000","last_undegraded":"2024-08-17T17:26:19.851012+0000","last_fullsized":"2024-08-17T17:26:19.851012+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1637,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.388901+0000","last_change":"2024-08-17T17:24:20.649977+0000","last_active":"2024-08-17T17:26:24.388901+0000","last_peered":"2024-08-17T17:26:24.388901+0000","last_clean":"2024-08-17T17:26:24.388901+0000","last_became_active":"2024-08-17T17:24:20.649336+0000","last_became_peered":"2024-08-17T17:24:20.649336+0000","last_unstale":"2024-08-17T17:26:24.388901+0000","last_undegraded":"2024-08-17T17:26:24.388901+0000","last_fullsized":"2024-08-17T17:26:24.388901+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:57.187271+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1206,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.900256+0000","last_change":"2024-08-17T17:21:06.442796+0000","last_active":"2024-08-17T17:26:19.900256+0000","last_peered":"2024-08-17T17:26:19.900256+0000","last_clean":"2024-08-17T17:26:19.900256+0000","last_became_active":"2024-08-17T17:21:06.442466+0000","last_became_peered":"2024-08-17T17:21:06.442466+0000","last_unstale":"2024-08-17T17:26:19.900256+0000","last_undegraded":"2024-08-17T17:26:19.900256+0000","last_fullsized":"2024-08-17T17:26:19.900256+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:36.366620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1560,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.404796+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:26:34.404796+0000","last_peered":"2024-08-17T17:26:34.404796+0000","last_clean":"2024-08-17T17:26:34.404796+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:26:34.404796+0000","last_undegraded":"2024-08-17T17:26:34.404796+0000","last_fullsized":"2024-08-17T17:26:34.404796+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1424,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.851151+0000","last_change":"2024-08-17T17:25:07.472056+0000","last_active":"2024-08-17T17:26:19.851151+0000","last_peered":"2024-08-17T17:26:19.851151+0000","last_clean":"2024-08-17T17:26:19.851151+0000","last_became_active":"2024-08-17T17:24:40.883356+0000","last_became_peered":"2024-08-17T17:24:40.883356+0000","last_unstale":"2024-08-17T17:26:19.851151+0000","last_undegraded":"2024-08-17T17:26:19.851151+0000","last_fullsized":"2024-08-17T17:26:19.851151+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:43:27.883917+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1042,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.900053+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:26:19.900053+0000","last_peered":"2024-08-17T17:26:19.900053+0000","last_clean":"2024-08-17T17:26:19.900053+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:26:19.900053+0000","last_undegraded":"2024-08-17T17:26:19.900053+0000","last_fullsized":"2024-08-17T17:26:19.900053+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1593,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.404527+0000","last_change":"2024-08-17T17:21:06.468788+0000","last_active":"2024-08-17T17:26:34.404527+0000","last_peered":"2024-08-17T17:26:34.404527+0000","last_clean":"2024-08-17T17:26:34.404527+0000","last_became_active":"2024-08-17T17:21:06.467141+0000","last_became_peered":"2024-08-17T17:21:06.467141+0000","last_unstale":"2024-08-17T17:26:34.404527+0000","last_undegraded":"2024-08-17T17:26:34.404527+0000","last_fullsized":"2024-08-17T17:26:34.404527+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:51.301359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1424,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.899649+0000","last_change":"2024-08-17T17:24:53.130774+0000","last_active":"2024-08-17T17:26:19.899649+0000","last_peered":"2024-08-17T17:26:19.899649+0000","last_clean":"2024-08-17T17:26:19.899649+0000","last_became_active":"2024-08-17T17:24:40.883388+0000","last_became_peered":"2024-08-17T17:24:40.883388+0000","last_unstale":"2024-08-17T17:26:19.899649+0000","last_undegraded":"2024-08-17T17:26:19.899649+0000","last_fullsized":"2024-08-17T17:26:19.899649+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:36.772875+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1208,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.375760+0000","last_change":"2024-08-17T17:21:06.435998+0000","last_active":"2024-08-17T17:26:19.375760+0000","last_peered":"2024-08-17T17:26:19.375760+0000","last_clean":"2024-08-17T17:26:19.375760+0000","last_became_active":"2024-08-17T17:21:06.435579+0000","last_became_peered":"2024-08-17T17:21:06.435579+0000","last_unstale":"2024-08-17T17:26:19.375760+0000","last_undegraded":"2024-08-17T17:26:19.375760+0000","last_fullsized":"2024-08-17T17:26:19.375760+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:01.743074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1622,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.404703+0000","last_change":"2024-08-17T17:21:06.473627+0000","last_active":"2024-08-17T17:26:34.404703+0000","last_peered":"2024-08-17T17:26:34.404703+0000","last_clean":"2024-08-17T17:26:34.404703+0000","last_became_active":"2024-08-17T17:21:06.473062+0000","last_became_peered":"2024-08-17T17:21:06.473062+0000","last_unstale":"2024-08-17T17:26:34.404703+0000","last_undegraded":"2024-08-17T17:26:34.404703+0000","last_fullsized":"2024-08-17T17:26:34.404703+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:21.413163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":784,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.902697+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:26:19.902697+0000","last_peered":"2024-08-17T17:26:19.902697+0000","last_clean":"2024-08-17T17:26:19.902697+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:26:19.902697+0000","last_undegraded":"2024-08-17T17:26:19.902697+0000","last_fullsized":"2024-08-17T17:26:19.902697+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1003,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.850695+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:26:19.850695+0000","last_peered":"2024-08-17T17:26:19.850695+0000","last_clean":"2024-08-17T17:26:19.850695+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:26:19.850695+0000","last_undegraded":"2024-08-17T17:26:19.850695+0000","last_fullsized":"2024-08-17T17:26:19.850695+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1113,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.405024+0000","last_change":"2024-08-17T17:21:06.471265+0000","last_active":"2024-08-17T17:26:34.405024+0000","last_peered":"2024-08-17T17:26:34.405024+0000","last_clean":"2024-08-17T17:26:34.405024+0000","last_became_active":"2024-08-17T17:21:06.470943+0000","last_became_peered":"2024-08-17T17:21:06.470943+0000","last_unstale":"2024-08-17T17:26:34.405024+0000","last_undegraded":"2024-08-17T17:26:34.405024+0000","last_fullsized":"2024-08-17T17:26:34.405024+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:41.070024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1556,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.906460+0000","last_change":"2024-08-17T17:25:02.696608+0000","last_active":"2024-08-17T17:26:19.906460+0000","last_peered":"2024-08-17T17:26:19.906460+0000","last_clean":"2024-08-17T17:26:19.906460+0000","last_became_active":"2024-08-17T17:24:40.877715+0000","last_became_peered":"2024-08-17T17:24:40.877715+0000","last_unstale":"2024-08-17T17:26:19.906460+0000","last_undegraded":"2024-08-17T17:26:19.906460+0000","last_fullsized":"2024-08-17T17:26:19.906460+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:48:00.941285+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1599,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.405411+0000","last_change":"2024-08-17T17:22:48.377351+0000","last_active":"2024-08-17T17:26:34.405411+0000","last_peered":"2024-08-17T17:26:34.405411+0000","last_clean":"2024-08-17T17:26:34.405411+0000","last_became_active":"2024-08-17T17:22:48.377059+0000","last_became_peered":"2024-08-17T17:22:48.377059+0000","last_unstale":"2024-08-17T17:26:34.405411+0000","last_undegraded":"2024-08-17T17:26:34.405411+0000","last_fullsized":"2024-08-17T17:26:34.405411+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:15.250902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1226,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.901132+0000","last_change":"2024-08-17T17:21:06.459139+0000","last_active":"2024-08-17T17:26:19.901132+0000","last_peered":"2024-08-17T17:26:19.901132+0000","last_clean":"2024-08-17T17:26:19.901132+0000","last_became_active":"2024-08-17T17:21:06.457850+0000","last_became_peered":"2024-08-17T17:21:06.457850+0000","last_unstale":"2024-08-17T17:26:19.901132+0000","last_undegraded":"2024-08-17T17:26:19.901132+0000","last_fullsized":"2024-08-17T17:26:19.901132+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:13.888576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1500,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.404633+0000","last_change":"2024-08-17T17:24:49.911734+0000","last_active":"2024-08-17T17:26:34.404633+0000","last_peered":"2024-08-17T17:26:34.404633+0000","last_clean":"2024-08-17T17:26:34.404633+0000","last_became_active":"2024-08-17T17:24:40.899102+0000","last_became_peered":"2024-08-17T17:24:40.899102+0000","last_unstale":"2024-08-17T17:26:34.404633+0000","last_undegraded":"2024-08-17T17:26:34.404633+0000","last_fullsized":"2024-08-17T17:26:34.404633+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.911622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.911622+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:30:27.987033+0000","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1636,"reported_epoch":1246,"state":"active+clean","last_fresh":"2024-08-17T17:26:24.389162+0000","last_change":"2024-08-17T17:21:06.446321+0000","last_active":"2024-08-17T17:26:24.389162+0000","last_peered":"2024-08-17T17:26:24.389162+0000","last_clean":"2024-08-17T17:26:24.389162+0000","last_became_active":"2024-08-17T17:21:06.446019+0000","last_became_peered":"2024-08-17T17:21:06.446019+0000","last_unstale":"2024-08-17T17:26:24.389162+0000","last_undegraded":"2024-08-17T17:26:24.389162+0000","last_fullsized":"2024-08-17T17:26:24.389162+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:26.750070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":195,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.898969+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:26:19.898969+0000","last_peered":"2024-08-17T17:26:19.898969+0000","last_clean":"2024-08-17T17:26:19.898969+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:26:19.898969+0000","last_undegraded":"2024-08-17T17:26:19.898969+0000","last_fullsized":"2024-08-17T17:26:19.898969+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":704,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.405791+0000","last_change":"2024-08-17T17:24:46.946320+0000","last_active":"2024-08-17T17:26:34.405791+0000","last_peered":"2024-08-17T17:26:34.405791+0000","last_clean":"2024-08-17T17:26:34.405791+0000","last_became_active":"2024-08-17T17:24:40.880380+0000","last_became_peered":"2024-08-17T17:24:40.880380+0000","last_unstale":"2024-08-17T17:26:34.405791+0000","last_undegraded":"2024-08-17T17:26:34.405791+0000","last_fullsized":"2024-08-17T17:26:34.405791+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:54:30.637162+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1224,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.899169+0000","last_change":"2024-08-17T17:21:06.465804+0000","last_active":"2024-08-17T17:26:19.899169+0000","last_peered":"2024-08-17T17:26:19.899169+0000","last_clean":"2024-08-17T17:26:19.899169+0000","last_became_active":"2024-08-17T17:21:06.465461+0000","last_became_peered":"2024-08-17T17:21:06.465461+0000","last_unstale":"2024-08-17T17:26:19.899169+0000","last_undegraded":"2024-08-17T17:26:19.899169+0000","last_fullsized":"2024-08-17T17:26:19.899169+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:06.443667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1216,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.905423+0000","last_change":"2024-08-17T17:21:06.481634+0000","last_active":"2024-08-17T17:26:19.905423+0000","last_peered":"2024-08-17T17:26:19.905423+0000","last_clean":"2024-08-17T17:26:19.905423+0000","last_became_active":"2024-08-17T17:21:06.469081+0000","last_became_peered":"2024-08-17T17:21:06.469081+0000","last_unstale":"2024-08-17T17:26:19.905423+0000","last_undegraded":"2024-08-17T17:26:19.905423+0000","last_fullsized":"2024-08-17T17:26:19.905423+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:53:26.954738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1224,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.900993+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:26:19.900993+0000","last_peered":"2024-08-17T17:26:19.900993+0000","last_clean":"2024-08-17T17:26:19.900993+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:26:19.900993+0000","last_undegraded":"2024-08-17T17:26:19.900993+0000","last_fullsized":"2024-08-17T17:26:19.900993+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1175,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.904229+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:26:19.904229+0000","last_peered":"2024-08-17T17:26:19.904229+0000","last_clean":"2024-08-17T17:26:19.904229+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:26:19.904229+0000","last_undegraded":"2024-08-17T17:26:19.904229+0000","last_fullsized":"2024-08-17T17:26:19.904229+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1615,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.850256+0000","last_change":"2024-08-17T17:25:02.417545+0000","last_active":"2024-08-17T17:26:19.850256+0000","last_peered":"2024-08-17T17:26:19.850256+0000","last_clean":"2024-08-17T17:26:19.850256+0000","last_became_active":"2024-08-17T17:24:40.883220+0000","last_became_peered":"2024-08-17T17:24:40.883220+0000","last_unstale":"2024-08-17T17:26:19.850256+0000","last_undegraded":"2024-08-17T17:26:19.850256+0000","last_fullsized":"2024-08-17T17:26:19.850256+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:28.257805+0000","scrub_duration":0.0084952199999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1598,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.405177+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:26:34.405177+0000","last_peered":"2024-08-17T17:26:34.405177+0000","last_clean":"2024-08-17T17:26:34.405177+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:26:34.405177+0000","last_undegraded":"2024-08-17T17:26:34.405177+0000","last_fullsized":"2024-08-17T17:26:34.405177+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1574,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.406004+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:26:34.406004+0000","last_peered":"2024-08-17T17:26:34.406004+0000","last_clean":"2024-08-17T17:26:34.406004+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:26:34.406004+0000","last_undegraded":"2024-08-17T17:26:34.406004+0000","last_fullsized":"2024-08-17T17:26:34.406004+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1448,"reported_epoch":1248,"state":"active+clean","last_fresh":"2024-08-17T17:26:34.405300+0000","last_change":"2024-08-17T17:24:47.961645+0000","last_active":"2024-08-17T17:26:34.405300+0000","last_peered":"2024-08-17T17:26:34.405300+0000","last_clean":"2024-08-17T17:26:34.405300+0000","last_became_active":"2024-08-17T17:24:40.897539+0000","last_became_peered":"2024-08-17T17:24:40.897539+0000","last_unstale":"2024-08-17T17:26:34.405300+0000","last_undegraded":"2024-08-17T17:26:34.405300+0000","last_fullsized":"2024-08-17T17:26:34.405300+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:47.961445+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:47.961445+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:04:32.031483+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1209,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.898614+0000","last_change":"2024-08-17T17:21:06.483469+0000","last_active":"2024-08-17T17:26:19.898614+0000","last_peered":"2024-08-17T17:26:19.898614+0000","last_clean":"2024-08-17T17:26:19.898614+0000","last_became_active":"2024-08-17T17:21:06.483162+0000","last_became_peered":"2024-08-17T17:21:06.483162+0000","last_unstale":"2024-08-17T17:26:19.898614+0000","last_undegraded":"2024-08-17T17:26:19.898614+0000","last_fullsized":"2024-08-17T17:26:19.898614+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:24.898353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":218,"reported_epoch":1245,"state":"active+clean","last_fresh":"2024-08-17T17:26:19.899976+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:26:19.899976+0000","last_peered":"2024-08-17T17:26:19.899976+0000","last_clean":"2024-08-17T17:26:19.899976+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:26:19.899976+0000","last_undegraded":"2024-08-17T17:26:19.899976+0000","last_fullsized":"2024-08-17T17:26:19.899976+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":12,"acting":12,"num_store_stats":1},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201701,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":444664,"kb_used_data":8364,"kb_used_omap":1,"kb_used_meta":436286,"kb_avail":93300488,"statfs":{"total":95995035648,"available":95539699712,"internally_reserved":0,"allocated":8564736,"data_stored":5996239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":446757323},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463338,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":378284,"kb_used_data":8416,"kb_used_omap":1,"kb_used_meta":369854,"kb_avail":93366868,"statfs":{"total":95995035648,"available":95607672832,"internally_reserved":0,"allocated":8617984,"data_stored":6061104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":378730953},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626980,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":618100,"kb_used_data":8424,"kb_used_omap":42,"kb_used_meta":609621,"kb_avail":93127052,"statfs":{"total":95995035648,"available":95362101248,"internally_reserved":0,"allocated":8626176,"data_stored":6073267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":624251964},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785457,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":437404,"kb_used_data":8080,"kb_used_omap":48,"kb_used_meta":429263,"kb_avail":93307748,"statfs":{"total":95995035648,"available":95547133952,"internally_reserved":0,"allocated":8273920,"data_stored":5728610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":439565998},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724974,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":458600,"kb_used_data":8348,"kb_used_omap":1,"kb_used_meta":450238,"kb_avail":93286552,"statfs":{"total":95995035648,"available":95525429248,"internally_reserved":0,"allocated":8548352,"data_stored":5991938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":461044166},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986611,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":432112,"kb_used_data":8100,"kb_used_omap":1,"kb_used_meta":423998,"kb_avail":93313040,"statfs":{"total":95995035648,"available":95552552960,"internally_reserved":0,"allocated":8294400,"data_stored":5733481,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":434174406},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706218,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":469348,"kb_used_data":8344,"kb_used_omap":35,"kb_used_meta":460956,"kb_avail":93275804,"statfs":{"total":95995035648,"available":95514423296,"internally_reserved":0,"allocated":8544256,"data_stored":5991895,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":472018999},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831991,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":568416,"kb_used_data":8404,"kb_used_omap":65,"kb_used_meta":559934,"kb_avail":93176736,"statfs":{"total":95995035648,"available":95412977664,"internally_reserved":0,"allocated":8605696,"data_stored":6056875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":573372603},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:26:41.478 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:26:41.532 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:41.556 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:41.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:41.657 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:41.682 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:41.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:41.768 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:41 smithi063 ceph-mon[114756]: pgmap v1692: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:41.769 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:41 smithi063 ceph-mon[109843]: pgmap v1692: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:41.784 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:41.807 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:41.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:41.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:41 smithi162 ceph-mon[104668]: pgmap v1692: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:41.909 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:41.934 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:41.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:41.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:42.035 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:42.059 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:42.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:42.161 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:42.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.185 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:42.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:42.286 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:42.310 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:42.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:42.412 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:42.425 INFO:teuthology.orchestra.run.smithi063.stderr:enabled application 'rados' on pool 'unique_pool_4' 2024-08-17T17:26:42.427 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:26:42.437 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:42.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:42.539 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:42.561 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:42.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:42.663 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:42.669 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:42 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/339722065' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:42.670 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:42 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:42.670 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:42 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3024408723' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:26:42.670 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:42 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/339722065' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:42.670 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:42 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:42.670 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:42 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3024408723' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:26:42.686 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:42.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:42.788 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:42.811 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:42.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:42.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:42 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/339722065' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:42.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:42 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:26:42.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:42 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3024408723' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:26:42.912 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:42.935 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:42.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:42.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:43.036 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:43.061 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:43.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:43.163 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:43.187 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:43.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:43.264 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:26:43.264 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:26:43.289 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:43.312 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:43.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:43.414 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:43.419 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:26:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:26:43] "GET /metrics HTTP/1.1" 200 34040 "" "Prometheus/2.51.0" 2024-08-17T17:26:43.437 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:43.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:43.539 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:43.561 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:43.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:43.662 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:43.688 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:43.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:43.760 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:43 smithi063 ceph-mon[114756]: pgmap v1693: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:43.760 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:43 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:43.760 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:43 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:43.760 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:43 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3024408723' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:26:43.760 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:43 smithi063 ceph-mon[114756]: osdmap e1250: 8 total, 8 up, 8 in 2024-08-17T17:26:43.760 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:43 smithi063 ceph-mon[109843]: pgmap v1693: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:43.761 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:43 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:43.761 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:43 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:43.761 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:43 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3024408723' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:26:43.761 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:43 smithi063 ceph-mon[109843]: osdmap e1250: 8 total, 8 up, 8 in 2024-08-17T17:26:43.790 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:43.815 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:43.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:43.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:43 smithi162 ceph-mon[104668]: pgmap v1693: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:43.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:43 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:43.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:43 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:26:43.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:43 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3024408723' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:26:43.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:43 smithi162 ceph-mon[104668]: osdmap e1250: 8 total, 8 up, 8 in 2024-08-17T17:26:43.917 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:43.943 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:43.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:43.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:44.044 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:44.067 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:44.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:44.169 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:44.192 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:44.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:44.294 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:44.320 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:44.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:44.422 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:44.448 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:44.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:44.549 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:44.574 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:44.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:44.676 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:44.701 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:44.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:44.767 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:44 smithi063 ceph-mon[114756]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:26:44.767 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:44 smithi063 ceph-mon[109843]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:26:44.788 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:26:44.789 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:26:44.802 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:44.805 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:26:44.805 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:26:44.822 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:26:44.822 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:26:44.826 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:44.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:44.839 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:26:44.839 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:26:44.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:44 smithi162 ceph-mon[104668]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:26:44.856 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:26:44.856 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:26:44.873 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:26:44.873 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:26:44.889 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:26:44.890 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:26:44.907 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:26:44.907 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:26:44.928 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:44.952 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:44.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:44.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:45.054 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:45.079 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:45.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:45.180 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:45.203 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:45.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:45.305 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:45.330 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:45.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:45.433 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:45.455 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:45.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:45.557 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:45.584 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:45.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:45.686 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:45.709 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:45.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:45.758 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:45 smithi063 ceph-mon[114756]: pgmap v1695: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:45.759 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:45 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1716587898' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:45.759 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:45 smithi063 ceph-mon[109843]: pgmap v1695: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:45.759 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:45 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1716587898' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:45.811 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:45.834 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:45.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:45.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:45 smithi162 ceph-mon[104668]: pgmap v1695: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:45.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:45 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1716587898' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:45.936 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:45.959 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:45.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:45.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:46.022 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:26:46.033 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:46.062 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:46.088 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:46.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:46.189 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:46.212 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:46.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:46.261 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:26:46.313 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:46.334 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:46.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:46.401 DEBUG:teuthology.orchestra.run.smithi063:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2024-08-17T17:26:46.404 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:26:46.404 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:26:46.436 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:46.454 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:hints = 1 2024-08-17T17:26:46.454 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:26:46.454 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Object prefix: benchmark_data_smithi063_263014 2024-08-17T17:26:46.455 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:26:46.455 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:26:46.460 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:46.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:46.562 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:46.601 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:46.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:46.606 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:26:46.682 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:46.683 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:46.703 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:46.730 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:26:46.738 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:46.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:46.840 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:46.881 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:46.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:46.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:46.983 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:47.017 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:47.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:47.119 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:47.155 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:47.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:47.257 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:47.299 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:47.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:47.401 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:47.437 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:47.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:47.455 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 1 16 3766 3750 234.362 234.375 0.00458477 0.00425494 2024-08-17T17:26:47.538 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:47.576 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:47.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:47.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:47 smithi162 ceph-mon[104668]: pgmap v1696: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:47.678 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:47.680 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:47 smithi063 ceph-mon[114756]: pgmap v1696: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:47.681 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:47 smithi063 ceph-mon[109843]: pgmap v1696: 119 pgs: 119 active+clean; 582 KiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-08-17T17:26:47.693 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:47.715 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:47.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:47.818 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:47.857 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:47.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:47.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:47.960 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:48.003 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:48.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:48.105 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:48.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.143 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:48.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:48.244 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:48.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.287 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:48.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:48.387 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:48.421 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:48.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:48.455 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 2 16 7012 6996 218.609 202.875 0.00584406 0.00456601 2024-08-17T17:26:48.466 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:26:48.466 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:26:48.467 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:26:48.467 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:7030 9.4 9:2469073b:::benchmark_data_smithi063_263014_object7029:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-08-17T17:26:48.467 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:26:48.461770+0000", 2024-08-17T17:26:48.467 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.001746173, 2024-08-17T17:26:48.467 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001779334, 2024-08-17T17:26:48.467 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:26:48.467 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:26:48.467 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:26:48.467 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:26:48.467 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:26:48.467 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 7030 2024-08-17T17:26:48.467 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:26:48.467 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:26:48.468 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:26:48.468 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:26:48.468 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:48.461770+0000", 2024-08-17T17:26:48.468 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:26:48.468 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:26:48.468 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:26:48.468 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:26:48.468 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:48.461770+0000", 2024-08-17T17:26:48.468 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:26:48.468 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:26:48.468 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:26:48.468 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:26:48.469 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:48.461768+0000", 2024-08-17T17:26:48.469 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:26:48.469 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:26:48.469 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:26:48.469 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:26:48.469 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:48.461831+0000", 2024-08-17T17:26:48.469 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2491999999999997e-05 2024-08-17T17:26:48.469 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:26:48.469 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:26:48.469 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:26:48.469 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:48.461863+0000", 2024-08-17T17:26:48.469 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2604999999999997e-05 2024-08-17T17:26:48.469 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:26:48.469 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:26:48.470 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:26:48.470 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:48.461872+0000", 2024-08-17T17:26:48.470 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3489999999999992e-06 2024-08-17T17:26:48.470 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:26:48.470 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:26:48.470 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:26:48.470 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:48.461922+0000", 2024-08-17T17:26:48.470 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0454000000000002e-05 2024-08-17T17:26:48.470 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:26:48.470 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:26:48.470 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:26:48.470 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:48.462118+0000", 2024-08-17T17:26:48.470 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00019582900000000001 2024-08-17T17:26:48.471 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:26:48.471 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:26:48.471 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:26:48.471 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:48.462267+0000", 2024-08-17T17:26:48.471 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014956 2024-08-17T17:26:48.471 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:26:48.471 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:26:48.471 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:26:48.471 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:48.463081+0000", 2024-08-17T17:26:48.471 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00081357200000000004 2024-08-17T17:26:48.471 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:26:48.471 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:26:48.472 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:26:48.472 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:26:48.472 INFO:teuthology.orchestra.run.smithi162.stdout: ], 2024-08-17T17:26:48.472 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 1 2024-08-17T17:26:48.472 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:26:48.523 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:48.552 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:48.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:48.654 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:48.691 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:48.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:48.758 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:26:48.793 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:48.825 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:48.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:48.927 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:48.959 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:48.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:48.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:49.062 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:49.103 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:49.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:49.204 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:49.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.244 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:49.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:49.345 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:49.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.386 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:49.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:49.456 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 3 16 9911 9895 206.13 181.188 0.0050547 0.00484225 2024-08-17T17:26:49.487 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:49.500 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:26:49.500 INFO:teuthology.orchestra.run.smithi063.stdout:{"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1488,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1250,"num_osds":8,"num_up_osds":8,"osd_up_since":1723914464,"num_in_osds":8,"osd_in_since":1723915265,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":119}],"num_pgs":119,"num_pools":5,"num_objects":3186,"data_bytes":207689701,"bytes_used":4151996416,"bytes_avail":763808288768,"bytes_total":767960285184,"write_bytes_sec":20699369,"read_op_per_sec":0,"write_op_per_sec":315},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi063.ultpqj","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.63:8443/","prometheus":"http://172.21.15.63:9283/"}},"servicemap":{"epoch":24,"modified":"2024-08-17T17:15:48.097710+0000","services":{}},"progress_events":{}} 2024-08-17T17:26:49.522 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:49.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:49.580 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:49 smithi063 ceph-mon[114756]: pgmap v1697: 119 pgs: 119 active+clean; 198 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s wr, 315 op/s 2024-08-17T17:26:49.581 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:49 smithi063 ceph-mon[109843]: pgmap v1697: 119 pgs: 119 active+clean; 198 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s wr, 315 op/s 2024-08-17T17:26:49.624 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:49.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.662 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:49.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:49.763 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:49.799 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:49.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:49.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:49 smithi162 ceph-mon[104668]: pgmap v1697: 119 pgs: 119 active+clean; 198 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s wr, 315 op/s 2024-08-17T17:26:49.901 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:49.939 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:49.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:49.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:50.040 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:50.075 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:50.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:50.178 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:50.215 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:50.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:50.317 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:50.355 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:50.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:50.455 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 4 15 12737 12722 198.765 176.688 0.00807529 0.00502443 2024-08-17T17:26:50.458 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:50.493 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:50.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:50.583 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:26:50.596 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:50.629 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:50.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:50.715 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2024-08-17T17:26:50.716 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:26:50.731 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:50.756 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:50 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/968181333' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:26:50.757 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:50 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/968181333' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:26:50.767 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:50.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:50.776 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:50.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:50 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/968181333' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:26:50.870 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:50.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:50.913 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:50.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:50.962 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:26:51.013 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:51.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.050 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:51.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:51.151 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:51.180 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:51.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:51.282 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:51.319 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:51.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:51.421 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:51.455 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 5 16 15417 15401 192.495 167.438 0.00304604 0.00518706 2024-08-17T17:26:51.459 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:51.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:51.561 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:51.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.583 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:51.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:51.628 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:26:51.684 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:51.721 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:51.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:51.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:51 smithi063 ceph-mon[114756]: pgmap v1698: 119 pgs: 119 active+clean; 198 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s wr, 315 op/s 2024-08-17T17:26:51.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:51 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3688754672' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:26:51.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:51 smithi063 ceph-mon[109843]: pgmap v1698: 119 pgs: 119 active+clean; 198 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s wr, 315 op/s 2024-08-17T17:26:51.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:51 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3688754672' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:26:51.823 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:51 smithi162 ceph-mon[104668]: pgmap v1698: 119 pgs: 119 active+clean; 198 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 20 MiB/s wr, 315 op/s 2024-08-17T17:26:51.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:51 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3688754672' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:26:51.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.864 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:51.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:51.966 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:51.989 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:51.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:51.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:52.091 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:52.115 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:52.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:52.217 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:52.253 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:52.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:52.354 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:52.387 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:52.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:52.431 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:26:52.455 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 6 16 17575 17559 182.887 134.875 0.00762374 0.00545997 2024-08-17T17:26:52.489 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:52.519 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:52.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:52.621 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:52.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.659 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:52.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:52.760 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:52.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:52 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:52.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:52 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3688754672' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:26:52.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:52 smithi063 ceph-mon[114756]: osdmap e1251: 8 total, 8 up, 8 in 2024-08-17T17:26:52.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:52 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:52.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:52 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3688754672' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:26:52.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:52 smithi063 ceph-mon[109843]: osdmap e1251: 8 total, 8 up, 8 in 2024-08-17T17:26:52.794 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:52.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:52.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:52 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:26:52.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:52 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3688754672' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:26:52.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:52 smithi162 ceph-mon[104668]: osdmap e1251: 8 total, 8 up, 8 in 2024-08-17T17:26:52.896 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:52.924 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:52.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:52.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:53.026 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:53.055 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:53.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:53.157 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:53.195 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:53.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:53.297 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:53.319 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:53.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:53.421 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:53.456 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 7 16 19619 19603 175.009 127.75 0.00712918 0.00570815 2024-08-17T17:26:53.457 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:53.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:53.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:26:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:26:53] "GET /metrics HTTP/1.1" 200 36336 "" "Prometheus/2.51.0" 2024-08-17T17:26:53.559 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:53.596 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:53.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:53.698 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:53.719 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:53.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:53.821 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:53.860 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:53.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:53.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:53.955 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:53 smithi063 ceph-mon[114756]: pgmap v1700: 119 pgs: 119 active+clean; 455 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 46 MiB/s wr, 731 op/s 2024-08-17T17:26:53.955 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:53 smithi063 ceph-mon[109843]: pgmap v1700: 119 pgs: 119 active+clean; 455 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 46 MiB/s wr, 731 op/s 2024-08-17T17:26:53.962 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:54.001 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:54.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:54.087 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:53 smithi162 ceph-mon[104668]: pgmap v1700: 119 pgs: 119 active+clean; 455 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 46 MiB/s wr, 731 op/s 2024-08-17T17:26:54.103 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:54.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.141 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:54.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:54.241 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:54.284 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:54.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:54.386 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:54.416 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:54.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:54.456 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 8 16 21914 21898 171.061 143.438 0.00703451 0.00583953 2024-08-17T17:26:54.519 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:54.553 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:54.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:54.656 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:54.692 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:54.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:54.793 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:54.820 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:54.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:54.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:54.958 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:54.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:54.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:54.999 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:26:55.061 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:55.094 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:55.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:55.157 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:26:55.158 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:26:55.158 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:26:55.158 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:26:55.158 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:26:55.195 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:55.222 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:55.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:55.324 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:55.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.362 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:55.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:55.456 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 9 16 23994 23978 166.498 130 0.00225754 0.0059976 2024-08-17T17:26:55.463 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:55.493 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:55.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:55.596 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:55.618 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:55.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:55.720 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:55.756 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:55.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:55.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:55 smithi063 ceph-mon[114756]: pgmap v1701: 119 pgs: 119 active+clean; 1.0 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-08-17T17:26:55.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:55 smithi063 ceph-mon[109843]: pgmap v1701: 119 pgs: 119 active+clean; 1.0 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-08-17T17:26:55.858 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:55.895 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:55.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:55.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:55.997 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:56.031 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:56.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:55 smithi162 ceph-mon[104668]: pgmap v1701: 119 pgs: 119 active+clean; 1.0 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 104 MiB/s wr, 1.67k op/s 2024-08-17T17:26:56.133 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:56.158 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:56.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:56.262 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:56.297 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:56.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:56.399 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:56.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.438 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:56.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:56.456 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 10 16 25726 25710 160.672 108.25 0.00170612 0.00621297 2024-08-17T17:26:56.540 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:56.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.564 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:56.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:56.665 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:56.702 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:56.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:56.804 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:26:56.835 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:56.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:26:56.937 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:56.958 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:56.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:56.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:57.060 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:57.084 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:57.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:57.186 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:57.221 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:57.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:57.323 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:57.359 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:57.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:57.456 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 11 16 26785 26769 152.082 66.1875 0.00557176 0.00656909 2024-08-17T17:26:57.461 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:57.489 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:57.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:57.591 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:57.628 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:57.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:57.730 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:57.750 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:57.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:57.852 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:57.887 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:57.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:57.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:57.944 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:57 smithi063 ceph-mon[109843]: pgmap v1702: 119 pgs: 119 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-08-17T17:26:57.944 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:57 smithi063 ceph-mon[114756]: pgmap v1702: 119 pgs: 119 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-08-17T17:26:57.989 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:58.018 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:58.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:57 smithi162 ceph-mon[104668]: pgmap v1702: 119 pgs: 119 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2024-08-17T17:26:58.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:58.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.158 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:58.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:58.259 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:26:58.282 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:58.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:26:58.383 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:58.422 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:58.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:58.456 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 12 16 28640 28624 149.069 115.938 0.00587444 0.00670261 2024-08-17T17:26:58.525 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:58.545 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:58.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:58.647 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:58.683 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:58.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:58.785 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:58.821 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:58.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:58.923 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:58.947 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:58.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:58.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:59.049 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:26:59.087 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:59.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:26:59.189 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:26:59.227 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:59.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:26:59.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:58 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2147032288' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:26:58 smithi063 ceph-mon[114756]: pgmap v1703: 119 pgs: 119 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-08-17T17:26:59.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:58 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2147032288' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:59.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:26:58 smithi063 ceph-mon[109843]: pgmap v1703: 119 pgs: 119 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-08-17T17:26:59.299 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:26:59.299 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:26:59.299 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:26:59.300 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:26:59.300 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:26:59.300 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:26:59.300 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:26:59.300 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:26:59.300 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:26:59.300 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:26:59.300 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:26:59.300 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:26:59.301 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:26:59.301 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:26:59.301 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:26:59.301 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:26:59.329 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:26:59.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:58 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2147032288' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:26:59.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:26:58 smithi162 ceph-mon[104668]: pgmap v1703: 119 pgs: 119 active+clean; 1.6 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-08-17T17:26:59.362 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:59.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:26:59.451 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:26:59.456 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 13 16 30760 30744 147.794 132.5 0.00704661 0.00676001 2024-08-17T17:26:59.463 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:26:59.544 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:59.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:26:59.647 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:26:59.683 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:26:59.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:26:59.784 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:59.828 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:26:59.831 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:59.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:26:59.933 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:26:59.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:26:59.970 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:26:59.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:00.071 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:00.086 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:00.106 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:00.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:00.208 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:00.247 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:00.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:00.349 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:00.392 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:00.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:00.456 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 14 16 32879 32863 146.697 132.438 0.00560923 0.00681165 2024-08-17T17:27:00.494 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:00.530 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:00.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:00.632 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:00.679 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:00.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:00.781 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:00.816 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:00.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:00.918 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:00.952 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:00.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:00.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:01.054 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:01.088 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:01.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:01.190 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:01.228 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:01.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:01.262 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:27:01.264 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:27:01.330 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:01.364 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:01.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:01.456 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 15 16 35004 34988 145.77 132.812 0.00726822 0.00685488 2024-08-17T17:27:01.466 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:01.507 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:01.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:01.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:01 smithi063 ceph-mon[109843]: pgmap v1704: 119 pgs: 119 active+clean; 1.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-08-17T17:27:01.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:01 smithi063 ceph-mon[114756]: pgmap v1704: 119 pgs: 119 active+clean; 1.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-08-17T17:27:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:01.609 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:01.645 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:01.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:01.747 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:01.774 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:01 smithi162 ceph-mon[104668]: pgmap v1704: 119 pgs: 119 active+clean; 1.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s 2024-08-17T17:27:01.775 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:01.784 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:01.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:01.821 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:27:01.821 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:27:01.886 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:01.927 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:01.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:01.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:02.028 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:02.061 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:27:02.063 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:02.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.065 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1704,"stamp":"2024-08-17T17:27:00.369155+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1723930597,"num_objects":26322,"num_object_clones":0,"num_object_copies":78966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26360,"num_write_kb":1682977,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1246,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":30974,"ondisk_log_size":30974,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9008184,"kb_used_data":5052600,"kb_used_omap":199,"kb_used_meta":3955320,"kb_avail":740953032,"statfs":{"total":767960285184,"available":758735904768,"internally_reserved":0,"allocated":5173862400,"data_stored":5153275159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":4050248444},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,3,3,2,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1516240896,"num_objects":23136,"num_object_clones":0,"num_object_copies":69408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23136,"num_write_kb":1480704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003031"},"pg_stats":[{"pgid":"1.11","version":"369'149","reported_seq":1574,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801265+0000","last_change":"2024-08-17T17:24:49.721626+0000","last_active":"2024-08-17T17:26:42.801265+0000","last_peered":"2024-08-17T17:26:42.801265+0000","last_clean":"2024-08-17T17:26:42.801265+0000","last_became_active":"2024-08-17T17:24:20.649128+0000","last_became_peered":"2024-08-17T17:24:20.649128+0000","last_unstale":"2024-08-17T17:26:42.801265+0000","last_undegraded":"2024-08-17T17:26:42.801265+0000","last_fullsized":"2024-08-17T17:26:42.801265+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:03:15.343059+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1599,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.643202+0000","last_change":"2024-08-17T17:21:06.463828+0000","last_active":"2024-08-17T17:26:51.643202+0000","last_peered":"2024-08-17T17:26:51.643202+0000","last_clean":"2024-08-17T17:26:51.643202+0000","last_became_active":"2024-08-17T17:21:06.463464+0000","last_became_peered":"2024-08-17T17:21:06.463464+0000","last_unstale":"2024-08-17T17:26:51.643202+0000","last_undegraded":"2024-08-17T17:26:51.643202+0000","last_fullsized":"2024-08-17T17:26:51.643202+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:46.465006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1181,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801823+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:26:42.801823+0000","last_peered":"2024-08-17T17:26:42.801823+0000","last_clean":"2024-08-17T17:26:42.801823+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:26:42.801823+0000","last_undegraded":"2024-08-17T17:26:42.801823+0000","last_fullsized":"2024-08-17T17:26:42.801823+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1505,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801003+0000","last_change":"2024-08-17T17:24:48.765312+0000","last_active":"2024-08-17T17:26:42.801003+0000","last_peered":"2024-08-17T17:26:42.801003+0000","last_clean":"2024-08-17T17:26:42.801003+0000","last_became_active":"2024-08-17T17:24:20.642731+0000","last_became_peered":"2024-08-17T17:24:20.642731+0000","last_unstale":"2024-08-17T17:26:42.801003+0000","last_undegraded":"2024-08-17T17:26:42.801003+0000","last_fullsized":"2024-08-17T17:26:42.801003+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:08:36.138960+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1192,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802338+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:26:42.802338+0000","last_peered":"2024-08-17T17:26:42.802338+0000","last_clean":"2024-08-17T17:26:42.802338+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:26:42.802338+0000","last_undegraded":"2024-08-17T17:26:42.802338+0000","last_fullsized":"2024-08-17T17:26:42.802338+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1048,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:47.421288+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:26:47.421288+0000","last_peered":"2024-08-17T17:26:47.421288+0000","last_clean":"2024-08-17T17:26:47.421288+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:26:47.421288+0000","last_undegraded":"2024-08-17T17:26:47.421288+0000","last_fullsized":"2024-08-17T17:26:47.421288+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":813,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.643514+0000","last_change":"2024-08-17T17:24:50.880046+0000","last_active":"2024-08-17T17:26:51.643514+0000","last_peered":"2024-08-17T17:26:51.643514+0000","last_clean":"2024-08-17T17:26:51.643514+0000","last_became_active":"2024-08-17T17:24:20.652165+0000","last_became_peered":"2024-08-17T17:24:20.652165+0000","last_unstale":"2024-08-17T17:26:51.643514+0000","last_undegraded":"2024-08-17T17:26:51.643514+0000","last_fullsized":"2024-08-17T17:26:51.643514+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:00.416810+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1150,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.643593+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:26:51.643593+0000","last_peered":"2024-08-17T17:26:51.643593+0000","last_clean":"2024-08-17T17:26:51.643593+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:26:51.643593+0000","last_undegraded":"2024-08-17T17:26:51.643593+0000","last_fullsized":"2024-08-17T17:26:51.643593+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1164,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801297+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:26:42.801297+0000","last_peered":"2024-08-17T17:26:42.801297+0000","last_clean":"2024-08-17T17:26:42.801297+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:26:42.801297+0000","last_undegraded":"2024-08-17T17:26:42.801297+0000","last_fullsized":"2024-08-17T17:26:42.801297+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1510,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801556+0000","last_change":"2024-08-17T17:24:50.727183+0000","last_active":"2024-08-17T17:26:42.801556+0000","last_peered":"2024-08-17T17:26:42.801556+0000","last_clean":"2024-08-17T17:26:42.801556+0000","last_became_active":"2024-08-17T17:24:20.646409+0000","last_became_peered":"2024-08-17T17:24:20.646409+0000","last_unstale":"2024-08-17T17:26:42.801556+0000","last_undegraded":"2024-08-17T17:26:42.801556+0000","last_fullsized":"2024-08-17T17:26:42.801556+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.086193+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1598,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.644476+0000","last_change":"2024-08-17T17:21:06.442598+0000","last_active":"2024-08-17T17:26:51.644476+0000","last_peered":"2024-08-17T17:26:51.644476+0000","last_clean":"2024-08-17T17:26:51.644476+0000","last_became_active":"2024-08-17T17:21:06.442264+0000","last_became_peered":"2024-08-17T17:21:06.442264+0000","last_unstale":"2024-08-17T17:26:51.644476+0000","last_undegraded":"2024-08-17T17:26:51.644476+0000","last_fullsized":"2024-08-17T17:26:51.644476+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:35.003620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1241,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426112+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:26:42.426112+0000","last_peered":"2024-08-17T17:26:42.426112+0000","last_clean":"2024-08-17T17:26:42.426112+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:26:42.426112+0000","last_undegraded":"2024-08-17T17:26:42.426112+0000","last_fullsized":"2024-08-17T17:26:42.426112+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"1251'1711","reported_seq":1724,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.709386+0000","last_change":"2024-08-17T17:26:34.519835+0000","last_active":"2024-08-17T17:26:57.709386+0000","last_peered":"2024-08-17T17:26:57.709386+0000","last_clean":"2024-08-17T17:26:57.709386+0000","last_became_active":"2024-08-17T17:26:34.519427+0000","last_became_peered":"2024-08-17T17:26:34.519427+0000","last_unstale":"2024-08-17T17:26:57.709386+0000","last_undegraded":"2024-08-17T17:26:57.709386+0000","last_fullsized":"2024-08-17T17:26:57.709386+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1711,"log_dups_size":0,"ondisk_log_size":1711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:25.352736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112132096,"num_objects":1711,"num_object_clones":0,"num_object_copies":5133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1711,"num_write_kb":109504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1448,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802144+0000","last_change":"2024-08-17T17:24:50.410621+0000","last_active":"2024-08-17T17:26:42.802144+0000","last_peered":"2024-08-17T17:26:42.802144+0000","last_clean":"2024-08-17T17:26:42.802144+0000","last_became_active":"2024-08-17T17:24:31.863062+0000","last_became_peered":"2024-08-17T17:24:31.863062+0000","last_unstale":"2024-08-17T17:26:42.802144+0000","last_undegraded":"2024-08-17T17:26:42.802144+0000","last_fullsized":"2024-08-17T17:26:42.802144+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.410491+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.410491+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:11.717684+0000","scrub_duration":0.0066315480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1206,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801954+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:26:42.801954+0000","last_peered":"2024-08-17T17:26:42.801954+0000","last_clean":"2024-08-17T17:26:42.801954+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:26:42.801954+0000","last_undegraded":"2024-08-17T17:26:42.801954+0000","last_fullsized":"2024-08-17T17:26:42.801954+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1179,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801281+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:26:42.801281+0000","last_peered":"2024-08-17T17:26:42.801281+0000","last_clean":"2024-08-17T17:26:42.801281+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:26:42.801281+0000","last_undegraded":"2024-08-17T17:26:42.801281+0000","last_fullsized":"2024-08-17T17:26:42.801281+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"1251'1659","reported_seq":1672,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.339638+0000","last_change":"2024-08-17T17:26:34.715261+0000","last_active":"2024-08-17T17:26:57.339638+0000","last_peered":"2024-08-17T17:26:57.339638+0000","last_clean":"2024-08-17T17:26:57.339638+0000","last_became_active":"2024-08-17T17:26:34.714853+0000","last_became_peered":"2024-08-17T17:26:34.714853+0000","last_unstale":"2024-08-17T17:26:57.339638+0000","last_undegraded":"2024-08-17T17:26:57.339638+0000","last_fullsized":"2024-08-17T17:26:57.339638+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1659,"log_dups_size":0,"ondisk_log_size":1659,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:18.673096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108724224,"num_objects":1659,"num_object_clones":0,"num_object_copies":4977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1659,"num_write_kb":106176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1681,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.643899+0000","last_change":"2024-08-17T17:24:45.985725+0000","last_active":"2024-08-17T17:26:51.643899+0000","last_peered":"2024-08-17T17:26:51.643899+0000","last_clean":"2024-08-17T17:26:51.643899+0000","last_became_active":"2024-08-17T17:24:31.795836+0000","last_became_peered":"2024-08-17T17:24:31.795836+0000","last_unstale":"2024-08-17T17:26:51.643899+0000","last_undegraded":"2024-08-17T17:26:51.643899+0000","last_fullsized":"2024-08-17T17:26:51.643899+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:45.985596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:45.985596+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:17.897808+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1207,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.800641+0000","last_change":"2024-08-17T17:21:06.540689+0000","last_active":"2024-08-17T17:26:42.800641+0000","last_peered":"2024-08-17T17:26:42.800641+0000","last_clean":"2024-08-17T17:26:42.800641+0000","last_became_active":"2024-08-17T17:21:06.461614+0000","last_became_peered":"2024-08-17T17:21:06.461614+0000","last_unstale":"2024-08-17T17:26:42.800641+0000","last_undegraded":"2024-08-17T17:26:42.800641+0000","last_fullsized":"2024-08-17T17:26:42.800641+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:11.696898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1231,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.427055+0000","last_change":"2024-08-17T17:21:06.470389+0000","last_active":"2024-08-17T17:26:42.427055+0000","last_peered":"2024-08-17T17:26:42.427055+0000","last_clean":"2024-08-17T17:26:42.427055+0000","last_became_active":"2024-08-17T17:21:06.469966+0000","last_became_peered":"2024-08-17T17:21:06.469966+0000","last_unstale":"2024-08-17T17:26:42.427055+0000","last_undegraded":"2024-08-17T17:26:42.427055+0000","last_fullsized":"2024-08-17T17:26:42.427055+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:56.140616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"1251'1755","reported_seq":1768,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.704225+0000","last_change":"2024-08-17T17:26:34.412902+0000","last_active":"2024-08-17T17:26:57.704225+0000","last_peered":"2024-08-17T17:26:57.704225+0000","last_clean":"2024-08-17T17:26:57.704225+0000","last_became_active":"2024-08-17T17:26:34.412543+0000","last_became_peered":"2024-08-17T17:26:34.412543+0000","last_unstale":"2024-08-17T17:26:57.704225+0000","last_undegraded":"2024-08-17T17:26:57.704225+0000","last_fullsized":"2024-08-17T17:26:57.704225+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1755,"log_dups_size":0,"ondisk_log_size":1755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:20.767004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115015680,"num_objects":1755,"num_object_clones":0,"num_object_copies":5265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1755,"num_write_kb":112320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":816,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426889+0000","last_change":"2024-08-17T17:24:48.192447+0000","last_active":"2024-08-17T17:26:42.426889+0000","last_peered":"2024-08-17T17:26:42.426889+0000","last_clean":"2024-08-17T17:26:42.426889+0000","last_became_active":"2024-08-17T17:24:20.644240+0000","last_became_peered":"2024-08-17T17:24:20.644240+0000","last_unstale":"2024-08-17T17:26:42.426889+0000","last_undegraded":"2024-08-17T17:26:42.426889+0000","last_fullsized":"2024-08-17T17:26:42.426889+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:56.795337+0000","scrub_duration":0.010920255,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1165,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802332+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:26:42.802332+0000","last_peered":"2024-08-17T17:26:42.802332+0000","last_clean":"2024-08-17T17:26:42.802332+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:26:42.802332+0000","last_undegraded":"2024-08-17T17:26:42.802332+0000","last_fullsized":"2024-08-17T17:26:42.802332+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1213,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426503+0000","last_change":"2024-08-17T17:21:06.446798+0000","last_active":"2024-08-17T17:26:42.426503+0000","last_peered":"2024-08-17T17:26:42.426503+0000","last_clean":"2024-08-17T17:26:42.426503+0000","last_became_active":"2024-08-17T17:21:06.446205+0000","last_became_peered":"2024-08-17T17:21:06.446205+0000","last_unstale":"2024-08-17T17:26:42.426503+0000","last_undegraded":"2024-08-17T17:26:42.426503+0000","last_fullsized":"2024-08-17T17:26:42.426503+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:28.584048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1251'1704","reported_seq":1717,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.715302+0000","last_change":"2024-08-17T17:26:35.023050+0000","last_active":"2024-08-17T17:26:57.715302+0000","last_peered":"2024-08-17T17:26:57.715302+0000","last_clean":"2024-08-17T17:26:57.715302+0000","last_became_active":"2024-08-17T17:26:35.022784+0000","last_became_peered":"2024-08-17T17:26:35.022784+0000","last_unstale":"2024-08-17T17:26:57.715302+0000","last_undegraded":"2024-08-17T17:26:57.715302+0000","last_fullsized":"2024-08-17T17:26:57.715302+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1704,"log_dups_size":0,"ondisk_log_size":1704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:04.319458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111673344,"num_objects":1704,"num_object_clones":0,"num_object_copies":5112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1704,"num_write_kb":109056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":724,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426223+0000","last_change":"2024-08-17T17:24:52.102415+0000","last_active":"2024-08-17T17:26:42.426223+0000","last_peered":"2024-08-17T17:26:42.426223+0000","last_clean":"2024-08-17T17:26:42.426223+0000","last_became_active":"2024-08-17T17:24:31.795843+0000","last_became_peered":"2024-08-17T17:24:31.795843+0000","last_unstale":"2024-08-17T17:26:42.426223+0000","last_undegraded":"2024-08-17T17:26:42.426223+0000","last_fullsized":"2024-08-17T17:26:42.426223+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:32:08.434197+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1194,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802135+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:26:42.802135+0000","last_peered":"2024-08-17T17:26:42.802135+0000","last_clean":"2024-08-17T17:26:42.802135+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:26:42.802135+0000","last_undegraded":"2024-08-17T17:26:42.802135+0000","last_fullsized":"2024-08-17T17:26:42.802135+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":976,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426326+0000","last_change":"2024-08-17T17:21:06.460905+0000","last_active":"2024-08-17T17:26:42.426326+0000","last_peered":"2024-08-17T17:26:42.426326+0000","last_clean":"2024-08-17T17:26:42.426326+0000","last_became_active":"2024-08-17T17:21:06.460592+0000","last_became_peered":"2024-08-17T17:21:06.460592+0000","last_unstale":"2024-08-17T17:26:42.426326+0000","last_undegraded":"2024-08-17T17:26:42.426326+0000","last_fullsized":"2024-08-17T17:26:42.426326+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:10.146916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.1","version":"1251'1471","reported_seq":1490,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:55.668635+0000","last_change":"2024-08-17T17:26:34.410513+0000","last_active":"2024-08-17T17:26:55.668635+0000","last_peered":"2024-08-17T17:26:55.668635+0000","last_clean":"2024-08-17T17:26:55.668635+0000","last_became_active":"2024-08-17T17:26:34.410199+0000","last_became_peered":"2024-08-17T17:26:34.410199+0000","last_unstale":"2024-08-17T17:26:55.668635+0000","last_undegraded":"2024-08-17T17:26:55.668635+0000","last_fullsized":"2024-08-17T17:26:55.668635+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1471,"log_dups_size":0,"ondisk_log_size":1471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:35.837825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96403456,"num_objects":1471,"num_object_clones":0,"num_object_copies":4413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1471,"num_write_kb":94144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1594,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801103+0000","last_change":"2024-08-17T17:24:54.418765+0000","last_active":"2024-08-17T17:26:42.801103+0000","last_peered":"2024-08-17T17:26:42.801103+0000","last_clean":"2024-08-17T17:26:42.801103+0000","last_became_active":"2024-08-17T17:24:31.800263+0000","last_became_peered":"2024-08-17T17:24:31.800263+0000","last_unstale":"2024-08-17T17:26:42.801103+0000","last_undegraded":"2024-08-17T17:26:42.801103+0000","last_fullsized":"2024-08-17T17:26:42.801103+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:54.418644+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:54.418644+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:11.024294+0000","scrub_duration":0.013346135,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1620,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.642779+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:26:51.642779+0000","last_peered":"2024-08-17T17:26:51.642779+0000","last_clean":"2024-08-17T17:26:51.642779+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:26:51.642779+0000","last_undegraded":"2024-08-17T17:26:51.642779+0000","last_fullsized":"2024-08-17T17:26:51.642779+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1197,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.427219+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:26:42.427219+0000","last_peered":"2024-08-17T17:26:42.427219+0000","last_clean":"2024-08-17T17:26:42.427219+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:26:42.427219+0000","last_undegraded":"2024-08-17T17:26:42.427219+0000","last_fullsized":"2024-08-17T17:26:42.427219+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"1251'1655","reported_seq":1668,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.417718+0000","last_change":"2024-08-17T17:26:35.023056+0000","last_active":"2024-08-17T17:26:57.417718+0000","last_peered":"2024-08-17T17:26:57.417718+0000","last_clean":"2024-08-17T17:26:57.417718+0000","last_became_active":"2024-08-17T17:26:35.022671+0000","last_became_peered":"2024-08-17T17:26:35.022671+0000","last_unstale":"2024-08-17T17:26:57.417718+0000","last_undegraded":"2024-08-17T17:26:57.417718+0000","last_fullsized":"2024-08-17T17:26:57.417718+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1655,"log_dups_size":0,"ondisk_log_size":1655,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:43.531150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108462080,"num_objects":1655,"num_object_clones":0,"num_object_copies":4965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1655,"num_write_kb":105920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1540,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801088+0000","last_change":"2024-08-17T17:24:58.712331+0000","last_active":"2024-08-17T17:26:42.801088+0000","last_peered":"2024-08-17T17:26:42.801088+0000","last_clean":"2024-08-17T17:26:42.801088+0000","last_became_active":"2024-08-17T17:24:31.802584+0000","last_became_peered":"2024-08-17T17:24:31.802584+0000","last_unstale":"2024-08-17T17:26:42.801088+0000","last_undegraded":"2024-08-17T17:26:42.801088+0000","last_fullsized":"2024-08-17T17:26:42.801088+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:59.616217+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1214,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426669+0000","last_change":"2024-08-17T17:21:06.446841+0000","last_active":"2024-08-17T17:26:42.426669+0000","last_peered":"2024-08-17T17:26:42.426669+0000","last_clean":"2024-08-17T17:26:42.426669+0000","last_became_active":"2024-08-17T17:21:06.446271+0000","last_became_peered":"2024-08-17T17:21:06.446271+0000","last_unstale":"2024-08-17T17:26:42.426669+0000","last_undegraded":"2024-08-17T17:26:42.426669+0000","last_fullsized":"2024-08-17T17:26:42.426669+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:20.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1230,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426472+0000","last_change":"2024-08-17T17:21:06.480911+0000","last_active":"2024-08-17T17:26:42.426472+0000","last_peered":"2024-08-17T17:26:42.426472+0000","last_clean":"2024-08-17T17:26:42.426472+0000","last_became_active":"2024-08-17T17:21:06.480551+0000","last_became_peered":"2024-08-17T17:21:06.480551+0000","last_unstale":"2024-08-17T17:26:42.426472+0000","last_undegraded":"2024-08-17T17:26:42.426472+0000","last_fullsized":"2024-08-17T17:26:42.426472+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:26.003704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"1251'1670","reported_seq":1683,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.407297+0000","last_change":"2024-08-17T17:26:34.715560+0000","last_active":"2024-08-17T17:26:57.407297+0000","last_peered":"2024-08-17T17:26:57.407297+0000","last_clean":"2024-08-17T17:26:57.407297+0000","last_became_active":"2024-08-17T17:26:34.714955+0000","last_became_peered":"2024-08-17T17:26:34.714955+0000","last_unstale":"2024-08-17T17:26:57.407297+0000","last_undegraded":"2024-08-17T17:26:57.407297+0000","last_fullsized":"2024-08-17T17:26:57.407297+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1670,"log_dups_size":0,"ondisk_log_size":1670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:46.367495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109445120,"num_objects":1670,"num_object_clones":0,"num_object_copies":5010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1670,"num_write_kb":106880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":773,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801670+0000","last_change":"2024-08-17T17:24:57.682573+0000","last_active":"2024-08-17T17:26:42.801670+0000","last_peered":"2024-08-17T17:26:42.801670+0000","last_clean":"2024-08-17T17:26:42.801670+0000","last_became_active":"2024-08-17T17:24:31.869575+0000","last_became_peered":"2024-08-17T17:24:31.869575+0000","last_unstale":"2024-08-17T17:26:42.801670+0000","last_undegraded":"2024-08-17T17:26:42.801670+0000","last_fullsized":"2024-08-17T17:26:42.801670+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:57.682409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:57.682409+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:42.144615+0000","scrub_duration":0.0066879779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1199,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.427134+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:26:42.427134+0000","last_peered":"2024-08-17T17:26:42.427134+0000","last_clean":"2024-08-17T17:26:42.427134+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:26:42.427134+0000","last_undegraded":"2024-08-17T17:26:42.427134+0000","last_fullsized":"2024-08-17T17:26:42.427134+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1215,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.427476+0000","last_change":"2024-08-17T17:21:06.451521+0000","last_active":"2024-08-17T17:26:42.427476+0000","last_peered":"2024-08-17T17:26:42.427476+0000","last_clean":"2024-08-17T17:26:42.427476+0000","last_became_active":"2024-08-17T17:21:06.451197+0000","last_became_peered":"2024-08-17T17:21:06.451197+0000","last_unstale":"2024-08-17T17:26:42.427476+0000","last_undegraded":"2024-08-17T17:26:42.427476+0000","last_fullsized":"2024-08-17T17:26:42.427476+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:04.000692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"1251'1502","reported_seq":1521,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:55.667806+0000","last_change":"2024-08-17T17:26:35.023654+0000","last_active":"2024-08-17T17:26:55.667806+0000","last_peered":"2024-08-17T17:26:55.667806+0000","last_clean":"2024-08-17T17:26:55.667806+0000","last_became_active":"2024-08-17T17:26:35.022968+0000","last_became_peered":"2024-08-17T17:26:35.022968+0000","last_unstale":"2024-08-17T17:26:55.667806+0000","last_undegraded":"2024-08-17T17:26:55.667806+0000","last_fullsized":"2024-08-17T17:26:55.667806+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1502,"log_dups_size":0,"ondisk_log_size":1502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:25:10.494807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98435072,"num_objects":1502,"num_object_clones":0,"num_object_copies":4506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1502,"num_write_kb":96128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1540,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801483+0000","last_change":"2024-08-17T17:24:52.716059+0000","last_active":"2024-08-17T17:26:42.801483+0000","last_peered":"2024-08-17T17:26:42.801483+0000","last_clean":"2024-08-17T17:26:42.801483+0000","last_became_active":"2024-08-17T17:24:31.798173+0000","last_became_peered":"2024-08-17T17:24:31.798173+0000","last_unstale":"2024-08-17T17:26:42.801483+0000","last_undegraded":"2024-08-17T17:26:42.801483+0000","last_fullsized":"2024-08-17T17:26:42.801483+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:04:56.094994+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1601,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.642611+0000","last_change":"2024-08-17T17:21:06.441118+0000","last_active":"2024-08-17T17:26:51.642611+0000","last_peered":"2024-08-17T17:26:51.642611+0000","last_clean":"2024-08-17T17:26:51.642611+0000","last_became_active":"2024-08-17T17:21:06.440800+0000","last_became_peered":"2024-08-17T17:21:06.440800+0000","last_unstale":"2024-08-17T17:26:51.642611+0000","last_undegraded":"2024-08-17T17:26:51.642611+0000","last_fullsized":"2024-08-17T17:26:51.642611+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:08.091528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":107,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.646730+0000","last_change":"2024-08-17T17:22:48.374766+0000","last_active":"2024-08-17T17:26:51.646730+0000","last_peered":"2024-08-17T17:26:51.646730+0000","last_clean":"2024-08-17T17:26:51.646730+0000","last_became_active":"2024-08-17T17:22:48.373556+0000","last_became_peered":"2024-08-17T17:22:48.373556+0000","last_unstale":"2024-08-17T17:26:51.646730+0000","last_undegraded":"2024-08-17T17:26:51.646730+0000","last_fullsized":"2024-08-17T17:26:51.646730+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:33.362080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.8","version":"1251'1541","reported_seq":1560,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:55.671243+0000","last_change":"2024-08-17T17:26:34.519719+0000","last_active":"2024-08-17T17:26:55.671243+0000","last_peered":"2024-08-17T17:26:55.671243+0000","last_clean":"2024-08-17T17:26:55.671243+0000","last_became_active":"2024-08-17T17:26:34.519368+0000","last_became_peered":"2024-08-17T17:26:34.519368+0000","last_unstale":"2024-08-17T17:26:55.671243+0000","last_undegraded":"2024-08-17T17:26:55.671243+0000","last_fullsized":"2024-08-17T17:26:55.671243+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1541,"log_dups_size":0,"ondisk_log_size":1541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:44.575457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100990976,"num_objects":1541,"num_object_clones":0,"num_object_copies":4623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1541,"num_write_kb":98624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1595,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801010+0000","last_change":"2024-08-17T17:24:13.542832+0000","last_active":"2024-08-17T17:26:42.801010+0000","last_peered":"2024-08-17T17:26:42.801010+0000","last_clean":"2024-08-17T17:26:42.801010+0000","last_became_active":"2024-08-17T17:24:11.570282+0000","last_became_peered":"2024-08-17T17:24:11.570282+0000","last_unstale":"2024-08-17T17:26:42.801010+0000","last_undegraded":"2024-08-17T17:26:42.801010+0000","last_fullsized":"2024-08-17T17:26:42.801010+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:49.414162+0000","scrub_duration":0.017597167,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1209,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801889+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:26:42.801889+0000","last_peered":"2024-08-17T17:26:42.801889+0000","last_clean":"2024-08-17T17:26:42.801889+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:26:42.801889+0000","last_undegraded":"2024-08-17T17:26:42.801889+0000","last_fullsized":"2024-08-17T17:26:42.801889+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1567,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.642491+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:26:51.642491+0000","last_peered":"2024-08-17T17:26:51.642491+0000","last_clean":"2024-08-17T17:26:51.642491+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:26:51.642491+0000","last_undegraded":"2024-08-17T17:26:51.642491+0000","last_fullsized":"2024-08-17T17:26:51.642491+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1180,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.427596+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:26:42.427596+0000","last_peered":"2024-08-17T17:26:42.427596+0000","last_clean":"2024-08-17T17:26:42.427596+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:26:42.427596+0000","last_undegraded":"2024-08-17T17:26:42.427596+0000","last_fullsized":"2024-08-17T17:26:42.427596+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"1251'1695","reported_seq":1708,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.405986+0000","last_change":"2024-08-17T17:26:34.519823+0000","last_active":"2024-08-17T17:26:57.405986+0000","last_peered":"2024-08-17T17:26:57.405986+0000","last_clean":"2024-08-17T17:26:57.405986+0000","last_became_active":"2024-08-17T17:26:34.519497+0000","last_became_peered":"2024-08-17T17:26:34.519497+0000","last_unstale":"2024-08-17T17:26:57.405986+0000","last_undegraded":"2024-08-17T17:26:57.405986+0000","last_fullsized":"2024-08-17T17:26:57.405986+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1695,"log_dups_size":0,"ondisk_log_size":1695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:59:31.807854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111083520,"num_objects":1695,"num_object_clones":0,"num_object_copies":5085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1695,"num_write_kb":108480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":834,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:47.440094+0000","last_change":"2024-08-17T17:24:11.936322+0000","last_active":"2024-08-17T17:26:47.440094+0000","last_peered":"2024-08-17T17:26:47.440094+0000","last_clean":"2024-08-17T17:26:47.440094+0000","last_became_active":"2024-08-17T17:24:11.574846+0000","last_became_peered":"2024-08-17T17:24:11.574846+0000","last_unstale":"2024-08-17T17:26:47.440094+0000","last_undegraded":"2024-08-17T17:26:47.440094+0000","last_fullsized":"2024-08-17T17:26:47.440094+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:11.936236+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:11.936236+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:31.674851+0000","scrub_duration":0.0035797139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1245,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801078+0000","last_change":"2024-08-17T17:21:06.430639+0000","last_active":"2024-08-17T17:26:42.801078+0000","last_peered":"2024-08-17T17:26:42.801078+0000","last_clean":"2024-08-17T17:26:42.801078+0000","last_became_active":"2024-08-17T17:21:06.430297+0000","last_became_peered":"2024-08-17T17:21:06.430297+0000","last_unstale":"2024-08-17T17:26:42.801078+0000","last_undegraded":"2024-08-17T17:26:42.801078+0000","last_fullsized":"2024-08-17T17:26:42.801078+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:57.794620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1048,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426778+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:26:42.426778+0000","last_peered":"2024-08-17T17:26:42.426778+0000","last_clean":"2024-08-17T17:26:42.426778+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:26:42.426778+0000","last_undegraded":"2024-08-17T17:26:42.426778+0000","last_fullsized":"2024-08-17T17:26:42.426778+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":59,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802435+0000","last_change":"2024-08-17T17:22:48.370769+0000","last_active":"2024-08-17T17:26:42.802435+0000","last_peered":"2024-08-17T17:26:42.802435+0000","last_clean":"2024-08-17T17:26:42.802435+0000","last_became_active":"2024-08-17T17:22:48.370531+0000","last_became_peered":"2024-08-17T17:22:48.370531+0000","last_unstale":"2024-08-17T17:26:42.802435+0000","last_undegraded":"2024-08-17T17:26:42.802435+0000","last_fullsized":"2024-08-17T17:26:42.802435+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:02.943823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"1251'1700","reported_seq":1713,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.716308+0000","last_change":"2024-08-17T17:26:34.715167+0000","last_active":"2024-08-17T17:26:57.716308+0000","last_peered":"2024-08-17T17:26:57.716308+0000","last_clean":"2024-08-17T17:26:57.716308+0000","last_became_active":"2024-08-17T17:26:34.714868+0000","last_became_peered":"2024-08-17T17:26:34.714868+0000","last_unstale":"2024-08-17T17:26:57.716308+0000","last_undegraded":"2024-08-17T17:26:57.716308+0000","last_fullsized":"2024-08-17T17:26:57.716308+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1700,"log_dups_size":0,"ondisk_log_size":1700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:19.957252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111411200,"num_objects":1700,"num_object_clones":0,"num_object_copies":5100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1700,"num_write_kb":108800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1616,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801433+0000","last_change":"2024-08-17T17:24:12.519946+0000","last_active":"2024-08-17T17:26:42.801433+0000","last_peered":"2024-08-17T17:26:42.801433+0000","last_clean":"2024-08-17T17:26:42.801433+0000","last_became_active":"2024-08-17T17:24:11.569747+0000","last_became_peered":"2024-08-17T17:24:11.569747+0000","last_unstale":"2024-08-17T17:26:42.801433+0000","last_undegraded":"2024-08-17T17:26:42.801433+0000","last_fullsized":"2024-08-17T17:26:42.801433+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.519838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.519838+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:56.003993+0000","scrub_duration":0.0068224740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1664,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.645432+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:26:51.645432+0000","last_peered":"2024-08-17T17:26:51.645432+0000","last_clean":"2024-08-17T17:26:51.645432+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:26:51.645432+0000","last_undegraded":"2024-08-17T17:26:51.645432+0000","last_fullsized":"2024-08-17T17:26:51.645432+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1056,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.645821+0000","last_change":"2024-08-17T17:21:06.428684+0000","last_active":"2024-08-17T17:26:51.645821+0000","last_peered":"2024-08-17T17:26:51.645821+0000","last_clean":"2024-08-17T17:26:51.645821+0000","last_became_active":"2024-08-17T17:21:06.428178+0000","last_became_peered":"2024-08-17T17:21:06.428178+0000","last_unstale":"2024-08-17T17:26:51.645821+0000","last_undegraded":"2024-08-17T17:26:51.645821+0000","last_fullsized":"2024-08-17T17:26:51.645821+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:14.836119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":892,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.800780+0000","last_change":"2024-08-17T17:21:06.443565+0000","last_active":"2024-08-17T17:26:42.800780+0000","last_peered":"2024-08-17T17:26:42.800780+0000","last_clean":"2024-08-17T17:26:42.800780+0000","last_became_active":"2024-08-17T17:21:06.443215+0000","last_became_peered":"2024-08-17T17:21:06.443215+0000","last_unstale":"2024-08-17T17:26:42.800780+0000","last_undegraded":"2024-08-17T17:26:42.800780+0000","last_fullsized":"2024-08-17T17:26:42.800780+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:47.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"1251'1704","reported_seq":1717,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.417913+0000","last_change":"2024-08-17T17:26:34.519795+0000","last_active":"2024-08-17T17:26:57.417913+0000","last_peered":"2024-08-17T17:26:57.417913+0000","last_clean":"2024-08-17T17:26:57.417913+0000","last_became_active":"2024-08-17T17:26:34.519426+0000","last_became_peered":"2024-08-17T17:26:34.519426+0000","last_unstale":"2024-08-17T17:26:57.417913+0000","last_undegraded":"2024-08-17T17:26:57.417913+0000","last_fullsized":"2024-08-17T17:26:57.417913+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1704,"log_dups_size":0,"ondisk_log_size":1704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:14.719799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111673344,"num_objects":1704,"num_object_clones":0,"num_object_copies":5112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1704,"num_write_kb":109056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1558,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.645243+0000","last_change":"2024-08-17T17:24:13.795227+0000","last_active":"2024-08-17T17:26:51.645243+0000","last_peered":"2024-08-17T17:26:51.645243+0000","last_clean":"2024-08-17T17:26:51.645243+0000","last_became_active":"2024-08-17T17:24:11.578352+0000","last_became_peered":"2024-08-17T17:24:11.578352+0000","last_unstale":"2024-08-17T17:26:51.645243+0000","last_undegraded":"2024-08-17T17:26:51.645243+0000","last_fullsized":"2024-08-17T17:26:51.645243+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:36:39.440808+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1236,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.800685+0000","last_change":"2024-08-17T17:21:06.456978+0000","last_active":"2024-08-17T17:26:42.800685+0000","last_peered":"2024-08-17T17:26:42.800685+0000","last_clean":"2024-08-17T17:26:42.800685+0000","last_became_active":"2024-08-17T17:21:06.452265+0000","last_became_peered":"2024-08-17T17:21:06.452265+0000","last_unstale":"2024-08-17T17:26:42.800685+0000","last_undegraded":"2024-08-17T17:26:42.800685+0000","last_fullsized":"2024-08-17T17:26:42.800685+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:52.893746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1218,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.640706+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:26:51.640706+0000","last_peered":"2024-08-17T17:26:51.640706+0000","last_clean":"2024-08-17T17:26:51.640706+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:26:51.640706+0000","last_undegraded":"2024-08-17T17:26:51.640706+0000","last_fullsized":"2024-08-17T17:26:51.640706+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1132,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.646271+0000","last_change":"2024-08-17T17:21:06.435754+0000","last_active":"2024-08-17T17:26:51.646271+0000","last_peered":"2024-08-17T17:26:51.646271+0000","last_clean":"2024-08-17T17:26:51.646271+0000","last_became_active":"2024-08-17T17:21:06.435442+0000","last_became_peered":"2024-08-17T17:21:06.435442+0000","last_unstale":"2024-08-17T17:26:51.646271+0000","last_undegraded":"2024-08-17T17:26:51.646271+0000","last_fullsized":"2024-08-17T17:26:51.646271+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:34.849042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"1251'1645","reported_seq":1658,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.721945+0000","last_change":"2024-08-17T17:26:34.716730+0000","last_active":"2024-08-17T17:26:57.721945+0000","last_peered":"2024-08-17T17:26:57.721945+0000","last_clean":"2024-08-17T17:26:57.721945+0000","last_became_active":"2024-08-17T17:26:34.716457+0000","last_became_peered":"2024-08-17T17:26:34.716457+0000","last_unstale":"2024-08-17T17:26:57.721945+0000","last_undegraded":"2024-08-17T17:26:57.721945+0000","last_fullsized":"2024-08-17T17:26:57.721945+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1645,"log_dups_size":0,"ondisk_log_size":1645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:51.133413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107806720,"num_objects":1645,"num_object_clones":0,"num_object_copies":4935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1645,"num_write_kb":105280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":820,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426844+0000","last_change":"2024-08-17T17:24:55.041464+0000","last_active":"2024-08-17T17:26:42.426844+0000","last_peered":"2024-08-17T17:26:42.426844+0000","last_clean":"2024-08-17T17:26:42.426844+0000","last_became_active":"2024-08-17T17:24:31.800289+0000","last_became_peered":"2024-08-17T17:24:31.800289+0000","last_unstale":"2024-08-17T17:26:42.426844+0000","last_undegraded":"2024-08-17T17:26:42.426844+0000","last_fullsized":"2024-08-17T17:26:42.426844+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:55.041378+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:55.041378+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:43.974501+0000","scrub_duration":0.001623642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1210,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802417+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:26:42.802417+0000","last_peered":"2024-08-17T17:26:42.802417+0000","last_clean":"2024-08-17T17:26:42.802417+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:26:42.802417+0000","last_undegraded":"2024-08-17T17:26:42.802417+0000","last_fullsized":"2024-08-17T17:26:42.802417+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":901,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802036+0000","last_change":"2024-08-17T17:21:06.434991+0000","last_active":"2024-08-17T17:26:42.802036+0000","last_peered":"2024-08-17T17:26:42.802036+0000","last_clean":"2024-08-17T17:26:42.802036+0000","last_became_active":"2024-08-17T17:21:06.434630+0000","last_became_peered":"2024-08-17T17:21:06.434630+0000","last_unstale":"2024-08-17T17:26:42.802036+0000","last_undegraded":"2024-08-17T17:26:42.802036+0000","last_fullsized":"2024-08-17T17:26:42.802036+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:24.104609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":843,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.800632+0000","last_change":"2024-08-17T17:21:06.462532+0000","last_active":"2024-08-17T17:26:42.800632+0000","last_peered":"2024-08-17T17:26:42.800632+0000","last_clean":"2024-08-17T17:26:42.800632+0000","last_became_active":"2024-08-17T17:21:06.462183+0000","last_became_peered":"2024-08-17T17:21:06.462183+0000","last_unstale":"2024-08-17T17:26:42.800632+0000","last_undegraded":"2024-08-17T17:26:42.800632+0000","last_fullsized":"2024-08-17T17:26:42.800632+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:50.033322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.e","version":"1251'1775","reported_seq":1788,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.421430+0000","last_change":"2024-08-17T17:26:34.716166+0000","last_active":"2024-08-17T17:26:57.421430+0000","last_peered":"2024-08-17T17:26:57.421430+0000","last_clean":"2024-08-17T17:26:57.421430+0000","last_became_active":"2024-08-17T17:26:34.715299+0000","last_became_peered":"2024-08-17T17:26:34.715299+0000","last_unstale":"2024-08-17T17:26:57.421430+0000","last_undegraded":"2024-08-17T17:26:57.421430+0000","last_fullsized":"2024-08-17T17:26:57.421430+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1775,"log_dups_size":0,"ondisk_log_size":1775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:37.004999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116326400,"num_objects":1775,"num_object_clones":0,"num_object_copies":5325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1775,"num_write_kb":113600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":837,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.427072+0000","last_change":"2024-08-17T17:24:53.103270+0000","last_active":"2024-08-17T17:26:42.427072+0000","last_peered":"2024-08-17T17:26:42.427072+0000","last_clean":"2024-08-17T17:26:42.427072+0000","last_became_active":"2024-08-17T17:24:31.800823+0000","last_became_peered":"2024-08-17T17:24:31.800823+0000","last_unstale":"2024-08-17T17:26:42.427072+0000","last_undegraded":"2024-08-17T17:26:42.427072+0000","last_fullsized":"2024-08-17T17:26:42.427072+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.103133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.103133+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:52.444781+0000","scrub_duration":0.0074281759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1055,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.645990+0000","last_change":"2024-08-17T17:21:06.433720+0000","last_active":"2024-08-17T17:26:51.645990+0000","last_peered":"2024-08-17T17:26:51.645990+0000","last_clean":"2024-08-17T17:26:51.645990+0000","last_became_active":"2024-08-17T17:21:06.433292+0000","last_became_peered":"2024-08-17T17:21:06.433292+0000","last_unstale":"2024-08-17T17:26:51.645990+0000","last_undegraded":"2024-08-17T17:26:51.645990+0000","last_fullsized":"2024-08-17T17:26:51.645990+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:32.246594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1211,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802406+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:26:42.802406+0000","last_peered":"2024-08-17T17:26:42.802406+0000","last_clean":"2024-08-17T17:26:42.802406+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:26:42.802406+0000","last_undegraded":"2024-08-17T17:26:42.802406+0000","last_fullsized":"2024-08-17T17:26:42.802406+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":803,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801746+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:26:42.801746+0000","last_peered":"2024-08-17T17:26:42.801746+0000","last_clean":"2024-08-17T17:26:42.801746+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:26:42.801746+0000","last_undegraded":"2024-08-17T17:26:42.801746+0000","last_fullsized":"2024-08-17T17:26:42.801746+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"1251'1679","reported_seq":1692,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.410134+0000","last_change":"2024-08-17T17:26:34.715123+0000","last_active":"2024-08-17T17:26:57.410134+0000","last_peered":"2024-08-17T17:26:57.410134+0000","last_clean":"2024-08-17T17:26:57.410134+0000","last_became_active":"2024-08-17T17:26:34.714789+0000","last_became_peered":"2024-08-17T17:26:34.714789+0000","last_unstale":"2024-08-17T17:26:57.410134+0000","last_undegraded":"2024-08-17T17:26:57.410134+0000","last_fullsized":"2024-08-17T17:26:57.410134+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1679,"log_dups_size":0,"ondisk_log_size":1679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:19.314570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110034944,"num_objects":1679,"num_object_clones":0,"num_object_copies":5037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1679,"num_write_kb":107456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1562,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.640953+0000","last_change":"2024-08-17T17:24:49.119578+0000","last_active":"2024-08-17T17:26:51.640953+0000","last_peered":"2024-08-17T17:26:51.640953+0000","last_clean":"2024-08-17T17:26:51.640953+0000","last_became_active":"2024-08-17T17:24:31.807987+0000","last_became_peered":"2024-08-17T17:24:31.807987+0000","last_unstale":"2024-08-17T17:26:51.640953+0000","last_undegraded":"2024-08-17T17:26:51.640953+0000","last_fullsized":"2024-08-17T17:26:51.640953+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:34.052786+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1620,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.644696+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:26:51.644696+0000","last_peered":"2024-08-17T17:26:51.644696+0000","last_clean":"2024-08-17T17:26:51.644696+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:26:51.644696+0000","last_undegraded":"2024-08-17T17:26:51.644696+0000","last_fullsized":"2024-08-17T17:26:51.644696+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":853,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.427255+0000","last_change":"2024-08-17T17:21:06.452395+0000","last_active":"2024-08-17T17:26:42.427255+0000","last_peered":"2024-08-17T17:26:42.427255+0000","last_clean":"2024-08-17T17:26:42.427255+0000","last_became_active":"2024-08-17T17:21:06.451867+0000","last_became_peered":"2024-08-17T17:21:06.451867+0000","last_unstale":"2024-08-17T17:26:42.427255+0000","last_undegraded":"2024-08-17T17:26:42.427255+0000","last_fullsized":"2024-08-17T17:26:42.427255+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:50.597092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1610,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.642347+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:26:51.642347+0000","last_peered":"2024-08-17T17:26:51.642347+0000","last_clean":"2024-08-17T17:26:51.642347+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:26:51.642347+0000","last_undegraded":"2024-08-17T17:26:51.642347+0000","last_fullsized":"2024-08-17T17:26:51.642347+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"1251'1430","reported_seq":1449,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:55.659497+0000","last_change":"2024-08-17T17:26:34.412017+0000","last_active":"2024-08-17T17:26:55.659497+0000","last_peered":"2024-08-17T17:26:55.659497+0000","last_clean":"2024-08-17T17:26:55.659497+0000","last_became_active":"2024-08-17T17:26:34.411768+0000","last_became_peered":"2024-08-17T17:26:34.411768+0000","last_unstale":"2024-08-17T17:26:55.659497+0000","last_undegraded":"2024-08-17T17:26:55.659497+0000","last_fullsized":"2024-08-17T17:26:55.659497+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":1430,"log_dups_size":0,"ondisk_log_size":1430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:06.384935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93716480,"num_objects":1430,"num_object_clones":0,"num_object_copies":4290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1430,"num_write_kb":91520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1646,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.642966+0000","last_change":"2024-08-17T17:24:12.033217+0000","last_active":"2024-08-17T17:26:51.642966+0000","last_peered":"2024-08-17T17:26:51.642966+0000","last_clean":"2024-08-17T17:26:51.642966+0000","last_became_active":"2024-08-17T17:24:11.572877+0000","last_became_peered":"2024-08-17T17:24:11.572877+0000","last_unstale":"2024-08-17T17:26:51.642966+0000","last_undegraded":"2024-08-17T17:26:51.642966+0000","last_fullsized":"2024-08-17T17:26:51.642966+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.033101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.033101+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.264475+0000","scrub_duration":20,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":222,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.427435+0000","last_change":"2024-08-17T17:21:06.519965+0000","last_active":"2024-08-17T17:26:42.427435+0000","last_peered":"2024-08-17T17:26:42.427435+0000","last_clean":"2024-08-17T17:26:42.427435+0000","last_became_active":"2024-08-17T17:21:06.459952+0000","last_became_peered":"2024-08-17T17:21:06.459952+0000","last_unstale":"2024-08-17T17:26:42.427435+0000","last_undegraded":"2024-08-17T17:26:42.427435+0000","last_fullsized":"2024-08-17T17:26:42.427435+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:27.828098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1207,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426969+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:26:42.426969+0000","last_peered":"2024-08-17T17:26:42.426969+0000","last_clean":"2024-08-17T17:26:42.426969+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:26:42.426969+0000","last_undegraded":"2024-08-17T17:26:42.426969+0000","last_fullsized":"2024-08-17T17:26:42.426969+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1184,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.630868+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:26:51.630868+0000","last_peered":"2024-08-17T17:26:51.630868+0000","last_clean":"2024-08-17T17:26:51.630868+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:26:51.630868+0000","last_undegraded":"2024-08-17T17:26:51.630868+0000","last_fullsized":"2024-08-17T17:26:51.630868+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":710,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.643110+0000","last_change":"2024-08-17T17:24:48.935651+0000","last_active":"2024-08-17T17:26:51.643110+0000","last_peered":"2024-08-17T17:26:51.643110+0000","last_clean":"2024-08-17T17:26:51.643110+0000","last_became_active":"2024-08-17T17:24:40.897753+0000","last_became_peered":"2024-08-17T17:24:40.897753+0000","last_unstale":"2024-08-17T17:26:51.643110+0000","last_undegraded":"2024-08-17T17:26:51.643110+0000","last_fullsized":"2024-08-17T17:26:51.643110+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:10:15.118428+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1215,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426999+0000","last_change":"2024-08-17T17:21:06.458000+0000","last_active":"2024-08-17T17:26:42.426999+0000","last_peered":"2024-08-17T17:26:42.426999+0000","last_clean":"2024-08-17T17:26:42.426999+0000","last_became_active":"2024-08-17T17:21:06.457411+0000","last_became_peered":"2024-08-17T17:21:06.457411+0000","last_unstale":"2024-08-17T17:26:42.426999+0000","last_undegraded":"2024-08-17T17:26:42.426999+0000","last_fullsized":"2024-08-17T17:26:42.426999+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:52.740639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1206,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.427342+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:26:42.427342+0000","last_peered":"2024-08-17T17:26:42.427342+0000","last_clean":"2024-08-17T17:26:42.427342+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:26:42.427342+0000","last_undegraded":"2024-08-17T17:26:42.427342+0000","last_fullsized":"2024-08-17T17:26:42.427342+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":682,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802499+0000","last_change":"2024-08-17T17:25:03.663370+0000","last_active":"2024-08-17T17:26:42.802499+0000","last_peered":"2024-08-17T17:26:42.802499+0000","last_clean":"2024-08-17T17:26:42.802499+0000","last_became_active":"2024-08-17T17:24:40.886214+0000","last_became_peered":"2024-08-17T17:24:40.886214+0000","last_unstale":"2024-08-17T17:26:42.802499+0000","last_undegraded":"2024-08-17T17:26:42.802499+0000","last_fullsized":"2024-08-17T17:26:42.802499+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:11:39.272029+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":875,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426586+0000","last_change":"2024-08-17T17:21:06.454587+0000","last_active":"2024-08-17T17:26:42.426586+0000","last_peered":"2024-08-17T17:26:42.426586+0000","last_clean":"2024-08-17T17:26:42.426586+0000","last_became_active":"2024-08-17T17:21:06.454229+0000","last_became_peered":"2024-08-17T17:21:06.454229+0000","last_unstale":"2024-08-17T17:26:42.426586+0000","last_undegraded":"2024-08-17T17:26:42.426586+0000","last_fullsized":"2024-08-17T17:26:42.426586+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:38.340263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1181,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802264+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:26:42.802264+0000","last_peered":"2024-08-17T17:26:42.802264+0000","last_clean":"2024-08-17T17:26:42.802264+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:26:42.802264+0000","last_undegraded":"2024-08-17T17:26:42.802264+0000","last_fullsized":"2024-08-17T17:26:42.802264+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1648,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.645864+0000","last_change":"2024-08-17T17:24:20.649977+0000","last_active":"2024-08-17T17:26:51.645864+0000","last_peered":"2024-08-17T17:26:51.645864+0000","last_clean":"2024-08-17T17:26:51.645864+0000","last_became_active":"2024-08-17T17:24:20.649336+0000","last_became_peered":"2024-08-17T17:24:20.649336+0000","last_unstale":"2024-08-17T17:26:51.645864+0000","last_undegraded":"2024-08-17T17:26:51.645864+0000","last_fullsized":"2024-08-17T17:26:51.645864+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:57.187271+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1211,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802453+0000","last_change":"2024-08-17T17:21:06.442796+0000","last_active":"2024-08-17T17:26:42.802453+0000","last_peered":"2024-08-17T17:26:42.802453+0000","last_clean":"2024-08-17T17:26:42.802453+0000","last_became_active":"2024-08-17T17:21:06.442466+0000","last_became_peered":"2024-08-17T17:21:06.442466+0000","last_unstale":"2024-08-17T17:26:42.802453+0000","last_undegraded":"2024-08-17T17:26:42.802453+0000","last_fullsized":"2024-08-17T17:26:42.802453+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:36.366620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1566,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.642562+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:26:51.642562+0000","last_peered":"2024-08-17T17:26:51.642562+0000","last_clean":"2024-08-17T17:26:51.642562+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:26:51.642562+0000","last_undegraded":"2024-08-17T17:26:51.642562+0000","last_fullsized":"2024-08-17T17:26:51.642562+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1429,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802348+0000","last_change":"2024-08-17T17:25:07.472056+0000","last_active":"2024-08-17T17:26:42.802348+0000","last_peered":"2024-08-17T17:26:42.802348+0000","last_clean":"2024-08-17T17:26:42.802348+0000","last_became_active":"2024-08-17T17:24:40.883356+0000","last_became_peered":"2024-08-17T17:24:40.883356+0000","last_unstale":"2024-08-17T17:26:42.802348+0000","last_undegraded":"2024-08-17T17:26:42.802348+0000","last_fullsized":"2024-08-17T17:26:42.802348+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:43:27.883917+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1047,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802646+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:26:42.802646+0000","last_peered":"2024-08-17T17:26:42.802646+0000","last_clean":"2024-08-17T17:26:42.802646+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:26:42.802646+0000","last_undegraded":"2024-08-17T17:26:42.802646+0000","last_fullsized":"2024-08-17T17:26:42.802646+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1599,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.642279+0000","last_change":"2024-08-17T17:21:06.468788+0000","last_active":"2024-08-17T17:26:51.642279+0000","last_peered":"2024-08-17T17:26:51.642279+0000","last_clean":"2024-08-17T17:26:51.642279+0000","last_became_active":"2024-08-17T17:21:06.467141+0000","last_became_peered":"2024-08-17T17:21:06.467141+0000","last_unstale":"2024-08-17T17:26:51.642279+0000","last_undegraded":"2024-08-17T17:26:51.642279+0000","last_fullsized":"2024-08-17T17:26:51.642279+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:51.301359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1429,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426177+0000","last_change":"2024-08-17T17:24:53.130774+0000","last_active":"2024-08-17T17:26:42.426177+0000","last_peered":"2024-08-17T17:26:42.426177+0000","last_clean":"2024-08-17T17:26:42.426177+0000","last_became_active":"2024-08-17T17:24:40.883388+0000","last_became_peered":"2024-08-17T17:24:40.883388+0000","last_unstale":"2024-08-17T17:26:42.426177+0000","last_undegraded":"2024-08-17T17:26:42.426177+0000","last_fullsized":"2024-08-17T17:26:42.426177+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:36.772875+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1214,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.641493+0000","last_change":"2024-08-17T17:21:06.435998+0000","last_active":"2024-08-17T17:26:51.641493+0000","last_peered":"2024-08-17T17:26:51.641493+0000","last_clean":"2024-08-17T17:26:51.641493+0000","last_became_active":"2024-08-17T17:21:06.435579+0000","last_became_peered":"2024-08-17T17:21:06.435579+0000","last_unstale":"2024-08-17T17:26:51.641493+0000","last_undegraded":"2024-08-17T17:26:51.641493+0000","last_fullsized":"2024-08-17T17:26:51.641493+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:01.743074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1628,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.642510+0000","last_change":"2024-08-17T17:21:06.473627+0000","last_active":"2024-08-17T17:26:51.642510+0000","last_peered":"2024-08-17T17:26:51.642510+0000","last_clean":"2024-08-17T17:26:51.642510+0000","last_became_active":"2024-08-17T17:21:06.473062+0000","last_became_peered":"2024-08-17T17:21:06.473062+0000","last_unstale":"2024-08-17T17:26:51.642510+0000","last_undegraded":"2024-08-17T17:26:51.642510+0000","last_fullsized":"2024-08-17T17:26:51.642510+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:21.413163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":789,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801587+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:26:42.801587+0000","last_peered":"2024-08-17T17:26:42.801587+0000","last_clean":"2024-08-17T17:26:42.801587+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:26:42.801587+0000","last_undegraded":"2024-08-17T17:26:42.801587+0000","last_fullsized":"2024-08-17T17:26:42.801587+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1008,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802057+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:26:42.802057+0000","last_peered":"2024-08-17T17:26:42.802057+0000","last_clean":"2024-08-17T17:26:42.802057+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:26:42.802057+0000","last_undegraded":"2024-08-17T17:26:42.802057+0000","last_fullsized":"2024-08-17T17:26:42.802057+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1119,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.642729+0000","last_change":"2024-08-17T17:21:06.471265+0000","last_active":"2024-08-17T17:26:51.642729+0000","last_peered":"2024-08-17T17:26:51.642729+0000","last_clean":"2024-08-17T17:26:51.642729+0000","last_became_active":"2024-08-17T17:21:06.470943+0000","last_became_peered":"2024-08-17T17:21:06.470943+0000","last_unstale":"2024-08-17T17:26:51.642729+0000","last_undegraded":"2024-08-17T17:26:51.642729+0000","last_fullsized":"2024-08-17T17:26:51.642729+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:41.070024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1562,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.644537+0000","last_change":"2024-08-17T17:25:02.696608+0000","last_active":"2024-08-17T17:26:51.644537+0000","last_peered":"2024-08-17T17:26:51.644537+0000","last_clean":"2024-08-17T17:26:51.644537+0000","last_became_active":"2024-08-17T17:24:40.877715+0000","last_became_peered":"2024-08-17T17:24:40.877715+0000","last_unstale":"2024-08-17T17:26:51.644537+0000","last_undegraded":"2024-08-17T17:26:51.644537+0000","last_fullsized":"2024-08-17T17:26:51.644537+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:48:00.941285+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1605,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.642830+0000","last_change":"2024-08-17T17:22:48.377351+0000","last_active":"2024-08-17T17:26:51.642830+0000","last_peered":"2024-08-17T17:26:51.642830+0000","last_clean":"2024-08-17T17:26:51.642830+0000","last_became_active":"2024-08-17T17:22:48.377059+0000","last_became_peered":"2024-08-17T17:22:48.377059+0000","last_unstale":"2024-08-17T17:26:51.642830+0000","last_undegraded":"2024-08-17T17:26:51.642830+0000","last_fullsized":"2024-08-17T17:26:51.642830+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:15.250902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1231,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426552+0000","last_change":"2024-08-17T17:21:06.459139+0000","last_active":"2024-08-17T17:26:42.426552+0000","last_peered":"2024-08-17T17:26:42.426552+0000","last_clean":"2024-08-17T17:26:42.426552+0000","last_became_active":"2024-08-17T17:21:06.457850+0000","last_became_peered":"2024-08-17T17:21:06.457850+0000","last_unstale":"2024-08-17T17:26:42.426552+0000","last_undegraded":"2024-08-17T17:26:42.426552+0000","last_fullsized":"2024-08-17T17:26:42.426552+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:13.888576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1506,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.642346+0000","last_change":"2024-08-17T17:24:49.911734+0000","last_active":"2024-08-17T17:26:51.642346+0000","last_peered":"2024-08-17T17:26:51.642346+0000","last_clean":"2024-08-17T17:26:51.642346+0000","last_became_active":"2024-08-17T17:24:40.899102+0000","last_became_peered":"2024-08-17T17:24:40.899102+0000","last_unstale":"2024-08-17T17:26:51.642346+0000","last_undegraded":"2024-08-17T17:26:51.642346+0000","last_fullsized":"2024-08-17T17:26:51.642346+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.911622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.911622+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:30:27.987033+0000","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1646,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.646088+0000","last_change":"2024-08-17T17:21:06.446321+0000","last_active":"2024-08-17T17:26:51.646088+0000","last_peered":"2024-08-17T17:26:51.646088+0000","last_clean":"2024-08-17T17:26:51.646088+0000","last_became_active":"2024-08-17T17:21:06.446019+0000","last_became_peered":"2024-08-17T17:21:06.446019+0000","last_unstale":"2024-08-17T17:26:51.646088+0000","last_undegraded":"2024-08-17T17:26:51.646088+0000","last_fullsized":"2024-08-17T17:26:51.646088+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:26.750070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":200,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426430+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:26:42.426430+0000","last_peered":"2024-08-17T17:26:42.426430+0000","last_clean":"2024-08-17T17:26:42.426430+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:26:42.426430+0000","last_undegraded":"2024-08-17T17:26:42.426430+0000","last_fullsized":"2024-08-17T17:26:42.426430+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":711,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.643499+0000","last_change":"2024-08-17T17:24:46.946320+0000","last_active":"2024-08-17T17:26:51.643499+0000","last_peered":"2024-08-17T17:26:51.643499+0000","last_clean":"2024-08-17T17:26:51.643499+0000","last_became_active":"2024-08-17T17:24:40.880380+0000","last_became_peered":"2024-08-17T17:24:40.880380+0000","last_unstale":"2024-08-17T17:26:51.643499+0000","last_undegraded":"2024-08-17T17:26:51.643499+0000","last_fullsized":"2024-08-17T17:26:51.643499+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:54:30.637162+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1229,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426662+0000","last_change":"2024-08-17T17:21:06.465804+0000","last_active":"2024-08-17T17:26:42.426662+0000","last_peered":"2024-08-17T17:26:42.426662+0000","last_clean":"2024-08-17T17:26:42.426662+0000","last_became_active":"2024-08-17T17:21:06.465461+0000","last_became_peered":"2024-08-17T17:21:06.465461+0000","last_unstale":"2024-08-17T17:26:42.426662+0000","last_undegraded":"2024-08-17T17:26:42.426662+0000","last_fullsized":"2024-08-17T17:26:42.426662+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:06.443667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1221,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.800638+0000","last_change":"2024-08-17T17:21:06.481634+0000","last_active":"2024-08-17T17:26:42.800638+0000","last_peered":"2024-08-17T17:26:42.800638+0000","last_clean":"2024-08-17T17:26:42.800638+0000","last_became_active":"2024-08-17T17:21:06.469081+0000","last_became_peered":"2024-08-17T17:21:06.469081+0000","last_unstale":"2024-08-17T17:26:42.800638+0000","last_undegraded":"2024-08-17T17:26:42.800638+0000","last_fullsized":"2024-08-17T17:26:42.800638+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:53:26.954738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1229,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.426392+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:26:42.426392+0000","last_peered":"2024-08-17T17:26:42.426392+0000","last_clean":"2024-08-17T17:26:42.426392+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:26:42.426392+0000","last_undegraded":"2024-08-17T17:26:42.426392+0000","last_fullsized":"2024-08-17T17:26:42.426392+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1180,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802383+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:26:42.802383+0000","last_peered":"2024-08-17T17:26:42.802383+0000","last_clean":"2024-08-17T17:26:42.802383+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:26:42.802383+0000","last_undegraded":"2024-08-17T17:26:42.802383+0000","last_fullsized":"2024-08-17T17:26:42.802383+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1620,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.801814+0000","last_change":"2024-08-17T17:25:02.417545+0000","last_active":"2024-08-17T17:26:42.801814+0000","last_peered":"2024-08-17T17:26:42.801814+0000","last_clean":"2024-08-17T17:26:42.801814+0000","last_became_active":"2024-08-17T17:24:40.883220+0000","last_became_peered":"2024-08-17T17:24:40.883220+0000","last_unstale":"2024-08-17T17:26:42.801814+0000","last_undegraded":"2024-08-17T17:26:42.801814+0000","last_fullsized":"2024-08-17T17:26:42.801814+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:28.257805+0000","scrub_duration":0.0084952199999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1604,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.643179+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:26:51.643179+0000","last_peered":"2024-08-17T17:26:51.643179+0000","last_clean":"2024-08-17T17:26:51.643179+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:26:51.643179+0000","last_undegraded":"2024-08-17T17:26:51.643179+0000","last_fullsized":"2024-08-17T17:26:51.643179+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1580,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.643683+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:26:51.643683+0000","last_peered":"2024-08-17T17:26:51.643683+0000","last_clean":"2024-08-17T17:26:51.643683+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:26:51.643683+0000","last_undegraded":"2024-08-17T17:26:51.643683+0000","last_fullsized":"2024-08-17T17:26:51.643683+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1454,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.642757+0000","last_change":"2024-08-17T17:24:47.961645+0000","last_active":"2024-08-17T17:26:51.642757+0000","last_peered":"2024-08-17T17:26:51.642757+0000","last_clean":"2024-08-17T17:26:51.642757+0000","last_became_active":"2024-08-17T17:24:40.897539+0000","last_became_peered":"2024-08-17T17:24:40.897539+0000","last_unstale":"2024-08-17T17:26:51.642757+0000","last_undegraded":"2024-08-17T17:26:51.642757+0000","last_fullsized":"2024-08-17T17:26:51.642757+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:47.961445+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:47.961445+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:04:32.031483+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1214,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.425998+0000","last_change":"2024-08-17T17:21:06.483469+0000","last_active":"2024-08-17T17:26:42.425998+0000","last_peered":"2024-08-17T17:26:42.425998+0000","last_clean":"2024-08-17T17:26:42.425998+0000","last_became_active":"2024-08-17T17:21:06.483162+0000","last_became_peered":"2024-08-17T17:21:06.483162+0000","last_unstale":"2024-08-17T17:26:42.425998+0000","last_undegraded":"2024-08-17T17:26:42.425998+0000","last_fullsized":"2024-08-17T17:26:42.425998+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:24.898353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":223,"reported_epoch":1250,"state":"active+clean","last_fresh":"2024-08-17T17:26:42.802642+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:26:42.802642+0000","last_peered":"2024-08-17T17:26:42.802642+0000","last_clean":"2024-08-17T17:26:42.802642+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:26:42.802642+0000","last_undegraded":"2024-08-17T17:26:42.802642+0000","last_fullsized":"2024-08-17T17:26:42.802642+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1723334656,"num_objects":26296,"num_object_clones":0,"num_object_copies":78888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26296,"num_write_kb":1682944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5249564672,"data_stored":5249564672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":26296,"ondisk_log_size":26296,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201706,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1416432,"kb_used_data":948136,"kb_used_omap":1,"kb_used_meta":468286,"kb_avail":92328720,"statfs":{"total":95995035648,"available":94544609280,"internally_reserved":0,"allocated":970891264,"data_stored":968303667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":479525323},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463343,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":711332,"kb_used_data":319324,"kb_used_omap":1,"kb_used_meta":391998,"kb_avail":93033820,"statfs":{"total":95995035648,"available":95266631680,"internally_reserved":0,"allocated":326987776,"data_stored":324411796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":401406409},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,1],"upper_bound":256},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626984,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1375956,"kb_used_data":762124,"kb_used_omap":42,"kb_used_meta":613781,"kb_avail":92369196,"statfs":{"total":95995035648,"available":94586056704,"internally_reserved":0,"allocated":780414976,"data_stored":777850637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":628511804},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785462,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1485972,"kb_used_data":1002508,"kb_used_omap":48,"kb_used_meta":483407,"kb_avail":92259180,"statfs":{"total":95995035648,"available":94473400320,"internally_reserved":0,"allocated":1026568192,"data_stored":1024003782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":495009454},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724979,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1444192,"kb_used_data":957848,"kb_used_omap":1,"kb_used_meta":486334,"kb_avail":92300960,"statfs":{"total":95995035648,"available":94516183040,"internally_reserved":0,"allocated":980836352,"data_stored":978260838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":498006470},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986616,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":744744,"kb_used_data":320736,"kb_used_omap":1,"kb_used_meta":423998,"kb_avail":93000408,"statfs":{"total":95995035648,"available":95232417792,"internally_reserved":0,"allocated":328433664,"data_stored":325853645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":434174406},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706223,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":834140,"kb_used_data":314836,"kb_used_omap":35,"kb_used_meta":519260,"kb_avail":92911012,"statfs":{"total":95995035648,"available":95140876288,"internally_reserved":0,"allocated":322392064,"data_stored":319820603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":531722295},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831996,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":995416,"kb_used_data":427088,"kb_used_omap":65,"kb_used_meta":568254,"kb_avail":92749736,"statfs":{"total":95995035648,"available":94975729664,"internally_reserved":0,"allocated":437338112,"data_stored":434770191,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":581892283},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":431423488,"data_stored":431423488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1059258368,"data_stored":1059258368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":318570496,"data_stored":318570496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":783679488,"data_stored":783679488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":328335360,"data_stored":328335360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1011548160,"data_stored":1011548160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":326107136,"data_stored":326107136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":990642176,"data_stored":990642176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:27:02.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:02.167 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:02.210 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:02.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:02.312 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:02.345 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:02.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:02.448 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:02.456 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 16 16 36931 36915 144.186 120.438 0.00990856 0.00693009 2024-08-17T17:27:02.483 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:02.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:02.585 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:02.607 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:02.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:02.709 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:02.744 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:02.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:02.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:02 smithi063 ceph-mon[114756]: from='client.35601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:27:02.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:02 smithi063 ceph-mon[109843]: from='client.35601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:27:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:02 smithi162 ceph-mon[104668]: from='client.35601 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:27:02.846 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:02.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.869 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:02.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:02.970 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:02.992 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:02.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:02.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:03.093 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:03.126 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:03.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:03.228 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:03.266 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:03.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:03.368 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:03.406 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:03.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:03.457 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 17 16 38582 38566 141.772 103.188 0.00500921 0.0070478 2024-08-17T17:27:03.508 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:03.530 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:03.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:03.632 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:03.668 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:03.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:03.770 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:03.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:03 smithi063 ceph-mon[109843]: pgmap v1705: 119 pgs: 119 active+clean; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-08-17T17:27:03.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:27:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:27:03] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.51.0" 2024-08-17T17:27:03.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:03 smithi063 ceph-mon[114756]: pgmap v1705: 119 pgs: 119 active+clean; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-08-17T17:27:03.794 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:03 smithi162 ceph-mon[104668]: pgmap v1705: 119 pgs: 119 active+clean; 1.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-08-17T17:27:03.805 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:03.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:03.907 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:03.930 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:03.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:03.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:04.032 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:04.072 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:04.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:04.173 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:04.212 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:04.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.213 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:04.314 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:04.339 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:04.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:04.365 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:04.441 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:04.457 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 18 16 39958 39942 138.67 86 0.00471752 0.00720647 2024-08-17T17:27:04.482 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:04.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:04.583 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:04.622 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:04.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:04.724 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:04.746 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:04.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:04.848 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:04.880 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:04.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:04.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:04.982 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:05.005 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:05.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:05.071 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:27:05.107 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:05.146 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:05.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:05.248 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:05.269 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:05.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:05.371 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:05.403 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:05.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:05.454 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:27:05.457 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 19 16 41028 41012 134.891 66.875 0.00450359 0.00740835 2024-08-17T17:27:05.505 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:05.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.545 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:05.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:05.646 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:05.680 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:05.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:05.716 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:05 smithi162 ceph-mon[104668]: pgmap v1706: 119 pgs: 119 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-08-17T17:27:05.716 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:05 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1449120745' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:05.717 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:27:05.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:05 smithi063 ceph-mon[109843]: pgmap v1706: 119 pgs: 119 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-08-17T17:27:05.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:05 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1449120745' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:05.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:05 smithi063 ceph-mon[114756]: pgmap v1706: 119 pgs: 119 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s 2024-08-17T17:27:05.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:05 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1449120745' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:05.782 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:05.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.843 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:05.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:05.944 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:05.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:05.996 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:05.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:06.097 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:06.146 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:06.149 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:06.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:06.251 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:06.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.291 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:06.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:06.392 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:06.430 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:06.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:06.457 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:27:06.457639+0000 min lat: 0.00127496 max lat: 0.229915 avg lat: 0.00747795 2024-08-17T17:27:06.457 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:27:06.458 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 20 16 42781 42765 133.624 109.562 0.0129837 0.00747795 2024-08-17T17:27:06.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:06 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:27:06.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:06 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:27:06.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:06 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1449120745' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:06.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:06 smithi063 ceph-mon[109843]: osdmap e1252: 8 total, 8 up, 8 in 2024-08-17T17:27:06.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:06 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:27:06.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:06 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:27:06.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:06 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1449120745' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:06.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:06 smithi063 ceph-mon[114756]: osdmap e1252: 8 total, 8 up, 8 in 2024-08-17T17:27:06.531 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:06.563 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:06.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:06.665 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:06.698 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:06.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:06.716 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:06 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:27:06.716 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:06 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:27:06.716 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:06 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1449120745' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:06.716 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:06 smithi162 ceph-mon[104668]: osdmap e1252: 8 total, 8 up, 8 in 2024-08-17T17:27:06.801 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:06.837 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:06.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:06.940 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:06.976 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:06.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:06.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:07.078 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:07.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.112 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:07.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:07.193 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:07.214 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:07.234 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:07.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:07.337 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:07.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.364 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:07.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:07.457 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 21 16 44206 44190 131.502 89.0625 0.00223938 0.00755684 2024-08-17T17:27:07.465 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:07.487 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:07.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:07.588 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:07.610 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:07.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:07.647 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:07 smithi063 ceph-mon[114756]: pgmap v1708: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2024-08-17T17:27:07.649 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:07 smithi063 ceph-mon[109843]: pgmap v1708: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2024-08-17T17:27:07.712 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:07.747 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:07.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:07.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:07 smithi162 ceph-mon[104668]: pgmap v1708: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 118 MiB/s wr, 1.88k op/s 2024-08-17T17:27:07.849 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:07.875 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:07.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:07.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:07.977 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:08.018 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:08.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:08.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:08.142 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:08.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:08.244 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:08.258 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:27:08.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.266 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:08.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:08.368 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:08.410 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:08.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:08.458 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 22 16 45047 45031 127.913 52.5625 0.00698581 0.00781119 2024-08-17T17:27:08.513 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:08.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.537 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:08.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:08.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:08 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/290085123' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:08.638 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:08.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.676 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:08.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:08.777 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:08.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:08 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/290085123' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:08.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:08 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/290085123' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:08.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.828 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:08.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:08.929 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:08.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:08.965 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:08.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:09.066 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:09.112 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:09.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:09.214 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:09.252 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:09.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:09.284 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:27:09.284 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:27:09.284 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:27:09.285 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:27:09.285 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:27:09.285 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:27:09.285 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:27:09.285 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:27:09.285 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:27:09.285 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:27:09.285 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:27:09.286 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:27:09.286 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:27:09.286 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:27:09.286 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:27:09.286 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:27:09.354 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:09.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.391 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:09.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:09.416 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:27:09.417 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:27:09.417 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.417 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:46427 9.7 9:e69b692a:::benchmark_data_smithi063_263014_object46426:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:27:09.417 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:09.405045+0000", 2024-08-17T17:27:09.417 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0088372169999999996, 2024-08-17T17:27:09.417 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0088703459999999994, 2024-08-17T17:27:09.417 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:09.417 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:27:09.417 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:09.417 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:09.417 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:09.417 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 46427 2024-08-17T17:27:09.418 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.418 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:09.418 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.418 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:09.418 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405045+0000", 2024-08-17T17:27:09.418 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.418 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.418 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.418 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:09.418 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405045+0000", 2024-08-17T17:27:09.418 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.419 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.419 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.419 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:09.419 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405044+0000", 2024-08-17T17:27:09.419 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:27:09.419 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.419 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.419 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:09.419 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405082+0000", 2024-08-17T17:27:09.419 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8102000000000002e-05 2024-08-17T17:27:09.419 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.419 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.419 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:09.420 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405093+0000", 2024-08-17T17:27:09.420 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1576000000000001e-05 2024-08-17T17:27:09.420 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.420 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.420 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:09.420 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405100+0000", 2024-08-17T17:27:09.420 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9249999999999998e-06 2024-08-17T17:27:09.420 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.420 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.420 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:09.420 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.406566+0000", 2024-08-17T17:27:09.420 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0014661139999999999 2024-08-17T17:27:09.420 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.420 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.421 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:09.421 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.406696+0000", 2024-08-17T17:27:09.421 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000130035 2024-08-17T17:27:09.421 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.421 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.421 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:27:09.421 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.406828+0000", 2024-08-17T17:27:09.421 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000131798 2024-08-17T17:27:09.421 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.421 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.421 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:09.421 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410587+0000", 2024-08-17T17:27:09.422 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0037590969999999999 2024-08-17T17:27:09.422 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.422 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.422 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:09.422 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410595+0000", 2024-08-17T17:27:09.422 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3440000000000003e-06 2024-08-17T17:27:09.422 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.422 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:09.422 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.422 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.422 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.422 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:46428 9.a 9:5cf68b89:::benchmark_data_smithi063_263014_object46427:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:27:09.422 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:09.405118+0000", 2024-08-17T17:27:09.423 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0087645750000000001, 2024-08-17T17:27:09.423 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0088834440000000008, 2024-08-17T17:27:09.423 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:09.423 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:27:09.423 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:09.423 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:09.423 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:09.423 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 46428 2024-08-17T17:27:09.423 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.423 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:09.423 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.423 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:09.424 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405118+0000", 2024-08-17T17:27:09.424 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.424 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.424 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.424 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:09.424 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405118+0000", 2024-08-17T17:27:09.424 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.424 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.424 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.424 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:09.424 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405117+0000", 2024-08-17T17:27:09.425 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:09.425 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.425 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.425 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:09.425 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405153+0000", 2024-08-17T17:27:09.425 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6100000000000003e-05 2024-08-17T17:27:09.425 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.425 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.425 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:09.425 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405162+0000", 2024-08-17T17:27:09.425 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0360000000000006e-06 2024-08-17T17:27:09.425 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.425 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.425 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:09.426 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405168+0000", 2024-08-17T17:27:09.426 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1659999999999999e-06 2024-08-17T17:27:09.426 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.426 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.426 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:09.426 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.406854+0000", 2024-08-17T17:27:09.426 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016863080000000001 2024-08-17T17:27:09.426 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.426 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.426 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:09.426 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.406981+0000", 2024-08-17T17:27:09.426 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000126945 2024-08-17T17:27:09.427 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.427 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.427 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:27:09.427 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.407096+0000", 2024-08-17T17:27:09.427 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011469800000000001 2024-08-17T17:27:09.427 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.427 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.427 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:09.427 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410898+0000", 2024-08-17T17:27:09.427 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0038025910000000001 2024-08-17T17:27:09.427 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.427 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.428 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:09.428 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410916+0000", 2024-08-17T17:27:09.428 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7448999999999998e-05 2024-08-17T17:27:09.428 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.428 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:09.428 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.428 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.428 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.428 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:46429 9.5 9:a8a185ab:::benchmark_data_smithi063_263014_object46428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:27:09.428 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:09.405435+0000", 2024-08-17T17:27:09.428 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0084474919999999992, 2024-08-17T17:27:09.428 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0086442680000000001, 2024-08-17T17:27:09.429 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:09.429 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:27:09.429 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:09.429 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:09.429 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:09.429 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 46429 2024-08-17T17:27:09.429 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.429 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:09.429 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.429 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:09.429 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405435+0000", 2024-08-17T17:27:09.429 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.429 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.430 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.430 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:09.430 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405435+0000", 2024-08-17T17:27:09.430 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.430 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.430 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.430 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:09.430 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405434+0000", 2024-08-17T17:27:09.430 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:09.430 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.430 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.430 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:09.430 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405478+0000", 2024-08-17T17:27:09.431 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3303000000000002e-05 2024-08-17T17:27:09.431 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.431 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.431 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:09.431 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405497+0000", 2024-08-17T17:27:09.431 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9386999999999999e-05 2024-08-17T17:27:09.431 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.431 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.431 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:09.431 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405504+0000", 2024-08-17T17:27:09.431 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8709999999999999e-06 2024-08-17T17:27:09.431 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.432 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.432 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:09.432 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405783+0000", 2024-08-17T17:27:09.432 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00027887000000000002 2024-08-17T17:27:09.432 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.432 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.432 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:09.432 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.405966+0000", 2024-08-17T17:27:09.432 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00018305900000000001 2024-08-17T17:27:09.432 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.432 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.432 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:27:09.432 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.406108+0000", 2024-08-17T17:27:09.433 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014207099999999999 2024-08-17T17:27:09.433 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.433 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.433 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:09.433 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410235+0000", 2024-08-17T17:27:09.433 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0041266519999999998 2024-08-17T17:27:09.433 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.433 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.433 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:09.433 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410242+0000", 2024-08-17T17:27:09.433 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1810000000000004e-06 2024-08-17T17:27:09.433 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.434 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:09.434 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.434 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.434 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.434 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:46432 9.7 9:ebdbc781:::benchmark_data_smithi063_263014_object46431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:27:09.434 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:09.408436+0000", 2024-08-17T17:27:09.434 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0054461609999999997, 2024-08-17T17:27:09.434 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0057203619999999997, 2024-08-17T17:27:09.434 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:09.434 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:27:09.434 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:09.435 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:09.435 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:09.435 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 46432 2024-08-17T17:27:09.435 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.435 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:09.435 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.435 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:09.435 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.408436+0000", 2024-08-17T17:27:09.435 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.435 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.435 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.435 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:09.436 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.408436+0000", 2024-08-17T17:27:09.436 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.436 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.436 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.436 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:09.436 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.408435+0000", 2024-08-17T17:27:09.436 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:09.436 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.436 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.436 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:09.436 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.408467+0000", 2024-08-17T17:27:09.436 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2054000000000002e-05 2024-08-17T17:27:09.436 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.437 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.437 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:09.437 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.408476+0000", 2024-08-17T17:27:09.437 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4160000000000004e-06 2024-08-17T17:27:09.437 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.437 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.437 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:09.437 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.408481+0000", 2024-08-17T17:27:09.437 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1529999999999996e-06 2024-08-17T17:27:09.437 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.437 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.437 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:09.438 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.408755+0000", 2024-08-17T17:27:09.438 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00027396499999999998 2024-08-17T17:27:09.438 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.438 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.438 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:09.438 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.408907+0000", 2024-08-17T17:27:09.438 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015222799999999999 2024-08-17T17:27:09.438 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.438 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.438 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:27:09.438 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.409039+0000", 2024-08-17T17:27:09.439 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000132146 2024-08-17T17:27:09.439 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.439 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.439 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:09.439 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.411240+0000", 2024-08-17T17:27:09.439 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0022007210000000001 2024-08-17T17:27:09.439 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.439 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:09.439 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.439 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.440 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.440 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:46435 9.7 9:ea3ca095:::benchmark_data_smithi063_263014_object46434:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:27:09.440 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:09.410623+0000", 2024-08-17T17:27:09.440 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.003259826, 2024-08-17T17:27:09.440 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003604744, 2024-08-17T17:27:09.440 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:09.440 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:27:09.440 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:09.440 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:09.440 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:09.440 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 46435 2024-08-17T17:27:09.440 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.440 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:09.441 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.441 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:09.441 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410623+0000", 2024-08-17T17:27:09.441 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.441 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.441 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.441 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:09.441 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410623+0000", 2024-08-17T17:27:09.441 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.441 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.441 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.441 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:09.441 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410621+0000", 2024-08-17T17:27:09.442 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:09.442 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.442 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.442 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:09.442 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410658+0000", 2024-08-17T17:27:09.442 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6788000000000001e-05 2024-08-17T17:27:09.442 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.442 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.442 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:09.442 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410670+0000", 2024-08-17T17:27:09.442 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1582e-05 2024-08-17T17:27:09.442 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.442 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.443 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:09.443 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410677+0000", 2024-08-17T17:27:09.443 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9539999999999998e-06 2024-08-17T17:27:09.443 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.443 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.443 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:09.443 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.411268+0000", 2024-08-17T17:27:09.443 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00059075399999999998 2024-08-17T17:27:09.443 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.443 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.443 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:09.443 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.411392+0000", 2024-08-17T17:27:09.443 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000124117 2024-08-17T17:27:09.444 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.444 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.444 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,7", 2024-08-17T17:27:09.444 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.411507+0000", 2024-08-17T17:27:09.444 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000115495 2024-08-17T17:27:09.444 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.444 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:09.444 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.444 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.444 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.444 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:46436 9.f 9:fc40c1dd:::benchmark_data_smithi063_263014_object46435:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:27:09.444 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:09.410688+0000", 2024-08-17T17:27:09.444 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0031948250000000001, 2024-08-17T17:27:09.445 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0036050679999999999, 2024-08-17T17:27:09.445 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:09.445 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:27:09.445 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:09.445 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:09.445 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:09.445 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 46436 2024-08-17T17:27:09.445 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.445 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:09.445 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.445 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:09.445 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410688+0000", 2024-08-17T17:27:09.445 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.446 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.446 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.446 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:09.446 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410688+0000", 2024-08-17T17:27:09.446 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.446 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.446 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.446 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:09.446 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410687+0000", 2024-08-17T17:27:09.446 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:09.446 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.446 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.447 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:09.447 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410719+0000", 2024-08-17T17:27:09.447 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1838000000000002e-05 2024-08-17T17:27:09.447 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.447 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.447 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:09.447 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410726+0000", 2024-08-17T17:27:09.447 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8070000000000002e-06 2024-08-17T17:27:09.448 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.448 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.448 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:09.448 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410731+0000", 2024-08-17T17:27:09.448 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0000000000000004e-06 2024-08-17T17:27:09.448 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.448 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.448 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:09.448 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.412938+0000", 2024-08-17T17:27:09.448 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0022072630000000001 2024-08-17T17:27:09.448 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.448 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.448 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:09.449 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.413083+0000", 2024-08-17T17:27:09.449 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014470299999999999 2024-08-17T17:27:09.449 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.449 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.449 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:27:09.449 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.413224+0000", 2024-08-17T17:27:09.449 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000141707 2024-08-17T17:27:09.449 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.449 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:09.449 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.449 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.449 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.449 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:46437 9.6 9:6cf97d3c:::benchmark_data_smithi063_263014_object46436:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:27:09.450 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:09.410735+0000", 2024-08-17T17:27:09.450 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.003146996, 2024-08-17T17:27:09.450 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0036251730000000002, 2024-08-17T17:27:09.450 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:09.450 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:27:09.450 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:09.450 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:09.450 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:09.450 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 46437 2024-08-17T17:27:09.450 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.450 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:09.450 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.451 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:09.451 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410735+0000", 2024-08-17T17:27:09.451 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.451 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.451 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.451 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:09.451 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410735+0000", 2024-08-17T17:27:09.451 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.451 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.451 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.451 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:09.451 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410735+0000", 2024-08-17T17:27:09.451 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:09.452 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.452 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.452 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:09.452 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410762+0000", 2024-08-17T17:27:09.452 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6517999999999999e-05 2024-08-17T17:27:09.452 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.452 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.452 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:09.452 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410769+0000", 2024-08-17T17:27:09.452 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0790000000000003e-06 2024-08-17T17:27:09.452 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.452 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.453 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:09.453 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410774+0000", 2024-08-17T17:27:09.453 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3499999999999996e-06 2024-08-17T17:27:09.453 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.453 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.453 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:09.453 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410822+0000", 2024-08-17T17:27:09.453 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.791e-05 2024-08-17T17:27:09.453 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.453 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.453 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:09.453 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.410962+0000", 2024-08-17T17:27:09.454 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014024299999999999 2024-08-17T17:27:09.454 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.454 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.454 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:27:09.454 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.411088+0000", 2024-08-17T17:27:09.454 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000126136 2024-08-17T17:27:09.454 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.454 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:09.454 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.454 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.454 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.454 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop_reply(client.45521.0:46437 9.6 e1252/1247)", 2024-08-17T17:27:09.455 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:09.413086+0000", 2024-08-17T17:27:09.455 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.00079622000000000004, 2024-08-17T17:27:09.455 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013339910000000001, 2024-08-17T17:27:09.455 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:09.455 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "queued for pg", 2024-08-17T17:27:09.455 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:09.455 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.455 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:09.455 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.413086+0000", 2024-08-17T17:27:09.455 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.455 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.455 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.456 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:09.456 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.413086+0000", 2024-08-17T17:27:09.456 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.456 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.456 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.456 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:09.456 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.413086+0000", 2024-08-17T17:27:09.456 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:09.456 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.456 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.456 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:09.456 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.413089+0000", 2024-08-17T17:27:09.457 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.766e-06 2024-08-17T17:27:09.457 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.457 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.457 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:09.457 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.413100+0000", 2024-08-17T17:27:09.457 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.116e-05 2024-08-17T17:27:09.457 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.457 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.457 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:09.457 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.413108+0000", 2024-08-17T17:27:09.457 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7649999999999993e-06 2024-08-17T17:27:09.457 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.457 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:09.457 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.458 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.458 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.458 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:46441 9.f 9.b29833af (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:27:09.458 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:09.413673+0000", 2024-08-17T17:27:09.458 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.00020956499999999999, 2024-08-17T17:27:09.458 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000789407, 2024-08-17T17:27:09.458 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:09.458 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "queued for pg", 2024-08-17T17:27:09.458 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:09.458 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:09.458 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:09.458 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 46441 2024-08-17T17:27:09.458 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.459 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:09.459 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.459 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:09.459 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.413673+0000", 2024-08-17T17:27:09.459 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.459 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.459 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.459 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:09.459 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.413673+0000", 2024-08-17T17:27:09.459 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:09.459 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.460 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.460 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:09.460 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.413672+0000", 2024-08-17T17:27:09.460 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:09.460 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.460 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.460 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:09.460 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.413710+0000", 2024-08-17T17:27:09.460 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8407e-05 2024-08-17T17:27:09.460 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.460 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.460 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:09.460 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.413726+0000", 2024-08-17T17:27:09.461 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5849e-05 2024-08-17T17:27:09.461 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:09.461 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:09.461 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:09.461 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:09.413734+0000", 2024-08-17T17:27:09.461 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9889999999999992e-06 2024-08-17T17:27:09.461 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.461 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:09.461 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.461 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:09.461 INFO:teuthology.orchestra.run.smithi162.stdout: ], 2024-08-17T17:27:09.461 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 9 2024-08-17T17:27:09.461 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:27:09.462 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 23 16 46488 46472 126.267 90.0625 0.0168505 0.00791244 2024-08-17T17:27:09.492 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:09.525 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:09.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:09.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:09 smithi162 ceph-mon[104668]: pgmap v1709: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-08-17T17:27:09.627 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:09.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.667 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:09.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:09.675 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:09 smithi063 ceph-mon[114756]: pgmap v1709: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-08-17T17:27:09.676 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:09 smithi063 ceph-mon[109843]: pgmap v1709: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-08-17T17:27:09.768 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:09.806 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:09.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:09.908 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:09.945 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:09.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:09.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:10.047 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:10.083 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:10.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:10.175 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:27:10.185 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:10.221 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:10.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:10.323 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:10.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.370 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:10.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:10.458 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 24 16 48228 48212 125.537 108.75 0.00804487 0.00796042 2024-08-17T17:27:10.471 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:10.512 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:10.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:10.554 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:10.614 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:10.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.675 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:10.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:10.776 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:10.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.818 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:10.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:10.919 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:10.959 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:10.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:10.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:11.061 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:11.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.098 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:11.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:11.154 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:11.199 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:11.250 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:11.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:11.352 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:11.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.392 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:11.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:11.458 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 25 16 49880 49864 124.645 103.25 0.00487509 0.00801721 2024-08-17T17:27:11.492 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:11.536 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:11.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:11.565 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:27:11.567 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:27:11.639 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:11.669 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:11.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:11.771 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:11.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:11 smithi063 ceph-mon[114756]: pgmap v1710: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-08-17T17:27:11.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:11 smithi063 ceph-mon[109843]: pgmap v1710: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-08-17T17:27:11.808 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:11.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:11.839 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:11 smithi162 ceph-mon[104668]: pgmap v1710: 119 pgs: 119 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2024-08-17T17:27:11.910 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:11.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:11.959 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:11.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:12.060 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:12.098 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:12.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:12.201 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:12.234 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:12.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:12.336 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:12.372 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:12.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:12.458 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 26 16 51565 51549 123.901 105.312 0.00620142 0.00806561 2024-08-17T17:27:12.475 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:12.522 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:12.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:12.527 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:27:12.527 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:27:12.527 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:27:12.527 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:27:12.527 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:27:12.625 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:12.644 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:27:12.659 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:12.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:12.760 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:12.776 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:12 smithi063 ceph-mon[114756]: from='client.55222 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:27:12.776 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:12 smithi063 ceph-mon[109843]: from='client.55222 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:27:12.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.814 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:12.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:12.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:12 smithi162 ceph-mon[104668]: from='client.55222 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:27:12.914 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:12.951 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:12.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:12.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:13.053 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:13.089 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:13.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:13.191 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:13.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.244 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:13.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:13.265 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:27:13.345 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:13.386 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:13.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:13.420 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:27:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:27:13] "GET /metrics HTTP/1.1" 200 36393 "" "Prometheus/2.51.0" 2024-08-17T17:27:13.458 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 27 16 53181 53165 123.053 101 0.0262392 0.00812179 2024-08-17T17:27:13.488 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:13.523 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:13.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:13.625 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:13.653 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:13.659 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:13.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.660 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:13.731 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:13 smithi063 ceph-mon[114756]: pgmap v1711: 119 pgs: 119 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-08-17T17:27:13.732 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:13 smithi063 ceph-mon[109843]: pgmap v1711: 119 pgs: 119 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-08-17T17:27:13.761 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:13.780 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1710,"stamp":"2024-08-17T17:27:10.372275+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2860390373,"num_objects":43663,"num_object_clones":0,"num_object_copies":130989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":43701,"num_write_kb":2792801,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1246,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":48315,"ondisk_log_size":48315,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12471900,"kb_used_data":8404044,"kb_used_omap":199,"kb_used_meta":4067576,"kb_avail":737489316,"statfs":{"total":767960285184,"available":755189059584,"internally_reserved":0,"allocated":8605741056,"data_stored":8585125619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":4165198588},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1136459776,"num_objects":17341,"num_object_clones":0,"num_object_copies":52023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":17341,"num_write_kb":1109824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003119"},"pg_stats":[{"pgid":"1.11","version":"369'149","reported_seq":1575,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.645141+0000","last_change":"2024-08-17T17:24:49.721626+0000","last_active":"2024-08-17T17:26:51.645141+0000","last_peered":"2024-08-17T17:26:51.645141+0000","last_clean":"2024-08-17T17:26:51.645141+0000","last_became_active":"2024-08-17T17:24:20.649128+0000","last_became_peered":"2024-08-17T17:24:20.649128+0000","last_unstale":"2024-08-17T17:26:51.645141+0000","last_undegraded":"2024-08-17T17:26:51.645141+0000","last_fullsized":"2024-08-17T17:26:51.645141+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:03:15.343059+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1601,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.459600+0000","last_change":"2024-08-17T17:21:06.463828+0000","last_active":"2024-08-17T17:27:05.459600+0000","last_peered":"2024-08-17T17:27:05.459600+0000","last_clean":"2024-08-17T17:27:05.459600+0000","last_became_active":"2024-08-17T17:21:06.463464+0000","last_became_peered":"2024-08-17T17:21:06.463464+0000","last_unstale":"2024-08-17T17:27:05.459600+0000","last_undegraded":"2024-08-17T17:27:05.459600+0000","last_fullsized":"2024-08-17T17:27:05.459600+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:46.465006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1182,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.648246+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:26:51.648246+0000","last_peered":"2024-08-17T17:26:51.648246+0000","last_clean":"2024-08-17T17:26:51.648246+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:26:51.648246+0000","last_undegraded":"2024-08-17T17:26:51.648246+0000","last_fullsized":"2024-08-17T17:26:51.648246+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1506,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.644341+0000","last_change":"2024-08-17T17:24:48.765312+0000","last_active":"2024-08-17T17:26:51.644341+0000","last_peered":"2024-08-17T17:26:51.644341+0000","last_clean":"2024-08-17T17:26:51.644341+0000","last_became_active":"2024-08-17T17:24:20.642731+0000","last_became_peered":"2024-08-17T17:24:20.642731+0000","last_unstale":"2024-08-17T17:26:51.644341+0000","last_undegraded":"2024-08-17T17:26:51.644341+0000","last_fullsized":"2024-08-17T17:26:51.644341+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:08:36.138960+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1193,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.644890+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:26:51.644890+0000","last_peered":"2024-08-17T17:26:51.644890+0000","last_clean":"2024-08-17T17:26:51.644890+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:26:51.644890+0000","last_undegraded":"2024-08-17T17:26:51.644890+0000","last_fullsized":"2024-08-17T17:26:51.644890+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1049,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.647986+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:26:51.647986+0000","last_peered":"2024-08-17T17:26:51.647986+0000","last_clean":"2024-08-17T17:26:51.647986+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:26:51.647986+0000","last_undegraded":"2024-08-17T17:26:51.647986+0000","last_fullsized":"2024-08-17T17:26:51.647986+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":815,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.458832+0000","last_change":"2024-08-17T17:24:50.880046+0000","last_active":"2024-08-17T17:27:05.458832+0000","last_peered":"2024-08-17T17:27:05.458832+0000","last_clean":"2024-08-17T17:27:05.458832+0000","last_became_active":"2024-08-17T17:24:20.652165+0000","last_became_peered":"2024-08-17T17:24:20.652165+0000","last_unstale":"2024-08-17T17:27:05.458832+0000","last_undegraded":"2024-08-17T17:27:05.458832+0000","last_fullsized":"2024-08-17T17:27:05.458832+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:00.416810+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1152,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.459765+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:27:05.459765+0000","last_peered":"2024-08-17T17:27:05.459765+0000","last_clean":"2024-08-17T17:27:05.459765+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:27:05.459765+0000","last_undegraded":"2024-08-17T17:27:05.459765+0000","last_fullsized":"2024-08-17T17:27:05.459765+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1165,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.648187+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:26:51.648187+0000","last_peered":"2024-08-17T17:26:51.648187+0000","last_clean":"2024-08-17T17:26:51.648187+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:26:51.648187+0000","last_undegraded":"2024-08-17T17:26:51.648187+0000","last_fullsized":"2024-08-17T17:26:51.648187+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1511,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.645558+0000","last_change":"2024-08-17T17:24:50.727183+0000","last_active":"2024-08-17T17:26:51.645558+0000","last_peered":"2024-08-17T17:26:51.645558+0000","last_clean":"2024-08-17T17:26:51.645558+0000","last_became_active":"2024-08-17T17:24:20.646409+0000","last_became_peered":"2024-08-17T17:24:20.646409+0000","last_unstale":"2024-08-17T17:26:51.645558+0000","last_undegraded":"2024-08-17T17:26:51.645558+0000","last_fullsized":"2024-08-17T17:26:51.645558+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.086193+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1601,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.459424+0000","last_change":"2024-08-17T17:21:06.442598+0000","last_active":"2024-08-17T17:27:05.459424+0000","last_peered":"2024-08-17T17:27:05.459424+0000","last_clean":"2024-08-17T17:27:05.459424+0000","last_became_active":"2024-08-17T17:21:06.442264+0000","last_became_peered":"2024-08-17T17:21:06.442264+0000","last_unstale":"2024-08-17T17:27:05.459424+0000","last_undegraded":"2024-08-17T17:27:05.459424+0000","last_fullsized":"2024-08-17T17:27:05.459424+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:35.003620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1242,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.629905+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:26:51.629905+0000","last_peered":"2024-08-17T17:26:51.629905+0000","last_clean":"2024-08-17T17:26:51.629905+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:26:51.629905+0000","last_undegraded":"2024-08-17T17:26:51.629905+0000","last_fullsized":"2024-08-17T17:26:51.629905+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"1252'2788","reported_seq":2802,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.802984+0000","last_change":"2024-08-17T17:26:34.519835+0000","last_active":"2024-08-17T17:27:07.802984+0000","last_peered":"2024-08-17T17:27:07.802984+0000","last_clean":"2024-08-17T17:27:07.802984+0000","last_became_active":"2024-08-17T17:26:34.519427+0000","last_became_peered":"2024-08-17T17:26:34.519427+0000","last_unstale":"2024-08-17T17:27:07.802984+0000","last_undegraded":"2024-08-17T17:27:07.802984+0000","last_fullsized":"2024-08-17T17:27:07.802984+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2788,"log_dups_size":0,"ondisk_log_size":2788,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:25.352736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182714368,"num_objects":2788,"num_object_clones":0,"num_object_copies":8364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2788,"num_write_kb":178432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1449,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.648853+0000","last_change":"2024-08-17T17:24:50.410621+0000","last_active":"2024-08-17T17:26:51.648853+0000","last_peered":"2024-08-17T17:26:51.648853+0000","last_clean":"2024-08-17T17:26:51.648853+0000","last_became_active":"2024-08-17T17:24:31.863062+0000","last_became_peered":"2024-08-17T17:24:31.863062+0000","last_unstale":"2024-08-17T17:26:51.648853+0000","last_undegraded":"2024-08-17T17:26:51.648853+0000","last_fullsized":"2024-08-17T17:26:51.648853+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.410491+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.410491+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:11.717684+0000","scrub_duration":0.0066315480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1207,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.649673+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:26:51.649673+0000","last_peered":"2024-08-17T17:26:51.649673+0000","last_clean":"2024-08-17T17:26:51.649673+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:26:51.649673+0000","last_undegraded":"2024-08-17T17:26:51.649673+0000","last_fullsized":"2024-08-17T17:26:51.649673+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1180,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.644622+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:26:51.644622+0000","last_peered":"2024-08-17T17:26:51.644622+0000","last_clean":"2024-08-17T17:26:51.644622+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:26:51.644622+0000","last_undegraded":"2024-08-17T17:26:51.644622+0000","last_fullsized":"2024-08-17T17:26:51.644622+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"1252'2724","reported_seq":2736,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.319713+0000","last_change":"2024-08-17T17:26:34.715261+0000","last_active":"2024-08-17T17:27:07.319713+0000","last_peered":"2024-08-17T17:27:07.319713+0000","last_clean":"2024-08-17T17:27:07.319713+0000","last_became_active":"2024-08-17T17:26:34.714853+0000","last_became_peered":"2024-08-17T17:26:34.714853+0000","last_unstale":"2024-08-17T17:27:07.319713+0000","last_undegraded":"2024-08-17T17:27:07.319713+0000","last_fullsized":"2024-08-17T17:27:07.319713+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2724,"log_dups_size":0,"ondisk_log_size":2724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:18.673096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178520064,"num_objects":2724,"num_object_clones":0,"num_object_copies":8172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2724,"num_write_kb":174336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1683,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.459464+0000","last_change":"2024-08-17T17:24:45.985725+0000","last_active":"2024-08-17T17:27:05.459464+0000","last_peered":"2024-08-17T17:27:05.459464+0000","last_clean":"2024-08-17T17:27:05.459464+0000","last_became_active":"2024-08-17T17:24:31.795836+0000","last_became_peered":"2024-08-17T17:24:31.795836+0000","last_unstale":"2024-08-17T17:27:05.459464+0000","last_undegraded":"2024-08-17T17:27:05.459464+0000","last_fullsized":"2024-08-17T17:27:05.459464+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:45.985596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:45.985596+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:17.897808+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1208,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.649359+0000","last_change":"2024-08-17T17:21:06.540689+0000","last_active":"2024-08-17T17:26:51.649359+0000","last_peered":"2024-08-17T17:26:51.649359+0000","last_clean":"2024-08-17T17:26:51.649359+0000","last_became_active":"2024-08-17T17:21:06.461614+0000","last_became_peered":"2024-08-17T17:21:06.461614+0000","last_unstale":"2024-08-17T17:26:51.649359+0000","last_undegraded":"2024-08-17T17:26:51.649359+0000","last_fullsized":"2024-08-17T17:26:51.649359+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:11.696898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1232,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.631028+0000","last_change":"2024-08-17T17:21:06.470389+0000","last_active":"2024-08-17T17:26:51.631028+0000","last_peered":"2024-08-17T17:26:51.631028+0000","last_clean":"2024-08-17T17:26:51.631028+0000","last_became_active":"2024-08-17T17:21:06.469966+0000","last_became_peered":"2024-08-17T17:21:06.469966+0000","last_unstale":"2024-08-17T17:26:51.631028+0000","last_undegraded":"2024-08-17T17:26:51.631028+0000","last_fullsized":"2024-08-17T17:26:51.631028+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:56.140616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"1252'2829","reported_seq":2843,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.795914+0000","last_change":"2024-08-17T17:26:34.412902+0000","last_active":"2024-08-17T17:27:07.795914+0000","last_peered":"2024-08-17T17:27:07.795914+0000","last_clean":"2024-08-17T17:27:07.795914+0000","last_became_active":"2024-08-17T17:26:34.412543+0000","last_became_peered":"2024-08-17T17:26:34.412543+0000","last_unstale":"2024-08-17T17:27:07.795914+0000","last_undegraded":"2024-08-17T17:27:07.795914+0000","last_fullsized":"2024-08-17T17:27:07.795914+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2829,"log_dups_size":0,"ondisk_log_size":2829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:20.767004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185401344,"num_objects":2829,"num_object_clones":0,"num_object_copies":8487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2829,"num_write_kb":181056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":817,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.641476+0000","last_change":"2024-08-17T17:24:48.192447+0000","last_active":"2024-08-17T17:26:51.641476+0000","last_peered":"2024-08-17T17:26:51.641476+0000","last_clean":"2024-08-17T17:26:51.641476+0000","last_became_active":"2024-08-17T17:24:20.644240+0000","last_became_peered":"2024-08-17T17:24:20.644240+0000","last_unstale":"2024-08-17T17:26:51.641476+0000","last_undegraded":"2024-08-17T17:26:51.641476+0000","last_fullsized":"2024-08-17T17:26:51.641476+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:56.795337+0000","scrub_duration":0.010920255,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1166,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.649231+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:26:51.649231+0000","last_peered":"2024-08-17T17:26:51.649231+0000","last_clean":"2024-08-17T17:26:51.649231+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:26:51.649231+0000","last_undegraded":"2024-08-17T17:26:51.649231+0000","last_fullsized":"2024-08-17T17:26:51.649231+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1214,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.629935+0000","last_change":"2024-08-17T17:21:06.446798+0000","last_active":"2024-08-17T17:26:51.629935+0000","last_peered":"2024-08-17T17:26:51.629935+0000","last_clean":"2024-08-17T17:26:51.629935+0000","last_became_active":"2024-08-17T17:21:06.446205+0000","last_became_peered":"2024-08-17T17:21:06.446205+0000","last_unstale":"2024-08-17T17:26:51.629935+0000","last_undegraded":"2024-08-17T17:26:51.629935+0000","last_fullsized":"2024-08-17T17:26:51.629935+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:28.584048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1252'2821","reported_seq":2835,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.800121+0000","last_change":"2024-08-17T17:26:35.023050+0000","last_active":"2024-08-17T17:27:07.800121+0000","last_peered":"2024-08-17T17:27:07.800121+0000","last_clean":"2024-08-17T17:27:07.800121+0000","last_became_active":"2024-08-17T17:26:35.022784+0000","last_became_peered":"2024-08-17T17:26:35.022784+0000","last_unstale":"2024-08-17T17:27:07.800121+0000","last_undegraded":"2024-08-17T17:27:07.800121+0000","last_fullsized":"2024-08-17T17:27:07.800121+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2821,"log_dups_size":0,"ondisk_log_size":2821,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:04.319458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184877056,"num_objects":2821,"num_object_clones":0,"num_object_copies":8463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2821,"num_write_kb":180544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":725,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.640667+0000","last_change":"2024-08-17T17:24:52.102415+0000","last_active":"2024-08-17T17:26:51.640667+0000","last_peered":"2024-08-17T17:26:51.640667+0000","last_clean":"2024-08-17T17:26:51.640667+0000","last_became_active":"2024-08-17T17:24:31.795843+0000","last_became_peered":"2024-08-17T17:24:31.795843+0000","last_unstale":"2024-08-17T17:26:51.640667+0000","last_undegraded":"2024-08-17T17:26:51.640667+0000","last_fullsized":"2024-08-17T17:26:51.640667+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:32:08.434197+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1195,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.644935+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:26:51.644935+0000","last_peered":"2024-08-17T17:26:51.644935+0000","last_clean":"2024-08-17T17:26:51.644935+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:26:51.644935+0000","last_undegraded":"2024-08-17T17:26:51.644935+0000","last_fullsized":"2024-08-17T17:26:51.644935+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":977,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.630993+0000","last_change":"2024-08-17T17:21:06.460905+0000","last_active":"2024-08-17T17:26:51.630993+0000","last_peered":"2024-08-17T17:26:51.630993+0000","last_clean":"2024-08-17T17:26:51.630993+0000","last_became_active":"2024-08-17T17:21:06.460592+0000","last_became_peered":"2024-08-17T17:21:06.460592+0000","last_unstale":"2024-08-17T17:26:51.630993+0000","last_undegraded":"2024-08-17T17:26:51.630993+0000","last_fullsized":"2024-08-17T17:26:51.630993+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:10.146916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.1","version":"1252'2523","reported_seq":2544,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.650610+0000","last_change":"2024-08-17T17:26:34.410513+0000","last_active":"2024-08-17T17:27:05.650610+0000","last_peered":"2024-08-17T17:27:05.650610+0000","last_clean":"2024-08-17T17:27:05.650610+0000","last_became_active":"2024-08-17T17:26:34.410199+0000","last_became_peered":"2024-08-17T17:26:34.410199+0000","last_unstale":"2024-08-17T17:27:05.650610+0000","last_undegraded":"2024-08-17T17:27:05.650610+0000","last_fullsized":"2024-08-17T17:27:05.650610+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2523,"log_dups_size":0,"ondisk_log_size":2523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:35.837825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165347328,"num_objects":2523,"num_object_clones":0,"num_object_copies":7569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2523,"num_write_kb":161472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1596,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:27:02.454700+0000","last_change":"2024-08-17T17:24:54.418765+0000","last_active":"2024-08-17T17:27:02.454700+0000","last_peered":"2024-08-17T17:27:02.454700+0000","last_clean":"2024-08-17T17:27:02.454700+0000","last_became_active":"2024-08-17T17:24:31.800263+0000","last_became_peered":"2024-08-17T17:24:31.800263+0000","last_unstale":"2024-08-17T17:27:02.454700+0000","last_undegraded":"2024-08-17T17:27:02.454700+0000","last_fullsized":"2024-08-17T17:27:02.454700+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:54.418644+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:54.418644+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:11.024294+0000","scrub_duration":0.013346135,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1622,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.459963+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:27:05.459963+0000","last_peered":"2024-08-17T17:27:05.459963+0000","last_clean":"2024-08-17T17:27:05.459963+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:27:05.459963+0000","last_undegraded":"2024-08-17T17:27:05.459963+0000","last_fullsized":"2024-08-17T17:27:05.459963+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1198,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.631186+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:26:51.631186+0000","last_peered":"2024-08-17T17:26:51.631186+0000","last_clean":"2024-08-17T17:26:51.631186+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:26:51.631186+0000","last_undegraded":"2024-08-17T17:26:51.631186+0000","last_fullsized":"2024-08-17T17:26:51.631186+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"1252'2693","reported_seq":2707,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.348770+0000","last_change":"2024-08-17T17:26:35.023056+0000","last_active":"2024-08-17T17:27:07.348770+0000","last_peered":"2024-08-17T17:27:07.348770+0000","last_clean":"2024-08-17T17:27:07.348770+0000","last_became_active":"2024-08-17T17:26:35.022671+0000","last_became_peered":"2024-08-17T17:26:35.022671+0000","last_unstale":"2024-08-17T17:27:07.348770+0000","last_undegraded":"2024-08-17T17:27:07.348770+0000","last_fullsized":"2024-08-17T17:27:07.348770+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2693,"log_dups_size":0,"ondisk_log_size":2693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:43.531150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176488448,"num_objects":2693,"num_object_clones":0,"num_object_copies":8079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2693,"num_write_kb":172352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1541,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.644393+0000","last_change":"2024-08-17T17:24:58.712331+0000","last_active":"2024-08-17T17:26:51.644393+0000","last_peered":"2024-08-17T17:26:51.644393+0000","last_clean":"2024-08-17T17:26:51.644393+0000","last_became_active":"2024-08-17T17:24:31.802584+0000","last_became_peered":"2024-08-17T17:24:31.802584+0000","last_unstale":"2024-08-17T17:26:51.644393+0000","last_undegraded":"2024-08-17T17:26:51.644393+0000","last_fullsized":"2024-08-17T17:26:51.644393+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:59.616217+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1215,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.630020+0000","last_change":"2024-08-17T17:21:06.446841+0000","last_active":"2024-08-17T17:26:51.630020+0000","last_peered":"2024-08-17T17:26:51.630020+0000","last_clean":"2024-08-17T17:26:51.630020+0000","last_became_active":"2024-08-17T17:21:06.446271+0000","last_became_peered":"2024-08-17T17:21:06.446271+0000","last_unstale":"2024-08-17T17:26:51.630020+0000","last_undegraded":"2024-08-17T17:26:51.630020+0000","last_fullsized":"2024-08-17T17:26:51.630020+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:20.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1231,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.640795+0000","last_change":"2024-08-17T17:21:06.480911+0000","last_active":"2024-08-17T17:26:51.640795+0000","last_peered":"2024-08-17T17:26:51.640795+0000","last_clean":"2024-08-17T17:26:51.640795+0000","last_became_active":"2024-08-17T17:21:06.480551+0000","last_became_peered":"2024-08-17T17:21:06.480551+0000","last_unstale":"2024-08-17T17:26:51.640795+0000","last_undegraded":"2024-08-17T17:26:51.640795+0000","last_fullsized":"2024-08-17T17:26:51.640795+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:26.003704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"1252'2742","reported_seq":2756,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.338649+0000","last_change":"2024-08-17T17:26:34.715560+0000","last_active":"2024-08-17T17:27:07.338649+0000","last_peered":"2024-08-17T17:27:07.338649+0000","last_clean":"2024-08-17T17:27:07.338649+0000","last_became_active":"2024-08-17T17:26:34.714955+0000","last_became_peered":"2024-08-17T17:26:34.714955+0000","last_unstale":"2024-08-17T17:27:07.338649+0000","last_undegraded":"2024-08-17T17:27:07.338649+0000","last_fullsized":"2024-08-17T17:27:07.338649+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2742,"log_dups_size":0,"ondisk_log_size":2742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:46.367495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179699712,"num_objects":2742,"num_object_clones":0,"num_object_copies":8226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2742,"num_write_kb":175488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":774,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.645081+0000","last_change":"2024-08-17T17:24:57.682573+0000","last_active":"2024-08-17T17:26:51.645081+0000","last_peered":"2024-08-17T17:26:51.645081+0000","last_clean":"2024-08-17T17:26:51.645081+0000","last_became_active":"2024-08-17T17:24:31.869575+0000","last_became_peered":"2024-08-17T17:24:31.869575+0000","last_unstale":"2024-08-17T17:26:51.645081+0000","last_undegraded":"2024-08-17T17:26:51.645081+0000","last_fullsized":"2024-08-17T17:26:51.645081+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:57.682409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:57.682409+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:42.144615+0000","scrub_duration":0.0066879779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1200,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.631050+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:26:51.631050+0000","last_peered":"2024-08-17T17:26:51.631050+0000","last_clean":"2024-08-17T17:26:51.631050+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:26:51.631050+0000","last_undegraded":"2024-08-17T17:26:51.631050+0000","last_fullsized":"2024-08-17T17:26:51.631050+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1216,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.640491+0000","last_change":"2024-08-17T17:21:06.451521+0000","last_active":"2024-08-17T17:26:51.640491+0000","last_peered":"2024-08-17T17:26:51.640491+0000","last_clean":"2024-08-17T17:26:51.640491+0000","last_became_active":"2024-08-17T17:21:06.451197+0000","last_became_peered":"2024-08-17T17:21:06.451197+0000","last_unstale":"2024-08-17T17:26:51.640491+0000","last_undegraded":"2024-08-17T17:26:51.640491+0000","last_fullsized":"2024-08-17T17:26:51.640491+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:04.000692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"1252'2606","reported_seq":2627,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.667176+0000","last_change":"2024-08-17T17:26:35.023654+0000","last_active":"2024-08-17T17:27:05.667176+0000","last_peered":"2024-08-17T17:27:05.667176+0000","last_clean":"2024-08-17T17:27:05.667176+0000","last_became_active":"2024-08-17T17:26:35.022968+0000","last_became_peered":"2024-08-17T17:26:35.022968+0000","last_unstale":"2024-08-17T17:27:05.667176+0000","last_undegraded":"2024-08-17T17:27:05.667176+0000","last_fullsized":"2024-08-17T17:27:05.667176+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2606,"log_dups_size":0,"ondisk_log_size":2606,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:25:10.494807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170786816,"num_objects":2606,"num_object_clones":0,"num_object_copies":7818,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2606,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2606,"num_write_kb":166784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1541,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.646144+0000","last_change":"2024-08-17T17:24:52.716059+0000","last_active":"2024-08-17T17:26:51.646144+0000","last_peered":"2024-08-17T17:26:51.646144+0000","last_clean":"2024-08-17T17:26:51.646144+0000","last_became_active":"2024-08-17T17:24:31.798173+0000","last_became_peered":"2024-08-17T17:24:31.798173+0000","last_unstale":"2024-08-17T17:26:51.646144+0000","last_undegraded":"2024-08-17T17:26:51.646144+0000","last_fullsized":"2024-08-17T17:26:51.646144+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:04:56.094994+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1603,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.458996+0000","last_change":"2024-08-17T17:21:06.441118+0000","last_active":"2024-08-17T17:27:05.458996+0000","last_peered":"2024-08-17T17:27:05.458996+0000","last_clean":"2024-08-17T17:27:05.458996+0000","last_became_active":"2024-08-17T17:21:06.440800+0000","last_became_peered":"2024-08-17T17:21:06.440800+0000","last_unstale":"2024-08-17T17:27:05.458996+0000","last_undegraded":"2024-08-17T17:27:05.458996+0000","last_fullsized":"2024-08-17T17:27:05.458996+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:08.091528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":109,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.463169+0000","last_change":"2024-08-17T17:22:48.374766+0000","last_active":"2024-08-17T17:27:05.463169+0000","last_peered":"2024-08-17T17:27:05.463169+0000","last_clean":"2024-08-17T17:27:05.463169+0000","last_became_active":"2024-08-17T17:22:48.373556+0000","last_became_peered":"2024-08-17T17:22:48.373556+0000","last_unstale":"2024-08-17T17:27:05.463169+0000","last_undegraded":"2024-08-17T17:27:05.463169+0000","last_fullsized":"2024-08-17T17:27:05.463169+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:33.362080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.8","version":"1252'2632","reported_seq":2653,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.672462+0000","last_change":"2024-08-17T17:26:34.519719+0000","last_active":"2024-08-17T17:27:05.672462+0000","last_peered":"2024-08-17T17:27:05.672462+0000","last_clean":"2024-08-17T17:27:05.672462+0000","last_became_active":"2024-08-17T17:26:34.519368+0000","last_became_peered":"2024-08-17T17:26:34.519368+0000","last_unstale":"2024-08-17T17:27:05.672462+0000","last_undegraded":"2024-08-17T17:27:05.672462+0000","last_fullsized":"2024-08-17T17:27:05.672462+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2632,"log_dups_size":0,"ondisk_log_size":2632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:44.575457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172490752,"num_objects":2632,"num_object_clones":0,"num_object_copies":7896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2632,"num_write_kb":168448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1596,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.647921+0000","last_change":"2024-08-17T17:24:13.542832+0000","last_active":"2024-08-17T17:26:51.647921+0000","last_peered":"2024-08-17T17:26:51.647921+0000","last_clean":"2024-08-17T17:26:51.647921+0000","last_became_active":"2024-08-17T17:24:11.570282+0000","last_became_peered":"2024-08-17T17:24:11.570282+0000","last_unstale":"2024-08-17T17:26:51.647921+0000","last_undegraded":"2024-08-17T17:26:51.647921+0000","last_fullsized":"2024-08-17T17:26:51.647921+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:49.414162+0000","scrub_duration":0.017597167,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1210,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.645269+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:26:51.645269+0000","last_peered":"2024-08-17T17:26:51.645269+0000","last_clean":"2024-08-17T17:26:51.645269+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:26:51.645269+0000","last_undegraded":"2024-08-17T17:26:51.645269+0000","last_fullsized":"2024-08-17T17:26:51.645269+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1569,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.460177+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:27:05.460177+0000","last_peered":"2024-08-17T17:27:05.460177+0000","last_clean":"2024-08-17T17:27:05.460177+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:27:05.460177+0000","last_undegraded":"2024-08-17T17:27:05.460177+0000","last_fullsized":"2024-08-17T17:27:05.460177+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1182,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.459751+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:27:05.459751+0000","last_peered":"2024-08-17T17:27:05.459751+0000","last_clean":"2024-08-17T17:27:05.459751+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:27:05.459751+0000","last_undegraded":"2024-08-17T17:27:05.459751+0000","last_fullsized":"2024-08-17T17:27:05.459751+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"1252'2795","reported_seq":2809,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.315876+0000","last_change":"2024-08-17T17:26:34.519823+0000","last_active":"2024-08-17T17:27:07.315876+0000","last_peered":"2024-08-17T17:27:07.315876+0000","last_clean":"2024-08-17T17:27:07.315876+0000","last_became_active":"2024-08-17T17:26:34.519497+0000","last_became_peered":"2024-08-17T17:26:34.519497+0000","last_unstale":"2024-08-17T17:27:07.315876+0000","last_undegraded":"2024-08-17T17:27:07.315876+0000","last_fullsized":"2024-08-17T17:27:07.315876+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2795,"log_dups_size":0,"ondisk_log_size":2795,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:59:31.807854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183173120,"num_objects":2795,"num_object_clones":0,"num_object_copies":8385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2795,"num_write_kb":178880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":835,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.630141+0000","last_change":"2024-08-17T17:24:11.936322+0000","last_active":"2024-08-17T17:26:51.630141+0000","last_peered":"2024-08-17T17:26:51.630141+0000","last_clean":"2024-08-17T17:26:51.630141+0000","last_became_active":"2024-08-17T17:24:11.574846+0000","last_became_peered":"2024-08-17T17:24:11.574846+0000","last_unstale":"2024-08-17T17:26:51.630141+0000","last_undegraded":"2024-08-17T17:26:51.630141+0000","last_fullsized":"2024-08-17T17:26:51.630141+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:11.936236+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:11.936236+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:31.674851+0000","scrub_duration":0.0035797139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1246,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.647777+0000","last_change":"2024-08-17T17:21:06.430639+0000","last_active":"2024-08-17T17:26:51.647777+0000","last_peered":"2024-08-17T17:26:51.647777+0000","last_clean":"2024-08-17T17:26:51.647777+0000","last_became_active":"2024-08-17T17:21:06.430297+0000","last_became_peered":"2024-08-17T17:21:06.430297+0000","last_unstale":"2024-08-17T17:26:51.647777+0000","last_undegraded":"2024-08-17T17:26:51.647777+0000","last_fullsized":"2024-08-17T17:26:51.647777+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:57.794620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1049,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.630869+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:26:51.630869+0000","last_peered":"2024-08-17T17:26:51.630869+0000","last_clean":"2024-08-17T17:26:51.630869+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:26:51.630869+0000","last_undegraded":"2024-08-17T17:26:51.630869+0000","last_fullsized":"2024-08-17T17:26:51.630869+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":60,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.645638+0000","last_change":"2024-08-17T17:22:48.370769+0000","last_active":"2024-08-17T17:26:51.645638+0000","last_peered":"2024-08-17T17:26:51.645638+0000","last_clean":"2024-08-17T17:26:51.645638+0000","last_became_active":"2024-08-17T17:22:48.370531+0000","last_became_peered":"2024-08-17T17:22:48.370531+0000","last_unstale":"2024-08-17T17:26:51.645638+0000","last_undegraded":"2024-08-17T17:26:51.645638+0000","last_fullsized":"2024-08-17T17:26:51.645638+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:02.943823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"1252'2759","reported_seq":2773,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.807743+0000","last_change":"2024-08-17T17:26:34.715167+0000","last_active":"2024-08-17T17:27:07.807743+0000","last_peered":"2024-08-17T17:27:07.807743+0000","last_clean":"2024-08-17T17:27:07.807743+0000","last_became_active":"2024-08-17T17:26:34.714868+0000","last_became_peered":"2024-08-17T17:26:34.714868+0000","last_unstale":"2024-08-17T17:27:07.807743+0000","last_undegraded":"2024-08-17T17:27:07.807743+0000","last_fullsized":"2024-08-17T17:27:07.807743+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2759,"log_dups_size":0,"ondisk_log_size":2759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:19.957252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180813824,"num_objects":2759,"num_object_clones":0,"num_object_copies":8277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2759,"num_write_kb":176576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1617,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.647897+0000","last_change":"2024-08-17T17:24:12.519946+0000","last_active":"2024-08-17T17:26:51.647897+0000","last_peered":"2024-08-17T17:26:51.647897+0000","last_clean":"2024-08-17T17:26:51.647897+0000","last_became_active":"2024-08-17T17:24:11.569747+0000","last_became_peered":"2024-08-17T17:24:11.569747+0000","last_unstale":"2024-08-17T17:26:51.647897+0000","last_undegraded":"2024-08-17T17:26:51.647897+0000","last_fullsized":"2024-08-17T17:26:51.647897+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.519838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.519838+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:56.003993+0000","scrub_duration":0.0068224740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1666,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.462726+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:27:05.462726+0000","last_peered":"2024-08-17T17:27:05.462726+0000","last_clean":"2024-08-17T17:27:05.462726+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:27:05.462726+0000","last_undegraded":"2024-08-17T17:27:05.462726+0000","last_fullsized":"2024-08-17T17:27:05.462726+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1058,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.462703+0000","last_change":"2024-08-17T17:21:06.428684+0000","last_active":"2024-08-17T17:27:05.462703+0000","last_peered":"2024-08-17T17:27:05.462703+0000","last_clean":"2024-08-17T17:27:05.462703+0000","last_became_active":"2024-08-17T17:21:06.428178+0000","last_became_peered":"2024-08-17T17:21:06.428178+0000","last_unstale":"2024-08-17T17:27:05.462703+0000","last_undegraded":"2024-08-17T17:27:05.462703+0000","last_fullsized":"2024-08-17T17:27:05.462703+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:14.836119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":893,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.646111+0000","last_change":"2024-08-17T17:21:06.443565+0000","last_active":"2024-08-17T17:26:51.646111+0000","last_peered":"2024-08-17T17:26:51.646111+0000","last_clean":"2024-08-17T17:26:51.646111+0000","last_became_active":"2024-08-17T17:21:06.443215+0000","last_became_peered":"2024-08-17T17:21:06.443215+0000","last_unstale":"2024-08-17T17:26:51.646111+0000","last_undegraded":"2024-08-17T17:26:51.646111+0000","last_fullsized":"2024-08-17T17:26:51.646111+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:47.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"1252'2875","reported_seq":2889,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.352533+0000","last_change":"2024-08-17T17:26:34.519795+0000","last_active":"2024-08-17T17:27:07.352533+0000","last_peered":"2024-08-17T17:27:07.352533+0000","last_clean":"2024-08-17T17:27:07.352533+0000","last_became_active":"2024-08-17T17:26:34.519426+0000","last_became_peered":"2024-08-17T17:26:34.519426+0000","last_unstale":"2024-08-17T17:27:07.352533+0000","last_undegraded":"2024-08-17T17:27:07.352533+0000","last_fullsized":"2024-08-17T17:27:07.352533+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2875,"log_dups_size":0,"ondisk_log_size":2875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:14.719799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188416000,"num_objects":2875,"num_object_clones":0,"num_object_copies":8625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2875,"num_write_kb":184000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1559,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:52.783448+0000","last_change":"2024-08-17T17:24:13.795227+0000","last_active":"2024-08-17T17:26:52.783448+0000","last_peered":"2024-08-17T17:26:52.783448+0000","last_clean":"2024-08-17T17:26:52.783448+0000","last_became_active":"2024-08-17T17:24:11.578352+0000","last_became_peered":"2024-08-17T17:24:11.578352+0000","last_unstale":"2024-08-17T17:26:52.783448+0000","last_undegraded":"2024-08-17T17:26:52.783448+0000","last_fullsized":"2024-08-17T17:26:52.783448+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:36:39.440808+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1237,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.645777+0000","last_change":"2024-08-17T17:21:06.456978+0000","last_active":"2024-08-17T17:26:51.645777+0000","last_peered":"2024-08-17T17:26:51.645777+0000","last_clean":"2024-08-17T17:26:51.645777+0000","last_became_active":"2024-08-17T17:21:06.452265+0000","last_became_peered":"2024-08-17T17:21:06.452265+0000","last_unstale":"2024-08-17T17:26:51.645777+0000","last_undegraded":"2024-08-17T17:26:51.645777+0000","last_fullsized":"2024-08-17T17:26:51.645777+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:52.893746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1219,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.354924+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:26:57.354924+0000","last_peered":"2024-08-17T17:26:57.354924+0000","last_clean":"2024-08-17T17:26:57.354924+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:26:57.354924+0000","last_undegraded":"2024-08-17T17:26:57.354924+0000","last_fullsized":"2024-08-17T17:26:57.354924+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1134,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.462978+0000","last_change":"2024-08-17T17:21:06.435754+0000","last_active":"2024-08-17T17:27:05.462978+0000","last_peered":"2024-08-17T17:27:05.462978+0000","last_clean":"2024-08-17T17:27:05.462978+0000","last_became_active":"2024-08-17T17:21:06.435442+0000","last_became_peered":"2024-08-17T17:21:06.435442+0000","last_unstale":"2024-08-17T17:27:05.462978+0000","last_undegraded":"2024-08-17T17:27:05.462978+0000","last_fullsized":"2024-08-17T17:27:05.462978+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:34.849042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"1252'2653","reported_seq":2667,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.800445+0000","last_change":"2024-08-17T17:26:34.716730+0000","last_active":"2024-08-17T17:27:07.800445+0000","last_peered":"2024-08-17T17:27:07.800445+0000","last_clean":"2024-08-17T17:27:07.800445+0000","last_became_active":"2024-08-17T17:26:34.716457+0000","last_became_peered":"2024-08-17T17:26:34.716457+0000","last_unstale":"2024-08-17T17:27:07.800445+0000","last_undegraded":"2024-08-17T17:27:07.800445+0000","last_fullsized":"2024-08-17T17:27:07.800445+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2653,"log_dups_size":0,"ondisk_log_size":2653,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:51.133413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173867008,"num_objects":2653,"num_object_clones":0,"num_object_copies":7959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2653,"num_write_kb":169792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":822,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.460366+0000","last_change":"2024-08-17T17:24:55.041464+0000","last_active":"2024-08-17T17:27:05.460366+0000","last_peered":"2024-08-17T17:27:05.460366+0000","last_clean":"2024-08-17T17:27:05.460366+0000","last_became_active":"2024-08-17T17:24:31.800289+0000","last_became_peered":"2024-08-17T17:24:31.800289+0000","last_unstale":"2024-08-17T17:27:05.460366+0000","last_undegraded":"2024-08-17T17:27:05.460366+0000","last_fullsized":"2024-08-17T17:27:05.460366+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:55.041378+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:55.041378+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:43.974501+0000","scrub_duration":0.001623642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1212,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:27:02.809354+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:27:02.809354+0000","last_peered":"2024-08-17T17:27:02.809354+0000","last_clean":"2024-08-17T17:27:02.809354+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:27:02.809354+0000","last_undegraded":"2024-08-17T17:27:02.809354+0000","last_fullsized":"2024-08-17T17:27:02.809354+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":902,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.648740+0000","last_change":"2024-08-17T17:21:06.434991+0000","last_active":"2024-08-17T17:26:51.648740+0000","last_peered":"2024-08-17T17:26:51.648740+0000","last_clean":"2024-08-17T17:26:51.648740+0000","last_became_active":"2024-08-17T17:21:06.434630+0000","last_became_peered":"2024-08-17T17:21:06.434630+0000","last_unstale":"2024-08-17T17:26:51.648740+0000","last_undegraded":"2024-08-17T17:26:51.648740+0000","last_fullsized":"2024-08-17T17:26:51.648740+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:24.104609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":844,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.645638+0000","last_change":"2024-08-17T17:21:06.462532+0000","last_active":"2024-08-17T17:26:51.645638+0000","last_peered":"2024-08-17T17:26:51.645638+0000","last_clean":"2024-08-17T17:26:51.645638+0000","last_became_active":"2024-08-17T17:21:06.462183+0000","last_became_peered":"2024-08-17T17:21:06.462183+0000","last_unstale":"2024-08-17T17:26:51.645638+0000","last_undegraded":"2024-08-17T17:26:51.645638+0000","last_fullsized":"2024-08-17T17:26:51.645638+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:50.033322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.e","version":"1252'2870","reported_seq":2884,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.311961+0000","last_change":"2024-08-17T17:26:34.716166+0000","last_active":"2024-08-17T17:27:07.311961+0000","last_peered":"2024-08-17T17:27:07.311961+0000","last_clean":"2024-08-17T17:27:07.311961+0000","last_became_active":"2024-08-17T17:26:34.715299+0000","last_became_peered":"2024-08-17T17:26:34.715299+0000","last_unstale":"2024-08-17T17:27:07.311961+0000","last_undegraded":"2024-08-17T17:27:07.311961+0000","last_fullsized":"2024-08-17T17:27:07.311961+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2870,"log_dups_size":0,"ondisk_log_size":2870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:37.004999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188088320,"num_objects":2870,"num_object_clones":0,"num_object_copies":8610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2870,"num_write_kb":183680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":839,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.460027+0000","last_change":"2024-08-17T17:24:53.103270+0000","last_active":"2024-08-17T17:27:05.460027+0000","last_peered":"2024-08-17T17:27:05.460027+0000","last_clean":"2024-08-17T17:27:05.460027+0000","last_became_active":"2024-08-17T17:24:31.800823+0000","last_became_peered":"2024-08-17T17:24:31.800823+0000","last_unstale":"2024-08-17T17:27:05.460027+0000","last_undegraded":"2024-08-17T17:27:05.460027+0000","last_fullsized":"2024-08-17T17:27:05.460027+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.103133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.103133+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:52.444781+0000","scrub_duration":0.0074281759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1057,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.462880+0000","last_change":"2024-08-17T17:21:06.433720+0000","last_active":"2024-08-17T17:27:05.462880+0000","last_peered":"2024-08-17T17:27:05.462880+0000","last_clean":"2024-08-17T17:27:05.462880+0000","last_became_active":"2024-08-17T17:21:06.433292+0000","last_became_peered":"2024-08-17T17:21:06.433292+0000","last_unstale":"2024-08-17T17:27:05.462880+0000","last_undegraded":"2024-08-17T17:27:05.462880+0000","last_fullsized":"2024-08-17T17:27:05.462880+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:32.246594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1212,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.649478+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:26:51.649478+0000","last_peered":"2024-08-17T17:26:51.649478+0000","last_clean":"2024-08-17T17:26:51.649478+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:26:51.649478+0000","last_undegraded":"2024-08-17T17:26:51.649478+0000","last_fullsized":"2024-08-17T17:26:51.649478+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":804,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.645158+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:26:51.645158+0000","last_peered":"2024-08-17T17:26:51.645158+0000","last_clean":"2024-08-17T17:26:51.645158+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:26:51.645158+0000","last_undegraded":"2024-08-17T17:26:51.645158+0000","last_fullsized":"2024-08-17T17:26:51.645158+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"1252'2786","reported_seq":2800,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.342761+0000","last_change":"2024-08-17T17:26:34.715123+0000","last_active":"2024-08-17T17:27:07.342761+0000","last_peered":"2024-08-17T17:27:07.342761+0000","last_clean":"2024-08-17T17:27:07.342761+0000","last_became_active":"2024-08-17T17:26:34.714789+0000","last_became_peered":"2024-08-17T17:26:34.714789+0000","last_unstale":"2024-08-17T17:27:07.342761+0000","last_undegraded":"2024-08-17T17:27:07.342761+0000","last_fullsized":"2024-08-17T17:27:07.342761+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2786,"log_dups_size":0,"ondisk_log_size":2786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:19.314570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182583296,"num_objects":2786,"num_object_clones":0,"num_object_copies":8358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2786,"num_write_kb":178304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1563,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:52.353747+0000","last_change":"2024-08-17T17:24:49.119578+0000","last_active":"2024-08-17T17:26:52.353747+0000","last_peered":"2024-08-17T17:26:52.353747+0000","last_clean":"2024-08-17T17:26:52.353747+0000","last_became_active":"2024-08-17T17:24:31.807987+0000","last_became_peered":"2024-08-17T17:24:31.807987+0000","last_unstale":"2024-08-17T17:26:52.353747+0000","last_undegraded":"2024-08-17T17:26:52.353747+0000","last_fullsized":"2024-08-17T17:26:52.353747+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:34.052786+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1622,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.459520+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:27:05.459520+0000","last_peered":"2024-08-17T17:27:05.459520+0000","last_clean":"2024-08-17T17:27:05.459520+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:27:05.459520+0000","last_undegraded":"2024-08-17T17:27:05.459520+0000","last_fullsized":"2024-08-17T17:27:05.459520+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":854,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.641296+0000","last_change":"2024-08-17T17:21:06.452395+0000","last_active":"2024-08-17T17:26:51.641296+0000","last_peered":"2024-08-17T17:26:51.641296+0000","last_clean":"2024-08-17T17:26:51.641296+0000","last_became_active":"2024-08-17T17:21:06.451867+0000","last_became_peered":"2024-08-17T17:21:06.451867+0000","last_unstale":"2024-08-17T17:26:51.641296+0000","last_undegraded":"2024-08-17T17:26:51.641296+0000","last_fullsized":"2024-08-17T17:26:51.641296+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:50.597092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1612,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.458629+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:27:05.458629+0000","last_peered":"2024-08-17T17:27:05.458629+0000","last_clean":"2024-08-17T17:27:05.458629+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:27:05.458629+0000","last_undegraded":"2024-08-17T17:27:05.458629+0000","last_fullsized":"2024-08-17T17:27:05.458629+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"1252'2541","reported_seq":2562,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.661737+0000","last_change":"2024-08-17T17:26:34.412017+0000","last_active":"2024-08-17T17:27:05.661737+0000","last_peered":"2024-08-17T17:27:05.661737+0000","last_clean":"2024-08-17T17:27:05.661737+0000","last_became_active":"2024-08-17T17:26:34.411768+0000","last_became_peered":"2024-08-17T17:26:34.411768+0000","last_unstale":"2024-08-17T17:27:05.661737+0000","last_undegraded":"2024-08-17T17:27:05.661737+0000","last_fullsized":"2024-08-17T17:27:05.661737+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":2541,"log_dups_size":0,"ondisk_log_size":2541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:06.384935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166526976,"num_objects":2541,"num_object_clones":0,"num_object_copies":7623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2541,"num_write_kb":162624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1648,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.459303+0000","last_change":"2024-08-17T17:24:12.033217+0000","last_active":"2024-08-17T17:27:05.459303+0000","last_peered":"2024-08-17T17:27:05.459303+0000","last_clean":"2024-08-17T17:27:05.459303+0000","last_became_active":"2024-08-17T17:24:11.572877+0000","last_became_peered":"2024-08-17T17:24:11.572877+0000","last_unstale":"2024-08-17T17:27:05.459303+0000","last_undegraded":"2024-08-17T17:27:05.459303+0000","last_fullsized":"2024-08-17T17:27:05.459303+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.033101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.033101+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.264475+0000","scrub_duration":20,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":223,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.640413+0000","last_change":"2024-08-17T17:21:06.519965+0000","last_active":"2024-08-17T17:26:51.640413+0000","last_peered":"2024-08-17T17:26:51.640413+0000","last_clean":"2024-08-17T17:26:51.640413+0000","last_became_active":"2024-08-17T17:21:06.459952+0000","last_became_peered":"2024-08-17T17:21:06.459952+0000","last_unstale":"2024-08-17T17:26:51.640413+0000","last_undegraded":"2024-08-17T17:26:51.640413+0000","last_fullsized":"2024-08-17T17:26:51.640413+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:27.828098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1209,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.460667+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:27:05.460667+0000","last_peered":"2024-08-17T17:27:05.460667+0000","last_clean":"2024-08-17T17:27:05.460667+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:27:05.460667+0000","last_undegraded":"2024-08-17T17:27:05.460667+0000","last_fullsized":"2024-08-17T17:27:05.460667+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1185,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.354763+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:26:57.354763+0000","last_peered":"2024-08-17T17:26:57.354763+0000","last_clean":"2024-08-17T17:26:57.354763+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:26:57.354763+0000","last_undegraded":"2024-08-17T17:26:57.354763+0000","last_fullsized":"2024-08-17T17:26:57.354763+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":713,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.459523+0000","last_change":"2024-08-17T17:24:48.935651+0000","last_active":"2024-08-17T17:27:05.459523+0000","last_peered":"2024-08-17T17:27:05.459523+0000","last_clean":"2024-08-17T17:27:05.459523+0000","last_became_active":"2024-08-17T17:24:40.897753+0000","last_became_peered":"2024-08-17T17:24:40.897753+0000","last_unstale":"2024-08-17T17:27:05.459523+0000","last_undegraded":"2024-08-17T17:27:05.459523+0000","last_fullsized":"2024-08-17T17:27:05.459523+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:10:15.118428+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1216,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.640757+0000","last_change":"2024-08-17T17:21:06.458000+0000","last_active":"2024-08-17T17:26:51.640757+0000","last_peered":"2024-08-17T17:26:51.640757+0000","last_clean":"2024-08-17T17:26:51.640757+0000","last_became_active":"2024-08-17T17:21:06.457411+0000","last_became_peered":"2024-08-17T17:21:06.457411+0000","last_unstale":"2024-08-17T17:26:51.640757+0000","last_undegraded":"2024-08-17T17:26:51.640757+0000","last_fullsized":"2024-08-17T17:26:51.640757+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:52.740639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1207,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.640488+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:26:51.640488+0000","last_peered":"2024-08-17T17:26:51.640488+0000","last_clean":"2024-08-17T17:26:51.640488+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:26:51.640488+0000","last_undegraded":"2024-08-17T17:26:51.640488+0000","last_fullsized":"2024-08-17T17:26:51.640488+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":684,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:27:02.809410+0000","last_change":"2024-08-17T17:25:03.663370+0000","last_active":"2024-08-17T17:27:02.809410+0000","last_peered":"2024-08-17T17:27:02.809410+0000","last_clean":"2024-08-17T17:27:02.809410+0000","last_became_active":"2024-08-17T17:24:40.886214+0000","last_became_peered":"2024-08-17T17:24:40.886214+0000","last_unstale":"2024-08-17T17:27:02.809410+0000","last_undegraded":"2024-08-17T17:27:02.809410+0000","last_fullsized":"2024-08-17T17:27:02.809410+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:11:39.272029+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":876,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.629987+0000","last_change":"2024-08-17T17:21:06.454587+0000","last_active":"2024-08-17T17:26:51.629987+0000","last_peered":"2024-08-17T17:26:51.629987+0000","last_clean":"2024-08-17T17:26:51.629987+0000","last_became_active":"2024-08-17T17:21:06.454229+0000","last_became_peered":"2024-08-17T17:21:06.454229+0000","last_unstale":"2024-08-17T17:26:51.629987+0000","last_undegraded":"2024-08-17T17:26:51.629987+0000","last_fullsized":"2024-08-17T17:26:51.629987+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:38.340263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1182,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.649455+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:26:51.649455+0000","last_peered":"2024-08-17T17:26:51.649455+0000","last_clean":"2024-08-17T17:26:51.649455+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:26:51.649455+0000","last_undegraded":"2024-08-17T17:26:51.649455+0000","last_fullsized":"2024-08-17T17:26:51.649455+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1650,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.462840+0000","last_change":"2024-08-17T17:24:20.649977+0000","last_active":"2024-08-17T17:27:05.462840+0000","last_peered":"2024-08-17T17:27:05.462840+0000","last_clean":"2024-08-17T17:27:05.462840+0000","last_became_active":"2024-08-17T17:24:20.649336+0000","last_became_peered":"2024-08-17T17:24:20.649336+0000","last_unstale":"2024-08-17T17:27:05.462840+0000","last_undegraded":"2024-08-17T17:27:05.462840+0000","last_fullsized":"2024-08-17T17:27:05.462840+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:57.187271+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1212,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.645715+0000","last_change":"2024-08-17T17:21:06.442796+0000","last_active":"2024-08-17T17:26:51.645715+0000","last_peered":"2024-08-17T17:26:51.645715+0000","last_clean":"2024-08-17T17:26:51.645715+0000","last_became_active":"2024-08-17T17:21:06.442466+0000","last_became_peered":"2024-08-17T17:21:06.442466+0000","last_unstale":"2024-08-17T17:26:51.645715+0000","last_undegraded":"2024-08-17T17:26:51.645715+0000","last_fullsized":"2024-08-17T17:26:51.645715+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:36.366620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1568,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.460861+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:27:05.460861+0000","last_peered":"2024-08-17T17:27:05.460861+0000","last_clean":"2024-08-17T17:27:05.460861+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:27:05.460861+0000","last_undegraded":"2024-08-17T17:27:05.460861+0000","last_fullsized":"2024-08-17T17:27:05.460861+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1431,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:27:02.454973+0000","last_change":"2024-08-17T17:25:07.472056+0000","last_active":"2024-08-17T17:27:02.454973+0000","last_peered":"2024-08-17T17:27:02.454973+0000","last_clean":"2024-08-17T17:27:02.454973+0000","last_became_active":"2024-08-17T17:24:40.883356+0000","last_became_peered":"2024-08-17T17:24:40.883356+0000","last_unstale":"2024-08-17T17:27:02.454973+0000","last_undegraded":"2024-08-17T17:27:02.454973+0000","last_fullsized":"2024-08-17T17:27:02.454973+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:43:27.883917+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1048,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.646686+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:26:51.646686+0000","last_peered":"2024-08-17T17:26:51.646686+0000","last_clean":"2024-08-17T17:26:51.646686+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:26:51.646686+0000","last_undegraded":"2024-08-17T17:26:51.646686+0000","last_fullsized":"2024-08-17T17:26:51.646686+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1601,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.458525+0000","last_change":"2024-08-17T17:21:06.468788+0000","last_active":"2024-08-17T17:27:05.458525+0000","last_peered":"2024-08-17T17:27:05.458525+0000","last_clean":"2024-08-17T17:27:05.458525+0000","last_became_active":"2024-08-17T17:21:06.467141+0000","last_became_peered":"2024-08-17T17:21:06.467141+0000","last_unstale":"2024-08-17T17:27:05.458525+0000","last_undegraded":"2024-08-17T17:27:05.458525+0000","last_fullsized":"2024-08-17T17:27:05.458525+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:51.301359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1431,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.469263+0000","last_change":"2024-08-17T17:24:53.130774+0000","last_active":"2024-08-17T17:27:05.469263+0000","last_peered":"2024-08-17T17:27:05.469263+0000","last_clean":"2024-08-17T17:27:05.469263+0000","last_became_active":"2024-08-17T17:24:40.883388+0000","last_became_peered":"2024-08-17T17:24:40.883388+0000","last_unstale":"2024-08-17T17:27:05.469263+0000","last_undegraded":"2024-08-17T17:27:05.469263+0000","last_fullsized":"2024-08-17T17:27:05.469263+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:36.772875+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1215,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.441295+0000","last_change":"2024-08-17T17:21:06.435998+0000","last_active":"2024-08-17T17:26:57.441295+0000","last_peered":"2024-08-17T17:26:57.441295+0000","last_clean":"2024-08-17T17:26:57.441295+0000","last_became_active":"2024-08-17T17:21:06.435579+0000","last_became_peered":"2024-08-17T17:21:06.435579+0000","last_unstale":"2024-08-17T17:26:57.441295+0000","last_undegraded":"2024-08-17T17:26:57.441295+0000","last_fullsized":"2024-08-17T17:26:57.441295+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:01.743074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1631,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.458912+0000","last_change":"2024-08-17T17:21:06.473627+0000","last_active":"2024-08-17T17:27:05.458912+0000","last_peered":"2024-08-17T17:27:05.458912+0000","last_clean":"2024-08-17T17:27:05.458912+0000","last_became_active":"2024-08-17T17:21:06.473062+0000","last_became_peered":"2024-08-17T17:21:06.473062+0000","last_unstale":"2024-08-17T17:27:05.458912+0000","last_undegraded":"2024-08-17T17:27:05.458912+0000","last_fullsized":"2024-08-17T17:27:05.458912+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:21.413163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":791,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:27:02.809266+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:27:02.809266+0000","last_peered":"2024-08-17T17:27:02.809266+0000","last_clean":"2024-08-17T17:27:02.809266+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:27:02.809266+0000","last_undegraded":"2024-08-17T17:27:02.809266+0000","last_fullsized":"2024-08-17T17:27:02.809266+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1010,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.464193+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:27:05.464193+0000","last_peered":"2024-08-17T17:27:05.464193+0000","last_clean":"2024-08-17T17:27:05.464193+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:27:05.464193+0000","last_undegraded":"2024-08-17T17:27:05.464193+0000","last_fullsized":"2024-08-17T17:27:05.464193+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1121,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.460953+0000","last_change":"2024-08-17T17:21:06.471265+0000","last_active":"2024-08-17T17:27:05.460953+0000","last_peered":"2024-08-17T17:27:05.460953+0000","last_clean":"2024-08-17T17:27:05.460953+0000","last_became_active":"2024-08-17T17:21:06.470943+0000","last_became_peered":"2024-08-17T17:21:06.470943+0000","last_unstale":"2024-08-17T17:27:05.460953+0000","last_undegraded":"2024-08-17T17:27:05.460953+0000","last_fullsized":"2024-08-17T17:27:05.460953+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:41.070024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1563,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:57.803690+0000","last_change":"2024-08-17T17:25:02.696608+0000","last_active":"2024-08-17T17:26:57.803690+0000","last_peered":"2024-08-17T17:26:57.803690+0000","last_clean":"2024-08-17T17:26:57.803690+0000","last_became_active":"2024-08-17T17:24:40.877715+0000","last_became_peered":"2024-08-17T17:24:40.877715+0000","last_unstale":"2024-08-17T17:26:57.803690+0000","last_undegraded":"2024-08-17T17:26:57.803690+0000","last_fullsized":"2024-08-17T17:26:57.803690+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:48:00.941285+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1607,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.459227+0000","last_change":"2024-08-17T17:22:48.377351+0000","last_active":"2024-08-17T17:27:05.459227+0000","last_peered":"2024-08-17T17:27:05.459227+0000","last_clean":"2024-08-17T17:27:05.459227+0000","last_became_active":"2024-08-17T17:22:48.377059+0000","last_became_peered":"2024-08-17T17:22:48.377059+0000","last_unstale":"2024-08-17T17:27:05.459227+0000","last_undegraded":"2024-08-17T17:27:05.459227+0000","last_fullsized":"2024-08-17T17:27:05.459227+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:15.250902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1232,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.631128+0000","last_change":"2024-08-17T17:21:06.459139+0000","last_active":"2024-08-17T17:26:51.631128+0000","last_peered":"2024-08-17T17:26:51.631128+0000","last_clean":"2024-08-17T17:26:51.631128+0000","last_became_active":"2024-08-17T17:21:06.457850+0000","last_became_peered":"2024-08-17T17:21:06.457850+0000","last_unstale":"2024-08-17T17:26:51.631128+0000","last_undegraded":"2024-08-17T17:26:51.631128+0000","last_fullsized":"2024-08-17T17:26:51.631128+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:13.888576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1509,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.460623+0000","last_change":"2024-08-17T17:24:49.911734+0000","last_active":"2024-08-17T17:27:05.460623+0000","last_peered":"2024-08-17T17:27:05.460623+0000","last_clean":"2024-08-17T17:27:05.460623+0000","last_became_active":"2024-08-17T17:24:40.899102+0000","last_became_peered":"2024-08-17T17:24:40.899102+0000","last_unstale":"2024-08-17T17:27:05.460623+0000","last_undegraded":"2024-08-17T17:27:05.460623+0000","last_fullsized":"2024-08-17T17:27:05.460623+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.911622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.911622+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:30:27.987033+0000","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1648,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.462950+0000","last_change":"2024-08-17T17:21:06.446321+0000","last_active":"2024-08-17T17:27:05.462950+0000","last_peered":"2024-08-17T17:27:05.462950+0000","last_clean":"2024-08-17T17:27:05.462950+0000","last_became_active":"2024-08-17T17:21:06.446019+0000","last_became_peered":"2024-08-17T17:21:06.446019+0000","last_unstale":"2024-08-17T17:27:05.462950+0000","last_undegraded":"2024-08-17T17:27:05.462950+0000","last_fullsized":"2024-08-17T17:27:05.462950+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:26.750070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":201,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.631133+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:26:51.631133+0000","last_peered":"2024-08-17T17:26:51.631133+0000","last_clean":"2024-08-17T17:26:51.631133+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:26:51.631133+0000","last_undegraded":"2024-08-17T17:26:51.631133+0000","last_fullsized":"2024-08-17T17:26:51.631133+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":713,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.459681+0000","last_change":"2024-08-17T17:24:46.946320+0000","last_active":"2024-08-17T17:27:05.459681+0000","last_peered":"2024-08-17T17:27:05.459681+0000","last_clean":"2024-08-17T17:27:05.459681+0000","last_became_active":"2024-08-17T17:24:40.880380+0000","last_became_peered":"2024-08-17T17:24:40.880380+0000","last_unstale":"2024-08-17T17:27:05.459681+0000","last_undegraded":"2024-08-17T17:27:05.459681+0000","last_fullsized":"2024-08-17T17:27:05.459681+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:54:30.637162+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1230,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.631231+0000","last_change":"2024-08-17T17:21:06.465804+0000","last_active":"2024-08-17T17:26:51.631231+0000","last_peered":"2024-08-17T17:26:51.631231+0000","last_clean":"2024-08-17T17:26:51.631231+0000","last_became_active":"2024-08-17T17:21:06.465461+0000","last_became_peered":"2024-08-17T17:21:06.465461+0000","last_unstale":"2024-08-17T17:26:51.631231+0000","last_undegraded":"2024-08-17T17:26:51.631231+0000","last_fullsized":"2024-08-17T17:26:51.631231+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:06.443667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1222,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.644853+0000","last_change":"2024-08-17T17:21:06.481634+0000","last_active":"2024-08-17T17:26:51.644853+0000","last_peered":"2024-08-17T17:26:51.644853+0000","last_clean":"2024-08-17T17:26:51.644853+0000","last_became_active":"2024-08-17T17:21:06.469081+0000","last_became_peered":"2024-08-17T17:21:06.469081+0000","last_unstale":"2024-08-17T17:26:51.644853+0000","last_undegraded":"2024-08-17T17:26:51.644853+0000","last_fullsized":"2024-08-17T17:26:51.644853+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:53:26.954738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1230,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.631029+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:26:51.631029+0000","last_peered":"2024-08-17T17:26:51.631029+0000","last_clean":"2024-08-17T17:26:51.631029+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:26:51.631029+0000","last_undegraded":"2024-08-17T17:26:51.631029+0000","last_fullsized":"2024-08-17T17:26:51.631029+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1181,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.644976+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:26:51.644976+0000","last_peered":"2024-08-17T17:26:51.644976+0000","last_clean":"2024-08-17T17:26:51.644976+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:26:51.644976+0000","last_undegraded":"2024-08-17T17:26:51.644976+0000","last_fullsized":"2024-08-17T17:26:51.644976+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1622,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.463555+0000","last_change":"2024-08-17T17:25:02.417545+0000","last_active":"2024-08-17T17:27:05.463555+0000","last_peered":"2024-08-17T17:27:05.463555+0000","last_clean":"2024-08-17T17:27:05.463555+0000","last_became_active":"2024-08-17T17:24:40.883220+0000","last_became_peered":"2024-08-17T17:24:40.883220+0000","last_unstale":"2024-08-17T17:27:05.463555+0000","last_undegraded":"2024-08-17T17:27:05.463555+0000","last_fullsized":"2024-08-17T17:27:05.463555+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:28.257805+0000","scrub_duration":0.0084952199999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1607,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.461112+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:27:05.461112+0000","last_peered":"2024-08-17T17:27:05.461112+0000","last_clean":"2024-08-17T17:27:05.461112+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:27:05.461112+0000","last_undegraded":"2024-08-17T17:27:05.461112+0000","last_fullsized":"2024-08-17T17:27:05.461112+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1582,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.459846+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:27:05.459846+0000","last_peered":"2024-08-17T17:27:05.459846+0000","last_clean":"2024-08-17T17:27:05.459846+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:27:05.459846+0000","last_undegraded":"2024-08-17T17:27:05.459846+0000","last_fullsized":"2024-08-17T17:27:05.459846+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1456,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.459992+0000","last_change":"2024-08-17T17:24:47.961645+0000","last_active":"2024-08-17T17:27:05.459992+0000","last_peered":"2024-08-17T17:27:05.459992+0000","last_clean":"2024-08-17T17:27:05.459992+0000","last_became_active":"2024-08-17T17:24:40.897539+0000","last_became_peered":"2024-08-17T17:24:40.897539+0000","last_unstale":"2024-08-17T17:27:05.459992+0000","last_undegraded":"2024-08-17T17:27:05.459992+0000","last_fullsized":"2024-08-17T17:27:05.459992+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:47.961445+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:47.961445+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:04:32.031483+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1215,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.630820+0000","last_change":"2024-08-17T17:21:06.483469+0000","last_active":"2024-08-17T17:26:51.630820+0000","last_peered":"2024-08-17T17:26:51.630820+0000","last_clean":"2024-08-17T17:26:51.630820+0000","last_became_active":"2024-08-17T17:21:06.483162+0000","last_became_peered":"2024-08-17T17:21:06.483162+0000","last_unstale":"2024-08-17T17:26:51.630820+0000","last_undegraded":"2024-08-17T17:26:51.630820+0000","last_fullsized":"2024-08-17T17:26:51.630820+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:24.898353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":224,"reported_epoch":1251,"state":"active+clean","last_fresh":"2024-08-17T17:26:51.646667+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:26:51.646667+0000","last_peered":"2024-08-17T17:26:51.646667+0000","last_clean":"2024-08-17T17:26:51.646667+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:26:51.646667+0000","last_undegraded":"2024-08-17T17:26:51.646667+0000","last_fullsized":"2024-08-17T17:26:51.646667+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2859794432,"num_objects":43637,"num_object_clones":0,"num_object_copies":130911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":43637,"num_write_kb":2792768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8631943168,"data_stored":8631943168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":43637,"ondisk_log_size":43637,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201708,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2116544,"kb_used_data":1594100,"kb_used_omap":1,"kb_used_meta":522430,"kb_avail":91628608,"statfs":{"total":95995035648,"available":93827694592,"internally_reserved":0,"allocated":1632358400,"data_stored":1629766775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":534968779},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463345,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":918644,"kb_used_data":530728,"kb_used_omap":1,"kb_used_meta":387902,"kb_avail":92826508,"statfs":{"total":95995035648,"available":95054344192,"internally_reserved":0,"allocated":543465472,"data_stored":540885464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":397212105},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626986,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1930068,"kb_used_data":1302348,"kb_used_omap":42,"kb_used_meta":627669,"kb_avail":91815084,"statfs":{"total":95995035648,"available":94018646016,"internally_reserved":0,"allocated":1333604352,"data_stored":1331040013,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":642733116},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785464,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2111072,"kb_used_data":1609560,"kb_used_omap":48,"kb_used_meta":501455,"kb_avail":91634080,"statfs":{"total":95995035648,"available":93833297920,"internally_reserved":0,"allocated":1648189440,"data_stored":1645621002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":513490606},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724981,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2106860,"kb_used_data":1602468,"kb_used_omap":1,"kb_used_meta":504382,"kb_avail":91638292,"statfs":{"total":95995035648,"available":93837611008,"internally_reserved":0,"allocated":1640927232,"data_stored":1638347690,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":516487622},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986618,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":975032,"kb_used_data":534636,"kb_used_omap":1,"kb_used_meta":440382,"kb_avail":92770120,"statfs":{"total":95995035648,"available":94996602880,"internally_reserved":0,"allocated":547467264,"data_stored":544883217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":450951622},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706225,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1050088,"kb_used_data":518304,"kb_used_omap":35,"kb_used_meta":531740,"kb_avail":92695064,"statfs":{"total":95995035648,"available":94919745536,"internally_reserved":0,"allocated":530743296,"data_stored":528167807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":544501815},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980831998,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1263592,"kb_used_data":711900,"kb_used_omap":65,"kb_used_meta":551614,"kb_avail":92481560,"statfs":{"total":95995035648,"available":94701117440,"internally_reserved":0,"allocated":728985600,"data_stored":726413651,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":564852923},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":726138880,"data_stored":726138880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1657536512,"data_stored":1657536512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":535822336,"data_stored":535822336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1346502656,"data_stored":1346502656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":543293440,"data_stored":543293440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1649737728,"data_stored":1649737728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":535953408,"data_stored":535953408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1636958208,"data_stored":1636958208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:27:13.785 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:27:13.785 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:27:13.786 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2024-08-17T17:27:13.786 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:27:13.794 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:13 smithi162 ceph-mon[104668]: pgmap v1711: 119 pgs: 119 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.62k op/s 2024-08-17T17:27:13.810 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:13.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:13.913 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:13.951 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:13.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:13.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:14.053 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:14.089 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:14.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:14.191 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:14.230 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:14.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:14.259 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:14.332 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:14.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.386 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:14.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:14.459 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 28 16 54742 54726 122.139 97.5625 0.00689583 0.00818264 2024-08-17T17:27:14.487 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:14.536 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:14.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:14.638 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:14.674 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:14.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:14.776 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:14.813 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:14.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:14.915 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:14.953 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:14.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:14.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:15.055 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:15.098 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:15.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:15.144 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:15.200 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:15.234 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:15.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:15.336 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:15.373 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:15.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:15.459 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 29 16 56308 56292 121.302 97.875 0.0075285 0.0082387 2024-08-17T17:27:15.476 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:15.506 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:27:15.511 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:15.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:15.524 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:15 smithi063 ceph-mon[114756]: pgmap v1712: 119 pgs: 119 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-08-17T17:27:15.524 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:15 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4066364698' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:27:15.524 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:15 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:27:15.525 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:15 smithi063 ceph-mon[109843]: pgmap v1712: 119 pgs: 119 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-08-17T17:27:15.525 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:15 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4066364698' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:27:15.525 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:15 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:27:15.612 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:15.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.651 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:15.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:15.688 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:27:15.688 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:27:15.688 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:27:15.688 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:27:15.689 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.689 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88382 7.0 7:01e65490:::benchmark_data_smithi063_196002_object88381:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:27:15.689 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:00.856365+0000", 2024-08-17T17:27:15.689 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 494.82662447500002, 2024-08-17T17:27:15.689 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.84903507199999995, 2024-08-17T17:27:15.689 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.689 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.689 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.689 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:15.689 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:15.689 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88382 2024-08-17T17:27:15.689 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.690 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.690 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.690 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.690 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.856365+0000", 2024-08-17T17:27:15.690 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.690 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.690 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.690 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.690 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.856365+0000", 2024-08-17T17:27:15.690 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.690 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.690 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.691 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.691 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.856362+0000", 2024-08-17T17:27:15.691 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999976 2024-08-17T17:27:15.691 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.691 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.691 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.691 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.856412+0000", 2024-08-17T17:27:15.691 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9360000000000002e-05 2024-08-17T17:27:15.691 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.691 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.691 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.691 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.856429+0000", 2024-08-17T17:27:15.692 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7076999999999999e-05 2024-08-17T17:27:15.692 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.692 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.692 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.692 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.856438+0000", 2024-08-17T17:27:15.692 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9719999999999998e-06 2024-08-17T17:27:15.692 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.692 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.693 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.693 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.856501+0000", 2024-08-17T17:27:15.694 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3445e-05 2024-08-17T17:27:15.694 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.694 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.694 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.694 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.856511+0000", 2024-08-17T17:27:15.694 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7780000000000002e-06 2024-08-17T17:27:15.694 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.694 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.694 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.694 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.662054+0000", 2024-08-17T17:27:15.694 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.80554313 2024-08-17T17:27:15.695 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.695 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.695 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.695 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.662289+0000", 2024-08-17T17:27:15.695 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00023489999999999999 2024-08-17T17:27:15.695 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.695 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.695 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:27:15.695 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.662480+0000", 2024-08-17T17:27:15.695 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000190834 2024-08-17T17:27:15.695 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.695 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.695 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.696 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.664477+0000", 2024-08-17T17:27:15.696 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0019967959999999999 2024-08-17T17:27:15.696 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.696 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.696 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.696 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.674142+0000", 2024-08-17T17:27:15.696 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0096652030000000007 2024-08-17T17:27:15.696 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.696 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.696 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.696 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.705274+0000", 2024-08-17T17:27:15.696 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.031131927 2024-08-17T17:27:15.696 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.697 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.697 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.697 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.705359+0000", 2024-08-17T17:27:15.697 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.5195e-05 2024-08-17T17:27:15.697 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.697 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.697 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.697 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.705400+0000", 2024-08-17T17:27:15.697 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0630000000000002e-05 2024-08-17T17:27:15.697 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.697 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.697 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.697 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.698 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.698 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186368 7.1b 7:d82f1c34:::benchmark_data_smithi063_196002_object74275:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1172)", 2024-08-17T17:27:15.698 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.413998+0000", 2024-08-17T17:27:15.698 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 370.26899120100001, 2024-08-17T17:27:15.698 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0345649890000002, 2024-08-17T17:27:15.698 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.698 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:27:15.698 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.698 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:15.698 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:15.698 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186368 2024-08-17T17:27:15.698 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.698 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.699 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.699 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.699 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.413998+0000", 2024-08-17T17:27:15.699 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.699 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.699 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.699 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.699 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.413998+0000", 2024-08-17T17:27:15.699 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.699 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.699 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.700 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.700 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.413996+0000", 2024-08-17T17:27:15.700 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999976 2024-08-17T17:27:15.700 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.700 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.700 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.700 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.414003+0000", 2024-08-17T17:27:15.700 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.644e-06 2024-08-17T17:27:15.700 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.700 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.700 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.701 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.414018+0000", 2024-08-17T17:27:15.701 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5046e-05 2024-08-17T17:27:15.701 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.701 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.701 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.701 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.414027+0000", 2024-08-17T17:27:15.701 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4929999999999998e-06 2024-08-17T17:27:15.701 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.701 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.701 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.701 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.414088+0000", 2024-08-17T17:27:15.701 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1426000000000004e-05 2024-08-17T17:27:15.702 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.702 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.702 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.702 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.414237+0000", 2024-08-17T17:27:15.702 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014835599999999999 2024-08-17T17:27:15.702 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.702 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.702 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2,6", 2024-08-17T17:27:15.702 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.414706+0000", 2024-08-17T17:27:15.702 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00046953799999999998 2024-08-17T17:27:15.702 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.702 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.703 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.703 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.417185+0000", 2024-08-17T17:27:15.703 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002478334 2024-08-17T17:27:15.703 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.703 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.703 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.703 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.417499+0000", 2024-08-17T17:27:15.703 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00031453399999999999 2024-08-17T17:27:15.703 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.703 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.703 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.703 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.417901+0000", 2024-08-17T17:27:15.703 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00040172899999999998 2024-08-17T17:27:15.704 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.704 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.704 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.704 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.429809+0000", 2024-08-17T17:27:15.704 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011907778000000001 2024-08-17T17:27:15.704 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.704 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.704 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.704 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.429831+0000", 2024-08-17T17:27:15.704 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2631999999999999e-05 2024-08-17T17:27:15.704 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.704 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.704 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.705 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.432460+0000", 2024-08-17T17:27:15.705 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0026290170000001 2024-08-17T17:27:15.705 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.705 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.705 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.705 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.432468+0000", 2024-08-17T17:27:15.705 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0790000000000001e-06 2024-08-17T17:27:15.705 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.705 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.705 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.705 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.448507+0000", 2024-08-17T17:27:15.705 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0160391150000001 2024-08-17T17:27:15.706 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.706 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.706 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.706 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.448563+0000", 2024-08-17T17:27:15.706 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5458999999999999e-05 2024-08-17T17:27:15.706 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.706 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.706 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.706 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.706 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.706 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186370 7.10 7:0adb90b3:::benchmark_data_smithi063_196002_object74277:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1172)", 2024-08-17T17:27:15.706 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.415940+0000", 2024-08-17T17:27:15.707 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 370.26704935599997, 2024-08-17T17:27:15.707 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2171005479999999, 2024-08-17T17:27:15.707 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.707 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:27:15.707 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.707 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:15.707 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:15.707 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186370 2024-08-17T17:27:15.707 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.707 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.707 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.707 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.707 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.415940+0000", 2024-08-17T17:27:15.708 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.708 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.708 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.708 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.708 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.415940+0000", 2024-08-17T17:27:15.708 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.708 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.708 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.708 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.708 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.415938+0000", 2024-08-17T17:27:15.708 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:27:15.708 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.708 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.709 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.709 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.415949+0000", 2024-08-17T17:27:15.709 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0644e-05 2024-08-17T17:27:15.709 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.709 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.709 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.709 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.415962+0000", 2024-08-17T17:27:15.709 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3839000000000001e-05 2024-08-17T17:27:15.709 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.709 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.709 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.709 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.415970+0000", 2024-08-17T17:27:15.709 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.289e-06 2024-08-17T17:27:15.710 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.710 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.710 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.710 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416038+0000", 2024-08-17T17:27:15.710 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8509000000000004e-05 2024-08-17T17:27:15.710 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.710 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.710 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.710 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416191+0000", 2024-08-17T17:27:15.710 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015260600000000001 2024-08-17T17:27:15.710 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.710 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.710 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:27:15.711 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416788+0000", 2024-08-17T17:27:15.711 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00059681700000000003 2024-08-17T17:27:15.711 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.711 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.711 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.711 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.433219+0000", 2024-08-17T17:27:15.711 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.016431016999999999 2024-08-17T17:27:15.711 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.711 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.711 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.711 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.433229+0000", 2024-08-17T17:27:15.711 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0749e-05 2024-08-17T17:27:15.711 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.712 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.712 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.712 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.541033+0000", 2024-08-17T17:27:15.712 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1078034649999999 2024-08-17T17:27:15.712 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.712 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.712 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:27:15.712 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.541350+0000", 2024-08-17T17:27:15.712 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00031702299999999999 2024-08-17T17:27:15.712 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.712 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.712 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.713 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.632954+0000", 2024-08-17T17:27:15.713 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.091603852999999999 2024-08-17T17:27:15.713 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.713 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.713 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.713 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.633040+0000", 2024-08-17T17:27:15.713 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6596999999999998e-05 2024-08-17T17:27:15.713 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.713 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.713 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.713 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.713 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.713 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186382 7.1b 7:d8b6fe7e:::benchmark_data_smithi063_196002_object74289:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1173)", 2024-08-17T17:27:15.714 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.445523+0000", 2024-08-17T17:27:15.714 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 370.23746589500001, 2024-08-17T17:27:15.714 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0130498399999999, 2024-08-17T17:27:15.714 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.714 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.714 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.714 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:15.714 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:15.714 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186382 2024-08-17T17:27:15.714 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.714 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.714 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.714 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.715 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445523+0000", 2024-08-17T17:27:15.715 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.715 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.715 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.715 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.715 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445523+0000", 2024-08-17T17:27:15.715 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.715 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.715 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.715 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.715 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445522+0000", 2024-08-17T17:27:15.715 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:15.716 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.716 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.716 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.716 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445528+0000", 2024-08-17T17:27:15.716 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2809999999999997e-06 2024-08-17T17:27:15.716 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.716 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.716 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.716 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445540+0000", 2024-08-17T17:27:15.716 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2225000000000001e-05 2024-08-17T17:27:15.716 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.716 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.716 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.716 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445546+0000", 2024-08-17T17:27:15.717 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8359999999999997e-06 2024-08-17T17:27:15.717 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.717 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.717 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.717 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445594+0000", 2024-08-17T17:27:15.717 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7546000000000002e-05 2024-08-17T17:27:15.717 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.717 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.717 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.717 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445601+0000", 2024-08-17T17:27:15.717 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0509999999999999e-06 2024-08-17T17:27:15.717 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.718 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.718 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.718 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.432490+0000", 2024-08-17T17:27:15.718 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98688914699999997 2024-08-17T17:27:15.718 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.718 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.718 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.718 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.432496+0000", 2024-08-17T17:27:15.718 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0340000000000002e-06 2024-08-17T17:27:15.718 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.718 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.718 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.719 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.448806+0000", 2024-08-17T17:27:15.719 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.016310134 2024-08-17T17:27:15.719 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.719 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.719 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.719 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.448890+0000", 2024-08-17T17:27:15.719 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4312000000000002e-05 2024-08-17T17:27:15.719 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.719 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.719 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:27:15.719 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.449176+0000", 2024-08-17T17:27:15.719 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00028566600000000001 2024-08-17T17:27:15.719 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.720 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.720 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.720 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.454102+0000", 2024-08-17T17:27:15.720 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.004925825 2024-08-17T17:27:15.721 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.721 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.721 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.721 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.454111+0000", 2024-08-17T17:27:15.721 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.5089999999999999e-06 2024-08-17T17:27:15.721 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.721 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.721 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.721 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.458505+0000", 2024-08-17T17:27:15.722 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0043937789999999996 2024-08-17T17:27:15.722 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.722 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.722 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.722 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.458534+0000", 2024-08-17T17:27:15.722 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9164000000000001e-05 2024-08-17T17:27:15.722 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.722 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.722 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.722 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.458573+0000", 2024-08-17T17:27:15.723 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8726000000000002e-05 2024-08-17T17:27:15.723 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.723 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.723 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.723 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.723 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.723 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186383 7.b 7:d027c7e4:::benchmark_data_smithi063_196002_object74290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1173)", 2024-08-17T17:27:15.723 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.446820+0000", 2024-08-17T17:27:15.723 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 370.23616948799997, 2024-08-17T17:27:15.723 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.042344328, 2024-08-17T17:27:15.723 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.723 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.723 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.724 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:15.724 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:15.724 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186383 2024-08-17T17:27:15.724 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.724 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.724 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.724 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.724 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446820+0000", 2024-08-17T17:27:15.724 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.724 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.724 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.724 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.724 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446820+0000", 2024-08-17T17:27:15.725 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.725 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.725 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.725 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.725 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446819+0000", 2024-08-17T17:27:15.725 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:15.725 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.725 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.725 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.725 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446823+0000", 2024-08-17T17:27:15.725 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9720000000000003e-06 2024-08-17T17:27:15.725 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.725 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.726 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.726 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446836+0000", 2024-08-17T17:27:15.726 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3427999999999999e-05 2024-08-17T17:27:15.726 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.726 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.726 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.726 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446842+0000", 2024-08-17T17:27:15.726 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4550000000000003e-06 2024-08-17T17:27:15.726 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.726 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.726 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.726 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446889+0000", 2024-08-17T17:27:15.727 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7132000000000002e-05 2024-08-17T17:27:15.727 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.727 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.727 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.727 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446896+0000", 2024-08-17T17:27:15.727 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3499999999999999e-06 2024-08-17T17:27:15.727 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.727 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.727 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.727 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.432421+0000", 2024-08-17T17:27:15.727 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98552497100000003 2024-08-17T17:27:15.727 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.728 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.728 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.728 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.432432+0000", 2024-08-17T17:27:15.728 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0730999999999999e-05 2024-08-17T17:27:15.728 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.728 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.728 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.728 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.457123+0000", 2024-08-17T17:27:15.728 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.024691437 2024-08-17T17:27:15.728 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.728 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.728 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.728 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.457230+0000", 2024-08-17T17:27:15.729 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000106691 2024-08-17T17:27:15.729 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.729 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.729 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:27:15.729 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.457673+0000", 2024-08-17T17:27:15.729 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000443153 2024-08-17T17:27:15.729 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.729 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.729 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.729 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.463204+0000", 2024-08-17T17:27:15.729 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0055307159999999998 2024-08-17T17:27:15.729 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.729 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.730 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.730 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.465008+0000", 2024-08-17T17:27:15.730 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001804225 2024-08-17T17:27:15.730 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.730 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.730 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.730 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.489076+0000", 2024-08-17T17:27:15.730 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.024068220000000001 2024-08-17T17:27:15.730 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.730 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.730 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.731 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.489124+0000", 2024-08-17T17:27:15.731 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7203999999999997e-05 2024-08-17T17:27:15.731 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.731 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.731 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.731 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.489164+0000", 2024-08-17T17:27:15.731 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0289e-05 2024-08-17T17:27:15.731 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.731 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.731 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.731 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.731 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.731 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186421 7.b 7:d3c8aa29:::benchmark_data_smithi063_196002_object74328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1174)", 2024-08-17T17:27:15.731 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:06.649818+0000", 2024-08-17T17:27:15.732 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 369.03317130699998, 2024-08-17T17:27:15.732 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.83941269600000001, 2024-08-17T17:27:15.732 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.732 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.732 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.732 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:15.732 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:15.732 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186421 2024-08-17T17:27:15.732 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.732 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.732 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.732 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.732 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649818+0000", 2024-08-17T17:27:15.732 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.732 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.733 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.733 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.733 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649818+0000", 2024-08-17T17:27:15.733 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.733 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.733 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.733 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.733 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649817+0000", 2024-08-17T17:27:15.733 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:15.733 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.733 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.733 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.734 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649822+0000", 2024-08-17T17:27:15.734 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0509999999999996e-06 2024-08-17T17:27:15.734 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.734 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.734 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.734 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649834+0000", 2024-08-17T17:27:15.734 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1656e-05 2024-08-17T17:27:15.734 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.734 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.734 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.734 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649840+0000", 2024-08-17T17:27:15.734 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7379999999999997e-06 2024-08-17T17:27:15.734 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.735 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.735 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.735 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649913+0000", 2024-08-17T17:27:15.735 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2510999999999997e-05 2024-08-17T17:27:15.735 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.735 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.735 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.735 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649922+0000", 2024-08-17T17:27:15.735 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.0920000000000004e-06 2024-08-17T17:27:15.735 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.735 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.735 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.736 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.459587+0000", 2024-08-17T17:27:15.736 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.80966524500000003 2024-08-17T17:27:15.736 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.736 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.736 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.736 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.459695+0000", 2024-08-17T17:27:15.736 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000107626 2024-08-17T17:27:15.736 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.736 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.736 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:27:15.736 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.460016+0000", 2024-08-17T17:27:15.736 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00032096400000000003 2024-08-17T17:27:15.736 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.737 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.737 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.737 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.465014+0000", 2024-08-17T17:27:15.737 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0049983270000000003 2024-08-17T17:27:15.737 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.737 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.737 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.737 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.465040+0000", 2024-08-17T17:27:15.737 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6228e-05 2024-08-17T17:27:15.737 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.737 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.737 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.737 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.489189+0000", 2024-08-17T17:27:15.738 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.024148613999999999 2024-08-17T17:27:15.738 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.738 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.738 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.738 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.489210+0000", 2024-08-17T17:27:15.738 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0996e-05 2024-08-17T17:27:15.738 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.738 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.738 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.738 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.489230+0000", 2024-08-17T17:27:15.738 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0514e-05 2024-08-17T17:27:15.738 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.739 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.739 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.739 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.739 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.739 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186422 7.1b 7:d892c863:::benchmark_data_smithi063_196002_object74329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1174)", 2024-08-17T17:27:15.739 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:06.650101+0000", 2024-08-17T17:27:15.739 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 369.03288777699998, 2024-08-17T17:27:15.739 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.80855219099999998, 2024-08-17T17:27:15.739 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.739 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.739 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.739 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:15.740 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:15.740 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186422 2024-08-17T17:27:15.740 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.740 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.740 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.740 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.740 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650101+0000", 2024-08-17T17:27:15.740 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.740 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.740 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.740 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.740 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650101+0000", 2024-08-17T17:27:15.740 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.741 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.741 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.741 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.741 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650101+0000", 2024-08-17T17:27:15.741 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:15.741 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.741 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.741 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.741 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650105+0000", 2024-08-17T17:27:15.741 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2649999999999998e-06 2024-08-17T17:27:15.741 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.741 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.742 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.742 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650115+0000", 2024-08-17T17:27:15.742 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0387999999999999e-05 2024-08-17T17:27:15.742 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.742 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.742 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.742 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650122+0000", 2024-08-17T17:27:15.742 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7880000000000001e-06 2024-08-17T17:27:15.742 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.742 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.742 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.742 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650163+0000", 2024-08-17T17:27:15.742 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0861000000000001e-05 2024-08-17T17:27:15.743 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.743 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.743 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.743 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650168+0000", 2024-08-17T17:27:15.743 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3140000000000003e-06 2024-08-17T17:27:15.743 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.743 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.743 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.743 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.450388+0000", 2024-08-17T17:27:15.743 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.80021958400000004 2024-08-17T17:27:15.743 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.743 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.744 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.744 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.450459+0000", 2024-08-17T17:27:15.744 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1019000000000006e-05 2024-08-17T17:27:15.744 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.744 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.744 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:27:15.744 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.450615+0000", 2024-08-17T17:27:15.744 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015638100000000001 2024-08-17T17:27:15.744 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.744 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.744 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.744 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.454118+0000", 2024-08-17T17:27:15.745 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0035029509999999998 2024-08-17T17:27:15.745 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.745 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.745 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.745 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.454146+0000", 2024-08-17T17:27:15.745 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8038000000000002e-05 2024-08-17T17:27:15.745 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.745 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.745 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.745 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.458598+0000", 2024-08-17T17:27:15.745 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.004451389 2024-08-17T17:27:15.745 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.745 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.746 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.746 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.458622+0000", 2024-08-17T17:27:15.746 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4394000000000001e-05 2024-08-17T17:27:15.746 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.746 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.746 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.746 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.458653+0000", 2024-08-17T17:27:15.746 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1575999999999997e-05 2024-08-17T17:27:15.746 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.747 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.747 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.747 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.747 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.747 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186448 7.1b 7:dcfed570:::benchmark_data_smithi063_196002_object74355:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1174)", 2024-08-17T17:27:15.747 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:06.676011+0000", 2024-08-17T17:27:15.747 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 369.006977699, 2024-08-17T17:27:15.747 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.79170734899999995, 2024-08-17T17:27:15.747 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.747 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.747 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.747 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:15.747 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:15.748 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186448 2024-08-17T17:27:15.748 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.748 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.748 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.748 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.748 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676011+0000", 2024-08-17T17:27:15.748 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.748 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.748 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.748 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.748 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676011+0000", 2024-08-17T17:27:15.748 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.748 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.749 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.749 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.749 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676009+0000", 2024-08-17T17:27:15.749 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:27:15.749 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.749 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.749 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.749 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676017+0000", 2024-08-17T17:27:15.749 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0490000000000004e-06 2024-08-17T17:27:15.749 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.749 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.749 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.749 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676034+0000", 2024-08-17T17:27:15.750 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6784000000000001e-05 2024-08-17T17:27:15.750 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.750 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.750 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.750 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676042+0000", 2024-08-17T17:27:15.750 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9999999999999996e-06 2024-08-17T17:27:15.750 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.750 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.750 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.750 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676094+0000", 2024-08-17T17:27:15.750 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2318000000000003e-05 2024-08-17T17:27:15.750 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.751 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.751 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.751 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676104+0000", 2024-08-17T17:27:15.751 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3470000000000005e-06 2024-08-17T17:27:15.751 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.751 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.751 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.751 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.451598+0000", 2024-08-17T17:27:15.751 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.77549462999999996 2024-08-17T17:27:15.751 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.751 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.751 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.751 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.451728+0000", 2024-08-17T17:27:15.752 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012943399999999999 2024-08-17T17:27:15.752 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.752 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.752 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:27:15.752 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.451963+0000", 2024-08-17T17:27:15.752 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00023507199999999999 2024-08-17T17:27:15.752 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.752 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.752 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.752 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.454124+0000", 2024-08-17T17:27:15.752 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021607419999999998 2024-08-17T17:27:15.752 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.753 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.753 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.753 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.454192+0000", 2024-08-17T17:27:15.753 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8164000000000007e-05 2024-08-17T17:27:15.753 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.753 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.753 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.753 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.467648+0000", 2024-08-17T17:27:15.753 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.013455801999999999 2024-08-17T17:27:15.753 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.753 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.753 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.754 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.467682+0000", 2024-08-17T17:27:15.754 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3977999999999999e-05 2024-08-17T17:27:15.754 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.754 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.754 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.754 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.467719+0000", 2024-08-17T17:27:15.754 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7069e-05 2024-08-17T17:27:15.754 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.754 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.754 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.754 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.754 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.755 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:188170 7.10 7:0e1463ac:::benchmark_data_smithi063_196002_object76077:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:27:15.755 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:09.020045+0000", 2024-08-17T17:27:15.755 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 366.662944036, 2024-08-17T17:27:15.755 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.90828101699999997, 2024-08-17T17:27:15.755 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.755 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.755 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.755 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:15.755 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:15.755 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 188170 2024-08-17T17:27:15.755 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.755 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.756 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.756 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.756 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020045+0000", 2024-08-17T17:27:15.756 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.756 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.756 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.756 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.756 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020045+0000", 2024-08-17T17:27:15.756 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.756 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.756 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.756 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.757 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020044+0000", 2024-08-17T17:27:15.757 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:15.757 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.757 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.757 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.757 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020050+0000", 2024-08-17T17:27:15.757 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3389999999999998e-06 2024-08-17T17:27:15.757 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.757 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.757 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.757 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020065+0000", 2024-08-17T17:27:15.757 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5390999999999999e-05 2024-08-17T17:27:15.757 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.758 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.758 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.758 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020073+0000", 2024-08-17T17:27:15.758 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3649999999999998e-06 2024-08-17T17:27:15.758 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.758 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.758 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.758 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020139+0000", 2024-08-17T17:27:15.758 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5932999999999994e-05 2024-08-17T17:27:15.758 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.758 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.758 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.759 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020151+0000", 2024-08-17T17:27:15.759 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.226e-05 2024-08-17T17:27:15.759 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.759 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.759 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.759 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.916518+0000", 2024-08-17T17:27:15.759 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.89636652999999999 2024-08-17T17:27:15.759 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.759 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.759 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.759 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.916620+0000", 2024-08-17T17:27:15.759 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010269500000000001 2024-08-17T17:27:15.760 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.760 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.760 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:27:15.760 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.916708+0000", 2024-08-17T17:27:15.760 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7652999999999996e-05 2024-08-17T17:27:15.760 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.760 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.760 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.760 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.922303+0000", 2024-08-17T17:27:15.761 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0055949839999999999 2024-08-17T17:27:15.761 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.761 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.761 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.761 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.923563+0000", 2024-08-17T17:27:15.761 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012598240000000001 2024-08-17T17:27:15.761 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.761 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.761 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.761 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.928276+0000", 2024-08-17T17:27:15.761 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0047131489999999998 2024-08-17T17:27:15.761 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.762 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.762 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.762 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.928304+0000", 2024-08-17T17:27:15.762 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8513e-05 2024-08-17T17:27:15.762 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.762 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.762 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.762 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.928326+0000", 2024-08-17T17:27:15.762 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.1733e-05 2024-08-17T17:27:15.762 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.762 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.762 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.762 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.763 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.763 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:188192 7.10 7:087427ab:::benchmark_data_smithi063_196002_object76099:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:27:15.763 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:09.034613+0000", 2024-08-17T17:27:15.763 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 366.64837624199998, 2024-08-17T17:27:15.763 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.89933586499999996, 2024-08-17T17:27:15.763 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.763 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.763 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.763 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:15.763 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:15.763 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 188192 2024-08-17T17:27:15.763 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.764 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.764 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.764 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.764 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034613+0000", 2024-08-17T17:27:15.764 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.764 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.764 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.764 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.764 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034613+0000", 2024-08-17T17:27:15.764 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.764 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.764 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.765 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.765 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034612+0000", 2024-08-17T17:27:15.765 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:15.765 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.765 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.765 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.765 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034617+0000", 2024-08-17T17:27:15.765 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4349999999999996e-06 2024-08-17T17:27:15.765 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.765 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.765 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.765 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034628+0000", 2024-08-17T17:27:15.766 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0743e-05 2024-08-17T17:27:15.766 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.766 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.766 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.766 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034633+0000", 2024-08-17T17:27:15.766 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3730000000000001e-06 2024-08-17T17:27:15.766 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.766 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.766 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.766 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034684+0000", 2024-08-17T17:27:15.766 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0871999999999997e-05 2024-08-17T17:27:15.766 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.767 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.767 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.767 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034697+0000", 2024-08-17T17:27:15.767 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3054000000000001e-05 2024-08-17T17:27:15.767 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.767 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.767 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.767 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:15.768 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.918720+0000", 2024-08-17T17:27:15.768 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.88402291600000005 2024-08-17T17:27:15.769 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.769 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.769 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.769 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.918786+0000", 2024-08-17T17:27:15.769 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6181999999999998e-05 2024-08-17T17:27:15.769 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.769 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.769 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:27:15.769 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.918842+0000", 2024-08-17T17:27:15.769 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5640000000000003e-05 2024-08-17T17:27:15.769 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.770 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.770 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.770 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.923571+0000", 2024-08-17T17:27:15.770 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0047287379999999997 2024-08-17T17:27:15.770 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.770 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.770 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.770 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.933868+0000", 2024-08-17T17:27:15.770 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010297059000000001 2024-08-17T17:27:15.770 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.770 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.770 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.770 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.933899+0000", 2024-08-17T17:27:15.771 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1095e-05 2024-08-17T17:27:15.771 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.771 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.771 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.771 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.933927+0000", 2024-08-17T17:27:15.771 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8529000000000001e-05 2024-08-17T17:27:15.771 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.771 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.771 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.771 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.933949+0000", 2024-08-17T17:27:15.771 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.1302999999999999e-05 2024-08-17T17:27:15.771 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.771 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.772 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.772 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.772 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.772 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:188201 7.10 7:080e429a:::benchmark_data_smithi063_196002_object76108:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:27:15.772 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:09.041737+0000", 2024-08-17T17:27:15.772 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 366.641251813, 2024-08-17T17:27:15.772 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.89234012900000004, 2024-08-17T17:27:15.772 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.772 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.772 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.772 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:15.773 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:15.773 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 188201 2024-08-17T17:27:15.773 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.773 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.773 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.773 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.773 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041737+0000", 2024-08-17T17:27:15.773 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.773 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.773 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.773 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.773 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041737+0000", 2024-08-17T17:27:15.774 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.774 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.774 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.774 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.774 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041735+0000", 2024-08-17T17:27:15.774 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:27:15.774 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.774 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.774 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.774 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041745+0000", 2024-08-17T17:27:15.774 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8300000000000008e-06 2024-08-17T17:27:15.775 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.775 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.775 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.775 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041757+0000", 2024-08-17T17:27:15.775 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1226999999999999e-05 2024-08-17T17:27:15.775 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.775 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.775 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.775 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041762+0000", 2024-08-17T17:27:15.775 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5740000000000003e-06 2024-08-17T17:27:15.775 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.775 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.776 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.776 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041821+0000", 2024-08-17T17:27:15.776 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8814999999999998e-05 2024-08-17T17:27:15.776 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.776 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.776 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.776 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041830+0000", 2024-08-17T17:27:15.776 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9789999999999999e-06 2024-08-17T17:27:15.776 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.776 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.776 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.776 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.919950+0000", 2024-08-17T17:27:15.777 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.87812030600000002 2024-08-17T17:27:15.777 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.777 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.777 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.777 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.920014+0000", 2024-08-17T17:27:15.777 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3324000000000006e-05 2024-08-17T17:27:15.777 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.777 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.777 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:27:15.777 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.920069+0000", 2024-08-17T17:27:15.777 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5655000000000002e-05 2024-08-17T17:27:15.777 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.778 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.778 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.778 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.923576+0000", 2024-08-17T17:27:15.778 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0035071730000000001 2024-08-17T17:27:15.778 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.778 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.778 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.778 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.934000+0000", 2024-08-17T17:27:15.778 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010423798999999999 2024-08-17T17:27:15.778 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.778 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.778 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.779 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.934037+0000", 2024-08-17T17:27:15.779 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6585000000000003e-05 2024-08-17T17:27:15.779 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.779 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.779 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.779 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.934059+0000", 2024-08-17T17:27:15.779 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2503000000000001e-05 2024-08-17T17:27:15.779 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.779 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.779 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.779 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.934077+0000", 2024-08-17T17:27:15.779 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.8151e-05 2024-08-17T17:27:15.780 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.780 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.780 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.780 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.780 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.780 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:188245 7.10 7:0874df2b:::benchmark_data_smithi063_196002_object76152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:27:15.780 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:09.075632+0000", 2024-08-17T17:27:15.780 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 366.60735692100002, 2024-08-17T17:27:15.780 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.86299068199999995, 2024-08-17T17:27:15.780 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.780 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.780 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.780 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:15.781 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:15.781 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 188245 2024-08-17T17:27:15.781 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.781 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.781 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.781 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.781 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075632+0000", 2024-08-17T17:27:15.781 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.781 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.781 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.781 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.781 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075632+0000", 2024-08-17T17:27:15.781 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.782 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.782 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.782 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.782 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075631+0000", 2024-08-17T17:27:15.782 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:15.782 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.782 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.782 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.782 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075638+0000", 2024-08-17T17:27:15.782 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.114e-06 2024-08-17T17:27:15.782 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.782 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.782 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.783 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075656+0000", 2024-08-17T17:27:15.783 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7855000000000001e-05 2024-08-17T17:27:15.783 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.783 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.783 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.783 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075664+0000", 2024-08-17T17:27:15.783 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7409999999999998e-06 2024-08-17T17:27:15.783 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.783 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.783 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.783 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075739+0000", 2024-08-17T17:27:15.783 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4977999999999999e-05 2024-08-17T17:27:15.783 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.784 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.784 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.784 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075756+0000", 2024-08-17T17:27:15.784 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6467999999999999e-05 2024-08-17T17:27:15.784 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.784 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.784 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.784 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.921100+0000", 2024-08-17T17:27:15.784 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.84534372899999999 2024-08-17T17:27:15.784 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.784 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.784 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.785 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.921169+0000", 2024-08-17T17:27:15.785 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9863999999999994e-05 2024-08-17T17:27:15.785 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.785 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.785 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:27:15.785 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.921236+0000", 2024-08-17T17:27:15.785 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6625000000000007e-05 2024-08-17T17:27:15.785 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.785 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.785 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.785 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.925891+0000", 2024-08-17T17:27:15.785 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0046546620000000004 2024-08-17T17:27:15.785 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.786 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.786 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.786 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.934115+0000", 2024-08-17T17:27:15.786 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0082241750000000002 2024-08-17T17:27:15.786 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.786 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.786 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.786 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.938555+0000", 2024-08-17T17:27:15.786 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0044405190000000004 2024-08-17T17:27:15.786 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.786 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.786 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.787 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.938584+0000", 2024-08-17T17:27:15.787 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8129000000000001e-05 2024-08-17T17:27:15.787 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.787 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.787 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.787 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.938623+0000", 2024-08-17T17:27:15.787 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9240999999999999e-05 2024-08-17T17:27:15.787 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.787 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.787 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.787 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.787 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.788 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:188257 7.10 7:0b1312c5:::benchmark_data_smithi063_196002_object76164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:27:15.788 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:09.085121+0000", 2024-08-17T17:27:15.788 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 366.59786772799998, 2024-08-17T17:27:15.788 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.86507842300000004, 2024-08-17T17:27:15.788 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.788 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.788 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.788 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:15.788 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:15.788 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 188257 2024-08-17T17:27:15.788 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.788 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.789 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.789 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.789 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085121+0000", 2024-08-17T17:27:15.789 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.789 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.789 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.789 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.789 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085121+0000", 2024-08-17T17:27:15.789 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.789 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.789 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.789 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.790 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085120+0000", 2024-08-17T17:27:15.790 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:15.790 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.790 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.790 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.790 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085127+0000", 2024-08-17T17:27:15.790 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8580000000000002e-06 2024-08-17T17:27:15.790 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.790 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.790 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.790 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085145+0000", 2024-08-17T17:27:15.790 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.8308000000000002e-05 2024-08-17T17:27:15.790 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.791 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.791 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.791 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085154+0000", 2024-08-17T17:27:15.791 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8240000000000006e-06 2024-08-17T17:27:15.791 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.791 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.791 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.791 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085234+0000", 2024-08-17T17:27:15.791 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9925999999999993e-05 2024-08-17T17:27:15.791 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.791 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.791 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.792 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085247+0000", 2024-08-17T17:27:15.792 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3182e-05 2024-08-17T17:27:15.792 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.792 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.792 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.792 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.922382+0000", 2024-08-17T17:27:15.792 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.83713534300000003 2024-08-17T17:27:15.792 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.792 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.792 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.792 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.922448+0000", 2024-08-17T17:27:15.792 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6243999999999998e-05 2024-08-17T17:27:15.793 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.793 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.793 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:27:15.793 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.922510+0000", 2024-08-17T17:27:15.793 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1130000000000006e-05 2024-08-17T17:27:15.793 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.793 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.793 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.793 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.928234+0000", 2024-08-17T17:27:15.793 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0057246600000000003 2024-08-17T17:27:15.793 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.793 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.793 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.794 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.938660+0000", 2024-08-17T17:27:15.794 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010425528 2024-08-17T17:27:15.794 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.794 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.794 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.794 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.950147+0000", 2024-08-17T17:27:15.794 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011487338 2024-08-17T17:27:15.794 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.794 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.794 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.794 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.950177+0000", 2024-08-17T17:27:15.794 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.0428000000000001e-05 2024-08-17T17:27:15.795 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.795 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.795 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.795 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.950200+0000", 2024-08-17T17:27:15.795 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.228e-05 2024-08-17T17:27:15.795 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.795 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.795 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.795 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.795 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.795 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:188342 7.10 7:0e94f6c4:::benchmark_data_smithi063_196002_object76249:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:27:15.795 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:09.152746+0000", 2024-08-17T17:27:15.795 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 366.53024258699998, 2024-08-17T17:27:15.796 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.79753565299999996, 2024-08-17T17:27:15.796 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.796 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.796 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.796 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:15.796 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:15.796 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 188342 2024-08-17T17:27:15.796 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.796 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.796 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.796 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.796 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152746+0000", 2024-08-17T17:27:15.796 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.797 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.797 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.797 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.797 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152746+0000", 2024-08-17T17:27:15.797 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.797 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.797 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.797 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.797 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152745+0000", 2024-08-17T17:27:15.797 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:27:15.797 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.797 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.797 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.798 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152753+0000", 2024-08-17T17:27:15.798 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0819999999999999e-06 2024-08-17T17:27:15.798 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.798 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.798 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.798 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152769+0000", 2024-08-17T17:27:15.798 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6642000000000001e-05 2024-08-17T17:27:15.798 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.798 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.798 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.798 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152778+0000", 2024-08-17T17:27:15.798 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9849999999999995e-06 2024-08-17T17:27:15.798 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.799 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.799 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.799 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152847+0000", 2024-08-17T17:27:15.799 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8341000000000002e-05 2024-08-17T17:27:15.799 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.799 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.799 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:15.799 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152867+0000", 2024-08-17T17:27:15.799 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9862000000000001e-05 2024-08-17T17:27:15.799 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.799 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.799 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.799 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.923594+0000", 2024-08-17T17:27:15.800 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.77072738900000004 2024-08-17T17:27:15.800 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.800 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.800 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.800 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.923658+0000", 2024-08-17T17:27:15.800 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3749000000000002e-05 2024-08-17T17:27:15.800 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.800 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.800 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:27:15.800 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.923717+0000", 2024-08-17T17:27:15.800 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9061999999999998e-05 2024-08-17T17:27:15.800 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.801 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.801 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.801 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.928243+0000", 2024-08-17T17:27:15.801 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0045256849999999998 2024-08-17T17:27:15.801 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.801 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.801 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.801 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.940172+0000", 2024-08-17T17:27:15.801 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011929194000000001 2024-08-17T17:27:15.801 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.801 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.801 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.801 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.950236+0000", 2024-08-17T17:27:15.802 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010064149999999999 2024-08-17T17:27:15.802 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.802 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.802 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.802 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.950261+0000", 2024-08-17T17:27:15.802 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5369000000000001e-05 2024-08-17T17:27:15.802 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.802 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.802 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.802 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.950282+0000", 2024-08-17T17:27:15.802 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0922999999999999e-05 2024-08-17T17:27:15.802 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.802 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.803 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.803 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.803 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.803 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:235996 8.3 8:c78032a6:::benchmark_data_smithi063_230025_object109506:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1241)", 2024-08-17T17:27:15.803 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:25:54.297302+0000", 2024-08-17T17:27:15.803 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 81.385687163, 2024-08-17T17:27:15.803 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33830727199999999, 2024-08-17T17:27:15.803 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.803 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.803 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.803 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:27:15.803 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:27:15.803 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 235996 2024-08-17T17:27:15.804 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.804 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.804 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.804 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.804 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297302+0000", 2024-08-17T17:27:15.804 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.804 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.804 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.804 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.804 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297302+0000", 2024-08-17T17:27:15.804 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.804 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.804 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.805 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.805 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297301+0000", 2024-08-17T17:27:15.805 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:15.805 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.805 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.805 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.805 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297307+0000", 2024-08-17T17:27:15.805 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4049999999999997e-06 2024-08-17T17:27:15.805 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.805 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.805 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.805 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297319+0000", 2024-08-17T17:27:15.806 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1793999999999999e-05 2024-08-17T17:27:15.806 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.806 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.806 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.806 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297327+0000", 2024-08-17T17:27:15.806 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.5179999999999998e-06 2024-08-17T17:27:15.806 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.806 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.806 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.806 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297372+0000", 2024-08-17T17:27:15.806 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5195999999999999e-05 2024-08-17T17:27:15.806 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.806 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.807 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.807 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297499+0000", 2024-08-17T17:27:15.807 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012701300000000001 2024-08-17T17:27:15.807 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.807 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.807 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:27:15.807 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297621+0000", 2024-08-17T17:27:15.807 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012220500000000001 2024-08-17T17:27:15.807 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.807 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.807 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.807 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.304840+0000", 2024-08-17T17:27:15.807 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0072189680000000001 2024-08-17T17:27:15.808 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.808 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.808 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.808 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.309170+0000", 2024-08-17T17:27:15.808 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0043299300000000001 2024-08-17T17:27:15.808 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.808 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.808 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.808 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.634866+0000", 2024-08-17T17:27:15.808 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.32569575299999998 2024-08-17T17:27:15.808 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.808 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.808 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.809 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.634913+0000", 2024-08-17T17:27:15.809 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7450000000000001e-05 2024-08-17T17:27:15.809 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.809 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.809 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.809 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.635609+0000", 2024-08-17T17:27:15.809 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00069607700000000002 2024-08-17T17:27:15.809 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.809 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.809 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.809 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.809 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.809 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:235998 8.4 8:2ca03464:::benchmark_data_smithi063_230025_object109508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1241)", 2024-08-17T17:27:15.810 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:25:54.297556+0000", 2024-08-17T17:27:15.810 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 81.385432795, 2024-08-17T17:27:15.810 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.34206344, 2024-08-17T17:27:15.810 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.810 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.810 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.810 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:27:15.810 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:27:15.810 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 235998 2024-08-17T17:27:15.810 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.810 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.810 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.810 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.811 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297556+0000", 2024-08-17T17:27:15.811 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.811 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.811 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.811 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.811 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297556+0000", 2024-08-17T17:27:15.811 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.811 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.811 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.811 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.811 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297555+0000", 2024-08-17T17:27:15.811 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:15.812 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.812 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.812 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.812 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297560+0000", 2024-08-17T17:27:15.812 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5199999999999999e-06 2024-08-17T17:27:15.812 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.812 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.812 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.812 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297568+0000", 2024-08-17T17:27:15.812 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3010000000000007e-06 2024-08-17T17:27:15.812 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.812 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.812 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.812 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297575+0000", 2024-08-17T17:27:15.813 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5300000000000002e-06 2024-08-17T17:27:15.813 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.813 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.813 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.813 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.301305+0000", 2024-08-17T17:27:15.813 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0037297519999999998 2024-08-17T17:27:15.813 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.813 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.813 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.813 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.301423+0000", 2024-08-17T17:27:15.813 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011865400000000001 2024-08-17T17:27:15.813 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.813 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.814 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:27:15.814 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.301540+0000", 2024-08-17T17:27:15.814 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:15 smithi162 ceph-mon[104668]: pgmap v1712: 119 pgs: 119 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2024-08-17T17:27:15.814 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:15 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4066364698' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:27:15.814 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:15 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:27:15.815 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011626299999999999 2024-08-17T17:27:15.815 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.815 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.815 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.815 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305339+0000", 2024-08-17T17:27:15.815 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003799331 2024-08-17T17:27:15.815 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.816 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.816 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.816 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.309072+0000", 2024-08-17T17:27:15.816 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0037334859999999998 2024-08-17T17:27:15.816 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.816 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.816 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.816 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.639385+0000", 2024-08-17T17:27:15.816 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33031314000000001 2024-08-17T17:27:15.816 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.816 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.816 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.816 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.639486+0000", 2024-08-17T17:27:15.817 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010067499999999999 2024-08-17T17:27:15.817 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.817 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.817 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.817 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.639620+0000", 2024-08-17T17:27:15.817 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013358200000000001 2024-08-17T17:27:15.817 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.817 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.817 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.817 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.817 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.817 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:235999 8.d 8:b3cee540:::benchmark_data_smithi063_230025_object109509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1241)", 2024-08-17T17:27:15.817 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:25:54.299954+0000", 2024-08-17T17:27:15.818 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 81.383035488999994, 2024-08-17T17:27:15.818 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.34301663399999999, 2024-08-17T17:27:15.818 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.818 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.818 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.818 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:27:15.818 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:27:15.818 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 235999 2024-08-17T17:27:15.818 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.818 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.818 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.818 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.818 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.299954+0000", 2024-08-17T17:27:15.819 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.819 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.819 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.819 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.819 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.299954+0000", 2024-08-17T17:27:15.819 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.819 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.819 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.819 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.819 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.299952+0000", 2024-08-17T17:27:15.819 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:15.819 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.819 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.820 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.820 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.299959+0000", 2024-08-17T17:27:15.820 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4509999999999997e-06 2024-08-17T17:27:15.820 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.820 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.820 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.820 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.299971+0000", 2024-08-17T17:27:15.820 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2335e-05 2024-08-17T17:27:15.820 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.820 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.820 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.820 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.299978+0000", 2024-08-17T17:27:15.820 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3860000000000001e-06 2024-08-17T17:27:15.821 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.821 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.821 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.821 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.301412+0000", 2024-08-17T17:27:15.821 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0014331979999999999 2024-08-17T17:27:15.821 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.821 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.821 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.821 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.301514+0000", 2024-08-17T17:27:15.821 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010192300000000001 2024-08-17T17:27:15.821 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.821 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.821 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:27:15.822 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.301621+0000", 2024-08-17T17:27:15.822 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010751600000000001 2024-08-17T17:27:15.822 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.822 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.822 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.822 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305430+0000", 2024-08-17T17:27:15.822 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0038086729999999998 2024-08-17T17:27:15.822 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.822 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.822 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.822 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.642833+0000", 2024-08-17T17:27:15.822 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.337403067 2024-08-17T17:27:15.823 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.823 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.823 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.823 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.642883+0000", 2024-08-17T17:27:15.823 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0062999999999998e-05 2024-08-17T17:27:15.823 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.823 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.823 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.823 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.642931+0000", 2024-08-17T17:27:15.823 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7683999999999999e-05 2024-08-17T17:27:15.823 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.823 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.823 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.824 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.642970+0000", 2024-08-17T17:27:15.824 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9653999999999997e-05 2024-08-17T17:27:15.824 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.824 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.824 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.824 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.824 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.824 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:236007 8.d 8:b706bdcc:::benchmark_data_smithi063_230025_object109517:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1241)", 2024-08-17T17:27:15.824 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:25:54.305395+0000", 2024-08-17T17:27:15.824 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 81.377594373999997, 2024-08-17T17:27:15.824 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33775034999999998, 2024-08-17T17:27:15.824 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.824 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.825 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.825 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:27:15.825 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:27:15.825 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 236007 2024-08-17T17:27:15.825 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.825 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.825 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.825 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.825 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305395+0000", 2024-08-17T17:27:15.825 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.825 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.825 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.825 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.826 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305395+0000", 2024-08-17T17:27:15.826 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.826 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.826 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.826 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.826 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305393+0000", 2024-08-17T17:27:15.826 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:15.826 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.826 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.826 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.826 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305399+0000", 2024-08-17T17:27:15.826 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8699999999999997e-06 2024-08-17T17:27:15.826 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.827 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.827 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.827 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305410+0000", 2024-08-17T17:27:15.827 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0278999999999999e-05 2024-08-17T17:27:15.827 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.827 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.827 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.827 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305417+0000", 2024-08-17T17:27:15.827 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0809999999999996e-06 2024-08-17T17:27:15.827 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.827 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.827 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.828 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305455+0000", 2024-08-17T17:27:15.828 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8733999999999999e-05 2024-08-17T17:27:15.828 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.828 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.828 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.828 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305560+0000", 2024-08-17T17:27:15.828 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010471699999999999 2024-08-17T17:27:15.828 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.828 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.828 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:27:15.828 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305698+0000", 2024-08-17T17:27:15.828 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013761099999999999 2024-08-17T17:27:15.828 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.829 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.829 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.829 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.642983+0000", 2024-08-17T17:27:15.829 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.337285216 2024-08-17T17:27:15.829 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.829 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.829 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.829 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643027+0000", 2024-08-17T17:27:15.829 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4542000000000001e-05 2024-08-17T17:27:15.829 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.829 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.829 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.829 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643073+0000", 2024-08-17T17:27:15.830 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5969e-05 2024-08-17T17:27:15.830 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.830 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.830 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.830 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643112+0000", 2024-08-17T17:27:15.830 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8430999999999998e-05 2024-08-17T17:27:15.830 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.830 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.830 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.830 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643145+0000", 2024-08-17T17:27:15.830 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3185000000000002e-05 2024-08-17T17:27:15.830 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.831 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.831 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.831 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.831 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.831 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:236009 8.d 8:b1d34fc9:::benchmark_data_smithi063_230025_object109519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1241)", 2024-08-17T17:27:15.831 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:25:54.305466+0000", 2024-08-17T17:27:15.831 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 81.377523474, 2024-08-17T17:27:15.831 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33783398199999998, 2024-08-17T17:27:15.831 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.831 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:15.831 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:15.831 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:27:15.832 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:27:15.832 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 236009 2024-08-17T17:27:15.832 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.832 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.832 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.832 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.832 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305466+0000", 2024-08-17T17:27:15.832 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.832 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.832 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.832 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.832 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305466+0000", 2024-08-17T17:27:15.833 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.833 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.833 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.833 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.833 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305465+0000", 2024-08-17T17:27:15.833 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:15.833 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.833 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.833 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.833 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305468+0000", 2024-08-17T17:27:15.833 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2090000000000001e-06 2024-08-17T17:27:15.833 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.834 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.834 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.834 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305475+0000", 2024-08-17T17:27:15.834 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6409999999999996e-06 2024-08-17T17:27:15.834 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.834 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.834 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.834 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305480+0000", 2024-08-17T17:27:15.834 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1760000000000001e-06 2024-08-17T17:27:15.834 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.834 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.834 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.835 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.309513+0000", 2024-08-17T17:27:15.835 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0040332379999999998 2024-08-17T17:27:15.835 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.835 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.835 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.835 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.309627+0000", 2024-08-17T17:27:15.835 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000113669 2024-08-17T17:27:15.835 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.835 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.835 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:27:15.835 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.634933+0000", 2024-08-17T17:27:15.835 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.32530621300000001 2024-08-17T17:27:15.835 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.836 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.836 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:15.836 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643155+0000", 2024-08-17T17:27:15.836 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0082218229999999996 2024-08-17T17:27:15.836 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.836 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.836 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.836 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643190+0000", 2024-08-17T17:27:15.836 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5298e-05 2024-08-17T17:27:15.836 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.836 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.836 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:15.836 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643234+0000", 2024-08-17T17:27:15.837 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4044e-05 2024-08-17T17:27:15.837 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.837 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.837 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:15.837 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643270+0000", 2024-08-17T17:27:15.837 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6094999999999999e-05 2024-08-17T17:27:15.837 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.837 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.837 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.837 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643300+0000", 2024-08-17T17:27:15.837 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9274000000000001e-05 2024-08-17T17:27:15.837 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.837 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.838 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.838 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.838 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.838 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop_reply(client.45218.0:235999 8.d e1241/1195 ondisk, result = 0)", 2024-08-17T17:27:15.838 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:25:54.305839+0000", 2024-08-17T17:27:15.838 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 81.377149947999996, 2024-08-17T17:27:15.838 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.336997665, 2024-08-17T17:27:15.838 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:15.838 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:27:15.838 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:15.838 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.838 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:15.839 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305839+0000", 2024-08-17T17:27:15.839 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.839 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.839 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.839 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:15.839 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305839+0000", 2024-08-17T17:27:15.839 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:15.839 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.839 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.839 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:15.839 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305838+0000", 2024-08-17T17:27:15.839 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:15.839 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.840 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.840 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:15.840 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305843+0000", 2024-08-17T17:27:15.840 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8250000000000004e-06 2024-08-17T17:27:15.840 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.840 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.840 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:15.840 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305852+0000", 2024-08-17T17:27:15.840 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8890000000000008e-06 2024-08-17T17:27:15.840 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.840 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.840 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:15.841 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305866+0000", 2024-08-17T17:27:15.841 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3957e-05 2024-08-17T17:27:15.841 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.841 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.841 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:15.841 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.642816+0000", 2024-08-17T17:27:15.841 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33695024499999998 2024-08-17T17:27:15.841 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.841 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.841 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:15.841 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.642825+0000", 2024-08-17T17:27:15.841 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.0750000000000004e-06 2024-08-17T17:27:15.842 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:15.842 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:15.842 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:15.842 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.642837+0000", 2024-08-17T17:27:15.842 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1569000000000001e-05 2024-08-17T17:27:15.842 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.842 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.842 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.842 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:15.842 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:15.842 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:27:15.849 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:15.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:15.951 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:15.991 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:15.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:15.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:16.093 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:16.136 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:16.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:16.238 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:16.275 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:16.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:16.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:16.399 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:16.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:16.459 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 30 16 57724 57708 120.209 88.5 0.0104193 0.00829052 2024-08-17T17:27:16.501 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:16.529 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:16.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:16.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:16 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:16.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:16 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:27:16.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:16 smithi162 ceph-mon[104668]: osdmap e1253: 8 total, 8 up, 8 in 2024-08-17T17:27:16.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:16.631 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:16.647 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:16 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:16.647 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:16 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:27:16.648 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:16 smithi063 ceph-mon[114756]: osdmap e1253: 8 total, 8 up, 8 in 2024-08-17T17:27:16.648 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:16.648 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:16 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:16.648 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:16 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:27:16.648 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:16 smithi063 ceph-mon[109843]: osdmap e1253: 8 total, 8 up, 8 in 2024-08-17T17:27:16.648 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:16.667 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:16.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:16.769 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:16.811 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:16.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:16.913 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:16.946 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:16.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:16.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:17.037 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:27:17.037 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:27:17.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:27:17.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:27:17.038 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:27:17.038 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:27:17.038 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:27:17.038 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:27:17.048 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:17.052 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:27:17.052 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:27:17.085 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:27:17.085 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:27:17.088 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:17.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:17.104 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:27:17.104 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:27:17.135 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:27:17.135 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:27:17.190 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:17.235 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:17.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:17.336 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:17.373 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:17.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:17.459 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 31 16 59020 59004 118.944 81 0.0127969 0.00840199 2024-08-17T17:27:17.474 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:17.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.509 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:17.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:17.567 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:27:17.610 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:17.648 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:17.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:17.749 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:17.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:17 smithi063 ceph-mon[114756]: pgmap v1714: 119 pgs: 119 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-17T17:27:17.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:17 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3131581636' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:17.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:17 smithi063 ceph-mon[109843]: pgmap v1714: 119 pgs: 119 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-17T17:27:17.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:17 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3131581636' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:17.789 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:17.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:17.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:17 smithi162 ceph-mon[104668]: pgmap v1714: 119 pgs: 119 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-17T17:27:17.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:17 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3131581636' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:17.890 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:17.926 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:17.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:17.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:17.933 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:18.027 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:18.068 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:18.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:18.169 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:18.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.209 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:18.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:18.310 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:18.345 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:18.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:18.360 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:27:18.446 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:18.461 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 32 16 60394 60378 117.91 85.875 0.0177831 0.00847588 2024-08-17T17:27:18.482 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:18.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:18.585 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:18.619 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:18.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:18.721 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:18.751 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:18.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:18.853 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:18.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:18.889 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:18.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:18.990 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:19.025 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:19.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:19.127 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:19.164 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:19.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:19.266 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:19.300 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:19.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:19.402 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:19.441 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:19.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:19.459 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 33 16 61758 61742 116.92 85.25 0.00768864 0.00854812 2024-08-17T17:27:19.543 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:19.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.591 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:19.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:19.693 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:19.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.731 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:19.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:19.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:19 smithi063 ceph-mon[114756]: pgmap v1715: 119 pgs: 119 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:27:19.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:19 smithi063 ceph-mon[109843]: pgmap v1715: 119 pgs: 119 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:27:19.832 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:19.834 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:19 smithi162 ceph-mon[104668]: pgmap v1715: 119 pgs: 119 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:27:19.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:19.868 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:19.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:19.906 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:27:19.906 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:27:19.970 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:20.012 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:20.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:20.114 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:20.149 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:20.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:20.251 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:20.284 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:20.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:20.386 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:20.428 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:20.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:20.459 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 34 16 63220 63204 116.169 91.375 0.00903963 0.00860319 2024-08-17T17:27:20.530 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:20.570 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:20.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:20.621 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:27:20.671 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:20.701 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:20.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:20.803 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:20.837 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:20.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:20.939 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:20.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:20.979 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:20.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:21.001 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:21.080 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:21.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.117 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:21.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:21.218 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:21.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.250 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:21.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:21.351 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:21.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.391 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:21.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:21.460 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 35 16 64652 64636 115.406 89.5 0.00506289 0.00865932 2024-08-17T17:27:21.491 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:21.495 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:21.522 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:21.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:21.623 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:21.669 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:21.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:21.772 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:21.772 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:21 smithi063 ceph-mon[109843]: pgmap v1716: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:27:21.773 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:21 smithi063 ceph-mon[114756]: pgmap v1716: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:27:21.803 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:21 smithi162 ceph-mon[104668]: pgmap v1716: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:27:21.806 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:21.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:21.901 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:27:21.903 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:27:21.908 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:21.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:21.946 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:21.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:22.049 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:22.082 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:22.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:22.184 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:22.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.219 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:22.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:22.321 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:22.343 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:22.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:22.445 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:22.461 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 36 16 65942 65926 114.44 80.625 0.00522174 0.00873348 2024-08-17T17:27:22.485 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:22.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:22.587 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:22.623 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:22.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:22.724 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:22.760 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:22.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:22.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:22 smithi162 ceph-mon[104668]: from='client.55240 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:27:22.862 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:22.889 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:22.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:22.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:22.951 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:27:22.991 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:22.992 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:22 smithi063 ceph-mon[114756]: from='client.55240 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:27:22.992 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:22 smithi063 ceph-mon[109843]: from='client.55240 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:27:23.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.026 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:23.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:23.127 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:23.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.174 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:23.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:23.275 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:23.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.313 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:23.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:23.414 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:23.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.448 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:23.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:23.459 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 37 16 67165 67149 113.413 76.4375 0.0107025 0.00881174 2024-08-17T17:27:23.546 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:27:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:27:23] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.51.0" 2024-08-17T17:27:23.548 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:23.576 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:23.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:23.678 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:23.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.707 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:23.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:23.808 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:23.844 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:23.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:23 smithi162 ceph-mon[104668]: pgmap v1717: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 93 MiB/s wr, 1.50k op/s 2024-08-17T17:27:23.946 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:23.950 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:23 smithi063 ceph-mon[114756]: pgmap v1717: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 93 MiB/s wr, 1.50k op/s 2024-08-17T17:27:23.950 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:23 smithi063 ceph-mon[109843]: pgmap v1717: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 93 MiB/s wr, 1.50k op/s 2024-08-17T17:27:23.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:23.986 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:23.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:24.086 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:24.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.126 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:24.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:24.227 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:24.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.268 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:24.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:24.346 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1716,"stamp":"2024-08-17T17:27:20.377697+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3839825893,"num_objects":58608,"num_object_clones":0,"num_object_copies":175824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":58646,"num_write_kb":3749281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1246,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":63260,"ondisk_log_size":63260,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15342588,"kb_used_data":11250988,"kb_used_omap":199,"kb_used_meta":4091320,"kb_avail":734618628,"statfs":{"total":767960285184,"available":752249475072,"internally_reserved":0,"allocated":11521011712,"data_stored":11500364051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":4189512444},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,6,8,5,5,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":979435520,"num_objects":14945,"num_object_clones":0,"num_object_copies":44835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14945,"num_write_kb":956480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005421"},"pg_stats":[{"pgid":"1.11","version":"369'149","reported_seq":1576,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.465370+0000","last_change":"2024-08-17T17:24:49.721626+0000","last_active":"2024-08-17T17:27:05.465370+0000","last_peered":"2024-08-17T17:27:05.465370+0000","last_clean":"2024-08-17T17:27:05.465370+0000","last_became_active":"2024-08-17T17:24:20.649128+0000","last_became_peered":"2024-08-17T17:24:20.649128+0000","last_unstale":"2024-08-17T17:27:05.465370+0000","last_undegraded":"2024-08-17T17:27:05.465370+0000","last_fullsized":"2024-08-17T17:27:05.465370+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:03:15.343059+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1603,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.513982+0000","last_change":"2024-08-17T17:21:06.463828+0000","last_active":"2024-08-17T17:27:15.513982+0000","last_peered":"2024-08-17T17:27:15.513982+0000","last_clean":"2024-08-17T17:27:15.513982+0000","last_became_active":"2024-08-17T17:21:06.463464+0000","last_became_peered":"2024-08-17T17:21:06.463464+0000","last_unstale":"2024-08-17T17:27:15.513982+0000","last_undegraded":"2024-08-17T17:27:15.513982+0000","last_fullsized":"2024-08-17T17:27:15.513982+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:46.465006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1183,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.463370+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:27:05.463370+0000","last_peered":"2024-08-17T17:27:05.463370+0000","last_clean":"2024-08-17T17:27:05.463370+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:27:05.463370+0000","last_undegraded":"2024-08-17T17:27:05.463370+0000","last_fullsized":"2024-08-17T17:27:05.463370+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1507,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.465156+0000","last_change":"2024-08-17T17:24:48.765312+0000","last_active":"2024-08-17T17:27:05.465156+0000","last_peered":"2024-08-17T17:27:05.465156+0000","last_clean":"2024-08-17T17:27:05.465156+0000","last_became_active":"2024-08-17T17:24:20.642731+0000","last_became_peered":"2024-08-17T17:24:20.642731+0000","last_unstale":"2024-08-17T17:27:05.465156+0000","last_undegraded":"2024-08-17T17:27:05.465156+0000","last_fullsized":"2024-08-17T17:27:05.465156+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:08:36.138960+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1194,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.466332+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:27:05.466332+0000","last_peered":"2024-08-17T17:27:05.466332+0000","last_clean":"2024-08-17T17:27:05.466332+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:27:05.466332+0000","last_undegraded":"2024-08-17T17:27:05.466332+0000","last_fullsized":"2024-08-17T17:27:05.466332+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1050,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.463311+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:27:05.463311+0000","last_peered":"2024-08-17T17:27:05.463311+0000","last_clean":"2024-08-17T17:27:05.463311+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:27:05.463311+0000","last_undegraded":"2024-08-17T17:27:05.463311+0000","last_fullsized":"2024-08-17T17:27:05.463311+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":817,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.513038+0000","last_change":"2024-08-17T17:24:50.880046+0000","last_active":"2024-08-17T17:27:15.513038+0000","last_peered":"2024-08-17T17:27:15.513038+0000","last_clean":"2024-08-17T17:27:15.513038+0000","last_became_active":"2024-08-17T17:24:20.652165+0000","last_became_peered":"2024-08-17T17:24:20.652165+0000","last_unstale":"2024-08-17T17:27:15.513038+0000","last_undegraded":"2024-08-17T17:27:15.513038+0000","last_fullsized":"2024-08-17T17:27:15.513038+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:00.416810+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1155,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.514077+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:27:15.514077+0000","last_peered":"2024-08-17T17:27:15.514077+0000","last_clean":"2024-08-17T17:27:15.514077+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:27:15.514077+0000","last_undegraded":"2024-08-17T17:27:15.514077+0000","last_fullsized":"2024-08-17T17:27:15.514077+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1166,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.465730+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:27:05.465730+0000","last_peered":"2024-08-17T17:27:05.465730+0000","last_clean":"2024-08-17T17:27:05.465730+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:27:05.465730+0000","last_undegraded":"2024-08-17T17:27:05.465730+0000","last_fullsized":"2024-08-17T17:27:05.465730+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1512,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.466005+0000","last_change":"2024-08-17T17:24:50.727183+0000","last_active":"2024-08-17T17:27:05.466005+0000","last_peered":"2024-08-17T17:27:05.466005+0000","last_clean":"2024-08-17T17:27:05.466005+0000","last_became_active":"2024-08-17T17:24:20.646409+0000","last_became_peered":"2024-08-17T17:24:20.646409+0000","last_unstale":"2024-08-17T17:27:05.466005+0000","last_undegraded":"2024-08-17T17:27:05.466005+0000","last_fullsized":"2024-08-17T17:27:05.466005+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.086193+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1603,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.514502+0000","last_change":"2024-08-17T17:21:06.442598+0000","last_active":"2024-08-17T17:27:15.514502+0000","last_peered":"2024-08-17T17:27:15.514502+0000","last_clean":"2024-08-17T17:27:15.514502+0000","last_became_active":"2024-08-17T17:21:06.442264+0000","last_became_peered":"2024-08-17T17:21:06.442264+0000","last_unstale":"2024-08-17T17:27:15.514502+0000","last_undegraded":"2024-08-17T17:27:15.514502+0000","last_fullsized":"2024-08-17T17:27:15.514502+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:35.003620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1243,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.460876+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:27:05.460876+0000","last_peered":"2024-08-17T17:27:05.460876+0000","last_clean":"2024-08-17T17:27:05.460876+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:27:05.460876+0000","last_undegraded":"2024-08-17T17:27:05.460876+0000","last_fullsized":"2024-08-17T17:27:05.460876+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"1253'3739","reported_seq":3754,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:17.805835+0000","last_change":"2024-08-17T17:26:34.519835+0000","last_active":"2024-08-17T17:27:17.805835+0000","last_peered":"2024-08-17T17:27:17.805835+0000","last_clean":"2024-08-17T17:27:17.805835+0000","last_became_active":"2024-08-17T17:26:34.519427+0000","last_became_peered":"2024-08-17T17:26:34.519427+0000","last_unstale":"2024-08-17T17:27:17.805835+0000","last_undegraded":"2024-08-17T17:27:17.805835+0000","last_fullsized":"2024-08-17T17:27:17.805835+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3739,"log_dups_size":0,"ondisk_log_size":3739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:25.352736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245039104,"num_objects":3739,"num_object_clones":0,"num_object_copies":11217,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3739,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3739,"num_write_kb":239296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1450,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.463796+0000","last_change":"2024-08-17T17:24:50.410621+0000","last_active":"2024-08-17T17:27:05.463796+0000","last_peered":"2024-08-17T17:27:05.463796+0000","last_clean":"2024-08-17T17:27:05.463796+0000","last_became_active":"2024-08-17T17:24:31.863062+0000","last_became_peered":"2024-08-17T17:24:31.863062+0000","last_unstale":"2024-08-17T17:27:05.463796+0000","last_undegraded":"2024-08-17T17:27:05.463796+0000","last_fullsized":"2024-08-17T17:27:05.463796+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.410491+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.410491+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:11.717684+0000","scrub_duration":0.0066315480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1208,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.464243+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:27:05.464243+0000","last_peered":"2024-08-17T17:27:05.464243+0000","last_clean":"2024-08-17T17:27:05.464243+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:27:05.464243+0000","last_undegraded":"2024-08-17T17:27:05.464243+0000","last_fullsized":"2024-08-17T17:27:05.464243+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1181,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.466100+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:27:05.466100+0000","last_peered":"2024-08-17T17:27:05.466100+0000","last_clean":"2024-08-17T17:27:05.466100+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:27:05.466100+0000","last_undegraded":"2024-08-17T17:27:05.466100+0000","last_fullsized":"2024-08-17T17:27:05.466100+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"1253'3669","reported_seq":3684,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:17.623810+0000","last_change":"2024-08-17T17:26:34.715261+0000","last_active":"2024-08-17T17:27:17.623810+0000","last_peered":"2024-08-17T17:27:17.623810+0000","last_clean":"2024-08-17T17:27:17.623810+0000","last_became_active":"2024-08-17T17:26:34.714853+0000","last_became_peered":"2024-08-17T17:26:34.714853+0000","last_unstale":"2024-08-17T17:27:17.623810+0000","last_undegraded":"2024-08-17T17:27:17.623810+0000","last_fullsized":"2024-08-17T17:27:17.623810+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3669,"log_dups_size":0,"ondisk_log_size":3669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:18.673096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240451584,"num_objects":3669,"num_object_clones":0,"num_object_copies":11007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3669,"num_write_kb":234816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1685,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.513804+0000","last_change":"2024-08-17T17:24:45.985725+0000","last_active":"2024-08-17T17:27:15.513804+0000","last_peered":"2024-08-17T17:27:15.513804+0000","last_clean":"2024-08-17T17:27:15.513804+0000","last_became_active":"2024-08-17T17:24:31.795836+0000","last_became_peered":"2024-08-17T17:24:31.795836+0000","last_unstale":"2024-08-17T17:27:15.513804+0000","last_undegraded":"2024-08-17T17:27:15.513804+0000","last_fullsized":"2024-08-17T17:27:15.513804+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:45.985596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:45.985596+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:17.897808+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1209,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.464274+0000","last_change":"2024-08-17T17:21:06.540689+0000","last_active":"2024-08-17T17:27:05.464274+0000","last_peered":"2024-08-17T17:27:05.464274+0000","last_clean":"2024-08-17T17:27:05.464274+0000","last_became_active":"2024-08-17T17:21:06.461614+0000","last_became_peered":"2024-08-17T17:21:06.461614+0000","last_unstale":"2024-08-17T17:27:05.464274+0000","last_undegraded":"2024-08-17T17:27:05.464274+0000","last_fullsized":"2024-08-17T17:27:05.464274+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:11.696898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1233,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.469089+0000","last_change":"2024-08-17T17:21:06.470389+0000","last_active":"2024-08-17T17:27:05.469089+0000","last_peered":"2024-08-17T17:27:05.469089+0000","last_clean":"2024-08-17T17:27:05.469089+0000","last_became_active":"2024-08-17T17:21:06.469966+0000","last_became_peered":"2024-08-17T17:21:06.469966+0000","last_unstale":"2024-08-17T17:27:05.469089+0000","last_undegraded":"2024-08-17T17:27:05.469089+0000","last_fullsized":"2024-08-17T17:27:05.469089+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:56.140616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"1253'3794","reported_seq":3808,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:17.818631+0000","last_change":"2024-08-17T17:26:34.412902+0000","last_active":"2024-08-17T17:27:17.818631+0000","last_peered":"2024-08-17T17:27:17.818631+0000","last_clean":"2024-08-17T17:27:17.818631+0000","last_became_active":"2024-08-17T17:26:34.412543+0000","last_became_peered":"2024-08-17T17:26:34.412543+0000","last_unstale":"2024-08-17T17:27:17.818631+0000","last_undegraded":"2024-08-17T17:27:17.818631+0000","last_fullsized":"2024-08-17T17:27:17.818631+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3794,"log_dups_size":0,"ondisk_log_size":3794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:20.767004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248643584,"num_objects":3794,"num_object_clones":0,"num_object_copies":11382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3794,"num_write_kb":242816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":818,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.460479+0000","last_change":"2024-08-17T17:24:48.192447+0000","last_active":"2024-08-17T17:27:05.460479+0000","last_peered":"2024-08-17T17:27:05.460479+0000","last_clean":"2024-08-17T17:27:05.460479+0000","last_became_active":"2024-08-17T17:24:20.644240+0000","last_became_peered":"2024-08-17T17:24:20.644240+0000","last_unstale":"2024-08-17T17:27:05.460479+0000","last_undegraded":"2024-08-17T17:27:05.460479+0000","last_fullsized":"2024-08-17T17:27:05.460479+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:56.795337+0000","scrub_duration":0.010920255,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1167,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.464135+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:27:05.464135+0000","last_peered":"2024-08-17T17:27:05.464135+0000","last_clean":"2024-08-17T17:27:05.464135+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:27:05.464135+0000","last_undegraded":"2024-08-17T17:27:05.464135+0000","last_fullsized":"2024-08-17T17:27:05.464135+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1215,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.460900+0000","last_change":"2024-08-17T17:21:06.446798+0000","last_active":"2024-08-17T17:27:05.460900+0000","last_peered":"2024-08-17T17:27:05.460900+0000","last_clean":"2024-08-17T17:27:05.460900+0000","last_became_active":"2024-08-17T17:21:06.446205+0000","last_became_peered":"2024-08-17T17:21:06.446205+0000","last_unstale":"2024-08-17T17:27:05.460900+0000","last_undegraded":"2024-08-17T17:27:05.460900+0000","last_fullsized":"2024-08-17T17:27:05.460900+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:28.584048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1253'3735","reported_seq":3750,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:17.766829+0000","last_change":"2024-08-17T17:26:35.023050+0000","last_active":"2024-08-17T17:27:17.766829+0000","last_peered":"2024-08-17T17:27:17.766829+0000","last_clean":"2024-08-17T17:27:17.766829+0000","last_became_active":"2024-08-17T17:26:35.022784+0000","last_became_peered":"2024-08-17T17:26:35.022784+0000","last_unstale":"2024-08-17T17:27:17.766829+0000","last_undegraded":"2024-08-17T17:27:17.766829+0000","last_fullsized":"2024-08-17T17:27:17.766829+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3735,"log_dups_size":0,"ondisk_log_size":3735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:04.319458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244776960,"num_objects":3735,"num_object_clones":0,"num_object_copies":11205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3735,"num_write_kb":239040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":726,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.470536+0000","last_change":"2024-08-17T17:24:52.102415+0000","last_active":"2024-08-17T17:27:05.470536+0000","last_peered":"2024-08-17T17:27:05.470536+0000","last_clean":"2024-08-17T17:27:05.470536+0000","last_became_active":"2024-08-17T17:24:31.795843+0000","last_became_peered":"2024-08-17T17:24:31.795843+0000","last_unstale":"2024-08-17T17:27:05.470536+0000","last_undegraded":"2024-08-17T17:27:05.470536+0000","last_fullsized":"2024-08-17T17:27:05.470536+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:32:08.434197+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1196,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.465606+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:27:05.465606+0000","last_peered":"2024-08-17T17:27:05.465606+0000","last_clean":"2024-08-17T17:27:05.465606+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:27:05.465606+0000","last_undegraded":"2024-08-17T17:27:05.465606+0000","last_fullsized":"2024-08-17T17:27:05.465606+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":978,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.469749+0000","last_change":"2024-08-17T17:21:06.460905+0000","last_active":"2024-08-17T17:27:05.469749+0000","last_peered":"2024-08-17T17:27:05.469749+0000","last_clean":"2024-08-17T17:27:05.469749+0000","last_became_active":"2024-08-17T17:21:06.460592+0000","last_became_peered":"2024-08-17T17:21:06.460592+0000","last_unstale":"2024-08-17T17:27:05.469749+0000","last_undegraded":"2024-08-17T17:27:05.469749+0000","last_fullsized":"2024-08-17T17:27:05.469749+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:10.146916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.1","version":"1253'3457","reported_seq":3480,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.659417+0000","last_change":"2024-08-17T17:26:34.410513+0000","last_active":"2024-08-17T17:27:15.659417+0000","last_peered":"2024-08-17T17:27:15.659417+0000","last_clean":"2024-08-17T17:27:15.659417+0000","last_became_active":"2024-08-17T17:26:34.410199+0000","last_became_peered":"2024-08-17T17:26:34.410199+0000","last_unstale":"2024-08-17T17:27:15.659417+0000","last_undegraded":"2024-08-17T17:27:15.659417+0000","last_fullsized":"2024-08-17T17:27:15.659417+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3457,"log_dups_size":0,"ondisk_log_size":3457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:35.837825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226557952,"num_objects":3457,"num_object_clones":0,"num_object_copies":10371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3457,"num_write_kb":221248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1597,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.464990+0000","last_change":"2024-08-17T17:24:54.418765+0000","last_active":"2024-08-17T17:27:05.464990+0000","last_peered":"2024-08-17T17:27:05.464990+0000","last_clean":"2024-08-17T17:27:05.464990+0000","last_became_active":"2024-08-17T17:24:31.800263+0000","last_became_peered":"2024-08-17T17:24:31.800263+0000","last_unstale":"2024-08-17T17:27:05.464990+0000","last_undegraded":"2024-08-17T17:27:05.464990+0000","last_fullsized":"2024-08-17T17:27:05.464990+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:54.418644+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:54.418644+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:11.024294+0000","scrub_duration":0.013346135,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1624,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.513131+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:27:15.513131+0000","last_peered":"2024-08-17T17:27:15.513131+0000","last_clean":"2024-08-17T17:27:15.513131+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:27:15.513131+0000","last_undegraded":"2024-08-17T17:27:15.513131+0000","last_fullsized":"2024-08-17T17:27:15.513131+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1199,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.470713+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:27:05.470713+0000","last_peered":"2024-08-17T17:27:05.470713+0000","last_clean":"2024-08-17T17:27:05.470713+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:27:05.470713+0000","last_undegraded":"2024-08-17T17:27:05.470713+0000","last_fullsized":"2024-08-17T17:27:05.470713+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"1253'3636","reported_seq":3651,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:17.427142+0000","last_change":"2024-08-17T17:26:35.023056+0000","last_active":"2024-08-17T17:27:17.427142+0000","last_peered":"2024-08-17T17:27:17.427142+0000","last_clean":"2024-08-17T17:27:17.427142+0000","last_became_active":"2024-08-17T17:26:35.022671+0000","last_became_peered":"2024-08-17T17:26:35.022671+0000","last_unstale":"2024-08-17T17:27:17.427142+0000","last_undegraded":"2024-08-17T17:27:17.427142+0000","last_fullsized":"2024-08-17T17:27:17.427142+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3636,"log_dups_size":0,"ondisk_log_size":3636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:43.531150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238288896,"num_objects":3636,"num_object_clones":0,"num_object_copies":10908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3636,"num_write_kb":232704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1543,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:12.817410+0000","last_change":"2024-08-17T17:24:58.712331+0000","last_active":"2024-08-17T17:27:12.817410+0000","last_peered":"2024-08-17T17:27:12.817410+0000","last_clean":"2024-08-17T17:27:12.817410+0000","last_became_active":"2024-08-17T17:24:31.802584+0000","last_became_peered":"2024-08-17T17:24:31.802584+0000","last_unstale":"2024-08-17T17:27:12.817410+0000","last_undegraded":"2024-08-17T17:27:12.817410+0000","last_fullsized":"2024-08-17T17:27:12.817410+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:59.616217+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1216,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.461047+0000","last_change":"2024-08-17T17:21:06.446841+0000","last_active":"2024-08-17T17:27:05.461047+0000","last_peered":"2024-08-17T17:27:05.461047+0000","last_clean":"2024-08-17T17:27:05.461047+0000","last_became_active":"2024-08-17T17:21:06.446271+0000","last_became_peered":"2024-08-17T17:21:06.446271+0000","last_unstale":"2024-08-17T17:27:05.461047+0000","last_undegraded":"2024-08-17T17:27:05.461047+0000","last_fullsized":"2024-08-17T17:27:05.461047+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:20.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1232,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.470681+0000","last_change":"2024-08-17T17:21:06.480911+0000","last_active":"2024-08-17T17:27:05.470681+0000","last_peered":"2024-08-17T17:27:05.470681+0000","last_clean":"2024-08-17T17:27:05.470681+0000","last_became_active":"2024-08-17T17:21:06.480551+0000","last_became_peered":"2024-08-17T17:21:06.480551+0000","last_unstale":"2024-08-17T17:27:05.470681+0000","last_undegraded":"2024-08-17T17:27:05.470681+0000","last_fullsized":"2024-08-17T17:27:05.470681+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:26.003704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"1253'3667","reported_seq":3682,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:17.452559+0000","last_change":"2024-08-17T17:26:34.715560+0000","last_active":"2024-08-17T17:27:17.452559+0000","last_peered":"2024-08-17T17:27:17.452559+0000","last_clean":"2024-08-17T17:27:17.452559+0000","last_became_active":"2024-08-17T17:26:34.714955+0000","last_became_peered":"2024-08-17T17:26:34.714955+0000","last_unstale":"2024-08-17T17:27:17.452559+0000","last_undegraded":"2024-08-17T17:27:17.452559+0000","last_fullsized":"2024-08-17T17:27:17.452559+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3667,"log_dups_size":0,"ondisk_log_size":3667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:46.367495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240320512,"num_objects":3667,"num_object_clones":0,"num_object_copies":11001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3667,"num_write_kb":234688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":775,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.466155+0000","last_change":"2024-08-17T17:24:57.682573+0000","last_active":"2024-08-17T17:27:05.466155+0000","last_peered":"2024-08-17T17:27:05.466155+0000","last_clean":"2024-08-17T17:27:05.466155+0000","last_became_active":"2024-08-17T17:24:31.869575+0000","last_became_peered":"2024-08-17T17:24:31.869575+0000","last_unstale":"2024-08-17T17:27:05.466155+0000","last_undegraded":"2024-08-17T17:27:05.466155+0000","last_fullsized":"2024-08-17T17:27:05.466155+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:57.682409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:57.682409+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:42.144615+0000","scrub_duration":0.0066879779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1201,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.469164+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:27:05.469164+0000","last_peered":"2024-08-17T17:27:05.469164+0000","last_clean":"2024-08-17T17:27:05.469164+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:27:05.469164+0000","last_undegraded":"2024-08-17T17:27:05.469164+0000","last_fullsized":"2024-08-17T17:27:05.469164+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1217,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.470696+0000","last_change":"2024-08-17T17:21:06.451521+0000","last_active":"2024-08-17T17:27:05.470696+0000","last_peered":"2024-08-17T17:27:05.470696+0000","last_clean":"2024-08-17T17:27:05.470696+0000","last_became_active":"2024-08-17T17:21:06.451197+0000","last_became_peered":"2024-08-17T17:21:06.451197+0000","last_unstale":"2024-08-17T17:27:05.470696+0000","last_undegraded":"2024-08-17T17:27:05.470696+0000","last_fullsized":"2024-08-17T17:27:05.470696+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:04.000692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"1253'3548","reported_seq":3571,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.627997+0000","last_change":"2024-08-17T17:26:35.023654+0000","last_active":"2024-08-17T17:27:15.627997+0000","last_peered":"2024-08-17T17:27:15.627997+0000","last_clean":"2024-08-17T17:27:15.627997+0000","last_became_active":"2024-08-17T17:26:35.022968+0000","last_became_peered":"2024-08-17T17:26:35.022968+0000","last_unstale":"2024-08-17T17:27:15.627997+0000","last_undegraded":"2024-08-17T17:27:15.627997+0000","last_fullsized":"2024-08-17T17:27:15.627997+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3548,"log_dups_size":0,"ondisk_log_size":3548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:25:10.494807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232521728,"num_objects":3548,"num_object_clones":0,"num_object_copies":10644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3548,"num_write_kb":227072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1543,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:12.817472+0000","last_change":"2024-08-17T17:24:52.716059+0000","last_active":"2024-08-17T17:27:12.817472+0000","last_peered":"2024-08-17T17:27:12.817472+0000","last_clean":"2024-08-17T17:27:12.817472+0000","last_became_active":"2024-08-17T17:24:31.798173+0000","last_became_peered":"2024-08-17T17:24:31.798173+0000","last_unstale":"2024-08-17T17:27:12.817472+0000","last_undegraded":"2024-08-17T17:27:12.817472+0000","last_fullsized":"2024-08-17T17:27:12.817472+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:04:56.094994+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1606,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.513564+0000","last_change":"2024-08-17T17:21:06.441118+0000","last_active":"2024-08-17T17:27:15.513564+0000","last_peered":"2024-08-17T17:27:15.513564+0000","last_clean":"2024-08-17T17:27:15.513564+0000","last_became_active":"2024-08-17T17:21:06.440800+0000","last_became_peered":"2024-08-17T17:21:06.440800+0000","last_unstale":"2024-08-17T17:27:15.513564+0000","last_undegraded":"2024-08-17T17:27:15.513564+0000","last_fullsized":"2024-08-17T17:27:15.513564+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:08.091528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":111,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.511317+0000","last_change":"2024-08-17T17:22:48.374766+0000","last_active":"2024-08-17T17:27:15.511317+0000","last_peered":"2024-08-17T17:27:15.511317+0000","last_clean":"2024-08-17T17:27:15.511317+0000","last_became_active":"2024-08-17T17:22:48.373556+0000","last_became_peered":"2024-08-17T17:22:48.373556+0000","last_unstale":"2024-08-17T17:27:15.511317+0000","last_undegraded":"2024-08-17T17:27:15.511317+0000","last_fullsized":"2024-08-17T17:27:15.511317+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:33.362080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.8","version":"1253'3567","reported_seq":3590,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.682991+0000","last_change":"2024-08-17T17:26:34.519719+0000","last_active":"2024-08-17T17:27:15.682991+0000","last_peered":"2024-08-17T17:27:15.682991+0000","last_clean":"2024-08-17T17:27:15.682991+0000","last_became_active":"2024-08-17T17:26:34.519368+0000","last_became_peered":"2024-08-17T17:26:34.519368+0000","last_unstale":"2024-08-17T17:27:15.682991+0000","last_undegraded":"2024-08-17T17:27:15.682991+0000","last_fullsized":"2024-08-17T17:27:15.682991+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3567,"log_dups_size":0,"ondisk_log_size":3567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:44.575457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233766912,"num_objects":3567,"num_object_clones":0,"num_object_copies":10701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3567,"num_write_kb":228288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1597,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.463357+0000","last_change":"2024-08-17T17:24:13.542832+0000","last_active":"2024-08-17T17:27:05.463357+0000","last_peered":"2024-08-17T17:27:05.463357+0000","last_clean":"2024-08-17T17:27:05.463357+0000","last_became_active":"2024-08-17T17:24:11.570282+0000","last_became_peered":"2024-08-17T17:24:11.570282+0000","last_unstale":"2024-08-17T17:27:05.463357+0000","last_undegraded":"2024-08-17T17:27:05.463357+0000","last_fullsized":"2024-08-17T17:27:05.463357+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:49.414162+0000","scrub_duration":0.017597167,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1211,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.466043+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:27:05.466043+0000","last_peered":"2024-08-17T17:27:05.466043+0000","last_clean":"2024-08-17T17:27:05.466043+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:27:05.466043+0000","last_undegraded":"2024-08-17T17:27:05.466043+0000","last_fullsized":"2024-08-17T17:27:05.466043+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1571,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.513353+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:27:15.513353+0000","last_peered":"2024-08-17T17:27:15.513353+0000","last_clean":"2024-08-17T17:27:15.513353+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:27:15.513353+0000","last_undegraded":"2024-08-17T17:27:15.513353+0000","last_fullsized":"2024-08-17T17:27:15.513353+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1183,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.445376+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:27:07.445376+0000","last_peered":"2024-08-17T17:27:07.445376+0000","last_clean":"2024-08-17T17:27:07.445376+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:27:07.445376+0000","last_undegraded":"2024-08-17T17:27:07.445376+0000","last_fullsized":"2024-08-17T17:27:07.445376+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"1253'3692","reported_seq":3707,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:17.428127+0000","last_change":"2024-08-17T17:26:34.519823+0000","last_active":"2024-08-17T17:27:17.428127+0000","last_peered":"2024-08-17T17:27:17.428127+0000","last_clean":"2024-08-17T17:27:17.428127+0000","last_became_active":"2024-08-17T17:26:34.519497+0000","last_became_peered":"2024-08-17T17:26:34.519497+0000","last_unstale":"2024-08-17T17:27:17.428127+0000","last_undegraded":"2024-08-17T17:27:17.428127+0000","last_fullsized":"2024-08-17T17:27:17.428127+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3692,"log_dups_size":0,"ondisk_log_size":3692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:59:31.807854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241958912,"num_objects":3692,"num_object_clones":0,"num_object_copies":11076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3692,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3692,"num_write_kb":236288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":836,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.459488+0000","last_change":"2024-08-17T17:24:11.936322+0000","last_active":"2024-08-17T17:27:05.459488+0000","last_peered":"2024-08-17T17:27:05.459488+0000","last_clean":"2024-08-17T17:27:05.459488+0000","last_became_active":"2024-08-17T17:24:11.574846+0000","last_became_peered":"2024-08-17T17:24:11.574846+0000","last_unstale":"2024-08-17T17:27:05.459488+0000","last_undegraded":"2024-08-17T17:27:05.459488+0000","last_fullsized":"2024-08-17T17:27:05.459488+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:11.936236+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:11.936236+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:31.674851+0000","scrub_duration":0.0035797139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1247,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.463534+0000","last_change":"2024-08-17T17:21:06.430639+0000","last_active":"2024-08-17T17:27:05.463534+0000","last_peered":"2024-08-17T17:27:05.463534+0000","last_clean":"2024-08-17T17:27:05.463534+0000","last_became_active":"2024-08-17T17:21:06.430297+0000","last_became_peered":"2024-08-17T17:21:06.430297+0000","last_unstale":"2024-08-17T17:27:05.463534+0000","last_undegraded":"2024-08-17T17:27:05.463534+0000","last_fullsized":"2024-08-17T17:27:05.463534+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:57.794620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1050,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.468819+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:27:05.468819+0000","last_peered":"2024-08-17T17:27:05.468819+0000","last_clean":"2024-08-17T17:27:05.468819+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:27:05.468819+0000","last_undegraded":"2024-08-17T17:27:05.468819+0000","last_fullsized":"2024-08-17T17:27:05.468819+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":62,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:12.820565+0000","last_change":"2024-08-17T17:22:48.370769+0000","last_active":"2024-08-17T17:27:12.820565+0000","last_peered":"2024-08-17T17:27:12.820565+0000","last_clean":"2024-08-17T17:27:12.820565+0000","last_became_active":"2024-08-17T17:22:48.370531+0000","last_became_peered":"2024-08-17T17:22:48.370531+0000","last_unstale":"2024-08-17T17:27:12.820565+0000","last_undegraded":"2024-08-17T17:27:12.820565+0000","last_fullsized":"2024-08-17T17:27:12.820565+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:02.943823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"1253'3726","reported_seq":3741,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:17.805608+0000","last_change":"2024-08-17T17:26:34.715167+0000","last_active":"2024-08-17T17:27:17.805608+0000","last_peered":"2024-08-17T17:27:17.805608+0000","last_clean":"2024-08-17T17:27:17.805608+0000","last_became_active":"2024-08-17T17:26:34.714868+0000","last_became_peered":"2024-08-17T17:26:34.714868+0000","last_unstale":"2024-08-17T17:27:17.805608+0000","last_undegraded":"2024-08-17T17:27:17.805608+0000","last_fullsized":"2024-08-17T17:27:17.805608+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3726,"log_dups_size":0,"ondisk_log_size":3726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:19.957252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244187136,"num_objects":3726,"num_object_clones":0,"num_object_copies":11178,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3726,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3726,"num_write_kb":238464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1618,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.463229+0000","last_change":"2024-08-17T17:24:12.519946+0000","last_active":"2024-08-17T17:27:05.463229+0000","last_peered":"2024-08-17T17:27:05.463229+0000","last_clean":"2024-08-17T17:27:05.463229+0000","last_became_active":"2024-08-17T17:24:11.569747+0000","last_became_peered":"2024-08-17T17:24:11.569747+0000","last_unstale":"2024-08-17T17:27:05.463229+0000","last_undegraded":"2024-08-17T17:27:05.463229+0000","last_fullsized":"2024-08-17T17:27:05.463229+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.519838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.519838+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:56.003993+0000","scrub_duration":0.0068224740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1668,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.511316+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:27:15.511316+0000","last_peered":"2024-08-17T17:27:15.511316+0000","last_clean":"2024-08-17T17:27:15.511316+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:27:15.511316+0000","last_undegraded":"2024-08-17T17:27:15.511316+0000","last_fullsized":"2024-08-17T17:27:15.511316+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1060,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.511201+0000","last_change":"2024-08-17T17:21:06.428684+0000","last_active":"2024-08-17T17:27:15.511201+0000","last_peered":"2024-08-17T17:27:15.511201+0000","last_clean":"2024-08-17T17:27:15.511201+0000","last_became_active":"2024-08-17T17:21:06.428178+0000","last_became_peered":"2024-08-17T17:21:06.428178+0000","last_unstale":"2024-08-17T17:27:15.511201+0000","last_undegraded":"2024-08-17T17:27:15.511201+0000","last_fullsized":"2024-08-17T17:27:15.511201+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:14.836119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":894,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.465870+0000","last_change":"2024-08-17T17:21:06.443565+0000","last_active":"2024-08-17T17:27:05.465870+0000","last_peered":"2024-08-17T17:27:05.465870+0000","last_clean":"2024-08-17T17:27:05.465870+0000","last_became_active":"2024-08-17T17:21:06.443215+0000","last_became_peered":"2024-08-17T17:21:06.443215+0000","last_unstale":"2024-08-17T17:27:05.465870+0000","last_undegraded":"2024-08-17T17:27:05.465870+0000","last_fullsized":"2024-08-17T17:27:05.465870+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:47.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"1253'3780","reported_seq":3793,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:17.450710+0000","last_change":"2024-08-17T17:26:34.519795+0000","last_active":"2024-08-17T17:27:17.450710+0000","last_peered":"2024-08-17T17:27:17.450710+0000","last_clean":"2024-08-17T17:27:17.450710+0000","last_became_active":"2024-08-17T17:26:34.519426+0000","last_became_peered":"2024-08-17T17:26:34.519426+0000","last_unstale":"2024-08-17T17:27:17.450710+0000","last_undegraded":"2024-08-17T17:27:17.450710+0000","last_fullsized":"2024-08-17T17:27:17.450710+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3780,"log_dups_size":0,"ondisk_log_size":3780,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:14.719799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247726080,"num_objects":3780,"num_object_clones":0,"num_object_copies":11340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3780,"num_write_kb":241920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1560,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.465506+0000","last_change":"2024-08-17T17:24:13.795227+0000","last_active":"2024-08-17T17:27:05.465506+0000","last_peered":"2024-08-17T17:27:05.465506+0000","last_clean":"2024-08-17T17:27:05.465506+0000","last_became_active":"2024-08-17T17:24:11.578352+0000","last_became_peered":"2024-08-17T17:24:11.578352+0000","last_unstale":"2024-08-17T17:27:05.465506+0000","last_undegraded":"2024-08-17T17:27:05.465506+0000","last_fullsized":"2024-08-17T17:27:05.465506+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:36:39.440808+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1238,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.466182+0000","last_change":"2024-08-17T17:21:06.456978+0000","last_active":"2024-08-17T17:27:05.466182+0000","last_peered":"2024-08-17T17:27:05.466182+0000","last_clean":"2024-08-17T17:27:05.466182+0000","last_became_active":"2024-08-17T17:21:06.452265+0000","last_became_peered":"2024-08-17T17:21:06.452265+0000","last_unstale":"2024-08-17T17:27:05.466182+0000","last_undegraded":"2024-08-17T17:27:05.466182+0000","last_fullsized":"2024-08-17T17:27:05.466182+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:52.893746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1220,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.470289+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:27:05.470289+0000","last_peered":"2024-08-17T17:27:05.470289+0000","last_clean":"2024-08-17T17:27:05.470289+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:27:05.470289+0000","last_undegraded":"2024-08-17T17:27:05.470289+0000","last_fullsized":"2024-08-17T17:27:05.470289+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1136,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.511761+0000","last_change":"2024-08-17T17:21:06.435754+0000","last_active":"2024-08-17T17:27:15.511761+0000","last_peered":"2024-08-17T17:27:15.511761+0000","last_clean":"2024-08-17T17:27:15.511761+0000","last_became_active":"2024-08-17T17:21:06.435442+0000","last_became_peered":"2024-08-17T17:21:06.435442+0000","last_unstale":"2024-08-17T17:27:15.511761+0000","last_undegraded":"2024-08-17T17:27:15.511761+0000","last_fullsized":"2024-08-17T17:27:15.511761+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:34.849042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"1253'3637","reported_seq":3652,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:17.805845+0000","last_change":"2024-08-17T17:26:34.716730+0000","last_active":"2024-08-17T17:27:17.805845+0000","last_peered":"2024-08-17T17:27:17.805845+0000","last_clean":"2024-08-17T17:27:17.805845+0000","last_became_active":"2024-08-17T17:26:34.716457+0000","last_became_peered":"2024-08-17T17:26:34.716457+0000","last_unstale":"2024-08-17T17:27:17.805845+0000","last_undegraded":"2024-08-17T17:27:17.805845+0000","last_fullsized":"2024-08-17T17:27:17.805845+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3637,"log_dups_size":0,"ondisk_log_size":3637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:51.133413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238354432,"num_objects":3637,"num_object_clones":0,"num_object_copies":10911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3637,"num_write_kb":232768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":823,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.445408+0000","last_change":"2024-08-17T17:24:55.041464+0000","last_active":"2024-08-17T17:27:07.445408+0000","last_peered":"2024-08-17T17:27:07.445408+0000","last_clean":"2024-08-17T17:27:07.445408+0000","last_became_active":"2024-08-17T17:24:31.800289+0000","last_became_peered":"2024-08-17T17:24:31.800289+0000","last_unstale":"2024-08-17T17:27:07.445408+0000","last_undegraded":"2024-08-17T17:27:07.445408+0000","last_fullsized":"2024-08-17T17:27:07.445408+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:55.041378+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:55.041378+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:43.974501+0000","scrub_duration":0.001623642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1213,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.466474+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:27:05.466474+0000","last_peered":"2024-08-17T17:27:05.466474+0000","last_clean":"2024-08-17T17:27:05.466474+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:27:05.466474+0000","last_undegraded":"2024-08-17T17:27:05.466474+0000","last_fullsized":"2024-08-17T17:27:05.466474+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":903,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.463800+0000","last_change":"2024-08-17T17:21:06.434991+0000","last_active":"2024-08-17T17:27:05.463800+0000","last_peered":"2024-08-17T17:27:05.463800+0000","last_clean":"2024-08-17T17:27:05.463800+0000","last_became_active":"2024-08-17T17:21:06.434630+0000","last_became_peered":"2024-08-17T17:21:06.434630+0000","last_unstale":"2024-08-17T17:27:05.463800+0000","last_undegraded":"2024-08-17T17:27:05.463800+0000","last_fullsized":"2024-08-17T17:27:05.463800+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:24.104609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":845,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.466084+0000","last_change":"2024-08-17T17:21:06.462532+0000","last_active":"2024-08-17T17:27:05.466084+0000","last_peered":"2024-08-17T17:27:05.466084+0000","last_clean":"2024-08-17T17:27:05.466084+0000","last_became_active":"2024-08-17T17:21:06.462183+0000","last_became_peered":"2024-08-17T17:21:06.462183+0000","last_unstale":"2024-08-17T17:27:05.466084+0000","last_undegraded":"2024-08-17T17:27:05.466084+0000","last_fullsized":"2024-08-17T17:27:05.466084+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:50.033322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.e","version":"1253'3774","reported_seq":3789,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:17.425236+0000","last_change":"2024-08-17T17:26:34.716166+0000","last_active":"2024-08-17T17:27:17.425236+0000","last_peered":"2024-08-17T17:27:17.425236+0000","last_clean":"2024-08-17T17:27:17.425236+0000","last_became_active":"2024-08-17T17:26:34.715299+0000","last_became_peered":"2024-08-17T17:26:34.715299+0000","last_unstale":"2024-08-17T17:27:17.425236+0000","last_undegraded":"2024-08-17T17:27:17.425236+0000","last_fullsized":"2024-08-17T17:27:17.425236+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3774,"log_dups_size":0,"ondisk_log_size":3774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:37.004999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247332864,"num_objects":3774,"num_object_clones":0,"num_object_copies":11322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3774,"num_write_kb":241536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":840,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.445389+0000","last_change":"2024-08-17T17:24:53.103270+0000","last_active":"2024-08-17T17:27:07.445389+0000","last_peered":"2024-08-17T17:27:07.445389+0000","last_clean":"2024-08-17T17:27:07.445389+0000","last_became_active":"2024-08-17T17:24:31.800823+0000","last_became_peered":"2024-08-17T17:24:31.800823+0000","last_unstale":"2024-08-17T17:27:07.445389+0000","last_undegraded":"2024-08-17T17:27:07.445389+0000","last_fullsized":"2024-08-17T17:27:07.445389+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.103133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.103133+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:52.444781+0000","scrub_duration":0.0074281759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1059,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.511576+0000","last_change":"2024-08-17T17:21:06.433720+0000","last_active":"2024-08-17T17:27:15.511576+0000","last_peered":"2024-08-17T17:27:15.511576+0000","last_clean":"2024-08-17T17:27:15.511576+0000","last_became_active":"2024-08-17T17:21:06.433292+0000","last_became_peered":"2024-08-17T17:21:06.433292+0000","last_unstale":"2024-08-17T17:27:15.511576+0000","last_undegraded":"2024-08-17T17:27:15.511576+0000","last_fullsized":"2024-08-17T17:27:15.511576+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:32.246594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1213,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.464305+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:27:05.464305+0000","last_peered":"2024-08-17T17:27:05.464305+0000","last_clean":"2024-08-17T17:27:05.464305+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:27:05.464305+0000","last_undegraded":"2024-08-17T17:27:05.464305+0000","last_fullsized":"2024-08-17T17:27:05.464305+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":805,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.465968+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:27:05.465968+0000","last_peered":"2024-08-17T17:27:05.465968+0000","last_clean":"2024-08-17T17:27:05.465968+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:27:05.465968+0000","last_undegraded":"2024-08-17T17:27:05.465968+0000","last_fullsized":"2024-08-17T17:27:05.465968+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"1253'3727","reported_seq":3741,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:17.456578+0000","last_change":"2024-08-17T17:26:34.715123+0000","last_active":"2024-08-17T17:27:17.456578+0000","last_peered":"2024-08-17T17:27:17.456578+0000","last_clean":"2024-08-17T17:27:17.456578+0000","last_became_active":"2024-08-17T17:26:34.714789+0000","last_became_peered":"2024-08-17T17:26:34.714789+0000","last_unstale":"2024-08-17T17:27:17.456578+0000","last_undegraded":"2024-08-17T17:27:17.456578+0000","last_fullsized":"2024-08-17T17:27:17.456578+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3727,"log_dups_size":0,"ondisk_log_size":3727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:19.314570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244252672,"num_objects":3727,"num_object_clones":0,"num_object_copies":11181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3727,"num_write_kb":238528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1564,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.470822+0000","last_change":"2024-08-17T17:24:49.119578+0000","last_active":"2024-08-17T17:27:05.470822+0000","last_peered":"2024-08-17T17:27:05.470822+0000","last_clean":"2024-08-17T17:27:05.470822+0000","last_became_active":"2024-08-17T17:24:31.807987+0000","last_became_peered":"2024-08-17T17:24:31.807987+0000","last_unstale":"2024-08-17T17:27:05.470822+0000","last_undegraded":"2024-08-17T17:27:05.470822+0000","last_fullsized":"2024-08-17T17:27:05.470822+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:34.052786+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1625,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.514780+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:27:15.514780+0000","last_peered":"2024-08-17T17:27:15.514780+0000","last_clean":"2024-08-17T17:27:15.514780+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:27:15.514780+0000","last_undegraded":"2024-08-17T17:27:15.514780+0000","last_fullsized":"2024-08-17T17:27:15.514780+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":855,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.460218+0000","last_change":"2024-08-17T17:21:06.452395+0000","last_active":"2024-08-17T17:27:05.460218+0000","last_peered":"2024-08-17T17:27:05.460218+0000","last_clean":"2024-08-17T17:27:05.460218+0000","last_became_active":"2024-08-17T17:21:06.451867+0000","last_became_peered":"2024-08-17T17:21:06.451867+0000","last_unstale":"2024-08-17T17:27:05.460218+0000","last_undegraded":"2024-08-17T17:27:05.460218+0000","last_fullsized":"2024-08-17T17:27:05.460218+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:50.597092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1614,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.513260+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:27:15.513260+0000","last_peered":"2024-08-17T17:27:15.513260+0000","last_clean":"2024-08-17T17:27:15.513260+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:27:15.513260+0000","last_undegraded":"2024-08-17T17:27:15.513260+0000","last_fullsized":"2024-08-17T17:27:15.513260+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"1253'3434","reported_seq":3457,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.676624+0000","last_change":"2024-08-17T17:26:34.412017+0000","last_active":"2024-08-17T17:27:15.676624+0000","last_peered":"2024-08-17T17:27:15.676624+0000","last_clean":"2024-08-17T17:27:15.676624+0000","last_became_active":"2024-08-17T17:26:34.411768+0000","last_became_peered":"2024-08-17T17:26:34.411768+0000","last_unstale":"2024-08-17T17:27:15.676624+0000","last_undegraded":"2024-08-17T17:27:15.676624+0000","last_fullsized":"2024-08-17T17:27:15.676624+0000","mapping_epoch":1247,"log_start":"0'0","ondisk_log_start":"0'0","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":3434,"log_dups_size":0,"ondisk_log_size":3434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:33:06.384935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225050624,"num_objects":3434,"num_object_clones":0,"num_object_copies":10302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3434,"num_write_kb":219776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1650,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.513704+0000","last_change":"2024-08-17T17:24:12.033217+0000","last_active":"2024-08-17T17:27:15.513704+0000","last_peered":"2024-08-17T17:27:15.513704+0000","last_clean":"2024-08-17T17:27:15.513704+0000","last_became_active":"2024-08-17T17:24:11.572877+0000","last_became_peered":"2024-08-17T17:24:11.572877+0000","last_unstale":"2024-08-17T17:27:15.513704+0000","last_undegraded":"2024-08-17T17:27:15.513704+0000","last_fullsized":"2024-08-17T17:27:15.513704+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.033101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.033101+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.264475+0000","scrub_duration":20,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":224,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.470621+0000","last_change":"2024-08-17T17:21:06.519965+0000","last_active":"2024-08-17T17:27:05.470621+0000","last_peered":"2024-08-17T17:27:05.470621+0000","last_clean":"2024-08-17T17:27:05.470621+0000","last_became_active":"2024-08-17T17:21:06.459952+0000","last_became_peered":"2024-08-17T17:21:06.459952+0000","last_unstale":"2024-08-17T17:27:05.470621+0000","last_undegraded":"2024-08-17T17:27:05.470621+0000","last_fullsized":"2024-08-17T17:27:05.470621+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:27.828098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1210,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.445437+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:27:07.445437+0000","last_peered":"2024-08-17T17:27:07.445437+0000","last_clean":"2024-08-17T17:27:07.445437+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:27:07.445437+0000","last_undegraded":"2024-08-17T17:27:07.445437+0000","last_fullsized":"2024-08-17T17:27:07.445437+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1186,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.468911+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:27:05.468911+0000","last_peered":"2024-08-17T17:27:05.468911+0000","last_clean":"2024-08-17T17:27:05.468911+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:27:05.468911+0000","last_undegraded":"2024-08-17T17:27:05.468911+0000","last_fullsized":"2024-08-17T17:27:05.468911+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":715,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.513493+0000","last_change":"2024-08-17T17:24:48.935651+0000","last_active":"2024-08-17T17:27:15.513493+0000","last_peered":"2024-08-17T17:27:15.513493+0000","last_clean":"2024-08-17T17:27:15.513493+0000","last_became_active":"2024-08-17T17:24:40.897753+0000","last_became_peered":"2024-08-17T17:24:40.897753+0000","last_unstale":"2024-08-17T17:27:15.513493+0000","last_undegraded":"2024-08-17T17:27:15.513493+0000","last_fullsized":"2024-08-17T17:27:15.513493+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:10:15.118428+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1217,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.470115+0000","last_change":"2024-08-17T17:21:06.458000+0000","last_active":"2024-08-17T17:27:05.470115+0000","last_peered":"2024-08-17T17:27:05.470115+0000","last_clean":"2024-08-17T17:27:05.470115+0000","last_became_active":"2024-08-17T17:21:06.457411+0000","last_became_peered":"2024-08-17T17:21:06.457411+0000","last_unstale":"2024-08-17T17:27:05.470115+0000","last_undegraded":"2024-08-17T17:27:05.470115+0000","last_fullsized":"2024-08-17T17:27:05.470115+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:52.740639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1208,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.471398+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:27:05.471398+0000","last_peered":"2024-08-17T17:27:05.471398+0000","last_clean":"2024-08-17T17:27:05.471398+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:27:05.471398+0000","last_undegraded":"2024-08-17T17:27:05.471398+0000","last_fullsized":"2024-08-17T17:27:05.471398+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":685,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.465788+0000","last_change":"2024-08-17T17:25:03.663370+0000","last_active":"2024-08-17T17:27:05.465788+0000","last_peered":"2024-08-17T17:27:05.465788+0000","last_clean":"2024-08-17T17:27:05.465788+0000","last_became_active":"2024-08-17T17:24:40.886214+0000","last_became_peered":"2024-08-17T17:24:40.886214+0000","last_unstale":"2024-08-17T17:27:05.465788+0000","last_undegraded":"2024-08-17T17:27:05.465788+0000","last_fullsized":"2024-08-17T17:27:05.465788+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:11:39.272029+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":877,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.460971+0000","last_change":"2024-08-17T17:21:06.454587+0000","last_active":"2024-08-17T17:27:05.460971+0000","last_peered":"2024-08-17T17:27:05.460971+0000","last_clean":"2024-08-17T17:27:05.460971+0000","last_became_active":"2024-08-17T17:21:06.454229+0000","last_became_peered":"2024-08-17T17:21:06.454229+0000","last_unstale":"2024-08-17T17:27:05.460971+0000","last_undegraded":"2024-08-17T17:27:05.460971+0000","last_fullsized":"2024-08-17T17:27:05.460971+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:38.340263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1183,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.464010+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:27:05.464010+0000","last_peered":"2024-08-17T17:27:05.464010+0000","last_clean":"2024-08-17T17:27:05.464010+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:27:05.464010+0000","last_undegraded":"2024-08-17T17:27:05.464010+0000","last_fullsized":"2024-08-17T17:27:05.464010+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1652,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.510962+0000","last_change":"2024-08-17T17:24:20.649977+0000","last_active":"2024-08-17T17:27:15.510962+0000","last_peered":"2024-08-17T17:27:15.510962+0000","last_clean":"2024-08-17T17:27:15.510962+0000","last_became_active":"2024-08-17T17:24:20.649336+0000","last_became_peered":"2024-08-17T17:24:20.649336+0000","last_unstale":"2024-08-17T17:27:15.510962+0000","last_undegraded":"2024-08-17T17:27:15.510962+0000","last_fullsized":"2024-08-17T17:27:15.510962+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:57.187271+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1213,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.465820+0000","last_change":"2024-08-17T17:21:06.442796+0000","last_active":"2024-08-17T17:27:05.465820+0000","last_peered":"2024-08-17T17:27:05.465820+0000","last_clean":"2024-08-17T17:27:05.465820+0000","last_became_active":"2024-08-17T17:21:06.442466+0000","last_became_peered":"2024-08-17T17:21:06.442466+0000","last_unstale":"2024-08-17T17:27:05.465820+0000","last_undegraded":"2024-08-17T17:27:05.465820+0000","last_fullsized":"2024-08-17T17:27:05.465820+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:36.366620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1570,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.518019+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:27:15.518019+0000","last_peered":"2024-08-17T17:27:15.518019+0000","last_clean":"2024-08-17T17:27:15.518019+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:27:15.518019+0000","last_undegraded":"2024-08-17T17:27:15.518019+0000","last_fullsized":"2024-08-17T17:27:15.518019+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1432,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.464218+0000","last_change":"2024-08-17T17:25:07.472056+0000","last_active":"2024-08-17T17:27:05.464218+0000","last_peered":"2024-08-17T17:27:05.464218+0000","last_clean":"2024-08-17T17:27:05.464218+0000","last_became_active":"2024-08-17T17:24:40.883356+0000","last_became_peered":"2024-08-17T17:24:40.883356+0000","last_unstale":"2024-08-17T17:27:05.464218+0000","last_undegraded":"2024-08-17T17:27:05.464218+0000","last_fullsized":"2024-08-17T17:27:05.464218+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:43:27.883917+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1050,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:12.820662+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:27:12.820662+0000","last_peered":"2024-08-17T17:27:12.820662+0000","last_clean":"2024-08-17T17:27:12.820662+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:27:12.820662+0000","last_undegraded":"2024-08-17T17:27:12.820662+0000","last_fullsized":"2024-08-17T17:27:12.820662+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1603,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.513175+0000","last_change":"2024-08-17T17:21:06.468788+0000","last_active":"2024-08-17T17:27:15.513175+0000","last_peered":"2024-08-17T17:27:15.513175+0000","last_clean":"2024-08-17T17:27:15.513175+0000","last_became_active":"2024-08-17T17:21:06.467141+0000","last_became_peered":"2024-08-17T17:21:06.467141+0000","last_unstale":"2024-08-17T17:27:15.513175+0000","last_undegraded":"2024-08-17T17:27:15.513175+0000","last_fullsized":"2024-08-17T17:27:15.513175+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:51.301359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1432,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.635982+0000","last_change":"2024-08-17T17:24:53.130774+0000","last_active":"2024-08-17T17:27:07.635982+0000","last_peered":"2024-08-17T17:27:07.635982+0000","last_clean":"2024-08-17T17:27:07.635982+0000","last_became_active":"2024-08-17T17:24:40.883388+0000","last_became_peered":"2024-08-17T17:24:40.883388+0000","last_unstale":"2024-08-17T17:27:07.635982+0000","last_undegraded":"2024-08-17T17:27:07.635982+0000","last_fullsized":"2024-08-17T17:27:07.635982+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:36.772875+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1216,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.460590+0000","last_change":"2024-08-17T17:21:06.435998+0000","last_active":"2024-08-17T17:27:05.460590+0000","last_peered":"2024-08-17T17:27:05.460590+0000","last_clean":"2024-08-17T17:27:05.460590+0000","last_became_active":"2024-08-17T17:21:06.435579+0000","last_became_peered":"2024-08-17T17:21:06.435579+0000","last_unstale":"2024-08-17T17:27:05.460590+0000","last_undegraded":"2024-08-17T17:27:05.460590+0000","last_fullsized":"2024-08-17T17:27:05.460590+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:01.743074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1633,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.513437+0000","last_change":"2024-08-17T17:21:06.473627+0000","last_active":"2024-08-17T17:27:15.513437+0000","last_peered":"2024-08-17T17:27:15.513437+0000","last_clean":"2024-08-17T17:27:15.513437+0000","last_became_active":"2024-08-17T17:21:06.473062+0000","last_became_peered":"2024-08-17T17:21:06.473062+0000","last_unstale":"2024-08-17T17:27:15.513437+0000","last_undegraded":"2024-08-17T17:27:15.513437+0000","last_fullsized":"2024-08-17T17:27:15.513437+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:21.413163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":792,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.465756+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:27:05.465756+0000","last_peered":"2024-08-17T17:27:05.465756+0000","last_clean":"2024-08-17T17:27:05.465756+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:27:05.465756+0000","last_undegraded":"2024-08-17T17:27:05.465756+0000","last_fullsized":"2024-08-17T17:27:05.465756+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1011,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.455648+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:27:07.455648+0000","last_peered":"2024-08-17T17:27:07.455648+0000","last_clean":"2024-08-17T17:27:07.455648+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:27:07.455648+0000","last_undegraded":"2024-08-17T17:27:07.455648+0000","last_fullsized":"2024-08-17T17:27:07.455648+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1123,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.518205+0000","last_change":"2024-08-17T17:21:06.471265+0000","last_active":"2024-08-17T17:27:15.518205+0000","last_peered":"2024-08-17T17:27:15.518205+0000","last_clean":"2024-08-17T17:27:15.518205+0000","last_became_active":"2024-08-17T17:21:06.470943+0000","last_became_peered":"2024-08-17T17:21:06.470943+0000","last_unstale":"2024-08-17T17:27:15.518205+0000","last_undegraded":"2024-08-17T17:27:15.518205+0000","last_fullsized":"2024-08-17T17:27:15.518205+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:41.070024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1564,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.466028+0000","last_change":"2024-08-17T17:25:02.696608+0000","last_active":"2024-08-17T17:27:05.466028+0000","last_peered":"2024-08-17T17:27:05.466028+0000","last_clean":"2024-08-17T17:27:05.466028+0000","last_became_active":"2024-08-17T17:24:40.877715+0000","last_became_peered":"2024-08-17T17:24:40.877715+0000","last_unstale":"2024-08-17T17:27:05.466028+0000","last_undegraded":"2024-08-17T17:27:05.466028+0000","last_fullsized":"2024-08-17T17:27:05.466028+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:48:00.941285+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1609,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.513203+0000","last_change":"2024-08-17T17:22:48.377351+0000","last_active":"2024-08-17T17:27:15.513203+0000","last_peered":"2024-08-17T17:27:15.513203+0000","last_clean":"2024-08-17T17:27:15.513203+0000","last_became_active":"2024-08-17T17:22:48.377059+0000","last_became_peered":"2024-08-17T17:22:48.377059+0000","last_unstale":"2024-08-17T17:27:15.513203+0000","last_undegraded":"2024-08-17T17:27:15.513203+0000","last_fullsized":"2024-08-17T17:27:15.513203+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:15.250902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1233,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.469592+0000","last_change":"2024-08-17T17:21:06.459139+0000","last_active":"2024-08-17T17:27:05.469592+0000","last_peered":"2024-08-17T17:27:05.469592+0000","last_clean":"2024-08-17T17:27:05.469592+0000","last_became_active":"2024-08-17T17:21:06.457850+0000","last_became_peered":"2024-08-17T17:21:06.457850+0000","last_unstale":"2024-08-17T17:27:05.469592+0000","last_undegraded":"2024-08-17T17:27:05.469592+0000","last_fullsized":"2024-08-17T17:27:05.469592+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:13.888576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1511,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.514665+0000","last_change":"2024-08-17T17:24:49.911734+0000","last_active":"2024-08-17T17:27:15.514665+0000","last_peered":"2024-08-17T17:27:15.514665+0000","last_clean":"2024-08-17T17:27:15.514665+0000","last_became_active":"2024-08-17T17:24:40.899102+0000","last_became_peered":"2024-08-17T17:24:40.899102+0000","last_unstale":"2024-08-17T17:27:15.514665+0000","last_undegraded":"2024-08-17T17:27:15.514665+0000","last_fullsized":"2024-08-17T17:27:15.514665+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.911622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.911622+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:30:27.987033+0000","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1651,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.511667+0000","last_change":"2024-08-17T17:21:06.446321+0000","last_active":"2024-08-17T17:27:15.511667+0000","last_peered":"2024-08-17T17:27:15.511667+0000","last_clean":"2024-08-17T17:27:15.511667+0000","last_became_active":"2024-08-17T17:21:06.446019+0000","last_became_peered":"2024-08-17T17:21:06.446019+0000","last_unstale":"2024-08-17T17:27:15.511667+0000","last_undegraded":"2024-08-17T17:27:15.511667+0000","last_fullsized":"2024-08-17T17:27:15.511667+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:26.750070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":202,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.469836+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:27:05.469836+0000","last_peered":"2024-08-17T17:27:05.469836+0000","last_clean":"2024-08-17T17:27:05.469836+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:27:05.469836+0000","last_undegraded":"2024-08-17T17:27:05.469836+0000","last_fullsized":"2024-08-17T17:27:05.469836+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":715,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.513876+0000","last_change":"2024-08-17T17:24:46.946320+0000","last_active":"2024-08-17T17:27:15.513876+0000","last_peered":"2024-08-17T17:27:15.513876+0000","last_clean":"2024-08-17T17:27:15.513876+0000","last_became_active":"2024-08-17T17:24:40.880380+0000","last_became_peered":"2024-08-17T17:24:40.880380+0000","last_unstale":"2024-08-17T17:27:15.513876+0000","last_undegraded":"2024-08-17T17:27:15.513876+0000","last_fullsized":"2024-08-17T17:27:15.513876+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:54:30.637162+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1231,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.470042+0000","last_change":"2024-08-17T17:21:06.465804+0000","last_active":"2024-08-17T17:27:05.470042+0000","last_peered":"2024-08-17T17:27:05.470042+0000","last_clean":"2024-08-17T17:27:05.470042+0000","last_became_active":"2024-08-17T17:21:06.465461+0000","last_became_peered":"2024-08-17T17:21:06.465461+0000","last_unstale":"2024-08-17T17:27:05.470042+0000","last_undegraded":"2024-08-17T17:27:05.470042+0000","last_fullsized":"2024-08-17T17:27:05.470042+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:06.443667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1223,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.465500+0000","last_change":"2024-08-17T17:21:06.481634+0000","last_active":"2024-08-17T17:27:05.465500+0000","last_peered":"2024-08-17T17:27:05.465500+0000","last_clean":"2024-08-17T17:27:05.465500+0000","last_became_active":"2024-08-17T17:21:06.469081+0000","last_became_peered":"2024-08-17T17:21:06.469081+0000","last_unstale":"2024-08-17T17:27:05.465500+0000","last_undegraded":"2024-08-17T17:27:05.465500+0000","last_fullsized":"2024-08-17T17:27:05.465500+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:53:26.954738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1231,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.469445+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:27:05.469445+0000","last_peered":"2024-08-17T17:27:05.469445+0000","last_clean":"2024-08-17T17:27:05.469445+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:27:05.469445+0000","last_undegraded":"2024-08-17T17:27:05.469445+0000","last_fullsized":"2024-08-17T17:27:05.469445+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1182,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.466406+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:27:05.466406+0000","last_peered":"2024-08-17T17:27:05.466406+0000","last_clean":"2024-08-17T17:27:05.466406+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:27:05.466406+0000","last_undegraded":"2024-08-17T17:27:05.466406+0000","last_fullsized":"2024-08-17T17:27:05.466406+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1623,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:07.455592+0000","last_change":"2024-08-17T17:25:02.417545+0000","last_active":"2024-08-17T17:27:07.455592+0000","last_peered":"2024-08-17T17:27:07.455592+0000","last_clean":"2024-08-17T17:27:07.455592+0000","last_became_active":"2024-08-17T17:24:40.883220+0000","last_became_peered":"2024-08-17T17:24:40.883220+0000","last_unstale":"2024-08-17T17:27:07.455592+0000","last_undegraded":"2024-08-17T17:27:07.455592+0000","last_fullsized":"2024-08-17T17:27:07.455592+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:28.257805+0000","scrub_duration":0.0084952199999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1609,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.518347+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:27:15.518347+0000","last_peered":"2024-08-17T17:27:15.518347+0000","last_clean":"2024-08-17T17:27:15.518347+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:27:15.518347+0000","last_undegraded":"2024-08-17T17:27:15.518347+0000","last_fullsized":"2024-08-17T17:27:15.518347+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1584,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.514230+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:27:15.514230+0000","last_peered":"2024-08-17T17:27:15.514230+0000","last_clean":"2024-08-17T17:27:15.514230+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:27:15.514230+0000","last_undegraded":"2024-08-17T17:27:15.514230+0000","last_fullsized":"2024-08-17T17:27:15.514230+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1459,"reported_epoch":1253,"state":"active+clean","last_fresh":"2024-08-17T17:27:15.513097+0000","last_change":"2024-08-17T17:24:47.961645+0000","last_active":"2024-08-17T17:27:15.513097+0000","last_peered":"2024-08-17T17:27:15.513097+0000","last_clean":"2024-08-17T17:27:15.513097+0000","last_became_active":"2024-08-17T17:24:40.897539+0000","last_became_peered":"2024-08-17T17:24:40.897539+0000","last_unstale":"2024-08-17T17:27:15.513097+0000","last_undegraded":"2024-08-17T17:27:15.513097+0000","last_fullsized":"2024-08-17T17:27:15.513097+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:47.961445+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:47.961445+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:04:32.031483+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1216,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.468711+0000","last_change":"2024-08-17T17:21:06.483469+0000","last_active":"2024-08-17T17:27:05.468711+0000","last_peered":"2024-08-17T17:27:05.468711+0000","last_clean":"2024-08-17T17:27:05.468711+0000","last_became_active":"2024-08-17T17:21:06.483162+0000","last_became_peered":"2024-08-17T17:21:06.483162+0000","last_unstale":"2024-08-17T17:27:05.468711+0000","last_undegraded":"2024-08-17T17:27:05.468711+0000","last_fullsized":"2024-08-17T17:27:05.468711+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:24.898353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":225,"reported_epoch":1252,"state":"active+clean","last_fresh":"2024-08-17T17:27:05.466340+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:27:05.466340+0000","last_peered":"2024-08-17T17:27:05.466340+0000","last_clean":"2024-08-17T17:27:05.466340+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:27:05.466340+0000","last_undegraded":"2024-08-17T17:27:05.466340+0000","last_fullsized":"2024-08-17T17:27:05.466340+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3839229952,"num_objects":58582,"num_object_clones":0,"num_object_copies":175746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":58582,"num_write_kb":3749248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11587092480,"data_stored":11587092480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":58582,"ondisk_log_size":58582,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201710,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2634312,"kb_used_data":2115968,"kb_used_omap":1,"kb_used_meta":518334,"kb_avail":91110840,"statfs":{"total":95995035648,"available":93297500160,"internally_reserved":0,"allocated":2166751232,"data_stored":2164155579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":530774475},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463347,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1101760,"kb_used_data":705652,"kb_used_omap":1,"kb_used_meta":396094,"kb_avail":92643392,"statfs":{"total":95995035648,"available":94866833408,"internally_reserved":0,"allocated":722587648,"data_stored":720003612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":405600713},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887626988,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2416736,"kb_used_data":1793176,"kb_used_omap":42,"kb_used_meta":623509,"kb_avail":91328416,"statfs":{"total":95995035648,"available":93520297984,"internally_reserved":0,"allocated":1836212224,"data_stored":1833643857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":638473276},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,4,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785466,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2653804,"kb_used_data":2156452,"kb_used_omap":48,"kb_used_meta":497295,"kb_avail":91091348,"statfs":{"total":95995035648,"available":93277540352,"internally_reserved":0,"allocated":2208206848,"data_stored":2205634382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":509230766},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724983,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2629628,"kb_used_data":2129328,"kb_used_omap":1,"kb_used_meta":500286,"kb_avail":91115524,"statfs":{"total":95995035648,"available":93302296576,"internally_reserved":0,"allocated":2180431872,"data_stored":2177848302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":512293318},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3,0,1,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986620,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1158208,"kb_used_data":707960,"kb_used_omap":1,"kb_used_meta":450238,"kb_avail":92586944,"statfs":{"total":95995035648,"available":94809030656,"internally_reserved":0,"allocated":724951040,"data_stored":722362965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":461044166},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706227,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1222648,"kb_used_data":699180,"kb_used_omap":35,"kb_used_meta":523420,"kb_avail":92522504,"statfs":{"total":95995035648,"available":94743044096,"internally_reserved":0,"allocated":715960320,"data_stored":713380803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":535982135},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832000,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1525492,"kb_used_data":943272,"kb_used_omap":65,"kb_used_meta":582142,"kb_avail":92219660,"statfs":{"total":95995035648,"available":94432931840,"internally_reserved":0,"allocated":965910528,"data_stored":963334551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":596113595},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":964952064,"data_stored":964952064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2230255616,"data_stored":2230255616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":718143488,"data_stored":718143488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1842610176,"data_stored":1842610176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":724500480,"data_stored":724500480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2206859264,"data_stored":2206859264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":717357056,"data_stored":717357056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2182414336,"data_stored":2182414336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:27:24.351 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_4 pgp_num 16 2024-08-17T17:27:24.369 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:24.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.423 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:24.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:24.460 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 38 16 68162 68146 112.068 62.3125 0.00776001 0.00891764 2024-08-17T17:27:24.524 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:24.559 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:24.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:24.661 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:24.683 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:24.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:24.785 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:24.817 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:24.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:24.919 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:24.949 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:24.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:24.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:25.024 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:25.050 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:25.095 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:25.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:25.197 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:25.239 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:25.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:25.341 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:25.362 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:25.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:25.460 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 39 16 69266 69250 110.963 69 0.0142116 0.00900618 2024-08-17T17:27:25.464 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:25.498 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:25.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:25.600 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:25.622 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:25.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:25.724 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:25.744 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:25.768 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:25.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:25.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:25 smithi063 ceph-mon[109843]: pgmap v1718: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-17T17:27:25.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:25 smithi063 ceph-mon[114756]: pgmap v1718: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-17T17:27:25.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:25 smithi162 ceph-mon[104668]: pgmap v1718: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-08-17T17:27:25.870 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:25.937 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:25.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:25.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:26.040 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:26.075 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:26.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:26.176 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:26.215 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:26.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:26.317 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:26.351 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:26.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:26.453 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:26.460 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:27:26.460228+0000 min lat: 0.00127496 max lat: 0.34753 avg lat: 0.00907162 2024-08-17T17:27:26.460 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:27:26.460 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 40 16 70534 70518 110.17 79.25 0.0194124 0.00907162 2024-08-17T17:27:26.487 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:26.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:26.584 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:27:26.589 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:26.621 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:26.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:26.723 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:26.752 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:26.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:26.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:26 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3677372847' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:26.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:26 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:26.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:26 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3677372847' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:26.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:26 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:26 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3677372847' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:26.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:26 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:26.853 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:26.900 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:26.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:26.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:27.002 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:27.026 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:27.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:27.128 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:27.168 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:27.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:27.270 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:27.300 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:27.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:27.401 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:27.442 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:27.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:27.460 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 41 16 71775 71759 109.375 77.5625 0.00614755 0.00913764 2024-08-17T17:27:27.545 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:27.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.578 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:27.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:27.679 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:27.711 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:27.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:27.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:27 smithi063 ceph-mon[114756]: pgmap v1719: 119 pgs: 119 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-17T17:27:27.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:27 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:27.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:27 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:27.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:27 smithi063 ceph-mon[114756]: osdmap e1254: 8 total, 8 up, 8 in 2024-08-17T17:27:27.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:27 smithi063 ceph-mon[109843]: pgmap v1719: 119 pgs: 119 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-17T17:27:27.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:27 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:27.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:27 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:27.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:27 smithi063 ceph-mon[109843]: osdmap e1254: 8 total, 8 up, 8 in 2024-08-17T17:27:27.813 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:27 smithi162 ceph-mon[104668]: pgmap v1719: 119 pgs: 119 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-08-17T17:27:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:27 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:27 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:27 smithi162 ceph-mon[104668]: osdmap e1254: 8 total, 8 up, 8 in 2024-08-17T17:27:27.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:27.854 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:27.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:27.955 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:27.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.040 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:28.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:28.141 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:28.182 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:28.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:28.284 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:28.319 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:28.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:28.421 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:28.447 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:28.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:28.460 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 42 16 72958 72942 108.531 73.9375 0.00311082 0.00920237 2024-08-17T17:27:28.549 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:28.572 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:28.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:28.674 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:28.696 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:28.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:28.798 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:28.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.821 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:28.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:28.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:28.958 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:28.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:28.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:29.060 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:29.104 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:29.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:29.206 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:29.244 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:29.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:29.346 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:29.384 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:29.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:29.425 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:27:29.461 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 43 16 73969 73953 107.474 63.1875 0.0117277 0.00929912 2024-08-17T17:27:29.466 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:27:29.466 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:27:29.486 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:29.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.538 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:29.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:29.639 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:29.672 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:29.708 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:27:29.709 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:27:29.709 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:29 smithi063 ceph-mon[109843]: pgmap v1721: 119 pgs: 119 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-17T17:27:29.709 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:29 smithi063 ceph-mon[114756]: pgmap v1721: 119 pgs: 119 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-17T17:27:29.744 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:27:29.745 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:27:29.774 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:29.780 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:27:29.780 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:27:29.805 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:29.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:29.812 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:27:29.812 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:27:29.823 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:29 smithi162 ceph-mon[104668]: pgmap v1721: 119 pgs: 119 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-17T17:27:29.845 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:27:29.845 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:27:29.879 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:27:29.880 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:27:29.907 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:29.931 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:27:29.931 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:27:29.957 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:29.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:29.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:29.973 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:27:29.973 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:27:30.059 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:30.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.105 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:30.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:30.163 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:27:30.206 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:30.252 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:30.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:30.354 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:30.399 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:30.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:30.463 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 44 16 75177 75161 106.744 75.5 0.00492057 0.00936384 2024-08-17T17:27:30.501 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:30.541 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:30.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:30.552 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:30.642 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:30.681 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:30.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:30.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:30 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1800416878' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:30.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:30 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1800416878' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:30.783 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:30.823 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:30.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:30.841 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:30 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1800416878' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:30.925 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:30.939 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:30.979 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:30.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:30.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:31.081 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:31.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.118 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:31.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:31.219 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:31.266 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:31.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:31.368 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:31.410 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:31.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:31.463 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 45 16 76478 76462 106.179 81.3125 0.0184389 0.00941336 2024-08-17T17:27:31.511 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:31.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:31.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:31.618 INFO:teuthology.orchestra.run.smithi063.stderr:set pool 9 pgp_num to 16 2024-08-17T17:27:31.648 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:31.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.694 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:31.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:31.795 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:31.831 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:31.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:31.843 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:31 smithi162 ceph-mon[104668]: pgmap v1722: 119 pgs: 119 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-17T17:27:31.843 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:31 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3766220071' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:27:31.843 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:31.933 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:31.955 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:31.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:31.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:31 smithi063 ceph-mon[114756]: pgmap v1722: 119 pgs: 119 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-17T17:27:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:31 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3766220071' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:27:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:31 smithi063 ceph-mon[109843]: pgmap v1722: 119 pgs: 119 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-08-17T17:27:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:31 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3766220071' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"} : dispatch 2024-08-17T17:27:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:32.057 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:32.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.095 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:32.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:32.196 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:32.225 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:32.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:32.327 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:32.366 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:32.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:32.463 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 46 16 77743 77727 105.589 79.0625 0.0082587 0.00946537 2024-08-17T17:27:32.468 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:32.516 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:32.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:32.525 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:27:32.525 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:27:32.525 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:27:32.526 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:27:32.526 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.526 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65735 7.7 7:e7f48e21:::benchmark_data_smithi063_196002_object65734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-08-17T17:27:32.526 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.525390+0000", 2024-08-17T17:27:32.526 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 541.99638700900005, 2024-08-17T17:27:32.526 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4912375070000001, 2024-08-17T17:27:32.526 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.526 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.526 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.526 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:32.526 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:32.527 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65735 2024-08-17T17:27:32.527 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.527 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.527 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.527 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.527 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525390+0000", 2024-08-17T17:27:32.527 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.527 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.527 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.527 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.527 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525390+0000", 2024-08-17T17:27:32.527 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.527 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.528 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.528 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.528 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525388+0000", 2024-08-17T17:27:32.528 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:27:32.528 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.528 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.528 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.528 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525427+0000", 2024-08-17T17:27:32.528 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8878000000000003e-05 2024-08-17T17:27:32.528 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.528 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.528 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.529 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525439+0000", 2024-08-17T17:27:32.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2507000000000001e-05 2024-08-17T17:27:32.529 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.529 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.529 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.529 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.525447+0000", 2024-08-17T17:27:32.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7530000000000001e-06 2024-08-17T17:27:32.529 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.529 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.529 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.529 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.533687+0000", 2024-08-17T17:27:32.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0082400100000000007 2024-08-17T17:27:32.530 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.530 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.530 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.530 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.533697+0000", 2024-08-17T17:27:32.530 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0455000000000001e-05 2024-08-17T17:27:32.530 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.530 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.530 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.530 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523807+0000", 2024-08-17T17:27:32.530 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99010963600000002 2024-08-17T17:27:32.530 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.530 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.531 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.531 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523817+0000", 2024-08-17T17:27:32.531 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7189999999999996e-06 2024-08-17T17:27:32.531 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.531 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.531 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.531 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:32.998106+0000", 2024-08-17T17:27:32.531 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4742891280000001 2024-08-17T17:27:32.532 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.532 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.532 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:32.998241+0000", 2024-08-17T17:27:32.532 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013504000000000001 2024-08-17T17:27:32.532 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.532 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.532 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:27:32.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:32.998370+0000", 2024-08-17T17:27:32.532 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012903799999999999 2024-08-17T17:27:32.532 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.532 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.533 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.533 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.003851+0000", 2024-08-17T17:27:32.533 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0054806999999999998 2024-08-17T17:27:32.533 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.533 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.533 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.533 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.005429+0000", 2024-08-17T17:27:32.533 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015783819999999999 2024-08-17T17:27:32.533 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.533 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.533 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.533 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.014199+0000", 2024-08-17T17:27:32.534 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0087699130000000007 2024-08-17T17:27:32.534 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.534 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.534 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.534 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.014257+0000", 2024-08-17T17:27:32.534 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8193999999999999e-05 2024-08-17T17:27:32.534 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.534 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.534 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.534 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.016627+0000", 2024-08-17T17:27:32.534 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023701730000000002 2024-08-17T17:27:32.534 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.535 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.535 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.535 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.535 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.535 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65738 7.d 7:b7ed0697:::benchmark_data_smithi063_196002_object65737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:27:32.535 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.535426+0000", 2024-08-17T17:27:32.535 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 541.98635091699998, 2024-08-17T17:27:32.535 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5234001109999999, 2024-08-17T17:27:32.535 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.535 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.535 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.535 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:32.535 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:32.535 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65738 2024-08-17T17:27:32.536 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.536 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.536 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.536 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.536 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535426+0000", 2024-08-17T17:27:32.536 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.536 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.536 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.536 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.536 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535426+0000", 2024-08-17T17:27:32.536 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.536 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.536 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.537 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.537 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535424+0000", 2024-08-17T17:27:32.537 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:32.537 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.537 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.537 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.537 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535475+0000", 2024-08-17T17:27:32.537 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0512e-05 2024-08-17T17:27:32.537 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.537 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.537 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.537 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535486+0000", 2024-08-17T17:27:32.537 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0689999999999999e-05 2024-08-17T17:27:32.537 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.538 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.538 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.538 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535492+0000", 2024-08-17T17:27:32.538 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7379999999999997e-06 2024-08-17T17:27:32.538 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.538 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.538 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.538 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537456+0000", 2024-08-17T17:27:32.538 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0019633559999999999 2024-08-17T17:27:32.538 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.538 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.538 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.538 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537467+0000", 2024-08-17T17:27:32.539 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0784999999999999e-05 2024-08-17T17:27:32.539 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.539 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.539 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.539 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523461+0000", 2024-08-17T17:27:32.539 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.985994009 2024-08-17T17:27:32.539 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.539 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.539 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.539 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523472+0000", 2024-08-17T17:27:32.539 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1806e-05 2024-08-17T17:27:32.539 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.540 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.540 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.540 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.030195+0000", 2024-08-17T17:27:32.540 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.506722414 2024-08-17T17:27:32.540 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.540 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.540 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.540 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.030370+0000", 2024-08-17T17:27:32.540 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017517499999999999 2024-08-17T17:27:32.540 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.540 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.540 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:27:32.540 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.030534+0000", 2024-08-17T17:27:32.541 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016449999999999999 2024-08-17T17:27:32.541 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.541 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.541 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.038628+0000", 2024-08-17T17:27:32.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0080940379999999996 2024-08-17T17:27:32.542 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.542 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.542 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.542 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.038638+0000", 2024-08-17T17:27:32.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8770000000000005e-06 2024-08-17T17:27:32.542 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.542 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.542 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.542 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.056336+0000", 2024-08-17T17:27:32.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.017698034000000001 2024-08-17T17:27:32.543 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.543 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.543 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.543 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.056374+0000", 2024-08-17T17:27:32.543 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7753000000000003e-05 2024-08-17T17:27:32.543 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.543 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.543 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.543 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.058826+0000", 2024-08-17T17:27:32.543 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0024519149999999998 2024-08-17T17:27:32.543 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.543 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.543 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.544 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.544 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.544 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65740 7.7 7:e4a48506:::benchmark_data_smithi063_196002_object65739:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:27:32.544 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.537667+0000", 2024-08-17T17:27:32.544 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 541.98410940500003, 2024-08-17T17:27:32.544 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4812937370000001, 2024-08-17T17:27:32.544 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.544 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.544 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.544 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:32.544 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:32.544 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65740 2024-08-17T17:27:32.544 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.545 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.545 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.545 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.545 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537667+0000", 2024-08-17T17:27:32.545 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.545 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.545 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.545 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.545 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537667+0000", 2024-08-17T17:27:32.545 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.545 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.545 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.546 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.546 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537666+0000", 2024-08-17T17:27:32.546 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:32.546 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.546 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.546 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.546 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537699+0000", 2024-08-17T17:27:32.546 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2543000000000003e-05 2024-08-17T17:27:32.546 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.546 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.546 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.546 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537708+0000", 2024-08-17T17:27:32.547 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9700000000000005e-06 2024-08-17T17:27:32.547 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.547 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.547 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.547 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537715+0000", 2024-08-17T17:27:32.547 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1860000000000003e-06 2024-08-17T17:27:32.547 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.547 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.547 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.547 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537786+0000", 2024-08-17T17:27:32.547 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0732e-05 2024-08-17T17:27:32.547 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.547 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.548 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.548 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.537795+0000", 2024-08-17T17:27:32.548 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7370000000000001e-06 2024-08-17T17:27:32.548 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.548 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.548 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.548 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524063+0000", 2024-08-17T17:27:32.548 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98626732800000005 2024-08-17T17:27:32.548 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.548 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.548 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.548 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524072+0000", 2024-08-17T17:27:32.548 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.1949999999999992e-06 2024-08-17T17:27:32.549 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.549 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.001760+0000", 2024-08-17T17:27:32.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4776884450000001 2024-08-17T17:27:32.549 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.549 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.001916+0000", 2024-08-17T17:27:32.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015600400000000001 2024-08-17T17:27:32.549 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.549 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:27:32.550 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.002089+0000", 2024-08-17T17:27:32.550 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017285900000000001 2024-08-17T17:27:32.550 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.550 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.550 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.550 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.006924+0000", 2024-08-17T17:27:32.550 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0048349899999999999 2024-08-17T17:27:32.550 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.550 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.550 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.550 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.006974+0000", 2024-08-17T17:27:32.550 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9648999999999998e-05 2024-08-17T17:27:32.551 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.551 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.551 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.551 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.016708+0000", 2024-08-17T17:27:32.551 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0097340340000000008 2024-08-17T17:27:32.551 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.551 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.551 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.551 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.016776+0000", 2024-08-17T17:27:32.551 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8320999999999999e-05 2024-08-17T17:27:32.551 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.551 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.551 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.551 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.018961+0000", 2024-08-17T17:27:32.552 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021849840000000001 2024-08-17T17:27:32.552 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.552 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.552 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.552 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.552 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.552 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65748 7.d 7:bfbcc155:::benchmark_data_smithi063_196002_object65747:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:27:32.552 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.545206+0000", 2024-08-17T17:27:32.552 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 541.97657117000006, 2024-08-17T17:27:32.552 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5163881620000002, 2024-08-17T17:27:32.552 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.552 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.552 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.553 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:32.553 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:32.553 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65748 2024-08-17T17:27:32.553 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.553 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.553 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.553 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.553 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545206+0000", 2024-08-17T17:27:32.553 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.553 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.553 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.553 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.553 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545206+0000", 2024-08-17T17:27:32.554 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.554 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.554 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.554 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.554 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545205+0000", 2024-08-17T17:27:32.554 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:32.554 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.554 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.554 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.554 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545338+0000", 2024-08-17T17:27:32.554 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000133375 2024-08-17T17:27:32.554 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.554 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.555 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.555 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545351+0000", 2024-08-17T17:27:32.555 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2901000000000001e-05 2024-08-17T17:27:32.555 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.555 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.555 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.555 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545358+0000", 2024-08-17T17:27:32.555 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9009999999999997e-06 2024-08-17T17:27:32.555 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.555 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.555 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.555 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545409+0000", 2024-08-17T17:27:32.555 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1038999999999997e-05 2024-08-17T17:27:32.556 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.556 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.556 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.545418+0000", 2024-08-17T17:27:32.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.0990000000000005e-06 2024-08-17T17:27:32.556 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.556 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.556 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523596+0000", 2024-08-17T17:27:32.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97817841900000002 2024-08-17T17:27:32.556 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.556 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.556 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.523605+0000", 2024-08-17T17:27:32.557 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.1630000000000002e-06 2024-08-17T17:27:32.557 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.557 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.035987+0000", 2024-08-17T17:27:32.557 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.512381864 2024-08-17T17:27:32.557 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.557 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.036136+0000", 2024-08-17T17:27:32.557 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014822899999999999 2024-08-17T17:27:32.557 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.558 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.558 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:27:32.558 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.036293+0000", 2024-08-17T17:27:32.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000157305 2024-08-17T17:27:32.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.558 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.558 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.558 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.041715+0000", 2024-08-17T17:27:32.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.005422105 2024-08-17T17:27:32.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.558 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.558 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.041772+0000", 2024-08-17T17:27:32.559 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7457999999999998e-05 2024-08-17T17:27:32.559 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.559 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.559 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.058975+0000", 2024-08-17T17:27:32.559 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.017202696999999999 2024-08-17T17:27:32.559 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.559 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.559 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.059136+0000", 2024-08-17T17:27:32.559 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016129800000000001 2024-08-17T17:27:32.559 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.560 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.560 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.560 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.061594+0000", 2024-08-17T17:27:32.560 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002457416 2024-08-17T17:27:32.560 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.560 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.560 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.560 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.560 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.560 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65753 7.7 7:e08ff533:::benchmark_data_smithi063_196002_object65752:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:27:32.560 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.548721+0000", 2024-08-17T17:27:32.560 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 541.97305586699997, 2024-08-17T17:27:32.560 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.472712064, 2024-08-17T17:27:32.561 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.561 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.561 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.561 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:32.561 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:32.561 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65753 2024-08-17T17:27:32.561 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.561 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.561 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.561 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.561 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548721+0000", 2024-08-17T17:27:32.561 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.562 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.562 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.562 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.562 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548721+0000", 2024-08-17T17:27:32.562 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.562 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.562 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.562 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.562 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548720+0000", 2024-08-17T17:27:32.562 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:32.562 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.562 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.562 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548841+0000", 2024-08-17T17:27:32.563 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012082399999999999 2024-08-17T17:27:32.563 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548851+0000", 2024-08-17T17:27:32.563 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9660000000000008e-06 2024-08-17T17:27:32.563 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548862+0000", 2024-08-17T17:27:32.563 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0838e-05 2024-08-17T17:27:32.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.564 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.564 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548911+0000", 2024-08-17T17:27:32.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9004e-05 2024-08-17T17:27:32.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.564 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.564 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.548921+0000", 2024-08-17T17:27:32.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9180000000000006e-06 2024-08-17T17:27:32.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524131+0000", 2024-08-17T17:27:32.565 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.975210566 2024-08-17T17:27:32.565 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524140+0000", 2024-08-17T17:27:32.565 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.242e-06 2024-08-17T17:27:32.565 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.003888+0000", 2024-08-17T17:27:32.565 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4797479820000001 2024-08-17T17:27:32.566 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.566 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.566 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.566 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.004030+0000", 2024-08-17T17:27:32.566 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000142495 2024-08-17T17:27:32.566 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.566 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.566 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:27:32.566 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.004221+0000", 2024-08-17T17:27:32.566 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00019079500000000001 2024-08-17T17:27:32.566 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.566 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.567 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.567 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.006933+0000", 2024-08-17T17:27:32.567 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00271244 2024-08-17T17:27:32.567 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.567 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.567 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.567 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.008656+0000", 2024-08-17T17:27:32.567 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001723154 2024-08-17T17:27:32.567 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.567 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.567 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.567 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.019021+0000", 2024-08-17T17:27:32.567 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010364310999999999 2024-08-17T17:27:32.568 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.568 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.568 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.568 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.019082+0000", 2024-08-17T17:27:32.568 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0998000000000001e-05 2024-08-17T17:27:32.568 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.568 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.568 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.568 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.021433+0000", 2024-08-17T17:27:32.568 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023512530000000002 2024-08-17T17:27:32.568 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.568 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.568 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.569 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.569 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.569 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65763 7.7 7:e13a5ff4:::benchmark_data_smithi063_196002_object65762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:27:32.569 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.559250+0000", 2024-08-17T17:27:32.569 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 541.96252666500004, 2024-08-17T17:27:32.569 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4649766419999999, 2024-08-17T17:27:32.569 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.569 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.569 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.569 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:32.569 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:32.569 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65763 2024-08-17T17:27:32.569 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.570 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.570 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.570 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.570 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559250+0000", 2024-08-17T17:27:32.570 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.570 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.570 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.570 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.570 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559250+0000", 2024-08-17T17:27:32.570 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.570 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.570 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.571 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.571 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559249+0000", 2024-08-17T17:27:32.571 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:32.571 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.571 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.571 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.571 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559294+0000", 2024-08-17T17:27:32.571 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5008e-05 2024-08-17T17:27:32.571 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.571 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.571 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.572 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559304+0000", 2024-08-17T17:27:32.572 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0543999999999999e-05 2024-08-17T17:27:32.572 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.572 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.572 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.572 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559311+0000", 2024-08-17T17:27:32.572 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9999999999999999e-06 2024-08-17T17:27:32.572 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.572 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.572 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.572 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559352+0000", 2024-08-17T17:27:32.572 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0099999999999999e-05 2024-08-17T17:27:32.572 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.572 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.573 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.573 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.559360+0000", 2024-08-17T17:27:32.573 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4090000000000003e-06 2024-08-17T17:27:32.573 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.573 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.573 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.573 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524193+0000", 2024-08-17T17:27:32.573 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.96483273400000003 2024-08-17T17:27:32.573 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.573 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.573 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.573 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.524201+0000", 2024-08-17T17:27:32.573 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.5280000000000003e-06 2024-08-17T17:27:32.574 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.574 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.574 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.574 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.005532+0000", 2024-08-17T17:27:32.574 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4813310959999999 2024-08-17T17:27:32.574 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.574 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.574 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.574 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.005618+0000", 2024-08-17T17:27:32.574 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6101999999999997e-05 2024-08-17T17:27:32.574 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.574 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.575 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:27:32.575 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.005742+0000", 2024-08-17T17:27:32.575 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012364099999999999 2024-08-17T17:27:32.575 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.575 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.575 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.575 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.007064+0000", 2024-08-17T17:27:32.575 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013215550000000001 2024-08-17T17:27:32.575 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.575 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.575 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.575 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.008743+0000", 2024-08-17T17:27:32.575 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016792140000000001 2024-08-17T17:27:32.576 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.576 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.576 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.576 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.021506+0000", 2024-08-17T17:27:32.576 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012763480000000001 2024-08-17T17:27:32.576 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.576 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.576 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.576 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.021565+0000", 2024-08-17T17:27:32.576 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8486000000000002e-05 2024-08-17T17:27:32.576 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.576 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.577 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.577 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.024227+0000", 2024-08-17T17:27:32.577 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026619759999999999 2024-08-17T17:27:32.577 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.577 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.577 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.577 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.577 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.577 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88363 7.7 7:e6097857:::benchmark_data_smithi063_196002_object88362:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:27:32.577 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:00.712141+0000", 2024-08-17T17:27:32.577 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 511.80963579199999, 2024-08-17T17:27:32.577 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9602415930000001, 2024-08-17T17:27:32.577 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.578 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:27:32.578 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.578 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:32.578 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:32.578 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88363 2024-08-17T17:27:32.578 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.578 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.578 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.578 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.578 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712141+0000", 2024-08-17T17:27:32.578 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.578 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.579 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.579 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.579 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712141+0000", 2024-08-17T17:27:32.579 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.579 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.579 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.579 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.579 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712138+0000", 2024-08-17T17:27:32.579 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999971 2024-08-17T17:27:32.579 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.579 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.579 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.580 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712181+0000", 2024-08-17T17:27:32.580 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3390999999999998e-05 2024-08-17T17:27:32.580 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.580 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.580 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.580 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712208+0000", 2024-08-17T17:27:32.580 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6259e-05 2024-08-17T17:27:32.580 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.580 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.580 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.580 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.712219+0000", 2024-08-17T17:27:32.580 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1686e-05 2024-08-17T17:27:32.580 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.581 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.581 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.581 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.771097+0000", 2024-08-17T17:27:32.581 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.058877691000000003 2024-08-17T17:27:32.581 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.581 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.581 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.581 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.771106+0000", 2024-08-17T17:27:32.581 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6079999999999996e-06 2024-08-17T17:27:32.581 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.581 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.581 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.581 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.650544+0000", 2024-08-17T17:27:32.581 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.879438045 2024-08-17T17:27:32.582 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.582 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.582 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.582 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.650553+0000", 2024-08-17T17:27:32.582 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.6520000000000001e-06 2024-08-17T17:27:32.582 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.582 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.582 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.582 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.672283+0000", 2024-08-17T17:27:32.582 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0217297430000001 2024-08-17T17:27:32.582 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.582 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.582 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.583 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.672383+0000", 2024-08-17T17:27:32.583 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9489999999999998e-05 2024-08-17T17:27:32.583 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.583 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.583 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.583 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.583 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.583 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:170142 7.d 7:b454d72d:::benchmark_data_smithi063_196002_object58049:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1164)", 2024-08-17T17:27:32.583 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:20:43.923974+0000", 2024-08-17T17:27:32.583 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 408.59780290399999, 2024-08-17T17:27:32.584 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98031507699999998, 2024-08-17T17:27:32.584 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.584 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.584 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.584 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:32.584 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:32.584 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 170142 2024-08-17T17:27:32.584 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.584 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.584 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.584 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.584 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923974+0000", 2024-08-17T17:27:32.584 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.585 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.585 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.585 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.585 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923974+0000", 2024-08-17T17:27:32.585 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.585 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.585 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.585 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.585 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923973+0000", 2024-08-17T17:27:32.585 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:32.585 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.585 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.585 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.585 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923978+0000", 2024-08-17T17:27:32.586 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9819999999999999e-06 2024-08-17T17:27:32.586 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.586 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.587 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.587 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923989+0000", 2024-08-17T17:27:32.587 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1197e-05 2024-08-17T17:27:32.587 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.587 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.587 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.587 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.923994+0000", 2024-08-17T17:27:32.587 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9599999999999999e-06 2024-08-17T17:27:32.587 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.587 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.587 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.588 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.924041+0000", 2024-08-17T17:27:32.588 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7633999999999998e-05 2024-08-17T17:27:32.588 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.588 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.588 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.588 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:43.924050+0000", 2024-08-17T17:27:32.588 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0940000000000008e-06 2024-08-17T17:27:32.588 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.588 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.588 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.588 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.863783+0000", 2024-08-17T17:27:32.588 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.93973320699999996 2024-08-17T17:27:32.589 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.589 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.589 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for degraded object", 2024-08-17T17:27:32.589 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.864178+0000", 2024-08-17T17:27:32.589 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00039516399999999999 2024-08-17T17:27:32.589 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.589 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.589 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.589 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.885932+0000", 2024-08-17T17:27:32.589 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.021753828999999999 2024-08-17T17:27:32.589 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.589 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.589 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.590 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.886015+0000", 2024-08-17T17:27:32.590 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3139999999999993e-05 2024-08-17T17:27:32.590 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.590 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.590 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:27:32.590 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.886349+0000", 2024-08-17T17:27:32.590 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000334555 2024-08-17T17:27:32.590 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.590 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.590 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.590 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.891760+0000", 2024-08-17T17:27:32.590 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0054106279999999998 2024-08-17T17:27:32.591 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.591 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.591 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.591 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.900574+0000", 2024-08-17T17:27:32.591 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0088143300000000004 2024-08-17T17:27:32.591 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.591 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.591 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.591 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.903791+0000", 2024-08-17T17:27:32.591 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0032164749999999999 2024-08-17T17:27:32.591 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.591 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.591 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.592 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.903846+0000", 2024-08-17T17:27:32.592 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5198000000000002e-05 2024-08-17T17:27:32.592 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.592 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.592 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.592 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:20:44.904289+0000", 2024-08-17T17:27:32.592 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00044295599999999999 2024-08-17T17:27:32.592 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.592 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.592 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.592 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.592 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.592 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186372 7.d 7:b527266b:::benchmark_data_smithi063_196002_object74279:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1172)", 2024-08-17T17:27:32.593 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.416634+0000", 2024-08-17T17:27:32.593 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 387.10514310399998, 2024-08-17T17:27:32.593 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.260521416, 2024-08-17T17:27:32.593 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.593 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:27:32.593 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.593 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:32.593 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:32.593 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186372 2024-08-17T17:27:32.593 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.593 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.593 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.593 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.593 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416634+0000", 2024-08-17T17:27:32.594 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.594 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.594 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.594 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.594 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416634+0000", 2024-08-17T17:27:32.594 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.594 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.594 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.594 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.594 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416632+0000", 2024-08-17T17:27:32.594 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:32.594 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.595 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.595 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.595 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416640+0000", 2024-08-17T17:27:32.595 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4399999999999999e-06 2024-08-17T17:27:32.595 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.595 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.595 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.595 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416653+0000", 2024-08-17T17:27:32.595 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3383999999999999e-05 2024-08-17T17:27:32.595 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.595 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.595 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.595 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416661+0000", 2024-08-17T17:27:32.596 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3100000000000001e-06 2024-08-17T17:27:32.596 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.596 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.596 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.596 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416714+0000", 2024-08-17T17:27:32.596 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3134999999999999e-05 2024-08-17T17:27:32.596 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.596 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.596 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.596 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416894+0000", 2024-08-17T17:27:32.596 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017996599999999999 2024-08-17T17:27:32.596 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.596 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.597 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:27:32.597 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.417361+0000", 2024-08-17T17:27:32.597 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00046639499999999998 2024-08-17T17:27:32.597 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.597 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.597 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.597 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.432229+0000", 2024-08-17T17:27:32.597 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.014867811 2024-08-17T17:27:32.597 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.597 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.597 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.597 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.432244+0000", 2024-08-17T17:27:32.597 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5260999999999998e-05 2024-08-17T17:27:32.598 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.598 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.598 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.598 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.623931+0000", 2024-08-17T17:27:32.598 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1916872569999999 2024-08-17T17:27:32.598 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.598 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.598 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:27:32.598 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.624058+0000", 2024-08-17T17:27:32.598 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000127033 2024-08-17T17:27:32.598 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.598 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.598 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.599 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.677057+0000", 2024-08-17T17:27:32.599 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.052998521999999999 2024-08-17T17:27:32.599 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.599 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.599 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.599 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.677155+0000", 2024-08-17T17:27:32.599 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8443999999999995e-05 2024-08-17T17:27:32.599 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.599 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.599 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.599 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.599 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.600 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186392 7.d 7:b05cf35d:::benchmark_data_smithi063_196002_object74299:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1173)", 2024-08-17T17:27:32.600 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.464967+0000", 2024-08-17T17:27:32.600 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 387.05680997299999, 2024-08-17T17:27:32.600 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2400478340000001, 2024-08-17T17:27:32.600 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.600 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.600 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.600 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:32.600 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:32.600 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186392 2024-08-17T17:27:32.600 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.600 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.600 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.600 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.601 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464967+0000", 2024-08-17T17:27:32.601 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.601 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.601 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.601 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.601 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464967+0000", 2024-08-17T17:27:32.601 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.601 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.601 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.601 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.601 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464966+0000", 2024-08-17T17:27:32.601 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:32.601 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.602 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.602 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.602 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464971+0000", 2024-08-17T17:27:32.602 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5700000000000003e-06 2024-08-17T17:27:32.602 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.602 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.602 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.602 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464979+0000", 2024-08-17T17:27:32.602 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.5019999999999992e-06 2024-08-17T17:27:32.602 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.602 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.602 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.603 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.464985+0000", 2024-08-17T17:27:32.603 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6919999999999999e-06 2024-08-17T17:27:32.603 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.603 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.603 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.603 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.472207+0000", 2024-08-17T17:27:32.603 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0072221009999999999 2024-08-17T17:27:32.603 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.603 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.603 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.603 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.472220+0000", 2024-08-17T17:27:32.603 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.314e-05 2024-08-17T17:27:32.604 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.604 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.604 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.604 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.677176+0000", 2024-08-17T17:27:32.604 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.204956286 2024-08-17T17:27:32.604 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.604 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.604 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.604 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.677275+0000", 2024-08-17T17:27:32.604 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9032999999999999e-05 2024-08-17T17:27:32.604 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.604 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.604 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:27:32.605 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.677671+0000", 2024-08-17T17:27:32.605 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00039520700000000001 2024-08-17T17:27:32.605 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.605 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.605 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.605 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.682047+0000", 2024-08-17T17:27:32.605 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0043766289999999999 2024-08-17T17:27:32.605 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.605 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.605 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.605 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.686155+0000", 2024-08-17T17:27:32.605 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0041077830000000003 2024-08-17T17:27:32.606 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.606 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.606 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.606 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.704798+0000", 2024-08-17T17:27:32.606 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.018642549000000001 2024-08-17T17:27:32.606 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.606 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.606 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.606 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.704836+0000", 2024-08-17T17:27:32.606 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8770999999999999e-05 2024-08-17T17:27:32.606 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.606 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.606 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.607 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.705015+0000", 2024-08-17T17:27:32.607 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017840600000000001 2024-08-17T17:27:32.607 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.607 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.607 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.607 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.607 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.607 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186408 7.1d 7:bfe64f7d:::benchmark_data_smithi063_196002_object74315:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1173)", 2024-08-17T17:27:32.607 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.493415+0000", 2024-08-17T17:27:32.607 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 387.02836186799999, 2024-08-17T17:27:32.607 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1831635359999999, 2024-08-17T17:27:32.607 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.607 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.608 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.608 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:32.608 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:32.608 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186408 2024-08-17T17:27:32.608 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.608 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.608 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.608 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.608 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493415+0000", 2024-08-17T17:27:32.608 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.608 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.608 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.609 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.609 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493415+0000", 2024-08-17T17:27:32.609 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.609 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.609 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.609 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.609 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493414+0000", 2024-08-17T17:27:32.609 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:32.609 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.609 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.609 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.609 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493419+0000", 2024-08-17T17:27:32.609 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4320000000000001e-06 2024-08-17T17:27:32.610 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.610 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.610 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.610 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493429+0000", 2024-08-17T17:27:32.610 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0558e-05 2024-08-17T17:27:32.610 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.610 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.610 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.610 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493436+0000", 2024-08-17T17:27:32.610 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4459999999999998e-06 2024-08-17T17:27:32.610 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.610 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.610 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.611 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493483+0000", 2024-08-17T17:27:32.611 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7105999999999999e-05 2024-08-17T17:27:32.611 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.611 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.611 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:32.611 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.493491+0000", 2024-08-17T17:27:32.611 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4919999999999993e-06 2024-08-17T17:27:32.611 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.611 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.611 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.611 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.540092+0000", 2024-08-17T17:27:32.611 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.046600636 2024-08-17T17:27:32.611 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.612 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.612 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.612 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.540231+0000", 2024-08-17T17:27:32.612 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013884100000000001 2024-08-17T17:27:32.612 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.612 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.612 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:27:32.612 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.540568+0000", 2024-08-17T17:27:32.612 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00033741400000000002 2024-08-17T17:27:32.612 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.612 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.612 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.612 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.546007+0000", 2024-08-17T17:27:32.613 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.005438539 2024-08-17T17:27:32.613 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.613 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.613 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.613 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.546046+0000", 2024-08-17T17:27:32.613 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9446000000000001e-05 2024-08-17T17:27:32.613 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.613 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.613 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.613 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676197+0000", 2024-08-17T17:27:32.613 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.13015059400000001 2024-08-17T17:27:32.613 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.613 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.614 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.614 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676334+0000", 2024-08-17T17:27:32.614 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000137718 2024-08-17T17:27:32.614 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.614 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.614 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.614 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676579+0000", 2024-08-17T17:27:32.614 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00024415 2024-08-17T17:27:32.614 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.614 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.614 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.614 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.614 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.615 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:235947 8.5 8:a52fb054:::benchmark_data_smithi063_230025_object109457:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1241)", 2024-08-17T17:27:32.615 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:25:54.057385+0000", 2024-08-17T17:27:32.615 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 98.464391993000007, 2024-08-17T17:27:32.615 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.24082837500000001, 2024-08-17T17:27:32.615 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.615 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.615 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.615 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:27:32.615 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:27:32.615 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 235947 2024-08-17T17:27:32.615 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.615 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.616 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.616 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.616 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.057385+0000", 2024-08-17T17:27:32.616 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.616 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.616 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.616 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.616 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.057385+0000", 2024-08-17T17:27:32.616 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.616 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.616 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.616 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.616 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.057384+0000", 2024-08-17T17:27:32.617 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:32.617 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.617 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.617 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.617 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.057393+0000", 2024-08-17T17:27:32.617 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8869999999999998e-06 2024-08-17T17:27:32.617 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.617 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.617 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.617 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.057404+0000", 2024-08-17T17:27:32.617 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1313e-05 2024-08-17T17:27:32.617 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.618 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.618 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.618 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.057411+0000", 2024-08-17T17:27:32.618 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3220000000000004e-06 2024-08-17T17:27:32.618 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.618 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.618 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.618 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.059166+0000", 2024-08-17T17:27:32.618 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001755211 2024-08-17T17:27:32.618 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.618 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.618 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.618 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.059281+0000", 2024-08-17T17:27:32.619 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000114948 2024-08-17T17:27:32.619 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.619 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.619 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:27:32.619 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.283985+0000", 2024-08-17T17:27:32.619 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.22470329999999999 2024-08-17T17:27:32.619 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.619 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.619 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.619 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.295649+0000", 2024-08-17T17:27:32.619 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011664033000000001 2024-08-17T17:27:32.619 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.619 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.620 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.620 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.296606+0000", 2024-08-17T17:27:32.620 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00095769699999999998 2024-08-17T17:27:32.620 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.620 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.620 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.620 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.298123+0000", 2024-08-17T17:27:32.620 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015162299999999999 2024-08-17T17:27:32.620 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.620 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.620 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.620 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.298180+0000", 2024-08-17T17:27:32.621 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7583000000000001e-05 2024-08-17T17:27:32.621 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.621 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.621 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.621 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.298213+0000", 2024-08-17T17:27:32.621 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3053999999999999e-05 2024-08-17T17:27:32.621 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.621 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.621 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.621 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.621 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.621 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.45218.0:235953 8.3 e1241/1195 8:c3c2d9eb:::benchmark_data_smithi063_230025_object109463:head v 1241'14899, mlcod=1241'14895)", 2024-08-17T17:27:32.621 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:25:54.061107+0000", 2024-08-17T17:27:32.622 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 98.460670089999994, 2024-08-17T17:27:32.622 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.24117237999999999, 2024-08-17T17:27:32.622 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.622 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.622 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.622 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.622 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.622 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.061107+0000", 2024-08-17T17:27:32.622 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.622 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.622 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.622 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.622 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.061107+0000", 2024-08-17T17:27:32.623 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.623 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.623 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.623 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.623 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.061106+0000", 2024-08-17T17:27:32.623 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:32.623 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.623 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.623 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.623 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.061112+0000", 2024-08-17T17:27:32.623 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3980000000000004e-06 2024-08-17T17:27:32.623 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.623 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.624 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.624 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.283670+0000", 2024-08-17T17:27:32.624 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.22255772900000001 2024-08-17T17:27:32.624 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.624 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.624 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.624 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.283688+0000", 2024-08-17T17:27:32.624 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7909000000000001e-05 2024-08-17T17:27:32.624 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.624 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.624 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.624 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.283759+0000", 2024-08-17T17:27:32.624 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1029e-05 2024-08-17T17:27:32.625 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.625 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.625 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.625 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.283782+0000", 2024-08-17T17:27:32.625 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2986e-05 2024-08-17T17:27:32.625 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.625 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.625 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.625 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.302137+0000", 2024-08-17T17:27:32.625 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.018355106 2024-08-17T17:27:32.625 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.625 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.625 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.626 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.302279+0000", 2024-08-17T17:27:32.626 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000142201 2024-08-17T17:27:32.626 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.626 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.626 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.626 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.626 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.626 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:236010 8.9 8:9c9fd98f:::benchmark_data_smithi063_230025_object109520:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1241)", 2024-08-17T17:27:32.626 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:25:54.305492+0000", 2024-08-17T17:27:32.626 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 98.216284926, 2024-08-17T17:27:32.626 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.34211961699999999, 2024-08-17T17:27:32.626 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.626 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.627 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.627 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:27:32.627 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:27:32.627 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 236010 2024-08-17T17:27:32.627 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.627 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.627 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.627 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.627 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305492+0000", 2024-08-17T17:27:32.627 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.627 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.627 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.627 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.628 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305492+0000", 2024-08-17T17:27:32.628 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.628 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.628 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.628 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.628 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305490+0000", 2024-08-17T17:27:32.628 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:32.628 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.628 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.628 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.628 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305498+0000", 2024-08-17T17:27:32.628 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7570000000000004e-06 2024-08-17T17:27:32.628 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.628 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.629 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.629 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305514+0000", 2024-08-17T17:27:32.629 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5753e-05 2024-08-17T17:27:32.629 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.629 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.629 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.629 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305523+0000", 2024-08-17T17:27:32.629 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8070000000000006e-06 2024-08-17T17:27:32.629 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.629 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.629 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.629 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305577+0000", 2024-08-17T17:27:32.629 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4302999999999998e-05 2024-08-17T17:27:32.630 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.630 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.630 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.630 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305717+0000", 2024-08-17T17:27:32.630 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013970700000000001 2024-08-17T17:27:32.630 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.630 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.630 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:27:32.630 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305845+0000", 2024-08-17T17:27:32.630 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000128403 2024-08-17T17:27:32.630 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.630 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.630 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.631 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.309581+0000", 2024-08-17T17:27:32.631 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0037357129999999999 2024-08-17T17:27:32.631 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.631 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.631 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.631 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.310757+0000", 2024-08-17T17:27:32.631 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001176428 2024-08-17T17:27:32.631 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.631 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.631 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:32.632 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.633 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.647480+0000", 2024-08-17T17:27:32.633 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33672268500000002 2024-08-17T17:27:32.633 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.633 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.633 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.633 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.647567+0000", 2024-08-17T17:27:32.633 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7318000000000007e-05 2024-08-17T17:27:32.633 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.633 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.633 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.633 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.647612+0000", 2024-08-17T17:27:32.633 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4226e-05 2024-08-17T17:27:32.633 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.634 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.634 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.634 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.634 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.634 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:236021 8.9 8:9553e800:::benchmark_data_smithi063_230025_object109531:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1241)", 2024-08-17T17:27:32.634 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:25:54.326491+0000", 2024-08-17T17:27:32.634 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 98.195285773999998, 2024-08-17T17:27:32.634 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.32122857100000002, 2024-08-17T17:27:32.634 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.634 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.634 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.634 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:27:32.635 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:27:32.635 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 236021 2024-08-17T17:27:32.635 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.635 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.635 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.635 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.635 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.326491+0000", 2024-08-17T17:27:32.635 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.635 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.635 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.635 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.635 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.326491+0000", 2024-08-17T17:27:32.636 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.636 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.636 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.636 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.636 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.326490+0000", 2024-08-17T17:27:32.636 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:32.636 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.636 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.636 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.636 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.326497+0000", 2024-08-17T17:27:32.636 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2069999999999998e-06 2024-08-17T17:27:32.636 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.636 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.637 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.637 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.326511+0000", 2024-08-17T17:27:32.637 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4216000000000001e-05 2024-08-17T17:27:32.637 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.637 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.637 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.637 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.326521+0000", 2024-08-17T17:27:32.637 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4939999999999992e-06 2024-08-17T17:27:32.637 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.637 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.637 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.637 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.326561+0000", 2024-08-17T17:27:32.637 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0083000000000002e-05 2024-08-17T17:27:32.638 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.638 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.638 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.638 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.326647+0000", 2024-08-17T17:27:32.638 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6075999999999994e-05 2024-08-17T17:27:32.638 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.638 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.638 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:27:32.638 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.326723+0000", 2024-08-17T17:27:32.638 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6104000000000002e-05 2024-08-17T17:27:32.638 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.638 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.639 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.639 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.329549+0000", 2024-08-17T17:27:32.639 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002825686 2024-08-17T17:27:32.639 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.639 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.639 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.639 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.329909+0000", 2024-08-17T17:27:32.639 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00036013199999999999 2024-08-17T17:27:32.639 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.639 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.639 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.639 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.647651+0000", 2024-08-17T17:27:32.639 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.31774275000000002 2024-08-17T17:27:32.639 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.640 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.640 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.640 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.647688+0000", 2024-08-17T17:27:32.640 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6891000000000003e-05 2024-08-17T17:27:32.640 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.640 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.640 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.640 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.647720+0000", 2024-08-17T17:27:32.640 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1344999999999999e-05 2024-08-17T17:27:32.640 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.640 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.640 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.640 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.641 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.641 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:44828 9.0 9:085cc4e2:::benchmark_data_smithi063_263014_object44827:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:27:32.641 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:08.020757+0000", 2024-08-17T17:27:32.641 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 24.501019934999999, 2024-08-17T17:27:32.641 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.34445221999999998, 2024-08-17T17:27:32.641 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.641 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.641 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.641 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:32.641 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:32.641 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 44828 2024-08-17T17:27:32.641 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.641 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.642 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.642 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.642 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.020757+0000", 2024-08-17T17:27:32.642 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.642 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.642 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.642 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.642 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.020757+0000", 2024-08-17T17:27:32.642 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.642 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.642 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.642 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.643 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.020755+0000", 2024-08-17T17:27:32.643 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:32.643 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.643 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.643 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.643 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.020948+0000", 2024-08-17T17:27:32.643 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00019298300000000001 2024-08-17T17:27:32.643 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.643 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.643 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.643 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.020961+0000", 2024-08-17T17:27:32.643 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2672e-05 2024-08-17T17:27:32.643 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.644 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.644 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.644 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.020967+0000", 2024-08-17T17:27:32.644 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1659999999999999e-06 2024-08-17T17:27:32.644 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.644 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.644 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.644 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.021021+0000", 2024-08-17T17:27:32.644 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3649000000000001e-05 2024-08-17T17:27:32.644 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.644 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.644 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.644 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.021166+0000", 2024-08-17T17:27:32.644 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000144659 2024-08-17T17:27:32.645 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.645 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.645 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:27:32.645 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.021312+0000", 2024-08-17T17:27:32.645 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000146296 2024-08-17T17:27:32.645 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.645 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.645 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.645 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.023230+0000", 2024-08-17T17:27:32.645 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001918285 2024-08-17T17:27:32.645 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.645 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.645 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.646 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.023758+0000", 2024-08-17T17:27:32.646 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00052815100000000003 2024-08-17T17:27:32.646 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.646 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.646 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.646 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.365114+0000", 2024-08-17T17:27:32.646 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.34135574400000002 2024-08-17T17:27:32.646 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.646 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.646 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.646 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.365169+0000", 2024-08-17T17:27:32.646 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4728000000000002e-05 2024-08-17T17:27:32.647 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.647 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.647 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.647 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.365209+0000", 2024-08-17T17:27:32.647 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0386999999999998e-05 2024-08-17T17:27:32.647 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.647 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.647 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.647 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.647 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.647 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:44843 9.0 9:06a3cb59:::benchmark_data_smithi063_263014_object44842:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:27:32.647 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:08.028819+0000", 2024-08-17T17:27:32.648 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 24.492957344000001, 2024-08-17T17:27:32.648 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33648468500000001, 2024-08-17T17:27:32.648 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.648 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.648 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.648 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:32.648 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:32.648 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 44843 2024-08-17T17:27:32.648 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.648 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.648 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.648 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.648 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.028819+0000", 2024-08-17T17:27:32.649 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.649 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.649 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.649 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.649 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.028819+0000", 2024-08-17T17:27:32.649 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.649 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.649 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.649 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.649 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.028819+0000", 2024-08-17T17:27:32.649 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:32.649 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.649 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.650 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.650 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.028893+0000", 2024-08-17T17:27:32.650 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4518999999999996e-05 2024-08-17T17:27:32.650 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.650 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.650 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.650 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.028908+0000", 2024-08-17T17:27:32.650 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4786e-05 2024-08-17T17:27:32.650 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.650 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.650 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.650 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.028916+0000", 2024-08-17T17:27:32.651 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9699999999999999e-06 2024-08-17T17:27:32.651 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.651 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.651 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.651 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.028966+0000", 2024-08-17T17:27:32.651 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0146999999999999e-05 2024-08-17T17:27:32.651 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.651 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.651 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.651 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.029099+0000", 2024-08-17T17:27:32.651 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013351500000000001 2024-08-17T17:27:32.651 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.651 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.652 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:27:32.652 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.029234+0000", 2024-08-17T17:27:32.652 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013504200000000001 2024-08-17T17:27:32.652 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.652 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.652 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.652 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.033167+0000", 2024-08-17T17:27:32.652 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0039330010000000002 2024-08-17T17:27:32.652 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.652 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.652 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.652 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.033203+0000", 2024-08-17T17:27:32.653 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5176999999999999e-05 2024-08-17T17:27:32.653 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.653 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.653 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.653 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.365249+0000", 2024-08-17T17:27:32.653 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.332046495 2024-08-17T17:27:32.653 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.653 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.653 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.653 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.365281+0000", 2024-08-17T17:27:32.653 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2098999999999999e-05 2024-08-17T17:27:32.653 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.653 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.654 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.654 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.365304+0000", 2024-08-17T17:27:32.654 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2909e-05 2024-08-17T17:27:32.654 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.654 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.654 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.654 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.654 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.654 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:44845 9.0 9:0386fd05:::benchmark_data_smithi063_263014_object44844:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:27:32.654 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:08.029831+0000", 2024-08-17T17:27:32.654 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 24.491946235, 2024-08-17T17:27:32.654 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33563528100000001, 2024-08-17T17:27:32.655 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.655 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.655 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.655 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:32.655 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:32.655 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 44845 2024-08-17T17:27:32.655 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.655 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.655 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.655 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.655 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.029831+0000", 2024-08-17T17:27:32.655 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.655 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.656 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.656 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.656 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.029831+0000", 2024-08-17T17:27:32.656 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.656 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.656 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.656 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.656 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.029829+0000", 2024-08-17T17:27:32.656 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:32.656 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.656 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.656 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.656 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.029950+0000", 2024-08-17T17:27:32.657 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000120789 2024-08-17T17:27:32.657 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.657 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.657 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.657 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.029962+0000", 2024-08-17T17:27:32.657 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2012e-05 2024-08-17T17:27:32.657 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.657 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.657 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.657 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.029970+0000", 2024-08-17T17:27:32.657 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4789999999999999e-06 2024-08-17T17:27:32.657 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.658 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.658 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.658 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.030886+0000", 2024-08-17T17:27:32.658 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00091675600000000002 2024-08-17T17:27:32.658 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.658 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.658 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.658 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.031015+0000", 2024-08-17T17:27:32.658 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012893299999999999 2024-08-17T17:27:32.658 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.658 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.658 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:27:32.659 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.031184+0000", 2024-08-17T17:27:32.659 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000168901 2024-08-17T17:27:32.659 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.659 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.659 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.659 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.033342+0000", 2024-08-17T17:27:32.659 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021578529999999999 2024-08-17T17:27:32.659 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.659 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.659 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.659 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.033832+0000", 2024-08-17T17:27:32.659 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00049035099999999998 2024-08-17T17:27:32.659 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.659 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.660 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.660 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.365344+0000", 2024-08-17T17:27:32.660 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33151195700000002 2024-08-17T17:27:32.660 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.660 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.660 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.660 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.365410+0000", 2024-08-17T17:27:32.660 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5945999999999995e-05 2024-08-17T17:27:32.660 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.660 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.660 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.660 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.365466+0000", 2024-08-17T17:27:32.661 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5677000000000002e-05 2024-08-17T17:27:32.661 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.661 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.661 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.661 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.661 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.661 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:44867 9.0 9:0591ddba:::benchmark_data_smithi063_263014_object44866:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:27:32.661 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:08.042197+0000", 2024-08-17T17:27:32.661 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 24.479579548, 2024-08-17T17:27:32.661 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.323431091, 2024-08-17T17:27:32.661 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.661 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.661 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.661 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:32.662 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:32.662 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 44867 2024-08-17T17:27:32.662 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.662 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.662 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.662 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.662 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.042197+0000", 2024-08-17T17:27:32.662 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.662 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.662 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.662 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.662 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.042197+0000", 2024-08-17T17:27:32.663 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.663 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.663 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.663 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.663 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.042196+0000", 2024-08-17T17:27:32.663 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:32.663 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.663 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.663 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.663 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.042296+0000", 2024-08-17T17:27:32.663 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9947999999999999e-05 2024-08-17T17:27:32.663 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.664 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.664 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.664 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.042312+0000", 2024-08-17T17:27:32.664 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5947000000000002e-05 2024-08-17T17:27:32.664 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.664 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.664 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.664 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.042319+0000", 2024-08-17T17:27:32.664 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1729999999999998e-06 2024-08-17T17:27:32.664 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.664 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.664 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.664 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.042368+0000", 2024-08-17T17:27:32.665 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8578999999999997e-05 2024-08-17T17:27:32.665 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.665 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.665 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.665 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.042481+0000", 2024-08-17T17:27:32.665 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000113291 2024-08-17T17:27:32.665 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.665 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.665 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:27:32.665 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.042606+0000", 2024-08-17T17:27:32.665 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000125062 2024-08-17T17:27:32.665 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.665 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.665 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.666 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.044370+0000", 2024-08-17T17:27:32.666 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0017644220000000001 2024-08-17T17:27:32.666 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.666 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.666 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.666 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.046193+0000", 2024-08-17T17:27:32.666 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0018223530000000001 2024-08-17T17:27:32.666 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.666 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.666 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.666 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.365533+0000", 2024-08-17T17:27:32.666 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.31934083699999999 2024-08-17T17:27:32.667 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.667 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.667 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.667 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.365585+0000", 2024-08-17T17:27:32.667 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1892999999999999e-05 2024-08-17T17:27:32.667 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.667 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.667 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.667 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.365628+0000", 2024-08-17T17:27:32.667 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3009000000000001e-05 2024-08-17T17:27:32.667 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.667 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.667 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.668 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.668 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.668 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:44884 9.0 9:0bbdd76d:::benchmark_data_smithi063_263014_object44883:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:27:32.668 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:08.057454+0000", 2024-08-17T17:27:32.668 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 24.464323277999998, 2024-08-17T17:27:32.668 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.30831374299999997, 2024-08-17T17:27:32.668 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:32.668 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:32.668 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:32.668 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:32.668 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:32.668 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 44884 2024-08-17T17:27:32.668 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.669 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:32.669 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.669 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:32.669 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.057454+0000", 2024-08-17T17:27:32.669 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.669 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.669 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.669 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:32.669 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.057454+0000", 2024-08-17T17:27:32.669 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:32.669 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.669 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.669 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:32.670 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.057452+0000", 2024-08-17T17:27:32.670 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:32.670 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.670 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.670 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:32.670 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.057510+0000", 2024-08-17T17:27:32.670 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7828999999999998e-05 2024-08-17T17:27:32.670 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.670 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.670 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:32.670 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.057521+0000", 2024-08-17T17:27:32.670 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0351e-05 2024-08-17T17:27:32.670 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.671 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.671 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:32.671 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.057526+0000", 2024-08-17T17:27:32.671 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1950000000000002e-06 2024-08-17T17:27:32.671 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.671 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.671 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:32.671 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.057563+0000", 2024-08-17T17:27:32.671 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6971000000000002e-05 2024-08-17T17:27:32.671 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.671 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.671 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:32.671 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.057650+0000", 2024-08-17T17:27:32.672 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6959000000000005e-05 2024-08-17T17:27:32.672 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.672 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.672 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:27:32.672 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.057737+0000", 2024-08-17T17:27:32.672 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6952000000000004e-05 2024-08-17T17:27:32.672 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.672 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.672 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:32.672 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.059022+0000", 2024-08-17T17:27:32.672 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012851550000000001 2024-08-17T17:27:32.672 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.672 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.672 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.673 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.059343+0000", 2024-08-17T17:27:32.673 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00032132599999999998 2024-08-17T17:27:32.673 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.673 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.673 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:32.673 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.365680+0000", 2024-08-17T17:27:32.673 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.30633652900000002 2024-08-17T17:27:32.673 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.673 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.673 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:32.673 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.365723+0000", 2024-08-17T17:27:32.673 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3727999999999999e-05 2024-08-17T17:27:32.673 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:32.674 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:32.674 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:32.674 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.365767+0000", 2024-08-17T17:27:32.674 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3890000000000002e-05 2024-08-17T17:27:32.674 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.674 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.674 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.674 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:32.674 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:32.674 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:27:32.708 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:32.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:32.810 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:32.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:32 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3766220071' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-08-17T17:27:32.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:32 smithi162 ceph-mon[104668]: osdmap e1255: 8 total, 8 up, 8 in 2024-08-17T17:27:32.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.854 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:32.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:32.955 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:32.956 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:27:32.972 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:32 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3766220071' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-08-17T17:27:32.972 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:32 smithi063 ceph-mon[109843]: osdmap e1255: 8 total, 8 up, 8 in 2024-08-17T17:27:32.973 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:32 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3766220071' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2024-08-17T17:27:32.973 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:32 smithi063 ceph-mon[114756]: osdmap e1255: 8 total, 8 up, 8 in 2024-08-17T17:27:32.990 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:32.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:32.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:33.092 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:33.136 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:33.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:33.238 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:33.275 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:33.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:33.377 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:33.415 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:33.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:33.463 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 47 16 79038 79022 105.065 80.9375 0.0164337 0.00951222 2024-08-17T17:27:33.517 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:33.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:27:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:27:33] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.51.0" 2024-08-17T17:27:33.552 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:33.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:33.576 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:27:33.655 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:33.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.701 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:33.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:33.802 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:33.842 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:33.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:33.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:33 smithi162 ceph-mon[104668]: pgmap v1724: 119 pgs: 119 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2024-08-17T17:27:33.944 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:33.996 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:33.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:33.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:33 smithi063 ceph-mon[114756]: pgmap v1724: 119 pgs: 119 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2024-08-17T17:27:34.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:33 smithi063 ceph-mon[109843]: pgmap v1724: 119 pgs: 119 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s 2024-08-17T17:27:34.077 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:34.097 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:34.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.148 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:34.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:34.249 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:34.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.284 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:34.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:34.385 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:34.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.422 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:34.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:34.463 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 48 16 80235 80219 104.434 74.8125 0.0125322 0.00957 2024-08-17T17:27:34.523 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:34.571 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:34.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:34.673 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:34.712 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:34.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:34.814 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:34.851 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:34.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:34.858 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:34.953 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:34.989 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:34.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:34.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:35.091 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:35.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.130 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:35.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:35.231 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:35.264 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:35.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:35.366 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:35.401 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:35.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:35.463 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 49 16 81421 81405 103.816 74.125 0.00905727 0.0096269 2024-08-17T17:27:35.503 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:35.543 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:35.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.544 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:35.636 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:27:35.645 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:35.671 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:35 smithi162 ceph-mon[104668]: pgmap v1725: 119 pgs: 119 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-17T17:27:35.671 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:35 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/587563884' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:27:35.680 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:35.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:35.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:35 smithi063 ceph-mon[114756]: pgmap v1725: 119 pgs: 119 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-17T17:27:35.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:35 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/587563884' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:27:35.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:35 smithi063 ceph-mon[109843]: pgmap v1725: 119 pgs: 119 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-08-17T17:27:35.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:35 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/587563884' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:27:35.782 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:35.830 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:35.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:35.932 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:35.970 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:35.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:35.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:36.072 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:36.128 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:36.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:36.157 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:27:36.157 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:27:36.157 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.157 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:82200 9.d 9:ba1549dd:::benchmark_data_smithi063_263014_object82199:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1256)", 2024-08-17T17:27:36.157 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:36.146185+0000", 2024-08-17T17:27:36.157 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0071643699999999998, 2024-08-17T17:27:36.158 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0071980200000000003, 2024-08-17T17:27:36.158 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:36.158 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:27:36.158 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:36.158 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:36.158 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:36.158 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 82200 2024-08-17T17:27:36.158 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.158 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:36.158 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.158 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:36.159 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146185+0000", 2024-08-17T17:27:36.159 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:36.159 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.159 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.159 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:36.159 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146185+0000", 2024-08-17T17:27:36.159 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:36.159 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.159 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.159 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:36.159 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146183+0000", 2024-08-17T17:27:36.159 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:36.159 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.160 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.160 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:36.160 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146272+0000", 2024-08-17T17:27:36.160 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8416000000000002e-05 2024-08-17T17:27:36.160 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.160 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.160 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:36.160 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146312+0000", 2024-08-17T17:27:36.160 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0227e-05 2024-08-17T17:27:36.160 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.160 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.160 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:36.160 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146320+0000", 2024-08-17T17:27:36.161 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9449999999999993e-06 2024-08-17T17:27:36.161 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.161 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.161 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:36.161 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146497+0000", 2024-08-17T17:27:36.161 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000176875 2024-08-17T17:27:36.161 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.161 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.161 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:36.161 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146647+0000", 2024-08-17T17:27:36.161 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015018799999999999 2024-08-17T17:27:36.161 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.161 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.162 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,5", 2024-08-17T17:27:36.162 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146792+0000", 2024-08-17T17:27:36.162 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000144554 2024-08-17T17:27:36.162 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.162 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.162 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:36.162 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.150867+0000", 2024-08-17T17:27:36.162 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0040758189999999996 2024-08-17T17:27:36.162 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:36.162 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:36.162 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:36.162 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.163 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.163 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:82202 9.3 9:c12e34cc:::benchmark_data_smithi063_263014_object82201:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1256)", 2024-08-17T17:27:36.163 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:36.146349+0000", 2024-08-17T17:27:36.163 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0069997540000000004, 2024-08-17T17:27:36.163 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0071116449999999998, 2024-08-17T17:27:36.163 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:36.163 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:27:36.163 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:36.163 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:36.163 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:36.163 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 82202 2024-08-17T17:27:36.163 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.163 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:36.164 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.164 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:36.164 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146349+0000", 2024-08-17T17:27:36.164 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:36.164 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.164 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.164 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:36.164 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146349+0000", 2024-08-17T17:27:36.164 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:36.164 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.164 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.164 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:36.164 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146349+0000", 2024-08-17T17:27:36.165 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:36.165 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.165 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.165 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:36.165 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146384+0000", 2024-08-17T17:27:36.165 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4919000000000002e-05 2024-08-17T17:27:36.165 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.165 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.165 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:36.165 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146392+0000", 2024-08-17T17:27:36.165 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4810000000000006e-06 2024-08-17T17:27:36.165 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.165 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.165 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:36.166 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146400+0000", 2024-08-17T17:27:36.166 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.1769999999999998e-06 2024-08-17T17:27:36.166 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.166 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.166 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:36.166 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.147812+0000", 2024-08-17T17:27:36.166 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001411315 2024-08-17T17:27:36.166 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.166 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.166 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:36.166 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.147956+0000", 2024-08-17T17:27:36.166 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000144747 2024-08-17T17:27:36.167 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.167 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.167 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:27:36.167 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.148084+0000", 2024-08-17T17:27:36.167 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000127646 2024-08-17T17:27:36.167 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.167 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.167 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:36.167 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.152913+0000", 2024-08-17T17:27:36.167 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0048288619999999997 2024-08-17T17:27:36.167 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:36.167 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:36.167 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:36.168 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.168 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.168 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:82203 9.9 9:98f8b0eb:::benchmark_data_smithi063_263014_object82202:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1256)", 2024-08-17T17:27:36.168 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:36.146414+0000", 2024-08-17T17:27:36.168 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0069355290000000002, 2024-08-17T17:27:36.168 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0071264049999999997, 2024-08-17T17:27:36.168 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:36.168 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:27:36.168 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:36.168 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:36.168 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:36.168 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 82203 2024-08-17T17:27:36.169 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.169 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:36.169 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.169 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:36.169 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146414+0000", 2024-08-17T17:27:36.169 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:36.169 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.169 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.169 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:36.169 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146414+0000", 2024-08-17T17:27:36.169 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:36.169 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.169 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.169 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:36.170 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146413+0000", 2024-08-17T17:27:36.170 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:36.170 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.170 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.170 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:36.170 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146438+0000", 2024-08-17T17:27:36.170 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5340000000000001e-05 2024-08-17T17:27:36.170 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.170 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.170 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:36.170 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146446+0000", 2024-08-17T17:27:36.170 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2350000000000002e-06 2024-08-17T17:27:36.170 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.171 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.171 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:36.171 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146451+0000", 2024-08-17T17:27:36.171 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1050000000000003e-06 2024-08-17T17:27:36.171 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.171 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.171 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:36.171 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146648+0000", 2024-08-17T17:27:36.171 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00019766599999999999 2024-08-17T17:27:36.172 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.172 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.172 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:36.172 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146779+0000", 2024-08-17T17:27:36.172 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000130099 2024-08-17T17:27:36.172 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.172 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.172 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:27:36.172 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.146939+0000", 2024-08-17T17:27:36.172 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000160896 2024-08-17T17:27:36.172 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.173 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.173 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:36.173 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.150824+0000", 2024-08-17T17:27:36.173 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0038846710000000001 2024-08-17T17:27:36.173 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:36.173 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:36.173 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:36.173 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.173 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.173 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.45521.0:82201 9.1 e1256/1247 9:8bf25770:::benchmark_data_smithi063_263014_object82200:head v 1256'5120, mlcod=1256'5118)", 2024-08-17T17:27:36.173 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:36.149978+0000", 2024-08-17T17:27:36.173 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.003370657, 2024-08-17T17:27:36.174 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003637216, 2024-08-17T17:27:36.174 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:36.174 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:27:36.174 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:36.174 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.174 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:36.174 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.149978+0000", 2024-08-17T17:27:36.174 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:36.174 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.175 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.175 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:36.175 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.149978+0000", 2024-08-17T17:27:36.175 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:36.175 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.175 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.175 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:36.175 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.149977+0000", 2024-08-17T17:27:36.175 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:36.175 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.175 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.175 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:36.175 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.150016+0000", 2024-08-17T17:27:36.176 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8884000000000002e-05 2024-08-17T17:27:36.176 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.176 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.176 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:36.176 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.150060+0000", 2024-08-17T17:27:36.176 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4147999999999998e-05 2024-08-17T17:27:36.176 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.176 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.176 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:36.176 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.150070+0000", 2024-08-17T17:27:36.176 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7909999999999999e-06 2024-08-17T17:27:36.177 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.177 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.177 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:36.177 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.150105+0000", 2024-08-17T17:27:36.177 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4477999999999998e-05 2024-08-17T17:27:36.177 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.177 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.177 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:36.177 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.150125+0000", 2024-08-17T17:27:36.177 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0452999999999999e-05 2024-08-17T17:27:36.178 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:36.178 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:36.178 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:36.178 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.178 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.178 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:82205 9.9 9:9d520c5d:::benchmark_data_smithi063_263014_object82204:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1256)", 2024-08-17T17:27:36.178 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:36.151016+0000", 2024-08-17T17:27:36.178 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0023326940000000002, 2024-08-17T17:27:36.178 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026574369999999999, 2024-08-17T17:27:36.178 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:36.179 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:27:36.179 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:36.179 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:36.179 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:36.179 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 82205 2024-08-17T17:27:36.179 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.179 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:36.179 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.179 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:36.179 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.151016+0000", 2024-08-17T17:27:36.180 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:36.180 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.180 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.180 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:36.180 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.151016+0000", 2024-08-17T17:27:36.180 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:36.180 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.180 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.180 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:36.180 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.151014+0000", 2024-08-17T17:27:36.181 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999976 2024-08-17T17:27:36.181 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.181 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.181 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:36.181 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.151052+0000", 2024-08-17T17:27:36.181 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7474000000000001e-05 2024-08-17T17:27:36.181 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.181 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.181 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:36.181 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.151067+0000", 2024-08-17T17:27:36.182 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5603999999999998e-05 2024-08-17T17:27:36.182 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.182 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.182 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:36.182 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.151075+0000", 2024-08-17T17:27:36.182 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.8220000000000007e-06 2024-08-17T17:27:36.182 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.182 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.182 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:36.182 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.151178+0000", 2024-08-17T17:27:36.183 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000103071 2024-08-17T17:27:36.183 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.183 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.183 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:36.183 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.151329+0000", 2024-08-17T17:27:36.183 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000150953 2024-08-17T17:27:36.183 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.183 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.183 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:27:36.184 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.151468+0000", 2024-08-17T17:27:36.184 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000139219 2024-08-17T17:27:36.184 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:36.184 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:36.184 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:36.184 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.184 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.184 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:82207 9.9 9.a548c539 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1256)", 2024-08-17T17:27:36.185 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:36.151085+0000", 2024-08-17T17:27:36.185 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0022636980000000002, 2024-08-17T17:27:36.185 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026553879999999998, 2024-08-17T17:27:36.185 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:36.185 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "queued for pg", 2024-08-17T17:27:36.185 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:36.185 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:36.185 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:36.185 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 82207 2024-08-17T17:27:36.185 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.185 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:36.185 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.186 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:36.186 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.151085+0000", 2024-08-17T17:27:36.186 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:36.186 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.186 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.186 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:36.186 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.151085+0000", 2024-08-17T17:27:36.186 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:36.186 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.186 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.186 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:36.186 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.151085+0000", 2024-08-17T17:27:36.186 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:36.187 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.187 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.187 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:36.187 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.151111+0000", 2024-08-17T17:27:36.187 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5780999999999999e-05 2024-08-17T17:27:36.187 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.187 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.187 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:36.187 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.151118+0000", 2024-08-17T17:27:36.187 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.99e-06 2024-08-17T17:27:36.187 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.187 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.188 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:36.188 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.151122+0000", 2024-08-17T17:27:36.188 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6059999999999996e-06 2024-08-17T17:27:36.188 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:36.188 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:36.188 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:36.188 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.188 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.188 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:82208 9.3 9:cde8ccf8:::benchmark_data_smithi063_263014_object82207:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1256)", 2024-08-17T17:27:36.188 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:36.153126+0000", 2024-08-17T17:27:36.188 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.000223487, 2024-08-17T17:27:36.188 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00067217700000000004, 2024-08-17T17:27:36.188 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:36.188 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:27:36.189 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:36.189 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:36.189 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:36.189 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 82208 2024-08-17T17:27:36.189 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.189 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:36.189 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.189 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:36.189 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.153126+0000", 2024-08-17T17:27:36.189 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:36.190 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.190 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.190 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:36.190 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.153126+0000", 2024-08-17T17:27:36.190 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:36.190 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.190 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.190 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:36.190 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.153124+0000", 2024-08-17T17:27:36.190 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:36.190 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.190 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.191 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:36.191 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.153266+0000", 2024-08-17T17:27:36.191 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014178199999999999 2024-08-17T17:27:36.191 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.191 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.191 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:36.191 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.153281+0000", 2024-08-17T17:27:36.191 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5238e-05 2024-08-17T17:27:36.191 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.191 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.191 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:36.191 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.153288+0000", 2024-08-17T17:27:36.192 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5320000000000003e-06 2024-08-17T17:27:36.192 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.192 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.192 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:36.192 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.153355+0000", 2024-08-17T17:27:36.192 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6812999999999999e-05 2024-08-17T17:27:36.192 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.192 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.192 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:36.192 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.153515+0000", 2024-08-17T17:27:36.193 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016021000000000001 2024-08-17T17:27:36.193 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:36.193 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:36.193 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:27:36.193 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:36.153666+0000", 2024-08-17T17:27:36.193 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015102900000000001 2024-08-17T17:27:36.193 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:36.193 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:36.194 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:36.194 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:36.194 INFO:teuthology.orchestra.run.smithi162.stdout: ], 2024-08-17T17:27:36.194 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 7 2024-08-17T17:27:36.194 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:27:36.231 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:36.276 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:36.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.277 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:36.378 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:36.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.423 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:36.423 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:36.463 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 50 16 82551 82535 103.151 70.625 0.0201501 0.00968889 2024-08-17T17:27:36.524 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:36.591 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:36.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:36.693 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:36.730 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:36.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:36.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:36 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:36.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:36 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/587563884' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:27:36.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:36 smithi063 ceph-mon[114756]: osdmap e1256: 8 total, 8 up, 8 in 2024-08-17T17:27:36.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:36 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:27:36.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:36 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:36.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:36 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/587563884' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:27:36.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:36 smithi063 ceph-mon[109843]: osdmap e1256: 8 total, 8 up, 8 in 2024-08-17T17:27:36.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:36 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:27:36.832 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:36.872 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:36.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:36.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:36.973 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:37.008 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:37.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:36 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:36 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/587563884' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:27:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:36 smithi162 ceph-mon[104668]: osdmap e1256: 8 total, 8 up, 8 in 2024-08-17T17:27:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:36 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:27:37.109 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:37.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.138 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:37.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:37.239 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:37.279 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:37.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:37.380 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:37.414 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:37.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:37.464 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 51 16 83641 83625 102.464 68.125 0.0183223 0.00975446 2024-08-17T17:27:37.516 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:37.553 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:37.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:37.565 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:27:37.640 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:27:37.640 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:27:37.655 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:37.675 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:27:37.675 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:27:37.705 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:27:37.705 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:27:37.706 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:37 smithi063 ceph-mon[109843]: pgmap v1727: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2024-08-17T17:27:37.706 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:37 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/433243155' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:37.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:37 smithi063 ceph-mon[114756]: pgmap v1727: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2024-08-17T17:27:37.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:37 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/433243155' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:37.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.740 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:37.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:37.745 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:27:37.745 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:27:37.787 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:27:37.788 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:27:37.818 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:27:37.819 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:27:37.841 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:37.845 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:27:37.845 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:27:37.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:37 smithi162 ceph-mon[104668]: pgmap v1727: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 937 op/s 2024-08-17T17:27:37.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:37 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/433243155' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:37.883 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:27:37.883 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:27:37.899 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:37.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:37.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:38.000 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:38.037 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:38.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:38.139 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:38.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.184 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:38.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:38.286 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:38.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.334 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:38.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:38.435 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:38.464 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 52 16 84695 84679 101.761 65.875 0.00993792 0.00982154 2024-08-17T17:27:38.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.477 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:38.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:38.517 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:27:38.578 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:38.619 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:38.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:38.686 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:27:38.686 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:27:38.686 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-08-17T17:27:38.686 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:27:38.686 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:27:38.687 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-08-17T17:27:38.722 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:38.744 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:38.751 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:38.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:38.852 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:38.896 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:38.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:38.997 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:39.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.039 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:39.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:39.140 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:39.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.177 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:39.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:39.278 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:39.313 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:39.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:39.415 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:39.427 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:39.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.464 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:39.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:39.465 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 53 16 85711 85695 101.039 63.5 0.0197388 0.00989197 2024-08-17T17:27:39.565 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:39.593 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:39.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:39.694 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:39.757 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:39.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:39.862 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:39.884 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:39.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:39.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:39.889 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:39 smithi063 ceph-mon[114756]: pgmap v1728: 119 pgs: 119 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:27:39.889 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:39 smithi063 ceph-mon[109843]: pgmap v1728: 119 pgs: 119 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:27:39.986 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:40.009 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:40.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:40.032 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:39 smithi162 ceph-mon[104668]: pgmap v1728: 119 pgs: 119 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-08-17T17:27:40.110 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:40.146 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:40.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:40.248 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:40.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.278 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:40.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:40.379 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:40.412 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:40.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:40.464 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 54 16 86435 86419 100.006 45.25 0.00642762 0.009988 2024-08-17T17:27:40.515 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:40.549 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:40.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:40.650 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:40.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.689 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:40.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:40.790 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:40.824 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:40.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:40.848 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:27:40.926 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:40.974 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:40.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:40.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:41.075 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:41.103 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:41.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:41.205 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:41.239 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:41.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:41.341 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:41.367 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:41.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:41.410 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:41.465 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 55 16 87396 87380 99.2763 60.0625 0.0119261 0.0100674 2024-08-17T17:27:41.469 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:41.503 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:41.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:41.605 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:41.638 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:41.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:41.740 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:41.763 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:41.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:41.833 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:41 smithi063 ceph-mon[114756]: pgmap v1729: 119 pgs: 119 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:27:41.833 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:41 smithi063 ceph-mon[109843]: pgmap v1729: 119 pgs: 119 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:27:41.865 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:41.911 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:41.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:41.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:41.958 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:41 smithi162 ceph-mon[104668]: pgmap v1729: 119 pgs: 119 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-08-17T17:27:42.013 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:42.039 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:42.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:42.141 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:42.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.176 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:42.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:42.277 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:42.329 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:42.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:42.431 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:42.462 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:42.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:42.465 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 56 16 88484 88468 98.7177 68 0.021732 0.0101248 2024-08-17T17:27:42.564 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:42.600 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:42.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:42.682 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:27:42.702 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:42.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.740 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:42.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:42.841 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:42.874 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:42.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:42.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:42.976 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:43.013 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:42 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2156891360' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:43.014 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:43.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:42 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2156891360' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:43.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:42 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2156891360' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:27:43.116 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:43.148 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:43.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:43.250 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:43.306 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:43.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:43.408 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:43.446 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:43.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:43.465 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 57 16 89667 89651 98.2829 73.9375 0.0216185 0.0101692 2024-08-17T17:27:43.517 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:27:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:27:43] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.51.0" 2024-08-17T17:27:43.548 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:43.588 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:43.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:43.677 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:43.690 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:43.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.736 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:43.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:43.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:43 smithi063 ceph-mon[109843]: pgmap v1730: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-08-17T17:27:43.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:43 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:27:43.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:43 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:27:43.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:43 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2156891360' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:43.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:43 smithi063 ceph-mon[109843]: osdmap e1257: 8 total, 8 up, 8 in 2024-08-17T17:27:43.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:43 smithi063 ceph-mon[114756]: pgmap v1730: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-08-17T17:27:43.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:43 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:27:43.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:43 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:27:43.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:43 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2156891360' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:43.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:43 smithi063 ceph-mon[114756]: osdmap e1257: 8 total, 8 up, 8 in 2024-08-17T17:27:43.837 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:43.883 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:43.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:43.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:43.985 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:44.016 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:43 smithi162 ceph-mon[104668]: pgmap v1730: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-08-17T17:27:44.016 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:43 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:27:44.016 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:43 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:27:44.017 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:43 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2156891360' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:27:44.017 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:43 smithi162 ceph-mon[104668]: osdmap e1257: 8 total, 8 up, 8 in 2024-08-17T17:27:44.034 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:44.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:44.137 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:44.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.182 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:44.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:44.283 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:44.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.327 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:44.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:44.428 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:44.466 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 58 16 90763 90747 97.7693 68.5 0.00421231 0.0102227 2024-08-17T17:27:44.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.470 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:44.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:44.571 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:44.619 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:44.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:44.721 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:44.762 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:44.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:44.864 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:44.905 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:44.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:44.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:45.007 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:45.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.065 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:45.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:45.166 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:45.216 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:45.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:45.318 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:45.355 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:45.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:45.458 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:45.466 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 59 16 91864 91848 97.2785 68.8125 0.0113001 0.0102743 2024-08-17T17:27:45.502 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:45.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:45.555 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:27:45.604 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:45.606 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:27:45.606 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:27:45.641 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:45.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:45.742 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:45.770 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:45.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:45.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:45 smithi063 ceph-mon[109843]: pgmap v1732: 119 pgs: 119 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-08-17T17:27:45.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:45 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1123554793' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:45.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:45 smithi063 ceph-mon[114756]: pgmap v1732: 119 pgs: 119 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-08-17T17:27:45.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:45 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1123554793' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:45.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:45 smithi162 ceph-mon[104668]: pgmap v1732: 119 pgs: 119 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2024-08-17T17:27:45.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:45 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1123554793' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:45.872 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:45.914 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:45.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:45.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:45.915 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:27:45.915 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:27:45.916 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:27:45.916 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:27:45.916 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:27:45.916 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:27:45.916 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:27:45.916 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:27:45.916 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:27:45.916 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:27:45.916 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:27:45.916 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:27:45.916 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:27:45.917 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:27:45.917 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:27:45.917 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:27:46.016 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:46.051 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:46.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:46.153 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:46.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.206 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:46.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:46.307 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:46.327 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:27:46.355 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:46.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:46.461 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:46.466 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:27:46.466241+0000 min lat: 0.00127496 max lat: 0.34753 avg lat: 0.0103248 2024-08-17T17:27:46.466 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:27:46.466 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 60 16 92965 92949 96.804 68.8125 0.0203652 0.0103248 2024-08-17T17:27:46.507 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:46.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:46.609 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:46.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.661 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:46.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:46.762 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:46.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.801 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:46.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:46.902 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:46.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:46.944 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:46.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:47.045 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:47.083 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:47.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.091 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:47.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:47.192 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:47.246 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:47.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:47.348 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:47.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.384 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:47.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:47.467 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 61 16 94041 94025 96.3194 67.25 0.013097 0.010377 2024-08-17T17:27:47.484 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:47.520 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:47.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:47.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:47.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:47.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:47 smithi063 ceph-mon[114756]: pgmap v1733: 119 pgs: 119 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:27:47.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:47.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:47.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:47 smithi063 ceph-mon[109843]: pgmap v1733: 119 pgs: 119 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:27:47.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:47.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:27:47.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:47 smithi162 ceph-mon[104668]: pgmap v1733: 119 pgs: 119 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-08-17T17:27:47.622 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:47.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.670 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:47.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:47.707 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:27:47.707 INFO:teuthology.orchestra.run.smithi162.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:27:47.707 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:27:47.771 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:47.816 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:47.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:47.918 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:47.954 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:47.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:47.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:48.056 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:48.097 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:48.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:48.199 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:48.232 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:48.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:48.334 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:48.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.369 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:48.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:48.466 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 62 16 95186 95170 95.92 71.5625 0.00686716 0.0104201 2024-08-17T17:27:48.470 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:48.511 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:48.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:48.552 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:27:48.613 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:48.651 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:48.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:48.753 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:48.811 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:48.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:48.913 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:48.946 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:48.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:48.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:49.024 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:27:49.048 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:49.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.138 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:49.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:49.239 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:49.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.275 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:49.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:49.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:49.433 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:49.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:49.466 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 63 16 96226 96210 95.4292 65 0.0188433 0.0104739 2024-08-17T17:27:49.535 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:49.564 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:49.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:49 smithi162 ceph-mon[104668]: pgmap v1734: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:27:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:49 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:49.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:49 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:49.666 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:49.737 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:49.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:49.747 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:49.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:49 smithi063 ceph-mon[109843]: pgmap v1734: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:27:49.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:49 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:49.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:49 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:49.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:49 smithi063 ceph-mon[114756]: pgmap v1734: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:27:49.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:49 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:49.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:49 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:49.838 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:49.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:49.890 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:49.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:49.991 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:50.039 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:50.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:50.141 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:50.178 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:50.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:50.279 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:50.332 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:50.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:50.434 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:50.459 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:27:50.466 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 64 16 97219 97203 94.9078 62.0625 0.017797 0.01053 2024-08-17T17:27:50.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.482 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:50.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:50.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:50 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3707065764' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:27:50.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:50 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3707065764' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:27:50.584 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:50.619 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:50.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:50.721 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:50.755 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:50.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:50.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:50 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3707065764' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:27:50.857 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:50.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:50.893 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:50.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:50.994 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:51.044 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:51.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:51.146 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:51.190 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:51.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:51.291 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:51.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.333 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:51.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:51.434 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:51.467 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 65 16 98300 98284 94.4867 67.5625 0.0102376 0.0105782 2024-08-17T17:27:51.468 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:51.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:51.570 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:51.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:51 smithi162 ceph-mon[104668]: pgmap v1735: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:27:51.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:51 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:51.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:51 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3707065764' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:27:51.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:51 smithi162 ceph-mon[104668]: osdmap e1258: 8 total, 8 up, 8 in 2024-08-17T17:27:51.611 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:51.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:51.713 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:51.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.750 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:51.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:51.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:51 smithi063 ceph-mon[114756]: pgmap v1735: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:27:51.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:51 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:51.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:51 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3707065764' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:27:51.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:51 smithi063 ceph-mon[114756]: osdmap e1258: 8 total, 8 up, 8 in 2024-08-17T17:27:51.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:51 smithi063 ceph-mon[109843]: pgmap v1735: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-08-17T17:27:51.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:51 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:27:51.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:51 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3707065764' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:27:51.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:51 smithi063 ceph-mon[109843]: osdmap e1258: 8 total, 8 up, 8 in 2024-08-17T17:27:51.851 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:51.891 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:51.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:51.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:51.993 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:52.034 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:52.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:52.135 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:52.165 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:52.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:52.186 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:27:52.186 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:27:52.186 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:27:52.186 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:27:52.187 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.187 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65731 7.6 7:6df89fef:::benchmark_data_smithi063_196002_object65730:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1142)", 2024-08-17T17:27:52.187 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.520165+0000", 2024-08-17T17:27:52.187 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 561.66040513400003, 2024-08-17T17:27:52.187 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5085655259999999, 2024-08-17T17:27:52.187 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.187 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:27:52.187 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.187 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:52.187 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:52.187 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65731 2024-08-17T17:27:52.187 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.187 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.187 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.188 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.188 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520165+0000", 2024-08-17T17:27:52.188 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.188 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.188 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.188 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.188 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520165+0000", 2024-08-17T17:27:52.188 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.188 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.188 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.188 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.188 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520161+0000", 2024-08-17T17:27:52.188 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999957 2024-08-17T17:27:52.189 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.189 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.189 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.189 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520732+0000", 2024-08-17T17:27:52.189 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00057097799999999998 2024-08-17T17:27:52.189 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.189 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.189 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.189 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520747+0000", 2024-08-17T17:27:52.189 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5429000000000001e-05 2024-08-17T17:27:52.189 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.189 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.189 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.190 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520755+0000", 2024-08-17T17:27:52.190 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.8019999999999992e-06 2024-08-17T17:27:52.190 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.190 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.190 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.190 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520822+0000", 2024-08-17T17:27:52.190 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6693000000000007e-05 2024-08-17T17:27:52.190 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.190 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.190 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.190 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.520994+0000", 2024-08-17T17:27:52.190 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017211400000000001 2024-08-17T17:27:52.191 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.191 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.191 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:27:52.191 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.521137+0000", 2024-08-17T17:27:52.191 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014272500000000001 2024-08-17T17:27:52.191 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.191 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.191 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.191 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.526213+0000", 2024-08-17T17:27:52.191 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0050766630000000004 2024-08-17T17:27:52.191 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.191 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.191 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.192 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.540120+0000", 2024-08-17T17:27:52.192 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.013906114000000001 2024-08-17T17:27:52.192 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.192 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.192 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.192 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.540132+0000", 2024-08-17T17:27:52.192 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2480000000000001e-05 2024-08-17T17:27:52.192 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.192 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.192 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.192 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.533968+0000", 2024-08-17T17:27:52.192 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99383588099999998 2024-08-17T17:27:52.192 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.193 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.193 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.193 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.533982+0000", 2024-08-17T17:27:52.193 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4029000000000001e-05 2024-08-17T17:27:52.193 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.193 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.193 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.193 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.028588+0000", 2024-08-17T17:27:52.193 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.494605636 2024-08-17T17:27:52.193 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.193 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.193 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.193 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.028731+0000", 2024-08-17T17:27:52.194 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014346399999999999 2024-08-17T17:27:52.194 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.194 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.194 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.194 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.194 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.194 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65739 7.5 7:a65210cc:::benchmark_data_smithi063_196002_object65738:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:27:52.194 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.535601+0000", 2024-08-17T17:27:52.194 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 561.64496933800001, 2024-08-17T17:27:52.194 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4809891020000001, 2024-08-17T17:27:52.194 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.194 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.194 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.195 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:52.195 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:52.195 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65739 2024-08-17T17:27:52.195 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.195 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.195 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.195 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.195 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535601+0000", 2024-08-17T17:27:52.195 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.195 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.195 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.195 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.195 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535601+0000", 2024-08-17T17:27:52.196 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.196 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.196 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.196 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.196 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535600+0000", 2024-08-17T17:27:52.196 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:27:52.196 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.196 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.196 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.196 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.535635+0000", 2024-08-17T17:27:52.196 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5670000000000002e-05 2024-08-17T17:27:52.196 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.196 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.196 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.197 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.536766+0000", 2024-08-17T17:27:52.197 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001130818 2024-08-17T17:27:52.197 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.197 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.197 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.197 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.536777+0000", 2024-08-17T17:27:52.197 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0709e-05 2024-08-17T17:27:52.197 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.197 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.197 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.197 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.538938+0000", 2024-08-17T17:27:52.197 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021608600000000001 2024-08-17T17:27:52.197 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.197 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.198 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.198 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.538947+0000", 2024-08-17T17:27:52.198 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3510000000000008e-06 2024-08-17T17:27:52.198 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.198 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.198 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.198 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.530306+0000", 2024-08-17T17:27:52.198 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99135938499999998 2024-08-17T17:27:52.198 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.198 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.198 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.198 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.530319+0000", 2024-08-17T17:27:52.198 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2928999999999999e-05 2024-08-17T17:27:52.199 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.199 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.199 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.199 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:32.999604+0000", 2024-08-17T17:27:52.199 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.46928456 2024-08-17T17:27:52.199 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.199 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.199 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.199 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:32.999714+0000", 2024-08-17T17:27:52.199 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000110207 2024-08-17T17:27:52.199 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.199 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.199 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:27:52.200 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:32.999808+0000", 2024-08-17T17:27:52.200 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4368000000000002e-05 2024-08-17T17:27:52.200 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.200 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.200 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.200 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.003364+0000", 2024-08-17T17:27:52.200 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0035553030000000001 2024-08-17T17:27:52.200 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.200 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.200 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.200 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.003393+0000", 2024-08-17T17:27:52.200 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9266999999999999e-05 2024-08-17T17:27:52.200 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.200 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.201 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.201 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.014163+0000", 2024-08-17T17:27:52.201 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.01077003 2024-08-17T17:27:52.201 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.201 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.201 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.201 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.014211+0000", 2024-08-17T17:27:52.201 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7732999999999999e-05 2024-08-17T17:27:52.201 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.201 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.201 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.201 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.016590+0000", 2024-08-17T17:27:52.201 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023796709999999999 2024-08-17T17:27:52.202 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.202 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.202 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.202 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.202 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.202 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65747 7.6 7:6fcb91c8:::benchmark_data_smithi063_196002_object65746:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:27:52.202 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.544342+0000", 2024-08-17T17:27:52.202 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 561.63622877, 2024-08-17T17:27:52.202 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5295821300000001, 2024-08-17T17:27:52.202 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.202 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.202 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.202 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:52.203 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:52.203 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65747 2024-08-17T17:27:52.203 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.203 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.203 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.203 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.203 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.544342+0000", 2024-08-17T17:27:52.203 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.203 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.203 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.203 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.203 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.544342+0000", 2024-08-17T17:27:52.203 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.203 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.204 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.204 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.204 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.544340+0000", 2024-08-17T17:27:52.204 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:27:52.204 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.204 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.204 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.204 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.544384+0000", 2024-08-17T17:27:52.204 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3909999999999998e-05 2024-08-17T17:27:52.204 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.204 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.204 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.205 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.544395+0000", 2024-08-17T17:27:52.205 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1348e-05 2024-08-17T17:27:52.205 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.205 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.205 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.205 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.544402+0000", 2024-08-17T17:27:52.205 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7240000000000003e-06 2024-08-17T17:27:52.205 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.205 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.205 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.205 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.544616+0000", 2024-08-17T17:27:52.205 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000214045 2024-08-17T17:27:52.205 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.206 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.206 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.206 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.544725+0000", 2024-08-17T17:27:52.206 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000108578 2024-08-17T17:27:52.206 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.206 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.206 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.206 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.534010+0000", 2024-08-17T17:27:52.206 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98928472099999998 2024-08-17T17:27:52.206 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.206 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.206 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.206 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.534020+0000", 2024-08-17T17:27:52.207 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0267e-05 2024-08-17T17:27:52.207 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.207 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.207 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.207 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.035317+0000", 2024-08-17T17:27:52.207 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.501297433 2024-08-17T17:27:52.207 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.207 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.207 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.207 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.035480+0000", 2024-08-17T17:27:52.207 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016316900000000001 2024-08-17T17:27:52.207 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.207 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.208 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:27:52.208 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.035715+0000", 2024-08-17T17:27:52.208 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00023457299999999999 2024-08-17T17:27:52.208 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.208 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.208 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.208 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.044875+0000", 2024-08-17T17:27:52.208 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0091599430000000003 2024-08-17T17:27:52.208 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.208 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.208 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.208 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.044884+0000", 2024-08-17T17:27:52.208 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4739999999999994e-06 2024-08-17T17:27:52.209 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.209 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.209 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.209 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.071002+0000", 2024-08-17T17:27:52.209 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.026117838000000001 2024-08-17T17:27:52.209 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.209 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.209 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.209 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.071067+0000", 2024-08-17T17:27:52.209 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4300000000000004e-05 2024-08-17T17:27:52.209 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.209 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.209 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.209 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.073924+0000", 2024-08-17T17:27:52.210 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002857428 2024-08-17T17:27:52.210 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.210 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.210 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.210 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.210 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.210 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65755 7.6 7:619e2726:::benchmark_data_smithi063_196002_object65754:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:27:52.210 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.550248+0000", 2024-08-17T17:27:52.210 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 561.63032244600004, 2024-08-17T17:27:52.210 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.526426533, 2024-08-17T17:27:52.210 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.210 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.210 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.211 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:52.211 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:52.211 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65755 2024-08-17T17:27:52.211 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.211 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.211 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.211 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.211 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.550248+0000", 2024-08-17T17:27:52.211 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.211 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.211 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.211 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.211 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.550248+0000", 2024-08-17T17:27:52.211 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.212 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.212 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.212 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.212 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.550244+0000", 2024-08-17T17:27:52.212 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999952 2024-08-17T17:27:52.212 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.212 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.212 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.212 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.550283+0000", 2024-08-17T17:27:52.212 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9468999999999998e-05 2024-08-17T17:27:52.212 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.212 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.212 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.213 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.550299+0000", 2024-08-17T17:27:52.213 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6243e-05 2024-08-17T17:27:52.213 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.213 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.213 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.213 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.550306+0000", 2024-08-17T17:27:52.213 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1879999999999996e-06 2024-08-17T17:27:52.213 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.213 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.213 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.213 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.550371+0000", 2024-08-17T17:27:52.213 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4268000000000002e-05 2024-08-17T17:27:52.213 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.214 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.214 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.214 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.550382+0000", 2024-08-17T17:27:52.214 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1517e-05 2024-08-17T17:27:52.214 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.214 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.214 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.214 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.534042+0000", 2024-08-17T17:27:52.214 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98365983999999995 2024-08-17T17:27:52.214 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.214 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.214 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.214 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.534051+0000", 2024-08-17T17:27:52.215 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3500000000000003e-06 2024-08-17T17:27:52.215 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.215 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.215 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.216 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.039052+0000", 2024-08-17T17:27:52.216 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5050007320000001 2024-08-17T17:27:52.216 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.216 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.216 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.216 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.039380+0000", 2024-08-17T17:27:52.216 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00032762500000000001 2024-08-17T17:27:52.216 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.216 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.216 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:27:52.216 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.039709+0000", 2024-08-17T17:27:52.217 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00032969800000000001 2024-08-17T17:27:52.217 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.217 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.217 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.217 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.044913+0000", 2024-08-17T17:27:52.217 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0052034029999999997 2024-08-17T17:27:52.217 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.217 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.217 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.217 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.047303+0000", 2024-08-17T17:27:52.217 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023902250000000002 2024-08-17T17:27:52.217 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.217 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.217 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.218 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.073979+0000", 2024-08-17T17:27:52.218 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.026676060000000001 2024-08-17T17:27:52.218 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.218 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.218 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.218 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.074071+0000", 2024-08-17T17:27:52.218 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2003000000000007e-05 2024-08-17T17:27:52.218 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.218 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.218 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.218 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.076675+0000", 2024-08-17T17:27:52.218 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026035289999999998 2024-08-17T17:27:52.218 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.219 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.219 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.219 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.219 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.219 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65758 7.6 7:63955631:::benchmark_data_smithi063_196002_object65757:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:27:52.219 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.552011+0000", 2024-08-17T17:27:52.219 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 561.62855986199997, 2024-08-17T17:27:52.219 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5262674000000001, 2024-08-17T17:27:52.219 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.219 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.219 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.219 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:52.220 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:52.220 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65758 2024-08-17T17:27:52.220 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.220 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.220 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.220 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.220 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.552011+0000", 2024-08-17T17:27:52.220 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.220 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.220 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.220 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.220 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.552011+0000", 2024-08-17T17:27:52.220 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.220 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.221 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.221 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.221 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.552010+0000", 2024-08-17T17:27:52.221 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:52.221 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.221 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.221 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.221 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.552077+0000", 2024-08-17T17:27:52.221 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6804000000000006e-05 2024-08-17T17:27:52.221 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.221 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.221 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.221 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.552088+0000", 2024-08-17T17:27:52.222 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.097e-05 2024-08-17T17:27:52.222 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.222 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.222 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.222 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.552095+0000", 2024-08-17T17:27:52.222 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2540000000000003e-06 2024-08-17T17:27:52.222 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.222 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.222 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.222 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.552132+0000", 2024-08-17T17:27:52.222 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6541000000000001e-05 2024-08-17T17:27:52.222 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.223 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.223 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.223 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.552139+0000", 2024-08-17T17:27:52.223 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6320000000000008e-06 2024-08-17T17:27:52.223 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.223 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.223 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.223 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.534075+0000", 2024-08-17T17:27:52.223 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98193567000000004 2024-08-17T17:27:52.223 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.223 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.223 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.223 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.534084+0000", 2024-08-17T17:27:52.224 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2990000000000002e-06 2024-08-17T17:27:52.224 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.224 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.224 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.224 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.042458+0000", 2024-08-17T17:27:52.224 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.508374101 2024-08-17T17:27:52.224 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.224 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.224 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.224 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.042616+0000", 2024-08-17T17:27:52.224 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015792099999999999 2024-08-17T17:27:52.224 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.224 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.224 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:27:52.225 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.042783+0000", 2024-08-17T17:27:52.225 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016671399999999999 2024-08-17T17:27:52.225 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.225 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.225 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.225 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.047313+0000", 2024-08-17T17:27:52.225 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.004529918 2024-08-17T17:27:52.225 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.225 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.225 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.225 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.049839+0000", 2024-08-17T17:27:52.225 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0025265890000000001 2024-08-17T17:27:52.225 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.226 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.226 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.226 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.076725+0000", 2024-08-17T17:27:52.226 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.026885639999999999 2024-08-17T17:27:52.226 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.226 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.226 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.226 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.076765+0000", 2024-08-17T17:27:52.226 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9879999999999998e-05 2024-08-17T17:27:52.226 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.226 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.226 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.226 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.078278+0000", 2024-08-17T17:27:52.227 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015132539999999999 2024-08-17T17:27:52.227 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.227 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.227 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.227 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.227 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.227 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:65760 7.6 7:64dd68dd:::benchmark_data_smithi063_196002_object65759:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1143)", 2024-08-17T17:27:52.227 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:18:30.555542+0000", 2024-08-17T17:27:52.227 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 561.62502881199998, 2024-08-17T17:27:52.227 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5240997699999999, 2024-08-17T17:27:52.227 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.227 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.227 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.228 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:52.228 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:52.228 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 65760 2024-08-17T17:27:52.228 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.228 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.228 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.228 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.228 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.555542+0000", 2024-08-17T17:27:52.228 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.228 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.228 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.228 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.229 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.555542+0000", 2024-08-17T17:27:52.229 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.229 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.229 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.229 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.229 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.555541+0000", 2024-08-17T17:27:52.229 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:52.229 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.229 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.229 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.229 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.555598+0000", 2024-08-17T17:27:52.229 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7515000000000001e-05 2024-08-17T17:27:52.229 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.229 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.230 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.230 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.555609+0000", 2024-08-17T17:27:52.230 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0518e-05 2024-08-17T17:27:52.230 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.230 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.230 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.230 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.555616+0000", 2024-08-17T17:27:52.230 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2579999999999998e-06 2024-08-17T17:27:52.230 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.230 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.230 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.230 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.555655+0000", 2024-08-17T17:27:52.231 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8756e-05 2024-08-17T17:27:52.231 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.231 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.231 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.231 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:30.555663+0000", 2024-08-17T17:27:52.231 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9419999999999995e-06 2024-08-17T17:27:52.231 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.231 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.231 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.231 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.534107+0000", 2024-08-17T17:27:52.231 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97844418799999999 2024-08-17T17:27:52.231 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.231 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.232 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.232 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:31.534117+0000", 2024-08-17T17:27:52.232 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.6809999999999993e-06 2024-08-17T17:27:52.232 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.232 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.232 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.232 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.044933+0000", 2024-08-17T17:27:52.232 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.510816175 2024-08-17T17:27:52.232 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.232 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.232 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.232 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.045036+0000", 2024-08-17T17:27:52.232 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000102685 2024-08-17T17:27:52.233 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.233 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.233 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:27:52.233 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.045168+0000", 2024-08-17T17:27:52.233 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000132688 2024-08-17T17:27:52.233 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.233 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.233 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.233 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.047497+0000", 2024-08-17T17:27:52.233 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023291789999999998 2024-08-17T17:27:52.233 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.233 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.233 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.234 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.049887+0000", 2024-08-17T17:27:52.234 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023896490000000002 2024-08-17T17:27:52.234 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.234 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.234 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.234 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.078336+0000", 2024-08-17T17:27:52.234 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.028448803000000002 2024-08-17T17:27:52.234 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.234 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.234 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.234 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.078395+0000", 2024-08-17T17:27:52.234 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9135000000000002e-05 2024-08-17T17:27:52.234 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.235 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.235 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.235 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:18:33.079642+0000", 2024-08-17T17:27:52.235 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012465110000000001 2024-08-17T17:27:52.235 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.235 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.235 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.235 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.235 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.235 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88364 7.6 7:671e019e:::benchmark_data_smithi063_196002_object88363:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:27:52.235 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:00.720918+0000", 2024-08-17T17:27:52.235 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 531.45965280400003, 2024-08-17T17:27:52.235 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9732610319999999, 2024-08-17T17:27:52.236 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.236 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:27:52.236 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.236 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:52.236 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:52.236 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88364 2024-08-17T17:27:52.236 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.236 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.236 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.236 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.236 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.720918+0000", 2024-08-17T17:27:52.236 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.236 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.237 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.237 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.237 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.720918+0000", 2024-08-17T17:27:52.237 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.237 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.237 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.237 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.237 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.720915+0000", 2024-08-17T17:27:52.237 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999967 2024-08-17T17:27:52.237 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.237 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.237 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.237 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.720977+0000", 2024-08-17T17:27:52.238 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1952999999999995e-05 2024-08-17T17:27:52.238 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.238 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.238 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.238 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721015+0000", 2024-08-17T17:27:52.238 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7889000000000003e-05 2024-08-17T17:27:52.238 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.238 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.238 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.238 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721029+0000", 2024-08-17T17:27:52.238 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4025999999999999e-05 2024-08-17T17:27:52.238 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.238 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.238 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.239 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.789772+0000", 2024-08-17T17:27:52.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.068743471 2024-08-17T17:27:52.239 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.239 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.239 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.239 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.789781+0000", 2024-08-17T17:27:52.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8750000000000006e-06 2024-08-17T17:27:52.239 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.239 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.239 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.239 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.649177+0000", 2024-08-17T17:27:52.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.859395829 2024-08-17T17:27:52.239 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.240 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.240 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.240 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.649188+0000", 2024-08-17T17:27:52.240 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1219999999999999e-05 2024-08-17T17:27:52.240 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.240 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.240 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.240 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.694077+0000", 2024-08-17T17:27:52.240 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0448895309999999 2024-08-17T17:27:52.240 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.240 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.240 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.241 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.694179+0000", 2024-08-17T17:27:52.241 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000101369 2024-08-17T17:27:52.241 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.241 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.241 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.241 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.241 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.241 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88365 7.16 7:6872d9c5:::benchmark_data_smithi063_196002_object88364:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:27:52.241 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:00.721043+0000", 2024-08-17T17:27:52.241 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 531.459527772, 2024-08-17T17:27:52.241 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.067687582, 2024-08-17T17:27:52.241 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.241 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.242 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.242 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:52.242 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:52.242 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88365 2024-08-17T17:27:52.242 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.242 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.242 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.242 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.242 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721043+0000", 2024-08-17T17:27:52.242 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.242 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.242 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.242 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.243 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721043+0000", 2024-08-17T17:27:52.243 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.243 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.243 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.243 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.243 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721042+0000", 2024-08-17T17:27:52.243 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:52.243 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.243 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.243 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.243 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721088+0000", 2024-08-17T17:27:52.243 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5148000000000002e-05 2024-08-17T17:27:52.243 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.244 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.244 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.244 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721094+0000", 2024-08-17T17:27:52.244 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5509999999999996e-06 2024-08-17T17:27:52.244 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.244 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.244 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.244 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721099+0000", 2024-08-17T17:27:52.244 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1200000000000001e-06 2024-08-17T17:27:52.244 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.244 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.244 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.245 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.790366+0000", 2024-08-17T17:27:52.245 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.069267251000000002 2024-08-17T17:27:52.245 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.245 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.245 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.245 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.790374+0000", 2024-08-17T17:27:52.245 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.8720000000000002e-06 2024-08-17T17:27:52.245 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.245 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.245 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.245 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.769317+0000", 2024-08-17T17:27:52.245 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97894276199999997 2024-08-17T17:27:52.245 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.246 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.246 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.246 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.769496+0000", 2024-08-17T17:27:52.246 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017906700000000001 2024-08-17T17:27:52.246 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.246 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.246 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:27:52.246 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.769685+0000", 2024-08-17T17:27:52.246 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00018840600000000001 2024-08-17T17:27:52.246 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.247 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.247 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.247 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.787874+0000", 2024-08-17T17:27:52.247 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.018189338999999999 2024-08-17T17:27:52.247 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.247 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.247 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.247 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.787895+0000", 2024-08-17T17:27:52.247 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.1577999999999999e-05 2024-08-17T17:27:52.247 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.247 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.247 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.248 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.788201+0000", 2024-08-17T17:27:52.248 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00030538599999999999 2024-08-17T17:27:52.248 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.248 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.248 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.248 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.788243+0000", 2024-08-17T17:27:52.248 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1764999999999997e-05 2024-08-17T17:27:52.248 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.248 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.248 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.248 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.788730+0000", 2024-08-17T17:27:52.248 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00048783399999999998 2024-08-17T17:27:52.249 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.249 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.249 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.249 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.249 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.249 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88371 7.6 7:6649df65:::benchmark_data_smithi063_196002_object88370:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:27:52.249 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:00.721104+0000", 2024-08-17T17:27:52.249 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 531.45946645100003, 2024-08-17T17:27:52.249 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.986046859, 2024-08-17T17:27:52.249 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.249 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.249 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.249 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:52.250 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:52.250 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88371 2024-08-17T17:27:52.250 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.250 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.250 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.250 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.250 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721104+0000", 2024-08-17T17:27:52.250 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.250 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.250 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.250 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.250 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721104+0000", 2024-08-17T17:27:52.250 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.251 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.251 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.251 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.251 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721104+0000", 2024-08-17T17:27:52.251 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:52.251 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.251 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.251 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.251 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721145+0000", 2024-08-17T17:27:52.251 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1221999999999999e-05 2024-08-17T17:27:52.251 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.251 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.251 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.252 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721151+0000", 2024-08-17T17:27:52.252 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1789999999999996e-06 2024-08-17T17:27:52.252 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.252 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.252 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.252 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721155+0000", 2024-08-17T17:27:52.252 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0289999999999999e-06 2024-08-17T17:27:52.252 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.252 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.252 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.252 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.789812+0000", 2024-08-17T17:27:52.252 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.068656462000000001 2024-08-17T17:27:52.252 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.253 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.253 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.253 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.789818+0000", 2024-08-17T17:27:52.253 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0700000000000003e-06 2024-08-17T17:27:52.253 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.253 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.253 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.253 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.649213+0000", 2024-08-17T17:27:52.253 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.85939523699999998 2024-08-17T17:27:52.253 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.253 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.253 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.253 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.649222+0000", 2024-08-17T17:27:52.254 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6319999999999997e-06 2024-08-17T17:27:52.254 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.254 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.254 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.254 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.695192+0000", 2024-08-17T17:27:52.254 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0459707110000001 2024-08-17T17:27:52.254 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.254 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.254 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.254 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.695366+0000", 2024-08-17T17:27:52.254 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000174024 2024-08-17T17:27:52.254 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.254 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.255 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:27:52.255 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.695548+0000", 2024-08-17T17:27:52.255 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00018159599999999999 2024-08-17T17:27:52.255 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.255 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.255 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.255 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.700528+0000", 2024-08-17T17:27:52.255 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0049801330000000003 2024-08-17T17:27:52.255 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.255 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.255 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.255 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.703416+0000", 2024-08-17T17:27:52.255 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0028876560000000002 2024-08-17T17:27:52.255 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.256 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.256 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.256 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.705782+0000", 2024-08-17T17:27:52.256 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002366047 2024-08-17T17:27:52.256 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.256 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.256 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.256 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.705834+0000", 2024-08-17T17:27:52.256 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2556000000000003e-05 2024-08-17T17:27:52.256 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.256 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.256 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.256 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.707151+0000", 2024-08-17T17:27:52.257 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013167070000000001 2024-08-17T17:27:52.257 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.257 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.257 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.257 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.257 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.257 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88376 7.6 7:64605e94:::benchmark_data_smithi063_196002_object88375:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1151)", 2024-08-17T17:27:52.257 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:00.721160+0000", 2024-08-17T17:27:52.257 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 531.45941100599998, 2024-08-17T17:27:52.257 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.987210446, 2024-08-17T17:27:52.257 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.257 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.257 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.258 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:52.258 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:52.258 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88376 2024-08-17T17:27:52.258 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.258 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.258 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.258 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.258 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721160+0000", 2024-08-17T17:27:52.258 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.258 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.258 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.259 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.259 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721160+0000", 2024-08-17T17:27:52.259 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.259 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.259 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.259 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.259 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721159+0000", 2024-08-17T17:27:52.259 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:52.259 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.259 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.260 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.260 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721198+0000", 2024-08-17T17:27:52.260 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8661999999999997e-05 2024-08-17T17:27:52.260 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.260 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.261 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.261 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721204+0000", 2024-08-17T17:27:52.261 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1279999999999996e-06 2024-08-17T17:27:52.261 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.261 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.261 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.261 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.721208+0000", 2024-08-17T17:27:52.261 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0929999999999996e-06 2024-08-17T17:27:52.261 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.261 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.261 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.261 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.789834+0000", 2024-08-17T17:27:52.262 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.068625969999999994 2024-08-17T17:27:52.262 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.262 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.262 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.262 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:00.789840+0000", 2024-08-17T17:27:52.262 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9619999999999998e-06 2024-08-17T17:27:52.262 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.262 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.262 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.262 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.649348+0000", 2024-08-17T17:27:52.262 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.85950773999999996 2024-08-17T17:27:52.262 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.262 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.263 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.263 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.649358+0000", 2024-08-17T17:27:52.263 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9960000000000005e-06 2024-08-17T17:27:52.263 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.263 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.263 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.263 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.697349+0000", 2024-08-17T17:27:52.263 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0479912570000001 2024-08-17T17:27:52.263 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.263 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.263 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.263 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.697478+0000", 2024-08-17T17:27:52.263 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012920100000000001 2024-08-17T17:27:52.264 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.264 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.264 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:27:52.264 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.697630+0000", 2024-08-17T17:27:52.264 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015209500000000001 2024-08-17T17:27:52.264 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.264 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.264 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.264 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.703423+0000", 2024-08-17T17:27:52.264 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0057923810000000001 2024-08-17T17:27:52.264 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.264 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.264 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.264 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.704352+0000", 2024-08-17T17:27:52.265 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00092935499999999998 2024-08-17T17:27:52.265 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.265 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.265 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.265 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.707253+0000", 2024-08-17T17:27:52.265 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0029011499999999999 2024-08-17T17:27:52.265 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.265 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.265 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.265 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.707301+0000", 2024-08-17T17:27:52.265 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7689999999999999e-05 2024-08-17T17:27:52.265 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.265 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.266 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.266 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.708370+0000", 2024-08-17T17:27:52.266 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0010691159999999999 2024-08-17T17:27:52.266 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.266 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.266 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.266 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.266 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.266 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88407 7.6 7:657e6986:::benchmark_data_smithi063_196002_object88406:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1152)", 2024-08-17T17:27:52.266 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.728376+0000", 2024-08-17T17:27:52.266 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 530.45219430099996, 2024-08-17T17:27:52.266 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.00199964, 2024-08-17T17:27:52.267 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.267 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.267 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.267 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:52.267 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:52.267 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88407 2024-08-17T17:27:52.267 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.267 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.267 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.267 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.267 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.728376+0000", 2024-08-17T17:27:52.267 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.267 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.267 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.268 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.268 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.728376+0000", 2024-08-17T17:27:52.268 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.268 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.268 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.268 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.268 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.728374+0000", 2024-08-17T17:27:52.268 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999976 2024-08-17T17:27:52.268 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.268 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.268 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.268 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.728450+0000", 2024-08-17T17:27:52.269 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.5836999999999999e-05 2024-08-17T17:27:52.269 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.269 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.269 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.269 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.728468+0000", 2024-08-17T17:27:52.269 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7927e-05 2024-08-17T17:27:52.269 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.269 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.269 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.269 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.728477+0000", 2024-08-17T17:27:52.269 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2580000000000001e-06 2024-08-17T17:27:52.269 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.269 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.269 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.270 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.728543+0000", 2024-08-17T17:27:52.270 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6383000000000005e-05 2024-08-17T17:27:52.270 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.270 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.270 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.270 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.728555+0000", 2024-08-17T17:27:52.270 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1739999999999999e-05 2024-08-17T17:27:52.270 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.270 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.270 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.270 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.699053+0000", 2024-08-17T17:27:52.270 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97049734499999996 2024-08-17T17:27:52.271 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.271 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.271 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.271 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.699182+0000", 2024-08-17T17:27:52.271 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012976700000000001 2024-08-17T17:27:52.271 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.271 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.271 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:27:52.271 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.699311+0000", 2024-08-17T17:27:52.271 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000128727 2024-08-17T17:27:52.271 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.271 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.272 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.272 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.704361+0000", 2024-08-17T17:27:52.272 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0050494880000000004 2024-08-17T17:27:52.272 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.272 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.272 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.272 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.708421+0000", 2024-08-17T17:27:52.272 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0040604539999999998 2024-08-17T17:27:52.272 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.272 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.272 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.272 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.729061+0000", 2024-08-17T17:27:52.272 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.020639520000000001 2024-08-17T17:27:52.273 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.273 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.273 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.273 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.729110+0000", 2024-08-17T17:27:52.273 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9086999999999999e-05 2024-08-17T17:27:52.273 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.273 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.273 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.273 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.730376+0000", 2024-08-17T17:27:52.273 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012663430000000001 2024-08-17T17:27:52.273 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.273 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.273 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.274 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.274 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.274 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:88413 7.6 7:62cf965b:::benchmark_data_smithi063_196002_object88412:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1152)", 2024-08-17T17:27:52.274 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:19:01.778995+0000", 2024-08-17T17:27:52.274 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 530.40157592699995, 2024-08-17T17:27:52.274 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95251618400000004, 2024-08-17T17:27:52.274 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.274 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.274 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.274 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:27:52.274 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:27:52.274 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 88413 2024-08-17T17:27:52.274 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.275 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.275 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.275 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.275 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.778995+0000", 2024-08-17T17:27:52.275 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.275 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.275 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.275 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.275 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.778995+0000", 2024-08-17T17:27:52.275 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.275 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.275 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.275 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.276 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.778994+0000", 2024-08-17T17:27:52.276 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:52.276 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.276 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.276 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.276 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.779035+0000", 2024-08-17T17:27:52.276 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0852999999999997e-05 2024-08-17T17:27:52.276 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.276 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.276 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.276 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.779046+0000", 2024-08-17T17:27:52.276 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.19e-05 2024-08-17T17:27:52.276 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.277 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.277 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.277 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.779055+0000", 2024-08-17T17:27:52.277 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9020000000000005e-06 2024-08-17T17:27:52.277 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.277 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.277 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.277 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.787916+0000", 2024-08-17T17:27:52.277 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0088602390000000007 2024-08-17T17:27:52.277 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.277 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.277 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:27:52.277 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:01.787922+0000", 2024-08-17T17:27:52.277 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8419999999999999e-06 2024-08-17T17:27:52.278 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.278 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.278 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.278 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.700555+0000", 2024-08-17T17:27:52.278 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.91263279399999997 2024-08-17T17:27:52.278 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.278 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.278 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.278 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.700660+0000", 2024-08-17T17:27:52.278 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010457400000000001 2024-08-17T17:27:52.278 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.278 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.279 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:27:52.279 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.700751+0000", 2024-08-17T17:27:52.279 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.1155000000000004e-05 2024-08-17T17:27:52.279 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.279 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.279 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.279 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.704367+0000", 2024-08-17T17:27:52.279 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0036157519999999999 2024-08-17T17:27:52.279 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.279 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.279 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.279 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.708952+0000", 2024-08-17T17:27:52.279 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0045855310000000003 2024-08-17T17:27:52.280 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.280 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.280 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.280 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.730429+0000", 2024-08-17T17:27:52.280 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.021476888999999999 2024-08-17T17:27:52.280 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.280 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.280 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.280 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.730505+0000", 2024-08-17T17:27:52.280 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6190000000000001e-05 2024-08-17T17:27:52.280 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.280 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.280 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.281 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:19:02.731511+0000", 2024-08-17T17:27:52.281 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001005515 2024-08-17T17:27:52.281 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.281 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.281 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.281 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.281 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.281 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:25692 9.9 9:909dc1e3:::benchmark_data_smithi063_263014_object25691:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1251)", 2024-08-17T17:27:52.281 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:26:56.438654+0000", 2024-08-17T17:27:52.281 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 55.741916158999999, 2024-08-17T17:27:52.281 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.20093865399999999, 2024-08-17T17:27:52.281 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.281 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.282 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.282 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:52.282 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:52.282 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 25692 2024-08-17T17:27:52.282 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.282 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.282 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.282 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.282 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.438654+0000", 2024-08-17T17:27:52.282 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.282 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.282 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.282 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.283 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.438654+0000", 2024-08-17T17:27:52.283 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.283 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.283 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.283 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.283 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.438653+0000", 2024-08-17T17:27:52.283 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:52.283 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.283 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.283 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.283 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.438862+0000", 2024-08-17T17:27:52.283 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00020918 2024-08-17T17:27:52.283 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.283 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.284 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.284 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.438873+0000", 2024-08-17T17:27:52.284 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0874e-05 2024-08-17T17:27:52.284 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.284 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.284 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.284 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.438915+0000", 2024-08-17T17:27:52.284 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1532000000000001e-05 2024-08-17T17:27:52.284 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.284 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.284 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.284 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.439009+0000", 2024-08-17T17:27:52.284 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4129999999999995e-05 2024-08-17T17:27:52.285 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.285 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.285 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.285 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.439136+0000", 2024-08-17T17:27:52.285 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000126618 2024-08-17T17:27:52.285 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.285 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.285 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:27:52.285 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.439266+0000", 2024-08-17T17:27:52.285 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000130073 2024-08-17T17:27:52.285 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.285 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.285 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.286 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.443559+0000", 2024-08-17T17:27:52.286 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0042933429999999998 2024-08-17T17:27:52.286 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.286 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.286 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.286 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.443976+0000", 2024-08-17T17:27:52.286 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00041653600000000001 2024-08-17T17:27:52.286 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.286 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.286 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.286 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.639506+0000", 2024-08-17T17:27:52.286 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.195530021 2024-08-17T17:27:52.286 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.287 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.287 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.287 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.639548+0000", 2024-08-17T17:27:52.287 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2110999999999997e-05 2024-08-17T17:27:52.287 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.287 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.287 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.287 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.639593+0000", 2024-08-17T17:27:52.287 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5420999999999998e-05 2024-08-17T17:27:52.287 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.287 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.287 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.287 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.288 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.288 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:25701 9.9 9:9c6b84ec:::benchmark_data_smithi063_263014_object25700:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1251)", 2024-08-17T17:27:52.288 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:26:56.440847+0000", 2024-08-17T17:27:52.288 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 55.739723140000002, 2024-08-17T17:27:52.288 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.201000071, 2024-08-17T17:27:52.288 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.288 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.288 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.288 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:52.288 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:52.288 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 25701 2024-08-17T17:27:52.288 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.288 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.288 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.289 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.289 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.440847+0000", 2024-08-17T17:27:52.289 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.289 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.289 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.289 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.289 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.440847+0000", 2024-08-17T17:27:52.289 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.289 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.289 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.289 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.289 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.440847+0000", 2024-08-17T17:27:52.290 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:52.290 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.290 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.290 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.290 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.440900+0000", 2024-08-17T17:27:52.290 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3437999999999999e-05 2024-08-17T17:27:52.290 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.290 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.290 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.290 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.440909+0000", 2024-08-17T17:27:52.290 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.149e-06 2024-08-17T17:27:52.290 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.290 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.291 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.291 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.440916+0000", 2024-08-17T17:27:52.291 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2389999999999999e-06 2024-08-17T17:27:52.291 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.291 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.291 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.291 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.440950+0000", 2024-08-17T17:27:52.291 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4548999999999997e-05 2024-08-17T17:27:52.291 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.291 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.291 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.291 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.441095+0000", 2024-08-17T17:27:52.291 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014452900000000001 2024-08-17T17:27:52.292 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.292 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.292 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:27:52.292 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.441566+0000", 2024-08-17T17:27:52.292 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00047116999999999998 2024-08-17T17:27:52.292 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.292 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.292 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.292 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.443731+0000", 2024-08-17T17:27:52.292 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021654840000000001 2024-08-17T17:27:52.292 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.292 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.292 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.293 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.444121+0000", 2024-08-17T17:27:52.293 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00039001299999999998 2024-08-17T17:27:52.293 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.293 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.293 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.293 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.641764+0000", 2024-08-17T17:27:52.293 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.197642506 2024-08-17T17:27:52.293 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.293 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.293 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.293 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.641811+0000", 2024-08-17T17:27:52.293 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6623999999999999e-05 2024-08-17T17:27:52.293 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.294 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.294 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.294 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.641848+0000", 2024-08-17T17:27:52.294 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6980000000000002e-05 2024-08-17T17:27:52.294 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.294 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.294 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.294 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.294 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.294 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:25976 9.e 9:7222fd7c:::benchmark_data_smithi063_263014_object25975:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1251)", 2024-08-17T17:27:52.294 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:26:56.853139+0000", 2024-08-17T17:27:52.294 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 55.327431560999997, 2024-08-17T17:27:52.294 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.22521754399999999, 2024-08-17T17:27:52.295 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.295 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.295 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.295 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:52.295 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:52.295 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 25976 2024-08-17T17:27:52.295 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.295 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.295 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.295 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.295 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.853139+0000", 2024-08-17T17:27:52.295 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.295 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.296 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.296 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.296 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.853139+0000", 2024-08-17T17:27:52.296 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.296 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.296 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.296 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.296 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.853138+0000", 2024-08-17T17:27:52.296 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:52.296 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.296 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.296 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.296 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.853173+0000", 2024-08-17T17:27:52.297 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4359000000000001e-05 2024-08-17T17:27:52.297 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.297 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.297 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.297 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.853182+0000", 2024-08-17T17:27:52.297 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3659999999999998e-06 2024-08-17T17:27:52.297 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.297 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.297 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.297 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.853188+0000", 2024-08-17T17:27:52.297 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7230000000000001e-06 2024-08-17T17:27:52.297 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.297 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.298 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.298 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.854715+0000", 2024-08-17T17:27:52.298 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015268809999999999 2024-08-17T17:27:52.298 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.298 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.298 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.298 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.854833+0000", 2024-08-17T17:27:52.298 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011790000000000001 2024-08-17T17:27:52.298 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.298 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.298 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:27:52.298 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.854969+0000", 2024-08-17T17:27:52.298 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013638699999999999 2024-08-17T17:27:52.299 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.299 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.299 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.299 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.856478+0000", 2024-08-17T17:27:52.299 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015094399999999999 2024-08-17T17:27:52.299 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.299 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.299 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.299 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.858966+0000", 2024-08-17T17:27:52.299 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0024873740000000001 2024-08-17T17:27:52.299 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.299 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.299 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.300 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.078254+0000", 2024-08-17T17:27:52.300 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.21928797999999999 2024-08-17T17:27:52.300 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.300 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.300 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.300 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.078310+0000", 2024-08-17T17:27:52.300 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5874000000000001e-05 2024-08-17T17:27:52.300 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.300 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.300 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.300 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.078357+0000", 2024-08-17T17:27:52.300 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6916999999999998e-05 2024-08-17T17:27:52.300 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.300 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.301 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.301 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.301 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.301 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:26007 9.e 9:77785aa7:::benchmark_data_smithi063_263014_object26006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1251)", 2024-08-17T17:27:52.301 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:26:56.863833+0000", 2024-08-17T17:27:52.301 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 55.316737388999996, 2024-08-17T17:27:52.301 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.21645382399999999, 2024-08-17T17:27:52.301 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.301 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.301 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.301 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:52.301 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:52.301 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 26007 2024-08-17T17:27:52.302 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.302 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.302 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.302 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.302 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.863833+0000", 2024-08-17T17:27:52.302 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.302 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.302 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.302 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.302 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.863833+0000", 2024-08-17T17:27:52.302 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.302 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.302 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.303 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.303 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.863832+0000", 2024-08-17T17:27:52.303 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:52.303 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.303 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.303 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.303 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.863978+0000", 2024-08-17T17:27:52.303 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014578700000000001 2024-08-17T17:27:52.303 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.303 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:52.305 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.305 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.305 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.863993+0000", 2024-08-17T17:27:52.305 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4704e-05 2024-08-17T17:27:52.305 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.305 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.305 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.305 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.864000+0000", 2024-08-17T17:27:52.305 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.8290000000000008e-06 2024-08-17T17:27:52.305 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.305 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.306 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.306 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.864051+0000", 2024-08-17T17:27:52.306 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0355000000000002e-05 2024-08-17T17:27:52.306 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.306 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.306 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.306 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.864180+0000", 2024-08-17T17:27:52.306 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012905499999999999 2024-08-17T17:27:52.306 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.306 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.306 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:27:52.306 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.864313+0000", 2024-08-17T17:27:52.307 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013292100000000001 2024-08-17T17:27:52.307 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.307 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.307 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.307 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.865373+0000", 2024-08-17T17:27:52.307 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0010607139999999999 2024-08-17T17:27:52.307 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.307 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.307 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.307 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.865931+0000", 2024-08-17T17:27:52.307 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00055763700000000004 2024-08-17T17:27:52.307 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.307 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.308 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.308 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.080144+0000", 2024-08-17T17:27:52.308 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.21421327100000001 2024-08-17T17:27:52.308 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.308 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.308 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.308 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.080192+0000", 2024-08-17T17:27:52.308 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7651000000000002e-05 2024-08-17T17:27:52.308 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.308 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.308 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.308 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.080287+0000", 2024-08-17T17:27:52.308 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.5084e-05 2024-08-17T17:27:52.309 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.309 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.309 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.309 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.309 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.309 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:26020 9.e 9:76c9ba14:::benchmark_data_smithi063_263014_object26019:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1251)", 2024-08-17T17:27:52.309 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:26:56.871889+0000", 2024-08-17T17:27:52.309 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 55.308681542000002, 2024-08-17T17:27:52.309 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.20855354500000001, 2024-08-17T17:27:52.309 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.309 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.309 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.309 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:52.310 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:52.310 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 26020 2024-08-17T17:27:52.310 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.310 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.310 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.310 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.310 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.871889+0000", 2024-08-17T17:27:52.310 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.310 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.310 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.310 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.310 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.871889+0000", 2024-08-17T17:27:52.311 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.311 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.311 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.311 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.311 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.871888+0000", 2024-08-17T17:27:52.311 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:52.311 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.311 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.311 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.311 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.871997+0000", 2024-08-17T17:27:52.311 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000108533 2024-08-17T17:27:52.311 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.311 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.312 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.312 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.872008+0000", 2024-08-17T17:27:52.312 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0784999999999999e-05 2024-08-17T17:27:52.312 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.312 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.312 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.312 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.872014+0000", 2024-08-17T17:27:52.312 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0109999999999997e-06 2024-08-17T17:27:52.312 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.312 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.312 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.312 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.872049+0000", 2024-08-17T17:27:52.312 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4968000000000001e-05 2024-08-17T17:27:52.313 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.313 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.313 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.313 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.872137+0000", 2024-08-17T17:27:52.313 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8789999999999995e-05 2024-08-17T17:27:52.313 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.313 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.313 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:27:52.313 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.872215+0000", 2024-08-17T17:27:52.313 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7228e-05 2024-08-17T17:27:52.313 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.313 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.314 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.314 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.873277+0000", 2024-08-17T17:27:52.314 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001062068 2024-08-17T17:27:52.314 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.314 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.314 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.314 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.873527+0000", 2024-08-17T17:27:52.314 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00025069900000000001 2024-08-17T17:27:52.314 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.314 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.314 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.314 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.080360+0000", 2024-08-17T17:27:52.314 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.20683270500000001 2024-08-17T17:27:52.315 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.315 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.315 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.315 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.080405+0000", 2024-08-17T17:27:52.315 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4569e-05 2024-08-17T17:27:52.315 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.315 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.315 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.315 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.080443+0000", 2024-08-17T17:27:52.315 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7871999999999999e-05 2024-08-17T17:27:52.315 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.315 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.315 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.315 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.316 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.316 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:45037 9.9 9:98aac709:::benchmark_data_smithi063_263014_object45036:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:27:52.316 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:08.450929+0000", 2024-08-17T17:27:52.316 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 43.729641426000001, 2024-08-17T17:27:52.316 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.17910045399999999, 2024-08-17T17:27:52.316 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.316 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.316 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.316 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:52.316 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:52.316 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 45037 2024-08-17T17:27:52.316 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.317 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.317 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.317 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.317 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.450929+0000", 2024-08-17T17:27:52.317 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.317 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.317 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.317 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.317 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.450929+0000", 2024-08-17T17:27:52.317 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.317 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.317 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.317 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.317 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.450929+0000", 2024-08-17T17:27:52.317 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:27:52.318 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.318 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.318 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.318 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.451023+0000", 2024-08-17T17:27:52.318 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4011999999999994e-05 2024-08-17T17:27:52.318 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.318 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.318 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.318 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.451033+0000", 2024-08-17T17:27:52.318 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0040999999999999e-05 2024-08-17T17:27:52.318 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.318 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.319 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.319 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.451039+0000", 2024-08-17T17:27:52.319 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6279999999999999e-06 2024-08-17T17:27:52.319 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.319 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.319 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.319 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.451136+0000", 2024-08-17T17:27:52.319 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7048999999999999e-05 2024-08-17T17:27:52.319 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.319 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.319 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.319 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.451252+0000", 2024-08-17T17:27:52.320 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000115299 2024-08-17T17:27:52.320 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.320 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.320 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:27:52.320 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.451368+0000", 2024-08-17T17:27:52.320 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011603999999999999 2024-08-17T17:27:52.320 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.320 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.320 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.320 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.457201+0000", 2024-08-17T17:27:52.320 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0058331490000000001 2024-08-17T17:27:52.320 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.320 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.321 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.321 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.457522+0000", 2024-08-17T17:27:52.321 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00032107700000000002 2024-08-17T17:27:52.321 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.321 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.321 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.321 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.629959+0000", 2024-08-17T17:27:52.321 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.172437221 2024-08-17T17:27:52.321 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.321 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.321 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.321 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.630001+0000", 2024-08-17T17:27:52.321 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1522e-05 2024-08-17T17:27:52.322 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.322 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.322 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.322 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.630030+0000", 2024-08-17T17:27:52.322 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9085999999999999e-05 2024-08-17T17:27:52.322 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.322 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.322 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.322 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.322 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.322 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:45038 9.9 9:94fc2236:::benchmark_data_smithi063_263014_object45037:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:27:52.322 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:08.451060+0000", 2024-08-17T17:27:52.323 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 43.729510331, 2024-08-17T17:27:52.323 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.17906725900000001, 2024-08-17T17:27:52.323 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.323 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.323 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:27:52.323 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:27:52.323 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:27:52.323 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 45038 2024-08-17T17:27:52.323 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.323 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.323 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.323 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.323 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.451060+0000", 2024-08-17T17:27:52.324 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.324 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.324 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.324 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.324 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.451060+0000", 2024-08-17T17:27:52.324 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.324 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.324 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.324 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.324 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.451059+0000", 2024-08-17T17:27:52.324 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:52.324 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.324 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.325 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.325 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.451124+0000", 2024-08-17T17:27:52.325 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4686999999999999e-05 2024-08-17T17:27:52.325 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.325 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.325 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.325 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.451132+0000", 2024-08-17T17:27:52.325 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4769999999999997e-06 2024-08-17T17:27:52.325 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.325 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.325 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.325 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.451137+0000", 2024-08-17T17:27:52.326 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9250000000000003e-06 2024-08-17T17:27:52.326 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.326 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.326 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.326 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.452990+0000", 2024-08-17T17:27:52.326 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0018533740000000001 2024-08-17T17:27:52.326 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.326 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.326 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.326 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.453134+0000", 2024-08-17T17:27:52.326 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014373700000000001 2024-08-17T17:27:52.326 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.326 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.327 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:27:52.327 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.453280+0000", 2024-08-17T17:27:52.327 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000146675 2024-08-17T17:27:52.327 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.327 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.327 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.327 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.457555+0000", 2024-08-17T17:27:52.327 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00427459 2024-08-17T17:27:52.327 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.327 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.327 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:27:52.327 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.457563+0000", 2024-08-17T17:27:52.327 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6960000000000005e-06 2024-08-17T17:27:52.328 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.328 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.328 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:27:52.328 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.630068+0000", 2024-08-17T17:27:52.328 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.17250523500000001 2024-08-17T17:27:52.328 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.328 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.328 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.328 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.630105+0000", 2024-08-17T17:27:52.328 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7135000000000003e-05 2024-08-17T17:27:52.328 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.328 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.328 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.328 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:08.630128+0000", 2024-08-17T17:27:52.329 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2529000000000001e-05 2024-08-17T17:27:52.329 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.329 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.329 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.329 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.329 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.329 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.45521.0:57705 9.a e1253/1247 9:563675ec:::benchmark_data_smithi063_263014_object57704:head v 1253'3603, mlcod=1253'3599)", 2024-08-17T17:27:52.329 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:16.371231+0000", 2024-08-17T17:27:52.329 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 35.809339809000001, 2024-08-17T17:27:52.329 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.153911045, 2024-08-17T17:27:52.329 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:27:52.329 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:27:52.330 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:27:52.330 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.330 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:27:52.330 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:16.371231+0000", 2024-08-17T17:27:52.330 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.330 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.330 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.330 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:27:52.330 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:16.371231+0000", 2024-08-17T17:27:52.330 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:27:52.330 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.330 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.331 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:27:52.331 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:16.371230+0000", 2024-08-17T17:27:52.331 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:27:52.331 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.331 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.331 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:27:52.331 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:16.371254+0000", 2024-08-17T17:27:52.331 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3688000000000001e-05 2024-08-17T17:27:52.331 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.331 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.331 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:27:52.331 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:16.511694+0000", 2024-08-17T17:27:52.331 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14044052000000001 2024-08-17T17:27:52.331 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.332 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.332 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:27:52.332 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:16.511714+0000", 2024-08-17T17:27:52.332 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9706000000000001e-05 2024-08-17T17:27:52.332 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.332 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.332 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:27:52.332 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:16.512390+0000", 2024-08-17T17:27:52.332 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00067597600000000001 2024-08-17T17:27:52.332 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.332 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.332 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:27:52.332 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:16.512412+0000", 2024-08-17T17:27:52.333 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.1878e-05 2024-08-17T17:27:52.333 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.333 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.333 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:27:52.333 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:16.525098+0000", 2024-08-17T17:27:52.333 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.01268587 2024-08-17T17:27:52.333 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:27:52.333 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:27:52.333 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:27:52.333 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:16.525142+0000", 2024-08-17T17:27:52.333 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4143000000000001e-05 2024-08-17T17:27:52.333 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.333 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.334 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.334 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:27:52.334 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:27:52.334 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:27:52.339 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:52.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:52.441 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:52.467 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 66 16 99314 99298 94.0153 63.375 0.0170979 0.0106317 2024-08-17T17:27:52.475 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:52.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:52.577 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:52.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.609 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:52.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:52.710 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:52.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.742 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:52.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:52.843 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:52.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:52.888 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:52.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:52.989 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:53.022 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:53.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:53.124 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:53.167 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:53.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:53.269 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:53.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.310 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:53.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:53.411 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:53.449 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:53.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:53.467 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 67 16 100357 100341 93.585 65.1875 0.0352719 0.0106804 2024-08-17T17:27:53.472 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:53 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:53.472 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:53 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:53.472 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:53 smithi162 ceph-mon[104668]: pgmap v1737: 119 pgs: 119 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s 2024-08-17T17:27:53.472 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:53 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:27:53.472 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:53 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:27:53.472 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:53 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:53.474 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:27:53.514 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:27:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:27:53] "GET /metrics HTTP/1.1" 200 36400 "" "Prometheus/2.51.0" 2024-08-17T17:27:53.514 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:53 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:53.515 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:53 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:53.515 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:53 smithi063 ceph-mon[109843]: pgmap v1737: 119 pgs: 119 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s 2024-08-17T17:27:53.515 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:53 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:27:53.515 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:53 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:27:53.515 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:53 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:53.515 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:53 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:53.515 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:53 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:53.515 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:53 smithi063 ceph-mon[114756]: pgmap v1737: 119 pgs: 119 active+clean; 5.8 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s 2024-08-17T17:27:53.516 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:53 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:27:53.516 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:53 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:27:53.516 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:53 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:27:53.533 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:27:53.533 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:27:53.551 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:53.670 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:53.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:53.772 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:53.808 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:53.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:53.910 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:53.941 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:53.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:53.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:54.042 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:54.083 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:54.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:54.185 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:54.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.226 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:54.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:54.327 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:54.366 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:54.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:54.467 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 68 16 101331 101315 93.1039 60.875 0.015173 0.0107352 2024-08-17T17:27:54.467 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:54.502 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:54.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:54.604 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:54.635 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:54.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:54.737 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:54.777 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:54.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:54.879 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:54.914 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:54.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:54.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:55.016 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:55.052 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:55.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:55.154 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:55.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.202 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:55.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:55.303 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:55.332 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:55.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:55.342 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:55.434 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:55.467 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 69 16 102342 102326 92.6703 63.1875 0.013987 0.0107854 2024-08-17T17:27:55.471 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:55.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:55.573 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:55.585 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:27:55.600 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:55.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:55.702 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:55.742 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:55.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:55.751 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:55 smithi063 ceph-mon[114756]: pgmap v1738: 119 pgs: 119 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-08-17T17:27:55.752 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:55 smithi063 ceph-mon[109843]: pgmap v1738: 119 pgs: 119 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-08-17T17:27:55.843 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:55.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:55 smithi162 ceph-mon[104668]: pgmap v1738: 119 pgs: 119 active+clean; 6.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-08-17T17:27:55.882 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:55.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:55.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:55.984 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:56.015 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:56.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:56.117 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:56.138 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:56.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:56.240 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:56.263 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:56.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:56.365 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:56.399 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:56.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:56.467 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 70 16 103087 103071 92.0116 46.5625 0.0109077 0.0108629 2024-08-17T17:27:56.501 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:27:56.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.538 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:56.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:27:56.543 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:27:56.639 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:56.688 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:56.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:56.790 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:56.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.826 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:56.826 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:56.927 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:56.985 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:56.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:56.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:57.035 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:27:57.035 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:27:57.062 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:27:57.062 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:27:57.087 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:57.110 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:27:57.110 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:27:57.127 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:57.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:57.145 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:27:57.145 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:27:57.177 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:27:57.178 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:27:57.213 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:27:57.213 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:27:57.229 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:57.252 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:27:57.252 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:27:57.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.269 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:57.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:57.286 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:27:57.286 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:27:57.369 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:27:57.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.401 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:57.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:27:57.467 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 71 16 104138 104122 91.6408 65.6875 0.00673489 0.0109072 2024-08-17T17:27:57.473 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:57 smithi063 ceph-mon[114756]: pgmap v1739: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T17:27:57.473 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:57 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1613858487' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:57.474 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:57 smithi063 ceph-mon[109843]: pgmap v1739: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T17:27:57.474 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:57 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1613858487' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:57.501 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:57.539 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:57.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:57.642 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:57.683 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:57.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:57.785 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:27:57.816 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:57.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:27:57.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:57 smithi162 ceph-mon[104668]: pgmap v1739: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-08-17T17:27:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:57 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1613858487' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:27:57.918 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:57.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:57.961 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:57.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:58.062 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:58.101 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:58.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:58.203 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:58.239 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:58.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:58.341 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:27:58.381 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:27:58.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:27:58.467 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 72 16 105292 105276 91.3696 72.125 0.00888023 0.0109397 2024-08-17T17:27:58.484 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:58.521 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:58.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:58.623 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:58.668 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:58.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:58.770 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:58.807 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:58.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:58.909 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:58.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:58.960 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:58.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:59.061 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:27:59.102 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:59.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:27:59.204 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:59.244 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:59.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:27:59.346 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:59.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.383 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:59.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:59.467 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 73 16 106421 106405 91.0846 70.5625 0.0110395 0.0109733 2024-08-17T17:27:59.484 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:27:59.521 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:27:59 smithi063 ceph-mon[109843]: pgmap v1740: 119 pgs: 119 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:27:59.521 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:27:59 smithi063 ceph-mon[114756]: pgmap v1740: 119 pgs: 119 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:27:59.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.525 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:59.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:27:59.577 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:27:59.577 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:27:59.626 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:59.667 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:59.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:59.769 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:27:59.801 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:59.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:27:59.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:27:59 smithi162 ceph-mon[104668]: pgmap v1740: 119 pgs: 119 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:27:59.903 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:27:59.946 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:27:59.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:27:59.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:00.048 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:00.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.091 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:00.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:00.192 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:00.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.233 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:00.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:00.334 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:00.360 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:28:00.369 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:00.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:00.412 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:00.468 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 74 15 107510 107495 90.7742 68.125 0.0151704 0.011011 2024-08-17T17:28:00.471 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:00.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.509 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:00.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:00.610 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:00.650 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:00.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:00.753 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:00.794 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:00.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:00.895 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:00.952 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:28:00.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:00.958 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:00.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:01.059 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:01.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.120 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:01.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:01.221 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:01.290 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:01.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:01.393 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:01.432 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:01.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:01.468 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 75 16 108549 108533 90.4289 64.875 0.00865049 0.0110534 2024-08-17T17:28:01.534 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:01.567 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:01.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:01.629 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:01 smithi063 ceph-mon[114756]: pgmap v1741: 119 pgs: 119 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:28:01.629 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:28:01.630 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:01.630 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:01 smithi063 ceph-mon[109843]: pgmap v1741: 119 pgs: 119 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:28:01.630 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:28:01.630 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:01.669 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:01.712 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:01.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:01.813 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:01 smithi162 ceph-mon[104668]: pgmap v1741: 119 pgs: 119 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-08-17T17:28:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:28:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:01.849 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:01.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:01.951 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:01.985 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:01.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:01.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:02.087 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:02.126 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:02.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:02.212 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:02.228 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:02.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.262 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:02.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:02.363 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:02.376 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:02.395 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:02.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:02.467 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:28:02.470 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 76 16 109605 109589 90.105 66 0.0225101 0.0110927 2024-08-17T17:28:02.500 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:02.546 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:02.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:02.594 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:02 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4197183881' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:28:02.595 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:02 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4197183881' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:28:02.649 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:02.686 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:02.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:02.788 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:02.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.824 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:02.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:02 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4197183881' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:28:02.925 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:02.966 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:02.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:02.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:03.068 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:03.105 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:03.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:03.206 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:03.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.244 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:03.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:03.345 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:03.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.383 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:03.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:03.470 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 77 16 110708 110692 89.8296 68.9375 0.0103764 0.0111272 2024-08-17T17:28:03.484 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:03.519 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:03.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:03.548 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:28:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:28:03] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.51.0" 2024-08-17T17:28:03.550 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:03 smithi063 ceph-mon[109843]: pgmap v1742: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s 2024-08-17T17:28:03.550 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:03 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4197183881' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:28:03.550 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:03 smithi063 ceph-mon[109843]: osdmap e1259: 8 total, 8 up, 8 in 2024-08-17T17:28:03.550 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:03 smithi063 ceph-mon[114756]: pgmap v1742: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s 2024-08-17T17:28:03.550 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:03 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4197183881' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:28:03.550 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:03 smithi063 ceph-mon[114756]: osdmap e1259: 8 total, 8 up, 8 in 2024-08-17T17:28:03.621 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:03.657 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:03.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:03.746 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:03 smithi162 ceph-mon[104668]: pgmap v1742: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s 2024-08-17T17:28:03.746 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:03 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4197183881' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:28:03.747 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:03 smithi162 ceph-mon[104668]: osdmap e1259: 8 total, 8 up, 8 in 2024-08-17T17:28:03.759 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:03.798 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:03.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:03.900 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:03.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:03.948 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:03.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:04.049 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:04.075 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:28:04.090 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:04.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:04.195 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:04.242 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:04.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:04.344 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:04.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.392 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:04.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:04.470 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 78 16 111740 111724 89.5048 64.5 0.0216066 0.011167 2024-08-17T17:28:04.493 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:04.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.530 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:04.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:04.631 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:04.670 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:04.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:04.772 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:04.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.809 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:04.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:04.910 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:04.947 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:04.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:04.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:05.048 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:05.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.081 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:05.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:05.182 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:05.223 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:05.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:05.325 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:05.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.363 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:05.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:05.464 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:05.470 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 79 16 112747 112731 89.1685 62.9375 0.009255 0.0112081 2024-08-17T17:28:05.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.499 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:05.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:05.600 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:05.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.640 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:05.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:05.742 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:05.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.775 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:05.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:05.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:05 smithi063 ceph-mon[109843]: pgmap v1744: 119 pgs: 119 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:28:05.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:05 smithi063 ceph-mon[114756]: pgmap v1744: 119 pgs: 119 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:28:05.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:05 smithi162 ceph-mon[104668]: pgmap v1744: 119 pgs: 119 active+clean; 6.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:28:05.876 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:05.907 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:05.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:05.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:06.009 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:06.042 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:06.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:06.144 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:06.179 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:06.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:06.280 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:06.322 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:06.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:06.423 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:06.455 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:06.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:06.470 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:28:06.470686+0000 min lat: 0.00127496 max lat: 0.360103 avg lat: 0.0112493 2024-08-17T17:28:06.470 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:28:06.470 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 80 16 113772 113756 88.8546 64.0625 0.0110969 0.0112493 2024-08-17T17:28:06.557 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:06.598 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:06.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:06.700 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:06.731 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:06.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:06.759 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:28:06.759 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:28:06.793 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:28:06.793 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:28:06.833 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:06.841 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:28:06.841 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:28:06.866 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:06.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:06.886 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:28:06.887 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:28:06.919 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:28:06.919 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:28:06.963 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:28:06.963 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:28:06.968 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:06.974 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:28:06.974 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:28:06.974 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:28:06.974 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:28:06.974 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:28:06.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:06.996 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:06.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:07.008 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:28:07.008 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:28:07.051 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:28:07.051 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:28:07.096 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:07.133 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:07.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:07.235 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:07.268 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:07.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:07.370 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:07.408 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:07.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:07.471 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 81 16 114792 114776 88.5446 63.75 0.0154569 0.0112886 2024-08-17T17:28:07.510 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:07.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.549 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:07.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:07.650 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:07.682 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:07.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:07 smithi063 ceph-mon[114756]: pgmap v1745: 119 pgs: 119 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:28:07.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:07 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2974838591' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:07.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:07 smithi063 ceph-mon[109843]: pgmap v1745: 119 pgs: 119 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:28:07.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:07 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2974838591' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:07.783 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:07.822 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:07.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:07.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:07 smithi162 ceph-mon[104668]: pgmap v1745: 119 pgs: 119 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-08-17T17:28:07.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:07 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2974838591' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:07.924 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:07.956 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:07.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:07.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:08.058 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:08.097 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:08.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:08.198 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:08.237 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:08.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:08.325 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:08.338 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:08.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.370 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:08.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:08.471 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 82 16 115799 115783 88.2322 62.9375 0.0066011 0.0113222 2024-08-17T17:28:08.471 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:08.494 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:08.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:08.500 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:28:08.596 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:08.631 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:08.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:08.733 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:08.767 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:08 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2756124702' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:08.767 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:08 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:08.768 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:08.769 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:08 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2756124702' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:08.769 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:08 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:08.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:08.806 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:28:08.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:08 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2756124702' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:08.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:08 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:08.870 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:08.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:08.928 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:08.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:09.029 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:09.072 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:09.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:09.174 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:09.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.207 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:09.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:09.308 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:09.349 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:09.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:09.451 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:09.471 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 83 16 116854 116838 87.9627 65.9375 0.016412 0.0113634 2024-08-17T17:28:09.488 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:09.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:09.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:09 smithi063 ceph-mon[114756]: pgmap v1746: 119 pgs: 119 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T17:28:09.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:09 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:09.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:09 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:28:09.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:09 smithi063 ceph-mon[114756]: osdmap e1260: 8 total, 8 up, 8 in 2024-08-17T17:28:09.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:09 smithi063 ceph-mon[109843]: pgmap v1746: 119 pgs: 119 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T17:28:09.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:09 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:09.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:09 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:28:09.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:09 smithi063 ceph-mon[109843]: osdmap e1260: 8 total, 8 up, 8 in 2024-08-17T17:28:09.589 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:09.630 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:09.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:09.732 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:09.766 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:09.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:09.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:09 smithi162 ceph-mon[104668]: pgmap v1746: 119 pgs: 119 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-08-17T17:28:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:09 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:09 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:28:09.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:09 smithi162 ceph-mon[104668]: osdmap e1260: 8 total, 8 up, 8 in 2024-08-17T17:28:09.868 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:09.901 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:09.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:09.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:10.004 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:10.043 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:10.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:10.121 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:28:10.145 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:10.180 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:10.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:10.281 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:10.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.335 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:10.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:10.436 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:10.471 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 84 16 117919 117903 87.7079 66.5625 0.0129897 0.0113962 2024-08-17T17:28:10.473 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:10.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:10.576 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:10.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.616 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:10.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:10.717 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:10.752 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:10.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:10.854 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:10.894 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:10.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:10.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:10.957 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:10.996 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:11.052 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:11.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:11.154 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:11.196 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:11.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:11.298 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:11.341 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:11.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:11.443 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:11.472 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 85 16 118995 118979 87.4672 67.25 0.00895104 0.0114282 2024-08-17T17:28:11.478 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:11.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:11.580 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:11.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.615 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:11.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:11.716 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:11.718 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:28:11.759 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:11.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:11.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:11 smithi063 ceph-mon[114756]: pgmap v1748: 119 pgs: 119 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-08-17T17:28:11.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:11 smithi063 ceph-mon[109843]: pgmap v1748: 119 pgs: 119 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-08-17T17:28:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:11 smithi162 ceph-mon[104668]: pgmap v1748: 119 pgs: 119 active+clean; 7.0 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s 2024-08-17T17:28:11.862 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:11.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:11.897 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:11.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:11.928 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:11.997 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:12.029 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:12.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:12.130 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:12.163 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:12.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:12.265 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:12.303 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:12.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:12.406 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:12.444 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:12.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:12.472 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 86 16 120017 120001 87.1928 63.875 0.0153082 0.0114643 2024-08-17T17:28:12.514 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:28:12.545 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:12.583 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:12.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.584 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:12.685 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:12.722 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:12.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:12.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:12 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/521277869' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:28:12.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:12 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:28:12.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:12 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/521277869' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:28:12.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:12 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:28:12.824 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:12.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:12 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/521277869' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:28:12.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:12 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:28:12.857 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:12.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:12.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:12.959 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:13.008 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:13.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:13.110 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:13.133 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:13.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:13.235 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:13.273 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:13.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:13.375 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:13.412 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:13.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:13.472 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 87 16 121083 121067 86.9564 66.625 0.0127539 0.0114952 2024-08-17T17:28:13.514 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:13.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.552 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:13.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:13.653 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:13.690 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:13.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:13.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:13 smithi063 ceph-mon[109843]: pgmap v1749: 119 pgs: 119 active+clean; 7.1 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s 2024-08-17T17:28:13.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:13 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:28:13.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:13 smithi063 ceph-mon[109843]: osdmap e1261: 8 total, 8 up, 8 in 2024-08-17T17:28:13.780 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:28:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:28:13] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.51.0" 2024-08-17T17:28:13.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:13 smithi063 ceph-mon[114756]: pgmap v1749: 119 pgs: 119 active+clean; 7.1 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s 2024-08-17T17:28:13.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:13 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:28:13.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:13 smithi063 ceph-mon[114756]: osdmap e1261: 8 total, 8 up, 8 in 2024-08-17T17:28:13.791 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:13.824 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:13.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:13.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:13 smithi162 ceph-mon[104668]: pgmap v1749: 119 pgs: 119 active+clean; 7.1 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 62 MiB/s wr, 995 op/s 2024-08-17T17:28:13.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:13 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:28:13.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:13 smithi162 ceph-mon[104668]: osdmap e1261: 8 total, 8 up, 8 in 2024-08-17T17:28:13.926 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:13.964 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:13.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:13.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:14.066 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:14.124 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:14.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:14.226 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:14.264 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:14.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:14.366 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:14.404 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:14.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:14.472 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 88 16 122176 122160 86.7445 68.3125 0.0124794 0.0115234 2024-08-17T17:28:14.506 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:14.536 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:14.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:14.637 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:14.675 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:14.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:14.777 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:14.809 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:14.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:14.911 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:14.945 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:14.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:14.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:15.047 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:15.086 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:15.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:15.188 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:15.226 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:15.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:15.328 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:15.370 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:15.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:15.472 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:15.474 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 89 16 123265 123249 86.533 68.0625 0.0122303 0.0115514 2024-08-17T17:28:15.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.510 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:15.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:15.611 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:15.655 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:15.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:15.757 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:15.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:15 smithi063 ceph-mon[114756]: pgmap v1751: 119 pgs: 119 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T17:28:15.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:15 smithi063 ceph-mon[109843]: pgmap v1751: 119 pgs: 119 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T17:28:15.795 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:15.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:15.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:15 smithi162 ceph-mon[104668]: pgmap v1751: 119 pgs: 119 active+clean; 7.3 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-08-17T17:28:15.898 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:15.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:15.931 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:15.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:16.032 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:16.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.068 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:16.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:16.169 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:16.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.202 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:16.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:16.210 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:28:16.303 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:16.348 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:16.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:16.450 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:16.474 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Total time run: 90.0073 2024-08-17T17:28:16.474 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Total writes made: 124335 2024-08-17T17:28:16.474 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Write size: 65536 2024-08-17T17:28:16.474 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Object size: 65536 2024-08-17T17:28:16.474 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Bandwidth (MB/sec): 86.3367 2024-08-17T17:28:16.474 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev Bandwidth: 34.1995 2024-08-17T17:28:16.474 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max bandwidth (MB/sec): 234.375 2024-08-17T17:28:16.474 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min bandwidth (MB/sec): 45.25 2024-08-17T17:28:16.474 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Average IOPS: 1381 2024-08-17T17:28:16.474 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev IOPS: 547.191 2024-08-17T17:28:16.474 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max IOPS: 3750 2024-08-17T17:28:16.474 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min IOPS: 724 2024-08-17T17:28:16.475 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Average Latency(s): 0.0115783 2024-08-17T17:28:16.475 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev Latency(s): 0.0124167 2024-08-17T17:28:16.475 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max latency(s): 0.360103 2024-08-17T17:28:16.475 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min latency(s): 0.00127496 2024-08-17T17:28:16.479 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:28:16.481 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:16.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:16.583 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:16.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.615 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:16.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:16.716 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:16.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.755 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:16.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:16.775 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:16.776 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:16.847 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:28:16.847 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:28:16.847 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:28:16.847 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:28:16.847 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.848 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "MOSDPGPush(7.5 1149/1144 [PushOp(7:aa8dfb9c:::benchmark_data_smithi063_196002_object9253:head, version: 1132'606, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:aa8dfb9c:::benchmark_data_smithi063_196002_object9253:head@1132'606, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:28:16.848 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:18:53.656740+0000", 2024-08-17T17:28:16.848 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 563.18780294400005, 2024-08-17T17:28:16.848 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.152369581, 2024-08-17T17:28:16.848 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.848 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.848 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:28:16.848 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.848 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.848 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.848 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656740+0000", 2024-08-17T17:28:16.849 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.849 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.849 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.849 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.849 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656740+0000", 2024-08-17T17:28:16.849 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.849 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.849 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.849 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.849 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656741+0000", 2024-08-17T17:28:16.849 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0079999999999999e-06 2024-08-17T17:28:16.849 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.849 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.850 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.850 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656947+0000", 2024-08-17T17:28:16.850 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00020619200000000001 2024-08-17T17:28:16.850 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.850 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.850 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.850 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656975+0000", 2024-08-17T17:28:16.850 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.7746000000000002e-05 2024-08-17T17:28:16.850 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.850 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.850 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.851 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.656998+0000", 2024-08-17T17:28:16.851 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3006e-05 2024-08-17T17:28:16.851 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.851 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.851 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.851 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.803287+0000", 2024-08-17T17:28:16.851 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14628891699999999 2024-08-17T17:28:16.851 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.851 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.851 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.851 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.803301+0000", 2024-08-17T17:28:16.851 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3920000000000001e-05 2024-08-17T17:28:16.852 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.852 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.852 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.852 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:18:53.809110+0000", 2024-08-17T17:28:16.852 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0058087920000000001 2024-08-17T17:28:16.852 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.852 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.852 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.852 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.852 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.852 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88367 7.d e1150/1144 7:bbf555f3:::benchmark_data_smithi063_196002_object88366:head v 1150'5489, mlcod=0'0)", 2024-08-17T17:28:16.852 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.645168+0000", 2024-08-17T17:28:16.853 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 556.19937544200002, 2024-08-17T17:28:16.853 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.210459275, 2024-08-17T17:28:16.853 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.853 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.853 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.853 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.853 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.853 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.853 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.645168+0000", 2024-08-17T17:28:16.853 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.853 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.853 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.853 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.854 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.645168+0000", 2024-08-17T17:28:16.854 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.854 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.854 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.854 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.854 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.645169+0000", 2024-08-17T17:28:16.854 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1969999999999999e-06 2024-08-17T17:28:16.854 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.854 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.854 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.854 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646316+0000", 2024-08-17T17:28:16.855 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0011472229999999999 2024-08-17T17:28:16.855 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.855 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.855 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.855 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646328+0000", 2024-08-17T17:28:16.855 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1657999999999999e-05 2024-08-17T17:28:16.855 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.855 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.855 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.855 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646333+0000", 2024-08-17T17:28:16.855 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.733e-06 2024-08-17T17:28:16.855 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.855 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.856 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.856 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646392+0000", 2024-08-17T17:28:16.856 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8805999999999999e-05 2024-08-17T17:28:16.856 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.856 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.856 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.856 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646423+0000", 2024-08-17T17:28:16.856 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1179000000000001e-05 2024-08-17T17:28:16.856 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.856 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.856 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.856 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.855586+0000", 2024-08-17T17:28:16.856 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20916273699999999 2024-08-17T17:28:16.857 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.857 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.857 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.857 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.855627+0000", 2024-08-17T17:28:16.857 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.0741999999999997e-05 2024-08-17T17:28:16.857 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.857 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.860 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.861 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.861 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.861 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88364 7.6 e1150/1144 7:671e019e:::benchmark_data_smithi063_196002_object88363:head v 1150'5532, mlcod=0'0)", 2024-08-17T17:28:16.861 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.646551+0000", 2024-08-17T17:28:16.861 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 556.19799176399999, 2024-08-17T17:28:16.861 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.22748069200000001, 2024-08-17T17:28:16.862 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.862 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.862 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:28:16.862 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.862 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.862 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.863 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646551+0000", 2024-08-17T17:28:16.863 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.863 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.863 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.863 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.863 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646551+0000", 2024-08-17T17:28:16.864 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.864 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.864 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.864 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.864 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646552+0000", 2024-08-17T17:28:16.864 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.29e-07 2024-08-17T17:28:16.864 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.865 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.865 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.865 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646636+0000", 2024-08-17T17:28:16.865 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4449000000000004e-05 2024-08-17T17:28:16.865 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.866 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.866 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.866 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646645+0000", 2024-08-17T17:28:16.866 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7099999999999996e-06 2024-08-17T17:28:16.866 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.866 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.866 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.866 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646650+0000", 2024-08-17T17:28:16.866 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1780000000000002e-06 2024-08-17T17:28:16.866 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.867 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.867 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.867 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646720+0000", 2024-08-17T17:28:16.867 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9924000000000003e-05 2024-08-17T17:28:16.867 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.867 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.867 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.867 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.646752+0000", 2024-08-17T17:28:16.867 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1919999999999999e-05 2024-08-17T17:28:16.867 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.867 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.867 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.868 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.874032+0000", 2024-08-17T17:28:16.868 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.227280282 2024-08-17T17:28:16.868 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.868 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.868 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.868 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.868 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.868 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:16.869 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "MOSDPGPush(7.5 1150/1144 [PushOp(7:a927a2c1:::benchmark_data_smithi063_196002_object12871:head, version: 1132'815, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:a927a2c1:::benchmark_data_smithi063_196002_object12871:head@1132'815, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:28:16.869 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.650543+0000", 2024-08-17T17:28:16.869 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 556.19400054799996, 2024-08-17T17:28:16.870 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20508452899999999, 2024-08-17T17:28:16.870 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.870 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.870 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:28:16.870 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.870 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.870 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.870 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650543+0000", 2024-08-17T17:28:16.870 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.870 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.870 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.870 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.871 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650543+0000", 2024-08-17T17:28:16.871 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.871 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.871 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.871 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.871 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650544+0000", 2024-08-17T17:28:16.871 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3659999999999999e-06 2024-08-17T17:28:16.871 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.871 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.871 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.871 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650585+0000", 2024-08-17T17:28:16.871 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.1542999999999998e-05 2024-08-17T17:28:16.872 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.872 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.872 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.872 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650612+0000", 2024-08-17T17:28:16.872 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6242999999999999e-05 2024-08-17T17:28:16.872 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.872 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.872 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.872 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.650629+0000", 2024-08-17T17:28:16.872 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.747e-05 2024-08-17T17:28:16.873 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.873 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.873 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.873 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.855605+0000", 2024-08-17T17:28:16.873 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20497623200000001 2024-08-17T17:28:16.874 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.874 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.874 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.874 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.855627+0000", 2024-08-17T17:28:16.874 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1675000000000002e-05 2024-08-17T17:28:16.874 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.875 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.875 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.875 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.875 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.875 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88374 7.d e1150/1144 7:b7ece4c7:::benchmark_data_smithi063_196002_object88373:head v 1150'5490, mlcod=0'0)", 2024-08-17T17:28:16.876 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.651831+0000", 2024-08-17T17:28:16.876 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 556.19271210800002, 2024-08-17T17:28:16.876 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20382080899999999, 2024-08-17T17:28:16.876 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.876 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.876 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.876 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.876 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.876 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.877 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651831+0000", 2024-08-17T17:28:16.877 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.877 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.877 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.877 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.877 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651831+0000", 2024-08-17T17:28:16.877 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.877 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.877 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.877 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.877 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651832+0000", 2024-08-17T17:28:16.878 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.06e-06 2024-08-17T17:28:16.878 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.878 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.878 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.878 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651872+0000", 2024-08-17T17:28:16.878 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9501e-05 2024-08-17T17:28:16.878 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.878 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.878 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.878 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651889+0000", 2024-08-17T17:28:16.878 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7815000000000001e-05 2024-08-17T17:28:16.878 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.878 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.879 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.879 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.651898+0000", 2024-08-17T17:28:16.879 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.5040000000000002e-06 2024-08-17T17:28:16.879 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.879 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.879 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.879 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.653677+0000", 2024-08-17T17:28:16.879 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001778899 2024-08-17T17:28:16.879 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.879 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.879 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.880 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.653704+0000", 2024-08-17T17:28:16.880 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.7161999999999998e-05 2024-08-17T17:28:16.880 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.880 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.880 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.880 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.855630+0000", 2024-08-17T17:28:16.880 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20192644600000001 2024-08-17T17:28:16.880 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.880 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.880 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.880 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.855652+0000", 2024-08-17T17:28:16.881 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1421999999999999e-05 2024-08-17T17:28:16.881 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.881 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.881 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.881 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.881 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.881 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88377 7.e e1150/1144 7:72a20e0d:::benchmark_data_smithi063_196002_object88376:head v 1150'5626, mlcod=0'0)", 2024-08-17T17:28:16.881 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.682989+0000", 2024-08-17T17:28:16.881 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 556.16155411199998, 2024-08-17T17:28:16.881 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.25261817399999997, 2024-08-17T17:28:16.881 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.881 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.882 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.882 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.882 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.882 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.882 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.682989+0000", 2024-08-17T17:28:16.882 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.882 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.882 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.882 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.882 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.682989+0000", 2024-08-17T17:28:16.882 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.882 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.882 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.883 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.883 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.682989+0000", 2024-08-17T17:28:16.883 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3599999999999999e-07 2024-08-17T17:28:16.883 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.883 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.883 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.883 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.683026+0000", 2024-08-17T17:28:16.883 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7171e-05 2024-08-17T17:28:16.883 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.883 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.883 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.883 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.683049+0000", 2024-08-17T17:28:16.883 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2577000000000001e-05 2024-08-17T17:28:16.883 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.884 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.884 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.884 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.683062+0000", 2024-08-17T17:28:16.884 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3134e-05 2024-08-17T17:28:16.884 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.884 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.884 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.884 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.875331+0000", 2024-08-17T17:28:16.884 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19226912900000001 2024-08-17T17:28:16.884 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.884 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.884 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.884 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.875369+0000", 2024-08-17T17:28:16.885 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7978000000000001e-05 2024-08-17T17:28:16.885 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.885 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.885 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.885 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.935541+0000", 2024-08-17T17:28:16.885 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.060171988000000003 2024-08-17T17:28:16.885 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.885 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.885 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.885 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.935607+0000", 2024-08-17T17:28:16.885 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5860999999999998e-05 2024-08-17T17:28:16.885 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.885 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.886 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.886 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.886 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.886 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.34965.0:88379 7.e e1151/1144 7:77a5a758:::benchmark_data_smithi063_196002_object88378:head v 1151'5627, mlcod=0'0)", 2024-08-17T17:28:16.886 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:00.707652+0000", 2024-08-17T17:28:16.886 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 556.13689133699995, 2024-08-17T17:28:16.886 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.22874829799999999, 2024-08-17T17:28:16.886 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.886 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.886 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.886 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.887 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.887 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.887 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.707652+0000", 2024-08-17T17:28:16.887 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.887 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.887 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.887 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.887 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.707652+0000", 2024-08-17T17:28:16.887 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.887 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.887 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.887 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.888 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.707654+0000", 2024-08-17T17:28:16.888 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0449999999999999e-06 2024-08-17T17:28:16.888 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.888 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.888 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.888 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.707692+0000", 2024-08-17T17:28:16.888 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8145999999999997e-05 2024-08-17T17:28:16.888 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.888 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.888 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.888 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.707715+0000", 2024-08-17T17:28:16.888 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3397e-05 2024-08-17T17:28:16.888 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.889 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.889 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.889 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.707728+0000", 2024-08-17T17:28:16.889 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2893e-05 2024-08-17T17:28:16.889 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.889 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.889 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.889 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.887668+0000", 2024-08-17T17:28:16.889 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.179940088 2024-08-17T17:28:16.889 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.889 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.889 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.890 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.887705+0000", 2024-08-17T17:28:16.890 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6947999999999999e-05 2024-08-17T17:28:16.890 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.890 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.890 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.890 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.936360+0000", 2024-08-17T17:28:16.890 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.048654898000000002 2024-08-17T17:28:16.890 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.890 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.890 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.890 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:00.936400+0000", 2024-08-17T17:28:16.890 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9882999999999997e-05 2024-08-17T17:28:16.891 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.891 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.891 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.891 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.891 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.891 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "MOSDPGPush(7.e 1154/1144 [PushOp(7:7d25040c:::benchmark_data_smithi063_196002_object8959:head, version: 1132'552, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:7d25040c:::benchmark_data_smithi063_196002_object8959:head@1132'552, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:28:16.891 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:19:20.740336+0000", 2024-08-17T17:28:16.891 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 536.10420669500002, 2024-08-17T17:28:16.891 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.151667202, 2024-08-17T17:28:16.891 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.891 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.891 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:28:16.892 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.892 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.892 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.892 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.740336+0000", 2024-08-17T17:28:16.892 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.892 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.892 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.892 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.892 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.740336+0000", 2024-08-17T17:28:16.892 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.892 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.892 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.892 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.893 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.740337+0000", 2024-08-17T17:28:16.893 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.1800000000000001e-07 2024-08-17T17:28:16.893 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.893 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.893 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.893 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.740368+0000", 2024-08-17T17:28:16.893 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1130999999999997e-05 2024-08-17T17:28:16.893 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.893 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.893 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.893 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.740384+0000", 2024-08-17T17:28:16.893 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5610000000000001e-05 2024-08-17T17:28:16.893 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.894 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.894 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.894 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.740397+0000", 2024-08-17T17:28:16.894 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3380000000000001e-05 2024-08-17T17:28:16.894 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.894 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.894 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.894 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.882222+0000", 2024-08-17T17:28:16.894 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14182467000000001 2024-08-17T17:28:16.894 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.894 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.894 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.895 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.882251+0000", 2024-08-17T17:28:16.895 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.9852000000000001e-05 2024-08-17T17:28:16.895 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.895 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.895 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.895 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:19:20.892004+0000", 2024-08-17T17:28:16.895 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0097521410000000006 2024-08-17T17:28:16.895 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.895 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.895 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.895 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.895 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.895 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.7.0:82783 7.5 e1180/1152 7:a27210bc:::benchmark_data_smithi063_196002_object109261:head v 1180'9710, mlcod=1180'9693)", 2024-08-17T17:28:16.896 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:21:32.185748+0000", 2024-08-17T17:28:16.896 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 404.65879471099998, 2024-08-17T17:28:16.896 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.107565682, 2024-08-17T17:28:16.896 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.896 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.896 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.896 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.896 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.896 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.896 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.185748+0000", 2024-08-17T17:28:16.896 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.896 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.897 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.897 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.897 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.185748+0000", 2024-08-17T17:28:16.897 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.897 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.897 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.897 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.897 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.185749+0000", 2024-08-17T17:28:16.897 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7700000000000004e-07 2024-08-17T17:28:16.897 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.897 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.897 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.897 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.185759+0000", 2024-08-17T17:28:16.898 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.6199999999999994e-06 2024-08-17T17:28:16.898 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.898 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.898 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.898 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.185774+0000", 2024-08-17T17:28:16.898 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5131e-05 2024-08-17T17:28:16.898 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.898 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.898 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.898 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.185781+0000", 2024-08-17T17:28:16.898 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5020000000000003e-06 2024-08-17T17:28:16.898 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.898 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.899 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.899 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.241151+0000", 2024-08-17T17:28:16.899 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.055370115999999997 2024-08-17T17:28:16.899 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.899 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.899 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.899 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.241171+0000", 2024-08-17T17:28:16.899 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9585e-05 2024-08-17T17:28:16.899 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.900 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.900 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.900 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.292583+0000", 2024-08-17T17:28:16.900 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.051412226999999998 2024-08-17T17:28:16.900 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.900 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.900 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.900 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:21:32.293314+0000", 2024-08-17T17:28:16.900 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00073082400000000004 2024-08-17T17:28:16.900 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.900 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.900 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.900 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.901 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.901 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45218.0:138112 8.2 e1213/1195 8:4cfb624c:::benchmark_data_smithi063_230025_object11622:head v 1213'8819, mlcod=1213'8817)", 2024-08-17T17:28:16.901 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:24:10.386314+0000", 2024-08-17T17:28:16.901 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 246.45822929100001, 2024-08-17T17:28:16.901 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.123093868, 2024-08-17T17:28:16.901 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.901 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.901 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.901 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.901 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.901 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.901 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386314+0000", 2024-08-17T17:28:16.902 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.902 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.902 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.902 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.902 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386314+0000", 2024-08-17T17:28:16.902 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.902 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.902 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.903 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.903 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386315+0000", 2024-08-17T17:28:16.903 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0449999999999999e-06 2024-08-17T17:28:16.903 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.903 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.903 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.903 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386325+0000", 2024-08-17T17:28:16.903 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0037999999999999e-05 2024-08-17T17:28:16.903 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.903 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.903 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.903 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386337+0000", 2024-08-17T17:28:16.904 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1958e-05 2024-08-17T17:28:16.904 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.904 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.904 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.904 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386345+0000", 2024-08-17T17:28:16.904 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.1489999999999994e-06 2024-08-17T17:28:16.904 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.904 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.904 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.904 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386423+0000", 2024-08-17T17:28:16.904 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7682999999999995e-05 2024-08-17T17:28:16.904 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.904 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.905 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.905 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.386445+0000", 2024-08-17T17:28:16.905 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2558e-05 2024-08-17T17:28:16.905 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.905 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.905 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.905 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390898+0000", 2024-08-17T17:28:16.905 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044525980000000003 2024-08-17T17:28:16.905 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.905 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.905 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.905 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509408+0000", 2024-08-17T17:28:16.905 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.11850983900000001 2024-08-17T17:28:16.906 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.906 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.906 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.906 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.906 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.906 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45218.0:138114 8.6 e1213/1195 8:6a96a6b7:::benchmark_data_smithi063_230025_object11624:head v 1213'8609, mlcod=1213'8607)", 2024-08-17T17:28:16.906 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:24:10.389955+0000", 2024-08-17T17:28:16.906 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 246.45458840500001, 2024-08-17T17:28:16.906 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.12685882800000001, 2024-08-17T17:28:16.906 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.906 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.906 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.906 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.907 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.907 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.907 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.389955+0000", 2024-08-17T17:28:16.907 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.907 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.907 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.907 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.907 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.389955+0000", 2024-08-17T17:28:16.907 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.907 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.907 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.907 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.907 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.389956+0000", 2024-08-17T17:28:16.908 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8700000000000004e-07 2024-08-17T17:28:16.908 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.908 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.908 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.908 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.389960+0000", 2024-08-17T17:28:16.908 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9490000000000004e-06 2024-08-17T17:28:16.908 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.908 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.908 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.908 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.389974+0000", 2024-08-17T17:28:16.908 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3152999999999999e-05 2024-08-17T17:28:16.908 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.908 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.909 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.909 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.389982+0000", 2024-08-17T17:28:16.909 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9889999999999992e-06 2024-08-17T17:28:16.909 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.909 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.909 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.909 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390058+0000", 2024-08-17T17:28:16.909 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6427000000000005e-05 2024-08-17T17:28:16.909 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.909 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.909 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.909 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390078+0000", 2024-08-17T17:28:16.909 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9629000000000001e-05 2024-08-17T17:28:16.910 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.910 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.910 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.910 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.516764+0000", 2024-08-17T17:28:16.910 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.126685933 2024-08-17T17:28:16.910 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.910 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.910 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.910 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.516813+0000", 2024-08-17T17:28:16.910 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9861000000000003e-05 2024-08-17T17:28:16.910 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.910 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.910 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.911 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.911 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.911 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45218.0:138103 8.c e1213/1195 8:30f0a028:::benchmark_data_smithi063_230025_object11613:head v 1213'8745, mlcod=1213'8743)", 2024-08-17T17:28:16.911 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:24:10.390810+0000", 2024-08-17T17:28:16.911 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 246.45373304099999, 2024-08-17T17:28:16.911 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14214605299999999, 2024-08-17T17:28:16.911 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.911 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.911 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.911 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.911 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.911 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.911 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390810+0000", 2024-08-17T17:28:16.912 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.912 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.912 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.912 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.912 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390810+0000", 2024-08-17T17:28:16.912 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.912 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.912 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.912 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.912 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390811+0000", 2024-08-17T17:28:16.912 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7999999999999995e-07 2024-08-17T17:28:16.912 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.912 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.913 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.913 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390817+0000", 2024-08-17T17:28:16.913 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0100000000000001e-06 2024-08-17T17:28:16.913 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.913 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.913 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.913 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509388+0000", 2024-08-17T17:28:16.913 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.118571464 2024-08-17T17:28:16.913 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.913 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.913 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.913 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509396+0000", 2024-08-17T17:28:16.913 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9980000000000003e-06 2024-08-17T17:28:16.914 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.914 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.914 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.914 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509570+0000", 2024-08-17T17:28:16.914 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000174279 2024-08-17T17:28:16.914 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.914 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.914 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.914 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509590+0000", 2024-08-17T17:28:16.914 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9083999999999999e-05 2024-08-17T17:28:16.914 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.914 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.915 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.915 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.532822+0000", 2024-08-17T17:28:16.915 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.023232017000000001 2024-08-17T17:28:16.915 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.915 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.915 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.915 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.532956+0000", 2024-08-17T17:28:16.915 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013452099999999999 2024-08-17T17:28:16.915 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.915 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.915 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.915 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.915 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.916 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45218.0:138118 8.2 e1213/1195 8:48763a67:::benchmark_data_smithi063_230025_object11628:head v 1213'8820, mlcod=1213'8817)", 2024-08-17T17:28:16.916 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:24:10.393238+0000", 2024-08-17T17:28:16.916 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 246.45130519899999, 2024-08-17T17:28:16.916 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.123133243, 2024-08-17T17:28:16.916 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.916 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.916 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.916 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.916 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.916 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.916 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.393238+0000", 2024-08-17T17:28:16.916 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.917 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.917 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.917 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.918 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.393238+0000", 2024-08-17T17:28:16.918 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.918 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.918 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.918 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.918 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.393239+0000", 2024-08-17T17:28:16.918 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0839999999999999e-06 2024-08-17T17:28:16.918 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.918 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.918 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.918 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.393250+0000", 2024-08-17T17:28:16.919 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.063e-05 2024-08-17T17:28:16.919 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.919 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.919 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.919 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509130+0000", 2024-08-17T17:28:16.919 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.11588066599999999 2024-08-17T17:28:16.919 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.919 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.919 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.919 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509164+0000", 2024-08-17T17:28:16.919 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3503000000000001e-05 2024-08-17T17:28:16.919 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.920 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.920 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.920 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509437+0000", 2024-08-17T17:28:16.920 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000272902 2024-08-17T17:28:16.920 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.920 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.920 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.920 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509462+0000", 2024-08-17T17:28:16.920 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5718999999999999e-05 2024-08-17T17:28:16.920 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.920 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.920 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.920 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.516304+0000", 2024-08-17T17:28:16.921 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0068414399999999998 2024-08-17T17:28:16.921 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.921 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.921 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.921 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.516371+0000", 2024-08-17T17:28:16.921 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7298999999999994e-05 2024-08-17T17:28:16.921 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.921 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.921 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.921 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.921 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.921 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45218.0:138121 8.9 e1213/1195 8:980b75d7:::benchmark_data_smithi063_230025_object11631:head v 1213'8648, mlcod=1213'8646)", 2024-08-17T17:28:16.922 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:24:10.395445+0000", 2024-08-17T17:28:16.922 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 246.44909806199999, 2024-08-17T17:28:16.922 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.129388477, 2024-08-17T17:28:16.922 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.922 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.922 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.922 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.922 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.922 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.922 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.395445+0000", 2024-08-17T17:28:16.922 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.922 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.923 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.923 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.923 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.395445+0000", 2024-08-17T17:28:16.923 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.923 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.923 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.923 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.923 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.395446+0000", 2024-08-17T17:28:16.923 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.1500000000000003e-07 2024-08-17T17:28:16.923 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.923 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.923 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.923 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.395454+0000", 2024-08-17T17:28:16.924 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7700000000000001e-06 2024-08-17T17:28:16.924 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.924 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.924 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.924 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509168+0000", 2024-08-17T17:28:16.924 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.11371437500000001 2024-08-17T17:28:16.924 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.924 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.924 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.924 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509196+0000", 2024-08-17T17:28:16.924 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8255999999999998e-05 2024-08-17T17:28:16.924 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.924 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.925 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.925 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509487+0000", 2024-08-17T17:28:16.925 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00029032400000000001 2024-08-17T17:28:16.925 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.925 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.925 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.925 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509507+0000", 2024-08-17T17:28:16.925 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9964000000000001e-05 2024-08-17T17:28:16.925 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.925 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.925 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.925 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.524782+0000", 2024-08-17T17:28:16.926 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.015275476 2024-08-17T17:28:16.926 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.926 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.926 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.926 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.524833+0000", 2024-08-17T17:28:16.926 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1397000000000003e-05 2024-08-17T17:28:16.926 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.926 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.926 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.926 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.926 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.926 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:25690 9.5 e1251/1247 9:aa569b13:::benchmark_data_smithi063_263014_object25689:head v 1251'1611, mlcod=1251'1609)", 2024-08-17T17:28:16.926 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:26:56.437973+0000", 2024-08-17T17:28:16.927 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 80.406569743000006, 2024-08-17T17:28:16.927 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.202847471, 2024-08-17T17:28:16.927 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.927 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.927 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.927 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.927 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.927 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.927 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.437973+0000", 2024-08-17T17:28:16.927 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.927 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.927 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.927 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.928 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.437973+0000", 2024-08-17T17:28:16.928 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.928 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.928 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.928 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.928 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.437974+0000", 2024-08-17T17:28:16.928 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8999999999999995e-07 2024-08-17T17:28:16.928 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.928 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.928 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.928 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.438031+0000", 2024-08-17T17:28:16.928 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6320000000000003e-05 2024-08-17T17:28:16.928 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.929 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.929 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.929 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.438042+0000", 2024-08-17T17:28:16.929 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1209999999999999e-05 2024-08-17T17:28:16.929 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.929 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.929 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.929 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.438049+0000", 2024-08-17T17:28:16.929 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3180000000000001e-06 2024-08-17T17:28:16.929 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.929 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.929 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.929 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.438111+0000", 2024-08-17T17:28:16.930 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1484999999999998e-05 2024-08-17T17:28:16.930 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.930 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.930 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.930 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.438131+0000", 2024-08-17T17:28:16.930 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0409999999999999e-05 2024-08-17T17:28:16.930 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.930 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.930 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.930 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.640763+0000", 2024-08-17T17:28:16.930 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.202632328 2024-08-17T17:28:16.930 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.930 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.931 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.931 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.640821+0000", 2024-08-17T17:28:16.931 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7510000000000003e-05 2024-08-17T17:28:16.931 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.931 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.931 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.931 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.931 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.931 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:25692 9.9 e1251/1247 9:909dc1e3:::benchmark_data_smithi063_263014_object25691:head v 1251'1639, mlcod=1251'1637)", 2024-08-17T17:28:16.931 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:26:56.440325+0000", 2024-08-17T17:28:16.931 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 80.404218330999996, 2024-08-17T17:28:16.931 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19897859300000001, 2024-08-17T17:28:16.931 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.440325+0000", 2024-08-17T17:28:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.440325+0000", 2024-08-17T17:28:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.932 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.933 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.933 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.933 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.440326+0000", 2024-08-17T17:28:16.933 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:16.933 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.02e-06 2024-08-17T17:28:16.934 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.934 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.934 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.934 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.440419+0000", 2024-08-17T17:28:16.934 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3542999999999995e-05 2024-08-17T17:28:16.934 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.934 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.934 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.934 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.637224+0000", 2024-08-17T17:28:16.934 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.196805127 2024-08-17T17:28:16.934 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.935 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.935 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.935 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.637238+0000", 2024-08-17T17:28:16.935 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3341e-05 2024-08-17T17:28:16.935 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.935 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.935 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.935 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.637298+0000", 2024-08-17T17:28:16.935 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0081999999999998e-05 2024-08-17T17:28:16.935 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.935 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.937 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.938 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.637319+0000", 2024-08-17T17:28:16.938 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1596000000000001e-05 2024-08-17T17:28:16.938 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.938 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.938 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.938 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.639257+0000", 2024-08-17T17:28:16.938 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0019377520000000001 2024-08-17T17:28:16.938 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.938 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.938 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.938 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.639303+0000", 2024-08-17T17:28:16.938 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6131999999999998e-05 2024-08-17T17:28:16.938 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.939 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.939 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.939 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.939 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.939 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:25698 9.5 e1251/1247 9:ae5e9e69:::benchmark_data_smithi063_263014_object25697:head v 1251'1612, mlcod=1251'1609)", 2024-08-17T17:28:16.939 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:26:56.441132+0000", 2024-08-17T17:28:16.939 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 80.403411328000004, 2024-08-17T17:28:16.939 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20151798300000001, 2024-08-17T17:28:16.939 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.939 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.939 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.939 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.939 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.940 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.940 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.441132+0000", 2024-08-17T17:28:16.940 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.940 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.940 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.940 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.940 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.441132+0000", 2024-08-17T17:28:16.940 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.940 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.940 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.940 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.940 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.441133+0000", 2024-08-17T17:28:16.940 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.9100000000000002e-07 2024-08-17T17:28:16.941 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.941 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.941 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.941 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.441167+0000", 2024-08-17T17:28:16.941 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4740999999999998e-05 2024-08-17T17:28:16.941 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.941 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.941 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.941 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.637228+0000", 2024-08-17T17:28:16.941 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.196060242 2024-08-17T17:28:16.941 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.941 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.941 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.941 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.637237+0000", 2024-08-17T17:28:16.942 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.662e-06 2024-08-17T17:28:16.942 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.942 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.942 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.942 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.638332+0000", 2024-08-17T17:28:16.942 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0010948119999999999 2024-08-17T17:28:16.942 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.942 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.942 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.942 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.638351+0000", 2024-08-17T17:28:16.942 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8592000000000001e-05 2024-08-17T17:28:16.942 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.943 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.943 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.943 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.642625+0000", 2024-08-17T17:28:16.943 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0042740950000000003 2024-08-17T17:28:16.943 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.943 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.943 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.943 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.642650+0000", 2024-08-17T17:28:16.943 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4947999999999999e-05 2024-08-17T17:28:16.943 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.943 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.943 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.943 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.943 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.944 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:45004 9.8 e1252/1247 9:152681de:::benchmark_data_smithi063_263014_object45003:head v 1252'2841, mlcod=1252'2838)", 2024-08-17T17:28:16.944 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:08.432336+0000", 2024-08-17T17:28:16.944 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 68.412207530000003, 2024-08-17T17:28:16.944 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19930678199999999, 2024-08-17T17:28:16.944 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.944 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.944 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.944 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.944 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.944 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.944 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.432336+0000", 2024-08-17T17:28:16.944 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.944 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.945 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.945 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.945 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.432336+0000", 2024-08-17T17:28:16.945 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.945 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.945 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.945 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.945 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.432337+0000", 2024-08-17T17:28:16.945 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3829999999999999e-06 2024-08-17T17:28:16.945 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.945 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.945 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.945 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.432500+0000", 2024-08-17T17:28:16.946 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016271499999999999 2024-08-17T17:28:16.946 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.946 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.946 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.946 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.432513+0000", 2024-08-17T17:28:16.946 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3733999999999999e-05 2024-08-17T17:28:16.946 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.946 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.946 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.946 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.432521+0000", 2024-08-17T17:28:16.946 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9969999999999998e-06 2024-08-17T17:28:16.946 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.947 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.947 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.947 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627136+0000", 2024-08-17T17:28:16.947 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19461452200000001 2024-08-17T17:28:16.947 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.947 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.947 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.947 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627166+0000", 2024-08-17T17:28:16.947 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0463999999999998e-05 2024-08-17T17:28:16.947 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.947 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.947 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.947 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.631598+0000", 2024-08-17T17:28:16.948 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044317799999999997 2024-08-17T17:28:16.948 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.948 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.948 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.948 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.631642+0000", 2024-08-17T17:28:16.948 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4187000000000002e-05 2024-08-17T17:28:16.948 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.948 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.948 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.948 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.948 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.948 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:45014 9.8 e1252/1247 9:1f47c66c:::benchmark_data_smithi063_263014_object45013:head v 1252'2842, mlcod=1252'2839)", 2024-08-17T17:28:16.948 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:08.436477+0000", 2024-08-17T17:28:16.949 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 68.408065922999995, 2024-08-17T17:28:16.949 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19519346600000001, 2024-08-17T17:28:16.949 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.949 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.949 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.949 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.949 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.949 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.949 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.436477+0000", 2024-08-17T17:28:16.949 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.949 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.949 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.949 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.950 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.436477+0000", 2024-08-17T17:28:16.950 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.950 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.950 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.950 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.950 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.436479+0000", 2024-08-17T17:28:16.950 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7150000000000001e-06 2024-08-17T17:28:16.950 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.950 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.950 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.950 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.436532+0000", 2024-08-17T17:28:16.950 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2707000000000002e-05 2024-08-17T17:28:16.950 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.951 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.951 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.951 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627125+0000", 2024-08-17T17:28:16.951 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19059372699999999 2024-08-17T17:28:16.951 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.951 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.951 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.951 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627142+0000", 2024-08-17T17:28:16.951 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.66e-05 2024-08-17T17:28:16.951 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.951 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.951 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.952 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.628712+0000", 2024-08-17T17:28:16.952 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001569892 2024-08-17T17:28:16.952 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.952 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.952 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.952 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.628731+0000", 2024-08-17T17:28:16.952 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9286000000000001e-05 2024-08-17T17:28:16.952 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.952 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.952 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.952 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.631647+0000", 2024-08-17T17:28:16.952 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0029157549999999999 2024-08-17T17:28:16.952 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.953 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.953 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.953 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.631671+0000", 2024-08-17T17:28:16.953 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3784000000000001e-05 2024-08-17T17:28:16.953 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.953 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.953 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.953 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.953 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.953 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:45017 9.b e1252/1247 9:d02eb79b:::benchmark_data_smithi063_263014_object45016:head v 1252'2851, mlcod=1252'2849)", 2024-08-17T17:28:16.953 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:08.439705+0000", 2024-08-17T17:28:16.953 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 68.404838006000006, 2024-08-17T17:28:16.954 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19242014800000001, 2024-08-17T17:28:16.954 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:28:16.954 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:28:16.954 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:28:16.954 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:28:16.954 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.954 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:28:16.954 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.439705+0000", 2024-08-17T17:28:16.954 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.954 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.954 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.954 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:28:16.954 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.439705+0000", 2024-08-17T17:28:16.955 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:28:16.955 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.955 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.955 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:28:16.955 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.439706+0000", 2024-08-17T17:28:16.955 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2580000000000001e-06 2024-08-17T17:28:16.955 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.955 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.955 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:28:16.955 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.439739+0000", 2024-08-17T17:28:16.955 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.2904999999999998e-05 2024-08-17T17:28:16.955 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.955 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.956 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:28:16.956 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627128+0000", 2024-08-17T17:28:16.956 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.187389153 2024-08-17T17:28:16.956 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.956 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.956 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:28:16.956 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627141+0000", 2024-08-17T17:28:16.956 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.27e-05 2024-08-17T17:28:16.956 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.956 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.956 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:28:16.956 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627246+0000", 2024-08-17T17:28:16.956 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000104615 2024-08-17T17:28:16.957 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.957 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.957 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:28:16.957 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627266+0000", 2024-08-17T17:28:16.957 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9843999999999998e-05 2024-08-17T17:28:16.957 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.957 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.957 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:28:16.957 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.632045+0000", 2024-08-17T17:28:16.957 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0047795329999999999 2024-08-17T17:28:16.957 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:28:16.957 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:28:16.957 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:28:16.958 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.632125+0000", 2024-08-17T17:28:16.958 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0140000000000002e-05 2024-08-17T17:28:16.958 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.958 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.958 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.958 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:28:16.958 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:28:16.958 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:28:16.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:16.960 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:16.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:17.061 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:17.105 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:17.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:17.207 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:17.240 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:17.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:17.341 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:17.376 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:17.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:17.478 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:17.517 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:17.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:17.619 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:17.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.654 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:17.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:17.755 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:17.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.795 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:17.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:17.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:17 smithi162 ceph-mon[104668]: pgmap v1752: 119 pgs: 119 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-08-17T17:28:17.896 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:17.933 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:17.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:17.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:17.989 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:17 smithi063 ceph-mon[109843]: pgmap v1752: 119 pgs: 119 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-08-17T17:28:17.990 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:17 smithi063 ceph-mon[114756]: pgmap v1752: 119 pgs: 119 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s 2024-08-17T17:28:18.035 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:18.075 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:18.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:18.177 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:18.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.211 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:18.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:18.312 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:18.343 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:18.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:18.444 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:18.466 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:18.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:18.569 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:18.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.606 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:18.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:18.707 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:18.741 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:18.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:18.843 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:18.878 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:18.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:18.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:18.981 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:19.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.015 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:19.015 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:19.116 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:19.151 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:19.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:19.253 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:19.288 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:19.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:19.389 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:19.422 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:19.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:19.481 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:19.524 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:19.560 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:19.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:19.662 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:19.701 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:19.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:19.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:19 smithi063 ceph-mon[109843]: pgmap v1753: 119 pgs: 119 active+clean; 7.5 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 103 B/s rd, 61 MiB/s wr, 1.08k op/s 2024-08-17T17:28:19.789 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:19 smithi063 ceph-mon[114756]: pgmap v1753: 119 pgs: 119 active+clean; 7.5 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 103 B/s rd, 61 MiB/s wr, 1.08k op/s 2024-08-17T17:28:19.803 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:19.844 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:19.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:19 smithi162 ceph-mon[104668]: pgmap v1753: 119 pgs: 119 active+clean; 7.5 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 103 B/s rd, 61 MiB/s wr, 1.08k op/s 2024-08-17T17:28:19.946 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:19.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:19.980 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:19.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:20.081 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:20.121 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:20.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:20.222 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:20.259 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:20.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:20.360 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:20.394 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:20.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:20.496 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:20.532 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:20.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:20.633 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:20.663 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:20.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:20.676 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:28:20.765 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:20.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.807 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:20.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:20.908 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:20.936 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:20.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:21.037 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:21.081 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:21.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:21.183 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:21.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.217 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:21.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:21.319 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:21.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.354 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:21.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:21.424 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:28:21.425 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:28:21.455 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:21.457 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:28:21.457 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:28:21.488 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:28:21.489 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:28:21.491 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:21.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:21.525 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:28:21.525 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:28:21.561 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:28:21.561 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:28:21.592 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:21.593 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:28:21.594 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:28:21.627 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:28:21.627 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:28:21.633 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:21.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:21.654 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:28:21.654 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:28:21.735 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:21.779 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:21 smithi063 ceph-mon[114756]: pgmap v1754: 119 pgs: 119 active+clean; 7.5 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 60 MiB/s wr, 1.07k op/s 2024-08-17T17:28:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:21 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4180479254' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:21 smithi063 ceph-mon[109843]: pgmap v1754: 119 pgs: 119 active+clean; 7.5 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 60 MiB/s wr, 1.07k op/s 2024-08-17T17:28:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:21 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4180479254' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:21.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:21.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:21 smithi162 ceph-mon[104668]: pgmap v1754: 119 pgs: 119 active+clean; 7.5 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 60 MiB/s wr, 1.07k op/s 2024-08-17T17:28:21.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:21 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4180479254' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:21.882 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:21.914 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:21.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:21.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:22.016 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:22.047 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:22.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:22.149 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:22.186 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:22.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:22.289 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:22.326 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:22.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:22.428 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:22.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.476 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:22.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:22.577 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:22.621 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:22.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:22.723 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:22.755 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:22.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:22.857 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:22.899 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:22.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:22.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:23.001 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:23.037 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:23.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:23.138 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:23.173 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:23.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:23.275 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:23.314 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:23.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:23.416 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:23.451 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:23.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:23.534 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:28:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:28:23] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.51.0" 2024-08-17T17:28:23.553 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:23.585 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:23.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:23.687 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:23.715 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:23.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:23.818 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:23.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:23 smithi162 ceph-mon[104668]: pgmap v1755: 119 pgs: 119 active+clean; 7.5 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 53 MiB/s wr, 1.08k op/s 2024-08-17T17:28:23.863 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:23 smithi063 ceph-mon[114756]: pgmap v1755: 119 pgs: 119 active+clean; 7.5 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 53 MiB/s wr, 1.08k op/s 2024-08-17T17:28:23.864 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:23 smithi063 ceph-mon[109843]: pgmap v1755: 119 pgs: 119 active+clean; 7.5 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 53 MiB/s wr, 1.08k op/s 2024-08-17T17:28:23.865 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:23.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:23.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:23.968 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:24.022 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:24.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:24.124 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:24.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.173 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:24.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:24.274 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:24.323 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:24.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:24.424 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:24.427 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:24.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.485 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:24.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:24.577 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:28:24.586 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:24.641 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:24.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:24.733 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:24 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/302028359' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:24.733 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:24 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:24.734 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:24 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/302028359' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:24.734 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:24 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:24.743 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:24.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.773 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:24.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:24.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:24 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/302028359' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:24.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:24 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:28:24.874 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:24.905 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:24.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:24.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:25.007 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:25.047 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:25.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:25.149 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:25.185 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:25.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:25.287 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:25.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.322 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:25.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:25.392 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:28:25.424 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:25.456 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:25.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:25.558 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:25.598 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:25.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:25.699 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:25.737 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:25.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:25.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:25 smithi063 ceph-mon[114756]: pgmap v1756: 119 pgs: 119 active+clean; 7.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 86 B/s rd, 41 MiB/s wr, 1.21k op/s 2024-08-17T17:28:25.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:25 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:25.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:25 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:28:25.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:25 smithi063 ceph-mon[114756]: osdmap e1262: 8 total, 8 up, 8 in 2024-08-17T17:28:25.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:25 smithi063 ceph-mon[109843]: pgmap v1756: 119 pgs: 119 active+clean; 7.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 86 B/s rd, 41 MiB/s wr, 1.21k op/s 2024-08-17T17:28:25.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:25 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:25.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:25 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:28:25.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:25 smithi063 ceph-mon[109843]: osdmap e1262: 8 total, 8 up, 8 in 2024-08-17T17:28:25.839 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:25.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:25 smithi162 ceph-mon[104668]: pgmap v1756: 119 pgs: 119 active+clean; 7.2 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 86 B/s rd, 41 MiB/s wr, 1.21k op/s 2024-08-17T17:28:25.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:25 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:25 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:28:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:25 smithi162 ceph-mon[104668]: osdmap e1262: 8 total, 8 up, 8 in 2024-08-17T17:28:25.872 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:25.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:25.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:25.974 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:26.017 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:26.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:26.119 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:26.152 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:26.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:26.253 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:26.302 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:26.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:26.404 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:26.440 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:26.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:26.542 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:26.587 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:26.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:26.689 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:26.710 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:26.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:26.812 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:26.847 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:26.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:26.949 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:26.984 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:26.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:26.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:27.086 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:27.130 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:27.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:27.232 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:27.273 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:27.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:27.375 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:27.408 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:27.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:27.510 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:27.538 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:27.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:27.640 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:27.670 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:27.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:27.697 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:27 smithi063 ceph-mon[109843]: pgmap v1758: 119 pgs: 119 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 20 MiB/s wr, 1.10k op/s 2024-08-17T17:28:27.697 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:27 smithi063 ceph-mon[114756]: pgmap v1758: 119 pgs: 119 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 20 MiB/s wr, 1.10k op/s 2024-08-17T17:28:27.772 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:27.797 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:27.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:27.838 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:27.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:27 smithi162 ceph-mon[104668]: pgmap v1758: 119 pgs: 119 active+clean; 7.1 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 102 B/s rd, 20 MiB/s wr, 1.10k op/s 2024-08-17T17:28:27.869 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:28:27.869 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:28:27.900 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:27.928 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:27.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:27.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:28.030 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:28.072 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:28.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:28.174 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:28.215 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:28.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:28.317 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:28.357 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:28.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:28.459 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:28.487 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:28.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:28.586 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:28:28.588 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:28.622 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:28.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:28.724 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:28.755 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:28.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:28.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:28 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1207050801' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:28:28.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:28 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:28:28.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:28 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1207050801' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:28:28.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:28 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:28:28.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:28 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1207050801' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:28:28.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:28 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:28:28.857 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:28.890 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:28.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:28.891 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:28.956 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:28:28.992 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:29.027 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:29.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:29.129 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:29.148 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:28:29.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.172 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:29.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:29.273 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:29.301 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:29.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:29.402 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:29.451 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:29.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:29.554 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:29.590 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:29.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:29.692 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:29.731 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:29.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:29.832 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:29.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:29 smithi162 ceph-mon[104668]: pgmap v1759: 119 pgs: 119 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 1.0 MiB/s wr, 1.10k op/s 2024-08-17T17:28:29.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:29 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:28:29.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:29 smithi162 ceph-mon[104668]: osdmap e1263: 8 total, 8 up, 8 in 2024-08-17T17:28:29.854 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:29.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:29.956 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:29.978 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:29.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:29.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:29 smithi063 ceph-mon[109843]: pgmap v1759: 119 pgs: 119 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 1.0 MiB/s wr, 1.10k op/s 2024-08-17T17:28:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:29 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:28:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:29 smithi063 ceph-mon[109843]: osdmap e1263: 8 total, 8 up, 8 in 2024-08-17T17:28:30.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:29 smithi063 ceph-mon[114756]: pgmap v1759: 119 pgs: 119 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 1.0 MiB/s wr, 1.10k op/s 2024-08-17T17:28:30.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:29 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:28:30.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:29 smithi063 ceph-mon[114756]: osdmap e1263: 8 total, 8 up, 8 in 2024-08-17T17:28:30.080 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:30.122 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:30.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:30.224 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:30.261 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:30.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:30.363 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:30.402 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:30.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:30.504 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:30.538 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:30.640 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:30.675 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:30.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:30.777 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:30.812 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:30.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:30.913 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:30.934 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:30.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:30.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:31.036 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:31.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.070 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:31.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:31.171 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:31.193 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:31.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:31.295 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:31.329 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:31.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:31.433 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:31.471 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:31.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:31.573 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:31.612 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:31.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:31.713 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:31.750 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:31.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:31.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:31 smithi162 ceph-mon[104668]: pgmap v1761: 119 pgs: 119 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:28:31.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:31.854 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:31.890 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:31.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:31.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:31.992 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:32.017 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:32.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:31 smithi063 ceph-mon[114756]: pgmap v1761: 119 pgs: 119 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:28:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:31 smithi063 ceph-mon[109843]: pgmap v1761: 119 pgs: 119 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:28:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:32.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:32.164 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:32.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:32.266 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:32.298 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:32.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:32.400 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:32.433 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:32.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:32.535 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:32.577 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:32.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:32.594 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:28:32.679 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:32.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.722 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:32.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:32.823 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:32.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:32.870 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:32.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:32.971 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:33.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.017 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:33.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:33.118 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:33.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.183 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:33.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:33.218 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:33.283 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:33.298 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:33.343 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:33.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:33.395 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:28:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:28:33] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.51.0" 2024-08-17T17:28:33.445 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:33.482 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:33.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:33.584 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:33.620 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:33.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:33.671 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:33 smithi063 ceph-mon[114756]: pgmap v1762: 119 pgs: 119 active+clean; 6.8 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2024-08-17T17:28:33.671 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:33 smithi063 ceph-mon[109843]: pgmap v1762: 119 pgs: 119 active+clean; 6.8 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2024-08-17T17:28:33.722 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:33.766 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:33.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:33.868 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:33.899 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:33.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:33.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:34.003 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:34.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.047 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:34.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:34.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:33 smithi162 ceph-mon[104668]: pgmap v1762: 119 pgs: 119 active+clean; 6.8 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2024-08-17T17:28:34.148 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:34.188 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:34.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:34.290 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:34.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.326 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:34.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:34.340 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:34.427 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:34.453 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:34.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:34.470 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:28:34.470 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:28:34.470 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:28:34.470 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:28:34.470 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:28:34.471 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:28:34.471 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:28:34.471 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:28:34.471 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:28:34.471 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:28:34.471 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:28:34.471 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:28:34.471 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:28:34.471 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:28:34.471 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:28:34.472 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:28:34.555 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:34.591 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:34.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:34.693 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:34.729 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:34.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:34.831 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:34.862 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:34.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:34.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:34.959 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:34 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3324461889' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:34.959 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:34 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3324461889' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:34.965 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:35.009 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:35.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:34 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3324461889' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:35.111 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:35.153 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:35.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:35.255 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:35.288 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:35.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:35.390 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:35.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.431 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:35.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:35.532 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:35.563 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:35.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:35.665 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:35.702 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:35.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:35.804 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:35.840 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:35.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:35.892 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:35 smithi063 ceph-mon[114756]: pgmap v1763: 119 pgs: 119 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:28:35.893 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:35 smithi063 ceph-mon[109843]: pgmap v1763: 119 pgs: 119 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:28:35.942 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:35.976 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:35.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:35.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:36.078 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:35 smithi162 ceph-mon[104668]: pgmap v1763: 119 pgs: 119 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:28:36.116 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:36.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:36.218 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:36.252 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:36.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:36.354 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:36.392 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:36.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:36.494 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:36.529 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:36.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:36.631 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:36.668 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:36.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:36.770 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:36.803 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:36.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:36.905 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:36.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:36.956 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:36.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:37.057 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:37.090 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:37.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:37.192 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:37.226 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:37.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:37.328 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:37.370 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:37.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:37.472 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:37.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.512 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:37.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:37.613 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:37.657 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:37.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:37.730 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:37 smithi063 ceph-mon[109843]: pgmap v1764: 119 pgs: 119 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:28:37.731 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:37 smithi063 ceph-mon[114756]: pgmap v1764: 119 pgs: 119 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:28:37.759 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:37.803 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:37.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:37.905 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:37.930 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:28:37.941 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:37.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:37.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:38.043 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:38.089 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:38.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:37 smithi162 ceph-mon[104668]: pgmap v1764: 119 pgs: 119 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:28:38.192 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:38.227 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:38.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:38.328 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:38.371 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:38.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:38.474 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:38.507 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:38.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:38.609 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:38.641 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:38.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:38.655 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:28:38.743 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:38.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.784 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:38.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:38.827 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:38 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/171501780' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:38.827 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:38 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:38.827 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:38 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/171501780' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:38.827 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:38 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:38.885 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:38.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:38.921 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:38.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:39.022 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:39.059 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:39.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:39.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:38 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/171501780' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:38 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:28:39.161 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:39.200 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:39.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:39.302 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:39.341 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:39.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:39.443 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:39.478 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:39.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:39.580 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:39.623 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:39.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:39.725 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:39.758 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:39.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:39.860 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:39.887 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:39 smithi063 ceph-mon[114756]: pgmap v1765: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:28:39.888 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:39 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:28:39.888 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:39 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:28:39.888 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:39 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:28:39.888 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:39 smithi063 ceph-mon[114756]: osdmap e1264: 8 total, 8 up, 8 in 2024-08-17T17:28:39.888 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:39 smithi063 ceph-mon[109843]: pgmap v1765: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:28:39.889 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:39 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:28:39.889 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:39 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:28:39.889 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:39 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:28:39.889 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:39 smithi063 ceph-mon[109843]: osdmap e1264: 8 total, 8 up, 8 in 2024-08-17T17:28:39.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:39.891 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:39.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:39.992 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:40.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.027 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:40.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:40.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:39 smithi162 ceph-mon[104668]: pgmap v1765: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:28:40.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:39 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:28:40.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:39 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:28:40.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:39 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:28:40.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:39 smithi162 ceph-mon[104668]: osdmap e1264: 8 total, 8 up, 8 in 2024-08-17T17:28:40.108 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:28:40.108 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:28:40.108 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:28:40.108 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:28:40.108 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:28:40.128 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:40.166 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:40.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:40.267 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:40.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:40.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:40.400 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:40.439 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:40.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:40.540 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:40.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.587 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:40.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:40.689 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:40.718 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:40.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:40.820 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:40.854 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:40.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:40.956 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:40.991 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:40.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:40.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:41.007 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:41.037 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:28:41.093 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:41.125 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:41.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:41.227 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:41.293 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:41.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:41.395 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:41.396 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:28:41.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.451 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:41.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:41.552 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:41.576 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:41.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:41.653 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:28:41.678 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:41.710 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:41.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:41.812 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:41.854 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:41.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:41.907 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:41 smithi063 ceph-mon[114756]: pgmap v1767: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:28:41.907 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:41 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1242283854' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:28:41.907 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:41 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:28:41.907 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:41 smithi063 ceph-mon[109843]: pgmap v1767: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:28:41.907 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:41 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1242283854' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:28:41.907 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:41 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:28:41.956 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:41.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:41.999 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:41.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:41 smithi162 ceph-mon[104668]: pgmap v1767: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-08-17T17:28:42.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:41 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1242283854' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:28:42.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:41 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:28:42.100 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:42.131 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:42.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:42.233 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:42.271 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:42.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:42.373 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:42.407 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:42.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:42.509 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:42.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.547 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:42.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:42.648 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:42.685 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:28:42.692 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:42.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:42.794 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:42.814 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:42.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:42.917 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:42.963 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:42.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:42.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:42 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:28:43.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:42 smithi063 ceph-mon[114756]: osdmap e1265: 8 total, 8 up, 8 in 2024-08-17T17:28:43.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:42 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:28:43.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:42 smithi063 ceph-mon[109843]: osdmap e1265: 8 total, 8 up, 8 in 2024-08-17T17:28:43.065 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:43.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:42 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:28:43.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:42 smithi162 ceph-mon[104668]: osdmap e1265: 8 total, 8 up, 8 in 2024-08-17T17:28:43.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.102 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:43.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:43.154 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:43.203 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:43.229 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:43.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:43.242 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:43.331 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:43.365 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:43.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:43.467 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:43.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.497 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:43.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:43.526 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:28:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:28:43] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.51.0" 2024-08-17T17:28:43.598 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:43.638 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:43.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:43.740 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:43.775 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:43.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:43.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:43 smithi063 ceph-mon[109843]: pgmap v1769: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2024-08-17T17:28:43.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:43 smithi063 ceph-mon[114756]: pgmap v1769: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2024-08-17T17:28:43.877 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:43.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:43.912 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:43.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:44.013 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:44.059 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:44.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:44.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:43 smithi162 ceph-mon[104668]: pgmap v1769: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2024-08-17T17:28:44.162 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:44.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.209 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:44.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:44.308 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:28:44.308 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:28:44.310 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:44.347 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:44.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:44.353 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:28:44.353 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:28:44.397 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:28:44.397 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:28:44.440 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:28:44.440 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:28:44.449 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:44.473 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:28:44.473 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:28:44.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.483 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:44.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:44.510 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:28:44.511 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:28:44.541 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:28:44.541 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:28:44.575 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:28:44.575 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:28:44.584 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:44.635 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:44.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:44.737 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:44.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:44 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3523291433' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:44.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:44 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3523291433' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:44.783 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:44.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:44.885 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:44.924 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:44.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:44.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:45.026 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:45.060 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:45.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:45.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:44 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3523291433' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:28:45.162 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:45.203 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:45.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:45.305 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:45.344 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:45.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:45.446 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:45.479 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:45.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:45.580 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:45.614 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:45.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:45.716 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:45.759 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:45.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:45.861 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:45.890 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:45.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:45.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:45.956 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:45 smithi063 ceph-mon[109843]: pgmap v1770: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:28:45.956 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:45 smithi063 ceph-mon[114756]: pgmap v1770: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:28:45.991 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:46.028 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:46.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:45 smithi162 ceph-mon[104668]: pgmap v1770: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:28:46.129 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:46.166 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:46.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:46.268 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:46.302 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:46.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:46.404 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:46.464 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:46.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:46.511 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:46.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:46.599 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:46.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:46.703 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:46.739 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:46.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:46.841 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:46.884 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:46.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:46.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:46.986 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:47.021 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:47.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:47.123 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:47.168 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:47.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:47.270 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:47.310 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:47.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:47.407 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:28:47.412 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:47.479 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:28:47.479 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:47.480 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:47 smithi063 ceph-mon[114756]: pgmap v1771: 119 pgs: 119 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2024-08-17T17:28:47.481 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:28:47.481 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:47.481 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:47 smithi063 ceph-mon[109843]: pgmap v1771: 119 pgs: 119 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2024-08-17T17:28:47.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.492 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:47.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:47.593 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:47.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.622 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:47.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:47.723 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:47.771 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:47.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:47.792 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:28:47.792 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:28:47.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:28:47.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:28:47.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:47 smithi162 ceph-mon[104668]: pgmap v1771: 119 pgs: 119 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2024-08-17T17:28:47.873 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:47.907 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:47.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:47.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:48.009 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:48.051 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:48.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:48.153 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:48.177 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:48.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:48.279 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:48.318 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:48.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:48.394 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:28:48.420 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:48.474 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:48.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:48.576 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:48.613 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:48.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:48.715 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:48.753 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:48.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:48.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:48 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3914179349' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:28:48.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:48 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:28:48.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:48 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3914179349' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:28:48.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:48 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:28:48.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:48 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3914179349' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:28:48.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:48 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:28:48.857 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:48.891 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:48.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:48.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:48.992 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:49.024 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:49.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:49.126 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:49.162 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:49.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:49.264 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:49.304 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:49.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:49.406 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:49.447 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:49.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:49.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:49 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:49.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:49 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:28:49.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:49 smithi063 ceph-mon[114756]: osdmap e1266: 8 total, 8 up, 8 in 2024-08-17T17:28:49.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:49 smithi063 ceph-mon[114756]: pgmap v1773: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.38k op/s 2024-08-17T17:28:49.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:49 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:49.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:49 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:28:49.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:49 smithi063 ceph-mon[109843]: osdmap e1266: 8 total, 8 up, 8 in 2024-08-17T17:28:49.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:49 smithi063 ceph-mon[109843]: pgmap v1773: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.38k op/s 2024-08-17T17:28:49.549 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:49.585 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:49.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:49.687 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:49.721 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:49.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:49.823 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:49 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:28:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:49 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:28:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:49 smithi162 ceph-mon[104668]: osdmap e1266: 8 total, 8 up, 8 in 2024-08-17T17:28:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:49 smithi162 ceph-mon[104668]: pgmap v1773: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.38k op/s 2024-08-17T17:28:49.856 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:49.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:49.957 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:49.998 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:49.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:49.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:50.100 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:50.140 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:50.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:50.242 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:50.258 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:28:50.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.288 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:50.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:50.390 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:50.425 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:50.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:50.527 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:50.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.588 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:50.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:50.689 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:50.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.729 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:50.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:50.831 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:50.862 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:50.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:50.963 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:50.997 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:50.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:50.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:51.099 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:51.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.133 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:51.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:51.235 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:51.268 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:51.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:51.370 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:51.403 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:51.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:51.505 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:51.541 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:51.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:51.643 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:51.686 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:51.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:51.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:51 smithi063 ceph-mon[114756]: pgmap v1774: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:28:51.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:51 smithi063 ceph-mon[109843]: pgmap v1774: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:28:51.787 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:51.812 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:51.818 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:51.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:51.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:51 smithi162 ceph-mon[104668]: pgmap v1774: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:28:51.920 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:51.954 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:51.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:51.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:52.056 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:52.094 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:52.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:52.196 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:52.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.231 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:52.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:52.332 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:52.361 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:52.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:52.463 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:52.500 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:52.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:52.601 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:52.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.634 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:52.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:52.735 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:52.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.765 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:52.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:52.826 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:52.865 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:52.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:52.897 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:52.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:52.998 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:53.039 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:53.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:53.141 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:53.171 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:28:53.176 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:53.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:53.277 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:53.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.340 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:53.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:53.441 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:53.474 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:53.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:53.576 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:53.621 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:53.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:53.647 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:53 smithi063 ceph-mon[109843]: pgmap v1775: 119 pgs: 119 active+clean; 5.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:28:53.647 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:53 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:28:53.647 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:28:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:28:53] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.51.0" 2024-08-17T17:28:53.648 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:53 smithi063 ceph-mon[114756]: pgmap v1775: 119 pgs: 119 active+clean; 5.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:28:53.648 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:53 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:28:53.724 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:53.752 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:53.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:53.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:53 smithi162 ceph-mon[104668]: pgmap v1775: 119 pgs: 119 active+clean; 5.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:28:53.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:53 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:28:53.854 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:53.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:53.898 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:53.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:53.999 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:54.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.038 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:54.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:54.139 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:54.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.178 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:54.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:54.279 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:54.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.330 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:54.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:54.431 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:54.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.468 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:54.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:54.569 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:54.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.609 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:54.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:54.710 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:54.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.746 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:54.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:54.847 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:54.879 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:54.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:54.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:54.981 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:55.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.042 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:55.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:55.143 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:55.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.205 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:55.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:55.307 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:55.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.351 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:55.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:55.452 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:55.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.480 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:55.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:55.581 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:55.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.618 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:55.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:55.719 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:55.755 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:55.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:55.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:55 smithi063 ceph-mon[114756]: pgmap v1776: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:28:55.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:55 smithi063 ceph-mon[109843]: pgmap v1776: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:28:55.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:55 smithi162 ceph-mon[104668]: pgmap v1776: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:28:55.857 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:55.886 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:55.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:55.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:55.988 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:56.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.032 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:56.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:56.133 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:56.169 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:56.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:56.270 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:56.303 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:56.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:56.405 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:56.440 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:56.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:28:56.500 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:28:56.542 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:56.574 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:56.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:56.676 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:56.722 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:56.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:56.758 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:56 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2126914045' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:28:56.758 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:56 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:28:56.759 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:56 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2126914045' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:28:56.759 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:56 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:28:56.824 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:56.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:56 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2126914045' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:28:56.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:56 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:28:56.863 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:56.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:56.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:56.965 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:56.998 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:56.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:57.100 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:57.123 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:57.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:57.224 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:57.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.266 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:57.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:57.367 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:57.388 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:57.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:57.489 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:57.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.523 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:57.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:57.607 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:28:57.624 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:57.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.691 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:57.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:57.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:57 smithi063 ceph-mon[114756]: pgmap v1777: 119 pgs: 119 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:28:57.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:57 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:28:57.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:57 smithi063 ceph-mon[114756]: osdmap e1267: 8 total, 8 up, 8 in 2024-08-17T17:28:57.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:57 smithi063 ceph-mon[109843]: pgmap v1777: 119 pgs: 119 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:28:57.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:57 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:28:57.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:57 smithi063 ceph-mon[109843]: osdmap e1267: 8 total, 8 up, 8 in 2024-08-17T17:28:57.792 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:57.839 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:57.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:57.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:57 smithi162 ceph-mon[104668]: pgmap v1777: 119 pgs: 119 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:28:57.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:57 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:28:57.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:57 smithi162 ceph-mon[104668]: osdmap e1267: 8 total, 8 up, 8 in 2024-08-17T17:28:57.940 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:57.989 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:57.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:57.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:58.091 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:58.126 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:58.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:58.227 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:58.252 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:58.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:58.353 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:58.391 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:58.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:58.493 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:58.514 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:58.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:58.517 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:28:58.517 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:28:58.616 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:58.638 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:58.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:58.740 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:28:58.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.772 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:58.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:28:58.873 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:28:58.906 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:58.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:58.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:28:59.008 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:28:59.046 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:59.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:28:59.148 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:28:59.207 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:59.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:28:59.309 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:59.315 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:28:59.340 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:59.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:59.345 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:28:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=cleanup t=2024-08-17T17:28:59.009307813Z level=info msg="Completed cleanup jobs" duration=5.545154ms 2024-08-17T17:28:59.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:28:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=grafana.update.checker t=2024-08-17T17:28:59.149042795Z level=info msg="Update check succeeded" duration=71.482395ms 2024-08-17T17:28:59.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:28:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=plugins.update.checker t=2024-08-17T17:28:59.153347043Z level=info msg="Update check succeeded" duration=66.272114ms 2024-08-17T17:28:59.442 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:28:59.512 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:59.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:28:59.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:28:59 smithi063 ceph-mon[114756]: pgmap v1779: 119 pgs: 119 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:28:59.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:28:59 smithi063 ceph-mon[109843]: pgmap v1779: 119 pgs: 119 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:28:59.614 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:28:59.642 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:28:59.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:28:59.660 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:59.673 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:28:59.674 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:28:59.719 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:28:59.719 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:28:59.742 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:28:59.744 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:28:59.755 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:28:59.755 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:28:59.805 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:28:59.805 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:28:59.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.808 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:59.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:28:59.836 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:28:59.836 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:28:59.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:28:59 smithi162 ceph-mon[104668]: pgmap v1779: 119 pgs: 119 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:28:59.869 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:28:59.869 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:28:59.902 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:28:59.903 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:28:59.909 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:28:59.937 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:28:59.937 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:28:59.950 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:28:59.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:28:59.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:00.052 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:00.094 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:00.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:00.195 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:00.232 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:00.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:00.334 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:00.368 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:00.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:00.470 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:00.506 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:00.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:00.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:00 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2559346957' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:00.608 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:00.646 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:00.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:00.748 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:00.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:00 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2559346957' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:00.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:00 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2559346957' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:00.798 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:00.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:00.900 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:00.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:00.937 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:00.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:01.038 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:01.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.074 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:01.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:01.175 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:01.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.215 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:01.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:01.316 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:01.356 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:01.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:01.458 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:01.482 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:01.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:01.584 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:01.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.611 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:01.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:01.712 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:01.750 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:01.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:01 smithi063 ceph-mon[114756]: pgmap v1780: 119 pgs: 119 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:29:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:01 smithi063 ceph-mon[109843]: pgmap v1780: 119 pgs: 119 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:29:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:01.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:01 smithi162 ceph-mon[104668]: pgmap v1780: 119 pgs: 119 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:29:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:01.852 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:01.872 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:01.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:01.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:01.974 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:02.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.009 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:02.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:02.110 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:02.170 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:02.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:02.272 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:02.318 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:02.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:02.419 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:02.460 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:02.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:02.562 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:02.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.603 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:02.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:02.704 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:02.738 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:02.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:02.840 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:02.877 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:02.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:02.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:02.978 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:03.015 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:03.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:03.117 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:03.160 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:03.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:03.262 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:03.297 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:03.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:03.399 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:03.436 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:03.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:03.513 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:29:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:29:03] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.51.0" 2024-08-17T17:29:03.514 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:03 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:03.514 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:03 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:03.514 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:03 smithi063 ceph-mon[114756]: pgmap v1781: 119 pgs: 119 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:29:03.514 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:03 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:03.515 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:03 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:03.515 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:03 smithi063 ceph-mon[109843]: pgmap v1781: 119 pgs: 119 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:29:03.537 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:03.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.575 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:03.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:03.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:03 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:03.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:03 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:03.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:03 smithi162 ceph-mon[104668]: pgmap v1781: 119 pgs: 119 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-08-17T17:29:03.676 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:03.718 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:03.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:03.819 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:03.820 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:03.848 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:03.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:03.866 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:29:03.950 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:03.992 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:03.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:03.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:04.093 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:04.141 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:04.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:04.243 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:04.282 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:04.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:04.384 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:04.417 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:04.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:04.519 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:04.565 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:04.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:04.667 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:04.713 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:04.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:04.815 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:04.849 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:04.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:04.951 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:04.991 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:04.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:04.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:05.093 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:05.130 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:05.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:05.232 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:05.263 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:05.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:05.398 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:05.434 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:05.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:05.492 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:29:05.537 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:05.575 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:05.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:05.677 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:05.717 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:05.718 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:05 smithi063 ceph-mon[109843]: pgmap v1782: 119 pgs: 119 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 999 op/s 2024-08-17T17:29:05.718 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:05 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2548192367' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:29:05.718 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:05 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:29:05.718 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:05 smithi063 ceph-mon[114756]: pgmap v1782: 119 pgs: 119 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 999 op/s 2024-08-17T17:29:05.718 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:05 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2548192367' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:29:05.719 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:05 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:29:05.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:05.788 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:05 smithi162 ceph-mon[104668]: pgmap v1782: 119 pgs: 119 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 999 op/s 2024-08-17T17:29:05.788 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:05 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2548192367' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:29:05.788 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:05 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:29:05.820 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:05.852 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:05.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:05.954 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:05.993 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:05.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:05.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:06.095 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:06.128 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:06.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:06.230 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:06.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.268 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:06.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:06.368 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:06.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.409 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:06.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:06.510 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:06.530 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:29:06.551 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:06.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:06.653 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:06.685 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:06.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:06.742 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:06 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:29:06.742 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:06 smithi063 ceph-mon[109843]: osdmap e1268: 8 total, 8 up, 8 in 2024-08-17T17:29:06.743 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:06 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:29:06.743 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:06 smithi063 ceph-mon[114756]: osdmap e1268: 8 total, 8 up, 8 in 2024-08-17T17:29:06.787 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:06.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.812 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:06.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:06.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:06 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:29:06.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:06 smithi162 ceph-mon[104668]: osdmap e1268: 8 total, 8 up, 8 in 2024-08-17T17:29:06.913 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:06.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:06.976 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:06.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:07.077 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:07.109 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:07.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:07.210 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:07.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.251 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:07.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:07.352 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:07.389 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:07.428 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:07.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:07.445 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:29:07.446 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:29:07.446 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:29:07.446 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:29:07.446 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:29:07.516 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:29:07.531 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:07.573 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:07.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:07.674 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:07.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.708 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:07.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:07.725 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:07 smithi063 ceph-mon[109843]: pgmap v1784: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-08-17T17:29:07.725 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:07 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3812761621' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:07.726 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:07 smithi063 ceph-mon[114756]: pgmap v1784: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-08-17T17:29:07.726 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:07 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3812761621' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:07.809 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:07.840 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:07.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:07.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:07 smithi162 ceph-mon[104668]: pgmap v1784: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-08-17T17:29:07.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:07 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3812761621' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:07.941 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:07.983 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:07.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:07.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:08.084 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:08.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.126 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:08.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:08.227 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:08.269 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:08.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:08.370 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:08.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.401 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:08.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:08.502 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:08.538 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:08.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:08.610 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:08 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:08.610 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:08 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3812761621' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:08.610 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:08 smithi063 ceph-mon[109843]: osdmap e1269: 8 total, 8 up, 8 in 2024-08-17T17:29:08.610 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:08 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:08.611 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:08 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3812761621' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:08.611 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:08 smithi063 ceph-mon[114756]: osdmap e1269: 8 total, 8 up, 8 in 2024-08-17T17:29:08.640 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:08.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.678 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:08.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:08.779 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:08.821 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:08.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:08.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:08 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:08.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:08 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3812761621' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:08.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:08 smithi162 ceph-mon[104668]: osdmap e1269: 8 total, 8 up, 8 in 2024-08-17T17:29:08.923 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:08.956 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:08.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:08.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:09.058 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:09.066 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:09.069 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:29:09.109 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:09.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:09.211 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:09.267 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:09.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:09.369 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:09.407 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:09.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:09.509 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:09.571 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:09.672 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:09.703 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:09.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:09.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:09 smithi063 ceph-mon[109843]: pgmap v1786: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.27k op/s 2024-08-17T17:29:09.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:09 smithi063 ceph-mon[114756]: pgmap v1786: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.27k op/s 2024-08-17T17:29:09.805 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:09.838 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:09.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:09.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:09 smithi162 ceph-mon[104668]: pgmap v1786: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.27k op/s 2024-08-17T17:29:09.940 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:09.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:09.975 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:09.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:10.076 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:10.121 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:10.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:10.223 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:10.254 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:10.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:10.356 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:10.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.391 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:10.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:10.492 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:10.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.534 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:10.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:10.635 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:10.671 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:10.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:10.773 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:10.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.810 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:10.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:10.911 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:10.946 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:10.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:10.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:11.049 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:11.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.090 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:11.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:11.191 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:11.226 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:11.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:11.328 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:11.365 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:11.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:11.467 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:11.526 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:11.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:11.527 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:29:11.629 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:11.639 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:11 smithi063 ceph-mon[114756]: pgmap v1787: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:29:11.639 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:11 smithi063 ceph-mon[109843]: pgmap v1787: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:29:11.681 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:11.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:11.783 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:11.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.834 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:11.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:11.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:11 smithi162 ceph-mon[104668]: pgmap v1787: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:29:11.935 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:11.968 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:11.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:11.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:12.069 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:12.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.104 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:12.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:12.205 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:12.238 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:12.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:12.340 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:12.371 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:12.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:12.473 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:12.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.515 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:12.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:12.616 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:12.649 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:12.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:12.751 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:12.787 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:12.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:12.889 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:12.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:12.925 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:12.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:13.026 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:13.037 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:13.051 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:13.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:13.123 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:29:13.123 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:29:13.153 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:13.165 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:29:13.165 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:29:13.200 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:29:13.200 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:29:13.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.211 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:13.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:13.235 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:29:13.235 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:29:13.271 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:29:13.271 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:29:13.302 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:29:13.302 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:29:13.313 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:13.335 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:29:13.335 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:29:13.347 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:13.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:13.371 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:29:13.372 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:29:13.448 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:13.482 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:13.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:29:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:29:13] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.51.0" 2024-08-17T17:29:13.585 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:13.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.620 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:13.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:13.721 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:13.756 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:13.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:13 smithi162 ceph-mon[104668]: pgmap v1788: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2024-08-17T17:29:13.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:13 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2244271766' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:13.857 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:13.862 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:13 smithi063 ceph-mon[114756]: pgmap v1788: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2024-08-17T17:29:13.862 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:13 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2244271766' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:13.862 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:13 smithi063 ceph-mon[109843]: pgmap v1788: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2024-08-17T17:29:13.862 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:13 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2244271766' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:13.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:13.890 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:13.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:13.991 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:14.033 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:14.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:14.135 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:14.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.172 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:14.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:14.273 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:14.313 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:14.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:14.415 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:14.449 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:14.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:14.551 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:14.590 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:14.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:14.692 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:14.728 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:14.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:14.829 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:14.858 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:14.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:14.961 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:14.996 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:14.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:14.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:15.099 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:15.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.147 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:15.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:15.248 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:15.284 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:15.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:15.386 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:15.424 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:15.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:15.526 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:15.558 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:15.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:15.660 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:15.694 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:15.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:15.698 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:15 smithi063 ceph-mon[114756]: pgmap v1789: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:29:15.699 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:15 smithi063 ceph-mon[109843]: pgmap v1789: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:29:15.796 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:15.832 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:15.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:15.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:15 smithi162 ceph-mon[104668]: pgmap v1789: 119 pgs: 119 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.21k op/s 2024-08-17T17:29:15.933 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:15.978 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:15.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:15.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:16.080 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:16.122 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:16.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:16.224 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:16.269 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:16.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:16.371 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:16.405 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:16.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:16.506 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:16.546 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:16.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:16.572 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:29:16.648 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:16.689 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:16.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:16.780 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:29:16.791 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:16.814 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:16 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3074568235' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:29:16.814 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:16.815 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:16 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3074568235' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:29:16.815 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:16 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3074568235' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:29:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:16.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.856 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:16.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:16.957 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:16.978 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:16.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:16.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:17.080 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:17.116 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:17.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:17.218 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:17.252 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:17.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:17.354 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:17.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.392 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:17.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:17.493 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:17.531 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:17.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:17.633 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:17.675 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:17.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:17.777 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:17.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.817 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:17.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:17.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:17 smithi162 ceph-mon[104668]: pgmap v1790: 119 pgs: 119 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:29:17.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:17 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3074568235' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:29:17.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:17 smithi162 ceph-mon[104668]: osdmap e1270: 8 total, 8 up, 8 in 2024-08-17T17:29:17.918 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:17.963 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:17.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:17.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:17 smithi063 ceph-mon[109843]: pgmap v1790: 119 pgs: 119 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:29:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:17 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3074568235' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:29:18.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:17 smithi063 ceph-mon[109843]: osdmap e1270: 8 total, 8 up, 8 in 2024-08-17T17:29:18.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:17 smithi063 ceph-mon[114756]: pgmap v1790: 119 pgs: 119 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:29:18.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:17 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3074568235' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:29:18.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:17 smithi063 ceph-mon[114756]: osdmap e1270: 8 total, 8 up, 8 in 2024-08-17T17:29:18.066 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:18.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.093 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:18.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:18.194 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:18.231 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:18.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:18.333 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:18.365 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:18.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:18.467 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:18.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.507 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:18.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:18.608 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:18.641 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:18.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:18.743 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:18.784 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:18.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:18.886 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:18.917 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:18.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:18.918 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:19.019 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:19.055 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:19.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:19.157 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:19.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.198 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:19.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:19.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:19 smithi063 ceph-mon[109843]: pgmap v1792: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:29:19.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:19 smithi063 ceph-mon[114756]: pgmap v1792: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:29:19.300 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:19.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:19 smithi162 ceph-mon[104668]: pgmap v1792: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:29:19.350 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:19.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:19.452 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:19.512 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:19.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:19.614 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:19.650 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:19.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:19.716 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:19.751 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:19.786 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:19.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:19.888 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:19.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:19.931 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:19.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:20.032 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:20.064 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:20.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:20.165 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:20.197 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:20.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:20.299 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:20.325 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:20.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:20.427 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:20.445 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:20 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:20.445 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:20 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:20.445 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:20 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:20.445 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:20 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:20.469 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:20.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:20.571 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:20.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:20 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:20.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:20 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:20.600 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:20.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:20.702 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:20.712 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:29:20.741 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:20.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:20.843 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:20.872 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:20.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:20.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:20.974 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:21.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.020 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:21.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:21.121 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:21.157 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:21.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:21.259 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:21.302 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:21.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:21.404 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:21.434 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:21 smithi063 ceph-mon[114756]: pgmap v1793: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:29:21.434 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:21 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:21.434 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:21 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:29:21.434 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:21 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:21.434 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:21 smithi063 ceph-mon[109843]: pgmap v1793: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:29:21.435 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:21 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:21.435 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:21 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:29:21.435 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:21 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:21.438 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:21.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:21.442 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:29:21.442 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:29:21.539 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:21.576 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:21.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:21.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:21 smithi162 ceph-mon[104668]: pgmap v1793: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:29:21.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:21 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:29:21.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:21 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:29:21.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:21 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:21.677 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:21.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.713 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:21.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:21.814 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:21.852 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:21.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:21.953 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:21.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:21.985 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:21.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:22.086 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:22.120 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:22.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:22.222 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:22.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.287 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:22.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:22.388 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:22.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.423 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:22.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:22.524 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:22.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.564 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:22.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:22.665 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:22.696 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:22.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:22.798 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:22.831 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:22.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:22.933 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:22.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:22.968 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:22.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:22.981 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:29:23.017 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:23.068 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:23.100 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:23.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:23.201 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:23.249 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:23.252 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:23.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:23.356 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:23.413 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:23.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:23.511 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:29:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:29:23] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.51.0" 2024-08-17T17:29:23.511 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:23 smithi063 ceph-mon[114756]: pgmap v1794: 119 pgs: 119 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:29:23.514 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:23 smithi063 ceph-mon[109843]: pgmap v1794: 119 pgs: 119 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:29:23.514 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:23.541 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:23.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:23.642 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:23.699 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:23.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:23.801 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:23.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:23 smithi162 ceph-mon[104668]: pgmap v1794: 119 pgs: 119 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:29:23.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.849 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:23.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:23.950 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:23.988 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:23.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:23.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:24.090 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:24.130 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:24.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:24.232 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:24.276 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:24.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:24.378 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:24.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.411 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:24.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:24.513 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:24.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.548 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:24.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:24.649 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:24.693 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:24.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:24.754 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:24 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2064427974' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:29:24.754 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:24 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:29:24.754 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:24 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2064427974' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:29:24.754 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:24 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:29:24.795 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:24.824 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:24.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:24.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:24 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2064427974' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:29:24.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:24 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:29:24.926 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:24.966 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:24.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:24.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:25.068 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:25.103 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:25.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:25.205 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:25.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.239 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:25.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:25.340 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:25.375 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:25.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:25.477 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:25.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.518 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:25.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:25.619 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:25.651 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:25 smithi063 ceph-mon[114756]: pgmap v1795: 119 pgs: 119 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:29:25.651 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:25 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:25.651 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:25 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:29:25.651 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:25 smithi063 ceph-mon[114756]: osdmap e1271: 8 total, 8 up, 8 in 2024-08-17T17:29:25.651 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:25 smithi063 ceph-mon[109843]: pgmap v1795: 119 pgs: 119 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:29:25.651 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:25 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:25.651 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:25 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:29:25.651 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:25 smithi063 ceph-mon[109843]: osdmap e1271: 8 total, 8 up, 8 in 2024-08-17T17:29:25.659 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:25.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:25.761 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:25.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.795 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:25.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:25.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:25 smithi162 ceph-mon[104668]: pgmap v1795: 119 pgs: 119 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s 2024-08-17T17:29:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:25 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:25 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:29:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:25 smithi162 ceph-mon[104668]: osdmap e1271: 8 total, 8 up, 8 in 2024-08-17T17:29:25.895 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:25.933 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:25.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:25.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:26.035 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:26.076 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:26.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:26.144 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:29:26.177 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:26.199 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:26.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:26.301 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:26.339 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:26.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:26.441 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:26.479 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:26.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:26.581 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:26.605 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:26.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:26.707 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:26.728 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:26.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:26.830 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:26.862 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:26.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:26.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:26.964 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:27.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.001 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:27.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:27.101 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:27.138 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:27.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:27.241 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:27.277 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:27.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:27.379 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:27.415 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:27.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:27.517 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:27.548 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:27.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:27.650 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:27.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.686 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:27.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:27.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:27 smithi063 ceph-mon[114756]: pgmap v1797: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:29:27.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:27 smithi063 ceph-mon[109843]: pgmap v1797: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:29:27.787 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:27.822 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:27.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:27.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:27 smithi162 ceph-mon[104668]: pgmap v1797: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.12k op/s 2024-08-17T17:29:27.924 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:27.963 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:27.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:27.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:28.067 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:28.103 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:28.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:28.205 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:28.242 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:28.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:28.344 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:28.371 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:28.372 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:28.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:28.474 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:28.502 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:28.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:28.604 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:28.644 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:28.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:28.746 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:28.782 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:28.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:28.884 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:28.915 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:28.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:28.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:29.018 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:29.052 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:29.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:29.153 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:29.200 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:29.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:29.302 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:29.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.339 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:29.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:29.440 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:29.472 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:29.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:29.574 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:29.613 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:29.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:29.617 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:29 smithi063 ceph-mon[109843]: pgmap v1798: 119 pgs: 119 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:29:29.620 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:29 smithi063 ceph-mon[114756]: pgmap v1798: 119 pgs: 119 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:29:29.715 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:29.743 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:29.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:29.845 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:29.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:29 smithi162 ceph-mon[104668]: pgmap v1798: 119 pgs: 119 active+clean; 2.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:29:29.895 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:29.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:29.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:29.997 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:30.027 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:30.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:30.118 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:29:30.118 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:29:30.129 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:30.156 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:30.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:30.158 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:29:30.158 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:29:30.194 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:29:30.195 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:29:30.221 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:29:30.222 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:29:30.258 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:30.266 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:29:30.266 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:29:30.285 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:30.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:30.305 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:29:30.305 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:29:30.344 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:29:30.344 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:29:30.370 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:29:30.384 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:29:30.384 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:29:30.386 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:30.433 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:30.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:30.534 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:30.535 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:29:30.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.588 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:30.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:30.689 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:30.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.718 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:30.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:30.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:30 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2359366113' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:29:30.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:30 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:29:30.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:30 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3178761203' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:30.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:30 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2359366113' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:29:30.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:30 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:29:30.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:30 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3178761203' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:30.819 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:30.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:30 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2359366113' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:29:30.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:30 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:29:30.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:30 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3178761203' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:30.859 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:30.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:30.961 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:30.994 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:30.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:30.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:31.096 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:31.134 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:31.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:31.236 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:31.267 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:31.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:31.369 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:31.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.406 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:31.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:31.443 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:29:31.443 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:29:31.508 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:31.537 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:31.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:31.639 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:31.677 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:31.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:31.779 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:31.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:31 smithi063 ceph-mon[114756]: pgmap v1799: 119 pgs: 119 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:29:31.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:31 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:29:31.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:31 smithi063 ceph-mon[114756]: osdmap e1272: 8 total, 8 up, 8 in 2024-08-17T17:29:31.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:31 smithi063 ceph-mon[109843]: pgmap v1799: 119 pgs: 119 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:29:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:31 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:29:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:31 smithi063 ceph-mon[109843]: osdmap e1272: 8 total, 8 up, 8 in 2024-08-17T17:29:31.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:31.820 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:31.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:31.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:31 smithi162 ceph-mon[104668]: pgmap v1799: 119 pgs: 119 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:29:31.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:31 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:29:31.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:31 smithi162 ceph-mon[104668]: osdmap e1272: 8 total, 8 up, 8 in 2024-08-17T17:29:31.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:31.921 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:31.958 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:31.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:31.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:32.060 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:32.101 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:32.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:32.203 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:32.241 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:32.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:32.343 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:32.371 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:32.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:32.473 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:32.494 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:32.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:32.596 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:32.626 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:32.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:32.728 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:32.767 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:32.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:32.869 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:32.918 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:32.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:32.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:33.019 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:33.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.058 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:33.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:33.159 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:33.193 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:33.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:33.276 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:33.294 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:33.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.365 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:33.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:33.440 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:29:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:29:33] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.51.0" 2024-08-17T17:29:33.467 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:33.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.511 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:33.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:33.597 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:33.612 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:33.660 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:33.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:33.765 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:33.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:33 smithi063 ceph-mon[114756]: pgmap v1801: 119 pgs: 119 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 824 op/s 2024-08-17T17:29:33.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:33 smithi063 ceph-mon[109843]: pgmap v1801: 119 pgs: 119 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 824 op/s 2024-08-17T17:29:33.802 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:33.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:33.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:33 smithi162 ceph-mon[104668]: pgmap v1801: 119 pgs: 119 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 824 op/s 2024-08-17T17:29:33.904 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:33.943 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:33.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:33.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:34.045 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:34.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.096 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:34.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:34.198 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:34.203 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:29:34.203 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:29:34.203 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.203 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:208303 9.4 e1272/1247 9:2cde31f5:::benchmark_data_smithi063_263014_object83965:head v 1272'12975, mlcod=1272'12971)", 2024-08-17T17:29:34.203 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:29:34.189591+0000", 2024-08-17T17:29:34.203 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.011018032000000001, 2024-08-17T17:29:34.203 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011044939, 2024-08-17T17:29:34.203 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:29:34.204 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:29:34.204 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:29:34.204 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:29:34.204 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.204 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:29:34.204 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.189591+0000", 2024-08-17T17:29:34.204 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.204 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.204 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.204 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:29:34.204 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.189591+0000", 2024-08-17T17:29:34.204 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.205 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.205 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.205 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:29:34.205 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.189592+0000", 2024-08-17T17:29:34.205 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.8700000000000004e-07 2024-08-17T17:29:34.205 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.205 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.205 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:29:34.205 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.189598+0000", 2024-08-17T17:29:34.205 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5979999999999996e-06 2024-08-17T17:29:34.205 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.205 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.205 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:29:34.206 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.189611+0000", 2024-08-17T17:29:34.206 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.296e-05 2024-08-17T17:29:34.206 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.206 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.206 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:29:34.206 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.189619+0000", 2024-08-17T17:29:34.206 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9710000000000004e-06 2024-08-17T17:29:34.206 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.206 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.206 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:29:34.206 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.189836+0000", 2024-08-17T17:29:34.206 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000217493 2024-08-17T17:29:34.207 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.207 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.207 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:29:34.207 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.189856+0000", 2024-08-17T17:29:34.207 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9408000000000001e-05 2024-08-17T17:29:34.207 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.207 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:29:34.207 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.207 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.207 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.207 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:208304 9.c 9:322e9c56:::benchmark_data_smithi063_263014_object83966:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1272)", 2024-08-17T17:29:34.207 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:29:34.189696+0000", 2024-08-17T17:29:34.208 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.010913424, 2024-08-17T17:29:34.208 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.010998346000000001, 2024-08-17T17:29:34.208 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:29:34.208 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:29:34.208 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:29:34.208 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:29:34.208 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:29:34.208 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:29:34.208 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 208304 2024-08-17T17:29:34.208 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.208 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:29:34.208 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.209 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:29:34.209 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.189696+0000", 2024-08-17T17:29:34.209 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.209 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.209 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.209 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:29:34.209 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.189696+0000", 2024-08-17T17:29:34.209 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.209 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.209 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.209 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:29:34.209 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.189697+0000", 2024-08-17T17:29:34.210 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5200000000000001e-06 2024-08-17T17:29:34.210 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.210 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.210 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:29:34.210 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.189701+0000", 2024-08-17T17:29:34.210 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8190000000000002e-06 2024-08-17T17:29:34.210 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.210 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.210 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:29:34.210 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.189711+0000", 2024-08-17T17:29:34.210 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9569999999999997e-06 2024-08-17T17:29:34.210 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.211 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.211 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:29:34.211 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.189718+0000", 2024-08-17T17:29:34.211 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0350000000000004e-06 2024-08-17T17:29:34.211 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.211 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.211 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:29:34.211 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190627+0000", 2024-08-17T17:29:34.211 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00090850900000000005 2024-08-17T17:29:34.211 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.211 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.211 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:29:34.211 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190744+0000", 2024-08-17T17:29:34.212 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000117599 2024-08-17T17:29:34.212 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.212 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.212 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:29:34.212 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190855+0000", 2024-08-17T17:29:34.212 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000110844 2024-08-17T17:29:34.212 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.212 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:29:34.212 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.212 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.212 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.212 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:208309 9.1 9:81cc68f1:::benchmark_data_smithi063_263014_object83971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1272)", 2024-08-17T17:29:34.212 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:29:34.190535+0000", 2024-08-17T17:29:34.213 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.010074002, 2024-08-17T17:29:34.213 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.010224171000000001, 2024-08-17T17:29:34.213 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:29:34.213 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:29:34.213 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:29:34.213 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:29:34.213 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:29:34.213 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:29:34.213 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 208309 2024-08-17T17:29:34.213 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.213 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:29:34.213 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.213 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:29:34.213 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190535+0000", 2024-08-17T17:29:34.214 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.214 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.214 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.214 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:29:34.214 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190535+0000", 2024-08-17T17:29:34.214 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.214 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.214 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.214 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:29:34.214 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190536+0000", 2024-08-17T17:29:34.214 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2099999999999995e-07 2024-08-17T17:29:34.214 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.214 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.215 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:29:34.215 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190540+0000", 2024-08-17T17:29:34.215 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6909999999999999e-06 2024-08-17T17:29:34.215 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.215 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.215 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:29:34.215 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190550+0000", 2024-08-17T17:29:34.215 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0729e-05 2024-08-17T17:29:34.215 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.215 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.215 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:29:34.215 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190557+0000", 2024-08-17T17:29:34.215 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6200000000000001e-06 2024-08-17T17:29:34.216 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.216 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.216 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:29:34.216 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190667+0000", 2024-08-17T17:29:34.216 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000109425 2024-08-17T17:29:34.216 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.216 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.216 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:29:34.216 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190781+0000", 2024-08-17T17:29:34.216 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000114419 2024-08-17T17:29:34.216 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.216 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.216 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 6,7", 2024-08-17T17:29:34.217 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190891+0000", 2024-08-17T17:29:34.217 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000110389 2024-08-17T17:29:34.217 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.217 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.217 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:34.217 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.197254+0000", 2024-08-17T17:29:34.217 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0063629309999999996 2024-08-17T17:29:34.217 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.217 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.217 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:29:34.217 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.197899+0000", 2024-08-17T17:29:34.217 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00064482500000000002 2024-08-17T17:29:34.217 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.217 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:29:34.218 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.218 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.218 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.218 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:208310 9.c 9:3e200965:::benchmark_data_smithi063_263014_object83972:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1272)", 2024-08-17T17:29:34.218 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:29:34.190642+0000", 2024-08-17T17:29:34.219 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0099674550000000001, 2024-08-17T17:29:34.219 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.01018984, 2024-08-17T17:29:34.219 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:29:34.219 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:29:34.219 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:29:34.219 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:29:34.219 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:29:34.219 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:29:34.219 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 208310 2024-08-17T17:29:34.219 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.219 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:29:34.219 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.220 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:29:34.220 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190642+0000", 2024-08-17T17:29:34.220 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.220 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.220 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.220 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:29:34.220 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190642+0000", 2024-08-17T17:29:34.220 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.220 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.220 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.220 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:29:34.220 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190642+0000", 2024-08-17T17:29:34.220 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9500000000000002e-07 2024-08-17T17:29:34.221 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.221 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.221 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:29:34.221 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190646+0000", 2024-08-17T17:29:34.221 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1470000000000001e-06 2024-08-17T17:29:34.221 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.221 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.221 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:29:34.221 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190654+0000", 2024-08-17T17:29:34.221 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0679999999999997e-06 2024-08-17T17:29:34.221 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.221 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.221 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:29:34.222 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190659+0000", 2024-08-17T17:29:34.222 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7119999999999997e-06 2024-08-17T17:29:34.222 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.222 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.222 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:29:34.222 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.193881+0000", 2024-08-17T17:29:34.222 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0032211219999999999 2024-08-17T17:29:34.222 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.222 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.222 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:29:34.222 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.194137+0000", 2024-08-17T17:29:34.222 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00025666300000000002 2024-08-17T17:29:34.222 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.223 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.223 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:29:34.223 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.194259+0000", 2024-08-17T17:29:34.223 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000121897 2024-08-17T17:29:34.223 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.223 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:29:34.223 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.223 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.223 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.223 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:208311 9.2 9:4149dbf1:::benchmark_data_smithi063_263014_object83973:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1272)", 2024-08-17T17:29:34.223 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:29:34.190670+0000", 2024-08-17T17:29:34.223 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0099394010000000005, 2024-08-17T17:29:34.224 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.01022081, 2024-08-17T17:29:34.224 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:29:34.224 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:29:34.224 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:29:34.224 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:29:34.224 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:29:34.224 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:29:34.224 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 208311 2024-08-17T17:29:34.224 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.224 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:29:34.224 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.224 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:29:34.224 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190670+0000", 2024-08-17T17:29:34.225 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.225 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.225 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.225 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:29:34.225 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190670+0000", 2024-08-17T17:29:34.225 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.225 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.225 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.225 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:29:34.226 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190670+0000", 2024-08-17T17:29:34.226 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2599999999999998e-07 2024-08-17T17:29:34.226 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.226 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.226 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:29:34.226 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190673+0000", 2024-08-17T17:29:34.226 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.283e-06 2024-08-17T17:29:34.226 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.226 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.226 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:29:34.226 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190679+0000", 2024-08-17T17:29:34.226 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0360000000000003e-06 2024-08-17T17:29:34.227 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.227 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.227 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:29:34.227 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.190684+0000", 2024-08-17T17:29:34.227 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0640000000000001e-06 2024-08-17T17:29:34.227 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.227 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.227 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:29:34.227 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.193817+0000", 2024-08-17T17:29:34.227 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.003133135 2024-08-17T17:29:34.227 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.227 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.227 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:29:34.228 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.193941+0000", 2024-08-17T17:29:34.228 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012443799999999999 2024-08-17T17:29:34.228 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.228 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.228 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:29:34.228 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.194055+0000", 2024-08-17T17:29:34.228 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0001133 2024-08-17T17:29:34.228 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.228 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.228 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:29:34.228 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.198057+0000", 2024-08-17T17:29:34.228 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0040027580000000004 2024-08-17T17:29:34.228 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.228 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.229 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:29:34.229 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.200645+0000", 2024-08-17T17:29:34.229 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0025881099999999998 2024-08-17T17:29:34.229 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.229 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:29:34.229 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.229 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.229 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.229 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:208308 9.3 e1272/1247 9:c7ae7816:::benchmark_data_smithi063_263014_object83970:head v 1272'12965, mlcod=1272'12963)", 2024-08-17T17:29:34.229 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:29:34.195340+0000", 2024-08-17T17:29:34.229 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0052696979999999997, 2024-08-17T17:29:34.229 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0056304670000000001, 2024-08-17T17:29:34.229 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:29:34.230 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:29:34.230 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:29:34.230 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:29:34.230 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.230 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:29:34.230 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.195340+0000", 2024-08-17T17:29:34.230 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.230 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.230 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.230 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:29:34.230 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.195340+0000", 2024-08-17T17:29:34.230 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.231 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.231 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.231 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:29:34.231 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.195340+0000", 2024-08-17T17:29:34.231 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8599999999999997e-07 2024-08-17T17:29:34.231 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.231 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.231 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:29:34.231 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.195346+0000", 2024-08-17T17:29:34.231 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9030000000000001e-06 2024-08-17T17:29:34.232 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.232 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.233 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:29:34.233 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.195359+0000", 2024-08-17T17:29:34.233 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3067999999999999e-05 2024-08-17T17:29:34.233 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.233 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.233 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:29:34.234 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.195366+0000", 2024-08-17T17:29:34.234 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2650000000000002e-06 2024-08-17T17:29:34.234 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.235 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.235 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:29:34.235 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.195441+0000", 2024-08-17T17:29:34.235 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5712999999999998e-05 2024-08-17T17:29:34.235 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.235 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.235 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:29:34.236 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.195462+0000", 2024-08-17T17:29:34.236 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0959e-05 2024-08-17T17:29:34.236 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.236 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:29:34.236 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.237 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.237 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.237 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:208316 9.3 e1272/1247 9:c6d6ca50:::benchmark_data_smithi063_263014_object83978:head v 1272'12966, mlcod=1272'12963)", 2024-08-17T17:29:34.237 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:29:34.196477+0000", 2024-08-17T17:29:34.237 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0041324689999999997, 2024-08-17T17:29:34.237 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0045418730000000001, 2024-08-17T17:29:34.238 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:29:34.238 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:29:34.238 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:29:34.238 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:29:34.238 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.238 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:29:34.239 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.196477+0000", 2024-08-17T17:29:34.239 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.239 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.239 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.239 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:29:34.240 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.196477+0000", 2024-08-17T17:29:34.240 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.240 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.240 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.240 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:29:34.240 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.196477+0000", 2024-08-17T17:29:34.241 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1900000000000003e-07 2024-08-17T17:29:34.241 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.241 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.241 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:29:34.241 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.196481+0000", 2024-08-17T17:29:34.241 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7689999999999998e-06 2024-08-17T17:29:34.242 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.242 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.242 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:29:34.242 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.196491+0000", 2024-08-17T17:29:34.242 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0295999999999999e-05 2024-08-17T17:29:34.242 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.243 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.243 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:29:34.243 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.196498+0000", 2024-08-17T17:29:34.243 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2770000000000002e-06 2024-08-17T17:29:34.243 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.244 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.244 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:29:34.244 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.198381+0000", 2024-08-17T17:29:34.244 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0018836339999999999 2024-08-17T17:29:34.244 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.244 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.245 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:29:34.245 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.198402+0000", 2024-08-17T17:29:34.245 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0358000000000001e-05 2024-08-17T17:29:34.245 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.245 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:29:34.245 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.246 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.246 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.246 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:208319 9.c 9:3d49d288:::benchmark_data_smithi063_263014_object83981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1272)", 2024-08-17T17:29:34.246 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:29:34.198135+0000", 2024-08-17T17:29:34.246 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.002474502, 2024-08-17T17:29:34.247 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002934131, 2024-08-17T17:29:34.247 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:29:34.247 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:29:34.247 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:29:34.247 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:29:34.247 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:29:34.248 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:29:34.248 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 208319 2024-08-17T17:29:34.248 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.248 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:29:34.248 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.249 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:29:34.249 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.198135+0000", 2024-08-17T17:29:34.249 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.249 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.249 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.249 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:29:34.249 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.198135+0000", 2024-08-17T17:29:34.250 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.250 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.250 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.250 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:29:34.250 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.198136+0000", 2024-08-17T17:29:34.250 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.35e-06 2024-08-17T17:29:34.251 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.251 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.251 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:29:34.251 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.198141+0000", 2024-08-17T17:29:34.251 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2839999999999997e-06 2024-08-17T17:29:34.251 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.251 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.251 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:29:34.251 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.198158+0000", 2024-08-17T17:29:34.251 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6277999999999999e-05 2024-08-17T17:29:34.251 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.251 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.251 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:29:34.252 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.198165+0000", 2024-08-17T17:29:34.252 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4170000000000003e-06 2024-08-17T17:29:34.252 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.252 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.252 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:29:34.252 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.199621+0000", 2024-08-17T17:29:34.252 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0014563530000000001 2024-08-17T17:29:34.252 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.252 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.252 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:29:34.252 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.199745+0000", 2024-08-17T17:29:34.252 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012367899999999999 2024-08-17T17:29:34.252 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.253 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.253 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:29:34.253 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.199862+0000", 2024-08-17T17:29:34.253 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000117136 2024-08-17T17:29:34.253 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.253 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:29:34.253 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.253 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.253 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.253 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:208318 9.3 e1272/1247)", 2024-08-17T17:29:34.253 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:29:34.199557+0000", 2024-08-17T17:29:34.253 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.00105273, 2024-08-17T17:29:34.253 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001567049, 2024-08-17T17:29:34.254 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:29:34.254 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:29:34.254 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "queued for pg", 2024-08-17T17:29:34.254 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:29:34.254 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.254 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:29:34.254 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.199557+0000", 2024-08-17T17:29:34.254 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.254 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.254 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.254 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:29:34.254 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.199557+0000", 2024-08-17T17:29:34.254 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:29:34.255 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.255 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.255 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:29:34.255 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.199557+0000", 2024-08-17T17:29:34.255 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7700000000000004e-07 2024-08-17T17:29:34.255 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.255 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.255 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:29:34.255 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.199562+0000", 2024-08-17T17:29:34.255 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0119999999999996e-06 2024-08-17T17:29:34.255 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.255 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.255 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:29:34.256 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.199576+0000", 2024-08-17T17:29:34.256 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3839000000000001e-05 2024-08-17T17:29:34.256 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:29:34.256 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:29:34.256 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:29:34.256 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:34.199583+0000", 2024-08-17T17:29:34.256 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4930000000000003e-06 2024-08-17T17:29:34.256 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.256 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:29:34.256 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.256 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:29:34.256 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-08-17T17:29:34.256 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 9 2024-08-17T17:29:34.257 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:29:34.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.258 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:34.258 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:34.359 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:34.392 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:34.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:34.494 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:34.529 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:34.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:34.632 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:34.670 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:34.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:34.772 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:34.802 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:34.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:34.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:34.899 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:29:34.905 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:35.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.001 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:35.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:35.102 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:35.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.138 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:35.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:35.240 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:35.276 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:35.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:35.359 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:29:35.378 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:35.462 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:35.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:35.564 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:35.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.596 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:35.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:35.634 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:35 smithi063 ceph-mon[109843]: pgmap v1802: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:29:35.634 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:35 smithi063 ceph-mon[114756]: pgmap v1802: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:29:35.697 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:35.733 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:35.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:35.835 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:35.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:35 smithi162 ceph-mon[104668]: pgmap v1802: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s 2024-08-17T17:29:35.866 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:35.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:35.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:35.968 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:36.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.027 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:36.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:36.128 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:36.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.164 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:36.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:36.265 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:36.305 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:36.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:36.406 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:36.443 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:36.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:36.545 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:36.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.587 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:36.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:36.688 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:36.729 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:36.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:36.831 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:36.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:36.881 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:36.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:36.982 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:37.015 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:37.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:37.117 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:37.152 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:37.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:37.254 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:37.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.288 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:37.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:37.389 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:37.422 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:37.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:37.438 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:37.524 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:37.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.582 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:37.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:37.684 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:37.692 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:37 smithi063 ceph-mon[114756]: pgmap v1803: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:29:37.692 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:37 smithi063 ceph-mon[109843]: pgmap v1803: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:29:37.702 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:37.714 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:37.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:37.816 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:37.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:37 smithi162 ceph-mon[104668]: pgmap v1803: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-08-17T17:29:37.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:37.867 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:37.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:37.968 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:38.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.008 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:38.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:38.109 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:38.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.149 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:38.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:38.250 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:38.288 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:38.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:38.390 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:38.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.423 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:38.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:38.524 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:38.540 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:38 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2078867193' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:38.540 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:38 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2078867193' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:38.540 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:38 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:38.557 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:38.557 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:29:38.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:38.659 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:38.711 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:38.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:38.813 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:38.845 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:38.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:38.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:38 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2078867193' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:38.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:38 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:38.947 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:38.965 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:38 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:29:38.975 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:29:38.978 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:29:38.980 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:38.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:38.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:39.081 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:39.113 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:39.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:39.215 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:39.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.249 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:39.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:39.350 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:39.386 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:39.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:39.488 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:39.525 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:39.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:39.627 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:39.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.669 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:39.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:39.770 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:39.800 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:39.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:39.823 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:39 smithi063 ceph-mon[114756]: pgmap v1804: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:29:39.823 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:39 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:29:39.823 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:39 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:29:39.823 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:39 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:39.823 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:39 smithi063 ceph-mon[114756]: osdmap e1273: 8 total, 8 up, 8 in 2024-08-17T17:29:39.823 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:39 smithi063 ceph-mon[114756]: from='client.55330 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:29:39.823 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:39 smithi063 ceph-mon[109843]: pgmap v1804: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:29:39.823 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:39 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:29:39.824 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:39 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:29:39.824 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:39 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:39.824 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:39 smithi063 ceph-mon[109843]: osdmap e1273: 8 total, 8 up, 8 in 2024-08-17T17:29:39.824 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:39 smithi063 ceph-mon[109843]: from='client.55330 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:29:39.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:39 smithi162 ceph-mon[104668]: pgmap v1804: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:29:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:39 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:29:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:39 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:29:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:39 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:29:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:39 smithi162 ceph-mon[104668]: osdmap e1273: 8 total, 8 up, 8 in 2024-08-17T17:29:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:39 smithi162 ceph-mon[104668]: from='client.55330 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:29:39.860 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1805,"stamp":"2024-08-17T17:29:38.551976+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2414352381,"num_objects":36858,"num_object_clones":0,"num_object_copies":110574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36858,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":211904,"num_write_kb":7957474,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1246,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106818,"ondisk_log_size":106818,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12120548,"kb_used_data":7392036,"kb_used_omap":199,"kb_used_meta":4728248,"kb_avail":737840668,"statfs":{"total":767960285184,"available":755548844032,"internally_reserved":0,"allocated":7569444864,"data_stored":7548171935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":4841726716},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,7,3,11,4,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-713162752,"num_objects":-10882,"num_object_clones":0,"num_object_copies":-32646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-10882,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10882,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.121973"},"pg_stats":[{"pgid":"1.11","version":"369'149","reported_seq":1596,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.503107+0000","last_change":"2024-08-17T17:24:49.721626+0000","last_active":"2024-08-17T17:29:24.503107+0000","last_peered":"2024-08-17T17:29:24.503107+0000","last_clean":"2024-08-17T17:29:24.503107+0000","last_became_active":"2024-08-17T17:24:20.649128+0000","last_became_peered":"2024-08-17T17:24:20.649128+0000","last_unstale":"2024-08-17T17:29:24.503107+0000","last_undegraded":"2024-08-17T17:29:24.503107+0000","last_fullsized":"2024-08-17T17:29:24.503107+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:03:15.343059+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1642,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.539508+0000","last_change":"2024-08-17T17:21:06.463828+0000","last_active":"2024-08-17T17:29:30.539508+0000","last_peered":"2024-08-17T17:29:30.539508+0000","last_clean":"2024-08-17T17:29:30.539508+0000","last_became_active":"2024-08-17T17:21:06.463464+0000","last_became_peered":"2024-08-17T17:21:06.463464+0000","last_unstale":"2024-08-17T17:29:30.539508+0000","last_undegraded":"2024-08-17T17:29:30.539508+0000","last_fullsized":"2024-08-17T17:29:30.539508+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:46.465006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1203,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.507521+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:29:24.507521+0000","last_peered":"2024-08-17T17:29:24.507521+0000","last_clean":"2024-08-17T17:29:24.507521+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:29:24.507521+0000","last_undegraded":"2024-08-17T17:29:24.507521+0000","last_fullsized":"2024-08-17T17:29:24.507521+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1527,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.504815+0000","last_change":"2024-08-17T17:24:48.765312+0000","last_active":"2024-08-17T17:29:24.504815+0000","last_peered":"2024-08-17T17:29:24.504815+0000","last_clean":"2024-08-17T17:29:24.504815+0000","last_became_active":"2024-08-17T17:24:20.642731+0000","last_became_peered":"2024-08-17T17:24:20.642731+0000","last_unstale":"2024-08-17T17:29:24.504815+0000","last_undegraded":"2024-08-17T17:29:24.504815+0000","last_fullsized":"2024-08-17T17:29:24.504815+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:08:36.138960+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1214,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.506654+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:29:24.506654+0000","last_peered":"2024-08-17T17:29:24.506654+0000","last_clean":"2024-08-17T17:29:24.506654+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:29:24.506654+0000","last_undegraded":"2024-08-17T17:29:24.506654+0000","last_fullsized":"2024-08-17T17:29:24.506654+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1070,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.507938+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:29:24.507938+0000","last_peered":"2024-08-17T17:29:24.507938+0000","last_clean":"2024-08-17T17:29:24.507938+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:29:24.507938+0000","last_undegraded":"2024-08-17T17:29:24.507938+0000","last_fullsized":"2024-08-17T17:29:24.507938+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":856,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.534508+0000","last_change":"2024-08-17T17:24:50.880046+0000","last_active":"2024-08-17T17:29:30.534508+0000","last_peered":"2024-08-17T17:29:30.534508+0000","last_clean":"2024-08-17T17:29:30.534508+0000","last_became_active":"2024-08-17T17:24:20.652165+0000","last_became_peered":"2024-08-17T17:24:20.652165+0000","last_unstale":"2024-08-17T17:29:30.534508+0000","last_undegraded":"2024-08-17T17:29:30.534508+0000","last_fullsized":"2024-08-17T17:29:30.534508+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:00.416810+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1194,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.534665+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:29:30.534665+0000","last_peered":"2024-08-17T17:29:30.534665+0000","last_clean":"2024-08-17T17:29:30.534665+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:29:30.534665+0000","last_undegraded":"2024-08-17T17:29:30.534665+0000","last_fullsized":"2024-08-17T17:29:30.534665+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1186,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.509064+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:29:24.509064+0000","last_peered":"2024-08-17T17:29:24.509064+0000","last_clean":"2024-08-17T17:29:24.509064+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:29:24.509064+0000","last_undegraded":"2024-08-17T17:29:24.509064+0000","last_fullsized":"2024-08-17T17:29:24.509064+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1532,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.506113+0000","last_change":"2024-08-17T17:24:50.727183+0000","last_active":"2024-08-17T17:29:24.506113+0000","last_peered":"2024-08-17T17:29:24.506113+0000","last_clean":"2024-08-17T17:29:24.506113+0000","last_became_active":"2024-08-17T17:24:20.646409+0000","last_became_peered":"2024-08-17T17:24:20.646409+0000","last_unstale":"2024-08-17T17:29:24.506113+0000","last_undegraded":"2024-08-17T17:29:24.506113+0000","last_fullsized":"2024-08-17T17:29:24.506113+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.086193+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1642,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.532803+0000","last_change":"2024-08-17T17:21:06.442598+0000","last_active":"2024-08-17T17:29:30.532803+0000","last_peered":"2024-08-17T17:29:30.532803+0000","last_clean":"2024-08-17T17:29:30.532803+0000","last_became_active":"2024-08-17T17:21:06.442264+0000","last_became_peered":"2024-08-17T17:21:06.442264+0000","last_unstale":"2024-08-17T17:29:30.532803+0000","last_undegraded":"2024-08-17T17:29:30.532803+0000","last_fullsized":"2024-08-17T17:29:30.532803+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:35.003620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1263,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.499998+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:29:24.499998+0000","last_peered":"2024-08-17T17:29:24.499998+0000","last_clean":"2024-08-17T17:29:24.499998+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:29:24.499998+0000","last_undegraded":"2024-08-17T17:29:24.499998+0000","last_fullsized":"2024-08-17T17:29:24.499998+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"1272'12985","reported_seq":13020,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.175443+0000","last_change":"2024-08-17T17:26:34.519835+0000","last_active":"2024-08-17T17:29:38.175443+0000","last_peered":"2024-08-17T17:29:38.175443+0000","last_clean":"2024-08-17T17:29:38.175443+0000","last_became_active":"2024-08-17T17:26:34.519427+0000","last_became_peered":"2024-08-17T17:26:34.519427+0000","last_unstale":"2024-08-17T17:29:38.175443+0000","last_undegraded":"2024-08-17T17:29:38.175443+0000","last_fullsized":"2024-08-17T17:29:38.175443+0000","mapping_epoch":1247,"log_start":"1258'6000","ondisk_log_start":"1258'6000","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":6985,"log_dups_size":0,"ondisk_log_size":6985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:25.352736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146472960,"num_objects":2235,"num_object_clones":0,"num_object_copies":6705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12985,"num_write_kb":487040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1470,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.508199+0000","last_change":"2024-08-17T17:24:50.410621+0000","last_active":"2024-08-17T17:29:24.508199+0000","last_peered":"2024-08-17T17:29:24.508199+0000","last_clean":"2024-08-17T17:29:24.508199+0000","last_became_active":"2024-08-17T17:24:31.863062+0000","last_became_peered":"2024-08-17T17:24:31.863062+0000","last_unstale":"2024-08-17T17:29:24.508199+0000","last_undegraded":"2024-08-17T17:29:24.508199+0000","last_fullsized":"2024-08-17T17:29:24.508199+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.410491+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.410491+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:11.717684+0000","scrub_duration":0.0066315480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1228,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.507870+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:29:24.507870+0000","last_peered":"2024-08-17T17:29:24.507870+0000","last_clean":"2024-08-17T17:29:24.507870+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:29:24.507870+0000","last_undegraded":"2024-08-17T17:29:24.507870+0000","last_fullsized":"2024-08-17T17:29:24.507870+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1201,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.505089+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:29:24.505089+0000","last_peered":"2024-08-17T17:29:24.505089+0000","last_clean":"2024-08-17T17:29:24.505089+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:29:24.505089+0000","last_undegraded":"2024-08-17T17:29:24.505089+0000","last_fullsized":"2024-08-17T17:29:24.505089+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"1272'13203","reported_seq":13238,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:37.731995+0000","last_change":"2024-08-17T17:26:34.715261+0000","last_active":"2024-08-17T17:29:37.731995+0000","last_peered":"2024-08-17T17:29:37.731995+0000","last_clean":"2024-08-17T17:29:37.731995+0000","last_became_active":"2024-08-17T17:26:34.714853+0000","last_became_peered":"2024-08-17T17:26:34.714853+0000","last_unstale":"2024-08-17T17:29:37.731995+0000","last_undegraded":"2024-08-17T17:29:37.731995+0000","last_fullsized":"2024-08-17T17:29:37.731995+0000","mapping_epoch":1247,"log_start":"1259'6800","ondisk_log_start":"1259'6800","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":6403,"log_dups_size":0,"ondisk_log_size":6403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:18.673096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152764416,"num_objects":2331,"num_object_clones":0,"num_object_copies":6993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13203,"num_write_kb":497088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1724,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.534915+0000","last_change":"2024-08-17T17:24:45.985725+0000","last_active":"2024-08-17T17:29:30.534915+0000","last_peered":"2024-08-17T17:29:30.534915+0000","last_clean":"2024-08-17T17:29:30.534915+0000","last_became_active":"2024-08-17T17:24:31.795836+0000","last_became_peered":"2024-08-17T17:24:31.795836+0000","last_unstale":"2024-08-17T17:29:30.534915+0000","last_undegraded":"2024-08-17T17:29:30.534915+0000","last_fullsized":"2024-08-17T17:29:30.534915+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:45.985596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:45.985596+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:17.897808+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1229,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.507951+0000","last_change":"2024-08-17T17:21:06.540689+0000","last_active":"2024-08-17T17:29:24.507951+0000","last_peered":"2024-08-17T17:29:24.507951+0000","last_clean":"2024-08-17T17:29:24.507951+0000","last_became_active":"2024-08-17T17:21:06.461614+0000","last_became_peered":"2024-08-17T17:21:06.461614+0000","last_unstale":"2024-08-17T17:29:24.507951+0000","last_undegraded":"2024-08-17T17:29:24.507951+0000","last_fullsized":"2024-08-17T17:29:24.507951+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:11.696898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1253,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.519254+0000","last_change":"2024-08-17T17:21:06.470389+0000","last_active":"2024-08-17T17:29:24.519254+0000","last_peered":"2024-08-17T17:29:24.519254+0000","last_clean":"2024-08-17T17:29:24.519254+0000","last_became_active":"2024-08-17T17:21:06.469966+0000","last_became_peered":"2024-08-17T17:21:06.469966+0000","last_unstale":"2024-08-17T17:29:24.519254+0000","last_undegraded":"2024-08-17T17:29:24.519254+0000","last_fullsized":"2024-08-17T17:29:24.519254+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:56.140616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"1272'13423","reported_seq":13458,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.235113+0000","last_change":"2024-08-17T17:26:34.412902+0000","last_active":"2024-08-17T17:29:38.235113+0000","last_peered":"2024-08-17T17:29:38.235113+0000","last_clean":"2024-08-17T17:29:38.235113+0000","last_became_active":"2024-08-17T17:26:34.412543+0000","last_became_peered":"2024-08-17T17:26:34.412543+0000","last_unstale":"2024-08-17T17:29:38.235113+0000","last_undegraded":"2024-08-17T17:29:38.235113+0000","last_fullsized":"2024-08-17T17:29:38.235113+0000","mapping_epoch":1247,"log_start":"1258'6400","ondisk_log_start":"1258'6400","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":7023,"log_dups_size":0,"ondisk_log_size":7023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:20.767004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147128320,"num_objects":2245,"num_object_clones":0,"num_object_copies":6735,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2245,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13423,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":838,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.500602+0000","last_change":"2024-08-17T17:24:48.192447+0000","last_active":"2024-08-17T17:29:24.500602+0000","last_peered":"2024-08-17T17:29:24.500602+0000","last_clean":"2024-08-17T17:29:24.500602+0000","last_became_active":"2024-08-17T17:24:20.644240+0000","last_became_peered":"2024-08-17T17:24:20.644240+0000","last_unstale":"2024-08-17T17:29:24.500602+0000","last_undegraded":"2024-08-17T17:29:24.500602+0000","last_fullsized":"2024-08-17T17:29:24.500602+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:56.795337+0000","scrub_duration":0.010920255,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1188,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.532960+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:29:30.532960+0000","last_peered":"2024-08-17T17:29:30.532960+0000","last_clean":"2024-08-17T17:29:30.532960+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:29:30.532960+0000","last_undegraded":"2024-08-17T17:29:30.532960+0000","last_fullsized":"2024-08-17T17:29:30.532960+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1235,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.500019+0000","last_change":"2024-08-17T17:21:06.446798+0000","last_active":"2024-08-17T17:29:24.500019+0000","last_peered":"2024-08-17T17:29:24.500019+0000","last_clean":"2024-08-17T17:29:24.500019+0000","last_became_active":"2024-08-17T17:21:06.446205+0000","last_became_peered":"2024-08-17T17:21:06.446205+0000","last_unstale":"2024-08-17T17:29:24.500019+0000","last_undegraded":"2024-08-17T17:29:24.500019+0000","last_fullsized":"2024-08-17T17:29:24.500019+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:28.584048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1272'13426","reported_seq":13461,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.243988+0000","last_change":"2024-08-17T17:26:35.023050+0000","last_active":"2024-08-17T17:29:38.243988+0000","last_peered":"2024-08-17T17:29:38.243988+0000","last_clean":"2024-08-17T17:29:38.243988+0000","last_became_active":"2024-08-17T17:26:35.022784+0000","last_became_peered":"2024-08-17T17:26:35.022784+0000","last_unstale":"2024-08-17T17:29:38.243988+0000","last_undegraded":"2024-08-17T17:29:38.243988+0000","last_fullsized":"2024-08-17T17:29:38.243988+0000","mapping_epoch":1247,"log_start":"1258'6400","ondisk_log_start":"1258'6400","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":7026,"log_dups_size":0,"ondisk_log_size":7026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:04.319458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147456000,"num_objects":2250,"num_object_clones":0,"num_object_copies":6750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13426,"num_write_kb":501632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":746,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.520135+0000","last_change":"2024-08-17T17:24:52.102415+0000","last_active":"2024-08-17T17:29:24.520135+0000","last_peered":"2024-08-17T17:29:24.520135+0000","last_clean":"2024-08-17T17:29:24.520135+0000","last_became_active":"2024-08-17T17:24:31.795843+0000","last_became_peered":"2024-08-17T17:24:31.795843+0000","last_unstale":"2024-08-17T17:29:24.520135+0000","last_undegraded":"2024-08-17T17:29:24.520135+0000","last_fullsized":"2024-08-17T17:29:24.520135+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:32:08.434197+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1217,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530527+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:29:30.530527+0000","last_peered":"2024-08-17T17:29:30.530527+0000","last_clean":"2024-08-17T17:29:30.530527+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:29:30.530527+0000","last_undegraded":"2024-08-17T17:29:30.530527+0000","last_fullsized":"2024-08-17T17:29:30.530527+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":998,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.519143+0000","last_change":"2024-08-17T17:21:06.460905+0000","last_active":"2024-08-17T17:29:24.519143+0000","last_peered":"2024-08-17T17:29:24.519143+0000","last_clean":"2024-08-17T17:29:24.519143+0000","last_became_active":"2024-08-17T17:21:06.460592+0000","last_became_peered":"2024-08-17T17:21:06.460592+0000","last_unstale":"2024-08-17T17:29:24.519143+0000","last_undegraded":"2024-08-17T17:29:24.519143+0000","last_fullsized":"2024-08-17T17:29:24.519143+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:10.146916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.1","version":"1272'13153","reported_seq":13216,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:35.971238+0000","last_change":"2024-08-17T17:26:34.410513+0000","last_active":"2024-08-17T17:29:35.971238+0000","last_peered":"2024-08-17T17:29:35.971238+0000","last_clean":"2024-08-17T17:29:35.971238+0000","last_became_active":"2024-08-17T17:26:34.410199+0000","last_became_peered":"2024-08-17T17:26:34.410199+0000","last_unstale":"2024-08-17T17:29:35.971238+0000","last_undegraded":"2024-08-17T17:29:35.971238+0000","last_fullsized":"2024-08-17T17:29:35.971238+0000","mapping_epoch":1247,"log_start":"1260'7300","ondisk_log_start":"1260'7300","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5853,"log_dups_size":0,"ondisk_log_size":5853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:06.961811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159842304,"num_objects":2439,"num_object_clones":0,"num_object_copies":7317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13153,"num_write_kb":498944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1617,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.508967+0000","last_change":"2024-08-17T17:24:54.418765+0000","last_active":"2024-08-17T17:29:24.508967+0000","last_peered":"2024-08-17T17:29:24.508967+0000","last_clean":"2024-08-17T17:29:24.508967+0000","last_became_active":"2024-08-17T17:24:31.800263+0000","last_became_peered":"2024-08-17T17:24:31.800263+0000","last_unstale":"2024-08-17T17:29:24.508967+0000","last_undegraded":"2024-08-17T17:29:24.508967+0000","last_fullsized":"2024-08-17T17:29:24.508967+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:54.418644+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:54.418644+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:11.024294+0000","scrub_duration":0.013346135,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1663,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.534660+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:29:30.534660+0000","last_peered":"2024-08-17T17:29:30.534660+0000","last_clean":"2024-08-17T17:29:30.534660+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:29:30.534660+0000","last_undegraded":"2024-08-17T17:29:30.534660+0000","last_fullsized":"2024-08-17T17:29:30.534660+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1219,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.519750+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:29:24.519750+0000","last_peered":"2024-08-17T17:29:24.519750+0000","last_clean":"2024-08-17T17:29:24.519750+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:29:24.519750+0000","last_undegraded":"2024-08-17T17:29:24.519750+0000","last_fullsized":"2024-08-17T17:29:24.519750+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"1272'13266","reported_seq":13300,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:37.514623+0000","last_change":"2024-08-17T17:26:35.023056+0000","last_active":"2024-08-17T17:29:37.514623+0000","last_peered":"2024-08-17T17:29:37.514623+0000","last_clean":"2024-08-17T17:29:37.514623+0000","last_became_active":"2024-08-17T17:26:35.022671+0000","last_became_peered":"2024-08-17T17:26:35.022671+0000","last_unstale":"2024-08-17T17:29:37.514623+0000","last_undegraded":"2024-08-17T17:29:37.514623+0000","last_fullsized":"2024-08-17T17:29:37.514623+0000","mapping_epoch":1247,"log_start":"1258'6200","ondisk_log_start":"1258'6200","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":7066,"log_dups_size":0,"ondisk_log_size":7066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:43.531150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152567808,"num_objects":2328,"num_object_clones":0,"num_object_copies":6984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13266,"num_write_kb":499008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1563,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.504893+0000","last_change":"2024-08-17T17:24:58.712331+0000","last_active":"2024-08-17T17:29:24.504893+0000","last_peered":"2024-08-17T17:29:24.504893+0000","last_clean":"2024-08-17T17:29:24.504893+0000","last_became_active":"2024-08-17T17:24:31.802584+0000","last_became_peered":"2024-08-17T17:24:31.802584+0000","last_unstale":"2024-08-17T17:29:24.504893+0000","last_undegraded":"2024-08-17T17:29:24.504893+0000","last_fullsized":"2024-08-17T17:29:24.504893+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:59.616217+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1236,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.500096+0000","last_change":"2024-08-17T17:21:06.446841+0000","last_active":"2024-08-17T17:29:24.500096+0000","last_peered":"2024-08-17T17:29:24.500096+0000","last_clean":"2024-08-17T17:29:24.500096+0000","last_became_active":"2024-08-17T17:21:06.446271+0000","last_became_peered":"2024-08-17T17:21:06.446271+0000","last_unstale":"2024-08-17T17:29:24.500096+0000","last_undegraded":"2024-08-17T17:29:24.500096+0000","last_fullsized":"2024-08-17T17:29:24.500096+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:20.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1252,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.520104+0000","last_change":"2024-08-17T17:21:06.480911+0000","last_active":"2024-08-17T17:29:24.520104+0000","last_peered":"2024-08-17T17:29:24.520104+0000","last_clean":"2024-08-17T17:29:24.520104+0000","last_became_active":"2024-08-17T17:21:06.480551+0000","last_became_peered":"2024-08-17T17:21:06.480551+0000","last_unstale":"2024-08-17T17:29:24.520104+0000","last_undegraded":"2024-08-17T17:29:24.520104+0000","last_fullsized":"2024-08-17T17:29:24.520104+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:26.003704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"1272'13193","reported_seq":13228,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:37.671407+0000","last_change":"2024-08-17T17:26:34.715560+0000","last_active":"2024-08-17T17:29:37.671407+0000","last_peered":"2024-08-17T17:29:37.671407+0000","last_clean":"2024-08-17T17:29:37.671407+0000","last_became_active":"2024-08-17T17:26:34.714955+0000","last_became_peered":"2024-08-17T17:26:34.714955+0000","last_unstale":"2024-08-17T17:29:37.671407+0000","last_undegraded":"2024-08-17T17:29:37.671407+0000","last_fullsized":"2024-08-17T17:29:37.671407+0000","mapping_epoch":1247,"log_start":"1260'7400","ondisk_log_start":"1260'7400","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5793,"log_dups_size":0,"ondisk_log_size":5793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:46.367495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145424384,"num_objects":2219,"num_object_clones":0,"num_object_copies":6657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13193,"num_write_kb":493184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":795,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.504608+0000","last_change":"2024-08-17T17:24:57.682573+0000","last_active":"2024-08-17T17:29:24.504608+0000","last_peered":"2024-08-17T17:29:24.504608+0000","last_clean":"2024-08-17T17:29:24.504608+0000","last_became_active":"2024-08-17T17:24:31.869575+0000","last_became_peered":"2024-08-17T17:24:31.869575+0000","last_unstale":"2024-08-17T17:29:24.504608+0000","last_undegraded":"2024-08-17T17:29:24.504608+0000","last_fullsized":"2024-08-17T17:29:24.504608+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:57.682409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:57.682409+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:42.144615+0000","scrub_duration":0.0066879779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1221,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.519320+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:29:24.519320+0000","last_peered":"2024-08-17T17:29:24.519320+0000","last_clean":"2024-08-17T17:29:24.519320+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:29:24.519320+0000","last_undegraded":"2024-08-17T17:29:24.519320+0000","last_fullsized":"2024-08-17T17:29:24.519320+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1237,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.520358+0000","last_change":"2024-08-17T17:21:06.451521+0000","last_active":"2024-08-17T17:29:24.520358+0000","last_peered":"2024-08-17T17:29:24.520358+0000","last_clean":"2024-08-17T17:29:24.520358+0000","last_became_active":"2024-08-17T17:21:06.451197+0000","last_became_peered":"2024-08-17T17:21:06.451197+0000","last_unstale":"2024-08-17T17:29:24.520358+0000","last_undegraded":"2024-08-17T17:29:24.520358+0000","last_fullsized":"2024-08-17T17:29:24.520358+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:04.000692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"1272'13134","reported_seq":13197,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:35.969650+0000","last_change":"2024-08-17T17:26:35.023654+0000","last_active":"2024-08-17T17:29:35.969650+0000","last_peered":"2024-08-17T17:29:35.969650+0000","last_clean":"2024-08-17T17:29:35.969650+0000","last_became_active":"2024-08-17T17:26:35.022968+0000","last_became_peered":"2024-08-17T17:26:35.022968+0000","last_unstale":"2024-08-17T17:29:35.969650+0000","last_undegraded":"2024-08-17T17:29:35.969650+0000","last_fullsized":"2024-08-17T17:29:35.969650+0000","mapping_epoch":1247,"log_start":"1260'7300","ondisk_log_start":"1260'7300","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5834,"log_dups_size":0,"ondisk_log_size":5834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:15:51.584797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154533888,"num_objects":2358,"num_object_clones":0,"num_object_copies":7074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13134,"num_write_kb":495744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1563,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.506034+0000","last_change":"2024-08-17T17:24:52.716059+0000","last_active":"2024-08-17T17:29:24.506034+0000","last_peered":"2024-08-17T17:29:24.506034+0000","last_clean":"2024-08-17T17:29:24.506034+0000","last_became_active":"2024-08-17T17:24:31.798173+0000","last_became_peered":"2024-08-17T17:24:31.798173+0000","last_unstale":"2024-08-17T17:29:24.506034+0000","last_undegraded":"2024-08-17T17:29:24.506034+0000","last_fullsized":"2024-08-17T17:29:24.506034+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:04:56.094994+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1645,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.532256+0000","last_change":"2024-08-17T17:21:06.441118+0000","last_active":"2024-08-17T17:29:30.532256+0000","last_peered":"2024-08-17T17:29:30.532256+0000","last_clean":"2024-08-17T17:29:30.532256+0000","last_became_active":"2024-08-17T17:21:06.440800+0000","last_became_peered":"2024-08-17T17:21:06.440800+0000","last_unstale":"2024-08-17T17:29:30.532256+0000","last_undegraded":"2024-08-17T17:29:30.532256+0000","last_fullsized":"2024-08-17T17:29:30.532256+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:08.091528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":151,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.543302+0000","last_change":"2024-08-17T17:22:48.374766+0000","last_active":"2024-08-17T17:29:30.543302+0000","last_peered":"2024-08-17T17:29:30.543302+0000","last_clean":"2024-08-17T17:29:30.543302+0000","last_became_active":"2024-08-17T17:22:48.373556+0000","last_became_peered":"2024-08-17T17:22:48.373556+0000","last_unstale":"2024-08-17T17:29:30.543302+0000","last_undegraded":"2024-08-17T17:29:30.543302+0000","last_fullsized":"2024-08-17T17:29:30.543302+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:33.362080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.8","version":"1272'13273","reported_seq":13336,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:35.962348+0000","last_change":"2024-08-17T17:26:34.519719+0000","last_active":"2024-08-17T17:29:35.962348+0000","last_peered":"2024-08-17T17:29:35.962348+0000","last_clean":"2024-08-17T17:29:35.962348+0000","last_became_active":"2024-08-17T17:26:34.519368+0000","last_became_peered":"2024-08-17T17:26:34.519368+0000","last_unstale":"2024-08-17T17:29:35.962348+0000","last_undegraded":"2024-08-17T17:29:35.962348+0000","last_fullsized":"2024-08-17T17:29:35.962348+0000","mapping_epoch":1247,"log_start":"1260'7500","ondisk_log_start":"1260'7500","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5773,"log_dups_size":0,"ondisk_log_size":5773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:32.666019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153944064,"num_objects":2349,"num_object_clones":0,"num_object_copies":7047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13273,"num_write_kb":499904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1617,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.507415+0000","last_change":"2024-08-17T17:24:13.542832+0000","last_active":"2024-08-17T17:29:24.507415+0000","last_peered":"2024-08-17T17:29:24.507415+0000","last_clean":"2024-08-17T17:29:24.507415+0000","last_became_active":"2024-08-17T17:24:11.570282+0000","last_became_peered":"2024-08-17T17:24:11.570282+0000","last_unstale":"2024-08-17T17:29:24.507415+0000","last_undegraded":"2024-08-17T17:29:24.507415+0000","last_fullsized":"2024-08-17T17:29:24.507415+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:49.414162+0000","scrub_duration":0.017597167,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1231,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.504759+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:29:24.504759+0000","last_peered":"2024-08-17T17:29:24.504759+0000","last_clean":"2024-08-17T17:29:24.504759+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:29:24.504759+0000","last_undegraded":"2024-08-17T17:29:24.504759+0000","last_fullsized":"2024-08-17T17:29:24.504759+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1610,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.533457+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:29:30.533457+0000","last_peered":"2024-08-17T17:29:30.533457+0000","last_clean":"2024-08-17T17:29:30.533457+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:29:30.533457+0000","last_undegraded":"2024-08-17T17:29:30.533457+0000","last_fullsized":"2024-08-17T17:29:30.533457+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1203,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.500322+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:29:24.500322+0000","last_peered":"2024-08-17T17:29:24.500322+0000","last_clean":"2024-08-17T17:29:24.500322+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:29:24.500322+0000","last_undegraded":"2024-08-17T17:29:24.500322+0000","last_fullsized":"2024-08-17T17:29:24.500322+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"1272'13173","reported_seq":13208,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:37.512552+0000","last_change":"2024-08-17T17:26:34.519823+0000","last_active":"2024-08-17T17:29:37.512552+0000","last_peered":"2024-08-17T17:29:37.512552+0000","last_clean":"2024-08-17T17:29:37.512552+0000","last_became_active":"2024-08-17T17:26:34.519497+0000","last_became_peered":"2024-08-17T17:26:34.519497+0000","last_unstale":"2024-08-17T17:29:37.512552+0000","last_undegraded":"2024-08-17T17:29:37.512552+0000","last_fullsized":"2024-08-17T17:29:37.512552+0000","mapping_epoch":1247,"log_start":"1258'6100","ondisk_log_start":"1258'6100","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":7073,"log_dups_size":0,"ondisk_log_size":7073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:59:31.807854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149487616,"num_objects":2281,"num_object_clones":0,"num_object_copies":6843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13173,"num_write_kb":494528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":856,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.500033+0000","last_change":"2024-08-17T17:24:11.936322+0000","last_active":"2024-08-17T17:29:24.500033+0000","last_peered":"2024-08-17T17:29:24.500033+0000","last_clean":"2024-08-17T17:29:24.500033+0000","last_became_active":"2024-08-17T17:24:11.574846+0000","last_became_peered":"2024-08-17T17:24:11.574846+0000","last_unstale":"2024-08-17T17:29:24.500033+0000","last_undegraded":"2024-08-17T17:29:24.500033+0000","last_fullsized":"2024-08-17T17:29:24.500033+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:11.936236+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:11.936236+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:31.674851+0000","scrub_duration":0.0035797139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1267,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.507473+0000","last_change":"2024-08-17T17:21:06.430639+0000","last_active":"2024-08-17T17:29:24.507473+0000","last_peered":"2024-08-17T17:29:24.507473+0000","last_clean":"2024-08-17T17:29:24.507473+0000","last_became_active":"2024-08-17T17:21:06.430297+0000","last_became_peered":"2024-08-17T17:21:06.430297+0000","last_unstale":"2024-08-17T17:29:24.507473+0000","last_undegraded":"2024-08-17T17:29:24.507473+0000","last_fullsized":"2024-08-17T17:29:24.507473+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:57.794620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1070,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.519006+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:29:24.519006+0000","last_peered":"2024-08-17T17:29:24.519006+0000","last_clean":"2024-08-17T17:29:24.519006+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:29:24.519006+0000","last_undegraded":"2024-08-17T17:29:24.519006+0000","last_fullsized":"2024-08-17T17:29:24.519006+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":82,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.502355+0000","last_change":"2024-08-17T17:22:48.370769+0000","last_active":"2024-08-17T17:29:24.502355+0000","last_peered":"2024-08-17T17:29:24.502355+0000","last_clean":"2024-08-17T17:29:24.502355+0000","last_became_active":"2024-08-17T17:22:48.370531+0000","last_became_peered":"2024-08-17T17:22:48.370531+0000","last_unstale":"2024-08-17T17:29:24.502355+0000","last_undegraded":"2024-08-17T17:29:24.502355+0000","last_fullsized":"2024-08-17T17:29:24.502355+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:02.943823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"1272'13308","reported_seq":13343,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.184602+0000","last_change":"2024-08-17T17:26:34.715167+0000","last_active":"2024-08-17T17:29:38.184602+0000","last_peered":"2024-08-17T17:29:38.184602+0000","last_clean":"2024-08-17T17:29:38.184602+0000","last_became_active":"2024-08-17T17:26:34.714868+0000","last_became_peered":"2024-08-17T17:26:34.714868+0000","last_unstale":"2024-08-17T17:29:38.184602+0000","last_undegraded":"2024-08-17T17:29:38.184602+0000","last_fullsized":"2024-08-17T17:29:38.184602+0000","mapping_epoch":1247,"log_start":"1258'6300","ondisk_log_start":"1258'6300","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":7008,"log_dups_size":0,"ondisk_log_size":7008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:19.957252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148373504,"num_objects":2264,"num_object_clones":0,"num_object_copies":6792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13308,"num_write_kb":498304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1638,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.507961+0000","last_change":"2024-08-17T17:24:12.519946+0000","last_active":"2024-08-17T17:29:24.507961+0000","last_peered":"2024-08-17T17:29:24.507961+0000","last_clean":"2024-08-17T17:29:24.507961+0000","last_became_active":"2024-08-17T17:24:11.569747+0000","last_became_peered":"2024-08-17T17:24:11.569747+0000","last_unstale":"2024-08-17T17:29:24.507961+0000","last_undegraded":"2024-08-17T17:29:24.507961+0000","last_fullsized":"2024-08-17T17:29:24.507961+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.519838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.519838+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:56.003993+0000","scrub_duration":0.0068224740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1707,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.535978+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:29:30.535978+0000","last_peered":"2024-08-17T17:29:30.535978+0000","last_clean":"2024-08-17T17:29:30.535978+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:29:30.535978+0000","last_undegraded":"2024-08-17T17:29:30.535978+0000","last_fullsized":"2024-08-17T17:29:30.535978+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1099,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.535951+0000","last_change":"2024-08-17T17:21:06.428684+0000","last_active":"2024-08-17T17:29:30.535951+0000","last_peered":"2024-08-17T17:29:30.535951+0000","last_clean":"2024-08-17T17:29:30.535951+0000","last_became_active":"2024-08-17T17:21:06.428178+0000","last_became_peered":"2024-08-17T17:21:06.428178+0000","last_unstale":"2024-08-17T17:29:30.535951+0000","last_undegraded":"2024-08-17T17:29:30.535951+0000","last_fullsized":"2024-08-17T17:29:30.535951+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:14.836119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":914,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.505319+0000","last_change":"2024-08-17T17:21:06.443565+0000","last_active":"2024-08-17T17:29:24.505319+0000","last_peered":"2024-08-17T17:29:24.505319+0000","last_clean":"2024-08-17T17:29:24.505319+0000","last_became_active":"2024-08-17T17:21:06.443215+0000","last_became_peered":"2024-08-17T17:21:06.443215+0000","last_unstale":"2024-08-17T17:29:24.505319+0000","last_undegraded":"2024-08-17T17:29:24.505319+0000","last_fullsized":"2024-08-17T17:29:24.505319+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:47.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"1272'13581","reported_seq":13616,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:37.725460+0000","last_change":"2024-08-17T17:26:34.519795+0000","last_active":"2024-08-17T17:29:37.725460+0000","last_peered":"2024-08-17T17:29:37.725460+0000","last_clean":"2024-08-17T17:29:37.725460+0000","last_became_active":"2024-08-17T17:26:34.519426+0000","last_became_peered":"2024-08-17T17:26:34.519426+0000","last_unstale":"2024-08-17T17:29:37.725460+0000","last_undegraded":"2024-08-17T17:29:37.725460+0000","last_fullsized":"2024-08-17T17:29:37.725460+0000","mapping_epoch":1247,"log_start":"1261'7800","ondisk_log_start":"1261'7800","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5781,"log_dups_size":0,"ondisk_log_size":5781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:14.719799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147652608,"num_objects":2253,"num_object_clones":0,"num_object_copies":6759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13581,"num_write_kb":506688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1580,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.503233+0000","last_change":"2024-08-17T17:24:13.795227+0000","last_active":"2024-08-17T17:29:24.503233+0000","last_peered":"2024-08-17T17:29:24.503233+0000","last_clean":"2024-08-17T17:29:24.503233+0000","last_became_active":"2024-08-17T17:24:11.578352+0000","last_became_peered":"2024-08-17T17:24:11.578352+0000","last_unstale":"2024-08-17T17:29:24.503233+0000","last_undegraded":"2024-08-17T17:29:24.503233+0000","last_fullsized":"2024-08-17T17:29:24.503233+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:36:39.440808+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1258,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.506311+0000","last_change":"2024-08-17T17:21:06.456978+0000","last_active":"2024-08-17T17:29:24.506311+0000","last_peered":"2024-08-17T17:29:24.506311+0000","last_clean":"2024-08-17T17:29:24.506311+0000","last_became_active":"2024-08-17T17:21:06.452265+0000","last_became_peered":"2024-08-17T17:21:06.452265+0000","last_unstale":"2024-08-17T17:29:24.506311+0000","last_undegraded":"2024-08-17T17:29:24.506311+0000","last_fullsized":"2024-08-17T17:29:24.506311+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:52.893746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1240,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.520152+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:29:24.520152+0000","last_peered":"2024-08-17T17:29:24.520152+0000","last_clean":"2024-08-17T17:29:24.520152+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:29:24.520152+0000","last_undegraded":"2024-08-17T17:29:24.520152+0000","last_fullsized":"2024-08-17T17:29:24.520152+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1175,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.543183+0000","last_change":"2024-08-17T17:21:06.435754+0000","last_active":"2024-08-17T17:29:30.543183+0000","last_peered":"2024-08-17T17:29:30.543183+0000","last_clean":"2024-08-17T17:29:30.543183+0000","last_became_active":"2024-08-17T17:21:06.435442+0000","last_became_peered":"2024-08-17T17:21:06.435442+0000","last_unstale":"2024-08-17T17:29:30.543183+0000","last_undegraded":"2024-08-17T17:29:30.543183+0000","last_fullsized":"2024-08-17T17:29:30.543183+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:34.849042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"1272'13052","reported_seq":13087,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.202327+0000","last_change":"2024-08-17T17:26:34.716730+0000","last_active":"2024-08-17T17:29:38.202327+0000","last_peered":"2024-08-17T17:29:38.202327+0000","last_clean":"2024-08-17T17:29:38.202327+0000","last_became_active":"2024-08-17T17:26:34.716457+0000","last_became_peered":"2024-08-17T17:26:34.716457+0000","last_unstale":"2024-08-17T17:29:38.202327+0000","last_undegraded":"2024-08-17T17:29:38.202327+0000","last_fullsized":"2024-08-17T17:29:38.202327+0000","mapping_epoch":1247,"log_start":"1258'6000","ondisk_log_start":"1258'6000","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":7052,"log_dups_size":0,"ondisk_log_size":7052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:51.133413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149684224,"num_objects":2284,"num_object_clones":0,"num_object_copies":6852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13052,"num_write_kb":490752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":843,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.500536+0000","last_change":"2024-08-17T17:24:55.041464+0000","last_active":"2024-08-17T17:29:24.500536+0000","last_peered":"2024-08-17T17:29:24.500536+0000","last_clean":"2024-08-17T17:29:24.500536+0000","last_became_active":"2024-08-17T17:24:31.800289+0000","last_became_peered":"2024-08-17T17:24:31.800289+0000","last_unstale":"2024-08-17T17:29:24.500536+0000","last_undegraded":"2024-08-17T17:29:24.500536+0000","last_fullsized":"2024-08-17T17:29:24.500536+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:55.041378+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:55.041378+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:43.974501+0000","scrub_duration":0.001623642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1233,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.506733+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:29:24.506733+0000","last_peered":"2024-08-17T17:29:24.506733+0000","last_clean":"2024-08-17T17:29:24.506733+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:29:24.506733+0000","last_undegraded":"2024-08-17T17:29:24.506733+0000","last_fullsized":"2024-08-17T17:29:24.506733+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":923,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.507678+0000","last_change":"2024-08-17T17:21:06.434991+0000","last_active":"2024-08-17T17:29:24.507678+0000","last_peered":"2024-08-17T17:29:24.507678+0000","last_clean":"2024-08-17T17:29:24.507678+0000","last_became_active":"2024-08-17T17:21:06.434630+0000","last_became_peered":"2024-08-17T17:21:06.434630+0000","last_unstale":"2024-08-17T17:29:24.507678+0000","last_undegraded":"2024-08-17T17:29:24.507678+0000","last_fullsized":"2024-08-17T17:29:24.507678+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:24.104609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":865,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.506183+0000","last_change":"2024-08-17T17:21:06.462532+0000","last_active":"2024-08-17T17:29:24.506183+0000","last_peered":"2024-08-17T17:29:24.506183+0000","last_clean":"2024-08-17T17:29:24.506183+0000","last_became_active":"2024-08-17T17:21:06.462183+0000","last_became_peered":"2024-08-17T17:21:06.462183+0000","last_unstale":"2024-08-17T17:29:24.506183+0000","last_undegraded":"2024-08-17T17:29:24.506183+0000","last_fullsized":"2024-08-17T17:29:24.506183+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:50.033322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.e","version":"1272'13572","reported_seq":13607,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:37.722804+0000","last_change":"2024-08-17T17:26:34.716166+0000","last_active":"2024-08-17T17:29:37.722804+0000","last_peered":"2024-08-17T17:29:37.722804+0000","last_clean":"2024-08-17T17:29:37.722804+0000","last_became_active":"2024-08-17T17:26:34.715299+0000","last_became_peered":"2024-08-17T17:26:34.715299+0000","last_unstale":"2024-08-17T17:29:37.722804+0000","last_undegraded":"2024-08-17T17:29:37.722804+0000","last_fullsized":"2024-08-17T17:29:37.722804+0000","mapping_epoch":1247,"log_start":"1261'7800","ondisk_log_start":"1261'7800","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5772,"log_dups_size":0,"ondisk_log_size":5772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:37.004999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153878528,"num_objects":2348,"num_object_clones":0,"num_object_copies":7044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13572,"num_write_kb":509440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":860,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.500476+0000","last_change":"2024-08-17T17:24:53.103270+0000","last_active":"2024-08-17T17:29:24.500476+0000","last_peered":"2024-08-17T17:29:24.500476+0000","last_clean":"2024-08-17T17:29:24.500476+0000","last_became_active":"2024-08-17T17:24:31.800823+0000","last_became_peered":"2024-08-17T17:24:31.800823+0000","last_unstale":"2024-08-17T17:29:24.500476+0000","last_undegraded":"2024-08-17T17:29:24.500476+0000","last_fullsized":"2024-08-17T17:29:24.500476+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.103133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.103133+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:52.444781+0000","scrub_duration":0.0074281759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1098,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.543004+0000","last_change":"2024-08-17T17:21:06.433720+0000","last_active":"2024-08-17T17:29:30.543004+0000","last_peered":"2024-08-17T17:29:30.543004+0000","last_clean":"2024-08-17T17:29:30.543004+0000","last_became_active":"2024-08-17T17:21:06.433292+0000","last_became_peered":"2024-08-17T17:21:06.433292+0000","last_unstale":"2024-08-17T17:29:30.543004+0000","last_undegraded":"2024-08-17T17:29:30.543004+0000","last_fullsized":"2024-08-17T17:29:30.543004+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:32.246594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1233,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.508412+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:29:24.508412+0000","last_peered":"2024-08-17T17:29:24.508412+0000","last_clean":"2024-08-17T17:29:24.508412+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:29:24.508412+0000","last_undegraded":"2024-08-17T17:29:24.508412+0000","last_fullsized":"2024-08-17T17:29:24.508412+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":825,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.504689+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:29:24.504689+0000","last_peered":"2024-08-17T17:29:24.504689+0000","last_clean":"2024-08-17T17:29:24.504689+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:29:24.504689+0000","last_undegraded":"2024-08-17T17:29:24.504689+0000","last_fullsized":"2024-08-17T17:29:24.504689+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"1272'13252","reported_seq":13288,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:37.700803+0000","last_change":"2024-08-17T17:26:34.715123+0000","last_active":"2024-08-17T17:29:37.700803+0000","last_peered":"2024-08-17T17:29:37.700803+0000","last_clean":"2024-08-17T17:29:37.700803+0000","last_became_active":"2024-08-17T17:26:34.714789+0000","last_became_peered":"2024-08-17T17:26:34.714789+0000","last_unstale":"2024-08-17T17:29:37.700803+0000","last_undegraded":"2024-08-17T17:29:37.700803+0000","last_fullsized":"2024-08-17T17:29:37.700803+0000","mapping_epoch":1247,"log_start":"1260'7400","ondisk_log_start":"1260'7400","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5852,"log_dups_size":0,"ondisk_log_size":5852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:19.314570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146210840,"num_objects":2232,"num_object_clones":0,"num_object_copies":6696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2232,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13252,"num_write_kb":495425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1584,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.520238+0000","last_change":"2024-08-17T17:24:49.119578+0000","last_active":"2024-08-17T17:29:24.520238+0000","last_peered":"2024-08-17T17:29:24.520238+0000","last_clean":"2024-08-17T17:29:24.520238+0000","last_became_active":"2024-08-17T17:24:31.807987+0000","last_became_peered":"2024-08-17T17:24:31.807987+0000","last_unstale":"2024-08-17T17:29:24.520238+0000","last_undegraded":"2024-08-17T17:29:24.520238+0000","last_fullsized":"2024-08-17T17:29:24.520238+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:34.052786+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1664,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.539470+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:29:30.539470+0000","last_peered":"2024-08-17T17:29:30.539470+0000","last_clean":"2024-08-17T17:29:30.539470+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:29:30.539470+0000","last_undegraded":"2024-08-17T17:29:30.539470+0000","last_fullsized":"2024-08-17T17:29:30.539470+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":875,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.500551+0000","last_change":"2024-08-17T17:21:06.452395+0000","last_active":"2024-08-17T17:29:24.500551+0000","last_peered":"2024-08-17T17:29:24.500551+0000","last_clean":"2024-08-17T17:29:24.500551+0000","last_became_active":"2024-08-17T17:21:06.451867+0000","last_became_peered":"2024-08-17T17:21:06.451867+0000","last_unstale":"2024-08-17T17:29:24.500551+0000","last_undegraded":"2024-08-17T17:29:24.500551+0000","last_fullsized":"2024-08-17T17:29:24.500551+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:50.597092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1653,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.532565+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:29:30.532565+0000","last_peered":"2024-08-17T17:29:30.532565+0000","last_clean":"2024-08-17T17:29:30.532565+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:29:30.532565+0000","last_undegraded":"2024-08-17T17:29:30.532565+0000","last_fullsized":"2024-08-17T17:29:30.532565+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"1272'12846","reported_seq":12909,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:35.937842+0000","last_change":"2024-08-17T17:26:34.412017+0000","last_active":"2024-08-17T17:29:35.937842+0000","last_peered":"2024-08-17T17:29:35.937842+0000","last_clean":"2024-08-17T17:29:35.937842+0000","last_became_active":"2024-08-17T17:26:34.411768+0000","last_became_peered":"2024-08-17T17:26:34.411768+0000","last_unstale":"2024-08-17T17:29:35.937842+0000","last_undegraded":"2024-08-17T17:29:35.937842+0000","last_fullsized":"2024-08-17T17:29:35.937842+0000","mapping_epoch":1247,"log_start":"1259'7000","ondisk_log_start":"1259'7000","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5846,"log_dups_size":0,"ondisk_log_size":5846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:23:07.054750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158334976,"num_objects":2416,"num_object_clones":0,"num_object_copies":7248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12846,"num_write_kb":488384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1689,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.539544+0000","last_change":"2024-08-17T17:24:12.033217+0000","last_active":"2024-08-17T17:29:30.539544+0000","last_peered":"2024-08-17T17:29:30.539544+0000","last_clean":"2024-08-17T17:29:30.539544+0000","last_became_active":"2024-08-17T17:24:11.572877+0000","last_became_peered":"2024-08-17T17:24:11.572877+0000","last_unstale":"2024-08-17T17:29:30.539544+0000","last_undegraded":"2024-08-17T17:29:30.539544+0000","last_fullsized":"2024-08-17T17:29:30.539544+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.033101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.033101+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.264475+0000","scrub_duration":20,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":244,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.520322+0000","last_change":"2024-08-17T17:21:06.519965+0000","last_active":"2024-08-17T17:29:24.520322+0000","last_peered":"2024-08-17T17:29:24.520322+0000","last_clean":"2024-08-17T17:29:24.520322+0000","last_became_active":"2024-08-17T17:21:06.459952+0000","last_became_peered":"2024-08-17T17:21:06.459952+0000","last_unstale":"2024-08-17T17:29:24.520322+0000","last_undegraded":"2024-08-17T17:29:24.520322+0000","last_fullsized":"2024-08-17T17:29:24.520322+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:27.828098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1230,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.500692+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:29:24.500692+0000","last_peered":"2024-08-17T17:29:24.500692+0000","last_clean":"2024-08-17T17:29:24.500692+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:29:24.500692+0000","last_undegraded":"2024-08-17T17:29:24.500692+0000","last_fullsized":"2024-08-17T17:29:24.500692+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1206,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.519086+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:29:24.519086+0000","last_peered":"2024-08-17T17:29:24.519086+0000","last_clean":"2024-08-17T17:29:24.519086+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:29:24.519086+0000","last_undegraded":"2024-08-17T17:29:24.519086+0000","last_fullsized":"2024-08-17T17:29:24.519086+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":754,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.534063+0000","last_change":"2024-08-17T17:24:48.935651+0000","last_active":"2024-08-17T17:29:30.534063+0000","last_peered":"2024-08-17T17:29:30.534063+0000","last_clean":"2024-08-17T17:29:30.534063+0000","last_became_active":"2024-08-17T17:24:40.897753+0000","last_became_peered":"2024-08-17T17:24:40.897753+0000","last_unstale":"2024-08-17T17:29:30.534063+0000","last_undegraded":"2024-08-17T17:29:30.534063+0000","last_fullsized":"2024-08-17T17:29:30.534063+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:10:15.118428+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1237,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.520193+0000","last_change":"2024-08-17T17:21:06.458000+0000","last_active":"2024-08-17T17:29:24.520193+0000","last_peered":"2024-08-17T17:29:24.520193+0000","last_clean":"2024-08-17T17:29:24.520193+0000","last_became_active":"2024-08-17T17:21:06.457411+0000","last_became_peered":"2024-08-17T17:21:06.457411+0000","last_unstale":"2024-08-17T17:29:24.520193+0000","last_undegraded":"2024-08-17T17:29:24.520193+0000","last_fullsized":"2024-08-17T17:29:24.520193+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:52.740639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1228,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.519913+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:29:24.519913+0000","last_peered":"2024-08-17T17:29:24.519913+0000","last_clean":"2024-08-17T17:29:24.519913+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:29:24.519913+0000","last_undegraded":"2024-08-17T17:29:24.519913+0000","last_fullsized":"2024-08-17T17:29:24.519913+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":705,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:28.222834+0000","last_change":"2024-08-17T17:25:03.663370+0000","last_active":"2024-08-17T17:29:28.222834+0000","last_peered":"2024-08-17T17:29:28.222834+0000","last_clean":"2024-08-17T17:29:28.222834+0000","last_became_active":"2024-08-17T17:24:40.886214+0000","last_became_peered":"2024-08-17T17:24:40.886214+0000","last_unstale":"2024-08-17T17:29:28.222834+0000","last_undegraded":"2024-08-17T17:29:28.222834+0000","last_fullsized":"2024-08-17T17:29:28.222834+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:11:39.272029+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":897,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.500038+0000","last_change":"2024-08-17T17:21:06.454587+0000","last_active":"2024-08-17T17:29:24.500038+0000","last_peered":"2024-08-17T17:29:24.500038+0000","last_clean":"2024-08-17T17:29:24.500038+0000","last_became_active":"2024-08-17T17:21:06.454229+0000","last_became_peered":"2024-08-17T17:21:06.454229+0000","last_unstale":"2024-08-17T17:29:24.500038+0000","last_undegraded":"2024-08-17T17:29:24.500038+0000","last_fullsized":"2024-08-17T17:29:24.500038+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:38.340263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1203,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.508288+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:29:24.508288+0000","last_peered":"2024-08-17T17:29:24.508288+0000","last_clean":"2024-08-17T17:29:24.508288+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:29:24.508288+0000","last_undegraded":"2024-08-17T17:29:24.508288+0000","last_fullsized":"2024-08-17T17:29:24.508288+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1691,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.542863+0000","last_change":"2024-08-17T17:24:20.649977+0000","last_active":"2024-08-17T17:29:30.542863+0000","last_peered":"2024-08-17T17:29:30.542863+0000","last_clean":"2024-08-17T17:29:30.542863+0000","last_became_active":"2024-08-17T17:24:20.649336+0000","last_became_peered":"2024-08-17T17:24:20.649336+0000","last_unstale":"2024-08-17T17:29:30.542863+0000","last_undegraded":"2024-08-17T17:29:30.542863+0000","last_fullsized":"2024-08-17T17:29:30.542863+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:57.187271+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1233,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.503543+0000","last_change":"2024-08-17T17:21:06.442796+0000","last_active":"2024-08-17T17:29:24.503543+0000","last_peered":"2024-08-17T17:29:24.503543+0000","last_clean":"2024-08-17T17:29:24.503543+0000","last_became_active":"2024-08-17T17:21:06.442466+0000","last_became_peered":"2024-08-17T17:21:06.442466+0000","last_unstale":"2024-08-17T17:29:24.503543+0000","last_undegraded":"2024-08-17T17:29:24.503543+0000","last_fullsized":"2024-08-17T17:29:24.503543+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:36.366620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1609,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.532641+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:29:30.532641+0000","last_peered":"2024-08-17T17:29:30.532641+0000","last_clean":"2024-08-17T17:29:30.532641+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:29:30.532641+0000","last_undegraded":"2024-08-17T17:29:30.532641+0000","last_fullsized":"2024-08-17T17:29:30.532641+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1452,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:27.724525+0000","last_change":"2024-08-17T17:25:07.472056+0000","last_active":"2024-08-17T17:29:27.724525+0000","last_peered":"2024-08-17T17:29:27.724525+0000","last_clean":"2024-08-17T17:29:27.724525+0000","last_became_active":"2024-08-17T17:24:40.883356+0000","last_became_peered":"2024-08-17T17:24:40.883356+0000","last_unstale":"2024-08-17T17:29:27.724525+0000","last_undegraded":"2024-08-17T17:29:27.724525+0000","last_fullsized":"2024-08-17T17:29:27.724525+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:43:27.883917+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1070,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530711+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:29:30.530711+0000","last_peered":"2024-08-17T17:29:30.530711+0000","last_clean":"2024-08-17T17:29:30.530711+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:29:30.530711+0000","last_undegraded":"2024-08-17T17:29:30.530711+0000","last_fullsized":"2024-08-17T17:29:30.530711+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1642,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.532465+0000","last_change":"2024-08-17T17:21:06.468788+0000","last_active":"2024-08-17T17:29:30.532465+0000","last_peered":"2024-08-17T17:29:30.532465+0000","last_clean":"2024-08-17T17:29:30.532465+0000","last_became_active":"2024-08-17T17:21:06.467141+0000","last_became_peered":"2024-08-17T17:21:06.467141+0000","last_unstale":"2024-08-17T17:29:30.532465+0000","last_undegraded":"2024-08-17T17:29:30.532465+0000","last_fullsized":"2024-08-17T17:29:30.532465+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:51.301359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1452,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.535802+0000","last_change":"2024-08-17T17:24:53.130774+0000","last_active":"2024-08-17T17:29:30.535802+0000","last_peered":"2024-08-17T17:29:30.535802+0000","last_clean":"2024-08-17T17:29:30.535802+0000","last_became_active":"2024-08-17T17:24:40.883388+0000","last_became_peered":"2024-08-17T17:24:40.883388+0000","last_unstale":"2024-08-17T17:29:30.535802+0000","last_undegraded":"2024-08-17T17:29:30.535802+0000","last_fullsized":"2024-08-17T17:29:30.535802+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:36.772875+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1236,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.500614+0000","last_change":"2024-08-17T17:21:06.435998+0000","last_active":"2024-08-17T17:29:24.500614+0000","last_peered":"2024-08-17T17:29:24.500614+0000","last_clean":"2024-08-17T17:29:24.500614+0000","last_became_active":"2024-08-17T17:21:06.435579+0000","last_became_peered":"2024-08-17T17:21:06.435579+0000","last_unstale":"2024-08-17T17:29:24.500614+0000","last_undegraded":"2024-08-17T17:29:24.500614+0000","last_fullsized":"2024-08-17T17:29:24.500614+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:01.743074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1672,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.532737+0000","last_change":"2024-08-17T17:21:06.473627+0000","last_active":"2024-08-17T17:29:30.532737+0000","last_peered":"2024-08-17T17:29:30.532737+0000","last_clean":"2024-08-17T17:29:30.532737+0000","last_became_active":"2024-08-17T17:21:06.473062+0000","last_became_peered":"2024-08-17T17:21:06.473062+0000","last_unstale":"2024-08-17T17:29:30.532737+0000","last_undegraded":"2024-08-17T17:29:30.532737+0000","last_fullsized":"2024-08-17T17:29:30.532737+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:21.413163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":812,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530355+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:29:30.530355+0000","last_peered":"2024-08-17T17:29:30.530355+0000","last_clean":"2024-08-17T17:29:30.530355+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:29:30.530355+0000","last_undegraded":"2024-08-17T17:29:30.530355+0000","last_fullsized":"2024-08-17T17:29:30.530355+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1031,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.507825+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:29:24.507825+0000","last_peered":"2024-08-17T17:29:24.507825+0000","last_clean":"2024-08-17T17:29:24.507825+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:29:24.507825+0000","last_undegraded":"2024-08-17T17:29:24.507825+0000","last_fullsized":"2024-08-17T17:29:24.507825+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1162,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.532832+0000","last_change":"2024-08-17T17:21:06.471265+0000","last_active":"2024-08-17T17:29:30.532832+0000","last_peered":"2024-08-17T17:29:30.532832+0000","last_clean":"2024-08-17T17:29:30.532832+0000","last_became_active":"2024-08-17T17:21:06.470943+0000","last_became_peered":"2024-08-17T17:21:06.470943+0000","last_unstale":"2024-08-17T17:29:30.532832+0000","last_undegraded":"2024-08-17T17:29:30.532832+0000","last_fullsized":"2024-08-17T17:29:30.532832+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:41.070024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1584,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.505023+0000","last_change":"2024-08-17T17:25:02.696608+0000","last_active":"2024-08-17T17:29:24.505023+0000","last_peered":"2024-08-17T17:29:24.505023+0000","last_clean":"2024-08-17T17:29:24.505023+0000","last_became_active":"2024-08-17T17:24:40.877715+0000","last_became_peered":"2024-08-17T17:24:40.877715+0000","last_unstale":"2024-08-17T17:29:24.505023+0000","last_undegraded":"2024-08-17T17:29:24.505023+0000","last_fullsized":"2024-08-17T17:29:24.505023+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:48:00.941285+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1648,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.533682+0000","last_change":"2024-08-17T17:22:48.377351+0000","last_active":"2024-08-17T17:29:30.533682+0000","last_peered":"2024-08-17T17:29:30.533682+0000","last_clean":"2024-08-17T17:29:30.533682+0000","last_became_active":"2024-08-17T17:22:48.377059+0000","last_became_peered":"2024-08-17T17:22:48.377059+0000","last_unstale":"2024-08-17T17:29:30.533682+0000","last_undegraded":"2024-08-17T17:29:30.533682+0000","last_fullsized":"2024-08-17T17:29:30.533682+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:15.250902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1253,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.519433+0000","last_change":"2024-08-17T17:21:06.459139+0000","last_active":"2024-08-17T17:29:24.519433+0000","last_peered":"2024-08-17T17:29:24.519433+0000","last_clean":"2024-08-17T17:29:24.519433+0000","last_became_active":"2024-08-17T17:21:06.457850+0000","last_became_peered":"2024-08-17T17:21:06.457850+0000","last_unstale":"2024-08-17T17:29:24.519433+0000","last_undegraded":"2024-08-17T17:29:24.519433+0000","last_fullsized":"2024-08-17T17:29:24.519433+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:13.888576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1550,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.532426+0000","last_change":"2024-08-17T17:24:49.911734+0000","last_active":"2024-08-17T17:29:30.532426+0000","last_peered":"2024-08-17T17:29:30.532426+0000","last_clean":"2024-08-17T17:29:30.532426+0000","last_became_active":"2024-08-17T17:24:40.899102+0000","last_became_peered":"2024-08-17T17:24:40.899102+0000","last_unstale":"2024-08-17T17:29:30.532426+0000","last_undegraded":"2024-08-17T17:29:30.532426+0000","last_fullsized":"2024-08-17T17:29:30.532426+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.911622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.911622+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:30:27.987033+0000","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1689,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.543100+0000","last_change":"2024-08-17T17:21:06.446321+0000","last_active":"2024-08-17T17:29:30.543100+0000","last_peered":"2024-08-17T17:29:30.543100+0000","last_clean":"2024-08-17T17:29:30.543100+0000","last_became_active":"2024-08-17T17:21:06.446019+0000","last_became_peered":"2024-08-17T17:21:06.446019+0000","last_unstale":"2024-08-17T17:29:30.543100+0000","last_undegraded":"2024-08-17T17:29:30.543100+0000","last_fullsized":"2024-08-17T17:29:30.543100+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:26.750070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":222,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.519922+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:29:24.519922+0000","last_peered":"2024-08-17T17:29:24.519922+0000","last_clean":"2024-08-17T17:29:24.519922+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:29:24.519922+0000","last_undegraded":"2024-08-17T17:29:24.519922+0000","last_fullsized":"2024-08-17T17:29:24.519922+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":754,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.534284+0000","last_change":"2024-08-17T17:24:46.946320+0000","last_active":"2024-08-17T17:29:30.534284+0000","last_peered":"2024-08-17T17:29:30.534284+0000","last_clean":"2024-08-17T17:29:30.534284+0000","last_became_active":"2024-08-17T17:24:40.880380+0000","last_became_peered":"2024-08-17T17:24:40.880380+0000","last_unstale":"2024-08-17T17:29:30.534284+0000","last_undegraded":"2024-08-17T17:29:30.534284+0000","last_fullsized":"2024-08-17T17:29:30.534284+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:54:30.637162+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1251,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.519778+0000","last_change":"2024-08-17T17:21:06.465804+0000","last_active":"2024-08-17T17:29:24.519778+0000","last_peered":"2024-08-17T17:29:24.519778+0000","last_clean":"2024-08-17T17:29:24.519778+0000","last_became_active":"2024-08-17T17:21:06.465461+0000","last_became_peered":"2024-08-17T17:21:06.465461+0000","last_unstale":"2024-08-17T17:29:24.519778+0000","last_undegraded":"2024-08-17T17:29:24.519778+0000","last_fullsized":"2024-08-17T17:29:24.519778+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:06.443667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1243,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.502395+0000","last_change":"2024-08-17T17:21:06.481634+0000","last_active":"2024-08-17T17:29:24.502395+0000","last_peered":"2024-08-17T17:29:24.502395+0000","last_clean":"2024-08-17T17:29:24.502395+0000","last_became_active":"2024-08-17T17:21:06.469081+0000","last_became_peered":"2024-08-17T17:21:06.469081+0000","last_unstale":"2024-08-17T17:29:24.502395+0000","last_undegraded":"2024-08-17T17:29:24.502395+0000","last_fullsized":"2024-08-17T17:29:24.502395+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:53:26.954738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1251,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.519296+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:29:24.519296+0000","last_peered":"2024-08-17T17:29:24.519296+0000","last_clean":"2024-08-17T17:29:24.519296+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:29:24.519296+0000","last_undegraded":"2024-08-17T17:29:24.519296+0000","last_fullsized":"2024-08-17T17:29:24.519296+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1202,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.506670+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:29:24.506670+0000","last_peered":"2024-08-17T17:29:24.506670+0000","last_clean":"2024-08-17T17:29:24.506670+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:29:24.506670+0000","last_undegraded":"2024-08-17T17:29:24.506670+0000","last_fullsized":"2024-08-17T17:29:24.506670+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1643,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.525232+0000","last_change":"2024-08-17T17:25:02.417545+0000","last_active":"2024-08-17T17:29:30.525232+0000","last_peered":"2024-08-17T17:29:30.525232+0000","last_clean":"2024-08-17T17:29:30.525232+0000","last_became_active":"2024-08-17T17:24:40.883220+0000","last_became_peered":"2024-08-17T17:24:40.883220+0000","last_unstale":"2024-08-17T17:29:30.525232+0000","last_undegraded":"2024-08-17T17:29:30.525232+0000","last_fullsized":"2024-08-17T17:29:30.525232+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:28.257805+0000","scrub_duration":0.0084952199999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1648,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.533077+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:29:30.533077+0000","last_peered":"2024-08-17T17:29:30.533077+0000","last_clean":"2024-08-17T17:29:30.533077+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:29:30.533077+0000","last_undegraded":"2024-08-17T17:29:30.533077+0000","last_fullsized":"2024-08-17T17:29:30.533077+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1623,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.534541+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:29:30.534541+0000","last_peered":"2024-08-17T17:29:30.534541+0000","last_clean":"2024-08-17T17:29:30.534541+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:29:30.534541+0000","last_undegraded":"2024-08-17T17:29:30.534541+0000","last_fullsized":"2024-08-17T17:29:30.534541+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1497,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.533578+0000","last_change":"2024-08-17T17:24:47.961645+0000","last_active":"2024-08-17T17:29:30.533578+0000","last_peered":"2024-08-17T17:29:30.533578+0000","last_clean":"2024-08-17T17:29:30.533578+0000","last_became_active":"2024-08-17T17:24:40.897539+0000","last_became_peered":"2024-08-17T17:24:40.897539+0000","last_unstale":"2024-08-17T17:29:30.533578+0000","last_undegraded":"2024-08-17T17:29:30.533578+0000","last_fullsized":"2024-08-17T17:29:30.533578+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:47.961445+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:47.961445+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:04:32.031483+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1236,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.518982+0000","last_change":"2024-08-17T17:21:06.483469+0000","last_active":"2024-08-17T17:29:24.518982+0000","last_peered":"2024-08-17T17:29:24.518982+0000","last_clean":"2024-08-17T17:29:24.518982+0000","last_became_active":"2024-08-17T17:21:06.483162+0000","last_became_peered":"2024-08-17T17:21:06.483162+0000","last_unstale":"2024-08-17T17:29:24.518982+0000","last_undegraded":"2024-08-17T17:29:24.518982+0000","last_fullsized":"2024-08-17T17:29:24.518982+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:24.898353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":245,"reported_epoch":1271,"state":"active+clean","last_fresh":"2024-08-17T17:29:24.505549+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:29:24.505549+0000","last_peered":"2024-08-17T17:29:24.505549+0000","last_clean":"2024-08-17T17:29:24.505549+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:29:24.505549+0000","last_undegraded":"2024-08-17T17:29:24.505549+0000","last_fullsized":"2024-08-17T17:29:24.505549+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2413756440,"num_objects":36832,"num_object_clones":0,"num_object_copies":110496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36832,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":211840,"num_write_kb":7957441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7399419904,"data_stored":7399407688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102140,"ondisk_log_size":102140,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201738,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1959408,"kb_used_data":1342632,"kb_used_omap":1,"kb_used_meta":616766,"kb_avail":91785744,"statfs":{"total":95995035648,"available":93988601856,"internally_reserved":0,"allocated":1374855168,"data_stored":1372179315,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":631568843},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,1,2,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463375,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":892832,"kb_used_data":460632,"kb_used_omap":1,"kb_used_meta":432190,"kb_avail":92852320,"statfs":{"total":95995035648,"available":95080775680,"internally_reserved":0,"allocated":471687168,"data_stored":469027004,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":442563017},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627016,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1978896,"kb_used_data":1238728,"kb_used_omap":42,"kb_used_meta":740117,"kb_avail":91766256,"statfs":{"total":95995035648,"available":93968646144,"internally_reserved":0,"allocated":1268457472,"data_stored":1265808949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":757879868},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,2,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785493,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2110356,"kb_used_data":1510284,"kb_used_omap":48,"kb_used_meta":600015,"kb_avail":91634796,"statfs":{"total":95995035648,"available":93834031104,"internally_reserved":0,"allocated":1546530816,"data_stored":1543878150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":614416046},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725011,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1915872,"kb_used_data":1317144,"kb_used_omap":1,"kb_used_meta":598718,"kb_avail":91829280,"statfs":{"total":95995035648,"available":94033182720,"internally_reserved":0,"allocated":1348755456,"data_stored":1346091686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":613087686},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,0,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986648,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":960356,"kb_used_data":455964,"kb_used_omap":1,"kb_used_meta":504382,"kb_avail":92784796,"statfs":{"total":95995035648,"available":95011631104,"internally_reserved":0,"allocated":466907136,"data_stored":464242933,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":516487622},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706255,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1020760,"kb_used_data":461200,"kb_used_omap":35,"kb_used_meta":559516,"kb_avail":92724392,"statfs":{"total":95995035648,"available":94949777408,"internally_reserved":0,"allocated":472268800,"data_stored":469613155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":572944439},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832028,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1282068,"kb_used_data":605452,"kb_used_omap":65,"kb_used_meta":676542,"kb_avail":92463084,"statfs":{"total":95995035648,"available":94682198016,"internally_reserved":0,"allocated":619982848,"data_stored":617330743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":692779195},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":602275840,"data_stored":602275840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1514082304,"data_stored":1514078232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":456458240,"data_stored":456458240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1257635840,"data_stored":1257635840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":455999488,"data_stored":455999488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1317605376,"data_stored":1317601304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":453705728,"data_stored":453705728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1341657088,"data_stored":1341653016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:29:39.866 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:29:39.902 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:39.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:39.938 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:39.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:40.039 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:40.080 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:40.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:40.181 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:40.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.221 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:40.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:40.322 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:40.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.357 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:40.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:40.458 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:40.493 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:40.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:40.595 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:40.635 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:40.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:40.737 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:40.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.768 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:40.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:40.869 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:40.905 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:40.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:40.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:41.007 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:41.039 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:41.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:41.141 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:41.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.179 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:41.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:41.281 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:41.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.320 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:41.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:41.421 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:41.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.459 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:41.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:41.560 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:41.605 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:41.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:41.654 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:41 smithi063 ceph-mon[114756]: pgmap v1806: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:29:41.655 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:41 smithi063 ceph-mon[109843]: pgmap v1806: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:29:41.706 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:41.708 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:29:41.708 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:29:41.708 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:29:41.708 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:29:41.708 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:29:41.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.760 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:41.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:41.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:41 smithi162 ceph-mon[104668]: pgmap v1806: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:29:41.861 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:41.894 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:41.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:41.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:41.996 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:42.052 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:42.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:42.154 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:42.188 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:42.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.189 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:42.290 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:42.332 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:42.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:42.434 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:42.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.468 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:42.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:42.569 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:42.607 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:42.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:42.708 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:42.718 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:42.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.765 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:42.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:42.866 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:42.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:42.912 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:42.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:43.013 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:43.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.046 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:43.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:43.090 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:29:43.147 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:43.178 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:43.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:43.280 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:43.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.315 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:43.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:43.416 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:43.455 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:43.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:43.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:29:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:29:43] "GET /metrics HTTP/1.1" 200 36403 "" "Prometheus/2.51.0" 2024-08-17T17:29:43.556 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:43.595 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:43.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:43.696 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:43.733 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:43.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:43.835 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:43.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:43 smithi162 ceph-mon[104668]: pgmap v1807: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:29:43.871 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:43.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:43.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:43.973 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:44.007 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:44.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:43 smithi063 ceph-mon[114756]: pgmap v1807: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:29:44.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:43 smithi063 ceph-mon[109843]: pgmap v1807: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:29:44.109 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:44.144 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:44.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:44.246 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:44.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.280 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:44.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:44.333 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:29:44.381 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:44.435 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:44.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:44.536 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:44.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.573 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:44.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:44.674 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:44.705 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:44.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:44.807 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:44.843 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:44.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:44.945 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:44.983 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:44.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:44.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:45.085 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:45.119 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:45.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:45.222 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:45.266 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:45.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:45.368 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:45.401 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:45.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:45.503 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:45.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.536 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:45.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:45.637 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:45.683 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:45.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:45.754 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:45 smithi063 ceph-mon[109843]: pgmap v1808: 119 pgs: 119 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:29:45.754 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:45 smithi063 ceph-mon[114756]: pgmap v1808: 119 pgs: 119 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:29:45.785 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:45.821 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:45.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:45.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:45 smithi162 ceph-mon[104668]: pgmap v1808: 119 pgs: 119 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:29:45.923 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:45.966 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:45.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:45.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:46.068 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:46.100 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:46.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:46.202 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:46.240 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:46.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:46.342 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:46.377 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:46.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:46.479 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:46.517 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:46.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:46.610 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:46.619 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:46.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.651 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:46.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:46.752 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:46.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.811 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:46.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:46.912 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:46.922 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:29:46.923 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:29:46.945 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:46.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:46.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:46.960 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:29:46.960 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:29:47.000 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:29:47.001 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:29:47.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:29:47.038 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:29:47.047 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:47.070 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:29:47.070 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:29:47.091 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:47.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:47.102 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:29:47.102 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:29:47.138 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:29:47.138 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:29:47.175 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:29:47.175 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:29:47.193 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:47.227 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:47.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:47.330 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:47.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.373 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:47.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:47.474 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:47.508 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:47.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:47.609 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:47.652 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:47.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:47.754 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:47.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:47.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:47.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:47 smithi063 ceph-mon[109843]: pgmap v1809: 119 pgs: 119 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:29:47.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:47 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4009158838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:47.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:47.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:47.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:47 smithi063 ceph-mon[114756]: pgmap v1809: 119 pgs: 119 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:29:47.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:47 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4009158838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:47.792 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:47.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:47.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:29:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:29:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:47 smithi162 ceph-mon[104668]: pgmap v1809: 119 pgs: 119 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-08-17T17:29:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:47 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4009158838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:47.894 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:47.926 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:47.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:47.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:48.028 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:48.065 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:48.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:48.167 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:48.202 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:48.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:48.304 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:48.341 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:48.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:48.444 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:48.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.481 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:48.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:48.582 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:48.614 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:48.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.615 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:48.706 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:29:48.707 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:29:48.716 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:48.746 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:48.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:48.848 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:48.885 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:48.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:48.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:48.987 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:49.012 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:49.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:49.114 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:29:49.147 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:49.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:29:49.249 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:49.292 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:49.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:49.394 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:49.429 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:49.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:49.531 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:49.553 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:49.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:49.654 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:49.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.692 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:49.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:49.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:49 smithi063 ceph-mon[114756]: pgmap v1810: 119 pgs: 119 active+clean; 1.6 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-17T17:29:49.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:49 smithi063 ceph-mon[114756]: from='client.35790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:29:49.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:49 smithi063 ceph-mon[109843]: pgmap v1810: 119 pgs: 119 active+clean; 1.6 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-17T17:29:49.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:49 smithi063 ceph-mon[109843]: from='client.35790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:29:49.793 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:49.827 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:49.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:49.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:49 smithi162 ceph-mon[104668]: pgmap v1810: 119 pgs: 119 active+clean; 1.6 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s 2024-08-17T17:29:49.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:49 smithi162 ceph-mon[104668]: from='client.35790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:29:49.928 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:49.963 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:49.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:49.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:50.064 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:50.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.095 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:50.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:50.142 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:29:50.196 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:50.221 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:50.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:50.323 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:50.374 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:50.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.375 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:50.476 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:50.514 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:50.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:50.616 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:50.657 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:50.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:50.759 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:50.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.804 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:50.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:50.905 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:50.950 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:50.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:50.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:50.984 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:51.017 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1810,"stamp":"2024-08-17T17:29:48.438030+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1739200509,"num_objects":26556,"num_object_clones":0,"num_object_copies":79668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26556,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":222206,"num_write_kb":7957474,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":1246,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106720,"ondisk_log_size":106720,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10055236,"kb_used_data":5345860,"kb_used_omap":199,"kb_used_meta":4708984,"kb_avail":739905980,"statfs":{"total":767960285184,"available":757663723520,"internally_reserved":0,"allocated":5474160640,"data_stored":5452855487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":4822000380},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,6,7,13,9,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-675151872,"num_objects":-10302,"num_object_clones":0,"num_object_copies":-30906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-10302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10302,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005831"},"pg_stats":[{"pgid":"1.11","version":"369'149","reported_seq":1597,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530134+0000","last_change":"2024-08-17T17:24:49.721626+0000","last_active":"2024-08-17T17:29:30.530134+0000","last_peered":"2024-08-17T17:29:30.530134+0000","last_clean":"2024-08-17T17:29:30.530134+0000","last_became_active":"2024-08-17T17:24:20.649128+0000","last_became_peered":"2024-08-17T17:24:20.649128+0000","last_unstale":"2024-08-17T17:29:30.530134+0000","last_undegraded":"2024-08-17T17:29:30.530134+0000","last_fullsized":"2024-08-17T17:29:30.530134+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:03:15.343059+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1644,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.576588+0000","last_change":"2024-08-17T17:21:06.463828+0000","last_active":"2024-08-17T17:29:38.576588+0000","last_peered":"2024-08-17T17:29:38.576588+0000","last_clean":"2024-08-17T17:29:38.576588+0000","last_became_active":"2024-08-17T17:21:06.463464+0000","last_became_peered":"2024-08-17T17:21:06.463464+0000","last_unstale":"2024-08-17T17:29:38.576588+0000","last_undegraded":"2024-08-17T17:29:38.576588+0000","last_fullsized":"2024-08-17T17:29:38.576588+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:46.465006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1204,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.529402+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:29:30.529402+0000","last_peered":"2024-08-17T17:29:30.529402+0000","last_clean":"2024-08-17T17:29:30.529402+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:29:30.529402+0000","last_undegraded":"2024-08-17T17:29:30.529402+0000","last_fullsized":"2024-08-17T17:29:30.529402+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1528,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530584+0000","last_change":"2024-08-17T17:24:48.765312+0000","last_active":"2024-08-17T17:29:30.530584+0000","last_peered":"2024-08-17T17:29:30.530584+0000","last_clean":"2024-08-17T17:29:30.530584+0000","last_became_active":"2024-08-17T17:24:20.642731+0000","last_became_peered":"2024-08-17T17:24:20.642731+0000","last_unstale":"2024-08-17T17:29:30.530584+0000","last_undegraded":"2024-08-17T17:29:30.530584+0000","last_fullsized":"2024-08-17T17:29:30.530584+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:08:36.138960+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1215,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.533847+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:29:30.533847+0000","last_peered":"2024-08-17T17:29:30.533847+0000","last_clean":"2024-08-17T17:29:30.533847+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:29:30.533847+0000","last_undegraded":"2024-08-17T17:29:30.533847+0000","last_fullsized":"2024-08-17T17:29:30.533847+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1071,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.525126+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:29:30.525126+0000","last_peered":"2024-08-17T17:29:30.525126+0000","last_clean":"2024-08-17T17:29:30.525126+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:29:30.525126+0000","last_undegraded":"2024-08-17T17:29:30.525126+0000","last_fullsized":"2024-08-17T17:29:30.525126+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":858,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.564566+0000","last_change":"2024-08-17T17:24:50.880046+0000","last_active":"2024-08-17T17:29:38.564566+0000","last_peered":"2024-08-17T17:29:38.564566+0000","last_clean":"2024-08-17T17:29:38.564566+0000","last_became_active":"2024-08-17T17:24:20.652165+0000","last_became_peered":"2024-08-17T17:24:20.652165+0000","last_unstale":"2024-08-17T17:29:38.564566+0000","last_undegraded":"2024-08-17T17:29:38.564566+0000","last_fullsized":"2024-08-17T17:29:38.564566+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:00.416810+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1196,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.570273+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:29:38.570273+0000","last_peered":"2024-08-17T17:29:38.570273+0000","last_clean":"2024-08-17T17:29:38.570273+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:29:38.570273+0000","last_undegraded":"2024-08-17T17:29:38.570273+0000","last_fullsized":"2024-08-17T17:29:38.570273+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1187,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.525024+0000","last_change":"2024-08-17T16:53:36.874104+0000","last_active":"2024-08-17T17:29:30.525024+0000","last_peered":"2024-08-17T17:29:30.525024+0000","last_clean":"2024-08-17T17:29:30.525024+0000","last_became_active":"2024-08-17T16:53:36.873834+0000","last_became_peered":"2024-08-17T16:53:36.873834+0000","last_unstale":"2024-08-17T17:29:30.525024+0000","last_undegraded":"2024-08-17T17:29:30.525024+0000","last_fullsized":"2024-08-17T17:29:30.525024+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:20:15.028628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1533,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530678+0000","last_change":"2024-08-17T17:24:50.727183+0000","last_active":"2024-08-17T17:29:30.530678+0000","last_peered":"2024-08-17T17:29:30.530678+0000","last_clean":"2024-08-17T17:29:30.530678+0000","last_became_active":"2024-08-17T17:24:20.646409+0000","last_became_peered":"2024-08-17T17:24:20.646409+0000","last_unstale":"2024-08-17T17:29:30.530678+0000","last_undegraded":"2024-08-17T17:29:30.530678+0000","last_fullsized":"2024-08-17T17:29:30.530678+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.086193+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1644,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.573074+0000","last_change":"2024-08-17T17:21:06.442598+0000","last_active":"2024-08-17T17:29:38.573074+0000","last_peered":"2024-08-17T17:29:38.573074+0000","last_clean":"2024-08-17T17:29:38.573074+0000","last_became_active":"2024-08-17T17:21:06.442264+0000","last_became_peered":"2024-08-17T17:21:06.442264+0000","last_unstale":"2024-08-17T17:29:38.573074+0000","last_undegraded":"2024-08-17T17:29:38.573074+0000","last_fullsized":"2024-08-17T17:29:38.573074+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:35.003620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1264,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.529253+0000","last_change":"2024-08-17T17:18:31.546175+0000","last_active":"2024-08-17T17:29:30.529253+0000","last_peered":"2024-08-17T17:29:30.529253+0000","last_clean":"2024-08-17T17:29:30.529253+0000","last_became_active":"2024-08-17T17:18:31.531640+0000","last_became_peered":"2024-08-17T17:18:31.531640+0000","last_unstale":"2024-08-17T17:29:30.529253+0000","last_undegraded":"2024-08-17T17:29:30.529253+0000","last_fullsized":"2024-08-17T17:29:30.529253+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:42.162499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.5","version":"1273'13586","reported_seq":13622,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:46.837648+0000","last_change":"2024-08-17T17:26:34.519835+0000","last_active":"2024-08-17T17:29:46.837648+0000","last_peered":"2024-08-17T17:29:46.837648+0000","last_clean":"2024-08-17T17:29:46.837648+0000","last_became_active":"2024-08-17T17:26:34.519427+0000","last_became_peered":"2024-08-17T17:26:34.519427+0000","last_unstale":"2024-08-17T17:29:46.837648+0000","last_undegraded":"2024-08-17T17:29:46.837648+0000","last_fullsized":"2024-08-17T17:29:46.837648+0000","mapping_epoch":1247,"log_start":"1258'6600","ondisk_log_start":"1258'6600","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":6986,"log_dups_size":0,"ondisk_log_size":6986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:31:25.352736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107085824,"num_objects":1634,"num_object_clones":0,"num_object_copies":4902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13586,"num_write_kb":487040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1471,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.529749+0000","last_change":"2024-08-17T17:24:50.410621+0000","last_active":"2024-08-17T17:29:30.529749+0000","last_peered":"2024-08-17T17:29:30.529749+0000","last_clean":"2024-08-17T17:29:30.529749+0000","last_became_active":"2024-08-17T17:24:31.863062+0000","last_became_peered":"2024-08-17T17:24:31.863062+0000","last_unstale":"2024-08-17T17:29:30.529749+0000","last_undegraded":"2024-08-17T17:29:30.529749+0000","last_fullsized":"2024-08-17T17:29:30.529749+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.410491+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.410491+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:11.717684+0000","scrub_duration":0.0066315480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1230,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.568796+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:29:38.568796+0000","last_peered":"2024-08-17T17:29:38.568796+0000","last_clean":"2024-08-17T17:29:38.568796+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:29:38.568796+0000","last_undegraded":"2024-08-17T17:29:38.568796+0000","last_fullsized":"2024-08-17T17:29:38.568796+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1202,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530356+0000","last_change":"2024-08-17T16:53:37.118736+0000","last_active":"2024-08-17T17:29:30.530356+0000","last_peered":"2024-08-17T17:29:30.530356+0000","last_clean":"2024-08-17T17:29:30.530356+0000","last_became_active":"2024-08-17T16:53:37.118400+0000","last_became_peered":"2024-08-17T16:53:37.118400+0000","last_unstale":"2024-08-17T17:29:30.530356+0000","last_undegraded":"2024-08-17T17:29:30.530356+0000","last_fullsized":"2024-08-17T17:29:30.530356+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:52.728955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"1273'13851","reported_seq":13887,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:46.817172+0000","last_change":"2024-08-17T17:26:34.715261+0000","last_active":"2024-08-17T17:29:46.817172+0000","last_peered":"2024-08-17T17:29:46.817172+0000","last_clean":"2024-08-17T17:29:46.817172+0000","last_became_active":"2024-08-17T17:26:34.714853+0000","last_became_peered":"2024-08-17T17:26:34.714853+0000","last_unstale":"2024-08-17T17:29:46.817172+0000","last_undegraded":"2024-08-17T17:29:46.817172+0000","last_fullsized":"2024-08-17T17:29:46.817172+0000","mapping_epoch":1247,"log_start":"1260'7400","ondisk_log_start":"1260'7400","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":6451,"log_dups_size":0,"ondisk_log_size":6451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:18.673096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110297088,"num_objects":1683,"num_object_clones":0,"num_object_copies":5049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13851,"num_write_kb":497088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1726,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.576274+0000","last_change":"2024-08-17T17:24:45.985725+0000","last_active":"2024-08-17T17:29:38.576274+0000","last_peered":"2024-08-17T17:29:38.576274+0000","last_clean":"2024-08-17T17:29:38.576274+0000","last_became_active":"2024-08-17T17:24:31.795836+0000","last_became_peered":"2024-08-17T17:24:31.795836+0000","last_unstale":"2024-08-17T17:29:38.576274+0000","last_undegraded":"2024-08-17T17:29:38.576274+0000","last_fullsized":"2024-08-17T17:29:38.576274+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:45.985596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:45.985596+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:17.897808+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1231,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.569268+0000","last_change":"2024-08-17T17:21:06.540689+0000","last_active":"2024-08-17T17:29:38.569268+0000","last_peered":"2024-08-17T17:29:38.569268+0000","last_clean":"2024-08-17T17:29:38.569268+0000","last_became_active":"2024-08-17T17:21:06.461614+0000","last_became_peered":"2024-08-17T17:21:06.461614+0000","last_unstale":"2024-08-17T17:29:38.569268+0000","last_undegraded":"2024-08-17T17:29:38.569268+0000","last_fullsized":"2024-08-17T17:29:38.569268+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:11.696898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1254,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.535929+0000","last_change":"2024-08-17T17:21:06.470389+0000","last_active":"2024-08-17T17:29:30.535929+0000","last_peered":"2024-08-17T17:29:30.535929+0000","last_clean":"2024-08-17T17:29:30.535929+0000","last_became_active":"2024-08-17T17:21:06.469966+0000","last_became_peered":"2024-08-17T17:21:06.469966+0000","last_unstale":"2024-08-17T17:29:30.535929+0000","last_undegraded":"2024-08-17T17:29:30.535929+0000","last_fullsized":"2024-08-17T17:29:30.535929+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:56.140616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"1273'14034","reported_seq":14070,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:46.782174+0000","last_change":"2024-08-17T17:26:34.412902+0000","last_active":"2024-08-17T17:29:46.782174+0000","last_peered":"2024-08-17T17:29:46.782174+0000","last_clean":"2024-08-17T17:29:46.782174+0000","last_became_active":"2024-08-17T17:26:34.412543+0000","last_became_peered":"2024-08-17T17:26:34.412543+0000","last_unstale":"2024-08-17T17:29:46.782174+0000","last_undegraded":"2024-08-17T17:29:46.782174+0000","last_fullsized":"2024-08-17T17:29:46.782174+0000","mapping_epoch":1247,"log_start":"1259'7000","ondisk_log_start":"1259'7000","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":7034,"log_dups_size":0,"ondisk_log_size":7034,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:56:20.767004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107085824,"num_objects":1634,"num_object_clones":0,"num_object_copies":4902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14034,"num_write_kb":501376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":839,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.529809+0000","last_change":"2024-08-17T17:24:48.192447+0000","last_active":"2024-08-17T17:29:30.529809+0000","last_peered":"2024-08-17T17:29:30.529809+0000","last_clean":"2024-08-17T17:29:30.529809+0000","last_became_active":"2024-08-17T17:24:20.644240+0000","last_became_peered":"2024-08-17T17:24:20.644240+0000","last_unstale":"2024-08-17T17:29:30.529809+0000","last_undegraded":"2024-08-17T17:29:30.529809+0000","last_fullsized":"2024-08-17T17:29:30.529809+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:56.795337+0000","scrub_duration":0.010920255,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1189,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:37.740091+0000","last_change":"2024-08-17T16:53:36.872329+0000","last_active":"2024-08-17T17:29:37.740091+0000","last_peered":"2024-08-17T17:29:37.740091+0000","last_clean":"2024-08-17T17:29:37.740091+0000","last_became_active":"2024-08-17T16:53:36.872064+0000","last_became_peered":"2024-08-17T16:53:36.872064+0000","last_unstale":"2024-08-17T17:29:37.740091+0000","last_undegraded":"2024-08-17T17:29:37.740091+0000","last_fullsized":"2024-08-17T17:29:37.740091+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:00:53.076558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1236,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.529287+0000","last_change":"2024-08-17T17:21:06.446798+0000","last_active":"2024-08-17T17:29:30.529287+0000","last_peered":"2024-08-17T17:29:30.529287+0000","last_clean":"2024-08-17T17:29:30.529287+0000","last_became_active":"2024-08-17T17:21:06.446205+0000","last_became_peered":"2024-08-17T17:21:06.446205+0000","last_unstale":"2024-08-17T17:29:30.529287+0000","last_undegraded":"2024-08-17T17:29:30.529287+0000","last_fullsized":"2024-08-17T17:29:30.529287+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:20:28.584048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"1273'14014","reported_seq":14049,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:46.837863+0000","last_change":"2024-08-17T17:26:35.023050+0000","last_active":"2024-08-17T17:29:46.837863+0000","last_peered":"2024-08-17T17:29:46.837863+0000","last_clean":"2024-08-17T17:29:46.837863+0000","last_became_active":"2024-08-17T17:26:35.022784+0000","last_became_peered":"2024-08-17T17:26:35.022784+0000","last_unstale":"2024-08-17T17:29:46.837863+0000","last_undegraded":"2024-08-17T17:29:46.837863+0000","last_fullsized":"2024-08-17T17:29:46.837863+0000","mapping_epoch":1247,"log_start":"1259'7000","ondisk_log_start":"1259'7000","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":7014,"log_dups_size":0,"ondisk_log_size":7014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:36:04.319458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108920832,"num_objects":1662,"num_object_clones":0,"num_object_copies":4986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14014,"num_write_kb":501632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":747,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.536861+0000","last_change":"2024-08-17T17:24:52.102415+0000","last_active":"2024-08-17T17:29:30.536861+0000","last_peered":"2024-08-17T17:29:30.536861+0000","last_clean":"2024-08-17T17:29:30.536861+0000","last_became_active":"2024-08-17T17:24:31.795843+0000","last_became_peered":"2024-08-17T17:24:31.795843+0000","last_unstale":"2024-08-17T17:29:30.536861+0000","last_undegraded":"2024-08-17T17:29:30.536861+0000","last_fullsized":"2024-08-17T17:29:30.536861+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:32:08.434197+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1218,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.245268+0000","last_change":"2024-08-17T17:07:45.683370+0000","last_active":"2024-08-17T17:29:38.245268+0000","last_peered":"2024-08-17T17:29:38.245268+0000","last_clean":"2024-08-17T17:29:38.245268+0000","last_became_active":"2024-08-17T17:07:45.682999+0000","last_became_peered":"2024-08-17T17:07:45.682999+0000","last_unstale":"2024-08-17T17:29:38.245268+0000","last_undegraded":"2024-08-17T17:29:38.245268+0000","last_fullsized":"2024-08-17T17:29:38.245268+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:27:09.704300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":999,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.536149+0000","last_change":"2024-08-17T17:21:06.460905+0000","last_active":"2024-08-17T17:29:30.536149+0000","last_peered":"2024-08-17T17:29:30.536149+0000","last_clean":"2024-08-17T17:29:30.536149+0000","last_became_active":"2024-08-17T17:21:06.460592+0000","last_became_peered":"2024-08-17T17:21:06.460592+0000","last_unstale":"2024-08-17T17:29:30.536149+0000","last_undegraded":"2024-08-17T17:29:30.536149+0000","last_fullsized":"2024-08-17T17:29:30.536149+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:53:10.146916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.1","version":"1273'13899","reported_seq":13964,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:45.990936+0000","last_change":"2024-08-17T17:26:34.410513+0000","last_active":"2024-08-17T17:29:45.990936+0000","last_peered":"2024-08-17T17:29:45.990936+0000","last_clean":"2024-08-17T17:29:45.990936+0000","last_became_active":"2024-08-17T17:26:34.410199+0000","last_became_peered":"2024-08-17T17:26:34.410199+0000","last_unstale":"2024-08-17T17:29:45.990936+0000","last_undegraded":"2024-08-17T17:29:45.990936+0000","last_fullsized":"2024-08-17T17:29:45.990936+0000","mapping_epoch":1247,"log_start":"1261'8100","ondisk_log_start":"1261'8100","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5799,"log_dups_size":0,"ondisk_log_size":5799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:06.961811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110952448,"num_objects":1693,"num_object_clones":0,"num_object_copies":5079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13899,"num_write_kb":498944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1618,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.525107+0000","last_change":"2024-08-17T17:24:54.418765+0000","last_active":"2024-08-17T17:29:30.525107+0000","last_peered":"2024-08-17T17:29:30.525107+0000","last_clean":"2024-08-17T17:29:30.525107+0000","last_became_active":"2024-08-17T17:24:31.800263+0000","last_became_peered":"2024-08-17T17:24:31.800263+0000","last_unstale":"2024-08-17T17:29:30.525107+0000","last_undegraded":"2024-08-17T17:29:30.525107+0000","last_fullsized":"2024-08-17T17:29:30.525107+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:54.418644+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:54.418644+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:11.024294+0000","scrub_duration":0.013346135,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1665,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.569935+0000","last_change":"2024-08-17T17:18:31.559328+0000","last_active":"2024-08-17T17:29:38.569935+0000","last_peered":"2024-08-17T17:29:38.569935+0000","last_clean":"2024-08-17T17:29:38.569935+0000","last_became_active":"2024-08-17T17:18:31.538861+0000","last_became_peered":"2024-08-17T17:18:31.538861+0000","last_unstale":"2024-08-17T17:29:38.569935+0000","last_undegraded":"2024-08-17T17:29:38.569935+0000","last_fullsized":"2024-08-17T17:29:38.569935+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:16:29.895105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1221,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.551299+0000","last_change":"2024-08-17T17:07:45.684413+0000","last_active":"2024-08-17T17:29:38.551299+0000","last_peered":"2024-08-17T17:29:38.551299+0000","last_clean":"2024-08-17T17:29:38.551299+0000","last_became_active":"2024-08-17T17:07:45.684149+0000","last_became_peered":"2024-08-17T17:07:45.684149+0000","last_unstale":"2024-08-17T17:29:38.551299+0000","last_undegraded":"2024-08-17T17:29:38.551299+0000","last_fullsized":"2024-08-17T17:29:38.551299+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:46:42.757309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"1273'13955","reported_seq":13991,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:46.825822+0000","last_change":"2024-08-17T17:26:35.023056+0000","last_active":"2024-08-17T17:29:46.825822+0000","last_peered":"2024-08-17T17:29:46.825822+0000","last_clean":"2024-08-17T17:29:46.825822+0000","last_became_active":"2024-08-17T17:26:35.022671+0000","last_became_peered":"2024-08-17T17:26:35.022671+0000","last_unstale":"2024-08-17T17:29:46.825822+0000","last_undegraded":"2024-08-17T17:29:46.825822+0000","last_fullsized":"2024-08-17T17:29:46.825822+0000","mapping_epoch":1247,"log_start":"1259'6900","ondisk_log_start":"1259'6900","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":7055,"log_dups_size":0,"ondisk_log_size":7055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:43.531150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107413504,"num_objects":1639,"num_object_clones":0,"num_object_copies":4917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13955,"num_write_kb":499008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1564,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530177+0000","last_change":"2024-08-17T17:24:58.712331+0000","last_active":"2024-08-17T17:29:30.530177+0000","last_peered":"2024-08-17T17:29:30.530177+0000","last_clean":"2024-08-17T17:29:30.530177+0000","last_became_active":"2024-08-17T17:24:31.802584+0000","last_became_peered":"2024-08-17T17:24:31.802584+0000","last_unstale":"2024-08-17T17:29:30.530177+0000","last_undegraded":"2024-08-17T17:29:30.530177+0000","last_fullsized":"2024-08-17T17:29:30.530177+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:59.616217+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1237,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.529423+0000","last_change":"2024-08-17T17:21:06.446841+0000","last_active":"2024-08-17T17:29:30.529423+0000","last_peered":"2024-08-17T17:29:30.529423+0000","last_clean":"2024-08-17T17:29:30.529423+0000","last_became_active":"2024-08-17T17:21:06.446271+0000","last_became_peered":"2024-08-17T17:21:06.446271+0000","last_unstale":"2024-08-17T17:29:30.529423+0000","last_undegraded":"2024-08-17T17:29:30.529423+0000","last_fullsized":"2024-08-17T17:29:30.529423+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:20.576826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1254,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.552930+0000","last_change":"2024-08-17T17:21:06.480911+0000","last_active":"2024-08-17T17:29:38.552930+0000","last_peered":"2024-08-17T17:29:38.552930+0000","last_clean":"2024-08-17T17:29:38.552930+0000","last_became_active":"2024-08-17T17:21:06.480551+0000","last_became_peered":"2024-08-17T17:21:06.480551+0000","last_unstale":"2024-08-17T17:29:38.552930+0000","last_undegraded":"2024-08-17T17:29:38.552930+0000","last_fullsized":"2024-08-17T17:29:38.552930+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:47:26.003704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"1273'13816","reported_seq":13851,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:46.818873+0000","last_change":"2024-08-17T17:26:34.715560+0000","last_active":"2024-08-17T17:29:46.818873+0000","last_peered":"2024-08-17T17:29:46.818873+0000","last_clean":"2024-08-17T17:29:46.818873+0000","last_became_active":"2024-08-17T17:26:34.714955+0000","last_became_peered":"2024-08-17T17:26:34.714955+0000","last_unstale":"2024-08-17T17:29:46.818873+0000","last_undegraded":"2024-08-17T17:29:46.818873+0000","last_fullsized":"2024-08-17T17:29:46.818873+0000","mapping_epoch":1247,"log_start":"1261'8000","ondisk_log_start":"1261'8000","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5816,"log_dups_size":0,"ondisk_log_size":5816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:35:46.367495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104595456,"num_objects":1596,"num_object_clones":0,"num_object_copies":4788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13816,"num_write_kb":493184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":796,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530385+0000","last_change":"2024-08-17T17:24:57.682573+0000","last_active":"2024-08-17T17:29:30.530385+0000","last_peered":"2024-08-17T17:29:30.530385+0000","last_clean":"2024-08-17T17:29:30.530385+0000","last_became_active":"2024-08-17T17:24:31.869575+0000","last_became_peered":"2024-08-17T17:24:31.869575+0000","last_unstale":"2024-08-17T17:29:30.530385+0000","last_undegraded":"2024-08-17T17:29:30.530385+0000","last_fullsized":"2024-08-17T17:29:30.530385+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:57.682409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:57.682409+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:40:42.144615+0000","scrub_duration":0.0066879779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1222,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.535995+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:29:30.535995+0000","last_peered":"2024-08-17T17:29:30.535995+0000","last_clean":"2024-08-17T17:29:30.535995+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:29:30.535995+0000","last_undegraded":"2024-08-17T17:29:30.535995+0000","last_fullsized":"2024-08-17T17:29:30.535995+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1239,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.551461+0000","last_change":"2024-08-17T17:21:06.451521+0000","last_active":"2024-08-17T17:29:38.551461+0000","last_peered":"2024-08-17T17:29:38.551461+0000","last_clean":"2024-08-17T17:29:38.551461+0000","last_became_active":"2024-08-17T17:21:06.451197+0000","last_became_peered":"2024-08-17T17:21:06.451197+0000","last_unstale":"2024-08-17T17:29:38.551461+0000","last_undegraded":"2024-08-17T17:29:38.551461+0000","last_fullsized":"2024-08-17T17:29:38.551461+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:04.000692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"1273'13779","reported_seq":13844,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:46.007262+0000","last_change":"2024-08-17T17:26:35.023654+0000","last_active":"2024-08-17T17:29:46.007262+0000","last_peered":"2024-08-17T17:29:46.007262+0000","last_clean":"2024-08-17T17:29:46.007262+0000","last_became_active":"2024-08-17T17:26:35.022968+0000","last_became_peered":"2024-08-17T17:26:35.022968+0000","last_unstale":"2024-08-17T17:29:46.007262+0000","last_undegraded":"2024-08-17T17:29:46.007262+0000","last_fullsized":"2024-08-17T17:29:46.007262+0000","mapping_epoch":1247,"log_start":"1261'8000","ondisk_log_start":"1261'8000","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5779,"log_dups_size":0,"ondisk_log_size":5779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:15:51.584797+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112263168,"num_objects":1713,"num_object_clones":0,"num_object_copies":5139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13779,"num_write_kb":495744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1564,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530657+0000","last_change":"2024-08-17T17:24:52.716059+0000","last_active":"2024-08-17T17:29:30.530657+0000","last_peered":"2024-08-17T17:29:30.530657+0000","last_clean":"2024-08-17T17:29:30.530657+0000","last_became_active":"2024-08-17T17:24:31.798173+0000","last_became_peered":"2024-08-17T17:24:31.798173+0000","last_unstale":"2024-08-17T17:29:30.530657+0000","last_undegraded":"2024-08-17T17:29:30.530657+0000","last_fullsized":"2024-08-17T17:29:30.530657+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:04:56.094994+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1647,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.576753+0000","last_change":"2024-08-17T17:21:06.441118+0000","last_active":"2024-08-17T17:29:38.576753+0000","last_peered":"2024-08-17T17:29:38.576753+0000","last_clean":"2024-08-17T17:29:38.576753+0000","last_became_active":"2024-08-17T17:21:06.440800+0000","last_became_peered":"2024-08-17T17:21:06.440800+0000","last_unstale":"2024-08-17T17:29:38.576753+0000","last_undegraded":"2024-08-17T17:29:38.576753+0000","last_fullsized":"2024-08-17T17:29:38.576753+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:08.091528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":153,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.563360+0000","last_change":"2024-08-17T17:22:48.374766+0000","last_active":"2024-08-17T17:29:38.563360+0000","last_peered":"2024-08-17T17:29:38.563360+0000","last_clean":"2024-08-17T17:29:38.563360+0000","last_became_active":"2024-08-17T17:22:48.373556+0000","last_became_peered":"2024-08-17T17:22:48.373556+0000","last_unstale":"2024-08-17T17:29:38.563360+0000","last_undegraded":"2024-08-17T17:29:38.563360+0000","last_fullsized":"2024-08-17T17:29:38.563360+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:33.362080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.8","version":"1273'13922","reported_seq":13985,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:45.998821+0000","last_change":"2024-08-17T17:26:34.519719+0000","last_active":"2024-08-17T17:29:45.998821+0000","last_peered":"2024-08-17T17:29:45.998821+0000","last_clean":"2024-08-17T17:29:45.998821+0000","last_became_active":"2024-08-17T17:26:34.519368+0000","last_became_peered":"2024-08-17T17:26:34.519368+0000","last_unstale":"2024-08-17T17:29:45.998821+0000","last_undegraded":"2024-08-17T17:29:45.998821+0000","last_fullsized":"2024-08-17T17:29:45.998821+0000","mapping_epoch":1247,"log_start":"1261'8100","ondisk_log_start":"1261'8100","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5822,"log_dups_size":0,"ondisk_log_size":5822,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:32.666019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111411200,"num_objects":1700,"num_object_clones":0,"num_object_copies":5100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13922,"num_write_kb":499904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1618,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.524643+0000","last_change":"2024-08-17T17:24:13.542832+0000","last_active":"2024-08-17T17:29:30.524643+0000","last_peered":"2024-08-17T17:29:30.524643+0000","last_clean":"2024-08-17T17:29:30.524643+0000","last_became_active":"2024-08-17T17:24:11.570282+0000","last_became_peered":"2024-08-17T17:24:11.570282+0000","last_unstale":"2024-08-17T17:29:30.524643+0000","last_undegraded":"2024-08-17T17:29:30.524643+0000","last_fullsized":"2024-08-17T17:29:30.524643+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:49.414162+0000","scrub_duration":0.017597167,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1232,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530465+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:29:30.530465+0000","last_peered":"2024-08-17T17:29:30.530465+0000","last_clean":"2024-08-17T17:29:30.530465+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:29:30.530465+0000","last_undegraded":"2024-08-17T17:29:30.530465+0000","last_fullsized":"2024-08-17T17:29:30.530465+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1612,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.564347+0000","last_change":"2024-08-17T17:07:45.681714+0000","last_active":"2024-08-17T17:29:38.564347+0000","last_peered":"2024-08-17T17:29:38.564347+0000","last_clean":"2024-08-17T17:29:38.564347+0000","last_became_active":"2024-08-17T17:07:45.680973+0000","last_became_peered":"2024-08-17T17:07:45.680973+0000","last_unstale":"2024-08-17T17:29:38.564347+0000","last_undegraded":"2024-08-17T17:29:38.564347+0000","last_fullsized":"2024-08-17T17:29:38.564347+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:02.180012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1204,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.529407+0000","last_change":"2024-08-17T16:53:37.119070+0000","last_active":"2024-08-17T17:29:30.529407+0000","last_peered":"2024-08-17T17:29:30.529407+0000","last_clean":"2024-08-17T17:29:30.529407+0000","last_became_active":"2024-08-17T16:53:37.118437+0000","last_became_peered":"2024-08-17T16:53:37.118437+0000","last_unstale":"2024-08-17T17:29:30.529407+0000","last_undegraded":"2024-08-17T17:29:30.529407+0000","last_fullsized":"2024-08-17T17:29:30.529407+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:11:10.290927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"1273'13795","reported_seq":13831,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:46.829107+0000","last_change":"2024-08-17T17:26:34.519823+0000","last_active":"2024-08-17T17:29:46.829107+0000","last_peered":"2024-08-17T17:29:46.829107+0000","last_clean":"2024-08-17T17:29:46.829107+0000","last_became_active":"2024-08-17T17:26:34.519497+0000","last_became_peered":"2024-08-17T17:26:34.519497+0000","last_unstale":"2024-08-17T17:29:46.829107+0000","last_undegraded":"2024-08-17T17:29:46.829107+0000","last_fullsized":"2024-08-17T17:29:46.829107+0000","mapping_epoch":1247,"log_start":"1258'6800","ondisk_log_start":"1258'6800","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":6995,"log_dups_size":0,"ondisk_log_size":6995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:59:31.807854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108724224,"num_objects":1659,"num_object_clones":0,"num_object_copies":4977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13795,"num_write_kb":494528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":857,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.529165+0000","last_change":"2024-08-17T17:24:11.936322+0000","last_active":"2024-08-17T17:29:30.529165+0000","last_peered":"2024-08-17T17:29:30.529165+0000","last_clean":"2024-08-17T17:29:30.529165+0000","last_became_active":"2024-08-17T17:24:11.574846+0000","last_became_peered":"2024-08-17T17:24:11.574846+0000","last_unstale":"2024-08-17T17:29:30.529165+0000","last_undegraded":"2024-08-17T17:29:30.529165+0000","last_fullsized":"2024-08-17T17:29:30.529165+0000","mapping_epoch":1176,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:11.936236+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:11.936236+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:31.674851+0000","scrub_duration":0.0035797139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1268,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.524576+0000","last_change":"2024-08-17T17:21:06.430639+0000","last_active":"2024-08-17T17:29:30.524576+0000","last_peered":"2024-08-17T17:29:30.524576+0000","last_clean":"2024-08-17T17:29:30.524576+0000","last_became_active":"2024-08-17T17:21:06.430297+0000","last_became_peered":"2024-08-17T17:21:06.430297+0000","last_unstale":"2024-08-17T17:29:30.524576+0000","last_undegraded":"2024-08-17T17:29:30.524576+0000","last_fullsized":"2024-08-17T17:29:30.524576+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:36:57.794620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1072,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.551988+0000","last_change":"2024-08-17T17:18:31.543656+0000","last_active":"2024-08-17T17:29:38.551988+0000","last_peered":"2024-08-17T17:29:38.551988+0000","last_clean":"2024-08-17T17:29:38.551988+0000","last_became_active":"2024-08-17T17:18:31.543126+0000","last_became_peered":"2024-08-17T17:18:31.543126+0000","last_unstale":"2024-08-17T17:29:38.551988+0000","last_undegraded":"2024-08-17T17:29:38.551988+0000","last_fullsized":"2024-08-17T17:29:38.551988+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:37:31.298235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":83,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530593+0000","last_change":"2024-08-17T17:22:48.370769+0000","last_active":"2024-08-17T17:29:30.530593+0000","last_peered":"2024-08-17T17:29:30.530593+0000","last_clean":"2024-08-17T17:29:30.530593+0000","last_became_active":"2024-08-17T17:22:48.370531+0000","last_became_peered":"2024-08-17T17:22:48.370531+0000","last_unstale":"2024-08-17T17:29:30.530593+0000","last_undegraded":"2024-08-17T17:29:30.530593+0000","last_fullsized":"2024-08-17T17:29:30.530593+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:07:02.943823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.a","version":"1273'13907","reported_seq":13942,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:46.842303+0000","last_change":"2024-08-17T17:26:34.715167+0000","last_active":"2024-08-17T17:29:46.842303+0000","last_peered":"2024-08-17T17:29:46.842303+0000","last_clean":"2024-08-17T17:29:46.842303+0000","last_became_active":"2024-08-17T17:26:34.714868+0000","last_became_peered":"2024-08-17T17:26:34.714868+0000","last_unstale":"2024-08-17T17:29:46.842303+0000","last_undegraded":"2024-08-17T17:29:46.842303+0000","last_fullsized":"2024-08-17T17:29:46.842303+0000","mapping_epoch":1247,"log_start":"1259'6900","ondisk_log_start":"1259'6900","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":7007,"log_dups_size":0,"ondisk_log_size":7007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:53:19.957252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109117440,"num_objects":1665,"num_object_clones":0,"num_object_copies":4995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13907,"num_write_kb":498304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1639,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.525047+0000","last_change":"2024-08-17T17:24:12.519946+0000","last_active":"2024-08-17T17:29:30.525047+0000","last_peered":"2024-08-17T17:29:30.525047+0000","last_clean":"2024-08-17T17:29:30.525047+0000","last_became_active":"2024-08-17T17:24:11.569747+0000","last_became_peered":"2024-08-17T17:24:11.569747+0000","last_unstale":"2024-08-17T17:29:30.525047+0000","last_undegraded":"2024-08-17T17:29:30.525047+0000","last_fullsized":"2024-08-17T17:29:30.525047+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.519838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.519838+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:56.003993+0000","scrub_duration":0.0068224740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1709,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.561416+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:29:38.561416+0000","last_peered":"2024-08-17T17:29:38.561416+0000","last_clean":"2024-08-17T17:29:38.561416+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:29:38.561416+0000","last_undegraded":"2024-08-17T17:29:38.561416+0000","last_fullsized":"2024-08-17T17:29:38.561416+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1102,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.563977+0000","last_change":"2024-08-17T17:21:06.428684+0000","last_active":"2024-08-17T17:29:38.563977+0000","last_peered":"2024-08-17T17:29:38.563977+0000","last_clean":"2024-08-17T17:29:38.563977+0000","last_became_active":"2024-08-17T17:21:06.428178+0000","last_became_peered":"2024-08-17T17:21:06.428178+0000","last_unstale":"2024-08-17T17:29:38.563977+0000","last_undegraded":"2024-08-17T17:29:38.563977+0000","last_fullsized":"2024-08-17T17:29:38.563977+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:14.836119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":915,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530548+0000","last_change":"2024-08-17T17:21:06.443565+0000","last_active":"2024-08-17T17:29:30.530548+0000","last_peered":"2024-08-17T17:29:30.530548+0000","last_clean":"2024-08-17T17:29:30.530548+0000","last_became_active":"2024-08-17T17:21:06.443215+0000","last_became_peered":"2024-08-17T17:21:06.443215+0000","last_unstale":"2024-08-17T17:29:30.530548+0000","last_undegraded":"2024-08-17T17:29:30.530548+0000","last_fullsized":"2024-08-17T17:29:30.530548+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:39:47.873057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.9","version":"1273'14191","reported_seq":14227,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:46.827923+0000","last_change":"2024-08-17T17:26:34.519795+0000","last_active":"2024-08-17T17:29:46.827923+0000","last_peered":"2024-08-17T17:29:46.827923+0000","last_clean":"2024-08-17T17:29:46.827923+0000","last_became_active":"2024-08-17T17:26:34.519426+0000","last_became_peered":"2024-08-17T17:26:34.519426+0000","last_unstale":"2024-08-17T17:29:46.827923+0000","last_undegraded":"2024-08-17T17:29:46.827923+0000","last_fullsized":"2024-08-17T17:29:46.827923+0000","mapping_epoch":1247,"log_start":"1261'8400","ondisk_log_start":"1261'8400","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5791,"log_dups_size":0,"ondisk_log_size":5791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:32:14.719799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107675648,"num_objects":1643,"num_object_clones":0,"num_object_copies":4929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14191,"num_write_kb":506688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1581,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530217+0000","last_change":"2024-08-17T17:24:13.795227+0000","last_active":"2024-08-17T17:29:30.530217+0000","last_peered":"2024-08-17T17:29:30.530217+0000","last_clean":"2024-08-17T17:29:30.530217+0000","last_became_active":"2024-08-17T17:24:11.578352+0000","last_became_peered":"2024-08-17T17:24:11.578352+0000","last_unstale":"2024-08-17T17:29:30.530217+0000","last_undegraded":"2024-08-17T17:29:30.530217+0000","last_fullsized":"2024-08-17T17:29:30.530217+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:36:39.440808+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1259,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530791+0000","last_change":"2024-08-17T17:21:06.456978+0000","last_active":"2024-08-17T17:29:30.530791+0000","last_peered":"2024-08-17T17:29:30.530791+0000","last_clean":"2024-08-17T17:29:30.530791+0000","last_became_active":"2024-08-17T17:21:06.452265+0000","last_became_peered":"2024-08-17T17:21:06.452265+0000","last_unstale":"2024-08-17T17:29:30.530791+0000","last_undegraded":"2024-08-17T17:29:30.530791+0000","last_fullsized":"2024-08-17T17:29:30.530791+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:18:52.893746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1241,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.536494+0000","last_change":"2024-08-17T17:18:31.537144+0000","last_active":"2024-08-17T17:29:30.536494+0000","last_peered":"2024-08-17T17:29:30.536494+0000","last_clean":"2024-08-17T17:29:30.536494+0000","last_became_active":"2024-08-17T17:18:31.536673+0000","last_became_peered":"2024-08-17T17:18:31.536673+0000","last_unstale":"2024-08-17T17:29:30.536494+0000","last_undegraded":"2024-08-17T17:29:30.536494+0000","last_fullsized":"2024-08-17T17:29:30.536494+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:29:05.983910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1177,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.562173+0000","last_change":"2024-08-17T17:21:06.435754+0000","last_active":"2024-08-17T17:29:38.562173+0000","last_peered":"2024-08-17T17:29:38.562173+0000","last_clean":"2024-08-17T17:29:38.562173+0000","last_became_active":"2024-08-17T17:21:06.435442+0000","last_became_peered":"2024-08-17T17:21:06.435442+0000","last_unstale":"2024-08-17T17:29:38.562173+0000","last_undegraded":"2024-08-17T17:29:38.562173+0000","last_fullsized":"2024-08-17T17:29:38.562173+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:35:34.849042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.f","version":"1273'13693","reported_seq":13729,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:46.842288+0000","last_change":"2024-08-17T17:26:34.716730+0000","last_active":"2024-08-17T17:29:46.842288+0000","last_peered":"2024-08-17T17:29:46.842288+0000","last_clean":"2024-08-17T17:29:46.842288+0000","last_became_active":"2024-08-17T17:26:34.716457+0000","last_became_peered":"2024-08-17T17:26:34.716457+0000","last_unstale":"2024-08-17T17:29:46.842288+0000","last_undegraded":"2024-08-17T17:29:46.842288+0000","last_fullsized":"2024-08-17T17:29:46.842288+0000","mapping_epoch":1247,"log_start":"1258'6700","ondisk_log_start":"1258'6700","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":6993,"log_dups_size":0,"ondisk_log_size":6993,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:53:51.133413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107675648,"num_objects":1643,"num_object_clones":0,"num_object_copies":4929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13693,"num_write_kb":490752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":844,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.529712+0000","last_change":"2024-08-17T17:24:55.041464+0000","last_active":"2024-08-17T17:29:30.529712+0000","last_peered":"2024-08-17T17:29:30.529712+0000","last_clean":"2024-08-17T17:29:30.529712+0000","last_became_active":"2024-08-17T17:24:31.800289+0000","last_became_peered":"2024-08-17T17:24:31.800289+0000","last_unstale":"2024-08-17T17:29:30.529712+0000","last_undegraded":"2024-08-17T17:29:30.529712+0000","last_fullsized":"2024-08-17T17:29:30.529712+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:55.041378+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:55.041378+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:43.974501+0000","scrub_duration":0.001623642,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1234,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.534155+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:29:30.534155+0000","last_peered":"2024-08-17T17:29:30.534155+0000","last_clean":"2024-08-17T17:29:30.534155+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:29:30.534155+0000","last_undegraded":"2024-08-17T17:29:30.534155+0000","last_fullsized":"2024-08-17T17:29:30.534155+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":924,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.525404+0000","last_change":"2024-08-17T17:21:06.434991+0000","last_active":"2024-08-17T17:29:30.525404+0000","last_peered":"2024-08-17T17:29:30.525404+0000","last_clean":"2024-08-17T17:29:30.525404+0000","last_became_active":"2024-08-17T17:21:06.434630+0000","last_became_peered":"2024-08-17T17:21:06.434630+0000","last_unstale":"2024-08-17T17:29:30.525404+0000","last_undegraded":"2024-08-17T17:29:30.525404+0000","last_fullsized":"2024-08-17T17:29:30.525404+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:27:24.104609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":866,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530729+0000","last_change":"2024-08-17T17:21:06.462532+0000","last_active":"2024-08-17T17:29:30.530729+0000","last_peered":"2024-08-17T17:29:30.530729+0000","last_clean":"2024-08-17T17:29:30.530729+0000","last_became_active":"2024-08-17T17:21:06.462183+0000","last_became_peered":"2024-08-17T17:21:06.462183+0000","last_unstale":"2024-08-17T17:29:30.530729+0000","last_undegraded":"2024-08-17T17:29:30.530729+0000","last_fullsized":"2024-08-17T17:29:30.530729+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:37:50.033322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.e","version":"1273'14243","reported_seq":14279,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:46.821460+0000","last_change":"2024-08-17T17:26:34.716166+0000","last_active":"2024-08-17T17:29:46.821460+0000","last_peered":"2024-08-17T17:29:46.821460+0000","last_clean":"2024-08-17T17:29:46.821460+0000","last_became_active":"2024-08-17T17:26:34.715299+0000","last_became_peered":"2024-08-17T17:26:34.715299+0000","last_unstale":"2024-08-17T17:29:46.821460+0000","last_undegraded":"2024-08-17T17:29:46.821460+0000","last_fullsized":"2024-08-17T17:29:46.821460+0000","mapping_epoch":1247,"log_start":"1261'8400","ondisk_log_start":"1261'8400","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5843,"log_dups_size":0,"ondisk_log_size":5843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:37.004999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109903872,"num_objects":1677,"num_object_clones":0,"num_object_copies":5031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14243,"num_write_kb":509440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":861,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.529548+0000","last_change":"2024-08-17T17:24:53.103270+0000","last_active":"2024-08-17T17:29:30.529548+0000","last_peered":"2024-08-17T17:29:30.529548+0000","last_clean":"2024-08-17T17:29:30.529548+0000","last_became_active":"2024-08-17T17:24:31.800823+0000","last_became_peered":"2024-08-17T17:24:31.800823+0000","last_unstale":"2024-08-17T17:29:30.529548+0000","last_undegraded":"2024-08-17T17:29:30.529548+0000","last_fullsized":"2024-08-17T17:29:30.529548+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.103133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.103133+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:52:52.444781+0000","scrub_duration":0.0074281759999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1100,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.562006+0000","last_change":"2024-08-17T17:21:06.433720+0000","last_active":"2024-08-17T17:29:38.562006+0000","last_peered":"2024-08-17T17:29:38.562006+0000","last_clean":"2024-08-17T17:29:38.562006+0000","last_became_active":"2024-08-17T17:21:06.433292+0000","last_became_peered":"2024-08-17T17:21:06.433292+0000","last_unstale":"2024-08-17T17:29:38.562006+0000","last_undegraded":"2024-08-17T17:29:38.562006+0000","last_fullsized":"2024-08-17T17:29:38.562006+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:12:32.246594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1234,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.533006+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:29:30.533006+0000","last_peered":"2024-08-17T17:29:30.533006+0000","last_clean":"2024-08-17T17:29:30.533006+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:29:30.533006+0000","last_undegraded":"2024-08-17T17:29:30.533006+0000","last_fullsized":"2024-08-17T17:29:30.533006+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":827,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.566879+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:29:38.566879+0000","last_peered":"2024-08-17T17:29:38.566879+0000","last_clean":"2024-08-17T17:29:38.566879+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:29:38.566879+0000","last_undegraded":"2024-08-17T17:29:38.566879+0000","last_fullsized":"2024-08-17T17:29:38.566879+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"1273'13888","reported_seq":13925,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:46.810103+0000","last_change":"2024-08-17T17:26:34.715123+0000","last_active":"2024-08-17T17:29:46.810103+0000","last_peered":"2024-08-17T17:29:46.810103+0000","last_clean":"2024-08-17T17:29:46.810103+0000","last_became_active":"2024-08-17T17:26:34.714789+0000","last_became_peered":"2024-08-17T17:26:34.714789+0000","last_unstale":"2024-08-17T17:29:46.810103+0000","last_undegraded":"2024-08-17T17:29:46.810103+0000","last_fullsized":"2024-08-17T17:29:46.810103+0000","mapping_epoch":1247,"log_start":"1261'8100","ondisk_log_start":"1261'8100","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5788,"log_dups_size":0,"ondisk_log_size":5788,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:19.314570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104529944,"num_objects":1596,"num_object_clones":0,"num_object_copies":4788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1596,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13888,"num_write_kb":495425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1585,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.536883+0000","last_change":"2024-08-17T17:24:49.119578+0000","last_active":"2024-08-17T17:29:30.536883+0000","last_peered":"2024-08-17T17:29:30.536883+0000","last_clean":"2024-08-17T17:29:30.536883+0000","last_became_active":"2024-08-17T17:24:31.807987+0000","last_became_peered":"2024-08-17T17:24:31.807987+0000","last_unstale":"2024-08-17T17:29:30.536883+0000","last_undegraded":"2024-08-17T17:29:30.536883+0000","last_fullsized":"2024-08-17T17:29:30.536883+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:26:34.052786+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1666,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.573389+0000","last_change":"2024-08-17T17:18:31.557510+0000","last_active":"2024-08-17T17:29:38.573389+0000","last_peered":"2024-08-17T17:29:38.573389+0000","last_clean":"2024-08-17T17:29:38.573389+0000","last_became_active":"2024-08-17T17:18:31.537594+0000","last_became_peered":"2024-08-17T17:18:31.537594+0000","last_unstale":"2024-08-17T17:29:38.573389+0000","last_undegraded":"2024-08-17T17:29:38.573389+0000","last_fullsized":"2024-08-17T17:29:38.573389+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:41.156598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":876,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.529677+0000","last_change":"2024-08-17T17:21:06.452395+0000","last_active":"2024-08-17T17:29:30.529677+0000","last_peered":"2024-08-17T17:29:30.529677+0000","last_clean":"2024-08-17T17:29:30.529677+0000","last_became_active":"2024-08-17T17:21:06.451867+0000","last_became_peered":"2024-08-17T17:21:06.451867+0000","last_unstale":"2024-08-17T17:29:30.529677+0000","last_undegraded":"2024-08-17T17:29:30.529677+0000","last_fullsized":"2024-08-17T17:29:30.529677+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:50.597092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1656,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.576452+0000","last_change":"2024-08-17T17:18:31.541109+0000","last_active":"2024-08-17T17:29:38.576452+0000","last_peered":"2024-08-17T17:29:38.576452+0000","last_clean":"2024-08-17T17:29:38.576452+0000","last_became_active":"2024-08-17T17:18:31.540777+0000","last_became_peered":"2024-08-17T17:18:31.540777+0000","last_unstale":"2024-08-17T17:29:38.576452+0000","last_undegraded":"2024-08-17T17:29:38.576452+0000","last_fullsized":"2024-08-17T17:29:38.576452+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:49:15.216931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"1273'13569","reported_seq":13634,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:46.006804+0000","last_change":"2024-08-17T17:26:34.412017+0000","last_active":"2024-08-17T17:29:46.006804+0000","last_peered":"2024-08-17T17:29:46.006804+0000","last_clean":"2024-08-17T17:29:46.006804+0000","last_became_active":"2024-08-17T17:26:34.411768+0000","last_became_peered":"2024-08-17T17:26:34.411768+0000","last_unstale":"2024-08-17T17:29:46.006804+0000","last_undegraded":"2024-08-17T17:29:46.006804+0000","last_fullsized":"2024-08-17T17:29:46.006804+0000","mapping_epoch":1247,"log_start":"1261'7700","ondisk_log_start":"1261'7700","created":1247,"last_epoch_clean":1248,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:26:33.391667+0000","last_clean_scrub_stamp":"2024-08-17T17:26:33.391667+0000","objects_scrubbed":0,"log_size":5869,"log_dups_size":0,"ondisk_log_size":5869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:23:07.054750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110952448,"num_objects":1693,"num_object_clones":0,"num_object_copies":5079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13569,"num_write_kb":488384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1691,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.573474+0000","last_change":"2024-08-17T17:24:12.033217+0000","last_active":"2024-08-17T17:29:38.573474+0000","last_peered":"2024-08-17T17:29:38.573474+0000","last_clean":"2024-08-17T17:29:38.573474+0000","last_became_active":"2024-08-17T17:24:11.572877+0000","last_became_peered":"2024-08-17T17:24:11.572877+0000","last_unstale":"2024-08-17T17:29:38.573474+0000","last_undegraded":"2024-08-17T17:29:38.573474+0000","last_fullsized":"2024-08-17T17:29:38.573474+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.033101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.033101+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.264475+0000","scrub_duration":20,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":245,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.537067+0000","last_change":"2024-08-17T17:21:06.519965+0000","last_active":"2024-08-17T17:29:30.537067+0000","last_peered":"2024-08-17T17:29:30.537067+0000","last_clean":"2024-08-17T17:29:30.537067+0000","last_became_active":"2024-08-17T17:21:06.459952+0000","last_became_peered":"2024-08-17T17:21:06.459952+0000","last_unstale":"2024-08-17T17:29:30.537067+0000","last_undegraded":"2024-08-17T17:29:30.537067+0000","last_fullsized":"2024-08-17T17:29:30.537067+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:27.828098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1231,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.529894+0000","last_change":"2024-08-17T17:07:12.645438+0000","last_active":"2024-08-17T17:29:30.529894+0000","last_peered":"2024-08-17T17:29:30.529894+0000","last_clean":"2024-08-17T17:29:30.529894+0000","last_became_active":"2024-08-17T17:07:12.645126+0000","last_became_peered":"2024-08-17T17:07:12.645126+0000","last_unstale":"2024-08-17T17:29:30.529894+0000","last_undegraded":"2024-08-17T17:29:30.529894+0000","last_fullsized":"2024-08-17T17:29:30.529894+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:00:06.726603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1207,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.535755+0000","last_change":"2024-08-17T16:53:36.899161+0000","last_active":"2024-08-17T17:29:30.535755+0000","last_peered":"2024-08-17T17:29:30.535755+0000","last_clean":"2024-08-17T17:29:30.535755+0000","last_became_active":"2024-08-17T16:53:36.898263+0000","last_became_peered":"2024-08-17T16:53:36.898263+0000","last_unstale":"2024-08-17T17:29:30.535755+0000","last_undegraded":"2024-08-17T17:29:30.535755+0000","last_fullsized":"2024-08-17T17:29:30.535755+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:22:32.564582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":756,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.564652+0000","last_change":"2024-08-17T17:24:48.935651+0000","last_active":"2024-08-17T17:29:38.564652+0000","last_peered":"2024-08-17T17:29:38.564652+0000","last_clean":"2024-08-17T17:29:38.564652+0000","last_became_active":"2024-08-17T17:24:40.897753+0000","last_became_peered":"2024-08-17T17:24:40.897753+0000","last_unstale":"2024-08-17T17:29:38.564652+0000","last_undegraded":"2024-08-17T17:29:38.564652+0000","last_fullsized":"2024-08-17T17:29:38.564652+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:10:15.118428+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1238,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.536407+0000","last_change":"2024-08-17T17:21:06.458000+0000","last_active":"2024-08-17T17:29:30.536407+0000","last_peered":"2024-08-17T17:29:30.536407+0000","last_clean":"2024-08-17T17:29:30.536407+0000","last_became_active":"2024-08-17T17:21:06.457411+0000","last_became_peered":"2024-08-17T17:21:06.457411+0000","last_unstale":"2024-08-17T17:29:30.536407+0000","last_undegraded":"2024-08-17T17:29:30.536407+0000","last_fullsized":"2024-08-17T17:29:30.536407+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:52.740639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1229,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.537247+0000","last_change":"2024-08-17T16:53:36.875444+0000","last_active":"2024-08-17T17:29:30.537247+0000","last_peered":"2024-08-17T17:29:30.537247+0000","last_clean":"2024-08-17T17:29:30.537247+0000","last_became_active":"2024-08-17T16:53:36.875106+0000","last_became_peered":"2024-08-17T16:53:36.875106+0000","last_unstale":"2024-08-17T17:29:30.537247+0000","last_undegraded":"2024-08-17T17:29:30.537247+0000","last_fullsized":"2024-08-17T17:29:30.537247+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:32.683969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":706,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530456+0000","last_change":"2024-08-17T17:25:03.663370+0000","last_active":"2024-08-17T17:29:30.530456+0000","last_peered":"2024-08-17T17:29:30.530456+0000","last_clean":"2024-08-17T17:29:30.530456+0000","last_became_active":"2024-08-17T17:24:40.886214+0000","last_became_peered":"2024-08-17T17:24:40.886214+0000","last_unstale":"2024-08-17T17:29:30.530456+0000","last_undegraded":"2024-08-17T17:29:30.530456+0000","last_fullsized":"2024-08-17T17:29:30.530456+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:11:39.272029+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":898,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.529268+0000","last_change":"2024-08-17T17:21:06.454587+0000","last_active":"2024-08-17T17:29:30.529268+0000","last_peered":"2024-08-17T17:29:30.529268+0000","last_clean":"2024-08-17T17:29:30.529268+0000","last_became_active":"2024-08-17T17:21:06.454229+0000","last_became_peered":"2024-08-17T17:21:06.454229+0000","last_unstale":"2024-08-17T17:29:30.529268+0000","last_undegraded":"2024-08-17T17:29:30.529268+0000","last_fullsized":"2024-08-17T17:29:30.529268+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:38.340263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1205,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.568853+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:29:38.568853+0000","last_peered":"2024-08-17T17:29:38.568853+0000","last_clean":"2024-08-17T17:29:38.568853+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:29:38.568853+0000","last_undegraded":"2024-08-17T17:29:38.568853+0000","last_fullsized":"2024-08-17T17:29:38.568853+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1693,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.561897+0000","last_change":"2024-08-17T17:24:20.649977+0000","last_active":"2024-08-17T17:29:38.561897+0000","last_peered":"2024-08-17T17:29:38.561897+0000","last_clean":"2024-08-17T17:29:38.561897+0000","last_became_active":"2024-08-17T17:24:20.649336+0000","last_became_peered":"2024-08-17T17:24:20.649336+0000","last_unstale":"2024-08-17T17:29:38.561897+0000","last_undegraded":"2024-08-17T17:29:38.561897+0000","last_fullsized":"2024-08-17T17:29:38.561897+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:57.187271+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1234,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530667+0000","last_change":"2024-08-17T17:21:06.442796+0000","last_active":"2024-08-17T17:29:30.530667+0000","last_peered":"2024-08-17T17:29:30.530667+0000","last_clean":"2024-08-17T17:29:30.530667+0000","last_became_active":"2024-08-17T17:21:06.442466+0000","last_became_peered":"2024-08-17T17:21:06.442466+0000","last_unstale":"2024-08-17T17:29:30.530667+0000","last_undegraded":"2024-08-17T17:29:30.530667+0000","last_fullsized":"2024-08-17T17:29:30.530667+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:34:36.366620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1611,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.564608+0000","last_change":"2024-08-17T17:07:45.684718+0000","last_active":"2024-08-17T17:29:38.564608+0000","last_peered":"2024-08-17T17:29:38.564608+0000","last_clean":"2024-08-17T17:29:38.564608+0000","last_became_active":"2024-08-17T17:07:45.683715+0000","last_became_peered":"2024-08-17T17:07:45.683715+0000","last_unstale":"2024-08-17T17:29:38.564608+0000","last_undegraded":"2024-08-17T17:29:38.564608+0000","last_fullsized":"2024-08-17T17:29:38.564608+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:19:28.413519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1453,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.533026+0000","last_change":"2024-08-17T17:25:07.472056+0000","last_active":"2024-08-17T17:29:30.533026+0000","last_peered":"2024-08-17T17:29:30.533026+0000","last_clean":"2024-08-17T17:29:30.533026+0000","last_became_active":"2024-08-17T17:24:40.883356+0000","last_became_peered":"2024-08-17T17:24:40.883356+0000","last_unstale":"2024-08-17T17:29:30.533026+0000","last_undegraded":"2024-08-17T17:29:30.533026+0000","last_fullsized":"2024-08-17T17:29:30.533026+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:43:27.883917+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1071,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.251979+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:29:38.251979+0000","last_peered":"2024-08-17T17:29:38.251979+0000","last_clean":"2024-08-17T17:29:38.251979+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:29:38.251979+0000","last_undegraded":"2024-08-17T17:29:38.251979+0000","last_fullsized":"2024-08-17T17:29:38.251979+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1644,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.573835+0000","last_change":"2024-08-17T17:21:06.468788+0000","last_active":"2024-08-17T17:29:38.573835+0000","last_peered":"2024-08-17T17:29:38.573835+0000","last_clean":"2024-08-17T17:29:38.573835+0000","last_became_active":"2024-08-17T17:21:06.467141+0000","last_became_peered":"2024-08-17T17:21:06.467141+0000","last_unstale":"2024-08-17T17:29:38.573835+0000","last_undegraded":"2024-08-17T17:29:38.573835+0000","last_fullsized":"2024-08-17T17:29:38.573835+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:05:51.301359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1453,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:37.740282+0000","last_change":"2024-08-17T17:24:53.130774+0000","last_active":"2024-08-17T17:29:37.740282+0000","last_peered":"2024-08-17T17:29:37.740282+0000","last_clean":"2024-08-17T17:29:37.740282+0000","last_became_active":"2024-08-17T17:24:40.883388+0000","last_became_peered":"2024-08-17T17:24:40.883388+0000","last_unstale":"2024-08-17T17:29:37.740282+0000","last_undegraded":"2024-08-17T17:29:37.740282+0000","last_fullsized":"2024-08-17T17:29:37.740282+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:42:36.772875+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1237,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.529811+0000","last_change":"2024-08-17T17:21:06.435998+0000","last_active":"2024-08-17T17:29:30.529811+0000","last_peered":"2024-08-17T17:29:30.529811+0000","last_clean":"2024-08-17T17:29:30.529811+0000","last_became_active":"2024-08-17T17:21:06.435579+0000","last_became_peered":"2024-08-17T17:21:06.435579+0000","last_unstale":"2024-08-17T17:29:30.529811+0000","last_undegraded":"2024-08-17T17:29:30.529811+0000","last_fullsized":"2024-08-17T17:29:30.529811+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:01.743074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1674,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.576657+0000","last_change":"2024-08-17T17:21:06.473627+0000","last_active":"2024-08-17T17:29:38.576657+0000","last_peered":"2024-08-17T17:29:38.576657+0000","last_clean":"2024-08-17T17:29:38.576657+0000","last_became_active":"2024-08-17T17:21:06.473062+0000","last_became_peered":"2024-08-17T17:21:06.473062+0000","last_unstale":"2024-08-17T17:29:38.576657+0000","last_undegraded":"2024-08-17T17:29:38.576657+0000","last_fullsized":"2024-08-17T17:29:38.576657+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:50:21.413163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":813,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:33.229480+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:29:33.229480+0000","last_peered":"2024-08-17T17:29:33.229480+0000","last_clean":"2024-08-17T17:29:33.229480+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:29:33.229480+0000","last_undegraded":"2024-08-17T17:29:33.229480+0000","last_fullsized":"2024-08-17T17:29:33.229480+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1032,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.525572+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:29:30.525572+0000","last_peered":"2024-08-17T17:29:30.525572+0000","last_clean":"2024-08-17T17:29:30.525572+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:29:30.525572+0000","last_undegraded":"2024-08-17T17:29:30.525572+0000","last_fullsized":"2024-08-17T17:29:30.525572+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1164,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.568627+0000","last_change":"2024-08-17T17:21:06.471265+0000","last_active":"2024-08-17T17:29:38.568627+0000","last_peered":"2024-08-17T17:29:38.568627+0000","last_clean":"2024-08-17T17:29:38.568627+0000","last_became_active":"2024-08-17T17:21:06.470943+0000","last_became_peered":"2024-08-17T17:21:06.470943+0000","last_unstale":"2024-08-17T17:29:38.568627+0000","last_undegraded":"2024-08-17T17:29:38.568627+0000","last_fullsized":"2024-08-17T17:29:38.568627+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:41.070024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1585,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530309+0000","last_change":"2024-08-17T17:25:02.696608+0000","last_active":"2024-08-17T17:29:30.530309+0000","last_peered":"2024-08-17T17:29:30.530309+0000","last_clean":"2024-08-17T17:29:30.530309+0000","last_became_active":"2024-08-17T17:24:40.877715+0000","last_became_peered":"2024-08-17T17:24:40.877715+0000","last_unstale":"2024-08-17T17:29:30.530309+0000","last_undegraded":"2024-08-17T17:29:30.530309+0000","last_fullsized":"2024-08-17T17:29:30.530309+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:48:00.941285+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1650,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.564322+0000","last_change":"2024-08-17T17:22:48.377351+0000","last_active":"2024-08-17T17:29:38.564322+0000","last_peered":"2024-08-17T17:29:38.564322+0000","last_clean":"2024-08-17T17:29:38.564322+0000","last_became_active":"2024-08-17T17:22:48.377059+0000","last_became_peered":"2024-08-17T17:22:48.377059+0000","last_unstale":"2024-08-17T17:29:38.564322+0000","last_undegraded":"2024-08-17T17:29:38.564322+0000","last_fullsized":"2024-08-17T17:29:38.564322+0000","mapping_epoch":1203,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:15.250902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1254,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.536102+0000","last_change":"2024-08-17T17:21:06.459139+0000","last_active":"2024-08-17T17:29:30.536102+0000","last_peered":"2024-08-17T17:29:30.536102+0000","last_clean":"2024-08-17T17:29:30.536102+0000","last_became_active":"2024-08-17T17:21:06.457850+0000","last_became_peered":"2024-08-17T17:21:06.457850+0000","last_unstale":"2024-08-17T17:29:30.536102+0000","last_undegraded":"2024-08-17T17:29:30.536102+0000","last_fullsized":"2024-08-17T17:29:30.536102+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:13.888576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1552,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.564265+0000","last_change":"2024-08-17T17:24:49.911734+0000","last_active":"2024-08-17T17:29:38.564265+0000","last_peered":"2024-08-17T17:29:38.564265+0000","last_clean":"2024-08-17T17:29:38.564265+0000","last_became_active":"2024-08-17T17:24:40.899102+0000","last_became_peered":"2024-08-17T17:24:40.899102+0000","last_unstale":"2024-08-17T17:29:38.564265+0000","last_undegraded":"2024-08-17T17:29:38.564265+0000","last_fullsized":"2024-08-17T17:29:38.564265+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.911622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.911622+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:30:27.987033+0000","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1692,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.562131+0000","last_change":"2024-08-17T17:21:06.446321+0000","last_active":"2024-08-17T17:29:38.562131+0000","last_peered":"2024-08-17T17:29:38.562131+0000","last_clean":"2024-08-17T17:29:38.562131+0000","last_became_active":"2024-08-17T17:21:06.446019+0000","last_became_peered":"2024-08-17T17:21:06.446019+0000","last_unstale":"2024-08-17T17:29:38.562131+0000","last_undegraded":"2024-08-17T17:29:38.562131+0000","last_fullsized":"2024-08-17T17:29:38.562131+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:17:26.750070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":223,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.536200+0000","last_change":"2024-08-17T17:18:31.537927+0000","last_active":"2024-08-17T17:29:30.536200+0000","last_peered":"2024-08-17T17:29:30.536200+0000","last_clean":"2024-08-17T17:29:30.536200+0000","last_became_active":"2024-08-17T17:18:31.537522+0000","last_became_peered":"2024-08-17T17:18:31.537522+0000","last_unstale":"2024-08-17T17:29:30.536200+0000","last_undegraded":"2024-08-17T17:29:30.536200+0000","last_fullsized":"2024-08-17T17:29:30.536200+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:29:48.791612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":756,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.570101+0000","last_change":"2024-08-17T17:24:46.946320+0000","last_active":"2024-08-17T17:29:38.570101+0000","last_peered":"2024-08-17T17:29:38.570101+0000","last_clean":"2024-08-17T17:29:38.570101+0000","last_became_active":"2024-08-17T17:24:40.880380+0000","last_became_peered":"2024-08-17T17:24:40.880380+0000","last_unstale":"2024-08-17T17:29:38.570101+0000","last_undegraded":"2024-08-17T17:29:38.570101+0000","last_fullsized":"2024-08-17T17:29:38.570101+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:54:30.637162+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1252,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.536301+0000","last_change":"2024-08-17T17:21:06.465804+0000","last_active":"2024-08-17T17:29:30.536301+0000","last_peered":"2024-08-17T17:29:30.536301+0000","last_clean":"2024-08-17T17:29:30.536301+0000","last_became_active":"2024-08-17T17:21:06.465461+0000","last_became_peered":"2024-08-17T17:21:06.465461+0000","last_unstale":"2024-08-17T17:29:30.536301+0000","last_undegraded":"2024-08-17T17:29:30.536301+0000","last_fullsized":"2024-08-17T17:29:30.536301+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:46:06.443667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1244,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.530478+0000","last_change":"2024-08-17T17:21:06.481634+0000","last_active":"2024-08-17T17:29:30.530478+0000","last_peered":"2024-08-17T17:29:30.530478+0000","last_clean":"2024-08-17T17:29:30.530478+0000","last_became_active":"2024-08-17T17:21:06.469081+0000","last_became_peered":"2024-08-17T17:21:06.469081+0000","last_unstale":"2024-08-17T17:29:30.530478+0000","last_undegraded":"2024-08-17T17:29:30.530478+0000","last_fullsized":"2024-08-17T17:29:30.530478+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:53:26.954738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1252,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.535973+0000","last_change":"2024-08-17T17:18:31.538878+0000","last_active":"2024-08-17T17:29:30.535973+0000","last_peered":"2024-08-17T17:29:30.535973+0000","last_clean":"2024-08-17T17:29:30.535973+0000","last_became_active":"2024-08-17T17:18:31.538543+0000","last_became_peered":"2024-08-17T17:18:31.538543+0000","last_unstale":"2024-08-17T17:29:30.535973+0000","last_undegraded":"2024-08-17T17:29:30.535973+0000","last_fullsized":"2024-08-17T17:29:30.535973+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:53:10.521061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1203,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.534057+0000","last_change":"2024-08-17T16:53:36.874839+0000","last_active":"2024-08-17T17:29:30.534057+0000","last_peered":"2024-08-17T17:29:30.534057+0000","last_clean":"2024-08-17T17:29:30.534057+0000","last_became_active":"2024-08-17T16:53:36.874476+0000","last_became_peered":"2024-08-17T16:53:36.874476+0000","last_unstale":"2024-08-17T17:29:30.534057+0000","last_undegraded":"2024-08-17T17:29:30.534057+0000","last_fullsized":"2024-08-17T17:29:30.534057+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:25:25.803379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1644,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:37.740029+0000","last_change":"2024-08-17T17:25:02.417545+0000","last_active":"2024-08-17T17:29:37.740029+0000","last_peered":"2024-08-17T17:29:37.740029+0000","last_clean":"2024-08-17T17:29:37.740029+0000","last_became_active":"2024-08-17T17:24:40.883220+0000","last_became_peered":"2024-08-17T17:24:40.883220+0000","last_unstale":"2024-08-17T17:29:37.740029+0000","last_undegraded":"2024-08-17T17:29:37.740029+0000","last_fullsized":"2024-08-17T17:29:37.740029+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:39:28.257805+0000","scrub_duration":0.0084952199999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1650,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.568902+0000","last_change":"2024-08-17T17:18:31.536378+0000","last_active":"2024-08-17T17:29:38.568902+0000","last_peered":"2024-08-17T17:29:38.568902+0000","last_clean":"2024-08-17T17:29:38.568902+0000","last_became_active":"2024-08-17T17:18:31.536043+0000","last_became_peered":"2024-08-17T17:18:31.536043+0000","last_unstale":"2024-08-17T17:29:38.568902+0000","last_undegraded":"2024-08-17T17:29:38.568902+0000","last_fullsized":"2024-08-17T17:29:38.568902+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:45:34.518793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1626,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.570417+0000","last_change":"2024-08-17T17:07:45.686261+0000","last_active":"2024-08-17T17:29:38.570417+0000","last_peered":"2024-08-17T17:29:38.570417+0000","last_clean":"2024-08-17T17:29:38.570417+0000","last_became_active":"2024-08-17T17:07:45.682615+0000","last_became_peered":"2024-08-17T17:07:45.682615+0000","last_unstale":"2024-08-17T17:29:38.570417+0000","last_undegraded":"2024-08-17T17:29:38.570417+0000","last_fullsized":"2024-08-17T17:29:38.570417+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:26:20.565799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1500,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.564225+0000","last_change":"2024-08-17T17:24:47.961645+0000","last_active":"2024-08-17T17:29:38.564225+0000","last_peered":"2024-08-17T17:29:38.564225+0000","last_clean":"2024-08-17T17:29:38.564225+0000","last_became_active":"2024-08-17T17:24:40.897539+0000","last_became_peered":"2024-08-17T17:24:40.897539+0000","last_unstale":"2024-08-17T17:29:38.564225+0000","last_undegraded":"2024-08-17T17:29:38.564225+0000","last_fullsized":"2024-08-17T17:29:38.564225+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:47.961445+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:47.961445+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:04:32.031483+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1237,"reported_epoch":1272,"state":"active+clean","last_fresh":"2024-08-17T17:29:30.535658+0000","last_change":"2024-08-17T17:21:06.483469+0000","last_active":"2024-08-17T17:29:30.535658+0000","last_peered":"2024-08-17T17:29:30.535658+0000","last_clean":"2024-08-17T17:29:30.535658+0000","last_became_active":"2024-08-17T17:21:06.483162+0000","last_became_peered":"2024-08-17T17:21:06.483162+0000","last_unstale":"2024-08-17T17:29:30.535658+0000","last_undegraded":"2024-08-17T17:29:30.535658+0000","last_fullsized":"2024-08-17T17:29:30.535658+0000","mapping_epoch":1173,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:48:24.898353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":247,"reported_epoch":1273,"state":"active+clean","last_fresh":"2024-08-17T17:29:38.566725+0000","last_change":"2024-08-17T17:18:31.548983+0000","last_active":"2024-08-17T17:29:38.566725+0000","last_peered":"2024-08-17T17:29:38.566725+0000","last_clean":"2024-08-17T17:29:38.566725+0000","last_became_active":"2024-08-17T17:18:31.535592+0000","last_became_peered":"2024-08-17T17:18:31.535592+0000","last_unstale":"2024-08-17T17:29:38.566725+0000","last_undegraded":"2024-08-17T17:29:38.566725+0000","last_fullsized":"2024-08-17T17:29:38.566725+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:02:43.913313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1738604568,"num_objects":26530,"num_object_clones":0,"num_object_copies":79590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26530,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":222142,"num_write_kb":7957441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5326778368,"data_stored":5326766152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102042,"ondisk_log_size":102042,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201740,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1577340,"kb_used_data":972788,"kb_used_omap":1,"kb_used_meta":604478,"kb_avail":92167812,"statfs":{"total":95995035648,"available":94379839488,"internally_reserved":0,"allocated":996134912,"data_stored":993455031,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":618985931},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,3,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463377,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":765228,"kb_used_data":324836,"kb_used_omap":1,"kb_used_meta":440382,"kb_avail":92979924,"statfs":{"total":95995035648,"available":95211442176,"internally_reserved":0,"allocated":332632064,"data_stored":329967872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":450951625},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627018,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1622236,"kb_used_data":882068,"kb_used_omap":42,"kb_used_meta":740117,"kb_avail":92122916,"statfs":{"total":95995035648,"available":94333865984,"internally_reserved":0,"allocated":903237632,"data_stored":900585081,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":757879868},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,5,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785495,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1700832,"kb_used_data":1113240,"kb_used_omap":48,"kb_used_meta":587535,"kb_avail":92044320,"statfs":{"total":95995035648,"available":94253383680,"internally_reserved":0,"allocated":1139957760,"data_stored":1137301066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":601636526},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725013,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1562476,"kb_used_data":957988,"kb_used_omap":1,"kb_used_meta":604478,"kb_avail":92182676,"statfs":{"total":95995035648,"available":94395060224,"internally_reserved":0,"allocated":980979712,"data_stored":978311914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":618985926},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,3,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986650,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":843888,"kb_used_data":331304,"kb_used_omap":1,"kb_used_meta":512574,"kb_avail":92901264,"statfs":{"total":95995035648,"available":95130894336,"internally_reserved":0,"allocated":339255296,"data_stored":336587065,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":524876230},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706257,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":880996,"kb_used_data":329692,"kb_used_omap":35,"kb_used_meta":551196,"kb_avail":92864156,"statfs":{"total":95995035648,"available":95092895744,"internally_reserved":0,"allocated":337604608,"data_stored":334944935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":564424759},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832030,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1102240,"kb_used_data":433944,"kb_used_omap":65,"kb_used_meta":668222,"kb_avail":92642912,"statfs":{"total":95995035648,"available":94866341888,"internally_reserved":0,"allocated":444358656,"data_stored":441702523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":684259515},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":427425792,"data_stored":427425792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1103499264,"data_stored":1103495192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":322174976,"data_stored":322174976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":891355136,"data_stored":891355136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327286784,"data_stored":327286784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":967446528,"data_stored":967442456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":323092480,"data_stored":323092480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":964497408,"data_stored":964493336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:29:51.022 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:29:51.022 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:29:51.022 INFO:tasks.thrashosds.thrasher:Killing osd 6, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2024-08-17T17:29:51.022 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-08-17T17:29:51.022 DEBUG:teuthology.orchestra.run.smithi162:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.6 2024-08-17T17:29:51.052 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:51.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.099 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:51.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:51.200 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:51.235 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:51.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:51.337 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:51.346 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:29:51 smithi162 systemd[1]: Stopping Ceph osd.6 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:29:51.375 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:51.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:51.477 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:51.513 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:51.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:51.615 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:51.620 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:51 smithi063 ceph-mon[114756]: pgmap v1811: 119 pgs: 119 active+clean; 1.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-08-17T17:29:51.620 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:51 smithi063 ceph-mon[109843]: pgmap v1811: 119 pgs: 119 active+clean; 1.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-08-17T17:29:51.653 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:51.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:51.688 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:51.755 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:51.773 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:51 smithi162 ceph-mon[104668]: pgmap v1811: 119 pgs: 119 active+clean; 1.6 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-08-17T17:29:51.792 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:51.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:51.894 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:51.944 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:51.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:51.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:52.046 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:52.080 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:52.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:52.096 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:29:51 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[51613]: 2024-08-17T17:29:51.768+0000 7efc1d5fe640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:29:52.096 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:29:51 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[51613]: 2024-08-17T17:29:51.768+0000 7efc1d5fe640 -1 osd.6 1273 *** Got signal Terminated *** 2024-08-17T17:29:52.096 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:29:51 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[51613]: 2024-08-17T17:29:51.768+0000 7efc1d5fe640 -1 osd.6 1273 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:29:52.182 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:52.219 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:52.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:52.321 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:52.356 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:52.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:52.458 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:52.480 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:52.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:52.533 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:29:52.582 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:52.611 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:52.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:52.713 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:52.735 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:52.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:52.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:52 smithi063 ceph-mon[114756]: osd.6 marked itself down and dead 2024-08-17T17:29:52.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:52 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1978331301' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:29:52.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:52 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:29:52.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:52 smithi063 ceph-mon[109843]: osd.6 marked itself down and dead 2024-08-17T17:29:52.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:52 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1978331301' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:29:52.782 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:52 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:29:52.837 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:52.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:52 smithi162 ceph-mon[104668]: osd.6 marked itself down and dead 2024-08-17T17:29:52.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:52 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1978331301' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:29:52.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:52 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:29:52.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.858 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:52.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:52.959 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:52.982 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:52.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:52.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:53.083 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:53.106 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:53.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:53.207 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:53.230 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:53.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:53.332 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:53.354 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:53.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:53.455 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:53.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.478 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:53.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:53.530 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:29:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:29:53] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.51.0" 2024-08-17T17:29:53.531 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:29:53 smithi162 podman[198300]: 2024-08-17 17:29:53.143568971 +0000 UTC m=+1.772989501 container died af91211ce044714f807af4d1608d4132e11b598fde640a0c9beac04804153ac5 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-08-17T17:29:53.531 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:53 smithi162 ceph-mon[104668]: pgmap v1812: 119 pgs: 119 active+clean; 1.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.30k op/s 2024-08-17T17:29:53.532 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:53 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:53.532 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:53 smithi162 ceph-mon[104668]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:29:53.532 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:53 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:29:53.532 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:53 smithi162 ceph-mon[104668]: osdmap e1274: 8 total, 7 up, 8 in 2024-08-17T17:29:53.578 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:53.610 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:53.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:53.712 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:53.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.746 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:53.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:53.824 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:53 smithi063 ceph-mon[114756]: pgmap v1812: 119 pgs: 119 active+clean; 1.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.30k op/s 2024-08-17T17:29:53.824 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:53 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:53.824 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:53 smithi063 ceph-mon[114756]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:29:53.824 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:53 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:29:53.824 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:53 smithi063 ceph-mon[114756]: osdmap e1274: 8 total, 7 up, 8 in 2024-08-17T17:29:53.825 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:53 smithi063 ceph-mon[109843]: pgmap v1812: 119 pgs: 119 active+clean; 1.3 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.30k op/s 2024-08-17T17:29:53.825 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:53 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:29:53.825 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:53 smithi063 ceph-mon[109843]: Health check failed: 1 osds down (OSD_DOWN) 2024-08-17T17:29:53.825 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:53 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:29:53.825 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:53 smithi063 ceph-mon[109843]: osdmap e1274: 8 total, 7 up, 8 in 2024-08-17T17:29:53.847 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:53.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:53.884 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:53.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:53.985 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:54.023 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:54.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:54.125 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:54.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.153 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:54.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:54.254 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:54.283 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:54.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:54.385 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:54.414 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:54.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:54.515 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:54.561 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:54.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:54.663 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:54.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.716 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:54.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:54.741 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:54 smithi063 ceph-mon[109843]: osdmap e1275: 8 total, 7 up, 8 in 2024-08-17T17:29:54.742 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:54 smithi063 ceph-mon[114756]: osdmap e1275: 8 total, 7 up, 8 in 2024-08-17T17:29:54.818 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:54.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:54 smithi162 ceph-mon[104668]: osdmap e1275: 8 total, 7 up, 8 in 2024-08-17T17:29:54.857 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:54.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:54.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:54.959 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:55.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.012 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:55.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:55.113 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:55.146 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:55.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:55.247 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:55.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.284 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:55.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:55.385 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:55.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.431 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:55.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:55.438 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:29:55.438 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:29:55.532 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:55.578 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:55.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:55.680 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:55.704 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:29:55.704 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:29:55.714 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:55.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:55.753 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:29:55.753 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:29:55.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:55 smithi063 ceph-mon[114756]: pgmap v1815: 119 pgs: 14 stale+active+clean, 105 active+clean; 1.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:29:55.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:55 smithi063 ceph-mon[109843]: pgmap v1815: 119 pgs: 14 stale+active+clean, 105 active+clean; 1.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:29:55.785 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:29:55.785 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:29:55.807 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:29:55.808 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:29:55.816 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:55.824 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:29:55.825 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:29:55.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:55 smithi162 ceph-mon[104668]: pgmap v1815: 119 pgs: 14 stale+active+clean, 105 active+clean; 1.3 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-08-17T17:29:55.853 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:55.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:55.858 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:29:55.858 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:29:55.863 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:29:55.895 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:29:55.895 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:29:55.955 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:55.984 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:55.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:55.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:56.086 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:56.120 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:56.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:56.175 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:29:56.222 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:56.248 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:56.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:56.350 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:56.403 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:56.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:56.452 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:29:56.504 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:56.549 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:56.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:56.650 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:56.690 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:56.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:56.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:56 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/79741994' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:56 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/79741994' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:56.791 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:56.827 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:56.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:56.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:56 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/79741994' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:29:56.929 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:56.964 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:56.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:56.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:57.066 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:57.110 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:57.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:57.173 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:29:57.174 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:29:57.212 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:57.219 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:57.237 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:57.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:57.339 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:57.377 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:57.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:57.481 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:57.509 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:57.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:57.611 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:57.646 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:57.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:57.747 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:57.759 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:57 smithi063 ceph-mon[114756]: pgmap v1816: 119 pgs: 8 active+undersized, 12 stale+active+clean, 2 active+undersized+degraded, 97 active+clean; 1.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 4.5 KiB/s rd, 14 KiB/s wr, 795 op/s; 1112/60654 objects degraded (1.833%); 9.8 KiB/s, 0 objects/s recovering 2024-08-17T17:29:57.760 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:57 smithi063 ceph-mon[114756]: Health check failed: Degraded data redundancy: 1112/60654 objects degraded (1.833%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:29:57.760 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:57 smithi063 ceph-mon[109843]: pgmap v1816: 119 pgs: 8 active+undersized, 12 stale+active+clean, 2 active+undersized+degraded, 97 active+clean; 1.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 4.5 KiB/s rd, 14 KiB/s wr, 795 op/s; 1112/60654 objects degraded (1.833%); 9.8 KiB/s, 0 objects/s recovering 2024-08-17T17:29:57.760 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:57 smithi063 ceph-mon[109843]: Health check failed: Degraded data redundancy: 1112/60654 objects degraded (1.833%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:29:57.771 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:57.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:57.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:57 smithi162 ceph-mon[104668]: pgmap v1816: 119 pgs: 8 active+undersized, 12 stale+active+clean, 2 active+undersized+degraded, 97 active+clean; 1.2 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 4.5 KiB/s rd, 14 KiB/s wr, 795 op/s; 1112/60654 objects degraded (1.833%); 9.8 KiB/s, 0 objects/s recovering 2024-08-17T17:29:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:57 smithi162 ceph-mon[104668]: Health check failed: Degraded data redundancy: 1112/60654 objects degraded (1.833%), 2 pgs degraded (PG_DEGRADED) 2024-08-17T17:29:57.873 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:57.899 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:57.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:57.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:58.001 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:58.034 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:58.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:58.135 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:29:58.173 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:58.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:29:58.275 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:58.320 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:58.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:58.422 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:58.460 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:58.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:58.562 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:58.590 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:58.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:58.692 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:29:58.734 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:58.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:29:58.836 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:58.876 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:58.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:58.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:58.978 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:59.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.013 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:59.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:59.084 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:29:58 smithi162 podman[198300]: 2024-08-17 17:29:58.596137715 +0000 UTC m=+7.225558255 container cleanup af91211ce044714f807af4d1608d4132e11b598fde640a0c9beac04804153ac5 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-08-17T17:29:59.084 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:29:58 smithi162 bash[198300]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6 2024-08-17T17:29:59.114 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:59.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.164 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:59.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:59.175 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:29:59.182 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:29:59.266 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:29:59.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.302 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:59.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:29:59.346 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:29:59 smithi162 podman[198438]: 2024-08-17 17:29:59.083822533 +0000 UTC m=+5.723396050 container remove af91211ce044714f807af4d1608d4132e11b598fde640a0c9beac04804153ac5 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-08-17T17:29:59.402 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:29:59.427 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:59.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:29:59.529 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:29:59.585 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:59.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:29:59.607 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:29:59.687 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:29:59.697 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:29:59 smithi162 ceph-mon[104668]: pgmap v1817: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 1.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 7.6 KiB/s rd, 23 KiB/s wr, 1.16k op/s; 3235/51825 objects degraded (6.242%); 16 KiB/s, 0 objects/s recovering 2024-08-17T17:29:59.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.732 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:29:59.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:29:59.754 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:29:59 smithi063 ceph-mon[114756]: pgmap v1817: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 1.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 7.6 KiB/s rd, 23 KiB/s wr, 1.16k op/s; 3235/51825 objects degraded (6.242%); 16 KiB/s, 0 objects/s recovering 2024-08-17T17:29:59.754 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:29:59 smithi063 ceph-mon[109843]: pgmap v1817: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 1.1 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 7.6 KiB/s rd, 23 KiB/s wr, 1.16k op/s; 3235/51825 objects degraded (6.242%); 16 KiB/s, 0 objects/s recovering 2024-08-17T17:29:59.833 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:29:59.863 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:29:59.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:29:59.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:29:59.965 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:00.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.008 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:00.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:00.109 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:00.147 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:00.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:00.250 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:00.290 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:00.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:00.392 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:00.435 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:00.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:00.537 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:00.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.576 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:00.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:00.598 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:30:00.677 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:00.713 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:00.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:00.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/330246440' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 3235/51825 objects degraded (6.242%), 9 pgs degraded 2024-08-17T17:30:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-08-17T17:30:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: [WRN] OSD_DOWN: 1 osds down 2024-08-17T17:30:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: osd.6 (root=default,host=smithi162) is down 2024-08-17T17:30:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: [WRN] PG_DEGRADED: Degraded data redundancy: 3235/51825 objects degraded (6.242%), 9 pgs degraded 2024-08-17T17:30:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: pg 1.c is active+undersized+degraded, acting [3,0] 2024-08-17T17:30:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-08-17T17:30:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: pg 3.d is active+undersized+degraded, acting [7,5] 2024-08-17T17:30:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: pg 3.10 is active+undersized+degraded, acting [5,0] 2024-08-17T17:30:00.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-08-17T17:30:00.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-08-17T17:30:00.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: pg 9.0 is active+undersized+degraded, acting [2,1] 2024-08-17T17:30:00.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: pg 9.1 is active+undersized+degraded, acting [3,7] 2024-08-17T17:30:00.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[109843]: pg 9.b is active+undersized+degraded, acting [5,0] 2024-08-17T17:30:00.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/330246440' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:00.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:00.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 3235/51825 objects degraded (6.242%), 9 pgs degraded 2024-08-17T17:30:00.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-08-17T17:30:00.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: [WRN] OSD_DOWN: 1 osds down 2024-08-17T17:30:00.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: osd.6 (root=default,host=smithi162) is down 2024-08-17T17:30:00.781 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: [WRN] PG_DEGRADED: Degraded data redundancy: 3235/51825 objects degraded (6.242%), 9 pgs degraded 2024-08-17T17:30:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: pg 1.c is active+undersized+degraded, acting [3,0] 2024-08-17T17:30:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-08-17T17:30:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: pg 3.d is active+undersized+degraded, acting [7,5] 2024-08-17T17:30:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: pg 3.10 is active+undersized+degraded, acting [5,0] 2024-08-17T17:30:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-08-17T17:30:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-08-17T17:30:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: pg 9.0 is active+undersized+degraded, acting [2,1] 2024-08-17T17:30:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: pg 9.1 is active+undersized+degraded, acting [3,7] 2024-08-17T17:30:00.782 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:00 smithi063 ceph-mon[114756]: pg 9.b is active+undersized+degraded, acting [5,0] 2024-08-17T17:30:00.816 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/330246440' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:00.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:00.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 3235/51825 objects degraded (6.242%), 9 pgs degraded 2024-08-17T17:30:00.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-08-17T17:30:00.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: [WRN] OSD_DOWN: 1 osds down 2024-08-17T17:30:00.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: osd.6 (root=default,host=smithi162) is down 2024-08-17T17:30:00.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: [WRN] PG_DEGRADED: Degraded data redundancy: 3235/51825 objects degraded (6.242%), 9 pgs degraded 2024-08-17T17:30:00.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: pg 1.c is active+undersized+degraded, acting [3,0] 2024-08-17T17:30:00.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: pg 2.2 is active+undersized+degraded, acting [5,1] 2024-08-17T17:30:00.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: pg 3.d is active+undersized+degraded, acting [7,5] 2024-08-17T17:30:00.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: pg 3.10 is active+undersized+degraded, acting [5,0] 2024-08-17T17:30:00.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: pg 3.14 is active+undersized+degraded, acting [4,7] 2024-08-17T17:30:00.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: pg 3.1d is active+undersized+degraded, acting [5,4] 2024-08-17T17:30:00.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: pg 9.0 is active+undersized+degraded, acting [2,1] 2024-08-17T17:30:00.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: pg 9.1 is active+undersized+degraded, acting [3,7] 2024-08-17T17:30:00.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:00 smithi162 ceph-mon[104668]: pg 9.b is active+undersized+degraded, acting [5,0] 2024-08-17T17:30:00.858 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:00.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:00.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:00.961 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:01.017 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:01.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:01.119 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:01.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.171 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:01.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:01.272 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:01.290 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:30:01.290 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:30:01.290 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 0 2024-08-17T17:30:01.290 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:30:01.308 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:01.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:01.410 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:01.444 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:01.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:01.533 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:30:01 smithi162 podman[199064]: 2024-08-17 17:30:01.270453812 +0000 UTC m=+0.121291728 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:30:01.533 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:30:01 smithi162 podman[199064]: 2024-08-17 17:30:01.402403709 +0000 UTC m=+0.253241607 container create 06650a273ca8a8e9157b9a4bbe57bea0a22c1d9e291c18ed4990f503b1907cee (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=reef-ced649f) 2024-08-17T17:30:01.547 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:01.587 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:01.689 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:01.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.720 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:01.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:01.821 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:01 smithi162 ceph-mon[104668]: pgmap v1818: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 1.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 7.6 KiB/s rd, 23 KiB/s wr, 494 op/s; 3235/51825 objects degraded (6.242%); 16 KiB/s, 0 objects/s recovering 2024-08-17T17:30:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:01 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:01 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:01 smithi162 ceph-mon[104668]: osdmap e1276: 8 total, 7 up, 8 in 2024-08-17T17:30:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:01.873 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:01.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:01.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:01.975 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:02.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.019 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:02.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:01 smithi063 ceph-mon[109843]: pgmap v1818: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 1.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 7.6 KiB/s rd, 23 KiB/s wr, 494 op/s; 3235/51825 objects degraded (6.242%); 16 KiB/s, 0 objects/s recovering 2024-08-17T17:30:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:01 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:01 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:01 smithi063 ceph-mon[109843]: osdmap e1276: 8 total, 7 up, 8 in 2024-08-17T17:30:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:02.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:01 smithi063 ceph-mon[114756]: pgmap v1818: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 1.1 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 7.6 KiB/s rd, 23 KiB/s wr, 494 op/s; 3235/51825 objects degraded (6.242%); 16 KiB/s, 0 objects/s recovering 2024-08-17T17:30:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:01 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:01 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:01 smithi063 ceph-mon[114756]: osdmap e1276: 8 total, 7 up, 8 in 2024-08-17T17:30:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:02.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:02.129 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:30:01 smithi162 podman[199064]: 2024-08-17 17:30:01.869242044 +0000 UTC m=+0.720079945 container init 06650a273ca8a8e9157b9a4bbe57bea0a22c1d9e291c18ed4990f503b1907cee (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f) 2024-08-17T17:30:02.129 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:30:01 smithi162 podman[199064]: 2024-08-17 17:30:01.880433311 +0000 UTC m=+0.731271204 container start 06650a273ca8a8e9157b9a4bbe57bea0a22c1d9e291c18ed4990f503b1907cee (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-08-17T17:30:02.129 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:30:02 smithi162 podman[199064]: 2024-08-17 17:30:02.008200506 +0000 UTC m=+0.859038394 container attach 06650a273ca8a8e9157b9a4bbe57bea0a22c1d9e291c18ed4990f503b1907cee (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=) 2024-08-17T17:30:02.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.165 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:02.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:02.266 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:02.303 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:02.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:02.406 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:02.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.459 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:02.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:02.560 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:02.596 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:30:02 smithi162 podman[199064]: 2024-08-17 17:30:02.353470337 +0000 UTC m=+1.204308233 container died 06650a273ca8a8e9157b9a4bbe57bea0a22c1d9e291c18ed4990f503b1907cee (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:30:02.606 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:02.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:02.706 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:02 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 2203/35472 objects degraded (6.211%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:02.706 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:02 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 2203/35472 objects degraded (6.211%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:02.709 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:02.725 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:30:02.734 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:02.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:02.750 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:30:02.751 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:30:02.782 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:30:02.782 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:30:02.814 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:30:02.815 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:30:02.836 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:02.853 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:30:02.854 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:30:02.871 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:02.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:02.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:02.887 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:30:02.887 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:30:02.919 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:30:02.919 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:30:02.921 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:30:02.954 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:30:02.955 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:30:02.974 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:03.009 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:03.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:03.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:02 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 2203/35472 objects degraded (6.211%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:03.111 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:03.151 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:03.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:03.189 DEBUG:teuthology.orchestra.run.smithi162:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.6.service' 2024-08-17T17:30:03.253 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:03.270 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:30:03.270 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-08-17T17:30:03.306 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:03.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:03.326 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:03.409 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:03.447 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:03.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:03.549 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:03.589 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:03.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:03.657 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:30:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:30:03] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.51.0" 2024-08-17T17:30:03.691 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:03.728 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:03.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:03.760 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:30:03.760 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:30:03.830 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:03.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:03 smithi162 ceph-mon[104668]: pgmap v1820: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 738 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 6.8 KiB/s rd, 21 KiB/s wr, 1.05k op/s; 2203/35472 objects degraded (6.211%); 15 KiB/s, 0 objects/s recovering 2024-08-17T17:30:03.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:03 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2486181749' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:03.863 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:03.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:03.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:03.965 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:04.003 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:04.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:04.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:03 smithi063 ceph-mon[109843]: pgmap v1820: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 738 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 6.8 KiB/s rd, 21 KiB/s wr, 1.05k op/s; 2203/35472 objects degraded (6.211%); 15 KiB/s, 0 objects/s recovering 2024-08-17T17:30:04.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:03 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2486181749' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:04.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:03 smithi063 ceph-mon[114756]: pgmap v1820: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 738 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 6.8 KiB/s rd, 21 KiB/s wr, 1.05k op/s; 2203/35472 objects degraded (6.211%); 15 KiB/s, 0 objects/s recovering 2024-08-17T17:30:04.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:03 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2486181749' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:04.105 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:04.137 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:04.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:04.161 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:30:04.239 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:04.277 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:04.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:04.379 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:04.426 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:04.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:04.528 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:04.572 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:04.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:04.585 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:04.674 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:04.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.713 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:04.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:04.814 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:04.850 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:04.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:04.952 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:04.975 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:04.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:04.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:05.076 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:05.104 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:05.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:05.206 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:05.241 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:05.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:05.343 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:05.384 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:05.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:05.486 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:05.530 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:05.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:05.632 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:05.659 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:05.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:05.761 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:05.762 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:30:05.792 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:05.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:05.894 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:05.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:05.931 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:05.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:05 smithi063 ceph-mon[114756]: pgmap v1821: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 738 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 6.1 KiB/s rd, 18 KiB/s wr, 940 op/s; 2203/35472 objects degraded (6.211%); 13 KiB/s, 0 objects/s recovering 2024-08-17T17:30:06.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:05 smithi063 ceph-mon[109843]: pgmap v1821: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 738 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 6.1 KiB/s rd, 18 KiB/s wr, 940 op/s; 2203/35472 objects degraded (6.211%); 13 KiB/s, 0 objects/s recovering 2024-08-17T17:30:06.032 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:06.072 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:06.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:05 smithi162 ceph-mon[104668]: pgmap v1821: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 738 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 6.1 KiB/s rd, 18 KiB/s wr, 940 op/s; 2203/35472 objects degraded (6.211%); 13 KiB/s, 0 objects/s recovering 2024-08-17T17:30:06.174 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:06.212 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:06.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:06.314 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:06.350 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:06.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:06.453 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:06.490 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:06.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:06.592 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:06.631 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:06.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:06.733 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:06.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.769 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:06.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:06.870 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:06.902 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:06.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:06.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:06.986 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:30:06.986 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:30:07.005 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:07.037 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:07.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:07.139 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:07.175 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:07.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:07.276 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:07.319 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:07.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:07.421 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:07.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.458 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:07.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:07.559 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:07.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.598 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:07.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:07.699 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:07.731 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:07.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:07.753 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:30:07.833 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:07.866 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:07.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:07.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:07.959 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:07 smithi063 ceph-mon[109843]: pgmap v1822: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 653 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.5 KiB/s rd, 7.3 KiB/s wr, 976 op/s; 1874/31395 objects degraded (5.969%); 5.2 KiB/s, 0 objects/s recovering 2024-08-17T17:30:07.960 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:07 smithi063 ceph-mon[114756]: pgmap v1822: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 653 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.5 KiB/s rd, 7.3 KiB/s wr, 976 op/s; 1874/31395 objects degraded (5.969%); 5.2 KiB/s, 0 objects/s recovering 2024-08-17T17:30:07.968 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:07.999 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:07.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:08.050 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:07 smithi162 ceph-mon[104668]: pgmap v1822: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 653 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 2.5 KiB/s rd, 7.3 KiB/s wr, 976 op/s; 1874/31395 objects degraded (5.969%); 5.2 KiB/s, 0 objects/s recovering 2024-08-17T17:30:08.093 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:08.100 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:08.145 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:08.146 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:08.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:08.248 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:08.271 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0, 1] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:30:08.271 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:30:08.271 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [2, 3, 4, 5, 6, 7, 0, 1] 2024-08-17T17:30:08.271 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd out 1 2024-08-17T17:30:08.291 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:08.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:08.315 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:30:08 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=infra.usagestats t=2024-08-17T17:30:08.04809842Z level=info msg="Usage stats are ready to report" 2024-08-17T17:30:08.393 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:08.422 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:08.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:08.524 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:08.566 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:08.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:08.670 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:08.704 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:08.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:08.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:08 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 1874/31395 objects degraded (5.969%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:08.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:08 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 1874/31395 objects degraded (5.969%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:08.806 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:08.836 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:08.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.837 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:08.885 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:08 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 1874/31395 objects degraded (5.969%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:08.938 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:08.974 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:08.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:08.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:09.075 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:09.119 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:09.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:09.220 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:09.264 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:09.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:09.366 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:09.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.401 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:09.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:09.502 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:09.537 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:09.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:09.639 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:09.676 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:09.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:09.700 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:30:09.777 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:09.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.821 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:09.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:09.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:09.966 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:09.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:09.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:09.977 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:09 smithi162 ceph-mon[104668]: pgmap v1823: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 392 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 1194/18885 objects degraded (6.322%) 2024-08-17T17:30:09.977 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:09 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2154221476' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:30:09.977 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:09 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:30:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:09 smithi063 ceph-mon[114756]: pgmap v1823: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 392 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 1194/18885 objects degraded (6.322%) 2024-08-17T17:30:10.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:09 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2154221476' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:30:10.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:09 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:30:10.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:09 smithi063 ceph-mon[109843]: pgmap v1823: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 392 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 1194/18885 objects degraded (6.322%) 2024-08-17T17:30:10.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:09 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2154221476' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:30:10.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:09 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:30:10.067 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:10.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.103 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:10.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:10.204 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:10.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.241 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:10.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:10.286 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:30:10.287 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:30:10.341 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:10.378 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:10.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:10.480 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:10.506 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:10.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:10.608 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:10.629 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:10.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:10.731 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:10.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.768 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:10.768 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:10.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:10 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:10.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:10 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:30:10.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:10 smithi063 ceph-mon[114756]: osdmap e1277: 8 total, 7 up, 8 in 2024-08-17T17:30:10.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:10 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:10.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:10 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:30:10.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:10 smithi063 ceph-mon[109843]: osdmap e1277: 8 total, 7 up, 8 in 2024-08-17T17:30:10.869 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:10.900 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:10 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:10.901 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:10 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:30:10.901 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:10 smithi162 ceph-mon[104668]: osdmap e1277: 8 total, 7 up, 8 in 2024-08-17T17:30:10.902 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:10.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:10.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:10.936 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:30:11.004 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:11.039 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:11.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:11.141 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:11.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.173 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:11.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:11.252 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:11.274 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:11.312 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:11.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:11.413 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:11.452 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:11.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:11.553 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:11.592 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:11.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:11.677 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:11.694 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:11.730 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:11.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:11.768 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:30:11.768 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:30:11.802 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:30:11.802 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:30:11.822 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:11 smithi063 ceph-mon[109843]: pgmap v1825: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 392 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 1194/18885 objects degraded (6.322%) 2024-08-17T17:30:11.823 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:11 smithi063 ceph-mon[114756]: pgmap v1825: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 392 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 1194/18885 objects degraded (6.322%) 2024-08-17T17:30:11.832 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:11.839 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:30:11.839 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:30:11.862 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:11.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:11.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:11.876 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:30:11.877 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:30:11.909 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:30:11.909 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:30:11.954 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:30:11.955 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:30:11.955 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:30:11.963 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:11.990 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:30:11.991 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:30:12.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.013 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:12.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:11 smithi162 ceph-mon[104668]: pgmap v1825: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 392 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 1194/18885 objects degraded (6.322%) 2024-08-17T17:30:12.113 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:12.153 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:12.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:12.254 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:12.288 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:12.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:12.390 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:12.412 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:12.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:12.509 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:30:12.513 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:12.535 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:12.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:12.637 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:12.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.684 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:12.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:12.752 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Removed 124335 objects 2024-08-17T17:30:12.752 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Clean up completed and total clean up time :116.273 2024-08-17T17:30:12.785 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:12.807 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:12.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:12.909 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:12.931 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:12.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:12.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:12.989 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:12 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3632649681' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:13.014 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:30:13.014 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:30:13.014 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 0 2024-08-17T17:30:13.015 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:30:13.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:12 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3632649681' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:13.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:12 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3632649681' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:13.034 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:13.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.059 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:13.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:13.159 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:13.182 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:13.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:13.284 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:13.308 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:13.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:13.410 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:13.434 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:13.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:30:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:30:13] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.51.0" 2024-08-17T17:30:13.536 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:13.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.560 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:13.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:13.661 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:13.664 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:30:13.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.684 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:13.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:13.786 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:13.807 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:13.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:13.848 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2024-08-17T17:30:13.848 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2024-08-17T17:30:13.875 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:13 smithi063 ceph-mon[114756]: pgmap v1826: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 53 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 153/2604 objects degraded (5.876%) 2024-08-17T17:30:13.875 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:13 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 153/2604 objects degraded (5.876%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:13.875 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:13 smithi063 ceph-mon[109843]: pgmap v1826: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 53 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 153/2604 objects degraded (5.876%) 2024-08-17T17:30:13.876 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:13 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 153/2604 objects degraded (5.876%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:13.903 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:13.910 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:13.933 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:13.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:13.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:14.025 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:13 smithi162 ceph-mon[104668]: pgmap v1826: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 53 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 153/2604 objects degraded (5.876%) 2024-08-17T17:30:14.025 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:13 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 153/2604 objects degraded (5.876%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:14.035 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:14.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.057 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:14.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:14.158 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:14.179 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:14.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:14.281 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:14.304 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:14.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:14.406 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:14.429 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:30:14.429 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:30:14.429 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:14.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:14.531 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:14.554 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:14.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:14.655 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:14.678 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:14.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:14.780 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:14.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.804 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:14.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:14.905 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:14.922 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:14.931 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:14.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:14.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:14.936 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:15.034 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:15.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.057 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:15.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:15.158 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:15.180 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:15.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:15.282 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:15.304 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:15.304 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:15 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a[109820]: 2024-08-17T17:30:15.112+0000 7f66fd20a640 -1 mon.a@0(leader).osd e1277 definitely_dead 0 2024-08-17T17:30:15.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:15.406 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:15.429 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:15.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:15.531 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:15.552 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:15.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:15.654 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:15.676 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:15.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:15.732 INFO:teuthology.orchestra.run.smithi063.stderr:marked out osd.1. 2024-08-17T17:30:15.779 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:15.799 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:15.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:15.901 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:15.924 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:15.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:15.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:16.012 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:15 smithi162 ceph-mon[104668]: pgmap v1827: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 53 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 153/2604 objects degraded (5.876%) 2024-08-17T17:30:16.013 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:15 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1468800005' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T17:30:16.013 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:15 smithi162 ceph-mon[104668]: Client client.admin marked osd.1 out, while it was still marked up 2024-08-17T17:30:16.026 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:15 smithi063 ceph-mon[109843]: pgmap v1827: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 53 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 153/2604 objects degraded (5.876%) 2024-08-17T17:30:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:15 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1468800005' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T17:30:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:15 smithi063 ceph-mon[109843]: Client client.admin marked osd.1 out, while it was still marked up 2024-08-17T17:30:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:15 smithi063 ceph-mon[114756]: pgmap v1827: 119 pgs: 34 active+undersized, 9 active+undersized+degraded, 76 active+clean; 53 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 153/2604 objects degraded (5.876%) 2024-08-17T17:30:16.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:15 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1468800005' entity='client.admin' cmd={"prefix": "osd out", "ids": ["1"]} : dispatch 2024-08-17T17:30:16.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:15 smithi063 ceph-mon[114756]: Client client.admin marked osd.1 out, while it was still marked up 2024-08-17T17:30:16.050 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:16.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:16.112 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:30:16.112 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:30:16.112 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:30:16.112 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:30:16.112 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:30:16.152 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:16.174 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:16.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:16.276 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:16.299 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:16.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:16.401 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:16.424 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:16.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:16.430 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:30:16.526 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:16.550 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:16.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:16.652 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:16.674 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:16.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:16.721 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:30:16.776 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:16.798 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:16.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:16.900 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:16.922 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:16.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:16.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:16.934 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:17.024 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:16 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1468800005' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-08-17T17:30:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:16 smithi063 ceph-mon[114756]: osdmap e1278: 8 total, 7 up, 7 in 2024-08-17T17:30:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:16 smithi063 ceph-mon[114756]: osdmap e1279: 8 total, 7 up, 7 in 2024-08-17T17:30:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:16 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1468800005' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-08-17T17:30:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:16 smithi063 ceph-mon[109843]: osdmap e1278: 8 total, 7 up, 7 in 2024-08-17T17:30:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:16 smithi063 ceph-mon[109843]: osdmap e1279: 8 total, 7 up, 7 in 2024-08-17T17:30:17.045 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:17.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:17.057 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:16 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1468800005' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2024-08-17T17:30:17.057 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:16 smithi162 ceph-mon[104668]: osdmap e1278: 8 total, 7 up, 7 in 2024-08-17T17:30:17.057 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:17.057 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:16 smithi162 ceph-mon[104668]: osdmap e1279: 8 total, 7 up, 7 in 2024-08-17T17:30:17.147 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:17.170 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:17.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:17.272 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:17.295 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:17.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:17.397 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:17.419 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:17.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:17.521 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:17.544 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:17.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:17.646 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:17.668 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:17.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:17.770 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:17.804 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:17.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:17.905 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:17.928 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:17.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:17.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:17 smithi063 ceph-mon[109843]: pgmap v1829: 119 pgs: 1 unknown, 11 peering, 34 active+undersized, 8 active+undersized+degraded, 65 active+clean; 30 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 725 op/s; 68/1470 objects degraded (4.626%) 2024-08-17T17:30:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:17 smithi063 ceph-mon[109843]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:30:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:17 smithi063 ceph-mon[109843]: osdmap e1280: 8 total, 7 up, 7 in 2024-08-17T17:30:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:17 smithi063 ceph-mon[114756]: pgmap v1829: 119 pgs: 1 unknown, 11 peering, 34 active+undersized, 8 active+undersized+degraded, 65 active+clean; 30 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 725 op/s; 68/1470 objects degraded (4.626%) 2024-08-17T17:30:18.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:17 smithi063 ceph-mon[114756]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:30:18.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:17 smithi063 ceph-mon[114756]: osdmap e1280: 8 total, 7 up, 7 in 2024-08-17T17:30:18.030 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:18.053 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:18.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:17 smithi162 ceph-mon[104668]: pgmap v1829: 119 pgs: 1 unknown, 11 peering, 34 active+undersized, 8 active+undersized+degraded, 65 active+clean; 30 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 725 op/s; 68/1470 objects degraded (4.626%) 2024-08-17T17:30:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:17 smithi162 ceph-mon[104668]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:30:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:17 smithi162 ceph-mon[104668]: osdmap e1280: 8 total, 7 up, 7 in 2024-08-17T17:30:18.155 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:18.177 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:18.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:18.279 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:18.301 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:18.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:18.403 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:18.424 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:18.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:18.526 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:18.551 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:18.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:18.653 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:18.675 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:18.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:18.737 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:30:18.737 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:30:18.737 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:30:18.738 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:30:18.738 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.738 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186368 7.1b 7:d82f1c34:::benchmark_data_smithi063_196002_object74275:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1172)", 2024-08-17T17:30:18.738 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.413998+0000", 2024-08-17T17:30:18.738 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 553.31471600600003, 2024-08-17T17:30:18.738 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0345649890000002, 2024-08-17T17:30:18.738 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.739 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:30:18.739 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.739 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:30:18.739 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:30:18.739 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186368 2024-08-17T17:30:18.739 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.739 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.739 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.739 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.739 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.413998+0000", 2024-08-17T17:30:18.739 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.739 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.740 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.740 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.740 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.413998+0000", 2024-08-17T17:30:18.740 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.740 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.740 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.740 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.740 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.413996+0000", 2024-08-17T17:30:18.740 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999976 2024-08-17T17:30:18.740 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.740 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.740 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.740 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.414003+0000", 2024-08-17T17:30:18.741 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.644e-06 2024-08-17T17:30:18.741 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.741 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.741 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.741 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.414018+0000", 2024-08-17T17:30:18.741 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5046e-05 2024-08-17T17:30:18.741 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.741 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.741 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.741 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.414027+0000", 2024-08-17T17:30:18.742 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4929999999999998e-06 2024-08-17T17:30:18.742 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.742 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.742 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.742 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.414088+0000", 2024-08-17T17:30:18.742 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1426000000000004e-05 2024-08-17T17:30:18.742 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.742 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.742 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.742 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.414237+0000", 2024-08-17T17:30:18.743 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014835599999999999 2024-08-17T17:30:18.743 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.743 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.743 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2,6", 2024-08-17T17:30:18.743 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.414706+0000", 2024-08-17T17:30:18.743 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00046953799999999998 2024-08-17T17:30:18.743 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.743 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.744 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.744 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.417185+0000", 2024-08-17T17:30:18.744 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002478334 2024-08-17T17:30:18.744 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.744 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.744 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.744 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.417499+0000", 2024-08-17T17:30:18.744 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00031453399999999999 2024-08-17T17:30:18.744 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.744 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.744 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.744 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.417901+0000", 2024-08-17T17:30:18.744 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00040172899999999998 2024-08-17T17:30:18.745 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.745 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.745 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.745 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.429809+0000", 2024-08-17T17:30:18.745 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011907778000000001 2024-08-17T17:30:18.745 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.745 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.745 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.745 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.429831+0000", 2024-08-17T17:30:18.746 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2631999999999999e-05 2024-08-17T17:30:18.746 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.746 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.746 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.746 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.432460+0000", 2024-08-17T17:30:18.746 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0026290170000001 2024-08-17T17:30:18.746 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.746 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.746 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.746 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.432468+0000", 2024-08-17T17:30:18.747 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0790000000000001e-06 2024-08-17T17:30:18.747 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.747 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.747 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.747 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.448507+0000", 2024-08-17T17:30:18.747 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0160391150000001 2024-08-17T17:30:18.747 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.747 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.747 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.748 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.448563+0000", 2024-08-17T17:30:18.748 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5458999999999999e-05 2024-08-17T17:30:18.748 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.748 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.748 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.748 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.748 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.748 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186370 7.10 7:0adb90b3:::benchmark_data_smithi063_196002_object74277:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1172)", 2024-08-17T17:30:18.748 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.415940+0000", 2024-08-17T17:30:18.748 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 553.31277416099999, 2024-08-17T17:30:18.748 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2171005479999999, 2024-08-17T17:30:18.748 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.749 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:30:18.749 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.749 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:30:18.749 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:30:18.749 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186370 2024-08-17T17:30:18.749 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.749 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.749 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.749 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.749 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.415940+0000", 2024-08-17T17:30:18.749 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.749 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.750 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.750 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.750 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.415940+0000", 2024-08-17T17:30:18.750 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.750 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.750 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.750 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.750 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.415938+0000", 2024-08-17T17:30:18.750 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:30:18.750 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.750 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.750 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.750 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.415949+0000", 2024-08-17T17:30:18.751 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0644e-05 2024-08-17T17:30:18.751 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.751 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.751 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.751 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.415962+0000", 2024-08-17T17:30:18.751 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3839000000000001e-05 2024-08-17T17:30:18.751 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.751 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.751 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.751 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.415970+0000", 2024-08-17T17:30:18.751 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.289e-06 2024-08-17T17:30:18.751 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.751 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.752 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.752 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416038+0000", 2024-08-17T17:30:18.752 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8509000000000004e-05 2024-08-17T17:30:18.752 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.752 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.752 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.752 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416191+0000", 2024-08-17T17:30:18.752 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015260600000000001 2024-08-17T17:30:18.752 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.752 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.752 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:30:18.752 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.416788+0000", 2024-08-17T17:30:18.752 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00059681700000000003 2024-08-17T17:30:18.752 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.753 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.753 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.753 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.433219+0000", 2024-08-17T17:30:18.753 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.016431016999999999 2024-08-17T17:30:18.753 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.753 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.753 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.753 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.433229+0000", 2024-08-17T17:30:18.753 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0749e-05 2024-08-17T17:30:18.753 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.753 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.753 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.753 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.541033+0000", 2024-08-17T17:30:18.753 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1078034649999999 2024-08-17T17:30:18.754 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.754 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.754 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:30:18.754 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.541350+0000", 2024-08-17T17:30:18.754 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00031702299999999999 2024-08-17T17:30:18.754 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.754 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.754 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.754 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.632954+0000", 2024-08-17T17:30:18.754 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.091603852999999999 2024-08-17T17:30:18.754 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.755 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.755 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.755 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.633040+0000", 2024-08-17T17:30:18.755 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6596999999999998e-05 2024-08-17T17:30:18.755 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.755 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.755 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.755 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.755 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.755 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186382 7.1b 7:d8b6fe7e:::benchmark_data_smithi063_196002_object74289:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1173)", 2024-08-17T17:30:18.755 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.445523+0000", 2024-08-17T17:30:18.756 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 553.28319069999998, 2024-08-17T17:30:18.756 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0130498399999999, 2024-08-17T17:30:18.756 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.756 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.756 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.756 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:30:18.756 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:30:18.756 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186382 2024-08-17T17:30:18.756 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.756 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.756 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.756 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.757 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445523+0000", 2024-08-17T17:30:18.757 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.757 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.757 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.757 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.757 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445523+0000", 2024-08-17T17:30:18.757 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.757 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.757 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.757 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.757 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445522+0000", 2024-08-17T17:30:18.757 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:18.758 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.758 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.758 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.758 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445528+0000", 2024-08-17T17:30:18.758 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2809999999999997e-06 2024-08-17T17:30:18.758 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.758 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.758 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.758 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445540+0000", 2024-08-17T17:30:18.758 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2225000000000001e-05 2024-08-17T17:30:18.758 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.759 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.759 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.759 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445546+0000", 2024-08-17T17:30:18.759 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8359999999999997e-06 2024-08-17T17:30:18.759 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.759 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.759 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.759 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445594+0000", 2024-08-17T17:30:18.759 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7546000000000002e-05 2024-08-17T17:30:18.759 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.759 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.759 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.760 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.445601+0000", 2024-08-17T17:30:18.760 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0509999999999999e-06 2024-08-17T17:30:18.760 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.760 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.760 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.760 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.432490+0000", 2024-08-17T17:30:18.760 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98688914699999997 2024-08-17T17:30:18.760 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.760 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.760 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.760 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.432496+0000", 2024-08-17T17:30:18.760 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0340000000000002e-06 2024-08-17T17:30:18.760 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.761 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.761 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.761 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.448806+0000", 2024-08-17T17:30:18.761 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.016310134 2024-08-17T17:30:18.761 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.761 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.761 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.761 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.448890+0000", 2024-08-17T17:30:18.761 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4312000000000002e-05 2024-08-17T17:30:18.761 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.761 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.761 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:30:18.762 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.449176+0000", 2024-08-17T17:30:18.762 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00028566600000000001 2024-08-17T17:30:18.762 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.762 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.762 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.762 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.454102+0000", 2024-08-17T17:30:18.762 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.004925825 2024-08-17T17:30:18.762 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.762 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.762 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.762 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.454111+0000", 2024-08-17T17:30:18.762 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.5089999999999999e-06 2024-08-17T17:30:18.762 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.763 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.763 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.763 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.458505+0000", 2024-08-17T17:30:18.763 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0043937789999999996 2024-08-17T17:30:18.763 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.763 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.763 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.763 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.458534+0000", 2024-08-17T17:30:18.763 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9164000000000001e-05 2024-08-17T17:30:18.763 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.763 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.763 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.764 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.458573+0000", 2024-08-17T17:30:18.764 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8726000000000002e-05 2024-08-17T17:30:18.764 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.764 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.764 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.764 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.764 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.764 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186383 7.b 7:d027c7e4:::benchmark_data_smithi063_196002_object74290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1173)", 2024-08-17T17:30:18.764 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:05.446820+0000", 2024-08-17T17:30:18.764 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 553.28189429300005, 2024-08-17T17:30:18.765 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.042344328, 2024-08-17T17:30:18.765 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.765 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.765 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.765 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:30:18.765 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:30:18.765 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186383 2024-08-17T17:30:18.765 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.765 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.765 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.766 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.766 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446820+0000", 2024-08-17T17:30:18.766 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.766 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.766 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.766 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.766 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446820+0000", 2024-08-17T17:30:18.766 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.766 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.766 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.766 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.767 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446819+0000", 2024-08-17T17:30:18.767 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:18.767 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.767 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.767 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.767 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446823+0000", 2024-08-17T17:30:18.767 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9720000000000003e-06 2024-08-17T17:30:18.767 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.767 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.767 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.767 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446836+0000", 2024-08-17T17:30:18.767 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3427999999999999e-05 2024-08-17T17:30:18.768 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.768 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.768 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.768 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446842+0000", 2024-08-17T17:30:18.768 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4550000000000003e-06 2024-08-17T17:30:18.768 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.768 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.768 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.768 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446889+0000", 2024-08-17T17:30:18.768 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7132000000000002e-05 2024-08-17T17:30:18.768 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.768 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.768 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.769 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:05.446896+0000", 2024-08-17T17:30:18.769 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3499999999999999e-06 2024-08-17T17:30:18.769 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.769 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.769 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.769 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.432421+0000", 2024-08-17T17:30:18.769 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98552497100000003 2024-08-17T17:30:18.769 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.769 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.769 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.769 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.432432+0000", 2024-08-17T17:30:18.770 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0730999999999999e-05 2024-08-17T17:30:18.770 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.770 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.770 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.770 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.457123+0000", 2024-08-17T17:30:18.770 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.024691437 2024-08-17T17:30:18.770 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.770 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.770 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.770 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.457230+0000", 2024-08-17T17:30:18.771 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000106691 2024-08-17T17:30:18.771 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.771 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.771 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:30:18.771 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.457673+0000", 2024-08-17T17:30:18.771 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000443153 2024-08-17T17:30:18.771 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.771 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.771 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.771 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.463204+0000", 2024-08-17T17:30:18.771 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0055307159999999998 2024-08-17T17:30:18.771 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.771 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.772 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.772 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.465008+0000", 2024-08-17T17:30:18.772 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001804225 2024-08-17T17:30:18.772 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.772 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.772 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.772 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.489076+0000", 2024-08-17T17:30:18.772 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.024068220000000001 2024-08-17T17:30:18.772 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.772 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.772 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.772 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.489124+0000", 2024-08-17T17:30:18.772 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7203999999999997e-05 2024-08-17T17:30:18.773 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.773 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.773 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.773 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.489164+0000", 2024-08-17T17:30:18.773 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0289e-05 2024-08-17T17:30:18.773 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.773 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.773 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.773 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.773 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.773 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186421 7.b 7:d3c8aa29:::benchmark_data_smithi063_196002_object74328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1174)", 2024-08-17T17:30:18.773 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:06.649818+0000", 2024-08-17T17:30:18.773 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 552.07889611200005, 2024-08-17T17:30:18.774 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.83941269600000001, 2024-08-17T17:30:18.774 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.774 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.774 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.774 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:30:18.774 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:30:18.774 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186421 2024-08-17T17:30:18.774 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.774 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.774 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.774 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.775 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649818+0000", 2024-08-17T17:30:18.775 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.775 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.775 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.775 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.775 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649818+0000", 2024-08-17T17:30:18.775 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.775 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.775 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.775 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.775 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649817+0000", 2024-08-17T17:30:18.775 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:30:18.775 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.775 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.776 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.776 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649822+0000", 2024-08-17T17:30:18.776 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0509999999999996e-06 2024-08-17T17:30:18.776 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.776 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.776 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.776 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649834+0000", 2024-08-17T17:30:18.776 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1656e-05 2024-08-17T17:30:18.776 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.776 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.776 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.776 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649840+0000", 2024-08-17T17:30:18.776 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7379999999999997e-06 2024-08-17T17:30:18.776 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.777 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.777 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.777 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649913+0000", 2024-08-17T17:30:18.777 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2510999999999997e-05 2024-08-17T17:30:18.777 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.777 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.777 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.777 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.649922+0000", 2024-08-17T17:30:18.777 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.0920000000000004e-06 2024-08-17T17:30:18.777 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.777 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.777 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.777 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.459587+0000", 2024-08-17T17:30:18.777 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.80966524500000003 2024-08-17T17:30:18.778 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.778 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.778 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.778 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.459695+0000", 2024-08-17T17:30:18.778 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000107626 2024-08-17T17:30:18.778 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.778 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.778 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:30:18.778 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.460016+0000", 2024-08-17T17:30:18.778 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00032096400000000003 2024-08-17T17:30:18.778 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.778 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.779 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.779 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.465014+0000", 2024-08-17T17:30:18.779 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0049983270000000003 2024-08-17T17:30:18.779 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.779 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.779 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.779 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.465040+0000", 2024-08-17T17:30:18.779 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6228e-05 2024-08-17T17:30:18.779 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.779 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.779 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.779 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.489189+0000", 2024-08-17T17:30:18.779 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.024148613999999999 2024-08-17T17:30:18.780 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.780 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.780 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.780 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.489210+0000", 2024-08-17T17:30:18.780 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0996e-05 2024-08-17T17:30:18.780 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.780 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.780 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.781 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.489230+0000", 2024-08-17T17:30:18.781 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0514e-05 2024-08-17T17:30:18.781 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.781 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.781 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.782 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.782 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.782 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186422 7.1b 7:d892c863:::benchmark_data_smithi063_196002_object74329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1174)", 2024-08-17T17:30:18.782 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:06.650101+0000", 2024-08-17T17:30:18.782 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 552.07861258200001, 2024-08-17T17:30:18.782 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.80855219099999998, 2024-08-17T17:30:18.782 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.782 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.782 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.782 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:30:18.782 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:30:18.782 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186422 2024-08-17T17:30:18.783 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.783 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.783 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.783 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.783 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650101+0000", 2024-08-17T17:30:18.783 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.783 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.783 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.783 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.783 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650101+0000", 2024-08-17T17:30:18.783 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.783 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.784 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.784 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.784 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650101+0000", 2024-08-17T17:30:18.784 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:30:18.784 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.784 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.784 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.784 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650105+0000", 2024-08-17T17:30:18.784 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2649999999999998e-06 2024-08-17T17:30:18.784 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.784 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.784 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.784 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650115+0000", 2024-08-17T17:30:18.785 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0387999999999999e-05 2024-08-17T17:30:18.785 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.785 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.785 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.785 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650122+0000", 2024-08-17T17:30:18.785 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7880000000000001e-06 2024-08-17T17:30:18.785 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.785 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.785 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.785 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650163+0000", 2024-08-17T17:30:18.785 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0861000000000001e-05 2024-08-17T17:30:18.785 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.785 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.786 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.786 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.650168+0000", 2024-08-17T17:30:18.786 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3140000000000003e-06 2024-08-17T17:30:18.786 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.786 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.786 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.786 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.450388+0000", 2024-08-17T17:30:18.786 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.80021958400000004 2024-08-17T17:30:18.786 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.786 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.786 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.786 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.450459+0000", 2024-08-17T17:30:18.787 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:18.789 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1019000000000006e-05 2024-08-17T17:30:18.789 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.789 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.789 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:30:18.789 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.450615+0000", 2024-08-17T17:30:18.789 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015638100000000001 2024-08-17T17:30:18.789 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.789 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.789 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.789 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.454118+0000", 2024-08-17T17:30:18.790 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0035029509999999998 2024-08-17T17:30:18.790 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.790 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.790 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.790 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.454146+0000", 2024-08-17T17:30:18.790 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8038000000000002e-05 2024-08-17T17:30:18.790 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.790 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.790 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.790 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.458598+0000", 2024-08-17T17:30:18.790 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.004451389 2024-08-17T17:30:18.790 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.791 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.791 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.791 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.458622+0000", 2024-08-17T17:30:18.791 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4394000000000001e-05 2024-08-17T17:30:18.791 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.791 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.791 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.791 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.458653+0000", 2024-08-17T17:30:18.791 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1575999999999997e-05 2024-08-17T17:30:18.791 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.791 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.791 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.792 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.792 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.792 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:186448 7.1b 7:dcfed570:::benchmark_data_smithi063_196002_object74355:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1174)", 2024-08-17T17:30:18.792 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:06.676011+0000", 2024-08-17T17:30:18.792 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 552.05270250399997, 2024-08-17T17:30:18.792 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.79170734899999995, 2024-08-17T17:30:18.792 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.792 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.792 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.792 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:30:18.792 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:30:18.793 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 186448 2024-08-17T17:30:18.793 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.793 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.793 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.793 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.793 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676011+0000", 2024-08-17T17:30:18.793 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.793 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.793 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.793 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.793 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676011+0000", 2024-08-17T17:30:18.793 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.793 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.794 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.794 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.794 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676009+0000", 2024-08-17T17:30:18.794 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:30:18.794 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.794 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.794 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.794 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676017+0000", 2024-08-17T17:30:18.794 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0490000000000004e-06 2024-08-17T17:30:18.794 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.794 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.794 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.795 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676034+0000", 2024-08-17T17:30:18.795 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6784000000000001e-05 2024-08-17T17:30:18.795 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.795 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.795 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.795 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676042+0000", 2024-08-17T17:30:18.795 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9999999999999996e-06 2024-08-17T17:30:18.795 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.795 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.795 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.795 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676094+0000", 2024-08-17T17:30:18.796 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2318000000000003e-05 2024-08-17T17:30:18.796 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.796 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.796 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.796 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:06.676104+0000", 2024-08-17T17:30:18.796 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3470000000000005e-06 2024-08-17T17:30:18.796 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.796 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.796 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.796 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.451598+0000", 2024-08-17T17:30:18.796 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.77549462999999996 2024-08-17T17:30:18.796 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.796 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.796 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.797 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.451728+0000", 2024-08-17T17:30:18.797 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012943399999999999 2024-08-17T17:30:18.797 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.797 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.797 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:30:18.797 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.451963+0000", 2024-08-17T17:30:18.797 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00023507199999999999 2024-08-17T17:30:18.797 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.797 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.797 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.797 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.454124+0000", 2024-08-17T17:30:18.797 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021607419999999998 2024-08-17T17:30:18.797 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.797 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.798 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.798 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.454192+0000", 2024-08-17T17:30:18.798 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8164000000000007e-05 2024-08-17T17:30:18.798 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.798 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.798 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.798 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.467648+0000", 2024-08-17T17:30:18.798 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.013455801999999999 2024-08-17T17:30:18.798 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.798 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.798 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.798 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.467682+0000", 2024-08-17T17:30:18.798 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3977999999999999e-05 2024-08-17T17:30:18.799 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.799 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.799 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.799 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:07.467719+0000", 2024-08-17T17:30:18.799 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7069e-05 2024-08-17T17:30:18.799 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.799 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.799 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.799 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.799 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.799 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:188170 7.10 7:0e1463ac:::benchmark_data_smithi063_196002_object76077:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:30:18.799 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:09.020045+0000", 2024-08-17T17:30:18.799 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 549.70866884099996, 2024-08-17T17:30:18.800 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.90828101699999997, 2024-08-17T17:30:18.800 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.800 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.800 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.800 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:30:18.800 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:30:18.800 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 188170 2024-08-17T17:30:18.800 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.800 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.800 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.800 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.800 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020045+0000", 2024-08-17T17:30:18.800 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.800 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.801 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.801 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.801 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020045+0000", 2024-08-17T17:30:18.801 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.801 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.801 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.801 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.801 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020044+0000", 2024-08-17T17:30:18.801 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:18.801 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.801 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.801 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.801 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020050+0000", 2024-08-17T17:30:18.802 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3389999999999998e-06 2024-08-17T17:30:18.802 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.802 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.802 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.802 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020065+0000", 2024-08-17T17:30:18.802 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5390999999999999e-05 2024-08-17T17:30:18.802 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.802 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.802 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.802 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020073+0000", 2024-08-17T17:30:18.802 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3649999999999998e-06 2024-08-17T17:30:18.802 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.802 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.803 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.803 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020139+0000", 2024-08-17T17:30:18.803 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5932999999999994e-05 2024-08-17T17:30:18.803 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.803 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.803 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.803 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.020151+0000", 2024-08-17T17:30:18.803 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.226e-05 2024-08-17T17:30:18.803 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.803 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.803 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.803 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.916518+0000", 2024-08-17T17:30:18.803 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.89636652999999999 2024-08-17T17:30:18.804 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.804 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.804 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.804 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.916620+0000", 2024-08-17T17:30:18.804 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010269500000000001 2024-08-17T17:30:18.804 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.804 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.804 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:30:18.804 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.916708+0000", 2024-08-17T17:30:18.804 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7652999999999996e-05 2024-08-17T17:30:18.804 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.804 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.805 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.805 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.922303+0000", 2024-08-17T17:30:18.805 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0055949839999999999 2024-08-17T17:30:18.805 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.805 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.805 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.805 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.923563+0000", 2024-08-17T17:30:18.805 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012598240000000001 2024-08-17T17:30:18.805 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.805 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.805 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.805 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.928276+0000", 2024-08-17T17:30:18.805 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0047131489999999998 2024-08-17T17:30:18.806 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.806 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.806 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.806 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.928304+0000", 2024-08-17T17:30:18.806 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8513e-05 2024-08-17T17:30:18.806 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.806 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.806 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.806 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.928326+0000", 2024-08-17T17:30:18.806 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.1733e-05 2024-08-17T17:30:18.806 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.807 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.807 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.807 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.807 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.807 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:188192 7.10 7:087427ab:::benchmark_data_smithi063_196002_object76099:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:30:18.807 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:09.034613+0000", 2024-08-17T17:30:18.807 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 549.694101047, 2024-08-17T17:30:18.807 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.89933586499999996, 2024-08-17T17:30:18.807 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.807 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.807 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.807 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:30:18.807 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:30:18.808 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 188192 2024-08-17T17:30:18.808 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.808 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.808 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.808 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.808 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034613+0000", 2024-08-17T17:30:18.808 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.808 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.808 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.808 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.808 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034613+0000", 2024-08-17T17:30:18.808 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.808 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.809 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.809 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.809 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034612+0000", 2024-08-17T17:30:18.809 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:30:18.809 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.809 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.809 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.809 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034617+0000", 2024-08-17T17:30:18.809 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4349999999999996e-06 2024-08-17T17:30:18.809 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.809 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.809 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.810 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034628+0000", 2024-08-17T17:30:18.810 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0743e-05 2024-08-17T17:30:18.810 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.810 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.810 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.810 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034633+0000", 2024-08-17T17:30:18.810 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3730000000000001e-06 2024-08-17T17:30:18.810 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.810 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.810 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.810 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034684+0000", 2024-08-17T17:30:18.810 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0871999999999997e-05 2024-08-17T17:30:18.811 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.811 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.811 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.811 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.034697+0000", 2024-08-17T17:30:18.811 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3054000000000001e-05 2024-08-17T17:30:18.811 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.811 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.811 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.811 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.918720+0000", 2024-08-17T17:30:18.811 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.88402291600000005 2024-08-17T17:30:18.811 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.811 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.811 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.812 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.918786+0000", 2024-08-17T17:30:18.812 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6181999999999998e-05 2024-08-17T17:30:18.812 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.812 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.812 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:30:18.812 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.918842+0000", 2024-08-17T17:30:18.812 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5640000000000003e-05 2024-08-17T17:30:18.812 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.812 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.812 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.812 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.923571+0000", 2024-08-17T17:30:18.812 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0047287379999999997 2024-08-17T17:30:18.812 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.813 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.813 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.813 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.933868+0000", 2024-08-17T17:30:18.813 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010297059000000001 2024-08-17T17:30:18.813 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.813 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.813 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.813 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.933899+0000", 2024-08-17T17:30:18.813 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1095e-05 2024-08-17T17:30:18.813 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.813 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.813 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.813 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.933927+0000", 2024-08-17T17:30:18.814 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8529000000000001e-05 2024-08-17T17:30:18.814 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.814 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.814 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.814 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.933949+0000", 2024-08-17T17:30:18.814 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.1302999999999999e-05 2024-08-17T17:30:18.814 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.814 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.814 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.814 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.814 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.814 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:188201 7.10 7:080e429a:::benchmark_data_smithi063_196002_object76108:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:30:18.815 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:09.041737+0000", 2024-08-17T17:30:18.815 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 549.68697661800002, 2024-08-17T17:30:18.815 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.89234012900000004, 2024-08-17T17:30:18.815 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.815 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.815 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.815 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:30:18.815 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:30:18.815 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 188201 2024-08-17T17:30:18.815 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.815 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.815 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.816 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.816 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041737+0000", 2024-08-17T17:30:18.816 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.816 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.816 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.816 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.816 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041737+0000", 2024-08-17T17:30:18.816 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.816 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.816 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.816 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.816 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041735+0000", 2024-08-17T17:30:18.817 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:30:18.817 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.817 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.817 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.817 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041745+0000", 2024-08-17T17:30:18.817 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8300000000000008e-06 2024-08-17T17:30:18.817 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.817 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.817 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.817 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041757+0000", 2024-08-17T17:30:18.817 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1226999999999999e-05 2024-08-17T17:30:18.817 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.817 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.818 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.818 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041762+0000", 2024-08-17T17:30:18.818 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5740000000000003e-06 2024-08-17T17:30:18.818 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.818 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.818 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.818 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041821+0000", 2024-08-17T17:30:18.818 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8814999999999998e-05 2024-08-17T17:30:18.818 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.818 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.818 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.818 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.041830+0000", 2024-08-17T17:30:18.818 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9789999999999999e-06 2024-08-17T17:30:18.819 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.819 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.819 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.819 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.919950+0000", 2024-08-17T17:30:18.819 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.87812030600000002 2024-08-17T17:30:18.819 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.819 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.819 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.819 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.920014+0000", 2024-08-17T17:30:18.819 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3324000000000006e-05 2024-08-17T17:30:18.819 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.819 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.820 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:30:18.820 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.920069+0000", 2024-08-17T17:30:18.820 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5655000000000002e-05 2024-08-17T17:30:18.820 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.820 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.820 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.820 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.923576+0000", 2024-08-17T17:30:18.820 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0035071730000000001 2024-08-17T17:30:18.820 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.820 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.820 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.820 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.934000+0000", 2024-08-17T17:30:18.821 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010423798999999999 2024-08-17T17:30:18.821 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.821 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.821 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.821 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.934037+0000", 2024-08-17T17:30:18.821 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6585000000000003e-05 2024-08-17T17:30:18.821 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.821 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.821 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.821 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.934059+0000", 2024-08-17T17:30:18.821 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2503000000000001e-05 2024-08-17T17:30:18.821 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.821 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.822 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.822 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.934077+0000", 2024-08-17T17:30:18.822 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.8151e-05 2024-08-17T17:30:18.822 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.822 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.822 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.822 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.822 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.822 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:188245 7.10 7:0874df2b:::benchmark_data_smithi063_196002_object76152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:30:18.822 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:09.075632+0000", 2024-08-17T17:30:18.822 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 549.65308172599998, 2024-08-17T17:30:18.822 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.86299068199999995, 2024-08-17T17:30:18.823 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.823 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.823 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.823 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:30:18.823 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:30:18.823 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 188245 2024-08-17T17:30:18.823 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.823 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.823 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.823 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.823 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075632+0000", 2024-08-17T17:30:18.823 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.823 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.824 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.824 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.824 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075632+0000", 2024-08-17T17:30:18.824 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.824 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.824 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.824 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.824 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075631+0000", 2024-08-17T17:30:18.824 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:18.824 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.824 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.824 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.825 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075638+0000", 2024-08-17T17:30:18.825 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.114e-06 2024-08-17T17:30:18.825 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.825 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.825 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.825 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075656+0000", 2024-08-17T17:30:18.825 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7855000000000001e-05 2024-08-17T17:30:18.825 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.825 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.825 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.825 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075664+0000", 2024-08-17T17:30:18.825 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7409999999999998e-06 2024-08-17T17:30:18.826 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.826 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.826 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.826 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075739+0000", 2024-08-17T17:30:18.826 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4977999999999999e-05 2024-08-17T17:30:18.826 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.826 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.826 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.826 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.075756+0000", 2024-08-17T17:30:18.826 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6467999999999999e-05 2024-08-17T17:30:18.826 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.826 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.827 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.827 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.921100+0000", 2024-08-17T17:30:18.827 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.84534372899999999 2024-08-17T17:30:18.827 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.827 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.827 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.827 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.921169+0000", 2024-08-17T17:30:18.827 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9863999999999994e-05 2024-08-17T17:30:18.827 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.827 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.827 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:30:18.827 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.921236+0000", 2024-08-17T17:30:18.828 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6625000000000007e-05 2024-08-17T17:30:18.828 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.828 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.828 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.828 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.925891+0000", 2024-08-17T17:30:18.828 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0046546620000000004 2024-08-17T17:30:18.828 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.828 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.828 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.828 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.934115+0000", 2024-08-17T17:30:18.828 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0082241750000000002 2024-08-17T17:30:18.828 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.828 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.829 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.829 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.938555+0000", 2024-08-17T17:30:18.829 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0044405190000000004 2024-08-17T17:30:18.829 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.829 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.829 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.829 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.938584+0000", 2024-08-17T17:30:18.829 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8129000000000001e-05 2024-08-17T17:30:18.829 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.829 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.829 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.829 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.938623+0000", 2024-08-17T17:30:18.829 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9240999999999999e-05 2024-08-17T17:30:18.830 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.830 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.830 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.830 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.830 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.830 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:188257 7.10 7:0b1312c5:::benchmark_data_smithi063_196002_object76164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:30:18.830 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:09.085121+0000", 2024-08-17T17:30:18.830 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 549.64359253299995, 2024-08-17T17:30:18.830 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.86507842300000004, 2024-08-17T17:30:18.830 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.830 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.831 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.831 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:30:18.831 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:30:18.831 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 188257 2024-08-17T17:30:18.831 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.831 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.831 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.831 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.831 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085121+0000", 2024-08-17T17:30:18.831 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.831 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.831 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.831 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.832 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085121+0000", 2024-08-17T17:30:18.832 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.832 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.832 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.832 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.832 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085120+0000", 2024-08-17T17:30:18.832 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:18.832 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.832 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.832 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.832 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085127+0000", 2024-08-17T17:30:18.832 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8580000000000002e-06 2024-08-17T17:30:18.832 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.833 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.833 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.833 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085145+0000", 2024-08-17T17:30:18.833 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.8308000000000002e-05 2024-08-17T17:30:18.833 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.833 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.833 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.833 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085154+0000", 2024-08-17T17:30:18.833 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8240000000000006e-06 2024-08-17T17:30:18.833 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.833 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.833 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.833 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085234+0000", 2024-08-17T17:30:18.834 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9925999999999993e-05 2024-08-17T17:30:18.834 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.834 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.834 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.834 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.085247+0000", 2024-08-17T17:30:18.834 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3182e-05 2024-08-17T17:30:18.834 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.834 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.834 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.834 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.922382+0000", 2024-08-17T17:30:18.834 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.83713534300000003 2024-08-17T17:30:18.834 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.835 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.835 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.835 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.922448+0000", 2024-08-17T17:30:18.835 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6243999999999998e-05 2024-08-17T17:30:18.835 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.835 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.835 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:30:18.835 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.922510+0000", 2024-08-17T17:30:18.835 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1130000000000006e-05 2024-08-17T17:30:18.835 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.835 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.835 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.835 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.928234+0000", 2024-08-17T17:30:18.836 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0057246600000000003 2024-08-17T17:30:18.836 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.836 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.836 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.836 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.938660+0000", 2024-08-17T17:30:18.836 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010425528 2024-08-17T17:30:18.836 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.836 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.836 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.836 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.950147+0000", 2024-08-17T17:30:18.836 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011487338 2024-08-17T17:30:18.836 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.836 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.837 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.837 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.950177+0000", 2024-08-17T17:30:18.837 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.0428000000000001e-05 2024-08-17T17:30:18.837 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.837 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.837 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.837 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.950200+0000", 2024-08-17T17:30:18.837 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.228e-05 2024-08-17T17:30:18.837 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.837 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.837 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.838 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.838 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.838 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.34965.0:188342 7.10 7:0e94f6c4:::benchmark_data_smithi063_196002_object76249:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1177)", 2024-08-17T17:30:18.838 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:21:09.152746+0000", 2024-08-17T17:30:18.838 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 549.57596739200005, 2024-08-17T17:30:18.838 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.79753565299999996, 2024-08-17T17:30:18.838 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.838 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.838 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.838 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.34965", 2024-08-17T17:30:18.838 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/2653054194", 2024-08-17T17:30:18.839 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 188342 2024-08-17T17:30:18.839 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.839 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.839 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.839 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.839 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152746+0000", 2024-08-17T17:30:18.839 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.839 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.839 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.839 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.839 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152746+0000", 2024-08-17T17:30:18.840 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.840 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.840 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.840 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.840 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152745+0000", 2024-08-17T17:30:18.840 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:30:18.840 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.840 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.840 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.840 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152753+0000", 2024-08-17T17:30:18.840 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0819999999999999e-06 2024-08-17T17:30:18.840 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.840 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.840 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.841 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152769+0000", 2024-08-17T17:30:18.841 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6642000000000001e-05 2024-08-17T17:30:18.841 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.841 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.841 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.841 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152778+0000", 2024-08-17T17:30:18.841 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9849999999999995e-06 2024-08-17T17:30:18.841 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.841 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.841 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.841 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152847+0000", 2024-08-17T17:30:18.841 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8341000000000002e-05 2024-08-17T17:30:18.841 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.842 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.842 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:30:18.842 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.152867+0000", 2024-08-17T17:30:18.842 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9862000000000001e-05 2024-08-17T17:30:18.842 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.842 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.842 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.842 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.923594+0000", 2024-08-17T17:30:18.842 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.77072738900000004 2024-08-17T17:30:18.842 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.842 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.843 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.843 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.923658+0000", 2024-08-17T17:30:18.843 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3749000000000002e-05 2024-08-17T17:30:18.843 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.843 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.843 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:30:18.843 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.923717+0000", 2024-08-17T17:30:18.843 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9061999999999998e-05 2024-08-17T17:30:18.843 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.843 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.843 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.843 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.928243+0000", 2024-08-17T17:30:18.843 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0045256849999999998 2024-08-17T17:30:18.844 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.844 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.844 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.844 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.940172+0000", 2024-08-17T17:30:18.844 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011929194000000001 2024-08-17T17:30:18.844 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.844 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.844 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.844 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.950236+0000", 2024-08-17T17:30:18.844 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010064149999999999 2024-08-17T17:30:18.844 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.844 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.844 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.845 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.950261+0000", 2024-08-17T17:30:18.845 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5369000000000001e-05 2024-08-17T17:30:18.845 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.845 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.845 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.845 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:21:09.950282+0000", 2024-08-17T17:30:18.845 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0922999999999999e-05 2024-08-17T17:30:18.845 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.845 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.845 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.845 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.845 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.845 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:235996 8.3 8:c78032a6:::benchmark_data_smithi063_230025_object109506:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1241)", 2024-08-17T17:30:18.845 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:25:54.297302+0000", 2024-08-17T17:30:18.846 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 264.43141196800002, 2024-08-17T17:30:18.846 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33830727199999999, 2024-08-17T17:30:18.846 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.846 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.846 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.846 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:30:18.846 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:30:18.846 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 235996 2024-08-17T17:30:18.846 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.846 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.846 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.846 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.846 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297302+0000", 2024-08-17T17:30:18.847 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.847 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.847 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.847 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.847 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297302+0000", 2024-08-17T17:30:18.847 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.847 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.847 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.847 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.847 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297301+0000", 2024-08-17T17:30:18.847 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:30:18.847 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.847 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.848 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.848 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297307+0000", 2024-08-17T17:30:18.848 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4049999999999997e-06 2024-08-17T17:30:18.848 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.848 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.848 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.848 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297319+0000", 2024-08-17T17:30:18.848 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1793999999999999e-05 2024-08-17T17:30:18.848 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.848 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.848 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.848 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297327+0000", 2024-08-17T17:30:18.848 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.5179999999999998e-06 2024-08-17T17:30:18.849 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.849 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.849 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.849 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297372+0000", 2024-08-17T17:30:18.849 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5195999999999999e-05 2024-08-17T17:30:18.849 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.849 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.849 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.849 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297499+0000", 2024-08-17T17:30:18.849 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012701300000000001 2024-08-17T17:30:18.849 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.849 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.850 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,6", 2024-08-17T17:30:18.850 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297621+0000", 2024-08-17T17:30:18.850 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012220500000000001 2024-08-17T17:30:18.850 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.850 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.850 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.850 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.304840+0000", 2024-08-17T17:30:18.850 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0072189680000000001 2024-08-17T17:30:18.850 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.850 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.850 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.850 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.309170+0000", 2024-08-17T17:30:18.850 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0043299300000000001 2024-08-17T17:30:18.851 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.851 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.851 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.851 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.634866+0000", 2024-08-17T17:30:18.851 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.32569575299999998 2024-08-17T17:30:18.851 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.851 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.851 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.851 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.634913+0000", 2024-08-17T17:30:18.851 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7450000000000001e-05 2024-08-17T17:30:18.851 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.851 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.851 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.852 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.635609+0000", 2024-08-17T17:30:18.852 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00069607700000000002 2024-08-17T17:30:18.852 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.852 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.852 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.852 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.852 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.852 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:235998 8.4 8:2ca03464:::benchmark_data_smithi063_230025_object109508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1241)", 2024-08-17T17:30:18.852 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:25:54.297556+0000", 2024-08-17T17:30:18.852 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 264.43115760000001, 2024-08-17T17:30:18.852 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.34206344, 2024-08-17T17:30:18.852 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.852 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.853 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.853 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:30:18.853 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:30:18.853 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 235998 2024-08-17T17:30:18.853 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.853 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.853 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.853 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.853 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297556+0000", 2024-08-17T17:30:18.853 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.853 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.853 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.853 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.853 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297556+0000", 2024-08-17T17:30:18.854 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.854 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.854 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.854 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.854 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297555+0000", 2024-08-17T17:30:18.854 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:30:18.854 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.854 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.854 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.854 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297560+0000", 2024-08-17T17:30:18.854 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5199999999999999e-06 2024-08-17T17:30:18.854 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.854 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.855 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.855 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297568+0000", 2024-08-17T17:30:18.855 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3010000000000007e-06 2024-08-17T17:30:18.855 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.855 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.855 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.855 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.297575+0000", 2024-08-17T17:30:18.855 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5300000000000002e-06 2024-08-17T17:30:18.855 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.855 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.855 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.855 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.301305+0000", 2024-08-17T17:30:18.855 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0037297519999999998 2024-08-17T17:30:18.856 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.856 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.856 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.856 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.301423+0000", 2024-08-17T17:30:18.856 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011865400000000001 2024-08-17T17:30:18.856 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.856 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.856 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:30:18.856 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.301540+0000", 2024-08-17T17:30:18.856 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011626299999999999 2024-08-17T17:30:18.856 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.856 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.856 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.857 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305339+0000", 2024-08-17T17:30:18.857 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003799331 2024-08-17T17:30:18.857 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.857 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.857 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.857 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.309072+0000", 2024-08-17T17:30:18.857 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0037334859999999998 2024-08-17T17:30:18.857 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.857 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.857 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.857 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.639385+0000", 2024-08-17T17:30:18.857 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33031314000000001 2024-08-17T17:30:18.857 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.858 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.858 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.858 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.639486+0000", 2024-08-17T17:30:18.858 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010067499999999999 2024-08-17T17:30:18.858 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.858 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.858 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.858 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.639620+0000", 2024-08-17T17:30:18.858 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013358200000000001 2024-08-17T17:30:18.858 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.858 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.858 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.858 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.859 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.859 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:235999 8.d 8:b3cee540:::benchmark_data_smithi063_230025_object109509:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1241)", 2024-08-17T17:30:18.859 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:25:54.299954+0000", 2024-08-17T17:30:18.859 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 264.42876029400003, 2024-08-17T17:30:18.859 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.34301663399999999, 2024-08-17T17:30:18.859 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.859 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.859 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.859 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:30:18.859 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:30:18.859 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 235999 2024-08-17T17:30:18.859 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.859 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.860 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.860 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.860 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.299954+0000", 2024-08-17T17:30:18.860 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.860 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.860 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.860 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.860 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.299954+0000", 2024-08-17T17:30:18.860 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.860 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.860 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.860 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.860 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.299952+0000", 2024-08-17T17:30:18.861 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:18.861 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.861 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.861 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.861 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.299959+0000", 2024-08-17T17:30:18.861 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4509999999999997e-06 2024-08-17T17:30:18.861 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.861 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.861 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.861 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.299971+0000", 2024-08-17T17:30:18.861 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2335e-05 2024-08-17T17:30:18.861 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.861 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.862 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.862 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.299978+0000", 2024-08-17T17:30:18.862 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3860000000000001e-06 2024-08-17T17:30:18.862 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.862 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.862 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.862 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.301412+0000", 2024-08-17T17:30:18.862 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0014331979999999999 2024-08-17T17:30:18.862 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.862 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.862 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.862 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.301514+0000", 2024-08-17T17:30:18.862 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010192300000000001 2024-08-17T17:30:18.863 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.863 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.863 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:30:18.863 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.301621+0000", 2024-08-17T17:30:18.863 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010751600000000001 2024-08-17T17:30:18.863 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.863 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.863 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.863 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305430+0000", 2024-08-17T17:30:18.863 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0038086729999999998 2024-08-17T17:30:18.863 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.863 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.863 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.864 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.642833+0000", 2024-08-17T17:30:18.864 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.337403067 2024-08-17T17:30:18.864 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.864 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.864 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.864 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.642883+0000", 2024-08-17T17:30:18.864 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0062999999999998e-05 2024-08-17T17:30:18.864 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.864 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.864 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.864 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.642931+0000", 2024-08-17T17:30:18.864 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7683999999999999e-05 2024-08-17T17:30:18.865 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.865 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.865 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.865 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.642970+0000", 2024-08-17T17:30:18.865 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9653999999999997e-05 2024-08-17T17:30:18.865 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.865 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.865 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.865 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.865 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.865 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:236007 8.d 8:b706bdcc:::benchmark_data_smithi063_230025_object109517:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1241)", 2024-08-17T17:30:18.865 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:25:54.305395+0000", 2024-08-17T17:30:18.865 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 264.42331917899998, 2024-08-17T17:30:18.866 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33775034999999998, 2024-08-17T17:30:18.866 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.866 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.866 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.866 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:30:18.866 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:30:18.866 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 236007 2024-08-17T17:30:18.866 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.866 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.866 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.866 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.866 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305395+0000", 2024-08-17T17:30:18.867 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.867 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.867 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.867 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.867 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305395+0000", 2024-08-17T17:30:18.867 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.867 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.867 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.867 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.867 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305393+0000", 2024-08-17T17:30:18.867 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:18.867 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.867 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.868 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.868 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305399+0000", 2024-08-17T17:30:18.868 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8699999999999997e-06 2024-08-17T17:30:18.868 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.868 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.868 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.868 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305410+0000", 2024-08-17T17:30:18.868 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0278999999999999e-05 2024-08-17T17:30:18.868 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.868 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.868 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.868 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305417+0000", 2024-08-17T17:30:18.868 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0809999999999996e-06 2024-08-17T17:30:18.869 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.869 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.869 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.869 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305455+0000", 2024-08-17T17:30:18.869 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8733999999999999e-05 2024-08-17T17:30:18.869 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.869 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.869 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.869 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305560+0000", 2024-08-17T17:30:18.869 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010471699999999999 2024-08-17T17:30:18.869 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.869 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.870 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:30:18.870 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305698+0000", 2024-08-17T17:30:18.870 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013761099999999999 2024-08-17T17:30:18.870 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.870 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.870 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.870 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.642983+0000", 2024-08-17T17:30:18.870 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.337285216 2024-08-17T17:30:18.870 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.870 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.870 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.870 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643027+0000", 2024-08-17T17:30:18.871 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4542000000000001e-05 2024-08-17T17:30:18.871 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.871 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.871 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.871 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643073+0000", 2024-08-17T17:30:18.871 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5969e-05 2024-08-17T17:30:18.871 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.871 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.871 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.871 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643112+0000", 2024-08-17T17:30:18.871 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8430999999999998e-05 2024-08-17T17:30:18.871 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.871 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.872 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.872 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643145+0000", 2024-08-17T17:30:18.872 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3185000000000002e-05 2024-08-17T17:30:18.872 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.872 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.872 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.872 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.872 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.872 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45218.0:236009 8.d 8:b1d34fc9:::benchmark_data_smithi063_230025_object109519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1241)", 2024-08-17T17:30:18.872 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:25:54.305466+0000", 2024-08-17T17:30:18.872 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 264.42324827900001, 2024-08-17T17:30:18.872 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33783398199999998, 2024-08-17T17:30:18.872 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.873 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.873 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.873 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45218", 2024-08-17T17:30:18.873 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:30:18.873 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 236009 2024-08-17T17:30:18.873 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.873 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.873 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.873 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.873 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305466+0000", 2024-08-17T17:30:18.873 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.873 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.874 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.874 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.874 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305466+0000", 2024-08-17T17:30:18.874 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.874 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.874 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.874 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.874 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305465+0000", 2024-08-17T17:30:18.874 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:18.874 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.874 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.874 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.875 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305468+0000", 2024-08-17T17:30:18.875 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2090000000000001e-06 2024-08-17T17:30:18.875 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.875 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.875 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.875 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305475+0000", 2024-08-17T17:30:18.875 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6409999999999996e-06 2024-08-17T17:30:18.875 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.875 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.875 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.875 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.305480+0000", 2024-08-17T17:30:18.875 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1760000000000001e-06 2024-08-17T17:30:18.876 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.876 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.876 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.876 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.309513+0000", 2024-08-17T17:30:18.876 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0040332379999999998 2024-08-17T17:30:18.876 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.876 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.876 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.876 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.309627+0000", 2024-08-17T17:30:18.876 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000113669 2024-08-17T17:30:18.876 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.876 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.876 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,7", 2024-08-17T17:30:18.876 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.634933+0000", 2024-08-17T17:30:18.877 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.32530621300000001 2024-08-17T17:30:18.877 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.877 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.877 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.877 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643155+0000", 2024-08-17T17:30:18.877 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0082218229999999996 2024-08-17T17:30:18.877 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.877 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.877 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.877 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643190+0000", 2024-08-17T17:30:18.877 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5298e-05 2024-08-17T17:30:18.877 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.877 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.878 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.878 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643234+0000", 2024-08-17T17:30:18.878 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4044e-05 2024-08-17T17:30:18.878 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.878 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.878 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.878 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643270+0000", 2024-08-17T17:30:18.878 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6094999999999999e-05 2024-08-17T17:30:18.878 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.878 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.878 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.878 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:25:54.643300+0000", 2024-08-17T17:30:18.879 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9274000000000001e-05 2024-08-17T17:30:18.879 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.879 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.879 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.879 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.879 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.879 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:102923 9.6 9:69981af7:::benchmark_data_smithi063_263014_object102922:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-08-17T17:30:18.879 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:56.019188+0000", 2024-08-17T17:30:18.879 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 142.70952638399999, 2024-08-17T17:30:18.879 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33817375300000002, 2024-08-17T17:30:18.879 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.879 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.880 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.880 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:30:18.880 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:30:18.880 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 102923 2024-08-17T17:30:18.880 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.880 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.880 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.880 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.880 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.019188+0000", 2024-08-17T17:30:18.880 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.880 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.880 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.880 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.881 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.019188+0000", 2024-08-17T17:30:18.881 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.881 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.881 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.881 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.881 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.019186+0000", 2024-08-17T17:30:18.881 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:30:18.881 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.881 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.881 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.881 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.019236+0000", 2024-08-17T17:30:18.881 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0643000000000003e-05 2024-08-17T17:30:18.881 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.882 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.882 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.882 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.019253+0000", 2024-08-17T17:30:18.882 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6711e-05 2024-08-17T17:30:18.882 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.882 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.882 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.882 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.019260+0000", 2024-08-17T17:30:18.882 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1999999999999997e-06 2024-08-17T17:30:18.882 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.882 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.882 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.883 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.019310+0000", 2024-08-17T17:30:18.883 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0039e-05 2024-08-17T17:30:18.883 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.883 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.883 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.883 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.019458+0000", 2024-08-17T17:30:18.883 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014769800000000001 2024-08-17T17:30:18.883 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.883 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.883 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:30:18.883 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.019612+0000", 2024-08-17T17:30:18.883 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015339800000000001 2024-08-17T17:30:18.884 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.884 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.884 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.884 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.026690+0000", 2024-08-17T17:30:18.884 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0070783820000000003 2024-08-17T17:30:18.884 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.884 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.884 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.884 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.026707+0000", 2024-08-17T17:30:18.884 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7215999999999999e-05 2024-08-17T17:30:18.884 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.885 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.885 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.885 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.357280+0000", 2024-08-17T17:30:18.885 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33057263599999998 2024-08-17T17:30:18.885 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.885 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.885 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.885 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.357328+0000", 2024-08-17T17:30:18.885 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7762000000000001e-05 2024-08-17T17:30:18.885 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.885 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.886 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.886 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.357361+0000", 2024-08-17T17:30:18.886 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3732000000000002e-05 2024-08-17T17:30:18.886 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.886 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.886 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.886 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.886 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.886 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:248513 9.f 9:fa4cd51e:::benchmark_data_smithi063_263014_object124175:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1277)", 2024-08-17T17:30:18.886 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:30:12.343661+0000", 2024-08-17T17:30:18.886 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 6.3850533360000004, 2024-08-17T17:30:18.886 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.28721632499999999, 2024-08-17T17:30:18.887 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:30:18.887 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:30:18.887 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:30:18.887 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:30:18.887 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:30:18.887 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 248513 2024-08-17T17:30:18.887 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.887 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:30:18.887 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.887 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:30:18.887 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.343661+0000", 2024-08-17T17:30:18.887 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.887 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.888 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.888 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:30:18.888 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.343661+0000", 2024-08-17T17:30:18.888 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:30:18.888 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.888 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.888 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:30:18.888 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.343660+0000", 2024-08-17T17:30:18.888 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:30:18.888 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.888 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.888 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:30:18.888 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.343664+0000", 2024-08-17T17:30:18.889 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3309999999999997e-06 2024-08-17T17:30:18.889 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.889 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.889 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:30:18.889 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.343676+0000", 2024-08-17T17:30:18.889 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1907e-05 2024-08-17T17:30:18.889 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.889 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.889 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:30:18.889 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.343681+0000", 2024-08-17T17:30:18.889 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.327e-06 2024-08-17T17:30:18.889 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.890 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.890 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:30:18.890 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.344409+0000", 2024-08-17T17:30:18.890 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000727489 2024-08-17T17:30:18.890 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.890 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.890 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:30:18.890 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.344519+0000", 2024-08-17T17:30:18.890 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000109787 2024-08-17T17:30:18.890 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.890 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.890 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:30:18.890 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.344635+0000", 2024-08-17T17:30:18.891 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011677599999999999 2024-08-17T17:30:18.891 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.891 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.891 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:30:18.891 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.348097+0000", 2024-08-17T17:30:18.891 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0034618420000000001 2024-08-17T17:30:18.891 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.891 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.891 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.891 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.465675+0000", 2024-08-17T17:30:18.891 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.117577179 2024-08-17T17:30:18.891 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.892 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.892 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:30:18.892 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.630753+0000", 2024-08-17T17:30:18.892 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.165078843 2024-08-17T17:30:18.892 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.892 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.892 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:30:18.892 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.630833+0000", 2024-08-17T17:30:18.892 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9747999999999996e-05 2024-08-17T17:30:18.892 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:30:18.892 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:30:18.892 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:30:18.892 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.630877+0000", 2024-08-17T17:30:18.893 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3773000000000003e-05 2024-08-17T17:30:18.893 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.893 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.893 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.893 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:30:18.893 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:30:18.893 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:30:18.914 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:18.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:18.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:18.984 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:19.015 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[114756]: 9.0 starting backfill to osd.0 from (0'0,0'0] MAX to 1277'15594 2024-08-17T17:30:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[114756]: 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 1277'15492 2024-08-17T17:30:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[114756]: 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 1277'15534 2024-08-17T17:30:19.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[114756]: 9.a starting backfill to osd.3 from (0'0,0'0] MAX to 1277'15572 2024-08-17T17:30:19.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[114756]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 1277'15484 2024-08-17T17:30:19.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[114756]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 1277'15412 2024-08-17T17:30:19.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[114756]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 1277'15920 2024-08-17T17:30:19.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[114756]: 9.f starting backfill to osd.2 from (0'0,0'0] MAX to 1277'15336 2024-08-17T17:30:19.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 68/1470 objects degraded (4.626%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:19.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[114756]: osdmap e1281: 8 total, 7 up, 7 in 2024-08-17T17:30:19.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[109843]: 9.0 starting backfill to osd.0 from (0'0,0'0] MAX to 1277'15594 2024-08-17T17:30:19.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[109843]: 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 1277'15492 2024-08-17T17:30:19.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[109843]: 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 1277'15534 2024-08-17T17:30:19.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[109843]: 9.a starting backfill to osd.3 from (0'0,0'0] MAX to 1277'15572 2024-08-17T17:30:19.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[109843]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 1277'15484 2024-08-17T17:30:19.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[109843]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 1277'15412 2024-08-17T17:30:19.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[109843]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 1277'15920 2024-08-17T17:30:19.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[109843]: 9.f starting backfill to osd.2 from (0'0,0'0] MAX to 1277'15336 2024-08-17T17:30:19.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 68/1470 objects degraded (4.626%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:19.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:18 smithi063 ceph-mon[109843]: osdmap e1281: 8 total, 7 up, 7 in 2024-08-17T17:30:19.052 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:19.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:18 smithi162 ceph-mon[104668]: 9.0 starting backfill to osd.0 from (0'0,0'0] MAX to 1277'15594 2024-08-17T17:30:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:18 smithi162 ceph-mon[104668]: 9.2 starting backfill to osd.7 from (0'0,0'0] MAX to 1277'15492 2024-08-17T17:30:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:18 smithi162 ceph-mon[104668]: 9.4 starting backfill to osd.0 from (0'0,0'0] MAX to 1277'15534 2024-08-17T17:30:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:18 smithi162 ceph-mon[104668]: 9.a starting backfill to osd.3 from (0'0,0'0] MAX to 1277'15572 2024-08-17T17:30:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:18 smithi162 ceph-mon[104668]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 1277'15484 2024-08-17T17:30:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:18 smithi162 ceph-mon[104668]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 1277'15412 2024-08-17T17:30:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:18 smithi162 ceph-mon[104668]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 1277'15920 2024-08-17T17:30:19.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:18 smithi162 ceph-mon[104668]: 9.f starting backfill to osd.2 from (0'0,0'0] MAX to 1277'15336 2024-08-17T17:30:19.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:18 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 68/1470 objects degraded (4.626%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:19.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:18 smithi162 ceph-mon[104668]: osdmap e1281: 8 total, 7 up, 7 in 2024-08-17T17:30:19.154 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:19.176 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:19.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:19.278 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:19.301 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:19.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:19.403 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:19.427 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:19.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:19.528 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:19.550 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:19.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:19.652 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:19.672 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:30:19.675 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:19.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:19.777 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:19.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.801 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:19.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:19.874 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:19.902 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:19.926 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:19.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:19.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:20.028 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:20.040 INFO:teuthology.orchestra.run.smithi063.stderr:pool 'unique_pool_4' removed 2024-08-17T17:30:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.063 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:20.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:20.163 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:20.193 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:20.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:20.272 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:20 smithi162 ceph-mon[104668]: pgmap v1833: 119 pgs: 29 remapped+peering, 1 unknown, 11 peering, 23 active+undersized, 5 active+undersized+degraded, 50 active+clean; 626 KiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 11/78 objects degraded (14.103%); 0 B/s, 0 objects/s recovering 2024-08-17T17:30:20.272 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:20 smithi162 ceph-mon[104668]: osdmap e1282: 8 total, 7 up, 7 in 2024-08-17T17:30:20.272 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:20 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2821159371' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:30:20.272 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:20 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:30:20.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:20 smithi063 ceph-mon[114756]: pgmap v1833: 119 pgs: 29 remapped+peering, 1 unknown, 11 peering, 23 active+undersized, 5 active+undersized+degraded, 50 active+clean; 626 KiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 11/78 objects degraded (14.103%); 0 B/s, 0 objects/s recovering 2024-08-17T17:30:20.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:20 smithi063 ceph-mon[114756]: osdmap e1282: 8 total, 7 up, 7 in 2024-08-17T17:30:20.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:20 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2821159371' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:30:20.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:20 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:30:20.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:20 smithi063 ceph-mon[109843]: pgmap v1833: 119 pgs: 29 remapped+peering, 1 unknown, 11 peering, 23 active+undersized, 5 active+undersized+degraded, 50 active+clean; 626 KiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 11/78 objects degraded (14.103%); 0 B/s, 0 objects/s recovering 2024-08-17T17:30:20.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:20 smithi063 ceph-mon[109843]: osdmap e1282: 8 total, 7 up, 7 in 2024-08-17T17:30:20.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:20 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2821159371' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:30:20.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:20 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:30:20.296 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:20.318 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:20.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:20.420 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:20.441 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:20.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:20.543 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:20.565 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:20.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:20.667 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:20.690 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:20.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:20.792 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:20.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.814 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:20.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:20.915 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:20.939 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:20.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:20.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:21.042 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:21.065 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:21 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:30:21.065 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:21 smithi063 ceph-mon[114756]: osdmap e1283: 8 total, 7 up, 7 in 2024-08-17T17:30:21.065 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:21 smithi063 ceph-mon[114756]: pgmap v1836: 103 pgs: 21 remapped+peering, 1 unknown, 10 peering, 21 active+undersized, 5 active+undersized+degraded, 45 active+clean; 626 KiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 0 B/s, 0 objects/s recovering 2024-08-17T17:30:21.065 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:21 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:21.066 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:21 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:30:21.066 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:21 smithi063 ceph-mon[109843]: osdmap e1283: 8 total, 7 up, 7 in 2024-08-17T17:30:21.066 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:21 smithi063 ceph-mon[109843]: pgmap v1836: 103 pgs: 21 remapped+peering, 1 unknown, 10 peering, 21 active+undersized, 5 active+undersized+degraded, 45 active+clean; 626 KiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 0 B/s, 0 objects/s recovering 2024-08-17T17:30:21.066 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:21 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:21.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.066 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:21.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:21.167 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:21.189 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:21.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:21.290 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:21.315 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:21.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:21.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:21 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:30:21.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:21 smithi162 ceph-mon[104668]: osdmap e1283: 8 total, 7 up, 7 in 2024-08-17T17:30:21.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:21 smithi162 ceph-mon[104668]: pgmap v1836: 103 pgs: 21 remapped+peering, 1 unknown, 10 peering, 21 active+undersized, 5 active+undersized+degraded, 45 active+clean; 626 KiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 0 B/s, 0 objects/s recovering 2024-08-17T17:30:21.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:21 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:30:21.417 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:21.440 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:21.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:21.535 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0] out_osds: [1] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:30:21.536 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:30:21.536 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:30:21.536 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.303058 2024-08-17T17:30:21.536 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 3 0.3030578996637753 2024-08-17T17:30:21.542 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:21.563 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:21.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:21.665 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:21.674 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:30:21.674 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:30:21.690 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:21.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:21.707 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:30:21.707 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:30:21.708 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2024-08-17T17:30:21.708 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_5 16 2024-08-17T17:30:21.792 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:21.813 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:21.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:21.820 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:30:21.820 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:30:21.820 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:30:21.820 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:30:21.821 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:30:21.821 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:30:21.821 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:30:21.821 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:30:21.821 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:30:21.821 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:30:21.822 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:30:21.822 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:30:21.822 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:30:21.822 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:30:21.822 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:30:21.916 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:21.938 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:21.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:21.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:22.040 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:22.063 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:22.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:22.149 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:30:22.165 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:22.188 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:22.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:22.290 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:22.312 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:22.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:22.325 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:22 smithi162 ceph-mon[104668]: 1.3 scrub starts 2024-08-17T17:30:22.326 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:22 smithi162 ceph-mon[104668]: 1.3 scrub ok 2024-08-17T17:30:22.326 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:22 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:22.326 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:22 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:22.326 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:22 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/875966780' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:22.326 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:22 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1214999253' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:22.326 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:22 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:22.326 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:22 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:22.346 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:30:22.414 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:22.436 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:22.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:22.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[114756]: 1.3 scrub starts 2024-08-17T17:30:22.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[114756]: 1.3 scrub ok 2024-08-17T17:30:22.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:22.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:22.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/875966780' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:22.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1214999253' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:22.518 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:22.518 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:22.518 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[109843]: 1.3 scrub starts 2024-08-17T17:30:22.518 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[109843]: 1.3 scrub ok 2024-08-17T17:30:22.518 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:22.518 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:22.518 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/875966780' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:22.518 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1214999253' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:22.518 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:22.518 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:22 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:22.538 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:22.539 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:30:22.539 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:30:22.601 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:22.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:22.610 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:22.703 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:22.725 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:22.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:22.782 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:22.823 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:22.826 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:22.847 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:22.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:22.949 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:22.971 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:22.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:22.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:23.073 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:23.095 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:23.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:23.198 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:23.220 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:23.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:23.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:23 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:30:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:23 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1214999253' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:23 smithi063 ceph-mon[114756]: osdmap e1284: 8 total, 7 up, 7 in 2024-08-17T17:30:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:23 smithi063 ceph-mon[114756]: pgmap v1838: 103 pgs: 20 active+clean+remapped, 21 active+undersized, 5 active+undersized+degraded, 57 active+clean; 608 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%); 0 B/s, 0 objects/s recovering 2024-08-17T17:30:23.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:23 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 11/78 objects degraded (14.103%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:23.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:23 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:30:23.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:23 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1214999253' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:23.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:23 smithi063 ceph-mon[109843]: osdmap e1284: 8 total, 7 up, 7 in 2024-08-17T17:30:23.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:23 smithi063 ceph-mon[109843]: pgmap v1838: 103 pgs: 20 active+clean+remapped, 21 active+undersized, 5 active+undersized+degraded, 57 active+clean; 608 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%); 0 B/s, 0 objects/s recovering 2024-08-17T17:30:23.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:23 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 11/78 objects degraded (14.103%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:23.322 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:23.343 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:23.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:23.397 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:23 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:30:23.398 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:23 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1214999253' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:23.398 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:23 smithi162 ceph-mon[104668]: osdmap e1284: 8 total, 7 up, 7 in 2024-08-17T17:30:23.398 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:23 smithi162 ceph-mon[104668]: pgmap v1838: 103 pgs: 20 active+clean+remapped, 21 active+undersized, 5 active+undersized+degraded, 57 active+clean; 608 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%); 0 B/s, 0 objects/s recovering 2024-08-17T17:30:23.398 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:23 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 11/78 objects degraded (14.103%), 5 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:23.445 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:23.469 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:23.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:23.565 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:30:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:30:23] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.51.0" 2024-08-17T17:30:23.572 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:23.593 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:23.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:23.695 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:23.716 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:23.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:23.818 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:23.841 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:23.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:23.943 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:23.965 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:23.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:23.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:24.067 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:24.090 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:24.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:24.191 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:24.214 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:24.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:24.316 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:24.338 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:24.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:24.429 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:24 smithi162 ceph-mon[104668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:30:24.440 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:24.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.464 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:24.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:24.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:24 smithi063 ceph-mon[114756]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:30:24.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:24 smithi063 ceph-mon[109843]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:30:24.540 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:30:24.565 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:24.587 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:24.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:24.688 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:24.711 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:24.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:24.813 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:24.835 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:24.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:24.937 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:24.961 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:24.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:24.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:24.982 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:30:24.982 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:30:25.062 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:25.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.085 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:25.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:25.186 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:25.199 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:25 smithi063 ceph-mon[114756]: pgmap v1839: 103 pgs: 20 active+clean+remapped, 21 active+undersized, 5 active+undersized+degraded, 57 active+clean; 608 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%); 0 B/s, 0 objects/s recovering 2024-08-17T17:30:25.200 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:25 smithi063 ceph-mon[109843]: pgmap v1839: 103 pgs: 20 active+clean+remapped, 21 active+undersized, 5 active+undersized+degraded, 57 active+clean; 608 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%); 0 B/s, 0 objects/s recovering 2024-08-17T17:30:25.209 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:25.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:25.285 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:25 smithi162 ceph-mon[104668]: pgmap v1839: 103 pgs: 20 active+clean+remapped, 21 active+undersized, 5 active+undersized+degraded, 57 active+clean; 608 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%); 0 B/s, 0 objects/s recovering 2024-08-17T17:30:25.311 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:25.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.333 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:25.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:25.434 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:25.457 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:25.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:25.559 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:25.580 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:25.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:25.596 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:30:25.641 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:30:25.682 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:25.705 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:25.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:25.806 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:25.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.829 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:25.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:25.930 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:25.952 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:25.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:25.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:26.053 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:26.076 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:26.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:26.153 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:30:26.178 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:26.200 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:26.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:26.302 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:26.324 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:26.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:26.426 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:26.447 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:26.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:26.475 INFO:teuthology.orchestra.run.smithi063.stderr:set osd.3 primary-affinity to 0.303058 (8198612) 2024-08-17T17:30:26.549 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:26.572 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:26.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:26.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:26 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2702704995' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.3030578996637753} : dispatch 2024-08-17T17:30:26.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:26 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.3030578996637753} : dispatch 2024-08-17T17:30:26.674 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:26.696 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:26.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:26.749 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:26 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2702704995' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.3030578996637753} : dispatch 2024-08-17T17:30:26.750 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:26 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.3030578996637753} : dispatch 2024-08-17T17:30:26.750 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:26 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2702704995' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.3030578996637753} : dispatch 2024-08-17T17:30:26.750 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:26 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.3030578996637753} : dispatch 2024-08-17T17:30:26.797 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:26.821 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:26.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:26.923 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:26.944 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:26.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:26.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:27.002 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:27.002 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:27.046 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:27.068 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:27.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.069 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:27.170 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:27.192 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:27.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:27.294 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:27.315 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:27.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:27.416 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:27.440 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:27.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:27.541 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:27.564 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:27.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:27.665 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:27.672 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:27 smithi162 ceph-mon[104668]: pgmap v1840: 103 pgs: 20 active+clean+remapped, 21 active+undersized, 5 active+undersized+degraded, 57 active+clean; 608 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%); 0 B/s, 0 objects/s recovering 2024-08-17T17:30:27.672 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:27 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.3030578996637753}]': finished 2024-08-17T17:30:27.672 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:27 smithi162 ceph-mon[104668]: osdmap e1285: 8 total, 7 up, 7 in 2024-08-17T17:30:27.673 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:27 smithi162 ceph-mon[104668]: osdmap e1286: 8 total, 7 up, 7 in 2024-08-17T17:30:27.688 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:27.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:27.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:27 smithi063 ceph-mon[109843]: pgmap v1840: 103 pgs: 20 active+clean+remapped, 21 active+undersized, 5 active+undersized+degraded, 57 active+clean; 608 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%); 0 B/s, 0 objects/s recovering 2024-08-17T17:30:27.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:27 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.3030578996637753}]': finished 2024-08-17T17:30:27.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:27 smithi063 ceph-mon[109843]: osdmap e1285: 8 total, 7 up, 7 in 2024-08-17T17:30:27.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:27 smithi063 ceph-mon[109843]: osdmap e1286: 8 total, 7 up, 7 in 2024-08-17T17:30:27.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:27 smithi063 ceph-mon[114756]: pgmap v1840: 103 pgs: 20 active+clean+remapped, 21 active+undersized, 5 active+undersized+degraded, 57 active+clean; 608 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%); 0 B/s, 0 objects/s recovering 2024-08-17T17:30:27.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:27 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.3030578996637753}]': finished 2024-08-17T17:30:27.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:27 smithi063 ceph-mon[114756]: osdmap e1285: 8 total, 7 up, 7 in 2024-08-17T17:30:27.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:27 smithi063 ceph-mon[114756]: osdmap e1286: 8 total, 7 up, 7 in 2024-08-17T17:30:27.789 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:27.811 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:27.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:27.912 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:27.936 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:27.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:27.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:28.038 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:28.061 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:28.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:28.162 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:28.185 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:28.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:28.218 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:30:28.218 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:30:28.286 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:28.307 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:28.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:28.409 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:28.430 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:28.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:28.532 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:28.553 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:28.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:28.655 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:28.676 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:28.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:28.778 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:28.799 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:28.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:28.868 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:30:28.901 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:28.932 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:28.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:28.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:29.034 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:29.057 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:29.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:29.158 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:29.180 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:29.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:29.282 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:29.304 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:29.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:29.406 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:29.427 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:29.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:29.529 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:29.551 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:29.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:29.642 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:29 smithi162 ceph-mon[104668]: pgmap v1843: 103 pgs: 15 peering, 20 active+clean+remapped, 15 active+undersized, 5 active+undersized+degraded, 48 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 3.2 KiB/s rd, 8.7 KiB/s wr, 2 op/s; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%); 8.3 KiB/s, 0 objects/s recovering 2024-08-17T17:30:29.642 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:29 smithi162 ceph-mon[104668]: osdmap e1287: 8 total, 7 up, 7 in 2024-08-17T17:30:29.643 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:29 smithi162 ceph-mon[104668]: 1.1d scrub starts 2024-08-17T17:30:29.643 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:29 smithi162 ceph-mon[104668]: 1.1d scrub ok 2024-08-17T17:30:29.643 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:29 smithi162 ceph-mon[104668]: Health check failed: Reduced data availability: 6 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2024-08-17T17:30:29.643 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:29 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1874587257' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:30:29.643 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:29 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:30:29.652 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:29.673 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:29.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:29.775 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:29.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:29 smithi063 ceph-mon[114756]: pgmap v1843: 103 pgs: 15 peering, 20 active+clean+remapped, 15 active+undersized, 5 active+undersized+degraded, 48 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 3.2 KiB/s rd, 8.7 KiB/s wr, 2 op/s; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%); 8.3 KiB/s, 0 objects/s recovering 2024-08-17T17:30:29.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:29 smithi063 ceph-mon[114756]: osdmap e1287: 8 total, 7 up, 7 in 2024-08-17T17:30:29.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:29 smithi063 ceph-mon[114756]: 1.1d scrub starts 2024-08-17T17:30:29.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:29 smithi063 ceph-mon[114756]: 1.1d scrub ok 2024-08-17T17:30:29.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:29 smithi063 ceph-mon[114756]: Health check failed: Reduced data availability: 6 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2024-08-17T17:30:29.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:29 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1874587257' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:30:29.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:29 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:30:29.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:29 smithi063 ceph-mon[109843]: pgmap v1843: 103 pgs: 15 peering, 20 active+clean+remapped, 15 active+undersized, 5 active+undersized+degraded, 48 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 3.2 KiB/s rd, 8.7 KiB/s wr, 2 op/s; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%); 8.3 KiB/s, 0 objects/s recovering 2024-08-17T17:30:29.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:29 smithi063 ceph-mon[109843]: osdmap e1287: 8 total, 7 up, 7 in 2024-08-17T17:30:29.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:29 smithi063 ceph-mon[109843]: 1.1d scrub starts 2024-08-17T17:30:29.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:29 smithi063 ceph-mon[109843]: 1.1d scrub ok 2024-08-17T17:30:29.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:29 smithi063 ceph-mon[109843]: Health check failed: Reduced data availability: 6 pgs inactive, 10 pgs peering (PG_AVAILABILITY) 2024-08-17T17:30:29.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:29 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1874587257' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:30:29.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:29 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-08-17T17:30:29.798 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:29.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:29.899 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:29.922 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:29.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:29.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:30.024 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:30.033 INFO:teuthology.orchestra.run.smithi063.stderr:pool 'unique_pool_5' created 2024-08-17T17:30:30.049 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:30.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:30.150 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:30.172 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:30.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:30.274 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:30.295 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:30.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:30.397 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:30.419 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:30.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:30.521 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:30.542 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:30.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:30.644 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:30.665 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:30.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:30.767 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:30.788 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:30.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:30.890 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:30.910 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:30.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:30.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:31.012 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:31.038 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:31.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:31.140 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:31.162 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:31.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:31.265 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:31.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:31 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-08-17T17:30:31.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:31 smithi063 ceph-mon[109843]: osdmap e1288: 8 total, 7 up, 7 in 2024-08-17T17:30:31.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:31 smithi063 ceph-mon[109843]: pgmap v1846: 119 pgs: 16 unknown, 15 peering, 20 active+clean+remapped, 15 active+undersized, 5 active+undersized+degraded, 48 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 5.0 KiB/s rd, 14 KiB/s wr, 3 op/s; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%); 13 KiB/s, 0 objects/s recovering 2024-08-17T17:30:31.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:31 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-08-17T17:30:31.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:31 smithi063 ceph-mon[114756]: osdmap e1288: 8 total, 7 up, 7 in 2024-08-17T17:30:31.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:31 smithi063 ceph-mon[114756]: pgmap v1846: 119 pgs: 16 unknown, 15 peering, 20 active+clean+remapped, 15 active+undersized, 5 active+undersized+degraded, 48 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 5.0 KiB/s rd, 14 KiB/s wr, 3 op/s; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%); 13 KiB/s, 0 objects/s recovering 2024-08-17T17:30:31.288 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:31.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:31.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:31 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-08-17T17:30:31.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:31 smithi162 ceph-mon[104668]: osdmap e1288: 8 total, 7 up, 7 in 2024-08-17T17:30:31.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:31 smithi162 ceph-mon[104668]: pgmap v1846: 119 pgs: 16 unknown, 15 peering, 20 active+clean+remapped, 15 active+undersized, 5 active+undersized+degraded, 48 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 5.0 KiB/s rd, 14 KiB/s wr, 3 op/s; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%); 13 KiB/s, 0 objects/s recovering 2024-08-17T17:30:31.390 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:31.412 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:31.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:31.514 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:31.536 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:31.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:31.638 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:31.659 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:31.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:31.698 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2024-08-17T17:30:31.761 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:31.783 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:31.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:31.885 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:31.905 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:31.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:31.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:31.933 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:30:31.933 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:30:31.951 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:30:31.951 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:30:31.970 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:30:31.970 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:30:31.976 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:31.995 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:30:31.996 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:30:32.007 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:32.014 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:30:32.015 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:30:32.031 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:32.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:32.032 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:30:32.032 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:30:32.033 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:30:32.043 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:30:32.052 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:30:32.053 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:30:32.133 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:32.155 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:32.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:32.226 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[114756]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:30:32.226 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[114756]: osdmap e1289: 8 total, 7 up, 7 in 2024-08-17T17:30:32.226 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3483703780' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:30:32.226 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:32.227 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:32.227 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:32.227 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:32.227 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1259602105' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:32.227 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[109843]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:30:32.227 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[109843]: osdmap e1289: 8 total, 7 up, 7 in 2024-08-17T17:30:32.227 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3483703780' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:30:32.227 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:32.227 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:32.227 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:32.227 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:32.228 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:32 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1259602105' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:32.257 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:32.278 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:32.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:32.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:32 smithi162 ceph-mon[104668]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:30:32.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:32 smithi162 ceph-mon[104668]: osdmap e1289: 8 total, 7 up, 7 in 2024-08-17T17:30:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:32 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3483703780' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:30:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:32 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:32 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:32 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:32 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:32 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1259602105' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:32.380 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:32.403 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:32.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:32.505 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:32.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.528 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:32.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:32.628 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:32.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.651 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:32.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:32.752 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:32.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.776 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:32.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:32.802 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0] out_osds: [1] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:30:32.802 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:30:32.802 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-08-17T17:30:32.802 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd test-reweight-by-utilization 110 0.05 2024-08-17T17:30:32.877 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:32.898 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:32.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:32.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:33.000 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:33.021 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:33.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:33.123 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:33.145 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:33.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:33.247 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:33.269 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:33.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:33.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:33 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:33 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3483703780' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:30:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:33 smithi162 ceph-mon[104668]: osdmap e1290: 8 total, 7 up, 7 in 2024-08-17T17:30:33.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:33 smithi162 ceph-mon[104668]: pgmap v1849: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:33.371 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:33.393 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:33.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:33.495 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:33.517 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:33.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:33.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:30:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:30:33] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.51.0" 2024-08-17T17:30:33.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:33 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:33.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:33 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3483703780' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:30:33.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:33 smithi063 ceph-mon[109843]: osdmap e1290: 8 total, 7 up, 7 in 2024-08-17T17:30:33.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:33 smithi063 ceph-mon[109843]: pgmap v1849: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:33.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:33 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:33.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:33 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3483703780' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:30:33.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:33 smithi063 ceph-mon[114756]: osdmap e1290: 8 total, 7 up, 7 in 2024-08-17T17:30:33.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:33 smithi063 ceph-mon[114756]: pgmap v1849: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:33.619 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:33.641 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:33.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:33.683 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:30:33.683 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:30:33.743 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:33.765 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:33.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:33.867 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:33.888 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:33.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:33.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:33.991 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:34.012 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:34.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:34.113 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:34.115 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:34.143 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:34.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:34.245 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:34.267 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:34.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:34.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:34 smithi162 ceph-mon[104668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 10 pgs peering) 2024-08-17T17:30:34.369 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:34.391 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:34.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:34.493 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:34.514 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:34.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:34.517 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:34 smithi063 ceph-mon[114756]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 10 pgs peering) 2024-08-17T17:30:34.517 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:34 smithi063 ceph-mon[109843]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive, 10 pgs peering) 2024-08-17T17:30:34.589 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:34.616 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:34.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.639 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:34.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:34.740 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:34.762 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:34.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:34.864 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:34.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:34.887 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:34.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:34.988 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:35.012 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:35.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:35.114 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:35.127 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:30:35.137 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:35.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:35.196 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:30:35.196 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:30:35.196 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 0 2024-08-17T17:30:35.196 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:30:35.239 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:35.264 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:35.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:35.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:35 smithi162 ceph-mon[104668]: pgmap v1850: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:35.367 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:35.390 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:35.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:35.492 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:35.495 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:35 smithi063 ceph-mon[109843]: pgmap v1850: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:35.495 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:35 smithi063 ceph-mon[114756]: pgmap v1850: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:35.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.515 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:35.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:35.616 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:35.642 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:35.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:35.684 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:30:35.744 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:35.767 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:35.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:35.869 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:35.884 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:30:35.894 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:35.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:35.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:35.996 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:36.022 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:36.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:36.124 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:36.147 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:36.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:36.249 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:36.272 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:36.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:36.296 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:36.374 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:36.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.397 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:36.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:36.498 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:36.524 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:36.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:36.626 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:36.652 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:36.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:36.753 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:36.776 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:36.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:36.878 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:36.904 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:36.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:36.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:37.006 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:37.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.031 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:37.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:37.133 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:37.158 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:37.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:37.260 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:37.283 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:37.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:37.385 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:37.407 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:37.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:37.544 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:37.569 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:37.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:37.670 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:37.695 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:37.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:37.767 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:37 smithi063 ceph-mon[114756]: pgmap v1851: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:37.767 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:37 smithi063 ceph-mon[109843]: pgmap v1851: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:37.796 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:37.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.821 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:37.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:37.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:37 smithi162 ceph-mon[104668]: pgmap v1851: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:37.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:37.944 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:37.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:37.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:38.046 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:38.068 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:38.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:38.170 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:38.192 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:38.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:38.294 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:38.319 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:38.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:38.421 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:38.444 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:38.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:38.546 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:38.568 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:38.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:38.670 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:38.692 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:38.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:38.794 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:38.820 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:38.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:38.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:38.945 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:38.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:38.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:39.046 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:39.072 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:39.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:39.174 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:39.196 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:39.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:39.298 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:39.324 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:39.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:39.425 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:39.448 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:39.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:39.517 INFO:teuthology.orchestra.run.smithi063.stderr:enabled application 'rados' on pool 'unique_pool_5' 2024-08-17T17:30:39.550 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:39.572 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:39.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:39.674 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:39.700 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:39.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:39.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:39 smithi063 ceph-mon[114756]: pgmap v1852: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:39.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:39 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1146225187' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:30:39.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:39 smithi063 ceph-mon[109843]: pgmap v1852: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:39.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:39 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1146225187' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:30:39.801 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:39.824 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:39.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:39 smithi162 ceph-mon[104668]: pgmap v1852: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:39.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:39 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1146225187' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:30:39.927 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:39.951 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:39.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:39.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:40.053 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:40.079 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:40.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:40.180 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:40.203 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:40.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:40.305 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:40.330 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:40.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:40.432 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:40.457 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:40.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:40.559 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:40.581 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:40.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:40.683 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:40.706 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:40.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:40.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:40 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1146225187' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:30:40.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:40 smithi063 ceph-mon[109843]: osdmap e1291: 8 total, 7 up, 7 in 2024-08-17T17:30:40.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:40 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1146225187' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:30:40.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:40 smithi063 ceph-mon[114756]: osdmap e1291: 8 total, 7 up, 7 in 2024-08-17T17:30:40.808 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:40.830 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:40.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:40.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:40 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1146225187' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:30:40.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:40 smithi162 ceph-mon[104668]: osdmap e1291: 8 total, 7 up, 7 in 2024-08-17T17:30:40.932 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:40.954 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:40.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:40.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:41.056 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:41.078 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:41.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:41.180 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:41.209 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:41.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:41.311 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:41.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.334 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:41.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:41.404 INFO:teuthology.orchestra.run.smithi063.stdout:moved 0 / 357 (0%) 2024-08-17T17:30:41.405 INFO:teuthology.orchestra.run.smithi063.stdout:avg 59.5 2024-08-17T17:30:41.405 INFO:teuthology.orchestra.run.smithi063.stdout:stddev 11.5722 -> 11.5722 (expected baseline 7.04154) 2024-08-17T17:30:41.405 INFO:teuthology.orchestra.run.smithi063.stdout:min osd.2 with 38 -> 38 pgs (0.638655 -> 0.638655 * mean) 2024-08-17T17:30:41.405 INFO:teuthology.orchestra.run.smithi063.stdout:max osd.3 with 58 -> 58 pgs (0.97479 -> 0.97479 * mean) 2024-08-17T17:30:41.405 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:30:41.405 INFO:teuthology.orchestra.run.smithi063.stdout:oload 110 2024-08-17T17:30:41.405 INFO:teuthology.orchestra.run.smithi063.stdout:max_change 0.05 2024-08-17T17:30:41.405 INFO:teuthology.orchestra.run.smithi063.stdout:max_change_osds 4 2024-08-17T17:30:41.405 INFO:teuthology.orchestra.run.smithi063.stdout:average_utilization 0.0070 2024-08-17T17:30:41.405 INFO:teuthology.orchestra.run.smithi063.stdout:overload_utilization 0.0076 2024-08-17T17:30:41.405 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 weight 1.0000 -> 0.9500 2024-08-17T17:30:41.406 INFO:teuthology.orchestra.run.smithi063.stderr:no change 2024-08-17T17:30:41.434 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:41.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.457 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:41.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:41.558 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:41.581 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:41.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:41.683 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:41.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.707 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:41.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:41.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:41 smithi063 ceph-mon[114756]: pgmap v1854: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:41.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:41 smithi063 ceph-mon[114756]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:30:41.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:30:41 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T17:30:41.403+0000 7f4652b73640 -1 mgr.server reply reply (1) Operation not permitted no change 2024-08-17T17:30:41.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:41 smithi063 ceph-mon[109843]: pgmap v1854: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:41.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:41 smithi063 ceph-mon[109843]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:30:41.808 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:41.834 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:41.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:41.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:41 smithi162 ceph-mon[104668]: pgmap v1854: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:41.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:41 smithi162 ceph-mon[104668]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:30:41.936 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:41.961 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:41.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:41.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:42.062 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:42.085 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:42.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:42.187 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:42.212 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:42.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:42.262 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-08-17T17:30:42.262 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd test-reweight-by-utilization 110 0.05 2024-08-17T17:30:42.313 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:42.336 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:42.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:42.437 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:42.464 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:42.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:42.480 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:42.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:42.589 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:42.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:42.690 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:42.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.715 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:42.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:42.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:42 smithi063 ceph-mon[114756]: from='client.45803 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:42.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:42 smithi063 ceph-mon[109843]: from='client.45803 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:42.816 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:42.838 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:42.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:42.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:42 smithi162 ceph-mon[104668]: from='client.45803 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:42.940 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:42.966 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:42.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:42.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:43.068 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:43.092 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:43.194 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:43.219 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:43.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:43.320 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:43.343 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:43.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:43.416 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:43.446 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:43.468 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:43.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:43.492 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:30:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:30:43] "GET /metrics HTTP/1.1" 200 36481 "" "Prometheus/2.51.0" 2024-08-17T17:30:43.547 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:30:43.570 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:43.593 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:43.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:43.695 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:43.717 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:43.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:43.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:43 smithi063 ceph-mon[109843]: pgmap v1855: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:43.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:43 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/927493086' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:43.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:43 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:43.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:43 smithi063 ceph-mon[114756]: pgmap v1855: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:43.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:43 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/927493086' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:43.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:43 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:43.818 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:43.841 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:43.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.842 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:43.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:43 smithi162 ceph-mon[104668]: pgmap v1855: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:43.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:43 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/927493086' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:43.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:43 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:30:43.942 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:43.966 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:43.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:43.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:44.068 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:44.091 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:44.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:44.193 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:44.216 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:44.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:44.317 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:44.341 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:44.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:44.443 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:44.466 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:44.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:44.567 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:44.592 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:44.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:44.694 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:44.717 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:44.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:44.818 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:44.831 DEBUG:teuthology.orchestra.run.smithi063:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2024-08-17T17:30:44.833 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:30:44.833 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:30:44.840 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:44.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:44.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:44 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:44.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:44 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:44.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:44 smithi162 ceph-mon[104668]: osdmap e1292: 8 total, 7 up, 7 in 2024-08-17T17:30:44.941 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:44.964 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:44.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:44.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:44 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:44 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:44 smithi063 ceph-mon[114756]: osdmap e1292: 8 total, 7 up, 7 in 2024-08-17T17:30:45.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:44 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:30:45.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:44 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:30:45.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:44 smithi063 ceph-mon[109843]: osdmap e1292: 8 total, 7 up, 7 in 2024-08-17T17:30:45.034 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:hints = 1 2024-08-17T17:30:45.034 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:30:45.034 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Object prefix: benchmark_data_smithi063_294527 2024-08-17T17:30:45.035 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:30:45.035 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:30:45.065 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:45.105 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:45.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:45.207 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:45.241 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:45.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:45.343 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:45.382 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:45.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:45.484 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:45.516 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:45.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:45.528 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:45.618 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:45.655 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:45.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:45.756 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:45 smithi063 ceph-mon[109843]: pgmap v1857: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:45.757 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:45.757 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:45 smithi063 ceph-mon[114756]: pgmap v1857: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:45.799 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:45.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:45.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:45 smithi162 ceph-mon[104668]: pgmap v1857: 119 pgs: 20 active+clean+remapped, 28 active+undersized, 5 active+undersized+degraded, 66 active+clean; 621 KiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 11/78 objects degraded (14.103%); 2/78 objects misplaced (2.564%) 2024-08-17T17:30:45.901 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:45.937 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:45.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:45.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:46.035 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 1 16 4446 4430 276.859 276.875 0.00253989 0.00360254 2024-08-17T17:30:46.038 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:46.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.072 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:46.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:46.173 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:46.215 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:46.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:46.317 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:46.353 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:46.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:46.455 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:46.491 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:46.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:46.593 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:46.629 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:46.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:46.731 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:46.768 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:46.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:46.870 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:46.902 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:46.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:46.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:47.004 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:47.034 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 2 15 8442 8427 263.324 249.812 0.00469007 0.00379069 2024-08-17T17:30:47.043 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:47.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:47.145 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:47.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.180 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:47.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:47.281 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:47.312 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:47.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:47.354 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:30:47.354 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:30:47.354 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:30:47.354 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:30:47.354 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:30:47.415 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:47.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.448 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:47.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:47.532 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:30:47.548 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:47.571 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:47.571 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:47.571 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:47 smithi063 ceph-mon[114756]: pgmap v1858: 119 pgs: 20 active+clean+remapped, 27 active+undersized, 6 active+undersized+degraded, 66 active+clean; 19 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 2.3 MiB/s wr, 36 op/s; 302/951 objects degraded (31.756%); 2/951 objects misplaced (0.210%) 2024-08-17T17:30:47.571 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:47.572 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:47.572 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:47 smithi063 ceph-mon[109843]: pgmap v1858: 119 pgs: 20 active+clean+remapped, 27 active+undersized, 6 active+undersized+degraded, 66 active+clean; 19 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 2.3 MiB/s wr, 36 op/s; 302/951 objects degraded (31.756%); 2/951 objects misplaced (0.210%) 2024-08-17T17:30:47.609 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:47.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:47.711 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:47.757 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:47.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:30:47.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:47 smithi162 ceph-mon[104668]: pgmap v1858: 119 pgs: 20 active+clean+remapped, 27 active+undersized, 6 active+undersized+degraded, 66 active+clean; 19 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 2.3 MiB/s wr, 36 op/s; 302/951 objects degraded (31.756%); 2/951 objects misplaced (0.210%) 2024-08-17T17:30:47.858 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:47.887 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:47.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:47.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:47.989 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:48.022 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:48.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:48.034 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 3 16 12088 12072 251.483 227.812 0.00476168 0.00396964 2024-08-17T17:30:48.124 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:48.172 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:48.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:48.274 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:48.305 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:48.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:48.399 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:48 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 302/951 objects degraded (31.756%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:48.399 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:48 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 302/951 objects degraded (31.756%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:48.406 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:48.446 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:48.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:48.547 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:48.586 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:48.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:48.688 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:48.727 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:48.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:48.829 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:48.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:48 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 302/951 objects degraded (31.756%), 6 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:48.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:48.862 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:48.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:48.963 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:49.009 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:49.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:49.035 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 4 16 15480 15464 241.608 212 0.0034305 0.00413333 2024-08-17T17:30:49.111 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:49.146 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:49.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:49.247 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:49.286 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:49.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:49.388 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:49.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.425 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:49.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:49.526 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:49.565 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:49.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:49.667 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:49.703 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:49.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:49.805 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:49.828 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:49.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:49.930 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:49.963 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:49.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:49.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:49 smithi063 ceph-mon[114756]: pgmap v1859: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 495 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 55 MiB/s wr, 884 op/s; 3300/23790 objects degraded (13.871%); 2/23790 objects misplaced (0.008%) 2024-08-17T17:30:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:49 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:50.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:49 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:49 smithi063 ceph-mon[109843]: pgmap v1859: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 495 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 55 MiB/s wr, 884 op/s; 3300/23790 objects degraded (13.871%); 2/23790 objects misplaced (0.008%) 2024-08-17T17:30:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:49 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:49 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:50.035 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 5 16 18539 18523 231.52 191.188 0.00812196 0.0043139 2024-08-17T17:30:50.065 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:49 smithi162 ceph-mon[104668]: pgmap v1859: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 495 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 55 MiB/s wr, 884 op/s; 3300/23790 objects degraded (13.871%); 2/23790 objects misplaced (0.008%) 2024-08-17T17:30:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:49 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:49 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:50.096 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:50.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:50.198 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:50.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.225 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:50.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:50.326 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:50.362 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:50.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:50.464 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:50.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.486 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:50.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:50.587 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:50.626 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:50.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:50.728 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:50.753 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:50.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:50.855 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:50.887 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:50.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:50.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:50.989 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:51.011 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:51.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:51.035 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 6 16 21308 21292 221.775 173.062 0.00341487 0.00450014 2024-08-17T17:30:51.113 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:51.154 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:51.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:51.256 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:51.294 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:51.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:51.396 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:51.434 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:51.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:51.536 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:51.572 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:51.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:51.646 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:30:51.674 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:51.703 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:51.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:51.805 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:51.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:51.870 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:51.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:51.971 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:52.011 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:52.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:51 smithi063 ceph-mon[114756]: pgmap v1860: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 495 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 49 MiB/s wr, 790 op/s; 3300/23790 objects degraded (13.871%); 2/23790 objects misplaced (0.008%) 2024-08-17T17:30:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:51 smithi063 ceph-mon[109843]: pgmap v1860: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 495 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 49 MiB/s wr, 790 op/s; 3300/23790 objects degraded (13.871%); 2/23790 objects misplaced (0.008%) 2024-08-17T17:30:52.035 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 7 16 24169 24153 215.635 178.812 0.0039797 0.00463278 2024-08-17T17:30:52.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:51 smithi162 ceph-mon[104668]: pgmap v1860: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 495 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 49 MiB/s wr, 790 op/s; 3300/23790 objects degraded (13.871%); 2/23790 objects misplaced (0.008%) 2024-08-17T17:30:52.113 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:52.159 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:52.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:52.260 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:52.293 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:52.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:52.394 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:52.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.453 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:52.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:52.554 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:52.597 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:52.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:52.699 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:52.729 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:52.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:52.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:52 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:30:52.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:52 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:30:52.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:52 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:52.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:52 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:30:52.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:52 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:30:52.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:52 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:52.816 INFO:teuthology.orchestra.run.smithi063.stdout:moved 3 / 357 (0.840336%) 2024-08-17T17:30:52.816 INFO:teuthology.orchestra.run.smithi063.stdout:avg 59.5 2024-08-17T17:30:52.816 INFO:teuthology.orchestra.run.smithi063.stdout:stddev 11.5722 -> 11.2064 (expected baseline 7.04154) 2024-08-17T17:30:52.816 INFO:teuthology.orchestra.run.smithi063.stdout:min osd.2 with 38 -> 39 pgs (0.638655 -> 0.655462 * mean) 2024-08-17T17:30:52.816 INFO:teuthology.orchestra.run.smithi063.stdout:max osd.3 with 58 -> 59 pgs (0.97479 -> 0.991597 * mean) 2024-08-17T17:30:52.816 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:30:52.816 INFO:teuthology.orchestra.run.smithi063.stdout:oload 110 2024-08-17T17:30:52.816 INFO:teuthology.orchestra.run.smithi063.stdout:max_change 0.05 2024-08-17T17:30:52.817 INFO:teuthology.orchestra.run.smithi063.stdout:max_change_osds 4 2024-08-17T17:30:52.817 INFO:teuthology.orchestra.run.smithi063.stdout:average_utilization 0.0127 2024-08-17T17:30:52.817 INFO:teuthology.orchestra.run.smithi063.stdout:overload_utilization 0.0140 2024-08-17T17:30:52.817 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 weight 1.0000 -> 0.9500 2024-08-17T17:30:52.817 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 weight 1.0000 -> 0.9500 2024-08-17T17:30:52.817 INFO:teuthology.orchestra.run.smithi063.stderr:no change 2024-08-17T17:30:52.831 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:52.876 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:52.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:52.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:52.978 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:53.017 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:53.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:53.035 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 8 16 26754 26738 208.874 161.562 0.00402457 0.0047822 2024-08-17T17:30:53.068 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:30:52 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T17:30:52.812+0000 7f4652b73640 -1 mgr.server reply reply (2) No such file or directory no change 2024-08-17T17:30:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:52 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:30:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:52 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:30:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:52 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:30:53.119 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:53.154 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:53.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:53.185 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:30:53.185 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:30:53.185 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:30:53.185 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:30:53.185 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:30:53.186 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:30:53.186 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:30:53.186 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:30:53.186 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:30:53.186 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:30:53.197 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:53.206 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:30:53.206 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:30:53.206 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:30:53.240 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:30:53.241 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:30:53.257 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:53.292 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:53.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:53.337 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:30:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:30:53] "GET /metrics HTTP/1.1" 200 36475 "" "Prometheus/2.51.0" 2024-08-17T17:30:53.394 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:53.435 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:53.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:53.537 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:53.571 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:53.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:53.673 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:53.705 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:53.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:53.807 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:53.844 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:53.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:53.946 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:53.976 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:53 smithi063 ceph-mon[114756]: pgmap v1861: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.4 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 148 MiB/s wr, 2.37k op/s; 10297/71295 objects degraded (14.443%); 2/71295 objects misplaced (0.003%) 2024-08-17T17:30:53.976 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:53 smithi063 ceph-mon[114756]: from='client.45824 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:53.976 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:53 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 10297/71295 objects degraded (14.443%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:53.976 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:53 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1334624538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:53.976 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:53 smithi063 ceph-mon[109843]: pgmap v1861: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.4 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 148 MiB/s wr, 2.37k op/s; 10297/71295 objects degraded (14.443%); 2/71295 objects misplaced (0.003%) 2024-08-17T17:30:53.976 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:53 smithi063 ceph-mon[109843]: from='client.45824 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:53.977 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:53 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 10297/71295 objects degraded (14.443%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:53.977 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:53 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1334624538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:53.979 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:53.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:53.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:54.019 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-08-17T17:30:54.019 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight-by-utilization 1000 1.0 2024-08-17T17:30:54.035 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 9 16 29495 29479 204.697 171.312 0.00700598 0.00488023 2024-08-17T17:30:54.081 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:53 smithi162 ceph-mon[104668]: pgmap v1861: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.4 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 148 MiB/s wr, 2.37k op/s; 10297/71295 objects degraded (14.443%); 2/71295 objects misplaced (0.003%) 2024-08-17T17:30:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:53 smithi162 ceph-mon[104668]: from='client.45824 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:30:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:53 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 10297/71295 objects degraded (14.443%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:30:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:53 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1334624538' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:30:54.109 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:54.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:54.211 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:54.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.255 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:54.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:54.356 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:54.391 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:54.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:54.493 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:54.530 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:54.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:54.632 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:54.670 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:54.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:54.772 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:54.805 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:54.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:54.906 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:54.933 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:54.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:54.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:55.035 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:55.036 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 10 16 31716 31700 198.107 138.812 0.00733301 0.00504252 2024-08-17T17:30:55.064 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:55.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:55.166 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:55.199 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:55.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:55.301 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:55.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.339 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:55.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:55.440 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:55.470 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:55.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:55.573 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:55.595 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:55.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:55.697 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:55.730 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:55.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:55.832 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:55.868 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:55.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:55.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:55.971 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:56.005 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:56.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:55 smithi063 ceph-mon[114756]: pgmap v1862: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.4 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s; 10297/71295 objects degraded (14.443%); 2/71295 objects misplaced (0.003%) 2024-08-17T17:30:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:55 smithi063 ceph-mon[109843]: pgmap v1862: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.4 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s; 10297/71295 objects degraded (14.443%); 2/71295 objects misplaced (0.003%) 2024-08-17T17:30:56.035 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 11 16 33843 33827 192.182 132.938 0.00636663 0.00519836 2024-08-17T17:30:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:55 smithi162 ceph-mon[104668]: pgmap v1862: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.4 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s; 10297/71295 objects degraded (14.443%); 2/71295 objects misplaced (0.003%) 2024-08-17T17:30:56.107 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:56.144 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:56.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:56.210 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:30:56.211 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:30:56.246 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:56.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.286 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:56.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:56.387 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:56.418 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:56.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:56.520 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:56.553 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:56.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:56.655 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:56.664 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:56.683 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:56.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:56.692 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:30:56.785 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:56.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.823 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:56.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:56.924 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:56.961 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:56.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:56.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:57.036 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 12 16 36181 36165 188.342 146.125 0.00355017 0.00530468 2024-08-17T17:30:57.062 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:57.094 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:57.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:57.196 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:57.247 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:57.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:57.349 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:57.390 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:57.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:57.492 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:57.521 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:57.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:57.623 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:57.658 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:57.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:57.759 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:30:57.800 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:57.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:30:57.854 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:57 smithi063 ceph-mon[114756]: pgmap v1863: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.5 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 128 MiB/s wr, 2.04k op/s; 11110/73734 objects degraded (15.068%); 2/73734 objects misplaced (0.003%) 2024-08-17T17:30:57.854 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:57 smithi063 ceph-mon[109843]: pgmap v1863: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.5 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 128 MiB/s wr, 2.04k op/s; 11110/73734 objects degraded (15.068%); 2/73734 objects misplaced (0.003%) 2024-08-17T17:30:57.902 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:57.940 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:57.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:57.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:58.036 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 13 16 38456 38440 184.791 142.188 0.00663489 0.00540628 2024-08-17T17:30:58.042 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:58.077 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:58.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:57 smithi162 ceph-mon[104668]: pgmap v1863: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 1.5 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 128 MiB/s wr, 2.04k op/s; 11110/73734 objects degraded (15.068%); 2/73734 objects misplaced (0.003%) 2024-08-17T17:30:58.179 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:58.212 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:30:58.214 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:58.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:58.316 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:58.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.353 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:58.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:58.454 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:58.491 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:58.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:58.592 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:58.629 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:58.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:30:58.731 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:58.778 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:58.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:58.880 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:58.915 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:58.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:58.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:59.017 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:59.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:58 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 11110/73734 objects degraded (15.068%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:59.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:58 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 11110/73734 objects degraded (15.068%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:59.036 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 14 16 40763 40747 181.89 144.188 0.00887429 0.00549249 2024-08-17T17:30:59.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.049 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:59.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:58 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 11110/73734 objects degraded (15.068%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:30:59.150 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:59.185 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:59.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:59.287 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:30:59.321 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:59.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:30:59.423 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:30:59.460 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:59.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:30:59.561 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:30:59.597 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:59.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:30:59.699 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:30:59.729 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:59.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:30:59.830 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:30:59.851 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:30:59.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:30:59.953 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:30:59.991 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:30:59.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:30:59.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:00.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:30:59 smithi063 ceph-mon[109843]: pgmap v1864: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 185 MiB/s wr, 2.96k op/s; 15599/107625 objects degraded (14.494%); 2/107625 objects misplaced (0.002%) 2024-08-17T17:31:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:30:59 smithi063 ceph-mon[114756]: pgmap v1864: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 185 MiB/s wr, 2.96k op/s; 15599/107625 objects degraded (14.494%); 2/107625 objects misplaced (0.002%) 2024-08-17T17:31:00.036 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 15 16 42847 42831 178.446 130.25 0.00812061 0.00559949 2024-08-17T17:31:00.092 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:30:59 smithi162 ceph-mon[104668]: pgmap v1864: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 185 MiB/s wr, 2.96k op/s; 15599/107625 objects degraded (14.494%); 2/107625 objects misplaced (0.002%) 2024-08-17T17:31:00.130 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:00.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:00.232 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:00.267 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:00.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:00.369 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:00.409 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:00.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:00.510 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:00.543 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:00.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:00.645 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:00.681 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:00.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:00.783 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:00.805 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:00.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:00.907 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:00.929 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:00.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:00.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:01.030 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:01.036 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 16 16 44521 44505 173.832 104.625 0.00763684 0.0057481 2024-08-17T17:31:01.065 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:01.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:01.167 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:01.207 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:01.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.208 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:01.309 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:01.340 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:01.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:01.442 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:01.479 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:01.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:01.580 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:01.613 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:01.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:01.715 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:01.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.761 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:01.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:01.862 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:01.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:01.889 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:01.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:01.990 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:02.002 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:31:02.002 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:01 smithi063 ceph-mon[109843]: pgmap v1865: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 145 MiB/s wr, 2.33k op/s; 15599/107625 objects degraded (14.494%); 2/107625 objects misplaced (0.002%) 2024-08-17T17:31:02.002 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:02.003 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:01 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3497661492' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:02.003 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:01 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:02.003 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:01 smithi063 ceph-mon[114756]: pgmap v1865: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 145 MiB/s wr, 2.33k op/s; 15599/107625 objects degraded (14.494%); 2/107625 objects misplaced (0.002%) 2024-08-17T17:31:02.003 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:02.003 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:01 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3497661492' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:02.003 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:01 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:02.020 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:02.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:02.036 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 17 16 46476 46460 170.794 122.188 0.00597251 0.00585006 2024-08-17T17:31:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:01 smithi162 ceph-mon[104668]: pgmap v1865: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 145 MiB/s wr, 2.33k op/s; 15599/107625 objects degraded (14.494%); 2/107625 objects misplaced (0.002%) 2024-08-17T17:31:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:01 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3497661492' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:01 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:02.122 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:02.155 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:02.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:02.257 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:02.283 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:02.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:02.385 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:02.421 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:02.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:02.523 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:02.559 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:02.661 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:02.683 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:02.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:02.784 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:02.817 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:02.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:02.919 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:02.955 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:02.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:02.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:02 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:02 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:31:03.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:02 smithi063 ceph-mon[114756]: osdmap e1293: 8 total, 7 up, 7 in 2024-08-17T17:31:03.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:02 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:03.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:02 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:31:03.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:02 smithi063 ceph-mon[109843]: osdmap e1293: 8 total, 7 up, 7 in 2024-08-17T17:31:03.036 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 18 16 48542 48526 168.478 129.125 0.00730872 0.00593053 2024-08-17T17:31:03.057 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:03.078 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:03.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:02 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:02 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:31:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:02 smithi162 ceph-mon[104668]: osdmap e1293: 8 total, 7 up, 7 in 2024-08-17T17:31:03.180 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:03.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.216 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:03.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:03.317 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:03.357 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:03.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:03.459 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:03.493 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:03.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:03.595 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:03.632 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:03.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:03.734 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:03.743 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:31:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:31:03] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.51.0" 2024-08-17T17:31:03.769 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:03.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:03.871 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:03.912 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:03.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:03.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:03.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:03 smithi162 ceph-mon[104668]: pgmap v1867: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 140 MiB/s wr, 2.25k op/s; 20147/138741 objects degraded (14.521%); 2/138741 objects misplaced (0.001%) 2024-08-17T17:31:03.996 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:03 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 20147/138741 objects degraded (14.521%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:04.015 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:04.016 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:03 smithi063 ceph-mon[114756]: pgmap v1867: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 140 MiB/s wr, 2.25k op/s; 20147/138741 objects degraded (14.521%); 2/138741 objects misplaced (0.001%) 2024-08-17T17:31:04.016 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:03 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 20147/138741 objects degraded (14.521%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:04.017 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:03 smithi063 ceph-mon[109843]: pgmap v1867: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 140 MiB/s wr, 2.25k op/s; 20147/138741 objects degraded (14.521%); 2/138741 objects misplaced (0.001%) 2024-08-17T17:31:04.017 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:03 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 20147/138741 objects degraded (14.521%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:04.036 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 19 16 50510 50494 166.08 123 0.00876324 0.0060165 2024-08-17T17:31:04.055 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:04.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:04.158 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:04.190 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:04.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:04.292 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:04.324 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:04.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:04.339 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:31:04.339 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:31:04.426 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:04.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.464 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:04.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:04.565 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:04.602 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:04.705 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:04.744 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:04.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:04.846 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:04.882 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:04.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:04.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:04.984 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:05.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.018 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:05.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:05.037 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:31:05.037508+0000 min lat: 0.00111601 max lat: 0.162415 avg lat: 0.00609646 2024-08-17T17:31:05.037 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:31:05.038 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 20 16 52475 52459 163.912 122.812 0.00674937 0.00609646 2024-08-17T17:31:05.119 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:05.154 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:05.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:05.256 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:05.287 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:05.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:05.388 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:05.428 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:05.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:05.530 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:05.563 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:05.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:05.664 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:05.703 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:05.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:05.805 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:05.837 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:05.837 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:31:05.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:05.876 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:05 smithi063 ceph-mon[109843]: pgmap v1868: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 140 MiB/s wr, 2.25k op/s; 20147/138741 objects degraded (14.521%); 2/138741 objects misplaced (0.001%) 2024-08-17T17:31:05.876 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:05 smithi063 ceph-mon[114756]: pgmap v1868: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 140 MiB/s wr, 2.25k op/s; 20147/138741 objects degraded (14.521%); 2/138741 objects misplaced (0.001%) 2024-08-17T17:31:05.939 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:05.981 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:05 smithi162 ceph-mon[104668]: pgmap v1868: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 140 MiB/s wr, 2.25k op/s; 20147/138741 objects degraded (14.521%); 2/138741 objects misplaced (0.001%) 2024-08-17T17:31:05.990 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:05.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:05.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:06.026 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:31:06.037 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 21 16 54307 54291 161.558 114.5 0.00355835 0.00618398 2024-08-17T17:31:06.092 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:06.150 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:06.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:06.251 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:06.318 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:06.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:06.340 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:06.353 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:06.420 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:06.450 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:06.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:06.552 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:06.591 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:06.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:06.640 INFO:teuthology.orchestra.run.smithi063.stdout:moved 0 / 357 (0%) 2024-08-17T17:31:06.641 INFO:teuthology.orchestra.run.smithi063.stdout:avg 59.5 2024-08-17T17:31:06.641 INFO:teuthology.orchestra.run.smithi063.stdout:stddev 11.5722 -> 11.5722 (expected baseline 7.04154) 2024-08-17T17:31:06.641 INFO:teuthology.orchestra.run.smithi063.stdout:min osd.2 with 38 -> 38 pgs (0.638655 -> 0.638655 * mean) 2024-08-17T17:31:06.641 INFO:teuthology.orchestra.run.smithi063.stdout:max osd.3 with 58 -> 58 pgs (0.97479 -> 0.97479 * mean) 2024-08-17T17:31:06.641 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:31:06.641 INFO:teuthology.orchestra.run.smithi063.stdout:oload 1000 2024-08-17T17:31:06.641 INFO:teuthology.orchestra.run.smithi063.stdout:max_change 1 2024-08-17T17:31:06.641 INFO:teuthology.orchestra.run.smithi063.stdout:max_change_osds 4 2024-08-17T17:31:06.641 INFO:teuthology.orchestra.run.smithi063.stdout:average_utilization 0.0190 2024-08-17T17:31:06.641 INFO:teuthology.orchestra.run.smithi063.stdout:overload_utilization 0.1898 2024-08-17T17:31:06.642 INFO:teuthology.orchestra.run.smithi063.stderr:no change 2024-08-17T17:31:06.693 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:06.725 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:06.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:06.827 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:06.859 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:06.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:06.961 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:06.998 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:06.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:06.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:07.037 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 22 16 55846 55830 158.586 96.1875 0.00458416 0.00630052 2024-08-17T17:31:07.100 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:07.140 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:07.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:07.242 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:07.278 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:07.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:07.380 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:07.411 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:07.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:07.513 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:07.552 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:07.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:07.654 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:07.688 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:07.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:07.790 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:07.827 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:07.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:07.929 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:07.973 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:07.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:07.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:08.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:07 smithi063 ceph-mon[114756]: pgmap v1869: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 139 MiB/s wr, 2.23k op/s; 20760/140580 objects degraded (14.767%); 2/140580 objects misplaced (0.001%) 2024-08-17T17:31:08.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:07 smithi063 ceph-mon[114756]: from='client.55441 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:08.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:07 smithi063 ceph-mon[109843]: pgmap v1869: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 139 MiB/s wr, 2.23k op/s; 20760/140580 objects degraded (14.767%); 2/140580 objects misplaced (0.001%) 2024-08-17T17:31:08.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:07 smithi063 ceph-mon[109843]: from='client.55441 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:08.038 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 23 16 57712 57696 156.759 116.625 0.010078 0.00637427 2024-08-17T17:31:08.074 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:07 smithi162 ceph-mon[104668]: pgmap v1869: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 139 MiB/s wr, 2.23k op/s; 20760/140580 objects degraded (14.767%); 2/140580 objects misplaced (0.001%) 2024-08-17T17:31:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:07 smithi162 ceph-mon[104668]: from='client.55441 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:08.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.112 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:08.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:08.213 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:08.248 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:08.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:08.350 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:08.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.391 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:08.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:08.492 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:08.526 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:08.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:08.627 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:08.660 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:08.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.661 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:08.762 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:08.801 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:08.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:08.903 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:08.936 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:08.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:08.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:08.990 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:08 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 20760/140580 objects degraded (14.767%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:08.990 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:08 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 20760/140580 objects degraded (14.767%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:09.032 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-08-17T17:31:09.032 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight-by-utilization 110 0.05 2024-08-17T17:31:09.037 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:09.038 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 24 16 59578 59562 155.086 116.625 0.0201154 0.00644325 2024-08-17T17:31:09.041 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:09.083 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:09.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:09.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:08 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 20760/140580 objects degraded (14.767%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:09.185 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:09.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.224 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:09.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:09.325 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:09.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.358 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:09.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:09.458 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:09.503 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:09.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:09.605 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:09.648 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:09.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:09.750 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:09.778 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:09.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:09.880 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:09.907 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:09.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:09.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:10.009 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:09 smithi063 ceph-mon[109843]: pgmap v1870: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s; 24223/166962 objects degraded (14.508%); 2/166962 objects misplaced (0.001%) 2024-08-17T17:31:10.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:09 smithi063 ceph-mon[114756]: pgmap v1870: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s; 24223/166962 objects degraded (14.508%); 2/166962 objects misplaced (0.001%) 2024-08-17T17:31:10.038 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 25 16 61208 61192 152.957 101.875 0.0104418 0.00653294 2024-08-17T17:31:10.047 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:10.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:09 smithi162 ceph-mon[104668]: pgmap v1870: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s; 24223/166962 objects degraded (14.508%); 2/166962 objects misplaced (0.001%) 2024-08-17T17:31:10.149 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:10.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.181 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:10.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:10.282 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:10.323 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:10.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:10.425 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:10.461 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:10.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:10.562 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:10.596 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:10.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:10.698 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:10.731 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:10.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:10.773 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:10 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/355040140' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:10.780 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:31:10.834 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:10.869 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:10.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:10.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:10.971 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:11.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.012 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:11.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:10 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/355040140' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:11.038 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 26 16 63014 62998 151.415 112.875 0.00707116 0.00659973 2024-08-17T17:31:11.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:10 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/355040140' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:11.113 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:11.152 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:11.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:11.254 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:11.293 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:11.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:11.395 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:11.434 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:11.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:11.536 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:11.567 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:11.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:11.669 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:11.703 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:11.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:11.805 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:11.825 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:11 smithi063 ceph-mon[109843]: pgmap v1871: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s; 24223/166962 objects degraded (14.508%); 2/166962 objects misplaced (0.001%) 2024-08-17T17:31:11.825 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:11 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:31:11.825 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:11 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/355040140' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:11.825 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:11 smithi063 ceph-mon[109843]: osdmap e1294: 8 total, 7 up, 7 in 2024-08-17T17:31:11.826 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:11 smithi063 ceph-mon[114756]: pgmap v1871: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s; 24223/166962 objects degraded (14.508%); 2/166962 objects misplaced (0.001%) 2024-08-17T17:31:11.826 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:11 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:31:11.826 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:11 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/355040140' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:11.826 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:11 smithi063 ceph-mon[114756]: osdmap e1294: 8 total, 7 up, 7 in 2024-08-17T17:31:11.840 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:11.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:11.942 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:11.975 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:11.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:11.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:12.038 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 27 16 64772 64756 149.876 109.875 0.00891381 0.00666701 2024-08-17T17:31:12.077 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:11 smithi162 ceph-mon[104668]: pgmap v1871: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 124 MiB/s wr, 1.98k op/s; 24223/166962 objects degraded (14.508%); 2/166962 objects misplaced (0.001%) 2024-08-17T17:31:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:11 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:31:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:11 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/355040140' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:11 smithi162 ceph-mon[104668]: osdmap e1294: 8 total, 7 up, 7 in 2024-08-17T17:31:12.112 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:12.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:12.214 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:12.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.264 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:12.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:12.365 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:12.375 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:12.415 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:12.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:12.441 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:31:12.441 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:31:12.475 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:31:12.475 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:31:12.505 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:31:12.505 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:31:12.517 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:12.540 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:31:12.540 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:31:12.555 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:12.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:12.573 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:31:12.573 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:31:12.610 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:31:12.610 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:31:12.612 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:31:12.650 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:31:12.651 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:31:12.657 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:12.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.707 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:12.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:12.808 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:12.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.852 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:12.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:12.922 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:12 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3997449120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:12.922 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:12 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3997449120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:12.954 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:12.997 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:12.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:12.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:13.038 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 28 16 66347 66331 148.039 98.4375 0.0118457 0.00675066 2024-08-17T17:31:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:12 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3997449120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:13.099 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:13.137 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:13.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:13.239 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:13.269 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:13.371 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:13.407 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:13.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:13.509 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:13.539 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:13.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:13.586 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:31:13.621 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:31:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:31:13] "GET /metrics HTTP/1.1" 200 36538 "" "Prometheus/2.51.0" 2024-08-17T17:31:13.640 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:13.672 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:13.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:13.774 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:13.802 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:31:13.803 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:31:13.805 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:13.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:13.875 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:13 smithi063 ceph-mon[109843]: pgmap v1873: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 114 MiB/s wr, 1.83k op/s; 28127/193698 objects degraded (14.521%); 2/193698 objects misplaced (0.001%) 2024-08-17T17:31:13.875 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:13 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 28127/193698 objects degraded (14.521%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:13.875 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:13 smithi063 ceph-mon[114756]: pgmap v1873: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 114 MiB/s wr, 1.83k op/s; 28127/193698 objects degraded (14.521%); 2/193698 objects misplaced (0.001%) 2024-08-17T17:31:13.875 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:13 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 28127/193698 objects degraded (14.521%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:13.907 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:13.933 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:13.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:13.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:14.035 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:14.039 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 29 16 67980 67964 146.45 102.062 0.00905819 0.00682323 2024-08-17T17:31:14.049 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:31:14.050 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:31:14.050 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 0 2024-08-17T17:31:14.050 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:31:14.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.088 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:14.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:13 smithi162 ceph-mon[104668]: pgmap v1873: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 114 MiB/s wr, 1.83k op/s; 28127/193698 objects degraded (14.521%); 2/193698 objects misplaced (0.001%) 2024-08-17T17:31:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:13 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 28127/193698 objects degraded (14.521%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:14.189 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:14.224 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:14.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:14.326 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:14.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.368 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:14.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:14.468 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:14.505 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:14.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:14.606 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:14.644 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:14.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:14.746 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:14.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.792 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:14.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:14.893 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:14.930 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:14.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:14.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:15.032 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:15.039 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 30 16 69653 69637 145.053 104.562 0.00898885 0.00688849 2024-08-17T17:31:15.061 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:15.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:15.163 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:15.200 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:15.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:15.304 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:15.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.351 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:15.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:15.452 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:15.499 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:15.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:15.601 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:15.640 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:15.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:15.742 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:15.776 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:15.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:15.803 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:31:15.854 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:15 smithi063 ceph-mon[114756]: pgmap v1874: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 114 MiB/s wr, 1.83k op/s; 28127/193698 objects degraded (14.521%); 2/193698 objects misplaced (0.001%) 2024-08-17T17:31:15.854 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:15 smithi063 ceph-mon[109843]: pgmap v1874: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 114 MiB/s wr, 1.83k op/s; 28127/193698 objects degraded (14.521%); 2/193698 objects misplaced (0.001%) 2024-08-17T17:31:15.877 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:15.922 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:15.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:15.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:16.018 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:31:16.024 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:16.039 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 31 16 71250 71234 143.594 99.8125 0.019762 0.00695961 2024-08-17T17:31:16.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.094 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:16.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:15 smithi162 ceph-mon[104668]: pgmap v1874: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 114 MiB/s wr, 1.83k op/s; 28127/193698 objects degraded (14.521%); 2/193698 objects misplaced (0.001%) 2024-08-17T17:31:16.180 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:16.196 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:16.223 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:16.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:16.324 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:16.368 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:16.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:16.470 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:16.498 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:16.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:16.600 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:16.631 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:16.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:16.733 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:16.772 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:16.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:16.873 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:16.901 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:16.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:16.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:17.003 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:17.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.041 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:17.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:17.042 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 32 16 72693 72677 141.916 90.1875 0.00923953 0.00704162 2024-08-17T17:31:17.143 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:17.181 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:17.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:17.284 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:17.320 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:17.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:17.422 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:17.456 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:17.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:17.558 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:17.593 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:17.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:17.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:17 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:31:17.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:17 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:17.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:17 smithi162 ceph-mon[104668]: pgmap v1875: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s; 28659/195294 objects degraded (14.675%); 2/195294 objects misplaced (0.001%) 2024-08-17T17:31:17.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:17 smithi162 ceph-mon[104668]: from='client.35925 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:17.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:17 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','4':'62260'}"} : dispatch 2024-08-17T17:31:17.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:17 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','4':'62260'}"} : dispatch 2024-08-17T17:31:17.628 INFO:teuthology.orchestra.run.smithi063.stdout:moved 3 / 357 (0.840336%) 2024-08-17T17:31:17.628 INFO:teuthology.orchestra.run.smithi063.stdout:avg 59.5 2024-08-17T17:31:17.628 INFO:teuthology.orchestra.run.smithi063.stdout:stddev 11.5722 -> 11.2064 (expected baseline 7.04154) 2024-08-17T17:31:17.628 INFO:teuthology.orchestra.run.smithi063.stdout:min osd.2 with 38 -> 39 pgs (0.638655 -> 0.655462 * mean) 2024-08-17T17:31:17.628 INFO:teuthology.orchestra.run.smithi063.stdout:max osd.3 with 58 -> 59 pgs (0.97479 -> 0.991597 * mean) 2024-08-17T17:31:17.628 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:31:17.628 INFO:teuthology.orchestra.run.smithi063.stdout:oload 110 2024-08-17T17:31:17.628 INFO:teuthology.orchestra.run.smithi063.stdout:max_change 0.05 2024-08-17T17:31:17.629 INFO:teuthology.orchestra.run.smithi063.stdout:max_change_osds 4 2024-08-17T17:31:17.629 INFO:teuthology.orchestra.run.smithi063.stdout:average_utilization 0.0236 2024-08-17T17:31:17.629 INFO:teuthology.orchestra.run.smithi063.stdout:overload_utilization 0.0259 2024-08-17T17:31:17.629 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3 weight 1.0000 -> 0.9500 2024-08-17T17:31:17.629 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 weight 1.0000 -> 0.9500 2024-08-17T17:31:17.629 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 weight 1.0000 -> 0.9500 2024-08-17T17:31:17.695 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:17.728 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:17.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:17.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:17 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:31:17.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:17 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:17.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:17 smithi063 ceph-mon[109843]: pgmap v1875: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s; 28659/195294 objects degraded (14.675%); 2/195294 objects misplaced (0.001%) 2024-08-17T17:31:17.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:17 smithi063 ceph-mon[109843]: from='client.35925 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:17.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:17 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','4':'62260'}"} : dispatch 2024-08-17T17:31:17.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:17 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','4':'62260'}"} : dispatch 2024-08-17T17:31:17.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:17 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:31:17.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:17 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:17.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:17 smithi063 ceph-mon[114756]: pgmap v1875: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 114 MiB/s wr, 1.82k op/s; 28659/195294 objects degraded (14.675%); 2/195294 objects misplaced (0.001%) 2024-08-17T17:31:17.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:17 smithi063 ceph-mon[114756]: from='client.35925 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:17.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:17 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','4':'62260'}"} : dispatch 2024-08-17T17:31:17.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:17 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','4':'62260'}"} : dispatch 2024-08-17T17:31:17.830 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:17.865 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:17.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:17.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:17.967 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:18.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.007 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:18.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:18.042 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 33 16 74217 74201 140.502 95.25 0.00519501 0.00711214 2024-08-17T17:31:18.108 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:18.145 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:18.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:18.246 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:18.280 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:18.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:18.382 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:18.425 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:18.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:18.519 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-08-17T17:31:18.519 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight-by-utilization 110 0.05 2024-08-17T17:31:18.528 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:18.557 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:18.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:18.659 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:18.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.700 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:18.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:18 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','4':'62260'}"}]': finished 2024-08-17T17:31:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:18 smithi063 ceph-mon[114756]: osdmap e1295: 8 total, 7 up, 7 in 2024-08-17T17:31:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:18 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 28659/195294 objects degraded (14.675%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:18 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','4':'62260'}"}]': finished 2024-08-17T17:31:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:18 smithi063 ceph-mon[109843]: osdmap e1295: 8 total, 7 up, 7 in 2024-08-17T17:31:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:18 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 28659/195294 objects degraded (14.675%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:18.801 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:18.831 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:18.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:18.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:18 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','4':'62260'}"}]': finished 2024-08-17T17:31:18.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:18 smithi162 ceph-mon[104668]: osdmap e1295: 8 total, 7 up, 7 in 2024-08-17T17:31:18.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:18 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 28659/195294 objects degraded (14.675%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:18.933 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:18.972 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:18.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:18.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:19.042 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 34 16 75762 75746 139.207 96.5625 0.00585927 0.00717901 2024-08-17T17:31:19.074 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:19.106 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:19.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:19.207 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:19.243 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:19.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:19.344 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:19.379 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:19.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:19.467 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:19.473 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:19.481 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:19.506 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:19.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:19.608 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:19.645 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:19 smithi063 ceph-mon[114756]: osdmap e1296: 8 total, 7 up, 7 in 2024-08-17T17:31:19.646 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:19 smithi063 ceph-mon[114756]: pgmap v1878: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 137 MiB/s wr, 2.20k op/s; 31693/217677 objects degraded (14.560%); 2/217677 objects misplaced (0.001%) 2024-08-17T17:31:19.646 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:19 smithi063 ceph-mon[109843]: osdmap e1296: 8 total, 7 up, 7 in 2024-08-17T17:31:19.646 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:19 smithi063 ceph-mon[109843]: pgmap v1878: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 137 MiB/s wr, 2.20k op/s; 31693/217677 objects degraded (14.560%); 2/217677 objects misplaced (0.001%) 2024-08-17T17:31:19.651 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:19.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:19.753 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:19.795 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:19.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:19 smithi162 ceph-mon[104668]: osdmap e1296: 8 total, 7 up, 7 in 2024-08-17T17:31:19.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:19 smithi162 ceph-mon[104668]: pgmap v1878: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 137 MiB/s wr, 2.20k op/s; 31693/217677 objects degraded (14.560%); 2/217677 objects misplaced (0.001%) 2024-08-17T17:31:19.897 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:19.935 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:19.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:19.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:20.036 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:20.042 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 35 16 77242 77226 137.873 92.5 0.0202462 0.00724797 2024-08-17T17:31:20.067 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:20.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:20.169 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:20.208 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:20.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:20.310 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:20.339 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:20.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:20.441 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:20.484 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:20.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:20.587 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:20.624 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:20.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:20.726 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:20.764 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:20.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:20.866 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:20.892 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:20.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:20.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:20.993 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:21.039 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:21.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:21.042 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 36 16 78752 78736 136.664 94.375 0.0133618 0.00731199 2024-08-17T17:31:21.141 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:21.180 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:21.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:21.282 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:21.313 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:21.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:21.415 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:21.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.449 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:21.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:21.456 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:21.534 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:31:21.550 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:21.577 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:21.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:21.678 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:21.721 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:21.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:21 smithi063 ceph-mon[114756]: pgmap v1879: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 62 MiB/s wr, 998 op/s; 31693/217677 objects degraded (14.560%); 2/217677 objects misplaced (0.001%) 2024-08-17T17:31:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:21 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1549925992' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:31:21.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:21 smithi063 ceph-mon[109843]: pgmap v1879: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 62 MiB/s wr, 998 op/s; 31693/217677 objects degraded (14.560%); 2/217677 objects misplaced (0.001%) 2024-08-17T17:31:21.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:21 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1549925992' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:31:21.823 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:21.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:21 smithi162 ceph-mon[104668]: pgmap v1879: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 62 MiB/s wr, 998 op/s; 31693/217677 objects degraded (14.560%); 2/217677 objects misplaced (0.001%) 2024-08-17T17:31:21.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:21 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1549925992' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:31:21.861 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:21.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:21.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:21.963 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:22.001 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:22.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:22.042 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 37 16 80195 80179 135.408 90.1875 0.00624147 0.00738064 2024-08-17T17:31:22.103 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:22.137 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:22.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:22.239 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:22.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.275 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:22.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:22.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:22.420 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:22.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:22.522 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:22.558 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:22.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:22.660 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:22.698 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:22.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:22.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:22 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:22.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:22 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1549925992' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:31:22.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:22 smithi063 ceph-mon[114756]: osdmap e1297: 8 total, 7 up, 7 in 2024-08-17T17:31:22.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:22 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:22.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:22 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1549925992' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:31:22.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:22 smithi063 ceph-mon[109843]: osdmap e1297: 8 total, 7 up, 7 in 2024-08-17T17:31:22.800 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:22.834 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:22.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:22.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:22 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:22.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:22 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1549925992' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:31:22.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:22 smithi162 ceph-mon[104668]: osdmap e1297: 8 total, 7 up, 7 in 2024-08-17T17:31:22.936 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:22.976 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:22.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:22.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:23.043 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 38 16 81617 81601 134.183 88.875 0.00589189 0.00744005 2024-08-17T17:31:23.077 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:23.113 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:23.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:23.215 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:23.250 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:23.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:23.351 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:23.378 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:23.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:23.480 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:23.508 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:23.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:23.564 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:31:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:31:23] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.51.0" 2024-08-17T17:31:23.610 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:23.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.645 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:23.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:23.746 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:23.771 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:23.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:23 smithi162 ceph-mon[104668]: pgmap v1881: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 156 MiB/s wr, 2.49k op/s; 34965/240141 objects degraded (14.560%); 2/240141 objects misplaced (0.001%) 2024-08-17T17:31:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:23 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 34965/240141 objects degraded (14.560%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:23.873 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:23.910 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:23.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:23.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:23.928 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:23 smithi063 ceph-mon[114756]: pgmap v1881: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 156 MiB/s wr, 2.49k op/s; 34965/240141 objects degraded (14.560%); 2/240141 objects misplaced (0.001%) 2024-08-17T17:31:23.928 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:23 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 34965/240141 objects degraded (14.560%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:23.929 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:23 smithi063 ceph-mon[109843]: pgmap v1881: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 156 MiB/s wr, 2.49k op/s; 34965/240141 objects degraded (14.560%); 2/240141 objects misplaced (0.001%) 2024-08-17T17:31:23.929 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:23 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 34965/240141 objects degraded (14.560%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:24.011 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:24.044 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 39 16 82898 82882 132.796 80.0625 0.0170378 0.00752543 2024-08-17T17:31:24.045 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:24.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:24.146 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:24.190 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:24.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:24.291 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:24.312 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:24.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:24.414 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:24.454 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:24.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:24.556 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:24.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.593 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:24.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:24.694 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:24.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.734 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:24.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:24.835 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:24.858 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:24.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:24.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:24.960 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:25.001 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:25.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:25.043 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:31:25.043019+0000 min lat: 0.00111601 max lat: 0.162415 avg lat: 0.00760484 2024-08-17T17:31:25.043 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:31:25.043 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 40 16 84131 84115 131.402 77.0625 0.0253353 0.00760484 2024-08-17T17:31:25.103 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:25.126 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:25.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:25.227 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:25.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.263 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:25.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:25.364 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:25.399 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:25.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:25.501 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:25.523 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:25.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:25.625 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:25.661 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:25.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:25.763 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:25.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.801 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:25.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:25.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:25 smithi162 ceph-mon[104668]: pgmap v1882: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 34965/240141 objects degraded (14.560%); 2/240141 objects misplaced (0.001%) 2024-08-17T17:31:25.902 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:25.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:25.937 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:25.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:25 smithi063 ceph-mon[114756]: pgmap v1882: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 34965/240141 objects degraded (14.560%); 2/240141 objects misplaced (0.001%) 2024-08-17T17:31:26.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:25 smithi063 ceph-mon[109843]: pgmap v1882: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 34965/240141 objects degraded (14.560%); 2/240141 objects misplaced (0.001%) 2024-08-17T17:31:26.038 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:26.043 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 41 16 85379 85363 130.099 78 0.00841326 0.00768148 2024-08-17T17:31:26.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.076 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:26.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:26.177 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:26.208 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:26.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:26.310 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:26.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.343 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:26.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:26.444 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:26.467 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:26.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:26.569 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:26.590 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:26.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:26.692 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:26.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.721 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:26.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:26.822 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:26.851 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:26.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:26.953 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:26.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:26.991 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:26.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:27.043 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 42 16 86607 86591 128.829 76.75 0.0110048 0.00775742 2024-08-17T17:31:27.092 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:27.116 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:27.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:27.218 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:27.253 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:27.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:27.355 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:27.394 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:27.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:27.496 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:27.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.530 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:27.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:27.631 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:27.655 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:27.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:27.757 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:27.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.796 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:27.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:27.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:27 smithi162 ceph-mon[104668]: pgmap v1883: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 61 MiB/s wr, 976 op/s; 35376/241374 objects degraded (14.656%); 2/241374 objects misplaced (0.001%) 2024-08-17T17:31:27.897 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:27.929 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:27.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:27.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:27 smithi063 ceph-mon[109843]: pgmap v1883: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 61 MiB/s wr, 976 op/s; 35376/241374 objects degraded (14.656%); 2/241374 objects misplaced (0.001%) 2024-08-17T17:31:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:27 smithi063 ceph-mon[114756]: pgmap v1883: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 61 MiB/s wr, 976 op/s; 35376/241374 objects degraded (14.656%); 2/241374 objects misplaced (0.001%) 2024-08-17T17:31:28.031 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:28.043 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 43 16 87986 87970 127.837 86.1875 0.0135646 0.00781743 2024-08-17T17:31:28.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.066 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:28.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:28.167 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:28.204 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:28.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:28.305 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:28.341 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:28.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:28.443 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:28.483 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:28.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:28.585 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:28.618 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:28.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:28.721 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:28.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.752 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:28.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:28.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:28 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 35376/241374 objects degraded (14.656%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:28.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:28 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 35376/241374 objects degraded (14.656%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:28.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:28 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 35376/241374 objects degraded (14.656%), 12 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:28.853 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:28.887 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:28.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:28.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:28.989 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:29.018 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:29.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:29.044 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 44 16 89362 89346 126.885 86 0.00917418 0.00787635 2024-08-17T17:31:29.120 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:29.150 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:29.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:29.252 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:29.281 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:29.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:29.383 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:29.422 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:29.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:29.524 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:29.570 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:29.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:29.672 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:29.698 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:29.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:29.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:29 smithi063 ceph-mon[109843]: pgmap v1884: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s; 37757/259422 objects degraded (14.554%); 2/259422 objects misplaced (0.001%) 2024-08-17T17:31:29.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:29 smithi063 ceph-mon[114756]: pgmap v1884: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s; 37757/259422 objects degraded (14.554%); 2/259422 objects misplaced (0.001%) 2024-08-17T17:31:29.800 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:29.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:29 smithi162 ceph-mon[104668]: pgmap v1884: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s; 37757/259422 objects degraded (14.554%); 2/259422 objects misplaced (0.001%) 2024-08-17T17:31:29.848 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:29.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:29.950 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:29.988 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:29.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:29.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:30.044 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 45 16 90716 90700 125.946 84.625 0.0159744 0.00793512 2024-08-17T17:31:30.090 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:30.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.130 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:30.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:30.231 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:30.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.263 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:30.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:30.364 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:30.365 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:31:30.366 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:31:30.403 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:31:30.403 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:31:30.405 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:30.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:30.448 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:31:30.448 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:31:30.483 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:31:30.483 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:31:30.495 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:31:30.495 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:31:30.507 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:30.531 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:31:30.532 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:31:30.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.553 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:30.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:30.564 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:31:30.564 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:31:30.567 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:31:30.599 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:31:30.599 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:31:30.654 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:30.664 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:30 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/636237051' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:30.664 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:30 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/636237051' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:30.699 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:30.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.700 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:30.801 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:30.840 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:30.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:30.942 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:30.977 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:30.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:30.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:31.044 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 46 16 91998 91982 124.95 80.125 0.0153371 0.00799881 2024-08-17T17:31:31.079 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:31.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:30 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/636237051' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:31.117 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:31.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:31.220 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:31.253 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:31.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:31.355 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:31.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.388 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:31.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:31.489 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:31.499 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:31:31.500 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:31:31.518 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:31.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:31.567 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:31:31.619 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:31.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.665 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:31.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:31.766 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:31 smithi063 ceph-mon[109843]: pgmap v1885: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s; 37757/259422 objects degraded (14.554%); 2/259422 objects misplaced (0.001%) 2024-08-17T17:31:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:31:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:31.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:31 smithi063 ceph-mon[114756]: pgmap v1885: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s; 37757/259422 objects degraded (14.554%); 2/259422 objects misplaced (0.001%) 2024-08-17T17:31:31.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:31:31.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:31.812 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:31.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:31.913 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:31:31.914 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:31.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:31.964 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:31.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:32.044 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 47 16 93328 93312 124.06 83.125 0.00393726 0.00805558 2024-08-17T17:31:32.065 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:31 smithi162 ceph-mon[104668]: pgmap v1885: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s; 37757/259422 objects degraded (14.554%); 2/259422 objects misplaced (0.001%) 2024-08-17T17:31:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:31:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:32.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.113 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:32.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:32.214 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:32.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.252 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:32.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:32.353 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:32.390 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:32.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:32.491 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:32.530 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:32.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:32.632 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:32.692 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:32.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:32.794 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:32.837 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:32.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:32.924 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:32.939 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:32.985 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:32.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:32.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:33.044 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 48 16 94609 94593 123.143 80.0625 0.0131811 0.00811602 2024-08-17T17:31:33.087 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:33.138 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:33.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:33.240 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:33.273 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:33.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:33.322 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:31:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:31:33] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-08-17T17:31:33.375 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:33.414 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:33.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:33.471 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:33.501 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:31:33.515 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:33.542 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:33.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:33.643 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:33.671 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:33.672 INFO:teuthology.orchestra.run.smithi063.stdout:moved 8 / 357 (2.2409%) 2024-08-17T17:31:33.672 INFO:teuthology.orchestra.run.smithi063.stdout:avg 59.5 2024-08-17T17:31:33.672 INFO:teuthology.orchestra.run.smithi063.stdout:stddev 11.2064 -> 10.8896 (expected baseline 7.04154) 2024-08-17T17:31:33.672 INFO:teuthology.orchestra.run.smithi063.stdout:min osd.2 with 39 -> 40 pgs (0.655462 -> 0.672269 * mean) 2024-08-17T17:31:33.672 INFO:teuthology.orchestra.run.smithi063.stdout:max osd.3 with 59 -> 59 pgs (0.991597 -> 0.991597 * mean) 2024-08-17T17:31:33.672 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:31:33.672 INFO:teuthology.orchestra.run.smithi063.stdout:oload 110 2024-08-17T17:31:33.672 INFO:teuthology.orchestra.run.smithi063.stdout:max_change 0.05 2024-08-17T17:31:33.672 INFO:teuthology.orchestra.run.smithi063.stdout:max_change_osds 4 2024-08-17T17:31:33.672 INFO:teuthology.orchestra.run.smithi063.stdout:average_utilization 0.0306 2024-08-17T17:31:33.673 INFO:teuthology.orchestra.run.smithi063.stdout:overload_utilization 0.0337 2024-08-17T17:31:33.673 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0 weight 0.9500 -> 0.9000 2024-08-17T17:31:33.673 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4 weight 0.9500 -> 0.9000 2024-08-17T17:31:33.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:33.775 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:33.803 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:33.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:33.897 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:33 smithi162 ceph-mon[104668]: pgmap v1886: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 115 MiB/s wr, 1.85k op/s; 40794/279675 objects degraded (14.586%); 2/279675 objects misplaced (0.001%) 2024-08-17T17:31:33.897 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:33 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 40794/279675 objects degraded (14.586%), 12 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:33.906 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:33.929 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:33.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:33.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:33 smithi063 ceph-mon[114756]: pgmap v1886: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 115 MiB/s wr, 1.85k op/s; 40794/279675 objects degraded (14.586%); 2/279675 objects misplaced (0.001%) 2024-08-17T17:31:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:33 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 40794/279675 objects degraded (14.586%), 12 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:34.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:33 smithi063 ceph-mon[109843]: pgmap v1886: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 115 MiB/s wr, 1.85k op/s; 40794/279675 objects degraded (14.586%); 2/279675 objects misplaced (0.001%) 2024-08-17T17:31:34.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:33 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 40794/279675 objects degraded (14.586%), 12 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:34.031 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:34.044 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 49 16 95473 95457 121.732 54 0.0032087 0.00816393 2024-08-17T17:31:34.053 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:34.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:34.155 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:34.178 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:34.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:34.280 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:34.305 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:34.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:34.406 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:34.431 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:34.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:34.533 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:34.557 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:34.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:34.659 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:34.690 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:34.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:34.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:34 smithi063 ceph-mon[109843]: from='client.45860 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:34.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:34 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'58984','4':'58984'}"} : dispatch 2024-08-17T17:31:34.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:34 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'58984','4':'58984'}"} : dispatch 2024-08-17T17:31:34.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:34 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'58984','4':'58984'}"}]': finished 2024-08-17T17:31:34.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:34 smithi063 ceph-mon[109843]: osdmap e1298: 8 total, 7 up, 7 in 2024-08-17T17:31:34.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:34 smithi063 ceph-mon[114756]: from='client.45860 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:34.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:34 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'58984','4':'58984'}"} : dispatch 2024-08-17T17:31:34.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:34 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'58984','4':'58984'}"} : dispatch 2024-08-17T17:31:34.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:34 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'58984','4':'58984'}"}]': finished 2024-08-17T17:31:34.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:34 smithi063 ceph-mon[114756]: osdmap e1298: 8 total, 7 up, 7 in 2024-08-17T17:31:34.792 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:34.818 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:34.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:34.919 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:34.943 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:34.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:34.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:35.044 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 50 16 95473 95457 119.298 0 - 0.00816393 2024-08-17T17:31:35.046 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:35.069 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:35.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:34 smithi162 ceph-mon[104668]: from='client.45860 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-08-17T17:31:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:34 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'58984','4':'58984'}"} : dispatch 2024-08-17T17:31:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:34 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'58984','4':'58984'}"} : dispatch 2024-08-17T17:31:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:34 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'58984','4':'58984'}"}]': finished 2024-08-17T17:31:35.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:34 smithi162 ceph-mon[104668]: osdmap e1298: 8 total, 7 up, 7 in 2024-08-17T17:31:35.171 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:35.194 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:35.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:35.296 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:35.318 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:35.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:35.420 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:35.442 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:35.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:35.544 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:35.566 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:35.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:35.667 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:35.695 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:35.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:35.796 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:35.818 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:35.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:35.920 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:35.952 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:35.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:35.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:35 smithi063 ceph-mon[114756]: pgmap v1888: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s; 40794/279675 objects degraded (14.586%); 2/279675 objects misplaced (0.001%) 2024-08-17T17:31:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:35 smithi063 ceph-mon[114756]: osdmap e1299: 8 total, 7 up, 7 in 2024-08-17T17:31:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:35 smithi063 ceph-mon[109843]: pgmap v1888: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s; 40794/279675 objects degraded (14.586%); 2/279675 objects misplaced (0.001%) 2024-08-17T17:31:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:35 smithi063 ceph-mon[109843]: osdmap e1299: 8 total, 7 up, 7 in 2024-08-17T17:31:36.044 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 51 16 95636 95620 117.158 5.09375 0.00896812 0.00853011 2024-08-17T17:31:36.054 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:36.093 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:36.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:35 smithi162 ceph-mon[104668]: pgmap v1888: 119 pgs: 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 66 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 82 MiB/s wr, 1.32k op/s; 40794/279675 objects degraded (14.586%); 2/279675 objects misplaced (0.001%) 2024-08-17T17:31:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:35 smithi162 ceph-mon[104668]: osdmap e1299: 8 total, 7 up, 7 in 2024-08-17T17:31:36.195 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:36.227 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:36.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:36.328 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:36.369 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:36.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:36.470 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:36.516 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:36.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:36.618 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:36.645 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:36.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:36.747 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:36.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.778 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:36.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:36.882 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:36.907 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:36 smithi063 ceph-mon[114756]: osdmap e1300: 8 total, 7 up, 7 in 2024-08-17T17:31:36.907 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:36 smithi063 ceph-mon[109843]: osdmap e1300: 8 total, 7 up, 7 in 2024-08-17T17:31:36.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:36.924 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:36.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:37.025 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:37.048 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 52 16 96775 96759 116.27 71.1875 0.0140904 0.00859541 2024-08-17T17:31:37.070 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:37.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:36 smithi162 ceph-mon[104668]: osdmap e1300: 8 total, 7 up, 7 in 2024-08-17T17:31:37.172 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:37.203 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:37.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:37.306 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:37.365 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:37.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:37.467 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:37.520 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:37.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:37.621 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:37.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.655 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:37.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:37.703 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:31:37.713 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:37 smithi063 ceph-mon[114756]: pgmap v1891: 119 pgs: 1 active+undersized+remapped, 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 65 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 41017/280344 objects degraded (14.631%); 2/280344 objects misplaced (0.001%) 2024-08-17T17:31:37.756 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:37.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.786 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:37.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:37.856 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:31:37.856 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:31:37.887 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:37.915 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:37.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:37.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:38.017 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:37 smithi063 ceph-mon[109843]: pgmap v1891: 119 pgs: 1 active+undersized+remapped, 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 65 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 41017/280344 objects degraded (14.631%); 2/280344 objects misplaced (0.001%) 2024-08-17T17:31:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:37 smithi063 ceph-mon[109843]: osdmap e1301: 8 total, 7 up, 7 in 2024-08-17T17:31:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:37 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3081458175' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:37 smithi063 ceph-mon[114756]: osdmap e1301: 8 total, 7 up, 7 in 2024-08-17T17:31:38.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:37 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3081458175' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:38.046 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 53 16 97848 97832 115.342 67.0625 0.00766392 0.0086642 2024-08-17T17:31:38.060 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:38.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:37 smithi162 ceph-mon[104668]: pgmap v1891: 119 pgs: 1 active+undersized+remapped, 20 active+clean+remapped, 21 active+undersized, 12 active+undersized+degraded, 65 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 41017/280344 objects degraded (14.631%); 2/280344 objects misplaced (0.001%) 2024-08-17T17:31:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:37 smithi162 ceph-mon[104668]: osdmap e1301: 8 total, 7 up, 7 in 2024-08-17T17:31:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:37 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3081458175' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:31:38.162 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:38.197 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:38.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:38.299 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:38.328 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:38.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:38.430 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:38.464 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:38.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.465 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:38.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:38.606 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:38.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:38.708 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:38.742 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:38.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:38.794 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:38.843 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:38.872 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:38.880 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:38 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:38.880 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:38 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3081458175' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:38.880 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:38 smithi063 ceph-mon[109843]: osdmap e1302: 8 total, 7 up, 7 in 2024-08-17T17:31:38.880 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:38 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 41017/280344 objects degraded (14.631%), 12 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:38.881 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:38 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:38.881 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:38 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3081458175' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:38.881 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:38 smithi063 ceph-mon[114756]: osdmap e1302: 8 total, 7 up, 7 in 2024-08-17T17:31:38.881 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:38 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 41017/280344 objects degraded (14.631%), 12 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:38.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:38.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:38.983 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:39.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.019 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:39.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:39.046 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 54 16 99042 99026 114.588 74.625 0.0128063 0.00872228 2024-08-17T17:31:39.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:38 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:38 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3081458175' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:31:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:38 smithi162 ceph-mon[104668]: osdmap e1302: 8 total, 7 up, 7 in 2024-08-17T17:31:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:38 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 41017/280344 objects degraded (14.631%), 12 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:39.120 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:39.155 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:39.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:39.256 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:39.293 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:39.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:39.395 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:39.441 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:39.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:39.543 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:39.547 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:31:39.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.573 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:39.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:39.674 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:39.698 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:39.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:39.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:39 smithi063 ceph-mon[109843]: pgmap v1894: 119 pgs: 1 active+undersized+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 64 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s; 42321/290052 objects degraded (14.591%); 5871/290052 objects misplaced (2.024%); 336 KiB/s, 5 objects/s recovering 2024-08-17T17:31:39.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:39 smithi063 ceph-mon[114756]: pgmap v1894: 119 pgs: 1 active+undersized+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 64 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s; 42321/290052 objects degraded (14.591%); 5871/290052 objects misplaced (2.024%); 336 KiB/s, 5 objects/s recovering 2024-08-17T17:31:39.800 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:39.836 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:39.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:39.938 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:39.978 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:39.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:39.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:40.047 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 55 16 100229 100213 113.853 74.1875 0.0128963 0.0087788 2024-08-17T17:31:40.079 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:39 smithi162 ceph-mon[104668]: pgmap v1894: 119 pgs: 1 active+undersized+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 64 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 54 MiB/s wr, 864 op/s; 42321/290052 objects degraded (14.591%); 5871/290052 objects misplaced (2.024%); 336 KiB/s, 5 objects/s recovering 2024-08-17T17:31:40.112 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:40.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:40.214 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:40.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.255 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:40.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:40.356 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:40.392 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:40.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:40.494 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:40.522 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:40.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:40.624 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:40.654 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:40.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:40.756 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:40.796 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:40.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:40.874 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:40.883 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:31:40.899 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:40.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:40.983 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:40.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:41.047 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 56 16 101363 101347 113.086 70.875 0.0129222 0.00883607 2024-08-17T17:31:41.083 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:41.118 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0] out_osds: [1] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:31:41.118 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:31:41.119 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2024-08-17T17:31:41.119 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:31:41.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.120 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:41.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:41.221 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:41.250 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:41.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:41.351 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:41.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.397 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:41.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:41.498 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:41.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.548 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:41.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:41.649 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:41.695 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:41.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:41.797 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:41.845 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:41.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:41.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:41 smithi063 ceph-mon[114756]: pgmap v1895: 119 pgs: 1 active+undersized+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 64 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 37 MiB/s wr, 594 op/s; 42321/290052 objects degraded (14.591%); 5871/290052 objects misplaced (2.024%); 231 KiB/s, 3 objects/s recovering 2024-08-17T17:31:41.911 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:41 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3488234778' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:41.911 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:41 smithi063 ceph-mon[109843]: pgmap v1895: 119 pgs: 1 active+undersized+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 64 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 37 MiB/s wr, 594 op/s; 42321/290052 objects degraded (14.591%); 5871/290052 objects misplaced (2.024%); 231 KiB/s, 3 objects/s recovering 2024-08-17T17:31:41.911 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:41 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3488234778' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:41.947 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:41.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:41.981 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:41.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:42.047 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 57 16 102409 102393 112.248 65.375 0.00778315 0.00890349 2024-08-17T17:31:42.080 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:31:42.080 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:31:42.080 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:31:42.080 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:31:42.080 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:31:42.080 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:31:42.080 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:31:42.081 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:31:42.081 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:31:42.081 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:31:42.081 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:31:42.081 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:31:42.081 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:31:42.081 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:31:42.081 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:31:42.082 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:41 smithi162 ceph-mon[104668]: pgmap v1895: 119 pgs: 1 active+undersized+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 64 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 37 MiB/s wr, 594 op/s; 42321/290052 objects degraded (14.591%); 5871/290052 objects misplaced (2.024%); 231 KiB/s, 3 objects/s recovering 2024-08-17T17:31:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:41 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3488234778' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:42.116 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:42.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:42.218 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:42.254 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:42.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:42.356 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:42.385 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:42.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:42.486 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:42.526 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:42.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:42.628 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:42.661 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:42.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:42.763 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:42.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.799 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:42.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:42.900 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:42.925 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:31:42.926 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:31:42.936 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:42.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:42.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:43.038 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:43.049 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 58 16 103563 103547 111.554 72.125 0.0488051 0.00895833 2024-08-17T17:31:43.077 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:43.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:43.178 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:43.210 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:43.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:43.312 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:43.339 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:43.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:43.441 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:43.478 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:43.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:43.532 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:43.580 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:43.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.642 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:43.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:43.689 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:31:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:31:43] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-08-17T17:31:43.743 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:43.782 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:43.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:43.863 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:43.884 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:43.912 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:43.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:43.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:43.941 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:43 smithi063 ceph-mon[109843]: pgmap v1896: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s; 44779/307023 objects degraded (14.585%); 5776/307023 objects misplaced (1.881%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:31:43.941 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:43 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 44779/307023 objects degraded (14.585%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:43.942 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:43 smithi063 ceph-mon[114756]: pgmap v1896: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s; 44779/307023 objects degraded (14.585%); 5776/307023 objects misplaced (1.881%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:31:43.942 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:43 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 44779/307023 objects degraded (14.585%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:44.014 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:44.050 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 59 16 104673 104657 110.837 69.375 0.0210309 0.00901746 2024-08-17T17:31:44.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.051 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:44.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:44.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:43 smithi162 ceph-mon[104668]: pgmap v1896: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s; 44779/307023 objects degraded (14.585%); 5776/307023 objects misplaced (1.881%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:31:44.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:43 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 44779/307023 objects degraded (14.585%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:44.152 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:44.191 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:44.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:44.293 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:44.324 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:44.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:44.426 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:44.468 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:44.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:44.570 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:44.613 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:44.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:44.715 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:44.751 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:44.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:44.853 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:44.894 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:44.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:44.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:44.927 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:31:44.996 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:45.035 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:45.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:45.049 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:31:45.050014+0000 min lat: 0.00111601 max lat: 2.21144 avg lat: 0.00907157 2024-08-17T17:31:45.050 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:31:45.050 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 60 16 105815 105799 110.179 71.375 0.00698324 0.00907157 2024-08-17T17:31:45.137 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:45.193 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:45.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:45.294 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:45.324 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:45.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:45.426 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:45.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.466 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:45.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:45.567 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:45.601 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:45.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:45.704 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:45.748 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:45.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:45.850 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:45.888 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:45.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:45.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:45.991 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:46.010 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:45 smithi063 ceph-mon[114756]: pgmap v1897: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 44779/307023 objects degraded (14.585%); 5776/307023 objects misplaced (1.881%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:31:46.011 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:45 smithi063 ceph-mon[109843]: pgmap v1897: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 44779/307023 objects degraded (14.585%); 5776/307023 objects misplaced (1.881%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:31:46.025 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:46.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:46.050 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 61 16 106940 106924 109.526 70.3125 0.0166381 0.00912454 2024-08-17T17:31:46.086 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:31:46.086 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:31:46.086 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.086 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:106973 10.5 10:a1ca0048:::benchmark_data_smithi063_294527_object106972:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1302)", 2024-08-17T17:31:46.086 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:31:46.080630+0000", 2024-08-17T17:31:46.086 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0026658609999999998, 2024-08-17T17:31:46.086 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0026994779999999999, 2024-08-17T17:31:46.087 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:31:46.087 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:31:46.087 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:31:46.087 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:31:46.087 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:31:46.087 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:31:46.087 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 106973 2024-08-17T17:31:46.087 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.087 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:31:46.087 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.087 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:31:46.087 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.080630+0000", 2024-08-17T17:31:46.088 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.088 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.088 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.088 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:31:46.088 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.080630+0000", 2024-08-17T17:31:46.088 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.088 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.088 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.088 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:31:46.088 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.080632+0000", 2024-08-17T17:31:46.088 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4580000000000001e-06 2024-08-17T17:31:46.088 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.088 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.089 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:31:46.089 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.080680+0000", 2024-08-17T17:31:46.089 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8162999999999999e-05 2024-08-17T17:31:46.089 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.089 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.089 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:31:46.089 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.080693+0000", 2024-08-17T17:31:46.089 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3104e-05 2024-08-17T17:31:46.089 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.089 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.089 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:31:46.089 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.080700+0000", 2024-08-17T17:31:46.089 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9739999999999996e-06 2024-08-17T17:31:46.089 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.090 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.090 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:31:46.090 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.080753+0000", 2024-08-17T17:31:46.090 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3474999999999999e-05 2024-08-17T17:31:46.090 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.090 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.090 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:31:46.090 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.080904+0000", 2024-08-17T17:31:46.090 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015066999999999999 2024-08-17T17:31:46.090 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.090 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.090 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:31:46.090 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081062+0000", 2024-08-17T17:31:46.091 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000157551 2024-08-17T17:31:46.091 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.091 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:31:46.091 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.091 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.091 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.091 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:106975 10.5 10.2ecfb1f5 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1302)", 2024-08-17T17:31:46.091 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:31:46.081298+0000", 2024-08-17T17:31:46.091 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0019977739999999999, 2024-08-17T17:31:46.091 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002108122, 2024-08-17T17:31:46.091 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:31:46.091 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:31:46.091 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "queued for pg", 2024-08-17T17:31:46.092 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:31:46.092 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:31:46.092 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:31:46.092 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 106975 2024-08-17T17:31:46.092 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.092 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:31:46.092 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.092 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:31:46.092 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081298+0000", 2024-08-17T17:31:46.092 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.092 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.092 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.093 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:31:46.093 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081298+0000", 2024-08-17T17:31:46.093 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.093 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.093 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.093 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:31:46.093 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081299+0000", 2024-08-17T17:31:46.093 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0380000000000001e-06 2024-08-17T17:31:46.093 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.093 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.093 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:31:46.093 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081323+0000", 2024-08-17T17:31:46.094 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3130999999999999e-05 2024-08-17T17:31:46.094 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.094 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.094 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:31:46.094 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081337+0000", 2024-08-17T17:31:46.094 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4083999999999999e-05 2024-08-17T17:31:46.094 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.094 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.094 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:31:46.094 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081344+0000", 2024-08-17T17:31:46.094 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6760000000000007e-06 2024-08-17T17:31:46.094 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.094 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:31:46.095 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.095 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.095 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.095 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:106974 10.e e1302/1288 10:743c72fb:::benchmark_data_smithi063_294527_object106973:head v 1302'6728, mlcod=1302'6725)", 2024-08-17T17:31:46.095 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:31:46.081372+0000", 2024-08-17T17:31:46.095 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.001923892, 2024-08-17T17:31:46.095 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0020903300000000001, 2024-08-17T17:31:46.095 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:31:46.095 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:31:46.095 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:31:46.095 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:31:46.095 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.095 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:31:46.095 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081372+0000", 2024-08-17T17:31:46.096 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.096 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.096 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.096 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:31:46.096 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081372+0000", 2024-08-17T17:31:46.096 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.096 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.096 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.096 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:31:46.096 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081373+0000", 2024-08-17T17:31:46.096 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1099999999999995e-07 2024-08-17T17:31:46.096 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.096 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.096 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:31:46.097 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081437+0000", 2024-08-17T17:31:46.097 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4282999999999994e-05 2024-08-17T17:31:46.097 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.097 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.097 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:31:46.097 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081446+0000", 2024-08-17T17:31:46.097 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8580000000000006e-06 2024-08-17T17:31:46.097 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.097 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.097 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:31:46.097 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081452+0000", 2024-08-17T17:31:46.097 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0549999999999996e-06 2024-08-17T17:31:46.097 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.098 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.098 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:31:46.098 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081516+0000", 2024-08-17T17:31:46.098 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4338999999999995e-05 2024-08-17T17:31:46.098 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.098 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.098 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:31:46.098 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081536+0000", 2024-08-17T17:31:46.098 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9049999999999999e-05 2024-08-17T17:31:46.098 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.098 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:31:46.098 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.098 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.099 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.099 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.45818.0:106962 10.5 e1302/1288)", 2024-08-17T17:31:46.099 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:31:46.081468+0000", 2024-08-17T17:31:46.099 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.001827902, 2024-08-17T17:31:46.099 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0020449650000000002, 2024-08-17T17:31:46.099 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:31:46.099 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:31:46.099 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "queued for pg", 2024-08-17T17:31:46.099 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:31:46.099 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.099 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:31:46.099 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081468+0000", 2024-08-17T17:31:46.099 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.099 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.100 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.100 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:31:46.100 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081468+0000", 2024-08-17T17:31:46.100 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.100 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.100 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.100 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:31:46.100 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081469+0000", 2024-08-17T17:31:46.100 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4000000000000002e-07 2024-08-17T17:31:46.100 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.100 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.100 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:31:46.100 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081472+0000", 2024-08-17T17:31:46.101 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7840000000000001e-06 2024-08-17T17:31:46.101 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.101 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.101 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:31:46.101 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:45 smithi162 ceph-mon[104668]: pgmap v1897: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.2 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 69 MiB/s wr, 1.11k op/s; 44779/307023 objects degraded (14.585%); 5776/307023 objects misplaced (1.881%); 928 KiB/s, 14 objects/s recovering 2024-08-17T17:31:46.102 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081483+0000", 2024-08-17T17:31:46.102 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0332e-05 2024-08-17T17:31:46.102 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.102 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.102 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:31:46.102 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.081490+0000", 2024-08-17T17:31:46.102 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8090000000000004e-06 2024-08-17T17:31:46.102 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.102 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:31:46.102 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.103 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.103 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.103 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:106976 10.8 10.a9c6fb28 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1302)", 2024-08-17T17:31:46.103 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:31:46.082620+0000", 2024-08-17T17:31:46.103 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.00067585500000000005, 2024-08-17T17:31:46.103 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00093908100000000003, 2024-08-17T17:31:46.103 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:31:46.103 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:31:46.103 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "queued for pg", 2024-08-17T17:31:46.103 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:31:46.103 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:31:46.103 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:31:46.103 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 106976 2024-08-17T17:31:46.103 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.104 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:31:46.104 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.104 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:31:46.104 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.082620+0000", 2024-08-17T17:31:46.104 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.104 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.104 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.104 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:31:46.104 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.082620+0000", 2024-08-17T17:31:46.104 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.104 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.104 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.104 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:31:46.105 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.082622+0000", 2024-08-17T17:31:46.105 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.31e-06 2024-08-17T17:31:46.105 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.105 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.105 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:31:46.105 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.082645+0000", 2024-08-17T17:31:46.105 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3308000000000001e-05 2024-08-17T17:31:46.105 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.105 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.105 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:31:46.105 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.082662+0000", 2024-08-17T17:31:46.105 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7592e-05 2024-08-17T17:31:46.105 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.105 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.106 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:31:46.106 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.082671+0000", 2024-08-17T17:31:46.106 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2870000000000004e-06 2024-08-17T17:31:46.106 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.106 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:31:46.106 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.106 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.106 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.106 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop_reply(client.45818.0:106962 10.5 e1302/1288)", 2024-08-17T17:31:46.106 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:31:46.082853+0000", 2024-08-17T17:31:46.106 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.00044356799999999998, 2024-08-17T17:31:46.106 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00075433300000000004, 2024-08-17T17:31:46.107 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:31:46.107 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:31:46.107 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "queued for pg", 2024-08-17T17:31:46.107 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:31:46.107 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.107 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:31:46.107 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.082853+0000", 2024-08-17T17:31:46.107 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.107 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.107 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.107 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:31:46.107 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.082853+0000", 2024-08-17T17:31:46.107 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.108 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.108 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.108 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:31:46.108 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.082853+0000", 2024-08-17T17:31:46.108 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.1900000000000001e-07 2024-08-17T17:31:46.108 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.108 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.108 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:31:46.108 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.082858+0000", 2024-08-17T17:31:46.108 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3789999999999999e-06 2024-08-17T17:31:46.108 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.109 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.109 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:31:46.109 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.082868+0000", 2024-08-17T17:31:46.109 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0237999999999999e-05 2024-08-17T17:31:46.109 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.109 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.109 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:31:46.109 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.082874+0000", 2024-08-17T17:31:46.109 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9359999999999996e-06 2024-08-17T17:31:46.109 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.109 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:31:46.109 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.109 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.110 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.110 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:106956 10.a e1302/1299 10:5fa020d4:::benchmark_data_smithi063_294527_object106955:head v 1302'6624, mlcod=1292'187)", 2024-08-17T17:31:46.110 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:31:46.064587+0000", 2024-08-17T17:31:46.110 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.018708998000000001, 2024-08-17T17:31:46.110 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.019065601000000001, 2024-08-17T17:31:46.110 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:31:46.110 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:31:46.110 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:31:46.110 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:31:46.110 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.110 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:31:46.110 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.064587+0000", 2024-08-17T17:31:46.111 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.111 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.111 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.111 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:31:46.111 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.064587+0000", 2024-08-17T17:31:46.111 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.111 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.111 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.111 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:31:46.111 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.064588+0000", 2024-08-17T17:31:46.111 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.1100000000000005e-07 2024-08-17T17:31:46.112 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.112 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.112 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:31:46.112 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.064616+0000", 2024-08-17T17:31:46.112 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8442e-05 2024-08-17T17:31:46.112 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.112 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.112 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:31:46.112 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.064630+0000", 2024-08-17T17:31:46.112 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3974e-05 2024-08-17T17:31:46.112 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.112 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.112 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:31:46.113 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.064637+0000", 2024-08-17T17:31:46.113 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9460000000000001e-06 2024-08-17T17:31:46.113 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.113 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.113 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:31:46.113 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.076193+0000", 2024-08-17T17:31:46.113 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011555786 2024-08-17T17:31:46.113 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.113 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.113 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:31:46.113 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.076214+0000", 2024-08-17T17:31:46.113 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0506999999999999e-05 2024-08-17T17:31:46.114 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.114 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:31:46.114 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.114 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.114 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.114 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:106962 10.5 10:ad40ef26:::benchmark_data_smithi063_294527_object106961:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1302)", 2024-08-17T17:31:46.114 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:31:46.067391+0000", 2024-08-17T17:31:46.114 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.015904651999999998, 2024-08-17T17:31:46.114 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.016316635999999999, 2024-08-17T17:31:46.114 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:31:46.114 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:31:46.114 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:31:46.115 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:31:46.115 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:31:46.115 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:31:46.115 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 106962 2024-08-17T17:31:46.115 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.115 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:31:46.115 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.115 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:31:46.115 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.067391+0000", 2024-08-17T17:31:46.115 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.115 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.115 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.115 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:31:46.116 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.067391+0000", 2024-08-17T17:31:46.116 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.116 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.116 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.116 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:31:46.116 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.067392+0000", 2024-08-17T17:31:46.116 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8000000000000005e-07 2024-08-17T17:31:46.116 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.116 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.116 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:31:46.116 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.067413+0000", 2024-08-17T17:31:46.116 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0846e-05 2024-08-17T17:31:46.117 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.117 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.117 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:31:46.117 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.067481+0000", 2024-08-17T17:31:46.117 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7514000000000004e-05 2024-08-17T17:31:46.117 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.117 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.117 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:31:46.117 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.067489+0000", 2024-08-17T17:31:46.117 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0350000000000001e-06 2024-08-17T17:31:46.117 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.117 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.117 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:31:46.118 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.071382+0000", 2024-08-17T17:31:46.118 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0038931769999999998 2024-08-17T17:31:46.118 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.118 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.118 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:31:46.118 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.071550+0000", 2024-08-17T17:31:46.118 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016808099999999999 2024-08-17T17:31:46.118 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.118 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.118 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:31:46.118 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.071695+0000", 2024-08-17T17:31:46.118 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000145279 2024-08-17T17:31:46.119 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.119 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.119 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:31:46.119 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.076513+0000", 2024-08-17T17:31:46.119 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00481819 2024-08-17T17:31:46.119 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.119 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:31:46.119 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.119 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.119 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.119 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:106957 10.a e1302/1299 10:538f88c8:::benchmark_data_smithi063_294527_object106956:head v 1302'6625, mlcod=1292'187)", 2024-08-17T17:31:46.119 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:31:46.069619+0000", 2024-08-17T17:31:46.120 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.013676949000000001, 2024-08-17T17:31:46.120 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.01416561, 2024-08-17T17:31:46.120 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:31:46.120 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:31:46.120 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:31:46.120 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:31:46.120 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.120 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:31:46.120 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.069619+0000", 2024-08-17T17:31:46.120 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.120 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.120 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.120 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:31:46.121 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.069619+0000", 2024-08-17T17:31:46.121 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.121 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.121 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.121 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:31:46.121 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.069620+0000", 2024-08-17T17:31:46.121 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.9199999999999999e-07 2024-08-17T17:31:46.121 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.121 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.121 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:31:46.121 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.069660+0000", 2024-08-17T17:31:46.121 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.0037999999999999e-05 2024-08-17T17:31:46.122 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.122 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.122 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:31:46.122 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.069678+0000", 2024-08-17T17:31:46.122 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7425e-05 2024-08-17T17:31:46.122 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.122 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.122 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:31:46.122 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.069685+0000", 2024-08-17T17:31:46.122 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2980000000000003e-06 2024-08-17T17:31:46.122 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.122 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.123 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:31:46.123 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.083555+0000", 2024-08-17T17:31:46.123 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.013870362000000001 2024-08-17T17:31:46.123 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.123 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.123 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:31:46.123 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.083576+0000", 2024-08-17T17:31:46.123 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0496000000000002e-05 2024-08-17T17:31:46.123 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.123 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:31:46.123 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.123 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.123 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.123 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:106958 10.a e1302/1299)", 2024-08-17T17:31:46.123 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:31:46.071831+0000", 2024-08-17T17:31:46.124 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.011465042, 2024-08-17T17:31:46.124 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.012004406, 2024-08-17T17:31:46.124 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:31:46.124 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:31:46.124 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "queued for pg", 2024-08-17T17:31:46.124 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:31:46.124 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.124 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:31:46.124 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.071831+0000", 2024-08-17T17:31:46.124 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.124 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.124 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.124 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:31:46.124 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.071831+0000", 2024-08-17T17:31:46.125 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.125 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.125 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.125 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:31:46.125 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.071832+0000", 2024-08-17T17:31:46.125 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.95e-07 2024-08-17T17:31:46.125 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.125 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.125 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:31:46.125 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.071872+0000", 2024-08-17T17:31:46.125 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.0080999999999998e-05 2024-08-17T17:31:46.125 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.125 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.125 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:31:46.125 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.071897+0000", 2024-08-17T17:31:46.126 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4400999999999999e-05 2024-08-17T17:31:46.126 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.126 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.126 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:31:46.126 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.071913+0000", 2024-08-17T17:31:46.126 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6310000000000001e-05 2024-08-17T17:31:46.126 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.126 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:31:46.126 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.126 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.126 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.126 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:106969 10.8 10:1029cb4d:::benchmark_data_smithi063_294527_object106968:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1302)", 2024-08-17T17:31:46.126 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:31:46.072638+0000", 2024-08-17T17:31:46.126 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.010657890999999999, 2024-08-17T17:31:46.127 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011249162, 2024-08-17T17:31:46.127 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:31:46.127 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:31:46.127 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:31:46.127 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:31:46.127 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:31:46.127 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:31:46.127 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 106969 2024-08-17T17:31:46.127 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.127 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:31:46.127 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.127 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:31:46.128 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.072638+0000", 2024-08-17T17:31:46.128 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.128 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.128 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.128 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:31:46.128 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.072638+0000", 2024-08-17T17:31:46.128 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:31:46.128 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.128 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.128 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:31:46.128 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.072639+0000", 2024-08-17T17:31:46.128 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.15e-06 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.072662+0000", 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2493999999999998e-05 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.072672+0000", 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0356999999999999e-05 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.072680+0000", 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4399999999999999e-06 2024-08-17T17:31:46.129 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.130 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.130 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:31:46.130 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.079556+0000", 2024-08-17T17:31:46.130 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0068762889999999998 2024-08-17T17:31:46.130 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.130 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.130 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:31:46.130 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.079721+0000", 2024-08-17T17:31:46.130 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016491800000000001 2024-08-17T17:31:46.130 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:31:46.130 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:31:46.130 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:31:46.130 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:31:46.079869+0000", 2024-08-17T17:31:46.131 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014859899999999999 2024-08-17T17:31:46.131 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.131 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:31:46.131 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.131 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:31:46.131 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-08-17T17:31:46.131 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 11 2024-08-17T17:31:46.131 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:31:46.131 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:46.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.164 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:46.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:46.265 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:46.295 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:46.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:46.396 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:46.435 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:46.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.436 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:46.537 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:46.572 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:46.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:46.674 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:46.708 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:46.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:46.809 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:46.844 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:46.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:46.946 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:46.979 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:46.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:46.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:47.002 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:47.003 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:47.044 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:31:47.050 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 62 16 108126 108110 108.955 74.125 0.00474879 0.00917266 2024-08-17T17:31:47.080 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:47.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:31:47.136 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:47.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:47.238 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:47.294 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:47.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:47.396 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:47.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.441 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:47.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:47.542 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:47.571 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:47.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:47.673 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:47.712 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:47.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:47.814 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:47.867 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:47.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:47.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:47.969 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:48.013 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:48.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:48.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:47 smithi063 ceph-mon[109843]: pgmap v1898: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 59 MiB/s wr, 941 op/s; 45102/307992 objects degraded (14.644%); 5776/307992 objects misplaced (1.875%); 759 KiB/s, 11 objects/s recovering 2024-08-17T17:31:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:47 smithi063 ceph-mon[114756]: pgmap v1898: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 59 MiB/s wr, 941 op/s; 45102/307992 objects degraded (14.644%); 5776/307992 objects misplaced (1.875%); 759 KiB/s, 11 objects/s recovering 2024-08-17T17:31:48.050 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 63 16 109188 109172 108.279 66.375 0.0126127 0.00923056 2024-08-17T17:31:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:47 smithi162 ceph-mon[104668]: pgmap v1898: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.3 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 59 MiB/s wr, 941 op/s; 45102/307992 objects degraded (14.644%); 5776/307992 objects misplaced (1.875%); 759 KiB/s, 11 objects/s recovering 2024-08-17T17:31:48.116 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:48.161 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:48.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:48.263 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:48.305 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:48.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:48.407 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:48.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.458 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:48.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:48.559 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:48.590 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:48.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:48.692 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:48.724 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:48.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:48.760 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:48.760 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:48.795 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:31:48.825 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:48.854 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:48.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:48.926 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:48 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 45102/307992 objects degraded (14.644%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:48.926 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:48 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1816153775' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:48.927 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:48 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:48.927 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:48 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 45102/307992 objects degraded (14.644%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:48.927 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:48 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1816153775' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:48.927 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:48 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:48.955 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:48.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:48.991 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:48.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:49.050 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 64 16 110167 110151 107.543 61.1875 0.0116526 0.00929398 2024-08-17T17:31:49.081 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:31:49.083 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:31:49.091 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:48 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 45102/307992 objects degraded (14.644%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:48 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1816153775' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:48 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:31:49.135 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:49.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.137 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:49.238 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:49.267 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:49.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:49.369 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:49.407 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:49.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:49.509 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:49.545 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:49.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:49.647 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:49.678 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:49.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:49.779 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:49.808 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:49.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:49.909 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:49.947 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:49.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:49.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:49 smithi063 ceph-mon[114756]: pgmap v1899: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 47254/324219 objects degraded (14.575%); 5682/324219 objects misplaced (1.753%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:31:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:49 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:49 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:31:50.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:49 smithi063 ceph-mon[114756]: osdmap e1303: 8 total, 7 up, 7 in 2024-08-17T17:31:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:49 smithi063 ceph-mon[109843]: pgmap v1899: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 47254/324219 objects degraded (14.575%); 5682/324219 objects misplaced (1.753%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:31:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:49 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:49 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:31:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:49 smithi063 ceph-mon[109843]: osdmap e1303: 8 total, 7 up, 7 in 2024-08-17T17:31:50.048 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:50.050 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 65 16 111315 111299 106.992 71.75 0.0115264 0.00934165 2024-08-17T17:31:50.081 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:50.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:49 smithi162 ceph-mon[104668]: pgmap v1899: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 66 MiB/s wr, 1.05k op/s; 47254/324219 objects degraded (14.575%); 5682/324219 objects misplaced (1.753%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:31:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:49 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:31:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:49 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:31:50.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:49 smithi162 ceph-mon[104668]: osdmap e1303: 8 total, 7 up, 7 in 2024-08-17T17:31:50.183 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:50.214 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:50.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:50.316 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:50.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.359 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:50.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:50.460 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:50.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.493 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:50.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:50.593 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:50.621 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:50.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:50.723 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:50.754 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:50.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:50.856 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:50.891 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:50.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:50.993 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:51.026 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:51.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:51.050 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 66 16 112472 112456 106.467 72.3125 0.00692457 0.00938725 2024-08-17T17:31:51.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:50 smithi162 ceph-mon[104668]: from='client.55477 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:31:51.128 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:51.163 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:51.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:51.266 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:51.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:50 smithi063 ceph-mon[114756]: from='client.55477 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:31:51.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:50 smithi063 ceph-mon[109843]: from='client.55477 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:31:51.304 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:51.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:51.406 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:51.429 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1900,"stamp":"2024-08-17T17:31:48.793521+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7081604385,"num_objects":108073,"num_object_clones":0,"num_object_copies":324219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47254,"num_objects_misplaced":5682,"num_objects_unfound":0,"num_objects_dirty":108073,"num_whiteouts":0,"num_read":26,"num_read_kb":56,"num_write":108125,"num_write_kb":6915206,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":230,"num_bytes_recovered":13897999,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101362,"ondisk_log_size":101362,"up":303,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":351,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":22686496,"kb_used_data":17984104,"kb_used_omap":150,"kb_used_meta":4702185,"kb_avail":633529568,"statfs":{"total":671965249536,"available":648734277632,"internally_reserved":0,"allocated":18415722496,"data_stored":18395878530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":154546,"internal_metadata":4815037518},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,5,3,5,7,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":746389504,"num_objects":11389,"num_object_clones":0,"num_object_copies":34167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4933,"num_objects_misplaced":-189,"num_objects_unfound":0,"num_objects_dirty":11389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11389,"num_write_kb":728896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":189,"num_bytes_recovered":12386304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.309838"},"pg_stats":[{"pgid":"1.11","version":"369'149","reported_seq":1627,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.708840+0000","last_change":"2024-08-17T17:24:49.721626+0000","last_active":"2024-08-17T17:31:36.708840+0000","last_peered":"2024-08-17T17:31:36.708840+0000","last_clean":"2024-08-17T17:31:36.708840+0000","last_became_active":"2024-08-17T17:24:20.649128+0000","last_became_peered":"2024-08-17T17:24:20.649128+0000","last_unstale":"2024-08-17T17:31:36.708840+0000","last_undegraded":"2024-08-17T17:31:36.708840+0000","last_fullsized":"2024-08-17T17:31:36.708840+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:03:15.343059+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1681,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.715179+0000","last_change":"2024-08-17T17:30:27.480287+0000","last_active":"2024-08-17T17:31:36.715179+0000","last_peered":"2024-08-17T17:31:36.715179+0000","last_clean":"2024-08-17T17:31:36.715179+0000","last_became_active":"2024-08-17T17:30:27.480010+0000","last_became_peered":"2024-08-17T17:30:27.480010+0000","last_unstale":"2024-08-17T17:31:36.715179+0000","last_undegraded":"2024-08-17T17:31:36.715179+0000","last_fullsized":"2024-08-17T17:31:36.715179+0000","mapping_epoch":1285,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:40:33.880970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1234,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.716035+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:31:36.716035+0000","last_peered":"2024-08-17T17:31:36.716035+0000","last_clean":"2024-08-17T17:31:36.716035+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:31:36.716035+0000","last_undegraded":"2024-08-17T17:31:36.716035+0000","last_fullsized":"2024-08-17T17:31:36.716035+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1564,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.708235+0000","last_change":"2024-08-17T17:29:53.529377+0000","last_active":"2024-08-17T17:31:36.708235+0000","last_peered":"2024-08-17T17:31:36.708235+0000","last_clean":"2024-08-17T17:29:38.563824+0000","last_became_active":"2024-08-17T17:29:53.529377+0000","last_became_peered":"2024-08-17T17:29:53.529377+0000","last_unstale":"2024-08-17T17:31:36.708235+0000","last_undegraded":"2024-08-17T17:31:36.708235+0000","last_fullsized":"2024-08-17T17:29:53.524218+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:52.315542+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1245,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.709567+0000","last_change":"2024-08-17T17:07:45.680174+0000","last_active":"2024-08-17T17:31:36.709567+0000","last_peered":"2024-08-17T17:31:36.709567+0000","last_clean":"2024-08-17T17:31:36.709567+0000","last_became_active":"2024-08-17T17:07:45.679852+0000","last_became_peered":"2024-08-17T17:07:45.679852+0000","last_unstale":"2024-08-17T17:31:36.709567+0000","last_undegraded":"2024-08-17T17:31:36.709567+0000","last_fullsized":"2024-08-17T17:31:36.709567+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:12.728052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1101,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.713585+0000","last_change":"2024-08-17T17:18:31.537952+0000","last_active":"2024-08-17T17:31:36.713585+0000","last_peered":"2024-08-17T17:31:36.713585+0000","last_clean":"2024-08-17T17:31:36.713585+0000","last_became_active":"2024-08-17T17:18:31.536006+0000","last_became_peered":"2024-08-17T17:18:31.536006+0000","last_unstale":"2024-08-17T17:31:36.713585+0000","last_undegraded":"2024-08-17T17:31:36.713585+0000","last_fullsized":"2024-08-17T17:31:36.713585+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:05:23.181305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":937,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.725479+0000","last_change":"2024-08-17T17:31:37.725404+0000","last_active":"2024-08-17T17:31:37.725479+0000","last_peered":"2024-08-17T17:31:37.725479+0000","last_clean":"2024-08-17T17:31:37.725479+0000","last_became_active":"2024-08-17T17:31:37.723892+0000","last_became_peered":"2024-08-17T17:31:37.723892+0000","last_unstale":"2024-08-17T17:31:37.725479+0000","last_undegraded":"2024-08-17T17:31:37.725479+0000","last_fullsized":"2024-08-17T17:31:37.725479+0000","mapping_epoch":1301,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1302,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:26:03.186537+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1254,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.717098+0000","last_change":"2024-08-17T17:18:31.543764+0000","last_active":"2024-08-17T17:31:37.717098+0000","last_peered":"2024-08-17T17:31:37.717098+0000","last_clean":"2024-08-17T17:31:37.717098+0000","last_became_active":"2024-08-17T17:18:31.543186+0000","last_became_peered":"2024-08-17T17:18:31.543186+0000","last_unstale":"2024-08-17T17:31:37.717098+0000","last_undegraded":"2024-08-17T17:31:37.717098+0000","last_fullsized":"2024-08-17T17:31:37.717098+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:20.521126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1223,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.716512+0000","last_change":"2024-08-17T17:29:53.532808+0000","last_active":"2024-08-17T17:31:36.716512+0000","last_peered":"2024-08-17T17:31:36.716512+0000","last_clean":"2024-08-17T17:29:38.568865+0000","last_became_active":"2024-08-17T17:29:53.532808+0000","last_became_peered":"2024-08-17T17:29:53.532808+0000","last_unstale":"2024-08-17T17:31:36.716512+0000","last_undegraded":"2024-08-17T17:31:36.716512+0000","last_fullsized":"2024-08-17T17:29:53.525701+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:13:38.674828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1586,"reported_epoch":1301,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:36.708806+0000","last_change":"2024-08-17T17:30:19.041192+0000","last_active":"2024-08-17T17:31:36.708806+0000","last_peered":"2024-08-17T17:31:36.708806+0000","last_clean":"2024-08-17T17:31:36.708806+0000","last_became_active":"2024-08-17T17:30:19.040728+0000","last_became_peered":"2024-08-17T17:30:19.040728+0000","last_unstale":"2024-08-17T17:31:36.708806+0000","last_undegraded":"2024-08-17T17:31:36.708806+0000","last_fullsized":"2024-08-17T17:31:36.708806+0000","mapping_epoch":1281,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1282,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:31:11.985066+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7,1],"avail_no_missing":["5","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1709,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.710714+0000","last_change":"2024-08-17T17:30:16.752326+0000","last_active":"2024-08-17T17:31:37.710714+0000","last_peered":"2024-08-17T17:31:37.710714+0000","last_clean":"2024-08-17T17:31:37.710714+0000","last_became_active":"2024-08-17T17:30:16.751906+0000","last_became_peered":"2024-08-17T17:30:16.751906+0000","last_unstale":"2024-08-17T17:31:37.710714+0000","last_undegraded":"2024-08-17T17:31:37.710714+0000","last_fullsized":"2024-08-17T17:31:37.710714+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:30:16.625419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":48,"reported_epoch":1301,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:31:36.708438+0000","last_change":"2024-08-17T17:29:53.530420+0000","last_active":"2024-08-17T17:31:36.708438+0000","last_peered":"2024-08-17T17:31:36.708438+0000","last_clean":"2024-08-17T16:53:38.920551+0000","last_became_active":"2024-08-17T17:29:53.530420+0000","last_became_peered":"2024-08-17T17:29:53.530420+0000","last_unstale":"2024-08-17T17:31:36.708438+0000","last_undegraded":"2024-08-17T17:29:53.525718+0000","last_fullsized":"2024-08-17T17:29:53.525585+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:16:06.719747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":4}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.6","version":"1302'6820","reported_seq":6842,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:47.022377+0000","last_change":"2024-08-17T17:30:31.048590+0000","last_active":"2024-08-17T17:31:47.022377+0000","last_peered":"2024-08-17T17:31:47.022377+0000","last_clean":"2024-08-17T17:31:47.022377+0000","last_became_active":"2024-08-17T17:30:31.048314+0000","last_became_peered":"2024-08-17T17:30:31.048314+0000","last_unstale":"2024-08-17T17:31:47.022377+0000","last_undegraded":"2024-08-17T17:31:47.022377+0000","last_fullsized":"2024-08-17T17:31:47.022377+0000","mapping_epoch":1288,"log_start":"1292'200","ondisk_log_start":"1292'200","created":1288,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6620,"log_dups_size":0,"ondisk_log_size":6620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:05:50.558381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":446955520,"num_objects":6820,"num_object_clones":0,"num_object_copies":20460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6820,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6820,"num_write_kb":436480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1518,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.713964+0000","last_change":"2024-08-17T17:30:19.090966+0000","last_active":"2024-08-17T17:31:36.713964+0000","last_peered":"2024-08-17T17:31:36.713964+0000","last_clean":"2024-08-17T17:31:36.713964+0000","last_became_active":"2024-08-17T17:30:19.050098+0000","last_became_peered":"2024-08-17T17:30:19.050098+0000","last_unstale":"2024-08-17T17:31:36.713964+0000","last_undegraded":"2024-08-17T17:31:36.713964+0000","last_fullsized":"2024-08-17T17:31:36.713964+0000","mapping_epoch":1281,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1282,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.410491+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.410491+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:23:16.295685+0000","scrub_duration":0.0066315480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1259,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.715521+0000","last_change":"2024-08-17T17:18:31.555799+0000","last_active":"2024-08-17T17:31:36.715521+0000","last_peered":"2024-08-17T17:31:36.715521+0000","last_clean":"2024-08-17T17:31:36.715521+0000","last_became_active":"2024-08-17T17:18:31.539713+0000","last_became_peered":"2024-08-17T17:18:31.539713+0000","last_unstale":"2024-08-17T17:31:36.715521+0000","last_undegraded":"2024-08-17T17:31:36.715521+0000","last_fullsized":"2024-08-17T17:31:36.715521+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:04:19.285010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1238,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.708643+0000","last_change":"2024-08-17T17:29:53.529490+0000","last_active":"2024-08-17T17:31:36.708643+0000","last_peered":"2024-08-17T17:31:36.708643+0000","last_clean":"2024-08-17T17:29:38.563813+0000","last_became_active":"2024-08-17T17:29:53.529490+0000","last_became_peered":"2024-08-17T17:29:53.529490+0000","last_unstale":"2024-08-17T17:31:36.708643+0000","last_undegraded":"2024-08-17T17:31:36.708643+0000","last_fullsized":"2024-08-17T17:29:53.525273+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:35.412627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"1302'6689","reported_seq":6707,"reported_epoch":1302,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:31:47.034669+0000","last_change":"2024-08-17T17:30:45.041965+0000","last_active":"2024-08-17T17:31:47.034669+0000","last_peered":"2024-08-17T17:31:47.034669+0000","last_clean":"2024-08-17T17:30:30.026678+0000","last_became_active":"2024-08-17T17:30:31.048834+0000","last_became_peered":"2024-08-17T17:30:31.048834+0000","last_unstale":"2024-08-17T17:31:47.034669+0000","last_undegraded":"2024-08-17T17:30:45.041131+0000","last_fullsized":"2024-08-17T17:30:31.045171+0000","mapping_epoch":1288,"log_start":"1292'100","ondisk_log_start":"1292'100","created":1288,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6589,"log_dups_size":0,"ondisk_log_size":6589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:52.286672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":438370304,"num_objects":6689,"num_object_clones":0,"num_object_copies":20067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6689,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6689,"num_write_kb":428096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":6689}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1789,"reported_epoch":1302,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:31:37.712697+0000","last_change":"2024-08-17T17:29:53.528197+0000","last_active":"2024-08-17T17:31:37.712697+0000","last_peered":"2024-08-17T17:31:37.712697+0000","last_clean":"2024-08-17T17:29:38.576329+0000","last_became_active":"2024-08-17T17:29:53.528197+0000","last_became_peered":"2024-08-17T17:29:53.528197+0000","last_unstale":"2024-08-17T17:31:37.712697+0000","last_undegraded":"2024-08-17T17:29:53.523689+0000","last_fullsized":"2024-08-17T17:29:53.523571+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:45.985596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:45.985596+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:58:34.889041+0000","scrub_duration":10,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1273,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.715012+0000","last_change":"2024-08-17T17:30:16.762210+0000","last_active":"2024-08-17T17:31:36.715012+0000","last_peered":"2024-08-17T17:31:36.715012+0000","last_clean":"2024-08-17T17:31:36.715012+0000","last_became_active":"2024-08-17T17:30:16.753615+0000","last_became_peered":"2024-08-17T17:30:16.753615+0000","last_unstale":"2024-08-17T17:31:36.715012+0000","last_undegraded":"2024-08-17T17:31:36.715012+0000","last_fullsized":"2024-08-17T17:31:36.715012+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:14:51.581788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1294,"reported_epoch":1301,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:36.716591+0000","last_change":"2024-08-17T17:30:17.751104+0000","last_active":"2024-08-17T17:31:36.716591+0000","last_peered":"2024-08-17T17:31:36.716591+0000","last_clean":"2024-08-17T17:31:36.716591+0000","last_became_active":"2024-08-17T17:30:17.750695+0000","last_became_peered":"2024-08-17T17:30:17.750695+0000","last_unstale":"2024-08-17T17:31:36.716591+0000","last_undegraded":"2024-08-17T17:31:36.716591+0000","last_fullsized":"2024-08-17T17:31:36.716591+0000","mapping_epoch":1279,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:38.049076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,1],"avail_no_missing":["4","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"1302'6837","reported_seq":6858,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:46.974109+0000","last_change":"2024-08-17T17:30:31.047987+0000","last_active":"2024-08-17T17:31:46.974109+0000","last_peered":"2024-08-17T17:31:46.974109+0000","last_clean":"2024-08-17T17:31:46.974109+0000","last_became_active":"2024-08-17T17:30:31.047644+0000","last_became_peered":"2024-08-17T17:30:31.047644+0000","last_unstale":"2024-08-17T17:31:46.974109+0000","last_undegraded":"2024-08-17T17:31:46.974109+0000","last_fullsized":"2024-08-17T17:31:46.974109+0000","mapping_epoch":1288,"log_start":"1292'1199","ondisk_log_start":"1292'1199","created":1288,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":5638,"log_dups_size":0,"ondisk_log_size":5638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:22:30.407991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":448069632,"num_objects":6837,"num_object_clones":0,"num_object_copies":20511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6837,"num_write_kb":437568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":791,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.710296+0000","last_change":"2024-08-17T17:29:53.531472+0000","last_active":"2024-08-17T17:31:36.710296+0000","last_peered":"2024-08-17T17:31:36.710296+0000","last_clean":"2024-08-17T17:24:12.583154+0000","last_became_active":"2024-08-17T17:29:53.531472+0000","last_became_peered":"2024-08-17T17:29:53.531472+0000","last_unstale":"2024-08-17T17:31:36.710296+0000","last_undegraded":"2024-08-17T17:31:36.710296+0000","last_fullsized":"2024-08-17T17:29:53.527758+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1224,"reported_epoch":1301,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:31:36.714216+0000","last_change":"2024-08-17T17:29:53.534444+0000","last_active":"2024-08-17T17:31:36.714216+0000","last_peered":"2024-08-17T17:31:36.714216+0000","last_clean":"2024-08-17T17:29:38.569012+0000","last_became_active":"2024-08-17T17:29:53.534444+0000","last_became_peered":"2024-08-17T17:29:53.534444+0000","last_unstale":"2024-08-17T17:31:36.714216+0000","last_undegraded":"2024-08-17T17:29:53.527480+0000","last_fullsized":"2024-08-17T17:29:53.527380+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:20:42.295839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1170,"reported_epoch":1301,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:36.713666+0000","last_change":"2024-08-17T17:30:17.779046+0000","last_active":"2024-08-17T17:31:36.713666+0000","last_peered":"2024-08-17T17:31:36.713666+0000","last_clean":"2024-08-17T17:31:36.713666+0000","last_became_active":"2024-08-17T17:30:17.778674+0000","last_became_peered":"2024-08-17T17:30:17.778674+0000","last_unstale":"2024-08-17T17:31:36.713666+0000","last_undegraded":"2024-08-17T17:31:36.713666+0000","last_fullsized":"2024-08-17T17:31:36.713666+0000","mapping_epoch":1279,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:49:44.282833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5,1],"avail_no_missing":["7","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.5","version":"1302'6803","reported_seq":6842,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:47.207307+0000","last_change":"2024-08-17T17:30:31.046937+0000","last_active":"2024-08-17T17:31:47.207307+0000","last_peered":"2024-08-17T17:31:47.207307+0000","last_clean":"2024-08-17T17:31:47.207307+0000","last_became_active":"2024-08-17T17:30:31.046647+0000","last_became_peered":"2024-08-17T17:30:31.046647+0000","last_unstale":"2024-08-17T17:31:47.207307+0000","last_undegraded":"2024-08-17T17:31:47.207307+0000","last_fullsized":"2024-08-17T17:31:47.207307+0000","mapping_epoch":1288,"log_start":"0'0","ondisk_log_start":"0'0","created":1288,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6803,"log_dups_size":0,"ondisk_log_size":6803,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:03.220444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":445841408,"num_objects":6803,"num_object_clones":0,"num_object_copies":20409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6803,"num_write_kb":435392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":777,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.715416+0000","last_change":"2024-08-17T17:24:52.102415+0000","last_active":"2024-08-17T17:31:36.715416+0000","last_peered":"2024-08-17T17:31:36.715416+0000","last_clean":"2024-08-17T17:31:36.715416+0000","last_became_active":"2024-08-17T17:24:31.795843+0000","last_became_peered":"2024-08-17T17:24:31.795843+0000","last_unstale":"2024-08-17T17:31:36.715416+0000","last_undegraded":"2024-08-17T17:31:36.715416+0000","last_fullsized":"2024-08-17T17:31:36.715416+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:32:08.434197+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1253,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.709154+0000","last_change":"2024-08-17T17:29:53.531726+0000","last_active":"2024-08-17T17:31:36.709154+0000","last_peered":"2024-08-17T17:31:36.709154+0000","last_clean":"2024-08-17T17:29:38.563933+0000","last_became_active":"2024-08-17T17:29:53.531726+0000","last_became_peered":"2024-08-17T17:29:53.531726+0000","last_unstale":"2024-08-17T17:31:36.709154+0000","last_undegraded":"2024-08-17T17:31:36.709154+0000","last_fullsized":"2024-08-17T17:29:53.525159+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:01:38.841719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1038,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.715760+0000","last_change":"2024-08-17T17:30:16.745698+0000","last_active":"2024-08-17T17:31:36.715760+0000","last_peered":"2024-08-17T17:31:36.715760+0000","last_clean":"2024-08-17T17:31:36.715760+0000","last_became_active":"2024-08-17T17:30:16.745339+0000","last_became_peered":"2024-08-17T17:30:16.745339+0000","last_unstale":"2024-08-17T17:31:36.715760+0000","last_undegraded":"2024-08-17T17:31:36.715760+0000","last_fullsized":"2024-08-17T17:31:36.715760+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:32.351164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.2","version":"1302'6808","reported_seq":6831,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:46.959432+0000","last_change":"2024-08-17T17:30:31.049978+0000","last_active":"2024-08-17T17:31:46.959432+0000","last_peered":"2024-08-17T17:31:46.959432+0000","last_clean":"2024-08-17T17:31:46.959432+0000","last_became_active":"2024-08-17T17:30:31.049740+0000","last_became_peered":"2024-08-17T17:30:31.049740+0000","last_unstale":"2024-08-17T17:31:46.959432+0000","last_undegraded":"2024-08-17T17:31:46.959432+0000","last_fullsized":"2024-08-17T17:31:46.959432+0000","mapping_epoch":1288,"log_start":"1292'1246","ondisk_log_start":"1292'1246","created":1288,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":5562,"log_dups_size":0,"ondisk_log_size":5562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:18:12.681396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":446169088,"num_objects":6808,"num_object_clones":0,"num_object_copies":20424,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6808,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6808,"num_write_kb":435712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1648,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.716268+0000","last_change":"2024-08-17T17:24:54.418765+0000","last_active":"2024-08-17T17:31:36.716268+0000","last_peered":"2024-08-17T17:31:36.716268+0000","last_clean":"2024-08-17T17:31:36.716268+0000","last_became_active":"2024-08-17T17:24:31.800263+0000","last_became_peered":"2024-08-17T17:24:31.800263+0000","last_unstale":"2024-08-17T17:31:36.716268+0000","last_undegraded":"2024-08-17T17:31:36.716268+0000","last_fullsized":"2024-08-17T17:31:36.716268+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:54.418644+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:54.418644+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:56:11.024294+0000","scrub_duration":0.013346135,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":40,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.715840+0000","last_change":"2024-08-17T17:30:27.481673+0000","last_active":"2024-08-17T17:31:36.715840+0000","last_peered":"2024-08-17T17:31:36.715840+0000","last_clean":"2024-08-17T17:31:36.715840+0000","last_became_active":"2024-08-17T17:30:27.481211+0000","last_became_peered":"2024-08-17T17:30:27.481211+0000","last_unstale":"2024-08-17T17:31:36.715840+0000","last_undegraded":"2024-08-17T17:31:36.715840+0000","last_fullsized":"2024-08-17T17:31:36.715840+0000","mapping_epoch":1285,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:37:06.870290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1256,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.714810+0000","last_change":"2024-08-17T17:29:53.537940+0000","last_active":"2024-08-17T17:31:36.714810+0000","last_peered":"2024-08-17T17:31:36.714810+0000","last_clean":"2024-08-17T17:29:42.740989+0000","last_became_active":"2024-08-17T17:29:53.537940+0000","last_became_peered":"2024-08-17T17:29:53.537940+0000","last_unstale":"2024-08-17T17:31:36.714810+0000","last_undegraded":"2024-08-17T17:31:36.714810+0000","last_fullsized":"2024-08-17T17:29:53.527066+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:49.456097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"1302'6749","reported_seq":6788,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:47.195616+0000","last_change":"2024-08-17T17:30:31.048657+0000","last_active":"2024-08-17T17:31:47.195616+0000","last_peered":"2024-08-17T17:31:47.195616+0000","last_clean":"2024-08-17T17:31:47.195616+0000","last_became_active":"2024-08-17T17:30:31.048345+0000","last_became_peered":"2024-08-17T17:30:31.048345+0000","last_unstale":"2024-08-17T17:31:47.195616+0000","last_undegraded":"2024-08-17T17:31:47.195616+0000","last_fullsized":"2024-08-17T17:31:47.195616+0000","mapping_epoch":1288,"log_start":"0'0","ondisk_log_start":"0'0","created":1288,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6749,"log_dups_size":0,"ondisk_log_size":6749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:49:09.069771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":442302464,"num_objects":6749,"num_object_clones":0,"num_object_copies":20247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6749,"num_write_kb":431936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1594,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.708346+0000","last_change":"2024-08-17T17:24:58.712331+0000","last_active":"2024-08-17T17:31:36.708346+0000","last_peered":"2024-08-17T17:31:36.708346+0000","last_clean":"2024-08-17T17:31:36.708346+0000","last_became_active":"2024-08-17T17:24:31.802584+0000","last_became_peered":"2024-08-17T17:24:31.802584+0000","last_unstale":"2024-08-17T17:31:36.708346+0000","last_undegraded":"2024-08-17T17:31:36.708346+0000","last_fullsized":"2024-08-17T17:31:36.708346+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:04:59.616217+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1171,"reported_epoch":1301,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:36.713888+0000","last_change":"2024-08-17T17:30:17.761479+0000","last_active":"2024-08-17T17:31:36.713888+0000","last_peered":"2024-08-17T17:31:36.713888+0000","last_clean":"2024-08-17T17:31:36.713888+0000","last_became_active":"2024-08-17T17:30:17.761084+0000","last_became_peered":"2024-08-17T17:30:17.761084+0000","last_unstale":"2024-08-17T17:31:36.713888+0000","last_undegraded":"2024-08-17T17:31:36.713888+0000","last_fullsized":"2024-08-17T17:31:36.713888+0000","mapping_epoch":1279,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:15:48.571230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,1],"avail_no_missing":["4","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1279,"reported_epoch":1302,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:37.716413+0000","last_change":"2024-08-17T17:31:35.880844+0000","last_active":"2024-08-17T17:31:37.716413+0000","last_peered":"2024-08-17T17:31:37.716413+0000","last_clean":"2024-08-17T17:31:37.716413+0000","last_became_active":"2024-08-17T17:31:35.880326+0000","last_became_peered":"2024-08-17T17:31:35.880326+0000","last_unstale":"2024-08-17T17:31:37.716413+0000","last_undegraded":"2024-08-17T17:31:37.716413+0000","last_fullsized":"2024-08-17T17:31:37.716413+0000","mapping_epoch":1299,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:29:15.775330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0,4],"avail_no_missing":["2","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"1302'6766","reported_seq":6786,"reported_epoch":1302,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:31:46.983185+0000","last_change":"2024-08-17T17:30:45.041100+0000","last_active":"2024-08-17T17:31:46.983185+0000","last_peered":"2024-08-17T17:31:46.983185+0000","last_clean":"2024-08-17T17:30:30.026678+0000","last_became_active":"2024-08-17T17:30:31.046734+0000","last_became_peered":"2024-08-17T17:30:31.046734+0000","last_unstale":"2024-08-17T17:31:46.983185+0000","last_undegraded":"2024-08-17T17:30:45.040211+0000","last_fullsized":"2024-08-17T17:30:31.041942+0000","mapping_epoch":1288,"log_start":"1292'1148","ondisk_log_start":"1292'1148","created":1288,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":5618,"log_dups_size":0,"ondisk_log_size":5618,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:24:18.619486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":443416576,"num_objects":6766,"num_object_clones":0,"num_object_copies":20298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6766,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6766,"num_write_kb":433024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":6766}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":843,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.710944+0000","last_change":"2024-08-17T17:30:19.042789+0000","last_active":"2024-08-17T17:31:36.710944+0000","last_peered":"2024-08-17T17:31:36.710944+0000","last_clean":"2024-08-17T17:31:36.710944+0000","last_became_active":"2024-08-17T17:30:19.041286+0000","last_became_peered":"2024-08-17T17:30:19.041286+0000","last_unstale":"2024-08-17T17:31:36.710944+0000","last_undegraded":"2024-08-17T17:31:36.710944+0000","last_fullsized":"2024-08-17T17:31:36.710944+0000","mapping_epoch":1281,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1282,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:57.682409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:57.682409+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:23.528243+0000","scrub_duration":0.0066879779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1252,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.716704+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:31:36.716704+0000","last_peered":"2024-08-17T17:31:36.716704+0000","last_clean":"2024-08-17T17:31:36.716704+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:31:36.716704+0000","last_undegraded":"2024-08-17T17:31:36.716704+0000","last_fullsized":"2024-08-17T17:31:36.716704+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1284,"reported_epoch":1301,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:36.715584+0000","last_change":"2024-08-17T17:30:17.755538+0000","last_active":"2024-08-17T17:31:36.715584+0000","last_peered":"2024-08-17T17:31:36.715584+0000","last_clean":"2024-08-17T17:31:36.715584+0000","last_became_active":"2024-08-17T17:30:17.754295+0000","last_became_peered":"2024-08-17T17:30:17.754295+0000","last_unstale":"2024-08-17T17:31:36.715584+0000","last_undegraded":"2024-08-17T17:31:36.715584+0000","last_fullsized":"2024-08-17T17:31:36.715584+0000","mapping_epoch":1279,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:12:41.792642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,1],"avail_no_missing":["4","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"1302'6685","reported_seq":6707,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:46.980121+0000","last_change":"2024-08-17T17:30:31.049101+0000","last_active":"2024-08-17T17:31:46.980121+0000","last_peered":"2024-08-17T17:31:46.980121+0000","last_clean":"2024-08-17T17:31:46.980121+0000","last_became_active":"2024-08-17T17:30:31.048798+0000","last_became_peered":"2024-08-17T17:30:31.048798+0000","last_unstale":"2024-08-17T17:31:46.980121+0000","last_undegraded":"2024-08-17T17:31:46.980121+0000","last_fullsized":"2024-08-17T17:31:46.980121+0000","mapping_epoch":1288,"log_start":"1292'1039","ondisk_log_start":"1292'1039","created":1288,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":5646,"log_dups_size":0,"ondisk_log_size":5646,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:45:08.180351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":438108160,"num_objects":6685,"num_object_clones":0,"num_object_copies":20055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6685,"num_write_kb":427840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1600,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:37.000289+0000","last_change":"2024-08-17T17:29:53.533887+0000","last_active":"2024-08-17T17:31:37.000289+0000","last_peered":"2024-08-17T17:31:37.000289+0000","last_clean":"2024-08-17T17:29:38.563668+0000","last_became_active":"2024-08-17T17:29:53.533887+0000","last_became_peered":"2024-08-17T17:29:53.533887+0000","last_unstale":"2024-08-17T17:31:37.000289+0000","last_undegraded":"2024-08-17T17:31:37.000289+0000","last_fullsized":"2024-08-17T17:29:53.527142+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:59:52.156417+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1713,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.710983+0000","last_change":"2024-08-17T17:30:16.751744+0000","last_active":"2024-08-17T17:31:37.710983+0000","last_peered":"2024-08-17T17:31:37.710983+0000","last_clean":"2024-08-17T17:31:37.710983+0000","last_became_active":"2024-08-17T17:30:16.751311+0000","last_became_peered":"2024-08-17T17:30:16.751311+0000","last_unstale":"2024-08-17T17:31:37.710983+0000","last_undegraded":"2024-08-17T17:31:37.710983+0000","last_fullsized":"2024-08-17T17:31:37.710983+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:45.925275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":891,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.713187+0000","last_change":"2024-08-17T17:30:27.482317+0000","last_active":"2024-08-17T17:31:36.713187+0000","last_peered":"2024-08-17T17:31:36.713187+0000","last_clean":"2024-08-17T17:31:36.713187+0000","last_became_active":"2024-08-17T17:30:27.482045+0000","last_became_peered":"2024-08-17T17:30:27.482045+0000","last_unstale":"2024-08-17T17:31:36.713187+0000","last_undegraded":"2024-08-17T17:31:36.713187+0000","last_fullsized":"2024-08-17T17:31:36.713187+0000","mapping_epoch":1285,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:57:54.583008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.b","version":"1302'6802","reported_seq":6822,"reported_epoch":1302,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:31:46.992053+0000","last_change":"2024-08-17T17:30:45.048593+0000","last_active":"2024-08-17T17:31:46.992053+0000","last_peered":"2024-08-17T17:31:46.992053+0000","last_clean":"2024-08-17T17:30:30.026678+0000","last_became_active":"2024-08-17T17:30:31.048664+0000","last_became_peered":"2024-08-17T17:30:31.048664+0000","last_unstale":"2024-08-17T17:31:46.992053+0000","last_undegraded":"2024-08-17T17:30:45.045686+0000","last_fullsized":"2024-08-17T17:30:31.043605+0000","mapping_epoch":1288,"log_start":"1292'1193","ondisk_log_start":"1292'1193","created":1288,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":5609,"log_dups_size":0,"ondisk_log_size":5609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:06:36.830681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":445775872,"num_objects":6802,"num_object_clones":0,"num_object_copies":20406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6802,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6802,"num_write_kb":435328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":6802}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1654,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.717803+0000","last_change":"2024-08-17T17:29:53.531198+0000","last_active":"2024-08-17T17:31:36.717803+0000","last_peered":"2024-08-17T17:31:36.717803+0000","last_clean":"2024-08-17T17:29:38.568060+0000","last_became_active":"2024-08-17T17:29:53.531198+0000","last_became_peered":"2024-08-17T17:29:53.531198+0000","last_unstale":"2024-08-17T17:31:36.717803+0000","last_undegraded":"2024-08-17T17:31:36.717803+0000","last_fullsized":"2024-08-17T17:29:53.525184+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:00:25.966007+0000","scrub_duration":0.017597167,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1262,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.711086+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:31:36.711086+0000","last_peered":"2024-08-17T17:31:36.711086+0000","last_clean":"2024-08-17T17:31:36.711086+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:31:36.711086+0000","last_undegraded":"2024-08-17T17:31:36.711086+0000","last_fullsized":"2024-08-17T17:31:36.711086+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1638,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.710225+0000","last_change":"2024-08-17T17:30:27.482181+0000","last_active":"2024-08-17T17:31:36.710225+0000","last_peered":"2024-08-17T17:31:36.710225+0000","last_clean":"2024-08-17T17:29:38.564389+0000","last_became_active":"2024-08-17T17:30:27.482181+0000","last_became_peered":"2024-08-17T17:30:27.482181+0000","last_unstale":"2024-08-17T17:31:36.710225+0000","last_undegraded":"2024-08-17T17:31:36.710225+0000","last_fullsized":"2024-08-17T17:30:27.477725+0000","mapping_epoch":1285,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:43:23.393244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":36,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.713152+0000","last_change":"2024-08-17T17:29:53.530094+0000","last_active":"2024-08-17T17:31:36.713152+0000","last_peered":"2024-08-17T17:31:36.713152+0000","last_clean":"2024-08-17T16:53:36.856414+0000","last_became_active":"2024-08-17T17:29:53.530094+0000","last_became_peered":"2024-08-17T17:29:53.530094+0000","last_unstale":"2024-08-17T17:31:36.713152+0000","last_undegraded":"2024-08-17T17:31:36.713152+0000","last_fullsized":"2024-08-17T17:29:53.525569+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.793336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"1302'6702","reported_seq":6741,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:47.171188+0000","last_change":"2024-08-17T17:30:31.046435+0000","last_active":"2024-08-17T17:31:47.171188+0000","last_peered":"2024-08-17T17:31:47.171188+0000","last_clean":"2024-08-17T17:31:47.171188+0000","last_became_active":"2024-08-17T17:30:31.046121+0000","last_became_peered":"2024-08-17T17:30:31.046121+0000","last_unstale":"2024-08-17T17:31:47.171188+0000","last_undegraded":"2024-08-17T17:31:47.171188+0000","last_fullsized":"2024-08-17T17:31:47.171188+0000","mapping_epoch":1288,"log_start":"0'0","ondisk_log_start":"0'0","created":1288,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6702,"log_dups_size":0,"ondisk_log_size":6702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:08:15.648250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439222272,"num_objects":6702,"num_object_clones":0,"num_object_copies":20106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6702,"num_write_kb":428928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":700,"reported_epoch":1301,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:36.715643+0000","last_change":"2024-08-17T17:30:20.754461+0000","last_active":"2024-08-17T17:31:36.715643+0000","last_peered":"2024-08-17T17:31:36.715643+0000","last_clean":"2024-08-17T17:31:36.715643+0000","last_became_active":"2024-08-17T17:30:19.042543+0000","last_became_peered":"2024-08-17T17:30:19.042543+0000","last_unstale":"2024-08-17T17:31:36.715643+0000","last_undegraded":"2024-08-17T17:31:36.715643+0000","last_fullsized":"2024-08-17T17:31:36.715643+0000","mapping_epoch":1281,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1282,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:20.754326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:20.754326+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:19:23.993964+0000","scrub_duration":0.0023303870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,1],"avail_no_missing":["4","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1307,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.717639+0000","last_change":"2024-08-17T17:30:16.753763+0000","last_active":"2024-08-17T17:31:36.717639+0000","last_peered":"2024-08-17T17:31:36.717639+0000","last_clean":"2024-08-17T17:31:36.717639+0000","last_became_active":"2024-08-17T17:30:16.753456+0000","last_became_peered":"2024-08-17T17:30:16.753456+0000","last_unstale":"2024-08-17T17:31:36.717639+0000","last_undegraded":"2024-08-17T17:31:36.717639+0000","last_fullsized":"2024-08-17T17:31:36.717639+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:10:40.618605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":953,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.710546+0000","last_change":"2024-08-17T17:31:18.406597+0000","last_active":"2024-08-17T17:31:37.710546+0000","last_peered":"2024-08-17T17:31:37.710546+0000","last_clean":"2024-08-17T17:31:37.710546+0000","last_became_active":"2024-08-17T17:31:18.406248+0000","last_became_peered":"2024-08-17T17:31:18.406248+0000","last_unstale":"2024-08-17T17:31:37.710546+0000","last_undegraded":"2024-08-17T17:31:37.710546+0000","last_fullsized":"2024-08-17T17:31:37.710546+0000","mapping_epoch":1295,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:20:57.414829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":980,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.716065+0000","last_change":"2024-08-17T17:30:16.751988+0000","last_active":"2024-08-17T17:31:37.716065+0000","last_peered":"2024-08-17T17:31:37.716065+0000","last_clean":"2024-08-17T17:31:37.716065+0000","last_became_active":"2024-08-17T17:30:16.751459+0000","last_became_peered":"2024-08-17T17:30:16.751459+0000","last_unstale":"2024-08-17T17:31:37.716065+0000","last_undegraded":"2024-08-17T17:31:37.716065+0000","last_fullsized":"2024-08-17T17:31:37.716065+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:19:35.996084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"1302'6671","reported_seq":6694,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:47.016508+0000","last_change":"2024-08-17T17:30:31.047456+0000","last_active":"2024-08-17T17:31:47.016508+0000","last_peered":"2024-08-17T17:31:47.016508+0000","last_clean":"2024-08-17T17:31:47.016508+0000","last_became_active":"2024-08-17T17:30:31.047147+0000","last_became_peered":"2024-08-17T17:30:31.047147+0000","last_unstale":"2024-08-17T17:31:47.016508+0000","last_undegraded":"2024-08-17T17:31:47.016508+0000","last_fullsized":"2024-08-17T17:31:47.016508+0000","mapping_epoch":1288,"log_start":"1292'100","ondisk_log_start":"1292'100","created":1288,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6571,"log_dups_size":0,"ondisk_log_size":6571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:45:18.819118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437190656,"num_objects":6671,"num_object_clones":0,"num_object_copies":20013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6671,"num_write_kb":426944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1669,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.713524+0000","last_change":"2024-08-17T17:24:12.519946+0000","last_active":"2024-08-17T17:31:36.713524+0000","last_peered":"2024-08-17T17:31:36.713524+0000","last_clean":"2024-08-17T17:31:36.713524+0000","last_became_active":"2024-08-17T17:24:11.569747+0000","last_became_peered":"2024-08-17T17:24:11.569747+0000","last_unstale":"2024-08-17T17:31:36.713524+0000","last_undegraded":"2024-08-17T17:31:36.713524+0000","last_fullsized":"2024-08-17T17:31:36.713524+0000","mapping_epoch":1146,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.519838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.519838+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:17:56.003993+0000","scrub_duration":0.0068224740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1768,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.716751+0000","last_change":"2024-08-17T17:18:31.535299+0000","last_active":"2024-08-17T17:31:37.716751+0000","last_peered":"2024-08-17T17:31:37.716751+0000","last_clean":"2024-08-17T17:31:37.716751+0000","last_became_active":"2024-08-17T17:18:31.534429+0000","last_became_peered":"2024-08-17T17:18:31.534429+0000","last_unstale":"2024-08-17T17:31:37.716751+0000","last_undegraded":"2024-08-17T17:31:37.716751+0000","last_fullsized":"2024-08-17T17:31:37.716751+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:35.828308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1171,"reported_epoch":1302,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:37.715845+0000","last_change":"2024-08-17T17:30:17.759199+0000","last_active":"2024-08-17T17:31:37.715845+0000","last_peered":"2024-08-17T17:31:37.715845+0000","last_clean":"2024-08-17T17:31:37.715845+0000","last_became_active":"2024-08-17T17:30:17.758799+0000","last_became_peered":"2024-08-17T17:30:17.758799+0000","last_unstale":"2024-08-17T17:31:37.715845+0000","last_undegraded":"2024-08-17T17:31:37.715845+0000","last_fullsized":"2024-08-17T17:31:37.715845+0000","mapping_epoch":1279,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:16:07.552945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0,1],"avail_no_missing":["2","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":954,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.710246+0000","last_change":"2024-08-17T17:30:16.752136+0000","last_active":"2024-08-17T17:31:36.710246+0000","last_peered":"2024-08-17T17:31:36.710246+0000","last_clean":"2024-08-17T17:31:36.710246+0000","last_became_active":"2024-08-17T17:30:16.751575+0000","last_became_peered":"2024-08-17T17:30:16.751575+0000","last_unstale":"2024-08-17T17:31:36.710246+0000","last_undegraded":"2024-08-17T17:31:36.710246+0000","last_fullsized":"2024-08-17T17:31:36.710246+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:52.867557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"1302'6678","reported_seq":7127,"reported_epoch":1302,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:31:46.987348+0000","last_change":"2024-08-17T17:31:35.901344+0000","last_active":"2024-08-17T17:31:46.987348+0000","last_peered":"2024-08-17T17:31:46.987348+0000","last_clean":"2024-08-17T17:30:30.026678+0000","last_became_active":"2024-08-17T17:31:35.854230+0000","last_became_peered":"2024-08-17T17:31:35.854230+0000","last_unstale":"2024-08-17T17:31:46.987348+0000","last_undegraded":"2024-08-17T17:31:35.707617+0000","last_fullsized":"2024-08-17T17:31:35.675247+0000","mapping_epoch":1299,"log_start":"1292'100","ondisk_log_start":"1292'100","created":1288,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6578,"log_dups_size":0,"ondisk_log_size":6578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:10.484940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437649408,"num_objects":6678,"num_object_clones":0,"num_object_copies":20034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6678,"num_objects_misplaced":5678,"num_objects_unfound":0,"num_objects_dirty":6678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6678,"num_write_kb":427392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":210,"num_bytes_recovered":13762560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":1000}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1611,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.708934+0000","last_change":"2024-08-17T17:24:13.795227+0000","last_active":"2024-08-17T17:31:36.708934+0000","last_peered":"2024-08-17T17:31:36.708934+0000","last_clean":"2024-08-17T17:31:36.708934+0000","last_became_active":"2024-08-17T17:24:11.578352+0000","last_became_peered":"2024-08-17T17:24:11.578352+0000","last_unstale":"2024-08-17T17:31:36.708934+0000","last_undegraded":"2024-08-17T17:31:36.708934+0000","last_fullsized":"2024-08-17T17:31:36.708934+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:36:39.440808+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1309,"reported_epoch":1301,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:36.716289+0000","last_change":"2024-08-17T17:30:17.787708+0000","last_active":"2024-08-17T17:31:36.716289+0000","last_peered":"2024-08-17T17:31:36.716289+0000","last_clean":"2024-08-17T17:31:36.716289+0000","last_became_active":"2024-08-17T17:30:17.775352+0000","last_became_peered":"2024-08-17T17:30:17.775352+0000","last_unstale":"2024-08-17T17:31:36.716289+0000","last_undegraded":"2024-08-17T17:31:36.716289+0000","last_fullsized":"2024-08-17T17:31:36.716289+0000","mapping_epoch":1279,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:06:34.015530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0,1],"avail_no_missing":["5","0","1"],"object_location_counts":[{"shards":"0,1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1277,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.716068+0000","last_change":"2024-08-17T17:29:53.530759+0000","last_active":"2024-08-17T17:31:36.716068+0000","last_peered":"2024-08-17T17:31:36.716068+0000","last_clean":"2024-08-17T17:29:38.552381+0000","last_became_active":"2024-08-17T17:29:53.530759+0000","last_became_peered":"2024-08-17T17:29:53.530759+0000","last_unstale":"2024-08-17T17:31:36.716068+0000","last_undegraded":"2024-08-17T17:31:36.716068+0000","last_fullsized":"2024-08-17T17:29:53.525624+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:40.072138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1242,"reported_epoch":1302,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:37.715475+0000","last_change":"2024-08-17T17:30:17.750509+0000","last_active":"2024-08-17T17:31:37.715475+0000","last_peered":"2024-08-17T17:31:37.715475+0000","last_clean":"2024-08-17T17:31:37.715475+0000","last_became_active":"2024-08-17T17:30:17.750080+0000","last_became_peered":"2024-08-17T17:30:17.750080+0000","last_unstale":"2024-08-17T17:31:37.715475+0000","last_undegraded":"2024-08-17T17:31:37.715475+0000","last_fullsized":"2024-08-17T17:31:37.715475+0000","mapping_epoch":1279,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:54:23.717351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3,1],"avail_no_missing":["2","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.c","version":"1302'6729","reported_seq":6752,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:46.937547+0000","last_change":"2024-08-17T17:30:31.050402+0000","last_active":"2024-08-17T17:31:46.937547+0000","last_peered":"2024-08-17T17:31:46.937547+0000","last_clean":"2024-08-17T17:31:46.937547+0000","last_became_active":"2024-08-17T17:30:31.050164+0000","last_became_peered":"2024-08-17T17:30:31.050164+0000","last_unstale":"2024-08-17T17:31:46.937547+0000","last_undegraded":"2024-08-17T17:31:46.937547+0000","last_fullsized":"2024-08-17T17:31:46.937547+0000","mapping_epoch":1288,"log_start":"1292'1176","ondisk_log_start":"1292'1176","created":1288,"last_epoch_clean":1289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":5553,"log_dups_size":0,"ondisk_log_size":5553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:00:15.774986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":440991744,"num_objects":6729,"num_object_clones":0,"num_object_copies":20187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6729,"num_write_kb":430656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":1451,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.715259+0000","last_change":"2024-08-17T17:30:27.481545+0000","last_active":"2024-08-17T17:31:36.715259+0000","last_peered":"2024-08-17T17:31:36.715259+0000","last_clean":"2024-08-17T17:18:22.182098+0000","last_became_active":"2024-08-17T17:30:27.481545+0000","last_became_peered":"2024-08-17T17:30:27.481545+0000","last_unstale":"2024-08-17T17:31:36.715259+0000","last_undegraded":"2024-08-17T17:31:36.715259+0000","last_fullsized":"2024-08-17T17:30:27.474302+0000","mapping_epoch":1285,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:55.041378+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:55.041378+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":19,"num_write":4,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53669,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1264,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.709746+0000","last_change":"2024-08-17T17:07:45.680191+0000","last_active":"2024-08-17T17:31:36.709746+0000","last_peered":"2024-08-17T17:31:36.709746+0000","last_clean":"2024-08-17T17:31:36.709746+0000","last_became_active":"2024-08-17T17:07:45.679885+0000","last_became_peered":"2024-08-17T17:07:45.679885+0000","last_unstale":"2024-08-17T17:31:36.709746+0000","last_undegraded":"2024-08-17T17:31:36.709746+0000","last_fullsized":"2024-08-17T17:31:36.709746+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:47.590203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":963,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.715417+0000","last_change":"2024-08-17T17:30:16.758025+0000","last_active":"2024-08-17T17:31:36.715417+0000","last_peered":"2024-08-17T17:31:36.715417+0000","last_clean":"2024-08-17T17:31:36.715417+0000","last_became_active":"2024-08-17T17:30:16.757704+0000","last_became_peered":"2024-08-17T17:30:16.757704+0000","last_unstale":"2024-08-17T17:31:36.715417+0000","last_undegraded":"2024-08-17T17:31:36.715417+0000","last_fullsized":"2024-08-17T17:31:36.715417+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:27.711448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":905,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.710011+0000","last_change":"2024-08-17T17:30:16.747909+0000","last_active":"2024-08-17T17:31:36.710011+0000","last_peered":"2024-08-17T17:31:36.710011+0000","last_clean":"2024-08-17T17:31:36.710011+0000","last_became_active":"2024-08-17T17:30:16.747542+0000","last_became_peered":"2024-08-17T17:30:16.747542+0000","last_unstale":"2024-08-17T17:31:36.710011+0000","last_undegraded":"2024-08-17T17:31:36.710011+0000","last_fullsized":"2024-08-17T17:31:36.710011+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:40.571895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"1302'6841","reported_seq":6861,"reported_epoch":1302,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:31:46.983090+0000","last_change":"2024-08-17T17:30:45.038994+0000","last_active":"2024-08-17T17:31:46.983090+0000","last_peered":"2024-08-17T17:31:46.983090+0000","last_clean":"2024-08-17T17:30:30.026678+0000","last_became_active":"2024-08-17T17:30:31.046228+0000","last_became_peered":"2024-08-17T17:30:31.046228+0000","last_unstale":"2024-08-17T17:31:46.983090+0000","last_undegraded":"2024-08-17T17:30:45.038835+0000","last_fullsized":"2024-08-17T17:30:31.044006+0000","mapping_epoch":1288,"log_start":"1292'1209","ondisk_log_start":"1292'1209","created":1288,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":5632,"log_dups_size":0,"ondisk_log_size":5632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:28:39.199905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":448331776,"num_objects":6841,"num_object_clones":0,"num_object_copies":20523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6841,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6841,"num_write_kb":437824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":6841}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":700,"reported_epoch":1302,"state":"active+undersized","last_fresh":"2024-08-17T17:31:37.712064+0000","last_change":"2024-08-17T17:30:27.482210+0000","last_active":"2024-08-17T17:31:37.712064+0000","last_peered":"2024-08-17T17:31:37.712064+0000","last_clean":"2024-08-17T17:18:28.526985+0000","last_became_active":"2024-08-17T17:30:27.482210+0000","last_became_peered":"2024-08-17T17:30:27.482210+0000","last_unstale":"2024-08-17T17:31:37.712064+0000","last_undegraded":"2024-08-17T17:31:37.712064+0000","last_fullsized":"2024-08-17T17:30:27.474804+0000","mapping_epoch":1285,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.103133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.103133+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:31:38.613340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1166,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.715261+0000","last_change":"2024-08-17T17:30:16.747211+0000","last_active":"2024-08-17T17:31:37.715261+0000","last_peered":"2024-08-17T17:31:37.715261+0000","last_clean":"2024-08-17T17:31:37.715261+0000","last_became_active":"2024-08-17T17:30:16.746880+0000","last_became_peered":"2024-08-17T17:30:16.746880+0000","last_unstale":"2024-08-17T17:31:37.715261+0000","last_undegraded":"2024-08-17T17:31:37.715261+0000","last_fullsized":"2024-08-17T17:31:37.715261+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:43:15.602906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1264,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.714411+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:31:36.714411+0000","last_peered":"2024-08-17T17:31:36.714411+0000","last_clean":"2024-08-17T17:31:36.714411+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:31:36.714411+0000","last_undegraded":"2024-08-17T17:31:36.714411+0000","last_fullsized":"2024-08-17T17:31:36.714411+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":857,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.706133+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:31:37.706133+0000","last_peered":"2024-08-17T17:31:37.706133+0000","last_clean":"2024-08-17T17:31:37.706133+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:31:37.706133+0000","last_undegraded":"2024-08-17T17:31:37.706133+0000","last_fullsized":"2024-08-17T17:31:37.706133+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"1302'6792","reported_seq":6812,"reported_epoch":1302,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:31:46.944380+0000","last_change":"2024-08-17T17:30:45.040437+0000","last_active":"2024-08-17T17:31:46.944380+0000","last_peered":"2024-08-17T17:31:46.944380+0000","last_clean":"2024-08-17T17:30:30.026678+0000","last_became_active":"2024-08-17T17:30:31.047154+0000","last_became_peered":"2024-08-17T17:30:31.047154+0000","last_unstale":"2024-08-17T17:31:46.944380+0000","last_undegraded":"2024-08-17T17:30:45.040259+0000","last_fullsized":"2024-08-17T17:30:31.043359+0000","mapping_epoch":1288,"log_start":"1292'1153","ondisk_log_start":"1292'1153","created":1288,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":5639,"log_dups_size":0,"ondisk_log_size":5639,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:43.577566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":445120512,"num_objects":6792,"num_object_clones":0,"num_object_copies":20376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6792,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6792,"num_write_kb":434688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":6792}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1621,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.713376+0000","last_change":"2024-08-17T17:29:53.529747+0000","last_active":"2024-08-17T17:31:36.713376+0000","last_peered":"2024-08-17T17:31:36.713376+0000","last_clean":"2024-08-17T17:29:38.553022+0000","last_became_active":"2024-08-17T17:29:53.529747+0000","last_became_peered":"2024-08-17T17:29:53.529747+0000","last_unstale":"2024-08-17T17:31:36.713376+0000","last_undegraded":"2024-08-17T17:31:36.713376+0000","last_fullsized":"2024-08-17T17:29:53.526039+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:45:41.319012+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":57,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.715435+0000","last_change":"2024-08-17T17:30:27.481627+0000","last_active":"2024-08-17T17:31:36.715435+0000","last_peered":"2024-08-17T17:31:36.715435+0000","last_clean":"2024-08-17T17:31:36.715435+0000","last_became_active":"2024-08-17T17:30:27.481296+0000","last_became_peered":"2024-08-17T17:30:27.481296+0000","last_unstale":"2024-08-17T17:31:36.715435+0000","last_undegraded":"2024-08-17T17:31:36.715435+0000","last_fullsized":"2024-08-17T17:31:36.715435+0000","mapping_epoch":1285,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:01:14.582369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":810,"reported_epoch":1301,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:36.715918+0000","last_change":"2024-08-17T17:30:17.755184+0000","last_active":"2024-08-17T17:31:36.715918+0000","last_peered":"2024-08-17T17:31:36.715918+0000","last_clean":"2024-08-17T17:31:36.715918+0000","last_became_active":"2024-08-17T17:30:17.754471+0000","last_became_peered":"2024-08-17T17:30:17.754471+0000","last_unstale":"2024-08-17T17:31:36.715918+0000","last_undegraded":"2024-08-17T17:31:36.715918+0000","last_fullsized":"2024-08-17T17:31:36.715918+0000","mapping_epoch":1279,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:39:33.883393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,1],"avail_no_missing":["4","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1419,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.714410+0000","last_change":"2024-08-17T17:31:34.681452+0000","last_active":"2024-08-17T17:31:36.714410+0000","last_peered":"2024-08-17T17:31:36.714410+0000","last_clean":"2024-08-17T17:31:36.714410+0000","last_became_active":"2024-08-17T17:31:34.681033+0000","last_became_peered":"2024-08-17T17:31:34.681033+0000","last_unstale":"2024-08-17T17:31:36.714410+0000","last_undegraded":"2024-08-17T17:31:36.714410+0000","last_fullsized":"2024-08-17T17:31:36.714410+0000","mapping_epoch":1298,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:27:03.626909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"1302'6675","reported_seq":6711,"reported_epoch":1302,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:31:46.058664+0000","last_change":"2024-08-17T17:30:45.054716+0000","last_active":"2024-08-17T17:31:46.058664+0000","last_peered":"2024-08-17T17:31:46.058664+0000","last_clean":"2024-08-17T17:30:30.026678+0000","last_became_active":"2024-08-17T17:30:31.047056+0000","last_became_peered":"2024-08-17T17:30:31.047056+0000","last_unstale":"2024-08-17T17:31:46.058664+0000","last_undegraded":"2024-08-17T17:30:45.047432+0000","last_fullsized":"2024-08-17T17:30:31.042363+0000","mapping_epoch":1288,"log_start":"1292'1500","ondisk_log_start":"1292'1500","created":1288,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":5175,"log_dups_size":0,"ondisk_log_size":5175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:17.503513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437452800,"num_objects":6675,"num_object_clones":0,"num_object_copies":20025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6675,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6675,"num_write_kb":427200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":6675}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1750,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.712606+0000","last_change":"2024-08-17T17:24:12.033217+0000","last_active":"2024-08-17T17:31:37.712606+0000","last_peered":"2024-08-17T17:31:37.712606+0000","last_clean":"2024-08-17T17:31:37.712606+0000","last_became_active":"2024-08-17T17:24:11.572877+0000","last_became_peered":"2024-08-17T17:24:11.572877+0000","last_unstale":"2024-08-17T17:31:37.712606+0000","last_undegraded":"2024-08-17T17:31:37.712606+0000","last_fullsized":"2024-08-17T17:31:37.712606+0000","mapping_epoch":1141,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.033101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.033101+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.264475+0000","scrub_duration":20,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":288,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.715503+0000","last_change":"2024-08-17T17:30:16.764107+0000","last_active":"2024-08-17T17:31:36.715503+0000","last_peered":"2024-08-17T17:31:36.715503+0000","last_clean":"2024-08-17T17:31:36.715503+0000","last_became_active":"2024-08-17T17:30:16.756141+0000","last_became_peered":"2024-08-17T17:30:16.756141+0000","last_unstale":"2024-08-17T17:31:36.715503+0000","last_undegraded":"2024-08-17T17:31:36.715503+0000","last_fullsized":"2024-08-17T17:31:36.715503+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:17.491834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":781,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.715605+0000","last_change":"2024-08-17T17:29:53.532379+0000","last_active":"2024-08-17T17:31:36.715605+0000","last_peered":"2024-08-17T17:31:36.715605+0000","last_clean":"2024-08-17T17:06:58.643061+0000","last_became_active":"2024-08-17T17:29:53.532379+0000","last_became_peered":"2024-08-17T17:29:53.532379+0000","last_unstale":"2024-08-17T17:31:36.715605+0000","last_undegraded":"2024-08-17T17:31:36.715605+0000","last_fullsized":"2024-08-17T17:29:53.528598+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:55:06.971090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1243,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.716365+0000","last_change":"2024-08-17T17:29:53.529700+0000","last_active":"2024-08-17T17:31:36.716365+0000","last_peered":"2024-08-17T17:31:36.716365+0000","last_clean":"2024-08-17T17:29:38.552083+0000","last_became_active":"2024-08-17T17:29:53.529700+0000","last_became_peered":"2024-08-17T17:29:53.529700+0000","last_unstale":"2024-08-17T17:31:36.716365+0000","last_undegraded":"2024-08-17T17:31:36.716365+0000","last_fullsized":"2024-08-17T17:29:53.524158+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:03.647039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":794,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.716024+0000","last_change":"2024-08-17T17:30:27.481908+0000","last_active":"2024-08-17T17:31:36.716024+0000","last_peered":"2024-08-17T17:31:36.716024+0000","last_clean":"2024-08-17T17:31:36.716024+0000","last_became_active":"2024-08-17T17:30:27.481281+0000","last_became_peered":"2024-08-17T17:30:27.481281+0000","last_unstale":"2024-08-17T17:31:36.716024+0000","last_undegraded":"2024-08-17T17:31:36.716024+0000","last_fullsized":"2024-08-17T17:31:36.716024+0000","mapping_epoch":1285,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1286,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:07:43.667627+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1284,"reported_epoch":1301,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:36.715932+0000","last_change":"2024-08-17T17:30:17.748645+0000","last_active":"2024-08-17T17:31:36.715932+0000","last_peered":"2024-08-17T17:31:36.715932+0000","last_clean":"2024-08-17T17:31:36.715932+0000","last_became_active":"2024-08-17T17:30:17.747064+0000","last_became_peered":"2024-08-17T17:30:17.747064+0000","last_unstale":"2024-08-17T17:31:36.715932+0000","last_undegraded":"2024-08-17T17:31:36.715932+0000","last_fullsized":"2024-08-17T17:31:36.715932+0000","mapping_epoch":1279,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:53:26.532821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,1],"avail_no_missing":["4","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1265,"reported_epoch":1301,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:31:36.715016+0000","last_change":"2024-08-17T17:29:53.531085+0000","last_active":"2024-08-17T17:31:36.715016+0000","last_peered":"2024-08-17T17:31:36.715016+0000","last_clean":"2024-08-17T17:29:38.551404+0000","last_became_active":"2024-08-17T17:29:53.531085+0000","last_became_peered":"2024-08-17T17:29:53.531085+0000","last_unstale":"2024-08-17T17:31:36.715016+0000","last_undegraded":"2024-08-17T17:29:53.527582+0000","last_fullsized":"2024-08-17T17:29:53.527482+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:01.176057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":741,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.710162+0000","last_change":"2024-08-17T17:29:53.529304+0000","last_active":"2024-08-17T17:31:36.710162+0000","last_peered":"2024-08-17T17:31:36.710162+0000","last_clean":"2024-08-17T17:29:38.565104+0000","last_became_active":"2024-08-17T17:29:53.529304+0000","last_became_peered":"2024-08-17T17:29:53.529304+0000","last_unstale":"2024-08-17T17:31:36.710162+0000","last_undegraded":"2024-08-17T17:31:36.710162+0000","last_fullsized":"2024-08-17T17:29:53.525070+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:04.056586+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":857,"reported_epoch":1302,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:37.720433+0000","last_change":"2024-08-17T17:30:17.749028+0000","last_active":"2024-08-17T17:31:37.720433+0000","last_peered":"2024-08-17T17:31:37.720433+0000","last_clean":"2024-08-17T17:31:37.720433+0000","last_became_active":"2024-08-17T17:30:17.748576+0000","last_became_peered":"2024-08-17T17:30:17.748576+0000","last_unstale":"2024-08-17T17:31:37.720433+0000","last_undegraded":"2024-08-17T17:31:37.720433+0000","last_fullsized":"2024-08-17T17:31:37.720433+0000","mapping_epoch":1279,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:09:27.716931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0,1],"avail_no_missing":["2","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1234,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.714100+0000","last_change":"2024-08-17T17:07:45.684388+0000","last_active":"2024-08-17T17:31:36.714100+0000","last_peered":"2024-08-17T17:31:36.714100+0000","last_clean":"2024-08-17T17:31:36.714100+0000","last_became_active":"2024-08-17T17:07:45.684023+0000","last_became_peered":"2024-08-17T17:07:45.684023+0000","last_unstale":"2024-08-17T17:31:36.714100+0000","last_undegraded":"2024-08-17T17:31:36.714100+0000","last_fullsized":"2024-08-17T17:31:36.714100+0000","mapping_epoch":837,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:58:13.846820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1752,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.714897+0000","last_change":"2024-08-17T17:24:20.649977+0000","last_active":"2024-08-17T17:31:37.714897+0000","last_peered":"2024-08-17T17:31:37.714897+0000","last_clean":"2024-08-17T17:31:37.714897+0000","last_became_active":"2024-08-17T17:24:20.649336+0000","last_became_peered":"2024-08-17T17:24:20.649336+0000","last_unstale":"2024-08-17T17:31:37.714897+0000","last_undegraded":"2024-08-17T17:31:37.714897+0000","last_fullsized":"2024-08-17T17:31:37.714897+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:57.187271+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1273,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.711382+0000","last_change":"2024-08-17T17:30:16.751480+0000","last_active":"2024-08-17T17:31:36.711382+0000","last_peered":"2024-08-17T17:31:36.711382+0000","last_clean":"2024-08-17T17:31:36.711382+0000","last_became_active":"2024-08-17T17:30:16.751075+0000","last_became_peered":"2024-08-17T17:30:16.751075+0000","last_unstale":"2024-08-17T17:31:36.711382+0000","last_undegraded":"2024-08-17T17:31:36.711382+0000","last_fullsized":"2024-08-17T17:31:36.711382+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:37:31.651833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1637,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.716351+0000","last_change":"2024-08-17T17:30:27.479605+0000","last_active":"2024-08-17T17:31:36.716351+0000","last_peered":"2024-08-17T17:31:36.716351+0000","last_clean":"2024-08-17T17:29:38.564651+0000","last_became_active":"2024-08-17T17:30:27.479605+0000","last_became_peered":"2024-08-17T17:30:27.479605+0000","last_unstale":"2024-08-17T17:31:36.716351+0000","last_undegraded":"2024-08-17T17:31:36.716351+0000","last_fullsized":"2024-08-17T17:30:27.473361+0000","mapping_epoch":1285,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:15:50.842983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1499,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.714343+0000","last_change":"2024-08-17T17:30:19.090893+0000","last_active":"2024-08-17T17:31:36.714343+0000","last_peered":"2024-08-17T17:31:36.714343+0000","last_clean":"2024-08-17T17:31:36.714343+0000","last_became_active":"2024-08-17T17:30:19.051974+0000","last_became_peered":"2024-08-17T17:30:19.051974+0000","last_unstale":"2024-08-17T17:31:36.714343+0000","last_undegraded":"2024-08-17T17:31:36.714343+0000","last_fullsized":"2024-08-17T17:31:36.714343+0000","mapping_epoch":1281,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1282,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:10:10.502399+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1100,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.710438+0000","last_change":"2024-08-17T17:18:31.536710+0000","last_active":"2024-08-17T17:31:36.710438+0000","last_peered":"2024-08-17T17:31:36.710438+0000","last_clean":"2024-08-17T17:31:36.710438+0000","last_became_active":"2024-08-17T17:18:31.536365+0000","last_became_peered":"2024-08-17T17:18:31.536365+0000","last_unstale":"2024-08-17T17:31:36.710438+0000","last_undegraded":"2024-08-17T17:31:36.710438+0000","last_fullsized":"2024-08-17T17:31:36.710438+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:57:08.297188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1713,"reported_epoch":1302,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:37.710123+0000","last_change":"2024-08-17T17:30:17.754077+0000","last_active":"2024-08-17T17:31:37.710123+0000","last_peered":"2024-08-17T17:31:37.710123+0000","last_clean":"2024-08-17T17:31:37.710123+0000","last_became_active":"2024-08-17T17:30:17.753119+0000","last_became_peered":"2024-08-17T17:30:17.753119+0000","last_unstale":"2024-08-17T17:31:37.710123+0000","last_undegraded":"2024-08-17T17:31:37.710123+0000","last_fullsized":"2024-08-17T17:31:37.710123+0000","mapping_epoch":1279,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:33.582544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,1],"avail_no_missing":["3","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1499,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.712619+0000","last_change":"2024-08-17T17:30:19.042254+0000","last_active":"2024-08-17T17:31:36.712619+0000","last_peered":"2024-08-17T17:31:36.712619+0000","last_clean":"2024-08-17T17:31:36.712619+0000","last_became_active":"2024-08-17T17:30:19.041923+0000","last_became_peered":"2024-08-17T17:30:19.041923+0000","last_unstale":"2024-08-17T17:31:36.712619+0000","last_undegraded":"2024-08-17T17:31:36.712619+0000","last_fullsized":"2024-08-17T17:31:36.712619+0000","mapping_epoch":1281,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1282,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:14:54.737444+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1171,"reported_epoch":1301,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:36.710319+0000","last_change":"2024-08-17T17:30:17.779072+0000","last_active":"2024-08-17T17:31:36.710319+0000","last_peered":"2024-08-17T17:31:36.710319+0000","last_clean":"2024-08-17T17:31:36.710319+0000","last_became_active":"2024-08-17T17:30:17.778695+0000","last_became_peered":"2024-08-17T17:30:17.778695+0000","last_unstale":"2024-08-17T17:31:36.710319+0000","last_undegraded":"2024-08-17T17:31:36.710319+0000","last_fullsized":"2024-08-17T17:31:36.710319+0000","mapping_epoch":1279,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:47.959101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0,1],"avail_no_missing":["5","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1735,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.715546+0000","last_change":"2024-08-17T17:31:34.682271+0000","last_active":"2024-08-17T17:31:37.715546+0000","last_peered":"2024-08-17T17:31:37.715546+0000","last_clean":"2024-08-17T17:31:37.715546+0000","last_became_active":"2024-08-17T17:31:34.681791+0000","last_became_peered":"2024-08-17T17:31:34.681791+0000","last_unstale":"2024-08-17T17:31:37.715546+0000","last_undegraded":"2024-08-17T17:31:37.715546+0000","last_fullsized":"2024-08-17T17:31:37.715546+0000","mapping_epoch":1298,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:11:18.367177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":842,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.709851+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:31:36.709851+0000","last_peered":"2024-08-17T17:31:36.709851+0000","last_clean":"2024-08-17T17:31:36.709851+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:31:36.709851+0000","last_undegraded":"2024-08-17T17:31:36.709851+0000","last_fullsized":"2024-08-17T17:31:36.709851+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1062,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.714925+0000","last_change":"2024-08-17T17:18:31.544057+0000","last_active":"2024-08-17T17:31:36.714925+0000","last_peered":"2024-08-17T17:31:36.714925+0000","last_clean":"2024-08-17T17:31:36.714925+0000","last_became_active":"2024-08-17T17:18:31.543574+0000","last_became_peered":"2024-08-17T17:18:31.543574+0000","last_unstale":"2024-08-17T17:31:36.714925+0000","last_undegraded":"2024-08-17T17:31:36.714925+0000","last_fullsized":"2024-08-17T17:31:36.714925+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:42:50.090844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1235,"reported_epoch":1302,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:37.710650+0000","last_change":"2024-08-17T17:31:35.881783+0000","last_active":"2024-08-17T17:31:37.710650+0000","last_peered":"2024-08-17T17:31:37.710650+0000","last_clean":"2024-08-17T17:31:37.710650+0000","last_became_active":"2024-08-17T17:31:35.881322+0000","last_became_peered":"2024-08-17T17:31:35.881322+0000","last_unstale":"2024-08-17T17:31:37.710650+0000","last_undegraded":"2024-08-17T17:31:37.710650+0000","last_fullsized":"2024-08-17T17:31:37.710650+0000","mapping_epoch":1299,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:34:10.402503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7,4],"avail_no_missing":["3","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1614,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.708556+0000","last_change":"2024-08-17T17:25:02.696608+0000","last_active":"2024-08-17T17:31:36.708556+0000","last_peered":"2024-08-17T17:31:36.708556+0000","last_clean":"2024-08-17T17:31:36.708556+0000","last_became_active":"2024-08-17T17:24:40.877715+0000","last_became_peered":"2024-08-17T17:24:40.877715+0000","last_unstale":"2024-08-17T17:31:36.708556+0000","last_undegraded":"2024-08-17T17:31:36.708556+0000","last_fullsized":"2024-08-17T17:31:36.708556+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:48:00.941285+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1688,"reported_epoch":1301,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:36.713030+0000","last_change":"2024-08-17T17:30:28.579385+0000","last_active":"2024-08-17T17:31:36.713030+0000","last_peered":"2024-08-17T17:31:36.713030+0000","last_clean":"2024-08-17T17:31:36.713030+0000","last_became_active":"2024-08-17T17:30:28.578992+0000","last_became_peered":"2024-08-17T17:30:28.578992+0000","last_unstale":"2024-08-17T17:31:36.713030+0000","last_undegraded":"2024-08-17T17:31:36.713030+0000","last_fullsized":"2024-08-17T17:31:36.713030+0000","mapping_epoch":1286,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1287,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:11:12.881824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,2],"avail_no_missing":["4","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1293,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.714042+0000","last_change":"2024-08-17T17:30:16.753726+0000","last_active":"2024-08-17T17:31:36.714042+0000","last_peered":"2024-08-17T17:31:36.714042+0000","last_clean":"2024-08-17T17:31:36.714042+0000","last_became_active":"2024-08-17T17:30:16.753339+0000","last_became_peered":"2024-08-17T17:30:16.753339+0000","last_unstale":"2024-08-17T17:31:36.714042+0000","last_undegraded":"2024-08-17T17:31:36.714042+0000","last_fullsized":"2024-08-17T17:31:36.714042+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:38:25.336519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1614,"reported_epoch":1302,"state":"active+undersized","last_fresh":"2024-08-17T17:31:37.710196+0000","last_change":"2024-08-17T17:29:53.532468+0000","last_active":"2024-08-17T17:31:37.710196+0000","last_peered":"2024-08-17T17:31:37.710196+0000","last_clean":"2024-08-17T17:29:38.564315+0000","last_became_active":"2024-08-17T17:29:53.532468+0000","last_became_peered":"2024-08-17T17:29:53.532468+0000","last_unstale":"2024-08-17T17:31:37.710196+0000","last_undegraded":"2024-08-17T17:31:37.710196+0000","last_fullsized":"2024-08-17T17:29:53.523273+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.911622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.911622+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:32:03.833400+0000","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1764,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.715377+0000","last_change":"2024-08-17T17:31:34.680586+0000","last_active":"2024-08-17T17:31:37.715377+0000","last_peered":"2024-08-17T17:31:37.715377+0000","last_clean":"2024-08-17T17:31:37.715377+0000","last_became_active":"2024-08-17T17:31:34.680221+0000","last_became_peered":"2024-08-17T17:31:34.680221+0000","last_unstale":"2024-08-17T17:31:37.715377+0000","last_undegraded":"2024-08-17T17:31:37.715377+0000","last_fullsized":"2024-08-17T17:31:37.715377+0000","mapping_epoch":1298,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:51.409775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":262,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.715793+0000","last_change":"2024-08-17T17:31:18.402460+0000","last_active":"2024-08-17T17:31:36.715793+0000","last_peered":"2024-08-17T17:31:36.715793+0000","last_clean":"2024-08-17T17:31:36.715793+0000","last_became_active":"2024-08-17T17:31:18.402008+0000","last_became_peered":"2024-08-17T17:31:18.402008+0000","last_unstale":"2024-08-17T17:31:36.715793+0000","last_undegraded":"2024-08-17T17:31:36.715793+0000","last_fullsized":"2024-08-17T17:31:36.715793+0000","mapping_epoch":1295,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:21.445057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":685,"reported_epoch":1301,"state":"active+undersized","last_fresh":"2024-08-17T17:31:36.714696+0000","last_change":"2024-08-17T17:30:27.481185+0000","last_active":"2024-08-17T17:31:36.714696+0000","last_peered":"2024-08-17T17:31:36.714696+0000","last_clean":"2024-08-17T17:18:28.526985+0000","last_became_active":"2024-08-17T17:30:27.481185+0000","last_became_peered":"2024-08-17T17:30:27.481185+0000","last_unstale":"2024-08-17T17:31:36.714696+0000","last_undegraded":"2024-08-17T17:31:36.714696+0000","last_fullsized":"2024-08-17T17:30:27.474163+0000","mapping_epoch":1285,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":9,"num_read_kb":19,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1291,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.715894+0000","last_change":"2024-08-17T17:30:16.749595+0000","last_active":"2024-08-17T17:31:36.715894+0000","last_peered":"2024-08-17T17:31:36.715894+0000","last_clean":"2024-08-17T17:31:36.715894+0000","last_became_active":"2024-08-17T17:30:16.749250+0000","last_became_peered":"2024-08-17T17:30:16.749250+0000","last_unstale":"2024-08-17T17:31:36.715894+0000","last_undegraded":"2024-08-17T17:31:36.715894+0000","last_fullsized":"2024-08-17T17:31:36.715894+0000","mapping_epoch":1278,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1279,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:21.136051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1284,"reported_epoch":1301,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:36.709034+0000","last_change":"2024-08-17T17:30:17.751668+0000","last_active":"2024-08-17T17:31:36.709034+0000","last_peered":"2024-08-17T17:31:36.709034+0000","last_clean":"2024-08-17T17:31:36.709034+0000","last_became_active":"2024-08-17T17:30:17.750230+0000","last_became_peered":"2024-08-17T17:30:17.750230+0000","last_unstale":"2024-08-17T17:31:36.709034+0000","last_undegraded":"2024-08-17T17:31:36.709034+0000","last_fullsized":"2024-08-17T17:31:36.709034+0000","mapping_epoch":1279,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:33.723276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,1],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":24,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.720253+0000","last_change":"2024-08-17T17:31:18.419785+0000","last_active":"2024-08-17T17:31:37.720253+0000","last_peered":"2024-08-17T17:31:37.720253+0000","last_clean":"2024-08-17T17:31:37.720253+0000","last_became_active":"2024-08-17T17:31:18.419485+0000","last_became_peered":"2024-08-17T17:31:18.419485+0000","last_unstale":"2024-08-17T17:31:37.720253+0000","last_undegraded":"2024-08-17T17:31:37.720253+0000","last_fullsized":"2024-08-17T17:31:37.720253+0000","mapping_epoch":1295,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:05.421481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1239,"reported_epoch":1301,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:31:36.709645+0000","last_change":"2024-08-17T17:29:53.531923+0000","last_active":"2024-08-17T17:31:36.709645+0000","last_peered":"2024-08-17T17:31:36.709645+0000","last_clean":"2024-08-17T17:29:38.564569+0000","last_became_active":"2024-08-17T17:29:53.531923+0000","last_became_peered":"2024-08-17T17:29:53.531923+0000","last_unstale":"2024-08-17T17:31:36.709645+0000","last_undegraded":"2024-08-17T17:29:53.527534+0000","last_fullsized":"2024-08-17T17:29:53.527422+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:28.673742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1696,"reported_epoch":1301,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:36.715044+0000","last_change":"2024-08-17T17:30:19.046612+0000","last_active":"2024-08-17T17:31:36.715044+0000","last_peered":"2024-08-17T17:31:36.715044+0000","last_clean":"2024-08-17T17:31:36.715044+0000","last_became_active":"2024-08-17T17:30:19.046125+0000","last_became_peered":"2024-08-17T17:30:19.046125+0000","last_unstale":"2024-08-17T17:31:36.715044+0000","last_undegraded":"2024-08-17T17:31:36.715044+0000","last_fullsized":"2024-08-17T17:31:36.715044+0000","mapping_epoch":1281,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1282,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:15.044058+0000","scrub_duration":0.0084952199999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5,1],"avail_no_missing":["7","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1422,"reported_epoch":1302,"state":"active+clean","last_fresh":"2024-08-17T17:31:37.716664+0000","last_change":"2024-08-17T17:30:27.482509+0000","last_active":"2024-08-17T17:31:37.716664+0000","last_peered":"2024-08-17T17:31:37.716664+0000","last_clean":"2024-08-17T17:31:37.716664+0000","last_became_active":"2024-08-17T17:30:27.482108+0000","last_became_peered":"2024-08-17T17:30:27.482108+0000","last_unstale":"2024-08-17T17:31:37.716664+0000","last_undegraded":"2024-08-17T17:31:37.716664+0000","last_fullsized":"2024-08-17T17:31:37.716664+0000","mapping_epoch":1285,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:15.888367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":43,"reported_epoch":1302,"state":"active+undersized","last_fresh":"2024-08-17T17:31:37.717506+0000","last_change":"2024-08-17T17:30:27.483260+0000","last_active":"2024-08-17T17:31:37.717506+0000","last_peered":"2024-08-17T17:31:37.717506+0000","last_clean":"2024-08-17T16:53:35.850756+0000","last_became_active":"2024-08-17T17:30:27.483260+0000","last_became_peered":"2024-08-17T17:30:27.483260+0000","last_unstale":"2024-08-17T17:31:37.717506+0000","last_undegraded":"2024-08-17T17:31:37.717506+0000","last_fullsized":"2024-08-17T17:30:27.478705+0000","mapping_epoch":1285,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:44:47.867684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1543,"reported_epoch":1301,"state":"active+clean","last_fresh":"2024-08-17T17:31:36.712905+0000","last_change":"2024-08-17T17:30:28.798844+0000","last_active":"2024-08-17T17:31:36.712905+0000","last_peered":"2024-08-17T17:31:36.712905+0000","last_clean":"2024-08-17T17:31:36.712905+0000","last_became_active":"2024-08-17T17:30:27.480204+0000","last_became_peered":"2024-08-17T17:30:27.480204+0000","last_unstale":"2024-08-17T17:31:36.712905+0000","last_undegraded":"2024-08-17T17:31:36.712905+0000","last_fullsized":"2024-08-17T17:31:36.712905+0000","mapping_epoch":1285,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1286,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:49.496641+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1283,"reported_epoch":1301,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:36.717151+0000","last_change":"2024-08-17T17:30:17.752628+0000","last_active":"2024-08-17T17:31:36.717151+0000","last_peered":"2024-08-17T17:31:36.717151+0000","last_clean":"2024-08-17T17:31:36.717151+0000","last_became_active":"2024-08-17T17:30:17.752263+0000","last_became_peered":"2024-08-17T17:30:17.752263+0000","last_unstale":"2024-08-17T17:31:36.717151+0000","last_undegraded":"2024-08-17T17:31:36.717151+0000","last_fullsized":"2024-08-17T17:31:36.717151+0000","mapping_epoch":1279,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:44:30.019294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,1],"avail_no_missing":["4","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":287,"reported_epoch":1302,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:31:37.706130+0000","last_change":"2024-08-17T17:31:35.678605+0000","last_active":"2024-08-17T17:31:37.706130+0000","last_peered":"2024-08-17T17:31:37.706130+0000","last_clean":"2024-08-17T17:31:37.706130+0000","last_became_active":"2024-08-17T17:31:35.678065+0000","last_became_peered":"2024-08-17T17:31:35.678065+0000","last_unstale":"2024-08-17T17:31:37.706130+0000","last_undegraded":"2024-08-17T17:31:37.706130+0000","last_fullsized":"2024-08-17T17:31:37.706130+0000","mapping_epoch":1299,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:59.133394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":7080968192,"num_objects":108047,"num_object_clones":0,"num_object_copies":324141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47243,"num_objects_misplaced":5678,"num_objects_unfound":0,"num_objects_dirty":108047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":108047,"num_write_kb":6915008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":210,"num_bytes_recovered":13762560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18196856832,"data_stored":18196856832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96684,"ondisk_log_size":96684,"up":41,"acting":41,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":80,"acting":91,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":3,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1208,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":82,"acting":87,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":21,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":630620,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":26,"num_read_kb":56,"num_write":14,"num_write_kb":165,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":134174,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":79,"acting":82,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201764,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2862008,"kb_used_data":2145136,"kb_used_omap":1,"kb_used_meta":716862,"kb_avail":90883144,"statfs":{"total":95995035648,"available":93064339456,"internally_reserved":0,"allocated":2196619264,"data_stored":2193743526,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":734067147},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463378,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":695916,"kb_used_data":271908,"kb_used_omap":1,"kb_used_meta":423998,"kb_avail":93049236,"statfs":{"total":95995035648,"available":95282417664,"internally_reserved":0,"allocated":278433792,"data_stored":275769600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":434174409},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627042,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3379548,"kb_used_data":2571348,"kb_used_omap":42,"kb_used_meta":808149,"kb_avail":90365604,"statfs":{"total":95995035648,"available":92534378496,"internally_reserved":0,"allocated":2633060352,"data_stored":2630207872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":827544636},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":0,"seq":3311419785520,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725037,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2838184,"kb_used_data":2139360,"kb_used_omap":1,"kb_used_meta":698814,"kb_avail":90906968,"statfs":{"total":95995035648,"available":93088735232,"internally_reserved":0,"allocated":2190704640,"data_stored":2187840985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":715585990},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986674,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4504304,"kb_used_data":3881768,"kb_used_omap":1,"kb_used_meta":622526,"kb_avail":89240848,"statfs":{"total":95995035648,"available":91382628352,"internally_reserved":0,"allocated":3974930432,"data_stored":3972062272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":637467078},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706281,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3289892,"kb_used_data":2648412,"kb_used_omap":35,"kb_used_meta":641436,"kb_avail":90455260,"statfs":{"total":95995035648,"available":92626186240,"internally_reserved":0,"allocated":2711973888,"data_stored":2709114286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":656830519},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,1,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832054,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5116644,"kb_used_data":4326172,"kb_used_omap":65,"kb_used_meta":790398,"kb_avail":88628508,"statfs":{"total":95995035648,"available":90755592192,"internally_reserved":0,"allocated":4430000128,"data_stored":4427139989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":809367739},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4441047040,"data_stored":4441047040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2719744000,"data_stored":2719744000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2633695232,"data_stored":2633695232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4003332096,"data_stored":4003332096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2202075136,"data_stored":2202075136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2196963328,"data_stored":2196963328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:31:51.434 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-08-17T17:31:51.434 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_5 pg_num 26 2024-08-17T17:31:51.438 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:51.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:51.540 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:51.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.587 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:51.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:51.688 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:51.742 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:51.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:51.844 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:51.888 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:51.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:51.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:51.990 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:52.027 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:52.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:51 smithi063 ceph-mon[114756]: pgmap v1901: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 47254/324219 objects degraded (14.575%); 5682/324219 objects misplaced (1.753%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:31:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:51 smithi063 ceph-mon[109843]: pgmap v1901: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 47254/324219 objects degraded (14.575%); 5682/324219 objects misplaced (1.753%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:31:52.050 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 67 16 113576 113560 105.908 69 0.00841886 0.00943754 2024-08-17T17:31:52.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:51 smithi162 ceph-mon[104668]: pgmap v1901: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.6 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 71 MiB/s wr, 1.14k op/s; 47254/324219 objects degraded (14.575%); 5682/324219 objects misplaced (1.753%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:31:52.130 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:52.167 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:52.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:52.269 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:52.318 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:52.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:52.420 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:52.451 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:52.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:52.553 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:52.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.594 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:52.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:52.695 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:52.731 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:52.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:52.833 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:52.869 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:52.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:52.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:52.972 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:53.001 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:53.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:53.050 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 68 16 114594 114578 105.286 63.625 0.0161988 0.00949234 2024-08-17T17:31:53.069 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:52 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:53.102 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:53.139 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:53.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:53.241 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:53.275 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:53.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:53.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:52 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:53.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:52 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:31:53.377 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:53.404 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:53.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:53.506 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:53.538 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:53.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:53.642 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:53.674 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:53.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:53.775 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:53.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:31:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:31:53] "GET /metrics HTTP/1.1" 200 36597 "" "Prometheus/2.51.0" 2024-08-17T17:31:53.810 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:53.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:53.912 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:53.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:53.948 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:53.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:54.049 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:54.054 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 69 16 115733 115717 104.792 71.1875 0.00431476 0.00953689 2024-08-17T17:31:54.064 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:53 smithi063 ceph-mon[114756]: pgmap v1902: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 49683/340578 objects degraded (14.588%); 5586/340578 objects misplaced (1.640%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:31:54.064 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:53 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 49683/340578 objects degraded (14.588%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:54.064 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:53 smithi063 ceph-mon[109843]: pgmap v1902: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 49683/340578 objects degraded (14.588%); 5586/340578 objects misplaced (1.640%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:31:54.064 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:53 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 49683/340578 objects degraded (14.588%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:54.094 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:54.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:53 smithi162 ceph-mon[104668]: pgmap v1902: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 49683/340578 objects degraded (14.588%); 5586/340578 objects misplaced (1.640%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:31:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:53 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 49683/340578 objects degraded (14.588%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:54.196 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:54.224 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:54.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:54.231 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:31:54.325 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:54.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.372 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:54.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:54.473 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:54.503 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:54.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:54.604 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:54.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.648 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:54.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:54.749 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:54.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.779 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:54.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:54.880 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:54.924 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:54.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:54.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:55.026 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:55.050 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 70 16 116672 116656 104.133 58.6875 0.0217305 0.0095977 2024-08-17T17:31:55.053 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:55.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:55.155 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:55.189 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:55.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:55.291 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:55.329 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:55.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:55.430 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:55.460 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:55.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:55.562 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:55.592 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:55.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:55.694 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:55.733 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:55.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:55.834 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:55.861 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:55.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:55.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:55.963 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:56.003 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:56.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:56.051 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 71 16 117713 117697 103.582 65.0625 0.0188202 0.00964894 2024-08-17T17:31:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:55 smithi162 ceph-mon[104668]: pgmap v1903: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 49683/340578 objects degraded (14.588%); 5586/340578 objects misplaced (1.640%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:31:56.105 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:56.140 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:56.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:56.242 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:56.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.274 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:56.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:56.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:55 smithi063 ceph-mon[114756]: pgmap v1903: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 49683/340578 objects degraded (14.588%); 5586/340578 objects misplaced (1.640%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:31:56.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:55 smithi063 ceph-mon[109843]: pgmap v1903: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 49683/340578 objects degraded (14.588%); 5586/340578 objects misplaced (1.640%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:31:56.375 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:56.404 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:56.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:56.506 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:56.526 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:56.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:56.628 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:56.669 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:56.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:56.771 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:56.826 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:56.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:56.927 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:56.969 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:56.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:56.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:57.051 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 72 16 118743 118727 103.038 64.375 0.0037266 0.00969977 2024-08-17T17:31:57.072 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:57.098 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:57.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:57.200 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:57.237 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:57.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:57.292 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:31:57.292 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:31:57.292 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:31:57.292 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:31:57.292 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:31:57.340 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:57.389 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:57.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:57.490 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:57.522 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:57.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:57.624 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:57.653 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:57.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:57.755 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:31:57.789 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:57.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:31:57.890 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:57.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:57.934 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:57.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:57.991 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:58.035 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:58.048 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:31:58.048 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:31:58.054 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 73 16 119675 119659 102.42 58.25 0.0209686 0.00975669 2024-08-17T17:31:58.090 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:57 smithi063 ceph-mon[114756]: pgmap v1904: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 49995/341514 objects degraded (14.639%); 5586/341514 objects misplaced (1.636%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:31:58.090 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:57 smithi063 ceph-mon[109843]: pgmap v1904: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 49995/341514 objects degraded (14.639%); 5586/341514 objects misplaced (1.636%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:31:58.091 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:31:58.091 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:31:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:57 smithi162 ceph-mon[104668]: pgmap v1904: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 49995/341514 objects degraded (14.639%); 5586/341514 objects misplaced (1.636%); 1.2 MiB/s, 18 objects/s recovering 2024-08-17T17:31:58.096 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:58.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:58.127 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:31:58.128 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:31:58.160 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:31:58.160 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:31:58.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:31:58.196 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:31:58.199 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:58.228 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:31:58.228 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:31:58.229 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:31:58.245 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:58.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:58.264 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:31:58.264 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:31:58.347 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:58.395 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:58.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:58.497 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:31:58.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.544 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:58.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:31:58.645 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:58.686 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:58.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:58.788 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:58.835 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:58.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:58.937 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:58.970 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:31:58.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:58.984 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:58.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:59.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:58 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3994665291' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:59.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:58 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 49995/341514 objects degraded (14.639%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:59.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:58 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3994665291' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:59.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:58 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 49995/341514 objects degraded (14.639%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:59.054 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 74 16 120703 120687 101.904 64.25 0.00681858 0.0098078 2024-08-17T17:31:59.085 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:58 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3994665291' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:31:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:58 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 49995/341514 objects degraded (14.639%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:31:59.114 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:59.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:59.215 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:31:59.258 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:59.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:31:59.359 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:31:59.394 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:59.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:31:59.496 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:31:59.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.526 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:31:59.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:31:59.550 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:31:59.550 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:31:59.627 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:31:59.673 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:59.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:31:59.776 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:59.816 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:59.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:31:59.918 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:31:59.960 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:31:59.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:31:59.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:00.055 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 75 16 121723 121707 101.395 63.75 0.0169541 0.00985667 2024-08-17T17:32:00.062 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:00.084 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:31:59 smithi063 ceph-mon[109843]: pgmap v1905: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 7.2 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 51893/356145 objects degraded (14.571%); 5487/356145 objects misplaced (1.541%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:32:00.084 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:31:59 smithi063 ceph-mon[114756]: pgmap v1905: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 7.2 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 51893/356145 objects degraded (14.571%); 5487/356145 objects misplaced (1.541%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:32:00.091 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:00.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:31:59 smithi162 ceph-mon[104668]: pgmap v1905: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 7.2 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 51893/356145 objects degraded (14.571%); 5487/356145 objects misplaced (1.541%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:32:00.193 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:00.240 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:00.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:00.341 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:00.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.378 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:00.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:00.479 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:00.517 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:00.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:00.619 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:00.658 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:00.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:00.760 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:00.797 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:00.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:00.899 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:00.929 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:00.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:00.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:01.031 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:01.056 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 76 16 122780 122764 100.928 66.0625 0.00598857 0.00989963 2024-08-17T17:32:01.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.074 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:01.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:01.175 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:01.205 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:01.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:01.307 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:01.338 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:01.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:01.439 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:01.477 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:01.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:01.551 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:32:01.579 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:01.652 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:01.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:01.754 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:01.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.789 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:01.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:01.890 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:01.922 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:01.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:01.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:02.023 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:02.056 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 77 16 123797 123781 100.443 63.5625 0.0166617 0.00995111 2024-08-17T17:32:02.062 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:02.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:01 smithi162 ceph-mon[104668]: pgmap v1906: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 7.2 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 57 MiB/s wr, 909 op/s; 51893/356145 objects degraded (14.571%); 5487/356145 objects misplaced (1.541%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:32:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:02.137 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:01 smithi063 ceph-mon[109843]: pgmap v1906: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 7.2 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 57 MiB/s wr, 909 op/s; 51893/356145 objects degraded (14.571%); 5487/356145 objects misplaced (1.541%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:32:02.137 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:02.137 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:01 smithi063 ceph-mon[114756]: pgmap v1906: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 7.2 GiB data, 23 GiB used, 602 GiB / 626 GiB avail; 57 MiB/s wr, 909 op/s; 51893/356145 objects degraded (14.571%); 5487/356145 objects misplaced (1.541%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:32:02.138 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:02.164 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:02.177 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:32:02.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.224 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:02.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:02.325 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:02.352 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:02.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:02.454 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:02.516 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:02.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:02.618 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:02.662 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:02.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:02.764 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:02.810 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:02.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:02.836 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:32:02.912 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:02.952 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:02.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:02.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:03.055 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:03.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 78 16 124797 124781 99.9562 62.5 0.00392403 0.00999914 2024-08-17T17:32:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:02 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2594088877' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:03.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:02 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:03.096 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:03.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:03.178 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:02 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2594088877' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:03.178 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:02 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:03.178 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:02 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2594088877' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:03.178 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:02 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:03.198 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:03.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.240 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:03.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:03.341 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:03.378 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:03.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:03.480 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:03.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.512 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:03.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:03.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:32:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:32:03] "GET /metrics HTTP/1.1" 200 36604 "" "Prometheus/2.51.0" 2024-08-17T17:32:03.593 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:03.605 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:03.613 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:03.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.653 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:03.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:03.754 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:03.788 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:03.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:03.890 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:03.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:03.925 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:03.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:03.956 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:03 smithi162 ceph-mon[104668]: pgmap v1907: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 7.6 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 54086/371352 objects degraded (14.565%); 5392/371352 objects misplaced (1.452%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:32:03.956 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:03 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:32:03.956 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:03 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:03.956 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:03 smithi162 ceph-mon[104668]: osdmap e1304: 8 total, 7 up, 7 in 2024-08-17T17:32:03.956 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:03 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 54086/371352 objects degraded (14.565%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:03.957 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:03 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:03.957 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:03 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:04.026 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:04.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 79 16 125770 125754 99.4607 60.8125 0.0097991 0.0100491 2024-08-17T17:32:04.059 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:04.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:04.113 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:03 smithi063 ceph-mon[109843]: pgmap v1907: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 7.6 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 54086/371352 objects degraded (14.565%); 5392/371352 objects misplaced (1.452%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:32:04.113 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:03 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:32:04.113 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:03 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:04.113 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:03 smithi063 ceph-mon[109843]: osdmap e1304: 8 total, 7 up, 7 in 2024-08-17T17:32:04.114 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:03 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 54086/371352 objects degraded (14.565%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:04.114 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:03 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:04.114 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:03 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:04.114 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:03 smithi063 ceph-mon[114756]: pgmap v1907: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 7.6 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 54086/371352 objects degraded (14.565%); 5392/371352 objects misplaced (1.452%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:32:04.114 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:03 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:32:04.114 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:03 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:04.114 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:03 smithi063 ceph-mon[114756]: osdmap e1304: 8 total, 7 up, 7 in 2024-08-17T17:32:04.114 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:03 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 54086/371352 objects degraded (14.565%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:04.114 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:03 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:04.114 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:03 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:04.161 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:04.194 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:04.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:04.296 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:04.336 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:04.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:04.438 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:04.476 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:04.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:04.577 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:04.608 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:04.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:04.710 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:04.748 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:04.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:04.850 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:04.860 INFO:teuthology.orchestra.run.smithi063.stderr:set pool 10 pg_num to 26 2024-08-17T17:32:04.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:04.899 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:04.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:05.000 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:05.032 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:05.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:05.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:32:05.057210+0000 min lat: 0.00111601 max lat: 2.21144 avg lat: 0.0100965 2024-08-17T17:32:05.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:32:05.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 80 16 126769 126753 98.998 62.4375 0.0161292 0.0100965 2024-08-17T17:32:05.098 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:04 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/57062525' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:32:05.098 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:04 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:32:05.098 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:04 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/57062525' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:32:05.098 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:04 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:32:05.134 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:05.173 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:05.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:05.249 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:04 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/57062525' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:32:05.249 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:04 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"} : dispatch 2024-08-17T17:32:05.274 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:05.311 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:05.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:05.413 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:05.453 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:05.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:05.554 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:05.598 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:05.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:05.625 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:32:05.700 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:05.753 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:05.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:05.855 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:05.884 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:05.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:05.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:05.985 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:06.026 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:06.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:06.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 81 16 127876 127860 98.6299 69.1875 0.010358 0.010134 2024-08-17T17:32:06.128 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:06.137 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:05 smithi063 ceph-mon[114756]: pgmap v1909: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 7.6 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 54086/371352 objects degraded (14.565%); 5392/371352 objects misplaced (1.452%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:32:06.137 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:05 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:32:06.137 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:05 smithi063 ceph-mon[114756]: osdmap e1305: 8 total, 7 up, 7 in 2024-08-17T17:32:06.137 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:05 smithi063 ceph-mon[109843]: pgmap v1909: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 7.6 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 54086/371352 objects degraded (14.565%); 5392/371352 objects misplaced (1.452%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:32:06.137 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:05 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:32:06.137 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:05 smithi063 ceph-mon[109843]: osdmap e1305: 8 total, 7 up, 7 in 2024-08-17T17:32:06.167 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:06.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:06.269 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:06.307 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:06.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:06.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:05 smithi162 ceph-mon[104668]: pgmap v1909: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 7.6 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 54086/371352 objects degraded (14.565%); 5392/371352 objects misplaced (1.452%); 1.2 MiB/s, 19 objects/s recovering 2024-08-17T17:32:06.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:05 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2024-08-17T17:32:06.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:05 smithi162 ceph-mon[104668]: osdmap e1305: 8 total, 7 up, 7 in 2024-08-17T17:32:06.409 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:06.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.444 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:06.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:06.545 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:06.572 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:06.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:06.674 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:06.705 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:06.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:06.807 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:06.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.841 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:06.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:06.942 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:06.973 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:06.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:06.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:07.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 82 16 128770 128754 98.1085 55.875 0.0432758 0.0101658 2024-08-17T17:32:07.076 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:07.113 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:07.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:07.215 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:07.236 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:07.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:07.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:06 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:32:07.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:06 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:32:07.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:06 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:32:07.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:06 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:32:07.338 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:07.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:06 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:32:07.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:06 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"} : dispatch 2024-08-17T17:32:07.361 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:07.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:07.463 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:07.484 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:07.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:07.586 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:07.608 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:07.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:07.710 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:07.731 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:07.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:07.833 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:07.854 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:07.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:07.958 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:07.987 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:07.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:07.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:08.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 83 16 128984 128968 97.0878 13.375 0.00343143 0.0102933 2024-08-17T17:32:08.089 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:08.116 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:08.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:08.173 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:07 smithi063 ceph-mon[109843]: pgmap v1911: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 7.6 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 42 MiB/s wr, 675 op/s; 54422/372360 objects degraded (14.615%); 5392/372360 objects misplaced (1.448%); 760 KiB/s, 11 objects/s recovering 2024-08-17T17:32:08.174 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:07 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:32:08.174 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:07 smithi063 ceph-mon[109843]: osdmap e1306: 8 total, 7 up, 7 in 2024-08-17T17:32:08.174 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:07 smithi063 ceph-mon[114756]: pgmap v1911: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 7.6 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 42 MiB/s wr, 675 op/s; 54422/372360 objects degraded (14.615%); 5392/372360 objects misplaced (1.448%); 760 KiB/s, 11 objects/s recovering 2024-08-17T17:32:08.174 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:07 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:32:08.174 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:07 smithi063 ceph-mon[114756]: osdmap e1306: 8 total, 7 up, 7 in 2024-08-17T17:32:08.218 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:08.239 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:08.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:08.341 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:08.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:07 smithi162 ceph-mon[104668]: pgmap v1911: 119 pgs: 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 11 active+undersized+degraded, 65 active+clean; 7.6 GiB data, 24 GiB used, 601 GiB / 626 GiB avail; 42 MiB/s wr, 675 op/s; 54422/372360 objects degraded (14.615%); 5392/372360 objects misplaced (1.448%); 760 KiB/s, 11 objects/s recovering 2024-08-17T17:32:08.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:07 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2024-08-17T17:32:08.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:07 smithi162 ceph-mon[104668]: osdmap e1306: 8 total, 7 up, 7 in 2024-08-17T17:32:08.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.381 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:08.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:08.482 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:08.515 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:08.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:08.617 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:08.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.655 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:08.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:08.756 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:08.792 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:08.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:08.894 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:08.921 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:08.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:08.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:09.022 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:09.052 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:09.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:09.057 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 84 16 130256 130240 96.8784 79.5 0.00478758 0.0103178 2024-08-17T17:32:09.096 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:08 smithi063 ceph-mon[109843]: osdmap e1307: 8 total, 7 up, 7 in 2024-08-17T17:32:09.097 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:08 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 54422/372360 objects degraded (14.615%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:09.097 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:08 smithi063 ceph-mon[114756]: osdmap e1307: 8 total, 7 up, 7 in 2024-08-17T17:32:09.097 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:08 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 54422/372360 objects degraded (14.615%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:09.097 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:32:09.097 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:32:09.154 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:09.178 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:09.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:09.283 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:09.321 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:09.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:09.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:08 smithi162 ceph-mon[104668]: osdmap e1307: 8 total, 7 up, 7 in 2024-08-17T17:32:09.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:08 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 54422/372360 objects degraded (14.615%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:09.424 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:09.455 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:09.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:09.557 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:09.594 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:09.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:09.714 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:09.747 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:09.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:09.849 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:09.880 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:09.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:09.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:09.982 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:10.009 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:10.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:10.059 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 85 16 131745 131729 96.8319 93.0625 0.0126309 0.0103214 2024-08-17T17:32:10.093 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:32:10.110 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:10.152 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:09 smithi063 ceph-mon[109843]: pgmap v1914: 129 pgs: 10 peering, 10 unknown, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 9 active+undersized+degraded, 57 active+clean; 7.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 9.4 MiB/s wr, 151 op/s; 40290/373920 objects degraded (10.775%); 5289/373920 objects misplaced (1.414%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:32:10.152 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:09 smithi063 ceph-mon[114756]: pgmap v1914: 129 pgs: 10 peering, 10 unknown, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 9 active+undersized+degraded, 57 active+clean; 7.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 9.4 MiB/s wr, 151 op/s; 40290/373920 objects degraded (10.775%); 5289/373920 objects misplaced (1.414%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:32:10.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.203 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:10.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:10.304 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:10.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:09 smithi162 ceph-mon[104668]: pgmap v1914: 129 pgs: 10 peering, 10 unknown, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 9 active+undersized+degraded, 57 active+clean; 7.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 9.4 MiB/s wr, 151 op/s; 40290/373920 objects degraded (10.775%); 5289/373920 objects misplaced (1.414%); 1.1 MiB/s, 18 objects/s recovering 2024-08-17T17:32:10.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.358 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:10.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:10.455 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:10.460 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:10.473 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:32:10.474 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:32:10.520 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:32:10.520 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:32:10.536 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:10.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:10.554 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:32:10.554 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:32:10.588 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:32:10.588 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:32:10.619 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:32:10.619 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:32:10.638 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:10.653 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:32:10.654 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:32:10.659 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:32:10.689 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:10.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:10.692 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:32:10.692 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:32:10.791 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:10.826 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:10.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:10.928 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:10.959 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:10.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:10.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:10 smithi063 ceph-mon[114756]: 10.8 scrub starts 2024-08-17T17:32:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:10 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/480082074' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:10 smithi063 ceph-mon[109843]: 10.8 scrub starts 2024-08-17T17:32:11.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:10 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/480082074' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:11.059 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 86 16 132875 132859 96.5272 70.625 0.00316698 0.0103532 2024-08-17T17:32:11.061 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:11.088 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:11.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:11.180 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0] out_osds: [1] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:32:11.180 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:32:11.180 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-08-17T17:32:11.180 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:32:11.180 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:32:11.181 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-08-17T17:32:11.189 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:11.209 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:10 smithi162 ceph-mon[104668]: 10.8 scrub starts 2024-08-17T17:32:11.209 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:10 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/480082074' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:11.230 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:11.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:11.332 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:11.375 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:11.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:11.477 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:11.483 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:32:11.505 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:11.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:11.607 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:11.642 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:11.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:11.744 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:11.768 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:11.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:11.792 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:32:11.792 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:32:11.870 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:11.909 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:11.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:11.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:12.011 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:11 smithi063 ceph-mon[109843]: pgmap v1915: 129 pgs: 10 peering, 10 unknown, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 9 active+undersized+degraded, 57 active+clean; 7.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 8.9 MiB/s wr, 142 op/s; 40290/373920 objects degraded (10.775%); 5289/373920 objects misplaced (1.414%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:32:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:11 smithi063 ceph-mon[109843]: 10.5 deep-scrub starts 2024-08-17T17:32:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:11 smithi063 ceph-mon[109843]: 10.8 scrub ok 2024-08-17T17:32:12.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:11 smithi063 ceph-mon[114756]: pgmap v1915: 129 pgs: 10 peering, 10 unknown, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 9 active+undersized+degraded, 57 active+clean; 7.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 8.9 MiB/s wr, 142 op/s; 40290/373920 objects degraded (10.775%); 5289/373920 objects misplaced (1.414%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:32:12.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:11 smithi063 ceph-mon[114756]: 10.5 deep-scrub starts 2024-08-17T17:32:12.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:11 smithi063 ceph-mon[114756]: 10.8 scrub ok 2024-08-17T17:32:12.048 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:12.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:12.059 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 87 16 134046 134030 96.2589 73.1875 0.043318 0.0103829 2024-08-17T17:32:12.150 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:12.188 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:12.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:12.290 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:12.336 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:12.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:12.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:11 smithi162 ceph-mon[104668]: pgmap v1915: 129 pgs: 10 peering, 10 unknown, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 9 active+undersized+degraded, 57 active+clean; 7.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 8.9 MiB/s wr, 142 op/s; 40290/373920 objects degraded (10.775%); 5289/373920 objects misplaced (1.414%); 1.1 MiB/s, 17 objects/s recovering 2024-08-17T17:32:12.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:11 smithi162 ceph-mon[104668]: 10.5 deep-scrub starts 2024-08-17T17:32:12.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:11 smithi162 ceph-mon[104668]: 10.8 scrub ok 2024-08-17T17:32:12.438 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:12.477 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:12.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:12.579 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:12.609 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:12.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:12.711 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:12.754 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:12.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:12.856 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:12.892 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:12.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:12.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:12.994 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:13.018 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:13.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:13.059 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 88 16 135210 135194 95.9917 72.75 0.0100738 0.0104125 2024-08-17T17:32:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:12 smithi162 ceph-mon[104668]: 10.15 scrub starts 2024-08-17T17:32:13.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:13.152 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:13.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:13.254 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:13.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:12 smithi063 ceph-mon[114756]: 10.15 scrub starts 2024-08-17T17:32:13.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:12 smithi063 ceph-mon[109843]: 10.15 scrub starts 2024-08-17T17:32:13.295 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:13.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:13.397 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:13.422 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:13.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:13.524 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:13.557 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:13.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:13.659 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:13.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.690 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:13.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:13.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:32:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:32:13] "GET /metrics HTTP/1.1" 200 36604 "" "Prometheus/2.51.0" 2024-08-17T17:32:13.792 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:13.793 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:32:13.817 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:32:13.817 INFO:teuthology.orchestra.run.smithi162.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:32:13.817 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:32:13.830 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:13.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:13.932 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:13.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:13.977 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:13.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:14.059 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 89 16 136193 136177 95.6035 61.4375 0.0241341 0.0104547 2024-08-17T17:32:14.078 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:13 smithi162 ceph-mon[104668]: pgmap v1916: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 1 active+clean+scrubbing+deep, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 69 active+clean; 8.2 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 51 MiB/s wr, 811 op/s; 58515/402375 objects degraded (14.542%); 5147/402375 objects misplaced (1.279%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:32:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:13 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 58515/402375 objects degraded (14.542%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:14.109 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:14.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:14.211 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:14.233 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:14.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:14.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:13 smithi063 ceph-mon[114756]: pgmap v1916: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 1 active+clean+scrubbing+deep, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 69 active+clean; 8.2 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 51 MiB/s wr, 811 op/s; 58515/402375 objects degraded (14.542%); 5147/402375 objects misplaced (1.279%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:32:14.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:13 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 58515/402375 objects degraded (14.542%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:14.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:13 smithi063 ceph-mon[109843]: pgmap v1916: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 1 active+clean+scrubbing+deep, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 69 active+clean; 8.2 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 51 MiB/s wr, 811 op/s; 58515/402375 objects degraded (14.542%); 5147/402375 objects misplaced (1.279%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:32:14.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:13 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 58515/402375 objects degraded (14.542%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:14.335 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:14.361 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:14.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:14.463 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:14.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.500 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:14.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:14.601 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:14.639 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:14.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:14.741 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:14.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.768 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:14.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:14.869 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:14.898 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:14.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:14.916 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:14.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:15.017 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:15.057 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:15.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:15.059 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 90 3 137182 137179 95.237 62.625 0.0787464 0.0104942 2024-08-17T17:32:15.158 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:15.182 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:15.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:15.283 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:15.306 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:15.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:15.408 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:15.430 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:15.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:15.531 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:15.558 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:15.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:15.660 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:15.682 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:15.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:15.784 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:15.809 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:15.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:15.910 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:15.937 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:15.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:15.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:15.981 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:15 smithi063 ceph-mon[114756]: pgmap v1917: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 1 active+clean+scrubbing+deep, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 69 active+clean; 8.2 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 46 MiB/s wr, 732 op/s; 58515/402375 objects degraded (14.542%); 5147/402375 objects misplaced (1.279%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:32:15.981 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:15 smithi063 ceph-mon[109843]: pgmap v1917: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 1 active+clean+scrubbing+deep, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 69 active+clean; 8.2 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 46 MiB/s wr, 732 op/s; 58515/402375 objects degraded (14.542%); 5147/402375 objects misplaced (1.279%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:32:16.040 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:16.059 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Total time run: 90.0442 2024-08-17T17:32:16.059 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Total writes made: 137182 2024-08-17T17:32:16.059 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Write size: 65536 2024-08-17T17:32:16.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Object size: 65536 2024-08-17T17:32:16.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Bandwidth (MB/sec): 95.2186 2024-08-17T17:32:16.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev Bandwidth: 46.8628 2024-08-17T17:32:16.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max bandwidth (MB/sec): 276.875 2024-08-17T17:32:16.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min bandwidth (MB/sec): 0 2024-08-17T17:32:16.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Average IOPS: 1523 2024-08-17T17:32:16.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev IOPS: 749.816 2024-08-17T17:32:16.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max IOPS: 4430 2024-08-17T17:32:16.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min IOPS: 0 2024-08-17T17:32:16.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Average Latency(s): 0.0104949 2024-08-17T17:32:16.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev Latency(s): 0.0274826 2024-08-17T17:32:16.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max latency(s): 2.21144 2024-08-17T17:32:16.060 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min latency(s): 0.00111601 2024-08-17T17:32:16.065 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:16.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:16.071 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:32:16.167 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:16.207 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:16.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:16.309 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:16.333 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:16.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:16.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:15 smithi162 ceph-mon[104668]: pgmap v1917: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 1 active+clean+scrubbing+deep, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 69 active+clean; 8.2 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 46 MiB/s wr, 732 op/s; 58515/402375 objects degraded (14.542%); 5147/402375 objects misplaced (1.279%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:32:16.436 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:16.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.475 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:16.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:16.577 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:16.601 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:16.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:16.703 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:16.739 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:16.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:16.841 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:16.868 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:16.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:16.970 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:16.996 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:16.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:16.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:17.098 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:17.126 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:17.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:17.192 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:16 smithi063 ceph-mon[109843]: 10.15 scrub ok 2024-08-17T17:32:17.192 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:16 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:17.192 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:17.192 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:16 smithi063 ceph-mon[114756]: 10.15 scrub ok 2024-08-17T17:32:17.192 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:16 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:17.193 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:17.228 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:17.269 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:17.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:17.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:16 smithi162 ceph-mon[104668]: 10.15 scrub ok 2024-08-17T17:32:17.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:16 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:17.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:17.370 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:17.398 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:17.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:17.500 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:17.536 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:17.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:17.637 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:17.665 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:17.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:17.767 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:17.797 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:17.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:17.899 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:17.939 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:17.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:17.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:17.968 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:32:17.969 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:32:17.969 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.969 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.2.0:120544 10.5 e1307/1306 10:a359c566:::benchmark_data_smithi063_294527_object105132:head v 1307'9752, mlcod=1307'9740)", 2024-08-17T17:32:17.969 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:17.955813+0000", 2024-08-17T17:32:17.969 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.011278794, 2024-08-17T17:32:17.969 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011304514, 2024-08-17T17:32:17.969 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:17.969 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:17.969 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:32:17.969 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:17.969 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.969 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:17.970 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.955813+0000", 2024-08-17T17:32:17.970 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:17.970 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.970 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.970 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:17.970 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.955813+0000", 2024-08-17T17:32:17.970 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:17.970 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.970 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.970 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:17.970 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.955813+0000", 2024-08-17T17:32:17.970 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3200000000000001e-07 2024-08-17T17:32:17.970 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.971 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.971 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:17.971 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.955817+0000", 2024-08-17T17:32:17.971 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4659999999999999e-06 2024-08-17T17:32:17.971 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.971 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.971 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:17.971 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.955823+0000", 2024-08-17T17:32:17.971 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2589999999999997e-06 2024-08-17T17:32:17.971 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.971 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.971 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:17.971 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.955827+0000", 2024-08-17T17:32:17.972 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2549999999999999e-06 2024-08-17T17:32:17.972 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.972 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.972 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:17.972 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.958848+0000", 2024-08-17T17:32:17.972 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0030204609999999999 2024-08-17T17:32:17.972 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.972 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.972 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:17.972 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.958869+0000", 2024-08-17T17:32:17.972 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1461999999999999e-05 2024-08-17T17:32:17.972 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:17.972 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:17.972 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:17.973 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.973 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.973 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.2.0:120546 10.5 e1307/1306 10:a35a55c7:::benchmark_data_smithi063_294527_object17208:head v 1307'9753, mlcod=1307'9740)", 2024-08-17T17:32:17.973 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:17.959248+0000", 2024-08-17T17:32:17.973 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0078437209999999997, 2024-08-17T17:32:17.973 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00792146, 2024-08-17T17:32:17.973 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:17.973 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:17.973 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:32:17.973 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:17.973 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.973 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:17.973 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.959248+0000", 2024-08-17T17:32:17.974 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:17.974 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.974 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.974 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:17.974 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.959248+0000", 2024-08-17T17:32:17.974 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:17.974 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.974 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.974 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:17.974 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.959248+0000", 2024-08-17T17:32:17.974 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.39e-07 2024-08-17T17:32:17.974 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.974 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.974 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:17.975 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.959252+0000", 2024-08-17T17:32:17.975 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9280000000000003e-06 2024-08-17T17:32:17.975 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.975 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.975 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:17.975 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.959259+0000", 2024-08-17T17:32:17.975 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9959999999999996e-06 2024-08-17T17:32:17.975 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.975 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.975 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:17.975 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.959264+0000", 2024-08-17T17:32:17.975 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1939999999999997e-06 2024-08-17T17:32:17.975 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.976 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.976 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:17.976 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.961931+0000", 2024-08-17T17:32:17.976 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0026662299999999999 2024-08-17T17:32:17.976 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.976 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.976 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:17.976 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.961950+0000", 2024-08-17T17:32:17.976 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8970999999999998e-05 2024-08-17T17:32:17.976 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:17.976 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:17.976 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:17.976 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.976 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.977 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.2.0:120548 10.5 e1307/1306 10:a35b36cc:::benchmark_data_smithi063_294527_object82002:head v 1307'9754, mlcod=1307'9740)", 2024-08-17T17:32:17.977 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:17.962674+0000", 2024-08-17T17:32:17.977 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0044177660000000001, 2024-08-17T17:32:17.977 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0045465159999999996, 2024-08-17T17:32:17.977 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:17.977 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:17.977 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:32:17.977 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:17.977 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.977 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:17.977 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.962674+0000", 2024-08-17T17:32:17.977 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:17.978 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.978 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.978 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:17.978 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.962674+0000", 2024-08-17T17:32:17.978 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:17.978 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.978 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.978 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:17.978 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.962675+0000", 2024-08-17T17:32:17.978 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7700000000000003e-07 2024-08-17T17:32:17.978 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.978 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.978 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:17.979 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.962681+0000", 2024-08-17T17:32:17.979 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3470000000000004e-06 2024-08-17T17:32:17.979 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.979 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.979 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:17.979 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.962692+0000", 2024-08-17T17:32:17.979 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0322e-05 2024-08-17T17:32:17.979 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.979 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.979 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:17.979 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.962698+0000", 2024-08-17T17:32:17.979 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8199999999999999e-06 2024-08-17T17:32:17.979 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.980 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.980 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:17.980 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.964717+0000", 2024-08-17T17:32:17.980 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0020182059999999998 2024-08-17T17:32:17.980 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.980 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.980 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:17.980 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.964735+0000", 2024-08-17T17:32:17.980 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8598e-05 2024-08-17T17:32:17.980 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:17.980 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:17.980 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:17.980 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.981 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.981 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.2.0:120549 10.5 e1307/1306)", 2024-08-17T17:32:17.981 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:17.964211+0000", 2024-08-17T17:32:17.981 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0028808879999999999, 2024-08-17T17:32:17.981 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0030567210000000001, 2024-08-17T17:32:17.981 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:17.981 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:17.981 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "queued for pg", 2024-08-17T17:32:17.981 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:17.981 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.981 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:17.981 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.964211+0000", 2024-08-17T17:32:17.981 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:17.982 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.982 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.982 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:17.982 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.964211+0000", 2024-08-17T17:32:17.982 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:17.982 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.982 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.982 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:17.982 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.964212+0000", 2024-08-17T17:32:17.982 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1600000000000001e-07 2024-08-17T17:32:17.982 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.982 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.983 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:17.983 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.964218+0000", 2024-08-17T17:32:17.983 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2620000000000004e-06 2024-08-17T17:32:17.983 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.983 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.983 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:17.983 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.964275+0000", 2024-08-17T17:32:17.983 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6848999999999997e-05 2024-08-17T17:32:17.983 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.983 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.983 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:17.983 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.964284+0000", 2024-08-17T17:32:17.983 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7769999999999997e-06 2024-08-17T17:32:17.984 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:17.984 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:17.984 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:17.984 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.984 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.984 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:139380 10.2 e1307/1306 10:43a746e6:::benchmark_data_smithi063_294527_object2195:head v 1307'8448, mlcod=1307'8446)", 2024-08-17T17:32:17.984 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:17.966996+0000", 2024-08-17T17:32:17.984 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 9.5730000000000007e-05, 2024-08-17T17:32:17.984 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00031454399999999999, 2024-08-17T17:32:17.984 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:17.984 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:17.984 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:32:17.984 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:17.984 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.985 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:17.985 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.966996+0000", 2024-08-17T17:32:17.985 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:17.985 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.985 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.985 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:17.985 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.966996+0000", 2024-08-17T17:32:17.985 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:17.985 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.985 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.985 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:17.985 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.966997+0000", 2024-08-17T17:32:17.985 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2900000000000002e-07 2024-08-17T17:32:17.985 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.986 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.986 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:17.986 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.967004+0000", 2024-08-17T17:32:17.986 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8870000000000003e-06 2024-08-17T17:32:17.986 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.986 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.986 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:17.986 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.967020+0000", 2024-08-17T17:32:17.986 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5959000000000001e-05 2024-08-17T17:32:17.986 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.986 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.986 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:17.987 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.967028+0000", 2024-08-17T17:32:17.987 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.5620000000000004e-06 2024-08-17T17:32:17.987 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.987 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.987 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:17.987 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.967167+0000", 2024-08-17T17:32:17.987 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013915800000000001 2024-08-17T17:32:17.987 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:17.987 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:17.987 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:17.987 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:17.967187+0000", 2024-08-17T17:32:17.987 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.995e-05 2024-08-17T17:32:17.987 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:17.987 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:17.988 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:17.988 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:17.988 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-08-17T17:32:17.988 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 5 2024-08-17T17:32:17.988 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:32:18.041 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:18.066 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:18.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:18.152 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:17 smithi063 ceph-mon[114756]: pgmap v1918: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 1 active+clean+scrubbing+deep, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 69 active+clean; 8.2 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 40 MiB/s wr, 637 op/s; 58787/403191 objects degraded (14.580%); 5147/403191 objects misplaced (1.277%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:32:18.152 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:17 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/19931774' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:32:18.152 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:17 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:32:18.156 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:17 smithi063 ceph-mon[109843]: pgmap v1918: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 1 active+clean+scrubbing+deep, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 69 active+clean; 8.2 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 40 MiB/s wr, 637 op/s; 58787/403191 objects degraded (14.580%); 5147/403191 objects misplaced (1.277%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:32:18.156 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:17 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/19931774' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:32:18.156 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:17 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:32:18.167 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:18.203 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:18.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:18.304 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:18.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:17 smithi162 ceph-mon[104668]: pgmap v1918: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 1 active+clean+scrubbing+deep, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 69 active+clean; 8.2 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 40 MiB/s wr, 637 op/s; 58787/403191 objects degraded (14.580%); 5147/403191 objects misplaced (1.277%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:32:18.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:17 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/19931774' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:32:18.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:17 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:32:18.347 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:18.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:18.386 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:32:18.449 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:18.486 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:18.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:18.588 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:18.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.628 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:18.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:18.729 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:18.764 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:18.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:18.866 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:18.904 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:18.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:18.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:19.006 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:19.037 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:19.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:19.139 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:19.167 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:19.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:19.268 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:19.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:18 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 58787/403191 objects degraded (14.580%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:19.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:18 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:19.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:18 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:32:19.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:18 smithi063 ceph-mon[114756]: osdmap e1308: 8 total, 7 up, 7 in 2024-08-17T17:32:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:18 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 58787/403191 objects degraded (14.580%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:18 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:18 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:32:19.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:18 smithi063 ceph-mon[109843]: osdmap e1308: 8 total, 7 up, 7 in 2024-08-17T17:32:19.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.310 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:19.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:19.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:18 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 58787/403191 objects degraded (14.580%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:19.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:18 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:19.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:18 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:32:19.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:18 smithi162 ceph-mon[104668]: osdmap e1308: 8 total, 7 up, 7 in 2024-08-17T17:32:19.411 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:19.454 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:19.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:19.555 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:19.606 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:19.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:19.708 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:19.736 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:19.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:19.838 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:19.853 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0] out_osds: [1] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:32:19.853 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:32:19.853 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:32:19.854 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.395891 2024-08-17T17:32:19.854 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 0 0.3958913496135553 2024-08-17T17:32:19.873 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:19.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:19.899 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:19.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:19.911 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:32:20.000 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:20.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.026 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:20.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:20.130 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:20.205 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:20.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:20.214 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:19 smithi063 ceph-mon[114756]: pgmap v1920: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 8.3 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 102 B/s rd, 26 MiB/s wr, 548 op/s; 59431/407889 objects degraded (14.570%); 4959/407889 objects misplaced (1.216%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:32:20.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:19 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:20.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:19 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:20.216 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:19 smithi063 ceph-mon[109843]: pgmap v1920: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 8.3 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 102 B/s rd, 26 MiB/s wr, 548 op/s; 59431/407889 objects degraded (14.570%); 4959/407889 objects misplaced (1.216%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:32:20.216 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:19 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:20.217 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:19 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:20.307 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:20.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:19 smithi162 ceph-mon[104668]: pgmap v1920: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 8.3 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 102 B/s rd, 26 MiB/s wr, 548 op/s; 59431/407889 objects degraded (14.570%); 4959/407889 objects misplaced (1.216%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:32:20.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:19 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:20.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:19 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:20.349 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:20.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:20.451 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:20.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.478 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:20.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:20.579 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:20.621 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:20.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:20.722 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:20.756 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:20.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:20.857 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:20.884 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:20.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:20.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:20.986 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:21.024 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:21.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:21.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:20 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:21.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:20 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:21.042 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:20 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:21.042 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:20 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:21.042 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:20 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:21.043 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:20 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:21.126 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:21.170 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:21.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:21.272 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:21.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:21.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:21.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:20 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:32:21.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:20 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:32:21.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:20 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:21.401 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:21.423 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:21.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:21.525 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:21.554 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:21.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:21.656 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:21.678 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:21.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:21.779 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:21.808 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:21.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:21.910 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:21.936 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:21.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:21.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:22.002 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:22.037 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:22.074 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:22.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:22.176 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:22.208 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:22.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:22.261 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:21 smithi063 ceph-mon[114756]: pgmap v1921: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 8.3 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 102 B/s rd, 26 MiB/s wr, 548 op/s; 59431/407889 objects degraded (14.570%); 4959/407889 objects misplaced (1.216%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:32:22.262 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:21 smithi063 ceph-mon[109843]: pgmap v1921: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 8.3 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 102 B/s rd, 26 MiB/s wr, 548 op/s; 59431/407889 objects degraded (14.570%); 4959/407889 objects misplaced (1.216%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:32:22.310 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:22.336 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:22.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:22.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:21 smithi162 ceph-mon[104668]: pgmap v1921: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 8.3 GiB data, 27 GiB used, 599 GiB / 626 GiB avail; 102 B/s rd, 26 MiB/s wr, 548 op/s; 59431/407889 objects degraded (14.570%); 4959/407889 objects misplaced (1.216%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:32:22.439 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:22.478 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:22.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:22.580 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:22.626 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:22.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:22.728 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:22.757 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:22.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:22.825 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:32:22.858 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:22.883 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:22.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:22.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:22.985 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:23.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.026 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:23.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:23.083 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:22 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1624863887' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:23.083 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:22 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1624863887' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:23.127 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:23.171 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:23.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:23.240 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:32:23.240 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:32:23.240 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:32:23.240 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:32:23.240 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:32:23.241 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:32:23.241 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:32:23.241 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:32:23.241 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:32:23.241 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:32:23.241 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:32:23.242 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:32:23.242 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:32:23.242 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:32:23.242 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:32:23.272 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:23.298 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:23.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:23.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:22 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1624863887' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:23.400 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:23.436 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:23.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:23.444 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:23.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:32:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:32:23] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.51.0" 2024-08-17T17:32:23.538 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:23.567 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:23.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:23.669 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:23.700 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:23.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.701 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:23.802 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:23.835 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:23.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:23.936 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:23.975 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:23.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:23.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:24.077 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:24.124 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:24.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:24.225 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:24.252 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:24.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:24.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:23 smithi063 ceph-mon[114756]: pgmap v1922: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 8.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 0 B/s wr, 572 op/s; 56967/391140 objects degraded (14.564%); 4831/391140 objects misplaced (1.235%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:32:24.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:23 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 56967/391140 objects degraded (14.564%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:24.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:23 smithi063 ceph-mon[109843]: pgmap v1922: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 8.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 0 B/s wr, 572 op/s; 56967/391140 objects degraded (14.564%); 4831/391140 objects misplaced (1.235%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:32:24.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:23 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 56967/391140 objects degraded (14.564%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:24.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:23 smithi162 ceph-mon[104668]: pgmap v1922: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 8.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 0 B/s wr, 572 op/s; 56967/391140 objects degraded (14.564%); 4831/391140 objects misplaced (1.235%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:32:24.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:23 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 56967/391140 objects degraded (14.564%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:24.354 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:24.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.397 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:24.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:24.498 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:24.540 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:24.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:24.642 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:24.674 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:24.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:24.776 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:24.799 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:24.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:24.900 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:24.928 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:24.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:24.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:25.047 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:25.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.085 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:25.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:25.186 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:25.221 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:25.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:25.322 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:25.362 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:25.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:25.464 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:25.492 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:25.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:25.594 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:25.627 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:25.729 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:25.774 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:25.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:25.876 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:25.911 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:25.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:25.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:26.013 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:26.051 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:26.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:26.153 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:26.192 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:26.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:26.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:25 smithi063 ceph-mon[114756]: pgmap v1923: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 8.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 0 B/s wr, 572 op/s; 56967/391140 objects degraded (14.564%); 4831/391140 objects misplaced (1.235%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:32:26.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:25 smithi063 ceph-mon[109843]: pgmap v1923: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 8.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 0 B/s wr, 572 op/s; 56967/391140 objects degraded (14.564%); 4831/391140 objects misplaced (1.235%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:32:26.293 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:26.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.334 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:26.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:26.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:25 smithi162 ceph-mon[104668]: pgmap v1923: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 8.0 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 0 B/s wr, 572 op/s; 56967/391140 objects degraded (14.564%); 4831/391140 objects misplaced (1.235%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:32:26.435 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:26.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.470 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:26.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:26.571 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:26.604 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:26.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:26.706 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:26.743 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:26.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:26.844 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:26.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:26.871 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:26.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:26.972 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:27.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.010 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:27.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:27.111 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:27.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.149 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:27.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:27.250 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:27.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.293 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:27.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:27.393 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:27.417 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:27.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:27.519 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:27.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.556 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:27.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:27.657 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:27.693 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:27.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:27.794 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:27.821 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:27.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:27.923 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:27.956 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:27.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:27.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:28.058 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:28.092 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:28.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:28.194 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:28.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.229 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:28.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:28.266 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:27 smithi063 ceph-mon[114756]: pgmap v1924: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 7.9 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 0 B/s wr, 577 op/s; 56633/390138 objects degraded (14.516%); 4831/390138 objects misplaced (1.238%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:32:28.270 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:27 smithi063 ceph-mon[109843]: pgmap v1924: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 7.9 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 0 B/s wr, 577 op/s; 56633/390138 objects degraded (14.516%); 4831/390138 objects misplaced (1.238%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:32:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:32:28.307 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:32:28.330 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:28.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:27 smithi162 ceph-mon[104668]: pgmap v1924: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 7.9 GiB data, 26 GiB used, 600 GiB / 626 GiB avail; 102 B/s rd, 0 B/s wr, 577 op/s; 56633/390138 objects degraded (14.516%); 4831/390138 objects misplaced (1.238%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:32:28.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.376 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:28.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:28.477 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:28.514 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:28.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:28.616 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:28.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.652 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:28.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:28.753 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:28.788 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:28.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:28.890 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:28.922 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:28.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:28.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:29.023 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:29.051 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:29.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:29.153 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:29.194 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:29.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:29.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:28 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 56633/390138 objects degraded (14.516%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:29.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:28 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 56633/390138 objects degraded (14.516%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:29.297 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:29.327 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:29.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:29.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:28 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 56633/390138 objects degraded (14.516%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:29.429 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:29.472 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:29.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.497 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:29.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:29.598 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:29.655 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:29.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:29.757 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:29.784 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:29.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:29.885 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:29.911 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:29.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:29.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:30.013 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:30.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.053 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:30.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:30.154 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:30.196 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:30.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:30.244 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:29 smithi063 ceph-mon[109843]: pgmap v1925: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 7.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 54578/374814 objects degraded (14.561%); 4696/374814 objects misplaced (1.253%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:32:30.244 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:29 smithi063 ceph-mon[114756]: pgmap v1925: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 7.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 54578/374814 objects degraded (14.561%); 4696/374814 objects misplaced (1.253%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:32:30.298 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:30.308 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:32:30.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.337 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:30.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:30.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:29 smithi162 ceph-mon[104668]: pgmap v1925: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 7.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 54578/374814 objects degraded (14.561%); 4696/374814 objects misplaced (1.253%); 1.6 MiB/s, 25 objects/s recovering 2024-08-17T17:32:30.438 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:30.489 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:30.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:30.590 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:30.642 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:30.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:30.721 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:32:30.721 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:32:30.744 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:30.767 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:30.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:30.869 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:30.902 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:30.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:30.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:31.004 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:31.008 INFO:teuthology.orchestra.run.smithi063.stderr:set osd.0 primary-affinity to 0.395891 (8259452) 2024-08-17T17:32:31.030 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:31.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:31.132 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:31.159 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:31.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:31.261 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:31.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:31 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1011001135' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.3958913496135553} : dispatch 2024-08-17T17:32:31.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:31 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.3958913496135553} : dispatch 2024-08-17T17:32:31.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:31 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1011001135' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.3958913496135553} : dispatch 2024-08-17T17:32:31.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:31 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.3958913496135553} : dispatch 2024-08-17T17:32:31.301 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:31.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:31.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:31 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1011001135' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.3958913496135553} : dispatch 2024-08-17T17:32:31.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:31 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 0.3958913496135553} : dispatch 2024-08-17T17:32:31.403 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:31.445 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:31.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:31.547 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:31.586 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:31.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:31.688 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:31.712 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:31.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:31.737 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:32:31.814 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:31.839 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:31.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:31.941 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:31.968 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:31.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:31.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:32.070 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:32.099 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:32.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:32.201 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:32.229 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:32.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:32.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:32 smithi063 ceph-mon[109843]: pgmap v1926: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 7.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 918 op/s; 54578/374814 objects degraded (14.561%); 4696/374814 objects misplaced (1.253%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:32:32.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:32 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.3958913496135553}]': finished 2024-08-17T17:32:32.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:32 smithi063 ceph-mon[109843]: osdmap e1309: 8 total, 7 up, 7 in 2024-08-17T17:32:32.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:32 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:32.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:32 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:32.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:32 smithi063 ceph-mon[114756]: pgmap v1926: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 7.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 918 op/s; 54578/374814 objects degraded (14.561%); 4696/374814 objects misplaced (1.253%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:32:32.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:32 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.3958913496135553}]': finished 2024-08-17T17:32:32.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:32 smithi063 ceph-mon[114756]: osdmap e1309: 8 total, 7 up, 7 in 2024-08-17T17:32:32.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:32 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:32.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:32 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:32.330 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:32.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:32 smithi162 ceph-mon[104668]: pgmap v1926: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 7.6 GiB data, 25 GiB used, 601 GiB / 626 GiB avail; 0 B/s wr, 918 op/s; 54578/374814 objects degraded (14.561%); 4696/374814 objects misplaced (1.253%); 1.4 MiB/s, 21 objects/s recovering 2024-08-17T17:32:32.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:32 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.3958913496135553}]': finished 2024-08-17T17:32:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:32 smithi162 ceph-mon[104668]: osdmap e1309: 8 total, 7 up, 7 in 2024-08-17T17:32:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:32 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:32:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:32 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:32.371 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:32.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:32.473 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:32.511 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:32.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:32.612 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:32.641 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:32.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:32.743 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:32.771 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:32.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:32.873 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:32.915 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:32.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:32.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:33.017 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:33.051 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:33.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:33.153 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:33.178 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:33.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:33.280 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:33.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:33 smithi063 ceph-mon[109843]: osdmap e1310: 8 total, 7 up, 7 in 2024-08-17T17:32:33.289 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:33 smithi063 ceph-mon[109843]: 10.5 deep-scrub ok 2024-08-17T17:32:33.289 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:32:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:32:33] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.51.0" 2024-08-17T17:32:33.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:33 smithi063 ceph-mon[114756]: osdmap e1310: 8 total, 7 up, 7 in 2024-08-17T17:32:33.289 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:33 smithi063 ceph-mon[114756]: 10.5 deep-scrub ok 2024-08-17T17:32:33.323 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:33.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:33.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:33 smithi162 ceph-mon[104668]: osdmap e1310: 8 total, 7 up, 7 in 2024-08-17T17:32:33.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:33 smithi162 ceph-mon[104668]: 10.5 deep-scrub ok 2024-08-17T17:32:33.425 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:33.462 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:33.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:33.564 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:33.601 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:33.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:33.703 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:33.740 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:33.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.761 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:33.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:33.763 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:33.863 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:33.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:33.899 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:33.899 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:34.000 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:34.028 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:34.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:34.129 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:34.138 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:34 smithi162 ceph-mon[104668]: pgmap v1929: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 1.5 KiB/s rd, 4.6 KiB/s wr, 1.24k op/s; 52199/361323 objects degraded (14.447%); 4546/361323 objects misplaced (1.258%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:32:34.138 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:34 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 52199/361323 objects degraded (14.447%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:34.161 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:34.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:34.263 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:34.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:34 smithi063 ceph-mon[114756]: pgmap v1929: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 1.5 KiB/s rd, 4.6 KiB/s wr, 1.24k op/s; 52199/361323 objects degraded (14.447%); 4546/361323 objects misplaced (1.258%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:32:34.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:34 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 52199/361323 objects degraded (14.447%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:34.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:34 smithi063 ceph-mon[109843]: pgmap v1929: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 7.4 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 1.5 KiB/s rd, 4.6 KiB/s wr, 1.24k op/s; 52199/361323 objects degraded (14.447%); 4546/361323 objects misplaced (1.258%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:32:34.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:34 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 52199/361323 objects degraded (14.447%), 14 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:34.297 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:34.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:34.399 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:34.439 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:34.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:34.541 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:34.578 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:34.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:34.680 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:34.717 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:34.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:34.819 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:34.854 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:34.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:34.955 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:34.992 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:34.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:34.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:35.094 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:35.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.130 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:35.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:35.231 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:35.259 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:35.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:35.361 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:35.402 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:35.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:35.504 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:35.530 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:35.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:35.631 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:35.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.669 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:35.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:35.770 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:35.808 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:35.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:35.910 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:35.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:35.940 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:35.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:36.041 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:36.043 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:32:36.063 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:36.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:36.165 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:36.201 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:36.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:36.303 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:36.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.337 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:36.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:36.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:36 smithi162 ceph-mon[104668]: pgmap v1930: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 7.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 1.5 KiB/s rd, 0 B/s wr, 776 op/s; 52199/358053 objects degraded (14.579%); 4546/358053 objects misplaced (1.270%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:32:36.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:36 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4238008625' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:36.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:36 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:36.438 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:36.460 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:36.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:36.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:36 smithi063 ceph-mon[114756]: pgmap v1930: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 7.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 1.5 KiB/s rd, 0 B/s wr, 776 op/s; 52199/358053 objects degraded (14.579%); 4546/358053 objects misplaced (1.270%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:32:36.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:36 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4238008625' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:36.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:36 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:36.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:36 smithi063 ceph-mon[109843]: pgmap v1930: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 7.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 1.5 KiB/s rd, 0 B/s wr, 776 op/s; 52199/358053 objects degraded (14.579%); 4546/358053 objects misplaced (1.270%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:32:36.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:36 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4238008625' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:36.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:36 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:36.562 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:36.600 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:36.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:36.702 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:36.737 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:36.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:36.838 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:36.869 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:36.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:36.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:36.970 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:37.002 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:37.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:37.103 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:37.142 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:37.244 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:37.275 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:37.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:37.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:37 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:37.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:37 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:37.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:37 smithi063 ceph-mon[114756]: osdmap e1311: 8 total, 7 up, 7 in 2024-08-17T17:32:37.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:37 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:37.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:37 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:37.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:37 smithi063 ceph-mon[109843]: osdmap e1311: 8 total, 7 up, 7 in 2024-08-17T17:32:37.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:37 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:37.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:37 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:37.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:37 smithi162 ceph-mon[104668]: osdmap e1311: 8 total, 7 up, 7 in 2024-08-17T17:32:37.377 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:37.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.421 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:37.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:37.522 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:37.558 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:37.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:37.660 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:37.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.699 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:37.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:37.800 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:37.841 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:37.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:37.943 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:37.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:37.979 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:37.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:38.080 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:38.117 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:38.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:38.192 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0] out_osds: [1] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:32:38.192 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:32:38.193 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-08-17T17:32:38.193 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.223868 2024-08-17T17:32:38.193 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 7 0.22386826037767393 2024-08-17T17:32:38.220 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:38.289 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:38.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:38.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:38 smithi162 ceph-mon[104668]: pgmap v1932: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 7.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 2.0 KiB/s rd, 0 B/s wr, 255 op/s; 51774/356778 objects degraded (14.512%); 4546/356778 objects misplaced (1.274%); 1.6 MiB/s, 24 objects/s recovering 2024-08-17T17:32:38.358 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:38 smithi063 ceph-mon[109843]: pgmap v1932: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 7.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 2.0 KiB/s rd, 0 B/s wr, 255 op/s; 51774/356778 objects degraded (14.512%); 4546/356778 objects misplaced (1.274%); 1.6 MiB/s, 24 objects/s recovering 2024-08-17T17:32:38.358 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:38 smithi063 ceph-mon[114756]: pgmap v1932: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 7.3 GiB data, 24 GiB used, 602 GiB / 626 GiB avail; 2.0 KiB/s rd, 0 B/s wr, 255 op/s; 51774/356778 objects degraded (14.512%); 4546/356778 objects misplaced (1.274%); 1.6 MiB/s, 24 objects/s recovering 2024-08-17T17:32:38.392 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:38.425 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:38.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:38.526 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:38.560 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:38.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:38.662 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:38.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.704 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:38.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:38.805 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:38.838 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:38.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:38.940 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:38.976 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:38.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:38.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:39.078 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:39.122 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:39.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.123 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:39.224 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:39.253 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:39.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:39.272 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:39 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 51774/356778 objects degraded (14.512%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:39.272 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:39 smithi063 ceph-mon[114756]: pgmap v1933: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 1.6 KiB/s rd, 0 B/s wr, 1.08k op/s; 49121/336966 objects degraded (14.577%); 4452/336966 objects misplaced (1.321%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:32:39.272 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:39 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 51774/356778 objects degraded (14.512%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:39.272 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:39 smithi063 ceph-mon[109843]: pgmap v1933: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 1.6 KiB/s rd, 0 B/s wr, 1.08k op/s; 49121/336966 objects degraded (14.577%); 4452/336966 objects misplaced (1.321%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:32:39.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:39 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 51774/356778 objects degraded (14.512%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:39.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:39 smithi162 ceph-mon[104668]: pgmap v1933: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 1.6 KiB/s rd, 0 B/s wr, 1.08k op/s; 49121/336966 objects degraded (14.577%); 4452/336966 objects misplaced (1.321%); 2.0 MiB/s, 32 objects/s recovering 2024-08-17T17:32:39.355 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:39.394 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:39.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:39.446 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:32:39.446 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:32:39.476 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:32:39.477 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:32:39.496 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:39.513 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:32:39.514 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:32:39.531 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:39.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:39.548 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:32:39.548 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:32:39.590 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:32:39.590 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:32:39.626 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:32:39.626 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:32:39.630 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:32:39.633 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:39.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.674 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:39.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:39.677 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:32:39.677 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:32:39.775 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:39.807 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:39.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:39.908 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:39.951 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:39.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:39.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:39.972 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:32:39.972 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:32:39.973 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:32:39.973 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:32:39.973 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.973 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:44821 9.2 9:42c04f20:::benchmark_data_smithi063_263014_object44820:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:32:39.973 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:08.014872+0000", 2024-08-17T17:32:39.973 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 331.95471281900001, 2024-08-17T17:32:39.973 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.347331799, 2024-08-17T17:32:39.973 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:39.973 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:39.973 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:39.973 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:32:39.974 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:32:39.974 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:32:39.974 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 44821 2024-08-17T17:32:39.974 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.974 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:39.974 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.974 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:39.974 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.014872+0000", 2024-08-17T17:32:39.974 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:39.974 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.974 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.974 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:39.974 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.014872+0000", 2024-08-17T17:32:39.975 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:39.975 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.975 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.975 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:39.975 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.014874+0000", 2024-08-17T17:32:39.975 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.677e-06 2024-08-17T17:32:39.975 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.975 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.975 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:39.975 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.014910+0000", 2024-08-17T17:32:39.975 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5738999999999998e-05 2024-08-17T17:32:39.975 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.975 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.975 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:39.976 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.014929+0000", 2024-08-17T17:32:39.976 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8845999999999999e-05 2024-08-17T17:32:39.976 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.976 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.976 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:39.976 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.014936+0000", 2024-08-17T17:32:39.976 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1860000000000003e-06 2024-08-17T17:32:39.976 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.976 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.976 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:39.976 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.015044+0000", 2024-08-17T17:32:39.976 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000107877 2024-08-17T17:32:39.976 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.977 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.977 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:39.977 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.015196+0000", 2024-08-17T17:32:39.977 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015212400000000001 2024-08-17T17:32:39.977 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.977 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.977 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:32:39.977 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.015363+0000", 2024-08-17T17:32:39.977 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016737300000000001 2024-08-17T17:32:39.977 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.977 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.977 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:32:39.978 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019403+0000", 2024-08-17T17:32:39.978 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0040394519999999998 2024-08-17T17:32:39.978 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.978 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.978 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:39.978 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019531+0000", 2024-08-17T17:32:39.978 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000128269 2024-08-17T17:32:39.978 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.978 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.978 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:39.978 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.362090+0000", 2024-08-17T17:32:39.978 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.34255893199999998 2024-08-17T17:32:39.979 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.979 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.979 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:39.979 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.362154+0000", 2024-08-17T17:32:39.979 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4572000000000005e-05 2024-08-17T17:32:39.979 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.979 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.979 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:39.979 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.362204+0000", 2024-08-17T17:32:39.979 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9752000000000001e-05 2024-08-17T17:32:39.979 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:39.979 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:39.980 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:39.980 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.980 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.980 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:44824 9.2 9:4d91f1c9:::benchmark_data_smithi063_263014_object44823:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:32:39.980 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:08.017287+0000", 2024-08-17T17:32:39.980 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 331.95229809599999, 2024-08-17T17:32:39.980 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.34720898700000002, 2024-08-17T17:32:39.980 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:39.980 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:39.980 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:39.981 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:32:39.981 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:32:39.981 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:32:39.981 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 44824 2024-08-17T17:32:39.981 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.981 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:39.981 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.981 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:39.981 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017287+0000", 2024-08-17T17:32:39.981 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:39.981 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.982 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.982 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:39.982 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017287+0000", 2024-08-17T17:32:39.982 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:39.982 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.982 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.982 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:39.982 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017288+0000", 2024-08-17T17:32:39.982 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.1800000000000005e-07 2024-08-17T17:32:39.982 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.983 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.983 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:39.983 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017307+0000", 2024-08-17T17:32:39.983 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9488e-05 2024-08-17T17:32:39.983 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.983 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.983 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:39.983 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017319+0000", 2024-08-17T17:32:39.983 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1854e-05 2024-08-17T17:32:39.983 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.983 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.983 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:39.983 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017327+0000", 2024-08-17T17:32:39.983 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9149999999999996e-06 2024-08-17T17:32:39.984 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.984 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.984 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:39.984 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017499+0000", 2024-08-17T17:32:39.984 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00017178699999999999 2024-08-17T17:32:39.984 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.984 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.984 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:39.984 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017626+0000", 2024-08-17T17:32:39.984 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012685100000000001 2024-08-17T17:32:39.984 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.984 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.984 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:32:39.984 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017750+0000", 2024-08-17T17:32:39.985 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000124218 2024-08-17T17:32:39.985 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.985 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.985 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:39.985 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.022751+0000", 2024-08-17T17:32:39.985 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.005000984 2024-08-17T17:32:39.985 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.985 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.985 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:32:39.985 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.022772+0000", 2024-08-17T17:32:39.985 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0888000000000001e-05 2024-08-17T17:32:39.985 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.985 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.985 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:39.985 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364384+0000", 2024-08-17T17:32:39.986 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.34161243699999999 2024-08-17T17:32:39.986 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.986 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.986 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:39.986 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364448+0000", 2024-08-17T17:32:39.986 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3516999999999995e-05 2024-08-17T17:32:39.986 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.986 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.986 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:39.986 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364496+0000", 2024-08-17T17:32:39.986 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8229999999999997e-05 2024-08-17T17:32:39.986 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:39.987 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:39.987 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:39.987 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.987 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.987 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:44825 9.2 9:495368c0:::benchmark_data_smithi063_263014_object44824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:32:39.987 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:08.019614+0000", 2024-08-17T17:32:39.987 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 331.94997104499998, 2024-08-17T17:32:39.987 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.34501749799999998, 2024-08-17T17:32:39.987 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:39.987 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:39.987 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:39.988 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:32:39.988 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:32:39.988 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:32:39.988 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 44825 2024-08-17T17:32:39.988 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.988 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:39.988 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.989 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:39.989 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019614+0000", 2024-08-17T17:32:39.989 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:39.989 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.989 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.989 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:39.989 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019614+0000", 2024-08-17T17:32:39.989 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:39.989 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.989 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.989 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:39.989 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019615+0000", 2024-08-17T17:32:39.989 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.24e-06 2024-08-17T17:32:39.989 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.990 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.990 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:39.990 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019643+0000", 2024-08-17T17:32:39.990 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.7710000000000001e-05 2024-08-17T17:32:39.990 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.990 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.990 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:39.990 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019653+0000", 2024-08-17T17:32:39.990 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0088999999999999e-05 2024-08-17T17:32:39.990 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.990 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.990 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:39.990 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019660+0000", 2024-08-17T17:32:39.990 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8469999999999998e-06 2024-08-17T17:32:39.991 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.991 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.991 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:39.991 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019845+0000", 2024-08-17T17:32:39.991 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00018503399999999999 2024-08-17T17:32:39.991 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.991 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.991 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:39.991 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.020016+0000", 2024-08-17T17:32:39.991 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00017090399999999999 2024-08-17T17:32:39.991 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.991 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.991 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:32:39.991 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.020159+0000", 2024-08-17T17:32:39.992 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014291799999999999 2024-08-17T17:32:39.992 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.992 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.992 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:32:39.992 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.022872+0000", 2024-08-17T17:32:39.992 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0027137279999999999 2024-08-17T17:32:39.992 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.992 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.992 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:39.992 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.025059+0000", 2024-08-17T17:32:39.992 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0021862729999999999 2024-08-17T17:32:39.992 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.992 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.992 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:39.993 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364543+0000", 2024-08-17T17:32:39.993 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.33948450099999999 2024-08-17T17:32:39.993 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.993 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.993 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:39.993 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364596+0000", 2024-08-17T17:32:39.993 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2871999999999998e-05 2024-08-17T17:32:39.993 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.993 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.993 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:39.993 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364632+0000", 2024-08-17T17:32:39.993 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5382000000000001e-05 2024-08-17T17:32:39.993 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:39.994 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:39.994 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:39.994 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.994 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.994 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:44847 9.2 9:4ffc50cb:::benchmark_data_smithi063_263014_object44846:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:32:39.994 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:08.029913+0000", 2024-08-17T17:32:39.994 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 331.939672358, 2024-08-17T17:32:39.994 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.33484943499999997, 2024-08-17T17:32:39.994 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:39.994 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:39.994 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:39.994 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:32:39.994 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:32:39.994 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:32:39.995 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 44847 2024-08-17T17:32:39.995 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.995 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:39.995 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.995 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:39.995 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.029913+0000", 2024-08-17T17:32:39.995 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:39.995 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.995 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.995 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:39.995 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.029913+0000", 2024-08-17T17:32:39.995 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:39.995 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.995 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.996 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:39.996 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.029914+0000", 2024-08-17T17:32:39.996 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.37e-06 2024-08-17T17:32:39.996 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.996 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.996 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:39.996 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.029976+0000", 2024-08-17T17:32:39.996 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1852000000000003e-05 2024-08-17T17:32:39.996 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.996 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.996 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:39.996 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.029991+0000", 2024-08-17T17:32:39.997 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5037e-05 2024-08-17T17:32:39.997 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.997 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.997 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:39.997 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.029998+0000", 2024-08-17T17:32:39.997 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2320000000000004e-06 2024-08-17T17:32:39.997 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.997 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.997 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:39.997 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.030064+0000", 2024-08-17T17:32:39.997 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5420999999999996e-05 2024-08-17T17:32:39.997 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.998 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.998 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:39.998 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.030246+0000", 2024-08-17T17:32:39.998 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00018207 2024-08-17T17:32:39.998 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.998 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.998 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:32:39.998 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.030401+0000", 2024-08-17T17:32:39.998 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000155026 2024-08-17T17:32:39.998 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.998 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.998 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:32:39.999 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.032588+0000", 2024-08-17T17:32:39.999 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0021871360000000001 2024-08-17T17:32:39.999 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.999 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.999 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:39.999 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.032753+0000", 2024-08-17T17:32:39.999 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016551200000000001 2024-08-17T17:32:39.999 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:39.999 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:39.999 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:39.999 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364680+0000", 2024-08-17T17:32:39.999 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.33192645300000001 2024-08-17T17:32:40.000 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.000 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.000 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.000 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364726+0000", 2024-08-17T17:32:40.000 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6159000000000003e-05 2024-08-17T17:32:40.000 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.000 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.000 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.000 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364762+0000", 2024-08-17T17:32:40.000 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6167000000000001e-05 2024-08-17T17:32:40.000 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.000 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.001 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.001 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.001 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.001 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:102903 9.2 9:4b6cfc07:::benchmark_data_smithi063_263014_object102902:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-08-17T17:32:40.001 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:55.985471+0000", 2024-08-17T17:32:40.001 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 283.98411360300003, 2024-08-17T17:32:40.001 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.36001304699999997, 2024-08-17T17:32:40.001 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.001 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.001 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.001 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:32:40.001 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:32:40.002 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:32:40.002 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 102903 2024-08-17T17:32:40.002 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.002 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.002 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.002 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.002 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:55.985471+0000", 2024-08-17T17:32:40.002 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.002 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.002 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.002 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.003 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:55.985471+0000", 2024-08-17T17:32:40.003 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.003 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.003 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.003 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.003 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:55.985473+0000", 2024-08-17T17:32:40.003 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.544e-06 2024-08-17T17:32:40.003 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.003 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.003 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.003 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:55.985497+0000", 2024-08-17T17:32:40.004 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3995e-05 2024-08-17T17:32:40.004 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.004 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.004 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.004 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.002738+0000", 2024-08-17T17:32:40.004 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.017241324999999998 2024-08-17T17:32:40.004 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.004 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.004 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.004 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.002748+0000", 2024-08-17T17:32:40.004 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4259999999999992e-06 2024-08-17T17:32:40.004 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.005 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.005 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.005 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.003040+0000", 2024-08-17T17:32:40.005 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00029239799999999997 2024-08-17T17:32:40.005 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.005 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.005 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.005 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.003185+0000", 2024-08-17T17:32:40.005 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000144571 2024-08-17T17:32:40.005 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.005 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.005 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:32:40.005 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.003341+0000", 2024-08-17T17:32:40.006 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000156317 2024-08-17T17:32:40.006 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.006 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.006 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:32:40.006 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.009881+0000", 2024-08-17T17:32:40.006 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0065399220000000001 2024-08-17T17:32:40.006 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.006 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.006 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.006 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.014340+0000", 2024-08-17T17:32:40.006 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044590910000000001 2024-08-17T17:32:40.007 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.007 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.007 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.007 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.345341+0000", 2024-08-17T17:32:40.007 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.33100101799999998 2024-08-17T17:32:40.007 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.007 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.007 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.007 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.345400+0000", 2024-08-17T17:32:40.007 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8921000000000001e-05 2024-08-17T17:32:40.007 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.007 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.008 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.008 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.345485+0000", 2024-08-17T17:32:40.008 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4518999999999995e-05 2024-08-17T17:32:40.008 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.008 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.008 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.008 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.008 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.008 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:102911 9.2 9:4a48d03c:::benchmark_data_smithi063_263014_object102910:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-08-17T17:32:40.008 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:56.002912+0000", 2024-08-17T17:32:40.008 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 283.96667318499999, 2024-08-17T17:32:40.008 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.34287475299999998, 2024-08-17T17:32:40.009 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.009 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.009 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.009 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:32:40.009 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:32:40.009 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:32:40.009 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 102911 2024-08-17T17:32:40.009 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.009 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.009 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.009 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.009 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.002912+0000", 2024-08-17T17:32:40.009 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.010 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.010 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.010 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.010 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.002912+0000", 2024-08-17T17:32:40.010 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.010 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.010 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.010 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.010 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.002912+0000", 2024-08-17T17:32:40.010 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2899999999999999e-07 2024-08-17T17:32:40.010 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.010 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.010 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.011 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.002944+0000", 2024-08-17T17:32:40.011 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1674999999999997e-05 2024-08-17T17:32:40.011 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.011 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.011 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.011 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.002952+0000", 2024-08-17T17:32:40.011 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3499999999999997e-06 2024-08-17T17:32:40.011 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.011 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.011 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.011 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.002957+0000", 2024-08-17T17:32:40.011 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.062e-06 2024-08-17T17:32:40.012 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.012 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.012 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.012 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.006406+0000", 2024-08-17T17:32:40.012 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0034489019999999998 2024-08-17T17:32:40.012 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.012 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.012 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.012 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.006546+0000", 2024-08-17T17:32:40.012 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013922899999999999 2024-08-17T17:32:40.012 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.012 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.012 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:32:40.012 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.006696+0000", 2024-08-17T17:32:40.012 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015076600000000001 2024-08-17T17:32:40.013 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.013 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.013 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:32:40.013 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.013746+0000", 2024-08-17T17:32:40.013 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0070498130000000003 2024-08-17T17:32:40.013 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.013 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.013 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.013 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.018198+0000", 2024-08-17T17:32:40.013 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044515270000000003 2024-08-17T17:32:40.013 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.014 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.014 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.014 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.345526+0000", 2024-08-17T17:32:40.014 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.32732798200000002 2024-08-17T17:32:40.014 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.014 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.014 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.014 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.345748+0000", 2024-08-17T17:32:40.014 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00022286100000000001 2024-08-17T17:32:40.014 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.014 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.014 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.014 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.345787+0000", 2024-08-17T17:32:40.015 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8157000000000001e-05 2024-08-17T17:32:40.015 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.015 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.015 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.015 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.015 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.015 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:102912 9.2 9:48e20027:::benchmark_data_smithi063_263014_object102911:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-08-17T17:32:40.015 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:56.002965+0000", 2024-08-17T17:32:40.015 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 283.966620471, 2024-08-17T17:32:40.015 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.343703015, 2024-08-17T17:32:40.015 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.015 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.015 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.016 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:32:40.016 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:32:40.016 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:32:40.016 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 102912 2024-08-17T17:32:40.016 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.016 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.016 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.016 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.016 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.002965+0000", 2024-08-17T17:32:40.016 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.016 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.016 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.016 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.017 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.002965+0000", 2024-08-17T17:32:40.017 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.017 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.017 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.017 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.017 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.002965+0000", 2024-08-17T17:32:40.017 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4499999999999997e-07 2024-08-17T17:32:40.017 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.017 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.017 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.017 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.003115+0000", 2024-08-17T17:32:40.017 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014981 2024-08-17T17:32:40.018 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.018 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.018 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.018 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.003122+0000", 2024-08-17T17:32:40.018 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8619999999999997e-06 2024-08-17T17:32:40.018 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.018 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.018 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.018 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.003127+0000", 2024-08-17T17:32:40.018 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2499999999999997e-06 2024-08-17T17:32:40.018 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.018 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.019 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.019 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.009909+0000", 2024-08-17T17:32:40.019 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0067816789999999997 2024-08-17T17:32:40.019 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.019 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.019 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.019 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.010041+0000", 2024-08-17T17:32:40.019 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013193800000000001 2024-08-17T17:32:40.019 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.019 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.019 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:32:40.019 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.010179+0000", 2024-08-17T17:32:40.019 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000138195 2024-08-17T17:32:40.020 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.020 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.020 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:32:40.020 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.013752+0000", 2024-08-17T17:32:40.020 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.003573503 2024-08-17T17:32:40.020 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.020 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.020 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.020 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.020194+0000", 2024-08-17T17:32:40.020 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0064413760000000004 2024-08-17T17:32:40.020 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.020 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.020 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.021 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.345824+0000", 2024-08-17T17:32:40.021 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.32563071500000002 2024-08-17T17:32:40.021 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.021 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.021 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.021 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.346080+0000", 2024-08-17T17:32:40.021 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00025565800000000002 2024-08-17T17:32:40.021 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.021 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.021 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.021 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.346668+0000", 2024-08-17T17:32:40.021 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00058758400000000004 2024-08-17T17:32:40.022 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.022 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.022 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.022 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.022 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.022 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:102922 9.2 9:49d4c915:::benchmark_data_smithi063_263014_object102921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-08-17T17:32:40.022 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:56.014561+0000", 2024-08-17T17:32:40.022 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 283.95502414999999, 2024-08-17T17:32:40.022 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.33406101599999999, 2024-08-17T17:32:40.022 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.022 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.022 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.022 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:32:40.023 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:32:40.023 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:32:40.023 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 102922 2024-08-17T17:32:40.023 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.023 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.023 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.023 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.023 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.014561+0000", 2024-08-17T17:32:40.023 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.023 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.023 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.023 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.024 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.014561+0000", 2024-08-17T17:32:40.024 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.024 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.024 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.024 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.024 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.014562+0000", 2024-08-17T17:32:40.024 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4330000000000001e-06 2024-08-17T17:32:40.024 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.024 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.024 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.024 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.014584+0000", 2024-08-17T17:32:40.025 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1705e-05 2024-08-17T17:32:40.025 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.025 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.025 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.025 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.014595+0000", 2024-08-17T17:32:40.025 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1447999999999999e-05 2024-08-17T17:32:40.025 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.025 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.025 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.025 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.014603+0000", 2024-08-17T17:32:40.025 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0330000000000003e-06 2024-08-17T17:32:40.026 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.026 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.026 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.026 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.014659+0000", 2024-08-17T17:32:40.026 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6078000000000001e-05 2024-08-17T17:32:40.026 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.026 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.026 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.026 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.014821+0000", 2024-08-17T17:32:40.026 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016266100000000001 2024-08-17T17:32:40.026 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.026 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.026 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:32:40.027 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.014987+0000", 2024-08-17T17:32:40.027 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016576500000000001 2024-08-17T17:32:40.027 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.027 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.027 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:32:40.027 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.018301+0000", 2024-08-17T17:32:40.027 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0033135550000000001 2024-08-17T17:32:40.027 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.027 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.027 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.027 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.020870+0000", 2024-08-17T17:32:40.028 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00256931 2024-08-17T17:32:40.028 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.028 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.028 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.028 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.348519+0000", 2024-08-17T17:32:40.028 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.32764934099999998 2024-08-17T17:32:40.028 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.028 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.028 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.028 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.348580+0000", 2024-08-17T17:32:40.028 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0677999999999997e-05 2024-08-17T17:32:40.028 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.028 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.029 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.029 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.348622+0000", 2024-08-17T17:32:40.029 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2008999999999997e-05 2024-08-17T17:32:40.029 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.029 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.029 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.029 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.029 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.029 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:172893 9.c 9:3412a79b:::benchmark_data_smithi063_263014_object48555:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T17:32:40.029 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:29:01.400423+0000", 2024-08-17T17:32:40.029 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 218.569161581, 2024-08-17T17:32:40.029 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.189272936, 2024-08-17T17:32:40.029 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.030 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.030 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.030 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:32:40.030 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:32:40.030 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:32:40.030 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 172893 2024-08-17T17:32:40.030 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.030 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.030 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.030 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.030 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.400423+0000", 2024-08-17T17:32:40.031 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.031 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.031 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.031 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.031 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.400423+0000", 2024-08-17T17:32:40.031 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.031 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.031 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.031 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.031 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.400424+0000", 2024-08-17T17:32:40.031 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9599999999999998e-07 2024-08-17T17:32:40.031 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.032 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.032 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.032 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.400429+0000", 2024-08-17T17:32:40.032 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.634e-06 2024-08-17T17:32:40.032 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.032 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.032 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.032 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.400440+0000", 2024-08-17T17:32:40.032 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1438e-05 2024-08-17T17:32:40.032 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.032 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.032 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.033 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.400448+0000", 2024-08-17T17:32:40.033 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4089999999999997e-06 2024-08-17T17:32:40.033 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.033 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.033 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.033 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.400499+0000", 2024-08-17T17:32:40.033 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1742e-05 2024-08-17T17:32:40.033 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.033 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.033 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.033 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.400636+0000", 2024-08-17T17:32:40.033 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000136379 2024-08-17T17:32:40.033 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.034 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.034 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:32:40.034 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.400758+0000", 2024-08-17T17:32:40.035 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012167699999999999 2024-08-17T17:32:40.035 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.035 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.035 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.035 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.413080+0000", 2024-08-17T17:32:40.035 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.012322405 2024-08-17T17:32:40.035 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.035 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.035 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:32:40.035 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.413304+0000", 2024-08-17T17:32:40.035 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00022375800000000001 2024-08-17T17:32:40.035 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.035 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.035 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.036 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.589617+0000", 2024-08-17T17:32:40.036 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17631340000000001 2024-08-17T17:32:40.036 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.036 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.036 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.036 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.589663+0000", 2024-08-17T17:32:40.036 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5608000000000001e-05 2024-08-17T17:32:40.036 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.037 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.037 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.037 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.589696+0000", 2024-08-17T17:32:40.037 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3689999999999998e-05 2024-08-17T17:32:40.037 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.037 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.038 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.038 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.038 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.038 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:172898 9.c 9:35b463e6:::benchmark_data_smithi063_263014_object48560:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T17:32:40.038 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:29:01.405746+0000", 2024-08-17T17:32:40.038 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 218.563839527, 2024-08-17T17:32:40.038 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19636144899999999, 2024-08-17T17:32:40.038 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.038 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.038 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.038 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:32:40.038 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:32:40.039 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:32:40.039 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 172898 2024-08-17T17:32:40.039 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.039 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.039 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.039 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.039 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.405746+0000", 2024-08-17T17:32:40.039 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.039 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.039 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.039 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.039 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.405746+0000", 2024-08-17T17:32:40.040 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.040 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.040 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.040 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.040 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.405747+0000", 2024-08-17T17:32:40.040 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.327e-06 2024-08-17T17:32:40.040 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.040 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.040 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.040 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.405751+0000", 2024-08-17T17:32:40.040 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2640000000000002e-06 2024-08-17T17:32:40.040 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.041 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.041 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.041 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.405765+0000", 2024-08-17T17:32:40.041 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3475000000000001e-05 2024-08-17T17:32:40.041 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.041 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.041 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.041 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.405772+0000", 2024-08-17T17:32:40.041 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.294e-06 2024-08-17T17:32:40.041 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.041 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.041 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.041 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.405916+0000", 2024-08-17T17:32:40.042 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014422 2024-08-17T17:32:40.042 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.042 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.042 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.042 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.406048+0000", 2024-08-17T17:32:40.042 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013168399999999999 2024-08-17T17:32:40.042 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.042 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.042 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:32:40.042 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.406172+0000", 2024-08-17T17:32:40.042 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012436399999999999 2024-08-17T17:32:40.042 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.042 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.043 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.043 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.413165+0000", 2024-08-17T17:32:40.043 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.006992369 2024-08-17T17:32:40.043 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.043 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.043 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:32:40.043 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.413311+0000", 2024-08-17T17:32:40.043 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014679600000000001 2024-08-17T17:32:40.043 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.043 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.043 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.043 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.601988+0000", 2024-08-17T17:32:40.043 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.188676818 2024-08-17T17:32:40.044 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.044 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.044 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.044 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.602046+0000", 2024-08-17T17:32:40.044 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7479000000000003e-05 2024-08-17T17:32:40.044 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.044 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.044 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.044 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:01.602107+0000", 2024-08-17T17:32:40.044 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1359000000000007e-05 2024-08-17T17:32:40.044 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.044 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.045 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.045 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.045 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.045 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:248518 9.2 9:4568ceb0:::benchmark_data_smithi063_263014_object124180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1277)", 2024-08-17T17:32:40.045 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:30:12.449390+0000", 2024-08-17T17:32:40.045 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 147.52019469300001, 2024-08-17T17:32:40.045 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.18799277, 2024-08-17T17:32:40.045 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.045 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.045 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.045 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:32:40.045 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:32:40.046 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:32:40.046 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 248518 2024-08-17T17:32:40.046 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.046 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.046 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.046 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.046 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.449390+0000", 2024-08-17T17:32:40.046 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.046 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.046 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.046 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.046 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.449390+0000", 2024-08-17T17:32:40.047 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.047 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.047 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.047 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.047 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.449391+0000", 2024-08-17T17:32:40.047 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9600000000000008e-07 2024-08-17T17:32:40.047 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.047 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.047 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.047 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.449395+0000", 2024-08-17T17:32:40.047 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9709999999999998e-06 2024-08-17T17:32:40.047 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.047 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.047 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.048 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.449403+0000", 2024-08-17T17:32:40.048 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.79e-06 2024-08-17T17:32:40.048 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.048 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.048 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.048 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.449410+0000", 2024-08-17T17:32:40.048 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7120000000000003e-06 2024-08-17T17:32:40.048 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.048 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.048 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.048 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.453008+0000", 2024-08-17T17:32:40.048 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.003598558 2024-08-17T17:32:40.048 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.048 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.049 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.049 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.453115+0000", 2024-08-17T17:32:40.049 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000106734 2024-08-17T17:32:40.049 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.049 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.049 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:32:40.049 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.453212+0000", 2024-08-17T17:32:40.049 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7337999999999995e-05 2024-08-17T17:32:40.049 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.049 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.049 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:32:40.049 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.461999+0000", 2024-08-17T17:32:40.049 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0087869220000000008 2024-08-17T17:32:40.049 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.050 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.050 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.050 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.464474+0000", 2024-08-17T17:32:40.050 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0024745729999999999 2024-08-17T17:32:40.050 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.050 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.050 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.050 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.637288+0000", 2024-08-17T17:32:40.050 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17281374899999999 2024-08-17T17:32:40.050 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.050 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.050 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.051 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.637338+0000", 2024-08-17T17:32:40.051 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9957000000000003e-05 2024-08-17T17:32:40.051 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.051 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.051 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.051 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.637383+0000", 2024-08-17T17:32:40.051 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5469999999999997e-05 2024-08-17T17:32:40.051 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.051 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.051 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.051 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.051 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.051 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:248522 9.2 9:479aa9d9:::benchmark_data_smithi063_263014_object124184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1277)", 2024-08-17T17:32:40.051 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:30:12.450244+0000", 2024-08-17T17:32:40.052 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 147.519341252, 2024-08-17T17:32:40.052 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.187361053, 2024-08-17T17:32:40.052 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.052 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.052 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.052 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:32:40.052 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:32:40.052 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:32:40.052 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 248522 2024-08-17T17:32:40.052 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.052 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.052 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.052 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.052 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.450244+0000", 2024-08-17T17:32:40.053 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.053 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.053 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.053 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.053 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.450244+0000", 2024-08-17T17:32:40.053 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.053 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.053 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.053 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.053 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.450244+0000", 2024-08-17T17:32:40.053 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7599999999999998e-07 2024-08-17T17:32:40.053 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.053 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.054 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.054 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.450246+0000", 2024-08-17T17:32:40.054 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1610000000000001e-06 2024-08-17T17:32:40.054 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.054 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.054 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.054 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.450252+0000", 2024-08-17T17:32:40.054 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7790000000000001e-06 2024-08-17T17:32:40.054 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.054 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.054 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.054 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.450257+0000", 2024-08-17T17:32:40.055 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8550000000000001e-06 2024-08-17T17:32:40.055 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.055 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.055 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.055 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.459252+0000", 2024-08-17T17:32:40.055 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0089952190000000005 2024-08-17T17:32:40.055 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.055 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.055 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.055 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.459363+0000", 2024-08-17T17:32:40.055 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000111267 2024-08-17T17:32:40.055 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.055 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.055 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:32:40.055 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.459574+0000", 2024-08-17T17:32:40.056 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00021022099999999999 2024-08-17T17:32:40.056 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.056 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.056 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:32:40.056 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.468285+0000", 2024-08-17T17:32:40.056 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0087115119999999994 2024-08-17T17:32:40.056 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.056 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.056 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.056 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.468332+0000", 2024-08-17T17:32:40.056 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7290000000000003e-05 2024-08-17T17:32:40.056 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.056 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.057 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:32:40.057 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.637530+0000", 2024-08-17T17:32:40.057 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.16919764400000001 2024-08-17T17:32:40.057 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.057 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.057 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.057 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.637572+0000", 2024-08-17T17:32:40.057 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2280999999999997e-05 2024-08-17T17:32:40.057 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.057 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.057 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.058 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:30:12.637605+0000", 2024-08-17T17:32:40.058 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.2447999999999998e-05 2024-08-17T17:32:40.058 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.058 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.058 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.058 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.058 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.058 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.2.0:119170 10.5 e1307/1306 10:a17140fc:::benchmark_data_smithi063_294527_object48763:head v 1307'8837, mlcod=1307'8825)", 2024-08-17T17:32:40.058 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:14.156615+0000", 2024-08-17T17:32:40.058 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 25.812970272000001, 2024-08-17T17:32:40.058 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19435047999999999, 2024-08-17T17:32:40.058 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.059 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.059 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.059 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.059 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.059 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.059 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.156615+0000", 2024-08-17T17:32:40.059 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.059 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.059 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.059 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.059 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.156615+0000", 2024-08-17T17:32:40.059 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.059 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.060 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.060 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.060 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.156616+0000", 2024-08-17T17:32:40.060 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.035e-06 2024-08-17T17:32:40.060 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.060 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.060 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.060 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.156624+0000", 2024-08-17T17:32:40.060 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7749999999999993e-06 2024-08-17T17:32:40.060 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.060 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.060 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.061 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.156636+0000", 2024-08-17T17:32:40.061 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2061e-05 2024-08-17T17:32:40.061 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.061 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.061 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.061 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.156644+0000", 2024-08-17T17:32:40.061 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0020000000000006e-06 2024-08-17T17:32:40.061 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.061 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.061 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.061 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.165193+0000", 2024-08-17T17:32:40.061 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0085491130000000005 2024-08-17T17:32:40.061 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.061 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.062 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.062 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.165212+0000", 2024-08-17T17:32:40.062 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8904000000000001e-05 2024-08-17T17:32:40.062 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.062 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.062 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.062 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.350915+0000", 2024-08-17T17:32:40.062 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.18570366999999999 2024-08-17T17:32:40.062 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.062 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.062 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.062 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.350965+0000", 2024-08-17T17:32:40.062 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9920000000000003e-05 2024-08-17T17:32:40.063 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.063 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.063 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.063 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.063 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.063 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.2.0:119171 10.5 e1307/1306 10:a171c6af:::benchmark_data_smithi063_294527_object17270:head v 1307'8838, mlcod=1307'8825)", 2024-08-17T17:32:40.063 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:14.159316+0000", 2024-08-17T17:32:40.063 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 25.810269348999999, 2024-08-17T17:32:40.063 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20066993799999999, 2024-08-17T17:32:40.063 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.063 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.063 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.063 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.064 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.064 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.064 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.159316+0000", 2024-08-17T17:32:40.064 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.064 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.064 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.064 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.064 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.159316+0000", 2024-08-17T17:32:40.064 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.064 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.064 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.064 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.064 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.159317+0000", 2024-08-17T17:32:40.065 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.47e-07 2024-08-17T17:32:40.065 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.065 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.065 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.065 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.159323+0000", 2024-08-17T17:32:40.065 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2689999999999996e-06 2024-08-17T17:32:40.065 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.065 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.065 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.065 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.159333+0000", 2024-08-17T17:32:40.065 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0516999999999999e-05 2024-08-17T17:32:40.065 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.066 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.066 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.066 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.159341+0000", 2024-08-17T17:32:40.066 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.289e-06 2024-08-17T17:32:40.066 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.066 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.066 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.066 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.169158+0000", 2024-08-17T17:32:40.066 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0098170489999999996 2024-08-17T17:32:40.066 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.066 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.066 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.067 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.169170+0000", 2024-08-17T17:32:40.067 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.258e-05 2024-08-17T17:32:40.067 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.067 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.067 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.067 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.359944+0000", 2024-08-17T17:32:40.067 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19077418500000001 2024-08-17T17:32:40.067 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.067 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.067 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.067 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.359986+0000", 2024-08-17T17:32:40.067 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.1202000000000003e-05 2024-08-17T17:32:40.067 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.068 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.068 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.068 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.068 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.068 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.2.0:119172 10.5 e1307/1306 10:a17394cb:::benchmark_data_smithi063_294527_object102437:head v 1307'8839, mlcod=1307'8825)", 2024-08-17T17:32:40.068 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:14.161164+0000", 2024-08-17T17:32:40.068 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 25.808420747, 2024-08-17T17:32:40.068 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19884744400000001, 2024-08-17T17:32:40.068 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.068 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.068 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.068 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.069 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.069 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.069 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.161164+0000", 2024-08-17T17:32:40.069 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.069 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.069 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.069 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.069 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.161164+0000", 2024-08-17T17:32:40.069 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.069 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.069 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.069 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.070 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.161165+0000", 2024-08-17T17:32:40.070 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.54e-07 2024-08-17T17:32:40.070 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.070 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.070 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.070 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.161174+0000", 2024-08-17T17:32:40.070 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7290000000000006e-06 2024-08-17T17:32:40.070 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.070 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.070 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.070 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.161186+0000", 2024-08-17T17:32:40.070 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2218999999999999e-05 2024-08-17T17:32:40.070 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.071 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.071 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.071 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.161193+0000", 2024-08-17T17:32:40.071 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1620000000000002e-06 2024-08-17T17:32:40.071 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.071 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.071 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.071 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.171387+0000", 2024-08-17T17:32:40.071 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.010193643 2024-08-17T17:32:40.071 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.071 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.071 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.071 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.171408+0000", 2024-08-17T17:32:40.071 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0778e-05 2024-08-17T17:32:40.072 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.072 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.072 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.072 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.359990+0000", 2024-08-17T17:32:40.072 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.188581953 2024-08-17T17:32:40.072 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.072 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.072 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.072 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.360012+0000", 2024-08-17T17:32:40.072 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2106000000000001e-05 2024-08-17T17:32:40.072 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.072 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.072 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.073 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.073 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.073 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.2.0:119173 10.5 e1307/1306 10:a17474fa:::benchmark_data_smithi063_294527_object99799:head v 1307'8840, mlcod=1307'8825)", 2024-08-17T17:32:40.073 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:14.164690+0000", 2024-08-17T17:32:40.073 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 25.804895181999999, 2024-08-17T17:32:40.073 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19762239500000001, 2024-08-17T17:32:40.073 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.073 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.073 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.073 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.073 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.073 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.074 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.164690+0000", 2024-08-17T17:32:40.074 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.074 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.074 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.074 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.074 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.164690+0000", 2024-08-17T17:32:40.074 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.074 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.074 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.074 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.074 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.164690+0000", 2024-08-17T17:32:40.074 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6100000000000002e-07 2024-08-17T17:32:40.075 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.075 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.075 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.075 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.164694+0000", 2024-08-17T17:32:40.075 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2429999999999999e-06 2024-08-17T17:32:40.075 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.075 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.075 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.075 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.164701+0000", 2024-08-17T17:32:40.075 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.815e-06 2024-08-17T17:32:40.075 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.075 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.075 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.076 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.164706+0000", 2024-08-17T17:32:40.076 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0549999999999999e-06 2024-08-17T17:32:40.076 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.076 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.076 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.076 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.348373+0000", 2024-08-17T17:32:40.076 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.183666526 2024-08-17T17:32:40.076 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.076 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.076 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.076 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.348397+0000", 2024-08-17T17:32:40.076 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3870999999999999e-05 2024-08-17T17:32:40.077 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.077 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.077 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.077 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.362272+0000", 2024-08-17T17:32:40.077 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.013875567 2024-08-17T17:32:40.077 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.077 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.077 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.077 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.362312+0000", 2024-08-17T17:32:40.077 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9956999999999997e-05 2024-08-17T17:32:40.077 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.077 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.078 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.078 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.078 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.078 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.2.0:119174 10.5 e1307/1306 10:a1757489:::benchmark_data_smithi063_294527_object32471:head v 1307'8841, mlcod=1307'8825)", 2024-08-17T17:32:40.078 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:14.165750+0000", 2024-08-17T17:32:40.078 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 25.803834736999999, 2024-08-17T17:32:40.078 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19659244200000001, 2024-08-17T17:32:40.078 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.078 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.078 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.078 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.078 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.078 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.078 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.165750+0000", 2024-08-17T17:32:40.079 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.079 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.079 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.079 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.079 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.165750+0000", 2024-08-17T17:32:40.079 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.079 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.079 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.079 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.079 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.165751+0000", 2024-08-17T17:32:40.079 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1900000000000002e-07 2024-08-17T17:32:40.079 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.079 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.079 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.080 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.165757+0000", 2024-08-17T17:32:40.080 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5860000000000004e-06 2024-08-17T17:32:40.080 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.080 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.080 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.080 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.165767+0000", 2024-08-17T17:32:40.080 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0631e-05 2024-08-17T17:32:40.080 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.080 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.080 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.080 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:40.081 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.165774+0000", 2024-08-17T17:32:40.081 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6839999999999999e-06 2024-08-17T17:32:40.081 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.081 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.081 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.082 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.350996+0000", 2024-08-17T17:32:40.082 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.18522177100000001 2024-08-17T17:32:40.082 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.082 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.082 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.082 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.351015+0000", 2024-08-17T17:32:40.082 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9653e-05 2024-08-17T17:32:40.082 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.082 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.082 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.082 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.362315+0000", 2024-08-17T17:32:40.082 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011299273 2024-08-17T17:32:40.082 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.083 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.083 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.083 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.362343+0000", 2024-08-17T17:32:40.083 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8225000000000002e-05 2024-08-17T17:32:40.083 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.083 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.083 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.083 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.083 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.083 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.2.0:119177 10.5 e1307/1306 10:a1768431:::benchmark_data_smithi063_294527_object75652:head v 1307'8842, mlcod=1307'8825)", 2024-08-17T17:32:40.083 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:14.168553+0000", 2024-08-17T17:32:40.083 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 25.801032112000001, 2024-08-17T17:32:40.083 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19381780200000001, 2024-08-17T17:32:40.083 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.084 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.084 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.084 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.084 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.084 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.084 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.168553+0000", 2024-08-17T17:32:40.084 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.084 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.084 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.084 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.084 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.168553+0000", 2024-08-17T17:32:40.084 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.084 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.084 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.085 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.085 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.168554+0000", 2024-08-17T17:32:40.085 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1230000000000001e-06 2024-08-17T17:32:40.085 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.085 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.085 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.085 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.168560+0000", 2024-08-17T17:32:40.085 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7089999999999999e-06 2024-08-17T17:32:40.085 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.085 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.085 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.085 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.168570+0000", 2024-08-17T17:32:40.085 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0105999999999999e-05 2024-08-17T17:32:40.085 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.086 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.086 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.086 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.168577+0000", 2024-08-17T17:32:40.086 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8499999999999996e-06 2024-08-17T17:32:40.086 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.086 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.086 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.086 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.353033+0000", 2024-08-17T17:32:40.086 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.18445613599999999 2024-08-17T17:32:40.086 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.086 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.086 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.086 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.353056+0000", 2024-08-17T17:32:40.086 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3589999999999999e-05 2024-08-17T17:32:40.087 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.087 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.087 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.087 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.362346+0000", 2024-08-17T17:32:40.087 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0092892630000000007 2024-08-17T17:32:40.087 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.087 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.087 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.087 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.362371+0000", 2024-08-17T17:32:40.087 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5024999999999998e-05 2024-08-17T17:32:40.087 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.087 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.088 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.088 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.088 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.088 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(osd.2.0:119178 10.5 e1307/1306 10:a1776d4b:::benchmark_data_smithi063_294527_object104116:head v 1307'8843, mlcod=1307'8825)", 2024-08-17T17:32:40.088 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:14.170776+0000", 2024-08-17T17:32:40.088 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 25.798808645000001, 2024-08-17T17:32:40.088 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.191626344, 2024-08-17T17:32:40.088 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.088 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.088 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.088 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.088 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.088 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.089 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.170776+0000", 2024-08-17T17:32:40.089 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.089 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.089 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.089 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.089 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.170776+0000", 2024-08-17T17:32:40.089 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.089 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.089 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.089 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.089 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.170777+0000", 2024-08-17T17:32:40.089 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4499999999999996e-07 2024-08-17T17:32:40.089 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.089 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.090 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.090 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.170783+0000", 2024-08-17T17:32:40.090 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9200000000000001e-06 2024-08-17T17:32:40.090 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.090 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.090 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.090 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.170795+0000", 2024-08-17T17:32:40.090 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1379000000000001e-05 2024-08-17T17:32:40.090 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.090 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.090 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.090 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.170802+0000", 2024-08-17T17:32:40.090 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5140000000000004e-06 2024-08-17T17:32:40.090 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.091 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.091 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.091 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.355198+0000", 2024-08-17T17:32:40.091 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.184396326 2024-08-17T17:32:40.091 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.091 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.091 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.091 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.355216+0000", 2024-08-17T17:32:40.091 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7527999999999999e-05 2024-08-17T17:32:40.091 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.091 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.091 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.092 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.362372+0000", 2024-08-17T17:32:40.092 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0071564640000000004 2024-08-17T17:32:40.092 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.092 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.092 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.092 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.362403+0000", 2024-08-17T17:32:40.092 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0368000000000001e-05 2024-08-17T17:32:40.092 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.092 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.092 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.092 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.092 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.092 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:136286 10.5 e1307/1306 10:a342eec6:::benchmark_data_smithi063_294527_object136285:head v 1307'8844, mlcod=1307'8825)", 2024-08-17T17:32:40.092 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:14.173926+0000", 2024-08-17T17:32:40.093 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 25.795659063999999, 2024-08-17T17:32:40.093 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.190978975, 2024-08-17T17:32:40.093 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:32:40.093 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:32:40.093 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:32:40.093 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:32:40.093 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.093 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:32:40.093 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.173926+0000", 2024-08-17T17:32:40.093 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.093 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.093 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.093 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:32:40.093 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.173926+0000", 2024-08-17T17:32:40.094 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:32:40.094 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.094 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.094 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:32:40.094 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.173927+0000", 2024-08-17T17:32:40.094 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9699999999999995e-07 2024-08-17T17:32:40.094 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.094 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.094 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:32:40.094 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.173951+0000", 2024-08-17T17:32:40.094 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4406999999999999e-05 2024-08-17T17:32:40.094 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.094 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.094 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:32:40.095 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.348324+0000", 2024-08-17T17:32:40.095 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.17437304300000001 2024-08-17T17:32:40.095 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.095 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.095 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:32:40.095 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.348334+0000", 2024-08-17T17:32:40.095 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7310000000000005e-06 2024-08-17T17:32:40.095 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.095 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.095 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:32:40.095 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.357408+0000", 2024-08-17T17:32:40.095 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0090735799999999995 2024-08-17T17:32:40.095 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.095 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.095 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:32:40.096 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.357428+0000", 2024-08-17T17:32:40.096 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0156000000000001e-05 2024-08-17T17:32:40.096 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.096 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.096 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:32:40.096 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.364860+0000", 2024-08-17T17:32:40.096 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0074318839999999997 2024-08-17T17:32:40.096 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:32:40.096 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:32:40.096 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:32:40.096 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.364905+0000", 2024-08-17T17:32:40.096 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5377000000000003e-05 2024-08-17T17:32:40.097 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.097 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.097 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.097 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:32:40.097 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:32:40.097 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:32:40.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.129 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:40.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:40.230 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:40.263 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:40.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:40.280 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:32:40.319 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:40 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/387260011' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:40.319 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:40 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/387260011' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:40.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:40 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/387260011' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:40.365 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:40.406 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:40.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:40.508 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:40.556 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:40.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:40.658 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:40.686 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:40.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:40.788 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:40.819 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:40.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:40.921 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:40.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:40.957 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:40.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:41.058 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:41.093 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:41.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:41.195 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:41.220 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:41.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:41.322 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:41.363 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:41.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:41.372 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:41 smithi063 ceph-mon[109843]: pgmap v1934: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 1.4 KiB/s rd, 0 B/s wr, 875 op/s; 49121/336966 objects degraded (14.577%); 4452/336966 objects misplaced (1.321%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:32:41.372 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:41 smithi063 ceph-mon[114756]: pgmap v1934: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 1.4 KiB/s rd, 0 B/s wr, 875 op/s; 49121/336966 objects degraded (14.577%); 4452/336966 objects misplaced (1.321%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:32:41.465 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:41.501 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:41.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:41.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:41 smithi162 ceph-mon[104668]: pgmap v1934: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 6.9 GiB data, 23 GiB used, 603 GiB / 626 GiB avail; 1.4 KiB/s rd, 0 B/s wr, 875 op/s; 49121/336966 objects degraded (14.577%); 4452/336966 objects misplaced (1.321%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:32:41.604 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:41.632 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:41.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.633 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:41.734 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:41.760 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:41.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:41.862 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:41.901 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:41.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:41.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:42.002 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:42.040 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:32:42.040 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:32:42.056 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:42.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:42.158 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:42.193 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:42.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:42.295 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:42.327 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:42.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:42.429 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:42.461 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:42.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:42.563 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:42.565 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:32:42.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.657 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:42.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:42.700 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:42.737 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:42.758 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:42.788 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:42.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:42.880 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:32:42.891 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:42.932 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:42.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:42.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:43.034 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:43.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.079 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:43.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:43.180 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:43.219 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:43.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:43.321 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:43.357 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:43.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:43.388 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:32:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:32:43] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.51.0" 2024-08-17T17:32:43.458 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:43.495 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:43.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:43.596 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:43.631 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:43.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:43.733 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:43.760 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:43.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:43.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:43 smithi063 ceph-mon[114756]: pgmap v1935: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 920 op/s; 46073/320238 objects degraded (14.387%); 4334/320238 objects misplaced (1.353%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:32:43.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:43 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 46073/320238 objects degraded (14.387%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:43.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:43 smithi063 ceph-mon[109843]: pgmap v1935: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 920 op/s; 46073/320238 objects degraded (14.387%); 4334/320238 objects misplaced (1.353%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:32:43.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:43 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 46073/320238 objects degraded (14.387%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:43.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:43 smithi162 ceph-mon[104668]: pgmap v1935: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 6.5 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 920 op/s; 46073/320238 objects degraded (14.387%); 4334/320238 objects misplaced (1.353%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:32:43.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:43 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 46073/320238 objects degraded (14.387%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:43.863 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:43.903 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:43.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:43.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:44.005 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:44.035 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:44.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:44.041 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:32:44.137 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:44.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.181 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:44.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:44.282 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:44.355 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:44.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:44.457 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:44.491 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:44.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:44.593 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:44.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.629 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:44.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:44.730 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:44.768 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:44.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:44.870 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:44.910 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:44.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:44.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:45.012 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:45.045 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:45.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:45.147 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:45.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.193 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:45.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:45.294 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:45.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.346 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:45.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:45.447 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:45.476 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:45.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:45.576 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:45 smithi162 ceph-mon[104668]: pgmap v1936: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.40k op/s; 46073/316116 objects degraded (14.575%); 4334/316116 objects misplaced (1.371%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:32:45.578 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:45.613 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:45.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:45.664 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:45 smithi063 ceph-mon[114756]: pgmap v1936: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.40k op/s; 46073/316116 objects degraded (14.575%); 4334/316116 objects misplaced (1.371%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:32:45.664 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:45 smithi063 ceph-mon[109843]: pgmap v1936: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.40k op/s; 46073/316116 objects degraded (14.575%); 4334/316116 objects misplaced (1.371%); 1.3 MiB/s, 21 objects/s recovering 2024-08-17T17:32:45.715 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:45.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.750 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:45.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:45.851 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:45.900 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:45.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:45.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:45.908 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:45.921 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:32:45.921 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:32:45.921 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:32:45.921 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:174202 10.12 10:4cc65325:::benchmark_data_smithi063_294527_object37017:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1311)", 2024-08-17T17:32:45.922 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:32:45.918068+0000", 2024-08-17T17:32:45.922 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0024828129999999999, 2024-08-17T17:32:45.922 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0025032890000000001, 2024-08-17T17:32:45.922 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:32:45.922 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "waiting for sub ops", 2024-08-17T17:32:45.922 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:32:45.922 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:32:45.922 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:32:45.922 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 174202 2024-08-17T17:32:45.922 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:32:45.922 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:32:45.923 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:32:45.923 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:32:45.923 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:45.918068+0000", 2024-08-17T17:32:45.923 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:32:45.923 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:32:45.923 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:32:45.923 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:32:45.923 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:45.918068+0000", 2024-08-17T17:32:45.923 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:32:45.923 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:32:45.923 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:32:45.923 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:32:45.924 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:45.918067+0000", 2024-08-17T17:32:45.924 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:32:45.924 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:32:45.924 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:32:45.924 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:32:45.924 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:45.918073+0000", 2024-08-17T17:32:45.924 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8599999999999998e-06 2024-08-17T17:32:45.924 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:32:45.924 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:32:45.924 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:32:45.924 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:45.918085+0000", 2024-08-17T17:32:45.925 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2078e-05 2024-08-17T17:32:45.925 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:32:45.925 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:32:45.925 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:32:45.925 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:45.918091+0000", 2024-08-17T17:32:45.925 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8760000000000002e-06 2024-08-17T17:32:45.925 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:32:45.925 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:32:45.925 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:32:45.925 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:45.918131+0000", 2024-08-17T17:32:45.925 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9928000000000001e-05 2024-08-17T17:32:45.925 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:32:45.926 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:32:45.926 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:32:45.926 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:45.918217+0000", 2024-08-17T17:32:45.926 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6528999999999998e-05 2024-08-17T17:32:45.926 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:32:45.926 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:32:45.926 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:32:45.926 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:45.918546+0000", 2024-08-17T17:32:45.926 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00032902700000000002 2024-08-17T17:32:45.926 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:32:45.926 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:32:45.926 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:32:45.926 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:45.919952+0000", 2024-08-17T17:32:45.927 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0014060769999999999 2024-08-17T17:32:45.927 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:32:45.927 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:32:45.927 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:32:45.927 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:32:45.927 INFO:teuthology.orchestra.run.smithi162.stdout: ], 2024-08-17T17:32:45.927 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 1 2024-08-17T17:32:45.927 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:32:46.003 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:46.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.053 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:46.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:46.154 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:46.184 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:46.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:46.286 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:46.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.310 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:46.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:46.411 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:46.454 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:46.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:46.556 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:46.577 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:46.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:46.581 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:32:46.581 INFO:teuthology.orchestra.run.smithi063.stderr:set osd.7 primary-affinity to 0.223868 (8146712) 2024-08-17T17:32:46.678 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:46.717 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:46.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:46.818 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:46.820 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:32:46.844 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:46.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:46.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:46 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4090053594' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.22386826037767393} : dispatch 2024-08-17T17:32:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:46 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.22386826037767393} : dispatch 2024-08-17T17:32:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:46 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1425828580' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:46.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:46 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:46.945 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:46.968 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:46.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:46.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:46 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4090053594' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.22386826037767393} : dispatch 2024-08-17T17:32:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:46 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.22386826037767393} : dispatch 2024-08-17T17:32:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:46 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1425828580' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:47.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:46 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:46 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4090053594' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.22386826037767393} : dispatch 2024-08-17T17:32:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:46 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 0.22386826037767393} : dispatch 2024-08-17T17:32:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:32:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:46 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1425828580' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:46 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:32:47.050 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:32:47.070 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:47.093 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:47.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:47.195 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:47.217 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:47.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:47.319 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:47.341 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:47.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:47.443 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:47.465 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:47.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:47.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:47.593 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:47.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:47.695 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:47.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.723 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:47.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:47.824 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:47.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:47 smithi162 ceph-mon[104668]: pgmap v1937: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.38k op/s; 45634/314799 objects degraded (14.496%); 4334/314799 objects misplaced (1.377%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:32:47.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:47 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:47.848 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:47 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.22386826037767393}]': finished 2024-08-17T17:32:47.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:47 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:32:47.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:47 smithi162 ceph-mon[104668]: osdmap e1312: 8 total, 7 up, 7 in 2024-08-17T17:32:47.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:47 smithi162 ceph-mon[104668]: osdmap e1313: 8 total, 7 up, 7 in 2024-08-17T17:32:47.865 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:47.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:47.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:47.967 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:48.001 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:48.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:47 smithi063 ceph-mon[114756]: pgmap v1937: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.38k op/s; 45634/314799 objects degraded (14.496%); 4334/314799 objects misplaced (1.377%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:32:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:47 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:47 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.22386826037767393}]': finished 2024-08-17T17:32:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:47 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:32:48.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:47 smithi063 ceph-mon[114756]: osdmap e1312: 8 total, 7 up, 7 in 2024-08-17T17:32:48.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:47 smithi063 ceph-mon[114756]: osdmap e1313: 8 total, 7 up, 7 in 2024-08-17T17:32:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:47 smithi063 ceph-mon[109843]: pgmap v1937: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 6.4 GiB data, 22 GiB used, 604 GiB / 626 GiB avail; 0 B/s wr, 1.38k op/s; 45634/314799 objects degraded (14.496%); 4334/314799 objects misplaced (1.377%); 1.3 MiB/s, 20 objects/s recovering 2024-08-17T17:32:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:47 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:32:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:47 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.22386826037767393}]': finished 2024-08-17T17:32:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:47 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:32:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:47 smithi063 ceph-mon[109843]: osdmap e1312: 8 total, 7 up, 7 in 2024-08-17T17:32:48.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:47 smithi063 ceph-mon[109843]: osdmap e1313: 8 total, 7 up, 7 in 2024-08-17T17:32:48.103 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:48.145 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:48.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:48.247 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:48.274 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:48.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:48.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:48.402 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:48.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:48.504 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:48.537 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:48.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:48.639 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:48.673 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:48.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:48.775 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:48.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.804 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:48.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:48.899 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:48 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 45634/314799 objects degraded (14.496%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:48.899 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:48 smithi162 ceph-mon[104668]: osdmap e1314: 8 total, 7 up, 7 in 2024-08-17T17:32:48.905 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:48.942 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:48.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:48.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:48 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 45634/314799 objects degraded (14.496%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:48 smithi063 ceph-mon[109843]: osdmap e1314: 8 total, 7 up, 7 in 2024-08-17T17:32:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:48 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 45634/314799 objects degraded (14.496%), 14 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:49.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:48 smithi063 ceph-mon[114756]: osdmap e1314: 8 total, 7 up, 7 in 2024-08-17T17:32:49.044 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:49.055 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:32:49.055 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:32:49.055 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:32:49.055 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:32:49.055 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:32:49.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.085 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:49.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:49.186 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:49.218 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:49.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:49.319 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:49.351 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:49.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:49.453 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:49.484 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:49.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:49.586 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:49.623 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:49.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:49.721 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:49 smithi162 ceph-mon[104668]: pgmap v1940: 129 pgs: 2 remapped+peering, 17 peering, 2 active+recovery_wait, 19 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 12 active+undersized+degraded, 58 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 9.5 KiB/s rd, 27 KiB/s wr, 1.64k op/s; 43293/297708 objects degraded (14.542%); 4197/297708 objects misplaced (1.410%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:32:49.722 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:49 smithi162 ceph-mon[104668]: Health check failed: Reduced data availability: 12 pgs inactive, 16 pgs peering (PG_AVAILABILITY) 2024-08-17T17:32:49.722 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:49 smithi162 ceph-mon[104668]: 1.9 scrub starts 2024-08-17T17:32:49.722 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:49 smithi162 ceph-mon[104668]: 1.9 scrub ok 2024-08-17T17:32:49.725 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:49.758 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:32:49.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.759 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:49.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:49.860 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:49.891 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:49 smithi063 ceph-mon[109843]: pgmap v1940: 129 pgs: 2 remapped+peering, 17 peering, 2 active+recovery_wait, 19 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 12 active+undersized+degraded, 58 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 9.5 KiB/s rd, 27 KiB/s wr, 1.64k op/s; 43293/297708 objects degraded (14.542%); 4197/297708 objects misplaced (1.410%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:32:49.891 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:49 smithi063 ceph-mon[109843]: Health check failed: Reduced data availability: 12 pgs inactive, 16 pgs peering (PG_AVAILABILITY) 2024-08-17T17:32:49.891 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:49 smithi063 ceph-mon[109843]: 1.9 scrub starts 2024-08-17T17:32:49.891 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:49 smithi063 ceph-mon[109843]: 1.9 scrub ok 2024-08-17T17:32:49.892 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:49 smithi063 ceph-mon[114756]: pgmap v1940: 129 pgs: 2 remapped+peering, 17 peering, 2 active+recovery_wait, 19 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 12 active+undersized+degraded, 58 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 9.5 KiB/s rd, 27 KiB/s wr, 1.64k op/s; 43293/297708 objects degraded (14.542%); 4197/297708 objects misplaced (1.410%); 2.0 MiB/s, 31 objects/s recovering 2024-08-17T17:32:49.892 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:49 smithi063 ceph-mon[114756]: Health check failed: Reduced data availability: 12 pgs inactive, 16 pgs peering (PG_AVAILABILITY) 2024-08-17T17:32:49.892 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:49 smithi063 ceph-mon[114756]: 1.9 scrub starts 2024-08-17T17:32:49.892 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:49 smithi063 ceph-mon[114756]: 1.9 scrub ok 2024-08-17T17:32:49.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:49.893 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:49.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:49.994 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:50.027 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:50.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:50.106 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:32:50.129 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:50.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.168 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:50.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:50.269 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:50.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:50.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:50.401 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:50.435 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:50.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:50.537 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:50.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.571 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:50.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:50.672 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:50.711 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:50.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:50.813 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:50.845 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:50.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:50.947 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:50.977 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:50.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:50.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:51.079 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:51.124 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:51.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:51.226 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:51.257 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:51.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:51.359 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:51.389 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:51.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:51.491 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:51.514 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:51.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:51.618 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:51.650 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:51.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:51.752 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:51.787 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:51.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:51.889 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:51.926 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:51.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:51.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:52.028 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:52.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:51 smithi063 ceph-mon[114756]: pgmap v1942: 129 pgs: 2 remapped+peering, 17 peering, 2 active+recovery_wait, 19 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 12 active+undersized+degraded, 58 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 13 KiB/s rd, 36 KiB/s wr, 1.03k op/s; 43293/297708 objects degraded (14.542%); 4197/297708 objects misplaced (1.410%); 1.5 MiB/s, 22 objects/s recovering 2024-08-17T17:32:52.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:51 smithi063 ceph-mon[109843]: pgmap v1942: 129 pgs: 2 remapped+peering, 17 peering, 2 active+recovery_wait, 19 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 12 active+undersized+degraded, 58 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 13 KiB/s rd, 36 KiB/s wr, 1.03k op/s; 43293/297708 objects degraded (14.542%); 4197/297708 objects misplaced (1.410%); 1.5 MiB/s, 22 objects/s recovering 2024-08-17T17:32:52.064 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:52.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:52.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:51 smithi162 ceph-mon[104668]: pgmap v1942: 129 pgs: 2 remapped+peering, 17 peering, 2 active+recovery_wait, 19 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 18 active+undersized, 12 active+undersized+degraded, 58 active+clean; 6.1 GiB data, 21 GiB used, 605 GiB / 626 GiB avail; 13 KiB/s rd, 36 KiB/s wr, 1.03k op/s; 43293/297708 objects degraded (14.542%); 4197/297708 objects misplaced (1.410%); 1.5 MiB/s, 22 objects/s recovering 2024-08-17T17:32:52.167 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:52.216 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:52.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:52.318 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:52.350 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:52.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:52.368 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:32:52.369 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:32:52.404 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:32:52.404 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:32:52.440 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:32:52.440 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:32:52.452 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:52.472 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:32:52.472 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:32:52.491 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:52.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:52.499 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:32:52.499 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:32:52.522 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:32:52.523 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:32:52.523 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:32:52.551 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:32:52.551 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:32:52.593 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:52.620 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:52.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:52.722 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:52.758 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:52.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:52.861 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:52.896 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:52.896 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:52 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1754251307' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:52.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:52.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:52.998 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:52 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1754251307' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:52 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1754251307' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:32:53.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.032 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:53.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:53.134 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:53.170 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:53.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:53.272 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:53.308 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:53.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:53.362 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:32:53.362 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:32:53.410 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:53.436 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:32:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:32:53] "GET /metrics HTTP/1.1" 200 36603 "" "Prometheus/2.51.0" 2024-08-17T17:32:53.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.451 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:53.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:53.473 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:32:53.474 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:32:53.552 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:53.586 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:53.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:53.687 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:53.715 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:53.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:53.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:53 smithi063 ceph-mon[109843]: pgmap v1943: 129 pgs: 1 active+clean+scrubbing, 4 peering, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 20 active+undersized, 13 active+undersized+degraded, 67 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 9.3 KiB/s rd, 27 KiB/s wr, 1.78k op/s; 40645/282813 objects degraded (14.372%); 4048/282813 objects misplaced (1.431%); 3.0 MiB/s, 46 objects/s recovering 2024-08-17T17:32:53.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:53 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 40645/282813 objects degraded (14.372%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:53.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:53 smithi063 ceph-mon[114756]: pgmap v1943: 129 pgs: 1 active+clean+scrubbing, 4 peering, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 20 active+undersized, 13 active+undersized+degraded, 67 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 9.3 KiB/s rd, 27 KiB/s wr, 1.78k op/s; 40645/282813 objects degraded (14.372%); 4048/282813 objects misplaced (1.431%); 3.0 MiB/s, 46 objects/s recovering 2024-08-17T17:32:53.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:53 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 40645/282813 objects degraded (14.372%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:53.817 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:53.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.854 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:53.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:53.955 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:53.990 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:53.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:53.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:54.050 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:53 smithi162 ceph-mon[104668]: pgmap v1943: 129 pgs: 1 active+clean+scrubbing, 4 peering, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 20 active+undersized, 13 active+undersized+degraded, 67 active+clean; 5.8 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 9.3 KiB/s rd, 27 KiB/s wr, 1.78k op/s; 40645/282813 objects degraded (14.372%); 4048/282813 objects misplaced (1.431%); 3.0 MiB/s, 46 objects/s recovering 2024-08-17T17:32:54.051 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:53 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 40645/282813 objects degraded (14.372%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:54.088 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:32:54.092 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:54.103 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:32:54.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.135 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:54.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:54.236 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:54.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.273 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:54.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:54.374 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:54.387 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:32:54.417 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:54.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:54.519 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:54.540 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:54.561 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:54.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:54.663 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:54.697 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:54.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:54.799 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:54.821 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:54 smithi162 ceph-mon[104668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-08-17T17:32:54.848 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:54.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:54.861 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:54 smithi063 ceph-mon[114756]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-08-17T17:32:54.861 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:54 smithi063 ceph-mon[109843]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-08-17T17:32:54.950 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:54.981 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:54.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:54.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:55.083 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:55.125 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:55.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:55.226 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:55.263 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:55.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:55.365 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:55.399 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:55.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:55.474 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:32:55.500 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:55.561 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:55.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:55.662 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:55.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.753 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:55.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:55.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:55 smithi063 ceph-mon[114756]: pgmap v1944: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 2.3 KiB/s rd, 6.9 KiB/s wr, 1.49k op/s; 40645/279219 objects degraded (14.557%); 4048/279219 objects misplaced (1.450%); 2.3 MiB/s, 35 objects/s recovering 2024-08-17T17:32:55.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:55 smithi063 ceph-mon[109843]: pgmap v1944: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 2.3 KiB/s rd, 6.9 KiB/s wr, 1.49k op/s; 40645/279219 objects degraded (14.557%); 4048/279219 objects misplaced (1.450%); 2.3 MiB/s, 35 objects/s recovering 2024-08-17T17:32:55.855 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:55.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:55.895 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:55.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:55.968 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:55 smithi162 ceph-mon[104668]: pgmap v1944: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 2.3 KiB/s rd, 6.9 KiB/s wr, 1.49k op/s; 40645/279219 objects degraded (14.557%); 4048/279219 objects misplaced (1.450%); 2.3 MiB/s, 35 objects/s recovering 2024-08-17T17:32:55.996 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:56.034 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:56.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:56.136 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:56.171 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:56.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:56.273 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:56.287 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:32:56.287 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:32:56.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.309 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:56.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:56.375 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0] out_osds: [1] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:32:56.376 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:32:56.376 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2024-08-17T17:32:56.376 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:32:56.376 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:32:56.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 config set filestore_inject_stall 3 2024-08-17T17:32:56.410 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:56.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.470 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:56.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:56.571 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:56.599 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:56.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:56.701 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:56.758 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:56.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:56.860 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:56.899 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:56.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:56.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:57.001 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:57.038 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:57.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:57.043 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:32:57.129 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:57.140 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:57.168 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:57.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:57.205 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:32:57.270 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:57.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.323 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:57.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:57.334 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:32:57.424 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:57.453 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:57.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:57.555 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:57.584 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:57.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:57.686 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:57.708 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:32:57.730 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:57.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:57.755 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:57 smithi063 ceph-mon[114756]: pgmap v1945: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 40283/278133 objects degraded (14.483%); 4048/278133 objects misplaced (1.455%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:32:57.755 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:57 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1117918442' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:57.755 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:57 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:57.756 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:57 smithi063 ceph-mon[109843]: pgmap v1945: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 40283/278133 objects degraded (14.483%); 4048/278133 objects misplaced (1.455%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:32:57.756 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:57 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1117918442' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:57.756 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:57 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:57.832 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:32:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:57 smithi162 ceph-mon[104668]: pgmap v1945: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 5.7 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 40283/278133 objects degraded (14.483%); 4048/278133 objects misplaced (1.455%); 1.0 MiB/s, 16 objects/s recovering 2024-08-17T17:32:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:57 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1117918442' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:57 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:32:57.869 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:57.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:57.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:32:57.971 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:58.011 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:58.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:58.113 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:58.140 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:58.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:58.242 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:58.274 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:58.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:58.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:58.409 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:58.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:32:58.511 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:58.542 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:58.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:58.643 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:32:58.690 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:58.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:32:58.791 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:58.821 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:58.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:58.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:58.948 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:58 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:32:58.948 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:58 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:58.949 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:58 smithi063 ceph-mon[109843]: osdmap e1315: 8 total, 7 up, 7 in 2024-08-17T17:32:58.949 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:58 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 40283/278133 objects degraded (14.483%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:58.949 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:58 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:32:58.949 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:58 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:58.949 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:58 smithi063 ceph-mon[114756]: osdmap e1315: 8 total, 7 up, 7 in 2024-08-17T17:32:58.949 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:58 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 40283/278133 objects degraded (14.483%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:58.957 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:58.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:58.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:59.059 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:59.081 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:59.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:58 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:32:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:58 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:32:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:58 smithi162 ceph-mon[104668]: osdmap e1315: 8 total, 7 up, 7 in 2024-08-17T17:32:59.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:58 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 40283/278133 objects degraded (14.483%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:32:59.183 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:59.222 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:59.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:59.324 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:32:59.358 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:59.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:32:59.460 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:32:59.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.492 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:59.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:32:59.593 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:32:59.626 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:32:59.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:32:59.728 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:32:59.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.763 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:59.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:32:59.864 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:32:59.899 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:32:59.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:32:59.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:00.001 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:00.025 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:00.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:32:59 smithi063 ceph-mon[114756]: pgmap v1947: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 37988/260475 objects degraded (14.584%); 3914/260475 objects misplaced (1.503%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:33:00.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:32:59 smithi063 ceph-mon[109843]: pgmap v1947: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 37988/260475 objects degraded (14.584%); 3914/260475 objects misplaced (1.503%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:33:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:32:59 smithi162 ceph-mon[104668]: pgmap v1947: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.25k op/s; 37988/260475 objects degraded (14.584%); 3914/260475 objects misplaced (1.503%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:33:00.127 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:00.156 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:00.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:00.258 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:00.294 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:00.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:00.396 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:00.431 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:00.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:00.533 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:00.559 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:00.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:00.661 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:00.700 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:00.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:00.803 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:00.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.838 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:00.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:00.939 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:00.966 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:00.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:00.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:01.068 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:01.098 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:01.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:01.200 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:01.234 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:01.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:01.335 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:01.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.383 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:01.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:01.431 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:33:01.431 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:33:01.434 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:33:01.454 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:33:01.454 INFO:teuthology.orchestra.run.smithi063.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:33:01.455 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:33:01.467 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:33:01.467 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:33:01.485 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:01.513 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:33:01.513 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:33:01.531 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:01.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:01.548 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:33:01.548 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:33:01.579 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:33:01.580 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:33:01.613 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:33:01.613 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:33:01.614 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:33:01.633 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:01.645 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:33:01.646 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:33:01.660 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:01.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:01.762 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:01 smithi063 ceph-mon[114756]: pgmap v1948: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.24k op/s; 37988/260475 objects degraded (14.584%); 3914/260475 objects misplaced (1.503%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:33:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:01 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/520446254' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:01.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:01 smithi063 ceph-mon[109843]: pgmap v1948: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.24k op/s; 37988/260475 objects degraded (14.584%); 3914/260475 objects misplaced (1.503%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:33:01.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:01.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:01.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:01 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/520446254' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:01.806 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:01.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:01.908 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:01.936 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:01.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:01.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:02.038 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:02.081 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:02.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:01 smithi162 ceph-mon[104668]: pgmap v1948: 129 pgs: 1 active+clean+scrubbing, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 1.24k op/s; 37988/260475 objects degraded (14.584%); 3914/260475 objects misplaced (1.503%); 1.8 MiB/s, 28 objects/s recovering 2024-08-17T17:33:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:01 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/520446254' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:02.182 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:02.213 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:02.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:02.314 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:02.345 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:02.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:02.446 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:02.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.479 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:02.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:02.580 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:02.614 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:02.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:02.715 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:02.751 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:02.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:02.853 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:02.874 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:02.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:02.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:02.976 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:03.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.009 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:03.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:03.111 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:03.142 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:03.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:03.244 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:03.280 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:03.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:03.382 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:03.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.406 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:03.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:03.507 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:03.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:33:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:33:03] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-08-17T17:33:03.530 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:03.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:03.631 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:03.652 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:03.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:03.754 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:03.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.792 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:03.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:03.894 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:03.919 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:03.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:03.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:04.020 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:04.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:03 smithi063 ceph-mon[114756]: pgmap v1949: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 35345/245814 objects degraded (14.379%); 3781/245814 objects misplaced (1.538%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:33:04.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:03 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 35345/245814 objects degraded (14.379%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:04.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:03 smithi063 ceph-mon[109843]: pgmap v1949: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 35345/245814 objects degraded (14.379%); 3781/245814 objects misplaced (1.538%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:33:04.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:03 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 35345/245814 objects degraded (14.379%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:04.045 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:04.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:04.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:03 smithi162 ceph-mon[104668]: pgmap v1949: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 35345/245814 objects degraded (14.379%); 3781/245814 objects misplaced (1.538%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:33:04.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:03 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 35345/245814 objects degraded (14.379%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:04.147 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:04.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.170 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:04.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:04.271 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:04.315 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:04.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:04.418 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:04.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.453 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:04.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:04.554 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:04.590 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:04.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:04.691 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:04.724 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:04.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:04.826 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:04.863 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:04.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:04.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:04.965 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:05.001 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:05.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:05.103 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:05.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.147 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:05.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:05.248 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:05.281 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:05.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:05.383 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:05.429 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:05.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:05.445 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:33:05.445 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:33:05.510 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:05.530 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:05.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.578 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:05.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:05.679 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:05.719 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:05.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:05.821 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:05.865 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:05.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:05.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:05.967 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:05.989 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:05 smithi063 ceph-mon[114756]: pgmap v1950: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 35345/242310 objects degraded (14.587%); 3781/242310 objects misplaced (1.560%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:33:05.990 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:05 smithi063 ceph-mon[109843]: pgmap v1950: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 35345/242310 objects degraded (14.587%); 3781/242310 objects misplaced (1.560%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:33:06.003 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:06.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:05 smithi162 ceph-mon[104668]: pgmap v1950: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 35345/242310 objects degraded (14.587%); 3781/242310 objects misplaced (1.560%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:33:06.105 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:06.130 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:06.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:06.232 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:06.263 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:06.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:06.365 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:06.406 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:06.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:06.508 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:06.541 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:06.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:06.643 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:06.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.677 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:06.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:06.778 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:06.812 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:06.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:06.913 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:06.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:06.945 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:06.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:07.046 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:07.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.080 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:07.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:07.181 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:07.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.221 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:07.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:07.322 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:07.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.363 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:07.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:07.446 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:33:07.465 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:07.496 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:07.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:07.598 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:07.638 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:07.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:07.739 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:07.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.788 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:07.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:07.809 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:07.889 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:07.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:07.927 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:07.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:08.028 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:07 smithi063 ceph-mon[114756]: pgmap v1951: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 35008/241299 objects degraded (14.508%); 3781/241299 objects misplaced (1.567%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:33:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:07 smithi063 ceph-mon[109843]: pgmap v1951: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 35008/241299 objects degraded (14.508%); 3781/241299 objects misplaced (1.567%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:33:08.059 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:07 smithi162 ceph-mon[104668]: pgmap v1951: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 1.23k op/s; 35008/241299 objects degraded (14.508%); 3781/241299 objects misplaced (1.567%); 1.7 MiB/s, 26 objects/s recovering 2024-08-17T17:33:08.066 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:08.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:08.168 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:08.207 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:08.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:08.308 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:08.345 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:08.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:08.374 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:33:08.374 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:33:08.374 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:33:08.374 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:33:08.375 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:33:08.447 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:08.487 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:08.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:08.589 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:08.628 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:08.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:08.729 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:08.761 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:33:08.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.770 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:08.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:08.871 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:08.906 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:08.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:08.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:09.007 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:09.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:08 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2805960256' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:08 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:09.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:08 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 35008/241299 objects degraded (14.508%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:08 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2805960256' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:08 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:09.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:08 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 35008/241299 objects degraded (14.508%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:09.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.045 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:09.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:09.066 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:08 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2805960256' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:09.066 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:08 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:33:09.066 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:08 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 35008/241299 objects degraded (14.508%), 14 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:09.102 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:33:09.146 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:09.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.178 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:09.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:09.278 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:09.321 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:09.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:09.423 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:09.456 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:09.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.463 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:09.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:09.564 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:09.600 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:09.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:09.702 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:09.743 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:09.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:09.809 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:09 smithi063 ceph-mon[109843]: pgmap v1952: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 32946/225606 objects degraded (14.603%); 3583/225606 objects misplaced (1.588%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:33:09.809 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:09 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:09.809 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:09 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:33:09.809 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:09 smithi063 ceph-mon[109843]: osdmap e1316: 8 total, 7 up, 7 in 2024-08-17T17:33:09.809 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:09 smithi063 ceph-mon[114756]: pgmap v1952: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 32946/225606 objects degraded (14.603%); 3583/225606 objects misplaced (1.588%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:33:09.809 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:09 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:09.809 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:09 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:33:09.810 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:09 smithi063 ceph-mon[114756]: osdmap e1316: 8 total, 7 up, 7 in 2024-08-17T17:33:09.845 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:09.849 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:09 smithi162 ceph-mon[104668]: pgmap v1952: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.07k op/s; 32946/225606 objects degraded (14.603%); 3583/225606 objects misplaced (1.588%); 1.9 MiB/s, 30 objects/s recovering 2024-08-17T17:33:09.850 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:09 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:09.850 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:09 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:33:09.850 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:09 smithi162 ceph-mon[104668]: osdmap e1316: 8 total, 7 up, 7 in 2024-08-17T17:33:09.881 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:09.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:09.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:09.983 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:10.022 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:10.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:10.037 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0] out_osds: [1] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:33:10.037 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:33:10.037 INFO:tasks.thrashosds.thrasher:Reweighting osd 3 to 0.2641110121186118 2024-08-17T17:33:10.037 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 3 0.2641110121186118 2024-08-17T17:33:10.124 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:10.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.180 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:10.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:10.280 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:10.308 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:10.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:10.410 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.453 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:10.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:10.492 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:33:10.493 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:33:10.506 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:10.525 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:33:10.525 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:33:10.554 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:10.562 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:33:10.563 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:33:10.596 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:10.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:10.598 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:33:10.598 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:33:10.634 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:33:10.634 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:33:10.667 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:33:10.668 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:33:10.671 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:33:10.699 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:10.705 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:33:10.705 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:33:10.749 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:10.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:10.851 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:10.885 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:10.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:10.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:10.987 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:11.015 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:11.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:11.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:10 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4157640437' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:11.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:10 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4157640437' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:11.076 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:10 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4157640437' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:11.117 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:11.160 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:11.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:11.262 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:11.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:11.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:11.400 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:11.438 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:11.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:11.536 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:33:11.537 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:33:11.540 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:11.577 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:11.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:11.679 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:11.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.717 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:11.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:11.727 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:33:11.727 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:33:11.818 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:11.848 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:11.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:11.950 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:11.980 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:11.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:11.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:11 smithi063 ceph-mon[114756]: pgmap v1954: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s; 32946/225606 objects degraded (14.603%); 3583/225606 objects misplaced (1.588%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:33:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:11 smithi063 ceph-mon[109843]: pgmap v1954: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s; 32946/225606 objects degraded (14.603%); 3583/225606 objects misplaced (1.588%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:33:12.083 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:11 smithi162 ceph-mon[104668]: pgmap v1954: 129 pgs: 1 active+clean+scrubbing+deep, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 70 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.16k op/s; 32946/225606 objects degraded (14.603%); 3583/225606 objects misplaced (1.588%); 2.1 MiB/s, 33 objects/s recovering 2024-08-17T17:33:12.119 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:12.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:12.156 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:33:12.221 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:12.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.250 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:12.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:12.260 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:33:12.351 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:12.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.392 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:12.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:12.493 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:12.530 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:12.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:12.632 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:12.639 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:12.675 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:12.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:12.776 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:12.809 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:12.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:12.912 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:12.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:12.952 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:12.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:13.053 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:13.097 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:13.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:13.200 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:13.222 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:13.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:13.324 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:13.351 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:13.357 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:13.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:13.459 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:13.500 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:13.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:13.503 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:33:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:33:13] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-08-17T17:33:13.603 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:13.631 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:13.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:13.728 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:33:13.733 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:13.813 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.3 to 0.264111 (439c) 2024-08-17T17:33:13.838 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:13.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:13.941 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:13.962 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:13.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:13.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:13 smithi063 ceph-mon[109843]: pgmap v1955: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 30214/210522 objects degraded (14.352%); 3437/210522 objects misplaced (1.633%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:33:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:13 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 30214/210522 objects degraded (14.352%), 14 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:13 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1542679005' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 0.2641110121186118} : dispatch 2024-08-17T17:33:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:13 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 0.2641110121186118} : dispatch 2024-08-17T17:33:14.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:13 smithi063 ceph-mon[114756]: pgmap v1955: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 30214/210522 objects degraded (14.352%); 3437/210522 objects misplaced (1.633%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:33:14.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:13 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 30214/210522 objects degraded (14.352%), 14 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:14.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:13 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1542679005' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 0.2641110121186118} : dispatch 2024-08-17T17:33:14.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:13 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 0.2641110121186118} : dispatch 2024-08-17T17:33:14.063 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:14.085 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:14.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:13 smithi162 ceph-mon[104668]: pgmap v1955: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 4.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.18k op/s; 30214/210522 objects degraded (14.352%); 3437/210522 objects misplaced (1.633%); 2.1 MiB/s, 34 objects/s recovering 2024-08-17T17:33:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:13 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 30214/210522 objects degraded (14.352%), 14 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:13 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1542679005' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 0.2641110121186118} : dispatch 2024-08-17T17:33:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:13 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 0.2641110121186118} : dispatch 2024-08-17T17:33:14.187 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:14.209 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:14.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:14.311 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:14.334 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:14.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:14.436 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:14.458 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:14.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:14.560 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:14.582 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:14.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:14.684 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:14.706 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:14.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:14.807 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:14.838 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:14.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:14.940 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:14.962 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:14.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:14.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:15.064 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:15.086 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:15.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:15.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:14 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.2641110121186118}]': finished 2024-08-17T17:33:15.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:14 smithi162 ceph-mon[104668]: osdmap e1317: 8 total, 7 up, 7 in 2024-08-17T17:33:15.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:14 smithi162 ceph-mon[104668]: osdmap e1318: 8 total, 7 up, 7 in 2024-08-17T17:33:15.188 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:15.209 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:15.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:15.211 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:14 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.2641110121186118}]': finished 2024-08-17T17:33:15.212 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:14 smithi063 ceph-mon[114756]: osdmap e1317: 8 total, 7 up, 7 in 2024-08-17T17:33:15.212 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:14 smithi063 ceph-mon[114756]: osdmap e1318: 8 total, 7 up, 7 in 2024-08-17T17:33:15.212 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:14 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 0.2641110121186118}]': finished 2024-08-17T17:33:15.212 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:14 smithi063 ceph-mon[109843]: osdmap e1317: 8 total, 7 up, 7 in 2024-08-17T17:33:15.212 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:14 smithi063 ceph-mon[109843]: osdmap e1318: 8 total, 7 up, 7 in 2024-08-17T17:33:15.311 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:15.334 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:15.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:15.436 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:15.459 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:15.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:15.561 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:15.582 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:15.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:15.684 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:15.705 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:15.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:15.807 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:15.846 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:15.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:15.948 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:15.973 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:15.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:15.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:15.977 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:15 smithi063 ceph-mon[109843]: pgmap v1957: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.42k op/s; 30214/207129 objects degraded (14.587%); 3437/207129 objects misplaced (1.659%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:33:15.977 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:15 smithi063 ceph-mon[109843]: osdmap e1319: 8 total, 7 up, 7 in 2024-08-17T17:33:15.977 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:15 smithi063 ceph-mon[114756]: pgmap v1957: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.42k op/s; 30214/207129 objects degraded (14.587%); 3437/207129 objects misplaced (1.659%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:33:15.978 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:15 smithi063 ceph-mon[114756]: osdmap e1319: 8 total, 7 up, 7 in 2024-08-17T17:33:16.075 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:15 smithi162 ceph-mon[104668]: pgmap v1957: 129 pgs: 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 21 active+undersized, 13 active+undersized+degraded, 71 active+clean; 4.2 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 1.42k op/s; 30214/207129 objects degraded (14.587%); 3437/207129 objects misplaced (1.659%); 2.7 MiB/s, 42 objects/s recovering 2024-08-17T17:33:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:15 smithi162 ceph-mon[104668]: osdmap e1319: 8 total, 7 up, 7 in 2024-08-17T17:33:16.116 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:16.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:16.218 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:16.258 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:16.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:16.305 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:16.361 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:16.387 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:16.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:16.488 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:16.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.530 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:16.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:16.631 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:16.654 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:16.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:16.756 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:16.800 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:16.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:16.813 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:33:16.903 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:16.935 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:16.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:16.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:17.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[109843]: 10.18 scrub starts 2024-08-17T17:33:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[109843]: 10.13 starting backfill to osd.4 from (0'0,0'0] MAX to 1316'10439 2024-08-17T17:33:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[109843]: 10.3 starting backfill to osd.4 from (0'0,0'0] MAX to 1316'10541 2024-08-17T17:33:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[109843]: 10.7 starting backfill to osd.2 from (0'0,0'0] MAX to 1316'10386 2024-08-17T17:33:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[109843]: 10.17 starting backfill to osd.2 from (0'0,0'0] MAX to 1316'10403 2024-08-17T17:33:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[109843]: 10.15 starting backfill to osd.7 from (0'0,0'0] MAX to 1316'10498 2024-08-17T17:33:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[109843]: 10.5 starting backfill to osd.7 from (0'0,0'0] MAX to 1316'13786 2024-08-17T17:33:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2068947784' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:17.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:17.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:17.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:17.031 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[109843]: osdmap e1320: 8 total, 7 up, 7 in 2024-08-17T17:33:17.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[114756]: 10.18 scrub starts 2024-08-17T17:33:17.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[114756]: 10.13 starting backfill to osd.4 from (0'0,0'0] MAX to 1316'10439 2024-08-17T17:33:17.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[114756]: 10.3 starting backfill to osd.4 from (0'0,0'0] MAX to 1316'10541 2024-08-17T17:33:17.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[114756]: 10.7 starting backfill to osd.2 from (0'0,0'0] MAX to 1316'10386 2024-08-17T17:33:17.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[114756]: 10.17 starting backfill to osd.2 from (0'0,0'0] MAX to 1316'10403 2024-08-17T17:33:17.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[114756]: 10.15 starting backfill to osd.7 from (0'0,0'0] MAX to 1316'10498 2024-08-17T17:33:17.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[114756]: 10.5 starting backfill to osd.7 from (0'0,0'0] MAX to 1316'13786 2024-08-17T17:33:17.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:17.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:17.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2068947784' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:17.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:17.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:17.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:17.032 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:16 smithi063 ceph-mon[114756]: osdmap e1320: 8 total, 7 up, 7 in 2024-08-17T17:33:17.038 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:17.083 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:17.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:16 smithi162 ceph-mon[104668]: 10.18 scrub starts 2024-08-17T17:33:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:16 smithi162 ceph-mon[104668]: 10.13 starting backfill to osd.4 from (0'0,0'0] MAX to 1316'10439 2024-08-17T17:33:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:16 smithi162 ceph-mon[104668]: 10.3 starting backfill to osd.4 from (0'0,0'0] MAX to 1316'10541 2024-08-17T17:33:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:16 smithi162 ceph-mon[104668]: 10.7 starting backfill to osd.2 from (0'0,0'0] MAX to 1316'10386 2024-08-17T17:33:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:16 smithi162 ceph-mon[104668]: 10.17 starting backfill to osd.2 from (0'0,0'0] MAX to 1316'10403 2024-08-17T17:33:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:16 smithi162 ceph-mon[104668]: 10.15 starting backfill to osd.7 from (0'0,0'0] MAX to 1316'10498 2024-08-17T17:33:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:16 smithi162 ceph-mon[104668]: 10.5 starting backfill to osd.7 from (0'0,0'0] MAX to 1316'13786 2024-08-17T17:33:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:16 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:16 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2068947784' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:16 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:16 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:17.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:16 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:17.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:16 smithi162 ceph-mon[104668]: osdmap e1320: 8 total, 7 up, 7 in 2024-08-17T17:33:17.185 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:17.212 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:17.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:17.314 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:17.348 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:17.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:17.450 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:17.487 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:17.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:17.588 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:17.610 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:17.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:17.712 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:17.733 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:17.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:17.834 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:17.877 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:17.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:17.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:17.978 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:18.011 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:18.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:17 smithi063 ceph-mon[109843]: pgmap v1960: 129 pgs: 2 active+undersized+remapped, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 13 active+undersized+degraded, 71 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 30004/206499 objects degraded (14.530%); 3437/206499 objects misplaced (1.664%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:33:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:17 smithi063 ceph-mon[109843]: osdmap e1321: 8 total, 7 up, 7 in 2024-08-17T17:33:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:17 smithi063 ceph-mon[114756]: pgmap v1960: 129 pgs: 2 active+undersized+remapped, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 13 active+undersized+degraded, 71 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 30004/206499 objects degraded (14.530%); 3437/206499 objects misplaced (1.664%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:33:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:17 smithi063 ceph-mon[114756]: osdmap e1321: 8 total, 7 up, 7 in 2024-08-17T17:33:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:17 smithi162 ceph-mon[104668]: pgmap v1960: 129 pgs: 2 active+undersized+remapped, 2 active+recovery_wait, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 19 active+undersized, 13 active+undersized+degraded, 71 active+clean; 4.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 1.06k op/s; 30004/206499 objects degraded (14.530%); 3437/206499 objects misplaced (1.664%); 1.5 MiB/s, 24 objects/s recovering 2024-08-17T17:33:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:17 smithi162 ceph-mon[104668]: osdmap e1321: 8 total, 7 up, 7 in 2024-08-17T17:33:18.113 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:18.146 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:18.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:18.248 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:18.278 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:18.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:18.380 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:18.422 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:18.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.423 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:18.523 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:18.572 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:18.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:18.674 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:18.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.709 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:18.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:18.809 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:18.839 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:18.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:18.932 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:33:18.932 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:33:18.941 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:18.966 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:33:18.966 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:33:18.973 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:18.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:18.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:19.002 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:33:19.003 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:33:19.035 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:33:19.035 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:33:19.073 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:33:19.074 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:33:19.075 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:19.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:18 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 30004/206499 objects degraded (14.530%), 14 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:19.107 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:33:19.107 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:33:19.108 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:18 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 30004/206499 objects degraded (14.530%), 14 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:19.108 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:18 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 30004/206499 objects degraded (14.530%), 14 pgs degraded, 27 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:19.108 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:33:19.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.113 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:19.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:19.145 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:33:19.145 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:33:19.215 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:19.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.257 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:19.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:19.358 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:19.392 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:19.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:19.494 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:19.528 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:19.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:19.630 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:19.662 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:19.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:19.717 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:33:19.764 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:19.821 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:19.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:19.924 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:19.954 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0] out_osds: [1] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:33:19.954 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:33:19.955 INFO:tasks.thrashosds.thrasher:Adding osd 1 2024-08-17T17:33:19.955 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd in 1 2024-08-17T17:33:19.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:19.961 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:19.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:19.994 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:19 smithi063 ceph-mon[109843]: pgmap v1963: 129 pgs: 1 active+remapped+backfilling, 4 peering, 2 active+undersized+degraded+remapped+backfill_wait, 5 remapped+peering, 36 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 10 active+undersized+degraded, 53 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 16 KiB/s rd, 46 KiB/s wr, 804 op/s; 28805/197610 objects degraded (14.577%); 19103/197610 objects misplaced (9.667%); 8.3 MiB/s, 130 objects/s recovering 2024-08-17T17:33:19.994 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:19 smithi063 ceph-mon[109843]: 1.19 scrub starts 2024-08-17T17:33:19.994 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:19 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1247166509' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:19.994 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:19 smithi063 ceph-mon[114756]: pgmap v1963: 129 pgs: 1 active+remapped+backfilling, 4 peering, 2 active+undersized+degraded+remapped+backfill_wait, 5 remapped+peering, 36 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 10 active+undersized+degraded, 53 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 16 KiB/s rd, 46 KiB/s wr, 804 op/s; 28805/197610 objects degraded (14.577%); 19103/197610 objects misplaced (9.667%); 8.3 MiB/s, 130 objects/s recovering 2024-08-17T17:33:19.994 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:19 smithi063 ceph-mon[114756]: 1.19 scrub starts 2024-08-17T17:33:19.994 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:19 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1247166509' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:20.062 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:20.099 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:20.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:20.111 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:33:20.111 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:33:20.111 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.111 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.45818.0:211872 10.0 e1321/1306 10:038af43e:::benchmark_data_smithi063_294527_object74687:head v 1321'10643, mlcod=1321'10639)", 2024-08-17T17:33:20.112 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:20.104242+0000", 2024-08-17T17:33:20.112 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.0055463539999999999, 2024-08-17T17:33:20.112 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0055759299999999998, 2024-08-17T17:33:20.112 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:20.112 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:33:20.112 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:20.112 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.112 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:20.112 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.104242+0000", 2024-08-17T17:33:20.112 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:20.112 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.113 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.113 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:20.113 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.104242+0000", 2024-08-17T17:33:20.113 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:20.113 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.113 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.113 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:20.113 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.104241+0000", 2024-08-17T17:33:20.113 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:33:20.113 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.113 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.113 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:20.114 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.104247+0000", 2024-08-17T17:33:20.114 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.229e-06 2024-08-17T17:33:20.114 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.114 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.114 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:20.114 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.104257+0000", 2024-08-17T17:33:20.114 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.6879999999999994e-06 2024-08-17T17:33:20.114 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.114 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.114 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:20.114 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.104263+0000", 2024-08-17T17:33:20.114 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3099999999999997e-06 2024-08-17T17:33:20.114 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.115 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.115 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:20.115 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.107724+0000", 2024-08-17T17:33:20.115 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0034609100000000002 2024-08-17T17:33:20.115 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.115 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.115 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:20.115 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.107761+0000", 2024-08-17T17:33:20.115 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6594000000000002e-05 2024-08-17T17:33:20.115 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:20.115 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:20.115 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:20.115 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.116 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.116 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.45818.0:211875 10.19 e1321/1306 10:98fbcd0a:::benchmark_data_smithi063_294527_object74690:head v 1321'10541, mlcod=1321'10539)", 2024-08-17T17:33:20.116 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:20.106872+0000", 2024-08-17T17:33:20.116 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.002915937, 2024-08-17T17:33:20.116 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0030192169999999998, 2024-08-17T17:33:20.116 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:20.116 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:33:20.116 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:20.116 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.116 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:20.116 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.106872+0000", 2024-08-17T17:33:20.116 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:20.116 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.116 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.117 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:20.117 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.106872+0000", 2024-08-17T17:33:20.117 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:20.117 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.117 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.117 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:20.117 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.106871+0000", 2024-08-17T17:33:20.117 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:33:20.117 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.117 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.117 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:20.117 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.106880+0000", 2024-08-17T17:33:20.117 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.1389999999999995e-06 2024-08-17T17:33:20.118 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.118 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.118 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:20.118 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.106895+0000", 2024-08-17T17:33:20.118 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5566e-05 2024-08-17T17:33:20.118 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.118 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.118 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:20.118 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.106903+0000", 2024-08-17T17:33:20.118 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4739999999999999e-06 2024-08-17T17:33:20.118 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.118 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.119 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:20.119 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.106945+0000", 2024-08-17T17:33:20.119 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2898999999999997e-05 2024-08-17T17:33:20.119 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.119 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.119 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:20.119 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.106965+0000", 2024-08-17T17:33:20.119 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9029999999999999e-05 2024-08-17T17:33:20.119 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:20.119 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:20.119 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:20.119 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.119 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.119 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.45818.0:211874 10.5 e1321/1318 10:a411d791:::benchmark_data_smithi063_294527_object74689:head v 1321'13928, mlcod=1321'13925)", 2024-08-17T17:33:20.119 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:20.108561+0000", 2024-08-17T17:33:20.120 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.001227176, 2024-08-17T17:33:20.120 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0013884990000000001, 2024-08-17T17:33:20.120 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:20.120 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:33:20.120 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:20.120 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.120 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:20.120 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.108561+0000", 2024-08-17T17:33:20.120 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:20.120 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.120 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.120 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:20.120 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.108561+0000", 2024-08-17T17:33:20.121 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:20.121 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.121 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.121 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:20.121 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.108560+0000", 2024-08-17T17:33:20.121 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:33:20.121 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.121 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.121 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:20.121 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.108567+0000", 2024-08-17T17:33:20.121 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1489999999999997e-06 2024-08-17T17:33:20.121 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.121 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.121 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:20.121 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.108580+0000", 2024-08-17T17:33:20.122 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.223e-05 2024-08-17T17:33:20.122 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.122 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.122 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:20.122 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.108586+0000", 2024-08-17T17:33:20.122 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.28e-06 2024-08-17T17:33:20.122 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.122 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.122 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:20.122 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.108618+0000", 2024-08-17T17:33:20.122 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1701e-05 2024-08-17T17:33:20.122 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:20.122 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:20.122 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:20.122 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:20.108638+0000", 2024-08-17T17:33:20.123 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0186999999999998e-05 2024-08-17T17:33:20.123 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:20.123 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:20.123 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:20.123 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:20.123 INFO:teuthology.orchestra.run.smithi162.stdout: ], 2024-08-17T17:33:20.123 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 3 2024-08-17T17:33:20.123 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:33:20.123 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:19 smithi162 ceph-mon[104668]: pgmap v1963: 129 pgs: 1 active+remapped+backfilling, 4 peering, 2 active+undersized+degraded+remapped+backfill_wait, 5 remapped+peering, 36 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 10 active+undersized+degraded, 53 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 16 KiB/s rd, 46 KiB/s wr, 804 op/s; 28805/197610 objects degraded (14.577%); 19103/197610 objects misplaced (9.667%); 8.3 MiB/s, 130 objects/s recovering 2024-08-17T17:33:20.123 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:19 smithi162 ceph-mon[104668]: 1.19 scrub starts 2024-08-17T17:33:20.124 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:19 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1247166509' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:20.201 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:20.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.245 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:20.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:20.346 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:20.376 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:20.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:20.404 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:33:20.405 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:33:20.477 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:20.541 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:20.542 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:20.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:20.544 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:20.643 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:20.677 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:20.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:20.778 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:20.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.822 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:20.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:20.830 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:33:20.924 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:20.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:20.971 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:20.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:20 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:33:21.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:20 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:33:21.072 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:21.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:20 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:33:21.128 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:21.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:21.230 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:21.268 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:21.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:21.424 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:21.457 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:21.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:21.559 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:21.598 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:21.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.599 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:21.699 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:21.738 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:21.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:21.840 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:21.881 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:21.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:21.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:21.883 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:21.983 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:22.032 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:22.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:22.038 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:21 smithi162 ceph-mon[104668]: pgmap v1964: 129 pgs: 1 active+remapped+backfilling, 4 peering, 2 active+undersized+degraded+remapped+backfill_wait, 5 remapped+peering, 36 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 10 active+undersized+degraded, 53 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 11 KiB/s rd, 32 KiB/s wr, 561 op/s; 28805/197610 objects degraded (14.577%); 19103/197610 objects misplaced (9.667%); 5.8 MiB/s, 91 objects/s recovering 2024-08-17T17:33:22.134 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:22.169 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:22.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:22.213 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:21 smithi063 ceph-mon[114756]: pgmap v1964: 129 pgs: 1 active+remapped+backfilling, 4 peering, 2 active+undersized+degraded+remapped+backfill_wait, 5 remapped+peering, 36 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 10 active+undersized+degraded, 53 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 11 KiB/s rd, 32 KiB/s wr, 561 op/s; 28805/197610 objects degraded (14.577%); 19103/197610 objects misplaced (9.667%); 5.8 MiB/s, 91 objects/s recovering 2024-08-17T17:33:22.213 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:21 smithi063 ceph-mon[109843]: pgmap v1964: 129 pgs: 1 active+remapped+backfilling, 4 peering, 2 active+undersized+degraded+remapped+backfill_wait, 5 remapped+peering, 36 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 10 active+undersized+degraded, 53 active+clean; 4.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 11 KiB/s rd, 32 KiB/s wr, 561 op/s; 28805/197610 objects degraded (14.577%); 19103/197610 objects misplaced (9.667%); 5.8 MiB/s, 91 objects/s recovering 2024-08-17T17:33:22.271 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:22.317 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:22.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:22.406 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:33:22.419 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:22.489 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:22.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:22.591 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:22.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.641 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:22.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:22.742 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:22.784 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:22.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:22.885 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:22.921 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:22.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:22.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:23.023 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:23.057 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:23.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:23.084 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:22 smithi063 ceph-mon[109843]: 10.18 scrub starts 2024-08-17T17:33:23.084 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:22 smithi063 ceph-mon[114756]: 10.18 scrub starts 2024-08-17T17:33:23.160 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:23.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.199 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:23.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:23.300 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:23.331 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:23.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:23.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:22 smithi162 ceph-mon[104668]: 10.18 scrub starts 2024-08-17T17:33:23.433 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:23.466 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:33:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:33:23] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.51.0" 2024-08-17T17:33:23.469 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:23.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:23.571 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:23.599 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:23.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:23.701 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:23.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.734 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:23.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:23.835 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:23.867 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:23.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:23.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:23.969 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:24.000 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:24.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:23 smithi063 ceph-mon[109843]: pgmap v1965: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped+scrubbing, 1 peering, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 10 active+undersized+degraded, 56 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 9.4 KiB/s rd, 27 KiB/s wr, 1.19k op/s; 26171/183159 objects degraded (14.289%); 18218/183159 objects misplaced (9.947%); 6.7 MiB/s, 105 objects/s recovering 2024-08-17T17:33:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:23 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 26171/183159 objects degraded (14.289%), 13 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:24.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:23 smithi063 ceph-mon[114756]: pgmap v1965: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped+scrubbing, 1 peering, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 10 active+undersized+degraded, 56 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 9.4 KiB/s rd, 27 KiB/s wr, 1.19k op/s; 26171/183159 objects degraded (14.289%); 18218/183159 objects misplaced (9.947%); 6.7 MiB/s, 105 objects/s recovering 2024-08-17T17:33:24.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:23 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 26171/183159 objects degraded (14.289%), 13 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:23 smithi162 ceph-mon[104668]: pgmap v1965: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped+scrubbing, 1 peering, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 10 active+undersized+degraded, 56 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 9.4 KiB/s rd, 27 KiB/s wr, 1.19k op/s; 26171/183159 objects degraded (14.289%); 18218/183159 objects misplaced (9.947%); 6.7 MiB/s, 105 objects/s recovering 2024-08-17T17:33:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:23 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 26171/183159 objects degraded (14.289%), 13 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:24.102 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:24.127 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:24.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:24.229 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:24.278 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:24.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:24.350 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:24 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a[109820]: 2024-08-17T17:33:24.220+0000 7f66fd20a640 -1 mon.a@0(leader).osd e1321 definitely_dead 0 2024-08-17T17:33:24.355 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:24.380 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:24.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.442 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:24.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:24.543 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:24.555 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:33:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:33:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:33:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:33:24.555 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:33:24.565 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:24.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:24.667 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:24.699 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:24.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:24.801 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:24.836 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:24.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:24.894 INFO:teuthology.orchestra.run.smithi063.stderr:marked in osd.1. 2024-08-17T17:33:24.896 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:33:24.937 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:24.980 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:24.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:24.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:25.082 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:25.103 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:25.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:25.205 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:25.230 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:25.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:24 smithi063 ceph-mon[109843]: 1.19 scrub starts 2024-08-17T17:33:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:24 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1711976227' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T17:33:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:24 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/387204286' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:24 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:24 smithi063 ceph-mon[114756]: 1.19 scrub starts 2024-08-17T17:33:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:24 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1711976227' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T17:33:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:24 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/387204286' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:24 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:25.286 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:24 smithi162 ceph-mon[104668]: 1.19 scrub starts 2024-08-17T17:33:25.287 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:24 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1711976227' entity='client.admin' cmd={"prefix": "osd in", "ids": ["1"]} : dispatch 2024-08-17T17:33:25.287 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:24 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/387204286' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:25.287 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:24 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:33:25.313 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:33:25.332 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:25.378 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:25.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:25.479 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:25.504 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:25.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:25.566 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:33:25.606 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:25.629 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:25.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:25.731 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:25.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.754 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:25.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:25.855 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:25.878 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:25.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:25.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:25.979 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:25.982 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:25 smithi162 ceph-mon[104668]: pgmap v1966: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped+scrubbing, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 10 active+undersized+degraded, 57 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 7.9 KiB/s rd, 23 KiB/s wr, 1.12k op/s; 26171/179784 objects degraded (14.557%); 16921/179784 objects misplaced (9.412%); 10 MiB/s, 162 objects/s recovering 2024-08-17T17:33:25.982 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:25 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:25.982 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:25 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1711976227' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-08-17T17:33:25.983 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:25 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:33:25.983 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:25 smithi162 ceph-mon[104668]: osdmap e1322: 8 total, 7 up, 8 in 2024-08-17T17:33:26.009 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:26.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:26.111 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:26.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.135 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:26.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:26.236 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:26.260 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:26.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:26.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:25 smithi063 ceph-mon[114756]: pgmap v1966: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped+scrubbing, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 10 active+undersized+degraded, 57 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 7.9 KiB/s rd, 23 KiB/s wr, 1.12k op/s; 26171/179784 objects degraded (14.557%); 16921/179784 objects misplaced (9.412%); 10 MiB/s, 162 objects/s recovering 2024-08-17T17:33:26.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:25 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:26.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:25 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1711976227' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-08-17T17:33:26.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:25 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:33:26.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:25 smithi063 ceph-mon[114756]: osdmap e1322: 8 total, 7 up, 8 in 2024-08-17T17:33:26.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:25 smithi063 ceph-mon[109843]: pgmap v1966: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped+scrubbing, 2 active+undersized+degraded+remapped+backfill_wait, 40 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 10 active+undersized+degraded, 57 active+clean; 3.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 7.9 KiB/s rd, 23 KiB/s wr, 1.12k op/s; 26171/179784 objects degraded (14.557%); 16921/179784 objects misplaced (9.412%); 10 MiB/s, 162 objects/s recovering 2024-08-17T17:33:26.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:25 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:33:26.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:25 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1711976227' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2024-08-17T17:33:26.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:25 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:33:26.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:25 smithi063 ceph-mon[109843]: osdmap e1322: 8 total, 7 up, 8 in 2024-08-17T17:33:26.363 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:26.388 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:26.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:26.490 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:26.513 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:26.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:26.615 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:26.640 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:26.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:26.742 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:26.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.767 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:26.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:26.868 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:26.892 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:26.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:26.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:26.994 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:27.029 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:27.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:26 smithi162 ceph-mon[104668]: osdmap e1323: 8 total, 7 up, 8 in 2024-08-17T17:33:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:26 smithi162 ceph-mon[104668]: 10.2 starting backfill to osd.1 from (0'0,0'0] MAX to 1321'10941 2024-08-17T17:33:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:26 smithi162 ceph-mon[104668]: 10.12 starting backfill to osd.1 from (0'0,0'0] MAX to 1321'10907 2024-08-17T17:33:27.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:26 smithi162 ceph-mon[104668]: osdmap e1324: 8 total, 7 up, 8 in 2024-08-17T17:33:27.131 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:27.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.176 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:27.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:27.277 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:27.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:26 smithi063 ceph-mon[109843]: osdmap e1323: 8 total, 7 up, 8 in 2024-08-17T17:33:27.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:26 smithi063 ceph-mon[109843]: 10.2 starting backfill to osd.1 from (0'0,0'0] MAX to 1321'10941 2024-08-17T17:33:27.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:26 smithi063 ceph-mon[109843]: 10.12 starting backfill to osd.1 from (0'0,0'0] MAX to 1321'10907 2024-08-17T17:33:27.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:26 smithi063 ceph-mon[109843]: osdmap e1324: 8 total, 7 up, 8 in 2024-08-17T17:33:27.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:26 smithi063 ceph-mon[114756]: osdmap e1323: 8 total, 7 up, 8 in 2024-08-17T17:33:27.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:26 smithi063 ceph-mon[114756]: 10.2 starting backfill to osd.1 from (0'0,0'0] MAX to 1321'10941 2024-08-17T17:33:27.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:26 smithi063 ceph-mon[114756]: 10.12 starting backfill to osd.1 from (0'0,0'0] MAX to 1321'10907 2024-08-17T17:33:27.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:26 smithi063 ceph-mon[114756]: osdmap e1324: 8 total, 7 up, 8 in 2024-08-17T17:33:27.309 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:27.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:27.411 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:27.452 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:27.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:27.554 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:27.589 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:27.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:27.658 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:33:27.658 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:33:27.658 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:33:27.658 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:33:27.658 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.658 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:25701 9.9 9:9c6b84ec:::benchmark_data_smithi063_263014_object25700:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1251)", 2024-08-17T17:33:27.658 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:26:56.440847+0000", 2024-08-17T17:33:27.658 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 391.21323472300003, 2024-08-17T17:33:27.659 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.201000071, 2024-08-17T17:33:27.659 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.659 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.659 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.659 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.659 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.659 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 25701 2024-08-17T17:33:27.659 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.659 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.659 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.659 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.659 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.440847+0000", 2024-08-17T17:33:27.659 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.660 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.660 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.660 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.660 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.440847+0000", 2024-08-17T17:33:27.660 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.660 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.660 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.660 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.660 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.440847+0000", 2024-08-17T17:33:27.660 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:27.660 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.661 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.661 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.661 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.440900+0000", 2024-08-17T17:33:27.661 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3437999999999999e-05 2024-08-17T17:33:27.661 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.661 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.661 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.661 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.440909+0000", 2024-08-17T17:33:27.661 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.149e-06 2024-08-17T17:33:27.661 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.661 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.661 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.661 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.440916+0000", 2024-08-17T17:33:27.661 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2389999999999999e-06 2024-08-17T17:33:27.662 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.662 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.662 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.662 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.440950+0000", 2024-08-17T17:33:27.662 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4548999999999997e-05 2024-08-17T17:33:27.662 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.662 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.662 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.662 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.441095+0000", 2024-08-17T17:33:27.662 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014452900000000001 2024-08-17T17:33:27.662 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.662 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.663 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:33:27.663 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.441566+0000", 2024-08-17T17:33:27.663 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00047116999999999998 2024-08-17T17:33:27.663 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.663 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.663 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.663 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.443731+0000", 2024-08-17T17:33:27.663 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021654840000000001 2024-08-17T17:33:27.663 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.663 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.663 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.664 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.444121+0000", 2024-08-17T17:33:27.664 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00039001299999999998 2024-08-17T17:33:27.664 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.664 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.664 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.664 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.641764+0000", 2024-08-17T17:33:27.664 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.197642506 2024-08-17T17:33:27.665 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.665 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.665 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.665 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.641811+0000", 2024-08-17T17:33:27.665 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6623999999999999e-05 2024-08-17T17:33:27.665 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.665 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.665 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.665 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.641848+0000", 2024-08-17T17:33:27.665 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6980000000000002e-05 2024-08-17T17:33:27.665 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.665 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.665 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.665 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.666 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.666 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:25976 9.e 9:7222fd7c:::benchmark_data_smithi063_263014_object25975:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1251)", 2024-08-17T17:33:27.666 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:26:56.853139+0000", 2024-08-17T17:33:27.666 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 390.80094314399997, 2024-08-17T17:33:27.666 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.22521754399999999, 2024-08-17T17:33:27.666 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.666 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.666 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.666 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.666 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.666 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 25976 2024-08-17T17:33:27.666 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.666 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.667 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.667 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.667 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.853139+0000", 2024-08-17T17:33:27.667 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.667 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.667 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.667 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.667 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.853139+0000", 2024-08-17T17:33:27.667 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.667 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.667 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.667 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.667 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.853138+0000", 2024-08-17T17:33:27.667 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:27.668 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.668 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.668 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.668 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.853173+0000", 2024-08-17T17:33:27.668 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4359000000000001e-05 2024-08-17T17:33:27.668 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.668 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.668 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.668 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.853182+0000", 2024-08-17T17:33:27.668 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3659999999999998e-06 2024-08-17T17:33:27.668 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.668 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.668 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.668 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.853188+0000", 2024-08-17T17:33:27.669 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7230000000000001e-06 2024-08-17T17:33:27.669 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.669 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.669 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.669 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.854715+0000", 2024-08-17T17:33:27.669 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015268809999999999 2024-08-17T17:33:27.669 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.669 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.669 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.669 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.854833+0000", 2024-08-17T17:33:27.669 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011790000000000001 2024-08-17T17:33:27.669 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.669 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.669 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:33:27.670 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.854969+0000", 2024-08-17T17:33:27.670 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013638699999999999 2024-08-17T17:33:27.670 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.670 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.670 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.670 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.856478+0000", 2024-08-17T17:33:27.670 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015094399999999999 2024-08-17T17:33:27.670 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.670 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.670 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.670 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.858966+0000", 2024-08-17T17:33:27.670 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0024873740000000001 2024-08-17T17:33:27.670 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.671 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.671 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.671 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.078254+0000", 2024-08-17T17:33:27.671 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.21928797999999999 2024-08-17T17:33:27.671 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.671 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.671 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.671 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.078310+0000", 2024-08-17T17:33:27.671 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5874000000000001e-05 2024-08-17T17:33:27.671 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.671 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.671 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.671 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.078357+0000", 2024-08-17T17:33:27.671 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6916999999999998e-05 2024-08-17T17:33:27.672 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.672 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.672 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.672 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.672 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.672 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:26007 9.e 9:77785aa7:::benchmark_data_smithi063_263014_object26006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1251)", 2024-08-17T17:33:27.672 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:26:56.863833+0000", 2024-08-17T17:33:27.672 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 390.79024897199997, 2024-08-17T17:33:27.672 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.21645382399999999, 2024-08-17T17:33:27.672 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.672 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.672 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.673 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.673 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.673 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 26007 2024-08-17T17:33:27.673 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.673 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.673 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.673 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.673 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.863833+0000", 2024-08-17T17:33:27.674 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.674 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.674 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.674 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.674 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.863833+0000", 2024-08-17T17:33:27.674 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.674 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.674 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.674 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.674 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.863832+0000", 2024-08-17T17:33:27.674 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:33:27.674 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.675 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.675 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.675 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.863978+0000", 2024-08-17T17:33:27.675 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014578700000000001 2024-08-17T17:33:27.675 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.675 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.675 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.675 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.863993+0000", 2024-08-17T17:33:27.675 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4704e-05 2024-08-17T17:33:27.675 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.675 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.675 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.676 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.864000+0000", 2024-08-17T17:33:27.676 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.8290000000000008e-06 2024-08-17T17:33:27.676 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.676 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.676 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.676 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.864051+0000", 2024-08-17T17:33:27.676 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0355000000000002e-05 2024-08-17T17:33:27.676 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.676 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.676 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.676 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.864180+0000", 2024-08-17T17:33:27.676 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012905499999999999 2024-08-17T17:33:27.676 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.676 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.677 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:33:27.677 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.864313+0000", 2024-08-17T17:33:27.677 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013292100000000001 2024-08-17T17:33:27.677 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.677 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.677 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.677 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.865373+0000", 2024-08-17T17:33:27.677 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0010607139999999999 2024-08-17T17:33:27.677 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.677 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.677 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.677 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.865931+0000", 2024-08-17T17:33:27.677 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00055763700000000004 2024-08-17T17:33:27.677 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.678 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.678 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.678 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.080144+0000", 2024-08-17T17:33:27.678 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.21421327100000001 2024-08-17T17:33:27.678 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.678 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.678 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.678 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.080192+0000", 2024-08-17T17:33:27.678 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7651000000000002e-05 2024-08-17T17:33:27.678 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.678 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.679 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.679 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.080287+0000", 2024-08-17T17:33:27.679 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.5084e-05 2024-08-17T17:33:27.679 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.679 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.679 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.679 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.679 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.679 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:26020 9.e 9:76c9ba14:::benchmark_data_smithi063_263014_object26019:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1251)", 2024-08-17T17:33:27.679 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:26:56.871889+0000", 2024-08-17T17:33:27.679 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 390.78219312499999, 2024-08-17T17:33:27.679 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.20855354500000001, 2024-08-17T17:33:27.679 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.679 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.680 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.680 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.680 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.680 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 26020 2024-08-17T17:33:27.680 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.680 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.680 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.680 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.680 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.871889+0000", 2024-08-17T17:33:27.680 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.680 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.680 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.680 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.680 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.871889+0000", 2024-08-17T17:33:27.680 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.681 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.681 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.681 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.681 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.871888+0000", 2024-08-17T17:33:27.681 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:27.681 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.681 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.681 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.681 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.871997+0000", 2024-08-17T17:33:27.681 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000108533 2024-08-17T17:33:27.681 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.681 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.681 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.681 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.872008+0000", 2024-08-17T17:33:27.681 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0784999999999999e-05 2024-08-17T17:33:27.682 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.682 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.682 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.682 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.872014+0000", 2024-08-17T17:33:27.682 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0109999999999997e-06 2024-08-17T17:33:27.682 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.682 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.682 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.682 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.872049+0000", 2024-08-17T17:33:27.682 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4968000000000001e-05 2024-08-17T17:33:27.682 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.682 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.683 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.683 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.872137+0000", 2024-08-17T17:33:27.683 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8789999999999995e-05 2024-08-17T17:33:27.683 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.683 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.683 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:33:27.683 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.872215+0000", 2024-08-17T17:33:27.683 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7228e-05 2024-08-17T17:33:27.683 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.683 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.684 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.684 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.873277+0000", 2024-08-17T17:33:27.684 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001062068 2024-08-17T17:33:27.684 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.684 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.684 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.684 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:56.873527+0000", 2024-08-17T17:33:27.684 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00025069900000000001 2024-08-17T17:33:27.684 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.684 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.684 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.685 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.080360+0000", 2024-08-17T17:33:27.685 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.20683270500000001 2024-08-17T17:33:27.685 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.685 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.685 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.685 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.080405+0000", 2024-08-17T17:33:27.685 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4569e-05 2024-08-17T17:33:27.685 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.685 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.685 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.685 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:26:57.080443+0000", 2024-08-17T17:33:27.685 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7871999999999999e-05 2024-08-17T17:33:27.685 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.686 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.686 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.686 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.686 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.686 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:102917 9.e 9:7370cc17:::benchmark_data_smithi063_263014_object102916:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-08-17T17:33:27.686 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:56.005049+0000", 2024-08-17T17:33:27.686 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 331.64903309200002, 2024-08-17T17:33:27.686 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33703233399999999, 2024-08-17T17:33:27.686 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.686 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.686 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.686 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.687 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.687 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 102917 2024-08-17T17:33:27.687 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.687 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.687 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.687 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.687 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.005049+0000", 2024-08-17T17:33:27.687 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.687 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.687 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.687 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.687 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.005049+0000", 2024-08-17T17:33:27.688 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.688 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.688 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.688 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.688 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.005048+0000", 2024-08-17T17:33:27.688 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:27.688 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.688 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.688 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.688 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.005108+0000", 2024-08-17T17:33:27.688 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0532000000000003e-05 2024-08-17T17:33:27.688 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.689 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.689 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.689 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.005122+0000", 2024-08-17T17:33:27.689 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3613e-05 2024-08-17T17:33:27.689 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.689 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.689 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.689 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.005129+0000", 2024-08-17T17:33:27.689 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8079999999999999e-06 2024-08-17T17:33:27.689 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.689 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.689 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.690 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.005176+0000", 2024-08-17T17:33:27.690 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6949000000000001e-05 2024-08-17T17:33:27.690 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.690 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.690 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.690 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.005281+0000", 2024-08-17T17:33:27.690 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010506 2024-08-17T17:33:27.690 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.690 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.690 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:33:27.690 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.005376+0000", 2024-08-17T17:33:27.690 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.5461e-05 2024-08-17T17:33:27.690 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.691 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.691 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.691 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.007998+0000", 2024-08-17T17:33:27.691 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026220789999999998 2024-08-17T17:33:27.691 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.691 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.691 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.691 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.008651+0000", 2024-08-17T17:33:27.691 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00065313100000000005 2024-08-17T17:33:27.691 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.691 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.691 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.691 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.341993+0000", 2024-08-17T17:33:27.691 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.33334163500000003 2024-08-17T17:33:27.692 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.692 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.692 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.692 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.342038+0000", 2024-08-17T17:33:27.692 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4787000000000003e-05 2024-08-17T17:33:27.692 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.692 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.692 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.692 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.342081+0000", 2024-08-17T17:33:27.692 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3707e-05 2024-08-17T17:33:27.692 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.692 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.692 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.692 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.693 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.693 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:102936 9.3 9:caa1e09a:::benchmark_data_smithi063_263014_object102935:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-08-17T17:33:27.693 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:56.030326+0000", 2024-08-17T17:33:27.693 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 331.62375655300002, 2024-08-17T17:33:27.693 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.31609246200000002, 2024-08-17T17:33:27.693 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.693 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.693 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.693 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.693 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.693 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 102936 2024-08-17T17:33:27.693 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.693 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.693 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.694 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.694 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.030326+0000", 2024-08-17T17:33:27.694 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.694 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.694 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.694 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.694 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.030326+0000", 2024-08-17T17:33:27.694 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.694 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.694 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.694 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.694 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.030324+0000", 2024-08-17T17:33:27.694 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:27.695 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.695 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.695 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.695 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.030387+0000", 2024-08-17T17:33:27.695 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2792000000000005e-05 2024-08-17T17:33:27.695 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.695 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.695 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.695 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.030399+0000", 2024-08-17T17:33:27.695 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2317000000000001e-05 2024-08-17T17:33:27.695 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.695 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.695 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.696 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.030406+0000", 2024-08-17T17:33:27.696 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5200000000000003e-06 2024-08-17T17:33:27.696 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.696 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.696 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.696 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.030444+0000", 2024-08-17T17:33:27.696 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8141e-05 2024-08-17T17:33:27.696 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.696 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.696 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.696 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.030572+0000", 2024-08-17T17:33:27.696 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012744899999999999 2024-08-17T17:33:27.696 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.696 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.697 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:33:27.697 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.030696+0000", 2024-08-17T17:33:27.697 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000124895 2024-08-17T17:33:27.697 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.697 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.697 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.697 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.033490+0000", 2024-08-17T17:33:27.697 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0027935270000000001 2024-08-17T17:33:27.697 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.697 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.697 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.697 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.033693+0000", 2024-08-17T17:33:27.697 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00020267700000000001 2024-08-17T17:33:27.698 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.698 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.698 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.698 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.346330+0000", 2024-08-17T17:33:27.698 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.312637469 2024-08-17T17:33:27.698 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.698 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.698 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.698 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.346381+0000", 2024-08-17T17:33:27.698 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1238e-05 2024-08-17T17:33:27.698 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.698 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.698 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.699 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.346418+0000", 2024-08-17T17:33:27.699 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6711000000000001e-05 2024-08-17T17:33:27.699 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.699 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.699 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.699 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.699 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.699 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:102939 9.e 9:7325d85a:::benchmark_data_smithi063_263014_object102938:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-08-17T17:33:27.699 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:56.033523+0000", 2024-08-17T17:33:27.699 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 331.620559442, 2024-08-17T17:33:27.699 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.32321910100000001, 2024-08-17T17:33:27.699 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.699 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.700 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.700 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.700 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.700 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 102939 2024-08-17T17:33:27.700 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.700 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.700 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.700 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.700 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.033523+0000", 2024-08-17T17:33:27.700 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.700 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.700 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.700 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.700 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.033523+0000", 2024-08-17T17:33:27.701 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.701 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.701 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.701 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.701 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.033521+0000", 2024-08-17T17:33:27.701 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:27.701 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.701 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.701 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.701 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.033577+0000", 2024-08-17T17:33:27.701 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5875999999999998e-05 2024-08-17T17:33:27.701 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.701 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.701 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.702 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.033588+0000", 2024-08-17T17:33:27.702 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0606e-05 2024-08-17T17:33:27.702 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.702 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.702 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.702 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.033597+0000", 2024-08-17T17:33:27.702 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.6979999999999993e-06 2024-08-17T17:33:27.702 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.702 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.702 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.702 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.033644+0000", 2024-08-17T17:33:27.702 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6631000000000001e-05 2024-08-17T17:33:27.702 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.702 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.703 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.703 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.033802+0000", 2024-08-17T17:33:27.703 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015802599999999999 2024-08-17T17:33:27.703 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.703 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.703 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:33:27.703 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.034113+0000", 2024-08-17T17:33:27.703 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00031094800000000002 2024-08-17T17:33:27.703 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.703 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.703 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.703 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.037204+0000", 2024-08-17T17:33:27.704 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0030911879999999999 2024-08-17T17:33:27.704 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.704 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.704 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.704 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.037583+0000", 2024-08-17T17:33:27.704 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00037890899999999997 2024-08-17T17:33:27.704 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.704 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.704 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.704 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.356662+0000", 2024-08-17T17:33:27.704 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.31907885699999999 2024-08-17T17:33:27.704 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.704 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.704 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.705 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.356708+0000", 2024-08-17T17:33:27.705 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5945000000000001e-05 2024-08-17T17:33:27.705 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.705 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.705 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.705 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.356742+0000", 2024-08-17T17:33:27.705 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3912999999999999e-05 2024-08-17T17:33:27.705 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.705 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.705 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.705 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.705 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.705 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:102941 9.d 9:ba238f05:::benchmark_data_smithi063_263014_object102940:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-08-17T17:33:27.706 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:27:56.034420+0000", 2024-08-17T17:33:27.706 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 331.61966181100001, 2024-08-17T17:33:27.706 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.31113930499999998, 2024-08-17T17:33:27.706 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.706 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.706 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.706 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.706 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.706 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 102941 2024-08-17T17:33:27.706 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.706 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.706 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.707 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.707 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.034420+0000", 2024-08-17T17:33:27.707 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.707 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.707 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.707 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.707 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.034420+0000", 2024-08-17T17:33:27.707 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.707 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.707 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.707 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.707 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.034419+0000", 2024-08-17T17:33:27.708 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:33:27.708 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.708 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.708 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.708 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.034481+0000", 2024-08-17T17:33:27.708 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1327000000000004e-05 2024-08-17T17:33:27.708 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.708 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.708 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.708 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.034493+0000", 2024-08-17T17:33:27.708 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2282e-05 2024-08-17T17:33:27.708 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.708 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.708 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.034500+0000", 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.613e-06 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.034544+0000", 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3915999999999998e-05 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.034678+0000", 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013428299999999999 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,5", 2024-08-17T17:33:27.709 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.034815+0000", 2024-08-17T17:33:27.710 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000137052 2024-08-17T17:33:27.710 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.710 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.710 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.710 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.037348+0000", 2024-08-17T17:33:27.710 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0025326419999999999 2024-08-17T17:33:27.710 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.710 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.710 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.710 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.038184+0000", 2024-08-17T17:33:27.710 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00083690200000000005 2024-08-17T17:33:27.710 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.710 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.710 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.711 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.345469+0000", 2024-08-17T17:33:27.711 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.30728421299999997 2024-08-17T17:33:27.711 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.711 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.711 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.711 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.345522+0000", 2024-08-17T17:33:27.711 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3196999999999999e-05 2024-08-17T17:33:27.711 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.711 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.711 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.711 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:27:56.345560+0000", 2024-08-17T17:33:27.711 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7855000000000003e-05 2024-08-17T17:33:27.711 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.711 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.711 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.712 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.712 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.712 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:172910 9.e 9:7ba1b61c:::benchmark_data_smithi063_263014_object48572:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T17:33:27.712 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:29:01.413706+0000", 2024-08-17T17:33:27.712 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 266.240376508, 2024-08-17T17:33:27.712 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.18999054000000001, 2024-08-17T17:33:27.712 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.712 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.712 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.712 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.712 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.712 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 172910 2024-08-17T17:33:27.712 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.713 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.713 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.713 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.713 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.413706+0000", 2024-08-17T17:33:27.713 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.713 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.713 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.713 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.713 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.413706+0000", 2024-08-17T17:33:27.713 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.713 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.713 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.713 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.714 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.413705+0000", 2024-08-17T17:33:27.714 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:33:27.714 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.714 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.714 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.714 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.413709+0000", 2024-08-17T17:33:27.714 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4299999999999999e-06 2024-08-17T17:33:27.714 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.714 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.714 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.714 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.413718+0000", 2024-08-17T17:33:27.714 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.5869999999999993e-06 2024-08-17T17:33:27.715 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.715 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.715 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.715 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.413723+0000", 2024-08-17T17:33:27.715 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8319999999999996e-06 2024-08-17T17:33:27.715 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.715 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.715 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.715 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.413763+0000", 2024-08-17T17:33:27.715 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0666e-05 2024-08-17T17:33:27.715 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.715 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.715 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.715 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.413873+0000", 2024-08-17T17:33:27.716 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000110133 2024-08-17T17:33:27.716 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.716 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.716 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:33:27.716 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.413947+0000", 2024-08-17T17:33:27.716 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3719000000000003e-05 2024-08-17T17:33:27.716 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.716 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.716 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:27.717 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.717 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.416873+0000", 2024-08-17T17:33:27.717 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0029254810000000002 2024-08-17T17:33:27.717 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.718 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.718 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.718 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.420018+0000", 2024-08-17T17:33:27.718 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003145133 2024-08-17T17:33:27.718 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.718 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.718 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.718 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.603603+0000", 2024-08-17T17:33:27.718 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.183584835 2024-08-17T17:33:27.718 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.718 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.718 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.719 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.603654+0000", 2024-08-17T17:33:27.719 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1078999999999997e-05 2024-08-17T17:33:27.719 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.719 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.719 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.719 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.603696+0000", 2024-08-17T17:33:27.719 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2521999999999997e-05 2024-08-17T17:33:27.719 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.719 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.719 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.719 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.719 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.719 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:172920 9.e 9:74ff90e8:::benchmark_data_smithi063_263014_object48582:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T17:33:27.720 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:29:01.421723+0000", 2024-08-17T17:33:27.720 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 266.23235908999999, 2024-08-17T17:33:27.720 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.18213111500000001, 2024-08-17T17:33:27.720 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.720 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.720 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.720 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.720 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.720 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 172920 2024-08-17T17:33:27.720 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.720 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.720 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.720 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.721 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.421723+0000", 2024-08-17T17:33:27.721 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.721 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.721 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.721 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.721 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.421723+0000", 2024-08-17T17:33:27.721 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.721 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.721 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.721 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.721 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.421722+0000", 2024-08-17T17:33:27.721 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:33:27.721 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.722 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.722 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.722 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.421728+0000", 2024-08-17T17:33:27.722 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.614e-06 2024-08-17T17:33:27.722 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.722 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.722 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.722 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.421740+0000", 2024-08-17T17:33:27.722 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2456000000000001e-05 2024-08-17T17:33:27.722 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.722 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.722 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.723 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.421747+0000", 2024-08-17T17:33:27.723 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8639999999999998e-06 2024-08-17T17:33:27.723 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.723 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.723 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.723 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.421779+0000", 2024-08-17T17:33:27.723 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2240999999999998e-05 2024-08-17T17:33:27.723 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.723 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.723 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.723 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.421867+0000", 2024-08-17T17:33:27.723 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7681000000000003e-05 2024-08-17T17:33:27.723 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.724 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.724 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:33:27.724 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.421940+0000", 2024-08-17T17:33:27.724 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2768000000000005e-05 2024-08-17T17:33:27.724 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.724 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.724 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.724 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.427972+0000", 2024-08-17T17:33:27.724 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0060319800000000002 2024-08-17T17:33:27.724 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.724 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.724 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.724 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.428005+0000", 2024-08-17T17:33:27.725 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3736999999999999e-05 2024-08-17T17:33:27.725 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.725 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.725 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.725 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.603742+0000", 2024-08-17T17:33:27.725 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.17573704000000001 2024-08-17T17:33:27.725 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.725 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.725 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.725 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.603818+0000", 2024-08-17T17:33:27.725 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.5141000000000004e-05 2024-08-17T17:33:27.725 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.725 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.726 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.726 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.603854+0000", 2024-08-17T17:33:27.726 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6724000000000003e-05 2024-08-17T17:33:27.726 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.726 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.726 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.726 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.726 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.726 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:172923 9.e 9:78fd8532:::benchmark_data_smithi063_263014_object48585:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T17:33:27.726 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:29:01.422240+0000", 2024-08-17T17:33:27.726 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 266.23184237300001, 2024-08-17T17:33:27.726 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.18182452700000001, 2024-08-17T17:33:27.726 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.727 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.727 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.727 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.727 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.727 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 172923 2024-08-17T17:33:27.727 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.727 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.727 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.727 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.727 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.422240+0000", 2024-08-17T17:33:27.727 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.727 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.727 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.728 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.728 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.422240+0000", 2024-08-17T17:33:27.728 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.728 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.728 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.728 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.728 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.422239+0000", 2024-08-17T17:33:27.728 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:33:27.728 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.728 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.728 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.728 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.422243+0000", 2024-08-17T17:33:27.728 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2699999999999998e-06 2024-08-17T17:33:27.729 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.729 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.729 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.729 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.422253+0000", 2024-08-17T17:33:27.729 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7659999999999993e-06 2024-08-17T17:33:27.729 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.729 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.729 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.729 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.422259+0000", 2024-08-17T17:33:27.729 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1519999999999997e-06 2024-08-17T17:33:27.729 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.729 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.729 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.730 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.424926+0000", 2024-08-17T17:33:27.730 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026670550000000002 2024-08-17T17:33:27.730 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.730 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.730 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.730 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.425034+0000", 2024-08-17T17:33:27.730 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000107526 2024-08-17T17:33:27.730 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.730 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.730 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:33:27.730 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.425145+0000", 2024-08-17T17:33:27.730 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000110778 2024-08-17T17:33:27.731 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.731 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.731 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.731 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.428097+0000", 2024-08-17T17:33:27.731 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0029521669999999999 2024-08-17T17:33:27.731 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.731 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.731 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.731 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.430088+0000", 2024-08-17T17:33:27.731 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0019908999999999999 2024-08-17T17:33:27.731 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.731 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.731 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.732 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.603906+0000", 2024-08-17T17:33:27.732 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.17381835300000001 2024-08-17T17:33:27.732 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.732 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.732 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.732 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.604029+0000", 2024-08-17T17:33:27.732 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012261600000000001 2024-08-17T17:33:27.732 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.732 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.732 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.732 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.604064+0000", 2024-08-17T17:33:27.732 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5732999999999998e-05 2024-08-17T17:33:27.733 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.733 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.733 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.733 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.733 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.733 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:172933 9.e 9:76514b18:::benchmark_data_smithi063_263014_object48595:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T17:33:27.733 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:29:01.433873+0000", 2024-08-17T17:33:27.733 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 266.22020882800001, 2024-08-17T17:33:27.733 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.170326586, 2024-08-17T17:33:27.733 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.733 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.733 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.733 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.734 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.734 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 172933 2024-08-17T17:33:27.734 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.734 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.734 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.734 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.734 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.433873+0000", 2024-08-17T17:33:27.734 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.734 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.734 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.734 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.734 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.433873+0000", 2024-08-17T17:33:27.735 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.735 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.735 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.735 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.735 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.433872+0000", 2024-08-17T17:33:27.735 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:33:27.735 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.735 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.735 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.735 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.433880+0000", 2024-08-17T17:33:27.735 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7829999999999999e-06 2024-08-17T17:33:27.735 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.735 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.735 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.736 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.433892+0000", 2024-08-17T17:33:27.736 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1539000000000001e-05 2024-08-17T17:33:27.736 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.736 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.736 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.736 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.433898+0000", 2024-08-17T17:33:27.736 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.365e-06 2024-08-17T17:33:27.736 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.736 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.736 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.736 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.433948+0000", 2024-08-17T17:33:27.736 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9622e-05 2024-08-17T17:33:27.736 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.737 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.737 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.737 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.434054+0000", 2024-08-17T17:33:27.737 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000106587 2024-08-17T17:33:27.737 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.737 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.737 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:33:27.737 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.434164+0000", 2024-08-17T17:33:27.737 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000109669 2024-08-17T17:33:27.737 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.737 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.737 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.737 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.437208+0000", 2024-08-17T17:33:27.738 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0030440010000000002 2024-08-17T17:33:27.738 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.738 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.738 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.738 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.438128+0000", 2024-08-17T17:33:27.738 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00092013299999999995 2024-08-17T17:33:27.738 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.738 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.738 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.738 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.604099+0000", 2024-08-17T17:33:27.738 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.165970437 2024-08-17T17:33:27.738 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.738 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.739 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.739 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.604165+0000", 2024-08-17T17:33:27.739 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6840000000000004e-05 2024-08-17T17:33:27.739 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.739 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.739 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.739 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.604200+0000", 2024-08-17T17:33:27.739 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4578e-05 2024-08-17T17:33:27.739 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.739 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.739 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.739 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.739 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.739 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:172941 9.e 9:71b95911:::benchmark_data_smithi063_263014_object48603:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1267)", 2024-08-17T17:33:27.740 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:29:01.440467+0000", 2024-08-17T17:33:27.740 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 266.21361514, 2024-08-17T17:33:27.740 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.16537352499999999, 2024-08-17T17:33:27.740 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.740 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.740 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.740 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.740 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.740 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 172941 2024-08-17T17:33:27.740 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.740 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.740 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.740 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.740 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.440467+0000", 2024-08-17T17:33:27.741 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.741 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.741 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.741 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.741 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.440467+0000", 2024-08-17T17:33:27.741 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.741 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.741 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.741 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.741 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.440466+0000", 2024-08-17T17:33:27.741 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:33:27.741 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.741 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.742 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.742 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.440471+0000", 2024-08-17T17:33:27.742 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2989999999999996e-06 2024-08-17T17:33:27.742 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.742 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.742 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.742 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.440486+0000", 2024-08-17T17:33:27.742 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4778e-05 2024-08-17T17:33:27.742 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.742 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.742 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.742 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.440493+0000", 2024-08-17T17:33:27.742 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7390000000000002e-06 2024-08-17T17:33:27.743 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.743 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.743 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.743 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.440536+0000", 2024-08-17T17:33:27.743 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3201999999999997e-05 2024-08-17T17:33:27.743 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.743 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.743 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.743 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.440649+0000", 2024-08-17T17:33:27.743 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000112674 2024-08-17T17:33:27.743 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.743 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.744 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:33:27.744 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.440765+0000", 2024-08-17T17:33:27.744 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000116431 2024-08-17T17:33:27.744 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.744 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.744 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.744 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.443778+0000", 2024-08-17T17:33:27.744 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003013104 2024-08-17T17:33:27.744 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.744 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.744 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.744 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.444169+0000", 2024-08-17T17:33:27.744 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00039093100000000002 2024-08-17T17:33:27.745 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.745 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.745 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.745 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.605712+0000", 2024-08-17T17:33:27.745 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.16154253900000001 2024-08-17T17:33:27.745 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.745 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.745 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.745 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.605801+0000", 2024-08-17T17:33:27.745 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9761999999999999e-05 2024-08-17T17:33:27.745 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.745 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.745 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.746 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:01.605841+0000", 2024-08-17T17:33:27.746 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9181999999999999e-05 2024-08-17T17:33:27.746 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.746 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.746 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.746 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.746 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.746 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:206378 9.9 9:96b77632:::benchmark_data_smithi063_263014_object82040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1272)", 2024-08-17T17:33:27.746 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:29:32.322352+0000", 2024-08-17T17:33:27.746 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 235.33172981600001, 2024-08-17T17:33:27.746 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.169222867, 2024-08-17T17:33:27.746 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.746 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.747 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.747 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.747 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.747 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 206378 2024-08-17T17:33:27.747 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.747 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.747 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.747 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.747 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.322352+0000", 2024-08-17T17:33:27.747 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.747 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.747 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.748 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.748 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.322352+0000", 2024-08-17T17:33:27.748 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.748 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.748 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.748 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.748 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.322352+0000", 2024-08-17T17:33:27.748 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:27.748 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.748 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.748 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.748 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.322355+0000", 2024-08-17T17:33:27.748 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.411e-06 2024-08-17T17:33:27.749 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.749 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.749 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.749 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.322365+0000", 2024-08-17T17:33:27.749 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7210000000000006e-06 2024-08-17T17:33:27.749 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.749 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.749 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.749 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.322370+0000", 2024-08-17T17:33:27.749 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0089999999999998e-06 2024-08-17T17:33:27.749 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.749 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.749 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.750 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.329268+0000", 2024-08-17T17:33:27.750 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.006898019 2024-08-17T17:33:27.750 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.750 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.750 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.750 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.329374+0000", 2024-08-17T17:33:27.750 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000105933 2024-08-17T17:33:27.750 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.750 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.750 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:33:27.750 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.329478+0000", 2024-08-17T17:33:27.750 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000103912 2024-08-17T17:33:27.751 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.751 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.751 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.751 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.337101+0000", 2024-08-17T17:33:27.751 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0076235649999999997 2024-08-17T17:33:27.751 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.751 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.751 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.751 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.342180+0000", 2024-08-17T17:33:27.751 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0050788819999999998 2024-08-17T17:33:27.751 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.751 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.751 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.751 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.491479+0000", 2024-08-17T17:33:27.752 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14929852599999999 2024-08-17T17:33:27.752 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.752 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.752 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.752 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.491533+0000", 2024-08-17T17:33:27.752 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4070999999999998e-05 2024-08-17T17:33:27.752 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.752 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.752 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.752 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.491575+0000", 2024-08-17T17:33:27.752 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2497999999999999e-05 2024-08-17T17:33:27.752 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.753 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.753 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.753 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.753 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.753 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:206387 9.9 9:9c6b4511:::benchmark_data_smithi063_263014_object82049:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1272)", 2024-08-17T17:33:27.753 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:29:32.333260+0000", 2024-08-17T17:33:27.753 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 235.320822529, 2024-08-17T17:33:27.753 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.17277848900000001, 2024-08-17T17:33:27.753 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.753 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.753 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.753 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.753 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.753 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 206387 2024-08-17T17:33:27.754 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.754 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.754 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.754 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.754 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.333260+0000", 2024-08-17T17:33:27.754 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.754 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.754 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.754 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.754 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.333260+0000", 2024-08-17T17:33:27.754 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.754 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.755 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.755 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.758 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.333259+0000", 2024-08-17T17:33:27.758 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:33:27.758 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.758 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.758 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.758 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.333264+0000", 2024-08-17T17:33:27.758 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7080000000000002e-06 2024-08-17T17:33:27.758 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.758 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.758 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.759 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.333280+0000", 2024-08-17T17:33:27.759 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5908000000000001e-05 2024-08-17T17:33:27.759 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.759 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.759 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.759 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.333287+0000", 2024-08-17T17:33:27.759 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2150000000000004e-06 2024-08-17T17:33:27.759 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.759 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.759 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.759 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.333917+0000", 2024-08-17T17:33:27.759 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00062969600000000001 2024-08-17T17:33:27.760 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.760 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.760 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.760 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.334019+0000", 2024-08-17T17:33:27.760 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000101679 2024-08-17T17:33:27.760 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.760 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.760 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:33:27.760 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.334125+0000", 2024-08-17T17:33:27.760 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000106185 2024-08-17T17:33:27.760 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.760 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.760 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.761 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.341199+0000", 2024-08-17T17:33:27.761 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00707451 2024-08-17T17:33:27.761 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.761 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.761 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.761 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.342215+0000", 2024-08-17T17:33:27.761 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001015382 2024-08-17T17:33:27.761 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.761 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.761 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.761 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.505912+0000", 2024-08-17T17:33:27.761 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.16369752600000001 2024-08-17T17:33:27.761 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.762 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.762 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.762 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.505997+0000", 2024-08-17T17:33:27.762 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4611999999999996e-05 2024-08-17T17:33:27.762 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.762 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.762 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.762 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.506038+0000", 2024-08-17T17:33:27.762 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1161000000000001e-05 2024-08-17T17:33:27.762 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.762 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.762 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.763 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.763 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.763 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:206389 9.9 9:933f16c7:::benchmark_data_smithi063_263014_object82051:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1272)", 2024-08-17T17:33:27.763 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:29:32.336234+0000", 2024-08-17T17:33:27.763 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 235.31784788600001, 2024-08-17T17:33:27.763 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.175550437, 2024-08-17T17:33:27.763 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.763 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.763 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.763 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.763 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.763 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 206389 2024-08-17T17:33:27.763 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.764 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.764 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.764 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.764 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.336234+0000", 2024-08-17T17:33:27.764 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.764 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.764 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.764 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.765 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.336234+0000", 2024-08-17T17:33:27.765 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.765 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.765 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.765 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.765 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.336233+0000", 2024-08-17T17:33:27.765 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:27.765 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.765 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.766 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.766 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.336239+0000", 2024-08-17T17:33:27.766 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6039999999999998e-06 2024-08-17T17:33:27.766 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.766 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.767 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.767 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.336253+0000", 2024-08-17T17:33:27.767 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3213e-05 2024-08-17T17:33:27.767 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.767 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.767 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.767 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.336260+0000", 2024-08-17T17:33:27.767 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0829999999999998e-06 2024-08-17T17:33:27.767 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.767 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.767 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.767 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.337133+0000", 2024-08-17T17:33:27.768 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00087345800000000003 2024-08-17T17:33:27.768 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.768 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.768 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.768 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.337222+0000", 2024-08-17T17:33:27.768 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8514e-05 2024-08-17T17:33:27.768 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.768 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.768 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:33:27.768 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.337300+0000", 2024-08-17T17:33:27.768 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.8207000000000005e-05 2024-08-17T17:33:27.768 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.768 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.769 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.769 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.341206+0000", 2024-08-17T17:33:27.769 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0039061270000000001 2024-08-17T17:33:27.769 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.769 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.769 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.769 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.342239+0000", 2024-08-17T17:33:27.769 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0010332869999999999 2024-08-17T17:33:27.769 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.769 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.769 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.769 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.511675+0000", 2024-08-17T17:33:27.769 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.16943546900000001 2024-08-17T17:33:27.770 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.770 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.770 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.770 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.511749+0000", 2024-08-17T17:33:27.770 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4172999999999996e-05 2024-08-17T17:33:27.770 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.770 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.770 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.770 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.511785+0000", 2024-08-17T17:33:27.770 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5849999999999997e-05 2024-08-17T17:33:27.770 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.770 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.771 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.771 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.771 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.771 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:206407 9.9 9:940be49a:::benchmark_data_smithi063_263014_object82069:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1272)", 2024-08-17T17:33:27.771 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:29:32.353191+0000", 2024-08-17T17:33:27.771 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 235.30089094499999, 2024-08-17T17:33:27.771 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.158748686, 2024-08-17T17:33:27.771 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.771 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.771 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.771 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.771 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.771 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 206407 2024-08-17T17:33:27.772 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.772 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.772 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.772 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.772 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.353191+0000", 2024-08-17T17:33:27.772 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.772 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.772 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.772 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.772 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.353191+0000", 2024-08-17T17:33:27.772 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.772 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.772 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.773 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.773 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.353190+0000", 2024-08-17T17:33:27.773 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:33:27.773 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.773 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.773 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.773 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.353196+0000", 2024-08-17T17:33:27.773 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2739999999999998e-06 2024-08-17T17:33:27.773 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.773 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.773 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.773 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.353212+0000", 2024-08-17T17:33:27.773 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6413e-05 2024-08-17T17:33:27.773 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.774 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.774 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.774 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.353219+0000", 2024-08-17T17:33:27.774 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1620000000000002e-06 2024-08-17T17:33:27.774 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.774 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.774 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.774 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.353267+0000", 2024-08-17T17:33:27.774 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7877000000000003e-05 2024-08-17T17:33:27.774 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.774 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.774 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.774 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.353371+0000", 2024-08-17T17:33:27.775 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000103546 2024-08-17T17:33:27.775 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.775 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.775 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:33:27.775 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.353491+0000", 2024-08-17T17:33:27.775 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000120109 2024-08-17T17:33:27.775 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.775 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.775 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.775 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.362052+0000", 2024-08-17T17:33:27.775 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0085613830000000005 2024-08-17T17:33:27.775 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.775 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.775 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.776 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.362059+0000", 2024-08-17T17:33:27.776 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5910000000000001e-06 2024-08-17T17:33:27.776 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.776 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.776 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.776 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.511835+0000", 2024-08-17T17:33:27.776 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14977627099999999 2024-08-17T17:33:27.776 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.776 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.776 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.776 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.511907+0000", 2024-08-17T17:33:27.776 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2187000000000005e-05 2024-08-17T17:33:27.776 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.777 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.777 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.777 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.511940+0000", 2024-08-17T17:33:27.777 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2882999999999997e-05 2024-08-17T17:33:27.777 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.777 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.777 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.777 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.777 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.777 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:206408 9.9 9:99ffcc12:::benchmark_data_smithi063_263014_object82070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1272)", 2024-08-17T17:33:27.777 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:29:32.356601+0000", 2024-08-17T17:33:27.777 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 235.29748169000001, 2024-08-17T17:33:27.777 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.155607786, 2024-08-17T17:33:27.778 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.778 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.778 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.778 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.778 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.778 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 206408 2024-08-17T17:33:27.778 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.778 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.778 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.778 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.778 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.356601+0000", 2024-08-17T17:33:27.778 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.779 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.779 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.779 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.779 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.356601+0000", 2024-08-17T17:33:27.779 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.779 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.779 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.779 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.779 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.356599+0000", 2024-08-17T17:33:27.779 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:27.779 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.779 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.779 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.779 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.356608+0000", 2024-08-17T17:33:27.780 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.0909999999999999e-06 2024-08-17T17:33:27.780 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.780 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.780 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.780 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.356620+0000", 2024-08-17T17:33:27.780 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.152e-05 2024-08-17T17:33:27.780 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.780 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.780 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.780 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.356626+0000", 2024-08-17T17:33:27.780 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2530000000000001e-06 2024-08-17T17:33:27.780 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.780 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.781 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.781 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.357689+0000", 2024-08-17T17:33:27.781 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0010629560000000001 2024-08-17T17:33:27.781 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.781 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.781 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.781 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.357797+0000", 2024-08-17T17:33:27.781 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000108084 2024-08-17T17:33:27.781 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.781 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.781 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,5", 2024-08-17T17:33:27.781 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.358168+0000", 2024-08-17T17:33:27.781 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00037155700000000001 2024-08-17T17:33:27.782 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.782 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.782 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.782 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.362135+0000", 2024-08-17T17:33:27.782 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0039669759999999997 2024-08-17T17:33:27.782 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.782 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.782 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.782 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.362282+0000", 2024-08-17T17:33:27.782 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014667400000000001 2024-08-17T17:33:27.782 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.782 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.782 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.783 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.512112+0000", 2024-08-17T17:33:27.783 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.14983017000000001 2024-08-17T17:33:27.783 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.783 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.783 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.783 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.512174+0000", 2024-08-17T17:33:27.783 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1994000000000003e-05 2024-08-17T17:33:27.783 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.783 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.783 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.783 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:29:32.512208+0000", 2024-08-17T17:33:27.783 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4004000000000002e-05 2024-08-17T17:33:27.783 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.784 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.784 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.784 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.784 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.784 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:248514 9.d 9:bf10c044:::benchmark_data_smithi063_263014_object124176:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1277)", 2024-08-17T17:33:27.784 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:30:12.343796+0000", 2024-08-17T17:33:27.784 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 195.31028605099999, 2024-08-17T17:33:27.784 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.273035044, 2024-08-17T17:33:27.784 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.785 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.785 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.785 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.785 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.785 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 248514 2024-08-17T17:33:27.785 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.785 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.785 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.785 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.785 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.343796+0000", 2024-08-17T17:33:27.785 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.785 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.786 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.786 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.786 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.343796+0000", 2024-08-17T17:33:27.786 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.786 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.786 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.786 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.786 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.343795+0000", 2024-08-17T17:33:27.786 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:33:27.786 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.786 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.786 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.786 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.343800+0000", 2024-08-17T17:33:27.787 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4759999999999997e-06 2024-08-17T17:33:27.787 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.787 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.787 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.787 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.343811+0000", 2024-08-17T17:33:27.787 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0247e-05 2024-08-17T17:33:27.787 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.787 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.787 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.787 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.343817+0000", 2024-08-17T17:33:27.787 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.404e-06 2024-08-17T17:33:27.787 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.787 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.788 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.788 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.343867+0000", 2024-08-17T17:33:27.788 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9678000000000001e-05 2024-08-17T17:33:27.788 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.788 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.788 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.788 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.343981+0000", 2024-08-17T17:33:27.788 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011388899999999999 2024-08-17T17:33:27.788 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.788 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.788 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,5", 2024-08-17T17:33:27.788 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.344094+0000", 2024-08-17T17:33:27.789 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011362399999999999 2024-08-17T17:33:27.789 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.789 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.789 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.789 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.349017+0000", 2024-08-17T17:33:27.789 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0049224259999999997 2024-08-17T17:33:27.789 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.789 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.789 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.789 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.349043+0000", 2024-08-17T17:33:27.789 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5959e-05 2024-08-17T17:33:27.789 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.789 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.790 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.790 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.616745+0000", 2024-08-17T17:33:27.790 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.26770211199999999 2024-08-17T17:33:27.790 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.790 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.790 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.790 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.616794+0000", 2024-08-17T17:33:27.790 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8912999999999997e-05 2024-08-17T17:33:27.790 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.790 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.790 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.790 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.616831+0000", 2024-08-17T17:33:27.791 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7613000000000001e-05 2024-08-17T17:33:27.791 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.791 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.791 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.791 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.791 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.791 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45521.0:248534 9.d 9:bd42dfce:::benchmark_data_smithi063_263014_object124196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1277)", 2024-08-17T17:33:27.791 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:30:12.465666+0000", 2024-08-17T17:33:27.791 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 195.18841584099999, 2024-08-17T17:33:27.791 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.15130177, 2024-08-17T17:33:27.791 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:33:27.791 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:33:27.791 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:33:27.792 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45521", 2024-08-17T17:33:27.792 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:33:27.792 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 248534 2024-08-17T17:33:27.792 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.792 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:33:27.792 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.792 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:33:27.792 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.465666+0000", 2024-08-17T17:33:27.792 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.792 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.792 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.792 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:33:27.792 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.465666+0000", 2024-08-17T17:33:27.792 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:33:27.792 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.793 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.793 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:33:27.793 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.465666+0000", 2024-08-17T17:33:27.793 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:33:27.793 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.793 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.793 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:33:27.793 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.465670+0000", 2024-08-17T17:33:27.793 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9620000000000004e-06 2024-08-17T17:33:27.793 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.793 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.793 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:33:27.793 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.465678+0000", 2024-08-17T17:33:27.793 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4260000000000003e-06 2024-08-17T17:33:27.794 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.794 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.794 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:33:27.794 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.465684+0000", 2024-08-17T17:33:27.794 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.806e-06 2024-08-17T17:33:27.794 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.794 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.794 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:33:27.794 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.465728+0000", 2024-08-17T17:33:27.794 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4267000000000002e-05 2024-08-17T17:33:27.794 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.794 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.794 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:33:27.795 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.465805+0000", 2024-08-17T17:33:27.795 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7170999999999997e-05 2024-08-17T17:33:27.795 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.795 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.795 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,5", 2024-08-17T17:33:27.795 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.465886+0000", 2024-08-17T17:33:27.795 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0626999999999999e-05 2024-08-17T17:33:27.795 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.795 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.795 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:33:27.795 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.469008+0000", 2024-08-17T17:33:27.795 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0031219770000000002 2024-08-17T17:33:27.795 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.795 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.796 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.796 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.469136+0000", 2024-08-17T17:33:27.796 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012775199999999999 2024-08-17T17:33:27.796 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.796 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.796 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:33:27.796 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.616885+0000", 2024-08-17T17:33:27.796 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.147749557 2024-08-17T17:33:27.796 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.796 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.796 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:33:27.796 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.616930+0000", 2024-08-17T17:33:27.796 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4623000000000003e-05 2024-08-17T17:33:27.796 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:33:27.796 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:33:27.797 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:33:27.797 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:30:12.616968+0000", 2024-08-17T17:33:27.797 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8306000000000002e-05 2024-08-17T17:33:27.797 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.797 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.797 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.797 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:33:27.797 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:33:27.797 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:33:27.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.798 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:27.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:27.899 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:27.939 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:27.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:27.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:28.041 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:28.071 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:28.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:27 smithi162 ceph-mon[104668]: pgmap v1969: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped+scrubbing, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 39 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 10 active+undersized+degraded, 57 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 774 op/s; 25912/179007 objects degraded (14.475%); 16921/179007 objects misplaced (9.453%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T17:33:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:27 smithi162 ceph-mon[104668]: 10.18 scrub starts 2024-08-17T17:33:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:27 smithi162 ceph-mon[104668]: 10.10 starting backfill to osd.1 from (0'0,0'0] MAX to 1321'10711 2024-08-17T17:33:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:27 smithi162 ceph-mon[104668]: 10.0 starting backfill to osd.1 from (0'0,0'0] MAX to 1321'10796 2024-08-17T17:33:28.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:27 smithi162 ceph-mon[104668]: 10.b starting backfill to osd.1 from (0'0,0'0] MAX to 1321'13733 2024-08-17T17:33:28.173 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:28.214 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:28.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:28.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:27 smithi063 ceph-mon[114756]: pgmap v1969: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped+scrubbing, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 39 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 10 active+undersized+degraded, 57 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 774 op/s; 25912/179007 objects degraded (14.475%); 16921/179007 objects misplaced (9.453%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T17:33:28.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:27 smithi063 ceph-mon[114756]: 10.18 scrub starts 2024-08-17T17:33:28.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:27 smithi063 ceph-mon[114756]: 10.10 starting backfill to osd.1 from (0'0,0'0] MAX to 1321'10711 2024-08-17T17:33:28.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:27 smithi063 ceph-mon[114756]: 10.0 starting backfill to osd.1 from (0'0,0'0] MAX to 1321'10796 2024-08-17T17:33:28.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:27 smithi063 ceph-mon[114756]: 10.b starting backfill to osd.1 from (0'0,0'0] MAX to 1321'13733 2024-08-17T17:33:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:27 smithi063 ceph-mon[109843]: pgmap v1969: 129 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped+scrubbing, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 39 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 10 active+undersized+degraded, 57 active+clean; 3.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 774 op/s; 25912/179007 objects degraded (14.475%); 16921/179007 objects misplaced (9.453%); 6.1 MiB/s, 97 objects/s recovering 2024-08-17T17:33:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:27 smithi063 ceph-mon[109843]: 10.18 scrub starts 2024-08-17T17:33:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:27 smithi063 ceph-mon[109843]: 10.10 starting backfill to osd.1 from (0'0,0'0] MAX to 1321'10711 2024-08-17T17:33:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:27 smithi063 ceph-mon[109843]: 10.0 starting backfill to osd.1 from (0'0,0'0] MAX to 1321'10796 2024-08-17T17:33:28.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:27 smithi063 ceph-mon[109843]: 10.b starting backfill to osd.1 from (0'0,0'0] MAX to 1321'13733 2024-08-17T17:33:28.316 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:28.347 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:28.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:28.449 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:28.487 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:28.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:28.588 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:28.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.630 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:28.630 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:28.731 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:28.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.771 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:28.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:28.872 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:28.908 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:28.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:28.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:29.010 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:29.053 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:29.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:29.071 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:28 smithi162 ceph-mon[104668]: 10.18 scrub ok 2024-08-17T17:33:29.071 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:28 smithi162 ceph-mon[104668]: osdmap e1325: 8 total, 7 up, 8 in 2024-08-17T17:33:29.071 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:28 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 25912/179007 objects degraded (14.475%), 13 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:29.071 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:28 smithi162 ceph-mon[104668]: osdmap e1326: 8 total, 7 up, 8 in 2024-08-17T17:33:29.110 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:33:29.155 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:29.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.187 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:29.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:29.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:28 smithi063 ceph-mon[114756]: 10.18 scrub ok 2024-08-17T17:33:29.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:28 smithi063 ceph-mon[114756]: osdmap e1325: 8 total, 7 up, 8 in 2024-08-17T17:33:29.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:28 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 25912/179007 objects degraded (14.475%), 13 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:29.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:28 smithi063 ceph-mon[114756]: osdmap e1326: 8 total, 7 up, 8 in 2024-08-17T17:33:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:28 smithi063 ceph-mon[109843]: 10.18 scrub ok 2024-08-17T17:33:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:28 smithi063 ceph-mon[109843]: osdmap e1325: 8 total, 7 up, 8 in 2024-08-17T17:33:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:28 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 25912/179007 objects degraded (14.475%), 13 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:29.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:28 smithi063 ceph-mon[109843]: osdmap e1326: 8 total, 7 up, 8 in 2024-08-17T17:33:29.288 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:29.350 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:29.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:29.454 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:29.485 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:29.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:29.587 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:29.625 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:29.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:29.727 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:29.759 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:29.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:29.860 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:29.895 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:29.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:29.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:29.996 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:30.028 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:30.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:30.129 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:30.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.169 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:30.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:30.189 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:29 smithi063 ceph-mon[114756]: pgmap v1972: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 8 active+undersized+remapped, 1 active+remapped+backfilling, 1 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 55 active+clean; 3.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 24614/169371 objects degraded (14.533%); 21395/169371 objects misplaced (12.632%); 27 MiB/s, 428 objects/s recovering 2024-08-17T17:33:30.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:29 smithi063 ceph-mon[109843]: pgmap v1972: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 8 active+undersized+remapped, 1 active+remapped+backfilling, 1 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 55 active+clean; 3.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 24614/169371 objects degraded (14.533%); 21395/169371 objects misplaced (12.632%); 27 MiB/s, 428 objects/s recovering 2024-08-17T17:33:30.270 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:30.301 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:30.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:29 smithi162 ceph-mon[104668]: pgmap v1972: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 8 active+undersized+remapped, 1 active+remapped+backfilling, 1 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 55 active+clean; 3.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 24614/169371 objects degraded (14.533%); 21395/169371 objects misplaced (12.632%); 27 MiB/s, 428 objects/s recovering 2024-08-17T17:33:30.403 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:30.436 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:30.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:30.538 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:30.578 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:30.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:30.680 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:30.728 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:30.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:30.830 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:30.866 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:30.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:30.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:30.968 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:31.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.012 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:31.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:31.113 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:31.149 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:31.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:31.250 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:31.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.293 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:31.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:31.393 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:31.440 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:31.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:31.542 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:31.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.585 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:31.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:31.686 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:31.744 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:31.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:31.846 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:31.883 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:31.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:31.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:31.985 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:32.025 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:32.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:32.126 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:32.167 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:32.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:32.269 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:32.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:31 smithi063 ceph-mon[109843]: pgmap v1974: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 8 active+undersized+remapped, 1 active+remapped+backfilling, 1 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 55 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 24614/169371 objects degraded (14.533%); 21395/169371 objects misplaced (12.632%); 23 MiB/s, 368 objects/s recovering 2024-08-17T17:33:32.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:32.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:32.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:32.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:31 smithi063 ceph-mon[114756]: pgmap v1974: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 8 active+undersized+remapped, 1 active+remapped+backfilling, 1 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 55 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 24614/169371 objects degraded (14.533%); 21395/169371 objects misplaced (12.632%); 23 MiB/s, 368 objects/s recovering 2024-08-17T17:33:32.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:32.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:32.282 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:32.317 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:32.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:31 smithi162 ceph-mon[104668]: pgmap v1974: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 8 active+undersized+remapped, 1 active+remapped+backfilling, 1 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 55 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 24614/169371 objects degraded (14.533%); 21395/169371 objects misplaced (12.632%); 23 MiB/s, 368 objects/s recovering 2024-08-17T17:33:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:32.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:32.418 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:32.448 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:32.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:32.550 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:32.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.603 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:32.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:32.706 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:32.751 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:32.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:32.853 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:32.900 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:32.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:32.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:33.002 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:33.039 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:33.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:33.141 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:33.171 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:33.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:33.175 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:33.216 INFO:tasks.thrashosds.thrasher:Added osd 1 2024-08-17T17:33:33.246 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:33:33.273 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:33.342 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:33.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:33.444 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:33.447 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:33:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:33:33] "GET /metrics HTTP/1.1" 200 36608 "" "Prometheus/2.51.0" 2024-08-17T17:33:33.447 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:33 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:33.447 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:33 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:33.447 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:33 smithi063 ceph-mon[109843]: pgmap v1975: 129 pgs: 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped+scrubbing, 1 active+remapped+backfilling, 4 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 21516/156156 objects degraded (13.779%); 19853/156156 objects misplaced (12.714%); 29 MiB/s, 460 objects/s recovering 2024-08-17T17:33:33.447 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:33 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 24614/169371 objects degraded (14.533%), 13 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:33.448 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:33 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:33.448 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:33 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:33.448 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:33 smithi063 ceph-mon[114756]: pgmap v1975: 129 pgs: 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped+scrubbing, 1 active+remapped+backfilling, 4 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 21516/156156 objects degraded (13.779%); 19853/156156 objects misplaced (12.714%); 29 MiB/s, 460 objects/s recovering 2024-08-17T17:33:33.448 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:33 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 24614/169371 objects degraded (14.533%), 13 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:33.472 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:33.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:33.498 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:33 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:33.498 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:33 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:33.498 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:33 smithi162 ceph-mon[104668]: pgmap v1975: 129 pgs: 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped+scrubbing, 1 active+remapped+backfilling, 4 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 21516/156156 objects degraded (13.779%); 19853/156156 objects misplaced (12.714%); 29 MiB/s, 460 objects/s recovering 2024-08-17T17:33:33.498 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:33 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 24614/169371 objects degraded (14.533%), 13 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:33.574 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:33.609 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:33.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:33.711 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:33.754 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:33.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:33.856 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:33.913 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:33:33.913 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:33:33.913 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:33:33.913 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:33:33.915 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:33.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:33.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:33.948 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:33:33.948 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:33:33.983 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:33:33.983 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:33:34.017 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:33:34.018 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:33:34.018 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:34.060 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:33:34.060 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:33:34.062 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:33:34.072 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:34.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:34.103 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:33:34.103 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:33:34.174 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:34.208 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:34.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:34.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:34 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3806782121' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:34.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:34 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3806782121' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:34.309 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:34.353 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:34.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:34.455 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:34.513 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:34.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:34.589 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:34 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3806782121' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:34.619 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:34.652 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:34.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:34.754 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:34.783 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:34.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:34.884 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:34.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:34.916 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:34.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:35.017 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:35.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.053 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:35.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:35.154 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:35.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.200 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:35.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:35.215 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:33:35.215 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:33:35.301 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:35.328 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:35.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:35.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:35 smithi162 ceph-mon[104668]: pgmap v1976: 129 pgs: 2 active+remapped+backfilling, 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped+scrubbing, 4 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 237 op/s; 21516/153090 objects degraded (14.054%); 18432/153090 objects misplaced (12.040%); 31 MiB/s, 490 objects/s recovering 2024-08-17T17:33:35.429 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:35.452 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:35 smithi063 ceph-mon[109843]: pgmap v1976: 129 pgs: 2 active+remapped+backfilling, 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped+scrubbing, 4 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 237 op/s; 21516/153090 objects degraded (14.054%); 18432/153090 objects misplaced (12.040%); 31 MiB/s, 490 objects/s recovering 2024-08-17T17:33:35.453 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:35 smithi063 ceph-mon[114756]: pgmap v1976: 129 pgs: 2 active+remapped+backfilling, 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped+scrubbing, 4 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 237 op/s; 21516/153090 objects degraded (14.054%); 18432/153090 objects misplaced (12.040%); 31 MiB/s, 490 objects/s recovering 2024-08-17T17:33:35.457 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:35.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:35.559 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:35.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.594 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:35.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:35.695 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:35.772 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:35.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:35.779 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:35.874 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:35.932 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:35.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:35.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:36.034 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:36.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.076 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:36.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:36.177 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:36.222 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:36.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:36.324 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:36.351 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:36.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:36.453 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:36.480 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:36.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:36.581 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:36.622 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:36.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:36.645 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:33:36.645 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:33:36.724 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:36.758 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:36.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:36.860 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:36.893 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:36.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:36.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:36.995 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:37.032 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:37.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:37.133 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:37.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.176 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:37.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:37.217 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:33:37.277 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:37.327 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:37.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:37.430 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:37.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.484 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:37.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:37.584 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:37.623 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:37.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:37.724 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:37.759 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:37.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:37.861 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:37.886 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:37.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:37.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:37.988 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:38.018 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:38.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:38.028 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:37 smithi063 ceph-mon[114756]: pgmap v1977: 129 pgs: 2 active+remapped+backfilling, 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped+scrubbing, 4 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 21172/152058 objects degraded (13.924%); 18432/152058 objects misplaced (12.122%); 15 MiB/s, 235 objects/s recovering 2024-08-17T17:33:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:37 smithi063 ceph-mon[109843]: pgmap v1977: 129 pgs: 2 active+remapped+backfilling, 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped+scrubbing, 4 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 21172/152058 objects degraded (13.924%); 18432/152058 objects misplaced (12.122%); 15 MiB/s, 235 objects/s recovering 2024-08-17T17:33:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:37 smithi162 ceph-mon[104668]: pgmap v1977: 129 pgs: 2 active+remapped+backfilling, 1 active+clean+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+remapped+scrubbing, 4 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 21172/152058 objects degraded (13.924%); 18432/152058 objects misplaced (12.122%); 15 MiB/s, 235 objects/s recovering 2024-08-17T17:33:38.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:38.142 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:38.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:38.216 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0, 1] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:33:38.217 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:33:38.217 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2024-08-17T17:33:38.217 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:33:38.243 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:38.268 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:38.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:38.370 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:38.392 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:38.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:38.493 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:38.537 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:38.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:38.639 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:38.661 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:38.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:38.719 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:38 smithi063 ceph-mon[114756]: osdmap e1327: 8 total, 7 up, 8 in 2024-08-17T17:33:38.719 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:38 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 21172/152058 objects degraded (13.924%), 13 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:38.720 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:38 smithi063 ceph-mon[114756]: osdmap e1328: 8 total, 7 up, 8 in 2024-08-17T17:33:38.720 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:38 smithi063 ceph-mon[109843]: osdmap e1327: 8 total, 7 up, 8 in 2024-08-17T17:33:38.720 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:38 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 21172/152058 objects degraded (13.924%), 13 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:38.720 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:38 smithi063 ceph-mon[109843]: osdmap e1328: 8 total, 7 up, 8 in 2024-08-17T17:33:38.763 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:38.792 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:38.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:38.894 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:38.926 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:38.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:38.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:38.969 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:33:39.028 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:39.066 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:39.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:39.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:38 smithi162 ceph-mon[104668]: osdmap e1327: 8 total, 7 up, 8 in 2024-08-17T17:33:39.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:38 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 21172/152058 objects degraded (13.924%), 13 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:38 smithi162 ceph-mon[104668]: osdmap e1328: 8 total, 7 up, 8 in 2024-08-17T17:33:39.168 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:39.194 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:39.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:39.296 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:39.321 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:39.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:39.423 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:39.452 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:39.470 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:39.481 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:39.555 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:39.585 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:39.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:39.687 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:39.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.751 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:39.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:39.853 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:39.867 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:39 smithi063 ceph-mon[109843]: pgmap v1979: 129 pgs: 1 active+undersized+remapped, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 1 peering, 1 active+clean+remapped+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 18630/138336 objects degraded (13.467%); 15528/138336 objects misplaced (11.225%); 28 MiB/s, 445 objects/s recovering 2024-08-17T17:33:39.867 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:39 smithi063 ceph-mon[109843]: osdmap e1329: 8 total, 7 up, 8 in 2024-08-17T17:33:39.872 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:39 smithi063 ceph-mon[114756]: pgmap v1979: 129 pgs: 1 active+undersized+remapped, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 1 peering, 1 active+clean+remapped+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 18630/138336 objects degraded (13.467%); 15528/138336 objects misplaced (11.225%); 28 MiB/s, 445 objects/s recovering 2024-08-17T17:33:39.872 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:39 smithi063 ceph-mon[114756]: osdmap e1329: 8 total, 7 up, 8 in 2024-08-17T17:33:39.899 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:39.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:39.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:40.001 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:40.034 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:40.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:40.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:39 smithi162 ceph-mon[104668]: pgmap v1979: 129 pgs: 1 active+undersized+remapped, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 1 peering, 1 active+clean+remapped+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 18630/138336 objects degraded (13.467%); 15528/138336 objects misplaced (11.225%); 28 MiB/s, 445 objects/s recovering 2024-08-17T17:33:40.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:39 smithi162 ceph-mon[104668]: osdmap e1329: 8 total, 7 up, 8 in 2024-08-17T17:33:40.113 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:40.136 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:40.161 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:40.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:40.262 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:40.290 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:40.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:40.392 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:40.421 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:40.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:40.523 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:40.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:40.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:40.604 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:33:40.647 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:40.670 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:40.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:40.773 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:40.799 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:40.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:40.875 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:40 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/95519171' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:40.875 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:40 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:33:40.875 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:40 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/95519171' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:40.875 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:40 smithi063 ceph-mon[109843]: osdmap e1330: 8 total, 7 up, 8 in 2024-08-17T17:33:40.875 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:40 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/95519171' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:40.876 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:40 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:33:40.876 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:40 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/95519171' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:40.876 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:40 smithi063 ceph-mon[114756]: osdmap e1330: 8 total, 7 up, 8 in 2024-08-17T17:33:40.901 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:40.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:40.937 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:40.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:41.023 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:40 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/95519171' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:33:41.023 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:40 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:33:41.023 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:40 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/95519171' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:33:41.023 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:40 smithi162 ceph-mon[104668]: osdmap e1330: 8 total, 7 up, 8 in 2024-08-17T17:33:41.037 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:41.067 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:41.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:41.168 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:41.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.199 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:41.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:41.300 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:41.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.335 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:41.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:41.436 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:41.472 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:41.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:41.574 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:41.607 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:41.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:41.708 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:41.740 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:41.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:41.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:41 smithi063 ceph-mon[114756]: pgmap v1982: 129 pgs: 1 active+undersized+remapped, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 1 peering, 1 active+clean+remapped+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 18630/138336 objects degraded (13.467%); 15528/138336 objects misplaced (11.225%); 24 MiB/s, 376 objects/s recovering 2024-08-17T17:33:41.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:41 smithi063 ceph-mon[109843]: pgmap v1982: 129 pgs: 1 active+undersized+remapped, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 1 peering, 1 active+clean+remapped+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 18630/138336 objects degraded (13.467%); 15528/138336 objects misplaced (11.225%); 24 MiB/s, 376 objects/s recovering 2024-08-17T17:33:41.871 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:41.915 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:41.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:41.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:42.017 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:42.019 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:41 smithi162 ceph-mon[104668]: pgmap v1982: 129 pgs: 1 active+undersized+remapped, 1 active+clean+scrubbing, 2 active+remapped+backfilling, 1 peering, 1 active+clean+remapped+scrubbing, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 32 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 18630/138336 objects degraded (13.467%); 15528/138336 objects misplaced (11.225%); 24 MiB/s, 376 objects/s recovering 2024-08-17T17:33:42.049 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:42.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:42.151 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:42.182 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:42.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:42.284 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:42.319 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:42.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:42.421 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:42.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.462 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:42.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:42.563 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:42.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.604 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:42.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:42.705 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:42.744 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:42.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:42.846 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:42.883 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:42.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:42.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:42.985 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:43.023 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:43.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:43.125 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:43.153 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:43.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:43.255 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:43.290 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:43.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:43.392 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:43.424 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:43.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:43.526 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:43.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:33:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:33:43] "GET /metrics HTTP/1.1" 200 36608 "" "Prometheus/2.51.0" 2024-08-17T17:33:43.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.596 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:43.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:43.697 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:43.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.731 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:43.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:43.832 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:43.854 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:43.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:43.956 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:43.994 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:43.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:43.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:43.996 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:43 smithi063 ceph-mon[114756]: pgmap v1984: 129 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 16046/130128 objects degraded (12.331%); 14238/130128 objects misplaced (10.942%); 35 MiB/s, 604 objects/s recovering 2024-08-17T17:33:43.996 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:43 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 16046/130128 objects degraded (12.331%), 12 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:43.996 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:43 smithi063 ceph-mon[109843]: pgmap v1984: 129 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 16046/130128 objects degraded (12.331%); 14238/130128 objects misplaced (10.942%); 35 MiB/s, 604 objects/s recovering 2024-08-17T17:33:43.996 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:43 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 16046/130128 objects degraded (12.331%), 12 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:44.013 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:43 smithi162 ceph-mon[104668]: pgmap v1984: 129 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 62 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 734 op/s; 16046/130128 objects degraded (12.331%); 14238/130128 objects misplaced (10.942%); 35 MiB/s, 604 objects/s recovering 2024-08-17T17:33:44.013 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:43 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 16046/130128 objects degraded (12.331%), 12 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:44.096 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:44.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.143 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:44.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:44.244 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:44.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.304 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:44.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:44.406 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:44.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.452 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:44.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:44.553 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:44.591 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:44.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:44.692 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:44.733 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:44.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:44.835 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:44.871 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:44.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:44.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:44.973 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:45.015 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:45.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:45.117 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:45.158 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:45.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:45.260 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:45.292 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:45.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:45.394 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:45.435 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:45.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:45.536 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:45.569 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:45.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:45.582 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:33:45.671 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:45.745 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:45.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.747 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:45.771 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:33:45.771 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:33:45.772 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:45 smithi063 ceph-mon[109843]: pgmap v1985: 129 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 65 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 16046/128298 objects degraded (12.507%); 12585/128298 objects misplaced (9.809%); 31 MiB/s, 540 objects/s recovering 2024-08-17T17:33:45.772 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:45 smithi063 ceph-mon[114756]: pgmap v1985: 129 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 65 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 16046/128298 objects degraded (12.507%); 12585/128298 objects misplaced (9.809%); 31 MiB/s, 540 objects/s recovering 2024-08-17T17:33:45.848 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:45.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:45.892 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:45.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:45.992 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:46.021 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:46.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:45 smithi162 ceph-mon[104668]: pgmap v1985: 129 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 65 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 16046/128298 objects degraded (12.507%); 12585/128298 objects misplaced (9.809%); 31 MiB/s, 540 objects/s recovering 2024-08-17T17:33:46.123 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:46.150 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:46.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:46.252 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:46.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.276 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:46.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:46.377 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:46.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.401 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:46.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:46.502 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:46.526 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:46.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:46.628 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:46.651 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:46.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:46.749 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:33:46.753 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:46.831 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:46.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:46.933 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:46.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:46.964 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:46.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:46.967 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:46 smithi063 ceph-mon[114756]: osdmap e1331: 8 total, 7 up, 8 in 2024-08-17T17:33:46.967 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:46.967 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:46.968 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:46 smithi063 ceph-mon[114756]: osdmap e1332: 8 total, 7 up, 8 in 2024-08-17T17:33:46.969 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:46 smithi063 ceph-mon[109843]: osdmap e1331: 8 total, 7 up, 8 in 2024-08-17T17:33:46.969 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:46.969 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:46.969 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:46 smithi063 ceph-mon[109843]: osdmap e1332: 8 total, 7 up, 8 in 2024-08-17T17:33:47.066 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:46 smithi162 ceph-mon[104668]: osdmap e1331: 8 total, 7 up, 8 in 2024-08-17T17:33:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:33:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:46 smithi162 ceph-mon[104668]: osdmap e1332: 8 total, 7 up, 8 in 2024-08-17T17:33:47.103 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:47.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:47.205 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:47.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.236 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:47.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:47.338 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:47.340 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:33:47.341 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:33:47.359 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:33:47.359 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:33:47.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.364 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:47.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:47.404 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:33:47.404 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:33:47.436 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:33:47.436 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:33:47.466 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:47.479 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:33:47.479 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:33:47.525 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:33:47.525 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:33:47.526 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:33:47.529 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:47.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:47.570 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:33:47.571 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:33:47.631 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:47.676 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:47.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:47.778 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:47.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:47 smithi063 ceph-mon[114756]: pgmap v1987: 129 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 65 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 15726/127338 objects degraded (12.350%); 12585/127338 objects misplaced (9.883%); 27 MiB/s, 467 objects/s recovering 2024-08-17T17:33:47.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:47 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2800715562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:47.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:47 smithi063 ceph-mon[109843]: pgmap v1987: 129 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 65 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 15726/127338 objects degraded (12.350%); 12585/127338 objects misplaced (9.883%); 27 MiB/s, 467 objects/s recovering 2024-08-17T17:33:47.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:47 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2800715562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:47.807 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:47.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:47.909 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:47.945 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:47.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:47.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:47.982 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:33:47.984 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:33:48.048 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:48.082 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:48.082 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:47 smithi162 ceph-mon[104668]: pgmap v1987: 129 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 65 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 528 op/s; 15726/127338 objects degraded (12.350%); 12585/127338 objects misplaced (9.883%); 27 MiB/s, 467 objects/s recovering 2024-08-17T17:33:48.083 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:47 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2800715562' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:33:48.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:48.184 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:48.213 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:48.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:48.315 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:48.350 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:48.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:48.452 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:48.487 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:48.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:48.589 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:48.610 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:48.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:48.712 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:48.754 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:48.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:48.856 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:48.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:48.887 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:48.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:48.988 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:49.017 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:49.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:48 smithi063 ceph-mon[109843]: from='client.46001 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:33:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:48 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 15726/127338 objects degraded (12.350%), 12 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:48 smithi063 ceph-mon[114756]: from='client.46001 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:33:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:48 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 15726/127338 objects degraded (12.350%), 12 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:48 smithi162 ceph-mon[104668]: from='client.46001 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:33:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:48 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 15726/127338 objects degraded (12.350%), 12 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:49.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:49.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.144 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:49.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:49.245 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:49.270 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:49.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:49.371 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:49.394 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:49.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:49.496 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:49.518 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:49.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:49.620 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:49.644 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:49.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:49.746 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:49.773 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:49.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:49.875 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:49.914 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:49.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:49.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:49.945 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:49 smithi063 ceph-mon[109843]: pgmap v1989: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+scrubbing, 3 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 66 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 13199/114111 objects degraded (11.567%); 10694/114111 objects misplaced (9.372%); 35 MiB/s, 640 objects/s recovering 2024-08-17T17:33:49.946 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:49 smithi063 ceph-mon[109843]: osdmap e1333: 8 total, 7 up, 8 in 2024-08-17T17:33:49.946 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:49 smithi063 ceph-mon[114756]: pgmap v1989: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+scrubbing, 3 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 66 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 13199/114111 objects degraded (11.567%); 10694/114111 objects misplaced (9.372%); 35 MiB/s, 640 objects/s recovering 2024-08-17T17:33:49.946 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:49 smithi063 ceph-mon[114756]: osdmap e1333: 8 total, 7 up, 8 in 2024-08-17T17:33:50.017 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:50.060 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:50.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:49 smithi162 ceph-mon[104668]: pgmap v1989: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+scrubbing, 3 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 66 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 13199/114111 objects degraded (11.567%); 10694/114111 objects misplaced (9.372%); 35 MiB/s, 640 objects/s recovering 2024-08-17T17:33:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:49 smithi162 ceph-mon[104668]: osdmap e1333: 8 total, 7 up, 8 in 2024-08-17T17:33:50.162 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:50.197 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:50.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:50.299 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:50.331 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:50.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:50.433 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:50.442 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":1988,"stamp":"2024-08-17T17:33:46.693139+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2780576584,"num_objects":42446,"num_object_clones":0,"num_object_copies":127338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15726,"num_objects_misplaced":12585,"num_objects_unfound":0,"num_objects_dirty":42446,"num_whiteouts":0,"num_read":71,"num_read_kb":150,"num_write":212407,"num_write_kb":7695834,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13030,"num_bytes_recovered":838713422,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":182467,"ondisk_log_size":182467,"up":327,"acting":361,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":418,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14035956,"kb_used_data":8000660,"kb_used_omap":199,"kb_used_meta":6034936,"kb_avail":735925260,"statfs":{"total":767960285184,"available":753587466240,"internally_reserved":0,"allocated":8192675840,"data_stored":8168609025,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":204036,"internal_metadata":6179775228},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-240254976,"num_objects":-3666,"num_object_clones":0,"num_object_copies":-10998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2904,"num_objects_misplaced":-2943,"num_objects_unfound":0,"num_objects_dirty":-3666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3666,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3246,"num_bytes_recovered":198246400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.150073"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":315,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.615337+0000","last_change":"2024-08-17T17:31:35.678605+0000","last_active":"2024-08-17T17:33:39.615337+0000","last_peered":"2024-08-17T17:33:39.615337+0000","last_clean":"2024-08-17T17:33:39.615337+0000","last_became_active":"2024-08-17T17:31:35.678065+0000","last_became_peered":"2024-08-17T17:31:35.678065+0000","last_unstale":"2024-08-17T17:33:39.615337+0000","last_undegraded":"2024-08-17T17:33:39.615337+0000","last_fullsized":"2024-08-17T17:33:39.615337+0000","mapping_epoch":1299,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1300,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:52:59.133394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1332,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.612056+0000","last_change":"2024-08-17T17:33:27.034479+0000","last_active":"2024-08-17T17:33:39.612056+0000","last_peered":"2024-08-17T17:33:39.612056+0000","last_clean":"2024-08-17T17:33:39.612056+0000","last_became_active":"2024-08-17T17:33:27.034087+0000","last_became_peered":"2024-08-17T17:33:27.034087+0000","last_unstale":"2024-08-17T17:33:39.612056+0000","last_undegraded":"2024-08-17T17:33:39.612056+0000","last_fullsized":"2024-08-17T17:33:39.612056+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:33:11.617604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,2],"avail_no_missing":["4","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1606,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.611785+0000","last_change":"2024-08-17T17:33:28.932679+0000","last_active":"2024-08-17T17:33:39.611785+0000","last_peered":"2024-08-17T17:33:39.611785+0000","last_clean":"2024-08-17T17:33:39.611785+0000","last_became_active":"2024-08-17T17:33:28.932314+0000","last_became_peered":"2024-08-17T17:33:28.932314+0000","last_unstale":"2024-08-17T17:33:39.611785+0000","last_undegraded":"2024-08-17T17:33:39.611785+0000","last_fullsized":"2024-08-17T17:33:39.611785+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:21:33.466698+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.16","version":"1330'11335","reported_seq":11391,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:42.311944+0000","last_change":"2024-08-17T17:32:07.915562+0000","last_active":"2024-08-17T17:33:42.311944+0000","last_peered":"2024-08-17T17:33:42.311944+0000","last_clean":"2024-08-17T17:33:42.311944+0000","last_became_active":"2024-08-17T17:32:07.912432+0000","last_became_peered":"2024-08-17T17:32:07.912432+0000","last_unstale":"2024-08-17T17:33:42.311944+0000","last_undegraded":"2024-08-17T17:33:42.311944+0000","last_fullsized":"2024-08-17T17:33:42.311944+0000","mapping_epoch":1306,"log_start":"1294'4247","ondisk_log_start":"1294'4247","created":1306,"last_epoch_clean":1307,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":7088,"log_dups_size":0,"ondisk_log_size":7088,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87261184,"num_objects":1332,"num_object_clones":0,"num_object_copies":3996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7276,"num_write_kb":275424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":112,"reported_epoch":1330,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:40.611155+0000","last_change":"2024-08-17T17:33:26.973715+0000","last_active":"2024-08-17T17:33:40.611155+0000","last_peered":"2024-08-17T17:33:40.611155+0000","last_clean":"2024-08-17T17:33:40.611155+0000","last_became_active":"2024-08-17T17:33:26.973165+0000","last_became_peered":"2024-08-17T17:33:26.973165+0000","last_unstale":"2024-08-17T17:33:40.611155+0000","last_undegraded":"2024-08-17T17:33:40.611155+0000","last_fullsized":"2024-08-17T17:33:40.611155+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:00:37.903328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1,7],"avail_no_missing":["2","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1486,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.610698+0000","last_change":"2024-08-17T17:33:14.822835+0000","last_active":"2024-08-17T17:33:40.610698+0000","last_peered":"2024-08-17T17:33:40.610698+0000","last_clean":"2024-08-17T17:33:40.610698+0000","last_became_active":"2024-08-17T17:33:14.822423+0000","last_became_peered":"2024-08-17T17:33:14.822423+0000","last_unstale":"2024-08-17T17:33:40.610698+0000","last_undegraded":"2024-08-17T17:33:40.610698+0000","last_fullsized":"2024-08-17T17:33:40.610698+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:34.055293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1724,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.616438+0000","last_change":"2024-08-17T17:33:26.930905+0000","last_active":"2024-08-17T17:33:39.616438+0000","last_peered":"2024-08-17T17:33:39.616438+0000","last_clean":"2024-08-17T17:33:39.616438+0000","last_became_active":"2024-08-17T17:33:26.930418+0000","last_became_peered":"2024-08-17T17:33:26.930418+0000","last_unstale":"2024-08-17T17:33:39.616438+0000","last_undegraded":"2024-08-17T17:33:39.616438+0000","last_fullsized":"2024-08-17T17:33:39.616438+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:21.094599+0000","scrub_duration":0.0084952199999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1,5],"avail_no_missing":["7","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.17","version":"1330'11146","reported_seq":11211,"reported_epoch":1330,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T17:33:42.286889+0000","last_change":"2024-08-17T17:33:27.081103+0000","last_active":"2024-08-17T17:33:42.286889+0000","last_peered":"2024-08-17T17:33:42.286889+0000","last_clean":"2024-08-17T17:30:30.026678+0000","last_became_active":"2024-08-17T17:33:15.958861+0000","last_became_peered":"2024-08-17T17:33:15.958861+0000","last_unstale":"2024-08-17T17:33:42.286889+0000","last_undegraded":"2024-08-17T17:33:15.839724+0000","last_fullsized":"2024-08-17T17:33:15.812467+0000","mapping_epoch":1318,"log_start":"1294'3944","ondisk_log_start":"1294'3944","created":1306,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":7202,"log_dups_size":0,"ondisk_log_size":7202,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83165184,"num_objects":1269,"num_object_clones":0,"num_object_copies":3807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1269,"num_objects_misplaced":1269,"num_objects_unfound":0,"num_objects_dirty":1269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7165,"num_write_kb":269888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":1269}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1267,"reported_epoch":1329,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:33:39.614331+0000","last_change":"2024-08-17T17:29:53.531923+0000","last_active":"2024-08-17T17:33:39.614331+0000","last_peered":"2024-08-17T17:33:39.614331+0000","last_clean":"2024-08-17T17:29:38.564569+0000","last_became_active":"2024-08-17T17:29:53.531923+0000","last_became_peered":"2024-08-17T17:29:53.531923+0000","last_unstale":"2024-08-17T17:33:39.614331+0000","last_undegraded":"2024-08-17T17:29:53.527534+0000","last_fullsized":"2024-08-17T17:29:53.527422+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:28.673742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":80,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.612892+0000","last_change":"2024-08-17T17:31:18.419785+0000","last_active":"2024-08-17T17:33:40.612892+0000","last_peered":"2024-08-17T17:33:40.612892+0000","last_clean":"2024-08-17T17:33:40.612892+0000","last_became_active":"2024-08-17T17:31:18.419485+0000","last_became_peered":"2024-08-17T17:31:18.419485+0000","last_unstale":"2024-08-17T17:33:40.612892+0000","last_undegraded":"2024-08-17T17:33:40.612892+0000","last_fullsized":"2024-08-17T17:33:40.612892+0000","mapping_epoch":1295,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:18:05.421481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.14","version":"1330'11494","reported_seq":11555,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:42.188221+0000","last_change":"2024-08-17T17:32:07.919261+0000","last_active":"2024-08-17T17:33:42.188221+0000","last_peered":"2024-08-17T17:33:42.188221+0000","last_clean":"2024-08-17T17:33:42.188221+0000","last_became_active":"2024-08-17T17:32:07.916886+0000","last_became_peered":"2024-08-17T17:32:07.916886+0000","last_unstale":"2024-08-17T17:33:42.188221+0000","last_undegraded":"2024-08-17T17:33:42.188221+0000","last_fullsized":"2024-08-17T17:33:42.188221+0000","mapping_epoch":1306,"log_start":"1297'5247","ondisk_log_start":"1297'5247","created":1306,"last_epoch_clean":1307,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6247,"log_dups_size":0,"ondisk_log_size":6247,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92372992,"num_objects":1410,"num_object_clones":0,"num_object_copies":4230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7402,"num_write_kb":281952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1322,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.614097+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:33:39.614097+0000","last_peered":"2024-08-17T17:33:39.614097+0000","last_clean":"2024-08-17T17:33:39.614097+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:33:39.614097+0000","last_undegraded":"2024-08-17T17:33:39.614097+0000","last_fullsized":"2024-08-17T17:33:39.614097+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1329,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.730028+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:33:40.730028+0000","last_peered":"2024-08-17T17:33:40.730028+0000","last_clean":"2024-08-17T17:33:40.730028+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:33:40.730028+0000","last_undegraded":"2024-08-17T17:33:40.730028+0000","last_fullsized":"2024-08-17T17:33:40.730028+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":820,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.612648+0000","last_change":"2024-08-17T17:33:17.829007+0000","last_active":"2024-08-17T17:33:39.612648+0000","last_peered":"2024-08-17T17:33:39.612648+0000","last_clean":"2024-08-17T17:33:39.612648+0000","last_became_active":"2024-08-17T17:33:17.828301+0000","last_became_peered":"2024-08-17T17:33:17.828301+0000","last_unstale":"2024-08-17T17:33:39.612648+0000","last_undegraded":"2024-08-17T17:33:39.612648+0000","last_fullsized":"2024-08-17T17:33:39.612648+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:55.848017+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,3],"avail_no_missing":["4","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.15","version":"1330'11297","reported_seq":14572,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:42.664477+0000","last_change":"2024-08-17T17:33:40.748701+0000","last_active":"2024-08-17T17:33:42.664477+0000","last_peered":"2024-08-17T17:33:42.664477+0000","last_clean":"2024-08-17T17:33:42.664477+0000","last_became_active":"2024-08-17T17:33:40.730218+0000","last_became_peered":"2024-08-17T17:33:40.730218+0000","last_unstale":"2024-08-17T17:33:42.664477+0000","last_undegraded":"2024-08-17T17:33:42.664477+0000","last_fullsized":"2024-08-17T17:33:42.664477+0000","mapping_epoch":1329,"log_start":"1292'1953","ondisk_log_start":"1292'1953","created":1306,"last_epoch_clean":1330,"parent":"0.0","parent_split_bits":5,"last_scrub":"1307'8312","last_scrub_stamp":"2024-08-17T17:32:15.481063+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:32:15.481063+0000","objects_scrubbed":4292,"log_size":9344,"log_dups_size":0,"ondisk_log_size":9344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":4,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:29:57.971830+0000","scrub_duration":3761,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88211456,"num_objects":1346,"num_object_clones":0,"num_object_copies":4038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3024,"num_write_kb":2496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1548,"num_bytes_recovered":101449728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":311,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.611825+0000","last_change":"2024-08-17T17:33:26.969793+0000","last_active":"2024-08-17T17:33:39.611825+0000","last_peered":"2024-08-17T17:33:39.611825+0000","last_clean":"2024-08-17T17:33:39.611825+0000","last_became_active":"2024-08-17T17:33:26.965090+0000","last_became_peered":"2024-08-17T17:33:26.965090+0000","last_unstale":"2024-08-17T17:33:39.611825+0000","last_undegraded":"2024-08-17T17:33:39.611825+0000","last_fullsized":"2024-08-17T17:33:39.611825+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:55:40.904290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1835,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.611471+0000","last_change":"2024-08-17T17:33:25.949892+0000","last_active":"2024-08-17T17:33:40.611471+0000","last_peered":"2024-08-17T17:33:40.611471+0000","last_clean":"2024-08-17T17:33:40.611471+0000","last_became_active":"2024-08-17T17:33:25.947722+0000","last_became_peered":"2024-08-17T17:33:25.947722+0000","last_unstale":"2024-08-17T17:33:40.611471+0000","last_undegraded":"2024-08-17T17:33:40.611471+0000","last_fullsized":"2024-08-17T17:33:40.611471+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:07.290679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1483,"reported_epoch":1330,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:40.608876+0000","last_change":"2024-08-17T17:33:40.608876+0000","last_active":"2024-08-17T17:33:40.608876+0000","last_peered":"2024-08-17T17:33:40.608876+0000","last_clean":"2024-08-17T17:33:40.608876+0000","last_became_active":"2024-08-17T17:33:17.833356+0000","last_became_peered":"2024-08-17T17:33:17.833356+0000","last_unstale":"2024-08-17T17:33:40.608876+0000","last_undegraded":"2024-08-17T17:33:40.608876+0000","last_fullsized":"2024-08-17T17:33:40.608876+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.911622+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.911622+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:33:44.824434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":18,"num_read_kb":38,"num_write":10,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":107340,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7,3],"avail_no_missing":["0","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.12","version":"1330'11335","reported_seq":11414,"reported_epoch":1330,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:33:42.226288+0000","last_change":"2024-08-17T17:33:26.064035+0000","last_active":"2024-08-17T17:33:42.226288+0000","last_peered":"2024-08-17T17:33:42.226288+0000","last_clean":"2024-08-17T17:33:24.895665+0000","last_became_active":"2024-08-17T17:33:26.039946+0000","last_became_peered":"2024-08-17T17:33:26.039946+0000","last_unstale":"2024-08-17T17:33:42.226288+0000","last_undegraded":"2024-08-17T17:33:42.226288+0000","last_fullsized":"2024-08-17T17:33:42.226288+0000","mapping_epoch":1322,"log_start":"1297'5104","ondisk_log_start":"1297'5104","created":1306,"last_epoch_clean":1319,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6231,"log_dups_size":0,"ondisk_log_size":6231,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87752704,"num_objects":1339,"num_object_clones":0,"num_object_copies":4017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1339,"num_objects_unfound":0,"num_objects_dirty":1339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7285,"num_write_kb":275968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,0,3],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":1339}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1331,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.612340+0000","last_change":"2024-08-17T17:33:26.040114+0000","last_active":"2024-08-17T17:33:39.612340+0000","last_peered":"2024-08-17T17:33:39.612340+0000","last_clean":"2024-08-17T17:33:39.612340+0000","last_became_active":"2024-08-17T17:33:26.039801+0000","last_became_peered":"2024-08-17T17:33:26.039801+0000","last_unstale":"2024-08-17T17:33:39.612340+0000","last_undegraded":"2024-08-17T17:33:39.612340+0000","last_fullsized":"2024-08-17T17:33:39.612340+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:39:04.901831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1737,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.612340+0000","last_change":"2024-08-17T17:33:26.966472+0000","last_active":"2024-08-17T17:33:39.612340+0000","last_peered":"2024-08-17T17:33:39.612340+0000","last_clean":"2024-08-17T17:33:39.612340+0000","last_became_active":"2024-08-17T17:33:26.964811+0000","last_became_peered":"2024-08-17T17:33:26.964811+0000","last_unstale":"2024-08-17T17:33:39.612340+0000","last_undegraded":"2024-08-17T17:33:39.612340+0000","last_fullsized":"2024-08-17T17:33:39.612340+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:12:09.271941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,5],"avail_no_missing":["4","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1643,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.613498+0000","last_change":"2024-08-17T17:25:02.696608+0000","last_active":"2024-08-17T17:33:39.613498+0000","last_peered":"2024-08-17T17:33:39.613498+0000","last_clean":"2024-08-17T17:33:39.613498+0000","last_became_active":"2024-08-17T17:24:40.877715+0000","last_became_peered":"2024-08-17T17:24:40.877715+0000","last_unstale":"2024-08-17T17:33:39.613498+0000","last_undegraded":"2024-08-17T17:33:39.613498+0000","last_fullsized":"2024-08-17T17:33:39.613498+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:48:00.941285+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.13","version":"1330'11156","reported_seq":14746,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:42.667590+0000","last_change":"2024-08-17T17:33:39.985648+0000","last_active":"2024-08-17T17:33:42.667590+0000","last_peered":"2024-08-17T17:33:42.667590+0000","last_clean":"2024-08-17T17:33:42.667590+0000","last_became_active":"2024-08-17T17:33:39.615241+0000","last_became_peered":"2024-08-17T17:33:39.615241+0000","last_unstale":"2024-08-17T17:33:42.667590+0000","last_undegraded":"2024-08-17T17:33:42.667590+0000","last_fullsized":"2024-08-17T17:33:42.667590+0000","mapping_epoch":1328,"log_start":"1292'1632","ondisk_log_start":"1292'1632","created":1306,"last_epoch_clean":1329,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":9524,"log_dups_size":0,"ondisk_log_size":9524,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:33:44.934576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87949312,"num_objects":1342,"num_object_clones":0,"num_object_copies":4026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7146,"num_write_kb":271616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1727,"num_bytes_recovered":113180672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1300,"reported_epoch":1331,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:45.707155+0000","last_change":"2024-08-17T17:33:26.931912+0000","last_active":"2024-08-17T17:33:45.707155+0000","last_peered":"2024-08-17T17:33:45.707155+0000","last_clean":"2024-08-17T17:33:45.707155+0000","last_became_active":"2024-08-17T17:33:26.931472+0000","last_became_peered":"2024-08-17T17:33:26.931472+0000","last_unstale":"2024-08-17T17:33:45.707155+0000","last_undegraded":"2024-08-17T17:33:45.707155+0000","last_fullsized":"2024-08-17T17:33:45.707155+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:32:16.983718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1,7],"avail_no_missing":["3","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":932,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.608294+0000","last_change":"2024-08-17T17:33:25.945328+0000","last_active":"2024-08-17T17:33:40.608294+0000","last_peered":"2024-08-17T17:33:40.608294+0000","last_clean":"2024-08-17T17:33:40.608294+0000","last_became_active":"2024-08-17T17:33:25.944618+0000","last_became_peered":"2024-08-17T17:33:25.944618+0000","last_unstale":"2024-08-17T17:33:40.608294+0000","last_undegraded":"2024-08-17T17:33:40.608294+0000","last_fullsized":"2024-08-17T17:33:40.608294+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:01.231899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":871,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.613716+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:33:39.613716+0000","last_peered":"2024-08-17T17:33:39.613716+0000","last_clean":"2024-08-17T17:33:39.613716+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:33:39.613716+0000","last_undegraded":"2024-08-17T17:33:39.613716+0000","last_fullsized":"2024-08-17T17:33:39.613716+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.10","version":"1330'11141","reported_seq":14448,"reported_epoch":1330,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:42.261790+0000","last_change":"2024-08-17T17:33:38.685201+0000","last_active":"2024-08-17T17:33:42.261790+0000","last_peered":"2024-08-17T17:33:42.261790+0000","last_clean":"2024-08-17T17:33:42.261790+0000","last_became_active":"2024-08-17T17:33:38.611020+0000","last_became_peered":"2024-08-17T17:33:38.611020+0000","last_unstale":"2024-08-17T17:33:42.261790+0000","last_undegraded":"2024-08-17T17:33:42.261790+0000","last_fullsized":"2024-08-17T17:33:42.261790+0000","mapping_epoch":1327,"log_start":"1296'4788","ondisk_log_start":"1296'4788","created":1306,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6353,"log_dups_size":0,"ondisk_log_size":6353,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91324416,"num_objects":1394,"num_object_clones":0,"num_object_copies":4182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1394,"num_objects_unfound":0,"num_objects_dirty":1394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7132,"num_write_kb":272800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1616,"num_bytes_recovered":105906176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,7],"avail_no_missing":["4","1","7"],"object_location_counts":[{"shards":"1,4,7","objects":1394}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1806,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.610582+0000","last_change":"2024-08-17T17:33:25.949368+0000","last_active":"2024-08-17T17:33:40.610582+0000","last_peered":"2024-08-17T17:33:40.610582+0000","last_clean":"2024-08-17T17:33:40.610582+0000","last_became_active":"2024-08-17T17:33:25.947760+0000","last_became_peered":"2024-08-17T17:33:25.947760+0000","last_unstale":"2024-08-17T17:33:40.610582+0000","last_undegraded":"2024-08-17T17:33:40.610582+0000","last_fullsized":"2024-08-17T17:33:40.610582+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:31.935292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1210,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.615267+0000","last_change":"2024-08-17T17:33:26.935867+0000","last_active":"2024-08-17T17:33:39.615267+0000","last_peered":"2024-08-17T17:33:39.615267+0000","last_clean":"2024-08-17T17:33:39.615267+0000","last_became_active":"2024-08-17T17:33:26.935438+0000","last_became_peered":"2024-08-17T17:33:26.935438+0000","last_unstale":"2024-08-17T17:33:39.615267+0000","last_undegraded":"2024-08-17T17:33:39.615267+0000","last_fullsized":"2024-08-17T17:33:39.615267+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:04:21.910082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1,0],"avail_no_missing":["5","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1545,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.611963+0000","last_change":"2024-08-17T17:33:28.924960+0000","last_active":"2024-08-17T17:33:39.611963+0000","last_peered":"2024-08-17T17:33:39.611963+0000","last_clean":"2024-08-17T17:33:39.611963+0000","last_became_active":"2024-08-17T17:33:28.924558+0000","last_became_peered":"2024-08-17T17:33:28.924558+0000","last_unstale":"2024-08-17T17:33:39.611963+0000","last_undegraded":"2024-08-17T17:33:39.611963+0000","last_fullsized":"2024-08-17T17:33:39.611963+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:29.596215+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.11","version":"1330'11248","reported_seq":11312,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:42.268173+0000","last_change":"2024-08-17T17:33:14.828533+0000","last_active":"2024-08-17T17:33:42.268173+0000","last_peered":"2024-08-17T17:33:42.268173+0000","last_clean":"2024-08-17T17:33:42.268173+0000","last_became_active":"2024-08-17T17:32:07.932531+0000","last_became_peered":"2024-08-17T17:32:07.932531+0000","last_unstale":"2024-08-17T17:33:42.268173+0000","last_undegraded":"2024-08-17T17:33:42.268173+0000","last_fullsized":"2024-08-17T17:33:42.268173+0000","mapping_epoch":1306,"log_start":"1296'4886","ondisk_log_start":"1296'4886","created":1306,"last_epoch_clean":1318,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6362,"log_dups_size":0,"ondisk_log_size":6362,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87588864,"num_objects":1337,"num_object_clones":0,"num_object_copies":4011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7217,"num_write_kb":273696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1691,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.611450+0000","last_change":"2024-08-17T17:33:26.965038+0000","last_active":"2024-08-17T17:33:39.611450+0000","last_peered":"2024-08-17T17:33:39.611450+0000","last_clean":"2024-08-17T17:33:39.611450+0000","last_became_active":"2024-08-17T17:33:26.964597+0000","last_became_peered":"2024-08-17T17:33:26.964597+0000","last_unstale":"2024-08-17T17:33:39.611450+0000","last_undegraded":"2024-08-17T17:33:39.611450+0000","last_fullsized":"2024-08-17T17:33:39.611450+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:45:15.954214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,5],"avail_no_missing":["4","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1139,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.615400+0000","last_change":"2024-08-17T17:33:15.842385+0000","last_active":"2024-08-17T17:33:39.615400+0000","last_peered":"2024-08-17T17:33:39.615400+0000","last_clean":"2024-08-17T17:33:39.615400+0000","last_became_active":"2024-08-17T17:33:15.841697+0000","last_became_peered":"2024-08-17T17:33:15.841697+0000","last_unstale":"2024-08-17T17:33:39.615400+0000","last_undegraded":"2024-08-17T17:33:39.615400+0000","last_fullsized":"2024-08-17T17:33:39.615400+0000","mapping_epoch":1318,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:48:13.291929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0,3],"avail_no_missing":["5","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1545,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.617265+0000","last_change":"2024-08-17T17:33:28.921346+0000","last_active":"2024-08-17T17:33:39.617265+0000","last_peered":"2024-08-17T17:33:39.617265+0000","last_clean":"2024-08-17T17:33:39.617265+0000","last_became_active":"2024-08-17T17:33:28.920705+0000","last_became_peered":"2024-08-17T17:33:28.920705+0000","last_unstale":"2024-08-17T17:33:39.617265+0000","last_undegraded":"2024-08-17T17:33:39.617265+0000","last_fullsized":"2024-08-17T17:33:39.617265+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:49.824465+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1682,"reported_epoch":1331,"state":"active+undersized","last_fresh":"2024-08-17T17:33:45.707718+0000","last_change":"2024-08-17T17:32:47.688690+0000","last_active":"2024-08-17T17:33:45.707718+0000","last_peered":"2024-08-17T17:33:45.707718+0000","last_clean":"2024-08-17T17:29:38.564651+0000","last_became_active":"2024-08-17T17:32:47.688690+0000","last_became_peered":"2024-08-17T17:32:47.688690+0000","last_unstale":"2024-08-17T17:33:45.707718+0000","last_undegraded":"2024-08-17T17:33:45.707718+0000","last_fullsized":"2024-08-17T17:32:47.584197+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:02:47.789049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1320,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.614305+0000","last_change":"2024-08-17T17:33:25.932066+0000","last_active":"2024-08-17T17:33:39.614305+0000","last_peered":"2024-08-17T17:33:39.614305+0000","last_clean":"2024-08-17T17:33:39.614305+0000","last_became_active":"2024-08-17T17:33:25.931851+0000","last_became_peered":"2024-08-17T17:33:25.931851+0000","last_unstale":"2024-08-17T17:33:39.614305+0000","last_undegraded":"2024-08-17T17:33:39.614305+0000","last_fullsized":"2024-08-17T17:33:39.614305+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:04:59.835356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1809,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.610812+0000","last_change":"2024-08-17T17:24:20.649977+0000","last_active":"2024-08-17T17:33:40.610812+0000","last_peered":"2024-08-17T17:33:40.610812+0000","last_clean":"2024-08-17T17:33:40.610812+0000","last_became_active":"2024-08-17T17:24:20.649336+0000","last_became_peered":"2024-08-17T17:24:20.649336+0000","last_unstale":"2024-08-17T17:33:40.610812+0000","last_undegraded":"2024-08-17T17:33:40.610812+0000","last_fullsized":"2024-08-17T17:33:40.610812+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:31:57.187271+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":61,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-08-17T17:33:45.708080+0000","last_change":"2024-08-17T17:32:47.693814+0000","last_active":"2024-08-17T17:33:45.708080+0000","last_peered":"2024-08-17T17:33:45.708080+0000","last_clean":"2024-08-17T17:33:45.708080+0000","last_became_active":"2024-08-17T17:32:47.693067+0000","last_became_peered":"2024-08-17T17:32:47.693067+0000","last_unstale":"2024-08-17T17:33:45.708080+0000","last_undegraded":"2024-08-17T17:33:45.708080+0000","last_fullsized":"2024-08-17T17:33:45.708080+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:16:32.934429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":920,"reported_epoch":1330,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:40.613010+0000","last_change":"2024-08-17T17:33:26.972019+0000","last_active":"2024-08-17T17:33:40.613010+0000","last_peered":"2024-08-17T17:33:40.613010+0000","last_clean":"2024-08-17T17:33:40.613010+0000","last_became_active":"2024-08-17T17:33:26.969237+0000","last_became_peered":"2024-08-17T17:33:26.969237+0000","last_unstale":"2024-08-17T17:33:40.613010+0000","last_undegraded":"2024-08-17T17:33:40.613010+0000","last_fullsized":"2024-08-17T17:33:40.613010+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:11:40.846511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1,0],"avail_no_missing":["2","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":770,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T17:33:39.614168+0000","last_change":"2024-08-17T17:29:53.529304+0000","last_active":"2024-08-17T17:33:39.614168+0000","last_peered":"2024-08-17T17:33:39.614168+0000","last_clean":"2024-08-17T17:29:38.565104+0000","last_became_active":"2024-08-17T17:29:53.529304+0000","last_became_peered":"2024-08-17T17:29:53.529304+0000","last_unstale":"2024-08-17T17:33:39.614168+0000","last_undegraded":"2024-08-17T17:33:39.614168+0000","last_fullsized":"2024-08-17T17:29:53.525070+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:04.056586+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1294,"reported_epoch":1329,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:33:39.612071+0000","last_change":"2024-08-17T17:29:53.531085+0000","last_active":"2024-08-17T17:33:39.612071+0000","last_peered":"2024-08-17T17:33:39.612071+0000","last_clean":"2024-08-17T17:29:38.551404+0000","last_became_active":"2024-08-17T17:29:53.531085+0000","last_became_peered":"2024-08-17T17:29:53.531085+0000","last_unstale":"2024-08-17T17:33:39.612071+0000","last_undegraded":"2024-08-17T17:29:53.527582+0000","last_fullsized":"2024-08-17T17:29:53.527482+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:15:01.176057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1323,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.612094+0000","last_change":"2024-08-17T17:33:26.967279+0000","last_active":"2024-08-17T17:33:39.612094+0000","last_peered":"2024-08-17T17:33:39.612094+0000","last_clean":"2024-08-17T17:33:39.612094+0000","last_became_active":"2024-08-17T17:33:26.964525+0000","last_became_peered":"2024-08-17T17:33:26.964525+0000","last_unstale":"2024-08-17T17:33:39.612094+0000","last_undegraded":"2024-08-17T17:33:39.612094+0000","last_fullsized":"2024-08-17T17:33:39.612094+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:19:24.671619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,5],"avail_no_missing":["4","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":857,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.612742+0000","last_change":"2024-08-17T17:33:28.933709+0000","last_active":"2024-08-17T17:33:39.612742+0000","last_peered":"2024-08-17T17:33:39.612742+0000","last_clean":"2024-08-17T17:33:39.612742+0000","last_became_active":"2024-08-17T17:33:28.933340+0000","last_became_peered":"2024-08-17T17:33:28.933340+0000","last_unstale":"2024-08-17T17:33:39.612742+0000","last_undegraded":"2024-08-17T17:33:39.612742+0000","last_fullsized":"2024-08-17T17:33:39.612742+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:23:38.510059+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1272,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T17:33:39.611582+0000","last_change":"2024-08-17T17:29:53.529700+0000","last_active":"2024-08-17T17:33:39.611582+0000","last_peered":"2024-08-17T17:33:39.611582+0000","last_clean":"2024-08-17T17:29:38.552083+0000","last_became_active":"2024-08-17T17:29:53.529700+0000","last_became_peered":"2024-08-17T17:29:53.529700+0000","last_unstale":"2024-08-17T17:33:39.611582+0000","last_undegraded":"2024-08-17T17:33:39.611582+0000","last_fullsized":"2024-08-17T17:29:53.524158+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:53:03.647039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":791,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T17:33:40.611771+0000","last_change":"2024-08-17T17:32:47.589237+0000","last_active":"2024-08-17T17:33:40.611771+0000","last_peered":"2024-08-17T17:33:40.611771+0000","last_clean":"2024-08-17T17:06:58.643061+0000","last_became_active":"2024-08-17T17:32:47.589237+0000","last_became_peered":"2024-08-17T17:32:47.589237+0000","last_unstale":"2024-08-17T17:33:40.611771+0000","last_undegraded":"2024-08-17T17:33:40.611771+0000","last_fullsized":"2024-08-17T17:32:47.587138+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:50:06.708475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":136,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.608384+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:33:40.608384+0000","last_peered":"2024-08-17T17:33:40.608384+0000","last_clean":"2024-08-17T17:33:40.608384+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:33:40.608384+0000","last_undegraded":"2024-08-17T17:33:40.608384+0000","last_fullsized":"2024-08-17T17:33:40.608384+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1407,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.613953+0000","last_change":"2024-08-17T17:33:15.826182+0000","last_active":"2024-08-17T17:33:39.613953+0000","last_peered":"2024-08-17T17:33:39.613953+0000","last_clean":"2024-08-17T17:33:39.613953+0000","last_became_active":"2024-08-17T17:33:15.824020+0000","last_became_peered":"2024-08-17T17:33:15.824020+0000","last_unstale":"2024-08-17T17:33:39.613953+0000","last_undegraded":"2024-08-17T17:33:39.613953+0000","last_fullsized":"2024-08-17T17:33:39.613953+0000","mapping_epoch":1318,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1319,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.033101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.033101+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":12,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7,3],"avail_no_missing":["5","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.f","version":"1331'14757","reported_seq":14854,"reported_epoch":1331,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:33:46.195753+0000","last_change":"2024-08-17T17:30:45.054716+0000","last_active":"2024-08-17T17:33:46.195753+0000","last_peered":"2024-08-17T17:33:46.195753+0000","last_clean":"2024-08-17T17:30:30.026678+0000","last_became_active":"2024-08-17T17:30:31.047056+0000","last_became_peered":"2024-08-17T17:30:31.047056+0000","last_unstale":"2024-08-17T17:33:46.195753+0000","last_undegraded":"2024-08-17T17:30:45.047432+0000","last_fullsized":"2024-08-17T17:30:31.042363+0000","mapping_epoch":1288,"log_start":"1307'8281","ondisk_log_start":"1307'8281","created":1288,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6476,"log_dups_size":0,"ondisk_log_size":6476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:33:58.197297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157614080,"num_objects":2405,"num_object_clones":0,"num_object_copies":7215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2405,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14757,"num_write_kb":549184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"2,3","objects":2405}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1456,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.613102+0000","last_change":"2024-08-17T17:33:26.901705+0000","last_active":"2024-08-17T17:33:39.613102+0000","last_peered":"2024-08-17T17:33:39.613102+0000","last_clean":"2024-08-17T17:33:39.613102+0000","last_became_active":"2024-08-17T17:33:26.901418+0000","last_became_peered":"2024-08-17T17:33:26.901418+0000","last_unstale":"2024-08-17T17:33:39.613102+0000","last_undegraded":"2024-08-17T17:33:39.613102+0000","last_fullsized":"2024-08-17T17:33:39.613102+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:27.165882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":849,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.612942+0000","last_change":"2024-08-17T17:33:26.966545+0000","last_active":"2024-08-17T17:33:39.612942+0000","last_peered":"2024-08-17T17:33:39.612942+0000","last_clean":"2024-08-17T17:33:39.612942+0000","last_became_active":"2024-08-17T17:33:26.965388+0000","last_became_peered":"2024-08-17T17:33:26.965388+0000","last_unstale":"2024-08-17T17:33:39.612942+0000","last_undegraded":"2024-08-17T17:33:39.612942+0000","last_fullsized":"2024-08-17T17:33:39.612942+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:51.795975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,3],"avail_no_missing":["4","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1435,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.607795+0000","last_change":"2024-08-17T17:32:47.697847+0000","last_active":"2024-08-17T17:33:40.607795+0000","last_peered":"2024-08-17T17:33:40.607795+0000","last_clean":"2024-08-17T17:33:40.607795+0000","last_became_active":"2024-08-17T17:32:47.695811+0000","last_became_peered":"2024-08-17T17:32:47.695811+0000","last_unstale":"2024-08-17T17:33:40.607795+0000","last_undegraded":"2024-08-17T17:33:40.607795+0000","last_fullsized":"2024-08-17T17:33:40.607795+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:28:15.895052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1650,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T17:33:39.612710+0000","last_change":"2024-08-17T17:29:53.529747+0000","last_active":"2024-08-17T17:33:39.612710+0000","last_peered":"2024-08-17T17:33:39.612710+0000","last_clean":"2024-08-17T17:29:38.553022+0000","last_became_active":"2024-08-17T17:29:53.529747+0000","last_became_peered":"2024-08-17T17:29:53.529747+0000","last_unstale":"2024-08-17T17:33:39.612710+0000","last_undegraded":"2024-08-17T17:33:39.612710+0000","last_fullsized":"2024-08-17T17:29:53.526039+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:45:41.319012+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.e","version":"1330'14512","reported_seq":14560,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:33:42.257057+0000","last_change":"2024-08-17T17:30:45.040437+0000","last_active":"2024-08-17T17:33:42.257057+0000","last_peered":"2024-08-17T17:33:42.257057+0000","last_clean":"2024-08-17T17:30:30.026678+0000","last_became_active":"2024-08-17T17:30:31.047154+0000","last_became_peered":"2024-08-17T17:30:31.047154+0000","last_unstale":"2024-08-17T17:33:42.257057+0000","last_undegraded":"2024-08-17T17:30:45.040259+0000","last_fullsized":"2024-08-17T17:30:31.043359+0000","mapping_epoch":1288,"log_start":"1310'9704","ondisk_log_start":"1310'9704","created":1288,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":4808,"log_dups_size":0,"ondisk_log_size":4808,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:43.577566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172883968,"num_objects":2638,"num_object_clones":0,"num_object_copies":7914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2638,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14512,"num_write_kb":548800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":2638}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":885,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.613647+0000","last_change":"2024-08-17T17:18:31.540502+0000","last_active":"2024-08-17T17:33:39.613647+0000","last_peered":"2024-08-17T17:33:39.613647+0000","last_clean":"2024-08-17T17:33:39.613647+0000","last_became_active":"2024-08-17T17:18:31.540165+0000","last_became_peered":"2024-08-17T17:18:31.540165+0000","last_unstale":"2024-08-17T17:33:39.613647+0000","last_undegraded":"2024-08-17T17:33:39.613647+0000","last_fullsized":"2024-08-17T17:33:39.613647+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:20:09.623551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1293,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.617398+0000","last_change":"2024-08-17T17:18:31.540222+0000","last_active":"2024-08-17T17:33:39.617398+0000","last_peered":"2024-08-17T17:33:39.617398+0000","last_clean":"2024-08-17T17:33:39.617398+0000","last_became_active":"2024-08-17T17:18:31.539585+0000","last_became_peered":"2024-08-17T17:18:31.539585+0000","last_unstale":"2024-08-17T17:33:39.617398+0000","last_undegraded":"2024-08-17T17:33:39.617398+0000","last_fullsized":"2024-08-17T17:33:39.617398+0000","mapping_epoch":1143,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:51:12.651046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1230,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.610915+0000","last_change":"2024-08-17T17:33:25.953838+0000","last_active":"2024-08-17T17:33:40.610915+0000","last_peered":"2024-08-17T17:33:40.610915+0000","last_clean":"2024-08-17T17:33:40.610915+0000","last_became_active":"2024-08-17T17:33:25.953470+0000","last_became_peered":"2024-08-17T17:33:25.953470+0000","last_unstale":"2024-08-17T17:33:40.610915+0000","last_undegraded":"2024-08-17T17:33:40.610915+0000","last_fullsized":"2024-08-17T17:33:40.610915+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:33:06.374801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":752,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.612360+0000","last_change":"2024-08-17T17:33:17.829266+0000","last_active":"2024-08-17T17:33:39.612360+0000","last_peered":"2024-08-17T17:33:39.612360+0000","last_clean":"2024-08-17T17:33:39.612360+0000","last_became_active":"2024-08-17T17:33:17.828516+0000","last_became_peered":"2024-08-17T17:33:17.828516+0000","last_unstale":"2024-08-17T17:33:39.612360+0000","last_undegraded":"2024-08-17T17:33:39.612360+0000","last_fullsized":"2024-08-17T17:33:39.612360+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.103133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.103133+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,3],"avail_no_missing":["4","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.d","version":"1330'14727","reported_seq":14777,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:33:42.277245+0000","last_change":"2024-08-17T17:30:45.038994+0000","last_active":"2024-08-17T17:33:42.277245+0000","last_peered":"2024-08-17T17:33:42.277245+0000","last_clean":"2024-08-17T17:30:30.026678+0000","last_became_active":"2024-08-17T17:30:31.046228+0000","last_became_peered":"2024-08-17T17:30:31.046228+0000","last_unstale":"2024-08-17T17:33:42.277245+0000","last_undegraded":"2024-08-17T17:30:45.038835+0000","last_fullsized":"2024-08-17T17:30:31.044006+0000","mapping_epoch":1288,"log_start":"1310'9925","ondisk_log_start":"1310'9925","created":1288,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":4802,"log_dups_size":0,"ondisk_log_size":4802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:28:39.199905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178257944,"num_objects":2721,"num_object_clones":0,"num_object_copies":8163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2721,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2721,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14727,"num_write_kb":558273,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":2721}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":943,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.613708+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:33:39.613708+0000","last_peered":"2024-08-17T17:33:39.613708+0000","last_clean":"2024-08-17T17:33:39.613708+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:33:39.613708+0000","last_undegraded":"2024-08-17T17:33:39.613708+0000","last_fullsized":"2024-08-17T17:33:39.613708+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.681497+0000","last_change":"2024-08-17T17:33:25.950484+0000","last_active":"2024-08-17T17:33:40.681497+0000","last_peered":"2024-08-17T17:33:40.681497+0000","last_clean":"2024-08-17T17:33:40.681497+0000","last_became_active":"2024-08-17T17:33:25.950200+0000","last_became_peered":"2024-08-17T17:33:25.950200+0000","last_unstale":"2024-08-17T17:33:40.681497+0000","last_undegraded":"2024-08-17T17:33:40.681497+0000","last_fullsized":"2024-08-17T17:33:40.681497+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:51:43.186014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1303,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.614524+0000","last_change":"2024-08-17T17:33:15.866959+0000","last_active":"2024-08-17T17:33:39.614524+0000","last_peered":"2024-08-17T17:33:39.614524+0000","last_clean":"2024-08-17T17:33:39.614524+0000","last_became_active":"2024-08-17T17:33:15.866527+0000","last_became_peered":"2024-08-17T17:33:15.866527+0000","last_unstale":"2024-08-17T17:33:39.614524+0000","last_undegraded":"2024-08-17T17:33:39.614524+0000","last_fullsized":"2024-08-17T17:33:39.614524+0000","mapping_epoch":1318,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:08:21.746930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2,3],"avail_no_missing":["5","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":701,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.613848+0000","last_change":"2024-08-17T17:33:17.827266+0000","last_active":"2024-08-17T17:33:39.613848+0000","last_peered":"2024-08-17T17:33:39.613848+0000","last_clean":"2024-08-17T17:33:39.613848+0000","last_became_active":"2024-08-17T17:33:17.826863+0000","last_became_peered":"2024-08-17T17:33:17.826863+0000","last_unstale":"2024-08-17T17:33:39.613848+0000","last_undegraded":"2024-08-17T17:33:39.613848+0000","last_fullsized":"2024-08-17T17:33:39.613848+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:55.041378+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:55.041378+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":4,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7,3],"avail_no_missing":["5","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.c","version":"1330'14353","reported_seq":14412,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:42.290069+0000","last_change":"2024-08-17T17:32:47.605709+0000","last_active":"2024-08-17T17:33:42.290069+0000","last_peered":"2024-08-17T17:33:42.290069+0000","last_clean":"2024-08-17T17:33:42.290069+0000","last_became_active":"2024-08-17T17:32:47.589849+0000","last_became_peered":"2024-08-17T17:32:47.589849+0000","last_unstale":"2024-08-17T17:33:42.290069+0000","last_undegraded":"2024-08-17T17:33:42.290069+0000","last_fullsized":"2024-08-17T17:33:42.290069+0000","mapping_epoch":1312,"log_start":"1308'9507","ondisk_log_start":"1308'9507","created":1288,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":4846,"log_dups_size":0,"ondisk_log_size":4846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:37.070162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171507712,"num_objects":2617,"num_object_clones":0,"num_object_copies":7851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14353,"num_write_kb":543040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1311,"reported_epoch":1330,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:40.611655+0000","last_change":"2024-08-17T17:33:26.964063+0000","last_active":"2024-08-17T17:33:40.611655+0000","last_peered":"2024-08-17T17:33:40.611655+0000","last_clean":"2024-08-17T17:33:40.611655+0000","last_became_active":"2024-08-17T17:33:26.963682+0000","last_became_peered":"2024-08-17T17:33:26.963682+0000","last_unstale":"2024-08-17T17:33:40.611655+0000","last_undegraded":"2024-08-17T17:33:40.611655+0000","last_fullsized":"2024-08-17T17:33:40.611655+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:54:19.032737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1,7],"avail_no_missing":["2","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1306,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T17:33:39.613206+0000","last_change":"2024-08-17T17:29:53.530759+0000","last_active":"2024-08-17T17:33:39.613206+0000","last_peered":"2024-08-17T17:33:39.613206+0000","last_clean":"2024-08-17T17:29:38.552381+0000","last_became_active":"2024-08-17T17:29:53.530759+0000","last_became_peered":"2024-08-17T17:29:53.530759+0000","last_unstale":"2024-08-17T17:33:39.613206+0000","last_undegraded":"2024-08-17T17:33:39.613206+0000","last_fullsized":"2024-08-17T17:29:53.525624+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:10:40.072138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1348,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.614430+0000","last_change":"2024-08-17T17:33:26.936290+0000","last_active":"2024-08-17T17:33:39.614430+0000","last_peered":"2024-08-17T17:33:39.614430+0000","last_clean":"2024-08-17T17:33:39.614430+0000","last_became_active":"2024-08-17T17:33:26.935774+0000","last_became_peered":"2024-08-17T17:33:26.935774+0000","last_unstale":"2024-08-17T17:33:39.614430+0000","last_undegraded":"2024-08-17T17:33:39.614430+0000","last_fullsized":"2024-08-17T17:33:39.614430+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:03:52.529138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1,0],"avail_no_missing":["5","0","1"],"object_location_counts":[{"shards":"0,1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1640,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.613322+0000","last_change":"2024-08-17T17:24:13.795227+0000","last_active":"2024-08-17T17:33:39.613322+0000","last_peered":"2024-08-17T17:33:39.613322+0000","last_clean":"2024-08-17T17:33:39.613322+0000","last_became_active":"2024-08-17T17:24:11.578352+0000","last_became_peered":"2024-08-17T17:24:11.578352+0000","last_unstale":"2024-08-17T17:33:39.613322+0000","last_undegraded":"2024-08-17T17:33:39.613322+0000","last_fullsized":"2024-08-17T17:33:39.613322+0000","mapping_epoch":1147,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:36:39.440808+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"1330'14303","reported_seq":22885,"reported_epoch":1330,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:33:42.315523+0000","last_change":"2024-08-17T17:33:27.133079+0000","last_active":"2024-08-17T17:33:42.315523+0000","last_peered":"2024-08-17T17:33:42.315523+0000","last_clean":"2024-08-17T17:30:30.026678+0000","last_became_active":"2024-08-17T17:31:35.854230+0000","last_became_peered":"2024-08-17T17:31:35.854230+0000","last_unstale":"2024-08-17T17:33:42.315523+0000","last_undegraded":"2024-08-17T17:31:35.707617+0000","last_fullsized":"2024-08-17T17:31:35.675247+0000","mapping_epoch":1299,"log_start":"1308'9124","ondisk_log_start":"1308'9124","created":1288,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":5179,"log_dups_size":0,"ondisk_log_size":5179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:10.484940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173473792,"num_objects":2647,"num_object_clones":0,"num_object_copies":7941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2647,"num_objects_misplaced":1630,"num_objects_unfound":0,"num_objects_dirty":2647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14303,"num_write_kb":542400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4258,"num_bytes_recovered":264568832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":1017}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":992,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.614436+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:33:39.614436+0000","last_peered":"2024-08-17T17:33:39.614436+0000","last_clean":"2024-08-17T17:33:39.614436+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:33:39.614436+0000","last_undegraded":"2024-08-17T17:33:39.614436+0000","last_fullsized":"2024-08-17T17:33:39.614436+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1234,"reported_epoch":1330,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:40.611066+0000","last_change":"2024-08-17T17:33:27.035430+0000","last_active":"2024-08-17T17:33:40.611066+0000","last_peered":"2024-08-17T17:33:40.611066+0000","last_clean":"2024-08-17T17:33:40.611066+0000","last_became_active":"2024-08-17T17:33:27.034349+0000","last_became_peered":"2024-08-17T17:33:27.034349+0000","last_unstale":"2024-08-17T17:33:40.611066+0000","last_undegraded":"2024-08-17T17:33:40.611066+0000","last_fullsized":"2024-08-17T17:33:40.611066+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:55:44.159308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1,0],"avail_no_missing":["2","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1837,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.610789+0000","last_change":"2024-08-17T17:33:26.962759+0000","last_active":"2024-08-17T17:33:40.610789+0000","last_peered":"2024-08-17T17:33:40.610789+0000","last_clean":"2024-08-17T17:33:40.610789+0000","last_became_active":"2024-08-17T17:33:26.962465+0000","last_became_peered":"2024-08-17T17:33:26.962465+0000","last_unstale":"2024-08-17T17:33:40.610789+0000","last_undegraded":"2024-08-17T17:33:40.610789+0000","last_fullsized":"2024-08-17T17:33:40.610789+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:58:50.157084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1460,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.612003+0000","last_change":"2024-08-17T17:32:47.592186+0000","last_active":"2024-08-17T17:33:39.612003+0000","last_peered":"2024-08-17T17:33:39.612003+0000","last_clean":"2024-08-17T17:33:39.612003+0000","last_became_active":"2024-08-17T17:32:47.591889+0000","last_became_peered":"2024-08-17T17:32:47.591889+0000","last_unstale":"2024-08-17T17:33:39.612003+0000","last_undegraded":"2024-08-17T17:33:39.612003+0000","last_fullsized":"2024-08-17T17:33:39.612003+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:12.519838+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:12.519838+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":18,"num_read_kb":38,"num_write":10,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":107340,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.9","version":"1330'11003","reported_seq":11064,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:42.286794+0000","last_change":"2024-08-17T17:32:07.889683+0000","last_active":"2024-08-17T17:33:42.286794+0000","last_peered":"2024-08-17T17:33:42.286794+0000","last_clean":"2024-08-17T17:33:42.286794+0000","last_became_active":"2024-08-17T17:32:07.885995+0000","last_became_peered":"2024-08-17T17:32:07.885995+0000","last_unstale":"2024-08-17T17:33:42.286794+0000","last_undegraded":"2024-08-17T17:33:42.286794+0000","last_fullsized":"2024-08-17T17:33:42.286794+0000","mapping_epoch":1288,"log_start":"1293'3721","ondisk_log_start":"1293'3721","created":1288,"last_epoch_clean":1307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":7282,"log_dups_size":0,"ondisk_log_size":7282,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91488256,"num_objects":1396,"num_object_clones":0,"num_object_copies":4188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7032,"num_write_kb":269696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":983,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.608605+0000","last_change":"2024-08-17T17:33:25.946767+0000","last_active":"2024-08-17T17:33:40.608605+0000","last_peered":"2024-08-17T17:33:40.608605+0000","last_clean":"2024-08-17T17:33:40.608605+0000","last_became_active":"2024-08-17T17:33:25.946410+0000","last_became_peered":"2024-08-17T17:33:25.946410+0000","last_unstale":"2024-08-17T17:33:40.608605+0000","last_undegraded":"2024-08-17T17:33:40.608605+0000","last_fullsized":"2024-08-17T17:33:40.608605+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:12:49.063222+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":954,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.613131+0000","last_change":"2024-08-17T17:33:14.823355+0000","last_active":"2024-08-17T17:33:39.613131+0000","last_peered":"2024-08-17T17:33:39.613131+0000","last_clean":"2024-08-17T17:33:39.613131+0000","last_became_active":"2024-08-17T17:33:14.822746+0000","last_became_peered":"2024-08-17T17:33:14.822746+0000","last_unstale":"2024-08-17T17:33:39.613131+0000","last_undegraded":"2024-08-17T17:33:39.613131+0000","last_fullsized":"2024-08-17T17:33:39.613131+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:08:12.120599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1328,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.607538+0000","last_change":"2024-08-17T17:33:25.943112+0000","last_active":"2024-08-17T17:33:40.607538+0000","last_peered":"2024-08-17T17:33:40.607538+0000","last_clean":"2024-08-17T17:33:40.607538+0000","last_became_active":"2024-08-17T17:33:25.942528+0000","last_became_peered":"2024-08-17T17:33:25.942528+0000","last_unstale":"2024-08-17T17:33:40.607538+0000","last_undegraded":"2024-08-17T17:33:40.607538+0000","last_fullsized":"2024-08-17T17:33:40.607538+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:05:07.892214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":739,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.611626+0000","last_change":"2024-08-17T17:33:26.967670+0000","last_active":"2024-08-17T17:33:39.611626+0000","last_peered":"2024-08-17T17:33:39.611626+0000","last_clean":"2024-08-17T17:33:39.611626+0000","last_became_active":"2024-08-17T17:33:26.966800+0000","last_became_peered":"2024-08-17T17:33:26.966800+0000","last_unstale":"2024-08-17T17:33:39.611626+0000","last_undegraded":"2024-08-17T17:33:39.611626+0000","last_fullsized":"2024-08-17T17:33:39.611626+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:20.754326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:20.754326+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:06:34.619107+0000","scrub_duration":0.0023303870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,5],"avail_no_missing":["4","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.8","version":"1330'11236","reported_seq":11390,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:42.636341+0000","last_change":"2024-08-17T17:32:10.911297+0000","last_active":"2024-08-17T17:33:42.636341+0000","last_peered":"2024-08-17T17:33:42.636341+0000","last_clean":"2024-08-17T17:33:42.636341+0000","last_became_active":"2024-08-17T17:32:07.897878+0000","last_became_peered":"2024-08-17T17:32:07.897878+0000","last_unstale":"2024-08-17T17:33:42.636341+0000","last_undegraded":"2024-08-17T17:33:42.636341+0000","last_fullsized":"2024-08-17T17:33:42.636341+0000","mapping_epoch":1288,"log_start":"1292'1792","ondisk_log_start":"1292'1792","created":1288,"last_epoch_clean":1307,"parent":"0.0","parent_split_bits":0,"last_scrub":"1307'8157","last_scrub_stamp":"2024-08-17T17:32:10.910607+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:32:10.910607+0000","objects_scrubbed":4139,"log_size":9444,"log_dups_size":0,"ondisk_log_size":9444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:24:09.503398+0000","scrub_duration":1272,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87097344,"num_objects":1329,"num_object_clones":0,"num_object_copies":3987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3106,"num_write_kb":9472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":65,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T17:33:39.612480+0000","last_change":"2024-08-17T17:29:53.530094+0000","last_active":"2024-08-17T17:33:39.612480+0000","last_peered":"2024-08-17T17:33:39.612480+0000","last_clean":"2024-08-17T16:53:36.856414+0000","last_became_active":"2024-08-17T17:29:53.530094+0000","last_became_peered":"2024-08-17T17:29:53.530094+0000","last_unstale":"2024-08-17T17:33:39.612480+0000","last_undegraded":"2024-08-17T17:33:39.612480+0000","last_fullsized":"2024-08-17T17:29:53.525569+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.793336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1677,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.613949+0000","last_change":"2024-08-17T17:33:15.849568+0000","last_active":"2024-08-17T17:33:39.613949+0000","last_peered":"2024-08-17T17:33:39.613949+0000","last_clean":"2024-08-17T17:33:39.613949+0000","last_became_active":"2024-08-17T17:33:15.849213+0000","last_became_peered":"2024-08-17T17:33:15.849213+0000","last_unstale":"2024-08-17T17:33:39.613949+0000","last_undegraded":"2024-08-17T17:33:39.613949+0000","last_fullsized":"2024-08-17T17:33:39.613949+0000","mapping_epoch":1318,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:32:48.644431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,3],"avail_no_missing":["5","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1291,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.613796+0000","last_change":"2024-08-17T17:07:12.645556+0000","last_active":"2024-08-17T17:33:39.613796+0000","last_peered":"2024-08-17T17:33:39.613796+0000","last_clean":"2024-08-17T17:33:39.613796+0000","last_became_active":"2024-08-17T17:07:12.645205+0000","last_became_peered":"2024-08-17T17:07:12.645205+0000","last_unstale":"2024-08-17T17:33:39.613796+0000","last_undegraded":"2024-08-17T17:33:39.613796+0000","last_fullsized":"2024-08-17T17:33:39.613796+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:33:09.741032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1492,"reported_epoch":1330,"state":"active+undersized","last_fresh":"2024-08-17T17:33:40.608111+0000","last_change":"2024-08-17T17:32:47.587161+0000","last_active":"2024-08-17T17:33:40.608111+0000","last_peered":"2024-08-17T17:33:40.608111+0000","last_clean":"2024-08-17T17:18:28.512669+0000","last_became_active":"2024-08-17T17:32:47.587161+0000","last_became_peered":"2024-08-17T17:32:47.587161+0000","last_unstale":"2024-08-17T17:33:40.608111+0000","last_undegraded":"2024-08-17T17:33:40.608111+0000","last_fullsized":"2024-08-17T17:32:47.582453+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1215,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:33:42.741706+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.b","version":"1330'14590","reported_seq":17148,"reported_epoch":1330,"state":"active+undersized+degraded+remapped+backfilling","last_fresh":"2024-08-17T17:33:42.281362+0000","last_change":"2024-08-17T17:33:37.182288+0000","last_active":"2024-08-17T17:33:42.281362+0000","last_peered":"2024-08-17T17:33:42.281362+0000","last_clean":"2024-08-17T17:30:30.026678+0000","last_became_active":"2024-08-17T17:33:27.041515+0000","last_became_peered":"2024-08-17T17:33:27.041515+0000","last_unstale":"2024-08-17T17:33:42.281362+0000","last_undegraded":"2024-08-17T17:33:26.920494+0000","last_fullsized":"2024-08-17T17:33:26.902470+0000","mapping_epoch":1323,"log_start":"1310'9767","ondisk_log_start":"1310'9767","created":1288,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":4823,"log_dups_size":0,"ondisk_log_size":4823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:04.330648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171048960,"num_objects":2610,"num_object_clones":0,"num_object_copies":7830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1417,"num_objects_misplaced":2610,"num_objects_unfound":0,"num_objects_dirty":2610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14590,"num_write_kb":550400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1250,"num_bytes_recovered":81854464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":2610}],"blocked_by":[],"up_primary":1,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":929,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.612618+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:33:39.612618+0000","last_peered":"2024-08-17T17:33:39.612618+0000","last_clean":"2024-08-17T17:33:39.612618+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:33:39.612618+0000","last_undegraded":"2024-08-17T17:33:39.612618+0000","last_fullsized":"2024-08-17T17:33:39.612618+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1689,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.616109+0000","last_change":"2024-08-17T17:33:25.945385+0000","last_active":"2024-08-17T17:33:39.616109+0000","last_peered":"2024-08-17T17:33:39.616109+0000","last_clean":"2024-08-17T17:33:39.616109+0000","last_became_active":"2024-08-17T17:33:25.945132+0000","last_became_peered":"2024-08-17T17:33:25.945132+0000","last_unstale":"2024-08-17T17:33:39.616109+0000","last_undegraded":"2024-08-17T17:33:39.616109+0000","last_fullsized":"2024-08-17T17:33:39.616109+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:45.745740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1628,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T17:33:39.613414+0000","last_change":"2024-08-17T17:29:53.533887+0000","last_active":"2024-08-17T17:33:39.613414+0000","last_peered":"2024-08-17T17:33:39.613414+0000","last_clean":"2024-08-17T17:29:38.563668+0000","last_became_active":"2024-08-17T17:29:53.533887+0000","last_became_peered":"2024-08-17T17:29:53.533887+0000","last_unstale":"2024-08-17T17:33:39.613414+0000","last_undegraded":"2024-08-17T17:33:39.613414+0000","last_fullsized":"2024-08-17T17:29:53.527142+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:59:52.156417+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.1","version":"1330'11228","reported_seq":11289,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:42.268618+0000","last_change":"2024-08-17T17:32:07.909748+0000","last_active":"2024-08-17T17:33:42.268618+0000","last_peered":"2024-08-17T17:33:42.268618+0000","last_clean":"2024-08-17T17:33:42.268618+0000","last_became_active":"2024-08-17T17:32:07.907907+0000","last_became_peered":"2024-08-17T17:32:07.907907+0000","last_unstale":"2024-08-17T17:33:42.268618+0000","last_undegraded":"2024-08-17T17:33:42.268618+0000","last_fullsized":"2024-08-17T17:33:42.268618+0000","mapping_epoch":1288,"log_start":"1296'4895","ondisk_log_start":"1296'4895","created":1288,"last_epoch_clean":1307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6333,"log_dups_size":0,"ondisk_log_size":6333,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92438528,"num_objects":1410,"num_object_clones":0,"num_object_copies":4230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7196,"num_write_kb":275424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1333,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.612789+0000","last_change":"2024-08-17T17:33:26.966212+0000","last_active":"2024-08-17T17:33:39.612789+0000","last_peered":"2024-08-17T17:33:39.612789+0000","last_clean":"2024-08-17T17:33:39.612789+0000","last_became_active":"2024-08-17T17:33:26.965142+0000","last_became_peered":"2024-08-17T17:33:26.965142+0000","last_unstale":"2024-08-17T17:33:39.612789+0000","last_undegraded":"2024-08-17T17:33:39.612789+0000","last_fullsized":"2024-08-17T17:33:39.612789+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:37:42.965448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,5],"avail_no_missing":["4","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1281,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.612001+0000","last_change":"2024-08-17T17:07:12.651442+0000","last_active":"2024-08-17T17:33:39.612001+0000","last_peered":"2024-08-17T17:33:39.612001+0000","last_clean":"2024-08-17T17:33:39.612001+0000","last_became_active":"2024-08-17T17:07:12.651112+0000","last_became_peered":"2024-08-17T17:07:12.651112+0000","last_unstale":"2024-08-17T17:33:39.612001+0000","last_undegraded":"2024-08-17T17:33:39.612001+0000","last_fullsized":"2024-08-17T17:33:39.612001+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:28.587036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":785,"reported_epoch":1330,"state":"active+clean+scrubbing","last_fresh":"2024-08-17T17:33:40.608621+0000","last_change":"2024-08-17T17:33:40.871086+0000","last_active":"2024-08-17T17:33:40.608621+0000","last_peered":"2024-08-17T17:33:40.608621+0000","last_clean":"2024-08-17T17:33:40.608621+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:33:40.608621+0000","last_undegraded":"2024-08-17T17:33:40.608621+0000","last_fullsized":"2024-08-17T17:33:40.608621+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:57.682409+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:57.682409+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":6,"num_read_kb":12,"num_write":3,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.0","version":"1330'11226","reported_seq":11292,"reported_epoch":1330,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T17:33:42.277787+0000","last_change":"2024-08-17T17:33:27.126169+0000","last_active":"2024-08-17T17:33:42.277787+0000","last_peered":"2024-08-17T17:33:42.277787+0000","last_clean":"2024-08-17T17:30:30.026678+0000","last_became_active":"2024-08-17T17:33:27.034254+0000","last_became_peered":"2024-08-17T17:33:27.034254+0000","last_unstale":"2024-08-17T17:33:42.277787+0000","last_undegraded":"2024-08-17T17:33:26.916781+0000","last_fullsized":"2024-08-17T17:33:26.898679+0000","mapping_epoch":1323,"log_start":"1296'4865","ondisk_log_start":"1296'4865","created":1288,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6361,"log_dups_size":0,"ondisk_log_size":6361,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85622784,"num_objects":1306,"num_object_clones":0,"num_object_copies":3918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1306,"num_objects_misplaced":1306,"num_objects_unfound":0,"num_objects_dirty":1306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7216,"num_write_kb":272736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":1306}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1341,"reported_epoch":1330,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:40.612551+0000","last_change":"2024-08-17T17:33:26.963460+0000","last_active":"2024-08-17T17:33:40.612551+0000","last_peered":"2024-08-17T17:33:40.612551+0000","last_clean":"2024-08-17T17:33:40.612551+0000","last_became_active":"2024-08-17T17:33:26.963074+0000","last_became_peered":"2024-08-17T17:33:26.963074+0000","last_unstale":"2024-08-17T17:33:40.612551+0000","last_undegraded":"2024-08-17T17:33:40.612551+0000","last_fullsized":"2024-08-17T17:33:40.612551+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:24:53.228278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1,0],"avail_no_missing":["2","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1210,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.612132+0000","last_change":"2024-08-17T17:33:26.967352+0000","last_active":"2024-08-17T17:33:39.612132+0000","last_peered":"2024-08-17T17:33:39.612132+0000","last_clean":"2024-08-17T17:33:39.612132+0000","last_became_active":"2024-08-17T17:33:26.966838+0000","last_became_peered":"2024-08-17T17:33:26.966838+0000","last_unstale":"2024-08-17T17:33:39.612132+0000","last_undegraded":"2024-08-17T17:33:39.612132+0000","last_fullsized":"2024-08-17T17:33:39.612132+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:54:55.824323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,3],"avail_no_missing":["4","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1633,"reported_epoch":1330,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:40.607192+0000","last_change":"2024-08-17T17:33:15.825629+0000","last_active":"2024-08-17T17:33:40.607192+0000","last_peered":"2024-08-17T17:33:40.607192+0000","last_clean":"2024-08-17T17:33:40.607192+0000","last_became_active":"2024-08-17T17:33:15.825173+0000","last_became_peered":"2024-08-17T17:33:15.825173+0000","last_unstale":"2024-08-17T17:33:40.607192+0000","last_undegraded":"2024-08-17T17:33:40.607192+0000","last_fullsized":"2024-08-17T17:33:40.607192+0000","mapping_epoch":1318,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1319,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:20.323218+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2,3],"avail_no_missing":["5","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.3","version":"1330'11335","reported_seq":14708,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:42.674985+0000","last_change":"2024-08-17T17:33:38.897329+0000","last_active":"2024-08-17T17:33:42.674985+0000","last_peered":"2024-08-17T17:33:42.674985+0000","last_clean":"2024-08-17T17:33:42.674985+0000","last_became_active":"2024-08-17T17:33:38.609800+0000","last_became_peered":"2024-08-17T17:33:38.609800+0000","last_unstale":"2024-08-17T17:33:42.674985+0000","last_undegraded":"2024-08-17T17:33:42.674985+0000","last_fullsized":"2024-08-17T17:33:42.674985+0000","mapping_epoch":1327,"log_start":"1292'2027","ondisk_log_start":"1292'2027","created":1288,"last_epoch_clean":1328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":9308,"log_dups_size":0,"ondisk_log_size":9308,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:33:43.897027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80805888,"num_objects":1233,"num_object_clones":0,"num_object_copies":3699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7325,"num_write_kb":273856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1620,"num_bytes_recovered":106168320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1285,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T17:33:39.611903+0000","last_change":"2024-08-17T17:29:53.537940+0000","last_active":"2024-08-17T17:33:39.611903+0000","last_peered":"2024-08-17T17:33:39.611903+0000","last_clean":"2024-08-17T17:29:42.740989+0000","last_became_active":"2024-08-17T17:29:53.537940+0000","last_became_peered":"2024-08-17T17:29:53.537940+0000","last_unstale":"2024-08-17T17:33:39.611903+0000","last_undegraded":"2024-08-17T17:33:39.611903+0000","last_fullsized":"2024-08-17T17:29:53.527066+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":838,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:20:49.456097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":95,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.611914+0000","last_change":"2024-08-17T17:33:25.965486+0000","last_active":"2024-08-17T17:33:39.611914+0000","last_peered":"2024-08-17T17:33:39.611914+0000","last_clean":"2024-08-17T17:33:39.611914+0000","last_became_active":"2024-08-17T17:33:25.947710+0000","last_became_peered":"2024-08-17T17:33:25.947710+0000","last_unstale":"2024-08-17T17:33:39.611914+0000","last_undegraded":"2024-08-17T17:33:39.611914+0000","last_fullsized":"2024-08-17T17:33:39.611914+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:38:15.394927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1496,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.610500+0000","last_change":"2024-08-17T17:33:17.830896+0000","last_active":"2024-08-17T17:33:40.610500+0000","last_peered":"2024-08-17T17:33:40.610500+0000","last_clean":"2024-08-17T17:33:40.610500+0000","last_became_active":"2024-08-17T17:33:17.830347+0000","last_became_peered":"2024-08-17T17:33:17.830347+0000","last_unstale":"2024-08-17T17:33:40.610500+0000","last_undegraded":"2024-08-17T17:33:40.610500+0000","last_fullsized":"2024-08-17T17:33:40.610500+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:51:43.814393+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.2","version":"1330'11365","reported_seq":11450,"reported_epoch":1330,"state":"active+remapped+backfill_wait","last_fresh":"2024-08-17T17:33:42.282810+0000","last_change":"2024-08-17T17:33:27.123051+0000","last_active":"2024-08-17T17:33:42.282810+0000","last_peered":"2024-08-17T17:33:42.282810+0000","last_clean":"2024-08-17T17:33:24.882495+0000","last_became_active":"2024-08-17T17:33:26.001332+0000","last_became_peered":"2024-08-17T17:33:26.001332+0000","last_unstale":"2024-08-17T17:33:42.282810+0000","last_undegraded":"2024-08-17T17:33:42.282810+0000","last_fullsized":"2024-08-17T17:33:42.282810+0000","mapping_epoch":1322,"log_start":"1296'5037","ondisk_log_start":"1296'5037","created":1288,"last_epoch_clean":1319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6328,"log_dups_size":0,"ondisk_log_size":6328,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90112000,"num_objects":1375,"num_object_clones":0,"num_object_copies":4125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1375,"num_objects_unfound":0,"num_objects_dirty":1375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7315,"num_write_kb":278080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,0,3],"avail_no_missing":["4","0","3"],"object_location_counts":[{"shards":"0,3,4","objects":1375}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1073,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.608889+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:33:40.608889+0000","last_peered":"2024-08-17T17:33:40.608889+0000","last_clean":"2024-08-17T17:33:40.608889+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:33:40.608889+0000","last_undegraded":"2024-08-17T17:33:40.608889+0000","last_fullsized":"2024-08-17T17:33:40.608889+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1302,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.614173+0000","last_change":"2024-08-17T17:33:26.935893+0000","last_active":"2024-08-17T17:33:39.614173+0000","last_peered":"2024-08-17T17:33:39.614173+0000","last_clean":"2024-08-17T17:33:39.614173+0000","last_became_active":"2024-08-17T17:33:26.935392+0000","last_became_peered":"2024-08-17T17:33:26.935392+0000","last_unstale":"2024-08-17T17:33:39.614173+0000","last_undegraded":"2024-08-17T17:33:39.614173+0000","last_fullsized":"2024-08-17T17:33:39.614173+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:14:23.278076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1,0],"avail_no_missing":["5","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":806,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.612588+0000","last_change":"2024-08-17T17:24:52.102415+0000","last_active":"2024-08-17T17:33:39.612588+0000","last_peered":"2024-08-17T17:33:39.612588+0000","last_clean":"2024-08-17T17:33:39.612588+0000","last_became_active":"2024-08-17T17:24:31.795843+0000","last_became_peered":"2024-08-17T17:24:31.795843+0000","last_unstale":"2024-08-17T17:33:39.612588+0000","last_undegraded":"2024-08-17T17:33:39.612588+0000","last_fullsized":"2024-08-17T17:33:39.612588+0000","mapping_epoch":1220,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1224,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:32:08.434197+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.5","version":"1330'14551","reported_seq":16816,"reported_epoch":1330,"state":"active+remapped+backfilling","last_fresh":"2024-08-17T17:33:42.691455+0000","last_change":"2024-08-17T17:33:38.626473+0000","last_active":"2024-08-17T17:33:42.691455+0000","last_peered":"2024-08-17T17:33:42.691455+0000","last_clean":"2024-08-17T17:33:13.815261+0000","last_became_active":"2024-08-17T17:33:16.054283+0000","last_became_peered":"2024-08-17T17:33:16.054283+0000","last_unstale":"2024-08-17T17:33:42.691455+0000","last_undegraded":"2024-08-17T17:33:42.691455+0000","last_fullsized":"2024-08-17T17:33:42.691455+0000","mapping_epoch":1318,"log_start":"1307'8188","ondisk_log_start":"1307'8188","created":1288,"last_epoch_clean":1307,"parent":"0.0","parent_split_bits":0,"last_scrub":"1310'12132","last_scrub_stamp":"2024-08-17T17:32:32.155899+0000","last_deep_scrub":"1310'12132","last_deep_scrub_stamp":"2024-08-17T17:32:32.155899+0000","last_clean_scrub_stamp":"2024-08-17T17:32:32.155899+0000","objects_scrubbed":4054,"log_size":6363,"log_dups_size":0,"ondisk_log_size":6363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":22,"scrub_schedule":"no scrub is scheduled","scrub_duration":21478,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84082688,"num_objects":1283,"num_object_clones":0,"num_object_copies":3849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":346,"num_objects_unfound":0,"num_objects_dirty":1283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2801,"num_write_kb":960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":989,"num_bytes_recovered":64815104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,3],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":1283}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1166,"reported_epoch":1330,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:40.681219+0000","last_change":"2024-08-17T17:33:26.935752+0000","last_active":"2024-08-17T17:33:40.681219+0000","last_peered":"2024-08-17T17:33:40.681219+0000","last_clean":"2024-08-17T17:33:40.681219+0000","last_became_active":"2024-08-17T17:33:26.935306+0000","last_became_peered":"2024-08-17T17:33:26.935306+0000","last_unstale":"2024-08-17T17:33:40.681219+0000","last_undegraded":"2024-08-17T17:33:40.681219+0000","last_fullsized":"2024-08-17T17:33:40.681219+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:51.839882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7,5],"avail_no_missing":["1","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":38,"reported_epoch":1329,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:33:39.614175+0000","last_change":"2024-08-17T17:32:47.591039+0000","last_active":"2024-08-17T17:33:39.614175+0000","last_peered":"2024-08-17T17:33:39.614175+0000","last_clean":"2024-08-17T16:53:40.260059+0000","last_became_active":"2024-08-17T17:32:47.591039+0000","last_became_peered":"2024-08-17T17:32:47.591039+0000","last_unstale":"2024-08-17T17:33:39.614175+0000","last_undegraded":"2024-08-17T17:32:47.585493+0000","last_fullsized":"2024-08-17T17:32:47.585390+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:02:23.899625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":820,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T17:33:39.614523+0000","last_change":"2024-08-17T17:29:53.531472+0000","last_active":"2024-08-17T17:33:39.614523+0000","last_peered":"2024-08-17T17:33:39.614523+0000","last_clean":"2024-08-17T17:24:12.583154+0000","last_became_active":"2024-08-17T17:29:53.531472+0000","last_became_peered":"2024-08-17T17:29:53.531472+0000","last_unstale":"2024-08-17T17:33:39.614523+0000","last_undegraded":"2024-08-17T17:33:39.614523+0000","last_fullsized":"2024-08-17T17:29:53.527758+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.4","version":"1330'11423","reported_seq":11485,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:42.176867+0000","last_change":"2024-08-17T17:32:07.911726+0000","last_active":"2024-08-17T17:33:42.176867+0000","last_peered":"2024-08-17T17:33:42.176867+0000","last_clean":"2024-08-17T17:33:42.176867+0000","last_became_active":"2024-08-17T17:32:07.907756+0000","last_became_peered":"2024-08-17T17:32:07.907756+0000","last_unstale":"2024-08-17T17:33:42.176867+0000","last_undegraded":"2024-08-17T17:33:42.176867+0000","last_fullsized":"2024-08-17T17:33:42.176867+0000","mapping_epoch":1288,"log_start":"1297'5175","ondisk_log_start":"1297'5175","created":1288,"last_epoch_clean":1307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":6248,"log_dups_size":0,"ondisk_log_size":6248,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88768512,"num_objects":1354,"num_object_clones":0,"num_object_copies":4062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7330,"num_write_kb":277920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1343,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.612113+0000","last_change":"2024-08-17T17:33:26.966387+0000","last_active":"2024-08-17T17:33:39.612113+0000","last_peered":"2024-08-17T17:33:39.612113+0000","last_clean":"2024-08-17T17:33:39.612113+0000","last_became_active":"2024-08-17T17:33:26.964496+0000","last_became_peered":"2024-08-17T17:33:26.964496+0000","last_unstale":"2024-08-17T17:33:39.612113+0000","last_undegraded":"2024-08-17T17:33:39.612113+0000","last_fullsized":"2024-08-17T17:33:39.612113+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:09:33.441628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,0],"avail_no_missing":["4","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":54,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.612826+0000","last_change":"2024-08-17T17:33:25.963275+0000","last_active":"2024-08-17T17:33:39.612826+0000","last_peered":"2024-08-17T17:33:39.612826+0000","last_clean":"2024-08-17T17:33:39.612826+0000","last_became_active":"2024-08-17T17:33:25.947299+0000","last_became_peered":"2024-08-17T17:33:25.947299+0000","last_unstale":"2024-08-17T17:33:39.612826+0000","last_undegraded":"2024-08-17T17:33:39.612826+0000","last_fullsized":"2024-08-17T17:33:39.612826+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:36:34.476397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1597,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.611584+0000","last_change":"2024-08-17T17:33:17.829060+0000","last_active":"2024-08-17T17:33:39.611584+0000","last_peered":"2024-08-17T17:33:39.611584+0000","last_clean":"2024-08-17T17:33:39.611584+0000","last_became_active":"2024-08-17T17:33:17.828558+0000","last_became_peered":"2024-08-17T17:33:17.828558+0000","last_unstale":"2024-08-17T17:33:39.611584+0000","last_undegraded":"2024-08-17T17:33:39.611584+0000","last_fullsized":"2024-08-17T17:33:39.611584+0000","mapping_epoch":1320,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1321,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:45.985596+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:45.985596+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,3],"avail_no_missing":["4","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.7","version":"1330'11138","reported_seq":11204,"reported_epoch":1330,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2024-08-17T17:33:42.254826+0000","last_change":"2024-08-17T17:33:25.972604+0000","last_active":"2024-08-17T17:33:42.254826+0000","last_peered":"2024-08-17T17:33:42.254826+0000","last_clean":"2024-08-17T17:30:30.026678+0000","last_became_active":"2024-08-17T17:33:15.927496+0000","last_became_peered":"2024-08-17T17:33:15.927496+0000","last_unstale":"2024-08-17T17:33:42.254826+0000","last_undegraded":"2024-08-17T17:33:15.838505+0000","last_fullsized":"2024-08-17T17:33:15.808983+0000","mapping_epoch":1318,"log_start":"1294'4003","ondisk_log_start":"1294'4003","created":1288,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":7135,"log_dups_size":0,"ondisk_log_size":7135,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86048768,"num_objects":1313,"num_object_clones":0,"num_object_copies":3939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1313,"num_objects_misplaced":1313,"num_objects_unfound":0,"num_objects_dirty":1313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7157,"num_write_kb":271040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":1313}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1267,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T17:33:39.613584+0000","last_change":"2024-08-17T17:29:53.529490+0000","last_active":"2024-08-17T17:33:39.613584+0000","last_peered":"2024-08-17T17:33:39.613584+0000","last_clean":"2024-08-17T17:29:38.563813+0000","last_became_active":"2024-08-17T17:29:53.529490+0000","last_became_peered":"2024-08-17T17:29:53.529490+0000","last_unstale":"2024-08-17T17:33:39.613584+0000","last_undegraded":"2024-08-17T17:33:39.613584+0000","last_fullsized":"2024-08-17T17:29:53.525273+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:50:35.412627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":41,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.612655+0000","last_change":"2024-08-17T17:32:47.590429+0000","last_active":"2024-08-17T17:33:39.612655+0000","last_peered":"2024-08-17T17:33:39.612655+0000","last_clean":"2024-08-17T17:33:39.612655+0000","last_became_active":"2024-08-17T17:32:47.590098+0000","last_became_peered":"2024-08-17T17:32:47.590098+0000","last_unstale":"2024-08-17T17:33:39.612655+0000","last_undegraded":"2024-08-17T17:33:39.612655+0000","last_fullsized":"2024-08-17T17:33:39.612655+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:16:52.106213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1549,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.613048+0000","last_change":"2024-08-17T17:33:28.933742+0000","last_active":"2024-08-17T17:33:39.613048+0000","last_peered":"2024-08-17T17:33:39.613048+0000","last_clean":"2024-08-17T17:33:39.613048+0000","last_became_active":"2024-08-17T17:33:28.932774+0000","last_became_peered":"2024-08-17T17:33:28.932774+0000","last_unstale":"2024-08-17T17:33:39.613048+0000","last_undegraded":"2024-08-17T17:33:39.613048+0000","last_fullsized":"2024-08-17T17:33:39.613048+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.410491+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.410491+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.115365091,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.6","version":"1330'11359","reported_seq":11418,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:42.306377+0000","last_change":"2024-08-17T17:32:07.884875+0000","last_active":"2024-08-17T17:33:42.306377+0000","last_peered":"2024-08-17T17:33:42.306377+0000","last_clean":"2024-08-17T17:33:42.306377+0000","last_became_active":"2024-08-17T17:32:07.881371+0000","last_became_peered":"2024-08-17T17:32:07.881371+0000","last_unstale":"2024-08-17T17:33:42.306377+0000","last_undegraded":"2024-08-17T17:33:42.306377+0000","last_fullsized":"2024-08-17T17:33:42.306377+0000","mapping_epoch":1288,"log_start":"1294'4217","ondisk_log_start":"1294'4217","created":1288,"last_epoch_clean":1307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":7142,"log_dups_size":0,"ondisk_log_size":7142,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89358336,"num_objects":1363,"num_object_clones":0,"num_object_copies":4089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7299,"num_write_kb":277216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.19","version":"1330'11167","reported_seq":11227,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:42.309386+0000","last_change":"2024-08-17T17:32:07.917364+0000","last_active":"2024-08-17T17:33:42.309386+0000","last_peered":"2024-08-17T17:33:42.309386+0000","last_clean":"2024-08-17T17:33:42.309386+0000","last_became_active":"2024-08-17T17:32:07.913540+0000","last_became_peered":"2024-08-17T17:32:07.913540+0000","last_unstale":"2024-08-17T17:33:42.309386+0000","last_undegraded":"2024-08-17T17:33:42.309386+0000","last_fullsized":"2024-08-17T17:33:42.309386+0000","mapping_epoch":1306,"log_start":"1294'4020","ondisk_log_start":"1294'4020","created":1306,"last_epoch_clean":1307,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:30:30.026678+0000","objects_scrubbed":0,"log_size":7147,"log_dups_size":0,"ondisk_log_size":7147,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87556096,"num_objects":1336,"num_object_clones":0,"num_object_copies":4008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7196,"num_write_kb":273024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1161,"reported_epoch":1330,"state":"active+undersized+degraded","last_fresh":"2024-08-17T17:33:40.608289+0000","last_change":"2024-08-17T17:32:32.030588+0000","last_active":"2024-08-17T17:33:40.608289+0000","last_peered":"2024-08-17T17:33:40.608289+0000","last_clean":"2024-08-17T17:18:28.516042+0000","last_became_active":"2024-08-17T17:32:32.030588+0000","last_became_peered":"2024-08-17T17:32:32.030588+0000","last_unstale":"2024-08-17T17:33:40.608289+0000","last_undegraded":"2024-08-17T17:32:32.020552+0000","last_fullsized":"2024-08-17T17:32:32.020421+0000","mapping_epoch":1309,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:19:55.892444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1695,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.611512+0000","last_change":"2024-08-17T17:33:26.962690+0000","last_active":"2024-08-17T17:33:40.611512+0000","last_peered":"2024-08-17T17:33:40.611512+0000","last_clean":"2024-08-17T17:33:40.611512+0000","last_became_active":"2024-08-17T17:33:26.962351+0000","last_became_peered":"2024-08-17T17:33:26.962351+0000","last_unstale":"2024-08-17T17:33:40.611512+0000","last_undegraded":"2024-08-17T17:33:40.611512+0000","last_fullsized":"2024-08-17T17:33:40.611512+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:56:34.728644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1625,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.613561+0000","last_change":"2024-08-17T17:33:26.936016+0000","last_active":"2024-08-17T17:33:39.613561+0000","last_peered":"2024-08-17T17:33:39.613561+0000","last_clean":"2024-08-17T17:33:39.613561+0000","last_became_active":"2024-08-17T17:33:26.935372+0000","last_became_peered":"2024-08-17T17:33:26.935372+0000","last_unstale":"2024-08-17T17:33:39.613561+0000","last_undegraded":"2024-08-17T17:33:39.613561+0000","last_fullsized":"2024-08-17T17:33:39.613561+0000","mapping_epoch":1323,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1324,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:54:10.938003+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1,7],"avail_no_missing":["5","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.18","version":"1330'11256","reported_seq":11394,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:42.619068+0000","last_change":"2024-08-17T17:33:27.059419+0000","last_active":"2024-08-17T17:33:42.619068+0000","last_peered":"2024-08-17T17:33:42.619068+0000","last_clean":"2024-08-17T17:33:42.619068+0000","last_became_active":"2024-08-17T17:32:07.914430+0000","last_became_peered":"2024-08-17T17:32:07.914430+0000","last_unstale":"2024-08-17T17:33:42.619068+0000","last_undegraded":"2024-08-17T17:33:42.619068+0000","last_fullsized":"2024-08-17T17:33:42.619068+0000","mapping_epoch":1306,"log_start":"1292'1843","ondisk_log_start":"1292'1843","created":1306,"last_epoch_clean":1318,"parent":"0.0","parent_split_bits":5,"last_scrub":"1323'10813","last_scrub_stamp":"2024-08-17T17:33:27.059257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:30:30.026678+0000","last_clean_scrub_stamp":"2024-08-17T17:33:27.059257+0000","objects_scrubbed":1759,"log_size":9413,"log_dups_size":0,"ondisk_log_size":9413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:01:22.112309+0000","scrub_duration":273,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86245376,"num_objects":1316,"num_object_clones":0,"num_object_copies":3948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":443,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1215,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T17:33:39.611688+0000","last_change":"2024-08-17T17:32:47.590499+0000","last_active":"2024-08-17T17:33:39.611688+0000","last_peered":"2024-08-17T17:33:39.611688+0000","last_clean":"2024-08-17T17:29:38.568865+0000","last_became_active":"2024-08-17T17:32:47.590499+0000","last_became_peered":"2024-08-17T17:32:47.590499+0000","last_unstale":"2024-08-17T17:33:39.611688+0000","last_undegraded":"2024-08-17T17:33:39.611688+0000","last_fullsized":"2024-08-17T17:32:47.584568+0000","mapping_epoch":1312,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:34:47.373717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":36,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.610981+0000","last_change":"2024-08-17T17:33:14.819390+0000","last_active":"2024-08-17T17:33:40.610981+0000","last_peered":"2024-08-17T17:33:40.610981+0000","last_clean":"2024-08-17T17:33:40.610981+0000","last_became_active":"2024-08-17T17:33:14.816211+0000","last_became_peered":"2024-08-17T17:33:14.816211+0000","last_unstale":"2024-08-17T17:33:40.610981+0000","last_undegraded":"2024-08-17T17:33:40.610981+0000","last_fullsized":"2024-08-17T17:33:40.610981+0000","mapping_epoch":1317,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1318,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:43:59.156987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":961,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.613270+0000","last_change":"2024-08-17T17:33:28.932441+0000","last_active":"2024-08-17T17:33:39.613270+0000","last_peered":"2024-08-17T17:33:39.613270+0000","last_clean":"2024-08-17T17:33:39.613270+0000","last_became_active":"2024-08-17T17:33:28.932014+0000","last_became_peered":"2024-08-17T17:33:28.932014+0000","last_unstale":"2024-08-17T17:33:39.613270+0000","last_undegraded":"2024-08-17T17:33:39.613270+0000","last_fullsized":"2024-08-17T17:33:39.613270+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:22:46.346372+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":972,"reported_epoch":1330,"state":"active+clean","last_fresh":"2024-08-17T17:33:40.608117+0000","last_change":"2024-08-17T17:33:25.946272+0000","last_active":"2024-08-17T17:33:40.608117+0000","last_peered":"2024-08-17T17:33:40.608117+0000","last_clean":"2024-08-17T17:33:40.608117+0000","last_became_active":"2024-08-17T17:33:25.945791+0000","last_became_peered":"2024-08-17T17:33:25.945791+0000","last_unstale":"2024-08-17T17:33:40.608117+0000","last_undegraded":"2024-08-17T17:33:40.608117+0000","last_fullsized":"2024-08-17T17:33:40.608117+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:41:15.286694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1284,"reported_epoch":1329,"state":"active+clean+remapped","last_fresh":"2024-08-17T17:33:39.614347+0000","last_change":"2024-08-17T17:33:15.824395+0000","last_active":"2024-08-17T17:33:39.614347+0000","last_peered":"2024-08-17T17:33:39.614347+0000","last_clean":"2024-08-17T17:33:39.614347+0000","last_became_active":"2024-08-17T17:33:15.823810+0000","last_became_peered":"2024-08-17T17:33:15.823810+0000","last_unstale":"2024-08-17T17:33:39.614347+0000","last_undegraded":"2024-08-17T17:33:39.614347+0000","last_fullsized":"2024-08-17T17:33:39.614347+0000","mapping_epoch":1318,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1319,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:43:42.060991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7,3],"avail_no_missing":["5","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1593,"reported_epoch":1329,"state":"active+undersized","last_fresh":"2024-08-17T17:33:39.613214+0000","last_change":"2024-08-17T17:29:53.529377+0000","last_active":"2024-08-17T17:33:39.613214+0000","last_peered":"2024-08-17T17:33:39.613214+0000","last_clean":"2024-08-17T17:29:38.563824+0000","last_became_active":"2024-08-17T17:29:53.529377+0000","last_became_peered":"2024-08-17T17:29:53.529377+0000","last_unstale":"2024-08-17T17:33:39.613214+0000","last_undegraded":"2024-08-17T17:33:39.613214+0000","last_fullsized":"2024-08-17T17:29:53.524218+0000","mapping_epoch":1274,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1221,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:57:52.315542+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1263,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.616326+0000","last_change":"2024-08-17T17:07:12.646175+0000","last_active":"2024-08-17T17:33:39.616326+0000","last_peered":"2024-08-17T17:33:39.616326+0000","last_clean":"2024-08-17T17:33:39.616326+0000","last_became_active":"2024-08-17T17:07:12.645825+0000","last_became_peered":"2024-08-17T17:07:12.645825+0000","last_unstale":"2024-08-17T17:33:39.616326+0000","last_undegraded":"2024-08-17T17:33:39.616326+0000","last_fullsized":"2024-08-17T17:33:39.616326+0000","mapping_epoch":804,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":805,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:27.572532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1701,"reported_epoch":1331,"state":"active+clean","last_fresh":"2024-08-17T17:33:45.706082+0000","last_change":"2024-08-17T17:33:25.940723+0000","last_active":"2024-08-17T17:33:45.706082+0000","last_peered":"2024-08-17T17:33:45.706082+0000","last_clean":"2024-08-17T17:33:45.706082+0000","last_became_active":"2024-08-17T17:33:25.940315+0000","last_became_peered":"2024-08-17T17:33:25.940315+0000","last_unstale":"2024-08-17T17:33:45.706082+0000","last_undegraded":"2024-08-17T17:33:45.706082+0000","last_fullsized":"2024-08-17T17:33:45.706082+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:46:26.251170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1690,"reported_epoch":1329,"state":"active+clean","last_fresh":"2024-08-17T17:33:39.613305+0000","last_change":"2024-08-17T17:33:28.948818+0000","last_active":"2024-08-17T17:33:39.613305+0000","last_peered":"2024-08-17T17:33:39.613305+0000","last_clean":"2024-08-17T17:33:39.613305+0000","last_became_active":"2024-08-17T17:33:28.948424+0000","last_became_peered":"2024-08-17T17:33:28.948424+0000","last_unstale":"2024-08-17T17:33:39.613305+0000","last_undegraded":"2024-08-17T17:33:39.613305+0000","last_fullsized":"2024-08-17T17:33:39.613305+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:34.770786+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":2780037144,"num_objects":42421,"num_object_clones":0,"num_object_copies":127263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15716,"num_objects_misplaced":12582,"num_objects_unfound":0,"num_objects_dirty":42421,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":212305,"num_write_kb":7695361,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13010,"num_bytes_recovered":838008832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7741054976,"data_stored":7741046832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":177789,"ondisk_log_size":177789,"up":69,"acting":70,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":79,"acting":91,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":1164,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11692,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":81,"acting":90,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":76,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":21,"acting":23,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":533867,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":70,"num_read_kb":149,"num_write":38,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":703369,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":77,"acting":87,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201787,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1844408,"kb_used_data":1029104,"kb_used_omap":1,"kb_used_meta":815294,"kb_avail":91900744,"statfs":{"total":95995035648,"available":94106361856,"internally_reserved":0,"allocated":1053802496,"data_stored":1050733143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":834861515},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463378,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":695916,"kb_used_data":271908,"kb_used_omap":1,"kb_used_meta":423998,"kb_avail":93049236,"statfs":{"total":95995035648,"available":95282417664,"internally_reserved":0,"allocated":278433792,"data_stored":275769600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1591,"internal_metadata":434174409},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627066,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1838960,"kb_used_data":936356,"kb_used_omap":42,"kb_used_meta":902549,"kb_avail":91906192,"statfs":{"total":95995035648,"available":94111940608,"internally_reserved":0,"allocated":958828544,"data_stored":955775600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":924210236},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785543,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":813792,"kb_used_data":177620,"kb_used_omap":48,"kb_used_meta":636111,"kb_avail":92931360,"statfs":{"total":95995035648,"available":95161712640,"internally_reserved":0,"allocated":181882880,"data_stored":178836714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":651378350},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725060,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1634736,"kb_used_data":863712,"kb_used_omap":1,"kb_used_meta":771006,"kb_avail":92110416,"statfs":{"total":95995035648,"available":94321065984,"internally_reserved":0,"allocated":884441088,"data_stored":881383818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":789510598},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986697,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2542456,"kb_used_data":1767276,"kb_used_omap":1,"kb_used_meta":775102,"kb_avail":91202696,"statfs":{"total":95995035648,"available":93391560704,"internally_reserved":0,"allocated":1809690624,"data_stored":1806624777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":793704902},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706304,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1985512,"kb_used_data":1209372,"kb_used_omap":35,"kb_used_meta":776092,"kb_avail":91759640,"statfs":{"total":95995035648,"available":93961871360,"internally_reserved":0,"allocated":1238396928,"data_stored":1235343711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":794718263},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832077,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2680176,"kb_used_data":1745312,"kb_used_omap":65,"kb_used_meta":934782,"kb_avail":91064976,"statfs":{"total":95995035648,"available":93250535424,"internally_reserved":0,"allocated":1787199488,"data_stored":1784141662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":957216955},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1755385856,"data_stored":1755381784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":174194688,"data_stored":174194688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1204879360,"data_stored":1204879360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":947912704,"data_stored":947912704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1757089792,"data_stored":1757085720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":869138432,"data_stored":869138432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1032454144,"data_stored":1032454144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:33:50.449 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set unique_pool_5 pgp_num 26 2024-08-17T17:33:50.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.485 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:50.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:50.586 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:50.618 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:50.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:50.719 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:50.776 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:50.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:50.857 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:33:50.858 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:33:50.878 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:50.912 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:50.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:50.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:51.014 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:50 smithi063 ceph-mon[114756]: osdmap e1334: 8 total, 7 up, 8 in 2024-08-17T17:33:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:50 smithi063 ceph-mon[114756]: 1.19 deep-scrub starts 2024-08-17T17:33:51.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:50 smithi063 ceph-mon[114756]: 1.19 deep-scrub ok 2024-08-17T17:33:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:50 smithi063 ceph-mon[109843]: osdmap e1334: 8 total, 7 up, 8 in 2024-08-17T17:33:51.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:50 smithi063 ceph-mon[109843]: 1.19 deep-scrub starts 2024-08-17T17:33:51.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:50 smithi063 ceph-mon[109843]: 1.19 deep-scrub ok 2024-08-17T17:33:51.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.047 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:51.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:51.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:50 smithi162 ceph-mon[104668]: osdmap e1334: 8 total, 7 up, 8 in 2024-08-17T17:33:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:50 smithi162 ceph-mon[104668]: 1.19 deep-scrub starts 2024-08-17T17:33:51.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:50 smithi162 ceph-mon[104668]: 1.19 deep-scrub ok 2024-08-17T17:33:51.096 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:51.148 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:51.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.186 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:51.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:51.287 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:51.327 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:51.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:51.429 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:51.458 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:51.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:51.560 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:51.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.594 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:51.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:51.634 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:51.694 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:51.753 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:51.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.787 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:51.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:51.830 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:51 smithi063 ceph-mon[114756]: pgmap v1992: 129 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 3 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 66 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 13008/113793 objects degraded (11.431%); 8354/113793 objects misplaced (7.341%); 21 MiB/s, 418 objects/s recovering 2024-08-17T17:33:51.830 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:51 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:51.830 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:51 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:51.831 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:51 smithi063 ceph-mon[109843]: pgmap v1992: 129 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 3 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 66 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 13008/113793 objects degraded (11.431%); 8354/113793 objects misplaced (7.341%); 21 MiB/s, 418 objects/s recovering 2024-08-17T17:33:51.831 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:51 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:51.831 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:51 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:51.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:51 smithi162 ceph-mon[104668]: pgmap v1992: 129 pgs: 1 active+clean+scrubbing, 1 remapped+peering, 3 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 66 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 13008/113793 objects degraded (11.431%); 8354/113793 objects misplaced (7.341%); 21 MiB/s, 418 objects/s recovering 2024-08-17T17:33:51.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:51 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:51.847 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:51 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:51.889 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:51.917 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:51.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:51.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:52.019 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:52.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.060 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:52.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:52.161 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:52.202 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:52.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:52.304 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:52.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.343 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:52.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:52.444 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:52.491 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:52.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:52.592 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:52.621 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:52.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:52.722 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:52.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.757 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:52.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:52.858 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:52.859 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:33:52.914 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:52.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:52.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:53.016 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:53.045 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:53.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:53.147 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:53.172 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:53.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:53.277 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:53.315 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:53.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:53.417 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:53.446 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:53.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:53.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:33:53] "GET /metrics HTTP/1.1" 200 36604 "" "Prometheus/2.51.0" 2024-08-17T17:33:53.549 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:53.572 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:53.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:53.674 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:53.697 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:53.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:53.798 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:53.822 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:53.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:53.923 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:53.947 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:53.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:53.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:54.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[109843]: pgmap v1993: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+scrubbing, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 66 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.34k op/s; 10472/103227 objects degraded (10.145%); 6882/103227 objects misplaced (6.667%); 28 MiB/s, 599 objects/s recovering 2024-08-17T17:33:54.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 10472/103227 objects degraded (10.145%), 11 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:54.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[109843]: osdmap e1335: 8 total, 7 up, 8 in 2024-08-17T17:33:54.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:33:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:33:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:54.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[114756]: pgmap v1993: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+scrubbing, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 66 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.34k op/s; 10472/103227 objects degraded (10.145%); 6882/103227 objects misplaced (6.667%); 28 MiB/s, 599 objects/s recovering 2024-08-17T17:33:54.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 10472/103227 objects degraded (10.145%), 11 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:54.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[114756]: osdmap e1335: 8 total, 7 up, 8 in 2024-08-17T17:33:54.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:54.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:54.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:33:54.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:33:54.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:53 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:54.049 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:54.073 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:54.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:53 smithi162 ceph-mon[104668]: pgmap v1993: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+scrubbing, 1 remapped+peering, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 66 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.34k op/s; 10472/103227 objects degraded (10.145%); 6882/103227 objects misplaced (6.667%); 28 MiB/s, 599 objects/s recovering 2024-08-17T17:33:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:53 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 10472/103227 objects degraded (10.145%), 11 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:53 smithi162 ceph-mon[104668]: osdmap e1335: 8 total, 7 up, 8 in 2024-08-17T17:33:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:53 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:53 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:53 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:33:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:53 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:33:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:53 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:33:54.175 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:54.196 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:54.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:54.298 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:54.340 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:54.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:54.441 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:54.493 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:54.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:54.595 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:54.627 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:54.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:54.729 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:54.757 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:54.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:54.859 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:54.887 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:54.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:54.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:54.988 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:55.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.019 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:55.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:55.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:55.155 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:55.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:55.257 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:55.292 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:55.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:55.394 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:55.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.426 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:55.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:55.527 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:55.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:55 smithi063 ceph-mon[109843]: osdmap e1336: 8 total, 7 up, 8 in 2024-08-17T17:33:55.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:55 smithi063 ceph-mon[114756]: osdmap e1336: 8 total, 7 up, 8 in 2024-08-17T17:33:55.566 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:55.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:55.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:55 smithi162 ceph-mon[104668]: osdmap e1336: 8 total, 7 up, 8 in 2024-08-17T17:33:55.668 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:55.700 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:55.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:55.802 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:55.847 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:55.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:55.948 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:55.977 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:55.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:55.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:56.079 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:56.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.111 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:56.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:56.212 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:56.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.246 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:56.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:56.347 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:56.387 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:56.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:56.400 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:56 smithi063 ceph-mon[114756]: pgmap v1996: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+scrubbing, 2 active+undersized+degraded+remapped+backfill_wait, 35 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 66 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 10472/99789 objects degraded (10.494%); 8847/99789 objects misplaced (8.866%); 13 MiB/s, 301 objects/s recovering 2024-08-17T17:33:56.401 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:56 smithi063 ceph-mon[109843]: pgmap v1996: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+scrubbing, 2 active+undersized+degraded+remapped+backfill_wait, 35 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 66 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 10472/99789 objects degraded (10.494%); 8847/99789 objects misplaced (8.866%); 13 MiB/s, 301 objects/s recovering 2024-08-17T17:33:56.490 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:56.527 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:56.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:56.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:56 smithi162 ceph-mon[104668]: pgmap v1996: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+scrubbing, 2 active+undersized+degraded+remapped+backfill_wait, 35 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 66 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 10472/99789 objects degraded (10.494%); 8847/99789 objects misplaced (8.866%); 13 MiB/s, 301 objects/s recovering 2024-08-17T17:33:56.631 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:56.679 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:56.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:56.781 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:56.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.820 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:56.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:56.921 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:56.951 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:56.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:56.952 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:57.052 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:57.094 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:57.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:57.196 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:57.234 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:57.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:57.336 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:57.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.377 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:57.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:57.479 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:57.512 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:57.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:57.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:57 smithi063 ceph-mon[114756]: pgmap v1997: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+scrubbing, 2 active+undersized+degraded+remapped+backfill_wait, 35 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 66 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 10207/98994 objects degraded (10.311%); 8847/98994 objects misplaced (8.937%); 9.4 MiB/s, 229 objects/s recovering 2024-08-17T17:33:57.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:57 smithi063 ceph-mon[109843]: pgmap v1997: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+scrubbing, 2 active+undersized+degraded+remapped+backfill_wait, 35 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 66 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 10207/98994 objects degraded (10.311%); 8847/98994 objects misplaced (8.937%); 9.4 MiB/s, 229 objects/s recovering 2024-08-17T17:33:57.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:57 smithi162 ceph-mon[104668]: pgmap v1997: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+clean+scrubbing, 2 active+undersized+degraded+remapped+backfill_wait, 35 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 14 active+undersized, 7 active+undersized+degraded, 66 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 10207/98994 objects degraded (10.311%); 8847/98994 objects misplaced (8.937%); 9.4 MiB/s, 229 objects/s recovering 2024-08-17T17:33:57.614 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:57.645 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:57.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:57.747 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:57.773 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:57.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:57.875 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:57.899 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:57.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:57.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:58.001 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:58.024 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:58.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:58.126 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:58.152 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:58.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:58.254 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:58.286 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:58.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:58.388 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:58.411 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:58.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:58.513 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:58.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:58 smithi063 ceph-mon[114756]: osdmap e1337: 8 total, 7 up, 8 in 2024-08-17T17:33:58.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:58 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 10207/98994 objects degraded (10.311%), 11 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:58.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:33:58 smithi063 ceph-mon[114756]: osdmap e1338: 8 total, 7 up, 8 in 2024-08-17T17:33:58.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:58 smithi063 ceph-mon[109843]: osdmap e1337: 8 total, 7 up, 8 in 2024-08-17T17:33:58.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:58 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 10207/98994 objects degraded (10.311%), 11 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:58.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:33:58 smithi063 ceph-mon[109843]: osdmap e1338: 8 total, 7 up, 8 in 2024-08-17T17:33:58.537 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:58.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:58.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:58 smithi162 ceph-mon[104668]: osdmap e1337: 8 total, 7 up, 8 in 2024-08-17T17:33:58.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:58 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 10207/98994 objects degraded (10.311%), 11 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:33:58.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:33:58 smithi162 ceph-mon[104668]: osdmap e1338: 8 total, 7 up, 8 in 2024-08-17T17:33:58.639 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:58.662 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:58.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:58.764 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:33:58.789 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:58.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:33:58.890 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:58.916 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:58.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:58.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:59.018 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:59.041 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:59.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:59.143 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:59.169 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:59.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:33:59.272 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:33:59.304 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:59.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:33:59.407 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:33:59.435 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:59.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:33:59.537 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:33:59.556 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:33:59.569 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:59.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:33:59.671 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:33:59.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.693 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:59.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:33:59.794 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:33:59.819 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:33:59.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:33:59.921 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:33:59.942 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:33:59.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:33:59.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:00.044 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:00.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.067 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:00.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:00.168 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:00.215 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:00.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:00.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:00 smithi063 ceph-mon[109843]: pgmap v2000: 129 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 36 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 7 active+undersized+degraded, 67 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 8954/86181 objects degraded (10.390%); 6419/86181 objects misplaced (7.448%); 13 MiB/s, 311 objects/s recovering 2024-08-17T17:34:00.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:00 smithi063 ceph-mon[109843]: osdmap e1339: 8 total, 7 up, 8 in 2024-08-17T17:34:00.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:00 smithi063 ceph-mon[114756]: pgmap v2000: 129 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 36 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 7 active+undersized+degraded, 67 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 8954/86181 objects degraded (10.390%); 6419/86181 objects misplaced (7.448%); 13 MiB/s, 311 objects/s recovering 2024-08-17T17:34:00.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:00 smithi063 ceph-mon[114756]: osdmap e1339: 8 total, 7 up, 8 in 2024-08-17T17:34:00.317 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:00.356 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:00.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:00.458 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:00.486 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:00.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:00.588 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:00.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:00 smithi162 ceph-mon[104668]: pgmap v2000: 129 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 36 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 7 active+undersized+degraded, 67 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 8954/86181 objects degraded (10.390%); 6419/86181 objects misplaced (7.448%); 13 MiB/s, 311 objects/s recovering 2024-08-17T17:34:00.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:00 smithi162 ceph-mon[104668]: osdmap e1339: 8 total, 7 up, 8 in 2024-08-17T17:34:00.623 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:00.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:00.724 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:00.750 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:00.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.751 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:00.852 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:00.879 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:00.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:00.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:00.981 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:01.003 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:01.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:01.093 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:34:01.093 INFO:teuthology.orchestra.run.smithi063.stderr:set pool 10 pgp_num to 26 2024-08-17T17:34:01.104 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:01.125 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:01.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:01.227 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:01.254 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:01.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:01.355 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:01.386 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:01.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:01.488 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:01.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.513 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:01.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:01.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[114756]: osdmap e1340: 8 total, 7 up, 8 in 2024-08-17T17:34:01.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[114756]: pgmap v2003: 129 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 36 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 7 active+undersized+degraded, 67 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 8954/85470 objects degraded (10.476%); 6182/85470 objects misplaced (7.233%); 18 MiB/s, 424 objects/s recovering 2024-08-17T17:34:01.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2783319861' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:34:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3783322341' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:34:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2783319861' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:34:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3783322341' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2024-08-17T17:34:01.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[114756]: osdmap e1341: 8 total, 7 up, 8 in 2024-08-17T17:34:01.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[109843]: osdmap e1340: 8 total, 7 up, 8 in 2024-08-17T17:34:01.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[109843]: pgmap v2003: 129 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 36 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 7 active+undersized+degraded, 67 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 8954/85470 objects degraded (10.476%); 6182/85470 objects misplaced (7.233%); 18 MiB/s, 424 objects/s recovering 2024-08-17T17:34:01.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2783319861' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:34:01.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3783322341' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:34:01.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:01.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2783319861' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:34:01.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3783322341' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2024-08-17T17:34:01.531 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:01 smithi063 ceph-mon[109843]: osdmap e1341: 8 total, 7 up, 8 in 2024-08-17T17:34:01.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:01 smithi162 ceph-mon[104668]: osdmap e1340: 8 total, 7 up, 8 in 2024-08-17T17:34:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:01 smithi162 ceph-mon[104668]: pgmap v2003: 129 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 36 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+undersized, 7 active+undersized+degraded, 67 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 8954/85470 objects degraded (10.476%); 6182/85470 objects misplaced (7.233%); 18 MiB/s, 424 objects/s recovering 2024-08-17T17:34:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:01 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2783319861' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:34:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:01 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3783322341' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"} : dispatch 2024-08-17T17:34:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:01 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:01 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2783319861' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:34:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:01 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3783322341' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2024-08-17T17:34:01.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:01 smithi162 ceph-mon[104668]: osdmap e1341: 8 total, 7 up, 8 in 2024-08-17T17:34:01.614 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:01.645 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:01.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:01.747 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:01.784 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:01.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:01.886 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:01.924 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:01.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:01.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:02.026 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:02.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.069 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:02.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:02.170 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:02.213 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:02.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:02.315 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:02.356 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:02.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:02.458 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:02.500 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:02.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:02.601 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:02.635 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:02.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:02.659 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:02 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:34:02.659 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:02 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:02.660 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:02 smithi063 ceph-mon[114756]: 10.13 scrub starts 2024-08-17T17:34:02.660 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:02 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:34:02.660 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:02 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:02.660 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:02 smithi063 ceph-mon[109843]: 10.13 scrub starts 2024-08-17T17:34:02.738 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:02.761 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:02.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:02.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:02 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:34:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:02 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:02.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:02 smithi162 ceph-mon[104668]: 10.13 scrub starts 2024-08-17T17:34:02.863 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:02.884 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:02.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:02.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:02.986 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:03.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.008 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:03.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:03.109 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:03.133 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:03.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:03.234 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:03.255 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:03.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:03.357 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:03.380 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:03.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:03.482 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:03.521 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:03.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:03.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:34:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:34:03] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-08-17T17:34:03.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:03 smithi063 ceph-mon[114756]: osdmap e1342: 8 total, 7 up, 8 in 2024-08-17T17:34:03.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:03 smithi063 ceph-mon[114756]: pgmap v2006: 129 pgs: 1 active+undersized+remapped, 38 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 4898/78654 objects degraded (6.227%); 6373/78654 objects misplaced (8.103%); 39 MiB/s, 656 objects/s recovering 2024-08-17T17:34:03.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:03 smithi063 ceph-mon[114756]: 10.3 scrub starts 2024-08-17T17:34:03.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:03 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 8954/85470 objects degraded (10.476%), 10 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:03.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:03 smithi063 ceph-mon[109843]: osdmap e1342: 8 total, 7 up, 8 in 2024-08-17T17:34:03.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:03 smithi063 ceph-mon[109843]: pgmap v2006: 129 pgs: 1 active+undersized+remapped, 38 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 4898/78654 objects degraded (6.227%); 6373/78654 objects misplaced (8.103%); 39 MiB/s, 656 objects/s recovering 2024-08-17T17:34:03.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:03 smithi063 ceph-mon[109843]: 10.3 scrub starts 2024-08-17T17:34:03.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:03 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 8954/85470 objects degraded (10.476%), 10 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:03.623 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:03.654 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:03.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:03.756 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:03.787 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:03.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:03.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:03 smithi162 ceph-mon[104668]: osdmap e1342: 8 total, 7 up, 8 in 2024-08-17T17:34:03.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:03 smithi162 ceph-mon[104668]: pgmap v2006: 129 pgs: 1 active+undersized+remapped, 38 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 626 op/s; 4898/78654 objects degraded (6.227%); 6373/78654 objects misplaced (8.103%); 39 MiB/s, 656 objects/s recovering 2024-08-17T17:34:03.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:03 smithi162 ceph-mon[104668]: 10.3 scrub starts 2024-08-17T17:34:03.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:03 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 8954/85470 objects degraded (10.476%), 10 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:03.889 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:03.915 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:03.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:03.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:04.017 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:04.060 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:04.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:04.162 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:04.188 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:04.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:04.290 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:04.330 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:04.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:04.353 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:34:04.354 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:34:04.354 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:34:04.354 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:34:04.354 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.354 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45218.0:138103 8.c e1213/1195 8:30f0a028:::benchmark_data_smithi063_230025_object11613:head v 1213'8745, mlcod=1213'8743)", 2024-08-17T17:34:04.354 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:24:10.390810+0000", 2024-08-17T17:34:04.354 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 593.53136249500005, 2024-08-17T17:34:04.354 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.14214605299999999, 2024-08-17T17:34:04.354 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.354 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.354 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.354 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.355 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.355 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.355 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390810+0000", 2024-08-17T17:34:04.355 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.355 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.355 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.355 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.355 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390810+0000", 2024-08-17T17:34:04.355 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.355 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.355 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.355 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.355 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390811+0000", 2024-08-17T17:34:04.355 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7999999999999995e-07 2024-08-17T17:34:04.355 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.356 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.356 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.356 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.390817+0000", 2024-08-17T17:34:04.356 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0100000000000001e-06 2024-08-17T17:34:04.356 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.356 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.356 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.356 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509388+0000", 2024-08-17T17:34:04.356 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.118571464 2024-08-17T17:34:04.356 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.356 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.356 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.356 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509396+0000", 2024-08-17T17:34:04.356 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9980000000000003e-06 2024-08-17T17:34:04.357 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.357 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.357 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.357 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509570+0000", 2024-08-17T17:34:04.357 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000174279 2024-08-17T17:34:04.357 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.357 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.357 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.357 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509590+0000", 2024-08-17T17:34:04.357 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9083999999999999e-05 2024-08-17T17:34:04.357 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.357 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.358 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.358 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.532822+0000", 2024-08-17T17:34:04.358 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.023232017000000001 2024-08-17T17:34:04.358 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.358 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.358 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.358 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.532956+0000", 2024-08-17T17:34:04.358 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013452099999999999 2024-08-17T17:34:04.358 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.358 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.358 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.358 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.358 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.359 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45218.0:138121 8.9 e1213/1195 8:980b75d7:::benchmark_data_smithi063_230025_object11631:head v 1213'8648, mlcod=1213'8646)", 2024-08-17T17:34:04.359 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:24:10.395445+0000", 2024-08-17T17:34:04.359 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 593.52672751600005, 2024-08-17T17:34:04.359 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.129388477, 2024-08-17T17:34:04.359 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.359 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.359 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.359 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.359 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.359 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.359 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.395445+0000", 2024-08-17T17:34:04.359 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.359 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.360 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.360 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.360 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.395445+0000", 2024-08-17T17:34:04.360 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.360 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.360 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.360 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.360 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.395446+0000", 2024-08-17T17:34:04.360 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.1500000000000003e-07 2024-08-17T17:34:04.360 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.360 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.360 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.361 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.395454+0000", 2024-08-17T17:34:04.361 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7700000000000001e-06 2024-08-17T17:34:04.361 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.361 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.361 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.361 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509168+0000", 2024-08-17T17:34:04.361 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.11371437500000001 2024-08-17T17:34:04.361 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.361 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.361 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.361 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509196+0000", 2024-08-17T17:34:04.361 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8255999999999998e-05 2024-08-17T17:34:04.361 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.362 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.362 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.362 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509487+0000", 2024-08-17T17:34:04.362 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00029032400000000001 2024-08-17T17:34:04.362 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.362 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.362 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.362 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.509507+0000", 2024-08-17T17:34:04.362 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9964000000000001e-05 2024-08-17T17:34:04.362 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.362 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.362 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.363 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.524782+0000", 2024-08-17T17:34:04.363 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.015275476 2024-08-17T17:34:04.363 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.363 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.363 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.363 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:24:10.524833+0000", 2024-08-17T17:34:04.363 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1397000000000003e-05 2024-08-17T17:34:04.363 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.363 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.363 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.363 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.363 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.363 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:25690 9.5 e1251/1247 9:aa569b13:::benchmark_data_smithi063_263014_object25689:head v 1251'1611, mlcod=1251'1609)", 2024-08-17T17:34:04.364 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:26:56.437973+0000", 2024-08-17T17:34:04.364 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 427.48419919700001, 2024-08-17T17:34:04.364 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.202847471, 2024-08-17T17:34:04.364 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.364 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.364 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.364 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.364 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.364 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.364 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.437973+0000", 2024-08-17T17:34:04.364 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.364 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.364 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.365 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.365 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.437973+0000", 2024-08-17T17:34:04.365 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.365 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.365 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.365 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.365 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.437974+0000", 2024-08-17T17:34:04.365 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8999999999999995e-07 2024-08-17T17:34:04.365 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.365 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.365 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.365 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.438031+0000", 2024-08-17T17:34:04.365 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6320000000000003e-05 2024-08-17T17:34:04.365 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.366 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.366 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.366 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.438042+0000", 2024-08-17T17:34:04.366 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1209999999999999e-05 2024-08-17T17:34:04.366 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.366 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.366 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.366 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.438049+0000", 2024-08-17T17:34:04.366 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3180000000000001e-06 2024-08-17T17:34:04.366 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.366 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.366 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.366 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.438111+0000", 2024-08-17T17:34:04.366 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1484999999999998e-05 2024-08-17T17:34:04.367 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.367 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.367 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.367 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.438131+0000", 2024-08-17T17:34:04.367 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0409999999999999e-05 2024-08-17T17:34:04.367 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.367 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.367 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.367 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.640763+0000", 2024-08-17T17:34:04.367 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.202632328 2024-08-17T17:34:04.367 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.367 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.367 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.368 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.640821+0000", 2024-08-17T17:34:04.368 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7510000000000003e-05 2024-08-17T17:34:04.368 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.368 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.368 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.368 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.368 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.368 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:25692 9.9 e1251/1247 9:909dc1e3:::benchmark_data_smithi063_263014_object25691:head v 1251'1639, mlcod=1251'1637)", 2024-08-17T17:34:04.368 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:26:56.440325+0000", 2024-08-17T17:34:04.368 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 427.48184778500001, 2024-08-17T17:34:04.368 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19897859300000001, 2024-08-17T17:34:04.368 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.368 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.369 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.369 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.369 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.369 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.369 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.440325+0000", 2024-08-17T17:34:04.369 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.369 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.369 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.369 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.369 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.440325+0000", 2024-08-17T17:34:04.369 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.369 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.369 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.370 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.370 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.440326+0000", 2024-08-17T17:34:04.370 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.02e-06 2024-08-17T17:34:04.370 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.370 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.370 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.370 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.440419+0000", 2024-08-17T17:34:04.370 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3542999999999995e-05 2024-08-17T17:34:04.370 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.370 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.370 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.370 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.637224+0000", 2024-08-17T17:34:04.370 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.196805127 2024-08-17T17:34:04.371 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.371 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.371 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.371 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.637238+0000", 2024-08-17T17:34:04.371 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3341e-05 2024-08-17T17:34:04.371 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.371 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.371 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.371 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.637298+0000", 2024-08-17T17:34:04.371 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0081999999999998e-05 2024-08-17T17:34:04.371 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.371 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.371 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.372 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.637319+0000", 2024-08-17T17:34:04.372 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1596000000000001e-05 2024-08-17T17:34:04.372 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.372 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.372 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.372 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.639257+0000", 2024-08-17T17:34:04.372 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0019377520000000001 2024-08-17T17:34:04.372 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.372 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.372 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.372 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.639303+0000", 2024-08-17T17:34:04.372 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6131999999999998e-05 2024-08-17T17:34:04.373 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.373 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.373 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.373 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.373 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.373 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:25698 9.5 e1251/1247 9:ae5e9e69:::benchmark_data_smithi063_263014_object25697:head v 1251'1612, mlcod=1251'1609)", 2024-08-17T17:34:04.373 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:26:56.441132+0000", 2024-08-17T17:34:04.373 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 427.48104078199998, 2024-08-17T17:34:04.373 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20151798300000001, 2024-08-17T17:34:04.373 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.373 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.373 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.373 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.374 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.374 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.374 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.441132+0000", 2024-08-17T17:34:04.374 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.374 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.374 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.374 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.374 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.441132+0000", 2024-08-17T17:34:04.374 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.374 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.374 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.375 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.375 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.441133+0000", 2024-08-17T17:34:04.375 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.9100000000000002e-07 2024-08-17T17:34:04.375 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.375 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.375 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.375 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.441167+0000", 2024-08-17T17:34:04.375 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4740999999999998e-05 2024-08-17T17:34:04.375 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.375 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.375 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.375 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.637228+0000", 2024-08-17T17:34:04.375 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.196060242 2024-08-17T17:34:04.375 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.376 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.376 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.376 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.637237+0000", 2024-08-17T17:34:04.376 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.662e-06 2024-08-17T17:34:04.376 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.376 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.376 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.376 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.638332+0000", 2024-08-17T17:34:04.376 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0010948119999999999 2024-08-17T17:34:04.376 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.376 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.376 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.376 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.638351+0000", 2024-08-17T17:34:04.376 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8592000000000001e-05 2024-08-17T17:34:04.377 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.377 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.377 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.377 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.642625+0000", 2024-08-17T17:34:04.377 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0042740950000000003 2024-08-17T17:34:04.377 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.377 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.377 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.377 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:26:56.642650+0000", 2024-08-17T17:34:04.377 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4947999999999999e-05 2024-08-17T17:34:04.377 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.377 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.377 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.377 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.378 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.378 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:45004 9.8 e1252/1247 9:152681de:::benchmark_data_smithi063_263014_object45003:head v 1252'2841, mlcod=1252'2838)", 2024-08-17T17:34:04.378 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:08.432336+0000", 2024-08-17T17:34:04.378 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 415.48983698400002, 2024-08-17T17:34:04.378 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19930678199999999, 2024-08-17T17:34:04.378 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.378 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.378 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.378 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.378 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.378 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.378 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.432336+0000", 2024-08-17T17:34:04.378 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.379 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.379 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.379 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.379 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.432336+0000", 2024-08-17T17:34:04.379 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.379 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.379 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.379 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.379 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.432337+0000", 2024-08-17T17:34:04.379 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3829999999999999e-06 2024-08-17T17:34:04.379 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.379 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.379 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.379 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.432500+0000", 2024-08-17T17:34:04.380 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016271499999999999 2024-08-17T17:34:04.380 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.380 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.380 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.380 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.432513+0000", 2024-08-17T17:34:04.380 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3733999999999999e-05 2024-08-17T17:34:04.380 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.380 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.380 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.380 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.432521+0000", 2024-08-17T17:34:04.380 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9969999999999998e-06 2024-08-17T17:34:04.380 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.380 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.381 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.381 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627136+0000", 2024-08-17T17:34:04.381 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19461452200000001 2024-08-17T17:34:04.381 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.381 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.381 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.381 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627166+0000", 2024-08-17T17:34:04.381 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0463999999999998e-05 2024-08-17T17:34:04.381 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.381 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.381 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.381 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.631598+0000", 2024-08-17T17:34:04.381 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044317799999999997 2024-08-17T17:34:04.382 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.382 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.382 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.382 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.631642+0000", 2024-08-17T17:34:04.382 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4187000000000002e-05 2024-08-17T17:34:04.382 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.382 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.382 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.382 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.382 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.382 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:45014 9.8 e1252/1247 9:1f47c66c:::benchmark_data_smithi063_263014_object45013:head v 1252'2842, mlcod=1252'2839)", 2024-08-17T17:34:04.382 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:08.436477+0000", 2024-08-17T17:34:04.382 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 415.48569537700001, 2024-08-17T17:34:04.383 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19519346600000001, 2024-08-17T17:34:04.383 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.383 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.383 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.383 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.383 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.383 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.383 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.436477+0000", 2024-08-17T17:34:04.383 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.383 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.383 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.383 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.383 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.436477+0000", 2024-08-17T17:34:04.383 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.383 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.384 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.384 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.384 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.436479+0000", 2024-08-17T17:34:04.384 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7150000000000001e-06 2024-08-17T17:34:04.384 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.384 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.384 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.384 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.436532+0000", 2024-08-17T17:34:04.384 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2707000000000002e-05 2024-08-17T17:34:04.384 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.384 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.384 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.384 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627125+0000", 2024-08-17T17:34:04.385 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19059372699999999 2024-08-17T17:34:04.385 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.385 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.385 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.385 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627142+0000", 2024-08-17T17:34:04.385 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.66e-05 2024-08-17T17:34:04.385 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.385 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.385 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.385 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.628712+0000", 2024-08-17T17:34:04.385 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001569892 2024-08-17T17:34:04.385 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.385 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.386 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.386 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.628731+0000", 2024-08-17T17:34:04.386 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9286000000000001e-05 2024-08-17T17:34:04.386 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.386 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.386 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.386 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.631647+0000", 2024-08-17T17:34:04.386 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0029157549999999999 2024-08-17T17:34:04.386 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.386 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.386 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.386 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.631671+0000", 2024-08-17T17:34:04.386 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3784000000000001e-05 2024-08-17T17:34:04.387 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.387 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.387 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.387 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.387 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.387 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:45017 9.b e1252/1247 9:d02eb79b:::benchmark_data_smithi063_263014_object45016:head v 1252'2851, mlcod=1252'2849)", 2024-08-17T17:34:04.387 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:08.439705+0000", 2024-08-17T17:34:04.387 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 415.48246746000001, 2024-08-17T17:34:04.387 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.19242014800000001, 2024-08-17T17:34:04.387 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.387 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.387 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.387 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.388 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.388 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.388 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.439705+0000", 2024-08-17T17:34:04.388 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.388 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.388 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.388 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.388 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.439705+0000", 2024-08-17T17:34:04.388 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.388 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.388 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.388 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.388 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.439706+0000", 2024-08-17T17:34:04.388 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2580000000000001e-06 2024-08-17T17:34:04.388 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.389 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.389 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.389 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.439739+0000", 2024-08-17T17:34:04.389 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.2904999999999998e-05 2024-08-17T17:34:04.389 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.389 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.389 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.389 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627128+0000", 2024-08-17T17:34:04.389 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.187389153 2024-08-17T17:34:04.389 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.389 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.389 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.389 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627141+0000", 2024-08-17T17:34:04.389 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.27e-05 2024-08-17T17:34:04.390 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.390 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.390 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.390 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627246+0000", 2024-08-17T17:34:04.390 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000104615 2024-08-17T17:34:04.390 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.390 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.390 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.390 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.627266+0000", 2024-08-17T17:34:04.390 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9843999999999998e-05 2024-08-17T17:34:04.390 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.390 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.390 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.391 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.632045+0000", 2024-08-17T17:34:04.391 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0047795329999999999 2024-08-17T17:34:04.391 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.391 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.391 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.391 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.632125+0000", 2024-08-17T17:34:04.391 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0140000000000002e-05 2024-08-17T17:34:04.391 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.391 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.391 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.391 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.391 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.391 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:206378 9.9 e1272/1247 9:96b77632:::benchmark_data_smithi063_263014_object82040:head v 1272'13200, mlcod=1272'13197)", 2024-08-17T17:34:04.391 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:29:32.329541+0000", 2024-08-17T17:34:04.391 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 271.59263104899998, 2024-08-17T17:34:04.392 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.16146491900000001, 2024-08-17T17:34:04.392 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.392 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.392 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.392 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.392 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.392 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.392 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329541+0000", 2024-08-17T17:34:04.392 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.392 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.392 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.392 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.392 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329541+0000", 2024-08-17T17:34:04.392 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.392 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.393 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.393 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.393 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329542+0000", 2024-08-17T17:34:04.393 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.5000000000000001e-07 2024-08-17T17:34:04.393 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.393 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.393 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.393 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329548+0000", 2024-08-17T17:34:04.393 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0599999999999996e-06 2024-08-17T17:34:04.393 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.393 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.393 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.393 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329559+0000", 2024-08-17T17:34:04.393 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0682999999999999e-05 2024-08-17T17:34:04.394 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.394 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.394 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.394 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329566+0000", 2024-08-17T17:34:04.394 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.579e-06 2024-08-17T17:34:04.394 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.394 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.394 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.394 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329627+0000", 2024-08-17T17:34:04.394 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1527999999999998e-05 2024-08-17T17:34:04.394 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.394 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.394 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.394 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329645+0000", 2024-08-17T17:34:04.395 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7652999999999999e-05 2024-08-17T17:34:04.395 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.395 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.395 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.395 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.332505+0000", 2024-08-17T17:34:04.395 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0028606069999999998 2024-08-17T17:34:04.395 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.395 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.395 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.395 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.491006+0000", 2024-08-17T17:34:04.395 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.158500959 2024-08-17T17:34:04.395 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.395 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.395 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.395 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.396 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.396 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:206370 9.5 e1272/1247 9:a7b70f1c:::benchmark_data_smithi063_263014_object82032:head v 1272'12623, mlcod=1272'12620)", 2024-08-17T17:34:04.396 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:29:32.329581+0000", 2024-08-17T17:34:04.396 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 271.59259111099999, 2024-08-17T17:34:04.396 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.16146875099999999, 2024-08-17T17:34:04.396 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.396 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.396 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.396 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.396 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.396 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.396 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329581+0000", 2024-08-17T17:34:04.396 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.396 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.396 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.397 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.397 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329581+0000", 2024-08-17T17:34:04.397 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.397 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.397 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.397 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.397 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329582+0000", 2024-08-17T17:34:04.397 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1500000000000004e-07 2024-08-17T17:34:04.397 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.397 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.397 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.398 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329588+0000", 2024-08-17T17:34:04.398 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8510000000000004e-06 2024-08-17T17:34:04.398 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.398 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.398 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.398 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329598+0000", 2024-08-17T17:34:04.398 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0446e-05 2024-08-17T17:34:04.398 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.398 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.398 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.398 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329604+0000", 2024-08-17T17:34:04.399 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1399999999999997e-06 2024-08-17T17:34:04.399 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.399 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.399 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.399 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329672+0000", 2024-08-17T17:34:04.399 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7558999999999995e-05 2024-08-17T17:34:04.399 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.399 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.399 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.399 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.329691+0000", 2024-08-17T17:34:04.399 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9431000000000001e-05 2024-08-17T17:34:04.399 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.400 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.400 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.400 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.490997+0000", 2024-08-17T17:34:04.400 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.161305368 2024-08-17T17:34:04.400 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.400 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.400 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.400 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.491050+0000", 2024-08-17T17:34:04.400 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3341000000000003e-05 2024-08-17T17:34:04.400 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.401 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.401 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.401 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.401 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.401 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:206377 9.5 e1272/1247 9:a0112b9a:::benchmark_data_smithi063_263014_object82039:head v 1272'12624, mlcod=1272'12620)", 2024-08-17T17:34:04.401 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:29:32.330955+0000", 2024-08-17T17:34:04.401 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 271.59121742500002, 2024-08-17T17:34:04.401 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.166790033, 2024-08-17T17:34:04.401 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.401 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.401 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.401 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.402 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.402 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.402 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.330955+0000", 2024-08-17T17:34:04.402 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.402 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.402 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.402 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.402 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.330955+0000", 2024-08-17T17:34:04.402 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.402 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.402 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.402 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.403 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.330956+0000", 2024-08-17T17:34:04.403 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.37e-07 2024-08-17T17:34:04.403 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.403 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.403 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.403 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.330961+0000", 2024-08-17T17:34:04.403 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6099999999999997e-06 2024-08-17T17:34:04.403 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.403 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.403 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.403 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.330972+0000", 2024-08-17T17:34:04.403 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0518e-05 2024-08-17T17:34:04.403 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.404 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.404 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.404 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.330978+0000", 2024-08-17T17:34:04.404 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3539999999999996e-06 2024-08-17T17:34:04.404 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.404 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.404 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.404 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.491074+0000", 2024-08-17T17:34:04.404 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.16009595800000001 2024-08-17T17:34:04.404 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.404 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.404 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.404 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.491089+0000", 2024-08-17T17:34:04.405 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4799e-05 2024-08-17T17:34:04.405 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.405 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.405 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.405 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.497715+0000", 2024-08-17T17:34:04.405 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.006626084 2024-08-17T17:34:04.405 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.405 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.405 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.405 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.497745+0000", 2024-08-17T17:34:04.405 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.9972999999999999e-05 2024-08-17T17:34:04.405 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.406 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.406 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.406 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.406 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.406 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:206387 9.9 e1272/1247 9:9c6b4511:::benchmark_data_smithi063_263014_object82049:head v 1272'13201, mlcod=1272'13198)", 2024-08-17T17:34:04.406 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:29:32.335011+0000", 2024-08-17T17:34:04.406 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 271.58716197899997, 2024-08-17T17:34:04.406 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.169889283, 2024-08-17T17:34:04.406 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.406 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.406 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.407 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.407 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.407 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.407 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.335011+0000", 2024-08-17T17:34:04.407 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.407 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.407 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.407 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.407 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.335011+0000", 2024-08-17T17:34:04.407 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.407 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.407 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.407 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.408 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.335011+0000", 2024-08-17T17:34:04.408 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0999999999999998e-07 2024-08-17T17:34:04.408 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.408 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.408 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.408 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.335017+0000", 2024-08-17T17:34:04.408 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3600000000000004e-06 2024-08-17T17:34:04.408 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.408 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.408 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.408 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.491151+0000", 2024-08-17T17:34:04.408 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.15613448499999999 2024-08-17T17:34:04.408 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.409 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.409 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.409 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.491162+0000", 2024-08-17T17:34:04.409 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1381e-05 2024-08-17T17:34:04.409 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.409 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.409 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.409 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.491216+0000", 2024-08-17T17:34:04.409 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3297999999999997e-05 2024-08-17T17:34:04.409 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.409 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.409 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.409 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.491234+0000", 2024-08-17T17:34:04.410 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8148999999999999e-05 2024-08-17T17:34:04.410 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.410 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.410 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.410 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.504813+0000", 2024-08-17T17:34:04.410 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.013579242 2024-08-17T17:34:04.410 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.410 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.410 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.410 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:29:32.504900+0000", 2024-08-17T17:34:04.410 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6657999999999996e-05 2024-08-17T17:34:04.411 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.411 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.411 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.411 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.411 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.411 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:194842 10.12 e1315/1306 10:4a1efe0c:::benchmark_data_smithi063_294527_object57657:head v 1315'10202, mlcod=1315'10200)", 2024-08-17T17:34:04.411 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.321450+0000", 2024-08-17T17:34:04.411 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 60.600722017000002, 2024-08-17T17:34:04.411 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41592500300000002, 2024-08-17T17:34:04.411 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.411 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.411 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.411 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.412 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.412 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.412 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321450+0000", 2024-08-17T17:34:04.412 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.412 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.412 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.412 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.412 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321450+0000", 2024-08-17T17:34:04.412 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.412 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.412 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.412 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.412 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321451+0000", 2024-08-17T17:34:04.413 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.67e-07 2024-08-17T17:34:04.413 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.413 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.413 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.413 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321454+0000", 2024-08-17T17:34:04.413 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5300000000000001e-06 2024-08-17T17:34:04.413 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.413 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.413 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.413 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321460+0000", 2024-08-17T17:34:04.413 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6330000000000001e-06 2024-08-17T17:34:04.413 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.413 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.414 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.414 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321465+0000", 2024-08-17T17:34:04.414 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9300000000000002e-06 2024-08-17T17:34:04.414 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.414 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.414 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.414 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321512+0000", 2024-08-17T17:34:04.414 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7454999999999999e-05 2024-08-17T17:34:04.414 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.414 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.414 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.414 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321531+0000", 2024-08-17T17:34:04.415 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8476999999999999e-05 2024-08-17T17:34:04.415 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.415 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.415 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.415 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.737340+0000", 2024-08-17T17:34:04.415 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41580924000000002 2024-08-17T17:34:04.415 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.415 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.415 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.415 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.737375+0000", 2024-08-17T17:34:04.415 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5471e-05 2024-08-17T17:34:04.415 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.415 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.415 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.416 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.416 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.416 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:194846 10.b e1315/1288 10:dcd470cd:::benchmark_data_smithi063_294527_object57661:head v 1315'12297, mlcod=1315'12295)", 2024-08-17T17:34:04.416 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.321477+0000", 2024-08-17T17:34:04.416 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 60.600695876000003, 2024-08-17T17:34:04.416 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41552626500000001, 2024-08-17T17:34:04.416 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.416 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.416 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.416 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.416 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.416 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.416 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321477+0000", 2024-08-17T17:34:04.417 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.417 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.417 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.417 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.417 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321477+0000", 2024-08-17T17:34:04.417 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.417 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.417 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.417 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.417 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321477+0000", 2024-08-17T17:34:04.417 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6600000000000003e-07 2024-08-17T17:34:04.417 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.417 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.418 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.418 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321484+0000", 2024-08-17T17:34:04.418 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9129999999999997e-06 2024-08-17T17:34:04.418 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.418 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.418 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.418 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321491+0000", 2024-08-17T17:34:04.418 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3150000000000003e-06 2024-08-17T17:34:04.418 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.418 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.418 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.418 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321496+0000", 2024-08-17T17:34:04.419 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2000000000000002e-06 2024-08-17T17:34:04.419 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.419 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.419 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.419 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321546+0000", 2024-08-17T17:34:04.419 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9416000000000003e-05 2024-08-17T17:34:04.419 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.419 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.419 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.419 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321565+0000", 2024-08-17T17:34:04.419 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9171e-05 2024-08-17T17:34:04.419 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.419 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.419 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.419 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.736955+0000", 2024-08-17T17:34:04.420 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41538978599999998 2024-08-17T17:34:04.420 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.420 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.420 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.420 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.737003+0000", 2024-08-17T17:34:04.420 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8198000000000002e-05 2024-08-17T17:34:04.420 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.420 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.420 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.420 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.420 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.420 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:194840 10.4 e1315/1306 10:21446871:::benchmark_data_smithi063_294527_object57655:head v 1315'10259, mlcod=1315'10257)", 2024-08-17T17:34:04.420 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.321510+0000", 2024-08-17T17:34:04.420 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 60.600662690999997, 2024-08-17T17:34:04.421 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.416863237, 2024-08-17T17:34:04.421 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.421 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.421 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.421 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.421 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.421 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.421 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321510+0000", 2024-08-17T17:34:04.421 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.421 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.421 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.421 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.422 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321510+0000", 2024-08-17T17:34:04.422 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.422 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.422 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.422 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.422 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321510+0000", 2024-08-17T17:34:04.422 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5699999999999999e-07 2024-08-17T17:34:04.422 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.422 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.422 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.422 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321512+0000", 2024-08-17T17:34:04.422 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.34e-06 2024-08-17T17:34:04.422 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.423 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.423 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.423 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321516+0000", 2024-08-17T17:34:04.423 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8940000000000003e-06 2024-08-17T17:34:04.423 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.423 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.423 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.423 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321519+0000", 2024-08-17T17:34:04.423 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5890000000000001e-06 2024-08-17T17:34:04.423 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.423 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.423 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.423 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321564+0000", 2024-08-17T17:34:04.423 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4878e-05 2024-08-17T17:34:04.423 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.424 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.424 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.424 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321579+0000", 2024-08-17T17:34:04.424 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5821999999999999e-05 2024-08-17T17:34:04.424 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.424 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.424 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.424 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.738341+0000", 2024-08-17T17:34:04.424 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41676199000000003 2024-08-17T17:34:04.424 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.424 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.424 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.424 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.738373+0000", 2024-08-17T17:34:04.424 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1566999999999998e-05 2024-08-17T17:34:04.424 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.425 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.425 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.425 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.425 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.425 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:194844 10.12 e1315/1306 10:49a75d7b:::benchmark_data_smithi063_294527_object57659:head v 1315'10203, mlcod=1315'10200)", 2024-08-17T17:34:04.425 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.322823+0000", 2024-08-17T17:34:04.425 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 60.599349451999998, 2024-08-17T17:34:04.425 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41833177599999999, 2024-08-17T17:34:04.425 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.425 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.425 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.425 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.425 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.426 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.426 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.322823+0000", 2024-08-17T17:34:04.426 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.426 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.426 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.426 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.426 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.322823+0000", 2024-08-17T17:34:04.426 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.426 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.426 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.426 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.427 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.322824+0000", 2024-08-17T17:34:04.427 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6700000000000003e-07 2024-08-17T17:34:04.427 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.427 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.427 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.427 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.322830+0000", 2024-08-17T17:34:04.427 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1789999999999996e-06 2024-08-17T17:34:04.427 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.427 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.427 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.427 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.322840+0000", 2024-08-17T17:34:04.427 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0342e-05 2024-08-17T17:34:04.428 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.428 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.428 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.428 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.322847+0000", 2024-08-17T17:34:04.428 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8770000000000004e-06 2024-08-17T17:34:04.428 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.428 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.428 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.428 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.323796+0000", 2024-08-17T17:34:04.428 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00094924600000000005 2024-08-17T17:34:04.428 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.428 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.428 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.428 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.323880+0000", 2024-08-17T17:34:04.429 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3496000000000002e-05 2024-08-17T17:34:04.429 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.429 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.429 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.429 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.741122+0000", 2024-08-17T17:34:04.429 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41724261200000001 2024-08-17T17:34:04.429 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.429 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.429 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.429 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.741155+0000", 2024-08-17T17:34:04.429 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.2357000000000002e-05 2024-08-17T17:34:04.429 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.429 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.429 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.430 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.430 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.430 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:194849 10.c e1315/1312 10:38a65d47:::benchmark_data_smithi063_294527_object57664:head v 1315'12025, mlcod=1315'12023)", 2024-08-17T17:34:04.430 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.324072+0000", 2024-08-17T17:34:04.430 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 60.598100123000002, 2024-08-17T17:34:04.430 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41871652599999998, 2024-08-17T17:34:04.430 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.430 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.430 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.430 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.430 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.430 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.430 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.324072+0000", 2024-08-17T17:34:04.430 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.324072+0000", 2024-08-17T17:34:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.324073+0000", 2024-08-17T17:34:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3600000000000004e-07 2024-08-17T17:34:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.324078+0000", 2024-08-17T17:34:04.431 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7129999999999996e-06 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.324087+0000", 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.149e-06 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.324092+0000", 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6450000000000002e-06 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.324161+0000", 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8206000000000004e-05 2024-08-17T17:34:04.432 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.433 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.433 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.433 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.736990+0000", 2024-08-17T17:34:04.433 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41282972299999998 2024-08-17T17:34:04.433 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.433 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.433 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.433 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.742729+0000", 2024-08-17T17:34:04.433 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0057386039999999996 2024-08-17T17:34:04.433 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.433 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.433 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.433 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.742789+0000", 2024-08-17T17:34:04.433 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.995e-05 2024-08-17T17:34:04.433 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.434 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.434 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.434 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.434 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.434 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:194852 10.15 e1315/1306 10:a9b1e74c:::benchmark_data_smithi063_294527_object57667:head v 1315'10134, mlcod=1315'10132)", 2024-08-17T17:34:04.434 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.325815+0000", 2024-08-17T17:34:04.434 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 60.596357844000003, 2024-08-17T17:34:04.434 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41519754199999998, 2024-08-17T17:34:04.434 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.434 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.434 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.434 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.434 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.435 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.435 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325815+0000", 2024-08-17T17:34:04.435 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.435 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.435 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.435 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.435 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325815+0000", 2024-08-17T17:34:04.435 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.435 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.435 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.435 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.435 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325815+0000", 2024-08-17T17:34:04.435 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2900000000000003e-07 2024-08-17T17:34:04.436 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.436 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.436 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.436 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325822+0000", 2024-08-17T17:34:04.436 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8159999999999996e-06 2024-08-17T17:34:04.436 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.436 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.436 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.436 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.736930+0000", 2024-08-17T17:34:04.436 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.411107687 2024-08-17T17:34:04.436 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.436 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.436 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.437 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.736944+0000", 2024-08-17T17:34:04.437 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3869999999999999e-05 2024-08-17T17:34:04.437 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.437 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.437 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.437 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.737286+0000", 2024-08-17T17:34:04.437 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00034230699999999997 2024-08-17T17:34:04.437 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.437 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.437 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.437 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.737305+0000", 2024-08-17T17:34:04.437 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9351000000000002e-05 2024-08-17T17:34:04.437 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.438 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.438 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.438 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.740983+0000", 2024-08-17T17:34:04.438 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0036774379999999999 2024-08-17T17:34:04.438 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.438 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.438 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.438 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.741012+0000", 2024-08-17T17:34:04.438 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.9343999999999999e-05 2024-08-17T17:34:04.438 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.438 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.438 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.438 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.439 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.439 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:194850 10.b e1315/1288 10:d5f8e4e1:::benchmark_data_smithi063_294527_object57665:head v 1315'12298, mlcod=1315'12295)", 2024-08-17T17:34:04.439 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.328059+0000", 2024-08-17T17:34:04.439 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 60.594113878000002, 2024-08-17T17:34:04.439 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.412432615, 2024-08-17T17:34:04.439 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.439 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.439 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.439 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.439 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.439 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.439 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.328059+0000", 2024-08-17T17:34:04.439 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.439 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.439 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.328059+0000", 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.328059+0000", 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0699999999999996e-07 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.328067+0000", 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.199e-06 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.440 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.441 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.737195+0000", 2024-08-17T17:34:04.441 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.40912816800000001 2024-08-17T17:34:04.441 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.441 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.441 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.441 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.737203+0000", 2024-08-17T17:34:04.441 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.5890000000000003e-06 2024-08-17T17:34:04.441 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.441 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.441 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.441 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.737292+0000", 2024-08-17T17:34:04.441 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8620000000000002e-05 2024-08-17T17:34:04.441 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.441 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.441 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.442 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.737316+0000", 2024-08-17T17:34:04.442 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3969999999999999e-05 2024-08-17T17:34:04.442 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.442 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.442 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.442 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.740341+0000", 2024-08-17T17:34:04.442 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0030250540000000001 2024-08-17T17:34:04.442 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.442 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.442 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.442 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.740491+0000", 2024-08-17T17:34:04.442 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015020800000000001 2024-08-17T17:34:04.443 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.443 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.443 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.443 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.443 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.443 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:194867 10.a e1315/1299 10:5aba3e88:::benchmark_data_smithi063_294527_object57682:head v 1315'12056, mlcod=1292'259)", 2024-08-17T17:34:04.443 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.336729+0000", 2024-08-17T17:34:04.443 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 60.585443138000002, 2024-08-17T17:34:04.443 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.40791139900000001, 2024-08-17T17:34:04.443 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:04.443 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:04.443 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:04.443 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:04.443 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.443 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:04.444 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.336729+0000", 2024-08-17T17:34:04.444 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.444 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.444 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.444 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:04.444 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.336729+0000", 2024-08-17T17:34:04.444 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:04.444 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.444 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.444 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:04.444 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.336731+0000", 2024-08-17T17:34:04.444 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3859999999999999e-06 2024-08-17T17:34:04.444 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.444 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.444 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:04.445 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.336740+0000", 2024-08-17T17:34:04.445 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.2979999999999997e-06 2024-08-17T17:34:04.445 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.445 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.445 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:04.445 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.736925+0000", 2024-08-17T17:34:04.445 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.40018459299999998 2024-08-17T17:34:04.445 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.445 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.445 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:04.445 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.736951+0000", 2024-08-17T17:34:04.445 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6665e-05 2024-08-17T17:34:04.445 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.445 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.445 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:04.446 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.738471+0000", 2024-08-17T17:34:04.446 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0015192140000000001 2024-08-17T17:34:04.446 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.446 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.446 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:04.446 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.738492+0000", 2024-08-17T17:34:04.446 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1759999999999998e-05 2024-08-17T17:34:04.446 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.446 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.446 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:04.446 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.744507+0000", 2024-08-17T17:34:04.446 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0060146219999999999 2024-08-17T17:34:04.446 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:04.447 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:04.447 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:04.447 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.744641+0000", 2024-08-17T17:34:04.447 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013386100000000001 2024-08-17T17:34:04.447 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.447 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.447 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.447 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:04.447 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:04.447 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:34:04.447 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:04.476 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:04.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:04.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:04 smithi063 ceph-mon[114756]: osdmap e1343: 8 total, 7 up, 8 in 2024-08-17T17:34:04.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:04 smithi063 ceph-mon[114756]: 1.b scrub starts 2024-08-17T17:34:04.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:04 smithi063 ceph-mon[109843]: osdmap e1343: 8 total, 7 up, 8 in 2024-08-17T17:34:04.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:04 smithi063 ceph-mon[109843]: 1.b scrub starts 2024-08-17T17:34:04.579 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:04.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.612 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:04.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:04.713 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:04.744 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:04.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:04.746 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:04 smithi162 ceph-mon[104668]: osdmap e1343: 8 total, 7 up, 8 in 2024-08-17T17:34:04.746 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:04 smithi162 ceph-mon[104668]: 1.b scrub starts 2024-08-17T17:34:04.846 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:04.882 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:04.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:04.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:04.984 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:05.019 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:05.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:05.121 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:05.162 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:05.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:05.264 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:05.295 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:34:05.300 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:05.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:05.402 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:05.448 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:05.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:05.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:05 smithi063 ceph-mon[114756]: pgmap v2008: 129 pgs: 1 active+undersized+remapped, 38 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 4898/76161 objects degraded (6.431%); 6139/76161 objects misplaced (8.061%); 35 MiB/s, 589 objects/s recovering 2024-08-17T17:34:05.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:05 smithi063 ceph-mon[109843]: pgmap v2008: 129 pgs: 1 active+undersized+remapped, 38 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 4898/76161 objects degraded (6.431%); 6139/76161 objects misplaced (8.061%); 35 MiB/s, 589 objects/s recovering 2024-08-17T17:34:05.550 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:05.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:05 smithi162 ceph-mon[104668]: pgmap v2008: 129 pgs: 1 active+undersized+remapped, 38 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 4898/76161 objects degraded (6.431%); 6139/76161 objects misplaced (8.061%); 35 MiB/s, 589 objects/s recovering 2024-08-17T17:34:05.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.596 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:05.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:05.604 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:34:05.697 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:05.742 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:05.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:05.843 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:05.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:05.910 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:05.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:06.011 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:06.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.044 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:06.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:06.145 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:06.180 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:06.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:06.282 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:06.293 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:06.310 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:06.313 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:06.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:06.414 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:06.446 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:06.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:06.548 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:06.594 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:06.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:06.696 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:06.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.736 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:06.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:06.837 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:06.871 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:06.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:06.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:06.951 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:34:06.952 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:34:06.952 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:34:06.952 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:34:06.952 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:34:06.952 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:34:06.952 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:34:06.952 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:34:06.952 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:34:06.953 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:34:06.953 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:34:06.953 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:34:06.953 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:34:06.953 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:34:06.953 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:34:06.973 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:07.002 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:07.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:07.104 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:07.144 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:07.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:07.246 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:07.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.275 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:07.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:07.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:07.414 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:07.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:07.515 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:07.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:07.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:07.648 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:07.680 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:07.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:07.750 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:34:07.751 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:34:07.782 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:07.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.812 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:07.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:07.913 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:07.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:07.951 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:07.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:07.965 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:07 smithi063 ceph-mon[109843]: pgmap v2009: 129 pgs: 1 active+undersized+remapped, 38 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 4529/75054 objects degraded (6.034%); 6139/75054 objects misplaced (8.179%); 26 MiB/s, 437 objects/s recovering 2024-08-17T17:34:07.965 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:07 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2781130149' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:07.966 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:07 smithi063 ceph-mon[114756]: pgmap v2009: 129 pgs: 1 active+undersized+remapped, 38 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 4529/75054 objects degraded (6.034%); 6139/75054 objects misplaced (8.179%); 26 MiB/s, 437 objects/s recovering 2024-08-17T17:34:07.966 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:07 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2781130149' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:08.052 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:08.083 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:08.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:07 smithi162 ceph-mon[104668]: pgmap v2009: 129 pgs: 1 active+undersized+remapped, 38 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 4529/75054 objects degraded (6.034%); 6139/75054 objects misplaced (8.179%); 26 MiB/s, 437 objects/s recovering 2024-08-17T17:34:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:07 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2781130149' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:08.185 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:08.220 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:08.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:08.322 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:08.360 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:08.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:08.462 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:08.497 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:08.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:08.602 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:08.643 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:08.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:08.744 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:08.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:08 smithi063 ceph-mon[109843]: 10.13 scrub starts 2024-08-17T17:34:08.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:08 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 4529/75054 objects degraded (6.034%), 7 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:08.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:08 smithi063 ceph-mon[114756]: 10.13 scrub starts 2024-08-17T17:34:08.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:08 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 4529/75054 objects degraded (6.034%), 7 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:08.784 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:08.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:08.886 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:08.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:08.918 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:08.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:09.019 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:09.061 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:09.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:08 smithi162 ceph-mon[104668]: 10.13 scrub starts 2024-08-17T17:34:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:08 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 4529/75054 objects degraded (6.034%), 7 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:09.162 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:09.190 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:09.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:09.202 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 4, 5, 6, 7, 0, 1] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:34:09.202 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:34:09.202 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [2, 3, 4, 5, 6, 7, 0, 1] 2024-08-17T17:34:09.202 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd out 2 2024-08-17T17:34:09.292 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:09.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.356 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:09.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:09.458 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:09.506 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:09.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:09.607 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:09.652 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:09.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:09.752 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:34:09.754 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:09.778 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:09.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:09.880 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:09.906 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:09.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:09.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:09.920 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:09 smithi063 ceph-mon[109843]: 10.3 scrub starts 2024-08-17T17:34:09.920 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:09 smithi063 ceph-mon[109843]: pgmap v2010: 129 pgs: 1 active+clean+scrubbing, 39 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 68 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 3897/60924 objects degraded (6.396%); 5142/60924 objects misplaced (8.440%); 21 MiB/s, 352 objects/s recovering 2024-08-17T17:34:09.921 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:09 smithi063 ceph-mon[109843]: 1.b scrub starts 2024-08-17T17:34:09.921 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:09 smithi063 ceph-mon[114756]: 10.3 scrub starts 2024-08-17T17:34:09.921 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:09 smithi063 ceph-mon[114756]: pgmap v2010: 129 pgs: 1 active+clean+scrubbing, 39 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 68 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 3897/60924 objects degraded (6.396%); 5142/60924 objects misplaced (8.440%); 21 MiB/s, 352 objects/s recovering 2024-08-17T17:34:09.921 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:09 smithi063 ceph-mon[114756]: 1.b scrub starts 2024-08-17T17:34:10.007 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:10.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.051 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:10.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:09 smithi162 ceph-mon[104668]: 10.3 scrub starts 2024-08-17T17:34:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:09 smithi162 ceph-mon[104668]: pgmap v2010: 129 pgs: 1 active+clean+scrubbing, 39 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 68 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 3897/60924 objects degraded (6.396%); 5142/60924 objects misplaced (8.440%); 21 MiB/s, 352 objects/s recovering 2024-08-17T17:34:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:09 smithi162 ceph-mon[104668]: 1.b scrub starts 2024-08-17T17:34:10.152 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:10.184 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:10.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.185 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:10.286 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:10.326 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:10.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:10.428 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:10.467 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:10.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:10.568 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:10.595 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:10.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:10.681 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:34:10.681 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:34:10.696 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:10.754 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:10.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:10.856 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:10.895 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:10.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:10.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:10.955 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:10.997 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:11.024 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:11.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:11.126 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:11.159 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:11.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:11.261 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:11.297 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:11.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:11.399 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:11.428 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:11.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:11.530 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:11.560 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:11.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:11.584 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:34:11.637 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:34:11.662 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:11.674 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:11.737 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:11.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:11.841 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:11.884 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:11.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:11.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:11.987 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:11.999 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:11 smithi063 ceph-mon[114756]: pgmap v2011: 129 pgs: 1 active+clean+scrubbing, 39 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 68 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 3897/59724 objects degraded (6.525%); 4742/59724 objects misplaced (7.940%) 2024-08-17T17:34:11.999 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:11 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4108105169' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:11.999 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:11 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:12.000 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:11 smithi063 ceph-mon[109843]: pgmap v2011: 129 pgs: 1 active+clean+scrubbing, 39 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 68 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 3897/59724 objects degraded (6.525%); 4742/59724 objects misplaced (7.940%) 2024-08-17T17:34:12.000 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:11 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4108105169' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:12.000 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:11 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:12.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.019 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:12.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:11 smithi162 ceph-mon[104668]: pgmap v2011: 129 pgs: 1 active+clean+scrubbing, 39 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 68 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 3897/59724 objects degraded (6.525%); 4742/59724 objects misplaced (7.940%) 2024-08-17T17:34:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:11 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4108105169' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:11 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:12.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:12.155 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:12.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:12.257 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:12.286 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:12.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:12.389 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:12.427 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:12.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:12.529 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:12.571 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:12.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:12.673 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:12.704 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:12.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:12.773 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:12 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:12.774 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:12 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:34:12.774 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:12 smithi063 ceph-mon[109843]: osdmap e1344: 8 total, 7 up, 8 in 2024-08-17T17:34:12.774 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:12 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:12.774 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:12 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:34:12.774 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:12 smithi063 ceph-mon[114756]: osdmap e1344: 8 total, 7 up, 8 in 2024-08-17T17:34:12.806 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:12.844 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:12.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:12.946 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:12.974 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:12.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:12.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:13.076 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:12 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:12 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:34:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:12 smithi162 ceph-mon[104668]: osdmap e1344: 8 total, 7 up, 8 in 2024-08-17T17:34:13.117 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:13.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:13.219 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:13.248 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:13.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:13.349 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:13.383 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:13.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:13.485 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:13.520 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:13.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:13.551 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:34:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:34:13] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.51.0" 2024-08-17T17:34:13.622 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:13.652 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:13.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:13.753 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:13.793 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:13.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:13.869 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:13 smithi063 ceph-mon[109843]: 10.13 scrub starts 2024-08-17T17:34:13.869 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:13 smithi063 ceph-mon[109843]: pgmap v2013: 129 pgs: 39 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 953 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 2735/45789 objects degraded (5.973%); 3593/45789 objects misplaced (7.847%) 2024-08-17T17:34:13.869 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:13 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 2735/45789 objects degraded (5.973%), 7 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:13.870 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:13 smithi063 ceph-mon[114756]: 10.13 scrub starts 2024-08-17T17:34:13.870 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:13 smithi063 ceph-mon[114756]: pgmap v2013: 129 pgs: 39 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 953 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 2735/45789 objects degraded (5.973%); 3593/45789 objects misplaced (7.847%) 2024-08-17T17:34:13.870 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:13 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 2735/45789 objects degraded (5.973%), 7 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:13.895 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:13.923 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:13.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:13.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:14.025 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:14.066 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:14.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:13 smithi162 ceph-mon[104668]: 10.13 scrub starts 2024-08-17T17:34:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:13 smithi162 ceph-mon[104668]: pgmap v2013: 129 pgs: 39 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 953 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 2735/45789 objects degraded (5.973%); 3593/45789 objects misplaced (7.847%) 2024-08-17T17:34:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:13 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 2735/45789 objects degraded (5.973%), 7 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:14.168 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.207 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:14.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:14.308 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:14.324 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:14.335 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:14.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:14.437 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:14.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.478 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:14.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:14.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:34:14.579 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:14.606 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:14.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:14.708 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:14.734 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:14.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:14.836 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:14.875 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:14.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:14.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:14.977 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:15.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.007 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:15.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:15.108 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:15.138 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:15.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:15.240 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:15.276 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:15.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:15.378 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:15.418 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:15.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:15.519 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:15.562 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:15.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:15.664 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:15.693 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:15.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:15.795 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:15.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.838 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:15.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:15.939 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:15.974 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:15.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:15.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:15.997 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:15 smithi063 ceph-mon[109843]: 10.3 scrub starts 2024-08-17T17:34:15.997 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:15 smithi063 ceph-mon[109843]: pgmap v2014: 129 pgs: 39 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 856 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s; 2735/41157 objects degraded (6.645%); 3212/41157 objects misplaced (7.804%) 2024-08-17T17:34:15.997 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:15 smithi063 ceph-mon[109843]: 1.b scrub starts 2024-08-17T17:34:15.997 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:15 smithi063 ceph-mon[114756]: 10.3 scrub starts 2024-08-17T17:34:15.997 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:15 smithi063 ceph-mon[114756]: pgmap v2014: 129 pgs: 39 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 856 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s; 2735/41157 objects degraded (6.645%); 3212/41157 objects misplaced (7.804%) 2024-08-17T17:34:15.997 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:15 smithi063 ceph-mon[114756]: 1.b scrub starts 2024-08-17T17:34:16.023 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:15 smithi162 ceph-mon[104668]: 10.3 scrub starts 2024-08-17T17:34:16.024 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:15 smithi162 ceph-mon[104668]: pgmap v2014: 129 pgs: 39 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 856 MiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s; 2735/41157 objects degraded (6.645%); 3212/41157 objects misplaced (7.804%) 2024-08-17T17:34:16.024 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:15 smithi162 ceph-mon[104668]: 1.b scrub starts 2024-08-17T17:34:16.076 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:16.103 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:16.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:16.205 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:16.247 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:16.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:16.349 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:16.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.394 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:16.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:16.496 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:16.534 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:16.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:16.635 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:16.668 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:16.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:16.689 INFO:teuthology.orchestra.run.smithi063.stderr:marked out osd.2. 2024-08-17T17:34:16.770 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:16.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:16.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:16 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1354921020' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-08-17T17:34:16.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:16 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-08-17T17:34:16.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:16 smithi063 ceph-mon[114756]: Client client.admin marked osd.2 out, while it was still marked up 2024-08-17T17:34:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:16 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a[109820]: 2024-08-17T17:34:16.524+0000 7f66fd20a640 -1 mon.a@0(leader).osd e1344 definitely_dead 0 2024-08-17T17:34:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:16 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1354921020' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-08-17T17:34:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:16 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-08-17T17:34:16.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:16 smithi063 ceph-mon[109843]: Client client.admin marked osd.2 out, while it was still marked up 2024-08-17T17:34:16.791 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:16.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:16.893 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:16.916 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:16.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:16.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:16.990 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:16.990 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:16 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1354921020' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-08-17T17:34:16.990 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:16 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd out", "ids": ["2"]} : dispatch 2024-08-17T17:34:16.991 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:16 smithi162 ceph-mon[104668]: Client client.admin marked osd.2 out, while it was still marked up 2024-08-17T17:34:17.018 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:17.040 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:17.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:17.142 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:17.164 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:17.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:17.266 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:17.287 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:17.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:17.389 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:17.411 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:17.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:17.512 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:17.534 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:17.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:17.636 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:17.659 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:17.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:17.761 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:17.783 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:17.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:17.885 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:17.907 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:17.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:17.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:17.969 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:17 smithi063 ceph-mon[109843]: pgmap v2015: 129 pgs: 39 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 830 MiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s; 2322/39918 objects degraded (5.817%); 3212/39918 objects misplaced (8.046%) 2024-08-17T17:34:17.969 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:17 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-08-17T17:34:17.969 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:17 smithi063 ceph-mon[109843]: osdmap e1345: 8 total, 7 up, 7 in 2024-08-17T17:34:17.970 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:17 smithi063 ceph-mon[114756]: pgmap v2015: 129 pgs: 39 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 830 MiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s; 2322/39918 objects degraded (5.817%); 3212/39918 objects misplaced (8.046%) 2024-08-17T17:34:17.970 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:17 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-08-17T17:34:17.970 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:17 smithi063 ceph-mon[114756]: osdmap e1345: 8 total, 7 up, 7 in 2024-08-17T17:34:18.009 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:18.030 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:18.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:17 smithi162 ceph-mon[104668]: pgmap v2015: 129 pgs: 39 active+clean+remapped, 14 active+undersized, 7 active+undersized+degraded, 69 active+clean; 830 MiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.17k op/s; 2322/39918 objects degraded (5.817%); 3212/39918 objects misplaced (8.046%) 2024-08-17T17:34:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:17 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2024-08-17T17:34:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:17 smithi162 ceph-mon[104668]: osdmap e1345: 8 total, 7 up, 7 in 2024-08-17T17:34:18.132 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:18.154 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:18.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:18.256 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:18.278 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:18.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:18.380 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:18.402 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:18.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:18.430 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:18.444 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:34:18.444 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:34:18.463 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:34:18.463 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:34:18.480 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:34:18.480 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:34:18.496 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:34:18.496 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:34:18.504 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:18.513 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:34:18.513 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:34:18.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.526 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:18.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:18.529 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:34:18.529 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:34:18.530 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:34:18.546 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:34:18.546 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:34:18.628 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:18.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.652 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:18.652 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:18.753 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:18.776 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:18.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:18 smithi063 ceph-mon[114756]: osdmap e1346: 8 total, 7 up, 7 in 2024-08-17T17:34:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:18 smithi063 ceph-mon[114756]: 10.a starting backfill to osd.7 from (0'0,0'0] MAX to 1344'16441 2024-08-17T17:34:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:18 smithi063 ceph-mon[114756]: 10.7 starting backfill to osd.4 from (0'0,0'0] MAX to 1344'12192 2024-08-17T17:34:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:18 smithi063 ceph-mon[114756]: 10.17 starting backfill to osd.4 from (0'0,0'0] MAX to 1344'12186 2024-08-17T17:34:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:18 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 2322/39918 objects degraded (5.817%), 7 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:18 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2007201154' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:18 smithi063 ceph-mon[114756]: osdmap e1347: 8 total, 7 up, 7 in 2024-08-17T17:34:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:18 smithi063 ceph-mon[109843]: osdmap e1346: 8 total, 7 up, 7 in 2024-08-17T17:34:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:18 smithi063 ceph-mon[109843]: 10.a starting backfill to osd.7 from (0'0,0'0] MAX to 1344'16441 2024-08-17T17:34:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:18 smithi063 ceph-mon[109843]: 10.7 starting backfill to osd.4 from (0'0,0'0] MAX to 1344'12192 2024-08-17T17:34:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:18 smithi063 ceph-mon[109843]: 10.17 starting backfill to osd.4 from (0'0,0'0] MAX to 1344'12186 2024-08-17T17:34:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:18 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 2322/39918 objects degraded (5.817%), 7 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:18 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2007201154' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:18.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:18 smithi063 ceph-mon[109843]: osdmap e1347: 8 total, 7 up, 7 in 2024-08-17T17:34:18.879 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:18.921 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:18.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:18.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:19.022 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:19.047 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:19.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:18 smithi162 ceph-mon[104668]: osdmap e1346: 8 total, 7 up, 7 in 2024-08-17T17:34:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:18 smithi162 ceph-mon[104668]: 10.a starting backfill to osd.7 from (0'0,0'0] MAX to 1344'16441 2024-08-17T17:34:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:18 smithi162 ceph-mon[104668]: 10.7 starting backfill to osd.4 from (0'0,0'0] MAX to 1344'12192 2024-08-17T17:34:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:18 smithi162 ceph-mon[104668]: 10.17 starting backfill to osd.4 from (0'0,0'0] MAX to 1344'12186 2024-08-17T17:34:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:18 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 2322/39918 objects degraded (5.817%), 7 pgs degraded, 21 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:18 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2007201154' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:18 smithi162 ceph-mon[104668]: osdmap e1347: 8 total, 7 up, 7 in 2024-08-17T17:34:19.148 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:19.179 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:19.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:19.281 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:19.306 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:19.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:19.407 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:19.431 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:19.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:19.533 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:19.556 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:19.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:19.658 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:19.682 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:19.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:19.784 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:19.816 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:19.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:19.838 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:34:19.839 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:34:19.918 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:19.955 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:19.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:19.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:19.986 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[109843]: pgmap v2018: 129 pgs: 11 unknown, 8 remapped+peering, 16 peering, 33 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 44 active+clean; 540 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.57k op/s; 1133/25968 objects degraded (4.363%); 1217/25968 objects misplaced (4.687%) 2024-08-17T17:34:19.986 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[109843]: 10.5 starting backfill to osd.5 from (0'0,0'0] MAX to 1344'15560 2024-08-17T17:34:19.986 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[109843]: 10.3 starting backfill to osd.0 from (0'0,0'0] MAX to 1344'12346 2024-08-17T17:34:19.986 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[109843]: 10.13 starting backfill to osd.0 from (0'0,0'0] MAX to 1344'12217 2024-08-17T17:34:19.986 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[109843]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 1344'16641 2024-08-17T17:34:19.986 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[109843]: 10.9 starting backfill to osd.4 from (0'0,0'0] MAX to 1344'12132 2024-08-17T17:34:19.987 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[109843]: 10.19 starting backfill to osd.4 from (0'0,0'0] MAX to 1344'12234 2024-08-17T17:34:19.987 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[109843]: 10.15 starting backfill to osd.5 from (0'0,0'0] MAX to 1344'12356 2024-08-17T17:34:19.987 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[109843]: Health check failed: Reduced data availability: 5 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T17:34:19.987 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[109843]: 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 1344'16650 2024-08-17T17:34:19.987 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[109843]: 10.18 starting backfill to osd.5 from (0'0,0'0] MAX to 1344'12294 2024-08-17T17:34:19.987 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[109843]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 1344'12311 2024-08-17T17:34:19.987 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[109843]: osdmap e1348: 8 total, 7 up, 7 in 2024-08-17T17:34:19.987 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[114756]: pgmap v2018: 129 pgs: 11 unknown, 8 remapped+peering, 16 peering, 33 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 44 active+clean; 540 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.57k op/s; 1133/25968 objects degraded (4.363%); 1217/25968 objects misplaced (4.687%) 2024-08-17T17:34:19.987 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[114756]: 10.5 starting backfill to osd.5 from (0'0,0'0] MAX to 1344'15560 2024-08-17T17:34:19.987 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[114756]: 10.3 starting backfill to osd.0 from (0'0,0'0] MAX to 1344'12346 2024-08-17T17:34:19.988 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[114756]: 10.13 starting backfill to osd.0 from (0'0,0'0] MAX to 1344'12217 2024-08-17T17:34:19.988 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[114756]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 1344'16641 2024-08-17T17:34:19.988 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[114756]: 10.9 starting backfill to osd.4 from (0'0,0'0] MAX to 1344'12132 2024-08-17T17:34:19.988 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[114756]: 10.19 starting backfill to osd.4 from (0'0,0'0] MAX to 1344'12234 2024-08-17T17:34:19.988 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[114756]: 10.15 starting backfill to osd.5 from (0'0,0'0] MAX to 1344'12356 2024-08-17T17:34:19.988 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[114756]: Health check failed: Reduced data availability: 5 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T17:34:19.988 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[114756]: 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 1344'16650 2024-08-17T17:34:19.988 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[114756]: 10.18 starting backfill to osd.5 from (0'0,0'0] MAX to 1344'12294 2024-08-17T17:34:19.988 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[114756]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 1344'12311 2024-08-17T17:34:19.988 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:19 smithi063 ceph-mon[114756]: osdmap e1348: 8 total, 7 up, 7 in 2024-08-17T17:34:20.058 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:20.095 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:19 smithi162 ceph-mon[104668]: pgmap v2018: 129 pgs: 11 unknown, 8 remapped+peering, 16 peering, 33 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 44 active+clean; 540 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.57k op/s; 1133/25968 objects degraded (4.363%); 1217/25968 objects misplaced (4.687%) 2024-08-17T17:34:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:19 smithi162 ceph-mon[104668]: 10.5 starting backfill to osd.5 from (0'0,0'0] MAX to 1344'15560 2024-08-17T17:34:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:19 smithi162 ceph-mon[104668]: 10.3 starting backfill to osd.0 from (0'0,0'0] MAX to 1344'12346 2024-08-17T17:34:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:19 smithi162 ceph-mon[104668]: 10.13 starting backfill to osd.0 from (0'0,0'0] MAX to 1344'12217 2024-08-17T17:34:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:19 smithi162 ceph-mon[104668]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 1344'16641 2024-08-17T17:34:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:19 smithi162 ceph-mon[104668]: 10.9 starting backfill to osd.4 from (0'0,0'0] MAX to 1344'12132 2024-08-17T17:34:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:19 smithi162 ceph-mon[104668]: 10.19 starting backfill to osd.4 from (0'0,0'0] MAX to 1344'12234 2024-08-17T17:34:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:19 smithi162 ceph-mon[104668]: 10.15 starting backfill to osd.5 from (0'0,0'0] MAX to 1344'12356 2024-08-17T17:34:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:19 smithi162 ceph-mon[104668]: Health check failed: Reduced data availability: 5 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-08-17T17:34:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:19 smithi162 ceph-mon[104668]: 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 1344'16650 2024-08-17T17:34:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:19 smithi162 ceph-mon[104668]: 10.18 starting backfill to osd.5 from (0'0,0'0] MAX to 1344'12294 2024-08-17T17:34:20.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:19 smithi162 ceph-mon[104668]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 1344'12311 2024-08-17T17:34:20.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:19 smithi162 ceph-mon[104668]: osdmap e1348: 8 total, 7 up, 7 in 2024-08-17T17:34:20.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:20.198 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:20.236 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:20.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:20.338 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:20.378 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:20.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:20.413 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:34:20.413 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:34:20.413 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:34:20.413 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:34:20.414 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:34:20.480 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:20.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.515 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:20.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:20.615 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:20.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.654 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:20.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:20.755 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:20.783 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:20.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:20.885 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:20.912 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:20.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:20.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:21.013 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:21.038 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:21.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:21.140 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:21.167 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:21.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:21.173 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:34:21.269 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:21.295 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:21.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:21.397 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:21.420 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:21.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:21.522 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:21.545 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:21.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:21.647 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:21.670 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:21.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:21.772 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:21.799 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:21.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:21.840 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:34:21.901 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:21.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:21.925 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:21.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:22.026 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:21 smithi063 ceph-mon[109843]: pgmap v2021: 129 pgs: 11 unknown, 8 remapped+peering, 16 peering, 33 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 44 active+clean; 526 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 1133/25326 objects degraded (4.474%); 1003/25326 objects misplaced (3.960%) 2024-08-17T17:34:22.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:21 smithi063 ceph-mon[109843]: osdmap e1349: 8 total, 7 up, 7 in 2024-08-17T17:34:22.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:21 smithi063 ceph-mon[114756]: pgmap v2021: 129 pgs: 11 unknown, 8 remapped+peering, 16 peering, 33 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 44 active+clean; 526 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 1133/25326 objects degraded (4.474%); 1003/25326 objects misplaced (3.960%) 2024-08-17T17:34:22.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:21 smithi063 ceph-mon[114756]: osdmap e1349: 8 total, 7 up, 7 in 2024-08-17T17:34:22.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.051 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:22.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:21 smithi162 ceph-mon[104668]: pgmap v2021: 129 pgs: 11 unknown, 8 remapped+peering, 16 peering, 33 active+clean+remapped, 11 active+undersized, 6 active+undersized+degraded, 44 active+clean; 526 MiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.22k op/s; 1133/25326 objects degraded (4.474%); 1003/25326 objects misplaced (3.960%) 2024-08-17T17:34:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:21 smithi162 ceph-mon[104668]: osdmap e1349: 8 total, 7 up, 7 in 2024-08-17T17:34:22.152 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:22.175 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:22.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:22.277 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:22.299 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:22.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:22.401 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:22.425 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:22.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:22.526 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:22.549 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:22.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:22.651 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:22.673 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:22.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:22.775 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:22.806 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:22.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:22.907 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:22.928 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:22.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:22.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:22.959 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:22 smithi063 ceph-mon[114756]: osdmap e1350: 8 total, 7 up, 7 in 2024-08-17T17:34:22.959 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:22 smithi063 ceph-mon[109843]: osdmap e1350: 8 total, 7 up, 7 in 2024-08-17T17:34:22.996 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 7, 0, 1] out_osds: [2] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:34:22.996 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:34:22.996 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-08-17T17:34:22.997 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:34:22.997 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:34:22.997 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-08-17T17:34:23.020 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:23.030 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:23.053 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:23.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:23.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:22 smithi162 ceph-mon[104668]: osdmap e1350: 8 total, 7 up, 7 in 2024-08-17T17:34:23.155 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:23.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.192 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:23.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:23.293 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:23.318 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:23.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:23.420 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:23.450 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:23.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:23.552 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:23.574 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:23.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:23.647 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:34:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:34:23] "GET /metrics HTTP/1.1" 200 36583 "" "Prometheus/2.51.0" 2024-08-17T17:34:23.675 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:23.697 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:23.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:23.799 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:23.820 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:23.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:23.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:23.934 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:23 smithi063 ceph-mon[114756]: pgmap v2024: 129 pgs: 1 active+remapped+backfilling, 5 remapped+peering, 3 peering, 48 active+clean+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 5 active+undersized+degraded, 55 active+clean; 431 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 436 op/s; 472/20727 objects degraded (2.277%); 1772/20727 objects misplaced (8.549%); 45 MiB/s, 715 objects/s recovering 2024-08-17T17:34:23.934 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:23 smithi063 ceph-mon[114756]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 4 pgs peering) 2024-08-17T17:34:23.934 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:23 smithi063 ceph-mon[114756]: osdmap e1351: 8 total, 7 up, 7 in 2024-08-17T17:34:23.934 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:23 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 472/20727 objects degraded (2.277%), 5 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:23.934 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:23 smithi063 ceph-mon[114756]: osdmap e1352: 8 total, 7 up, 7 in 2024-08-17T17:34:23.935 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:23 smithi063 ceph-mon[109843]: pgmap v2024: 129 pgs: 1 active+remapped+backfilling, 5 remapped+peering, 3 peering, 48 active+clean+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 5 active+undersized+degraded, 55 active+clean; 431 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 436 op/s; 472/20727 objects degraded (2.277%); 1772/20727 objects misplaced (8.549%); 45 MiB/s, 715 objects/s recovering 2024-08-17T17:34:23.935 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:23 smithi063 ceph-mon[109843]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 4 pgs peering) 2024-08-17T17:34:23.935 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:23 smithi063 ceph-mon[109843]: osdmap e1351: 8 total, 7 up, 7 in 2024-08-17T17:34:23.935 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:23 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 472/20727 objects degraded (2.277%), 5 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:23.935 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:23 smithi063 ceph-mon[109843]: osdmap e1352: 8 total, 7 up, 7 in 2024-08-17T17:34:23.945 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:23.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:23.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:24.046 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:24.068 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:24.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:23 smithi162 ceph-mon[104668]: pgmap v2024: 129 pgs: 1 active+remapped+backfilling, 5 remapped+peering, 3 peering, 48 active+clean+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 5 active+undersized+degraded, 55 active+clean; 431 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 436 op/s; 472/20727 objects degraded (2.277%); 1772/20727 objects misplaced (8.549%); 45 MiB/s, 715 objects/s recovering 2024-08-17T17:34:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:23 smithi162 ceph-mon[104668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 4 pgs peering) 2024-08-17T17:34:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:23 smithi162 ceph-mon[104668]: osdmap e1351: 8 total, 7 up, 7 in 2024-08-17T17:34:24.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:23 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 472/20727 objects degraded (2.277%), 5 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:24.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:23 smithi162 ceph-mon[104668]: osdmap e1352: 8 total, 7 up, 7 in 2024-08-17T17:34:24.170 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:24.199 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:24.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:24.300 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:24.328 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:24.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:24.430 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:24.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.463 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:24.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:24.564 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:24.611 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:24.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:24.713 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:24.748 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:24.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:24.850 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:24.886 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:24.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:24.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:24.988 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:25.023 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:25.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:25.125 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:25.162 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:25.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:25.263 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:25 smithi063 ceph-mon[109843]: osdmap e1353: 8 total, 7 up, 7 in 2024-08-17T17:34:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:25 smithi063 ceph-mon[109843]: pgmap v2028: 129 pgs: 1 active+remapped+backfilling, 5 remapped+peering, 3 peering, 48 active+clean+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 5 active+undersized+degraded, 55 active+clean; 421 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 472/20286 objects degraded (2.327%); 1625/20286 objects misplaced (8.010%); 46 MiB/s, 739 objects/s recovering 2024-08-17T17:34:25.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:25 smithi063 ceph-mon[114756]: osdmap e1353: 8 total, 7 up, 7 in 2024-08-17T17:34:25.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:25 smithi063 ceph-mon[114756]: pgmap v2028: 129 pgs: 1 active+remapped+backfilling, 5 remapped+peering, 3 peering, 48 active+clean+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 5 active+undersized+degraded, 55 active+clean; 421 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 472/20286 objects degraded (2.327%); 1625/20286 objects misplaced (8.010%); 46 MiB/s, 739 objects/s recovering 2024-08-17T17:34:25.294 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:25.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:25.396 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:25.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.433 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:25.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:25.533 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:25.568 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:25.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:25.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:25 smithi162 ceph-mon[104668]: osdmap e1353: 8 total, 7 up, 7 in 2024-08-17T17:34:25.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:25 smithi162 ceph-mon[104668]: pgmap v2028: 129 pgs: 1 active+remapped+backfilling, 5 remapped+peering, 3 peering, 48 active+clean+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 5 active+undersized+degraded, 55 active+clean; 421 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 472/20286 objects degraded (2.327%); 1625/20286 objects misplaced (8.010%); 46 MiB/s, 739 objects/s recovering 2024-08-17T17:34:25.670 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:25.705 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:25.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:25.807 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:25.836 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:25.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:25.938 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:25.971 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:25.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:25.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:26.073 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:26.111 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:26.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:26.131 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:34:26.213 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:26.234 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:26.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:26.336 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:26.357 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:26.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:26.442 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:26 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2127345602' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:34:26.442 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:26 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:34:26.443 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:26 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2127345602' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:34:26.443 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:26 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:34:26.458 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:26.489 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:26.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:26.590 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:26.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:26 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2127345602' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:34:26.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:26 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:34:26.624 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:26.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:26.725 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:26.754 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:26.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:26.856 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:26.889 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:26.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:26.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:26.991 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:27.017 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:27.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:27.119 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:27.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.154 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:27.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:27.255 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:27.289 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:27.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:27.381 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:27 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:27.381 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:27 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:34:27.381 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:27 smithi063 ceph-mon[109843]: osdmap e1354: 8 total, 7 up, 7 in 2024-08-17T17:34:27.381 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:27 smithi063 ceph-mon[109843]: 10.4 scrub starts 2024-08-17T17:34:27.381 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:27 smithi063 ceph-mon[109843]: 10.4 scrub ok 2024-08-17T17:34:27.381 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:27 smithi063 ceph-mon[109843]: pgmap v2030: 129 pgs: 1 active+remapped+backfilling, 4 remapped+peering, 3 peering, 49 active+clean+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 5 active+undersized+degraded, 55 active+clean; 408 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 472/19659 objects degraded (2.401%); 1845/19659 objects misplaced (9.385%) 2024-08-17T17:34:27.381 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:27 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:27.381 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:27 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:34:27.382 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:27 smithi063 ceph-mon[114756]: osdmap e1354: 8 total, 7 up, 7 in 2024-08-17T17:34:27.382 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:27 smithi063 ceph-mon[114756]: 10.4 scrub starts 2024-08-17T17:34:27.382 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:27 smithi063 ceph-mon[114756]: 10.4 scrub ok 2024-08-17T17:34:27.382 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:27 smithi063 ceph-mon[114756]: pgmap v2030: 129 pgs: 1 active+remapped+backfilling, 4 remapped+peering, 3 peering, 49 active+clean+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 5 active+undersized+degraded, 55 active+clean; 408 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 472/19659 objects degraded (2.401%); 1845/19659 objects misplaced (9.385%) 2024-08-17T17:34:27.390 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:27.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.426 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:27.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:27.527 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:27.562 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:27.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:27.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:27 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:27.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:27 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:34:27.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:27 smithi162 ceph-mon[104668]: osdmap e1354: 8 total, 7 up, 7 in 2024-08-17T17:34:27.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:27 smithi162 ceph-mon[104668]: 10.4 scrub starts 2024-08-17T17:34:27.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:27 smithi162 ceph-mon[104668]: 10.4 scrub ok 2024-08-17T17:34:27.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:27 smithi162 ceph-mon[104668]: pgmap v2030: 129 pgs: 1 active+remapped+backfilling, 4 remapped+peering, 3 peering, 49 active+clean+remapped, 1 active+remapped+backfill_wait, 11 active+undersized, 5 active+undersized+degraded, 55 active+clean; 408 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 88 op/s; 472/19659 objects degraded (2.401%); 1845/19659 objects misplaced (9.385%) 2024-08-17T17:34:27.663 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:27.695 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:27.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:27.797 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:27.827 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:27.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:27.929 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:27.965 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:27.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:27.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:28.066 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:28.096 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:28.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:28.198 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:28.231 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:28.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:28.333 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:28.373 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:28.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:28.475 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:28.513 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:28.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:28.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:28 smithi063 ceph-mon[109843]: 1.b scrub starts 2024-08-17T17:34:28.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:28 smithi063 ceph-mon[109843]: 1.b scrub ok 2024-08-17T17:34:28.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:28 smithi063 ceph-mon[109843]: 10.6 scrub starts 2024-08-17T17:34:28.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:28 smithi063 ceph-mon[109843]: 10.6 scrub ok 2024-08-17T17:34:28.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:28 smithi063 ceph-mon[109843]: 10.1 scrub starts 2024-08-17T17:34:28.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:28 smithi063 ceph-mon[109843]: 10.1 scrub ok 2024-08-17T17:34:28.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:28 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 472/19659 objects degraded (2.401%), 5 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:28.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:28 smithi063 ceph-mon[114756]: 1.b scrub starts 2024-08-17T17:34:28.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:28 smithi063 ceph-mon[114756]: 1.b scrub ok 2024-08-17T17:34:28.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:28 smithi063 ceph-mon[114756]: 10.6 scrub starts 2024-08-17T17:34:28.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:28 smithi063 ceph-mon[114756]: 10.6 scrub ok 2024-08-17T17:34:28.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:28 smithi063 ceph-mon[114756]: 10.1 scrub starts 2024-08-17T17:34:28.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:28 smithi063 ceph-mon[114756]: 10.1 scrub ok 2024-08-17T17:34:28.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:28 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 472/19659 objects degraded (2.401%), 5 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:28.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:28 smithi162 ceph-mon[104668]: 1.b scrub starts 2024-08-17T17:34:28.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:28 smithi162 ceph-mon[104668]: 1.b scrub ok 2024-08-17T17:34:28.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:28 smithi162 ceph-mon[104668]: 10.6 scrub starts 2024-08-17T17:34:28.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:28 smithi162 ceph-mon[104668]: 10.6 scrub ok 2024-08-17T17:34:28.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:28 smithi162 ceph-mon[104668]: 10.1 scrub starts 2024-08-17T17:34:28.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:28 smithi162 ceph-mon[104668]: 10.1 scrub ok 2024-08-17T17:34:28.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:28 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 472/19659 objects degraded (2.401%), 5 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:28.616 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:28.652 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:28.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:28.657 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Removed 137182 objects 2024-08-17T17:34:28.657 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Clean up completed and total clean up time :132.585 2024-08-17T17:34:28.696 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:28.755 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:28.777 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:28.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:28.879 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:28.901 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:28.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:28.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:28.966 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:29.003 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:29.027 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:29.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:29.128 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:29.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.152 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:29.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:29.253 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:29.268 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:29 smithi063 ceph-mon[114756]: 10.16 scrub starts 2024-08-17T17:34:29.268 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:29 smithi063 ceph-mon[114756]: 10.16 scrub ok 2024-08-17T17:34:29.269 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:29 smithi063 ceph-mon[114756]: pgmap v2031: 129 pgs: 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 63 active+clean; 141 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 136/6849 objects degraded (1.986%); 1036/6849 objects misplaced (15.126%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:34:29.269 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:29 smithi063 ceph-mon[109843]: 10.16 scrub starts 2024-08-17T17:34:29.269 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:29 smithi063 ceph-mon[109843]: 10.16 scrub ok 2024-08-17T17:34:29.269 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:29 smithi063 ceph-mon[109843]: pgmap v2031: 129 pgs: 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 63 active+clean; 141 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 136/6849 objects degraded (1.986%); 1036/6849 objects misplaced (15.126%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:34:29.275 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:34:29.275 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:34:29.275 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:34:29.275 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:34:29.275 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.275 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:217506 10.b 10.9360ed7b (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T17:34:29.276 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:24.905438+0000", 2024-08-17T17:34:29.276 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 64.367042487999996, 2024-08-17T17:34:29.276 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0222552789999999, 2024-08-17T17:34:29.276 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.276 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.276 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:34:29.276 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.276 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.276 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.276 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 217506 2024-08-17T17:34:29.277 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.277 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.277 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.277 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.277 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.905438+0000", 2024-08-17T17:34:29.277 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.277 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.277 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.277 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.277 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.905438+0000", 2024-08-17T17:34:29.277 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.278 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.278 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.278 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.278 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.905439+0000", 2024-08-17T17:34:29.278 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4640000000000001e-06 2024-08-17T17:34:29.278 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.278 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.278 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.278 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.905444+0000", 2024-08-17T17:34:29.278 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8010000000000003e-06 2024-08-17T17:34:29.278 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.278 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.279 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.279 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.905459+0000", 2024-08-17T17:34:29.279 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5187e-05 2024-08-17T17:34:29.279 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.279 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.279 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.279 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.905470+0000", 2024-08-17T17:34:29.279 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0499999999999999e-05 2024-08-17T17:34:29.279 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.279 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.279 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.279 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.912193+0000", 2024-08-17T17:34:29.280 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0067235760000000002 2024-08-17T17:34:29.280 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.280 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.280 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.280 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.912202+0000", 2024-08-17T17:34:29.280 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4509999999999992e-06 2024-08-17T17:34:29.280 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.280 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.280 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.280 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.927680+0000", 2024-08-17T17:34:29.280 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0154780729999999 2024-08-17T17:34:29.280 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.280 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.281 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.281 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.927693+0000", 2024-08-17T17:34:29.281 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3227000000000001e-05 2024-08-17T17:34:29.281 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.281 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.281 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.281 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.281 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.281 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:217510 10.b 10.49d38fbb (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T17:34:29.281 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:24.905582+0000", 2024-08-17T17:34:29.281 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 64.366898276000001, 2024-08-17T17:34:29.281 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.022145431, 2024-08-17T17:34:29.282 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.282 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.282 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:34:29.282 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.282 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.282 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.282 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 217510 2024-08-17T17:34:29.282 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.282 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.282 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.282 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.282 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.905582+0000", 2024-08-17T17:34:29.282 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.283 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.283 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.283 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.283 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.905582+0000", 2024-08-17T17:34:29.283 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.283 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.283 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.283 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.283 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.905582+0000", 2024-08-17T17:34:29.283 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1500000000000005e-07 2024-08-17T17:34:29.283 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.283 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.283 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.284 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.905588+0000", 2024-08-17T17:34:29.284 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8209999999999998e-06 2024-08-17T17:34:29.284 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.284 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.284 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.284 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.905598+0000", 2024-08-17T17:34:29.284 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0016e-05 2024-08-17T17:34:29.284 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.284 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.284 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.284 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.905604+0000", 2024-08-17T17:34:29.285 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9560000000000002e-06 2024-08-17T17:34:29.285 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.285 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.285 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.285 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.912222+0000", 2024-08-17T17:34:29.285 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0066179469999999999 2024-08-17T17:34:29.285 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.285 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.285 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.285 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.912227+0000", 2024-08-17T17:34:29.285 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9280000000000001e-06 2024-08-17T17:34:29.286 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.286 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.286 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.286 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.927720+0000", 2024-08-17T17:34:29.286 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.015492558 2024-08-17T17:34:29.286 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.286 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.286 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.286 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.927727+0000", 2024-08-17T17:34:29.286 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6899999999999992e-06 2024-08-17T17:34:29.286 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.287 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.287 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.287 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.287 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.287 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:217523 10.b 10.bd5e52db (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T17:34:29.287 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:24.919995+0000", 2024-08-17T17:34:29.287 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 64.352485419999994, 2024-08-17T17:34:29.287 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0077610770000001, 2024-08-17T17:34:29.287 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.287 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.288 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:34:29.288 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.288 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.288 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.288 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 217523 2024-08-17T17:34:29.288 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.288 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.288 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.288 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.288 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.919995+0000", 2024-08-17T17:34:29.288 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.288 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.289 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.289 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.289 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.919995+0000", 2024-08-17T17:34:29.289 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.289 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.289 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.289 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.289 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.919997+0000", 2024-08-17T17:34:29.289 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9530000000000002e-06 2024-08-17T17:34:29.289 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.290 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.290 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.290 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.920004+0000", 2024-08-17T17:34:29.290 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0369999999999997e-06 2024-08-17T17:34:29.290 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.290 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.290 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.290 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.920022+0000", 2024-08-17T17:34:29.290 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8128999999999999e-05 2024-08-17T17:34:29.290 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.290 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.290 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.291 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.920032+0000", 2024-08-17T17:34:29.291 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0212e-05 2024-08-17T17:34:29.291 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.291 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.291 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.291 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.920106+0000", 2024-08-17T17:34:29.291 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3930999999999994e-05 2024-08-17T17:34:29.291 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.291 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.291 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.291 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.920115+0000", 2024-08-17T17:34:29.292 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.098e-06 2024-08-17T17:34:29.292 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.292 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.292 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.292 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.927748+0000", 2024-08-17T17:34:29.292 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0076332859999999 2024-08-17T17:34:29.292 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.292 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.292 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.292 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.927756+0000", 2024-08-17T17:34:29.292 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4309999999999997e-06 2024-08-17T17:34:29.293 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.293 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.293 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.293 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.293 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.293 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:217549 10.b 10.f7f5cf4b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T17:34:29.293 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:24.964708+0000", 2024-08-17T17:34:29.293 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 64.307772599000003, 2024-08-17T17:34:29.293 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.96307531899999999, 2024-08-17T17:34:29.293 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.293 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.293 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:34:29.294 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.294 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.294 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.294 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 217549 2024-08-17T17:34:29.294 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.294 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.294 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.294 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.294 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.964708+0000", 2024-08-17T17:34:29.295 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.295 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.295 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.295 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.295 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.964708+0000", 2024-08-17T17:34:29.295 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.295 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.295 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.295 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.295 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.964710+0000", 2024-08-17T17:34:29.295 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6989999999999999e-06 2024-08-17T17:34:29.296 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.296 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.296 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.296 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.964720+0000", 2024-08-17T17:34:29.296 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7340000000000003e-06 2024-08-17T17:34:29.296 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.296 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.296 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.296 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.964749+0000", 2024-08-17T17:34:29.296 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.9099999999999999e-05 2024-08-17T17:34:29.296 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.297 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.297 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.297 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.964760+0000", 2024-08-17T17:34:29.297 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0745e-05 2024-08-17T17:34:29.297 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.297 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.297 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.297 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.964831+0000", 2024-08-17T17:34:29.297 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1575000000000005e-05 2024-08-17T17:34:29.297 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.297 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.297 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.964841+0000", 2024-08-17T17:34:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4469999999999995e-06 2024-08-17T17:34:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.927776+0000", 2024-08-17T17:34:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.96293531499999996 2024-08-17T17:34:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.298 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.927783+0000", 2024-08-17T17:34:29.299 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7039999999999997e-06 2024-08-17T17:34:29.299 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.299 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.299 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.299 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.299 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.299 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:217570 10.b 10.45bfe3cb (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T17:34:29.299 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:24.998012+0000", 2024-08-17T17:34:29.299 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 64.274468475999996, 2024-08-17T17:34:29.299 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.929800501, 2024-08-17T17:34:29.299 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.300 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.300 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:34:29.300 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.300 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.300 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.300 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 217570 2024-08-17T17:34:29.300 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.300 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.300 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.300 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.301 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.998012+0000", 2024-08-17T17:34:29.301 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.301 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.301 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.301 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.301 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.998012+0000", 2024-08-17T17:34:29.301 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.301 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.301 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.301 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.301 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.998014+0000", 2024-08-17T17:34:29.301 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4169999999999999e-06 2024-08-17T17:34:29.302 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.302 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.302 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.302 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.998027+0000", 2024-08-17T17:34:29.302 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2775999999999999e-05 2024-08-17T17:34:29.302 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.302 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.302 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.302 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.998053+0000", 2024-08-17T17:34:29.302 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6393e-05 2024-08-17T17:34:29.302 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.303 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.303 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.303 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.998064+0000", 2024-08-17T17:34:29.303 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0582e-05 2024-08-17T17:34:29.303 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.303 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.303 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.303 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.998131+0000", 2024-08-17T17:34:29.303 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7082000000000006e-05 2024-08-17T17:34:29.303 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.303 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.303 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.303 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:24.998178+0000", 2024-08-17T17:34:29.304 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7370000000000002e-05 2024-08-17T17:34:29.304 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.304 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.304 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.304 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.927805+0000", 2024-08-17T17:34:29.304 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.92962700399999998 2024-08-17T17:34:29.304 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.304 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.304 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.304 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.927812+0000", 2024-08-17T17:34:29.304 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8770000000000004e-06 2024-08-17T17:34:29.304 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.304 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.305 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.305 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.305 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.305 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:217580 10.b 10.64f8b89b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T17:34:29.305 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:25.008801+0000", 2024-08-17T17:34:29.305 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 64.263679026999995, 2024-08-17T17:34:29.305 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.91903957599999997, 2024-08-17T17:34:29.306 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.306 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.306 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:34:29.306 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.306 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.306 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.306 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 217580 2024-08-17T17:34:29.306 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.306 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.306 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.306 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.306 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.008801+0000", 2024-08-17T17:34:29.307 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.307 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.307 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.307 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.307 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.008801+0000", 2024-08-17T17:34:29.307 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.307 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.307 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.307 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.307 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.008803+0000", 2024-08-17T17:34:29.307 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9769999999999999e-06 2024-08-17T17:34:29.308 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.308 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.308 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.308 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.008810+0000", 2024-08-17T17:34:29.308 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2919999999999998e-06 2024-08-17T17:34:29.308 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.308 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.308 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.308 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.008829+0000", 2024-08-17T17:34:29.308 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8138000000000002e-05 2024-08-17T17:34:29.308 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.308 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.309 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.309 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.008837+0000", 2024-08-17T17:34:29.309 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.32e-06 2024-08-17T17:34:29.309 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.309 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.309 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.309 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.008893+0000", 2024-08-17T17:34:29.309 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5753e-05 2024-08-17T17:34:29.309 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.309 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.309 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.309 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.008899+0000", 2024-08-17T17:34:29.309 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6089999999999997e-06 2024-08-17T17:34:29.310 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.310 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.310 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.311 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:29.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:29.312 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.927833+0000", 2024-08-17T17:34:29.312 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.91893400599999997 2024-08-17T17:34:29.312 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.312 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.313 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.313 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:25.927841+0000", 2024-08-17T17:34:29.313 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.481e-06 2024-08-17T17:34:29.313 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.313 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.313 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.313 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.313 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.313 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:237930 10.b 10:db75a794:::benchmark_data_smithi063_294527_object100745:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1333)", 2024-08-17T17:34:29.313 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:48.750134+0000", 2024-08-17T17:34:29.313 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 40.522345829999999, 2024-08-17T17:34:29.313 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0556535039999999, 2024-08-17T17:34:29.314 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.314 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.314 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:29.314 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.314 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.314 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.314 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 237930 2024-08-17T17:34:29.314 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.314 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.314 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.314 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.314 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.750134+0000", 2024-08-17T17:34:29.314 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.315 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.315 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.315 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.315 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.750134+0000", 2024-08-17T17:34:29.315 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.315 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.315 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.315 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.315 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.750136+0000", 2024-08-17T17:34:29.315 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8989999999999999e-06 2024-08-17T17:34:29.315 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.315 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.315 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.316 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.750142+0000", 2024-08-17T17:34:29.316 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2859999999999996e-06 2024-08-17T17:34:29.316 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.316 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.316 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.316 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.750160+0000", 2024-08-17T17:34:29.316 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7363e-05 2024-08-17T17:34:29.316 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.316 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.316 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.316 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.750168+0000", 2024-08-17T17:34:29.316 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.1079999999999993e-06 2024-08-17T17:34:29.316 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.317 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.317 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.317 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.751409+0000", 2024-08-17T17:34:29.317 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0012409350000000001 2024-08-17T17:34:29.317 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.317 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.317 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.317 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.751502+0000", 2024-08-17T17:34:29.317 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3256000000000003e-05 2024-08-17T17:34:29.317 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.317 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.318 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.318 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.799291+0000", 2024-08-17T17:34:29.318 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.047788597 2024-08-17T17:34:29.318 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.318 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.318 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:29.318 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.799439+0000", 2024-08-17T17:34:29.318 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000148348 2024-08-17T17:34:29.318 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.318 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.318 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:34:29.319 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.799584+0000", 2024-08-17T17:34:29.319 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014485 2024-08-17T17:34:29.319 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.319 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.319 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:29.319 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.803851+0000", 2024-08-17T17:34:29.319 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0042669689999999998 2024-08-17T17:34:29.319 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.319 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.319 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.319 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.805550+0000", 2024-08-17T17:34:29.319 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001698866 2024-08-17T17:34:29.319 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.320 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.320 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.320 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.805675+0000", 2024-08-17T17:34:29.320 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012545699999999999 2024-08-17T17:34:29.320 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.320 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.320 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:29.320 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.805709+0000", 2024-08-17T17:34:29.320 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3390999999999999e-05 2024-08-17T17:34:29.320 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.321 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.321 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.321 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.805788+0000", 2024-08-17T17:34:29.321 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9178999999999995e-05 2024-08-17T17:34:29.321 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.321 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.321 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.321 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.321 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.321 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:237933 10.b 10:d4867978:::benchmark_data_smithi063_294527_object100748:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1333)", 2024-08-17T17:34:29.321 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:48.752062+0000", 2024-08-17T17:34:29.321 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 40.520418079000002, 2024-08-17T17:34:29.321 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.058098202, 2024-08-17T17:34:29.321 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.321 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.322 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:29.322 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.322 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.322 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.322 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 237933 2024-08-17T17:34:29.322 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.322 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.322 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.322 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.322 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.752062+0000", 2024-08-17T17:34:29.322 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.322 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.322 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.323 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.323 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.752062+0000", 2024-08-17T17:34:29.323 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.323 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.323 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.323 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.323 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.752064+0000", 2024-08-17T17:34:29.323 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.443e-06 2024-08-17T17:34:29.323 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.323 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.323 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.323 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.752069+0000", 2024-08-17T17:34:29.323 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2619999999999998e-06 2024-08-17T17:34:29.324 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.324 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.324 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.324 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.752093+0000", 2024-08-17T17:34:29.324 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4655999999999999e-05 2024-08-17T17:34:29.324 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.324 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.324 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.324 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.752101+0000", 2024-08-17T17:34:29.324 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0479999999999999e-06 2024-08-17T17:34:29.324 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.324 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.324 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.324 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.752294+0000", 2024-08-17T17:34:29.325 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000192158 2024-08-17T17:34:29.325 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.325 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.325 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.325 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.752332+0000", 2024-08-17T17:34:29.325 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8690999999999999e-05 2024-08-17T17:34:29.325 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.325 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.325 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.325 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.802160+0000", 2024-08-17T17:34:29.325 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0498276719999999 2024-08-17T17:34:29.325 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.325 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.325 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:29.326 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.802269+0000", 2024-08-17T17:34:29.326 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010852 2024-08-17T17:34:29.326 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.326 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.326 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:34:29.326 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.802401+0000", 2024-08-17T17:34:29.326 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013216200000000001 2024-08-17T17:34:29.326 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.326 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.326 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.326 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.805838+0000", 2024-08-17T17:34:29.327 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0034371340000000001 2024-08-17T17:34:29.327 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.327 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.327 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:29.327 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.805846+0000", 2024-08-17T17:34:29.327 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.1000000000000004e-06 2024-08-17T17:34:29.327 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.327 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.327 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.327 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.810030+0000", 2024-08-17T17:34:29.327 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0041838839999999997 2024-08-17T17:34:29.327 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.328 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.328 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:29.328 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.810090+0000", 2024-08-17T17:34:29.328 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0047999999999998e-05 2024-08-17T17:34:29.328 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.328 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.328 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.328 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.810160+0000", 2024-08-17T17:34:29.328 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0424000000000002e-05 2024-08-17T17:34:29.328 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.328 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.328 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.329 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.329 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.329 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:237947 10.b 10:da475ad5:::benchmark_data_smithi063_294527_object100762:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1333)", 2024-08-17T17:34:29.329 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:48.765227+0000", 2024-08-17T17:34:29.329 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 40.507252721, 2024-08-17T17:34:29.329 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0451235430000001, 2024-08-17T17:34:29.329 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.329 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.329 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:29.329 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.329 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.329 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.329 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 237947 2024-08-17T17:34:29.330 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.330 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.330 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.330 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.330 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.765227+0000", 2024-08-17T17:34:29.330 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.330 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.330 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.330 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.330 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.765227+0000", 2024-08-17T17:34:29.330 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.330 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.331 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.331 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.331 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.765229+0000", 2024-08-17T17:34:29.331 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.454e-06 2024-08-17T17:34:29.331 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.331 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.331 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.331 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.765236+0000", 2024-08-17T17:34:29.331 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1250000000000004e-06 2024-08-17T17:34:29.331 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.331 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.331 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.331 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.765373+0000", 2024-08-17T17:34:29.331 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013737499999999999 2024-08-17T17:34:29.331 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.332 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.332 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.332 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.765382+0000", 2024-08-17T17:34:29.332 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0990000000000005e-06 2024-08-17T17:34:29.332 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.332 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.332 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.332 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.765707+0000", 2024-08-17T17:34:29.332 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00032420499999999999 2024-08-17T17:34:29.332 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.332 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.332 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.332 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.765717+0000", 2024-08-17T17:34:29.332 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0441999999999999e-05 2024-08-17T17:34:29.333 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.333 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.333 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.333 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.803897+0000", 2024-08-17T17:34:29.333 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.038179838 2024-08-17T17:34:29.333 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.333 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.333 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:29.333 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.804013+0000", 2024-08-17T17:34:29.333 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011652 2024-08-17T17:34:29.333 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.333 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.333 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:34:29.333 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.804130+0000", 2024-08-17T17:34:29.334 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000116173 2024-08-17T17:34:29.334 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.334 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.334 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:29.334 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.805851+0000", 2024-08-17T17:34:29.334 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0017210870000000001 2024-08-17T17:34:29.334 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.334 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.334 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.334 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.809967+0000", 2024-08-17T17:34:29.334 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0041157859999999998 2024-08-17T17:34:29.335 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.335 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.335 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.335 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.810249+0000", 2024-08-17T17:34:29.335 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00028282600000000002 2024-08-17T17:34:29.335 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.335 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.335 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:29.335 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.810283+0000", 2024-08-17T17:34:29.335 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3874999999999997e-05 2024-08-17T17:34:29.335 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.335 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.336 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.336 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.810351+0000", 2024-08-17T17:34:29.336 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7737999999999994e-05 2024-08-17T17:34:29.336 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.336 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.336 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.336 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.336 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.336 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:237966 10.b 10:d8d8801e:::benchmark_data_smithi063_294527_object100781:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1333)", 2024-08-17T17:34:29.336 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:48.781363+0000", 2024-08-17T17:34:29.336 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 40.491117095, 2024-08-17T17:34:29.336 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0337627970000001, 2024-08-17T17:34:29.336 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.337 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.337 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:29.337 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.337 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.337 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.337 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 237966 2024-08-17T17:34:29.337 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.337 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.337 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.337 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.337 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.781363+0000", 2024-08-17T17:34:29.337 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.337 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.337 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.338 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.338 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.781363+0000", 2024-08-17T17:34:29.338 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.338 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.338 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.338 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.338 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.781364+0000", 2024-08-17T17:34:29.338 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4330000000000001e-06 2024-08-17T17:34:29.338 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.338 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.338 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.338 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.781370+0000", 2024-08-17T17:34:29.339 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5890000000000002e-06 2024-08-17T17:34:29.339 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.339 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.339 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.339 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.781400+0000", 2024-08-17T17:34:29.339 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0287e-05 2024-08-17T17:34:29.339 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.339 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.339 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.339 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.781408+0000", 2024-08-17T17:34:29.339 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8890000000000002e-06 2024-08-17T17:34:29.339 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.339 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.340 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.340 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.781497+0000", 2024-08-17T17:34:29.340 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.9109000000000004e-05 2024-08-17T17:34:29.340 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.340 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.340 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.340 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.781508+0000", 2024-08-17T17:34:29.340 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.099e-05 2024-08-17T17:34:29.340 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.340 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.340 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.340 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.805884+0000", 2024-08-17T17:34:29.340 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0243753250000001 2024-08-17T17:34:29.341 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.341 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.341 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:29.341 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.805962+0000", 2024-08-17T17:34:29.341 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8434000000000002e-05 2024-08-17T17:34:29.341 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.341 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.341 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:34:29.341 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.806057+0000", 2024-08-17T17:34:29.341 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.5169999999999999e-05 2024-08-17T17:34:29.341 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.341 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.341 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:29.341 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.809988+0000", 2024-08-17T17:34:29.342 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0039306310000000004 2024-08-17T17:34:29.342 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.342 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.342 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.342 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.810209+0000", 2024-08-17T17:34:29.342 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00022144800000000001 2024-08-17T17:34:29.342 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.342 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.342 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.342 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.814938+0000", 2024-08-17T17:34:29.342 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0047283560000000004 2024-08-17T17:34:29.342 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.343 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.343 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:29.343 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.815000+0000", 2024-08-17T17:34:29.343 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2731999999999995e-05 2024-08-17T17:34:29.343 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.343 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.343 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.343 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.815126+0000", 2024-08-17T17:34:29.343 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012540399999999999 2024-08-17T17:34:29.343 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.343 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.343 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.343 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.344 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.344 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:237969 10.b 10:d6292662:::benchmark_data_smithi063_294527_object100784:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1333)", 2024-08-17T17:34:29.344 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:48.782144+0000", 2024-08-17T17:34:29.344 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 40.490335766999998, 2024-08-17T17:34:29.344 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0338661629999999, 2024-08-17T17:34:29.344 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.344 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.344 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:29.344 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.344 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.344 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.344 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 237969 2024-08-17T17:34:29.344 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.345 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.345 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.345 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.345 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.782144+0000", 2024-08-17T17:34:29.345 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.345 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.345 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.345 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.345 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.782144+0000", 2024-08-17T17:34:29.345 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.345 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.345 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.345 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.345 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.782145+0000", 2024-08-17T17:34:29.346 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.068e-06 2024-08-17T17:34:29.346 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.346 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.346 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.346 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.782150+0000", 2024-08-17T17:34:29.346 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7960000000000003e-06 2024-08-17T17:34:29.346 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.346 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.346 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.346 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.782165+0000", 2024-08-17T17:34:29.346 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5109999999999999e-05 2024-08-17T17:34:29.346 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.346 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.347 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.347 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.782173+0000", 2024-08-17T17:34:29.347 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4449999999999999e-06 2024-08-17T17:34:29.347 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.347 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.347 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.347 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.782232+0000", 2024-08-17T17:34:29.347 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9517e-05 2024-08-17T17:34:29.347 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.347 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.347 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.347 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.782242+0000", 2024-08-17T17:34:29.347 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3470000000000005e-06 2024-08-17T17:34:29.348 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.348 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.348 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.348 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.807901+0000", 2024-08-17T17:34:29.348 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0256588740000001 2024-08-17T17:34:29.348 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.348 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.348 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:29.348 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.808026+0000", 2024-08-17T17:34:29.348 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000125401 2024-08-17T17:34:29.348 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.348 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.348 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:34:29.348 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.808178+0000", 2024-08-17T17:34:29.349 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000151583 2024-08-17T17:34:29.349 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.349 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.349 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.349 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.810386+0000", 2024-08-17T17:34:29.349 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0022086689999999999 2024-08-17T17:34:29.349 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.349 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.349 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:29.349 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.814867+0000", 2024-08-17T17:34:29.349 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044809330000000003 2024-08-17T17:34:29.349 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.350 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.350 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.350 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.815195+0000", 2024-08-17T17:34:29.350 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000327886 2024-08-17T17:34:29.350 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.350 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.350 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:29.350 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.815831+0000", 2024-08-17T17:34:29.350 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000635645 2024-08-17T17:34:29.350 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.350 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.350 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.350 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.816011+0000", 2024-08-17T17:34:29.351 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000179889 2024-08-17T17:34:29.351 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.351 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.351 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.351 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.351 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.351 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:237990 10.b 10:d26db6e0:::benchmark_data_smithi063_294527_object100805:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1333)", 2024-08-17T17:34:29.351 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:48.799892+0000", 2024-08-17T17:34:29.351 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 40.472588268000003, 2024-08-17T17:34:29.351 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0165217150000001, 2024-08-17T17:34:29.351 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.351 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.352 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:29.352 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.352 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.352 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.352 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 237990 2024-08-17T17:34:29.352 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.352 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.352 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.352 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.352 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.799892+0000", 2024-08-17T17:34:29.352 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.352 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.352 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.352 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.353 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.799892+0000", 2024-08-17T17:34:29.353 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.353 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.353 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.353 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.353 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.799894+0000", 2024-08-17T17:34:29.353 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.052e-06 2024-08-17T17:34:29.353 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.353 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.353 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.353 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.799902+0000", 2024-08-17T17:34:29.353 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.5089999999999993e-06 2024-08-17T17:34:29.353 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.353 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.354 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.354 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.799927+0000", 2024-08-17T17:34:29.354 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4589000000000001e-05 2024-08-17T17:34:29.354 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.354 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.354 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.354 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.799935+0000", 2024-08-17T17:34:29.354 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9729999999999997e-06 2024-08-17T17:34:29.354 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.354 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.354 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.354 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.799997+0000", 2024-08-17T17:34:29.354 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2260000000000004e-05 2024-08-17T17:34:29.355 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.355 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.355 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.355 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.800008+0000", 2024-08-17T17:34:29.355 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0977e-05 2024-08-17T17:34:29.355 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.355 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.355 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.355 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.810404+0000", 2024-08-17T17:34:29.355 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0103960599999999 2024-08-17T17:34:29.355 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.355 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.355 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:29.355 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.810489+0000", 2024-08-17T17:34:29.356 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.5025999999999995e-05 2024-08-17T17:34:29.356 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.356 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.356 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:34:29.356 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.810629+0000", 2024-08-17T17:34:29.356 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013993799999999999 2024-08-17T17:34:29.356 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.356 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.356 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:29.356 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.816028+0000", 2024-08-17T17:34:29.356 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0053984840000000003 2024-08-17T17:34:29.356 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.356 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.357 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.357 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.816066+0000", 2024-08-17T17:34:29.357 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8483999999999999e-05 2024-08-17T17:34:29.357 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.357 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.357 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.357 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.816130+0000", 2024-08-17T17:34:29.357 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3633999999999994e-05 2024-08-17T17:34:29.357 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.357 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.357 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:29.358 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.816236+0000", 2024-08-17T17:34:29.358 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000105797 2024-08-17T17:34:29.358 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.358 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.358 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.358 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.816414+0000", 2024-08-17T17:34:29.358 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000177932 2024-08-17T17:34:29.358 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.358 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.358 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.358 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.359 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.359 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:238059 10.b 10:d96f01f2:::benchmark_data_smithi063_294527_object100874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1333)", 2024-08-17T17:34:29.359 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:48.857862+0000", 2024-08-17T17:34:29.359 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 40.414617853999999, 2024-08-17T17:34:29.359 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.96271203500000002, 2024-08-17T17:34:29.359 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.359 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.359 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:29.359 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.359 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.359 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.359 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 238059 2024-08-17T17:34:29.359 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.360 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.360 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.360 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.360 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.857862+0000", 2024-08-17T17:34:29.360 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.360 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.360 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.360 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.360 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.857862+0000", 2024-08-17T17:34:29.360 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.360 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.360 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.360 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.361 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.857864+0000", 2024-08-17T17:34:29.361 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.621e-06 2024-08-17T17:34:29.361 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.361 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.361 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.361 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.857869+0000", 2024-08-17T17:34:29.361 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0259999999999998e-06 2024-08-17T17:34:29.361 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.361 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.361 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.361 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.857882+0000", 2024-08-17T17:34:29.361 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2792999999999999e-05 2024-08-17T17:34:29.362 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.362 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.362 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.362 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.857889+0000", 2024-08-17T17:34:29.362 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6389999999999992e-06 2024-08-17T17:34:29.362 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.362 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.362 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.362 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.858034+0000", 2024-08-17T17:34:29.362 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000144888 2024-08-17T17:34:29.362 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.363 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.363 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.363 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.858052+0000", 2024-08-17T17:34:29.363 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7676000000000002e-05 2024-08-17T17:34:29.363 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.363 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.363 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.363 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.812511+0000", 2024-08-17T17:34:29.363 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95445865500000004 2024-08-17T17:34:29.363 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.363 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.363 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:29.363 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.812618+0000", 2024-08-17T17:34:29.363 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010717000000000001 2024-08-17T17:34:29.364 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.364 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.364 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:34:29.364 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.812754+0000", 2024-08-17T17:34:29.364 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000136334 2024-08-17T17:34:29.364 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.364 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.364 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:29.364 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.816077+0000", 2024-08-17T17:34:29.364 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0033229209999999999 2024-08-17T17:34:29.364 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.364 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.364 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.364 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.820317+0000", 2024-08-17T17:34:29.365 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0042394939999999999 2024-08-17T17:34:29.365 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.365 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.365 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.365 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.820396+0000", 2024-08-17T17:34:29.365 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9597000000000004e-05 2024-08-17T17:34:29.365 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.365 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.365 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:29.365 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.820467+0000", 2024-08-17T17:34:29.365 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0672000000000004e-05 2024-08-17T17:34:29.365 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.365 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.366 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.366 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.820574+0000", 2024-08-17T17:34:29.366 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000107549 2024-08-17T17:34:29.366 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.366 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.366 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.366 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.366 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.366 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:238067 10.b 10:de04ef45:::benchmark_data_smithi063_294527_object100882:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1333)", 2024-08-17T17:34:29.366 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:48.869150+0000", 2024-08-17T17:34:29.366 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 40.403330461000003, 2024-08-17T17:34:29.367 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.951831392, 2024-08-17T17:34:29.367 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.367 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.367 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:29.367 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.367 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.367 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.367 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 238067 2024-08-17T17:34:29.367 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.367 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.367 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.367 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.367 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.869150+0000", 2024-08-17T17:34:29.368 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.368 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.368 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.368 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.368 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.869150+0000", 2024-08-17T17:34:29.368 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.368 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.368 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.368 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.368 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.869151+0000", 2024-08-17T17:34:29.368 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.542e-06 2024-08-17T17:34:29.368 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.369 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.369 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.369 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.869157+0000", 2024-08-17T17:34:29.369 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9920000000000004e-06 2024-08-17T17:34:29.369 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.369 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.369 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.369 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.869170+0000", 2024-08-17T17:34:29.369 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3115999999999999e-05 2024-08-17T17:34:29.369 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.369 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.370 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.370 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.869176+0000", 2024-08-17T17:34:29.370 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2290000000000002e-06 2024-08-17T17:34:29.370 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.370 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.370 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.370 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.869223+0000", 2024-08-17T17:34:29.370 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7394000000000001e-05 2024-08-17T17:34:29.370 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.370 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.370 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.371 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.869229+0000", 2024-08-17T17:34:29.371 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4280000000000001e-06 2024-08-17T17:34:29.371 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.371 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.371 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.371 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.816450+0000", 2024-08-17T17:34:29.371 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.947220273 2024-08-17T17:34:29.371 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.371 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.371 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:29.371 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.816554+0000", 2024-08-17T17:34:29.371 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000104249 2024-08-17T17:34:29.372 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.372 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.372 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:34:29.372 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.816658+0000", 2024-08-17T17:34:29.372 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000103992 2024-08-17T17:34:29.372 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.372 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.372 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.372 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.820665+0000", 2024-08-17T17:34:29.373 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0040071589999999997 2024-08-17T17:34:29.373 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.373 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.373 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:29.373 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.820688+0000", 2024-08-17T17:34:29.373 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2467e-05 2024-08-17T17:34:29.373 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.373 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.373 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.373 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.820836+0000", 2024-08-17T17:34:29.373 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000148467 2024-08-17T17:34:29.373 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.374 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.374 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:29.374 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.820883+0000", 2024-08-17T17:34:29.374 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7154000000000003e-05 2024-08-17T17:34:29.374 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.374 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.374 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.374 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.820981+0000", 2024-08-17T17:34:29.374 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7930000000000006e-05 2024-08-17T17:34:29.374 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.374 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.374 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.375 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.375 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.375 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:238090 10.b 10:d15b138f:::benchmark_data_smithi063_294527_object100905:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1333)", 2024-08-17T17:34:29.375 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:48.888347+0000", 2024-08-17T17:34:29.375 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 40.384133005999999, 2024-08-17T17:34:29.375 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.94701250400000003, 2024-08-17T17:34:29.375 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.375 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.375 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:29.375 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.375 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.375 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.375 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 238090 2024-08-17T17:34:29.376 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.376 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.376 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.376 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.376 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.888347+0000", 2024-08-17T17:34:29.376 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.376 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.376 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.376 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.376 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.888347+0000", 2024-08-17T17:34:29.376 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.376 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.376 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.376 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.377 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.888348+0000", 2024-08-17T17:34:29.377 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3239999999999999e-06 2024-08-17T17:34:29.377 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.377 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.377 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.377 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.888355+0000", 2024-08-17T17:34:29.377 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2870000000000001e-06 2024-08-17T17:34:29.377 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.377 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.377 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.377 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.888365+0000", 2024-08-17T17:34:29.378 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0447e-05 2024-08-17T17:34:29.378 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.378 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.378 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.378 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.888371+0000", 2024-08-17T17:34:29.378 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6559999999999997e-06 2024-08-17T17:34:29.378 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.378 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.378 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.378 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.888420+0000", 2024-08-17T17:34:29.378 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8699999999999998e-05 2024-08-17T17:34:29.378 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.379 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.379 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.379 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.888427+0000", 2024-08-17T17:34:29.379 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.002e-06 2024-08-17T17:34:29.379 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.379 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.379 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.379 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.821021+0000", 2024-08-17T17:34:29.379 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.93259477000000002 2024-08-17T17:34:29.379 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.379 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.379 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:29.380 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.821127+0000", 2024-08-17T17:34:29.380 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000105957 2024-08-17T17:34:29.380 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.380 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.380 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:34:29.380 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.821263+0000", 2024-08-17T17:34:29.380 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013572899999999999 2024-08-17T17:34:29.380 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.380 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.380 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:29.380 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.830196+0000", 2024-08-17T17:34:29.380 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0089327279999999992 2024-08-17T17:34:29.380 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.381 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.381 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.381 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.830238+0000", 2024-08-17T17:34:29.381 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2682000000000002e-05 2024-08-17T17:34:29.381 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.381 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.381 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.381 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.835198+0000", 2024-08-17T17:34:29.381 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0049594119999999998 2024-08-17T17:34:29.381 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.381 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.381 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:29.381 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.835255+0000", 2024-08-17T17:34:29.381 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7188000000000002e-05 2024-08-17T17:34:29.382 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.382 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.382 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.382 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.835360+0000", 2024-08-17T17:34:29.382 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000104622 2024-08-17T17:34:29.382 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.382 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.382 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.382 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.382 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.382 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:238094 10.b 10:d4925b4d:::benchmark_data_smithi063_294527_object100909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1333)", 2024-08-17T17:34:29.383 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:48.892070+0000", 2024-08-17T17:34:29.383 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 40.380410605000002, 2024-08-17T17:34:29.383 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95158483699999996, 2024-08-17T17:34:29.383 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.383 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.383 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:29.383 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.383 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.383 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.383 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 238094 2024-08-17T17:34:29.383 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.383 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.383 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.384 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.384 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.892070+0000", 2024-08-17T17:34:29.384 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.384 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.384 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.384 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.384 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.892070+0000", 2024-08-17T17:34:29.384 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.384 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.384 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.384 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.384 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.892071+0000", 2024-08-17T17:34:29.384 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3510000000000001e-06 2024-08-17T17:34:29.385 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.385 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.385 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.385 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.892076+0000", 2024-08-17T17:34:29.385 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5079999999999996e-06 2024-08-17T17:34:29.385 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.385 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.385 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.385 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.892087+0000", 2024-08-17T17:34:29.385 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0436e-05 2024-08-17T17:34:29.385 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.386 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.386 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.386 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.892093+0000", 2024-08-17T17:34:29.386 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4640000000000003e-06 2024-08-17T17:34:29.386 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.386 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.386 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.386 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.892151+0000", 2024-08-17T17:34:29.386 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7268999999999997e-05 2024-08-17T17:34:29.386 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.386 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.387 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.387 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.892157+0000", 2024-08-17T17:34:29.387 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5320000000000003e-06 2024-08-17T17:34:29.387 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.387 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.387 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.387 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.823489+0000", 2024-08-17T17:34:29.387 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.931331625 2024-08-17T17:34:29.387 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.387 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.387 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:29.387 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.825413+0000", 2024-08-17T17:34:29.387 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0019238809999999999 2024-08-17T17:34:29.388 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.388 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.388 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:34:29.388 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.825534+0000", 2024-08-17T17:34:29.388 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000121229 2024-08-17T17:34:29.388 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.388 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.388 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.388 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.830280+0000", 2024-08-17T17:34:29.388 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0047459879999999996 2024-08-17T17:34:29.388 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.388 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.389 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.389 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.835399+0000", 2024-08-17T17:34:29.389 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0051189410000000001 2024-08-17T17:34:29.389 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.389 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.389 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:29.389 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.843493+0000", 2024-08-17T17:34:29.389 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0080942480000000001 2024-08-17T17:34:29.389 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.389 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.389 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:29.389 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.843546+0000", 2024-08-17T17:34:29.389 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2539e-05 2024-08-17T17:34:29.390 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.390 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.390 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.390 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.843654+0000", 2024-08-17T17:34:29.390 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000108826 2024-08-17T17:34:29.390 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.390 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.390 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.390 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.390 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.390 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:238096 10.b 10:d02fc9ce:::benchmark_data_smithi063_294527_object100911:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1333)", 2024-08-17T17:34:29.390 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:48.942277+0000", 2024-08-17T17:34:29.390 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 40.330203015999999, 2024-08-17T17:34:29.391 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.90151625999999996, 2024-08-17T17:34:29.391 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.391 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.391 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:29.391 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.391 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.391 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.391 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 238096 2024-08-17T17:34:29.391 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.391 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.393 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.393 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.393 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.942277+0000", 2024-08-17T17:34:29.393 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.394 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.394 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.394 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.394 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.942277+0000", 2024-08-17T17:34:29.394 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.394 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.394 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.394 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.394 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.942279+0000", 2024-08-17T17:34:29.394 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3999999999999999e-06 2024-08-17T17:34:29.394 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.394 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.395 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.395 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.942283+0000", 2024-08-17T17:34:29.395 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7439999999999998e-06 2024-08-17T17:34:29.395 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.395 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.395 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.395 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.942294+0000", 2024-08-17T17:34:29.395 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0857e-05 2024-08-17T17:34:29.395 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.395 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.395 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.395 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.942301+0000", 2024-08-17T17:34:29.395 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.311e-06 2024-08-17T17:34:29.395 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.396 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.396 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.396 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.942367+0000", 2024-08-17T17:34:29.396 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5862000000000001e-05 2024-08-17T17:34:29.396 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.396 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.396 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.396 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.942379+0000", 2024-08-17T17:34:29.396 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1375999999999999e-05 2024-08-17T17:34:29.396 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.396 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.396 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.396 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.827802+0000", 2024-08-17T17:34:29.396 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.88542292300000003 2024-08-17T17:34:29.397 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.397 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.397 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:29.397 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.827913+0000", 2024-08-17T17:34:29.397 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000111536 2024-08-17T17:34:29.397 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.397 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.397 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:34:29.397 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.828030+0000", 2024-08-17T17:34:29.397 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000116972 2024-08-17T17:34:29.397 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.397 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.397 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.397 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.835433+0000", 2024-08-17T17:34:29.397 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0074024729999999997 2024-08-17T17:34:29.398 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.398 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.398 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.398 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.837958+0000", 2024-08-17T17:34:29.398 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0025254980000000002 2024-08-17T17:34:29.398 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.398 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.398 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:29.398 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.843662+0000", 2024-08-17T17:34:29.398 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0057038339999999996 2024-08-17T17:34:29.398 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.398 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.398 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:29.399 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.843708+0000", 2024-08-17T17:34:29.399 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6513e-05 2024-08-17T17:34:29.399 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.399 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.399 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.399 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.843793+0000", 2024-08-17T17:34:29.399 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4961000000000002e-05 2024-08-17T17:34:29.399 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.399 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.399 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.399 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.399 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.399 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:238120 10.b 10:d50ea830:::benchmark_data_smithi063_294527_object100935:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1333)", 2024-08-17T17:34:29.400 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:48.966491+0000", 2024-08-17T17:34:29.400 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 40.305988954999997, 2024-08-17T17:34:29.400 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.87745561900000002, 2024-08-17T17:34:29.400 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.400 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.400 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:29.400 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.400 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.400 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.400 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 238120 2024-08-17T17:34:29.400 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.400 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.400 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.401 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.401 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.966491+0000", 2024-08-17T17:34:29.401 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.401 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.401 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.401 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.401 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.966491+0000", 2024-08-17T17:34:29.401 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.401 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.401 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.401 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.401 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.966493+0000", 2024-08-17T17:34:29.401 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5060000000000001e-06 2024-08-17T17:34:29.402 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.402 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.402 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.402 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.966498+0000", 2024-08-17T17:34:29.402 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7440000000000004e-06 2024-08-17T17:34:29.402 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.402 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.402 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.402 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.966510+0000", 2024-08-17T17:34:29.402 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1865000000000001e-05 2024-08-17T17:34:29.402 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.402 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.403 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.403 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.966522+0000", 2024-08-17T17:34:29.403 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1269e-05 2024-08-17T17:34:29.403 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.403 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.403 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.403 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.966584+0000", 2024-08-17T17:34:29.403 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2433000000000003e-05 2024-08-17T17:34:29.403 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.403 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.403 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.403 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:48.966594+0000", 2024-08-17T17:34:29.403 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0259e-05 2024-08-17T17:34:29.403 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.404 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.404 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.404 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.830309+0000", 2024-08-17T17:34:29.404 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.86371512299999997 2024-08-17T17:34:29.404 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.404 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.404 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:29.404 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.830418+0000", 2024-08-17T17:34:29.404 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010855 2024-08-17T17:34:29.404 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.404 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.404 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:34:29.405 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.830526+0000", 2024-08-17T17:34:29.405 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010833299999999999 2024-08-17T17:34:29.405 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.405 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.405 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.406 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.837996+0000", 2024-08-17T17:34:29.406 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0074698170000000001 2024-08-17T17:34:29.406 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.406 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.406 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.406 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.838047+0000", 2024-08-17T17:34:29.406 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1178999999999999e-05 2024-08-17T17:34:29.406 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.406 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.406 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:29.406 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.843800+0000", 2024-08-17T17:34:29.406 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0057525830000000003 2024-08-17T17:34:29.407 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.407 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.407 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:29.407 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.843848+0000", 2024-08-17T17:34:29.407 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8096999999999997e-05 2024-08-17T17:34:29.407 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.407 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.407 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.407 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.843947+0000", 2024-08-17T17:34:29.407 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.8861000000000001e-05 2024-08-17T17:34:29.407 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.407 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.407 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.408 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.408 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.408 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:238149 10.b 10:d99ea9e2:::benchmark_data_smithi063_294527_object100964:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1333)", 2024-08-17T17:34:29.408 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:49.001963+0000", 2024-08-17T17:34:29.408 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 40.270517276, 2024-08-17T17:34:29.408 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.84918955200000001, 2024-08-17T17:34:29.408 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.408 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.408 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:29.408 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.408 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.408 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.408 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 238149 2024-08-17T17:34:29.409 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.409 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.409 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.409 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.409 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.001963+0000", 2024-08-17T17:34:29.409 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.409 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.409 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.409 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.409 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.001963+0000", 2024-08-17T17:34:29.409 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.409 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.409 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.410 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.410 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.001965+0000", 2024-08-17T17:34:29.410 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7179999999999999e-06 2024-08-17T17:34:29.410 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.410 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.410 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.410 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.001971+0000", 2024-08-17T17:34:29.410 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.127e-06 2024-08-17T17:34:29.410 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.410 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.410 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.410 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.001984+0000", 2024-08-17T17:34:29.410 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3536000000000001e-05 2024-08-17T17:34:29.411 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.411 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.411 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.411 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.001992+0000", 2024-08-17T17:34:29.411 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0569999999999993e-06 2024-08-17T17:34:29.411 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.411 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.411 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.411 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.002064+0000", 2024-08-17T17:34:29.411 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1433000000000005e-05 2024-08-17T17:34:29.411 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.411 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.411 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.411 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.002076+0000", 2024-08-17T17:34:29.411 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2659e-05 2024-08-17T17:34:29.412 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.412 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.412 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.412 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.832890+0000", 2024-08-17T17:34:29.412 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.83081360000000004 2024-08-17T17:34:29.412 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.412 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.412 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:29.412 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.833014+0000", 2024-08-17T17:34:29.412 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012415200000000001 2024-08-17T17:34:29.412 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.412 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.412 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:34:29.413 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.833123+0000", 2024-08-17T17:34:29.413 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000108971 2024-08-17T17:34:29.413 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.413 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.413 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:29.413 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.843956+0000", 2024-08-17T17:34:29.413 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.01083278 2024-08-17T17:34:29.413 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.413 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.413 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.413 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.846646+0000", 2024-08-17T17:34:29.413 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002689954 2024-08-17T17:34:29.413 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.414 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.414 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.414 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.850963+0000", 2024-08-17T17:34:29.414 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0043173090000000001 2024-08-17T17:34:29.414 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.414 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.414 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:29.414 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.851037+0000", 2024-08-17T17:34:29.414 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3428000000000002e-05 2024-08-17T17:34:29.414 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.414 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.414 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.851152+0000", 2024-08-17T17:34:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000115828 2024-08-17T17:34:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:238163 10.b 10:d46e2727:::benchmark_data_smithi063_294527_object100978:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1333)", 2024-08-17T17:34:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:49.020140+0000", 2024-08-17T17:34:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 40.252340185000001, 2024-08-17T17:34:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.83145751599999995, 2024-08-17T17:34:29.415 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:29.416 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:29.416 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:29.416 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:29.416 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:29.416 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:29.416 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 238163 2024-08-17T17:34:29.416 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.416 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:29.416 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.416 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:29.416 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.020140+0000", 2024-08-17T17:34:29.416 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.416 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.417 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.417 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:29.417 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.020140+0000", 2024-08-17T17:34:29.417 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:29.417 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.417 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.417 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:29.417 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.020141+0000", 2024-08-17T17:34:29.417 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.268e-06 2024-08-17T17:34:29.417 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.417 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.417 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:29.417 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.020148+0000", 2024-08-17T17:34:29.417 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8430000000000004e-06 2024-08-17T17:34:29.418 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.418 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.418 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:29.418 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.020160+0000", 2024-08-17T17:34:29.418 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1722000000000001e-05 2024-08-17T17:34:29.418 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.418 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.418 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:29.418 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.020165+0000", 2024-08-17T17:34:29.418 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3140000000000003e-06 2024-08-17T17:34:29.418 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.418 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.418 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.419 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.020218+0000", 2024-08-17T17:34:29.419 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2695999999999998e-05 2024-08-17T17:34:29.419 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.419 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.419 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:34:29.419 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.020224+0000", 2024-08-17T17:34:29.419 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2169999999999999e-06 2024-08-17T17:34:29.419 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.419 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.419 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:29.419 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.835461+0000", 2024-08-17T17:34:29.419 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.81523674899999998 2024-08-17T17:34:29.419 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.419 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.419 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:29.420 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.835568+0000", 2024-08-17T17:34:29.420 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000106913 2024-08-17T17:34:29.420 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.420 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.420 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:34:29.420 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.835676+0000", 2024-08-17T17:34:29.420 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000107828 2024-08-17T17:34:29.420 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.420 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.420 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:29.420 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.843961+0000", 2024-08-17T17:34:29.420 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0082858280000000003 2024-08-17T17:34:29.421 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.421 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.421 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.421 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.846700+0000", 2024-08-17T17:34:29.421 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0027387700000000002 2024-08-17T17:34:29.421 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.421 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.421 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:29.421 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.851435+0000", 2024-08-17T17:34:29.421 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0047344229999999998 2024-08-17T17:34:29.421 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.421 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.421 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:29.422 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.851494+0000", 2024-08-17T17:34:29.422 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9057000000000001e-05 2024-08-17T17:34:29.422 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:29.422 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:29.422 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:29.422 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:49.851597+0000", 2024-08-17T17:34:29.422 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010388800000000001 2024-08-17T17:34:29.422 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.422 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.422 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.422 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:29.422 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:29.423 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:34:29.423 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:29.445 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:29.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:29.547 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:29.570 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:29.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:29.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:29 smithi162 ceph-mon[104668]: 10.16 scrub starts 2024-08-17T17:34:29.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:29 smithi162 ceph-mon[104668]: 10.16 scrub ok 2024-08-17T17:34:29.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:29 smithi162 ceph-mon[104668]: pgmap v2031: 129 pgs: 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 63 active+clean; 141 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 136/6849 objects degraded (1.986%); 1036/6849 objects misplaced (15.126%); 4.1 MiB/s, 65 objects/s recovering 2024-08-17T17:34:29.672 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:29.695 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:29.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:29.797 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:29.821 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:29.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:29.922 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:29.947 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:29.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:29.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:30.048 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:30.073 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:30.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:30.174 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:30.186 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:34:30.198 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:30.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:30.300 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:30.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.325 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:30.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:30.426 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:30.449 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:30.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:30.551 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:30.575 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:30.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:30.578 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2024-08-17T17:34:30.578 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2024-08-17T17:34:30.676 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:30.698 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:30.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:30.800 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:30.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.827 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:30.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:30.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:30 smithi162 ceph-mon[104668]: 10.14 scrub starts 2024-08-17T17:34:30.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:30 smithi162 ceph-mon[104668]: 10.14 scrub ok 2024-08-17T17:34:30.888 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:30 smithi063 ceph-mon[114756]: 10.14 scrub starts 2024-08-17T17:34:30.888 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:30 smithi063 ceph-mon[114756]: 10.14 scrub ok 2024-08-17T17:34:30.888 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:30 smithi063 ceph-mon[109843]: 10.14 scrub starts 2024-08-17T17:34:30.888 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:30 smithi063 ceph-mon[109843]: 10.14 scrub ok 2024-08-17T17:34:30.929 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:30.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:30.952 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:30.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:31.052 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:31.075 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:31.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:31.177 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:31.199 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:31.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:31.301 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:31.327 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:31.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:31.428 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:31.453 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:31.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:31.555 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:31.577 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:31.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:31.679 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:31.705 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:31.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:31.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:31 smithi063 ceph-mon[114756]: pgmap v2032: 129 pgs: 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 63 active+clean; 119 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 136/5793 objects degraded (2.348%); 684/5793 objects misplaced (11.807%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:34:31.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:31 smithi063 ceph-mon[109843]: pgmap v2032: 129 pgs: 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 63 active+clean; 119 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 136/5793 objects degraded (2.348%); 684/5793 objects misplaced (11.807%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:34:31.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:31.807 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:31.830 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:31.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:31.932 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:31.954 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:31.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:31.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:32.056 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:32.082 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:32.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:31 smithi162 ceph-mon[104668]: pgmap v2032: 129 pgs: 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 63 active+clean; 119 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 136/5793 objects degraded (2.348%); 684/5793 objects misplaced (11.807%); 3.2 MiB/s, 51 objects/s recovering 2024-08-17T17:34:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:32.184 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:32.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.208 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:32.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:32.309 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:32.332 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:32.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.333 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:32.433 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:32.456 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:32.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:32.557 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:32.572 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:34:32.581 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:32.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:32.683 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:32.706 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:32.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:32.793 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:34:32.793 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:34:32.808 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:32.812 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:34:32.812 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:34:32.813 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:32 smithi063 ceph-mon[114756]: 1.4 scrub starts 2024-08-17T17:34:32.813 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:32 smithi063 ceph-mon[114756]: 1.4 scrub ok 2024-08-17T17:34:32.813 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:32 smithi063 ceph-mon[114756]: 10.11 scrub starts 2024-08-17T17:34:32.813 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:32 smithi063 ceph-mon[114756]: 10.11 scrub ok 2024-08-17T17:34:32.813 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:32 smithi063 ceph-mon[109843]: 1.4 scrub starts 2024-08-17T17:34:32.813 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:32 smithi063 ceph-mon[109843]: 1.4 scrub ok 2024-08-17T17:34:32.813 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:32 smithi063 ceph-mon[109843]: 10.11 scrub starts 2024-08-17T17:34:32.813 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:32 smithi063 ceph-mon[109843]: 10.11 scrub ok 2024-08-17T17:34:32.831 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:32.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:32.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:32.841 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:34:32.841 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:34:32.858 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:34:32.858 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:34:32.875 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:34:32.875 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:34:32.892 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:34:32.892 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:34:32.892 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:34:32.909 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:34:32.910 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:34:32.933 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:33.009 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:33.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:32 smithi162 ceph-mon[104668]: 1.4 scrub starts 2024-08-17T17:34:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:32 smithi162 ceph-mon[104668]: 1.4 scrub ok 2024-08-17T17:34:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:32 smithi162 ceph-mon[104668]: 10.11 scrub starts 2024-08-17T17:34:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:32 smithi162 ceph-mon[104668]: 10.11 scrub ok 2024-08-17T17:34:33.111 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:33.133 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:33.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:33.235 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:33.254 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:33.259 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:33.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:33.361 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:33.387 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:33.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:33.488 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:33.500 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:34:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:34:33] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.51.0" 2024-08-17T17:34:33.512 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:33.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:33.614 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:33.639 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:33.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:33.740 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:33.751 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:33 smithi063 ceph-mon[109843]: pgmap v2033: 129 pgs: 1 active+clean+scrubbing, 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 62 active+clean; 9.5 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/510 objects degraded (1.961%); 41/510 objects misplaced (8.039%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:34:33.751 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:33 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1075682224' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:33.751 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:33 smithi063 ceph-mon[109843]: 1.7 scrub starts 2024-08-17T17:34:33.751 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:33 smithi063 ceph-mon[109843]: 1.7 scrub ok 2024-08-17T17:34:33.751 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:33 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 10/510 objects degraded (1.961%), 5 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:33.752 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:33 smithi063 ceph-mon[114756]: pgmap v2033: 129 pgs: 1 active+clean+scrubbing, 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 62 active+clean; 9.5 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/510 objects degraded (1.961%); 41/510 objects misplaced (8.039%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:34:33.752 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:33 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1075682224' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:33.752 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:33 smithi063 ceph-mon[114756]: 1.7 scrub starts 2024-08-17T17:34:33.752 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:33 smithi063 ceph-mon[114756]: 1.7 scrub ok 2024-08-17T17:34:33.752 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:33 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 10/510 objects degraded (1.961%), 5 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:33.766 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:33.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:33.868 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:33.890 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:33.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:33.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:33.927 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:33.995 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:34.018 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:34.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:34.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:33 smithi162 ceph-mon[104668]: pgmap v2033: 129 pgs: 1 active+clean+scrubbing, 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 62 active+clean; 9.5 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/510 objects degraded (1.961%); 41/510 objects misplaced (8.039%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:34:34.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:33 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1075682224' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:34.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:33 smithi162 ceph-mon[104668]: 1.7 scrub starts 2024-08-17T17:34:34.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:33 smithi162 ceph-mon[104668]: 1.7 scrub ok 2024-08-17T17:34:34.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:33 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 10/510 objects degraded (1.961%), 5 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:34.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:34.142 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:34.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:34.244 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:34.267 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:34.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:34.368 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:34.394 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:34.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.395 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:34.403 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:34:34.404 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:34:34.496 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:34.521 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:34.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:34.623 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:34.647 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:34.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:34.749 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:34.775 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:34.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:34.876 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:34.899 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:34.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:34.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:35.000 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:35.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.023 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:35.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:35.124 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:35.150 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:35.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:35.252 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:35.276 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:35.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:35.377 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:35.400 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:35.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:35.453 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:34:35.453 INFO:teuthology.orchestra.run.smithi063.stdout: "success": "" 2024-08-17T17:34:35.453 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:34:35.503 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:35.527 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:35.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:35.629 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:35.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.655 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:35.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:35.756 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:35.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:35 smithi063 ceph-mon[114756]: pgmap v2034: 129 pgs: 1 active+clean+scrubbing, 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 62 active+clean; 9.5 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/510 objects degraded (1.961%); 41/510 objects misplaced (8.039%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:34:35.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:35 smithi063 ceph-mon[109843]: pgmap v2034: 129 pgs: 1 active+clean+scrubbing, 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 62 active+clean; 9.5 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/510 objects degraded (1.961%); 41/510 objects misplaced (8.039%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:34:35.782 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:35.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:35.884 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:35.897 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:35 smithi162 ceph-mon[104668]: pgmap v2034: 129 pgs: 1 active+clean+scrubbing, 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 62 active+clean; 9.5 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/510 objects degraded (1.961%); 41/510 objects misplaced (8.039%); 2.4 MiB/s, 38 objects/s recovering 2024-08-17T17:34:35.909 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:35.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:35.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:36.011 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:36.035 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:36.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:36.137 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:36.161 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:36.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:36.263 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:36.289 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:36.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:36.391 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:36.404 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:34:36.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.415 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:36.415 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:36.516 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:36.541 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:36.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:36.643 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:36.668 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:36.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:36.753 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:36.769 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:36.793 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:36.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:36.895 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:36.918 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:36.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:36.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:37.020 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:37.042 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:37.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:37.144 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:37.167 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:37.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:37.269 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:37.295 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:37.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:37.397 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:37.419 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:37.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:37.521 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:37.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:37.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:37.648 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:37.670 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:37.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:37.772 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:37.797 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:37.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:37.899 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:37.924 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:37.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:37.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:37.985 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:37 smithi063 ceph-mon[109843]: pgmap v2035: 129 pgs: 1 active+clean+scrubbing, 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 62 active+clean; 9.5 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/510 objects degraded (1.961%); 41/510 objects misplaced (8.039%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:34:37.985 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:37 smithi063 ceph-mon[109843]: 10.3 scrub starts 2024-08-17T17:34:37.985 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:37 smithi063 ceph-mon[109843]: 10.3 scrub ok 2024-08-17T17:34:37.985 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:37 smithi063 ceph-mon[109843]: 1.2 scrub starts 2024-08-17T17:34:37.985 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:37 smithi063 ceph-mon[109843]: 1.2 scrub ok 2024-08-17T17:34:37.985 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:37 smithi063 ceph-mon[114756]: pgmap v2035: 129 pgs: 1 active+clean+scrubbing, 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 62 active+clean; 9.5 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/510 objects degraded (1.961%); 41/510 objects misplaced (8.039%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:34:37.985 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:37 smithi063 ceph-mon[114756]: 10.3 scrub starts 2024-08-17T17:34:37.985 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:37 smithi063 ceph-mon[114756]: 10.3 scrub ok 2024-08-17T17:34:37.985 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:37 smithi063 ceph-mon[114756]: 1.2 scrub starts 2024-08-17T17:34:37.986 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:37 smithi063 ceph-mon[114756]: 1.2 scrub ok 2024-08-17T17:34:38.026 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:38.049 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:38.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:37 smithi162 ceph-mon[104668]: pgmap v2035: 129 pgs: 1 active+clean+scrubbing, 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 62 active+clean; 9.5 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/510 objects degraded (1.961%); 41/510 objects misplaced (8.039%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:34:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:37 smithi162 ceph-mon[104668]: 10.3 scrub starts 2024-08-17T17:34:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:37 smithi162 ceph-mon[104668]: 10.3 scrub ok 2024-08-17T17:34:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:37 smithi162 ceph-mon[104668]: 1.2 scrub starts 2024-08-17T17:34:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:37 smithi162 ceph-mon[104668]: 1.2 scrub ok 2024-08-17T17:34:38.150 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:38.174 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:38.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:38.275 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:38.298 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:38.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:38.400 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:38.422 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:38.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:38.524 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:38.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.547 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:38.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:38.648 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:38.673 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:38.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:38.774 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:38.798 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:38.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:38.899 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:38.924 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:38.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:38.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:39.026 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:39.048 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:39.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:39.150 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:39.172 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:39.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:39.273 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:39.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:39 smithi063 ceph-mon[114756]: 1.c scrub starts 2024-08-17T17:34:39.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:39 smithi063 ceph-mon[114756]: 1.c scrub ok 2024-08-17T17:34:39.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:39 smithi063 ceph-mon[114756]: pgmap v2036: 129 pgs: 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 63 active+clean; 9.0 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/489 objects degraded (2.045%); 41/489 objects misplaced (8.384%) 2024-08-17T17:34:39.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:39 smithi063 ceph-mon[109843]: 1.c scrub starts 2024-08-17T17:34:39.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:39 smithi063 ceph-mon[109843]: 1.c scrub ok 2024-08-17T17:34:39.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:39 smithi063 ceph-mon[109843]: pgmap v2036: 129 pgs: 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 63 active+clean; 9.0 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/489 objects degraded (2.045%); 41/489 objects misplaced (8.384%) 2024-08-17T17:34:39.295 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:39.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:39.397 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:39.423 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:39.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:39.525 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:39.547 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:39.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:39.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:39 smithi162 ceph-mon[104668]: 1.c scrub starts 2024-08-17T17:34:39.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:39 smithi162 ceph-mon[104668]: 1.c scrub ok 2024-08-17T17:34:39.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:39 smithi162 ceph-mon[104668]: pgmap v2036: 129 pgs: 50 active+clean+remapped, 11 active+undersized, 5 active+undersized+degraded, 63 active+clean; 9.0 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/489 objects degraded (2.045%); 41/489 objects misplaced (8.384%) 2024-08-17T17:34:39.649 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:39.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.672 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:39.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:39.773 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:39.799 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:39.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:39.901 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:39.924 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:39.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:39.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:40.026 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:40.048 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:40.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:40.092 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:34:40.092 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:34:40.125 INFO:teuthology.orchestra.run.smithi063.stderr:pool 'unique_pool_5' removed 2024-08-17T17:34:40.151 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:40.177 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:40.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:40.279 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:40.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:40 smithi063 ceph-mon[109843]: 10.7 scrub starts 2024-08-17T17:34:40.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:40 smithi063 ceph-mon[109843]: 10.7 scrub ok 2024-08-17T17:34:40.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:40 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 10/489 objects degraded (2.045%), 5 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:40.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:40 smithi063 ceph-mon[109843]: 1.6 scrub starts 2024-08-17T17:34:40.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:40 smithi063 ceph-mon[109843]: 1.6 scrub ok 2024-08-17T17:34:40.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:40 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/105270179' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:34:40.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:40 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:34:40.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:40 smithi063 ceph-mon[114756]: 10.7 scrub starts 2024-08-17T17:34:40.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:40 smithi063 ceph-mon[114756]: 10.7 scrub ok 2024-08-17T17:34:40.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:40 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 10/489 objects degraded (2.045%), 5 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:40.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:40 smithi063 ceph-mon[114756]: 1.6 scrub starts 2024-08-17T17:34:40.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:40 smithi063 ceph-mon[114756]: 1.6 scrub ok 2024-08-17T17:34:40.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:40 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/105270179' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:34:40.281 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:40 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:34:40.303 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:40.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:40.405 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:40.430 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:40.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:40.532 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:40.556 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:40.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:40.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:40 smithi162 ceph-mon[104668]: 10.7 scrub starts 2024-08-17T17:34:40.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:40 smithi162 ceph-mon[104668]: 10.7 scrub ok 2024-08-17T17:34:40.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:40 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 10/489 objects degraded (2.045%), 5 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:40.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:40 smithi162 ceph-mon[104668]: 1.6 scrub starts 2024-08-17T17:34:40.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:40 smithi162 ceph-mon[104668]: 1.6 scrub ok 2024-08-17T17:34:40.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:40 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/105270179' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:34:40.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:40 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:34:40.658 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:40.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.682 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:40.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:40.784 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:40.807 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:40.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:40.810 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:34:40.910 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:40.933 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:40.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:40.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:41.035 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:41.064 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:41.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:41.166 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:41.197 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:41.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:41.263 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:41.299 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:41.321 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:41.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:41.384 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 7, 0, 1] out_osds: [2] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:34:41.384 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:34:41.384 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2024-08-17T17:34:41.384 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-08-17T17:34:41.384 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-08-17T17:34:41.385 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 config set filestore_inject_stall 3 2024-08-17T17:34:41.408 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:41 smithi063 ceph-mon[114756]: 10.17 scrub starts 2024-08-17T17:34:41.408 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:41 smithi063 ceph-mon[114756]: 10.17 scrub ok 2024-08-17T17:34:41.408 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:41 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:34:41.408 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:41 smithi063 ceph-mon[114756]: osdmap e1355: 8 total, 7 up, 7 in 2024-08-17T17:34:41.408 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:41 smithi063 ceph-mon[114756]: pgmap v2038: 103 pgs: 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 435 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/75 objects degraded (13.333%); 5/75 objects misplaced (6.667%) 2024-08-17T17:34:41.408 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:41 smithi063 ceph-mon[109843]: 10.17 scrub starts 2024-08-17T17:34:41.408 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:41 smithi063 ceph-mon[109843]: 10.17 scrub ok 2024-08-17T17:34:41.408 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:41 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:34:41.408 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:41 smithi063 ceph-mon[109843]: osdmap e1355: 8 total, 7 up, 7 in 2024-08-17T17:34:41.409 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:41 smithi063 ceph-mon[109843]: pgmap v2038: 103 pgs: 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 435 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/75 objects degraded (13.333%); 5/75 objects misplaced (6.667%) 2024-08-17T17:34:41.423 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:41.445 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:41.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:41.547 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:41.567 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:41.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:41.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:41 smithi162 ceph-mon[104668]: 10.17 scrub starts 2024-08-17T17:34:41.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:41 smithi162 ceph-mon[104668]: 10.17 scrub ok 2024-08-17T17:34:41.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:41 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:34:41.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:41 smithi162 ceph-mon[104668]: osdmap e1355: 8 total, 7 up, 7 in 2024-08-17T17:34:41.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:41 smithi162 ceph-mon[104668]: pgmap v2038: 103 pgs: 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 435 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/75 objects degraded (13.333%); 5/75 objects misplaced (6.667%) 2024-08-17T17:34:41.669 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:41.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.695 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:41.695 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:41.730 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-08-17T17:34:41.731 INFO:tasks.radosbench:Beginning radosbench... 2024-08-17T17:34:41.731 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_6 2024-08-17T17:34:41.731 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool create unique_pool_6 16 2024-08-17T17:34:41.795 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:41.816 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:41.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:41.918 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:41.941 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:41.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:41.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:42.043 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:42.064 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:42.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:42.124 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:34:42.166 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:42.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.190 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:42.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:42.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:42 smithi063 ceph-mon[114756]: 10.19 scrub starts 2024-08-17T17:34:42.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:42 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2304948572' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:42.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:42 smithi063 ceph-mon[109843]: 10.19 scrub starts 2024-08-17T17:34:42.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:42 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2304948572' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:42.291 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:42.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.315 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:42.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:42.416 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:42.438 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:42.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:42.540 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:42.562 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:42.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:42.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:42 smithi162 ceph-mon[104668]: 10.19 scrub starts 2024-08-17T17:34:42.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:42 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2304948572' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:34:42.664 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:42.686 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:42.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:42.788 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:42.810 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:42.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:42.912 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:42.934 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:42.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:42.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:43.036 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:43.059 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:43.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:43.160 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:43.182 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:43.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:43.278 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:43.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:43 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:34:43.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:43 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2304948572' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:34:43.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:43 smithi063 ceph-mon[109843]: osdmap e1356: 8 total, 7 up, 7 in 2024-08-17T17:34:43.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:43 smithi063 ceph-mon[109843]: 1.d scrub starts 2024-08-17T17:34:43.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:43 smithi063 ceph-mon[109843]: 1.d scrub ok 2024-08-17T17:34:43.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:43 smithi063 ceph-mon[109843]: pgmap v2040: 103 pgs: 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:43.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:43 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:34:43.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:43 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2304948572' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:34:43.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:43 smithi063 ceph-mon[114756]: osdmap e1356: 8 total, 7 up, 7 in 2024-08-17T17:34:43.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:43 smithi063 ceph-mon[114756]: 1.d scrub starts 2024-08-17T17:34:43.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:43 smithi063 ceph-mon[114756]: 1.d scrub ok 2024-08-17T17:34:43.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:43 smithi063 ceph-mon[114756]: pgmap v2040: 103 pgs: 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:43.284 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:43.284 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:43.290 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:43.313 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:43.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:43.415 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:43.437 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:43.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:43.539 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:43.561 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:43.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:43.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:43 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:34:43.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:43 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2304948572' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:34:43.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:43 smithi162 ceph-mon[104668]: osdmap e1356: 8 total, 7 up, 7 in 2024-08-17T17:34:43.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:43 smithi162 ceph-mon[104668]: 1.d scrub starts 2024-08-17T17:34:43.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:43 smithi162 ceph-mon[104668]: 1.d scrub ok 2024-08-17T17:34:43.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:43 smithi162 ceph-mon[104668]: pgmap v2040: 103 pgs: 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:43.646 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:34:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:34:43] "GET /metrics HTTP/1.1" 200 36593 "" "Prometheus/2.51.0" 2024-08-17T17:34:43.663 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:43.685 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:43.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:43.787 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:43.810 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:43.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:43.912 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:43.935 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:43.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:43.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:44.037 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:44.059 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:44.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:44.161 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:44.183 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:44.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:44.285 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:44.309 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:44.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:44.411 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:44.434 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:44.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:44.536 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:44.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.559 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:44.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:44.660 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:44.686 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:44.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:44.788 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:44.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.811 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:44.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:44.912 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:44.936 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:44.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:44.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:45.037 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:45.063 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:45.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:45.165 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:45.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.191 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:45.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:45.292 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:45.314 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:45.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:45.416 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:45.438 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:45.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.439 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:45.540 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:45.562 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:45.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:45.664 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:45.668 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:34:45.690 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:45.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:45.793 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:45.816 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:45.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:45.918 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:45.940 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:45.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:45.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:46.014 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:45 smithi063 ceph-mon[109843]: pgmap v2041: 103 pgs: 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:46.014 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:45 smithi063 ceph-mon[114756]: pgmap v2041: 103 pgs: 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:46.041 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:46.063 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:46.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:45 smithi162 ceph-mon[104668]: pgmap v2041: 103 pgs: 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:46.130 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:34:46.130 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:34:46.134 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:34:46.134 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:34:46.155 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:34:46.155 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:34:46.166 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:46.172 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:34:46.172 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:34:46.188 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:46.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:46.189 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:34:46.190 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:34:46.206 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:34:46.207 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:34:46.207 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:34:46.224 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:34:46.224 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:34:46.290 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:46.312 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:46.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:46.414 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:46.440 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:46.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:46.542 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:46.567 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:46.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:46.669 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:46.692 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:46.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:46.794 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:46.820 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:46.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:46.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:46.945 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:46.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:46.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:46 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/460744758' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:46 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:34:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:46 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/460744758' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:46 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:34:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:47.048 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:47.070 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:47.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:46 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/460744758' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:34:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:46 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:34:47.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:34:47.171 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:47.198 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:47.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:47.300 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:47.323 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:47.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:47.425 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:47.451 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:47.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:47.553 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:47.578 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:47.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:47.680 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:47.706 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:47.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:47.808 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:47.830 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:47.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:47.905 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:47 smithi063 ceph-mon[114756]: pgmap v2042: 103 pgs: 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:47.905 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:47 smithi063 ceph-mon[109843]: pgmap v2042: 103 pgs: 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:47.931 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:47.953 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:47.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:47.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:48.055 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:48.077 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:48.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:47 smithi162 ceph-mon[104668]: pgmap v2042: 103 pgs: 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:48.179 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:48.205 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:48.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:48.306 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:48.331 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:48.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:48.433 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:48.458 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:48.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:48.560 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:48.585 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:48.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:48.687 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:48.709 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:48.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:48.745 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:34:48.745 INFO:teuthology.orchestra.run.smithi063.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-08-17T17:34:48.745 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:34:48.811 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:48.836 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:48.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:48.938 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:48.962 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:48.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:48.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:48 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 10/78 objects degraded (12.821%), 4 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:48 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 10/78 objects degraded (12.821%), 4 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:49.064 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:49.088 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:49.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:48 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 10/78 objects degraded (12.821%), 4 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2024-08-17T17:34:49.190 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:49.212 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:49.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:49.314 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:49.319 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:34:49.319 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:34:49.319 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:34:49.319 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:34:49.319 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:34:49.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.338 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:49.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:49.439 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:49.462 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:49.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:49.563 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:49.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.587 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:49.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:49.652 INFO:teuthology.orchestra.run.smithi063.stderr:pool 'unique_pool_6' created 2024-08-17T17:34:49.688 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:49.711 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:49.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:49.813 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:49.839 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:49.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:49.941 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:49.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:49.966 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:49.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:49 smithi063 ceph-mon[114756]: pgmap v2043: 103 pgs: 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:49 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3311542803' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-08-17T17:34:50.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:49 smithi063 ceph-mon[109843]: pgmap v2043: 103 pgs: 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:50.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:49 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3311542803' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-08-17T17:34:50.067 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:50.090 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:50.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:49 smithi162 ceph-mon[104668]: pgmap v2043: 103 pgs: 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:49 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3311542803' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16} : dispatch 2024-08-17T17:34:50.192 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:50.219 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:50.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:50.306 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:34:50.321 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:50.372 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:50.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:50.474 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:50.498 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:50.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:50.600 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:50.623 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:50.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.624 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:50.724 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:50.746 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:50.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:50.848 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:50.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:50.876 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:50.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:50.971 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:50 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3311542803' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-08-17T17:34:50.971 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:50 smithi063 ceph-mon[109843]: osdmap e1357: 8 total, 7 up, 7 in 2024-08-17T17:34:50.971 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:50 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3311542803' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-08-17T17:34:50.971 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:50 smithi063 ceph-mon[114756]: osdmap e1357: 8 total, 7 up, 7 in 2024-08-17T17:34:50.977 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:51.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.003 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:51.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:51.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:50 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3311542803' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_6", "pg_num": 16}]': finished 2024-08-17T17:34:51.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:50 smithi162 ceph-mon[104668]: osdmap e1357: 8 total, 7 up, 7 in 2024-08-17T17:34:51.106 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:51.131 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:51.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:51.233 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:51.258 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:51.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:51.360 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:51.383 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:51.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:51.470 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool application enable unique_pool_6 rados --yes-i-really-mean-it || true 2024-08-17T17:34:51.485 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:51.510 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:51.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:51.612 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:51.634 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:51.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:51.736 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:51.736 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:51 smithi063 ceph-mon[109843]: pgmap v2045: 119 pgs: 16 unknown, 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:51.737 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:51 smithi063 ceph-mon[109843]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:34:51.737 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:51 smithi063 ceph-mon[109843]: osdmap e1358: 8 total, 7 up, 7 in 2024-08-17T17:34:51.737 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:51 smithi063 ceph-mon[114756]: pgmap v2045: 119 pgs: 16 unknown, 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:51.737 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:51 smithi063 ceph-mon[114756]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:34:51.737 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:51 smithi063 ceph-mon[114756]: osdmap e1358: 8 total, 7 up, 7 in 2024-08-17T17:34:51.759 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:51.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:51.860 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:51.886 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:51.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:51.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:51.988 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:52.010 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:52.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:52.059 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:34:52.060 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:34:52.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:51 smithi162 ceph-mon[104668]: pgmap v2045: 119 pgs: 16 unknown, 42 active+clean+remapped, 11 active+undersized, 4 active+undersized+degraded, 46 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:52.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:51 smithi162 ceph-mon[104668]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:34:52.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:51 smithi162 ceph-mon[104668]: osdmap e1358: 8 total, 7 up, 7 in 2024-08-17T17:34:52.111 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:52.111 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:52.113 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:52.115 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:34:52.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.140 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:52.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:52.241 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:52.267 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:52.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:52.369 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:52.392 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:52.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:52.494 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:52.516 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:52.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:52.618 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:52.640 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:52.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:52.668 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:52 smithi063 ceph-mon[109843]: osdmap e1359: 8 total, 7 up, 7 in 2024-08-17T17:34:52.742 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:52.768 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:52.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:52.870 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:52.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:52.894 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:52.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:52.983 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:52 smithi063 ceph-mon[114756]: osdmap e1359: 8 total, 7 up, 7 in 2024-08-17T17:34:52.995 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:53.018 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:53.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:52 smithi162 ceph-mon[104668]: osdmap e1359: 8 total, 7 up, 7 in 2024-08-17T17:34:53.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:53.141 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:53.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:53.246 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:53.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.270 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:53.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:53.370 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:53.393 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:53.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:53.495 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:53.517 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:53.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:53.604 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:34:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:34:53] "GET /metrics HTTP/1.1" 200 34164 "" "Prometheus/2.51.0" 2024-08-17T17:34:53.619 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:53.645 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:53.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:53.747 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:53.770 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:53.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:53.872 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:53.893 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:53.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:53.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:53.995 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:54.016 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:54.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:53 smithi063 ceph-mon[114756]: pgmap v2048: 119 pgs: 2 unknown, 42 active+clean+remapped, 19 active+undersized, 4 active+undersized+degraded, 52 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:54.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:53 smithi063 ceph-mon[109843]: pgmap v2048: 119 pgs: 2 unknown, 42 active+clean+remapped, 19 active+undersized, 4 active+undersized+degraded, 52 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:54.060 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:34:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:53 smithi162 ceph-mon[104668]: pgmap v2048: 119 pgs: 2 unknown, 42 active+clean+remapped, 19 active+undersized, 4 active+undersized+degraded, 52 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:54.119 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:54.139 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:54.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:54.241 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:54.263 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:54.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:54.365 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:54.387 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:54.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:54.489 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:54.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.512 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:54.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:54.613 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:54.634 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:54.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:54.736 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:34:54.757 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:54.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:34:54.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:54 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:34:54.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:54 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:34:54.860 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:54.882 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:54.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:54.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:54.985 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:55.006 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:55.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:55.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:54 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:34:55.108 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:55.134 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:55.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:55.237 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:55.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.261 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:55.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:55.362 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:55.384 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:55.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:55.486 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:55.509 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:55.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:55.611 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:55.634 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:55.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:55.736 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:55.759 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:55.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:55.861 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:55.883 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:55.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:55.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:55.920 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:34:55.920 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:34:55.920 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:34:55.920 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:34:55.920 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.920 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45218.0:236015 8.a 8:554a8f74:::benchmark_data_smithi063_230025_object109525:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1241)", 2024-08-17T17:34:55.921 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:25:54.315639+0000", 2024-08-17T17:34:55.921 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 541.55900170500001, 2024-08-17T17:34:55.921 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.32538034700000001, 2024-08-17T17:34:55.921 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:55.921 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:55.921 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:55.921 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:55.921 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45218", 2024-08-17T17:34:55.921 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/3307648358", 2024-08-17T17:34:55.921 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 236015 2024-08-17T17:34:55.921 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.922 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:55.922 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.922 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:55.922 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:25:54.315639+0000", 2024-08-17T17:34:55.922 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.922 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.922 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.922 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:55.922 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:25:54.315639+0000", 2024-08-17T17:34:55.922 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.922 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.922 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.923 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:55.923 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:25:54.315641+0000", 2024-08-17T17:34:55.923 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.655e-06 2024-08-17T17:34:55.923 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.923 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.923 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:55.923 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:25:54.315647+0000", 2024-08-17T17:34:55.923 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.178e-06 2024-08-17T17:34:55.923 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.923 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.923 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:55.923 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:25:54.315669+0000", 2024-08-17T17:34:55.923 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1829000000000001e-05 2024-08-17T17:34:55.924 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.924 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.924 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:55.924 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:25:54.315678+0000", 2024-08-17T17:34:55.924 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0929999999999992e-06 2024-08-17T17:34:55.924 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.924 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.924 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:55.924 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:25:54.315761+0000", 2024-08-17T17:34:55.924 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2966999999999994e-05 2024-08-17T17:34:55.924 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.924 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.925 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:55.925 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:25:54.315890+0000", 2024-08-17T17:34:55.925 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000128765 2024-08-17T17:34:55.925 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.925 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.925 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,5", 2024-08-17T17:34:55.925 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:25:54.316005+0000", 2024-08-17T17:34:55.925 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000115275 2024-08-17T17:34:55.925 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.925 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.925 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.925 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:25:54.323847+0000", 2024-08-17T17:34:55.925 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0078412719999999998 2024-08-17T17:34:55.926 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.926 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.926 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:55.926 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:25:54.323928+0000", 2024-08-17T17:34:55.926 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.1055000000000002e-05 2024-08-17T17:34:55.926 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.926 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.926 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.926 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:25:54.640898+0000", 2024-08-17T17:34:55.926 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.31697044099999999 2024-08-17T17:34:55.926 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.926 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.926 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:55.927 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:25:54.640980+0000", 2024-08-17T17:34:55.927 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2169000000000006e-05 2024-08-17T17:34:55.927 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.927 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.927 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:55.927 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:25:54.641020+0000", 2024-08-17T17:34:55.927 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9647999999999997e-05 2024-08-17T17:34:55.927 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.927 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:55.927 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.927 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.927 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.927 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:44821 9.2 e1252/1247 9:42c04f20:::benchmark_data_smithi063_263014_object44820:head v 1252'2818, mlcod=1252'2815)", 2024-08-17T17:34:55.927 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:08.016432+0000", 2024-08-17T17:34:55.928 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 467.85820910899997, 2024-08-17T17:34:55.928 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.34546906500000002, 2024-08-17T17:34:55.928 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:55.928 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:55.928 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:55.928 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:55.928 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.928 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:55.928 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.016432+0000", 2024-08-17T17:34:55.928 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.928 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.928 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.928 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:55.928 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.016432+0000", 2024-08-17T17:34:55.929 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.929 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.929 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.929 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:55.929 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.016433+0000", 2024-08-17T17:34:55.929 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9000000000000005e-07 2024-08-17T17:34:55.929 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.929 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.929 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:55.929 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.016496+0000", 2024-08-17T17:34:55.929 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2683999999999998e-05 2024-08-17T17:34:55.929 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.929 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.930 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:55.930 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.016507+0000", 2024-08-17T17:34:55.930 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.115e-05 2024-08-17T17:34:55.930 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.930 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.930 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:55.930 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.016514+0000", 2024-08-17T17:34:55.930 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9870000000000002e-06 2024-08-17T17:34:55.930 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.930 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.930 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:55.930 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.016557+0000", 2024-08-17T17:34:55.930 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3407e-05 2024-08-17T17:34:55.931 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.931 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.931 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:55.931 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.016576+0000", 2024-08-17T17:34:55.931 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8457e-05 2024-08-17T17:34:55.931 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.931 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.931 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:55.931 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.361852+0000", 2024-08-17T17:34:55.931 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.34527579899999999 2024-08-17T17:34:55.931 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.931 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.931 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:55.932 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.361901+0000", 2024-08-17T17:34:55.932 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9591e-05 2024-08-17T17:34:55.932 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.932 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:55.932 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.932 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.932 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.932 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45521.0:44824 9.2 e1252/1247 9:4d91f1c9:::benchmark_data_smithi063_263014_object44823:head v 1252'2819, mlcod=1252'2815)", 2024-08-17T17:34:55.932 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:08.017858+0000", 2024-08-17T17:34:55.932 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 467.85678320099998, 2024-08-17T17:34:55.932 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.34623521400000001, 2024-08-17T17:34:55.932 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:55.932 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:55.933 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:55.933 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:55.933 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.933 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:55.933 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017858+0000", 2024-08-17T17:34:55.933 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.933 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.933 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.933 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:55.933 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017858+0000", 2024-08-17T17:34:55.933 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.933 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.933 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.933 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:55.933 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017859+0000", 2024-08-17T17:34:55.934 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.9700000000000005e-07 2024-08-17T17:34:55.934 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.934 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.934 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:55.934 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.018217+0000", 2024-08-17T17:34:55.934 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00035857599999999999 2024-08-17T17:34:55.934 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.934 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.934 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:55.934 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.358524+0000", 2024-08-17T17:34:55.934 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.34030651000000001 2024-08-17T17:34:55.934 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.935 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.935 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:55.935 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.358538+0000", 2024-08-17T17:34:55.935 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3745e-05 2024-08-17T17:34:55.935 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.935 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.935 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:55.935 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.359500+0000", 2024-08-17T17:34:55.935 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00096184600000000003 2024-08-17T17:34:55.935 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.935 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.935 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:55.935 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.359517+0000", 2024-08-17T17:34:55.935 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7688000000000001e-05 2024-08-17T17:34:55.936 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.936 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.936 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:55.936 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364045+0000", 2024-08-17T17:34:55.936 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0045272580000000002 2024-08-17T17:34:55.936 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.936 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.936 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:55.936 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364093+0000", 2024-08-17T17:34:55.936 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8693999999999998e-05 2024-08-17T17:34:55.936 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.936 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:55.936 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.937 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.937 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.937 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "MOSDPGPush(10.a 1307/1299 [PushOp(10:5fd3f15c:::benchmark_data_smithi063_294527_object12573:head, version: 1292'773, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:5fd3f15c:::benchmark_data_smithi063_294527_object12573:head@1292'773, size: 65536, copy_subset: [(0, 65536)], clone_subset: {}, snapset: 0=[]:{}, object_exist: false), after_progress: ObjectRecoveryProgress(!first, data_recovered_to: 65536, data_complete: true, omap_recovered_to: , omap_complete: true, error: false), before_progress: ObjectRecoveryProgress(first, data_recovered_to: 0, data_complete: false, omap_recovered_to: , omap_complete: false, error: false))])", 2024-08-17T17:34:55.937 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:13.133412+0000", 2024-08-17T17:34:55.937 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 162.74122878099999, 2024-08-17T17:34:55.937 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.30574706600000001, 2024-08-17T17:34:55.937 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:55.937 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:55.937 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:34:55.937 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:55.937 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.937 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:55.937 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:13.133412+0000", 2024-08-17T17:34:55.938 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.938 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.938 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.938 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:55.938 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:13.133412+0000", 2024-08-17T17:34:55.938 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.938 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.938 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.938 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:55.938 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:13.133414+0000", 2024-08-17T17:34:55.938 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4980000000000001e-06 2024-08-17T17:34:55.938 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.938 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.939 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:55.939 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:13.133458+0000", 2024-08-17T17:34:55.939 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4094999999999997e-05 2024-08-17T17:34:55.939 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.939 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.939 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:55.939 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:13.133492+0000", 2024-08-17T17:34:55.939 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.3704000000000001e-05 2024-08-17T17:34:55.939 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.939 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.939 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:55.939 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:13.133511+0000", 2024-08-17T17:34:55.939 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9559e-05 2024-08-17T17:34:55.940 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.940 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.940 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:55.940 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:13.434436+0000", 2024-08-17T17:34:55.940 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.30092502799999998 2024-08-17T17:34:55.940 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.940 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.940 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:55.940 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:13.434448+0000", 2024-08-17T17:34:55.940 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1462e-05 2024-08-17T17:34:55.940 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.940 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.940 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:55.941 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:13.439159+0000", 2024-08-17T17:34:55.941 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00471172 2024-08-17T17:34:55.941 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.941 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:55.941 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.941 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.941 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.941 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:136286 10.5 10:a342eec6:::benchmark_data_smithi063_294527_object136285:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1307)", 2024-08-17T17:34:55.941 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:14.133233+0000", 2024-08-17T17:34:55.941 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 161.741407866, 2024-08-17T17:34:55.941 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.25713804200000001, 2024-08-17T17:34:55.941 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:55.941 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:55.942 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:55.942 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:55.942 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:55.942 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:55.942 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 136286 2024-08-17T17:34:55.942 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.942 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:55.942 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.942 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:55.942 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.133233+0000", 2024-08-17T17:34:55.942 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.942 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.942 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.942 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:55.943 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.133233+0000", 2024-08-17T17:34:55.943 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.943 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.943 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.943 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:55.943 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.133235+0000", 2024-08-17T17:34:55.943 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.209e-06 2024-08-17T17:34:55.943 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.943 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.943 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:55.943 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.133351+0000", 2024-08-17T17:34:55.943 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011694 2024-08-17T17:34:55.944 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.944 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.944 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:55.944 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.133363+0000", 2024-08-17T17:34:55.944 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1059e-05 2024-08-17T17:34:55.944 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.944 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.944 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:55.944 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.133369+0000", 2024-08-17T17:34:55.944 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9099999999999999e-06 2024-08-17T17:34:55.944 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.944 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.944 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:55.944 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.172981+0000", 2024-08-17T17:34:55.945 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.039611449999999999 2024-08-17T17:34:55.945 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.945 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.945 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:55.945 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.173156+0000", 2024-08-17T17:34:55.945 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000174642 2024-08-17T17:34:55.945 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.945 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.945 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:34:55.945 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.173517+0000", 2024-08-17T17:34:55.945 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000361644 2024-08-17T17:34:55.945 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.945 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.945 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:55.946 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.175900+0000", 2024-08-17T17:34:55.946 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002382824 2024-08-17T17:34:55.946 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.946 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.946 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.946 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.188968+0000", 2024-08-17T17:34:55.946 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.013068392 2024-08-17T17:34:55.946 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.946 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.946 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.946 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.390140+0000", 2024-08-17T17:34:55.946 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.201171931 2024-08-17T17:34:55.946 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.947 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.947 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:55.947 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.390207+0000", 2024-08-17T17:34:55.947 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7083999999999997e-05 2024-08-17T17:34:55.947 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.947 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.947 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:55.947 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.390371+0000", 2024-08-17T17:34:55.947 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000163957 2024-08-17T17:34:55.947 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.947 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:55.947 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.947 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.948 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.948 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:136287 10.5 10:a4ff40a8:::benchmark_data_smithi063_294527_object136286:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1307)", 2024-08-17T17:34:55.948 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:14.133387+0000", 2024-08-17T17:34:55.948 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 161.74125458500001, 2024-08-17T17:34:55.948 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.25718428599999998, 2024-08-17T17:34:55.948 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:55.948 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:55.948 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:55.948 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:55.948 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:55.948 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:55.948 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 136287 2024-08-17T17:34:55.948 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.949 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:55.949 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.949 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:55.949 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.133387+0000", 2024-08-17T17:34:55.949 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.949 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.949 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.949 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:55.949 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.133387+0000", 2024-08-17T17:34:55.949 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.949 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.949 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.949 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:55.949 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.133387+0000", 2024-08-17T17:34:55.950 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2099999999999996e-07 2024-08-17T17:34:55.950 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.950 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.950 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:55.950 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.133407+0000", 2024-08-17T17:34:55.950 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0029000000000001e-05 2024-08-17T17:34:55.950 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.950 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.950 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:55.950 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.133414+0000", 2024-08-17T17:34:55.950 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.08e-06 2024-08-17T17:34:55.950 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.950 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.951 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:55.951 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.133419+0000", 2024-08-17T17:34:55.951 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9869999999999999e-06 2024-08-17T17:34:55.951 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.951 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.951 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:55.951 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.177527+0000", 2024-08-17T17:34:55.951 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.044108069999999999 2024-08-17T17:34:55.951 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.951 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.951 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:55.951 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.177661+0000", 2024-08-17T17:34:55.951 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000133895 2024-08-17T17:34:55.952 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.952 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.952 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:34:55.952 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.178101+0000", 2024-08-17T17:34:55.952 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000439896 2024-08-17T17:34:55.952 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.952 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.952 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:55.952 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.185783+0000", 2024-08-17T17:34:55.952 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0076817229999999997 2024-08-17T17:34:55.953 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.953 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.953 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.953 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.190981+0000", 2024-08-17T17:34:55.953 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0051979569999999996 2024-08-17T17:34:55.953 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.953 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.953 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.953 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.390460+0000", 2024-08-17T17:34:55.953 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.199479513 2024-08-17T17:34:55.953 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.953 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.953 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:55.954 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.390520+0000", 2024-08-17T17:34:55.954 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9787000000000002e-05 2024-08-17T17:34:55.954 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.954 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.954 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:55.954 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.390571+0000", 2024-08-17T17:34:55.954 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0727999999999999e-05 2024-08-17T17:34:55.954 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.954 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:55.954 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.954 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.954 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.954 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:136291 10.15 10:acf7d48c:::benchmark_data_smithi063_294527_object136290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1307)", 2024-08-17T17:34:55.955 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:14.134505+0000", 2024-08-17T17:34:55.955 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 161.74013598499999, 2024-08-17T17:34:55.955 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.25577408299999999, 2024-08-17T17:34:55.955 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:55.955 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:55.955 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:55.955 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:55.955 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:55.955 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:55.955 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 136291 2024-08-17T17:34:55.955 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.955 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:55.955 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.956 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:55.956 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.134505+0000", 2024-08-17T17:34:55.956 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.956 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.956 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.956 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:55.956 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.134505+0000", 2024-08-17T17:34:55.956 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.956 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.956 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.956 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:55.956 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.134506+0000", 2024-08-17T17:34:55.957 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6699999999999999e-07 2024-08-17T17:34:55.957 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.957 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.957 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:55.957 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.134522+0000", 2024-08-17T17:34:55.957 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6350000000000001e-05 2024-08-17T17:34:55.957 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.957 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.957 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:55.957 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.134532+0000", 2024-08-17T17:34:55.957 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0311e-05 2024-08-17T17:34:55.957 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.957 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.958 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:55.958 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.134538+0000", 2024-08-17T17:34:55.958 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9479999999999996e-06 2024-08-17T17:34:55.958 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.958 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.958 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:55.958 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.176986+0000", 2024-08-17T17:34:55.958 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.042447592999999999 2024-08-17T17:34:55.958 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.958 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.958 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:55.958 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.177155+0000", 2024-08-17T17:34:55.958 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000169486 2024-08-17T17:34:55.959 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.959 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.959 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:34:55.959 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.177501+0000", 2024-08-17T17:34:55.959 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0003455 2024-08-17T17:34:55.959 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.959 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.959 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:55.959 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.180995+0000", 2024-08-17T17:34:55.959 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00349439 2024-08-17T17:34:55.959 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.959 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.959 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.960 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.187177+0000", 2024-08-17T17:34:55.960 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.006182056 2024-08-17T17:34:55.960 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.960 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.960 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.960 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.390118+0000", 2024-08-17T17:34:55.960 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20294103299999999 2024-08-17T17:34:55.960 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.960 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.960 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:55.960 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.390215+0000", 2024-08-17T17:34:55.960 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.6997000000000007e-05 2024-08-17T17:34:55.960 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.960 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.961 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:55.961 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.390279+0000", 2024-08-17T17:34:55.961 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4052000000000003e-05 2024-08-17T17:34:55.961 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.961 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:55.961 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.961 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.961 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.961 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:136299 10.15 10:acb86603:::benchmark_data_smithi063_294527_object136298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1307)", 2024-08-17T17:34:55.961 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:32:14.140314+0000", 2024-08-17T17:34:55.961 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 161.73432679000001, 2024-08-17T17:34:55.961 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.25049849299999999, 2024-08-17T17:34:55.961 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:55.962 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:55.962 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:55.962 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:55.962 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:55.962 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:55.962 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 136299 2024-08-17T17:34:55.962 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.962 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:55.962 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.962 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:55.962 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.140314+0000", 2024-08-17T17:34:55.962 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.962 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.963 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.963 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:55.963 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.140314+0000", 2024-08-17T17:34:55.963 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.963 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.963 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.963 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:55.963 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.140316+0000", 2024-08-17T17:34:55.963 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1620000000000001e-06 2024-08-17T17:34:55.963 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.963 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.963 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:55.963 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.140342+0000", 2024-08-17T17:34:55.963 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6120000000000001e-05 2024-08-17T17:34:55.964 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.964 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.964 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:55.964 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.140356+0000", 2024-08-17T17:34:55.964 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4776999999999999e-05 2024-08-17T17:34:55.964 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.964 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.964 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:55.964 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.140363+0000", 2024-08-17T17:34:55.964 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9809999999999997e-06 2024-08-17T17:34:55.964 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.964 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.964 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:55.964 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.179017+0000", 2024-08-17T17:34:55.965 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.038653060000000003 2024-08-17T17:34:55.965 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.965 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.965 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:55.965 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.179223+0000", 2024-08-17T17:34:55.965 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00020646400000000001 2024-08-17T17:34:55.965 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.965 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.965 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:34:55.965 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.179530+0000", 2024-08-17T17:34:55.965 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00030656299999999999 2024-08-17T17:34:55.965 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.965 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.965 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:55.966 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.187321+0000", 2024-08-17T17:34:55.966 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0077916340000000004 2024-08-17T17:34:55.966 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.966 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.966 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.966 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.189119+0000", 2024-08-17T17:34:55.966 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0017982969999999999 2024-08-17T17:34:55.966 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.966 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.966 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.966 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.390676+0000", 2024-08-17T17:34:55.966 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.20155627200000001 2024-08-17T17:34:55.967 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.967 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.967 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:55.967 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.390723+0000", 2024-08-17T17:34:55.967 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7623999999999997e-05 2024-08-17T17:34:55.967 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.967 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.967 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:55.967 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:32:14.390813+0000", 2024-08-17T17:34:55.967 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.9538999999999998e-05 2024-08-17T17:34:55.967 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.967 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:55.967 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.968 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.968 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.968 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:194852 10.15 10:a9b1e74c:::benchmark_data_smithi063_294527_object57667:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1315)", 2024-08-17T17:34:55.968 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.325062+0000", 2024-08-17T17:34:55.968 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 112.549579313, 2024-08-17T17:34:55.968 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41615492399999998, 2024-08-17T17:34:55.968 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:55.968 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:55.968 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:55.968 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:55.968 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:55.968 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:55.968 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 194852 2024-08-17T17:34:55.968 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.969 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:55.969 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.969 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:55.969 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325062+0000", 2024-08-17T17:34:55.969 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.969 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.969 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.969 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:55.969 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325062+0000", 2024-08-17T17:34:55.969 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.969 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.969 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.969 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:55.969 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325063+0000", 2024-08-17T17:34:55.970 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.096e-06 2024-08-17T17:34:55.970 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.970 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.970 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:55.970 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325070+0000", 2024-08-17T17:34:55.970 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1500000000000002e-06 2024-08-17T17:34:55.970 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.970 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.970 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:55.970 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325081+0000", 2024-08-17T17:34:55.970 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0804e-05 2024-08-17T17:34:55.970 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.971 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.971 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:55.971 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325088+0000", 2024-08-17T17:34:55.971 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.754e-06 2024-08-17T17:34:55.971 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.971 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.971 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:55.971 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325149+0000", 2024-08-17T17:34:55.971 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1291999999999995e-05 2024-08-17T17:34:55.971 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.971 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.971 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:55.971 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325255+0000", 2024-08-17T17:34:55.972 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010603900000000001 2024-08-17T17:34:55.972 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.972 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.972 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:34:55.972 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325724+0000", 2024-08-17T17:34:55.972 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00046944499999999998 2024-08-17T17:34:55.972 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.972 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.972 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:55.972 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.328147+0000", 2024-08-17T17:34:55.972 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002422895 2024-08-17T17:34:55.972 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.972 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.972 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.973 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.328804+0000", 2024-08-17T17:34:55.973 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00065621400000000002 2024-08-17T17:34:55.973 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.973 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.973 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.973 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.741137+0000", 2024-08-17T17:34:55.973 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41233342699999997 2024-08-17T17:34:55.973 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.973 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.973 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:55.973 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.741181+0000", 2024-08-17T17:34:55.973 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3937000000000003e-05 2024-08-17T17:34:55.973 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.974 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.974 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:55.974 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.741217+0000", 2024-08-17T17:34:55.974 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5871000000000003e-05 2024-08-17T17:34:55.974 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.974 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:55.974 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.974 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.974 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.974 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:194860 10.8 10:132c625d:::benchmark_data_smithi063_294527_object57675:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1315)", 2024-08-17T17:34:55.974 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.328608+0000", 2024-08-17T17:34:55.974 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 112.54603357400001, 2024-08-17T17:34:55.974 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41232137000000002, 2024-08-17T17:34:55.975 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:55.975 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:55.975 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:55.975 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:55.975 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:55.975 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:55.975 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 194860 2024-08-17T17:34:55.975 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.975 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:55.975 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.975 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:55.975 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.328608+0000", 2024-08-17T17:34:55.975 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.975 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.976 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.976 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:55.976 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.328608+0000", 2024-08-17T17:34:55.976 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.976 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.976 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.976 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:55.976 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.328609+0000", 2024-08-17T17:34:55.976 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0890000000000001e-06 2024-08-17T17:34:55.976 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.976 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.976 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:55.976 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.328612+0000", 2024-08-17T17:34:55.977 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6909999999999999e-06 2024-08-17T17:34:55.977 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.977 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.977 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:55.977 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.328628+0000", 2024-08-17T17:34:55.977 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5342999999999999e-05 2024-08-17T17:34:55.977 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.977 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.977 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:55.977 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.328636+0000", 2024-08-17T17:34:55.977 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.79e-06 2024-08-17T17:34:55.977 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.977 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.977 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:55.978 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.328706+0000", 2024-08-17T17:34:55.978 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0244e-05 2024-08-17T17:34:55.978 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.978 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.978 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:55.978 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.328840+0000", 2024-08-17T17:34:55.978 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000134561 2024-08-17T17:34:55.978 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.978 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.978 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:34:55.978 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.329304+0000", 2024-08-17T17:34:55.978 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00046334499999999999 2024-08-17T17:34:55.979 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.979 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.979 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:55.979 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.331482+0000", 2024-08-17T17:34:55.979 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0021783229999999998 2024-08-17T17:34:55.979 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.979 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.979 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.979 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.334002+0000", 2024-08-17T17:34:55.979 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0025196849999999998 2024-08-17T17:34:55.979 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.979 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.979 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.979 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.740815+0000", 2024-08-17T17:34:55.979 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.40681344899999999 2024-08-17T17:34:55.980 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.980 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.980 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:55.980 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.740873+0000", 2024-08-17T17:34:55.980 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7485999999999998e-05 2024-08-17T17:34:55.980 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.980 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.980 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:55.980 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.740929+0000", 2024-08-17T17:34:55.980 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6363999999999998e-05 2024-08-17T17:34:55.980 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.980 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:55.980 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.981 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.981 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.981 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:194864 10.5 10:a7e82a38:::benchmark_data_smithi063_294527_object57679:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1315)", 2024-08-17T17:34:55.981 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.333926+0000", 2024-08-17T17:34:55.981 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 112.540715167, 2024-08-17T17:34:55.981 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.43303359000000002, 2024-08-17T17:34:55.981 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:55.981 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:55.981 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:55.981 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:55.981 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:55.981 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:55.981 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 194864 2024-08-17T17:34:55.982 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.982 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:55.982 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.982 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:55.982 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.333926+0000", 2024-08-17T17:34:55.982 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.982 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.982 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.982 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:55.982 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.333926+0000", 2024-08-17T17:34:55.982 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.982 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.982 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.982 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:55.983 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.333927+0000", 2024-08-17T17:34:55.983 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0299999999999998e-07 2024-08-17T17:34:55.983 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.983 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.983 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:55.983 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.333930+0000", 2024-08-17T17:34:55.983 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.512e-06 2024-08-17T17:34:55.983 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.983 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.983 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:55.983 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.333940+0000", 2024-08-17T17:34:55.983 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0156999999999999e-05 2024-08-17T17:34:55.983 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.984 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.984 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:55.984 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.333947+0000", 2024-08-17T17:34:55.984 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.212e-06 2024-08-17T17:34:55.984 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.984 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.984 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:55.984 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.333996+0000", 2024-08-17T17:34:55.984 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9786000000000001e-05 2024-08-17T17:34:55.984 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.984 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.984 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:55.984 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.334106+0000", 2024-08-17T17:34:55.985 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010990200000000001 2024-08-17T17:34:55.985 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.985 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.985 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:34:55.985 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.334612+0000", 2024-08-17T17:34:55.985 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00050531399999999998 2024-08-17T17:34:55.985 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.985 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.985 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:55.985 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.337683+0000", 2024-08-17T17:34:55.985 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0030710799999999999 2024-08-17T17:34:55.985 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.985 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.986 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.986 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.337892+0000", 2024-08-17T17:34:55.986 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000209073 2024-08-17T17:34:55.986 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.986 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.986 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.986 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.766804+0000", 2024-08-17T17:34:55.986 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.428911928 2024-08-17T17:34:55.986 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.986 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.986 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:55.986 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.766902+0000", 2024-08-17T17:34:55.987 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.8641000000000006e-05 2024-08-17T17:34:55.987 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.987 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.987 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:55.987 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.766960+0000", 2024-08-17T17:34:55.987 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7281999999999998e-05 2024-08-17T17:34:55.987 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.987 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:55.987 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.987 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.987 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.987 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:194871 10.18 10:19570010:::benchmark_data_smithi063_294527_object57686:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1315)", 2024-08-17T17:34:55.987 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.342863+0000", 2024-08-17T17:34:55.987 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 112.53177777099999, 2024-08-17T17:34:55.988 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.39921943500000001, 2024-08-17T17:34:55.988 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:55.988 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:55.988 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:55.988 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:34:55.988 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:34:55.988 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:34:55.988 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 194871 2024-08-17T17:34:55.988 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.988 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:55.988 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.988 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:55.988 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.342863+0000", 2024-08-17T17:34:55.989 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.989 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.989 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.989 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:55.989 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.342863+0000", 2024-08-17T17:34:55.989 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.989 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.989 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.989 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:55.989 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.342864+0000", 2024-08-17T17:34:55.989 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9900000000000009e-07 2024-08-17T17:34:55.989 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.989 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.990 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:55.990 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.342868+0000", 2024-08-17T17:34:55.990 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5159999999999999e-06 2024-08-17T17:34:55.990 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.990 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.990 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:55.990 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.342878+0000", 2024-08-17T17:34:55.990 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0112000000000001e-05 2024-08-17T17:34:55.990 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.990 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.990 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:55.990 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.342884+0000", 2024-08-17T17:34:55.990 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5029999999999997e-06 2024-08-17T17:34:55.990 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.991 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.991 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:55.991 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.342945+0000", 2024-08-17T17:34:55.991 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1713999999999999e-05 2024-08-17T17:34:55.991 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.991 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.991 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:55.991 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.343026+0000", 2024-08-17T17:34:55.991 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0592000000000004e-05 2024-08-17T17:34:55.991 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.991 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.991 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:34:55.991 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.343399+0000", 2024-08-17T17:34:55.992 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00037350699999999997 2024-08-17T17:34:55.992 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.992 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.992 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:34:55.992 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.345410+0000", 2024-08-17T17:34:55.992 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002010852 2024-08-17T17:34:55.992 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.992 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.992 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.992 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.345901+0000", 2024-08-17T17:34:55.992 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000490896 2024-08-17T17:34:55.992 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.993 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.993 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:34:55.993 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.740851+0000", 2024-08-17T17:34:55.993 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.39494993699999997 2024-08-17T17:34:55.993 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.993 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.993 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:55.993 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.740900+0000", 2024-08-17T17:34:55.993 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8828000000000001e-05 2024-08-17T17:34:55.993 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.993 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.993 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:55.993 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.742083+0000", 2024-08-17T17:34:55.993 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001182979 2024-08-17T17:34:55.994 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.994 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:55.994 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.994 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.994 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.994 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:271087 10.7 e1354/1350 10:e43105b8:::benchmark_data_smithi063_294527_object133902:head v 1354'12349, mlcod=1353'12347)", 2024-08-17T17:34:55.994 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:26.138942+0000", 2024-08-17T17:34:55.994 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 29.735699684, 2024-08-17T17:34:55.994 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.23474012399999999, 2024-08-17T17:34:55.994 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:55.994 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:55.994 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:55.994 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:55.995 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.995 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:55.995 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.138942+0000", 2024-08-17T17:34:55.995 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.995 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.995 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.995 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:55.995 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.138942+0000", 2024-08-17T17:34:55.995 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:55.995 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.995 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.996 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:55.996 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.138942+0000", 2024-08-17T17:34:55.996 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.06e-07 2024-08-17T17:34:55.996 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.996 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.996 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:55.996 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.138947+0000", 2024-08-17T17:34:55.996 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2499999999999997e-06 2024-08-17T17:34:55.996 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.996 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.996 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:55.996 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.138985+0000", 2024-08-17T17:34:55.997 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8050000000000003e-05 2024-08-17T17:34:55.997 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.997 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.997 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:55.997 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.138992+0000", 2024-08-17T17:34:55.997 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4579999999999998e-06 2024-08-17T17:34:55.997 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.997 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.997 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:55.997 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.367286+0000", 2024-08-17T17:34:55.997 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.22829414000000001 2024-08-17T17:34:55.997 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.998 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.998 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:55.998 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.367314+0000", 2024-08-17T17:34:55.998 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8050000000000001e-05 2024-08-17T17:34:55.998 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.998 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.998 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:55.998 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.373639+0000", 2024-08-17T17:34:55.998 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0063244820000000002 2024-08-17T17:34:55.998 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.998 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.998 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:55.998 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.373682+0000", 2024-08-17T17:34:55.999 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3087999999999998e-05 2024-08-17T17:34:55.999 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.999 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:55.999 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:55.999 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:55.999 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:55.999 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:271096 10.f e1354/1349 10:f0592efe:::benchmark_data_smithi063_294527_object133911:head v 1354'16943, mlcod=1353'16941)", 2024-08-17T17:34:55.999 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:26.141094+0000", 2024-08-17T17:34:55.999 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 29.733547267999999, 2024-08-17T17:34:55.999 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.238525602, 2024-08-17T17:34:55.999 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:34:55.999 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:34:55.999 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:34:56.000 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:34:56.000 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:56.000 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:34:56.000 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.141094+0000", 2024-08-17T17:34:56.000 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:56.000 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:56.000 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:56.000 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:34:56.000 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.141094+0000", 2024-08-17T17:34:56.000 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:34:56.000 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:56.000 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:56.000 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:34:56.000 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.141094+0000", 2024-08-17T17:34:56.000 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.03e-07 2024-08-17T17:34:56.001 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:56.001 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:56.001 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:34:56.001 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.141099+0000", 2024-08-17T17:34:56.001 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2739999999999998e-06 2024-08-17T17:34:56.001 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:56.001 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:56.001 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:34:56.001 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.367203+0000", 2024-08-17T17:34:56.001 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.22610355300000001 2024-08-17T17:34:56.001 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:56.001 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:56.001 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:34:56.001 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.367215+0000", 2024-08-17T17:34:56.002 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1809e-05 2024-08-17T17:34:56.002 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:56.002 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:56.002 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:34:56.002 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.367347+0000", 2024-08-17T17:34:56.002 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013215500000000001 2024-08-17T17:34:56.002 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:56.002 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:56.002 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:34:56.002 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.367361+0000", 2024-08-17T17:34:56.002 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4404e-05 2024-08-17T17:34:56.002 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:56.002 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:56.002 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:34:56.003 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.379575+0000", 2024-08-17T17:34:56.003 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.012213576 2024-08-17T17:34:56.003 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:34:56.003 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:34:56.003 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:34:56.003 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:26.379620+0000", 2024-08-17T17:34:56.003 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4628e-05 2024-08-17T17:34:56.003 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:56.003 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:56.003 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:56.003 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:34:56.003 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:34:56.003 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:34:56.005 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:56.005 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 7, 0, 1] out_osds: [2] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:34:56.005 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:34:56.006 INFO:tasks.thrashosds.thrasher:Killing osd 7, live_osds are [0, 1, 2, 3, 4, 5, 7] 2024-08-17T17:34:56.006 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-08-17T17:34:56.006 DEBUG:teuthology.orchestra.run.smithi162:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.7 2024-08-17T17:34:56.027 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:56.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:56.028 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:55 smithi162 ceph-mon[104668]: pgmap v2049: 119 pgs: 42 active+clean+remapped, 20 active+undersized, 4 active+undersized+degraded, 53 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:55 smithi063 ceph-mon[109843]: pgmap v2049: 119 pgs: 42 active+clean+remapped, 20 active+undersized, 4 active+undersized+degraded, 53 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:55 smithi063 ceph-mon[114756]: pgmap v2049: 119 pgs: 42 active+clean+remapped, 20 active+undersized, 4 active+undersized+degraded, 53 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:56.129 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:56.151 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:56.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:56.253 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:56.277 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:56.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:56.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:34:56 smithi162 systemd[1]: Stopping Ceph osd.7 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:34:56.379 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:56.401 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:56.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:56.502 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:56.524 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:56.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:56.626 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:56.649 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:56.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:56.751 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:56.774 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:56.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:56.876 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:56.900 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:56.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:56.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:57.001 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:57.024 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:57.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:57.125 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:57.147 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:57.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:57.249 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:57.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.272 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:57.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:57.373 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:57.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.400 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:57.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:57.500 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:57.522 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:57.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:57.543 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:34:57 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[57708]: 2024-08-17T17:34:57.161+0000 7f061b29d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:34:57.543 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:34:57 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[57708]: 2024-08-17T17:34:57.161+0000 7f061b29d640 -1 osd.7 1359 *** Got signal Terminated *** 2024-08-17T17:34:57.543 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:34:57 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[57708]: 2024-08-17T17:34:57.161+0000 7f061b29d640 -1 osd.7 1359 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:34:57.625 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:57.646 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:57.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:57.748 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:57.770 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:57.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:57.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:57 smithi162 ceph-mon[104668]: pgmap v2050: 119 pgs: 42 active+clean+remapped, 20 active+undersized, 4 active+undersized+degraded, 53 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:57.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:57 smithi162 ceph-mon[104668]: osd.7 marked itself down and dead 2024-08-17T17:34:57.849 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:57 smithi063 ceph-mon[114756]: pgmap v2050: 119 pgs: 42 active+clean+remapped, 20 active+undersized, 4 active+undersized+degraded, 53 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:57.850 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:57 smithi063 ceph-mon[114756]: osd.7 marked itself down and dead 2024-08-17T17:34:57.850 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:57 smithi063 ceph-mon[109843]: pgmap v2050: 119 pgs: 42 active+clean+remapped, 20 active+undersized, 4 active+undersized+degraded, 53 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:57.850 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:57 smithi063 ceph-mon[109843]: osd.7 marked itself down and dead 2024-08-17T17:34:57.872 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:57.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:57.896 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:57.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:57.997 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:58.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.020 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:58.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:58.121 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:58.144 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:58.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:58.246 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:58.267 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:58.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:58.369 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:58.392 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:58.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:58.493 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:58.516 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:58.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:58.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:34:58 smithi162 podman[224045]: 2024-08-17 17:34:58.33859996 +0000 UTC m=+1.703447474 container died b8c383163607c3a4d09a69e1fe97db0abcd3bfde527d46dc71cdca4a0179423d (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, ceph=True) 2024-08-17T17:34:58.617 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:58.639 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:58.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:58.681 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:34:58.681 INFO:teuthology.orchestra.run.smithi063.stderr:enabled application 'rados' on pool 'unique_pool_6' 2024-08-17T17:34:58.741 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:34:58.753 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:34:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.765 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:58.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:34:58.866 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:34:58.890 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:58.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:58.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:34:58.932 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[114756]: Health check update: 2 osds down (OSD_DOWN) 2024-08-17T17:34:58.933 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[114756]: osdmap e1360: 8 total, 6 up, 7 in 2024-08-17T17:34:58.933 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3805382876' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:34:58.933 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2108944194' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:34:58.933 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:58.933 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3805382876' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:34:58.933 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2108944194' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:34:58.933 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[114756]: osdmap e1361: 8 total, 6 up, 7 in 2024-08-17T17:34:58.933 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[109843]: Health check update: 2 osds down (OSD_DOWN) 2024-08-17T17:34:58.933 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[109843]: osdmap e1360: 8 total, 6 up, 7 in 2024-08-17T17:34:58.933 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3805382876' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:34:58.934 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2108944194' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:34:58.934 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:58.934 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3805382876' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:34:58.934 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2108944194' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:34:58.934 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:58 smithi063 ceph-mon[109843]: osdmap e1361: 8 total, 6 up, 7 in 2024-08-17T17:34:58.949 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:58 smithi162 ceph-mon[104668]: Health check update: 2 osds down (OSD_DOWN) 2024-08-17T17:34:58.949 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:58 smithi162 ceph-mon[104668]: osdmap e1360: 8 total, 6 up, 7 in 2024-08-17T17:34:58.949 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:58 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3805382876' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-08-17T17:34:58.949 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:58 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2108944194' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:34:58.950 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:58 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:34:58.950 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:58 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3805382876' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_6", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-08-17T17:34:58.950 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:58 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2108944194' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:34:58.950 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:58 smithi162 ceph-mon[104668]: osdmap e1361: 8 total, 6 up, 7 in 2024-08-17T17:34:58.992 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:59.014 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:59.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:59.115 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:59.138 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:59.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:59.240 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:59.263 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:59.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:59.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:34:58 smithi162 podman[224045]: 2024-08-17 17:34:58.948834793 +0000 UTC m=+2.313682300 container cleanup b8c383163607c3a4d09a69e1fe97db0abcd3bfde527d46dc71cdca4a0179423d (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=reef-ced649f, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True) 2024-08-17T17:34:59.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:34:58 smithi162 bash[224045]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7 2024-08-17T17:34:59.365 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:59.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.389 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:59.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:59.491 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:34:59.512 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:34:59.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:34:59.614 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:34:59.637 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:59.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:34:59.739 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:34:59.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.763 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:59.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:34:59.864 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:34:59.869 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:59 smithi063 ceph-mon[114756]: pgmap v2052: 119 pgs: 2 stale+active+undersized, 5 stale+active+clean+remapped, 4 stale+active+clean, 37 active+clean+remapped, 18 active+undersized, 4 active+undersized+degraded, 49 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:59.869 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:59 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:34:59.869 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:34:59 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:34:59.870 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:59 smithi063 ceph-mon[109843]: pgmap v2052: 119 pgs: 2 stale+active+undersized, 5 stale+active+clean+remapped, 4 stale+active+clean, 37 active+clean+remapped, 18 active+undersized, 4 active+undersized+degraded, 49 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:59.870 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:59 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:34:59.870 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:34:59 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:34:59.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:34:59.887 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:34:59.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:34:59.939 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:34:59 smithi162 podman[224332]: 2024-08-17 17:34:59.593662644 +0000 UTC m=+1.250707149 container remove b8c383163607c3a4d09a69e1fe97db0abcd3bfde527d46dc71cdca4a0179423d (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=reef-ced649f) 2024-08-17T17:34:59.940 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:59 smithi162 ceph-mon[104668]: pgmap v2052: 119 pgs: 2 stale+active+undersized, 5 stale+active+clean+remapped, 4 stale+active+clean, 37 active+clean+remapped, 18 active+undersized, 4 active+undersized+degraded, 49 active+clean; 582 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/78 objects degraded (12.821%); 6/78 objects misplaced (7.692%) 2024-08-17T17:34:59.940 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:59 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:34:59.940 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:34:59 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:34:59.988 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:00.010 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:00.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:00.112 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:00.135 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:00.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:00.198 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:34:59 smithi162 podman[224602]: 2024-08-17 17:34:59.841713277 +0000 UTC m=+0.017383784 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:35:00.198 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:34:59 smithi162 podman[224602]: 2024-08-17 17:34:59.96156559 +0000 UTC m=+0.137236126 container create 01e9ed357dfc338b581aa117cf0f2c22314d4e3402c23837e7eafebea40ad530 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD) 2024-08-17T17:35:00.198 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:00 smithi162 podman[224602]: 2024-08-17 17:35:00.197608369 +0000 UTC m=+0.373278878 container init 01e9ed357dfc338b581aa117cf0f2c22314d4e3402c23837e7eafebea40ad530 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0) 2024-08-17T17:35:00.236 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:00.245 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:00.246 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:00.263 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:00.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:00.365 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:00.388 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:00.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:00.489 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:00.505 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:00 smithi162 podman[224602]: 2024-08-17 17:35:00.202351566 +0000 UTC m=+0.378022061 container start 01e9ed357dfc338b581aa117cf0f2c22314d4e3402c23837e7eafebea40ad530 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-08-17T17:35:00.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.514 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:00.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:00.604 DEBUG:teuthology.orchestra.run.smithi063:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_6 bench 90 write' 2024-08-17T17:35:00.607 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-08-17T17:35:00.607 INFO:teuthology.orchestra.run:waiting for 3000 2024-08-17T17:35:00.615 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:00.638 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:00.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:00.659 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:hints = 1 2024-08-17T17:35:00.659 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-08-17T17:35:00.659 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Object prefix: benchmark_data_smithi063_328699 2024-08-17T17:35:00.659 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:35:00.659 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 0 0 0 0 0 0 - 0 2024-08-17T17:35:00.740 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:00.763 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:00.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:00 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 10/81 objects degraded (12.346%), 4 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:00.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:00 smithi162 ceph-mon[104668]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:35:00.846 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:00 smithi162 podman[224602]: 2024-08-17 17:35:00.505642005 +0000 UTC m=+0.681312512 container attach 01e9ed357dfc338b581aa117cf0f2c22314d4e3402c23837e7eafebea40ad530 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, ceph=True) 2024-08-17T17:35:00.846 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:00 smithi162 podman[224602]: 2024-08-17 17:35:00.587489083 +0000 UTC m=+0.763159584 container died 01e9ed357dfc338b581aa117cf0f2c22314d4e3402c23837e7eafebea40ad530 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-08-17T17:35:00.865 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:00.887 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:00.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:00.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:00.989 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:01.011 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:01.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:00 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 10/81 objects degraded (12.346%), 4 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:01.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:00 smithi063 ceph-mon[114756]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:35:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:00 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 10/81 objects degraded (12.346%), 4 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:01.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:00 smithi063 ceph-mon[109843]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-08-17T17:35:01.113 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:01.135 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:01.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:01.237 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:01.259 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:01.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:01.297 DEBUG:teuthology.orchestra.run.smithi162:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.7.service' 2024-08-17T17:35:01.335 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:35:01.335 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-08-17T17:35:01.361 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:01.384 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:01.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.385 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:01.486 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:01.509 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:01.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:01.611 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:01.632 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:01.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:01.659 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 1 16 50 34 2.12474 2.125 0.00125123 0.00209173 2024-08-17T17:35:01.734 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:01.757 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:01.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:01.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:01 smithi063 ceph-mon[109843]: pgmap v2054: 119 pgs: 6 active+undersized+remapped, 1 stale+active+clean+remapped, 3 stale+active+clean, 35 active+clean+remapped, 3 undersized+peered, 20 active+undersized, 4 active+undersized+degraded, 47 active+clean; 742 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/81 objects degraded (12.346%); 8/81 objects misplaced (9.877%) 2024-08-17T17:35:01.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:01.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:01 smithi063 ceph-mon[114756]: pgmap v2054: 119 pgs: 6 active+undersized+remapped, 1 stale+active+clean+remapped, 3 stale+active+clean, 35 active+clean+remapped, 3 undersized+peered, 20 active+undersized, 4 active+undersized+degraded, 47 active+clean; 742 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/81 objects degraded (12.346%); 8/81 objects misplaced (9.877%) 2024-08-17T17:35:01.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:01.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:01.858 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:01.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:01.881 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:01.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:01.982 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:02.004 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:02.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:01 smithi162 ceph-mon[104668]: pgmap v2054: 119 pgs: 6 active+undersized+remapped, 1 stale+active+clean+remapped, 3 stale+active+clean, 35 active+clean+remapped, 3 undersized+peered, 20 active+undersized, 4 active+undersized+degraded, 47 active+clean; 742 KiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 10/81 objects degraded (12.346%); 8/81 objects misplaced (9.877%) 2024-08-17T17:35:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:02.106 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:02.128 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:02.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:02.150 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:35:02.230 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:02.251 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:02.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:02.353 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:02.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.377 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:02.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:02.478 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:02.500 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:02.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:02.602 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:02.624 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:02.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:02.659 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 2 16 50 34 1.06237 0 - 0.00209173 2024-08-17T17:35:02.726 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:02.748 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:02.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:02.850 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:02.872 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:02.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:02.974 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:02.997 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:02.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:02.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:03.100 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:03.123 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:03.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:03.225 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:03.249 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:03.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:03.351 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:03.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.378 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:03.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:03.456 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:35:03.457 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:35:03.457 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 0 2024-08-17T17:35:03.457 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:35:03.479 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:03.503 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:03.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:03.605 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:03.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.632 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:03.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:03.660 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 3 16 50 34 0.708244 0 - 0.00209173 2024-08-17T17:35:03.733 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:03.747 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:35:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:35:03] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.51.0" 2024-08-17T17:35:03.758 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:03.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:03.860 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:03.886 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:03.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:03.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:03.988 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:04.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.013 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:04.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:04.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:03 smithi063 ceph-mon[114756]: pgmap v2055: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 31 active+undersized, 2 undersized+degraded+peered, 14 active+undersized+degraded, 22 active+clean; 2.5 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 2.4 KiB/s rd, 239 KiB/s wr, 5 op/s; 42/168 objects degraded (25.000%); 10/168 objects misplaced (5.952%); 4.9 KiB/s, 0 objects/s recovering 2024-08-17T17:35:04.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:03 smithi063 ceph-mon[109843]: pgmap v2055: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 31 active+undersized, 2 undersized+degraded+peered, 14 active+undersized+degraded, 22 active+clean; 2.5 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 2.4 KiB/s rd, 239 KiB/s wr, 5 op/s; 42/168 objects degraded (25.000%); 10/168 objects misplaced (5.952%); 4.9 KiB/s, 0 objects/s recovering 2024-08-17T17:35:04.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:03 smithi162 ceph-mon[104668]: pgmap v2055: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 31 active+undersized, 2 undersized+degraded+peered, 14 active+undersized+degraded, 22 active+clean; 2.5 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 2.4 KiB/s rd, 239 KiB/s wr, 5 op/s; 42/168 objects degraded (25.000%); 10/168 objects misplaced (5.952%); 4.9 KiB/s, 0 objects/s recovering 2024-08-17T17:35:04.115 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:04.141 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:04.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:04.161 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:35:04.243 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:04.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.267 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:04.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:04.368 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:04.394 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:04.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:04.496 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:04.518 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:04.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:04.620 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:04.642 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:04.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:04.659 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 4 16 50 34 0.531188 0 - 0.00209173 2024-08-17T17:35:04.705 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:04.744 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:04.771 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:04.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:04.873 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:04.895 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:04.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:04.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:04.997 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:05.024 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:05.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:05.125 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:05.152 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:05.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:05.253 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:05.276 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:05.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:05.378 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:05.400 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:05.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:05.503 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:05.529 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:05.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:05.631 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:05.653 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:05.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:05.659 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 5 16 50 34 0.424954 0 - 0.00209173 2024-08-17T17:35:05.755 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:05.780 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:05.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:05.809 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:35:05.809 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:35:05.827 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:35:05.827 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:35:05.844 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:35:05.844 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:35:05.865 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:35:05.866 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:35:05.882 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:05.892 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:35:05.892 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:35:05.908 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:05.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:05.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:05.910 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:35:05.910 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:35:05.910 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:35:05.910 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:35:05.991 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:05 smithi063 ceph-mon[114756]: pgmap v2056: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 29 active+undersized, 2 undersized+degraded+peered, 16 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 2.4 KiB/s rd, 279 KiB/s wr, 5 op/s; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%); 4.9 KiB/s, 0 objects/s recovering 2024-08-17T17:35:05.991 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:05 smithi063 ceph-mon[109843]: pgmap v2056: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 29 active+undersized, 2 undersized+degraded+peered, 16 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 2.4 KiB/s rd, 279 KiB/s wr, 5 op/s; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%); 4.9 KiB/s, 0 objects/s recovering 2024-08-17T17:35:06.010 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:06.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.034 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:06.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:05 smithi162 ceph-mon[104668]: pgmap v2056: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 29 active+undersized, 2 undersized+degraded+peered, 16 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 2.4 KiB/s rd, 279 KiB/s wr, 5 op/s; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%); 4.9 KiB/s, 0 objects/s recovering 2024-08-17T17:35:06.135 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:06.157 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:06.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:06.259 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:06.281 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:06.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:06.336 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 7, 0, 1] out_osds: [2] dead_osds: [6, 7] live_osds: [0, 1, 2, 3, 4, 5] 2024-08-17T17:35:06.336 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:35:06.337 INFO:tasks.thrashosds.thrasher:Adding osd 2 2024-08-17T17:35:06.337 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd in 2 2024-08-17T17:35:06.383 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:06.405 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:06.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:06.507 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:06.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.529 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:06.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:06.630 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:06.632 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:06.654 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:06.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:06.661 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 6 16 50 34 0.35413 0 - 0.00209173 2024-08-17T17:35:06.757 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:06.779 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:06.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:06.881 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:06.906 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:06.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:06.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:07.007 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:07.028 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:07.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:06 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2943295979' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:07.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:06 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2943295979' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:07.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:07.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:06 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2943295979' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:07.131 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:07.154 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:07.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:07.256 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:07.278 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:07.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:07.380 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:07.401 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:07.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:07.503 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:07.526 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:07.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:07.628 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:07.649 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:07.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:07.660 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 7 16 50 34 0.303541 0 - 0.00209173 2024-08-17T17:35:07.751 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:07.774 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:07.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:07.820 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:35:07.876 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:07.899 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:07.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:07.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:08.001 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:08.012 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:07 smithi063 ceph-mon[114756]: pgmap v2057: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 29 active+undersized, 2 undersized+degraded+peered, 16 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 2.1 KiB/s rd, 251 KiB/s wr, 5 op/s; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%); 4.4 KiB/s, 0 objects/s recovering 2024-08-17T17:35:08.012 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:07 smithi063 ceph-mon[109843]: pgmap v2057: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 29 active+undersized, 2 undersized+degraded+peered, 16 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 2.1 KiB/s rd, 251 KiB/s wr, 5 op/s; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%); 4.4 KiB/s, 0 objects/s recovering 2024-08-17T17:35:08.024 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:08.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:08.036 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:35:08.036 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:35:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:07 smithi162 ceph-mon[104668]: pgmap v2057: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 29 active+undersized, 2 undersized+degraded+peered, 16 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 2.1 KiB/s rd, 251 KiB/s wr, 5 op/s; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%); 4.4 KiB/s, 0 objects/s recovering 2024-08-17T17:35:08.126 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:08.147 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:08.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:08.249 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:08.271 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:08.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:08.374 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:08.397 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:08.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:08.499 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:08.521 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:08.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:08.622 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:08.645 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:08.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:08.660 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 8 16 50 34 0.265597 0 - 0.00209173 2024-08-17T17:35:08.747 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:08.771 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:08.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.772 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:08.873 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:08.896 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:08.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:08.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:08.997 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:09.021 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:08 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/763584143' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:09.022 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:08 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:09.022 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:08 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:09.022 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:08 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:09.022 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:08 smithi063 ceph-mon[114756]: osdmap e1362: 8 total, 6 up, 7 in 2024-08-17T17:35:09.022 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:08 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 47/183 objects degraded (25.683%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:09.022 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:08 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/763584143' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:09.022 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:08 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:09.022 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:08 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:09.023 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:08 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:09.023 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:08 smithi063 ceph-mon[109843]: osdmap e1362: 8 total, 6 up, 7 in 2024-08-17T17:35:09.023 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:08 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 47/183 objects degraded (25.683%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:09.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.024 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:09.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:09.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:08 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/763584143' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:08 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:08 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:08 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:08 smithi162 ceph-mon[104668]: osdmap e1362: 8 total, 6 up, 7 in 2024-08-17T17:35:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:08 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 47/183 objects degraded (25.683%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:09.126 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:09.151 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:09.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:09.253 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:09.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.277 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:09.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:09.378 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:09.399 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:09.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:09.501 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:09.527 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:09.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:09.629 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:09.652 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:09.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:09.660 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 9 16 50 34 0.236087 0 - 0.00209173 2024-08-17T17:35:09.754 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:09.777 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:09.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:09.878 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:09.904 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:09.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:09.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:10.006 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:09 smithi063 ceph-mon[114756]: pgmap v2059: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 29 active+undersized, 2 undersized+degraded+peered, 16 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.9 KiB/s rd, 225 KiB/s wr, 4 op/s; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%); 4.0 KiB/s, 0 objects/s recovering 2024-08-17T17:35:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:09 smithi063 ceph-mon[109843]: pgmap v2059: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 29 active+undersized, 2 undersized+degraded+peered, 16 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.9 KiB/s rd, 225 KiB/s wr, 4 op/s; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%); 4.0 KiB/s, 0 objects/s recovering 2024-08-17T17:35:10.032 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:10.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:10.037 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:35:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:09 smithi162 ceph-mon[104668]: pgmap v2059: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 29 active+undersized, 2 undersized+degraded+peered, 16 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.9 KiB/s rd, 225 KiB/s wr, 4 op/s; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%); 4.0 KiB/s, 0 objects/s recovering 2024-08-17T17:35:10.115 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:10.133 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:10.160 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:10.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:10.262 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:10.285 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:10.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:10.387 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:10.412 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:10.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:10.514 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:10.536 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:10.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:10.638 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:10.660 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 10 16 50 34 0.212478 0 - 0.00209173 2024-08-17T17:35:10.661 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:10.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:10.762 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:10.785 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:10.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:10.887 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:10.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:10.911 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:10.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:11.012 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:11.024 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:35:11.035 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:11.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:11.137 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:11.160 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:11.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:11.262 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:11.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.286 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:11.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:11.387 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:11.410 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:11.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:11.512 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:11.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.535 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:11.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:11.636 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:11.658 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:11.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:11.660 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 11 16 50 34 0.193162 0 - 0.00209173 2024-08-17T17:35:11.761 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:11.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.784 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:11.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:11.885 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:11.907 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:11.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:11.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:11.994 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:35:11.994 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:35:12.009 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:12.035 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:12.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:11 smithi162 ceph-mon[104668]: pgmap v2060: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 29 active+undersized, 2 undersized+degraded+peered, 16 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.9 KiB/s rd, 223 KiB/s wr, 4 op/s; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%); 3.9 KiB/s, 0 objects/s recovering 2024-08-17T17:35:12.137 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:12.162 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:12.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:12.264 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:12.272 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:11 smithi063 ceph-mon[114756]: pgmap v2060: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 29 active+undersized, 2 undersized+degraded+peered, 16 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.9 KiB/s rd, 223 KiB/s wr, 4 op/s; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%); 3.9 KiB/s, 0 objects/s recovering 2024-08-17T17:35:12.273 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:11 smithi063 ceph-mon[109843]: pgmap v2060: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 29 active+undersized, 2 undersized+degraded+peered, 16 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 1.9 KiB/s rd, 223 KiB/s wr, 4 op/s; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%); 3.9 KiB/s, 0 objects/s recovering 2024-08-17T17:35:12.289 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:12.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:12.391 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:12.414 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:12.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:12.515 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:12.538 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:12.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:12.641 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:12.660 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 12 16 50 34 0.177065 0 - 0.00209173 2024-08-17T17:35:12.667 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:12.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:12.669 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:35:12.768 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:12.790 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:12.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:12.892 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:12.919 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:12.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:12.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:13.021 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:13.043 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:13.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:13.144 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:13.166 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:13.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:13.269 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:13.291 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:13.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:13.329 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:35:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:35:13] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.51.0" 2024-08-17T17:35:13.361 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:13.393 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:13.415 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:13.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:13.517 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:13.539 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:13.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:13.641 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:13.660 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 13 16 50 34 0.163445 0 - 0.00209173 2024-08-17T17:35:13.664 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:13.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:13.766 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:13.791 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:13.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:13.823 INFO:teuthology.orchestra.run.smithi063.stderr:marked in osd.2. 2024-08-17T17:35:13.844 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:13 smithi063 ceph-mon[114756]: pgmap v2061: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 28 active+undersized, 2 undersized+degraded+peered, 17 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 32 KiB/s wr, 0 op/s; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%) 2024-08-17T17:35:13.844 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:13 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 47/183 objects degraded (25.683%), 19 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:13.844 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:13 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/333627816' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-08-17T17:35:13.845 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a[109820]: 2024-08-17T17:35:13.781+0000 7f66fd20a640 -1 mon.a@0(leader).osd e1362 definitely_dead 0 2024-08-17T17:35:13.845 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:13 smithi063 ceph-mon[109843]: pgmap v2061: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 28 active+undersized, 2 undersized+degraded+peered, 17 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 32 KiB/s wr, 0 op/s; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%) 2024-08-17T17:35:13.845 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:13 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 47/183 objects degraded (25.683%), 19 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:13.845 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:13 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/333627816' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-08-17T17:35:13.893 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:13.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:13.916 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:13.916 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:14.017 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:14.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.040 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:14.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:13 smithi162 ceph-mon[104668]: pgmap v2061: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 28 active+undersized, 2 undersized+degraded+peered, 17 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 32 KiB/s wr, 0 op/s; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%) 2024-08-17T17:35:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:13 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 47/183 objects degraded (25.683%), 19 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:13 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/333627816' entity='client.admin' cmd={"prefix": "osd in", "ids": ["2"]} : dispatch 2024-08-17T17:35:14.141 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:14.167 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:14.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:14.269 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:14.293 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:14.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:14.395 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:14.421 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:14.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:14.523 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:14.548 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:14.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:14.650 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:14.660 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 14 16 50 34 0.151771 0 - 0.00209173 2024-08-17T17:35:14.673 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:14.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:14.774 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:14.800 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:14.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:14.861 INFO:tasks.thrashosds.thrasher:Added osd 2 2024-08-17T17:35:14.902 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:14.926 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:14.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:14.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:14.965 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:14.965 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:15.027 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:15.051 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:15.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:15.084 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:14 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/333627816' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-08-17T17:35:15.084 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:14 smithi063 ceph-mon[114756]: osdmap e1363: 8 total, 6 up, 8 in 2024-08-17T17:35:15.084 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:14 smithi063 ceph-mon[114756]: osdmap e1364: 8 total, 6 up, 8 in 2024-08-17T17:35:15.084 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:14 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/333627816' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-08-17T17:35:15.084 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:14 smithi063 ceph-mon[109843]: osdmap e1363: 8 total, 6 up, 8 in 2024-08-17T17:35:15.084 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:14 smithi063 ceph-mon[109843]: osdmap e1364: 8 total, 6 up, 8 in 2024-08-17T17:35:15.153 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:15.175 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:15.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:15.277 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:15.300 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:15.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:15.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:14 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/333627816' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2024-08-17T17:35:15.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:14 smithi162 ceph-mon[104668]: osdmap e1363: 8 total, 6 up, 8 in 2024-08-17T17:35:15.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:14 smithi162 ceph-mon[104668]: osdmap e1364: 8 total, 6 up, 8 in 2024-08-17T17:35:15.402 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:15.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.426 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:15.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:15.527 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:15.551 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:15.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:15.653 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:15.660 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 15 16 50 34 0.141653 0 - 0.00209173 2024-08-17T17:35:15.677 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:15.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:15.779 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:15.805 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:15.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:15.907 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:15.930 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:15.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:15.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:16.032 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:16.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.055 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:16.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:16.156 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:16.182 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:16.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:16.217 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:15 smithi063 ceph-mon[114756]: pgmap v2063: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 28 active+undersized, 2 undersized+degraded+peered, 17 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%) 2024-08-17T17:35:16.218 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:15 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:16.218 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:15 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:16.218 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:15 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:35:16.218 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:15 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:35:16.218 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:15 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:16.219 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:15 smithi063 ceph-mon[114756]: osdmap e1365: 8 total, 6 up, 8 in 2024-08-17T17:35:16.219 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:15 smithi063 ceph-mon[109843]: pgmap v2063: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 28 active+undersized, 2 undersized+degraded+peered, 17 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%) 2024-08-17T17:35:16.219 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:15 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:16.220 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:15 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:16.220 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:15 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:35:16.220 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:15 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:35:16.220 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:15 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:16.220 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:15 smithi063 ceph-mon[109843]: osdmap e1365: 8 total, 6 up, 8 in 2024-08-17T17:35:16.284 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:16.306 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:16.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:15 smithi162 ceph-mon[104668]: pgmap v2063: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 28 active+undersized, 2 undersized+degraded+peered, 17 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%) 2024-08-17T17:35:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:15 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:15 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:15 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:35:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:15 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:35:16.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:15 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:16.347 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:15 smithi162 ceph-mon[104668]: osdmap e1365: 8 total, 6 up, 8 in 2024-08-17T17:35:16.408 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:16.431 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:16.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:16.532 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:16.554 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:16.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:16.656 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:16.660 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 16 16 67 51 0.199199 0.0708333 0.00122145 0.895292 2024-08-17T17:35:16.680 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:16.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:16.782 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:16.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.805 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:16.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:16.906 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:16.931 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:16.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:16.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:17.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:16 smithi063 ceph-mon[114756]: osdmap e1366: 8 total, 6 up, 8 in 2024-08-17T17:35:17.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:16 smithi063 ceph-mon[109843]: osdmap e1366: 8 total, 6 up, 8 in 2024-08-17T17:35:17.033 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:17.057 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:17.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:17.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:16 smithi162 ceph-mon[104668]: osdmap e1366: 8 total, 6 up, 8 in 2024-08-17T17:35:17.159 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:17.185 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:17.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:17.287 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:17.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.311 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:17.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:17.412 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:17.433 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:17.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:17.535 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:17.558 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:17.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:17.660 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:17.661 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 17 16 67 51 0.187481 0 - 0.895292 2024-08-17T17:35:17.682 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:17.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:17.783 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:17.806 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:17.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:17.908 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:17.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:17.934 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:17.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:17.938 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:17 smithi063 ceph-mon[114756]: pgmap v2066: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 28 active+undersized, 2 undersized+degraded+peered, 17 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%) 2024-08-17T17:35:17.938 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:17 smithi063 ceph-mon[114756]: osdmap e1367: 8 total, 6 up, 8 in 2024-08-17T17:35:17.938 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:17 smithi063 ceph-mon[109843]: pgmap v2066: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 28 active+undersized, 2 undersized+degraded+peered, 17 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%) 2024-08-17T17:35:17.938 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:17 smithi063 ceph-mon[109843]: osdmap e1367: 8 total, 6 up, 8 in 2024-08-17T17:35:18.035 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:18.058 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:18.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:18.160 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:18.183 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:18.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:18.285 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:18.307 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:18.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:18.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:17 smithi162 ceph-mon[104668]: pgmap v2066: 119 pgs: 15 active+undersized+remapped, 25 active+clean+remapped, 10 undersized+peered, 28 active+undersized, 2 undersized+degraded+peered, 17 active+undersized+degraded, 22 active+clean; 2.9 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 47/183 objects degraded (25.683%); 10/183 objects misplaced (5.464%) 2024-08-17T17:35:18.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:17 smithi162 ceph-mon[104668]: osdmap e1367: 8 total, 6 up, 8 in 2024-08-17T17:35:18.409 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:18.431 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:18.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:18.498 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:35:18.499 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:35:18.499 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:35:18.499 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:35:18.499 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.499 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:44821 9.2 9:42c04f20:::benchmark_data_smithi063_263014_object44820:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:35:18.499 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:08.014872+0000", 2024-08-17T17:35:18.499 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 490.481608666, 2024-08-17T17:35:18.499 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.347331799, 2024-08-17T17:35:18.499 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.499 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.500 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.500 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.500 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:35:18.500 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:35:18.500 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 44821 2024-08-17T17:35:18.500 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.500 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.500 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.500 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.500 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.014872+0000", 2024-08-17T17:35:18.500 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.500 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.501 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.501 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.501 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.014872+0000", 2024-08-17T17:35:18.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.501 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.501 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.501 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.501 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.014874+0000", 2024-08-17T17:35:18.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.677e-06 2024-08-17T17:35:18.501 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.501 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.501 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.502 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.014910+0000", 2024-08-17T17:35:18.502 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5738999999999998e-05 2024-08-17T17:35:18.502 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.502 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.502 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.502 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.014929+0000", 2024-08-17T17:35:18.502 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8845999999999999e-05 2024-08-17T17:35:18.502 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.502 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.502 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.502 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.014936+0000", 2024-08-17T17:35:18.502 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1860000000000003e-06 2024-08-17T17:35:18.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.503 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.503 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.503 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.015044+0000", 2024-08-17T17:35:18.503 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000107877 2024-08-17T17:35:18.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.503 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.503 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.503 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.015196+0000", 2024-08-17T17:35:18.503 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015212400000000001 2024-08-17T17:35:18.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.503 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.504 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:35:18.504 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.015363+0000", 2024-08-17T17:35:18.504 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016737300000000001 2024-08-17T17:35:18.504 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.504 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.504 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.504 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019403+0000", 2024-08-17T17:35:18.504 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0040394519999999998 2024-08-17T17:35:18.504 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.504 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.504 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.504 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019531+0000", 2024-08-17T17:35:18.505 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000128269 2024-08-17T17:35:18.505 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.505 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.505 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.505 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.362090+0000", 2024-08-17T17:35:18.505 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.34255893199999998 2024-08-17T17:35:18.505 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.505 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.505 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.505 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.362154+0000", 2024-08-17T17:35:18.505 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4572000000000005e-05 2024-08-17T17:35:18.506 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.506 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.506 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.506 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.362204+0000", 2024-08-17T17:35:18.506 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9752000000000001e-05 2024-08-17T17:35:18.506 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.506 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.506 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.506 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.506 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.506 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:44824 9.2 9:4d91f1c9:::benchmark_data_smithi063_263014_object44823:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:35:18.507 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:08.017287+0000", 2024-08-17T17:35:18.507 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 490.47919394299998, 2024-08-17T17:35:18.507 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.34720898700000002, 2024-08-17T17:35:18.507 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.507 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.507 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.507 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.507 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:35:18.507 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:35:18.507 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 44824 2024-08-17T17:35:18.507 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.507 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.508 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.508 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.508 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017287+0000", 2024-08-17T17:35:18.508 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.508 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.508 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.508 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.508 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017287+0000", 2024-08-17T17:35:18.508 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.508 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.508 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.508 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.508 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017288+0000", 2024-08-17T17:35:18.509 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.1800000000000005e-07 2024-08-17T17:35:18.509 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.509 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.509 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.509 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017307+0000", 2024-08-17T17:35:18.509 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9488e-05 2024-08-17T17:35:18.509 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.509 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.509 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.509 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017319+0000", 2024-08-17T17:35:18.510 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1854e-05 2024-08-17T17:35:18.510 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.510 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.510 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.510 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017327+0000", 2024-08-17T17:35:18.510 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9149999999999996e-06 2024-08-17T17:35:18.510 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.510 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.510 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.511 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017499+0000", 2024-08-17T17:35:18.511 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00017178699999999999 2024-08-17T17:35:18.511 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.511 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.511 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.511 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017626+0000", 2024-08-17T17:35:18.511 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012685100000000001 2024-08-17T17:35:18.511 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.511 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.511 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:35:18.511 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.017750+0000", 2024-08-17T17:35:18.512 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000124218 2024-08-17T17:35:18.512 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.512 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.512 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.512 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.022751+0000", 2024-08-17T17:35:18.512 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.005000984 2024-08-17T17:35:18.512 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.512 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.512 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.512 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.022772+0000", 2024-08-17T17:35:18.512 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0888000000000001e-05 2024-08-17T17:35:18.513 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.513 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.513 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.513 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364384+0000", 2024-08-17T17:35:18.513 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.34161243699999999 2024-08-17T17:35:18.513 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.513 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.513 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.513 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364448+0000", 2024-08-17T17:35:18.513 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3516999999999995e-05 2024-08-17T17:35:18.514 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.514 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.514 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.514 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364496+0000", 2024-08-17T17:35:18.514 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8229999999999997e-05 2024-08-17T17:35:18.514 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.514 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.514 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.514 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.514 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.514 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:44825 9.2 9:495368c0:::benchmark_data_smithi063_263014_object44824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1252)", 2024-08-17T17:35:18.515 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:08.019614+0000", 2024-08-17T17:35:18.515 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 490.47686689199998, 2024-08-17T17:35:18.515 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.34501749799999998, 2024-08-17T17:35:18.515 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.515 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.515 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.516 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.516 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:35:18.516 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:35:18.516 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 44825 2024-08-17T17:35:18.516 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.516 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.517 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.517 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019614+0000", 2024-08-17T17:35:18.517 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.517 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.517 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.517 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019614+0000", 2024-08-17T17:35:18.517 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.517 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.518 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.518 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.518 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019615+0000", 2024-08-17T17:35:18.518 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.24e-06 2024-08-17T17:35:18.518 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.518 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.518 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.518 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019643+0000", 2024-08-17T17:35:18.518 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.7710000000000001e-05 2024-08-17T17:35:18.518 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.519 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.519 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.519 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019653+0000", 2024-08-17T17:35:18.519 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0088999999999999e-05 2024-08-17T17:35:18.519 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.519 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.519 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.519 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019660+0000", 2024-08-17T17:35:18.519 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8469999999999998e-06 2024-08-17T17:35:18.519 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.520 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.520 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.520 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.019845+0000", 2024-08-17T17:35:18.520 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00018503399999999999 2024-08-17T17:35:18.520 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.520 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.520 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.520 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.020016+0000", 2024-08-17T17:35:18.520 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00017090399999999999 2024-08-17T17:35:18.520 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.520 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.521 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:35:18.521 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.020159+0000", 2024-08-17T17:35:18.521 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014291799999999999 2024-08-17T17:35:18.521 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.521 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.521 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.521 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.022872+0000", 2024-08-17T17:35:18.521 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0027137279999999999 2024-08-17T17:35:18.521 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.521 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.521 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.522 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.025059+0000", 2024-08-17T17:35:18.522 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0021862729999999999 2024-08-17T17:35:18.522 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.522 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.522 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.522 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364543+0000", 2024-08-17T17:35:18.522 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.33948450099999999 2024-08-17T17:35:18.522 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.522 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.522 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.522 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364596+0000", 2024-08-17T17:35:18.522 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2871999999999998e-05 2024-08-17T17:35:18.523 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.523 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.523 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.523 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:08.364632+0000", 2024-08-17T17:35:18.523 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5382000000000001e-05 2024-08-17T17:35:18.523 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.523 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.523 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.523 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.523 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.523 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45521.0:102903 9.2 9:4b6cfc07:::benchmark_data_smithi063_263014_object102902:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1258)", 2024-08-17T17:35:18.524 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:27:55.985471+0000", 2024-08-17T17:35:18.524 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 442.51100945000002, 2024-08-17T17:35:18.524 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.36001304699999997, 2024-08-17T17:35:18.524 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.524 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.524 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.524 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.524 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45521", 2024-08-17T17:35:18.524 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/295198673", 2024-08-17T17:35:18.524 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 102903 2024-08-17T17:35:18.525 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.525 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.525 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.525 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.525 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:55.985471+0000", 2024-08-17T17:35:18.525 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.525 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.525 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.525 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.525 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:55.985471+0000", 2024-08-17T17:35:18.525 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.526 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.526 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.526 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.526 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:55.985473+0000", 2024-08-17T17:35:18.526 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.544e-06 2024-08-17T17:35:18.526 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.526 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.526 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.526 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:55.985497+0000", 2024-08-17T17:35:18.526 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3995e-05 2024-08-17T17:35:18.526 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.527 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.527 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.527 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.002738+0000", 2024-08-17T17:35:18.527 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.017241324999999998 2024-08-17T17:35:18.527 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.527 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.527 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.527 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.002748+0000", 2024-08-17T17:35:18.527 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4259999999999992e-06 2024-08-17T17:35:18.527 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.527 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.527 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.528 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.003040+0000", 2024-08-17T17:35:18.528 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00029239799999999997 2024-08-17T17:35:18.528 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.528 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.528 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.528 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.003185+0000", 2024-08-17T17:35:18.528 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000144571 2024-08-17T17:35:18.528 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.528 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.528 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,2", 2024-08-17T17:35:18.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.003341+0000", 2024-08-17T17:35:18.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000156317 2024-08-17T17:35:18.529 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.529 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.009881+0000", 2024-08-17T17:35:18.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0065399220000000001 2024-08-17T17:35:18.529 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.529 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.014340+0000", 2024-08-17T17:35:18.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044590910000000001 2024-08-17T17:35:18.529 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.530 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.530 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.530 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.345341+0000", 2024-08-17T17:35:18.530 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.33100101799999998 2024-08-17T17:35:18.530 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.530 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.530 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.530 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.345400+0000", 2024-08-17T17:35:18.530 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8921000000000001e-05 2024-08-17T17:35:18.530 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.530 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.530 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.531 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:27:56.345485+0000", 2024-08-17T17:35:18.531 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4518999999999995e-05 2024-08-17T17:35:18.531 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.531 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.531 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.531 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.531 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.531 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267379 10.f 10:f6c7ed9c:::benchmark_data_smithi063_294527_object130194:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1348)", 2024-08-17T17:35:18.531 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.680475+0000", 2024-08-17T17:35:18.531 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.816006231000003, 2024-08-17T17:35:18.531 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0741320969999999, 2024-08-17T17:35:18.532 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.532 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.532 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:35:18.532 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.532 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.532 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.532 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267379 2024-08-17T17:35:18.532 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.532 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.532 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.533 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.533 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680475+0000", 2024-08-17T17:35:18.533 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.533 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.533 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.533 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.533 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680475+0000", 2024-08-17T17:35:18.533 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.533 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.533 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.533 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.533 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680476+0000", 2024-08-17T17:35:18.534 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.403e-06 2024-08-17T17:35:18.534 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.534 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.534 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.534 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680481+0000", 2024-08-17T17:35:18.534 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2499999999999997e-06 2024-08-17T17:35:18.534 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.534 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.534 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.534 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680498+0000", 2024-08-17T17:35:18.534 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6568000000000001e-05 2024-08-17T17:35:18.535 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.535 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.535 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.535 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680511+0000", 2024-08-17T17:35:18.535 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3091000000000001e-05 2024-08-17T17:35:18.535 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.535 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.535 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.535 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680583+0000", 2024-08-17T17:35:18.536 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2421000000000003e-05 2024-08-17T17:35:18.536 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.536 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.536 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.536 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680703+0000", 2024-08-17T17:35:18.536 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0001192 2024-08-17T17:35:18.536 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.536 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.536 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.536 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680808+0000", 2024-08-17T17:35:18.536 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010527999999999999 2024-08-17T17:35:18.537 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.537 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.537 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.537 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.685424+0000", 2024-08-17T17:35:18.537 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0046164320000000002 2024-08-17T17:35:18.537 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.537 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.537 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.537 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.687850+0000", 2024-08-17T17:35:18.537 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0024256009999999999 2024-08-17T17:35:18.537 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.538 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.538 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.538 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.693025+0000", 2024-08-17T17:35:18.538 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0051753099999999998 2024-08-17T17:35:18.538 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.538 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.538 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.538 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.693041+0000", 2024-08-17T17:35:18.538 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6231000000000001e-05 2024-08-17T17:35:18.538 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.538 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.539 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.539 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.754514+0000", 2024-08-17T17:35:18.539 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.061472953 2024-08-17T17:35:18.539 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.539 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.539 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.539 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.754607+0000", 2024-08-17T17:35:18.539 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.2356999999999997e-05 2024-08-17T17:35:18.539 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.539 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.539 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.539 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.540 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.540 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267392 10.f 10:f2eb3fe1:::benchmark_data_smithi063_294527_object130207:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:35:18.540 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.696596+0000", 2024-08-17T17:35:18.540 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.799885009999997, 2024-08-17T17:35:18.540 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.067108331, 2024-08-17T17:35:18.540 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.540 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.540 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.540 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.540 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.540 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.540 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267392 2024-08-17T17:35:18.541 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.541 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.541 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.541 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.541 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.696596+0000", 2024-08-17T17:35:18.541 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.541 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.541 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.541 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.541 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.696596+0000", 2024-08-17T17:35:18.541 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.541 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.541 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.542 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.542 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.696598+0000", 2024-08-17T17:35:18.542 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7990000000000001e-06 2024-08-17T17:35:18.542 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.542 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.542 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.542 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.696603+0000", 2024-08-17T17:35:18.542 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4970000000000001e-06 2024-08-17T17:35:18.542 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.542 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.543 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.543 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.696621+0000", 2024-08-17T17:35:18.543 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7563000000000001e-05 2024-08-17T17:35:18.543 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.543 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.543 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.543 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.696628+0000", 2024-08-17T17:35:18.543 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2949999999999996e-06 2024-08-17T17:35:18.543 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.543 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.544 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.544 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.697082+0000", 2024-08-17T17:35:18.544 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00045373100000000002 2024-08-17T17:35:18.544 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.544 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.544 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.544 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.697091+0000", 2024-08-17T17:35:18.544 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4909999999999994e-06 2024-08-17T17:35:18.544 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.544 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.544 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.544 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.755363+0000", 2024-08-17T17:35:18.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0582717079999999 2024-08-17T17:35:18.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.545 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.545 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.755462+0000", 2024-08-17T17:35:18.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9500000000000006e-05 2024-08-17T17:35:18.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.545 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.545 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.755541+0000", 2024-08-17T17:35:18.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8914999999999999e-05 2024-08-17T17:35:18.546 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.546 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.546 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763339+0000", 2024-08-17T17:35:18.546 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0077973590000000002 2024-08-17T17:35:18.546 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.546 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.546 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763479+0000", 2024-08-17T17:35:18.546 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000139961 2024-08-17T17:35:18.546 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.547 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.547 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.547 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763540+0000", 2024-08-17T17:35:18.547 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1356e-05 2024-08-17T17:35:18.547 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.547 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.547 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.547 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763609+0000", 2024-08-17T17:35:18.547 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8900999999999996e-05 2024-08-17T17:35:18.547 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.548 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.548 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.548 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763704+0000", 2024-08-17T17:35:18.548 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.5254999999999995e-05 2024-08-17T17:35:18.548 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.548 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.548 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.548 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.548 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.548 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267452 10.f 10:f299a0d0:::benchmark_data_smithi063_294527_object130267:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:35:18.548 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.745106+0000", 2024-08-17T17:35:18.549 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.751374697999999, 2024-08-17T17:35:18.549 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.018779831, 2024-08-17T17:35:18.549 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.549 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.549 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.549 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.549 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.549 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.549 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267452 2024-08-17T17:35:18.549 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.549 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.549 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.550 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.550 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745106+0000", 2024-08-17T17:35:18.550 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.550 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.550 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.550 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.553 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745106+0000", 2024-08-17T17:35:18.553 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.554 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.554 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.554 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.554 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745108+0000", 2024-08-17T17:35:18.554 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8279999999999999e-06 2024-08-17T17:35:18.554 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.554 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.554 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.554 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745115+0000", 2024-08-17T17:35:18.555 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9330000000000004e-06 2024-08-17T17:35:18.555 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.555 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.555 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.555 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745132+0000", 2024-08-17T17:35:18.555 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6818000000000001e-05 2024-08-17T17:35:18.555 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.555 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.555 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.555 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745141+0000", 2024-08-17T17:35:18.555 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7940000000000008e-06 2024-08-17T17:35:18.556 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.556 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.556 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.556 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745201+0000", 2024-08-17T17:35:18.556 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0409999999999999e-05 2024-08-17T17:35:18.556 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.556 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.556 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.556 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745208+0000", 2024-08-17T17:35:18.556 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0119999999999999e-06 2024-08-17T17:35:18.556 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.557 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.557 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.557 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.759735+0000", 2024-08-17T17:35:18.557 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0145268999999999 2024-08-17T17:35:18.557 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.557 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.557 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.557 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.759830+0000", 2024-08-17T17:35:18.557 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.5042000000000002e-05 2024-08-17T17:35:18.557 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.557 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.558 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.558 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.759902+0000", 2024-08-17T17:35:18.558 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2287999999999996e-05 2024-08-17T17:35:18.558 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.558 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.558 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.558 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763526+0000", 2024-08-17T17:35:18.558 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0036238720000000002 2024-08-17T17:35:18.558 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.558 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.559 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.559 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763719+0000", 2024-08-17T17:35:18.559 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00019284799999999999 2024-08-17T17:35:18.559 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.559 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.559 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.559 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763758+0000", 2024-08-17T17:35:18.559 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9137999999999997e-05 2024-08-17T17:35:18.559 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.559 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763811+0000", 2024-08-17T17:35:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3329999999999999e-05 2024-08-17T17:35:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763886+0000", 2024-08-17T17:35:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4617999999999996e-05 2024-08-17T17:35:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.560 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.561 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.561 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.561 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267466 10.f 10:f92259a0:::benchmark_data_smithi063_294527_object130281:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:35:18.561 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.757244+0000", 2024-08-17T17:35:18.561 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.739237185999997, 2024-08-17T17:35:18.561 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.015260455, 2024-08-17T17:35:18.561 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.561 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.561 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.561 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.561 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.562 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.562 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267466 2024-08-17T17:35:18.562 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.562 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.562 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.562 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.562 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757244+0000", 2024-08-17T17:35:18.562 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.562 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.562 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.562 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.563 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757244+0000", 2024-08-17T17:35:18.563 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.563 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.563 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.563 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.563 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757246+0000", 2024-08-17T17:35:18.563 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.294e-06 2024-08-17T17:35:18.563 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.563 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.563 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.563 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757253+0000", 2024-08-17T17:35:18.564 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8940000000000004e-06 2024-08-17T17:35:18.564 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.564 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.564 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.564 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757270+0000", 2024-08-17T17:35:18.564 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7520000000000002e-05 2024-08-17T17:35:18.564 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.564 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.564 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.564 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757280+0000", 2024-08-17T17:35:18.565 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3670000000000003e-06 2024-08-17T17:35:18.565 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.565 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.565 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.565 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757341+0000", 2024-08-17T17:35:18.565 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1328000000000006e-05 2024-08-17T17:35:18.565 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.565 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.565 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.565 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757352+0000", 2024-08-17T17:35:18.566 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1072e-05 2024-08-17T17:35:18.566 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.566 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.566 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.566 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763943+0000", 2024-08-17T17:35:18.566 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.006590825 2024-08-17T17:35:18.566 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.566 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.566 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.566 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.764061+0000", 2024-08-17T17:35:18.567 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000117793 2024-08-17T17:35:18.567 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.567 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.567 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.567 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.764180+0000", 2024-08-17T17:35:18.567 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000118709 2024-08-17T17:35:18.567 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.567 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.567 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.567 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772142+0000", 2024-08-17T17:35:18.568 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0079620249999999993 2024-08-17T17:35:18.568 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.568 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.568 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.568 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772235+0000", 2024-08-17T17:35:18.568 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3037999999999999e-05 2024-08-17T17:35:18.568 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.568 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.568 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.568 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772339+0000", 2024-08-17T17:35:18.568 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010490299999999999 2024-08-17T17:35:18.569 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.569 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.569 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.569 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772402+0000", 2024-08-17T17:35:18.569 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2700000000000006e-05 2024-08-17T17:35:18.569 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.569 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.569 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.569 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772504+0000", 2024-08-17T17:35:18.569 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000101987 2024-08-17T17:35:18.569 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.570 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.570 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.570 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.570 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.570 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267538 10.f 10:f588c7f4:::benchmark_data_smithi063_294527_object130353:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:35:18.570 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.818626+0000", 2024-08-17T17:35:18.570 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.67785482, 2024-08-17T17:35:18.570 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95837228299999999, 2024-08-17T17:35:18.570 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.570 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.571 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.571 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.571 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.571 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.571 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267538 2024-08-17T17:35:18.571 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.571 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.571 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.571 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.571 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818626+0000", 2024-08-17T17:35:18.571 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.571 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.572 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.572 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.572 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818626+0000", 2024-08-17T17:35:18.572 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:18.574 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.574 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.574 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.574 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.574 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818628+0000", 2024-08-17T17:35:18.575 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9740000000000001e-06 2024-08-17T17:35:18.575 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.575 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.575 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.575 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818636+0000", 2024-08-17T17:35:18.575 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8949999999999998e-06 2024-08-17T17:35:18.575 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.575 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.575 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.575 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818657+0000", 2024-08-17T17:35:18.575 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1498e-05 2024-08-17T17:35:18.576 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.576 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.576 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.576 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818667+0000", 2024-08-17T17:35:18.576 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.6770000000000007e-06 2024-08-17T17:35:18.576 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.576 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.576 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.576 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818742+0000", 2024-08-17T17:35:18.576 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4572000000000004e-05 2024-08-17T17:35:18.576 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.577 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.577 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.577 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818751+0000", 2024-08-17T17:35:18.577 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7510000000000003e-06 2024-08-17T17:35:18.577 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.577 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.577 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.577 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.767638+0000", 2024-08-17T17:35:18.577 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.94888622300000003 2024-08-17T17:35:18.577 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.577 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.578 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.578 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.767770+0000", 2024-08-17T17:35:18.578 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013257300000000001 2024-08-17T17:35:18.578 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.578 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.578 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.578 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.767883+0000", 2024-08-17T17:35:18.578 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000112569 2024-08-17T17:35:18.578 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.578 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.578 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.579 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772322+0000", 2024-08-17T17:35:18.579 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044396710000000001 2024-08-17T17:35:18.579 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.579 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.579 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.579 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.776814+0000", 2024-08-17T17:35:18.579 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044919829999999997 2024-08-17T17:35:18.579 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.579 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.579 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.579 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.776837+0000", 2024-08-17T17:35:18.580 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2761000000000001e-05 2024-08-17T17:35:18.580 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.580 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.580 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.580 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.776898+0000", 2024-08-17T17:35:18.580 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0705999999999998e-05 2024-08-17T17:35:18.580 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.580 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.580 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.581 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.776998+0000", 2024-08-17T17:35:18.581 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010043 2024-08-17T17:35:18.581 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.581 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.581 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.581 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.581 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.581 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267541 10.f 10:fa0f1a18:::benchmark_data_smithi063_294527_object130356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:35:18.581 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.820949+0000", 2024-08-17T17:35:18.581 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.675532304000001, 2024-08-17T17:35:18.582 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.96002311799999995, 2024-08-17T17:35:18.582 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.582 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.582 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.582 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.582 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.582 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.582 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267541 2024-08-17T17:35:18.582 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.582 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.583 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.583 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.583 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.820949+0000", 2024-08-17T17:35:18.583 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.583 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.583 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.583 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.583 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.820949+0000", 2024-08-17T17:35:18.583 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.583 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.583 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.583 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.584 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.820950+0000", 2024-08-17T17:35:18.584 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.31e-06 2024-08-17T17:35:18.584 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.584 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.584 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.584 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.820955+0000", 2024-08-17T17:35:18.584 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9840000000000001e-06 2024-08-17T17:35:18.584 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.584 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.584 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.584 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.820969+0000", 2024-08-17T17:35:18.585 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3891e-05 2024-08-17T17:35:18.585 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.585 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.585 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.585 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.820976+0000", 2024-08-17T17:35:18.585 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6899999999999992e-06 2024-08-17T17:35:18.585 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.585 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.585 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.585 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.821034+0000", 2024-08-17T17:35:18.585 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7296999999999997e-05 2024-08-17T17:35:18.586 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.586 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.586 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.586 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.821042+0000", 2024-08-17T17:35:18.586 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8900000000000007e-06 2024-08-17T17:35:18.586 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.586 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.586 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.586 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772561+0000", 2024-08-17T17:35:18.586 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95151887499999999 2024-08-17T17:35:18.587 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.587 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.587 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.587 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772658+0000", 2024-08-17T17:35:18.587 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7894999999999997e-05 2024-08-17T17:35:18.587 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.587 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.587 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.587 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772760+0000", 2024-08-17T17:35:18.587 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010155200000000001 2024-08-17T17:35:18.587 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.588 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.588 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.588 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777043+0000", 2024-08-17T17:35:18.588 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0042826269999999998 2024-08-17T17:35:18.588 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.588 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.588 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.588 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777055+0000", 2024-08-17T17:35:18.588 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2758000000000001e-05 2024-08-17T17:35:18.588 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.589 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.589 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.589 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.780809+0000", 2024-08-17T17:35:18.589 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0037537180000000001 2024-08-17T17:35:18.589 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.589 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.589 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.589 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.780855+0000", 2024-08-17T17:35:18.589 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6026000000000003e-05 2024-08-17T17:35:18.589 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.589 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.590 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.590 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.780972+0000", 2024-08-17T17:35:18.590 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000116605 2024-08-17T17:35:18.590 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.590 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.590 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.590 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.590 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.590 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267557 10.f 10:f1c06cfb:::benchmark_data_smithi063_294527_object130372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:35:18.590 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.835168+0000", 2024-08-17T17:35:18.590 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.661312531, 2024-08-17T17:35:18.591 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.954258312, 2024-08-17T17:35:18.591 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.591 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.591 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.591 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.591 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267557 2024-08-17T17:35:18.591 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.591 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.591 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.591 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.592 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835168+0000", 2024-08-17T17:35:18.592 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.592 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.592 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.592 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.592 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835168+0000", 2024-08-17T17:35:18.592 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.592 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.592 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.592 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.593 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835170+0000", 2024-08-17T17:35:18.593 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.669e-06 2024-08-17T17:35:18.593 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.593 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.593 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.593 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835176+0000", 2024-08-17T17:35:18.593 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3620000000000002e-06 2024-08-17T17:35:18.593 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.593 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.593 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.593 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835193+0000", 2024-08-17T17:35:18.593 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6181e-05 2024-08-17T17:35:18.593 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.594 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.594 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.594 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835200+0000", 2024-08-17T17:35:18.594 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5279999999999998e-06 2024-08-17T17:35:18.594 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.594 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.594 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.594 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835276+0000", 2024-08-17T17:35:18.594 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6150999999999996e-05 2024-08-17T17:35:18.594 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.595 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.595 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.595 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835284+0000", 2024-08-17T17:35:18.595 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.038e-06 2024-08-17T17:35:18.595 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.595 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.595 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.595 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777083+0000", 2024-08-17T17:35:18.595 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.94179912300000002 2024-08-17T17:35:18.595 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.595 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.595 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.596 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777200+0000", 2024-08-17T17:35:18.596 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011640400000000001 2024-08-17T17:35:18.596 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.596 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.596 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.596 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777266+0000", 2024-08-17T17:35:18.596 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6346000000000005e-05 2024-08-17T17:35:18.596 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.596 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.596 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.597 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.781026+0000", 2024-08-17T17:35:18.597 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0037601959999999999 2024-08-17T17:35:18.597 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.597 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.597 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.597 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789265+0000", 2024-08-17T17:35:18.597 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0082385630000000008 2024-08-17T17:35:18.597 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.597 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.597 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.597 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789306+0000", 2024-08-17T17:35:18.598 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.0571999999999997e-05 2024-08-17T17:35:18.598 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.598 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.598 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.598 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789352+0000", 2024-08-17T17:35:18.598 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6347999999999997e-05 2024-08-17T17:35:18.598 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.598 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.598 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.598 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789427+0000", 2024-08-17T17:35:18.598 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4831000000000002e-05 2024-08-17T17:35:18.599 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.599 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.599 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.599 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.599 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.599 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267575 10.f 10:fab61f3b:::benchmark_data_smithi063_294527_object130390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:35:18.599 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.846798+0000", 2024-08-17T17:35:18.599 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.649682609000003, 2024-08-17T17:35:18.599 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.942820518, 2024-08-17T17:35:18.599 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.599 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.599 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.600 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.600 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.600 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.600 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267575 2024-08-17T17:35:18.600 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.600 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.600 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.600 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.600 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846798+0000", 2024-08-17T17:35:18.600 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.600 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.600 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.600 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.601 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846798+0000", 2024-08-17T17:35:18.601 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.601 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.601 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.601 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.601 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846800+0000", 2024-08-17T17:35:18.601 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.86e-06 2024-08-17T17:35:18.601 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.601 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.601 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.601 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846807+0000", 2024-08-17T17:35:18.601 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6089999999999997e-06 2024-08-17T17:35:18.602 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.602 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.602 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.602 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846828+0000", 2024-08-17T17:35:18.602 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0812e-05 2024-08-17T17:35:18.602 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.602 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.602 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.602 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846839+0000", 2024-08-17T17:35:18.602 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0969e-05 2024-08-17T17:35:18.603 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.603 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.603 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.603 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846893+0000", 2024-08-17T17:35:18.603 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4716999999999998e-05 2024-08-17T17:35:18.603 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.603 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.603 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.603 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846900+0000", 2024-08-17T17:35:18.603 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8499999999999996e-06 2024-08-17T17:35:18.603 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.603 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.603 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.604 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.781090+0000", 2024-08-17T17:35:18.604 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.93418942699999996 2024-08-17T17:35:18.604 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.604 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.604 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.604 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.781222+0000", 2024-08-17T17:35:18.604 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013291099999999999 2024-08-17T17:35:18.604 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.604 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.604 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.604 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.781363+0000", 2024-08-17T17:35:18.604 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014084199999999999 2024-08-17T17:35:18.605 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.605 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.605 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.605 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789243+0000", 2024-08-17T17:35:18.605 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0078801830000000007 2024-08-17T17:35:18.605 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.605 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.605 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.605 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789497+0000", 2024-08-17T17:35:18.605 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000253687 2024-08-17T17:35:18.605 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.606 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.606 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.606 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789534+0000", 2024-08-17T17:35:18.606 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6947999999999999e-05 2024-08-17T17:35:18.606 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.606 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.606 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.606 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789563+0000", 2024-08-17T17:35:18.606 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8702999999999999e-05 2024-08-17T17:35:18.606 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.606 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.607 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.607 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789619+0000", 2024-08-17T17:35:18.607 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5999999999999999e-05 2024-08-17T17:35:18.607 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.607 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.607 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.607 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.607 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.607 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267580 10.f 10:ff03e0ba:::benchmark_data_smithi063_294527_object130395:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:35:18.607 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.852353+0000", 2024-08-17T17:35:18.607 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.644127443000002, 2024-08-17T17:35:18.608 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.93742878200000002, 2024-08-17T17:35:18.608 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.608 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.608 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.608 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.608 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.608 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.608 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267580 2024-08-17T17:35:18.608 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.608 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.608 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.609 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.609 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852353+0000", 2024-08-17T17:35:18.609 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.609 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.609 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.609 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.609 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852353+0000", 2024-08-17T17:35:18.609 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.609 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.609 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.609 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.610 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852355+0000", 2024-08-17T17:35:18.610 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4980000000000001e-06 2024-08-17T17:35:18.610 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.610 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.610 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.610 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852360+0000", 2024-08-17T17:35:18.610 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4310000000000002e-06 2024-08-17T17:35:18.610 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.610 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.610 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.610 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852376+0000", 2024-08-17T17:35:18.611 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5373999999999999e-05 2024-08-17T17:35:18.611 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.611 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.611 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.611 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852384+0000", 2024-08-17T17:35:18.611 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8390000000000007e-06 2024-08-17T17:35:18.611 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.611 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.611 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.611 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852452+0000", 2024-08-17T17:35:18.611 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8458000000000001e-05 2024-08-17T17:35:18.612 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.612 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.612 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.612 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852463+0000", 2024-08-17T17:35:18.612 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0448999999999999e-05 2024-08-17T17:35:18.612 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.612 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.612 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.612 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.785113+0000", 2024-08-17T17:35:18.612 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.93265094699999995 2024-08-17T17:35:18.612 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.612 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.613 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.613 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.785224+0000", 2024-08-17T17:35:18.613 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011068799999999999 2024-08-17T17:35:18.613 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.613 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.613 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.613 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.785345+0000", 2024-08-17T17:35:18.613 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000121298 2024-08-17T17:35:18.613 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.613 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.613 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.614 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789481+0000", 2024-08-17T17:35:18.614 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0041354390000000003 2024-08-17T17:35:18.614 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.614 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.614 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.614 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789670+0000", 2024-08-17T17:35:18.614 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00018873300000000001 2024-08-17T17:35:18.614 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.614 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.614 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.614 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789678+0000", 2024-08-17T17:35:18.615 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9289999999999998e-06 2024-08-17T17:35:18.615 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.615 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.615 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.615 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789705+0000", 2024-08-17T17:35:18.615 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.7120000000000001e-05 2024-08-17T17:35:18.615 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.615 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.615 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.615 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789782+0000", 2024-08-17T17:35:18.615 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7578999999999997e-05 2024-08-17T17:35:18.616 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.616 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.616 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.616 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.616 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.616 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267583 10.f 10:f7f0ea77:::benchmark_data_smithi063_294527_object130398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:35:18.616 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.853732+0000", 2024-08-17T17:35:18.616 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.642748935999997, 2024-08-17T17:35:18.616 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95321229200000002, 2024-08-17T17:35:18.616 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.617 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.617 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.617 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.617 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.617 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.617 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267583 2024-08-17T17:35:18.617 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.617 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.617 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.617 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.617 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853732+0000", 2024-08-17T17:35:18.618 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.618 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.618 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.618 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.618 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853732+0000", 2024-08-17T17:35:18.618 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.618 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.618 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.618 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.618 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853733+0000", 2024-08-17T17:35:18.618 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.046e-06 2024-08-17T17:35:18.619 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.619 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.619 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.619 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853738+0000", 2024-08-17T17:35:18.619 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6290000000000001e-06 2024-08-17T17:35:18.619 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.619 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.619 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.619 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853754+0000", 2024-08-17T17:35:18.619 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5928000000000001e-05 2024-08-17T17:35:18.619 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.620 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.620 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.620 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853761+0000", 2024-08-17T17:35:18.620 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1890000000000001e-06 2024-08-17T17:35:18.620 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.620 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.620 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.620 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853820+0000", 2024-08-17T17:35:18.620 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9242e-05 2024-08-17T17:35:18.620 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.620 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.621 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.621 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853828+0000", 2024-08-17T17:35:18.621 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4579999999999993e-06 2024-08-17T17:35:18.621 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.621 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.621 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.621 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789813+0000", 2024-08-17T17:35:18.621 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.93598451400000005 2024-08-17T17:35:18.621 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.621 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.621 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.622 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789927+0000", 2024-08-17T17:35:18.622 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000114324 2024-08-17T17:35:18.622 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.622 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.622 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.622 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.790017+0000", 2024-08-17T17:35:18.622 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0160000000000004e-05 2024-08-17T17:35:18.622 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.622 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.622 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.622 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.798343+0000", 2024-08-17T17:35:18.622 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0083259660000000006 2024-08-17T17:35:18.623 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.623 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.623 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.623 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.798372+0000", 2024-08-17T17:35:18.623 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8609e-05 2024-08-17T17:35:18.623 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.623 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.623 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.623 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.806451+0000", 2024-08-17T17:35:18.623 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0080793810000000001 2024-08-17T17:35:18.623 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.624 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.624 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.624 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.806698+0000", 2024-08-17T17:35:18.624 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00024614800000000002 2024-08-17T17:35:18.624 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.624 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.624 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.624 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.806944+0000", 2024-08-17T17:35:18.624 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000246698 2024-08-17T17:35:18.624 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.624 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.624 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.625 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.625 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.625 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267588 10.f 10:ff638821:::benchmark_data_smithi063_294527_object130403:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:35:18.625 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.857910+0000", 2024-08-17T17:35:18.625 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.638570614000002, 2024-08-17T17:35:18.625 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95390986700000002, 2024-08-17T17:35:18.625 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.625 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.625 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.625 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.626 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.626 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.626 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267588 2024-08-17T17:35:18.626 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.626 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.626 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.626 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.626 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.857910+0000", 2024-08-17T17:35:18.626 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.626 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.626 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.627 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.627 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.857910+0000", 2024-08-17T17:35:18.627 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.627 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.627 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.627 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.627 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.857912+0000", 2024-08-17T17:35:18.627 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.584e-06 2024-08-17T17:35:18.627 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.627 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.628 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.628 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.857916+0000", 2024-08-17T17:35:18.628 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5229999999999997e-06 2024-08-17T17:35:18.628 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.628 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.628 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.628 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.857935+0000", 2024-08-17T17:35:18.628 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8380999999999999e-05 2024-08-17T17:35:18.628 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.628 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.628 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.629 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.857943+0000", 2024-08-17T17:35:18.629 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.1009999999999992e-06 2024-08-17T17:35:18.629 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.629 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.629 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.629 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.858006+0000", 2024-08-17T17:35:18.629 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2819999999999998e-05 2024-08-17T17:35:18.629 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.629 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.629 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.629 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.858015+0000", 2024-08-17T17:35:18.629 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.2390000000000008e-06 2024-08-17T17:35:18.630 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.630 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.630 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.630 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.793977+0000", 2024-08-17T17:35:18.630 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.93596159899999998 2024-08-17T17:35:18.630 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.630 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.630 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.630 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.794084+0000", 2024-08-17T17:35:18.630 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010793099999999999 2024-08-17T17:35:18.630 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.631 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.631 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.631 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.794194+0000", 2024-08-17T17:35:18.631 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000109067 2024-08-17T17:35:18.631 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.631 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.631 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.631 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.798432+0000", 2024-08-17T17:35:18.631 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0042388269999999997 2024-08-17T17:35:18.631 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.631 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.632 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.632 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.806955+0000", 2024-08-17T17:35:18.632 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0085225140000000001 2024-08-17T17:35:18.632 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.632 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.632 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.632 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811658+0000", 2024-08-17T17:35:18.632 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0047027170000000004 2024-08-17T17:35:18.632 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.632 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.632 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.633 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811719+0000", 2024-08-17T17:35:18.633 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1008999999999998e-05 2024-08-17T17:35:18.633 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.633 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.633 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.633 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811820+0000", 2024-08-17T17:35:18.633 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000101555 2024-08-17T17:35:18.633 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.633 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.633 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.633 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.634 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.634 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267591 10.f 10:f43063b1:::benchmark_data_smithi063_294527_object130406:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:35:18.634 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.861930+0000", 2024-08-17T17:35:18.634 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.634550865999998, 2024-08-17T17:35:18.634 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95045635799999995, 2024-08-17T17:35:18.634 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.634 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.634 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.635 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.635 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.635 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.635 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267591 2024-08-17T17:35:18.635 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.635 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.635 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.635 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.635 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.861930+0000", 2024-08-17T17:35:18.635 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.635 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.636 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.636 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.636 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.861930+0000", 2024-08-17T17:35:18.636 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.636 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.636 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.636 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.636 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.861932+0000", 2024-08-17T17:35:18.636 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7179999999999999e-06 2024-08-17T17:35:18.637 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.637 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.637 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.637 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.861938+0000", 2024-08-17T17:35:18.637 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4620000000000001e-06 2024-08-17T17:35:18.637 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.637 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.637 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.637 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.861957+0000", 2024-08-17T17:35:18.638 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9267999999999999e-05 2024-08-17T17:35:18.638 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.638 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.638 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.638 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.861965+0000", 2024-08-17T17:35:18.638 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8320000000000006e-06 2024-08-17T17:35:18.638 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.638 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.638 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.638 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.862025+0000", 2024-08-17T17:35:18.638 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9435000000000003e-05 2024-08-17T17:35:18.639 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.639 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.639 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.639 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.862031+0000", 2024-08-17T17:35:18.639 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3609999999999997e-06 2024-08-17T17:35:18.639 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.639 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.639 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.639 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.798461+0000", 2024-08-17T17:35:18.639 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.93642974599999995 2024-08-17T17:35:18.640 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.640 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.640 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.640 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.798565+0000", 2024-08-17T17:35:18.640 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000104264 2024-08-17T17:35:18.640 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.640 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.640 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.640 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.798688+0000", 2024-08-17T17:35:18.640 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012238099999999999 2024-08-17T17:35:18.640 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.640 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.641 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.641 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811574+0000", 2024-08-17T17:35:18.641 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.012886806000000001 2024-08-17T17:35:18.641 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.641 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.641 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.641 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811601+0000", 2024-08-17T17:35:18.641 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6605e-05 2024-08-17T17:35:18.641 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.641 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.641 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.642 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811960+0000", 2024-08-17T17:35:18.642 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000359086 2024-08-17T17:35:18.642 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.642 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.642 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.642 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.812014+0000", 2024-08-17T17:35:18.642 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4048999999999997e-05 2024-08-17T17:35:18.642 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.642 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.642 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.642 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.812386+0000", 2024-08-17T17:35:18.643 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000372345 2024-08-17T17:35:18.643 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.643 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.643 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.643 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.643 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.643 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267604 10.f 10:fae658d4:::benchmark_data_smithi063_294527_object130419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:35:18.643 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.872469+0000", 2024-08-17T17:35:18.643 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.624011420000002, 2024-08-17T17:35:18.643 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.94023317100000003, 2024-08-17T17:35:18.643 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.644 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.644 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.644 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.644 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.644 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.644 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267604 2024-08-17T17:35:18.644 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.644 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.644 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.644 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.645 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872469+0000", 2024-08-17T17:35:18.645 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.645 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.645 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.645 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.645 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872469+0000", 2024-08-17T17:35:18.645 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.645 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.645 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.645 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.645 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872471+0000", 2024-08-17T17:35:18.646 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.55e-06 2024-08-17T17:35:18.646 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.646 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.646 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.646 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872476+0000", 2024-08-17T17:35:18.646 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1449999999999999e-06 2024-08-17T17:35:18.646 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.646 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.646 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.646 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872490+0000", 2024-08-17T17:35:18.647 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3725e-05 2024-08-17T17:35:18.647 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.647 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.647 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.647 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872499+0000", 2024-08-17T17:35:18.647 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3349999999999996e-06 2024-08-17T17:35:18.647 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.647 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.647 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.647 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872554+0000", 2024-08-17T17:35:18.647 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4503000000000003e-05 2024-08-17T17:35:18.648 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.648 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.648 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.648 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872563+0000", 2024-08-17T17:35:18.648 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4439999999999997e-06 2024-08-17T17:35:18.648 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.648 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.648 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.648 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.802449+0000", 2024-08-17T17:35:18.648 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.92988571900000005 2024-08-17T17:35:18.648 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.649 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.649 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.649 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.802561+0000", 2024-08-17T17:35:18.649 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000112218 2024-08-17T17:35:18.649 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.649 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.649 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.649 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.802673+0000", 2024-08-17T17:35:18.649 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011205900000000001 2024-08-17T17:35:18.649 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.649 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.650 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.650 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811607+0000", 2024-08-17T17:35:18.650 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0089336009999999993 2024-08-17T17:35:18.650 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.650 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.650 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.650 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811879+0000", 2024-08-17T17:35:18.650 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00027259400000000003 2024-08-17T17:35:18.650 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.651 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.651 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.651 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.812523+0000", 2024-08-17T17:35:18.651 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00064338100000000005 2024-08-17T17:35:18.651 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.651 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.651 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.651 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.812589+0000", 2024-08-17T17:35:18.651 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6017000000000002e-05 2024-08-17T17:35:18.652 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.652 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.652 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.652 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.812703+0000", 2024-08-17T17:35:18.652 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011388 2024-08-17T17:35:18.652 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.652 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.652 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.652 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.653 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.653 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267639 10.f 10:fc605646:::benchmark_data_smithi063_294527_object130454:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:35:18.653 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.921459+0000", 2024-08-17T17:35:18.653 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.575021808000002, 2024-08-17T17:35:18.653 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.90069574100000005, 2024-08-17T17:35:18.653 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.653 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.653 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.653 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.653 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.653 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.653 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267639 2024-08-17T17:35:18.654 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.654 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.654 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.654 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.654 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921459+0000", 2024-08-17T17:35:18.654 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.654 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.654 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.654 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.654 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921459+0000", 2024-08-17T17:35:18.654 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.655 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.655 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.655 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.655 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921461+0000", 2024-08-17T17:35:18.655 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3489999999999999e-06 2024-08-17T17:35:18.655 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.655 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.655 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.655 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921471+0000", 2024-08-17T17:35:18.655 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3640000000000005e-06 2024-08-17T17:35:18.655 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.656 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.656 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.656 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921492+0000", 2024-08-17T17:35:18.656 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1596000000000001e-05 2024-08-17T17:35:18.656 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.656 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.656 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.656 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921504+0000", 2024-08-17T17:35:18.656 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1153e-05 2024-08-17T17:35:18.656 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.656 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.657 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.657 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921571+0000", 2024-08-17T17:35:18.657 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7308e-05 2024-08-17T17:35:18.657 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.657 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.657 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.657 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921581+0000", 2024-08-17T17:35:18.657 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0497e-05 2024-08-17T17:35:18.657 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.657 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.657 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.657 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.807005+0000", 2024-08-17T17:35:18.658 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.88542341400000002 2024-08-17T17:35:18.658 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.658 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.658 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.658 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.807122+0000", 2024-08-17T17:35:18.658 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011692300000000001 2024-08-17T17:35:18.658 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.658 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.658 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.658 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.807232+0000", 2024-08-17T17:35:18.658 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000110369 2024-08-17T17:35:18.659 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.659 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.659 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.659 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811893+0000", 2024-08-17T17:35:18.659 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00466103 2024-08-17T17:35:18.659 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.659 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.659 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.659 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.812464+0000", 2024-08-17T17:35:18.659 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00057084599999999996 2024-08-17T17:35:18.659 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.660 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.660 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.660 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.821975+0000", 2024-08-17T17:35:18.660 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0095109449999999998 2024-08-17T17:35:18.660 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.660 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.660 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.660 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.822046+0000", 2024-08-17T17:35:18.660 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1000999999999993e-05 2024-08-17T17:35:18.660 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.660 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.660 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.661 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.822155+0000", 2024-08-17T17:35:18.661 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000108946 2024-08-17T17:35:18.661 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.661 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.661 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.661 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.661 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.661 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267644 10.f 10:fe3b370b:::benchmark_data_smithi063_294527_object130459:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:35:18.661 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.930279+0000", 2024-08-17T17:35:18.661 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.566202351999998, 2024-08-17T17:35:18.662 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.89220029700000003, 2024-08-17T17:35:18.662 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.662 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.662 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.662 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.662 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.662 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.662 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267644 2024-08-17T17:35:18.662 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.662 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.662 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.663 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.663 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.930279+0000", 2024-08-17T17:35:18.663 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.663 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.663 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.663 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.663 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.930279+0000", 2024-08-17T17:35:18.663 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.663 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.663 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.663 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.664 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.930280+0000", 2024-08-17T17:35:18.664 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6249999999999999e-06 2024-08-17T17:35:18.664 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.664 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.664 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.664 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.930285+0000", 2024-08-17T17:35:18.664 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6240000000000001e-06 2024-08-17T17:35:18.664 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.664 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.664 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.664 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.930298+0000", 2024-08-17T17:35:18.664 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2846e-05 2024-08-17T17:35:18.665 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.665 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.665 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.665 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.930306+0000", 2024-08-17T17:35:18.665 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0499999999999992e-06 2024-08-17T17:35:18.665 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.665 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.665 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.665 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.930368+0000", 2024-08-17T17:35:18.665 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2720999999999998e-05 2024-08-17T17:35:18.665 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.666 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.666 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.666 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.930376+0000", 2024-08-17T17:35:18.666 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9419999999999995e-06 2024-08-17T17:35:18.666 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.666 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.666 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.666 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.812756+0000", 2024-08-17T17:35:18.666 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.88237979 2024-08-17T17:35:18.666 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.666 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.667 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.667 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.812867+0000", 2024-08-17T17:35:18.667 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000110584 2024-08-17T17:35:18.667 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.667 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.667 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.667 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.812994+0000", 2024-08-17T17:35:18.667 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012696100000000001 2024-08-17T17:35:18.667 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.667 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.667 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.668 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.822168+0000", 2024-08-17T17:35:18.668 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0091745439999999998 2024-08-17T17:35:18.668 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.668 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.668 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.668 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.822235+0000", 2024-08-17T17:35:18.668 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6322000000000006e-05 2024-08-17T17:35:18.668 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.668 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.668 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.668 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.822302+0000", 2024-08-17T17:35:18.668 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7779000000000003e-05 2024-08-17T17:35:18.669 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.669 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.669 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.669 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.822365+0000", 2024-08-17T17:35:18.669 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2347000000000005e-05 2024-08-17T17:35:18.669 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.669 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.669 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.669 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.822479+0000", 2024-08-17T17:35:18.669 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000114162 2024-08-17T17:35:18.669 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.670 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.670 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.670 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.670 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.670 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267651 10.f 10:fcd2ef07:::benchmark_data_smithi063_294527_object130466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:35:18.670 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.951716+0000", 2024-08-17T17:35:18.670 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 57.544764772999997, 2024-08-17T17:35:18.670 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.87099803399999998, 2024-08-17T17:35:18.670 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:35:18.670 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:35:18.671 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:18.671 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:35:18.671 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:35:18.671 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:18.671 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267651 2024-08-17T17:35:18.671 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.671 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:35:18.671 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.671 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:35:18.671 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.951716+0000", 2024-08-17T17:35:18.672 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.672 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.672 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.672 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:35:18.672 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.951716+0000", 2024-08-17T17:35:18.672 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:35:18.672 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.672 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.672 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:35:18.672 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.951718+0000", 2024-08-17T17:35:18.672 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7370000000000001e-06 2024-08-17T17:35:18.672 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.673 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.673 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:35:18.673 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.951728+0000", 2024-08-17T17:35:18.673 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0213e-05 2024-08-17T17:35:18.673 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.673 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.673 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:35:18.673 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.951746+0000", 2024-08-17T17:35:18.673 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8172000000000002e-05 2024-08-17T17:35:18.673 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.673 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.673 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:35:18.674 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.951754+0000", 2024-08-17T17:35:18.674 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7579999999999992e-06 2024-08-17T17:35:18.674 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.674 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.674 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.674 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.951823+0000", 2024-08-17T17:35:18.674 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8829000000000001e-05 2024-08-17T17:35:18.675 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.675 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.675 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:35:18.675 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.951835+0000", 2024-08-17T17:35:18.675 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2363e-05 2024-08-17T17:35:18.675 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.675 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.675 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:35:18.675 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.817439+0000", 2024-08-17T17:35:18.675 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.86560404599999996 2024-08-17T17:35:18.675 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.676 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.676 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:35:18.676 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.817579+0000", 2024-08-17T17:35:18.676 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000140197 2024-08-17T17:35:18.676 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.676 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.676 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:35:18.676 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.817706+0000", 2024-08-17T17:35:18.676 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000127021 2024-08-17T17:35:18.676 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.677 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.677 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:35:18.677 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.822494+0000", 2024-08-17T17:35:18.677 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0047871140000000003 2024-08-17T17:35:18.677 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.677 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.677 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.677 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.822532+0000", 2024-08-17T17:35:18.677 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7948000000000003e-05 2024-08-17T17:35:18.677 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.677 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.677 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:18.678 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.822582+0000", 2024-08-17T17:35:18.678 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0670000000000001e-05 2024-08-17T17:35:18.678 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.678 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.678 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:35:18.678 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.822636+0000", 2024-08-17T17:35:18.678 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4135999999999998e-05 2024-08-17T17:35:18.678 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:35:18.678 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:35:18.678 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:35:18.678 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.822714+0000", 2024-08-17T17:35:18.679 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7830000000000005e-05 2024-08-17T17:35:18.679 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.679 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.679 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.679 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:35:18.679 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:35:18.679 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:35:18.680 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 18 16 67 51 0.177066 0 - 0.895292 2024-08-17T17:35:18.701 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:18.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:18.802 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:18.824 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:18.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:18.926 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:18.949 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:18.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:18.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:19.051 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:19.072 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:19.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:19.174 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:19.196 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:19.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:19.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:18 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2044848804' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:19.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:18 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2044848804' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:19.298 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:19.324 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:19.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:19.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:18 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2044848804' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:19.391 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:35:19.391 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:35:19.391 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:35:19.391 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:35:19.391 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:35:19.392 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:35:19.392 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:35:19.392 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:35:19.392 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:35:19.392 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:35:19.392 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:35:19.392 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:35:19.393 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:35:19.393 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:35:19.426 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:19.452 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:19.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:19.554 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:19.576 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:19.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:19.661 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 19 16 67 51 0.167747 0 - 0.895292 2024-08-17T17:35:19.678 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:19.704 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:19.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:19.806 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:19.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.833 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:19.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:19.862 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 6, 7, 0, 1, 2] out_osds: [] dead_osds: [6, 7] live_osds: [0, 1, 2, 3, 4, 5] 2024-08-17T17:35:19.862 INFO:tasks.thrashosds.thrasher:Reviving osd 7 2024-08-17T17:35:19.863 INFO:tasks.cephadm.osd.7:Restarting osd.7 (starting--it wasn't running)... 2024-08-17T17:35:19.863 DEBUG:teuthology.orchestra.run.smithi162:osd.7> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.7.service 2024-08-17T17:35:19.875 DEBUG:teuthology.orchestra.run.smithi162:> sudo systemctl start ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.7 2024-08-17T17:35:19.934 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:19.957 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:19.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:19.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:19 smithi063 ceph-mon[114756]: pgmap v2069: 119 pgs: 2 remapped+peering, 2 peering, 9 active+undersized+remapped, 31 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 24 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 256 KiB/s wr, 3 op/s; 32/231 objects degraded (13.853%); 32/231 objects misplaced (13.853%); 416 KiB/s, 5 objects/s recovering 2024-08-17T17:35:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:19 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 32/231 objects degraded (13.853%), 12 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:19 smithi063 ceph-mon[109843]: pgmap v2069: 119 pgs: 2 remapped+peering, 2 peering, 9 active+undersized+remapped, 31 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 24 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 256 KiB/s wr, 3 op/s; 32/231 objects degraded (13.853%); 32/231 objects misplaced (13.853%); 416 KiB/s, 5 objects/s recovering 2024-08-17T17:35:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:19 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 32/231 objects degraded (13.853%), 12 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:20.050 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:35:20.051 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:35:20.059 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:20.081 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:20.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:20.183 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:20.206 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:20.207 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:19 smithi162 ceph-mon[104668]: pgmap v2069: 119 pgs: 2 remapped+peering, 2 peering, 9 active+undersized+remapped, 31 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 24 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 256 KiB/s wr, 3 op/s; 32/231 objects degraded (13.853%); 32/231 objects misplaced (13.853%); 416 KiB/s, 5 objects/s recovering 2024-08-17T17:35:20.207 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:19 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 32/231 objects degraded (13.853%), 12 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:20.207 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:19 smithi162 systemd[1]: Starting Ceph osd.7 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:35:20.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:20.309 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:20.333 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:20.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:20.434 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:20.458 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:20.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:20.462 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 podman[225801]: 2024-08-17 17:35:20.108478528 +0000 UTC m=+0.017723350 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:35:20.462 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 podman[225801]: 2024-08-17 17:35:20.225467558 +0000 UTC m=+0.134712380 container create 760d5d9f0d228ba54098eba1c258928079611e491d4e70b4f1d43881081556fb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-activate, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:35:20.462 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 podman[225801]: 2024-08-17 17:35:20.46155482 +0000 UTC m=+0.370799649 container init 760d5d9f0d228ba54098eba1c258928079611e491d4e70b4f1d43881081556fb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:35:20.560 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:20.583 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:20.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:20.661 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:35:20.661694+0000 min lat: 0.000948653 max lat: 15.1952 avg lat: 0.895292 2024-08-17T17:35:20.661 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:35:20.662 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 20 16 67 51 0.159358 0 - 0.895292 2024-08-17T17:35:20.684 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:20.708 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:20.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:20.797 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 podman[225801]: 2024-08-17 17:35:20.466986949 +0000 UTC m=+0.376231773 container start 760d5d9f0d228ba54098eba1c258928079611e491d4e70b4f1d43881081556fb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.36.0) 2024-08-17T17:35:20.797 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 podman[225801]: 2024-08-17 17:35:20.544477326 +0000 UTC m=+0.453722169 container attach 760d5d9f0d228ba54098eba1c258928079611e491d4e70b4f1d43881081556fb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, CEPH_POINT_RELEASE=) 2024-08-17T17:35:20.810 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:20.835 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:20.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:20.937 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:20.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:20.960 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:20.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:21.061 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:21.085 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:21.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:21.096 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-activate[225816]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:35:21.096 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 bash[225801]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:35:21.096 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-activate[225816]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-08-17T17:35:21.096 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 bash[225801]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-08-17T17:35:21.096 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-activate[225816]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-08-17T17:35:21.096 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 bash[225801]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-08-17T17:35:21.096 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-activate[225816]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:35:21.096 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 bash[225801]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-08-17T17:35:21.096 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-activate[225816]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-08-17T17:35:21.097 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 bash[225801]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-08-17T17:35:21.097 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-activate[225816]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:35:21.097 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 bash[225801]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-08-17T17:35:21.097 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-activate[225816]: --> ceph-volume raw activate successful for osd ID: 7 2024-08-17T17:35:21.097 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 bash[225801]: --> ceph-volume raw activate successful for osd ID: 7 2024-08-17T17:35:21.097 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:20 smithi162 podman[225801]: 2024-08-17 17:35:20.846449987 +0000 UTC m=+0.755694810 container died 760d5d9f0d228ba54098eba1c258928079611e491d4e70b4f1d43881081556fb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-activate, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:35:21.187 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:21.211 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:21.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:21.313 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:21.335 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:21.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:21.437 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:21.459 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:21.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:21.561 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:21.581 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:35:21.583 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:21.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:21.662 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 21 16 67 51 0.151769 0 - 0.895292 2024-08-17T17:35:21.685 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:21.710 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:21.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:21.812 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:21.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.835 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:21.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:21.839 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:21 smithi162 podman[225964]: 2024-08-17 17:35:21.49029014 +0000 UTC m=+0.638033006 container remove 760d5d9f0d228ba54098eba1c258928079611e491d4e70b4f1d43881081556fb (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-activate, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:35:21.839 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:21 smithi162 podman[226086]: 2024-08-17 17:35:21.707956918 +0000 UTC m=+0.114642272 container create 9afc092f538774cb0dfbccc98065aeaed3052f50b417ef19f04840a76360ee48 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:35:21.839 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:21 smithi162 podman[226086]: 2024-08-17 17:35:21.612205613 +0000 UTC m=+0.018890981 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:35:21.936 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:21.959 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:21.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:21.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:22.052 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:35:22.061 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:22.074 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:35:22.075 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:21 smithi063 ceph-mon[114756]: pgmap v2070: 119 pgs: 2 remapped+peering, 2 peering, 9 active+undersized+remapped, 31 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 24 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 189 KiB/s wr, 2 op/s; 33/234 objects degraded (14.103%); 32/234 objects misplaced (13.675%); 289 KiB/s, 4 objects/s recovering 2024-08-17T17:35:22.075 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:21 smithi063 ceph-mon[109843]: pgmap v2070: 119 pgs: 2 remapped+peering, 2 peering, 9 active+undersized+remapped, 31 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 24 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 189 KiB/s wr, 2 op/s; 33/234 objects degraded (14.103%); 32/234 objects misplaced (13.675%); 289 KiB/s, 4 objects/s recovering 2024-08-17T17:35:22.096 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:22.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:22.107 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:22.198 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:22.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.222 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:22.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:22.235 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:21 smithi162 ceph-mon[104668]: pgmap v2070: 119 pgs: 2 remapped+peering, 2 peering, 9 active+undersized+remapped, 31 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 24 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 189 KiB/s wr, 2 op/s; 33/234 objects degraded (14.103%); 32/234 objects misplaced (13.675%); 289 KiB/s, 4 objects/s recovering 2024-08-17T17:35:22.235 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:21 smithi162 podman[226086]: 2024-08-17 17:35:21.952274098 +0000 UTC m=+0.358959460 container init 9afc092f538774cb0dfbccc98065aeaed3052f50b417ef19f04840a76360ee48 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-08-17T17:35:22.235 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:21 smithi162 podman[226086]: 2024-08-17 17:35:21.957700769 +0000 UTC m=+0.364386125 container start 9afc092f538774cb0dfbccc98065aeaed3052f50b417ef19f04840a76360ee48 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-08-17T17:35:22.235 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:22 smithi162 ceph-osd[226163]: -- 172.21.15.162:0/947691594 <== mon.1 v2:172.21.15.162:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556283a351e0 con 0x5562847b4400 2024-08-17T17:35:22.235 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:22 smithi162 bash[226086]: 9afc092f538774cb0dfbccc98065aeaed3052f50b417ef19f04840a76360ee48 2024-08-17T17:35:22.235 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:22 smithi162 systemd[1]: Started Ceph osd.7 for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:35:22.265 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:22.323 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:22.346 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:22.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:22.450 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:22.473 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:22.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:22.575 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:22.598 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:22.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:22.662 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 22 16 67 51 0.14487 0 - 0.895292 2024-08-17T17:35:22.700 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:22.723 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:22.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:22.825 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:22.845 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:35:22.549+0000 7f3a5185e640 -1 Falling back to public interface 2024-08-17T17:35:22.847 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:22.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:22.869 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:35:22.887 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:22.949 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:22.970 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:22.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:22.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:22 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1451248065' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:35:23.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:22 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1451248065' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:35:23.072 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:23.093 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:23.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:23.195 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:23.218 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:23.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:23.319 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:23.341 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:23.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:23.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:22 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1451248065' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:35:23.443 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:23.464 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:23.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:23.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:23.575 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:35:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:35:23] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.51.0" 2024-08-17T17:35:23.592 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:23.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:23.662 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 23 16 67 51 0.138571 0 - 0.895292 2024-08-17T17:35:23.694 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:23.718 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:23.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:23.820 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:23.843 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:23.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:23.944 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:23.963 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:35:23.963 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:35:23.963 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 0 2024-08-17T17:35:23.963 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:35:23.963 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:23 smithi162 ceph-mon[104668]: pgmap v2071: 119 pgs: 1 remapped+peering, 9 active+undersized+remapped, 32 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 161 KiB/s wr, 2 op/s; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%); 246 KiB/s, 3 objects/s recovering 2024-08-17T17:35:23.963 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:23 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:23.963 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:23 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1451248065' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:35:23.964 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:23 smithi162 ceph-mon[104668]: osdmap e1368: 8 total, 6 up, 8 in 2024-08-17T17:35:23.967 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:23.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:23.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:24.069 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:24.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.093 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:24.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:24.142 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:23 smithi063 ceph-mon[114756]: pgmap v2071: 119 pgs: 1 remapped+peering, 9 active+undersized+remapped, 32 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 161 KiB/s wr, 2 op/s; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%); 246 KiB/s, 3 objects/s recovering 2024-08-17T17:35:24.210 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:23 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:24.210 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:23 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1451248065' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:35:24.210 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:23 smithi063 ceph-mon[114756]: osdmap e1368: 8 total, 6 up, 8 in 2024-08-17T17:35:24.210 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:23 smithi063 ceph-mon[109843]: pgmap v2071: 119 pgs: 1 remapped+peering, 9 active+undersized+remapped, 32 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 161 KiB/s wr, 2 op/s; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%); 246 KiB/s, 3 objects/s recovering 2024-08-17T17:35:24.211 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:23 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:24.211 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:23 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1451248065' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:35:24.211 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:23 smithi063 ceph-mon[109843]: osdmap e1368: 8 total, 6 up, 8 in 2024-08-17T17:35:24.211 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:24.234 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:24.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:24.336 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:24.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.361 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:24.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:24.462 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:24.485 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:24.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:24.541 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:35:24.541 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:35:24.553 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:35:24.553 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:35:24.569 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:35:24.570 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:35:24.586 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:35:24.586 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:35:24.588 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:24.603 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:35:24.604 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:35:24.610 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:24.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:24.621 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:35:24.621 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:35:24.621 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:35:24.622 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:35:24.662 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 24 16 67 51 0.132798 0 - 0.895292 2024-08-17T17:35:24.712 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:24.735 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:24.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:24.836 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:24.860 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:24.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:24.962 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:24.984 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:24.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:24.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:25.007 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:35:25.086 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:25.108 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:25.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:25.179 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:24 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/257719740' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:25.210 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:25.232 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:25.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:25.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:24 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/257719740' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:25.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:24 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/257719740' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:25.334 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:25.357 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:25.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.358 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:25.409 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:35:25.409 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:35:25.459 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:25.481 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:25.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.482 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:25.583 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:25.600 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:35:25.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.607 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:25.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:25.662 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 25 16 67 51 0.127486 0 - 0.895292 2024-08-17T17:35:25.708 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:25.730 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:25.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:25.818 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:25.833 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:25.855 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:25.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:25.940 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:25 smithi063 ceph-mon[114756]: pgmap v2073: 119 pgs: 9 active+undersized+remapped, 33 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 140 KiB/s wr, 2 op/s; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%); 215 KiB/s, 3 objects/s recovering 2024-08-17T17:35:25.940 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:25 smithi063 ceph-mon[109843]: pgmap v2073: 119 pgs: 9 active+undersized+remapped, 33 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 140 KiB/s wr, 2 op/s; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%); 215 KiB/s, 3 objects/s recovering 2024-08-17T17:35:25.957 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:25.979 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:25.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:25.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:26.081 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:26.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:25 smithi162 ceph-mon[104668]: pgmap v2073: 119 pgs: 9 active+undersized+remapped, 33 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 140 KiB/s wr, 2 op/s; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%); 215 KiB/s, 3 objects/s recovering 2024-08-17T17:35:26.104 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:26.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:26.206 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:26.228 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:26.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:26.330 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:26.355 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:26.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:26.457 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:26.479 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:26.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:26.581 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:26.605 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:26.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:26.662 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 26 16 67 51 0.122582 0 - 0.895292 2024-08-17T17:35:26.676 INFO:teuthology.orchestra.run.smithi162.stderr:no valid command found; 10 closest matches: 2024-08-17T17:35:26.676 INFO:teuthology.orchestra.run.smithi162.stderr:0 2024-08-17T17:35:26.676 INFO:teuthology.orchestra.run.smithi162.stderr:1 2024-08-17T17:35:26.676 INFO:teuthology.orchestra.run.smithi162.stderr:2 2024-08-17T17:35:26.677 INFO:teuthology.orchestra.run.smithi162.stderr:abort 2024-08-17T17:35:26.677 INFO:teuthology.orchestra.run.smithi162.stderr:assert 2024-08-17T17:35:26.677 INFO:teuthology.orchestra.run.smithi162.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:35:26.677 INFO:teuthology.orchestra.run.smithi162.stderr:bluefs files list 2024-08-17T17:35:26.677 INFO:teuthology.orchestra.run.smithi162.stderr:bluefs stats 2024-08-17T17:35:26.677 INFO:teuthology.orchestra.run.smithi162.stderr:bluestore bluefs device info [] 2024-08-17T17:35:26.677 INFO:teuthology.orchestra.run.smithi162.stderr:config diff 2024-08-17T17:35:26.677 INFO:teuthology.orchestra.run.smithi162.stderr:admin_socket: invalid command 2024-08-17T17:35:26.707 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:26.728 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:26.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:26.830 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:26.855 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:26.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:26.878 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:26.957 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:26.979 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:26.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:26.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:27.081 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:27.103 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:27.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:27.205 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:27.226 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:27.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:27.328 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:27.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.353 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:27.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:27.410 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:35:27.454 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:27.477 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:27.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:27.478 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:35:27.479 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-7, ['dump_ops_in_flight'] 2024-08-17T17:35:27.578 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:27.599 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:27.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:27.662 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 27 16 67 51 0.118042 0 - 0.895292 2024-08-17T17:35:27.701 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:27.723 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:27.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:27.825 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:27.847 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:27.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:27.876 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:35:27.949 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:27.974 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:27.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:27.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:28.077 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:27 smithi162 ceph-mon[104668]: pgmap v2074: 119 pgs: 9 active+undersized+remapped, 33 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 7.3 KiB/s wr, 0 op/s; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%) 2024-08-17T17:35:28.078 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:27 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4001284183' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:28.079 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:28.101 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:28.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:28.202 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:28.224 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:28.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:28.271 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:27 smithi063 ceph-mon[114756]: pgmap v2074: 119 pgs: 9 active+undersized+remapped, 33 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 7.3 KiB/s wr, 0 op/s; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%) 2024-08-17T17:35:28.271 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:27 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4001284183' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:28.272 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:27 smithi063 ceph-mon[109843]: pgmap v2074: 119 pgs: 9 active+undersized+remapped, 33 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 7.3 KiB/s wr, 0 op/s; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%) 2024-08-17T17:35:28.272 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:27 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4001284183' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:28.326 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:28.349 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:28.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:28.451 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:28.473 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:28.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:28.574 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:28.597 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:28.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:28.662 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 28 16 67 51 0.113827 0 - 0.895292 2024-08-17T17:35:28.699 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:28.724 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:28.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:28.730 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:28.826 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:28.848 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:28.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:28.950 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:28.973 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:28.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:28.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:28.977 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:28 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:35:28.977 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:28 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4001284183' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:28.977 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:28 smithi063 ceph-mon[109843]: osdmap e1369: 8 total, 6 up, 8 in 2024-08-17T17:35:28.977 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:28 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 33/234 objects degraded (14.103%), 12 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:28.977 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:28 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:35:28.978 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:28 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4001284183' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:28.978 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:28 smithi063 ceph-mon[114756]: osdmap e1369: 8 total, 6 up, 8 in 2024-08-17T17:35:28.978 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:28 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 33/234 objects degraded (14.103%), 12 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:29.074 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:29.097 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:29.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:29.193 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:28 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:35:29.193 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:28 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4001284183' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:29.193 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:28 smithi162 ceph-mon[104668]: osdmap e1369: 8 total, 6 up, 8 in 2024-08-17T17:35:29.194 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:28 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 33/234 objects degraded (14.103%), 12 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:29.199 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:29.221 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:29.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:29.323 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:29.344 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:29.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:29.427 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:35:29.427 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:35:29.446 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:29.468 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:29.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:29.570 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:29.594 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:29.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:29.662 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 29 16 67 51 0.109902 0 - 0.895292 2024-08-17T17:35:29.696 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:29.718 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:29.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:29.820 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:29.842 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:29.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:29.944 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:29.967 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:29.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:29.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:30.069 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:30.092 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:30.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:29 smithi162 ceph-mon[104668]: pgmap v2076: 119 pgs: 9 active+undersized+remapped, 33 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%) 2024-08-17T17:35:30.137 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:35:30.146 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:29 smithi063 ceph-mon[114756]: pgmap v2076: 119 pgs: 9 active+undersized+remapped, 33 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%) 2024-08-17T17:35:30.146 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:29 smithi063 ceph-mon[109843]: pgmap v2076: 119 pgs: 9 active+undersized+remapped, 33 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%) 2024-08-17T17:35:30.193 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:30.215 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:30.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:30.317 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:30.341 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:30.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:30.349 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:30.443 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:30.464 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:30.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:30.498 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:35:30.499 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:35:30.515 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:35:30.515 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:35:30.531 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:35:30.532 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:35:30.548 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:35:30.548 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:35:30.564 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:35:30.564 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:35:30.567 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:30.580 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:35:30.581 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:35:30.581 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:35:30.581 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.7 2024-08-17T17:35:30.590 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:30.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:30.663 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 30 16 67 51 0.106238 0 - 0.895292 2024-08-17T17:35:30.692 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:30.715 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:30.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:30.716 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:35:30.817 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:30.839 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:30.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:30.941 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:30.963 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:30.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:30.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:31.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:30 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1453312216' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:30 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1453312216' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:31.065 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:31.087 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:31.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:31.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:30 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1453312216' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:31.189 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:31.211 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:31.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:31.313 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:31.336 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:31.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:31.369 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:35:31.370 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:35:31.438 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:31.461 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:31.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:31.501 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:31.522 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:31 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:35:31.324+0000 7f3a5185e640 -1 osd.7 1359 log_to_monitors true 2024-08-17T17:35:31.563 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:31.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.587 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:31.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:31.663 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 31 16 67 51 0.102811 0 - 0.895292 2024-08-17T17:35:31.688 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:31.709 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:31.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:31.811 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:31.834 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:31.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:31.935 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:31.958 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:31.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:31.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:32.060 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:32.082 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:32.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:32.091 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:35:31 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:35:31.944+0000 7f3a45600640 -1 osd.7 1370 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:35:32.091 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:31 smithi162 ceph-mon[104668]: pgmap v2077: 119 pgs: 9 active+undersized+remapped, 33 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%) 2024-08-17T17:35:32.091 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:31 smithi162 ceph-mon[104668]: from='osd.7 [v2:172.21.15.162:6816/3585957098,v1:172.21.15.162:6817/3585957098]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:35:32.091 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:31 smithi162 ceph-mon[104668]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:35:32.091 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:32.092 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:32.092 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:35:32.092 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:35:32.092 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:35:32.092 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:35:32.092 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.092 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:174887 10.c 10:3cb5f4d4:::benchmark_data_smithi063_294527_object37702:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1312)", 2024-08-17T17:35:32.093 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:32:46.588852+0000", 2024-08-17T17:35:32.093 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 165.500445974, 2024-08-17T17:35:32.093 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0166613250000001, 2024-08-17T17:35:32.093 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.093 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:35:32.093 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.093 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.093 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.093 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 174887 2024-08-17T17:35:32.093 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.093 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.093 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.094 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.094 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:46.588852+0000", 2024-08-17T17:35:32.094 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.094 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.094 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.094 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.094 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:46.588852+0000", 2024-08-17T17:35:32.094 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.094 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.094 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.094 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.094 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:46.588851+0000", 2024-08-17T17:35:32.095 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:35:32.095 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.095 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.095 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.095 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:46.588883+0000", 2024-08-17T17:35:32.095 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.2239000000000001e-05 2024-08-17T17:35:32.095 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.095 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.095 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.095 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:46.588894+0000", 2024-08-17T17:35:32.095 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0285999999999999e-05 2024-08-17T17:35:32.095 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.096 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.096 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.096 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:46.588902+0000", 2024-08-17T17:35:32.096 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.038e-06 2024-08-17T17:35:32.096 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.096 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.096 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.096 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:46.591630+0000", 2024-08-17T17:35:32.096 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0027280270000000001 2024-08-17T17:35:32.096 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.096 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.096 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.096 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:46.591680+0000", 2024-08-17T17:35:32.097 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0614e-05 2024-08-17T17:35:32.097 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.097 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.097 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.097 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:47.604422+0000", 2024-08-17T17:35:32.097 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0127418050000001 2024-08-17T17:35:32.097 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.097 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.097 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.097 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:32:47.605513+0000", 2024-08-17T17:35:32.097 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001091315 2024-08-17T17:35:32.097 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.097 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.098 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.098 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.098 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.098 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217515 10.2 10:436ade26:::benchmark_data_smithi063_294527_object80330:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1321)", 2024-08-17T17:35:32.098 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:24.896915+0000", 2024-08-17T17:35:32.098 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 127.192382786, 2024-08-17T17:35:32.098 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.131623177, 2024-08-17T17:35:32.098 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.098 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:35:32.098 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.098 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.098 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.099 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217515 2024-08-17T17:35:32.099 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.099 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.099 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.099 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.099 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.896915+0000", 2024-08-17T17:35:32.099 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.099 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.099 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.099 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.099 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.896915+0000", 2024-08-17T17:35:32.099 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.099 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.100 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.100 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.100 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.896915+0000", 2024-08-17T17:35:32.100 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:35:32.100 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.100 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.100 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.100 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.896919+0000", 2024-08-17T17:35:32.100 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7190000000000001e-06 2024-08-17T17:35:32.100 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.100 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.100 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.101 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.896932+0000", 2024-08-17T17:35:32.101 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2968e-05 2024-08-17T17:35:32.101 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.101 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.101 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.101 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.896939+0000", 2024-08-17T17:35:32.101 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9519999999999997e-06 2024-08-17T17:35:32.101 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.101 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.101 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.101 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.896984+0000", 2024-08-17T17:35:32.101 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4591e-05 2024-08-17T17:35:32.101 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.101 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.101 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.102 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.897094+0000", 2024-08-17T17:35:32.102 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000110202 2024-08-17T17:35:32.102 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.102 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.102 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:35:32.102 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.897490+0000", 2024-08-17T17:35:32.102 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00039553899999999998 2024-08-17T17:35:32.102 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.102 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.102 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.102 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.903817+0000", 2024-08-17T17:35:32.102 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0063278350000000004 2024-08-17T17:35:32.102 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.103 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.103 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.103 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.903830+0000", 2024-08-17T17:35:32.103 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2867000000000001e-05 2024-08-17T17:35:32.103 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.103 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.103 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.103 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.022871+0000", 2024-08-17T17:35:32.103 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.119041041 2024-08-17T17:35:32.103 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.103 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.103 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:35:32.103 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.022963+0000", 2024-08-17T17:35:32.103 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.1442999999999998e-05 2024-08-17T17:35:32.104 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.104 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.104 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.104 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.028440+0000", 2024-08-17T17:35:32.104 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0054767920000000003 2024-08-17T17:35:32.104 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.104 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.104 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.104 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.028538+0000", 2024-08-17T17:35:32.104 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8899000000000003e-05 2024-08-17T17:35:32.104 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.104 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.104 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.105 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.105 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.105 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217518 10.0 10:04aa3b8c:::benchmark_data_smithi063_294527_object80333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T17:35:32.105 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:24.912852+0000", 2024-08-17T17:35:32.105 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 127.176445644, 2024-08-17T17:35:32.105 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.166729997, 2024-08-17T17:35:32.105 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.105 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.105 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.105 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.105 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.105 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217518 2024-08-17T17:35:32.106 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.106 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.106 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.106 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.106 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.912852+0000", 2024-08-17T17:35:32.106 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.106 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.106 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.106 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.106 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.912852+0000", 2024-08-17T17:35:32.106 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.106 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.106 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.106 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.107 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.912851+0000", 2024-08-17T17:35:32.107 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:35:32.107 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.107 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.107 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.107 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.912865+0000", 2024-08-17T17:35:32.107 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4208e-05 2024-08-17T17:35:32.108 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.108 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.108 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.108 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.912878+0000", 2024-08-17T17:35:32.108 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3089e-05 2024-08-17T17:35:32.108 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.108 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.108 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.109 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.912886+0000", 2024-08-17T17:35:32.109 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2579999999999998e-06 2024-08-17T17:35:32.109 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.109 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.109 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.109 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.912920+0000", 2024-08-17T17:35:32.109 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4014999999999999e-05 2024-08-17T17:35:32.109 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.109 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.109 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.109 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.912929+0000", 2024-08-17T17:35:32.109 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7499999999999998e-06 2024-08-17T17:35:32.109 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.110 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.110 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.110 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.915416+0000", 2024-08-17T17:35:32.110 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.002486808 2024-08-17T17:35:32.110 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.110 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.110 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.110 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.915429+0000", 2024-08-17T17:35:32.110 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2391999999999999e-05 2024-08-17T17:35:32.110 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.110 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.110 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.110 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.072427+0000", 2024-08-17T17:35:32.111 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.156998339 2024-08-17T17:35:32.111 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.111 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.111 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.111 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.072567+0000", 2024-08-17T17:35:32.111 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000140338 2024-08-17T17:35:32.111 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.111 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.111 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:35:32.111 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.073006+0000", 2024-08-17T17:35:32.111 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000438663 2024-08-17T17:35:32.111 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.112 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.112 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.112 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.076820+0000", 2024-08-17T17:35:32.112 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0038138989999999999 2024-08-17T17:35:32.112 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.112 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.112 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.112 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.077002+0000", 2024-08-17T17:35:32.112 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00018168000000000001 2024-08-17T17:35:32.112 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.112 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.112 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.112 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.079137+0000", 2024-08-17T17:35:32.113 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021350240000000001 2024-08-17T17:35:32.113 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.113 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.113 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.113 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.079278+0000", 2024-08-17T17:35:32.113 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014168699999999999 2024-08-17T17:35:32.113 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.113 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.113 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.113 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.079582+0000", 2024-08-17T17:35:32.113 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000304161 2024-08-17T17:35:32.113 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.113 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.114 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.114 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.114 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.114 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217538 10.12 10:4aefbbe3:::benchmark_data_smithi063_294527_object80353:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T17:35:32.114 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:24.948052+0000", 2024-08-17T17:35:32.114 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 127.14124650399999, 2024-08-17T17:35:32.114 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1184878279999999, 2024-08-17T17:35:32.114 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.114 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.114 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.114 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.114 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.114 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217538 2024-08-17T17:35:32.115 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.115 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.115 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.115 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.115 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.948052+0000", 2024-08-17T17:35:32.115 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.115 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.115 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.115 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.115 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.948052+0000", 2024-08-17T17:35:32.115 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.115 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.115 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.116 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.116 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.948051+0000", 2024-08-17T17:35:32.116 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:35:32.116 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.116 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.116 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.116 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.948057+0000", 2024-08-17T17:35:32.116 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9200000000000001e-06 2024-08-17T17:35:32.116 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.116 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.116 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.116 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.948067+0000", 2024-08-17T17:35:32.117 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0011e-05 2024-08-17T17:35:32.117 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.117 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.117 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.117 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.948072+0000", 2024-08-17T17:35:32.117 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1220000000000003e-06 2024-08-17T17:35:32.117 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.117 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.117 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.117 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.948119+0000", 2024-08-17T17:35:32.117 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7119000000000001e-05 2024-08-17T17:35:32.117 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.117 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.118 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.118 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.948128+0000", 2024-08-17T17:35:32.118 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.1050000000000001e-06 2024-08-17T17:35:32.118 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.118 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.118 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.118 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.061250+0000", 2024-08-17T17:35:32.118 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.113122315 2024-08-17T17:35:32.118 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.118 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.118 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.118 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.061374+0000", 2024-08-17T17:35:32.119 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012367300000000001 2024-08-17T17:35:32.119 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.119 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.119 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1,3", 2024-08-17T17:35:32.119 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.061847+0000", 2024-08-17T17:35:32.119 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000473179 2024-08-17T17:35:32.119 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.119 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.119 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.119 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.066311+0000", 2024-08-17T17:35:32.119 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.004463247 2024-08-17T17:35:32.119 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.119 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.120 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.120 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.066348+0000", 2024-08-17T17:35:32.120 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7939999999999999e-05 2024-08-17T17:35:32.120 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.120 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.120 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.120 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.066387+0000", 2024-08-17T17:35:32.120 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8679e-05 2024-08-17T17:35:32.120 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.120 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.120 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.120 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.066418+0000", 2024-08-17T17:35:32.120 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.0787999999999997e-05 2024-08-17T17:35:32.120 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.121 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.121 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.121 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.066462+0000", 2024-08-17T17:35:32.121 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3823999999999999e-05 2024-08-17T17:35:32.121 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.121 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.121 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.121 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.066539+0000", 2024-08-17T17:35:32.121 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7577000000000006e-05 2024-08-17T17:35:32.121 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.121 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.121 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.122 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.122 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.122 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217541 10.12 10:4aae98bd:::benchmark_data_smithi063_294527_object80356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T17:35:32.122 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:24.953166+0000", 2024-08-17T17:35:32.122 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 127.13613206799999, 2024-08-17T17:35:32.122 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1142808230000001, 2024-08-17T17:35:32.122 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.122 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.122 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.122 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.122 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.122 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217541 2024-08-17T17:35:32.122 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.123 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.123 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.123 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.123 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.953166+0000", 2024-08-17T17:35:32.123 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.123 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.123 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.123 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.123 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.953166+0000", 2024-08-17T17:35:32.123 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.123 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.123 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.124 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.124 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.953165+0000", 2024-08-17T17:35:32.124 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:35:32.124 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.124 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.124 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.124 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.953171+0000", 2024-08-17T17:35:32.124 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7420000000000003e-06 2024-08-17T17:35:32.124 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.124 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.124 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.124 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.953183+0000", 2024-08-17T17:35:32.124 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.253e-05 2024-08-17T17:35:32.125 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.125 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.125 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.125 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.953190+0000", 2024-08-17T17:35:32.125 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3840000000000002e-06 2024-08-17T17:35:32.125 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.125 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.125 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.125 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.953235+0000", 2024-08-17T17:35:32.125 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5080999999999997e-05 2024-08-17T17:35:32.125 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.125 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.125 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.126 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.953244+0000", 2024-08-17T17:35:32.126 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8400000000000001e-06 2024-08-17T17:35:32.126 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.126 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.126 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.126 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.064136+0000", 2024-08-17T17:35:32.126 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.110892789 2024-08-17T17:35:32.126 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.126 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.126 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.126 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.064255+0000", 2024-08-17T17:35:32.126 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000118838 2024-08-17T17:35:32.127 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.127 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.127 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1,3", 2024-08-17T17:35:32.127 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.064594+0000", 2024-08-17T17:35:32.127 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00033872399999999999 2024-08-17T17:35:32.127 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.127 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.127 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.127 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.066554+0000", 2024-08-17T17:35:32.128 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0019602349999999998 2024-08-17T17:35:32.128 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.128 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.128 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.128 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.066575+0000", 2024-08-17T17:35:32.128 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0687999999999999e-05 2024-08-17T17:35:32.128 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.128 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.128 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.128 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.067130+0000", 2024-08-17T17:35:32.128 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00055555299999999997 2024-08-17T17:35:32.128 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.129 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.129 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.129 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.067304+0000", 2024-08-17T17:35:32.129 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000173685 2024-08-17T17:35:32.129 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.129 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.129 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.129 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.067348+0000", 2024-08-17T17:35:32.129 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3590000000000001e-05 2024-08-17T17:35:32.129 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.129 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.129 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.129 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.067447+0000", 2024-08-17T17:35:32.130 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9145e-05 2024-08-17T17:35:32.130 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.130 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.130 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.130 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.130 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.130 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217544 10.10 10:0a09eaf9:::benchmark_data_smithi063_294527_object80359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T17:35:32.130 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:24.958013+0000", 2024-08-17T17:35:32.130 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 127.13128474600001, 2024-08-17T17:35:32.130 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.106239628, 2024-08-17T17:35:32.130 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.130 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.131 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.131 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.131 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.131 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217544 2024-08-17T17:35:32.131 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.131 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.131 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.131 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.131 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.958013+0000", 2024-08-17T17:35:32.131 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.131 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.131 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.131 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.132 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.958013+0000", 2024-08-17T17:35:32.132 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.132 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.132 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.132 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.132 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.958013+0000", 2024-08-17T17:35:32.132 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:35:32.132 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.132 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.132 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.132 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.958015+0000", 2024-08-17T17:35:32.132 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5909999999999998e-06 2024-08-17T17:35:32.132 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.132 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.132 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.133 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.958023+0000", 2024-08-17T17:35:32.133 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3490000000000003e-06 2024-08-17T17:35:32.133 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.133 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.133 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.133 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.958026+0000", 2024-08-17T17:35:32.133 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4010000000000001e-06 2024-08-17T17:35:32.133 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.133 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.133 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.133 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.958076+0000", 2024-08-17T17:35:32.133 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0167000000000003e-05 2024-08-17T17:35:32.133 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.134 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.134 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.134 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.958087+0000", 2024-08-17T17:35:32.134 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.04e-05 2024-08-17T17:35:32.134 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.134 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.134 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.134 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.915410+0000", 2024-08-17T17:35:32.134 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95732366999999996 2024-08-17T17:35:32.134 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.134 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.134 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.134 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.915422+0000", 2024-08-17T17:35:32.135 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1654000000000001e-05 2024-08-17T17:35:32.135 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.135 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.135 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.135 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.056729+0000", 2024-08-17T17:35:32.135 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1413069300000001 2024-08-17T17:35:32.135 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.135 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.135 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.135 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.056811+0000", 2024-08-17T17:35:32.135 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.2159999999999999e-05 2024-08-17T17:35:32.135 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.135 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.136 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:35:32.136 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.057208+0000", 2024-08-17T17:35:32.136 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00039660200000000002 2024-08-17T17:35:32.136 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.136 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.136 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.136 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.059203+0000", 2024-08-17T17:35:32.136 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0019953979999999998 2024-08-17T17:35:32.136 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.136 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.136 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.136 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.061652+0000", 2024-08-17T17:35:32.136 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0024484060000000002 2024-08-17T17:35:32.137 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.137 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.137 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.137 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.064102+0000", 2024-08-17T17:35:32.137 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0024506749999999998 2024-08-17T17:35:32.137 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.137 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.137 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.137 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.064169+0000", 2024-08-17T17:35:32.137 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6464000000000006e-05 2024-08-17T17:35:32.137 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.137 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.137 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.137 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.064253+0000", 2024-08-17T17:35:32.138 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4233000000000005e-05 2024-08-17T17:35:32.138 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.138 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.138 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.138 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.138 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.138 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217560 10.10 10:0d1396f6:::benchmark_data_smithi063_294527_object80375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T17:35:32.138 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:24.979517+0000", 2024-08-17T17:35:32.138 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 127.109781359, 2024-08-17T17:35:32.138 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0850348259999998, 2024-08-17T17:35:32.138 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.138 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.138 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.139 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.139 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.139 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217560 2024-08-17T17:35:32.139 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.139 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.139 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.139 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.139 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.979517+0000", 2024-08-17T17:35:32.139 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.139 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.139 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.139 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.139 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.979517+0000", 2024-08-17T17:35:32.140 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.140 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.140 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.140 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.140 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.979515+0000", 2024-08-17T17:35:32.140 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:35:32.140 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.140 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.140 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.140 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.979521+0000", 2024-08-17T17:35:32.140 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7620000000000001e-06 2024-08-17T17:35:32.140 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.141 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.141 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.141 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.979534+0000", 2024-08-17T17:35:32.141 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3278999999999999e-05 2024-08-17T17:35:32.141 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.141 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.141 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.141 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.979542+0000", 2024-08-17T17:35:32.141 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.306e-06 2024-08-17T17:35:32.141 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.141 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.141 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.142 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.979602+0000", 2024-08-17T17:35:32.142 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0334000000000002e-05 2024-08-17T17:35:32.142 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.142 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.142 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.142 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.979612+0000", 2024-08-17T17:35:32.142 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0271e-05 2024-08-17T17:35:32.142 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.142 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.142 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.142 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.915448+0000", 2024-08-17T17:35:32.142 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.935835113 2024-08-17T17:35:32.142 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.143 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.143 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.143 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.915456+0000", 2024-08-17T17:35:32.143 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6910000000000003e-06 2024-08-17T17:35:32.143 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.143 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.143 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.143 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.059239+0000", 2024-08-17T17:35:32.143 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.143783196 2024-08-17T17:35:32.143 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.143 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.143 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.143 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.059375+0000", 2024-08-17T17:35:32.144 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000135635 2024-08-17T17:35:32.144 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.144 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.144 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:35:32.144 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.059730+0000", 2024-08-17T17:35:32.144 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00035485699999999998 2024-08-17T17:35:32.144 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.144 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.144 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.144 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.064302+0000", 2024-08-17T17:35:32.144 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0045718649999999996 2024-08-17T17:35:32.144 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.144 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.145 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.145 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.064341+0000", 2024-08-17T17:35:32.145 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8748000000000002e-05 2024-08-17T17:35:32.145 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.145 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.145 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.145 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.064350+0000", 2024-08-17T17:35:32.145 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4380000000000001e-06 2024-08-17T17:35:32.145 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.145 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.145 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.145 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.064431+0000", 2024-08-17T17:35:32.145 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0605000000000005e-05 2024-08-17T17:35:32.146 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.146 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.146 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.146 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.064551+0000", 2024-08-17T17:35:32.146 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000120933 2024-08-17T17:35:32.146 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.146 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.146 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.146 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.146 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.146 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217569 10.10 10:0833c3f3:::benchmark_data_smithi063_294527_object80384:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T17:35:32.146 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:24.997286+0000", 2024-08-17T17:35:32.146 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 127.092012497, 2024-08-17T17:35:32.147 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0675317889999998, 2024-08-17T17:35:32.147 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.147 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.147 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.147 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.147 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.147 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217569 2024-08-17T17:35:32.147 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.147 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.147 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.147 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.147 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.997286+0000", 2024-08-17T17:35:32.147 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.147 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.148 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.148 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.148 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.997286+0000", 2024-08-17T17:35:32.148 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.148 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.148 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.148 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.148 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.997285+0000", 2024-08-17T17:35:32.148 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:35:32.148 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.148 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.148 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.148 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.997292+0000", 2024-08-17T17:35:32.148 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7320000000000001e-06 2024-08-17T17:35:32.149 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.149 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.149 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.149 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.997304+0000", 2024-08-17T17:35:32.149 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2215000000000001e-05 2024-08-17T17:35:32.149 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.149 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.149 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.149 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.997311+0000", 2024-08-17T17:35:32.149 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3649999999999998e-06 2024-08-17T17:35:32.149 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.149 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.149 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.150 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.997366+0000", 2024-08-17T17:35:32.150 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4970999999999999e-05 2024-08-17T17:35:32.150 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.150 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.150 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.150 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:24.997376+0000", 2024-08-17T17:35:32.150 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.499e-06 2024-08-17T17:35:32.150 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.150 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.150 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.150 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.915491+0000", 2024-08-17T17:35:32.150 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.91811499900000004 2024-08-17T17:35:32.150 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.151 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.151 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.151 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.915499+0000", 2024-08-17T17:35:32.151 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7090000000000008e-06 2024-08-17T17:35:32.151 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.151 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.151 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.151 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.061696+0000", 2024-08-17T17:35:32.151 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.146196808 2024-08-17T17:35:32.151 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.151 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.151 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.151 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.061860+0000", 2024-08-17T17:35:32.152 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016396999999999999 2024-08-17T17:35:32.152 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.152 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.152 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:35:32.152 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.062146+0000", 2024-08-17T17:35:32.152 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00028628699999999998 2024-08-17T17:35:32.152 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.152 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.152 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.152 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.064563+0000", 2024-08-17T17:35:32.152 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002416695 2024-08-17T17:35:32.152 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.152 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.153 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.153 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.064620+0000", 2024-08-17T17:35:32.153 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6498e-05 2024-08-17T17:35:32.153 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.153 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.153 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.153 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.064667+0000", 2024-08-17T17:35:32.153 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6999999999999997e-05 2024-08-17T17:35:32.153 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.153 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.153 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.153 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.064715+0000", 2024-08-17T17:35:32.153 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8439000000000002e-05 2024-08-17T17:35:32.154 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.154 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.154 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.154 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.064817+0000", 2024-08-17T17:35:32.154 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000102354 2024-08-17T17:35:32.154 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.154 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.154 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.154 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.154 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.154 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217579 10.2 10:4267e1f9:::benchmark_data_smithi063_294527_object80394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T17:35:32.155 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:25.008612+0000", 2024-08-17T17:35:32.155 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 127.08068591599999, 2024-08-17T17:35:32.155 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0310009899999999, 2024-08-17T17:35:32.155 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.155 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.155 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.155 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.155 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.155 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217579 2024-08-17T17:35:32.155 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.155 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.155 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.155 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.156 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.008612+0000", 2024-08-17T17:35:32.156 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.156 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.156 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.156 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.156 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.008612+0000", 2024-08-17T17:35:32.156 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.156 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.156 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.156 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.156 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.008611+0000", 2024-08-17T17:35:32.156 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:35:32.157 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.157 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.157 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.157 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.008617+0000", 2024-08-17T17:35:32.157 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7049999999999996e-06 2024-08-17T17:35:32.157 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.157 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.157 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.157 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.008627+0000", 2024-08-17T17:35:32.157 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0456e-05 2024-08-17T17:35:32.157 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.157 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.157 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.158 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.008634+0000", 2024-08-17T17:35:32.158 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8530000000000003e-06 2024-08-17T17:35:32.158 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.158 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.158 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.158 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.008686+0000", 2024-08-17T17:35:32.158 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2244000000000003e-05 2024-08-17T17:35:32.158 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.158 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.158 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.158 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.008698+0000", 2024-08-17T17:35:32.158 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1378e-05 2024-08-17T17:35:32.158 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.159 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.159 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.159 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.023157+0000", 2024-08-17T17:35:32.159 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0144596859999999 2024-08-17T17:35:32.159 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.159 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.159 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.159 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.023270+0000", 2024-08-17T17:35:32.159 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000112633 2024-08-17T17:35:32.159 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.159 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.159 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1,3", 2024-08-17T17:35:32.159 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.023723+0000", 2024-08-17T17:35:32.160 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0004529 2024-08-17T17:35:32.160 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.160 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.160 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.160 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.025937+0000", 2024-08-17T17:35:32.160 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00221404 2024-08-17T17:35:32.160 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.160 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.160 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.160 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.026615+0000", 2024-08-17T17:35:32.160 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00067837699999999997 2024-08-17T17:35:32.160 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.161 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.161 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.161 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.028573+0000", 2024-08-17T17:35:32.161 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001958024 2024-08-17T17:35:32.161 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.161 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.161 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.161 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.039423+0000", 2024-08-17T17:35:32.161 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010849523999999999 2024-08-17T17:35:32.161 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.161 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.161 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.161 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.039483+0000", 2024-08-17T17:35:32.162 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9778999999999998e-05 2024-08-17T17:35:32.162 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.162 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.162 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.162 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.039613+0000", 2024-08-17T17:35:32.162 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013054200000000001 2024-08-17T17:35:32.162 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.162 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.162 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.162 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.162 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.163 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217583 10.10 10:0f14c327:::benchmark_data_smithi063_294527_object80398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T17:35:32.163 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:25.013330+0000", 2024-08-17T17:35:32.163 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 127.075967959, 2024-08-17T17:35:32.163 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.063382335, 2024-08-17T17:35:32.163 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.163 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.163 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.163 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.163 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.163 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217583 2024-08-17T17:35:32.163 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.163 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.163 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.164 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.164 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.013330+0000", 2024-08-17T17:35:32.164 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.164 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.164 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.164 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.164 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.013330+0000", 2024-08-17T17:35:32.164 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.164 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.164 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.164 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.164 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.013329+0000", 2024-08-17T17:35:32.164 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:35:32.165 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.165 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.165 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.165 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.013334+0000", 2024-08-17T17:35:32.165 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7400000000000004e-06 2024-08-17T17:35:32.165 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.165 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.165 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.165 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.013343+0000", 2024-08-17T17:35:32.165 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2099999999999999e-06 2024-08-17T17:35:32.165 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.165 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.165 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.165 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.013349+0000", 2024-08-17T17:35:32.166 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0689999999999998e-06 2024-08-17T17:35:32.166 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.166 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.166 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.166 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.013391+0000", 2024-08-17T17:35:32.166 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1937999999999997e-05 2024-08-17T17:35:32.166 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.166 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.166 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.166 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.013399+0000", 2024-08-17T17:35:32.166 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3980000000000002e-06 2024-08-17T17:35:32.166 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.166 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.167 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.167 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.915536+0000", 2024-08-17T17:35:32.167 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.90213739699999995 2024-08-17T17:35:32.167 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.167 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.167 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.167 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.915544+0000", 2024-08-17T17:35:32.167 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3809999999999999e-06 2024-08-17T17:35:32.167 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.167 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.167 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.168 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.064848+0000", 2024-08-17T17:35:32.169 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1493039409999999 2024-08-17T17:35:32.169 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.169 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.169 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.169 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.064971+0000", 2024-08-17T17:35:32.169 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012219999999999999 2024-08-17T17:35:32.169 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.169 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.169 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:35:32.169 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.065295+0000", 2024-08-17T17:35:32.169 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00032476299999999999 2024-08-17T17:35:32.169 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.170 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.170 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.170 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.072319+0000", 2024-08-17T17:35:32.170 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0070238119999999999 2024-08-17T17:35:32.170 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.170 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.170 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.170 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.072358+0000", 2024-08-17T17:35:32.170 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8461000000000003e-05 2024-08-17T17:35:32.170 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.170 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.170 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.170 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.075996+0000", 2024-08-17T17:35:32.171 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0036388219999999999 2024-08-17T17:35:32.171 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.171 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.171 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.171 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.076047+0000", 2024-08-17T17:35:32.171 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0135e-05 2024-08-17T17:35:32.171 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.171 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.171 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.171 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.076712+0000", 2024-08-17T17:35:32.171 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00066587999999999999 2024-08-17T17:35:32.171 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.171 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.172 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.172 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.172 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.172 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217593 10.10 10:089e3aea:::benchmark_data_smithi063_294527_object80408:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-08-17T17:35:32.172 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:25.055172+0000", 2024-08-17T17:35:32.172 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 127.034125917, 2024-08-17T17:35:32.172 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0218211940000002, 2024-08-17T17:35:32.172 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.172 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.172 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.172 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.172 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.173 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217593 2024-08-17T17:35:32.173 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.173 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.173 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.173 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.173 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.055172+0000", 2024-08-17T17:35:32.173 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.173 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.173 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.173 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.173 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.055172+0000", 2024-08-17T17:35:32.173 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.173 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.173 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.174 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.174 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.055171+0000", 2024-08-17T17:35:32.174 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:35:32.174 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.174 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.174 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.174 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.055177+0000", 2024-08-17T17:35:32.174 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5990000000000001e-06 2024-08-17T17:35:32.174 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.174 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.174 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.174 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.055188+0000", 2024-08-17T17:35:32.175 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.149e-05 2024-08-17T17:35:32.175 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.175 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.175 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.175 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.055193+0000", 2024-08-17T17:35:32.175 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7099999999999998e-06 2024-08-17T17:35:32.175 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.175 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.175 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.175 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.055228+0000", 2024-08-17T17:35:32.175 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5465e-05 2024-08-17T17:35:32.175 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.175 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.176 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.176 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.055235+0000", 2024-08-17T17:35:32.176 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9450000000000004e-06 2024-08-17T17:35:32.176 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.176 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.176 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.176 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.915583+0000", 2024-08-17T17:35:32.176 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.86034776599999996 2024-08-17T17:35:32.176 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.176 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.176 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.176 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.915589+0000", 2024-08-17T17:35:32.176 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6879999999999996e-06 2024-08-17T17:35:32.177 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.177 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.177 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.177 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.069689+0000", 2024-08-17T17:35:32.177 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.154100731 2024-08-17T17:35:32.177 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.177 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.177 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.177 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.069877+0000", 2024-08-17T17:35:32.177 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000187116 2024-08-17T17:35:32.177 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.177 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.177 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:35:32.177 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.070316+0000", 2024-08-17T17:35:32.178 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000439851 2024-08-17T17:35:32.178 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.178 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.178 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.178 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.072394+0000", 2024-08-17T17:35:32.178 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0020778250000000002 2024-08-17T17:35:32.178 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.178 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.178 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.178 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.076783+0000", 2024-08-17T17:35:32.178 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0043887240000000001 2024-08-17T17:35:32.178 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.178 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.179 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.179 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.076838+0000", 2024-08-17T17:35:32.179 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5191000000000001e-05 2024-08-17T17:35:32.179 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.179 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.179 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.179 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.076886+0000", 2024-08-17T17:35:32.179 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8275000000000001e-05 2024-08-17T17:35:32.179 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.179 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.179 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.179 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.076993+0000", 2024-08-17T17:35:32.180 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000106908 2024-08-17T17:35:32.180 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.180 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.180 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.180 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.180 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.180 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217506 10.b 10:deb706c9:::benchmark_data_smithi063_294527_object80321:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1323)", 2024-08-17T17:35:32.180 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:25.916019+0000", 2024-08-17T17:35:32.180 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 126.17327920699999, 2024-08-17T17:35:32.180 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.153235598, 2024-08-17T17:35:32.180 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.180 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:35:32.180 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.181 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.181 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.181 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217506 2024-08-17T17:35:32.181 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.181 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.181 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.181 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.181 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916019+0000", 2024-08-17T17:35:32.181 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.181 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.181 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.181 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.182 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916019+0000", 2024-08-17T17:35:32.182 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.182 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.182 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.182 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.182 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916017+0000", 2024-08-17T17:35:32.182 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:35:32.182 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.182 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.182 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.182 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916024+0000", 2024-08-17T17:35:32.182 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5910000000000001e-06 2024-08-17T17:35:32.182 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.183 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.183 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.183 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916037+0000", 2024-08-17T17:35:32.183 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3175e-05 2024-08-17T17:35:32.183 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.183 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.183 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.183 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916047+0000", 2024-08-17T17:35:32.183 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9599999999999995e-06 2024-08-17T17:35:32.183 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.183 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.183 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.183 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916087+0000", 2024-08-17T17:35:32.184 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9771000000000003e-05 2024-08-17T17:35:32.184 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.184 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.184 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.184 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916092+0000", 2024-08-17T17:35:32.184 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0560000000000004e-06 2024-08-17T17:35:32.184 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.184 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.184 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.184 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.069165+0000", 2024-08-17T17:35:32.184 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.153072791 2024-08-17T17:35:32.184 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.184 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.185 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.185 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.069254+0000", 2024-08-17T17:35:32.185 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9802999999999994e-05 2024-08-17T17:35:32.185 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.185 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.185 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.185 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.185 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.185 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217510 10.b 10:ddf1cb92:::benchmark_data_smithi063_294527_object80325:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1323)", 2024-08-17T17:35:32.185 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:25.916057+0000", 2024-08-17T17:35:32.185 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 126.173241285, 2024-08-17T17:35:32.185 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1617807820000001, 2024-08-17T17:35:32.185 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.186 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:35:32.186 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.186 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.186 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.186 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217510 2024-08-17T17:35:32.186 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.186 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.186 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.186 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.186 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916057+0000", 2024-08-17T17:35:32.186 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.186 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.186 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.187 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.187 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916057+0000", 2024-08-17T17:35:32.187 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.187 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.187 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.187 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.187 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916056+0000", 2024-08-17T17:35:32.187 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:35:32.187 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.187 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.187 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.187 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916059+0000", 2024-08-17T17:35:32.187 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.0130000000000002e-06 2024-08-17T17:35:32.188 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.188 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.188 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.188 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916067+0000", 2024-08-17T17:35:32.188 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.43e-06 2024-08-17T17:35:32.188 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.188 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.188 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.188 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916072+0000", 2024-08-17T17:35:32.188 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6790000000000002e-06 2024-08-17T17:35:32.188 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.188 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.188 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.189 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916113+0000", 2024-08-17T17:35:32.189 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0389999999999998e-05 2024-08-17T17:35:32.189 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.189 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.189 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.189 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916121+0000", 2024-08-17T17:35:32.189 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4689999999999997e-06 2024-08-17T17:35:32.189 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.189 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.189 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.189 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.069523+0000", 2024-08-17T17:35:32.189 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1534013169999999 2024-08-17T17:35:32.189 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.189 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.190 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:35:32.190 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.069642+0000", 2024-08-17T17:35:32.190 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011981600000000001 2024-08-17T17:35:32.190 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.190 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.190 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.190 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.077739+0000", 2024-08-17T17:35:32.190 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0080963500000000004 2024-08-17T17:35:32.190 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.190 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.190 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.190 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.077838+0000", 2024-08-17T17:35:32.190 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.8771999999999996e-05 2024-08-17T17:35:32.191 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.191 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.191 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.191 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.191 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.191 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217523 10.b 10:db4a7abd:::benchmark_data_smithi063_294527_object80338:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1323)", 2024-08-17T17:35:32.191 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:25.916085+0000", 2024-08-17T17:35:32.191 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 126.173213339, 2024-08-17T17:35:32.191 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1619718699999999, 2024-08-17T17:35:32.191 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.191 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.191 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.191 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.191 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.192 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217523 2024-08-17T17:35:32.192 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.192 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.192 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.192 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.192 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916085+0000", 2024-08-17T17:35:32.192 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.192 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.192 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.192 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.192 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916085+0000", 2024-08-17T17:35:32.192 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.192 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.193 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.193 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.193 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916084+0000", 2024-08-17T17:35:32.193 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:35:32.193 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.193 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.193 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.193 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916087+0000", 2024-08-17T17:35:32.193 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.385e-06 2024-08-17T17:35:32.193 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.193 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.193 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.193 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916093+0000", 2024-08-17T17:35:32.193 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5529999999999998e-06 2024-08-17T17:35:32.194 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.194 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.194 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.194 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916099+0000", 2024-08-17T17:35:32.194 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0270000000000001e-06 2024-08-17T17:35:32.194 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.194 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.194 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.194 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916146+0000", 2024-08-17T17:35:32.194 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.71e-05 2024-08-17T17:35:32.194 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.194 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.194 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.195 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916150+0000", 2024-08-17T17:35:32.195 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8929999999999998e-06 2024-08-17T17:35:32.195 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.195 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.195 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.195 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.069721+0000", 2024-08-17T17:35:32.195 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.153570894 2024-08-17T17:35:32.195 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.195 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.195 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.195 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.069844+0000", 2024-08-17T17:35:32.195 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000122516 2024-08-17T17:35:32.195 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.195 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.196 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:35:32.196 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.069985+0000", 2024-08-17T17:35:32.196 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000141206 2024-08-17T17:35:32.196 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.196 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.196 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.196 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.072167+0000", 2024-08-17T17:35:32.196 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002181723 2024-08-17T17:35:32.196 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.196 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.196 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.196 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.077303+0000", 2024-08-17T17:35:32.196 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0051359719999999999 2024-08-17T17:35:32.196 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.197 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.197 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.197 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.077921+0000", 2024-08-17T17:35:32.197 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00061814300000000003 2024-08-17T17:35:32.197 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.197 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.197 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.197 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.077969+0000", 2024-08-17T17:35:32.197 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8789000000000003e-05 2024-08-17T17:35:32.197 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.197 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.197 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.197 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.078057+0000", 2024-08-17T17:35:32.198 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7106000000000003e-05 2024-08-17T17:35:32.198 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.198 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.198 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.198 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.198 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.198 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217549 10.b 10:d2f3afef:::benchmark_data_smithi063_294527_object80364:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1323)", 2024-08-17T17:35:32.198 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:25.916105+0000", 2024-08-17T17:35:32.198 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 126.1731928, 2024-08-17T17:35:32.198 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1687483860000001, 2024-08-17T17:35:32.198 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.198 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.199 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.199 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.199 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.199 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217549 2024-08-17T17:35:32.199 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.199 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.199 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.199 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.199 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916105+0000", 2024-08-17T17:35:32.199 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.199 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.199 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.199 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.200 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916105+0000", 2024-08-17T17:35:32.200 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.200 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.200 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.200 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.200 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916105+0000", 2024-08-17T17:35:32.200 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:35:32.200 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.200 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.200 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.200 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916107+0000", 2024-08-17T17:35:32.200 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.1739999999999999e-06 2024-08-17T17:35:32.200 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.201 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.201 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.201 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916113+0000", 2024-08-17T17:35:32.201 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9859999999999999e-06 2024-08-17T17:35:32.201 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.201 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.201 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.201 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916118+0000", 2024-08-17T17:35:32.201 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0379999999999999e-06 2024-08-17T17:35:32.201 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.201 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.201 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.201 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916178+0000", 2024-08-17T17:35:32.202 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0143999999999999e-05 2024-08-17T17:35:32.202 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.202 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.202 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.202 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916398+0000", 2024-08-17T17:35:32.202 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000219316 2024-08-17T17:35:32.202 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.202 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.202 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.202 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.072194+0000", 2024-08-17T17:35:32.202 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1557966150000001 2024-08-17T17:35:32.202 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.202 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.203 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.203 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.072336+0000", 2024-08-17T17:35:32.203 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000142235 2024-08-17T17:35:32.203 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.203 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.203 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:35:32.203 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.072473+0000", 2024-08-17T17:35:32.203 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013664699999999999 2024-08-17T17:35:32.203 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.203 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.203 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.203 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.077425+0000", 2024-08-17T17:35:32.204 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0049516910000000003 2024-08-17T17:35:32.204 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.204 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.204 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.204 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.077883+0000", 2024-08-17T17:35:32.204 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00045853000000000002 2024-08-17T17:35:32.204 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.204 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.204 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.204 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.084716+0000", 2024-08-17T17:35:32.204 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0068322829999999998 2024-08-17T17:35:32.204 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.204 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.205 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.205 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.084763+0000", 2024-08-17T17:35:32.205 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7531000000000003e-05 2024-08-17T17:35:32.205 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.205 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.205 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.205 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.084854+0000", 2024-08-17T17:35:32.205 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.0567000000000002e-05 2024-08-17T17:35:32.205 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.205 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.205 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.205 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.205 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.206 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217570 10.b 10:d3c7fda2:::benchmark_data_smithi063_294527_object80385:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1323)", 2024-08-17T17:35:32.206 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:25.916123+0000", 2024-08-17T17:35:32.206 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 126.173174692, 2024-08-17T17:35:32.206 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1689921160000001, 2024-08-17T17:35:32.206 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.206 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.206 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.206 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.206 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.206 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217570 2024-08-17T17:35:32.206 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.206 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.206 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.207 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.207 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916123+0000", 2024-08-17T17:35:32.207 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.207 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.207 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.207 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.207 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916123+0000", 2024-08-17T17:35:32.207 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.207 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.207 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.207 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.207 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916123+0000", 2024-08-17T17:35:32.207 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:35:32.208 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.208 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.208 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.208 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916125+0000", 2024-08-17T17:35:32.208 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9750000000000001e-06 2024-08-17T17:35:32.208 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.208 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.208 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.208 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916131+0000", 2024-08-17T17:35:32.208 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0050000000000001e-06 2024-08-17T17:35:32.208 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.208 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.208 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.209 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916135+0000", 2024-08-17T17:35:32.209 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5059999999999997e-06 2024-08-17T17:35:32.209 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.209 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.209 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.209 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916417+0000", 2024-08-17T17:35:32.209 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00028201300000000001 2024-08-17T17:35:32.209 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.209 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.209 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.209 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916421+0000", 2024-08-17T17:35:32.209 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3469999999999999e-06 2024-08-17T17:35:32.209 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.210 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.210 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.210 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.074796+0000", 2024-08-17T17:35:32.210 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1583748090000001 2024-08-17T17:35:32.210 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.210 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.210 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.210 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.074924+0000", 2024-08-17T17:35:32.210 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000128479 2024-08-17T17:35:32.210 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.210 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.210 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:35:32.211 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.075048+0000", 2024-08-17T17:35:32.211 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000124156 2024-08-17T17:35:32.211 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.211 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.211 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.211 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.077715+0000", 2024-08-17T17:35:32.211 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026663680000000001 2024-08-17T17:35:32.211 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.211 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.211 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.211 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.084905+0000", 2024-08-17T17:35:32.211 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0071902329999999999 2024-08-17T17:35:32.211 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.212 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.212 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.212 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.084940+0000", 2024-08-17T17:35:32.212 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5466000000000002e-05 2024-08-17T17:35:32.212 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.212 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.212 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.212 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.085030+0000", 2024-08-17T17:35:32.212 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9943000000000003e-05 2024-08-17T17:35:32.212 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.212 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.212 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.212 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.085115+0000", 2024-08-17T17:35:32.213 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.5167999999999995e-05 2024-08-17T17:35:32.213 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.213 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.213 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.213 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.213 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.213 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217580 10.b 10:d91d1f26:::benchmark_data_smithi063_294527_object80395:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1323)", 2024-08-17T17:35:32.213 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:25.916145+0000", 2024-08-17T17:35:32.213 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 126.173152751, 2024-08-17T17:35:32.213 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.169392089, 2024-08-17T17:35:32.213 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.213 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.213 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.214 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.214 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.214 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217580 2024-08-17T17:35:32.214 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.214 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.214 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.214 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.214 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916145+0000", 2024-08-17T17:35:32.214 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.214 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.214 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.214 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.215 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916145+0000", 2024-08-17T17:35:32.215 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.215 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.215 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.215 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.215 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916145+0000", 2024-08-17T17:35:32.215 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:35:32.215 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.215 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.215 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.215 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916147+0000", 2024-08-17T17:35:32.215 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6950000000000001e-06 2024-08-17T17:35:32.215 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.216 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.216 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.216 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916150+0000", 2024-08-17T17:35:32.216 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4060000000000001e-06 2024-08-17T17:35:32.216 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.216 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.216 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.216 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916154+0000", 2024-08-17T17:35:32.216 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6689999999999999e-06 2024-08-17T17:35:32.216 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.216 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.216 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.216 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916434+0000", 2024-08-17T17:35:32.217 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00028033000000000002 2024-08-17T17:35:32.217 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.217 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.217 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.217 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:25.916438+0000", 2024-08-17T17:35:32.217 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9009999999999996e-06 2024-08-17T17:35:32.217 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.217 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.217 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.217 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.078087+0000", 2024-08-17T17:35:32.217 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1616489839999999 2024-08-17T17:35:32.217 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.217 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.218 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.218 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.078207+0000", 2024-08-17T17:35:32.218 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000120013 2024-08-17T17:35:32.218 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.218 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.218 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:35:32.218 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.078349+0000", 2024-08-17T17:35:32.218 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014165400000000001 2024-08-17T17:35:32.218 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.218 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.218 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.218 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.084870+0000", 2024-08-17T17:35:32.218 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0065218630000000001 2024-08-17T17:35:32.219 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.219 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.219 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.219 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.085160+0000", 2024-08-17T17:35:32.219 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00028950200000000002 2024-08-17T17:35:32.219 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.219 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.219 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.219 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.085194+0000", 2024-08-17T17:35:32.219 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4192000000000001e-05 2024-08-17T17:35:32.219 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.219 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.219 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.220 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.085235+0000", 2024-08-17T17:35:32.220 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1242000000000003e-05 2024-08-17T17:35:32.220 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.220 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.220 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.220 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.085537+0000", 2024-08-17T17:35:32.220 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00030194500000000002 2024-08-17T17:35:32.220 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.220 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.220 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.220 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.220 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.220 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:217601 10.b 10:d7a18532:::benchmark_data_smithi063_294527_object80416:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1323)", 2024-08-17T17:35:32.221 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:26.090134+0000", 2024-08-17T17:35:32.221 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 125.99916438699999, 2024-08-17T17:35:32.221 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99560673700000002, 2024-08-17T17:35:32.221 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.221 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.221 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.221 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.221 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.221 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 217601 2024-08-17T17:35:32.221 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.221 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.221 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.221 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.222 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.090134+0000", 2024-08-17T17:35:32.222 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.222 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.222 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.222 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.222 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.090134+0000", 2024-08-17T17:35:32.222 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.222 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.222 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.222 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.222 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.090132+0000", 2024-08-17T17:35:32.222 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:35:32.223 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.223 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.223 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.223 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.090139+0000", 2024-08-17T17:35:32.223 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5130000000000002e-06 2024-08-17T17:35:32.223 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.223 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.223 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.223 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.090152+0000", 2024-08-17T17:35:32.223 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3750999999999999e-05 2024-08-17T17:35:32.223 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.223 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.223 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.224 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.090160+0000", 2024-08-17T17:35:32.224 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7449999999999995e-06 2024-08-17T17:35:32.224 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.224 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.224 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.224 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.090199+0000", 2024-08-17T17:35:32.224 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8906999999999999e-05 2024-08-17T17:35:32.224 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.224 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.224 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.224 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:26.090208+0000", 2024-08-17T17:35:32.224 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2280000000000004e-06 2024-08-17T17:35:32.224 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.225 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.225 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.225 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.080957+0000", 2024-08-17T17:35:32.225 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99074836700000002 2024-08-17T17:35:32.225 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.225 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.225 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.225 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.081079+0000", 2024-08-17T17:35:32.225 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000122254 2024-08-17T17:35:32.225 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.225 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.225 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:35:32.225 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.081207+0000", 2024-08-17T17:35:32.225 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000128155 2024-08-17T17:35:32.226 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.226 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.226 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.226 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.085581+0000", 2024-08-17T17:35:32.226 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0043737089999999999 2024-08-17T17:35:32.226 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.226 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.226 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.226 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.085588+0000", 2024-08-17T17:35:32.226 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3590000000000002e-06 2024-08-17T17:35:32.226 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.226 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.227 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.227 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.085621+0000", 2024-08-17T17:35:32.227 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3222000000000001e-05 2024-08-17T17:35:32.227 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.227 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.227 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.227 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.085660+0000", 2024-08-17T17:35:32.227 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8976000000000001e-05 2024-08-17T17:35:32.227 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.227 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.227 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.227 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:27.085740+0000", 2024-08-17T17:35:32.228 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0000000000000007e-05 2024-08-17T17:35:32.228 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.228 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.228 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.228 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.228 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.228 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:241949 10.0 10:03d27464:::benchmark_data_smithi063_294527_object104764:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-08-17T17:35:32.228 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:53.098314+0000", 2024-08-17T17:35:32.229 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 98.990983627000006, 2024-08-17T17:35:32.229 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0206755729999999, 2024-08-17T17:35:32.229 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.229 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.229 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.229 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.229 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.229 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 241949 2024-08-17T17:35:32.229 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.229 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.229 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.229 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.229 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:53.098314+0000", 2024-08-17T17:35:32.230 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.230 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.230 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.230 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.230 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:53.098314+0000", 2024-08-17T17:35:32.230 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.230 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.230 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.230 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.230 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:53.098314+0000", 2024-08-17T17:35:32.230 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:35:32.230 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.230 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.231 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.231 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:53.098317+0000", 2024-08-17T17:35:32.231 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5689999999999999e-06 2024-08-17T17:35:32.231 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.231 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.231 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.231 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:53.098324+0000", 2024-08-17T17:35:32.231 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6249999999999998e-06 2024-08-17T17:35:32.231 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.231 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.231 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.231 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:53.098330+0000", 2024-08-17T17:35:32.231 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2670000000000003e-06 2024-08-17T17:35:32.232 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.232 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.232 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.232 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:53.098378+0000", 2024-08-17T17:35:32.232 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7509999999999997e-05 2024-08-17T17:35:32.232 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.232 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.232 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.232 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:53.098388+0000", 2024-08-17T17:35:32.232 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0297e-05 2024-08-17T17:35:32.232 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.232 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.232 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.233 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:54.112409+0000", 2024-08-17T17:35:32.233 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0140209760000001 2024-08-17T17:35:32.233 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.233 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.233 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.233 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:54.112527+0000", 2024-08-17T17:35:32.233 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000117507 2024-08-17T17:35:32.233 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.233 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.233 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:35:32.233 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:54.112976+0000", 2024-08-17T17:35:32.233 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000449622 2024-08-17T17:35:32.233 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.234 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.234 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.234 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:54.118768+0000", 2024-08-17T17:35:32.234 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0057914949999999998 2024-08-17T17:35:32.234 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.234 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.234 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.234 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:54.118822+0000", 2024-08-17T17:35:32.234 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3687999999999998e-05 2024-08-17T17:35:32.234 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.234 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.234 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.234 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:54.118882+0000", 2024-08-17T17:35:32.235 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0708999999999998e-05 2024-08-17T17:35:32.235 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.235 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.235 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.235 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:54.118920+0000", 2024-08-17T17:35:32.235 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7549999999999998e-05 2024-08-17T17:35:32.235 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.235 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.235 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.235 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:54.118990+0000", 2024-08-17T17:35:32.235 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0158000000000002e-05 2024-08-17T17:35:32.235 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.235 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.236 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.236 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.236 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.236 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:246486 10.2 10:41361ec0:::benchmark_data_smithi063_294527_object109301:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1339)", 2024-08-17T17:35:32.236 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:59.132643+0000", 2024-08-17T17:35:32.236 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 92.956654576999995, 2024-08-17T17:35:32.236 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0007147709999999, 2024-08-17T17:35:32.236 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:32.236 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:35:32.236 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:35:32.236 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:35:32.236 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:35:32.236 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 246486 2024-08-17T17:35:32.236 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.237 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:32.237 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.237 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:32.237 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.132643+0000", 2024-08-17T17:35:32.237 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.237 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.237 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.237 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:32.237 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.132643+0000", 2024-08-17T17:35:32.237 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:32.237 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.237 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.237 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:32.237 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.132642+0000", 2024-08-17T17:35:32.238 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:35:32.238 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.238 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.238 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:32.238 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.132650+0000", 2024-08-17T17:35:32.238 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9980000000000003e-06 2024-08-17T17:35:32.238 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.238 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.238 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:32.238 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.132667+0000", 2024-08-17T17:35:32.238 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6569e-05 2024-08-17T17:35:32.238 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.238 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.239 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:32.239 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.132675+0000", 2024-08-17T17:35:32.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4470000000000006e-06 2024-08-17T17:35:32.239 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.239 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.239 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.239 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.132719+0000", 2024-08-17T17:35:32.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4094999999999997e-05 2024-08-17T17:35:32.239 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.239 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.239 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:35:32.239 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.132725+0000", 2024-08-17T17:35:32.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6579999999999999e-06 2024-08-17T17:35:32.239 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.240 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.240 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:32.240 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.127277+0000", 2024-08-17T17:35:32.240 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99455265999999998 2024-08-17T17:35:32.240 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.240 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.240 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:32.240 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.127411+0000", 2024-08-17T17:35:32.240 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000134027 2024-08-17T17:35:32.240 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.240 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.240 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,1", 2024-08-17T17:35:32.240 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.127824+0000", 2024-08-17T17:35:32.241 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00041265799999999999 2024-08-17T17:35:32.241 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.241 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.241 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.241 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.133269+0000", 2024-08-17T17:35:32.241 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0054446269999999996 2024-08-17T17:35:32.241 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.241 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.241 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:35:32.241 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.133280+0000", 2024-08-17T17:35:32.241 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0890000000000001e-05 2024-08-17T17:35:32.241 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.241 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.241 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:35:32.242 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.133305+0000", 2024-08-17T17:35:32.242 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5533000000000001e-05 2024-08-17T17:35:32.242 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.242 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.242 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:35:32.242 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.133334+0000", 2024-08-17T17:35:32.242 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9195000000000001e-05 2024-08-17T17:35:32.242 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:32.242 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:32.242 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:35:32.242 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.133358+0000", 2024-08-17T17:35:32.242 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3864000000000001e-05 2024-08-17T17:35:32.243 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.243 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.243 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.243 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:32.243 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:32.243 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:35:32.243 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:32.266 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:32.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:32.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:31 smithi063 ceph-mon[114756]: pgmap v2077: 119 pgs: 9 active+undersized+remapped, 33 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%) 2024-08-17T17:35:32.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:31 smithi063 ceph-mon[114756]: from='osd.7 [v2:172.21.15.162:6816/3585957098,v1:172.21.15.162:6817/3585957098]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:35:32.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:31 smithi063 ceph-mon[114756]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:35:32.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:32.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:32.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:31 smithi063 ceph-mon[109843]: pgmap v2077: 119 pgs: 9 active+undersized+remapped, 33 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%) 2024-08-17T17:35:32.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:31 smithi063 ceph-mon[109843]: from='osd.7 [v2:172.21.15.162:6816/3585957098,v1:172.21.15.162:6817/3585957098]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:35:32.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:31 smithi063 ceph-mon[109843]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-08-17T17:35:32.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:32.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:32.367 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:32.390 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:32.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:32.480 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:35:32.492 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:32.514 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:32.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:32.615 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:32.637 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:32.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:32.663 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 32 16 67 51 0.0995982 0 - 0.895292 2024-08-17T17:35:32.739 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:32.762 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:32.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:32.864 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:32.886 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:32.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:32.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:32.988 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:33.010 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:33.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:33.019 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:35:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:32 smithi063 ceph-mon[109843]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:35:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:32 smithi063 ceph-mon[109843]: from='osd.7 [v2:172.21.15.162:6816/3585957098,v1:172.21.15.162:6817/3585957098]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi162", "root=default"]} : dispatch 2024-08-17T17:35:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:32 smithi063 ceph-mon[109843]: osdmap e1370: 8 total, 6 up, 8 in 2024-08-17T17:35:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:32 smithi063 ceph-mon[109843]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi162", "root=default"]} : dispatch 2024-08-17T17:35:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:32 smithi063 ceph-mon[114756]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:35:33.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:32 smithi063 ceph-mon[114756]: from='osd.7 [v2:172.21.15.162:6816/3585957098,v1:172.21.15.162:6817/3585957098]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi162", "root=default"]} : dispatch 2024-08-17T17:35:33.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:32 smithi063 ceph-mon[114756]: osdmap e1370: 8 total, 6 up, 8 in 2024-08-17T17:35:33.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:32 smithi063 ceph-mon[114756]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi162", "root=default"]} : dispatch 2024-08-17T17:35:33.113 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:33.119 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:33.152 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:33.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:33.254 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:33.264 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:32 smithi162 ceph-mon[104668]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-08-17T17:35:33.264 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:32 smithi162 ceph-mon[104668]: from='osd.7 [v2:172.21.15.162:6816/3585957098,v1:172.21.15.162:6817/3585957098]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi162", "root=default"]} : dispatch 2024-08-17T17:35:33.264 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:32 smithi162 ceph-mon[104668]: osdmap e1370: 8 total, 6 up, 8 in 2024-08-17T17:35:33.264 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:32 smithi162 ceph-mon[104668]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi162", "root=default"]} : dispatch 2024-08-17T17:35:33.275 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:33.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:33.371 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:35:33.377 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:33.394 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:35:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:35:33] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.51.0" 2024-08-17T17:35:33.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.417 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:33.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:33.518 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:33.540 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:33.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:33.583 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:33.642 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:33.666 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 33 16 131 115 0.217779 0.235294 0.00123446 4.52324 2024-08-17T17:35:33.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.667 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:33.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:33.768 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:33.790 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:33.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:33.892 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:33.914 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:33.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:33.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:33.957 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:33 smithi063 ceph-mon[114756]: pgmap v2079: 119 pgs: 9 active+undersized+remapped, 33 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%) 2024-08-17T17:35:33.957 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:33 smithi063 ceph-mon[114756]: Health check update: 1 osds down (OSD_DOWN) 2024-08-17T17:35:33.957 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:33 smithi063 ceph-mon[114756]: osd.7 [v2:172.21.15.162:6816/3585957098,v1:172.21.15.162:6817/3585957098] boot 2024-08-17T17:35:33.957 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:33 smithi063 ceph-mon[114756]: osdmap e1371: 8 total, 7 up, 8 in 2024-08-17T17:35:33.957 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:33 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:35:33.957 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:33 smithi063 ceph-mon[114756]: osdmap e1372: 8 total, 7 up, 8 in 2024-08-17T17:35:33.957 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:33 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2385676429' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:35:33.958 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:33 smithi063 ceph-mon[109843]: pgmap v2079: 119 pgs: 9 active+undersized+remapped, 33 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%) 2024-08-17T17:35:33.958 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:33 smithi063 ceph-mon[109843]: Health check update: 1 osds down (OSD_DOWN) 2024-08-17T17:35:33.958 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:33 smithi063 ceph-mon[109843]: osd.7 [v2:172.21.15.162:6816/3585957098,v1:172.21.15.162:6817/3585957098] boot 2024-08-17T17:35:33.958 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:33 smithi063 ceph-mon[109843]: osdmap e1371: 8 total, 7 up, 8 in 2024-08-17T17:35:33.958 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:33 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:35:33.958 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:33 smithi063 ceph-mon[109843]: osdmap e1372: 8 total, 7 up, 8 in 2024-08-17T17:35:33.958 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:33 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2385676429' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:35:34.016 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:34.038 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:34.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:34.119 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:35:34.141 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:34.170 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:34.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:34.271 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:34.280 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:33 smithi162 ceph-mon[104668]: pgmap v2079: 119 pgs: 9 active+undersized+remapped, 33 active+clean+remapped, 9 undersized+peered, 30 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 26 active+clean; 3.9 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 33/234 objects degraded (14.103%); 33/234 objects misplaced (14.103%) 2024-08-17T17:35:34.280 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:33 smithi162 ceph-mon[104668]: Health check update: 1 osds down (OSD_DOWN) 2024-08-17T17:35:34.280 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:33 smithi162 ceph-mon[104668]: osd.7 [v2:172.21.15.162:6816/3585957098,v1:172.21.15.162:6817/3585957098] boot 2024-08-17T17:35:34.280 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:33 smithi162 ceph-mon[104668]: osdmap e1371: 8 total, 7 up, 8 in 2024-08-17T17:35:34.280 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:33 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-08-17T17:35:34.280 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:33 smithi162 ceph-mon[104668]: osdmap e1372: 8 total, 7 up, 8 in 2024-08-17T17:35:34.280 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:33 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2385676429' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:35:34.304 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:34.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:34.393 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:34.405 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:34.448 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:34.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:34.550 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:34.590 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:34.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:34.663 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 34 16 2141 2125 3.90582 125.625 0.00346199 0.256006 2024-08-17T17:35:34.692 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:34.730 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:34.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:34.832 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:34.871 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:34.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:34.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:34.973 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:35.013 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:35.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:35.114 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:35.153 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:35.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:35.186 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:35:35.186 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:35:35.186 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 0 2024-08-17T17:35:35.186 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:35:35.186 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:34 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:35.186 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:34 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2385676429' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:35:35.187 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:34 smithi162 ceph-mon[104668]: osdmap e1373: 8 total, 7 up, 8 in 2024-08-17T17:35:35.255 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:35.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:34 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:35.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:34 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2385676429' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:35:35.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:34 smithi063 ceph-mon[109843]: osdmap e1373: 8 total, 7 up, 8 in 2024-08-17T17:35:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:34 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:34 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2385676429' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:35:35.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:34 smithi063 ceph-mon[114756]: osdmap e1373: 8 total, 7 up, 8 in 2024-08-17T17:35:35.295 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:35.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:35.397 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:35.428 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:35.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:35.530 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:35.565 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:35.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:35.663 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 35 16 5681 5665 10.115 221.25 0.00325713 0.0988527 2024-08-17T17:35:35.666 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:35.698 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:35.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.699 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:35.800 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:35.838 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:35.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:35.940 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:35.981 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:35.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:35.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:36.084 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:36.122 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:36.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:36.164 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:35 smithi063 ceph-mon[114756]: pgmap v2083: 119 pgs: 1 remapped+peering, 8 active+undersized+remapped, 33 active+clean+remapped, 8 undersized+peered, 29 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 28 active+clean; 4.7 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 208 KiB/s wr, 3 op/s; 41/273 objects degraded (15.018%); 33/273 objects misplaced (12.088%); 48 KiB/s, 0 objects/s recovering 2024-08-17T17:35:36.165 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:35 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 41/273 objects degraded (15.018%), 12 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:36.165 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:35 smithi063 ceph-mon[109843]: pgmap v2083: 119 pgs: 1 remapped+peering, 8 active+undersized+remapped, 33 active+clean+remapped, 8 undersized+peered, 29 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 28 active+clean; 4.7 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 208 KiB/s wr, 3 op/s; 41/273 objects degraded (15.018%); 33/273 objects misplaced (12.088%); 48 KiB/s, 0 objects/s recovering 2024-08-17T17:35:36.165 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:35 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 41/273 objects degraded (15.018%), 12 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:36.225 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:36.263 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:36.294 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:35 smithi162 ceph-mon[104668]: pgmap v2083: 119 pgs: 1 remapped+peering, 8 active+undersized+remapped, 33 active+clean+remapped, 8 undersized+peered, 29 active+undersized, 2 undersized+degraded+peered, 10 active+undersized+degraded, 28 active+clean; 4.7 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 208 KiB/s wr, 3 op/s; 41/273 objects degraded (15.018%); 33/273 objects misplaced (12.088%); 48 KiB/s, 0 objects/s recovering 2024-08-17T17:35:36.295 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:35 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 41/273 objects degraded (15.018%), 12 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:36.324 INFO:tasks.thrashosds.thrasher:Reviving osd 6 2024-08-17T17:35:36.325 INFO:tasks.cephadm.osd.6:Restarting osd.6 (starting--it wasn't running)... 2024-08-17T17:35:36.325 DEBUG:teuthology.orchestra.run.smithi162:osd.6> sudo journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.6.service 2024-08-17T17:35:36.327 DEBUG:teuthology.orchestra.run.smithi162:> sudo systemctl start ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.6 2024-08-17T17:35:36.364 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:36.398 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:36.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:36.500 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:36.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.546 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:36.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:36.596 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:36 smithi162 systemd[1]: Starting Ceph osd.6 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:35:36.646 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:36.663 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 36 16 8954 8938 15.5157 204.562 0.00591786 0.0644431 2024-08-17T17:35:36.673 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:35:36.673 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:35:36.677 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:36.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:36.711 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:35:36.711 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:35:36.748 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:35:36.749 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:35:36.779 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:36.790 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:35:36.791 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:35:36.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.818 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:36.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:36.820 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:35:36.820 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:35:36.855 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:35:36.855 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:35:36.856 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:35:36.894 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:35:36.894 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_enable_op_tracker = '' osd_max_backfills = '' osd_pg_delete_cost = '' (not observed, change may require restart) osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' 2024-08-17T17:35:36.919 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:36.956 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:36.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:36.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:36.976 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:36 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3828866101' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:37.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:36 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3828866101' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:37.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:36 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3828866101' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:37.058 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:37.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.092 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:37.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:37.193 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:37.228 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:37.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:37.330 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:37.368 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:37.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:37.469 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:37.500 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:35:37.500 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:35:37.506 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:37.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:37.610 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:37.640 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:37.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:37.663 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 37 16 12099 12083 20.4082 196.562 0.00564691 0.0489934 2024-08-17T17:35:37.711 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:35:37.712 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:35:37.742 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:37.774 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:37.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:37.876 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:37.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:37.908 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:37.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:38.009 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:38.026 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:35:38.050 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:38.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.051 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:38.152 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:38.177 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:37 smithi162 ceph-mon[104668]: pgmap v2084: 119 pgs: 1 remapped+peering, 8 active+undersized+remapped, 34 active+clean+remapped, 7 undersized+peered, 28 active+undersized, 2 undersized+degraded+peered, 9 active+undersized+degraded, 30 active+clean; 4.7 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 177 KiB/s wr, 2 op/s; 39/273 objects degraded (14.286%); 33/273 objects misplaced (12.088%); 41 KiB/s, 0 objects/s recovering 2024-08-17T17:35:38.183 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:38.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:38.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:37 smithi063 ceph-mon[114756]: pgmap v2084: 119 pgs: 1 remapped+peering, 8 active+undersized+remapped, 34 active+clean+remapped, 7 undersized+peered, 28 active+undersized, 2 undersized+degraded+peered, 9 active+undersized+degraded, 30 active+clean; 4.7 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 177 KiB/s wr, 2 op/s; 39/273 objects degraded (14.286%); 33/273 objects misplaced (12.088%); 41 KiB/s, 0 objects/s recovering 2024-08-17T17:35:38.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:37 smithi063 ceph-mon[109843]: pgmap v2084: 119 pgs: 1 remapped+peering, 8 active+undersized+remapped, 34 active+clean+remapped, 7 undersized+peered, 28 active+undersized, 2 undersized+degraded+peered, 9 active+undersized+degraded, 30 active+clean; 4.7 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 177 KiB/s wr, 2 op/s; 39/273 objects degraded (14.286%); 33/273 objects misplaced (12.088%); 41 KiB/s, 0 objects/s recovering 2024-08-17T17:35:38.285 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:38.326 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:38.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:38.428 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:38.441 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:38.457 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:38.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:38.559 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:38.561 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:35:38.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.635 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:38.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:38.663 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 38 16 15028 15012 24.6881 183.062 0.00328435 0.0404971 2024-08-17T17:35:38.736 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:38.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.779 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:38.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:38.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:38 smithi162 podman[228389]: 2024-08-17 17:35:38.568777788 +0000 UTC m=+1.062607738 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:35:38.880 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:38.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:38.917 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:38.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:39.018 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:39.057 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:39.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:39.159 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:39.188 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:39.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:39.291 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:39.331 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:39.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:39.345 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 podman[228389]: 2024-08-17 17:35:39.075878131 +0000 UTC m=+1.569708075 container create 605fb581df8431fda5fcb2e72f2224b76ff21ad64a8867bd7a33190353ce9c69 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-activate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-08-17T17:35:39.433 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:39.455 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:39.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:39.556 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:39.568 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:39.593 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:39.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:39.663 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 39 16 17471 17455 27.9697 152.688 0.00444154 0.0357458 2024-08-17T17:35:39.695 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:39.703 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 podman[228389]: 2024-08-17 17:35:39.37912128 +0000 UTC m=+1.872951235 container init 605fb581df8431fda5fcb2e72f2224b76ff21ad64a8867bd7a33190353ce9c69 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-activate, io.buildah.version=1.36.0, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-08-17T17:35:39.703 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 podman[228389]: 2024-08-17 17:35:39.384353694 +0000 UTC m=+1.878183638 container start 605fb581df8431fda5fcb2e72f2224b76ff21ad64a8867bd7a33190353ce9c69 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-activate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:35:39.703 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 podman[228389]: 2024-08-17 17:35:39.462227302 +0000 UTC m=+1.956057254 container attach 605fb581df8431fda5fcb2e72f2224b76ff21ad64a8867bd7a33190353ce9c69 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-activate, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0) 2024-08-17T17:35:39.712 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:35:39.725 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:39.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:39.827 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:39.874 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:39.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:39.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:39.976 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:40.007 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:40.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:40.073 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:39 smithi162 ceph-mon[104668]: pgmap v2085: 119 pgs: 1 remapped+peering, 39 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 642 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 2764/30864 objects degraded (8.955%); 2980/30864 objects misplaced (9.655%); 153 KiB/s, 2 objects/s recovering 2024-08-17T17:35:40.074 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-activate[228572]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-08-17T17:35:40.074 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 bash[228389]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-08-17T17:35:40.074 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-activate[228572]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-08-17T17:35:40.074 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 bash[228389]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-08-17T17:35:40.074 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-activate[228572]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-08-17T17:35:40.074 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 bash[228389]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-08-17T17:35:40.074 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-activate[228572]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T17:35:40.074 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 bash[228389]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-08-17T17:35:40.074 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-activate[228572]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-08-17T17:35:40.074 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 bash[228389]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-08-17T17:35:40.075 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-activate[228572]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-08-17T17:35:40.075 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 bash[228389]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-08-17T17:35:40.075 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-activate[228572]: --> ceph-volume raw activate successful for osd ID: 6 2024-08-17T17:35:40.075 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:39 smithi162 bash[228389]: --> ceph-volume raw activate successful for osd ID: 6 2024-08-17T17:35:40.108 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:40.152 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:40.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:40.254 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:40.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:39 smithi063 ceph-mon[109843]: pgmap v2085: 119 pgs: 1 remapped+peering, 39 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 642 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 2764/30864 objects degraded (8.955%); 2980/30864 objects misplaced (9.655%); 153 KiB/s, 2 objects/s recovering 2024-08-17T17:35:40.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:39 smithi063 ceph-mon[114756]: pgmap v2085: 119 pgs: 1 remapped+peering, 39 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 642 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 2764/30864 objects degraded (8.955%); 2980/30864 objects misplaced (9.655%); 153 KiB/s, 2 objects/s recovering 2024-08-17T17:35:40.284 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:40.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:40.345 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:40 smithi162 podman[228389]: 2024-08-17 17:35:40.20187762 +0000 UTC m=+2.695707569 container died 605fb581df8431fda5fcb2e72f2224b76ff21ad64a8867bd7a33190353ce9c69 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-activate, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-ced649f, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-08-17T17:35:40.386 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:40.415 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:40.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:40.508 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:40.517 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:40.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.565 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:40.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:40.663 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:35:40.663801+0000 min lat: 0.000948653 max lat: 32.4964 avg lat: 0.0320429 2024-08-17T17:35:40.664 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:35:40.664 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 40 16 19988 19972 31.2029 157.312 0.00499123 0.0320429 2024-08-17T17:35:40.666 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:40.697 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:40.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:40.798 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:40.827 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:40.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:40.930 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:40.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:40.970 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:40.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:41.071 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:41.098 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:41.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:41.200 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:41.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.221 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:41.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:41.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:40 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 3707/36684 objects degraded (10.105%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:41.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:40 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 3707/36684 objects degraded (10.105%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:41.290 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:40 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 3707/36684 objects degraded (10.105%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:41.323 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:41.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.360 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:41.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:41.461 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:41.499 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:41.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:41.595 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:41 smithi162 podman[228785]: 2024-08-17 17:35:41.289885602 +0000 UTC m=+1.408950010 container remove 605fb581df8431fda5fcb2e72f2224b76ff21ad64a8867bd7a33190353ce9c69 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True) 2024-08-17T17:35:41.601 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:41.641 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:41.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:41.663 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 41 16 22382 22366 34.0908 149.625 0.0113447 0.0293274 2024-08-17T17:35:41.743 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:41.782 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:41.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:41.884 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:41.913 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:41.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:41.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:42.002 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:35:42.015 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:42.043 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:42.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:42.145 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:42.185 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:42.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:42.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:42 smithi063 ceph-mon[109843]: pgmap v2086: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 763 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 3707/36684 objects degraded (10.105%); 2980/36684 objects misplaced (8.123%); 120 KiB/s, 1 objects/s recovering 2024-08-17T17:35:42.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:42 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2591388208' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:42.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:42 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:42.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:42 smithi063 ceph-mon[114756]: pgmap v2086: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 763 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 3707/36684 objects degraded (10.105%); 2980/36684 objects misplaced (8.123%); 120 KiB/s, 1 objects/s recovering 2024-08-17T17:35:42.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:42 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2591388208' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:42.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:42 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:42.287 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:42.316 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:42 smithi162 ceph-mon[104668]: pgmap v2086: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 763 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 3707/36684 objects degraded (10.105%); 2980/36684 objects misplaced (8.123%); 120 KiB/s, 1 objects/s recovering 2024-08-17T17:35:42.316 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:42 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2591388208' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:42.316 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:42 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:35:42.322 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:42.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:42.367 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:35:42.368 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:35:42.424 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:42.461 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:42.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:42.563 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:42.596 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:42 smithi162 podman[229079]: 2024-08-17 17:35:42.357301022 +0000 UTC m=+0.258026182 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:35:42.596 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:42 smithi162 podman[229079]: 2024-08-17 17:35:42.490984956 +0000 UTC m=+0.391710098 container create d6247a6fefe8228f90acf6fd9e143c7cdcd057b04d0827f8bba50570ea2198a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-08-17T17:35:42.600 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:42.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:42.664 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 42 16 24793 24777 36.8666 150.688 0.00407221 0.0271125 2024-08-17T17:35:42.701 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:42.731 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:42.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:42.833 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:42.870 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:42.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:42.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:42.972 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:43.004 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:43.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:43.077 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:35:43.107 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:43.145 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:43.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:43.247 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:43.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:43 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:43.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:43 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:43.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:43 smithi063 ceph-mon[114756]: osdmap e1374: 8 total, 7 up, 8 in 2024-08-17T17:35:43.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:43 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:43.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:43 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:43.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:43 smithi063 ceph-mon[109843]: osdmap e1374: 8 total, 7 up, 8 in 2024-08-17T17:35:43.290 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:43.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:43.346 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:42 smithi162 podman[229079]: 2024-08-17 17:35:42.962244901 +0000 UTC m=+0.862970063 container init d6247a6fefe8228f90acf6fd9e143c7cdcd057b04d0827f8bba50570ea2198a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6, GIT_CLEAN=True, io.buildah.version=1.36.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-08-17T17:35:43.346 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:42 smithi162 podman[229079]: 2024-08-17 17:35:42.969823522 +0000 UTC m=+0.870548670 container start d6247a6fefe8228f90acf6fd9e143c7cdcd057b04d0827f8bba50570ea2198a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, RELEASE=reef-ced649f) 2024-08-17T17:35:43.346 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:43 smithi162 bash[229079]: d6247a6fefe8228f90acf6fd9e143c7cdcd057b04d0827f8bba50570ea2198a4 2024-08-17T17:35:43.346 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:43 smithi162 systemd[1]: Started Ceph osd.6 for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:35:43.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:43 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:43.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:43 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:35:43.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:43 smithi162 ceph-mon[104668]: osdmap e1374: 8 total, 7 up, 8 in 2024-08-17T17:35:43.393 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:43.434 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:43.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:43.536 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:43.571 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:43.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:43.664 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 43 16 26932 26916 39.1179 133.688 0.00414644 0.0255376 2024-08-17T17:35:43.673 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:43.711 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:43.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:43.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:35:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:35:43] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.51.0" 2024-08-17T17:35:43.812 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:43.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:43 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:35:43.535+0000 7fabeabad640 -1 Falling back to public interface 2024-08-17T17:35:43.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.851 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:43.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:43.952 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:43.973 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:43.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:43.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:44.075 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:44.096 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:44.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:44.198 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:44.234 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:44.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:44.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:44 smithi063 ceph-mon[114756]: pgmap v2088: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 1.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s; 6943/53583 objects degraded (12.957%); 3827/53583 objects misplaced (7.142%); 86 KiB/s, 1 objects/s recovering 2024-08-17T17:35:44.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:44 smithi063 ceph-mon[109843]: pgmap v2088: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 1.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s; 6943/53583 objects degraded (12.957%); 3827/53583 objects misplaced (7.142%); 86 KiB/s, 1 objects/s recovering 2024-08-17T17:35:44.325 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:44 smithi162 ceph-mon[104668]: pgmap v2088: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 1.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 131 MiB/s wr, 2.10k op/s; 6943/53583 objects degraded (12.957%); 3827/53583 objects misplaced (7.142%); 86 KiB/s, 1 objects/s recovering 2024-08-17T17:35:44.337 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:44.364 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:35:44.370 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:44.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:44.472 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:44.497 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:44.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:44.505 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:44.598 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:44.636 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:44.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:44.664 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 44 16 28654 28638 40.6746 107.625 0.00717307 0.0245801 2024-08-17T17:35:44.742 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:44.768 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:44.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:44.870 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:44.909 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:44.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:44.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:45.011 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:45.033 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:45.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:45.135 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:45.169 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:45.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:45.271 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:45.300 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:45.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.301 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:45.402 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:45.423 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:45.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:45.525 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:45.560 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:45.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:45.661 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:45.664 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 45 16 30345 30329 42.1191 105.688 0.00226918 0.0237255 2024-08-17T17:35:45.688 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:45.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:45.790 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:45.816 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:45.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:45.917 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:45.954 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:45.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:45.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:46.056 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:46.085 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:46.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:46.187 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:46.218 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:46.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:46.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:46 smithi063 ceph-mon[109843]: pgmap v2089: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 155 MiB/s wr, 2.48k op/s; 7711/74622 objects degraded (10.333%); 6216/74622 objects misplaced (8.330%); 73 KiB/s, 1 objects/s recovering 2024-08-17T17:35:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:46 smithi063 ceph-mon[114756]: pgmap v2089: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 155 MiB/s wr, 2.48k op/s; 7711/74622 objects degraded (10.333%); 6216/74622 objects misplaced (8.330%); 73 KiB/s, 1 objects/s recovering 2024-08-17T17:35:46.321 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:46.342 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:46.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:46.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:46 smithi162 ceph-mon[104668]: pgmap v2089: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 155 MiB/s wr, 2.48k op/s; 7711/74622 objects degraded (10.333%); 6216/74622 objects misplaced (8.330%); 73 KiB/s, 1 objects/s recovering 2024-08-17T17:35:46.444 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:46.482 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:46.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:46.584 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:46.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.627 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:46.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:46.664 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 46 16 32130 32114 43.6285 111.562 0.00216458 0.0229094 2024-08-17T17:35:46.729 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:46.759 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:46.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:46.861 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:46.883 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:46.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:46.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:46.985 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:47.024 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:47.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:47.126 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:47.157 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:47.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:47.200 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:47.259 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:47.282 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:47.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:47.383 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:47.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.420 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:47.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:47.521 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:47.554 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:47.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:47.656 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:47.661 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:47.661 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:47.661 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:47 smithi162 ceph-mon[104668]: pgmap v2090: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 155 MiB/s wr, 2.48k op/s; 7711/74622 objects degraded (10.333%); 6216/74622 objects misplaced (8.330%); 73 KiB/s, 1 objects/s recovering 2024-08-17T17:35:47.664 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 47 16 33803 33787 44.9247 104.562 0.00207478 0.022225 2024-08-17T17:35:47.678 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:47.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:47.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:47.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:47.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:47 smithi063 ceph-mon[114756]: pgmap v2090: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 155 MiB/s wr, 2.48k op/s; 7711/74622 objects degraded (10.333%); 6216/74622 objects misplaced (8.330%); 73 KiB/s, 1 objects/s recovering 2024-08-17T17:35:47.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:35:47.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:35:47.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:47 smithi063 ceph-mon[109843]: pgmap v2090: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 155 MiB/s wr, 2.48k op/s; 7711/74622 objects degraded (10.333%); 6216/74622 objects misplaced (8.330%); 73 KiB/s, 1 objects/s recovering 2024-08-17T17:35:47.781 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:47.817 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:47.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:47.919 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:47.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:47.956 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:47.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:48.057 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:48.094 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:48.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:48.196 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:48.241 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:48.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:48.343 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:48.382 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:48.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:48.484 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:48.519 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:48.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:48.552 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:48 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 7711/74622 objects degraded (10.333%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:48.589 INFO:teuthology.orchestra.run.smithi162.stderr:no valid command found; 10 closest matches: 2024-08-17T17:35:48.589 INFO:teuthology.orchestra.run.smithi162.stderr:0 2024-08-17T17:35:48.589 INFO:teuthology.orchestra.run.smithi162.stderr:1 2024-08-17T17:35:48.589 INFO:teuthology.orchestra.run.smithi162.stderr:2 2024-08-17T17:35:48.589 INFO:teuthology.orchestra.run.smithi162.stderr:abort 2024-08-17T17:35:48.589 INFO:teuthology.orchestra.run.smithi162.stderr:assert 2024-08-17T17:35:48.589 INFO:teuthology.orchestra.run.smithi162.stderr:bluefs debug_inject_read_zeros 2024-08-17T17:35:48.589 INFO:teuthology.orchestra.run.smithi162.stderr:bluefs files list 2024-08-17T17:35:48.589 INFO:teuthology.orchestra.run.smithi162.stderr:bluefs stats 2024-08-17T17:35:48.589 INFO:teuthology.orchestra.run.smithi162.stderr:bluestore bluefs device info [] 2024-08-17T17:35:48.589 INFO:teuthology.orchestra.run.smithi162.stderr:config diff 2024-08-17T17:35:48.590 INFO:teuthology.orchestra.run.smithi162.stderr:admin_socket: invalid command 2024-08-17T17:35:48.621 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:48.651 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:48.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:48.664 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 48 16 35828 35812 46.6252 126.562 0.0107303 0.0214422 2024-08-17T17:35:48.754 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:48.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:48 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 7711/74622 objects degraded (10.333%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:48.781 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:48 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 7711/74622 objects degraded (10.333%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:48.793 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:48.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:48.895 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:48.918 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:48.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:48.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:49.020 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:49.060 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:49.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:49.161 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:49.201 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:49.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:49.302 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:49.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.343 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:49.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:49.444 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:49.481 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:49.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:49.583 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:49.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:49 smithi162 ceph-mon[104668]: pgmap v2091: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s; 10121/98535 objects degraded (10.271%); 8451/98535 objects misplaced (8.577%) 2024-08-17T17:35:49.618 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:49.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:49.664 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 49 16 37853 37837 48.2564 126.562 0.00556873 0.0207169 2024-08-17T17:35:49.720 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:49.742 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:49.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:49.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:49 smithi063 ceph-mon[109843]: pgmap v2091: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s; 10121/98535 objects degraded (10.271%); 8451/98535 objects misplaced (8.577%) 2024-08-17T17:35:49.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:49 smithi063 ceph-mon[114756]: pgmap v2091: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 141 MiB/s wr, 2.25k op/s; 10121/98535 objects degraded (10.271%); 8451/98535 objects misplaced (8.577%) 2024-08-17T17:35:49.843 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:49.875 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:49.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:49.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:49.976 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:50.014 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:50.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:50.116 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:50.154 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:50.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:50.256 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:50.293 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:50.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:50.398 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:50.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.437 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:50.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:50.538 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:50.566 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:50.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:50.579 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:35:50.580 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:35:50.664 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 50 16 39665 39649 49.556 113.25 0.0033927 0.0201738 2024-08-17T17:35:50.668 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:50.709 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:50.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:50.811 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:50.847 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:50.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:50.949 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:50.988 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:50.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:50.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:51.089 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:51.118 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:51.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:51.220 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:51.255 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:51.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:51.337 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:35:51.357 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:51.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.405 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:51.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:51.506 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:51.548 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:51.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:51.650 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:51.665 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 51 16 41570 41554 50.9182 119.062 0.0106344 0.0196323 2024-08-17T17:35:51.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.692 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:51.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:51.751 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:35:51.751 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:35:51.789 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:35:51.789 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:35:51.793 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:51.823 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:51 smithi063 ceph-mon[109843]: pgmap v2092: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s; 10660/101883 objects degraded (10.463%); 8451/101883 objects misplaced (8.295%) 2024-08-17T17:35:51.823 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:51 smithi063 ceph-mon[114756]: pgmap v2092: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s; 10660/101883 objects degraded (10.463%); 8451/101883 objects misplaced (8.295%) 2024-08-17T17:35:51.824 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:35:51.824 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:35:51.833 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:51.834 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-08-17T17:35:51.834 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-6, ['dump_ops_in_flight'] 2024-08-17T17:35:51.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:51.855 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:35:51.855 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:35:51.889 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:35:51.890 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:35:51.899 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:51.923 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:35:51.924 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:35:51.925 INFO:teuthology.orchestra.run.smithi063.stderr:Error ENXIO: problem getting command descriptions from osd.6 2024-08-17T17:35:51.936 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:51.959 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:35:51.959 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:35:51.978 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:51.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:51.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:52.030 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:51 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:35:51.659+0000 7fabeabad640 -1 osd.6 1273 log_to_monitors true 2024-08-17T17:35:52.031 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:51 smithi162 ceph-mon[104668]: pgmap v2092: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 136 MiB/s wr, 2.17k op/s; 10660/101883 objects degraded (10.463%); 8451/101883 objects misplaced (8.295%) 2024-08-17T17:35:52.080 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:52.111 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:52.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:52.213 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:52.245 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:52.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:52.346 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:52.376 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:52.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:52.478 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:52.516 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:52.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:52.618 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:52.661 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:52.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:52.665 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 52 16 43446 43430 52.1936 117.25 0.0100095 0.0191539 2024-08-17T17:35:52.764 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:52.769 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-08-17T17:35:52.770 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-08-17T17:35:52.785 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:52.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:52.887 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:52.904 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:35:52.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:52.914 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:52.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:52.935 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:52 smithi063 ceph-mon[114756]: from='osd.6 [v2:172.21.15.162:6824/1307501458,v1:172.21.15.162:6825/1307501458]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-08-17T17:35:52.935 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:52 smithi063 ceph-mon[114756]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-08-17T17:35:52.935 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:52 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/180521517' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:52.936 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:52 smithi063 ceph-mon[109843]: from='osd.6 [v2:172.21.15.162:6824/1307501458,v1:172.21.15.162:6825/1307501458]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-08-17T17:35:52.936 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:52 smithi063 ceph-mon[109843]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-08-17T17:35:52.936 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:52 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/180521517' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:53.015 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:53.039 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:53.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:52 smithi162 ceph-mon[104668]: from='osd.6 [v2:172.21.15.162:6824/1307501458,v1:172.21.15.162:6825/1307501458]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-08-17T17:35:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:52 smithi162 ceph-mon[104668]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-08-17T17:35:53.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:52 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/180521517' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:53.141 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:53.190 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:53.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:53.229 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:53.292 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:53.331 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:53.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:53.433 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:53.454 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:35:53 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:35:53.332+0000 7fabde94f640 -1 osd.6 1375 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-08-17T17:35:53.475 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:53.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:53.577 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:53.611 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:35:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:35:53] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-08-17T17:35:53.618 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:53.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:53.665 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 53 16 44596 44580 52.5648 71.875 0.00820321 0.0190188 2024-08-17T17:35:53.716 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:35:53.717 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:35:53.717 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:53.717 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:44667 11.5 e1375/1371 11:a122f0be:::benchmark_data_smithi063_328699_object44666:head v 1375'2820, mlcod=1375'2818)", 2024-08-17T17:35:53.717 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:53.715210+0000", 2024-08-17T17:35:53.717 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 0.00014536599999999999, 2024-08-17T17:35:53.717 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00017111200000000001, 2024-08-17T17:35:53.717 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:35:53.717 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:35:53.717 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:35:53.717 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:53.717 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:35:53.717 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:53.715210+0000", 2024-08-17T17:35:53.717 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:53.717 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:53.718 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:53.718 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:35:53.718 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:53.715210+0000", 2024-08-17T17:35:53.718 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:35:53.718 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:53.718 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:53.718 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:35:53.718 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:53.715209+0000", 2024-08-17T17:35:53.718 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:35:53.718 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:53.718 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:53.718 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:35:53.718 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:53.715233+0000", 2024-08-17T17:35:53.718 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3791999999999999e-05 2024-08-17T17:35:53.719 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:53.719 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:53.719 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:35:53.719 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:53.715243+0000", 2024-08-17T17:35:53.719 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.3449999999999995e-06 2024-08-17T17:35:53.719 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:53.719 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:53.719 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:35:53.719 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:53.715247+0000", 2024-08-17T17:35:53.719 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.498e-06 2024-08-17T17:35:53.719 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:53.719 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:53.719 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:35:53.720 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:53.715288+0000", 2024-08-17T17:35:53.720 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0398999999999997e-05 2024-08-17T17:35:53.720 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:35:53.720 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:35:53.720 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:35:53.720 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:53.715310+0000", 2024-08-17T17:35:53.720 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2376999999999999e-05 2024-08-17T17:35:53.720 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:53.720 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:35:53.720 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:53.720 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:35:53.720 INFO:teuthology.orchestra.run.smithi162.stdout: ], 2024-08-17T17:35:53.721 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 1 2024-08-17T17:35:53.721 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:35:53.721 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:53 smithi162 ceph-mon[104668]: pgmap v2093: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s; 13039/114513 objects degraded (11.386%); 9073/114513 objects misplaced (7.923%) 2024-08-17T17:35:53.721 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:53 smithi162 ceph-mon[104668]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T17:35:53.721 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:53 smithi162 ceph-mon[104668]: from='osd.6 [v2:172.21.15.162:6824/1307501458,v1:172.21.15.162:6825/1307501458]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi162", "root=default"]} : dispatch 2024-08-17T17:35:53.721 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:53 smithi162 ceph-mon[104668]: osdmap e1375: 8 total, 7 up, 8 in 2024-08-17T17:35:53.721 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:53 smithi162 ceph-mon[104668]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi162", "root=default"]} : dispatch 2024-08-17T17:35:53.721 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:53 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 13039/114513 objects degraded (11.386%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:53.721 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:53 smithi162 ceph-mon[104668]: from='osd.6 ' entity='osd.6' 2024-08-17T17:35:53.722 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:53.763 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:53.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:53.865 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:53.908 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:53.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:53.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:54.010 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:53 smithi063 ceph-mon[114756]: pgmap v2093: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s; 13039/114513 objects degraded (11.386%); 9073/114513 objects misplaced (7.923%) 2024-08-17T17:35:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:53 smithi063 ceph-mon[114756]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T17:35:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:53 smithi063 ceph-mon[114756]: from='osd.6 [v2:172.21.15.162:6824/1307501458,v1:172.21.15.162:6825/1307501458]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi162", "root=default"]} : dispatch 2024-08-17T17:35:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:53 smithi063 ceph-mon[114756]: osdmap e1375: 8 total, 7 up, 8 in 2024-08-17T17:35:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:53 smithi063 ceph-mon[114756]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi162", "root=default"]} : dispatch 2024-08-17T17:35:54.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:53 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 13039/114513 objects degraded (11.386%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:54.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:53 smithi063 ceph-mon[114756]: from='osd.6 ' entity='osd.6' 2024-08-17T17:35:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:53 smithi063 ceph-mon[109843]: pgmap v2093: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 148 MiB/s wr, 2.37k op/s; 13039/114513 objects degraded (11.386%); 9073/114513 objects misplaced (7.923%) 2024-08-17T17:35:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:53 smithi063 ceph-mon[109843]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-08-17T17:35:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:53 smithi063 ceph-mon[109843]: from='osd.6 [v2:172.21.15.162:6824/1307501458,v1:172.21.15.162:6825/1307501458]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi162", "root=default"]} : dispatch 2024-08-17T17:35:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:53 smithi063 ceph-mon[109843]: osdmap e1375: 8 total, 7 up, 8 in 2024-08-17T17:35:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:53 smithi063 ceph-mon[109843]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi162", "root=default"]} : dispatch 2024-08-17T17:35:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:53 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 13039/114513 objects degraded (11.386%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:54.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:53 smithi063 ceph-mon[109843]: from='osd.6 ' entity='osd.6' 2024-08-17T17:35:54.039 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:54.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:54.140 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:54.166 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:54.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:54.269 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:54.292 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:54.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:54.349 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:35:54.393 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:54.416 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:54.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:54.518 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:54.540 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:54.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:54.551 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:54.642 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:54.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.665 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:54.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:54.666 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 54 16 45474 45458 52.6076 54.875 0.00224989 0.0188395 2024-08-17T17:35:54.766 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:54.771 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:35:54.788 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:54.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:54.890 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:54.915 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:54.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:54.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:54.962 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:54 smithi162 ceph-mon[104668]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:35:54.963 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:54 smithi162 ceph-mon[104668]: osd.6 [v2:172.21.15.162:6824/1307501458,v1:172.21.15.162:6825/1307501458] boot 2024-08-17T17:35:54.963 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:54 smithi162 ceph-mon[104668]: osdmap e1376: 8 total, 8 up, 8 in 2024-08-17T17:35:54.963 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:54 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-08-17T17:35:55.017 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:55.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:54 smithi063 ceph-mon[109843]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:35:55.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:54 smithi063 ceph-mon[109843]: osd.6 [v2:172.21.15.162:6824/1307501458,v1:172.21.15.162:6825/1307501458] boot 2024-08-17T17:35:55.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:54 smithi063 ceph-mon[109843]: osdmap e1376: 8 total, 8 up, 8 in 2024-08-17T17:35:55.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:54 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-08-17T17:35:55.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:54 smithi063 ceph-mon[114756]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-08-17T17:35:55.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:54 smithi063 ceph-mon[114756]: osd.6 [v2:172.21.15.162:6824/1307501458,v1:172.21.15.162:6825/1307501458] boot 2024-08-17T17:35:55.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:54 smithi063 ceph-mon[114756]: osdmap e1376: 8 total, 8 up, 8 in 2024-08-17T17:35:55.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:54 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-08-17T17:35:55.039 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:55.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:55.141 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:55.165 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:55.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:55.253 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:35:55.267 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:55.290 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:55.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:55.391 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:55.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.415 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:55.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:55.516 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:55.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.539 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:55.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:55.640 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:55.663 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:55.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:55.665 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 55 16 45474 45458 51.6511 0 - 0.0188395 2024-08-17T17:35:55.754 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:55 smithi063 ceph-mon[109843]: pgmap v2096: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s; 13579/130182 objects degraded (10.431%); 10836/130182 objects misplaced (8.324%) 2024-08-17T17:35:55.754 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:55 smithi063 ceph-mon[109843]: osdmap e1377: 8 total, 8 up, 8 in 2024-08-17T17:35:55.754 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:55 smithi063 ceph-mon[114756]: pgmap v2096: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s; 13579/130182 objects degraded (10.431%); 10836/130182 objects misplaced (8.324%) 2024-08-17T17:35:55.754 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:55 smithi063 ceph-mon[114756]: osdmap e1377: 8 total, 8 up, 8 in 2024-08-17T17:35:55.765 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:55.788 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:55.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:55.890 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:55.910 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:55 smithi162 ceph-mon[104668]: pgmap v2096: 119 pgs: 40 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 59 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 145 MiB/s wr, 2.31k op/s; 13579/130182 objects degraded (10.431%); 10836/130182 objects misplaced (8.324%) 2024-08-17T17:35:55.910 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:55 smithi162 ceph-mon[104668]: osdmap e1377: 8 total, 8 up, 8 in 2024-08-17T17:35:55.913 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:55.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:55.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:56.015 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:56.037 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:56.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:56.139 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:56.160 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:56.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:56.183 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:35:56.262 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:56.291 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:56.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:56.393 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:56.416 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:56.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:56.453 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:35:56.453 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:35:56.453 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:35:56.453 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:35:56.453 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:35:56.518 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:56.553 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:56.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:56.655 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:56.665 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 56 16 45741 45725 51.0267 8.34375 0.00678727 0.0195915 2024-08-17T17:35:56.690 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:56.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:56.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:56 smithi063 ceph-mon[114756]: 1.0 scrub starts 2024-08-17T17:35:56.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:56 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3177102444' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:35:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:56 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:56 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3177102444' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:35:56.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:56 smithi063 ceph-mon[114756]: osdmap e1378: 8 total, 8 up, 8 in 2024-08-17T17:35:56.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:56 smithi063 ceph-mon[109843]: 1.0 scrub starts 2024-08-17T17:35:56.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:56 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3177102444' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:35:56.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:56 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:56.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:56 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3177102444' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:35:56.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:56 smithi063 ceph-mon[109843]: osdmap e1378: 8 total, 8 up, 8 in 2024-08-17T17:35:56.792 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:56.828 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:56.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:56.835 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:35:56.930 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:35:56.974 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:56.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:56.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:35:57.076 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:57.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:56 smithi162 ceph-mon[104668]: 1.0 scrub starts 2024-08-17T17:35:57.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:56 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3177102444' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:35:57.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:56 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:35:57.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:56 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3177102444' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:35:57.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:56 smithi162 ceph-mon[104668]: osdmap e1378: 8 total, 8 up, 8 in 2024-08-17T17:35:57.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.124 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:57.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:57.225 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:35:57.265 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:57.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:35:57.367 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:57.404 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:57.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:57.506 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:57.545 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:57.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:57.647 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:57.665 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 57 16 47025 47009 51.5391 80.25 0.0047003 0.0193935 2024-08-17T17:35:57.671 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:57.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:57.773 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:57.811 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:57.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:57.913 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:57.942 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:57 smithi063 ceph-mon[114756]: pgmap v2099: 119 pgs: 39 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 60 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s; 13579/130182 objects degraded (10.431%); 10836/130182 objects misplaced (8.324%) 2024-08-17T17:35:57.942 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:57 smithi063 ceph-mon[114756]: osdmap e1379: 8 total, 8 up, 8 in 2024-08-17T17:35:57.943 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:57 smithi063 ceph-mon[109843]: pgmap v2099: 119 pgs: 39 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 60 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s; 13579/130182 objects degraded (10.431%); 10836/130182 objects misplaced (8.324%) 2024-08-17T17:35:57.943 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:57 smithi063 ceph-mon[109843]: osdmap e1379: 8 total, 8 up, 8 in 2024-08-17T17:35:57.949 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:57.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:57.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:58.051 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:58.092 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:58.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:57 smithi162 ceph-mon[104668]: pgmap v2099: 119 pgs: 39 active+clean+remapped, 11 active+undersized, 9 active+undersized+degraded, 60 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 82 MiB/s wr, 1.30k op/s; 13579/130182 objects degraded (10.431%); 10836/130182 objects misplaced (8.324%) 2024-08-17T17:35:58.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:57 smithi162 ceph-mon[104668]: osdmap e1379: 8 total, 8 up, 8 in 2024-08-17T17:35:58.194 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:58.226 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:58.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:58.298 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:35:58.298 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:35:58.328 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:58.332 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:35:58.332 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:35:58.359 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:58.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:58.366 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:35:58.366 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:35:58.398 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:35:58.399 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:35:58.432 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:35:58.433 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:35:58.461 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:58.469 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:35:58.470 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:35:58.500 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:58.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:58.503 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:35:58.503 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:35:58.535 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:35:58.535 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:35:58.602 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:58.641 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:58.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:35:58.666 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 58 16 48228 48212 51.9467 75.1875 0.027238 0.0192447 2024-08-17T17:35:58.742 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:58.775 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:58.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:58.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:58 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 13579/130182 objects degraded (10.431%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:58.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:58 smithi063 ceph-mon[109843]: osdmap e1380: 8 total, 8 up, 8 in 2024-08-17T17:35:58.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:58 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1042832690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:58.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:58 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 13579/130182 objects degraded (10.431%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:58.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:58 smithi063 ceph-mon[114756]: osdmap e1380: 8 total, 8 up, 8 in 2024-08-17T17:35:58.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:58 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1042832690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:58.877 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:58.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:58.914 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:58.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:58.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:58 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 13579/130182 objects degraded (10.431%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2024-08-17T17:35:58.960 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:58 smithi162 ceph-mon[104668]: osdmap e1380: 8 total, 8 up, 8 in 2024-08-17T17:35:58.961 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:58 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1042832690' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:35:59.009 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:35:59.014 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:59.129 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:59.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:59.181 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:35:59.231 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:35:59.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.287 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:59.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:35:59.388 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:35:59.426 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:59.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:35:59.528 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:35:59.593 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:35:59.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:35:59.666 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 59 16 49414 49398 52.3225 74.125 0.0138974 0.0191061 2024-08-17T17:35:59.689 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:35:59.694 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:35:59.768 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:59.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:35:59.870 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:35:59.912 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:35:59.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:35:59.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:00.013 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:35:59 smithi063 ceph-mon[114756]: pgmap v2102: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 active+undersized+degraded, 108 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 15 KiB/s rd, 43 MiB/s wr, 702 op/s; 25215/138480 objects degraded (18.208%); 833 KiB/s, 11 objects/s recovering 2024-08-17T17:36:00.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:35:59 smithi063 ceph-mon[109843]: pgmap v2102: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 active+undersized+degraded, 108 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 15 KiB/s rd, 43 MiB/s wr, 702 op/s; 25215/138480 objects degraded (18.208%); 833 KiB/s, 11 objects/s recovering 2024-08-17T17:36:00.052 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:00.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:35:59 smithi162 ceph-mon[104668]: pgmap v2102: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 peering, 3 active+undersized+degraded, 108 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 15 KiB/s rd, 43 MiB/s wr, 702 op/s; 25215/138480 objects degraded (18.208%); 833 KiB/s, 11 objects/s recovering 2024-08-17T17:36:00.153 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:00.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.192 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:00.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:00.293 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:00.328 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:00.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:00.430 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:00.450 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:00.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:00.552 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:00.610 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:00.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:00.666 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:36:00.666214+0000 min lat: 0.000948653 max lat: 32.4964 avg lat: 0.0190429 2024-08-17T17:36:00.666 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:36:00.666 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 60 16 50418 50402 52.4962 62.75 0.0257231 0.0190429 2024-08-17T17:36:00.712 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:00.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.749 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:00.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:00.850 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:00.877 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:00.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:00.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:00.979 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:01.018 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:01.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:01.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:01.156 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:01.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:01.226 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:01.226 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:01.258 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:01.301 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:01.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:01.358 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:01.403 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:01.433 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:01.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:01.535 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:01.573 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:01.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:01.666 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 61 16 51633 51617 52.8804 75.9375 0.00648757 0.0189043 2024-08-17T17:36:01.676 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:01.716 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:01.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:01.818 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:01 smithi162 ceph-mon[104668]: pgmap v2103: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 peering, 108 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 11 KiB/s rd, 46 MiB/s wr, 737 op/s; 25596/142035 objects degraded (18.021%); 615 KiB/s, 8 objects/s recovering 2024-08-17T17:36:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:01.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:01.856 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:01.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:01.904 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:01.933 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:36:01.957 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:01 smithi063 ceph-mon[114756]: pgmap v2103: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 peering, 108 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 11 KiB/s rd, 46 MiB/s wr, 737 op/s; 25596/142035 objects degraded (18.021%); 615 KiB/s, 8 objects/s recovering 2024-08-17T17:36:01.957 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:01.957 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:01.958 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:01 smithi063 ceph-mon[109843]: pgmap v2103: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 peering, 108 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 11 KiB/s rd, 46 MiB/s wr, 737 op/s; 25596/142035 objects degraded (18.021%); 615 KiB/s, 8 objects/s recovering 2024-08-17T17:36:01.958 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:01.958 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:01.958 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:01.979 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:01.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:01.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:02.081 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:02.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.136 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:02.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:02.237 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:02.275 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:02.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:02.378 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:02.411 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:02.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:02.513 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:02.544 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:02.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:02.646 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:02.666 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 62 16 52618 52602 53.0203 61.5625 0.0100237 0.0188547 2024-08-17T17:36:02.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.691 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:02.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:02.792 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:02.836 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:02.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:02.938 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:02.979 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:02.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:02.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:03.081 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:03.119 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:03.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:03.211 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:36:03.221 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:03.294 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:03.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:03.396 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:03.432 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:03.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:03.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:36:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:36:03] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-17T17:36:03.535 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:03.570 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:03.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:03.666 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 63 16 53727 53711 53.2788 69.3125 0.0127313 0.0187639 2024-08-17T17:36:03.672 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:03.709 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:03.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:03.811 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:03.854 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:03.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:03.955 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:03.956 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:03 smithi162 ceph-mon[104668]: pgmap v2104: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 8 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 9.5 KiB/s rd, 57 MiB/s wr, 912 op/s; 25353/147603 objects degraded (17.176%); 2.9 MiB/s, 45 objects/s recovering 2024-08-17T17:36:03.957 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:03 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 25353/147603 objects degraded (17.176%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:03.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:03.997 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:03.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:04.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:03 smithi063 ceph-mon[114756]: pgmap v2104: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 8 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 9.5 KiB/s rd, 57 MiB/s wr, 912 op/s; 25353/147603 objects degraded (17.176%); 2.9 MiB/s, 45 objects/s recovering 2024-08-17T17:36:04.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:03 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 25353/147603 objects degraded (17.176%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:04.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:03 smithi063 ceph-mon[109843]: pgmap v2104: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 8 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 9.5 KiB/s rd, 57 MiB/s wr, 912 op/s; 25353/147603 objects degraded (17.176%); 2.9 MiB/s, 45 objects/s recovering 2024-08-17T17:36:04.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:03 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 25353/147603 objects degraded (17.176%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:04.098 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:04.134 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:04.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:04.236 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:04.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.258 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:04.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:04.359 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:04.397 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:04.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:04.499 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:04.539 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:04.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:04.622 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:04.622 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:04.641 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:04.666 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 64 16 54753 54737 53.4482 64.125 0.0129175 0.0187036 2024-08-17T17:36:04.677 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:04.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:04.779 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:04.818 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:04.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:04.923 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:04.960 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:04.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:04.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:05.062 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:05.093 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:05.248 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:05.249 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:05.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:05.350 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:05.381 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:36:05.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.391 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:05.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:05.492 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:05.532 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:05.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:05.634 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:05.666 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:05.666 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 65 16 55912 55896 53.7402 72.4375 0.0131393 0.0186002 2024-08-17T17:36:05.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:05.736 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:05.768 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:05.816 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:05.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:05.917 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:05.931 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:05 smithi063 ceph-mon[114756]: pgmap v2105: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 8 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 7.6 KiB/s rd, 75 MiB/s wr, 1.21k op/s; 25353/159078 objects degraded (15.937%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:36:05.931 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:05 smithi063 ceph-mon[109843]: pgmap v2105: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 8 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 7.6 KiB/s rd, 75 MiB/s wr, 1.21k op/s; 25353/159078 objects degraded (15.937%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:36:05.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:05.955 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:05.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:06.031 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:05 smithi162 ceph-mon[104668]: pgmap v2105: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 8 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 7.6 KiB/s rd, 75 MiB/s wr, 1.21k op/s; 25353/159078 objects degraded (15.937%); 2.3 MiB/s, 36 objects/s recovering 2024-08-17T17:36:06.057 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:06.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.092 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:06.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:06.193 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:06.232 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:06.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:06.334 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:06.375 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:06.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:06.477 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:06.516 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:06.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:06.618 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:06.655 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:06.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:06.666 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 66 16 57061 57045 54.0139 71.8125 0.0120272 0.0185081 2024-08-17T17:36:06.757 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:06.796 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:06.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:06.898 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:06.937 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:06.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:06.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:07.039 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:07.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.078 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:07.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:07.179 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:07.213 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:07.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:07.314 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:07.335 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:07.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:07.437 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:07.474 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:07.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.475 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:07.576 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:07.612 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:07.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:07.666 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 67 16 58010 57994 54.0929 59.3125 0.00783208 0.01848 2024-08-17T17:36:07.714 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:07.762 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:07.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:07.864 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:07.907 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:07.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:07.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:08.009 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:07 smithi063 ceph-mon[109843]: pgmap v2106: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 8 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 6.5 KiB/s rd, 65 MiB/s wr, 1.05k op/s; 25174/159723 objects degraded (15.761%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:36:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:07 smithi063 ceph-mon[114756]: pgmap v2106: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 8 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 6.5 KiB/s rd, 65 MiB/s wr, 1.05k op/s; 25174/159723 objects degraded (15.761%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:36:08.049 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:07 smithi162 ceph-mon[104668]: pgmap v2106: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 8 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 6.5 KiB/s rd, 65 MiB/s wr, 1.05k op/s; 25174/159723 objects degraded (15.761%); 3.1 MiB/s, 49 objects/s recovering 2024-08-17T17:36:08.052 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:08.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:08.154 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:08.185 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:08.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:08.286 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:08.324 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:08.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:08.425 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:08.462 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:08.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:08.564 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:08.602 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:08.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:08.666 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 68 16 59007 58991 54.2137 62.3125 0.0177881 0.0184381 2024-08-17T17:36:08.704 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:08.740 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:08.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:08.835 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:08.836 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:08.842 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:08.872 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:08.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:08.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:08.974 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:09.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.007 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:09.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:09.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:08 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 25174/159723 objects degraded (15.761%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:09.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:08 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 25174/159723 objects degraded (15.761%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:08 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 25174/159723 objects degraded (15.761%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:09.109 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:09.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.154 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:09.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:09.256 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:09.294 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:09.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:09.396 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:09.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.444 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:09.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:09.545 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:09.569 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:36:09.590 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:09.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:09.667 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 69 16 60089 60073 54.408 67.625 0.00439338 0.0183733 2024-08-17T17:36:09.692 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:09.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.726 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:09.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:09.827 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:09.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:09 smithi162 ceph-mon[104668]: pgmap v2107: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 8 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 25174/170631 objects degraded (14.753%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:36:09.873 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:09.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:09.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:09.948 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:09.975 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:10.011 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:10.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:09 smithi063 ceph-mon[109843]: pgmap v2107: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 8 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 25174/170631 objects degraded (14.753%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:36:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:09 smithi063 ceph-mon[114756]: pgmap v2107: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 8 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 25174/170631 objects degraded (14.753%); 2.5 MiB/s, 40 objects/s recovering 2024-08-17T17:36:10.113 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:10.147 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:10.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:10.249 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:10.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.285 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:10.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:10.386 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:10.425 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:10.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:10.527 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:10.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.571 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:10.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:10.667 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 70 16 61160 61144 54.5869 66.9375 0.0142646 0.0183132 2024-08-17T17:36:10.672 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:10.702 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:36:10.702 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:36:10.708 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:10.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:10.721 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:36:10.742 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:36:10.742 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:36:10.794 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:36:10.794 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:36:10.810 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:10.831 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:36:10.832 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:36:10.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:10 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3975198141' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:10.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:10 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3116457604' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:10.857 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:10.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:10.869 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:36:10.869 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:36:10.880 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:10 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3975198141' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:10.880 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:10 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3116457604' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:10.880 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:10 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3975198141' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:10.880 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:10 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3116457604' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:10.910 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:36:10.910 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:36:10.954 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:36:10.954 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:36:10.959 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:10.986 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:36:10.987 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:36:10.993 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:10.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:10.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:11.096 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:11.138 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:11.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:11.240 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:11.284 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:11.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:11.385 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:11.423 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:11.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:11.525 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:11.564 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:11.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:11.666 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:11.667 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 71 16 62210 62194 54.7423 65.625 0.0073159 0.018262 2024-08-17T17:36:11.694 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:11.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:11.796 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:11.819 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:11.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:11.921 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:11.943 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:11.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:11.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:12.022 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:11 smithi162 ceph-mon[104668]: pgmap v2108: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 8 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 25174/174792 objects degraded (14.402%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:36:12.023 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:11 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:36:12.023 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:11 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3975198141' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:12.023 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:11 smithi162 ceph-mon[104668]: osdmap e1381: 8 total, 8 up, 8 in 2024-08-17T17:36:12.023 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:11 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3851647988' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:36:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:11 smithi063 ceph-mon[109843]: pgmap v2108: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 8 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 25174/174792 objects degraded (14.402%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:36:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:11 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:36:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:11 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3975198141' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:11 smithi063 ceph-mon[109843]: osdmap e1381: 8 total, 8 up, 8 in 2024-08-17T17:36:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:11 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3851647988' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:36:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:11 smithi063 ceph-mon[114756]: pgmap v2108: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 8 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 25174/174792 objects degraded (14.402%); 2.2 MiB/s, 35 objects/s recovering 2024-08-17T17:36:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:11 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:36:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:11 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3975198141' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:12.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:11 smithi063 ceph-mon[114756]: osdmap e1381: 8 total, 8 up, 8 in 2024-08-17T17:36:12.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:11 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3851647988' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:36:12.045 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:12.068 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:12.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:12.170 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:12.193 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:12.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:12.295 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:12.318 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:12.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:12.420 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:12.443 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:12.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:12.545 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:12.566 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:12.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.567 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:12.637 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:36:12.667 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 72 16 62304 62288 54.0636 5.875 0.00429509 0.0182617 2024-08-17T17:36:12.668 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:12.691 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:12.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:12.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:12 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3851647988' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:36:12.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:12 smithi063 ceph-mon[109843]: osdmap e1382: 8 total, 8 up, 8 in 2024-08-17T17:36:12.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:12 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3851647988' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:36:12.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:12 smithi063 ceph-mon[114756]: osdmap e1382: 8 total, 8 up, 8 in 2024-08-17T17:36:12.793 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:12.820 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:12.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:12.921 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:12.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:12.945 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:12.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:13.046 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:13.068 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:13.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:13.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:12 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3851647988' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:36:13.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:12 smithi162 ceph-mon[104668]: osdmap e1382: 8 total, 8 up, 8 in 2024-08-17T17:36:13.170 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:13.193 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:13.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:13.247 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:36:13.294 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:13.316 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:13.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:13.418 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:13.440 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:13.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:13.462 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:36:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:36:13] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.51.0" 2024-08-17T17:36:13.543 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:13.564 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:13.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:13.666 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:13.667 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 73 16 62310 62294 53.3281 0.375 0.00332489 0.0182956 2024-08-17T17:36:13.689 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:13.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:13.743 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:36:13.791 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:13.815 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:13.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:13.900 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:13 smithi063 ceph-mon[114756]: pgmap v2111: 119 pgs: 1 remapped+peering, 11 peering, 2 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 798 op/s; 21961/178251 objects degraded (12.320%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:36:13.900 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:13 smithi063 ceph-mon[114756]: osdmap e1383: 8 total, 8 up, 8 in 2024-08-17T17:36:13.900 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:13 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 21961/178251 objects degraded (12.320%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:13.900 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:13 smithi063 ceph-mon[109843]: pgmap v2111: 119 pgs: 1 remapped+peering, 11 peering, 2 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 798 op/s; 21961/178251 objects degraded (12.320%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:36:13.900 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:13 smithi063 ceph-mon[109843]: osdmap e1383: 8 total, 8 up, 8 in 2024-08-17T17:36:13.900 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:13 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 21961/178251 objects degraded (12.320%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:13.917 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:13.942 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:13.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:13.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:14.021 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:13 smithi162 ceph-mon[104668]: pgmap v2111: 119 pgs: 1 remapped+peering, 11 peering, 2 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 798 op/s; 21961/178251 objects degraded (12.320%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:36:14.021 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:13 smithi162 ceph-mon[104668]: osdmap e1383: 8 total, 8 up, 8 in 2024-08-17T17:36:14.021 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:13 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 21961/178251 objects degraded (12.320%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:14.044 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:14.075 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:14.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:14.177 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:14.203 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:14.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:14.305 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:14.334 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:14.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:14.370 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:36:14.370 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:36:14.370 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:36:14.370 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:36:14.370 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:36:14.436 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:14.463 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:14.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:14.551 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:36:14.565 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:14.588 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:14.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:14.667 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 74 16 62374 62358 52.6615 4 0.00513387 0.0185837 2024-08-17T17:36:14.690 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:14.712 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:14.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:14.814 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:14.837 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:14.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:14.939 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:14.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:14.962 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:14.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:14.967 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:14 smithi063 ceph-mon[109843]: osdmap e1384: 8 total, 8 up, 8 in 2024-08-17T17:36:14.967 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:14 smithi063 ceph-mon[114756]: osdmap e1384: 8 total, 8 up, 8 in 2024-08-17T17:36:15.063 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:15.084 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:14 smithi162 ceph-mon[104668]: osdmap e1384: 8 total, 8 up, 8 in 2024-08-17T17:36:15.085 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:15.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:15.116 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:36:15.187 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:15.220 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:15.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:15.322 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:15.354 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:15.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:15.395 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:36:15.458 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:15.480 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:15.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:15.582 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:15.606 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:15.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:15.667 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 75 16 62374 62358 51.9593 0 - 0.0185837 2024-08-17T17:36:15.708 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:15.741 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:15.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:15.838 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:15.843 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:15.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:15 smithi162 ceph-mon[104668]: pgmap v2114: 119 pgs: 1 active+undersized+remapped, 9 remapped+peering, 11 peering, 2 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s; 17381/185877 objects degraded (9.351%); 5.1 MiB/s, 81 objects/s recovering 2024-08-17T17:36:15.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:15 smithi162 ceph-mon[104668]: osdmap e1385: 8 total, 8 up, 8 in 2024-08-17T17:36:15.868 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:15.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:15.882 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:15.963 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:15 smithi063 ceph-mon[114756]: pgmap v2114: 119 pgs: 1 active+undersized+remapped, 9 remapped+peering, 11 peering, 2 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s; 17381/185877 objects degraded (9.351%); 5.1 MiB/s, 81 objects/s recovering 2024-08-17T17:36:15.963 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:15 smithi063 ceph-mon[114756]: osdmap e1385: 8 total, 8 up, 8 in 2024-08-17T17:36:15.964 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:15 smithi063 ceph-mon[109843]: pgmap v2114: 119 pgs: 1 active+undersized+remapped, 9 remapped+peering, 11 peering, 2 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 58 MiB/s wr, 923 op/s; 17381/185877 objects degraded (9.351%); 5.1 MiB/s, 81 objects/s recovering 2024-08-17T17:36:15.964 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:15 smithi063 ceph-mon[109843]: osdmap e1385: 8 total, 8 up, 8 in 2024-08-17T17:36:15.970 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:15.992 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:15.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:15.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:16.094 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:16.128 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:16.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:16.229 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:16.260 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:16.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:16.362 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:16.388 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:16.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:16.490 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:16.512 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:16.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:16.613 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:16.636 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:16.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:16.667 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 76 16 62374 62358 51.2756 0 - 0.0185837 2024-08-17T17:36:16.738 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:16.760 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:16.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:16.771 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:36:16.862 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:16.891 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:16.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:16.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:16.976 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:16 smithi162 ceph-mon[104668]: osdmap e1386: 8 total, 8 up, 8 in 2024-08-17T17:36:16.976 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:36:16.977 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:16 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3109370048' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:36:16.977 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:16 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:36:16.977 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:16 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:16.977 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:16.993 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:17.027 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:17.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:17.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:16 smithi063 ceph-mon[109843]: osdmap e1386: 8 total, 8 up, 8 in 2024-08-17T17:36:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:36:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:16 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3109370048' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:36:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:16 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:36:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:16 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:17.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:17.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:16 smithi063 ceph-mon[114756]: osdmap e1386: 8 total, 8 up, 8 in 2024-08-17T17:36:17.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:36:17.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:16 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3109370048' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:36:17.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:16 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:36:17.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:16 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:17.031 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:17.129 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:17.153 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:17.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:17.255 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:17.281 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:17.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:17.383 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:17.411 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:17.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:17.513 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:17.544 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:36:17.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:17.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:17.646 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:17.667 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 77 16 62374 62358 50.6097 0 - 0.0185837 2024-08-17T17:36:17.673 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:17.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:17.774 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:17 smithi063 ceph-mon[114756]: pgmap v2117: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 9 remapped+peering, 11 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 46 MiB/s wr, 728 op/s; 25099/186993 objects degraded (13.422%); 704 KiB/s, 10 objects/s recovering 2024-08-17T17:36:17.774 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:17 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:36:17.774 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:17 smithi063 ceph-mon[114756]: osdmap e1387: 8 total, 8 up, 8 in 2024-08-17T17:36:17.774 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:17 smithi063 ceph-mon[114756]: osdmap e1388: 8 total, 8 up, 8 in 2024-08-17T17:36:17.776 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:17.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.810 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:17.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:17.911 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:17.936 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:17.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:17.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:17 smithi063 ceph-mon[109843]: pgmap v2117: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 9 remapped+peering, 11 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 46 MiB/s wr, 728 op/s; 25099/186993 objects degraded (13.422%); 704 KiB/s, 10 objects/s recovering 2024-08-17T17:36:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:17 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:36:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:17 smithi063 ceph-mon[109843]: osdmap e1387: 8 total, 8 up, 8 in 2024-08-17T17:36:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:17 smithi063 ceph-mon[109843]: osdmap e1388: 8 total, 8 up, 8 in 2024-08-17T17:36:18.038 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:18.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.061 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:18.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:17 smithi162 ceph-mon[104668]: pgmap v2117: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 9 remapped+peering, 11 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 46 MiB/s wr, 728 op/s; 25099/186993 objects degraded (13.422%); 704 KiB/s, 10 objects/s recovering 2024-08-17T17:36:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:17 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:36:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:17 smithi162 ceph-mon[104668]: osdmap e1387: 8 total, 8 up, 8 in 2024-08-17T17:36:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:17 smithi162 ceph-mon[104668]: osdmap e1388: 8 total, 8 up, 8 in 2024-08-17T17:36:18.161 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:18.184 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:18.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:18.286 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:18.313 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:18.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:18.356 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:36:18.356 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:36:18.356 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:36:18.356 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:36:18.356 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.356 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:45440 11.9 11.32a99749 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1376)", 2024-08-17T17:36:18.357 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:54.209364+0000", 2024-08-17T17:36:18.357 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 23.531289977, 2024-08-17T17:36:18.357 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98234849700000004, 2024-08-17T17:36:18.357 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.357 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:36:18.357 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:36:18.357 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:36:18.357 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:36:18.357 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 45440 2024-08-17T17:36:18.357 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.357 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.357 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.358 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.358 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209364+0000", 2024-08-17T17:36:18.358 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.358 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.358 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.358 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.358 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209364+0000", 2024-08-17T17:36:18.358 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.358 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.358 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.358 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.359 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209363+0000", 2024-08-17T17:36:18.359 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:18.359 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.359 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.359 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.359 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209417+0000", 2024-08-17T17:36:18.359 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4178999999999997e-05 2024-08-17T17:36:18.359 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.359 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.359 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.359 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209434+0000", 2024-08-17T17:36:18.359 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7286999999999999e-05 2024-08-17T17:36:18.359 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.360 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.360 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.360 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209443+0000", 2024-08-17T17:36:18.360 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9109999999999999e-06 2024-08-17T17:36:18.360 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.360 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.360 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.360 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209549+0000", 2024-08-17T17:36:18.360 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00010602700000000001 2024-08-17T17:36:18.360 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.360 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.360 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:36:18.360 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209565+0000", 2024-08-17T17:36:18.361 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6331e-05 2024-08-17T17:36:18.361 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.361 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.361 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.361 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.191702+0000", 2024-08-17T17:36:18.361 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98213637499999995 2024-08-17T17:36:18.361 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.361 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.361 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.361 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.191713+0000", 2024-08-17T17:36:18.361 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1242e-05 2024-08-17T17:36:18.361 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.361 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.361 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.362 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.362 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.362 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:45448 11.9 11.e30821d9 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1376)", 2024-08-17T17:36:18.362 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:54.209460+0000", 2024-08-17T17:36:18.362 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 23.531194298999999, 2024-08-17T17:36:18.362 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98228465499999995, 2024-08-17T17:36:18.362 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.362 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:36:18.362 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:36:18.362 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:36:18.362 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:36:18.362 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 45448 2024-08-17T17:36:18.363 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.363 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.363 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.363 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.363 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209460+0000", 2024-08-17T17:36:18.363 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.363 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.363 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.363 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.363 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209460+0000", 2024-08-17T17:36:18.363 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.363 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.363 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.363 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.364 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209460+0000", 2024-08-17T17:36:18.364 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:36:18.364 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.364 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.364 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.364 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209586+0000", 2024-08-17T17:36:18.364 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00012697199999999999 2024-08-17T17:36:18.364 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.364 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.364 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.364 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209601+0000", 2024-08-17T17:36:18.364 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4513e-05 2024-08-17T17:36:18.364 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.364 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.365 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.365 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209612+0000", 2024-08-17T17:36:18.365 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0682999999999999e-05 2024-08-17T17:36:18.365 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.365 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.365 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.365 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209679+0000", 2024-08-17T17:36:18.365 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7058999999999996e-05 2024-08-17T17:36:18.365 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.365 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.365 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:36:18.365 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209685+0000", 2024-08-17T17:36:18.365 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5699999999999998e-06 2024-08-17T17:36:18.366 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.366 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.366 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.366 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.191737+0000", 2024-08-17T17:36:18.366 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98205150399999996 2024-08-17T17:36:18.366 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.366 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.366 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.366 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.191745+0000", 2024-08-17T17:36:18.366 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9230000000000002e-06 2024-08-17T17:36:18.366 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.366 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.366 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.366 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.367 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.367 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:45462 11.0 11.bc8c9820 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1376)", 2024-08-17T17:36:18.367 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:54.209629+0000", 2024-08-17T17:36:18.367 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 23.531025189000001, 2024-08-17T17:36:18.367 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99344323999999995, 2024-08-17T17:36:18.367 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.367 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:36:18.367 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:36:18.367 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:36:18.367 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:36:18.367 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 45462 2024-08-17T17:36:18.367 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.368 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.368 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.368 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.368 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209629+0000", 2024-08-17T17:36:18.368 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.368 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.368 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.368 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.368 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209629+0000", 2024-08-17T17:36:18.368 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.368 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.368 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.368 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.369 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209628+0000", 2024-08-17T17:36:18.369 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:36:18.369 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.369 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.369 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.369 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209658+0000", 2024-08-17T17:36:18.369 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9785999999999999e-05 2024-08-17T17:36:18.369 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.369 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.369 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.369 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209667+0000", 2024-08-17T17:36:18.369 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8270000000000004e-06 2024-08-17T17:36:18.370 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.370 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.370 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.370 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209673+0000", 2024-08-17T17:36:18.370 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9660000000000001e-06 2024-08-17T17:36:18.370 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.370 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.370 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.370 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209785+0000", 2024-08-17T17:36:18.370 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000112364 2024-08-17T17:36:18.370 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.370 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.370 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:36:18.371 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209794+0000", 2024-08-17T17:36:18.371 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4270000000000008e-06 2024-08-17T17:36:18.371 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.371 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.371 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.371 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.203063+0000", 2024-08-17T17:36:18.371 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99326882000000005 2024-08-17T17:36:18.371 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.371 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.371 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.371 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.203072+0000", 2024-08-17T17:36:18.371 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7669999999999998e-06 2024-08-17T17:36:18.372 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.372 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.372 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.372 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.372 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.372 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:45470 11.3 11.aa96c633 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1376)", 2024-08-17T17:36:18.372 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:54.214839+0000", 2024-08-17T17:36:18.372 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 23.525815075000001, 2024-08-17T17:36:18.372 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98449979399999998, 2024-08-17T17:36:18.372 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.372 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:36:18.372 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:36:18.372 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:36:18.373 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:36:18.373 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 45470 2024-08-17T17:36:18.373 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.373 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.373 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.373 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.373 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.214839+0000", 2024-08-17T17:36:18.373 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.373 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.373 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.373 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.373 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.214839+0000", 2024-08-17T17:36:18.373 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.374 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.374 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.374 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.374 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.214838+0000", 2024-08-17T17:36:18.374 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:36:18.374 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.374 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.374 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.374 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.214947+0000", 2024-08-17T17:36:18.374 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000108482 2024-08-17T17:36:18.374 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.374 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.375 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.375 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.214963+0000", 2024-08-17T17:36:18.375 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6552999999999999e-05 2024-08-17T17:36:18.375 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.375 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.375 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.375 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.214972+0000", 2024-08-17T17:36:18.375 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6090000000000001e-06 2024-08-17T17:36:18.375 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.375 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.375 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.375 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.215020+0000", 2024-08-17T17:36:18.376 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7550999999999999e-05 2024-08-17T17:36:18.376 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.376 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.376 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:36:18.376 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.215028+0000", 2024-08-17T17:36:18.376 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.772e-06 2024-08-17T17:36:18.376 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.376 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.376 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.376 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.199330+0000", 2024-08-17T17:36:18.376 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.984301553 2024-08-17T17:36:18.376 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.376 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.376 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.377 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.199339+0000", 2024-08-17T17:36:18.377 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.1910000000000006e-06 2024-08-17T17:36:18.377 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.377 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.377 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.377 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.377 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.377 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:45474 11.3 11.4aad86b3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1376)", 2024-08-17T17:36:18.377 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:54.220249+0000", 2024-08-17T17:36:18.377 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 23.520405277000002, 2024-08-17T17:36:18.377 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97911945499999997, 2024-08-17T17:36:18.377 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.377 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:36:18.378 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:36:18.378 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:36:18.378 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:36:18.378 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 45474 2024-08-17T17:36:18.378 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.378 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.378 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.378 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.378 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.220249+0000", 2024-08-17T17:36:18.378 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.378 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.379 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.379 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.379 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.220249+0000", 2024-08-17T17:36:18.379 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.379 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.379 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.379 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.379 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.220247+0000", 2024-08-17T17:36:18.379 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:36:18.379 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.379 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.379 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.379 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.220358+0000", 2024-08-17T17:36:18.380 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000110604 2024-08-17T17:36:18.380 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.380 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.380 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.380 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.220387+0000", 2024-08-17T17:36:18.380 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8983e-05 2024-08-17T17:36:18.380 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.380 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.380 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.380 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.220400+0000", 2024-08-17T17:36:18.380 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3696999999999999e-05 2024-08-17T17:36:18.380 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.380 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.380 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.381 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.220541+0000", 2024-08-17T17:36:18.381 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000141027 2024-08-17T17:36:18.381 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.381 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.381 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:36:18.381 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.220557+0000", 2024-08-17T17:36:18.381 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5227e-05 2024-08-17T17:36:18.381 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.381 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.381 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.381 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.199362+0000", 2024-08-17T17:36:18.381 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97880558299999998 2024-08-17T17:36:18.381 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.382 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.382 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.382 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.199369+0000", 2024-08-17T17:36:18.382 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4269999999999996e-06 2024-08-17T17:36:18.382 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.382 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.382 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.382 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.382 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.382 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:45646 11.1 e1378/1377 11:8f5a5526:::benchmark_data_smithi063_328699_object45645:head v 1378'2839, mlcod=0'0)", 2024-08-17T17:36:18.382 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:56.577930+0000", 2024-08-17T17:36:18.382 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 21.162724447999999, 2024-08-17T17:36:18.383 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.030510908999999999, 2024-08-17T17:36:18.383 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.383 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:18.383 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.383 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.383 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.383 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.577930+0000", 2024-08-17T17:36:18.383 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.383 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.383 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.383 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.383 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.577930+0000", 2024-08-17T17:36:18.383 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.383 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.384 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.384 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.384 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.577929+0000", 2024-08-17T17:36:18.384 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:36:18.384 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.384 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.384 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.384 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.577994+0000", 2024-08-17T17:36:18.384 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4766999999999998e-05 2024-08-17T17:36:18.384 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.384 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.384 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.384 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.578001+0000", 2024-08-17T17:36:18.384 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5289999999999997e-06 2024-08-17T17:36:18.385 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.385 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.385 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.385 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.578004+0000", 2024-08-17T17:36:18.385 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7419999999999999e-06 2024-08-17T17:36:18.385 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.385 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.385 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.385 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.578042+0000", 2024-08-17T17:36:18.385 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7407000000000003e-05 2024-08-17T17:36:18.385 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.385 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.385 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:36:18.386 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.578063+0000", 2024-08-17T17:36:18.386 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.1610000000000001e-05 2024-08-17T17:36:18.386 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.386 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.386 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:36:18.386 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.608349+0000", 2024-08-17T17:36:18.386 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.030285954 2024-08-17T17:36:18.386 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.386 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.386 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.386 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.608441+0000", 2024-08-17T17:36:18.386 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.1454999999999998e-05 2024-08-17T17:36:18.386 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.386 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.387 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.387 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.387 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.387 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:46224 11.3 e1378/1377 11:c8d96e1f:::benchmark_data_smithi063_328699_object46223:head v 1378'2901, mlcod=0'0)", 2024-08-17T17:36:18.387 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:57.035501+0000", 2024-08-17T17:36:18.387 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 20.70515378, 2024-08-17T17:36:18.387 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.039345974999999998, 2024-08-17T17:36:18.387 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.387 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:18.387 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.387 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.387 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.387 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.035501+0000", 2024-08-17T17:36:18.388 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.388 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.388 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.388 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.388 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.035501+0000", 2024-08-17T17:36:18.388 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.388 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.388 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.388 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.388 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.035500+0000", 2024-08-17T17:36:18.388 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:36:18.388 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.388 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.388 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.389 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.035570+0000", 2024-08-17T17:36:18.389 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0567000000000004e-05 2024-08-17T17:36:18.389 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.389 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.389 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.389 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.057458+0000", 2024-08-17T17:36:18.389 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.021887446000000001 2024-08-17T17:36:18.389 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.389 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.389 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.389 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.057466+0000", 2024-08-17T17:36:18.389 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.4940000000000003e-06 2024-08-17T17:36:18.390 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.390 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.390 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.390 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.062349+0000", 2024-08-17T17:36:18.390 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0048823149999999999 2024-08-17T17:36:18.390 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.390 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.390 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:36:18.390 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.062371+0000", 2024-08-17T17:36:18.390 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2330000000000001e-05 2024-08-17T17:36:18.390 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.390 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.390 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:36:18.391 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.074812+0000", 2024-08-17T17:36:18.391 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012441020000000001 2024-08-17T17:36:18.391 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.391 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.391 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.391 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.074847+0000", 2024-08-17T17:36:18.391 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4486999999999997e-05 2024-08-17T17:36:18.391 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.391 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.391 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.391 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.391 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.391 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:46655 11.5 e1379/1377 11:a2fbd958:::benchmark_data_smithi063_328699_object46654:head v 1379'2928, mlcod=0'0)", 2024-08-17T17:36:18.392 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:57.396369+0000", 2024-08-17T17:36:18.392 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 20.344285115000002, 2024-08-17T17:36:18.392 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.037003486000000002, 2024-08-17T17:36:18.392 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.392 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:18.392 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.392 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.392 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.392 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.396369+0000", 2024-08-17T17:36:18.392 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.392 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.392 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.392 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.392 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.396369+0000", 2024-08-17T17:36:18.393 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.393 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.393 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.393 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.393 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.396368+0000", 2024-08-17T17:36:18.393 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:36:18.393 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.393 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.393 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.393 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.396395+0000", 2024-08-17T17:36:18.393 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6499000000000002e-05 2024-08-17T17:36:18.393 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.393 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.394 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.394 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.396406+0000", 2024-08-17T17:36:18.394 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1107000000000001e-05 2024-08-17T17:36:18.394 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.394 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.394 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.394 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.396413+0000", 2024-08-17T17:36:18.394 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5849999999999996e-06 2024-08-17T17:36:18.394 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.394 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.394 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.394 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.397991+0000", 2024-08-17T17:36:18.395 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015783430000000001 2024-08-17T17:36:18.395 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.395 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.395 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:36:18.395 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.398019+0000", 2024-08-17T17:36:18.395 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8498999999999999e-05 2024-08-17T17:36:18.395 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.395 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.395 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:36:18.395 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.432908+0000", 2024-08-17T17:36:18.395 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.034888633000000002 2024-08-17T17:36:18.395 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.396 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.396 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.396 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.433373+0000", 2024-08-17T17:36:18.396 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00046464100000000001 2024-08-17T17:36:18.396 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.396 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.396 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.396 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.396 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.396 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:46649 11.d e1379/1377 11:b8b04b83:::benchmark_data_smithi063_328699_object46648:head v 1379'2920, mlcod=0'0)", 2024-08-17T17:36:18.396 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:57.398491+0000", 2024-08-17T17:36:18.396 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 20.342163645999999, 2024-08-17T17:36:18.396 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.037010217999999998, 2024-08-17T17:36:18.397 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.397 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:18.397 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.397 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.397 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.397 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.398491+0000", 2024-08-17T17:36:18.397 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.397 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.397 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.397 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.397 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.398491+0000", 2024-08-17T17:36:18.397 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.397 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.398 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.398 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.398 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.398490+0000", 2024-08-17T17:36:18.398 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:36:18.398 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.398 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.398 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.398 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.398547+0000", 2024-08-17T17:36:18.398 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7766999999999998e-05 2024-08-17T17:36:18.398 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.398 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.398 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.399 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.398563+0000", 2024-08-17T17:36:18.399 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5099000000000001e-05 2024-08-17T17:36:18.399 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.399 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.399 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.399 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.398569+0000", 2024-08-17T17:36:18.399 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7549999999999997e-06 2024-08-17T17:36:18.399 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.399 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.399 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.399 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.400762+0000", 2024-08-17T17:36:18.399 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002192481 2024-08-17T17:36:18.399 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.399 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.399 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:36:18.400 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.400780+0000", 2024-08-17T17:36:18.400 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.8447000000000001e-05 2024-08-17T17:36:18.400 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.400 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.400 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:36:18.400 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.435466+0000", 2024-08-17T17:36:18.400 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.034685973000000002 2024-08-17T17:36:18.400 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.400 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.400 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.400 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.435501+0000", 2024-08-17T17:36:18.400 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4764999999999997e-05 2024-08-17T17:36:18.400 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.400 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.401 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.401 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.401 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.401 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:46652 11.d e1379/1377 11:b8f5fbef:::benchmark_data_smithi063_328699_object46651:head v 1379'2921, mlcod=0'0)", 2024-08-17T17:36:18.401 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:57.399458+0000", 2024-08-17T17:36:18.401 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 20.341196703000001, 2024-08-17T17:36:18.401 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.036065343, 2024-08-17T17:36:18.401 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.401 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:18.401 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.401 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.401 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.401 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.399458+0000", 2024-08-17T17:36:18.402 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.402 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.402 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.402 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.402 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.399458+0000", 2024-08-17T17:36:18.402 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.402 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.402 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.402 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.402 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.399457+0000", 2024-08-17T17:36:18.402 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:36:18.402 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.403 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.403 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.403 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.399520+0000", 2024-08-17T17:36:18.403 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3658999999999995e-05 2024-08-17T17:36:18.403 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.403 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.403 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.403 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.399531+0000", 2024-08-17T17:36:18.403 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0917000000000001e-05 2024-08-17T17:36:18.403 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.403 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.403 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.403 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.399537+0000", 2024-08-17T17:36:18.403 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5690000000000004e-06 2024-08-17T17:36:18.404 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.404 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.404 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.404 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.409752+0000", 2024-08-17T17:36:18.404 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010214867000000001 2024-08-17T17:36:18.404 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.404 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.404 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:36:18.404 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.409776+0000", 2024-08-17T17:36:18.404 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4725000000000001e-05 2024-08-17T17:36:18.404 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.404 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.405 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:36:18.405 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.435506+0000", 2024-08-17T17:36:18.405 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.025729126000000001 2024-08-17T17:36:18.405 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.405 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.405 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.405 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.435523+0000", 2024-08-17T17:36:18.405 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7487999999999999e-05 2024-08-17T17:36:18.405 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.405 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.405 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.405 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.405 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.406 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:46661 11.5 e1379/1377 11:ae8d7763:::benchmark_data_smithi063_328699_object46660:head v 1379'2929, mlcod=0'0)", 2024-08-17T17:36:18.406 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:57.402284+0000", 2024-08-17T17:36:18.406 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 20.338370654999999, 2024-08-17T17:36:18.406 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.033312950000000001, 2024-08-17T17:36:18.406 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.406 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:18.406 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.406 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.406 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.406 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.402284+0000", 2024-08-17T17:36:18.406 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.406 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.407 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.407 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.407 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.402284+0000", 2024-08-17T17:36:18.407 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.407 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.407 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.407 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.407 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.402283+0000", 2024-08-17T17:36:18.407 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:36:18.407 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.407 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.407 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.407 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.402309+0000", 2024-08-17T17:36:18.408 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6115e-05 2024-08-17T17:36:18.408 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.408 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.408 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.408 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.402319+0000", 2024-08-17T17:36:18.408 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9389999999999992e-06 2024-08-17T17:36:18.408 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.408 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.408 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.408 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.402325+0000", 2024-08-17T17:36:18.408 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0159999999999997e-06 2024-08-17T17:36:18.408 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.408 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.409 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.409 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.409786+0000", 2024-08-17T17:36:18.409 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0074605380000000001 2024-08-17T17:36:18.409 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.409 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.409 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:36:18.409 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.409801+0000", 2024-08-17T17:36:18.409 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5665e-05 2024-08-17T17:36:18.409 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.409 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.409 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:36:18.409 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.435525+0000", 2024-08-17T17:36:18.409 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.025723676000000001 2024-08-17T17:36:18.410 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.410 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.410 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.410 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:57.435597+0000", 2024-08-17T17:36:18.410 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1678000000000001e-05 2024-08-17T17:36:18.410 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.410 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.410 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.410 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.410 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.410 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:47598 11.9 e1379/1377 11:9f580121:::benchmark_data_smithi063_328699_object47597:head v 1379'3028, mlcod=1373'83)", 2024-08-17T17:36:18.410 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:58.176685+0000", 2024-08-17T17:36:18.410 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 19.563969801999999, 2024-08-17T17:36:18.411 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.032764279, 2024-08-17T17:36:18.411 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.411 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:18.411 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.411 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.411 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.411 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:58.176685+0000", 2024-08-17T17:36:18.411 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.411 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.411 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.411 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.411 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:58.176685+0000", 2024-08-17T17:36:18.411 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.411 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.412 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.412 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.412 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:58.176684+0000", 2024-08-17T17:36:18.412 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:36:18.412 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.412 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.412 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.412 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:58.176742+0000", 2024-08-17T17:36:18.412 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7742999999999999e-05 2024-08-17T17:36:18.412 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.412 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.412 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.412 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:58.176752+0000", 2024-08-17T17:36:18.413 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0801e-05 2024-08-17T17:36:18.413 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.413 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.413 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.413 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:58.176758+0000", 2024-08-17T17:36:18.413 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8989999999999998e-06 2024-08-17T17:36:18.413 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.413 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.413 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.413 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:58.177898+0000", 2024-08-17T17:36:18.413 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001140255 2024-08-17T17:36:18.413 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.413 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.414 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:36:18.414 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:58.177919+0000", 2024-08-17T17:36:18.414 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0942999999999999e-05 2024-08-17T17:36:18.414 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.414 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.414 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:36:18.414 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:58.209400+0000", 2024-08-17T17:36:18.414 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.031480923000000001 2024-08-17T17:36:18.414 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.414 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.414 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.414 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:58.209449+0000", 2024-08-17T17:36:18.415 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8526000000000003e-05 2024-08-17T17:36:18.415 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.415 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.415 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.415 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.415 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.415 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:61902 11.8 e1381/1377 11:11169c47:::benchmark_data_smithi063_328699_object61901:head v 1381'3935, mlcod=0'0)", 2024-08-17T17:36:18.415 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:11.369358+0000", 2024-08-17T17:36:18.415 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 6.3712964740000002, 2024-08-17T17:36:18.415 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.033613430999999999, 2024-08-17T17:36:18.415 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.415 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:18.415 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.415 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.416 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.416 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.369358+0000", 2024-08-17T17:36:18.416 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.416 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.416 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.416 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.416 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.369358+0000", 2024-08-17T17:36:18.416 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.416 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.416 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.416 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.416 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.369357+0000", 2024-08-17T17:36:18.416 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:36:18.417 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.417 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.417 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.417 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.369382+0000", 2024-08-17T17:36:18.417 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.4967e-05 2024-08-17T17:36:18.417 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.417 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.417 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.417 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.369394+0000", 2024-08-17T17:36:18.417 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2167999999999999e-05 2024-08-17T17:36:18.417 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.417 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.418 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.418 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.369400+0000", 2024-08-17T17:36:18.418 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1970000000000001e-06 2024-08-17T17:36:18.418 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.418 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.418 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.418 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.369506+0000", 2024-08-17T17:36:18.418 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000105362 2024-08-17T17:36:18.418 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.418 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.418 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:36:18.418 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.369530+0000", 2024-08-17T17:36:18.419 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3983000000000001e-05 2024-08-17T17:36:18.419 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.419 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.419 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:36:18.419 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.402924+0000", 2024-08-17T17:36:18.419 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.033393883999999999 2024-08-17T17:36:18.419 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.419 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.419 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.419 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.402971+0000", 2024-08-17T17:36:18.419 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7814e-05 2024-08-17T17:36:18.419 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.419 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.420 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.420 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.420 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.420 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:61907 11.0 e1381/1377 11:05eafba5:::benchmark_data_smithi063_328699_object61906:head v 1381'3894, mlcod=0'0)", 2024-08-17T17:36:18.420 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:11.374475+0000", 2024-08-17T17:36:18.420 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 6.3661793449999999, 2024-08-17T17:36:18.420 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.037101919999999997, 2024-08-17T17:36:18.420 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.420 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:18.420 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.420 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.420 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.421 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.374475+0000", 2024-08-17T17:36:18.421 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.421 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.421 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.421 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.421 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.374475+0000", 2024-08-17T17:36:18.421 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.421 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.421 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.421 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.421 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.374474+0000", 2024-08-17T17:36:18.421 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:36:18.422 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.422 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.422 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.422 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.374694+0000", 2024-08-17T17:36:18.422 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00021987100000000001 2024-08-17T17:36:18.422 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.422 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.422 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.422 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.374711+0000", 2024-08-17T17:36:18.422 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7309999999999999e-05 2024-08-17T17:36:18.422 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.422 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.423 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.423 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.374717+0000", 2024-08-17T17:36:18.423 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1859999999999997e-06 2024-08-17T17:36:18.423 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.423 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.423 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.423 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.394119+0000", 2024-08-17T17:36:18.423 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.019401913 2024-08-17T17:36:18.423 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.423 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.423 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:36:18.423 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.394142+0000", 2024-08-17T17:36:18.423 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2812000000000001e-05 2024-08-17T17:36:18.424 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.424 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.424 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:36:18.424 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.411463+0000", 2024-08-17T17:36:18.424 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.017320939 2024-08-17T17:36:18.424 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.424 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.424 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.424 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.411577+0000", 2024-08-17T17:36:18.424 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000113807 2024-08-17T17:36:18.424 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.424 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.425 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.425 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.425 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.425 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:61906 11.3 e1381/1377 11:c8f0b015:::benchmark_data_smithi063_328699_object61905:head v 1381'3866, mlcod=0'0)", 2024-08-17T17:36:18.425 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:11.375357+0000", 2024-08-17T17:36:18.425 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 6.365297408, 2024-08-17T17:36:18.425 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.037991714000000003, 2024-08-17T17:36:18.425 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.425 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:18.425 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.425 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.425 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.425 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.375357+0000", 2024-08-17T17:36:18.425 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.426 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.426 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.426 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.426 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.375357+0000", 2024-08-17T17:36:18.426 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.426 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.426 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.426 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.426 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.375356+0000", 2024-08-17T17:36:18.426 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:36:18.426 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.426 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.426 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.427 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.375424+0000", 2024-08-17T17:36:18.427 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8017999999999998e-05 2024-08-17T17:36:18.427 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.427 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.427 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.427 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.393910+0000", 2024-08-17T17:36:18.427 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.018486183 2024-08-17T17:36:18.427 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.427 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.427 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.427 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.393926+0000", 2024-08-17T17:36:18.427 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.556e-05 2024-08-17T17:36:18.428 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.428 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.428 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.428 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.393974+0000", 2024-08-17T17:36:18.428 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7908000000000003e-05 2024-08-17T17:36:18.428 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.428 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.428 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:36:18.428 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.393996+0000", 2024-08-17T17:36:18.428 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2677e-05 2024-08-17T17:36:18.428 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.428 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.429 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:36:18.429 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.413313+0000", 2024-08-17T17:36:18.429 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.019316149000000001 2024-08-17T17:36:18.429 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.429 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.429 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.429 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.413349+0000", 2024-08-17T17:36:18.429 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6179e-05 2024-08-17T17:36:18.429 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.429 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.429 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.429 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.429 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.429 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:61910 11.9 e1381/1377 11:9e07e64a:::benchmark_data_smithi063_328699_object61909:head v 1381'3910, mlcod=1373'720)", 2024-08-17T17:36:18.430 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:11.376593+0000", 2024-08-17T17:36:18.430 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 6.3640610009999996, 2024-08-17T17:36:18.430 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.037775128999999998, 2024-08-17T17:36:18.430 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.430 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:18.430 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.430 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.430 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.430 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.376593+0000", 2024-08-17T17:36:18.430 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.430 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.430 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.430 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.431 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.376593+0000", 2024-08-17T17:36:18.431 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.431 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.431 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.431 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.431 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.376593+0000", 2024-08-17T17:36:18.431 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:36:18.431 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.431 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.431 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.431 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.376662+0000", 2024-08-17T17:36:18.431 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9492999999999993e-05 2024-08-17T17:36:18.432 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.432 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.432 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.432 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.395287+0000", 2024-08-17T17:36:18.432 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.018625354 2024-08-17T17:36:18.432 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.432 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.432 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.432 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.395296+0000", 2024-08-17T17:36:18.432 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8310000000000007e-06 2024-08-17T17:36:18.432 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.432 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.433 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.433 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.395825+0000", 2024-08-17T17:36:18.433 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00052915899999999997 2024-08-17T17:36:18.433 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.433 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.433 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:36:18.433 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.395852+0000", 2024-08-17T17:36:18.433 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6406000000000001e-05 2024-08-17T17:36:18.433 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.433 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.433 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:36:18.433 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.414330+0000", 2024-08-17T17:36:18.433 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.018477938999999999 2024-08-17T17:36:18.434 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.434 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.434 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.434 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.414369+0000", 2024-08-17T17:36:18.434 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8801999999999999e-05 2024-08-17T17:36:18.434 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.434 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.434 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.434 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.434 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.434 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:61916 11.8 e1381/1377 11:17407f1f:::benchmark_data_smithi063_328699_object61915:head v 1381'3937, mlcod=0'0)", 2024-08-17T17:36:18.434 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:11.381543+0000", 2024-08-17T17:36:18.435 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 6.3591117759999998, 2024-08-17T17:36:18.435 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.033927117999999999, 2024-08-17T17:36:18.435 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.435 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:18.435 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.435 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.435 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.435 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.381543+0000", 2024-08-17T17:36:18.435 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.435 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.435 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.435 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.436 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.381543+0000", 2024-08-17T17:36:18.436 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.436 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.436 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.436 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.436 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.381542+0000", 2024-08-17T17:36:18.436 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:36:18.436 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.436 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.436 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.436 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.381565+0000", 2024-08-17T17:36:18.436 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.3254000000000001e-05 2024-08-17T17:36:18.436 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.437 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.437 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.437 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.395066+0000", 2024-08-17T17:36:18.437 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.013500956 2024-08-17T17:36:18.437 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.437 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.437 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.437 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.395076+0000", 2024-08-17T17:36:18.437 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0417e-05 2024-08-17T17:36:18.437 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.437 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.437 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.437 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.398509+0000", 2024-08-17T17:36:18.437 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003432212 2024-08-17T17:36:18.438 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.438 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.438 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:36:18.438 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.398535+0000", 2024-08-17T17:36:18.438 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.6415e-05 2024-08-17T17:36:18.438 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.438 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.438 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:36:18.438 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.415434+0000", 2024-08-17T17:36:18.438 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.016899214999999999 2024-08-17T17:36:18.438 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.438 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.439 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.439 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.415470+0000", 2024-08-17T17:36:18.439 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5553999999999999e-05 2024-08-17T17:36:18.439 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.439 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.439 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.439 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.439 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.439 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:61944 11.4 e1381/1377 11:2c153d50:::benchmark_data_smithi063_328699_object61943:head v 1381'3884, mlcod=0'0)", 2024-08-17T17:36:18.439 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:11.416727+0000", 2024-08-17T17:36:18.439 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 6.3239278470000002, 2024-08-17T17:36:18.439 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.041169290999999997, 2024-08-17T17:36:18.439 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.439 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:18.440 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.440 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.440 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.440 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.416727+0000", 2024-08-17T17:36:18.440 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.440 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.440 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.440 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.440 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.416727+0000", 2024-08-17T17:36:18.440 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.440 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.440 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.440 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.441 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.416725+0000", 2024-08-17T17:36:18.441 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:36:18.441 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.441 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.441 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.441 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.416806+0000", 2024-08-17T17:36:18.441 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0668999999999996e-05 2024-08-17T17:36:18.441 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.441 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.441 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.441 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.416819+0000", 2024-08-17T17:36:18.441 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3013999999999999e-05 2024-08-17T17:36:18.441 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.441 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.442 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.442 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.416827+0000", 2024-08-17T17:36:18.442 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9519999999999994e-06 2024-08-17T17:36:18.442 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.442 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.442 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.442 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.416993+0000", 2024-08-17T17:36:18.442 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016662599999999999 2024-08-17T17:36:18.442 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.442 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.442 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:36:18.442 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.417014+0000", 2024-08-17T17:36:18.442 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0621000000000001e-05 2024-08-17T17:36:18.443 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.443 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.443 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:36:18.443 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.457855+0000", 2024-08-17T17:36:18.443 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.040840886 2024-08-17T17:36:18.443 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.443 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.443 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.443 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.457896+0000", 2024-08-17T17:36:18.443 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1010999999999997e-05 2024-08-17T17:36:18.443 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.443 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.443 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.444 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.444 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.444 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:61946 11.4 e1381/1377 11:25b47efa:::benchmark_data_smithi063_328699_object61945:head v 1381'3885, mlcod=0'0)", 2024-08-17T17:36:18.444 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:11.419787+0000", 2024-08-17T17:36:18.444 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 6.3208670319999998, 2024-08-17T17:36:18.444 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.038130519000000002, 2024-08-17T17:36:18.444 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.444 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:18.444 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.444 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.444 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.444 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.419787+0000", 2024-08-17T17:36:18.444 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.445 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.445 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.445 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.445 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.419787+0000", 2024-08-17T17:36:18.445 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.445 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.445 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.445 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.445 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.419787+0000", 2024-08-17T17:36:18.445 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:36:18.445 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.445 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.445 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.446 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.419838+0000", 2024-08-17T17:36:18.446 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1705e-05 2024-08-17T17:36:18.446 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.446 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.446 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.446 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.419849+0000", 2024-08-17T17:36:18.446 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0454e-05 2024-08-17T17:36:18.446 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.446 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.446 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.446 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.419855+0000", 2024-08-17T17:36:18.446 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.0360000000000003e-06 2024-08-17T17:36:18.446 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.447 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.447 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.447 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.423149+0000", 2024-08-17T17:36:18.447 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003294688 2024-08-17T17:36:18.447 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.447 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.447 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:36:18.447 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.423172+0000", 2024-08-17T17:36:18.447 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2619000000000001e-05 2024-08-17T17:36:18.447 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.447 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.447 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:36:18.447 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.457899+0000", 2024-08-17T17:36:18.448 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.034726656000000002 2024-08-17T17:36:18.448 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.448 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.448 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.448 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.457918+0000", 2024-08-17T17:36:18.448 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9128999999999999e-05 2024-08-17T17:36:18.448 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.448 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.448 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.448 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.448 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.448 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:61950 11.4 e1381/1377 11:2c077c0e:::benchmark_data_smithi063_328699_object61949:head v 1381'3886, mlcod=0'0)", 2024-08-17T17:36:18.448 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:11.424446+0000", 2024-08-17T17:36:18.449 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 6.3162081050000003, 2024-08-17T17:36:18.449 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.033493676, 2024-08-17T17:36:18.449 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:36:18.449 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:18.449 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:36:18.449 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.449 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:36:18.449 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.424446+0000", 2024-08-17T17:36:18.449 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.449 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.449 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.449 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:36:18.449 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.424446+0000", 2024-08-17T17:36:18.449 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:36:18.449 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.450 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.450 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:36:18.450 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.424445+0000", 2024-08-17T17:36:18.450 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:36:18.450 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.450 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.450 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:36:18.450 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.424490+0000", 2024-08-17T17:36:18.450 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.5358999999999997e-05 2024-08-17T17:36:18.450 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.450 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.450 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:36:18.451 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.424500+0000", 2024-08-17T17:36:18.451 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2329999999999995e-06 2024-08-17T17:36:18.451 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.451 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.451 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:36:18.451 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.424506+0000", 2024-08-17T17:36:18.451 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7800000000000003e-06 2024-08-17T17:36:18.451 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.451 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.451 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:36:18.451 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.429593+0000", 2024-08-17T17:36:18.451 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0050860269999999999 2024-08-17T17:36:18.451 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.451 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.452 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:36:18.452 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.429615+0000", 2024-08-17T17:36:18.452 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2577000000000001e-05 2024-08-17T17:36:18.452 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.452 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.452 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:36:18.452 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.457922+0000", 2024-08-17T17:36:18.452 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.028306946999999999 2024-08-17T17:36:18.452 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:36:18.452 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:36:18.452 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:36:18.452 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.457940+0000", 2024-08-17T17:36:18.452 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7912999999999999e-05 2024-08-17T17:36:18.452 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.453 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.453 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.453 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:36:18.453 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:36:18.453 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:36:18.453 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:18.477 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:18.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:18.578 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:18.605 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:18.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:18.668 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 78 16 62374 62358 49.9609 0 - 0.0185837 2024-08-17T17:36:18.707 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:18.732 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:18.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:18.794 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:18.834 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:18.861 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:18.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:18.963 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:18.997 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:18.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:18.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:19.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:18 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 25099/186993 objects degraded (13.422%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:19.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:18 smithi063 ceph-mon[109843]: osdmap e1389: 8 total, 8 up, 8 in 2024-08-17T17:36:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:18 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 25099/186993 objects degraded (13.422%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:18 smithi063 ceph-mon[114756]: osdmap e1389: 8 total, 8 up, 8 in 2024-08-17T17:36:19.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:18 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 25099/186993 objects degraded (13.422%), 8 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:19.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:18 smithi162 ceph-mon[104668]: osdmap e1389: 8 total, 8 up, 8 in 2024-08-17T17:36:19.099 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:19.126 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:19.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:19.228 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:19.251 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:19.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:19.353 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:19.375 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:19.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:19.477 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:19.503 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:19.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:19.605 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:19.639 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:19.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:19.668 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 79 16 62374 62358 49.3285 0 - 0.0185837 2024-08-17T17:36:19.741 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:19.775 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:19.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:19.877 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:19.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:19.901 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:19.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:20.003 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:20.025 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:20.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:20.045 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:19 smithi162 ceph-mon[104668]: pgmap v2120: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 1 unknown, 7 remapped+peering, 4 peering, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 28325/175110 objects degraded (16.176%); 9.9 MiB/s, 158 objects/s recovering 2024-08-17T17:36:20.045 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:19 smithi162 ceph-mon[104668]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:36:20.120 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:19 smithi063 ceph-mon[114756]: pgmap v2120: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 1 unknown, 7 remapped+peering, 4 peering, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 28325/175110 objects degraded (16.176%); 9.9 MiB/s, 158 objects/s recovering 2024-08-17T17:36:20.120 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:19 smithi063 ceph-mon[114756]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:36:20.120 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:19 smithi063 ceph-mon[109843]: pgmap v2120: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 1 active+undersized+remapped, 1 unknown, 7 remapped+peering, 4 peering, 7 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 28325/175110 objects degraded (16.176%); 9.9 MiB/s, 158 objects/s recovering 2024-08-17T17:36:20.120 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:19 smithi063 ceph-mon[109843]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-08-17T17:36:20.127 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:20.149 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:20.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:20.251 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:20.273 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:20.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:20.375 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:20.397 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:20.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:20.429 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:36:20.500 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:20.525 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:20.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:20.627 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:20.651 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:20.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:20.668 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:2024-08-17T17:36:20.668229+0000 min lat: 0.000948653 max lat: 32.4964 avg lat: 0.0193629 2024-08-17T17:36:20.668 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-08-17T17:36:20.668 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 80 16 62412 62396 48.7416 0.395833 0.00424076 0.0193629 2024-08-17T17:36:20.754 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:20.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.777 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:20.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:20.878 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:20.904 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:20.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:20.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:21.006 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:21.029 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:21.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:21.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:20 smithi162 ceph-mon[104668]: osdmap e1390: 8 total, 8 up, 8 in 2024-08-17T17:36:21.131 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:21.157 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:21.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:21.187 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:20 smithi063 ceph-mon[109843]: osdmap e1390: 8 total, 8 up, 8 in 2024-08-17T17:36:21.187 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:20 smithi063 ceph-mon[114756]: osdmap e1390: 8 total, 8 up, 8 in 2024-08-17T17:36:21.259 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:21.283 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:21.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:21.385 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:21.410 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:21.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:21.512 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:21.535 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:21.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:21.637 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:21.661 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:21.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:21.668 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 81 16 62412 62396 48.1398 0 - 0.0193629 2024-08-17T17:36:21.762 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:21.788 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:21.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:21.818 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:36:21.890 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:21.917 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:21.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:21.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:22.019 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:22.042 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:22.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:22.052 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:21 smithi063 ceph-mon[109843]: pgmap v2123: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 1 unknown, 7 remapped+peering, 4 peering, 7 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 28325/175137 objects degraded (16.173%); 9.2 MiB/s, 147 objects/s recovering 2024-08-17T17:36:22.052 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:21 smithi063 ceph-mon[109843]: osdmap e1391: 8 total, 8 up, 8 in 2024-08-17T17:36:22.052 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:21 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3253863010' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:22.052 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:21 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:22.052 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:21 smithi063 ceph-mon[114756]: pgmap v2123: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 1 unknown, 7 remapped+peering, 4 peering, 7 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 28325/175137 objects degraded (16.173%); 9.2 MiB/s, 147 objects/s recovering 2024-08-17T17:36:22.052 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:21 smithi063 ceph-mon[114756]: osdmap e1391: 8 total, 8 up, 8 in 2024-08-17T17:36:22.052 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:21 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3253863010' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:22.052 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:21 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:21 smithi162 ceph-mon[104668]: pgmap v2123: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 1 unknown, 7 remapped+peering, 4 peering, 7 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 28325/175137 objects degraded (16.173%); 9.2 MiB/s, 147 objects/s recovering 2024-08-17T17:36:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:21 smithi162 ceph-mon[104668]: osdmap e1391: 8 total, 8 up, 8 in 2024-08-17T17:36:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:21 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3253863010' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:21 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:36:22.144 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:22.169 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:22.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:22.272 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:22.294 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:22.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:22.396 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:22.418 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:22.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:22.520 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:22.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:22.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:22.567 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:36:22.568 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:36:22.575 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:36:22.575 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:36:22.596 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:36:22.597 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:36:22.616 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:36:22.616 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:36:22.634 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:36:22.634 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:36:22.647 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:22.652 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:36:22.652 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:36:22.668 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 82 16 62412 62396 47.5527 0 - 0.0193629 2024-08-17T17:36:22.668 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:22.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:22.770 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:22.793 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:22.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:22.894 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:22.917 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:22.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:22.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:23.018 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:22 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:22 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:36:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:22 smithi063 ceph-mon[114756]: osdmap e1392: 8 total, 8 up, 8 in 2024-08-17T17:36:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:22 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2705842615' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:23.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:22 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:23.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:22 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:36:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:22 smithi063 ceph-mon[109843]: osdmap e1392: 8 total, 8 up, 8 in 2024-08-17T17:36:23.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:22 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2705842615' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:23.042 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:23.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:22 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:22 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:36:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:22 smithi162 ceph-mon[104668]: osdmap e1392: 8 total, 8 up, 8 in 2024-08-17T17:36:23.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:22 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2705842615' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:23.103 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:36:23.103 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:36:23.128 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:36:23.129 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:36:23.147 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:23.181 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:23.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:23.283 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:23.310 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:23.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.311 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:23.412 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:23.439 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:23.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:23.541 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:23.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.567 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:23.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:23.602 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:36:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:36:23] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-08-17T17:36:23.668 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:23.669 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 83 16 62561 62545 47.092 3.10417 0.00672803 0.0211993 2024-08-17T17:36:23.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.699 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:23.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:23.801 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:23.827 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:23.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:23.913 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:23 smithi063 ceph-mon[114756]: pgmap v2126: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 activating+undersized+degraded+remapped, 7 remapped+peering, 1 active+clean+scrubbing, 7 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 27698/187170 objects degraded (14.798%); 4006/187170 objects misplaced (2.140%); 9.8 MiB/s, 157 objects/s recovering 2024-08-17T17:36:23.913 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:23 smithi063 ceph-mon[114756]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:36:23.913 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:23 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 27698/187170 objects degraded (14.798%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:23.913 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:23 smithi063 ceph-mon[114756]: osdmap e1393: 8 total, 8 up, 8 in 2024-08-17T17:36:23.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:23 smithi063 ceph-mon[109843]: pgmap v2126: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 activating+undersized+degraded+remapped, 7 remapped+peering, 1 active+clean+scrubbing, 7 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 27698/187170 objects degraded (14.798%); 4006/187170 objects misplaced (2.140%); 9.8 MiB/s, 157 objects/s recovering 2024-08-17T17:36:23.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:23 smithi063 ceph-mon[109843]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:36:23.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:23 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 27698/187170 objects degraded (14.798%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:23.913 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:23 smithi063 ceph-mon[109843]: osdmap e1393: 8 total, 8 up, 8 in 2024-08-17T17:36:23.930 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:23.955 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:23.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:23.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:23.965 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:24.057 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:24.091 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:24.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:24.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:23 smithi162 ceph-mon[104668]: pgmap v2126: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 activating+undersized+degraded+remapped, 7 remapped+peering, 1 active+clean+scrubbing, 7 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 27698/187170 objects degraded (14.798%); 4006/187170 objects misplaced (2.140%); 9.8 MiB/s, 157 objects/s recovering 2024-08-17T17:36:24.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:23 smithi162 ceph-mon[104668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-08-17T17:36:24.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:23 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 27698/187170 objects degraded (14.798%), 9 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:24.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:23 smithi162 ceph-mon[104668]: osdmap e1393: 8 total, 8 up, 8 in 2024-08-17T17:36:24.194 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:24.219 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:24.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:24.321 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:24.353 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:24.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:24.455 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:24.492 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:24.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:24.594 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:24.622 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:24.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:24.668 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 84 16 62838 62822 46.7375 17.3125 0.00542006 0.0213565 2024-08-17T17:36:24.724 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:24.752 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:24.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:24.854 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:24.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:24.882 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:24.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:24.983 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:25.009 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:25.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:25.110 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:25.112 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:36:25.145 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:25.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:25.247 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:25.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.277 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:25.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:25.330 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:25 smithi063 ceph-mon[109843]: osdmap e1394: 8 total, 8 up, 8 in 2024-08-17T17:36:25.331 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:25 smithi063 ceph-mon[109843]: pgmap v2129: 119 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 activating+undersized+degraded+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 47951/187362 objects degraded (25.593%); 4006/187362 objects misplaced (2.138%); 9.8 MiB/s, 157 objects/s recovering 2024-08-17T17:36:25.331 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:25 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:25.331 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:25 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:25.331 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:25 smithi063 ceph-mon[114756]: osdmap e1394: 8 total, 8 up, 8 in 2024-08-17T17:36:25.331 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:25 smithi063 ceph-mon[114756]: pgmap v2129: 119 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 activating+undersized+degraded+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 47951/187362 objects degraded (25.593%); 4006/187362 objects misplaced (2.138%); 9.8 MiB/s, 157 objects/s recovering 2024-08-17T17:36:25.331 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:25 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:25.331 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:25 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:25.378 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:25.410 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:25.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:25.512 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:25.537 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:25.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:25.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:25 smithi162 ceph-mon[104668]: osdmap e1394: 8 total, 8 up, 8 in 2024-08-17T17:36:25.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:25 smithi162 ceph-mon[104668]: pgmap v2129: 119 pgs: 1 active+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 activating+undersized+degraded+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 47951/187362 objects degraded (25.593%); 4006/187362 objects misplaced (2.138%); 9.8 MiB/s, 157 objects/s recovering 2024-08-17T17:36:25.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:25 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:25.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:25 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:25.639 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:25.668 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:25.669 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 85 16 63084 63068 46.3685 15.375 0.00700082 0.0215192 2024-08-17T17:36:25.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:25.771 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:25.798 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:25.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:25.899 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:25.927 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:25.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:25.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:26.029 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:26.054 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:26.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:26.151 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:36:26.156 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:26.194 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:26.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:26.296 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:26.319 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:26.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:26.421 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:26.444 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:26.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:26.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:26 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2269971207' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:36:26.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:26 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2269971207' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:36:26.547 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:26.581 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:26.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:26.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:26 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2269971207' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:36:26.668 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 86 16 63349 63333 46.0219 16.5625 0.233534 0.021699 2024-08-17T17:36:26.683 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:26.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.709 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:26.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:26.810 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:26.842 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:26.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:26.943 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:26.968 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:26.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:26.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:27.070 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:27.103 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:27.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:27.205 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:27.229 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:27.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:27.331 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:27.360 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:27.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:27.462 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:27.484 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:27 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2269971207' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:36:27.484 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:27 smithi063 ceph-mon[114756]: osdmap e1395: 8 total, 8 up, 8 in 2024-08-17T17:36:27.484 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:27 smithi063 ceph-mon[114756]: pgmap v2131: 119 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 2.5 MiB/s wr, 40 op/s; 47949/187740 objects degraded (25.540%); 4006/187740 objects misplaced (2.134%); 8.2 MiB/s, 130 objects/s recovering 2024-08-17T17:36:27.484 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:27 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2269971207' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:36:27.484 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:27 smithi063 ceph-mon[109843]: osdmap e1395: 8 total, 8 up, 8 in 2024-08-17T17:36:27.484 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:27 smithi063 ceph-mon[109843]: pgmap v2131: 119 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 2.5 MiB/s wr, 40 op/s; 47949/187740 objects degraded (25.540%); 4006/187740 objects misplaced (2.134%); 8.2 MiB/s, 130 objects/s recovering 2024-08-17T17:36:27.485 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:27.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:27.588 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:27.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:27 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2269971207' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:36:27.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:27 smithi162 ceph-mon[104668]: osdmap e1395: 8 total, 8 up, 8 in 2024-08-17T17:36:27.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:27 smithi162 ceph-mon[104668]: pgmap v2131: 119 pgs: 1 active+recovery_wait+remapped, 1 active+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 2.5 MiB/s wr, 40 op/s; 47949/187740 objects degraded (25.540%); 4006/187740 objects misplaced (2.134%); 8.2 MiB/s, 130 objects/s recovering 2024-08-17T17:36:27.619 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:27.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:27.669 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 87 16 63659 63643 45.7156 19.375 0.00648678 0.0218404 2024-08-17T17:36:27.721 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:27.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.752 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:27.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:27.853 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:27.881 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:27.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:27.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:27.984 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:28.011 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:28.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:28.113 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:28.130 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:36:28.141 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:28.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:28.243 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:28.273 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:28.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:28.370 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:28 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 47949/187740 objects degraded (25.540%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:28.370 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:28 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 47949/187740 objects degraded (25.540%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:28.376 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:28.379 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:28.405 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:28.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:28.507 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:28.536 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:28.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:28.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:28 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 47949/187740 objects degraded (25.540%), 12 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:28.639 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:28.657 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:28.658 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:28.666 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:28.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:28.669 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 88 16 63936 63920 45.3928 17.3125 0.0065802 0.0219934 2024-08-17T17:36:28.768 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:28.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.799 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:28.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:28.900 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:28.926 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:28.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:28.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:29.028 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:29.051 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:29.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:29.153 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:29.187 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:29.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:29.249 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:29 smithi063 ceph-mon[114756]: pgmap v2132: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 11 MiB/s wr, 181 op/s; 47199/190434 objects degraded (24.785%); 4006/190434 objects misplaced (2.104%); 7.8 MiB/s, 125 objects/s recovering 2024-08-17T17:36:29.250 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:29 smithi063 ceph-mon[109843]: pgmap v2132: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 11 MiB/s wr, 181 op/s; 47199/190434 objects degraded (24.785%); 4006/190434 objects misplaced (2.104%); 7.8 MiB/s, 125 objects/s recovering 2024-08-17T17:36:29.289 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:29.311 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:29.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:29.413 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:29.439 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:29.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:29.508 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:36:29.540 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:29.567 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:29.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:29.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:29 smithi162 ceph-mon[104668]: pgmap v2132: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 11 MiB/s wr, 181 op/s; 47199/190434 objects degraded (24.785%); 4006/190434 objects misplaced (2.104%); 7.8 MiB/s, 125 objects/s recovering 2024-08-17T17:36:29.669 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 89 16 64190 64174 45.0611 15.875 0.0962074 0.022137 2024-08-17T17:36:29.669 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:29.699 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:29.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:29.801 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:29.832 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:29.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:29.933 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:29.965 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:29.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:29.966 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:30.022 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:36:30.022 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:30.067 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:30.092 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:30.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:30.196 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:30.219 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:30.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:30.321 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:30.343 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:30.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:30.444 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:30.469 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:30.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:30.571 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:30.595 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:30.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:30.669 INFO:tasks.radosbench.radosbench.0.smithi063.stdout: 90 12 64440 64428 44.7368 15.875 0.0906238 0.0223199 2024-08-17T17:36:30.697 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:30.722 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:30.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:30.824 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:30.847 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:30.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:30.949 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:30.971 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:30.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:30.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:31.073 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:31.095 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:31.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.096 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:31.196 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:31.218 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:31.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:31.320 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:31.341 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:31.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:31.443 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:31.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.465 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:31.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:31.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:31.590 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:31.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:31.669 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Total time run: 90.2388 2024-08-17T17:36:31.669 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Total writes made: 64440 2024-08-17T17:36:31.670 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Write size: 65536 2024-08-17T17:36:31.670 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Object size: 65536 2024-08-17T17:36:31.670 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Bandwidth (MB/sec): 44.6316 2024-08-17T17:36:31.670 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev Bandwidth: 59.1963 2024-08-17T17:36:31.670 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max bandwidth (MB/sec): 221.25 2024-08-17T17:36:31.670 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min bandwidth (MB/sec): 0 2024-08-17T17:36:31.670 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Average IOPS: 714 2024-08-17T17:36:31.670 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev IOPS: 947.16 2024-08-17T17:36:31.670 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max IOPS: 3540 2024-08-17T17:36:31.670 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min IOPS: 0 2024-08-17T17:36:31.670 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Average Latency(s): 0.0223652 2024-08-17T17:36:31.671 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Stddev Latency(s): 0.511026 2024-08-17T17:36:31.671 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Max latency(s): 32.4964 2024-08-17T17:36:31.671 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Min latency(s): 0.000948653 2024-08-17T17:36:31.689 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Cleaning up (deleting benchmark objects) 2024-08-17T17:36:31.692 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:31.719 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:31.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:31.821 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:31.847 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:31.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:31.949 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:31.973 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:31.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:31.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:31 smithi063 ceph-mon[114756]: pgmap v2133: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 10 MiB/s wr, 166 op/s; 47199/190968 objects degraded (24.716%); 4006/190968 objects misplaced (2.098%); 6.3 MiB/s, 100 objects/s recovering 2024-08-17T17:36:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:32.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:31 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1203073234' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:31 smithi063 ceph-mon[109843]: pgmap v2133: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 10 MiB/s wr, 166 op/s; 47199/190968 objects degraded (24.716%); 4006/190968 objects misplaced (2.098%); 6.3 MiB/s, 100 objects/s recovering 2024-08-17T17:36:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:32.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:31 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1203073234' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:32.075 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:31 smithi162 ceph-mon[104668]: pgmap v2133: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 10 MiB/s wr, 166 op/s; 47199/190968 objects degraded (24.716%); 4006/190968 objects misplaced (2.098%); 6.3 MiB/s, 100 objects/s recovering 2024-08-17T17:36:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:31 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1203073234' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:36:32.102 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:32.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:32.205 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:32.228 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:32.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:32.329 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:32.353 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:32.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:32.455 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:32.478 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:32.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:32.580 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:32.602 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:32.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:32.704 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:32.734 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:32.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:32.836 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:32.864 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:32.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:32.966 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:32.991 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:32.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:32.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:32 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:32 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1203073234' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:33.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:32 smithi063 ceph-mon[109843]: osdmap e1396: 8 total, 8 up, 8 in 2024-08-17T17:36:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:32 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:33.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:32 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1203073234' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:33.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:32 smithi063 ceph-mon[114756]: osdmap e1396: 8 total, 8 up, 8 in 2024-08-17T17:36:33.089 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:36:33.093 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:32 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:32 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1203073234' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:36:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:32 smithi162 ceph-mon[104668]: osdmap e1396: 8 total, 8 up, 8 in 2024-08-17T17:36:33.116 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:33.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:33.218 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:33.246 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:33.290 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:33.347 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:33.371 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:33.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:33.394 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:36:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:36:33] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-17T17:36:33.474 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:33.499 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:33.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:33.601 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:33.626 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:33.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:33.728 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:33.754 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:33.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:33.856 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:33.879 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:33.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:33.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:33.949 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:33 smithi063 ceph-mon[114756]: pgmap v2135: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 11 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 12 MiB/s wr, 197 op/s; 46399/192033 objects degraded (24.162%); 4006/192033 objects misplaced (2.086%); 12 MiB/s, 193 objects/s recovering 2024-08-17T17:36:33.949 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:33 smithi063 ceph-mon[114756]: osdmap e1397: 8 total, 8 up, 8 in 2024-08-17T17:36:33.949 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:33 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 46399/192033 objects degraded (24.162%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:33.950 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:33 smithi063 ceph-mon[109843]: pgmap v2135: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 11 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 12 MiB/s wr, 197 op/s; 46399/192033 objects degraded (24.162%); 4006/192033 objects misplaced (2.086%); 12 MiB/s, 193 objects/s recovering 2024-08-17T17:36:33.950 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:33 smithi063 ceph-mon[109843]: osdmap e1397: 8 total, 8 up, 8 in 2024-08-17T17:36:33.950 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:33 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 46399/192033 objects degraded (24.162%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:33.981 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:34.004 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:34.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:34.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:33 smithi162 ceph-mon[104668]: pgmap v2135: 119 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 11 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 12 MiB/s wr, 197 op/s; 46399/192033 objects degraded (24.162%); 4006/192033 objects misplaced (2.086%); 12 MiB/s, 193 objects/s recovering 2024-08-17T17:36:34.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:33 smithi162 ceph-mon[104668]: osdmap e1397: 8 total, 8 up, 8 in 2024-08-17T17:36:34.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:33 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 46399/192033 objects degraded (24.162%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:34.106 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:34.130 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:34.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:34.232 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:34.256 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:34.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:34.357 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:34.383 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:34.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:34.485 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:34.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.511 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:34.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:34.612 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:34.637 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:34.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:34.739 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:34.762 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:34.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:34.864 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:34.889 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:34.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:34.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:34.991 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:35.019 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:35.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:35.121 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:35.144 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:35.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:35.246 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:35.274 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:35.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:35.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:35.404 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:35.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:35.506 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:35.532 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:35.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:35.634 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:35.659 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:35.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:35.761 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:35.788 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:35.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:35.821 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:36:35.932 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:35.950 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:35 smithi162 ceph-mon[104668]: pgmap v2137: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 10 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 127 B/s rd, 15 MiB/s wr, 261 op/s; 45854/192798 objects degraded (23.783%); 4006/192798 objects misplaced (2.078%); 16 MiB/s, 255 objects/s recovering 2024-08-17T17:36:35.956 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:35 smithi063 ceph-mon[114756]: pgmap v2137: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 10 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 127 B/s rd, 15 MiB/s wr, 261 op/s; 45854/192798 objects degraded (23.783%); 4006/192798 objects misplaced (2.078%); 16 MiB/s, 255 objects/s recovering 2024-08-17T17:36:35.956 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:35 smithi063 ceph-mon[109843]: pgmap v2137: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 10 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 127 B/s rd, 15 MiB/s wr, 261 op/s; 45854/192798 objects degraded (23.783%); 4006/192798 objects misplaced (2.078%); 16 MiB/s, 255 objects/s recovering 2024-08-17T17:36:35.959 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:35.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:35.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:36.060 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:36.084 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:36.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:36.186 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:36.213 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:36.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:36.315 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:36.338 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:36.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:36.440 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:36.465 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:36.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:36.567 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:36.596 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:36.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:36.657 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:36.698 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:36.725 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:36.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:36.826 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:36.851 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:36.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:36.954 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:36.981 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:36.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:36.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:37.083 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:37.112 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:37.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:37.214 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:37.241 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:37.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:37.343 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:37.371 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:37.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:37.473 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:37.499 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:37.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:37.601 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:37.626 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:37.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:37.709 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:36:37.729 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:37.759 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:37.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:37.861 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:37.890 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:37.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:37.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:37.992 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:38.017 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:38.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:37 smithi063 ceph-mon[114756]: pgmap v2138: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 peering, 10 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 127 B/s rd, 7.7 MiB/s wr, 162 op/s; 45854/192462 objects degraded (23.825%); 4006/192462 objects misplaced (2.081%); 10 MiB/s, 161 objects/s recovering 2024-08-17T17:36:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:37 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2226343741' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:36:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:37 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:36:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:37 smithi063 ceph-mon[109843]: pgmap v2138: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 peering, 10 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 127 B/s rd, 7.7 MiB/s wr, 162 op/s; 45854/192462 objects degraded (23.825%); 4006/192462 objects misplaced (2.081%); 10 MiB/s, 161 objects/s recovering 2024-08-17T17:36:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:37 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2226343741' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:36:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:37 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:36:38.078 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:37 smithi162 ceph-mon[104668]: pgmap v2138: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 peering, 10 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 127 B/s rd, 7.7 MiB/s wr, 162 op/s; 45854/192462 objects degraded (23.825%); 4006/192462 objects misplaced (2.081%); 10 MiB/s, 161 objects/s recovering 2024-08-17T17:36:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:37 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2226343741' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:36:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:37 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:36:38.119 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:38.143 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:38.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:38.245 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:38.268 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:38.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:38.370 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:38.394 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:38.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:38.496 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:38.518 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:38.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:38.620 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:38.630 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:36:38.645 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:38.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:38.747 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:38.779 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:38.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:38.832 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:36:38.832 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:36:38.881 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:38.905 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:38.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:38.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:38.937 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:38 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:36:38.937 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:38 smithi063 ceph-mon[109843]: osdmap e1398: 8 total, 8 up, 8 in 2024-08-17T17:36:38.937 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:38 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 45854/192462 objects degraded (23.825%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:38.937 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:38 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:36:38.937 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:38 smithi063 ceph-mon[114756]: osdmap e1398: 8 total, 8 up, 8 in 2024-08-17T17:36:38.937 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:38 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 45854/192462 objects degraded (23.825%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:39.007 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:39.030 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:39.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:39.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:38 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:36:39.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:38 smithi162 ceph-mon[104668]: osdmap e1398: 8 total, 8 up, 8 in 2024-08-17T17:36:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:38 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 45854/192462 objects degraded (23.825%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:39.132 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:39.155 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:39.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:39.257 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:39.281 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:39.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:39.383 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:39.406 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:39.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:39.508 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:39.531 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:39.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:39.634 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:39.658 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:39.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:39.760 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:39.784 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:39.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:39.886 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:39.890 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:39 smithi063 ceph-mon[114756]: pgmap v2140: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 147 B/s rd, 4.9 MiB/s wr, 190 op/s; 45292/191079 objects degraded (23.703%); 4006/191079 objects misplaced (2.097%); 17 MiB/s, 267 objects/s recovering 2024-08-17T17:36:39.891 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:39 smithi063 ceph-mon[109843]: pgmap v2140: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 147 B/s rd, 4.9 MiB/s wr, 190 op/s; 45292/191079 objects degraded (23.703%); 4006/191079 objects misplaced (2.097%); 17 MiB/s, 267 objects/s recovering 2024-08-17T17:36:39.911 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:39.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:39.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:40.013 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:40.040 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:40.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:39 smithi162 ceph-mon[104668]: pgmap v2140: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 147 B/s rd, 4.9 MiB/s wr, 190 op/s; 45292/191079 objects degraded (23.703%); 4006/191079 objects misplaced (2.097%); 17 MiB/s, 267 objects/s recovering 2024-08-17T17:36:40.142 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:40.169 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:40.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:40.272 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:40.296 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:40.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:40.399 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:40.424 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:40.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:40.526 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:40.550 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:40.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:40.651 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:40.676 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:40.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:40.778 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:40.802 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:40.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:40.904 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:40.929 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:40.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:40.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:41.031 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:41.059 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:41.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.060 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:41.161 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:41.186 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:41.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:41.288 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:41.318 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:41.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:41.420 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:41.448 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:41.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:41.550 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:41.577 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:41.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:41.679 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:41.709 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:41.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:41.740 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:36:41.811 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:41.841 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:41.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:41.943 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:41.961 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:41 smithi063 ceph-mon[109843]: pgmap v2141: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 127 B/s rd, 3.5 MiB/s wr, 166 op/s; 45041/190707 objects degraded (23.618%); 4006/190707 objects misplaced (2.101%); 9.2 MiB/s, 147 objects/s recovering 2024-08-17T17:36:41.961 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:41 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:41.961 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:41 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:41.961 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:41 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:36:41.961 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:41 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:36:41.961 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:41 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:41.967 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:41.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:41.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:42.069 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:42.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.097 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:42.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:42.198 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:42.223 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:42.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:42.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:41 smithi063 ceph-mon[114756]: pgmap v2141: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 127 B/s rd, 3.5 MiB/s wr, 166 op/s; 45041/190707 objects degraded (23.618%); 4006/190707 objects misplaced (2.101%); 9.2 MiB/s, 147 objects/s recovering 2024-08-17T17:36:42.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:41 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:42.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:41 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:42.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:41 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:36:42.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:41 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:36:42.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:41 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:42.325 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:41 smithi162 ceph-mon[104668]: pgmap v2141: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 127 B/s rd, 3.5 MiB/s wr, 166 op/s; 45041/190707 objects degraded (23.618%); 4006/190707 objects misplaced (2.101%); 9.2 MiB/s, 147 objects/s recovering 2024-08-17T17:36:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:41 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:41 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:41 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:36:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:41 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:36:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:41 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:36:42.349 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:42.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:42.451 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:42.475 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:42.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:42.577 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:42.604 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:42.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:42.706 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:42.731 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:42.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:42.833 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:42.859 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:42.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:42.888 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:42.962 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:42.985 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:42.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:42.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:43.087 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:43.114 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:43.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:43.216 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:43.242 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:43.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:43.343 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:43.367 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:43.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:43.469 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:43.495 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:43.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:43.509 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:36:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:36:43] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.51.0" 2024-08-17T17:36:43.597 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:43.622 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:43.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:43.724 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:43.751 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:43.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:43.853 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:43.879 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:43.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:43.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:43.981 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:44.009 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:44.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:44.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:36:44.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:36:44.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:36:44.039 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:36:44.039 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:36:44.039 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:36:44.039 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:36:44.039 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:36:44.039 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:36:44.039 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:36:44.039 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:36:44.039 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:36:44.039 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:36:44.040 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:36:44.040 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:36:44.040 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:36:44.111 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:44.134 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:44.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:44.236 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:44.262 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:44.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:44.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:43 smithi063 ceph-mon[114756]: pgmap v2142: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 10 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 103 B/s rd, 2.8 MiB/s wr, 156 op/s; 45041/190041 objects degraded (23.701%); 4006/190041 objects misplaced (2.108%); 7.5 MiB/s, 119 objects/s recovering 2024-08-17T17:36:44.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:43 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 45041/190041 objects degraded (23.701%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:44.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:43 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2507587347' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:44.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:43 smithi063 ceph-mon[109843]: pgmap v2142: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 10 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 103 B/s rd, 2.8 MiB/s wr, 156 op/s; 45041/190041 objects degraded (23.701%); 4006/190041 objects misplaced (2.108%); 7.5 MiB/s, 119 objects/s recovering 2024-08-17T17:36:44.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:43 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 45041/190041 objects degraded (23.701%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:44.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:43 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2507587347' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:44.284 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:43 smithi162 ceph-mon[104668]: pgmap v2142: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 10 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 103 B/s rd, 2.8 MiB/s wr, 156 op/s; 45041/190041 objects degraded (23.701%); 4006/190041 objects misplaced (2.108%); 7.5 MiB/s, 119 objects/s recovering 2024-08-17T17:36:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:43 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 45041/190041 objects degraded (23.701%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:43 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2507587347' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:36:44.364 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:44.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.390 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:44.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:44.491 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:44.517 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:44.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:44.619 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:44.644 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:44.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:44.746 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:44.773 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:44.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:44.875 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:44.901 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:44.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:44.902 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:45.003 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:45.028 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:45.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:45.130 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:45.154 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:45.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:45.256 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:45.281 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:45.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:45.383 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:45.411 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:45.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:45.513 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:45.541 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:45.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:45.643 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:45.667 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:45.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:45.768 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:45.794 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:45.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:45.896 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:45.922 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:45.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:45.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:45.965 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:36:46.024 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:46.049 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:46.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:46.151 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:46.173 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:46.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:46.275 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:45 smithi063 ceph-mon[114756]: pgmap v2143: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 10 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 140 op/s; 43803/188571 objects degraded (23.229%); 4006/188571 objects misplaced (2.124%); 11 MiB/s, 179 objects/s recovering 2024-08-17T17:36:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:45 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1109719802' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:36:46.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:45 smithi063 ceph-mon[109843]: pgmap v2143: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 10 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 140 op/s; 43803/188571 objects degraded (23.229%); 4006/188571 objects misplaced (2.124%); 11 MiB/s, 179 objects/s recovering 2024-08-17T17:36:46.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:45 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1109719802' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:36:46.302 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:46.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:46.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:45 smithi162 ceph-mon[104668]: pgmap v2143: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 10 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 140 op/s; 43803/188571 objects degraded (23.229%); 4006/188571 objects misplaced (2.124%); 11 MiB/s, 179 objects/s recovering 2024-08-17T17:36:46.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:45 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1109719802' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:36:46.404 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:46.431 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:46.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:46.533 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:46.558 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:46.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:46.660 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:46.686 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:46.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:46.788 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:46.810 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:46.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:46.912 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:46.934 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:46.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:46.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:47.036 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:47.071 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:47.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:47.089 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:36:47.173 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:47.198 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:47.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.199 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:47.248 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:46 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:47.248 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:46 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1109719802' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:36:47.248 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:46 smithi063 ceph-mon[114756]: osdmap e1399: 8 total, 8 up, 8 in 2024-08-17T17:36:47.248 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:47.248 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:46 smithi063 ceph-mon[114756]: osdmap e1400: 8 total, 8 up, 8 in 2024-08-17T17:36:47.249 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:46 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:47.249 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:46 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1109719802' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:36:47.249 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:46 smithi063 ceph-mon[109843]: osdmap e1399: 8 total, 8 up, 8 in 2024-08-17T17:36:47.249 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:47.249 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:46 smithi063 ceph-mon[109843]: osdmap e1400: 8 total, 8 up, 8 in 2024-08-17T17:36:47.299 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:47.323 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:47.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:47.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:46 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:36:47.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:46 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1109719802' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:36:47.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:46 smithi162 ceph-mon[104668]: osdmap e1399: 8 total, 8 up, 8 in 2024-08-17T17:36:47.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:36:47.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:46 smithi162 ceph-mon[104668]: osdmap e1400: 8 total, 8 up, 8 in 2024-08-17T17:36:47.425 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:47.449 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:47.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:47.551 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:47.577 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:47.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:47.679 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:47.705 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:47.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:47.806 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:47.831 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:47.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:47.933 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:47.961 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:47.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:47.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:48.063 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:48.090 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:48.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:48.192 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:48.218 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:48.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:48.254 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:47 smithi063 ceph-mon[114756]: pgmap v2145: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 10 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 139 op/s; 43803/188184 objects degraded (23.277%); 4006/188184 objects misplaced (2.129%); 8.6 MiB/s, 138 objects/s recovering 2024-08-17T17:36:48.255 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:47 smithi063 ceph-mon[109843]: pgmap v2145: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 10 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 139 op/s; 43803/188184 objects degraded (23.277%); 4006/188184 objects misplaced (2.129%); 8.6 MiB/s, 138 objects/s recovering 2024-08-17T17:36:48.320 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:48.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:47 smithi162 ceph-mon[104668]: pgmap v2145: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 10 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 139 op/s; 43803/188184 objects degraded (23.277%); 4006/188184 objects misplaced (2.129%); 8.6 MiB/s, 138 objects/s recovering 2024-08-17T17:36:48.346 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:48.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:48.448 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:48.471 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:48.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:48.573 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:48.597 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:48.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:48.699 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:48.726 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:48.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.727 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:48.828 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:48.853 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:48.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:48.956 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:48.983 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:48.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:48.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:48.997 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:36:49.056 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:36:49.085 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:49.111 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:49.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:49.213 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:49.246 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:49.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:49.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:49 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3462958899' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:36:49.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:49 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 43803/188184 objects degraded (23.277%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:49.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:49 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3462958899' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:36:49.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:49 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 43803/188184 objects degraded (23.277%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:49.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:48 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3462958899' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:36:49.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:48 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 43803/188184 objects degraded (23.277%), 11 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:49.348 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:49.373 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:49.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:49.475 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:49.501 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:49.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:49.602 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:49.628 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:49.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:49.729 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:49.754 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:49.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:49.856 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:49.879 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:49.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:49.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:49.981 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:50.009 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:50.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:50.110 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:50.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.135 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:50.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:50.236 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:50.260 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:50.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:50 smithi063 ceph-mon[114756]: pgmap v2147: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 41117/186927 objects degraded (21.996%); 6442/186927 objects misplaced (3.446%); 11 MiB/s, 169 objects/s recovering 2024-08-17T17:36:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:50 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3462958899' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:36:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:50 smithi063 ceph-mon[114756]: osdmap e1401: 8 total, 8 up, 8 in 2024-08-17T17:36:50.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:49 smithi063 ceph-mon[109843]: pgmap v2147: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 41117/186927 objects degraded (21.996%); 6442/186927 objects misplaced (3.446%); 11 MiB/s, 169 objects/s recovering 2024-08-17T17:36:50.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:49 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3462958899' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:36:50.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:49 smithi063 ceph-mon[109843]: osdmap e1401: 8 total, 8 up, 8 in 2024-08-17T17:36:50.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:49 smithi162 ceph-mon[104668]: pgmap v2147: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 41117/186927 objects degraded (21.996%); 6442/186927 objects misplaced (3.446%); 11 MiB/s, 169 objects/s recovering 2024-08-17T17:36:50.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:49 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3462958899' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:36:50.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:49 smithi162 ceph-mon[104668]: osdmap e1401: 8 total, 8 up, 8 in 2024-08-17T17:36:50.362 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:50.389 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:50.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:50.491 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:50.517 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:50.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:50.520 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:36:50.619 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:50.647 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:50.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:50.749 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:50.778 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:50.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:50.879 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:50.905 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:50.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:50.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:51.007 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:51.032 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:51.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.033 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:51.045 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:51.053 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:36:51.053 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:36:51.053 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:36:51.053 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:36:51.054 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.054 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:194842 10.12 e1315/1306 10:4a1efe0c:::benchmark_data_smithi063_294527_object57657:head v 1315'10202, mlcod=1315'10200)", 2024-08-17T17:36:51.054 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.321450+0000", 2024-08-17T17:36:51.054 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 227.726634182, 2024-08-17T17:36:51.054 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41592500300000002, 2024-08-17T17:36:51.054 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.054 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.054 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.054 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.054 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.054 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.055 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321450+0000", 2024-08-17T17:36:51.055 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.055 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.055 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.055 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.055 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321450+0000", 2024-08-17T17:36:51.055 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.055 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.055 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.055 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.055 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321451+0000", 2024-08-17T17:36:51.055 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.67e-07 2024-08-17T17:36:51.055 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.055 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.056 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.056 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321454+0000", 2024-08-17T17:36:51.056 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5300000000000001e-06 2024-08-17T17:36:51.056 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.056 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.056 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.056 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321460+0000", 2024-08-17T17:36:51.056 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6330000000000001e-06 2024-08-17T17:36:51.056 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.056 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.056 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.056 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321465+0000", 2024-08-17T17:36:51.056 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9300000000000002e-06 2024-08-17T17:36:51.057 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.057 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.057 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.057 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321512+0000", 2024-08-17T17:36:51.057 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7454999999999999e-05 2024-08-17T17:36:51.057 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.057 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.057 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.057 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321531+0000", 2024-08-17T17:36:51.057 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8476999999999999e-05 2024-08-17T17:36:51.057 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.057 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.058 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.058 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.737340+0000", 2024-08-17T17:36:51.058 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41580924000000002 2024-08-17T17:36:51.058 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.058 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.058 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.058 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.737375+0000", 2024-08-17T17:36:51.058 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.5471e-05 2024-08-17T17:36:51.058 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.058 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.058 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.058 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.058 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.059 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:194846 10.b e1315/1288 10:dcd470cd:::benchmark_data_smithi063_294527_object57661:head v 1315'12297, mlcod=1315'12295)", 2024-08-17T17:36:51.059 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.321477+0000", 2024-08-17T17:36:51.059 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 227.72660804099999, 2024-08-17T17:36:51.059 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41552626500000001, 2024-08-17T17:36:51.059 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.059 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.059 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.059 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.059 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.059 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.059 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321477+0000", 2024-08-17T17:36:51.059 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.060 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.060 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.060 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.060 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321477+0000", 2024-08-17T17:36:51.060 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.060 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.060 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.060 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.060 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321477+0000", 2024-08-17T17:36:51.060 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6600000000000003e-07 2024-08-17T17:36:51.060 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.060 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.060 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.060 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321484+0000", 2024-08-17T17:36:51.061 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9129999999999997e-06 2024-08-17T17:36:51.061 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.061 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.061 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.061 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321491+0000", 2024-08-17T17:36:51.061 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3150000000000003e-06 2024-08-17T17:36:51.061 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.061 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.061 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.061 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321496+0000", 2024-08-17T17:36:51.061 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2000000000000002e-06 2024-08-17T17:36:51.061 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.062 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.062 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.062 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321546+0000", 2024-08-17T17:36:51.062 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9416000000000003e-05 2024-08-17T17:36:51.062 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.062 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.062 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.062 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321565+0000", 2024-08-17T17:36:51.062 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9171e-05 2024-08-17T17:36:51.063 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.063 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.063 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.063 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.736955+0000", 2024-08-17T17:36:51.063 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41538978599999998 2024-08-17T17:36:51.063 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.063 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.063 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.063 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.737003+0000", 2024-08-17T17:36:51.063 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8198000000000002e-05 2024-08-17T17:36:51.063 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.063 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.063 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.064 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.064 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.064 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:194840 10.4 e1315/1306 10:21446871:::benchmark_data_smithi063_294527_object57655:head v 1315'10259, mlcod=1315'10257)", 2024-08-17T17:36:51.064 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.321510+0000", 2024-08-17T17:36:51.064 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 227.72657485600001, 2024-08-17T17:36:51.064 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.416863237, 2024-08-17T17:36:51.064 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.064 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.064 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.064 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.064 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.064 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.064 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321510+0000", 2024-08-17T17:36:51.065 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.065 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.065 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.065 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.065 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321510+0000", 2024-08-17T17:36:51.065 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.065 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.065 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.065 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.065 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321510+0000", 2024-08-17T17:36:51.065 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5699999999999999e-07 2024-08-17T17:36:51.065 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.065 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.065 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.066 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321512+0000", 2024-08-17T17:36:51.066 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.34e-06 2024-08-17T17:36:51.066 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.066 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.066 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.066 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321516+0000", 2024-08-17T17:36:51.066 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8940000000000003e-06 2024-08-17T17:36:51.066 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.066 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.066 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.066 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321519+0000", 2024-08-17T17:36:51.066 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.5890000000000001e-06 2024-08-17T17:36:51.066 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.066 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.067 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.067 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321564+0000", 2024-08-17T17:36:51.067 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4878e-05 2024-08-17T17:36:51.067 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.067 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.067 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.067 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.321579+0000", 2024-08-17T17:36:51.067 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5821999999999999e-05 2024-08-17T17:36:51.067 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.067 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.067 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.067 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.738341+0000", 2024-08-17T17:36:51.067 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41676199000000003 2024-08-17T17:36:51.067 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.068 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.068 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.068 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.738373+0000", 2024-08-17T17:36:51.068 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1566999999999998e-05 2024-08-17T17:36:51.068 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.068 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.068 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.068 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.068 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.068 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:194844 10.12 e1315/1306 10:49a75d7b:::benchmark_data_smithi063_294527_object57659:head v 1315'10203, mlcod=1315'10200)", 2024-08-17T17:36:51.068 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.322823+0000", 2024-08-17T17:36:51.068 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 227.725261617, 2024-08-17T17:36:51.069 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41833177599999999, 2024-08-17T17:36:51.069 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.069 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.069 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.069 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.069 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.069 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.069 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.322823+0000", 2024-08-17T17:36:51.069 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.069 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.069 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.069 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.069 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.322823+0000", 2024-08-17T17:36:51.069 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.070 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.070 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.070 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.070 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.322824+0000", 2024-08-17T17:36:51.070 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6700000000000003e-07 2024-08-17T17:36:51.070 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.070 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.070 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.070 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.322830+0000", 2024-08-17T17:36:51.070 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1789999999999996e-06 2024-08-17T17:36:51.070 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.070 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.070 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.071 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.322840+0000", 2024-08-17T17:36:51.071 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0342e-05 2024-08-17T17:36:51.071 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.071 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.071 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.071 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.322847+0000", 2024-08-17T17:36:51.071 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8770000000000004e-06 2024-08-17T17:36:51.071 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.071 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.071 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.071 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.323796+0000", 2024-08-17T17:36:51.071 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00094924600000000005 2024-08-17T17:36:51.071 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.072 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.072 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.072 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.323880+0000", 2024-08-17T17:36:51.072 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3496000000000002e-05 2024-08-17T17:36:51.072 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.072 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.072 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.072 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.741122+0000", 2024-08-17T17:36:51.072 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41724261200000001 2024-08-17T17:36:51.072 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.072 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.072 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.072 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.741155+0000", 2024-08-17T17:36:51.072 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.2357000000000002e-05 2024-08-17T17:36:51.072 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.073 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.073 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.073 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.073 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.073 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:194849 10.c e1315/1312 10:38a65d47:::benchmark_data_smithi063_294527_object57664:head v 1315'12025, mlcod=1315'12023)", 2024-08-17T17:36:51.073 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.324072+0000", 2024-08-17T17:36:51.073 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 227.72401228800001, 2024-08-17T17:36:51.073 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41871652599999998, 2024-08-17T17:36:51.073 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.073 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.073 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.073 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.073 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.073 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.073 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.324072+0000", 2024-08-17T17:36:51.074 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.074 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.074 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.074 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.074 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.324072+0000", 2024-08-17T17:36:51.074 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.074 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.074 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.074 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.074 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.324073+0000", 2024-08-17T17:36:51.074 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3600000000000004e-07 2024-08-17T17:36:51.074 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.074 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.074 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.075 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.324078+0000", 2024-08-17T17:36:51.075 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.7129999999999996e-06 2024-08-17T17:36:51.075 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.075 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.075 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.075 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.324087+0000", 2024-08-17T17:36:51.075 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.149e-06 2024-08-17T17:36:51.075 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.075 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.075 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.075 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.324092+0000", 2024-08-17T17:36:51.075 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6450000000000002e-06 2024-08-17T17:36:51.075 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.076 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.076 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.076 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.324161+0000", 2024-08-17T17:36:51.076 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8206000000000004e-05 2024-08-17T17:36:51.076 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.076 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.076 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.076 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.736990+0000", 2024-08-17T17:36:51.076 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41282972299999998 2024-08-17T17:36:51.076 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.076 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.076 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.076 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.742729+0000", 2024-08-17T17:36:51.077 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0057386039999999996 2024-08-17T17:36:51.077 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.077 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.077 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.077 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.742789+0000", 2024-08-17T17:36:51.077 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.995e-05 2024-08-17T17:36:51.077 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.077 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.077 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.077 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.077 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.077 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.45818.0:194852 10.15 e1315/1306 10:a9b1e74c:::benchmark_data_smithi063_294527_object57667:head v 1315'10134, mlcod=1315'10132)", 2024-08-17T17:36:51.077 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:33:03.325815+0000", 2024-08-17T17:36:51.078 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 227.722270009, 2024-08-17T17:36:51.078 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.41519754199999998, 2024-08-17T17:36:51.078 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.078 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.078 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.078 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.078 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.078 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.078 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325815+0000", 2024-08-17T17:36:51.078 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.078 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.078 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.078 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.079 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325815+0000", 2024-08-17T17:36:51.079 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.079 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.079 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.079 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.079 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325815+0000", 2024-08-17T17:36:51.079 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2900000000000003e-07 2024-08-17T17:36:51.079 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.079 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.079 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.079 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.325822+0000", 2024-08-17T17:36:51.079 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8159999999999996e-06 2024-08-17T17:36:51.079 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.079 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.079 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.080 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.736930+0000", 2024-08-17T17:36:51.080 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.411107687 2024-08-17T17:36:51.080 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.080 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.080 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.080 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.736944+0000", 2024-08-17T17:36:51.080 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3869999999999999e-05 2024-08-17T17:36:51.080 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.080 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.080 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.080 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.737286+0000", 2024-08-17T17:36:51.080 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00034230699999999997 2024-08-17T17:36:51.080 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.080 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.080 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.081 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.737305+0000", 2024-08-17T17:36:51.081 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9351000000000002e-05 2024-08-17T17:36:51.081 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.081 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.081 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.081 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.740983+0000", 2024-08-17T17:36:51.081 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0036774379999999999 2024-08-17T17:36:51.081 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.081 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.081 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.081 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:33:03.741012+0000", 2024-08-17T17:36:51.081 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.9343999999999999e-05 2024-08-17T17:36:51.081 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.081 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.081 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.082 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.082 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.082 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267660 10.5 10:a665299f:::benchmark_data_smithi063_294527_object130475:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1350)", 2024-08-17T17:36:51.082 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:21.777080+0000", 2024-08-17T17:36:51.082 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 149.27100485899999, 2024-08-17T17:36:51.082 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.96950791000000003, 2024-08-17T17:36:51.082 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.082 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.082 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.082 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:36:51.083 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:36:51.083 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:36:51.083 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267660 2024-08-17T17:36:51.083 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.083 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.083 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.084 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.084 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777080+0000", 2024-08-17T17:36:51.084 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.084 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.084 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.084 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.084 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777080+0000", 2024-08-17T17:36:51.084 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.084 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.084 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.084 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.084 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777081+0000", 2024-08-17T17:36:51.084 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5290000000000001e-06 2024-08-17T17:36:51.084 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.084 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.085 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.085 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777087+0000", 2024-08-17T17:36:51.085 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7819999999999999e-06 2024-08-17T17:36:51.085 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.085 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.085 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.085 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777103+0000", 2024-08-17T17:36:51.085 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6328e-05 2024-08-17T17:36:51.085 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.085 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.085 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.085 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777111+0000", 2024-08-17T17:36:51.085 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4880000000000001e-06 2024-08-17T17:36:51.086 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.086 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.086 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.086 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777174+0000", 2024-08-17T17:36:51.086 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3415000000000002e-05 2024-08-17T17:36:51.086 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.086 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.086 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.086 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777183+0000", 2024-08-17T17:36:51.086 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.1039999999999996e-06 2024-08-17T17:36:51.086 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.086 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.086 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.087 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.735896+0000", 2024-08-17T17:36:51.087 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95871258599999998 2024-08-17T17:36:51.087 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.087 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.087 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.087 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.735970+0000", 2024-08-17T17:36:51.087 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3584999999999994e-05 2024-08-17T17:36:51.087 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.087 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.087 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:36:51.087 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.736040+0000", 2024-08-17T17:36:51.087 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0869000000000002e-05 2024-08-17T17:36:51.087 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.087 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.088 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:36:51.088 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.743286+0000", 2024-08-17T17:36:51.088 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0072453639999999998 2024-08-17T17:36:51.088 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.088 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.088 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.088 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.743339+0000", 2024-08-17T17:36:51.088 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3399999999999997e-05 2024-08-17T17:36:51.088 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.088 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.088 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.088 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.746509+0000", 2024-08-17T17:36:51.088 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0031696350000000001 2024-08-17T17:36:51.089 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.089 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.089 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.089 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.746553+0000", 2024-08-17T17:36:51.089 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4338999999999997e-05 2024-08-17T17:36:51.089 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.089 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.089 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.089 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.746588+0000", 2024-08-17T17:36:51.089 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4486000000000002e-05 2024-08-17T17:36:51.089 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.089 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.089 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.090 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.090 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.090 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267665 10.15 10:add28e57:::benchmark_data_smithi063_294527_object130480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1350)", 2024-08-17T17:36:51.090 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:21.782485+0000", 2024-08-17T17:36:51.090 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 149.26559999099999, 2024-08-17T17:36:51.090 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.961089796, 2024-08-17T17:36:51.090 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.090 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.090 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.090 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:36:51.090 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:36:51.090 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:36:51.091 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267665 2024-08-17T17:36:51.091 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.091 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.091 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.091 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.091 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.782485+0000", 2024-08-17T17:36:51.091 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.091 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.091 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.091 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.091 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.782485+0000", 2024-08-17T17:36:51.091 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.091 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.091 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.092 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.092 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.782486+0000", 2024-08-17T17:36:51.092 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4219999999999999e-06 2024-08-17T17:36:51.092 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.092 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.092 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.092 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.782495+0000", 2024-08-17T17:36:51.092 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3449999999999995e-06 2024-08-17T17:36:51.092 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.092 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.092 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.092 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.782511+0000", 2024-08-17T17:36:51.092 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5631e-05 2024-08-17T17:36:51.092 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.093 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.093 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.093 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.782519+0000", 2024-08-17T17:36:51.093 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2320000000000001e-06 2024-08-17T17:36:51.093 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.093 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.093 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.093 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.782601+0000", 2024-08-17T17:36:51.093 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.1465999999999996e-05 2024-08-17T17:36:51.093 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.093 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.093 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.093 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.782612+0000", 2024-08-17T17:36:51.093 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1415e-05 2024-08-17T17:36:51.094 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.094 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.094 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.094 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.733425+0000", 2024-08-17T17:36:51.094 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95081299600000002 2024-08-17T17:36:51.094 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.094 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.094 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.094 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.733512+0000", 2024-08-17T17:36:51.094 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7131000000000003e-05 2024-08-17T17:36:51.094 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.094 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.094 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:36:51.095 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.733885+0000", 2024-08-17T17:36:51.095 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00037229299999999999 2024-08-17T17:36:51.095 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.095 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.095 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:36:51.095 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.737257+0000", 2024-08-17T17:36:51.095 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0033723690000000001 2024-08-17T17:36:51.095 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.095 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.095 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.095 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.737280+0000", 2024-08-17T17:36:51.095 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3008999999999999e-05 2024-08-17T17:36:51.096 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.096 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.096 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.096 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.743390+0000", 2024-08-17T17:36:51.096 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0061096500000000003 2024-08-17T17:36:51.096 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.096 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.096 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.096 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.743505+0000", 2024-08-17T17:36:51.096 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011558800000000001 2024-08-17T17:36:51.096 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.096 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.096 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.097 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.743574+0000", 2024-08-17T17:36:51.097 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9249000000000001e-05 2024-08-17T17:36:51.097 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.097 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.097 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.097 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.097 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.097 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267670 10.5 10:a6d6fd20:::benchmark_data_smithi063_294527_object130485:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1350)", 2024-08-17T17:36:51.097 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:21.789818+0000", 2024-08-17T17:36:51.097 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 149.25826632499999, 2024-08-17T17:36:51.097 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95889019099999995, 2024-08-17T17:36:51.097 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.098 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.098 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.098 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:36:51.098 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:36:51.098 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:36:51.098 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267670 2024-08-17T17:36:51.098 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.098 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.098 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.098 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.098 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789818+0000", 2024-08-17T17:36:51.098 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.099 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.099 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.099 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.099 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789818+0000", 2024-08-17T17:36:51.099 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.099 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.099 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.099 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.099 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789820+0000", 2024-08-17T17:36:51.099 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4160000000000001e-06 2024-08-17T17:36:51.099 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.099 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.099 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.099 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789825+0000", 2024-08-17T17:36:51.099 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5300000000000004e-06 2024-08-17T17:36:51.100 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.100 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.100 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.100 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789840+0000", 2024-08-17T17:36:51.100 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4455e-05 2024-08-17T17:36:51.100 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.100 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.100 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.100 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789847+0000", 2024-08-17T17:36:51.100 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6350000000000006e-06 2024-08-17T17:36:51.100 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.100 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.100 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.100 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789902+0000", 2024-08-17T17:36:51.101 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4741000000000003e-05 2024-08-17T17:36:51.101 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.101 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.101 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.101 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789937+0000", 2024-08-17T17:36:51.101 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4922999999999998e-05 2024-08-17T17:36:51.101 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.101 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.101 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.101 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.738261+0000", 2024-08-17T17:36:51.101 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.94832413100000001 2024-08-17T17:36:51.101 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.101 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.101 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.102 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.738405+0000", 2024-08-17T17:36:51.102 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014333099999999999 2024-08-17T17:36:51.102 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.102 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.102 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:36:51.102 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.738515+0000", 2024-08-17T17:36:51.102 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011097799999999999 2024-08-17T17:36:51.102 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.102 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.102 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.102 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.743444+0000", 2024-08-17T17:36:51.102 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0049283549999999997 2024-08-17T17:36:51.102 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.102 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.103 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:36:51.103 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.743456+0000", 2024-08-17T17:36:51.103 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1907e-05 2024-08-17T17:36:51.103 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.103 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.103 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.103 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.748619+0000", 2024-08-17T17:36:51.103 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0051628330000000004 2024-08-17T17:36:51.103 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.103 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.103 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.103 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.748668+0000", 2024-08-17T17:36:51.103 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9845999999999997e-05 2024-08-17T17:36:51.103 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.103 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.748709+0000", 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.0110000000000001e-05 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267698 10.5 10:a71e15da:::benchmark_data_smithi063_294527_object130513:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1350)", 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:21.825912+0000", 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 149.222172836, 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.92291534900000005, 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.104 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:36:51.105 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:36:51.105 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:36:51.105 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267698 2024-08-17T17:36:51.105 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.105 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.105 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.105 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.105 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.825912+0000", 2024-08-17T17:36:51.105 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.105 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.105 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.105 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.105 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.825912+0000", 2024-08-17T17:36:51.105 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.105 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.106 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.106 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.106 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.825913+0000", 2024-08-17T17:36:51.106 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1629999999999999e-06 2024-08-17T17:36:51.106 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.106 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.106 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.106 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.825917+0000", 2024-08-17T17:36:51.106 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3610000000000003e-06 2024-08-17T17:36:51.106 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.106 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.106 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.106 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.825939+0000", 2024-08-17T17:36:51.107 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1840000000000001e-05 2024-08-17T17:36:51.107 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.107 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.107 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.107 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.825946+0000", 2024-08-17T17:36:51.107 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1980000000000004e-06 2024-08-17T17:36:51.107 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.107 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.107 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.107 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.826003+0000", 2024-08-17T17:36:51.107 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6981000000000002e-05 2024-08-17T17:36:51.107 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.107 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.107 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.108 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.826012+0000", 2024-08-17T17:36:51.108 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6659999999999997e-06 2024-08-17T17:36:51.108 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.108 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.108 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.108 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.741047+0000", 2024-08-17T17:36:51.108 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.91503502999999997 2024-08-17T17:36:51.108 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.108 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.108 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.108 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.741183+0000", 2024-08-17T17:36:51.108 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013637100000000001 2024-08-17T17:36:51.108 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.108 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.108 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:36:51.109 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.741294+0000", 2024-08-17T17:36:51.109 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000111038 2024-08-17T17:36:51.109 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.109 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.109 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.109 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.746323+0000", 2024-08-17T17:36:51.109 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0050281550000000003 2024-08-17T17:36:51.109 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.109 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.109 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:36:51.109 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.746335+0000", 2024-08-17T17:36:51.109 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.252e-05 2024-08-17T17:36:51.109 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.109 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.109 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.110 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.748748+0000", 2024-08-17T17:36:51.110 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00241253 2024-08-17T17:36:51.110 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.110 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.110 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.110 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.748793+0000", 2024-08-17T17:36:51.110 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4818999999999999e-05 2024-08-17T17:36:51.110 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.110 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.110 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.110 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.748827+0000", 2024-08-17T17:36:51.110 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4677e-05 2024-08-17T17:36:51.110 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.111 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.111 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.111 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.111 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.111 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267715 10.5 10:a3d579c8:::benchmark_data_smithi063_294527_object130530:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1350)", 2024-08-17T17:36:51.111 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:21.843115+0000", 2024-08-17T17:36:51.111 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 149.20496926000001, 2024-08-17T17:36:51.111 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.90817942699999998, 2024-08-17T17:36:51.111 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.111 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.111 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.111 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:36:51.111 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:36:51.112 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:36:51.112 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267715 2024-08-17T17:36:51.112 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.112 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.112 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.112 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.112 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.843115+0000", 2024-08-17T17:36:51.112 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.112 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.112 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.112 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.112 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.843115+0000", 2024-08-17T17:36:51.112 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.112 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.112 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.113 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.113 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.843117+0000", 2024-08-17T17:36:51.113 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.621e-06 2024-08-17T17:36:51.113 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.113 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.113 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.113 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.843122+0000", 2024-08-17T17:36:51.113 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1839999999999998e-06 2024-08-17T17:36:51.113 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.113 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.113 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.113 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.843140+0000", 2024-08-17T17:36:51.113 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7708999999999999e-05 2024-08-17T17:36:51.113 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.113 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.114 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.114 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.843147+0000", 2024-08-17T17:36:51.114 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4580000000000004e-06 2024-08-17T17:36:51.114 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.114 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.114 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.114 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.843270+0000", 2024-08-17T17:36:51.114 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012255499999999999 2024-08-17T17:36:51.114 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.114 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.114 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.114 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.843281+0000", 2024-08-17T17:36:51.114 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0726e-05 2024-08-17T17:36:51.114 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.115 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.115 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.115 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.743481+0000", 2024-08-17T17:36:51.115 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.90020064499999997 2024-08-17T17:36:51.115 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.115 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.115 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.115 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.743601+0000", 2024-08-17T17:36:51.115 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000119707 2024-08-17T17:36:51.115 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.115 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.115 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:36:51.115 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.743706+0000", 2024-08-17T17:36:51.115 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010514399999999999 2024-08-17T17:36:51.115 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.749671+0000", 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0059646609999999996 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.749806+0000", 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000134819 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.751169+0000", 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00136287 2024-08-17T17:36:51.116 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.117 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.117 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.117 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.751222+0000", 2024-08-17T17:36:51.117 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3838000000000002e-05 2024-08-17T17:36:51.117 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.117 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.117 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.117 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.751295+0000", 2024-08-17T17:36:51.117 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2490000000000006e-05 2024-08-17T17:36:51.117 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.117 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.117 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.117 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.117 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.118 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267728 10.15 10:adc187b6:::benchmark_data_smithi063_294527_object130543:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1350)", 2024-08-17T17:36:51.118 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:21.867146+0000", 2024-08-17T17:36:51.118 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 149.180938392, 2024-08-17T17:36:51.118 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.87932255699999995, 2024-08-17T17:36:51.118 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.118 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.118 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.118 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:36:51.118 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:36:51.118 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:36:51.118 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267728 2024-08-17T17:36:51.118 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.118 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.118 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.119 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.119 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.867146+0000", 2024-08-17T17:36:51.119 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.119 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.119 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.119 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.119 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.867146+0000", 2024-08-17T17:36:51.119 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.119 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.119 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.119 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.119 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.867148+0000", 2024-08-17T17:36:51.119 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5349999999999999e-06 2024-08-17T17:36:51.120 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.120 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.120 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.120 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.867154+0000", 2024-08-17T17:36:51.120 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1140000000000002e-06 2024-08-17T17:36:51.120 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.120 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.120 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.120 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.867167+0000", 2024-08-17T17:36:51.120 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3540000000000001e-05 2024-08-17T17:36:51.120 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.120 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.120 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.120 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.867173+0000", 2024-08-17T17:36:51.120 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9780000000000002e-06 2024-08-17T17:36:51.121 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.121 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.121 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.121 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.867227+0000", 2024-08-17T17:36:51.121 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3520999999999998e-05 2024-08-17T17:36:51.121 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.121 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.121 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.121 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.867235+0000", 2024-08-17T17:36:51.121 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6170000000000001e-06 2024-08-17T17:36:51.121 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.121 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.121 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.122 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.735887+0000", 2024-08-17T17:36:51.122 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.868651952 2024-08-17T17:36:51.122 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.122 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.122 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.122 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.735963+0000", 2024-08-17T17:36:51.122 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6861999999999997e-05 2024-08-17T17:36:51.122 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.122 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.122 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 5,7", 2024-08-17T17:36:51.122 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.736173+0000", 2024-08-17T17:36:51.122 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000209366 2024-08-17T17:36:51.122 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.123 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.123 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:36:51.123 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.741016+0000", 2024-08-17T17:36:51.123 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0048427369999999997 2024-08-17T17:36:51.123 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.123 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.123 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.123 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.741072+0000", 2024-08-17T17:36:51.123 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6857999999999997e-05 2024-08-17T17:36:51.123 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.123 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.123 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.123 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.746369+0000", 2024-08-17T17:36:51.123 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0052963460000000004 2024-08-17T17:36:51.123 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.124 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.124 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.124 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.746426+0000", 2024-08-17T17:36:51.124 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7259999999999997e-05 2024-08-17T17:36:51.124 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.124 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.124 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.124 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:22.746469+0000", 2024-08-17T17:36:51.124 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.2870999999999997e-05 2024-08-17T17:36:51.124 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.124 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.124 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.124 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.124 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.125 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:45440 11.9 11:92e9954c:::benchmark_data_smithi063_328699_object45439:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1377)", 2024-08-17T17:36:51.125 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:55.193017+0000", 2024-08-17T17:36:51.125 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 55.855067708999997, 2024-08-17T17:36:51.125 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2299891139999999, 2024-08-17T17:36:51.125 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.125 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.125 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:36:51.126 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:36:51.126 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:36:51.126 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:36:51.126 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 45440 2024-08-17T17:36:51.126 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.126 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.126 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.126 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.126 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193017+0000", 2024-08-17T17:36:51.126 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.126 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.126 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.126 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.126 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193017+0000", 2024-08-17T17:36:51.126 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.127 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.127 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.127 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.127 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193019+0000", 2024-08-17T17:36:51.127 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8020000000000001e-06 2024-08-17T17:36:51.127 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.127 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.127 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.127 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193049+0000", 2024-08-17T17:36:51.127 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0513000000000001e-05 2024-08-17T17:36:51.127 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.127 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.127 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.127 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193069+0000", 2024-08-17T17:36:51.128 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9352999999999999e-05 2024-08-17T17:36:51.128 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.128 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.128 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.128 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193079+0000", 2024-08-17T17:36:51.128 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0845e-05 2024-08-17T17:36:51.128 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.128 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.128 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.128 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193217+0000", 2024-08-17T17:36:51.128 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013719700000000001 2024-08-17T17:36:51.128 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.128 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.128 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.129 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193227+0000", 2024-08-17T17:36:51.129 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0059e-05 2024-08-17T17:36:51.129 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.129 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.129 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.129 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.422882+0000", 2024-08-17T17:36:51.129 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2296553239999999 2024-08-17T17:36:51.129 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.129 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.129 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.129 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.423006+0000", 2024-08-17T17:36:51.129 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012402100000000001 2024-08-17T17:36:51.130 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.130 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.130 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.130 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.130 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.130 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:45448 11.9 11:9b8410c7:::benchmark_data_smithi063_328699_object45447:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1377)", 2024-08-17T17:36:51.130 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:55.193104+0000", 2024-08-17T17:36:51.130 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 55.854980971000003, 2024-08-17T17:36:51.130 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.231468666, 2024-08-17T17:36:51.130 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.130 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.130 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:36:51.130 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:36:51.130 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:36:51.131 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:36:51.131 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 45448 2024-08-17T17:36:51.131 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.131 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.131 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.131 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.131 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193104+0000", 2024-08-17T17:36:51.131 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.131 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.131 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.131 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.131 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193104+0000", 2024-08-17T17:36:51.131 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.131 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.131 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.132 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.132 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193104+0000", 2024-08-17T17:36:51.132 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.44e-07 2024-08-17T17:36:51.132 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.132 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.132 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.132 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193127+0000", 2024-08-17T17:36:51.132 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2549e-05 2024-08-17T17:36:51.132 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.132 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.132 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.132 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193136+0000", 2024-08-17T17:36:51.132 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7069999999999998e-06 2024-08-17T17:36:51.132 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.133 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.133 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.133 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193142+0000", 2024-08-17T17:36:51.133 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3230000000000003e-06 2024-08-17T17:36:51.133 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.133 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.133 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.133 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193254+0000", 2024-08-17T17:36:51.133 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000111681 2024-08-17T17:36:51.133 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.133 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.133 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.133 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.193260+0000", 2024-08-17T17:36:51.133 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9369999999999998e-06 2024-08-17T17:36:51.133 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.134 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.134 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.134 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.424511+0000", 2024-08-17T17:36:51.134 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.231250473 2024-08-17T17:36:51.134 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.134 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.134 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.134 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.424572+0000", 2024-08-17T17:36:51.134 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1451999999999994e-05 2024-08-17T17:36:51.134 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.134 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.134 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.134 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.134 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.135 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:62282 11.b 11:dacfc13a:::benchmark_data_smithi063_328699_object62281:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1383)", 2024-08-17T17:36:51.135 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:36:12.742240+0000", 2024-08-17T17:36:51.135 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 38.305844317000002, 2024-08-17T17:36:51.135 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3387509399999999, 2024-08-17T17:36:51.135 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.135 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.135 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.135 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:36:51.135 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:36:51.135 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:36:51.135 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 62282 2024-08-17T17:36:51.135 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.135 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.135 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.136 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.136 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742240+0000", 2024-08-17T17:36:51.136 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.136 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.136 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.136 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.136 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742240+0000", 2024-08-17T17:36:51.136 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.136 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.136 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.136 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.136 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742243+0000", 2024-08-17T17:36:51.136 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.362e-06 2024-08-17T17:36:51.136 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.137 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.137 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.137 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742274+0000", 2024-08-17T17:36:51.137 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.0927999999999999e-05 2024-08-17T17:36:51.137 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.137 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.137 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.137 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742289+0000", 2024-08-17T17:36:51.137 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5124e-05 2024-08-17T17:36:51.137 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.137 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.137 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.137 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742300+0000", 2024-08-17T17:36:51.137 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.112e-05 2024-08-17T17:36:51.137 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.138 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.138 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.138 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.743225+0000", 2024-08-17T17:36:51.138 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00092529099999999996 2024-08-17T17:36:51.138 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.138 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.138 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.138 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.743235+0000", 2024-08-17T17:36:51.138 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:51.139 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9180000000000006e-06 2024-08-17T17:36:51.139 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.139 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.139 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.139 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.070996+0000", 2024-08-17T17:36:51.139 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.327760641 2024-08-17T17:36:51.139 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.139 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.139 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.140 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.072103+0000", 2024-08-17T17:36:51.140 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001107385 2024-08-17T17:36:51.140 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.140 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.140 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:51.140 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.072207+0000", 2024-08-17T17:36:51.140 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010385799999999999 2024-08-17T17:36:51.140 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.140 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.140 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.140 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.078882+0000", 2024-08-17T17:36:51.140 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0066751400000000004 2024-08-17T17:36:51.140 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.140 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.140 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:36:51.141 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.078893+0000", 2024-08-17T17:36:51.141 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0859999999999999e-05 2024-08-17T17:36:51.141 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.141 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.141 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.141 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.078944+0000", 2024-08-17T17:36:51.141 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0809000000000001e-05 2024-08-17T17:36:51.141 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.141 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.141 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.141 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.079003+0000", 2024-08-17T17:36:51.141 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8959000000000003e-05 2024-08-17T17:36:51.141 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.141 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.142 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.142 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.080991+0000", 2024-08-17T17:36:51.142 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001988545 2024-08-17T17:36:51.142 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.142 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.142 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.142 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.142 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.142 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:62286 11.b 11:d14b6dea:::benchmark_data_smithi063_328699_object62285:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1383)", 2024-08-17T17:36:51.142 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:36:12.742333+0000", 2024-08-17T17:36:51.142 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 38.305751712000003, 2024-08-17T17:36:51.142 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.355419994, 2024-08-17T17:36:51.142 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.143 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.143 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.143 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:36:51.143 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:36:51.143 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:36:51.143 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 62286 2024-08-17T17:36:51.143 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.143 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.143 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.143 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.143 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742333+0000", 2024-08-17T17:36:51.143 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.143 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.143 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.144 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.144 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742333+0000", 2024-08-17T17:36:51.144 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.144 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.144 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.144 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.144 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742334+0000", 2024-08-17T17:36:51.144 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7700000000000004e-07 2024-08-17T17:36:51.144 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.144 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.144 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.144 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742360+0000", 2024-08-17T17:36:51.144 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6512999999999998e-05 2024-08-17T17:36:51.145 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.145 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.145 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.145 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742374+0000", 2024-08-17T17:36:51.145 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3689999999999999e-05 2024-08-17T17:36:51.145 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.145 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.145 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.145 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742381+0000", 2024-08-17T17:36:51.145 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1760000000000004e-06 2024-08-17T17:36:51.145 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.145 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.145 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.145 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.743270+0000", 2024-08-17T17:36:51.145 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00088935299999999995 2024-08-17T17:36:51.146 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.146 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.146 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.146 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.743279+0000", 2024-08-17T17:36:51.146 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4999999999999999e-06 2024-08-17T17:36:51.146 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.146 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.146 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.146 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.076605+0000", 2024-08-17T17:36:51.146 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3333258569999999 2024-08-17T17:36:51.146 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.146 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.146 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.147 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.076716+0000", 2024-08-17T17:36:51.147 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011156 2024-08-17T17:36:51.147 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.147 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.147 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:51.147 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.076816+0000", 2024-08-17T17:36:51.147 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9840000000000006e-05 2024-08-17T17:36:51.147 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.147 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.147 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.147 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.083331+0000", 2024-08-17T17:36:51.147 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0065153609999999999 2024-08-17T17:36:51.148 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.148 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.148 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:36:51.148 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.085677+0000", 2024-08-17T17:36:51.148 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00234542 2024-08-17T17:36:51.148 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.148 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.148 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.148 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.095402+0000", 2024-08-17T17:36:51.148 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0097249569999999994 2024-08-17T17:36:51.148 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.148 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.148 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.149 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.095446+0000", 2024-08-17T17:36:51.149 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.4079000000000003e-05 2024-08-17T17:36:51.149 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.149 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.149 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.149 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.097753+0000", 2024-08-17T17:36:51.149 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0023071110000000001 2024-08-17T17:36:51.149 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.149 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.149 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.149 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.149 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.149 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:62291 11.b 11:db026c7d:::benchmark_data_smithi063_328699_object62290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1383)", 2024-08-17T17:36:51.150 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:36:12.742391+0000", 2024-08-17T17:36:51.150 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 38.305693282, 2024-08-17T17:36:51.150 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.357514978, 2024-08-17T17:36:51.150 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.150 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.150 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.150 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:36:51.150 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:36:51.150 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:36:51.150 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 62291 2024-08-17T17:36:51.150 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.150 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.150 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.150 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.151 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742391+0000", 2024-08-17T17:36:51.151 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.151 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.151 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.151 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.151 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742391+0000", 2024-08-17T17:36:51.151 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.151 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.151 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.151 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.151 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742392+0000", 2024-08-17T17:36:51.151 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.44e-07 2024-08-17T17:36:51.151 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.151 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.152 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.152 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742416+0000", 2024-08-17T17:36:51.152 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4133000000000001e-05 2024-08-17T17:36:51.152 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.152 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.152 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.152 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742428+0000", 2024-08-17T17:36:51.152 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1525999999999999e-05 2024-08-17T17:36:51.152 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.152 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.152 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.152 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742433+0000", 2024-08-17T17:36:51.152 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5849999999999999e-06 2024-08-17T17:36:51.153 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.153 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.153 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.153 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.743302+0000", 2024-08-17T17:36:51.153 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00086895400000000002 2024-08-17T17:36:51.153 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.153 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.153 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.153 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.743308+0000", 2024-08-17T17:36:51.153 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3559999999999998e-06 2024-08-17T17:36:51.153 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.153 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.153 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.154 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.081036+0000", 2024-08-17T17:36:51.154 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3377278020000001 2024-08-17T17:36:51.154 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.154 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.154 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.154 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.081160+0000", 2024-08-17T17:36:51.154 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012334299999999999 2024-08-17T17:36:51.154 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.154 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.154 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:51.154 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.081263+0000", 2024-08-17T17:36:51.154 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010368499999999999 2024-08-17T17:36:51.155 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.155 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.155 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.155 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.087741+0000", 2024-08-17T17:36:51.155 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0064777289999999998 2024-08-17T17:36:51.155 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.155 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.155 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:36:51.155 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.087859+0000", 2024-08-17T17:36:51.155 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000117855 2024-08-17T17:36:51.155 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.155 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.155 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.156 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.097805+0000", 2024-08-17T17:36:51.156 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0099464089999999998 2024-08-17T17:36:51.156 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.156 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.156 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.156 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.097843+0000", 2024-08-17T17:36:51.156 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7237999999999998e-05 2024-08-17T17:36:51.156 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.156 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.156 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.156 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.099906+0000", 2024-08-17T17:36:51.156 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0020638190000000002 2024-08-17T17:36:51.156 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.157 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.157 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.157 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.157 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.157 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:62295 11.b 11:d7a3447f:::benchmark_data_smithi063_328699_object62294:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1383)", 2024-08-17T17:36:51.157 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:36:12.742442+0000", 2024-08-17T17:36:51.157 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 38.305642935000002, 2024-08-17T17:36:51.157 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.359753024, 2024-08-17T17:36:51.157 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.157 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.157 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.157 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:36:51.157 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:36:51.158 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:36:51.158 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 62295 2024-08-17T17:36:51.158 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.158 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.158 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.158 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.158 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742442+0000", 2024-08-17T17:36:51.158 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.158 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.158 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.158 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.159 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742442+0000", 2024-08-17T17:36:51.159 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.159 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.159 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.159 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.159 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742442+0000", 2024-08-17T17:36:51.159 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4899999999999995e-07 2024-08-17T17:36:51.159 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.159 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.159 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.159 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742466+0000", 2024-08-17T17:36:51.159 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3387000000000002e-05 2024-08-17T17:36:51.159 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.159 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.160 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.160 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742473+0000", 2024-08-17T17:36:51.160 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3300000000000001e-06 2024-08-17T17:36:51.160 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.160 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.160 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.160 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742478+0000", 2024-08-17T17:36:51.160 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2530000000000004e-06 2024-08-17T17:36:51.160 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.160 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.160 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.160 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.743330+0000", 2024-08-17T17:36:51.160 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00085178100000000002 2024-08-17T17:36:51.160 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.161 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.161 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.161 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.743337+0000", 2024-08-17T17:36:51.161 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5300000000000002e-06 2024-08-17T17:36:51.161 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.161 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.161 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.161 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.083368+0000", 2024-08-17T17:36:51.161 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3400309349999999 2024-08-17T17:36:51.161 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.161 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.161 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.162 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.083535+0000", 2024-08-17T17:36:51.162 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016720299999999999 2024-08-17T17:36:51.162 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.162 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.162 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:51.162 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.083684+0000", 2024-08-17T17:36:51.162 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014911899999999999 2024-08-17T17:36:51.162 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.162 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.162 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.162 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.087814+0000", 2024-08-17T17:36:51.162 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0041303049999999999 2024-08-17T17:36:51.162 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.163 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.163 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:36:51.163 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.087869+0000", 2024-08-17T17:36:51.163 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5047999999999999e-05 2024-08-17T17:36:51.163 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.163 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.163 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.163 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.099977+0000", 2024-08-17T17:36:51.163 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.012107498 2024-08-17T17:36:51.163 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.163 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.163 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.163 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.100038+0000", 2024-08-17T17:36:51.163 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1731999999999998e-05 2024-08-17T17:36:51.164 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.164 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.164 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.164 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.102195+0000", 2024-08-17T17:36:51.164 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0021563540000000001 2024-08-17T17:36:51.164 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.164 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.164 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.164 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.164 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.164 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:62300 11.b 11:d2c6eaaa:::benchmark_data_smithi063_328699_object62299:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1383)", 2024-08-17T17:36:51.164 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:36:12.742486+0000", 2024-08-17T17:36:51.164 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 38.305598918999998, 2024-08-17T17:36:51.165 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3609220909999999, 2024-08-17T17:36:51.165 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.165 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.165 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.165 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:36:51.165 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:36:51.165 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:36:51.165 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 62300 2024-08-17T17:36:51.165 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.165 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.165 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.165 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.166 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742486+0000", 2024-08-17T17:36:51.166 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.166 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.166 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.166 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.166 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742486+0000", 2024-08-17T17:36:51.166 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.166 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.166 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.166 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.166 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742486+0000", 2024-08-17T17:36:51.166 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5200000000000002e-07 2024-08-17T17:36:51.166 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.167 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.167 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.167 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742583+0000", 2024-08-17T17:36:51.167 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7120999999999994e-05 2024-08-17T17:36:51.167 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.167 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.167 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.167 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742595+0000", 2024-08-17T17:36:51.167 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2101e-05 2024-08-17T17:36:51.167 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.167 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.167 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.168 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.742602+0000", 2024-08-17T17:36:51.168 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.229e-06 2024-08-17T17:36:51.168 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.168 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.168 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.168 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.743358+0000", 2024-08-17T17:36:51.168 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00075607300000000001 2024-08-17T17:36:51.168 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.168 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.168 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.168 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.743364+0000", 2024-08-17T17:36:51.168 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1809999999999998e-06 2024-08-17T17:36:51.168 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.168 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.169 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.169 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.085724+0000", 2024-08-17T17:36:51.169 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3423604659999999 2024-08-17T17:36:51.169 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.169 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.169 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.169 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.085869+0000", 2024-08-17T17:36:51.169 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014451000000000001 2024-08-17T17:36:51.169 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.169 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.169 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:51.169 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.085986+0000", 2024-08-17T17:36:51.169 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000116821 2024-08-17T17:36:51.169 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.170 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.170 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:36:51.170 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.087876+0000", 2024-08-17T17:36:51.170 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0018899839999999999 2024-08-17T17:36:51.170 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.170 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.170 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.170 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.090171+0000", 2024-08-17T17:36:51.170 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0022952419999999999 2024-08-17T17:36:51.170 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.170 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.170 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.170 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.102249+0000", 2024-08-17T17:36:51.170 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.012077676000000001 2024-08-17T17:36:51.171 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.171 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.171 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.171 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.102286+0000", 2024-08-17T17:36:51.171 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.7030000000000003e-05 2024-08-17T17:36:51.171 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.171 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.171 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.171 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.103408+0000", 2024-08-17T17:36:51.171 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001122205 2024-08-17T17:36:51.171 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.171 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.171 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.172 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.172 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.172 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:62306 11.b 11:dfd96a93:::benchmark_data_smithi063_328699_object62305:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1383)", 2024-08-17T17:36:51.172 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:36:12.889720+0000", 2024-08-17T17:36:51.172 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 38.158364185000003, 2024-08-17T17:36:51.172 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2144806420000001, 2024-08-17T17:36:51.172 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:36:51.172 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:36:51.172 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:36:51.172 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:36:51.172 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:36:51.172 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:36:51.172 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 62306 2024-08-17T17:36:51.173 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.173 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:36:51.173 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.173 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:36:51.173 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.889720+0000", 2024-08-17T17:36:51.173 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.173 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.173 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.173 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:36:51.173 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.889720+0000", 2024-08-17T17:36:51.173 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:36:51.173 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.173 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.174 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:36:51.174 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.889722+0000", 2024-08-17T17:36:51.174 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9589999999999998e-06 2024-08-17T17:36:51.174 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.174 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.174 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:36:51.174 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.889751+0000", 2024-08-17T17:36:51.174 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8900000000000001e-05 2024-08-17T17:36:51.174 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.174 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.174 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:36:51.174 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.889772+0000", 2024-08-17T17:36:51.174 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0491000000000001e-05 2024-08-17T17:36:51.175 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.175 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.175 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:36:51.175 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.889781+0000", 2024-08-17T17:36:51.175 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.8340000000000005e-06 2024-08-17T17:36:51.175 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.175 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.175 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.175 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.889849+0000", 2024-08-17T17:36:51.175 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7892999999999995e-05 2024-08-17T17:36:51.175 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.175 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.175 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:36:51.175 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.889857+0000", 2024-08-17T17:36:51.176 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.14e-06 2024-08-17T17:36:51.176 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.176 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.176 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:36:51.176 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.087902+0000", 2024-08-17T17:36:51.176 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1980456959999999 2024-08-17T17:36:51.176 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.176 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.176 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:36:51.176 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.088057+0000", 2024-08-17T17:36:51.176 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000154291 2024-08-17T17:36:51.176 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.176 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.176 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,3", 2024-08-17T17:36:51.177 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.088200+0000", 2024-08-17T17:36:51.177 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014331599999999999 2024-08-17T17:36:51.177 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.177 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.177 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:36:51.177 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.090452+0000", 2024-08-17T17:36:51.177 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0022523679999999998 2024-08-17T17:36:51.177 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.177 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.177 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.177 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.090543+0000", 2024-08-17T17:36:51.177 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0704000000000004e-05 2024-08-17T17:36:51.177 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.177 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.178 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:36:51.178 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.103456+0000", 2024-08-17T17:36:51.178 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.012912956999999999 2024-08-17T17:36:51.178 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.178 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.178 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:36:51.178 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.103487+0000", 2024-08-17T17:36:51.178 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.129e-05 2024-08-17T17:36:51.178 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:36:51.178 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:36:51.178 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:36:51.178 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.104201+0000", 2024-08-17T17:36:51.178 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00071380300000000005 2024-08-17T17:36:51.179 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.179 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.179 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.179 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:36:51.179 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:36:51.179 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:36:51.179 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:51.204 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:51.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:51.306 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:51.331 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:51.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:51.433 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:51.459 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:51.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:51.561 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:51.586 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:51.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:51.688 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:51.715 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:51.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:51.817 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:51.845 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:51.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:51.947 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:51.971 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:51.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:51.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:52.073 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:52.098 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:52.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:52.199 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:52.225 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:52.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:52.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:52 smithi063 ceph-mon[114756]: pgmap v2149: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 39988/186324 objects degraded (21.462%); 6442/186324 objects misplaced (3.457%); 13 MiB/s, 207 objects/s recovering 2024-08-17T17:36:52.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:52 smithi063 ceph-mon[109843]: pgmap v2149: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 39988/186324 objects degraded (21.462%); 6442/186324 objects misplaced (3.457%); 13 MiB/s, 207 objects/s recovering 2024-08-17T17:36:52.327 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:52.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:52 smithi162 ceph-mon[104668]: pgmap v2149: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 124 op/s; 39988/186324 objects degraded (21.462%); 6442/186324 objects misplaced (3.457%); 13 MiB/s, 207 objects/s recovering 2024-08-17T17:36:52.350 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:52.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:52.451 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:52.475 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:52.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:52.576 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:52.602 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:52.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:52.704 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:52.727 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:52.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:52.829 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:52.855 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:52.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:52.957 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:52.981 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:52.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:52.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:53.083 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:53.108 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:53.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:53.211 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:53.235 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:53.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:53.337 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:53.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.363 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:53.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:53.464 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:53.486 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:53.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:53.588 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:53.611 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:53.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:53.713 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:53.738 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:53.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:53.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:36:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:36:53] "GET /metrics HTTP/1.1" 200 36585 "" "Prometheus/2.51.0" 2024-08-17T17:36:53.839 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:53.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.867 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:53.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:53.968 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:53.994 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:53.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:53.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:54.096 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:54.122 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:54.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:54.224 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:54.251 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:54.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:54.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:54 smithi063 ceph-mon[114756]: pgmap v2150: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 39988/185541 objects degraded (21.552%); 6442/185541 objects misplaced (3.472%); 12 MiB/s, 186 objects/s recovering 2024-08-17T17:36:54.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:54 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 39988/185541 objects degraded (21.552%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:54.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:54 smithi063 ceph-mon[109843]: pgmap v2150: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 39988/185541 objects degraded (21.552%); 6442/185541 objects misplaced (3.472%); 12 MiB/s, 186 objects/s recovering 2024-08-17T17:36:54.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:54 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 39988/185541 objects degraded (21.552%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:54.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:54 smithi162 ceph-mon[104668]: pgmap v2150: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 39988/185541 objects degraded (21.552%); 6442/185541 objects misplaced (3.472%); 12 MiB/s, 186 objects/s recovering 2024-08-17T17:36:54.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:54 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 39988/185541 objects degraded (21.552%), 10 pgs degraded (PG_DEGRADED) 2024-08-17T17:36:54.353 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:54.379 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:54.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:54.480 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:54.508 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:54.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:54.610 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:54.633 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:54.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:54.651 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:36:54.734 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:54.760 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:54.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:54.862 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:54.887 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:54.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:54.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:54.988 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:55.013 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:55.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:55.115 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:55.141 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:55.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:55.243 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:55.267 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:55.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:55.368 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:55.394 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:55.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:55.496 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:55.519 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:55.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:55.621 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:55.647 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:55.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:55.748 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:55.771 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:55.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:55.873 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:55.900 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:55.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:55.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:56.002 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:56.026 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:56.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:56.130 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:56.160 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:56.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:56.261 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:56.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:56 smithi063 ceph-mon[114756]: pgmap v2151: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 178 op/s; 39697/183903 objects degraded (21.586%); 5793/183903 objects misplaced (3.150%); 16 MiB/s, 254 objects/s recovering 2024-08-17T17:36:56.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:56 smithi063 ceph-mon[109843]: pgmap v2151: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 178 op/s; 39697/183903 objects degraded (21.586%); 5793/183903 objects misplaced (3.150%); 16 MiB/s, 254 objects/s recovering 2024-08-17T17:36:56.287 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:56.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:56.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:56 smithi162 ceph-mon[104668]: pgmap v2151: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 178 op/s; 39697/183903 objects degraded (21.586%); 5793/183903 objects misplaced (3.150%); 16 MiB/s, 254 objects/s recovering 2024-08-17T17:36:56.389 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:56.413 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:56.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:56.515 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:56.540 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:56.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:56.643 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:56.667 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:56.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:56.768 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:56.792 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:56.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:56.894 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:56.920 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:56.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:56.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:57.022 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:57.051 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:57.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:57.153 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:57.179 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:57.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:57.281 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:57.307 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:57.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:57.409 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:57.436 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:57.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:57.538 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:36:57.564 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:57.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:36:57.666 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:57.692 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:57.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:57.794 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:57.824 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:57.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:57.926 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:57.952 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:57.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:57.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:36:58.053 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:58.080 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:58.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:58.182 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:58.210 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:58.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:58.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:58 smithi063 ceph-mon[114756]: pgmap v2152: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 39697/183513 objects degraded (21.632%); 5793/183513 objects misplaced (3.157%); 13 MiB/s, 210 objects/s recovering 2024-08-17T17:36:58.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:58 smithi063 ceph-mon[109843]: pgmap v2152: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 39697/183513 objects degraded (21.632%); 5793/183513 objects misplaced (3.157%); 13 MiB/s, 210 objects/s recovering 2024-08-17T17:36:58.313 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:58.337 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:58.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:58.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:58 smithi162 ceph-mon[104668]: pgmap v2152: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 39697/183513 objects degraded (21.632%); 5793/183513 objects misplaced (3.157%); 13 MiB/s, 210 objects/s recovering 2024-08-17T17:36:58.439 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:58.464 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:58.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:58.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:58.590 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:58.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:58.693 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:36:58.722 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:58.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:36:58.824 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:58.847 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:58.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:58.949 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:58.973 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:58.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:58.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:59.074 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:36:59.097 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:59.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:36:59.199 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:59.226 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:59.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:59.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:36:59 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 39697/183513 objects degraded (21.632%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:59.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:36:59 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 39697/183513 objects degraded (21.632%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:59.328 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:36:59.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:36:59 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 39697/183513 objects degraded (21.632%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:36:59.352 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:59.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:36:59.454 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:59.485 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:59.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:59.587 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:36:59.610 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:59.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:36:59.712 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:36:59.735 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:36:59.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:36:59.837 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:36:59.863 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:59.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.864 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:36:59.965 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:36:59.992 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:36:59.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:36:59.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:00.094 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:00.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.123 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:00.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:00.223 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:00.249 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:00.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:00.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:00 smithi063 ceph-mon[109843]: pgmap v2153: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 39697/182352 objects degraded (21.769%); 4949/182352 objects misplaced (2.714%); 16 MiB/s, 262 objects/s recovering 2024-08-17T17:37:00.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:00 smithi063 ceph-mon[114756]: pgmap v2153: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 39697/182352 objects degraded (21.769%); 4949/182352 objects misplaced (2.714%); 16 MiB/s, 262 objects/s recovering 2024-08-17T17:37:00.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:00 smithi162 ceph-mon[104668]: pgmap v2153: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 152 op/s; 39697/182352 objects degraded (21.769%); 4949/182352 objects misplaced (2.714%); 16 MiB/s, 262 objects/s recovering 2024-08-17T17:37:00.353 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:00.376 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:00.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:00.478 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:00.505 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:00.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:00.606 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:00.630 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:00.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:00.733 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:00.761 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:00.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:00.863 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:00.890 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:00.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:00.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:00.992 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:01.018 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:01.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:01.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:01.145 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:01.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:01.247 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:01.271 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:01.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:01.372 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:01.396 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:01.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:01.498 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:01.522 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:01.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:01.624 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:01.648 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:01.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:01.749 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:01.778 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:01.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:01.880 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:01.903 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:01.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:01.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:02.005 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:02.030 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:02.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:02.132 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:02.155 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:02.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:02.258 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:02.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:02 smithi063 ceph-mon[109843]: pgmap v2154: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 39428/181821 objects degraded (21.685%); 4949/181821 objects misplaced (2.722%); 9.5 MiB/s, 152 objects/s recovering 2024-08-17T17:37:02.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:02 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:02.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:02 smithi063 ceph-mon[114756]: pgmap v2154: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 39428/181821 objects degraded (21.685%); 4949/181821 objects misplaced (2.722%); 9.5 MiB/s, 152 objects/s recovering 2024-08-17T17:37:02.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:02 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:02.285 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:02.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:02.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:02 smithi162 ceph-mon[104668]: pgmap v2154: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 129 op/s; 39428/181821 objects degraded (21.685%); 4949/181821 objects misplaced (2.722%); 9.5 MiB/s, 152 objects/s recovering 2024-08-17T17:37:02.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:02 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:02.387 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:02.410 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:02.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:02.512 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:02.536 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:02.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:02.638 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:02.667 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:02.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:02.769 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:02.792 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:02.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:02.894 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:02.921 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:02.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:02.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:03.023 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:03.051 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:03.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:03.153 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:03.175 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:03.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:03.277 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:03.301 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:03.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:03.403 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:03.427 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:03.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:03.528 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:03.552 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:03.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:03.654 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:03.679 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:03.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:03.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:37:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:37:03] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-08-17T17:37:03.781 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:03.805 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:03.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:03.907 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:03.930 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:03.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:03.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:04.032 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:04.065 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:04.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.066 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:04.167 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:04.192 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:04.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:04.293 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:04.318 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:04.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:04.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:04 smithi162 ceph-mon[104668]: pgmap v2155: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 39428/181086 objects degraded (21.773%); 4949/181086 objects misplaced (2.733%); 9.2 MiB/s, 147 objects/s recovering 2024-08-17T17:37:04.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:04 smithi162 ceph-mon[104668]: osdmap e1402: 8 total, 8 up, 8 in 2024-08-17T17:37:04.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:04 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 39428/181086 objects degraded (21.773%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:04.420 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:04.445 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:04.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:04.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:04 smithi063 ceph-mon[114756]: pgmap v2155: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 39428/181086 objects degraded (21.773%); 4949/181086 objects misplaced (2.733%); 9.2 MiB/s, 147 objects/s recovering 2024-08-17T17:37:04.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:04 smithi063 ceph-mon[114756]: osdmap e1402: 8 total, 8 up, 8 in 2024-08-17T17:37:04.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:04 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 39428/181086 objects degraded (21.773%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:04.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:04 smithi063 ceph-mon[109843]: pgmap v2155: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 9 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 39428/181086 objects degraded (21.773%); 4949/181086 objects misplaced (2.733%); 9.2 MiB/s, 147 objects/s recovering 2024-08-17T17:37:04.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:04 smithi063 ceph-mon[109843]: osdmap e1402: 8 total, 8 up, 8 in 2024-08-17T17:37:04.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:04 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 39428/181086 objects degraded (21.773%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:04.546 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:04.571 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:04.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:04.673 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:04.697 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:04.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.698 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:04.799 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:04.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.826 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:04.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:04.926 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:04.951 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:04.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:04.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:05.053 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:05.076 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:05.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:05.178 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:05.204 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:05.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:05.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:05 smithi063 ceph-mon[114756]: osdmap e1403: 8 total, 8 up, 8 in 2024-08-17T17:37:05.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:05 smithi063 ceph-mon[109843]: osdmap e1403: 8 total, 8 up, 8 in 2024-08-17T17:37:05.306 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:05.334 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:05.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:05.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:05 smithi162 ceph-mon[104668]: osdmap e1403: 8 total, 8 up, 8 in 2024-08-17T17:37:05.437 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:05.462 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:05.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:05.564 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:05.590 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:05.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:05.691 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:05.714 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:05.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:05.815 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:05.841 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:05.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:05.943 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:05.970 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:05.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:05.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:05.989 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:37:06.072 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:06.083 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:06 smithi063 ceph-mon[109843]: pgmap v2158: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 7 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 168 op/s; 32886/179457 objects degraded (18.325%); 10061/179457 objects misplaced (5.606%); 17 MiB/s, 284 objects/s recovering 2024-08-17T17:37:06.083 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:06 smithi063 ceph-mon[114756]: pgmap v2158: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 7 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 168 op/s; 32886/179457 objects degraded (18.325%); 10061/179457 objects misplaced (5.606%); 17 MiB/s, 284 objects/s recovering 2024-08-17T17:37:06.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.101 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:06.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:06.202 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:06.230 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:06.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:06.332 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:06.348 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:06 smithi162 ceph-mon[104668]: pgmap v2158: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 7 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 168 op/s; 32886/179457 objects degraded (18.325%); 10061/179457 objects misplaced (5.606%); 17 MiB/s, 284 objects/s recovering 2024-08-17T17:37:06.357 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:06.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:06.459 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:06.490 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:06.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:06.592 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:06.619 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:06.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:06.721 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:06.747 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:06.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:06.774 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:37:06.774 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:37:06.791 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:06.795 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:37:06.795 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:37:06.817 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:37:06.817 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:37:06.840 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:37:06.840 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:37:06.848 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:06.859 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:37:06.859 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:37:06.873 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:06.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:06.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:06.881 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:37:06.882 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:37:06.904 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:37:06.904 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:37:06.927 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:37:06.928 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:37:06.975 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:07.000 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:07.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:07.059 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:37:07.102 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:07.129 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:07.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:07.230 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:07.254 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:07.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:07.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:07 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2673908345' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:07.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:07 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:07.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:07 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2909906634' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:07.356 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:07.383 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:07.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:07.485 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:07.511 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:07.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:07.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:07 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2673908345' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:07.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:07 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:07.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:07 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2909906634' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:07.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:07 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2673908345' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:07.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:07 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:07.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:07 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2909906634' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:07.612 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:07.640 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:07.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:07.742 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:07.767 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:07.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:07.869 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:07.891 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:07.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:07.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:07.993 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:08.016 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:08.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:08.117 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:08.143 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:08.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:08.245 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:08.271 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:08.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:08.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:08 smithi063 ceph-mon[114756]: pgmap v2159: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 32885/179016 objects degraded (18.370%); 10061/179016 objects misplaced (5.620%); 10 MiB/s, 179 objects/s recovering 2024-08-17T17:37:08.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:08 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:37:08.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:08 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:08.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:08 smithi063 ceph-mon[114756]: osdmap e1404: 8 total, 8 up, 8 in 2024-08-17T17:37:08.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:08 smithi063 ceph-mon[109843]: pgmap v2159: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 32885/179016 objects degraded (18.370%); 10061/179016 objects misplaced (5.620%); 10 MiB/s, 179 objects/s recovering 2024-08-17T17:37:08.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:08 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:37:08.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:08 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:08.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:08 smithi063 ceph-mon[109843]: osdmap e1404: 8 total, 8 up, 8 in 2024-08-17T17:37:08.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:08 smithi162 ceph-mon[104668]: pgmap v2159: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 138 op/s; 32885/179016 objects degraded (18.370%); 10061/179016 objects misplaced (5.620%); 10 MiB/s, 179 objects/s recovering 2024-08-17T17:37:08.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:08 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:37:08.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:08 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:08.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:08 smithi162 ceph-mon[104668]: osdmap e1404: 8 total, 8 up, 8 in 2024-08-17T17:37:08.373 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:08.400 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:08.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:08.502 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:08.527 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:08.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:08.629 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:08.652 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:08.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:08.754 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:08.778 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:08.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:08.880 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:08.904 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:08.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:08.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:09.006 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:09.033 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:09.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:09.135 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:09.159 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:09.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:09.261 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:09.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:09 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 32885/179016 objects degraded (18.370%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:37:09.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:09 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 32885/179016 objects degraded (18.370%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:37:09.284 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:09.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:09.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:09 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 32885/179016 objects degraded (18.370%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:37:09.386 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:09.410 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:09.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:09.512 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:09.542 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:09.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:09.643 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:09.670 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:09.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:09.772 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:09.797 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:09.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:09.898 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:09.921 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:09.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:09.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:10.023 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:10.050 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:10.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:10.055 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:37:10.151 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:10.177 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:10.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:10.279 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:10.305 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:10.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:10.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:10 smithi162 ceph-mon[104668]: pgmap v2161: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 168 op/s; 32885/178044 objects degraded (18.470%); 9459/178044 objects misplaced (5.313%); 18 MiB/s, 316 objects/s recovering 2024-08-17T17:37:10.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:10 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4166354936' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:37:10.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:10 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:37:10.399 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:10 smithi063 ceph-mon[109843]: pgmap v2161: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 168 op/s; 32885/178044 objects degraded (18.470%); 9459/178044 objects misplaced (5.313%); 18 MiB/s, 316 objects/s recovering 2024-08-17T17:37:10.399 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:10 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4166354936' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:37:10.399 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:10 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:37:10.399 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:10 smithi063 ceph-mon[114756]: pgmap v2161: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 168 op/s; 32885/178044 objects degraded (18.470%); 9459/178044 objects misplaced (5.313%); 18 MiB/s, 316 objects/s recovering 2024-08-17T17:37:10.399 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:10 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4166354936' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:37:10.399 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:10 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:37:10.407 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:10.430 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:10.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:10.454 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:37:10.531 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:10.555 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:10.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:10.657 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:10.684 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:10.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.686 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:10.786 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:10.812 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:10.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:10.914 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:10.942 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:10.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:10.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:11.044 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:11.071 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:11.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:11.172 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:11.202 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:11.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:11.304 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:11.329 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:11.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:11.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:11 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:37:11.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:11 smithi162 ceph-mon[104668]: osdmap e1405: 8 total, 8 up, 8 in 2024-08-17T17:37:11.431 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:11.457 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:11.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:11.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:11 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:37:11.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:11 smithi063 ceph-mon[114756]: osdmap e1405: 8 total, 8 up, 8 in 2024-08-17T17:37:11.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:11 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:37:11.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:11 smithi063 ceph-mon[109843]: osdmap e1405: 8 total, 8 up, 8 in 2024-08-17T17:37:11.559 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:11.582 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:11.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:11.684 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:11.709 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:11.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:11.811 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:11.837 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:11.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:11.939 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:11.969 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:11.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:11.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:12.071 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:12.099 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:12.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:12.201 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:12.226 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:12.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:12.328 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:12.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:12 smithi162 ceph-mon[104668]: pgmap v2163: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 32655/177390 objects degraded (18.409%); 9426/177390 objects misplaced (5.314%); 8.3 MiB/s, 131 objects/s recovering 2024-08-17T17:37:12.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:12 smithi162 ceph-mon[104668]: osdmap e1406: 8 total, 8 up, 8 in 2024-08-17T17:37:12.354 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:12.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:12.456 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:12.480 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:12.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.481 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:12.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:12 smithi063 ceph-mon[114756]: pgmap v2163: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 32655/177390 objects degraded (18.409%); 9426/177390 objects misplaced (5.314%); 8.3 MiB/s, 131 objects/s recovering 2024-08-17T17:37:12.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:12 smithi063 ceph-mon[114756]: osdmap e1406: 8 total, 8 up, 8 in 2024-08-17T17:37:12.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:12 smithi063 ceph-mon[109843]: pgmap v2163: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 132 op/s; 32655/177390 objects degraded (18.409%); 9426/177390 objects misplaced (5.314%); 8.3 MiB/s, 131 objects/s recovering 2024-08-17T17:37:12.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:12 smithi063 ceph-mon[109843]: osdmap e1406: 8 total, 8 up, 8 in 2024-08-17T17:37:12.582 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:12.605 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:12.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:12.707 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:12.734 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:12.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:12.836 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:12.861 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:12.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:12.963 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:12.990 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:12.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:12.991 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:13.092 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:13.117 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:13.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:13.218 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:13.246 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:13.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:13.348 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:13.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.374 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:13.374 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:13.475 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:13.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.504 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:13.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:13.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:13 smithi063 ceph-mon[114756]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-08-17T17:37:13.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:13 smithi063 ceph-mon[109843]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-08-17T17:37:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:37:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:37:13] "GET /metrics HTTP/1.1" 200 36582 "" "Prometheus/2.51.0" 2024-08-17T17:37:13.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:13 smithi162 ceph-mon[104668]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-08-17T17:37:13.605 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:13.631 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:13.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:13.732 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:13.757 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:13.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:13.859 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:13.884 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:13.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:13.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:13.986 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:14.011 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:14.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:14.113 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:14.138 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:14.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:14.240 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:14.268 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:14.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:14.370 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:14.395 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:14.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:14.497 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:14.522 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:14.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:14.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:14 smithi063 ceph-mon[114756]: pgmap v2165: 119 pgs: 1 active+recovery_wait+remapped, 3 peering, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 32655/176469 objects degraded (18.505%); 9038/176469 objects misplaced (5.122%); 12 MiB/s, 185 objects/s recovering 2024-08-17T17:37:14.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:14 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 32655/176469 objects degraded (18.505%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:37:14.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:14 smithi063 ceph-mon[109843]: pgmap v2165: 119 pgs: 1 active+recovery_wait+remapped, 3 peering, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 32655/176469 objects degraded (18.505%); 9038/176469 objects misplaced (5.122%); 12 MiB/s, 185 objects/s recovering 2024-08-17T17:37:14.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:14 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 32655/176469 objects degraded (18.505%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:37:14.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:14 smithi162 ceph-mon[104668]: pgmap v2165: 119 pgs: 1 active+recovery_wait+remapped, 3 peering, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 32655/176469 objects degraded (18.505%); 9038/176469 objects misplaced (5.122%); 12 MiB/s, 185 objects/s recovering 2024-08-17T17:37:14.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:14 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 32655/176469 objects degraded (18.505%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2024-08-17T17:37:14.624 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:14.647 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:14.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:14.749 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:14.774 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:14.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:14.878 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:14.901 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:14.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:14.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:15.004 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:15.027 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:15.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:15.129 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:15.158 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:15.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:15.260 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:15.284 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:15.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:15.386 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:15.413 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:15.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:15.515 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:15.540 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:37:15.541 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:15.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:15.642 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:15.672 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:15.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:15.774 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:15.799 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:15.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:15.901 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:15.927 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:15.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:15.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:16.029 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:16.056 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:16.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:16.146 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:37:16.146 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:37:16.154 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:16.158 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:16.190 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:16.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:16.291 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:16.315 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:16.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:16.417 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:16.441 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:16.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:16.519 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:16 smithi063 ceph-mon[114756]: pgmap v2166: 119 pgs: 1 active+recovery_wait+remapped, 3 peering, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 34717/175584 objects degraded (19.772%); 6570/175584 objects misplaced (3.742%); 11 MiB/s, 182 objects/s recovering 2024-08-17T17:37:16.519 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:16 smithi063 ceph-mon[109843]: pgmap v2166: 119 pgs: 1 active+recovery_wait+remapped, 3 peering, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 34717/175584 objects degraded (19.772%); 6570/175584 objects misplaced (3.742%); 11 MiB/s, 182 objects/s recovering 2024-08-17T17:37:16.543 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:16.574 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:16.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:16.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:16 smithi162 ceph-mon[104668]: pgmap v2166: 119 pgs: 1 active+recovery_wait+remapped, 3 peering, 2 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 34717/175584 objects degraded (19.772%); 6570/175584 objects misplaced (3.742%); 11 MiB/s, 182 objects/s recovering 2024-08-17T17:37:16.676 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:16.709 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:16.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:16.811 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:16.839 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:16.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:16.941 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:16.967 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:16.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:16.968 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:17.069 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:17.098 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:17.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:17.200 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:17.224 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:17.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:17.299 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:37:17.326 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:17.366 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:17.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:17.468 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:17.492 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:17.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:17.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:17 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:37:17.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:17 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:17.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:17 smithi063 ceph-mon[109843]: pgmap v2167: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 34717/174357 objects degraded (19.911%); 5794/174357 objects misplaced (3.323%); 12 MiB/s, 194 objects/s recovering 2024-08-17T17:37:17.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:17 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:37:17.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:17 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:17.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:17 smithi063 ceph-mon[114756]: pgmap v2167: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 34717/174357 objects degraded (19.911%); 5794/174357 objects misplaced (3.323%); 12 MiB/s, 194 objects/s recovering 2024-08-17T17:37:17.594 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:17.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:17 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:37:17.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:17 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:17.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:17 smithi162 ceph-mon[104668]: pgmap v2167: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 34717/174357 objects degraded (19.911%); 5794/174357 objects misplaced (3.323%); 12 MiB/s, 194 objects/s recovering 2024-08-17T17:37:17.620 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:17.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:17.721 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:17.745 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:17.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:17.847 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:17.872 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:17.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:17.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:17.935 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:37:17.973 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:18.004 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:18.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:18.106 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:18.112 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:18.132 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:18.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:18.233 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:18.261 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:18.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:18.363 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:18.390 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:18.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:18.492 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:18.504 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:18 smithi063 ceph-mon[114756]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-08-17T17:37:18.504 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:18 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 34717/174357 objects degraded (19.911%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:18.505 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:18 smithi063 ceph-mon[109843]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-08-17T17:37:18.505 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:18 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 34717/174357 objects degraded (19.911%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:18.518 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:18.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:18.620 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:18.645 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:18.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:18.747 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:18.775 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:18.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:18.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:18 smithi162 ceph-mon[104668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-08-17T17:37:18.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:18 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 34717/174357 objects degraded (19.911%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:18.877 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:18.904 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:18.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:18.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:19.006 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:19.030 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:19.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:19.132 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:19.156 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:19.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:19.258 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:19.284 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:19.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:19.386 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:19.413 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:19.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:19.514 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:19.540 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:19.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:19.642 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:19.666 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:19.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:19.768 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:19.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:19 smithi063 ceph-mon[109843]: pgmap v2168: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 134 op/s; 34271/173931 objects degraded (19.704%); 5794/173931 objects misplaced (3.331%); 14 MiB/s, 218 objects/s recovering 2024-08-17T17:37:19.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:19 smithi063 ceph-mon[114756]: pgmap v2168: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 134 op/s; 34271/173931 objects degraded (19.704%); 5794/173931 objects misplaced (3.331%); 14 MiB/s, 218 objects/s recovering 2024-08-17T17:37:19.795 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:19.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:19 smithi162 ceph-mon[104668]: pgmap v2168: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 134 op/s; 34271/173931 objects degraded (19.704%); 5794/173931 objects misplaced (3.331%); 14 MiB/s, 218 objects/s recovering 2024-08-17T17:37:19.897 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:19.922 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:19.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:19.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:20.024 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:20.049 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:20.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:20.150 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:20.177 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:20.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:20.279 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:20.303 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:20.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:20.405 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:20.431 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:20.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:20.533 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:20.557 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:20.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:20.658 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:20.684 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:20.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:20.786 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:20.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.814 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:20.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:20.915 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:20.939 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:20.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:20.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:21.041 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:21.067 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:21.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:21.168 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:21.191 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:21.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:21.293 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:21.317 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:21.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:21.419 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:21.453 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:21.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:21.555 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:21.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.578 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:21.578 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:21.679 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:21.702 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:21.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:21 smithi063 ceph-mon[114756]: osdmap e1407: 8 total, 8 up, 8 in 2024-08-17T17:37:21.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:21 smithi063 ceph-mon[114756]: pgmap v2170: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 33997/173289 objects degraded (19.619%); 5794/173289 objects misplaced (3.344%); 11 MiB/s, 170 objects/s recovering 2024-08-17T17:37:21.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:21 smithi063 ceph-mon[109843]: osdmap e1407: 8 total, 8 up, 8 in 2024-08-17T17:37:21.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:21 smithi063 ceph-mon[109843]: pgmap v2170: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 33997/173289 objects degraded (19.619%); 5794/173289 objects misplaced (3.344%); 11 MiB/s, 170 objects/s recovering 2024-08-17T17:37:21.805 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:21.829 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:21.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:21.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:21 smithi162 ceph-mon[104668]: osdmap e1407: 8 total, 8 up, 8 in 2024-08-17T17:37:21.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:21 smithi162 ceph-mon[104668]: pgmap v2170: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 119 op/s; 33997/173289 objects degraded (19.619%); 5794/173289 objects misplaced (3.344%); 11 MiB/s, 170 objects/s recovering 2024-08-17T17:37:21.931 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:21.955 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:21.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:21.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:22.056 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:22.079 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:22.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:22.181 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:22.207 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:22.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:22.309 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:22.334 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:22.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:22.436 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:22.458 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:22.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:22.559 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:22.588 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:22.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:22.690 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:22.715 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:22.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:22.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:22 smithi063 ceph-mon[109843]: osdmap e1408: 8 total, 8 up, 8 in 2024-08-17T17:37:22.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:22 smithi063 ceph-mon[114756]: osdmap e1408: 8 total, 8 up, 8 in 2024-08-17T17:37:22.818 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:22.842 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:22.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:22.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:22 smithi162 ceph-mon[104668]: osdmap e1408: 8 total, 8 up, 8 in 2024-08-17T17:37:22.944 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:22.968 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:22.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:22.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:23.070 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:23.093 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:23.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:23.196 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:23.223 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:23.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:23.308 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:23.308 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:23.325 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:23.356 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:23.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:23.458 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:23.482 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:23.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:23.584 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:23.607 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:23.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:23.622 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:23 smithi063 ceph-mon[109843]: pgmap v2172: 119 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 33723/171909 objects degraded (19.617%); 5074/171909 objects misplaced (2.952%); 16 MiB/s, 294 objects/s recovering 2024-08-17T17:37:23.623 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:23 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 33997/173289 objects degraded (19.619%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:23.623 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:37:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:37:23] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.51.0" 2024-08-17T17:37:23.623 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:23 smithi063 ceph-mon[114756]: pgmap v2172: 119 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 33723/171909 objects degraded (19.617%); 5074/171909 objects misplaced (2.952%); 16 MiB/s, 294 objects/s recovering 2024-08-17T17:37:23.623 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:23 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 33997/173289 objects degraded (19.619%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:23.709 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:23.735 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:23.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:23.837 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:23.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:23 smithi162 ceph-mon[104668]: pgmap v2172: 119 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 153 op/s; 33723/171909 objects degraded (19.617%); 5074/171909 objects misplaced (2.952%); 16 MiB/s, 294 objects/s recovering 2024-08-17T17:37:23.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:23 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 33997/173289 objects degraded (19.619%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:23.862 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:23.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:23.964 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:23.987 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:23.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:23.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:24.089 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:24.117 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:24.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:24.218 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:24.246 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:24.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:24.348 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:24.375 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:24.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:24.477 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:24.500 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:24.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:24.602 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:24.628 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:24.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:24.730 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:24.754 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:24.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:24.856 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:24.883 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:24.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:24.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:24.985 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:25.009 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:25.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:25.111 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:25.139 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:25.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:25.241 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:25.266 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:25.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:25.368 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:25.393 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:25.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:25.495 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:25.524 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:25.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:25.626 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:25.651 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:25.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:25.753 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:25.776 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:25.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:25.878 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:25.901 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:25.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:25.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:25.983 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:37:25.984 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:37:25.984 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:25 smithi063 ceph-mon[114756]: pgmap v2173: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 106 op/s; 33194/171801 objects degraded (19.321%); 5074/171801 objects misplaced (2.953%); 14 MiB/s, 263 objects/s recovering 2024-08-17T17:37:25.984 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:25 smithi063 ceph-mon[109843]: pgmap v2173: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 106 op/s; 33194/171801 objects degraded (19.321%); 5074/171801 objects misplaced (2.953%); 14 MiB/s, 263 objects/s recovering 2024-08-17T17:37:26.003 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:26.005 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:37:26.005 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:37:26.024 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:37:26.024 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:37:26.026 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:26.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:26.043 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:37:26.043 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:37:26.066 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:37:26.067 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:37:26.084 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:37:26.085 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:37:26.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:25 smithi162 ceph-mon[104668]: pgmap v2173: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 106 op/s; 33194/171801 objects degraded (19.321%); 5074/171801 objects misplaced (2.953%); 14 MiB/s, 263 objects/s recovering 2024-08-17T17:37:26.109 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:37:26.110 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:37:26.128 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:26.132 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:37:26.132 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:37:26.157 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:26.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:26.259 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:26.286 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:26.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:26.388 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:26.416 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:26.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:26.518 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:26.544 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:26.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:26.646 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:26.674 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:26.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:26.776 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:26.798 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:26.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:26.899 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:26.924 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:26.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:26.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:26.947 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:26 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2920383477' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:26.947 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:26 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2920383477' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:27.026 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:27.077 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:27.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:27.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:26 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2920383477' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:27.179 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:27.202 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:27.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:27.304 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:27.327 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:27.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:27.429 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:27.453 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:27.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:27.555 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:27.580 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:27.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:27.682 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:27.707 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:27.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:27.808 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:27.831 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:27.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:27.932 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:27.957 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:27.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:27.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:27.990 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:27 smithi063 ceph-mon[114756]: pgmap v2174: 119 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 180 op/s; 32881/169605 objects degraded (19.387%); 5074/169605 objects misplaced (2.992%); 12 MiB/s, 227 objects/s recovering 2024-08-17T17:37:27.990 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:27 smithi063 ceph-mon[109843]: pgmap v2174: 119 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 180 op/s; 32881/169605 objects degraded (19.387%); 5074/169605 objects misplaced (2.992%); 12 MiB/s, 227 objects/s recovering 2024-08-17T17:37:28.059 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:28.085 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:28.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:27 smithi162 ceph-mon[104668]: pgmap v2174: 119 pgs: 1 active+recovery_wait+remapped, 1 active+clean+scrubbing+deep, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 180 op/s; 32881/169605 objects degraded (19.387%); 5074/169605 objects misplaced (2.992%); 12 MiB/s, 227 objects/s recovering 2024-08-17T17:37:28.188 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:28.212 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:28.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:28.313 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:28.337 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:28.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:28.439 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:28.464 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:28.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:28.566 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:28.591 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:28.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:28.693 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:28.719 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:28.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:28.786 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:37:28.787 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:37:28.787 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:37:28.787 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:37:28.787 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:37:28.820 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:28.846 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:28.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:28.850 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:37:28.914 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:28 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 32881/169605 objects degraded (19.387%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:28.914 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:28 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 32881/169605 objects degraded (19.387%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:28.948 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:28.973 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:28.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:28.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:29.074 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:29.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:28 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 32881/169605 objects degraded (19.387%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:29.099 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:29.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:29.201 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:29.226 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:29.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:29.327 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:29.349 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:29.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:29.451 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:29.474 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:29.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:29.576 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:29.600 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:29.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:29.700 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:37:29.701 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:29.725 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:29.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:29.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:29 smithi063 ceph-mon[109843]: pgmap v2175: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 170 op/s; 32303/169080 objects degraded (19.105%); 5074/169080 objects misplaced (3.001%); 15 MiB/s, 274 objects/s recovering 2024-08-17T17:37:29.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:29 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/649629821' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:37:29.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:29 smithi063 ceph-mon[114756]: pgmap v2175: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 170 op/s; 32303/169080 objects degraded (19.105%); 5074/169080 objects misplaced (3.001%); 15 MiB/s, 274 objects/s recovering 2024-08-17T17:37:29.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:29 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/649629821' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:37:29.827 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:29.851 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:29.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:29.953 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:29.979 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:37:29.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:29.980 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:29.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:30.081 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:29 smithi162 ceph-mon[104668]: pgmap v2175: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 170 op/s; 32303/169080 objects degraded (19.105%); 5074/169080 objects misplaced (3.001%); 15 MiB/s, 274 objects/s recovering 2024-08-17T17:37:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:29 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/649629821' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:37:30.106 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:30.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:30.208 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:30.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.235 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:30.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:30.336 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:30.364 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:30.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:30.466 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:30.491 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:30.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:30.593 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:30.618 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:30.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:30.705 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:37:30.715 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:30.720 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:30.753 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:37:30.759 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:30.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:30.862 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:30.885 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:30.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:30.887 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:30.987 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:31.012 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:31.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:30 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/649629821' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:37:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:30 smithi063 ceph-mon[114756]: osdmap e1409: 8 total, 8 up, 8 in 2024-08-17T17:37:31.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:30 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2782108498' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:31.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:30 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/649629821' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:37:31.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:30 smithi063 ceph-mon[109843]: osdmap e1409: 8 total, 8 up, 8 in 2024-08-17T17:37:31.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:30 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2782108498' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:31.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:30 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/649629821' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:37:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:30 smithi162 ceph-mon[104668]: osdmap e1409: 8 total, 8 up, 8 in 2024-08-17T17:37:31.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:30 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2782108498' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:37:31.114 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:31.138 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:31.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:31.240 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:31.267 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:31.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:31.369 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:31.394 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:31.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.395 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:31.496 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:31.521 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:31.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:31.624 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:31.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.651 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:31.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:31.753 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:31.776 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:31.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:31.877 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:31.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:31.903 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:31.903 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:31.975 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:31 smithi063 ceph-mon[114756]: pgmap v2177: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 136 op/s; 31994/168384 objects degraded (19.001%); 5074/168384 objects misplaced (3.013%); 9.6 MiB/s, 153 objects/s recovering 2024-08-17T17:37:31.975 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:31 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:31.975 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:31 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2782108498' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:37:31.975 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:31 smithi063 ceph-mon[114756]: osdmap e1410: 8 total, 8 up, 8 in 2024-08-17T17:37:31.975 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:37:31.975 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:31.976 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:31 smithi063 ceph-mon[109843]: pgmap v2177: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 136 op/s; 31994/168384 objects degraded (19.001%); 5074/168384 objects misplaced (3.013%); 9.6 MiB/s, 153 objects/s recovering 2024-08-17T17:37:31.976 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:31 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:31.976 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:31 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2782108498' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:37:31.976 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:31 smithi063 ceph-mon[109843]: osdmap e1410: 8 total, 8 up, 8 in 2024-08-17T17:37:31.976 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:37:31.976 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:32.004 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:32.030 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:32.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:31 smithi162 ceph-mon[104668]: pgmap v2177: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 136 op/s; 31994/168384 objects degraded (19.001%); 5074/168384 objects misplaced (3.013%); 9.6 MiB/s, 153 objects/s recovering 2024-08-17T17:37:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:31 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:31 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2782108498' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:37:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:31 smithi162 ceph-mon[104668]: osdmap e1410: 8 total, 8 up, 8 in 2024-08-17T17:37:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:37:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:32.132 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:32.157 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:32.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:32.259 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:32.288 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:32.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:32.309 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:32.310 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:32.390 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:32.416 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:32.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:32.518 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:32.543 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:32.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:32.645 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:32.672 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:32.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:32.774 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:32.799 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:32.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:32.901 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:32.930 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:32.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:32.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:33.032 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:33.056 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:33.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:33.158 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:33.182 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:33.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:33.284 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:33.311 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:33.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:33.413 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:33.437 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:33.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:33.469 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:37:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:37:33] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.51.0" 2024-08-17T17:37:33.539 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:33.567 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:33.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:33.669 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:33.692 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:33.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:33.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:33 smithi063 ceph-mon[114756]: pgmap v2179: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 193 op/s; 31994/167160 objects degraded (19.140%); 5074/167160 objects misplaced (3.035%); 7.0 MiB/s, 111 objects/s recovering 2024-08-17T17:37:33.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:33 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 31994/167160 objects degraded (19.140%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:33.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:33 smithi063 ceph-mon[109843]: pgmap v2179: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 193 op/s; 31994/167160 objects degraded (19.140%); 5074/167160 objects misplaced (3.035%); 7.0 MiB/s, 111 objects/s recovering 2024-08-17T17:37:33.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:33 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 31994/167160 objects degraded (19.140%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:33.794 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:33.818 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:33.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:33.852 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:37:33.920 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:33.942 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:33.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:33.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:34.044 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:34.067 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:34.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:34.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:33 smithi162 ceph-mon[104668]: pgmap v2179: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 193 op/s; 31994/167160 objects degraded (19.140%); 5074/167160 objects misplaced (3.035%); 7.0 MiB/s, 111 objects/s recovering 2024-08-17T17:37:34.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:33 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 31994/167160 objects degraded (19.140%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:34.168 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:34.195 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:34.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:34.297 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:34.319 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:34.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:34.421 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:34.445 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:34.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:34.547 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:34.572 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:34.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:34.674 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:34.699 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:34.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:34.801 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:34.824 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:34.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:34.926 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:34.951 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:34.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:34.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:35.053 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:35.078 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:35.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.079 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:35.180 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:35.203 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:35.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:35.305 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:35.328 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:35.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:35.430 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:35.453 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:35.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:35.554 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:35.578 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:35.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:35.680 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:35.706 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:35.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:35.808 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:35.832 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:35.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:35.934 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:35.958 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:35.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:35.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:35 smithi063 ceph-mon[114756]: pgmap v2180: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 31280/167010 objects degraded (18.729%); 5074/167010 objects misplaced (3.038%); 11 MiB/s, 180 objects/s recovering 2024-08-17T17:37:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:35 smithi063 ceph-mon[109843]: pgmap v2180: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 31280/167010 objects degraded (18.729%); 5074/167010 objects misplaced (3.038%); 11 MiB/s, 180 objects/s recovering 2024-08-17T17:37:36.063 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:36.089 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:36.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:35 smithi162 ceph-mon[104668]: pgmap v2180: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 31280/167010 objects degraded (18.729%); 5074/167010 objects misplaced (3.038%); 11 MiB/s, 180 objects/s recovering 2024-08-17T17:37:36.192 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:36.217 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:36.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:36.318 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:36.345 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:36.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:36.447 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:36.470 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:36.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:36.572 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:36.596 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:36.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:36.697 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:36.730 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:36.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:36.832 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:36.857 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:36.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:36.959 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:36.983 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:36.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:37.085 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:37.112 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:37.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:37.214 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:37.238 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:37.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:37.340 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:37.364 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:37.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:37.466 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:37.494 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:37.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:37.598 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:37.621 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:37.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.622 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:37.723 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:37.748 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:37.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:37.850 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:37.875 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:37.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:37.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:37.977 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:38.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.007 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:38.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:38.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:37 smithi063 ceph-mon[114756]: pgmap v2181: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 31003/165324 objects degraded (18.753%); 5074/165324 objects misplaced (3.069%); 7.9 MiB/s, 125 objects/s recovering 2024-08-17T17:37:38.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:37 smithi063 ceph-mon[109843]: pgmap v2181: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 31003/165324 objects degraded (18.753%); 5074/165324 objects misplaced (3.069%); 7.9 MiB/s, 125 objects/s recovering 2024-08-17T17:37:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:37 smithi162 ceph-mon[104668]: pgmap v2181: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 31003/165324 objects degraded (18.753%); 5074/165324 objects misplaced (3.069%); 7.9 MiB/s, 125 objects/s recovering 2024-08-17T17:37:38.108 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:38.134 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:38.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:38.236 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:38.262 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:38.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:38.364 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:38.390 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:38.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:38.492 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:38.518 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:38.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:38.620 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:38.645 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:38.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:38.747 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:38.773 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:38.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:38.875 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:38.905 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:38.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:38.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:39.007 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:38 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 31003/165324 objects degraded (18.753%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:39.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:38 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 31003/165324 objects degraded (18.753%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:39.033 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:39.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:39.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:38 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 31003/165324 objects degraded (18.753%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:39.135 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:39.159 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:39.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:39.261 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:39.287 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:39.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.288 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:39.389 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:39.413 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:39.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:39.475 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:37:39.475 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:37:39.503 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:37:39.503 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:37:39.515 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:39.532 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:37:39.533 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:37:39.539 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:39.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:39.557 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:37:39.557 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:37:39.576 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:37:39.577 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:37:39.594 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:37:39.595 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:37:39.615 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:37:39.615 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:37:39.634 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:37:39.634 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:37:39.641 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:39.665 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:39.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:39.767 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:39.794 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:39.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:39.895 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:39.920 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:39.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:39.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:40.021 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:39 smithi063 ceph-mon[114756]: pgmap v2182: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 30918/164868 objects degraded (18.753%); 4110/164868 objects misplaced (2.493%); 14 MiB/s, 230 objects/s recovering 2024-08-17T17:37:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:39 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2945626481' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:39 smithi063 ceph-mon[109843]: pgmap v2182: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 30918/164868 objects degraded (18.753%); 4110/164868 objects misplaced (2.493%); 14 MiB/s, 230 objects/s recovering 2024-08-17T17:37:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:39 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2945626481' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:40.046 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:40.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:39 smithi162 ceph-mon[104668]: pgmap v2182: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 157 op/s; 30918/164868 objects degraded (18.753%); 4110/164868 objects misplaced (2.493%); 14 MiB/s, 230 objects/s recovering 2024-08-17T17:37:40.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:39 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2945626481' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:37:40.147 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:40.175 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:40.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:40.277 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:40.300 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:40.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:40.402 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:40.425 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:40.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:40.527 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:40.553 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:40.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:40.655 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:40.677 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:40.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:40.779 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:40.815 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:40.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.816 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:40.917 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:40.941 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:40.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:40.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:41.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:40 smithi063 ceph-mon[109843]: osdmap e1411: 8 total, 8 up, 8 in 2024-08-17T17:37:41.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:40 smithi063 ceph-mon[114756]: osdmap e1411: 8 total, 8 up, 8 in 2024-08-17T17:37:41.044 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:41.067 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:41.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:41.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:40 smithi162 ceph-mon[104668]: osdmap e1411: 8 total, 8 up, 8 in 2024-08-17T17:37:41.170 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:41.193 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:41.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:41.295 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:41.319 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:41.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:41.421 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:41.449 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:41.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:41.551 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:41.574 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:41.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:41.676 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:41.703 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:41.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:41.804 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:41.830 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:41.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:41.931 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:41.959 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:41.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:41.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:41 smithi063 ceph-mon[114756]: pgmap v2184: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 30603/164091 objects degraded (18.650%); 4110/164091 objects misplaced (2.505%); 13 MiB/s, 207 objects/s recovering 2024-08-17T17:37:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:41 smithi063 ceph-mon[114756]: osdmap e1412: 8 total, 8 up, 8 in 2024-08-17T17:37:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:41 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:37:42.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:41 smithi063 ceph-mon[109843]: pgmap v2184: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 30603/164091 objects degraded (18.650%); 4110/164091 objects misplaced (2.505%); 13 MiB/s, 207 objects/s recovering 2024-08-17T17:37:42.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:41 smithi063 ceph-mon[109843]: osdmap e1412: 8 total, 8 up, 8 in 2024-08-17T17:37:42.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:41 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:37:42.061 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:42.086 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:42.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:41 smithi162 ceph-mon[104668]: pgmap v2184: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 6 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 111 op/s; 30603/164091 objects degraded (18.650%); 4110/164091 objects misplaced (2.505%); 13 MiB/s, 207 objects/s recovering 2024-08-17T17:37:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:41 smithi162 ceph-mon[104668]: osdmap e1412: 8 total, 8 up, 8 in 2024-08-17T17:37:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:41 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:37:42.188 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:42.213 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:42.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:42.315 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:42.341 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:42.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:42.443 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:42.467 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:42.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:42.569 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:42.593 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:42.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:42.695 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:42.719 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:42.821 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:42.846 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:42.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:42.948 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:42.974 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:42.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:42.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:43.076 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:43.102 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:43.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:43.204 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:43.228 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:43.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:43.330 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:43.331 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:37:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:37:43] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.51.0" 2024-08-17T17:37:43.360 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:43.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:43.368 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:37:43.368 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:37:43.463 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:43.489 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:43.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:43.591 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:43.616 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:43.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:43.718 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:43.744 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:43.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:43.846 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:43.873 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:43.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:43.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:43.975 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:44.009 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:44.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:44.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:43 smithi063 ceph-mon[109843]: pgmap v2186: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 191 op/s; 27361/162405 objects degraded (16.847%); 7176/162405 objects misplaced (4.419%); 12 MiB/s, 190 objects/s recovering 2024-08-17T17:37:44.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:43 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 27361/162405 objects degraded (16.847%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:43 smithi063 ceph-mon[114756]: pgmap v2186: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 191 op/s; 27361/162405 objects degraded (16.847%); 7176/162405 objects misplaced (4.419%); 12 MiB/s, 190 objects/s recovering 2024-08-17T17:37:44.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:43 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 27361/162405 objects degraded (16.847%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:44.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:43 smithi162 ceph-mon[104668]: pgmap v2186: 119 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 191 op/s; 27361/162405 objects degraded (16.847%); 7176/162405 objects misplaced (4.419%); 12 MiB/s, 190 objects/s recovering 2024-08-17T17:37:44.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:43 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 27361/162405 objects degraded (16.847%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:44.110 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:44.136 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:44.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:44.238 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:44.263 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:44.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:44.365 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:44.390 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:44.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:44.491 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:44.516 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:44.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:44.541 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:44.618 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:44.644 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:44.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:44.746 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:44.774 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:44.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:44.876 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:44.901 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:44.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:44.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:45.003 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:45.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.028 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:45.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:45.096 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:37:45.129 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:45.164 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:45.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:45.267 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:45.296 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:45.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:45.398 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:45.422 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:45.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.423 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:45.524 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:45.549 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:45.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:45.651 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:45.676 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:45.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:45.748 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:37:45.778 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:45.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.811 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:45.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:45.912 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:45.936 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:45.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:45.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:45 smithi063 ceph-mon[114756]: pgmap v2187: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 26997/162249 objects degraded (16.639%); 7072/162249 objects misplaced (4.359%); 12 MiB/s, 231 objects/s recovering 2024-08-17T17:37:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:45 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/378944107' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:37:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:45 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:37:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:45 smithi063 ceph-mon[109843]: pgmap v2187: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 26997/162249 objects degraded (16.639%); 7072/162249 objects misplaced (4.359%); 12 MiB/s, 231 objects/s recovering 2024-08-17T17:37:46.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:45 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/378944107' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:37:46.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:45 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:37:46.038 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:46.061 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:45 smithi162 ceph-mon[104668]: pgmap v2187: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 26997/162249 objects degraded (16.639%); 7072/162249 objects misplaced (4.359%); 12 MiB/s, 231 objects/s recovering 2024-08-17T17:37:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:45 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/378944107' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:37:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:45 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:37:46.108 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:37:46.163 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:46.190 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:46.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:46.292 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:46.318 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:46.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:46.420 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:46.446 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:46.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:46.548 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:46.571 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:46.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:46.673 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:46.703 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:46.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:46.806 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:46.832 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:46.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:46.934 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:46.957 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:46.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:46.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:46 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:37:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:46 smithi063 ceph-mon[114756]: osdmap e1413: 8 total, 8 up, 8 in 2024-08-17T17:37:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:46 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:37:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:46 smithi063 ceph-mon[109843]: osdmap e1413: 8 total, 8 up, 8 in 2024-08-17T17:37:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:47.058 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:47.088 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:47.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:46 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:37:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:46 smithi162 ceph-mon[104668]: osdmap e1413: 8 total, 8 up, 8 in 2024-08-17T17:37:47.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:37:47.189 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:47.223 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:47.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:47.325 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:47.348 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:47.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:47.451 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:47.477 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:47.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:47.579 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:47.607 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:47.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:47.709 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:47.735 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:47.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:47.837 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:47.861 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:47.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:47.963 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:47.990 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:47.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:47.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:48.092 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:47 smithi162 ceph-mon[104668]: pgmap v2189: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 228 op/s; 26700/160143 objects degraded (16.673%); 6601/160143 objects misplaced (4.122%); 9.6 MiB/s, 206 objects/s recovering 2024-08-17T17:37:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:37:48.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:37:48.117 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:48.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:48.219 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:48.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.245 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:48.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:47 smithi063 ceph-mon[114756]: pgmap v2189: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 228 op/s; 26700/160143 objects degraded (16.673%); 6601/160143 objects misplaced (4.122%); 9.6 MiB/s, 206 objects/s recovering 2024-08-17T17:37:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:37:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:37:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:47 smithi063 ceph-mon[109843]: pgmap v2189: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 228 op/s; 26700/160143 objects degraded (16.673%); 6601/160143 objects misplaced (4.122%); 9.6 MiB/s, 206 objects/s recovering 2024-08-17T17:37:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:37:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:37:48.346 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:48.376 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:48.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:48.478 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:48.503 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:48.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:48.605 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:48.631 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:48.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:48.733 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:48.762 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:48.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:48.864 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:48.888 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:48.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:48.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:48.962 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:37:48.963 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:37:48.990 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:49.019 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:49.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:49.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:48 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 26700/160143 objects degraded (16.673%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:49.121 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:49.130 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:48 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 26700/160143 objects degraded (16.673%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:49.130 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:48 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 26700/160143 objects degraded (16.673%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:49.152 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:49.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:49.254 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:49.277 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:49.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:49.379 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:49.404 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:49.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:49.487 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:49.506 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:49.518 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:49.546 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:49.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:49.648 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:49.675 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:49.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:49.777 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:49.801 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:49.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:49.902 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:49.927 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:49.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:49.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:50.029 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:50.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:49 smithi063 ceph-mon[114756]: pgmap v2190: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 184 op/s; 26700/159654 objects degraded (16.724%); 6601/159654 objects misplaced (4.135%); 7.0 MiB/s, 157 objects/s recovering 2024-08-17T17:37:50.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:49 smithi063 ceph-mon[109843]: pgmap v2190: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 184 op/s; 26700/159654 objects degraded (16.724%); 6601/159654 objects misplaced (4.135%); 7.0 MiB/s, 157 objects/s recovering 2024-08-17T17:37:50.054 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:50.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:49 smithi162 ceph-mon[104668]: pgmap v2190: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 184 op/s; 26700/159654 objects degraded (16.724%); 6601/159654 objects misplaced (4.135%); 7.0 MiB/s, 157 objects/s recovering 2024-08-17T17:37:50.156 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:50.181 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:50.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:50.284 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:50.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.310 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:50.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:50.411 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:50.434 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:50.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:50.535 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:50.560 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:50.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:50.662 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:50.689 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:50.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:50.791 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:50.815 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:50.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:50.917 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:50.942 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:50.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:50.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:51.044 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:51.070 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:51.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:51.172 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:51.196 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:51.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:51.298 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:51.322 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:51.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:51.424 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:51.446 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:51.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:51.548 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:51.571 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:51.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:51.673 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:51.698 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:51.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:51.800 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:51.810 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:37:51.828 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:51.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:51.930 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:51.954 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:51 smithi063 ceph-mon[114756]: pgmap v2191: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 26391/158862 objects degraded (16.613%); 6601/158862 objects misplaced (4.155%); 5.5 MiB/s, 125 objects/s recovering 2024-08-17T17:37:51.954 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:51 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2925575471' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:51.954 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:51 smithi063 ceph-mon[109843]: pgmap v2191: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 26391/158862 objects degraded (16.613%); 6601/158862 objects misplaced (4.155%); 5.5 MiB/s, 125 objects/s recovering 2024-08-17T17:37:51.955 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:51 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2925575471' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:51.956 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:51.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:51.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:52.057 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:52.081 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:52.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:52.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:51 smithi162 ceph-mon[104668]: pgmap v2191: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 26391/158862 objects degraded (16.613%); 6601/158862 objects misplaced (4.155%); 5.5 MiB/s, 125 objects/s recovering 2024-08-17T17:37:52.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:51 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2925575471' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:37:52.183 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:52.211 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:52.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:52.313 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:52.340 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:52.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:52.441 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:52.466 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:52.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:52.568 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:52.593 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:52.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:52.694 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:52.717 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:52.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:52.818 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:52.845 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:52.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:52.948 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:52.974 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:52.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:52.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:52 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:52 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2925575471' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:52 smithi063 ceph-mon[114756]: osdmap e1414: 8 total, 8 up, 8 in 2024-08-17T17:37:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:52 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:52 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2925575471' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:52 smithi063 ceph-mon[109843]: osdmap e1414: 8 total, 8 up, 8 in 2024-08-17T17:37:53.056 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:37:53.076 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:52 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:37:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:52 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2925575471' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:37:53.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:52 smithi162 ceph-mon[104668]: osdmap e1414: 8 total, 8 up, 8 in 2024-08-17T17:37:53.100 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:53.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:53.202 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:53.228 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:53.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:53.330 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:53.354 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:53.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:53.456 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:53.480 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:53.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:53.582 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:53.606 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:53.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:53.708 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:53.732 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:53.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:53.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:37:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:37:53] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.51.0" 2024-08-17T17:37:53.834 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:53.861 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:53.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:53.963 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:53.987 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:53.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:53.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:54.089 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:53 smithi162 ceph-mon[104668]: pgmap v2193: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 209 op/s; 26271/157230 objects degraded (16.709%); 6077/157230 objects misplaced (3.865%); 11 MiB/s, 169 objects/s recovering 2024-08-17T17:37:54.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:53 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 26271/157230 objects degraded (16.709%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:54.101 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:53 smithi063 ceph-mon[114756]: pgmap v2193: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 209 op/s; 26271/157230 objects degraded (16.709%); 6077/157230 objects misplaced (3.865%); 11 MiB/s, 169 objects/s recovering 2024-08-17T17:37:54.101 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:53 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 26271/157230 objects degraded (16.709%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:54.102 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:53 smithi063 ceph-mon[109843]: pgmap v2193: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 209 op/s; 26271/157230 objects degraded (16.709%); 6077/157230 objects misplaced (3.865%); 11 MiB/s, 169 objects/s recovering 2024-08-17T17:37:54.102 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:53 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 26271/157230 objects degraded (16.709%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:54.115 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:54.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:54.217 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:54.240 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:54.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:54.342 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:54.366 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:54.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.367 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:54.468 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:54.490 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:54.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:54.592 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:54.615 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:54.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:54.717 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:54.739 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:54.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:54.841 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:54.863 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:54.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:54.965 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:54.991 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:54.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:54.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:55.092 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:55.117 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:55.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:55.219 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:55.243 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:55.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:55.345 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:55.370 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:55.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:55.472 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:55.496 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:55.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:55.598 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:55.621 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:55.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.622 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:55.723 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:55.750 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:55.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:55.852 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:55.877 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:55.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:55.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:55.979 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:56.005 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:56.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:56.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:55 smithi063 ceph-mon[114756]: pgmap v2194: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 26271/157074 objects degraded (16.725%); 6077/157074 objects misplaced (3.869%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:37:56.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:55 smithi063 ceph-mon[109843]: pgmap v2194: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 26271/157074 objects degraded (16.725%); 6077/157074 objects misplaced (3.869%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:37:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:55 smithi162 ceph-mon[104668]: pgmap v2194: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 26271/157074 objects degraded (16.725%); 6077/157074 objects misplaced (3.869%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:37:56.107 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:56.132 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:56.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:56.233 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:56.260 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:56.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:56.362 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:56.386 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:56.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:56.488 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:56.515 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:56.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:56.617 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:56.640 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:56.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:56.741 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:37:56.774 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:56.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:37:56.876 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:56.903 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:56.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:56.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:57.006 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:57.029 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:57.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:57.131 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:57.158 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:57.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:57.259 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:57.283 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:57.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:57.385 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:57.413 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:57.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:57.515 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:57.538 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:57.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:57.639 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:57.663 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:57.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:57.765 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:57.789 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:57.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:57.891 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:57.916 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:57.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:57.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:58.017 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:58.040 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:58.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:57 smithi162 ceph-mon[104668]: pgmap v2195: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 164 op/s; 25880/155208 objects degraded (16.674%); 5457/155208 objects misplaced (3.516%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:37:58.119 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:57 smithi063 ceph-mon[114756]: pgmap v2195: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 164 op/s; 25880/155208 objects degraded (16.674%); 5457/155208 objects misplaced (3.516%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:37:58.120 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:57 smithi063 ceph-mon[109843]: pgmap v2195: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 164 op/s; 25880/155208 objects degraded (16.674%); 5457/155208 objects misplaced (3.516%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:37:58.141 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:58.163 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:58.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:58.265 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:58.294 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:58.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:58.396 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:37:58.421 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:58.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.422 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:37:58.523 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:58.547 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:58.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:58.649 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:58.675 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:58.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:58.777 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:58.802 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:58.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:58.904 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:58.927 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:58 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 25880/155208 objects degraded (16.674%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:58.927 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:58 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 25880/155208 objects degraded (16.674%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:58.928 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:58.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:58.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:59.029 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:59.057 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:59.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:58 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 25880/155208 objects degraded (16.674%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:37:59.159 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:37:59.184 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:59.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:37:59.286 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:37:59.312 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:59.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:37:59.414 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:37:59.438 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:59.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:37:59.540 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:37:59.564 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:59.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:37:59.666 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:37:59.693 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:37:59.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:37:59.795 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:59.824 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:59.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:59.913 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:37:59.913 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:37:59.926 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:37:59.937 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:37:59.937 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:37:59.951 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:37:59.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:37:59.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:37:59.956 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:37:59.956 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:37:59.976 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:37:59.977 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:37:59.998 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:37:59.998 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:38:00.018 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:38:00.018 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:38:00.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:37:59 smithi063 ceph-mon[109843]: pgmap v2196: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 162 op/s; 25880/154773 objects degraded (16.721%); 5457/154773 objects misplaced (3.526%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:38:00.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:37:59 smithi063 ceph-mon[114756]: pgmap v2196: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 162 op/s; 25880/154773 objects degraded (16.721%); 5457/154773 objects misplaced (3.526%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:38:00.038 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:38:00.038 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:38:00.053 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:00.055 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:38:00.055 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:38:00.077 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:00.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:37:59 smithi162 ceph-mon[104668]: pgmap v2196: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 162 op/s; 25880/154773 objects degraded (16.721%); 5457/154773 objects misplaced (3.526%); 9.7 MiB/s, 155 objects/s recovering 2024-08-17T17:38:00.180 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:00.205 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:00.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:00.306 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:00.331 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:00.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:00.433 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:00.459 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:00.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:00.561 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:00.589 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:00.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:00.691 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:00.699 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:38:00.715 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:00.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:00.817 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:00.842 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:00.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:00.872 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:00 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3365250860' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:00.872 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:00 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3365250860' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:00.945 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:00.969 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:00.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:00.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:01.071 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:01.094 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:01.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:00 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3365250860' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:01.197 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:01.223 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:01.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:01.325 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:01.351 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:01.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:01.453 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:01.477 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:01.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:01.579 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:01.604 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:01.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:01.707 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:01.733 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:01.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:01.835 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:01.860 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:01.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:01.962 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:01.990 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:01.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:01.992 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:01 smithi063 ceph-mon[114756]: pgmap v2197: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 25555/153909 objects degraded (16.604%); 5457/153909 objects misplaced (3.546%); 9.8 MiB/s, 156 objects/s recovering 2024-08-17T17:38:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:01 smithi063 ceph-mon[109843]: pgmap v2197: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 25555/153909 objects degraded (16.604%); 5457/153909 objects misplaced (3.546%); 9.8 MiB/s, 156 objects/s recovering 2024-08-17T17:38:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:02.092 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:02.118 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:02.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:02.201 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:38:02.202 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:38:02.202 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 0 2024-08-17T17:38:02.202 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:38:02.220 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:02.232 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:38:02.238 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:38:02.256 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:02.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:02.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:01 smithi162 ceph-mon[104668]: pgmap v2197: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 25555/153909 objects degraded (16.604%); 5457/153909 objects misplaced (3.546%); 9.8 MiB/s, 156 objects/s recovering 2024-08-17T17:38:02.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:02.358 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:02.381 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:02.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:02.483 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:02.507 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:02.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:02.608 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:02.633 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:02.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:02.663 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:02.734 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:02.759 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:02.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:02.861 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:02.884 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:02.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:02.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:02.986 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:03.011 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:03.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:03.101 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2197,"stamp":"2024-08-17T17:38:00.636620+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3361064625,"num_objects":51303,"num_object_clones":0,"num_object_copies":153909,"num_objects_missing_on_primary":6045,"num_objects_missing":6045,"num_objects_degraded":25555,"num_objects_misplaced":5457,"num_objects_unfound":0,"num_objects_dirty":51303,"num_whiteouts":0,"num_read":28,"num_read_kb":62,"num_write":77684,"num_write_kb":4124377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20474,"num_bytes_recovered":1290801885,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82183,"ondisk_log_size":82183,"up":357,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":376,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16092308,"kb_used_data":10279956,"kb_used_omap":298,"kb_used_meta":5811925,"kb_avail":733868908,"statfs":{"total":767960285184,"available":751481761792,"internally_reserved":0,"allocated":10526674944,"data_stored":10505622979,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":5951411729},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-108199936,"num_objects":-1651,"num_object_clones":0,"num_object_copies":-4953,"num_objects_missing_on_primary":-316,"num_objects_missing":-316,"num_objects_degraded":-836,"num_objects_misplaced":-1144,"num_objects_unfound":0,"num_objects_dirty":-1651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1651,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1564,"num_bytes_recovered":102498304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002840"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":34,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.815194+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:37:51.815194+0000","last_peered":"2024-08-17T17:37:51.815194+0000","last_clean":"2024-08-17T17:37:51.815194+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:37:51.815194+0000","last_undegraded":"2024-08-17T17:37:51.815194+0000","last_fullsized":"2024-08-17T17:37:51.815194+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1428,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.775189+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:37:45.775189+0000","last_peered":"2024-08-17T17:37:45.775189+0000","last_clean":"2024-08-17T17:37:45.775189+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:37:45.775189+0000","last_undegraded":"2024-08-17T17:37:45.775189+0000","last_fullsized":"2024-08-17T17:37:45.775189+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1743,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.830682+0000","last_change":"2024-08-17T17:36:15.771150+0000","last_active":"2024-08-17T17:37:51.830682+0000","last_peered":"2024-08-17T17:37:51.830682+0000","last_clean":"2024-08-17T17:37:51.830682+0000","last_became_active":"2024-08-17T17:36:15.770520+0000","last_became_peered":"2024-08-17T17:36:15.770520+0000","last_unstale":"2024-08-17T17:37:51.830682+0000","last_undegraded":"2024-08-17T17:37:51.830682+0000","last_fullsized":"2024-08-17T17:37:51.830682+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:17.552796+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":78,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.816411+0000","last_change":"2024-08-17T17:36:12.756709+0000","last_active":"2024-08-17T17:37:51.816411+0000","last_peered":"2024-08-17T17:37:51.816411+0000","last_clean":"2024-08-17T17:37:51.816411+0000","last_became_active":"2024-08-17T17:36:12.756385+0000","last_became_peered":"2024-08-17T17:36:12.756385+0000","last_unstale":"2024-08-17T17:37:51.816411+0000","last_undegraded":"2024-08-17T17:37:51.816411+0000","last_fullsized":"2024-08-17T17:37:51.816411+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:21.420282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":119,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.823439+0000","last_change":"2024-08-17T17:36:12.762800+0000","last_active":"2024-08-17T17:37:51.823439+0000","last_peered":"2024-08-17T17:37:51.823439+0000","last_clean":"2024-08-17T17:37:51.823439+0000","last_became_active":"2024-08-17T17:36:12.759977+0000","last_became_peered":"2024-08-17T17:36:12.759977+0000","last_unstale":"2024-08-17T17:37:51.823439+0000","last_undegraded":"2024-08-17T17:37:51.823439+0000","last_fullsized":"2024-08-17T17:37:51.823439+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:02.162681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1622,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.748960+0000","last_change":"2024-08-17T17:35:56.281561+0000","last_active":"2024-08-17T17:37:45.748960+0000","last_peered":"2024-08-17T17:37:45.748960+0000","last_clean":"2024-08-17T17:37:45.748960+0000","last_became_active":"2024-08-17T17:35:56.278638+0000","last_became_peered":"2024-08-17T17:35:56.278638+0000","last_unstale":"2024-08-17T17:37:45.748960+0000","last_undegraded":"2024-08-17T17:37:45.748960+0000","last_fullsized":"2024-08-17T17:37:45.748960+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1361,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.811157+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:37:45.811157+0000","last_peered":"2024-08-17T17:37:45.811157+0000","last_clean":"2024-08-17T17:37:45.811157+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:37:45.811157+0000","last_undegraded":"2024-08-17T17:37:45.811157+0000","last_fullsized":"2024-08-17T17:37:45.811157+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":37,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.775245+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:37:45.775245+0000","last_peered":"2024-08-17T17:37:45.775245+0000","last_clean":"2024-08-17T17:37:45.775245+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:37:45.775245+0000","last_undegraded":"2024-08-17T17:37:45.775245+0000","last_fullsized":"2024-08-17T17:37:45.775245+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1406,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.809099+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:37:45.809099+0000","last_peered":"2024-08-17T17:37:45.809099+0000","last_clean":"2024-08-17T17:37:45.809099+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:37:45.809099+0000","last_undegraded":"2024-08-17T17:37:45.809099+0000","last_fullsized":"2024-08-17T17:37:45.809099+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1414,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.776083+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:37:45.776083+0000","last_peered":"2024-08-17T17:37:45.776083+0000","last_clean":"2024-08-17T17:37:45.776083+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:37:45.776083+0000","last_undegraded":"2024-08-17T17:37:45.776083+0000","last_fullsized":"2024-08-17T17:37:45.776083+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":705,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.816273+0000","last_change":"2024-08-17T17:36:15.773315+0000","last_active":"2024-08-17T17:37:51.816273+0000","last_peered":"2024-08-17T17:37:51.816273+0000","last_clean":"2024-08-17T17:37:51.816273+0000","last_became_active":"2024-08-17T17:36:15.772961+0000","last_became_peered":"2024-08-17T17:36:15.772961+0000","last_unstale":"2024-08-17T17:37:51.816273+0000","last_undegraded":"2024-08-17T17:37:51.816273+0000","last_fullsized":"2024-08-17T17:37:51.816273+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:10.190221+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":28,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.815989+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:37:51.815989+0000","last_peered":"2024-08-17T17:37:51.815989+0000","last_clean":"2024-08-17T17:37:51.815989+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:37:51.815989+0000","last_undegraded":"2024-08-17T17:37:51.815989+0000","last_fullsized":"2024-08-17T17:37:51.815989+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":119,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.823913+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:37:51.823913+0000","last_peered":"2024-08-17T17:37:51.823913+0000","last_clean":"2024-08-17T17:37:51.823913+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:37:51.823913+0000","last_undegraded":"2024-08-17T17:37:51.823913+0000","last_fullsized":"2024-08-17T17:37:51.823913+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1495,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.785331+0000","last_change":"2024-08-17T17:36:24.225739+0000","last_active":"2024-08-17T17:37:45.785331+0000","last_peered":"2024-08-17T17:37:45.785331+0000","last_clean":"2024-08-17T17:37:45.785331+0000","last_became_active":"2024-08-17T17:36:24.225250+0000","last_became_peered":"2024-08-17T17:36:24.225250+0000","last_unstale":"2024-08-17T17:37:45.785331+0000","last_undegraded":"2024-08-17T17:37:45.785331+0000","last_fullsized":"2024-08-17T17:37:45.785331+0000","mapping_epoch":1393,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1394,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.290656+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1434,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.775968+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:37:45.775968+0000","last_peered":"2024-08-17T17:37:45.775968+0000","last_clean":"2024-08-17T17:37:45.775968+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:37:45.775968+0000","last_undegraded":"2024-08-17T17:37:45.775968+0000","last_fullsized":"2024-08-17T17:37:45.775968+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1841,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.776754+0000","last_change":"2024-08-17T17:36:12.752970+0000","last_active":"2024-08-17T17:37:45.776754+0000","last_peered":"2024-08-17T17:37:45.776754+0000","last_clean":"2024-08-17T17:37:45.776754+0000","last_became_active":"2024-08-17T17:36:12.752311+0000","last_became_peered":"2024-08-17T17:36:12.752311+0000","last_unstale":"2024-08-17T17:37:45.776754+0000","last_undegraded":"2024-08-17T17:37:45.776754+0000","last_fullsized":"2024-08-17T17:37:45.776754+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:38.131419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1789,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.810365+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:37:45.810365+0000","last_peered":"2024-08-17T17:37:45.810365+0000","last_clean":"2024-08-17T17:37:45.810365+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:37:45.810365+0000","last_undegraded":"2024-08-17T17:37:45.810365+0000","last_fullsized":"2024-08-17T17:37:45.810365+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1491,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.847536+0000","last_change":"2024-08-17T17:36:18.750494+0000","last_active":"2024-08-17T17:37:51.847536+0000","last_peered":"2024-08-17T17:37:51.847536+0000","last_clean":"2024-08-17T17:37:51.847536+0000","last_became_active":"2024-08-17T17:36:18.750092+0000","last_became_peered":"2024-08-17T17:36:18.750092+0000","last_unstale":"2024-08-17T17:37:51.847536+0000","last_undegraded":"2024-08-17T17:37:51.847536+0000","last_fullsized":"2024-08-17T17:37:51.847536+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:36.599727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1120,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.815244+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:37:51.815244+0000","last_peered":"2024-08-17T17:37:51.815244+0000","last_clean":"2024-08-17T17:37:51.815244+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:37:51.815244+0000","last_undegraded":"2024-08-17T17:37:51.815244+0000","last_fullsized":"2024-08-17T17:37:51.815244+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":956,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.808791+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:37:45.808791+0000","last_peered":"2024-08-17T17:37:45.808791+0000","last_clean":"2024-08-17T17:37:45.808791+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:37:45.808791+0000","last_undegraded":"2024-08-17T17:37:45.808791+0000","last_fullsized":"2024-08-17T17:37:45.808791+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":28,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.814844+0000","last_change":"2024-08-17T17:37:11.075505+0000","last_active":"2024-08-17T17:37:51.814844+0000","last_peered":"2024-08-17T17:37:51.814844+0000","last_clean":"2024-08-17T17:37:51.814844+0000","last_became_active":"2024-08-17T17:37:11.075131+0000","last_became_peered":"2024-08-17T17:37:11.075131+0000","last_unstale":"2024-08-17T17:37:51.814844+0000","last_undegraded":"2024-08-17T17:37:51.814844+0000","last_fullsized":"2024-08-17T17:37:51.814844+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:39.672054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1269,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.816916+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:37:51.816916+0000","last_peered":"2024-08-17T17:37:51.816916+0000","last_clean":"2024-08-17T17:37:51.816916+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:37:51.816916+0000","last_undegraded":"2024-08-17T17:37:51.816916+0000","last_fullsized":"2024-08-17T17:37:51.816916+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1646,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.775238+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:37:45.775238+0000","last_peered":"2024-08-17T17:37:45.775238+0000","last_clean":"2024-08-17T17:37:45.775238+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:37:45.775238+0000","last_undegraded":"2024-08-17T17:37:45.775238+0000","last_fullsized":"2024-08-17T17:37:45.775238+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":74,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.847086+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:37:51.847086+0000","last_peered":"2024-08-17T17:37:51.847086+0000","last_clean":"2024-08-17T17:37:51.847086+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:37:51.847086+0000","last_undegraded":"2024-08-17T17:37:51.847086+0000","last_fullsized":"2024-08-17T17:37:51.847086+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1243,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.810920+0000","last_change":"2024-08-17T17:36:12.755873+0000","last_active":"2024-08-17T17:37:45.810920+0000","last_peered":"2024-08-17T17:37:45.810920+0000","last_clean":"2024-08-17T17:37:45.810920+0000","last_became_active":"2024-08-17T17:36:12.755366+0000","last_became_peered":"2024-08-17T17:36:12.755366+0000","last_unstale":"2024-08-17T17:37:45.810920+0000","last_undegraded":"2024-08-17T17:37:45.810920+0000","last_fullsized":"2024-08-17T17:37:45.810920+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:36.523214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1597,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.834290+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:37:51.834290+0000","last_peered":"2024-08-17T17:37:51.834290+0000","last_clean":"2024-08-17T17:37:51.834290+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:37:51.834290+0000","last_undegraded":"2024-08-17T17:37:51.834290+0000","last_fullsized":"2024-08-17T17:37:51.834290+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":780,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.785488+0000","last_change":"2024-08-17T17:35:55.200519+0000","last_active":"2024-08-17T17:37:45.785488+0000","last_peered":"2024-08-17T17:37:45.785488+0000","last_clean":"2024-08-17T17:37:45.785488+0000","last_became_active":"2024-08-17T17:35:55.200230+0000","last_became_peered":"2024-08-17T17:35:55.200230+0000","last_unstale":"2024-08-17T17:37:45.785488+0000","last_undegraded":"2024-08-17T17:37:45.785488+0000","last_fullsized":"2024-08-17T17:37:45.785488+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:33.454851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1420,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.809488+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:37:45.809488+0000","last_peered":"2024-08-17T17:37:45.809488+0000","last_clean":"2024-08-17T17:37:45.809488+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:37:45.809488+0000","last_undegraded":"2024-08-17T17:37:45.809488+0000","last_fullsized":"2024-08-17T17:37:45.809488+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1692,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.823408+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:37:51.823408+0000","last_peered":"2024-08-17T17:37:51.823408+0000","last_clean":"2024-08-17T17:37:51.823408+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:37:51.823408+0000","last_undegraded":"2024-08-17T17:37:51.823408+0000","last_fullsized":"2024-08-17T17:37:51.823408+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:22.077229+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":239,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.848050+0000","last_change":"2024-08-17T17:35:33.145476+0000","last_active":"2024-08-17T17:37:51.848050+0000","last_peered":"2024-08-17T17:37:51.848050+0000","last_clean":"2024-08-17T17:37:51.848050+0000","last_became_active":"2024-08-17T17:35:33.145125+0000","last_became_peered":"2024-08-17T17:35:33.145125+0000","last_unstale":"2024-08-17T17:37:51.848050+0000","last_undegraded":"2024-08-17T17:37:51.848050+0000","last_fullsized":"2024-08-17T17:37:51.848050+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:45.298064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":924,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.748313+0000","last_change":"2024-08-17T17:35:56.248612+0000","last_active":"2024-08-17T17:37:45.748313+0000","last_peered":"2024-08-17T17:37:45.748313+0000","last_clean":"2024-08-17T17:37:45.748313+0000","last_became_active":"2024-08-17T17:35:56.244364+0000","last_became_peered":"2024-08-17T17:35:56.244364+0000","last_unstale":"2024-08-17T17:37:45.748313+0000","last_undegraded":"2024-08-17T17:37:45.748313+0000","last_fullsized":"2024-08-17T17:37:45.748313+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:41.064165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":864,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.810628+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:37:45.810628+0000","last_peered":"2024-08-17T17:37:45.810628+0000","last_clean":"2024-08-17T17:37:45.810628+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:37:45.810628+0000","last_undegraded":"2024-08-17T17:37:45.810628+0000","last_fullsized":"2024-08-17T17:37:45.810628+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1399,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.776598+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:37:45.776598+0000","last_peered":"2024-08-17T17:37:45.776598+0000","last_clean":"2024-08-17T17:37:45.776598+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:37:45.776598+0000","last_undegraded":"2024-08-17T17:37:45.776598+0000","last_fullsized":"2024-08-17T17:37:45.776598+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1418,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.776107+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:37:45.776107+0000","last_peered":"2024-08-17T17:37:45.776107+0000","last_clean":"2024-08-17T17:37:45.776107+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:37:45.776107+0000","last_undegraded":"2024-08-17T17:37:45.776107+0000","last_fullsized":"2024-08-17T17:37:45.776107+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":913,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.827461+0000","last_change":"2024-08-17T17:36:15.774881+0000","last_active":"2024-08-17T17:37:51.827461+0000","last_peered":"2024-08-17T17:37:51.827461+0000","last_clean":"2024-08-17T17:37:51.827461+0000","last_became_active":"2024-08-17T17:36:15.774435+0000","last_became_peered":"2024-08-17T17:36:15.774435+0000","last_unstale":"2024-08-17T17:37:51.827461+0000","last_undegraded":"2024-08-17T17:37:51.827461+0000","last_fullsized":"2024-08-17T17:37:51.827461+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:52.015854+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"1414'4990","reported_seq":5131,"reported_epoch":1414,"state":"active+recovery_wait+remapped","last_fresh":"2024-08-17T17:37:55.544080+0000","last_change":"2024-08-17T17:36:19.070399+0000","last_active":"2024-08-17T17:37:55.544080+0000","last_peered":"2024-08-17T17:37:55.544080+0000","last_clean":"2024-08-17T17:36:13.740039+0000","last_became_active":"2024-08-17T17:36:19.066494+0000","last_became_peered":"2024-08-17T17:36:19.066494+0000","last_unstale":"2024-08-17T17:37:55.544080+0000","last_undegraded":"2024-08-17T17:37:55.544080+0000","last_fullsized":"2024-08-17T17:37:55.544080+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4990,"log_dups_size":0,"ondisk_log_size":4990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216006656,"num_objects":3296,"num_object_clones":0,"num_object_copies":9888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4006,"num_objects_unfound":0,"num_objects_dirty":3296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4990,"num_write_kb":265152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[0,1,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":-710}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1367,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.775365+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:37:45.775365+0000","last_peered":"2024-08-17T17:37:45.775365+0000","last_clean":"2024-08-17T17:37:45.775365+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:37:45.775365+0000","last_undegraded":"2024-08-17T17:37:45.775365+0000","last_fullsized":"2024-08-17T17:37:45.775365+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1263,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.816805+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:37:51.816805+0000","last_peered":"2024-08-17T17:37:51.816805+0000","last_clean":"2024-08-17T17:37:51.816805+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:37:51.816805+0000","last_undegraded":"2024-08-17T17:37:51.816805+0000","last_fullsized":"2024-08-17T17:37:51.816805+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":306,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.815657+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:37:51.815657+0000","last_peered":"2024-08-17T17:37:51.815657+0000","last_clean":"2024-08-17T17:37:51.815657+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:37:51.815657+0000","last_undegraded":"2024-08-17T17:37:51.815657+0000","last_fullsized":"2024-08-17T17:37:51.815657+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1574,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.847171+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:37:51.847171+0000","last_peered":"2024-08-17T17:37:51.847171+0000","last_clean":"2024-08-17T17:37:51.847171+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:37:51.847171+0000","last_undegraded":"2024-08-17T17:37:51.847171+0000","last_fullsized":"2024-08-17T17:37:51.847171+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:25.751871+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.f","version":"1414'4923","reported_seq":5055,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:55.299520+0000","last_change":"2024-08-17T17:35:34.179168+0000","last_active":"2024-08-17T17:37:55.299520+0000","last_peered":"2024-08-17T17:37:55.299520+0000","last_clean":"2024-08-17T17:37:55.299520+0000","last_became_active":"2024-08-17T17:35:34.133493+0000","last_became_peered":"2024-08-17T17:35:34.133493+0000","last_unstale":"2024-08-17T17:37:55.299520+0000","last_undegraded":"2024-08-17T17:37:55.299520+0000","last_fullsized":"2024-08-17T17:37:55.299520+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4923,"log_dups_size":0,"ondisk_log_size":4923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:24:11.188566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215023616,"num_objects":3281,"num_object_clones":0,"num_object_copies":9843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4923,"num_write_kb":262528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":81,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.847180+0000","last_change":"2024-08-17T17:37:11.079712+0000","last_active":"2024-08-17T17:37:51.847180+0000","last_peered":"2024-08-17T17:37:51.847180+0000","last_clean":"2024-08-17T17:37:51.847180+0000","last_became_active":"2024-08-17T17:37:11.079253+0000","last_became_peered":"2024-08-17T17:37:11.079253+0000","last_unstale":"2024-08-17T17:37:51.847180+0000","last_undegraded":"2024-08-17T17:37:51.847180+0000","last_fullsized":"2024-08-17T17:37:51.847180+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:53.405657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":908,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.816192+0000","last_change":"2024-08-17T17:35:56.223396+0000","last_active":"2024-08-17T17:37:51.816192+0000","last_peered":"2024-08-17T17:37:51.816192+0000","last_clean":"2024-08-17T17:37:51.816192+0000","last_became_active":"2024-08-17T17:35:56.212413+0000","last_became_peered":"2024-08-17T17:35:56.212413+0000","last_unstale":"2024-08-17T17:37:51.816192+0000","last_undegraded":"2024-08-17T17:37:51.816192+0000","last_fullsized":"2024-08-17T17:37:51.816192+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:44.185174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1616,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.815416+0000","last_change":"2024-08-17T17:35:33.136068+0000","last_active":"2024-08-17T17:37:51.815416+0000","last_peered":"2024-08-17T17:37:51.815416+0000","last_clean":"2024-08-17T17:37:51.815416+0000","last_became_active":"2024-08-17T17:35:33.135591+0000","last_became_peered":"2024-08-17T17:35:33.135591+0000","last_unstale":"2024-08-17T17:37:51.815416+0000","last_undegraded":"2024-08-17T17:37:51.815416+0000","last_fullsized":"2024-08-17T17:37:51.815416+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:52.252561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1773,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.777130+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:37:45.777130+0000","last_peered":"2024-08-17T17:37:45.777130+0000","last_clean":"2024-08-17T17:37:45.777130+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:37:45.777130+0000","last_undegraded":"2024-08-17T17:37:45.777130+0000","last_fullsized":"2024-08-17T17:37:45.777130+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"1414'4826","reported_seq":4943,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:55.409991+0000","last_change":"2024-08-17T17:35:14.861823+0000","last_active":"2024-08-17T17:37:55.409991+0000","last_peered":"2024-08-17T17:37:55.409991+0000","last_clean":"2024-08-17T17:37:55.409991+0000","last_became_active":"2024-08-17T17:35:14.854109+0000","last_became_peered":"2024-08-17T17:35:14.854109+0000","last_unstale":"2024-08-17T17:37:55.409991+0000","last_undegraded":"2024-08-17T17:37:55.409991+0000","last_fullsized":"2024-08-17T17:37:55.409991+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4826,"log_dups_size":0,"ondisk_log_size":4826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:46:40.778197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211025920,"num_objects":3220,"num_object_clones":0,"num_object_copies":9660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4826,"num_write_kb":257472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":985,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.808924+0000","last_change":"2024-08-17T17:35:33.138467+0000","last_active":"2024-08-17T17:37:45.808924+0000","last_peered":"2024-08-17T17:37:45.808924+0000","last_clean":"2024-08-17T17:37:45.808924+0000","last_became_active":"2024-08-17T17:35:33.138174+0000","last_became_peered":"2024-08-17T17:35:33.138174+0000","last_unstale":"2024-08-17T17:37:45.808924+0000","last_undegraded":"2024-08-17T17:37:45.808924+0000","last_fullsized":"2024-08-17T17:37:45.808924+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:08.860439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1370,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.746658+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:37:45.746658+0000","last_peered":"2024-08-17T17:37:45.746658+0000","last_clean":"2024-08-17T17:37:45.746658+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:37:45.746658+0000","last_undegraded":"2024-08-17T17:37:45.746658+0000","last_fullsized":"2024-08-17T17:37:45.746658+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":112,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.823755+0000","last_change":"2024-08-17T17:35:14.854881+0000","last_active":"2024-08-17T17:37:51.823755+0000","last_peered":"2024-08-17T17:37:51.823755+0000","last_clean":"2024-08-17T17:37:51.823755+0000","last_became_active":"2024-08-17T17:35:14.854121+0000","last_became_peered":"2024-08-17T17:35:14.854121+0000","last_unstale":"2024-08-17T17:37:51.823755+0000","last_undegraded":"2024-08-17T17:37:51.823755+0000","last_fullsized":"2024-08-17T17:37:51.823755+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.863624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":927,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.816494+0000","last_change":"2024-08-17T17:36:15.773050+0000","last_active":"2024-08-17T17:37:51.816494+0000","last_peered":"2024-08-17T17:37:51.816494+0000","last_clean":"2024-08-17T17:37:51.816494+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:37:51.816494+0000","last_undegraded":"2024-08-17T17:37:51.816494+0000","last_fullsized":"2024-08-17T17:37:51.816494+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:39.212195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:39.212195+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"1414'4807","reported_seq":10630,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:55.535980+0000","last_change":"2024-08-17T17:37:21.409979+0000","last_active":"2024-08-17T17:37:55.535980+0000","last_peered":"2024-08-17T17:37:55.535980+0000","last_clean":"2024-08-17T17:37:55.535980+0000","last_became_active":"2024-08-17T17:37:21.398111+0000","last_became_peered":"2024-08-17T17:37:21.398111+0000","last_unstale":"2024-08-17T17:37:55.535980+0000","last_undegraded":"2024-08-17T17:37:55.535980+0000","last_fullsized":"2024-08-17T17:37:55.535980+0000","mapping_epoch":1407,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4807,"log_dups_size":0,"ondisk_log_size":4807,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:19.299395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211288088,"num_objects":3225,"num_object_clones":0,"num_object_copies":9675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3225,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4807,"num_write_kb":256961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2850,"num_bytes_recovered":168820736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1028,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.809748+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:37:45.809748+0000","last_peered":"2024-08-17T17:37:45.809748+0000","last_clean":"2024-08-17T17:37:45.809748+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:37:45.809748+0000","last_undegraded":"2024-08-17T17:37:45.809748+0000","last_fullsized":"2024-08-17T17:37:45.809748+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":206,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.826889+0000","last_change":"2024-08-17T17:35:33.145351+0000","last_active":"2024-08-17T17:37:51.826889+0000","last_peered":"2024-08-17T17:37:51.826889+0000","last_clean":"2024-08-17T17:37:51.826889+0000","last_became_active":"2024-08-17T17:35:33.145040+0000","last_became_peered":"2024-08-17T17:35:33.145040+0000","last_unstale":"2024-08-17T17:37:51.826889+0000","last_undegraded":"2024-08-17T17:37:51.826889+0000","last_fullsized":"2024-08-17T17:37:51.826889+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:02.124000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1438,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.811226+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:37:45.811226+0000","last_peered":"2024-08-17T17:37:45.811226+0000","last_clean":"2024-08-17T17:37:45.811226+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:37:45.811226+0000","last_undegraded":"2024-08-17T17:37:45.811226+0000","last_fullsized":"2024-08-17T17:37:45.811226+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":910,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.816729+0000","last_change":"2024-08-17T17:36:15.769890+0000","last_active":"2024-08-17T17:37:51.816729+0000","last_peered":"2024-08-17T17:37:51.816729+0000","last_clean":"2024-08-17T17:37:51.816729+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:37:51.816729+0000","last_undegraded":"2024-08-17T17:37:51.816729+0000","last_fullsized":"2024-08-17T17:37:51.816729+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:33.034306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:33.034306+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":4,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.b","version":"1414'4760","reported_seq":8443,"reported_epoch":1414,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:55.659103+0000","last_change":"2024-08-17T17:37:38.786235+0000","last_active":"2024-08-17T17:37:55.659103+0000","last_peered":"2024-08-17T17:37:55.659103+0000","last_clean":"2024-08-17T17:36:11.707166+0000","last_became_active":"2024-08-17T17:36:14.064459+0000","last_became_peered":"2024-08-17T17:36:14.064459+0000","last_unstale":"2024-08-17T17:37:55.659103+0000","last_undegraded":"2024-08-17T17:36:13.773134+0000","last_fullsized":"2024-08-17T17:36:13.738894+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4760,"log_dups_size":0,"ondisk_log_size":4760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209321984,"num_objects":3194,"num_object_clones":0,"num_object_copies":9582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":594,"num_objects_misplaced":1451,"num_objects_unfound":0,"num_objects_dirty":3194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4760,"num_write_kb":254528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1791,"num_bytes_recovered":117374976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[0,1],"avail_no_missing":["0","1","5"],"object_location_counts":[{"shards":"0,1","objects":1149},{"shards":"0,1,5","objects":1451}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1467,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.824081+0000","last_change":"2024-08-17T17:36:12.759743+0000","last_active":"2024-08-17T17:37:51.824081+0000","last_peered":"2024-08-17T17:37:51.824081+0000","last_clean":"2024-08-17T17:37:51.824081+0000","last_became_active":"2024-08-17T17:36:12.759334+0000","last_became_peered":"2024-08-17T17:36:12.759334+0000","last_unstale":"2024-08-17T17:37:51.824081+0000","last_undegraded":"2024-08-17T17:37:51.824081+0000","last_fullsized":"2024-08-17T17:37:51.824081+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:04.524384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1401,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.776314+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:37:45.776314+0000","last_peered":"2024-08-17T17:37:45.776314+0000","last_clean":"2024-08-17T17:37:45.776314+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:37:45.776314+0000","last_undegraded":"2024-08-17T17:37:45.776314+0000","last_fullsized":"2024-08-17T17:37:45.776314+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1443,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.809955+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:37:45.809955+0000","last_peered":"2024-08-17T17:37:45.809955+0000","last_clean":"2024-08-17T17:37:45.809955+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:37:45.809955+0000","last_undegraded":"2024-08-17T17:37:45.809955+0000","last_fullsized":"2024-08-17T17:37:45.809955+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1759,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.808583+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:37:45.808583+0000","last_peered":"2024-08-17T17:37:45.808583+0000","last_clean":"2024-08-17T17:37:45.808583+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:37:45.808583+0000","last_undegraded":"2024-08-17T17:37:45.808583+0000","last_fullsized":"2024-08-17T17:37:45.808583+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"1414'4962","reported_seq":7665,"reported_epoch":1414,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:00.467120+0000","last_change":"2024-08-17T17:36:22.955419+0000","last_active":"2024-08-17T17:38:00.467120+0000","last_peered":"2024-08-17T17:38:00.467120+0000","last_clean":"2024-08-17T17:35:54.187688+0000","last_became_active":"2024-08-17T17:36:22.904523+0000","last_became_peered":"2024-08-17T17:36:22.904523+0000","last_unstale":"2024-08-17T17:38:00.467120+0000","last_undegraded":"2024-08-17T17:36:20.560670+0000","last_fullsized":"2024-08-17T17:36:13.740491+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4962,"log_dups_size":0,"ondisk_log_size":4962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:57.525855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211419136,"num_objects":3226,"num_object_clones":0,"num_object_copies":9678,"num_objects_missing_on_primary":2013,"num_objects_missing":2013,"num_objects_degraded":5107,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4962,"num_write_kb":262016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":874,"num_bytes_recovered":57278464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":2013},{"shards":"2,4,5,6","objects":4},{"shards":"2,5,6","objects":1077},{"shards":"5,6","objects":132}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1077,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.810695+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:37:45.810695+0000","last_peered":"2024-08-17T17:37:45.810695+0000","last_clean":"2024-08-17T17:37:45.810695+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:37:45.810695+0000","last_undegraded":"2024-08-17T17:37:45.810695+0000","last_fullsized":"2024-08-17T17:37:45.810695+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1379,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.822791+0000","last_change":"2024-08-17T17:35:56.229622+0000","last_active":"2024-08-17T17:37:51.822791+0000","last_peered":"2024-08-17T17:37:51.822791+0000","last_clean":"2024-08-17T17:37:51.822791+0000","last_became_active":"2024-08-17T17:35:56.229324+0000","last_became_peered":"2024-08-17T17:35:56.229324+0000","last_unstale":"2024-08-17T17:37:51.822791+0000","last_undegraded":"2024-08-17T17:37:51.822791+0000","last_fullsized":"2024-08-17T17:37:51.822791+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:13.980316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1982,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.823514+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:37:51.823514+0000","last_peered":"2024-08-17T17:37:51.823514+0000","last_clean":"2024-08-17T17:37:51.823514+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:37:51.823514+0000","last_undegraded":"2024-08-17T17:37:51.823514+0000","last_fullsized":"2024-08-17T17:37:51.823514+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1573,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.775819+0000","last_change":"2024-08-17T17:35:33.137074+0000","last_active":"2024-08-17T17:37:45.775819+0000","last_peered":"2024-08-17T17:37:45.775819+0000","last_clean":"2024-08-17T17:37:45.775819+0000","last_became_active":"2024-08-17T17:35:33.136759+0000","last_became_peered":"2024-08-17T17:35:33.136759+0000","last_unstale":"2024-08-17T17:37:45.775819+0000","last_undegraded":"2024-08-17T17:37:45.775819+0000","last_fullsized":"2024-08-17T17:37:45.775819+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:37.263639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:37.263639+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:14.599780+0000","scrub_duration":0.0016181800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"1414'4887","reported_seq":6791,"reported_epoch":1414,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:55.536984+0000","last_change":"2024-08-17T17:36:23.019129+0000","last_active":"2024-08-17T17:37:55.536984+0000","last_peered":"2024-08-17T17:37:55.536984+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:23.013527+0000","last_became_peered":"2024-08-17T17:36:23.013527+0000","last_unstale":"2024-08-17T17:37:55.536984+0000","last_undegraded":"2024-08-17T17:36:13.874715+0000","last_fullsized":"2024-08-17T17:36:13.738828+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4887,"log_dups_size":0,"ondisk_log_size":4887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210567168,"num_objects":3213,"num_object_clones":0,"num_object_copies":9639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5861,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4887,"num_write_kb":259200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":838,"num_bytes_recovered":54919168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[0,6],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":2038},{"shards":"0,1,6","objects":1048},{"shards":"0,6","objects":-610}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1171,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.815796+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:37:51.815796+0000","last_peered":"2024-08-17T17:37:51.815796+0000","last_clean":"2024-08-17T17:37:51.815796+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:37:51.815796+0000","last_undegraded":"2024-08-17T17:37:51.815796+0000","last_fullsized":"2024-08-17T17:37:51.815796+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":37,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.775695+0000","last_change":"2024-08-17T17:36:17.774773+0000","last_active":"2024-08-17T17:37:45.775695+0000","last_peered":"2024-08-17T17:37:45.775695+0000","last_clean":"2024-08-17T17:37:45.775695+0000","last_became_active":"2024-08-17T17:36:17.774468+0000","last_became_peered":"2024-08-17T17:36:17.774468+0000","last_unstale":"2024-08-17T17:37:45.775695+0000","last_undegraded":"2024-08-17T17:37:45.775695+0000","last_fullsized":"2024-08-17T17:37:45.775695+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:54.231747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1509,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.814830+0000","last_change":"2024-08-17T17:35:33.137633+0000","last_active":"2024-08-17T17:37:51.814830+0000","last_peered":"2024-08-17T17:37:51.814830+0000","last_clean":"2024-08-17T17:37:51.814830+0000","last_became_active":"2024-08-17T17:35:33.137323+0000","last_became_peered":"2024-08-17T17:35:33.137323+0000","last_unstale":"2024-08-17T17:37:51.814830+0000","last_undegraded":"2024-08-17T17:37:51.814830+0000","last_fullsized":"2024-08-17T17:37:51.814830+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:03.359155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":889,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.815167+0000","last_change":"2024-08-17T17:35:56.264626+0000","last_active":"2024-08-17T17:37:51.815167+0000","last_peered":"2024-08-17T17:37:51.815167+0000","last_clean":"2024-08-17T17:37:51.815167+0000","last_became_active":"2024-08-17T17:35:56.264265+0000","last_became_peered":"2024-08-17T17:35:56.264265+0000","last_unstale":"2024-08-17T17:37:51.815167+0000","last_undegraded":"2024-08-17T17:37:51.815167+0000","last_fullsized":"2024-08-17T17:37:51.815167+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:20.754326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:20.754326+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"1414'4810","reported_seq":7294,"reported_epoch":1414,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:00.479639+0000","last_change":"2024-08-17T17:37:11.134558+0000","last_active":"2024-08-17T17:38:00.479639+0000","last_peered":"2024-08-17T17:38:00.479639+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:20.105110+0000","last_became_peered":"2024-08-17T17:36:20.105110+0000","last_unstale":"2024-08-17T17:38:00.479639+0000","last_undegraded":"2024-08-17T17:36:19.999272+0000","last_fullsized":"2024-08-17T17:36:13.740520+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4810,"log_dups_size":0,"ondisk_log_size":4810,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:45.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209977344,"num_objects":3204,"num_object_clones":0,"num_object_copies":9612,"num_objects_missing_on_primary":2013,"num_objects_missing":2013,"num_objects_degraded":5062,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4810,"num_write_kb":256448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":804,"num_bytes_recovered":52690944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":2013},{"shards":"1,5,6","objects":1036},{"shards":"5,6","objects":155}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1242,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.816152+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:37:51.816152+0000","last_peered":"2024-08-17T17:37:51.816152+0000","last_clean":"2024-08-17T17:37:51.816152+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:37:51.816152+0000","last_undegraded":"2024-08-17T17:37:51.816152+0000","last_fullsized":"2024-08-17T17:37:51.816152+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1781,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.809883+0000","last_change":"2024-08-17T17:36:12.758005+0000","last_active":"2024-08-17T17:37:45.809883+0000","last_peered":"2024-08-17T17:37:45.809883+0000","last_clean":"2024-08-17T17:37:45.809883+0000","last_became_active":"2024-08-17T17:36:12.757695+0000","last_became_peered":"2024-08-17T17:36:12.757695+0000","last_unstale":"2024-08-17T17:37:45.809883+0000","last_undegraded":"2024-08-17T17:37:45.809883+0000","last_fullsized":"2024-08-17T17:37:45.809883+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:51.792220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1419,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.809331+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:37:45.809331+0000","last_peered":"2024-08-17T17:37:45.809331+0000","last_clean":"2024-08-17T17:37:45.809331+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:37:45.809331+0000","last_undegraded":"2024-08-17T17:37:45.809331+0000","last_fullsized":"2024-08-17T17:37:45.809331+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1699,"reported_epoch":1414,"state":"active+clean+scrubbing","last_fresh":"2024-08-17T17:37:51.814969+0000","last_change":"2024-08-17T17:37:52.995110+0000","last_active":"2024-08-17T17:37:51.814969+0000","last_peered":"2024-08-17T17:37:51.814969+0000","last_clean":"2024-08-17T17:37:51.814969+0000","last_became_active":"2024-08-17T17:35:55.204861+0000","last_became_peered":"2024-08-17T17:35:55.204861+0000","last_unstale":"2024-08-17T17:37:51.814969+0000","last_undegraded":"2024-08-17T17:37:51.814969+0000","last_fullsized":"2024-08-17T17:37:51.814969+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.0","version":"1414'4878","reported_seq":12849,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:57.392366+0000","last_change":"2024-08-17T17:37:40.753703+0000","last_active":"2024-08-17T17:37:57.392366+0000","last_peered":"2024-08-17T17:37:57.392366+0000","last_clean":"2024-08-17T17:37:57.392366+0000","last_became_active":"2024-08-17T17:37:40.743458+0000","last_became_peered":"2024-08-17T17:37:40.743458+0000","last_unstale":"2024-08-17T17:37:57.392366+0000","last_undegraded":"2024-08-17T17:37:57.392366+0000","last_fullsized":"2024-08-17T17:37:57.392366+0000","mapping_epoch":1411,"log_start":"1373'100","ondisk_log_start":"1373'100","created":1357,"last_epoch_clean":1412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4778,"log_dups_size":0,"ondisk_log_size":4778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:14:02.340380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212205568,"num_objects":3238,"num_object_clones":0,"num_object_copies":9714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4878,"num_write_kb":259712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3914,"num_bytes_recovered":232652800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1015,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.777412+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:37:45.777412+0000","last_peered":"2024-08-17T17:37:45.777412+0000","last_clean":"2024-08-17T17:37:45.777412+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:37:45.777412+0000","last_undegraded":"2024-08-17T17:37:45.777412+0000","last_fullsized":"2024-08-17T17:37:45.777412+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.847343+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:37:51.847343+0000","last_peered":"2024-08-17T17:37:51.847343+0000","last_clean":"2024-08-17T17:37:51.847343+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:37:51.847343+0000","last_undegraded":"2024-08-17T17:37:51.847343+0000","last_fullsized":"2024-08-17T17:37:51.847343+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1750,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.809612+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:37:45.809612+0000","last_peered":"2024-08-17T17:37:45.809612+0000","last_clean":"2024-08-17T17:37:45.809612+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:37:45.809612+0000","last_undegraded":"2024-08-17T17:37:45.809612+0000","last_fullsized":"2024-08-17T17:37:45.809612+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1","version":"1414'4835","reported_seq":7350,"reported_epoch":1414,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:00.451279+0000","last_change":"2024-08-17T17:36:23.103523+0000","last_active":"2024-08-17T17:38:00.451279+0000","last_peered":"2024-08-17T17:38:00.451279+0000","last_clean":"2024-08-17T17:35:54.178280+0000","last_became_active":"2024-08-17T17:36:23.025910+0000","last_became_peered":"2024-08-17T17:36:23.025910+0000","last_unstale":"2024-08-17T17:38:00.451279+0000","last_undegraded":"2024-08-17T17:36:18.734637+0000","last_fullsized":"2024-08-17T17:36:13.740134+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4835,"log_dups_size":0,"ondisk_log_size":4835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:58.717055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210829312,"num_objects":3217,"num_object_clones":0,"num_object_copies":9651,"num_objects_missing_on_primary":2019,"num_objects_missing":2019,"num_objects_degraded":5112,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4835,"num_write_kb":257664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":811,"num_bytes_recovered":53149696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,0],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":2019},{"shards":"0,2,4,6","objects":4},{"shards":"0,2,6","objects":1070},{"shards":"0,6","objects":124}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1429,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.776816+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:37:45.776816+0000","last_peered":"2024-08-17T17:37:45.776816+0000","last_clean":"2024-08-17T17:37:45.776816+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:37:45.776816+0000","last_undegraded":"2024-08-17T17:37:45.776816+0000","last_fullsized":"2024-08-17T17:37:45.776816+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1402,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.775704+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:37:45.775704+0000","last_peered":"2024-08-17T17:37:45.775704+0000","last_clean":"2024-08-17T17:37:45.775704+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:37:45.775704+0000","last_undegraded":"2024-08-17T17:37:45.775704+0000","last_fullsized":"2024-08-17T17:37:45.775704+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":976,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.816175+0000","last_change":"2024-08-17T17:34:26.817339+0000","last_active":"2024-08-17T17:37:51.816175+0000","last_peered":"2024-08-17T17:37:51.816175+0000","last_clean":"2024-08-17T17:37:51.816175+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:37:51.816175+0000","last_undegraded":"2024-08-17T17:37:51.816175+0000","last_fullsized":"2024-08-17T17:37:51.816175+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:26.817223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:26.817223+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:30.089956+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"1414'4737","reported_seq":4879,"reported_epoch":1414,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:37:55.403149+0000","last_change":"2024-08-17T17:36:14.073155+0000","last_active":"2024-08-17T17:37:55.403149+0000","last_peered":"2024-08-17T17:37:55.403149+0000","last_clean":"2024-08-17T17:36:11.734683+0000","last_became_active":"2024-08-17T17:36:14.063794+0000","last_became_peered":"2024-08-17T17:36:14.063794+0000","last_unstale":"2024-08-17T17:37:55.403149+0000","last_undegraded":"2024-08-17T17:36:13.855770+0000","last_fullsized":"2024-08-17T17:36:13.742068+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4737,"log_dups_size":0,"ondisk_log_size":4737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207421440,"num_objects":3165,"num_object_clones":0,"num_object_copies":9495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3819,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4737,"num_write_kb":252864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-654},{"shards":"0,2,7","objects":3033}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":37,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.776881+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:37:45.776881+0000","last_peered":"2024-08-17T17:37:45.776881+0000","last_clean":"2024-08-17T17:37:45.776881+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:37:45.776881+0000","last_undegraded":"2024-08-17T17:37:45.776881+0000","last_fullsized":"2024-08-17T17:37:45.776881+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1263,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.748244+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:37:45.748244+0000","last_peered":"2024-08-17T17:37:45.748244+0000","last_clean":"2024-08-17T17:37:45.748244+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:37:45.748244+0000","last_undegraded":"2024-08-17T17:37:45.748244+0000","last_fullsized":"2024-08-17T17:37:45.748244+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1778,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.810238+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:37:45.810238+0000","last_peered":"2024-08-17T17:37:45.810238+0000","last_clean":"2024-08-17T17:37:45.810238+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:37:45.810238+0000","last_undegraded":"2024-08-17T17:37:45.810238+0000","last_fullsized":"2024-08-17T17:37:45.810238+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"1414'4907","reported_seq":10695,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:38:00.464230+0000","last_change":"2024-08-17T17:36:46.982456+0000","last_active":"2024-08-17T17:38:00.464230+0000","last_peered":"2024-08-17T17:38:00.464230+0000","last_clean":"2024-08-17T17:38:00.464230+0000","last_became_active":"2024-08-17T17:36:46.974782+0000","last_became_peered":"2024-08-17T17:36:46.974782+0000","last_unstale":"2024-08-17T17:38:00.464230+0000","last_undegraded":"2024-08-17T17:38:00.464230+0000","last_fullsized":"2024-08-17T17:38:00.464230+0000","mapping_epoch":1399,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4907,"log_dups_size":0,"ondisk_log_size":4907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:40.499321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207421440,"num_objects":3165,"num_object_clones":0,"num_object_copies":9495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4907,"num_write_kb":258304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2848,"num_bytes_recovered":186646528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1380,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.776373+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:37:45.776373+0000","last_peered":"2024-08-17T17:37:45.776373+0000","last_clean":"2024-08-17T17:37:45.776373+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:37:45.776373+0000","last_undegraded":"2024-08-17T17:37:45.776373+0000","last_fullsized":"2024-08-17T17:37:45.776373+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":194,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.776012+0000","last_change":"2024-08-17T17:36:12.754372+0000","last_active":"2024-08-17T17:37:45.776012+0000","last_peered":"2024-08-17T17:37:45.776012+0000","last_clean":"2024-08-17T17:37:45.776012+0000","last_became_active":"2024-08-17T17:36:12.746352+0000","last_became_peered":"2024-08-17T17:36:12.746352+0000","last_unstale":"2024-08-17T17:37:45.776012+0000","last_undegraded":"2024-08-17T17:37:45.776012+0000","last_fullsized":"2024-08-17T17:37:45.776012+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:07.807486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1648,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.823324+0000","last_change":"2024-08-17T17:36:15.771193+0000","last_active":"2024-08-17T17:37:51.823324+0000","last_peered":"2024-08-17T17:37:51.823324+0000","last_clean":"2024-08-17T17:37:51.823324+0000","last_became_active":"2024-08-17T17:36:15.770889+0000","last_became_peered":"2024-08-17T17:36:15.770889+0000","last_unstale":"2024-08-17T17:37:51.823324+0000","last_undegraded":"2024-08-17T17:37:51.823324+0000","last_fullsized":"2024-08-17T17:37:51.823324+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:22.978695+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.4","version":"1414'4872","reported_seq":10641,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:38:00.267579+0000","last_change":"2024-08-17T17:37:04.045576+0000","last_active":"2024-08-17T17:38:00.267579+0000","last_peered":"2024-08-17T17:38:00.267579+0000","last_clean":"2024-08-17T17:38:00.267579+0000","last_became_active":"2024-08-17T17:37:04.038810+0000","last_became_peered":"2024-08-17T17:37:04.038810+0000","last_unstale":"2024-08-17T17:38:00.267579+0000","last_undegraded":"2024-08-17T17:38:00.267579+0000","last_fullsized":"2024-08-17T17:38:00.267579+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4872,"log_dups_size":0,"ondisk_log_size":4872,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:27:04.725258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209321984,"num_objects":3194,"num_object_clones":0,"num_object_copies":9582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4872,"num_write_kb":258112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":176291840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1243,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.818081+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:37:51.818081+0000","last_peered":"2024-08-17T17:37:51.818081+0000","last_clean":"2024-08-17T17:37:51.818081+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:37:51.818081+0000","last_undegraded":"2024-08-17T17:37:51.818081+0000","last_fullsized":"2024-08-17T17:37:51.818081+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1406,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.809173+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:37:45.809173+0000","last_peered":"2024-08-17T17:37:45.809173+0000","last_clean":"2024-08-17T17:37:45.809173+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:37:45.809173+0000","last_undegraded":"2024-08-17T17:37:45.809173+0000","last_fullsized":"2024-08-17T17:37:45.809173+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":907,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:47.409717+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:37:47.409717+0000","last_peered":"2024-08-17T17:37:47.409717+0000","last_clean":"2024-08-17T17:37:47.409717+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:37:47.409717+0000","last_undegraded":"2024-08-17T17:37:47.409717+0000","last_fullsized":"2024-08-17T17:37:47.409717+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"1414'4905","reported_seq":10729,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:57.351736+0000","last_change":"2024-08-17T17:36:32.711490+0000","last_active":"2024-08-17T17:37:57.351736+0000","last_peered":"2024-08-17T17:37:57.351736+0000","last_clean":"2024-08-17T17:37:57.351736+0000","last_became_active":"2024-08-17T17:36:32.704370+0000","last_became_peered":"2024-08-17T17:36:32.704370+0000","last_unstale":"2024-08-17T17:37:57.351736+0000","last_undegraded":"2024-08-17T17:37:57.351736+0000","last_fullsized":"2024-08-17T17:37:57.351736+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4905,"log_dups_size":0,"ondisk_log_size":4905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:17.373879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209911808,"num_objects":3203,"num_object_clones":0,"num_object_copies":9609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4905,"num_write_kb":259456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2866,"num_bytes_recovered":187826176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1262,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.766700+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:37:45.766700+0000","last_peered":"2024-08-17T17:37:45.766700+0000","last_clean":"2024-08-17T17:37:45.766700+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:37:45.766700+0000","last_undegraded":"2024-08-17T17:37:45.766700+0000","last_fullsized":"2024-08-17T17:37:45.766700+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":143,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.811017+0000","last_change":"2024-08-17T17:35:55.217072+0000","last_active":"2024-08-17T17:37:45.811017+0000","last_peered":"2024-08-17T17:37:45.811017+0000","last_clean":"2024-08-17T17:37:45.811017+0000","last_became_active":"2024-08-17T17:35:55.216462+0000","last_became_peered":"2024-08-17T17:35:55.216462+0000","last_unstale":"2024-08-17T17:37:45.811017+0000","last_undegraded":"2024-08-17T17:37:45.811017+0000","last_fullsized":"2024-08-17T17:37:45.811017+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:34.937221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":889,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.749274+0000","last_change":"2024-08-17T17:35:55.215004+0000","last_active":"2024-08-17T17:37:45.749274+0000","last_peered":"2024-08-17T17:37:45.749274+0000","last_clean":"2024-08-17T17:37:45.749274+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:37:45.749274+0000","last_undegraded":"2024-08-17T17:37:45.749274+0000","last_fullsized":"2024-08-17T17:37:45.749274+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.6","version":"1414'4712","reported_seq":4830,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:55.305427+0000","last_change":"2024-08-17T17:35:14.868524+0000","last_active":"2024-08-17T17:37:55.305427+0000","last_peered":"2024-08-17T17:37:55.305427+0000","last_clean":"2024-08-17T17:37:55.305427+0000","last_became_active":"2024-08-17T17:35:14.854690+0000","last_became_peered":"2024-08-17T17:35:14.854690+0000","last_unstale":"2024-08-17T17:37:55.305427+0000","last_undegraded":"2024-08-17T17:37:55.305427+0000","last_fullsized":"2024-08-17T17:37:55.305427+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4712,"log_dups_size":0,"ondisk_log_size":4712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:05.388362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203423744,"num_objects":3104,"num_object_clones":0,"num_object_copies":9312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4712,"num_write_kb":250112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1448,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.775576+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:37:45.775576+0000","last_peered":"2024-08-17T17:37:45.775576+0000","last_clean":"2024-08-17T17:37:45.775576+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:37:45.775576+0000","last_undegraded":"2024-08-17T17:37:45.775576+0000","last_fullsized":"2024-08-17T17:37:45.775576+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":155,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.776911+0000","last_change":"2024-08-17T17:35:33.147672+0000","last_active":"2024-08-17T17:37:45.776911+0000","last_peered":"2024-08-17T17:37:45.776911+0000","last_clean":"2024-08-17T17:37:45.776911+0000","last_became_active":"2024-08-17T17:35:33.147283+0000","last_became_peered":"2024-08-17T17:35:33.147283+0000","last_unstale":"2024-08-17T17:37:45.776911+0000","last_undegraded":"2024-08-17T17:37:45.776911+0000","last_fullsized":"2024-08-17T17:37:45.776911+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:41.677709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1634,"reported_epoch":1414,"state":"active+clean+scrubbing","last_fresh":"2024-08-17T17:37:52.767469+0000","last_change":"2024-08-17T17:37:52.767469+0000","last_active":"2024-08-17T17:37:52.767469+0000","last_peered":"2024-08-17T17:37:52.767469+0000","last_clean":"2024-08-17T17:37:52.767469+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:37:52.767469+0000","last_undegraded":"2024-08-17T17:37:52.767469+0000","last_fullsized":"2024-08-17T17:37:52.767469+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:38.241235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:38.241235+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"1414'4794","reported_seq":4860,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:57.149431+0000","last_change":"2024-08-17T17:34:50.670013+0000","last_active":"2024-08-17T17:37:57.149431+0000","last_peered":"2024-08-17T17:37:57.149431+0000","last_clean":"2024-08-17T17:37:57.149431+0000","last_became_active":"2024-08-17T17:34:50.669299+0000","last_became_peered":"2024-08-17T17:34:50.669299+0000","last_unstale":"2024-08-17T17:37:57.149431+0000","last_undegraded":"2024-08-17T17:37:57.149431+0000","last_fullsized":"2024-08-17T17:37:57.149431+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4794,"log_dups_size":0,"ondisk_log_size":4794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:11.172227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205258752,"num_objects":3132,"num_object_clones":0,"num_object_copies":9396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4794,"num_write_kb":253632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1372,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.810432+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:37:45.810432+0000","last_peered":"2024-08-17T17:37:45.810432+0000","last_clean":"2024-08-17T17:37:45.810432+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:37:45.810432+0000","last_undegraded":"2024-08-17T17:37:45.810432+0000","last_fullsized":"2024-08-17T17:37:45.810432+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":142,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.777518+0000","last_change":"2024-08-17T17:35:33.140725+0000","last_active":"2024-08-17T17:37:45.777518+0000","last_peered":"2024-08-17T17:37:45.777518+0000","last_clean":"2024-08-17T17:37:45.777518+0000","last_became_active":"2024-08-17T17:35:33.140406+0000","last_became_peered":"2024-08-17T17:35:33.140406+0000","last_unstale":"2024-08-17T17:37:45.777518+0000","last_undegraded":"2024-08-17T17:37:45.777518+0000","last_fullsized":"2024-08-17T17:37:45.777518+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:54:51.875571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1662,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.776650+0000","last_change":"2024-08-17T17:35:33.138514+0000","last_active":"2024-08-17T17:37:45.776650+0000","last_peered":"2024-08-17T17:37:45.776650+0000","last_clean":"2024-08-17T17:37:45.776650+0000","last_became_active":"2024-08-17T17:35:33.137035+0000","last_became_peered":"2024-08-17T17:35:33.137035+0000","last_unstale":"2024-08-17T17:37:45.776650+0000","last_undegraded":"2024-08-17T17:37:45.776650+0000","last_fullsized":"2024-08-17T17:37:45.776650+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:50.565662+0000","scrub_duration":0.001487138,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1303,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.748073+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:37:45.748073+0000","last_peered":"2024-08-17T17:37:45.748073+0000","last_clean":"2024-08-17T17:37:45.748073+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:37:45.748073+0000","last_undegraded":"2024-08-17T17:37:45.748073+0000","last_fullsized":"2024-08-17T17:37:45.748073+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":74,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.847720+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:37:51.847720+0000","last_peered":"2024-08-17T17:37:51.847720+0000","last_clean":"2024-08-17T17:37:51.847720+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:37:51.847720+0000","last_undegraded":"2024-08-17T17:37:51.847720+0000","last_fullsized":"2024-08-17T17:37:51.847720+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1734,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.809684+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:37:45.809684+0000","last_peered":"2024-08-17T17:37:45.809684+0000","last_clean":"2024-08-17T17:37:45.809684+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:37:45.809684+0000","last_undegraded":"2024-08-17T17:37:45.809684+0000","last_fullsized":"2024-08-17T17:37:45.809684+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1321,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.775456+0000","last_change":"2024-08-17T17:35:55.205292+0000","last_active":"2024-08-17T17:37:45.775456+0000","last_peered":"2024-08-17T17:37:45.775456+0000","last_clean":"2024-08-17T17:37:45.775456+0000","last_became_active":"2024-08-17T17:35:55.204935+0000","last_became_peered":"2024-08-17T17:35:55.204935+0000","last_unstale":"2024-08-17T17:37:45.775456+0000","last_undegraded":"2024-08-17T17:37:45.775456+0000","last_fullsized":"2024-08-17T17:37:45.775456+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:52.277821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":74,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.847520+0000","last_change":"2024-08-17T17:36:12.749972+0000","last_active":"2024-08-17T17:37:51.847520+0000","last_peered":"2024-08-17T17:37:51.847520+0000","last_clean":"2024-08-17T17:37:51.847520+0000","last_became_active":"2024-08-17T17:36:12.749620+0000","last_became_peered":"2024-08-17T17:36:12.749620+0000","last_unstale":"2024-08-17T17:37:51.847520+0000","last_undegraded":"2024-08-17T17:37:51.847520+0000","last_fullsized":"2024-08-17T17:37:51.847520+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:04.994357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":950,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.846988+0000","last_change":"2024-08-17T17:36:20.832274+0000","last_active":"2024-08-17T17:37:51.846988+0000","last_peered":"2024-08-17T17:37:51.846988+0000","last_clean":"2024-08-17T17:37:51.846988+0000","last_became_active":"2024-08-17T17:36:20.831886+0000","last_became_peered":"2024-08-17T17:36:20.831886+0000","last_unstale":"2024-08-17T17:37:51.846988+0000","last_undegraded":"2024-08-17T17:37:51.846988+0000","last_fullsized":"2024-08-17T17:37:51.846988+0000","mapping_epoch":1390,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1391,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:05.248711+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1160,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.815054+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:37:51.815054+0000","last_peered":"2024-08-17T17:37:51.815054+0000","last_clean":"2024-08-17T17:37:51.815054+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:37:51.815054+0000","last_undegraded":"2024-08-17T17:37:51.815054+0000","last_fullsized":"2024-08-17T17:37:51.815054+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1404,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.811083+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:37:45.811083+0000","last_peered":"2024-08-17T17:37:45.811083+0000","last_clean":"2024-08-17T17:37:45.811083+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:37:45.811083+0000","last_undegraded":"2024-08-17T17:37:45.811083+0000","last_fullsized":"2024-08-17T17:37:45.811083+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1697,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.810166+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:37:45.810166+0000","last_peered":"2024-08-17T17:37:45.810166+0000","last_clean":"2024-08-17T17:37:45.810166+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:37:45.810166+0000","last_undegraded":"2024-08-17T17:37:45.810166+0000","last_fullsized":"2024-08-17T17:37:45.810166+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1346,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.834189+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:37:51.834189+0000","last_peered":"2024-08-17T17:37:51.834189+0000","last_clean":"2024-08-17T17:37:51.834189+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:37:51.834189+0000","last_undegraded":"2024-08-17T17:37:51.834189+0000","last_fullsized":"2024-08-17T17:37:51.834189+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1878,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-08-17T17:37:51.847269+0000","last_change":"2024-08-17T17:35:33.137411+0000","last_active":"2024-08-17T17:37:51.847269+0000","last_peered":"2024-08-17T17:37:51.847269+0000","last_clean":"2024-08-17T17:37:51.847269+0000","last_became_active":"2024-08-17T17:35:33.137073+0000","last_became_peered":"2024-08-17T17:35:33.137073+0000","last_unstale":"2024-08-17T17:37:51.847269+0000","last_undegraded":"2024-08-17T17:37:51.847269+0000","last_fullsized":"2024-08-17T17:37:51.847269+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:35.924539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1791,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-08-17T17:37:45.808494+0000","last_change":"2024-08-17T17:36:15.787445+0000","last_active":"2024-08-17T17:37:45.808494+0000","last_peered":"2024-08-17T17:37:45.808494+0000","last_clean":"2024-08-17T17:37:45.808494+0000","last_became_active":"2024-08-17T17:36:15.787099+0000","last_became_peered":"2024-08-17T17:36:15.787099+0000","last_unstale":"2024-08-17T17:37:45.808494+0000","last_undegraded":"2024-08-17T17:37:45.808494+0000","last_fullsized":"2024-08-17T17:37:45.808494+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:47.448168+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3360423960,"num_objects":51277,"num_object_clones":0,"num_object_copies":153831,"num_objects_missing_on_primary":6045,"num_objects_missing":6045,"num_objects_degraded":25555,"num_objects_misplaced":5457,"num_objects_unfound":0,"num_objects_dirty":51277,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":77605,"num_write_kb":4124161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20450,"num_bytes_recovered":1289486336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10458443776,"data_stored":10458431560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77505,"ondisk_log_size":77505,"up":48,"acting":42,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635092,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":27,"num_read_kb":61,"num_write":15,"num_write_kb":183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1314910,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162846,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1633792,"kb_used_data":1497080,"kb_used_omap":54,"kb_used_meta":136649,"kb_avail":92111360,"statfs":{"total":95995035648,"available":94322032640,"internally_reserved":0,"allocated":1533009920,"data_stored":1530367344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":139928597},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999321,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1356800,"kb_used_data":1194680,"kb_used_omap":47,"kb_used_meta":162064,"kb_avail":92388352,"statfs":{"total":95995035648,"available":94605672448,"internally_reserved":0,"allocated":1223352320,"data_stored":1220710721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":165954196},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627117,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1393252,"kb_used_data":386524,"kb_used_omap":42,"kb_used_meta":1006677,"kb_avail":92351900,"statfs":{"total":95995035648,"available":94568345600,"internally_reserved":0,"allocated":395800576,"data_stored":393181258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1030837308},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785594,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2953192,"kb_used_data":2144864,"kb_used_omap":48,"kb_used_meta":808271,"kb_avail":90791960,"statfs":{"total":95995035648,"available":92970967040,"internally_reserved":0,"allocated":2196340736,"data_stored":2193718293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":827670190},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725111,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1946032,"kb_used_data":1076584,"kb_used_omap":1,"kb_used_meta":869438,"kb_avail":91799120,"statfs":{"total":95995035648,"available":94002298880,"internally_reserved":0,"allocated":1102422016,"data_stored":1099791523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":890304966},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986748,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1878140,"kb_used_data":994740,"kb_used_omap":1,"kb_used_meta":883390,"kb_avail":91867012,"statfs":{"total":95995035648,"available":94071820288,"internally_reserved":0,"allocated":1018613760,"data_stored":1015974690,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":904591814},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706355,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2646064,"kb_used_data":1743592,"kb_used_omap":35,"kb_used_meta":902428,"kb_avail":91099088,"statfs":{"total":95995035648,"available":93285466112,"internally_reserved":0,"allocated":1785438208,"data_stored":1782807696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":924086327},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832128,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2285036,"kb_used_data":1241892,"kb_used_omap":65,"kb_used_meta":1043006,"kb_avail":91460116,"statfs":{"total":95995035648,"available":93655158784,"internally_reserved":0,"allocated":1271697408,"data_stored":1269071454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1068038331},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1261043712,"data_stored":1261043712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2185363456,"data_stored":2185363456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1775964160,"data_stored":1775960088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":395706368,"data_stored":395706368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1008603136,"data_stored":1008599064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1092026368,"data_stored":1092026368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1214648320,"data_stored":1214644248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1525088256,"data_stored":1525088256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:38:03.107 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:38:03.113 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:03.140 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:03.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:03.241 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:03.273 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:03.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:03.375 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:38:03.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:03.402 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:03.403 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:38:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:38:03] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.51.0" 2024-08-17T17:38:03.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:03.506 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:03.532 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:03.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:03.634 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:03.658 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:03.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:03.760 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:03.784 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:03.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:03.885 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:03.910 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:03.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:03.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:04.011 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:04.037 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:04.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:04.138 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:04.161 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:04.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:04.263 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:04.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:03 smithi063 ceph-mon[109843]: from='client.36330 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:04.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:03 smithi063 ceph-mon[109843]: pgmap v2198: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 25441/152253 objects degraded (16.710%); 4729/152253 objects misplaced (3.106%); 10 MiB/s, 163 objects/s recovering 2024-08-17T17:38:04.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:03 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 25441/152253 objects degraded (16.710%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:04.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:03 smithi063 ceph-mon[114756]: from='client.36330 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:04.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:03 smithi063 ceph-mon[114756]: pgmap v2198: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 25441/152253 objects degraded (16.710%); 4729/152253 objects misplaced (3.106%); 10 MiB/s, 163 objects/s recovering 2024-08-17T17:38:04.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:03 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 25441/152253 objects degraded (16.710%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:04.291 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:04.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:04.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:03 smithi162 ceph-mon[104668]: from='client.36330 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:04.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:03 smithi162 ceph-mon[104668]: pgmap v2198: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 161 op/s; 25441/152253 objects degraded (16.710%); 4729/152253 objects misplaced (3.106%); 10 MiB/s, 163 objects/s recovering 2024-08-17T17:38:04.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:03 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 25441/152253 objects degraded (16.710%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:04.393 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:04.416 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:04.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:04.517 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:04.542 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:04.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:04.644 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:04.667 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:04.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:04.769 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:04.793 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:04.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:04.895 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:04.923 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:04.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:04.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:05.025 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:05.049 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:05.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:05.151 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:05.174 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:05.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:05.276 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:05.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:05.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:05.401 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:05.426 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:05.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:05.528 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:05.555 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:05.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:05.657 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:05.685 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:05.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:05.786 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:05.810 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:05.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:05.912 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:05.936 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:05.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:05.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:06.038 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:06.062 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:06.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:06.147 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:05 smithi162 ceph-mon[104668]: pgmap v2199: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 25441/152139 objects degraded (16.722%); 4729/152139 objects misplaced (3.108%); 9.2 MiB/s, 147 objects/s recovering 2024-08-17T17:38:06.164 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:06.199 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:06.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:06.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:05 smithi063 ceph-mon[114756]: pgmap v2199: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 25441/152139 objects degraded (16.722%); 4729/152139 objects misplaced (3.108%); 9.2 MiB/s, 147 objects/s recovering 2024-08-17T17:38:06.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:05 smithi063 ceph-mon[109843]: pgmap v2199: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 141 op/s; 25441/152139 objects degraded (16.722%); 4729/152139 objects misplaced (3.108%); 9.2 MiB/s, 147 objects/s recovering 2024-08-17T17:38:06.301 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:06.330 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:06.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:06.432 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:06.459 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:06.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:06.561 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:06.586 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:06.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:06.688 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:06.717 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:06.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:06.819 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:06.846 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:06.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:06.948 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:06.971 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:06.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:06.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:07.073 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:07.098 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:07.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:07.200 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:07.223 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:07.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:07.325 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:07.349 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:07.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:07.450 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:07.478 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:07.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:07.580 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:07.608 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:07.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:07.710 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:07.734 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:07.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:07.836 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:07.859 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:07.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:07.961 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:07.984 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:07.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:07.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:08.086 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:08.106 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:07 smithi063 ceph-mon[114756]: pgmap v2200: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 197 op/s; 24494/149952 objects degraded (16.335%); 4006/149952 objects misplaced (2.672%); 14 MiB/s, 266 objects/s recovering 2024-08-17T17:38:08.107 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:07 smithi063 ceph-mon[114756]: osdmap e1415: 8 total, 8 up, 8 in 2024-08-17T17:38:08.107 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:07 smithi063 ceph-mon[109843]: pgmap v2200: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 197 op/s; 24494/149952 objects degraded (16.335%); 4006/149952 objects misplaced (2.672%); 14 MiB/s, 266 objects/s recovering 2024-08-17T17:38:08.107 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:07 smithi063 ceph-mon[109843]: osdmap e1415: 8 total, 8 up, 8 in 2024-08-17T17:38:08.111 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:08.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:08.213 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:08.237 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:08.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:08.339 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:08.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:07 smithi162 ceph-mon[104668]: pgmap v2200: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 197 op/s; 24494/149952 objects degraded (16.335%); 4006/149952 objects misplaced (2.672%); 14 MiB/s, 266 objects/s recovering 2024-08-17T17:38:08.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:07 smithi162 ceph-mon[104668]: osdmap e1415: 8 total, 8 up, 8 in 2024-08-17T17:38:08.364 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:08.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:08.466 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:08.491 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:08.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:08.535 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:38:08.592 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:08.619 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:08.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:08.722 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:08.750 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:08.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:08.852 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:08.885 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:38:08.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:08.902 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:08.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:09.003 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:09.028 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:09.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:09.130 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:09.155 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:09.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:09.215 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:08 smithi063 ceph-mon[114756]: osdmap e1416: 8 total, 8 up, 8 in 2024-08-17T17:38:09.216 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:08 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 24494/149952 objects degraded (16.335%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:09.216 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:08 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3850266809' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:38:09.216 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:08 smithi063 ceph-mon[109843]: osdmap e1416: 8 total, 8 up, 8 in 2024-08-17T17:38:09.216 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:08 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 24494/149952 objects degraded (16.335%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:09.216 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:08 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3850266809' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:38:09.257 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:09.284 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:09.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:09.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:08 smithi162 ceph-mon[104668]: osdmap e1416: 8 total, 8 up, 8 in 2024-08-17T17:38:09.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:08 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 24494/149952 objects degraded (16.335%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:09.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:08 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3850266809' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:38:09.385 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:09.410 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:09.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:09.512 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:09.536 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:09.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:09.639 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:09.668 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:09.671 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:09.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:09.673 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:09.774 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:09.802 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:09.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:09.905 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:09.931 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:09.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:09.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:09.962 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:09 smithi063 ceph-mon[109843]: pgmap v2203: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 185 op/s; 24494/149451 objects degraded (16.389%); 4006/149451 objects misplaced (2.680%); 14 MiB/s, 276 objects/s recovering 2024-08-17T17:38:09.962 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:09 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:09.962 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:09 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3850266809' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:38:09.962 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:09 smithi063 ceph-mon[109843]: osdmap e1417: 8 total, 8 up, 8 in 2024-08-17T17:38:09.963 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:09 smithi063 ceph-mon[114756]: pgmap v2203: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 185 op/s; 24494/149451 objects degraded (16.389%); 4006/149451 objects misplaced (2.680%); 14 MiB/s, 276 objects/s recovering 2024-08-17T17:38:09.963 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:09 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:09.963 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:09 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3850266809' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:38:09.963 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:09 smithi063 ceph-mon[114756]: osdmap e1417: 8 total, 8 up, 8 in 2024-08-17T17:38:10.033 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:10.056 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:10.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:10.158 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:10.188 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:10.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:10.290 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:10.316 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:10.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:10.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:09 smithi162 ceph-mon[104668]: pgmap v2203: 119 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 185 op/s; 24494/149451 objects degraded (16.389%); 4006/149451 objects misplaced (2.680%); 14 MiB/s, 276 objects/s recovering 2024-08-17T17:38:10.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:09 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:10.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:09 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3850266809' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:38:10.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:09 smithi162 ceph-mon[104668]: osdmap e1417: 8 total, 8 up, 8 in 2024-08-17T17:38:10.418 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:10.447 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:10.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:10.549 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:10.575 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:10.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:10.677 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:10.701 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:10.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:10.803 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:10.832 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:10.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:10.934 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:10.961 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:10.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:10.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:11.063 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:11.091 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:11.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:11.194 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:11.220 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:11.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.222 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:11.323 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:11.348 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:11.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:11.450 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:11.474 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:11.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:11.576 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:11.600 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:11.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:11.675 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:38:11.702 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:11.752 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:11.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:11.854 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:11.886 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:11.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:11.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:11.988 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:12.011 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:12.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:11 smithi063 ceph-mon[109843]: pgmap v2205: 119 pgs: 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 155 op/s; 24368/149346 objects degraded (16.316%); 4006/149346 objects misplaced (2.682%); 10 MiB/s, 259 objects/s recovering 2024-08-17T17:38:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:11 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:11 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:11 smithi063 ceph-mon[114756]: pgmap v2205: 119 pgs: 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 155 op/s; 24368/149346 objects degraded (16.316%); 4006/149346 objects misplaced (2.682%); 10 MiB/s, 259 objects/s recovering 2024-08-17T17:38:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:11 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:12.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:11 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:11 smithi162 ceph-mon[104668]: pgmap v2205: 119 pgs: 1 active+recovery_wait+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 155 op/s; 24368/149346 objects degraded (16.316%); 4006/149346 objects misplaced (2.682%); 10 MiB/s, 259 objects/s recovering 2024-08-17T17:38:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:11 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:12.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:11 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:12.113 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:12.136 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:12.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:12.238 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:12.263 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:12.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:12.365 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:12.393 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:12.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:12.496 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:12.524 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:12.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:12.627 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:12.651 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:12.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:12.754 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:12.778 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:12.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:12.880 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:12.910 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:12.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:12.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:13.012 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:13.038 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:13.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:13.140 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:13.167 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:13.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:13.269 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:13.293 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:13.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:13.395 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:13.422 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:13.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:13.524 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:13.548 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:13.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:13.650 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:13.676 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:13.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.678 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:13.779 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:13.780 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:38:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:38:13] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.51.0" 2024-08-17T17:38:13.805 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:13.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:13.907 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:13.929 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:13.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:13.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:14.031 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:14.056 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:14.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:13 smithi162 ceph-mon[104668]: pgmap v2206: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 19070/147216 objects degraded (12.954%); 8450/147216 objects misplaced (5.740%); 6.7 MiB/s, 127 objects/s recovering 2024-08-17T17:38:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:13 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 19070/147216 objects degraded (12.954%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:14.158 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:14.183 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:14.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:14.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:13 smithi063 ceph-mon[114756]: pgmap v2206: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 19070/147216 objects degraded (12.954%); 8450/147216 objects misplaced (5.740%); 6.7 MiB/s, 127 objects/s recovering 2024-08-17T17:38:14.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:13 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 19070/147216 objects degraded (12.954%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:14.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:13 smithi063 ceph-mon[109843]: pgmap v2206: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 151 op/s; 19070/147216 objects degraded (12.954%); 8450/147216 objects misplaced (5.740%); 6.7 MiB/s, 127 objects/s recovering 2024-08-17T17:38:14.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:13 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 19070/147216 objects degraded (12.954%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:14.285 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:14.309 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:14.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:14.411 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:14.437 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:14.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:14.539 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:14.563 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:14.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:14.664 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:14.687 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:14.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:14.789 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:14.814 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:14.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:14.916 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:14.939 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:14.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:14.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:15.041 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:15.066 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:15.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:15.168 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:15.192 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:15.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:15.294 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:15.321 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:15.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.322 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:15.423 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:15.447 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:15.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:15.549 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:15.579 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:15.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:15.681 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:15.707 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:15.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:15.809 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:15.836 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:15.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:15.938 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:15.961 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:15.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:15.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:16.063 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:16.089 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:16.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:16.094 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:15 smithi063 ceph-mon[114756]: pgmap v2207: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 19070/146889 objects degraded (12.983%); 8450/146889 objects misplaced (5.753%); 5.1 MiB/s, 98 objects/s recovering 2024-08-17T17:38:16.094 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:15 smithi063 ceph-mon[109843]: pgmap v2207: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 19070/146889 objects degraded (12.983%); 8450/146889 objects misplaced (5.753%); 5.1 MiB/s, 98 objects/s recovering 2024-08-17T17:38:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:15 smithi162 ceph-mon[104668]: pgmap v2207: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 131 op/s; 19070/146889 objects degraded (12.983%); 8450/146889 objects misplaced (5.753%); 5.1 MiB/s, 98 objects/s recovering 2024-08-17T17:38:16.190 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:16.217 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:16.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:16.319 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:16.344 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:16.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:16.446 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:16.473 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:16.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:16.574 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:16.599 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:16.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:16.701 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:16.730 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:16.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:16.832 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:16.856 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:16.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:16.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:16.936 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:16.958 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:17.006 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:17.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:17.072 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:17.072 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:17.107 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:17.132 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:17.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:17.234 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:17.259 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:17.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:17.362 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:17.389 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:17.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:17.491 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:17.519 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:17.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:17.622 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:17.648 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:17.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:17.750 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:17.776 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:17.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:17.878 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:17.904 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:17.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:17.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:18.005 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:18.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.034 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:18.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:18.135 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:18.161 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:18.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:18.263 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:18.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:18 smithi063 ceph-mon[114756]: pgmap v2208: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 175 op/s; 18737/144831 objects degraded (12.937%); 7843/144831 objects misplaced (5.415%); 9.6 MiB/s, 168 objects/s recovering 2024-08-17T17:38:18.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:18 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:18.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:18 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:18.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:18 smithi063 ceph-mon[109843]: pgmap v2208: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 175 op/s; 18737/144831 objects degraded (12.937%); 7843/144831 objects misplaced (5.415%); 9.6 MiB/s, 168 objects/s recovering 2024-08-17T17:38:18.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:18 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:18.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:18 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:18.289 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:18.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:18.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:18 smithi162 ceph-mon[104668]: pgmap v2208: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 175 op/s; 18737/144831 objects degraded (12.937%); 7843/144831 objects misplaced (5.415%); 9.6 MiB/s, 168 objects/s recovering 2024-08-17T17:38:18.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:18 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:18.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:18 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:18.392 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:18.419 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:18.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:18.466 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:38:18.468 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:38:18.522 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:18.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:18.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:18.648 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:18.673 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:18.776 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:18.802 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:18.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:18.885 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:18.904 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:18.931 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:18.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:18.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:19.033 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:19.058 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:19.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:19.160 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:19.187 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:19.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:19.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:19 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 18737/144831 objects degraded (12.937%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:19.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:19 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 18737/144831 objects degraded (12.937%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:19.289 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:19.312 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:19.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:19.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:19 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 18737/144831 objects degraded (12.937%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:19.366 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2208,"stamp":"2024-08-17T17:38:16.640829+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3162752689,"num_objects":48277,"num_object_clones":0,"num_object_copies":144831,"num_objects_missing_on_primary":5545,"num_objects_missing":5545,"num_objects_degraded":18737,"num_objects_misplaced":7843,"num_objects_unfound":0,"num_objects_dirty":48277,"num_whiteouts":0,"num_read":28,"num_read_kb":62,"num_write":80710,"num_write_kb":4124377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24169,"num_bytes_recovered":1494029021,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85209,"ondisk_log_size":85209,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":375,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15503732,"kb_used_data":9697972,"kb_used_omap":298,"kb_used_meta":5805397,"kb_avail":734457484,"statfs":{"total":767960285184,"available":752084463616,"internally_reserved":0,"allocated":9930723328,"data_stored":9909577171,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":5944727057},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,3,1,1,3,6,3],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-100925440,"num_objects":-1540,"num_object_clones":0,"num_object_copies":-4620,"num_objects_missing_on_primary":-328,"num_objects_missing":-328,"num_objects_degraded":-5757,"num_objects_misplaced":3837,"num_objects_unfound":0,"num_objects_dirty":-1540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1540,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1480,"num_bytes_recovered":88735744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.772394"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":40,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.887327+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:38:08.887327+0000","last_peered":"2024-08-17T17:38:08.887327+0000","last_clean":"2024-08-17T17:38:08.887327+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:38:08.887327+0000","last_undegraded":"2024-08-17T17:38:08.887327+0000","last_fullsized":"2024-08-17T17:38:08.887327+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1431,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.877472+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:38:07.877472+0000","last_peered":"2024-08-17T17:38:07.877472+0000","last_clean":"2024-08-17T17:38:07.877472+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:38:07.877472+0000","last_undegraded":"2024-08-17T17:38:07.877472+0000","last_fullsized":"2024-08-17T17:38:07.877472+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1746,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.878713+0000","last_change":"2024-08-17T17:36:15.771150+0000","last_active":"2024-08-17T17:38:07.878713+0000","last_peered":"2024-08-17T17:38:07.878713+0000","last_clean":"2024-08-17T17:38:07.878713+0000","last_became_active":"2024-08-17T17:36:15.770520+0000","last_became_peered":"2024-08-17T17:36:15.770520+0000","last_unstale":"2024-08-17T17:38:07.878713+0000","last_undegraded":"2024-08-17T17:38:07.878713+0000","last_fullsized":"2024-08-17T17:38:07.878713+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:17.552796+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":81,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.874086+0000","last_change":"2024-08-17T17:36:12.756709+0000","last_active":"2024-08-17T17:38:07.874086+0000","last_peered":"2024-08-17T17:38:07.874086+0000","last_clean":"2024-08-17T17:38:07.874086+0000","last_became_active":"2024-08-17T17:36:12.756385+0000","last_became_peered":"2024-08-17T17:36:12.756385+0000","last_unstale":"2024-08-17T17:38:07.874086+0000","last_undegraded":"2024-08-17T17:38:07.874086+0000","last_fullsized":"2024-08-17T17:38:07.874086+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:21.420282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":125,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.890967+0000","last_change":"2024-08-17T17:36:12.762800+0000","last_active":"2024-08-17T17:38:08.890967+0000","last_peered":"2024-08-17T17:38:08.890967+0000","last_clean":"2024-08-17T17:38:08.890967+0000","last_became_active":"2024-08-17T17:36:12.759977+0000","last_became_peered":"2024-08-17T17:36:12.759977+0000","last_unstale":"2024-08-17T17:38:08.890967+0000","last_undegraded":"2024-08-17T17:38:08.890967+0000","last_fullsized":"2024-08-17T17:38:08.890967+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:02.162681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1625,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.873657+0000","last_change":"2024-08-17T17:35:56.281561+0000","last_active":"2024-08-17T17:38:07.873657+0000","last_peered":"2024-08-17T17:38:07.873657+0000","last_clean":"2024-08-17T17:38:07.873657+0000","last_became_active":"2024-08-17T17:35:56.278638+0000","last_became_peered":"2024-08-17T17:35:56.278638+0000","last_unstale":"2024-08-17T17:38:07.873657+0000","last_undegraded":"2024-08-17T17:38:07.873657+0000","last_fullsized":"2024-08-17T17:38:07.873657+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1364,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.880475+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:38:07.880475+0000","last_peered":"2024-08-17T17:38:07.880475+0000","last_clean":"2024-08-17T17:38:07.880475+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:38:07.880475+0000","last_undegraded":"2024-08-17T17:38:07.880475+0000","last_fullsized":"2024-08-17T17:38:07.880475+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":40,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.877603+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:38:07.877603+0000","last_peered":"2024-08-17T17:38:07.877603+0000","last_clean":"2024-08-17T17:38:07.877603+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:38:07.877603+0000","last_undegraded":"2024-08-17T17:38:07.877603+0000","last_fullsized":"2024-08-17T17:38:07.877603+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1409,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.880039+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:38:07.880039+0000","last_peered":"2024-08-17T17:38:07.880039+0000","last_clean":"2024-08-17T17:38:07.880039+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:38:07.880039+0000","last_undegraded":"2024-08-17T17:38:07.880039+0000","last_fullsized":"2024-08-17T17:38:07.880039+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1417,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.878245+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:38:07.878245+0000","last_peered":"2024-08-17T17:38:07.878245+0000","last_clean":"2024-08-17T17:38:07.878245+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:38:07.878245+0000","last_undegraded":"2024-08-17T17:38:07.878245+0000","last_fullsized":"2024-08-17T17:38:07.878245+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":708,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.873958+0000","last_change":"2024-08-17T17:36:15.773315+0000","last_active":"2024-08-17T17:38:07.873958+0000","last_peered":"2024-08-17T17:38:07.873958+0000","last_clean":"2024-08-17T17:38:07.873958+0000","last_became_active":"2024-08-17T17:36:15.772961+0000","last_became_peered":"2024-08-17T17:36:15.772961+0000","last_unstale":"2024-08-17T17:38:07.873958+0000","last_undegraded":"2024-08-17T17:38:07.873958+0000","last_fullsized":"2024-08-17T17:38:07.873958+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:10.190221+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":34,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.882157+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:38:08.882157+0000","last_peered":"2024-08-17T17:38:08.882157+0000","last_clean":"2024-08-17T17:38:08.882157+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:38:08.882157+0000","last_undegraded":"2024-08-17T17:38:08.882157+0000","last_fullsized":"2024-08-17T17:38:08.882157+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":125,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.891807+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:38:08.891807+0000","last_peered":"2024-08-17T17:38:08.891807+0000","last_clean":"2024-08-17T17:38:08.891807+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:38:08.891807+0000","last_undegraded":"2024-08-17T17:38:08.891807+0000","last_fullsized":"2024-08-17T17:38:08.891807+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1498,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.930758+0000","last_change":"2024-08-17T17:36:24.225739+0000","last_active":"2024-08-17T17:38:07.930758+0000","last_peered":"2024-08-17T17:38:07.930758+0000","last_clean":"2024-08-17T17:38:07.930758+0000","last_became_active":"2024-08-17T17:36:24.225250+0000","last_became_peered":"2024-08-17T17:36:24.225250+0000","last_unstale":"2024-08-17T17:38:07.930758+0000","last_undegraded":"2024-08-17T17:38:07.930758+0000","last_fullsized":"2024-08-17T17:38:07.930758+0000","mapping_epoch":1393,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1394,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.290656+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1437,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.877973+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:38:07.877973+0000","last_peered":"2024-08-17T17:38:07.877973+0000","last_clean":"2024-08-17T17:38:07.877973+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:38:07.877973+0000","last_undegraded":"2024-08-17T17:38:07.877973+0000","last_fullsized":"2024-08-17T17:38:07.877973+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1845,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.878882+0000","last_change":"2024-08-17T17:36:12.752970+0000","last_active":"2024-08-17T17:38:07.878882+0000","last_peered":"2024-08-17T17:38:07.878882+0000","last_clean":"2024-08-17T17:38:07.878882+0000","last_became_active":"2024-08-17T17:36:12.752311+0000","last_became_peered":"2024-08-17T17:36:12.752311+0000","last_unstale":"2024-08-17T17:38:07.878882+0000","last_undegraded":"2024-08-17T17:38:07.878882+0000","last_fullsized":"2024-08-17T17:38:07.878882+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:38.131419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1792,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.884970+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:38:07.884970+0000","last_peered":"2024-08-17T17:38:07.884970+0000","last_clean":"2024-08-17T17:38:07.884970+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:38:07.884970+0000","last_undegraded":"2024-08-17T17:38:07.884970+0000","last_fullsized":"2024-08-17T17:38:07.884970+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1497,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.908267+0000","last_change":"2024-08-17T17:36:18.750494+0000","last_active":"2024-08-17T17:38:08.908267+0000","last_peered":"2024-08-17T17:38:08.908267+0000","last_clean":"2024-08-17T17:38:08.908267+0000","last_became_active":"2024-08-17T17:36:18.750092+0000","last_became_peered":"2024-08-17T17:36:18.750092+0000","last_unstale":"2024-08-17T17:38:08.908267+0000","last_undegraded":"2024-08-17T17:38:08.908267+0000","last_fullsized":"2024-08-17T17:38:08.908267+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:36.599727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1126,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.884108+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:38:08.884108+0000","last_peered":"2024-08-17T17:38:08.884108+0000","last_clean":"2024-08-17T17:38:08.884108+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:38:08.884108+0000","last_undegraded":"2024-08-17T17:38:08.884108+0000","last_fullsized":"2024-08-17T17:38:08.884108+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":959,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.879620+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:38:07.879620+0000","last_peered":"2024-08-17T17:38:07.879620+0000","last_clean":"2024-08-17T17:38:07.879620+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:38:07.879620+0000","last_undegraded":"2024-08-17T17:38:07.879620+0000","last_fullsized":"2024-08-17T17:38:07.879620+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":34,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.889606+0000","last_change":"2024-08-17T17:37:11.075505+0000","last_active":"2024-08-17T17:38:08.889606+0000","last_peered":"2024-08-17T17:38:08.889606+0000","last_clean":"2024-08-17T17:38:08.889606+0000","last_became_active":"2024-08-17T17:37:11.075131+0000","last_became_peered":"2024-08-17T17:37:11.075131+0000","last_unstale":"2024-08-17T17:38:08.889606+0000","last_undegraded":"2024-08-17T17:38:08.889606+0000","last_fullsized":"2024-08-17T17:38:08.889606+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:39.672054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1272,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.874448+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:38:07.874448+0000","last_peered":"2024-08-17T17:38:07.874448+0000","last_clean":"2024-08-17T17:38:07.874448+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:38:07.874448+0000","last_undegraded":"2024-08-17T17:38:07.874448+0000","last_fullsized":"2024-08-17T17:38:07.874448+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1649,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.877685+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:38:07.877685+0000","last_peered":"2024-08-17T17:38:07.877685+0000","last_clean":"2024-08-17T17:38:07.877685+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:38:07.877685+0000","last_undegraded":"2024-08-17T17:38:07.877685+0000","last_fullsized":"2024-08-17T17:38:07.877685+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":80,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.906342+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:38:08.906342+0000","last_peered":"2024-08-17T17:38:08.906342+0000","last_clean":"2024-08-17T17:38:08.906342+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:38:08.906342+0000","last_undegraded":"2024-08-17T17:38:08.906342+0000","last_fullsized":"2024-08-17T17:38:08.906342+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1246,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.881270+0000","last_change":"2024-08-17T17:36:12.755873+0000","last_active":"2024-08-17T17:38:07.881270+0000","last_peered":"2024-08-17T17:38:07.881270+0000","last_clean":"2024-08-17T17:38:07.881270+0000","last_became_active":"2024-08-17T17:36:12.755366+0000","last_became_peered":"2024-08-17T17:36:12.755366+0000","last_unstale":"2024-08-17T17:38:07.881270+0000","last_undegraded":"2024-08-17T17:38:07.881270+0000","last_fullsized":"2024-08-17T17:38:07.881270+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:36.523214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1600,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.917014+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:38:07.917014+0000","last_peered":"2024-08-17T17:38:07.917014+0000","last_clean":"2024-08-17T17:38:07.917014+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:38:07.917014+0000","last_undegraded":"2024-08-17T17:38:07.917014+0000","last_fullsized":"2024-08-17T17:38:07.917014+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":784,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.881374+0000","last_change":"2024-08-17T17:35:55.200519+0000","last_active":"2024-08-17T17:38:08.881374+0000","last_peered":"2024-08-17T17:38:08.881374+0000","last_clean":"2024-08-17T17:38:08.881374+0000","last_became_active":"2024-08-17T17:35:55.200230+0000","last_became_peered":"2024-08-17T17:35:55.200230+0000","last_unstale":"2024-08-17T17:38:08.881374+0000","last_undegraded":"2024-08-17T17:38:08.881374+0000","last_fullsized":"2024-08-17T17:38:08.881374+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:33.454851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1423,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.880631+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:38:07.880631+0000","last_peered":"2024-08-17T17:38:07.880631+0000","last_clean":"2024-08-17T17:38:07.880631+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:38:07.880631+0000","last_undegraded":"2024-08-17T17:38:07.880631+0000","last_fullsized":"2024-08-17T17:38:07.880631+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1698,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.891539+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:38:08.891539+0000","last_peered":"2024-08-17T17:38:08.891539+0000","last_clean":"2024-08-17T17:38:08.891539+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:38:08.891539+0000","last_undegraded":"2024-08-17T17:38:08.891539+0000","last_fullsized":"2024-08-17T17:38:08.891539+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:22.077229+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":245,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.905003+0000","last_change":"2024-08-17T17:35:33.145476+0000","last_active":"2024-08-17T17:38:08.905003+0000","last_peered":"2024-08-17T17:38:08.905003+0000","last_clean":"2024-08-17T17:38:08.905003+0000","last_became_active":"2024-08-17T17:35:33.145125+0000","last_became_peered":"2024-08-17T17:35:33.145125+0000","last_unstale":"2024-08-17T17:38:08.905003+0000","last_undegraded":"2024-08-17T17:38:08.905003+0000","last_fullsized":"2024-08-17T17:38:08.905003+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:45.298064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":928,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.888011+0000","last_change":"2024-08-17T17:35:56.248612+0000","last_active":"2024-08-17T17:38:08.888011+0000","last_peered":"2024-08-17T17:38:08.888011+0000","last_clean":"2024-08-17T17:38:08.888011+0000","last_became_active":"2024-08-17T17:35:56.244364+0000","last_became_peered":"2024-08-17T17:35:56.244364+0000","last_unstale":"2024-08-17T17:38:08.888011+0000","last_undegraded":"2024-08-17T17:38:08.888011+0000","last_fullsized":"2024-08-17T17:38:08.888011+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:41.064165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":867,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.880978+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:38:07.880978+0000","last_peered":"2024-08-17T17:38:07.880978+0000","last_clean":"2024-08-17T17:38:07.880978+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:38:07.880978+0000","last_undegraded":"2024-08-17T17:38:07.880978+0000","last_fullsized":"2024-08-17T17:38:07.880978+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1403,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.878810+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:38:07.878810+0000","last_peered":"2024-08-17T17:38:07.878810+0000","last_clean":"2024-08-17T17:38:07.878810+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:38:07.878810+0000","last_undegraded":"2024-08-17T17:38:07.878810+0000","last_fullsized":"2024-08-17T17:38:07.878810+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1422,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.878316+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:38:07.878316+0000","last_peered":"2024-08-17T17:38:07.878316+0000","last_clean":"2024-08-17T17:38:07.878316+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:38:07.878316+0000","last_undegraded":"2024-08-17T17:38:07.878316+0000","last_fullsized":"2024-08-17T17:38:07.878316+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":919,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.883008+0000","last_change":"2024-08-17T17:36:15.774881+0000","last_active":"2024-08-17T17:38:08.883008+0000","last_peered":"2024-08-17T17:38:08.883008+0000","last_clean":"2024-08-17T17:38:08.883008+0000","last_became_active":"2024-08-17T17:36:15.774435+0000","last_became_peered":"2024-08-17T17:36:15.774435+0000","last_unstale":"2024-08-17T17:38:08.883008+0000","last_undegraded":"2024-08-17T17:38:08.883008+0000","last_fullsized":"2024-08-17T17:38:08.883008+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:52.015854+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"1417'5205","reported_seq":7183,"reported_epoch":1417,"state":"active+recovering+remapped","last_fresh":"2024-08-17T17:38:15.664181+0000","last_change":"2024-08-17T17:38:05.928872+0000","last_active":"2024-08-17T17:38:15.664181+0000","last_peered":"2024-08-17T17:38:15.664181+0000","last_clean":"2024-08-17T17:36:13.740039+0000","last_became_active":"2024-08-17T17:36:19.066494+0000","last_became_peered":"2024-08-17T17:36:19.066494+0000","last_unstale":"2024-08-17T17:38:15.664181+0000","last_undegraded":"2024-08-17T17:38:15.664181+0000","last_fullsized":"2024-08-17T17:38:15.664181+0000","mapping_epoch":1388,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5205,"log_dups_size":0,"ondisk_log_size":5205,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201916416,"num_objects":3081,"num_object_clones":0,"num_object_copies":9243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3091,"num_objects_unfound":0,"num_objects_dirty":3081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5205,"num_write_kb":265152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":918,"num_bytes_recovered":60162048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[0,1,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":-10}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1370,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.877448+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:38:07.877448+0000","last_peered":"2024-08-17T17:38:07.877448+0000","last_clean":"2024-08-17T17:38:07.877448+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:38:07.877448+0000","last_undegraded":"2024-08-17T17:38:07.877448+0000","last_fullsized":"2024-08-17T17:38:07.877448+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1266,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.874313+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:38:07.874313+0000","last_peered":"2024-08-17T17:38:07.874313+0000","last_clean":"2024-08-17T17:38:07.874313+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:38:07.874313+0000","last_undegraded":"2024-08-17T17:38:07.874313+0000","last_fullsized":"2024-08-17T17:38:07.874313+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":312,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.885551+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:38:08.885551+0000","last_peered":"2024-08-17T17:38:08.885551+0000","last_clean":"2024-08-17T17:38:08.885551+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:38:08.885551+0000","last_undegraded":"2024-08-17T17:38:08.885551+0000","last_fullsized":"2024-08-17T17:38:08.885551+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1580,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.904510+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:38:08.904510+0000","last_peered":"2024-08-17T17:38:08.904510+0000","last_clean":"2024-08-17T17:38:08.904510+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:38:08.904510+0000","last_undegraded":"2024-08-17T17:38:08.904510+0000","last_fullsized":"2024-08-17T17:38:08.904510+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:25.751871+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.f","version":"1417'5165","reported_seq":5303,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:15.562049+0000","last_change":"2024-08-17T17:35:34.179168+0000","last_active":"2024-08-17T17:38:15.562049+0000","last_peered":"2024-08-17T17:38:15.562049+0000","last_clean":"2024-08-17T17:38:15.562049+0000","last_became_active":"2024-08-17T17:35:34.133493+0000","last_became_peered":"2024-08-17T17:35:34.133493+0000","last_unstale":"2024-08-17T17:38:15.562049+0000","last_undegraded":"2024-08-17T17:38:15.562049+0000","last_fullsized":"2024-08-17T17:38:15.562049+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5165,"log_dups_size":0,"ondisk_log_size":5165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:24:11.188566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199163904,"num_objects":3039,"num_object_clones":0,"num_object_copies":9117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5165,"num_write_kb":262528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":87,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.906115+0000","last_change":"2024-08-17T17:37:11.079712+0000","last_active":"2024-08-17T17:38:08.906115+0000","last_peered":"2024-08-17T17:38:08.906115+0000","last_clean":"2024-08-17T17:38:08.906115+0000","last_became_active":"2024-08-17T17:37:11.079253+0000","last_became_peered":"2024-08-17T17:37:11.079253+0000","last_unstale":"2024-08-17T17:38:08.906115+0000","last_undegraded":"2024-08-17T17:38:08.906115+0000","last_fullsized":"2024-08-17T17:38:08.906115+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:53.405657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":911,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.873883+0000","last_change":"2024-08-17T17:35:56.223396+0000","last_active":"2024-08-17T17:38:07.873883+0000","last_peered":"2024-08-17T17:38:07.873883+0000","last_clean":"2024-08-17T17:38:07.873883+0000","last_became_active":"2024-08-17T17:35:56.212413+0000","last_became_peered":"2024-08-17T17:35:56.212413+0000","last_unstale":"2024-08-17T17:38:07.873883+0000","last_undegraded":"2024-08-17T17:38:07.873883+0000","last_fullsized":"2024-08-17T17:38:07.873883+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:44.185174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1622,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.887557+0000","last_change":"2024-08-17T17:35:33.136068+0000","last_active":"2024-08-17T17:38:08.887557+0000","last_peered":"2024-08-17T17:38:08.887557+0000","last_clean":"2024-08-17T17:38:08.887557+0000","last_became_active":"2024-08-17T17:35:33.135591+0000","last_became_peered":"2024-08-17T17:35:33.135591+0000","last_unstale":"2024-08-17T17:38:08.887557+0000","last_undegraded":"2024-08-17T17:38:08.887557+0000","last_fullsized":"2024-08-17T17:38:08.887557+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:52.252561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1776,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.879205+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:38:07.879205+0000","last_peered":"2024-08-17T17:38:07.879205+0000","last_clean":"2024-08-17T17:38:07.879205+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:38:07.879205+0000","last_undegraded":"2024-08-17T17:38:07.879205+0000","last_fullsized":"2024-08-17T17:38:07.879205+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"1417'5025","reported_seq":5148,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:15.414028+0000","last_change":"2024-08-17T17:35:14.861823+0000","last_active":"2024-08-17T17:38:15.414028+0000","last_peered":"2024-08-17T17:38:15.414028+0000","last_clean":"2024-08-17T17:38:15.414028+0000","last_became_active":"2024-08-17T17:35:14.854109+0000","last_became_peered":"2024-08-17T17:35:14.854109+0000","last_unstale":"2024-08-17T17:38:15.414028+0000","last_undegraded":"2024-08-17T17:38:15.414028+0000","last_fullsized":"2024-08-17T17:38:15.414028+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5025,"log_dups_size":0,"ondisk_log_size":5025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:46:40.778197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197984256,"num_objects":3021,"num_object_clones":0,"num_object_copies":9063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5025,"num_write_kb":257472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":989,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.879751+0000","last_change":"2024-08-17T17:35:33.138467+0000","last_active":"2024-08-17T17:38:07.879751+0000","last_peered":"2024-08-17T17:38:07.879751+0000","last_clean":"2024-08-17T17:38:07.879751+0000","last_became_active":"2024-08-17T17:35:33.138174+0000","last_became_peered":"2024-08-17T17:35:33.138174+0000","last_unstale":"2024-08-17T17:38:07.879751+0000","last_undegraded":"2024-08-17T17:38:07.879751+0000","last_fullsized":"2024-08-17T17:38:07.879751+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:08.860439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1373,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.917088+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:38:07.917088+0000","last_peered":"2024-08-17T17:38:07.917088+0000","last_clean":"2024-08-17T17:38:07.917088+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:38:07.917088+0000","last_undegraded":"2024-08-17T17:38:07.917088+0000","last_fullsized":"2024-08-17T17:38:07.917088+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":118,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.891725+0000","last_change":"2024-08-17T17:35:14.854881+0000","last_active":"2024-08-17T17:38:08.891725+0000","last_peered":"2024-08-17T17:38:08.891725+0000","last_clean":"2024-08-17T17:38:08.891725+0000","last_became_active":"2024-08-17T17:35:14.854121+0000","last_became_peered":"2024-08-17T17:35:14.854121+0000","last_unstale":"2024-08-17T17:38:08.891725+0000","last_undegraded":"2024-08-17T17:38:08.891725+0000","last_fullsized":"2024-08-17T17:38:08.891725+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.863624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":930,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.874163+0000","last_change":"2024-08-17T17:36:15.773050+0000","last_active":"2024-08-17T17:38:07.874163+0000","last_peered":"2024-08-17T17:38:07.874163+0000","last_clean":"2024-08-17T17:38:07.874163+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:38:07.874163+0000","last_undegraded":"2024-08-17T17:38:07.874163+0000","last_fullsized":"2024-08-17T17:38:07.874163+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:39.212195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:39.212195+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"1417'5035","reported_seq":10864,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:15.229904+0000","last_change":"2024-08-17T17:37:21.409979+0000","last_active":"2024-08-17T17:38:15.229904+0000","last_peered":"2024-08-17T17:38:15.229904+0000","last_clean":"2024-08-17T17:38:15.229904+0000","last_became_active":"2024-08-17T17:37:21.398111+0000","last_became_peered":"2024-08-17T17:37:21.398111+0000","last_unstale":"2024-08-17T17:38:15.229904+0000","last_undegraded":"2024-08-17T17:38:15.229904+0000","last_fullsized":"2024-08-17T17:38:15.229904+0000","mapping_epoch":1407,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5035,"log_dups_size":0,"ondisk_log_size":5035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:19.299395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196345880,"num_objects":2997,"num_object_clones":0,"num_object_copies":8991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2997,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5035,"num_write_kb":256961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2850,"num_bytes_recovered":168820736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1031,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.879581+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:38:07.879581+0000","last_peered":"2024-08-17T17:38:07.879581+0000","last_clean":"2024-08-17T17:38:07.879581+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:38:07.879581+0000","last_undegraded":"2024-08-17T17:38:07.879581+0000","last_fullsized":"2024-08-17T17:38:07.879581+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":212,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.882689+0000","last_change":"2024-08-17T17:35:33.145351+0000","last_active":"2024-08-17T17:38:08.882689+0000","last_peered":"2024-08-17T17:38:08.882689+0000","last_clean":"2024-08-17T17:38:08.882689+0000","last_became_active":"2024-08-17T17:35:33.145040+0000","last_became_peered":"2024-08-17T17:35:33.145040+0000","last_unstale":"2024-08-17T17:38:08.882689+0000","last_undegraded":"2024-08-17T17:38:08.882689+0000","last_fullsized":"2024-08-17T17:38:08.882689+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:02.124000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1441,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.880549+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:38:07.880549+0000","last_peered":"2024-08-17T17:38:07.880549+0000","last_clean":"2024-08-17T17:38:07.880549+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:38:07.880549+0000","last_undegraded":"2024-08-17T17:38:07.880549+0000","last_fullsized":"2024-08-17T17:38:07.880549+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":913,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.874316+0000","last_change":"2024-08-17T17:36:15.769890+0000","last_active":"2024-08-17T17:38:07.874316+0000","last_peered":"2024-08-17T17:38:07.874316+0000","last_clean":"2024-08-17T17:38:07.874316+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:38:07.874316+0000","last_undegraded":"2024-08-17T17:38:07.874316+0000","last_fullsized":"2024-08-17T17:38:07.874316+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:33.034306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:33.034306+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":4,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.b","version":"1417'4943","reported_seq":12732,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:13.588390+0000","last_change":"2024-08-17T17:38:07.888612+0000","last_active":"2024-08-17T17:38:13.588390+0000","last_peered":"2024-08-17T17:38:13.588390+0000","last_clean":"2024-08-17T17:38:13.588390+0000","last_became_active":"2024-08-17T17:38:07.875196+0000","last_became_peered":"2024-08-17T17:38:07.875196+0000","last_unstale":"2024-08-17T17:38:13.588390+0000","last_undegraded":"2024-08-17T17:38:13.588390+0000","last_fullsized":"2024-08-17T17:38:13.588390+0000","mapping_epoch":1415,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4943,"log_dups_size":0,"ondisk_log_size":4943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:21.658004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197328896,"num_objects":3011,"num_object_clones":0,"num_object_copies":9033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4943,"num_write_kb":254528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3836,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1473,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.891944+0000","last_change":"2024-08-17T17:36:12.759743+0000","last_active":"2024-08-17T17:38:08.891944+0000","last_peered":"2024-08-17T17:38:08.891944+0000","last_clean":"2024-08-17T17:38:08.891944+0000","last_became_active":"2024-08-17T17:36:12.759334+0000","last_became_peered":"2024-08-17T17:36:12.759334+0000","last_unstale":"2024-08-17T17:38:08.891944+0000","last_undegraded":"2024-08-17T17:38:08.891944+0000","last_fullsized":"2024-08-17T17:38:08.891944+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:04.524384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1404,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.878526+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:38:07.878526+0000","last_peered":"2024-08-17T17:38:07.878526+0000","last_clean":"2024-08-17T17:38:07.878526+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:38:07.878526+0000","last_undegraded":"2024-08-17T17:38:07.878526+0000","last_fullsized":"2024-08-17T17:38:07.878526+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1446,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.879813+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:38:07.879813+0000","last_peered":"2024-08-17T17:38:07.879813+0000","last_clean":"2024-08-17T17:38:07.879813+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:38:07.879813+0000","last_undegraded":"2024-08-17T17:38:07.879813+0000","last_fullsized":"2024-08-17T17:38:07.879813+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1762,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.879452+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:38:07.879452+0000","last_peered":"2024-08-17T17:38:07.879452+0000","last_clean":"2024-08-17T17:38:07.879452+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:38:07.879452+0000","last_undegraded":"2024-08-17T17:38:07.879452+0000","last_fullsized":"2024-08-17T17:38:07.879452+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"1417'5141","reported_seq":8382,"reported_epoch":1417,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:38:15.807416+0000","last_change":"2024-08-17T17:38:05.991715+0000","last_active":"2024-08-17T17:38:15.807416+0000","last_peered":"2024-08-17T17:38:15.807416+0000","last_clean":"2024-08-17T17:35:54.187688+0000","last_became_active":"2024-08-17T17:36:22.904523+0000","last_became_peered":"2024-08-17T17:36:22.904523+0000","last_unstale":"2024-08-17T17:38:15.807416+0000","last_undegraded":"2024-08-17T17:38:15.807416+0000","last_fullsized":"2024-08-17T17:36:13.740491+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5141,"log_dups_size":0,"ondisk_log_size":5141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:57.525855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199688192,"num_objects":3047,"num_object_clones":0,"num_object_copies":9141,"num_objects_missing_on_primary":1836,"num_objects_missing":1836,"num_objects_degraded":0,"num_objects_misplaced":4752,"num_objects_unfound":0,"num_objects_dirty":3047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5141,"num_write_kb":262016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1052,"num_bytes_recovered":68943872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":1836},{"shards":"2,4,5,6","objects":3},{"shards":"2,5,6","objects":1077},{"shards":"5,6","objects":131}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1080,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.881048+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:38:07.881048+0000","last_peered":"2024-08-17T17:38:07.881048+0000","last_clean":"2024-08-17T17:38:07.881048+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:38:07.881048+0000","last_undegraded":"2024-08-17T17:38:07.881048+0000","last_fullsized":"2024-08-17T17:38:07.881048+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1385,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.890589+0000","last_change":"2024-08-17T17:35:56.229622+0000","last_active":"2024-08-17T17:38:08.890589+0000","last_peered":"2024-08-17T17:38:08.890589+0000","last_clean":"2024-08-17T17:38:08.890589+0000","last_became_active":"2024-08-17T17:35:56.229324+0000","last_became_peered":"2024-08-17T17:35:56.229324+0000","last_unstale":"2024-08-17T17:38:08.890589+0000","last_undegraded":"2024-08-17T17:38:08.890589+0000","last_fullsized":"2024-08-17T17:38:08.890589+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:13.980316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1988,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.891045+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:38:08.891045+0000","last_peered":"2024-08-17T17:38:08.891045+0000","last_clean":"2024-08-17T17:38:08.891045+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:38:08.891045+0000","last_undegraded":"2024-08-17T17:38:08.891045+0000","last_fullsized":"2024-08-17T17:38:08.891045+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1576,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.877781+0000","last_change":"2024-08-17T17:35:33.137074+0000","last_active":"2024-08-17T17:38:07.877781+0000","last_peered":"2024-08-17T17:38:07.877781+0000","last_clean":"2024-08-17T17:38:07.877781+0000","last_became_active":"2024-08-17T17:35:33.136759+0000","last_became_peered":"2024-08-17T17:35:33.136759+0000","last_unstale":"2024-08-17T17:38:07.877781+0000","last_undegraded":"2024-08-17T17:38:07.877781+0000","last_fullsized":"2024-08-17T17:38:07.877781+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:37.263639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:37.263639+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:14.599780+0000","scrub_duration":0.0016181800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"1417'5124","reported_seq":7744,"reported_epoch":1417,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:15.662278+0000","last_change":"2024-08-17T17:36:23.019129+0000","last_active":"2024-08-17T17:38:15.662278+0000","last_peered":"2024-08-17T17:38:15.662278+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:23.013527+0000","last_became_peered":"2024-08-17T17:36:23.013527+0000","last_unstale":"2024-08-17T17:38:15.662278+0000","last_undegraded":"2024-08-17T17:36:13.874715+0000","last_fullsized":"2024-08-17T17:36:13.738828+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5124,"log_dups_size":0,"ondisk_log_size":5124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195035136,"num_objects":2976,"num_object_clones":0,"num_object_copies":8928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5387,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5124,"num_write_kb":259200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1075,"num_bytes_recovered":70451200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[0,6],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":1801},{"shards":"0,1,6","objects":1048},{"shards":"0,6","objects":-610}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1177,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.889690+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:38:08.889690+0000","last_peered":"2024-08-17T17:38:08.889690+0000","last_clean":"2024-08-17T17:38:08.889690+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:38:08.889690+0000","last_undegraded":"2024-08-17T17:38:08.889690+0000","last_fullsized":"2024-08-17T17:38:08.889690+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":40,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.877354+0000","last_change":"2024-08-17T17:36:17.774773+0000","last_active":"2024-08-17T17:38:07.877354+0000","last_peered":"2024-08-17T17:38:07.877354+0000","last_clean":"2024-08-17T17:38:07.877354+0000","last_became_active":"2024-08-17T17:36:17.774468+0000","last_became_peered":"2024-08-17T17:36:17.774468+0000","last_unstale":"2024-08-17T17:38:07.877354+0000","last_undegraded":"2024-08-17T17:38:07.877354+0000","last_fullsized":"2024-08-17T17:38:07.877354+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:54.231747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1515,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.889742+0000","last_change":"2024-08-17T17:35:33.137633+0000","last_active":"2024-08-17T17:38:08.889742+0000","last_peered":"2024-08-17T17:38:08.889742+0000","last_clean":"2024-08-17T17:38:08.889742+0000","last_became_active":"2024-08-17T17:35:33.137323+0000","last_became_peered":"2024-08-17T17:35:33.137323+0000","last_unstale":"2024-08-17T17:38:08.889742+0000","last_undegraded":"2024-08-17T17:38:08.889742+0000","last_fullsized":"2024-08-17T17:38:08.889742+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:03.359155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":892,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.873633+0000","last_change":"2024-08-17T17:35:56.264626+0000","last_active":"2024-08-17T17:38:07.873633+0000","last_peered":"2024-08-17T17:38:07.873633+0000","last_clean":"2024-08-17T17:38:07.873633+0000","last_became_active":"2024-08-17T17:35:56.264265+0000","last_became_peered":"2024-08-17T17:35:56.264265+0000","last_unstale":"2024-08-17T17:38:07.873633+0000","last_undegraded":"2024-08-17T17:38:07.873633+0000","last_fullsized":"2024-08-17T17:38:07.873633+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:20.754326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:20.754326+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"1417'4961","reported_seq":7904,"reported_epoch":1417,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:15.832814+0000","last_change":"2024-08-17T17:37:11.134558+0000","last_active":"2024-08-17T17:38:15.832814+0000","last_peered":"2024-08-17T17:38:15.832814+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:20.105110+0000","last_became_peered":"2024-08-17T17:36:20.105110+0000","last_unstale":"2024-08-17T17:38:15.832814+0000","last_undegraded":"2024-08-17T17:36:19.999272+0000","last_fullsized":"2024-08-17T17:36:13.740520+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4961,"log_dups_size":0,"ondisk_log_size":4961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:45.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200081408,"num_objects":3053,"num_object_clones":0,"num_object_copies":9159,"num_objects_missing_on_primary":1857,"num_objects_missing":1857,"num_objects_degraded":4756,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4961,"num_write_kb":256448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":954,"num_bytes_recovered":62521344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1857},{"shards":"1,5,6","objects":1042},{"shards":"5,6","objects":154}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1245,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.874100+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:38:07.874100+0000","last_peered":"2024-08-17T17:38:07.874100+0000","last_clean":"2024-08-17T17:38:07.874100+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:38:07.874100+0000","last_undegraded":"2024-08-17T17:38:07.874100+0000","last_fullsized":"2024-08-17T17:38:07.874100+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1784,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.879732+0000","last_change":"2024-08-17T17:36:12.758005+0000","last_active":"2024-08-17T17:38:07.879732+0000","last_peered":"2024-08-17T17:38:07.879732+0000","last_clean":"2024-08-17T17:38:07.879732+0000","last_became_active":"2024-08-17T17:36:12.757695+0000","last_became_peered":"2024-08-17T17:36:12.757695+0000","last_unstale":"2024-08-17T17:38:07.879732+0000","last_undegraded":"2024-08-17T17:38:07.879732+0000","last_fullsized":"2024-08-17T17:38:07.879732+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:51.792220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1422,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.879829+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:38:07.879829+0000","last_peered":"2024-08-17T17:38:07.879829+0000","last_clean":"2024-08-17T17:38:07.879829+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:38:07.879829+0000","last_undegraded":"2024-08-17T17:38:07.879829+0000","last_fullsized":"2024-08-17T17:38:07.879829+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1709,"reported_epoch":1417,"state":"active+clean+scrubbing","last_fresh":"2024-08-17T17:38:09.044005+0000","last_change":"2024-08-17T17:38:09.044005+0000","last_active":"2024-08-17T17:38:09.044005+0000","last_peered":"2024-08-17T17:38:09.044005+0000","last_clean":"2024-08-17T17:38:09.044005+0000","last_became_active":"2024-08-17T17:35:55.204861+0000","last_became_peered":"2024-08-17T17:35:55.204861+0000","last_unstale":"2024-08-17T17:38:09.044005+0000","last_undegraded":"2024-08-17T17:38:09.044005+0000","last_fullsized":"2024-08-17T17:38:09.044005+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.0","version":"1417'5035","reported_seq":13012,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:12.602795+0000","last_change":"2024-08-17T17:37:40.753703+0000","last_active":"2024-08-17T17:38:12.602795+0000","last_peered":"2024-08-17T17:38:12.602795+0000","last_clean":"2024-08-17T17:38:12.602795+0000","last_became_active":"2024-08-17T17:37:40.743458+0000","last_became_peered":"2024-08-17T17:37:40.743458+0000","last_unstale":"2024-08-17T17:38:12.602795+0000","last_undegraded":"2024-08-17T17:38:12.602795+0000","last_fullsized":"2024-08-17T17:38:12.602795+0000","mapping_epoch":1411,"log_start":"1373'100","ondisk_log_start":"1373'100","created":1357,"last_epoch_clean":1412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4935,"log_dups_size":0,"ondisk_log_size":4935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:14:02.340380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201916416,"num_objects":3081,"num_object_clones":0,"num_object_copies":9243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5035,"num_write_kb":259712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3914,"num_bytes_recovered":232652800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1018,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.879554+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:38:07.879554+0000","last_peered":"2024-08-17T17:38:07.879554+0000","last_clean":"2024-08-17T17:38:07.879554+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:38:07.879554+0000","last_undegraded":"2024-08-17T17:38:07.879554+0000","last_fullsized":"2024-08-17T17:38:07.879554+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":80,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.906270+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:38:08.906270+0000","last_peered":"2024-08-17T17:38:08.906270+0000","last_clean":"2024-08-17T17:38:08.906270+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:38:08.906270+0000","last_undegraded":"2024-08-17T17:38:08.906270+0000","last_fullsized":"2024-08-17T17:38:08.906270+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1754,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.880800+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:38:07.880800+0000","last_peered":"2024-08-17T17:38:07.880800+0000","last_clean":"2024-08-17T17:38:07.880800+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:38:07.880800+0000","last_undegraded":"2024-08-17T17:38:07.880800+0000","last_fullsized":"2024-08-17T17:38:07.880800+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1","version":"1417'5005","reported_seq":8030,"reported_epoch":1417,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:15.834190+0000","last_change":"2024-08-17T17:36:23.103523+0000","last_active":"2024-08-17T17:38:15.834190+0000","last_peered":"2024-08-17T17:38:15.834190+0000","last_clean":"2024-08-17T17:35:54.178280+0000","last_became_active":"2024-08-17T17:36:23.025910+0000","last_became_peered":"2024-08-17T17:36:23.025910+0000","last_unstale":"2024-08-17T17:38:15.834190+0000","last_undegraded":"2024-08-17T17:36:18.734637+0000","last_fullsized":"2024-08-17T17:36:13.740134+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5005,"log_dups_size":0,"ondisk_log_size":5005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:58.717055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199688192,"num_objects":3047,"num_object_clones":0,"num_object_copies":9141,"num_objects_missing_on_primary":1852,"num_objects_missing":1852,"num_objects_degraded":4775,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5005,"num_write_kb":257664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":981,"num_bytes_recovered":64290816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,0],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":1852},{"shards":"0,2,4,6","objects":1},{"shards":"0,2,6","objects":1070},{"shards":"0,6","objects":124}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1432,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.879569+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:38:07.879569+0000","last_peered":"2024-08-17T17:38:07.879569+0000","last_clean":"2024-08-17T17:38:07.879569+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:38:07.879569+0000","last_undegraded":"2024-08-17T17:38:07.879569+0000","last_fullsized":"2024-08-17T17:38:07.879569+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1405,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.877712+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:38:07.877712+0000","last_peered":"2024-08-17T17:38:07.877712+0000","last_clean":"2024-08-17T17:38:07.877712+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:38:07.877712+0000","last_undegraded":"2024-08-17T17:38:07.877712+0000","last_fullsized":"2024-08-17T17:38:07.877712+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":982,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.882356+0000","last_change":"2024-08-17T17:34:26.817339+0000","last_active":"2024-08-17T17:38:08.882356+0000","last_peered":"2024-08-17T17:38:08.882356+0000","last_clean":"2024-08-17T17:38:08.882356+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:38:08.882356+0000","last_undegraded":"2024-08-17T17:38:08.882356+0000","last_fullsized":"2024-08-17T17:38:08.882356+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:26.817223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:26.817223+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:30.089956+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"1417'4943","reported_seq":5091,"reported_epoch":1417,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:15.561275+0000","last_change":"2024-08-17T17:36:14.073155+0000","last_active":"2024-08-17T17:38:15.561275+0000","last_peered":"2024-08-17T17:38:15.561275+0000","last_clean":"2024-08-17T17:36:11.734683+0000","last_became_active":"2024-08-17T17:36:14.063794+0000","last_became_peered":"2024-08-17T17:36:14.063794+0000","last_unstale":"2024-08-17T17:38:15.561275+0000","last_undegraded":"2024-08-17T17:36:13.855770+0000","last_fullsized":"2024-08-17T17:36:13.742068+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4943,"log_dups_size":0,"ondisk_log_size":4943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193921024,"num_objects":2959,"num_object_clones":0,"num_object_copies":8877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3819,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4943,"num_write_kb":252864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-860},{"shards":"0,2,7","objects":2827}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":40,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.879062+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:38:07.879062+0000","last_peered":"2024-08-17T17:38:07.879062+0000","last_clean":"2024-08-17T17:38:07.879062+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:38:07.879062+0000","last_undegraded":"2024-08-17T17:38:07.879062+0000","last_fullsized":"2024-08-17T17:38:07.879062+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1267,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.900309+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:38:07.900309+0000","last_peered":"2024-08-17T17:38:07.900309+0000","last_clean":"2024-08-17T17:38:07.900309+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:38:07.900309+0000","last_undegraded":"2024-08-17T17:38:07.900309+0000","last_fullsized":"2024-08-17T17:38:07.900309+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1782,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.884752+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:38:07.884752+0000","last_peered":"2024-08-17T17:38:07.884752+0000","last_clean":"2024-08-17T17:38:07.884752+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:38:07.884752+0000","last_undegraded":"2024-08-17T17:38:07.884752+0000","last_fullsized":"2024-08-17T17:38:07.884752+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"1417'5062","reported_seq":10853,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:15.508262+0000","last_change":"2024-08-17T17:36:46.982456+0000","last_active":"2024-08-17T17:38:15.508262+0000","last_peered":"2024-08-17T17:38:15.508262+0000","last_clean":"2024-08-17T17:38:15.508262+0000","last_became_active":"2024-08-17T17:36:46.974782+0000","last_became_peered":"2024-08-17T17:36:46.974782+0000","last_unstale":"2024-08-17T17:38:15.508262+0000","last_undegraded":"2024-08-17T17:38:15.508262+0000","last_fullsized":"2024-08-17T17:38:15.508262+0000","mapping_epoch":1399,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5062,"log_dups_size":0,"ondisk_log_size":5062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:40.499321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197263360,"num_objects":3010,"num_object_clones":0,"num_object_copies":9030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5062,"num_write_kb":258304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2848,"num_bytes_recovered":186646528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1383,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.878662+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:38:07.878662+0000","last_peered":"2024-08-17T17:38:07.878662+0000","last_clean":"2024-08-17T17:38:07.878662+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:38:07.878662+0000","last_undegraded":"2024-08-17T17:38:07.878662+0000","last_fullsized":"2024-08-17T17:38:07.878662+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":197,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.878103+0000","last_change":"2024-08-17T17:36:12.754372+0000","last_active":"2024-08-17T17:38:07.878103+0000","last_peered":"2024-08-17T17:38:07.878103+0000","last_clean":"2024-08-17T17:38:07.878103+0000","last_became_active":"2024-08-17T17:36:12.746352+0000","last_became_peered":"2024-08-17T17:36:12.746352+0000","last_unstale":"2024-08-17T17:38:07.878103+0000","last_undegraded":"2024-08-17T17:38:07.878103+0000","last_fullsized":"2024-08-17T17:38:07.878103+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:07.807486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1654,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.890869+0000","last_change":"2024-08-17T17:36:15.771193+0000","last_active":"2024-08-17T17:38:08.890869+0000","last_peered":"2024-08-17T17:38:08.890869+0000","last_clean":"2024-08-17T17:38:08.890869+0000","last_became_active":"2024-08-17T17:36:15.770889+0000","last_became_peered":"2024-08-17T17:36:15.770889+0000","last_unstale":"2024-08-17T17:38:08.890869+0000","last_undegraded":"2024-08-17T17:38:08.890869+0000","last_fullsized":"2024-08-17T17:38:08.890869+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:22.978695+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.4","version":"1417'5036","reported_seq":10808,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:15.830026+0000","last_change":"2024-08-17T17:37:04.045576+0000","last_active":"2024-08-17T17:38:15.830026+0000","last_peered":"2024-08-17T17:38:15.830026+0000","last_clean":"2024-08-17T17:38:15.830026+0000","last_became_active":"2024-08-17T17:37:04.038810+0000","last_became_peered":"2024-08-17T17:37:04.038810+0000","last_unstale":"2024-08-17T17:38:15.830026+0000","last_undegraded":"2024-08-17T17:38:15.830026+0000","last_fullsized":"2024-08-17T17:38:15.830026+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5036,"log_dups_size":0,"ondisk_log_size":5036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:27:04.725258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198574080,"num_objects":3030,"num_object_clones":0,"num_object_copies":9090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5036,"num_write_kb":258112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":176291840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1249,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.882735+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:38:08.882735+0000","last_peered":"2024-08-17T17:38:08.882735+0000","last_clean":"2024-08-17T17:38:08.882735+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:38:08.882735+0000","last_undegraded":"2024-08-17T17:38:08.882735+0000","last_fullsized":"2024-08-17T17:38:08.882735+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1409,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.880112+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:38:07.880112+0000","last_peered":"2024-08-17T17:38:07.880112+0000","last_clean":"2024-08-17T17:38:07.880112+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:38:07.880112+0000","last_undegraded":"2024-08-17T17:38:07.880112+0000","last_fullsized":"2024-08-17T17:38:07.880112+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":910,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.879088+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:38:07.879088+0000","last_peered":"2024-08-17T17:38:07.879088+0000","last_clean":"2024-08-17T17:38:07.879088+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:38:07.879088+0000","last_undegraded":"2024-08-17T17:38:07.879088+0000","last_fullsized":"2024-08-17T17:38:07.879088+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"1417'5067","reported_seq":10894,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:12.294462+0000","last_change":"2024-08-17T17:36:32.711490+0000","last_active":"2024-08-17T17:38:12.294462+0000","last_peered":"2024-08-17T17:38:12.294462+0000","last_clean":"2024-08-17T17:38:12.294462+0000","last_became_active":"2024-08-17T17:36:32.704370+0000","last_became_peered":"2024-08-17T17:36:32.704370+0000","last_unstale":"2024-08-17T17:38:12.294462+0000","last_undegraded":"2024-08-17T17:38:12.294462+0000","last_fullsized":"2024-08-17T17:38:12.294462+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5067,"log_dups_size":0,"ondisk_log_size":5067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:17.373879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199294976,"num_objects":3041,"num_object_clones":0,"num_object_copies":9123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5067,"num_write_kb":259456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2866,"num_bytes_recovered":187826176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1266,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.900867+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:38:07.900867+0000","last_peered":"2024-08-17T17:38:07.900867+0000","last_clean":"2024-08-17T17:38:07.900867+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:38:07.900867+0000","last_undegraded":"2024-08-17T17:38:07.900867+0000","last_fullsized":"2024-08-17T17:38:07.900867+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":146,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.880333+0000","last_change":"2024-08-17T17:35:55.217072+0000","last_active":"2024-08-17T17:38:07.880333+0000","last_peered":"2024-08-17T17:38:07.880333+0000","last_clean":"2024-08-17T17:38:07.880333+0000","last_became_active":"2024-08-17T17:35:55.216462+0000","last_became_peered":"2024-08-17T17:35:55.216462+0000","last_unstale":"2024-08-17T17:38:07.880333+0000","last_undegraded":"2024-08-17T17:38:07.880333+0000","last_fullsized":"2024-08-17T17:38:07.880333+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:34.937221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":893,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.882826+0000","last_change":"2024-08-17T17:35:55.215004+0000","last_active":"2024-08-17T17:38:08.882826+0000","last_peered":"2024-08-17T17:38:08.882826+0000","last_clean":"2024-08-17T17:38:08.882826+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:38:08.882826+0000","last_undegraded":"2024-08-17T17:38:08.882826+0000","last_fullsized":"2024-08-17T17:38:08.882826+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.6","version":"1417'4943","reported_seq":5067,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:15.582768+0000","last_change":"2024-08-17T17:35:14.868524+0000","last_active":"2024-08-17T17:38:15.582768+0000","last_peered":"2024-08-17T17:38:15.582768+0000","last_clean":"2024-08-17T17:38:15.582768+0000","last_became_active":"2024-08-17T17:35:14.854690+0000","last_became_peered":"2024-08-17T17:35:14.854690+0000","last_unstale":"2024-08-17T17:38:15.582768+0000","last_undegraded":"2024-08-17T17:38:15.582768+0000","last_fullsized":"2024-08-17T17:38:15.582768+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4943,"log_dups_size":0,"ondisk_log_size":4943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:05.388362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188284928,"num_objects":2873,"num_object_clones":0,"num_object_copies":8619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4943,"num_write_kb":250112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1451,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.877635+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:38:07.877635+0000","last_peered":"2024-08-17T17:38:07.877635+0000","last_clean":"2024-08-17T17:38:07.877635+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:38:07.877635+0000","last_undegraded":"2024-08-17T17:38:07.877635+0000","last_fullsized":"2024-08-17T17:38:07.877635+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":158,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.879655+0000","last_change":"2024-08-17T17:35:33.147672+0000","last_active":"2024-08-17T17:38:07.879655+0000","last_peered":"2024-08-17T17:38:07.879655+0000","last_clean":"2024-08-17T17:38:07.879655+0000","last_became_active":"2024-08-17T17:35:33.147283+0000","last_became_peered":"2024-08-17T17:35:33.147283+0000","last_unstale":"2024-08-17T17:38:07.879655+0000","last_undegraded":"2024-08-17T17:38:07.879655+0000","last_fullsized":"2024-08-17T17:38:07.879655+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:41.677709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1642,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.904411+0000","last_change":"2024-08-17T17:38:08.904411+0000","last_active":"2024-08-17T17:38:08.904411+0000","last_peered":"2024-08-17T17:38:08.904411+0000","last_clean":"2024-08-17T17:38:08.904411+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:38:08.904411+0000","last_undegraded":"2024-08-17T17:38:08.904411+0000","last_fullsized":"2024-08-17T17:38:08.904411+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:38.241235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:38.241235+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:38:13.659503+0000","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"1417'4941","reported_seq":5010,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:12.387648+0000","last_change":"2024-08-17T17:34:50.670013+0000","last_active":"2024-08-17T17:38:12.387648+0000","last_peered":"2024-08-17T17:38:12.387648+0000","last_clean":"2024-08-17T17:38:12.387648+0000","last_became_active":"2024-08-17T17:34:50.669299+0000","last_became_peered":"2024-08-17T17:34:50.669299+0000","last_unstale":"2024-08-17T17:38:12.387648+0000","last_undegraded":"2024-08-17T17:38:12.387648+0000","last_fullsized":"2024-08-17T17:38:12.387648+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":4941,"log_dups_size":0,"ondisk_log_size":4941,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:11.172227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195624960,"num_objects":2985,"num_object_clones":0,"num_object_copies":8955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4941,"num_write_kb":253632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1375,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.885043+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:38:07.885043+0000","last_peered":"2024-08-17T17:38:07.885043+0000","last_clean":"2024-08-17T17:38:07.885043+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:38:07.885043+0000","last_undegraded":"2024-08-17T17:38:07.885043+0000","last_fullsized":"2024-08-17T17:38:07.885043+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":145,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.879694+0000","last_change":"2024-08-17T17:35:33.140725+0000","last_active":"2024-08-17T17:38:07.879694+0000","last_peered":"2024-08-17T17:38:07.879694+0000","last_clean":"2024-08-17T17:38:07.879694+0000","last_became_active":"2024-08-17T17:35:33.140406+0000","last_became_peered":"2024-08-17T17:35:33.140406+0000","last_unstale":"2024-08-17T17:38:07.879694+0000","last_undegraded":"2024-08-17T17:38:07.879694+0000","last_fullsized":"2024-08-17T17:38:07.879694+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:54:51.875571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1665,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.878796+0000","last_change":"2024-08-17T17:35:33.138514+0000","last_active":"2024-08-17T17:38:07.878796+0000","last_peered":"2024-08-17T17:38:07.878796+0000","last_clean":"2024-08-17T17:38:07.878796+0000","last_became_active":"2024-08-17T17:35:33.137035+0000","last_became_peered":"2024-08-17T17:35:33.137035+0000","last_unstale":"2024-08-17T17:38:07.878796+0000","last_undegraded":"2024-08-17T17:38:07.878796+0000","last_fullsized":"2024-08-17T17:38:07.878796+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:50.565662+0000","scrub_duration":0.001487138,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1307,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.890603+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:38:08.890603+0000","last_peered":"2024-08-17T17:38:08.890603+0000","last_clean":"2024-08-17T17:38:08.890603+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:38:08.890603+0000","last_undegraded":"2024-08-17T17:38:08.890603+0000","last_fullsized":"2024-08-17T17:38:08.890603+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":80,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.905654+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:38:08.905654+0000","last_peered":"2024-08-17T17:38:08.905654+0000","last_clean":"2024-08-17T17:38:08.905654+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:38:08.905654+0000","last_undegraded":"2024-08-17T17:38:08.905654+0000","last_fullsized":"2024-08-17T17:38:08.905654+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1737,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.879509+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:38:07.879509+0000","last_peered":"2024-08-17T17:38:07.879509+0000","last_clean":"2024-08-17T17:38:07.879509+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:38:07.879509+0000","last_undegraded":"2024-08-17T17:38:07.879509+0000","last_fullsized":"2024-08-17T17:38:07.879509+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1324,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.877559+0000","last_change":"2024-08-17T17:35:55.205292+0000","last_active":"2024-08-17T17:38:07.877559+0000","last_peered":"2024-08-17T17:38:07.877559+0000","last_clean":"2024-08-17T17:38:07.877559+0000","last_became_active":"2024-08-17T17:35:55.204935+0000","last_became_peered":"2024-08-17T17:35:55.204935+0000","last_unstale":"2024-08-17T17:38:07.877559+0000","last_undegraded":"2024-08-17T17:38:07.877559+0000","last_fullsized":"2024-08-17T17:38:07.877559+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:52.277821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":80,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.905394+0000","last_change":"2024-08-17T17:36:12.749972+0000","last_active":"2024-08-17T17:38:08.905394+0000","last_peered":"2024-08-17T17:38:08.905394+0000","last_clean":"2024-08-17T17:38:08.905394+0000","last_became_active":"2024-08-17T17:36:12.749620+0000","last_became_peered":"2024-08-17T17:36:12.749620+0000","last_unstale":"2024-08-17T17:38:08.905394+0000","last_undegraded":"2024-08-17T17:38:08.905394+0000","last_fullsized":"2024-08-17T17:38:08.905394+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:04.994357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":956,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.904304+0000","last_change":"2024-08-17T17:36:20.832274+0000","last_active":"2024-08-17T17:38:08.904304+0000","last_peered":"2024-08-17T17:38:08.904304+0000","last_clean":"2024-08-17T17:38:08.904304+0000","last_became_active":"2024-08-17T17:36:20.831886+0000","last_became_peered":"2024-08-17T17:36:20.831886+0000","last_unstale":"2024-08-17T17:38:08.904304+0000","last_undegraded":"2024-08-17T17:38:08.904304+0000","last_fullsized":"2024-08-17T17:38:08.904304+0000","mapping_epoch":1390,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1391,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:05.248711+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1166,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.888024+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:38:08.888024+0000","last_peered":"2024-08-17T17:38:08.888024+0000","last_clean":"2024-08-17T17:38:08.888024+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:38:08.888024+0000","last_undegraded":"2024-08-17T17:38:08.888024+0000","last_fullsized":"2024-08-17T17:38:08.888024+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1407,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.880400+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:38:07.880400+0000","last_peered":"2024-08-17T17:38:07.880400+0000","last_clean":"2024-08-17T17:38:07.880400+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:38:07.880400+0000","last_undegraded":"2024-08-17T17:38:07.880400+0000","last_fullsized":"2024-08-17T17:38:07.880400+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1700,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.884667+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:38:07.884667+0000","last_peered":"2024-08-17T17:38:07.884667+0000","last_clean":"2024-08-17T17:38:07.884667+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:38:07.884667+0000","last_undegraded":"2024-08-17T17:38:07.884667+0000","last_fullsized":"2024-08-17T17:38:07.884667+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1349,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-08-17T17:38:07.913815+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:38:07.913815+0000","last_peered":"2024-08-17T17:38:07.913815+0000","last_clean":"2024-08-17T17:38:07.913815+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:38:07.913815+0000","last_undegraded":"2024-08-17T17:38:07.913815+0000","last_fullsized":"2024-08-17T17:38:07.913815+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1885,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.904604+0000","last_change":"2024-08-17T17:35:33.137411+0000","last_active":"2024-08-17T17:38:08.904604+0000","last_peered":"2024-08-17T17:38:08.904604+0000","last_clean":"2024-08-17T17:38:08.904604+0000","last_became_active":"2024-08-17T17:35:33.137073+0000","last_became_peered":"2024-08-17T17:35:33.137073+0000","last_unstale":"2024-08-17T17:38:08.904604+0000","last_undegraded":"2024-08-17T17:38:08.904604+0000","last_fullsized":"2024-08-17T17:38:08.904604+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:35.924539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1795,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-08-17T17:38:08.914920+0000","last_change":"2024-08-17T17:36:15.787445+0000","last_active":"2024-08-17T17:38:08.914920+0000","last_peered":"2024-08-17T17:38:08.914920+0000","last_clean":"2024-08-17T17:38:08.914920+0000","last_became_active":"2024-08-17T17:36:15.787099+0000","last_became_peered":"2024-08-17T17:36:15.787099+0000","last_unstale":"2024-08-17T17:38:08.914920+0000","last_undegraded":"2024-08-17T17:38:08.914920+0000","last_fullsized":"2024-08-17T17:38:08.914920+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:47.448168+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":3162112024,"num_objects":48251,"num_object_clones":0,"num_object_copies":144753,"num_objects_missing_on_primary":5545,"num_objects_missing":5545,"num_objects_degraded":18737,"num_objects_misplaced":7843,"num_objects_unfound":0,"num_objects_dirty":48251,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":80631,"num_write_kb":4124161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24145,"num_bytes_recovered":1492713472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9854398464,"data_stored":9854386248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":80531,"ondisk_log_size":80531,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635092,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":27,"num_read_kb":61,"num_write":15,"num_write_kb":183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1314910,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162849,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1577060,"kb_used_data":1434588,"kb_used_omap":54,"kb_used_meta":142409,"kb_avail":92168092,"statfs":{"total":95995035648,"available":94380126208,"internally_reserved":0,"allocated":1469018112,"data_stored":1466363762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":145826837},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999324,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1306980,"kb_used_data":1153052,"kb_used_omap":47,"kb_used_meta":153872,"kb_avail":92438172,"statfs":{"total":95995035648,"available":94656688128,"internally_reserved":0,"allocated":1180725248,"data_stored":1178071875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":157565588},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,3,1,1,3,6,3],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627120,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1447752,"kb_used_data":441024,"kb_used_omap":42,"kb_used_meta":1006677,"kb_avail":92297400,"statfs":{"total":95995035648,"available":94512537600,"internally_reserved":0,"allocated":451608576,"data_stored":448977484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1030837308},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785597,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2876300,"kb_used_data":2063812,"kb_used_omap":48,"kb_used_meta":812431,"kb_avail":90868852,"statfs":{"total":95995035648,"available":93049704448,"internally_reserved":0,"allocated":2113343488,"data_stored":2110709271,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":831930030},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725114,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1648468,"kb_used_data":791308,"kb_used_omap":1,"kb_used_meta":857150,"kb_avail":92096684,"statfs":{"total":95995035648,"available":94307004416,"internally_reserved":0,"allocated":810299392,"data_stored":807657125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":877722054},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986751,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1886688,"kb_used_data":995096,"kb_used_omap":1,"kb_used_meta":891582,"kb_avail":91858464,"statfs":{"total":95995035648,"available":94063067136,"internally_reserved":0,"allocated":1018978304,"data_stored":1016327460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":912980422},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706359,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2572052,"kb_used_data":1661260,"kb_used_omap":35,"kb_used_meta":910748,"kb_avail":91173100,"statfs":{"total":95995035648,"available":93361254400,"internally_reserved":0,"allocated":1701130240,"data_stored":1698487954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":932606007},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832132,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2188432,"kb_used_data":1157832,"kb_used_omap":65,"kb_used_meta":1030526,"kb_avail":91556720,"statfs":{"total":95995035648,"available":93754081280,"internally_reserved":0,"allocated":1185619968,"data_stored":1182982240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1055258811},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1175126016,"data_stored":1175126016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2101149696,"data_stored":2101149696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1690046464,"data_stored":1690042392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":441974784,"data_stored":441974784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1013256192,"data_stored":1013252120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":799997952,"data_stored":799997952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1171656704,"data_stored":1171652632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1461190656,"data_stored":1461190656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:38:19.372 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:38:19.414 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:19.441 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:19.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:19.542 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:19.567 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:19.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:19.669 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:19.698 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:19.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:19.800 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:19.824 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:19.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:19.926 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:19.954 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:19.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:19.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:20.056 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:20.080 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:20.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:20.182 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:20.212 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:20.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:20.214 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:20 smithi063 ceph-mon[109843]: from='client.36342 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:20.214 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:20 smithi063 ceph-mon[109843]: pgmap v2209: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 170 op/s; 18737/144321 objects degraded (12.983%); 7843/144321 objects misplaced (5.434%); 8.5 MiB/s, 147 objects/s recovering 2024-08-17T17:38:20.214 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:20 smithi063 ceph-mon[114756]: from='client.36342 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:20.214 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:20 smithi063 ceph-mon[114756]: pgmap v2209: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 170 op/s; 18737/144321 objects degraded (12.983%); 7843/144321 objects misplaced (5.434%); 8.5 MiB/s, 147 objects/s recovering 2024-08-17T17:38:20.314 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:20.339 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:20.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:20.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:20 smithi162 ceph-mon[104668]: from='client.36342 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:20.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:20 smithi162 ceph-mon[104668]: pgmap v2209: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 170 op/s; 18737/144321 objects degraded (12.983%); 7843/144321 objects misplaced (5.434%); 8.5 MiB/s, 147 objects/s recovering 2024-08-17T17:38:20.441 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:20.464 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:20.534 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:38:20.534 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:38:20.553 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:38:20.553 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:38:20.565 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:20.572 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:38:20.573 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:38:20.590 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:20.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:20.592 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:38:20.592 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:38:20.612 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:38:20.612 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:38:20.633 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:38:20.633 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:38:20.651 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:38:20.651 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:38:20.669 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:38:20.670 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:38:20.692 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:20.716 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:20.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:20.818 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:20.845 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:20.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:20.947 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:20.973 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:20.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:20.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:21.074 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:21.099 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:21.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:21.201 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:21.225 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:21.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:21.227 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:21 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4143724528' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:21.227 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:21 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4143724528' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:21.326 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:21.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:21 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4143724528' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:21.351 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:21.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:21.452 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:21.475 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:21.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:21.577 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:21.599 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:21.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:21.701 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:21.725 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:21.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:21.827 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:21.853 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:21.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.854 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:21.955 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:21.979 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:21.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:21.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:22.081 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:22.103 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:22.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:22.205 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:22.230 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:22.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:22.332 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:22.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:22 smithi162 ceph-mon[104668]: pgmap v2210: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 18737/144195 objects degraded (12.994%); 7843/144195 objects misplaced (5.439%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:38:22.356 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:22.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:22.458 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:22.483 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:22.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:22.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:22 smithi063 ceph-mon[114756]: pgmap v2210: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 18737/144195 objects degraded (12.994%); 7843/144195 objects misplaced (5.439%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:38:22.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:22 smithi063 ceph-mon[109843]: pgmap v2210: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 145 op/s; 18737/144195 objects degraded (12.994%); 7843/144195 objects misplaced (5.439%); 7.2 MiB/s, 115 objects/s recovering 2024-08-17T17:38:22.587 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:22.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.613 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:22.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:22.715 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:22.742 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:22.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:22.844 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:22.872 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:22.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:22.974 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:22.997 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:22.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:22.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:23.099 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:23.125 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:23.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:23.227 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:23.250 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:23.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:23.351 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:23.374 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:23.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:23.476 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:23.500 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:23.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:23.592 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:38:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:38:23] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.51.0" 2024-08-17T17:38:23.603 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:23.628 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:23.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:23.730 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:23.755 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:23.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:23.856 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:23.881 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:23.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:23.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:23.983 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:24.008 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:24.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:24.110 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:24.137 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:24.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:24.239 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:24.268 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:24.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.269 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:24.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:24 smithi063 ceph-mon[114756]: pgmap v2211: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 204 op/s; 18402/141990 objects degraded (12.960%); 7202/141990 objects misplaced (5.072%); 11 MiB/s, 175 objects/s recovering 2024-08-17T17:38:24.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:24 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 18402/141990 objects degraded (12.960%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:24.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:24 smithi063 ceph-mon[109843]: pgmap v2211: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 204 op/s; 18402/141990 objects degraded (12.960%); 7202/141990 objects misplaced (5.072%); 11 MiB/s, 175 objects/s recovering 2024-08-17T17:38:24.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:24 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 18402/141990 objects degraded (12.960%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:24.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:24 smithi162 ceph-mon[104668]: pgmap v2211: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 204 op/s; 18402/141990 objects degraded (12.960%); 7202/141990 objects misplaced (5.072%); 11 MiB/s, 175 objects/s recovering 2024-08-17T17:38:24.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:24 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 18402/141990 objects degraded (12.960%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:24.370 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:24.397 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:24.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:24.499 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:24.526 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:24.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:24.628 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:24.658 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:24.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:24.759 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:24.786 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:24.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:24.888 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:24.910 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:24.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:24.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:25.012 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:25.036 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:25.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:25.138 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:25.164 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:25.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:25.266 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:25.307 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:25.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:25.408 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:25.434 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:25.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.435 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:25.536 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:25.565 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:25.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:25.631 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:25.667 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:25.692 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:25.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:25.794 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:25.821 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:25.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:25.923 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:25.949 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:25.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:25.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:26.051 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:26.052 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:38:26.079 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:26.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:26.181 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:26.207 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:26.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:26.309 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:26.333 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:26.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:26.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:26 smithi162 ceph-mon[104668]: pgmap v2212: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 18402/141639 objects degraded (12.992%); 7202/141639 objects misplaced (5.085%); 7.6 MiB/s, 122 objects/s recovering 2024-08-17T17:38:26.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:26 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1521243834' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:26.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:26 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:26.435 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:26.460 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:26.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:26.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:26 smithi063 ceph-mon[114756]: pgmap v2212: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 18402/141639 objects degraded (12.992%); 7202/141639 objects misplaced (5.085%); 7.6 MiB/s, 122 objects/s recovering 2024-08-17T17:38:26.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:26 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1521243834' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:26.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:26 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:26.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:26 smithi063 ceph-mon[109843]: pgmap v2212: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 154 op/s; 18402/141639 objects degraded (12.992%); 7202/141639 objects misplaced (5.085%); 7.6 MiB/s, 122 objects/s recovering 2024-08-17T17:38:26.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:26 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1521243834' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:26.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:26 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:26.562 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:26.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.587 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:26.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:26.688 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:26.716 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:26.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:26.818 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:26.843 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:26.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:26.944 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:26.974 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:26.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:26.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:27.011 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:38:27.076 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:27.104 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:27.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:27.206 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:27.233 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:27.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:27.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:27 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:38:27.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:27 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:27.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:27 smithi063 ceph-mon[114756]: osdmap e1418: 8 total, 8 up, 8 in 2024-08-17T17:38:27.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:27 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:38:27.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:27 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:27.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:27 smithi063 ceph-mon[109843]: osdmap e1418: 8 total, 8 up, 8 in 2024-08-17T17:38:27.335 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:27.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:27 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:38:27.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:27 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:27.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:27 smithi162 ceph-mon[104668]: osdmap e1418: 8 total, 8 up, 8 in 2024-08-17T17:38:27.359 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:27.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:27.461 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:27.485 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:27.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:27.587 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:27.611 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:27.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:27.713 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:27.736 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:27.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:27.837 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:27.860 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:27.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:27.962 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:27.977 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:38:27.977 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:38:27.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:27.986 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:27.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:28.087 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:28.110 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:28.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:28.212 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:28.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.241 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:28.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:28.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:28 smithi063 ceph-mon[114756]: pgmap v2214: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 168 op/s; 18067/139782 objects degraded (12.925%); 6472/139782 objects misplaced (4.630%); 10 MiB/s, 159 objects/s recovering 2024-08-17T17:38:28.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:28 smithi063 ceph-mon[109843]: pgmap v2214: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 168 op/s; 18067/139782 objects degraded (12.925%); 6472/139782 objects misplaced (4.630%); 10 MiB/s, 159 objects/s recovering 2024-08-17T17:38:28.342 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:28.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:28 smithi162 ceph-mon[104668]: pgmap v2214: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 168 op/s; 18067/139782 objects degraded (12.925%); 6472/139782 objects misplaced (4.630%); 10 MiB/s, 159 objects/s recovering 2024-08-17T17:38:28.364 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:28.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:28.466 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:28.492 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:28.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:28.594 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:28.622 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:28.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.623 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:28.724 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:28.750 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:28.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:28.852 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:28.881 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:28.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:28.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:28.910 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:38:28.983 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:29.007 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:29.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:29.110 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:29.135 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:29.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:29.236 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:29.263 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:29.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:29.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:29 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 18067/139782 objects degraded (12.925%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:29.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:29 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 18067/139782 objects degraded (12.925%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:29.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:29 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 18067/139782 objects degraded (12.925%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:29.365 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:29.393 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:29.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:29.495 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:29.519 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:29.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:29.620 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:29.648 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:29.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:29.750 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:29.775 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:29.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:29.876 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:29.901 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:29.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:29.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:30.003 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:30.028 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:30.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:30.130 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:30.155 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:30.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:30.257 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:30.283 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:30.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:30.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:30 smithi162 ceph-mon[104668]: pgmap v2215: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 18067/139356 objects degraded (12.965%); 6472/139356 objects misplaced (4.644%); 10 MiB/s, 159 objects/s recovering 2024-08-17T17:38:30.385 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:30.410 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:30.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:30.512 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:30.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:30 smithi063 ceph-mon[114756]: pgmap v2215: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 18067/139356 objects degraded (12.965%); 6472/139356 objects misplaced (4.644%); 10 MiB/s, 159 objects/s recovering 2024-08-17T17:38:30.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:30 smithi063 ceph-mon[109843]: pgmap v2215: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 165 op/s; 18067/139356 objects degraded (12.965%); 6472/139356 objects misplaced (4.644%); 10 MiB/s, 159 objects/s recovering 2024-08-17T17:38:30.538 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:30.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:30.639 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:30.665 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:30.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:30.766 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:30.789 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:30.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:30.890 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:30.915 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:30.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:30.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:31.017 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:31.043 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:31.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:31.145 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:31.172 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:31.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:31.274 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:31.298 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:31.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:31.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:31 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/557083821' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:38:31.351 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:38:31.351 INFO:teuthology.orchestra.run.smithi063.stdout:{"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 18067/139356 objects degraded (12.965%), 4 pgs degraded, 5 pgs undersized","count":9},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2190,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1418,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916154,"num_in_osds":8,"osd_in_since":1723916113,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":111},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+clean+scrubbing","count":2},{"state_name":"active+recovering+remapped","count":1},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":46452,"data_bytes":3043149489,"bytes_used":15635562496,"bytes_avail":752324722688,"bytes_total":767960285184,"degraded_objects":18067,"degraded_total":139356,"degraded_ratio":0.12964637331725939,"misplaced_objects":6472,"misplaced_total":139356,"misplaced_ratio":0.046442205574212816,"recovering_objects_per_sec":159,"recovering_bytes_per_sec":10450104,"recovering_keys_per_sec":0,"num_objects_recovered":1595,"num_bytes_recovered":104529920,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":165},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi063.ultpqj","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.63:8443/","prometheus":"http://172.21.15.63:9283/"}},"servicemap":{"epoch":32,"modified":"2024-08-17T17:38:12.639909+0000","services":{"osd":{"daemons":{"summary":"","3":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}} 2024-08-17T17:38:31.352 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:31 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/557083821' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:38:31.352 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:31 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/557083821' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:38:31.400 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:31.425 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:31.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:31.527 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:31.561 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:31.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:31.662 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:31.687 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:31.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:31.789 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:31.814 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:31.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:31.915 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:31.941 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:31.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:31.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:32.045 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:32.072 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:32.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:32.173 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:32.204 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:32.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:32.306 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:32.332 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:32.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:32.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:32 smithi162 ceph-mon[104668]: pgmap v2216: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 18067/139191 objects degraded (12.980%); 6472/139191 objects misplaced (4.650%); 10 MiB/s, 159 objects/s recovering 2024-08-17T17:38:32.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:32 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:32.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:32 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:32.434 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:32.459 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:32.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:32.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:32 smithi063 ceph-mon[114756]: pgmap v2216: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 18067/139191 objects degraded (12.980%); 6472/139191 objects misplaced (4.650%); 10 MiB/s, 159 objects/s recovering 2024-08-17T17:38:32.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:32 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:32.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:32 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:32.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:32 smithi063 ceph-mon[109843]: pgmap v2216: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 166 op/s; 18067/139191 objects degraded (12.980%); 6472/139191 objects misplaced (4.650%); 10 MiB/s, 159 objects/s recovering 2024-08-17T17:38:32.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:32 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:32.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:32 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:32.561 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:32.593 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:32.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:32.694 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:32.723 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:32.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:32.825 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:32.854 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:32.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:32.956 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:32.984 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:32.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:32.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:33.085 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:33.113 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:33.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:33.215 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:33.238 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:33.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:33.340 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:33.367 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:33.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:33.377 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:38:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:38:33] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.51.0" 2024-08-17T17:38:33.469 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:33.498 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:33.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:33.600 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:33.625 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:33.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:33.727 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:33.754 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:33.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:33.856 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:33.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:33.886 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:33.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:33.987 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:34.011 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:34.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:34.113 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:34.138 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:34.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:34.240 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:34.269 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:34.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:34.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:34 smithi162 ceph-mon[104668]: pgmap v2217: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 173 op/s; 17710/136791 objects degraded (12.947%); 5597/136791 objects misplaced (4.092%); 11 MiB/s, 183 objects/s recovering 2024-08-17T17:38:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:34 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 17710/136791 objects degraded (12.947%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:34.371 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:34.380 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:34 smithi063 ceph-mon[114756]: pgmap v2217: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 173 op/s; 17710/136791 objects degraded (12.947%); 5597/136791 objects misplaced (4.092%); 11 MiB/s, 183 objects/s recovering 2024-08-17T17:38:34.380 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:34 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 17710/136791 objects degraded (12.947%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:34.380 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:34 smithi063 ceph-mon[109843]: pgmap v2217: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 173 op/s; 17710/136791 objects degraded (12.947%); 5597/136791 objects misplaced (4.092%); 11 MiB/s, 183 objects/s recovering 2024-08-17T17:38:34.380 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:34 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 17710/136791 objects degraded (12.947%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:34.399 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:34.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:34.502 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:34.528 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:34.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:34.630 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:34.654 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:34.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:34.756 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:34.780 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:34.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:34.882 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:34.904 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:34.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:34.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:35.006 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:35.034 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:35.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:35.036 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:38:35.037 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:38:35.053 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:35.055 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:35.136 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:35.160 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:35.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:35.262 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:35.287 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:35.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:35.388 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:35.417 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:35.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:35.519 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:35.545 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:35.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:35.646 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:35.677 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:35.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:35.779 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:35.803 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:35.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:35.905 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:35.932 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:35.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:35.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:36.034 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:36.060 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:36.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:36.122 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:38:36.162 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:36.190 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:36.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:36.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:36 smithi063 ceph-mon[114756]: pgmap v2218: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 172 op/s; 17710/136470 objects degraded (12.977%); 5597/136470 objects misplaced (4.101%); 11 MiB/s, 183 objects/s recovering 2024-08-17T17:38:36.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:36 smithi063 ceph-mon[109843]: pgmap v2218: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 172 op/s; 17710/136470 objects degraded (12.977%); 5597/136470 objects misplaced (4.101%); 11 MiB/s, 183 objects/s recovering 2024-08-17T17:38:36.292 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:36.324 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:36.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:36.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:36 smithi162 ceph-mon[104668]: pgmap v2218: 119 pgs: 1 active+recovering+remapped, 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 172 op/s; 17710/136470 objects degraded (12.977%); 5597/136470 objects misplaced (4.101%); 11 MiB/s, 183 objects/s recovering 2024-08-17T17:38:36.426 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:36.450 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:36.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:36.552 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:36.577 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:36.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:36.679 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:36.701 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:36.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:36.803 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:36.828 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:36.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:36.930 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:36.954 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:36.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:36.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:37.056 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:37.078 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:37.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:37.176 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:37 smithi063 ceph-mon[114756]: osdmap e1419: 8 total, 8 up, 8 in 2024-08-17T17:38:37.176 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:37 smithi063 ceph-mon[109843]: osdmap e1419: 8 total, 8 up, 8 in 2024-08-17T17:38:37.180 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:37.207 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:37.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:37.309 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:37.334 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:37.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:37.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:37 smithi162 ceph-mon[104668]: osdmap e1419: 8 total, 8 up, 8 in 2024-08-17T17:38:37.437 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:37.460 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:37.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:37.562 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:37.587 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:37.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.588 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:37.688 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:37.711 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:37.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:37.813 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:37.840 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:37.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:37.942 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:37.965 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:37.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:37.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:38.067 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:38.094 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:38.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:38.195 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:38.220 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:38.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:38.322 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:38.344 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:38.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:38.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:38 smithi162 ceph-mon[104668]: pgmap v2220: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 17371/134460 objects degraded (12.919%); 4334/134460 objects misplaced (3.223%); 9.0 MiB/s, 236 objects/s recovering 2024-08-17T17:38:38.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:38 smithi162 ceph-mon[104668]: osdmap e1420: 8 total, 8 up, 8 in 2024-08-17T17:38:38.446 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:38.469 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:38.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:38.486 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:38 smithi063 ceph-mon[114756]: pgmap v2220: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 17371/134460 objects degraded (12.919%); 4334/134460 objects misplaced (3.223%); 9.0 MiB/s, 236 objects/s recovering 2024-08-17T17:38:38.486 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:38 smithi063 ceph-mon[114756]: osdmap e1420: 8 total, 8 up, 8 in 2024-08-17T17:38:38.486 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:38 smithi063 ceph-mon[109843]: pgmap v2220: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 177 op/s; 17371/134460 objects degraded (12.919%); 4334/134460 objects misplaced (3.223%); 9.0 MiB/s, 236 objects/s recovering 2024-08-17T17:38:38.486 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:38 smithi063 ceph-mon[109843]: osdmap e1420: 8 total, 8 up, 8 in 2024-08-17T17:38:38.571 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:38.593 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:38.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:38.695 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:38.838 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:38.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.839 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:38.940 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:38.962 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:38.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:38.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:39.064 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:39.091 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:39.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:39.193 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:39.218 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:39.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:39.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:39 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 17371/134460 objects degraded (12.919%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:39.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:39 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 17371/134460 objects degraded (12.919%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:39.320 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:39.342 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:39.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:39.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:39 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 17371/134460 objects degraded (12.919%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:39.445 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:39.467 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:39.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:39.569 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:39.596 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:39.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:39.698 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:39.723 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:39.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:39.825 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:39.849 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:39.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:39.951 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:39.952 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:39.954 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:38:39.954 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:38:39.979 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:38:39.980 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:38:39.987 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:39.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:39.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:39.996 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:40.004 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:38:40.005 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:38:40.025 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:38:40.025 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:38:40.044 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:38:40.044 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:38:40.063 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:38:40.063 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:38:40.079 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:38:40.079 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:38:40.089 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:40.099 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:38:40.099 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:38:40.114 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:40.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:40.216 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:40.239 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:40 smithi063 ceph-mon[114756]: pgmap v2222: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 219 op/s; 17371/133911 objects degraded (12.972%); 4293/133911 objects misplaced (3.206%); 11 MiB/s, 301 objects/s recovering 2024-08-17T17:38:40.239 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:40 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:40.239 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:40 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:40.240 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:40 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:38:40.240 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:40 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:38:40.240 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:40 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:40.240 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:40 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4190504631' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:40.240 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:40.240 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:40 smithi063 ceph-mon[109843]: pgmap v2222: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 219 op/s; 17371/133911 objects degraded (12.972%); 4293/133911 objects misplaced (3.206%); 11 MiB/s, 301 objects/s recovering 2024-08-17T17:38:40.240 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:40 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:40.240 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:40 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:40.241 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:40 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:38:40.241 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:40 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:38:40.241 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:40 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:40.241 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:40 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4190504631' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:40.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:40.343 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:40 smithi162 ceph-mon[104668]: pgmap v2222: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 219 op/s; 17371/133911 objects degraded (12.972%); 4293/133911 objects misplaced (3.206%); 11 MiB/s, 301 objects/s recovering 2024-08-17T17:38:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:40 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:40 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:40 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:38:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:40 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:38:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:40 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:40 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4190504631' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:40.369 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:40.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:40.471 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:40.498 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:40.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:40.600 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:40.624 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:40.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:40.726 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:40.750 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:40.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:40.852 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:40.876 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:40.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:40.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:40.977 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:41.003 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:41.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:41.104 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:41.127 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:41.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:41.229 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:41.251 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:41.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:41.353 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:41.375 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:41.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:41.477 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:41.502 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:41.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:41.604 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:41.631 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:41.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:41.733 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:41.756 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:41.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:41.857 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:41.880 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:41.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:41.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:41.982 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:42.004 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:42.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:42.042 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:38:42.042 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:38:42.042 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:38:42.042 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:38:42.043 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.043 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:68 11.4 11:21df73c4:::benchmark_data_smithi063_328699_object67:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1372)", 2024-08-17T17:38:42.043 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:33.153455+0000", 2024-08-17T17:38:42.043 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 188.886476902, 2024-08-17T17:38:42.043 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98894179000000004, 2024-08-17T17:38:42.043 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.043 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.043 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:42.043 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.043 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.043 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.043 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 68 2024-08-17T17:38:42.044 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.044 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.044 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.044 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.044 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.153455+0000", 2024-08-17T17:38:42.044 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.044 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.044 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.044 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.044 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.153455+0000", 2024-08-17T17:38:42.044 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.045 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.045 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.045 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.045 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.153457+0000", 2024-08-17T17:38:42.045 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.632e-06 2024-08-17T17:38:42.045 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.045 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.045 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.045 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.153480+0000", 2024-08-17T17:38:42.045 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3078000000000002e-05 2024-08-17T17:38:42.045 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.045 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.046 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.046 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.153492+0000", 2024-08-17T17:38:42.046 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2024999999999999e-05 2024-08-17T17:38:42.046 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.046 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.046 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.046 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.153500+0000", 2024-08-17T17:38:42.046 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.2130000000000008e-06 2024-08-17T17:38:42.046 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.046 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.046 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.046 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.153571+0000", 2024-08-17T17:38:42.047 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0623000000000005e-05 2024-08-17T17:38:42.047 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.047 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.047 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.047 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.153585+0000", 2024-08-17T17:38:42.047 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4304999999999999e-05 2024-08-17T17:38:42.047 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.047 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.047 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.047 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.137384+0000", 2024-08-17T17:38:42.047 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98379832 2024-08-17T17:38:42.047 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.047 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.048 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.048 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.137597+0000", 2024-08-17T17:38:42.048 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000212859 2024-08-17T17:38:42.048 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.048 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.048 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:38:42.048 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.137751+0000", 2024-08-17T17:38:42.048 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015423800000000001 2024-08-17T17:38:42.048 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.048 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.048 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.048 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.139858+0000", 2024-08-17T17:38:42.049 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0021074620000000001 2024-08-17T17:38:42.049 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.049 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.049 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.049 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.140046+0000", 2024-08-17T17:38:42.049 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000187908 2024-08-17T17:38:42.049 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.049 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.049 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.049 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.142235+0000", 2024-08-17T17:38:42.049 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0021887429999999999 2024-08-17T17:38:42.049 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.049 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.050 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:38:42.050 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.142284+0000", 2024-08-17T17:38:42.050 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9162000000000001e-05 2024-08-17T17:38:42.050 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.050 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.050 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.050 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.142397+0000", 2024-08-17T17:38:42.050 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011322199999999999 2024-08-17T17:38:42.050 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.050 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.050 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.050 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.050 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.051 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:73 11.4 11:2d347ded:::benchmark_data_smithi063_328699_object72:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1372)", 2024-08-17T17:38:42.051 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:33.155188+0000", 2024-08-17T17:38:42.051 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 188.884744778, 2024-08-17T17:38:42.051 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98754444399999997, 2024-08-17T17:38:42.051 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.051 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.051 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:42.051 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.051 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.051 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.051 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 73 2024-08-17T17:38:42.051 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.051 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.052 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.052 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.052 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155188+0000", 2024-08-17T17:38:42.052 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.052 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.052 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.052 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.052 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155188+0000", 2024-08-17T17:38:42.052 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.052 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.052 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.052 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.052 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155191+0000", 2024-08-17T17:38:42.052 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.861e-06 2024-08-17T17:38:42.053 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.053 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.053 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.053 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155260+0000", 2024-08-17T17:38:42.053 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8292000000000003e-05 2024-08-17T17:38:42.053 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.053 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.053 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.053 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155275+0000", 2024-08-17T17:38:42.053 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5166e-05 2024-08-17T17:38:42.053 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.053 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.053 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.054 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155283+0000", 2024-08-17T17:38:42.054 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9510000000000006e-06 2024-08-17T17:38:42.054 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.054 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.054 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.054 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155539+0000", 2024-08-17T17:38:42.054 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00025604000000000002 2024-08-17T17:38:42.054 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.054 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.054 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.054 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155550+0000", 2024-08-17T17:38:42.054 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1486e-05 2024-08-17T17:38:42.054 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.055 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.055 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.055 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.138085+0000", 2024-08-17T17:38:42.055 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98253495000000002 2024-08-17T17:38:42.055 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.055 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.055 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.055 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.138210+0000", 2024-08-17T17:38:42.055 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012428800000000001 2024-08-17T17:38:42.055 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.055 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.055 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:38:42.055 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.138369+0000", 2024-08-17T17:38:42.055 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015961799999999999 2024-08-17T17:38:42.056 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.056 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.056 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.056 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.140093+0000", 2024-08-17T17:38:42.056 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0017240179999999999 2024-08-17T17:38:42.056 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.056 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.056 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.056 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.142028+0000", 2024-08-17T17:38:42.056 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0019348449999999999 2024-08-17T17:38:42.056 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.056 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.056 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.056 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.142442+0000", 2024-08-17T17:38:42.057 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00041389900000000002 2024-08-17T17:38:42.057 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.057 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.057 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:38:42.057 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.142492+0000", 2024-08-17T17:38:42.057 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0009000000000002e-05 2024-08-17T17:38:42.057 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.057 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.057 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.057 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.142732+0000", 2024-08-17T17:38:42.057 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00024002100000000001 2024-08-17T17:38:42.057 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.057 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.058 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.058 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.058 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.058 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:74 11.2 11:48d9252a:::benchmark_data_smithi063_328699_object73:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1372)", 2024-08-17T17:38:42.058 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:33.155306+0000", 2024-08-17T17:38:42.058 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 188.88462657100001, 2024-08-17T17:38:42.058 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98270905799999997, 2024-08-17T17:38:42.058 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.058 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.058 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:42.058 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.058 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.058 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.059 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 74 2024-08-17T17:38:42.059 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.059 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.059 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.059 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.059 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155306+0000", 2024-08-17T17:38:42.059 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.059 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.059 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.059 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.059 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155306+0000", 2024-08-17T17:38:42.059 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.060 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.060 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.060 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.060 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155306+0000", 2024-08-17T17:38:42.060 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4499999999999997e-07 2024-08-17T17:38:42.060 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.060 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.060 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.060 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155359+0000", 2024-08-17T17:38:42.060 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.24e-05 2024-08-17T17:38:42.060 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.060 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.060 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.061 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155369+0000", 2024-08-17T17:38:42.061 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0084e-05 2024-08-17T17:38:42.061 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.061 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.061 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.061 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155375+0000", 2024-08-17T17:38:42.061 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2639999999999997e-06 2024-08-17T17:38:42.061 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.061 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.061 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.061 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155440+0000", 2024-08-17T17:38:42.061 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4531999999999998e-05 2024-08-17T17:38:42.062 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.062 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.062 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.062 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155450+0000", 2024-08-17T17:38:42.062 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0509000000000001e-05 2024-08-17T17:38:42.062 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.062 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.062 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.062 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.130782+0000", 2024-08-17T17:38:42.062 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.975332223 2024-08-17T17:38:42.062 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.062 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.063 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.066 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.130957+0000", 2024-08-17T17:38:42.066 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00017452799999999999 2024-08-17T17:38:42.066 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.066 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.066 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:38:42.066 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.132023+0000", 2024-08-17T17:38:42.066 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001065823 2024-08-17T17:38:42.066 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.066 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.066 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.067 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.136862+0000", 2024-08-17T17:38:42.067 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0048389540000000003 2024-08-17T17:38:42.067 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.067 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.067 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.067 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.137619+0000", 2024-08-17T17:38:42.067 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00075684400000000001 2024-08-17T17:38:42.067 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.067 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.067 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.067 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.137836+0000", 2024-08-17T17:38:42.067 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000217073 2024-08-17T17:38:42.067 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.068 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.068 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:38:42.068 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.137890+0000", 2024-08-17T17:38:42.068 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4651000000000002e-05 2024-08-17T17:38:42.068 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.068 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.068 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.068 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.138015+0000", 2024-08-17T17:38:42.068 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000124528 2024-08-17T17:38:42.068 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.068 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.068 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.069 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.069 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.069 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:75 11.f 11:f35f8a88:::benchmark_data_smithi063_328699_object74:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1372)", 2024-08-17T17:38:42.069 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:33.155401+0000", 2024-08-17T17:38:42.069 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 188.88453114699999, 2024-08-17T17:38:42.069 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98579956000000002, 2024-08-17T17:38:42.069 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.069 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.069 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:42.069 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.069 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.069 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.070 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 75 2024-08-17T17:38:42.070 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.070 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.070 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.070 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.070 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155401+0000", 2024-08-17T17:38:42.070 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.070 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.070 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.070 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.070 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155401+0000", 2024-08-17T17:38:42.070 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.070 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.071 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.071 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.071 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155402+0000", 2024-08-17T17:38:42.071 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4799999999999998e-07 2024-08-17T17:38:42.071 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.071 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.071 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.071 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155451+0000", 2024-08-17T17:38:42.071 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8943000000000002e-05 2024-08-17T17:38:42.071 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.071 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.071 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.071 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155460+0000", 2024-08-17T17:38:42.071 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3959999999999995e-06 2024-08-17T17:38:42.072 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.072 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.072 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.072 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155472+0000", 2024-08-17T17:38:42.072 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2122e-05 2024-08-17T17:38:42.072 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.072 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.072 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.072 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155598+0000", 2024-08-17T17:38:42.072 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000126074 2024-08-17T17:38:42.072 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.072 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.072 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.073 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.155607+0000", 2024-08-17T17:38:42.073 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6200000000000005e-06 2024-08-17T17:38:42.073 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.073 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.073 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.073 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.133891+0000", 2024-08-17T17:38:42.073 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.97828377799999999 2024-08-17T17:38:42.073 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.073 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.073 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.073 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.134250+0000", 2024-08-17T17:38:42.073 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000358952 2024-08-17T17:38:42.073 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.073 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.074 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:38:42.074 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.134397+0000", 2024-08-17T17:38:42.074 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014684 2024-08-17T17:38:42.074 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.074 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.074 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.074 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.136095+0000", 2024-08-17T17:38:42.074 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0016985889999999999 2024-08-17T17:38:42.074 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.074 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.074 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.074 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.137776+0000", 2024-08-17T17:38:42.075 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0016807440000000001 2024-08-17T17:38:42.075 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.075 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.075 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.075 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.141046+0000", 2024-08-17T17:38:42.076 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0032702859999999999 2024-08-17T17:38:42.076 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.076 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.076 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:38:42.076 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.141096+0000", 2024-08-17T17:38:42.077 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0040999999999997e-05 2024-08-17T17:38:42.077 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.077 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.077 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.077 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.141201+0000", 2024-08-17T17:38:42.077 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000104627 2024-08-17T17:38:42.077 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.077 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.077 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.077 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.077 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.077 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:78 11.2 11:4bb3f07d:::benchmark_data_smithi063_328699_object77:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1372)", 2024-08-17T17:38:42.077 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:33.156622+0000", 2024-08-17T17:38:42.078 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 188.88330998000001, 2024-08-17T17:38:42.078 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.981767261, 2024-08-17T17:38:42.078 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.078 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.078 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:42.078 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.078 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.078 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.078 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 78 2024-08-17T17:38:42.078 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.078 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.079 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.079 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.079 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.156622+0000", 2024-08-17T17:38:42.079 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.079 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.079 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.079 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.079 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.156622+0000", 2024-08-17T17:38:42.079 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.079 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.079 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.079 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.079 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.156624+0000", 2024-08-17T17:38:42.080 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.178e-06 2024-08-17T17:38:42.080 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.080 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.080 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.080 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.156681+0000", 2024-08-17T17:38:42.080 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7442999999999999e-05 2024-08-17T17:38:42.080 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.080 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.080 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.080 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.156692+0000", 2024-08-17T17:38:42.080 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0631999999999999e-05 2024-08-17T17:38:42.080 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.080 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.081 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.081 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.156699+0000", 2024-08-17T17:38:42.081 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9890000000000003e-06 2024-08-17T17:38:42.081 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.081 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.081 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.081 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.156772+0000", 2024-08-17T17:38:42.081 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3121999999999995e-05 2024-08-17T17:38:42.081 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.081 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.081 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.081 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.156782+0000", 2024-08-17T17:38:42.081 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0526999999999999e-05 2024-08-17T17:38:42.082 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.082 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.082 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.082 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.132931+0000", 2024-08-17T17:38:42.082 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.97614832500000004 2024-08-17T17:38:42.082 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.082 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.082 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.082 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.133088+0000", 2024-08-17T17:38:42.082 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000157725 2024-08-17T17:38:42.082 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.082 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.082 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:38:42.083 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.133241+0000", 2024-08-17T17:38:42.083 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015315000000000001 2024-08-17T17:38:42.083 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.083 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.083 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.083 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.136943+0000", 2024-08-17T17:38:42.083 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0037015279999999999 2024-08-17T17:38:42.083 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.083 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.083 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.083 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.137657+0000", 2024-08-17T17:38:42.083 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00071401600000000004 2024-08-17T17:38:42.083 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.083 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.084 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.084 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.138025+0000", 2024-08-17T17:38:42.084 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00036757599999999999 2024-08-17T17:38:42.084 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.084 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.084 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:38:42.084 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.138071+0000", 2024-08-17T17:38:42.084 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5939999999999997e-05 2024-08-17T17:38:42.084 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.084 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.084 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.084 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.138390+0000", 2024-08-17T17:38:42.084 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00031911 2024-08-17T17:38:42.084 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.085 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.085 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.085 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.085 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.085 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:86 11.4 11:2fccc6bd:::benchmark_data_smithi063_328699_object85:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1372)", 2024-08-17T17:38:42.085 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:33.158676+0000", 2024-08-17T17:38:42.085 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 188.88125676600001, 2024-08-17T17:38:42.085 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98433958700000002, 2024-08-17T17:38:42.085 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.085 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.085 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:42.085 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.085 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.086 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.086 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 86 2024-08-17T17:38:42.086 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.086 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.086 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.086 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.086 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.158676+0000", 2024-08-17T17:38:42.086 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.086 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.086 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.086 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.086 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.158676+0000", 2024-08-17T17:38:42.087 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.087 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.087 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.087 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.087 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.158677+0000", 2024-08-17T17:38:42.087 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3379999999999999e-06 2024-08-17T17:38:42.087 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.087 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.087 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.087 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.158728+0000", 2024-08-17T17:38:42.087 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0921000000000003e-05 2024-08-17T17:38:42.087 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.088 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.088 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.088 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.158743+0000", 2024-08-17T17:38:42.088 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5235000000000001e-05 2024-08-17T17:38:42.088 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.088 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.088 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.088 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.158751+0000", 2024-08-17T17:38:42.088 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8879999999999997e-06 2024-08-17T17:38:42.088 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.088 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.089 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.089 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.158862+0000", 2024-08-17T17:38:42.089 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000111261 2024-08-17T17:38:42.089 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.089 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.089 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.089 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.158875+0000", 2024-08-17T17:38:42.089 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3020000000000001e-05 2024-08-17T17:38:42.089 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.089 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.089 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.089 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.139182+0000", 2024-08-17T17:38:42.089 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98030683799999996 2024-08-17T17:38:42.090 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.090 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.090 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.090 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.139341+0000", 2024-08-17T17:38:42.090 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000158548 2024-08-17T17:38:42.090 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.090 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.090 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:38:42.090 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.139494+0000", 2024-08-17T17:38:42.090 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015322799999999999 2024-08-17T17:38:42.090 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.090 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.090 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.091 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.142010+0000", 2024-08-17T17:38:42.091 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0025161620000000002 2024-08-17T17:38:42.091 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.091 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.091 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.091 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.142034+0000", 2024-08-17T17:38:42.091 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4039000000000001e-05 2024-08-17T17:38:42.091 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.091 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.091 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.091 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.142856+0000", 2024-08-17T17:38:42.091 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00082149700000000003 2024-08-17T17:38:42.092 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.092 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.092 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:38:42.092 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.142906+0000", 2024-08-17T17:38:42.092 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0784e-05 2024-08-17T17:38:42.092 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.092 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.092 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.092 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.143015+0000", 2024-08-17T17:38:42.092 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000108828 2024-08-17T17:38:42.092 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.092 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.093 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.093 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.093 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.093 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:87 11.2 11:4bff0942:::benchmark_data_smithi063_328699_object86:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1372)", 2024-08-17T17:38:42.093 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:33.159038+0000", 2024-08-17T17:38:42.093 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 188.88089420399999, 2024-08-17T17:38:42.093 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.97975204699999996, 2024-08-17T17:38:42.093 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.093 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.093 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:42.093 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.093 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.093 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.093 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 87 2024-08-17T17:38:42.094 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.094 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.094 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.094 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.094 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.159038+0000", 2024-08-17T17:38:42.094 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.094 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.094 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.094 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.094 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.159038+0000", 2024-08-17T17:38:42.094 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.094 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.094 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.095 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.095 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.159039+0000", 2024-08-17T17:38:42.095 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.2999999999999999e-07 2024-08-17T17:38:42.095 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.095 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.095 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.095 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.159097+0000", 2024-08-17T17:38:42.095 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7859000000000003e-05 2024-08-17T17:38:42.095 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.095 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.095 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.095 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.159111+0000", 2024-08-17T17:38:42.095 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4075e-05 2024-08-17T17:38:42.095 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.096 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.096 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.096 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.159118+0000", 2024-08-17T17:38:42.096 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.905e-06 2024-08-17T17:38:42.096 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.096 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.096 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.096 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.159185+0000", 2024-08-17T17:38:42.096 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6645999999999999e-05 2024-08-17T17:38:42.096 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.096 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.096 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.096 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.159208+0000", 2024-08-17T17:38:42.096 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3637000000000001e-05 2024-08-17T17:38:42.097 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.097 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.097 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.097 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.133532+0000", 2024-08-17T17:38:42.097 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.97432406900000001 2024-08-17T17:38:42.097 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.097 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.097 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.097 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.133671+0000", 2024-08-17T17:38:42.097 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000138455 2024-08-17T17:38:42.097 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.097 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.097 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:38:42.097 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.133816+0000", 2024-08-17T17:38:42.098 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014552400000000001 2024-08-17T17:38:42.098 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.098 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.098 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.098 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.137574+0000", 2024-08-17T17:38:42.098 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.003757308 2024-08-17T17:38:42.098 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.098 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.098 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.098 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.137695+0000", 2024-08-17T17:38:42.098 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000121295 2024-08-17T17:38:42.098 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.098 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.099 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.099 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.138491+0000", 2024-08-17T17:38:42.099 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00079655399999999999 2024-08-17T17:38:42.099 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.099 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.099 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:38:42.099 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.138669+0000", 2024-08-17T17:38:42.099 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00017748300000000001 2024-08-17T17:38:42.099 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.099 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.099 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.099 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.138790+0000", 2024-08-17T17:38:42.099 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000121307 2024-08-17T17:38:42.099 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.100 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.100 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.100 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.100 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.100 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:91 11.4 11:2b92b16f:::benchmark_data_smithi063_328699_object90:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1372)", 2024-08-17T17:38:42.100 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:33.160159+0000", 2024-08-17T17:38:42.100 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 188.87977374799999, 2024-08-17T17:38:42.100 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98452288700000001, 2024-08-17T17:38:42.100 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.100 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.100 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:42.100 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.100 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.101 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.101 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 91 2024-08-17T17:38:42.101 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.101 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.101 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.101 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.101 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.160159+0000", 2024-08-17T17:38:42.101 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.101 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.101 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.101 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.101 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.160159+0000", 2024-08-17T17:38:42.101 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.101 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.101 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.102 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.102 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.160160+0000", 2024-08-17T17:38:42.102 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.449e-06 2024-08-17T17:38:42.102 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.102 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.102 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.102 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.160210+0000", 2024-08-17T17:38:42.102 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9993000000000001e-05 2024-08-17T17:38:42.102 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.102 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.102 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.102 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.160226+0000", 2024-08-17T17:38:42.103 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6399999999999999e-05 2024-08-17T17:38:42.103 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.103 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.103 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.103 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.160235+0000", 2024-08-17T17:38:42.103 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.3119999999999994e-06 2024-08-17T17:38:42.103 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.103 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.103 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.103 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.160316+0000", 2024-08-17T17:38:42.103 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.1159e-05 2024-08-17T17:38:42.103 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.103 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.103 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.104 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.160329+0000", 2024-08-17T17:38:42.104 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2874000000000001e-05 2024-08-17T17:38:42.104 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.104 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.104 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.104 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.140130+0000", 2024-08-17T17:38:42.104 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.97980108799999999 2024-08-17T17:38:42.104 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.104 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.104 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.104 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.140266+0000", 2024-08-17T17:38:42.104 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013570699999999999 2024-08-17T17:38:42.104 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.104 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.105 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:38:42.105 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.140498+0000", 2024-08-17T17:38:42.105 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00023224500000000001 2024-08-17T17:38:42.105 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.105 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.105 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.105 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.142042+0000", 2024-08-17T17:38:42.105 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0015439329999999999 2024-08-17T17:38:42.105 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.105 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.105 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.105 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.142151+0000", 2024-08-17T17:38:42.106 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010938999999999999 2024-08-17T17:38:42.106 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.106 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.106 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.106 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.144501+0000", 2024-08-17T17:38:42.106 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0023501149999999998 2024-08-17T17:38:42.106 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.106 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.106 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:38:42.106 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.144564+0000", 2024-08-17T17:38:42.106 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2811999999999994e-05 2024-08-17T17:38:42.106 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.107 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.107 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.107 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.144682+0000", 2024-08-17T17:38:42.107 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011741 2024-08-17T17:38:42.107 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.107 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.107 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.107 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.107 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.107 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:96 11.4 11:245f544e:::benchmark_data_smithi063_328699_object95:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1372)", 2024-08-17T17:38:42.107 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:33.162311+0000", 2024-08-17T17:38:42.107 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 188.877621331, 2024-08-17T17:38:42.108 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98263752500000001, 2024-08-17T17:38:42.108 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.108 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.108 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:42.108 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.108 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.108 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.108 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 96 2024-08-17T17:38:42.108 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.108 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.108 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.108 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.108 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.162311+0000", 2024-08-17T17:38:42.108 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.108 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.109 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.109 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.109 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.162311+0000", 2024-08-17T17:38:42.109 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.109 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.109 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.109 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.109 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.162312+0000", 2024-08-17T17:38:42.109 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2200000000000004e-07 2024-08-17T17:38:42.109 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.109 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.109 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.109 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.162336+0000", 2024-08-17T17:38:42.109 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4329999999999999e-05 2024-08-17T17:38:42.110 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.110 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.110 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.110 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.162345+0000", 2024-08-17T17:38:42.110 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7399999999999993e-06 2024-08-17T17:38:42.110 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.110 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.110 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.110 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.162351+0000", 2024-08-17T17:38:42.110 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5760000000000002e-06 2024-08-17T17:38:42.110 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.110 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.110 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.111 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.162420+0000", 2024-08-17T17:38:42.111 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8898000000000003e-05 2024-08-17T17:38:42.111 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.111 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.111 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.111 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.162433+0000", 2024-08-17T17:38:42.111 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2371000000000001e-05 2024-08-17T17:38:42.111 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.111 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.111 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.111 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.140782+0000", 2024-08-17T17:38:42.111 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.97834915300000003 2024-08-17T17:38:42.111 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.111 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.112 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.112 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.140931+0000", 2024-08-17T17:38:42.112 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014906199999999999 2024-08-17T17:38:42.112 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.112 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.112 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:38:42.112 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.141094+0000", 2024-08-17T17:38:42.112 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016369299999999999 2024-08-17T17:38:42.112 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.112 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.112 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.112 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.142195+0000", 2024-08-17T17:38:42.112 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001100036 2024-08-17T17:38:42.113 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.113 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.113 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.113 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.144761+0000", 2024-08-17T17:38:42.113 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.002565987 2024-08-17T17:38:42.113 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.113 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.113 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.113 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.144778+0000", 2024-08-17T17:38:42.113 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7547999999999999e-05 2024-08-17T17:38:42.113 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.113 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.114 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:38:42.114 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.144830+0000", 2024-08-17T17:38:42.114 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1490000000000003e-05 2024-08-17T17:38:42.114 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.114 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.114 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.114 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.144949+0000", 2024-08-17T17:38:42.114 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011901899999999999 2024-08-17T17:38:42.114 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.114 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.114 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.115 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.115 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.115 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:102 11.4 11:2200b129:::benchmark_data_smithi063_328699_object101:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1372)", 2024-08-17T17:38:42.115 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:33.164585+0000", 2024-08-17T17:38:42.115 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 188.87534775, 2024-08-17T17:38:42.115 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98070656599999995, 2024-08-17T17:38:42.115 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.115 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.115 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:42.115 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.115 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.115 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.116 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 102 2024-08-17T17:38:42.116 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.116 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.116 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.116 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.116 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.164585+0000", 2024-08-17T17:38:42.116 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.116 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.116 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.116 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.116 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.164585+0000", 2024-08-17T17:38:42.116 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.116 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.117 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.117 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.117 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.164586+0000", 2024-08-17T17:38:42.117 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.277e-06 2024-08-17T17:38:42.117 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.117 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.117 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.117 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.164611+0000", 2024-08-17T17:38:42.117 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4970999999999999e-05 2024-08-17T17:38:42.117 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.117 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.117 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.117 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.164625+0000", 2024-08-17T17:38:42.118 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3935e-05 2024-08-17T17:38:42.118 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.118 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.118 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.118 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.164632+0000", 2024-08-17T17:38:42.118 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4580000000000004e-06 2024-08-17T17:38:42.118 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.118 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.118 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.118 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.164707+0000", 2024-08-17T17:38:42.118 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4331000000000004e-05 2024-08-17T17:38:42.118 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.118 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.119 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.119 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:33.164719+0000", 2024-08-17T17:38:42.119 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.208e-05 2024-08-17T17:38:42.119 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.119 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.119 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.119 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.143051+0000", 2024-08-17T17:38:42.119 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.97833277799999996 2024-08-17T17:38:42.119 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.119 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.119 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.119 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.143193+0000", 2024-08-17T17:38:42.119 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014188199999999999 2024-08-17T17:38:42.119 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.120 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.120 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:38:42.120 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.143474+0000", 2024-08-17T17:38:42.120 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000280659 2024-08-17T17:38:42.120 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.120 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.120 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.120 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.145026+0000", 2024-08-17T17:38:42.120 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0015521840000000001 2024-08-17T17:38:42.120 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.120 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.120 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.120 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.145072+0000", 2024-08-17T17:38:42.121 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6071e-05 2024-08-17T17:38:42.121 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.121 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.121 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.121 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.145126+0000", 2024-08-17T17:38:42.121 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4058999999999999e-05 2024-08-17T17:38:42.121 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.121 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.121 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:38:42.121 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.145178+0000", 2024-08-17T17:38:42.121 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1847e-05 2024-08-17T17:38:42.121 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.121 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.121 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.122 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:34.145291+0000", 2024-08-17T17:38:42.122 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000113034 2024-08-17T17:38:42.122 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.122 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.122 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.122 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.122 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.122 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:45453 11.1 11:8338c251:::benchmark_data_smithi063_328699_object45452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1375)", 2024-08-17T17:38:42.122 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:54.180810+0000", 2024-08-17T17:38:42.122 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 167.859122283, 2024-08-17T17:38:42.122 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2620342779999998, 2024-08-17T17:38:42.122 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.123 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.123 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:38:42.123 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.123 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.123 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.123 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 45453 2024-08-17T17:38:42.123 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.123 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.123 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.123 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.123 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.180810+0000", 2024-08-17T17:38:42.123 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.123 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.124 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.124 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.124 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.180810+0000", 2024-08-17T17:38:42.124 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.124 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.124 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.124 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.124 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.180811+0000", 2024-08-17T17:38:42.124 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1799999999999999e-06 2024-08-17T17:38:42.124 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.124 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.124 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.124 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.180839+0000", 2024-08-17T17:38:42.125 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.7881e-05 2024-08-17T17:38:42.125 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.125 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.125 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.125 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.180850+0000", 2024-08-17T17:38:42.125 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0749e-05 2024-08-17T17:38:42.125 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.125 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.125 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.125 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.180856+0000", 2024-08-17T17:38:42.125 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4679999999999997e-06 2024-08-17T17:38:42.125 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.126 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.126 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.126 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.180973+0000", 2024-08-17T17:38:42.126 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000116153 2024-08-17T17:38:42.126 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.126 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.126 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.126 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.181121+0000", 2024-08-17T17:38:42.126 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014842699999999999 2024-08-17T17:38:42.126 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.126 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.126 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:38:42.126 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.181263+0000", 2024-08-17T17:38:42.127 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014185900000000001 2024-08-17T17:38:42.127 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.127 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.127 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.127 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.183137+0000", 2024-08-17T17:38:42.127 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0018738520000000001 2024-08-17T17:38:42.127 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.127 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.127 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.127 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.187407+0000", 2024-08-17T17:38:42.127 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0042699499999999998 2024-08-17T17:38:42.127 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.127 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.128 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.128 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.198822+0000", 2024-08-17T17:38:42.128 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011415853 2024-08-17T17:38:42.128 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.128 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.128 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.128 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.198838+0000", 2024-08-17T17:38:42.128 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5486999999999999e-05 2024-08-17T17:38:42.128 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.128 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.128 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.128 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.199858+0000", 2024-08-17T17:38:42.128 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.00102051 2024-08-17T17:38:42.128 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.199870+0000", 2024-08-17T17:38:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1787000000000001e-05 2024-08-17T17:38:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.426543+0000", 2024-08-17T17:38:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2266722990000001 2024-08-17T17:38:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for degraded object", 2024-08-17T17:38:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.427417+0000", 2024-08-17T17:38:42.129 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00087453300000000004 2024-08-17T17:38:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.442781+0000", 2024-08-17T17:38:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.015364347 2024-08-17T17:38:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.442844+0000", 2024-08-17T17:38:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2942999999999997e-05 2024-08-17T17:38:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.130 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.131 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.131 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.131 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:45454 11.4 11:223d8872:::benchmark_data_smithi063_328699_object45453:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1375)", 2024-08-17T17:38:42.131 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:54.180885+0000", 2024-08-17T17:38:42.131 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 167.859047774, 2024-08-17T17:38:42.131 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2631166779999998, 2024-08-17T17:38:42.131 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.131 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.131 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:38:42.131 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.131 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.131 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.131 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 45454 2024-08-17T17:38:42.131 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.132 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.132 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.132 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.132 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.180885+0000", 2024-08-17T17:38:42.132 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.132 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.132 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.132 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.132 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.180885+0000", 2024-08-17T17:38:42.132 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.132 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.132 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.132 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.133 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.180885+0000", 2024-08-17T17:38:42.133 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4099999999999998e-07 2024-08-17T17:38:42.133 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.133 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.133 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.133 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.180908+0000", 2024-08-17T17:38:42.133 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3108e-05 2024-08-17T17:38:42.133 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.133 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.133 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.133 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.180923+0000", 2024-08-17T17:38:42.133 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5119e-05 2024-08-17T17:38:42.133 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.134 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.134 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.134 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.180930+0000", 2024-08-17T17:38:42.134 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2419999999999997e-06 2024-08-17T17:38:42.134 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.134 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.134 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.134 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.182059+0000", 2024-08-17T17:38:42.134 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001129768 2024-08-17T17:38:42.134 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.134 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.134 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.135 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.182199+0000", 2024-08-17T17:38:42.135 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013931499999999999 2024-08-17T17:38:42.135 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.135 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.135 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:38:42.135 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.182344+0000", 2024-08-17T17:38:42.135 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014483700000000001 2024-08-17T17:38:42.135 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.135 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.135 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.135 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.185020+0000", 2024-08-17T17:38:42.135 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00267636 2024-08-17T17:38:42.135 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.135 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.136 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.136 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.209655+0000", 2024-08-17T17:38:42.136 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.024634686999999999 2024-08-17T17:38:42.136 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.136 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.136 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.136 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.209667+0000", 2024-08-17T17:38:42.136 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2413e-05 2024-08-17T17:38:42.136 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.136 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.136 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.136 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.201715+0000", 2024-08-17T17:38:42.137 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.99204791199999998 2024-08-17T17:38:42.137 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.137 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.137 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.137 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.201729+0000", 2024-08-17T17:38:42.137 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3633e-05 2024-08-17T17:38:42.137 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.137 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.137 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.137 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.443909+0000", 2024-08-17T17:38:42.137 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2421807359999999 2024-08-17T17:38:42.137 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.138 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.138 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.138 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.444001+0000", 2024-08-17T17:38:42.138 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.1906999999999999e-05 2024-08-17T17:38:42.138 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.138 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.138 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.138 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.138 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.138 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:45455 11.d 11:bfce6b93:::benchmark_data_smithi063_328699_object45454:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1375)", 2024-08-17T17:38:42.138 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:54.182168+0000", 2024-08-17T17:38:42.138 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 167.85776451300001, 2024-08-17T17:38:42.139 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2696144199999999, 2024-08-17T17:38:42.139 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.139 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.139 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:38:42.139 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.139 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.139 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.139 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 45455 2024-08-17T17:38:42.139 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.139 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.139 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.139 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.139 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.182168+0000", 2024-08-17T17:38:42.140 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.140 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.140 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.140 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.140 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.182168+0000", 2024-08-17T17:38:42.140 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.140 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.140 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.140 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.140 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.182169+0000", 2024-08-17T17:38:42.140 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3e-06 2024-08-17T17:38:42.140 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.140 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.141 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.141 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.182193+0000", 2024-08-17T17:38:42.141 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.425e-05 2024-08-17T17:38:42.141 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.141 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.141 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.141 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.182206+0000", 2024-08-17T17:38:42.141 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2108e-05 2024-08-17T17:38:42.141 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.141 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.141 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.141 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.182213+0000", 2024-08-17T17:38:42.141 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7249999999999997e-06 2024-08-17T17:38:42.142 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.142 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.142 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.142 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.182280+0000", 2024-08-17T17:38:42.142 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6774999999999997e-05 2024-08-17T17:38:42.142 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.142 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.142 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.142 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.182470+0000", 2024-08-17T17:38:42.142 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00019022700000000001 2024-08-17T17:38:42.142 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.142 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.143 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:38:42.143 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.182634+0000", 2024-08-17T17:38:42.143 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016349100000000001 2024-08-17T17:38:42.143 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.143 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.143 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.143 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.186209+0000", 2024-08-17T17:38:42.143 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0035754409999999999 2024-08-17T17:38:42.143 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.143 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.143 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.143 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.189531+0000", 2024-08-17T17:38:42.143 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0033218750000000002 2024-08-17T17:38:42.143 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.144 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.144 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.144 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.203048+0000", 2024-08-17T17:38:42.144 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.013517032 2024-08-17T17:38:42.144 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.144 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.144 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.144 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.203062+0000", 2024-08-17T17:38:42.144 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3643e-05 2024-08-17T17:38:42.144 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.144 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.144 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.145 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.198676+0000", 2024-08-17T17:38:42.145 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.99561439900000004 2024-08-17T17:38:42.145 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.145 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.145 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.145 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.198691+0000", 2024-08-17T17:38:42.145 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.464e-05 2024-08-17T17:38:42.145 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.145 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.145 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.145 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.451652+0000", 2024-08-17T17:38:42.145 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2529616589999999 2024-08-17T17:38:42.146 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.146 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.146 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.146 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.451782+0000", 2024-08-17T17:38:42.146 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012985500000000001 2024-08-17T17:38:42.146 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.146 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.146 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.146 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.146 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.146 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:45458 11.4 11:2ec5c852:::benchmark_data_smithi063_328699_object45457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1375)", 2024-08-17T17:38:42.146 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:54.189726+0000", 2024-08-17T17:38:42.146 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 167.850206531, 2024-08-17T17:38:42.146 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2660037060000002, 2024-08-17T17:38:42.147 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.147 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.147 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:38:42.147 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.147 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.147 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.147 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 45458 2024-08-17T17:38:42.147 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.147 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.147 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.147 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.147 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.189726+0000", 2024-08-17T17:38:42.147 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.148 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.148 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.148 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.148 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.189726+0000", 2024-08-17T17:38:42.148 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.148 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.148 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.148 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.148 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.189727+0000", 2024-08-17T17:38:42.148 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.395e-06 2024-08-17T17:38:42.148 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.148 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.148 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.148 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.189761+0000", 2024-08-17T17:38:42.149 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.4192000000000001e-05 2024-08-17T17:38:42.149 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.149 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.149 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.149 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.189773+0000", 2024-08-17T17:38:42.149 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1201e-05 2024-08-17T17:38:42.149 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.149 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.149 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.149 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.189780+0000", 2024-08-17T17:38:42.149 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4229999999999999e-06 2024-08-17T17:38:42.149 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.149 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.150 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.150 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.189838+0000", 2024-08-17T17:38:42.150 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8068e-05 2024-08-17T17:38:42.150 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.150 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.150 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.150 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.190030+0000", 2024-08-17T17:38:42.150 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000192048 2024-08-17T17:38:42.150 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.150 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.150 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,7", 2024-08-17T17:38:42.151 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.190208+0000", 2024-08-17T17:38:42.151 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00017827500000000001 2024-08-17T17:38:42.151 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.151 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.151 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.151 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.204277+0000", 2024-08-17T17:38:42.151 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.014068381 2024-08-17T17:38:42.151 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.151 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.151 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.151 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.210029+0000", 2024-08-17T17:38:42.151 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0057517569999999997 2024-08-17T17:38:42.151 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.152 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.152 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.152 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.210038+0000", 2024-08-17T17:38:42.152 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.5529999999999998e-06 2024-08-17T17:38:42.152 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.152 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.152 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.152 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.201928+0000", 2024-08-17T17:38:42.152 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.99188996500000004 2024-08-17T17:38:42.152 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.152 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.152 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.152 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.201939+0000", 2024-08-17T17:38:42.153 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0749e-05 2024-08-17T17:38:42.153 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.153 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.153 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.153 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.446095+0000", 2024-08-17T17:38:42.153 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2441561509999999 2024-08-17T17:38:42.153 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.153 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.153 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for degraded object", 2024-08-17T17:38:42.153 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.447487+0000", 2024-08-17T17:38:42.153 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0013919710000000001 2024-08-17T17:38:42.153 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.153 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.153 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.153 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.455621+0000", 2024-08-17T17:38:42.154 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0081336940000000003 2024-08-17T17:38:42.154 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.154 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.154 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.154 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.455730+0000", 2024-08-17T17:38:42.154 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010888299999999999 2024-08-17T17:38:42.154 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.154 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.154 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.154 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.154 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.154 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:45460 11.d 11:b767b870:::benchmark_data_smithi063_328699_object45459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1375)", 2024-08-17T17:38:42.155 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:54.189799+0000", 2024-08-17T17:38:42.155 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 167.850133164, 2024-08-17T17:38:42.155 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2844039700000001, 2024-08-17T17:38:42.155 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.155 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.155 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:38:42.155 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.155 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.155 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.155 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 45460 2024-08-17T17:38:42.155 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.155 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.155 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.155 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.156 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.189799+0000", 2024-08-17T17:38:42.156 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.156 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.156 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.156 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.156 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.189799+0000", 2024-08-17T17:38:42.156 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.156 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.156 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.156 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.156 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.189801+0000", 2024-08-17T17:38:42.156 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.384e-06 2024-08-17T17:38:42.156 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.157 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.157 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.157 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.189824+0000", 2024-08-17T17:38:42.157 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3489000000000002e-05 2024-08-17T17:38:42.157 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.157 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.157 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.157 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.189831+0000", 2024-08-17T17:38:42.157 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7479999999999996e-06 2024-08-17T17:38:42.157 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.157 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.157 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.157 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.189837+0000", 2024-08-17T17:38:42.157 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9229999999999999e-06 2024-08-17T17:38:42.158 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.158 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.158 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.158 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.189881+0000", 2024-08-17T17:38:42.158 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.3828000000000001e-05 2024-08-17T17:38:42.158 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.158 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.158 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.158 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.190031+0000", 2024-08-17T17:38:42.158 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00015007099999999999 2024-08-17T17:38:42.158 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.158 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.158 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 1,4", 2024-08-17T17:38:42.159 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.190208+0000", 2024-08-17T17:38:42.159 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00017782200000000001 2024-08-17T17:38:42.159 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.159 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.159 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.159 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.197639+0000", 2024-08-17T17:38:42.159 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0074304280000000002 2024-08-17T17:38:42.159 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.159 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.159 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.159 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.203096+0000", 2024-08-17T17:38:42.159 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0054569029999999999 2024-08-17T17:38:42.160 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.160 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.160 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.160 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.203105+0000", 2024-08-17T17:38:42.160 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.329e-06 2024-08-17T17:38:42.160 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.160 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.160 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.160 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.198725+0000", 2024-08-17T17:38:42.160 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.99562017000000003 2024-08-17T17:38:42.160 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.160 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.161 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.161 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.198735+0000", 2024-08-17T17:38:42.161 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.764e-06 2024-08-17T17:38:42.161 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.161 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.161 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.161 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.453913+0000", 2024-08-17T17:38:42.161 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.255177736 2024-08-17T17:38:42.161 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.161 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.161 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for degraded object", 2024-08-17T17:38:42.161 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.454809+0000", 2024-08-17T17:38:42.162 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00089660900000000003 2024-08-17T17:38:42.162 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.162 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.162 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.162 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.474093+0000", 2024-08-17T17:38:42.162 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.019283494000000002 2024-08-17T17:38:42.162 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.162 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.162 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.162 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.474203+0000", 2024-08-17T17:38:42.162 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000110272 2024-08-17T17:38:42.162 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.162 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.163 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.163 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.163 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.163 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:45468 11.d 11:b6d439a0:::benchmark_data_smithi063_328699_object45467:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1376)", 2024-08-17T17:38:42.163 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:35:54.211269+0000", 2024-08-17T17:38:42.163 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 167.82866289099999, 2024-08-17T17:38:42.163 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2654245250000002, 2024-08-17T17:38:42.163 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.163 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.163 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:42.163 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.163 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.164 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:42.165 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.165 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 45468 2024-08-17T17:38:42.165 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.165 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.165 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.165 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.165 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.211269+0000", 2024-08-17T17:38:42.165 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.165 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.165 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.165 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.166 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.211269+0000", 2024-08-17T17:38:42.166 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.166 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.166 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.166 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.166 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.211271+0000", 2024-08-17T17:38:42.166 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.144e-06 2024-08-17T17:38:42.166 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.166 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.166 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.166 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.211294+0000", 2024-08-17T17:38:42.166 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3264999999999998e-05 2024-08-17T17:38:42.167 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.167 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.167 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.167 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.211307+0000", 2024-08-17T17:38:42.167 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.275e-05 2024-08-17T17:38:42.167 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.167 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.167 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.167 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.211315+0000", 2024-08-17T17:38:42.167 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9300000000000003e-06 2024-08-17T17:38:42.167 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.167 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.167 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.167 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.211365+0000", 2024-08-17T17:38:42.168 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.0612999999999998e-05 2024-08-17T17:38:42.168 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.168 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.168 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.168 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:54.211371+0000", 2024-08-17T17:38:42.168 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1970000000000001e-06 2024-08-17T17:38:42.168 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.168 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.168 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.168 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.198760+0000", 2024-08-17T17:38:42.168 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98738814200000002 2024-08-17T17:38:42.168 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.168 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.168 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.169 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:55.198767+0000", 2024-08-17T17:38:42.169 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6210000000000004e-06 2024-08-17T17:38:42.169 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.169 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.169 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.169 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.454856+0000", 2024-08-17T17:38:42.169 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.256088951 2024-08-17T17:38:42.169 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.169 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.169 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.169 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.455022+0000", 2024-08-17T17:38:42.169 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016581699999999999 2024-08-17T17:38:42.169 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.169 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.170 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 4,6", 2024-08-17T17:38:42.170 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.455198+0000", 2024-08-17T17:38:42.170 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00017656599999999999 2024-08-17T17:38:42.170 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.170 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.170 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.170 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.458897+0000", 2024-08-17T17:38:42.170 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0036982920000000002 2024-08-17T17:38:42.170 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.170 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.170 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.170 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.469373+0000", 2024-08-17T17:38:42.170 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.010476298 2024-08-17T17:38:42.171 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.171 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.171 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.171 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.474245+0000", 2024-08-17T17:38:42.171 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0048717259999999998 2024-08-17T17:38:42.171 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.171 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.171 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:38:42.171 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.474305+0000", 2024-08-17T17:38:42.171 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9849999999999998e-05 2024-08-17T17:38:42.171 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.171 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.171 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.171 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:35:56.476694+0000", 2024-08-17T17:38:42.172 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0023893629999999998 2024-08-17T17:38:42.172 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.172 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.172 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.172 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.172 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.172 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:62278 11.2 11:4f6a2035:::benchmark_data_smithi063_328699_object62277:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1381)", 2024-08-17T17:38:42.172 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:36:11.742879+0000", 2024-08-17T17:38:42.172 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 150.29705360700001, 2024-08-17T17:38:42.172 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.341815913, 2024-08-17T17:38:42.172 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.172 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.172 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:38:42.173 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.173 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.173 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.173 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 62278 2024-08-17T17:38:42.173 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.173 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.173 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.173 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.173 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.742879+0000", 2024-08-17T17:38:42.173 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.173 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.173 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.173 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.173 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.742879+0000", 2024-08-17T17:38:42.174 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.174 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.174 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.174 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.174 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.742880+0000", 2024-08-17T17:38:42.174 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6789999999999999e-06 2024-08-17T17:38:42.174 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.174 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.174 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.174 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.743221+0000", 2024-08-17T17:38:42.174 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00034102299999999998 2024-08-17T17:38:42.174 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.174 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.175 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.175 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.743270+0000", 2024-08-17T17:38:42.175 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9002999999999998e-05 2024-08-17T17:38:42.175 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.175 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.175 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.175 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.743281+0000", 2024-08-17T17:38:42.175 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0886e-05 2024-08-17T17:38:42.175 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.175 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.175 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.175 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.743348+0000", 2024-08-17T17:38:42.175 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6925e-05 2024-08-17T17:38:42.175 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.175 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.176 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.176 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.743532+0000", 2024-08-17T17:38:42.176 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000183976 2024-08-17T17:38:42.176 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.176 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.176 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,7", 2024-08-17T17:38:42.176 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.743695+0000", 2024-08-17T17:38:42.176 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00016296099999999999 2024-08-17T17:38:42.176 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.176 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.176 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.176 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.749976+0000", 2024-08-17T17:38:42.176 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.006280847 2024-08-17T17:38:42.176 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.177 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.177 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.177 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.749988+0000", 2024-08-17T17:38:42.177 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1494999999999999e-05 2024-08-17T17:38:42.177 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.177 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.177 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.177 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.749226+0000", 2024-08-17T17:38:42.177 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.99923878799999999 2024-08-17T17:38:42.177 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.177 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.177 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.177 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.749245+0000", 2024-08-17T17:38:42.177 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9108000000000001e-05 2024-08-17T17:38:42.178 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.178 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.178 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.178 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.072341+0000", 2024-08-17T17:38:42.178 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3230952949999999 2024-08-17T17:38:42.178 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.178 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.178 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for degraded object", 2024-08-17T17:38:42.178 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.073033+0000", 2024-08-17T17:38:42.178 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00069176700000000003 2024-08-17T17:38:42.178 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.178 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.178 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.178 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.084622+0000", 2024-08-17T17:38:42.178 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.011589088000000001 2024-08-17T17:38:42.179 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.179 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.179 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.179 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.084695+0000", 2024-08-17T17:38:42.179 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.3071999999999994e-05 2024-08-17T17:38:42.179 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.179 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.179 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.179 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.179 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.179 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:62280 11.2 11:470e6f04:::benchmark_data_smithi063_328699_object62279:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1382)", 2024-08-17T17:38:42.179 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:36:11.755412+0000", 2024-08-17T17:38:42.179 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 150.284520257, 2024-08-17T17:38:42.180 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3318083440000001, 2024-08-17T17:38:42.180 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.180 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.180 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:42.180 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.180 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.180 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.180 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 62280 2024-08-17T17:38:42.180 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.180 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.180 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.180 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.180 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.755412+0000", 2024-08-17T17:38:42.181 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.181 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.181 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.181 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.181 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.755412+0000", 2024-08-17T17:38:42.181 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.181 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.181 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.181 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.181 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.755413+0000", 2024-08-17T17:38:42.181 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9299999999999998e-07 2024-08-17T17:38:42.181 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.181 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.182 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.182 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.755437+0000", 2024-08-17T17:38:42.182 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.4009999999999999e-05 2024-08-17T17:38:42.182 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.182 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.182 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.182 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.755444+0000", 2024-08-17T17:38:42.182 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.807e-06 2024-08-17T17:38:42.182 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.182 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.182 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.182 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.755450+0000", 2024-08-17T17:38:42.182 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.6720000000000001e-06 2024-08-17T17:38:42.183 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.183 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.183 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.183 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.756946+0000", 2024-08-17T17:38:42.183 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0014963380000000001 2024-08-17T17:38:42.183 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.183 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.183 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.183 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.756959+0000", 2024-08-17T17:38:42.183 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.237e-05 2024-08-17T17:38:42.183 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.183 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.183 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.183 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.750673+0000", 2024-08-17T17:38:42.184 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.99371417299999998 2024-08-17T17:38:42.184 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.184 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.184 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.184 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.750682+0000", 2024-08-17T17:38:42.184 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.1099999999999992e-06 2024-08-17T17:38:42.184 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.184 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.184 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.184 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.075895+0000", 2024-08-17T17:38:42.184 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3252133500000001 2024-08-17T17:38:42.184 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.184 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.185 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.185 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.076030+0000", 2024-08-17T17:38:42.185 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000134497 2024-08-17T17:38:42.185 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.185 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.185 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:38:42.185 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.076136+0000", 2024-08-17T17:38:42.185 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010617599999999999 2024-08-17T17:38:42.185 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.185 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.185 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.185 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.084692+0000", 2024-08-17T17:38:42.185 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0085554470000000007 2024-08-17T17:38:42.186 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.186 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.186 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.186 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.084761+0000", 2024-08-17T17:38:42.186 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9868000000000003e-05 2024-08-17T17:38:42.186 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.186 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.186 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.186 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.084827+0000", 2024-08-17T17:38:42.186 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.5790000000000005e-05 2024-08-17T17:38:42.186 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.186 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.186 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:38:42.187 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.084881+0000", 2024-08-17T17:38:42.187 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3909000000000002e-05 2024-08-17T17:38:42.187 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.187 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.187 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.187 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.087220+0000", 2024-08-17T17:38:42.187 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0023393340000000002 2024-08-17T17:38:42.187 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.187 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.187 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.187 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.187 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.187 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:62287 11.1 11.1bbb9f21 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1382)", 2024-08-17T17:38:42.187 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:36:11.770653+0000", 2024-08-17T17:38:42.187 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 150.26927933600001, 2024-08-17T17:38:42.188 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.97995742799999996, 2024-08-17T17:38:42.188 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.188 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.188 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:38:42.188 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.188 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.188 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.188 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 62287 2024-08-17T17:38:42.188 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.188 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.188 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.188 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.188 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.770653+0000", 2024-08-17T17:38:42.189 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.189 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.189 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.189 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.189 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.770653+0000", 2024-08-17T17:38:42.189 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.189 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.189 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.189 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.189 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.770655+0000", 2024-08-17T17:38:42.189 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6419999999999999e-06 2024-08-17T17:38:42.189 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.189 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.189 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.190 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.770677+0000", 2024-08-17T17:38:42.190 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2793999999999999e-05 2024-08-17T17:38:42.190 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.190 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.190 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.190 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.770688+0000", 2024-08-17T17:38:42.190 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0681e-05 2024-08-17T17:38:42.190 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.190 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.190 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.190 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.770696+0000", 2024-08-17T17:38:42.190 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4150000000000002e-06 2024-08-17T17:38:42.190 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.191 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.191 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.191 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.771196+0000", 2024-08-17T17:38:42.191 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00050074099999999997 2024-08-17T17:38:42.191 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.191 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.191 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.191 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.771206+0000", 2024-08-17T17:38:42.191 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7520000000000008e-06 2024-08-17T17:38:42.191 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.191 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.191 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.191 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.750599+0000", 2024-08-17T17:38:42.192 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.97939335900000002 2024-08-17T17:38:42.192 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.192 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.192 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.192 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.750610+0000", 2024-08-17T17:38:42.192 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1044e-05 2024-08-17T17:38:42.192 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.192 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.192 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.192 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.192 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.192 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:62310 11.2 11:4258f068:::benchmark_data_smithi063_328699_object62309:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1383)", 2024-08-17T17:38:42.193 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:36:12.900962+0000", 2024-08-17T17:38:42.193 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 149.138970857, 2024-08-17T17:38:42.193 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.198921911, 2024-08-17T17:38:42.193 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:42.193 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:42.193 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:38:42.193 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:38:42.193 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:38:42.193 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:38:42.193 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 62310 2024-08-17T17:38:42.193 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.193 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:42.193 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.194 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:42.194 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.900962+0000", 2024-08-17T17:38:42.194 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.194 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.194 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.194 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:42.194 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.900962+0000", 2024-08-17T17:38:42.194 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:42.194 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.194 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.194 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:42.194 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.900963+0000", 2024-08-17T17:38:42.194 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.113e-06 2024-08-17T17:38:42.194 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.195 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.195 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:42.195 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.900982+0000", 2024-08-17T17:38:42.195 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9567000000000001e-05 2024-08-17T17:38:42.195 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.195 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.195 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:42.195 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.900994+0000", 2024-08-17T17:38:42.195 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1871e-05 2024-08-17T17:38:42.195 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.195 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.195 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:42.195 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.900999+0000", 2024-08-17T17:38:42.195 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1680000000000003e-06 2024-08-17T17:38:42.195 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.196 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.196 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.196 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.901052+0000", 2024-08-17T17:38:42.196 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2748999999999999e-05 2024-08-17T17:38:42.196 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.196 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.196 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:38:42.196 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.901059+0000", 2024-08-17T17:38:42.196 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6950000000000003e-06 2024-08-17T17:38:42.196 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.196 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.196 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:42.196 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.087280+0000", 2024-08-17T17:38:42.196 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1862214090000001 2024-08-17T17:38:42.197 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.197 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.197 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:42.197 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.087459+0000", 2024-08-17T17:38:42.197 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00017855900000000001 2024-08-17T17:38:42.197 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.197 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.197 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 0,3", 2024-08-17T17:38:42.197 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.087623+0000", 2024-08-17T17:38:42.197 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000164019 2024-08-17T17:38:42.197 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.197 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.197 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.197 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.093942+0000", 2024-08-17T17:38:42.197 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0063190090000000004 2024-08-17T17:38:42.198 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.198 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.198 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:38:42.198 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.093952+0000", 2024-08-17T17:38:42.198 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0772e-05 2024-08-17T17:38:42.198 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.198 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.198 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:38:42.198 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.097187+0000", 2024-08-17T17:38:42.198 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0032345260000000002 2024-08-17T17:38:42.198 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.198 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.198 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:38:42.199 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.097250+0000", 2024-08-17T17:38:42.199 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3200999999999993e-05 2024-08-17T17:38:42.199 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:42.199 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:42.199 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:38:42.199 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:14.099883+0000", 2024-08-17T17:38:42.199 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0026332529999999999 2024-08-17T17:38:42.199 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.199 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.199 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.199 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:42.199 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:42.199 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:38:42.222 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:42.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:42.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:42 smithi063 ceph-mon[109843]: pgmap v2223: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 17371/133716 objects degraded (12.991%); 4293/133716 objects misplaced (3.211%); 3.5 MiB/s, 176 objects/s recovering 2024-08-17T17:38:42.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:42 smithi063 ceph-mon[114756]: pgmap v2223: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 17371/133716 objects degraded (12.991%); 4293/133716 objects misplaced (3.211%); 3.5 MiB/s, 176 objects/s recovering 2024-08-17T17:38:42.324 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:42.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:42 smithi162 ceph-mon[104668]: pgmap v2223: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 128 op/s; 17371/133716 objects degraded (12.991%); 4293/133716 objects misplaced (3.211%); 3.5 MiB/s, 176 objects/s recovering 2024-08-17T17:38:42.347 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:42.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:42.449 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:42.471 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:42.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:42.573 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:42.596 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:42.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:42.698 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:42.724 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:42.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:42.825 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:42.847 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:42.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:42.949 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:42.973 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:42.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:42.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:43.074 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:43.108 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:43.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:43.210 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:43.235 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:43.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:43.337 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:43.361 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:43.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:43.463 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:43.488 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:43.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:43.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:38:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:38:43] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.51.0" 2024-08-17T17:38:43.590 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:43.616 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:43.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:43.718 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:43.744 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:43.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:43.846 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:43.869 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:43.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:43.971 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:43.994 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:43.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:43.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:44.095 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:44.118 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:44.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.119 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:44.220 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:44.242 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:44.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:44.344 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:44.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:44 smithi162 ceph-mon[104668]: pgmap v2224: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 17029/131361 objects degraded (12.964%); 4171/131361 objects misplaced (3.175%); 5.3 MiB/s, 204 objects/s recovering 2024-08-17T17:38:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:44 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 17029/131361 objects degraded (12.964%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:44.370 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:44.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:44.472 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:44.495 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:44.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:44.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:44 smithi063 ceph-mon[114756]: pgmap v2224: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 17029/131361 objects degraded (12.964%); 4171/131361 objects misplaced (3.175%); 5.3 MiB/s, 204 objects/s recovering 2024-08-17T17:38:44.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:44 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 17029/131361 objects degraded (12.964%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:44.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:44 smithi063 ceph-mon[109843]: pgmap v2224: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 17029/131361 objects degraded (12.964%); 4171/131361 objects misplaced (3.175%); 5.3 MiB/s, 204 objects/s recovering 2024-08-17T17:38:44.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:44 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 17029/131361 objects degraded (12.964%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:44.556 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:38:44.596 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:44.621 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:44.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:44.722 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:44.747 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:44.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:44.849 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:44.875 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:44.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:44.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:44.976 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:45.002 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:45.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:45.104 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:45.126 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:45.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:45.228 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:45.257 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:45.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:45.358 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:45.380 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:45.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:45.482 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:45.511 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:45.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:45.613 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:45.628 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:45.638 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:45.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:45.737 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:38:45.740 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:45.767 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:45.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:45.830 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:38:45.832 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:38:45.869 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:45.892 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:45.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:45.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:45.994 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:46.019 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:46.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:46.106 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:38:46.121 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:46.147 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:46.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:46.249 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:46.273 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:46.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:46.375 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:46.401 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:46.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:46.424 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:46 smithi063 ceph-mon[114756]: pgmap v2225: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 168 op/s; 17029/130995 objects degraded (13.000%); 4171/130995 objects misplaced (3.184%); 3.0 MiB/s, 52 objects/s recovering 2024-08-17T17:38:46.424 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:46 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/816184466' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:38:46.425 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:46 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:38:46.425 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:46 smithi063 ceph-mon[109843]: pgmap v2225: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 168 op/s; 17029/130995 objects degraded (13.000%); 4171/130995 objects misplaced (3.184%); 3.0 MiB/s, 52 objects/s recovering 2024-08-17T17:38:46.425 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:46 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/816184466' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:38:46.425 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:46 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:38:46.504 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:46.526 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:46.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:46.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:46 smithi162 ceph-mon[104668]: pgmap v2225: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 168 op/s; 17029/130995 objects degraded (13.000%); 4171/130995 objects misplaced (3.184%); 3.0 MiB/s, 52 objects/s recovering 2024-08-17T17:38:46.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:46 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/816184466' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:38:46.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:46 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:38:46.628 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:46.650 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:46.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:46.752 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:46.777 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:46.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:46.879 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:46.904 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:46.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:46.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:47.006 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:47.030 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:47.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:47.132 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:47.156 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:47.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:47.258 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:47.281 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:47.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:47.382 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:47.389 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:47 smithi063 ceph-mon[109843]: from='client.36378 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:47.390 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:47 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:47.390 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:47 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:38:47.390 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:47 smithi063 ceph-mon[109843]: osdmap e1421: 8 total, 8 up, 8 in 2024-08-17T17:38:47.390 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:47.390 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:47.390 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:47 smithi063 ceph-mon[114756]: from='client.36378 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:47.390 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:47 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:47.390 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:47 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:38:47.390 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:47 smithi063 ceph-mon[114756]: osdmap e1421: 8 total, 8 up, 8 in 2024-08-17T17:38:47.390 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:47.391 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:47.409 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:47.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:47.416 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2225,"stamp":"2024-08-17T17:38:44.649090+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2860500657,"num_objects":43665,"num_object_clones":0,"num_object_copies":130995,"num_objects_missing_on_primary":4660,"num_objects_missing":4660,"num_objects_degraded":17029,"num_objects_misplaced":4171,"num_objects_unfound":0,"num_objects_dirty":43665,"num_whiteouts":0,"num_read":28,"num_read_kb":62,"num_write":85322,"num_write_kb":4124377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28403,"num_bytes_recovered":1708528349,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89821,"ondisk_log_size":89821,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":375,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14872084,"kb_used_data":9087060,"kb_used_omap":298,"kb_used_meta":5784661,"kb_avail":735089132,"statfs":{"total":767960285184,"available":752731271168,"internally_reserved":0,"allocated":9305149440,"data_stored":9283908563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":5923493393},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-94568448,"num_objects":-1443,"num_object_clones":0,"num_object_copies":-4329,"num_objects_missing_on_primary":-354,"num_objects_missing":-354,"num_objects_degraded":-567,"num_objects_misplaced":-290,"num_objects_unfound":0,"num_objects_dirty":-1443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1443,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":447,"num_bytes_recovered":26607616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.573621"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":46,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081662+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:38:37.081662+0000","last_peered":"2024-08-17T17:38:37.081662+0000","last_clean":"2024-08-17T17:38:37.081662+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:38:37.081662+0000","last_undegraded":"2024-08-17T17:38:37.081662+0000","last_fullsized":"2024-08-17T17:38:37.081662+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1434,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.080221+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:38:36.080221+0000","last_peered":"2024-08-17T17:38:36.080221+0000","last_clean":"2024-08-17T17:38:36.080221+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:38:36.080221+0000","last_undegraded":"2024-08-17T17:38:36.080221+0000","last_fullsized":"2024-08-17T17:38:36.080221+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1749,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.081629+0000","last_change":"2024-08-17T17:36:15.771150+0000","last_active":"2024-08-17T17:38:36.081629+0000","last_peered":"2024-08-17T17:38:36.081629+0000","last_clean":"2024-08-17T17:38:36.081629+0000","last_became_active":"2024-08-17T17:36:15.770520+0000","last_became_peered":"2024-08-17T17:36:15.770520+0000","last_unstale":"2024-08-17T17:38:36.081629+0000","last_undegraded":"2024-08-17T17:38:36.081629+0000","last_fullsized":"2024-08-17T17:38:36.081629+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:17.552796+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":84,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.078655+0000","last_change":"2024-08-17T17:36:12.756709+0000","last_active":"2024-08-17T17:38:36.078655+0000","last_peered":"2024-08-17T17:38:36.078655+0000","last_clean":"2024-08-17T17:38:36.078655+0000","last_became_active":"2024-08-17T17:36:12.756385+0000","last_became_peered":"2024-08-17T17:36:12.756385+0000","last_unstale":"2024-08-17T17:38:36.078655+0000","last_undegraded":"2024-08-17T17:38:36.078655+0000","last_fullsized":"2024-08-17T17:38:36.078655+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:21.420282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":131,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.097915+0000","last_change":"2024-08-17T17:36:12.762800+0000","last_active":"2024-08-17T17:38:37.097915+0000","last_peered":"2024-08-17T17:38:37.097915+0000","last_clean":"2024-08-17T17:38:37.097915+0000","last_became_active":"2024-08-17T17:36:12.759977+0000","last_became_peered":"2024-08-17T17:36:12.759977+0000","last_unstale":"2024-08-17T17:38:37.097915+0000","last_undegraded":"2024-08-17T17:38:37.097915+0000","last_fullsized":"2024-08-17T17:38:37.097915+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:02.162681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1629,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.077810+0000","last_change":"2024-08-17T17:35:56.281561+0000","last_active":"2024-08-17T17:38:36.077810+0000","last_peered":"2024-08-17T17:38:36.077810+0000","last_clean":"2024-08-17T17:38:36.077810+0000","last_became_active":"2024-08-17T17:35:56.278638+0000","last_became_peered":"2024-08-17T17:35:56.278638+0000","last_unstale":"2024-08-17T17:38:36.077810+0000","last_undegraded":"2024-08-17T17:38:36.077810+0000","last_fullsized":"2024-08-17T17:38:36.077810+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1367,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.077706+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:38:36.077706+0000","last_peered":"2024-08-17T17:38:36.077706+0000","last_clean":"2024-08-17T17:38:36.077706+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:38:36.077706+0000","last_undegraded":"2024-08-17T17:38:36.077706+0000","last_fullsized":"2024-08-17T17:38:36.077706+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":43,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.080267+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:38:36.080267+0000","last_peered":"2024-08-17T17:38:36.080267+0000","last_clean":"2024-08-17T17:38:36.080267+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:38:36.080267+0000","last_undegraded":"2024-08-17T17:38:36.080267+0000","last_fullsized":"2024-08-17T17:38:36.080267+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1413,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081669+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:38:37.081669+0000","last_peered":"2024-08-17T17:38:37.081669+0000","last_clean":"2024-08-17T17:38:37.081669+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:38:37.081669+0000","last_undegraded":"2024-08-17T17:38:37.081669+0000","last_fullsized":"2024-08-17T17:38:37.081669+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1421,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084405+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:38:37.084405+0000","last_peered":"2024-08-17T17:38:37.084405+0000","last_clean":"2024-08-17T17:38:37.084405+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:38:37.084405+0000","last_undegraded":"2024-08-17T17:38:37.084405+0000","last_fullsized":"2024-08-17T17:38:37.084405+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":711,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.078552+0000","last_change":"2024-08-17T17:36:15.773315+0000","last_active":"2024-08-17T17:38:36.078552+0000","last_peered":"2024-08-17T17:38:36.078552+0000","last_clean":"2024-08-17T17:38:36.078552+0000","last_became_active":"2024-08-17T17:36:15.772961+0000","last_became_peered":"2024-08-17T17:36:15.772961+0000","last_unstale":"2024-08-17T17:38:36.078552+0000","last_undegraded":"2024-08-17T17:38:36.078552+0000","last_fullsized":"2024-08-17T17:38:36.078552+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:10.190221+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":40,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081624+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:38:37.081624+0000","last_peered":"2024-08-17T17:38:37.081624+0000","last_clean":"2024-08-17T17:38:37.081624+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:38:37.081624+0000","last_undegraded":"2024-08-17T17:38:37.081624+0000","last_fullsized":"2024-08-17T17:38:37.081624+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":131,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.098235+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:38:37.098235+0000","last_peered":"2024-08-17T17:38:37.098235+0000","last_clean":"2024-08-17T17:38:37.098235+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:38:37.098235+0000","last_undegraded":"2024-08-17T17:38:37.098235+0000","last_fullsized":"2024-08-17T17:38:37.098235+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1502,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.107562+0000","last_change":"2024-08-17T17:36:24.225739+0000","last_active":"2024-08-17T17:38:36.107562+0000","last_peered":"2024-08-17T17:38:36.107562+0000","last_clean":"2024-08-17T17:38:36.107562+0000","last_became_active":"2024-08-17T17:36:24.225250+0000","last_became_peered":"2024-08-17T17:36:24.225250+0000","last_unstale":"2024-08-17T17:38:36.107562+0000","last_undegraded":"2024-08-17T17:38:36.107562+0000","last_fullsized":"2024-08-17T17:38:36.107562+0000","mapping_epoch":1393,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1394,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.290656+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1441,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.080668+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:38:36.080668+0000","last_peered":"2024-08-17T17:38:36.080668+0000","last_clean":"2024-08-17T17:38:36.080668+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:38:36.080668+0000","last_undegraded":"2024-08-17T17:38:36.080668+0000","last_fullsized":"2024-08-17T17:38:36.080668+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1848,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.081795+0000","last_change":"2024-08-17T17:36:12.752970+0000","last_active":"2024-08-17T17:38:36.081795+0000","last_peered":"2024-08-17T17:38:36.081795+0000","last_clean":"2024-08-17T17:38:36.081795+0000","last_became_active":"2024-08-17T17:36:12.752311+0000","last_became_peered":"2024-08-17T17:36:12.752311+0000","last_unstale":"2024-08-17T17:38:36.081795+0000","last_undegraded":"2024-08-17T17:38:36.081795+0000","last_fullsized":"2024-08-17T17:38:36.081795+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:38.131419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1795,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.076954+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:38:36.076954+0000","last_peered":"2024-08-17T17:38:36.076954+0000","last_clean":"2024-08-17T17:38:36.076954+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:38:36.076954+0000","last_undegraded":"2024-08-17T17:38:36.076954+0000","last_fullsized":"2024-08-17T17:38:36.076954+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1503,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.088786+0000","last_change":"2024-08-17T17:36:18.750494+0000","last_active":"2024-08-17T17:38:37.088786+0000","last_peered":"2024-08-17T17:38:37.088786+0000","last_clean":"2024-08-17T17:38:37.088786+0000","last_became_active":"2024-08-17T17:36:18.750092+0000","last_became_peered":"2024-08-17T17:36:18.750092+0000","last_unstale":"2024-08-17T17:38:37.088786+0000","last_undegraded":"2024-08-17T17:38:37.088786+0000","last_fullsized":"2024-08-17T17:38:37.088786+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:36.599727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1132,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081739+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:38:37.081739+0000","last_peered":"2024-08-17T17:38:37.081739+0000","last_clean":"2024-08-17T17:38:37.081739+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:38:37.081739+0000","last_undegraded":"2024-08-17T17:38:37.081739+0000","last_fullsized":"2024-08-17T17:38:37.081739+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":962,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.076867+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:38:36.076867+0000","last_peered":"2024-08-17T17:38:36.076867+0000","last_clean":"2024-08-17T17:38:36.076867+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:38:36.076867+0000","last_undegraded":"2024-08-17T17:38:36.076867+0000","last_fullsized":"2024-08-17T17:38:36.076867+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":40,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081285+0000","last_change":"2024-08-17T17:37:11.075505+0000","last_active":"2024-08-17T17:38:37.081285+0000","last_peered":"2024-08-17T17:38:37.081285+0000","last_clean":"2024-08-17T17:38:37.081285+0000","last_became_active":"2024-08-17T17:37:11.075131+0000","last_became_peered":"2024-08-17T17:37:11.075131+0000","last_unstale":"2024-08-17T17:38:37.081285+0000","last_undegraded":"2024-08-17T17:38:37.081285+0000","last_fullsized":"2024-08-17T17:38:37.081285+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:39.672054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1275,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.079157+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:38:36.079157+0000","last_peered":"2024-08-17T17:38:36.079157+0000","last_clean":"2024-08-17T17:38:36.079157+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:38:36.079157+0000","last_undegraded":"2024-08-17T17:38:36.079157+0000","last_fullsized":"2024-08-17T17:38:36.079157+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1652,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.080354+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:38:36.080354+0000","last_peered":"2024-08-17T17:38:36.080354+0000","last_clean":"2024-08-17T17:38:36.080354+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:38:36.080354+0000","last_undegraded":"2024-08-17T17:38:36.080354+0000","last_fullsized":"2024-08-17T17:38:36.080354+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":87,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.086396+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:38:37.086396+0000","last_peered":"2024-08-17T17:38:37.086396+0000","last_clean":"2024-08-17T17:38:37.086396+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:38:37.086396+0000","last_undegraded":"2024-08-17T17:38:37.086396+0000","last_fullsized":"2024-08-17T17:38:37.086396+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1249,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.078069+0000","last_change":"2024-08-17T17:36:12.755873+0000","last_active":"2024-08-17T17:38:36.078069+0000","last_peered":"2024-08-17T17:38:36.078069+0000","last_clean":"2024-08-17T17:38:36.078069+0000","last_became_active":"2024-08-17T17:36:12.755366+0000","last_became_peered":"2024-08-17T17:36:12.755366+0000","last_unstale":"2024-08-17T17:38:36.078069+0000","last_undegraded":"2024-08-17T17:38:36.078069+0000","last_fullsized":"2024-08-17T17:38:36.078069+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:36.523214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1603,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.083492+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:38:36.083492+0000","last_peered":"2024-08-17T17:38:36.083492+0000","last_clean":"2024-08-17T17:38:36.083492+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:38:36.083492+0000","last_undegraded":"2024-08-17T17:38:36.083492+0000","last_fullsized":"2024-08-17T17:38:36.083492+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":787,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.107671+0000","last_change":"2024-08-17T17:35:55.200519+0000","last_active":"2024-08-17T17:38:36.107671+0000","last_peered":"2024-08-17T17:38:36.107671+0000","last_clean":"2024-08-17T17:38:36.107671+0000","last_became_active":"2024-08-17T17:35:55.200230+0000","last_became_peered":"2024-08-17T17:35:55.200230+0000","last_unstale":"2024-08-17T17:38:36.107671+0000","last_undegraded":"2024-08-17T17:38:36.107671+0000","last_fullsized":"2024-08-17T17:38:36.107671+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:33.454851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1426,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.078757+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:38:36.078757+0000","last_peered":"2024-08-17T17:38:36.078757+0000","last_clean":"2024-08-17T17:38:36.078757+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:38:36.078757+0000","last_undegraded":"2024-08-17T17:38:36.078757+0000","last_fullsized":"2024-08-17T17:38:36.078757+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1704,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.097933+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:38:37.097933+0000","last_peered":"2024-08-17T17:38:37.097933+0000","last_clean":"2024-08-17T17:38:37.097933+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:38:37.097933+0000","last_undegraded":"2024-08-17T17:38:37.097933+0000","last_fullsized":"2024-08-17T17:38:37.097933+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:22.077229+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":251,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085199+0000","last_change":"2024-08-17T17:35:33.145476+0000","last_active":"2024-08-17T17:38:37.085199+0000","last_peered":"2024-08-17T17:38:37.085199+0000","last_clean":"2024-08-17T17:38:37.085199+0000","last_became_active":"2024-08-17T17:35:33.145125+0000","last_became_peered":"2024-08-17T17:35:33.145125+0000","last_unstale":"2024-08-17T17:38:37.085199+0000","last_undegraded":"2024-08-17T17:38:37.085199+0000","last_fullsized":"2024-08-17T17:38:37.085199+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:45.298064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":931,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.093692+0000","last_change":"2024-08-17T17:35:56.248612+0000","last_active":"2024-08-17T17:38:36.093692+0000","last_peered":"2024-08-17T17:38:36.093692+0000","last_clean":"2024-08-17T17:38:36.093692+0000","last_became_active":"2024-08-17T17:35:56.244364+0000","last_became_peered":"2024-08-17T17:35:56.244364+0000","last_unstale":"2024-08-17T17:38:36.093692+0000","last_undegraded":"2024-08-17T17:38:36.093692+0000","last_fullsized":"2024-08-17T17:38:36.093692+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:41.064165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":870,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.077834+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:38:36.077834+0000","last_peered":"2024-08-17T17:38:36.077834+0000","last_clean":"2024-08-17T17:38:36.077834+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:38:36.077834+0000","last_undegraded":"2024-08-17T17:38:36.077834+0000","last_fullsized":"2024-08-17T17:38:36.077834+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1406,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.081659+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:38:36.081659+0000","last_peered":"2024-08-17T17:38:36.081659+0000","last_clean":"2024-08-17T17:38:36.081659+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:38:36.081659+0000","last_undegraded":"2024-08-17T17:38:36.081659+0000","last_fullsized":"2024-08-17T17:38:36.081659+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1425,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.081229+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:38:36.081229+0000","last_peered":"2024-08-17T17:38:36.081229+0000","last_clean":"2024-08-17T17:38:36.081229+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:38:36.081229+0000","last_undegraded":"2024-08-17T17:38:36.081229+0000","last_fullsized":"2024-08-17T17:38:36.081229+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":926,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.087692+0000","last_change":"2024-08-17T17:36:15.774881+0000","last_active":"2024-08-17T17:38:37.087692+0000","last_peered":"2024-08-17T17:38:37.087692+0000","last_clean":"2024-08-17T17:38:37.087692+0000","last_became_active":"2024-08-17T17:36:15.774435+0000","last_became_peered":"2024-08-17T17:36:15.774435+0000","last_unstale":"2024-08-17T17:38:37.087692+0000","last_undegraded":"2024-08-17T17:38:37.087692+0000","last_fullsized":"2024-08-17T17:38:37.087692+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:52.015854+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"1420'5495","reported_seq":13671,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:42.128723+0000","last_change":"2024-08-17T17:38:37.110784+0000","last_active":"2024-08-17T17:38:42.128723+0000","last_peered":"2024-08-17T17:38:42.128723+0000","last_clean":"2024-08-17T17:38:42.128723+0000","last_became_active":"2024-08-17T17:38:37.088380+0000","last_became_peered":"2024-08-17T17:38:37.088380+0000","last_unstale":"2024-08-17T17:38:42.128723+0000","last_undegraded":"2024-08-17T17:38:42.128723+0000","last_fullsized":"2024-08-17T17:38:42.128723+0000","mapping_epoch":1419,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5495,"log_dups_size":0,"ondisk_log_size":5495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:08.647903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182910976,"num_objects":2791,"num_object_clones":0,"num_object_copies":8373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5495,"num_write_kb":265152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4009,"num_bytes_recovered":199753728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1373,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.080343+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:38:36.080343+0000","last_peered":"2024-08-17T17:38:36.080343+0000","last_clean":"2024-08-17T17:38:36.080343+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:38:36.080343+0000","last_undegraded":"2024-08-17T17:38:36.080343+0000","last_fullsized":"2024-08-17T17:38:36.080343+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1269,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.079042+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:38:36.079042+0000","last_peered":"2024-08-17T17:38:36.079042+0000","last_clean":"2024-08-17T17:38:36.079042+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:38:36.079042+0000","last_undegraded":"2024-08-17T17:38:36.079042+0000","last_fullsized":"2024-08-17T17:38:36.079042+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":318,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.082004+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:38:37.082004+0000","last_peered":"2024-08-17T17:38:37.082004+0000","last_clean":"2024-08-17T17:38:37.082004+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:38:37.082004+0000","last_undegraded":"2024-08-17T17:38:37.082004+0000","last_fullsized":"2024-08-17T17:38:37.082004+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1587,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084821+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:38:37.084821+0000","last_peered":"2024-08-17T17:38:37.084821+0000","last_clean":"2024-08-17T17:38:37.084821+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:38:37.084821+0000","last_undegraded":"2024-08-17T17:38:37.084821+0000","last_fullsized":"2024-08-17T17:38:37.084821+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:25.751871+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.f","version":"1420'5413","reported_seq":5557,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:40.531517+0000","last_change":"2024-08-17T17:35:34.179168+0000","last_active":"2024-08-17T17:38:40.531517+0000","last_peered":"2024-08-17T17:38:40.531517+0000","last_clean":"2024-08-17T17:38:40.531517+0000","last_became_active":"2024-08-17T17:35:34.133493+0000","last_became_peered":"2024-08-17T17:35:34.133493+0000","last_unstale":"2024-08-17T17:38:40.531517+0000","last_undegraded":"2024-08-17T17:38:40.531517+0000","last_fullsized":"2024-08-17T17:38:40.531517+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5413,"log_dups_size":0,"ondisk_log_size":5413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:24:11.188566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182910976,"num_objects":2791,"num_object_clones":0,"num_object_copies":8373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5413,"num_write_kb":262528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":94,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.086492+0000","last_change":"2024-08-17T17:37:11.079712+0000","last_active":"2024-08-17T17:38:37.086492+0000","last_peered":"2024-08-17T17:38:37.086492+0000","last_clean":"2024-08-17T17:38:37.086492+0000","last_became_active":"2024-08-17T17:37:11.079253+0000","last_became_peered":"2024-08-17T17:37:11.079253+0000","last_unstale":"2024-08-17T17:38:37.086492+0000","last_undegraded":"2024-08-17T17:38:37.086492+0000","last_fullsized":"2024-08-17T17:38:37.086492+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:53.405657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":914,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.078462+0000","last_change":"2024-08-17T17:35:56.223396+0000","last_active":"2024-08-17T17:38:36.078462+0000","last_peered":"2024-08-17T17:38:36.078462+0000","last_clean":"2024-08-17T17:38:36.078462+0000","last_became_active":"2024-08-17T17:35:56.212413+0000","last_became_peered":"2024-08-17T17:35:56.212413+0000","last_unstale":"2024-08-17T17:38:36.078462+0000","last_undegraded":"2024-08-17T17:38:36.078462+0000","last_fullsized":"2024-08-17T17:38:36.078462+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:44.185174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1628,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081804+0000","last_change":"2024-08-17T17:35:33.136068+0000","last_active":"2024-08-17T17:38:37.081804+0000","last_peered":"2024-08-17T17:38:37.081804+0000","last_clean":"2024-08-17T17:38:37.081804+0000","last_became_active":"2024-08-17T17:35:33.135591+0000","last_became_peered":"2024-08-17T17:35:33.135591+0000","last_unstale":"2024-08-17T17:38:37.081804+0000","last_undegraded":"2024-08-17T17:38:37.081804+0000","last_fullsized":"2024-08-17T17:38:37.081804+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:52.252561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1779,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.082051+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:38:36.082051+0000","last_peered":"2024-08-17T17:38:36.082051+0000","last_clean":"2024-08-17T17:38:36.082051+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:38:36.082051+0000","last_undegraded":"2024-08-17T17:38:36.082051+0000","last_fullsized":"2024-08-17T17:38:36.082051+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"1420'5289","reported_seq":5418,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:40.571664+0000","last_change":"2024-08-17T17:35:14.861823+0000","last_active":"2024-08-17T17:38:40.571664+0000","last_peered":"2024-08-17T17:38:40.571664+0000","last_clean":"2024-08-17T17:38:40.571664+0000","last_became_active":"2024-08-17T17:35:14.854109+0000","last_became_peered":"2024-08-17T17:35:14.854109+0000","last_unstale":"2024-08-17T17:38:40.571664+0000","last_undegraded":"2024-08-17T17:38:40.571664+0000","last_fullsized":"2024-08-17T17:38:40.571664+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5289,"log_dups_size":0,"ondisk_log_size":5289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:46:40.778197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180682752,"num_objects":2757,"num_object_clones":0,"num_object_copies":8271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5289,"num_write_kb":257472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":992,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.077049+0000","last_change":"2024-08-17T17:35:33.138467+0000","last_active":"2024-08-17T17:38:36.077049+0000","last_peered":"2024-08-17T17:38:36.077049+0000","last_clean":"2024-08-17T17:38:36.077049+0000","last_became_active":"2024-08-17T17:35:33.138174+0000","last_became_peered":"2024-08-17T17:35:33.138174+0000","last_unstale":"2024-08-17T17:38:36.077049+0000","last_undegraded":"2024-08-17T17:38:36.077049+0000","last_fullsized":"2024-08-17T17:38:36.077049+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:08.860439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1376,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.083567+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:38:36.083567+0000","last_peered":"2024-08-17T17:38:36.083567+0000","last_clean":"2024-08-17T17:38:36.083567+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:38:36.083567+0000","last_undegraded":"2024-08-17T17:38:36.083567+0000","last_fullsized":"2024-08-17T17:38:36.083567+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":124,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.098191+0000","last_change":"2024-08-17T17:35:14.854881+0000","last_active":"2024-08-17T17:38:37.098191+0000","last_peered":"2024-08-17T17:38:37.098191+0000","last_clean":"2024-08-17T17:38:37.098191+0000","last_became_active":"2024-08-17T17:35:14.854121+0000","last_became_peered":"2024-08-17T17:35:14.854121+0000","last_unstale":"2024-08-17T17:38:37.098191+0000","last_undegraded":"2024-08-17T17:38:37.098191+0000","last_fullsized":"2024-08-17T17:38:37.098191+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.863624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":933,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.078662+0000","last_change":"2024-08-17T17:36:15.773050+0000","last_active":"2024-08-17T17:38:36.078662+0000","last_peered":"2024-08-17T17:38:36.078662+0000","last_clean":"2024-08-17T17:38:36.078662+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:38:36.078662+0000","last_undegraded":"2024-08-17T17:38:36.078662+0000","last_fullsized":"2024-08-17T17:38:36.078662+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:39.212195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:39.212195+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"1420'5309","reported_seq":11144,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:40.499893+0000","last_change":"2024-08-17T17:37:21.409979+0000","last_active":"2024-08-17T17:38:40.499893+0000","last_peered":"2024-08-17T17:38:40.499893+0000","last_clean":"2024-08-17T17:38:40.499893+0000","last_became_active":"2024-08-17T17:37:21.398111+0000","last_became_peered":"2024-08-17T17:37:21.398111+0000","last_unstale":"2024-08-17T17:38:40.499893+0000","last_undegraded":"2024-08-17T17:38:40.499893+0000","last_fullsized":"2024-08-17T17:38:40.499893+0000","mapping_epoch":1407,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5309,"log_dups_size":0,"ondisk_log_size":5309,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:19.299395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178389016,"num_objects":2723,"num_object_clones":0,"num_object_copies":8169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2723,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5309,"num_write_kb":256961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2850,"num_bytes_recovered":168820736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1035,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081374+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:38:37.081374+0000","last_peered":"2024-08-17T17:38:37.081374+0000","last_clean":"2024-08-17T17:38:37.081374+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:38:37.081374+0000","last_undegraded":"2024-08-17T17:38:37.081374+0000","last_fullsized":"2024-08-17T17:38:37.081374+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":218,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.087880+0000","last_change":"2024-08-17T17:35:33.145351+0000","last_active":"2024-08-17T17:38:37.087880+0000","last_peered":"2024-08-17T17:38:37.087880+0000","last_clean":"2024-08-17T17:38:37.087880+0000","last_became_active":"2024-08-17T17:35:33.145040+0000","last_became_peered":"2024-08-17T17:35:33.145040+0000","last_unstale":"2024-08-17T17:38:37.087880+0000","last_undegraded":"2024-08-17T17:38:37.087880+0000","last_fullsized":"2024-08-17T17:38:37.087880+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:02.124000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1444,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.077793+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:38:36.077793+0000","last_peered":"2024-08-17T17:38:36.077793+0000","last_clean":"2024-08-17T17:38:36.077793+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:38:36.077793+0000","last_undegraded":"2024-08-17T17:38:36.077793+0000","last_fullsized":"2024-08-17T17:38:36.077793+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":916,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.078925+0000","last_change":"2024-08-17T17:36:15.769890+0000","last_active":"2024-08-17T17:38:36.078925+0000","last_peered":"2024-08-17T17:38:36.078925+0000","last_clean":"2024-08-17T17:38:36.078925+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:38:36.078925+0000","last_undegraded":"2024-08-17T17:38:36.078925+0000","last_fullsized":"2024-08-17T17:38:36.078925+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:33.034306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:33.034306+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":4,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.b","version":"1420'5265","reported_seq":13060,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:43.514969+0000","last_change":"2024-08-17T17:38:07.888612+0000","last_active":"2024-08-17T17:38:43.514969+0000","last_peered":"2024-08-17T17:38:43.514969+0000","last_clean":"2024-08-17T17:38:43.514969+0000","last_became_active":"2024-08-17T17:38:07.875196+0000","last_became_peered":"2024-08-17T17:38:07.875196+0000","last_unstale":"2024-08-17T17:38:43.514969+0000","last_undegraded":"2024-08-17T17:38:43.514969+0000","last_fullsized":"2024-08-17T17:38:43.514969+0000","mapping_epoch":1415,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5265,"log_dups_size":0,"ondisk_log_size":5265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:21.658004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176226304,"num_objects":2689,"num_object_clones":0,"num_object_copies":8067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5265,"num_write_kb":254528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3836,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1479,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.098342+0000","last_change":"2024-08-17T17:36:12.759743+0000","last_active":"2024-08-17T17:38:37.098342+0000","last_peered":"2024-08-17T17:38:37.098342+0000","last_clean":"2024-08-17T17:38:37.098342+0000","last_became_active":"2024-08-17T17:36:12.759334+0000","last_became_peered":"2024-08-17T17:36:12.759334+0000","last_unstale":"2024-08-17T17:38:37.098342+0000","last_undegraded":"2024-08-17T17:38:37.098342+0000","last_fullsized":"2024-08-17T17:38:37.098342+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:04.524384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1407,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.081436+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:38:36.081436+0000","last_peered":"2024-08-17T17:38:36.081436+0000","last_clean":"2024-08-17T17:38:36.081436+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:38:36.081436+0000","last_undegraded":"2024-08-17T17:38:36.081436+0000","last_fullsized":"2024-08-17T17:38:36.081436+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1450,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.077513+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:38:36.077513+0000","last_peered":"2024-08-17T17:38:36.077513+0000","last_clean":"2024-08-17T17:38:36.077513+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:38:36.077513+0000","last_undegraded":"2024-08-17T17:38:36.077513+0000","last_fullsized":"2024-08-17T17:38:36.077513+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1765,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.076911+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:38:36.076911+0000","last_peered":"2024-08-17T17:38:36.076911+0000","last_clean":"2024-08-17T17:38:36.076911+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:38:36.076911+0000","last_undegraded":"2024-08-17T17:38:36.076911+0000","last_fullsized":"2024-08-17T17:38:36.076911+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"1420'5426","reported_seq":9536,"reported_epoch":1420,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:38:41.415327+0000","last_change":"2024-08-17T17:38:05.991715+0000","last_active":"2024-08-17T17:38:41.415327+0000","last_peered":"2024-08-17T17:38:41.415327+0000","last_clean":"2024-08-17T17:35:54.187688+0000","last_became_active":"2024-08-17T17:36:22.904523+0000","last_became_peered":"2024-08-17T17:36:22.904523+0000","last_unstale":"2024-08-17T17:38:41.415327+0000","last_undegraded":"2024-08-17T17:38:41.415327+0000","last_fullsized":"2024-08-17T17:36:13.740491+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5426,"log_dups_size":0,"ondisk_log_size":5426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:57.525855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181010432,"num_objects":2762,"num_object_clones":0,"num_object_copies":8286,"num_objects_missing_on_primary":1541,"num_objects_missing":1541,"num_objects_degraded":0,"num_objects_misplaced":4171,"num_objects_unfound":0,"num_objects_dirty":2762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5426,"num_write_kb":262016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1338,"num_bytes_recovered":87687168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":1541},{"shards":"2,4,5,6","objects":12},{"shards":"2,5,6","objects":1077},{"shards":"5,6","objects":132}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1083,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.077910+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:38:36.077910+0000","last_peered":"2024-08-17T17:38:36.077910+0000","last_clean":"2024-08-17T17:38:36.077910+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:38:36.077910+0000","last_undegraded":"2024-08-17T17:38:36.077910+0000","last_fullsized":"2024-08-17T17:38:36.077910+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1391,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.097784+0000","last_change":"2024-08-17T17:35:56.229622+0000","last_active":"2024-08-17T17:38:37.097784+0000","last_peered":"2024-08-17T17:38:37.097784+0000","last_clean":"2024-08-17T17:38:37.097784+0000","last_became_active":"2024-08-17T17:35:56.229324+0000","last_became_peered":"2024-08-17T17:35:56.229324+0000","last_unstale":"2024-08-17T17:38:37.097784+0000","last_undegraded":"2024-08-17T17:38:37.097784+0000","last_fullsized":"2024-08-17T17:38:37.097784+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:13.980316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1994,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.098035+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:38:37.098035+0000","last_peered":"2024-08-17T17:38:37.098035+0000","last_clean":"2024-08-17T17:38:37.098035+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:38:37.098035+0000","last_undegraded":"2024-08-17T17:38:37.098035+0000","last_fullsized":"2024-08-17T17:38:37.098035+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1579,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.080440+0000","last_change":"2024-08-17T17:35:33.137074+0000","last_active":"2024-08-17T17:38:36.080440+0000","last_peered":"2024-08-17T17:38:36.080440+0000","last_clean":"2024-08-17T17:38:36.080440+0000","last_became_active":"2024-08-17T17:35:33.136759+0000","last_became_peered":"2024-08-17T17:35:33.136759+0000","last_unstale":"2024-08-17T17:38:36.080440+0000","last_undegraded":"2024-08-17T17:38:36.080440+0000","last_fullsized":"2024-08-17T17:38:36.080440+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:37.263639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:37.263639+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:14.599780+0000","scrub_duration":0.0016181800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"1420'5385","reported_seq":8795,"reported_epoch":1420,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:40.608490+0000","last_change":"2024-08-17T17:36:23.019129+0000","last_active":"2024-08-17T17:38:40.608490+0000","last_peered":"2024-08-17T17:38:40.608490+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:23.013527+0000","last_became_peered":"2024-08-17T17:36:23.013527+0000","last_unstale":"2024-08-17T17:38:40.608490+0000","last_undegraded":"2024-08-17T17:36:13.874715+0000","last_fullsized":"2024-08-17T17:36:13.738828+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5385,"log_dups_size":0,"ondisk_log_size":5385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177930240,"num_objects":2715,"num_object_clones":0,"num_object_copies":8145,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4865,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2715,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5385,"num_write_kb":259200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1336,"num_bytes_recovered":87556096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[0,6],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":1540},{"shards":"0,1,6","objects":1048},{"shards":"0,6","objects":-610}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1183,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.082260+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:38:37.082260+0000","last_peered":"2024-08-17T17:38:37.082260+0000","last_clean":"2024-08-17T17:38:37.082260+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:38:37.082260+0000","last_undegraded":"2024-08-17T17:38:37.082260+0000","last_fullsized":"2024-08-17T17:38:37.082260+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":44,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.080134+0000","last_change":"2024-08-17T17:36:17.774773+0000","last_active":"2024-08-17T17:38:36.080134+0000","last_peered":"2024-08-17T17:38:36.080134+0000","last_clean":"2024-08-17T17:38:36.080134+0000","last_became_active":"2024-08-17T17:36:17.774468+0000","last_became_peered":"2024-08-17T17:36:17.774468+0000","last_unstale":"2024-08-17T17:38:36.080134+0000","last_undegraded":"2024-08-17T17:38:36.080134+0000","last_fullsized":"2024-08-17T17:38:36.080134+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:54.231747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1521,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081227+0000","last_change":"2024-08-17T17:35:33.137633+0000","last_active":"2024-08-17T17:38:37.081227+0000","last_peered":"2024-08-17T17:38:37.081227+0000","last_clean":"2024-08-17T17:38:37.081227+0000","last_became_active":"2024-08-17T17:35:33.137323+0000","last_became_peered":"2024-08-17T17:35:33.137323+0000","last_unstale":"2024-08-17T17:38:37.081227+0000","last_undegraded":"2024-08-17T17:38:37.081227+0000","last_fullsized":"2024-08-17T17:38:37.081227+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:03.359155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":895,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.077510+0000","last_change":"2024-08-17T17:35:56.264626+0000","last_active":"2024-08-17T17:38:36.077510+0000","last_peered":"2024-08-17T17:38:36.077510+0000","last_clean":"2024-08-17T17:38:36.077510+0000","last_became_active":"2024-08-17T17:35:56.264265+0000","last_became_peered":"2024-08-17T17:35:56.264265+0000","last_unstale":"2024-08-17T17:38:36.077510+0000","last_undegraded":"2024-08-17T17:38:36.077510+0000","last_fullsized":"2024-08-17T17:38:36.077510+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:20.754326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:20.754326+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"1420'5247","reported_seq":9049,"reported_epoch":1420,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:41.437910+0000","last_change":"2024-08-17T17:37:11.134558+0000","last_active":"2024-08-17T17:38:41.437910+0000","last_peered":"2024-08-17T17:38:41.437910+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:20.105110+0000","last_became_peered":"2024-08-17T17:36:20.105110+0000","last_unstale":"2024-08-17T17:38:41.437910+0000","last_undegraded":"2024-08-17T17:36:19.999272+0000","last_fullsized":"2024-08-17T17:36:13.740520+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5247,"log_dups_size":0,"ondisk_log_size":5247,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:45.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181338112,"num_objects":2767,"num_object_clones":0,"num_object_copies":8301,"num_objects_missing_on_primary":1576,"num_objects_missing":1576,"num_objects_degraded":4188,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2767,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5247,"num_write_kb":256448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1241,"num_bytes_recovered":81330176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1576},{"shards":"1,5,6","objects":1036},{"shards":"5,6","objects":155}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1248,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.078371+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:38:36.078371+0000","last_peered":"2024-08-17T17:38:36.078371+0000","last_clean":"2024-08-17T17:38:36.078371+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:38:36.078371+0000","last_undegraded":"2024-08-17T17:38:36.078371+0000","last_fullsized":"2024-08-17T17:38:36.078371+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1787,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.077435+0000","last_change":"2024-08-17T17:36:12.758005+0000","last_active":"2024-08-17T17:38:36.077435+0000","last_peered":"2024-08-17T17:38:36.077435+0000","last_clean":"2024-08-17T17:38:36.077435+0000","last_became_active":"2024-08-17T17:36:12.757695+0000","last_became_peered":"2024-08-17T17:36:12.757695+0000","last_unstale":"2024-08-17T17:38:36.077435+0000","last_undegraded":"2024-08-17T17:38:36.077435+0000","last_fullsized":"2024-08-17T17:38:36.077435+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:51.792220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1426,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.077083+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:38:36.077083+0000","last_peered":"2024-08-17T17:38:36.077083+0000","last_clean":"2024-08-17T17:38:36.077083+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:38:36.077083+0000","last_undegraded":"2024-08-17T17:38:36.077083+0000","last_fullsized":"2024-08-17T17:38:36.077083+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1719,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081337+0000","last_change":"2024-08-17T17:38:37.081337+0000","last_active":"2024-08-17T17:38:37.081337+0000","last_peered":"2024-08-17T17:38:37.081337+0000","last_clean":"2024-08-17T17:38:37.081337+0000","last_became_active":"2024-08-17T17:35:55.204861+0000","last_became_peered":"2024-08-17T17:35:55.204861+0000","last_unstale":"2024-08-17T17:38:37.081337+0000","last_undegraded":"2024-08-17T17:38:37.081337+0000","last_fullsized":"2024-08-17T17:38:37.081337+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:38:41.076782+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.0","version":"1420'5395","reported_seq":13378,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:42.626236+0000","last_change":"2024-08-17T17:37:40.753703+0000","last_active":"2024-08-17T17:38:42.626236+0000","last_peered":"2024-08-17T17:38:42.626236+0000","last_clean":"2024-08-17T17:38:42.626236+0000","last_became_active":"2024-08-17T17:37:40.743458+0000","last_became_peered":"2024-08-17T17:37:40.743458+0000","last_unstale":"2024-08-17T17:38:42.626236+0000","last_undegraded":"2024-08-17T17:38:42.626236+0000","last_fullsized":"2024-08-17T17:38:42.626236+0000","mapping_epoch":1411,"log_start":"1373'100","ondisk_log_start":"1373'100","created":1357,"last_epoch_clean":1412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5295,"log_dups_size":0,"ondisk_log_size":5295,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:14:02.340380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178323456,"num_objects":2721,"num_object_clones":0,"num_object_copies":8163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5395,"num_write_kb":259712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3914,"num_bytes_recovered":232652800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1021,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.082493+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:38:36.082493+0000","last_peered":"2024-08-17T17:38:36.082493+0000","last_clean":"2024-08-17T17:38:36.082493+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:38:36.082493+0000","last_undegraded":"2024-08-17T17:38:36.082493+0000","last_fullsized":"2024-08-17T17:38:36.082493+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":87,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.086637+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:38:37.086637+0000","last_peered":"2024-08-17T17:38:37.086637+0000","last_clean":"2024-08-17T17:38:37.086637+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:38:37.086637+0000","last_undegraded":"2024-08-17T17:38:37.086637+0000","last_fullsized":"2024-08-17T17:38:37.086637+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1757,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.076878+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:38:36.076878+0000","last_peered":"2024-08-17T17:38:36.076878+0000","last_clean":"2024-08-17T17:38:36.076878+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:38:36.076878+0000","last_undegraded":"2024-08-17T17:38:36.076878+0000","last_fullsized":"2024-08-17T17:38:36.076878+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1","version":"1420'5314","reported_seq":9270,"reported_epoch":1420,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:41.164170+0000","last_change":"2024-08-17T17:36:23.103523+0000","last_active":"2024-08-17T17:38:41.164170+0000","last_peered":"2024-08-17T17:38:41.164170+0000","last_clean":"2024-08-17T17:35:54.178280+0000","last_became_active":"2024-08-17T17:36:23.025910+0000","last_became_peered":"2024-08-17T17:36:23.025910+0000","last_unstale":"2024-08-17T17:38:41.164170+0000","last_undegraded":"2024-08-17T17:36:18.734637+0000","last_fullsized":"2024-08-17T17:36:13.740134+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5314,"log_dups_size":0,"ondisk_log_size":5314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:58.717055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179437568,"num_objects":2738,"num_object_clones":0,"num_object_copies":8214,"num_objects_missing_on_primary":1543,"num_objects_missing":1543,"num_objects_degraded":4157,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2738,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5314,"num_write_kb":257664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1290,"num_bytes_recovered":84541440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,0],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":1543},{"shards":"0,2,4,6","objects":1},{"shards":"0,2,6","objects":1070},{"shards":"0,6","objects":124}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1435,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.081739+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:38:36.081739+0000","last_peered":"2024-08-17T17:38:36.081739+0000","last_clean":"2024-08-17T17:38:36.081739+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:38:36.081739+0000","last_undegraded":"2024-08-17T17:38:36.081739+0000","last_fullsized":"2024-08-17T17:38:36.081739+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1408,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.080692+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:38:36.080692+0000","last_peered":"2024-08-17T17:38:36.080692+0000","last_clean":"2024-08-17T17:38:36.080692+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:38:36.080692+0000","last_undegraded":"2024-08-17T17:38:36.080692+0000","last_fullsized":"2024-08-17T17:38:36.080692+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":988,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081618+0000","last_change":"2024-08-17T17:34:26.817339+0000","last_active":"2024-08-17T17:38:37.081618+0000","last_peered":"2024-08-17T17:38:37.081618+0000","last_clean":"2024-08-17T17:38:37.081618+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:38:37.081618+0000","last_undegraded":"2024-08-17T17:38:37.081618+0000","last_fullsized":"2024-08-17T17:38:37.081618+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:26.817223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:26.817223+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:30.089956+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"1420'5230","reported_seq":5384,"reported_epoch":1420,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:40.583417+0000","last_change":"2024-08-17T17:36:14.073155+0000","last_active":"2024-08-17T17:38:40.583417+0000","last_peered":"2024-08-17T17:38:40.583417+0000","last_clean":"2024-08-17T17:36:11.734683+0000","last_became_active":"2024-08-17T17:36:14.063794+0000","last_became_peered":"2024-08-17T17:36:14.063794+0000","last_unstale":"2024-08-17T17:38:40.583417+0000","last_undegraded":"2024-08-17T17:36:13.855770+0000","last_fullsized":"2024-08-17T17:36:13.742068+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5230,"log_dups_size":0,"ondisk_log_size":5230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175112192,"num_objects":2672,"num_object_clones":0,"num_object_copies":8016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3819,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5230,"num_write_kb":252864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-1147},{"shards":"0,2,7","objects":2540}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":44,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.081915+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:38:36.081915+0000","last_peered":"2024-08-17T17:38:36.081915+0000","last_clean":"2024-08-17T17:38:36.081915+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:38:36.081915+0000","last_undegraded":"2024-08-17T17:38:36.081915+0000","last_fullsized":"2024-08-17T17:38:36.081915+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1270,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.093590+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:38:36.093590+0000","last_peered":"2024-08-17T17:38:36.093590+0000","last_clean":"2024-08-17T17:38:36.093590+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:38:36.093590+0000","last_undegraded":"2024-08-17T17:38:36.093590+0000","last_fullsized":"2024-08-17T17:38:36.093590+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1785,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.076770+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:38:36.076770+0000","last_peered":"2024-08-17T17:38:36.076770+0000","last_clean":"2024-08-17T17:38:36.076770+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:38:36.076770+0000","last_undegraded":"2024-08-17T17:38:36.076770+0000","last_fullsized":"2024-08-17T17:38:36.076770+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"1420'5324","reported_seq":11118,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:41.395653+0000","last_change":"2024-08-17T17:36:46.982456+0000","last_active":"2024-08-17T17:38:41.395653+0000","last_peered":"2024-08-17T17:38:41.395653+0000","last_clean":"2024-08-17T17:38:41.395653+0000","last_became_active":"2024-08-17T17:36:46.974782+0000","last_became_peered":"2024-08-17T17:36:46.974782+0000","last_unstale":"2024-08-17T17:38:41.395653+0000","last_undegraded":"2024-08-17T17:38:41.395653+0000","last_fullsized":"2024-08-17T17:38:41.395653+0000","mapping_epoch":1399,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5324,"log_dups_size":0,"ondisk_log_size":5324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:40.499321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180092928,"num_objects":2748,"num_object_clones":0,"num_object_copies":8244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5324,"num_write_kb":258304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2848,"num_bytes_recovered":186646528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1387,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085399+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:38:37.085399+0000","last_peered":"2024-08-17T17:38:37.085399+0000","last_clean":"2024-08-17T17:38:37.085399+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:38:37.085399+0000","last_undegraded":"2024-08-17T17:38:37.085399+0000","last_fullsized":"2024-08-17T17:38:37.085399+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":200,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.081012+0000","last_change":"2024-08-17T17:36:12.754372+0000","last_active":"2024-08-17T17:38:36.081012+0000","last_peered":"2024-08-17T17:38:36.081012+0000","last_clean":"2024-08-17T17:38:36.081012+0000","last_became_active":"2024-08-17T17:36:12.746352+0000","last_became_peered":"2024-08-17T17:36:12.746352+0000","last_unstale":"2024-08-17T17:38:36.081012+0000","last_undegraded":"2024-08-17T17:38:36.081012+0000","last_fullsized":"2024-08-17T17:38:36.081012+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:07.807486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1660,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.097832+0000","last_change":"2024-08-17T17:36:15.771193+0000","last_active":"2024-08-17T17:38:37.097832+0000","last_peered":"2024-08-17T17:38:37.097832+0000","last_clean":"2024-08-17T17:38:37.097832+0000","last_became_active":"2024-08-17T17:36:15.770889+0000","last_became_peered":"2024-08-17T17:36:15.770889+0000","last_unstale":"2024-08-17T17:38:37.097832+0000","last_undegraded":"2024-08-17T17:38:37.097832+0000","last_fullsized":"2024-08-17T17:38:37.097832+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:22.978695+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.4","version":"1420'5301","reported_seq":11076,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:41.328108+0000","last_change":"2024-08-17T17:37:04.045576+0000","last_active":"2024-08-17T17:38:41.328108+0000","last_peered":"2024-08-17T17:38:41.328108+0000","last_clean":"2024-08-17T17:38:41.328108+0000","last_became_active":"2024-08-17T17:37:04.038810+0000","last_became_peered":"2024-08-17T17:37:04.038810+0000","last_unstale":"2024-08-17T17:38:41.328108+0000","last_undegraded":"2024-08-17T17:38:41.328108+0000","last_fullsized":"2024-08-17T17:38:41.328108+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5301,"log_dups_size":0,"ondisk_log_size":5301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:27:04.725258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181207040,"num_objects":2765,"num_object_clones":0,"num_object_copies":8295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5301,"num_write_kb":258112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":176291840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1255,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081930+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:38:37.081930+0000","last_peered":"2024-08-17T17:38:37.081930+0000","last_clean":"2024-08-17T17:38:37.081930+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:38:37.081930+0000","last_undegraded":"2024-08-17T17:38:37.081930+0000","last_fullsized":"2024-08-17T17:38:37.081930+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1413,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081704+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:38:37.081704+0000","last_peered":"2024-08-17T17:38:37.081704+0000","last_clean":"2024-08-17T17:38:37.081704+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:38:37.081704+0000","last_undegraded":"2024-08-17T17:38:37.081704+0000","last_fullsized":"2024-08-17T17:38:37.081704+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":913,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.081598+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:38:36.081598+0000","last_peered":"2024-08-17T17:38:36.081598+0000","last_clean":"2024-08-17T17:38:36.081598+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:38:36.081598+0000","last_undegraded":"2024-08-17T17:38:36.081598+0000","last_fullsized":"2024-08-17T17:38:36.081598+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"1420'5391","reported_seq":11221,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:41.810590+0000","last_change":"2024-08-17T17:36:32.711490+0000","last_active":"2024-08-17T17:38:41.810590+0000","last_peered":"2024-08-17T17:38:41.810590+0000","last_clean":"2024-08-17T17:38:41.810590+0000","last_became_active":"2024-08-17T17:36:32.704370+0000","last_became_peered":"2024-08-17T17:36:32.704370+0000","last_unstale":"2024-08-17T17:38:41.810590+0000","last_undegraded":"2024-08-17T17:38:41.810590+0000","last_fullsized":"2024-08-17T17:38:41.810590+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5391,"log_dups_size":0,"ondisk_log_size":5391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:17.373879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178061312,"num_objects":2717,"num_object_clones":0,"num_object_copies":8151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5391,"num_write_kb":259456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2866,"num_bytes_recovered":187826176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1269,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.110651+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:38:36.110651+0000","last_peered":"2024-08-17T17:38:36.110651+0000","last_clean":"2024-08-17T17:38:36.110651+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:38:36.110651+0000","last_undegraded":"2024-08-17T17:38:36.110651+0000","last_fullsized":"2024-08-17T17:38:36.110651+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":149,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.077546+0000","last_change":"2024-08-17T17:35:55.217072+0000","last_active":"2024-08-17T17:38:36.077546+0000","last_peered":"2024-08-17T17:38:36.077546+0000","last_clean":"2024-08-17T17:38:36.077546+0000","last_became_active":"2024-08-17T17:35:55.216462+0000","last_became_peered":"2024-08-17T17:35:55.216462+0000","last_unstale":"2024-08-17T17:38:36.077546+0000","last_undegraded":"2024-08-17T17:38:36.077546+0000","last_fullsized":"2024-08-17T17:38:36.077546+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:34.937221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":896,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.079085+0000","last_change":"2024-08-17T17:35:55.215004+0000","last_active":"2024-08-17T17:38:36.079085+0000","last_peered":"2024-08-17T17:38:36.079085+0000","last_clean":"2024-08-17T17:38:36.079085+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:38:36.079085+0000","last_undegraded":"2024-08-17T17:38:36.079085+0000","last_fullsized":"2024-08-17T17:38:36.079085+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.6","version":"1420'5218","reported_seq":5348,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:40.515427+0000","last_change":"2024-08-17T17:35:14.868524+0000","last_active":"2024-08-17T17:38:40.515427+0000","last_peered":"2024-08-17T17:38:40.515427+0000","last_clean":"2024-08-17T17:38:40.515427+0000","last_became_active":"2024-08-17T17:35:14.854690+0000","last_became_peered":"2024-08-17T17:35:14.854690+0000","last_unstale":"2024-08-17T17:38:40.515427+0000","last_undegraded":"2024-08-17T17:38:40.515427+0000","last_fullsized":"2024-08-17T17:38:40.515427+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5218,"log_dups_size":0,"ondisk_log_size":5218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:05.388362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170262528,"num_objects":2598,"num_object_clones":0,"num_object_copies":7794,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2598,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5218,"num_write_kb":250112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1454,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.080548+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:38:36.080548+0000","last_peered":"2024-08-17T17:38:36.080548+0000","last_clean":"2024-08-17T17:38:36.080548+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:38:36.080548+0000","last_undegraded":"2024-08-17T17:38:36.080548+0000","last_fullsized":"2024-08-17T17:38:36.080548+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":161,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.081785+0000","last_change":"2024-08-17T17:35:33.147672+0000","last_active":"2024-08-17T17:38:36.081785+0000","last_peered":"2024-08-17T17:38:36.081785+0000","last_clean":"2024-08-17T17:38:36.081785+0000","last_became_active":"2024-08-17T17:35:33.147283+0000","last_became_peered":"2024-08-17T17:35:33.147283+0000","last_unstale":"2024-08-17T17:38:36.081785+0000","last_undegraded":"2024-08-17T17:38:36.081785+0000","last_fullsized":"2024-08-17T17:38:36.081785+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:41.677709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1654,"reported_epoch":1420,"state":"active+clean+scrubbing","last_fresh":"2024-08-17T17:38:37.084774+0000","last_change":"2024-08-17T17:38:40.861458+0000","last_active":"2024-08-17T17:38:37.084774+0000","last_peered":"2024-08-17T17:38:37.084774+0000","last_clean":"2024-08-17T17:38:37.084774+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:38:37.084774+0000","last_undegraded":"2024-08-17T17:38:37.084774+0000","last_fullsized":"2024-08-17T17:38:37.084774+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:38.241235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:38.241235+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"1420'5241","reported_seq":5313,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:42.360061+0000","last_change":"2024-08-17T17:34:50.670013+0000","last_active":"2024-08-17T17:38:42.360061+0000","last_peered":"2024-08-17T17:38:42.360061+0000","last_clean":"2024-08-17T17:38:42.360061+0000","last_became_active":"2024-08-17T17:34:50.669299+0000","last_became_peered":"2024-08-17T17:34:50.669299+0000","last_unstale":"2024-08-17T17:38:42.360061+0000","last_undegraded":"2024-08-17T17:38:42.360061+0000","last_fullsized":"2024-08-17T17:38:42.360061+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5241,"log_dups_size":0,"ondisk_log_size":5241,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:11.172227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175964160,"num_objects":2685,"num_object_clones":0,"num_object_copies":8055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5241,"num_write_kb":253632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1378,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.076850+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:38:36.076850+0000","last_peered":"2024-08-17T17:38:36.076850+0000","last_clean":"2024-08-17T17:38:36.076850+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:38:36.076850+0000","last_undegraded":"2024-08-17T17:38:36.076850+0000","last_fullsized":"2024-08-17T17:38:36.076850+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":148,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.082631+0000","last_change":"2024-08-17T17:35:33.140725+0000","last_active":"2024-08-17T17:38:36.082631+0000","last_peered":"2024-08-17T17:38:36.082631+0000","last_clean":"2024-08-17T17:38:36.082631+0000","last_became_active":"2024-08-17T17:35:33.140406+0000","last_became_peered":"2024-08-17T17:35:33.140406+0000","last_unstale":"2024-08-17T17:38:36.082631+0000","last_undegraded":"2024-08-17T17:38:36.082631+0000","last_fullsized":"2024-08-17T17:38:36.082631+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:54:51.875571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1668,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.081718+0000","last_change":"2024-08-17T17:35:33.138514+0000","last_active":"2024-08-17T17:38:36.081718+0000","last_peered":"2024-08-17T17:38:36.081718+0000","last_clean":"2024-08-17T17:38:36.081718+0000","last_became_active":"2024-08-17T17:35:33.137035+0000","last_became_peered":"2024-08-17T17:35:33.137035+0000","last_unstale":"2024-08-17T17:38:36.081718+0000","last_undegraded":"2024-08-17T17:38:36.081718+0000","last_fullsized":"2024-08-17T17:38:36.081718+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:50.565662+0000","scrub_duration":0.001487138,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1310,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.079489+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:38:36.079489+0000","last_peered":"2024-08-17T17:38:36.079489+0000","last_clean":"2024-08-17T17:38:36.079489+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:38:36.079489+0000","last_undegraded":"2024-08-17T17:38:36.079489+0000","last_fullsized":"2024-08-17T17:38:36.079489+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":87,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085729+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:38:37.085729+0000","last_peered":"2024-08-17T17:38:37.085729+0000","last_clean":"2024-08-17T17:38:37.085729+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:38:37.085729+0000","last_undegraded":"2024-08-17T17:38:37.085729+0000","last_fullsized":"2024-08-17T17:38:37.085729+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1741,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.077183+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:38:36.077183+0000","last_peered":"2024-08-17T17:38:36.077183+0000","last_clean":"2024-08-17T17:38:36.077183+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:38:36.077183+0000","last_undegraded":"2024-08-17T17:38:36.077183+0000","last_fullsized":"2024-08-17T17:38:36.077183+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1327,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.080420+0000","last_change":"2024-08-17T17:35:55.205292+0000","last_active":"2024-08-17T17:38:36.080420+0000","last_peered":"2024-08-17T17:38:36.080420+0000","last_clean":"2024-08-17T17:38:36.080420+0000","last_became_active":"2024-08-17T17:35:55.204935+0000","last_became_peered":"2024-08-17T17:35:55.204935+0000","last_unstale":"2024-08-17T17:38:36.080420+0000","last_undegraded":"2024-08-17T17:38:36.080420+0000","last_fullsized":"2024-08-17T17:38:36.080420+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:52.277821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":87,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.086124+0000","last_change":"2024-08-17T17:36:12.749972+0000","last_active":"2024-08-17T17:38:37.086124+0000","last_peered":"2024-08-17T17:38:37.086124+0000","last_clean":"2024-08-17T17:38:37.086124+0000","last_became_active":"2024-08-17T17:36:12.749620+0000","last_became_peered":"2024-08-17T17:36:12.749620+0000","last_unstale":"2024-08-17T17:38:37.086124+0000","last_undegraded":"2024-08-17T17:38:37.086124+0000","last_fullsized":"2024-08-17T17:38:37.086124+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:04.994357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":963,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085450+0000","last_change":"2024-08-17T17:36:20.832274+0000","last_active":"2024-08-17T17:38:37.085450+0000","last_peered":"2024-08-17T17:38:37.085450+0000","last_clean":"2024-08-17T17:38:37.085450+0000","last_became_active":"2024-08-17T17:36:20.831886+0000","last_became_peered":"2024-08-17T17:36:20.831886+0000","last_unstale":"2024-08-17T17:38:37.085450+0000","last_undegraded":"2024-08-17T17:38:37.085450+0000","last_fullsized":"2024-08-17T17:38:37.085450+0000","mapping_epoch":1390,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1391,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:05.248711+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1172,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081549+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:38:37.081549+0000","last_peered":"2024-08-17T17:38:37.081549+0000","last_clean":"2024-08-17T17:38:37.081549+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:38:37.081549+0000","last_undegraded":"2024-08-17T17:38:37.081549+0000","last_fullsized":"2024-08-17T17:38:37.081549+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1410,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.077621+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:38:36.077621+0000","last_peered":"2024-08-17T17:38:36.077621+0000","last_clean":"2024-08-17T17:38:36.077621+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:38:36.077621+0000","last_undegraded":"2024-08-17T17:38:36.077621+0000","last_fullsized":"2024-08-17T17:38:36.077621+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1704,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.076665+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:38:36.076665+0000","last_peered":"2024-08-17T17:38:36.076665+0000","last_clean":"2024-08-17T17:38:36.076665+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:38:36.076665+0000","last_undegraded":"2024-08-17T17:38:36.076665+0000","last_fullsized":"2024-08-17T17:38:36.076665+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1352,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.082951+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:38:36.082951+0000","last_peered":"2024-08-17T17:38:36.082951+0000","last_clean":"2024-08-17T17:38:36.082951+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:38:36.082951+0000","last_undegraded":"2024-08-17T17:38:36.082951+0000","last_fullsized":"2024-08-17T17:38:36.082951+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1891,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084902+0000","last_change":"2024-08-17T17:35:33.137411+0000","last_active":"2024-08-17T17:38:37.084902+0000","last_peered":"2024-08-17T17:38:37.084902+0000","last_clean":"2024-08-17T17:38:37.084902+0000","last_became_active":"2024-08-17T17:35:33.137073+0000","last_became_peered":"2024-08-17T17:35:33.137073+0000","last_unstale":"2024-08-17T17:38:37.084902+0000","last_undegraded":"2024-08-17T17:38:37.084902+0000","last_fullsized":"2024-08-17T17:38:37.084902+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:35.924539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1798,"reported_epoch":1419,"state":"active+clean","last_fresh":"2024-08-17T17:38:36.076802+0000","last_change":"2024-08-17T17:36:15.787445+0000","last_active":"2024-08-17T17:38:36.076802+0000","last_peered":"2024-08-17T17:38:36.076802+0000","last_clean":"2024-08-17T17:38:36.076802+0000","last_became_active":"2024-08-17T17:36:15.787099+0000","last_became_peered":"2024-08-17T17:36:15.787099+0000","last_unstale":"2024-08-17T17:38:36.076802+0000","last_undegraded":"2024-08-17T17:38:36.076802+0000","last_fullsized":"2024-08-17T17:38:36.076802+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:47.448168+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":2859859992,"num_objects":43639,"num_object_clones":0,"num_object_copies":130917,"num_objects_missing_on_primary":4660,"num_objects_missing":4660,"num_objects_degraded":17029,"num_objects_misplaced":4171,"num_objects_unfound":0,"num_objects_dirty":43639,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":85243,"num_write_kb":4124161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28379,"num_bytes_recovered":1707212800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9222893568,"data_stored":9222881352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85143,"ondisk_log_size":85143,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635092,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":27,"num_read_kb":61,"num_write":15,"num_write_kb":183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1314910,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162855,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1460872,"kb_used_data":1310208,"kb_used_omap":54,"kb_used_meta":150601,"kb_avail":92284280,"statfs":{"total":95995035648,"available":94499102720,"internally_reserved":0,"allocated":1341652992,"data_stored":1338986802,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":154215445},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999329,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1235784,"kb_used_data":1085952,"kb_used_omap":47,"kb_used_meta":149776,"kb_avail":92509368,"statfs":{"total":95995035648,"available":94729592832,"internally_reserved":0,"allocated":1112014848,"data_stored":1109349635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":153371284},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627126,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1404396,"kb_used_data":397668,"kb_used_omap":42,"kb_used_meta":1006677,"kb_avail":92340756,"statfs":{"total":95995035648,"available":94556934144,"internally_reserved":0,"allocated":407212032,"data_stored":404569100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1030837308},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785603,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2702192,"kb_used_data":1898024,"kb_used_omap":48,"kb_used_meta":804111,"kb_avail":91042960,"statfs":{"total":95995035648,"available":93227991040,"internally_reserved":0,"allocated":1943576576,"data_stored":1940930519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":823410350},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725120,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1568952,"kb_used_data":707696,"kb_used_omap":1,"kb_used_meta":861246,"kb_avail":92176200,"statfs":{"total":95995035648,"available":94388428800,"internally_reserved":0,"allocated":724680704,"data_stored":722026597,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":881916358},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986757,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1952708,"kb_used_data":1073404,"kb_used_omap":1,"kb_used_meta":879294,"kb_avail":91792444,"statfs":{"total":95995035648,"available":93995462656,"internally_reserved":0,"allocated":1099165696,"data_stored":1096503012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":900397510},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706364,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2457976,"kb_used_data":1559664,"kb_used_omap":35,"kb_used_meta":898268,"kb_avail":91287176,"statfs":{"total":95995035648,"available":93478068224,"internally_reserved":0,"allocated":1597095936,"data_stored":1594441810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":919826487},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832137,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2089204,"kb_used_data":1054444,"kb_used_omap":65,"kb_used_meta":1034686,"kb_avail":91655948,"statfs":{"total":95995035648,"available":93855690752,"internally_reserved":0,"allocated":1079750656,"data_stored":1077101088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1059518651},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1068630016,"data_stored":1068630016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1932918784,"data_stored":1932918784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1584467968,"data_stored":1584463896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":397541376,"data_stored":397541376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1090260992,"data_stored":1090256920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":714670080,"data_stored":714670080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1100943360,"data_stored":1100939288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1333460992,"data_stored":1333460992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:38:47.512 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:47.539 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:47.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:47.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:47 smithi162 ceph-mon[104668]: from='client.36378 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:38:47.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:47 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:47.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:47 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:38:47.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:47 smithi162 ceph-mon[104668]: osdmap e1421: 8 total, 8 up, 8 in 2024-08-17T17:38:47.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:38:47.597 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:38:47.641 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:47.668 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:47.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:47.769 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:47.793 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:47.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:47.895 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:47.917 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:47.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:47.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:48.019 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:48.044 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:48.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.045 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:48.146 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:48.174 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:48.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:48.263 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:48 smithi063 ceph-mon[114756]: pgmap v2227: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 192 op/s; 16665/128937 objects degraded (12.925%); 4044/128937 objects misplaced (3.136%); 3.1 MiB/s, 54 objects/s recovering 2024-08-17T17:38:48.264 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:48 smithi063 ceph-mon[109843]: pgmap v2227: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 192 op/s; 16665/128937 objects degraded (12.925%); 4044/128937 objects misplaced (3.136%); 3.1 MiB/s, 54 objects/s recovering 2024-08-17T17:38:48.270 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:48.276 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:48.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:48.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.300 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:48.400 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:48.423 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:48.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:48.525 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:48.551 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:48.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:48.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:48 smithi162 ceph-mon[104668]: pgmap v2227: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 192 op/s; 16665/128937 objects degraded (12.925%); 4044/128937 objects misplaced (3.136%); 3.1 MiB/s, 54 objects/s recovering 2024-08-17T17:38:48.653 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:48.676 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:48.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:48.777 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:48.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.809 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:48.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:48.843 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:38:48.910 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:48.933 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:48.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:48.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:49.035 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:49.062 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:49.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:49.164 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:49.180 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:38:49.180 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:38:49.180 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:49.180 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_repop(client.46088.0:86759 11.9 e1421/1383 11:964c20a2:::benchmark_data_smithi063_328699_object22316:head v 1421'5486, mlcod=1373'1435)", 2024-08-17T17:38:49.180 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:38:49.113036+0000", 2024-08-17T17:38:49.180 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 0.0018214290000000001, 2024-08-17T17:38:49.181 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.001855635, 2024-08-17T17:38:49.181 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:38:49.181 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:38:49.181 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "started", 2024-08-17T17:38:49.181 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:38:49.181 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:49.181 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:38:49.181 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:49.113036+0000", 2024-08-17T17:38:49.181 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:49.181 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:49.181 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:49.181 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:38:49.181 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:49.113036+0000", 2024-08-17T17:38:49.182 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:38:49.182 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:49.182 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:49.182 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:38:49.182 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:49.113038+0000", 2024-08-17T17:38:49.182 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.203e-06 2024-08-17T17:38:49.182 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:49.182 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:49.182 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:38:49.183 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:49.113047+0000", 2024-08-17T17:38:49.183 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.9069999999999996e-06 2024-08-17T17:38:49.183 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:49.183 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:49.183 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:38:49.183 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:49.113069+0000", 2024-08-17T17:38:49.183 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2600999999999999e-05 2024-08-17T17:38:49.183 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:49.183 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:49.184 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:38:49.184 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:49.113078+0000", 2024-08-17T17:38:49.184 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6570000000000003e-06 2024-08-17T17:38:49.184 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:49.184 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:49.184 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:38:49.184 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:49.113176+0000", 2024-08-17T17:38:49.184 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7886000000000004e-05 2024-08-17T17:38:49.184 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:38:49.184 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:38:49.184 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:38:49.184 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:49.113207+0000", 2024-08-17T17:38:49.184 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.1618000000000001e-05 2024-08-17T17:38:49.185 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:49.185 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:38:49.185 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:49.185 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:38:49.185 INFO:teuthology.orchestra.run.smithi063.stdout: ], 2024-08-17T17:38:49.185 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 1 2024-08-17T17:38:49.185 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:38:49.187 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:49.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:49.289 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:49.313 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:49.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:49.415 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:49.440 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:49.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:49.501 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:49 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 16665/128937 objects degraded (12.925%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:49.501 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:49 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 16665/128937 objects degraded (12.925%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:49.541 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:49.566 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:49.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:49.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:49 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 16665/128937 objects degraded (12.925%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:49.668 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:49.691 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:49.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:49.792 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:49.815 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:49.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:49.917 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:49.944 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:49.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:49.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:49.979 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:38:50.046 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:50.068 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:50.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:50.170 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:50.196 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:50.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:50 smithi063 ceph-mon[114756]: pgmap v2228: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 192 op/s; 16665/128142 objects degraded (13.005%); 4044/128142 objects misplaced (3.156%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T17:38:50.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:50 smithi063 ceph-mon[109843]: pgmap v2228: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 192 op/s; 16665/128142 objects degraded (13.005%); 4044/128142 objects misplaced (3.156%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T17:38:50.298 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:50.320 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:50.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:50.422 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:50.426 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:38:50.447 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:50.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:50.549 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:50.572 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:50.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:50.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:50 smithi162 ceph-mon[104668]: pgmap v2228: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 192 op/s; 16665/128142 objects degraded (13.005%); 4044/128142 objects misplaced (3.156%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T17:38:50.674 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:50.699 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:50.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:50.801 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:50.827 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:50.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:50.858 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:50.929 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:50.952 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:50.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:50.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:51.053 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:51.076 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:51.177 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:51.201 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:51.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:51.303 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:51.326 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:51.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:51.428 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:51.451 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:51.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:51.553 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:51.576 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:51.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:51.678 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:51.703 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:51.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:51.805 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:51.835 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:51.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:51.936 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:51.965 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:51.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:51.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:52.067 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:52.090 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:52.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:52.196 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:52.218 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:52.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:52.320 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:52.343 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:52.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:52.396 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:52 smithi063 ceph-mon[114756]: pgmap v2229: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 191 op/s; 16665/127968 objects degraded (13.023%); 4044/127968 objects misplaced (3.160%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T17:38:52.396 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:52 smithi063 ceph-mon[109843]: pgmap v2229: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 191 op/s; 16665/127968 objects degraded (13.023%); 4044/127968 objects misplaced (3.160%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T17:38:52.445 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:52.468 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:52.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:52.504 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:52.505 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:38:52.506 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:38:52.506 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:38:52.531 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:38:52.532 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:38:52.550 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:38:52.550 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:38:52.569 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:38:52.569 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:38:52.570 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:52.587 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:38:52.587 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:38:52.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:52 smithi162 ceph-mon[104668]: pgmap v2229: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 191 op/s; 16665/127968 objects degraded (13.023%); 4044/127968 objects misplaced (3.160%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T17:38:52.596 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:52.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:52.607 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:38:52.607 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:38:52.624 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:38:52.624 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:38:52.641 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:38:52.641 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:38:52.698 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:52.725 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:52.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:52.826 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:52.854 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:52.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:52.955 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:52.984 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:52.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:52.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:53.086 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:53.113 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:53.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:53.215 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:53.239 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:53.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:53.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:53 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1001167638' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:53.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:53 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1001167638' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:53.341 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:53.366 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:53.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:53.468 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:53.491 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:53.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:53.553 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:38:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:38:53] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.51.0" 2024-08-17T17:38:53.593 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:53.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:53 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1001167638' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:38:53.618 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:53.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:53.720 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:53.744 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:53.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:53.846 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:53.869 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:53.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:53.971 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:53.997 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:53.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:53.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:54.099 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:54.126 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:54.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:54.228 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:54.254 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:54.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:54.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:54 smithi063 ceph-mon[114756]: pgmap v2230: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 197 op/s; 16279/125421 objects degraded (12.979%); 3904/125421 objects misplaced (3.113%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:38:54.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:54 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 16279/125421 objects degraded (12.979%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:54.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:54 smithi063 ceph-mon[109843]: pgmap v2230: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 197 op/s; 16279/125421 objects degraded (12.979%); 3904/125421 objects misplaced (3.113%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:38:54.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:54 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 16279/125421 objects degraded (12.979%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:54.356 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:54.378 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:54.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:54.480 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:54.509 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:54.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:54.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:54 smithi162 ceph-mon[104668]: pgmap v2230: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 197 op/s; 16279/125421 objects degraded (12.979%); 3904/125421 objects misplaced (3.113%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:38:54.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:54 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 16279/125421 objects degraded (12.979%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:54.611 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:54.639 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:54.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:54.741 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:54.766 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:54.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:54.869 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:54.895 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:54.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:54.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:54.998 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:55.021 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:55.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:55.123 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:55.150 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:55.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:55.252 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:55.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.282 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:55.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:55.383 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:55.413 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:55.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:55.515 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:55.540 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:55.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:55.642 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:55.668 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:55.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:55.770 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:55.796 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:55.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:55.898 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:55.925 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:55.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:55.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:56.027 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:56.051 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:56.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:56.152 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:56.179 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:56.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:56.283 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:56.306 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:56.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:56.364 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:56 smithi063 ceph-mon[114756]: pgmap v2231: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 198 op/s; 16279/125040 objects degraded (13.019%); 3904/125040 objects misplaced (3.122%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:38:56.365 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:56 smithi063 ceph-mon[109843]: pgmap v2231: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 198 op/s; 16279/125040 objects degraded (13.019%); 3904/125040 objects misplaced (3.122%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:38:56.407 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:56.434 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:56.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:56.536 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:56.566 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:56.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.567 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:56.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:56 smithi162 ceph-mon[104668]: pgmap v2231: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 198 op/s; 16279/125040 objects degraded (13.019%); 3904/125040 objects misplaced (3.122%); 3.2 MiB/s, 50 objects/s recovering 2024-08-17T17:38:56.668 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:56.695 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:56.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:56.769 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:38:56.797 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:56.835 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:56.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:56.937 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:56.959 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:56.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:56.960 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:57.060 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:57.086 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:57.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:57.127 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:38:57.188 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:57.215 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:57.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:57.317 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:57.341 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:57.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:57.443 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:57.469 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:57.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:57.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:57 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3356518104' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:57.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:57 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:57.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:57 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3356518104' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:57.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:57 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:57.571 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:57.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:57 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3356518104' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:57.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:57 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:38:57.596 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:57.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:57.698 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:57.728 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:57.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:57.830 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:57.832 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:38:57.833 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:38:57.852 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:57.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:57.954 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:57.978 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:57.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:57.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:58.080 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:58.111 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:58.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:58.213 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:58.237 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:58.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.238 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:58.269 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:38:58.271 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:38:58.340 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:58.363 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:58.363 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:58 smithi063 ceph-mon[114756]: pgmap v2232: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 187 op/s; 16173/123933 objects degraded (13.050%); 3904/123933 objects misplaced (3.150%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T17:38:58.363 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:58 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:58.363 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:58 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:58.364 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:58 smithi063 ceph-mon[114756]: osdmap e1422: 8 total, 8 up, 8 in 2024-08-17T17:38:58.364 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:58 smithi063 ceph-mon[109843]: pgmap v2232: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 187 op/s; 16173/123933 objects degraded (13.050%); 3904/123933 objects misplaced (3.150%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T17:38:58.364 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:58 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:58.364 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:58 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:58.364 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:58 smithi063 ceph-mon[109843]: osdmap e1422: 8 total, 8 up, 8 in 2024-08-17T17:38:58.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:58.466 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:58.491 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:58.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:58.593 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:38:58.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:58 smithi162 ceph-mon[104668]: pgmap v2232: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 187 op/s; 16173/123933 objects degraded (13.050%); 3904/123933 objects misplaced (3.150%); 3.0 MiB/s, 47 objects/s recovering 2024-08-17T17:38:58.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:58 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:38:58.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:58 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:38:58.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:58 smithi162 ceph-mon[104668]: osdmap e1422: 8 total, 8 up, 8 in 2024-08-17T17:38:58.618 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:58.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:38:58.720 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:58.745 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:58.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:58.839 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:38:58.846 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:58.871 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:58.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:58.972 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:38:58.995 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:58.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:58.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:38:59.097 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:38:59.124 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:59.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:38:59.226 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:59.252 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:59.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:59.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:38:59 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 16173/123933 objects degraded (13.050%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:59.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:38:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=cleanup t=2024-08-17T17:38:59.017256101Z level=info msg="Completed cleanup jobs" duration=13.382537ms 2024-08-17T17:38:59.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:38:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=grafana.update.checker t=2024-08-17T17:38:59.139586393Z level=info msg="Update check succeeded" duration=63.012448ms 2024-08-17T17:38:59.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:38:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=plugins.update.checker t=2024-08-17T17:38:59.155773214Z level=info msg="Update check succeeded" duration=69.616885ms 2024-08-17T17:38:59.354 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:38:59.376 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:59.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:38:59.477 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:38:59.502 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:59.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:38:59.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:38:59 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 16173/123933 objects degraded (13.050%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:59.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:38:59 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 16173/123933 objects degraded (13.050%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:38:59.604 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:38:59.629 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:59.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:38:59.731 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:38:59.755 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:38:59.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:38:59.857 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:38:59.881 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:38:59.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:38:59.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:38:59.983 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:00.006 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:00.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:00.101 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2233,"stamp":"2024-08-17T17:38:57.122868+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2682635953,"num_objects":40951,"num_object_clones":0,"num_object_copies":122853,"num_objects_missing_on_primary":4069,"num_objects_missing":4069,"num_objects_degraded":15885,"num_objects_misplaced":3777,"num_objects_unfound":0,"num_objects_dirty":40951,"num_whiteouts":0,"num_read":28,"num_read_kb":62,"num_write":88036,"num_write_kb":4124377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29175,"num_bytes_recovered":1759122141,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92535,"ondisk_log_size":92535,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":375,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14530804,"kb_used_data":8686516,"kb_used_omap":298,"kb_used_meta":5843925,"kb_avail":735430412,"statfs":{"total":767960285184,"available":753080741888,"internally_reserved":0,"allocated":8894992384,"data_stored":8873719747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":5984179729},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-132907008,"num_objects":-2028,"num_object_clones":0,"num_object_copies":-6084,"num_objects_missing_on_primary":-404,"num_objects_missing":-404,"num_objects_degraded":-780,"num_objects_misplaced":-267,"num_objects_unfound":0,"num_objects_dirty":-2028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2028,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":526,"num_bytes_recovered":34471936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.473148"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":48,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.103993+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:38:46.103993+0000","last_peered":"2024-08-17T17:38:46.103993+0000","last_clean":"2024-08-17T17:38:46.103993+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:38:46.103993+0000","last_undegraded":"2024-08-17T17:38:46.103993+0000","last_fullsized":"2024-08-17T17:38:46.103993+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1435,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.083498+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:38:37.083498+0000","last_peered":"2024-08-17T17:38:37.083498+0000","last_clean":"2024-08-17T17:38:37.083498+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:38:37.083498+0000","last_undegraded":"2024-08-17T17:38:37.083498+0000","last_fullsized":"2024-08-17T17:38:37.083498+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1750,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.083481+0000","last_change":"2024-08-17T17:36:15.771150+0000","last_active":"2024-08-17T17:38:37.083481+0000","last_peered":"2024-08-17T17:38:37.083481+0000","last_clean":"2024-08-17T17:38:37.083481+0000","last_became_active":"2024-08-17T17:36:15.770520+0000","last_became_peered":"2024-08-17T17:36:15.770520+0000","last_unstale":"2024-08-17T17:38:37.083481+0000","last_undegraded":"2024-08-17T17:38:37.083481+0000","last_fullsized":"2024-08-17T17:38:37.083481+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:17.552796+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":85,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084165+0000","last_change":"2024-08-17T17:36:12.756709+0000","last_active":"2024-08-17T17:38:37.084165+0000","last_peered":"2024-08-17T17:38:37.084165+0000","last_clean":"2024-08-17T17:38:37.084165+0000","last_became_active":"2024-08-17T17:36:12.756385+0000","last_became_peered":"2024-08-17T17:36:12.756385+0000","last_unstale":"2024-08-17T17:38:37.084165+0000","last_undegraded":"2024-08-17T17:38:37.084165+0000","last_fullsized":"2024-08-17T17:38:37.084165+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:21.420282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":133,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.110450+0000","last_change":"2024-08-17T17:36:12.762800+0000","last_active":"2024-08-17T17:38:46.110450+0000","last_peered":"2024-08-17T17:38:46.110450+0000","last_clean":"2024-08-17T17:38:46.110450+0000","last_became_active":"2024-08-17T17:36:12.759977+0000","last_became_peered":"2024-08-17T17:36:12.759977+0000","last_unstale":"2024-08-17T17:38:46.110450+0000","last_undegraded":"2024-08-17T17:38:46.110450+0000","last_fullsized":"2024-08-17T17:38:46.110450+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:02.162681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1630,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.083678+0000","last_change":"2024-08-17T17:35:56.281561+0000","last_active":"2024-08-17T17:38:37.083678+0000","last_peered":"2024-08-17T17:38:37.083678+0000","last_clean":"2024-08-17T17:38:37.083678+0000","last_became_active":"2024-08-17T17:35:56.278638+0000","last_became_peered":"2024-08-17T17:35:56.278638+0000","last_unstale":"2024-08-17T17:38:37.083678+0000","last_undegraded":"2024-08-17T17:38:37.083678+0000","last_fullsized":"2024-08-17T17:38:37.083678+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1369,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.138774+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:38:46.138774+0000","last_peered":"2024-08-17T17:38:46.138774+0000","last_clean":"2024-08-17T17:38:46.138774+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:38:46.138774+0000","last_undegraded":"2024-08-17T17:38:46.138774+0000","last_fullsized":"2024-08-17T17:38:46.138774+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":45,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.103243+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:38:46.103243+0000","last_peered":"2024-08-17T17:38:46.103243+0000","last_clean":"2024-08-17T17:38:46.103243+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:38:46.103243+0000","last_undegraded":"2024-08-17T17:38:46.103243+0000","last_fullsized":"2024-08-17T17:38:46.103243+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1414,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.437981+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:38:37.437981+0000","last_peered":"2024-08-17T17:38:37.437981+0000","last_clean":"2024-08-17T17:38:37.437981+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:38:37.437981+0000","last_undegraded":"2024-08-17T17:38:37.437981+0000","last_fullsized":"2024-08-17T17:38:37.437981+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1422,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:42.421201+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:38:42.421201+0000","last_peered":"2024-08-17T17:38:42.421201+0000","last_clean":"2024-08-17T17:38:42.421201+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:38:42.421201+0000","last_undegraded":"2024-08-17T17:38:42.421201+0000","last_fullsized":"2024-08-17T17:38:42.421201+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":712,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084082+0000","last_change":"2024-08-17T17:36:15.773315+0000","last_active":"2024-08-17T17:38:37.084082+0000","last_peered":"2024-08-17T17:38:37.084082+0000","last_clean":"2024-08-17T17:38:37.084082+0000","last_became_active":"2024-08-17T17:36:15.772961+0000","last_became_peered":"2024-08-17T17:36:15.772961+0000","last_unstale":"2024-08-17T17:38:37.084082+0000","last_undegraded":"2024-08-17T17:38:37.084082+0000","last_fullsized":"2024-08-17T17:38:37.084082+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:10.190221+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":42,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.103769+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:38:46.103769+0000","last_peered":"2024-08-17T17:38:46.103769+0000","last_clean":"2024-08-17T17:38:46.103769+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:38:46.103769+0000","last_undegraded":"2024-08-17T17:38:46.103769+0000","last_fullsized":"2024-08-17T17:38:46.103769+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":133,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.110704+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:38:46.110704+0000","last_peered":"2024-08-17T17:38:46.110704+0000","last_clean":"2024-08-17T17:38:46.110704+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:38:46.110704+0000","last_undegraded":"2024-08-17T17:38:46.110704+0000","last_fullsized":"2024-08-17T17:38:46.110704+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1503,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.103260+0000","last_change":"2024-08-17T17:36:24.225739+0000","last_active":"2024-08-17T17:38:37.103260+0000","last_peered":"2024-08-17T17:38:37.103260+0000","last_clean":"2024-08-17T17:38:37.103260+0000","last_became_active":"2024-08-17T17:36:24.225250+0000","last_became_peered":"2024-08-17T17:36:24.225250+0000","last_unstale":"2024-08-17T17:38:37.103260+0000","last_undegraded":"2024-08-17T17:38:37.103260+0000","last_fullsized":"2024-08-17T17:38:37.103260+0000","mapping_epoch":1393,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1394,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.290656+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1442,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.083977+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:38:37.083977+0000","last_peered":"2024-08-17T17:38:37.083977+0000","last_clean":"2024-08-17T17:38:37.083977+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:38:37.083977+0000","last_undegraded":"2024-08-17T17:38:37.083977+0000","last_fullsized":"2024-08-17T17:38:37.083977+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1849,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085456+0000","last_change":"2024-08-17T17:36:12.752970+0000","last_active":"2024-08-17T17:38:37.085456+0000","last_peered":"2024-08-17T17:38:37.085456+0000","last_clean":"2024-08-17T17:38:37.085456+0000","last_became_active":"2024-08-17T17:36:12.752311+0000","last_became_peered":"2024-08-17T17:36:12.752311+0000","last_unstale":"2024-08-17T17:38:37.085456+0000","last_undegraded":"2024-08-17T17:38:37.085456+0000","last_fullsized":"2024-08-17T17:38:37.085456+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:38.131419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1796,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081274+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:38:37.081274+0000","last_peered":"2024-08-17T17:38:37.081274+0000","last_clean":"2024-08-17T17:38:37.081274+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:38:37.081274+0000","last_undegraded":"2024-08-17T17:38:37.081274+0000","last_fullsized":"2024-08-17T17:38:37.081274+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1505,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.140241+0000","last_change":"2024-08-17T17:36:18.750494+0000","last_active":"2024-08-17T17:38:46.140241+0000","last_peered":"2024-08-17T17:38:46.140241+0000","last_clean":"2024-08-17T17:38:46.140241+0000","last_became_active":"2024-08-17T17:36:18.750092+0000","last_became_peered":"2024-08-17T17:36:18.750092+0000","last_unstale":"2024-08-17T17:38:46.140241+0000","last_undegraded":"2024-08-17T17:38:46.140241+0000","last_fullsized":"2024-08-17T17:38:46.140241+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:36.599727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1135,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.104660+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:38:46.104660+0000","last_peered":"2024-08-17T17:38:46.104660+0000","last_clean":"2024-08-17T17:38:46.104660+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:38:46.104660+0000","last_undegraded":"2024-08-17T17:38:46.104660+0000","last_fullsized":"2024-08-17T17:38:46.104660+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":963,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081447+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:38:37.081447+0000","last_peered":"2024-08-17T17:38:37.081447+0000","last_clean":"2024-08-17T17:38:37.081447+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:38:37.081447+0000","last_undegraded":"2024-08-17T17:38:37.081447+0000","last_fullsized":"2024-08-17T17:38:37.081447+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":42,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.103399+0000","last_change":"2024-08-17T17:37:11.075505+0000","last_active":"2024-08-17T17:38:46.103399+0000","last_peered":"2024-08-17T17:38:46.103399+0000","last_clean":"2024-08-17T17:38:46.103399+0000","last_became_active":"2024-08-17T17:37:11.075131+0000","last_became_peered":"2024-08-17T17:37:11.075131+0000","last_unstale":"2024-08-17T17:38:46.103399+0000","last_undegraded":"2024-08-17T17:38:46.103399+0000","last_fullsized":"2024-08-17T17:38:46.103399+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:39.672054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1276,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084643+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:38:37.084643+0000","last_peered":"2024-08-17T17:38:37.084643+0000","last_clean":"2024-08-17T17:38:37.084643+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:38:37.084643+0000","last_undegraded":"2024-08-17T17:38:37.084643+0000","last_fullsized":"2024-08-17T17:38:37.084643+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1653,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.083697+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:38:37.083697+0000","last_peered":"2024-08-17T17:38:37.083697+0000","last_clean":"2024-08-17T17:38:37.083697+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:38:37.083697+0000","last_undegraded":"2024-08-17T17:38:37.083697+0000","last_fullsized":"2024-08-17T17:38:37.083697+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":89,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.139210+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:38:46.139210+0000","last_peered":"2024-08-17T17:38:46.139210+0000","last_clean":"2024-08-17T17:38:46.139210+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:38:46.139210+0000","last_undegraded":"2024-08-17T17:38:46.139210+0000","last_fullsized":"2024-08-17T17:38:46.139210+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1250,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081983+0000","last_change":"2024-08-17T17:36:12.755873+0000","last_active":"2024-08-17T17:38:37.081983+0000","last_peered":"2024-08-17T17:38:37.081983+0000","last_clean":"2024-08-17T17:38:37.081983+0000","last_became_active":"2024-08-17T17:36:12.755366+0000","last_became_peered":"2024-08-17T17:36:12.755366+0000","last_unstale":"2024-08-17T17:38:37.081983+0000","last_undegraded":"2024-08-17T17:38:37.081983+0000","last_fullsized":"2024-08-17T17:38:37.081983+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:36.523214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1604,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.087999+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:38:37.087999+0000","last_peered":"2024-08-17T17:38:37.087999+0000","last_clean":"2024-08-17T17:38:37.087999+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:38:37.087999+0000","last_undegraded":"2024-08-17T17:38:37.087999+0000","last_fullsized":"2024-08-17T17:38:37.087999+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":788,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.103460+0000","last_change":"2024-08-17T17:35:55.200519+0000","last_active":"2024-08-17T17:38:37.103460+0000","last_peered":"2024-08-17T17:38:37.103460+0000","last_clean":"2024-08-17T17:38:37.103460+0000","last_became_active":"2024-08-17T17:35:55.200230+0000","last_became_peered":"2024-08-17T17:35:55.200230+0000","last_unstale":"2024-08-17T17:38:37.103460+0000","last_undegraded":"2024-08-17T17:38:37.103460+0000","last_fullsized":"2024-08-17T17:38:37.103460+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:33.454851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1427,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081736+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:38:37.081736+0000","last_peered":"2024-08-17T17:38:37.081736+0000","last_clean":"2024-08-17T17:38:37.081736+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:38:37.081736+0000","last_undegraded":"2024-08-17T17:38:37.081736+0000","last_fullsized":"2024-08-17T17:38:37.081736+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1706,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.110566+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:38:46.110566+0000","last_peered":"2024-08-17T17:38:46.110566+0000","last_clean":"2024-08-17T17:38:46.110566+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:38:46.110566+0000","last_undegraded":"2024-08-17T17:38:46.110566+0000","last_fullsized":"2024-08-17T17:38:46.110566+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:22.077229+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":253,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.140193+0000","last_change":"2024-08-17T17:35:33.145476+0000","last_active":"2024-08-17T17:38:46.140193+0000","last_peered":"2024-08-17T17:38:46.140193+0000","last_clean":"2024-08-17T17:38:46.140193+0000","last_became_active":"2024-08-17T17:35:33.145125+0000","last_became_peered":"2024-08-17T17:35:33.145125+0000","last_unstale":"2024-08-17T17:38:46.140193+0000","last_undegraded":"2024-08-17T17:38:46.140193+0000","last_fullsized":"2024-08-17T17:38:46.140193+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:45.298064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":932,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.091538+0000","last_change":"2024-08-17T17:35:56.248612+0000","last_active":"2024-08-17T17:38:37.091538+0000","last_peered":"2024-08-17T17:38:37.091538+0000","last_clean":"2024-08-17T17:38:37.091538+0000","last_became_active":"2024-08-17T17:35:56.244364+0000","last_became_peered":"2024-08-17T17:35:56.244364+0000","last_unstale":"2024-08-17T17:38:37.091538+0000","last_undegraded":"2024-08-17T17:38:37.091538+0000","last_fullsized":"2024-08-17T17:38:37.091538+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:41.064165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":871,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081803+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:38:37.081803+0000","last_peered":"2024-08-17T17:38:37.081803+0000","last_clean":"2024-08-17T17:38:37.081803+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:38:37.081803+0000","last_undegraded":"2024-08-17T17:38:37.081803+0000","last_fullsized":"2024-08-17T17:38:37.081803+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1407,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085543+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:38:37.085543+0000","last_peered":"2024-08-17T17:38:37.085543+0000","last_clean":"2024-08-17T17:38:37.085543+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:38:37.085543+0000","last_undegraded":"2024-08-17T17:38:37.085543+0000","last_fullsized":"2024-08-17T17:38:37.085543+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1426,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084473+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:38:37.084473+0000","last_peered":"2024-08-17T17:38:37.084473+0000","last_clean":"2024-08-17T17:38:37.084473+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:38:37.084473+0000","last_undegraded":"2024-08-17T17:38:37.084473+0000","last_fullsized":"2024-08-17T17:38:37.084473+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":928,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.143331+0000","last_change":"2024-08-17T17:36:15.774881+0000","last_active":"2024-08-17T17:38:46.143331+0000","last_peered":"2024-08-17T17:38:46.143331+0000","last_clean":"2024-08-17T17:38:46.143331+0000","last_became_active":"2024-08-17T17:36:15.774435+0000","last_became_peered":"2024-08-17T17:36:15.774435+0000","last_unstale":"2024-08-17T17:38:46.143331+0000","last_undegraded":"2024-08-17T17:38:46.143331+0000","last_fullsized":"2024-08-17T17:38:46.143331+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:52.015854+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"1421'5619","reported_seq":13796,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:52.298687+0000","last_change":"2024-08-17T17:38:37.110784+0000","last_active":"2024-08-17T17:38:52.298687+0000","last_peered":"2024-08-17T17:38:52.298687+0000","last_clean":"2024-08-17T17:38:52.298687+0000","last_became_active":"2024-08-17T17:38:37.088380+0000","last_became_peered":"2024-08-17T17:38:37.088380+0000","last_unstale":"2024-08-17T17:38:52.298687+0000","last_undegraded":"2024-08-17T17:38:52.298687+0000","last_fullsized":"2024-08-17T17:38:52.298687+0000","mapping_epoch":1419,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5619,"log_dups_size":0,"ondisk_log_size":5619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:08.647903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174784512,"num_objects":2667,"num_object_clones":0,"num_object_copies":8001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5619,"num_write_kb":265152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4009,"num_bytes_recovered":199753728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1374,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084829+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:38:37.084829+0000","last_peered":"2024-08-17T17:38:37.084829+0000","last_clean":"2024-08-17T17:38:37.084829+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:38:37.084829+0000","last_undegraded":"2024-08-17T17:38:37.084829+0000","last_fullsized":"2024-08-17T17:38:37.084829+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1270,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084571+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:38:37.084571+0000","last_peered":"2024-08-17T17:38:37.084571+0000","last_clean":"2024-08-17T17:38:37.084571+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:38:37.084571+0000","last_undegraded":"2024-08-17T17:38:37.084571+0000","last_fullsized":"2024-08-17T17:38:37.084571+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":320,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.104772+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:38:46.104772+0000","last_peered":"2024-08-17T17:38:46.104772+0000","last_clean":"2024-08-17T17:38:46.104772+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:38:46.104772+0000","last_undegraded":"2024-08-17T17:38:46.104772+0000","last_fullsized":"2024-08-17T17:38:46.104772+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1589,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.140177+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:38:46.140177+0000","last_peered":"2024-08-17T17:38:46.140177+0000","last_clean":"2024-08-17T17:38:46.140177+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:38:46.140177+0000","last_undegraded":"2024-08-17T17:38:46.140177+0000","last_fullsized":"2024-08-17T17:38:46.140177+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:25.751871+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.f","version":"1421'5625","reported_seq":5771,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:55.562243+0000","last_change":"2024-08-17T17:35:34.179168+0000","last_active":"2024-08-17T17:38:55.562243+0000","last_peered":"2024-08-17T17:38:55.562243+0000","last_clean":"2024-08-17T17:38:55.562243+0000","last_became_active":"2024-08-17T17:35:34.133493+0000","last_became_peered":"2024-08-17T17:35:34.133493+0000","last_unstale":"2024-08-17T17:38:55.562243+0000","last_undegraded":"2024-08-17T17:38:55.562243+0000","last_fullsized":"2024-08-17T17:38:55.562243+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5625,"log_dups_size":0,"ondisk_log_size":5625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:24:11.188566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169017344,"num_objects":2579,"num_object_clones":0,"num_object_copies":7737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5625,"num_write_kb":262528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":96,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.139262+0000","last_change":"2024-08-17T17:37:11.079712+0000","last_active":"2024-08-17T17:38:46.139262+0000","last_peered":"2024-08-17T17:38:46.139262+0000","last_clean":"2024-08-17T17:38:46.139262+0000","last_became_active":"2024-08-17T17:37:11.079253+0000","last_became_peered":"2024-08-17T17:37:11.079253+0000","last_unstale":"2024-08-17T17:38:46.139262+0000","last_undegraded":"2024-08-17T17:38:46.139262+0000","last_fullsized":"2024-08-17T17:38:46.139262+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:53.405657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":915,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084071+0000","last_change":"2024-08-17T17:35:56.223396+0000","last_active":"2024-08-17T17:38:37.084071+0000","last_peered":"2024-08-17T17:38:37.084071+0000","last_clean":"2024-08-17T17:38:37.084071+0000","last_became_active":"2024-08-17T17:35:56.212413+0000","last_became_peered":"2024-08-17T17:35:56.212413+0000","last_unstale":"2024-08-17T17:38:37.084071+0000","last_undegraded":"2024-08-17T17:38:37.084071+0000","last_fullsized":"2024-08-17T17:38:37.084071+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:44.185174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1630,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.104276+0000","last_change":"2024-08-17T17:35:33.136068+0000","last_active":"2024-08-17T17:38:46.104276+0000","last_peered":"2024-08-17T17:38:46.104276+0000","last_clean":"2024-08-17T17:38:46.104276+0000","last_became_active":"2024-08-17T17:35:33.135591+0000","last_became_peered":"2024-08-17T17:35:33.135591+0000","last_unstale":"2024-08-17T17:38:46.104276+0000","last_undegraded":"2024-08-17T17:38:46.104276+0000","last_fullsized":"2024-08-17T17:38:46.104276+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:52.252561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1780,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085719+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:38:37.085719+0000","last_peered":"2024-08-17T17:38:37.085719+0000","last_clean":"2024-08-17T17:38:37.085719+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:38:37.085719+0000","last_undegraded":"2024-08-17T17:38:37.085719+0000","last_fullsized":"2024-08-17T17:38:37.085719+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"1421'5466","reported_seq":5597,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:55.398668+0000","last_change":"2024-08-17T17:35:14.861823+0000","last_active":"2024-08-17T17:38:55.398668+0000","last_peered":"2024-08-17T17:38:55.398668+0000","last_clean":"2024-08-17T17:38:55.398668+0000","last_became_active":"2024-08-17T17:35:14.854109+0000","last_became_peered":"2024-08-17T17:35:14.854109+0000","last_unstale":"2024-08-17T17:38:55.398668+0000","last_undegraded":"2024-08-17T17:38:55.398668+0000","last_fullsized":"2024-08-17T17:38:55.398668+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5466,"log_dups_size":0,"ondisk_log_size":5466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:46:40.778197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169082880,"num_objects":2580,"num_object_clones":0,"num_object_copies":7740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5466,"num_write_kb":257472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":993,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081554+0000","last_change":"2024-08-17T17:35:33.138467+0000","last_active":"2024-08-17T17:38:37.081554+0000","last_peered":"2024-08-17T17:38:37.081554+0000","last_clean":"2024-08-17T17:38:37.081554+0000","last_became_active":"2024-08-17T17:35:33.138174+0000","last_became_peered":"2024-08-17T17:35:33.138174+0000","last_unstale":"2024-08-17T17:38:37.081554+0000","last_undegraded":"2024-08-17T17:38:37.081554+0000","last_fullsized":"2024-08-17T17:38:37.081554+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:08.860439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1377,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.088079+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:38:37.088079+0000","last_peered":"2024-08-17T17:38:37.088079+0000","last_clean":"2024-08-17T17:38:37.088079+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:38:37.088079+0000","last_undegraded":"2024-08-17T17:38:37.088079+0000","last_fullsized":"2024-08-17T17:38:37.088079+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":126,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.110671+0000","last_change":"2024-08-17T17:35:14.854881+0000","last_active":"2024-08-17T17:38:46.110671+0000","last_peered":"2024-08-17T17:38:46.110671+0000","last_clean":"2024-08-17T17:38:46.110671+0000","last_became_active":"2024-08-17T17:35:14.854121+0000","last_became_peered":"2024-08-17T17:35:14.854121+0000","last_unstale":"2024-08-17T17:38:46.110671+0000","last_undegraded":"2024-08-17T17:38:46.110671+0000","last_fullsized":"2024-08-17T17:38:46.110671+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.863624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":934,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084256+0000","last_change":"2024-08-17T17:36:15.773050+0000","last_active":"2024-08-17T17:38:37.084256+0000","last_peered":"2024-08-17T17:38:37.084256+0000","last_clean":"2024-08-17T17:38:37.084256+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:38:37.084256+0000","last_undegraded":"2024-08-17T17:38:37.084256+0000","last_fullsized":"2024-08-17T17:38:37.084256+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:39.212195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:39.212195+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"1421'5477","reported_seq":11314,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:55.279244+0000","last_change":"2024-08-17T17:37:21.409979+0000","last_active":"2024-08-17T17:38:55.279244+0000","last_peered":"2024-08-17T17:38:55.279244+0000","last_clean":"2024-08-17T17:38:55.279244+0000","last_became_active":"2024-08-17T17:37:21.398111+0000","last_became_peered":"2024-08-17T17:37:21.398111+0000","last_unstale":"2024-08-17T17:38:55.279244+0000","last_undegraded":"2024-08-17T17:38:55.279244+0000","last_fullsized":"2024-08-17T17:38:55.279244+0000","mapping_epoch":1407,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5477,"log_dups_size":0,"ondisk_log_size":5477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:19.299395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167378968,"num_objects":2555,"num_object_clones":0,"num_object_copies":7665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2555,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5477,"num_write_kb":256961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2850,"num_bytes_recovered":168820736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1036,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.437939+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:38:37.437939+0000","last_peered":"2024-08-17T17:38:37.437939+0000","last_clean":"2024-08-17T17:38:37.437939+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:38:37.437939+0000","last_undegraded":"2024-08-17T17:38:37.437939+0000","last_fullsized":"2024-08-17T17:38:37.437939+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":220,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.143423+0000","last_change":"2024-08-17T17:35:33.145351+0000","last_active":"2024-08-17T17:38:46.143423+0000","last_peered":"2024-08-17T17:38:46.143423+0000","last_clean":"2024-08-17T17:38:46.143423+0000","last_became_active":"2024-08-17T17:35:33.145040+0000","last_became_peered":"2024-08-17T17:35:33.145040+0000","last_unstale":"2024-08-17T17:38:46.143423+0000","last_undegraded":"2024-08-17T17:38:46.143423+0000","last_fullsized":"2024-08-17T17:38:46.143423+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:02.124000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1445,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081618+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:38:37.081618+0000","last_peered":"2024-08-17T17:38:37.081618+0000","last_clean":"2024-08-17T17:38:37.081618+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:38:37.081618+0000","last_undegraded":"2024-08-17T17:38:37.081618+0000","last_fullsized":"2024-08-17T17:38:37.081618+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":917,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084546+0000","last_change":"2024-08-17T17:36:15.769890+0000","last_active":"2024-08-17T17:38:37.084546+0000","last_peered":"2024-08-17T17:38:37.084546+0000","last_clean":"2024-08-17T17:38:37.084546+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:38:37.084546+0000","last_undegraded":"2024-08-17T17:38:37.084546+0000","last_fullsized":"2024-08-17T17:38:37.084546+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:33.034306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:33.034306+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":4,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.b","version":"1421'5396","reported_seq":13193,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:53.448890+0000","last_change":"2024-08-17T17:38:07.888612+0000","last_active":"2024-08-17T17:38:53.448890+0000","last_peered":"2024-08-17T17:38:53.448890+0000","last_clean":"2024-08-17T17:38:53.448890+0000","last_became_active":"2024-08-17T17:38:07.875196+0000","last_became_peered":"2024-08-17T17:38:07.875196+0000","last_unstale":"2024-08-17T17:38:53.448890+0000","last_undegraded":"2024-08-17T17:38:53.448890+0000","last_fullsized":"2024-08-17T17:38:53.448890+0000","mapping_epoch":1415,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5396,"log_dups_size":0,"ondisk_log_size":5396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:21.658004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167641088,"num_objects":2558,"num_object_clones":0,"num_object_copies":7674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5396,"num_write_kb":254528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3836,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1481,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.110795+0000","last_change":"2024-08-17T17:36:12.759743+0000","last_active":"2024-08-17T17:38:46.110795+0000","last_peered":"2024-08-17T17:38:46.110795+0000","last_clean":"2024-08-17T17:38:46.110795+0000","last_became_active":"2024-08-17T17:36:12.759334+0000","last_became_peered":"2024-08-17T17:36:12.759334+0000","last_unstale":"2024-08-17T17:38:46.110795+0000","last_undegraded":"2024-08-17T17:38:46.110795+0000","last_fullsized":"2024-08-17T17:38:46.110795+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:04.524384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1408,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084663+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:38:37.084663+0000","last_peered":"2024-08-17T17:38:37.084663+0000","last_clean":"2024-08-17T17:38:37.084663+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:38:37.084663+0000","last_undegraded":"2024-08-17T17:38:37.084663+0000","last_fullsized":"2024-08-17T17:38:37.084663+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1451,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081488+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:38:37.081488+0000","last_peered":"2024-08-17T17:38:37.081488+0000","last_clean":"2024-08-17T17:38:37.081488+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:38:37.081488+0000","last_undegraded":"2024-08-17T17:38:37.081488+0000","last_fullsized":"2024-08-17T17:38:37.081488+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1766,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081199+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:38:37.081199+0000","last_peered":"2024-08-17T17:38:37.081199+0000","last_clean":"2024-08-17T17:38:37.081199+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:38:37.081199+0000","last_undegraded":"2024-08-17T17:38:37.081199+0000","last_fullsized":"2024-08-17T17:38:37.081199+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"1421'5629","reported_seq":10338,"reported_epoch":1421,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:38:56.730954+0000","last_change":"2024-08-17T17:38:05.991715+0000","last_active":"2024-08-17T17:38:56.730954+0000","last_peered":"2024-08-17T17:38:56.730954+0000","last_clean":"2024-08-17T17:35:54.187688+0000","last_became_active":"2024-08-17T17:36:22.904523+0000","last_became_peered":"2024-08-17T17:36:22.904523+0000","last_unstale":"2024-08-17T17:38:56.730954+0000","last_undegraded":"2024-08-17T17:38:56.730954+0000","last_fullsized":"2024-08-17T17:36:13.740491+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5629,"log_dups_size":0,"ondisk_log_size":5629,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:57.525855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167706624,"num_objects":2559,"num_object_clones":0,"num_object_copies":7677,"num_objects_missing_on_primary":1350,"num_objects_missing":1350,"num_objects_degraded":0,"num_objects_misplaced":3777,"num_objects_unfound":0,"num_objects_dirty":2559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5629,"num_write_kb":262016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1541,"num_bytes_recovered":100990976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":1350},{"shards":"2,5,6","objects":1077},{"shards":"5,6","objects":132}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1084,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081868+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:38:37.081868+0000","last_peered":"2024-08-17T17:38:37.081868+0000","last_clean":"2024-08-17T17:38:37.081868+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:38:37.081868+0000","last_undegraded":"2024-08-17T17:38:37.081868+0000","last_fullsized":"2024-08-17T17:38:37.081868+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1393,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.110351+0000","last_change":"2024-08-17T17:35:56.229622+0000","last_active":"2024-08-17T17:38:46.110351+0000","last_peered":"2024-08-17T17:38:46.110351+0000","last_clean":"2024-08-17T17:38:46.110351+0000","last_became_active":"2024-08-17T17:35:56.229324+0000","last_became_peered":"2024-08-17T17:35:56.229324+0000","last_unstale":"2024-08-17T17:38:46.110351+0000","last_undegraded":"2024-08-17T17:38:46.110351+0000","last_fullsized":"2024-08-17T17:38:46.110351+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:13.980316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1996,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.110474+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:38:46.110474+0000","last_peered":"2024-08-17T17:38:46.110474+0000","last_clean":"2024-08-17T17:38:46.110474+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:38:46.110474+0000","last_undegraded":"2024-08-17T17:38:46.110474+0000","last_fullsized":"2024-08-17T17:38:46.110474+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1580,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.083784+0000","last_change":"2024-08-17T17:35:33.137074+0000","last_active":"2024-08-17T17:38:37.083784+0000","last_peered":"2024-08-17T17:38:37.083784+0000","last_clean":"2024-08-17T17:38:37.083784+0000","last_became_active":"2024-08-17T17:35:33.136759+0000","last_became_peered":"2024-08-17T17:35:33.136759+0000","last_unstale":"2024-08-17T17:38:37.083784+0000","last_undegraded":"2024-08-17T17:38:37.083784+0000","last_fullsized":"2024-08-17T17:38:37.083784+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:37.263639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:37.263639+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:14.599780+0000","scrub_duration":0.0016181800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"1421'5560","reported_seq":9497,"reported_epoch":1421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:55.435777+0000","last_change":"2024-08-17T17:36:23.019129+0000","last_active":"2024-08-17T17:38:55.435777+0000","last_peered":"2024-08-17T17:38:55.435777+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:23.013527+0000","last_became_peered":"2024-08-17T17:36:23.013527+0000","last_unstale":"2024-08-17T17:38:55.435777+0000","last_undegraded":"2024-08-17T17:36:13.874715+0000","last_fullsized":"2024-08-17T17:36:13.738828+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5560,"log_dups_size":0,"ondisk_log_size":5560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166461440,"num_objects":2540,"num_object_clones":0,"num_object_copies":7620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4515,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5560,"num_write_kb":259200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1511,"num_bytes_recovered":99024896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[0,6],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":1365},{"shards":"0,1,6","objects":1048},{"shards":"0,6","objects":-610}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1185,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.104961+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:38:46.104961+0000","last_peered":"2024-08-17T17:38:46.104961+0000","last_clean":"2024-08-17T17:38:46.104961+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:38:46.104961+0000","last_undegraded":"2024-08-17T17:38:46.104961+0000","last_fullsized":"2024-08-17T17:38:46.104961+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":45,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084760+0000","last_change":"2024-08-17T17:36:17.774773+0000","last_active":"2024-08-17T17:38:37.084760+0000","last_peered":"2024-08-17T17:38:37.084760+0000","last_clean":"2024-08-17T17:38:37.084760+0000","last_became_active":"2024-08-17T17:36:17.774468+0000","last_became_peered":"2024-08-17T17:36:17.774468+0000","last_unstale":"2024-08-17T17:38:37.084760+0000","last_undegraded":"2024-08-17T17:38:37.084760+0000","last_fullsized":"2024-08-17T17:38:37.084760+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:54.231747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1524,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.103396+0000","last_change":"2024-08-17T17:35:33.137633+0000","last_active":"2024-08-17T17:38:46.103396+0000","last_peered":"2024-08-17T17:38:46.103396+0000","last_clean":"2024-08-17T17:38:46.103396+0000","last_became_active":"2024-08-17T17:35:33.137323+0000","last_became_peered":"2024-08-17T17:35:33.137323+0000","last_unstale":"2024-08-17T17:38:46.103396+0000","last_undegraded":"2024-08-17T17:38:46.103396+0000","last_fullsized":"2024-08-17T17:38:46.103396+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:03.359155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":896,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.082546+0000","last_change":"2024-08-17T17:35:56.264626+0000","last_active":"2024-08-17T17:38:37.082546+0000","last_peered":"2024-08-17T17:38:37.082546+0000","last_clean":"2024-08-17T17:38:37.082546+0000","last_became_active":"2024-08-17T17:35:56.264265+0000","last_became_peered":"2024-08-17T17:35:56.264265+0000","last_unstale":"2024-08-17T17:38:37.082546+0000","last_undegraded":"2024-08-17T17:38:37.082546+0000","last_fullsized":"2024-08-17T17:38:37.082546+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:20.754326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:20.754326+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"1421'5441","reported_seq":9828,"reported_epoch":1421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:56.747879+0000","last_change":"2024-08-17T17:37:11.134558+0000","last_active":"2024-08-17T17:38:56.747879+0000","last_peered":"2024-08-17T17:38:56.747879+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:20.105110+0000","last_became_peered":"2024-08-17T17:36:20.105110+0000","last_unstale":"2024-08-17T17:38:56.747879+0000","last_undegraded":"2024-08-17T17:36:19.999272+0000","last_fullsized":"2024-08-17T17:36:13.740520+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5441,"log_dups_size":0,"ondisk_log_size":5441,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:45.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168624128,"num_objects":2573,"num_object_clones":0,"num_object_copies":7719,"num_objects_missing_on_primary":1380,"num_objects_missing":1380,"num_objects_degraded":3798,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5441,"num_write_kb":256448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1435,"num_bytes_recovered":94044160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1380},{"shards":"1,5,6","objects":1038},{"shards":"5,6","objects":155}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1249,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084134+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:38:37.084134+0000","last_peered":"2024-08-17T17:38:37.084134+0000","last_clean":"2024-08-17T17:38:37.084134+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:38:37.084134+0000","last_undegraded":"2024-08-17T17:38:37.084134+0000","last_fullsized":"2024-08-17T17:38:37.084134+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1789,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.138476+0000","last_change":"2024-08-17T17:36:12.758005+0000","last_active":"2024-08-17T17:38:46.138476+0000","last_peered":"2024-08-17T17:38:46.138476+0000","last_clean":"2024-08-17T17:38:46.138476+0000","last_became_active":"2024-08-17T17:36:12.757695+0000","last_became_peered":"2024-08-17T17:36:12.757695+0000","last_unstale":"2024-08-17T17:38:46.138476+0000","last_undegraded":"2024-08-17T17:38:46.138476+0000","last_fullsized":"2024-08-17T17:38:46.138476+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:51.792220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1427,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081589+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:38:37.081589+0000","last_peered":"2024-08-17T17:38:37.081589+0000","last_clean":"2024-08-17T17:38:37.081589+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:38:37.081589+0000","last_undegraded":"2024-08-17T17:38:37.081589+0000","last_fullsized":"2024-08-17T17:38:37.081589+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1725,"reported_epoch":1421,"state":"active+clean+scrubbing","last_fresh":"2024-08-17T17:38:50.677829+0000","last_change":"2024-08-17T17:38:52.143613+0000","last_active":"2024-08-17T17:38:50.677829+0000","last_peered":"2024-08-17T17:38:50.677829+0000","last_clean":"2024-08-17T17:38:50.677829+0000","last_became_active":"2024-08-17T17:35:55.204861+0000","last_became_peered":"2024-08-17T17:35:55.204861+0000","last_unstale":"2024-08-17T17:38:50.677829+0000","last_undegraded":"2024-08-17T17:38:50.677829+0000","last_fullsized":"2024-08-17T17:38:50.677829+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.0","version":"1421'5509","reported_seq":13494,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:52.352140+0000","last_change":"2024-08-17T17:37:40.753703+0000","last_active":"2024-08-17T17:38:52.352140+0000","last_peered":"2024-08-17T17:38:52.352140+0000","last_clean":"2024-08-17T17:38:52.352140+0000","last_became_active":"2024-08-17T17:37:40.743458+0000","last_became_peered":"2024-08-17T17:37:40.743458+0000","last_unstale":"2024-08-17T17:38:52.352140+0000","last_undegraded":"2024-08-17T17:38:52.352140+0000","last_fullsized":"2024-08-17T17:38:52.352140+0000","mapping_epoch":1411,"log_start":"1373'100","ondisk_log_start":"1373'100","created":1357,"last_epoch_clean":1412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5409,"log_dups_size":0,"ondisk_log_size":5409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:14:02.340380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170852352,"num_objects":2607,"num_object_clones":0,"num_object_copies":7821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5509,"num_write_kb":259712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3914,"num_bytes_recovered":232652800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1022,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085138+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:38:37.085138+0000","last_peered":"2024-08-17T17:38:37.085138+0000","last_clean":"2024-08-17T17:38:37.085138+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:38:37.085138+0000","last_undegraded":"2024-08-17T17:38:37.085138+0000","last_fullsized":"2024-08-17T17:38:37.085138+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":89,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.139379+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:38:46.139379+0000","last_peered":"2024-08-17T17:38:46.139379+0000","last_clean":"2024-08-17T17:38:46.139379+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:38:46.139379+0000","last_undegraded":"2024-08-17T17:38:46.139379+0000","last_fullsized":"2024-08-17T17:38:46.139379+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1758,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081301+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:38:37.081301+0000","last_peered":"2024-08-17T17:38:37.081301+0000","last_clean":"2024-08-17T17:38:37.081301+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:38:37.081301+0000","last_undegraded":"2024-08-17T17:38:37.081301+0000","last_fullsized":"2024-08-17T17:38:37.081301+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1","version":"1421'5513","reported_seq":10073,"reported_epoch":1421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:56.725132+0000","last_change":"2024-08-17T17:36:23.103523+0000","last_active":"2024-08-17T17:38:56.725132+0000","last_peered":"2024-08-17T17:38:56.725132+0000","last_clean":"2024-08-17T17:35:54.178280+0000","last_became_active":"2024-08-17T17:36:23.025910+0000","last_became_peered":"2024-08-17T17:36:23.025910+0000","last_unstale":"2024-08-17T17:38:56.725132+0000","last_undegraded":"2024-08-17T17:36:18.734637+0000","last_fullsized":"2024-08-17T17:36:13.740134+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5513,"log_dups_size":0,"ondisk_log_size":5513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:58.717055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166395904,"num_objects":2539,"num_object_clones":0,"num_object_copies":7617,"num_objects_missing_on_primary":1339,"num_objects_missing":1339,"num_objects_degraded":3753,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5513,"num_write_kb":257664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1490,"num_bytes_recovered":97648640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,0],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":1339},{"shards":"0,2,4,6","objects":5},{"shards":"0,2,6","objects":1070},{"shards":"0,6","objects":125}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1437,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.104807+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:38:46.104807+0000","last_peered":"2024-08-17T17:38:46.104807+0000","last_clean":"2024-08-17T17:38:46.104807+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:38:46.104807+0000","last_undegraded":"2024-08-17T17:38:46.104807+0000","last_fullsized":"2024-08-17T17:38:46.104807+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1409,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085155+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:38:37.085155+0000","last_peered":"2024-08-17T17:38:37.085155+0000","last_clean":"2024-08-17T17:38:37.085155+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:38:37.085155+0000","last_undegraded":"2024-08-17T17:38:37.085155+0000","last_fullsized":"2024-08-17T17:38:37.085155+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":990,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.103946+0000","last_change":"2024-08-17T17:34:26.817339+0000","last_active":"2024-08-17T17:38:46.103946+0000","last_peered":"2024-08-17T17:38:46.103946+0000","last_clean":"2024-08-17T17:38:46.103946+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:38:46.103946+0000","last_undegraded":"2024-08-17T17:38:46.103946+0000","last_fullsized":"2024-08-17T17:38:46.103946+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:26.817223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:26.817223+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:30.089956+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"1421'5411","reported_seq":5567,"reported_epoch":1421,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:38:55.416515+0000","last_change":"2024-08-17T17:36:14.073155+0000","last_active":"2024-08-17T17:38:55.416515+0000","last_peered":"2024-08-17T17:38:55.416515+0000","last_clean":"2024-08-17T17:36:11.734683+0000","last_became_active":"2024-08-17T17:36:14.063794+0000","last_became_peered":"2024-08-17T17:36:14.063794+0000","last_unstale":"2024-08-17T17:38:55.416515+0000","last_undegraded":"2024-08-17T17:36:13.855770+0000","last_fullsized":"2024-08-17T17:36:13.742068+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5411,"log_dups_size":0,"ondisk_log_size":5411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163250176,"num_objects":2491,"num_object_clones":0,"num_object_copies":7473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3819,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5411,"num_write_kb":252864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-1328},{"shards":"0,2,7","objects":2359}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":45,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085580+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:38:37.085580+0000","last_peered":"2024-08-17T17:38:37.085580+0000","last_clean":"2024-08-17T17:38:37.085580+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:38:37.085580+0000","last_undegraded":"2024-08-17T17:38:37.085580+0000","last_fullsized":"2024-08-17T17:38:37.085580+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1271,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.091495+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:38:37.091495+0000","last_peered":"2024-08-17T17:38:37.091495+0000","last_clean":"2024-08-17T17:38:37.091495+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:38:37.091495+0000","last_undegraded":"2024-08-17T17:38:37.091495+0000","last_fullsized":"2024-08-17T17:38:37.091495+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1786,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081113+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:38:37.081113+0000","last_peered":"2024-08-17T17:38:37.081113+0000","last_clean":"2024-08-17T17:38:37.081113+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:38:37.081113+0000","last_undegraded":"2024-08-17T17:38:37.081113+0000","last_fullsized":"2024-08-17T17:38:37.081113+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"1421'5529","reported_seq":11325,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:56.658068+0000","last_change":"2024-08-17T17:36:46.982456+0000","last_active":"2024-08-17T17:38:56.658068+0000","last_peered":"2024-08-17T17:38:56.658068+0000","last_clean":"2024-08-17T17:38:56.658068+0000","last_became_active":"2024-08-17T17:36:46.974782+0000","last_became_peered":"2024-08-17T17:36:46.974782+0000","last_unstale":"2024-08-17T17:38:56.658068+0000","last_undegraded":"2024-08-17T17:38:56.658068+0000","last_fullsized":"2024-08-17T17:38:56.658068+0000","mapping_epoch":1399,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5529,"log_dups_size":0,"ondisk_log_size":5529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:40.499321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166658048,"num_objects":2543,"num_object_clones":0,"num_object_copies":7629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5529,"num_write_kb":258304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2848,"num_bytes_recovered":186646528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1388,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:42.421226+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:38:42.421226+0000","last_peered":"2024-08-17T17:38:42.421226+0000","last_clean":"2024-08-17T17:38:42.421226+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:38:42.421226+0000","last_undegraded":"2024-08-17T17:38:42.421226+0000","last_fullsized":"2024-08-17T17:38:42.421226+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":201,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084277+0000","last_change":"2024-08-17T17:36:12.754372+0000","last_active":"2024-08-17T17:38:37.084277+0000","last_peered":"2024-08-17T17:38:37.084277+0000","last_clean":"2024-08-17T17:38:37.084277+0000","last_became_active":"2024-08-17T17:36:12.746352+0000","last_became_peered":"2024-08-17T17:36:12.746352+0000","last_unstale":"2024-08-17T17:38:37.084277+0000","last_undegraded":"2024-08-17T17:38:37.084277+0000","last_fullsized":"2024-08-17T17:38:37.084277+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:07.807486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1662,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.110369+0000","last_change":"2024-08-17T17:36:15.771193+0000","last_active":"2024-08-17T17:38:46.110369+0000","last_peered":"2024-08-17T17:38:46.110369+0000","last_clean":"2024-08-17T17:38:46.110369+0000","last_became_active":"2024-08-17T17:36:15.770889+0000","last_became_peered":"2024-08-17T17:36:15.770889+0000","last_unstale":"2024-08-17T17:38:46.110369+0000","last_undegraded":"2024-08-17T17:38:46.110369+0000","last_fullsized":"2024-08-17T17:38:46.110369+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:22.978695+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.4","version":"1421'5489","reported_seq":11265,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:56.579872+0000","last_change":"2024-08-17T17:37:04.045576+0000","last_active":"2024-08-17T17:38:56.579872+0000","last_peered":"2024-08-17T17:38:56.579872+0000","last_clean":"2024-08-17T17:38:56.579872+0000","last_became_active":"2024-08-17T17:37:04.038810+0000","last_became_peered":"2024-08-17T17:37:04.038810+0000","last_unstale":"2024-08-17T17:38:56.579872+0000","last_undegraded":"2024-08-17T17:38:56.579872+0000","last_fullsized":"2024-08-17T17:38:56.579872+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5489,"log_dups_size":0,"ondisk_log_size":5489,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:27:04.725258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168886272,"num_objects":2577,"num_object_clones":0,"num_object_copies":7731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5489,"num_write_kb":258112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":176291840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1257,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.104285+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:38:46.104285+0000","last_peered":"2024-08-17T17:38:46.104285+0000","last_clean":"2024-08-17T17:38:46.104285+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:38:46.104285+0000","last_undegraded":"2024-08-17T17:38:46.104285+0000","last_fullsized":"2024-08-17T17:38:46.104285+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1414,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.437991+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:38:37.437991+0000","last_peered":"2024-08-17T17:38:37.437991+0000","last_clean":"2024-08-17T17:38:37.437991+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:38:37.437991+0000","last_undegraded":"2024-08-17T17:38:37.437991+0000","last_fullsized":"2024-08-17T17:38:37.437991+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":914,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085627+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:38:37.085627+0000","last_peered":"2024-08-17T17:38:37.085627+0000","last_clean":"2024-08-17T17:38:37.085627+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:38:37.085627+0000","last_undegraded":"2024-08-17T17:38:37.085627+0000","last_fullsized":"2024-08-17T17:38:37.085627+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"1421'5527","reported_seq":11358,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:52.372855+0000","last_change":"2024-08-17T17:36:32.711490+0000","last_active":"2024-08-17T17:38:52.372855+0000","last_peered":"2024-08-17T17:38:52.372855+0000","last_clean":"2024-08-17T17:38:52.372855+0000","last_became_active":"2024-08-17T17:36:32.704370+0000","last_became_peered":"2024-08-17T17:36:32.704370+0000","last_unstale":"2024-08-17T17:38:52.372855+0000","last_undegraded":"2024-08-17T17:38:52.372855+0000","last_fullsized":"2024-08-17T17:38:52.372855+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5527,"log_dups_size":0,"ondisk_log_size":5527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:17.373879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169148416,"num_objects":2581,"num_object_clones":0,"num_object_copies":7743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5527,"num_write_kb":259456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2866,"num_bytes_recovered":187826176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1270,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.094002+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:38:37.094002+0000","last_peered":"2024-08-17T17:38:37.094002+0000","last_clean":"2024-08-17T17:38:37.094002+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:38:37.094002+0000","last_undegraded":"2024-08-17T17:38:37.094002+0000","last_fullsized":"2024-08-17T17:38:37.094002+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":150,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081381+0000","last_change":"2024-08-17T17:35:55.217072+0000","last_active":"2024-08-17T17:38:37.081381+0000","last_peered":"2024-08-17T17:38:37.081381+0000","last_clean":"2024-08-17T17:38:37.081381+0000","last_became_active":"2024-08-17T17:35:55.216462+0000","last_became_peered":"2024-08-17T17:35:55.216462+0000","last_unstale":"2024-08-17T17:38:37.081381+0000","last_undegraded":"2024-08-17T17:38:37.081381+0000","last_fullsized":"2024-08-17T17:38:37.081381+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:34.937221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":897,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.084632+0000","last_change":"2024-08-17T17:35:55.215004+0000","last_active":"2024-08-17T17:38:37.084632+0000","last_peered":"2024-08-17T17:38:37.084632+0000","last_clean":"2024-08-17T17:38:37.084632+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:38:37.084632+0000","last_undegraded":"2024-08-17T17:38:37.084632+0000","last_fullsized":"2024-08-17T17:38:37.084632+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.6","version":"1421'5402","reported_seq":5534,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:55.413522+0000","last_change":"2024-08-17T17:35:14.868524+0000","last_active":"2024-08-17T17:38:55.413522+0000","last_peered":"2024-08-17T17:38:55.413522+0000","last_clean":"2024-08-17T17:38:55.413522+0000","last_became_active":"2024-08-17T17:35:14.854690+0000","last_became_peered":"2024-08-17T17:35:14.854690+0000","last_unstale":"2024-08-17T17:38:55.413522+0000","last_undegraded":"2024-08-17T17:38:55.413522+0000","last_fullsized":"2024-08-17T17:38:55.413522+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5402,"log_dups_size":0,"ondisk_log_size":5402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:05.388362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158203904,"num_objects":2414,"num_object_clones":0,"num_object_copies":7242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5402,"num_write_kb":250112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1455,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085011+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:38:37.085011+0000","last_peered":"2024-08-17T17:38:37.085011+0000","last_clean":"2024-08-17T17:38:37.085011+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:38:37.085011+0000","last_undegraded":"2024-08-17T17:38:37.085011+0000","last_fullsized":"2024-08-17T17:38:37.085011+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":162,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085793+0000","last_change":"2024-08-17T17:35:33.147672+0000","last_active":"2024-08-17T17:38:37.085793+0000","last_peered":"2024-08-17T17:38:37.085793+0000","last_clean":"2024-08-17T17:38:37.085793+0000","last_became_active":"2024-08-17T17:35:33.147283+0000","last_became_peered":"2024-08-17T17:35:33.147283+0000","last_unstale":"2024-08-17T17:38:37.085793+0000","last_undegraded":"2024-08-17T17:38:37.085793+0000","last_fullsized":"2024-08-17T17:38:37.085793+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:41.677709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1658,"reported_epoch":1421,"state":"active+clean+scrubbing","last_fresh":"2024-08-17T17:38:46.140168+0000","last_change":"2024-08-17T17:38:50.961038+0000","last_active":"2024-08-17T17:38:46.140168+0000","last_peered":"2024-08-17T17:38:46.140168+0000","last_clean":"2024-08-17T17:38:46.140168+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:38:46.140168+0000","last_undegraded":"2024-08-17T17:38:46.140168+0000","last_fullsized":"2024-08-17T17:38:46.140168+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:38.241235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:38.241235+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"1421'5364","reported_seq":5437,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:52.317660+0000","last_change":"2024-08-17T17:34:50.670013+0000","last_active":"2024-08-17T17:38:52.317660+0000","last_peered":"2024-08-17T17:38:52.317660+0000","last_clean":"2024-08-17T17:38:52.317660+0000","last_became_active":"2024-08-17T17:34:50.669299+0000","last_became_peered":"2024-08-17T17:34:50.669299+0000","last_unstale":"2024-08-17T17:38:52.317660+0000","last_undegraded":"2024-08-17T17:38:52.317660+0000","last_fullsized":"2024-08-17T17:38:52.317660+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5364,"log_dups_size":0,"ondisk_log_size":5364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:11.172227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167903232,"num_objects":2562,"num_object_clones":0,"num_object_copies":7686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5364,"num_write_kb":253632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1379,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081347+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:38:37.081347+0000","last_peered":"2024-08-17T17:38:37.081347+0000","last_clean":"2024-08-17T17:38:37.081347+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:38:37.081347+0000","last_undegraded":"2024-08-17T17:38:37.081347+0000","last_fullsized":"2024-08-17T17:38:37.081347+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":149,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085286+0000","last_change":"2024-08-17T17:35:33.140725+0000","last_active":"2024-08-17T17:38:37.085286+0000","last_peered":"2024-08-17T17:38:37.085286+0000","last_clean":"2024-08-17T17:38:37.085286+0000","last_became_active":"2024-08-17T17:35:33.140406+0000","last_became_peered":"2024-08-17T17:35:33.140406+0000","last_unstale":"2024-08-17T17:38:37.085286+0000","last_undegraded":"2024-08-17T17:38:37.085286+0000","last_fullsized":"2024-08-17T17:38:37.085286+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:54:51.875571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1669,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085381+0000","last_change":"2024-08-17T17:35:33.138514+0000","last_active":"2024-08-17T17:38:37.085381+0000","last_peered":"2024-08-17T17:38:37.085381+0000","last_clean":"2024-08-17T17:38:37.085381+0000","last_became_active":"2024-08-17T17:35:33.137035+0000","last_became_peered":"2024-08-17T17:35:33.137035+0000","last_unstale":"2024-08-17T17:38:37.085381+0000","last_undegraded":"2024-08-17T17:38:37.085381+0000","last_fullsized":"2024-08-17T17:38:37.085381+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:50.565662+0000","scrub_duration":0.001487138,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1311,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.082871+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:38:37.082871+0000","last_peered":"2024-08-17T17:38:37.082871+0000","last_clean":"2024-08-17T17:38:37.082871+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:38:37.082871+0000","last_undegraded":"2024-08-17T17:38:37.082871+0000","last_fullsized":"2024-08-17T17:38:37.082871+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":89,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.140294+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:38:46.140294+0000","last_peered":"2024-08-17T17:38:46.140294+0000","last_clean":"2024-08-17T17:38:46.140294+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:38:46.140294+0000","last_undegraded":"2024-08-17T17:38:46.140294+0000","last_fullsized":"2024-08-17T17:38:46.140294+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1742,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081330+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:38:37.081330+0000","last_peered":"2024-08-17T17:38:37.081330+0000","last_clean":"2024-08-17T17:38:37.081330+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:38:37.081330+0000","last_undegraded":"2024-08-17T17:38:37.081330+0000","last_fullsized":"2024-08-17T17:38:37.081330+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1328,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.085255+0000","last_change":"2024-08-17T17:35:55.205292+0000","last_active":"2024-08-17T17:38:37.085255+0000","last_peered":"2024-08-17T17:38:37.085255+0000","last_clean":"2024-08-17T17:38:37.085255+0000","last_became_active":"2024-08-17T17:35:55.204935+0000","last_became_peered":"2024-08-17T17:35:55.204935+0000","last_unstale":"2024-08-17T17:38:37.085255+0000","last_undegraded":"2024-08-17T17:38:37.085255+0000","last_fullsized":"2024-08-17T17:38:37.085255+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:52.277821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":89,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.140411+0000","last_change":"2024-08-17T17:36:12.749972+0000","last_active":"2024-08-17T17:38:46.140411+0000","last_peered":"2024-08-17T17:38:46.140411+0000","last_clean":"2024-08-17T17:38:46.140411+0000","last_became_active":"2024-08-17T17:36:12.749620+0000","last_became_peered":"2024-08-17T17:36:12.749620+0000","last_unstale":"2024-08-17T17:38:46.140411+0000","last_undegraded":"2024-08-17T17:38:46.140411+0000","last_fullsized":"2024-08-17T17:38:46.140411+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:04.994357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":965,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.139568+0000","last_change":"2024-08-17T17:36:20.832274+0000","last_active":"2024-08-17T17:38:46.139568+0000","last_peered":"2024-08-17T17:38:46.139568+0000","last_clean":"2024-08-17T17:38:46.139568+0000","last_became_active":"2024-08-17T17:36:20.831886+0000","last_became_peered":"2024-08-17T17:36:20.831886+0000","last_unstale":"2024-08-17T17:38:46.139568+0000","last_undegraded":"2024-08-17T17:38:46.139568+0000","last_fullsized":"2024-08-17T17:38:46.139568+0000","mapping_epoch":1390,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1391,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:05.248711+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1174,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.103621+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:38:46.103621+0000","last_peered":"2024-08-17T17:38:46.103621+0000","last_clean":"2024-08-17T17:38:46.103621+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:38:46.103621+0000","last_undegraded":"2024-08-17T17:38:46.103621+0000","last_fullsized":"2024-08-17T17:38:46.103621+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1411,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081460+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:38:37.081460+0000","last_peered":"2024-08-17T17:38:37.081460+0000","last_clean":"2024-08-17T17:38:37.081460+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:38:37.081460+0000","last_undegraded":"2024-08-17T17:38:37.081460+0000","last_fullsized":"2024-08-17T17:38:37.081460+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1705,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081834+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:38:37.081834+0000","last_peered":"2024-08-17T17:38:37.081834+0000","last_clean":"2024-08-17T17:38:37.081834+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:38:37.081834+0000","last_undegraded":"2024-08-17T17:38:37.081834+0000","last_fullsized":"2024-08-17T17:38:37.081834+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1353,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.087061+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:38:37.087061+0000","last_peered":"2024-08-17T17:38:37.087061+0000","last_clean":"2024-08-17T17:38:37.087061+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:38:37.087061+0000","last_undegraded":"2024-08-17T17:38:37.087061+0000","last_fullsized":"2024-08-17T17:38:37.087061+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1893,"reported_epoch":1421,"state":"active+clean","last_fresh":"2024-08-17T17:38:46.140223+0000","last_change":"2024-08-17T17:35:33.137411+0000","last_active":"2024-08-17T17:38:46.140223+0000","last_peered":"2024-08-17T17:38:46.140223+0000","last_clean":"2024-08-17T17:38:46.140223+0000","last_became_active":"2024-08-17T17:35:33.137073+0000","last_became_peered":"2024-08-17T17:35:33.137073+0000","last_unstale":"2024-08-17T17:38:46.140223+0000","last_undegraded":"2024-08-17T17:38:46.140223+0000","last_fullsized":"2024-08-17T17:38:46.140223+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:35.924539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1799,"reported_epoch":1420,"state":"active+clean","last_fresh":"2024-08-17T17:38:37.081113+0000","last_change":"2024-08-17T17:36:15.787445+0000","last_active":"2024-08-17T17:38:37.081113+0000","last_peered":"2024-08-17T17:38:37.081113+0000","last_clean":"2024-08-17T17:38:37.081113+0000","last_became_active":"2024-08-17T17:36:15.787099+0000","last_became_peered":"2024-08-17T17:36:15.787099+0000","last_unstale":"2024-08-17T17:38:37.081113+0000","last_undegraded":"2024-08-17T17:38:37.081113+0000","last_fullsized":"2024-08-17T17:38:37.081113+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:47.448168+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":2681995288,"num_objects":40925,"num_object_clones":0,"num_object_copies":122775,"num_objects_missing_on_primary":4069,"num_objects_missing":4069,"num_objects_degraded":15885,"num_objects_misplaced":3777,"num_objects_unfound":0,"num_objects_dirty":40925,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":87957,"num_write_kb":4124161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29151,"num_bytes_recovered":1757806592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8812834816,"data_stored":8812822600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87857,"ondisk_log_size":87857,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635092,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":27,"num_read_kb":61,"num_write":15,"num_write_kb":183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1314910,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162858,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1389012,"kb_used_data":1234252,"kb_used_omap":54,"kb_used_meta":154697,"kb_avail":92356140,"statfs":{"total":95995035648,"available":94572687360,"internally_reserved":0,"allocated":1263874048,"data_stored":1261203888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":158409749},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999332,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1195860,"kb_used_data":1033740,"kb_used_omap":47,"kb_used_meta":162064,"kb_avail":92549292,"statfs":{"total":95995035648,"available":94770475008,"internally_reserved":0,"allocated":1058549760,"data_stored":1055880577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":165954196},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627128,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1392632,"kb_used_data":381744,"kb_used_omap":42,"kb_used_meta":1010837,"kb_avail":92352520,"statfs":{"total":95995035648,"available":94568980480,"internally_reserved":0,"allocated":390905856,"data_stored":388258954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1035097148},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785605,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2642876,"kb_used_data":1834548,"kb_used_omap":48,"kb_used_meta":808271,"kb_avail":91102276,"statfs":{"total":95995035648,"available":93288730624,"internally_reserved":0,"allocated":1878577152,"data_stored":1875927125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":827670190},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725122,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1541380,"kb_used_data":676028,"kb_used_omap":1,"kb_used_meta":865342,"kb_avail":92203772,"statfs":{"total":95995035648,"available":94416662528,"internally_reserved":0,"allocated":692252672,"data_stored":689594595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":886110662},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986759,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1927568,"kb_used_data":1044168,"kb_used_omap":1,"kb_used_meta":883390,"kb_avail":91817584,"statfs":{"total":95995035648,"available":94021206016,"internally_reserved":0,"allocated":1069228032,"data_stored":1066561378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":904591814},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706367,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2408516,"kb_used_data":1487996,"kb_used_omap":35,"kb_used_meta":920476,"kb_avail":91336636,"statfs":{"total":95995035648,"available":93528715264,"internally_reserved":0,"allocated":1523707904,"data_stored":1521049808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":942567479},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832140,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2032960,"kb_used_data":994040,"kb_used_omap":65,"kb_used_meta":1038846,"kb_avail":91712192,"statfs":{"total":95995035648,"available":93913284608,"internally_reserved":0,"allocated":1017896960,"data_stored":1015243422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1063778491},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1007812608,"data_stored":1007812608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1867448320,"data_stored":1867448320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1512181760,"data_stored":1512177688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":381419520,"data_stored":381419520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1058476032,"data_stored":1058471960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":680853504,"data_stored":680853504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1049366528,"data_stored":1049362456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1255276544,"data_stored":1255276544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:39:00.106 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:39:00.107 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:00.114 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:00.133 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:00.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:00.234 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:00.259 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:00.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:00.361 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:00.387 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:00.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:00.396 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:00 smithi063 ceph-mon[114756]: from='client.46370 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:00.396 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:00 smithi063 ceph-mon[114756]: pgmap v2234: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 203 op/s; 15885/122025 objects degraded (13.018%); 3777/122025 objects misplaced (3.095%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:39:00.396 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:00 smithi063 ceph-mon[109843]: from='client.46370 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:00.396 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:00 smithi063 ceph-mon[109843]: pgmap v2234: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 203 op/s; 15885/122025 objects degraded (13.018%); 3777/122025 objects misplaced (3.095%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:39:00.489 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:00.513 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:00.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:00.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:00 smithi162 ceph-mon[104668]: from='client.46370 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:00.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:00 smithi162 ceph-mon[104668]: pgmap v2234: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 203 op/s; 15885/122025 objects degraded (13.018%); 3777/122025 objects misplaced (3.095%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:39:00.615 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:00.640 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:00.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:00.742 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:00.766 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:00.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:00.821 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:00.857 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:00.868 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:00.891 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:00.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:00.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:00.993 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:01.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.016 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:01.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:01.117 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:01.144 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:01.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:01.245 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:01.272 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:01.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:01.374 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:01.401 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:01.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:01.502 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:01.525 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:01.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:01.627 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:01.650 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:01.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:01.679 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:39:01.752 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:01.778 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:01.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:01.880 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:01.903 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:01.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:01.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:02.005 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:02.032 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:02.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:02.134 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:02.161 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:02.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:02.263 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:02.290 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:02.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:02.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:02 smithi063 ceph-mon[114756]: pgmap v2235: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 204 op/s; 15885/121827 objects degraded (13.039%); 3777/121827 objects misplaced (3.100%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:39:02.293 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:02 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:02.293 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:02 smithi063 ceph-mon[109843]: pgmap v2235: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 204 op/s; 15885/121827 objects degraded (13.039%); 3777/121827 objects misplaced (3.100%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:39:02.293 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:02 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:02.392 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:02.420 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:02.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:02.522 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:02.552 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:02.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:02.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:02 smithi162 ceph-mon[104668]: pgmap v2235: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 204 op/s; 15885/121827 objects degraded (13.039%); 3777/121827 objects misplaced (3.100%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:39:02.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:02 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:02.654 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:02.679 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:02.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:02.780 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:02.803 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:02.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:02.905 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:02.929 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:02.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:02.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:03.031 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:03.032 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:39:03.032 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:03.051 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:39:03.052 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:03.062 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:03.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:03.070 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:39:03.070 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:03.088 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:39:03.089 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:03.108 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:39:03.108 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:03.131 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:39:03.132 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:03.152 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:39:03.152 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:03.164 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:03.171 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:39:03.172 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:03.190 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:03.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:03.292 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:03.315 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:03.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:03.360 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:03 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/659410342' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:03.361 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:03 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/659410342' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:03.361 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:39:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:39:03] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.51.0" 2024-08-17T17:39:03.417 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:03.444 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:03.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:03.546 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:03.568 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:03.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:03.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:03 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/659410342' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:03.670 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:03.694 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:03.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.694 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:03.795 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:03.821 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:03.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:03.922 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:03.950 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:03.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:03.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:04.051 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:04.075 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:04.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:04.177 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:04.202 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:04.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:04.304 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:04.331 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:04.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:04.433 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:04.459 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:04.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:04.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:04 smithi063 ceph-mon[114756]: pgmap v2236: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 213 op/s; 15452/119013 objects degraded (12.983%); 3681/119013 objects misplaced (3.093%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:39:04.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:04 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 15452/119013 objects degraded (12.983%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:04.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:04 smithi063 ceph-mon[109843]: pgmap v2236: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 213 op/s; 15452/119013 objects degraded (12.983%); 3681/119013 objects misplaced (3.093%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:39:04.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:04 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 15452/119013 objects degraded (12.983%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:04.561 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:04.584 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:04.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:04.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:04 smithi162 ceph-mon[104668]: pgmap v2236: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 213 op/s; 15452/119013 objects degraded (12.983%); 3681/119013 objects misplaced (3.093%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:39:04.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:04 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 15452/119013 objects degraded (12.983%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:04.686 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:04.711 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:04.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:04.813 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:04.835 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:04.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:04.937 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:04.970 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:04.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:04.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:05.072 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:05.096 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:05.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:05.197 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:05.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.222 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:05.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:05.323 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:05.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.346 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:05.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:05.447 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:05.475 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:05.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:05.489 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:39:05.489 INFO:teuthology.orchestra.run.smithi063.stdout: "size": 20, 2024-08-17T17:39:05.489 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 600, 2024-08-17T17:39:05.489 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [ 2024-08-17T17:39:05.490 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.490 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267379 10.f 10:f6c7ed9c:::benchmark_data_smithi063_294527_object130194:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1348)", 2024-08-17T17:39:05.490 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.680475+0000", 2024-08-17T17:39:05.490 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 284.72136901699997, 2024-08-17T17:39:05.490 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0741320969999999, 2024-08-17T17:39:05.490 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.490 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.490 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:39:05.490 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.490 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:39:05.490 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:05.490 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267379 2024-08-17T17:39:05.491 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.491 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.491 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.491 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.491 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680475+0000", 2024-08-17T17:39:05.491 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.491 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.491 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.491 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.491 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680475+0000", 2024-08-17T17:39:05.491 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.491 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.491 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.491 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.492 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680476+0000", 2024-08-17T17:39:05.492 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.403e-06 2024-08-17T17:39:05.492 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.492 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.492 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.492 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680481+0000", 2024-08-17T17:39:05.492 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2499999999999997e-06 2024-08-17T17:39:05.492 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.492 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.492 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.492 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680498+0000", 2024-08-17T17:39:05.492 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6568000000000001e-05 2024-08-17T17:39:05.492 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.493 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.493 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.493 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680511+0000", 2024-08-17T17:39:05.493 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3091000000000001e-05 2024-08-17T17:39:05.493 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.493 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.493 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.493 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680583+0000", 2024-08-17T17:39:05.493 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2421000000000003e-05 2024-08-17T17:39:05.493 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.493 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.493 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:39:05.493 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680703+0000", 2024-08-17T17:39:05.493 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0001192 2024-08-17T17:39:05.494 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.494 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.494 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:39:05.494 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.680808+0000", 2024-08-17T17:39:05.494 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010527999999999999 2024-08-17T17:39:05.494 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.494 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.494 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.494 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.685424+0000", 2024-08-17T17:39:05.494 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0046164320000000002 2024-08-17T17:39:05.494 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.494 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.494 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:39:05.495 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.687850+0000", 2024-08-17T17:39:05.495 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0024256009999999999 2024-08-17T17:39:05.495 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.495 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.495 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.495 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.693025+0000", 2024-08-17T17:39:05.495 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0051753099999999998 2024-08-17T17:39:05.495 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.495 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.495 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.495 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.693041+0000", 2024-08-17T17:39:05.495 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6231000000000001e-05 2024-08-17T17:39:05.495 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.495 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.496 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.496 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.754514+0000", 2024-08-17T17:39:05.496 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.061472953 2024-08-17T17:39:05.496 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.496 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.496 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.496 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.754607+0000", 2024-08-17T17:39:05.496 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.2356999999999997e-05 2024-08-17T17:39:05.496 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.496 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.496 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.496 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.496 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.496 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267392 10.f 10:f2eb3fe1:::benchmark_data_smithi063_294527_object130207:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:39:05.497 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.696596+0000", 2024-08-17T17:39:05.497 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 284.70524779599998, 2024-08-17T17:39:05.497 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.067108331, 2024-08-17T17:39:05.497 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.497 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.497 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:05.497 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.497 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:39:05.497 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:05.497 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267392 2024-08-17T17:39:05.497 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.497 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.498 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.498 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.498 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.696596+0000", 2024-08-17T17:39:05.498 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.498 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.498 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.498 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.498 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.696596+0000", 2024-08-17T17:39:05.498 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.498 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.498 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.498 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.696598+0000", 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7990000000000001e-06 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.696603+0000", 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4970000000000001e-06 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.696621+0000", 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7563000000000001e-05 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.499 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.696628+0000", 2024-08-17T17:39:05.500 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2949999999999996e-06 2024-08-17T17:39:05.500 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.500 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.500 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.500 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.697082+0000", 2024-08-17T17:39:05.500 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00045373100000000002 2024-08-17T17:39:05.500 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.500 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.500 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.500 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.697091+0000", 2024-08-17T17:39:05.500 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4909999999999994e-06 2024-08-17T17:39:05.500 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.500 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.500 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.501 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.755363+0000", 2024-08-17T17:39:05.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0582717079999999 2024-08-17T17:39:05.501 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.501 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.501 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:39:05.501 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.755462+0000", 2024-08-17T17:39:05.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.9500000000000006e-05 2024-08-17T17:39:05.501 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.501 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.501 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:39:05.501 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.755541+0000", 2024-08-17T17:39:05.501 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8914999999999999e-05 2024-08-17T17:39:05.501 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.501 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.502 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.502 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763339+0000", 2024-08-17T17:39:05.502 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0077973590000000002 2024-08-17T17:39:05.502 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.502 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.502 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.502 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763479+0000", 2024-08-17T17:39:05.502 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000139961 2024-08-17T17:39:05.502 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.502 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.502 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:39:05.502 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763540+0000", 2024-08-17T17:39:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1356e-05 2024-08-17T17:39:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:39:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763609+0000", 2024-08-17T17:39:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8900999999999996e-05 2024-08-17T17:39:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763704+0000", 2024-08-17T17:39:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.5254999999999995e-05 2024-08-17T17:39:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.503 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.504 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.504 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.504 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267452 10.f 10:f299a0d0:::benchmark_data_smithi063_294527_object130267:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:39:05.504 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.745106+0000", 2024-08-17T17:39:05.504 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 284.65673748400002, 2024-08-17T17:39:05.504 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.018779831, 2024-08-17T17:39:05.504 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.504 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.504 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:05.504 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.504 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:39:05.504 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:05.504 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267452 2024-08-17T17:39:05.505 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.505 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.505 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.505 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.505 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745106+0000", 2024-08-17T17:39:05.505 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.505 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.505 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.505 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.505 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745106+0000", 2024-08-17T17:39:05.505 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.505 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.505 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.505 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.506 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745108+0000", 2024-08-17T17:39:05.506 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8279999999999999e-06 2024-08-17T17:39:05.506 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.506 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.506 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.506 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745115+0000", 2024-08-17T17:39:05.506 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.9330000000000004e-06 2024-08-17T17:39:05.506 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.506 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.506 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.506 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745132+0000", 2024-08-17T17:39:05.506 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6818000000000001e-05 2024-08-17T17:39:05.506 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.507 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.507 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.507 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745141+0000", 2024-08-17T17:39:05.507 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.7940000000000008e-06 2024-08-17T17:39:05.507 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.507 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.507 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.507 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745201+0000", 2024-08-17T17:39:05.507 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0409999999999999e-05 2024-08-17T17:39:05.507 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.507 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.507 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.507 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.745208+0000", 2024-08-17T17:39:05.507 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.0119999999999999e-06 2024-08-17T17:39:05.508 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.508 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.508 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.508 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.759735+0000", 2024-08-17T17:39:05.508 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0145268999999999 2024-08-17T17:39:05.508 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.508 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.508 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:39:05.508 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.759830+0000", 2024-08-17T17:39:05.508 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.5042000000000002e-05 2024-08-17T17:39:05.508 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.508 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.508 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:39:05.508 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.759902+0000", 2024-08-17T17:39:05.509 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2287999999999996e-05 2024-08-17T17:39:05.509 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.509 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.509 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.509 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763526+0000", 2024-08-17T17:39:05.509 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0036238720000000002 2024-08-17T17:39:05.509 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.509 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.509 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:39:05.509 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763719+0000", 2024-08-17T17:39:05.509 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00019284799999999999 2024-08-17T17:39:05.509 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.509 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.510 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.510 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763758+0000", 2024-08-17T17:39:05.510 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.9137999999999997e-05 2024-08-17T17:39:05.510 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.510 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.510 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:39:05.510 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763811+0000", 2024-08-17T17:39:05.510 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.3329999999999999e-05 2024-08-17T17:39:05.510 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.510 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.510 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.510 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763886+0000", 2024-08-17T17:39:05.510 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4617999999999996e-05 2024-08-17T17:39:05.511 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.511 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.511 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.511 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.511 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.511 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267466 10.f 10:f92259a0:::benchmark_data_smithi063_294527_object130281:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:39:05.511 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.757244+0000", 2024-08-17T17:39:05.511 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 284.64459997199998, 2024-08-17T17:39:05.511 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.015260455, 2024-08-17T17:39:05.511 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.512 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.512 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:05.512 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.512 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:39:05.512 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:05.512 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267466 2024-08-17T17:39:05.512 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.512 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.512 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.512 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.512 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757244+0000", 2024-08-17T17:39:05.512 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.512 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.512 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.513 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.513 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757244+0000", 2024-08-17T17:39:05.513 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.513 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.513 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.513 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.513 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757246+0000", 2024-08-17T17:39:05.513 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.294e-06 2024-08-17T17:39:05.513 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.513 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.513 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.513 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757253+0000", 2024-08-17T17:39:05.513 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8940000000000004e-06 2024-08-17T17:39:05.514 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.514 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.514 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.514 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757270+0000", 2024-08-17T17:39:05.514 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7520000000000002e-05 2024-08-17T17:39:05.514 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.514 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.514 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.514 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757280+0000", 2024-08-17T17:39:05.514 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3670000000000003e-06 2024-08-17T17:39:05.514 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.514 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.514 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.514 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757341+0000", 2024-08-17T17:39:05.515 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1328000000000006e-05 2024-08-17T17:39:05.515 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.515 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.515 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.515 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.757352+0000", 2024-08-17T17:39:05.515 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1072e-05 2024-08-17T17:39:05.515 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.515 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.515 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.515 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.763943+0000", 2024-08-17T17:39:05.515 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.006590825 2024-08-17T17:39:05.515 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.515 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.516 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:39:05.516 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.764061+0000", 2024-08-17T17:39:05.516 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000117793 2024-08-17T17:39:05.516 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.516 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.516 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:39:05.516 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.764180+0000", 2024-08-17T17:39:05.516 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000118709 2024-08-17T17:39:05.516 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.516 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.516 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.516 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772142+0000", 2024-08-17T17:39:05.516 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0079620249999999993 2024-08-17T17:39:05.516 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.517 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.517 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772235+0000", 2024-08-17T17:39:05.517 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3037999999999999e-05 2024-08-17T17:39:05.517 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.517 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:39:05.517 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772339+0000", 2024-08-17T17:39:05.517 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010490299999999999 2024-08-17T17:39:05.517 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.517 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.517 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:39:05.517 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772402+0000", 2024-08-17T17:39:05.518 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2700000000000006e-05 2024-08-17T17:39:05.518 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.518 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.518 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.518 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772504+0000", 2024-08-17T17:39:05.518 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000101987 2024-08-17T17:39:05.518 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.518 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.518 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.518 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.518 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.518 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267538 10.f 10:f588c7f4:::benchmark_data_smithi063_294527_object130353:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:39:05.519 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.818626+0000", 2024-08-17T17:39:05.519 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 284.58321760600001, 2024-08-17T17:39:05.519 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95837228299999999, 2024-08-17T17:39:05.519 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.519 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.519 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:05.519 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.519 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:39:05.519 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:05.519 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267538 2024-08-17T17:39:05.519 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.519 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.520 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.520 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.520 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818626+0000", 2024-08-17T17:39:05.520 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.520 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.520 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.520 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.520 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818626+0000", 2024-08-17T17:39:05.520 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.520 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.520 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.520 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.520 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818628+0000", 2024-08-17T17:39:05.520 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9740000000000001e-06 2024-08-17T17:39:05.521 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.521 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.521 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.521 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818636+0000", 2024-08-17T17:39:05.521 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8949999999999998e-06 2024-08-17T17:39:05.521 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.521 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.521 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.521 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818657+0000", 2024-08-17T17:39:05.521 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1498e-05 2024-08-17T17:39:05.521 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.521 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.521 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.521 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818667+0000", 2024-08-17T17:39:05.522 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.6770000000000007e-06 2024-08-17T17:39:05.522 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.522 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.522 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.522 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818742+0000", 2024-08-17T17:39:05.522 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4572000000000004e-05 2024-08-17T17:39:05.522 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.522 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.522 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.522 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.818751+0000", 2024-08-17T17:39:05.522 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7510000000000003e-06 2024-08-17T17:39:05.522 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.522 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.522 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.523 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.767638+0000", 2024-08-17T17:39:05.523 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.94888622300000003 2024-08-17T17:39:05.523 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.523 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.523 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:39:05.523 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.767770+0000", 2024-08-17T17:39:05.523 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013257300000000001 2024-08-17T17:39:05.523 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.523 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.523 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:39:05.523 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.767883+0000", 2024-08-17T17:39:05.523 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000112569 2024-08-17T17:39:05.524 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.524 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.524 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.524 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772322+0000", 2024-08-17T17:39:05.524 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044396710000000001 2024-08-17T17:39:05.524 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.524 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.524 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.524 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.776814+0000", 2024-08-17T17:39:05.524 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0044919829999999997 2024-08-17T17:39:05.524 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.524 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.524 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:39:05.524 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.776837+0000", 2024-08-17T17:39:05.524 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2761000000000001e-05 2024-08-17T17:39:05.525 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.525 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.525 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:39:05.525 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.776898+0000", 2024-08-17T17:39:05.525 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.0705999999999998e-05 2024-08-17T17:39:05.525 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.525 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.525 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.525 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.776998+0000", 2024-08-17T17:39:05.525 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010043 2024-08-17T17:39:05.525 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.525 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.525 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.525 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.525 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.526 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267541 10.f 10:fa0f1a18:::benchmark_data_smithi063_294527_object130356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:39:05.526 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.820949+0000", 2024-08-17T17:39:05.526 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 284.58089509000001, 2024-08-17T17:39:05.526 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.96002311799999995, 2024-08-17T17:39:05.526 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.526 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.526 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:05.526 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.526 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:39:05.526 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:05.526 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267541 2024-08-17T17:39:05.526 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.526 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.820949+0000", 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.820949+0000", 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.820950+0000", 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.31e-06 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.527 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.528 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.528 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.820955+0000", 2024-08-17T17:39:05.528 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9840000000000001e-06 2024-08-17T17:39:05.528 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.528 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.528 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.528 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.820969+0000", 2024-08-17T17:39:05.528 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3891e-05 2024-08-17T17:39:05.528 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.528 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.528 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.528 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.820976+0000", 2024-08-17T17:39:05.528 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6899999999999992e-06 2024-08-17T17:39:05.528 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.821034+0000", 2024-08-17T17:39:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.7296999999999997e-05 2024-08-17T17:39:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.821042+0000", 2024-08-17T17:39:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8900000000000007e-06 2024-08-17T17:39:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772561+0000", 2024-08-17T17:39:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95151887499999999 2024-08-17T17:39:05.529 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.530 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.530 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:39:05.530 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772658+0000", 2024-08-17T17:39:05.530 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.7894999999999997e-05 2024-08-17T17:39:05.530 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.530 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.530 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:39:05.530 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.772760+0000", 2024-08-17T17:39:05.530 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010155200000000001 2024-08-17T17:39:05.530 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.530 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.530 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.530 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777043+0000", 2024-08-17T17:39:05.530 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0042826269999999998 2024-08-17T17:39:05.530 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.531 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.531 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:39:05.531 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777055+0000", 2024-08-17T17:39:05.531 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2758000000000001e-05 2024-08-17T17:39:05.531 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.531 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.531 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.531 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.780809+0000", 2024-08-17T17:39:05.531 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0037537180000000001 2024-08-17T17:39:05.531 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.531 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.531 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:39:05.531 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.780855+0000", 2024-08-17T17:39:05.531 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6026000000000003e-05 2024-08-17T17:39:05.532 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.532 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.532 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.532 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.780972+0000", 2024-08-17T17:39:05.532 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000116605 2024-08-17T17:39:05.532 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.532 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.532 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.532 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.532 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.532 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267557 10.f 10:f1c06cfb:::benchmark_data_smithi063_294527_object130372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:39:05.532 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.835168+0000", 2024-08-17T17:39:05.532 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 284.56667531699998, 2024-08-17T17:39:05.533 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.954258312, 2024-08-17T17:39:05.533 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.533 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.533 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:05.533 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.533 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:39:05.533 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:05.533 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267557 2024-08-17T17:39:05.533 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.533 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.533 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.533 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.533 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835168+0000", 2024-08-17T17:39:05.534 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.534 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.534 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.534 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.534 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835168+0000", 2024-08-17T17:39:05.534 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.534 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.534 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.534 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.534 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835170+0000", 2024-08-17T17:39:05.534 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.669e-06 2024-08-17T17:39:05.534 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.534 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.535 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.535 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835176+0000", 2024-08-17T17:39:05.535 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3620000000000002e-06 2024-08-17T17:39:05.535 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.535 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.535 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.535 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835193+0000", 2024-08-17T17:39:05.535 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.6181e-05 2024-08-17T17:39:05.535 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.535 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.535 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.535 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835200+0000", 2024-08-17T17:39:05.536 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.5279999999999998e-06 2024-08-17T17:39:05.536 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.536 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.536 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.536 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835276+0000", 2024-08-17T17:39:05.536 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6150999999999996e-05 2024-08-17T17:39:05.536 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.536 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.536 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.536 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.835284+0000", 2024-08-17T17:39:05.536 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.038e-06 2024-08-17T17:39:05.536 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.536 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.537 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.537 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777083+0000", 2024-08-17T17:39:05.537 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.94179912300000002 2024-08-17T17:39:05.537 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.537 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.537 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:39:05.537 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777200+0000", 2024-08-17T17:39:05.537 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011640400000000001 2024-08-17T17:39:05.537 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.537 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.537 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:39:05.537 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.777266+0000", 2024-08-17T17:39:05.537 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6346000000000005e-05 2024-08-17T17:39:05.537 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.537 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.538 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.538 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.781026+0000", 2024-08-17T17:39:05.538 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0037601959999999999 2024-08-17T17:39:05.538 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.538 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.538 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:39:05.538 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789265+0000", 2024-08-17T17:39:05.538 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0082385630000000008 2024-08-17T17:39:05.538 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.538 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.538 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.538 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789306+0000", 2024-08-17T17:39:05.538 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.0571999999999997e-05 2024-08-17T17:39:05.539 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.539 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.539 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:39:05.539 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789352+0000", 2024-08-17T17:39:05.539 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6347999999999997e-05 2024-08-17T17:39:05.539 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.539 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.539 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.539 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789427+0000", 2024-08-17T17:39:05.539 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.4831000000000002e-05 2024-08-17T17:39:05.539 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.539 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.539 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.539 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.539 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.540 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267575 10.f 10:fab61f3b:::benchmark_data_smithi063_294527_object130390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:39:05.540 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.846798+0000", 2024-08-17T17:39:05.540 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 284.55504539499998, 2024-08-17T17:39:05.540 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.942820518, 2024-08-17T17:39:05.540 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.540 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.540 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:05.540 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.540 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:39:05.540 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:05.540 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267575 2024-08-17T17:39:05.540 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.540 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.541 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.541 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.541 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846798+0000", 2024-08-17T17:39:05.541 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.541 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.541 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.541 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.541 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846798+0000", 2024-08-17T17:39:05.541 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.541 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.541 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.541 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.541 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846800+0000", 2024-08-17T17:39:05.541 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.86e-06 2024-08-17T17:39:05.541 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.542 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.542 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.542 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846807+0000", 2024-08-17T17:39:05.542 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6089999999999997e-06 2024-08-17T17:39:05.542 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.542 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.542 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.542 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846828+0000", 2024-08-17T17:39:05.542 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0812e-05 2024-08-17T17:39:05.542 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.542 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.542 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.542 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846839+0000", 2024-08-17T17:39:05.543 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0969e-05 2024-08-17T17:39:05.543 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.543 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.543 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.543 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846893+0000", 2024-08-17T17:39:05.543 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4716999999999998e-05 2024-08-17T17:39:05.543 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.543 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.543 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.543 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.846900+0000", 2024-08-17T17:39:05.543 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8499999999999996e-06 2024-08-17T17:39:05.543 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.543 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.543 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.544 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.781090+0000", 2024-08-17T17:39:05.544 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.93418942699999996 2024-08-17T17:39:05.544 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.544 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.544 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:39:05.544 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.781222+0000", 2024-08-17T17:39:05.544 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00013291099999999999 2024-08-17T17:39:05.544 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.544 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.544 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:39:05.544 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.781363+0000", 2024-08-17T17:39:05.544 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00014084199999999999 2024-08-17T17:39:05.544 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.544 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.545 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789243+0000", 2024-08-17T17:39:05.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0078801830000000007 2024-08-17T17:39:05.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.545 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.545 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:39:05.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789497+0000", 2024-08-17T17:39:05.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000253687 2024-08-17T17:39:05.545 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.545 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.545 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.545 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789534+0000", 2024-08-17T17:39:05.545 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.6947999999999999e-05 2024-08-17T17:39:05.546 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.546 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.546 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:39:05.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789563+0000", 2024-08-17T17:39:05.546 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8702999999999999e-05 2024-08-17T17:39:05.546 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.546 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.546 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.546 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789619+0000", 2024-08-17T17:39:05.546 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.5999999999999999e-05 2024-08-17T17:39:05.546 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.546 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.546 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.547 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.547 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.547 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267580 10.f 10:ff03e0ba:::benchmark_data_smithi063_294527_object130395:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:39:05.547 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.852353+0000", 2024-08-17T17:39:05.547 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 284.54949022900001, 2024-08-17T17:39:05.547 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.93742878200000002, 2024-08-17T17:39:05.547 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.547 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.547 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:05.547 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.547 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:39:05.547 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:05.547 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267580 2024-08-17T17:39:05.548 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.548 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.548 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.548 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.548 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852353+0000", 2024-08-17T17:39:05.548 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.548 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.548 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.548 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.548 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852353+0000", 2024-08-17T17:39:05.548 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.548 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.549 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.549 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.549 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852355+0000", 2024-08-17T17:39:05.549 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.4980000000000001e-06 2024-08-17T17:39:05.549 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.549 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.549 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.549 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852360+0000", 2024-08-17T17:39:05.549 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4310000000000002e-06 2024-08-17T17:39:05.549 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.549 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.549 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.549 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852376+0000", 2024-08-17T17:39:05.549 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5373999999999999e-05 2024-08-17T17:39:05.550 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.550 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.550 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.550 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852384+0000", 2024-08-17T17:39:05.550 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8390000000000007e-06 2024-08-17T17:39:05.550 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.550 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.550 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.550 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852452+0000", 2024-08-17T17:39:05.550 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8458000000000001e-05 2024-08-17T17:39:05.550 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.550 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.550 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.852463+0000", 2024-08-17T17:39:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0448999999999999e-05 2024-08-17T17:39:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.785113+0000", 2024-08-17T17:39:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.93265094699999995 2024-08-17T17:39:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:39:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.785224+0000", 2024-08-17T17:39:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011068799999999999 2024-08-17T17:39:05.551 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.552 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.552 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:39:05.552 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.785345+0000", 2024-08-17T17:39:05.552 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000121298 2024-08-17T17:39:05.552 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.552 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.552 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.552 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789481+0000", 2024-08-17T17:39:05.552 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0041354390000000003 2024-08-17T17:39:05.552 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.552 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.552 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.552 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789670+0000", 2024-08-17T17:39:05.553 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00018873300000000001 2024-08-17T17:39:05.553 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.553 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.553 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:39:05.553 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789678+0000", 2024-08-17T17:39:05.553 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.9289999999999998e-06 2024-08-17T17:39:05.553 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.553 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.553 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:39:05.553 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789705+0000", 2024-08-17T17:39:05.553 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.7120000000000001e-05 2024-08-17T17:39:05.553 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.553 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.553 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.554 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789782+0000", 2024-08-17T17:39:05.554 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7578999999999997e-05 2024-08-17T17:39:05.554 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.554 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.554 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.554 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.554 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.554 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267583 10.f 10:f7f0ea77:::benchmark_data_smithi063_294527_object130398:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:39:05.554 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.853732+0000", 2024-08-17T17:39:05.554 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 284.54811172199999, 2024-08-17T17:39:05.554 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95321229200000002, 2024-08-17T17:39:05.554 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267583 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853732+0000", 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.555 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853732+0000", 2024-08-17T17:39:05.556 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.556 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.556 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.556 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.556 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853733+0000", 2024-08-17T17:39:05.556 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.046e-06 2024-08-17T17:39:05.556 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.556 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.556 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.556 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853738+0000", 2024-08-17T17:39:05.556 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.6290000000000001e-06 2024-08-17T17:39:05.556 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.556 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.557 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.557 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853754+0000", 2024-08-17T17:39:05.557 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.5928000000000001e-05 2024-08-17T17:39:05.557 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.557 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.557 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.557 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853761+0000", 2024-08-17T17:39:05.557 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1890000000000001e-06 2024-08-17T17:39:05.557 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.557 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.557 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.557 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853820+0000", 2024-08-17T17:39:05.557 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9242e-05 2024-08-17T17:39:05.557 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.558 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.558 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.558 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.853828+0000", 2024-08-17T17:39:05.558 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.4579999999999993e-06 2024-08-17T17:39:05.558 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.558 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.558 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.558 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789813+0000", 2024-08-17T17:39:05.558 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.93598451400000005 2024-08-17T17:39:05.558 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.558 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.558 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:39:05.558 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.789927+0000", 2024-08-17T17:39:05.558 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000114324 2024-08-17T17:39:05.559 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.559 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.559 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:39:05.559 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.790017+0000", 2024-08-17T17:39:05.559 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.0160000000000004e-05 2024-08-17T17:39:05.559 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.559 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.559 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.559 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.798343+0000", 2024-08-17T17:39:05.559 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0083259660000000006 2024-08-17T17:39:05.559 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.559 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.559 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.560 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.798372+0000", 2024-08-17T17:39:05.560 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8609e-05 2024-08-17T17:39:05.560 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.560 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.560 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:39:05.560 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.806451+0000", 2024-08-17T17:39:05.560 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0080793810000000001 2024-08-17T17:39:05.560 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.560 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.560 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:39:05.560 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.806698+0000", 2024-08-17T17:39:05.560 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00024614800000000002 2024-08-17T17:39:05.560 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.561 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.561 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.561 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.806944+0000", 2024-08-17T17:39:05.561 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000246698 2024-08-17T17:39:05.561 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.561 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.561 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.561 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.561 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.561 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267588 10.f 10:ff638821:::benchmark_data_smithi063_294527_object130403:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:39:05.561 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.857910+0000", 2024-08-17T17:39:05.561 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 284.54393340000001, 2024-08-17T17:39:05.562 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95390986700000002, 2024-08-17T17:39:05.562 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.562 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.562 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:05.562 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.562 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:39:05.562 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:05.562 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267588 2024-08-17T17:39:05.562 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.562 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.562 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.562 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.562 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.857910+0000", 2024-08-17T17:39:05.563 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.563 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.563 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.563 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.563 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.857910+0000", 2024-08-17T17:39:05.563 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.563 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.563 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.563 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.563 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.857912+0000", 2024-08-17T17:39:05.563 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.584e-06 2024-08-17T17:39:05.563 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.563 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.563 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.564 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.857916+0000", 2024-08-17T17:39:05.564 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.5229999999999997e-06 2024-08-17T17:39:05.564 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.564 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.564 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.564 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.857935+0000", 2024-08-17T17:39:05.564 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.8380999999999999e-05 2024-08-17T17:39:05.564 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.564 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.564 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.564 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.857943+0000", 2024-08-17T17:39:05.564 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.1009999999999992e-06 2024-08-17T17:39:05.565 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.565 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.565 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.565 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.858006+0000", 2024-08-17T17:39:05.565 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.2819999999999998e-05 2024-08-17T17:39:05.565 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.565 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.565 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.565 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.858015+0000", 2024-08-17T17:39:05.565 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.2390000000000008e-06 2024-08-17T17:39:05.565 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.565 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.565 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.565 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.793977+0000", 2024-08-17T17:39:05.566 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.93596159899999998 2024-08-17T17:39:05.566 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.566 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.566 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:39:05.566 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.794084+0000", 2024-08-17T17:39:05.566 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00010793099999999999 2024-08-17T17:39:05.566 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.566 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.566 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:39:05.566 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.794194+0000", 2024-08-17T17:39:05.566 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000109067 2024-08-17T17:39:05.566 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.566 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.566 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.567 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.798432+0000", 2024-08-17T17:39:05.567 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0042388269999999997 2024-08-17T17:39:05.567 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.567 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.567 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:39:05.567 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.806955+0000", 2024-08-17T17:39:05.567 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0085225140000000001 2024-08-17T17:39:05.567 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.567 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.567 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.567 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811658+0000", 2024-08-17T17:39:05.567 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0047027170000000004 2024-08-17T17:39:05.567 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.567 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.567 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:39:05.568 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811719+0000", 2024-08-17T17:39:05.568 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.1008999999999998e-05 2024-08-17T17:39:05.568 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.568 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.568 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.568 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811820+0000", 2024-08-17T17:39:05.568 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000101555 2024-08-17T17:39:05.568 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.568 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.568 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.568 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.568 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.568 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267591 10.f 10:f43063b1:::benchmark_data_smithi063_294527_object130406:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:39:05.568 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.861930+0000", 2024-08-17T17:39:05.568 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 284.539913652, 2024-08-17T17:39:05.569 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95045635799999995, 2024-08-17T17:39:05.569 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.569 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.569 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:05.569 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.569 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:39:05.569 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:05.569 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267591 2024-08-17T17:39:05.569 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.569 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.569 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.569 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.569 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.861930+0000", 2024-08-17T17:39:05.569 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.569 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.570 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.570 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.570 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.861930+0000", 2024-08-17T17:39:05.570 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.570 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.570 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.570 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.570 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.861932+0000", 2024-08-17T17:39:05.570 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.7179999999999999e-06 2024-08-17T17:39:05.570 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.570 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.570 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.570 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.861938+0000", 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.4620000000000001e-06 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.861957+0000", 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9267999999999999e-05 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.861965+0000", 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8320000000000006e-06 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.862025+0000", 2024-08-17T17:39:05.571 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.9435000000000003e-05 2024-08-17T17:39:05.572 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.572 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.572 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.572 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.862031+0000", 2024-08-17T17:39:05.572 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.3609999999999997e-06 2024-08-17T17:39:05.572 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.572 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.572 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.572 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.798461+0000", 2024-08-17T17:39:05.572 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.93642974599999995 2024-08-17T17:39:05.572 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.572 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.572 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:39:05.572 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.798565+0000", 2024-08-17T17:39:05.573 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000104264 2024-08-17T17:39:05.573 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.573 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.573 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:39:05.573 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.798688+0000", 2024-08-17T17:39:05.573 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00012238099999999999 2024-08-17T17:39:05.573 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.573 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.573 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.573 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811574+0000", 2024-08-17T17:39:05.573 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.012886806000000001 2024-08-17T17:39:05.573 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.574 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.574 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:39:05.574 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811601+0000", 2024-08-17T17:39:05.574 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6605e-05 2024-08-17T17:39:05.574 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.574 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.574 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.574 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811960+0000", 2024-08-17T17:39:05.574 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000359086 2024-08-17T17:39:05.574 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.574 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.574 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:39:05.575 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.812014+0000", 2024-08-17T17:39:05.575 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4048999999999997e-05 2024-08-17T17:39:05.575 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.575 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.575 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.575 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.812386+0000", 2024-08-17T17:39:05.575 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000372345 2024-08-17T17:39:05.575 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.575 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.575 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.575 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.575 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.575 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267604 10.f 10:fae658d4:::benchmark_data_smithi063_294527_object130419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:39:05.576 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.872469+0000", 2024-08-17T17:39:05.576 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 284.529374206, 2024-08-17T17:39:05.576 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.94023317100000003, 2024-08-17T17:39:05.576 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.576 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.576 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:05.576 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.576 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:39:05.576 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:05.576 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267604 2024-08-17T17:39:05.576 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.576 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.576 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.576 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.576 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872469+0000", 2024-08-17T17:39:05.577 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.577 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.577 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.577 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.577 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872469+0000", 2024-08-17T17:39:05.577 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.577 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.577 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.577 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.577 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872471+0000", 2024-08-17T17:39:05.577 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.55e-06 2024-08-17T17:39:05.577 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.577 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.577 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.578 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872476+0000", 2024-08-17T17:39:05.578 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.1449999999999999e-06 2024-08-17T17:39:05.578 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.578 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.578 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.578 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872490+0000", 2024-08-17T17:39:05.578 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3725e-05 2024-08-17T17:39:05.578 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.578 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.578 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.578 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872499+0000", 2024-08-17T17:39:05.578 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3349999999999996e-06 2024-08-17T17:39:05.578 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.578 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.579 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.579 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872554+0000", 2024-08-17T17:39:05.579 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.4503000000000003e-05 2024-08-17T17:39:05.579 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.579 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.579 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.579 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.872563+0000", 2024-08-17T17:39:05.579 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4439999999999997e-06 2024-08-17T17:39:05.579 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.579 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.579 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.579 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.802449+0000", 2024-08-17T17:39:05.579 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.92988571900000005 2024-08-17T17:39:05.579 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.579 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.580 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:39:05.580 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.802561+0000", 2024-08-17T17:39:05.580 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000112218 2024-08-17T17:39:05.580 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.580 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.580 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:39:05.580 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.802673+0000", 2024-08-17T17:39:05.580 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011205900000000001 2024-08-17T17:39:05.580 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.580 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.580 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:39:05.580 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811607+0000", 2024-08-17T17:39:05.581 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0089336009999999993 2024-08-17T17:39:05.581 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.581 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.581 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.581 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811879+0000", 2024-08-17T17:39:05.581 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00027259400000000003 2024-08-17T17:39:05.581 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.581 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.581 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.581 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.812523+0000", 2024-08-17T17:39:05.581 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00064338100000000005 2024-08-17T17:39:05.581 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.581 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.581 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:39:05.581 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.812589+0000", 2024-08-17T17:39:05.582 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.6017000000000002e-05 2024-08-17T17:39:05.582 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.582 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.582 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.582 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.812703+0000", 2024-08-17T17:39:05.582 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011388 2024-08-17T17:39:05.582 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.582 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.582 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.582 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.582 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.582 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.45818.0:267639 10.f 10:fc605646:::benchmark_data_smithi063_294527_object130454:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1349)", 2024-08-17T17:39:05.582 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:34:20.921459+0000", 2024-08-17T17:39:05.583 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 284.48038459399999, 2024-08-17T17:39:05.583 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.90069574100000005, 2024-08-17T17:39:05.583 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.583 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.583 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:05.583 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.583 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.45818", 2024-08-17T17:39:05.583 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:05.583 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 267639 2024-08-17T17:39:05.583 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.583 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.583 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.583 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.583 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921459+0000", 2024-08-17T17:39:05.584 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.584 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.584 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.584 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.584 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921459+0000", 2024-08-17T17:39:05.584 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.584 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.584 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.584 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.584 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921461+0000", 2024-08-17T17:39:05.584 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3489999999999999e-06 2024-08-17T17:39:05.584 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.584 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.584 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.585 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921471+0000", 2024-08-17T17:39:05.585 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.3640000000000005e-06 2024-08-17T17:39:05.585 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.585 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.585 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.585 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921492+0000", 2024-08-17T17:39:05.585 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.1596000000000001e-05 2024-08-17T17:39:05.585 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.585 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.585 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.585 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921504+0000", 2024-08-17T17:39:05.585 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.1153e-05 2024-08-17T17:39:05.585 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.585 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.586 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.586 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921571+0000", 2024-08-17T17:39:05.586 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.7308e-05 2024-08-17T17:39:05.586 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.586 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.586 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.586 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:20.921581+0000", 2024-08-17T17:39:05.586 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0497e-05 2024-08-17T17:39:05.586 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.586 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.586 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.586 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.807005+0000", 2024-08-17T17:39:05.586 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.88542341400000002 2024-08-17T17:39:05.586 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.587 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.587 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "started", 2024-08-17T17:39:05.587 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.807122+0000", 2024-08-17T17:39:05.587 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00011692300000000001 2024-08-17T17:39:05.587 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.587 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.587 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:39:05.587 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.807232+0000", 2024-08-17T17:39:05.587 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000110369 2024-08-17T17:39:05.587 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.587 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.587 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "op_commit", 2024-08-17T17:39:05.587 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.811893+0000", 2024-08-17T17:39:05.587 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00466103 2024-08-17T17:39:05.588 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.588 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.588 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.588 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.812464+0000", 2024-08-17T17:39:05.588 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.00057084599999999996 2024-08-17T17:39:05.588 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.588 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.588 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:05.588 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.821975+0000", 2024-08-17T17:39:05.588 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.0095109449999999998 2024-08-17T17:39:05.588 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.588 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.588 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "commit_sent", 2024-08-17T17:39:05.588 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.822046+0000", 2024-08-17T17:39:05.588 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1000999999999993e-05 2024-08-17T17:39:05.589 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.589 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.589 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.589 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:34:21.822155+0000", 2024-08-17T17:39:05.589 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.000108946 2024-08-17T17:39:05.589 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.589 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.589 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.589 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.589 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.589 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:62282 11.b 11.5c83f35b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1382)", 2024-08-17T17:39:05.589 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:36:11.761578+0000", 2024-08-17T17:39:05.589 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 173.64026560400001, 2024-08-17T17:39:05.590 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98266562899999998, 2024-08-17T17:39:05.590 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.590 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.590 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:39:05.590 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.590 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:39:05.590 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:05.590 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 62282 2024-08-17T17:39:05.590 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.590 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.590 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.590 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.590 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.761578+0000", 2024-08-17T17:39:05.590 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.591 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.591 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.591 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.591 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.761578+0000", 2024-08-17T17:39:05.591 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.591 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.591 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.591 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.591 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.761581+0000", 2024-08-17T17:39:05.591 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.678e-06 2024-08-17T17:39:05.591 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.591 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.591 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.592 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.761609+0000", 2024-08-17T17:39:05.592 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8195e-05 2024-08-17T17:39:05.592 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.592 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.592 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.592 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.761620+0000", 2024-08-17T17:39:05.592 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0909e-05 2024-08-17T17:39:05.592 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.592 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.592 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.592 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.761629+0000", 2024-08-17T17:39:05.592 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.2609999999999999e-06 2024-08-17T17:39:05.592 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.592 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.592 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.593 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.761698+0000", 2024-08-17T17:39:05.593 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8794000000000006e-05 2024-08-17T17:39:05.593 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.593 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.593 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.593 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.761707+0000", 2024-08-17T17:39:05.593 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.1500000000000005e-06 2024-08-17T17:39:05.593 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.593 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.593 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.593 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.744234+0000", 2024-08-17T17:39:05.593 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.98252719499999996 2024-08-17T17:39:05.593 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.593 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.594 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.594 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.744244+0000", 2024-08-17T17:39:05.594 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.4469999999999995e-06 2024-08-17T17:39:05.594 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.594 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.594 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.594 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.594 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.594 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:62286 11.b 11.57b6d28b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1382)", 2024-08-17T17:39:05.594 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:36:11.767574+0000", 2024-08-17T17:39:05.594 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 173.63427008100001, 2024-08-17T17:39:05.594 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.97670556100000006, 2024-08-17T17:39:05.594 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.595 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.595 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:39:05.595 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.595 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:39:05.595 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:05.595 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 62286 2024-08-17T17:39:05.595 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.595 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.595 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.595 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.595 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.767574+0000", 2024-08-17T17:39:05.595 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.595 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.595 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.596 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.596 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.767574+0000", 2024-08-17T17:39:05.596 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.596 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.596 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.596 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.596 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.767575+0000", 2024-08-17T17:39:05.596 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.641e-06 2024-08-17T17:39:05.596 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.596 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.596 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.596 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.767614+0000", 2024-08-17T17:39:05.596 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 3.8893999999999997e-05 2024-08-17T17:39:05.596 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.597 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.597 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.597 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.767643+0000", 2024-08-17T17:39:05.597 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8685000000000001e-05 2024-08-17T17:39:05.597 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.597 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.597 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.597 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.767719+0000", 2024-08-17T17:39:05.597 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.6048000000000001e-05 2024-08-17T17:39:05.597 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.597 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.597 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.597 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.767768+0000", 2024-08-17T17:39:05.597 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.9604999999999997e-05 2024-08-17T17:39:05.598 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.598 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.598 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.598 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.767776+0000", 2024-08-17T17:39:05.598 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.046e-06 2024-08-17T17:39:05.598 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.598 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.598 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.598 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.744271+0000", 2024-08-17T17:39:05.598 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.97649498099999998 2024-08-17T17:39:05.598 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.598 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.598 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.599 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.744279+0000", 2024-08-17T17:39:05.599 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6610000000000006e-06 2024-08-17T17:39:05.599 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.599 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.599 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.599 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.599 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.599 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:62291 11.b 11.be3640db (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1382)", 2024-08-17T17:39:05.599 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:36:11.774272+0000", 2024-08-17T17:39:05.599 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 173.62757199399999, 2024-08-17T17:39:05.599 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.97003763799999998, 2024-08-17T17:39:05.599 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.599 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.599 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:39:05.599 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.600 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:39:05.600 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:05.600 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 62291 2024-08-17T17:39:05.600 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.600 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.600 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.600 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.600 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.774272+0000", 2024-08-17T17:39:05.600 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.600 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.600 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.600 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.600 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.774272+0000", 2024-08-17T17:39:05.600 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.600 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.601 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.601 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.601 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.774273+0000", 2024-08-17T17:39:05.601 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.722e-06 2024-08-17T17:39:05.601 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.601 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.601 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.601 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.774297+0000", 2024-08-17T17:39:05.601 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3229000000000001e-05 2024-08-17T17:39:05.601 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.601 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.601 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.602 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.774317+0000", 2024-08-17T17:39:05.602 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9981999999999999e-05 2024-08-17T17:39:05.602 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.602 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.602 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.602 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.774325+0000", 2024-08-17T17:39:05.602 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.0800000000000006e-06 2024-08-17T17:39:05.602 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.602 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.602 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.602 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.774373+0000", 2024-08-17T17:39:05.602 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 4.8699000000000002e-05 2024-08-17T17:39:05.603 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.603 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.603 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.603 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.774379+0000", 2024-08-17T17:39:05.603 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.8819999999999998e-06 2024-08-17T17:39:05.603 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.603 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.603 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.603 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.744302+0000", 2024-08-17T17:39:05.603 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.96992279999999997 2024-08-17T17:39:05.603 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.603 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.604 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.604 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.744309+0000", 2024-08-17T17:39:05.604 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.2439999999999996e-06 2024-08-17T17:39:05.604 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.604 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.604 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.604 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.604 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.604 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:62295 11.b 11.fe22c5eb (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1382)", 2024-08-17T17:39:05.604 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:36:11.779485+0000", 2024-08-17T17:39:05.604 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 173.62235834000001, 2024-08-17T17:39:05.604 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.96485381800000003, 2024-08-17T17:39:05.604 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.604 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.605 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:39:05.605 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.605 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:39:05.605 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:05.605 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 62295 2024-08-17T17:39:05.605 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.605 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.605 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.605 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.605 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.779485+0000", 2024-08-17T17:39:05.605 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.605 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.605 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.605 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.606 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.779485+0000", 2024-08-17T17:39:05.606 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.606 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.606 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.606 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.606 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.779487+0000", 2024-08-17T17:39:05.606 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.9010000000000001e-06 2024-08-17T17:39:05.606 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.606 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.606 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.606 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.779516+0000", 2024-08-17T17:39:05.606 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.8785e-05 2024-08-17T17:39:05.606 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.607 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.607 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.607 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.779539+0000", 2024-08-17T17:39:05.607 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.2478999999999999e-05 2024-08-17T17:39:05.607 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.607 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.607 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.607 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.779549+0000", 2024-08-17T17:39:05.607 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.0203999999999999e-05 2024-08-17T17:39:05.607 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.607 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.607 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.607 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.779618+0000", 2024-08-17T17:39:05.608 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.8878e-05 2024-08-17T17:39:05.608 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.608 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.608 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.608 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.779627+0000", 2024-08-17T17:39:05.608 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 9.8689999999999999e-06 2024-08-17T17:39:05.608 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.608 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.608 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.608 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.744332+0000", 2024-08-17T17:39:05.608 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.96470478000000004 2024-08-17T17:39:05.608 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.608 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.608 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.609 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.744339+0000", 2024-08-17T17:39:05.609 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 6.922e-06 2024-08-17T17:39:05.609 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.609 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.609 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.609 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.609 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.609 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:62300 11.b 11.5557634b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1382)", 2024-08-17T17:39:05.609 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:36:11.790745+0000", 2024-08-17T17:39:05.609 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 173.611098889, 2024-08-17T17:39:05.609 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.953632916, 2024-08-17T17:39:05.609 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.609 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.609 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:39:05.609 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.610 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:39:05.610 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:05.610 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 62300 2024-08-17T17:39:05.610 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.610 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.610 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.610 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.610 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.790745+0000", 2024-08-17T17:39:05.610 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.610 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.610 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.610 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.610 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.790745+0000", 2024-08-17T17:39:05.611 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.611 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.611 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.611 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.611 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.790746+0000", 2024-08-17T17:39:05.611 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.658e-06 2024-08-17T17:39:05.611 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.611 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.611 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.611 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.790770+0000", 2024-08-17T17:39:05.611 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.3512000000000001e-05 2024-08-17T17:39:05.611 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.611 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.611 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.611 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.790791+0000", 2024-08-17T17:39:05.612 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.0890000000000002e-05 2024-08-17T17:39:05.612 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.612 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.612 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.612 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.790798+0000", 2024-08-17T17:39:05.612 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.1980000000000004e-06 2024-08-17T17:39:05.612 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.612 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.612 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.612 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.790850+0000", 2024-08-17T17:39:05.612 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.2179999999999998e-05 2024-08-17T17:39:05.612 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.612 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.613 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.613 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:11.790856+0000", 2024-08-17T17:39:05.613 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 5.553e-06 2024-08-17T17:39:05.613 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.613 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.613 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.613 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.744370+0000", 2024-08-17T17:39:05.613 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.95351413799999996 2024-08-17T17:39:05.613 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.613 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.613 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.613 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:36:12.744378+0000", 2024-08-17T17:39:05.613 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.7870000000000002e-06 2024-08-17T17:39:05.614 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.614 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.614 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.614 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.614 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.614 INFO:teuthology.orchestra.run.smithi063.stdout: "description": "osd_op(client.46088.0:79281 11.b 11:d992b94a:::benchmark_data_smithi063_328699_object14838:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1415)", 2024-08-17T17:39:05.614 INFO:teuthology.orchestra.run.smithi063.stdout: "initiated_at": "2024-08-17T17:38:06.894725+0000", 2024-08-17T17:39:05.614 INFO:teuthology.orchestra.run.smithi063.stdout: "age": 58.507118611999999, 2024-08-17T17:39:05.614 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.993561056, 2024-08-17T17:39:05.614 INFO:teuthology.orchestra.run.smithi063.stdout: "continuous": false, 2024-08-17T17:39:05.614 INFO:teuthology.orchestra.run.smithi063.stdout: "type_data": { 2024-08-17T17:39:05.614 INFO:teuthology.orchestra.run.smithi063.stdout: "flag_point": "reached pg", 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: "client_info": { 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: "client": "client.46088", 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: "tid": 79281 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: "events": [ 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "initiated", 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:06.894725+0000", 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "header_read", 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:06.894725+0000", 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0 2024-08-17T17:39:05.615 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.616 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.616 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "throttled", 2024-08-17T17:39:05.616 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:06.894728+0000", 2024-08-17T17:39:05.616 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.492e-06 2024-08-17T17:39:05.616 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.616 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.616 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "all_read", 2024-08-17T17:39:05.616 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:06.894740+0000", 2024-08-17T17:39:05.616 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2659e-05 2024-08-17T17:39:05.616 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.616 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.616 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "dispatched", 2024-08-17T17:39:05.616 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:06.894767+0000", 2024-08-17T17:39:05.616 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 2.6316999999999999e-05 2024-08-17T17:39:05.616 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.617 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.617 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "queued_for_pg", 2024-08-17T17:39:05.617 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:06.894779+0000", 2024-08-17T17:39:05.617 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.2833000000000001e-05 2024-08-17T17:39:05.617 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.617 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.617 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.617 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:06.894857+0000", 2024-08-17T17:39:05.617 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 7.8028000000000006e-05 2024-08-17T17:39:05.617 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.617 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.617 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "waiting for peered", 2024-08-17T17:39:05.617 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:06.894871+0000", 2024-08-17T17:39:05.617 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 1.3551e-05 2024-08-17T17:39:05.617 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.618 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.618 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "reached_pg", 2024-08-17T17:39:05.618 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:07.888200+0000", 2024-08-17T17:39:05.618 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 0.99332885299999996 2024-08-17T17:39:05.618 INFO:teuthology.orchestra.run.smithi063.stdout: }, 2024-08-17T17:39:05.618 INFO:teuthology.orchestra.run.smithi063.stdout: { 2024-08-17T17:39:05.618 INFO:teuthology.orchestra.run.smithi063.stdout: "event": "done", 2024-08-17T17:39:05.618 INFO:teuthology.orchestra.run.smithi063.stdout: "time": "2024-08-17T17:38:07.888286+0000", 2024-08-17T17:39:05.618 INFO:teuthology.orchestra.run.smithi063.stdout: "duration": 8.6322999999999993e-05 2024-08-17T17:39:05.618 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.618 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.618 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.618 INFO:teuthology.orchestra.run.smithi063.stdout: } 2024-08-17T17:39:05.618 INFO:teuthology.orchestra.run.smithi063.stdout: ] 2024-08-17T17:39:05.619 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:39:05.619 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:05.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.645 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:05.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:05.688 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:05.746 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:05.777 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:05.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:05.879 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:05.901 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:05.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:05.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:06.003 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:06.025 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:06.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:06.126 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:06.152 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:06.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:06.254 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:06.276 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:39:06.278 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:06.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:06.380 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:06.398 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:06 smithi162 ceph-mon[104668]: pgmap v2237: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 214 op/s; 15452/118617 objects degraded (13.027%); 3681/118617 objects misplaced (3.103%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:39:06.398 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:06 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3103063268' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:39:06.404 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:06.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:06.447 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:06 smithi063 ceph-mon[114756]: pgmap v2237: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 214 op/s; 15452/118617 objects degraded (13.027%); 3681/118617 objects misplaced (3.103%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:39:06.447 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:06 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3103063268' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:39:06.447 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:06 smithi063 ceph-mon[109843]: pgmap v2237: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 214 op/s; 15452/118617 objects degraded (13.027%); 3681/118617 objects misplaced (3.103%); 3.3 MiB/s, 52 objects/s recovering 2024-08-17T17:39:06.447 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:06 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3103063268' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:39:06.506 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:06.529 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:06.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:06.567 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:06.630 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:06.654 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:06.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:06.756 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:06.782 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:06.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:06.884 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:06.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:06.907 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:06.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:07.008 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:07.031 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:07.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:07.132 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:07.156 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:07.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:07.258 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:07.281 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:07.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:07.382 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:07.405 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:07.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:07.507 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:07.541 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:07.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:07.643 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:07.668 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:07.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:07.770 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:07.800 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:07.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:07.824 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:39:07.825 INFO:teuthology.orchestra.run.smithi063.stdout:{"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15452/118617 objects degraded (13.027%), 4 pgs degraded, 5 pgs undersized","count":9},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2225,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1422,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916154,"num_in_osds":8,"osd_in_since":1723916113,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":112},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+clean+scrubbing","count":2},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":39539,"data_bytes":2590099121,"bytes_used":14631260160,"bytes_avail":753329025024,"bytes_total":767960285184,"degraded_objects":15452,"degraded_total":118617,"degraded_ratio":0.13026800542923864,"misplaced_objects":3681,"misplaced_total":118617,"misplaced_ratio":0.031032651306305167,"recovering_objects_per_sec":52,"recovering_bytes_per_sec":3459237,"recovering_keys_per_sec":0,"num_objects_recovered":528,"num_bytes_recovered":34603008,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":214},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi063.ultpqj","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.63:8443/","prometheus":"http://172.21.15.63:9283/"}},"servicemap":{"epoch":33,"modified":"2024-08-17T17:38:44.649232+0000","services":{}},"progress_events":{}} 2024-08-17T17:39:07.902 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:07.928 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:07.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:07.929 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:08.030 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:08.061 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:08.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:08.163 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:08.190 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:08.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:08.292 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:08.316 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:08.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:08.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:08 smithi162 ceph-mon[104668]: pgmap v2238: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 222 op/s; 15298/117270 objects degraded (13.045%); 3681/117270 objects misplaced (3.139%); 3.4 MiB/s, 55 objects/s recovering 2024-08-17T17:39:08.418 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:08.440 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:08.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:08.481 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:08 smithi063 ceph-mon[109843]: pgmap v2238: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 222 op/s; 15298/117270 objects degraded (13.045%); 3681/117270 objects misplaced (3.139%); 3.4 MiB/s, 55 objects/s recovering 2024-08-17T17:39:08.481 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:08 smithi063 ceph-mon[114756]: pgmap v2238: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 222 op/s; 15298/117270 objects degraded (13.045%); 3681/117270 objects misplaced (3.139%); 3.4 MiB/s, 55 objects/s recovering 2024-08-17T17:39:08.503 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:39:08.503 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:39:08.542 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:08.569 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:08.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:08.671 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:08.697 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:08.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:08.798 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:08.828 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:08.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:08.930 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:08.958 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:08.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:08.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:09.060 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:09.087 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:09.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:09.108 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:39:09.108 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:39:09.108 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 0 2024-08-17T17:39:09.109 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:39:09.114 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:09.189 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:09.220 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:09.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:09.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:09 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 15298/117270 objects degraded (13.045%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:09.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:09 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 15298/117270 objects degraded (13.045%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:09.321 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:09.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:09 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 15298/117270 objects degraded (13.045%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:09.345 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:09.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:09.447 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:09.513 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:09.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:09.615 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:09.642 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:09.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:09.744 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:09.756 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:39:09.768 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:09.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:09.783 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:39:09.870 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:09.894 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:09.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:09.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:09.996 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:10.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.032 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:10.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:10.081 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:10.133 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:10.166 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:10.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:10.267 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:10.270 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:39:10.271 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:10 smithi063 ceph-mon[114756]: pgmap v2239: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 213 op/s; 15026/115467 objects degraded (13.013%); 3541/115467 objects misplaced (3.067%); 2.9 MiB/s, 47 objects/s recovering 2024-08-17T17:39:10.271 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:10 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2266734222' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:10.271 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:10 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:10.271 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:10 smithi063 ceph-mon[109843]: pgmap v2239: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 213 op/s; 15026/115467 objects degraded (13.013%); 3541/115467 objects misplaced (3.067%); 2.9 MiB/s, 47 objects/s recovering 2024-08-17T17:39:10.271 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:10 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2266734222' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:10.271 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:10 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:10.291 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:10.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:10.392 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:10.407 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:10 smithi162 ceph-mon[104668]: pgmap v2239: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 213 op/s; 15026/115467 objects degraded (13.013%); 3541/115467 objects misplaced (3.067%); 2.9 MiB/s, 47 objects/s recovering 2024-08-17T17:39:10.407 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:10 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2266734222' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:10.407 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:10 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:10.421 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:10.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:10.523 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:10.551 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:10.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:10.653 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:10.678 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:10.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:10.780 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:10.807 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:10.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:10.909 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:10.936 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:10.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:10.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:11.038 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:11.061 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:11.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:11.162 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:11.186 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:11.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:11.288 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:11.310 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:11.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.312 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:11.317 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:11 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:11.317 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:11 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:39:11.317 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:11 smithi162 ceph-mon[104668]: osdmap e1423: 8 total, 8 up, 8 in 2024-08-17T17:39:11.317 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:11 smithi162 ceph-mon[104668]: pgmap v2241: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 217 op/s; 15026/115293 objects degraded (13.033%); 3541/115293 objects misplaced (3.071%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:39:11.412 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:11.441 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:11.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:11.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:11 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:11.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:11 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:39:11.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:11 smithi063 ceph-mon[109843]: osdmap e1423: 8 total, 8 up, 8 in 2024-08-17T17:39:11.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:11 smithi063 ceph-mon[109843]: pgmap v2241: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 217 op/s; 15026/115293 objects degraded (13.033%); 3541/115293 objects misplaced (3.071%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:39:11.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:11 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:11.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:11 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:39:11.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:11 smithi063 ceph-mon[114756]: osdmap e1423: 8 total, 8 up, 8 in 2024-08-17T17:39:11.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:11 smithi063 ceph-mon[114756]: pgmap v2241: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 217 op/s; 15026/115293 objects degraded (13.033%); 3541/115293 objects misplaced (3.071%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:39:11.544 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:11.568 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:11.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:11.669 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:11.695 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:11.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:11.697 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:11.797 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:11.824 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:11.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:11.925 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:11.933 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:39:11.933 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:39:11.933 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:39:11.933 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:39:11.933 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:39:11.953 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:11.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:11.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:12.055 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:12.084 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:12.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:12.186 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:12.214 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:12.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:12.316 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:12.341 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:12.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:12.443 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:12.470 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:12.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:12.572 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:12.600 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:12.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:12.702 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:12.729 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:12.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:12.831 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:12.855 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:12.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:12.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:12.945 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:39:12.957 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:13.014 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:13.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:13.079 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:39:13.115 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:13.143 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:13.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:13.160 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:39:13.162 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:39:13.239 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:13.245 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:13.272 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:13.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:13.374 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:13.402 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:13.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:13.428 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:39:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:39:13] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.51.0" 2024-08-17T17:39:13.504 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:13.529 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:13.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:13.631 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:13.655 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:13.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.656 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:13.756 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:13.788 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:13.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:13.890 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:13.913 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:13.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:13.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:14.015 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:13 smithi063 ceph-mon[114756]: pgmap v2242: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 210 op/s; 14607/112683 objects degraded (12.963%); 3413/112683 objects misplaced (3.029%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T17:39:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:13 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 14607/112683 objects degraded (12.963%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:13 smithi063 ceph-mon[109843]: pgmap v2242: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 210 op/s; 14607/112683 objects degraded (12.963%); 3413/112683 objects misplaced (3.029%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T17:39:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:13 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 14607/112683 objects degraded (12.963%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:14.042 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:14.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:14.077 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2242,"stamp":"2024-08-17T17:39:12.656975+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2460468913,"num_objects":37561,"num_object_clones":0,"num_object_copies":112683,"num_objects_missing_on_primary":3451,"num_objects_missing":3451,"num_objects_degraded":14607,"num_objects_misplaced":3413,"num_objects_unfound":0,"num_objects_dirty":37561,"num_whiteouts":0,"num_read":28,"num_read_kb":62,"num_write":91426,"num_write_kb":4124377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29995,"num_bytes_recovered":1812861661,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95825,"ondisk_log_size":95825,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":375,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14060104,"kb_used_data":8161476,"kb_used_omap":298,"kb_used_meta":5898261,"kb_avail":735901112,"statfs":{"total":767960285184,"available":753562738688,"internally_reserved":0,"allocated":8357351424,"data_stored":8336031147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":6039819793},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1,2,0,0,4,8,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-138280960,"num_objects":-2110,"num_object_clones":0,"num_object_copies":-6330,"num_objects_missing_on_primary":-420,"num_objects_missing":-420,"num_objects_degraded":-845,"num_objects_misplaced":-268,"num_objects_unfound":0,"num_objects_dirty":-2110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2110,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":555,"num_bytes_recovered":36372480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002596"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":52,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.142015+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:39:10.142015+0000","last_peered":"2024-08-17T17:39:10.142015+0000","last_clean":"2024-08-17T17:39:10.142015+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:39:10.142015+0000","last_undegraded":"2024-08-17T17:39:10.142015+0000","last_fullsized":"2024-08-17T17:39:10.142015+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1437,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.144550+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:38:57.144550+0000","last_peered":"2024-08-17T17:38:57.144550+0000","last_clean":"2024-08-17T17:38:57.144550+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:38:57.144550+0000","last_undegraded":"2024-08-17T17:38:57.144550+0000","last_fullsized":"2024-08-17T17:38:57.144550+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1752,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145661+0000","last_change":"2024-08-17T17:36:15.771150+0000","last_active":"2024-08-17T17:38:57.145661+0000","last_peered":"2024-08-17T17:38:57.145661+0000","last_clean":"2024-08-17T17:38:57.145661+0000","last_became_active":"2024-08-17T17:36:15.770520+0000","last_became_peered":"2024-08-17T17:36:15.770520+0000","last_unstale":"2024-08-17T17:38:57.145661+0000","last_undegraded":"2024-08-17T17:38:57.145661+0000","last_fullsized":"2024-08-17T17:38:57.145661+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:17.552796+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":87,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122234+0000","last_change":"2024-08-17T17:36:12.756709+0000","last_active":"2024-08-17T17:38:57.122234+0000","last_peered":"2024-08-17T17:38:57.122234+0000","last_clean":"2024-08-17T17:38:57.122234+0000","last_became_active":"2024-08-17T17:36:12.756385+0000","last_became_peered":"2024-08-17T17:36:12.756385+0000","last_unstale":"2024-08-17T17:38:57.122234+0000","last_undegraded":"2024-08-17T17:38:57.122234+0000","last_fullsized":"2024-08-17T17:38:57.122234+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:21.420282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":137,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.156833+0000","last_change":"2024-08-17T17:36:12.762800+0000","last_active":"2024-08-17T17:39:10.156833+0000","last_peered":"2024-08-17T17:39:10.156833+0000","last_clean":"2024-08-17T17:39:10.156833+0000","last_became_active":"2024-08-17T17:36:12.759977+0000","last_became_peered":"2024-08-17T17:36:12.759977+0000","last_unstale":"2024-08-17T17:39:10.156833+0000","last_undegraded":"2024-08-17T17:39:10.156833+0000","last_fullsized":"2024-08-17T17:39:10.156833+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:02.162681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1632,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122092+0000","last_change":"2024-08-17T17:35:56.281561+0000","last_active":"2024-08-17T17:38:57.122092+0000","last_peered":"2024-08-17T17:38:57.122092+0000","last_clean":"2024-08-17T17:38:57.122092+0000","last_became_active":"2024-08-17T17:35:56.278638+0000","last_became_peered":"2024-08-17T17:35:56.278638+0000","last_unstale":"2024-08-17T17:38:57.122092+0000","last_undegraded":"2024-08-17T17:38:57.122092+0000","last_fullsized":"2024-08-17T17:38:57.122092+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1371,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.129014+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:38:57.129014+0000","last_peered":"2024-08-17T17:38:57.129014+0000","last_clean":"2024-08-17T17:38:57.129014+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:38:57.129014+0000","last_undegraded":"2024-08-17T17:38:57.129014+0000","last_fullsized":"2024-08-17T17:38:57.129014+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":47,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145121+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:38:57.145121+0000","last_peered":"2024-08-17T17:38:57.145121+0000","last_clean":"2024-08-17T17:38:57.145121+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:38:57.145121+0000","last_undegraded":"2024-08-17T17:38:57.145121+0000","last_fullsized":"2024-08-17T17:38:57.145121+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1416,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128876+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:38:57.128876+0000","last_peered":"2024-08-17T17:38:57.128876+0000","last_clean":"2024-08-17T17:38:57.128876+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:38:57.128876+0000","last_undegraded":"2024-08-17T17:38:57.128876+0000","last_fullsized":"2024-08-17T17:38:57.128876+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1424,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145344+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:38:57.145344+0000","last_peered":"2024-08-17T17:38:57.145344+0000","last_clean":"2024-08-17T17:38:57.145344+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:38:57.145344+0000","last_undegraded":"2024-08-17T17:38:57.145344+0000","last_fullsized":"2024-08-17T17:38:57.145344+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":714,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122135+0000","last_change":"2024-08-17T17:36:15.773315+0000","last_active":"2024-08-17T17:38:57.122135+0000","last_peered":"2024-08-17T17:38:57.122135+0000","last_clean":"2024-08-17T17:38:57.122135+0000","last_became_active":"2024-08-17T17:36:15.772961+0000","last_became_peered":"2024-08-17T17:36:15.772961+0000","last_unstale":"2024-08-17T17:38:57.122135+0000","last_undegraded":"2024-08-17T17:38:57.122135+0000","last_fullsized":"2024-08-17T17:38:57.122135+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:10.190221+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":46,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.142233+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:39:10.142233+0000","last_peered":"2024-08-17T17:39:10.142233+0000","last_clean":"2024-08-17T17:39:10.142233+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:39:10.142233+0000","last_undegraded":"2024-08-17T17:39:10.142233+0000","last_fullsized":"2024-08-17T17:39:10.142233+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":137,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.158413+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:39:10.158413+0000","last_peered":"2024-08-17T17:39:10.158413+0000","last_clean":"2024-08-17T17:39:10.158413+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:39:10.158413+0000","last_undegraded":"2024-08-17T17:39:10.158413+0000","last_fullsized":"2024-08-17T17:39:10.158413+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1505,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.179582+0000","last_change":"2024-08-17T17:36:24.225739+0000","last_active":"2024-08-17T17:38:57.179582+0000","last_peered":"2024-08-17T17:38:57.179582+0000","last_clean":"2024-08-17T17:38:57.179582+0000","last_became_active":"2024-08-17T17:36:24.225250+0000","last_became_peered":"2024-08-17T17:36:24.225250+0000","last_unstale":"2024-08-17T17:38:57.179582+0000","last_undegraded":"2024-08-17T17:38:57.179582+0000","last_fullsized":"2024-08-17T17:38:57.179582+0000","mapping_epoch":1393,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1394,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.290656+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1444,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145045+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:38:57.145045+0000","last_peered":"2024-08-17T17:38:57.145045+0000","last_clean":"2024-08-17T17:38:57.145045+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:38:57.145045+0000","last_undegraded":"2024-08-17T17:38:57.145045+0000","last_fullsized":"2024-08-17T17:38:57.145045+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1851,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145816+0000","last_change":"2024-08-17T17:36:12.752970+0000","last_active":"2024-08-17T17:38:57.145816+0000","last_peered":"2024-08-17T17:38:57.145816+0000","last_clean":"2024-08-17T17:38:57.145816+0000","last_became_active":"2024-08-17T17:36:12.752311+0000","last_became_peered":"2024-08-17T17:36:12.752311+0000","last_unstale":"2024-08-17T17:38:57.145816+0000","last_undegraded":"2024-08-17T17:38:57.145816+0000","last_fullsized":"2024-08-17T17:38:57.145816+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:38.131419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1799,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:39:07.441385+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:39:07.441385+0000","last_peered":"2024-08-17T17:39:07.441385+0000","last_clean":"2024-08-17T17:39:07.441385+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:39:07.441385+0000","last_undegraded":"2024-08-17T17:39:07.441385+0000","last_fullsized":"2024-08-17T17:39:07.441385+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1507,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128761+0000","last_change":"2024-08-17T17:36:18.750494+0000","last_active":"2024-08-17T17:38:57.128761+0000","last_peered":"2024-08-17T17:38:57.128761+0000","last_clean":"2024-08-17T17:38:57.128761+0000","last_became_active":"2024-08-17T17:36:18.750092+0000","last_became_peered":"2024-08-17T17:36:18.750092+0000","last_unstale":"2024-08-17T17:38:57.128761+0000","last_undegraded":"2024-08-17T17:38:57.128761+0000","last_fullsized":"2024-08-17T17:38:57.128761+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:36.599727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1139,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.142152+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:39:10.142152+0000","last_peered":"2024-08-17T17:39:10.142152+0000","last_clean":"2024-08-17T17:39:10.142152+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:39:10.142152+0000","last_undegraded":"2024-08-17T17:39:10.142152+0000","last_fullsized":"2024-08-17T17:39:10.142152+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":965,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128590+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:38:57.128590+0000","last_peered":"2024-08-17T17:38:57.128590+0000","last_clean":"2024-08-17T17:38:57.128590+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:38:57.128590+0000","last_undegraded":"2024-08-17T17:38:57.128590+0000","last_fullsized":"2024-08-17T17:38:57.128590+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":46,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.141409+0000","last_change":"2024-08-17T17:37:11.075505+0000","last_active":"2024-08-17T17:39:10.141409+0000","last_peered":"2024-08-17T17:39:10.141409+0000","last_clean":"2024-08-17T17:39:10.141409+0000","last_became_active":"2024-08-17T17:37:11.075131+0000","last_became_peered":"2024-08-17T17:37:11.075131+0000","last_unstale":"2024-08-17T17:39:10.141409+0000","last_undegraded":"2024-08-17T17:39:10.141409+0000","last_fullsized":"2024-08-17T17:39:10.141409+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:39.672054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1278,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122478+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:38:57.122478+0000","last_peered":"2024-08-17T17:38:57.122478+0000","last_clean":"2024-08-17T17:38:57.122478+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:38:57.122478+0000","last_undegraded":"2024-08-17T17:38:57.122478+0000","last_fullsized":"2024-08-17T17:38:57.122478+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1655,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.144768+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:38:57.144768+0000","last_peered":"2024-08-17T17:38:57.144768+0000","last_clean":"2024-08-17T17:38:57.144768+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:38:57.144768+0000","last_undegraded":"2024-08-17T17:38:57.144768+0000","last_fullsized":"2024-08-17T17:38:57.144768+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":91,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128346+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:38:57.128346+0000","last_peered":"2024-08-17T17:38:57.128346+0000","last_clean":"2024-08-17T17:38:57.128346+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:38:57.128346+0000","last_undegraded":"2024-08-17T17:38:57.128346+0000","last_fullsized":"2024-08-17T17:38:57.128346+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1252,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.129229+0000","last_change":"2024-08-17T17:36:12.755873+0000","last_active":"2024-08-17T17:38:57.129229+0000","last_peered":"2024-08-17T17:38:57.129229+0000","last_clean":"2024-08-17T17:38:57.129229+0000","last_became_active":"2024-08-17T17:36:12.755366+0000","last_became_peered":"2024-08-17T17:36:12.755366+0000","last_unstale":"2024-08-17T17:38:57.129229+0000","last_undegraded":"2024-08-17T17:38:57.129229+0000","last_fullsized":"2024-08-17T17:38:57.129229+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:36.523214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1606,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.140227+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:38:57.140227+0000","last_peered":"2024-08-17T17:38:57.140227+0000","last_clean":"2024-08-17T17:38:57.140227+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:38:57.140227+0000","last_undegraded":"2024-08-17T17:38:57.140227+0000","last_fullsized":"2024-08-17T17:38:57.140227+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":790,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.179781+0000","last_change":"2024-08-17T17:35:55.200519+0000","last_active":"2024-08-17T17:38:57.179781+0000","last_peered":"2024-08-17T17:38:57.179781+0000","last_clean":"2024-08-17T17:38:57.179781+0000","last_became_active":"2024-08-17T17:35:55.200230+0000","last_became_peered":"2024-08-17T17:35:55.200230+0000","last_unstale":"2024-08-17T17:38:57.179781+0000","last_undegraded":"2024-08-17T17:38:57.179781+0000","last_fullsized":"2024-08-17T17:38:57.179781+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:33.454851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1429,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128815+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:38:57.128815+0000","last_peered":"2024-08-17T17:38:57.128815+0000","last_clean":"2024-08-17T17:38:57.128815+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:38:57.128815+0000","last_undegraded":"2024-08-17T17:38:57.128815+0000","last_fullsized":"2024-08-17T17:38:57.128815+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1710,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.158125+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:39:10.158125+0000","last_peered":"2024-08-17T17:39:10.158125+0000","last_clean":"2024-08-17T17:39:10.158125+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:39:10.158125+0000","last_undegraded":"2024-08-17T17:39:10.158125+0000","last_fullsized":"2024-08-17T17:39:10.158125+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:22.077229+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":255,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128999+0000","last_change":"2024-08-17T17:35:33.145476+0000","last_active":"2024-08-17T17:38:57.128999+0000","last_peered":"2024-08-17T17:38:57.128999+0000","last_clean":"2024-08-17T17:38:57.128999+0000","last_became_active":"2024-08-17T17:35:33.145125+0000","last_became_peered":"2024-08-17T17:35:33.145125+0000","last_unstale":"2024-08-17T17:38:57.128999+0000","last_undegraded":"2024-08-17T17:38:57.128999+0000","last_fullsized":"2024-08-17T17:38:57.128999+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:45.298064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":934,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.134017+0000","last_change":"2024-08-17T17:35:56.248612+0000","last_active":"2024-08-17T17:38:57.134017+0000","last_peered":"2024-08-17T17:38:57.134017+0000","last_clean":"2024-08-17T17:38:57.134017+0000","last_became_active":"2024-08-17T17:35:56.244364+0000","last_became_peered":"2024-08-17T17:35:56.244364+0000","last_unstale":"2024-08-17T17:38:57.134017+0000","last_undegraded":"2024-08-17T17:38:57.134017+0000","last_fullsized":"2024-08-17T17:38:57.134017+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:41.064165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":874,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:39:07.441624+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:39:07.441624+0000","last_peered":"2024-08-17T17:39:07.441624+0000","last_clean":"2024-08-17T17:39:07.441624+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:39:07.441624+0000","last_undegraded":"2024-08-17T17:39:07.441624+0000","last_fullsized":"2024-08-17T17:39:07.441624+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1409,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.146549+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:38:57.146549+0000","last_peered":"2024-08-17T17:38:57.146549+0000","last_clean":"2024-08-17T17:38:57.146549+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:38:57.146549+0000","last_undegraded":"2024-08-17T17:38:57.146549+0000","last_fullsized":"2024-08-17T17:38:57.146549+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1428,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145371+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:38:57.145371+0000","last_peered":"2024-08-17T17:38:57.145371+0000","last_clean":"2024-08-17T17:38:57.145371+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:38:57.145371+0000","last_undegraded":"2024-08-17T17:38:57.145371+0000","last_fullsized":"2024-08-17T17:38:57.145371+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":930,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.124272+0000","last_change":"2024-08-17T17:36:15.774881+0000","last_active":"2024-08-17T17:38:57.124272+0000","last_peered":"2024-08-17T17:38:57.124272+0000","last_clean":"2024-08-17T17:38:57.124272+0000","last_became_active":"2024-08-17T17:36:15.774435+0000","last_became_peered":"2024-08-17T17:36:15.774435+0000","last_unstale":"2024-08-17T17:38:57.124272+0000","last_undegraded":"2024-08-17T17:38:57.124272+0000","last_fullsized":"2024-08-17T17:38:57.124272+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:52.015854+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"1423'5891","reported_seq":14070,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:12.290165+0000","last_change":"2024-08-17T17:38:37.110784+0000","last_active":"2024-08-17T17:39:12.290165+0000","last_peered":"2024-08-17T17:39:12.290165+0000","last_clean":"2024-08-17T17:39:12.290165+0000","last_became_active":"2024-08-17T17:38:37.088380+0000","last_became_peered":"2024-08-17T17:38:37.088380+0000","last_unstale":"2024-08-17T17:39:12.290165+0000","last_undegraded":"2024-08-17T17:39:12.290165+0000","last_fullsized":"2024-08-17T17:39:12.290165+0000","mapping_epoch":1419,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5891,"log_dups_size":0,"ondisk_log_size":5891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:08.647903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156958720,"num_objects":2395,"num_object_clones":0,"num_object_copies":7185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5891,"num_write_kb":265152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4009,"num_bytes_recovered":199753728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1376,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.144478+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:38:57.144478+0000","last_peered":"2024-08-17T17:38:57.144478+0000","last_clean":"2024-08-17T17:38:57.144478+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:38:57.144478+0000","last_undegraded":"2024-08-17T17:38:57.144478+0000","last_fullsized":"2024-08-17T17:38:57.144478+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1272,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122412+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:38:57.122412+0000","last_peered":"2024-08-17T17:38:57.122412+0000","last_clean":"2024-08-17T17:38:57.122412+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:38:57.122412+0000","last_undegraded":"2024-08-17T17:38:57.122412+0000","last_fullsized":"2024-08-17T17:38:57.122412+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":325,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.142293+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:39:10.142293+0000","last_peered":"2024-08-17T17:39:10.142293+0000","last_clean":"2024-08-17T17:39:10.142293+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:39:10.142293+0000","last_undegraded":"2024-08-17T17:39:10.142293+0000","last_fullsized":"2024-08-17T17:39:10.142293+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1591,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128724+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:38:57.128724+0000","last_peered":"2024-08-17T17:38:57.128724+0000","last_clean":"2024-08-17T17:38:57.128724+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:38:57.128724+0000","last_undegraded":"2024-08-17T17:38:57.128724+0000","last_fullsized":"2024-08-17T17:38:57.128724+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:25.751871+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.f","version":"1423'5833","reported_seq":5983,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.575929+0000","last_change":"2024-08-17T17:35:34.179168+0000","last_active":"2024-08-17T17:39:10.575929+0000","last_peered":"2024-08-17T17:39:10.575929+0000","last_clean":"2024-08-17T17:39:10.575929+0000","last_became_active":"2024-08-17T17:35:34.133493+0000","last_became_peered":"2024-08-17T17:35:34.133493+0000","last_unstale":"2024-08-17T17:39:10.575929+0000","last_undegraded":"2024-08-17T17:39:10.575929+0000","last_fullsized":"2024-08-17T17:39:10.575929+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5833,"log_dups_size":0,"ondisk_log_size":5833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:24:11.188566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155385856,"num_objects":2371,"num_object_clones":0,"num_object_copies":7113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5833,"num_write_kb":262528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":98,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128403+0000","last_change":"2024-08-17T17:37:11.079712+0000","last_active":"2024-08-17T17:38:57.128403+0000","last_peered":"2024-08-17T17:38:57.128403+0000","last_clean":"2024-08-17T17:38:57.128403+0000","last_became_active":"2024-08-17T17:37:11.079253+0000","last_became_peered":"2024-08-17T17:37:11.079253+0000","last_unstale":"2024-08-17T17:38:57.128403+0000","last_undegraded":"2024-08-17T17:38:57.128403+0000","last_fullsized":"2024-08-17T17:38:57.128403+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:53.405657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":917,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122092+0000","last_change":"2024-08-17T17:35:56.223396+0000","last_active":"2024-08-17T17:38:57.122092+0000","last_peered":"2024-08-17T17:38:57.122092+0000","last_clean":"2024-08-17T17:38:57.122092+0000","last_became_active":"2024-08-17T17:35:56.212413+0000","last_became_peered":"2024-08-17T17:35:56.212413+0000","last_unstale":"2024-08-17T17:38:57.122092+0000","last_undegraded":"2024-08-17T17:38:57.122092+0000","last_fullsized":"2024-08-17T17:38:57.122092+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:44.185174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1634,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.143000+0000","last_change":"2024-08-17T17:35:33.136068+0000","last_active":"2024-08-17T17:39:10.143000+0000","last_peered":"2024-08-17T17:39:10.143000+0000","last_clean":"2024-08-17T17:39:10.143000+0000","last_became_active":"2024-08-17T17:35:33.135591+0000","last_became_peered":"2024-08-17T17:35:33.135591+0000","last_unstale":"2024-08-17T17:39:10.143000+0000","last_undegraded":"2024-08-17T17:39:10.143000+0000","last_fullsized":"2024-08-17T17:39:10.143000+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:52.252561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1782,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.146082+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:38:57.146082+0000","last_peered":"2024-08-17T17:38:57.146082+0000","last_clean":"2024-08-17T17:38:57.146082+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:38:57.146082+0000","last_undegraded":"2024-08-17T17:38:57.146082+0000","last_fullsized":"2024-08-17T17:38:57.146082+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"1423'5686","reported_seq":5821,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.582655+0000","last_change":"2024-08-17T17:35:14.861823+0000","last_active":"2024-08-17T17:39:10.582655+0000","last_peered":"2024-08-17T17:39:10.582655+0000","last_clean":"2024-08-17T17:39:10.582655+0000","last_became_active":"2024-08-17T17:35:14.854109+0000","last_became_peered":"2024-08-17T17:35:14.854109+0000","last_unstale":"2024-08-17T17:39:10.582655+0000","last_undegraded":"2024-08-17T17:39:10.582655+0000","last_fullsized":"2024-08-17T17:39:10.582655+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:46:40.778197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154664960,"num_objects":2360,"num_object_clones":0,"num_object_copies":7080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5686,"num_write_kb":257472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":995,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128688+0000","last_change":"2024-08-17T17:35:33.138467+0000","last_active":"2024-08-17T17:38:57.128688+0000","last_peered":"2024-08-17T17:38:57.128688+0000","last_clean":"2024-08-17T17:38:57.128688+0000","last_became_active":"2024-08-17T17:35:33.138174+0000","last_became_peered":"2024-08-17T17:35:33.138174+0000","last_unstale":"2024-08-17T17:38:57.128688+0000","last_undegraded":"2024-08-17T17:38:57.128688+0000","last_fullsized":"2024-08-17T17:38:57.128688+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:08.860439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1379,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.140227+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:38:57.140227+0000","last_peered":"2024-08-17T17:38:57.140227+0000","last_clean":"2024-08-17T17:38:57.140227+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:38:57.140227+0000","last_undegraded":"2024-08-17T17:38:57.140227+0000","last_fullsized":"2024-08-17T17:38:57.140227+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":130,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.158323+0000","last_change":"2024-08-17T17:35:14.854881+0000","last_active":"2024-08-17T17:39:10.158323+0000","last_peered":"2024-08-17T17:39:10.158323+0000","last_clean":"2024-08-17T17:39:10.158323+0000","last_became_active":"2024-08-17T17:35:14.854121+0000","last_became_peered":"2024-08-17T17:35:14.854121+0000","last_unstale":"2024-08-17T17:39:10.158323+0000","last_undegraded":"2024-08-17T17:39:10.158323+0000","last_fullsized":"2024-08-17T17:39:10.158323+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.863624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":936,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122265+0000","last_change":"2024-08-17T17:36:15.773050+0000","last_active":"2024-08-17T17:38:57.122265+0000","last_peered":"2024-08-17T17:38:57.122265+0000","last_clean":"2024-08-17T17:38:57.122265+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:38:57.122265+0000","last_undegraded":"2024-08-17T17:38:57.122265+0000","last_fullsized":"2024-08-17T17:38:57.122265+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:39.212195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:39.212195+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"1423'5679","reported_seq":11520,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.592548+0000","last_change":"2024-08-17T17:37:21.409979+0000","last_active":"2024-08-17T17:39:10.592548+0000","last_peered":"2024-08-17T17:39:10.592548+0000","last_clean":"2024-08-17T17:39:10.592548+0000","last_became_active":"2024-08-17T17:37:21.398111+0000","last_became_peered":"2024-08-17T17:37:21.398111+0000","last_unstale":"2024-08-17T17:39:10.592548+0000","last_undegraded":"2024-08-17T17:39:10.592548+0000","last_fullsized":"2024-08-17T17:39:10.592548+0000","mapping_epoch":1407,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5679,"log_dups_size":0,"ondisk_log_size":5679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:19.299395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154140696,"num_objects":2353,"num_object_clones":0,"num_object_copies":7059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2353,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5679,"num_write_kb":256961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2850,"num_bytes_recovered":168820736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1038,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.129032+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:38:57.129032+0000","last_peered":"2024-08-17T17:38:57.129032+0000","last_clean":"2024-08-17T17:38:57.129032+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:38:57.129032+0000","last_undegraded":"2024-08-17T17:38:57.129032+0000","last_fullsized":"2024-08-17T17:38:57.129032+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":222,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.123945+0000","last_change":"2024-08-17T17:35:33.145351+0000","last_active":"2024-08-17T17:38:57.123945+0000","last_peered":"2024-08-17T17:38:57.123945+0000","last_clean":"2024-08-17T17:38:57.123945+0000","last_became_active":"2024-08-17T17:35:33.145040+0000","last_became_peered":"2024-08-17T17:35:33.145040+0000","last_unstale":"2024-08-17T17:38:57.123945+0000","last_undegraded":"2024-08-17T17:38:57.123945+0000","last_fullsized":"2024-08-17T17:38:57.123945+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:02.124000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1447,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.129055+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:38:57.129055+0000","last_peered":"2024-08-17T17:38:57.129055+0000","last_clean":"2024-08-17T17:38:57.129055+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:38:57.129055+0000","last_undegraded":"2024-08-17T17:38:57.129055+0000","last_fullsized":"2024-08-17T17:38:57.129055+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":919,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122393+0000","last_change":"2024-08-17T17:36:15.769890+0000","last_active":"2024-08-17T17:38:57.122393+0000","last_peered":"2024-08-17T17:38:57.122393+0000","last_clean":"2024-08-17T17:38:57.122393+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:38:57.122393+0000","last_undegraded":"2024-08-17T17:38:57.122393+0000","last_fullsized":"2024-08-17T17:38:57.122393+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:33.034306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:33.034306+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":4,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.b","version":"1422'5589","reported_seq":13388,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:39:08.349153+0000","last_change":"2024-08-17T17:38:07.888612+0000","last_active":"2024-08-17T17:39:08.349153+0000","last_peered":"2024-08-17T17:39:08.349153+0000","last_clean":"2024-08-17T17:39:08.349153+0000","last_became_active":"2024-08-17T17:38:07.875196+0000","last_became_peered":"2024-08-17T17:38:07.875196+0000","last_unstale":"2024-08-17T17:39:08.349153+0000","last_undegraded":"2024-08-17T17:39:08.349153+0000","last_fullsized":"2024-08-17T17:39:08.349153+0000","mapping_epoch":1415,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5589,"log_dups_size":0,"ondisk_log_size":5589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:21.658004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154992640,"num_objects":2365,"num_object_clones":0,"num_object_copies":7095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5589,"num_write_kb":254528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3836,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1485,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.158551+0000","last_change":"2024-08-17T17:36:12.759743+0000","last_active":"2024-08-17T17:39:10.158551+0000","last_peered":"2024-08-17T17:39:10.158551+0000","last_clean":"2024-08-17T17:39:10.158551+0000","last_became_active":"2024-08-17T17:36:12.759334+0000","last_became_peered":"2024-08-17T17:36:12.759334+0000","last_unstale":"2024-08-17T17:39:10.158551+0000","last_undegraded":"2024-08-17T17:39:10.158551+0000","last_fullsized":"2024-08-17T17:39:10.158551+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:04.524384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1410,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145484+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:38:57.145484+0000","last_peered":"2024-08-17T17:38:57.145484+0000","last_clean":"2024-08-17T17:38:57.145484+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:38:57.145484+0000","last_undegraded":"2024-08-17T17:38:57.145484+0000","last_fullsized":"2024-08-17T17:38:57.145484+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1453,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.130233+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:38:57.130233+0000","last_peered":"2024-08-17T17:38:57.130233+0000","last_clean":"2024-08-17T17:38:57.130233+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:38:57.130233+0000","last_undegraded":"2024-08-17T17:38:57.130233+0000","last_fullsized":"2024-08-17T17:38:57.130233+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1768,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128411+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:38:57.128411+0000","last_peered":"2024-08-17T17:38:57.128411+0000","last_clean":"2024-08-17T17:38:57.128411+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:38:57.128411+0000","last_undegraded":"2024-08-17T17:38:57.128411+0000","last_fullsized":"2024-08-17T17:38:57.128411+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"1423'5811","reported_seq":11068,"reported_epoch":1423,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:39:11.936916+0000","last_change":"2024-08-17T17:38:05.991715+0000","last_active":"2024-08-17T17:39:11.936916+0000","last_peered":"2024-08-17T17:39:11.936916+0000","last_clean":"2024-08-17T17:35:54.187688+0000","last_became_active":"2024-08-17T17:36:22.904523+0000","last_became_peered":"2024-08-17T17:36:22.904523+0000","last_unstale":"2024-08-17T17:39:11.936916+0000","last_undegraded":"2024-08-17T17:39:11.936916+0000","last_fullsized":"2024-08-17T17:36:13.740491+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5811,"log_dups_size":0,"ondisk_log_size":5811,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:57.525855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155779072,"num_objects":2377,"num_object_clones":0,"num_object_copies":7131,"num_objects_missing_on_primary":1167,"num_objects_missing":1167,"num_objects_degraded":0,"num_objects_misplaced":3413,"num_objects_unfound":0,"num_objects_dirty":2377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5811,"num_write_kb":262016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1722,"num_bytes_recovered":112852992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":1167},{"shards":"2,4,5,6","objects":2},{"shards":"2,5,6","objects":1077},{"shards":"5,6","objects":131}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1086,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.129118+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:38:57.129118+0000","last_peered":"2024-08-17T17:38:57.129118+0000","last_clean":"2024-08-17T17:38:57.129118+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:38:57.129118+0000","last_undegraded":"2024-08-17T17:38:57.129118+0000","last_fullsized":"2024-08-17T17:38:57.129118+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1397,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.156441+0000","last_change":"2024-08-17T17:35:56.229622+0000","last_active":"2024-08-17T17:39:10.156441+0000","last_peered":"2024-08-17T17:39:10.156441+0000","last_clean":"2024-08-17T17:39:10.156441+0000","last_became_active":"2024-08-17T17:35:56.229324+0000","last_became_peered":"2024-08-17T17:35:56.229324+0000","last_unstale":"2024-08-17T17:39:10.156441+0000","last_undegraded":"2024-08-17T17:39:10.156441+0000","last_fullsized":"2024-08-17T17:39:10.156441+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:13.980316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2000,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.156915+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:39:10.156915+0000","last_peered":"2024-08-17T17:39:10.156915+0000","last_clean":"2024-08-17T17:39:10.156915+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:39:10.156915+0000","last_undegraded":"2024-08-17T17:39:10.156915+0000","last_fullsized":"2024-08-17T17:39:10.156915+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1583,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:39:02.424954+0000","last_change":"2024-08-17T17:35:33.137074+0000","last_active":"2024-08-17T17:39:02.424954+0000","last_peered":"2024-08-17T17:39:02.424954+0000","last_clean":"2024-08-17T17:39:02.424954+0000","last_became_active":"2024-08-17T17:35:33.136759+0000","last_became_peered":"2024-08-17T17:35:33.136759+0000","last_unstale":"2024-08-17T17:39:02.424954+0000","last_undegraded":"2024-08-17T17:39:02.424954+0000","last_fullsized":"2024-08-17T17:39:02.424954+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:37.263639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:37.263639+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:14.599780+0000","scrub_duration":0.0016181800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"1423'5764","reported_seq":10318,"reported_epoch":1423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:10.493823+0000","last_change":"2024-08-17T17:36:23.019129+0000","last_active":"2024-08-17T17:39:10.493823+0000","last_peered":"2024-08-17T17:39:10.493823+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:23.013527+0000","last_became_peered":"2024-08-17T17:36:23.013527+0000","last_unstale":"2024-08-17T17:39:10.493823+0000","last_undegraded":"2024-08-17T17:36:13.874715+0000","last_fullsized":"2024-08-17T17:36:13.738828+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5764,"log_dups_size":0,"ondisk_log_size":5764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153092096,"num_objects":2336,"num_object_clones":0,"num_object_copies":7008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4107,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5764,"num_write_kb":259200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1715,"num_bytes_recovered":112394240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[0,6],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":1161},{"shards":"0,1,6","objects":1048},{"shards":"0,6","objects":-610}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1190,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.142418+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:39:10.142418+0000","last_peered":"2024-08-17T17:39:10.142418+0000","last_clean":"2024-08-17T17:39:10.142418+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:39:10.142418+0000","last_undegraded":"2024-08-17T17:39:10.142418+0000","last_fullsized":"2024-08-17T17:39:10.142418+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":47,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.144478+0000","last_change":"2024-08-17T17:36:17.774773+0000","last_active":"2024-08-17T17:38:57.144478+0000","last_peered":"2024-08-17T17:38:57.144478+0000","last_clean":"2024-08-17T17:38:57.144478+0000","last_became_active":"2024-08-17T17:36:17.774468+0000","last_became_peered":"2024-08-17T17:36:17.774468+0000","last_unstale":"2024-08-17T17:38:57.144478+0000","last_undegraded":"2024-08-17T17:38:57.144478+0000","last_fullsized":"2024-08-17T17:38:57.144478+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:54.231747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1528,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.141314+0000","last_change":"2024-08-17T17:35:33.137633+0000","last_active":"2024-08-17T17:39:10.141314+0000","last_peered":"2024-08-17T17:39:10.141314+0000","last_clean":"2024-08-17T17:39:10.141314+0000","last_became_active":"2024-08-17T17:35:33.137323+0000","last_became_peered":"2024-08-17T17:35:33.137323+0000","last_unstale":"2024-08-17T17:39:10.141314+0000","last_undegraded":"2024-08-17T17:39:10.141314+0000","last_fullsized":"2024-08-17T17:39:10.141314+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:03.359155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":898,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122009+0000","last_change":"2024-08-17T17:35:56.264626+0000","last_active":"2024-08-17T17:38:57.122009+0000","last_peered":"2024-08-17T17:38:57.122009+0000","last_clean":"2024-08-17T17:38:57.122009+0000","last_became_active":"2024-08-17T17:35:56.264265+0000","last_became_peered":"2024-08-17T17:35:56.264265+0000","last_unstale":"2024-08-17T17:38:57.122009+0000","last_undegraded":"2024-08-17T17:38:57.122009+0000","last_fullsized":"2024-08-17T17:38:57.122009+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:20.754326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:20.754326+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"1423'5677","reported_seq":10773,"reported_epoch":1423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:11.982058+0000","last_change":"2024-08-17T17:37:11.134558+0000","last_active":"2024-08-17T17:39:11.982058+0000","last_peered":"2024-08-17T17:39:11.982058+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:20.105110+0000","last_became_peered":"2024-08-17T17:36:20.105110+0000","last_unstale":"2024-08-17T17:39:11.982058+0000","last_undegraded":"2024-08-17T17:36:19.999272+0000","last_fullsized":"2024-08-17T17:36:13.740520+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5677,"log_dups_size":0,"ondisk_log_size":5677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:45.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153157632,"num_objects":2337,"num_object_clones":0,"num_object_copies":7011,"num_objects_missing_on_primary":1143,"num_objects_missing":1143,"num_objects_degraded":3326,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5677,"num_write_kb":256448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1670,"num_bytes_recovered":109445120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1143},{"shards":"1,5,6","objects":1040},{"shards":"5,6","objects":154}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1251,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122314+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:38:57.122314+0000","last_peered":"2024-08-17T17:38:57.122314+0000","last_clean":"2024-08-17T17:38:57.122314+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:38:57.122314+0000","last_undegraded":"2024-08-17T17:38:57.122314+0000","last_fullsized":"2024-08-17T17:38:57.122314+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1791,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.130137+0000","last_change":"2024-08-17T17:36:12.758005+0000","last_active":"2024-08-17T17:38:57.130137+0000","last_peered":"2024-08-17T17:38:57.130137+0000","last_clean":"2024-08-17T17:38:57.130137+0000","last_became_active":"2024-08-17T17:36:12.757695+0000","last_became_peered":"2024-08-17T17:36:12.757695+0000","last_unstale":"2024-08-17T17:38:57.130137+0000","last_undegraded":"2024-08-17T17:38:57.130137+0000","last_fullsized":"2024-08-17T17:38:57.130137+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:51.792220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1429,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128770+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:38:57.128770+0000","last_peered":"2024-08-17T17:38:57.128770+0000","last_clean":"2024-08-17T17:38:57.128770+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:38:57.128770+0000","last_undegraded":"2024-08-17T17:38:57.128770+0000","last_fullsized":"2024-08-17T17:38:57.128770+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1732,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.141505+0000","last_change":"2024-08-17T17:39:10.141505+0000","last_active":"2024-08-17T17:39:10.141505+0000","last_peered":"2024-08-17T17:39:10.141505+0000","last_clean":"2024-08-17T17:39:10.141505+0000","last_became_active":"2024-08-17T17:35:55.204861+0000","last_became_peered":"2024-08-17T17:35:55.204861+0000","last_unstale":"2024-08-17T17:39:10.141505+0000","last_undegraded":"2024-08-17T17:39:10.141505+0000","last_fullsized":"2024-08-17T17:39:10.141505+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:39:15.055842+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.0","version":"1422'5704","reported_seq":13692,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:39:07.404573+0000","last_change":"2024-08-17T17:37:40.753703+0000","last_active":"2024-08-17T17:39:07.404573+0000","last_peered":"2024-08-17T17:39:07.404573+0000","last_clean":"2024-08-17T17:39:07.404573+0000","last_became_active":"2024-08-17T17:37:40.743458+0000","last_became_peered":"2024-08-17T17:37:40.743458+0000","last_unstale":"2024-08-17T17:39:07.404573+0000","last_undegraded":"2024-08-17T17:39:07.404573+0000","last_fullsized":"2024-08-17T17:39:07.404573+0000","mapping_epoch":1411,"log_start":"1373'200","ondisk_log_start":"1373'200","created":1357,"last_epoch_clean":1412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5504,"log_dups_size":0,"ondisk_log_size":5504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:14:02.340380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158072832,"num_objects":2412,"num_object_clones":0,"num_object_copies":7236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5704,"num_write_kb":259712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3914,"num_bytes_recovered":232652800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1024,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145498+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:38:57.145498+0000","last_peered":"2024-08-17T17:38:57.145498+0000","last_clean":"2024-08-17T17:38:57.145498+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:38:57.145498+0000","last_undegraded":"2024-08-17T17:38:57.145498+0000","last_fullsized":"2024-08-17T17:38:57.145498+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":91,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128507+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:38:57.128507+0000","last_peered":"2024-08-17T17:38:57.128507+0000","last_clean":"2024-08-17T17:38:57.128507+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:38:57.128507+0000","last_undegraded":"2024-08-17T17:38:57.128507+0000","last_fullsized":"2024-08-17T17:38:57.128507+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1760,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128875+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:38:57.128875+0000","last_peered":"2024-08-17T17:38:57.128875+0000","last_clean":"2024-08-17T17:38:57.128875+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:38:57.128875+0000","last_undegraded":"2024-08-17T17:38:57.128875+0000","last_fullsized":"2024-08-17T17:38:57.128875+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1","version":"1423'5712","reported_seq":10872,"reported_epoch":1423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:11.986124+0000","last_change":"2024-08-17T17:36:23.103523+0000","last_active":"2024-08-17T17:39:11.986124+0000","last_peered":"2024-08-17T17:39:11.986124+0000","last_clean":"2024-08-17T17:35:54.178280+0000","last_became_active":"2024-08-17T17:36:23.025910+0000","last_became_peered":"2024-08-17T17:36:23.025910+0000","last_unstale":"2024-08-17T17:39:11.986124+0000","last_undegraded":"2024-08-17T17:36:18.734637+0000","last_fullsized":"2024-08-17T17:36:13.740134+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5712,"log_dups_size":0,"ondisk_log_size":5712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:58.717055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153354240,"num_objects":2340,"num_object_clones":0,"num_object_copies":7020,"num_objects_missing_on_primary":1141,"num_objects_missing":1141,"num_objects_degraded":3355,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5712,"num_write_kb":257664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1690,"num_bytes_recovered":110755840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,0],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":1141},{"shards":"0,2,4,6","objects":3},{"shards":"0,2,6","objects":1070},{"shards":"0,6","objects":126}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1439,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145962+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:38:57.145962+0000","last_peered":"2024-08-17T17:38:57.145962+0000","last_clean":"2024-08-17T17:38:57.145962+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:38:57.145962+0000","last_undegraded":"2024-08-17T17:38:57.145962+0000","last_fullsized":"2024-08-17T17:38:57.145962+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1412,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.182659+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:39:10.182659+0000","last_peered":"2024-08-17T17:39:10.182659+0000","last_clean":"2024-08-17T17:39:10.182659+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:39:10.182659+0000","last_undegraded":"2024-08-17T17:39:10.182659+0000","last_fullsized":"2024-08-17T17:39:10.182659+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":994,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.142403+0000","last_change":"2024-08-17T17:34:26.817339+0000","last_active":"2024-08-17T17:39:10.142403+0000","last_peered":"2024-08-17T17:39:10.142403+0000","last_clean":"2024-08-17T17:39:10.142403+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:39:10.142403+0000","last_undegraded":"2024-08-17T17:39:10.142403+0000","last_fullsized":"2024-08-17T17:39:10.142403+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:26.817223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:26.817223+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:30.089956+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"1423'5578","reported_seq":5738,"reported_epoch":1423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:10.249785+0000","last_change":"2024-08-17T17:36:14.073155+0000","last_active":"2024-08-17T17:39:10.249785+0000","last_peered":"2024-08-17T17:39:10.249785+0000","last_clean":"2024-08-17T17:36:11.734683+0000","last_became_active":"2024-08-17T17:36:14.063794+0000","last_became_peered":"2024-08-17T17:36:14.063794+0000","last_unstale":"2024-08-17T17:39:10.249785+0000","last_undegraded":"2024-08-17T17:36:13.855770+0000","last_fullsized":"2024-08-17T17:36:13.742068+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5578,"log_dups_size":0,"ondisk_log_size":5578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152305664,"num_objects":2324,"num_object_clones":0,"num_object_copies":6972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3819,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5578,"num_write_kb":252864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-1495},{"shards":"0,2,7","objects":2192}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":47,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145945+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:38:57.145945+0000","last_peered":"2024-08-17T17:38:57.145945+0000","last_clean":"2024-08-17T17:38:57.145945+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:38:57.145945+0000","last_undegraded":"2024-08-17T17:38:57.145945+0000","last_fullsized":"2024-08-17T17:38:57.145945+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1273,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.133942+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:38:57.133942+0000","last_peered":"2024-08-17T17:38:57.133942+0000","last_clean":"2024-08-17T17:38:57.133942+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:38:57.133942+0000","last_undegraded":"2024-08-17T17:38:57.133942+0000","last_fullsized":"2024-08-17T17:38:57.133942+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1788,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128483+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:38:57.128483+0000","last_peered":"2024-08-17T17:38:57.128483+0000","last_clean":"2024-08-17T17:38:57.128483+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:38:57.128483+0000","last_undegraded":"2024-08-17T17:38:57.128483+0000","last_fullsized":"2024-08-17T17:38:57.128483+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"1423'5745","reported_seq":11543,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:11.766232+0000","last_change":"2024-08-17T17:36:46.982456+0000","last_active":"2024-08-17T17:39:11.766232+0000","last_peered":"2024-08-17T17:39:11.766232+0000","last_clean":"2024-08-17T17:39:11.766232+0000","last_became_active":"2024-08-17T17:36:46.974782+0000","last_became_peered":"2024-08-17T17:36:46.974782+0000","last_unstale":"2024-08-17T17:39:11.766232+0000","last_undegraded":"2024-08-17T17:39:11.766232+0000","last_fullsized":"2024-08-17T17:39:11.766232+0000","mapping_epoch":1399,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:40.499321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152502272,"num_objects":2327,"num_object_clones":0,"num_object_copies":6981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5745,"num_write_kb":258304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2848,"num_bytes_recovered":186646528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1390,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.146460+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:38:57.146460+0000","last_peered":"2024-08-17T17:38:57.146460+0000","last_clean":"2024-08-17T17:38:57.146460+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:38:57.146460+0000","last_undegraded":"2024-08-17T17:38:57.146460+0000","last_fullsized":"2024-08-17T17:38:57.146460+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":203,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145260+0000","last_change":"2024-08-17T17:36:12.754372+0000","last_active":"2024-08-17T17:38:57.145260+0000","last_peered":"2024-08-17T17:38:57.145260+0000","last_clean":"2024-08-17T17:38:57.145260+0000","last_became_active":"2024-08-17T17:36:12.746352+0000","last_became_peered":"2024-08-17T17:36:12.746352+0000","last_unstale":"2024-08-17T17:38:57.145260+0000","last_undegraded":"2024-08-17T17:38:57.145260+0000","last_fullsized":"2024-08-17T17:38:57.145260+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:07.807486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1666,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.156738+0000","last_change":"2024-08-17T17:36:15.771193+0000","last_active":"2024-08-17T17:39:10.156738+0000","last_peered":"2024-08-17T17:39:10.156738+0000","last_clean":"2024-08-17T17:39:10.156738+0000","last_became_active":"2024-08-17T17:36:15.770889+0000","last_became_peered":"2024-08-17T17:36:15.770889+0000","last_unstale":"2024-08-17T17:39:10.156738+0000","last_undegraded":"2024-08-17T17:39:10.156738+0000","last_fullsized":"2024-08-17T17:39:10.156738+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:22.978695+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.4","version":"1423'5705","reported_seq":11484,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:11.831325+0000","last_change":"2024-08-17T17:37:04.045576+0000","last_active":"2024-08-17T17:39:11.831325+0000","last_peered":"2024-08-17T17:39:11.831325+0000","last_clean":"2024-08-17T17:39:11.831325+0000","last_became_active":"2024-08-17T17:37:04.038810+0000","last_became_peered":"2024-08-17T17:37:04.038810+0000","last_unstale":"2024-08-17T17:39:11.831325+0000","last_undegraded":"2024-08-17T17:39:11.831325+0000","last_fullsized":"2024-08-17T17:39:11.831325+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5705,"log_dups_size":0,"ondisk_log_size":5705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:27:04.725258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154730496,"num_objects":2361,"num_object_clones":0,"num_object_copies":7083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5705,"num_write_kb":258112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":176291840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1261,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.142693+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:39:10.142693+0000","last_peered":"2024-08-17T17:39:10.142693+0000","last_clean":"2024-08-17T17:39:10.142693+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:39:10.142693+0000","last_undegraded":"2024-08-17T17:39:10.142693+0000","last_fullsized":"2024-08-17T17:39:10.142693+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1416,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128915+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:38:57.128915+0000","last_peered":"2024-08-17T17:38:57.128915+0000","last_clean":"2024-08-17T17:38:57.128915+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:38:57.128915+0000","last_undegraded":"2024-08-17T17:38:57.128915+0000","last_fullsized":"2024-08-17T17:38:57.128915+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":916,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145735+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:38:57.145735+0000","last_peered":"2024-08-17T17:38:57.145735+0000","last_clean":"2024-08-17T17:38:57.145735+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:38:57.145735+0000","last_undegraded":"2024-08-17T17:38:57.145735+0000","last_fullsized":"2024-08-17T17:38:57.145735+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"1423'5774","reported_seq":11608,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:12.077816+0000","last_change":"2024-08-17T17:36:32.711490+0000","last_active":"2024-08-17T17:39:12.077816+0000","last_peered":"2024-08-17T17:39:12.077816+0000","last_clean":"2024-08-17T17:39:12.077816+0000","last_became_active":"2024-08-17T17:36:32.704370+0000","last_became_peered":"2024-08-17T17:36:32.704370+0000","last_unstale":"2024-08-17T17:39:12.077816+0000","last_undegraded":"2024-08-17T17:39:12.077816+0000","last_fullsized":"2024-08-17T17:39:12.077816+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5774,"log_dups_size":0,"ondisk_log_size":5774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:17.373879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152961024,"num_objects":2334,"num_object_clones":0,"num_object_copies":7002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5774,"num_write_kb":259456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2866,"num_bytes_recovered":187826176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1272,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.144876+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:38:57.144876+0000","last_peered":"2024-08-17T17:38:57.144876+0000","last_clean":"2024-08-17T17:38:57.144876+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:38:57.144876+0000","last_undegraded":"2024-08-17T17:38:57.144876+0000","last_fullsized":"2024-08-17T17:38:57.144876+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":152,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128927+0000","last_change":"2024-08-17T17:35:55.217072+0000","last_active":"2024-08-17T17:38:57.128927+0000","last_peered":"2024-08-17T17:38:57.128927+0000","last_clean":"2024-08-17T17:38:57.128927+0000","last_became_active":"2024-08-17T17:35:55.216462+0000","last_became_peered":"2024-08-17T17:35:55.216462+0000","last_unstale":"2024-08-17T17:38:57.128927+0000","last_undegraded":"2024-08-17T17:38:57.128927+0000","last_fullsized":"2024-08-17T17:38:57.128927+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:34.937221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":899,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122468+0000","last_change":"2024-08-17T17:35:55.215004+0000","last_active":"2024-08-17T17:38:57.122468+0000","last_peered":"2024-08-17T17:38:57.122468+0000","last_clean":"2024-08-17T17:38:57.122468+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:38:57.122468+0000","last_undegraded":"2024-08-17T17:38:57.122468+0000","last_fullsized":"2024-08-17T17:38:57.122468+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.6","version":"1423'5586","reported_seq":5722,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.588890+0000","last_change":"2024-08-17T17:35:14.868524+0000","last_active":"2024-08-17T17:39:10.588890+0000","last_peered":"2024-08-17T17:39:10.588890+0000","last_clean":"2024-08-17T17:39:10.588890+0000","last_became_active":"2024-08-17T17:35:14.854690+0000","last_became_peered":"2024-08-17T17:35:14.854690+0000","last_unstale":"2024-08-17T17:39:10.588890+0000","last_undegraded":"2024-08-17T17:39:10.588890+0000","last_fullsized":"2024-08-17T17:39:10.588890+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5586,"log_dups_size":0,"ondisk_log_size":5586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:05.388362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146145280,"num_objects":2230,"num_object_clones":0,"num_object_copies":6690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5586,"num_write_kb":250112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1457,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.144613+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:38:57.144613+0000","last_peered":"2024-08-17T17:38:57.144613+0000","last_clean":"2024-08-17T17:38:57.144613+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:38:57.144613+0000","last_undegraded":"2024-08-17T17:38:57.144613+0000","last_fullsized":"2024-08-17T17:38:57.144613+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":164,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.146041+0000","last_change":"2024-08-17T17:35:33.147672+0000","last_active":"2024-08-17T17:38:57.146041+0000","last_peered":"2024-08-17T17:38:57.146041+0000","last_clean":"2024-08-17T17:38:57.146041+0000","last_became_active":"2024-08-17T17:35:33.147283+0000","last_became_peered":"2024-08-17T17:35:33.147283+0000","last_unstale":"2024-08-17T17:38:57.146041+0000","last_undegraded":"2024-08-17T17:38:57.146041+0000","last_fullsized":"2024-08-17T17:38:57.146041+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:41.677709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1662,"reported_epoch":1422,"state":"active+clean+scrubbing","last_fresh":"2024-08-17T17:38:57.128717+0000","last_change":"2024-08-17T17:39:02.926756+0000","last_active":"2024-08-17T17:38:57.128717+0000","last_peered":"2024-08-17T17:38:57.128717+0000","last_clean":"2024-08-17T17:38:57.128717+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:38:57.128717+0000","last_undegraded":"2024-08-17T17:38:57.128717+0000","last_fullsized":"2024-08-17T17:38:57.128717+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:38.241235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:38.241235+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"1423'5613","reported_seq":5689,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:12.280928+0000","last_change":"2024-08-17T17:34:50.670013+0000","last_active":"2024-08-17T17:39:12.280928+0000","last_peered":"2024-08-17T17:39:12.280928+0000","last_clean":"2024-08-17T17:39:12.280928+0000","last_became_active":"2024-08-17T17:34:50.669299+0000","last_became_peered":"2024-08-17T17:34:50.669299+0000","last_unstale":"2024-08-17T17:39:12.280928+0000","last_undegraded":"2024-08-17T17:39:12.280928+0000","last_fullsized":"2024-08-17T17:39:12.280928+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5613,"log_dups_size":0,"ondisk_log_size":5613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:11.172227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151584768,"num_objects":2313,"num_object_clones":0,"num_object_copies":6939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5613,"num_write_kb":253632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1382,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.151040+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:39:10.151040+0000","last_peered":"2024-08-17T17:39:10.151040+0000","last_clean":"2024-08-17T17:39:10.151040+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:39:10.151040+0000","last_undegraded":"2024-08-17T17:39:10.151040+0000","last_fullsized":"2024-08-17T17:39:10.151040+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":151,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145640+0000","last_change":"2024-08-17T17:35:33.140725+0000","last_active":"2024-08-17T17:38:57.145640+0000","last_peered":"2024-08-17T17:38:57.145640+0000","last_clean":"2024-08-17T17:38:57.145640+0000","last_became_active":"2024-08-17T17:35:33.140406+0000","last_became_peered":"2024-08-17T17:35:33.140406+0000","last_unstale":"2024-08-17T17:38:57.145640+0000","last_undegraded":"2024-08-17T17:38:57.145640+0000","last_fullsized":"2024-08-17T17:38:57.145640+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:54:51.875571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1671,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145742+0000","last_change":"2024-08-17T17:35:33.138514+0000","last_active":"2024-08-17T17:38:57.145742+0000","last_peered":"2024-08-17T17:38:57.145742+0000","last_clean":"2024-08-17T17:38:57.145742+0000","last_became_active":"2024-08-17T17:35:33.137035+0000","last_became_peered":"2024-08-17T17:35:33.137035+0000","last_unstale":"2024-08-17T17:38:57.145742+0000","last_undegraded":"2024-08-17T17:38:57.145742+0000","last_fullsized":"2024-08-17T17:38:57.145742+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:50.565662+0000","scrub_duration":0.001487138,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1313,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.140471+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:38:57.140471+0000","last_peered":"2024-08-17T17:38:57.140471+0000","last_clean":"2024-08-17T17:38:57.140471+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:38:57.140471+0000","last_undegraded":"2024-08-17T17:38:57.140471+0000","last_fullsized":"2024-08-17T17:38:57.140471+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":91,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.129104+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:38:57.129104+0000","last_peered":"2024-08-17T17:38:57.129104+0000","last_clean":"2024-08-17T17:38:57.129104+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:38:57.129104+0000","last_undegraded":"2024-08-17T17:38:57.129104+0000","last_fullsized":"2024-08-17T17:38:57.129104+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1744,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128957+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:38:57.128957+0000","last_peered":"2024-08-17T17:38:57.128957+0000","last_clean":"2024-08-17T17:38:57.128957+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:38:57.128957+0000","last_undegraded":"2024-08-17T17:38:57.128957+0000","last_fullsized":"2024-08-17T17:38:57.128957+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1330,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.144560+0000","last_change":"2024-08-17T17:35:55.205292+0000","last_active":"2024-08-17T17:38:57.144560+0000","last_peered":"2024-08-17T17:38:57.144560+0000","last_clean":"2024-08-17T17:38:57.144560+0000","last_became_active":"2024-08-17T17:35:55.204935+0000","last_became_peered":"2024-08-17T17:35:55.204935+0000","last_unstale":"2024-08-17T17:38:57.144560+0000","last_undegraded":"2024-08-17T17:38:57.144560+0000","last_fullsized":"2024-08-17T17:38:57.144560+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:52.277821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":91,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.129143+0000","last_change":"2024-08-17T17:36:12.749972+0000","last_active":"2024-08-17T17:38:57.129143+0000","last_peered":"2024-08-17T17:38:57.129143+0000","last_clean":"2024-08-17T17:38:57.129143+0000","last_became_active":"2024-08-17T17:36:12.749620+0000","last_became_peered":"2024-08-17T17:36:12.749620+0000","last_unstale":"2024-08-17T17:38:57.129143+0000","last_undegraded":"2024-08-17T17:38:57.129143+0000","last_fullsized":"2024-08-17T17:38:57.129143+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:04.994357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":967,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128529+0000","last_change":"2024-08-17T17:36:20.832274+0000","last_active":"2024-08-17T17:38:57.128529+0000","last_peered":"2024-08-17T17:38:57.128529+0000","last_clean":"2024-08-17T17:38:57.128529+0000","last_became_active":"2024-08-17T17:36:20.831886+0000","last_became_peered":"2024-08-17T17:36:20.831886+0000","last_unstale":"2024-08-17T17:38:57.128529+0000","last_undegraded":"2024-08-17T17:38:57.128529+0000","last_fullsized":"2024-08-17T17:38:57.128529+0000","mapping_epoch":1390,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1391,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:05.248711+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1178,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.141910+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:39:10.141910+0000","last_peered":"2024-08-17T17:39:10.141910+0000","last_clean":"2024-08-17T17:39:10.141910+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:39:10.141910+0000","last_undegraded":"2024-08-17T17:39:10.141910+0000","last_fullsized":"2024-08-17T17:39:10.141910+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1413,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128973+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:38:57.128973+0000","last_peered":"2024-08-17T17:38:57.128973+0000","last_clean":"2024-08-17T17:38:57.128973+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:38:57.128973+0000","last_undegraded":"2024-08-17T17:38:57.128973+0000","last_fullsized":"2024-08-17T17:38:57.128973+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1707,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128396+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:38:57.128396+0000","last_peered":"2024-08-17T17:38:57.128396+0000","last_clean":"2024-08-17T17:38:57.128396+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:38:57.128396+0000","last_undegraded":"2024-08-17T17:38:57.128396+0000","last_fullsized":"2024-08-17T17:38:57.128396+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1355,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.139378+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:38:57.139378+0000","last_peered":"2024-08-17T17:38:57.139378+0000","last_clean":"2024-08-17T17:38:57.139378+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:38:57.139378+0000","last_undegraded":"2024-08-17T17:38:57.139378+0000","last_fullsized":"2024-08-17T17:38:57.139378+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1895,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128775+0000","last_change":"2024-08-17T17:35:33.137411+0000","last_active":"2024-08-17T17:38:57.128775+0000","last_peered":"2024-08-17T17:38:57.128775+0000","last_clean":"2024-08-17T17:38:57.128775+0000","last_became_active":"2024-08-17T17:35:33.137073+0000","last_became_peered":"2024-08-17T17:35:33.137073+0000","last_unstale":"2024-08-17T17:38:57.128775+0000","last_undegraded":"2024-08-17T17:38:57.128775+0000","last_fullsized":"2024-08-17T17:38:57.128775+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:35.924539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1801,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128483+0000","last_change":"2024-08-17T17:36:15.787445+0000","last_active":"2024-08-17T17:38:57.128483+0000","last_peered":"2024-08-17T17:38:57.128483+0000","last_clean":"2024-08-17T17:38:57.128483+0000","last_became_active":"2024-08-17T17:36:15.787099+0000","last_became_peered":"2024-08-17T17:36:15.787099+0000","last_unstale":"2024-08-17T17:38:57.128483+0000","last_undegraded":"2024-08-17T17:38:57.128483+0000","last_fullsized":"2024-08-17T17:38:57.128483+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:47.448168+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":2459828248,"num_objects":37535,"num_object_clones":0,"num_object_copies":112605,"num_objects_missing_on_primary":3451,"num_objects_missing":3451,"num_objects_degraded":14607,"num_objects_misplaced":3413,"num_objects_unfound":0,"num_objects_dirty":37535,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":91347,"num_write_kb":4124161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29971,"num_bytes_recovered":1811546112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8269934592,"data_stored":8269922376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":91147,"ondisk_log_size":91147,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635092,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":27,"num_read_kb":61,"num_write":15,"num_write_kb":183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1314910,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162861,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1304684,"kb_used_data":1162212,"kb_used_omap":54,"kb_used_meta":142409,"kb_avail":92440468,"statfs":{"total":95995035648,"available":94659039232,"internally_reserved":0,"allocated":1190105088,"data_stored":1187426988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":145826837},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999335,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1172080,"kb_used_data":982052,"kb_used_omap":47,"kb_used_meta":189968,"kb_avail":92573072,"statfs":{"total":95995035648,"available":94794825728,"internally_reserved":0,"allocated":1005621248,"data_stored":1002944125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":194527892},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,2,0,0,4,8,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627131,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1381700,"kb_used_data":356924,"kb_used_omap":42,"kb_used_meta":1024725,"kb_avail":92363452,"statfs":{"total":95995035648,"available":94580174848,"internally_reserved":0,"allocated":365490176,"data_stored":362839304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1049318460},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785608,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2544968,"kb_used_data":1728320,"kb_used_omap":48,"kb_used_meta":816591,"kb_avail":91200184,"statfs":{"total":95995035648,"available":93388988416,"internally_reserved":0,"allocated":1769799680,"data_stored":1767145683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":836189870},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725126,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1478172,"kb_used_data":608724,"kb_used_omap":1,"kb_used_meta":869438,"kb_avail":92266980,"statfs":{"total":95995035648,"available":94481387520,"internally_reserved":0,"allocated":623333376,"data_stored":620667359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":890304966},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986763,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1863912,"kb_used_data":976416,"kb_used_omap":1,"kb_used_meta":887486,"kb_avail":91881240,"statfs":{"total":95995035648,"available":94086389760,"internally_reserved":0,"allocated":999849984,"data_stored":997175390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":908786118},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706370,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2337808,"kb_used_data":1413128,"kb_used_omap":35,"kb_used_meta":924636,"kb_avail":91407344,"statfs":{"total":95995035648,"available":93601120256,"internally_reserved":0,"allocated":1447043072,"data_stored":1444381006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":946827319},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832143,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1976780,"kb_used_data":933700,"kb_used_omap":65,"kb_used_meta":1043006,"kb_avail":91768372,"statfs":{"total":95995035648,"available":93970812928,"internally_reserved":0,"allocated":956108800,"data_stored":953451292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1068038331},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":945029120,"data_stored":945029120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1760034816,"data_stored":1760034816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1434062848,"data_stored":1434058776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":356515840,"data_stored":356515840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":989007872,"data_stored":989003800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":612564992,"data_stored":612564992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":996282368,"data_stored":996278296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1176436736,"data_stored":1176436736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:39:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:13 smithi162 ceph-mon[104668]: pgmap v2242: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 210 op/s; 14607/112683 objects degraded (12.963%); 3413/112683 objects misplaced (3.029%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T17:39:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:13 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 14607/112683 objects degraded (12.963%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:14.145 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:14.169 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:14.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:14.271 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:14.293 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:14.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:14.395 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:14.420 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:14.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:14.522 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:14.544 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:14.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:14.645 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:14.671 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:14.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:14.773 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:14.796 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:14.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:14.802 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:14 smithi162 ceph-mon[104668]: from='client.46394 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:14.898 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:14.923 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:14.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:14.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:14.960 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:14.970 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:39:14.970 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:14.970 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:14 smithi063 ceph-mon[114756]: from='client.46394 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:14.971 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:14 smithi063 ceph-mon[109843]: from='client.46394 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:14.991 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:39:14.992 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:15.012 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:39:15.012 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:15.025 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:15.029 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:39:15.029 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:15.046 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:39:15.047 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:15.052 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:15.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:15.067 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:39:15.067 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:15.087 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:39:15.088 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:15.096 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:15.096 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:15.111 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:39:15.112 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:15.156 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:15.184 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:15.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:15.286 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:15.310 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:15.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:15.412 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:15.440 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:15.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:15.542 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:15.570 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:15.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:15.671 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:15.695 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:15.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:15.796 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:39:15.796 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:15.819 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:15.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:15.921 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:15.945 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:15.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:15.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:16.006 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:15 smithi063 ceph-mon[114756]: pgmap v2243: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 14607/112248 objects degraded (13.013%); 3413/112248 objects misplaced (3.041%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T17:39:16.006 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:15 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3529781087' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:16.006 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:15 smithi063 ceph-mon[109843]: pgmap v2243: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 14607/112248 objects degraded (13.013%); 3413/112248 objects misplaced (3.041%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T17:39:16.006 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:15 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3529781087' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:16.048 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:16.070 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:16.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:16.093 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:16.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:15 smithi162 ceph-mon[104668]: pgmap v2243: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 14607/112248 objects degraded (13.013%); 3413/112248 objects misplaced (3.041%); 3.5 MiB/s, 55 objects/s recovering 2024-08-17T17:39:16.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:15 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3529781087' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:16.171 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:16.201 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:16.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:16.303 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:16.329 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:16.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:16.431 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:16.457 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:16.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:16.558 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:16.581 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:16.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:16.683 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:16.707 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:16.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:16.809 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:16.832 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:16.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:16.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:16.934 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:16.958 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:16.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:16.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:16.978 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:16.978 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:17.060 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:17.084 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:39:17.087 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:17.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:17.189 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:17.213 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:17.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:17.315 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:17.349 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:17.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:17.450 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:17.477 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:17.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:17.569 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:17.579 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:17.608 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:17.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:17.710 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:17.738 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:17.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:17.839 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:17.871 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:17.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:17.973 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:17.997 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:17.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:17.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:17 smithi063 ceph-mon[114756]: pgmap v2244: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 14470/110880 objects degraded (13.050%); 3413/110880 objects misplaced (3.078%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:39:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:17 smithi063 ceph-mon[109843]: pgmap v2244: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 14470/110880 objects degraded (13.050%); 3413/110880 objects misplaced (3.078%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:39:18.092 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:17 smithi162 ceph-mon[104668]: pgmap v2244: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 14470/110880 objects degraded (13.050%); 3413/110880 objects misplaced (3.078%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:39:18.099 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:18.120 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:18.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:18.222 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:18.250 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:18.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:18.265 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:18.265 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:18.352 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:18.381 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:18.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:18.454 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:39:18.483 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:18.519 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:18.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:18.621 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:18.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.650 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:18.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:18.730 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:18 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2315110849' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:18.730 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:18 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:18.731 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:18 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2315110849' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:18.731 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:18 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:18.731 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:18 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 14470/110880 objects degraded (13.050%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:18.733 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:39:18.751 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:18.775 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:18.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:18.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:18 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2315110849' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:18.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:18 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:18.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:18 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 14470/110880 objects degraded (13.050%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:18.876 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:18.906 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:18.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:18.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:18.942 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:39:19.008 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:18 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 14470/110880 objects degraded (13.050%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:19.035 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:19.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:19.137 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:19.176 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:19.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:19.234 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:19.278 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:19.306 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:19.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:19.410 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:19.438 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:19.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:19.540 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:19.564 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:19.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:19.665 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:19.694 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:19.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:19.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:19 smithi063 ceph-mon[109843]: pgmap v2245: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 14165/108702 objects degraded (13.031%); 3267/108702 objects misplaced (3.005%); 3.6 MiB/s, 56 objects/s recovering 2024-08-17T17:39:19.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:19 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:39:19.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:19 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:19.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:19 smithi063 ceph-mon[109843]: osdmap e1424: 8 total, 8 up, 8 in 2024-08-17T17:39:19.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:19 smithi063 ceph-mon[114756]: pgmap v2245: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 14165/108702 objects degraded (13.031%); 3267/108702 objects misplaced (3.005%); 3.6 MiB/s, 56 objects/s recovering 2024-08-17T17:39:19.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:19 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:39:19.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:19 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:19.780 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:19 smithi063 ceph-mon[114756]: osdmap e1424: 8 total, 8 up, 8 in 2024-08-17T17:39:19.795 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:19.824 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:19.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:19.926 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:19.951 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:19.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:19.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:20.053 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:20.082 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:20.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:19 smithi162 ceph-mon[104668]: pgmap v2245: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 225 op/s; 14165/108702 objects degraded (13.031%); 3267/108702 objects misplaced (3.005%); 3.6 MiB/s, 56 objects/s recovering 2024-08-17T17:39:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:19 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:39:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:19 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:19 smithi162 ceph-mon[104668]: osdmap e1424: 8 total, 8 up, 8 in 2024-08-17T17:39:20.135 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:39:20.136 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:39:20.185 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:20.219 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:20.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:20.321 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:20.345 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:20.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:20.447 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:20.471 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:20.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:20.572 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:20.598 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:20.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:20.667 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:20.701 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:20.729 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:20.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:20.831 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:20.839 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:20 smithi162 ceph-mon[104668]: from='client.46412 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:20.858 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:20.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:20.960 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:20.988 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:20.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:20.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:21.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:20 smithi063 ceph-mon[114756]: from='client.46412 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:21.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:20 smithi063 ceph-mon[109843]: from='client.46412 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:21.090 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:21.119 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:21.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:21.127 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:21.127 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:21.221 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:21.245 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:21.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:21.346 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:21.370 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:21.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:21.396 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2246,"stamp":"2024-08-17T17:39:18.729778+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2369242801,"num_objects":36169,"num_object_clones":0,"num_object_copies":108507,"num_objects_missing_on_primary":3225,"num_objects_missing":3225,"num_objects_degraded":14165,"num_objects_misplaced":3267,"num_objects_unfound":0,"num_objects_dirty":36169,"num_whiteouts":0,"num_read":28,"num_read_kb":62,"num_write":92818,"num_write_kb":4124377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30288,"num_bytes_recovered":1832063709,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":96917,"ondisk_log_size":96917,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":375,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13837172,"kb_used_data":7917236,"kb_used_omap":298,"kb_used_meta":5919573,"kb_avail":736124044,"statfs":{"total":767960285184,"available":753791021056,"internally_reserved":0,"allocated":8107249664,"data_stored":8085913507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":6061643281},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,0,0,0,1,5,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-148242432,"num_objects":-2262,"num_object_clones":0,"num_object_copies":-6786,"num_objects_missing_on_primary":-435,"num_objects_missing":-435,"num_objects_degraded":-861,"num_objects_misplaced":-274,"num_objects_unfound":0,"num_objects_dirty":-2262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2262,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":570,"num_bytes_recovered":37355520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.589827"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":53,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.142065+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:39:10.142065+0000","last_peered":"2024-08-17T17:39:10.142065+0000","last_clean":"2024-08-17T17:39:10.142065+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:39:10.142065+0000","last_undegraded":"2024-08-17T17:39:10.142065+0000","last_fullsized":"2024-08-17T17:39:10.142065+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1437,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.144550+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:38:57.144550+0000","last_peered":"2024-08-17T17:38:57.144550+0000","last_clean":"2024-08-17T17:38:57.144550+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:38:57.144550+0000","last_undegraded":"2024-08-17T17:38:57.144550+0000","last_fullsized":"2024-08-17T17:38:57.144550+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1752,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145661+0000","last_change":"2024-08-17T17:36:15.771150+0000","last_active":"2024-08-17T17:38:57.145661+0000","last_peered":"2024-08-17T17:38:57.145661+0000","last_clean":"2024-08-17T17:38:57.145661+0000","last_became_active":"2024-08-17T17:36:15.770520+0000","last_became_peered":"2024-08-17T17:36:15.770520+0000","last_unstale":"2024-08-17T17:38:57.145661+0000","last_undegraded":"2024-08-17T17:38:57.145661+0000","last_fullsized":"2024-08-17T17:38:57.145661+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:17.552796+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":87,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122234+0000","last_change":"2024-08-17T17:36:12.756709+0000","last_active":"2024-08-17T17:38:57.122234+0000","last_peered":"2024-08-17T17:38:57.122234+0000","last_clean":"2024-08-17T17:38:57.122234+0000","last_became_active":"2024-08-17T17:36:12.756385+0000","last_became_peered":"2024-08-17T17:36:12.756385+0000","last_unstale":"2024-08-17T17:38:57.122234+0000","last_undegraded":"2024-08-17T17:38:57.122234+0000","last_fullsized":"2024-08-17T17:38:57.122234+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:21.420282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":137,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.156833+0000","last_change":"2024-08-17T17:36:12.762800+0000","last_active":"2024-08-17T17:39:10.156833+0000","last_peered":"2024-08-17T17:39:10.156833+0000","last_clean":"2024-08-17T17:39:10.156833+0000","last_became_active":"2024-08-17T17:36:12.759977+0000","last_became_peered":"2024-08-17T17:36:12.759977+0000","last_unstale":"2024-08-17T17:39:10.156833+0000","last_undegraded":"2024-08-17T17:39:10.156833+0000","last_fullsized":"2024-08-17T17:39:10.156833+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:02.162681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1632,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122092+0000","last_change":"2024-08-17T17:35:56.281561+0000","last_active":"2024-08-17T17:38:57.122092+0000","last_peered":"2024-08-17T17:38:57.122092+0000","last_clean":"2024-08-17T17:38:57.122092+0000","last_became_active":"2024-08-17T17:35:56.278638+0000","last_became_peered":"2024-08-17T17:35:56.278638+0000","last_unstale":"2024-08-17T17:38:57.122092+0000","last_undegraded":"2024-08-17T17:38:57.122092+0000","last_fullsized":"2024-08-17T17:38:57.122092+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1371,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.129014+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:38:57.129014+0000","last_peered":"2024-08-17T17:38:57.129014+0000","last_clean":"2024-08-17T17:38:57.129014+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:38:57.129014+0000","last_undegraded":"2024-08-17T17:38:57.129014+0000","last_fullsized":"2024-08-17T17:38:57.129014+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":47,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145121+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:38:57.145121+0000","last_peered":"2024-08-17T17:38:57.145121+0000","last_clean":"2024-08-17T17:38:57.145121+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:38:57.145121+0000","last_undegraded":"2024-08-17T17:38:57.145121+0000","last_fullsized":"2024-08-17T17:38:57.145121+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1416,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128876+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:38:57.128876+0000","last_peered":"2024-08-17T17:38:57.128876+0000","last_clean":"2024-08-17T17:38:57.128876+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:38:57.128876+0000","last_undegraded":"2024-08-17T17:38:57.128876+0000","last_fullsized":"2024-08-17T17:38:57.128876+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1424,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145344+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:38:57.145344+0000","last_peered":"2024-08-17T17:38:57.145344+0000","last_clean":"2024-08-17T17:38:57.145344+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:38:57.145344+0000","last_undegraded":"2024-08-17T17:38:57.145344+0000","last_fullsized":"2024-08-17T17:38:57.145344+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":714,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122135+0000","last_change":"2024-08-17T17:36:15.773315+0000","last_active":"2024-08-17T17:38:57.122135+0000","last_peered":"2024-08-17T17:38:57.122135+0000","last_clean":"2024-08-17T17:38:57.122135+0000","last_became_active":"2024-08-17T17:36:15.772961+0000","last_became_peered":"2024-08-17T17:36:15.772961+0000","last_unstale":"2024-08-17T17:38:57.122135+0000","last_undegraded":"2024-08-17T17:38:57.122135+0000","last_fullsized":"2024-08-17T17:38:57.122135+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:10.190221+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":46,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.142233+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:39:10.142233+0000","last_peered":"2024-08-17T17:39:10.142233+0000","last_clean":"2024-08-17T17:39:10.142233+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:39:10.142233+0000","last_undegraded":"2024-08-17T17:39:10.142233+0000","last_fullsized":"2024-08-17T17:39:10.142233+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":137,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.158413+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:39:10.158413+0000","last_peered":"2024-08-17T17:39:10.158413+0000","last_clean":"2024-08-17T17:39:10.158413+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:39:10.158413+0000","last_undegraded":"2024-08-17T17:39:10.158413+0000","last_fullsized":"2024-08-17T17:39:10.158413+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1505,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.179582+0000","last_change":"2024-08-17T17:36:24.225739+0000","last_active":"2024-08-17T17:38:57.179582+0000","last_peered":"2024-08-17T17:38:57.179582+0000","last_clean":"2024-08-17T17:38:57.179582+0000","last_became_active":"2024-08-17T17:36:24.225250+0000","last_became_peered":"2024-08-17T17:36:24.225250+0000","last_unstale":"2024-08-17T17:38:57.179582+0000","last_undegraded":"2024-08-17T17:38:57.179582+0000","last_fullsized":"2024-08-17T17:38:57.179582+0000","mapping_epoch":1393,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1394,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.290656+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1444,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145045+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:38:57.145045+0000","last_peered":"2024-08-17T17:38:57.145045+0000","last_clean":"2024-08-17T17:38:57.145045+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:38:57.145045+0000","last_undegraded":"2024-08-17T17:38:57.145045+0000","last_fullsized":"2024-08-17T17:38:57.145045+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1851,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145816+0000","last_change":"2024-08-17T17:36:12.752970+0000","last_active":"2024-08-17T17:38:57.145816+0000","last_peered":"2024-08-17T17:38:57.145816+0000","last_clean":"2024-08-17T17:38:57.145816+0000","last_became_active":"2024-08-17T17:36:12.752311+0000","last_became_peered":"2024-08-17T17:36:12.752311+0000","last_unstale":"2024-08-17T17:38:57.145816+0000","last_undegraded":"2024-08-17T17:38:57.145816+0000","last_fullsized":"2024-08-17T17:38:57.145816+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:38.131419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1799,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:39:07.441385+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:39:07.441385+0000","last_peered":"2024-08-17T17:39:07.441385+0000","last_clean":"2024-08-17T17:39:07.441385+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:39:07.441385+0000","last_undegraded":"2024-08-17T17:39:07.441385+0000","last_fullsized":"2024-08-17T17:39:07.441385+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1509,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.140839+0000","last_change":"2024-08-17T17:36:18.750494+0000","last_active":"2024-08-17T17:39:10.140839+0000","last_peered":"2024-08-17T17:39:10.140839+0000","last_clean":"2024-08-17T17:39:10.140839+0000","last_became_active":"2024-08-17T17:36:18.750092+0000","last_became_peered":"2024-08-17T17:36:18.750092+0000","last_unstale":"2024-08-17T17:39:10.140839+0000","last_undegraded":"2024-08-17T17:39:10.140839+0000","last_fullsized":"2024-08-17T17:39:10.140839+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:36.599727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1139,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.142152+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:39:10.142152+0000","last_peered":"2024-08-17T17:39:10.142152+0000","last_clean":"2024-08-17T17:39:10.142152+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:39:10.142152+0000","last_undegraded":"2024-08-17T17:39:10.142152+0000","last_fullsized":"2024-08-17T17:39:10.142152+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":965,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128590+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:38:57.128590+0000","last_peered":"2024-08-17T17:38:57.128590+0000","last_clean":"2024-08-17T17:38:57.128590+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:38:57.128590+0000","last_undegraded":"2024-08-17T17:38:57.128590+0000","last_fullsized":"2024-08-17T17:38:57.128590+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":46,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.141409+0000","last_change":"2024-08-17T17:37:11.075505+0000","last_active":"2024-08-17T17:39:10.141409+0000","last_peered":"2024-08-17T17:39:10.141409+0000","last_clean":"2024-08-17T17:39:10.141409+0000","last_became_active":"2024-08-17T17:37:11.075131+0000","last_became_peered":"2024-08-17T17:37:11.075131+0000","last_unstale":"2024-08-17T17:39:10.141409+0000","last_undegraded":"2024-08-17T17:39:10.141409+0000","last_fullsized":"2024-08-17T17:39:10.141409+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:39.672054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1278,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122478+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:38:57.122478+0000","last_peered":"2024-08-17T17:38:57.122478+0000","last_clean":"2024-08-17T17:38:57.122478+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:38:57.122478+0000","last_undegraded":"2024-08-17T17:38:57.122478+0000","last_fullsized":"2024-08-17T17:38:57.122478+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1655,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.144768+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:38:57.144768+0000","last_peered":"2024-08-17T17:38:57.144768+0000","last_clean":"2024-08-17T17:38:57.144768+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:38:57.144768+0000","last_undegraded":"2024-08-17T17:38:57.144768+0000","last_fullsized":"2024-08-17T17:38:57.144768+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":93,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.140568+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:39:10.140568+0000","last_peered":"2024-08-17T17:39:10.140568+0000","last_clean":"2024-08-17T17:39:10.140568+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:39:10.140568+0000","last_undegraded":"2024-08-17T17:39:10.140568+0000","last_fullsized":"2024-08-17T17:39:10.140568+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1253,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.151984+0000","last_change":"2024-08-17T17:36:12.755873+0000","last_active":"2024-08-17T17:39:10.151984+0000","last_peered":"2024-08-17T17:39:10.151984+0000","last_clean":"2024-08-17T17:39:10.151984+0000","last_became_active":"2024-08-17T17:36:12.755366+0000","last_became_peered":"2024-08-17T17:36:12.755366+0000","last_unstale":"2024-08-17T17:39:10.151984+0000","last_undegraded":"2024-08-17T17:39:10.151984+0000","last_fullsized":"2024-08-17T17:39:10.151984+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:36.523214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1606,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.140227+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:38:57.140227+0000","last_peered":"2024-08-17T17:38:57.140227+0000","last_clean":"2024-08-17T17:38:57.140227+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:38:57.140227+0000","last_undegraded":"2024-08-17T17:38:57.140227+0000","last_fullsized":"2024-08-17T17:38:57.140227+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":790,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.179781+0000","last_change":"2024-08-17T17:35:55.200519+0000","last_active":"2024-08-17T17:38:57.179781+0000","last_peered":"2024-08-17T17:38:57.179781+0000","last_clean":"2024-08-17T17:38:57.179781+0000","last_became_active":"2024-08-17T17:35:55.200230+0000","last_became_peered":"2024-08-17T17:35:55.200230+0000","last_unstale":"2024-08-17T17:38:57.179781+0000","last_undegraded":"2024-08-17T17:38:57.179781+0000","last_fullsized":"2024-08-17T17:38:57.179781+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:33.454851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1429,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128815+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:38:57.128815+0000","last_peered":"2024-08-17T17:38:57.128815+0000","last_clean":"2024-08-17T17:38:57.128815+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:38:57.128815+0000","last_undegraded":"2024-08-17T17:38:57.128815+0000","last_fullsized":"2024-08-17T17:38:57.128815+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1710,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.158125+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:39:10.158125+0000","last_peered":"2024-08-17T17:39:10.158125+0000","last_clean":"2024-08-17T17:39:10.158125+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:39:10.158125+0000","last_undegraded":"2024-08-17T17:39:10.158125+0000","last_fullsized":"2024-08-17T17:39:10.158125+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:22.077229+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":257,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.140734+0000","last_change":"2024-08-17T17:35:33.145476+0000","last_active":"2024-08-17T17:39:10.140734+0000","last_peered":"2024-08-17T17:39:10.140734+0000","last_clean":"2024-08-17T17:39:10.140734+0000","last_became_active":"2024-08-17T17:35:33.145125+0000","last_became_peered":"2024-08-17T17:35:33.145125+0000","last_unstale":"2024-08-17T17:39:10.140734+0000","last_undegraded":"2024-08-17T17:39:10.140734+0000","last_fullsized":"2024-08-17T17:39:10.140734+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:45.298064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":934,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.134017+0000","last_change":"2024-08-17T17:35:56.248612+0000","last_active":"2024-08-17T17:38:57.134017+0000","last_peered":"2024-08-17T17:38:57.134017+0000","last_clean":"2024-08-17T17:38:57.134017+0000","last_became_active":"2024-08-17T17:35:56.244364+0000","last_became_peered":"2024-08-17T17:35:56.244364+0000","last_unstale":"2024-08-17T17:38:57.134017+0000","last_undegraded":"2024-08-17T17:38:57.134017+0000","last_fullsized":"2024-08-17T17:38:57.134017+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:41.064165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":874,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:39:07.441624+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:39:07.441624+0000","last_peered":"2024-08-17T17:39:07.441624+0000","last_clean":"2024-08-17T17:39:07.441624+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:39:07.441624+0000","last_undegraded":"2024-08-17T17:39:07.441624+0000","last_fullsized":"2024-08-17T17:39:07.441624+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1409,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.146549+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:38:57.146549+0000","last_peered":"2024-08-17T17:38:57.146549+0000","last_clean":"2024-08-17T17:38:57.146549+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:38:57.146549+0000","last_undegraded":"2024-08-17T17:38:57.146549+0000","last_fullsized":"2024-08-17T17:38:57.146549+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1428,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145371+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:38:57.145371+0000","last_peered":"2024-08-17T17:38:57.145371+0000","last_clean":"2024-08-17T17:38:57.145371+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:38:57.145371+0000","last_undegraded":"2024-08-17T17:38:57.145371+0000","last_fullsized":"2024-08-17T17:38:57.145371+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":932,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.157008+0000","last_change":"2024-08-17T17:36:15.774881+0000","last_active":"2024-08-17T17:39:10.157008+0000","last_peered":"2024-08-17T17:39:10.157008+0000","last_clean":"2024-08-17T17:39:10.157008+0000","last_became_active":"2024-08-17T17:36:15.774435+0000","last_became_peered":"2024-08-17T17:36:15.774435+0000","last_unstale":"2024-08-17T17:39:10.157008+0000","last_undegraded":"2024-08-17T17:39:10.157008+0000","last_fullsized":"2024-08-17T17:39:10.157008+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:52.015854+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"1423'5967","reported_seq":14146,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:17.261343+0000","last_change":"2024-08-17T17:38:37.110784+0000","last_active":"2024-08-17T17:39:17.261343+0000","last_peered":"2024-08-17T17:39:17.261343+0000","last_clean":"2024-08-17T17:39:17.261343+0000","last_became_active":"2024-08-17T17:38:37.088380+0000","last_became_peered":"2024-08-17T17:38:37.088380+0000","last_unstale":"2024-08-17T17:39:17.261343+0000","last_undegraded":"2024-08-17T17:39:17.261343+0000","last_fullsized":"2024-08-17T17:39:17.261343+0000","mapping_epoch":1419,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5967,"log_dups_size":0,"ondisk_log_size":5967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:08.647903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151977984,"num_objects":2319,"num_object_clones":0,"num_object_copies":6957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5967,"num_write_kb":265152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4009,"num_bytes_recovered":199753728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1376,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.144478+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:38:57.144478+0000","last_peered":"2024-08-17T17:38:57.144478+0000","last_clean":"2024-08-17T17:38:57.144478+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:38:57.144478+0000","last_undegraded":"2024-08-17T17:38:57.144478+0000","last_fullsized":"2024-08-17T17:38:57.144478+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1272,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122412+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:38:57.122412+0000","last_peered":"2024-08-17T17:38:57.122412+0000","last_clean":"2024-08-17T17:38:57.122412+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:38:57.122412+0000","last_undegraded":"2024-08-17T17:38:57.122412+0000","last_fullsized":"2024-08-17T17:38:57.122412+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":325,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.142293+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:39:10.142293+0000","last_peered":"2024-08-17T17:39:10.142293+0000","last_clean":"2024-08-17T17:39:10.142293+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:39:10.142293+0000","last_undegraded":"2024-08-17T17:39:10.142293+0000","last_fullsized":"2024-08-17T17:39:10.142293+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1593,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.141482+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:39:10.141482+0000","last_peered":"2024-08-17T17:39:10.141482+0000","last_clean":"2024-08-17T17:39:10.141482+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:39:10.141482+0000","last_undegraded":"2024-08-17T17:39:10.141482+0000","last_fullsized":"2024-08-17T17:39:10.141482+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:25.751871+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.f","version":"1423'5920","reported_seq":6070,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.637707+0000","last_change":"2024-08-17T17:35:34.179168+0000","last_active":"2024-08-17T17:39:15.637707+0000","last_peered":"2024-08-17T17:39:15.637707+0000","last_clean":"2024-08-17T17:39:15.637707+0000","last_became_active":"2024-08-17T17:35:34.133493+0000","last_became_peered":"2024-08-17T17:35:34.133493+0000","last_unstale":"2024-08-17T17:39:15.637707+0000","last_undegraded":"2024-08-17T17:39:15.637707+0000","last_fullsized":"2024-08-17T17:39:15.637707+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5920,"log_dups_size":0,"ondisk_log_size":5920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:24:11.188566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149684224,"num_objects":2284,"num_object_clones":0,"num_object_copies":6852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5920,"num_write_kb":262528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":100,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.140666+0000","last_change":"2024-08-17T17:37:11.079712+0000","last_active":"2024-08-17T17:39:10.140666+0000","last_peered":"2024-08-17T17:39:10.140666+0000","last_clean":"2024-08-17T17:39:10.140666+0000","last_became_active":"2024-08-17T17:37:11.079253+0000","last_became_peered":"2024-08-17T17:37:11.079253+0000","last_unstale":"2024-08-17T17:39:10.140666+0000","last_undegraded":"2024-08-17T17:39:10.140666+0000","last_fullsized":"2024-08-17T17:39:10.140666+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:53.405657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":917,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122092+0000","last_change":"2024-08-17T17:35:56.223396+0000","last_active":"2024-08-17T17:38:57.122092+0000","last_peered":"2024-08-17T17:38:57.122092+0000","last_clean":"2024-08-17T17:38:57.122092+0000","last_became_active":"2024-08-17T17:35:56.212413+0000","last_became_peered":"2024-08-17T17:35:56.212413+0000","last_unstale":"2024-08-17T17:38:57.122092+0000","last_undegraded":"2024-08-17T17:38:57.122092+0000","last_fullsized":"2024-08-17T17:38:57.122092+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:44.185174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1635,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.143049+0000","last_change":"2024-08-17T17:35:33.136068+0000","last_active":"2024-08-17T17:39:10.143049+0000","last_peered":"2024-08-17T17:39:10.143049+0000","last_clean":"2024-08-17T17:39:10.143049+0000","last_became_active":"2024-08-17T17:35:33.135591+0000","last_became_peered":"2024-08-17T17:35:33.135591+0000","last_unstale":"2024-08-17T17:39:10.143049+0000","last_undegraded":"2024-08-17T17:39:10.143049+0000","last_fullsized":"2024-08-17T17:39:10.143049+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:52.252561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1782,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.146082+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:38:57.146082+0000","last_peered":"2024-08-17T17:38:57.146082+0000","last_clean":"2024-08-17T17:38:57.146082+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:38:57.146082+0000","last_undegraded":"2024-08-17T17:38:57.146082+0000","last_fullsized":"2024-08-17T17:38:57.146082+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"1423'5763","reported_seq":5898,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.641778+0000","last_change":"2024-08-17T17:35:14.861823+0000","last_active":"2024-08-17T17:39:15.641778+0000","last_peered":"2024-08-17T17:39:15.641778+0000","last_clean":"2024-08-17T17:39:15.641778+0000","last_became_active":"2024-08-17T17:35:14.854109+0000","last_became_peered":"2024-08-17T17:35:14.854109+0000","last_unstale":"2024-08-17T17:39:15.641778+0000","last_undegraded":"2024-08-17T17:39:15.641778+0000","last_fullsized":"2024-08-17T17:39:15.641778+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5763,"log_dups_size":0,"ondisk_log_size":5763,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:46:40.778197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149618688,"num_objects":2283,"num_object_clones":0,"num_object_copies":6849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5763,"num_write_kb":257472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":995,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128688+0000","last_change":"2024-08-17T17:35:33.138467+0000","last_active":"2024-08-17T17:38:57.128688+0000","last_peered":"2024-08-17T17:38:57.128688+0000","last_clean":"2024-08-17T17:38:57.128688+0000","last_became_active":"2024-08-17T17:35:33.138174+0000","last_became_peered":"2024-08-17T17:35:33.138174+0000","last_unstale":"2024-08-17T17:38:57.128688+0000","last_undegraded":"2024-08-17T17:38:57.128688+0000","last_fullsized":"2024-08-17T17:38:57.128688+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:08.860439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1379,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.140227+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:38:57.140227+0000","last_peered":"2024-08-17T17:38:57.140227+0000","last_clean":"2024-08-17T17:38:57.140227+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:38:57.140227+0000","last_undegraded":"2024-08-17T17:38:57.140227+0000","last_fullsized":"2024-08-17T17:38:57.140227+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":130,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.158323+0000","last_change":"2024-08-17T17:35:14.854881+0000","last_active":"2024-08-17T17:39:10.158323+0000","last_peered":"2024-08-17T17:39:10.158323+0000","last_clean":"2024-08-17T17:39:10.158323+0000","last_became_active":"2024-08-17T17:35:14.854121+0000","last_became_peered":"2024-08-17T17:35:14.854121+0000","last_unstale":"2024-08-17T17:39:10.158323+0000","last_undegraded":"2024-08-17T17:39:10.158323+0000","last_fullsized":"2024-08-17T17:39:10.158323+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.863624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":936,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122265+0000","last_change":"2024-08-17T17:36:15.773050+0000","last_active":"2024-08-17T17:38:57.122265+0000","last_peered":"2024-08-17T17:38:57.122265+0000","last_clean":"2024-08-17T17:38:57.122265+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:38:57.122265+0000","last_undegraded":"2024-08-17T17:38:57.122265+0000","last_fullsized":"2024-08-17T17:38:57.122265+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:39.212195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:39.212195+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"1423'5748","reported_seq":11589,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.385349+0000","last_change":"2024-08-17T17:37:21.409979+0000","last_active":"2024-08-17T17:39:15.385349+0000","last_peered":"2024-08-17T17:39:15.385349+0000","last_clean":"2024-08-17T17:39:15.385349+0000","last_became_active":"2024-08-17T17:37:21.398111+0000","last_became_peered":"2024-08-17T17:37:21.398111+0000","last_unstale":"2024-08-17T17:39:15.385349+0000","last_undegraded":"2024-08-17T17:39:15.385349+0000","last_fullsized":"2024-08-17T17:39:15.385349+0000","mapping_epoch":1407,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5748,"log_dups_size":0,"ondisk_log_size":5748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:19.299395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149618712,"num_objects":2284,"num_object_clones":0,"num_object_copies":6852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2284,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5748,"num_write_kb":256961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2850,"num_bytes_recovered":168820736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1038,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.129032+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:38:57.129032+0000","last_peered":"2024-08-17T17:38:57.129032+0000","last_clean":"2024-08-17T17:38:57.129032+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:38:57.129032+0000","last_undegraded":"2024-08-17T17:38:57.129032+0000","last_fullsized":"2024-08-17T17:38:57.129032+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":224,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.156526+0000","last_change":"2024-08-17T17:35:33.145351+0000","last_active":"2024-08-17T17:39:10.156526+0000","last_peered":"2024-08-17T17:39:10.156526+0000","last_clean":"2024-08-17T17:39:10.156526+0000","last_became_active":"2024-08-17T17:35:33.145040+0000","last_became_peered":"2024-08-17T17:35:33.145040+0000","last_unstale":"2024-08-17T17:39:10.156526+0000","last_undegraded":"2024-08-17T17:39:10.156526+0000","last_fullsized":"2024-08-17T17:39:10.156526+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:02.124000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1447,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.129055+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:38:57.129055+0000","last_peered":"2024-08-17T17:38:57.129055+0000","last_clean":"2024-08-17T17:38:57.129055+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:38:57.129055+0000","last_undegraded":"2024-08-17T17:38:57.129055+0000","last_fullsized":"2024-08-17T17:38:57.129055+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":919,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122393+0000","last_change":"2024-08-17T17:36:15.769890+0000","last_active":"2024-08-17T17:38:57.122393+0000","last_peered":"2024-08-17T17:38:57.122393+0000","last_clean":"2024-08-17T17:38:57.122393+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:38:57.122393+0000","last_undegraded":"2024-08-17T17:38:57.122393+0000","last_fullsized":"2024-08-17T17:38:57.122393+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:33.034306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:33.034306+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":4,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.b","version":"1423'5725","reported_seq":13526,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.365074+0000","last_change":"2024-08-17T17:38:07.888612+0000","last_active":"2024-08-17T17:39:18.365074+0000","last_peered":"2024-08-17T17:39:18.365074+0000","last_clean":"2024-08-17T17:39:18.365074+0000","last_became_active":"2024-08-17T17:38:07.875196+0000","last_became_peered":"2024-08-17T17:38:07.875196+0000","last_unstale":"2024-08-17T17:39:18.365074+0000","last_undegraded":"2024-08-17T17:39:18.365074+0000","last_fullsized":"2024-08-17T17:39:18.365074+0000","mapping_epoch":1415,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5725,"log_dups_size":0,"ondisk_log_size":5725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:21.658004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146079744,"num_objects":2229,"num_object_clones":0,"num_object_copies":6687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5725,"num_write_kb":254528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3836,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1485,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.158551+0000","last_change":"2024-08-17T17:36:12.759743+0000","last_active":"2024-08-17T17:39:10.158551+0000","last_peered":"2024-08-17T17:39:10.158551+0000","last_clean":"2024-08-17T17:39:10.158551+0000","last_became_active":"2024-08-17T17:36:12.759334+0000","last_became_peered":"2024-08-17T17:36:12.759334+0000","last_unstale":"2024-08-17T17:39:10.158551+0000","last_undegraded":"2024-08-17T17:39:10.158551+0000","last_fullsized":"2024-08-17T17:39:10.158551+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:04.524384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1410,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145484+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:38:57.145484+0000","last_peered":"2024-08-17T17:38:57.145484+0000","last_clean":"2024-08-17T17:38:57.145484+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:38:57.145484+0000","last_undegraded":"2024-08-17T17:38:57.145484+0000","last_fullsized":"2024-08-17T17:38:57.145484+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1453,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.130233+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:38:57.130233+0000","last_peered":"2024-08-17T17:38:57.130233+0000","last_clean":"2024-08-17T17:38:57.130233+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:38:57.130233+0000","last_undegraded":"2024-08-17T17:38:57.130233+0000","last_fullsized":"2024-08-17T17:38:57.130233+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1768,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128411+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:38:57.128411+0000","last_peered":"2024-08-17T17:38:57.128411+0000","last_clean":"2024-08-17T17:38:57.128411+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:38:57.128411+0000","last_undegraded":"2024-08-17T17:38:57.128411+0000","last_fullsized":"2024-08-17T17:38:57.128411+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"1423'5884","reported_seq":11361,"reported_epoch":1423,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:39:17.014293+0000","last_change":"2024-08-17T17:38:05.991715+0000","last_active":"2024-08-17T17:39:17.014293+0000","last_peered":"2024-08-17T17:39:17.014293+0000","last_clean":"2024-08-17T17:35:54.187688+0000","last_became_active":"2024-08-17T17:36:22.904523+0000","last_became_peered":"2024-08-17T17:36:22.904523+0000","last_unstale":"2024-08-17T17:39:17.014293+0000","last_undegraded":"2024-08-17T17:39:17.014293+0000","last_fullsized":"2024-08-17T17:36:13.740491+0000","mapping_epoch":1383,"log_start":"1373'100","ondisk_log_start":"1373'100","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5784,"log_dups_size":0,"ondisk_log_size":5784,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:57.525855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150994944,"num_objects":2304,"num_object_clones":0,"num_object_copies":6912,"num_objects_missing_on_primary":1095,"num_objects_missing":1095,"num_objects_degraded":0,"num_objects_misplaced":3267,"num_objects_unfound":0,"num_objects_dirty":2304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5884,"num_write_kb":262016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1796,"num_bytes_recovered":117702656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":1095},{"shards":"2,5,6","objects":1077},{"shards":"5,6","objects":132}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1086,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.129118+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:38:57.129118+0000","last_peered":"2024-08-17T17:38:57.129118+0000","last_clean":"2024-08-17T17:38:57.129118+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:38:57.129118+0000","last_undegraded":"2024-08-17T17:38:57.129118+0000","last_fullsized":"2024-08-17T17:38:57.129118+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1397,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.156441+0000","last_change":"2024-08-17T17:35:56.229622+0000","last_active":"2024-08-17T17:39:10.156441+0000","last_peered":"2024-08-17T17:39:10.156441+0000","last_clean":"2024-08-17T17:39:10.156441+0000","last_became_active":"2024-08-17T17:35:56.229324+0000","last_became_peered":"2024-08-17T17:35:56.229324+0000","last_unstale":"2024-08-17T17:39:10.156441+0000","last_undegraded":"2024-08-17T17:39:10.156441+0000","last_fullsized":"2024-08-17T17:39:10.156441+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:13.980316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2000,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.156915+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:39:10.156915+0000","last_peered":"2024-08-17T17:39:10.156915+0000","last_clean":"2024-08-17T17:39:10.156915+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:39:10.156915+0000","last_undegraded":"2024-08-17T17:39:10.156915+0000","last_fullsized":"2024-08-17T17:39:10.156915+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1583,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:39:02.424954+0000","last_change":"2024-08-17T17:35:33.137074+0000","last_active":"2024-08-17T17:39:02.424954+0000","last_peered":"2024-08-17T17:39:02.424954+0000","last_clean":"2024-08-17T17:39:02.424954+0000","last_became_active":"2024-08-17T17:35:33.136759+0000","last_became_peered":"2024-08-17T17:35:33.136759+0000","last_unstale":"2024-08-17T17:39:02.424954+0000","last_undegraded":"2024-08-17T17:39:02.424954+0000","last_fullsized":"2024-08-17T17:39:02.424954+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:37.263639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:37.263639+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:14.599780+0000","scrub_duration":0.0016181800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"1423'5832","reported_seq":10590,"reported_epoch":1423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:15.674868+0000","last_change":"2024-08-17T17:36:23.019129+0000","last_active":"2024-08-17T17:39:15.674868+0000","last_peered":"2024-08-17T17:39:15.674868+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:23.013527+0000","last_became_peered":"2024-08-17T17:36:23.013527+0000","last_unstale":"2024-08-17T17:39:15.674868+0000","last_undegraded":"2024-08-17T17:36:13.874715+0000","last_fullsized":"2024-08-17T17:36:13.738828+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5832,"log_dups_size":0,"ondisk_log_size":5832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148635648,"num_objects":2268,"num_object_clones":0,"num_object_copies":6804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3970,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5832,"num_write_kb":259200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1783,"num_bytes_recovered":116850688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[0,6],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":1092},{"shards":"0,1,3","objects":1},{"shards":"0,1,6","objects":1048},{"shards":"0,6","objects":-610}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1190,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.142418+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:39:10.142418+0000","last_peered":"2024-08-17T17:39:10.142418+0000","last_clean":"2024-08-17T17:39:10.142418+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:39:10.142418+0000","last_undegraded":"2024-08-17T17:39:10.142418+0000","last_fullsized":"2024-08-17T17:39:10.142418+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":47,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.144478+0000","last_change":"2024-08-17T17:36:17.774773+0000","last_active":"2024-08-17T17:38:57.144478+0000","last_peered":"2024-08-17T17:38:57.144478+0000","last_clean":"2024-08-17T17:38:57.144478+0000","last_became_active":"2024-08-17T17:36:17.774468+0000","last_became_peered":"2024-08-17T17:36:17.774468+0000","last_unstale":"2024-08-17T17:38:57.144478+0000","last_undegraded":"2024-08-17T17:38:57.144478+0000","last_fullsized":"2024-08-17T17:38:57.144478+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:54.231747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1528,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.141314+0000","last_change":"2024-08-17T17:35:33.137633+0000","last_active":"2024-08-17T17:39:10.141314+0000","last_peered":"2024-08-17T17:39:10.141314+0000","last_clean":"2024-08-17T17:39:10.141314+0000","last_became_active":"2024-08-17T17:35:33.137323+0000","last_became_peered":"2024-08-17T17:35:33.137323+0000","last_unstale":"2024-08-17T17:39:10.141314+0000","last_undegraded":"2024-08-17T17:39:10.141314+0000","last_fullsized":"2024-08-17T17:39:10.141314+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:03.359155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":898,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122009+0000","last_change":"2024-08-17T17:35:56.264626+0000","last_active":"2024-08-17T17:38:57.122009+0000","last_peered":"2024-08-17T17:38:57.122009+0000","last_clean":"2024-08-17T17:38:57.122009+0000","last_became_active":"2024-08-17T17:35:56.264265+0000","last_became_peered":"2024-08-17T17:35:56.264265+0000","last_unstale":"2024-08-17T17:38:57.122009+0000","last_undegraded":"2024-08-17T17:38:57.122009+0000","last_fullsized":"2024-08-17T17:38:57.122009+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:20.754326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:20.754326+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"1423'5755","reported_seq":11090,"reported_epoch":1423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:17.041832+0000","last_change":"2024-08-17T17:37:11.134558+0000","last_active":"2024-08-17T17:39:17.041832+0000","last_peered":"2024-08-17T17:39:17.041832+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:20.105110+0000","last_became_peered":"2024-08-17T17:36:20.105110+0000","last_unstale":"2024-08-17T17:39:17.041832+0000","last_undegraded":"2024-08-17T17:36:19.999272+0000","last_fullsized":"2024-08-17T17:36:13.740520+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5755,"log_dups_size":0,"ondisk_log_size":5755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:45.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148045824,"num_objects":2259,"num_object_clones":0,"num_object_copies":6777,"num_objects_missing_on_primary":1060,"num_objects_missing":1060,"num_objects_degraded":3165,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5755,"num_write_kb":256448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1748,"num_bytes_recovered":114556928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1060},{"shards":"1,5,6","objects":1045},{"shards":"5,6","objects":154}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1251,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122314+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:38:57.122314+0000","last_peered":"2024-08-17T17:38:57.122314+0000","last_clean":"2024-08-17T17:38:57.122314+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:38:57.122314+0000","last_undegraded":"2024-08-17T17:38:57.122314+0000","last_fullsized":"2024-08-17T17:38:57.122314+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1791,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.130137+0000","last_change":"2024-08-17T17:36:12.758005+0000","last_active":"2024-08-17T17:38:57.130137+0000","last_peered":"2024-08-17T17:38:57.130137+0000","last_clean":"2024-08-17T17:38:57.130137+0000","last_became_active":"2024-08-17T17:36:12.757695+0000","last_became_peered":"2024-08-17T17:36:12.757695+0000","last_unstale":"2024-08-17T17:38:57.130137+0000","last_undegraded":"2024-08-17T17:38:57.130137+0000","last_fullsized":"2024-08-17T17:38:57.130137+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:51.792220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1429,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128770+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:38:57.128770+0000","last_peered":"2024-08-17T17:38:57.128770+0000","last_clean":"2024-08-17T17:38:57.128770+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:38:57.128770+0000","last_undegraded":"2024-08-17T17:38:57.128770+0000","last_fullsized":"2024-08-17T17:38:57.128770+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1733,"reported_epoch":1423,"state":"active+clean+scrubbing","last_fresh":"2024-08-17T17:39:10.141550+0000","last_change":"2024-08-17T17:39:15.102008+0000","last_active":"2024-08-17T17:39:10.141550+0000","last_peered":"2024-08-17T17:39:10.141550+0000","last_clean":"2024-08-17T17:39:10.141550+0000","last_became_active":"2024-08-17T17:35:55.204861+0000","last_became_peered":"2024-08-17T17:35:55.204861+0000","last_unstale":"2024-08-17T17:39:10.141550+0000","last_undegraded":"2024-08-17T17:39:10.141550+0000","last_fullsized":"2024-08-17T17:39:10.141550+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.0","version":"1423'5859","reported_seq":13849,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:17.438784+0000","last_change":"2024-08-17T17:37:40.753703+0000","last_active":"2024-08-17T17:39:17.438784+0000","last_peered":"2024-08-17T17:39:17.438784+0000","last_clean":"2024-08-17T17:39:17.438784+0000","last_became_active":"2024-08-17T17:37:40.743458+0000","last_became_peered":"2024-08-17T17:37:40.743458+0000","last_unstale":"2024-08-17T17:39:17.438784+0000","last_undegraded":"2024-08-17T17:39:17.438784+0000","last_fullsized":"2024-08-17T17:39:17.438784+0000","mapping_epoch":1411,"log_start":"1373'400","ondisk_log_start":"1373'400","created":1357,"last_epoch_clean":1412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5459,"log_dups_size":0,"ondisk_log_size":5459,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:14:02.340380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147914752,"num_objects":2257,"num_object_clones":0,"num_object_copies":6771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5859,"num_write_kb":259712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3914,"num_bytes_recovered":232652800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1024,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145498+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:38:57.145498+0000","last_peered":"2024-08-17T17:38:57.145498+0000","last_clean":"2024-08-17T17:38:57.145498+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:38:57.145498+0000","last_undegraded":"2024-08-17T17:38:57.145498+0000","last_fullsized":"2024-08-17T17:38:57.145498+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":93,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.141001+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:39:10.141001+0000","last_peered":"2024-08-17T17:39:10.141001+0000","last_clean":"2024-08-17T17:39:10.141001+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:39:10.141001+0000","last_undegraded":"2024-08-17T17:39:10.141001+0000","last_fullsized":"2024-08-17T17:39:10.141001+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1760,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128875+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:38:57.128875+0000","last_peered":"2024-08-17T17:38:57.128875+0000","last_clean":"2024-08-17T17:38:57.128875+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:38:57.128875+0000","last_undegraded":"2024-08-17T17:38:57.128875+0000","last_fullsized":"2024-08-17T17:38:57.128875+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1","version":"1423'5786","reported_seq":11164,"reported_epoch":1423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:17.042045+0000","last_change":"2024-08-17T17:36:23.103523+0000","last_active":"2024-08-17T17:39:17.042045+0000","last_peered":"2024-08-17T17:39:17.042045+0000","last_clean":"2024-08-17T17:35:54.178280+0000","last_became_active":"2024-08-17T17:36:23.025910+0000","last_became_peered":"2024-08-17T17:36:23.025910+0000","last_unstale":"2024-08-17T17:39:17.042045+0000","last_undegraded":"2024-08-17T17:36:18.734637+0000","last_fullsized":"2024-08-17T17:36:13.740134+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5786,"log_dups_size":0,"ondisk_log_size":5786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:58.717055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148504576,"num_objects":2266,"num_object_clones":0,"num_object_copies":6798,"num_objects_missing_on_primary":1070,"num_objects_missing":1070,"num_objects_degraded":3211,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5786,"num_write_kb":257664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1763,"num_bytes_recovered":115539968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,0],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":1070},{"shards":"0,2,4,6","objects":1},{"shards":"0,2,6","objects":1070},{"shards":"0,6","objects":125}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1439,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145962+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:38:57.145962+0000","last_peered":"2024-08-17T17:38:57.145962+0000","last_clean":"2024-08-17T17:38:57.145962+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:38:57.145962+0000","last_undegraded":"2024-08-17T17:38:57.145962+0000","last_fullsized":"2024-08-17T17:38:57.145962+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1412,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.182659+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:39:10.182659+0000","last_peered":"2024-08-17T17:39:10.182659+0000","last_clean":"2024-08-17T17:39:10.182659+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:39:10.182659+0000","last_undegraded":"2024-08-17T17:39:10.182659+0000","last_fullsized":"2024-08-17T17:39:10.182659+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":995,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.142447+0000","last_change":"2024-08-17T17:34:26.817339+0000","last_active":"2024-08-17T17:39:10.142447+0000","last_peered":"2024-08-17T17:39:10.142447+0000","last_clean":"2024-08-17T17:39:10.142447+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:39:10.142447+0000","last_undegraded":"2024-08-17T17:39:10.142447+0000","last_fullsized":"2024-08-17T17:39:10.142447+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:26.817223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:26.817223+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:30.089956+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"1423'5664","reported_seq":5824,"reported_epoch":1423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:15.428633+0000","last_change":"2024-08-17T17:36:14.073155+0000","last_active":"2024-08-17T17:39:15.428633+0000","last_peered":"2024-08-17T17:39:15.428633+0000","last_clean":"2024-08-17T17:36:11.734683+0000","last_became_active":"2024-08-17T17:36:14.063794+0000","last_became_peered":"2024-08-17T17:36:14.063794+0000","last_unstale":"2024-08-17T17:39:15.428633+0000","last_undegraded":"2024-08-17T17:36:13.855770+0000","last_fullsized":"2024-08-17T17:36:13.742068+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5664,"log_dups_size":0,"ondisk_log_size":5664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146669568,"num_objects":2238,"num_object_clones":0,"num_object_copies":6714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3819,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5664,"num_write_kb":252864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-1581},{"shards":"0,2,7","objects":2106}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":47,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145945+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:38:57.145945+0000","last_peered":"2024-08-17T17:38:57.145945+0000","last_clean":"2024-08-17T17:38:57.145945+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:38:57.145945+0000","last_undegraded":"2024-08-17T17:38:57.145945+0000","last_fullsized":"2024-08-17T17:38:57.145945+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1273,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.133942+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:38:57.133942+0000","last_peered":"2024-08-17T17:38:57.133942+0000","last_clean":"2024-08-17T17:38:57.133942+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:38:57.133942+0000","last_undegraded":"2024-08-17T17:38:57.133942+0000","last_fullsized":"2024-08-17T17:38:57.133942+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1788,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128483+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:38:57.128483+0000","last_peered":"2024-08-17T17:38:57.128483+0000","last_clean":"2024-08-17T17:38:57.128483+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:38:57.128483+0000","last_undegraded":"2024-08-17T17:38:57.128483+0000","last_fullsized":"2024-08-17T17:38:57.128483+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"1423'5836","reported_seq":11634,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.989220+0000","last_change":"2024-08-17T17:36:46.982456+0000","last_active":"2024-08-17T17:39:16.989220+0000","last_peered":"2024-08-17T17:39:16.989220+0000","last_clean":"2024-08-17T17:39:16.989220+0000","last_became_active":"2024-08-17T17:36:46.974782+0000","last_became_peered":"2024-08-17T17:36:46.974782+0000","last_unstale":"2024-08-17T17:39:16.989220+0000","last_undegraded":"2024-08-17T17:39:16.989220+0000","last_fullsized":"2024-08-17T17:39:16.989220+0000","mapping_epoch":1399,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5836,"log_dups_size":0,"ondisk_log_size":5836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:40.499321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146538496,"num_objects":2236,"num_object_clones":0,"num_object_copies":6708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5836,"num_write_kb":258304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2848,"num_bytes_recovered":186646528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1390,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.146460+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:38:57.146460+0000","last_peered":"2024-08-17T17:38:57.146460+0000","last_clean":"2024-08-17T17:38:57.146460+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:38:57.146460+0000","last_undegraded":"2024-08-17T17:38:57.146460+0000","last_fullsized":"2024-08-17T17:38:57.146460+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":203,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145260+0000","last_change":"2024-08-17T17:36:12.754372+0000","last_active":"2024-08-17T17:38:57.145260+0000","last_peered":"2024-08-17T17:38:57.145260+0000","last_clean":"2024-08-17T17:38:57.145260+0000","last_became_active":"2024-08-17T17:36:12.746352+0000","last_became_peered":"2024-08-17T17:36:12.746352+0000","last_unstale":"2024-08-17T17:38:57.145260+0000","last_undegraded":"2024-08-17T17:38:57.145260+0000","last_fullsized":"2024-08-17T17:38:57.145260+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:07.807486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1666,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.156738+0000","last_change":"2024-08-17T17:36:15.771193+0000","last_active":"2024-08-17T17:39:10.156738+0000","last_peered":"2024-08-17T17:39:10.156738+0000","last_clean":"2024-08-17T17:39:10.156738+0000","last_became_active":"2024-08-17T17:36:15.770889+0000","last_became_peered":"2024-08-17T17:36:15.770889+0000","last_unstale":"2024-08-17T17:39:10.156738+0000","last_undegraded":"2024-08-17T17:39:10.156738+0000","last_fullsized":"2024-08-17T17:39:10.156738+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:22.978695+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.4","version":"1423'5790","reported_seq":11569,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:16.773074+0000","last_change":"2024-08-17T17:37:04.045576+0000","last_active":"2024-08-17T17:39:16.773074+0000","last_peered":"2024-08-17T17:39:16.773074+0000","last_clean":"2024-08-17T17:39:16.773074+0000","last_became_active":"2024-08-17T17:37:04.038810+0000","last_became_peered":"2024-08-17T17:37:04.038810+0000","last_unstale":"2024-08-17T17:39:16.773074+0000","last_undegraded":"2024-08-17T17:39:16.773074+0000","last_fullsized":"2024-08-17T17:39:16.773074+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5790,"log_dups_size":0,"ondisk_log_size":5790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:27:04.725258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149159936,"num_objects":2276,"num_object_clones":0,"num_object_copies":6828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5790,"num_write_kb":258112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":176291840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1262,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.142757+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:39:10.142757+0000","last_peered":"2024-08-17T17:39:10.142757+0000","last_clean":"2024-08-17T17:39:10.142757+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:39:10.142757+0000","last_undegraded":"2024-08-17T17:39:10.142757+0000","last_fullsized":"2024-08-17T17:39:10.142757+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1416,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128915+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:38:57.128915+0000","last_peered":"2024-08-17T17:38:57.128915+0000","last_clean":"2024-08-17T17:38:57.128915+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:38:57.128915+0000","last_undegraded":"2024-08-17T17:38:57.128915+0000","last_fullsized":"2024-08-17T17:38:57.128915+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":916,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145735+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:38:57.145735+0000","last_peered":"2024-08-17T17:38:57.145735+0000","last_clean":"2024-08-17T17:38:57.145735+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:38:57.145735+0000","last_undegraded":"2024-08-17T17:38:57.145735+0000","last_fullsized":"2024-08-17T17:38:57.145735+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"1423'5852","reported_seq":11686,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:17.417776+0000","last_change":"2024-08-17T17:36:32.711490+0000","last_active":"2024-08-17T17:39:17.417776+0000","last_peered":"2024-08-17T17:39:17.417776+0000","last_clean":"2024-08-17T17:39:17.417776+0000","last_became_active":"2024-08-17T17:36:32.704370+0000","last_became_peered":"2024-08-17T17:36:32.704370+0000","last_unstale":"2024-08-17T17:39:17.417776+0000","last_undegraded":"2024-08-17T17:39:17.417776+0000","last_fullsized":"2024-08-17T17:39:17.417776+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5852,"log_dups_size":0,"ondisk_log_size":5852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:17.373879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147849216,"num_objects":2256,"num_object_clones":0,"num_object_copies":6768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5852,"num_write_kb":259456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2866,"num_bytes_recovered":187826176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1272,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.144876+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:38:57.144876+0000","last_peered":"2024-08-17T17:38:57.144876+0000","last_clean":"2024-08-17T17:38:57.144876+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:38:57.144876+0000","last_undegraded":"2024-08-17T17:38:57.144876+0000","last_fullsized":"2024-08-17T17:38:57.144876+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":152,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128927+0000","last_change":"2024-08-17T17:35:55.217072+0000","last_active":"2024-08-17T17:38:57.128927+0000","last_peered":"2024-08-17T17:38:57.128927+0000","last_clean":"2024-08-17T17:38:57.128927+0000","last_became_active":"2024-08-17T17:35:55.216462+0000","last_became_peered":"2024-08-17T17:35:55.216462+0000","last_unstale":"2024-08-17T17:38:57.128927+0000","last_undegraded":"2024-08-17T17:38:57.128927+0000","last_fullsized":"2024-08-17T17:38:57.128927+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:34.937221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":899,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.122468+0000","last_change":"2024-08-17T17:35:55.215004+0000","last_active":"2024-08-17T17:38:57.122468+0000","last_peered":"2024-08-17T17:38:57.122468+0000","last_clean":"2024-08-17T17:38:57.122468+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:38:57.122468+0000","last_undegraded":"2024-08-17T17:38:57.122468+0000","last_fullsized":"2024-08-17T17:38:57.122468+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.6","version":"1423'5655","reported_seq":5791,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:15.596745+0000","last_change":"2024-08-17T17:35:14.868524+0000","last_active":"2024-08-17T17:39:15.596745+0000","last_peered":"2024-08-17T17:39:15.596745+0000","last_clean":"2024-08-17T17:39:15.596745+0000","last_became_active":"2024-08-17T17:35:14.854690+0000","last_became_peered":"2024-08-17T17:35:14.854690+0000","last_unstale":"2024-08-17T17:39:15.596745+0000","last_undegraded":"2024-08-17T17:39:15.596745+0000","last_fullsized":"2024-08-17T17:39:15.596745+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5655,"log_dups_size":0,"ondisk_log_size":5655,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:05.388362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141623296,"num_objects":2161,"num_object_clones":0,"num_object_copies":6483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5655,"num_write_kb":250112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1458,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.182578+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:39:10.182578+0000","last_peered":"2024-08-17T17:39:10.182578+0000","last_clean":"2024-08-17T17:39:10.182578+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:39:10.182578+0000","last_undegraded":"2024-08-17T17:39:10.182578+0000","last_fullsized":"2024-08-17T17:39:10.182578+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":164,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.146041+0000","last_change":"2024-08-17T17:35:33.147672+0000","last_active":"2024-08-17T17:38:57.146041+0000","last_peered":"2024-08-17T17:38:57.146041+0000","last_clean":"2024-08-17T17:38:57.146041+0000","last_became_active":"2024-08-17T17:35:33.147283+0000","last_became_peered":"2024-08-17T17:35:33.147283+0000","last_unstale":"2024-08-17T17:38:57.146041+0000","last_undegraded":"2024-08-17T17:38:57.146041+0000","last_fullsized":"2024-08-17T17:38:57.146041+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:41.677709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1667,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:13.923382+0000","last_change":"2024-08-17T17:39:18.685808+0000","last_active":"2024-08-17T17:39:13.923382+0000","last_peered":"2024-08-17T17:39:13.923382+0000","last_clean":"2024-08-17T17:39:13.923382+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:39:13.923382+0000","last_undegraded":"2024-08-17T17:39:13.923382+0000","last_fullsized":"2024-08-17T17:39:13.923382+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:38.241235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:38.241235+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:39:18.924101+0000","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"1423'5703","reported_seq":5779,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:17.295193+0000","last_change":"2024-08-17T17:34:50.670013+0000","last_active":"2024-08-17T17:39:17.295193+0000","last_peered":"2024-08-17T17:39:17.295193+0000","last_clean":"2024-08-17T17:39:17.295193+0000","last_became_active":"2024-08-17T17:34:50.669299+0000","last_became_peered":"2024-08-17T17:34:50.669299+0000","last_unstale":"2024-08-17T17:39:17.295193+0000","last_undegraded":"2024-08-17T17:39:17.295193+0000","last_fullsized":"2024-08-17T17:39:17.295193+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5703,"log_dups_size":0,"ondisk_log_size":5703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:11.172227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145686528,"num_objects":2223,"num_object_clones":0,"num_object_copies":6669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5703,"num_write_kb":253632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1382,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.151040+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:39:10.151040+0000","last_peered":"2024-08-17T17:39:10.151040+0000","last_clean":"2024-08-17T17:39:10.151040+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:39:10.151040+0000","last_undegraded":"2024-08-17T17:39:10.151040+0000","last_fullsized":"2024-08-17T17:39:10.151040+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":151,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145640+0000","last_change":"2024-08-17T17:35:33.140725+0000","last_active":"2024-08-17T17:38:57.145640+0000","last_peered":"2024-08-17T17:38:57.145640+0000","last_clean":"2024-08-17T17:38:57.145640+0000","last_became_active":"2024-08-17T17:35:33.140406+0000","last_became_peered":"2024-08-17T17:35:33.140406+0000","last_unstale":"2024-08-17T17:38:57.145640+0000","last_undegraded":"2024-08-17T17:38:57.145640+0000","last_fullsized":"2024-08-17T17:38:57.145640+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:54:51.875571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1671,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.145742+0000","last_change":"2024-08-17T17:35:33.138514+0000","last_active":"2024-08-17T17:38:57.145742+0000","last_peered":"2024-08-17T17:38:57.145742+0000","last_clean":"2024-08-17T17:38:57.145742+0000","last_became_active":"2024-08-17T17:35:33.137035+0000","last_became_peered":"2024-08-17T17:35:33.137035+0000","last_unstale":"2024-08-17T17:38:57.145742+0000","last_undegraded":"2024-08-17T17:38:57.145742+0000","last_fullsized":"2024-08-17T17:38:57.145742+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:50.565662+0000","scrub_duration":0.001487138,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1313,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.140471+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:38:57.140471+0000","last_peered":"2024-08-17T17:38:57.140471+0000","last_clean":"2024-08-17T17:38:57.140471+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:38:57.140471+0000","last_undegraded":"2024-08-17T17:38:57.140471+0000","last_fullsized":"2024-08-17T17:38:57.140471+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":93,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.141189+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:39:10.141189+0000","last_peered":"2024-08-17T17:39:10.141189+0000","last_clean":"2024-08-17T17:39:10.141189+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:39:10.141189+0000","last_undegraded":"2024-08-17T17:39:10.141189+0000","last_fullsized":"2024-08-17T17:39:10.141189+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1744,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128957+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:38:57.128957+0000","last_peered":"2024-08-17T17:38:57.128957+0000","last_clean":"2024-08-17T17:38:57.128957+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:38:57.128957+0000","last_undegraded":"2024-08-17T17:38:57.128957+0000","last_fullsized":"2024-08-17T17:38:57.128957+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1330,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.144560+0000","last_change":"2024-08-17T17:35:55.205292+0000","last_active":"2024-08-17T17:38:57.144560+0000","last_peered":"2024-08-17T17:38:57.144560+0000","last_clean":"2024-08-17T17:38:57.144560+0000","last_became_active":"2024-08-17T17:35:55.204935+0000","last_became_peered":"2024-08-17T17:35:55.204935+0000","last_unstale":"2024-08-17T17:38:57.144560+0000","last_undegraded":"2024-08-17T17:38:57.144560+0000","last_fullsized":"2024-08-17T17:38:57.144560+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:52.277821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":93,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.141140+0000","last_change":"2024-08-17T17:36:12.749972+0000","last_active":"2024-08-17T17:39:10.141140+0000","last_peered":"2024-08-17T17:39:10.141140+0000","last_clean":"2024-08-17T17:39:10.141140+0000","last_became_active":"2024-08-17T17:36:12.749620+0000","last_became_peered":"2024-08-17T17:36:12.749620+0000","last_unstale":"2024-08-17T17:39:10.141140+0000","last_undegraded":"2024-08-17T17:39:10.141140+0000","last_fullsized":"2024-08-17T17:39:10.141140+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:04.994357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":969,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.140504+0000","last_change":"2024-08-17T17:36:20.832274+0000","last_active":"2024-08-17T17:39:10.140504+0000","last_peered":"2024-08-17T17:39:10.140504+0000","last_clean":"2024-08-17T17:39:10.140504+0000","last_became_active":"2024-08-17T17:36:20.831886+0000","last_became_peered":"2024-08-17T17:36:20.831886+0000","last_unstale":"2024-08-17T17:39:10.140504+0000","last_undegraded":"2024-08-17T17:39:10.140504+0000","last_fullsized":"2024-08-17T17:39:10.140504+0000","mapping_epoch":1390,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1391,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:05.248711+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1178,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.141910+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:39:10.141910+0000","last_peered":"2024-08-17T17:39:10.141910+0000","last_clean":"2024-08-17T17:39:10.141910+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:39:10.141910+0000","last_undegraded":"2024-08-17T17:39:10.141910+0000","last_fullsized":"2024-08-17T17:39:10.141910+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1413,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128973+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:38:57.128973+0000","last_peered":"2024-08-17T17:38:57.128973+0000","last_clean":"2024-08-17T17:38:57.128973+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:38:57.128973+0000","last_undegraded":"2024-08-17T17:38:57.128973+0000","last_fullsized":"2024-08-17T17:38:57.128973+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1707,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128396+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:38:57.128396+0000","last_peered":"2024-08-17T17:38:57.128396+0000","last_clean":"2024-08-17T17:38:57.128396+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:38:57.128396+0000","last_undegraded":"2024-08-17T17:38:57.128396+0000","last_fullsized":"2024-08-17T17:38:57.128396+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1355,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.139378+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:38:57.139378+0000","last_peered":"2024-08-17T17:38:57.139378+0000","last_clean":"2024-08-17T17:38:57.139378+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:38:57.139378+0000","last_undegraded":"2024-08-17T17:38:57.139378+0000","last_fullsized":"2024-08-17T17:38:57.139378+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1897,"reported_epoch":1423,"state":"active+clean","last_fresh":"2024-08-17T17:39:10.141587+0000","last_change":"2024-08-17T17:35:33.137411+0000","last_active":"2024-08-17T17:39:10.141587+0000","last_peered":"2024-08-17T17:39:10.141587+0000","last_clean":"2024-08-17T17:39:10.141587+0000","last_became_active":"2024-08-17T17:35:33.137073+0000","last_became_peered":"2024-08-17T17:35:33.137073+0000","last_unstale":"2024-08-17T17:39:10.141587+0000","last_undegraded":"2024-08-17T17:39:10.141587+0000","last_fullsized":"2024-08-17T17:39:10.141587+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:35.924539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1801,"reported_epoch":1422,"state":"active+clean","last_fresh":"2024-08-17T17:38:57.128483+0000","last_change":"2024-08-17T17:36:15.787445+0000","last_active":"2024-08-17T17:38:57.128483+0000","last_peered":"2024-08-17T17:38:57.128483+0000","last_clean":"2024-08-17T17:38:57.128483+0000","last_became_active":"2024-08-17T17:36:15.787099+0000","last_became_peered":"2024-08-17T17:36:15.787099+0000","last_unstale":"2024-08-17T17:38:57.128483+0000","last_undegraded":"2024-08-17T17:38:57.128483+0000","last_fullsized":"2024-08-17T17:38:57.128483+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:47.448168+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":2368602136,"num_objects":36143,"num_object_clones":0,"num_object_copies":108429,"num_objects_missing_on_primary":3225,"num_objects_missing":3225,"num_objects_degraded":14165,"num_objects_misplaced":3267,"num_objects_unfound":0,"num_objects_dirty":36143,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":92739,"num_write_kb":4124161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30264,"num_bytes_recovered":1830748160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8022601728,"data_stored":8022589512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92239,"ondisk_log_size":92239,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635092,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":27,"num_read_kb":61,"num_write":15,"num_write_kb":183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1314910,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162862,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1271276,"kb_used_data":1124708,"kb_used_omap":54,"kb_used_meta":146505,"kb_avail":92473876,"statfs":{"total":95995035648,"available":94693249024,"internally_reserved":0,"allocated":1151700992,"data_stored":1149022892,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":150021141},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999336,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1155244,"kb_used_data":961124,"kb_used_omap":47,"kb_used_meta":194064,"kb_avail":92589908,"statfs":{"total":95995035648,"available":94812065792,"internally_reserved":0,"allocated":984190976,"data_stored":981513853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":198722196},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,1,5,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627133,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1396752,"kb_used_data":338056,"kb_used_omap":42,"kb_used_meta":1058645,"kb_avail":92348400,"statfs":{"total":95995035648,"available":94564761600,"internally_reserved":0,"allocated":346169344,"data_stored":343514502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1084052540},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785610,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2457620,"kb_used_data":1653452,"kb_used_omap":48,"kb_used_meta":804111,"kb_avail":91287532,"statfs":{"total":95995035648,"available":93478432768,"internally_reserved":0,"allocated":1693134848,"data_stored":1690476881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":823410350},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725127,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1459292,"kb_used_data":589844,"kb_used_omap":1,"kb_used_meta":869438,"kb_avail":92285860,"statfs":{"total":95995035648,"available":94500720640,"internally_reserved":0,"allocated":604000256,"data_stored":601334239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":890304966},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986764,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1843176,"kb_used_data":955680,"kb_used_omap":1,"kb_used_meta":887486,"kb_avail":91901976,"statfs":{"total":95995035648,"available":94107623424,"internally_reserved":0,"allocated":978616320,"data_stored":975941726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":908786118},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706371,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2295900,"kb_used_data":1383700,"kb_used_omap":35,"kb_used_meta":912156,"kb_avail":91449252,"statfs":{"total":95995035648,"available":93644034048,"internally_reserved":0,"allocated":1416908800,"data_stored":1414242764,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":934047799},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832144,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1957912,"kb_used_data":910672,"kb_used_omap":65,"kb_used_meta":1047166,"kb_avail":91787240,"statfs":{"total":95995035648,"available":93990133760,"internally_reserved":0,"allocated":932528128,"data_stored":929866650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1072298171},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":921698304,"data_stored":921698304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1681588224,"data_stored":1681588224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1403654144,"data_stored":1403650072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":336855040,"data_stored":336855040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":967970816,"data_stored":967966744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":591659008,"data_stored":591659008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":975376384,"data_stored":975372312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1143799808,"data_stored":1143799808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:39:21.401 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:39:21.473 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:21.496 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:21.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:21.598 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:21.628 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:21.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:21.730 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:21.756 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:21.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:21.841 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:21 smithi162 ceph-mon[104668]: pgmap v2247: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 226 op/s; 14165/108507 objects degraded (13.054%); 3267/108507 objects misplaced (3.011%); 3.6 MiB/s, 56 objects/s recovering 2024-08-17T17:39:21.858 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:21.882 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:39:21.887 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:21.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:21.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:21.988 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:22.012 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:22.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:22.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:21 smithi063 ceph-mon[114756]: pgmap v2247: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 226 op/s; 14165/108507 objects degraded (13.054%); 3267/108507 objects misplaced (3.011%); 3.6 MiB/s, 56 objects/s recovering 2024-08-17T17:39:22.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:21 smithi063 ceph-mon[109843]: pgmap v2247: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 226 op/s; 14165/108507 objects degraded (13.054%); 3267/108507 objects misplaced (3.011%); 3.6 MiB/s, 56 objects/s recovering 2024-08-17T17:39:22.114 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:22.138 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:22.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:22.172 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:22.240 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:22.269 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:22.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:22.371 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:22.398 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:22.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:22.412 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:22.451 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:39:22.500 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:22.527 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:22.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:22.629 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:22.657 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:22.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:22.759 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:22.785 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:22.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:22.886 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:22.917 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:22.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:22.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:23.019 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:23.047 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:23.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:23.149 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:23.177 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:23.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:23.279 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:23.302 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:23.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:23.404 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:23.428 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:23.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:23.430 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:39:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:39:23] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.51.0" 2024-08-17T17:39:23.530 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:23.555 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:23.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:23.657 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:23.686 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:23.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:23.788 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:23.815 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:23.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:23.917 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:23.939 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:23.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:23.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:23.964 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:39:23.964 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:39:23.965 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:39:23.965 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:39:23.965 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:39:23.965 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:23.965 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:23.965 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:23.965 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:23.965 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:23.965 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:39:23.966 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:39:23.966 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:39:23.966 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:23.966 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:23.966 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:23.966 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:23 smithi063 ceph-mon[114756]: pgmap v2248: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 254 op/s; 13665/105036 objects degraded (13.010%); 3120/105036 objects misplaced (2.970%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:39:23.967 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:23 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 13665/105036 objects degraded (13.010%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:23.967 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:23 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1408094048' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:23.967 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:23 smithi063 ceph-mon[109843]: pgmap v2248: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 254 op/s; 13665/105036 objects degraded (13.010%); 3120/105036 objects misplaced (2.970%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:39:23.967 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:23 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 13665/105036 objects degraded (13.010%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:23.967 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:23 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1408094048' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:24.041 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:24.068 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:24.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:24.094 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:23 smithi162 ceph-mon[104668]: pgmap v2248: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 254 op/s; 13665/105036 objects degraded (13.010%); 3120/105036 objects misplaced (2.970%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:39:24.094 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:23 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 13665/105036 objects degraded (13.010%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:24.094 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:23 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1408094048' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:24.170 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:24.195 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:24.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:24.297 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:24.320 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:24.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:24.418 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:39:24.419 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:39:24.419 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:39:24.419 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:39:24.419 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.419 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:246272 10.a 10:5cc562b4:::benchmark_data_smithi063_294527_object109087:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1338)", 2024-08-17T17:39:24.419 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:58.141613+0000", 2024-08-17T17:39:24.419 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 326.27411337400002, 2024-08-17T17:39:24.419 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.986951527, 2024-08-17T17:39:24.419 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.419 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.419 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.420 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:39:24.420 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:24.420 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 246272 2024-08-17T17:39:24.420 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.420 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.420 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.420 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.420 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.141613+0000", 2024-08-17T17:39:24.420 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.420 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.420 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.420 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.421 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.141613+0000", 2024-08-17T17:39:24.421 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.421 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.421 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.421 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.421 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.141612+0000", 2024-08-17T17:39:24.421 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:24.421 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.421 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.421 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.421 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.141618+0000", 2024-08-17T17:39:24.421 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8690000000000001e-06 2024-08-17T17:39:24.422 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.422 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.422 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.422 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.141632+0000", 2024-08-17T17:39:24.422 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4207e-05 2024-08-17T17:39:24.422 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.422 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.422 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.422 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.141640+0000", 2024-08-17T17:39:24.422 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.8699999999999992e-06 2024-08-17T17:39:24.423 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.423 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.423 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.423 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.141705+0000", 2024-08-17T17:39:24.423 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.5492000000000002e-05 2024-08-17T17:39:24.423 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.423 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.423 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.423 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.141716+0000", 2024-08-17T17:39:24.423 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.092e-05 2024-08-17T17:39:24.423 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.423 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.424 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.424 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.122879+0000", 2024-08-17T17:39:24.424 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98116315600000004 2024-08-17T17:39:24.424 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.424 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.424 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.424 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.122959+0000", 2024-08-17T17:39:24.424 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9303999999999998e-05 2024-08-17T17:39:24.424 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.424 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.424 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:39:24.424 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.123042+0000", 2024-08-17T17:39:24.424 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.3078000000000007e-05 2024-08-17T17:39:24.424 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.425 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.425 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.425 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.128398+0000", 2024-08-17T17:39:24.425 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0053567750000000003 2024-08-17T17:39:24.425 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.425 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.425 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.425 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.128427+0000", 2024-08-17T17:39:24.425 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8569999999999999e-05 2024-08-17T17:39:24.425 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.425 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.425 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.425 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.128464+0000", 2024-08-17T17:39:24.425 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.6928000000000003e-05 2024-08-17T17:39:24.426 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.426 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.426 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.426 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.128494+0000", 2024-08-17T17:39:24.426 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9709999999999998e-05 2024-08-17T17:39:24.426 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.426 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.426 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.426 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.128565+0000", 2024-08-17T17:39:24.426 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1038999999999995e-05 2024-08-17T17:39:24.426 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.426 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.426 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.427 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.427 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.427 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:246279 10.a 10:5beed8d7:::benchmark_data_smithi063_294527_object109094:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1338)", 2024-08-17T17:39:24.427 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:58.149646+0000", 2024-08-17T17:39:24.427 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 326.266080386, 2024-08-17T17:39:24.427 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98352370499999997, 2024-08-17T17:39:24.427 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.427 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.427 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.427 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:39:24.427 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:24.427 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 246279 2024-08-17T17:39:24.428 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.428 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.428 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.428 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.428 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.149646+0000", 2024-08-17T17:39:24.428 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.428 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.428 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.428 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.428 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.149646+0000", 2024-08-17T17:39:24.428 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.428 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.429 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.429 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.429 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.149645+0000", 2024-08-17T17:39:24.429 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:39:24.429 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.429 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.429 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.429 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.149650+0000", 2024-08-17T17:39:24.429 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1080000000000001e-06 2024-08-17T17:39:24.429 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.429 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.429 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.430 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.149665+0000", 2024-08-17T17:39:24.430 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4277000000000001e-05 2024-08-17T17:39:24.430 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.430 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.430 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.430 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.149672+0000", 2024-08-17T17:39:24.430 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3839999999999999e-06 2024-08-17T17:39:24.430 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.430 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.430 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.430 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.149759+0000", 2024-08-17T17:39:24.430 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6636000000000002e-05 2024-08-17T17:39:24.430 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.431 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.431 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.431 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.149769+0000", 2024-08-17T17:39:24.431 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0754000000000001e-05 2024-08-17T17:39:24.431 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.431 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.431 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.431 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.125608+0000", 2024-08-17T17:39:24.431 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97583824900000005 2024-08-17T17:39:24.431 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.431 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.431 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.431 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.125678+0000", 2024-08-17T17:39:24.431 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9944000000000007e-05 2024-08-17T17:39:24.432 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.432 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.432 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:39:24.432 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.125777+0000", 2024-08-17T17:39:24.432 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.9865999999999996e-05 2024-08-17T17:39:24.432 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.432 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.432 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.432 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.128617+0000", 2024-08-17T17:39:24.432 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002839416 2024-08-17T17:39:24.432 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.432 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.433 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.433 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.133058+0000", 2024-08-17T17:39:24.433 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0044410480000000004 2024-08-17T17:39:24.433 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.433 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.433 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.433 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.133094+0000", 2024-08-17T17:39:24.433 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5882e-05 2024-08-17T17:39:24.433 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.433 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.433 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.433 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.133120+0000", 2024-08-17T17:39:24.433 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5766e-05 2024-08-17T17:39:24.433 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.434 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.434 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.434 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.133170+0000", 2024-08-17T17:39:24.434 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0281000000000002e-05 2024-08-17T17:39:24.434 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.434 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.434 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.434 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.434 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.434 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:246327 10.a 10:53c75409:::benchmark_data_smithi063_294527_object109142:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1338)", 2024-08-17T17:39:24.434 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:58.191411+0000", 2024-08-17T17:39:24.434 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 326.22431542599998, 2024-08-17T17:39:24.434 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95205859699999995, 2024-08-17T17:39:24.435 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.435 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.435 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.435 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:39:24.435 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:24.435 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 246327 2024-08-17T17:39:24.435 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.435 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.435 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.435 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.435 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.191411+0000", 2024-08-17T17:39:24.435 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.435 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.436 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.436 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.436 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.191411+0000", 2024-08-17T17:39:24.436 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.436 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.436 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.436 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.436 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.191410+0000", 2024-08-17T17:39:24.436 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:24.436 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.436 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.436 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.436 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.191416+0000", 2024-08-17T17:39:24.436 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1580000000000002e-06 2024-08-17T17:39:24.437 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.437 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.437 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.437 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.191435+0000", 2024-08-17T17:39:24.437 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9368000000000002e-05 2024-08-17T17:39:24.437 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.437 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.437 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.437 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.191445+0000", 2024-08-17T17:39:24.437 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0168e-05 2024-08-17T17:39:24.437 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.437 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.437 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.438 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.191518+0000", 2024-08-17T17:39:24.438 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2723000000000002e-05 2024-08-17T17:39:24.438 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.438 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.438 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.438 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.191534+0000", 2024-08-17T17:39:24.438 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5616000000000001e-05 2024-08-17T17:39:24.438 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.438 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.438 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.438 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.127040+0000", 2024-08-17T17:39:24.438 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.93550599199999995 2024-08-17T17:39:24.439 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.439 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.439 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.439 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.127141+0000", 2024-08-17T17:39:24.439 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000101766 2024-08-17T17:39:24.439 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.439 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.439 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:39:24.439 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.127218+0000", 2024-08-17T17:39:24.439 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6870000000000001e-05 2024-08-17T17:39:24.439 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.439 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.440 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.440 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.131598+0000", 2024-08-17T17:39:24.440 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0043795830000000003 2024-08-17T17:39:24.440 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.440 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.440 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.440 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.139766+0000", 2024-08-17T17:39:24.440 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0081680699999999995 2024-08-17T17:39:24.440 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.440 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.440 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.440 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.143363+0000", 2024-08-17T17:39:24.440 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003596653 2024-08-17T17:39:24.440 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.441 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.441 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.441 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.143412+0000", 2024-08-17T17:39:24.441 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9580999999999998e-05 2024-08-17T17:39:24.441 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.441 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.441 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.441 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.143470+0000", 2024-08-17T17:39:24.441 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7506999999999997e-05 2024-08-17T17:39:24.441 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.441 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.441 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.441 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.441 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.442 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:246340 10.a 10:5e0706d7:::benchmark_data_smithi063_294527_object109155:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1338)", 2024-08-17T17:39:24.442 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:58.199794+0000", 2024-08-17T17:39:24.442 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 326.215932839, 2024-08-17T17:39:24.442 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.94989122000000004, 2024-08-17T17:39:24.442 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.442 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.442 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.442 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:39:24.442 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:24.442 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 246340 2024-08-17T17:39:24.442 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.442 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.442 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.443 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.443 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.199794+0000", 2024-08-17T17:39:24.443 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.443 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.443 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.443 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.443 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.199794+0000", 2024-08-17T17:39:24.443 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.443 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.443 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.443 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.443 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.199793+0000", 2024-08-17T17:39:24.443 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:39:24.444 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.444 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.444 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.444 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.199798+0000", 2024-08-17T17:39:24.444 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2959999999999998e-06 2024-08-17T17:39:24.444 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.444 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.444 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.444 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.199808+0000", 2024-08-17T17:39:24.444 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7880000000000001e-06 2024-08-17T17:39:24.444 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.444 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.445 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.445 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.199815+0000", 2024-08-17T17:39:24.445 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.221e-06 2024-08-17T17:39:24.445 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.445 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.445 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.445 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.199886+0000", 2024-08-17T17:39:24.445 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1341999999999996e-05 2024-08-17T17:39:24.445 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.445 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.445 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.445 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.199897+0000", 2024-08-17T17:39:24.445 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0703999999999999e-05 2024-08-17T17:39:24.445 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.446 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.446 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.446 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.128645+0000", 2024-08-17T17:39:24.446 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.92874814900000002 2024-08-17T17:39:24.446 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.446 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.446 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.446 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.128716+0000", 2024-08-17T17:39:24.446 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1087999999999994e-05 2024-08-17T17:39:24.446 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.446 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.446 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:39:24.446 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.128787+0000", 2024-08-17T17:39:24.447 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0415999999999998e-05 2024-08-17T17:39:24.447 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.447 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.447 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.447 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.133063+0000", 2024-08-17T17:39:24.447 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0042765640000000001 2024-08-17T17:39:24.447 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.447 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.447 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.447 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.139804+0000", 2024-08-17T17:39:24.447 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0067410689999999997 2024-08-17T17:39:24.447 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.447 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.448 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.448 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.149594+0000", 2024-08-17T17:39:24.448 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0097895120000000002 2024-08-17T17:39:24.448 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.448 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.448 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.448 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.149625+0000", 2024-08-17T17:39:24.448 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:24.450 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1786999999999999e-05 2024-08-17T17:39:24.450 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.450 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.450 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.450 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.149685+0000", 2024-08-17T17:39:24.450 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9420999999999999e-05 2024-08-17T17:39:24.450 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.450 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.450 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.451 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.451 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.451 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:246349 10.a 10:5bfa5980:::benchmark_data_smithi063_294527_object109164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1338)", 2024-08-17T17:39:24.451 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:58.207031+0000", 2024-08-17T17:39:24.451 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 326.20869505000002, 2024-08-17T17:39:24.451 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.94278713700000005, 2024-08-17T17:39:24.451 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.451 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.451 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.451 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:39:24.451 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:24.451 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 246349 2024-08-17T17:39:24.451 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.452 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.452 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.452 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.452 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.207031+0000", 2024-08-17T17:39:24.452 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.452 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.452 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.452 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.452 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.207031+0000", 2024-08-17T17:39:24.452 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.452 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.452 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.453 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.453 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.207030+0000", 2024-08-17T17:39:24.453 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:39:24.453 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.453 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.453 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.453 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.207035+0000", 2024-08-17T17:39:24.453 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0130000000000001e-06 2024-08-17T17:39:24.453 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.453 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.453 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.453 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.207047+0000", 2024-08-17T17:39:24.453 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.198e-05 2024-08-17T17:39:24.454 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.454 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.454 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.454 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.207055+0000", 2024-08-17T17:39:24.454 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.08e-06 2024-08-17T17:39:24.454 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.454 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.454 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.454 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.207130+0000", 2024-08-17T17:39:24.454 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.5139e-05 2024-08-17T17:39:24.454 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.454 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.454 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.455 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.207142+0000", 2024-08-17T17:39:24.455 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2809000000000001e-05 2024-08-17T17:39:24.455 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.455 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.455 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.455 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.130192+0000", 2024-08-17T17:39:24.455 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.92304980199999997 2024-08-17T17:39:24.455 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.455 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.455 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.455 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.130269+0000", 2024-08-17T17:39:24.455 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6456000000000001e-05 2024-08-17T17:39:24.456 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.456 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.456 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:39:24.456 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.130336+0000", 2024-08-17T17:39:24.456 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7590999999999997e-05 2024-08-17T17:39:24.456 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.456 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.456 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.456 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.134986+0000", 2024-08-17T17:39:24.456 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0046495620000000003 2024-08-17T17:39:24.456 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.456 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.456 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.456 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.143510+0000", 2024-08-17T17:39:24.457 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0085243449999999991 2024-08-17T17:39:24.457 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.457 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.457 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.457 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.149743+0000", 2024-08-17T17:39:24.457 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0062331640000000002 2024-08-17T17:39:24.457 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.457 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.457 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.457 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.149769+0000", 2024-08-17T17:39:24.457 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5593999999999999e-05 2024-08-17T17:39:24.457 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.457 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.457 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.458 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.149819+0000", 2024-08-17T17:39:24.458 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9580000000000003e-05 2024-08-17T17:39:24.458 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.458 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.458 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.458 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.458 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.458 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:246357 10.a 10:5c01e65f:::benchmark_data_smithi063_294527_object109172:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1338)", 2024-08-17T17:39:24.458 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:33:58.212382+0000", 2024-08-17T17:39:24.458 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 326.20334439700002, 2024-08-17T17:39:24.458 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95090854300000005, 2024-08-17T17:39:24.458 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.459 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.459 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.459 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:39:24.459 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:24.459 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 246357 2024-08-17T17:39:24.459 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.459 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.459 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.459 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.459 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.212382+0000", 2024-08-17T17:39:24.459 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.459 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.459 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.459 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.460 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.212382+0000", 2024-08-17T17:39:24.460 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.460 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.460 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.460 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.460 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.212381+0000", 2024-08-17T17:39:24.460 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:24.460 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.460 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.460 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.460 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.212387+0000", 2024-08-17T17:39:24.460 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.7479999999999999e-06 2024-08-17T17:39:24.460 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.460 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.461 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.461 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.212399+0000", 2024-08-17T17:39:24.461 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2767e-05 2024-08-17T17:39:24.461 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.461 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.461 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.461 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.212407+0000", 2024-08-17T17:39:24.461 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.089e-06 2024-08-17T17:39:24.461 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.461 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.461 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.461 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.212462+0000", 2024-08-17T17:39:24.461 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4562999999999999e-05 2024-08-17T17:39:24.462 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.462 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.462 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.462 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:58.212473+0000", 2024-08-17T17:39:24.462 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0573e-05 2024-08-17T17:39:24.462 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.462 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.462 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.462 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.131639+0000", 2024-08-17T17:39:24.462 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.919166029 2024-08-17T17:39:24.462 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.462 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.462 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.463 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.131735+0000", 2024-08-17T17:39:24.463 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.5975000000000002e-05 2024-08-17T17:39:24.463 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.463 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.463 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,2", 2024-08-17T17:39:24.463 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.131838+0000", 2024-08-17T17:39:24.463 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000103471 2024-08-17T17:39:24.463 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.463 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.463 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.463 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.138377+0000", 2024-08-17T17:39:24.463 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0065389189999999998 2024-08-17T17:39:24.464 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.464 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.464 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.464 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.149878+0000", 2024-08-17T17:39:24.464 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011501192 2024-08-17T17:39:24.464 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.464 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.464 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.464 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.163154+0000", 2024-08-17T17:39:24.464 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.013275729999999999 2024-08-17T17:39:24.464 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.464 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.464 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.464 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.163196+0000", 2024-08-17T17:39:24.465 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2258000000000001e-05 2024-08-17T17:39:24.465 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.465 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.465 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.465 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:33:59.163291+0000", 2024-08-17T17:39:24.465 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4532999999999998e-05 2024-08-17T17:39:24.465 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.465 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.465 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.465 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.465 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.465 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:246923 10.7 10:e0f4703d:::benchmark_data_smithi063_294527_object109738:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1340)", 2024-08-17T17:39:24.465 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:34:00.141593+0000", 2024-08-17T17:39:24.466 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 324.27413309100001, 2024-08-17T17:39:24.466 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.984131329, 2024-08-17T17:39:24.466 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.466 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.466 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.466 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:39:24.466 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:24.466 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 246923 2024-08-17T17:39:24.466 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.466 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.466 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.466 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.466 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.141593+0000", 2024-08-17T17:39:24.467 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.467 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.467 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.467 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.467 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.141593+0000", 2024-08-17T17:39:24.467 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.467 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.467 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.467 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.467 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.141592+0000", 2024-08-17T17:39:24.467 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:24.467 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.467 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.468 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.468 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.141598+0000", 2024-08-17T17:39:24.468 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.66e-06 2024-08-17T17:39:24.468 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.468 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.468 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.468 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.141611+0000", 2024-08-17T17:39:24.468 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3326000000000001e-05 2024-08-17T17:39:24.468 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.468 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.468 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.468 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.141625+0000", 2024-08-17T17:39:24.469 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3686000000000001e-05 2024-08-17T17:39:24.469 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.469 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.469 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.469 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.141659+0000", 2024-08-17T17:39:24.469 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4612e-05 2024-08-17T17:39:24.469 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.469 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.469 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.469 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.141665+0000", 2024-08-17T17:39:24.469 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5940000000000001e-06 2024-08-17T17:39:24.469 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.469 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.469 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.470 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.117378+0000", 2024-08-17T17:39:24.470 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97571269100000002 2024-08-17T17:39:24.470 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.470 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.470 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.470 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.117519+0000", 2024-08-17T17:39:24.470 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014144299999999999 2024-08-17T17:39:24.470 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.470 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.470 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:39:24.470 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.117974+0000", 2024-08-17T17:39:24.470 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00045487700000000002 2024-08-17T17:39:24.470 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.471 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.471 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.471 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.123814+0000", 2024-08-17T17:39:24.471 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0058404920000000001 2024-08-17T17:39:24.471 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.471 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.471 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.471 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.125523+0000", 2024-08-17T17:39:24.471 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0017087269999999999 2024-08-17T17:39:24.471 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.471 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.471 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.471 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.125570+0000", 2024-08-17T17:39:24.471 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6516e-05 2024-08-17T17:39:24.472 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.472 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.472 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.472 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.125624+0000", 2024-08-17T17:39:24.472 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4163000000000003e-05 2024-08-17T17:39:24.472 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.472 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.472 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.472 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.125725+0000", 2024-08-17T17:39:24.472 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000100926 2024-08-17T17:39:24.472 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.472 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.472 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.472 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.473 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.473 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:246948 10.7 10:e5b9ce73:::benchmark_data_smithi063_294527_object109763:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1340)", 2024-08-17T17:39:24.473 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:34:00.168342+0000", 2024-08-17T17:39:24.473 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 324.247384539, 2024-08-17T17:39:24.473 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.96957490499999999, 2024-08-17T17:39:24.473 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.473 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.473 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.473 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:39:24.473 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:24.473 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 246948 2024-08-17T17:39:24.473 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.473 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.474 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.474 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.474 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.168342+0000", 2024-08-17T17:39:24.474 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.474 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.474 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.474 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.474 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.168342+0000", 2024-08-17T17:39:24.474 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.474 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.474 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.474 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.474 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.168341+0000", 2024-08-17T17:39:24.475 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:39:24.475 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.475 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.475 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.475 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.168344+0000", 2024-08-17T17:39:24.475 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.0130000000000002e-06 2024-08-17T17:39:24.475 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.475 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.475 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.475 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.168353+0000", 2024-08-17T17:39:24.475 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0879999999999995e-06 2024-08-17T17:39:24.475 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.475 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.475 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.476 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.168358+0000", 2024-08-17T17:39:24.476 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3000000000000001e-06 2024-08-17T17:39:24.476 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.476 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.476 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.476 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.168403+0000", 2024-08-17T17:39:24.476 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4879999999999997e-05 2024-08-17T17:39:24.476 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.476 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.476 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.476 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:00.168413+0000", 2024-08-17T17:39:24.476 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0023999999999999e-05 2024-08-17T17:39:24.476 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.476 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.477 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.477 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.120323+0000", 2024-08-17T17:39:24.477 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95190994500000004 2024-08-17T17:39:24.477 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.477 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.477 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.477 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.120411+0000", 2024-08-17T17:39:24.477 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8614000000000002e-05 2024-08-17T17:39:24.477 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.477 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.477 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:39:24.477 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.120663+0000", 2024-08-17T17:39:24.477 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00025119299999999998 2024-08-17T17:39:24.478 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.478 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.478 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.478 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.125480+0000", 2024-08-17T17:39:24.478 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0048174949999999998 2024-08-17T17:39:24.478 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.478 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.478 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.478 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.125761+0000", 2024-08-17T17:39:24.478 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00028125300000000003 2024-08-17T17:39:24.478 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.478 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.478 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.478 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.137818+0000", 2024-08-17T17:39:24.479 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012056502 2024-08-17T17:39:24.479 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.479 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.479 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.479 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.137846+0000", 2024-08-17T17:39:24.479 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8218999999999999e-05 2024-08-17T17:39:24.479 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.479 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.479 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.479 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:01.137917+0000", 2024-08-17T17:39:24.479 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.0853999999999996e-05 2024-08-17T17:39:24.479 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.479 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.480 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.480 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.480 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.480 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:249102 10.17 10:eb5c6bdb:::benchmark_data_smithi063_294527_object111917:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1342)", 2024-08-17T17:39:24.480 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:34:02.464485+0000", 2024-08-17T17:39:24.480 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 321.95124125299998, 2024-08-17T17:39:24.480 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.96629201600000003, 2024-08-17T17:39:24.480 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.480 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.480 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.480 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:39:24.480 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:24.480 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 249102 2024-08-17T17:39:24.480 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.481 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.481 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.481 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.481 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.464485+0000", 2024-08-17T17:39:24.481 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.481 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.481 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.481 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.481 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.464485+0000", 2024-08-17T17:39:24.481 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.481 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.481 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.481 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.481 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.464484+0000", 2024-08-17T17:39:24.481 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:39:24.482 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.482 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.482 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.482 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.464489+0000", 2024-08-17T17:39:24.482 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7539999999999997e-06 2024-08-17T17:39:24.482 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.482 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.482 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.482 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.464499+0000", 2024-08-17T17:39:24.482 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0468999999999999e-05 2024-08-17T17:39:24.482 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.482 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.482 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.482 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.464504+0000", 2024-08-17T17:39:24.483 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9309999999999999e-06 2024-08-17T17:39:24.483 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.483 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.483 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.483 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.464544+0000", 2024-08-17T17:39:24.483 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9461e-05 2024-08-17T17:39:24.483 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.483 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.483 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.483 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.464551+0000", 2024-08-17T17:39:24.483 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8170000000000001e-06 2024-08-17T17:39:24.483 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.483 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.483 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.483 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.426666+0000", 2024-08-17T17:39:24.484 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.96211578499999995 2024-08-17T17:39:24.484 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.484 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.484 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.484 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.426787+0000", 2024-08-17T17:39:24.484 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000120202 2024-08-17T17:39:24.484 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.484 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.484 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:39:24.484 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.427085+0000", 2024-08-17T17:39:24.484 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00029837199999999998 2024-08-17T17:39:24.484 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.484 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.484 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.485 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.430574+0000", 2024-08-17T17:39:24.485 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0034893039999999999 2024-08-17T17:39:24.485 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.485 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.485 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.485 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.430581+0000", 2024-08-17T17:39:24.485 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.224e-06 2024-08-17T17:39:24.485 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.485 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.485 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.485 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.430681+0000", 2024-08-17T17:39:24.485 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000100296 2024-08-17T17:39:24.485 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.485 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.485 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.486 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.430720+0000", 2024-08-17T17:39:24.486 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.9507e-05 2024-08-17T17:39:24.486 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.486 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.486 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.486 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.430777+0000", 2024-08-17T17:39:24.486 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.6950000000000002e-05 2024-08-17T17:39:24.486 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.486 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.486 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.486 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.486 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.486 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:249122 10.17 10:efd10d53:::benchmark_data_smithi063_294527_object111937:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1342)", 2024-08-17T17:39:24.486 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:34:02.479767+0000", 2024-08-17T17:39:24.487 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 321.93595913000001, 2024-08-17T17:39:24.487 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95438025000000004, 2024-08-17T17:39:24.487 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.487 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.487 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.487 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:39:24.487 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:24.487 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 249122 2024-08-17T17:39:24.487 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.487 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.487 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.487 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.487 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.479767+0000", 2024-08-17T17:39:24.488 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.488 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.488 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.488 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.488 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.479767+0000", 2024-08-17T17:39:24.488 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.488 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.488 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.488 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.488 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.479767+0000", 2024-08-17T17:39:24.488 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:39:24.488 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.488 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.488 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.489 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.479770+0000", 2024-08-17T17:39:24.489 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.0199999999999999e-06 2024-08-17T17:39:24.489 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.489 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.489 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.489 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.479776+0000", 2024-08-17T17:39:24.489 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.9039999999999997e-06 2024-08-17T17:39:24.489 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.489 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.489 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.489 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.479779+0000", 2024-08-17T17:39:24.489 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4800000000000001e-06 2024-08-17T17:39:24.489 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.490 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.490 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.490 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.479822+0000", 2024-08-17T17:39:24.490 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.2845000000000001e-05 2024-08-17T17:39:24.490 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.490 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.490 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.490 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:02.479830+0000", 2024-08-17T17:39:24.490 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.1599999999999998e-06 2024-08-17T17:39:24.490 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.490 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.490 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.490 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.429101+0000", 2024-08-17T17:39:24.491 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.94927101300000005 2024-08-17T17:39:24.491 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.491 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.491 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.491 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.429184+0000", 2024-08-17T17:39:24.491 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.2746999999999999e-05 2024-08-17T17:39:24.491 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.491 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.491 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,3", 2024-08-17T17:39:24.491 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.429439+0000", 2024-08-17T17:39:24.491 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00025508099999999999 2024-08-17T17:39:24.491 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.491 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.492 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.492 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.433990+0000", 2024-08-17T17:39:24.492 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.004551172 2024-08-17T17:39:24.492 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.492 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.492 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.492 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.434035+0000", 2024-08-17T17:39:24.492 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4289999999999998e-05 2024-08-17T17:39:24.492 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.492 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.492 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.492 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.434066+0000", 2024-08-17T17:39:24.492 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1265e-05 2024-08-17T17:39:24.492 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.493 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.493 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.493 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.434095+0000", 2024-08-17T17:39:24.493 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8966999999999999e-05 2024-08-17T17:39:24.493 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.493 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.493 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.493 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:03.434148+0000", 2024-08-17T17:39:24.493 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2738999999999998e-05 2024-08-17T17:39:24.493 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.493 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.493 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.493 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.494 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.494 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:267653 10.3 10:c03404cc:::benchmark_data_smithi063_294527_object130468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1350)", 2024-08-17T17:39:24.494 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:34:21.758604+0000", 2024-08-17T17:39:24.494 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 302.657122259, 2024-08-17T17:39:24.494 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98898404200000001, 2024-08-17T17:39:24.494 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.494 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.494 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.494 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:39:24.494 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:24.494 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 267653 2024-08-17T17:39:24.494 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.494 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.494 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.495 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.495 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.758604+0000", 2024-08-17T17:39:24.495 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.495 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.495 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.495 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.495 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.758604+0000", 2024-08-17T17:39:24.495 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.495 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.495 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.495 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.495 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.758603+0000", 2024-08-17T17:39:24.495 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:39:24.495 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.495 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.496 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.496 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.758607+0000", 2024-08-17T17:39:24.496 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5140000000000002e-06 2024-08-17T17:39:24.496 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.496 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.496 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.496 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.758614+0000", 2024-08-17T17:39:24.496 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4109999999999999e-06 2024-08-17T17:39:24.496 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.496 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.496 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.496 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.758618+0000", 2024-08-17T17:39:24.496 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.0099999999999997e-06 2024-08-17T17:39:24.496 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.497 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.497 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.497 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.758650+0000", 2024-08-17T17:39:24.497 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1958999999999997e-05 2024-08-17T17:39:24.497 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.497 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.497 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.497 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.758657+0000", 2024-08-17T17:39:24.497 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9480000000000002e-06 2024-08-17T17:39:24.497 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.497 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.497 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.498 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.734064+0000", 2024-08-17T17:39:24.498 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97540663100000002 2024-08-17T17:39:24.498 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.498 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.498 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.498 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.734223+0000", 2024-08-17T17:39:24.498 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015952100000000001 2024-08-17T17:39:24.498 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.498 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.498 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:39:24.498 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.734666+0000", 2024-08-17T17:39:24.498 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00044259699999999998 2024-08-17T17:39:24.499 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.499 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.499 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.499 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.741393+0000", 2024-08-17T17:39:24.499 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0067266690000000002 2024-08-17T17:39:24.499 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.499 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.499 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.499 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.744142+0000", 2024-08-17T17:39:24.499 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002749769 2024-08-17T17:39:24.499 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.499 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.499 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.500 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.747512+0000", 2024-08-17T17:39:24.500 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003369103 2024-08-17T17:39:24.500 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.500 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.500 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.500 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.747553+0000", 2024-08-17T17:39:24.500 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1143999999999998e-05 2024-08-17T17:39:24.500 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.500 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.500 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.500 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.747588+0000", 2024-08-17T17:39:24.500 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5528000000000003e-05 2024-08-17T17:39:24.500 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.500 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.501 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.501 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.501 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.501 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:267655 10.3 10:c6f407ff:::benchmark_data_smithi063_294527_object130470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1350)", 2024-08-17T17:39:24.501 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:34:21.763951+0000", 2024-08-17T17:39:24.501 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 302.651775606, 2024-08-17T17:39:24.501 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.98379375800000002, 2024-08-17T17:39:24.501 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.501 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.501 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.501 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:39:24.501 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:24.501 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 267655 2024-08-17T17:39:24.501 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.502 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.502 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.502 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.502 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.763951+0000", 2024-08-17T17:39:24.502 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.502 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.502 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.502 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.502 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.763951+0000", 2024-08-17T17:39:24.502 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.502 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.502 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.503 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.503 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.763950+0000", 2024-08-17T17:39:24.503 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:39:24.503 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.503 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.503 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.503 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.763953+0000", 2024-08-17T17:39:24.503 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.807e-06 2024-08-17T17:39:24.503 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.503 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.503 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.503 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.763958+0000", 2024-08-17T17:39:24.503 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.0939999999999998e-06 2024-08-17T17:39:24.503 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.503 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.504 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.504 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.763962+0000", 2024-08-17T17:39:24.504 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.7270000000000001e-06 2024-08-17T17:39:24.504 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.504 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.504 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.504 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.763993+0000", 2024-08-17T17:39:24.504 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.1192000000000002e-05 2024-08-17T17:39:24.504 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.504 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.504 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.504 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:21.763999+0000", 2024-08-17T17:39:24.504 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3929999999999999e-06 2024-08-17T17:39:24.504 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.505 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.505 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.505 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.737141+0000", 2024-08-17T17:39:24.505 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.97314265 2024-08-17T17:39:24.505 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.505 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.505 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.505 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.737209+0000", 2024-08-17T17:39:24.505 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7417999999999997e-05 2024-08-17T17:39:24.505 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.505 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.505 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 0,4", 2024-08-17T17:39:24.505 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.737447+0000", 2024-08-17T17:39:24.505 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00023807400000000001 2024-08-17T17:39:24.505 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.506 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.506 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.506 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.744150+0000", 2024-08-17T17:39:24.506 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0067035209999999996 2024-08-17T17:39:24.506 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.506 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.506 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.506 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.744185+0000", 2024-08-17T17:39:24.506 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.4895999999999999e-05 2024-08-17T17:39:24.506 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.506 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.507 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.507 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.747618+0000", 2024-08-17T17:39:24.507 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0034333300000000001 2024-08-17T17:39:24.508 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.508 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.508 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.508 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.747691+0000", 2024-08-17T17:39:24.508 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2510999999999997e-05 2024-08-17T17:39:24.508 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.508 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.508 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.508 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:22.747745+0000", 2024-08-17T17:39:24.508 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3724999999999998e-05 2024-08-17T17:39:24.508 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.508 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.508 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.509 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.509 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.509 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:267853 10.9 10:915757ea:::benchmark_data_smithi063_294527_object130668:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1352)", 2024-08-17T17:39:24.509 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:34:23.161991+0000", 2024-08-17T17:39:24.509 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 301.25373559799999, 2024-08-17T17:39:24.509 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.967502482, 2024-08-17T17:39:24.509 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.509 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.509 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.509 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:39:24.509 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:24.509 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 267853 2024-08-17T17:39:24.509 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.509 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.509 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.510 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.510 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.161991+0000", 2024-08-17T17:39:24.510 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.510 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.510 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.510 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.510 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.161991+0000", 2024-08-17T17:39:24.510 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.510 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.510 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.510 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.510 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.161990+0000", 2024-08-17T17:39:24.510 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:39:24.510 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.511 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.511 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.511 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.161993+0000", 2024-08-17T17:39:24.511 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5859999999999999e-06 2024-08-17T17:39:24.511 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.511 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.511 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.511 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.162000+0000", 2024-08-17T17:39:24.511 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8639999999999998e-06 2024-08-17T17:39:24.511 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.511 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.511 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.511 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.162011+0000", 2024-08-17T17:39:24.511 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0862999999999999e-05 2024-08-17T17:39:24.512 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.512 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.512 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.512 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.163496+0000", 2024-08-17T17:39:24.512 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0014850169999999999 2024-08-17T17:39:24.512 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.512 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.512 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.512 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.163504+0000", 2024-08-17T17:39:24.512 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6799999999999999e-06 2024-08-17T17:39:24.512 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.512 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.512 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.512 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.123458+0000", 2024-08-17T17:39:24.512 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95995322599999999 2024-08-17T17:39:24.513 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.513 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.513 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.513 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.123547+0000", 2024-08-17T17:39:24.513 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9365999999999998e-05 2024-08-17T17:39:24.513 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.513 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.513 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:39:24.513 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.123851+0000", 2024-08-17T17:39:24.513 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00030409799999999999 2024-08-17T17:39:24.513 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.513 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.513 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.513 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.126024+0000", 2024-08-17T17:39:24.513 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021725009999999999 2024-08-17T17:39:24.514 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.514 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.514 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.514 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.126036+0000", 2024-08-17T17:39:24.514 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1990999999999999e-05 2024-08-17T17:39:24.514 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.514 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.514 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.514 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.129419+0000", 2024-08-17T17:39:24.514 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0033838050000000001 2024-08-17T17:39:24.515 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.515 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.515 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.515 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.129468+0000", 2024-08-17T17:39:24.515 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.8946999999999997e-05 2024-08-17T17:39:24.515 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.515 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.515 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.515 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.129493+0000", 2024-08-17T17:39:24.515 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.499e-05 2024-08-17T17:39:24.515 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.515 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.515 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.515 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.515 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.516 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.45818.0:267866 10.9 10:92a7aad4:::benchmark_data_smithi063_294527_object130681:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1352)", 2024-08-17T17:39:24.516 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:34:23.178640+0000", 2024-08-17T17:39:24.516 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 301.237086917, 2024-08-17T17:39:24.516 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.95101899099999998, 2024-08-17T17:39:24.516 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.516 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.516 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.516 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.45818", 2024-08-17T17:39:24.516 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/71900529", 2024-08-17T17:39:24.516 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 267866 2024-08-17T17:39:24.516 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.516 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.516 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.516 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.517 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.178640+0000", 2024-08-17T17:39:24.517 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.517 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.517 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.517 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.517 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.178640+0000", 2024-08-17T17:39:24.517 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.517 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.517 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.517 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.517 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.178638+0000", 2024-08-17T17:39:24.517 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:24.517 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.517 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.518 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.518 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.178645+0000", 2024-08-17T17:39:24.518 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3169999999999998e-06 2024-08-17T17:39:24.518 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.518 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.518 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.518 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.178657+0000", 2024-08-17T17:39:24.518 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2082999999999999e-05 2024-08-17T17:39:24.518 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.518 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.518 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.518 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.178664+0000", 2024-08-17T17:39:24.518 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4569999999999999e-06 2024-08-17T17:39:24.518 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.519 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.519 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.519 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.178710+0000", 2024-08-17T17:39:24.519 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6221999999999999e-05 2024-08-17T17:39:24.519 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.519 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.519 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.519 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:23.178720+0000", 2024-08-17T17:39:24.519 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2720000000000003e-06 2024-08-17T17:39:24.519 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.519 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.519 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.519 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.126079+0000", 2024-08-17T17:39:24.519 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.94735958899999995 2024-08-17T17:39:24.519 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.520 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.520 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.520 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.126158+0000", 2024-08-17T17:39:24.520 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.9105999999999998e-05 2024-08-17T17:39:24.520 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.520 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.520 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 4,7", 2024-08-17T17:39:24.520 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.126374+0000", 2024-08-17T17:39:24.520 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00021593 2024-08-17T17:39:24.520 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.520 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.520 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.520 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.129529+0000", 2024-08-17T17:39:24.520 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0031546310000000002 2024-08-17T17:39:24.520 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.521 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.521 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.521 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.129584+0000", 2024-08-17T17:39:24.521 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5022999999999998e-05 2024-08-17T17:39:24.521 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.521 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.521 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.521 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.129613+0000", 2024-08-17T17:39:24.521 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8704000000000002e-05 2024-08-17T17:39:24.521 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.521 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.521 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.521 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.129638+0000", 2024-08-17T17:39:24.522 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.5369999999999999e-05 2024-08-17T17:39:24.522 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.522 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.522 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.522 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:34:24.129659+0000", 2024-08-17T17:39:24.522 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.0483e-05 2024-08-17T17:39:24.522 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.522 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.522 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.522 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.522 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.522 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:45450 11.5 11:a5a92106:::benchmark_data_smithi063_328699_object45449:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1375)", 2024-08-17T17:39:24.522 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:54.179865+0000", 2024-08-17T17:39:24.523 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 210.235861382, 2024-08-17T17:39:24.523 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.262016547, 2024-08-17T17:39:24.523 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.523 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:39:24.523 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.523 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:24.523 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:24.523 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 45450 2024-08-17T17:39:24.523 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.523 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.523 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.523 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.523 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.179865+0000", 2024-08-17T17:39:24.523 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.524 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.524 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.524 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.524 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.179865+0000", 2024-08-17T17:39:24.524 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.524 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.524 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.524 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.524 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.179865+0000", 2024-08-17T17:39:24.524 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:39:24.524 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.524 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.524 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.524 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.179898+0000", 2024-08-17T17:39:24.525 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.307e-05 2024-08-17T17:39:24.525 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.525 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.525 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.525 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.179906+0000", 2024-08-17T17:39:24.525 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.7879999999999995e-06 2024-08-17T17:39:24.525 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.525 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.525 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.525 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.179913+0000", 2024-08-17T17:39:24.525 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.4810000000000003e-06 2024-08-17T17:39:24.525 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.525 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.525 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.525 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.180087+0000", 2024-08-17T17:39:24.526 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000174048 2024-08-17T17:39:24.526 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.526 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.526 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.526 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.180222+0000", 2024-08-17T17:39:24.526 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013555699999999999 2024-08-17T17:39:24.526 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.526 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.526 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 7", 2024-08-17T17:39:24.526 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.180385+0000", 2024-08-17T17:39:24.526 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00016259299999999999 2024-08-17T17:39:24.526 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.526 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.527 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.527 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.187584+0000", 2024-08-17T17:39:24.527 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0071993389999999999 2024-08-17T17:39:24.527 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.527 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.527 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.527 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.210572+0000", 2024-08-17T17:39:24.527 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.022987872999999999 2024-08-17T17:39:24.527 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.527 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.527 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.527 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.210585+0000", 2024-08-17T17:39:24.527 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2768e-05 2024-08-17T17:39:24.528 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.528 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.528 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.528 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.209609+0000", 2024-08-17T17:39:24.528 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99902418999999998 2024-08-17T17:39:24.528 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.528 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.528 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.528 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.209624+0000", 2024-08-17T17:39:24.528 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4406000000000001e-05 2024-08-17T17:39:24.528 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.528 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.528 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.528 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.441742+0000", 2024-08-17T17:39:24.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.232118818 2024-08-17T17:39:24.529 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.529 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.529 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.529 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.441882+0000", 2024-08-17T17:39:24.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00013917399999999999 2024-08-17T17:39:24.529 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.529 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.529 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.529 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.529 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.529 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:45459 11.8 11:106ee8dc:::benchmark_data_smithi063_328699_object45458:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1375)", 2024-08-17T17:39:24.529 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:54.189716+0000", 2024-08-17T17:39:24.529 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 210.22601036500001, 2024-08-17T17:39:24.529 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2688175680000002, 2024-08-17T17:39:24.530 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.530 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:39:24.530 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.530 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:24.530 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:24.530 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 45459 2024-08-17T17:39:24.530 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.530 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.530 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.530 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.530 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.189716+0000", 2024-08-17T17:39:24.530 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.530 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.530 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.531 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.531 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.189716+0000", 2024-08-17T17:39:24.531 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.531 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.531 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.531 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.531 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.189715+0000", 2024-08-17T17:39:24.531 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:24.531 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.531 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.531 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.531 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.190299+0000", 2024-08-17T17:39:24.531 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00058463999999999999 2024-08-17T17:39:24.531 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.531 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.532 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.190312+0000", 2024-08-17T17:39:24.532 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2388999999999999e-05 2024-08-17T17:39:24.532 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.532 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.532 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.190319+0000", 2024-08-17T17:39:24.532 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6170000000000001e-06 2024-08-17T17:39:24.532 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.532 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.532 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.532 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.190582+0000", 2024-08-17T17:39:24.532 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00026261500000000001 2024-08-17T17:39:24.532 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.533 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.533 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.533 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.198918+0000", 2024-08-17T17:39:24.533 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0083360699999999992 2024-08-17T17:39:24.533 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.533 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.533 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,4", 2024-08-17T17:39:24.533 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.199081+0000", 2024-08-17T17:39:24.533 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000162597 2024-08-17T17:39:24.533 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.533 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.533 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.534 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209951+0000", 2024-08-17T17:39:24.534 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010870716000000001 2024-08-17T17:39:24.534 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.534 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.534 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.534 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209963+0000", 2024-08-17T17:39:24.534 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1185e-05 2024-08-17T17:39:24.534 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.534 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.534 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.534 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.203722+0000", 2024-08-17T17:39:24.534 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99375982600000001 2024-08-17T17:39:24.535 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.535 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.535 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.535 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.203736+0000", 2024-08-17T17:39:24.535 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.377e-05 2024-08-17T17:39:24.535 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.535 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.535 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.535 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.443446+0000", 2024-08-17T17:39:24.535 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2397096409999999 2024-08-17T17:39:24.535 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.535 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.535 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for degraded object", 2024-08-17T17:39:24.535 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.444887+0000", 2024-08-17T17:39:24.536 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001440839 2024-08-17T17:39:24.536 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.536 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.536 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.536 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.458415+0000", 2024-08-17T17:39:24.536 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.013528277 2024-08-17T17:39:24.536 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.536 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.536 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.536 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.458534+0000", 2024-08-17T17:39:24.536 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00011879000000000001 2024-08-17T17:39:24.536 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.536 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.536 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.537 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.537 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.537 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:45463 11.5 11:ae6ebda3:::benchmark_data_smithi063_328699_object45462:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1376)", 2024-08-17T17:39:24.537 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:54.205660+0000", 2024-08-17T17:39:24.537 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 210.210066646, 2024-08-17T17:39:24.537 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2469025020000002, 2024-08-17T17:39:24.537 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.537 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.537 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.537 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:24.537 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:24.537 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 45463 2024-08-17T17:39:24.538 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.538 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.538 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.538 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.538 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.205660+0000", 2024-08-17T17:39:24.538 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.538 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.538 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.538 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.538 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.205660+0000", 2024-08-17T17:39:24.538 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.538 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.538 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.539 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.539 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.205658+0000", 2024-08-17T17:39:24.539 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:24.539 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.539 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.539 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.539 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.205688+0000", 2024-08-17T17:39:24.539 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.9638999999999999e-05 2024-08-17T17:39:24.539 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.539 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.539 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.539 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.205701+0000", 2024-08-17T17:39:24.539 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2476e-05 2024-08-17T17:39:24.539 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.540 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.540 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.540 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.205710+0000", 2024-08-17T17:39:24.540 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.1600000000000004e-06 2024-08-17T17:39:24.540 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.540 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.540 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.540 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.210615+0000", 2024-08-17T17:39:24.540 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0049054809999999997 2024-08-17T17:39:24.540 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.540 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.540 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.540 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.210624+0000", 2024-08-17T17:39:24.540 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.0110000000000006e-06 2024-08-17T17:39:24.540 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.541 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.541 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.209649+0000", 2024-08-17T17:39:24.541 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99902501899999996 2024-08-17T17:39:24.541 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.541 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.541 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.209658+0000", 2024-08-17T17:39:24.541 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.8470000000000002e-06 2024-08-17T17:39:24.541 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.541 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.541 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.541 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.443744+0000", 2024-08-17T17:39:24.541 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.234086378 2024-08-17T17:39:24.541 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.542 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.542 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.542 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.443846+0000", 2024-08-17T17:39:24.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000101377 2024-08-17T17:39:24.542 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.542 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.542 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 6,7", 2024-08-17T17:39:24.542 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.443963+0000", 2024-08-17T17:39:24.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000117409 2024-08-17T17:39:24.542 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.542 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.542 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.542 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.450270+0000", 2024-08-17T17:39:24.542 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0063068040000000001 2024-08-17T17:39:24.543 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.543 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.543 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.543 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.450304+0000", 2024-08-17T17:39:24.543 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.3621999999999998e-05 2024-08-17T17:39:24.543 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.543 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.543 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.543 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.450347+0000", 2024-08-17T17:39:24.543 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3547999999999997e-05 2024-08-17T17:39:24.543 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.543 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.543 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.543 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.450397+0000", 2024-08-17T17:39:24.543 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.9296999999999999e-05 2024-08-17T17:39:24.544 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.544 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.544 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.544 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.452562+0000", 2024-08-17T17:39:24.544 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021658179999999999 2024-08-17T17:39:24.544 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.544 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.544 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.544 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.544 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.544 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:45465 11.5 11:a5f199ae:::benchmark_data_smithi063_328699_object45464:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1376)", 2024-08-17T17:39:24.544 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:54.209706+0000", 2024-08-17T17:39:24.544 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 210.20602097099999, 2024-08-17T17:39:24.544 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.2474715590000001, 2024-08-17T17:39:24.545 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.545 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.545 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.545 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:24.545 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:24.545 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 45465 2024-08-17T17:39:24.545 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.545 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.545 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.545 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.545 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209706+0000", 2024-08-17T17:39:24.545 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.545 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.545 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.546 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.546 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209706+0000", 2024-08-17T17:39:24.546 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.546 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.546 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.546 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.546 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209704+0000", 2024-08-17T17:39:24.546 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:24.546 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.546 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.546 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.546 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209742+0000", 2024-08-17T17:39:24.547 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.8269000000000002e-05 2024-08-17T17:39:24.547 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.547 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.547 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.547 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209755+0000", 2024-08-17T17:39:24.547 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2901999999999999e-05 2024-08-17T17:39:24.547 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.547 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.547 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.547 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.209764+0000", 2024-08-17T17:39:24.547 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.789e-06 2024-08-17T17:39:24.547 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.548 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.548 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.548 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.210648+0000", 2024-08-17T17:39:24.548 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00088403799999999997 2024-08-17T17:39:24.548 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.548 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.548 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.548 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.210657+0000", 2024-08-17T17:39:24.548 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6940000000000001e-06 2024-08-17T17:39:24.548 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.548 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.548 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.209681+0000", 2024-08-17T17:39:24.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99902387000000004 2024-08-17T17:39:24.549 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.549 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.209689+0000", 2024-08-17T17:39:24.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.6810000000000004e-06 2024-08-17T17:39:24.549 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.549 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.549 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.549 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.445945+0000", 2024-08-17T17:39:24.549 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.236256147 2024-08-17T17:39:24.550 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.550 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.550 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.550 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.446103+0000", 2024-08-17T17:39:24.550 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015809400000000001 2024-08-17T17:39:24.550 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.550 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.550 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 6,7", 2024-08-17T17:39:24.550 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.446256+0000", 2024-08-17T17:39:24.550 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015296 2024-08-17T17:39:24.550 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.550 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.551 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.551 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.450312+0000", 2024-08-17T17:39:24.551 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0040553059999999998 2024-08-17T17:39:24.551 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.551 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.551 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.551 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.453563+0000", 2024-08-17T17:39:24.551 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003251711 2024-08-17T17:39:24.551 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.551 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.551 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.551 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.454799+0000", 2024-08-17T17:39:24.551 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012358849999999999 2024-08-17T17:39:24.551 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.552 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.552 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.552 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.454931+0000", 2024-08-17T17:39:24.552 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000131173 2024-08-17T17:39:24.552 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.552 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.552 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.552 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.457177+0000", 2024-08-17T17:39:24.552 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002246549 2024-08-17T17:39:24.552 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.552 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.552 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.552 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.553 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.553 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:45472 11.5 11:a8651b52:::benchmark_data_smithi063_328699_object45471:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1376)", 2024-08-17T17:39:24.553 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:35:54.217091+0000", 2024-08-17T17:39:24.553 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 210.19863532400001, 2024-08-17T17:39:24.553 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.243690322, 2024-08-17T17:39:24.553 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.553 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:24.553 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.553 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:24.553 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:24.553 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 45472 2024-08-17T17:39:24.553 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.553 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.554 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.554 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.554 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.217091+0000", 2024-08-17T17:39:24.554 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.554 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.554 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.554 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.554 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.217091+0000", 2024-08-17T17:39:24.554 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.554 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.554 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.554 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.554 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.217090+0000", 2024-08-17T17:39:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:39:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.217153+0000", 2024-08-17T17:39:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2960999999999996e-05 2024-08-17T17:39:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.217165+0000", 2024-08-17T17:39:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1617000000000001e-05 2024-08-17T17:39:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.555 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.217172+0000", 2024-08-17T17:39:24.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4279999999999999e-06 2024-08-17T17:39:24.556 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.556 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.556 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.217219+0000", 2024-08-17T17:39:24.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6582999999999998e-05 2024-08-17T17:39:24.556 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.556 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.556 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.556 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:54.217228+0000", 2024-08-17T17:39:24.556 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9760000000000001e-06 2024-08-17T17:39:24.556 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.557 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.209712+0000", 2024-08-17T17:39:24.557 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.99248398100000002 2024-08-17T17:39:24.557 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.557 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:55.209720+0000", 2024-08-17T17:39:24.557 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.5900000000000008e-06 2024-08-17T17:39:24.557 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.557 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.557 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.557 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.447935+0000", 2024-08-17T17:39:24.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.23821454 2024-08-17T17:39:24.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.558 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.558 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.558 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.448077+0000", 2024-08-17T17:39:24.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00014181799999999999 2024-08-17T17:39:24.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.558 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.558 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 6,7", 2024-08-17T17:39:24.558 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.448232+0000", 2024-08-17T17:39:24.558 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00015555699999999999 2024-08-17T17:39:24.558 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.559 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.559 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:24.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.452588+0000", 2024-08-17T17:39:24.559 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0043554800000000001 2024-08-17T17:39:24.559 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.559 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.559 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.457370+0000", 2024-08-17T17:39:24.559 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0047819220000000001 2024-08-17T17:39:24.559 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.559 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.559 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:24.559 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.458580+0000", 2024-08-17T17:39:24.560 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012100209999999999 2024-08-17T17:39:24.560 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.560 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.560 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:24.560 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.458626+0000", 2024-08-17T17:39:24.560 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.6557000000000002e-05 2024-08-17T17:39:24.560 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.560 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.560 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.560 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:35:56.460781+0000", 2024-08-17T17:39:24.560 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021553980000000002 2024-08-17T17:39:24.560 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.560 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.560 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.560 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.561 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.561 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:62271 11.8 11:13a623d9:::benchmark_data_smithi063_328699_object62270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1381)", 2024-08-17T17:39:24.561 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:11.731037+0000", 2024-08-17T17:39:24.561 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 192.684689924, 2024-08-17T17:39:24.561 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0247610199999999, 2024-08-17T17:39:24.561 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:24.561 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:39:24.561 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:24.561 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:24.561 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:24.561 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 62271 2024-08-17T17:39:24.561 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.561 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:24.561 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.562 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:24.562 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.731037+0000", 2024-08-17T17:39:24.562 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.562 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.562 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.562 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:24.562 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.731037+0000", 2024-08-17T17:39:24.562 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:24.562 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.562 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.562 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:24.562 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.731034+0000", 2024-08-17T17:39:24.562 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999976 2024-08-17T17:39:24.562 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:24.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.731103+0000", 2024-08-17T17:39:24.563 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9072000000000005e-05 2024-08-17T17:39:24.563 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:24.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.731116+0000", 2024-08-17T17:39:24.563 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2462e-05 2024-08-17T17:39:24.563 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.563 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.563 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:24.563 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.731123+0000", 2024-08-17T17:39:24.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9040000000000003e-06 2024-08-17T17:39:24.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.564 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.564 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.732732+0000", 2024-08-17T17:39:24.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016097310000000001 2024-08-17T17:39:24.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.564 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:24.564 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.732936+0000", 2024-08-17T17:39:24.564 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00020385299999999999 2024-08-17T17:39:24.564 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.564 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.564 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 2,6", 2024-08-17T17:39:24.564 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.733093+0000", 2024-08-17T17:39:24.565 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000156794 2024-08-17T17:39:24.565 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.750037+0000", 2024-08-17T17:39:24.565 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.016943634999999999 2024-08-17T17:39:24.565 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:24.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:11.750051+0000", 2024-08-17T17:39:24.565 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4279e-05 2024-08-17T17:39:24.565 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.565 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.565 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:24.565 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.755779+0000", 2024-08-17T17:39:24.566 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0057280529999999 2024-08-17T17:39:24.566 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:24.566 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:24.566 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:24.566 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.755798+0000", 2024-08-17T17:39:24.566 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.8535000000000001e-05 2024-08-17T17:39:24.566 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.566 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.566 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.566 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:24.566 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:24.566 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:39:24.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.568 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:24.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:24.669 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:24.694 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:24.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:24.796 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:24.825 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:24.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:24.927 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:24.955 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:24.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:24.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:25.057 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:25.088 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:25.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:25.190 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:25.214 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:25.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:25.315 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:25.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.347 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:25.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:25.350 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:25.364 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:39:25.448 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:25.475 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:25.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:25.577 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:25.603 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:25.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:25.680 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:25.705 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:25.732 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:25.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:25.834 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:25.865 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:25.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:25.967 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:25.989 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:25.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:25.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:25 smithi063 ceph-mon[114756]: pgmap v2249: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 246 op/s; 13665/104865 objects degraded (13.031%); 3120/104865 objects misplaced (2.975%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:39:26.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:25 smithi063 ceph-mon[109843]: pgmap v2249: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 246 op/s; 13665/104865 objects degraded (13.031%); 3120/104865 objects misplaced (2.975%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:39:26.092 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:26.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:25 smithi162 ceph-mon[104668]: pgmap v2249: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 246 op/s; 13665/104865 objects degraded (13.031%); 3120/104865 objects misplaced (2.975%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:39:26.122 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:26.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:26.223 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:26.251 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:26.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:26.353 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:26.378 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:26.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:26.480 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:26.505 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:26.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:26.573 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:39:26.607 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:26.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.634 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:26.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:26.735 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:26.762 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:26.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:26.864 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:26.887 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:26.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:26.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:26.892 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:39:26.892 INFO:teuthology.orchestra.run.smithi063.stdout:{"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 13520/103557 objects degraded (13.056%), 4 pgs degraded, 5 pgs undersized","count":9},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2246,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1424,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916154,"num_in_osds":8,"osd_in_since":1723916113,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":112},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+clean+scrubbing","count":2},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":34519,"data_bytes":2261108401,"bytes_used":13954273280,"bytes_avail":754006011904,"bytes_total":767960285184,"degraded_objects":13520,"degraded_total":103557,"degraded_ratio":0.13055611885241944,"misplaced_objects":3120,"misplaced_total":103557,"misplaced_ratio":0.030128335119789101,"recovering_objects_per_sec":61,"recovering_bytes_per_sec":4062033,"recovering_keys_per_sec":0,"num_objects_recovered":620,"num_bytes_recovered":40632320,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":244},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi063.ultpqj","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.63:8443/","prometheus":"http://172.21.15.63:9283/"}},"servicemap":{"epoch":33,"modified":"2024-08-17T17:38:44.649232+0000","services":{}},"progress_events":{}} 2024-08-17T17:39:26.989 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:27.018 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:27.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:27.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:27.147 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:27.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:27.249 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:27.275 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:27.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.275 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:27.376 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:27.400 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:27.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:27.502 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:27.527 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:27.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:27.629 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:27.657 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:27.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:27.744 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:27.759 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:27.791 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:27.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:27.873 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:39:27.873 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:39:27.874 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:27.874 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_repop(client.46088.0:95449 11.0 e1424/1411)", 2024-08-17T17:39:27.874 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:39:27.871576+0000", 2024-08-17T17:39:27.874 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 2.1463000000000001e-05, 2024-08-17T17:39:27.874 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.4691000000000003e-05, 2024-08-17T17:39:27.874 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:27.874 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "queued for pg", 2024-08-17T17:39:27.874 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:27.874 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:27.874 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:27.874 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:27.871576+0000", 2024-08-17T17:39:27.874 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:27.874 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:27.875 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:27.875 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:27.875 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:27.871576+0000", 2024-08-17T17:39:27.875 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:27.875 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:27.875 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:27.875 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:27.875 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:27.871575+0000", 2024-08-17T17:39:27.875 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:39:27.875 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:27.875 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:27.876 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:27.876 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:27.871581+0000", 2024-08-17T17:39:27.876 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7900000000000002e-06 2024-08-17T17:39:27.876 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:27.876 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:27.876 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:27.876 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:27.871593+0000", 2024-08-17T17:39:27.876 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1403000000000001e-05 2024-08-17T17:39:27.876 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:27.876 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:27.876 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:27.876 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:27.871601+0000", 2024-08-17T17:39:27.876 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0059999999999992e-06 2024-08-17T17:39:27.877 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:27.877 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:27.877 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:27.877 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:27.877 INFO:teuthology.orchestra.run.smithi162.stdout: ], 2024-08-17T17:39:27.877 INFO:teuthology.orchestra.run.smithi162.stdout: "num_ops": 1 2024-08-17T17:39:27.877 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:39:27.892 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:27.923 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:27.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:27.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:28.024 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:27 smithi063 ceph-mon[114756]: pgmap v2250: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 244 op/s; 13520/103557 objects degraded (13.056%); 3120/103557 objects misplaced (3.013%); 3.9 MiB/s, 61 objects/s recovering 2024-08-17T17:39:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:27 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3610966942' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:39:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:27 smithi063 ceph-mon[109843]: pgmap v2250: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 244 op/s; 13520/103557 objects degraded (13.056%); 3120/103557 objects misplaced (3.013%); 3.9 MiB/s, 61 objects/s recovering 2024-08-17T17:39:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:27 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3610966942' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:39:28.050 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:28.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:28.097 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:27 smithi162 ceph-mon[104668]: pgmap v2250: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 244 op/s; 13520/103557 objects degraded (13.056%); 3120/103557 objects misplaced (3.013%); 3.9 MiB/s, 61 objects/s recovering 2024-08-17T17:39:28.098 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:27 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3610966942' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:39:28.152 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:28.177 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:28.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:28.236 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:39:28.236 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:39:28.278 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:28.304 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:28.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:28.406 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:28.441 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:28.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:28.544 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:28.569 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:28.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:28.670 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:28.675 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:39:28.696 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:28.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:28.799 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:28.830 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:28.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:28.932 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:28.957 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:28.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:28.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:29.014 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:29.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:28 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 13520/103557 objects degraded (13.056%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:29.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:28 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 13520/103557 objects degraded (13.056%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:29.059 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:29.088 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:29.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:29.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:28 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 13520/103557 objects degraded (13.056%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:29.190 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:29.213 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:29.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.214 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:29.314 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:29.339 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:29.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:29.441 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:29.467 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:29.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:29.568 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:29.594 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:29.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:29.696 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:29.723 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:29.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:29.763 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:39:29.825 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:29.849 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:29.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:29.950 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:29.980 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:29.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:29.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:29 smithi063 ceph-mon[109843]: pgmap v2251: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 238 op/s; 13195/101550 objects degraded (12.994%); 2966/101550 objects misplaced (2.921%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:29 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3298793276' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:39:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:29 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:39:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:29 smithi063 ceph-mon[114756]: pgmap v2251: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 238 op/s; 13195/101550 objects degraded (12.994%); 2966/101550 objects misplaced (2.921%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:30.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:29 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3298793276' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:39:30.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:29 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:39:30.081 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:29 smithi162 ceph-mon[104668]: pgmap v2251: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 238 op/s; 13195/101550 objects degraded (12.994%); 2966/101550 objects misplaced (2.921%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:29 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3298793276' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:39:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:29 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:39:30.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.105 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:30.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:30.206 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:30.234 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:30.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:30.336 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:30.361 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:30.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:30.462 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:30.487 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:30.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:30.589 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:30.612 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:30.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:30.714 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:30.735 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:30.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:30.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:30 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:30.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:30 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:39:30.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:30 smithi063 ceph-mon[114756]: osdmap e1425: 8 total, 8 up, 8 in 2024-08-17T17:39:30.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:30 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:30.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:30 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:39:30.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:30 smithi063 ceph-mon[109843]: osdmap e1425: 8 total, 8 up, 8 in 2024-08-17T17:39:30.837 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:30.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:30 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:30.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:30 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:39:30.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:30 smithi162 ceph-mon[104668]: osdmap e1425: 8 total, 8 up, 8 in 2024-08-17T17:39:30.862 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:30.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:30.964 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:30.990 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:30.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:30.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:31.066 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:39:31.066 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:31.089 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:39:31.089 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:31.092 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:31.093 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:31.109 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:39:31.109 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:31.120 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:31.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:31.127 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:39:31.127 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:31.145 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:39:31.145 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:31.165 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:39:31.166 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:31.186 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:39:31.186 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:31.191 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:39:31.191 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:39:31.192 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:39:31.192 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:39:31.192 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:39:31.210 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:39:31.210 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:31.222 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:31.253 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:31.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:31.355 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:31.378 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:31.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:31.480 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:31.513 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:31.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:31.615 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:31.643 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:31.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:31.745 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:31.774 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:31.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:31.824 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:31 smithi162 ceph-mon[104668]: pgmap v2253: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 240 op/s; 13195/101298 objects degraded (13.026%); 2966/101298 objects misplaced (2.928%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:31.825 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:31 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1407677109' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:31.825 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:31.856 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:39:31.879 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:31.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:31.930 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:31.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:32.026 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:31 smithi063 ceph-mon[114756]: pgmap v2253: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 240 op/s; 13195/101298 objects degraded (13.026%); 2966/101298 objects misplaced (2.928%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:32.026 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:31 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1407677109' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:32.026 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:32.026 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:31 smithi063 ceph-mon[109843]: pgmap v2253: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 240 op/s; 13195/101298 objects degraded (13.026%); 2966/101298 objects misplaced (2.928%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:32.026 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:31 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1407677109' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:32.027 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:32.031 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:32.062 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:32.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:32.127 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:32.164 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:32.188 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:32.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:32.289 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:32.313 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:32.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:32.414 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:32.438 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:32.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:32.539 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:32.546 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:39:32.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.571 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:32.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:32.672 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:32.702 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:32.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:32.804 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:32.837 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:32.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:32.940 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:32.962 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:32.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:32.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:33.064 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:33.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.089 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:33.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:33.190 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:33.216 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:33.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:33.318 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:33.345 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:33.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:33.447 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:33.470 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:33.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:33.495 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:39:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:39:33] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.51.0" 2024-08-17T17:39:33.572 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:33.597 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:33.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:33.699 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:33.728 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:33.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:33.830 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:33.848 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:33.852 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:33.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.853 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:33.954 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:33.977 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:33.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:33.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:33.990 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:39:33.992 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:39:33.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:33 smithi162 ceph-mon[104668]: pgmap v2254: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 249 op/s; 12712/97563 objects degraded (13.030%); 2802/97563 objects misplaced (2.872%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:33.997 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:33 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 12712/97563 objects degraded (13.030%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:34.002 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:34.002 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:33 smithi063 ceph-mon[114756]: pgmap v2254: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 249 op/s; 12712/97563 objects degraded (13.030%); 2802/97563 objects misplaced (2.872%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:34.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:33 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 12712/97563 objects degraded (13.030%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:34.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:33 smithi063 ceph-mon[109843]: pgmap v2254: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 249 op/s; 12712/97563 objects degraded (13.030%); 2802/97563 objects misplaced (2.872%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:34.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:33 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 12712/97563 objects degraded (13.030%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:34.079 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:34.102 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:34.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:34.203 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:34.228 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:34.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:34.330 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:34.352 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:34.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:34.454 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:34.480 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:34.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:34.582 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:34.604 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:34.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:34.681 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:39:34.706 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:34.722 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:39:34.747 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:34.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:34.849 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:34.882 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:34.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:34.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:34.983 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:35.003 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:35.010 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:35.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:35.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:34 smithi063 ceph-mon[109843]: from='client.46442 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:35.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:34 smithi063 ceph-mon[114756]: from='client.46442 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:35.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:34 smithi162 ceph-mon[104668]: from='client.46442 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:35.113 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:35.138 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:35.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:35.240 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:35.275 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:35.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:35.377 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:35.402 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:35.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.403 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:35.504 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:35.530 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:35.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:35.632 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:35.662 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:35.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:35.763 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:35.789 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:35.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:35.891 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:35.916 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:35.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:35.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:36.018 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:35 smithi063 ceph-mon[114756]: pgmap v2255: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 12712/97341 objects degraded (13.059%); 2802/97341 objects misplaced (2.879%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:35 smithi063 ceph-mon[109843]: pgmap v2255: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 12712/97341 objects degraded (13.059%); 2802/97341 objects misplaced (2.879%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:36.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.042 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:36.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:35 smithi162 ceph-mon[104668]: pgmap v2255: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 12712/97341 objects degraded (13.059%); 2802/97341 objects misplaced (2.879%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:36.143 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:36.169 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:36.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:36.189 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2254,"stamp":"2024-08-17T17:39:32.662948+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2130167473,"num_objects":32521,"num_object_clones":0,"num_object_copies":97563,"num_objects_missing_on_primary":2521,"num_objects_missing":2521,"num_objects_degraded":12712,"num_objects_misplaced":2802,"num_objects_unfound":0,"num_objects_dirty":32521,"num_whiteouts":0,"num_read":28,"num_read_kb":62,"num_write":96466,"num_write_kb":4124377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31247,"num_bytes_recovered":1894912733,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99065,"ondisk_log_size":99065,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":375,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13249144,"kb_used_data":7343976,"kb_used_omap":298,"kb_used_meta":5904725,"kb_avail":736712072,"statfs":{"total":767960285184,"available":754393161728,"internally_reserved":0,"allocated":7520231424,"data_stored":7498835717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":6046438929},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,4,2,1,4,2,3,4,3],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-163250176,"num_objects":-2491,"num_object_clones":0,"num_object_copies":-7473,"num_objects_missing_on_primary":-474,"num_objects_missing":-474,"num_objects_degraded":-953,"num_objects_misplaced":-318,"num_objects_unfound":0,"num_objects_dirty":-2491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2491,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":637,"num_bytes_recovered":41746432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002989"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":57,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.760784+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:39:29.760784+0000","last_peered":"2024-08-17T17:39:29.760784+0000","last_clean":"2024-08-17T17:39:29.760784+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:39:29.760784+0000","last_undegraded":"2024-08-17T17:39:29.760784+0000","last_fullsized":"2024-08-17T17:39:29.760784+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1439,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751239+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:39:18.751239+0000","last_peered":"2024-08-17T17:39:18.751239+0000","last_clean":"2024-08-17T17:39:18.751239+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:39:18.751239+0000","last_undegraded":"2024-08-17T17:39:18.751239+0000","last_fullsized":"2024-08-17T17:39:18.751239+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1754,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751602+0000","last_change":"2024-08-17T17:36:15.771150+0000","last_active":"2024-08-17T17:39:18.751602+0000","last_peered":"2024-08-17T17:39:18.751602+0000","last_clean":"2024-08-17T17:39:18.751602+0000","last_became_active":"2024-08-17T17:36:15.770520+0000","last_became_peered":"2024-08-17T17:36:15.770520+0000","last_unstale":"2024-08-17T17:39:18.751602+0000","last_undegraded":"2024-08-17T17:39:18.751602+0000","last_fullsized":"2024-08-17T17:39:18.751602+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:17.552796+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":89,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.734725+0000","last_change":"2024-08-17T17:36:12.756709+0000","last_active":"2024-08-17T17:39:18.734725+0000","last_peered":"2024-08-17T17:39:18.734725+0000","last_clean":"2024-08-17T17:39:18.734725+0000","last_became_active":"2024-08-17T17:36:12.756385+0000","last_became_peered":"2024-08-17T17:36:12.756385+0000","last_unstale":"2024-08-17T17:39:18.734725+0000","last_undegraded":"2024-08-17T17:39:18.734725+0000","last_fullsized":"2024-08-17T17:39:18.734725+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:21.420282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":141,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.792660+0000","last_change":"2024-08-17T17:36:12.762800+0000","last_active":"2024-08-17T17:39:29.792660+0000","last_peered":"2024-08-17T17:39:29.792660+0000","last_clean":"2024-08-17T17:39:29.792660+0000","last_became_active":"2024-08-17T17:36:12.759977+0000","last_became_peered":"2024-08-17T17:36:12.759977+0000","last_unstale":"2024-08-17T17:39:29.792660+0000","last_undegraded":"2024-08-17T17:39:29.792660+0000","last_fullsized":"2024-08-17T17:39:29.792660+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:02.162681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1634,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.734997+0000","last_change":"2024-08-17T17:35:56.281561+0000","last_active":"2024-08-17T17:39:18.734997+0000","last_peered":"2024-08-17T17:39:18.734997+0000","last_clean":"2024-08-17T17:39:18.734997+0000","last_became_active":"2024-08-17T17:35:56.278638+0000","last_became_peered":"2024-08-17T17:35:56.278638+0000","last_unstale":"2024-08-17T17:39:18.734997+0000","last_undegraded":"2024-08-17T17:39:18.734997+0000","last_fullsized":"2024-08-17T17:39:18.734997+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1373,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.810829+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:39:18.810829+0000","last_peered":"2024-08-17T17:39:18.810829+0000","last_clean":"2024-08-17T17:39:18.810829+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:39:18.810829+0000","last_undegraded":"2024-08-17T17:39:18.810829+0000","last_fullsized":"2024-08-17T17:39:18.810829+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":49,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751184+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:39:18.751184+0000","last_peered":"2024-08-17T17:39:18.751184+0000","last_clean":"2024-08-17T17:39:18.751184+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:39:18.751184+0000","last_undegraded":"2024-08-17T17:39:18.751184+0000","last_fullsized":"2024-08-17T17:39:18.751184+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1418,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.810354+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:39:18.810354+0000","last_peered":"2024-08-17T17:39:18.810354+0000","last_clean":"2024-08-17T17:39:18.810354+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:39:18.810354+0000","last_undegraded":"2024-08-17T17:39:18.810354+0000","last_fullsized":"2024-08-17T17:39:18.810354+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1426,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751536+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:39:18.751536+0000","last_peered":"2024-08-17T17:39:18.751536+0000","last_clean":"2024-08-17T17:39:18.751536+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:39:18.751536+0000","last_undegraded":"2024-08-17T17:39:18.751536+0000","last_fullsized":"2024-08-17T17:39:18.751536+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":716,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.734594+0000","last_change":"2024-08-17T17:36:15.773315+0000","last_active":"2024-08-17T17:39:18.734594+0000","last_peered":"2024-08-17T17:39:18.734594+0000","last_clean":"2024-08-17T17:39:18.734594+0000","last_became_active":"2024-08-17T17:36:15.772961+0000","last_became_peered":"2024-08-17T17:36:15.772961+0000","last_unstale":"2024-08-17T17:39:18.734594+0000","last_undegraded":"2024-08-17T17:39:18.734594+0000","last_fullsized":"2024-08-17T17:39:18.734594+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:10.190221+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":50,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.760855+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:39:29.760855+0000","last_peered":"2024-08-17T17:39:29.760855+0000","last_clean":"2024-08-17T17:39:29.760855+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:39:29.760855+0000","last_undegraded":"2024-08-17T17:39:29.760855+0000","last_fullsized":"2024-08-17T17:39:29.760855+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":141,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.793578+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:39:29.793578+0000","last_peered":"2024-08-17T17:39:29.793578+0000","last_clean":"2024-08-17T17:39:29.793578+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:39:29.793578+0000","last_undegraded":"2024-08-17T17:39:29.793578+0000","last_fullsized":"2024-08-17T17:39:29.793578+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1507,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.741254+0000","last_change":"2024-08-17T17:36:24.225739+0000","last_active":"2024-08-17T17:39:18.741254+0000","last_peered":"2024-08-17T17:39:18.741254+0000","last_clean":"2024-08-17T17:39:18.741254+0000","last_became_active":"2024-08-17T17:36:24.225250+0000","last_became_peered":"2024-08-17T17:36:24.225250+0000","last_unstale":"2024-08-17T17:39:18.741254+0000","last_undegraded":"2024-08-17T17:39:18.741254+0000","last_fullsized":"2024-08-17T17:39:18.741254+0000","mapping_epoch":1393,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1394,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.290656+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1446,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751599+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:39:18.751599+0000","last_peered":"2024-08-17T17:39:18.751599+0000","last_clean":"2024-08-17T17:39:18.751599+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:39:18.751599+0000","last_undegraded":"2024-08-17T17:39:18.751599+0000","last_fullsized":"2024-08-17T17:39:18.751599+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1853,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751877+0000","last_change":"2024-08-17T17:36:12.752970+0000","last_active":"2024-08-17T17:39:18.751877+0000","last_peered":"2024-08-17T17:39:18.751877+0000","last_clean":"2024-08-17T17:39:18.751877+0000","last_became_active":"2024-08-17T17:36:12.752311+0000","last_became_peered":"2024-08-17T17:36:12.752311+0000","last_unstale":"2024-08-17T17:39:18.751877+0000","last_undegraded":"2024-08-17T17:39:18.751877+0000","last_fullsized":"2024-08-17T17:39:18.751877+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:38.131419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1801,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.810068+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:39:18.810068+0000","last_peered":"2024-08-17T17:39:18.810068+0000","last_clean":"2024-08-17T17:39:18.810068+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:39:18.810068+0000","last_undegraded":"2024-08-17T17:39:18.810068+0000","last_fullsized":"2024-08-17T17:39:18.810068+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1511,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.881425+0000","last_change":"2024-08-17T17:36:18.750494+0000","last_active":"2024-08-17T17:39:18.881425+0000","last_peered":"2024-08-17T17:39:18.881425+0000","last_clean":"2024-08-17T17:39:18.881425+0000","last_became_active":"2024-08-17T17:36:18.750092+0000","last_became_peered":"2024-08-17T17:36:18.750092+0000","last_unstale":"2024-08-17T17:39:18.881425+0000","last_undegraded":"2024-08-17T17:39:18.881425+0000","last_fullsized":"2024-08-17T17:39:18.881425+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:36.599727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1143,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.762549+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:39:29.762549+0000","last_peered":"2024-08-17T17:39:29.762549+0000","last_clean":"2024-08-17T17:39:29.762549+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:39:29.762549+0000","last_undegraded":"2024-08-17T17:39:29.762549+0000","last_fullsized":"2024-08-17T17:39:29.762549+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":968,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773514+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:39:29.773514+0000","last_peered":"2024-08-17T17:39:29.773514+0000","last_clean":"2024-08-17T17:39:29.773514+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:39:29.773514+0000","last_undegraded":"2024-08-17T17:39:29.773514+0000","last_fullsized":"2024-08-17T17:39:29.773514+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":50,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.760692+0000","last_change":"2024-08-17T17:37:11.075505+0000","last_active":"2024-08-17T17:39:29.760692+0000","last_peered":"2024-08-17T17:39:29.760692+0000","last_clean":"2024-08-17T17:39:29.760692+0000","last_became_active":"2024-08-17T17:37:11.075131+0000","last_became_peered":"2024-08-17T17:37:11.075131+0000","last_unstale":"2024-08-17T17:39:29.760692+0000","last_undegraded":"2024-08-17T17:39:29.760692+0000","last_fullsized":"2024-08-17T17:39:29.760692+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:39.672054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1280,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.734888+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:39:18.734888+0000","last_peered":"2024-08-17T17:39:18.734888+0000","last_clean":"2024-08-17T17:39:18.734888+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:39:18.734888+0000","last_undegraded":"2024-08-17T17:39:18.734888+0000","last_fullsized":"2024-08-17T17:39:18.734888+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1657,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751285+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:39:18.751285+0000","last_peered":"2024-08-17T17:39:18.751285+0000","last_clean":"2024-08-17T17:39:18.751285+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:39:18.751285+0000","last_undegraded":"2024-08-17T17:39:18.751285+0000","last_fullsized":"2024-08-17T17:39:18.751285+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":95,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.881020+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:39:18.881020+0000","last_peered":"2024-08-17T17:39:18.881020+0000","last_clean":"2024-08-17T17:39:18.881020+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:39:18.881020+0000","last_undegraded":"2024-08-17T17:39:18.881020+0000","last_fullsized":"2024-08-17T17:39:18.881020+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1255,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.810618+0000","last_change":"2024-08-17T17:36:12.755873+0000","last_active":"2024-08-17T17:39:18.810618+0000","last_peered":"2024-08-17T17:39:18.810618+0000","last_clean":"2024-08-17T17:39:18.810618+0000","last_became_active":"2024-08-17T17:36:12.755366+0000","last_became_peered":"2024-08-17T17:36:12.755366+0000","last_unstale":"2024-08-17T17:39:18.810618+0000","last_undegraded":"2024-08-17T17:39:18.810618+0000","last_fullsized":"2024-08-17T17:39:18.810618+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:36.523214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1608,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.859746+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:39:18.859746+0000","last_peered":"2024-08-17T17:39:18.859746+0000","last_clean":"2024-08-17T17:39:18.859746+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:39:18.859746+0000","last_undegraded":"2024-08-17T17:39:18.859746+0000","last_fullsized":"2024-08-17T17:39:18.859746+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":792,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.741377+0000","last_change":"2024-08-17T17:35:55.200519+0000","last_active":"2024-08-17T17:39:18.741377+0000","last_peered":"2024-08-17T17:39:18.741377+0000","last_clean":"2024-08-17T17:39:18.741377+0000","last_became_active":"2024-08-17T17:35:55.200230+0000","last_became_peered":"2024-08-17T17:35:55.200230+0000","last_unstale":"2024-08-17T17:39:18.741377+0000","last_undegraded":"2024-08-17T17:39:18.741377+0000","last_fullsized":"2024-08-17T17:39:18.741377+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:33.454851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1432,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773743+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:39:29.773743+0000","last_peered":"2024-08-17T17:39:29.773743+0000","last_clean":"2024-08-17T17:39:29.773743+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:39:29.773743+0000","last_undegraded":"2024-08-17T17:39:29.773743+0000","last_fullsized":"2024-08-17T17:39:29.773743+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1714,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.793288+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:39:29.793288+0000","last_peered":"2024-08-17T17:39:29.793288+0000","last_clean":"2024-08-17T17:39:29.793288+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:39:29.793288+0000","last_undegraded":"2024-08-17T17:39:29.793288+0000","last_fullsized":"2024-08-17T17:39:29.793288+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:22.077229+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":259,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.883180+0000","last_change":"2024-08-17T17:35:33.145476+0000","last_active":"2024-08-17T17:39:18.883180+0000","last_peered":"2024-08-17T17:39:18.883180+0000","last_clean":"2024-08-17T17:39:18.883180+0000","last_became_active":"2024-08-17T17:35:33.145125+0000","last_became_peered":"2024-08-17T17:35:33.145125+0000","last_unstale":"2024-08-17T17:39:18.883180+0000","last_undegraded":"2024-08-17T17:39:18.883180+0000","last_fullsized":"2024-08-17T17:39:18.883180+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:45.298064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":936,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.766510+0000","last_change":"2024-08-17T17:35:56.248612+0000","last_active":"2024-08-17T17:39:18.766510+0000","last_peered":"2024-08-17T17:39:18.766510+0000","last_clean":"2024-08-17T17:39:18.766510+0000","last_became_active":"2024-08-17T17:35:56.244364+0000","last_became_peered":"2024-08-17T17:35:56.244364+0000","last_unstale":"2024-08-17T17:39:18.766510+0000","last_undegraded":"2024-08-17T17:39:18.766510+0000","last_fullsized":"2024-08-17T17:39:18.766510+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:41.064165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":876,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.810338+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:39:18.810338+0000","last_peered":"2024-08-17T17:39:18.810338+0000","last_clean":"2024-08-17T17:39:18.810338+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:39:18.810338+0000","last_undegraded":"2024-08-17T17:39:18.810338+0000","last_fullsized":"2024-08-17T17:39:18.810338+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1411,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751767+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:39:18.751767+0000","last_peered":"2024-08-17T17:39:18.751767+0000","last_clean":"2024-08-17T17:39:18.751767+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:39:18.751767+0000","last_undegraded":"2024-08-17T17:39:18.751767+0000","last_fullsized":"2024-08-17T17:39:18.751767+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1430,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751505+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:39:18.751505+0000","last_peered":"2024-08-17T17:39:18.751505+0000","last_clean":"2024-08-17T17:39:18.751505+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:39:18.751505+0000","last_undegraded":"2024-08-17T17:39:18.751505+0000","last_fullsized":"2024-08-17T17:39:18.751505+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":936,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.761539+0000","last_change":"2024-08-17T17:36:15.774881+0000","last_active":"2024-08-17T17:39:29.761539+0000","last_peered":"2024-08-17T17:39:29.761539+0000","last_clean":"2024-08-17T17:39:29.761539+0000","last_became_active":"2024-08-17T17:36:15.774435+0000","last_became_peered":"2024-08-17T17:36:15.774435+0000","last_unstale":"2024-08-17T17:39:29.761539+0000","last_undegraded":"2024-08-17T17:39:29.761539+0000","last_fullsized":"2024-08-17T17:39:29.761539+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:52.015854+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"1425'6209","reported_seq":14390,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:32.123959+0000","last_change":"2024-08-17T17:38:37.110784+0000","last_active":"2024-08-17T17:39:32.123959+0000","last_peered":"2024-08-17T17:39:32.123959+0000","last_clean":"2024-08-17T17:39:32.123959+0000","last_became_active":"2024-08-17T17:38:37.088380+0000","last_became_peered":"2024-08-17T17:38:37.088380+0000","last_unstale":"2024-08-17T17:39:32.123959+0000","last_undegraded":"2024-08-17T17:39:32.123959+0000","last_fullsized":"2024-08-17T17:39:32.123959+0000","mapping_epoch":1419,"log_start":"1373'200","ondisk_log_start":"1373'200","created":1357,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6009,"log_dups_size":0,"ondisk_log_size":6009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:08.647903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136118272,"num_objects":2077,"num_object_clones":0,"num_object_copies":6231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6209,"num_write_kb":265152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4009,"num_bytes_recovered":199753728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1379,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:27.430577+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:39:27.430577+0000","last_peered":"2024-08-17T17:39:27.430577+0000","last_clean":"2024-08-17T17:39:27.430577+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:39:27.430577+0000","last_undegraded":"2024-08-17T17:39:27.430577+0000","last_fullsized":"2024-08-17T17:39:27.430577+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1274,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.734722+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:39:18.734722+0000","last_peered":"2024-08-17T17:39:18.734722+0000","last_clean":"2024-08-17T17:39:18.734722+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:39:18.734722+0000","last_undegraded":"2024-08-17T17:39:18.734722+0000","last_fullsized":"2024-08-17T17:39:18.734722+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":329,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.762649+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:39:29.762649+0000","last_peered":"2024-08-17T17:39:29.762649+0000","last_clean":"2024-08-17T17:39:29.762649+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:39:29.762649+0000","last_undegraded":"2024-08-17T17:39:29.762649+0000","last_fullsized":"2024-08-17T17:39:29.762649+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1595,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.883257+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:39:18.883257+0000","last_peered":"2024-08-17T17:39:18.883257+0000","last_clean":"2024-08-17T17:39:18.883257+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:39:18.883257+0000","last_undegraded":"2024-08-17T17:39:18.883257+0000","last_fullsized":"2024-08-17T17:39:18.883257+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:25.751871+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.f","version":"1425'6138","reported_seq":6292,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:30.632581+0000","last_change":"2024-08-17T17:35:34.179168+0000","last_active":"2024-08-17T17:39:30.632581+0000","last_peered":"2024-08-17T17:39:30.632581+0000","last_clean":"2024-08-17T17:39:30.632581+0000","last_became_active":"2024-08-17T17:35:34.133493+0000","last_became_peered":"2024-08-17T17:35:34.133493+0000","last_unstale":"2024-08-17T17:39:30.632581+0000","last_undegraded":"2024-08-17T17:39:30.632581+0000","last_fullsized":"2024-08-17T17:39:30.632581+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6138,"log_dups_size":0,"ondisk_log_size":6138,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:24:11.188566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135397376,"num_objects":2066,"num_object_clones":0,"num_object_copies":6198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6138,"num_write_kb":262528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":102,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.881157+0000","last_change":"2024-08-17T17:37:11.079712+0000","last_active":"2024-08-17T17:39:18.881157+0000","last_peered":"2024-08-17T17:39:18.881157+0000","last_clean":"2024-08-17T17:39:18.881157+0000","last_became_active":"2024-08-17T17:37:11.079253+0000","last_became_peered":"2024-08-17T17:37:11.079253+0000","last_unstale":"2024-08-17T17:39:18.881157+0000","last_undegraded":"2024-08-17T17:39:18.881157+0000","last_fullsized":"2024-08-17T17:39:18.881157+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:53.405657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":919,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.733946+0000","last_change":"2024-08-17T17:35:56.223396+0000","last_active":"2024-08-17T17:39:18.733946+0000","last_peered":"2024-08-17T17:39:18.733946+0000","last_clean":"2024-08-17T17:39:18.733946+0000","last_became_active":"2024-08-17T17:35:56.212413+0000","last_became_peered":"2024-08-17T17:35:56.212413+0000","last_unstale":"2024-08-17T17:39:18.733946+0000","last_undegraded":"2024-08-17T17:39:18.733946+0000","last_fullsized":"2024-08-17T17:39:18.733946+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:44.185174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1639,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.761039+0000","last_change":"2024-08-17T17:35:33.136068+0000","last_active":"2024-08-17T17:39:29.761039+0000","last_peered":"2024-08-17T17:39:29.761039+0000","last_clean":"2024-08-17T17:39:29.761039+0000","last_became_active":"2024-08-17T17:35:33.135591+0000","last_became_peered":"2024-08-17T17:35:33.135591+0000","last_unstale":"2024-08-17T17:39:29.761039+0000","last_undegraded":"2024-08-17T17:39:29.761039+0000","last_fullsized":"2024-08-17T17:39:29.761039+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:52.252561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1785,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:27.430779+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:39:27.430779+0000","last_peered":"2024-08-17T17:39:27.430779+0000","last_clean":"2024-08-17T17:39:27.430779+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:39:27.430779+0000","last_undegraded":"2024-08-17T17:39:27.430779+0000","last_fullsized":"2024-08-17T17:39:27.430779+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"1425'6011","reported_seq":6150,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:30.334034+0000","last_change":"2024-08-17T17:35:14.861823+0000","last_active":"2024-08-17T17:39:30.334034+0000","last_peered":"2024-08-17T17:39:30.334034+0000","last_clean":"2024-08-17T17:39:30.334034+0000","last_became_active":"2024-08-17T17:35:14.854109+0000","last_became_peered":"2024-08-17T17:35:14.854109+0000","last_unstale":"2024-08-17T17:39:30.334034+0000","last_undegraded":"2024-08-17T17:39:30.334034+0000","last_fullsized":"2024-08-17T17:39:30.334034+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6011,"log_dups_size":0,"ondisk_log_size":6011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:46:40.778197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133365760,"num_objects":2035,"num_object_clones":0,"num_object_copies":6105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6011,"num_write_kb":257472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":997,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.810396+0000","last_change":"2024-08-17T17:35:33.138467+0000","last_active":"2024-08-17T17:39:18.810396+0000","last_peered":"2024-08-17T17:39:18.810396+0000","last_clean":"2024-08-17T17:39:18.810396+0000","last_became_active":"2024-08-17T17:35:33.138174+0000","last_became_peered":"2024-08-17T17:35:33.138174+0000","last_unstale":"2024-08-17T17:39:18.810396+0000","last_undegraded":"2024-08-17T17:39:18.810396+0000","last_fullsized":"2024-08-17T17:39:18.810396+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:08.860439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1381,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.859821+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:39:18.859821+0000","last_peered":"2024-08-17T17:39:18.859821+0000","last_clean":"2024-08-17T17:39:18.859821+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:39:18.859821+0000","last_undegraded":"2024-08-17T17:39:18.859821+0000","last_fullsized":"2024-08-17T17:39:18.859821+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":135,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.793522+0000","last_change":"2024-08-17T17:35:14.854881+0000","last_active":"2024-08-17T17:39:29.793522+0000","last_peered":"2024-08-17T17:39:29.793522+0000","last_clean":"2024-08-17T17:39:29.793522+0000","last_became_active":"2024-08-17T17:35:14.854121+0000","last_became_peered":"2024-08-17T17:35:14.854121+0000","last_unstale":"2024-08-17T17:39:29.793522+0000","last_undegraded":"2024-08-17T17:39:29.793522+0000","last_fullsized":"2024-08-17T17:39:29.793522+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.863624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":938,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.734835+0000","last_change":"2024-08-17T17:36:15.773050+0000","last_active":"2024-08-17T17:39:18.734835+0000","last_peered":"2024-08-17T17:39:18.734835+0000","last_clean":"2024-08-17T17:39:18.734835+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:39:18.734835+0000","last_undegraded":"2024-08-17T17:39:18.734835+0000","last_fullsized":"2024-08-17T17:39:18.734835+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:39.212195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:39.212195+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"1425'6003","reported_seq":11848,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:30.426075+0000","last_change":"2024-08-17T17:37:21.409979+0000","last_active":"2024-08-17T17:39:30.426075+0000","last_peered":"2024-08-17T17:39:30.426075+0000","last_clean":"2024-08-17T17:39:30.426075+0000","last_became_active":"2024-08-17T17:37:21.398111+0000","last_became_peered":"2024-08-17T17:37:21.398111+0000","last_unstale":"2024-08-17T17:39:30.426075+0000","last_undegraded":"2024-08-17T17:39:30.426075+0000","last_fullsized":"2024-08-17T17:39:30.426075+0000","mapping_epoch":1407,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6003,"log_dups_size":0,"ondisk_log_size":6003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:19.299395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132907032,"num_objects":2029,"num_object_clones":0,"num_object_copies":6087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2029,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6003,"num_write_kb":256961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2850,"num_bytes_recovered":168820736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1040,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.810596+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:39:18.810596+0000","last_peered":"2024-08-17T17:39:18.810596+0000","last_clean":"2024-08-17T17:39:18.810596+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:39:18.810596+0000","last_undegraded":"2024-08-17T17:39:18.810596+0000","last_fullsized":"2024-08-17T17:39:18.810596+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":228,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.761974+0000","last_change":"2024-08-17T17:35:33.145351+0000","last_active":"2024-08-17T17:39:29.761974+0000","last_peered":"2024-08-17T17:39:29.761974+0000","last_clean":"2024-08-17T17:39:29.761974+0000","last_became_active":"2024-08-17T17:35:33.145040+0000","last_became_peered":"2024-08-17T17:35:33.145040+0000","last_unstale":"2024-08-17T17:39:29.761974+0000","last_undegraded":"2024-08-17T17:39:29.761974+0000","last_fullsized":"2024-08-17T17:39:29.761974+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:02.124000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1449,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.810876+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:39:18.810876+0000","last_peered":"2024-08-17T17:39:18.810876+0000","last_clean":"2024-08-17T17:39:18.810876+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:39:18.810876+0000","last_undegraded":"2024-08-17T17:39:18.810876+0000","last_fullsized":"2024-08-17T17:39:18.810876+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":921,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.734655+0000","last_change":"2024-08-17T17:36:15.769890+0000","last_active":"2024-08-17T17:39:18.734655+0000","last_peered":"2024-08-17T17:39:18.734655+0000","last_clean":"2024-08-17T17:39:18.734655+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:39:18.734655+0000","last_undegraded":"2024-08-17T17:39:18.734655+0000","last_fullsized":"2024-08-17T17:39:18.734655+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:33.034306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:33.034306+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":4,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.b","version":"1424'5866","reported_seq":13669,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:28.529840+0000","last_change":"2024-08-17T17:38:07.888612+0000","last_active":"2024-08-17T17:39:28.529840+0000","last_peered":"2024-08-17T17:39:28.529840+0000","last_clean":"2024-08-17T17:39:28.529840+0000","last_became_active":"2024-08-17T17:38:07.875196+0000","last_became_peered":"2024-08-17T17:38:07.875196+0000","last_unstale":"2024-08-17T17:39:28.529840+0000","last_undegraded":"2024-08-17T17:39:28.529840+0000","last_fullsized":"2024-08-17T17:39:28.529840+0000","mapping_epoch":1415,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5866,"log_dups_size":0,"ondisk_log_size":5866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:21.658004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136839168,"num_objects":2088,"num_object_clones":0,"num_object_copies":6264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5866,"num_write_kb":254528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3836,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1490,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.793646+0000","last_change":"2024-08-17T17:36:12.759743+0000","last_active":"2024-08-17T17:39:29.793646+0000","last_peered":"2024-08-17T17:39:29.793646+0000","last_clean":"2024-08-17T17:39:29.793646+0000","last_became_active":"2024-08-17T17:36:12.759334+0000","last_became_peered":"2024-08-17T17:36:12.759334+0000","last_unstale":"2024-08-17T17:39:29.793646+0000","last_undegraded":"2024-08-17T17:39:29.793646+0000","last_fullsized":"2024-08-17T17:39:29.793646+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:04.524384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1413,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:27.430709+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:39:27.430709+0000","last_peered":"2024-08-17T17:39:27.430709+0000","last_clean":"2024-08-17T17:39:27.430709+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:39:27.430709+0000","last_undegraded":"2024-08-17T17:39:27.430709+0000","last_fullsized":"2024-08-17T17:39:27.430709+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1455,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.812488+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:39:18.812488+0000","last_peered":"2024-08-17T17:39:18.812488+0000","last_clean":"2024-08-17T17:39:18.812488+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:39:18.812488+0000","last_undegraded":"2024-08-17T17:39:18.812488+0000","last_fullsized":"2024-08-17T17:39:18.812488+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1771,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:22.447198+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:39:22.447198+0000","last_peered":"2024-08-17T17:39:22.447198+0000","last_clean":"2024-08-17T17:39:22.447198+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:39:22.447198+0000","last_undegraded":"2024-08-17T17:39:22.447198+0000","last_fullsized":"2024-08-17T17:39:22.447198+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"1425'6114","reported_seq":12287,"reported_epoch":1425,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:39:32.213465+0000","last_change":"2024-08-17T17:38:05.991715+0000","last_active":"2024-08-17T17:39:32.213465+0000","last_peered":"2024-08-17T17:39:32.213465+0000","last_clean":"2024-08-17T17:35:54.187688+0000","last_became_active":"2024-08-17T17:36:22.904523+0000","last_became_peered":"2024-08-17T17:36:22.904523+0000","last_unstale":"2024-08-17T17:39:32.213465+0000","last_undegraded":"2024-08-17T17:39:32.213465+0000","last_fullsized":"2024-08-17T17:36:13.740491+0000","mapping_epoch":1383,"log_start":"1373'300","ondisk_log_start":"1373'300","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5814,"log_dups_size":0,"ondisk_log_size":5814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:57.525855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135921664,"num_objects":2074,"num_object_clones":0,"num_object_copies":6222,"num_objects_missing_on_primary":859,"num_objects_missing":859,"num_objects_degraded":0,"num_objects_misplaced":2802,"num_objects_unfound":0,"num_objects_dirty":2074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6114,"num_write_kb":262016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2025,"num_bytes_recovered":132710400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":859},{"shards":"2,4,5,6","objects":7},{"shards":"2,5,6","objects":1077},{"shards":"5,6","objects":131}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1089,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773958+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:39:29.773958+0000","last_peered":"2024-08-17T17:39:29.773958+0000","last_clean":"2024-08-17T17:39:29.773958+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:39:29.773958+0000","last_undegraded":"2024-08-17T17:39:29.773958+0000","last_fullsized":"2024-08-17T17:39:29.773958+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1401,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.792599+0000","last_change":"2024-08-17T17:35:56.229622+0000","last_active":"2024-08-17T17:39:29.792599+0000","last_peered":"2024-08-17T17:39:29.792599+0000","last_clean":"2024-08-17T17:39:29.792599+0000","last_became_active":"2024-08-17T17:35:56.229324+0000","last_became_peered":"2024-08-17T17:35:56.229324+0000","last_unstale":"2024-08-17T17:39:29.792599+0000","last_undegraded":"2024-08-17T17:39:29.792599+0000","last_fullsized":"2024-08-17T17:39:29.792599+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:13.980316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2005,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.792759+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:39:29.792759+0000","last_peered":"2024-08-17T17:39:29.792759+0000","last_clean":"2024-08-17T17:39:29.792759+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:39:29.792759+0000","last_undegraded":"2024-08-17T17:39:29.792759+0000","last_fullsized":"2024-08-17T17:39:29.792759+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1585,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751383+0000","last_change":"2024-08-17T17:35:33.137074+0000","last_active":"2024-08-17T17:39:18.751383+0000","last_peered":"2024-08-17T17:39:18.751383+0000","last_clean":"2024-08-17T17:39:18.751383+0000","last_became_active":"2024-08-17T17:35:33.136759+0000","last_became_peered":"2024-08-17T17:35:33.136759+0000","last_unstale":"2024-08-17T17:39:18.751383+0000","last_undegraded":"2024-08-17T17:39:18.751383+0000","last_fullsized":"2024-08-17T17:39:18.751383+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:37.263639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:37.263639+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:14.599780+0000","scrub_duration":0.0016181800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"1425'6088","reported_seq":11618,"reported_epoch":1425,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:30.630916+0000","last_change":"2024-08-17T17:36:23.019129+0000","last_active":"2024-08-17T17:39:30.630916+0000","last_peered":"2024-08-17T17:39:30.630916+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:23.013527+0000","last_became_peered":"2024-08-17T17:36:23.013527+0000","last_unstale":"2024-08-17T17:39:30.630916+0000","last_undegraded":"2024-08-17T17:36:13.874715+0000","last_fullsized":"2024-08-17T17:36:13.738828+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6088,"log_dups_size":0,"ondisk_log_size":6088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131858432,"num_objects":2012,"num_object_clones":0,"num_object_copies":6036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3459,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6088,"num_write_kb":259200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2039,"num_bytes_recovered":133627904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[0,6],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":837},{"shards":"0,1,6","objects":1048},{"shards":"0,6","objects":-610}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1194,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.762732+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:39:29.762732+0000","last_peered":"2024-08-17T17:39:29.762732+0000","last_clean":"2024-08-17T17:39:29.762732+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:39:29.762732+0000","last_undegraded":"2024-08-17T17:39:29.762732+0000","last_fullsized":"2024-08-17T17:39:29.762732+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":49,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751103+0000","last_change":"2024-08-17T17:36:17.774773+0000","last_active":"2024-08-17T17:39:18.751103+0000","last_peered":"2024-08-17T17:39:18.751103+0000","last_clean":"2024-08-17T17:39:18.751103+0000","last_became_active":"2024-08-17T17:36:17.774468+0000","last_became_peered":"2024-08-17T17:36:17.774468+0000","last_unstale":"2024-08-17T17:39:18.751103+0000","last_undegraded":"2024-08-17T17:39:18.751103+0000","last_fullsized":"2024-08-17T17:39:18.751103+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:54.231747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1532,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.760634+0000","last_change":"2024-08-17T17:35:33.137633+0000","last_active":"2024-08-17T17:39:29.760634+0000","last_peered":"2024-08-17T17:39:29.760634+0000","last_clean":"2024-08-17T17:39:29.760634+0000","last_became_active":"2024-08-17T17:35:33.137323+0000","last_became_peered":"2024-08-17T17:35:33.137323+0000","last_unstale":"2024-08-17T17:39:29.760634+0000","last_undegraded":"2024-08-17T17:39:29.760634+0000","last_fullsized":"2024-08-17T17:39:29.760634+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:03.359155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":900,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.733950+0000","last_change":"2024-08-17T17:35:56.264626+0000","last_active":"2024-08-17T17:39:18.733950+0000","last_peered":"2024-08-17T17:39:18.733950+0000","last_clean":"2024-08-17T17:39:18.733950+0000","last_became_active":"2024-08-17T17:35:56.264265+0000","last_became_peered":"2024-08-17T17:35:56.264265+0000","last_unstale":"2024-08-17T17:39:18.733950+0000","last_undegraded":"2024-08-17T17:39:18.733950+0000","last_fullsized":"2024-08-17T17:39:18.733950+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:20.754326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:20.754326+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"1425'6006","reported_seq":12090,"reported_epoch":1425,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:32.226733+0000","last_change":"2024-08-17T17:37:11.134558+0000","last_active":"2024-08-17T17:39:32.226733+0000","last_peered":"2024-08-17T17:39:32.226733+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:20.105110+0000","last_became_peered":"2024-08-17T17:36:20.105110+0000","last_unstale":"2024-08-17T17:39:32.226733+0000","last_undegraded":"2024-08-17T17:36:19.999272+0000","last_fullsized":"2024-08-17T17:36:13.740520+0000","mapping_epoch":1383,"log_start":"1373'200","ondisk_log_start":"1373'200","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5806,"log_dups_size":0,"ondisk_log_size":5806,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:45.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131596288,"num_objects":2008,"num_object_clones":0,"num_object_copies":6024,"num_objects_missing_on_primary":815,"num_objects_missing":815,"num_objects_degraded":2669,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6006,"num_write_kb":256448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1999,"num_bytes_recovered":131006464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":815},{"shards":"1,5,6","objects":1039},{"shards":"5,6","objects":154}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1253,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.734303+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:39:18.734303+0000","last_peered":"2024-08-17T17:39:18.734303+0000","last_clean":"2024-08-17T17:39:18.734303+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:39:18.734303+0000","last_undegraded":"2024-08-17T17:39:18.734303+0000","last_fullsized":"2024-08-17T17:39:18.734303+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1793,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.812438+0000","last_change":"2024-08-17T17:36:12.758005+0000","last_active":"2024-08-17T17:39:18.812438+0000","last_peered":"2024-08-17T17:39:18.812438+0000","last_clean":"2024-08-17T17:39:18.812438+0000","last_became_active":"2024-08-17T17:36:12.757695+0000","last_became_peered":"2024-08-17T17:36:12.757695+0000","last_unstale":"2024-08-17T17:39:18.812438+0000","last_undegraded":"2024-08-17T17:39:18.812438+0000","last_fullsized":"2024-08-17T17:39:18.812438+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:51.792220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1431,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.810473+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:39:18.810473+0000","last_peered":"2024-08-17T17:39:18.810473+0000","last_clean":"2024-08-17T17:39:18.810473+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:39:18.810473+0000","last_undegraded":"2024-08-17T17:39:18.810473+0000","last_fullsized":"2024-08-17T17:39:18.810473+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1739,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.760731+0000","last_change":"2024-08-17T17:39:29.760731+0000","last_active":"2024-08-17T17:39:29.760731+0000","last_peered":"2024-08-17T17:39:29.760731+0000","last_clean":"2024-08-17T17:39:29.760731+0000","last_became_active":"2024-08-17T17:35:55.204861+0000","last_became_peered":"2024-08-17T17:35:55.204861+0000","last_unstale":"2024-08-17T17:39:29.760731+0000","last_undegraded":"2024-08-17T17:39:29.760731+0000","last_fullsized":"2024-08-17T17:39:29.760731+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:39:30.204660+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.0","version":"1425'6096","reported_seq":14090,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:32.450555+0000","last_change":"2024-08-17T17:37:40.753703+0000","last_active":"2024-08-17T17:39:32.450555+0000","last_peered":"2024-08-17T17:39:32.450555+0000","last_clean":"2024-08-17T17:39:32.450555+0000","last_became_active":"2024-08-17T17:37:40.743458+0000","last_became_peered":"2024-08-17T17:37:40.743458+0000","last_unstale":"2024-08-17T17:39:32.450555+0000","last_undegraded":"2024-08-17T17:39:32.450555+0000","last_fullsized":"2024-08-17T17:39:32.450555+0000","mapping_epoch":1411,"log_start":"1373'600","ondisk_log_start":"1373'600","created":1357,"last_epoch_clean":1412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5496,"log_dups_size":0,"ondisk_log_size":5496,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:14:02.340380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132382720,"num_objects":2020,"num_object_clones":0,"num_object_copies":6060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6096,"num_write_kb":259712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3914,"num_bytes_recovered":232652800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1027,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:27.430836+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:39:27.430836+0000","last_peered":"2024-08-17T17:39:27.430836+0000","last_clean":"2024-08-17T17:39:27.430836+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:39:27.430836+0000","last_undegraded":"2024-08-17T17:39:27.430836+0000","last_fullsized":"2024-08-17T17:39:27.430836+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":95,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.883705+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:39:18.883705+0000","last_peered":"2024-08-17T17:39:18.883705+0000","last_clean":"2024-08-17T17:39:18.883705+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:39:18.883705+0000","last_undegraded":"2024-08-17T17:39:18.883705+0000","last_fullsized":"2024-08-17T17:39:18.883705+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1762,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.810182+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:39:18.810182+0000","last_peered":"2024-08-17T17:39:18.810182+0000","last_clean":"2024-08-17T17:39:18.810182+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:39:18.810182+0000","last_undegraded":"2024-08-17T17:39:18.810182+0000","last_fullsized":"2024-08-17T17:39:18.810182+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1","version":"1425'6010","reported_seq":12059,"reported_epoch":1425,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:32.095576+0000","last_change":"2024-08-17T17:36:23.103523+0000","last_active":"2024-08-17T17:39:32.095576+0000","last_peered":"2024-08-17T17:39:32.095576+0000","last_clean":"2024-08-17T17:35:54.178280+0000","last_became_active":"2024-08-17T17:36:23.025910+0000","last_became_peered":"2024-08-17T17:36:23.025910+0000","last_unstale":"2024-08-17T17:39:32.095576+0000","last_undegraded":"2024-08-17T17:36:18.734637+0000","last_fullsized":"2024-08-17T17:36:13.740134+0000","mapping_epoch":1383,"log_start":"1373'200","ondisk_log_start":"1373'200","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5810,"log_dups_size":0,"ondisk_log_size":5810,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:58.717055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133824512,"num_objects":2042,"num_object_clones":0,"num_object_copies":6126,"num_objects_missing_on_primary":847,"num_objects_missing":847,"num_objects_degraded":2765,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6010,"num_write_kb":257664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1986,"num_bytes_recovered":130154496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,0],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":847},{"shards":"0,2,4,6","objects":1},{"shards":"0,2,6","objects":1070},{"shards":"0,6","objects":124}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1441,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751948+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:39:18.751948+0000","last_peered":"2024-08-17T17:39:18.751948+0000","last_clean":"2024-08-17T17:39:18.751948+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:39:18.751948+0000","last_undegraded":"2024-08-17T17:39:18.751948+0000","last_fullsized":"2024-08-17T17:39:18.751948+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1414,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751394+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:39:18.751394+0000","last_peered":"2024-08-17T17:39:18.751394+0000","last_clean":"2024-08-17T17:39:18.751394+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:39:18.751394+0000","last_undegraded":"2024-08-17T17:39:18.751394+0000","last_fullsized":"2024-08-17T17:39:18.751394+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":999,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.762053+0000","last_change":"2024-08-17T17:34:26.817339+0000","last_active":"2024-08-17T17:39:29.762053+0000","last_peered":"2024-08-17T17:39:29.762053+0000","last_clean":"2024-08-17T17:39:29.762053+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:39:29.762053+0000","last_undegraded":"2024-08-17T17:39:29.762053+0000","last_fullsized":"2024-08-17T17:39:29.762053+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:26.817223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:26.817223+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:30.089956+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"1425'5880","reported_seq":6045,"reported_epoch":1425,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:30.508090+0000","last_change":"2024-08-17T17:36:14.073155+0000","last_active":"2024-08-17T17:39:30.508090+0000","last_peered":"2024-08-17T17:39:30.508090+0000","last_clean":"2024-08-17T17:36:11.734683+0000","last_became_active":"2024-08-17T17:36:14.063794+0000","last_became_peered":"2024-08-17T17:36:14.063794+0000","last_unstale":"2024-08-17T17:39:30.508090+0000","last_undegraded":"2024-08-17T17:36:13.855770+0000","last_fullsized":"2024-08-17T17:36:13.742068+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5880,"log_dups_size":0,"ondisk_log_size":5880,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132513792,"num_objects":2022,"num_object_clones":0,"num_object_copies":6066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3819,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5880,"num_write_kb":252864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-1797},{"shards":"0,2,7","objects":1890}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":49,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.752126+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:39:18.752126+0000","last_peered":"2024-08-17T17:39:18.752126+0000","last_clean":"2024-08-17T17:39:18.752126+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:39:18.752126+0000","last_undegraded":"2024-08-17T17:39:18.752126+0000","last_fullsized":"2024-08-17T17:39:18.752126+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1275,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.766439+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:39:18.766439+0000","last_peered":"2024-08-17T17:39:18.766439+0000","last_clean":"2024-08-17T17:39:18.766439+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:39:18.766439+0000","last_undegraded":"2024-08-17T17:39:18.766439+0000","last_fullsized":"2024-08-17T17:39:18.766439+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1790,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.810529+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:39:18.810529+0000","last_peered":"2024-08-17T17:39:18.810529+0000","last_clean":"2024-08-17T17:39:18.810529+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:39:18.810529+0000","last_undegraded":"2024-08-17T17:39:18.810529+0000","last_fullsized":"2024-08-17T17:39:18.810529+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"1425'6085","reported_seq":11885,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:31.978497+0000","last_change":"2024-08-17T17:36:46.982456+0000","last_active":"2024-08-17T17:39:31.978497+0000","last_peered":"2024-08-17T17:39:31.978497+0000","last_clean":"2024-08-17T17:39:31.978497+0000","last_became_active":"2024-08-17T17:36:46.974782+0000","last_became_peered":"2024-08-17T17:36:46.974782+0000","last_unstale":"2024-08-17T17:39:31.978497+0000","last_undegraded":"2024-08-17T17:39:31.978497+0000","last_fullsized":"2024-08-17T17:39:31.978497+0000","mapping_epoch":1399,"log_start":"1373'300","ondisk_log_start":"1373'300","created":1357,"last_epoch_clean":1400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5785,"log_dups_size":0,"ondisk_log_size":5785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:40.499321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130220032,"num_objects":1987,"num_object_clones":0,"num_object_copies":5961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6085,"num_write_kb":258304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2848,"num_bytes_recovered":186646528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1392,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751608+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:39:18.751608+0000","last_peered":"2024-08-17T17:39:18.751608+0000","last_clean":"2024-08-17T17:39:18.751608+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:39:18.751608+0000","last_undegraded":"2024-08-17T17:39:18.751608+0000","last_fullsized":"2024-08-17T17:39:18.751608+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":206,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:27.430673+0000","last_change":"2024-08-17T17:36:12.754372+0000","last_active":"2024-08-17T17:39:27.430673+0000","last_peered":"2024-08-17T17:39:27.430673+0000","last_clean":"2024-08-17T17:39:27.430673+0000","last_became_active":"2024-08-17T17:36:12.746352+0000","last_became_peered":"2024-08-17T17:36:12.746352+0000","last_unstale":"2024-08-17T17:39:27.430673+0000","last_undegraded":"2024-08-17T17:39:27.430673+0000","last_fullsized":"2024-08-17T17:39:27.430673+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:07.807486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1670,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.792655+0000","last_change":"2024-08-17T17:36:15.771193+0000","last_active":"2024-08-17T17:39:29.792655+0000","last_peered":"2024-08-17T17:39:29.792655+0000","last_clean":"2024-08-17T17:39:29.792655+0000","last_became_active":"2024-08-17T17:36:15.770889+0000","last_became_peered":"2024-08-17T17:36:15.770889+0000","last_unstale":"2024-08-17T17:39:29.792655+0000","last_undegraded":"2024-08-17T17:39:29.792655+0000","last_fullsized":"2024-08-17T17:39:29.792655+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:22.978695+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.4","version":"1425'5999","reported_seq":11780,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:32.052246+0000","last_change":"2024-08-17T17:37:04.045576+0000","last_active":"2024-08-17T17:39:32.052246+0000","last_peered":"2024-08-17T17:39:32.052246+0000","last_clean":"2024-08-17T17:39:32.052246+0000","last_became_active":"2024-08-17T17:37:04.038810+0000","last_became_peered":"2024-08-17T17:37:04.038810+0000","last_unstale":"2024-08-17T17:39:32.052246+0000","last_undegraded":"2024-08-17T17:39:32.052246+0000","last_fullsized":"2024-08-17T17:39:32.052246+0000","mapping_epoch":1402,"log_start":"1373'200","ondisk_log_start":"1373'200","created":1357,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5799,"log_dups_size":0,"ondisk_log_size":5799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:27:04.725258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135462912,"num_objects":2067,"num_object_clones":0,"num_object_copies":6201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5999,"num_write_kb":258112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":176291840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1266,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.762263+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:39:29.762263+0000","last_peered":"2024-08-17T17:39:29.762263+0000","last_clean":"2024-08-17T17:39:29.762263+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:39:29.762263+0000","last_undegraded":"2024-08-17T17:39:29.762263+0000","last_fullsized":"2024-08-17T17:39:29.762263+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1418,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.810436+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:39:18.810436+0000","last_peered":"2024-08-17T17:39:18.810436+0000","last_clean":"2024-08-17T17:39:18.810436+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:39:18.810436+0000","last_undegraded":"2024-08-17T17:39:18.810436+0000","last_fullsized":"2024-08-17T17:39:18.810436+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":918,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751868+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:39:18.751868+0000","last_peered":"2024-08-17T17:39:18.751868+0000","last_clean":"2024-08-17T17:39:18.751868+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:39:18.751868+0000","last_undegraded":"2024-08-17T17:39:18.751868+0000","last_fullsized":"2024-08-17T17:39:18.751868+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"1425'6078","reported_seq":11914,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:32.351240+0000","last_change":"2024-08-17T17:36:32.711490+0000","last_active":"2024-08-17T17:39:32.351240+0000","last_peered":"2024-08-17T17:39:32.351240+0000","last_clean":"2024-08-17T17:39:32.351240+0000","last_became_active":"2024-08-17T17:36:32.704370+0000","last_became_peered":"2024-08-17T17:36:32.704370+0000","last_unstale":"2024-08-17T17:39:32.351240+0000","last_undegraded":"2024-08-17T17:39:32.351240+0000","last_fullsized":"2024-08-17T17:39:32.351240+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6078,"log_dups_size":0,"ondisk_log_size":6078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:17.373879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133038080,"num_objects":2030,"num_object_clones":0,"num_object_copies":6090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6078,"num_write_kb":259456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2866,"num_bytes_recovered":187826176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1274,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.774564+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:39:18.774564+0000","last_peered":"2024-08-17T17:39:18.774564+0000","last_clean":"2024-08-17T17:39:18.774564+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:39:18.774564+0000","last_undegraded":"2024-08-17T17:39:18.774564+0000","last_fullsized":"2024-08-17T17:39:18.774564+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":155,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.774369+0000","last_change":"2024-08-17T17:35:55.217072+0000","last_active":"2024-08-17T17:39:29.774369+0000","last_peered":"2024-08-17T17:39:29.774369+0000","last_clean":"2024-08-17T17:39:29.774369+0000","last_became_active":"2024-08-17T17:35:55.216462+0000","last_became_peered":"2024-08-17T17:35:55.216462+0000","last_unstale":"2024-08-17T17:39:29.774369+0000","last_undegraded":"2024-08-17T17:39:29.774369+0000","last_fullsized":"2024-08-17T17:39:29.774369+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:34.937221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":901,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.734803+0000","last_change":"2024-08-17T17:35:55.215004+0000","last_active":"2024-08-17T17:39:18.734803+0000","last_peered":"2024-08-17T17:39:18.734803+0000","last_clean":"2024-08-17T17:39:18.734803+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:39:18.734803+0000","last_undegraded":"2024-08-17T17:39:18.734803+0000","last_fullsized":"2024-08-17T17:39:18.734803+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.6","version":"1425'5882","reported_seq":6022,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:30.540380+0000","last_change":"2024-08-17T17:35:14.868524+0000","last_active":"2024-08-17T17:39:30.540380+0000","last_peered":"2024-08-17T17:39:30.540380+0000","last_clean":"2024-08-17T17:39:30.540380+0000","last_became_active":"2024-08-17T17:35:14.854690+0000","last_became_peered":"2024-08-17T17:35:14.854690+0000","last_unstale":"2024-08-17T17:39:30.540380+0000","last_undegraded":"2024-08-17T17:39:30.540380+0000","last_fullsized":"2024-08-17T17:39:30.540380+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5882,"log_dups_size":0,"ondisk_log_size":5882,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:05.388362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126746624,"num_objects":1934,"num_object_clones":0,"num_object_copies":5802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5882,"num_write_kb":250112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1460,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751283+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:39:18.751283+0000","last_peered":"2024-08-17T17:39:18.751283+0000","last_clean":"2024-08-17T17:39:18.751283+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:39:18.751283+0000","last_undegraded":"2024-08-17T17:39:18.751283+0000","last_fullsized":"2024-08-17T17:39:18.751283+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":167,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.778952+0000","last_change":"2024-08-17T17:35:33.147672+0000","last_active":"2024-08-17T17:39:29.778952+0000","last_peered":"2024-08-17T17:39:29.778952+0000","last_clean":"2024-08-17T17:39:29.778952+0000","last_became_active":"2024-08-17T17:35:33.147283+0000","last_became_peered":"2024-08-17T17:35:33.147283+0000","last_unstale":"2024-08-17T17:39:29.778952+0000","last_undegraded":"2024-08-17T17:39:29.778952+0000","last_fullsized":"2024-08-17T17:39:29.778952+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:41.677709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1671,"reported_epoch":1424,"state":"active+clean+scrubbing","last_fresh":"2024-08-17T17:39:19.929485+0000","last_change":"2024-08-17T17:39:19.929485+0000","last_active":"2024-08-17T17:39:19.929485+0000","last_peered":"2024-08-17T17:39:19.929485+0000","last_clean":"2024-08-17T17:39:19.929485+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:39:19.929485+0000","last_undegraded":"2024-08-17T17:39:19.929485+0000","last_fullsized":"2024-08-17T17:39:19.929485+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:38.241235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:38.241235+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"1425'5922","reported_seq":6000,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:31.916974+0000","last_change":"2024-08-17T17:34:50.670013+0000","last_active":"2024-08-17T17:39:31.916974+0000","last_peered":"2024-08-17T17:39:31.916974+0000","last_clean":"2024-08-17T17:39:31.916974+0000","last_became_active":"2024-08-17T17:34:50.669299+0000","last_became_peered":"2024-08-17T17:34:50.669299+0000","last_unstale":"2024-08-17T17:39:31.916974+0000","last_undegraded":"2024-08-17T17:39:31.916974+0000","last_fullsized":"2024-08-17T17:39:31.916974+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5922,"log_dups_size":0,"ondisk_log_size":5922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:11.172227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131334144,"num_objects":2004,"num_object_clones":0,"num_object_copies":6012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5922,"num_write_kb":253632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1384,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.810150+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:39:18.810150+0000","last_peered":"2024-08-17T17:39:18.810150+0000","last_clean":"2024-08-17T17:39:18.810150+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:39:18.810150+0000","last_undegraded":"2024-08-17T17:39:18.810150+0000","last_fullsized":"2024-08-17T17:39:18.810150+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":154,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.779274+0000","last_change":"2024-08-17T17:35:33.140725+0000","last_active":"2024-08-17T17:39:29.779274+0000","last_peered":"2024-08-17T17:39:29.779274+0000","last_clean":"2024-08-17T17:39:29.779274+0000","last_became_active":"2024-08-17T17:35:33.140406+0000","last_became_peered":"2024-08-17T17:35:33.140406+0000","last_unstale":"2024-08-17T17:39:29.779274+0000","last_undegraded":"2024-08-17T17:39:29.779274+0000","last_fullsized":"2024-08-17T17:39:29.779274+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:54:51.875571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1674,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.778916+0000","last_change":"2024-08-17T17:35:33.138514+0000","last_active":"2024-08-17T17:39:29.778916+0000","last_peered":"2024-08-17T17:39:29.778916+0000","last_clean":"2024-08-17T17:39:29.778916+0000","last_became_active":"2024-08-17T17:35:33.137035+0000","last_became_peered":"2024-08-17T17:35:33.137035+0000","last_unstale":"2024-08-17T17:39:29.778916+0000","last_undegraded":"2024-08-17T17:39:29.778916+0000","last_fullsized":"2024-08-17T17:39:29.778916+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:50.565662+0000","scrub_duration":0.001487138,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1315,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.752112+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:39:18.752112+0000","last_peered":"2024-08-17T17:39:18.752112+0000","last_clean":"2024-08-17T17:39:18.752112+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:39:18.752112+0000","last_undegraded":"2024-08-17T17:39:18.752112+0000","last_fullsized":"2024-08-17T17:39:18.752112+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":95,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.882525+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:39:18.882525+0000","last_peered":"2024-08-17T17:39:18.882525+0000","last_clean":"2024-08-17T17:39:18.882525+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:39:18.882525+0000","last_undegraded":"2024-08-17T17:39:18.882525+0000","last_fullsized":"2024-08-17T17:39:18.882525+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1746,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.810271+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:39:18.810271+0000","last_peered":"2024-08-17T17:39:18.810271+0000","last_clean":"2024-08-17T17:39:18.810271+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:39:18.810271+0000","last_undegraded":"2024-08-17T17:39:18.810271+0000","last_fullsized":"2024-08-17T17:39:18.810271+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1332,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.751201+0000","last_change":"2024-08-17T17:35:55.205292+0000","last_active":"2024-08-17T17:39:18.751201+0000","last_peered":"2024-08-17T17:39:18.751201+0000","last_clean":"2024-08-17T17:39:18.751201+0000","last_became_active":"2024-08-17T17:35:55.204935+0000","last_became_peered":"2024-08-17T17:35:55.204935+0000","last_unstale":"2024-08-17T17:39:18.751201+0000","last_undegraded":"2024-08-17T17:39:18.751201+0000","last_fullsized":"2024-08-17T17:39:18.751201+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:52.277821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":95,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.883239+0000","last_change":"2024-08-17T17:36:12.749972+0000","last_active":"2024-08-17T17:39:18.883239+0000","last_peered":"2024-08-17T17:39:18.883239+0000","last_clean":"2024-08-17T17:39:18.883239+0000","last_became_active":"2024-08-17T17:36:12.749620+0000","last_became_peered":"2024-08-17T17:36:12.749620+0000","last_unstale":"2024-08-17T17:39:18.883239+0000","last_undegraded":"2024-08-17T17:39:18.883239+0000","last_fullsized":"2024-08-17T17:39:18.883239+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:04.994357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":971,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.882248+0000","last_change":"2024-08-17T17:36:20.832274+0000","last_active":"2024-08-17T17:39:18.882248+0000","last_peered":"2024-08-17T17:39:18.882248+0000","last_clean":"2024-08-17T17:39:18.882248+0000","last_became_active":"2024-08-17T17:36:20.831886+0000","last_became_peered":"2024-08-17T17:36:20.831886+0000","last_unstale":"2024-08-17T17:39:18.882248+0000","last_undegraded":"2024-08-17T17:39:18.882248+0000","last_fullsized":"2024-08-17T17:39:18.882248+0000","mapping_epoch":1390,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1391,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:05.248711+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1182,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.760987+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:39:29.760987+0000","last_peered":"2024-08-17T17:39:29.760987+0000","last_clean":"2024-08-17T17:39:29.760987+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:39:29.760987+0000","last_undegraded":"2024-08-17T17:39:29.760987+0000","last_fullsized":"2024-08-17T17:39:29.760987+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1416,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.774432+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:39:29.774432+0000","last_peered":"2024-08-17T17:39:29.774432+0000","last_clean":"2024-08-17T17:39:29.774432+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:39:29.774432+0000","last_undegraded":"2024-08-17T17:39:29.774432+0000","last_fullsized":"2024-08-17T17:39:29.774432+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1709,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.809903+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:39:18.809903+0000","last_peered":"2024-08-17T17:39:18.809903+0000","last_clean":"2024-08-17T17:39:18.809903+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:39:18.809903+0000","last_undegraded":"2024-08-17T17:39:18.809903+0000","last_fullsized":"2024-08-17T17:39:18.809903+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1357,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.859975+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:39:18.859975+0000","last_peered":"2024-08-17T17:39:18.859975+0000","last_clean":"2024-08-17T17:39:18.859975+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:39:18.859975+0000","last_undegraded":"2024-08-17T17:39:18.859975+0000","last_fullsized":"2024-08-17T17:39:18.859975+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1899,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.883355+0000","last_change":"2024-08-17T17:35:33.137411+0000","last_active":"2024-08-17T17:39:18.883355+0000","last_peered":"2024-08-17T17:39:18.883355+0000","last_clean":"2024-08-17T17:39:18.883355+0000","last_became_active":"2024-08-17T17:35:33.137073+0000","last_became_peered":"2024-08-17T17:35:33.137073+0000","last_unstale":"2024-08-17T17:39:18.883355+0000","last_undegraded":"2024-08-17T17:39:18.883355+0000","last_fullsized":"2024-08-17T17:39:18.883355+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:35.924539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1803,"reported_epoch":1424,"state":"active+clean","last_fresh":"2024-08-17T17:39:18.809984+0000","last_change":"2024-08-17T17:36:15.787445+0000","last_active":"2024-08-17T17:39:18.809984+0000","last_peered":"2024-08-17T17:39:18.809984+0000","last_clean":"2024-08-17T17:39:18.809984+0000","last_became_active":"2024-08-17T17:36:15.787099+0000","last_became_peered":"2024-08-17T17:36:15.787099+0000","last_unstale":"2024-08-17T17:39:18.809984+0000","last_undegraded":"2024-08-17T17:39:18.809984+0000","last_fullsized":"2024-08-17T17:39:18.809984+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:47.448168+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":2129526808,"num_objects":32495,"num_object_clones":0,"num_object_copies":97485,"num_objects_missing_on_primary":2521,"num_objects_missing":2521,"num_objects_degraded":12712,"num_objects_misplaced":2802,"num_objects_unfound":0,"num_objects_dirty":32495,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":96387,"num_write_kb":4124161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31223,"num_bytes_recovered":1893597184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7440838656,"data_stored":7440826440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94387,"ondisk_log_size":94387,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635092,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":27,"num_read_kb":61,"num_write":15,"num_write_kb":183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1314910,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162865,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1210056,"kb_used_data":1041340,"kb_used_omap":54,"kb_used_meta":168649,"kb_avail":92535096,"statfs":{"total":95995035648,"available":94755938304,"internally_reserved":0,"allocated":1066332160,"data_stored":1063646120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":172696597},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999339,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1090628,"kb_used_data":900540,"kb_used_omap":47,"kb_used_meta":189968,"kb_avail":92654524,"statfs":{"total":95995035648,"available":94878232576,"internally_reserved":0,"allocated":922152960,"data_stored":919467897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":194527892},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,0,1,2,0,3,4,3],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627135,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1369756,"kb_used_data":319380,"kb_used_omap":42,"kb_used_meta":1050325,"kb_avail":92375396,"statfs":{"total":95995035648,"available":94592405504,"internally_reserved":0,"allocated":327045120,"data_stored":324386308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1075532860},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785613,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2347564,"kb_used_data":1535076,"kb_used_omap":48,"kb_used_meta":812431,"kb_avail":91397588,"statfs":{"total":95995035648,"available":93591130112,"internally_reserved":0,"allocated":1571917824,"data_stored":1569251917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":831930030},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725130,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1384372,"kb_used_data":527212,"kb_used_omap":1,"kb_used_meta":857150,"kb_avail":92360780,"statfs":{"total":95995035648,"available":94577438720,"internally_reserved":0,"allocated":539865088,"data_stored":537191131,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":877722054},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986767,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1768772,"kb_used_data":893560,"kb_used_omap":1,"kb_used_meta":875198,"kb_avail":91976380,"statfs":{"total":95995035648,"available":94183813120,"internally_reserved":0,"allocated":915005440,"data_stored":912322906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":896203206},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706374,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2206072,"kb_used_data":1289708,"kb_used_omap":35,"kb_used_meta":916316,"kb_avail":91539080,"statfs":{"total":95995035648,"available":93736017920,"internally_reserved":0,"allocated":1320660992,"data_stored":1317987016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":938307639},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832147,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1871924,"kb_used_data":837160,"kb_used_omap":65,"kb_used_meta":1034686,"kb_avail":91873228,"statfs":{"total":95995035648,"available":94078185472,"internally_reserved":0,"allocated":857251840,"data_stored":854582422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1059518651},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,2,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":846725120,"data_stored":846725120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1562050560,"data_stored":1562050560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1312624640,"data_stored":1312620568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":317390848,"data_stored":317390848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":904794112,"data_stored":904790040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":531365888,"data_stored":531365888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":913182720,"data_stored":913178648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1052704768,"data_stored":1052704768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:39:36.272 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:36.308 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:36.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:36.410 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:36.434 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:36.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:36.535 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:36.560 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:36.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:36.661 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:36.698 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:36.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:36.738 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:36.800 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:36.825 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:36.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:36.877 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:36.877 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:36.927 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:36.953 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:36.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:36.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:37.055 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:37.077 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:37.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:37.179 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:37.219 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:37.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:37.320 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:37.344 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:37.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:37.446 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:37.471 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:37.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:37.573 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:37.597 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:37.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:37.650 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:39:37.700 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:37.723 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:37.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:37.783 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:39:37.825 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:37.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:37 smithi162 ceph-mon[104668]: pgmap v2256: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 12573/96030 objects degraded (13.093%); 2802/96030 objects misplaced (2.918%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:37.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:37 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1695191759' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:37.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:37 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:37.874 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:37.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:37.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:37.976 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:37.985 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:38.007 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:38.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:38.026 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:37 smithi063 ceph-mon[114756]: pgmap v2256: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 12573/96030 objects degraded (13.093%); 2802/96030 objects misplaced (2.918%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:38.026 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:37 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1695191759' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:38.026 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:37 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:38.026 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:37 smithi063 ceph-mon[109843]: pgmap v2256: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 250 op/s; 12573/96030 objects degraded (13.093%); 2802/96030 objects misplaced (2.918%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:38.026 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:37 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1695191759' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:38.026 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:37 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:38.109 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:38.138 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:38.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:38.240 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:38.270 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:38.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:38.373 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:38.405 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:38.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:38.507 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:38.530 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:38.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:38.632 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:38.654 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:38.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:38.756 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:38.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.785 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:38.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:38.886 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:38.908 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:38.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:38.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:39.010 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:38 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:38 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:38 smithi063 ceph-mon[114756]: osdmap e1426: 8 total, 8 up, 8 in 2024-08-17T17:39:39.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:38 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 12573/96030 objects degraded (13.093%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:39.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:38 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:39.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:38 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:39.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:38 smithi063 ceph-mon[109843]: osdmap e1426: 8 total, 8 up, 8 in 2024-08-17T17:39:39.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:38 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 12573/96030 objects degraded (13.093%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:39.042 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:39.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:38 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:38 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:39:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:38 smithi162 ceph-mon[104668]: osdmap e1426: 8 total, 8 up, 8 in 2024-08-17T17:39:39.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:38 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 12573/96030 objects degraded (13.093%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:39.143 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:39.174 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:39.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:39.195 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:39:39.276 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:39.299 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:39.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:39.364 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:39:39.365 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:39.386 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:39:39.386 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:39.401 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:39.408 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:39:39.408 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:39.428 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:39.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:39.436 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:39:39.436 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:39.454 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:39:39.454 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:39.473 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:39.476 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:39:39.476 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:39.497 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:39:39.497 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:39.519 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:39:39.519 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:39.530 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:39.560 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:39.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:39.662 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:39.690 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:39.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:39.791 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:39.820 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:39.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:39.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:39 smithi162 ceph-mon[104668]: pgmap v2258: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 12233/93825 objects degraded (13.038%); 2648/93825 objects misplaced (2.822%); 4.5 MiB/s, 71 objects/s recovering 2024-08-17T17:39:39.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:39 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2678714036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:39.888 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:39:39.888 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [], 2024-08-17T17:39:39.888 INFO:teuthology.orchestra.run.smithi162.stdout: "complaint_time": 30, 2024-08-17T17:39:39.888 INFO:teuthology.orchestra.run.smithi162.stdout: "num_blocked_ops": 0 2024-08-17T17:39:39.888 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:39:39.922 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:39.946 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:39.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:39.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:39 smithi063 ceph-mon[114756]: pgmap v2258: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 12233/93825 objects degraded (13.038%); 2648/93825 objects misplaced (2.822%); 4.5 MiB/s, 71 objects/s recovering 2024-08-17T17:39:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:39 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2678714036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:39 smithi063 ceph-mon[109843]: pgmap v2258: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 279 op/s; 12233/93825 objects degraded (13.038%); 2648/93825 objects misplaced (2.822%); 4.5 MiB/s, 71 objects/s recovering 2024-08-17T17:39:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:39 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2678714036' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:40.048 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:40.074 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:40.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:40.176 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:40.206 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:40.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:40.307 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:40.331 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:40.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:40.432 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:40.464 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:40.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:40.539 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:39:40.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:40.592 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:40.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:40.694 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:40.728 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:40.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:40.787 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:40 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:39:40.811 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:39:40.830 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:40.876 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:40.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:40.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:40.977 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:41.003 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:41.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:41.105 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:41.137 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:41.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:41.218 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:39:41.220 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:40 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:39:41.221 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:40 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:39:41.239 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:41.269 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:41.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:41.371 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:41.399 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:41.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:41.501 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:41.526 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:41.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:41.628 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:41.651 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:41.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:41.678 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:41.753 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:41.775 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:41.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:41.877 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:41.914 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:41.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:41.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:41.927 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:41 smithi063 ceph-mon[109843]: pgmap v2259: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 258 op/s; 12233/93537 objects degraded (13.078%); 2648/93537 objects misplaced (2.831%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:41.927 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:41 smithi063 ceph-mon[114756]: pgmap v2259: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 258 op/s; 12233/93537 objects degraded (13.078%); 2648/93537 objects misplaced (2.831%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:42.016 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:42.035 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:41 smithi162 ceph-mon[104668]: pgmap v2259: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 258 op/s; 12233/93537 objects degraded (13.078%); 2648/93537 objects misplaced (2.831%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:42.042 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:42.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:42.143 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:42.167 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:42.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:42.269 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:42.293 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:42.395 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:42.421 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:42.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:42.521 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:39:42.523 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:42.549 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:42.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:42.650 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:42.681 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:42.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:42.783 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:42.807 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:42.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:42.909 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:42.934 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:42.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:42.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:43.036 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:43.064 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:39:43.065 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:39:43.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.067 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:43.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:43.169 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:43.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.193 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:43.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:43.293 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:43.317 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:43.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:43.382 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:39:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:39:43] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.51.0" 2024-08-17T17:39:43.419 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:43.446 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:43.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:43.547 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:43.570 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:43.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:43.672 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:43.698 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:43.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:43.800 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:43.827 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:43.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:43.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:43 smithi162 ceph-mon[104668]: pgmap v2260: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 11774/89943 objects degraded (13.091%); 2471/89943 objects misplaced (2.747%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:43.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:43 smithi162 ceph-mon[104668]: from='client.46460 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:43.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:43 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 11774/89943 objects degraded (13.091%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:43.879 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:43.928 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:43.951 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:43.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:43.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:44.053 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:44.085 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:44.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:44.102 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:43 smithi063 ceph-mon[114756]: pgmap v2260: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 11774/89943 objects degraded (13.091%); 2471/89943 objects misplaced (2.747%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:44.102 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:43 smithi063 ceph-mon[114756]: from='client.46460 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:44.102 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:43 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 11774/89943 objects degraded (13.091%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:44.103 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:43 smithi063 ceph-mon[109843]: pgmap v2260: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 253 op/s; 11774/89943 objects degraded (13.091%); 2471/89943 objects misplaced (2.747%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:44.103 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:43 smithi063 ceph-mon[109843]: from='client.46460 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:39:44.103 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:43 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 11774/89943 objects degraded (13.091%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:44.187 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:44.223 INFO:teuthology.orchestra.run.smithi162.stdout:{ 2024-08-17T17:39:44.223 INFO:teuthology.orchestra.run.smithi162.stdout: "size": 20, 2024-08-17T17:39:44.223 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 600, 2024-08-17T17:39:44.223 INFO:teuthology.orchestra.run.smithi162.stdout: "ops": [ 2024-08-17T17:39:44.224 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.224 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:62271 11.8 11:13a623d9:::benchmark_data_smithi063_328699_object62270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1383)", 2024-08-17T17:39:44.224 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:12.741457+0000", 2024-08-17T17:39:44.224 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 211.47366763700001, 2024-08-17T17:39:44.224 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 10.253545902000001, 2024-08-17T17:39:44.224 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.224 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "reached pg", 2024-08-17T17:39:44.224 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.224 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.224 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.225 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 62271 2024-08-17T17:39:44.225 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.225 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.225 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.225 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.225 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741457+0000", 2024-08-17T17:39:44.225 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.225 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.225 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.225 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.225 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741457+0000", 2024-08-17T17:39:44.225 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.226 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.226 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.226 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.226 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741455+0000", 2024-08-17T17:39:44.226 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:44.226 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.226 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.226 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.226 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741621+0000", 2024-08-17T17:39:44.226 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000165286 2024-08-17T17:39:44.226 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.226 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.227 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.227 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741634+0000", 2024-08-17T17:39:44.227 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3066e-05 2024-08-17T17:39:44.227 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.227 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.227 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.227 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741642+0000", 2024-08-17T17:39:44.227 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.2930000000000001e-06 2024-08-17T17:39:44.227 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.227 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.227 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.227 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.751341+0000", 2024-08-17T17:39:44.227 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0096989499999999996 2024-08-17T17:39:44.228 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.228 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.228 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:44.228 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.751352+0000", 2024-08-17T17:39:44.228 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0478e-05 2024-08-17T17:39:44.228 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.228 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.228 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.228 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:22.936587+0000", 2024-08-17T17:39:44.228 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 10.185235643 2024-08-17T17:39:44.228 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.228 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.229 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:39:44.229 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:22.944642+0000", 2024-08-17T17:39:44.229 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0080547250000000004 2024-08-17T17:39:44.229 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.229 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.229 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.229 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:22.991232+0000", 2024-08-17T17:39:44.229 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.046589622999999997 2024-08-17T17:39:44.229 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.229 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.229 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.229 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:22.995003+0000", 2024-08-17T17:39:44.230 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0037714210000000001 2024-08-17T17:39:44.230 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.230 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.230 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.230 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.230 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.230 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:62284 11.a 11:57c07eba:::benchmark_data_smithi063_328699_object62283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1383)", 2024-08-17T17:39:44.230 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:12.741651+0000", 2024-08-17T17:39:44.230 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 211.473473278, 2024-08-17T17:39:44.230 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.455935459, 2024-08-17T17:39:44.230 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.230 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.231 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.231 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.231 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.231 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 62284 2024-08-17T17:39:44.231 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.231 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.231 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.231 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.231 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741651+0000", 2024-08-17T17:39:44.231 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.231 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.231 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.231 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.232 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741651+0000", 2024-08-17T17:39:44.232 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.232 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.232 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.232 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.232 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741651+0000", 2024-08-17T17:39:44.232 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:39:44.232 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.232 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.232 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.232 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741686+0000", 2024-08-17T17:39:44.232 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5453000000000001e-05 2024-08-17T17:39:44.232 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.233 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.233 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.233 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741694+0000", 2024-08-17T17:39:44.233 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7940000000000003e-06 2024-08-17T17:39:44.233 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.233 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.233 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.233 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741700+0000", 2024-08-17T17:39:44.233 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.8119999999999996e-06 2024-08-17T17:39:44.233 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.233 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.233 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.233 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.745984+0000", 2024-08-17T17:39:44.234 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0042847170000000004 2024-08-17T17:39:44.234 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.234 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.234 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:44.234 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.745995+0000", 2024-08-17T17:39:44.234 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0628000000000001e-05 2024-08-17T17:39:44.234 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.234 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.234 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.234 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.133911+0000", 2024-08-17T17:39:44.234 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.3879154309999997 2024-08-17T17:39:44.234 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.234 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.235 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.235 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.143194+0000", 2024-08-17T17:39:44.235 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0092832869999999994 2024-08-17T17:39:44.235 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.235 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.235 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.235 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.155507+0000", 2024-08-17T17:39:44.235 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012313166 2024-08-17T17:39:44.235 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.235 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.235 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.235 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.178090+0000", 2024-08-17T17:39:44.235 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.022583413 2024-08-17T17:39:44.236 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.236 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.236 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.236 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.179900+0000", 2024-08-17T17:39:44.236 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001809389 2024-08-17T17:39:44.236 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.236 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.236 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.236 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.181705+0000", 2024-08-17T17:39:44.236 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0018056750000000001 2024-08-17T17:39:44.236 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.236 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.236 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.236 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.184366+0000", 2024-08-17T17:39:44.237 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002660394 2024-08-17T17:39:44.237 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.237 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.237 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.237 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.197587+0000", 2024-08-17T17:39:44.237 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.01322099 2024-08-17T17:39:44.237 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.237 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.237 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.237 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.237 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.237 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:62304 11.a 11:5dee0cfc:::benchmark_data_smithi063_328699_object62303:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1383)", 2024-08-17T17:39:44.237 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:12.741904+0000", 2024-08-17T17:39:44.237 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 211.473220924, 2024-08-17T17:39:44.238 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.5334604729999999, 2024-08-17T17:39:44.238 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.238 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.238 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.238 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.238 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.238 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 62304 2024-08-17T17:39:44.238 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.238 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.239 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.239 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.239 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741904+0000", 2024-08-17T17:39:44.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.239 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.239 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.239 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.239 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741904+0000", 2024-08-17T17:39:44.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.239 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.239 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.239 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.239 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741903+0000", 2024-08-17T17:39:44.239 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999995 2024-08-17T17:39:44.240 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.240 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.240 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.240 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741934+0000", 2024-08-17T17:39:44.240 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.0450000000000002e-05 2024-08-17T17:39:44.240 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.240 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.240 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.240 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741941+0000", 2024-08-17T17:39:44.240 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4449999999999999e-06 2024-08-17T17:39:44.240 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.240 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.240 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.240 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.741947+0000", 2024-08-17T17:39:44.241 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3550000000000001e-06 2024-08-17T17:39:44.241 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.241 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.241 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.241 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.749347+0000", 2024-08-17T17:39:44.241 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0073995989999999998 2024-08-17T17:39:44.241 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.241 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.241 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for peered", 2024-08-17T17:39:44.241 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:12.749358+0000", 2024-08-17T17:39:44.241 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1375999999999999e-05 2024-08-17T17:39:44.241 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.241 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.242 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.242 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.201592+0000", 2024-08-17T17:39:44.242 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4522340150000002 2024-08-17T17:39:44.242 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.242 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.242 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.242 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.211646+0000", 2024-08-17T17:39:44.242 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010053945 2024-08-17T17:39:44.242 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.242 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.242 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.242 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.222352+0000", 2024-08-17T17:39:44.242 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010705668 2024-08-17T17:39:44.243 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.243 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.243 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.243 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.256947+0000", 2024-08-17T17:39:44.243 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.034595069999999999 2024-08-17T17:39:44.243 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.243 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.243 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.243 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.257469+0000", 2024-08-17T17:39:44.243 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00052186099999999998 2024-08-17T17:39:44.243 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.243 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.243 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.244 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.259617+0000", 2024-08-17T17:39:44.244 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021475460000000002 2024-08-17T17:39:44.244 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.244 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.244 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.244 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.262149+0000", 2024-08-17T17:39:44.244 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0025328569999999999 2024-08-17T17:39:44.244 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.244 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.244 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.244 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.275364+0000", 2024-08-17T17:39:44.244 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.013214867 2024-08-17T17:39:44.244 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.245 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.245 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.245 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.245 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.245 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:62311 11.8 11:1ef91939:::benchmark_data_smithi063_328699_object62310:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1384)", 2024-08-17T17:39:44.245 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:14.079264+0000", 2024-08-17T17:39:44.245 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 210.13586098100001, 2024-08-17T17:39:44.245 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9946967079999993, 2024-08-17T17:39:44.245 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.245 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.245 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.245 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.246 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.246 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 62311 2024-08-17T17:39:44.246 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.246 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.246 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.246 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.246 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.079264+0000", 2024-08-17T17:39:44.246 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.246 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.246 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.246 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.246 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.079264+0000", 2024-08-17T17:39:44.246 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.246 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.247 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.247 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.247 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.079262+0000", 2024-08-17T17:39:44.247 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:39:44.247 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.247 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.247 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.247 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.079382+0000", 2024-08-17T17:39:44.247 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000119948 2024-08-17T17:39:44.247 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.247 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.247 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.247 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.079396+0000", 2024-08-17T17:39:44.248 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3951000000000001e-05 2024-08-17T17:39:44.248 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.248 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.248 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.248 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.079404+0000", 2024-08-17T17:39:44.248 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.0579999999999998e-06 2024-08-17T17:39:44.248 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.248 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.248 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.248 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:22.995774+0000", 2024-08-17T17:39:44.248 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9163696059999999 2024-08-17T17:39:44.248 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.249 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.249 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.249 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:23.001485+0000", 2024-08-17T17:39:44.249 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0057115239999999999 2024-08-17T17:39:44.249 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.249 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.249 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.249 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:23.011884+0000", 2024-08-17T17:39:44.249 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010398900000000001 2024-08-17T17:39:44.249 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.249 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.249 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.249 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:23.047685+0000", 2024-08-17T17:39:44.250 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.035800812000000001 2024-08-17T17:39:44.250 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.250 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.250 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.250 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:23.050579+0000", 2024-08-17T17:39:44.250 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0028946559999999998 2024-08-17T17:39:44.250 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.250 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.250 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.250 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:23.055880+0000", 2024-08-17T17:39:44.250 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0053003800000000004 2024-08-17T17:39:44.250 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.251 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.251 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.251 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:23.059973+0000", 2024-08-17T17:39:44.251 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0040934150000000004 2024-08-17T17:39:44.251 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.251 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.251 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.251 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:23.073960+0000", 2024-08-17T17:39:44.251 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.013987169000000001 2024-08-17T17:39:44.251 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.251 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.251 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.251 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.252 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.252 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:62314 11.a 11:5e55e88c:::benchmark_data_smithi063_328699_object62313:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1384)", 2024-08-17T17:39:44.252 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:14.090016+0000", 2024-08-17T17:39:44.252 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 210.12510899, 2024-08-17T17:39:44.252 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.207289566, 2024-08-17T17:39:44.252 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.252 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.252 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.252 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.252 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.252 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 62314 2024-08-17T17:39:44.252 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.253 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.253 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.253 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.253 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.090016+0000", 2024-08-17T17:39:44.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.254 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:44.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:44.254 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.255 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.255 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.255 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.255 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.090016+0000", 2024-08-17T17:39:44.255 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.255 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.255 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.255 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.255 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.090014+0000", 2024-08-17T17:39:44.255 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:44.255 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.255 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.256 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.256 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.090066+0000", 2024-08-17T17:39:44.256 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.1854999999999997e-05 2024-08-17T17:39:44.256 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.256 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.256 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.256 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.090078+0000", 2024-08-17T17:39:44.256 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.19e-05 2024-08-17T17:39:44.256 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.256 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.256 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.256 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.090085+0000", 2024-08-17T17:39:44.256 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9999999999999999e-06 2024-08-17T17:39:44.257 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.257 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.257 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.257 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.229613+0000", 2024-08-17T17:39:44.257 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1395277549999996 2024-08-17T17:39:44.257 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.257 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.257 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.257 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.237707+0000", 2024-08-17T17:39:44.257 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0080945199999999991 2024-08-17T17:39:44.257 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.257 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.257 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.258 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.248539+0000", 2024-08-17T17:39:44.258 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010831372000000001 2024-08-17T17:39:44.258 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.258 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.258 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.258 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.276118+0000", 2024-08-17T17:39:44.258 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.027579830999999999 2024-08-17T17:39:44.258 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.258 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.258 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.258 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.277981+0000", 2024-08-17T17:39:44.258 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0018629549999999999 2024-08-17T17:39:44.259 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.259 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.259 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.259 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.280371+0000", 2024-08-17T17:39:44.259 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0023900779999999999 2024-08-17T17:39:44.259 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.259 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.259 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.259 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.283979+0000", 2024-08-17T17:39:44.259 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.003607734 2024-08-17T17:39:44.259 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.259 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.259 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.260 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.297305+0000", 2024-08-17T17:39:44.260 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.013326041 2024-08-17T17:39:44.260 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.260 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.260 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.260 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.260 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.260 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:62324 11.a 11:5aa7d49f:::benchmark_data_smithi063_328699_object62323:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1384)", 2024-08-17T17:39:44.260 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:14.109572+0000", 2024-08-17T17:39:44.260 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 210.10555295500001, 2024-08-17T17:39:44.260 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2628556809999996, 2024-08-17T17:39:44.260 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.260 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.260 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.261 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.261 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.261 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 62324 2024-08-17T17:39:44.261 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.261 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.261 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.261 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.261 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.109572+0000", 2024-08-17T17:39:44.261 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.261 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.261 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.261 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.262 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.109572+0000", 2024-08-17T17:39:44.262 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.262 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.262 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.262 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.262 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.109569+0000", 2024-08-17T17:39:44.262 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999976 2024-08-17T17:39:44.262 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.262 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.262 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.262 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.109637+0000", 2024-08-17T17:39:44.262 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.7994e-05 2024-08-17T17:39:44.262 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.263 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.263 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.263 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.109657+0000", 2024-08-17T17:39:44.263 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9899999999999999e-05 2024-08-17T17:39:44.263 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.263 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.263 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.263 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.109668+0000", 2024-08-17T17:39:44.263 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0448000000000001e-05 2024-08-17T17:39:44.263 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.265 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.265 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.265 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.298060+0000", 2024-08-17T17:39:44.265 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1883922010000001 2024-08-17T17:39:44.265 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.266 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.266 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.266 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.305056+0000", 2024-08-17T17:39:44.266 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0069964299999999997 2024-08-17T17:39:44.266 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.266 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.266 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.266 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.314207+0000", 2024-08-17T17:39:44.266 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.009150465 2024-08-17T17:39:44.266 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.266 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.266 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.267 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.351806+0000", 2024-08-17T17:39:44.267 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.037599625999999997 2024-08-17T17:39:44.267 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.267 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.267 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.267 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.353983+0000", 2024-08-17T17:39:44.267 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021769369999999999 2024-08-17T17:39:44.267 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.267 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.267 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.267 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.354637+0000", 2024-08-17T17:39:44.267 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000653679 2024-08-17T17:39:44.268 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.268 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.268 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.268 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.358087+0000", 2024-08-17T17:39:44.268 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0034499349999999999 2024-08-17T17:39:44.268 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.268 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.268 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.268 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.372427+0000", 2024-08-17T17:39:44.268 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.01434061 2024-08-17T17:39:44.268 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.268 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.268 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.269 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.269 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.269 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:62327 11.a 11:53d29a9b:::benchmark_data_smithi063_328699_object62326:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1384)", 2024-08-17T17:39:44.269 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:14.116158+0000", 2024-08-17T17:39:44.269 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 210.09896690299999, 2024-08-17T17:39:44.269 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2773710810000001, 2024-08-17T17:39:44.269 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.269 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.269 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.269 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.269 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.269 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 62327 2024-08-17T17:39:44.269 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.269 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.270 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.270 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.270 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.116158+0000", 2024-08-17T17:39:44.270 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.270 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.270 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.270 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.270 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.116158+0000", 2024-08-17T17:39:44.270 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.270 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.270 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.270 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.271 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.116156+0000", 2024-08-17T17:39:44.271 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:44.271 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.271 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.271 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.271 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.116210+0000", 2024-08-17T17:39:44.271 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3489999999999998e-05 2024-08-17T17:39:44.271 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.271 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.271 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.271 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.116223+0000", 2024-08-17T17:39:44.271 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3264e-05 2024-08-17T17:39:44.271 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.271 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.272 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.272 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.116230+0000", 2024-08-17T17:39:44.272 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.4939999999999997e-06 2024-08-17T17:39:44.272 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.272 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.272 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.272 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.321056+0000", 2024-08-17T17:39:44.272 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2048253349999998 2024-08-17T17:39:44.272 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.272 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.272 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.272 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.329092+0000", 2024-08-17T17:39:44.272 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0080364890000000008 2024-08-17T17:39:44.272 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.272 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.273 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.273 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.341823+0000", 2024-08-17T17:39:44.273 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012731025 2024-08-17T17:39:44.273 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.273 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.273 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.273 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.373075+0000", 2024-08-17T17:39:44.273 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.031252052000000002 2024-08-17T17:39:44.273 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.273 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.273 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.273 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.374775+0000", 2024-08-17T17:39:44.273 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016998219999999999 2024-08-17T17:39:44.273 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.274 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.274 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.274 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.376250+0000", 2024-08-17T17:39:44.274 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001474398 2024-08-17T17:39:44.274 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.274 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.274 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.274 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.378483+0000", 2024-08-17T17:39:44.274 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0022333209999999999 2024-08-17T17:39:44.274 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.274 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.274 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.274 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.393529+0000", 2024-08-17T17:39:44.275 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.015045973000000001 2024-08-17T17:39:44.275 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.275 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.275 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.275 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.275 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.275 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:62366 11.a 11:56edae78:::benchmark_data_smithi063_328699_object62365:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1384)", 2024-08-17T17:39:44.275 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:14.196895+0000", 2024-08-17T17:39:44.275 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 210.01823019, 2024-08-17T17:39:44.275 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2936742399999996, 2024-08-17T17:39:44.275 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.275 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.275 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.275 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.276 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.276 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 62366 2024-08-17T17:39:44.276 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.276 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.276 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.276 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.276 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.196895+0000", 2024-08-17T17:39:44.276 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.276 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.276 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.276 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.276 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.196895+0000", 2024-08-17T17:39:44.277 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.277 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.277 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.277 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.277 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.196893+0000", 2024-08-17T17:39:44.277 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:39:44.277 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.277 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.277 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.277 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.196945+0000", 2024-08-17T17:39:44.277 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.2475999999999997e-05 2024-08-17T17:39:44.277 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.277 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.277 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.277 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.196965+0000", 2024-08-17T17:39:44.278 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.9712e-05 2024-08-17T17:39:44.278 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.278 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.278 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.278 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.196974+0000", 2024-08-17T17:39:44.278 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.9949999999999994e-06 2024-08-17T17:39:44.278 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.278 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.278 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.278 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.394530+0000", 2024-08-17T17:39:44.278 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.1975559320000002 2024-08-17T17:39:44.278 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.278 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.279 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.279 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.407321+0000", 2024-08-17T17:39:44.279 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012791205 2024-08-17T17:39:44.279 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.279 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.279 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.279 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.427629+0000", 2024-08-17T17:39:44.279 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.020307405000000001 2024-08-17T17:39:44.279 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.279 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.279 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.279 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.467819+0000", 2024-08-17T17:39:44.279 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.040190591999999997 2024-08-17T17:39:44.279 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.279 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.280 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.280 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.470094+0000", 2024-08-17T17:39:44.280 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0022751450000000001 2024-08-17T17:39:44.280 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.280 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.280 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.280 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.470793+0000", 2024-08-17T17:39:44.280 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00069851400000000001 2024-08-17T17:39:44.280 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.280 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.280 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.280 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.474552+0000", 2024-08-17T17:39:44.280 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0037596600000000002 2024-08-17T17:39:44.280 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.281 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.281 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.281 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.490569+0000", 2024-08-17T17:39:44.281 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.016016308 2024-08-17T17:39:44.281 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.281 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.281 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.281 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.281 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.281 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:62374 11.a 11:5bac99f2:::benchmark_data_smithi063_328699_object62373:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1384)", 2024-08-17T17:39:44.281 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:14.224316+0000", 2024-08-17T17:39:44.281 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 209.99080901400001, 2024-08-17T17:39:44.281 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2898290230000002, 2024-08-17T17:39:44.281 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.282 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.282 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.282 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.282 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.282 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 62374 2024-08-17T17:39:44.282 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.282 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.282 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.282 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.282 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.224316+0000", 2024-08-17T17:39:44.282 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.282 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.282 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.283 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.283 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.224316+0000", 2024-08-17T17:39:44.283 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.283 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.283 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.283 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.283 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.224313+0000", 2024-08-17T17:39:44.283 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999976 2024-08-17T17:39:44.283 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.283 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.283 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.283 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.224348+0000", 2024-08-17T17:39:44.283 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 3.5203000000000002e-05 2024-08-17T17:39:44.284 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.284 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.284 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.284 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.224364+0000", 2024-08-17T17:39:44.284 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5451000000000002e-05 2024-08-17T17:39:44.284 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.284 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.284 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.284 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:14.224373+0000", 2024-08-17T17:39:44.284 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7000000000000003e-06 2024-08-17T17:39:44.284 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.284 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.284 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.284 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.435583+0000", 2024-08-17T17:39:44.285 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.2112099760000001 2024-08-17T17:39:44.285 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.285 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.285 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.285 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.444711+0000", 2024-08-17T17:39:44.285 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0091278569999999996 2024-08-17T17:39:44.285 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.285 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.285 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.285 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.457671+0000", 2024-08-17T17:39:44.285 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012959293 2024-08-17T17:39:44.285 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.286 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.286 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.286 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.491270+0000", 2024-08-17T17:39:44.286 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.033599535 2024-08-17T17:39:44.286 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.286 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.286 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.286 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.493401+0000", 2024-08-17T17:39:44.286 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021311870000000001 2024-08-17T17:39:44.286 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.286 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.286 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.287 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.495797+0000", 2024-08-17T17:39:44.287 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002395496 2024-08-17T17:39:44.287 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.287 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.287 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.287 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.499375+0000", 2024-08-17T17:39:44.287 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0035777740000000001 2024-08-17T17:39:44.287 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.287 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.287 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.287 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.514145+0000", 2024-08-17T17:39:44.287 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.014770146 2024-08-17T17:39:44.287 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.288 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.288 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.288 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.288 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.288 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "MOSDScrubReserve(1.0 REQUEST e1386)", 2024-08-17T17:39:44.288 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:36:16.339334+0000", 2024-08-17T17:39:44.288 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 207.87579088999999, 2024-08-17T17:39:44.288 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3007574640000001, 2024-08-17T17:39:44.288 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.288 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "started", 2024-08-17T17:39:44.288 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.288 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.288 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.288 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:16.339334+0000", 2024-08-17T17:39:44.288 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.289 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.289 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.289 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.289 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:16.339334+0000", 2024-08-17T17:39:44.289 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.289 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.289 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.289 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.289 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:16.339332+0000", 2024-08-17T17:39:44.289 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:39:44.289 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.289 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.289 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.290 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:16.339342+0000", 2024-08-17T17:39:44.290 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4630000000000007e-06 2024-08-17T17:39:44.290 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.290 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.290 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.290 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:16.339357+0000", 2024-08-17T17:39:44.290 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5370999999999999e-05 2024-08-17T17:39:44.290 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.290 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.290 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.290 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:16.339366+0000", 2024-08-17T17:39:44.290 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.1900000000000001e-06 2024-08-17T17:39:44.291 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.291 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.291 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.291 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.640048+0000", 2024-08-17T17:39:44.291 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.3006817460000004 2024-08-17T17:39:44.291 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.291 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.291 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.291 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.640063+0000", 2024-08-17T17:39:44.291 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5322999999999999e-05 2024-08-17T17:39:44.291 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.291 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.291 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.292 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:36:20.640091+0000", 2024-08-17T17:39:44.292 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.8116e-05 2024-08-17T17:39:44.292 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.292 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.292 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.292 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.292 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.292 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:68904 11.8 11:186e2c7b:::benchmark_data_smithi063_328699_object4461:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1401)", 2024-08-17T17:39:44.292 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:37:01.535183+0000", 2024-08-17T17:39:44.292 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 162.679941789, 2024-08-17T17:39:44.292 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.445871406, 2024-08-17T17:39:44.292 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.292 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.292 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.293 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.293 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.293 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 68904 2024-08-17T17:39:44.293 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.293 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.293 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.293 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.293 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:01.535183+0000", 2024-08-17T17:39:44.293 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.293 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.293 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.293 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.294 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:01.535183+0000", 2024-08-17T17:39:44.294 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.294 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.294 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.294 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.294 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:01.535181+0000", 2024-08-17T17:39:44.294 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:39:44.294 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.294 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.294 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.294 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:01.535188+0000", 2024-08-17T17:39:44.294 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.6139999999999997e-06 2024-08-17T17:39:44.295 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.295 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.295 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.295 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:01.535200+0000", 2024-08-17T17:39:44.295 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2371000000000001e-05 2024-08-17T17:39:44.295 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.295 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.295 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.295 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:01.535207+0000", 2024-08-17T17:39:44.295 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.3799999999999999e-06 2024-08-17T17:39:44.295 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.295 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.296 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.296 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:01.542177+0000", 2024-08-17T17:39:44.296 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0069705059999999996 2024-08-17T17:39:44.296 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.296 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.296 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:39:44.296 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:01.546819+0000", 2024-08-17T17:39:44.296 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.004641615 2024-08-17T17:39:44.296 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.296 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.296 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.296 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:01.601933+0000", 2024-08-17T17:39:44.296 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.055114682999999998 2024-08-17T17:39:44.297 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.297 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.297 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for rw locks", 2024-08-17T17:39:44.297 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:01.606755+0000", 2024-08-17T17:39:44.297 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0048213279999999997 2024-08-17T17:39:44.297 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.297 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.297 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.297 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:02.885598+0000", 2024-08-17T17:39:44.297 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2788435849999999 2024-08-17T17:39:44.297 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.297 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.298 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.298 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:02.889814+0000", 2024-08-17T17:39:44.298 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0042160230000000002 2024-08-17T17:39:44.298 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.298 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.298 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.298 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:02.897780+0000", 2024-08-17T17:39:44.298 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0079658320000000008 2024-08-17T17:39:44.298 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.298 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.298 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.298 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:02.960881+0000", 2024-08-17T17:39:44.299 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.063101298 2024-08-17T17:39:44.299 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.299 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.299 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.299 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:02.963144+0000", 2024-08-17T17:39:44.299 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0022626959999999998 2024-08-17T17:39:44.299 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.299 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.299 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.299 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:02.965774+0000", 2024-08-17T17:39:44.299 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0026295350000000001 2024-08-17T17:39:44.299 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.299 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.300 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.300 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:02.969839+0000", 2024-08-17T17:39:44.300 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0040655719999999999 2024-08-17T17:39:44.300 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.300 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.300 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.301 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:02.981054+0000", 2024-08-17T17:39:44.301 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011215105 2024-08-17T17:39:44.301 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.301 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.301 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.301 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.301 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.301 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:69307 11.8 11:16713e8e:::benchmark_data_smithi063_328699_object4864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1403)", 2024-08-17T17:39:44.301 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:37:04.279065+0000", 2024-08-17T17:39:44.301 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 159.936059497, 2024-08-17T17:39:44.302 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4943391109999999, 2024-08-17T17:39:44.302 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.302 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.302 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.302 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.302 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.302 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 69307 2024-08-17T17:39:44.302 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.302 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.302 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.302 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.302 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.279065+0000", 2024-08-17T17:39:44.302 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.303 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.303 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.303 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.303 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.279065+0000", 2024-08-17T17:39:44.303 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.303 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.303 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.303 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.303 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.279064+0000", 2024-08-17T17:39:44.303 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:39:44.303 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.303 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.303 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.304 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.279070+0000", 2024-08-17T17:39:44.304 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.3419999999999998e-06 2024-08-17T17:39:44.304 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.304 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.304 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.304 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.279080+0000", 2024-08-17T17:39:44.304 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.0495e-05 2024-08-17T17:39:44.304 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.304 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.304 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.304 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.279087+0000", 2024-08-17T17:39:44.304 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2910000000000002e-06 2024-08-17T17:39:44.304 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.304 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.305 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.305 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.505912+0000", 2024-08-17T17:39:44.305 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.22682419300000001 2024-08-17T17:39:44.305 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.305 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.305 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:39:44.305 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.510423+0000", 2024-08-17T17:39:44.305 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0045116599999999998 2024-08-17T17:39:44.305 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.305 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.305 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.305 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.546920+0000", 2024-08-17T17:39:44.305 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.036496328000000001 2024-08-17T17:39:44.305 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.305 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.306 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for rw locks", 2024-08-17T17:39:44.306 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.552072+0000", 2024-08-17T17:39:44.306 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0051527739999999997 2024-08-17T17:39:44.306 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.306 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.306 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.306 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.715270+0000", 2024-08-17T17:39:44.306 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1631980230000001 2024-08-17T17:39:44.306 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.306 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.306 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.306 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.719791+0000", 2024-08-17T17:39:44.306 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0045203279999999997 2024-08-17T17:39:44.307 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.307 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.307 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.307 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.726424+0000", 2024-08-17T17:39:44.307 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0066330399999999998 2024-08-17T17:39:44.307 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.307 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.307 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.307 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.755958+0000", 2024-08-17T17:39:44.307 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.029534079000000001 2024-08-17T17:39:44.307 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.307 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.307 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.308 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.756383+0000", 2024-08-17T17:39:44.308 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00042531600000000002 2024-08-17T17:39:44.308 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.308 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.308 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.308 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.758544+0000", 2024-08-17T17:39:44.308 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0021610650000000002 2024-08-17T17:39:44.308 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.308 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.308 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.308 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.761352+0000", 2024-08-17T17:39:44.308 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0028079089999999999 2024-08-17T17:39:44.308 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.308 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.309 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.309 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.773404+0000", 2024-08-17T17:39:44.309 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.012052225999999999 2024-08-17T17:39:44.309 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.309 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.309 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.309 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.309 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.309 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:69331 11.8 11:1763306e:::benchmark_data_smithi063_328699_object4888:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1403)", 2024-08-17T17:39:44.309 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:37:04.462438+0000", 2024-08-17T17:39:44.309 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 159.75268698599999, 2024-08-17T17:39:44.309 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.465448759, 2024-08-17T17:39:44.309 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.309 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.309 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.310 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.310 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.310 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 69331 2024-08-17T17:39:44.310 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.310 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.310 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.310 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.310 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.462438+0000", 2024-08-17T17:39:44.310 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.310 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.310 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.310 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.310 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.462438+0000", 2024-08-17T17:39:44.310 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.311 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.311 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.311 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.311 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.462436+0000", 2024-08-17T17:39:44.311 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999976 2024-08-17T17:39:44.311 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.311 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.311 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.311 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.462445+0000", 2024-08-17T17:39:44.311 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.4860000000000003e-06 2024-08-17T17:39:44.311 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.311 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.311 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.311 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.462459+0000", 2024-08-17T17:39:44.311 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4317e-05 2024-08-17T17:39:44.312 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.312 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.312 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.312 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.462469+0000", 2024-08-17T17:39:44.312 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.5419999999999994e-06 2024-08-17T17:39:44.312 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.312 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.312 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.312 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.511143+0000", 2024-08-17T17:39:44.312 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.048674596000000001 2024-08-17T17:39:44.312 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.312 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.312 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:39:44.312 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.516614+0000", 2024-08-17T17:39:44.312 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.005470964 2024-08-17T17:39:44.313 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.313 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.313 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.313 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.576058+0000", 2024-08-17T17:39:44.313 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.059443226000000002 2024-08-17T17:39:44.313 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.313 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.313 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for rw locks", 2024-08-17T17:39:44.313 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:04.580860+0000", 2024-08-17T17:39:44.313 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0048025619999999998 2024-08-17T17:39:44.313 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.313 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.313 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.313 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.817329+0000", 2024-08-17T17:39:44.314 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2364691249999999 2024-08-17T17:39:44.314 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.314 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.314 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.314 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.823907+0000", 2024-08-17T17:39:44.314 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0065781149999999998 2024-08-17T17:39:44.314 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.314 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.314 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.314 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.834686+0000", 2024-08-17T17:39:44.314 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010778307000000001 2024-08-17T17:39:44.314 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.314 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.315 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.315 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.870620+0000", 2024-08-17T17:39:44.315 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.035934031999999998 2024-08-17T17:39:44.315 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.315 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.315 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.315 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.912905+0000", 2024-08-17T17:39:44.315 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.042285659000000003 2024-08-17T17:39:44.315 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.315 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.315 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.315 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.914258+0000", 2024-08-17T17:39:44.315 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00135246 2024-08-17T17:39:44.316 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.316 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.316 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.316 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.916325+0000", 2024-08-17T17:39:44.316 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0020669149999999999 2024-08-17T17:39:44.316 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.316 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.316 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.316 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:37:05.927886+0000", 2024-08-17T17:39:44.316 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011561599000000001 2024-08-17T17:39:44.316 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.316 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.316 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.316 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.317 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.317 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:79371 11.8 11:16ba3828:::benchmark_data_smithi063_328699_object14928:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1415)", 2024-08-17T17:39:44.317 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:38:07.642373+0000", 2024-08-17T17:39:44.317 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 96.572752101999995, 2024-08-17T17:39:44.317 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.7079735920000001, 2024-08-17T17:39:44.317 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.317 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.317 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.317 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.317 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.317 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 79371 2024-08-17T17:39:44.317 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.317 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.317 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.317 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.318 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:07.642373+0000", 2024-08-17T17:39:44.318 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.318 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.318 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.318 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.318 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:07.642373+0000", 2024-08-17T17:39:44.318 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.318 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.318 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.318 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.318 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:07.642372+0000", 2024-08-17T17:39:44.318 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:39:44.318 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.319 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.319 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.319 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:07.642376+0000", 2024-08-17T17:39:44.319 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1849999999999997e-06 2024-08-17T17:39:44.319 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.319 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.319 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.319 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:07.642384+0000", 2024-08-17T17:39:44.319 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.6329999999999996e-06 2024-08-17T17:39:44.319 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.319 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.319 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.319 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:07.642388+0000", 2024-08-17T17:39:44.319 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.1529999999999999e-06 2024-08-17T17:39:44.320 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.320 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.320 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.320 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:07.656829+0000", 2024-08-17T17:39:44.320 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.01444149 2024-08-17T17:39:44.320 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.320 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.320 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:39:44.320 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:07.659535+0000", 2024-08-17T17:39:44.320 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0027061310000000001 2024-08-17T17:39:44.320 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.320 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.321 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.321 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:08.154247+0000", 2024-08-17T17:39:44.321 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.49471200199999998 2024-08-17T17:39:44.321 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.321 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.321 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for rw locks", 2024-08-17T17:39:44.321 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:08.157996+0000", 2024-08-17T17:39:44.321 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0037488650000000001 2024-08-17T17:39:44.321 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.321 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.321 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.321 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:09.278210+0000", 2024-08-17T17:39:44.321 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.120214102 2024-08-17T17:39:44.322 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.322 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.322 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.322 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:09.282024+0000", 2024-08-17T17:39:44.322 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0038141 2024-08-17T17:39:44.322 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.322 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.322 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.322 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:09.290789+0000", 2024-08-17T17:39:44.322 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0087647449999999991 2024-08-17T17:39:44.322 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.323 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.323 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.323 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:09.319902+0000", 2024-08-17T17:39:44.323 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.029112658999999999 2024-08-17T17:39:44.323 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.323 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.323 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.323 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:09.335936+0000", 2024-08-17T17:39:44.323 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.016034467 2024-08-17T17:39:44.323 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.323 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.323 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.323 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:09.337024+0000", 2024-08-17T17:39:44.323 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001087583 2024-08-17T17:39:44.324 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.324 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.324 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.324 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:09.338878+0000", 2024-08-17T17:39:44.324 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001853644 2024-08-17T17:39:44.324 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.324 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.324 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.324 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:09.350346+0000", 2024-08-17T17:39:44.324 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.011468608999999999 2024-08-17T17:39:44.324 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.324 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.324 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.324 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.324 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.325 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:85257 11.8 11:1ebaae2e:::benchmark_data_smithi063_328699_object20814:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1420)", 2024-08-17T17:39:44.325 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:38:41.463416+0000", 2024-08-17T17:39:44.325 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 62.751708610999998, 2024-08-17T17:39:44.325 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 2.051192098, 2024-08-17T17:39:44.325 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.325 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.325 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.325 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.325 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.325 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 85257 2024-08-17T17:39:44.325 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.325 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.325 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.325 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.326 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:41.463416+0000", 2024-08-17T17:39:44.326 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.326 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.326 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.326 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.326 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:41.463416+0000", 2024-08-17T17:39:44.326 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.326 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.326 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.326 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.326 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:41.463415+0000", 2024-08-17T17:39:44.326 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:39:44.326 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.326 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.326 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.327 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:41.463420+0000", 2024-08-17T17:39:44.327 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.5199999999999997e-06 2024-08-17T17:39:44.327 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.327 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.327 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.327 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:41.463434+0000", 2024-08-17T17:39:44.327 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3084e-05 2024-08-17T17:39:44.327 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.327 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.327 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.327 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:41.463441+0000", 2024-08-17T17:39:44.327 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.9990000000000002e-06 2024-08-17T17:39:44.327 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.327 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.328 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.328 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:41.483975+0000", 2024-08-17T17:39:44.328 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.020534899999999998 2024-08-17T17:39:44.328 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.328 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.328 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:39:44.328 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:41.486327+0000", 2024-08-17T17:39:44.328 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00235199 2024-08-17T17:39:44.328 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.328 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.328 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.328 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:42.150893+0000", 2024-08-17T17:39:44.328 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.66456510000000002 2024-08-17T17:39:44.329 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.329 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.329 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for rw locks", 2024-08-17T17:39:44.329 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:42.154534+0000", 2024-08-17T17:39:44.329 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0036415940000000002 2024-08-17T17:39:44.329 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.329 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.329 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.329 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:43.442139+0000", 2024-08-17T17:39:44.329 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2876053670000001 2024-08-17T17:39:44.329 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.329 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.329 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.329 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:43.445000+0000", 2024-08-17T17:39:44.330 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0028605530000000001 2024-08-17T17:39:44.330 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.330 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.330 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.330 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:43.449884+0000", 2024-08-17T17:39:44.330 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0048835659999999998 2024-08-17T17:39:44.330 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.330 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.330 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.330 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:43.499912+0000", 2024-08-17T17:39:44.330 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.050028759999999999 2024-08-17T17:39:44.330 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.330 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.331 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.331 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:43.501210+0000", 2024-08-17T17:39:44.331 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012977259999999999 2024-08-17T17:39:44.331 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.331 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.331 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.331 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:43.502234+0000", 2024-08-17T17:39:44.331 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001023843 2024-08-17T17:39:44.331 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.331 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.331 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.331 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:43.504463+0000", 2024-08-17T17:39:44.331 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0022293429999999999 2024-08-17T17:39:44.331 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.332 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.332 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.332 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:43.514608+0000", 2024-08-17T17:39:44.332 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010144887999999999 2024-08-17T17:39:44.332 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.332 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.332 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.332 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.332 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.332 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:86806 11.8 11:11cd46a7:::benchmark_data_smithi063_328699_object22363:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1421)", 2024-08-17T17:39:44.332 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:38:49.517407+0000", 2024-08-17T17:39:44.332 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 54.697717587, 2024-08-17T17:39:44.332 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5927827990000001, 2024-08-17T17:39:44.332 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.332 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.333 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.333 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.333 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.333 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 86806 2024-08-17T17:39:44.333 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.333 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.333 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.333 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.333 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:49.517407+0000", 2024-08-17T17:39:44.333 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.333 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.333 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.333 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.333 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:49.517407+0000", 2024-08-17T17:39:44.333 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.334 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.334 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.334 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.334 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:49.517406+0000", 2024-08-17T17:39:44.334 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:44.334 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.334 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.334 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.334 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:49.517413+0000", 2024-08-17T17:39:44.334 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 6.8739999999999997e-06 2024-08-17T17:39:44.334 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.334 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.334 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.334 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:49.517425+0000", 2024-08-17T17:39:44.335 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.2618e-05 2024-08-17T17:39:44.335 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.335 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.335 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.335 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:49.517433+0000", 2024-08-17T17:39:44.335 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.2880000000000004e-06 2024-08-17T17:39:44.335 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.335 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.335 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.335 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:49.548448+0000", 2024-08-17T17:39:44.335 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.031015740999999999 2024-08-17T17:39:44.335 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.335 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.336 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:39:44.336 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:49.549968+0000", 2024-08-17T17:39:44.336 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015199409999999999 2024-08-17T17:39:44.336 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.336 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.336 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.336 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:50.147018+0000", 2024-08-17T17:39:44.336 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.59705003999999995 2024-08-17T17:39:44.336 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.336 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.336 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for rw locks", 2024-08-17T17:39:44.336 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:50.151821+0000", 2024-08-17T17:39:44.336 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0048024579999999999 2024-08-17T17:39:44.336 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.336 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.337 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.337 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:51.059646+0000", 2024-08-17T17:39:44.337 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.90782546799999997 2024-08-17T17:39:44.337 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.337 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.337 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.337 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:51.062357+0000", 2024-08-17T17:39:44.337 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0027108779999999999 2024-08-17T17:39:44.337 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.337 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.337 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.337 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:51.067167+0000", 2024-08-17T17:39:44.337 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0048095250000000003 2024-08-17T17:39:44.337 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.338 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.338 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.338 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:51.095829+0000", 2024-08-17T17:39:44.338 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.028662823 2024-08-17T17:39:44.338 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.338 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.338 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.338 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:51.096303+0000", 2024-08-17T17:39:44.338 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00047358799999999999 2024-08-17T17:39:44.338 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.338 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.338 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.338 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:51.098177+0000", 2024-08-17T17:39:44.339 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001873687 2024-08-17T17:39:44.339 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.339 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.339 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.339 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:51.099745+0000", 2024-08-17T17:39:44.339 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0015682020000000001 2024-08-17T17:39:44.339 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.339 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.339 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.339 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:38:51.110190+0000", 2024-08-17T17:39:44.339 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010445022 2024-08-17T17:39:44.339 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.339 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.339 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.340 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.340 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.340 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:89825 11.8 11:1a452207:::benchmark_data_smithi063_328699_object25382:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1422)", 2024-08-17T17:39:44.340 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:39:03.425941+0000", 2024-08-17T17:39:44.340 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 40.789183600999998, 2024-08-17T17:39:44.340 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.523634497, 2024-08-17T17:39:44.340 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.340 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.340 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.340 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.340 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.340 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 89825 2024-08-17T17:39:44.340 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.340 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.340 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.341 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.341 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.425941+0000", 2024-08-17T17:39:44.341 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.341 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.341 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.341 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.341 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.425941+0000", 2024-08-17T17:39:44.341 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.341 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.341 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.341 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.341 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.425939+0000", 2024-08-17T17:39:44.341 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999981 2024-08-17T17:39:44.341 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.342 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.342 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.342 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.425949+0000", 2024-08-17T17:39:44.342 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.7010000000000008e-06 2024-08-17T17:39:44.342 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.342 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.342 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.342 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.425964+0000", 2024-08-17T17:39:44.342 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5041e-05 2024-08-17T17:39:44.342 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.342 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.342 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.342 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.425973+0000", 2024-08-17T17:39:44.342 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.2830000000000007e-06 2024-08-17T17:39:44.343 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.343 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.343 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.343 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.465678+0000", 2024-08-17T17:39:44.343 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.039704128999999998 2024-08-17T17:39:44.343 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.343 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.343 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:39:44.343 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.467327+0000", 2024-08-17T17:39:44.343 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0016498330000000001 2024-08-17T17:39:44.343 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.343 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.343 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.343 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.745664+0000", 2024-08-17T17:39:44.344 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.27833632899999999 2024-08-17T17:39:44.344 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.344 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.344 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for rw locks", 2024-08-17T17:39:44.344 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.748749+0000", 2024-08-17T17:39:44.344 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0030849789999999998 2024-08-17T17:39:44.344 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.344 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.344 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.344 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.921051+0000", 2024-08-17T17:39:44.344 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.172302154 2024-08-17T17:39:44.344 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.344 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.344 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.345 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.923929+0000", 2024-08-17T17:39:44.345 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002877803 2024-08-17T17:39:44.345 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.345 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.345 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.345 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.928412+0000", 2024-08-17T17:39:44.345 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0044837360000000003 2024-08-17T17:39:44.345 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.345 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.345 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.345 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.937812+0000", 2024-08-17T17:39:44.345 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0093994070000000002 2024-08-17T17:39:44.345 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.346 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.346 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.346 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.939051+0000", 2024-08-17T17:39:44.346 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0012389199999999999 2024-08-17T17:39:44.346 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.346 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.346 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.346 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.939447+0000", 2024-08-17T17:39:44.346 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000396291 2024-08-17T17:39:44.346 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.346 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.346 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.347 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.941360+0000", 2024-08-17T17:39:44.347 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001913159 2024-08-17T17:39:44.347 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.347 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.347 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.347 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.949576+0000", 2024-08-17T17:39:44.347 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0082154740000000004 2024-08-17T17:39:44.347 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.347 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.347 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.347 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.347 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.347 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:89913 11.8 11:1dbfb777:::benchmark_data_smithi063_328699_object25470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1422)", 2024-08-17T17:39:44.347 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:39:03.845672+0000", 2024-08-17T17:39:44.348 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 40.369452629000001, 2024-08-17T17:39:44.348 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.6210061009999999, 2024-08-17T17:39:44.348 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.348 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.348 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.348 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.348 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.348 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 89913 2024-08-17T17:39:44.348 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.348 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.348 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.348 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.348 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.845672+0000", 2024-08-17T17:39:44.348 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.349 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.349 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.349 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.349 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.845672+0000", 2024-08-17T17:39:44.349 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.349 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.349 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.349 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.349 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.845671+0000", 2024-08-17T17:39:44.349 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:44.349 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.349 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.350 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.350 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.845678+0000", 2024-08-17T17:39:44.350 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.1289999999999998e-06 2024-08-17T17:39:44.350 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.350 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.350 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.350 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.845692+0000", 2024-08-17T17:39:44.350 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.3869999999999999e-05 2024-08-17T17:39:44.350 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.350 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.350 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.350 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.845699+0000", 2024-08-17T17:39:44.350 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.7989999999999994e-06 2024-08-17T17:39:44.351 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.351 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.351 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.351 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.899464+0000", 2024-08-17T17:39:44.351 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.053764640000000002 2024-08-17T17:39:44.351 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.351 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.351 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:39:44.351 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:03.901084+0000", 2024-08-17T17:39:44.351 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001619498 2024-08-17T17:39:44.351 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.351 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.351 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.351 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.351530+0000", 2024-08-17T17:39:44.352 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.45044604900000002 2024-08-17T17:39:44.352 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.352 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.352 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for rw locks", 2024-08-17T17:39:44.352 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.354580+0000", 2024-08-17T17:39:44.352 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0030503700000000002 2024-08-17T17:39:44.352 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.352 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.352 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.352 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.422862+0000", 2024-08-17T17:39:44.352 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.068282143 2024-08-17T17:39:44.352 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.352 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.353 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.353 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.425809+0000", 2024-08-17T17:39:44.353 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0029465670000000002 2024-08-17T17:39:44.353 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.353 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.353 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.353 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.431089+0000", 2024-08-17T17:39:44.353 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.005280674 2024-08-17T17:39:44.353 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.353 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.353 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.353 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.454442+0000", 2024-08-17T17:39:44.353 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.023353046999999998 2024-08-17T17:39:44.354 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.354 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.354 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.354 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.454849+0000", 2024-08-17T17:39:44.354 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00040690799999999999 2024-08-17T17:39:44.354 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.354 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.354 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.354 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.456830+0000", 2024-08-17T17:39:44.354 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0019804969999999999 2024-08-17T17:39:44.354 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.354 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.354 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.355 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.458811+0000", 2024-08-17T17:39:44.355 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001980922 2024-08-17T17:39:44.355 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.355 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.355 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.355 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.466678+0000", 2024-08-17T17:39:44.355 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0078673739999999999 2024-08-17T17:39:44.355 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.355 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.355 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.355 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.355 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.355 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:90045 11.8 11:1135cc31:::benchmark_data_smithi063_328699_object25602:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1422)", 2024-08-17T17:39:44.356 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:39:04.433245+0000", 2024-08-17T17:39:44.356 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 39.781879445999998, 2024-08-17T17:39:44.356 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.4765460939999999, 2024-08-17T17:39:44.356 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.356 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.356 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.356 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.356 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:44.357 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.357 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 90045 2024-08-17T17:39:44.357 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.357 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.357 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.357 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.358 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.433245+0000", 2024-08-17T17:39:44.358 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.358 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.358 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.358 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.358 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.433245+0000", 2024-08-17T17:39:44.358 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.358 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.358 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.358 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.358 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.433244+0000", 2024-08-17T17:39:44.358 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.999999 2024-08-17T17:39:44.359 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.359 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.359 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.359 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.433250+0000", 2024-08-17T17:39:44.359 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 5.4140000000000002e-06 2024-08-17T17:39:44.359 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.359 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.359 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.359 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.433259+0000", 2024-08-17T17:39:44.359 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 9.0880000000000001e-06 2024-08-17T17:39:44.359 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.359 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.360 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.360 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.433263+0000", 2024-08-17T17:39:44.360 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4.7520000000000004e-06 2024-08-17T17:39:44.360 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.360 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.360 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.360 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.444056+0000", 2024-08-17T17:39:44.360 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.010792151999999999 2024-08-17T17:39:44.360 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.360 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.360 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:39:44.360 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.445439+0000", 2024-08-17T17:39:44.360 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.00138349 2024-08-17T17:39:44.360 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.361 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.361 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.361 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.974814+0000", 2024-08-17T17:39:44.361 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.52937497099999997 2024-08-17T17:39:44.361 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.361 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.361 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for rw locks", 2024-08-17T17:39:44.361 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:04.979227+0000", 2024-08-17T17:39:44.361 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0044124869999999997 2024-08-17T17:39:44.361 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.361 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.361 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.361 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.800285+0000", 2024-08-17T17:39:44.361 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.82105845499999996 2024-08-17T17:39:44.362 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.362 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.362 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.362 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.802959+0000", 2024-08-17T17:39:44.362 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002674006 2024-08-17T17:39:44.362 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.362 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.362 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.362 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.807432+0000", 2024-08-17T17:39:44.362 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0044732039999999997 2024-08-17T17:39:44.362 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.362 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.362 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.859265+0000", 2024-08-17T17:39:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.051832456999999998 2024-08-17T17:39:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.900323+0000", 2024-08-17T17:39:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.041057934999999997 2024-08-17T17:39:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.901480+0000", 2024-08-17T17:39:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0011572189999999999 2024-08-17T17:39:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.363 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.364 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.364 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.902911+0000", 2024-08-17T17:39:44.364 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0014309520000000001 2024-08-17T17:39:44.364 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.364 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.364 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.364 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:05.909791+0000", 2024-08-17T17:39:44.364 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.006880588 2024-08-17T17:39:44.364 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.364 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.364 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.364 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.364 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.364 INFO:teuthology.orchestra.run.smithi162.stdout: "description": "osd_op(client.46088.0:98746 11.8 11:18ceaa0c:::benchmark_data_smithi063_328699_object34303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1426)", 2024-08-17T17:39:44.365 INFO:teuthology.orchestra.run.smithi162.stdout: "initiated_at": "2024-08-17T17:39:40.783362+0000", 2024-08-17T17:39:44.365 INFO:teuthology.orchestra.run.smithi162.stdout: "age": 3.431763085, 2024-08-17T17:39:44.365 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.488006071, 2024-08-17T17:39:44.365 INFO:teuthology.orchestra.run.smithi162.stdout: "type_data": { 2024-08-17T17:39:44.365 INFO:teuthology.orchestra.run.smithi162.stdout: "flag_point": "commit sent; apply or cleanup", 2024-08-17T17:39:44.365 INFO:teuthology.orchestra.run.smithi162.stdout: "client_info": { 2024-08-17T17:39:44.365 INFO:teuthology.orchestra.run.smithi162.stdout: "client": "client.46088", 2024-08-17T17:39:44.365 INFO:teuthology.orchestra.run.smithi162.stdout: "client_addr": "172.21.15.63:0/4215654149", 2024-08-17T17:39:44.365 INFO:teuthology.orchestra.run.smithi162.stdout: "tid": 98746 2024-08-17T17:39:44.365 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.365 INFO:teuthology.orchestra.run.smithi162.stdout: "events": [ 2024-08-17T17:39:44.365 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.365 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "initiated", 2024-08-17T17:39:44.365 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:40.783362+0000", 2024-08-17T17:39:44.365 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.366 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.366 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.366 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "throttled", 2024-08-17T17:39:44.366 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:40.783362+0000", 2024-08-17T17:39:44.366 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0 2024-08-17T17:39:44.366 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.366 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.366 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "header_read", 2024-08-17T17:39:44.366 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:40.783360+0000", 2024-08-17T17:39:44.366 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 4294967295.9999986 2024-08-17T17:39:44.366 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.366 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.367 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "all_read", 2024-08-17T17:39:44.367 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:40.783368+0000", 2024-08-17T17:39:44.367 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 8.2819999999999996e-06 2024-08-17T17:39:44.367 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.367 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.367 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "dispatched", 2024-08-17T17:39:44.367 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:40.783384+0000", 2024-08-17T17:39:44.367 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.5271999999999999e-05 2024-08-17T17:39:44.367 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.367 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.367 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "queued_for_pg", 2024-08-17T17:39:44.367 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:40.783392+0000", 2024-08-17T17:39:44.367 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 7.8350000000000004e-06 2024-08-17T17:39:44.367 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.368 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.368 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.368 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:40.815725+0000", 2024-08-17T17:39:44.368 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.032333247000000002 2024-08-17T17:39:44.368 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.368 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.368 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for missing object", 2024-08-17T17:39:44.368 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:40.816854+0000", 2024-08-17T17:39:44.368 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001129122 2024-08-17T17:39:44.368 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.368 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.368 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.369 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:41.052704+0000", 2024-08-17T17:39:44.369 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.23585050599999999 2024-08-17T17:39:44.369 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.369 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.369 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for rw locks", 2024-08-17T17:39:44.369 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:41.056042+0000", 2024-08-17T17:39:44.369 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0033378729999999999 2024-08-17T17:39:44.369 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.369 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.369 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "reached_pg", 2024-08-17T17:39:44.369 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:42.219987+0000", 2024-08-17T17:39:44.369 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 1.1639450570000001 2024-08-17T17:39:44.369 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.370 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.370 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "started", 2024-08-17T17:39:44.370 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:42.222197+0000", 2024-08-17T17:39:44.370 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.002209468 2024-08-17T17:39:44.370 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.370 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.370 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "waiting for subops from 3,5", 2024-08-17T17:39:44.370 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:42.226348+0000", 2024-08-17T17:39:44.370 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0041507619999999997 2024-08-17T17:39:44.370 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.370 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.370 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "op_commit", 2024-08-17T17:39:44.370 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:42.259538+0000", 2024-08-17T17:39:44.370 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.033190452000000002 2024-08-17T17:39:44.371 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.371 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.371 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.371 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:42.264235+0000", 2024-08-17T17:39:44.371 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0046968139999999997 2024-08-17T17:39:44.371 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.371 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.371 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "sub_op_commit_rec", 2024-08-17T17:39:44.371 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:42.265220+0000", 2024-08-17T17:39:44.371 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.000984836 2024-08-17T17:39:44.371 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.371 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.371 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "commit_sent", 2024-08-17T17:39:44.372 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:42.266374+0000", 2024-08-17T17:39:44.372 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.001154314 2024-08-17T17:39:44.372 INFO:teuthology.orchestra.run.smithi162.stdout: }, 2024-08-17T17:39:44.372 INFO:teuthology.orchestra.run.smithi162.stdout: { 2024-08-17T17:39:44.372 INFO:teuthology.orchestra.run.smithi162.stdout: "event": "done", 2024-08-17T17:39:44.372 INFO:teuthology.orchestra.run.smithi162.stdout: "time": "2024-08-17T17:39:42.271368+0000", 2024-08-17T17:39:44.372 INFO:teuthology.orchestra.run.smithi162.stdout: "duration": 0.0049937050000000002 2024-08-17T17:39:44.372 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.372 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.372 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.372 INFO:teuthology.orchestra.run.smithi162.stdout: } 2024-08-17T17:39:44.372 INFO:teuthology.orchestra.run.smithi162.stdout: ] 2024-08-17T17:39:44.372 INFO:teuthology.orchestra.run.smithi162.stdout:} 2024-08-17T17:39:44.396 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:44.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:44.498 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:44.526 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:44.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:44.627 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:44.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.652 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:44.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:44.753 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:44.782 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:44.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:44.884 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:44.907 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:44.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:44.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:45.009 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:45.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.040 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:45.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:45.141 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:45.166 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:45.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:45.267 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:45.296 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:45.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:45.398 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:45.409 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-08-17T17:39:45.424 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:45.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:45.526 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:45.553 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:45.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:45.655 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:45.679 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:45.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:45.781 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:45.799 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:39:45.810 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:45.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:45.912 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:45.934 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:45.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:45.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:46.024 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:45 smithi162 ceph-mon[104668]: pgmap v2261: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 254 op/s; 11774/89691 objects degraded (13.127%); 2471/89691 objects misplaced (2.755%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:46.024 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:45 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2764646029' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:46.036 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:46.046 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:45 smithi063 ceph-mon[114756]: pgmap v2261: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 254 op/s; 11774/89691 objects degraded (13.127%); 2471/89691 objects misplaced (2.755%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:46.046 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:45 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2764646029' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:46.046 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:45 smithi063 ceph-mon[109843]: pgmap v2261: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 254 op/s; 11774/89691 objects degraded (13.127%); 2471/89691 objects misplaced (2.755%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:39:46.046 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:45 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2764646029' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:39:46.059 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:46.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:46.161 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:46.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.187 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:46.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:46.288 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:46.321 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:46.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:46.423 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:46.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.453 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:46.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:46.555 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:46.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.588 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:46.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:46.689 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:46.717 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:46.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:46.819 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:46.832 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:46 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:46.832 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:46 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2764646029' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:39:46.832 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:46 smithi162 ceph-mon[104668]: osdmap e1427: 8 total, 8 up, 8 in 2024-08-17T17:39:46.832 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:46.844 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:46.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:46.946 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:46.970 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:46.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:46.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:46 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:46 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2764646029' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:39:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:46 smithi063 ceph-mon[114756]: osdmap e1427: 8 total, 8 up, 8 in 2024-08-17T17:39:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:46 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:39:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:46 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2764646029' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:39:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:46 smithi063 ceph-mon[109843]: osdmap e1427: 8 total, 8 up, 8 in 2024-08-17T17:39:47.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:39:47.073 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:47.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.103 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:47.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:47.204 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:47.229 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:47.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:47.331 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:47.357 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:47.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:47.461 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:47.487 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:47.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:47.572 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2260,"stamp":"2024-08-17T17:39:42.665543+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1963706033,"num_objects":29981,"num_object_clones":0,"num_object_copies":89943,"num_objects_missing_on_primary":2027,"num_objects_missing":2027,"num_objects_degraded":11774,"num_objects_misplaced":2471,"num_objects_unfound":0,"num_objects_dirty":29981,"num_whiteouts":0,"num_read":28,"num_read_kb":62,"num_write":99006,"num_write_kb":4124377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31881,"num_bytes_recovered":1936462557,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100405,"ondisk_log_size":100405,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":375,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12935768,"kb_used_data":6953300,"kb_used_omap":298,"kb_used_meta":5982101,"kb_avail":737025448,"statfs":{"total":767960285184,"available":754714058752,"internally_reserved":0,"allocated":7120179200,"data_stored":7098747763,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":6125671953},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,4,0,3,1,6,5,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-166461440,"num_objects":-2540,"num_object_clones":0,"num_object_copies":-7620,"num_objects_missing_on_primary":-494,"num_objects_missing":-494,"num_objects_degraded":-938,"num_objects_misplaced":-331,"num_objects_unfound":0,"num_objects_dirty":-2540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2540,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":634,"num_bytes_recovered":41549824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002595"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":59,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.781866+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:39:37.781866+0000","last_peered":"2024-08-17T17:39:37.781866+0000","last_clean":"2024-08-17T17:39:37.781866+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:39:37.781866+0000","last_undegraded":"2024-08-17T17:39:37.781866+0000","last_fullsized":"2024-08-17T17:39:37.781866+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1441,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.781556+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:39:37.781556+0000","last_peered":"2024-08-17T17:39:37.781556+0000","last_clean":"2024-08-17T17:39:37.781556+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:39:37.781556+0000","last_undegraded":"2024-08-17T17:39:37.781556+0000","last_fullsized":"2024-08-17T17:39:37.781556+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1755,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.778826+0000","last_change":"2024-08-17T17:36:15.771150+0000","last_active":"2024-08-17T17:39:29.778826+0000","last_peered":"2024-08-17T17:39:29.778826+0000","last_clean":"2024-08-17T17:39:29.778826+0000","last_became_active":"2024-08-17T17:36:15.770520+0000","last_became_peered":"2024-08-17T17:36:15.770520+0000","last_unstale":"2024-08-17T17:39:29.778826+0000","last_undegraded":"2024-08-17T17:39:29.778826+0000","last_fullsized":"2024-08-17T17:39:29.778826+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:17.552796+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":90,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.761569+0000","last_change":"2024-08-17T17:36:12.756709+0000","last_active":"2024-08-17T17:39:29.761569+0000","last_peered":"2024-08-17T17:39:29.761569+0000","last_clean":"2024-08-17T17:39:29.761569+0000","last_became_active":"2024-08-17T17:36:12.756385+0000","last_became_peered":"2024-08-17T17:36:12.756385+0000","last_unstale":"2024-08-17T17:39:29.761569+0000","last_undegraded":"2024-08-17T17:39:29.761569+0000","last_fullsized":"2024-08-17T17:39:29.761569+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:21.420282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":143,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.789326+0000","last_change":"2024-08-17T17:36:12.762800+0000","last_active":"2024-08-17T17:39:37.789326+0000","last_peered":"2024-08-17T17:39:37.789326+0000","last_clean":"2024-08-17T17:39:37.789326+0000","last_became_active":"2024-08-17T17:36:12.759977+0000","last_became_peered":"2024-08-17T17:36:12.759977+0000","last_unstale":"2024-08-17T17:39:37.789326+0000","last_undegraded":"2024-08-17T17:39:37.789326+0000","last_fullsized":"2024-08-17T17:39:37.789326+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:02.162681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1635,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.761105+0000","last_change":"2024-08-17T17:35:56.281561+0000","last_active":"2024-08-17T17:39:29.761105+0000","last_peered":"2024-08-17T17:39:29.761105+0000","last_clean":"2024-08-17T17:39:29.761105+0000","last_became_active":"2024-08-17T17:35:56.278638+0000","last_became_peered":"2024-08-17T17:35:56.278638+0000","last_unstale":"2024-08-17T17:39:29.761105+0000","last_undegraded":"2024-08-17T17:39:29.761105+0000","last_fullsized":"2024-08-17T17:39:29.761105+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1374,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.774500+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:39:29.774500+0000","last_peered":"2024-08-17T17:39:29.774500+0000","last_clean":"2024-08-17T17:39:29.774500+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:39:29.774500+0000","last_undegraded":"2024-08-17T17:39:29.774500+0000","last_fullsized":"2024-08-17T17:39:29.774500+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":50,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.777917+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:39:29.777917+0000","last_peered":"2024-08-17T17:39:29.777917+0000","last_clean":"2024-08-17T17:39:29.777917+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:39:29.777917+0000","last_undegraded":"2024-08-17T17:39:29.777917+0000","last_fullsized":"2024-08-17T17:39:29.777917+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1419,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773361+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:39:29.773361+0000","last_peered":"2024-08-17T17:39:29.773361+0000","last_clean":"2024-08-17T17:39:29.773361+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:39:29.773361+0000","last_undegraded":"2024-08-17T17:39:29.773361+0000","last_fullsized":"2024-08-17T17:39:29.773361+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1427,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.778349+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:39:29.778349+0000","last_peered":"2024-08-17T17:39:29.778349+0000","last_clean":"2024-08-17T17:39:29.778349+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:39:29.778349+0000","last_undegraded":"2024-08-17T17:39:29.778349+0000","last_fullsized":"2024-08-17T17:39:29.778349+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":717,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.761436+0000","last_change":"2024-08-17T17:36:15.773315+0000","last_active":"2024-08-17T17:39:29.761436+0000","last_peered":"2024-08-17T17:39:29.761436+0000","last_clean":"2024-08-17T17:39:29.761436+0000","last_became_active":"2024-08-17T17:36:15.772961+0000","last_became_peered":"2024-08-17T17:36:15.772961+0000","last_unstale":"2024-08-17T17:39:29.761436+0000","last_undegraded":"2024-08-17T17:39:29.761436+0000","last_fullsized":"2024-08-17T17:39:29.761436+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:10.190221+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":53,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.781592+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:39:37.781592+0000","last_peered":"2024-08-17T17:39:37.781592+0000","last_clean":"2024-08-17T17:39:37.781592+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:39:37.781592+0000","last_undegraded":"2024-08-17T17:39:37.781592+0000","last_fullsized":"2024-08-17T17:39:37.781592+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":143,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.791060+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:39:37.791060+0000","last_peered":"2024-08-17T17:39:37.791060+0000","last_clean":"2024-08-17T17:39:37.791060+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:39:37.791060+0000","last_undegraded":"2024-08-17T17:39:37.791060+0000","last_fullsized":"2024-08-17T17:39:37.791060+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1508,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.780917+0000","last_change":"2024-08-17T17:36:24.225739+0000","last_active":"2024-08-17T17:39:29.780917+0000","last_peered":"2024-08-17T17:39:29.780917+0000","last_clean":"2024-08-17T17:39:29.780917+0000","last_became_active":"2024-08-17T17:36:24.225250+0000","last_became_peered":"2024-08-17T17:36:24.225250+0000","last_unstale":"2024-08-17T17:39:29.780917+0000","last_undegraded":"2024-08-17T17:39:29.780917+0000","last_fullsized":"2024-08-17T17:39:29.780917+0000","mapping_epoch":1393,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1394,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.290656+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1447,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.778122+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:39:29.778122+0000","last_peered":"2024-08-17T17:39:29.778122+0000","last_clean":"2024-08-17T17:39:29.778122+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:39:29.778122+0000","last_undegraded":"2024-08-17T17:39:29.778122+0000","last_fullsized":"2024-08-17T17:39:29.778122+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1854,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.778996+0000","last_change":"2024-08-17T17:36:12.752970+0000","last_active":"2024-08-17T17:39:29.778996+0000","last_peered":"2024-08-17T17:39:29.778996+0000","last_clean":"2024-08-17T17:39:29.778996+0000","last_became_active":"2024-08-17T17:36:12.752311+0000","last_became_peered":"2024-08-17T17:36:12.752311+0000","last_unstale":"2024-08-17T17:39:29.778996+0000","last_undegraded":"2024-08-17T17:39:29.778996+0000","last_fullsized":"2024-08-17T17:39:29.778996+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:38.131419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1802,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773331+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:39:29.773331+0000","last_peered":"2024-08-17T17:39:29.773331+0000","last_clean":"2024-08-17T17:39:29.773331+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:39:29.773331+0000","last_undegraded":"2024-08-17T17:39:29.773331+0000","last_fullsized":"2024-08-17T17:39:29.773331+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1515,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.801325+0000","last_change":"2024-08-17T17:36:18.750494+0000","last_active":"2024-08-17T17:39:37.801325+0000","last_peered":"2024-08-17T17:39:37.801325+0000","last_clean":"2024-08-17T17:39:37.801325+0000","last_became_active":"2024-08-17T17:36:18.750092+0000","last_became_peered":"2024-08-17T17:36:18.750092+0000","last_unstale":"2024-08-17T17:39:37.801325+0000","last_undegraded":"2024-08-17T17:39:37.801325+0000","last_fullsized":"2024-08-17T17:39:37.801325+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:36.599727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1145,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.782435+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:39:37.782435+0000","last_peered":"2024-08-17T17:39:37.782435+0000","last_clean":"2024-08-17T17:39:37.782435+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:39:37.782435+0000","last_undegraded":"2024-08-17T17:39:37.782435+0000","last_fullsized":"2024-08-17T17:39:37.782435+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":969,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:32.449394+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:39:32.449394+0000","last_peered":"2024-08-17T17:39:32.449394+0000","last_clean":"2024-08-17T17:39:32.449394+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:39:32.449394+0000","last_undegraded":"2024-08-17T17:39:32.449394+0000","last_fullsized":"2024-08-17T17:39:32.449394+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":53,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.781355+0000","last_change":"2024-08-17T17:37:11.075505+0000","last_active":"2024-08-17T17:39:37.781355+0000","last_peered":"2024-08-17T17:39:37.781355+0000","last_clean":"2024-08-17T17:39:37.781355+0000","last_became_active":"2024-08-17T17:37:11.075131+0000","last_became_peered":"2024-08-17T17:37:11.075131+0000","last_unstale":"2024-08-17T17:39:37.781355+0000","last_undegraded":"2024-08-17T17:39:37.781355+0000","last_fullsized":"2024-08-17T17:39:37.781355+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:39.672054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1281,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.761927+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:39:29.761927+0000","last_peered":"2024-08-17T17:39:29.761927+0000","last_clean":"2024-08-17T17:39:29.761927+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:39:29.761927+0000","last_undegraded":"2024-08-17T17:39:29.761927+0000","last_fullsized":"2024-08-17T17:39:29.761927+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1659,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.432158+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:39:37.432158+0000","last_peered":"2024-08-17T17:39:37.432158+0000","last_clean":"2024-08-17T17:39:37.432158+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:39:37.432158+0000","last_undegraded":"2024-08-17T17:39:37.432158+0000","last_fullsized":"2024-08-17T17:39:37.432158+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":99,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.794935+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:39:37.794935+0000","last_peered":"2024-08-17T17:39:37.794935+0000","last_clean":"2024-08-17T17:39:37.794935+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:39:37.794935+0000","last_undegraded":"2024-08-17T17:39:37.794935+0000","last_fullsized":"2024-08-17T17:39:37.794935+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1256,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.774162+0000","last_change":"2024-08-17T17:36:12.755873+0000","last_active":"2024-08-17T17:39:29.774162+0000","last_peered":"2024-08-17T17:39:29.774162+0000","last_clean":"2024-08-17T17:39:29.774162+0000","last_became_active":"2024-08-17T17:36:12.755366+0000","last_became_peered":"2024-08-17T17:36:12.755366+0000","last_unstale":"2024-08-17T17:39:29.774162+0000","last_undegraded":"2024-08-17T17:39:29.774162+0000","last_fullsized":"2024-08-17T17:39:29.774162+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:36.523214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1609,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.786321+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:39:29.786321+0000","last_peered":"2024-08-17T17:39:29.786321+0000","last_clean":"2024-08-17T17:39:29.786321+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:39:29.786321+0000","last_undegraded":"2024-08-17T17:39:29.786321+0000","last_fullsized":"2024-08-17T17:39:29.786321+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":793,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.781099+0000","last_change":"2024-08-17T17:35:55.200519+0000","last_active":"2024-08-17T17:39:29.781099+0000","last_peered":"2024-08-17T17:39:29.781099+0000","last_clean":"2024-08-17T17:39:29.781099+0000","last_became_active":"2024-08-17T17:35:55.200230+0000","last_became_peered":"2024-08-17T17:35:55.200230+0000","last_unstale":"2024-08-17T17:39:29.781099+0000","last_undegraded":"2024-08-17T17:39:29.781099+0000","last_fullsized":"2024-08-17T17:39:29.781099+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:33.454851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1433,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:32.449512+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:39:32.449512+0000","last_peered":"2024-08-17T17:39:32.449512+0000","last_clean":"2024-08-17T17:39:32.449512+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:39:32.449512+0000","last_undegraded":"2024-08-17T17:39:32.449512+0000","last_fullsized":"2024-08-17T17:39:32.449512+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1716,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.789456+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:39:37.789456+0000","last_peered":"2024-08-17T17:39:37.789456+0000","last_clean":"2024-08-17T17:39:37.789456+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:39:37.789456+0000","last_undegraded":"2024-08-17T17:39:37.789456+0000","last_fullsized":"2024-08-17T17:39:37.789456+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:22.077229+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":263,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.799663+0000","last_change":"2024-08-17T17:35:33.145476+0000","last_active":"2024-08-17T17:39:37.799663+0000","last_peered":"2024-08-17T17:39:37.799663+0000","last_clean":"2024-08-17T17:39:37.799663+0000","last_became_active":"2024-08-17T17:35:33.145125+0000","last_became_peered":"2024-08-17T17:35:33.145125+0000","last_unstale":"2024-08-17T17:39:37.799663+0000","last_undegraded":"2024-08-17T17:39:37.799663+0000","last_fullsized":"2024-08-17T17:39:37.799663+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:45.298064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":937,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.778730+0000","last_change":"2024-08-17T17:35:56.248612+0000","last_active":"2024-08-17T17:39:29.778730+0000","last_peered":"2024-08-17T17:39:29.778730+0000","last_clean":"2024-08-17T17:39:29.778730+0000","last_became_active":"2024-08-17T17:35:56.244364+0000","last_became_peered":"2024-08-17T17:35:56.244364+0000","last_unstale":"2024-08-17T17:39:29.778730+0000","last_undegraded":"2024-08-17T17:39:29.778730+0000","last_fullsized":"2024-08-17T17:39:29.778730+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:41.064165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":877,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773893+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:39:29.773893+0000","last_peered":"2024-08-17T17:39:29.773893+0000","last_clean":"2024-08-17T17:39:29.773893+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:39:29.773893+0000","last_undegraded":"2024-08-17T17:39:29.773893+0000","last_fullsized":"2024-08-17T17:39:29.773893+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1412,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.778602+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:39:29.778602+0000","last_peered":"2024-08-17T17:39:29.778602+0000","last_clean":"2024-08-17T17:39:29.778602+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:39:29.778602+0000","last_undegraded":"2024-08-17T17:39:29.778602+0000","last_fullsized":"2024-08-17T17:39:29.778602+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1431,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.778421+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:39:29.778421+0000","last_peered":"2024-08-17T17:39:29.778421+0000","last_clean":"2024-08-17T17:39:29.778421+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:39:29.778421+0000","last_undegraded":"2024-08-17T17:39:29.778421+0000","last_fullsized":"2024-08-17T17:39:29.778421+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":938,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.800884+0000","last_change":"2024-08-17T17:36:15.774881+0000","last_active":"2024-08-17T17:39:37.800884+0000","last_peered":"2024-08-17T17:39:37.800884+0000","last_clean":"2024-08-17T17:39:37.800884+0000","last_became_active":"2024-08-17T17:36:15.774435+0000","last_became_peered":"2024-08-17T17:36:15.774435+0000","last_unstale":"2024-08-17T17:39:37.800884+0000","last_undegraded":"2024-08-17T17:39:37.800884+0000","last_fullsized":"2024-08-17T17:39:37.800884+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:52.015854+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"1426'6387","reported_seq":14569,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.376682+0000","last_change":"2024-08-17T17:38:37.110784+0000","last_active":"2024-08-17T17:39:42.376682+0000","last_peered":"2024-08-17T17:39:42.376682+0000","last_clean":"2024-08-17T17:39:42.376682+0000","last_became_active":"2024-08-17T17:38:37.088380+0000","last_became_peered":"2024-08-17T17:38:37.088380+0000","last_unstale":"2024-08-17T17:39:42.376682+0000","last_undegraded":"2024-08-17T17:39:42.376682+0000","last_fullsized":"2024-08-17T17:39:42.376682+0000","mapping_epoch":1419,"log_start":"1373'300","ondisk_log_start":"1373'300","created":1357,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6087,"log_dups_size":0,"ondisk_log_size":6087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:08.647903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124452864,"num_objects":1899,"num_object_clones":0,"num_object_copies":5697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1899,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6387,"num_write_kb":265152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4009,"num_bytes_recovered":199753728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1380,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.777702+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:39:29.777702+0000","last_peered":"2024-08-17T17:39:29.777702+0000","last_clean":"2024-08-17T17:39:29.777702+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:39:29.777702+0000","last_undegraded":"2024-08-17T17:39:29.777702+0000","last_fullsized":"2024-08-17T17:39:29.777702+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1275,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.761839+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:39:29.761839+0000","last_peered":"2024-08-17T17:39:29.761839+0000","last_clean":"2024-08-17T17:39:29.761839+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:39:29.761839+0000","last_undegraded":"2024-08-17T17:39:29.761839+0000","last_fullsized":"2024-08-17T17:39:29.761839+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":331,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.782535+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:39:37.782535+0000","last_peered":"2024-08-17T17:39:37.782535+0000","last_clean":"2024-08-17T17:39:37.782535+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:39:37.782535+0000","last_undegraded":"2024-08-17T17:39:37.782535+0000","last_fullsized":"2024-08-17T17:39:37.782535+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1599,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.800739+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:39:37.800739+0000","last_peered":"2024-08-17T17:39:37.800739+0000","last_clean":"2024-08-17T17:39:37.800739+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:39:37.800739+0000","last_undegraded":"2024-08-17T17:39:37.800739+0000","last_fullsized":"2024-08-17T17:39:37.800739+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:25.751871+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.f","version":"1426'6279","reported_seq":6435,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:40.537754+0000","last_change":"2024-08-17T17:35:34.179168+0000","last_active":"2024-08-17T17:39:40.537754+0000","last_peered":"2024-08-17T17:39:40.537754+0000","last_clean":"2024-08-17T17:39:40.537754+0000","last_became_active":"2024-08-17T17:35:34.133493+0000","last_became_peered":"2024-08-17T17:35:34.133493+0000","last_unstale":"2024-08-17T17:39:40.537754+0000","last_undegraded":"2024-08-17T17:39:40.537754+0000","last_fullsized":"2024-08-17T17:39:40.537754+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6279,"log_dups_size":0,"ondisk_log_size":6279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:24:11.188566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126156800,"num_objects":1925,"num_object_clones":0,"num_object_copies":5775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6279,"num_write_kb":262528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":106,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.794957+0000","last_change":"2024-08-17T17:37:11.079712+0000","last_active":"2024-08-17T17:39:37.794957+0000","last_peered":"2024-08-17T17:39:37.794957+0000","last_clean":"2024-08-17T17:39:37.794957+0000","last_became_active":"2024-08-17T17:37:11.079253+0000","last_became_peered":"2024-08-17T17:37:11.079253+0000","last_unstale":"2024-08-17T17:39:37.794957+0000","last_undegraded":"2024-08-17T17:39:37.794957+0000","last_fullsized":"2024-08-17T17:39:37.794957+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:53.405657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":920,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.761355+0000","last_change":"2024-08-17T17:35:56.223396+0000","last_active":"2024-08-17T17:39:29.761355+0000","last_peered":"2024-08-17T17:39:29.761355+0000","last_clean":"2024-08-17T17:39:29.761355+0000","last_became_active":"2024-08-17T17:35:56.212413+0000","last_became_peered":"2024-08-17T17:35:56.212413+0000","last_unstale":"2024-08-17T17:39:29.761355+0000","last_undegraded":"2024-08-17T17:39:29.761355+0000","last_fullsized":"2024-08-17T17:39:29.761355+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:44.185174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1641,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.782105+0000","last_change":"2024-08-17T17:35:33.136068+0000","last_active":"2024-08-17T17:39:37.782105+0000","last_peered":"2024-08-17T17:39:37.782105+0000","last_clean":"2024-08-17T17:39:37.782105+0000","last_became_active":"2024-08-17T17:35:33.135591+0000","last_became_peered":"2024-08-17T17:35:33.135591+0000","last_unstale":"2024-08-17T17:39:37.782105+0000","last_undegraded":"2024-08-17T17:39:37.782105+0000","last_fullsized":"2024-08-17T17:39:37.782105+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:52.252561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1786,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.779390+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:39:29.779390+0000","last_peered":"2024-08-17T17:39:29.779390+0000","last_clean":"2024-08-17T17:39:29.779390+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:39:29.779390+0000","last_undegraded":"2024-08-17T17:39:29.779390+0000","last_fullsized":"2024-08-17T17:39:29.779390+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"1426'6160","reported_seq":6301,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:40.584119+0000","last_change":"2024-08-17T17:35:14.861823+0000","last_active":"2024-08-17T17:39:40.584119+0000","last_peered":"2024-08-17T17:39:40.584119+0000","last_clean":"2024-08-17T17:39:40.584119+0000","last_became_active":"2024-08-17T17:35:14.854109+0000","last_became_peered":"2024-08-17T17:35:14.854109+0000","last_unstale":"2024-08-17T17:39:40.584119+0000","last_undegraded":"2024-08-17T17:39:40.584119+0000","last_fullsized":"2024-08-17T17:39:40.584119+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6160,"log_dups_size":0,"ondisk_log_size":6160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:46:40.778197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123600896,"num_objects":1886,"num_object_clones":0,"num_object_copies":5658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6160,"num_write_kb":257472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":998,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773648+0000","last_change":"2024-08-17T17:35:33.138467+0000","last_active":"2024-08-17T17:39:29.773648+0000","last_peered":"2024-08-17T17:39:29.773648+0000","last_clean":"2024-08-17T17:39:29.773648+0000","last_became_active":"2024-08-17T17:35:33.138174+0000","last_became_peered":"2024-08-17T17:35:33.138174+0000","last_unstale":"2024-08-17T17:39:29.773648+0000","last_undegraded":"2024-08-17T17:39:29.773648+0000","last_fullsized":"2024-08-17T17:39:29.773648+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:08.860439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1383,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.790133+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:39:37.790133+0000","last_peered":"2024-08-17T17:39:37.790133+0000","last_clean":"2024-08-17T17:39:37.790133+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:39:37.790133+0000","last_undegraded":"2024-08-17T17:39:37.790133+0000","last_fullsized":"2024-08-17T17:39:37.790133+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":137,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.791041+0000","last_change":"2024-08-17T17:35:14.854881+0000","last_active":"2024-08-17T17:39:37.791041+0000","last_peered":"2024-08-17T17:39:37.791041+0000","last_clean":"2024-08-17T17:39:37.791041+0000","last_became_active":"2024-08-17T17:35:14.854121+0000","last_became_peered":"2024-08-17T17:35:14.854121+0000","last_unstale":"2024-08-17T17:39:37.791041+0000","last_undegraded":"2024-08-17T17:39:37.791041+0000","last_fullsized":"2024-08-17T17:39:37.791041+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.863624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":939,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.761659+0000","last_change":"2024-08-17T17:36:15.773050+0000","last_active":"2024-08-17T17:39:29.761659+0000","last_peered":"2024-08-17T17:39:29.761659+0000","last_clean":"2024-08-17T17:39:29.761659+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:39:29.761659+0000","last_undegraded":"2024-08-17T17:39:29.761659+0000","last_fullsized":"2024-08-17T17:39:29.761659+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:39.212195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:39.212195+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"1426'6163","reported_seq":12010,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:40.661520+0000","last_change":"2024-08-17T17:37:21.409979+0000","last_active":"2024-08-17T17:39:40.661520+0000","last_peered":"2024-08-17T17:39:40.661520+0000","last_clean":"2024-08-17T17:39:40.661520+0000","last_became_active":"2024-08-17T17:37:21.398111+0000","last_became_peered":"2024-08-17T17:37:21.398111+0000","last_unstale":"2024-08-17T17:39:40.661520+0000","last_undegraded":"2024-08-17T17:39:40.661520+0000","last_fullsized":"2024-08-17T17:39:40.661520+0000","mapping_epoch":1407,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6163,"log_dups_size":0,"ondisk_log_size":6163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:19.299395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122421272,"num_objects":1869,"num_object_clones":0,"num_object_copies":5607,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1869,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6163,"num_write_kb":256961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2850,"num_bytes_recovered":168820736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1041,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773418+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:39:29.773418+0000","last_peered":"2024-08-17T17:39:29.773418+0000","last_clean":"2024-08-17T17:39:29.773418+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:39:29.773418+0000","last_undegraded":"2024-08-17T17:39:29.773418+0000","last_fullsized":"2024-08-17T17:39:29.773418+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":230,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.800409+0000","last_change":"2024-08-17T17:35:33.145351+0000","last_active":"2024-08-17T17:39:37.800409+0000","last_peered":"2024-08-17T17:39:37.800409+0000","last_clean":"2024-08-17T17:39:37.800409+0000","last_became_active":"2024-08-17T17:35:33.145040+0000","last_became_peered":"2024-08-17T17:35:33.145040+0000","last_unstale":"2024-08-17T17:39:37.800409+0000","last_undegraded":"2024-08-17T17:39:37.800409+0000","last_fullsized":"2024-08-17T17:39:37.800409+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:02.124000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1451,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.450081+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:39:37.450081+0000","last_peered":"2024-08-17T17:39:37.450081+0000","last_clean":"2024-08-17T17:39:37.450081+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:39:37.450081+0000","last_undegraded":"2024-08-17T17:39:37.450081+0000","last_fullsized":"2024-08-17T17:39:37.450081+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":922,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.761841+0000","last_change":"2024-08-17T17:36:15.769890+0000","last_active":"2024-08-17T17:39:29.761841+0000","last_peered":"2024-08-17T17:39:29.761841+0000","last_clean":"2024-08-17T17:39:29.761841+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:39:29.761841+0000","last_undegraded":"2024-08-17T17:39:29.761841+0000","last_fullsized":"2024-08-17T17:39:29.761841+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:33.034306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:33.034306+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":4,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.b","version":"1426'6036","reported_seq":13843,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:38.472015+0000","last_change":"2024-08-17T17:38:07.888612+0000","last_active":"2024-08-17T17:39:38.472015+0000","last_peered":"2024-08-17T17:39:38.472015+0000","last_clean":"2024-08-17T17:39:38.472015+0000","last_became_active":"2024-08-17T17:38:07.875196+0000","last_became_peered":"2024-08-17T17:38:07.875196+0000","last_unstale":"2024-08-17T17:39:38.472015+0000","last_undegraded":"2024-08-17T17:39:38.472015+0000","last_fullsized":"2024-08-17T17:39:38.472015+0000","mapping_epoch":1415,"log_start":"1373'100","ondisk_log_start":"1373'100","created":1357,"last_epoch_clean":1416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5936,"log_dups_size":0,"ondisk_log_size":5936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:21.658004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125698048,"num_objects":1918,"num_object_clones":0,"num_object_copies":5754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6036,"num_write_kb":254528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3836,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1492,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.791142+0000","last_change":"2024-08-17T17:36:12.759743+0000","last_active":"2024-08-17T17:39:37.791142+0000","last_peered":"2024-08-17T17:39:37.791142+0000","last_clean":"2024-08-17T17:39:37.791142+0000","last_became_active":"2024-08-17T17:36:12.759334+0000","last_became_peered":"2024-08-17T17:36:12.759334+0000","last_unstale":"2024-08-17T17:39:37.791142+0000","last_undegraded":"2024-08-17T17:39:37.791142+0000","last_fullsized":"2024-08-17T17:39:37.791142+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:04.524384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1414,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.778620+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:39:29.778620+0000","last_peered":"2024-08-17T17:39:29.778620+0000","last_clean":"2024-08-17T17:39:29.778620+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:39:29.778620+0000","last_undegraded":"2024-08-17T17:39:29.778620+0000","last_fullsized":"2024-08-17T17:39:29.778620+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1456,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.775161+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:39:29.775161+0000","last_peered":"2024-08-17T17:39:29.775161+0000","last_clean":"2024-08-17T17:39:29.775161+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:39:29.775161+0000","last_undegraded":"2024-08-17T17:39:29.775161+0000","last_fullsized":"2024-08-17T17:39:29.775161+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1772,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773202+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:39:29.773202+0000","last_peered":"2024-08-17T17:39:29.773202+0000","last_clean":"2024-08-17T17:39:29.773202+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:39:29.773202+0000","last_undegraded":"2024-08-17T17:39:29.773202+0000","last_fullsized":"2024-08-17T17:39:29.773202+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"1426'6279","reported_seq":12948,"reported_epoch":1426,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:39:42.327546+0000","last_change":"2024-08-17T17:38:05.991715+0000","last_active":"2024-08-17T17:39:42.327546+0000","last_peered":"2024-08-17T17:39:42.327546+0000","last_clean":"2024-08-17T17:35:54.187688+0000","last_became_active":"2024-08-17T17:36:22.904523+0000","last_became_peered":"2024-08-17T17:36:22.904523+0000","last_unstale":"2024-08-17T17:39:42.327546+0000","last_undegraded":"2024-08-17T17:39:42.327546+0000","last_fullsized":"2024-08-17T17:36:13.740491+0000","mapping_epoch":1383,"log_start":"1373'500","ondisk_log_start":"1373'500","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5779,"log_dups_size":0,"ondisk_log_size":5779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:57.525855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125108224,"num_objects":1909,"num_object_clones":0,"num_object_copies":5727,"num_objects_missing_on_primary":693,"num_objects_missing":693,"num_objects_degraded":0,"num_objects_misplaced":2471,"num_objects_unfound":0,"num_objects_dirty":1909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6279,"num_write_kb":262016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2190,"num_bytes_recovered":143523840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":693},{"shards":"2,4,5,6","objects":8},{"shards":"2,5,6","objects":1077},{"shards":"5,6","objects":131}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1090,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:32.449547+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:39:32.449547+0000","last_peered":"2024-08-17T17:39:32.449547+0000","last_clean":"2024-08-17T17:39:32.449547+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:39:32.449547+0000","last_undegraded":"2024-08-17T17:39:32.449547+0000","last_fullsized":"2024-08-17T17:39:32.449547+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1403,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.789283+0000","last_change":"2024-08-17T17:35:56.229622+0000","last_active":"2024-08-17T17:39:37.789283+0000","last_peered":"2024-08-17T17:39:37.789283+0000","last_clean":"2024-08-17T17:39:37.789283+0000","last_became_active":"2024-08-17T17:35:56.229324+0000","last_became_peered":"2024-08-17T17:35:56.229324+0000","last_unstale":"2024-08-17T17:39:37.789283+0000","last_undegraded":"2024-08-17T17:39:37.789283+0000","last_fullsized":"2024-08-17T17:39:37.789283+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:13.980316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2007,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.789437+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:39:37.789437+0000","last_peered":"2024-08-17T17:39:37.789437+0000","last_clean":"2024-08-17T17:39:37.789437+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:39:37.789437+0000","last_undegraded":"2024-08-17T17:39:37.789437+0000","last_fullsized":"2024-08-17T17:39:37.789437+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1586,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.777998+0000","last_change":"2024-08-17T17:35:33.137074+0000","last_active":"2024-08-17T17:39:29.777998+0000","last_peered":"2024-08-17T17:39:29.777998+0000","last_clean":"2024-08-17T17:39:29.777998+0000","last_became_active":"2024-08-17T17:35:33.136759+0000","last_became_peered":"2024-08-17T17:35:33.136759+0000","last_unstale":"2024-08-17T17:39:29.777998+0000","last_undegraded":"2024-08-17T17:39:29.777998+0000","last_fullsized":"2024-08-17T17:39:29.777998+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:37.263639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:37.263639+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:14.599780+0000","scrub_duration":0.0016181800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"1426'6229","reported_seq":12184,"reported_epoch":1426,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:40.660569+0000","last_change":"2024-08-17T17:36:23.019129+0000","last_active":"2024-08-17T17:39:40.660569+0000","last_peered":"2024-08-17T17:39:40.660569+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:23.013527+0000","last_became_peered":"2024-08-17T17:36:23.013527+0000","last_unstale":"2024-08-17T17:39:40.660569+0000","last_undegraded":"2024-08-17T17:36:13.874715+0000","last_fullsized":"2024-08-17T17:36:13.738828+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6229,"log_dups_size":0,"ondisk_log_size":6229,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122617856,"num_objects":1871,"num_object_clones":0,"num_object_copies":5613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3177,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6229,"num_write_kb":259200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2180,"num_bytes_recovered":142868480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[0,6],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":696},{"shards":"0,1,6","objects":1048},{"shards":"0,6","objects":-610}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1196,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.782590+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:39:37.782590+0000","last_peered":"2024-08-17T17:39:37.782590+0000","last_clean":"2024-08-17T17:39:37.782590+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:39:37.782590+0000","last_undegraded":"2024-08-17T17:39:37.782590+0000","last_fullsized":"2024-08-17T17:39:37.782590+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":50,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.777626+0000","last_change":"2024-08-17T17:36:17.774773+0000","last_active":"2024-08-17T17:39:29.777626+0000","last_peered":"2024-08-17T17:39:29.777626+0000","last_clean":"2024-08-17T17:39:29.777626+0000","last_became_active":"2024-08-17T17:36:17.774468+0000","last_became_peered":"2024-08-17T17:36:17.774468+0000","last_unstale":"2024-08-17T17:39:29.777626+0000","last_undegraded":"2024-08-17T17:39:29.777626+0000","last_fullsized":"2024-08-17T17:39:29.777626+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:54.231747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1534,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.781302+0000","last_change":"2024-08-17T17:35:33.137633+0000","last_active":"2024-08-17T17:39:37.781302+0000","last_peered":"2024-08-17T17:39:37.781302+0000","last_clean":"2024-08-17T17:39:37.781302+0000","last_became_active":"2024-08-17T17:35:33.137323+0000","last_became_peered":"2024-08-17T17:35:33.137323+0000","last_unstale":"2024-08-17T17:39:37.781302+0000","last_undegraded":"2024-08-17T17:39:37.781302+0000","last_fullsized":"2024-08-17T17:39:37.781302+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:03.359155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":901,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.760970+0000","last_change":"2024-08-17T17:35:56.264626+0000","last_active":"2024-08-17T17:39:29.760970+0000","last_peered":"2024-08-17T17:39:29.760970+0000","last_clean":"2024-08-17T17:39:29.760970+0000","last_became_active":"2024-08-17T17:35:56.264265+0000","last_became_peered":"2024-08-17T17:35:56.264265+0000","last_unstale":"2024-08-17T17:39:29.760970+0000","last_undegraded":"2024-08-17T17:39:29.760970+0000","last_fullsized":"2024-08-17T17:39:29.760970+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:20.754326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:20.754326+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"1426'6170","reported_seq":12745,"reported_epoch":1426,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:42.349176+0000","last_change":"2024-08-17T17:37:11.134558+0000","last_active":"2024-08-17T17:39:42.349176+0000","last_peered":"2024-08-17T17:39:42.349176+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:20.105110+0000","last_became_peered":"2024-08-17T17:36:20.105110+0000","last_unstale":"2024-08-17T17:39:42.349176+0000","last_undegraded":"2024-08-17T17:36:19.999272+0000","last_fullsized":"2024-08-17T17:36:13.740520+0000","mapping_epoch":1383,"log_start":"1373'400","ondisk_log_start":"1373'400","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5770,"log_dups_size":0,"ondisk_log_size":5770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:45.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120848384,"num_objects":1844,"num_object_clones":0,"num_object_copies":5532,"num_objects_missing_on_primary":653,"num_objects_missing":653,"num_objects_degraded":2343,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6170,"num_write_kb":256448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2163,"num_bytes_recovered":141754368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":653},{"shards":"1,5,6","objects":1037},{"shards":"5,6","objects":154}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1254,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.761430+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:39:29.761430+0000","last_peered":"2024-08-17T17:39:29.761430+0000","last_clean":"2024-08-17T17:39:29.761430+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:39:29.761430+0000","last_undegraded":"2024-08-17T17:39:29.761430+0000","last_fullsized":"2024-08-17T17:39:29.761430+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1794,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.775114+0000","last_change":"2024-08-17T17:36:12.758005+0000","last_active":"2024-08-17T17:39:29.775114+0000","last_peered":"2024-08-17T17:39:29.775114+0000","last_clean":"2024-08-17T17:39:29.775114+0000","last_became_active":"2024-08-17T17:36:12.757695+0000","last_became_peered":"2024-08-17T17:36:12.757695+0000","last_unstale":"2024-08-17T17:39:29.775114+0000","last_undegraded":"2024-08-17T17:39:29.775114+0000","last_fullsized":"2024-08-17T17:39:29.775114+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:51.792220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1432,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773585+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:39:29.773585+0000","last_peered":"2024-08-17T17:39:29.773585+0000","last_clean":"2024-08-17T17:39:29.773585+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:39:29.773585+0000","last_undegraded":"2024-08-17T17:39:29.773585+0000","last_fullsized":"2024-08-17T17:39:29.773585+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1743,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.781414+0000","last_change":"2024-08-17T17:39:37.781414+0000","last_active":"2024-08-17T17:39:37.781414+0000","last_peered":"2024-08-17T17:39:37.781414+0000","last_clean":"2024-08-17T17:39:37.781414+0000","last_became_active":"2024-08-17T17:35:55.204861+0000","last_became_peered":"2024-08-17T17:35:55.204861+0000","last_unstale":"2024-08-17T17:39:37.781414+0000","last_undegraded":"2024-08-17T17:39:37.781414+0000","last_fullsized":"2024-08-17T17:39:37.781414+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:39:41.238980+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.0","version":"1426'6241","reported_seq":14237,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.639504+0000","last_change":"2024-08-17T17:37:40.753703+0000","last_active":"2024-08-17T17:39:42.639504+0000","last_peered":"2024-08-17T17:39:42.639504+0000","last_clean":"2024-08-17T17:39:42.639504+0000","last_became_active":"2024-08-17T17:37:40.743458+0000","last_became_peered":"2024-08-17T17:37:40.743458+0000","last_unstale":"2024-08-17T17:39:42.639504+0000","last_undegraded":"2024-08-17T17:39:42.639504+0000","last_fullsized":"2024-08-17T17:39:42.639504+0000","mapping_epoch":1411,"log_start":"1373'700","ondisk_log_start":"1373'700","created":1357,"last_epoch_clean":1412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5541,"log_dups_size":0,"ondisk_log_size":5541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:14:02.340380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122880000,"num_objects":1875,"num_object_clones":0,"num_object_copies":5625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6241,"num_write_kb":259712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3914,"num_bytes_recovered":232652800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1028,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.779138+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:39:29.779138+0000","last_peered":"2024-08-17T17:39:29.779138+0000","last_clean":"2024-08-17T17:39:29.779138+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:39:29.779138+0000","last_undegraded":"2024-08-17T17:39:29.779138+0000","last_fullsized":"2024-08-17T17:39:29.779138+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":99,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.795042+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:39:37.795042+0000","last_peered":"2024-08-17T17:39:37.795042+0000","last_clean":"2024-08-17T17:39:37.795042+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:39:37.795042+0000","last_undegraded":"2024-08-17T17:39:37.795042+0000","last_fullsized":"2024-08-17T17:39:37.795042+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1763,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773307+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:39:29.773307+0000","last_peered":"2024-08-17T17:39:29.773307+0000","last_clean":"2024-08-17T17:39:29.773307+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:39:29.773307+0000","last_undegraded":"2024-08-17T17:39:29.773307+0000","last_fullsized":"2024-08-17T17:39:29.773307+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1","version":"1426'6174","reported_seq":12718,"reported_epoch":1426,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:42.329716+0000","last_change":"2024-08-17T17:36:23.103523+0000","last_active":"2024-08-17T17:39:42.329716+0000","last_peered":"2024-08-17T17:39:42.329716+0000","last_clean":"2024-08-17T17:35:54.178280+0000","last_became_active":"2024-08-17T17:36:23.025910+0000","last_became_peered":"2024-08-17T17:36:23.025910+0000","last_unstale":"2024-08-17T17:39:42.329716+0000","last_undegraded":"2024-08-17T17:36:18.734637+0000","last_fullsized":"2024-08-17T17:36:13.740134+0000","mapping_epoch":1383,"log_start":"1373'400","ondisk_log_start":"1373'400","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5774,"log_dups_size":0,"ondisk_log_size":5774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:58.717055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123076608,"num_objects":1878,"num_object_clones":0,"num_object_copies":5634,"num_objects_missing_on_primary":681,"num_objects_missing":681,"num_objects_degraded":2435,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6174,"num_write_kb":257664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2150,"num_bytes_recovered":140902400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,0],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":681},{"shards":"0,2,4,6","objects":3},{"shards":"0,2,6","objects":1070},{"shards":"0,6","objects":124}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1442,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.778878+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:39:29.778878+0000","last_peered":"2024-08-17T17:39:29.778878+0000","last_clean":"2024-08-17T17:39:29.778878+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:39:29.778878+0000","last_undegraded":"2024-08-17T17:39:29.778878+0000","last_fullsized":"2024-08-17T17:39:29.778878+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1415,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.777875+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:39:29.777875+0000","last_peered":"2024-08-17T17:39:29.777875+0000","last_clean":"2024-08-17T17:39:29.777875+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:39:29.777875+0000","last_undegraded":"2024-08-17T17:39:29.777875+0000","last_fullsized":"2024-08-17T17:39:29.777875+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":1001,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.781679+0000","last_change":"2024-08-17T17:34:26.817339+0000","last_active":"2024-08-17T17:39:37.781679+0000","last_peered":"2024-08-17T17:39:37.781679+0000","last_clean":"2024-08-17T17:39:37.781679+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:39:37.781679+0000","last_undegraded":"2024-08-17T17:39:37.781679+0000","last_fullsized":"2024-08-17T17:39:37.781679+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:26.817223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:26.817223+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:30.089956+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"1426'6022","reported_seq":6189,"reported_epoch":1426,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:39:40.644588+0000","last_change":"2024-08-17T17:36:14.073155+0000","last_active":"2024-08-17T17:39:40.644588+0000","last_peered":"2024-08-17T17:39:40.644588+0000","last_clean":"2024-08-17T17:36:11.734683+0000","last_became_active":"2024-08-17T17:36:14.063794+0000","last_became_peered":"2024-08-17T17:36:14.063794+0000","last_unstale":"2024-08-17T17:39:40.644588+0000","last_undegraded":"2024-08-17T17:36:13.855770+0000","last_fullsized":"2024-08-17T17:36:13.742068+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6022,"log_dups_size":0,"ondisk_log_size":6022,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123207680,"num_objects":1880,"num_object_clones":0,"num_object_copies":5640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3819,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6022,"num_write_kb":252864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-1939},{"shards":"0,2,7","objects":1748}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":50,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.779128+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:39:29.779128+0000","last_peered":"2024-08-17T17:39:29.779128+0000","last_clean":"2024-08-17T17:39:29.779128+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:39:29.779128+0000","last_undegraded":"2024-08-17T17:39:29.779128+0000","last_fullsized":"2024-08-17T17:39:29.779128+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1276,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.778669+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:39:29.778669+0000","last_peered":"2024-08-17T17:39:29.778669+0000","last_clean":"2024-08-17T17:39:29.778669+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:39:29.778669+0000","last_undegraded":"2024-08-17T17:39:29.778669+0000","last_fullsized":"2024-08-17T17:39:29.778669+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1791,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773252+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:39:29.773252+0000","last_peered":"2024-08-17T17:39:29.773252+0000","last_clean":"2024-08-17T17:39:29.773252+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:39:29.773252+0000","last_undegraded":"2024-08-17T17:39:29.773252+0000","last_fullsized":"2024-08-17T17:39:29.773252+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"1426'6256","reported_seq":12057,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.241146+0000","last_change":"2024-08-17T17:36:46.982456+0000","last_active":"2024-08-17T17:39:42.241146+0000","last_peered":"2024-08-17T17:39:42.241146+0000","last_clean":"2024-08-17T17:39:42.241146+0000","last_became_active":"2024-08-17T17:36:46.974782+0000","last_became_peered":"2024-08-17T17:36:46.974782+0000","last_unstale":"2024-08-17T17:39:42.241146+0000","last_undegraded":"2024-08-17T17:39:42.241146+0000","last_fullsized":"2024-08-17T17:39:42.241146+0000","mapping_epoch":1399,"log_start":"1373'400","ondisk_log_start":"1373'400","created":1357,"last_epoch_clean":1400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5856,"log_dups_size":0,"ondisk_log_size":5856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:40.499321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119013376,"num_objects":1816,"num_object_clones":0,"num_object_copies":5448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6256,"num_write_kb":258304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2848,"num_bytes_recovered":186646528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1393,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.778472+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:39:29.778472+0000","last_peered":"2024-08-17T17:39:29.778472+0000","last_clean":"2024-08-17T17:39:29.778472+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:39:29.778472+0000","last_undegraded":"2024-08-17T17:39:29.778472+0000","last_fullsized":"2024-08-17T17:39:29.778472+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":207,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.778268+0000","last_change":"2024-08-17T17:36:12.754372+0000","last_active":"2024-08-17T17:39:29.778268+0000","last_peered":"2024-08-17T17:39:29.778268+0000","last_clean":"2024-08-17T17:39:29.778268+0000","last_became_active":"2024-08-17T17:36:12.746352+0000","last_became_peered":"2024-08-17T17:36:12.746352+0000","last_unstale":"2024-08-17T17:39:29.778268+0000","last_undegraded":"2024-08-17T17:39:29.778268+0000","last_fullsized":"2024-08-17T17:39:29.778268+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:07.807486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1673,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.789354+0000","last_change":"2024-08-17T17:36:15.771193+0000","last_active":"2024-08-17T17:39:37.789354+0000","last_peered":"2024-08-17T17:39:37.789354+0000","last_clean":"2024-08-17T17:39:37.789354+0000","last_became_active":"2024-08-17T17:36:15.770889+0000","last_became_peered":"2024-08-17T17:36:15.770889+0000","last_unstale":"2024-08-17T17:39:37.789354+0000","last_undegraded":"2024-08-17T17:39:37.789354+0000","last_fullsized":"2024-08-17T17:39:37.789354+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:22.978695+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.4","version":"1426'6173","reported_seq":11955,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.229320+0000","last_change":"2024-08-17T17:37:04.045576+0000","last_active":"2024-08-17T17:39:42.229320+0000","last_peered":"2024-08-17T17:39:42.229320+0000","last_clean":"2024-08-17T17:39:42.229320+0000","last_became_active":"2024-08-17T17:37:04.038810+0000","last_became_peered":"2024-08-17T17:37:04.038810+0000","last_unstale":"2024-08-17T17:39:42.229320+0000","last_undegraded":"2024-08-17T17:39:42.229320+0000","last_fullsized":"2024-08-17T17:39:42.229320+0000","mapping_epoch":1402,"log_start":"1373'400","ondisk_log_start":"1373'400","created":1357,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5773,"log_dups_size":0,"ondisk_log_size":5773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:27:04.725258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124059648,"num_objects":1893,"num_object_clones":0,"num_object_copies":5679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6173,"num_write_kb":258112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":176291840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1268,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.781937+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:39:37.781937+0000","last_peered":"2024-08-17T17:39:37.781937+0000","last_clean":"2024-08-17T17:39:37.781937+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:39:37.781937+0000","last_undegraded":"2024-08-17T17:39:37.781937+0000","last_fullsized":"2024-08-17T17:39:37.781937+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1419,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773435+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:39:29.773435+0000","last_peered":"2024-08-17T17:39:29.773435+0000","last_clean":"2024-08-17T17:39:29.773435+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:39:29.773435+0000","last_undegraded":"2024-08-17T17:39:29.773435+0000","last_fullsized":"2024-08-17T17:39:29.773435+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":919,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.777608+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:39:29.777608+0000","last_peered":"2024-08-17T17:39:29.777608+0000","last_clean":"2024-08-17T17:39:29.777608+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:39:29.777608+0000","last_undegraded":"2024-08-17T17:39:29.777608+0000","last_fullsized":"2024-08-17T17:39:29.777608+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"1426'6248","reported_seq":12085,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.388347+0000","last_change":"2024-08-17T17:36:32.711490+0000","last_active":"2024-08-17T17:39:42.388347+0000","last_peered":"2024-08-17T17:39:42.388347+0000","last_clean":"2024-08-17T17:39:42.388347+0000","last_became_active":"2024-08-17T17:36:32.704370+0000","last_became_peered":"2024-08-17T17:36:32.704370+0000","last_unstale":"2024-08-17T17:39:42.388347+0000","last_undegraded":"2024-08-17T17:39:42.388347+0000","last_fullsized":"2024-08-17T17:39:42.388347+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6248,"log_dups_size":0,"ondisk_log_size":6248,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:17.373879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121896960,"num_objects":1860,"num_object_clones":0,"num_object_copies":5580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6248,"num_write_kb":259456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2866,"num_bytes_recovered":187826176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1275,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.787875+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:39:29.787875+0000","last_peered":"2024-08-17T17:39:29.787875+0000","last_clean":"2024-08-17T17:39:29.787875+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:39:29.787875+0000","last_undegraded":"2024-08-17T17:39:29.787875+0000","last_fullsized":"2024-08-17T17:39:29.787875+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":156,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:32.449463+0000","last_change":"2024-08-17T17:35:55.217072+0000","last_active":"2024-08-17T17:39:32.449463+0000","last_peered":"2024-08-17T17:39:32.449463+0000","last_clean":"2024-08-17T17:39:32.449463+0000","last_became_active":"2024-08-17T17:35:55.216462+0000","last_became_peered":"2024-08-17T17:35:55.216462+0000","last_unstale":"2024-08-17T17:39:32.449463+0000","last_undegraded":"2024-08-17T17:39:32.449463+0000","last_fullsized":"2024-08-17T17:39:32.449463+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:34.937221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":902,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.761928+0000","last_change":"2024-08-17T17:35:55.215004+0000","last_active":"2024-08-17T17:39:29.761928+0000","last_peered":"2024-08-17T17:39:29.761928+0000","last_clean":"2024-08-17T17:39:29.761928+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:39:29.761928+0000","last_undegraded":"2024-08-17T17:39:29.761928+0000","last_fullsized":"2024-08-17T17:39:29.761928+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.6","version":"1426'6035","reported_seq":6178,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:40.598382+0000","last_change":"2024-08-17T17:35:14.868524+0000","last_active":"2024-08-17T17:39:40.598382+0000","last_peered":"2024-08-17T17:39:40.598382+0000","last_clean":"2024-08-17T17:39:40.598382+0000","last_became_active":"2024-08-17T17:35:14.854690+0000","last_became_peered":"2024-08-17T17:35:14.854690+0000","last_unstale":"2024-08-17T17:39:40.598382+0000","last_undegraded":"2024-08-17T17:39:40.598382+0000","last_fullsized":"2024-08-17T17:39:40.598382+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6035,"log_dups_size":0,"ondisk_log_size":6035,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:05.388362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116719616,"num_objects":1781,"num_object_clones":0,"num_object_copies":5343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6035,"num_write_kb":250112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1461,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.777801+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:39:29.777801+0000","last_peered":"2024-08-17T17:39:29.777801+0000","last_clean":"2024-08-17T17:39:29.777801+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:39:29.777801+0000","last_undegraded":"2024-08-17T17:39:29.777801+0000","last_fullsized":"2024-08-17T17:39:29.777801+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":168,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:32.431586+0000","last_change":"2024-08-17T17:35:33.147672+0000","last_active":"2024-08-17T17:39:32.431586+0000","last_peered":"2024-08-17T17:39:32.431586+0000","last_clean":"2024-08-17T17:39:32.431586+0000","last_became_active":"2024-08-17T17:35:33.147283+0000","last_became_peered":"2024-08-17T17:35:33.147283+0000","last_unstale":"2024-08-17T17:39:32.431586+0000","last_undegraded":"2024-08-17T17:39:32.431586+0000","last_fullsized":"2024-08-17T17:39:32.431586+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:41.677709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1677,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.800645+0000","last_change":"2024-08-17T17:39:37.800645+0000","last_active":"2024-08-17T17:39:37.800645+0000","last_peered":"2024-08-17T17:39:37.800645+0000","last_clean":"2024-08-17T17:39:37.800645+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:39:37.800645+0000","last_undegraded":"2024-08-17T17:39:37.800645+0000","last_fullsized":"2024-08-17T17:39:37.800645+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:38.241235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:38.241235+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:39:40.966245+0000","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"1426'6075","reported_seq":6154,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:42.418703+0000","last_change":"2024-08-17T17:34:50.670013+0000","last_active":"2024-08-17T17:39:42.418703+0000","last_peered":"2024-08-17T17:39:42.418703+0000","last_clean":"2024-08-17T17:39:42.418703+0000","last_became_active":"2024-08-17T17:34:50.669299+0000","last_became_peered":"2024-08-17T17:34:50.669299+0000","last_unstale":"2024-08-17T17:39:42.418703+0000","last_undegraded":"2024-08-17T17:39:42.418703+0000","last_fullsized":"2024-08-17T17:39:42.418703+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6075,"log_dups_size":0,"ondisk_log_size":6075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:11.172227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121307136,"num_objects":1851,"num_object_clones":0,"num_object_copies":5553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6075,"num_write_kb":253632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1385,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773366+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:39:29.773366+0000","last_peered":"2024-08-17T17:39:29.773366+0000","last_clean":"2024-08-17T17:39:29.773366+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:39:29.773366+0000","last_undegraded":"2024-08-17T17:39:29.773366+0000","last_fullsized":"2024-08-17T17:39:29.773366+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":155,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:32.431610+0000","last_change":"2024-08-17T17:35:33.140725+0000","last_active":"2024-08-17T17:39:32.431610+0000","last_peered":"2024-08-17T17:39:32.431610+0000","last_clean":"2024-08-17T17:39:32.431610+0000","last_became_active":"2024-08-17T17:35:33.140406+0000","last_became_peered":"2024-08-17T17:35:33.140406+0000","last_unstale":"2024-08-17T17:39:32.431610+0000","last_undegraded":"2024-08-17T17:39:32.431610+0000","last_fullsized":"2024-08-17T17:39:32.431610+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:54:51.875571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1675,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:32.431519+0000","last_change":"2024-08-17T17:35:33.138514+0000","last_active":"2024-08-17T17:39:32.431519+0000","last_peered":"2024-08-17T17:39:32.431519+0000","last_clean":"2024-08-17T17:39:32.431519+0000","last_became_active":"2024-08-17T17:35:33.137035+0000","last_became_peered":"2024-08-17T17:35:33.137035+0000","last_unstale":"2024-08-17T17:39:32.431519+0000","last_undegraded":"2024-08-17T17:39:32.431519+0000","last_fullsized":"2024-08-17T17:39:32.431519+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:50.565662+0000","scrub_duration":0.001487138,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1316,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.780335+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:39:29.780335+0000","last_peered":"2024-08-17T17:39:29.780335+0000","last_clean":"2024-08-17T17:39:29.780335+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:39:29.780335+0000","last_undegraded":"2024-08-17T17:39:29.780335+0000","last_fullsized":"2024-08-17T17:39:29.780335+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":99,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.800316+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:39:37.800316+0000","last_peered":"2024-08-17T17:39:37.800316+0000","last_clean":"2024-08-17T17:39:37.800316+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:39:37.800316+0000","last_undegraded":"2024-08-17T17:39:37.800316+0000","last_fullsized":"2024-08-17T17:39:37.800316+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1747,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773377+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:39:29.773377+0000","last_peered":"2024-08-17T17:39:29.773377+0000","last_clean":"2024-08-17T17:39:29.773377+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:39:29.773377+0000","last_undegraded":"2024-08-17T17:39:29.773377+0000","last_fullsized":"2024-08-17T17:39:29.773377+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1333,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.777736+0000","last_change":"2024-08-17T17:35:55.205292+0000","last_active":"2024-08-17T17:39:29.777736+0000","last_peered":"2024-08-17T17:39:29.777736+0000","last_clean":"2024-08-17T17:39:29.777736+0000","last_became_active":"2024-08-17T17:35:55.204935+0000","last_became_peered":"2024-08-17T17:35:55.204935+0000","last_unstale":"2024-08-17T17:39:29.777736+0000","last_undegraded":"2024-08-17T17:39:29.777736+0000","last_fullsized":"2024-08-17T17:39:29.777736+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:52.277821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":99,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.800066+0000","last_change":"2024-08-17T17:36:12.749972+0000","last_active":"2024-08-17T17:39:37.800066+0000","last_peered":"2024-08-17T17:39:37.800066+0000","last_clean":"2024-08-17T17:39:37.800066+0000","last_became_active":"2024-08-17T17:36:12.749620+0000","last_became_peered":"2024-08-17T17:36:12.749620+0000","last_unstale":"2024-08-17T17:39:37.800066+0000","last_undegraded":"2024-08-17T17:39:37.800066+0000","last_fullsized":"2024-08-17T17:39:37.800066+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:04.994357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":975,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.797911+0000","last_change":"2024-08-17T17:36:20.832274+0000","last_active":"2024-08-17T17:39:37.797911+0000","last_peered":"2024-08-17T17:39:37.797911+0000","last_clean":"2024-08-17T17:39:37.797911+0000","last_became_active":"2024-08-17T17:36:20.831886+0000","last_became_peered":"2024-08-17T17:36:20.831886+0000","last_unstale":"2024-08-17T17:39:37.797911+0000","last_undegraded":"2024-08-17T17:39:37.797911+0000","last_fullsized":"2024-08-17T17:39:37.797911+0000","mapping_epoch":1390,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1391,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:05.248711+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1184,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.781547+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:39:37.781547+0000","last_peered":"2024-08-17T17:39:37.781547+0000","last_clean":"2024-08-17T17:39:37.781547+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:39:37.781547+0000","last_undegraded":"2024-08-17T17:39:37.781547+0000","last_fullsized":"2024-08-17T17:39:37.781547+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1417,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:32.449480+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:39:32.449480+0000","last_peered":"2024-08-17T17:39:32.449480+0000","last_clean":"2024-08-17T17:39:32.449480+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:39:32.449480+0000","last_undegraded":"2024-08-17T17:39:32.449480+0000","last_fullsized":"2024-08-17T17:39:32.449480+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1710,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773175+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:39:29.773175+0000","last_peered":"2024-08-17T17:39:29.773175+0000","last_clean":"2024-08-17T17:39:29.773175+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:39:29.773175+0000","last_undegraded":"2024-08-17T17:39:29.773175+0000","last_fullsized":"2024-08-17T17:39:29.773175+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1358,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.785053+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:39:29.785053+0000","last_peered":"2024-08-17T17:39:29.785053+0000","last_clean":"2024-08-17T17:39:29.785053+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:39:29.785053+0000","last_undegraded":"2024-08-17T17:39:29.785053+0000","last_fullsized":"2024-08-17T17:39:29.785053+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1903,"reported_epoch":1426,"state":"active+clean","last_fresh":"2024-08-17T17:39:37.800831+0000","last_change":"2024-08-17T17:35:33.137411+0000","last_active":"2024-08-17T17:39:37.800831+0000","last_peered":"2024-08-17T17:39:37.800831+0000","last_clean":"2024-08-17T17:39:37.800831+0000","last_became_active":"2024-08-17T17:35:33.137073+0000","last_became_peered":"2024-08-17T17:35:33.137073+0000","last_unstale":"2024-08-17T17:39:37.800831+0000","last_undegraded":"2024-08-17T17:39:37.800831+0000","last_fullsized":"2024-08-17T17:39:37.800831+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:35.924539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1804,"reported_epoch":1425,"state":"active+clean","last_fresh":"2024-08-17T17:39:29.773109+0000","last_change":"2024-08-17T17:36:15.787445+0000","last_active":"2024-08-17T17:39:29.773109+0000","last_peered":"2024-08-17T17:39:29.773109+0000","last_clean":"2024-08-17T17:39:29.773109+0000","last_became_active":"2024-08-17T17:36:15.787099+0000","last_became_peered":"2024-08-17T17:36:15.787099+0000","last_unstale":"2024-08-17T17:39:29.773109+0000","last_undegraded":"2024-08-17T17:39:29.773109+0000","last_fullsized":"2024-08-17T17:39:29.773109+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:47.448168+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1963065368,"num_objects":29955,"num_object_clones":0,"num_object_copies":89865,"num_objects_missing_on_primary":2027,"num_objects_missing":2027,"num_objects_degraded":11774,"num_objects_misplaced":2471,"num_objects_unfound":0,"num_objects_dirty":29955,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":98927,"num_write_kb":4124161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31857,"num_bytes_recovered":1935147008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7034974208,"data_stored":7034961992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95727,"ondisk_log_size":95727,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635092,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":27,"num_read_kb":61,"num_write":15,"num_write_kb":183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1314910,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162867,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1148688,"kb_used_data":975880,"kb_used_omap":54,"kb_used_meta":172745,"kb_avail":92596464,"statfs":{"total":95995035648,"available":94818779136,"internally_reserved":0,"allocated":999301120,"data_stored":996611110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":176890901},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999341,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1057424,"kb_used_data":859208,"kb_used_omap":47,"kb_used_meta":198160,"kb_avail":92687728,"statfs":{"total":95995035648,"available":94912233472,"internally_reserved":0,"allocated":879828992,"data_stored":877139959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":202916500},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,4,0,3,1,6,5,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627137,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1358200,"kb_used_data":299500,"kb_used_omap":42,"kb_used_meta":1058645,"kb_avail":92386952,"statfs":{"total":95995035648,"available":94604238848,"internally_reserved":0,"allocated":306688000,"data_stored":304021248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1084052540},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785615,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2269368,"kb_used_data":1452720,"kb_used_omap":48,"kb_used_meta":816591,"kb_avail":91475784,"statfs":{"total":95995035648,"available":93671202816,"internally_reserved":0,"allocated":1487585280,"data_stored":1484915403,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":836189870},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725132,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1365696,"kb_used_data":486392,"kb_used_omap":1,"kb_used_meta":879294,"kb_avail":92379456,"statfs":{"total":95995035648,"available":94596562944,"internally_reserved":0,"allocated":498065408,"data_stored":495387481,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":900397510},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986769,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1733132,"kb_used_data":853828,"kb_used_omap":1,"kb_used_meta":879294,"kb_avail":92012020,"statfs":{"total":95995035648,"available":94220308480,"internally_reserved":0,"allocated":874319872,"data_stored":871633368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":900397510},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706376,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2175552,"kb_used_data":1236984,"kb_used_omap":35,"kb_used_meta":938524,"kb_avail":91569600,"statfs":{"total":95995035648,"available":93767270400,"internally_reserved":0,"allocated":1266671616,"data_stored":1263993670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":961048631},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832149,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1827708,"kb_used_data":788788,"kb_used_omap":65,"kb_used_meta":1038846,"kb_avail":91917444,"statfs":{"total":95995035648,"available":94123462656,"internally_reserved":0,"allocated":807718912,"data_stored":805045524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1063778491},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":796721152,"data_stored":796721152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1476853760,"data_stored":1476853760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1253380096,"data_stored":1253376024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":296681472,"data_stored":296681472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":862326784,"data_stored":862322712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":488898560,"data_stored":488898560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":868880384,"data_stored":868876312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":991232000,"data_stored":991232000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:39:47.578 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:39:47.589 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:47.645 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:47.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:47.747 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:47.770 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:47.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:47.872 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:47.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:47.901 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:47.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:48.002 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:48.027 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:48.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:47 smithi162 ceph-mon[104668]: pgmap v2263: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 216 op/s; 11608/88062 objects degraded (13.182%); 2471/88062 objects misplaced (2.806%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:39:48.123 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:47 smithi063 ceph-mon[114756]: pgmap v2263: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 216 op/s; 11608/88062 objects degraded (13.182%); 2471/88062 objects misplaced (2.806%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:39:48.123 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:47 smithi063 ceph-mon[109843]: pgmap v2263: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 216 op/s; 11608/88062 objects degraded (13.182%); 2471/88062 objects misplaced (2.806%); 2.8 MiB/s, 45 objects/s recovering 2024-08-17T17:39:48.129 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:48.140 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:39:48.140 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:39:48.157 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:48.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:48.175 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:39:48.175 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:39:48.203 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:48.204 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:39:48.205 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:39:48.223 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:48.234 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:39:48.234 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:39:48.259 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:48.271 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:39:48.271 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:39:48.285 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:48.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:48.293 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:39:48.293 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:39:48.320 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:39:48.320 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:39:48.348 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:39:48.348 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:39:48.387 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.412 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:48.412 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:48.513 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:48.540 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:48.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:48.641 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:48.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.667 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:48.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:48.743 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:39:48.768 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:48.794 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:48.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:48.896 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:48.922 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:48.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:48.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:49.023 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:48 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/524021967' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:48 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 11608/88062 objects degraded (13.182%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:48 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:39:49.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:48 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:39:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:48 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/524021967' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:49.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:48 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 11608/88062 objects degraded (13.182%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:49.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:48 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:39:49.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:48 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:39:49.050 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:49.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:49.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:48 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/524021967' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:39:49.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:48 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 11608/88062 objects degraded (13.182%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:49.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:48 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:39:49.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:48 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:39:49.152 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:49.177 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:49.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:49.279 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:49.302 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:49.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:49.404 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:49.430 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:49.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:49.531 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:49.556 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:49.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:49.658 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:49.686 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:49.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:49.788 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:49.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.812 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:49.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:49.913 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:49.940 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:49.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:49.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:50.042 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:50.066 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:50.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:50.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:49 smithi162 ceph-mon[104668]: pgmap v2264: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 273 op/s; 11237/85620 objects degraded (13.124%); 2306/85620 objects misplaced (2.693%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:39:50.169 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:50.197 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:50.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.198 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:50.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:49 smithi063 ceph-mon[109843]: pgmap v2264: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 273 op/s; 11237/85620 objects degraded (13.124%); 2306/85620 objects misplaced (2.693%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:39:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:49 smithi063 ceph-mon[114756]: pgmap v2264: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 273 op/s; 11237/85620 objects degraded (13.124%); 2306/85620 objects misplaced (2.693%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:39:50.299 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:50.324 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:50.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:50.426 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:50.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.472 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:50.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:50.573 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:50.597 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:50.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:50.698 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:50.722 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:50.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:50.824 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:50.850 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:50.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:50.952 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:50.977 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:50.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:50.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:51.079 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:51.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.103 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:51.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:51.205 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:51.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.231 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:51.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:51.332 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:51.363 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:51.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:51.464 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:51.488 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:51.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:51.590 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:51.617 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:51.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:51.719 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:51.745 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:51.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:51.847 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:51.878 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:51.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:51.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:51.979 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:52.003 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:52.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:52.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:51 smithi162 ceph-mon[104668]: pgmap v2265: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 273 op/s; 11237/85335 objects degraded (13.168%); 2306/85335 objects misplaced (2.702%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:39:52.105 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:52.129 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:52.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:52.205 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:51 smithi063 ceph-mon[109843]: pgmap v2265: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 273 op/s; 11237/85335 objects degraded (13.168%); 2306/85335 objects misplaced (2.702%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:39:52.205 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:51 smithi063 ceph-mon[114756]: pgmap v2265: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 273 op/s; 11237/85335 objects degraded (13.168%); 2306/85335 objects misplaced (2.702%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:39:52.231 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:52.260 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:52.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:52.267 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:52.361 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:52.388 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:52.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:52.490 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:52.518 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:52.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:52.620 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:52.642 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:52.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:52.744 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:52.774 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:52.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:52.876 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:52.902 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:52.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:52.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:53.004 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:53.027 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:53.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:53.129 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:53.154 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:53.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:53.184 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:39:53.256 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:53.280 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:53.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:53.382 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:53.410 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:53.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:53.423 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:39:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:39:53] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.51.0" 2024-08-17T17:39:53.512 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:53.541 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:53.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:53.644 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:53.671 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:53.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:53.773 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:53.796 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:53.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:53.897 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:53.922 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:53.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:53.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:54.024 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:54.051 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:54.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:54.080 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:39:54.080 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:39:54.080 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:53 smithi063 ceph-mon[114756]: pgmap v2266: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 280 op/s; 10668/81519 objects degraded (13.087%); 2118/81519 objects misplaced (2.598%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:39:54.080 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:53 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 10668/81519 objects degraded (13.087%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:54.081 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:53 smithi063 ceph-mon[109843]: pgmap v2266: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 280 op/s; 10668/81519 objects degraded (13.087%); 2118/81519 objects misplaced (2.598%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:39:54.081 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:53 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 10668/81519 objects degraded (13.087%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:53 smithi162 ceph-mon[104668]: pgmap v2266: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 280 op/s; 10668/81519 objects degraded (13.087%); 2118/81519 objects misplaced (2.598%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:39:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:53 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 10668/81519 objects degraded (13.087%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:54.153 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:54.179 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:54.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:54.281 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:54.305 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:54.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:54.408 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:54.435 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:54.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:54.537 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:54.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.565 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:54.566 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:54.666 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:54.696 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:54.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:54.798 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:54.824 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:54.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:54.926 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:54.948 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:54.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:54.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:55.050 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:55.075 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:55.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:55.177 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:55.203 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:55.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:55.305 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:55.329 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:55.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:55.431 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:55.460 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:55.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:55.562 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:55.590 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:55.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:55.606 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:39:55.607 INFO:teuthology.orchestra.run.smithi063.stdout:{"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 10668/80928 objects degraded (13.182%), 4 pgs degraded, 5 pgs undersized","count":9},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2275,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1427,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916154,"num_in_osds":8,"osd_in_since":1723916113,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":112},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+clean+scrubbing","count":2},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":26976,"data_bytes":1766770353,"bytes_used":12732059648,"bytes_avail":755228225536,"bytes_total":767960285184,"degraded_objects":10668,"degraded_total":80928,"degraded_ratio":0.1318208778173191,"misplaced_objects":2118,"misplaced_total":80928,"misplaced_ratio":0.026171411625148279,"recovering_objects_per_sec":73,"recovering_bytes_per_sec":4789286,"recovering_keys_per_sec":0,"num_objects_recovered":731,"num_bytes_recovered":47906816,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":292},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi063.ultpqj","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.63:8443/","prometheus":"http://172.21.15.63:9283/"}},"servicemap":{"epoch":33,"modified":"2024-08-17T17:38:44.649232+0000","services":{}},"progress_events":{}} 2024-08-17T17:39:55.693 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:55.720 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:55.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.721 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:55.822 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:55.849 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:55.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:55.951 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:55.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:55.975 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:55.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:56.076 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:55 smithi162 ceph-mon[104668]: pgmap v2267: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 292 op/s; 10668/80928 objects degraded (13.182%); 2118/80928 objects misplaced (2.617%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:39:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:55 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1748924087' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:39:56.107 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:56.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:56.147 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:55 smithi063 ceph-mon[114756]: pgmap v2267: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 292 op/s; 10668/80928 objects degraded (13.182%); 2118/80928 objects misplaced (2.617%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:39:56.147 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:55 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1748924087' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:39:56.148 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:55 smithi063 ceph-mon[109843]: pgmap v2267: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 292 op/s; 10668/80928 objects degraded (13.182%); 2118/80928 objects misplaced (2.617%); 4.6 MiB/s, 73 objects/s recovering 2024-08-17T17:39:56.148 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:55 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1748924087' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:39:56.208 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:56.238 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:56.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:56.340 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:56.366 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:56.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:56.467 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:56.495 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:56.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:56.597 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:56.627 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:56.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:56.729 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:56.752 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:56.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:56.854 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:56.881 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:56.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:56.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:56.982 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:57.007 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:57.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:57.109 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:57.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.135 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:57.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:57.236 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:57.269 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:57.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:57.371 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:57.392 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:57.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:57.493 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:57.517 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:57.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:57.619 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:57.648 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:57.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:57.750 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:57.775 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:57.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:57.876 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:39:57.901 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:57.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:57.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:39:58.002 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:39:58.002 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:58.027 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:58.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:58.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:57 smithi162 ceph-mon[104668]: pgmap v2268: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 269 op/s; 10492/79269 objects degraded (13.236%); 2118/79269 objects misplaced (2.672%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:39:58.129 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:58.132 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:57 smithi063 ceph-mon[114756]: pgmap v2268: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 269 op/s; 10492/79269 objects degraded (13.236%); 2118/79269 objects misplaced (2.672%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:39:58.133 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:57 smithi063 ceph-mon[109843]: pgmap v2268: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 269 op/s; 10492/79269 objects degraded (13.236%); 2118/79269 objects misplaced (2.672%); 4.2 MiB/s, 67 objects/s recovering 2024-08-17T17:39:58.154 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:58.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:58.256 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:58.278 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:58.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:58.379 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:58.406 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:58.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:58.507 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:58.534 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:58.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:58.634 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-08-17T17:39:58.636 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:58.664 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:58.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:58.766 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:39:58.791 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:58.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:39:58.825 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:39:58.893 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:58.916 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:58.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:58.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:58.968 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:58 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1399304678' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:58.968 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:58 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:58.968 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:58 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 10492/79269 objects degraded (13.236%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:58.968 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:58 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1399304678' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:58.968 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:58 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:58.969 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:58 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 10492/79269 objects degraded (13.236%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:59.001 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:39:59.002 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:39:59.019 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:59.045 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:59.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:39:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:58 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1399304678' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:58 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:39:59.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:58 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 10492/79269 objects degraded (13.236%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:39:59.147 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:39:59.172 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:59.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:39:59.274 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:39:59.301 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:59.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:39:59.403 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:59.430 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:59.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:59.532 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:39:59.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.562 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:59.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:39:59.663 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:39:59.691 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:59.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:39:59.793 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:39:59.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.828 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:39:59.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:39:59.929 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:39:59.958 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:39:59.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:39:59.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:00.060 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:00.084 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:00.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:59 smithi162 ceph-mon[104668]: pgmap v2269: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 319 op/s; 10067/76560 objects degraded (13.149%); 1915/76560 objects misplaced (2.501%); 5.5 MiB/s, 87 objects/s recovering 2024-08-17T17:40:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:59 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:40:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:59 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:00.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:39:59 smithi162 ceph-mon[104668]: osdmap e1428: 8 total, 8 up, 8 in 2024-08-17T17:40:00.187 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:00.191 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:59 smithi063 ceph-mon[109843]: pgmap v2269: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 319 op/s; 10067/76560 objects degraded (13.149%); 1915/76560 objects misplaced (2.501%); 5.5 MiB/s, 87 objects/s recovering 2024-08-17T17:40:00.191 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:59 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:40:00.191 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:59 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:00.191 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:39:59 smithi063 ceph-mon[109843]: osdmap e1428: 8 total, 8 up, 8 in 2024-08-17T17:40:00.191 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:59 smithi063 ceph-mon[114756]: pgmap v2269: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 319 op/s; 10067/76560 objects degraded (13.149%); 1915/76560 objects misplaced (2.501%); 5.5 MiB/s, 87 objects/s recovering 2024-08-17T17:40:00.192 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:59 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:40:00.192 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:59 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:00.192 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:39:59 smithi063 ceph-mon[114756]: osdmap e1428: 8 total, 8 up, 8 in 2024-08-17T17:40:00.209 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:00.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:00.311 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:00.342 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:00.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:00.444 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:00.469 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:00.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:00.571 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:00.593 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:00.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:00.695 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:00.720 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:00.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:00.822 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:00.846 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:00.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:00.948 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:00.970 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:00.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:00.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:01.013 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:00 smithi063 ceph-mon[114756]: Health detail: HEALTH_WARN Degraded data redundancy: 10067/76560 objects degraded (13.149%), 4 pgs degraded, 5 pgs undersized 2024-08-17T17:40:01.013 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:00 smithi063 ceph-mon[114756]: [WRN] PG_DEGRADED: Degraded data redundancy: 10067/76560 objects degraded (13.149%), 4 pgs degraded, 5 pgs undersized 2024-08-17T17:40:01.013 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:00 smithi063 ceph-mon[114756]: pg 11.1 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,0] 2024-08-17T17:40:01.013 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:00 smithi063 ceph-mon[114756]: pg 11.2 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,0] 2024-08-17T17:40:01.014 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:00 smithi063 ceph-mon[114756]: pg 11.8 is stuck undersized for 3m, current state active+recovering+undersized+remapped, last acting [6,5] 2024-08-17T17:40:01.014 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:00 smithi063 ceph-mon[114756]: pg 11.9 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,6] 2024-08-17T17:40:01.014 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:00 smithi063 ceph-mon[114756]: pg 11.a is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,5] 2024-08-17T17:40:01.014 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:00 smithi063 ceph-mon[109843]: Health detail: HEALTH_WARN Degraded data redundancy: 10067/76560 objects degraded (13.149%), 4 pgs degraded, 5 pgs undersized 2024-08-17T17:40:01.014 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:00 smithi063 ceph-mon[109843]: [WRN] PG_DEGRADED: Degraded data redundancy: 10067/76560 objects degraded (13.149%), 4 pgs degraded, 5 pgs undersized 2024-08-17T17:40:01.014 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:00 smithi063 ceph-mon[109843]: pg 11.1 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,0] 2024-08-17T17:40:01.014 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:00 smithi063 ceph-mon[109843]: pg 11.2 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,0] 2024-08-17T17:40:01.014 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:00 smithi063 ceph-mon[109843]: pg 11.8 is stuck undersized for 3m, current state active+recovering+undersized+remapped, last acting [6,5] 2024-08-17T17:40:01.014 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:00 smithi063 ceph-mon[109843]: pg 11.9 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,6] 2024-08-17T17:40:01.015 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:00 smithi063 ceph-mon[109843]: pg 11.a is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,5] 2024-08-17T17:40:01.072 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:01.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:00 smithi162 ceph-mon[104668]: Health detail: HEALTH_WARN Degraded data redundancy: 10067/76560 objects degraded (13.149%), 4 pgs degraded, 5 pgs undersized 2024-08-17T17:40:01.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:00 smithi162 ceph-mon[104668]: [WRN] PG_DEGRADED: Degraded data redundancy: 10067/76560 objects degraded (13.149%), 4 pgs degraded, 5 pgs undersized 2024-08-17T17:40:01.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:00 smithi162 ceph-mon[104668]: pg 11.1 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,0] 2024-08-17T17:40:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:00 smithi162 ceph-mon[104668]: pg 11.2 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,0] 2024-08-17T17:40:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:00 smithi162 ceph-mon[104668]: pg 11.8 is stuck undersized for 3m, current state active+recovering+undersized+remapped, last acting [6,5] 2024-08-17T17:40:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:00 smithi162 ceph-mon[104668]: pg 11.9 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,6] 2024-08-17T17:40:01.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:00 smithi162 ceph-mon[104668]: pg 11.a is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,5] 2024-08-17T17:40:01.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.097 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:01.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:01.198 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:01.225 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:01.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:01.327 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:01.355 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:01.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:01.458 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:01.489 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:01.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:01.591 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:01.620 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:40:01.620 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:40:01.623 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:01.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:01.645 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:40:01.645 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:40:01.680 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:40:01.680 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:40:01.708 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:40:01.708 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:40:01.726 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:01.735 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:40:01.736 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:40:01.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.753 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:01.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:01.759 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:40:01.759 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:40:01.790 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:40:01.791 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:40:01.812 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:40:01.812 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:40:01.854 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:01.883 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:01.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:01.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:01.985 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:02.013 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:02.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.014 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:01 smithi063 ceph-mon[109843]: pgmap v2271: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 10067/76236 objects degraded (13.205%); 1915/76236 objects misplaced (2.512%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:40:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:40:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:40:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:01 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:40:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:01 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1923241071' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:01 smithi063 ceph-mon[114756]: pgmap v2271: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 10067/76236 objects degraded (13.205%); 1915/76236 objects misplaced (2.512%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:40:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:40:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:40:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:01 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:40:02.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:01 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1923241071' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:01 smithi162 ceph-mon[104668]: pgmap v2271: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.5 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 303 op/s; 10067/76236 objects degraded (13.205%); 1915/76236 objects misplaced (2.512%); 4.9 MiB/s, 77 objects/s recovering 2024-08-17T17:40:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:40:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:40:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:01 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:40:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:01 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1923241071' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:02.115 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:02.141 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:02.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:02.242 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:02.265 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:02.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:02.367 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:02.397 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:02.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:02.498 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:02.521 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:02.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:02.623 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:02.647 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:02.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:02.748 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:02.777 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:02.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:02.879 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:02.905 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:02.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:02.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:03.006 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:03.040 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:03.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:03.044 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:03.141 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:03.180 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:03.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:03.282 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:03.309 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:03.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:03.411 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:03.440 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:03.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:03.541 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:03.565 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:03.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:03.650 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:40:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:40:03] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.51.0" 2024-08-17T17:40:03.667 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:03.696 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:03.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:03.797 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:03.824 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:03.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:03.871 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:03.926 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:03.950 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:03.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:03.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:03.987 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:03 smithi063 ceph-mon[114756]: pgmap v2272: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 331 op/s; 9436/71562 objects degraded (13.186%); 1691/71562 objects misplaced (2.363%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:40:03.987 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:03 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 9436/71562 objects degraded (13.186%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:03.987 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:03 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:40:03.987 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:03 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:40:03.988 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:03 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:40:03.988 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:03 smithi063 ceph-mon[109843]: pgmap v2272: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 331 op/s; 9436/71562 objects degraded (13.186%); 1691/71562 objects misplaced (2.363%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:40:03.988 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:03 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 9436/71562 objects degraded (13.186%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:03.988 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:03 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:40:03.988 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:03 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:40:03.988 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:03 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:40:04.053 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:04.081 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:04.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.082 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:04.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:03 smithi162 ceph-mon[104668]: pgmap v2272: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 331 op/s; 9436/71562 objects degraded (13.186%); 1691/71562 objects misplaced (2.363%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:40:04.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:03 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 9436/71562 objects degraded (13.186%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:04.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:03 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:40:04.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:03 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:40:04.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:03 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:40:04.183 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:04.185 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:40:04.208 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:04.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:04.310 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:04.332 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:04.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:04.433 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:04.457 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:04.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:04.559 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:04.584 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:04.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:04.686 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:04.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.731 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:04.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:04.832 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:04.857 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:04.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:04.958 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:04.988 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:04.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:04.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:05.056 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:40:05.090 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:05.126 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:05.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:05.228 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:05.258 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:05.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:05.360 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:05.385 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:05.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:05.412 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:05.487 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:05.510 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:05.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:05.612 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:05.649 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:05.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:05.751 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:05.777 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:05.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:05.879 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:05.907 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:05.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:05.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:05.941 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:40:05.943 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:40:06.009 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:06.032 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:06.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:05 smithi162 ceph-mon[104668]: pgmap v2273: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 322 op/s; 9436/71259 objects degraded (13.242%); 1691/71259 objects misplaced (2.373%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:40:06.134 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:06.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.158 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:06.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:06.259 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:06.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:05 smithi063 ceph-mon[114756]: pgmap v2273: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 322 op/s; 9436/71259 objects degraded (13.242%); 1691/71259 objects misplaced (2.373%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:40:06.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:05 smithi063 ceph-mon[109843]: pgmap v2273: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 322 op/s; 9436/71259 objects degraded (13.242%); 1691/71259 objects misplaced (2.373%); 5.2 MiB/s, 82 objects/s recovering 2024-08-17T17:40:06.286 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:06.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:06.388 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:06.417 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:06.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:06.519 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:06.546 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:06.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:06.648 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:06.671 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:06.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:06.773 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:06.798 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:06.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:06.899 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:06.925 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:06.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:06.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:07.027 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:07.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:06 smithi063 ceph-mon[109843]: from='client.46493 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:07.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:06 smithi063 ceph-mon[114756]: from='client.46493 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:07.049 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:07.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:07.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:06 smithi162 ceph-mon[104668]: from='client.46493 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:07.151 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:07.178 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:07.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:07.280 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:07.312 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:07.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:07.414 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:07.441 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:07.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:07.542 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:07.568 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:07.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:07.660 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2273,"stamp":"2024-08-17T17:40:04.672152+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1555547825,"num_objects":23753,"num_object_clones":0,"num_object_copies":71259,"num_objects_missing_on_primary":846,"num_objects_missing":846,"num_objects_degraded":9436,"num_objects_misplaced":1691,"num_objects_unfound":0,"num_objects_dirty":23753,"num_whiteouts":0,"num_read":28,"num_read_kb":62,"num_write":105234,"num_write_kb":4124377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33441,"num_bytes_recovered":2038698717,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103533,"ondisk_log_size":103533,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":375,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11960596,"kb_used_data":5982292,"kb_used_omap":298,"kb_used_meta":5977941,"kb_avail":738000620,"statfs":{"total":767960285184,"available":755712634880,"internally_reserved":0,"allocated":6125867008,"data_stored":6104372051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":6121412113},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,0,0,2,7,0,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-211222528,"num_objects":-3223,"num_object_clones":0,"num_object_copies":-9669,"num_objects_missing_on_primary":-627,"num_objects_missing":-627,"num_objects_degraded":-1232,"num_objects_misplaced":-427,"num_objects_unfound":0,"num_objects_dirty":-3223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3223,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":829,"num_bytes_recovered":54329344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002808"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":63,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823257+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:39:58.823257+0000","last_peered":"2024-08-17T17:39:58.823257+0000","last_clean":"2024-08-17T17:39:58.823257+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:39:58.823257+0000","last_undegraded":"2024-08-17T17:39:58.823257+0000","last_fullsized":"2024-08-17T17:39:58.823257+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1443,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.822732+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:39:45.822732+0000","last_peered":"2024-08-17T17:39:45.822732+0000","last_clean":"2024-08-17T17:39:45.822732+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:39:45.822732+0000","last_undegraded":"2024-08-17T17:39:45.822732+0000","last_fullsized":"2024-08-17T17:39:45.822732+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1757,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823432+0000","last_change":"2024-08-17T17:36:15.771150+0000","last_active":"2024-08-17T17:39:45.823432+0000","last_peered":"2024-08-17T17:39:45.823432+0000","last_clean":"2024-08-17T17:39:45.823432+0000","last_became_active":"2024-08-17T17:36:15.770520+0000","last_became_peered":"2024-08-17T17:36:15.770520+0000","last_unstale":"2024-08-17T17:39:45.823432+0000","last_undegraded":"2024-08-17T17:39:45.823432+0000","last_fullsized":"2024-08-17T17:39:45.823432+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:17.552796+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":93,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.827011+0000","last_change":"2024-08-17T17:36:12.756709+0000","last_active":"2024-08-17T17:39:58.827011+0000","last_peered":"2024-08-17T17:39:58.827011+0000","last_clean":"2024-08-17T17:39:58.827011+0000","last_became_active":"2024-08-17T17:36:12.756385+0000","last_became_peered":"2024-08-17T17:36:12.756385+0000","last_unstale":"2024-08-17T17:39:58.827011+0000","last_undegraded":"2024-08-17T17:39:58.827011+0000","last_fullsized":"2024-08-17T17:39:58.827011+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:21.420282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":147,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.892738+0000","last_change":"2024-08-17T17:36:12.762800+0000","last_active":"2024-08-17T17:39:58.892738+0000","last_peered":"2024-08-17T17:39:58.892738+0000","last_clean":"2024-08-17T17:39:58.892738+0000","last_became_active":"2024-08-17T17:36:12.759977+0000","last_became_peered":"2024-08-17T17:36:12.759977+0000","last_unstale":"2024-08-17T17:39:58.892738+0000","last_undegraded":"2024-08-17T17:39:58.892738+0000","last_fullsized":"2024-08-17T17:39:58.892738+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:02.162681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1637,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.803532+0000","last_change":"2024-08-17T17:35:56.281561+0000","last_active":"2024-08-17T17:39:45.803532+0000","last_peered":"2024-08-17T17:39:45.803532+0000","last_clean":"2024-08-17T17:39:45.803532+0000","last_became_active":"2024-08-17T17:35:56.278638+0000","last_became_peered":"2024-08-17T17:35:56.278638+0000","last_unstale":"2024-08-17T17:39:45.803532+0000","last_undegraded":"2024-08-17T17:39:45.803532+0000","last_fullsized":"2024-08-17T17:39:45.803532+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1376,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.802199+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:39:45.802199+0000","last_peered":"2024-08-17T17:39:45.802199+0000","last_clean":"2024-08-17T17:39:45.802199+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:39:45.802199+0000","last_undegraded":"2024-08-17T17:39:45.802199+0000","last_fullsized":"2024-08-17T17:39:45.802199+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":52,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.822836+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:39:45.822836+0000","last_peered":"2024-08-17T17:39:45.822836+0000","last_clean":"2024-08-17T17:39:45.822836+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:39:45.822836+0000","last_undegraded":"2024-08-17T17:39:45.822836+0000","last_fullsized":"2024-08-17T17:39:45.822836+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1421,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.801715+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:39:45.801715+0000","last_peered":"2024-08-17T17:39:45.801715+0000","last_clean":"2024-08-17T17:39:45.801715+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:39:45.801715+0000","last_undegraded":"2024-08-17T17:39:45.801715+0000","last_fullsized":"2024-08-17T17:39:45.801715+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1429,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823256+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:39:45.823256+0000","last_peered":"2024-08-17T17:39:45.823256+0000","last_clean":"2024-08-17T17:39:45.823256+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:39:45.823256+0000","last_undegraded":"2024-08-17T17:39:45.823256+0000","last_fullsized":"2024-08-17T17:39:45.823256+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":720,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.826864+0000","last_change":"2024-08-17T17:36:15.773315+0000","last_active":"2024-08-17T17:39:58.826864+0000","last_peered":"2024-08-17T17:39:58.826864+0000","last_clean":"2024-08-17T17:39:58.826864+0000","last_became_active":"2024-08-17T17:36:15.772961+0000","last_became_peered":"2024-08-17T17:36:15.772961+0000","last_unstale":"2024-08-17T17:39:58.826864+0000","last_undegraded":"2024-08-17T17:39:58.826864+0000","last_fullsized":"2024-08-17T17:39:58.826864+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:10.190221+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":57,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.822871+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:39:58.822871+0000","last_peered":"2024-08-17T17:39:58.822871+0000","last_clean":"2024-08-17T17:39:58.822871+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:39:58.822871+0000","last_undegraded":"2024-08-17T17:39:58.822871+0000","last_fullsized":"2024-08-17T17:39:58.822871+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":147,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.893173+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:39:58.893173+0000","last_peered":"2024-08-17T17:39:58.893173+0000","last_clean":"2024-08-17T17:39:58.893173+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:39:58.893173+0000","last_undegraded":"2024-08-17T17:39:58.893173+0000","last_fullsized":"2024-08-17T17:39:58.893173+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1510,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.816010+0000","last_change":"2024-08-17T17:36:24.225739+0000","last_active":"2024-08-17T17:39:45.816010+0000","last_peered":"2024-08-17T17:39:45.816010+0000","last_clean":"2024-08-17T17:39:45.816010+0000","last_became_active":"2024-08-17T17:36:24.225250+0000","last_became_peered":"2024-08-17T17:36:24.225250+0000","last_unstale":"2024-08-17T17:39:45.816010+0000","last_undegraded":"2024-08-17T17:39:45.816010+0000","last_fullsized":"2024-08-17T17:39:45.816010+0000","mapping_epoch":1393,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1394,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.290656+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1449,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823139+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:39:45.823139+0000","last_peered":"2024-08-17T17:39:45.823139+0000","last_clean":"2024-08-17T17:39:45.823139+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:39:45.823139+0000","last_undegraded":"2024-08-17T17:39:45.823139+0000","last_fullsized":"2024-08-17T17:39:45.823139+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1856,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823665+0000","last_change":"2024-08-17T17:36:12.752970+0000","last_active":"2024-08-17T17:39:45.823665+0000","last_peered":"2024-08-17T17:39:45.823665+0000","last_clean":"2024-08-17T17:39:45.823665+0000","last_became_active":"2024-08-17T17:36:12.752311+0000","last_became_peered":"2024-08-17T17:36:12.752311+0000","last_unstale":"2024-08-17T17:39:45.823665+0000","last_undegraded":"2024-08-17T17:39:45.823665+0000","last_fullsized":"2024-08-17T17:39:45.823665+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:38.131419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1804,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.800864+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:39:45.800864+0000","last_peered":"2024-08-17T17:39:45.800864+0000","last_clean":"2024-08-17T17:39:45.800864+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:39:45.800864+0000","last_undegraded":"2024-08-17T17:39:45.800864+0000","last_fullsized":"2024-08-17T17:39:45.800864+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1520,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.842550+0000","last_change":"2024-08-17T17:36:18.750494+0000","last_active":"2024-08-17T17:39:58.842550+0000","last_peered":"2024-08-17T17:39:58.842550+0000","last_clean":"2024-08-17T17:39:58.842550+0000","last_became_active":"2024-08-17T17:36:18.750092+0000","last_became_peered":"2024-08-17T17:36:18.750092+0000","last_unstale":"2024-08-17T17:39:58.842550+0000","last_undegraded":"2024-08-17T17:39:58.842550+0000","last_fullsized":"2024-08-17T17:39:58.842550+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:36.599727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1149,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823209+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:39:58.823209+0000","last_peered":"2024-08-17T17:39:58.823209+0000","last_clean":"2024-08-17T17:39:58.823209+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:39:58.823209+0000","last_undegraded":"2024-08-17T17:39:58.823209+0000","last_fullsized":"2024-08-17T17:39:58.823209+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":971,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.801253+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:39:45.801253+0000","last_peered":"2024-08-17T17:39:45.801253+0000","last_clean":"2024-08-17T17:39:45.801253+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:39:45.801253+0000","last_undegraded":"2024-08-17T17:39:45.801253+0000","last_fullsized":"2024-08-17T17:39:45.801253+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":57,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.822749+0000","last_change":"2024-08-17T17:37:11.075505+0000","last_active":"2024-08-17T17:39:58.822749+0000","last_peered":"2024-08-17T17:39:58.822749+0000","last_clean":"2024-08-17T17:39:58.822749+0000","last_became_active":"2024-08-17T17:37:11.075131+0000","last_became_peered":"2024-08-17T17:37:11.075131+0000","last_unstale":"2024-08-17T17:39:58.822749+0000","last_undegraded":"2024-08-17T17:39:58.822749+0000","last_fullsized":"2024-08-17T17:39:58.822749+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:39.672054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1284,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:57.583447+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:39:57.583447+0000","last_peered":"2024-08-17T17:39:57.583447+0000","last_clean":"2024-08-17T17:39:57.583447+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:39:57.583447+0000","last_undegraded":"2024-08-17T17:39:57.583447+0000","last_fullsized":"2024-08-17T17:39:57.583447+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1661,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.822880+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:39:45.822880+0000","last_peered":"2024-08-17T17:39:45.822880+0000","last_clean":"2024-08-17T17:39:45.822880+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:39:45.822880+0000","last_undegraded":"2024-08-17T17:39:45.822880+0000","last_fullsized":"2024-08-17T17:39:45.822880+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":103,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.843263+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:39:58.843263+0000","last_peered":"2024-08-17T17:39:58.843263+0000","last_clean":"2024-08-17T17:39:58.843263+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:39:58.843263+0000","last_undegraded":"2024-08-17T17:39:58.843263+0000","last_fullsized":"2024-08-17T17:39:58.843263+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1258,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.802382+0000","last_change":"2024-08-17T17:36:12.755873+0000","last_active":"2024-08-17T17:39:45.802382+0000","last_peered":"2024-08-17T17:39:45.802382+0000","last_clean":"2024-08-17T17:39:45.802382+0000","last_became_active":"2024-08-17T17:36:12.755366+0000","last_became_peered":"2024-08-17T17:36:12.755366+0000","last_unstale":"2024-08-17T17:39:45.802382+0000","last_undegraded":"2024-08-17T17:39:45.802382+0000","last_fullsized":"2024-08-17T17:39:45.802382+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:36.523214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1611,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.821511+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:39:45.821511+0000","last_peered":"2024-08-17T17:39:45.821511+0000","last_clean":"2024-08-17T17:39:45.821511+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:39:45.821511+0000","last_undegraded":"2024-08-17T17:39:45.821511+0000","last_fullsized":"2024-08-17T17:39:45.821511+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":795,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.816218+0000","last_change":"2024-08-17T17:35:55.200519+0000","last_active":"2024-08-17T17:39:45.816218+0000","last_peered":"2024-08-17T17:39:45.816218+0000","last_clean":"2024-08-17T17:39:45.816218+0000","last_became_active":"2024-08-17T17:35:55.200230+0000","last_became_peered":"2024-08-17T17:35:55.200230+0000","last_unstale":"2024-08-17T17:39:45.816218+0000","last_undegraded":"2024-08-17T17:39:45.816218+0000","last_fullsized":"2024-08-17T17:39:45.816218+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:33.454851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1435,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.801808+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:39:45.801808+0000","last_peered":"2024-08-17T17:39:45.801808+0000","last_clean":"2024-08-17T17:39:45.801808+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:39:45.801808+0000","last_undegraded":"2024-08-17T17:39:45.801808+0000","last_fullsized":"2024-08-17T17:39:45.801808+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1721,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.892768+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:39:58.892768+0000","last_peered":"2024-08-17T17:39:58.892768+0000","last_clean":"2024-08-17T17:39:58.892768+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:39:58.892768+0000","last_undegraded":"2024-08-17T17:39:58.892768+0000","last_fullsized":"2024-08-17T17:39:58.892768+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:22.077229+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":268,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.842714+0000","last_change":"2024-08-17T17:35:33.145476+0000","last_active":"2024-08-17T17:39:58.842714+0000","last_peered":"2024-08-17T17:39:58.842714+0000","last_clean":"2024-08-17T17:39:58.842714+0000","last_became_active":"2024-08-17T17:35:33.145125+0000","last_became_peered":"2024-08-17T17:35:33.145125+0000","last_unstale":"2024-08-17T17:39:58.842714+0000","last_undegraded":"2024-08-17T17:39:58.842714+0000","last_fullsized":"2024-08-17T17:39:58.842714+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:45.298064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":939,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.803362+0000","last_change":"2024-08-17T17:35:56.248612+0000","last_active":"2024-08-17T17:39:45.803362+0000","last_peered":"2024-08-17T17:39:45.803362+0000","last_clean":"2024-08-17T17:39:45.803362+0000","last_became_active":"2024-08-17T17:35:56.244364+0000","last_became_peered":"2024-08-17T17:35:56.244364+0000","last_unstale":"2024-08-17T17:39:45.803362+0000","last_undegraded":"2024-08-17T17:39:45.803362+0000","last_fullsized":"2024-08-17T17:39:45.803362+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:41.064165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":879,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.802179+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:39:45.802179+0000","last_peered":"2024-08-17T17:39:45.802179+0000","last_clean":"2024-08-17T17:39:45.802179+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:39:45.802179+0000","last_undegraded":"2024-08-17T17:39:45.802179+0000","last_fullsized":"2024-08-17T17:39:45.802179+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1414,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823365+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:39:45.823365+0000","last_peered":"2024-08-17T17:39:45.823365+0000","last_clean":"2024-08-17T17:39:45.823365+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:39:45.823365+0000","last_undegraded":"2024-08-17T17:39:45.823365+0000","last_fullsized":"2024-08-17T17:39:45.823365+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1433,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823330+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:39:45.823330+0000","last_peered":"2024-08-17T17:39:45.823330+0000","last_clean":"2024-08-17T17:39:45.823330+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:39:45.823330+0000","last_undegraded":"2024-08-17T17:39:45.823330+0000","last_fullsized":"2024-08-17T17:39:45.823330+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":942,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.877559+0000","last_change":"2024-08-17T17:36:15.774881+0000","last_active":"2024-08-17T17:39:58.877559+0000","last_peered":"2024-08-17T17:39:58.877559+0000","last_clean":"2024-08-17T17:39:58.877559+0000","last_became_active":"2024-08-17T17:36:15.774435+0000","last_became_peered":"2024-08-17T17:36:15.774435+0000","last_unstale":"2024-08-17T17:39:58.877559+0000","last_undegraded":"2024-08-17T17:39:58.877559+0000","last_fullsized":"2024-08-17T17:39:58.877559+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:52.015854+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"1428'6787","reported_seq":14971,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.371289+0000","last_change":"2024-08-17T17:38:37.110784+0000","last_active":"2024-08-17T17:40:02.371289+0000","last_peered":"2024-08-17T17:40:02.371289+0000","last_clean":"2024-08-17T17:40:02.371289+0000","last_became_active":"2024-08-17T17:38:37.088380+0000","last_became_peered":"2024-08-17T17:38:37.088380+0000","last_unstale":"2024-08-17T17:40:02.371289+0000","last_undegraded":"2024-08-17T17:40:02.371289+0000","last_fullsized":"2024-08-17T17:40:02.371289+0000","mapping_epoch":1419,"log_start":"1373'700","ondisk_log_start":"1373'700","created":1357,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6087,"log_dups_size":0,"ondisk_log_size":6087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:08.647903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98238464,"num_objects":1499,"num_object_clones":0,"num_object_copies":4497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6787,"num_write_kb":265152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4009,"num_bytes_recovered":199753728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1382,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.822776+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:39:45.822776+0000","last_peered":"2024-08-17T17:39:45.822776+0000","last_clean":"2024-08-17T17:39:45.822776+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:39:45.822776+0000","last_undegraded":"2024-08-17T17:39:45.822776+0000","last_fullsized":"2024-08-17T17:39:45.822776+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1277,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.803923+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:39:45.803923+0000","last_peered":"2024-08-17T17:39:45.803923+0000","last_clean":"2024-08-17T17:39:45.803923+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:39:45.803923+0000","last_undegraded":"2024-08-17T17:39:45.803923+0000","last_fullsized":"2024-08-17T17:39:45.803923+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":335,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823390+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:39:58.823390+0000","last_peered":"2024-08-17T17:39:58.823390+0000","last_clean":"2024-08-17T17:39:58.823390+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:39:58.823390+0000","last_undegraded":"2024-08-17T17:39:58.823390+0000","last_fullsized":"2024-08-17T17:39:58.823390+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1603,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.842631+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:39:58.842631+0000","last_peered":"2024-08-17T17:39:58.842631+0000","last_clean":"2024-08-17T17:39:58.842631+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:39:58.842631+0000","last_undegraded":"2024-08-17T17:39:58.842631+0000","last_fullsized":"2024-08-17T17:39:58.842631+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:25.751871+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.f","version":"1428'6667","reported_seq":6828,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:00.423815+0000","last_change":"2024-08-17T17:35:34.179168+0000","last_active":"2024-08-17T17:40:00.423815+0000","last_peered":"2024-08-17T17:40:00.423815+0000","last_clean":"2024-08-17T17:40:00.423815+0000","last_became_active":"2024-08-17T17:35:34.133493+0000","last_became_peered":"2024-08-17T17:35:34.133493+0000","last_unstale":"2024-08-17T17:40:00.423815+0000","last_undegraded":"2024-08-17T17:40:00.423815+0000","last_fullsized":"2024-08-17T17:40:00.423815+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6667,"log_dups_size":0,"ondisk_log_size":6667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:24:11.188566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100728832,"num_objects":1537,"num_object_clones":0,"num_object_copies":4611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6667,"num_write_kb":262528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":110,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.843322+0000","last_change":"2024-08-17T17:37:11.079712+0000","last_active":"2024-08-17T17:39:58.843322+0000","last_peered":"2024-08-17T17:39:58.843322+0000","last_clean":"2024-08-17T17:39:58.843322+0000","last_became_active":"2024-08-17T17:37:11.079253+0000","last_became_peered":"2024-08-17T17:37:11.079253+0000","last_unstale":"2024-08-17T17:39:58.843322+0000","last_undegraded":"2024-08-17T17:39:58.843322+0000","last_fullsized":"2024-08-17T17:39:58.843322+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:53.405657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":922,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.803368+0000","last_change":"2024-08-17T17:35:56.223396+0000","last_active":"2024-08-17T17:39:45.803368+0000","last_peered":"2024-08-17T17:39:45.803368+0000","last_clean":"2024-08-17T17:39:45.803368+0000","last_became_active":"2024-08-17T17:35:56.212413+0000","last_became_peered":"2024-08-17T17:35:56.212413+0000","last_unstale":"2024-08-17T17:39:45.803368+0000","last_undegraded":"2024-08-17T17:39:45.803368+0000","last_fullsized":"2024-08-17T17:39:45.803368+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:44.185174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1645,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823500+0000","last_change":"2024-08-17T17:35:33.136068+0000","last_active":"2024-08-17T17:39:58.823500+0000","last_peered":"2024-08-17T17:39:58.823500+0000","last_clean":"2024-08-17T17:39:58.823500+0000","last_became_active":"2024-08-17T17:35:33.135591+0000","last_became_peered":"2024-08-17T17:35:33.135591+0000","last_unstale":"2024-08-17T17:39:58.823500+0000","last_undegraded":"2024-08-17T17:39:58.823500+0000","last_fullsized":"2024-08-17T17:39:58.823500+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:52.252561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1788,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823765+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:39:45.823765+0000","last_peered":"2024-08-17T17:39:45.823765+0000","last_clean":"2024-08-17T17:39:45.823765+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:39:45.823765+0000","last_undegraded":"2024-08-17T17:39:45.823765+0000","last_fullsized":"2024-08-17T17:39:45.823765+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"1428'6539","reported_seq":6685,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:00.473150+0000","last_change":"2024-08-17T17:35:14.861823+0000","last_active":"2024-08-17T17:40:00.473150+0000","last_peered":"2024-08-17T17:40:00.473150+0000","last_clean":"2024-08-17T17:40:00.473150+0000","last_became_active":"2024-08-17T17:35:14.854109+0000","last_became_peered":"2024-08-17T17:35:14.854109+0000","last_unstale":"2024-08-17T17:40:00.473150+0000","last_undegraded":"2024-08-17T17:40:00.473150+0000","last_fullsized":"2024-08-17T17:40:00.473150+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6539,"log_dups_size":0,"ondisk_log_size":6539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:46:40.778197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98762752,"num_objects":1507,"num_object_clones":0,"num_object_copies":4521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6539,"num_write_kb":257472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1000,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.801382+0000","last_change":"2024-08-17T17:35:33.138467+0000","last_active":"2024-08-17T17:39:45.801382+0000","last_peered":"2024-08-17T17:39:45.801382+0000","last_clean":"2024-08-17T17:39:45.801382+0000","last_became_active":"2024-08-17T17:35:33.138174+0000","last_became_peered":"2024-08-17T17:35:33.138174+0000","last_unstale":"2024-08-17T17:39:45.801382+0000","last_undegraded":"2024-08-17T17:39:45.801382+0000","last_fullsized":"2024-08-17T17:39:45.801382+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:08.860439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1385,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.821506+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:39:45.821506+0000","last_peered":"2024-08-17T17:39:45.821506+0000","last_clean":"2024-08-17T17:39:45.821506+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:39:45.821506+0000","last_undegraded":"2024-08-17T17:39:45.821506+0000","last_fullsized":"2024-08-17T17:39:45.821506+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":141,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.893137+0000","last_change":"2024-08-17T17:35:14.854881+0000","last_active":"2024-08-17T17:39:58.893137+0000","last_peered":"2024-08-17T17:39:58.893137+0000","last_clean":"2024-08-17T17:39:58.893137+0000","last_became_active":"2024-08-17T17:35:14.854121+0000","last_became_peered":"2024-08-17T17:35:14.854121+0000","last_unstale":"2024-08-17T17:39:58.893137+0000","last_undegraded":"2024-08-17T17:39:58.893137+0000","last_fullsized":"2024-08-17T17:39:58.893137+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.863624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":941,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.804061+0000","last_change":"2024-08-17T17:36:15.773050+0000","last_active":"2024-08-17T17:39:45.804061+0000","last_peered":"2024-08-17T17:39:45.804061+0000","last_clean":"2024-08-17T17:39:45.804061+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:39:45.804061+0000","last_undegraded":"2024-08-17T17:39:45.804061+0000","last_fullsized":"2024-08-17T17:39:45.804061+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:39.212195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:39.212195+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"1428'6534","reported_seq":12386,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:00.384734+0000","last_change":"2024-08-17T17:37:21.409979+0000","last_active":"2024-08-17T17:40:00.384734+0000","last_peered":"2024-08-17T17:40:00.384734+0000","last_clean":"2024-08-17T17:40:00.384734+0000","last_became_active":"2024-08-17T17:37:21.398111+0000","last_became_peered":"2024-08-17T17:37:21.398111+0000","last_unstale":"2024-08-17T17:40:00.384734+0000","last_undegraded":"2024-08-17T17:40:00.384734+0000","last_fullsized":"2024-08-17T17:40:00.384734+0000","mapping_epoch":1407,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6534,"log_dups_size":0,"ondisk_log_size":6534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:19.299395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98107416,"num_objects":1498,"num_object_clones":0,"num_object_copies":4494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1498,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6534,"num_write_kb":256961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2850,"num_bytes_recovered":168820736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1043,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.801432+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:39:45.801432+0000","last_peered":"2024-08-17T17:39:45.801432+0000","last_clean":"2024-08-17T17:39:45.801432+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:39:45.801432+0000","last_undegraded":"2024-08-17T17:39:45.801432+0000","last_fullsized":"2024-08-17T17:39:45.801432+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":235,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.877544+0000","last_change":"2024-08-17T17:35:33.145351+0000","last_active":"2024-08-17T17:39:58.877544+0000","last_peered":"2024-08-17T17:39:58.877544+0000","last_clean":"2024-08-17T17:39:58.877544+0000","last_became_active":"2024-08-17T17:35:33.145040+0000","last_became_peered":"2024-08-17T17:35:33.145040+0000","last_unstale":"2024-08-17T17:39:58.877544+0000","last_undegraded":"2024-08-17T17:39:58.877544+0000","last_fullsized":"2024-08-17T17:39:58.877544+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:02.124000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1453,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.802191+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:39:45.802191+0000","last_peered":"2024-08-17T17:39:45.802191+0000","last_clean":"2024-08-17T17:39:45.802191+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:39:45.802191+0000","last_undegraded":"2024-08-17T17:39:45.802191+0000","last_fullsized":"2024-08-17T17:39:45.802191+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":924,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.803892+0000","last_change":"2024-08-17T17:36:15.769890+0000","last_active":"2024-08-17T17:39:45.803892+0000","last_peered":"2024-08-17T17:39:45.803892+0000","last_clean":"2024-08-17T17:39:45.803892+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:39:45.803892+0000","last_undegraded":"2024-08-17T17:39:45.803892+0000","last_fullsized":"2024-08-17T17:39:45.803892+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:33.034306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:33.034306+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":4,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.b","version":"1428'6512","reported_seq":14323,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:03.625234+0000","last_change":"2024-08-17T17:38:07.888612+0000","last_active":"2024-08-17T17:40:03.625234+0000","last_peered":"2024-08-17T17:40:03.625234+0000","last_clean":"2024-08-17T17:40:03.625234+0000","last_became_active":"2024-08-17T17:38:07.875196+0000","last_became_peered":"2024-08-17T17:38:07.875196+0000","last_unstale":"2024-08-17T17:40:03.625234+0000","last_undegraded":"2024-08-17T17:40:03.625234+0000","last_fullsized":"2024-08-17T17:40:03.625234+0000","mapping_epoch":1415,"log_start":"1373'600","ondisk_log_start":"1373'600","created":1357,"last_epoch_clean":1416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5912,"log_dups_size":0,"ondisk_log_size":5912,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:21.658004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94502912,"num_objects":1442,"num_object_clones":0,"num_object_copies":4326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6512,"num_write_kb":254528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3836,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1496,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.893265+0000","last_change":"2024-08-17T17:36:12.759743+0000","last_active":"2024-08-17T17:39:58.893265+0000","last_peered":"2024-08-17T17:39:58.893265+0000","last_clean":"2024-08-17T17:39:58.893265+0000","last_became_active":"2024-08-17T17:36:12.759334+0000","last_became_peered":"2024-08-17T17:36:12.759334+0000","last_unstale":"2024-08-17T17:39:58.893265+0000","last_undegraded":"2024-08-17T17:39:58.893265+0000","last_fullsized":"2024-08-17T17:39:58.893265+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:04.524384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1416,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823534+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:39:45.823534+0000","last_peered":"2024-08-17T17:39:45.823534+0000","last_clean":"2024-08-17T17:39:45.823534+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:39:45.823534+0000","last_undegraded":"2024-08-17T17:39:45.823534+0000","last_fullsized":"2024-08-17T17:39:45.823534+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1458,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.801641+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:39:45.801641+0000","last_peered":"2024-08-17T17:39:45.801641+0000","last_clean":"2024-08-17T17:39:45.801641+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:39:45.801641+0000","last_undegraded":"2024-08-17T17:39:45.801641+0000","last_fullsized":"2024-08-17T17:39:45.801641+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1774,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.801103+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:39:45.801103+0000","last_peered":"2024-08-17T17:39:45.801103+0000","last_clean":"2024-08-17T17:39:45.801103+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:39:45.801103+0000","last_undegraded":"2024-08-17T17:39:45.801103+0000","last_fullsized":"2024-08-17T17:39:45.801103+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"1428'6671","reported_seq":14515,"reported_epoch":1428,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:40:02.618179+0000","last_change":"2024-08-17T17:38:05.991715+0000","last_active":"2024-08-17T17:40:02.618179+0000","last_peered":"2024-08-17T17:40:02.618179+0000","last_clean":"2024-08-17T17:35:54.187688+0000","last_became_active":"2024-08-17T17:36:22.904523+0000","last_became_peered":"2024-08-17T17:36:22.904523+0000","last_unstale":"2024-08-17T17:40:02.618179+0000","last_undegraded":"2024-08-17T17:40:02.618179+0000","last_fullsized":"2024-08-17T17:36:13.740491+0000","mapping_epoch":1383,"log_start":"1373'900","ondisk_log_start":"1373'900","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5771,"log_dups_size":0,"ondisk_log_size":5771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:57.525855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99418112,"num_objects":1517,"num_object_clones":0,"num_object_copies":4551,"num_objects_missing_on_primary":306,"num_objects_missing":306,"num_objects_degraded":0,"num_objects_misplaced":1691,"num_objects_unfound":0,"num_objects_dirty":1517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6671,"num_write_kb":262016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2583,"num_bytes_recovered":169279488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":306},{"shards":"2,4,5,6","objects":2},{"shards":"2,5,6","objects":1077},{"shards":"5,6","objects":132}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1092,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.802254+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:39:45.802254+0000","last_peered":"2024-08-17T17:39:45.802254+0000","last_clean":"2024-08-17T17:39:45.802254+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:39:45.802254+0000","last_undegraded":"2024-08-17T17:39:45.802254+0000","last_fullsized":"2024-08-17T17:39:45.802254+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1408,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.892673+0000","last_change":"2024-08-17T17:35:56.229622+0000","last_active":"2024-08-17T17:39:58.892673+0000","last_peered":"2024-08-17T17:39:58.892673+0000","last_clean":"2024-08-17T17:39:58.892673+0000","last_became_active":"2024-08-17T17:35:56.229324+0000","last_became_peered":"2024-08-17T17:35:56.229324+0000","last_unstale":"2024-08-17T17:39:58.892673+0000","last_undegraded":"2024-08-17T17:39:58.892673+0000","last_fullsized":"2024-08-17T17:39:58.892673+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:13.980316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2011,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.892800+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:39:58.892800+0000","last_peered":"2024-08-17T17:39:58.892800+0000","last_clean":"2024-08-17T17:39:58.892800+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:39:58.892800+0000","last_undegraded":"2024-08-17T17:39:58.892800+0000","last_fullsized":"2024-08-17T17:39:58.892800+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1588,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.822942+0000","last_change":"2024-08-17T17:35:33.137074+0000","last_active":"2024-08-17T17:39:45.822942+0000","last_peered":"2024-08-17T17:39:45.822942+0000","last_clean":"2024-08-17T17:39:45.822942+0000","last_became_active":"2024-08-17T17:35:33.136759+0000","last_became_peered":"2024-08-17T17:35:33.136759+0000","last_unstale":"2024-08-17T17:39:45.822942+0000","last_undegraded":"2024-08-17T17:39:45.822942+0000","last_fullsized":"2024-08-17T17:39:45.822942+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:37.263639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:37.263639+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:14.599780+0000","scrub_duration":0.0016181800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"1428'6605","reported_seq":13694,"reported_epoch":1428,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:00.689976+0000","last_change":"2024-08-17T17:36:23.019129+0000","last_active":"2024-08-17T17:40:00.689976+0000","last_peered":"2024-08-17T17:40:00.689976+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:23.013527+0000","last_became_peered":"2024-08-17T17:36:23.013527+0000","last_unstale":"2024-08-17T17:40:00.689976+0000","last_undegraded":"2024-08-17T17:36:13.874715+0000","last_fullsized":"2024-08-17T17:36:13.738828+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6605,"log_dups_size":0,"ondisk_log_size":6605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97976320,"num_objects":1495,"num_object_clones":0,"num_object_copies":4485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2423,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6605,"num_write_kb":259200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2557,"num_bytes_recovered":167575552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[0,6],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":319},{"shards":"0,1,6","objects":1048},{"shards":"0,6","objects":-609}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1200,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823671+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:39:58.823671+0000","last_peered":"2024-08-17T17:39:58.823671+0000","last_clean":"2024-08-17T17:39:58.823671+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:39:58.823671+0000","last_undegraded":"2024-08-17T17:39:58.823671+0000","last_fullsized":"2024-08-17T17:39:58.823671+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.822764+0000","last_change":"2024-08-17T17:36:17.774773+0000","last_active":"2024-08-17T17:39:45.822764+0000","last_peered":"2024-08-17T17:39:45.822764+0000","last_clean":"2024-08-17T17:39:45.822764+0000","last_became_active":"2024-08-17T17:36:17.774468+0000","last_became_peered":"2024-08-17T17:36:17.774468+0000","last_unstale":"2024-08-17T17:39:45.822764+0000","last_undegraded":"2024-08-17T17:39:45.822764+0000","last_fullsized":"2024-08-17T17:39:45.822764+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:54.231747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1538,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.822697+0000","last_change":"2024-08-17T17:35:33.137633+0000","last_active":"2024-08-17T17:39:58.822697+0000","last_peered":"2024-08-17T17:39:58.822697+0000","last_clean":"2024-08-17T17:39:58.822697+0000","last_became_active":"2024-08-17T17:35:33.137323+0000","last_became_peered":"2024-08-17T17:35:33.137323+0000","last_unstale":"2024-08-17T17:39:58.822697+0000","last_undegraded":"2024-08-17T17:39:58.822697+0000","last_fullsized":"2024-08-17T17:39:58.822697+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:03.359155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":903,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.803315+0000","last_change":"2024-08-17T17:35:56.264626+0000","last_active":"2024-08-17T17:39:45.803315+0000","last_peered":"2024-08-17T17:39:45.803315+0000","last_clean":"2024-08-17T17:39:45.803315+0000","last_became_active":"2024-08-17T17:35:56.264265+0000","last_became_peered":"2024-08-17T17:35:56.264265+0000","last_unstale":"2024-08-17T17:39:45.803315+0000","last_undegraded":"2024-08-17T17:39:45.803315+0000","last_fullsized":"2024-08-17T17:39:45.803315+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:20.754326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:20.754326+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"1428'6574","reported_seq":14363,"reported_epoch":1428,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:02.625781+0000","last_change":"2024-08-17T17:37:11.134558+0000","last_active":"2024-08-17T17:40:02.625781+0000","last_peered":"2024-08-17T17:40:02.625781+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:20.105110+0000","last_became_peered":"2024-08-17T17:36:20.105110+0000","last_unstale":"2024-08-17T17:40:02.625781+0000","last_undegraded":"2024-08-17T17:36:19.999272+0000","last_fullsized":"2024-08-17T17:36:13.740520+0000","mapping_epoch":1383,"log_start":"1373'800","ondisk_log_start":"1373'800","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5774,"log_dups_size":0,"ondisk_log_size":5774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:45.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94371840,"num_objects":1440,"num_object_clones":0,"num_object_copies":4320,"num_objects_missing_on_primary":249,"num_objects_missing":249,"num_objects_degraded":1535,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6574,"num_write_kb":256448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2567,"num_bytes_recovered":168230912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":249},{"shards":"1,5,6","objects":1037},{"shards":"5,6","objects":154}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1256,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.804152+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:39:45.804152+0000","last_peered":"2024-08-17T17:39:45.804152+0000","last_clean":"2024-08-17T17:39:45.804152+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:39:45.804152+0000","last_undegraded":"2024-08-17T17:39:45.804152+0000","last_fullsized":"2024-08-17T17:39:45.804152+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1796,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.801570+0000","last_change":"2024-08-17T17:36:12.758005+0000","last_active":"2024-08-17T17:39:45.801570+0000","last_peered":"2024-08-17T17:39:45.801570+0000","last_clean":"2024-08-17T17:39:45.801570+0000","last_became_active":"2024-08-17T17:36:12.757695+0000","last_became_peered":"2024-08-17T17:36:12.757695+0000","last_unstale":"2024-08-17T17:39:45.801570+0000","last_undegraded":"2024-08-17T17:39:45.801570+0000","last_fullsized":"2024-08-17T17:39:45.801570+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:51.792220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1434,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.801462+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:39:45.801462+0000","last_peered":"2024-08-17T17:39:45.801462+0000","last_clean":"2024-08-17T17:39:45.801462+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:39:45.801462+0000","last_undegraded":"2024-08-17T17:39:45.801462+0000","last_fullsized":"2024-08-17T17:39:45.801462+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1751,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.822822+0000","last_change":"2024-08-17T17:39:58.822822+0000","last_active":"2024-08-17T17:39:58.822822+0000","last_peered":"2024-08-17T17:39:58.822822+0000","last_clean":"2024-08-17T17:39:58.822822+0000","last_became_active":"2024-08-17T17:35:55.204861+0000","last_became_peered":"2024-08-17T17:35:55.204861+0000","last_unstale":"2024-08-17T17:39:58.822822+0000","last_undegraded":"2024-08-17T17:39:58.822822+0000","last_fullsized":"2024-08-17T17:39:58.822822+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:40:01.491207+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.0","version":"1428'6623","reported_seq":14623,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.629303+0000","last_change":"2024-08-17T17:37:40.753703+0000","last_active":"2024-08-17T17:40:02.629303+0000","last_peered":"2024-08-17T17:40:02.629303+0000","last_clean":"2024-08-17T17:40:02.629303+0000","last_became_active":"2024-08-17T17:37:40.743458+0000","last_became_peered":"2024-08-17T17:37:40.743458+0000","last_unstale":"2024-08-17T17:40:02.629303+0000","last_undegraded":"2024-08-17T17:40:02.629303+0000","last_fullsized":"2024-08-17T17:40:02.629303+0000","mapping_epoch":1411,"log_start":"1373'1100","ondisk_log_start":"1373'1100","created":1357,"last_epoch_clean":1412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5523,"log_dups_size":0,"ondisk_log_size":5523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:14:02.340380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97845248,"num_objects":1493,"num_object_clones":0,"num_object_copies":4479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6623,"num_write_kb":259712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3914,"num_bytes_recovered":232652800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1030,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823320+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:39:45.823320+0000","last_peered":"2024-08-17T17:39:45.823320+0000","last_clean":"2024-08-17T17:39:45.823320+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:39:45.823320+0000","last_undegraded":"2024-08-17T17:39:45.823320+0000","last_fullsized":"2024-08-17T17:39:45.823320+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":103,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.843438+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:39:58.843438+0000","last_peered":"2024-08-17T17:39:58.843438+0000","last_clean":"2024-08-17T17:39:58.843438+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:39:58.843438+0000","last_undegraded":"2024-08-17T17:39:58.843438+0000","last_fullsized":"2024-08-17T17:39:58.843438+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1765,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.801275+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:39:45.801275+0000","last_peered":"2024-08-17T17:39:45.801275+0000","last_clean":"2024-08-17T17:39:45.801275+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:39:45.801275+0000","last_undegraded":"2024-08-17T17:39:45.801275+0000","last_fullsized":"2024-08-17T17:39:45.801275+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1","version":"1428'6560","reported_seq":14267,"reported_epoch":1428,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:02.620627+0000","last_change":"2024-08-17T17:36:23.103523+0000","last_active":"2024-08-17T17:40:02.620627+0000","last_peered":"2024-08-17T17:40:02.620627+0000","last_clean":"2024-08-17T17:35:54.178280+0000","last_became_active":"2024-08-17T17:36:23.025910+0000","last_became_peered":"2024-08-17T17:36:23.025910+0000","last_unstale":"2024-08-17T17:40:02.620627+0000","last_undegraded":"2024-08-17T17:36:18.734637+0000","last_fullsized":"2024-08-17T17:36:13.740134+0000","mapping_epoch":1383,"log_start":"1373'700","ondisk_log_start":"1373'700","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5860,"log_dups_size":0,"ondisk_log_size":5860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:58.717055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97779712,"num_objects":1492,"num_object_clones":0,"num_object_copies":4476,"num_objects_missing_on_primary":291,"num_objects_missing":291,"num_objects_degraded":1659,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6560,"num_write_kb":257664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2536,"num_bytes_recovered":166199296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,0],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":291},{"shards":"0,2,4,6","objects":7},{"shards":"0,2,6","objects":1070},{"shards":"0,6","objects":124}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1444,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823695+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:39:45.823695+0000","last_peered":"2024-08-17T17:39:45.823695+0000","last_clean":"2024-08-17T17:39:45.823695+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:39:45.823695+0000","last_undegraded":"2024-08-17T17:39:45.823695+0000","last_fullsized":"2024-08-17T17:39:45.823695+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1417,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823181+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:39:45.823181+0000","last_peered":"2024-08-17T17:39:45.823181+0000","last_clean":"2024-08-17T17:39:45.823181+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:39:45.823181+0000","last_undegraded":"2024-08-17T17:39:45.823181+0000","last_fullsized":"2024-08-17T17:39:45.823181+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":1005,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823076+0000","last_change":"2024-08-17T17:34:26.817339+0000","last_active":"2024-08-17T17:39:58.823076+0000","last_peered":"2024-08-17T17:39:58.823076+0000","last_clean":"2024-08-17T17:39:58.823076+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:39:58.823076+0000","last_undegraded":"2024-08-17T17:39:58.823076+0000","last_fullsized":"2024-08-17T17:39:58.823076+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:26.817223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:26.817223+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:30.089956+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"1428'6398","reported_seq":6569,"reported_epoch":1428,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:00.461469+0000","last_change":"2024-08-17T17:36:14.073155+0000","last_active":"2024-08-17T17:40:00.461469+0000","last_peered":"2024-08-17T17:40:00.461469+0000","last_clean":"2024-08-17T17:36:11.734683+0000","last_became_active":"2024-08-17T17:36:14.063794+0000","last_became_peered":"2024-08-17T17:36:14.063794+0000","last_unstale":"2024-08-17T17:40:00.461469+0000","last_undegraded":"2024-08-17T17:36:13.855770+0000","last_fullsized":"2024-08-17T17:36:13.742068+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6398,"log_dups_size":0,"ondisk_log_size":6398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98566144,"num_objects":1504,"num_object_clones":0,"num_object_copies":4512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3819,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6398,"num_write_kb":252864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-2315},{"shards":"0,2,7","objects":1372}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":52,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823678+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:39:45.823678+0000","last_peered":"2024-08-17T17:39:45.823678+0000","last_clean":"2024-08-17T17:39:45.823678+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:39:45.823678+0000","last_undegraded":"2024-08-17T17:39:45.823678+0000","last_fullsized":"2024-08-17T17:39:45.823678+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1278,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.803293+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:39:45.803293+0000","last_peered":"2024-08-17T17:39:45.803293+0000","last_clean":"2024-08-17T17:39:45.803293+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:39:45.803293+0000","last_undegraded":"2024-08-17T17:39:45.803293+0000","last_fullsized":"2024-08-17T17:39:45.803293+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1793,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.800773+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:39:45.800773+0000","last_peered":"2024-08-17T17:39:45.800773+0000","last_clean":"2024-08-17T17:39:45.800773+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:39:45.800773+0000","last_undegraded":"2024-08-17T17:39:45.800773+0000","last_fullsized":"2024-08-17T17:39:45.800773+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"1428'6623","reported_seq":12426,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.584457+0000","last_change":"2024-08-17T17:36:46.982456+0000","last_active":"2024-08-17T17:40:02.584457+0000","last_peered":"2024-08-17T17:40:02.584457+0000","last_clean":"2024-08-17T17:40:02.584457+0000","last_became_active":"2024-08-17T17:36:46.974782+0000","last_became_peered":"2024-08-17T17:36:46.974782+0000","last_unstale":"2024-08-17T17:40:02.584457+0000","last_undegraded":"2024-08-17T17:40:02.584457+0000","last_fullsized":"2024-08-17T17:40:02.584457+0000","mapping_epoch":1399,"log_start":"1373'800","ondisk_log_start":"1373'800","created":1357,"last_epoch_clean":1400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5823,"log_dups_size":0,"ondisk_log_size":5823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:40.499321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94961664,"num_objects":1449,"num_object_clones":0,"num_object_copies":4347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6623,"num_write_kb":258304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2848,"num_bytes_recovered":186646528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1395,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823273+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:39:45.823273+0000","last_peered":"2024-08-17T17:39:45.823273+0000","last_clean":"2024-08-17T17:39:45.823273+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:39:45.823273+0000","last_undegraded":"2024-08-17T17:39:45.823273+0000","last_fullsized":"2024-08-17T17:39:45.823273+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":209,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823126+0000","last_change":"2024-08-17T17:36:12.754372+0000","last_active":"2024-08-17T17:39:45.823126+0000","last_peered":"2024-08-17T17:39:45.823126+0000","last_clean":"2024-08-17T17:39:45.823126+0000","last_became_active":"2024-08-17T17:36:12.746352+0000","last_became_peered":"2024-08-17T17:36:12.746352+0000","last_unstale":"2024-08-17T17:39:45.823126+0000","last_undegraded":"2024-08-17T17:39:45.823126+0000","last_fullsized":"2024-08-17T17:39:45.823126+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:07.807486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1677,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.892674+0000","last_change":"2024-08-17T17:36:15.771193+0000","last_active":"2024-08-17T17:39:58.892674+0000","last_peered":"2024-08-17T17:39:58.892674+0000","last_clean":"2024-08-17T17:39:58.892674+0000","last_became_active":"2024-08-17T17:36:15.770889+0000","last_became_peered":"2024-08-17T17:36:15.770889+0000","last_unstale":"2024-08-17T17:39:58.892674+0000","last_undegraded":"2024-08-17T17:39:58.892674+0000","last_fullsized":"2024-08-17T17:39:58.892674+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:22.978695+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.4","version":"1428'6561","reported_seq":12345,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.452680+0000","last_change":"2024-08-17T17:37:04.045576+0000","last_active":"2024-08-17T17:40:02.452680+0000","last_peered":"2024-08-17T17:40:02.452680+0000","last_clean":"2024-08-17T17:40:02.452680+0000","last_became_active":"2024-08-17T17:37:04.038810+0000","last_became_peered":"2024-08-17T17:37:04.038810+0000","last_unstale":"2024-08-17T17:40:02.452680+0000","last_undegraded":"2024-08-17T17:40:02.452680+0000","last_fullsized":"2024-08-17T17:40:02.452680+0000","mapping_epoch":1402,"log_start":"1373'700","ondisk_log_start":"1373'700","created":1357,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5861,"log_dups_size":0,"ondisk_log_size":5861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:27:04.725258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98631680,"num_objects":1505,"num_object_clones":0,"num_object_copies":4515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6561,"num_write_kb":258112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":176291840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1272,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823440+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:39:58.823440+0000","last_peered":"2024-08-17T17:39:58.823440+0000","last_clean":"2024-08-17T17:39:58.823440+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:39:58.823440+0000","last_undegraded":"2024-08-17T17:39:58.823440+0000","last_fullsized":"2024-08-17T17:39:58.823440+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1421,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.801761+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:39:45.801761+0000","last_peered":"2024-08-17T17:39:45.801761+0000","last_clean":"2024-08-17T17:39:45.801761+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:39:45.801761+0000","last_undegraded":"2024-08-17T17:39:45.801761+0000","last_fullsized":"2024-08-17T17:39:45.801761+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":922,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887739+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:39:58.887739+0000","last_peered":"2024-08-17T17:39:58.887739+0000","last_clean":"2024-08-17T17:39:58.887739+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:39:58.887739+0000","last_undegraded":"2024-08-17T17:39:58.887739+0000","last_fullsized":"2024-08-17T17:39:58.887739+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"1428'6651","reported_seq":12490,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.423114+0000","last_change":"2024-08-17T17:36:32.711490+0000","last_active":"2024-08-17T17:40:02.423114+0000","last_peered":"2024-08-17T17:40:02.423114+0000","last_clean":"2024-08-17T17:40:02.423114+0000","last_became_active":"2024-08-17T17:36:32.704370+0000","last_became_peered":"2024-08-17T17:36:32.704370+0000","last_unstale":"2024-08-17T17:40:02.423114+0000","last_undegraded":"2024-08-17T17:40:02.423114+0000","last_fullsized":"2024-08-17T17:40:02.423114+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6651,"log_dups_size":0,"ondisk_log_size":6651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:17.373879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95485952,"num_objects":1457,"num_object_clones":0,"num_object_copies":4371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6651,"num_write_kb":259456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2866,"num_bytes_recovered":187826176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1277,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.811502+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:39:45.811502+0000","last_peered":"2024-08-17T17:39:45.811502+0000","last_clean":"2024-08-17T17:39:45.811502+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:39:45.811502+0000","last_undegraded":"2024-08-17T17:39:45.811502+0000","last_fullsized":"2024-08-17T17:39:45.811502+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":158,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.802031+0000","last_change":"2024-08-17T17:35:55.217072+0000","last_active":"2024-08-17T17:39:45.802031+0000","last_peered":"2024-08-17T17:39:45.802031+0000","last_clean":"2024-08-17T17:39:45.802031+0000","last_became_active":"2024-08-17T17:35:55.216462+0000","last_became_peered":"2024-08-17T17:35:55.216462+0000","last_unstale":"2024-08-17T17:39:45.802031+0000","last_undegraded":"2024-08-17T17:39:45.802031+0000","last_fullsized":"2024-08-17T17:39:45.802031+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:34.937221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":904,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.804015+0000","last_change":"2024-08-17T17:35:55.215004+0000","last_active":"2024-08-17T17:39:45.804015+0000","last_peered":"2024-08-17T17:39:45.804015+0000","last_clean":"2024-08-17T17:39:45.804015+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:39:45.804015+0000","last_undegraded":"2024-08-17T17:39:45.804015+0000","last_fullsized":"2024-08-17T17:39:45.804015+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.6","version":"1428'6373","reported_seq":6520,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:00.522967+0000","last_change":"2024-08-17T17:35:14.868524+0000","last_active":"2024-08-17T17:40:00.522967+0000","last_peered":"2024-08-17T17:40:00.522967+0000","last_clean":"2024-08-17T17:40:00.522967+0000","last_became_active":"2024-08-17T17:35:14.854690+0000","last_became_peered":"2024-08-17T17:35:14.854690+0000","last_unstale":"2024-08-17T17:40:00.522967+0000","last_undegraded":"2024-08-17T17:40:00.522967+0000","last_fullsized":"2024-08-17T17:40:00.522967+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6373,"log_dups_size":0,"ondisk_log_size":6373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:05.388362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94568448,"num_objects":1443,"num_object_clones":0,"num_object_copies":4329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6373,"num_write_kb":250112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1463,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.822944+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:39:45.822944+0000","last_peered":"2024-08-17T17:39:45.822944+0000","last_clean":"2024-08-17T17:39:45.822944+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:39:45.822944+0000","last_undegraded":"2024-08-17T17:39:45.822944+0000","last_fullsized":"2024-08-17T17:39:45.822944+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":170,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823743+0000","last_change":"2024-08-17T17:35:33.147672+0000","last_active":"2024-08-17T17:39:45.823743+0000","last_peered":"2024-08-17T17:39:45.823743+0000","last_clean":"2024-08-17T17:39:45.823743+0000","last_became_active":"2024-08-17T17:35:33.147283+0000","last_became_peered":"2024-08-17T17:35:33.147283+0000","last_unstale":"2024-08-17T17:39:45.823743+0000","last_undegraded":"2024-08-17T17:39:45.823743+0000","last_fullsized":"2024-08-17T17:39:45.823743+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:41.677709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1686,"reported_epoch":1428,"state":"active+clean+scrubbing","last_fresh":"2024-08-17T17:39:58.842553+0000","last_change":"2024-08-17T17:40:02.128679+0000","last_active":"2024-08-17T17:39:58.842553+0000","last_peered":"2024-08-17T17:39:58.842553+0000","last_clean":"2024-08-17T17:39:58.842553+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:39:58.842553+0000","last_undegraded":"2024-08-17T17:39:58.842553+0000","last_fullsized":"2024-08-17T17:39:58.842553+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:38.241235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:38.241235+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"1428'6477","reported_seq":6558,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.386990+0000","last_change":"2024-08-17T17:34:50.670013+0000","last_active":"2024-08-17T17:40:02.386990+0000","last_peered":"2024-08-17T17:40:02.386990+0000","last_clean":"2024-08-17T17:40:02.386990+0000","last_became_active":"2024-08-17T17:34:50.669299+0000","last_became_peered":"2024-08-17T17:34:50.669299+0000","last_unstale":"2024-08-17T17:40:02.386990+0000","last_undegraded":"2024-08-17T17:40:02.386990+0000","last_fullsized":"2024-08-17T17:40:02.386990+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6477,"log_dups_size":0,"ondisk_log_size":6477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:11.172227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94961664,"num_objects":1449,"num_object_clones":0,"num_object_copies":4347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6477,"num_write_kb":253632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1387,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.800887+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:39:45.800887+0000","last_peered":"2024-08-17T17:39:45.800887+0000","last_clean":"2024-08-17T17:39:45.800887+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:39:45.800887+0000","last_undegraded":"2024-08-17T17:39:45.800887+0000","last_fullsized":"2024-08-17T17:39:45.800887+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":157,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823454+0000","last_change":"2024-08-17T17:35:33.140725+0000","last_active":"2024-08-17T17:39:45.823454+0000","last_peered":"2024-08-17T17:39:45.823454+0000","last_clean":"2024-08-17T17:39:45.823454+0000","last_became_active":"2024-08-17T17:35:33.140406+0000","last_became_peered":"2024-08-17T17:35:33.140406+0000","last_unstale":"2024-08-17T17:39:45.823454+0000","last_undegraded":"2024-08-17T17:39:45.823454+0000","last_fullsized":"2024-08-17T17:39:45.823454+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:54:51.875571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1677,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.823554+0000","last_change":"2024-08-17T17:35:33.138514+0000","last_active":"2024-08-17T17:39:45.823554+0000","last_peered":"2024-08-17T17:39:45.823554+0000","last_clean":"2024-08-17T17:39:45.823554+0000","last_became_active":"2024-08-17T17:35:33.137035+0000","last_became_peered":"2024-08-17T17:35:33.137035+0000","last_unstale":"2024-08-17T17:39:45.823554+0000","last_undegraded":"2024-08-17T17:39:45.823554+0000","last_fullsized":"2024-08-17T17:39:45.823554+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:50.565662+0000","scrub_duration":0.001487138,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1318,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.808310+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:39:45.808310+0000","last_peered":"2024-08-17T17:39:45.808310+0000","last_clean":"2024-08-17T17:39:45.808310+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:39:45.808310+0000","last_undegraded":"2024-08-17T17:39:45.808310+0000","last_fullsized":"2024-08-17T17:39:45.808310+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":103,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.843150+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:39:58.843150+0000","last_peered":"2024-08-17T17:39:58.843150+0000","last_clean":"2024-08-17T17:39:58.843150+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:39:58.843150+0000","last_undegraded":"2024-08-17T17:39:58.843150+0000","last_fullsized":"2024-08-17T17:39:58.843150+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1749,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.801358+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:39:45.801358+0000","last_peered":"2024-08-17T17:39:45.801358+0000","last_clean":"2024-08-17T17:39:45.801358+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:39:45.801358+0000","last_undegraded":"2024-08-17T17:39:45.801358+0000","last_fullsized":"2024-08-17T17:39:45.801358+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1336,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:47.436688+0000","last_change":"2024-08-17T17:35:55.205292+0000","last_active":"2024-08-17T17:39:47.436688+0000","last_peered":"2024-08-17T17:39:47.436688+0000","last_clean":"2024-08-17T17:39:47.436688+0000","last_became_active":"2024-08-17T17:35:55.204935+0000","last_became_peered":"2024-08-17T17:35:55.204935+0000","last_unstale":"2024-08-17T17:39:47.436688+0000","last_undegraded":"2024-08-17T17:39:47.436688+0000","last_fullsized":"2024-08-17T17:39:47.436688+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:52.277821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":103,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.843028+0000","last_change":"2024-08-17T17:36:12.749972+0000","last_active":"2024-08-17T17:39:58.843028+0000","last_peered":"2024-08-17T17:39:58.843028+0000","last_clean":"2024-08-17T17:39:58.843028+0000","last_became_active":"2024-08-17T17:36:12.749620+0000","last_became_peered":"2024-08-17T17:36:12.749620+0000","last_unstale":"2024-08-17T17:39:58.843028+0000","last_undegraded":"2024-08-17T17:39:58.843028+0000","last_fullsized":"2024-08-17T17:39:58.843028+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:04.994357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":979,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.842757+0000","last_change":"2024-08-17T17:36:20.832274+0000","last_active":"2024-08-17T17:39:58.842757+0000","last_peered":"2024-08-17T17:39:58.842757+0000","last_clean":"2024-08-17T17:39:58.842757+0000","last_became_active":"2024-08-17T17:36:20.831886+0000","last_became_peered":"2024-08-17T17:36:20.831886+0000","last_unstale":"2024-08-17T17:39:58.842757+0000","last_undegraded":"2024-08-17T17:39:58.842757+0000","last_fullsized":"2024-08-17T17:39:58.842757+0000","mapping_epoch":1390,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1391,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:05.248711+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1189,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823186+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:39:58.823186+0000","last_peered":"2024-08-17T17:39:58.823186+0000","last_clean":"2024-08-17T17:39:58.823186+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:39:58.823186+0000","last_undegraded":"2024-08-17T17:39:58.823186+0000","last_fullsized":"2024-08-17T17:39:58.823186+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1419,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.802100+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:39:45.802100+0000","last_peered":"2024-08-17T17:39:45.802100+0000","last_clean":"2024-08-17T17:39:45.802100+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:39:45.802100+0000","last_undegraded":"2024-08-17T17:39:45.802100+0000","last_fullsized":"2024-08-17T17:39:45.802100+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1712,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.800725+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:39:45.800725+0000","last_peered":"2024-08-17T17:39:45.800725+0000","last_clean":"2024-08-17T17:39:45.800725+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:39:45.800725+0000","last_undegraded":"2024-08-17T17:39:45.800725+0000","last_fullsized":"2024-08-17T17:39:45.800725+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1360,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.820102+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:39:45.820102+0000","last_peered":"2024-08-17T17:39:45.820102+0000","last_clean":"2024-08-17T17:39:45.820102+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:39:45.820102+0000","last_undegraded":"2024-08-17T17:39:45.820102+0000","last_fullsized":"2024-08-17T17:39:45.820102+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1907,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.842727+0000","last_change":"2024-08-17T17:35:33.137411+0000","last_active":"2024-08-17T17:39:58.842727+0000","last_peered":"2024-08-17T17:39:58.842727+0000","last_clean":"2024-08-17T17:39:58.842727+0000","last_became_active":"2024-08-17T17:35:33.137073+0000","last_became_peered":"2024-08-17T17:35:33.137073+0000","last_unstale":"2024-08-17T17:39:58.842727+0000","last_undegraded":"2024-08-17T17:39:58.842727+0000","last_fullsized":"2024-08-17T17:39:58.842727+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:35.924539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1806,"reported_epoch":1427,"state":"active+clean","last_fresh":"2024-08-17T17:39:45.800966+0000","last_change":"2024-08-17T17:36:15.787445+0000","last_active":"2024-08-17T17:39:45.800966+0000","last_peered":"2024-08-17T17:39:45.800966+0000","last_clean":"2024-08-17T17:39:45.800966+0000","last_became_active":"2024-08-17T17:36:15.787099+0000","last_became_peered":"2024-08-17T17:36:15.787099+0000","last_unstale":"2024-08-17T17:39:45.800966+0000","last_undegraded":"2024-08-17T17:39:45.800966+0000","last_fullsized":"2024-08-17T17:39:45.800966+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:47.448168+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1554907160,"num_objects":23727,"num_object_clones":0,"num_object_copies":71181,"num_objects_missing_on_primary":846,"num_objects_missing":846,"num_objects_degraded":9436,"num_objects_misplaced":1691,"num_objects_unfound":0,"num_objects_dirty":23727,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":105155,"num_write_kb":4124161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33417,"num_bytes_recovered":2037383168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6043938816,"data_stored":6043926600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98855,"ondisk_log_size":98855,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635092,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":27,"num_read_kb":61,"num_write":15,"num_write_kb":183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1314910,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162871,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":991464,"kb_used_data":826848,"kb_used_omap":54,"kb_used_meta":164553,"kb_avail":92753688,"statfs":{"total":95995035648,"available":94979776512,"internally_reserved":0,"allocated":846692352,"data_stored":843994402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":168502293},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999345,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":955880,"kb_used_data":757664,"kb_used_omap":47,"kb_used_meta":198160,"kb_avail":92789272,"statfs":{"total":95995035648,"available":95016214528,"internally_reserved":0,"allocated":775847936,"data_stored":773150963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":202916500},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,0,2,7,0,2],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627142,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1301580,"kb_used_data":238724,"kb_used_omap":42,"kb_used_meta":1062805,"kb_avail":92443572,"statfs":{"total":95995035648,"available":94662217728,"internally_reserved":0,"allocated":244453376,"data_stored":241778684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1088312380},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785619,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2068624,"kb_used_data":1256136,"kb_used_omap":48,"kb_used_meta":812431,"kb_avail":91676528,"statfs":{"total":95995035648,"available":93876764672,"internally_reserved":0,"allocated":1286283264,"data_stored":1283605447,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":831930030},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725136,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1267928,"kb_used_data":384528,"kb_used_omap":1,"kb_used_meta":883390,"kb_avail":92477224,"statfs":{"total":95995035648,"available":94696677376,"internally_reserved":0,"allocated":393756672,"data_stored":391070805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":904591814},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986773,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1637604,"kb_used_data":754204,"kb_used_omap":1,"kb_used_meta":883390,"kb_avail":92107548,"statfs":{"total":95995035648,"available":94318129152,"internally_reserved":0,"allocated":772304896,"data_stored":769610452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":904591814},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706380,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2025944,"kb_used_data":1095696,"kb_used_omap":35,"kb_used_meta":930204,"kb_avail":91719208,"statfs":{"total":95995035648,"available":93920468992,"internally_reserved":0,"allocated":1121992704,"data_stored":1119306818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":952528951},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832153,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1711572,"kb_used_data":668492,"kb_used_omap":65,"kb_used_meta":1043006,"kb_avail":92033580,"statfs":{"total":95995035648,"available":94242385920,"internally_reserved":0,"allocated":684535808,"data_stored":681854480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1068038331},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":676069376,"data_stored":676069376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1276182528,"data_stored":1276182528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1107300352,"data_stored":1107296280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":234553344,"data_stored":234553344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":759369728,"data_stored":759365656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":384827392,"data_stored":384827392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":767365120,"data_stored":767361048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":838270976,"data_stored":838270976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:40:07.671 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:07.674 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:07.697 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:07.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:07.799 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:07.822 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:07.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:07.924 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:07.953 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:07.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:07.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:07 smithi063 ceph-mon[114756]: pgmap v2274: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 334 op/s; 9204/69228 objects degraded (13.295%); 1691/69228 objects misplaced (2.443%); 5.4 MiB/s, 85 objects/s recovering 2024-08-17T17:40:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:07 smithi063 ceph-mon[109843]: pgmap v2274: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 334 op/s; 9204/69228 objects degraded (13.295%); 1691/69228 objects misplaced (2.443%); 5.4 MiB/s, 85 objects/s recovering 2024-08-17T17:40:08.054 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:08.081 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:08.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:07 smithi162 ceph-mon[104668]: pgmap v2274: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 334 op/s; 9204/69228 objects degraded (13.295%); 1691/69228 objects misplaced (2.443%); 5.4 MiB/s, 85 objects/s recovering 2024-08-17T17:40:08.183 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:08.210 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:08.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:08.312 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:08.343 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:08.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:08.445 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:08.476 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:08.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:08.577 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:08.604 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:08.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:08.706 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:08.730 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:08.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:08.832 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:08.863 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:08.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:08.946 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:08 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 9204/69228 objects degraded (13.295%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:08.947 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:08 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 9204/69228 objects degraded (13.295%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:08.965 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:08.993 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:08.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:08.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:09.095 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:09.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:08 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 9204/69228 objects degraded (13.295%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:09.119 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:09.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:09.220 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:09.249 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:09.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:09.351 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:09.379 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:09.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:09.481 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:09.505 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:09.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:09.606 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:09.638 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:09.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:09.722 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:40:09.722 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:40:09.722 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:40:09.722 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:40:09.722 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:40:09.740 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:09.765 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:09.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:09.858 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:40:09.867 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:09.895 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:09.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:09.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:09.997 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:10.019 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:10.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:09 smithi162 ceph-mon[104668]: pgmap v2275: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 8774/66324 objects degraded (13.229%); 1444/66324 objects misplaced (2.177%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:40:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:09 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/922948288' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:10.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:09 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:10.121 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:10.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.151 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:10.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:10.252 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:10.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:09 smithi063 ceph-mon[109843]: pgmap v2275: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 8774/66324 objects degraded (13.229%); 1444/66324 objects misplaced (2.177%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:40:10.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:09 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/922948288' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:10.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:09 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:10.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:09 smithi063 ceph-mon[114756]: pgmap v2275: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 8774/66324 objects degraded (13.229%); 1444/66324 objects misplaced (2.177%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:40:10.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:09 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/922948288' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:10.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:09 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:40:10.280 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:10.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:10.381 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:10.406 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:10.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:10.508 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:10.531 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:10.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:10.633 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:10.658 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:10.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:10.666 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:40:10.761 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:10.786 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:10.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:10.888 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:10.915 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:10.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:10.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:10.917 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-08-17T17:40:10.981 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:10 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:10.982 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:10 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:40:10.982 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:10 smithi063 ceph-mon[109843]: osdmap e1429: 8 total, 8 up, 8 in 2024-08-17T17:40:10.982 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:10 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:10.982 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:10 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:40:10.982 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:10 smithi063 ceph-mon[114756]: osdmap e1429: 8 total, 8 up, 8 in 2024-08-17T17:40:11.017 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:11.041 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:11.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:11.143 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:11.172 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:11.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.173 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:11.274 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:11.303 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:11.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:11.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:10 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:11.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:10 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:40:11.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:10 smithi162 ceph-mon[104668]: osdmap e1429: 8 total, 8 up, 8 in 2024-08-17T17:40:11.404 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:11.428 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:11.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:11.530 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:11.558 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:11.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:11.660 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:11.694 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:11.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:11.796 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:11.822 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:11.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:11.924 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:11.947 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:11.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:11.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:11 smithi063 ceph-mon[114756]: pgmap v2277: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 8774/66000 objects degraded (13.294%); 1444/66000 objects misplaced (2.188%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:40:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:11 smithi063 ceph-mon[109843]: pgmap v2277: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 8774/66000 objects degraded (13.294%); 1444/66000 objects misplaced (2.188%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:40:12.049 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:12.078 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:12.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:12.124 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:12.180 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:12.208 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:12.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:12.310 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:12.337 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:12.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:12.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:11 smithi162 ceph-mon[104668]: pgmap v2277: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 341 op/s; 8774/66000 objects degraded (13.294%); 1444/66000 objects misplaced (2.188%); 5.5 MiB/s, 88 objects/s recovering 2024-08-17T17:40:12.439 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:12.472 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:12.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:12.574 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:12.602 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:12.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:12.704 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:12.729 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:12.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:12.831 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:12.857 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:12.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:12.959 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:12.983 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:12.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:12.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:13.085 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:13.111 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:13.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:13.213 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:13.236 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:13.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:13.339 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:13.372 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:13.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.373 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:13.423 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:13.437 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:40:13.437 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:40:13.471 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:40:13.472 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:40:13.474 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:13.496 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:40:13.497 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:40:13.505 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:13.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:13.529 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:40:13.529 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:40:13.553 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:40:13.553 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:40:13.554 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:40:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:40:13] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.51.0" 2024-08-17T17:40:13.573 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:40:13.573 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:40:13.592 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:40:13.592 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:40:13.606 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:13.615 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:40:13.615 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:40:13.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.633 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:13.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:13.663 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set nodeep-scrub 2024-08-17T17:40:13.734 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:13.766 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:13.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:13.867 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:13.893 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:13.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:13.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:13.995 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:14.022 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:14.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:14.114 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:13 smithi063 ceph-mon[114756]: pgmap v2278: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 299 op/s; 8596/62586 objects degraded (13.735%); 1444/62586 objects misplaced (2.307%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:40:14.114 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:13 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 8596/62586 objects degraded (13.735%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:14.114 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:13 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3536286368' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:14.114 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:13 smithi063 ceph-mon[109843]: pgmap v2278: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 299 op/s; 8596/62586 objects degraded (13.735%); 1444/62586 objects misplaced (2.307%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:40:14.114 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:13 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 8596/62586 objects degraded (13.735%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:14.114 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:13 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3536286368' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:14.124 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:14.170 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:14.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:14.272 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:14.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:14.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:14.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:13 smithi162 ceph-mon[104668]: pgmap v2278: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 299 op/s; 8596/62586 objects degraded (13.735%); 1444/62586 objects misplaced (2.307%); 3.4 MiB/s, 54 objects/s recovering 2024-08-17T17:40:14.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:13 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 8596/62586 objects degraded (13.735%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:14.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:13 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3536286368' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:14.401 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:14.427 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:14.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:14.529 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:14.553 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:14.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:14.655 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:14.694 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:14.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:14.795 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:14.839 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:14.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:14.941 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:14.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:14.994 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:14.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:15.095 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:15.121 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:15.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:15.223 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:15.252 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:40:15.253 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:40:15.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.256 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:15.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:15.357 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:15.402 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:15.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:15.504 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:15.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.529 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:15.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:15.630 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:15.658 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:15.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:15.760 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:15.787 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:15.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:15.889 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:15.928 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:15.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:15.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:16.030 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:16.057 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:16.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:16.159 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:16.200 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:16.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:16.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:15 smithi063 ceph-mon[114756]: pgmap v2279: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 364 op/s; 8118/60336 objects degraded (13.455%); 1185/60336 objects misplaced (1.964%); 5.7 MiB/s, 91 objects/s recovering 2024-08-17T17:40:16.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:15 smithi063 ceph-mon[109843]: pgmap v2279: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 364 op/s; 8118/60336 objects degraded (13.455%); 1185/60336 objects misplaced (1.964%); 5.7 MiB/s, 91 objects/s recovering 2024-08-17T17:40:16.302 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:16.330 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:16.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:16.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:15 smithi162 ceph-mon[104668]: pgmap v2279: 119 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 364 op/s; 8118/60336 objects degraded (13.455%); 1185/60336 objects misplaced (1.964%); 5.7 MiB/s, 91 objects/s recovering 2024-08-17T17:40:16.432 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:16.475 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:16.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:16.577 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:16.618 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:16.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:16.720 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:16.766 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:16.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:16.868 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:16.891 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:16.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:16.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:16.993 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:17.022 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:17.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:17.124 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:17.158 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:17.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:17.260 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:17.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:16 smithi063 ceph-mon[114756]: from='client.46511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:17.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:16 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:40:17.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:17.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:16 smithi063 ceph-mon[109843]: from='client.46511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:17.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:16 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:40:17.281 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:17.303 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:17.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:17.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:16 smithi162 ceph-mon[104668]: from='client.46511 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:17.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:16 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:40:17.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:17.405 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:17.440 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:17.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:17.542 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:17.601 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:17.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:17.703 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:17.737 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:17.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:17.839 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:17.876 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:17.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:17.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:17.978 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:18.014 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:18.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.015 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:18.116 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:18.152 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:18.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:18.254 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:18.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:17 smithi063 ceph-mon[114756]: pgmap v2280: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 7890/57390 objects degraded (13.748%); 1185/57390 objects misplaced (2.065%); 5.7 MiB/s, 90 objects/s recovering 2024-08-17T17:40:18.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:17 smithi063 ceph-mon[109843]: pgmap v2280: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 7890/57390 objects degraded (13.748%); 1185/57390 objects misplaced (2.065%); 5.7 MiB/s, 90 objects/s recovering 2024-08-17T17:40:18.292 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:18.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:18.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:17 smithi162 ceph-mon[104668]: pgmap v2280: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 394 op/s; 7890/57390 objects degraded (13.748%); 1185/57390 objects misplaced (2.065%); 5.7 MiB/s, 90 objects/s recovering 2024-08-17T17:40:18.395 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:18.412 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2279,"stamp":"2024-08-17T17:40:14.674693+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1316931249,"num_objects":20112,"num_object_clones":0,"num_object_copies":60336,"num_objects_missing_on_primary":138,"num_objects_missing":138,"num_objects_degraded":8118,"num_objects_misplaced":1185,"num_objects_unfound":0,"num_objects_dirty":20112,"num_whiteouts":0,"num_read":28,"num_read_kb":62,"num_write":108875,"num_write_kb":4124377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34352,"num_bytes_recovered":2098402013,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105274,"ondisk_log_size":105274,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":375,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11439356,"kb_used_data":5419444,"kb_used_omap":298,"kb_used_meta":6019477,"kb_avail":738521860,"statfs":{"total":767960285184,"available":756246384640,"internally_reserved":0,"allocated":5549510656,"data_stored":5527983939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":6163944977},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,3,2,5,1,3,5,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-238616576,"num_objects":-3641,"num_object_clones":0,"num_object_copies":-10923,"num_objects_missing_on_primary":-708,"num_objects_missing":-708,"num_objects_degraded":-1318,"num_objects_misplaced":-506,"num_objects_unfound":0,"num_objects_dirty":-3641,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3641,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":911,"num_bytes_recovered":59703296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002541"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":65,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.853814+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:40:09.853814+0000","last_peered":"2024-08-17T17:40:09.853814+0000","last_clean":"2024-08-17T17:40:09.853814+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:40:09.853814+0000","last_undegraded":"2024-08-17T17:40:09.853814+0000","last_fullsized":"2024-08-17T17:40:09.853814+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1444,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887412+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:39:58.887412+0000","last_peered":"2024-08-17T17:39:58.887412+0000","last_clean":"2024-08-17T17:39:58.887412+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:39:58.887412+0000","last_undegraded":"2024-08-17T17:39:58.887412+0000","last_fullsized":"2024-08-17T17:39:58.887412+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1758,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887791+0000","last_change":"2024-08-17T17:36:15.771150+0000","last_active":"2024-08-17T17:39:58.887791+0000","last_peered":"2024-08-17T17:39:58.887791+0000","last_clean":"2024-08-17T17:39:58.887791+0000","last_became_active":"2024-08-17T17:36:15.770520+0000","last_became_peered":"2024-08-17T17:36:15.770520+0000","last_unstale":"2024-08-17T17:39:58.887791+0000","last_undegraded":"2024-08-17T17:39:58.887791+0000","last_fullsized":"2024-08-17T17:39:58.887791+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:17.552796+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":94,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.646607+0000","last_change":"2024-08-17T17:36:12.756709+0000","last_active":"2024-08-17T17:40:02.646607+0000","last_peered":"2024-08-17T17:40:02.646607+0000","last_clean":"2024-08-17T17:40:02.646607+0000","last_became_active":"2024-08-17T17:36:12.756385+0000","last_became_peered":"2024-08-17T17:36:12.756385+0000","last_unstale":"2024-08-17T17:40:02.646607+0000","last_undegraded":"2024-08-17T17:40:02.646607+0000","last_fullsized":"2024-08-17T17:40:02.646607+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:21.420282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":150,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.856757+0000","last_change":"2024-08-17T17:36:12.762800+0000","last_active":"2024-08-17T17:40:09.856757+0000","last_peered":"2024-08-17T17:40:09.856757+0000","last_clean":"2024-08-17T17:40:09.856757+0000","last_became_active":"2024-08-17T17:36:12.759977+0000","last_became_peered":"2024-08-17T17:36:12.759977+0000","last_unstale":"2024-08-17T17:40:09.856757+0000","last_undegraded":"2024-08-17T17:40:09.856757+0000","last_fullsized":"2024-08-17T17:40:09.856757+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:02.162681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1638,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.826571+0000","last_change":"2024-08-17T17:35:56.281561+0000","last_active":"2024-08-17T17:39:58.826571+0000","last_peered":"2024-08-17T17:39:58.826571+0000","last_clean":"2024-08-17T17:39:58.826571+0000","last_became_active":"2024-08-17T17:35:56.278638+0000","last_became_peered":"2024-08-17T17:35:56.278638+0000","last_unstale":"2024-08-17T17:39:58.826571+0000","last_undegraded":"2024-08-17T17:39:58.826571+0000","last_fullsized":"2024-08-17T17:39:58.826571+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1377,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.824052+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:39:58.824052+0000","last_peered":"2024-08-17T17:39:58.824052+0000","last_clean":"2024-08-17T17:39:58.824052+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:39:58.824052+0000","last_undegraded":"2024-08-17T17:39:58.824052+0000","last_fullsized":"2024-08-17T17:39:58.824052+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":53,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887495+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:39:58.887495+0000","last_peered":"2024-08-17T17:39:58.887495+0000","last_clean":"2024-08-17T17:39:58.887495+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:39:58.887495+0000","last_undegraded":"2024-08-17T17:39:58.887495+0000","last_fullsized":"2024-08-17T17:39:58.887495+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1422,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823389+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:39:58.823389+0000","last_peered":"2024-08-17T17:39:58.823389+0000","last_clean":"2024-08-17T17:39:58.823389+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:39:58.823389+0000","last_undegraded":"2024-08-17T17:39:58.823389+0000","last_fullsized":"2024-08-17T17:39:58.823389+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1430,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.888150+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:39:58.888150+0000","last_peered":"2024-08-17T17:39:58.888150+0000","last_clean":"2024-08-17T17:39:58.888150+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:39:58.888150+0000","last_undegraded":"2024-08-17T17:39:58.888150+0000","last_fullsized":"2024-08-17T17:39:58.888150+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":721,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.646586+0000","last_change":"2024-08-17T17:36:15.773315+0000","last_active":"2024-08-17T17:40:02.646586+0000","last_peered":"2024-08-17T17:40:02.646586+0000","last_clean":"2024-08-17T17:40:02.646586+0000","last_became_active":"2024-08-17T17:36:15.772961+0000","last_became_peered":"2024-08-17T17:36:15.772961+0000","last_unstale":"2024-08-17T17:40:02.646586+0000","last_undegraded":"2024-08-17T17:40:02.646586+0000","last_fullsized":"2024-08-17T17:40:02.646586+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:10.190221+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":59,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.853956+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:40:09.853956+0000","last_peered":"2024-08-17T17:40:09.853956+0000","last_clean":"2024-08-17T17:40:09.853956+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:40:09.853956+0000","last_undegraded":"2024-08-17T17:40:09.853956+0000","last_fullsized":"2024-08-17T17:40:09.853956+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":149,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.856948+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:40:09.856948+0000","last_peered":"2024-08-17T17:40:09.856948+0000","last_clean":"2024-08-17T17:40:09.856948+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:40:09.856948+0000","last_undegraded":"2024-08-17T17:40:09.856948+0000","last_fullsized":"2024-08-17T17:40:09.856948+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1511,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.874120+0000","last_change":"2024-08-17T17:36:24.225739+0000","last_active":"2024-08-17T17:39:58.874120+0000","last_peered":"2024-08-17T17:39:58.874120+0000","last_clean":"2024-08-17T17:39:58.874120+0000","last_became_active":"2024-08-17T17:36:24.225250+0000","last_became_peered":"2024-08-17T17:36:24.225250+0000","last_unstale":"2024-08-17T17:39:58.874120+0000","last_undegraded":"2024-08-17T17:39:58.874120+0000","last_fullsized":"2024-08-17T17:39:58.874120+0000","mapping_epoch":1393,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1394,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.290656+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1450,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887707+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:39:58.887707+0000","last_peered":"2024-08-17T17:39:58.887707+0000","last_clean":"2024-08-17T17:39:58.887707+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:39:58.887707+0000","last_undegraded":"2024-08-17T17:39:58.887707+0000","last_fullsized":"2024-08-17T17:39:58.887707+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1857,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887957+0000","last_change":"2024-08-17T17:36:12.752970+0000","last_active":"2024-08-17T17:39:58.887957+0000","last_peered":"2024-08-17T17:39:58.887957+0000","last_clean":"2024-08-17T17:39:58.887957+0000","last_became_active":"2024-08-17T17:36:12.752311+0000","last_became_peered":"2024-08-17T17:36:12.752311+0000","last_unstale":"2024-08-17T17:39:58.887957+0000","last_undegraded":"2024-08-17T17:39:58.887957+0000","last_fullsized":"2024-08-17T17:39:58.887957+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:38.131419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1805,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823043+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:39:58.823043+0000","last_peered":"2024-08-17T17:39:58.823043+0000","last_clean":"2024-08-17T17:39:58.823043+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:39:58.823043+0000","last_undegraded":"2024-08-17T17:39:58.823043+0000","last_fullsized":"2024-08-17T17:39:58.823043+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1522,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.862000+0000","last_change":"2024-08-17T17:36:18.750494+0000","last_active":"2024-08-17T17:40:09.862000+0000","last_peered":"2024-08-17T17:40:09.862000+0000","last_clean":"2024-08-17T17:40:09.862000+0000","last_became_active":"2024-08-17T17:36:18.750092+0000","last_became_peered":"2024-08-17T17:36:18.750092+0000","last_unstale":"2024-08-17T17:40:09.862000+0000","last_undegraded":"2024-08-17T17:40:09.862000+0000","last_fullsized":"2024-08-17T17:40:09.862000+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:36.599727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1151,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.854712+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:40:09.854712+0000","last_peered":"2024-08-17T17:40:09.854712+0000","last_clean":"2024-08-17T17:40:09.854712+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:40:09.854712+0000","last_undegraded":"2024-08-17T17:40:09.854712+0000","last_fullsized":"2024-08-17T17:40:09.854712+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":972,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823267+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:39:58.823267+0000","last_peered":"2024-08-17T17:39:58.823267+0000","last_clean":"2024-08-17T17:39:58.823267+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:39:58.823267+0000","last_undegraded":"2024-08-17T17:39:58.823267+0000","last_fullsized":"2024-08-17T17:39:58.823267+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":59,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.853291+0000","last_change":"2024-08-17T17:37:11.075505+0000","last_active":"2024-08-17T17:40:09.853291+0000","last_peered":"2024-08-17T17:40:09.853291+0000","last_clean":"2024-08-17T17:40:09.853291+0000","last_became_active":"2024-08-17T17:37:11.075131+0000","last_became_peered":"2024-08-17T17:37:11.075131+0000","last_unstale":"2024-08-17T17:40:09.853291+0000","last_undegraded":"2024-08-17T17:40:09.853291+0000","last_fullsized":"2024-08-17T17:40:09.853291+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:39.672054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1285,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.827015+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:39:58.827015+0000","last_peered":"2024-08-17T17:39:58.827015+0000","last_clean":"2024-08-17T17:39:58.827015+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:39:58.827015+0000","last_undegraded":"2024-08-17T17:39:58.827015+0000","last_fullsized":"2024-08-17T17:39:58.827015+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1662,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887553+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:39:58.887553+0000","last_peered":"2024-08-17T17:39:58.887553+0000","last_clean":"2024-08-17T17:39:58.887553+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:39:58.887553+0000","last_undegraded":"2024-08-17T17:39:58.887553+0000","last_fullsized":"2024-08-17T17:39:58.887553+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":105,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.864511+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:40:09.864511+0000","last_peered":"2024-08-17T17:40:09.864511+0000","last_clean":"2024-08-17T17:40:09.864511+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:40:09.864511+0000","last_undegraded":"2024-08-17T17:40:09.864511+0000","last_fullsized":"2024-08-17T17:40:09.864511+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1259,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.824247+0000","last_change":"2024-08-17T17:36:12.755873+0000","last_active":"2024-08-17T17:39:58.824247+0000","last_peered":"2024-08-17T17:39:58.824247+0000","last_clean":"2024-08-17T17:39:58.824247+0000","last_became_active":"2024-08-17T17:36:12.755366+0000","last_became_peered":"2024-08-17T17:36:12.755366+0000","last_unstale":"2024-08-17T17:39:58.824247+0000","last_undegraded":"2024-08-17T17:39:58.824247+0000","last_fullsized":"2024-08-17T17:39:58.824247+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:36.523214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1612,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.880295+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:39:58.880295+0000","last_peered":"2024-08-17T17:39:58.880295+0000","last_clean":"2024-08-17T17:39:58.880295+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:39:58.880295+0000","last_undegraded":"2024-08-17T17:39:58.880295+0000","last_fullsized":"2024-08-17T17:39:58.880295+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":796,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.874198+0000","last_change":"2024-08-17T17:35:55.200519+0000","last_active":"2024-08-17T17:39:58.874198+0000","last_peered":"2024-08-17T17:39:58.874198+0000","last_clean":"2024-08-17T17:39:58.874198+0000","last_became_active":"2024-08-17T17:35:55.200230+0000","last_became_peered":"2024-08-17T17:35:55.200230+0000","last_unstale":"2024-08-17T17:39:58.874198+0000","last_undegraded":"2024-08-17T17:39:58.874198+0000","last_fullsized":"2024-08-17T17:39:58.874198+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:33.454851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1436,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823930+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:39:58.823930+0000","last_peered":"2024-08-17T17:39:58.823930+0000","last_clean":"2024-08-17T17:39:58.823930+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:39:58.823930+0000","last_undegraded":"2024-08-17T17:39:58.823930+0000","last_fullsized":"2024-08-17T17:39:58.823930+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1723,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.856630+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:40:09.856630+0000","last_peered":"2024-08-17T17:40:09.856630+0000","last_clean":"2024-08-17T17:40:09.856630+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:40:09.856630+0000","last_undegraded":"2024-08-17T17:40:09.856630+0000","last_fullsized":"2024-08-17T17:40:09.856630+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:22.077229+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":270,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.862069+0000","last_change":"2024-08-17T17:35:33.145476+0000","last_active":"2024-08-17T17:40:09.862069+0000","last_peered":"2024-08-17T17:40:09.862069+0000","last_clean":"2024-08-17T17:40:09.862069+0000","last_became_active":"2024-08-17T17:35:33.145125+0000","last_became_peered":"2024-08-17T17:35:33.145125+0000","last_unstale":"2024-08-17T17:40:09.862069+0000","last_undegraded":"2024-08-17T17:40:09.862069+0000","last_fullsized":"2024-08-17T17:40:09.862069+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:45.298064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":940,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.855038+0000","last_change":"2024-08-17T17:35:56.248612+0000","last_active":"2024-08-17T17:39:58.855038+0000","last_peered":"2024-08-17T17:39:58.855038+0000","last_clean":"2024-08-17T17:39:58.855038+0000","last_became_active":"2024-08-17T17:35:56.244364+0000","last_became_peered":"2024-08-17T17:35:56.244364+0000","last_unstale":"2024-08-17T17:39:58.855038+0000","last_undegraded":"2024-08-17T17:39:58.855038+0000","last_fullsized":"2024-08-17T17:39:58.855038+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:41.064165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":880,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.824071+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:39:58.824071+0000","last_peered":"2024-08-17T17:39:58.824071+0000","last_clean":"2024-08-17T17:39:58.824071+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:39:58.824071+0000","last_undegraded":"2024-08-17T17:39:58.824071+0000","last_fullsized":"2024-08-17T17:39:58.824071+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1415,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887993+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:39:58.887993+0000","last_peered":"2024-08-17T17:39:58.887993+0000","last_clean":"2024-08-17T17:39:58.887993+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:39:58.887993+0000","last_undegraded":"2024-08-17T17:39:58.887993+0000","last_fullsized":"2024-08-17T17:39:58.887993+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1434,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.888136+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:39:58.888136+0000","last_peered":"2024-08-17T17:39:58.888136+0000","last_clean":"2024-08-17T17:39:58.888136+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:39:58.888136+0000","last_undegraded":"2024-08-17T17:39:58.888136+0000","last_fullsized":"2024-08-17T17:39:58.888136+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":944,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.869098+0000","last_change":"2024-08-17T17:36:15.774881+0000","last_active":"2024-08-17T17:40:09.869098+0000","last_peered":"2024-08-17T17:40:09.869098+0000","last_clean":"2024-08-17T17:40:09.869098+0000","last_became_active":"2024-08-17T17:36:15.774435+0000","last_became_peered":"2024-08-17T17:36:15.774435+0000","last_unstale":"2024-08-17T17:40:09.869098+0000","last_undegraded":"2024-08-17T17:40:09.869098+0000","last_fullsized":"2024-08-17T17:40:09.869098+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:52.015854+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"1429'7009","reported_seq":15194,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:12.371864+0000","last_change":"2024-08-17T17:38:37.110784+0000","last_active":"2024-08-17T17:40:12.371864+0000","last_peered":"2024-08-17T17:40:12.371864+0000","last_clean":"2024-08-17T17:40:12.371864+0000","last_became_active":"2024-08-17T17:38:37.088380+0000","last_became_peered":"2024-08-17T17:38:37.088380+0000","last_unstale":"2024-08-17T17:40:12.371864+0000","last_undegraded":"2024-08-17T17:40:12.371864+0000","last_fullsized":"2024-08-17T17:40:12.371864+0000","mapping_epoch":1419,"log_start":"1373'1000","ondisk_log_start":"1373'1000","created":1357,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6009,"log_dups_size":0,"ondisk_log_size":6009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:08.647903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83689472,"num_objects":1277,"num_object_clones":0,"num_object_copies":3831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7009,"num_write_kb":265152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4009,"num_bytes_recovered":199753728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1383,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887400+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:39:58.887400+0000","last_peered":"2024-08-17T17:39:58.887400+0000","last_clean":"2024-08-17T17:39:58.887400+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:39:58.887400+0000","last_undegraded":"2024-08-17T17:39:58.887400+0000","last_fullsized":"2024-08-17T17:39:58.887400+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1279,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:07.716115+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:40:07.716115+0000","last_peered":"2024-08-17T17:40:07.716115+0000","last_clean":"2024-08-17T17:40:07.716115+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:40:07.716115+0000","last_undegraded":"2024-08-17T17:40:07.716115+0000","last_fullsized":"2024-08-17T17:40:07.716115+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":337,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.854876+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:40:09.854876+0000","last_peered":"2024-08-17T17:40:09.854876+0000","last_clean":"2024-08-17T17:40:09.854876+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:40:09.854876+0000","last_undegraded":"2024-08-17T17:40:09.854876+0000","last_fullsized":"2024-08-17T17:40:09.854876+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1605,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.862009+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:40:09.862009+0000","last_peered":"2024-08-17T17:40:09.862009+0000","last_clean":"2024-08-17T17:40:09.862009+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:40:09.862009+0000","last_undegraded":"2024-08-17T17:40:09.862009+0000","last_fullsized":"2024-08-17T17:40:09.862009+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:25.751871+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.f","version":"1429'6902","reported_seq":7065,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:10.645007+0000","last_change":"2024-08-17T17:35:34.179168+0000","last_active":"2024-08-17T17:40:10.645007+0000","last_peered":"2024-08-17T17:40:10.645007+0000","last_clean":"2024-08-17T17:40:10.645007+0000","last_became_active":"2024-08-17T17:35:34.133493+0000","last_became_peered":"2024-08-17T17:35:34.133493+0000","last_unstale":"2024-08-17T17:40:10.645007+0000","last_undegraded":"2024-08-17T17:40:10.645007+0000","last_fullsized":"2024-08-17T17:40:10.645007+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6902,"log_dups_size":0,"ondisk_log_size":6902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:24:11.188566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85327872,"num_objects":1302,"num_object_clones":0,"num_object_copies":3906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6902,"num_write_kb":262528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":112,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.864605+0000","last_change":"2024-08-17T17:37:11.079712+0000","last_active":"2024-08-17T17:40:09.864605+0000","last_peered":"2024-08-17T17:40:09.864605+0000","last_clean":"2024-08-17T17:40:09.864605+0000","last_became_active":"2024-08-17T17:37:11.079253+0000","last_became_peered":"2024-08-17T17:37:11.079253+0000","last_unstale":"2024-08-17T17:40:09.864605+0000","last_undegraded":"2024-08-17T17:40:09.864605+0000","last_fullsized":"2024-08-17T17:40:09.864605+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:53.405657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":924,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:07.716027+0000","last_change":"2024-08-17T17:35:56.223396+0000","last_active":"2024-08-17T17:40:07.716027+0000","last_peered":"2024-08-17T17:40:07.716027+0000","last_clean":"2024-08-17T17:40:07.716027+0000","last_became_active":"2024-08-17T17:35:56.212413+0000","last_became_peered":"2024-08-17T17:35:56.212413+0000","last_unstale":"2024-08-17T17:40:07.716027+0000","last_undegraded":"2024-08-17T17:40:07.716027+0000","last_fullsized":"2024-08-17T17:40:07.716027+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:44.185174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1647,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.854114+0000","last_change":"2024-08-17T17:35:33.136068+0000","last_active":"2024-08-17T17:40:09.854114+0000","last_peered":"2024-08-17T17:40:09.854114+0000","last_clean":"2024-08-17T17:40:09.854114+0000","last_became_active":"2024-08-17T17:35:33.135591+0000","last_became_peered":"2024-08-17T17:35:33.135591+0000","last_unstale":"2024-08-17T17:40:09.854114+0000","last_undegraded":"2024-08-17T17:40:09.854114+0000","last_fullsized":"2024-08-17T17:40:09.854114+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:52.252561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1789,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.888211+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:39:58.888211+0000","last_peered":"2024-08-17T17:39:58.888211+0000","last_clean":"2024-08-17T17:39:58.888211+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:39:58.888211+0000","last_undegraded":"2024-08-17T17:39:58.888211+0000","last_fullsized":"2024-08-17T17:39:58.888211+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"1429'6779","reported_seq":6927,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:10.347713+0000","last_change":"2024-08-17T17:35:14.861823+0000","last_active":"2024-08-17T17:40:10.347713+0000","last_peered":"2024-08-17T17:40:10.347713+0000","last_clean":"2024-08-17T17:40:10.347713+0000","last_became_active":"2024-08-17T17:35:14.854109+0000","last_became_peered":"2024-08-17T17:35:14.854109+0000","last_unstale":"2024-08-17T17:40:10.347713+0000","last_undegraded":"2024-08-17T17:40:10.347713+0000","last_fullsized":"2024-08-17T17:40:10.347713+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6779,"log_dups_size":0,"ondisk_log_size":6779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:46:40.778197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83034112,"num_objects":1267,"num_object_clones":0,"num_object_copies":3801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6779,"num_write_kb":257472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1002,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:07.461058+0000","last_change":"2024-08-17T17:35:33.138467+0000","last_active":"2024-08-17T17:40:07.461058+0000","last_peered":"2024-08-17T17:40:07.461058+0000","last_clean":"2024-08-17T17:40:07.461058+0000","last_became_active":"2024-08-17T17:35:33.138174+0000","last_became_peered":"2024-08-17T17:35:33.138174+0000","last_unstale":"2024-08-17T17:40:07.461058+0000","last_undegraded":"2024-08-17T17:40:07.461058+0000","last_fullsized":"2024-08-17T17:40:07.461058+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:08.860439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1386,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.880333+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:39:58.880333+0000","last_peered":"2024-08-17T17:39:58.880333+0000","last_clean":"2024-08-17T17:39:58.880333+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:39:58.880333+0000","last_undegraded":"2024-08-17T17:39:58.880333+0000","last_fullsized":"2024-08-17T17:39:58.880333+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":143,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.856840+0000","last_change":"2024-08-17T17:35:14.854881+0000","last_active":"2024-08-17T17:40:09.856840+0000","last_peered":"2024-08-17T17:40:09.856840+0000","last_clean":"2024-08-17T17:40:09.856840+0000","last_became_active":"2024-08-17T17:35:14.854121+0000","last_became_peered":"2024-08-17T17:35:14.854121+0000","last_unstale":"2024-08-17T17:40:09.856840+0000","last_undegraded":"2024-08-17T17:40:09.856840+0000","last_fullsized":"2024-08-17T17:40:09.856840+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.863624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":943,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:07.716074+0000","last_change":"2024-08-17T17:36:15.773050+0000","last_active":"2024-08-17T17:40:07.716074+0000","last_peered":"2024-08-17T17:40:07.716074+0000","last_clean":"2024-08-17T17:40:07.716074+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:40:07.716074+0000","last_undegraded":"2024-08-17T17:40:07.716074+0000","last_fullsized":"2024-08-17T17:40:07.716074+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:39.212195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:39.212195+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"1429'6763","reported_seq":12617,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:10.593800+0000","last_change":"2024-08-17T17:37:21.409979+0000","last_active":"2024-08-17T17:40:10.593800+0000","last_peered":"2024-08-17T17:40:10.593800+0000","last_clean":"2024-08-17T17:40:10.593800+0000","last_became_active":"2024-08-17T17:37:21.398111+0000","last_became_peered":"2024-08-17T17:37:21.398111+0000","last_unstale":"2024-08-17T17:40:10.593800+0000","last_undegraded":"2024-08-17T17:40:10.593800+0000","last_fullsized":"2024-08-17T17:40:10.593800+0000","mapping_epoch":1407,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6763,"log_dups_size":0,"ondisk_log_size":6763,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:19.299395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83099672,"num_objects":1269,"num_object_clones":0,"num_object_copies":3807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1269,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6763,"num_write_kb":256961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2850,"num_bytes_recovered":168820736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1044,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823561+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:39:58.823561+0000","last_peered":"2024-08-17T17:39:58.823561+0000","last_clean":"2024-08-17T17:39:58.823561+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:39:58.823561+0000","last_undegraded":"2024-08-17T17:39:58.823561+0000","last_fullsized":"2024-08-17T17:39:58.823561+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":237,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.869063+0000","last_change":"2024-08-17T17:35:33.145351+0000","last_active":"2024-08-17T17:40:09.869063+0000","last_peered":"2024-08-17T17:40:09.869063+0000","last_clean":"2024-08-17T17:40:09.869063+0000","last_became_active":"2024-08-17T17:35:33.145040+0000","last_became_peered":"2024-08-17T17:35:33.145040+0000","last_unstale":"2024-08-17T17:40:09.869063+0000","last_undegraded":"2024-08-17T17:40:09.869063+0000","last_fullsized":"2024-08-17T17:40:09.869063+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:02.124000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1454,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.824040+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:39:58.824040+0000","last_peered":"2024-08-17T17:39:58.824040+0000","last_clean":"2024-08-17T17:39:58.824040+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:39:58.824040+0000","last_undegraded":"2024-08-17T17:39:58.824040+0000","last_fullsized":"2024-08-17T17:39:58.824040+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":926,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:07.716093+0000","last_change":"2024-08-17T17:36:15.769890+0000","last_active":"2024-08-17T17:40:07.716093+0000","last_peered":"2024-08-17T17:40:07.716093+0000","last_clean":"2024-08-17T17:40:07.716093+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:40:07.716093+0000","last_undegraded":"2024-08-17T17:40:07.716093+0000","last_fullsized":"2024-08-17T17:40:07.716093+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:33.034306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:33.034306+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":4,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.b","version":"1429'6737","reported_seq":14550,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:13.538720+0000","last_change":"2024-08-17T17:38:07.888612+0000","last_active":"2024-08-17T17:40:13.538720+0000","last_peered":"2024-08-17T17:40:13.538720+0000","last_clean":"2024-08-17T17:40:13.538720+0000","last_became_active":"2024-08-17T17:38:07.875196+0000","last_became_peered":"2024-08-17T17:38:07.875196+0000","last_unstale":"2024-08-17T17:40:13.538720+0000","last_undegraded":"2024-08-17T17:40:13.538720+0000","last_fullsized":"2024-08-17T17:40:13.538720+0000","mapping_epoch":1415,"log_start":"1373'800","ondisk_log_start":"1373'800","created":1357,"last_epoch_clean":1416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5937,"log_dups_size":0,"ondisk_log_size":5937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:21.658004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79757312,"num_objects":1217,"num_object_clones":0,"num_object_copies":3651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6737,"num_write_kb":254528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3836,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1498,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.857099+0000","last_change":"2024-08-17T17:36:12.759743+0000","last_active":"2024-08-17T17:40:09.857099+0000","last_peered":"2024-08-17T17:40:09.857099+0000","last_clean":"2024-08-17T17:40:09.857099+0000","last_became_active":"2024-08-17T17:36:12.759334+0000","last_became_peered":"2024-08-17T17:36:12.759334+0000","last_unstale":"2024-08-17T17:40:09.857099+0000","last_undegraded":"2024-08-17T17:40:09.857099+0000","last_fullsized":"2024-08-17T17:40:09.857099+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:04.524384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1417,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.888300+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:39:58.888300+0000","last_peered":"2024-08-17T17:39:58.888300+0000","last_clean":"2024-08-17T17:39:58.888300+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:39:58.888300+0000","last_undegraded":"2024-08-17T17:39:58.888300+0000","last_fullsized":"2024-08-17T17:39:58.888300+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1459,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823792+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:39:58.823792+0000","last_peered":"2024-08-17T17:39:58.823792+0000","last_clean":"2024-08-17T17:39:58.823792+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:39:58.823792+0000","last_undegraded":"2024-08-17T17:39:58.823792+0000","last_fullsized":"2024-08-17T17:39:58.823792+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1775,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823039+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:39:58.823039+0000","last_peered":"2024-08-17T17:39:58.823039+0000","last_clean":"2024-08-17T17:39:58.823039+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:39:58.823039+0000","last_undegraded":"2024-08-17T17:39:58.823039+0000","last_fullsized":"2024-08-17T17:39:58.823039+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"1429'6923","reported_seq":15526,"reported_epoch":1429,"state":"active+recovering+undersized+remapped","last_fresh":"2024-08-17T17:40:12.713507+0000","last_change":"2024-08-17T17:38:05.991715+0000","last_active":"2024-08-17T17:40:12.713507+0000","last_peered":"2024-08-17T17:40:12.713507+0000","last_clean":"2024-08-17T17:35:54.187688+0000","last_became_active":"2024-08-17T17:36:22.904523+0000","last_became_peered":"2024-08-17T17:36:22.904523+0000","last_unstale":"2024-08-17T17:40:12.713507+0000","last_undegraded":"2024-08-17T17:40:12.713507+0000","last_fullsized":"2024-08-17T17:36:13.740491+0000","mapping_epoch":1383,"log_start":"1373'1100","ondisk_log_start":"1373'1100","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5823,"log_dups_size":0,"ondisk_log_size":5823,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:30:57.525855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82903040,"num_objects":1265,"num_object_clones":0,"num_object_copies":3795,"num_objects_missing_on_primary":51,"num_objects_missing":51,"num_objects_degraded":0,"num_objects_misplaced":1185,"num_objects_unfound":0,"num_objects_dirty":1265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6923,"num_write_kb":262016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2834,"num_bytes_recovered":185729024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["2","4","5"],"object_location_counts":[{"shards":"2,4,5","objects":51},{"shards":"2,4,5,6","objects":6},{"shards":"2,5,6","objects":1077},{"shards":"5,6","objects":131}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1093,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.824124+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:39:58.824124+0000","last_peered":"2024-08-17T17:39:58.824124+0000","last_clean":"2024-08-17T17:39:58.824124+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:39:58.824124+0000","last_undegraded":"2024-08-17T17:39:58.824124+0000","last_fullsized":"2024-08-17T17:39:58.824124+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1410,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.856402+0000","last_change":"2024-08-17T17:35:56.229622+0000","last_active":"2024-08-17T17:40:09.856402+0000","last_peered":"2024-08-17T17:40:09.856402+0000","last_clean":"2024-08-17T17:40:09.856402+0000","last_became_active":"2024-08-17T17:35:56.229324+0000","last_became_peered":"2024-08-17T17:35:56.229324+0000","last_unstale":"2024-08-17T17:40:09.856402+0000","last_undegraded":"2024-08-17T17:40:09.856402+0000","last_fullsized":"2024-08-17T17:40:09.856402+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:13.980316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2013,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.856656+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:40:09.856656+0000","last_peered":"2024-08-17T17:40:09.856656+0000","last_clean":"2024-08-17T17:40:09.856656+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:40:09.856656+0000","last_undegraded":"2024-08-17T17:40:09.856656+0000","last_fullsized":"2024-08-17T17:40:09.856656+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1589,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887586+0000","last_change":"2024-08-17T17:35:33.137074+0000","last_active":"2024-08-17T17:39:58.887586+0000","last_peered":"2024-08-17T17:39:58.887586+0000","last_clean":"2024-08-17T17:39:58.887586+0000","last_became_active":"2024-08-17T17:35:33.136759+0000","last_became_peered":"2024-08-17T17:35:33.136759+0000","last_unstale":"2024-08-17T17:39:58.887586+0000","last_undegraded":"2024-08-17T17:39:58.887586+0000","last_fullsized":"2024-08-17T17:39:58.887586+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:37.263639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:37.263639+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:14.599780+0000","scrub_duration":0.0016181800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"1429'6811","reported_seq":14519,"reported_epoch":1429,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:10.624000+0000","last_change":"2024-08-17T17:36:23.019129+0000","last_active":"2024-08-17T17:40:10.624000+0000","last_peered":"2024-08-17T17:40:10.624000+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:23.013527+0000","last_became_peered":"2024-08-17T17:36:23.013527+0000","last_unstale":"2024-08-17T17:40:10.624000+0000","last_undegraded":"2024-08-17T17:36:13.874715+0000","last_fullsized":"2024-08-17T17:36:13.738828+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6811,"log_dups_size":0,"ondisk_log_size":6811,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84475904,"num_objects":1289,"num_object_clones":0,"num_object_copies":3867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2013,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6811,"num_write_kb":259200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2762,"num_bytes_recovered":181010432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[0,6],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":114},{"shards":"0,1,6","objects":1048},{"shards":"0,6","objects":-610}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1202,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.855039+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:40:09.855039+0000","last_peered":"2024-08-17T17:40:09.855039+0000","last_clean":"2024-08-17T17:40:09.855039+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:40:09.855039+0000","last_undegraded":"2024-08-17T17:40:09.855039+0000","last_fullsized":"2024-08-17T17:40:09.855039+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887419+0000","last_change":"2024-08-17T17:36:17.774773+0000","last_active":"2024-08-17T17:39:58.887419+0000","last_peered":"2024-08-17T17:39:58.887419+0000","last_clean":"2024-08-17T17:39:58.887419+0000","last_became_active":"2024-08-17T17:36:17.774468+0000","last_became_peered":"2024-08-17T17:36:17.774468+0000","last_unstale":"2024-08-17T17:39:58.887419+0000","last_undegraded":"2024-08-17T17:39:58.887419+0000","last_fullsized":"2024-08-17T17:39:58.887419+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:54.231747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1540,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.853187+0000","last_change":"2024-08-17T17:35:33.137633+0000","last_active":"2024-08-17T17:40:09.853187+0000","last_peered":"2024-08-17T17:40:09.853187+0000","last_clean":"2024-08-17T17:40:09.853187+0000","last_became_active":"2024-08-17T17:35:33.137323+0000","last_became_peered":"2024-08-17T17:35:33.137323+0000","last_unstale":"2024-08-17T17:40:09.853187+0000","last_undegraded":"2024-08-17T17:40:09.853187+0000","last_fullsized":"2024-08-17T17:40:09.853187+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:03.359155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":905,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:07.715966+0000","last_change":"2024-08-17T17:35:56.264626+0000","last_active":"2024-08-17T17:40:07.715966+0000","last_peered":"2024-08-17T17:40:07.715966+0000","last_clean":"2024-08-17T17:40:07.715966+0000","last_became_active":"2024-08-17T17:35:56.264265+0000","last_became_peered":"2024-08-17T17:35:56.264265+0000","last_unstale":"2024-08-17T17:40:07.715966+0000","last_undegraded":"2024-08-17T17:40:07.715966+0000","last_fullsized":"2024-08-17T17:40:07.715966+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:20.754326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:20.754326+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"1429'6791","reported_seq":15231,"reported_epoch":1429,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:12.707473+0000","last_change":"2024-08-17T17:37:11.134558+0000","last_active":"2024-08-17T17:40:12.707473+0000","last_peered":"2024-08-17T17:40:12.707473+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:20.105110+0000","last_became_peered":"2024-08-17T17:36:20.105110+0000","last_unstale":"2024-08-17T17:40:12.707473+0000","last_undegraded":"2024-08-17T17:36:19.999272+0000","last_fullsized":"2024-08-17T17:36:13.740520+0000","mapping_epoch":1383,"log_start":"1373'1000","ondisk_log_start":"1373'1000","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5791,"log_dups_size":0,"ondisk_log_size":5791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:45.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80150528,"num_objects":1223,"num_object_clones":0,"num_object_copies":3669,"num_objects_missing_on_primary":33,"num_objects_missing":33,"num_objects_degraded":1102,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6791,"num_write_kb":256448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2784,"num_bytes_recovered":182452224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":33},{"shards":"1,5,6","objects":1036},{"shards":"5,6","objects":154}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1258,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:07.716010+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:40:07.716010+0000","last_peered":"2024-08-17T17:40:07.716010+0000","last_clean":"2024-08-17T17:40:07.716010+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:40:07.716010+0000","last_undegraded":"2024-08-17T17:40:07.716010+0000","last_fullsized":"2024-08-17T17:40:07.716010+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1797,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823702+0000","last_change":"2024-08-17T17:36:12.758005+0000","last_active":"2024-08-17T17:39:58.823702+0000","last_peered":"2024-08-17T17:39:58.823702+0000","last_clean":"2024-08-17T17:39:58.823702+0000","last_became_active":"2024-08-17T17:36:12.757695+0000","last_became_peered":"2024-08-17T17:36:12.757695+0000","last_unstale":"2024-08-17T17:39:58.823702+0000","last_undegraded":"2024-08-17T17:39:58.823702+0000","last_fullsized":"2024-08-17T17:39:58.823702+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:51.792220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1435,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823483+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:39:58.823483+0000","last_peered":"2024-08-17T17:39:58.823483+0000","last_clean":"2024-08-17T17:39:58.823483+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:39:58.823483+0000","last_undegraded":"2024-08-17T17:39:58.823483+0000","last_fullsized":"2024-08-17T17:39:58.823483+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1755,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.853394+0000","last_change":"2024-08-17T17:40:09.853394+0000","last_active":"2024-08-17T17:40:09.853394+0000","last_peered":"2024-08-17T17:40:09.853394+0000","last_clean":"2024-08-17T17:40:09.853394+0000","last_became_active":"2024-08-17T17:35:55.204861+0000","last_became_peered":"2024-08-17T17:35:55.204861+0000","last_unstale":"2024-08-17T17:40:09.853394+0000","last_undegraded":"2024-08-17T17:40:09.853394+0000","last_fullsized":"2024-08-17T17:40:09.853394+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:40:12.530088+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.0","version":"1429'6848","reported_seq":14850,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:12.526139+0000","last_change":"2024-08-17T17:37:40.753703+0000","last_active":"2024-08-17T17:40:12.526139+0000","last_peered":"2024-08-17T17:40:12.526139+0000","last_clean":"2024-08-17T17:40:12.526139+0000","last_became_active":"2024-08-17T17:37:40.743458+0000","last_became_peered":"2024-08-17T17:37:40.743458+0000","last_unstale":"2024-08-17T17:40:12.526139+0000","last_undegraded":"2024-08-17T17:40:12.526139+0000","last_fullsized":"2024-08-17T17:40:12.526139+0000","mapping_epoch":1411,"log_start":"1373'1300","ondisk_log_start":"1373'1300","created":1357,"last_epoch_clean":1412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5548,"log_dups_size":0,"ondisk_log_size":5548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:14:02.340380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83099648,"num_objects":1268,"num_object_clones":0,"num_object_copies":3804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6848,"num_write_kb":259712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3914,"num_bytes_recovered":232652800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1031,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887754+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:39:58.887754+0000","last_peered":"2024-08-17T17:39:58.887754+0000","last_clean":"2024-08-17T17:39:58.887754+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:39:58.887754+0000","last_undegraded":"2024-08-17T17:39:58.887754+0000","last_fullsized":"2024-08-17T17:39:58.887754+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":105,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.864673+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:40:09.864673+0000","last_peered":"2024-08-17T17:40:09.864673+0000","last_clean":"2024-08-17T17:40:09.864673+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:40:09.864673+0000","last_undegraded":"2024-08-17T17:40:09.864673+0000","last_fullsized":"2024-08-17T17:40:09.864673+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1766,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.822937+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:39:58.822937+0000","last_peered":"2024-08-17T17:39:58.822937+0000","last_clean":"2024-08-17T17:39:58.822937+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:39:58.822937+0000","last_undegraded":"2024-08-17T17:39:58.822937+0000","last_fullsized":"2024-08-17T17:39:58.822937+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1","version":"1429'6797","reported_seq":15218,"reported_epoch":1429,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:12.731137+0000","last_change":"2024-08-17T17:36:23.103523+0000","last_active":"2024-08-17T17:40:12.731137+0000","last_peered":"2024-08-17T17:40:12.731137+0000","last_clean":"2024-08-17T17:35:54.178280+0000","last_became_active":"2024-08-17T17:36:23.025910+0000","last_became_peered":"2024-08-17T17:36:23.025910+0000","last_unstale":"2024-08-17T17:40:12.731137+0000","last_undegraded":"2024-08-17T17:36:18.734637+0000","last_fullsized":"2024-08-17T17:36:13.740134+0000","mapping_epoch":1383,"log_start":"1373'1000","ondisk_log_start":"1373'1000","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5797,"log_dups_size":0,"ondisk_log_size":5797,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:58.717055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82247680,"num_objects":1255,"num_object_clones":0,"num_object_copies":3765,"num_objects_missing_on_primary":54,"num_objects_missing":54,"num_objects_degraded":1184,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6797,"num_write_kb":257664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2774,"num_bytes_recovered":181796864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,0],"avail_no_missing":["0","2","4"],"object_location_counts":[{"shards":"0,2,4","objects":54},{"shards":"0,2,4,6","objects":6},{"shards":"0,2,6","objects":1070},{"shards":"0,6","objects":125}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1445,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887985+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:39:58.887985+0000","last_peered":"2024-08-17T17:39:58.887985+0000","last_clean":"2024-08-17T17:39:58.887985+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:39:58.887985+0000","last_undegraded":"2024-08-17T17:39:58.887985+0000","last_fullsized":"2024-08-17T17:39:58.887985+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1418,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887653+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:39:58.887653+0000","last_peered":"2024-08-17T17:39:58.887653+0000","last_clean":"2024-08-17T17:39:58.887653+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:39:58.887653+0000","last_undegraded":"2024-08-17T17:39:58.887653+0000","last_fullsized":"2024-08-17T17:39:58.887653+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":1007,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.854146+0000","last_change":"2024-08-17T17:34:26.817339+0000","last_active":"2024-08-17T17:40:09.854146+0000","last_peered":"2024-08-17T17:40:09.854146+0000","last_clean":"2024-08-17T17:40:09.854146+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:40:09.854146+0000","last_undegraded":"2024-08-17T17:40:09.854146+0000","last_fullsized":"2024-08-17T17:40:09.854146+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:26.817223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:26.817223+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:30.089956+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"1429'6630","reported_seq":6803,"reported_epoch":1429,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:10.588718+0000","last_change":"2024-08-17T17:36:14.073155+0000","last_active":"2024-08-17T17:40:10.588718+0000","last_peered":"2024-08-17T17:40:10.588718+0000","last_clean":"2024-08-17T17:36:11.734683+0000","last_became_active":"2024-08-17T17:36:14.063794+0000","last_became_peered":"2024-08-17T17:36:14.063794+0000","last_unstale":"2024-08-17T17:40:10.588718+0000","last_undegraded":"2024-08-17T17:36:13.855770+0000","last_fullsized":"2024-08-17T17:36:13.742068+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6630,"log_dups_size":0,"ondisk_log_size":6630,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83361792,"num_objects":1272,"num_object_clones":0,"num_object_copies":3816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3819,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6630,"num_write_kb":252864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-2547},{"shards":"0,2,7","objects":1140}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":53,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.888077+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:39:58.888077+0000","last_peered":"2024-08-17T17:39:58.888077+0000","last_clean":"2024-08-17T17:39:58.888077+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:39:58.888077+0000","last_undegraded":"2024-08-17T17:39:58.888077+0000","last_fullsized":"2024-08-17T17:39:58.888077+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1279,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.854992+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:39:58.854992+0000","last_peered":"2024-08-17T17:39:58.854992+0000","last_clean":"2024-08-17T17:39:58.854992+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:39:58.854992+0000","last_undegraded":"2024-08-17T17:39:58.854992+0000","last_fullsized":"2024-08-17T17:39:58.854992+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1795,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.863981+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:40:09.863981+0000","last_peered":"2024-08-17T17:40:09.863981+0000","last_clean":"2024-08-17T17:40:09.863981+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:40:09.863981+0000","last_undegraded":"2024-08-17T17:40:09.863981+0000","last_fullsized":"2024-08-17T17:40:09.863981+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"1429'6843","reported_seq":12647,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:12.618494+0000","last_change":"2024-08-17T17:36:46.982456+0000","last_active":"2024-08-17T17:40:12.618494+0000","last_peered":"2024-08-17T17:40:12.618494+0000","last_clean":"2024-08-17T17:40:12.618494+0000","last_became_active":"2024-08-17T17:36:46.974782+0000","last_became_peered":"2024-08-17T17:36:46.974782+0000","last_unstale":"2024-08-17T17:40:12.618494+0000","last_undegraded":"2024-08-17T17:40:12.618494+0000","last_fullsized":"2024-08-17T17:40:12.618494+0000","mapping_epoch":1399,"log_start":"1373'1000","ondisk_log_start":"1373'1000","created":1357,"last_epoch_clean":1400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5843,"log_dups_size":0,"ondisk_log_size":5843,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:40.499321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80543744,"num_objects":1229,"num_object_clones":0,"num_object_copies":3687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6843,"num_write_kb":258304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2848,"num_bytes_recovered":186646528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1396,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887825+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:39:58.887825+0000","last_peered":"2024-08-17T17:39:58.887825+0000","last_clean":"2024-08-17T17:39:58.887825+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:39:58.887825+0000","last_undegraded":"2024-08-17T17:39:58.887825+0000","last_fullsized":"2024-08-17T17:39:58.887825+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":210,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887986+0000","last_change":"2024-08-17T17:36:12.754372+0000","last_active":"2024-08-17T17:39:58.887986+0000","last_peered":"2024-08-17T17:39:58.887986+0000","last_clean":"2024-08-17T17:39:58.887986+0000","last_became_active":"2024-08-17T17:36:12.746352+0000","last_became_peered":"2024-08-17T17:36:12.746352+0000","last_unstale":"2024-08-17T17:39:58.887986+0000","last_undegraded":"2024-08-17T17:39:58.887986+0000","last_fullsized":"2024-08-17T17:39:58.887986+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:07.807486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1679,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.856550+0000","last_change":"2024-08-17T17:36:15.771193+0000","last_active":"2024-08-17T17:40:09.856550+0000","last_peered":"2024-08-17T17:40:09.856550+0000","last_clean":"2024-08-17T17:40:09.856550+0000","last_became_active":"2024-08-17T17:36:15.770889+0000","last_became_peered":"2024-08-17T17:36:15.770889+0000","last_unstale":"2024-08-17T17:40:09.856550+0000","last_undegraded":"2024-08-17T17:40:09.856550+0000","last_fullsized":"2024-08-17T17:40:09.856550+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:22.978695+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.4","version":"1429'6810","reported_seq":12595,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:12.532923+0000","last_change":"2024-08-17T17:37:04.045576+0000","last_active":"2024-08-17T17:40:12.532923+0000","last_peered":"2024-08-17T17:40:12.532923+0000","last_clean":"2024-08-17T17:40:12.532923+0000","last_became_active":"2024-08-17T17:37:04.038810+0000","last_became_peered":"2024-08-17T17:37:04.038810+0000","last_unstale":"2024-08-17T17:40:12.532923+0000","last_undegraded":"2024-08-17T17:40:12.532923+0000","last_fullsized":"2024-08-17T17:40:12.532923+0000","mapping_epoch":1402,"log_start":"1373'1000","ondisk_log_start":"1373'1000","created":1357,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5810,"log_dups_size":0,"ondisk_log_size":5810,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:27:04.725258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82313216,"num_objects":1256,"num_object_clones":0,"num_object_copies":3768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6810,"num_write_kb":258112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":176291840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1274,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.854499+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:40:09.854499+0000","last_peered":"2024-08-17T17:40:09.854499+0000","last_clean":"2024-08-17T17:40:09.854499+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:40:09.854499+0000","last_undegraded":"2024-08-17T17:40:09.854499+0000","last_fullsized":"2024-08-17T17:40:09.854499+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1422,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823469+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:39:58.823469+0000","last_peered":"2024-08-17T17:39:58.823469+0000","last_clean":"2024-08-17T17:39:58.823469+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:39:58.823469+0000","last_undegraded":"2024-08-17T17:39:58.823469+0000","last_fullsized":"2024-08-17T17:39:58.823469+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":923,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:40:02.440409+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:40:02.440409+0000","last_peered":"2024-08-17T17:40:02.440409+0000","last_clean":"2024-08-17T17:40:02.440409+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:40:02.440409+0000","last_undegraded":"2024-08-17T17:40:02.440409+0000","last_fullsized":"2024-08-17T17:40:02.440409+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"1429'6870","reported_seq":12710,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:12.431344+0000","last_change":"2024-08-17T17:36:32.711490+0000","last_active":"2024-08-17T17:40:12.431344+0000","last_peered":"2024-08-17T17:40:12.431344+0000","last_clean":"2024-08-17T17:40:12.431344+0000","last_became_active":"2024-08-17T17:36:32.704370+0000","last_became_peered":"2024-08-17T17:36:32.704370+0000","last_unstale":"2024-08-17T17:40:12.431344+0000","last_undegraded":"2024-08-17T17:40:12.431344+0000","last_fullsized":"2024-08-17T17:40:12.431344+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6870,"log_dups_size":0,"ondisk_log_size":6870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:17.373879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81133568,"num_objects":1238,"num_object_clones":0,"num_object_copies":3714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6870,"num_write_kb":259456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2866,"num_bytes_recovered":187826176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1278,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.859970+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:39:58.859970+0000","last_peered":"2024-08-17T17:39:58.859970+0000","last_clean":"2024-08-17T17:39:58.859970+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:39:58.859970+0000","last_undegraded":"2024-08-17T17:39:58.859970+0000","last_fullsized":"2024-08-17T17:39:58.859970+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":159,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823897+0000","last_change":"2024-08-17T17:35:55.217072+0000","last_active":"2024-08-17T17:39:58.823897+0000","last_peered":"2024-08-17T17:39:58.823897+0000","last_clean":"2024-08-17T17:39:58.823897+0000","last_became_active":"2024-08-17T17:35:55.216462+0000","last_became_peered":"2024-08-17T17:35:55.216462+0000","last_unstale":"2024-08-17T17:39:58.823897+0000","last_undegraded":"2024-08-17T17:39:58.823897+0000","last_fullsized":"2024-08-17T17:39:58.823897+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:34.937221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":905,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.826986+0000","last_change":"2024-08-17T17:35:55.215004+0000","last_active":"2024-08-17T17:39:58.826986+0000","last_peered":"2024-08-17T17:39:58.826986+0000","last_clean":"2024-08-17T17:39:58.826986+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:39:58.826986+0000","last_undegraded":"2024-08-17T17:39:58.826986+0000","last_fullsized":"2024-08-17T17:39:58.826986+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.6","version":"1429'6578","reported_seq":6727,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:10.653982+0000","last_change":"2024-08-17T17:35:14.868524+0000","last_active":"2024-08-17T17:40:10.653982+0000","last_peered":"2024-08-17T17:40:10.653982+0000","last_clean":"2024-08-17T17:40:10.653982+0000","last_became_active":"2024-08-17T17:35:14.854690+0000","last_became_peered":"2024-08-17T17:35:14.854690+0000","last_unstale":"2024-08-17T17:40:10.653982+0000","last_undegraded":"2024-08-17T17:40:10.653982+0000","last_fullsized":"2024-08-17T17:40:10.653982+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6578,"log_dups_size":0,"ondisk_log_size":6578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:05.388362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81133568,"num_objects":1238,"num_object_clones":0,"num_object_copies":3714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6578,"num_write_kb":250112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1464,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887593+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:39:58.887593+0000","last_peered":"2024-08-17T17:39:58.887593+0000","last_clean":"2024-08-17T17:39:58.887593+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:39:58.887593+0000","last_undegraded":"2024-08-17T17:39:58.887593+0000","last_fullsized":"2024-08-17T17:39:58.887593+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":171,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.888067+0000","last_change":"2024-08-17T17:35:33.147672+0000","last_active":"2024-08-17T17:39:58.888067+0000","last_peered":"2024-08-17T17:39:58.888067+0000","last_clean":"2024-08-17T17:39:58.888067+0000","last_became_active":"2024-08-17T17:35:33.147283+0000","last_became_peered":"2024-08-17T17:35:33.147283+0000","last_unstale":"2024-08-17T17:39:58.888067+0000","last_undegraded":"2024-08-17T17:39:58.888067+0000","last_fullsized":"2024-08-17T17:39:58.888067+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:41.677709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1689,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.861894+0000","last_change":"2024-08-17T17:40:09.861894+0000","last_active":"2024-08-17T17:40:09.861894+0000","last_peered":"2024-08-17T17:40:09.861894+0000","last_clean":"2024-08-17T17:40:09.861894+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:40:09.861894+0000","last_undegraded":"2024-08-17T17:40:09.861894+0000","last_fullsized":"2024-08-17T17:40:09.861894+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:38.241235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:38.241235+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:40:13.117857+0000","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"1429'6705","reported_seq":6787,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:12.433626+0000","last_change":"2024-08-17T17:34:50.670013+0000","last_active":"2024-08-17T17:40:12.433626+0000","last_peered":"2024-08-17T17:40:12.433626+0000","last_clean":"2024-08-17T17:40:12.433626+0000","last_became_active":"2024-08-17T17:34:50.669299+0000","last_became_peered":"2024-08-17T17:34:50.669299+0000","last_unstale":"2024-08-17T17:40:12.433626+0000","last_undegraded":"2024-08-17T17:40:12.433626+0000","last_fullsized":"2024-08-17T17:40:12.433626+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6705,"log_dups_size":0,"ondisk_log_size":6705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:11.172227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80019456,"num_objects":1221,"num_object_clones":0,"num_object_copies":3663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6705,"num_write_kb":253632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1388,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823109+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:39:58.823109+0000","last_peered":"2024-08-17T17:39:58.823109+0000","last_clean":"2024-08-17T17:39:58.823109+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:39:58.823109+0000","last_undegraded":"2024-08-17T17:39:58.823109+0000","last_fullsized":"2024-08-17T17:39:58.823109+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":158,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887914+0000","last_change":"2024-08-17T17:35:33.140725+0000","last_active":"2024-08-17T17:39:58.887914+0000","last_peered":"2024-08-17T17:39:58.887914+0000","last_clean":"2024-08-17T17:39:58.887914+0000","last_became_active":"2024-08-17T17:35:33.140406+0000","last_became_peered":"2024-08-17T17:35:33.140406+0000","last_unstale":"2024-08-17T17:39:58.887914+0000","last_undegraded":"2024-08-17T17:39:58.887914+0000","last_fullsized":"2024-08-17T17:39:58.887914+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:54:51.875571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1678,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887886+0000","last_change":"2024-08-17T17:35:33.138514+0000","last_active":"2024-08-17T17:39:58.887886+0000","last_peered":"2024-08-17T17:39:58.887886+0000","last_clean":"2024-08-17T17:39:58.887886+0000","last_became_active":"2024-08-17T17:35:33.137035+0000","last_became_peered":"2024-08-17T17:35:33.137035+0000","last_unstale":"2024-08-17T17:39:58.887886+0000","last_undegraded":"2024-08-17T17:39:58.887886+0000","last_fullsized":"2024-08-17T17:39:58.887886+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:50.565662+0000","scrub_duration":0.001487138,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1319,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.826658+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:39:58.826658+0000","last_peered":"2024-08-17T17:39:58.826658+0000","last_clean":"2024-08-17T17:39:58.826658+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:39:58.826658+0000","last_undegraded":"2024-08-17T17:39:58.826658+0000","last_fullsized":"2024-08-17T17:39:58.826658+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":105,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.862412+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:40:09.862412+0000","last_peered":"2024-08-17T17:40:09.862412+0000","last_clean":"2024-08-17T17:40:09.862412+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:40:09.862412+0000","last_undegraded":"2024-08-17T17:40:09.862412+0000","last_fullsized":"2024-08-17T17:40:09.862412+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1750,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823027+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:39:58.823027+0000","last_peered":"2024-08-17T17:39:58.823027+0000","last_clean":"2024-08-17T17:39:58.823027+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:39:58.823027+0000","last_undegraded":"2024-08-17T17:39:58.823027+0000","last_fullsized":"2024-08-17T17:39:58.823027+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1337,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.887505+0000","last_change":"2024-08-17T17:35:55.205292+0000","last_active":"2024-08-17T17:39:58.887505+0000","last_peered":"2024-08-17T17:39:58.887505+0000","last_clean":"2024-08-17T17:39:58.887505+0000","last_became_active":"2024-08-17T17:35:55.204935+0000","last_became_peered":"2024-08-17T17:35:55.204935+0000","last_unstale":"2024-08-17T17:39:58.887505+0000","last_undegraded":"2024-08-17T17:39:58.887505+0000","last_fullsized":"2024-08-17T17:39:58.887505+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:52.277821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":105,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.862410+0000","last_change":"2024-08-17T17:36:12.749972+0000","last_active":"2024-08-17T17:40:09.862410+0000","last_peered":"2024-08-17T17:40:09.862410+0000","last_clean":"2024-08-17T17:40:09.862410+0000","last_became_active":"2024-08-17T17:36:12.749620+0000","last_became_peered":"2024-08-17T17:36:12.749620+0000","last_unstale":"2024-08-17T17:40:09.862410+0000","last_undegraded":"2024-08-17T17:40:09.862410+0000","last_fullsized":"2024-08-17T17:40:09.862410+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:04.994357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":981,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.861751+0000","last_change":"2024-08-17T17:36:20.832274+0000","last_active":"2024-08-17T17:40:09.861751+0000","last_peered":"2024-08-17T17:40:09.861751+0000","last_clean":"2024-08-17T17:40:09.861751+0000","last_became_active":"2024-08-17T17:36:20.831886+0000","last_became_peered":"2024-08-17T17:36:20.831886+0000","last_unstale":"2024-08-17T17:40:09.861751+0000","last_undegraded":"2024-08-17T17:40:09.861751+0000","last_fullsized":"2024-08-17T17:40:09.861751+0000","mapping_epoch":1390,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1391,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:05.248711+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1191,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.854895+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:40:09.854895+0000","last_peered":"2024-08-17T17:40:09.854895+0000","last_clean":"2024-08-17T17:40:09.854895+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:40:09.854895+0000","last_undegraded":"2024-08-17T17:40:09.854895+0000","last_fullsized":"2024-08-17T17:40:09.854895+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1420,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.823970+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:39:58.823970+0000","last_peered":"2024-08-17T17:39:58.823970+0000","last_clean":"2024-08-17T17:39:58.823970+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:39:58.823970+0000","last_undegraded":"2024-08-17T17:39:58.823970+0000","last_fullsized":"2024-08-17T17:39:58.823970+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1713,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.822954+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:39:58.822954+0000","last_peered":"2024-08-17T17:39:58.822954+0000","last_clean":"2024-08-17T17:39:58.822954+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:39:58.822954+0000","last_undegraded":"2024-08-17T17:39:58.822954+0000","last_fullsized":"2024-08-17T17:39:58.822954+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1362,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.865413+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:40:09.865413+0000","last_peered":"2024-08-17T17:40:09.865413+0000","last_clean":"2024-08-17T17:40:09.865413+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:40:09.865413+0000","last_undegraded":"2024-08-17T17:40:09.865413+0000","last_fullsized":"2024-08-17T17:40:09.865413+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1909,"reported_epoch":1429,"state":"active+clean","last_fresh":"2024-08-17T17:40:09.862115+0000","last_change":"2024-08-17T17:35:33.137411+0000","last_active":"2024-08-17T17:40:09.862115+0000","last_peered":"2024-08-17T17:40:09.862115+0000","last_clean":"2024-08-17T17:40:09.862115+0000","last_became_active":"2024-08-17T17:35:33.137073+0000","last_became_peered":"2024-08-17T17:35:33.137073+0000","last_unstale":"2024-08-17T17:40:09.862115+0000","last_undegraded":"2024-08-17T17:40:09.862115+0000","last_fullsized":"2024-08-17T17:40:09.862115+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:35.924539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1807,"reported_epoch":1428,"state":"active+clean","last_fresh":"2024-08-17T17:39:58.822936+0000","last_change":"2024-08-17T17:36:15.787445+0000","last_active":"2024-08-17T17:39:58.822936+0000","last_peered":"2024-08-17T17:39:58.822936+0000","last_clean":"2024-08-17T17:39:58.822936+0000","last_became_active":"2024-08-17T17:36:15.787099+0000","last_became_peered":"2024-08-17T17:36:15.787099+0000","last_unstale":"2024-08-17T17:39:58.822936+0000","last_undegraded":"2024-08-17T17:39:58.822936+0000","last_fullsized":"2024-08-17T17:39:58.822936+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:47.448168+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":1316290584,"num_objects":20086,"num_object_clones":0,"num_object_copies":60258,"num_objects_missing_on_primary":138,"num_objects_missing":138,"num_objects_degraded":8118,"num_objects_misplaced":1185,"num_objects_unfound":0,"num_objects_dirty":20086,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":108796,"num_write_kb":4124161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34328,"num_bytes_recovered":2097086464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5465518080,"data_stored":5465505864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100596,"ondisk_log_size":100596,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635092,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":27,"num_read_kb":61,"num_write":15,"num_write_kb":183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1314910,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162873,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":910580,"kb_used_data":741868,"kb_used_omap":54,"kb_used_meta":168649,"kb_avail":92834572,"statfs":{"total":95995035648,"available":95062601728,"internally_reserved":0,"allocated":759672832,"data_stored":756970912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":172696597},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999347,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":911476,"kb_used_data":699308,"kb_used_omap":47,"kb_used_meta":212112,"kb_avail":92833676,"statfs":{"total":95995035648,"available":95061684224,"internally_reserved":0,"allocated":716091392,"data_stored":713390449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":217203348},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,1,5,1,3,5,4],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627144,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1282328,"kb_used_data":209680,"kb_used_omap":42,"kb_used_meta":1072533,"kb_avail":92462824,"statfs":{"total":95995035648,"available":94681931776,"internally_reserved":0,"allocated":214712320,"data_stored":212033658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1098273852},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785621,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1942364,"kb_used_data":1138196,"kb_used_omap":48,"kb_used_meta":804111,"kb_avail":91802788,"statfs":{"total":95995035648,"available":94006054912,"internally_reserved":0,"allocated":1165512704,"data_stored":1162830917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":823410350},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725138,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1231588,"kb_used_data":326044,"kb_used_omap":1,"kb_used_meta":905534,"kb_avail":92513564,"statfs":{"total":95995035648,"available":94733889536,"internally_reserved":0,"allocated":333869056,"data_stored":331179219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":927267270},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986775,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1582192,"kb_used_data":694696,"kb_used_omap":1,"kb_used_meta":887486,"kb_avail":92162960,"statfs":{"total":95995035648,"available":94374871040,"internally_reserved":0,"allocated":711368704,"data_stored":708670290,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":908786118},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706382,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1947368,"kb_used_data":1008796,"kb_used_omap":35,"kb_used_meta":938524,"kb_avail":91797784,"statfs":{"total":95995035648,"available":94000930816,"internally_reserved":0,"allocated":1033007104,"data_stored":1030317248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":961048631},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832155,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1631460,"kb_used_data":600856,"kb_used_omap":65,"kb_used_meta":1030526,"kb_avail":92113692,"statfs":{"total":95995035648,"available":94324420608,"internally_reserved":0,"allocated":615276544,"data_stored":612591246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1055258811},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":605356032,"data_stored":605356032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1157300224,"data_stored":1157300224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1018236928,"data_stored":1018232856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":205717504,"data_stored":205717504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":697962496,"data_stored":697958424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":324927488,"data_stored":324927488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":705957888,"data_stored":705953816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":750059520,"data_stored":750059520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:40:18.417 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:40:18.424 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:18.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:18.526 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:18.549 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:18.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:18.651 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:18.674 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:18.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:18.775 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:18.810 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:18.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:18.912 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:18.954 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:18.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:18.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:19.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:18 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 7890/57390 objects degraded (13.748%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:19.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:18 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 7890/57390 objects degraded (13.748%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:19.056 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:19.080 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:19.084 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:19.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:19.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:18 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 7890/57390 objects degraded (13.748%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:19.186 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:19.214 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:19.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:19.316 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:19.344 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:19.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:19.435 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:40:19.436 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:40:19.446 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:19.487 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:19.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:19.589 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:19.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.634 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:19.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:19.735 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:19.780 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:19.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:19.882 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:19.913 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:19.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:19.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:20.014 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:20.048 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:20.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:20.150 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:20.178 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:20.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:20.208 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:19 smithi063 ceph-mon[109843]: pgmap v2281: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 7723/51393 objects degraded (15.027%); 874/51393 objects misplaced (1.701%); 5.7 MiB/s, 91 objects/s recovering 2024-08-17T17:40:20.208 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:19 smithi063 ceph-mon[114756]: pgmap v2281: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 7723/51393 objects degraded (15.027%); 874/51393 objects misplaced (1.701%); 5.7 MiB/s, 91 objects/s recovering 2024-08-17T17:40:20.248 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-08-17T17:40:20.280 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:20.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:19 smithi162 ceph-mon[104668]: pgmap v2281: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 497 op/s; 7723/51393 objects degraded (15.027%); 874/51393 objects misplaced (1.701%); 5.7 MiB/s, 91 objects/s recovering 2024-08-17T17:40:20.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.349 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:20.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:20.449 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:20.477 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:20.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:20.579 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:20.604 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:20.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:20.706 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:20.749 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:20.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:20.851 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:20.905 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:20.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:20.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:21.007 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:21.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.051 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:21.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:21.109 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:40:21.152 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:21.182 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:21.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:21.284 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:21.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.340 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:21.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:21.441 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:21.480 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:21.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:21.582 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:21.617 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:21.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:21.719 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:21.753 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:21.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:21.841 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:21.854 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:21.877 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:21.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:21.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:21.929 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:21.979 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:22.005 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:22.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:22.107 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:22.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.150 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:22.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:22.175 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:21 smithi063 ceph-mon[109843]: pgmap v2282: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 7723/50619 objects degraded (15.257%); 874/50619 objects misplaced (1.727%); 5.3 MiB/s, 84 objects/s recovering 2024-08-17T17:40:22.176 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:21 smithi063 ceph-mon[114756]: pgmap v2282: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 7723/50619 objects degraded (15.257%); 874/50619 objects misplaced (1.727%); 5.3 MiB/s, 84 objects/s recovering 2024-08-17T17:40:22.252 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:22.298 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:22.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.299 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:22.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:21 smithi162 ceph-mon[104668]: pgmap v2282: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 1.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 7723/50619 objects degraded (15.257%); 874/50619 objects misplaced (1.727%); 5.3 MiB/s, 84 objects/s recovering 2024-08-17T17:40:22.400 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:22.429 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:22.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.430 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:22.531 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:22.561 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:22.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:22.663 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:22.706 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:22.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:22.725 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:22.808 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:22.837 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:22.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:22.939 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:22.968 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:22.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:22.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:23.069 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:23.099 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:23.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:23.201 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:23.227 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:23.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:23.329 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:23.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.381 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:23.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:23.482 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:23.509 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:23.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:23.563 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:40:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:40:23] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.51.0" 2024-08-17T17:40:23.611 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:23.641 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:23.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:23.743 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:23.770 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:23.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:23.872 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:23.900 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:23.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:23.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:24.002 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:24.027 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:24.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:24.129 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:24.158 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:24.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:24.260 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:24.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:23 smithi063 ceph-mon[109843]: pgmap v2283: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 890 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 7723/42789 objects degraded (18.049%); 874/42789 objects misplaced (2.043%); 4.7 MiB/s, 75 objects/s recovering 2024-08-17T17:40:24.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:23 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 7723/42789 objects degraded (18.049%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:24.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:23 smithi063 ceph-mon[114756]: pgmap v2283: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 890 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 7723/42789 objects degraded (18.049%); 874/42789 objects misplaced (2.043%); 4.7 MiB/s, 75 objects/s recovering 2024-08-17T17:40:24.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:23 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 7723/42789 objects degraded (18.049%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:24.304 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:24.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:24.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:23 smithi162 ceph-mon[104668]: pgmap v2283: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 890 MiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 644 op/s; 7723/42789 objects degraded (18.049%); 874/42789 objects misplaced (2.043%); 4.7 MiB/s, 75 objects/s recovering 2024-08-17T17:40:24.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:23 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 7723/42789 objects degraded (18.049%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:24.406 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:24.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.454 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:24.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:24.555 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:24.581 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:24.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:24.683 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:24.728 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:24.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:24.830 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:24.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:24.857 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:24.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:24.922 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is set 2024-08-17T17:40:24.958 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:25.010 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:25.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:25.111 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:25.136 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:25.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:25.238 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:25.268 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:25.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:25.270 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:24 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/703546731' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:40:25.270 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:24 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/703546731' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:40:25.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:24 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/703546731' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:40:25.370 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:25.403 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:25.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:25.504 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:25.544 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:25.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:25.646 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:25.677 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:25.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:25.779 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:25.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.810 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:25.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:25.911 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:25.943 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:25.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:25.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:25.970 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:25 smithi063 ceph-mon[109843]: pgmap v2284: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 768 MiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 7727/36939 objects degraded (20.918%); 574/36939 objects misplaced (1.554%); 5.8 MiB/s, 93 objects/s recovering 2024-08-17T17:40:25.970 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:25 smithi063 ceph-mon[109843]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:25.970 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:25 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/703546731' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:25.970 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:25 smithi063 ceph-mon[109843]: osdmap e1430: 8 total, 8 up, 8 in 2024-08-17T17:40:25.970 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:25 smithi063 ceph-mon[114756]: pgmap v2284: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 768 MiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 7727/36939 objects degraded (20.918%); 574/36939 objects misplaced (1.554%); 5.8 MiB/s, 93 objects/s recovering 2024-08-17T17:40:25.970 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:25 smithi063 ceph-mon[114756]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:25.970 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:25 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/703546731' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:25.970 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:25 smithi063 ceph-mon[114756]: osdmap e1430: 8 total, 8 up, 8 in 2024-08-17T17:40:26.045 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:26.085 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:26.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:26.187 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:26.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.237 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:26.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:26.338 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:26.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:25 smithi162 ceph-mon[104668]: pgmap v2284: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 768 MiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 712 op/s; 7727/36939 objects degraded (20.918%); 574/36939 objects misplaced (1.554%); 5.8 MiB/s, 93 objects/s recovering 2024-08-17T17:40:26.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:25 smithi162 ceph-mon[104668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:26.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:25 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/703546731' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:26.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:25 smithi162 ceph-mon[104668]: osdmap e1430: 8 total, 8 up, 8 in 2024-08-17T17:40:26.365 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:26.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:26.467 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:26.507 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:26.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:26.610 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:26.658 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:26.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:26.760 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:26.818 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:26.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:26.919 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:26.957 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:26.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:26.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:27.058 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:27.097 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:27.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:27.199 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:27.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.226 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:27.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:27.254 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:40:27.255 INFO:teuthology.orchestra.run.smithi063.stdout:{"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 7727/31953 objects degraded (24.182%), 5 pgs degraded, 5 pgs undersized","count":10},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2306,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1430,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916154,"num_in_osds":8,"osd_in_since":1723916113,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":113},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+clean+scrubbing","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":10651,"data_bytes":696895153,"bytes_used":10261581824,"bytes_avail":757698703360,"bytes_total":767960285184,"degraded_objects":7727,"degraded_total":31953,"degraded_ratio":0.24182392889556537,"misplaced_objects":574,"misplaced_total":31953,"misplaced_ratio":0.017963884455293712,"recovering_objects_per_sec":63,"recovering_bytes_per_sec":4166565,"recovering_keys_per_sec":0,"num_objects_recovered":636,"num_bytes_recovered":41680896,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":847},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi063.ultpqj","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.63:8443/","prometheus":"http://172.21.15.63:9283/"}},"servicemap":{"epoch":33,"modified":"2024-08-17T17:38:44.649232+0000","services":{}},"progress_events":{}} 2024-08-17T17:40:27.327 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:27.355 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:27.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:27.456 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:27.492 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:27.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:27.594 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:27.628 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:27.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:27.730 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:27.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.782 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:27.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:27.883 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:27.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:27.927 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:27.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:28.021 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:27 smithi063 ceph-mon[114756]: pgmap v2286: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 665 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 7727/31953 objects degraded (24.182%); 574/31953 objects misplaced (1.796%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:40:28.021 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:27 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2705867686' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:40:28.022 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:27 smithi063 ceph-mon[109843]: pgmap v2286: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 665 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 7727/31953 objects degraded (24.182%); 574/31953 objects misplaced (1.796%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:40:28.022 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:27 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2705867686' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:40:28.028 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:28.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.052 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:28.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:28.153 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:28.200 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:28.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:28.302 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:28.327 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:28.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:28.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:27 smithi162 ceph-mon[104668]: pgmap v2286: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 665 MiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 7727/31953 objects degraded (24.182%); 574/31953 objects misplaced (1.796%); 4.0 MiB/s, 63 objects/s recovering 2024-08-17T17:40:28.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:27 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2705867686' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:40:28.429 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:28.445 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:40:28.469 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:28.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:28.571 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:28.626 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:28.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:28.728 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:28.752 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:28.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:28.823 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:40:28.823 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:40:28.854 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:28.887 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:28.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:28.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:28.989 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:29.019 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:29.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:29.121 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:29.163 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:29.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:29.265 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:29.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:28 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 7727/31953 objects degraded (24.182%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:29.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:28 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 7727/31953 objects degraded (24.182%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:29.291 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:29.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:29.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:28 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 7727/31953 objects degraded (24.182%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:29.394 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:29.416 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:29.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:29.518 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:29.564 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:29.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:29.616 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-08-17T17:40:29.616 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:40:29.666 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:29.711 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:29.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:29.813 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:29.845 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:29.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:29.947 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:29.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:29.978 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:29.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:30.039 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-08-17T17:40:30.079 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:30.160 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:30.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:30.194 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:29 smithi063 ceph-mon[109843]: pgmap v2287: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 511 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 7727/24579 objects degraded (31.437%); 254/24579 objects misplaced (1.033%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:40:30.194 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:29 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2046613262' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:30.194 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:29 smithi063 ceph-mon[114756]: pgmap v2287: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 511 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 7727/24579 objects degraded (31.437%); 254/24579 objects misplaced (1.033%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:40:30.195 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:29 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2046613262' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:30.195 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:40:30.195 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:40:30.195 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:40:30.195 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:40:30.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:40:30.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:40:30.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:40:30.196 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:40:30.197 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:40:30.197 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:40:30.197 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:40:30.197 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:40:30.197 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:40:30.197 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:40:30.198 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:40:30.198 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:40:30.262 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:30.321 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:30.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:30.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:29 smithi162 ceph-mon[104668]: pgmap v2287: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 511 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 7727/24579 objects degraded (31.437%); 254/24579 objects misplaced (1.033%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:40:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:29 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2046613262' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:30.422 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:30.453 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:30.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:30.555 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:30.586 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:30.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:30.688 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:30.729 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:30.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:30.831 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:30.856 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:30.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:30.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:30.958 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:31.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.040 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:31.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:31.073 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:31.111 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:31.136 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:31.141 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:31.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.200 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:31.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:31.300 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:31.342 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:31.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:31.444 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:31.470 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:31.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:31.571 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:31.600 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:31.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:31.702 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:31.724 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:31.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:31.826 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:31.853 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:31.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:31.954 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:31.995 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:31.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:31.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:32.097 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:32.122 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:32.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:32.158 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:31 smithi063 ceph-mon[114756]: pgmap v2288: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 495 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 7727/23802 objects degraded (32.464%); 254/23802 objects misplaced (1.067%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:40:32.158 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:32.158 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:31 smithi063 ceph-mon[109843]: pgmap v2288: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 495 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 7727/23802 objects degraded (32.464%); 254/23802 objects misplaced (1.067%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:40:32.158 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:32.225 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:32.247 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:32.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:32.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:31 smithi162 ceph-mon[104668]: pgmap v2288: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 495 MiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 893 op/s; 7727/23802 objects degraded (32.464%); 254/23802 objects misplaced (1.067%); 3.8 MiB/s, 61 objects/s recovering 2024-08-17T17:40:32.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:32.349 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:32.371 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:32.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:32.473 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:32.495 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:32.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:32.596 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:32.618 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:32.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:32.720 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:32.741 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:32.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:32.842 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:32.863 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:32.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:32.965 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:32.989 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:40:32.994 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:32.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:32.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:33.096 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:33.122 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:33.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:33.223 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:33.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.270 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:33.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:33.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:32 smithi063 ceph-mon[114756]: osdmap e1431: 8 total, 8 up, 8 in 2024-08-17T17:40:33.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:32 smithi063 ceph-mon[109843]: osdmap e1431: 8 total, 8 up, 8 in 2024-08-17T17:40:33.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:32 smithi162 ceph-mon[104668]: osdmap e1431: 8 total, 8 up, 8 in 2024-08-17T17:40:33.372 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:33.423 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:33.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.424 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:33.524 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:33.554 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:33.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:33.656 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:33.686 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:33.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:33.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:40:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:40:33] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.51.0" 2024-08-17T17:40:33.787 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:33.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.812 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:33.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:33.913 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:33.938 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:33.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:33.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:34.040 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:34.066 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:34.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:34.168 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:34.197 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:34.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:34.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:33 smithi063 ceph-mon[109843]: pgmap v2290: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 358 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 7727/17241 objects degraded (44.818%); 254/17241 objects misplaced (1.473%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:40:34.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:33 smithi063 ceph-mon[109843]: osdmap e1432: 8 total, 8 up, 8 in 2024-08-17T17:40:34.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:33 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 7727/17241 objects degraded (44.818%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:34.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:33 smithi063 ceph-mon[114756]: pgmap v2290: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 358 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 7727/17241 objects degraded (44.818%); 254/17241 objects misplaced (1.473%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:40:34.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:33 smithi063 ceph-mon[114756]: osdmap e1432: 8 total, 8 up, 8 in 2024-08-17T17:40:34.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:33 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 7727/17241 objects degraded (44.818%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:34.298 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:34.332 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:34.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:34.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:33 smithi162 ceph-mon[104668]: pgmap v2290: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 358 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 7727/17241 objects degraded (44.818%); 254/17241 objects misplaced (1.473%); 2.5 MiB/s, 39 objects/s recovering 2024-08-17T17:40:34.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:33 smithi162 ceph-mon[104668]: osdmap e1432: 8 total, 8 up, 8 in 2024-08-17T17:40:34.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:33 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 7727/17241 objects degraded (44.818%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:34.434 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:34.461 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:34.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:34.563 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:34.602 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:34.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:34.704 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:34.753 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:34.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:34.855 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:34.883 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:34.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:34.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:34.985 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:35.041 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:35.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:35.142 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:35.187 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:35.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:35.289 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:35.314 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:35.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:35.416 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:35.459 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:35.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:35.560 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:35.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.618 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:35.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:35.719 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:35.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.747 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:35.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:35.848 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:35.888 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:35.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:35.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:35.990 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:36.036 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:36.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:36.138 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:36.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.182 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:36.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:36.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:35 smithi063 ceph-mon[114756]: pgmap v2292: 119 pgs: 1 peering, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 257 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 7711/12408 objects degraded (62.145%); 4.5 MiB/s, 73 objects/s recovering 2024-08-17T17:40:36.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:35 smithi063 ceph-mon[109843]: pgmap v2292: 119 pgs: 1 peering, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 257 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 7711/12408 objects degraded (62.145%); 4.5 MiB/s, 73 objects/s recovering 2024-08-17T17:40:36.283 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:36.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.329 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:36.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:36.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:35 smithi162 ceph-mon[104668]: pgmap v2292: 119 pgs: 1 peering, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 257 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 813 op/s; 7711/12408 objects degraded (62.145%); 4.5 MiB/s, 73 objects/s recovering 2024-08-17T17:40:36.430 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:36.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.476 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:36.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:36.577 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:36.617 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:36.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.618 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:40:36.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:36.624 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:40:36.719 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:36.767 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:36.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:36.868 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:36.893 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:36.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:36.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:36.995 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:37.004 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:40:37.024 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:37.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:37.126 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:37.158 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:37.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:37.260 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:37.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:36 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1122210682' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:37.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:36 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:37.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:36 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1122210682' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:37.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:36 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:37.286 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:37.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:37.322 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:37.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:36 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1122210682' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:37.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:36 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:40:37.388 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:37.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.417 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:37.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:37.418 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Removed 64440 objects 2024-08-17T17:40:37.418 INFO:tasks.radosbench.radosbench.0.smithi063.stdout:Clean up completed and total clean up time :245.729 2024-08-17T17:40:37.518 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:37.542 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:37.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:37.643 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:37.665 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:37.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:37.766 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:37.787 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:37.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:37.889 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:37.910 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:37.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:37.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:38.012 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:38.033 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:38.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:38.135 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:38.156 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:38.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:38.197 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2292,"stamp":"2024-08-17T17:40:34.682849+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":269928113,"num_objects":4136,"num_object_clones":0,"num_object_copies":12408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7711,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4136,"num_whiteouts":0,"num_read":28,"num_read_kb":62,"num_write":124851,"num_write_kb":4124377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35692,"num_bytes_recovered":2185171677,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112350,"ondisk_log_size":112350,"up":357,"acting":353,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":375,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8819476,"kb_used_data":2700868,"kb_used_omap":298,"kb_used_meta":6118165,"kb_avail":741141740,"statfs":{"total":767960285184,"available":758929141760,"internally_reserved":0,"allocated":2765688832,"data_stored":2744114454,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":6265001489},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,1,1,0,4,8,3],"upper_bound":512},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-426967040,"num_objects":-6515,"num_object_clones":0,"num_object_copies":-19545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-16,"num_objects_misplaced":-574,"num_objects_unfound":0,"num_objects_dirty":-6515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6515,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":590,"num_bytes_recovered":37617664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003868"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":67,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.929353+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:40:24.929353+0000","last_peered":"2024-08-17T17:40:24.929353+0000","last_clean":"2024-08-17T17:40:24.929353+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:40:24.929353+0000","last_undegraded":"2024-08-17T17:40:24.929353+0000","last_fullsized":"2024-08-17T17:40:24.929353+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1446,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.933919+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:40:24.933919+0000","last_peered":"2024-08-17T17:40:24.933919+0000","last_clean":"2024-08-17T17:40:24.933919+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:40:24.933919+0000","last_undegraded":"2024-08-17T17:40:24.933919+0000","last_fullsized":"2024-08-17T17:40:24.933919+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1761,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:27.452885+0000","last_change":"2024-08-17T17:36:15.771150+0000","last_active":"2024-08-17T17:40:27.452885+0000","last_peered":"2024-08-17T17:40:27.452885+0000","last_clean":"2024-08-17T17:40:27.452885+0000","last_became_active":"2024-08-17T17:36:15.770520+0000","last_became_peered":"2024-08-17T17:36:15.770520+0000","last_unstale":"2024-08-17T17:40:27.452885+0000","last_undegraded":"2024-08-17T17:40:27.452885+0000","last_fullsized":"2024-08-17T17:40:27.452885+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:17.552796+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":96,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.922293+0000","last_change":"2024-08-17T17:36:12.756709+0000","last_active":"2024-08-17T17:40:24.922293+0000","last_peered":"2024-08-17T17:40:24.922293+0000","last_clean":"2024-08-17T17:40:24.922293+0000","last_became_active":"2024-08-17T17:36:12.756385+0000","last_became_peered":"2024-08-17T17:36:12.756385+0000","last_unstale":"2024-08-17T17:40:24.922293+0000","last_undegraded":"2024-08-17T17:40:24.922293+0000","last_fullsized":"2024-08-17T17:40:24.922293+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:21.420282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":152,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.925617+0000","last_change":"2024-08-17T17:36:12.762800+0000","last_active":"2024-08-17T17:40:24.925617+0000","last_peered":"2024-08-17T17:40:24.925617+0000","last_clean":"2024-08-17T17:40:24.925617+0000","last_became_active":"2024-08-17T17:36:12.759977+0000","last_became_peered":"2024-08-17T17:36:12.759977+0000","last_unstale":"2024-08-17T17:40:24.925617+0000","last_undegraded":"2024-08-17T17:40:24.925617+0000","last_fullsized":"2024-08-17T17:40:24.925617+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:02.162681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1640,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.921616+0000","last_change":"2024-08-17T17:35:56.281561+0000","last_active":"2024-08-17T17:40:24.921616+0000","last_peered":"2024-08-17T17:40:24.921616+0000","last_clean":"2024-08-17T17:40:24.921616+0000","last_became_active":"2024-08-17T17:35:56.278638+0000","last_became_peered":"2024-08-17T17:35:56.278638+0000","last_unstale":"2024-08-17T17:40:24.921616+0000","last_undegraded":"2024-08-17T17:40:24.921616+0000","last_fullsized":"2024-08-17T17:40:24.921616+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1379,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926657+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:40:24.926657+0000","last_peered":"2024-08-17T17:40:24.926657+0000","last_clean":"2024-08-17T17:40:24.926657+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:40:24.926657+0000","last_undegraded":"2024-08-17T17:40:24.926657+0000","last_fullsized":"2024-08-17T17:40:24.926657+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":55,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.933948+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:40:24.933948+0000","last_peered":"2024-08-17T17:40:24.933948+0000","last_clean":"2024-08-17T17:40:24.933948+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:40:24.933948+0000","last_undegraded":"2024-08-17T17:40:24.933948+0000","last_fullsized":"2024-08-17T17:40:24.933948+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1424,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926357+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:40:24.926357+0000","last_peered":"2024-08-17T17:40:24.926357+0000","last_clean":"2024-08-17T17:40:24.926357+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:40:24.926357+0000","last_undegraded":"2024-08-17T17:40:24.926357+0000","last_fullsized":"2024-08-17T17:40:24.926357+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1432,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.934641+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:40:24.934641+0000","last_peered":"2024-08-17T17:40:24.934641+0000","last_clean":"2024-08-17T17:40:24.934641+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:40:24.934641+0000","last_undegraded":"2024-08-17T17:40:24.934641+0000","last_fullsized":"2024-08-17T17:40:24.934641+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":723,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.921755+0000","last_change":"2024-08-17T17:36:15.773315+0000","last_active":"2024-08-17T17:40:24.921755+0000","last_peered":"2024-08-17T17:40:24.921755+0000","last_clean":"2024-08-17T17:40:24.921755+0000","last_became_active":"2024-08-17T17:36:15.772961+0000","last_became_peered":"2024-08-17T17:36:15.772961+0000","last_unstale":"2024-08-17T17:40:24.921755+0000","last_undegraded":"2024-08-17T17:40:24.921755+0000","last_fullsized":"2024-08-17T17:40:24.921755+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:10.190221+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":61,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.932036+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:40:24.932036+0000","last_peered":"2024-08-17T17:40:24.932036+0000","last_clean":"2024-08-17T17:40:24.932036+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:40:24.932036+0000","last_undegraded":"2024-08-17T17:40:24.932036+0000","last_fullsized":"2024-08-17T17:40:24.932036+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":152,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.927863+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:40:24.927863+0000","last_peered":"2024-08-17T17:40:24.927863+0000","last_clean":"2024-08-17T17:40:24.927863+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:40:24.927863+0000","last_undegraded":"2024-08-17T17:40:24.927863+0000","last_fullsized":"2024-08-17T17:40:24.927863+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1513,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.922655+0000","last_change":"2024-08-17T17:36:24.225739+0000","last_active":"2024-08-17T17:40:24.922655+0000","last_peered":"2024-08-17T17:40:24.922655+0000","last_clean":"2024-08-17T17:40:24.922655+0000","last_became_active":"2024-08-17T17:36:24.225250+0000","last_became_peered":"2024-08-17T17:36:24.225250+0000","last_unstale":"2024-08-17T17:40:24.922655+0000","last_undegraded":"2024-08-17T17:40:24.922655+0000","last_fullsized":"2024-08-17T17:40:24.922655+0000","mapping_epoch":1393,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1394,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.290656+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1452,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.934350+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:40:24.934350+0000","last_peered":"2024-08-17T17:40:24.934350+0000","last_clean":"2024-08-17T17:40:24.934350+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:40:24.934350+0000","last_undegraded":"2024-08-17T17:40:24.934350+0000","last_fullsized":"2024-08-17T17:40:24.934350+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1860,"reported_epoch":1431,"state":"active+clean","last_fresh":"2024-08-17T17:40:31.979777+0000","last_change":"2024-08-17T17:36:12.752970+0000","last_active":"2024-08-17T17:40:31.979777+0000","last_peered":"2024-08-17T17:40:31.979777+0000","last_clean":"2024-08-17T17:40:31.979777+0000","last_became_active":"2024-08-17T17:36:12.752311+0000","last_became_peered":"2024-08-17T17:36:12.752311+0000","last_unstale":"2024-08-17T17:40:31.979777+0000","last_undegraded":"2024-08-17T17:40:31.979777+0000","last_fullsized":"2024-08-17T17:40:31.979777+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:38.131419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1807,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.928653+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:40:24.928653+0000","last_peered":"2024-08-17T17:40:24.928653+0000","last_clean":"2024-08-17T17:40:24.928653+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:40:24.928653+0000","last_undegraded":"2024-08-17T17:40:24.928653+0000","last_fullsized":"2024-08-17T17:40:24.928653+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1528,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.982203+0000","last_change":"2024-08-17T17:36:18.750494+0000","last_active":"2024-08-17T17:40:32.982203+0000","last_peered":"2024-08-17T17:40:32.982203+0000","last_clean":"2024-08-17T17:40:32.982203+0000","last_became_active":"2024-08-17T17:36:18.750092+0000","last_became_peered":"2024-08-17T17:36:18.750092+0000","last_unstale":"2024-08-17T17:40:32.982203+0000","last_undegraded":"2024-08-17T17:40:32.982203+0000","last_fullsized":"2024-08-17T17:40:32.982203+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:36.599727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1153,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.930535+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:40:24.930535+0000","last_peered":"2024-08-17T17:40:24.930535+0000","last_clean":"2024-08-17T17:40:24.930535+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:40:24.930535+0000","last_undegraded":"2024-08-17T17:40:24.930535+0000","last_fullsized":"2024-08-17T17:40:24.930535+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":974,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926131+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:40:24.926131+0000","last_peered":"2024-08-17T17:40:24.926131+0000","last_clean":"2024-08-17T17:40:24.926131+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:40:24.926131+0000","last_undegraded":"2024-08-17T17:40:24.926131+0000","last_fullsized":"2024-08-17T17:40:24.926131+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":61,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.928786+0000","last_change":"2024-08-17T17:37:11.075505+0000","last_active":"2024-08-17T17:40:24.928786+0000","last_peered":"2024-08-17T17:40:24.928786+0000","last_clean":"2024-08-17T17:40:24.928786+0000","last_became_active":"2024-08-17T17:37:11.075131+0000","last_became_peered":"2024-08-17T17:37:11.075131+0000","last_unstale":"2024-08-17T17:40:24.928786+0000","last_undegraded":"2024-08-17T17:40:24.928786+0000","last_fullsized":"2024-08-17T17:40:24.928786+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:39.672054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1287,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.922213+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:40:24.922213+0000","last_peered":"2024-08-17T17:40:24.922213+0000","last_clean":"2024-08-17T17:40:24.922213+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:40:24.922213+0000","last_undegraded":"2024-08-17T17:40:24.922213+0000","last_fullsized":"2024-08-17T17:40:24.922213+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1664,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.934052+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:40:24.934052+0000","last_peered":"2024-08-17T17:40:24.934052+0000","last_clean":"2024-08-17T17:40:24.934052+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:40:24.934052+0000","last_undegraded":"2024-08-17T17:40:24.934052+0000","last_fullsized":"2024-08-17T17:40:24.934052+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":111,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.981563+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:40:32.981563+0000","last_peered":"2024-08-17T17:40:32.981563+0000","last_clean":"2024-08-17T17:40:32.981563+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:40:32.981563+0000","last_undegraded":"2024-08-17T17:40:32.981563+0000","last_fullsized":"2024-08-17T17:40:32.981563+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1261,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926695+0000","last_change":"2024-08-17T17:36:12.755873+0000","last_active":"2024-08-17T17:40:24.926695+0000","last_peered":"2024-08-17T17:40:24.926695+0000","last_clean":"2024-08-17T17:40:24.926695+0000","last_became_active":"2024-08-17T17:36:12.755366+0000","last_became_peered":"2024-08-17T17:36:12.755366+0000","last_unstale":"2024-08-17T17:40:24.926695+0000","last_undegraded":"2024-08-17T17:40:24.926695+0000","last_fullsized":"2024-08-17T17:40:24.926695+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:36.523214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1615,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.924618+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:40:24.924618+0000","last_peered":"2024-08-17T17:40:24.924618+0000","last_clean":"2024-08-17T17:40:24.924618+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:40:24.924618+0000","last_undegraded":"2024-08-17T17:40:24.924618+0000","last_fullsized":"2024-08-17T17:40:24.924618+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":798,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.922817+0000","last_change":"2024-08-17T17:35:55.200519+0000","last_active":"2024-08-17T17:40:24.922817+0000","last_peered":"2024-08-17T17:40:24.922817+0000","last_clean":"2024-08-17T17:40:24.922817+0000","last_became_active":"2024-08-17T17:35:55.200230+0000","last_became_peered":"2024-08-17T17:35:55.200230+0000","last_unstale":"2024-08-17T17:40:24.922817+0000","last_undegraded":"2024-08-17T17:40:24.922817+0000","last_fullsized":"2024-08-17T17:40:24.922817+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:33.454851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1438,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926410+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:40:24.926410+0000","last_peered":"2024-08-17T17:40:24.926410+0000","last_clean":"2024-08-17T17:40:24.926410+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:40:24.926410+0000","last_undegraded":"2024-08-17T17:40:24.926410+0000","last_fullsized":"2024-08-17T17:40:24.926410+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1725,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.927403+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:40:24.927403+0000","last_peered":"2024-08-17T17:40:24.927403+0000","last_clean":"2024-08-17T17:40:24.927403+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:40:24.927403+0000","last_undegraded":"2024-08-17T17:40:24.927403+0000","last_fullsized":"2024-08-17T17:40:24.927403+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:22.077229+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":276,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.982062+0000","last_change":"2024-08-17T17:35:33.145476+0000","last_active":"2024-08-17T17:40:32.982062+0000","last_peered":"2024-08-17T17:40:32.982062+0000","last_clean":"2024-08-17T17:40:32.982062+0000","last_became_active":"2024-08-17T17:35:33.145125+0000","last_became_peered":"2024-08-17T17:35:33.145125+0000","last_unstale":"2024-08-17T17:40:32.982062+0000","last_undegraded":"2024-08-17T17:40:32.982062+0000","last_fullsized":"2024-08-17T17:40:32.982062+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:45.298064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":942,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.921451+0000","last_change":"2024-08-17T17:35:56.248612+0000","last_active":"2024-08-17T17:40:24.921451+0000","last_peered":"2024-08-17T17:40:24.921451+0000","last_clean":"2024-08-17T17:40:24.921451+0000","last_became_active":"2024-08-17T17:35:56.244364+0000","last_became_peered":"2024-08-17T17:35:56.244364+0000","last_unstale":"2024-08-17T17:40:24.921451+0000","last_undegraded":"2024-08-17T17:40:24.921451+0000","last_fullsized":"2024-08-17T17:40:24.921451+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:41.064165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":882,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926494+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:40:24.926494+0000","last_peered":"2024-08-17T17:40:24.926494+0000","last_clean":"2024-08-17T17:40:24.926494+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:40:24.926494+0000","last_undegraded":"2024-08-17T17:40:24.926494+0000","last_fullsized":"2024-08-17T17:40:24.926494+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1418,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:27.452864+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:40:27.452864+0000","last_peered":"2024-08-17T17:40:27.452864+0000","last_clean":"2024-08-17T17:40:27.452864+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:40:27.452864+0000","last_undegraded":"2024-08-17T17:40:27.452864+0000","last_fullsized":"2024-08-17T17:40:27.452864+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1437,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:27.452824+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:40:27.452824+0000","last_peered":"2024-08-17T17:40:27.452824+0000","last_clean":"2024-08-17T17:40:27.452824+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:40:27.452824+0000","last_undegraded":"2024-08-17T17:40:27.452824+0000","last_fullsized":"2024-08-17T17:40:27.452824+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":948,"reported_epoch":1431,"state":"active+clean","last_fresh":"2024-08-17T17:40:31.979315+0000","last_change":"2024-08-17T17:36:15.774881+0000","last_active":"2024-08-17T17:40:31.979315+0000","last_peered":"2024-08-17T17:40:31.979315+0000","last_clean":"2024-08-17T17:40:31.979315+0000","last_became_active":"2024-08-17T17:36:15.774435+0000","last_became_peered":"2024-08-17T17:36:15.774435+0000","last_unstale":"2024-08-17T17:40:31.979315+0000","last_undegraded":"2024-08-17T17:40:31.979315+0000","last_fullsized":"2024-08-17T17:40:31.979315+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:52.015854+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"1431'8065","reported_seq":16252,"reported_epoch":1431,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.185799+0000","last_change":"2024-08-17T17:38:37.110784+0000","last_active":"2024-08-17T17:40:32.185799+0000","last_peered":"2024-08-17T17:40:32.185799+0000","last_clean":"2024-08-17T17:40:32.185799+0000","last_became_active":"2024-08-17T17:38:37.088380+0000","last_became_peered":"2024-08-17T17:38:37.088380+0000","last_unstale":"2024-08-17T17:40:32.185799+0000","last_undegraded":"2024-08-17T17:40:32.185799+0000","last_fullsized":"2024-08-17T17:40:32.185799+0000","mapping_epoch":1419,"log_start":"1374'2000","ondisk_log_start":"1374'2000","created":1357,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6065,"log_dups_size":0,"ondisk_log_size":6065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:08.647903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14483456,"num_objects":221,"num_object_clones":0,"num_object_copies":663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8065,"num_write_kb":265152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4009,"num_bytes_recovered":199753728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1385,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.933958+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:40:24.933958+0000","last_peered":"2024-08-17T17:40:24.933958+0000","last_clean":"2024-08-17T17:40:24.933958+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:40:24.933958+0000","last_undegraded":"2024-08-17T17:40:24.933958+0000","last_fullsized":"2024-08-17T17:40:24.933958+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1281,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.922064+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:40:24.922064+0000","last_peered":"2024-08-17T17:40:24.922064+0000","last_clean":"2024-08-17T17:40:24.922064+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:40:24.922064+0000","last_undegraded":"2024-08-17T17:40:24.922064+0000","last_fullsized":"2024-08-17T17:40:24.922064+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":339,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.930700+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:40:24.930700+0000","last_peered":"2024-08-17T17:40:24.930700+0000","last_clean":"2024-08-17T17:40:24.930700+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:40:24.930700+0000","last_undegraded":"2024-08-17T17:40:24.930700+0000","last_fullsized":"2024-08-17T17:40:24.930700+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1612,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.982287+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:40:32.982287+0000","last_peered":"2024-08-17T17:40:32.982287+0000","last_clean":"2024-08-17T17:40:32.982287+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:40:32.982287+0000","last_undegraded":"2024-08-17T17:40:32.982287+0000","last_fullsized":"2024-08-17T17:40:32.982287+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:25.751871+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.f","version":"1430'7891","reported_seq":8056,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:30.662949+0000","last_change":"2024-08-17T17:35:34.179168+0000","last_active":"2024-08-17T17:40:30.662949+0000","last_peered":"2024-08-17T17:40:30.662949+0000","last_clean":"2024-08-17T17:40:30.662949+0000","last_became_active":"2024-08-17T17:35:34.133493+0000","last_became_peered":"2024-08-17T17:35:34.133493+0000","last_unstale":"2024-08-17T17:40:30.662949+0000","last_undegraded":"2024-08-17T17:40:30.662949+0000","last_fullsized":"2024-08-17T17:40:30.662949+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":7891,"log_dups_size":0,"ondisk_log_size":7891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:24:11.188566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20512768,"num_objects":313,"num_object_clones":0,"num_object_copies":939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7891,"num_write_kb":262528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":119,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.981602+0000","last_change":"2024-08-17T17:37:11.079712+0000","last_active":"2024-08-17T17:40:32.981602+0000","last_peered":"2024-08-17T17:40:32.981602+0000","last_clean":"2024-08-17T17:40:32.981602+0000","last_became_active":"2024-08-17T17:37:11.079253+0000","last_became_peered":"2024-08-17T17:37:11.079253+0000","last_unstale":"2024-08-17T17:40:32.981602+0000","last_undegraded":"2024-08-17T17:40:32.981602+0000","last_fullsized":"2024-08-17T17:40:32.981602+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:53.405657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":926,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.921717+0000","last_change":"2024-08-17T17:35:56.223396+0000","last_active":"2024-08-17T17:40:24.921717+0000","last_peered":"2024-08-17T17:40:24.921717+0000","last_clean":"2024-08-17T17:40:24.921717+0000","last_became_active":"2024-08-17T17:35:56.212413+0000","last_became_peered":"2024-08-17T17:35:56.212413+0000","last_unstale":"2024-08-17T17:40:24.921717+0000","last_undegraded":"2024-08-17T17:40:24.921717+0000","last_fullsized":"2024-08-17T17:40:24.921717+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:44.185174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1649,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.929529+0000","last_change":"2024-08-17T17:35:33.136068+0000","last_active":"2024-08-17T17:40:24.929529+0000","last_peered":"2024-08-17T17:40:24.929529+0000","last_clean":"2024-08-17T17:40:24.929529+0000","last_became_active":"2024-08-17T17:35:33.135591+0000","last_became_peered":"2024-08-17T17:35:33.135591+0000","last_unstale":"2024-08-17T17:40:24.929529+0000","last_undegraded":"2024-08-17T17:40:24.929529+0000","last_fullsized":"2024-08-17T17:40:24.929529+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:52.252561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1791,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.936745+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:40:24.936745+0000","last_peered":"2024-08-17T17:40:24.936745+0000","last_clean":"2024-08-17T17:40:24.936745+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:40:24.936745+0000","last_undegraded":"2024-08-17T17:40:24.936745+0000","last_fullsized":"2024-08-17T17:40:24.936745+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"1430'7744","reported_seq":7894,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:30.693710+0000","last_change":"2024-08-17T17:35:14.861823+0000","last_active":"2024-08-17T17:40:30.693710+0000","last_peered":"2024-08-17T17:40:30.693710+0000","last_clean":"2024-08-17T17:40:30.693710+0000","last_became_active":"2024-08-17T17:35:14.854109+0000","last_became_peered":"2024-08-17T17:35:14.854109+0000","last_unstale":"2024-08-17T17:40:30.693710+0000","last_undegraded":"2024-08-17T17:40:30.693710+0000","last_fullsized":"2024-08-17T17:40:30.693710+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":7744,"log_dups_size":0,"ondisk_log_size":7744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:46:40.778197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19791872,"num_objects":302,"num_object_clones":0,"num_object_copies":906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7744,"num_write_kb":257472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1004,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926208+0000","last_change":"2024-08-17T17:35:33.138467+0000","last_active":"2024-08-17T17:40:24.926208+0000","last_peered":"2024-08-17T17:40:24.926208+0000","last_clean":"2024-08-17T17:40:24.926208+0000","last_became_active":"2024-08-17T17:35:33.138174+0000","last_became_peered":"2024-08-17T17:35:33.138174+0000","last_unstale":"2024-08-17T17:40:24.926208+0000","last_undegraded":"2024-08-17T17:40:24.926208+0000","last_fullsized":"2024-08-17T17:40:24.926208+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:08.860439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1388,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.924710+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:40:24.924710+0000","last_peered":"2024-08-17T17:40:24.924710+0000","last_clean":"2024-08-17T17:40:24.924710+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:40:24.924710+0000","last_undegraded":"2024-08-17T17:40:24.924710+0000","last_fullsized":"2024-08-17T17:40:24.924710+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":145,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.927762+0000","last_change":"2024-08-17T17:35:14.854881+0000","last_active":"2024-08-17T17:40:24.927762+0000","last_peered":"2024-08-17T17:40:24.927762+0000","last_clean":"2024-08-17T17:40:24.927762+0000","last_became_active":"2024-08-17T17:35:14.854121+0000","last_became_peered":"2024-08-17T17:35:14.854121+0000","last_unstale":"2024-08-17T17:40:24.927762+0000","last_undegraded":"2024-08-17T17:40:24.927762+0000","last_fullsized":"2024-08-17T17:40:24.927762+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.863624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":945,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.921854+0000","last_change":"2024-08-17T17:36:15.773050+0000","last_active":"2024-08-17T17:40:24.921854+0000","last_peered":"2024-08-17T17:40:24.921854+0000","last_clean":"2024-08-17T17:40:24.921854+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:40:24.921854+0000","last_undegraded":"2024-08-17T17:40:24.921854+0000","last_fullsized":"2024-08-17T17:40:24.921854+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:39.212195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:39.212195+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"1430'7737","reported_seq":13593,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:30.702202+0000","last_change":"2024-08-17T17:37:21.409979+0000","last_active":"2024-08-17T17:40:30.702202+0000","last_peered":"2024-08-17T17:40:30.702202+0000","last_clean":"2024-08-17T17:40:30.702202+0000","last_became_active":"2024-08-17T17:37:21.398111+0000","last_became_peered":"2024-08-17T17:37:21.398111+0000","last_unstale":"2024-08-17T17:40:30.702202+0000","last_undegraded":"2024-08-17T17:40:30.702202+0000","last_fullsized":"2024-08-17T17:40:30.702202+0000","mapping_epoch":1407,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":7737,"log_dups_size":0,"ondisk_log_size":7737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:19.299395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19267608,"num_objects":295,"num_object_clones":0,"num_object_copies":885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":295,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7737,"num_write_kb":256961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2850,"num_bytes_recovered":168820736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1046,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926257+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:40:24.926257+0000","last_peered":"2024-08-17T17:40:24.926257+0000","last_clean":"2024-08-17T17:40:24.926257+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:40:24.926257+0000","last_undegraded":"2024-08-17T17:40:24.926257+0000","last_fullsized":"2024-08-17T17:40:24.926257+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":241,"reported_epoch":1431,"state":"active+clean","last_fresh":"2024-08-17T17:40:31.979112+0000","last_change":"2024-08-17T17:35:33.145351+0000","last_active":"2024-08-17T17:40:31.979112+0000","last_peered":"2024-08-17T17:40:31.979112+0000","last_clean":"2024-08-17T17:40:31.979112+0000","last_became_active":"2024-08-17T17:35:33.145040+0000","last_became_peered":"2024-08-17T17:35:33.145040+0000","last_unstale":"2024-08-17T17:40:31.979112+0000","last_undegraded":"2024-08-17T17:40:31.979112+0000","last_fullsized":"2024-08-17T17:40:31.979112+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:02.124000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1456,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926733+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:40:24.926733+0000","last_peered":"2024-08-17T17:40:24.926733+0000","last_clean":"2024-08-17T17:40:24.926733+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:40:24.926733+0000","last_undegraded":"2024-08-17T17:40:24.926733+0000","last_fullsized":"2024-08-17T17:40:24.926733+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":928,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.921991+0000","last_change":"2024-08-17T17:36:15.769890+0000","last_active":"2024-08-17T17:40:24.921991+0000","last_peered":"2024-08-17T17:40:24.921991+0000","last_clean":"2024-08-17T17:40:24.921991+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:40:24.921991+0000","last_undegraded":"2024-08-17T17:40:24.921991+0000","last_fullsized":"2024-08-17T17:40:24.921991+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:33.034306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:33.034306+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":4,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.b","version":"1432'7758","reported_seq":15577,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.735669+0000","last_change":"2024-08-17T17:38:07.888612+0000","last_active":"2024-08-17T17:40:33.735669+0000","last_peered":"2024-08-17T17:40:33.735669+0000","last_clean":"2024-08-17T17:40:33.735669+0000","last_became_active":"2024-08-17T17:38:07.875196+0000","last_became_peered":"2024-08-17T17:38:07.875196+0000","last_unstale":"2024-08-17T17:40:33.735669+0000","last_undegraded":"2024-08-17T17:40:33.735669+0000","last_fullsized":"2024-08-17T17:40:33.735669+0000","mapping_epoch":1415,"log_start":"1374'1800","ondisk_log_start":"1374'1800","created":1357,"last_epoch_clean":1416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5958,"log_dups_size":0,"ondisk_log_size":5958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:21.658004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12845056,"num_objects":196,"num_object_clones":0,"num_object_copies":588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7758,"num_write_kb":254528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3836,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1500,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.933246+0000","last_change":"2024-08-17T17:36:12.759743+0000","last_active":"2024-08-17T17:40:24.933246+0000","last_peered":"2024-08-17T17:40:24.933246+0000","last_clean":"2024-08-17T17:40:24.933246+0000","last_became_active":"2024-08-17T17:36:12.759334+0000","last_became_peered":"2024-08-17T17:36:12.759334+0000","last_unstale":"2024-08-17T17:40:24.933246+0000","last_undegraded":"2024-08-17T17:40:24.933246+0000","last_fullsized":"2024-08-17T17:40:24.933246+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:04.524384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1419,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.934833+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:40:24.934833+0000","last_peered":"2024-08-17T17:40:24.934833+0000","last_clean":"2024-08-17T17:40:24.934833+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:40:24.934833+0000","last_undegraded":"2024-08-17T17:40:24.934833+0000","last_fullsized":"2024-08-17T17:40:24.934833+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1462,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.927852+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:40:24.927852+0000","last_peered":"2024-08-17T17:40:24.927852+0000","last_clean":"2024-08-17T17:40:24.927852+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:40:24.927852+0000","last_undegraded":"2024-08-17T17:40:24.927852+0000","last_fullsized":"2024-08-17T17:40:24.927852+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1777,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926083+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:40:24.926083+0000","last_peered":"2024-08-17T17:40:24.926083+0000","last_clean":"2024-08-17T17:40:24.926083+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:40:24.926083+0000","last_undegraded":"2024-08-17T17:40:24.926083+0000","last_fullsized":"2024-08-17T17:40:24.926083+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"1430'7939","reported_seq":18865,"reported_epoch":1431,"state":"peering","last_fresh":"2024-08-17T17:40:31.978876+0000","last_change":"2024-08-17T17:40:31.978876+0000","last_active":"2024-08-17T17:40:31.978571+0000","last_peered":"2024-08-17T17:40:31.967956+0000","last_clean":"2024-08-17T17:35:54.187688+0000","last_became_active":"2024-08-17T17:36:22.904523+0000","last_became_peered":"2024-08-17T17:36:22.904523+0000","last_unstale":"2024-08-17T17:40:31.978876+0000","last_undegraded":"2024-08-17T17:40:31.978876+0000","last_fullsized":"2024-08-17T17:40:31.978876+0000","mapping_epoch":1431,"log_start":"1374'2100","ondisk_log_start":"1374'2100","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5839,"log_dups_size":0,"ondisk_log_size":5839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:09.043902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16318464,"num_objects":249,"num_object_clones":0,"num_object_copies":747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7939,"num_write_kb":262016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3968,"num_bytes_recovered":258998272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1095,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926567+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:40:24.926567+0000","last_peered":"2024-08-17T17:40:24.926567+0000","last_clean":"2024-08-17T17:40:24.926567+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:40:24.926567+0000","last_undegraded":"2024-08-17T17:40:24.926567+0000","last_fullsized":"2024-08-17T17:40:24.926567+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1412,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926808+0000","last_change":"2024-08-17T17:35:56.229622+0000","last_active":"2024-08-17T17:40:24.926808+0000","last_peered":"2024-08-17T17:40:24.926808+0000","last_clean":"2024-08-17T17:40:24.926808+0000","last_became_active":"2024-08-17T17:35:56.229324+0000","last_became_peered":"2024-08-17T17:35:56.229324+0000","last_unstale":"2024-08-17T17:40:24.926808+0000","last_undegraded":"2024-08-17T17:40:24.926808+0000","last_fullsized":"2024-08-17T17:40:24.926808+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:13.980316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2015,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.925691+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:40:24.925691+0000","last_peered":"2024-08-17T17:40:24.925691+0000","last_clean":"2024-08-17T17:40:24.925691+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:40:24.925691+0000","last_undegraded":"2024-08-17T17:40:24.925691+0000","last_fullsized":"2024-08-17T17:40:24.925691+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1591,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.934159+0000","last_change":"2024-08-17T17:35:33.137074+0000","last_active":"2024-08-17T17:40:24.934159+0000","last_peered":"2024-08-17T17:40:24.934159+0000","last_clean":"2024-08-17T17:40:24.934159+0000","last_became_active":"2024-08-17T17:35:33.136759+0000","last_became_peered":"2024-08-17T17:35:33.136759+0000","last_unstale":"2024-08-17T17:40:24.934159+0000","last_undegraded":"2024-08-17T17:40:24.934159+0000","last_fullsized":"2024-08-17T17:40:24.934159+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:37.263639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:37.263639+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:14.599780+0000","scrub_duration":0.0016181800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"1430'7811","reported_seq":15862,"reported_epoch":1430,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:30.683299+0000","last_change":"2024-08-17T17:36:23.019129+0000","last_active":"2024-08-17T17:40:30.683299+0000","last_peered":"2024-08-17T17:40:30.683299+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:23.013527+0000","last_became_peered":"2024-08-17T17:36:23.013527+0000","last_unstale":"2024-08-17T17:40:30.683299+0000","last_undegraded":"2024-08-17T17:36:13.874715+0000","last_fullsized":"2024-08-17T17:36:13.738828+0000","mapping_epoch":1383,"log_start":"1373'800","ondisk_log_start":"1373'800","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":7011,"log_dups_size":0,"ondisk_log_size":7011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18939904,"num_objects":289,"num_object_clones":0,"num_object_copies":867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1785,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7811,"num_write_kb":259200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2876,"num_bytes_recovered":188481536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[0,6],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":162},{"shards":"0,6","objects":-1496}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1204,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.930854+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:40:24.930854+0000","last_peered":"2024-08-17T17:40:24.930854+0000","last_clean":"2024-08-17T17:40:24.930854+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:40:24.930854+0000","last_undegraded":"2024-08-17T17:40:24.930854+0000","last_fullsized":"2024-08-17T17:40:24.930854+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":56,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:27.452720+0000","last_change":"2024-08-17T17:36:17.774773+0000","last_active":"2024-08-17T17:40:27.452720+0000","last_peered":"2024-08-17T17:40:27.452720+0000","last_clean":"2024-08-17T17:40:27.452720+0000","last_became_active":"2024-08-17T17:36:17.774468+0000","last_became_peered":"2024-08-17T17:36:17.774468+0000","last_unstale":"2024-08-17T17:40:27.452720+0000","last_undegraded":"2024-08-17T17:40:27.452720+0000","last_fullsized":"2024-08-17T17:40:27.452720+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:54.231747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1542,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.928703+0000","last_change":"2024-08-17T17:35:33.137633+0000","last_active":"2024-08-17T17:40:24.928703+0000","last_peered":"2024-08-17T17:40:24.928703+0000","last_clean":"2024-08-17T17:40:24.928703+0000","last_became_active":"2024-08-17T17:35:33.137323+0000","last_became_peered":"2024-08-17T17:35:33.137323+0000","last_unstale":"2024-08-17T17:40:24.928703+0000","last_undegraded":"2024-08-17T17:40:24.928703+0000","last_fullsized":"2024-08-17T17:40:24.928703+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:03.359155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":907,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.921518+0000","last_change":"2024-08-17T17:35:56.264626+0000","last_active":"2024-08-17T17:40:24.921518+0000","last_peered":"2024-08-17T17:40:24.921518+0000","last_clean":"2024-08-17T17:40:24.921518+0000","last_became_active":"2024-08-17T17:35:56.264265+0000","last_became_peered":"2024-08-17T17:35:56.264265+0000","last_unstale":"2024-08-17T17:40:24.921518+0000","last_undegraded":"2024-08-17T17:40:24.921518+0000","last_fullsized":"2024-08-17T17:40:24.921518+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:20.754326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:20.754326+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"1431'7776","reported_seq":16317,"reported_epoch":1431,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:32.136440+0000","last_change":"2024-08-17T17:37:11.134558+0000","last_active":"2024-08-17T17:40:32.136440+0000","last_peered":"2024-08-17T17:40:32.136440+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:20.105110+0000","last_became_peered":"2024-08-17T17:36:20.105110+0000","last_unstale":"2024-08-17T17:40:32.136440+0000","last_undegraded":"2024-08-17T17:36:19.999272+0000","last_fullsized":"2024-08-17T17:36:13.740520+0000","mapping_epoch":1383,"log_start":"1374'2000","ondisk_log_start":"1374'2000","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5776,"log_dups_size":0,"ondisk_log_size":5776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:45.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15597568,"num_objects":238,"num_object_clones":0,"num_object_copies":714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1036,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7776,"num_write_kb":256448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2817,"num_bytes_recovered":184614912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"1,5,6","objects":84},{"shards":"5,6","objects":-798}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1260,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.923119+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:40:24.923119+0000","last_peered":"2024-08-17T17:40:24.923119+0000","last_clean":"2024-08-17T17:40:24.923119+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:40:24.923119+0000","last_undegraded":"2024-08-17T17:40:24.923119+0000","last_fullsized":"2024-08-17T17:40:24.923119+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1799,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.927801+0000","last_change":"2024-08-17T17:36:12.758005+0000","last_active":"2024-08-17T17:40:24.927801+0000","last_peered":"2024-08-17T17:40:24.927801+0000","last_clean":"2024-08-17T17:40:24.927801+0000","last_became_active":"2024-08-17T17:36:12.757695+0000","last_became_peered":"2024-08-17T17:36:12.757695+0000","last_unstale":"2024-08-17T17:40:24.927801+0000","last_undegraded":"2024-08-17T17:40:24.927801+0000","last_fullsized":"2024-08-17T17:40:24.927801+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:51.792220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1438,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926255+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:40:24.926255+0000","last_peered":"2024-08-17T17:40:24.926255+0000","last_clean":"2024-08-17T17:40:24.926255+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:40:24.926255+0000","last_undegraded":"2024-08-17T17:40:24.926255+0000","last_fullsized":"2024-08-17T17:40:24.926255+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1761,"reported_epoch":1430,"state":"active+clean+scrubbing","last_fresh":"2024-08-17T17:40:24.928900+0000","last_change":"2024-08-17T17:40:30.416217+0000","last_active":"2024-08-17T17:40:24.928900+0000","last_peered":"2024-08-17T17:40:24.928900+0000","last_clean":"2024-08-17T17:40:24.928900+0000","last_became_active":"2024-08-17T17:35:55.204861+0000","last_became_peered":"2024-08-17T17:35:55.204861+0000","last_unstale":"2024-08-17T17:40:24.928900+0000","last_undegraded":"2024-08-17T17:40:24.928900+0000","last_fullsized":"2024-08-17T17:40:24.928900+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.0","version":"1431'7873","reported_seq":15879,"reported_epoch":1431,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.096053+0000","last_change":"2024-08-17T17:37:40.753703+0000","last_active":"2024-08-17T17:40:32.096053+0000","last_peered":"2024-08-17T17:40:32.096053+0000","last_clean":"2024-08-17T17:40:32.096053+0000","last_became_active":"2024-08-17T17:37:40.743458+0000","last_became_peered":"2024-08-17T17:37:40.743458+0000","last_unstale":"2024-08-17T17:40:32.096053+0000","last_undegraded":"2024-08-17T17:40:32.096053+0000","last_fullsized":"2024-08-17T17:40:32.096053+0000","mapping_epoch":1411,"log_start":"1374'2400","ondisk_log_start":"1374'2400","created":1357,"last_epoch_clean":1412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5473,"log_dups_size":0,"ondisk_log_size":5473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:14:02.340380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15925248,"num_objects":243,"num_object_clones":0,"num_object_copies":729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7873,"num_write_kb":259712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3914,"num_bytes_recovered":232652800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1033,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.934461+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:40:24.934461+0000","last_peered":"2024-08-17T17:40:24.934461+0000","last_clean":"2024-08-17T17:40:24.934461+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:40:24.934461+0000","last_undegraded":"2024-08-17T17:40:24.934461+0000","last_fullsized":"2024-08-17T17:40:24.934461+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":111,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.981699+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:40:32.981699+0000","last_peered":"2024-08-17T17:40:32.981699+0000","last_clean":"2024-08-17T17:40:32.981699+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:40:32.981699+0000","last_undegraded":"2024-08-17T17:40:32.981699+0000","last_fullsized":"2024-08-17T17:40:32.981699+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1769,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926094+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:40:24.926094+0000","last_peered":"2024-08-17T17:40:24.926094+0000","last_clean":"2024-08-17T17:40:24.926094+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:40:24.926094+0000","last_undegraded":"2024-08-17T17:40:24.926094+0000","last_fullsized":"2024-08-17T17:40:24.926094+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1","version":"1431'7809","reported_seq":16406,"reported_epoch":1431,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:32.087422+0000","last_change":"2024-08-17T17:36:23.103523+0000","last_active":"2024-08-17T17:40:32.087422+0000","last_peered":"2024-08-17T17:40:32.087422+0000","last_clean":"2024-08-17T17:35:54.178280+0000","last_became_active":"2024-08-17T17:36:23.025910+0000","last_became_peered":"2024-08-17T17:36:23.025910+0000","last_unstale":"2024-08-17T17:40:32.087422+0000","last_undegraded":"2024-08-17T17:36:18.734637+0000","last_fullsized":"2024-08-17T17:36:13.740134+0000","mapping_epoch":1383,"log_start":"1374'2000","ondisk_log_start":"1374'2000","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5809,"log_dups_size":0,"ondisk_log_size":5809,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:58.717055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15925248,"num_objects":243,"num_object_clones":0,"num_object_copies":729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1071,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7809,"num_write_kb":257664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2833,"num_bytes_recovered":185663488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,0],"avail_no_missing":["6","0","2","4"],"object_location_counts":[{"shards":"0,2,6","objects":119},{"shards":"0,6","objects":-828}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1447,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.934596+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:40:24.934596+0000","last_peered":"2024-08-17T17:40:24.934596+0000","last_clean":"2024-08-17T17:40:24.934596+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:40:24.934596+0000","last_undegraded":"2024-08-17T17:40:24.934596+0000","last_fullsized":"2024-08-17T17:40:24.934596+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1420,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.934188+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:40:24.934188+0000","last_peered":"2024-08-17T17:40:24.934188+0000","last_clean":"2024-08-17T17:40:24.934188+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:40:24.934188+0000","last_undegraded":"2024-08-17T17:40:24.934188+0000","last_fullsized":"2024-08-17T17:40:24.934188+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":1009,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.932216+0000","last_change":"2024-08-17T17:34:26.817339+0000","last_active":"2024-08-17T17:40:24.932216+0000","last_peered":"2024-08-17T17:40:24.932216+0000","last_clean":"2024-08-17T17:40:24.932216+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:40:24.932216+0000","last_undegraded":"2024-08-17T17:40:24.932216+0000","last_fullsized":"2024-08-17T17:40:24.932216+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:26.817223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:26.817223+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:30.089956+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"1430'7595","reported_seq":7770,"reported_epoch":1430,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:30.665224+0000","last_change":"2024-08-17T17:36:14.073155+0000","last_active":"2024-08-17T17:40:30.665224+0000","last_peered":"2024-08-17T17:40:30.665224+0000","last_clean":"2024-08-17T17:36:11.734683+0000","last_became_active":"2024-08-17T17:36:14.063794+0000","last_became_peered":"2024-08-17T17:36:14.063794+0000","last_unstale":"2024-08-17T17:40:30.665224+0000","last_undegraded":"2024-08-17T17:36:13.855770+0000","last_fullsized":"2024-08-17T17:36:13.742068+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":7595,"log_dups_size":0,"ondisk_log_size":7595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20119552,"num_objects":307,"num_object_clones":0,"num_object_copies":921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3819,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":307,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7595,"num_write_kb":252864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":720896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-3512},{"shards":"0,2,7","objects":175}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":55,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.936624+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:40:24.936624+0000","last_peered":"2024-08-17T17:40:24.936624+0000","last_clean":"2024-08-17T17:40:24.936624+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:40:24.936624+0000","last_undegraded":"2024-08-17T17:40:24.936624+0000","last_fullsized":"2024-08-17T17:40:24.936624+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1282,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.923355+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:40:24.923355+0000","last_peered":"2024-08-17T17:40:24.923355+0000","last_clean":"2024-08-17T17:40:24.923355+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:40:24.923355+0000","last_undegraded":"2024-08-17T17:40:24.923355+0000","last_fullsized":"2024-08-17T17:40:24.923355+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1797,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.928568+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:40:24.928568+0000","last_peered":"2024-08-17T17:40:24.928568+0000","last_clean":"2024-08-17T17:40:24.928568+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:40:24.928568+0000","last_undegraded":"2024-08-17T17:40:24.928568+0000","last_fullsized":"2024-08-17T17:40:24.928568+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"1431'7824","reported_seq":13630,"reported_epoch":1431,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.081191+0000","last_change":"2024-08-17T17:36:46.982456+0000","last_active":"2024-08-17T17:40:32.081191+0000","last_peered":"2024-08-17T17:40:32.081191+0000","last_clean":"2024-08-17T17:40:32.081191+0000","last_became_active":"2024-08-17T17:36:46.974782+0000","last_became_peered":"2024-08-17T17:36:46.974782+0000","last_unstale":"2024-08-17T17:40:32.081191+0000","last_undegraded":"2024-08-17T17:40:32.081191+0000","last_fullsized":"2024-08-17T17:40:32.081191+0000","mapping_epoch":1399,"log_start":"1374'2000","ondisk_log_start":"1374'2000","created":1357,"last_epoch_clean":1400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:40.499321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16252928,"num_objects":248,"num_object_clones":0,"num_object_copies":744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7824,"num_write_kb":258304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2848,"num_bytes_recovered":186646528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1398,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.937435+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:40:24.937435+0000","last_peered":"2024-08-17T17:40:24.937435+0000","last_clean":"2024-08-17T17:40:24.937435+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:40:24.937435+0000","last_undegraded":"2024-08-17T17:40:24.937435+0000","last_fullsized":"2024-08-17T17:40:24.937435+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":212,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.934521+0000","last_change":"2024-08-17T17:36:12.754372+0000","last_active":"2024-08-17T17:40:24.934521+0000","last_peered":"2024-08-17T17:40:24.934521+0000","last_clean":"2024-08-17T17:40:24.934521+0000","last_became_active":"2024-08-17T17:36:12.746352+0000","last_became_peered":"2024-08-17T17:36:12.746352+0000","last_unstale":"2024-08-17T17:40:24.934521+0000","last_undegraded":"2024-08-17T17:40:24.934521+0000","last_fullsized":"2024-08-17T17:40:24.934521+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:07.807486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1681,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.925521+0000","last_change":"2024-08-17T17:36:15.771193+0000","last_active":"2024-08-17T17:40:24.925521+0000","last_peered":"2024-08-17T17:40:24.925521+0000","last_clean":"2024-08-17T17:40:24.925521+0000","last_became_active":"2024-08-17T17:36:15.770889+0000","last_became_peered":"2024-08-17T17:36:15.770889+0000","last_unstale":"2024-08-17T17:40:24.925521+0000","last_undegraded":"2024-08-17T17:40:24.925521+0000","last_fullsized":"2024-08-17T17:40:24.925521+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:22.978695+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.4","version":"1431'7840","reported_seq":13627,"reported_epoch":1431,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.140036+0000","last_change":"2024-08-17T17:37:04.045576+0000","last_active":"2024-08-17T17:40:32.140036+0000","last_peered":"2024-08-17T17:40:32.140036+0000","last_clean":"2024-08-17T17:40:32.140036+0000","last_became_active":"2024-08-17T17:37:04.038810+0000","last_became_peered":"2024-08-17T17:37:04.038810+0000","last_unstale":"2024-08-17T17:40:32.140036+0000","last_undegraded":"2024-08-17T17:40:32.140036+0000","last_fullsized":"2024-08-17T17:40:32.140036+0000","mapping_epoch":1402,"log_start":"1374'2000","ondisk_log_start":"1374'2000","created":1357,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5840,"log_dups_size":0,"ondisk_log_size":5840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:27:04.725258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14811136,"num_objects":226,"num_object_clones":0,"num_object_copies":678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7840,"num_write_kb":258112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":176291840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1276,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.936040+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:40:24.936040+0000","last_peered":"2024-08-17T17:40:24.936040+0000","last_clean":"2024-08-17T17:40:24.936040+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:40:24.936040+0000","last_undegraded":"2024-08-17T17:40:24.936040+0000","last_fullsized":"2024-08-17T17:40:24.936040+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1424,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926435+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:40:24.926435+0000","last_peered":"2024-08-17T17:40:24.926435+0000","last_clean":"2024-08-17T17:40:24.926435+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:40:24.926435+0000","last_undegraded":"2024-08-17T17:40:24.926435+0000","last_fullsized":"2024-08-17T17:40:24.926435+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":925,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.934381+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:40:24.934381+0000","last_peered":"2024-08-17T17:40:24.934381+0000","last_clean":"2024-08-17T17:40:24.934381+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:40:24.934381+0000","last_undegraded":"2024-08-17T17:40:24.934381+0000","last_fullsized":"2024-08-17T17:40:24.934381+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"1431'7860","reported_seq":13702,"reported_epoch":1431,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.138406+0000","last_change":"2024-08-17T17:36:32.711490+0000","last_active":"2024-08-17T17:40:32.138406+0000","last_peered":"2024-08-17T17:40:32.138406+0000","last_clean":"2024-08-17T17:40:32.138406+0000","last_became_active":"2024-08-17T17:36:32.704370+0000","last_became_peered":"2024-08-17T17:36:32.704370+0000","last_unstale":"2024-08-17T17:40:32.138406+0000","last_undegraded":"2024-08-17T17:40:32.138406+0000","last_fullsized":"2024-08-17T17:40:32.138406+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":7860,"log_dups_size":0,"ondisk_log_size":7860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:17.373879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16252928,"num_objects":248,"num_object_clones":0,"num_object_copies":744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7860,"num_write_kb":259456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2866,"num_bytes_recovered":187826176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1281,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.922617+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:40:24.922617+0000","last_peered":"2024-08-17T17:40:24.922617+0000","last_clean":"2024-08-17T17:40:24.922617+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:40:24.922617+0000","last_undegraded":"2024-08-17T17:40:24.922617+0000","last_fullsized":"2024-08-17T17:40:24.922617+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":161,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926497+0000","last_change":"2024-08-17T17:35:55.217072+0000","last_active":"2024-08-17T17:40:24.926497+0000","last_peered":"2024-08-17T17:40:24.926497+0000","last_clean":"2024-08-17T17:40:24.926497+0000","last_became_active":"2024-08-17T17:35:55.216462+0000","last_became_peered":"2024-08-17T17:35:55.216462+0000","last_unstale":"2024-08-17T17:40:24.926497+0000","last_undegraded":"2024-08-17T17:40:24.926497+0000","last_fullsized":"2024-08-17T17:40:24.926497+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:34.937221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":908,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:27.762080+0000","last_change":"2024-08-17T17:35:55.215004+0000","last_active":"2024-08-17T17:40:27.762080+0000","last_peered":"2024-08-17T17:40:27.762080+0000","last_clean":"2024-08-17T17:40:27.762080+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:40:27.762080+0000","last_undegraded":"2024-08-17T17:40:27.762080+0000","last_fullsized":"2024-08-17T17:40:27.762080+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.6","version":"1430'7544","reported_seq":7695,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:30.645745+0000","last_change":"2024-08-17T17:35:14.868524+0000","last_active":"2024-08-17T17:40:30.645745+0000","last_peered":"2024-08-17T17:40:30.645745+0000","last_clean":"2024-08-17T17:40:30.645745+0000","last_became_active":"2024-08-17T17:35:14.854690+0000","last_became_peered":"2024-08-17T17:35:14.854690+0000","last_unstale":"2024-08-17T17:40:30.645745+0000","last_undegraded":"2024-08-17T17:40:30.645745+0000","last_fullsized":"2024-08-17T17:40:30.645745+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":7544,"log_dups_size":0,"ondisk_log_size":7544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:05.388362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17825792,"num_objects":272,"num_object_clones":0,"num_object_copies":816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7544,"num_write_kb":250112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1466,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.934119+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:40:24.934119+0000","last_peered":"2024-08-17T17:40:24.934119+0000","last_clean":"2024-08-17T17:40:24.934119+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:40:24.934119+0000","last_undegraded":"2024-08-17T17:40:24.934119+0000","last_fullsized":"2024-08-17T17:40:24.934119+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":173,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.934668+0000","last_change":"2024-08-17T17:35:33.147672+0000","last_active":"2024-08-17T17:40:24.934668+0000","last_peered":"2024-08-17T17:40:24.934668+0000","last_clean":"2024-08-17T17:40:24.934668+0000","last_became_active":"2024-08-17T17:35:33.147283+0000","last_became_peered":"2024-08-17T17:35:33.147283+0000","last_unstale":"2024-08-17T17:40:24.934668+0000","last_undegraded":"2024-08-17T17:40:24.934668+0000","last_fullsized":"2024-08-17T17:40:24.934668+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:41.677709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1699,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.982261+0000","last_change":"2024-08-17T17:40:31.978943+0000","last_active":"2024-08-17T17:40:32.982261+0000","last_peered":"2024-08-17T17:40:32.982261+0000","last_clean":"2024-08-17T17:40:32.982261+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:40:32.982261+0000","last_undegraded":"2024-08-17T17:40:32.982261+0000","last_fullsized":"2024-08-17T17:40:32.982261+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:38.241235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:38.241235+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:40:34.271280+0000","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"1431'7706","reported_seq":7790,"reported_epoch":1431,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.148986+0000","last_change":"2024-08-17T17:34:50.670013+0000","last_active":"2024-08-17T17:40:32.148986+0000","last_peered":"2024-08-17T17:40:32.148986+0000","last_clean":"2024-08-17T17:40:32.148986+0000","last_became_active":"2024-08-17T17:34:50.669299+0000","last_became_peered":"2024-08-17T17:34:50.669299+0000","last_unstale":"2024-08-17T17:40:32.148986+0000","last_undegraded":"2024-08-17T17:40:32.148986+0000","last_fullsized":"2024-08-17T17:40:32.148986+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":7706,"log_dups_size":0,"ondisk_log_size":7706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:11.172227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14417920,"num_objects":220,"num_object_clones":0,"num_object_copies":660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7706,"num_write_kb":253632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1390,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.928647+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:40:24.928647+0000","last_peered":"2024-08-17T17:40:24.928647+0000","last_clean":"2024-08-17T17:40:24.928647+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:40:24.928647+0000","last_undegraded":"2024-08-17T17:40:24.928647+0000","last_fullsized":"2024-08-17T17:40:24.928647+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":160,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.934543+0000","last_change":"2024-08-17T17:35:33.140725+0000","last_active":"2024-08-17T17:40:24.934543+0000","last_peered":"2024-08-17T17:40:24.934543+0000","last_clean":"2024-08-17T17:40:24.934543+0000","last_became_active":"2024-08-17T17:35:33.140406+0000","last_became_peered":"2024-08-17T17:35:33.140406+0000","last_unstale":"2024-08-17T17:40:24.934543+0000","last_undegraded":"2024-08-17T17:40:24.934543+0000","last_fullsized":"2024-08-17T17:40:24.934543+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:54:51.875571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1680,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.936548+0000","last_change":"2024-08-17T17:35:33.138514+0000","last_active":"2024-08-17T17:40:24.936548+0000","last_peered":"2024-08-17T17:40:24.936548+0000","last_clean":"2024-08-17T17:40:24.936548+0000","last_became_active":"2024-08-17T17:35:33.137035+0000","last_became_peered":"2024-08-17T17:35:33.137035+0000","last_unstale":"2024-08-17T17:40:24.936548+0000","last_undegraded":"2024-08-17T17:40:24.936548+0000","last_fullsized":"2024-08-17T17:40:24.936548+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:50.565662+0000","scrub_duration":0.001487138,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1322,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:27.758640+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:40:27.758640+0000","last_peered":"2024-08-17T17:40:27.758640+0000","last_clean":"2024-08-17T17:40:27.758640+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:40:27.758640+0000","last_undegraded":"2024-08-17T17:40:27.758640+0000","last_fullsized":"2024-08-17T17:40:27.758640+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":111,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.982090+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:40:32.982090+0000","last_peered":"2024-08-17T17:40:32.982090+0000","last_clean":"2024-08-17T17:40:32.982090+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:40:32.982090+0000","last_undegraded":"2024-08-17T17:40:32.982090+0000","last_fullsized":"2024-08-17T17:40:32.982090+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1752,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926181+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:40:24.926181+0000","last_peered":"2024-08-17T17:40:24.926181+0000","last_clean":"2024-08-17T17:40:24.926181+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:40:24.926181+0000","last_undegraded":"2024-08-17T17:40:24.926181+0000","last_fullsized":"2024-08-17T17:40:24.926181+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1339,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.934035+0000","last_change":"2024-08-17T17:35:55.205292+0000","last_active":"2024-08-17T17:40:24.934035+0000","last_peered":"2024-08-17T17:40:24.934035+0000","last_clean":"2024-08-17T17:40:24.934035+0000","last_became_active":"2024-08-17T17:35:55.204935+0000","last_became_peered":"2024-08-17T17:35:55.204935+0000","last_unstale":"2024-08-17T17:40:24.934035+0000","last_undegraded":"2024-08-17T17:40:24.934035+0000","last_fullsized":"2024-08-17T17:40:24.934035+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:52.277821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":111,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.982549+0000","last_change":"2024-08-17T17:36:12.749972+0000","last_active":"2024-08-17T17:40:32.982549+0000","last_peered":"2024-08-17T17:40:32.982549+0000","last_clean":"2024-08-17T17:40:32.982549+0000","last_became_active":"2024-08-17T17:36:12.749620+0000","last_became_peered":"2024-08-17T17:36:12.749620+0000","last_unstale":"2024-08-17T17:40:32.982549+0000","last_undegraded":"2024-08-17T17:40:32.982549+0000","last_fullsized":"2024-08-17T17:40:32.982549+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:04.994357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":987,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.981645+0000","last_change":"2024-08-17T17:36:20.832274+0000","last_active":"2024-08-17T17:40:32.981645+0000","last_peered":"2024-08-17T17:40:32.981645+0000","last_clean":"2024-08-17T17:40:32.981645+0000","last_became_active":"2024-08-17T17:36:20.831886+0000","last_became_peered":"2024-08-17T17:36:20.831886+0000","last_unstale":"2024-08-17T17:40:32.981645+0000","last_undegraded":"2024-08-17T17:40:32.981645+0000","last_fullsized":"2024-08-17T17:40:32.981645+0000","mapping_epoch":1390,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1391,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:05.248711+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1193,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.929873+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:40:24.929873+0000","last_peered":"2024-08-17T17:40:24.929873+0000","last_clean":"2024-08-17T17:40:24.929873+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:40:24.929873+0000","last_undegraded":"2024-08-17T17:40:24.929873+0000","last_fullsized":"2024-08-17T17:40:24.929873+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1422,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.926574+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:40:24.926574+0000","last_peered":"2024-08-17T17:40:24.926574+0000","last_clean":"2024-08-17T17:40:24.926574+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:40:24.926574+0000","last_undegraded":"2024-08-17T17:40:24.926574+0000","last_fullsized":"2024-08-17T17:40:24.926574+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1715,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.928478+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:40:24.928478+0000","last_peered":"2024-08-17T17:40:24.928478+0000","last_clean":"2024-08-17T17:40:24.928478+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:40:24.928478+0000","last_undegraded":"2024-08-17T17:40:24.928478+0000","last_fullsized":"2024-08-17T17:40:24.928478+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1364,"reported_epoch":1430,"state":"active+clean","last_fresh":"2024-08-17T17:40:24.924838+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:40:24.924838+0000","last_peered":"2024-08-17T17:40:24.924838+0000","last_clean":"2024-08-17T17:40:24.924838+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:40:24.924838+0000","last_undegraded":"2024-08-17T17:40:24.924838+0000","last_fullsized":"2024-08-17T17:40:24.924838+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1916,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.982362+0000","last_change":"2024-08-17T17:35:33.137411+0000","last_active":"2024-08-17T17:40:32.982362+0000","last_peered":"2024-08-17T17:40:32.982362+0000","last_clean":"2024-08-17T17:40:32.982362+0000","last_became_active":"2024-08-17T17:35:33.137073+0000","last_became_peered":"2024-08-17T17:35:33.137073+0000","last_unstale":"2024-08-17T17:40:32.982362+0000","last_undegraded":"2024-08-17T17:40:32.982362+0000","last_fullsized":"2024-08-17T17:40:32.982362+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:35.924539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1810,"reported_epoch":1431,"state":"active+clean","last_fresh":"2024-08-17T17:40:31.974363+0000","last_change":"2024-08-17T17:36:15.787445+0000","last_active":"2024-08-17T17:40:31.974363+0000","last_peered":"2024-08-17T17:40:31.974363+0000","last_clean":"2024-08-17T17:40:31.974363+0000","last_became_active":"2024-08-17T17:36:15.787099+0000","last_became_peered":"2024-08-17T17:36:15.787099+0000","last_unstale":"2024-08-17T17:40:31.974363+0000","last_undegraded":"2024-08-17T17:40:31.974363+0000","last_fullsized":"2024-08-17T17:40:31.974363+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:47.448168+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":269287448,"num_objects":4110,"num_object_clones":0,"num_object_copies":12330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7711,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4110,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":124772,"num_write_kb":4124161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35668,"num_bytes_recovered":2183856128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2660052992,"data_stored":2660040776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107672,"ondisk_log_size":107672,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635092,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":27,"num_read_kb":61,"num_write":15,"num_write_kb":183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1314910,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162877,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":507520,"kb_used_data":347000,"kb_used_omap":54,"kb_used_meta":160457,"kb_avail":93237632,"statfs":{"total":95995035648,"available":95475335168,"internally_reserved":0,"allocated":355328000,"data_stored":352622110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":164307989},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999351,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":427600,"kb_used_data":197380,"kb_used_omap":47,"kb_used_meta":230160,"kb_avail":93317552,"statfs":{"total":95995035648,"available":95557173248,"internally_reserved":0,"allocated":202117120,"data_stored":199408237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":235684500},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,0,4,8,3],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627148,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1176452,"kb_used_data":89972,"kb_used_omap":42,"kb_used_meta":1086421,"kb_avail":92568700,"statfs":{"total":95995035648,"available":94790348800,"internally_reserved":0,"allocated":92131328,"data_stored":89440735,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1112495164},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785625,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1476344,"kb_used_data":623596,"kb_used_omap":48,"kb_used_meta":852687,"kb_avail":92268808,"statfs":{"total":95995035648,"available":94483259392,"internally_reserved":0,"allocated":638562304,"data_stored":635872577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":873152174},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725142,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":983664,"kb_used_data":78056,"kb_used_omap":1,"kb_used_meta":905534,"kb_avail":92761488,"statfs":{"total":95995035648,"available":94987763712,"internally_reserved":0,"allocated":79929344,"data_stored":77235537,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":927267270},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986779,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1324860,"kb_used_data":437364,"kb_used_omap":1,"kb_used_meta":887486,"kb_avail":92420292,"statfs":{"total":95995035648,"available":94638379008,"internally_reserved":0,"allocated":447860736,"data_stored":445158352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":908786118},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706386,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1580144,"kb_used_data":637416,"kb_used_omap":35,"kb_used_meta":942684,"kb_avail":92165008,"statfs":{"total":95995035648,"available":94376968192,"internally_reserved":0,"allocated":652713984,"data_stored":650020158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":965308471},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832159,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1342892,"kb_used_data":290084,"kb_used_omap":65,"kb_used_meta":1052734,"kb_avail":92402260,"statfs":{"total":95995035648,"available":94619914240,"internally_reserved":0,"allocated":297046016,"data_stored":294356748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1077999803},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":283508736,"data_stored":283508736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":630849536,"data_stored":630849536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":632229888,"data_stored":632225816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":76349440,"data_stored":76349440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":433197056,"data_stored":433192984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":62586880,"data_stored":62586880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":194056192,"data_stored":194052120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":347275264,"data_stored":347275264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:40:38.257 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:38.267 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:37 smithi162 ceph-mon[104668]: from='client.46532 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:38.267 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:37 smithi162 ceph-mon[104668]: pgmap v2293: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 176 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 7533/8523 objects degraded (88.384%); 2.7 MiB/s, 55 objects/s recovering 2024-08-17T17:40:38.267 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:37 smithi162 ceph-mon[104668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:38.267 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:37 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:40:38.268 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:37 smithi162 ceph-mon[104668]: osdmap e1433: 8 total, 8 up, 8 in 2024-08-17T17:40:38.278 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:38.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:37 smithi063 ceph-mon[109843]: from='client.46532 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:38.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:37 smithi063 ceph-mon[109843]: pgmap v2293: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 176 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 7533/8523 objects degraded (88.384%); 2.7 MiB/s, 55 objects/s recovering 2024-08-17T17:40:38.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:37 smithi063 ceph-mon[109843]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:38.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:37 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:40:38.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:37 smithi063 ceph-mon[109843]: osdmap e1433: 8 total, 8 up, 8 in 2024-08-17T17:40:38.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:37 smithi063 ceph-mon[114756]: from='client.46532 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:38.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:37 smithi063 ceph-mon[114756]: pgmap v2293: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 176 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 7533/8523 objects degraded (88.384%); 2.7 MiB/s, 55 objects/s recovering 2024-08-17T17:40:38.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:37 smithi063 ceph-mon[114756]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:40:38.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:37 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:40:38.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:37 smithi063 ceph-mon[114756]: osdmap e1433: 8 total, 8 up, 8 in 2024-08-17T17:40:38.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:38.381 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:38.402 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:38.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:38.504 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:38.527 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:38.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:38.629 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:38.652 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:38.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:38.754 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:38.777 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:38.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:38.878 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:38.903 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:38.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:38.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:39.005 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:39.027 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:39.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:39.129 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:39.151 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:39.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:39.253 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:39.276 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:39.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:39.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:38 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 7533/8523 objects degraded (88.384%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:39.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:38 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 7533/8523 objects degraded (88.384%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:39.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:38 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 7533/8523 objects degraded (88.384%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:39.378 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:39.403 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:39.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:39.504 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:39.528 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:39.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:39.630 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:39.655 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:39.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:39.749 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:40:39.756 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:39.782 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:39.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:39.883 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:39.908 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:39.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:39.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:40.010 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:40.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:39 smithi063 ceph-mon[114756]: pgmap v2295: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 43 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 7533/2115 objects degraded (356.170%); 3.2 MiB/s, 66 objects/s recovering 2024-08-17T17:40:40.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:39 smithi063 ceph-mon[109843]: pgmap v2295: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 43 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 7533/2115 objects degraded (356.170%); 3.2 MiB/s, 66 objects/s recovering 2024-08-17T17:40:40.032 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:40.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:40.135 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:40.160 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:40.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:40.262 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:40.284 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:40.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:40.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:39 smithi162 ceph-mon[104668]: pgmap v2295: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 43 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 7533/2115 objects degraded (356.170%); 3.2 MiB/s, 66 objects/s recovering 2024-08-17T17:40:40.387 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:40.409 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:40.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:40.487 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:40:40.487 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:40:40.487 INFO:teuthology.orchestra.run.smithi063.stdout: "complaint_time": 30, 2024-08-17T17:40:40.487 INFO:teuthology.orchestra.run.smithi063.stdout: "num_blocked_ops": 0 2024-08-17T17:40:40.487 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:40:40.510 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:40.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.537 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:40.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:40.638 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:40.664 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:40.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:40.766 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:40.792 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:40.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:40.894 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:40.917 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:40.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:40.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:41.018 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:41.041 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:41.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:41.143 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:41.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.166 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:41.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:41.203 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:40:41.267 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:41.293 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:41.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:41.395 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:41.417 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:41.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:41.519 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:41.542 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:41.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:41.643 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:41.671 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:41.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:41.773 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:41.798 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:41.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:41.899 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:41.922 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:41.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:41.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:42.024 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:42.050 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:42.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:42.152 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:42.174 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:42.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:42.276 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:42.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:41 smithi063 ceph-mon[114756]: pgmap v2296: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 31 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 7533/1527 objects degraded (493.320%); 2.7 MiB/s, 55 objects/s recovering 2024-08-17T17:40:42.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:41 smithi063 ceph-mon[109843]: pgmap v2296: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 31 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 7533/1527 objects degraded (493.320%); 2.7 MiB/s, 55 objects/s recovering 2024-08-17T17:40:42.301 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_6 2024-08-17T17:40:42.301 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool rm unique_pool_6 unique_pool_6 --yes-i-really-really-mean-it 2024-08-17T17:40:42.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.302 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:42.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:42.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:41 smithi162 ceph-mon[104668]: pgmap v2296: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 31 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 7533/1527 objects degraded (493.320%); 2.7 MiB/s, 55 objects/s recovering 2024-08-17T17:40:42.404 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:42.428 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:42.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:42.530 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:42.556 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:42.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:42.658 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:42.684 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:42.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:42.786 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:42.810 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:42.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.811 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:42.912 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:42.935 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:42.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:42.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:43.037 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:43.059 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:43.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:43.161 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:43.187 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:43.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:43.289 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:43.313 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:43.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.314 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:43.414 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:43.440 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:43.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:43.542 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:43.564 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:43.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:43.666 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:43.691 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:43.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:43.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:40:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:40:43] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.51.0" 2024-08-17T17:40:43.792 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:43.818 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:43.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:43.920 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:43.945 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:43.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:43.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:44.047 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:44.070 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:44.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:44.172 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:44.200 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:44.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:44.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:43 smithi063 ceph-mon[114756]: pgmap v2297: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 7067/78 objects degraded (9060.256%); 620 KiB/s, 66 objects/s recovering 2024-08-17T17:40:44.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:43 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 7067/78 objects degraded (9060.256%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:44.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:43 smithi063 ceph-mon[109843]: pgmap v2297: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 7067/78 objects degraded (9060.256%); 620 KiB/s, 66 objects/s recovering 2024-08-17T17:40:44.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:43 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 7067/78 objects degraded (9060.256%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:44.302 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:44.326 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:44.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:44.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:43 smithi162 ceph-mon[104668]: pgmap v2297: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 7067/78 objects degraded (9060.256%); 620 KiB/s, 66 objects/s recovering 2024-08-17T17:40:44.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:43 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 7067/78 objects degraded (9060.256%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:44.427 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:44.451 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:44.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:44.553 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:44.575 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:44.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:44.677 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:44.703 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:44.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:44.804 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:44.830 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:44.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:44.931 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:44.957 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:44.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:44.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:45.059 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:45.085 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:45.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:45.186 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:45.209 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:45.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:45.310 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:45.336 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:45.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:45.437 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:45.460 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:45.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:45.561 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:45.587 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:45.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:45.689 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:45.711 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:45.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:45.813 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:45.839 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:45.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:45.940 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:45.963 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:45.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:45.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:46.065 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:46.091 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:46.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:46.193 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:46.218 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:46.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:46.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:46 smithi063 ceph-mon[109843]: pgmap v2298: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 7067/78 objects degraded (9060.256%); 601 KiB/s, 64 objects/s recovering 2024-08-17T17:40:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:46 smithi063 ceph-mon[114756]: pgmap v2298: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 7067/78 objects degraded (9060.256%); 601 KiB/s, 64 objects/s recovering 2024-08-17T17:40:46.321 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:46.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:46 smithi162 ceph-mon[104668]: pgmap v2298: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 410 op/s; 7067/78 objects degraded (9060.256%); 601 KiB/s, 64 objects/s recovering 2024-08-17T17:40:46.346 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:46.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:46.448 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:46.470 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:46.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:46.572 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:46.599 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:46.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:46.700 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:46.723 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:46.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:46.825 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:46.851 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:46.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:46.953 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:46.976 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:46.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:46.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:47.078 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:47.101 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:47.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:47.203 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:47.228 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:47.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:47.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:47.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:47.330 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:47.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:40:47.355 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:47.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:47.456 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:47.479 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:47.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:47.580 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:47.603 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:47.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:47.705 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:47.731 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:47.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:47.833 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:47.856 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:47.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:47.957 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:47.980 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:47.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:47.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:48.083 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:48.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:48 smithi162 ceph-mon[104668]: pgmap v2299: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 6316/78 objects degraded (8097.436%); 0 B/s, 121 objects/s recovering 2024-08-17T17:40:48.106 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:48.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:48.208 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:48.230 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:48.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:48 smithi063 ceph-mon[114756]: pgmap v2299: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 6316/78 objects degraded (8097.436%); 0 B/s, 121 objects/s recovering 2024-08-17T17:40:48.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:48 smithi063 ceph-mon[109843]: pgmap v2299: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 281 op/s; 6316/78 objects degraded (8097.436%); 0 B/s, 121 objects/s recovering 2024-08-17T17:40:48.333 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:48.354 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:48.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:48.456 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:48.479 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:48.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:48.581 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:48.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.605 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:48.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:48.706 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:48.728 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:48.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:48.830 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:48.855 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:48.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:48.957 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:48.979 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:48.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:48.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:49.081 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:49.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.106 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:49.107 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:49.208 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:49.231 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:49.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:49.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:49 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 6316/78 objects degraded (8097.436%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:49.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:49 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 6316/78 objects degraded (8097.436%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:49.332 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:49.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:49 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 6316/78 objects degraded (8097.436%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:49.355 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:49.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:49.456 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:49.479 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:49.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:49.579 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-08-17T17:40:49.580 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:49.605 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:49.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:49.669 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:49.708 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:49.731 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:49.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:49.833 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:49.859 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:49.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:49.907 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:49.907 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:49.960 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:49.986 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:49.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:49.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:50.088 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:50.114 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:50.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:50.151 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:50 smithi063 ceph-mon[114756]: pgmap v2300: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 240 op/s; 6316/78 objects degraded (8097.436%); 0 B/s, 104 objects/s recovering 2024-08-17T17:40:50.151 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:50 smithi063 ceph-mon[109843]: pgmap v2300: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 240 op/s; 6316/78 objects degraded (8097.436%); 0 B/s, 104 objects/s recovering 2024-08-17T17:40:50.216 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:50.239 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:50.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:50.259 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:40:50.260 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:40:50.260 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:40:50.260 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:40:50.260 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:40:50.260 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:40:50.275 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:40:50.276 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:40:50.298 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:40:50.298 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:40:50.320 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:40:50.320 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:40:50.338 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:40:50.338 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:40:50.341 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:50.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:50 smithi162 ceph-mon[104668]: pgmap v2300: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 240 op/s; 6316/78 objects degraded (8097.436%); 0 B/s, 104 objects/s recovering 2024-08-17T17:40:50.355 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:40:50.355 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:40:50.365 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:50.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:50.468 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:50.492 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:50.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:50.594 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:50.618 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:50.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:50.720 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:50.746 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:50.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:50.848 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:50.873 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:50.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:50.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:50.975 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:51.000 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:51.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:51.102 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:51.128 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:51.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:51.176 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:51 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3727623224' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:51.176 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:51 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3727623224' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:51.230 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:51.255 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:51.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:51.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:51 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3727623224' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:40:51.357 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:51.382 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:51.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:51.484 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:51.508 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:51.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:51.610 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:51.633 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:51.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:51.735 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:51.759 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:51.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:51.860 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:51.900 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:51.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:51.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:52.002 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:52.026 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:52.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:52.127 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:52.153 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:52.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:52.255 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:52.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:52 smithi063 ceph-mon[114756]: pgmap v2301: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 56 op/s; 6316/78 objects degraded (8097.436%); 0 B/s, 101 objects/s recovering 2024-08-17T17:40:52.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:52 smithi063 ceph-mon[109843]: pgmap v2301: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 56 op/s; 6316/78 objects degraded (8097.436%); 0 B/s, 101 objects/s recovering 2024-08-17T17:40:52.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.280 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:52.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:52.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:52 smithi162 ceph-mon[104668]: pgmap v2301: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 56 op/s; 6316/78 objects degraded (8097.436%); 0 B/s, 101 objects/s recovering 2024-08-17T17:40:52.381 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:52.403 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:52.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:52.505 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:52.528 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:52.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:52.630 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:52.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.656 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:52.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:52.757 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:52.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.782 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:52.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:52.883 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:52.906 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:52.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:52.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:53.008 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:53.033 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:53.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:53.134 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:53.159 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:53.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:53.261 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:53.284 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:53.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:53.386 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:53.409 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:53.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:53.511 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:53.534 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:53.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:53.636 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:53.659 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:53.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:53.674 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:40:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:40:53] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.51.0" 2024-08-17T17:40:53.761 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:53.785 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:53.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:53.887 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:53.911 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:53.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:53.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:54.012 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:54.035 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:54.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:54.137 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:54.160 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:54.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.161 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:54.195 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:54 smithi063 ceph-mon[114756]: pgmap v2302: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 40 op/s; 5383/78 objects degraded (6901.282%); 0 B/s, 179 objects/s recovering 2024-08-17T17:40:54.195 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:54 smithi063 ceph-mon[114756]: Health check update: Degraded data redundancy: 5383/78 objects degraded (6901.282%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:54.195 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:54 smithi063 ceph-mon[109843]: pgmap v2302: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 40 op/s; 5383/78 objects degraded (6901.282%); 0 B/s, 179 objects/s recovering 2024-08-17T17:40:54.195 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:54 smithi063 ceph-mon[109843]: Health check update: Degraded data redundancy: 5383/78 objects degraded (6901.282%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:54.263 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:54.286 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:54.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:54.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:54 smithi162 ceph-mon[104668]: pgmap v2302: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 40 op/s; 5383/78 objects degraded (6901.282%); 0 B/s, 179 objects/s recovering 2024-08-17T17:40:54.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:54 smithi162 ceph-mon[104668]: Health check update: Degraded data redundancy: 5383/78 objects degraded (6901.282%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2024-08-17T17:40:54.388 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:54.412 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:54.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:54.513 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:54.539 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:54.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:54.640 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:54.665 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:54.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:54.725 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:40:54.727 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:40:54.767 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:54.793 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:54.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:54.888 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-08-17T17:40:54.894 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:54.919 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:54.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:54.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:55.021 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:55 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3441339291' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:40:55.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:55 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:40:55.044 INFO:teuthology.orchestra.run.smithi063.stderr:pool 'unique_pool_6' removed 2024-08-17T17:40:55.053 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:55.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:55.154 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:55.176 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:55.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:55.277 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:55.299 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:55.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:55.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:55 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3441339291' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:40:55.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:55 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:40:55.401 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:55.425 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:55.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:55.507 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:55 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3441339291' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:40:55.507 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:55 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true} : dispatch 2024-08-17T17:40:55.527 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:55.554 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:55.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:55.655 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:55.678 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:55.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:55.779 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:55.789 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:55.803 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:55.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:55.905 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:55.927 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:55.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:55.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:56.028 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:56.050 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:56.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:56.152 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:40:56.174 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:56.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:40:56.179 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:56 smithi063 ceph-mon[109843]: pgmap v2303: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 5383/78 objects degraded (6901.282%); 0 B/s, 140 objects/s recovering 2024-08-17T17:40:56.179 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:56 smithi063 ceph-mon[109843]: from='client.36579 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:56.179 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:56 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:40:56.179 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:56 smithi063 ceph-mon[109843]: osdmap e1434: 8 total, 8 up, 8 in 2024-08-17T17:40:56.179 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:56 smithi063 ceph-mon[114756]: pgmap v2303: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 5383/78 objects degraded (6901.282%); 0 B/s, 140 objects/s recovering 2024-08-17T17:40:56.179 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:56 smithi063 ceph-mon[114756]: from='client.36579 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:56.179 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:56 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:40:56.180 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:56 smithi063 ceph-mon[114756]: osdmap e1434: 8 total, 8 up, 8 in 2024-08-17T17:40:56.276 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:56.298 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:56.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:56.326 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2303,"stamp":"2024-08-17T17:40:54.687974+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":640665,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5383,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":28,"num_read_kb":62,"num_write":128961,"num_write_kb":4124377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38020,"num_bytes_recovered":2191332061,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114460,"ondisk_log_size":114460,"up":357,"acting":353,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":374,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7849856,"kb_used_data":1739496,"kb_used_omap":298,"kb_used_meta":6109973,"kb_avail":742111360,"statfs":{"total":767960285184,"available":759922032640,"internally_reserved":0,"allocated":1781243904,"data_stored":1759601963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":6256612881},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1684,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1684,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003034"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":73,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.007496+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:40:37.007496+0000","last_peered":"2024-08-17T17:40:37.007496+0000","last_clean":"2024-08-17T17:40:37.007496+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:40:37.007496+0000","last_undegraded":"2024-08-17T17:40:37.007496+0000","last_fullsized":"2024-08-17T17:40:37.007496+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1448,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002030+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:40:33.002030+0000","last_peered":"2024-08-17T17:40:33.002030+0000","last_clean":"2024-08-17T17:40:33.002030+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:40:33.002030+0000","last_undegraded":"2024-08-17T17:40:33.002030+0000","last_fullsized":"2024-08-17T17:40:33.002030+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1763,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002702+0000","last_change":"2024-08-17T17:36:15.771150+0000","last_active":"2024-08-17T17:40:33.002702+0000","last_peered":"2024-08-17T17:40:33.002702+0000","last_clean":"2024-08-17T17:40:33.002702+0000","last_became_active":"2024-08-17T17:36:15.770520+0000","last_became_peered":"2024-08-17T17:36:15.770520+0000","last_unstale":"2024-08-17T17:40:33.002702+0000","last_undegraded":"2024-08-17T17:40:33.002702+0000","last_fullsized":"2024-08-17T17:40:33.002702+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:17.552796+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":98,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.978323+0000","last_change":"2024-08-17T17:36:12.756709+0000","last_active":"2024-08-17T17:40:32.978323+0000","last_peered":"2024-08-17T17:40:32.978323+0000","last_clean":"2024-08-17T17:40:32.978323+0000","last_became_active":"2024-08-17T17:36:12.756385+0000","last_became_peered":"2024-08-17T17:36:12.756385+0000","last_unstale":"2024-08-17T17:40:32.978323+0000","last_undegraded":"2024-08-17T17:40:32.978323+0000","last_fullsized":"2024-08-17T17:40:32.978323+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:21.420282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":158,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.004483+0000","last_change":"2024-08-17T17:36:12.762800+0000","last_active":"2024-08-17T17:40:37.004483+0000","last_peered":"2024-08-17T17:40:37.004483+0000","last_clean":"2024-08-17T17:40:37.004483+0000","last_became_active":"2024-08-17T17:36:12.759977+0000","last_became_peered":"2024-08-17T17:36:12.759977+0000","last_unstale":"2024-08-17T17:40:37.004483+0000","last_undegraded":"2024-08-17T17:40:37.004483+0000","last_fullsized":"2024-08-17T17:40:37.004483+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:02.162681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1642,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.977970+0000","last_change":"2024-08-17T17:35:56.281561+0000","last_active":"2024-08-17T17:40:32.977970+0000","last_peered":"2024-08-17T17:40:32.977970+0000","last_clean":"2024-08-17T17:40:32.977970+0000","last_became_active":"2024-08-17T17:35:56.278638+0000","last_became_peered":"2024-08-17T17:35:56.278638+0000","last_unstale":"2024-08-17T17:40:32.977970+0000","last_undegraded":"2024-08-17T17:40:32.977970+0000","last_fullsized":"2024-08-17T17:40:32.977970+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.417151+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.417151+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1381,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.980310+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:40:32.980310+0000","last_peered":"2024-08-17T17:40:32.980310+0000","last_clean":"2024-08-17T17:40:32.980310+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:40:32.980310+0000","last_undegraded":"2024-08-17T17:40:32.980310+0000","last_fullsized":"2024-08-17T17:40:32.980310+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":57,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002086+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:40:33.002086+0000","last_peered":"2024-08-17T17:40:33.002086+0000","last_clean":"2024-08-17T17:40:33.002086+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:40:33.002086+0000","last_undegraded":"2024-08-17T17:40:33.002086+0000","last_fullsized":"2024-08-17T17:40:33.002086+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1426,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.980145+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:40:32.980145+0000","last_peered":"2024-08-17T17:40:32.980145+0000","last_clean":"2024-08-17T17:40:32.980145+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:40:32.980145+0000","last_undegraded":"2024-08-17T17:40:32.980145+0000","last_fullsized":"2024-08-17T17:40:32.980145+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1434,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002547+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:40:33.002547+0000","last_peered":"2024-08-17T17:40:33.002547+0000","last_clean":"2024-08-17T17:40:33.002547+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:40:33.002547+0000","last_undegraded":"2024-08-17T17:40:33.002547+0000","last_fullsized":"2024-08-17T17:40:33.002547+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":725,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.978020+0000","last_change":"2024-08-17T17:36:15.773315+0000","last_active":"2024-08-17T17:40:32.978020+0000","last_peered":"2024-08-17T17:40:32.978020+0000","last_clean":"2024-08-17T17:40:32.978020+0000","last_became_active":"2024-08-17T17:36:15.772961+0000","last_became_peered":"2024-08-17T17:36:15.772961+0000","last_unstale":"2024-08-17T17:40:32.978020+0000","last_undegraded":"2024-08-17T17:40:32.978020+0000","last_fullsized":"2024-08-17T17:40:32.978020+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:10.190221+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":67,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.006770+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:40:37.006770+0000","last_peered":"2024-08-17T17:40:37.006770+0000","last_clean":"2024-08-17T17:40:37.006770+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:40:37.006770+0000","last_undegraded":"2024-08-17T17:40:37.006770+0000","last_fullsized":"2024-08-17T17:40:37.006770+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":158,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.018463+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:40:37.018463+0000","last_peered":"2024-08-17T17:40:37.018463+0000","last_clean":"2024-08-17T17:40:37.018463+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:40:37.018463+0000","last_undegraded":"2024-08-17T17:40:37.018463+0000","last_fullsized":"2024-08-17T17:40:37.018463+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1516,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.010156+0000","last_change":"2024-08-17T17:36:24.225739+0000","last_active":"2024-08-17T17:40:37.010156+0000","last_peered":"2024-08-17T17:40:37.010156+0000","last_clean":"2024-08-17T17:40:37.010156+0000","last_became_active":"2024-08-17T17:36:24.225250+0000","last_became_peered":"2024-08-17T17:36:24.225250+0000","last_unstale":"2024-08-17T17:40:37.010156+0000","last_undegraded":"2024-08-17T17:40:37.010156+0000","last_fullsized":"2024-08-17T17:40:37.010156+0000","mapping_epoch":1393,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1394,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.290656+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1454,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002314+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:40:33.002314+0000","last_peered":"2024-08-17T17:40:33.002314+0000","last_clean":"2024-08-17T17:40:33.002314+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:40:33.002314+0000","last_undegraded":"2024-08-17T17:40:33.002314+0000","last_fullsized":"2024-08-17T17:40:33.002314+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1862,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002797+0000","last_change":"2024-08-17T17:36:12.752970+0000","last_active":"2024-08-17T17:40:33.002797+0000","last_peered":"2024-08-17T17:40:33.002797+0000","last_clean":"2024-08-17T17:40:33.002797+0000","last_became_active":"2024-08-17T17:36:12.752311+0000","last_became_peered":"2024-08-17T17:36:12.752311+0000","last_unstale":"2024-08-17T17:40:33.002797+0000","last_undegraded":"2024-08-17T17:40:33.002797+0000","last_fullsized":"2024-08-17T17:40:33.002797+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:38.131419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1809,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.980234+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:40:32.980234+0000","last_peered":"2024-08-17T17:40:32.980234+0000","last_clean":"2024-08-17T17:40:32.980234+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:40:32.980234+0000","last_undegraded":"2024-08-17T17:40:32.980234+0000","last_fullsized":"2024-08-17T17:40:32.980234+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1530,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.018889+0000","last_change":"2024-08-17T17:36:18.750494+0000","last_active":"2024-08-17T17:40:37.018889+0000","last_peered":"2024-08-17T17:40:37.018889+0000","last_clean":"2024-08-17T17:40:37.018889+0000","last_became_active":"2024-08-17T17:36:18.750092+0000","last_became_peered":"2024-08-17T17:36:18.750092+0000","last_unstale":"2024-08-17T17:40:37.018889+0000","last_undegraded":"2024-08-17T17:40:37.018889+0000","last_fullsized":"2024-08-17T17:40:37.018889+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:36.599727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1160,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.007514+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:40:37.007514+0000","last_peered":"2024-08-17T17:40:37.007514+0000","last_clean":"2024-08-17T17:40:37.007514+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:40:37.007514+0000","last_undegraded":"2024-08-17T17:40:37.007514+0000","last_fullsized":"2024-08-17T17:40:37.007514+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":976,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.979797+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:40:32.979797+0000","last_peered":"2024-08-17T17:40:32.979797+0000","last_clean":"2024-08-17T17:40:32.979797+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:40:32.979797+0000","last_undegraded":"2024-08-17T17:40:32.979797+0000","last_fullsized":"2024-08-17T17:40:32.979797+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":67,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.006224+0000","last_change":"2024-08-17T17:37:11.075505+0000","last_active":"2024-08-17T17:40:37.006224+0000","last_peered":"2024-08-17T17:40:37.006224+0000","last_clean":"2024-08-17T17:40:37.006224+0000","last_became_active":"2024-08-17T17:37:11.075131+0000","last_became_peered":"2024-08-17T17:37:11.075131+0000","last_unstale":"2024-08-17T17:40:37.006224+0000","last_undegraded":"2024-08-17T17:40:37.006224+0000","last_fullsized":"2024-08-17T17:40:37.006224+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:39.672054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1289,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.978467+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:40:32.978467+0000","last_peered":"2024-08-17T17:40:32.978467+0000","last_clean":"2024-08-17T17:40:32.978467+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:40:32.978467+0000","last_undegraded":"2024-08-17T17:40:32.978467+0000","last_fullsized":"2024-08-17T17:40:32.978467+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1666,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002119+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:40:33.002119+0000","last_peered":"2024-08-17T17:40:33.002119+0000","last_clean":"2024-08-17T17:40:33.002119+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:40:33.002119+0000","last_undegraded":"2024-08-17T17:40:33.002119+0000","last_fullsized":"2024-08-17T17:40:33.002119+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":114,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.014974+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:40:37.014974+0000","last_peered":"2024-08-17T17:40:37.014974+0000","last_clean":"2024-08-17T17:40:37.014974+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:40:37.014974+0000","last_undegraded":"2024-08-17T17:40:37.014974+0000","last_fullsized":"2024-08-17T17:40:37.014974+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1263,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.980358+0000","last_change":"2024-08-17T17:36:12.755873+0000","last_active":"2024-08-17T17:40:32.980358+0000","last_peered":"2024-08-17T17:40:32.980358+0000","last_clean":"2024-08-17T17:40:32.980358+0000","last_became_active":"2024-08-17T17:36:12.755366+0000","last_became_peered":"2024-08-17T17:36:12.755366+0000","last_unstale":"2024-08-17T17:40:32.980358+0000","last_undegraded":"2024-08-17T17:40:32.980358+0000","last_fullsized":"2024-08-17T17:40:32.980358+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:36.523214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1617,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.008802+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:40:33.008802+0000","last_peered":"2024-08-17T17:40:33.008802+0000","last_clean":"2024-08-17T17:40:33.008802+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:40:33.008802+0000","last_undegraded":"2024-08-17T17:40:33.008802+0000","last_fullsized":"2024-08-17T17:40:33.008802+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":801,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.010353+0000","last_change":"2024-08-17T17:35:55.200519+0000","last_active":"2024-08-17T17:40:37.010353+0000","last_peered":"2024-08-17T17:40:37.010353+0000","last_clean":"2024-08-17T17:40:37.010353+0000","last_became_active":"2024-08-17T17:35:55.200230+0000","last_became_peered":"2024-08-17T17:35:55.200230+0000","last_unstale":"2024-08-17T17:40:37.010353+0000","last_undegraded":"2024-08-17T17:40:37.010353+0000","last_fullsized":"2024-08-17T17:40:37.010353+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:33.454851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1440,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.979996+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:40:32.979996+0000","last_peered":"2024-08-17T17:40:32.979996+0000","last_clean":"2024-08-17T17:40:32.979996+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:40:32.979996+0000","last_undegraded":"2024-08-17T17:40:32.979996+0000","last_fullsized":"2024-08-17T17:40:32.979996+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1731,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.014685+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:40:37.014685+0000","last_peered":"2024-08-17T17:40:37.014685+0000","last_clean":"2024-08-17T17:40:37.014685+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:40:37.014685+0000","last_undegraded":"2024-08-17T17:40:37.014685+0000","last_fullsized":"2024-08-17T17:40:37.014685+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:22.077229+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":278,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.016568+0000","last_change":"2024-08-17T17:35:33.145476+0000","last_active":"2024-08-17T17:40:37.016568+0000","last_peered":"2024-08-17T17:40:37.016568+0000","last_clean":"2024-08-17T17:40:37.016568+0000","last_became_active":"2024-08-17T17:35:33.145125+0000","last_became_peered":"2024-08-17T17:35:33.145125+0000","last_unstale":"2024-08-17T17:40:37.016568+0000","last_undegraded":"2024-08-17T17:40:37.016568+0000","last_fullsized":"2024-08-17T17:40:37.016568+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:45.298064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":945,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.009813+0000","last_change":"2024-08-17T17:35:56.248612+0000","last_active":"2024-08-17T17:40:37.009813+0000","last_peered":"2024-08-17T17:40:37.009813+0000","last_clean":"2024-08-17T17:40:37.009813+0000","last_became_active":"2024-08-17T17:35:56.244364+0000","last_became_peered":"2024-08-17T17:35:56.244364+0000","last_unstale":"2024-08-17T17:40:37.009813+0000","last_undegraded":"2024-08-17T17:40:37.009813+0000","last_fullsized":"2024-08-17T17:40:37.009813+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:41.064165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":884,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.980003+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:40:32.980003+0000","last_peered":"2024-08-17T17:40:32.980003+0000","last_clean":"2024-08-17T17:40:32.980003+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:40:32.980003+0000","last_undegraded":"2024-08-17T17:40:32.980003+0000","last_fullsized":"2024-08-17T17:40:32.980003+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1420,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002604+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:40:33.002604+0000","last_peered":"2024-08-17T17:40:33.002604+0000","last_clean":"2024-08-17T17:40:33.002604+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:40:33.002604+0000","last_undegraded":"2024-08-17T17:40:33.002604+0000","last_fullsized":"2024-08-17T17:40:33.002604+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1439,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002618+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:40:33.002618+0000","last_peered":"2024-08-17T17:40:33.002618+0000","last_clean":"2024-08-17T17:40:33.002618+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:40:33.002618+0000","last_undegraded":"2024-08-17T17:40:33.002618+0000","last_fullsized":"2024-08-17T17:40:33.002618+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":952,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.014891+0000","last_change":"2024-08-17T17:36:15.774881+0000","last_active":"2024-08-17T17:40:37.014891+0000","last_peered":"2024-08-17T17:40:37.014891+0000","last_clean":"2024-08-17T17:40:37.014891+0000","last_became_active":"2024-08-17T17:36:15.774435+0000","last_became_peered":"2024-08-17T17:36:15.774435+0000","last_unstale":"2024-08-17T17:40:37.014891+0000","last_undegraded":"2024-08-17T17:40:37.014891+0000","last_fullsized":"2024-08-17T17:40:37.014891+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:52.015854+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"11.e","version":"1433'8286","reported_seq":16476,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.396902+0000","last_change":"2024-08-17T17:38:37.110784+0000","last_active":"2024-08-17T17:40:37.396902+0000","last_peered":"2024-08-17T17:40:37.396902+0000","last_clean":"2024-08-17T17:40:37.396902+0000","last_became_active":"2024-08-17T17:38:37.088380+0000","last_became_peered":"2024-08-17T17:38:37.088380+0000","last_unstale":"2024-08-17T17:40:37.396902+0000","last_undegraded":"2024-08-17T17:40:37.396902+0000","last_fullsized":"2024-08-17T17:40:37.396902+0000","mapping_epoch":1419,"log_start":"1374'2100","ondisk_log_start":"1374'2100","created":1357,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":6186,"log_dups_size":0,"ondisk_log_size":6186,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:11:08.647903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8286,"num_write_kb":265152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4009,"num_bytes_recovered":199753728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1387,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.001927+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:40:33.001927+0000","last_peered":"2024-08-17T17:40:33.001927+0000","last_clean":"2024-08-17T17:40:33.001927+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:40:33.001927+0000","last_undegraded":"2024-08-17T17:40:33.001927+0000","last_fullsized":"2024-08-17T17:40:33.001927+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1283,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.978382+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:40:32.978382+0000","last_peered":"2024-08-17T17:40:32.978382+0000","last_clean":"2024-08-17T17:40:32.978382+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:40:32.978382+0000","last_undegraded":"2024-08-17T17:40:32.978382+0000","last_fullsized":"2024-08-17T17:40:32.978382+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":345,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.007638+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:40:37.007638+0000","last_peered":"2024-08-17T17:40:37.007638+0000","last_clean":"2024-08-17T17:40:37.007638+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:40:37.007638+0000","last_undegraded":"2024-08-17T17:40:37.007638+0000","last_fullsized":"2024-08-17T17:40:37.007638+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1614,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.015729+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:40:37.015729+0000","last_peered":"2024-08-17T17:40:37.015729+0000","last_clean":"2024-08-17T17:40:37.015729+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:40:37.015729+0000","last_undegraded":"2024-08-17T17:40:37.015729+0000","last_fullsized":"2024-08-17T17:40:37.015729+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:25.751871+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.f","version":"1433'8204","reported_seq":8375,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.387712+0000","last_change":"2024-08-17T17:35:34.179168+0000","last_active":"2024-08-17T17:40:37.387712+0000","last_peered":"2024-08-17T17:40:37.387712+0000","last_clean":"2024-08-17T17:40:37.387712+0000","last_became_active":"2024-08-17T17:35:34.133493+0000","last_became_peered":"2024-08-17T17:35:34.133493+0000","last_unstale":"2024-08-17T17:40:37.387712+0000","last_undegraded":"2024-08-17T17:40:37.387712+0000","last_fullsized":"2024-08-17T17:40:37.387712+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":8204,"log_dups_size":0,"ondisk_log_size":8204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:24:11.188566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8204,"num_write_kb":262528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":458752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":121,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.015017+0000","last_change":"2024-08-17T17:37:11.079712+0000","last_active":"2024-08-17T17:40:37.015017+0000","last_peered":"2024-08-17T17:40:37.015017+0000","last_clean":"2024-08-17T17:40:37.015017+0000","last_became_active":"2024-08-17T17:37:11.079253+0000","last_became_peered":"2024-08-17T17:37:11.079253+0000","last_unstale":"2024-08-17T17:40:37.015017+0000","last_undegraded":"2024-08-17T17:40:37.015017+0000","last_fullsized":"2024-08-17T17:40:37.015017+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:53.405657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":928,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.978151+0000","last_change":"2024-08-17T17:35:56.223396+0000","last_active":"2024-08-17T17:40:32.978151+0000","last_peered":"2024-08-17T17:40:32.978151+0000","last_clean":"2024-08-17T17:40:32.978151+0000","last_became_active":"2024-08-17T17:35:56.212413+0000","last_became_peered":"2024-08-17T17:35:56.212413+0000","last_unstale":"2024-08-17T17:40:32.978151+0000","last_undegraded":"2024-08-17T17:40:32.978151+0000","last_fullsized":"2024-08-17T17:40:32.978151+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:44.185174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1655,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.007697+0000","last_change":"2024-08-17T17:35:33.136068+0000","last_active":"2024-08-17T17:40:37.007697+0000","last_peered":"2024-08-17T17:40:37.007697+0000","last_clean":"2024-08-17T17:40:37.007697+0000","last_became_active":"2024-08-17T17:35:33.135591+0000","last_became_peered":"2024-08-17T17:35:33.135591+0000","last_unstale":"2024-08-17T17:40:37.007697+0000","last_undegraded":"2024-08-17T17:40:37.007697+0000","last_fullsized":"2024-08-17T17:40:37.007697+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:52.252561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1793,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.003056+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:40:33.003056+0000","last_peered":"2024-08-17T17:40:33.003056+0000","last_clean":"2024-08-17T17:40:33.003056+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:40:33.003056+0000","last_undegraded":"2024-08-17T17:40:33.003056+0000","last_fullsized":"2024-08-17T17:40:33.003056+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.c","version":"1433'8046","reported_seq":8202,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.416006+0000","last_change":"2024-08-17T17:35:14.861823+0000","last_active":"2024-08-17T17:40:37.416006+0000","last_peered":"2024-08-17T17:40:37.416006+0000","last_clean":"2024-08-17T17:40:37.416006+0000","last_became_active":"2024-08-17T17:35:14.854109+0000","last_became_peered":"2024-08-17T17:35:14.854109+0000","last_unstale":"2024-08-17T17:40:37.416006+0000","last_undegraded":"2024-08-17T17:40:37.416006+0000","last_fullsized":"2024-08-17T17:40:37.416006+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":8046,"log_dups_size":0,"ondisk_log_size":8046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:46:40.778197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8046,"num_write_kb":257472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1006,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.979886+0000","last_change":"2024-08-17T17:35:33.138467+0000","last_active":"2024-08-17T17:40:32.979886+0000","last_peered":"2024-08-17T17:40:32.979886+0000","last_clean":"2024-08-17T17:40:32.979886+0000","last_became_active":"2024-08-17T17:35:33.138174+0000","last_became_peered":"2024-08-17T17:35:33.138174+0000","last_unstale":"2024-08-17T17:40:32.979886+0000","last_undegraded":"2024-08-17T17:40:32.979886+0000","last_fullsized":"2024-08-17T17:40:32.979886+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:08.860439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1390,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.008837+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:40:33.008837+0000","last_peered":"2024-08-17T17:40:33.008837+0000","last_clean":"2024-08-17T17:40:33.008837+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:40:33.008837+0000","last_undegraded":"2024-08-17T17:40:33.008837+0000","last_fullsized":"2024-08-17T17:40:33.008837+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":151,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.018202+0000","last_change":"2024-08-17T17:35:14.854881+0000","last_active":"2024-08-17T17:40:37.018202+0000","last_peered":"2024-08-17T17:40:37.018202+0000","last_clean":"2024-08-17T17:40:37.018202+0000","last_became_active":"2024-08-17T17:35:14.854121+0000","last_became_peered":"2024-08-17T17:35:14.854121+0000","last_unstale":"2024-08-17T17:40:37.018202+0000","last_undegraded":"2024-08-17T17:40:37.018202+0000","last_fullsized":"2024-08-17T17:40:37.018202+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.863624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":947,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.978325+0000","last_change":"2024-08-17T17:36:15.773050+0000","last_active":"2024-08-17T17:40:32.978325+0000","last_peered":"2024-08-17T17:40:32.978325+0000","last_clean":"2024-08-17T17:40:32.978325+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:40:32.978325+0000","last_undegraded":"2024-08-17T17:40:32.978325+0000","last_fullsized":"2024-08-17T17:40:32.978325+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:39.212195+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:39.212195+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.d","version":"1433'8032","reported_seq":13894,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.418073+0000","last_change":"2024-08-17T17:37:21.409979+0000","last_active":"2024-08-17T17:40:37.418073+0000","last_peered":"2024-08-17T17:40:37.418073+0000","last_clean":"2024-08-17T17:40:37.418073+0000","last_became_active":"2024-08-17T17:37:21.398111+0000","last_became_peered":"2024-08-17T17:37:21.398111+0000","last_unstale":"2024-08-17T17:40:37.418073+0000","last_undegraded":"2024-08-17T17:40:37.418073+0000","last_fullsized":"2024-08-17T17:40:37.418073+0000","mapping_epoch":1407,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":8032,"log_dups_size":0,"ondisk_log_size":8032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:19:19.299395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8032,"num_write_kb":256961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2850,"num_bytes_recovered":168820736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1049,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.001031+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:40:37.001031+0000","last_peered":"2024-08-17T17:40:37.001031+0000","last_clean":"2024-08-17T17:40:37.001031+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:40:37.001031+0000","last_undegraded":"2024-08-17T17:40:37.001031+0000","last_fullsized":"2024-08-17T17:40:37.001031+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":245,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.016185+0000","last_change":"2024-08-17T17:35:33.145351+0000","last_active":"2024-08-17T17:40:37.016185+0000","last_peered":"2024-08-17T17:40:37.016185+0000","last_clean":"2024-08-17T17:40:37.016185+0000","last_became_active":"2024-08-17T17:35:33.145040+0000","last_became_peered":"2024-08-17T17:35:33.145040+0000","last_unstale":"2024-08-17T17:40:37.016185+0000","last_undegraded":"2024-08-17T17:40:37.016185+0000","last_fullsized":"2024-08-17T17:40:37.016185+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:02.124000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1458,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.980325+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:40:32.980325+0000","last_peered":"2024-08-17T17:40:32.980325+0000","last_clean":"2024-08-17T17:40:32.980325+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:40:32.980325+0000","last_undegraded":"2024-08-17T17:40:32.980325+0000","last_fullsized":"2024-08-17T17:40:32.980325+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":930,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.978271+0000","last_change":"2024-08-17T17:36:15.769890+0000","last_active":"2024-08-17T17:40:32.978271+0000","last_peered":"2024-08-17T17:40:32.978271+0000","last_clean":"2024-08-17T17:40:32.978271+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:40:32.978271+0000","last_undegraded":"2024-08-17T17:40:32.978271+0000","last_fullsized":"2024-08-17T17:40:32.978271+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:33.034306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:33.034306+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":18,"num_write":4,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.b","version":"1433'7954","reported_seq":15776,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.358692+0000","last_change":"2024-08-17T17:38:07.888612+0000","last_active":"2024-08-17T17:40:37.358692+0000","last_peered":"2024-08-17T17:40:37.358692+0000","last_clean":"2024-08-17T17:40:37.358692+0000","last_became_active":"2024-08-17T17:38:07.875196+0000","last_became_peered":"2024-08-17T17:38:07.875196+0000","last_unstale":"2024-08-17T17:40:37.358692+0000","last_undegraded":"2024-08-17T17:40:37.358692+0000","last_fullsized":"2024-08-17T17:40:37.358692+0000","mapping_epoch":1415,"log_start":"1374'2000","ondisk_log_start":"1374'2000","created":1357,"last_epoch_clean":1416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5954,"log_dups_size":0,"ondisk_log_size":5954,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:00:21.658004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7954,"num_write_kb":254528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3836,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1506,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.018618+0000","last_change":"2024-08-17T17:36:12.759743+0000","last_active":"2024-08-17T17:40:37.018618+0000","last_peered":"2024-08-17T17:40:37.018618+0000","last_clean":"2024-08-17T17:40:37.018618+0000","last_became_active":"2024-08-17T17:36:12.759334+0000","last_became_peered":"2024-08-17T17:36:12.759334+0000","last_unstale":"2024-08-17T17:40:37.018618+0000","last_undegraded":"2024-08-17T17:40:37.018618+0000","last_fullsized":"2024-08-17T17:40:37.018618+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:04.524384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1421,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002814+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:40:33.002814+0000","last_peered":"2024-08-17T17:40:33.002814+0000","last_clean":"2024-08-17T17:40:33.002814+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:40:33.002814+0000","last_undegraded":"2024-08-17T17:40:33.002814+0000","last_fullsized":"2024-08-17T17:40:33.002814+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1464,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.979963+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:40:32.979963+0000","last_peered":"2024-08-17T17:40:32.979963+0000","last_clean":"2024-08-17T17:40:32.979963+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:40:32.979963+0000","last_undegraded":"2024-08-17T17:40:32.979963+0000","last_fullsized":"2024-08-17T17:40:32.979963+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1779,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.979703+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:40:32.979703+0000","last_peered":"2024-08-17T17:40:32.979703+0000","last_clean":"2024-08-17T17:40:32.979703+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:40:32.979703+0000","last_undegraded":"2024-08-17T17:40:32.979703+0000","last_fullsized":"2024-08-17T17:40:32.979703+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.8","version":"1433'8188","reported_seq":19124,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.411526+0000","last_change":"2024-08-17T17:40:33.000352+0000","last_active":"2024-08-17T17:40:37.411526+0000","last_peered":"2024-08-17T17:40:37.411526+0000","last_clean":"2024-08-17T17:40:37.411526+0000","last_became_active":"2024-08-17T17:40:32.982544+0000","last_became_peered":"2024-08-17T17:40:32.982544+0000","last_unstale":"2024-08-17T17:40:37.411526+0000","last_undegraded":"2024-08-17T17:40:37.411526+0000","last_fullsized":"2024-08-17T17:40:37.411526+0000","mapping_epoch":1431,"log_start":"1374'2400","ondisk_log_start":"1374'2400","created":1357,"last_epoch_clean":1432,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5788,"log_dups_size":0,"ondisk_log_size":5788,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:15:09.043902+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8188,"num_write_kb":262016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3968,"num_bytes_recovered":258998272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1097,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.980205+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:40:32.980205+0000","last_peered":"2024-08-17T17:40:32.980205+0000","last_clean":"2024-08-17T17:40:32.980205+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:40:32.980205+0000","last_undegraded":"2024-08-17T17:40:32.980205+0000","last_fullsized":"2024-08-17T17:40:32.980205+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1418,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.005119+0000","last_change":"2024-08-17T17:35:56.229622+0000","last_active":"2024-08-17T17:40:37.005119+0000","last_peered":"2024-08-17T17:40:37.005119+0000","last_clean":"2024-08-17T17:40:37.005119+0000","last_became_active":"2024-08-17T17:35:56.229324+0000","last_became_peered":"2024-08-17T17:35:56.229324+0000","last_unstale":"2024-08-17T17:40:37.005119+0000","last_undegraded":"2024-08-17T17:40:37.005119+0000","last_fullsized":"2024-08-17T17:40:37.005119+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:13.980316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2021,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.005202+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:40:37.005202+0000","last_peered":"2024-08-17T17:40:37.005202+0000","last_clean":"2024-08-17T17:40:37.005202+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:40:37.005202+0000","last_undegraded":"2024-08-17T17:40:37.005202+0000","last_fullsized":"2024-08-17T17:40:37.005202+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1593,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002218+0000","last_change":"2024-08-17T17:35:33.137074+0000","last_active":"2024-08-17T17:40:33.002218+0000","last_peered":"2024-08-17T17:40:33.002218+0000","last_clean":"2024-08-17T17:40:33.002218+0000","last_became_active":"2024-08-17T17:35:33.136759+0000","last_became_peered":"2024-08-17T17:35:33.136759+0000","last_unstale":"2024-08-17T17:40:33.002218+0000","last_undegraded":"2024-08-17T17:40:33.002218+0000","last_fullsized":"2024-08-17T17:40:33.002218+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:37.263639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:37.263639+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:14.599780+0000","scrub_duration":0.0016181800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.9","version":"1433'8100","reported_seq":16157,"reported_epoch":1433,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:37.404963+0000","last_change":"2024-08-17T17:36:23.019129+0000","last_active":"2024-08-17T17:40:37.404963+0000","last_peered":"2024-08-17T17:40:37.404963+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:23.013527+0000","last_became_peered":"2024-08-17T17:36:23.013527+0000","last_unstale":"2024-08-17T17:40:37.404963+0000","last_undegraded":"2024-08-17T17:36:13.874715+0000","last_fullsized":"2024-08-17T17:36:13.738828+0000","mapping_epoch":1383,"log_start":"1373'1100","ondisk_log_start":"1373'1100","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":7000,"log_dups_size":0,"ondisk_log_size":7000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1785,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8100,"num_write_kb":259200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2876,"num_bytes_recovered":188481536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[0,6],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,6","objects":-1785}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1210,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.007782+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:40:37.007782+0000","last_peered":"2024-08-17T17:40:37.007782+0000","last_clean":"2024-08-17T17:40:37.007782+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:40:37.007782+0000","last_undegraded":"2024-08-17T17:40:37.007782+0000","last_fullsized":"2024-08-17T17:40:37.007782+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":58,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.001932+0000","last_change":"2024-08-17T17:36:17.774773+0000","last_active":"2024-08-17T17:40:33.001932+0000","last_peered":"2024-08-17T17:40:33.001932+0000","last_clean":"2024-08-17T17:40:33.001932+0000","last_became_active":"2024-08-17T17:36:17.774468+0000","last_became_peered":"2024-08-17T17:36:17.774468+0000","last_unstale":"2024-08-17T17:40:33.001932+0000","last_undegraded":"2024-08-17T17:40:33.001932+0000","last_fullsized":"2024-08-17T17:40:33.001932+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:54.231747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1549,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.006178+0000","last_change":"2024-08-17T17:35:33.137633+0000","last_active":"2024-08-17T17:40:37.006178+0000","last_peered":"2024-08-17T17:40:37.006178+0000","last_clean":"2024-08-17T17:40:37.006178+0000","last_became_active":"2024-08-17T17:35:33.137323+0000","last_became_peered":"2024-08-17T17:35:33.137323+0000","last_unstale":"2024-08-17T17:40:37.006178+0000","last_undegraded":"2024-08-17T17:40:37.006178+0000","last_fullsized":"2024-08-17T17:40:37.006178+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:03.359155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":909,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.977934+0000","last_change":"2024-08-17T17:35:56.264626+0000","last_active":"2024-08-17T17:40:32.977934+0000","last_peered":"2024-08-17T17:40:32.977934+0000","last_clean":"2024-08-17T17:40:32.977934+0000","last_became_active":"2024-08-17T17:35:56.264265+0000","last_became_peered":"2024-08-17T17:35:56.264265+0000","last_unstale":"2024-08-17T17:40:32.977934+0000","last_undegraded":"2024-08-17T17:40:32.977934+0000","last_fullsized":"2024-08-17T17:40:32.977934+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:20.754326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:20.754326+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":25,"num_write":6,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.a","version":"1433'8014","reported_seq":16558,"reported_epoch":1433,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:37.406456+0000","last_change":"2024-08-17T17:37:11.134558+0000","last_active":"2024-08-17T17:40:37.406456+0000","last_peered":"2024-08-17T17:40:37.406456+0000","last_clean":"2024-08-17T17:34:49.643517+0000","last_became_active":"2024-08-17T17:36:20.105110+0000","last_became_peered":"2024-08-17T17:36:20.105110+0000","last_unstale":"2024-08-17T17:40:37.406456+0000","last_undegraded":"2024-08-17T17:36:19.999272+0000","last_fullsized":"2024-08-17T17:36:13.740520+0000","mapping_epoch":1383,"log_start":"1374'2200","ondisk_log_start":"1374'2200","created":1357,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5814,"log_dups_size":0,"ondisk_log_size":5814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:03:45.953593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1036,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8014,"num_write_kb":256448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2817,"num_bytes_recovered":184614912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"5,6","objects":-1036}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1262,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.978622+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:40:32.978622+0000","last_peered":"2024-08-17T17:40:32.978622+0000","last_clean":"2024-08-17T17:40:32.978622+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:40:32.978622+0000","last_undegraded":"2024-08-17T17:40:32.978622+0000","last_fullsized":"2024-08-17T17:40:32.978622+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1802,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.001170+0000","last_change":"2024-08-17T17:36:12.758005+0000","last_active":"2024-08-17T17:40:37.001170+0000","last_peered":"2024-08-17T17:40:37.001170+0000","last_clean":"2024-08-17T17:40:37.001170+0000","last_became_active":"2024-08-17T17:36:12.757695+0000","last_became_peered":"2024-08-17T17:36:12.757695+0000","last_unstale":"2024-08-17T17:40:37.001170+0000","last_undegraded":"2024-08-17T17:40:37.001170+0000","last_fullsized":"2024-08-17T17:40:37.001170+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:51.792220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1440,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.979908+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:40:32.979908+0000","last_peered":"2024-08-17T17:40:32.979908+0000","last_clean":"2024-08-17T17:40:32.979908+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:40:32.979908+0000","last_undegraded":"2024-08-17T17:40:32.979908+0000","last_fullsized":"2024-08-17T17:40:32.979908+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1771,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:47.444410+0000","last_change":"2024-08-17T17:40:50.736318+0000","last_active":"2024-08-17T17:40:47.444410+0000","last_peered":"2024-08-17T17:40:47.444410+0000","last_clean":"2024-08-17T17:40:47.444410+0000","last_became_active":"2024-08-17T17:35:55.204861+0000","last_became_peered":"2024-08-17T17:35:55.204861+0000","last_unstale":"2024-08-17T17:40:47.444410+0000","last_undegraded":"2024-08-17T17:40:47.444410+0000","last_fullsized":"2024-08-17T17:40:47.444410+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.542545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.542545+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-17T17:40:52.445115+0000","scrub_duration":0.0027678550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.0","version":"1433'8116","reported_seq":16126,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.399703+0000","last_change":"2024-08-17T17:37:40.753703+0000","last_active":"2024-08-17T17:40:37.399703+0000","last_peered":"2024-08-17T17:40:37.399703+0000","last_clean":"2024-08-17T17:40:37.399703+0000","last_became_active":"2024-08-17T17:37:40.743458+0000","last_became_peered":"2024-08-17T17:37:40.743458+0000","last_unstale":"2024-08-17T17:40:37.399703+0000","last_undegraded":"2024-08-17T17:40:37.399703+0000","last_fullsized":"2024-08-17T17:40:37.399703+0000","mapping_epoch":1411,"log_start":"1374'2600","ondisk_log_start":"1374'2600","created":1357,"last_epoch_clean":1412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5516,"log_dups_size":0,"ondisk_log_size":5516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:14:02.340380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8116,"num_write_kb":259712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3914,"num_bytes_recovered":232652800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1035,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002452+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:40:33.002452+0000","last_peered":"2024-08-17T17:40:33.002452+0000","last_clean":"2024-08-17T17:40:33.002452+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:40:33.002452+0000","last_undegraded":"2024-08-17T17:40:33.002452+0000","last_fullsized":"2024-08-17T17:40:33.002452+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":114,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.015180+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:40:37.015180+0000","last_peered":"2024-08-17T17:40:37.015180+0000","last_clean":"2024-08-17T17:40:37.015180+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:40:37.015180+0000","last_undegraded":"2024-08-17T17:40:37.015180+0000","last_fullsized":"2024-08-17T17:40:37.015180+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1771,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.979782+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:40:32.979782+0000","last_peered":"2024-08-17T17:40:32.979782+0000","last_clean":"2024-08-17T17:40:32.979782+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:40:32.979782+0000","last_undegraded":"2024-08-17T17:40:32.979782+0000","last_fullsized":"2024-08-17T17:40:32.979782+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.1","version":"1433'8052","reported_seq":16651,"reported_epoch":1433,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:37.386017+0000","last_change":"2024-08-17T17:36:23.103523+0000","last_active":"2024-08-17T17:40:37.386017+0000","last_peered":"2024-08-17T17:40:37.386017+0000","last_clean":"2024-08-17T17:35:54.178280+0000","last_became_active":"2024-08-17T17:36:23.025910+0000","last_became_peered":"2024-08-17T17:36:23.025910+0000","last_unstale":"2024-08-17T17:40:37.386017+0000","last_undegraded":"2024-08-17T17:36:18.734637+0000","last_fullsized":"2024-08-17T17:36:13.740134+0000","mapping_epoch":1383,"log_start":"1374'2200","ondisk_log_start":"1374'2200","created":1357,"last_epoch_clean":1365,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5852,"log_dups_size":0,"ondisk_log_size":5852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:06:58.717055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1071,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8052,"num_write_kb":257664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2833,"num_bytes_recovered":185663488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,0],"avail_no_missing":["6","0","2","4"],"object_location_counts":[{"shards":"0,6","objects":-1071}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1449,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.003104+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:40:33.003104+0000","last_peered":"2024-08-17T17:40:33.003104+0000","last_clean":"2024-08-17T17:40:33.003104+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:40:33.003104+0000","last_undegraded":"2024-08-17T17:40:33.003104+0000","last_fullsized":"2024-08-17T17:40:33.003104+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1422,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002551+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:40:33.002551+0000","last_peered":"2024-08-17T17:40:33.002551+0000","last_clean":"2024-08-17T17:40:33.002551+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:40:33.002551+0000","last_undegraded":"2024-08-17T17:40:33.002551+0000","last_fullsized":"2024-08-17T17:40:33.002551+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":1015,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.008877+0000","last_change":"2024-08-17T17:34:26.817339+0000","last_active":"2024-08-17T17:40:37.008877+0000","last_peered":"2024-08-17T17:40:37.008877+0000","last_clean":"2024-08-17T17:40:37.008877+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:40:37.008877+0000","last_undegraded":"2024-08-17T17:40:37.008877+0000","last_fullsized":"2024-08-17T17:40:37.008877+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:26.817223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:26.817223+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:30.089956+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"11.2","version":"1433'7902","reported_seq":12740,"reported_epoch":1433,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-08-17T17:40:50.731019+0000","last_change":"2024-08-17T17:40:31.635354+0000","last_active":"2024-08-17T17:40:50.731019+0000","last_peered":"2024-08-17T17:40:50.731019+0000","last_clean":"2024-08-17T17:36:11.734683+0000","last_became_active":"2024-08-17T17:36:14.063794+0000","last_became_peered":"2024-08-17T17:36:14.063794+0000","last_unstale":"2024-08-17T17:40:50.731019+0000","last_undegraded":"2024-08-17T17:36:13.855770+0000","last_fullsized":"2024-08-17T17:36:13.742068+0000","mapping_epoch":1383,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":7902,"log_dups_size":0,"ondisk_log_size":7902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1491,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7902,"num_write_kb":252864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2339,"num_bytes_recovered":6881280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-1491}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":58,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.006683+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:40:37.006683+0000","last_peered":"2024-08-17T17:40:37.006683+0000","last_clean":"2024-08-17T17:40:37.006683+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:40:37.006683+0000","last_undegraded":"2024-08-17T17:40:37.006683+0000","last_fullsized":"2024-08-17T17:40:37.006683+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1284,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.978202+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:40:32.978202+0000","last_peered":"2024-08-17T17:40:32.978202+0000","last_clean":"2024-08-17T17:40:32.978202+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:40:32.978202+0000","last_undegraded":"2024-08-17T17:40:32.978202+0000","last_fullsized":"2024-08-17T17:40:32.978202+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1799,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.980094+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:40:32.980094+0000","last_peered":"2024-08-17T17:40:32.980094+0000","last_clean":"2024-08-17T17:40:32.980094+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:40:32.980094+0000","last_undegraded":"2024-08-17T17:40:32.980094+0000","last_fullsized":"2024-08-17T17:40:32.980094+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"11.3","version":"1433'8072","reported_seq":13880,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.388078+0000","last_change":"2024-08-17T17:36:46.982456+0000","last_active":"2024-08-17T17:40:37.388078+0000","last_peered":"2024-08-17T17:40:37.388078+0000","last_clean":"2024-08-17T17:40:37.388078+0000","last_became_active":"2024-08-17T17:36:46.974782+0000","last_became_peered":"2024-08-17T17:36:46.974782+0000","last_unstale":"2024-08-17T17:40:37.388078+0000","last_undegraded":"2024-08-17T17:40:37.388078+0000","last_fullsized":"2024-08-17T17:40:37.388078+0000","mapping_epoch":1399,"log_start":"1374'2300","ondisk_log_start":"1374'2300","created":1357,"last_epoch_clean":1400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5772,"log_dups_size":0,"ondisk_log_size":5772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:55:40.499321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8072,"num_write_kb":258304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2848,"num_bytes_recovered":186646528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1400,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002468+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:40:33.002468+0000","last_peered":"2024-08-17T17:40:33.002468+0000","last_clean":"2024-08-17T17:40:33.002468+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:40:33.002468+0000","last_undegraded":"2024-08-17T17:40:33.002468+0000","last_fullsized":"2024-08-17T17:40:33.002468+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":214,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002361+0000","last_change":"2024-08-17T17:36:12.754372+0000","last_active":"2024-08-17T17:40:33.002361+0000","last_peered":"2024-08-17T17:40:33.002361+0000","last_clean":"2024-08-17T17:40:33.002361+0000","last_became_active":"2024-08-17T17:36:12.746352+0000","last_became_peered":"2024-08-17T17:36:12.746352+0000","last_unstale":"2024-08-17T17:40:33.002361+0000","last_undegraded":"2024-08-17T17:40:33.002361+0000","last_fullsized":"2024-08-17T17:40:33.002361+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:07.807486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1687,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.004367+0000","last_change":"2024-08-17T17:36:15.771193+0000","last_active":"2024-08-17T17:40:37.004367+0000","last_peered":"2024-08-17T17:40:37.004367+0000","last_clean":"2024-08-17T17:40:37.004367+0000","last_became_active":"2024-08-17T17:36:15.770889+0000","last_became_peered":"2024-08-17T17:36:15.770889+0000","last_unstale":"2024-08-17T17:40:37.004367+0000","last_undegraded":"2024-08-17T17:40:37.004367+0000","last_fullsized":"2024-08-17T17:40:37.004367+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:22.978695+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"11.4","version":"1433'8066","reported_seq":13855,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.414900+0000","last_change":"2024-08-17T17:37:04.045576+0000","last_active":"2024-08-17T17:40:37.414900+0000","last_peered":"2024-08-17T17:40:37.414900+0000","last_clean":"2024-08-17T17:40:37.414900+0000","last_became_active":"2024-08-17T17:37:04.038810+0000","last_became_peered":"2024-08-17T17:37:04.038810+0000","last_unstale":"2024-08-17T17:40:37.414900+0000","last_undegraded":"2024-08-17T17:40:37.414900+0000","last_fullsized":"2024-08-17T17:40:37.414900+0000","mapping_epoch":1402,"log_start":"1374'2200","ondisk_log_start":"1374'2200","created":1357,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":5866,"log_dups_size":0,"ondisk_log_size":5866,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:27:04.725258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8066,"num_write_kb":258112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":176291840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1282,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.010052+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:40:37.010052+0000","last_peered":"2024-08-17T17:40:37.010052+0000","last_clean":"2024-08-17T17:40:37.010052+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:40:37.010052+0000","last_undegraded":"2024-08-17T17:40:37.010052+0000","last_fullsized":"2024-08-17T17:40:37.010052+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1427,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.001282+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:40:37.001282+0000","last_peered":"2024-08-17T17:40:37.001282+0000","last_clean":"2024-08-17T17:40:37.001282+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:40:37.001282+0000","last_undegraded":"2024-08-17T17:40:37.001282+0000","last_fullsized":"2024-08-17T17:40:37.001282+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":927,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002720+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:40:33.002720+0000","last_peered":"2024-08-17T17:40:33.002720+0000","last_clean":"2024-08-17T17:40:33.002720+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:40:33.002720+0000","last_undegraded":"2024-08-17T17:40:33.002720+0000","last_fullsized":"2024-08-17T17:40:33.002720+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"11.5","version":"1433'8108","reported_seq":13952,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.417969+0000","last_change":"2024-08-17T17:36:32.711490+0000","last_active":"2024-08-17T17:40:37.417969+0000","last_peered":"2024-08-17T17:40:37.417969+0000","last_clean":"2024-08-17T17:40:37.417969+0000","last_became_active":"2024-08-17T17:36:32.704370+0000","last_became_peered":"2024-08-17T17:36:32.704370+0000","last_unstale":"2024-08-17T17:40:37.417969+0000","last_undegraded":"2024-08-17T17:40:37.417969+0000","last_fullsized":"2024-08-17T17:40:37.417969+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":8108,"log_dups_size":0,"ondisk_log_size":8108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:54:17.373879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8108,"num_write_kb":259456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2866,"num_bytes_recovered":187826176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1283,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.978428+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:40:32.978428+0000","last_peered":"2024-08-17T17:40:32.978428+0000","last_clean":"2024-08-17T17:40:32.978428+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:40:32.978428+0000","last_undegraded":"2024-08-17T17:40:32.978428+0000","last_fullsized":"2024-08-17T17:40:32.978428+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":163,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.980244+0000","last_change":"2024-08-17T17:35:55.217072+0000","last_active":"2024-08-17T17:40:32.980244+0000","last_peered":"2024-08-17T17:40:32.980244+0000","last_clean":"2024-08-17T17:40:32.980244+0000","last_became_active":"2024-08-17T17:35:55.216462+0000","last_became_peered":"2024-08-17T17:35:55.216462+0000","last_unstale":"2024-08-17T17:40:32.980244+0000","last_undegraded":"2024-08-17T17:40:32.980244+0000","last_fullsized":"2024-08-17T17:40:32.980244+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:34.937221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":910,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.978418+0000","last_change":"2024-08-17T17:35:55.215004+0000","last_active":"2024-08-17T17:40:32.978418+0000","last_peered":"2024-08-17T17:40:32.978418+0000","last_clean":"2024-08-17T17:40:32.978418+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:40:32.978418+0000","last_undegraded":"2024-08-17T17:40:32.978418+0000","last_fullsized":"2024-08-17T17:40:32.978418+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.192293+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.192293+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015613109999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"11.6","version":"1433'7816","reported_seq":7973,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.397898+0000","last_change":"2024-08-17T17:35:14.868524+0000","last_active":"2024-08-17T17:40:37.397898+0000","last_peered":"2024-08-17T17:40:37.397898+0000","last_clean":"2024-08-17T17:40:37.397898+0000","last_became_active":"2024-08-17T17:35:14.854690+0000","last_became_peered":"2024-08-17T17:35:14.854690+0000","last_unstale":"2024-08-17T17:40:37.397898+0000","last_undegraded":"2024-08-17T17:40:37.397898+0000","last_fullsized":"2024-08-17T17:40:37.397898+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":7816,"log_dups_size":0,"ondisk_log_size":7816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:05.388362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7816,"num_write_kb":250112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1468,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002361+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:40:33.002361+0000","last_peered":"2024-08-17T17:40:33.002361+0000","last_clean":"2024-08-17T17:40:33.002361+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:40:33.002361+0000","last_undegraded":"2024-08-17T17:40:33.002361+0000","last_fullsized":"2024-08-17T17:40:33.002361+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":175,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.003176+0000","last_change":"2024-08-17T17:35:33.147672+0000","last_active":"2024-08-17T17:40:33.003176+0000","last_peered":"2024-08-17T17:40:33.003176+0000","last_clean":"2024-08-17T17:40:33.003176+0000","last_became_active":"2024-08-17T17:35:33.147283+0000","last_became_peered":"2024-08-17T17:35:33.147283+0000","last_unstale":"2024-08-17T17:40:33.003176+0000","last_undegraded":"2024-08-17T17:40:33.003176+0000","last_fullsized":"2024-08-17T17:40:33.003176+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:41.677709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1705,"reported_epoch":1433,"state":"active+clean+scrubbing","last_fresh":"2024-08-17T17:40:45.357241+0000","last_change":"2024-08-17T17:40:40.274305+0000","last_active":"2024-08-17T17:40:45.357241+0000","last_peered":"2024-08-17T17:40:45.357241+0000","last_clean":"2024-08-17T17:40:45.357241+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:40:45.357241+0000","last_undegraded":"2024-08-17T17:40:45.357241+0000","last_fullsized":"2024-08-17T17:40:45.357241+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:38.241235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:38.241235+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":21,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"11.7","version":"1433'7926","reported_seq":8012,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.396969+0000","last_change":"2024-08-17T17:34:50.670013+0000","last_active":"2024-08-17T17:40:37.396969+0000","last_peered":"2024-08-17T17:40:37.396969+0000","last_clean":"2024-08-17T17:40:37.396969+0000","last_became_active":"2024-08-17T17:34:50.669299+0000","last_became_peered":"2024-08-17T17:34:50.669299+0000","last_unstale":"2024-08-17T17:40:37.396969+0000","last_undegraded":"2024-08-17T17:40:37.396969+0000","last_fullsized":"2024-08-17T17:40:37.396969+0000","mapping_epoch":1357,"log_start":"0'0","ondisk_log_start":"0'0","created":1357,"last_epoch_clean":1358,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T17:34:49.643517+0000","last_clean_scrub_stamp":"2024-08-17T17:34:49.643517+0000","objects_scrubbed":0,"log_size":7926,"log_dups_size":0,"ondisk_log_size":7926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:47:11.172227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7926,"num_write_kb":253632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1392,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.980318+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:40:32.980318+0000","last_peered":"2024-08-17T17:40:32.980318+0000","last_clean":"2024-08-17T17:40:32.980318+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:40:32.980318+0000","last_undegraded":"2024-08-17T17:40:32.980318+0000","last_fullsized":"2024-08-17T17:40:32.980318+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":162,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.004958+0000","last_change":"2024-08-17T17:35:33.140725+0000","last_active":"2024-08-17T17:40:33.004958+0000","last_peered":"2024-08-17T17:40:33.004958+0000","last_clean":"2024-08-17T17:40:33.004958+0000","last_became_active":"2024-08-17T17:35:33.140406+0000","last_became_peered":"2024-08-17T17:35:33.140406+0000","last_unstale":"2024-08-17T17:40:33.004958+0000","last_undegraded":"2024-08-17T17:40:33.004958+0000","last_fullsized":"2024-08-17T17:40:33.004958+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:54:51.875571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1682,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.003254+0000","last_change":"2024-08-17T17:35:33.138514+0000","last_active":"2024-08-17T17:40:33.003254+0000","last_peered":"2024-08-17T17:40:33.003254+0000","last_clean":"2024-08-17T17:40:33.003254+0000","last_became_active":"2024-08-17T17:35:33.137035+0000","last_became_peered":"2024-08-17T17:35:33.137035+0000","last_unstale":"2024-08-17T17:40:33.003254+0000","last_undegraded":"2024-08-17T17:40:33.003254+0000","last_fullsized":"2024-08-17T17:40:33.003254+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:50.565662+0000","scrub_duration":0.001487138,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1324,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.977777+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:40:32.977777+0000","last_peered":"2024-08-17T17:40:32.977777+0000","last_clean":"2024-08-17T17:40:32.977777+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:40:32.977777+0000","last_undegraded":"2024-08-17T17:40:32.977777+0000","last_fullsized":"2024-08-17T17:40:32.977777+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":113,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.016251+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:40:37.016251+0000","last_peered":"2024-08-17T17:40:37.016251+0000","last_clean":"2024-08-17T17:40:37.016251+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:40:37.016251+0000","last_undegraded":"2024-08-17T17:40:37.016251+0000","last_fullsized":"2024-08-17T17:40:37.016251+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1755,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.000960+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:40:37.000960+0000","last_peered":"2024-08-17T17:40:37.000960+0000","last_clean":"2024-08-17T17:40:37.000960+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:40:37.000960+0000","last_undegraded":"2024-08-17T17:40:37.000960+0000","last_fullsized":"2024-08-17T17:40:37.000960+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1341,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.002879+0000","last_change":"2024-08-17T17:35:55.205292+0000","last_active":"2024-08-17T17:40:33.002879+0000","last_peered":"2024-08-17T17:40:33.002879+0000","last_clean":"2024-08-17T17:40:33.002879+0000","last_became_active":"2024-08-17T17:35:55.204935+0000","last_became_peered":"2024-08-17T17:35:55.204935+0000","last_unstale":"2024-08-17T17:40:33.002879+0000","last_undegraded":"2024-08-17T17:40:33.002879+0000","last_fullsized":"2024-08-17T17:40:33.002879+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:52.277821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":113,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.016263+0000","last_change":"2024-08-17T17:36:12.749972+0000","last_active":"2024-08-17T17:40:37.016263+0000","last_peered":"2024-08-17T17:40:37.016263+0000","last_clean":"2024-08-17T17:40:37.016263+0000","last_became_active":"2024-08-17T17:36:12.749620+0000","last_became_peered":"2024-08-17T17:36:12.749620+0000","last_unstale":"2024-08-17T17:40:37.016263+0000","last_undegraded":"2024-08-17T17:40:37.016263+0000","last_fullsized":"2024-08-17T17:40:37.016263+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:04.994357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":989,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.015872+0000","last_change":"2024-08-17T17:36:20.832274+0000","last_active":"2024-08-17T17:40:37.015872+0000","last_peered":"2024-08-17T17:40:37.015872+0000","last_clean":"2024-08-17T17:40:37.015872+0000","last_became_active":"2024-08-17T17:36:20.831886+0000","last_became_peered":"2024-08-17T17:36:20.831886+0000","last_unstale":"2024-08-17T17:40:37.015872+0000","last_undegraded":"2024-08-17T17:40:37.015872+0000","last_fullsized":"2024-08-17T17:40:37.015872+0000","mapping_epoch":1390,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1391,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:05.248711+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1199,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.006942+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:40:37.006942+0000","last_peered":"2024-08-17T17:40:37.006942+0000","last_clean":"2024-08-17T17:40:37.006942+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:40:37.006942+0000","last_undegraded":"2024-08-17T17:40:37.006942+0000","last_fullsized":"2024-08-17T17:40:37.006942+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1424,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.980236+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:40:32.980236+0000","last_peered":"2024-08-17T17:40:32.980236+0000","last_clean":"2024-08-17T17:40:32.980236+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:40:32.980236+0000","last_undegraded":"2024-08-17T17:40:32.980236+0000","last_fullsized":"2024-08-17T17:40:32.980236+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1718,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.002320+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:40:37.002320+0000","last_peered":"2024-08-17T17:40:37.002320+0000","last_clean":"2024-08-17T17:40:37.002320+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:40:37.002320+0000","last_undegraded":"2024-08-17T17:40:37.002320+0000","last_fullsized":"2024-08-17T17:40:37.002320+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1366,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:33.008446+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:40:33.008446+0000","last_peered":"2024-08-17T17:40:33.008446+0000","last_clean":"2024-08-17T17:40:33.008446+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:40:33.008446+0000","last_undegraded":"2024-08-17T17:40:33.008446+0000","last_fullsized":"2024-08-17T17:40:33.008446+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1918,"reported_epoch":1433,"state":"active+clean","last_fresh":"2024-08-17T17:40:37.015818+0000","last_change":"2024-08-17T17:35:33.137411+0000","last_active":"2024-08-17T17:40:37.015818+0000","last_peered":"2024-08-17T17:40:37.015818+0000","last_clean":"2024-08-17T17:40:37.015818+0000","last_became_active":"2024-08-17T17:35:33.137073+0000","last_became_peered":"2024-08-17T17:35:33.137073+0000","last_unstale":"2024-08-17T17:40:37.015818+0000","last_undegraded":"2024-08-17T17:40:37.015818+0000","last_fullsized":"2024-08-17T17:40:37.015818+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:35.924539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1812,"reported_epoch":1432,"state":"active+clean","last_fresh":"2024-08-17T17:40:32.979658+0000","last_change":"2024-08-17T17:36:15.787445+0000","last_active":"2024-08-17T17:40:32.979658+0000","last_peered":"2024-08-17T17:40:32.979658+0000","last_clean":"2024-08-17T17:40:32.979658+0000","last_became_active":"2024-08-17T17:36:15.787099+0000","last_became_peered":"2024-08-17T17:36:15.787099+0000","last_unstale":"2024-08-17T17:40:32.979658+0000","last_undegraded":"2024-08-17T17:40:32.979658+0000","last_fullsized":"2024-08-17T17:40:32.979658+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:47.448168+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":11,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5383,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":128882,"num_write_kb":4124161,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37996,"num_bytes_recovered":2190016512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1717501952,"data_stored":1717501952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109782,"ondisk_log_size":109782,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":635092,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":27,"num_read_kb":61,"num_write":15,"num_write_kb":183,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1314910,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162881,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":420968,"kb_used_data":252252,"kb_used_omap":54,"kb_used_meta":168649,"kb_avail":93324184,"statfs":{"total":95995035648,"available":95563964416,"internally_reserved":0,"allocated":258306048,"data_stored":255588198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":172696597},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999355,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":314656,"kb_used_data":76248,"kb_used_omap":47,"kb_used_meta":238352,"kb_avail":93430496,"statfs":{"total":95995035648,"available":95672827904,"internally_reserved":0,"allocated":78077952,"data_stored":75365151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":244073108},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627152,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1106824,"kb_used_data":7872,"kb_used_omap":42,"kb_used_meta":1098901,"kb_avail":92638328,"statfs":{"total":95995035648,"available":94861647872,"internally_reserved":0,"allocated":8060928,"data_stored":5366336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1125274684},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785629,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1349712,"kb_used_data":505284,"kb_used_omap":48,"kb_used_meta":844367,"kb_avail":92395440,"statfs":{"total":95995035648,"available":94612930560,"internally_reserved":0,"allocated":517410816,"data_stored":514713099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":864632494},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725146,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":901080,"kb_used_data":7820,"kb_used_omap":1,"kb_used_meta":893246,"kb_avail":92844072,"statfs":{"total":95995035648,"available":95072329728,"internally_reserved":0,"allocated":8007680,"data_stored":5301913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":914684358},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986783,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1064032,"kb_used_data":188820,"kb_used_omap":1,"kb_used_meta":875198,"kb_avail":92681120,"statfs":{"total":95995035648,"available":94905466880,"internally_reserved":0,"allocated":193351680,"data_stored":190641408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":896203206},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706390,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1445652,"kb_used_data":511240,"kb_used_omap":35,"kb_used_meta":934364,"kb_avail":92299500,"statfs":{"total":95995035648,"available":94514688000,"internally_reserved":0,"allocated":523509760,"data_stored":520808046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":956788791},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832163,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1246932,"kb_used_data":189960,"kb_used_omap":65,"kb_used_meta":1056894,"kb_avail":92498220,"statfs":{"total":95995035648,"available":94718177280,"internally_reserved":0,"allocated":194519040,"data_stored":191817812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1082259643},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":186449920,"data_stored":186449920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":509673472,"data_stored":509673472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":515506176,"data_stored":515506176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":185597952,"data_stored":185597952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":69992448,"data_stored":69992448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":11,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":250281984,"data_stored":250281984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:40:56.331 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph status --format=json 2024-08-17T17:40:56.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:56 smithi162 ceph-mon[104668]: pgmap v2303: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 626 KiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 5383/78 objects degraded (6901.282%); 0 B/s, 140 objects/s recovering 2024-08-17T17:40:56.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:56 smithi162 ceph-mon[104668]: from='client.36579 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:40:56.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:56 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_6", "pool2": "unique_pool_6", "yes_i_really_really_mean_it": true}]': finished 2024-08-17T17:40:56.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:56 smithi162 ceph-mon[104668]: osdmap e1434: 8 total, 8 up, 8 in 2024-08-17T17:40:56.399 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:56.431 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:56.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:56.533 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:56.555 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:56.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:56.657 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:56.678 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:56.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:56.780 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:56.802 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:56.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:56.903 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:56.925 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:56.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:56.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:57.026 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:57.048 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:57.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:57.150 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:57.174 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:57.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:57.208 INFO:teuthology.task.sequential:In sequential, running task print... 2024-08-17T17:40:57.208 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-08-17T17:40:57.208 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-08-17T17:40:57.213 INFO:tasks.workunit:Pulling workunits from ref reef 2024-08-17T17:40:57.214 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-08-17T17:40:57.214 DEBUG:teuthology.orchestra.run.smithi063:> stat -- /home/ubuntu/cephtest/mnt.0 2024-08-17T17:40:57.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.273 INFO:teuthology.orchestra.run.smithi063.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-08-17T17:40:57.273 DEBUG:teuthology.orchestra.run.smithi063:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-08-17T17:40:57.275 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:57.321 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:57 smithi063 ceph-mon[114756]: 1.c scrub starts 2024-08-17T17:40:57.322 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:57 smithi063 ceph-mon[114756]: 1.c scrub ok 2024-08-17T17:40:57.322 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:57 smithi063 ceph-mon[114756]: 1.f scrub starts 2024-08-17T17:40:57.322 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:57 smithi063 ceph-mon[114756]: 1.f scrub ok 2024-08-17T17:40:57.322 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:57 smithi063 ceph-mon[109843]: 1.c scrub starts 2024-08-17T17:40:57.322 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:57 smithi063 ceph-mon[109843]: 1.c scrub ok 2024-08-17T17:40:57.322 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:57 smithi063 ceph-mon[109843]: 1.f scrub starts 2024-08-17T17:40:57.322 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:57 smithi063 ceph-mon[109843]: 1.f scrub ok 2024-08-17T17:40:57.335 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:57.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:57.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:57 smithi162 ceph-mon[104668]: 1.c scrub starts 2024-08-17T17:40:57.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:57 smithi162 ceph-mon[104668]: 1.c scrub ok 2024-08-17T17:40:57.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:57 smithi162 ceph-mon[104668]: 1.f scrub starts 2024-08-17T17:40:57.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:57 smithi162 ceph-mon[104668]: 1.f scrub ok 2024-08-17T17:40:57.354 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-08-17T17:40:57.355 DEBUG:teuthology.orchestra.run.smithi063:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-08-17T17:40:57.411 INFO:tasks.workunit:timeout=3h 2024-08-17T17:40:57.411 INFO:tasks.workunit:cleanup=True 2024-08-17T17:40:57.411 DEBUG:teuthology.orchestra.run.smithi063:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch reef https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-08-17T17:40:57.438 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:57.475 INFO:tasks.workunit.client.0.smithi063.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-08-17T17:40:57.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.479 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:57.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:57.579 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:57.603 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:57.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:57.705 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:57.728 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:57.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:57.829 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:40:57.852 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:57.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:40:57.953 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:57.977 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:57.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:57.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:58.041 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:40:58.078 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:58.101 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:58.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:58.203 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:58.225 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:58.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:58.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:58 smithi063 ceph-mon[114756]: pgmap v2305: 103 pgs: 1 active+clean+scrubbing, 102 active+clean; 626 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2024-08-17T17:40:58.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:58 smithi063 ceph-mon[114756]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5383/78 objects degraded (6901.282%), 4 pgs degraded, 4 pgs undersized) 2024-08-17T17:40:58.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:58 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1446612113' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:40:58.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:58 smithi063 ceph-mon[109843]: pgmap v2305: 103 pgs: 1 active+clean+scrubbing, 102 active+clean; 626 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2024-08-17T17:40:58.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:58 smithi063 ceph-mon[109843]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5383/78 objects degraded (6901.282%), 4 pgs degraded, 4 pgs undersized) 2024-08-17T17:40:58.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:58 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1446612113' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:40:58.282 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:58.326 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:58.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:58 smithi162 ceph-mon[104668]: pgmap v2305: 103 pgs: 1 active+clean+scrubbing, 102 active+clean; 626 KiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2024-08-17T17:40:58.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:58 smithi162 ceph-mon[104668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5383/78 objects degraded (6901.282%), 4 pgs degraded, 4 pgs undersized) 2024-08-17T17:40:58.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:58 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1446612113' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:40:58.351 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:58.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:58.453 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:58.478 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:58.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:58.498 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:40:58.579 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:58.603 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:58.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:58.705 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:40:58.730 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:58.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:40:58.831 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:58.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.856 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:58.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:40:58.957 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:58.981 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:58.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:58.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:59.082 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:59.107 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:59.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:59.209 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:59.231 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:59.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:59.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:59 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:40:59.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:59 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:40:59.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:59 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1446612113' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:59.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:40:59 smithi063 ceph-mon[109843]: osdmap e1435: 8 total, 8 up, 8 in 2024-08-17T17:40:59.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:59 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:40:59.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:59 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:40:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:59 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1446612113' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:59.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:40:59 smithi063 ceph-mon[114756]: osdmap e1435: 8 total, 8 up, 8 in 2024-08-17T17:40:59.333 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:59.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:59 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-08-17T17:40:59.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:59 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:40:59.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:59 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1446612113' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:40:59.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:40:59 smithi162 ceph-mon[104668]: osdmap e1435: 8 total, 8 up, 8 in 2024-08-17T17:40:59.356 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:59.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:59.458 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:40:59.481 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:59.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:40:59.583 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:40:59.605 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:40:59.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:40:59.707 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:40:59.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.731 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:59.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:40:59.834 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:40:59.857 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:59.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:40:59.959 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:40:59.982 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:40:59.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:40:59.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:00.084 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:00.106 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:00.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:00.208 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:00.231 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:00.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:00.333 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:00.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:00 smithi162 ceph-mon[104668]: 1.0 scrub starts 2024-08-17T17:41:00.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:00 smithi162 ceph-mon[104668]: 1.0 scrub ok 2024-08-17T17:41:00.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:00 smithi162 ceph-mon[104668]: pgmap v2307: 103 pgs: 1 active+clean+scrubbing, 102 active+clean; 626 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:00.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:00 smithi162 ceph-mon[104668]: 1.3 deep-scrub starts 2024-08-17T17:41:00.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:00 smithi162 ceph-mon[104668]: 1.3 deep-scrub ok 2024-08-17T17:41:00.356 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:00.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:00.457 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:00.481 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:00.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:00.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:00 smithi063 ceph-mon[114756]: 1.0 scrub starts 2024-08-17T17:41:00.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:00 smithi063 ceph-mon[114756]: 1.0 scrub ok 2024-08-17T17:41:00.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:00 smithi063 ceph-mon[114756]: pgmap v2307: 103 pgs: 1 active+clean+scrubbing, 102 active+clean; 626 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:00.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:00 smithi063 ceph-mon[114756]: 1.3 deep-scrub starts 2024-08-17T17:41:00.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:00 smithi063 ceph-mon[114756]: 1.3 deep-scrub ok 2024-08-17T17:41:00.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:00 smithi063 ceph-mon[109843]: 1.0 scrub starts 2024-08-17T17:41:00.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:00 smithi063 ceph-mon[109843]: 1.0 scrub ok 2024-08-17T17:41:00.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:00 smithi063 ceph-mon[109843]: pgmap v2307: 103 pgs: 1 active+clean+scrubbing, 102 active+clean; 626 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:00.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:00 smithi063 ceph-mon[109843]: 1.3 deep-scrub starts 2024-08-17T17:41:00.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:00 smithi063 ceph-mon[109843]: 1.3 deep-scrub ok 2024-08-17T17:41:00.583 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:00.607 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:00.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:00.709 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:00.731 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:00.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:00.833 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:00.855 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:00.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:00.957 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:00.979 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:00.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:00.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:01.081 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:01.104 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:01.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:01.206 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:01.231 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:01.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:01.333 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:01.357 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:01.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:01.459 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:01.481 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:01.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:01.583 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:01.609 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:01.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:01.711 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:01.734 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:01.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:01.788 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:01.788 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:01.836 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:01.862 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:01.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:01.964 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:01.989 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:01.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:01.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:02.091 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:02.117 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:02.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:02.219 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:02.242 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:02.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:02.344 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:02.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:02 smithi162 ceph-mon[104668]: pgmap v2308: 103 pgs: 103 active+clean; 786 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:02 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:02.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:02 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:02.366 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:02.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:02.429 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:02 smithi063 ceph-mon[109843]: pgmap v2308: 103 pgs: 103 active+clean; 786 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:02.429 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:02 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:02.429 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:02 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:02.429 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:02 smithi063 ceph-mon[114756]: pgmap v2308: 103 pgs: 103 active+clean; 786 KiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:02.429 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:02 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:02.429 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:02 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:02.468 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:02.494 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:02.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:02.547 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:41:02.547 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:41:02.550 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:41:02.551 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:41:02.557 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd set noscrub 2024-08-17T17:41:02.569 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:41:02.569 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:41:02.586 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:41:02.587 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:41:02.596 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:02.603 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:41:02.604 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:41:02.621 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:41:02.621 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:41:02.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.622 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:02.622 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:02.638 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:41:02.638 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:41:02.655 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:41:02.655 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:41:02.723 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:02.748 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:02.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:02.850 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:02.873 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:02.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:02.974 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:02.998 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:02.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:02.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:03.002 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-08-17T17:41:03.100 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:03.126 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:03.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:03.183 DEBUG:teuthology.orchestra.run.smithi063:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-08-17T17:41:03.227 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:03.233 INFO:tasks.workunit.client.0.smithi063.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-08-17T17:41:03.235 INFO:tasks.workunit.client.0.smithi063.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-17T17:41:03.235 INFO:tasks.workunit.client.0.smithi063.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-08-17T17:41:03.250 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:03.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:03.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:03 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3752120000' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:03.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:03 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3752120000' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:03.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:03 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3752120000' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:03.353 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:03.376 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:03.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:03.478 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:03.500 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:03.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:03.602 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:03.626 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:03.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:03.727 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:03.752 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:03.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:03.779 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:41:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:41:03] "GET /metrics HTTP/1.1" 200 34123 "" "Prometheus/2.51.0" 2024-08-17T17:41:03.853 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:03.875 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:03.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:03.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:03.977 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:03.999 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:03.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:04.096 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[114756]: pgmap v2309: 103 pgs: 103 active+clean; 786 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:04.096 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[114756]: 1.1c scrub starts 2024-08-17T17:41:04.096 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[114756]: 1.1c scrub ok 2024-08-17T17:41:04.096 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:41:04.096 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:04.096 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:04.096 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:04.096 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:04.097 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[109843]: pgmap v2309: 103 pgs: 103 active+clean; 786 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:04.097 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[109843]: 1.1c scrub starts 2024-08-17T17:41:04.097 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[109843]: 1.1c scrub ok 2024-08-17T17:41:04.097 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:41:04.097 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:04.097 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:04.097 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:04.097 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:04 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:04.101 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:04.122 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:04.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:04.224 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:04.245 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:04.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:04.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:04 smithi162 ceph-mon[104668]: pgmap v2309: 103 pgs: 103 active+clean; 786 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:04.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:04 smithi162 ceph-mon[104668]: 1.1c scrub starts 2024-08-17T17:41:04.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:04 smithi162 ceph-mon[104668]: 1.1c scrub ok 2024-08-17T17:41:04.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:04 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:41:04.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:04 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:04.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:04 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:04.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:04 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:04.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:04 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:04.347 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:04.370 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:04.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:04.472 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:04.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.496 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:04.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:04.597 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:04.608 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:04.609 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:04.618 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:04.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:04.720 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:04.741 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:04.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:04.846 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:04.871 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:04.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:04.972 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:04.996 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:04.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:04.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:05.098 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:05.119 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:05.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:05.221 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:05.244 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:05.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:05.346 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:05.366 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:05.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:05.468 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:05.490 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:05.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:05.592 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:05.614 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:05.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:05.715 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:05.737 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:05.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:05.769 INFO:tasks.workunit.client.0.smithi063.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-08-17T17:41:05.831 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-08-17T17:41:05.839 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:05.860 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:05.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:05.895 INFO:tasks.workunit.client.0.smithi063.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-08-17T17:41:05.938 INFO:tasks.workunit.client.0.smithi063.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-08-17T17:41:05.939 INFO:tasks.workunit.client.0.smithi063.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-17T17:41:05.940 INFO:tasks.workunit.client.0.smithi063.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-08-17T17:41:05.962 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:05.981 INFO:tasks.workunit.client.0.smithi063.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-08-17T17:41:05.985 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:05.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:05.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:06.001 DEBUG:teuthology.orchestra.run.smithi063:> set -ex 2024-08-17T17:41:06.001 DEBUG:teuthology.orchestra.run.smithi063:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-08-17T17:41:06.078 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-08-17T17:41:06.079 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-08-17T17:41:06.080 DEBUG:teuthology.orchestra.run.smithi063:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-08-17T17:41:06.088 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:06.109 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:06.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:06.211 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:06.232 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:06.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:06.260 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:06 smithi063 ceph-mon[114756]: pgmap v2310: 103 pgs: 103 active+clean; 768 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:06.261 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:06 smithi063 ceph-mon[109843]: pgmap v2310: 103 pgs: 103 active+clean; 768 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:06.320 INFO:tasks.workunit.client.0.smithi063.stdout:Running main() from gmock_main.cc 2024-08-17T17:41:06.320 INFO:tasks.workunit.client.0.smithi063.stdout:[==========] Running 70 tests from 1 test suite. 2024-08-17T17:41:06.321 INFO:tasks.workunit.client.0.smithi063.stdout:[----------] Global test environment set-up. 2024-08-17T17:41:06.321 INFO:tasks.workunit.client.0.smithi063.stdout:[----------] 70 tests from TestClsRbd 2024-08-17T17:41:06.334 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:06.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:06 smithi162 ceph-mon[104668]: pgmap v2310: 103 pgs: 103 active+clean; 768 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:06.357 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:06.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:06.459 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:06.481 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:06.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:06.583 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:06.605 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:06.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:06.707 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:06.731 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:06.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:06.833 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:06.856 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:06.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:06.957 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:06.980 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:06.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:06.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:07.081 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:07.104 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:07.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:07.108 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:07 smithi063 ceph-mon[109843]: 1.6 scrub starts 2024-08-17T17:41:07.108 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:07 smithi063 ceph-mon[109843]: 1.6 scrub ok 2024-08-17T17:41:07.108 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:07 smithi063 ceph-mon[114756]: 1.6 scrub starts 2024-08-17T17:41:07.108 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:07 smithi063 ceph-mon[114756]: 1.6 scrub ok 2024-08-17T17:41:07.206 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:07.229 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:07.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:07.331 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:07.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:07 smithi162 ceph-mon[104668]: 1.6 scrub starts 2024-08-17T17:41:07.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:07 smithi162 ceph-mon[104668]: 1.6 scrub ok 2024-08-17T17:41:07.354 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:07.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:07.391 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:41:07.392 INFO:teuthology.orchestra.run.smithi063.stdout:{"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2346,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1436,"num_osds":8,"num_up_osds":8,"osd_up_since":1723916154,"num_in_osds":8,"osd_in_since":1723916113,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":103}],"num_pgs":103,"num_pools":4,"num_objects":27,"data_bytes":786615,"bytes_used":6363643904,"bytes_avail":761596641280,"bytes_total":767960285184},"fsmap":{"epoch":8,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi063.ultpqj","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.63:8443/","prometheus":"http://172.21.15.63:9283/"}},"servicemap":{"epoch":33,"modified":"2024-08-17T17:38:44.649232+0000","services":{}},"progress_events":{}} 2024-08-17T17:41:07.455 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:07.478 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:07.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:07.580 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:07.603 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:07.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:07.705 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:07.727 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:07.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:07.754 INFO:teuthology.orchestra.run.smithi063.stdout:{ 2024-08-17T17:41:07.754 INFO:teuthology.orchestra.run.smithi063.stdout: "ops": [], 2024-08-17T17:41:07.754 INFO:teuthology.orchestra.run.smithi063.stdout: "num_ops": 0 2024-08-17T17:41:07.754 INFO:teuthology.orchestra.run.smithi063.stdout:} 2024-08-17T17:41:07.830 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:07.852 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:07.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:07.954 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:07.977 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:07.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:07.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:08.079 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:08.082 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.get_all_features 2024-08-17T17:41:08.098 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.get_all_features (16 ms) 2024-08-17T17:41:08.098 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.copyup 2024-08-17T17:41:08.109 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:08.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:08.212 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:08.233 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:08.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:08.278 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.copyup (181 ms) 2024-08-17T17:41:08.279 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-08-17T17:41:08.324 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.sparse_copyup (45 ms) 2024-08-17T17:41:08.324 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-08-17T17:41:08.330 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2024-08-17T17:41:08.331 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-08-17T17:41:08.335 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:08.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:08 smithi162 ceph-mon[104668]: pgmap v2311: 103 pgs: 103 active+clean; 768 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:08.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:08 smithi162 ceph-mon[104668]: osdmap e1436: 8 total, 8 up, 8 in 2024-08-17T17:41:08.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:08 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/711352209' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi063-380147-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:41:08.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:08 smithi162 ceph-mon[104668]: 1.7 scrub starts 2024-08-17T17:41:08.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:08 smithi162 ceph-mon[104668]: 1.7 scrub ok 2024-08-17T17:41:08.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:08 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1530849059' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:41:08.355 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:08.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.356 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:08.376 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:08 smithi063 ceph-mon[114756]: pgmap v2311: 103 pgs: 103 active+clean; 768 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:08.376 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:08 smithi063 ceph-mon[114756]: osdmap e1436: 8 total, 8 up, 8 in 2024-08-17T17:41:08.376 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:08 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/711352209' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi063-380147-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:41:08.376 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:08 smithi063 ceph-mon[114756]: 1.7 scrub starts 2024-08-17T17:41:08.376 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:08 smithi063 ceph-mon[114756]: 1.7 scrub ok 2024-08-17T17:41:08.376 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:08 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1530849059' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:41:08.376 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:08 smithi063 ceph-mon[109843]: pgmap v2311: 103 pgs: 103 active+clean; 768 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:08.377 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:08 smithi063 ceph-mon[109843]: osdmap e1436: 8 total, 8 up, 8 in 2024-08-17T17:41:08.377 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:08 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/711352209' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi063-380147-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-08-17T17:41:08.377 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:08 smithi063 ceph-mon[109843]: 1.7 scrub starts 2024-08-17T17:41:08.377 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:08 smithi063 ceph-mon[109843]: 1.7 scrub ok 2024-08-17T17:41:08.377 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:08 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1530849059' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-08-17T17:41:08.378 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.add_remove_child (48 ms) 2024-08-17T17:41:08.379 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.directory_methods 2024-08-17T17:41:08.402 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-08-17T17:41:08.406 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.directory_methods (27 ms) 2024-08-17T17:41:08.406 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.create 2024-08-17T17:41:08.427 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.create (21 ms) 2024-08-17T17:41:08.427 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.get_features 2024-08-17T17:41:08.430 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.get_features (3 ms) 2024-08-17T17:41:08.430 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-08-17T17:41:08.432 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2024-08-17T17:41:08.432 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-08-17T17:41:08.434 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2024-08-17T17:41:08.434 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-08-17T17:41:08.435 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2024-08-17T17:41:08.436 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-08-17T17:41:08.437 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-08-17T17:41:08.437 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-08-17T17:41:08.441 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2024-08-17T17:41:08.441 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.get_size 2024-08-17T17:41:08.446 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.get_size (5 ms) 2024-08-17T17:41:08.446 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.set_size 2024-08-17T17:41:08.451 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2024-08-17T17:41:08.451 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.protection_status 2024-08-17T17:41:08.457 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:08.472 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.protection_status (22 ms) 2024-08-17T17:41:08.473 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-08-17T17:41:08.480 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2024-08-17T17:41:08.480 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.parents_v1 2024-08-17T17:41:08.483 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:08.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:08.522 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.parents_v1 (42 ms) 2024-08-17T17:41:08.522 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.parents_v2 2024-08-17T17:41:08.551 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.parents_v2 (30 ms) 2024-08-17T17:41:08.552 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.snapshots 2024-08-17T17:41:08.574 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.snapshots (22 ms) 2024-08-17T17:41:08.574 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.snapid_race 2024-08-17T17:41:08.580 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.snapid_race (6 ms) 2024-08-17T17:41:08.580 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.stripingv2 2024-08-17T17:41:08.584 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:08.588 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.stripingv2 (8 ms) 2024-08-17T17:41:08.589 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.object_map_save 2024-08-17T17:41:08.592 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.object_map_save (4 ms) 2024-08-17T17:41:08.592 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-08-17T17:41:08.600 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.object_map_resize (8 ms) 2024-08-17T17:41:08.600 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.object_map_update 2024-08-17T17:41:08.607 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:08.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:08.609 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.object_map_update (9 ms) 2024-08-17T17:41:08.609 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-08-17T17:41:08.609 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-08-17T17:41:08.609 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-08-17T17:41:08.617 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.object_map_snap_add (8 ms) 2024-08-17T17:41:08.617 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-08-17T17:41:08.625 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.object_map_snap_remove (8 ms) 2024-08-17T17:41:08.625 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.flags 2024-08-17T17:41:08.630 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.flags (5 ms) 2024-08-17T17:41:08.630 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.metadata 2024-08-17T17:41:08.652 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.metadata (22 ms) 2024-08-17T17:41:08.652 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.set_features 2024-08-17T17:41:08.659 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2024-08-17T17:41:08.659 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.mirror 2024-08-17T17:41:08.694 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.mirror (36 ms) 2024-08-17T17:41:08.695 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.mirror_image 2024-08-17T17:41:08.709 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:08.710 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.mirror_image (15 ms) 2024-08-17T17:41:08.710 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-08-17T17:41:08.734 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:08.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.735 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:08.780 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:08.806 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2024-08-17T17:41:08.806 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:41:08.836 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:08.857 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:08.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:08.959 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:08.984 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:08.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:08.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:09.086 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:09.109 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:09.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:09.211 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:09.236 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:09.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:09.338 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:09.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:09 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/711352209' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi063-380147-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:41:09.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:09 smithi162 ceph-mon[104668]: osdmap e1437: 8 total, 8 up, 8 in 2024-08-17T17:41:09.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:09 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:09.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:09 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:09.360 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:09.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:09.461 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:09.487 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:09.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:09.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:09 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/711352209' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi063-380147-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:41:09.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:09 smithi063 ceph-mon[114756]: osdmap e1437: 8 total, 8 up, 8 in 2024-08-17T17:41:09.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:09.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:09 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:09.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:09 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/711352209' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi063-380147-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-08-17T17:41:09.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:09 smithi063 ceph-mon[109843]: osdmap e1437: 8 total, 8 up, 8 in 2024-08-17T17:41:09.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:09.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:09 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:09.589 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:09.616 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:09.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:09.718 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:09.743 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:09.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:09.845 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:09.875 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:09.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:09.977 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:09.998 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:09.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:09.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:10.100 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:10.125 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:10.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:10.227 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:10.252 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:10.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:10.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:10 smithi063 ceph-mon[114756]: pgmap v2314: 135 pgs: 18 creating+peering, 14 unknown, 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:10.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:10 smithi063 ceph-mon[114756]: osdmap e1438: 8 total, 8 up, 8 in 2024-08-17T17:41:10.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:10 smithi063 ceph-mon[109843]: pgmap v2314: 135 pgs: 18 creating+peering, 14 unknown, 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:10.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:10 smithi063 ceph-mon[109843]: osdmap e1438: 8 total, 8 up, 8 in 2024-08-17T17:41:10.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:10 smithi162 ceph-mon[104668]: pgmap v2314: 135 pgs: 18 creating+peering, 14 unknown, 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:10.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:10 smithi162 ceph-mon[104668]: osdmap e1438: 8 total, 8 up, 8 in 2024-08-17T17:41:10.354 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:10.374 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:10.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:10.476 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:10.502 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:10.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:10.603 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:10.632 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:10.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:10.734 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:10.758 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:10.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:10.860 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:10.885 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:10.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:10.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:10.987 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:11.008 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:11.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:11.110 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:11.130 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:11.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:11.232 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:11.264 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:11.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:11.303 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.mirror_image_status (2593 ms) 2024-08-17T17:41:11.303 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-08-17T17:41:11.366 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:11.392 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:11.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:11.394 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.mirror_image_map (91 ms) 2024-08-17T17:41:11.394 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-08-17T17:41:11.402 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2024-08-17T17:41:11.402 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-08-17T17:41:11.413 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.mirror_snapshot (12 ms) 2024-08-17T17:41:11.414 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-08-17T17:41:11.416 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2024-08-17T17:41:11.416 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-08-17T17:41:11.418 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-08-17T17:41:11.418 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-08-17T17:41:11.420 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-08-17T17:41:11.421 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-08-17T17:41:11.426 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_dir_rename (5 ms) 2024-08-17T17:41:11.426 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-08-17T17:41:11.429 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-08-17T17:41:11.429 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-08-17T17:41:11.432 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2024-08-17T17:41:11.433 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_image_add 2024-08-17T17:41:11.435 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2024-08-17T17:41:11.435 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-08-17T17:41:11.438 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-08-17T17:41:11.438 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_image_list 2024-08-17T17:41:11.440 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2024-08-17T17:41:11.441 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-08-17T17:41:11.445 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2024-08-17T17:41:11.445 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.image_group_add 2024-08-17T17:41:11.447 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2024-08-17T17:41:11.447 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-08-17T17:41:11.450 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-08-17T17:41:11.451 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.image_group_get 2024-08-17T17:41:11.453 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.image_group_get (3 ms) 2024-08-17T17:41:11.453 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-08-17T17:41:11.455 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-08-17T17:41:11.455 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-08-17T17:41:11.457 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-08-17T17:41:11.457 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-08-17T17:41:11.460 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2024-08-17T17:41:11.460 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-08-17T17:41:11.463 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2024-08-17T17:41:11.463 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-08-17T17:41:11.465 INFO:tasks.workunit.client.0.smithi063.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-719-gced649f8/rpm/el9/BUILD/ceph-18.2.4-719-gced649f8/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-08-17T17:41:11.465 INFO:tasks.workunit.client.0.smithi063.stdout:Expected equality of these values: 2024-08-17T17:41:11.466 INFO:tasks.workunit.client.0.smithi063.stdout: 1U 2024-08-17T17:41:11.466 INFO:tasks.workunit.client.0.smithi063.stdout: Which is: 1 2024-08-17T17:41:11.466 INFO:tasks.workunit.client.0.smithi063.stdout: keys.size() 2024-08-17T17:41:11.466 INFO:tasks.workunit.client.0.smithi063.stdout: Which is: 3 2024-08-17T17:41:11.466 INFO:tasks.workunit.client.0.smithi063.stdout:[ FAILED ] TestClsRbd.group_snap_set (2 ms) 2024-08-17T17:41:11.467 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-08-17T17:41:11.469 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_snap_list (4 ms) 2024-08-17T17:41:11.469 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-08-17T17:41:11.488 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (19 ms) 2024-08-17T17:41:11.488 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-08-17T17:41:11.494 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:11.525 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:11.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:11.529 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:11.628 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:11.649 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:11.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:11.729 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (241 ms) 2024-08-17T17:41:11.729 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-08-17T17:41:11.732 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2024-08-17T17:41:11.732 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-08-17T17:41:11.734 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (3 ms) 2024-08-17T17:41:11.734 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.trash_methods 2024-08-17T17:41:11.741 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2024-08-17T17:41:11.742 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.op_features 2024-08-17T17:41:11.749 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2024-08-17T17:41:11.749 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.clone_parent 2024-08-17T17:41:11.751 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:11.770 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.clone_parent (21 ms) 2024-08-17T17:41:11.770 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-08-17T17:41:11.775 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:11.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:11.778 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2024-08-17T17:41:11.778 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.clone_child 2024-08-17T17:41:11.789 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.clone_child (12 ms) 2024-08-17T17:41:11.790 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-08-17T17:41:11.796 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2024-08-17T17:41:11.797 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.migration 2024-08-17T17:41:11.806 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.migration (9 ms) 2024-08-17T17:41:11.806 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.migration_v1 2024-08-17T17:41:11.819 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.migration_v1 (13 ms) 2024-08-17T17:41:11.820 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-08-17T17:41:11.876 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:11.899 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:11.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:11.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:12.000 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:12.022 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:12.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:12.029 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:12.089 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is set 2024-08-17T17:41:12.124 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:12.147 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:12.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:12.249 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:12.271 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:12.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:12.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:12 smithi162 ceph-mon[104668]: pgmap v2316: 135 pgs: 18 creating+peering, 12 unknown, 105 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:12.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:12 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/935648289' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:41:12.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:12 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:41:12.372 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:12.381 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:12 smithi063 ceph-mon[114756]: pgmap v2316: 135 pgs: 18 creating+peering, 12 unknown, 105 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:12.381 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:12 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/935648289' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:41:12.381 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:12 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:41:12.381 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:12 smithi063 ceph-mon[109843]: pgmap v2316: 135 pgs: 18 creating+peering, 12 unknown, 105 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:12.381 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:12 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/935648289' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:41:12.381 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:12 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-08-17T17:41:12.394 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:12.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:12.496 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:12.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.518 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:12.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:12.619 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:12.642 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:12.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.643 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:12.744 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3 2024-08-17T17:41:12.767 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:12.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.3' 2024-08-17T17:41:12.869 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:12.892 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:12.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:12.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:12.994 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:13.018 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:13.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:13.105 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1286 ms) 2024-08-17T17:41:13.105 INFO:tasks.workunit.client.0.smithi063.stdout:[ RUN ] TestClsRbd.sparsify 2024-08-17T17:41:13.120 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:13.144 INFO:tasks.workunit.client.0.smithi063.stdout:[ OK ] TestClsRbd.sparsify (39 ms) 2024-08-17T17:41:13.144 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:13.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:13.207 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:13 smithi063 ceph-mon[109843]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:13.207 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:13 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:41:13.207 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:13 smithi063 ceph-mon[109843]: osdmap e1439: 8 total, 8 up, 8 in 2024-08-17T17:41:13.207 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:13 smithi063 ceph-mon[114756]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:13.207 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:13 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:41:13.207 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:13 smithi063 ceph-mon[114756]: osdmap e1439: 8 total, 8 up, 8 in 2024-08-17T17:41:13.246 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:13.268 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:13.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:13.369 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:13.392 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:13.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:13.493 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1 2024-08-17T17:41:13.516 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:13.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.1' 2024-08-17T17:41:13.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:41:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:41:13] "GET /metrics HTTP/1.1" 200 34123 "" "Prometheus/2.51.0" 2024-08-17T17:41:13.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:13 smithi162 ceph-mon[104668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-08-17T17:41:13.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:13 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-08-17T17:41:13.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:13 smithi162 ceph-mon[104668]: osdmap e1439: 8 total, 8 up, 8 in 2024-08-17T17:41:13.618 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4 2024-08-17T17:41:13.644 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:13.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.4' 2024-08-17T17:41:13.746 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6 2024-08-17T17:41:13.771 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:13.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.6' 2024-08-17T17:41:13.873 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:13.895 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:13.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:13.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:13.997 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:14.019 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:14.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:14.111 INFO:tasks.workunit.client.0.smithi063.stdout:[----------] 70 tests from TestClsRbd (5062 ms total) 2024-08-17T17:41:14.111 INFO:tasks.workunit.client.0.smithi063.stdout: 2024-08-17T17:41:14.111 INFO:tasks.workunit.client.0.smithi063.stdout:[----------] Global test environment tear-down 2024-08-17T17:41:14.111 INFO:tasks.workunit.client.0.smithi063.stdout:[==========] 70 tests from 1 test suite ran. (7791 ms total) 2024-08-17T17:41:14.111 INFO:tasks.workunit.client.0.smithi063.stdout:[ PASSED ] 69 tests. 2024-08-17T17:41:14.111 INFO:tasks.workunit.client.0.smithi063.stdout:[ FAILED ] 1 test, listed below: 2024-08-17T17:41:14.111 INFO:tasks.workunit.client.0.smithi063.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-08-17T17:41:14.111 INFO:tasks.workunit.client.0.smithi063.stdout: 2024-08-17T17:41:14.111 INFO:tasks.workunit.client.0.smithi063.stdout: 1 FAILED TEST 2024-08-17T17:41:14.122 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5 2024-08-17T17:41:14.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.145 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-08-17T17:41:14.145 DEBUG:teuthology.orchestra.run.smithi063:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-08-17T17:41:14.146 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:14.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.5' 2024-08-17T17:41:14.167 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:14 smithi063 ceph-mon[114756]: pgmap v2318: 135 pgs: 9 creating+peering, 126 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 98 KiB/s rd, 9.6 KiB/s wr, 142 op/s 2024-08-17T17:41:14.167 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:14 smithi063 ceph-mon[114756]: osdmap e1440: 8 total, 8 up, 8 in 2024-08-17T17:41:14.167 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:14 smithi063 ceph-mon[109843]: pgmap v2318: 135 pgs: 9 creating+peering, 126 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 98 KiB/s rd, 9.6 KiB/s wr, 142 op/s 2024-08-17T17:41:14.167 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:14 smithi063 ceph-mon[109843]: osdmap e1440: 8 total, 8 up, 8 in 2024-08-17T17:41:14.248 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:14.272 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:14.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:14.374 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0 2024-08-17T17:41:14.397 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:14.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.0' 2024-08-17T17:41:14.499 DEBUG:teuthology.orchestra.run.smithi162:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7 2024-08-17T17:41:14.523 INFO:teuthology.orchestra.run.smithi162.stderr:sudo: docker: command not found 2024-08-17T17:41:14.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi162 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.7' 2024-08-17T17:41:14.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:14 smithi162 ceph-mon[104668]: pgmap v2318: 135 pgs: 9 creating+peering, 126 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 98 KiB/s rd, 9.6 KiB/s wr, 142 op/s 2024-08-17T17:41:14.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:14 smithi162 ceph-mon[104668]: osdmap e1440: 8 total, 8 up, 8 in 2024-08-17T17:41:14.625 DEBUG:teuthology.orchestra.run.smithi063:> sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2 2024-08-17T17:41:14.646 INFO:tasks.thrashosds:joining thrashosds 2024-08-17T17:41:14.647 INFO:teuthology.orchestra.run.smithi063.stderr:sudo: docker: command not found 2024-08-17T17:41:14.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T17:41:14.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi063 with status 1: 'sudo docker kill -s 1 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd.2' 2024-08-17T17:41:15.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:15 smithi063 ceph-mon[109843]: osdmap e1441: 8 total, 8 up, 8 in 2024-08-17T17:41:15.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:15 smithi063 ceph-mon[114756]: osdmap e1441: 8 total, 8 up, 8 in 2024-08-17T17:41:15.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:15 smithi162 ceph-mon[104668]: osdmap e1441: 8 total, 8 up, 8 in 2024-08-17T17:41:15.614 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset noscrub 2024-08-17T17:41:16.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:16 smithi063 ceph-mon[114756]: pgmap v2321: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:16.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:16 smithi063 ceph-mon[109843]: pgmap v2321: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:16.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:16 smithi162 ceph-mon[104668]: pgmap v2321: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:17.699 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:17 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:17.699 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:17 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:17.699 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:17 smithi162 ceph-mon[104668]: pgmap v2322: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:17.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:17 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:17.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:17 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:17.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:17 smithi063 ceph-mon[114756]: pgmap v2322: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:17.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:17 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:17.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:17 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:17.780 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:17 smithi063 ceph-mon[109843]: pgmap v2322: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:18.485 INFO:teuthology.orchestra.run.smithi063.stdout:osd.0: {} 2024-08-17T17:41:18.485 INFO:teuthology.orchestra.run.smithi063.stderr:osd.0: osd_enable_op_tracker = '' 2024-08-17T17:41:18.503 INFO:teuthology.orchestra.run.smithi063.stdout:osd.1: {} 2024-08-17T17:41:18.503 INFO:teuthology.orchestra.run.smithi063.stderr:osd.1: osd_enable_op_tracker = '' 2024-08-17T17:41:18.526 INFO:teuthology.orchestra.run.smithi063.stdout:osd.2: {} 2024-08-17T17:41:18.526 INFO:teuthology.orchestra.run.smithi063.stderr:osd.2: osd_enable_op_tracker = '' 2024-08-17T17:41:18.554 INFO:teuthology.orchestra.run.smithi063.stdout:osd.3: {} 2024-08-17T17:41:18.554 INFO:teuthology.orchestra.run.smithi063.stderr:osd.3: osd_enable_op_tracker = '' 2024-08-17T17:41:18.572 INFO:teuthology.orchestra.run.smithi063.stdout:osd.4: {} 2024-08-17T17:41:18.572 INFO:teuthology.orchestra.run.smithi063.stderr:osd.4: osd_enable_op_tracker = '' 2024-08-17T17:41:18.588 INFO:teuthology.orchestra.run.smithi063.stdout:osd.5: {} 2024-08-17T17:41:18.589 INFO:teuthology.orchestra.run.smithi063.stderr:osd.5: osd_enable_op_tracker = '' 2024-08-17T17:41:18.605 INFO:teuthology.orchestra.run.smithi063.stdout:osd.6: {} 2024-08-17T17:41:18.606 INFO:teuthology.orchestra.run.smithi063.stderr:osd.6: osd_enable_op_tracker = '' 2024-08-17T17:41:18.623 INFO:teuthology.orchestra.run.smithi063.stdout:osd.7: {} 2024-08-17T17:41:18.624 INFO:teuthology.orchestra.run.smithi063.stderr:osd.7: osd_enable_op_tracker = '' 2024-08-17T17:41:18.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:18 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3668732499' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:18.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:18 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3668732499' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:18.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:18 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3668732499' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-08-17T17:41:19.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:19 smithi063 ceph-mon[114756]: pgmap v2323: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:19.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:19 smithi063 ceph-mon[109843]: pgmap v2323: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:19.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:19 smithi162 ceph-mon[104668]: pgmap v2323: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:20.087 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:41:20.088 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:41:20.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:20 smithi063 ceph-mon[114756]: from='client.36621 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:20.779 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:20 smithi063 ceph-mon[109843]: from='client.36621 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:20.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:20 smithi162 ceph-mon[104668]: from='client.36621 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:20.963 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2323,"stamp":"2024-08-17T17:41:18.693436+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":526015,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4678,"ondisk_log_size":4678,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":340,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6198964,"kb_used_data":63072,"kb_used_omap":298,"kb_used_meta":6135381,"kb_avail":743762252,"statfs":{"total":767960285184,"available":761612546048,"internally_reserved":0,"allocated":64585728,"data_stored":42628524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":6282630673},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-274,"num_objects":-18,"num_object_clones":0,"num_object_copies":-54,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-18,"num_whiteouts":0,"num_read":-507,"num_read_kb":-441,"num_write":-125,"num_write_kb":-134,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.607779"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":89,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.107703+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:41:14.107703+0000","last_peered":"2024-08-17T17:41:14.107703+0000","last_clean":"2024-08-17T17:41:14.107703+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:41:14.107703+0000","last_undegraded":"2024-08-17T17:41:14.107703+0000","last_fullsized":"2024-08-17T17:41:14.107703+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1456,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123659+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:41:13.123659+0000","last_peered":"2024-08-17T17:41:13.123659+0000","last_clean":"2024-08-17T17:41:13.123659+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:41:13.123659+0000","last_undegraded":"2024-08-17T17:41:13.123659+0000","last_fullsized":"2024-08-17T17:41:13.123659+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1771,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124170+0000","last_change":"2024-08-17T17:36:15.771150+0000","last_active":"2024-08-17T17:41:13.124170+0000","last_peered":"2024-08-17T17:41:13.124170+0000","last_clean":"2024-08-17T17:41:13.124170+0000","last_became_active":"2024-08-17T17:36:15.770520+0000","last_became_peered":"2024-08-17T17:36:15.770520+0000","last_unstale":"2024-08-17T17:41:13.124170+0000","last_undegraded":"2024-08-17T17:41:13.124170+0000","last_fullsized":"2024-08-17T17:41:13.124170+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:17.552796+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":106,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.101805+0000","last_change":"2024-08-17T17:36:12.756709+0000","last_active":"2024-08-17T17:41:13.101805+0000","last_peered":"2024-08-17T17:41:13.101805+0000","last_clean":"2024-08-17T17:41:13.101805+0000","last_became_active":"2024-08-17T17:36:12.756385+0000","last_became_peered":"2024-08-17T17:36:12.756385+0000","last_unstale":"2024-08-17T17:41:13.101805+0000","last_undegraded":"2024-08-17T17:41:13.101805+0000","last_fullsized":"2024-08-17T17:41:13.101805+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:21.420282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":174,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091082+0000","last_change":"2024-08-17T17:36:12.762800+0000","last_active":"2024-08-17T17:41:15.091082+0000","last_peered":"2024-08-17T17:41:15.091082+0000","last_clean":"2024-08-17T17:41:15.091082+0000","last_became_active":"2024-08-17T17:36:12.759977+0000","last_became_peered":"2024-08-17T17:36:12.759977+0000","last_unstale":"2024-08-17T17:41:15.091082+0000","last_undegraded":"2024-08-17T17:41:15.091082+0000","last_fullsized":"2024-08-17T17:41:15.091082+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:02.162681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1663,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.101553+0000","last_change":"2024-08-17T17:41:03.323402+0000","last_active":"2024-08-17T17:41:13.101553+0000","last_peered":"2024-08-17T17:41:13.101553+0000","last_clean":"2024-08-17T17:41:13.101553+0000","last_became_active":"2024-08-17T17:35:56.278638+0000","last_became_peered":"2024-08-17T17:35:56.278638+0000","last_unstale":"2024-08-17T17:41:13.101553+0000","last_undegraded":"2024-08-17T17:41:13.101553+0000","last_fullsized":"2024-08-17T17:41:13.101553+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:41:03.323294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:41:03.323294+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:28.872583+0000","scrub_duration":0.002060147,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1390,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.733244+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:41:14.733244+0000","last_peered":"2024-08-17T17:41:14.733244+0000","last_clean":"2024-08-17T17:41:14.733244+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:41:14.733244+0000","last_undegraded":"2024-08-17T17:41:14.733244+0000","last_fullsized":"2024-08-17T17:41:14.733244+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":66,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091151+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:41:15.091151+0000","last_peered":"2024-08-17T17:41:15.091151+0000","last_clean":"2024-08-17T17:41:15.091151+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:41:15.091151+0000","last_undegraded":"2024-08-17T17:41:15.091151+0000","last_fullsized":"2024-08-17T17:41:15.091151+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1435,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100722+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:41:13.100722+0000","last_peered":"2024-08-17T17:41:13.100722+0000","last_clean":"2024-08-17T17:41:13.100722+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:41:13.100722+0000","last_undegraded":"2024-08-17T17:41:13.100722+0000","last_fullsized":"2024-08-17T17:41:13.100722+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1443,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123904+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:41:13.123904+0000","last_peered":"2024-08-17T17:41:13.123904+0000","last_clean":"2024-08-17T17:41:13.123904+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:41:13.123904+0000","last_undegraded":"2024-08-17T17:41:13.123904+0000","last_fullsized":"2024-08-17T17:41:13.123904+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":733,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.101622+0000","last_change":"2024-08-17T17:36:15.773315+0000","last_active":"2024-08-17T17:41:13.101622+0000","last_peered":"2024-08-17T17:41:13.101622+0000","last_clean":"2024-08-17T17:41:13.101622+0000","last_became_active":"2024-08-17T17:36:15.772961+0000","last_became_peered":"2024-08-17T17:36:15.772961+0000","last_unstale":"2024-08-17T17:41:13.101622+0000","last_undegraded":"2024-08-17T17:41:13.101622+0000","last_fullsized":"2024-08-17T17:41:13.101622+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:10.190221+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":83,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.108196+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:41:14.108196+0000","last_peered":"2024-08-17T17:41:14.108196+0000","last_clean":"2024-08-17T17:41:14.108196+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:41:14.108196+0000","last_undegraded":"2024-08-17T17:41:14.108196+0000","last_fullsized":"2024-08-17T17:41:14.108196+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":174,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091099+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:41:15.091099+0000","last_peered":"2024-08-17T17:41:15.091099+0000","last_clean":"2024-08-17T17:41:15.091099+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:41:15.091099+0000","last_undegraded":"2024-08-17T17:41:15.091099+0000","last_fullsized":"2024-08-17T17:41:15.091099+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1524,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.101992+0000","last_change":"2024-08-17T17:36:24.225739+0000","last_active":"2024-08-17T17:41:13.101992+0000","last_peered":"2024-08-17T17:41:13.101992+0000","last_clean":"2024-08-17T17:41:13.101992+0000","last_became_active":"2024-08-17T17:36:24.225250+0000","last_became_peered":"2024-08-17T17:36:24.225250+0000","last_unstale":"2024-08-17T17:41:13.101992+0000","last_undegraded":"2024-08-17T17:41:13.101992+0000","last_fullsized":"2024-08-17T17:41:13.101992+0000","mapping_epoch":1393,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1394,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.290656+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1463,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123660+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:41:13.123660+0000","last_peered":"2024-08-17T17:41:13.123660+0000","last_clean":"2024-08-17T17:41:13.123660+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:41:13.123660+0000","last_undegraded":"2024-08-17T17:41:13.123660+0000","last_fullsized":"2024-08-17T17:41:13.123660+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1870,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124298+0000","last_change":"2024-08-17T17:36:12.752970+0000","last_active":"2024-08-17T17:41:13.124298+0000","last_peered":"2024-08-17T17:41:13.124298+0000","last_clean":"2024-08-17T17:41:13.124298+0000","last_became_active":"2024-08-17T17:36:12.752311+0000","last_became_peered":"2024-08-17T17:36:12.752311+0000","last_unstale":"2024-08-17T17:41:13.124298+0000","last_undegraded":"2024-08-17T17:41:13.124298+0000","last_fullsized":"2024-08-17T17:41:13.124298+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:38.131419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1817,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100043+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:41:13.100043+0000","last_peered":"2024-08-17T17:41:13.100043+0000","last_clean":"2024-08-17T17:41:13.100043+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:41:13.100043+0000","last_undegraded":"2024-08-17T17:41:13.100043+0000","last_fullsized":"2024-08-17T17:41:13.100043+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1544,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.160132+0000","last_change":"2024-08-17T17:36:18.750494+0000","last_active":"2024-08-17T17:41:13.160132+0000","last_peered":"2024-08-17T17:41:13.160132+0000","last_clean":"2024-08-17T17:41:13.160132+0000","last_became_active":"2024-08-17T17:36:18.750092+0000","last_became_peered":"2024-08-17T17:36:18.750092+0000","last_unstale":"2024-08-17T17:41:13.160132+0000","last_undegraded":"2024-08-17T17:41:13.160132+0000","last_fullsized":"2024-08-17T17:41:13.160132+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:36.599727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1176,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.108329+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:41:14.108329+0000","last_peered":"2024-08-17T17:41:14.108329+0000","last_clean":"2024-08-17T17:41:14.108329+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:41:14.108329+0000","last_undegraded":"2024-08-17T17:41:14.108329+0000","last_fullsized":"2024-08-17T17:41:14.108329+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":984,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100274+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:41:13.100274+0000","last_peered":"2024-08-17T17:41:13.100274+0000","last_clean":"2024-08-17T17:41:13.100274+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:41:13.100274+0000","last_undegraded":"2024-08-17T17:41:13.100274+0000","last_fullsized":"2024-08-17T17:41:13.100274+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":83,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.106460+0000","last_change":"2024-08-17T17:37:11.075505+0000","last_active":"2024-08-17T17:41:14.106460+0000","last_peered":"2024-08-17T17:41:14.106460+0000","last_clean":"2024-08-17T17:41:14.106460+0000","last_became_active":"2024-08-17T17:37:11.075131+0000","last_became_peered":"2024-08-17T17:37:11.075131+0000","last_unstale":"2024-08-17T17:41:14.106460+0000","last_undegraded":"2024-08-17T17:41:14.106460+0000","last_fullsized":"2024-08-17T17:41:14.106460+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:39.672054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1297,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.102893+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:41:13.102893+0000","last_peered":"2024-08-17T17:41:13.102893+0000","last_clean":"2024-08-17T17:41:13.102893+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:41:13.102893+0000","last_undegraded":"2024-08-17T17:41:13.102893+0000","last_fullsized":"2024-08-17T17:41:13.102893+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1674,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123237+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:41:13.123237+0000","last_peered":"2024-08-17T17:41:13.123237+0000","last_clean":"2024-08-17T17:41:13.123237+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:41:13.123237+0000","last_undegraded":"2024-08-17T17:41:13.123237+0000","last_fullsized":"2024-08-17T17:41:13.123237+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":128,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.159590+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:41:13.159590+0000","last_peered":"2024-08-17T17:41:13.159590+0000","last_clean":"2024-08-17T17:41:13.159590+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:41:13.159590+0000","last_undegraded":"2024-08-17T17:41:13.159590+0000","last_fullsized":"2024-08-17T17:41:13.159590+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1271,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.101366+0000","last_change":"2024-08-17T17:36:12.755873+0000","last_active":"2024-08-17T17:41:13.101366+0000","last_peered":"2024-08-17T17:41:13.101366+0000","last_clean":"2024-08-17T17:41:13.101366+0000","last_became_active":"2024-08-17T17:36:12.755366+0000","last_became_peered":"2024-08-17T17:36:12.755366+0000","last_unstale":"2024-08-17T17:41:13.101366+0000","last_undegraded":"2024-08-17T17:41:13.101366+0000","last_fullsized":"2024-08-17T17:41:13.101366+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:36.523214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1625,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.125309+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:41:13.125309+0000","last_peered":"2024-08-17T17:41:13.125309+0000","last_clean":"2024-08-17T17:41:13.125309+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:41:13.125309+0000","last_undegraded":"2024-08-17T17:41:13.125309+0000","last_fullsized":"2024-08-17T17:41:13.125309+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":809,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.102251+0000","last_change":"2024-08-17T17:35:55.200519+0000","last_active":"2024-08-17T17:41:13.102251+0000","last_peered":"2024-08-17T17:41:13.102251+0000","last_clean":"2024-08-17T17:41:13.102251+0000","last_became_active":"2024-08-17T17:35:55.200230+0000","last_became_peered":"2024-08-17T17:35:55.200230+0000","last_unstale":"2024-08-17T17:41:13.102251+0000","last_undegraded":"2024-08-17T17:41:13.102251+0000","last_fullsized":"2024-08-17T17:41:13.102251+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:33.454851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1448,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100673+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:41:13.100673+0000","last_peered":"2024-08-17T17:41:13.100673+0000","last_clean":"2024-08-17T17:41:13.100673+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:41:13.100673+0000","last_undegraded":"2024-08-17T17:41:13.100673+0000","last_fullsized":"2024-08-17T17:41:13.100673+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1747,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.090894+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:41:15.090894+0000","last_peered":"2024-08-17T17:41:15.090894+0000","last_clean":"2024-08-17T17:41:15.090894+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:41:15.090894+0000","last_undegraded":"2024-08-17T17:41:15.090894+0000","last_fullsized":"2024-08-17T17:41:15.090894+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:22.077229+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":292,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.160593+0000","last_change":"2024-08-17T17:35:33.145476+0000","last_active":"2024-08-17T17:41:13.160593+0000","last_peered":"2024-08-17T17:41:13.160593+0000","last_clean":"2024-08-17T17:41:13.160593+0000","last_became_active":"2024-08-17T17:35:33.145125+0000","last_became_peered":"2024-08-17T17:35:33.145125+0000","last_unstale":"2024-08-17T17:41:13.160593+0000","last_undegraded":"2024-08-17T17:41:13.160593+0000","last_fullsized":"2024-08-17T17:41:13.160593+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:45.298064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":953,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.102254+0000","last_change":"2024-08-17T17:35:56.248612+0000","last_active":"2024-08-17T17:41:13.102254+0000","last_peered":"2024-08-17T17:41:13.102254+0000","last_clean":"2024-08-17T17:41:13.102254+0000","last_became_active":"2024-08-17T17:35:56.244364+0000","last_became_peered":"2024-08-17T17:35:56.244364+0000","last_unstale":"2024-08-17T17:41:13.102254+0000","last_undegraded":"2024-08-17T17:41:13.102254+0000","last_fullsized":"2024-08-17T17:41:13.102254+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:41.064165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":892,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.101028+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:41:13.101028+0000","last_peered":"2024-08-17T17:41:13.101028+0000","last_clean":"2024-08-17T17:41:13.101028+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:41:13.101028+0000","last_undegraded":"2024-08-17T17:41:13.101028+0000","last_fullsized":"2024-08-17T17:41:13.101028+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1428,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123708+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:41:13.123708+0000","last_peered":"2024-08-17T17:41:13.123708+0000","last_clean":"2024-08-17T17:41:13.123708+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:41:13.123708+0000","last_undegraded":"2024-08-17T17:41:13.123708+0000","last_fullsized":"2024-08-17T17:41:13.123708+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1447,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124052+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:41:13.124052+0000","last_peered":"2024-08-17T17:41:13.124052+0000","last_clean":"2024-08-17T17:41:13.124052+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:41:13.124052+0000","last_undegraded":"2024-08-17T17:41:13.124052+0000","last_fullsized":"2024-08-17T17:41:13.124052+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":969,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.108710+0000","last_change":"2024-08-17T17:36:15.774881+0000","last_active":"2024-08-17T17:41:14.108710+0000","last_peered":"2024-08-17T17:41:14.108710+0000","last_clean":"2024-08-17T17:41:14.108710+0000","last_became_active":"2024-08-17T17:36:15.774435+0000","last_became_peered":"2024-08-17T17:36:15.774435+0000","last_unstale":"2024-08-17T17:41:14.108710+0000","last_undegraded":"2024-08-17T17:41:14.108710+0000","last_fullsized":"2024-08-17T17:41:14.108710+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:52.015854+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1395,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123115+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:41:13.123115+0000","last_peered":"2024-08-17T17:41:13.123115+0000","last_clean":"2024-08-17T17:41:13.123115+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:41:13.123115+0000","last_undegraded":"2024-08-17T17:41:13.123115+0000","last_fullsized":"2024-08-17T17:41:13.123115+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1291,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.102600+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:41:13.102600+0000","last_peered":"2024-08-17T17:41:13.102600+0000","last_clean":"2024-08-17T17:41:13.102600+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:41:13.102600+0000","last_undegraded":"2024-08-17T17:41:13.102600+0000","last_fullsized":"2024-08-17T17:41:13.102600+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":362,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.108390+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:41:14.108390+0000","last_peered":"2024-08-17T17:41:14.108390+0000","last_clean":"2024-08-17T17:41:14.108390+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:41:14.108390+0000","last_undegraded":"2024-08-17T17:41:14.108390+0000","last_fullsized":"2024-08-17T17:41:14.108390+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1628,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.160298+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:41:13.160298+0000","last_peered":"2024-08-17T17:41:13.160298+0000","last_clean":"2024-08-17T17:41:13.160298+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:41:13.160298+0000","last_undegraded":"2024-08-17T17:41:13.160298+0000","last_fullsized":"2024-08-17T17:41:13.160298+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:25.751871+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":135,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.159704+0000","last_change":"2024-08-17T17:37:11.079712+0000","last_active":"2024-08-17T17:41:13.159704+0000","last_peered":"2024-08-17T17:41:13.159704+0000","last_clean":"2024-08-17T17:41:13.159704+0000","last_became_active":"2024-08-17T17:37:11.079253+0000","last_became_peered":"2024-08-17T17:37:11.079253+0000","last_unstale":"2024-08-17T17:41:13.159704+0000","last_undegraded":"2024-08-17T17:41:13.159704+0000","last_fullsized":"2024-08-17T17:41:13.159704+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:53.405657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":936,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.101534+0000","last_change":"2024-08-17T17:35:56.223396+0000","last_active":"2024-08-17T17:41:13.101534+0000","last_peered":"2024-08-17T17:41:13.101534+0000","last_clean":"2024-08-17T17:41:13.101534+0000","last_became_active":"2024-08-17T17:35:56.212413+0000","last_became_peered":"2024-08-17T17:35:56.212413+0000","last_unstale":"2024-08-17T17:41:13.101534+0000","last_undegraded":"2024-08-17T17:41:13.101534+0000","last_fullsized":"2024-08-17T17:41:13.101534+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:44.185174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1671,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.107997+0000","last_change":"2024-08-17T17:35:33.136068+0000","last_active":"2024-08-17T17:41:14.107997+0000","last_peered":"2024-08-17T17:41:14.107997+0000","last_clean":"2024-08-17T17:41:14.107997+0000","last_became_active":"2024-08-17T17:35:33.135591+0000","last_became_peered":"2024-08-17T17:35:33.135591+0000","last_unstale":"2024-08-17T17:41:14.107997+0000","last_undegraded":"2024-08-17T17:41:14.107997+0000","last_fullsized":"2024-08-17T17:41:14.107997+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:52.252561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1801,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124433+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:41:13.124433+0000","last_peered":"2024-08-17T17:41:13.124433+0000","last_clean":"2024-08-17T17:41:13.124433+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:41:13.124433+0000","last_undegraded":"2024-08-17T17:41:13.124433+0000","last_fullsized":"2024-08-17T17:41:13.124433+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1014,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100776+0000","last_change":"2024-08-17T17:35:33.138467+0000","last_active":"2024-08-17T17:41:13.100776+0000","last_peered":"2024-08-17T17:41:13.100776+0000","last_clean":"2024-08-17T17:41:13.100776+0000","last_became_active":"2024-08-17T17:35:33.138174+0000","last_became_peered":"2024-08-17T17:35:33.138174+0000","last_unstale":"2024-08-17T17:41:13.100776+0000","last_undegraded":"2024-08-17T17:41:13.100776+0000","last_fullsized":"2024-08-17T17:41:13.100776+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:08.860439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1398,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.125391+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:41:13.125391+0000","last_peered":"2024-08-17T17:41:13.125391+0000","last_clean":"2024-08-17T17:41:13.125391+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:41:13.125391+0000","last_undegraded":"2024-08-17T17:41:13.125391+0000","last_fullsized":"2024-08-17T17:41:13.125391+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":167,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.090549+0000","last_change":"2024-08-17T17:35:14.854881+0000","last_active":"2024-08-17T17:41:15.090549+0000","last_peered":"2024-08-17T17:41:15.090549+0000","last_clean":"2024-08-17T17:41:15.090549+0000","last_became_active":"2024-08-17T17:35:14.854121+0000","last_became_peered":"2024-08-17T17:35:14.854121+0000","last_unstale":"2024-08-17T17:41:15.090549+0000","last_undegraded":"2024-08-17T17:41:15.090549+0000","last_fullsized":"2024-08-17T17:41:15.090549+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.863624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":967,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.101911+0000","last_change":"2024-08-17T17:41:06.387062+0000","last_active":"2024-08-17T17:41:13.101911+0000","last_peered":"2024-08-17T17:41:13.101911+0000","last_clean":"2024-08-17T17:41:13.101911+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:41:13.101911+0000","last_undegraded":"2024-08-17T17:41:13.101911+0000","last_fullsized":"2024-08-17T17:41:13.101911+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:41:06.386955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:41:06.386955+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:44:32.008526+0000","scrub_duration":0.0017975980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1057,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100426+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:41:13.100426+0000","last_peered":"2024-08-17T17:41:13.100426+0000","last_clean":"2024-08-17T17:41:13.100426+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:41:13.100426+0000","last_undegraded":"2024-08-17T17:41:13.100426+0000","last_fullsized":"2024-08-17T17:41:13.100426+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":261,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.108411+0000","last_change":"2024-08-17T17:35:33.145351+0000","last_active":"2024-08-17T17:41:14.108411+0000","last_peered":"2024-08-17T17:41:14.108411+0000","last_clean":"2024-08-17T17:41:14.108411+0000","last_became_active":"2024-08-17T17:35:33.145040+0000","last_became_peered":"2024-08-17T17:35:33.145040+0000","last_unstale":"2024-08-17T17:41:14.108411+0000","last_undegraded":"2024-08-17T17:41:14.108411+0000","last_fullsized":"2024-08-17T17:41:14.108411+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:02.124000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1466,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100949+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:41:13.100949+0000","last_peered":"2024-08-17T17:41:13.100949+0000","last_clean":"2024-08-17T17:41:13.100949+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:41:13.100949+0000","last_undegraded":"2024-08-17T17:41:13.100949+0000","last_fullsized":"2024-08-17T17:41:13.100949+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":950,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.102531+0000","last_change":"2024-08-17T17:41:07.361112+0000","last_active":"2024-08-17T17:41:13.102531+0000","last_peered":"2024-08-17T17:41:13.102531+0000","last_clean":"2024-08-17T17:41:13.102531+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:41:13.102531+0000","last_undegraded":"2024-08-17T17:41:13.102531+0000","last_fullsized":"2024-08-17T17:41:13.102531+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:41:07.360968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:41:07.360968+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:28.377019+0000","scrub_duration":0.001649951,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1522,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091404+0000","last_change":"2024-08-17T17:36:12.759743+0000","last_active":"2024-08-17T17:41:15.091404+0000","last_peered":"2024-08-17T17:41:15.091404+0000","last_clean":"2024-08-17T17:41:15.091404+0000","last_became_active":"2024-08-17T17:36:12.759334+0000","last_became_peered":"2024-08-17T17:36:12.759334+0000","last_unstale":"2024-08-17T17:41:15.091404+0000","last_undegraded":"2024-08-17T17:41:15.091404+0000","last_fullsized":"2024-08-17T17:41:15.091404+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:04.524384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1429,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124249+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:41:13.124249+0000","last_peered":"2024-08-17T17:41:13.124249+0000","last_clean":"2024-08-17T17:41:13.124249+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:41:13.124249+0000","last_undegraded":"2024-08-17T17:41:13.124249+0000","last_fullsized":"2024-08-17T17:41:13.124249+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1472,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100603+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:41:13.100603+0000","last_peered":"2024-08-17T17:41:13.100603+0000","last_clean":"2024-08-17T17:41:13.100603+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:41:13.100603+0000","last_undegraded":"2024-08-17T17:41:13.100603+0000","last_fullsized":"2024-08-17T17:41:13.100603+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1787,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100328+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:41:13.100328+0000","last_peered":"2024-08-17T17:41:13.100328+0000","last_clean":"2024-08-17T17:41:13.100328+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:41:13.100328+0000","last_undegraded":"2024-08-17T17:41:13.100328+0000","last_fullsized":"2024-08-17T17:41:13.100328+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1105,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.101157+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:41:13.101157+0000","last_peered":"2024-08-17T17:41:13.101157+0000","last_clean":"2024-08-17T17:41:13.101157+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:41:13.101157+0000","last_undegraded":"2024-08-17T17:41:13.101157+0000","last_fullsized":"2024-08-17T17:41:13.101157+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1434,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091570+0000","last_change":"2024-08-17T17:35:56.229622+0000","last_active":"2024-08-17T17:41:15.091570+0000","last_peered":"2024-08-17T17:41:15.091570+0000","last_clean":"2024-08-17T17:41:15.091570+0000","last_became_active":"2024-08-17T17:35:56.229324+0000","last_became_peered":"2024-08-17T17:35:56.229324+0000","last_unstale":"2024-08-17T17:41:15.091570+0000","last_undegraded":"2024-08-17T17:41:15.091570+0000","last_fullsized":"2024-08-17T17:41:15.091570+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:13.980316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2037,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091181+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:41:15.091181+0000","last_peered":"2024-08-17T17:41:15.091181+0000","last_clean":"2024-08-17T17:41:15.091181+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:41:15.091181+0000","last_undegraded":"2024-08-17T17:41:15.091181+0000","last_fullsized":"2024-08-17T17:41:15.091181+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1602,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123324+0000","last_change":"2024-08-17T17:35:33.137074+0000","last_active":"2024-08-17T17:41:13.123324+0000","last_peered":"2024-08-17T17:41:13.123324+0000","last_clean":"2024-08-17T17:41:13.123324+0000","last_became_active":"2024-08-17T17:35:33.136759+0000","last_became_peered":"2024-08-17T17:35:33.136759+0000","last_unstale":"2024-08-17T17:41:13.123324+0000","last_undegraded":"2024-08-17T17:41:13.123324+0000","last_fullsized":"2024-08-17T17:41:13.123324+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:37.263639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:37.263639+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:14.599780+0000","scrub_duration":0.0016181800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1227,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.109527+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:41:14.109527+0000","last_peered":"2024-08-17T17:41:14.109527+0000","last_clean":"2024-08-17T17:41:14.109527+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:41:14.109527+0000","last_undegraded":"2024-08-17T17:41:14.109527+0000","last_fullsized":"2024-08-17T17:41:14.109527+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":66,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123034+0000","last_change":"2024-08-17T17:36:17.774773+0000","last_active":"2024-08-17T17:41:13.123034+0000","last_peered":"2024-08-17T17:41:13.123034+0000","last_clean":"2024-08-17T17:41:13.123034+0000","last_became_active":"2024-08-17T17:36:17.774468+0000","last_became_peered":"2024-08-17T17:36:17.774468+0000","last_unstale":"2024-08-17T17:41:13.123034+0000","last_undegraded":"2024-08-17T17:41:13.123034+0000","last_fullsized":"2024-08-17T17:41:13.123034+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:54.231747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1565,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.106360+0000","last_change":"2024-08-17T17:35:33.137633+0000","last_active":"2024-08-17T17:41:14.106360+0000","last_peered":"2024-08-17T17:41:14.106360+0000","last_clean":"2024-08-17T17:41:14.106360+0000","last_became_active":"2024-08-17T17:35:33.137323+0000","last_became_peered":"2024-08-17T17:35:33.137323+0000","last_unstale":"2024-08-17T17:41:14.106360+0000","last_undegraded":"2024-08-17T17:41:14.106360+0000","last_fullsized":"2024-08-17T17:41:14.106360+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:03.359155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":929,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.101297+0000","last_change":"2024-08-17T17:40:59.325330+0000","last_active":"2024-08-17T17:41:13.101297+0000","last_peered":"2024-08-17T17:41:13.101297+0000","last_clean":"2024-08-17T17:41:13.101297+0000","last_became_active":"2024-08-17T17:35:56.264265+0000","last_became_peered":"2024-08-17T17:35:56.264265+0000","last_unstale":"2024-08-17T17:41:13.101297+0000","last_undegraded":"2024-08-17T17:41:13.101297+0000","last_fullsized":"2024-08-17T17:41:13.101297+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:59.325233+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:40:59.325233+0000","last_clean_scrub_stamp":"2024-08-17T17:40:59.325233+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:22:35.293983+0000","scrub_duration":0.0014772979999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1270,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.102433+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:41:13.102433+0000","last_peered":"2024-08-17T17:41:13.102433+0000","last_clean":"2024-08-17T17:41:13.102433+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:41:13.102433+0000","last_undegraded":"2024-08-17T17:41:13.102433+0000","last_fullsized":"2024-08-17T17:41:13.102433+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1810,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100517+0000","last_change":"2024-08-17T17:36:12.758005+0000","last_active":"2024-08-17T17:41:13.100517+0000","last_peered":"2024-08-17T17:41:13.100517+0000","last_clean":"2024-08-17T17:41:13.100517+0000","last_became_active":"2024-08-17T17:36:12.757695+0000","last_became_peered":"2024-08-17T17:36:12.757695+0000","last_unstale":"2024-08-17T17:41:13.100517+0000","last_undegraded":"2024-08-17T17:41:13.100517+0000","last_fullsized":"2024-08-17T17:41:13.100517+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:51.792220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1448,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100394+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:41:13.100394+0000","last_peered":"2024-08-17T17:41:13.100394+0000","last_clean":"2024-08-17T17:41:13.100394+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:41:13.100394+0000","last_undegraded":"2024-08-17T17:41:13.100394+0000","last_fullsized":"2024-08-17T17:41:13.100394+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1796,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.106560+0000","last_change":"2024-08-17T17:40:58.504477+0000","last_active":"2024-08-17T17:41:14.106560+0000","last_peered":"2024-08-17T17:41:14.106560+0000","last_clean":"2024-08-17T17:41:14.106560+0000","last_became_active":"2024-08-17T17:35:55.204861+0000","last_became_peered":"2024-08-17T17:35:55.204861+0000","last_unstale":"2024-08-17T17:41:14.106560+0000","last_undegraded":"2024-08-17T17:41:14.106560+0000","last_fullsized":"2024-08-17T17:41:14.106560+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:58.504387+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:40:58.504387+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:24:54.058501+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1043,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123373+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:41:13.123373+0000","last_peered":"2024-08-17T17:41:13.123373+0000","last_clean":"2024-08-17T17:41:13.123373+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:41:13.123373+0000","last_undegraded":"2024-08-17T17:41:13.123373+0000","last_fullsized":"2024-08-17T17:41:13.123373+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":128,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.159748+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:41:13.159748+0000","last_peered":"2024-08-17T17:41:13.159748+0000","last_clean":"2024-08-17T17:41:13.159748+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:41:13.159748+0000","last_undegraded":"2024-08-17T17:41:13.159748+0000","last_fullsized":"2024-08-17T17:41:13.159748+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1779,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100242+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:41:13.100242+0000","last_peered":"2024-08-17T17:41:13.100242+0000","last_clean":"2024-08-17T17:41:13.100242+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:41:13.100242+0000","last_undegraded":"2024-08-17T17:41:13.100242+0000","last_fullsized":"2024-08-17T17:41:13.100242+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1458,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124213+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:41:13.124213+0000","last_peered":"2024-08-17T17:41:13.124213+0000","last_clean":"2024-08-17T17:41:13.124213+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:41:13.124213+0000","last_undegraded":"2024-08-17T17:41:13.124213+0000","last_fullsized":"2024-08-17T17:41:13.124213+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1430,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123402+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:41:13.123402+0000","last_peered":"2024-08-17T17:41:13.123402+0000","last_clean":"2024-08-17T17:41:13.123402+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:41:13.123402+0000","last_undegraded":"2024-08-17T17:41:13.123402+0000","last_fullsized":"2024-08-17T17:41:13.123402+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":1031,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.108310+0000","last_change":"2024-08-17T17:34:26.817339+0000","last_active":"2024-08-17T17:41:14.108310+0000","last_peered":"2024-08-17T17:41:14.108310+0000","last_clean":"2024-08-17T17:41:14.108310+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:41:14.108310+0000","last_undegraded":"2024-08-17T17:41:14.108310+0000","last_fullsized":"2024-08-17T17:41:14.108310+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:26.817223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:26.817223+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:30.089956+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":66,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124383+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:41:13.124383+0000","last_peered":"2024-08-17T17:41:13.124383+0000","last_clean":"2024-08-17T17:41:13.124383+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:41:13.124383+0000","last_undegraded":"2024-08-17T17:41:13.124383+0000","last_fullsized":"2024-08-17T17:41:13.124383+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1292,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.102008+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:41:13.102008+0000","last_peered":"2024-08-17T17:41:13.102008+0000","last_clean":"2024-08-17T17:41:13.102008+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:41:13.102008+0000","last_undegraded":"2024-08-17T17:41:13.102008+0000","last_fullsized":"2024-08-17T17:41:13.102008+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1807,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.099925+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:41:13.099925+0000","last_peered":"2024-08-17T17:41:13.099925+0000","last_clean":"2024-08-17T17:41:13.099925+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:41:13.099925+0000","last_undegraded":"2024-08-17T17:41:13.099925+0000","last_fullsized":"2024-08-17T17:41:13.099925+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1409,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123562+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:41:13.123562+0000","last_peered":"2024-08-17T17:41:13.123562+0000","last_clean":"2024-08-17T17:41:13.123562+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:41:13.123562+0000","last_undegraded":"2024-08-17T17:41:13.123562+0000","last_fullsized":"2024-08-17T17:41:13.123562+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":222,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123749+0000","last_change":"2024-08-17T17:36:12.754372+0000","last_active":"2024-08-17T17:41:13.123749+0000","last_peered":"2024-08-17T17:41:13.123749+0000","last_clean":"2024-08-17T17:41:13.123749+0000","last_became_active":"2024-08-17T17:36:12.746352+0000","last_became_peered":"2024-08-17T17:36:12.746352+0000","last_unstale":"2024-08-17T17:41:13.123749+0000","last_undegraded":"2024-08-17T17:41:13.123749+0000","last_fullsized":"2024-08-17T17:41:13.123749+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:07.807486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1703,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.090894+0000","last_change":"2024-08-17T17:36:15.771193+0000","last_active":"2024-08-17T17:41:15.090894+0000","last_peered":"2024-08-17T17:41:15.090894+0000","last_clean":"2024-08-17T17:41:15.090894+0000","last_became_active":"2024-08-17T17:36:15.770889+0000","last_became_peered":"2024-08-17T17:36:15.770889+0000","last_unstale":"2024-08-17T17:41:15.090894+0000","last_undegraded":"2024-08-17T17:41:15.090894+0000","last_fullsized":"2024-08-17T17:41:15.090894+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:22.978695+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1298,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.109237+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:41:14.109237+0000","last_peered":"2024-08-17T17:41:14.109237+0000","last_clean":"2024-08-17T17:41:14.109237+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:41:14.109237+0000","last_undegraded":"2024-08-17T17:41:14.109237+0000","last_fullsized":"2024-08-17T17:41:14.109237+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1435,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100729+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:41:13.100729+0000","last_peered":"2024-08-17T17:41:13.100729+0000","last_clean":"2024-08-17T17:41:13.100729+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:41:13.100729+0000","last_undegraded":"2024-08-17T17:41:13.100729+0000","last_fullsized":"2024-08-17T17:41:13.100729+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":935,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123958+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:41:13.123958+0000","last_peered":"2024-08-17T17:41:13.123958+0000","last_clean":"2024-08-17T17:41:13.123958+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:41:13.123958+0000","last_undegraded":"2024-08-17T17:41:13.123958+0000","last_fullsized":"2024-08-17T17:41:13.123958+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1291,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.102128+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:41:13.102128+0000","last_peered":"2024-08-17T17:41:13.102128+0000","last_clean":"2024-08-17T17:41:13.102128+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:41:13.102128+0000","last_undegraded":"2024-08-17T17:41:13.102128+0000","last_fullsized":"2024-08-17T17:41:13.102128+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":171,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100848+0000","last_change":"2024-08-17T17:35:55.217072+0000","last_active":"2024-08-17T17:41:13.100848+0000","last_peered":"2024-08-17T17:41:13.100848+0000","last_clean":"2024-08-17T17:41:13.100848+0000","last_became_active":"2024-08-17T17:35:55.216462+0000","last_became_peered":"2024-08-17T17:35:55.216462+0000","last_unstale":"2024-08-17T17:41:13.100848+0000","last_undegraded":"2024-08-17T17:41:13.100848+0000","last_fullsized":"2024-08-17T17:41:13.100848+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:34.937221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":930,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.102810+0000","last_change":"2024-08-17T17:40:56.322072+0000","last_active":"2024-08-17T17:41:13.102810+0000","last_peered":"2024-08-17T17:41:13.102810+0000","last_clean":"2024-08-17T17:41:13.102810+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:41:13.102810+0000","last_undegraded":"2024-08-17T17:41:13.102810+0000","last_fullsized":"2024-08-17T17:41:13.102810+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:56.321976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:40:56.321976+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:45.244567+0000","scrub_duration":0.0014456440000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1476,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123324+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:41:13.123324+0000","last_peered":"2024-08-17T17:41:13.123324+0000","last_clean":"2024-08-17T17:41:13.123324+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:41:13.123324+0000","last_undegraded":"2024-08-17T17:41:13.123324+0000","last_fullsized":"2024-08-17T17:41:13.123324+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":183,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124294+0000","last_change":"2024-08-17T17:35:33.147672+0000","last_active":"2024-08-17T17:41:13.124294+0000","last_peered":"2024-08-17T17:41:13.124294+0000","last_clean":"2024-08-17T17:41:13.124294+0000","last_became_active":"2024-08-17T17:35:33.147283+0000","last_became_peered":"2024-08-17T17:35:33.147283+0000","last_unstale":"2024-08-17T17:41:13.124294+0000","last_undegraded":"2024-08-17T17:41:13.124294+0000","last_fullsized":"2024-08-17T17:41:13.124294+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:41.677709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1727,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.160293+0000","last_change":"2024-08-17T17:40:55.554364+0000","last_active":"2024-08-17T17:41:13.160293+0000","last_peered":"2024-08-17T17:41:13.160293+0000","last_clean":"2024-08-17T17:41:13.160293+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:41:13.160293+0000","last_undegraded":"2024-08-17T17:41:13.160293+0000","last_fullsized":"2024-08-17T17:41:13.160293+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:55.554234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:40:55.554234+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:49.333767+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1400,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100168+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:41:13.100168+0000","last_peered":"2024-08-17T17:41:13.100168+0000","last_clean":"2024-08-17T17:41:13.100168+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:41:13.100168+0000","last_undegraded":"2024-08-17T17:41:13.100168+0000","last_fullsized":"2024-08-17T17:41:13.100168+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":170,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123470+0000","last_change":"2024-08-17T17:35:33.140725+0000","last_active":"2024-08-17T17:41:13.123470+0000","last_peered":"2024-08-17T17:41:13.123470+0000","last_clean":"2024-08-17T17:41:13.123470+0000","last_became_active":"2024-08-17T17:35:33.140406+0000","last_became_peered":"2024-08-17T17:35:33.140406+0000","last_unstale":"2024-08-17T17:41:13.123470+0000","last_undegraded":"2024-08-17T17:41:13.123470+0000","last_fullsized":"2024-08-17T17:41:13.123470+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:54:51.875571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1690,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124257+0000","last_change":"2024-08-17T17:35:33.138514+0000","last_active":"2024-08-17T17:41:13.124257+0000","last_peered":"2024-08-17T17:41:13.124257+0000","last_clean":"2024-08-17T17:41:13.124257+0000","last_became_active":"2024-08-17T17:35:33.137035+0000","last_became_peered":"2024-08-17T17:35:33.137035+0000","last_unstale":"2024-08-17T17:41:13.124257+0000","last_undegraded":"2024-08-17T17:41:13.124257+0000","last_fullsized":"2024-08-17T17:41:13.124257+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:50.565662+0000","scrub_duration":0.001487138,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1332,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.101065+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:41:13.101065+0000","last_peered":"2024-08-17T17:41:13.101065+0000","last_clean":"2024-08-17T17:41:13.101065+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:41:13.101065+0000","last_undegraded":"2024-08-17T17:41:13.101065+0000","last_fullsized":"2024-08-17T17:41:13.101065+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":128,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.160702+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:41:13.160702+0000","last_peered":"2024-08-17T17:41:13.160702+0000","last_clean":"2024-08-17T17:41:13.160702+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:41:13.160702+0000","last_undegraded":"2024-08-17T17:41:13.160702+0000","last_fullsized":"2024-08-17T17:41:13.160702+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1763,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100341+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:41:13.100341+0000","last_peered":"2024-08-17T17:41:13.100341+0000","last_clean":"2024-08-17T17:41:13.100341+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:41:13.100341+0000","last_undegraded":"2024-08-17T17:41:13.100341+0000","last_fullsized":"2024-08-17T17:41:13.100341+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1349,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123194+0000","last_change":"2024-08-17T17:35:55.205292+0000","last_active":"2024-08-17T17:41:13.123194+0000","last_peered":"2024-08-17T17:41:13.123194+0000","last_clean":"2024-08-17T17:41:13.123194+0000","last_became_active":"2024-08-17T17:35:55.204935+0000","last_became_peered":"2024-08-17T17:35:55.204935+0000","last_unstale":"2024-08-17T17:41:13.123194+0000","last_undegraded":"2024-08-17T17:41:13.123194+0000","last_fullsized":"2024-08-17T17:41:13.123194+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:52.277821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":128,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.160673+0000","last_change":"2024-08-17T17:36:12.749972+0000","last_active":"2024-08-17T17:41:13.160673+0000","last_peered":"2024-08-17T17:41:13.160673+0000","last_clean":"2024-08-17T17:41:13.160673+0000","last_became_active":"2024-08-17T17:36:12.749620+0000","last_became_peered":"2024-08-17T17:36:12.749620+0000","last_unstale":"2024-08-17T17:41:13.160673+0000","last_undegraded":"2024-08-17T17:41:13.160673+0000","last_fullsized":"2024-08-17T17:41:13.160673+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:04.994357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":1004,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.160189+0000","last_change":"2024-08-17T17:36:20.832274+0000","last_active":"2024-08-17T17:41:13.160189+0000","last_peered":"2024-08-17T17:41:13.160189+0000","last_clean":"2024-08-17T17:41:13.160189+0000","last_became_active":"2024-08-17T17:36:20.831886+0000","last_became_peered":"2024-08-17T17:36:20.831886+0000","last_unstale":"2024-08-17T17:41:13.160189+0000","last_undegraded":"2024-08-17T17:41:13.160189+0000","last_fullsized":"2024-08-17T17:41:13.160189+0000","mapping_epoch":1390,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1391,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:05.248711+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1215,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.106053+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:41:14.106053+0000","last_peered":"2024-08-17T17:41:14.106053+0000","last_clean":"2024-08-17T17:41:14.106053+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:41:14.106053+0000","last_undegraded":"2024-08-17T17:41:14.106053+0000","last_fullsized":"2024-08-17T17:41:14.106053+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1432,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100839+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:41:13.100839+0000","last_peered":"2024-08-17T17:41:13.100839+0000","last_clean":"2024-08-17T17:41:13.100839+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:41:13.100839+0000","last_undegraded":"2024-08-17T17:41:13.100839+0000","last_fullsized":"2024-08-17T17:41:13.100839+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1726,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.099925+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:41:13.099925+0000","last_peered":"2024-08-17T17:41:13.099925+0000","last_clean":"2024-08-17T17:41:13.099925+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:41:13.099925+0000","last_undegraded":"2024-08-17T17:41:13.099925+0000","last_fullsized":"2024-08-17T17:41:13.099925+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1374,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124087+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:41:13.124087+0000","last_peered":"2024-08-17T17:41:13.124087+0000","last_clean":"2024-08-17T17:41:13.124087+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:41:13.124087+0000","last_undegraded":"2024-08-17T17:41:13.124087+0000","last_fullsized":"2024-08-17T17:41:13.124087+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1932,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.160408+0000","last_change":"2024-08-17T17:35:33.137411+0000","last_active":"2024-08-17T17:41:13.160408+0000","last_peered":"2024-08-17T17:41:13.160408+0000","last_clean":"2024-08-17T17:41:13.160408+0000","last_became_active":"2024-08-17T17:35:33.137073+0000","last_became_peered":"2024-08-17T17:35:33.137073+0000","last_unstale":"2024-08-17T17:41:13.160408+0000","last_undegraded":"2024-08-17T17:41:13.160408+0000","last_fullsized":"2024-08-17T17:41:13.160408+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:35.924539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1820,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100175+0000","last_change":"2024-08-17T17:36:15.787445+0000","last_active":"2024-08-17T17:41:13.100175+0000","last_peered":"2024-08-17T17:41:13.100175+0000","last_clean":"2024-08-17T17:41:13.100175+0000","last_became_active":"2024-08-17T17:36:15.787099+0000","last_became_peered":"2024-08-17T17:36:15.787099+0000","last_unstale":"2024-08-17T17:41:13.100175+0000","last_undegraded":"2024-08-17T17:41:13.100175+0000","last_fullsized":"2024-08-17T17:41:13.100175+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:47.448168+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162886,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":168464,"kb_used_data":7940,"kb_used_omap":54,"kb_used_meta":160457,"kb_avail":93576688,"statfs":{"total":95995035648,"available":95822528512,"internally_reserved":0,"allocated":8130560,"data_stored":5373368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":164307989},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999360,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":230024,"kb_used_data":7996,"kb_used_omap":47,"kb_used_meta":221968,"kb_avail":93515128,"statfs":{"total":95995035648,"available":95759491072,"internally_reserved":0,"allocated":8187904,"data_stored":5439275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":227295892},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627156,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1116660,"kb_used_data":7972,"kb_used_omap":42,"kb_used_meta":1108629,"kb_avail":92628492,"statfs":{"total":95995035648,"available":94851575808,"internally_reserved":0,"allocated":8163328,"data_stored":5425353,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1135236156},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785634,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":860460,"kb_used_data":7660,"kb_used_omap":48,"kb_used_meta":852687,"kb_avail":92884692,"statfs":{"total":95995035648,"available":95113924608,"internally_reserved":0,"allocated":7843840,"data_stored":5106781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":873152174},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725151,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":901184,"kb_used_data":7924,"kb_used_omap":1,"kb_used_meta":893246,"kb_avail":92843968,"statfs":{"total":95995035648,"available":95072223232,"internally_reserved":0,"allocated":8114176,"data_stored":5369067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":914684358},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986788,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":918984,"kb_used_data":7676,"kb_used_omap":1,"kb_used_meta":911294,"kb_avail":92826168,"statfs":{"total":95995035648,"available":95053996032,"internally_reserved":0,"allocated":7860224,"data_stored":5110610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":933165510},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706395,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":942384,"kb_used_data":7920,"kb_used_omap":35,"kb_used_meta":934364,"kb_avail":92802768,"statfs":{"total":95995035648,"available":95030034432,"internally_reserved":0,"allocated":8110080,"data_stored":5369024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":956788791},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832168,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1060804,"kb_used_data":7984,"kb_used_omap":65,"kb_used_meta":1052734,"kb_avail":92684348,"statfs":{"total":95995035648,"available":94908772352,"internally_reserved":0,"allocated":8175616,"data_stored":5435046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1077999803},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:41:21.445 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:21.446 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:21.799 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:21 smithi063 ceph-mon[114756]: pgmap v2324: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:21.799 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:21 smithi063 ceph-mon[109843]: pgmap v2324: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:21.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:21 smithi162 ceph-mon[104668]: pgmap v2324: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:22.214 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-08-17T17:41:22.590 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:23.277 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:23.748 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:41:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:41:23] "GET /metrics HTTP/1.1" 200 34114 "" "Prometheus/2.51.0" 2024-08-17T17:41:23.969 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:41:24.001 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:23 smithi063 ceph-mon[114756]: pgmap v2325: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:24.001 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:23 smithi063 ceph-mon[109843]: pgmap v2325: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:23 smithi162 ceph-mon[104668]: pgmap v2325: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:25.626 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:25.846 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:25 smithi063 ceph-mon[109843]: pgmap v2326: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:25.846 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:25 smithi063 ceph-mon[114756]: pgmap v2326: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:26.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:25 smithi162 ceph-mon[104668]: pgmap v2326: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:26.617 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:26.617 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:27.753 INFO:teuthology.orchestra.run.smithi063.stderr:noscrub is unset 2024-08-17T17:41:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:27 smithi063 ceph-mon[114756]: pgmap v2327: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:27 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/125312346' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:41:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:27 smithi063 ceph-mon[109843]: pgmap v2327: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:27 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/125312346' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:41:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:27 smithi162 ceph-mon[104668]: pgmap v2327: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:27 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/125312346' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-08-17T17:41:28.495 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-08-17T17:41:29.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:28 smithi063 ceph-mon[114756]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2024-08-17T17:41:29.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:28 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:41:29.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:28 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/125312346' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:41:29.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:28 smithi063 ceph-mon[114756]: osdmap e1442: 8 total, 8 up, 8 in 2024-08-17T17:41:29.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:28 smithi063 ceph-mon[109843]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2024-08-17T17:41:29.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:28 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:41:29.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:28 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/125312346' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:41:29.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:28 smithi063 ceph-mon[109843]: osdmap e1442: 8 total, 8 up, 8 in 2024-08-17T17:41:29.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:28 smithi162 ceph-mon[104668]: Health check cleared: OSDMAP_FLAGS (was: noscrub flag(s) set) 2024-08-17T17:41:29.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:28 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:41:29.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:28 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/125312346' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-08-17T17:41:29.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:28 smithi162 ceph-mon[104668]: osdmap e1442: 8 total, 8 up, 8 in 2024-08-17T17:41:29.152 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd unset nodeep-scrub 2024-08-17T17:41:30.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:29 smithi063 ceph-mon[109843]: pgmap v2329: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:30.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:29 smithi063 ceph-mon[114756]: pgmap v2329: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:30.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:29 smithi162 ceph-mon[104668]: pgmap v2329: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:30.180 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:30.581 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:41:30.582 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:41:31.213 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:31.213 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:31 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:31.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:31 smithi063 ceph-mon[109843]: from='client.36633 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:31.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:31 smithi063 ceph-mon[109843]: pgmap v2330: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:31.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:31.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:31 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:31.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:31 smithi063 ceph-mon[114756]: from='client.36633 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:31.530 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:31 smithi063 ceph-mon[114756]: pgmap v2330: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:31.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:31.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:31 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:31.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:31 smithi162 ceph-mon[104668]: from='client.36633 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:31.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:31 smithi162 ceph-mon[104668]: pgmap v2330: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:31.788 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:31.792 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2329,"stamp":"2024-08-17T17:41:28.695934+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":526015,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4678,"ondisk_log_size":4678,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6198964,"kb_used_data":63076,"kb_used_omap":298,"kb_used_meta":6135381,"kb_avail":743762252,"statfs":{"total":767960285184,"available":761612546048,"internally_reserved":0,"allocated":64589824,"data_stored":42636661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":6282630673},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002498"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":90,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.107763+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:41:14.107763+0000","last_peered":"2024-08-17T17:41:14.107763+0000","last_clean":"2024-08-17T17:41:14.107763+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:41:14.107763+0000","last_undegraded":"2024-08-17T17:41:14.107763+0000","last_fullsized":"2024-08-17T17:41:14.107763+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1456,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123659+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:41:13.123659+0000","last_peered":"2024-08-17T17:41:13.123659+0000","last_clean":"2024-08-17T17:41:13.123659+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:41:13.123659+0000","last_undegraded":"2024-08-17T17:41:13.123659+0000","last_fullsized":"2024-08-17T17:41:13.123659+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1771,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124170+0000","last_change":"2024-08-17T17:36:15.771150+0000","last_active":"2024-08-17T17:41:13.124170+0000","last_peered":"2024-08-17T17:41:13.124170+0000","last_clean":"2024-08-17T17:41:13.124170+0000","last_became_active":"2024-08-17T17:36:15.770520+0000","last_became_peered":"2024-08-17T17:36:15.770520+0000","last_unstale":"2024-08-17T17:41:13.124170+0000","last_undegraded":"2024-08-17T17:41:13.124170+0000","last_fullsized":"2024-08-17T17:41:13.124170+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:17.552796+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":107,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.809698+0000","last_change":"2024-08-17T17:36:12.756709+0000","last_active":"2024-08-17T17:41:14.809698+0000","last_peered":"2024-08-17T17:41:14.809698+0000","last_clean":"2024-08-17T17:41:14.809698+0000","last_became_active":"2024-08-17T17:36:12.756385+0000","last_became_peered":"2024-08-17T17:36:12.756385+0000","last_unstale":"2024-08-17T17:41:14.809698+0000","last_undegraded":"2024-08-17T17:41:14.809698+0000","last_fullsized":"2024-08-17T17:41:14.809698+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:21.420282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":174,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091082+0000","last_change":"2024-08-17T17:36:12.762800+0000","last_active":"2024-08-17T17:41:15.091082+0000","last_peered":"2024-08-17T17:41:15.091082+0000","last_clean":"2024-08-17T17:41:15.091082+0000","last_became_active":"2024-08-17T17:36:12.759977+0000","last_became_peered":"2024-08-17T17:36:12.759977+0000","last_unstale":"2024-08-17T17:41:15.091082+0000","last_undegraded":"2024-08-17T17:41:15.091082+0000","last_fullsized":"2024-08-17T17:41:15.091082+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:02.162681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1664,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.809358+0000","last_change":"2024-08-17T17:41:03.323402+0000","last_active":"2024-08-17T17:41:14.809358+0000","last_peered":"2024-08-17T17:41:14.809358+0000","last_clean":"2024-08-17T17:41:14.809358+0000","last_became_active":"2024-08-17T17:35:56.278638+0000","last_became_peered":"2024-08-17T17:35:56.278638+0000","last_unstale":"2024-08-17T17:41:14.809358+0000","last_undegraded":"2024-08-17T17:41:14.809358+0000","last_fullsized":"2024-08-17T17:41:14.809358+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:41:03.323294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:41:03.323294+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:28.872583+0000","scrub_duration":0.002060147,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1390,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.733244+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:41:14.733244+0000","last_peered":"2024-08-17T17:41:14.733244+0000","last_clean":"2024-08-17T17:41:14.733244+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:41:14.733244+0000","last_undegraded":"2024-08-17T17:41:14.733244+0000","last_fullsized":"2024-08-17T17:41:14.733244+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":66,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091151+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:41:15.091151+0000","last_peered":"2024-08-17T17:41:15.091151+0000","last_clean":"2024-08-17T17:41:15.091151+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:41:15.091151+0000","last_undegraded":"2024-08-17T17:41:15.091151+0000","last_fullsized":"2024-08-17T17:41:15.091151+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1435,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100722+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:41:13.100722+0000","last_peered":"2024-08-17T17:41:13.100722+0000","last_clean":"2024-08-17T17:41:13.100722+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:41:13.100722+0000","last_undegraded":"2024-08-17T17:41:13.100722+0000","last_fullsized":"2024-08-17T17:41:13.100722+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1443,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123904+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:41:13.123904+0000","last_peered":"2024-08-17T17:41:13.123904+0000","last_clean":"2024-08-17T17:41:13.123904+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:41:13.123904+0000","last_undegraded":"2024-08-17T17:41:13.123904+0000","last_fullsized":"2024-08-17T17:41:13.123904+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":734,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.809354+0000","last_change":"2024-08-17T17:36:15.773315+0000","last_active":"2024-08-17T17:41:14.809354+0000","last_peered":"2024-08-17T17:41:14.809354+0000","last_clean":"2024-08-17T17:41:14.809354+0000","last_became_active":"2024-08-17T17:36:15.772961+0000","last_became_peered":"2024-08-17T17:36:15.772961+0000","last_unstale":"2024-08-17T17:41:14.809354+0000","last_undegraded":"2024-08-17T17:41:14.809354+0000","last_fullsized":"2024-08-17T17:41:14.809354+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:10.190221+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":83,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.108196+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:41:14.108196+0000","last_peered":"2024-08-17T17:41:14.108196+0000","last_clean":"2024-08-17T17:41:14.108196+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:41:14.108196+0000","last_undegraded":"2024-08-17T17:41:14.108196+0000","last_fullsized":"2024-08-17T17:41:14.108196+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":174,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091099+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:41:15.091099+0000","last_peered":"2024-08-17T17:41:15.091099+0000","last_clean":"2024-08-17T17:41:15.091099+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:41:15.091099+0000","last_undegraded":"2024-08-17T17:41:15.091099+0000","last_fullsized":"2024-08-17T17:41:15.091099+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1525,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.808594+0000","last_change":"2024-08-17T17:36:24.225739+0000","last_active":"2024-08-17T17:41:14.808594+0000","last_peered":"2024-08-17T17:41:14.808594+0000","last_clean":"2024-08-17T17:41:14.808594+0000","last_became_active":"2024-08-17T17:36:24.225250+0000","last_became_peered":"2024-08-17T17:36:24.225250+0000","last_unstale":"2024-08-17T17:41:14.808594+0000","last_undegraded":"2024-08-17T17:41:14.808594+0000","last_fullsized":"2024-08-17T17:41:14.808594+0000","mapping_epoch":1393,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1394,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.290656+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1463,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123660+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:41:13.123660+0000","last_peered":"2024-08-17T17:41:13.123660+0000","last_clean":"2024-08-17T17:41:13.123660+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:41:13.123660+0000","last_undegraded":"2024-08-17T17:41:13.123660+0000","last_fullsized":"2024-08-17T17:41:13.123660+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1870,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124298+0000","last_change":"2024-08-17T17:36:12.752970+0000","last_active":"2024-08-17T17:41:13.124298+0000","last_peered":"2024-08-17T17:41:13.124298+0000","last_clean":"2024-08-17T17:41:13.124298+0000","last_became_active":"2024-08-17T17:36:12.752311+0000","last_became_peered":"2024-08-17T17:36:12.752311+0000","last_unstale":"2024-08-17T17:41:13.124298+0000","last_undegraded":"2024-08-17T17:41:13.124298+0000","last_fullsized":"2024-08-17T17:41:13.124298+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:38.131419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1817,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100043+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:41:13.100043+0000","last_peered":"2024-08-17T17:41:13.100043+0000","last_clean":"2024-08-17T17:41:13.100043+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:41:13.100043+0000","last_undegraded":"2024-08-17T17:41:13.100043+0000","last_fullsized":"2024-08-17T17:41:13.100043+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1546,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.796428+0000","last_change":"2024-08-17T17:36:18.750494+0000","last_active":"2024-08-17T17:41:14.796428+0000","last_peered":"2024-08-17T17:41:14.796428+0000","last_clean":"2024-08-17T17:41:14.796428+0000","last_became_active":"2024-08-17T17:36:18.750092+0000","last_became_peered":"2024-08-17T17:36:18.750092+0000","last_unstale":"2024-08-17T17:41:14.796428+0000","last_undegraded":"2024-08-17T17:41:14.796428+0000","last_fullsized":"2024-08-17T17:41:14.796428+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:36.599727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1176,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.108329+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:41:14.108329+0000","last_peered":"2024-08-17T17:41:14.108329+0000","last_clean":"2024-08-17T17:41:14.108329+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:41:14.108329+0000","last_undegraded":"2024-08-17T17:41:14.108329+0000","last_fullsized":"2024-08-17T17:41:14.108329+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":984,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100274+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:41:13.100274+0000","last_peered":"2024-08-17T17:41:13.100274+0000","last_clean":"2024-08-17T17:41:13.100274+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:41:13.100274+0000","last_undegraded":"2024-08-17T17:41:13.100274+0000","last_fullsized":"2024-08-17T17:41:13.100274+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":83,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.106460+0000","last_change":"2024-08-17T17:37:11.075505+0000","last_active":"2024-08-17T17:41:14.106460+0000","last_peered":"2024-08-17T17:41:14.106460+0000","last_clean":"2024-08-17T17:41:14.106460+0000","last_became_active":"2024-08-17T17:37:11.075131+0000","last_became_peered":"2024-08-17T17:37:11.075131+0000","last_unstale":"2024-08-17T17:41:14.106460+0000","last_undegraded":"2024-08-17T17:41:14.106460+0000","last_fullsized":"2024-08-17T17:41:14.106460+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:39.672054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1298,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.810476+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:41:14.810476+0000","last_peered":"2024-08-17T17:41:14.810476+0000","last_clean":"2024-08-17T17:41:14.810476+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:41:14.810476+0000","last_undegraded":"2024-08-17T17:41:14.810476+0000","last_fullsized":"2024-08-17T17:41:14.810476+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1674,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123237+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:41:13.123237+0000","last_peered":"2024-08-17T17:41:13.123237+0000","last_clean":"2024-08-17T17:41:13.123237+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:41:13.123237+0000","last_undegraded":"2024-08-17T17:41:13.123237+0000","last_fullsized":"2024-08-17T17:41:13.123237+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":130,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.796523+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:41:14.796523+0000","last_peered":"2024-08-17T17:41:14.796523+0000","last_clean":"2024-08-17T17:41:14.796523+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:41:14.796523+0000","last_undegraded":"2024-08-17T17:41:14.796523+0000","last_fullsized":"2024-08-17T17:41:14.796523+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1271,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.101366+0000","last_change":"2024-08-17T17:36:12.755873+0000","last_active":"2024-08-17T17:41:13.101366+0000","last_peered":"2024-08-17T17:41:13.101366+0000","last_clean":"2024-08-17T17:41:13.101366+0000","last_became_active":"2024-08-17T17:36:12.755366+0000","last_became_peered":"2024-08-17T17:36:12.755366+0000","last_unstale":"2024-08-17T17:41:13.101366+0000","last_undegraded":"2024-08-17T17:41:13.101366+0000","last_fullsized":"2024-08-17T17:41:13.101366+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:36.523214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1625,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.125309+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:41:13.125309+0000","last_peered":"2024-08-17T17:41:13.125309+0000","last_clean":"2024-08-17T17:41:13.125309+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:41:13.125309+0000","last_undegraded":"2024-08-17T17:41:13.125309+0000","last_fullsized":"2024-08-17T17:41:13.125309+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":810,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.809969+0000","last_change":"2024-08-17T17:35:55.200519+0000","last_active":"2024-08-17T17:41:14.809969+0000","last_peered":"2024-08-17T17:41:14.809969+0000","last_clean":"2024-08-17T17:41:14.809969+0000","last_became_active":"2024-08-17T17:35:55.200230+0000","last_became_peered":"2024-08-17T17:35:55.200230+0000","last_unstale":"2024-08-17T17:41:14.809969+0000","last_undegraded":"2024-08-17T17:41:14.809969+0000","last_fullsized":"2024-08-17T17:41:14.809969+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:33.454851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1448,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100673+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:41:13.100673+0000","last_peered":"2024-08-17T17:41:13.100673+0000","last_clean":"2024-08-17T17:41:13.100673+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:41:13.100673+0000","last_undegraded":"2024-08-17T17:41:13.100673+0000","last_fullsized":"2024-08-17T17:41:13.100673+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1747,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.090894+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:41:15.090894+0000","last_peered":"2024-08-17T17:41:15.090894+0000","last_clean":"2024-08-17T17:41:15.090894+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:41:15.090894+0000","last_undegraded":"2024-08-17T17:41:15.090894+0000","last_fullsized":"2024-08-17T17:41:15.090894+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:22.077229+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":294,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.797116+0000","last_change":"2024-08-17T17:35:33.145476+0000","last_active":"2024-08-17T17:41:14.797116+0000","last_peered":"2024-08-17T17:41:14.797116+0000","last_clean":"2024-08-17T17:41:14.797116+0000","last_became_active":"2024-08-17T17:35:33.145125+0000","last_became_peered":"2024-08-17T17:35:33.145125+0000","last_unstale":"2024-08-17T17:41:14.797116+0000","last_undegraded":"2024-08-17T17:41:14.797116+0000","last_fullsized":"2024-08-17T17:41:14.797116+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:45.298064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":954,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.808835+0000","last_change":"2024-08-17T17:35:56.248612+0000","last_active":"2024-08-17T17:41:14.808835+0000","last_peered":"2024-08-17T17:41:14.808835+0000","last_clean":"2024-08-17T17:41:14.808835+0000","last_became_active":"2024-08-17T17:35:56.244364+0000","last_became_peered":"2024-08-17T17:35:56.244364+0000","last_unstale":"2024-08-17T17:41:14.808835+0000","last_undegraded":"2024-08-17T17:41:14.808835+0000","last_fullsized":"2024-08-17T17:41:14.808835+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:41.064165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":892,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.101028+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:41:13.101028+0000","last_peered":"2024-08-17T17:41:13.101028+0000","last_clean":"2024-08-17T17:41:13.101028+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:41:13.101028+0000","last_undegraded":"2024-08-17T17:41:13.101028+0000","last_fullsized":"2024-08-17T17:41:13.101028+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1428,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123708+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:41:13.123708+0000","last_peered":"2024-08-17T17:41:13.123708+0000","last_clean":"2024-08-17T17:41:13.123708+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:41:13.123708+0000","last_undegraded":"2024-08-17T17:41:13.123708+0000","last_fullsized":"2024-08-17T17:41:13.123708+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1447,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124052+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:41:13.124052+0000","last_peered":"2024-08-17T17:41:13.124052+0000","last_clean":"2024-08-17T17:41:13.124052+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:41:13.124052+0000","last_undegraded":"2024-08-17T17:41:13.124052+0000","last_fullsized":"2024-08-17T17:41:13.124052+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":969,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.108710+0000","last_change":"2024-08-17T17:36:15.774881+0000","last_active":"2024-08-17T17:41:14.108710+0000","last_peered":"2024-08-17T17:41:14.108710+0000","last_clean":"2024-08-17T17:41:14.108710+0000","last_became_active":"2024-08-17T17:36:15.774435+0000","last_became_peered":"2024-08-17T17:36:15.774435+0000","last_unstale":"2024-08-17T17:41:14.108710+0000","last_undegraded":"2024-08-17T17:41:14.108710+0000","last_fullsized":"2024-08-17T17:41:14.108710+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:52.015854+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1395,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123115+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:41:13.123115+0000","last_peered":"2024-08-17T17:41:13.123115+0000","last_clean":"2024-08-17T17:41:13.123115+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:41:13.123115+0000","last_undegraded":"2024-08-17T17:41:13.123115+0000","last_fullsized":"2024-08-17T17:41:13.123115+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1292,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.810205+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:41:14.810205+0000","last_peered":"2024-08-17T17:41:14.810205+0000","last_clean":"2024-08-17T17:41:14.810205+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:41:14.810205+0000","last_undegraded":"2024-08-17T17:41:14.810205+0000","last_fullsized":"2024-08-17T17:41:14.810205+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":362,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.108390+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:41:14.108390+0000","last_peered":"2024-08-17T17:41:14.108390+0000","last_clean":"2024-08-17T17:41:14.108390+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:41:14.108390+0000","last_undegraded":"2024-08-17T17:41:14.108390+0000","last_fullsized":"2024-08-17T17:41:14.108390+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1630,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.797394+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:41:14.797394+0000","last_peered":"2024-08-17T17:41:14.797394+0000","last_clean":"2024-08-17T17:41:14.797394+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:41:14.797394+0000","last_undegraded":"2024-08-17T17:41:14.797394+0000","last_fullsized":"2024-08-17T17:41:14.797394+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:25.751871+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":137,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.796590+0000","last_change":"2024-08-17T17:37:11.079712+0000","last_active":"2024-08-17T17:41:14.796590+0000","last_peered":"2024-08-17T17:41:14.796590+0000","last_clean":"2024-08-17T17:41:14.796590+0000","last_became_active":"2024-08-17T17:37:11.079253+0000","last_became_peered":"2024-08-17T17:37:11.079253+0000","last_unstale":"2024-08-17T17:41:14.796590+0000","last_undegraded":"2024-08-17T17:41:14.796590+0000","last_fullsized":"2024-08-17T17:41:14.796590+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:53.405657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":937,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.809156+0000","last_change":"2024-08-17T17:35:56.223396+0000","last_active":"2024-08-17T17:41:14.809156+0000","last_peered":"2024-08-17T17:41:14.809156+0000","last_clean":"2024-08-17T17:41:14.809156+0000","last_became_active":"2024-08-17T17:35:56.212413+0000","last_became_peered":"2024-08-17T17:35:56.212413+0000","last_unstale":"2024-08-17T17:41:14.809156+0000","last_undegraded":"2024-08-17T17:41:14.809156+0000","last_fullsized":"2024-08-17T17:41:14.809156+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:44.185174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1672,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.108049+0000","last_change":"2024-08-17T17:35:33.136068+0000","last_active":"2024-08-17T17:41:14.108049+0000","last_peered":"2024-08-17T17:41:14.108049+0000","last_clean":"2024-08-17T17:41:14.108049+0000","last_became_active":"2024-08-17T17:35:33.135591+0000","last_became_peered":"2024-08-17T17:35:33.135591+0000","last_unstale":"2024-08-17T17:41:14.108049+0000","last_undegraded":"2024-08-17T17:41:14.108049+0000","last_fullsized":"2024-08-17T17:41:14.108049+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:52.252561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1801,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124433+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:41:13.124433+0000","last_peered":"2024-08-17T17:41:13.124433+0000","last_clean":"2024-08-17T17:41:13.124433+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:41:13.124433+0000","last_undegraded":"2024-08-17T17:41:13.124433+0000","last_fullsized":"2024-08-17T17:41:13.124433+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1014,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100776+0000","last_change":"2024-08-17T17:35:33.138467+0000","last_active":"2024-08-17T17:41:13.100776+0000","last_peered":"2024-08-17T17:41:13.100776+0000","last_clean":"2024-08-17T17:41:13.100776+0000","last_became_active":"2024-08-17T17:35:33.138174+0000","last_became_peered":"2024-08-17T17:35:33.138174+0000","last_unstale":"2024-08-17T17:41:13.100776+0000","last_undegraded":"2024-08-17T17:41:13.100776+0000","last_fullsized":"2024-08-17T17:41:13.100776+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:08.860439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1398,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.125391+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:41:13.125391+0000","last_peered":"2024-08-17T17:41:13.125391+0000","last_clean":"2024-08-17T17:41:13.125391+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:41:13.125391+0000","last_undegraded":"2024-08-17T17:41:13.125391+0000","last_fullsized":"2024-08-17T17:41:13.125391+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":167,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.090549+0000","last_change":"2024-08-17T17:35:14.854881+0000","last_active":"2024-08-17T17:41:15.090549+0000","last_peered":"2024-08-17T17:41:15.090549+0000","last_clean":"2024-08-17T17:41:15.090549+0000","last_became_active":"2024-08-17T17:35:14.854121+0000","last_became_peered":"2024-08-17T17:35:14.854121+0000","last_unstale":"2024-08-17T17:41:15.090549+0000","last_undegraded":"2024-08-17T17:41:15.090549+0000","last_fullsized":"2024-08-17T17:41:15.090549+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.863624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":968,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.809805+0000","last_change":"2024-08-17T17:41:06.387062+0000","last_active":"2024-08-17T17:41:14.809805+0000","last_peered":"2024-08-17T17:41:14.809805+0000","last_clean":"2024-08-17T17:41:14.809805+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:41:14.809805+0000","last_undegraded":"2024-08-17T17:41:14.809805+0000","last_fullsized":"2024-08-17T17:41:14.809805+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:41:06.386955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:41:06.386955+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:44:32.008526+0000","scrub_duration":0.0017975980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1057,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100426+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:41:13.100426+0000","last_peered":"2024-08-17T17:41:13.100426+0000","last_clean":"2024-08-17T17:41:13.100426+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:41:13.100426+0000","last_undegraded":"2024-08-17T17:41:13.100426+0000","last_fullsized":"2024-08-17T17:41:13.100426+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":261,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.108411+0000","last_change":"2024-08-17T17:35:33.145351+0000","last_active":"2024-08-17T17:41:14.108411+0000","last_peered":"2024-08-17T17:41:14.108411+0000","last_clean":"2024-08-17T17:41:14.108411+0000","last_became_active":"2024-08-17T17:35:33.145040+0000","last_became_peered":"2024-08-17T17:35:33.145040+0000","last_unstale":"2024-08-17T17:41:14.108411+0000","last_undegraded":"2024-08-17T17:41:14.108411+0000","last_fullsized":"2024-08-17T17:41:14.108411+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:02.124000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1466,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100949+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:41:13.100949+0000","last_peered":"2024-08-17T17:41:13.100949+0000","last_clean":"2024-08-17T17:41:13.100949+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:41:13.100949+0000","last_undegraded":"2024-08-17T17:41:13.100949+0000","last_fullsized":"2024-08-17T17:41:13.100949+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":951,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.810280+0000","last_change":"2024-08-17T17:41:07.361112+0000","last_active":"2024-08-17T17:41:14.810280+0000","last_peered":"2024-08-17T17:41:14.810280+0000","last_clean":"2024-08-17T17:41:14.810280+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:41:14.810280+0000","last_undegraded":"2024-08-17T17:41:14.810280+0000","last_fullsized":"2024-08-17T17:41:14.810280+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:41:07.360968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:41:07.360968+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:28.377019+0000","scrub_duration":0.001649951,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1522,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091404+0000","last_change":"2024-08-17T17:36:12.759743+0000","last_active":"2024-08-17T17:41:15.091404+0000","last_peered":"2024-08-17T17:41:15.091404+0000","last_clean":"2024-08-17T17:41:15.091404+0000","last_became_active":"2024-08-17T17:36:12.759334+0000","last_became_peered":"2024-08-17T17:36:12.759334+0000","last_unstale":"2024-08-17T17:41:15.091404+0000","last_undegraded":"2024-08-17T17:41:15.091404+0000","last_fullsized":"2024-08-17T17:41:15.091404+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:04.524384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1429,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124249+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:41:13.124249+0000","last_peered":"2024-08-17T17:41:13.124249+0000","last_clean":"2024-08-17T17:41:13.124249+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:41:13.124249+0000","last_undegraded":"2024-08-17T17:41:13.124249+0000","last_fullsized":"2024-08-17T17:41:13.124249+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1472,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100603+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:41:13.100603+0000","last_peered":"2024-08-17T17:41:13.100603+0000","last_clean":"2024-08-17T17:41:13.100603+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:41:13.100603+0000","last_undegraded":"2024-08-17T17:41:13.100603+0000","last_fullsized":"2024-08-17T17:41:13.100603+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1788,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.732798+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:41:14.732798+0000","last_peered":"2024-08-17T17:41:14.732798+0000","last_clean":"2024-08-17T17:41:14.732798+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:41:14.732798+0000","last_undegraded":"2024-08-17T17:41:14.732798+0000","last_fullsized":"2024-08-17T17:41:14.732798+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1105,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.101157+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:41:13.101157+0000","last_peered":"2024-08-17T17:41:13.101157+0000","last_clean":"2024-08-17T17:41:13.101157+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:41:13.101157+0000","last_undegraded":"2024-08-17T17:41:13.101157+0000","last_fullsized":"2024-08-17T17:41:13.101157+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1434,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091570+0000","last_change":"2024-08-17T17:35:56.229622+0000","last_active":"2024-08-17T17:41:15.091570+0000","last_peered":"2024-08-17T17:41:15.091570+0000","last_clean":"2024-08-17T17:41:15.091570+0000","last_became_active":"2024-08-17T17:35:56.229324+0000","last_became_peered":"2024-08-17T17:35:56.229324+0000","last_unstale":"2024-08-17T17:41:15.091570+0000","last_undegraded":"2024-08-17T17:41:15.091570+0000","last_fullsized":"2024-08-17T17:41:15.091570+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:13.980316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2037,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091181+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:41:15.091181+0000","last_peered":"2024-08-17T17:41:15.091181+0000","last_clean":"2024-08-17T17:41:15.091181+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:41:15.091181+0000","last_undegraded":"2024-08-17T17:41:15.091181+0000","last_fullsized":"2024-08-17T17:41:15.091181+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1602,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123324+0000","last_change":"2024-08-17T17:35:33.137074+0000","last_active":"2024-08-17T17:41:13.123324+0000","last_peered":"2024-08-17T17:41:13.123324+0000","last_clean":"2024-08-17T17:41:13.123324+0000","last_became_active":"2024-08-17T17:35:33.136759+0000","last_became_peered":"2024-08-17T17:35:33.136759+0000","last_unstale":"2024-08-17T17:41:13.123324+0000","last_undegraded":"2024-08-17T17:41:13.123324+0000","last_fullsized":"2024-08-17T17:41:13.123324+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:37.263639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:37.263639+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:14.599780+0000","scrub_duration":0.0016181800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1227,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.109527+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:41:14.109527+0000","last_peered":"2024-08-17T17:41:14.109527+0000","last_clean":"2024-08-17T17:41:14.109527+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:41:14.109527+0000","last_undegraded":"2024-08-17T17:41:14.109527+0000","last_fullsized":"2024-08-17T17:41:14.109527+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":66,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123034+0000","last_change":"2024-08-17T17:36:17.774773+0000","last_active":"2024-08-17T17:41:13.123034+0000","last_peered":"2024-08-17T17:41:13.123034+0000","last_clean":"2024-08-17T17:41:13.123034+0000","last_became_active":"2024-08-17T17:36:17.774468+0000","last_became_peered":"2024-08-17T17:36:17.774468+0000","last_unstale":"2024-08-17T17:41:13.123034+0000","last_undegraded":"2024-08-17T17:41:13.123034+0000","last_fullsized":"2024-08-17T17:41:13.123034+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:54.231747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1565,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.106360+0000","last_change":"2024-08-17T17:35:33.137633+0000","last_active":"2024-08-17T17:41:14.106360+0000","last_peered":"2024-08-17T17:41:14.106360+0000","last_clean":"2024-08-17T17:41:14.106360+0000","last_became_active":"2024-08-17T17:35:33.137323+0000","last_became_peered":"2024-08-17T17:35:33.137323+0000","last_unstale":"2024-08-17T17:41:14.106360+0000","last_undegraded":"2024-08-17T17:41:14.106360+0000","last_fullsized":"2024-08-17T17:41:14.106360+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:03.359155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":930,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.809113+0000","last_change":"2024-08-17T17:40:59.325330+0000","last_active":"2024-08-17T17:41:14.809113+0000","last_peered":"2024-08-17T17:41:14.809113+0000","last_clean":"2024-08-17T17:41:14.809113+0000","last_became_active":"2024-08-17T17:35:56.264265+0000","last_became_peered":"2024-08-17T17:35:56.264265+0000","last_unstale":"2024-08-17T17:41:14.809113+0000","last_undegraded":"2024-08-17T17:41:14.809113+0000","last_fullsized":"2024-08-17T17:41:14.809113+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:59.325233+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:40:59.325233+0000","last_clean_scrub_stamp":"2024-08-17T17:40:59.325233+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:22:35.293983+0000","scrub_duration":0.0014772979999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1271,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.811238+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:41:14.811238+0000","last_peered":"2024-08-17T17:41:14.811238+0000","last_clean":"2024-08-17T17:41:14.811238+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:41:14.811238+0000","last_undegraded":"2024-08-17T17:41:14.811238+0000","last_fullsized":"2024-08-17T17:41:14.811238+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1810,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100517+0000","last_change":"2024-08-17T17:36:12.758005+0000","last_active":"2024-08-17T17:41:13.100517+0000","last_peered":"2024-08-17T17:41:13.100517+0000","last_clean":"2024-08-17T17:41:13.100517+0000","last_became_active":"2024-08-17T17:36:12.757695+0000","last_became_peered":"2024-08-17T17:36:12.757695+0000","last_unstale":"2024-08-17T17:41:13.100517+0000","last_undegraded":"2024-08-17T17:41:13.100517+0000","last_fullsized":"2024-08-17T17:41:13.100517+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:51.792220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1448,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100394+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:41:13.100394+0000","last_peered":"2024-08-17T17:41:13.100394+0000","last_clean":"2024-08-17T17:41:13.100394+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:41:13.100394+0000","last_undegraded":"2024-08-17T17:41:13.100394+0000","last_fullsized":"2024-08-17T17:41:13.100394+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1796,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.106560+0000","last_change":"2024-08-17T17:40:58.504477+0000","last_active":"2024-08-17T17:41:14.106560+0000","last_peered":"2024-08-17T17:41:14.106560+0000","last_clean":"2024-08-17T17:41:14.106560+0000","last_became_active":"2024-08-17T17:35:55.204861+0000","last_became_peered":"2024-08-17T17:35:55.204861+0000","last_unstale":"2024-08-17T17:41:14.106560+0000","last_undegraded":"2024-08-17T17:41:14.106560+0000","last_fullsized":"2024-08-17T17:41:14.106560+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:58.504387+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:40:58.504387+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:24:54.058501+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1043,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123373+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:41:13.123373+0000","last_peered":"2024-08-17T17:41:13.123373+0000","last_clean":"2024-08-17T17:41:13.123373+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:41:13.123373+0000","last_undegraded":"2024-08-17T17:41:13.123373+0000","last_fullsized":"2024-08-17T17:41:13.123373+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":130,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.796680+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:41:14.796680+0000","last_peered":"2024-08-17T17:41:14.796680+0000","last_clean":"2024-08-17T17:41:14.796680+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:41:14.796680+0000","last_undegraded":"2024-08-17T17:41:14.796680+0000","last_fullsized":"2024-08-17T17:41:14.796680+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1779,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100242+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:41:13.100242+0000","last_peered":"2024-08-17T17:41:13.100242+0000","last_clean":"2024-08-17T17:41:13.100242+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:41:13.100242+0000","last_undegraded":"2024-08-17T17:41:13.100242+0000","last_fullsized":"2024-08-17T17:41:13.100242+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1458,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124213+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:41:13.124213+0000","last_peered":"2024-08-17T17:41:13.124213+0000","last_clean":"2024-08-17T17:41:13.124213+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:41:13.124213+0000","last_undegraded":"2024-08-17T17:41:13.124213+0000","last_fullsized":"2024-08-17T17:41:13.124213+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1431,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091679+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:41:15.091679+0000","last_peered":"2024-08-17T17:41:15.091679+0000","last_clean":"2024-08-17T17:41:15.091679+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:41:15.091679+0000","last_undegraded":"2024-08-17T17:41:15.091679+0000","last_fullsized":"2024-08-17T17:41:15.091679+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":1031,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.108310+0000","last_change":"2024-08-17T17:34:26.817339+0000","last_active":"2024-08-17T17:41:14.108310+0000","last_peered":"2024-08-17T17:41:14.108310+0000","last_clean":"2024-08-17T17:41:14.108310+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:41:14.108310+0000","last_undegraded":"2024-08-17T17:41:14.108310+0000","last_fullsized":"2024-08-17T17:41:14.108310+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:26.817223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:26.817223+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:30.089956+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":66,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124383+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:41:13.124383+0000","last_peered":"2024-08-17T17:41:13.124383+0000","last_clean":"2024-08-17T17:41:13.124383+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:41:13.124383+0000","last_undegraded":"2024-08-17T17:41:13.124383+0000","last_fullsized":"2024-08-17T17:41:13.124383+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1293,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.808799+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:41:14.808799+0000","last_peered":"2024-08-17T17:41:14.808799+0000","last_clean":"2024-08-17T17:41:14.808799+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:41:14.808799+0000","last_undegraded":"2024-08-17T17:41:14.808799+0000","last_fullsized":"2024-08-17T17:41:14.808799+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1807,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.099925+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:41:13.099925+0000","last_peered":"2024-08-17T17:41:13.099925+0000","last_clean":"2024-08-17T17:41:13.099925+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:41:13.099925+0000","last_undegraded":"2024-08-17T17:41:13.099925+0000","last_fullsized":"2024-08-17T17:41:13.099925+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1409,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123562+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:41:13.123562+0000","last_peered":"2024-08-17T17:41:13.123562+0000","last_clean":"2024-08-17T17:41:13.123562+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:41:13.123562+0000","last_undegraded":"2024-08-17T17:41:13.123562+0000","last_fullsized":"2024-08-17T17:41:13.123562+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":222,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123749+0000","last_change":"2024-08-17T17:36:12.754372+0000","last_active":"2024-08-17T17:41:13.123749+0000","last_peered":"2024-08-17T17:41:13.123749+0000","last_clean":"2024-08-17T17:41:13.123749+0000","last_became_active":"2024-08-17T17:36:12.746352+0000","last_became_peered":"2024-08-17T17:36:12.746352+0000","last_unstale":"2024-08-17T17:41:13.123749+0000","last_undegraded":"2024-08-17T17:41:13.123749+0000","last_fullsized":"2024-08-17T17:41:13.123749+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:07.807486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1703,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.090894+0000","last_change":"2024-08-17T17:36:15.771193+0000","last_active":"2024-08-17T17:41:15.090894+0000","last_peered":"2024-08-17T17:41:15.090894+0000","last_clean":"2024-08-17T17:41:15.090894+0000","last_became_active":"2024-08-17T17:36:15.770889+0000","last_became_peered":"2024-08-17T17:36:15.770889+0000","last_unstale":"2024-08-17T17:41:15.090894+0000","last_undegraded":"2024-08-17T17:41:15.090894+0000","last_fullsized":"2024-08-17T17:41:15.090894+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:22.978695+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1298,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.109237+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:41:14.109237+0000","last_peered":"2024-08-17T17:41:14.109237+0000","last_clean":"2024-08-17T17:41:14.109237+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:41:14.109237+0000","last_undegraded":"2024-08-17T17:41:14.109237+0000","last_fullsized":"2024-08-17T17:41:14.109237+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1435,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100729+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:41:13.100729+0000","last_peered":"2024-08-17T17:41:13.100729+0000","last_clean":"2024-08-17T17:41:13.100729+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:41:13.100729+0000","last_undegraded":"2024-08-17T17:41:13.100729+0000","last_fullsized":"2024-08-17T17:41:13.100729+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":935,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123958+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:41:13.123958+0000","last_peered":"2024-08-17T17:41:13.123958+0000","last_clean":"2024-08-17T17:41:13.123958+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:41:13.123958+0000","last_undegraded":"2024-08-17T17:41:13.123958+0000","last_fullsized":"2024-08-17T17:41:13.123958+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1292,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.808978+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:41:14.808978+0000","last_peered":"2024-08-17T17:41:14.808978+0000","last_clean":"2024-08-17T17:41:14.808978+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:41:14.808978+0000","last_undegraded":"2024-08-17T17:41:14.808978+0000","last_fullsized":"2024-08-17T17:41:14.808978+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":171,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100848+0000","last_change":"2024-08-17T17:35:55.217072+0000","last_active":"2024-08-17T17:41:13.100848+0000","last_peered":"2024-08-17T17:41:13.100848+0000","last_clean":"2024-08-17T17:41:13.100848+0000","last_became_active":"2024-08-17T17:35:55.216462+0000","last_became_peered":"2024-08-17T17:35:55.216462+0000","last_unstale":"2024-08-17T17:41:13.100848+0000","last_undegraded":"2024-08-17T17:41:13.100848+0000","last_fullsized":"2024-08-17T17:41:13.100848+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:34.937221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":931,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.810383+0000","last_change":"2024-08-17T17:40:56.322072+0000","last_active":"2024-08-17T17:41:14.810383+0000","last_peered":"2024-08-17T17:41:14.810383+0000","last_clean":"2024-08-17T17:41:14.810383+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:41:14.810383+0000","last_undegraded":"2024-08-17T17:41:14.810383+0000","last_fullsized":"2024-08-17T17:41:14.810383+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:56.321976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:40:56.321976+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:45.244567+0000","scrub_duration":0.0014456440000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1477,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091553+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:41:15.091553+0000","last_peered":"2024-08-17T17:41:15.091553+0000","last_clean":"2024-08-17T17:41:15.091553+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:41:15.091553+0000","last_undegraded":"2024-08-17T17:41:15.091553+0000","last_fullsized":"2024-08-17T17:41:15.091553+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":183,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124294+0000","last_change":"2024-08-17T17:35:33.147672+0000","last_active":"2024-08-17T17:41:13.124294+0000","last_peered":"2024-08-17T17:41:13.124294+0000","last_clean":"2024-08-17T17:41:13.124294+0000","last_became_active":"2024-08-17T17:35:33.147283+0000","last_became_peered":"2024-08-17T17:35:33.147283+0000","last_unstale":"2024-08-17T17:41:13.124294+0000","last_undegraded":"2024-08-17T17:41:13.124294+0000","last_fullsized":"2024-08-17T17:41:13.124294+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:41.677709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1729,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.797343+0000","last_change":"2024-08-17T17:40:55.554364+0000","last_active":"2024-08-17T17:41:14.797343+0000","last_peered":"2024-08-17T17:41:14.797343+0000","last_clean":"2024-08-17T17:41:14.797343+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:41:14.797343+0000","last_undegraded":"2024-08-17T17:41:14.797343+0000","last_fullsized":"2024-08-17T17:41:14.797343+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:55.554234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:40:55.554234+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:49.333767+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1401,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.732158+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:41:14.732158+0000","last_peered":"2024-08-17T17:41:14.732158+0000","last_clean":"2024-08-17T17:41:14.732158+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:41:14.732158+0000","last_undegraded":"2024-08-17T17:41:14.732158+0000","last_fullsized":"2024-08-17T17:41:14.732158+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":170,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123470+0000","last_change":"2024-08-17T17:35:33.140725+0000","last_active":"2024-08-17T17:41:13.123470+0000","last_peered":"2024-08-17T17:41:13.123470+0000","last_clean":"2024-08-17T17:41:13.123470+0000","last_became_active":"2024-08-17T17:35:33.140406+0000","last_became_peered":"2024-08-17T17:35:33.140406+0000","last_unstale":"2024-08-17T17:41:13.123470+0000","last_undegraded":"2024-08-17T17:41:13.123470+0000","last_fullsized":"2024-08-17T17:41:13.123470+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:54:51.875571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1690,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124257+0000","last_change":"2024-08-17T17:35:33.138514+0000","last_active":"2024-08-17T17:41:13.124257+0000","last_peered":"2024-08-17T17:41:13.124257+0000","last_clean":"2024-08-17T17:41:13.124257+0000","last_became_active":"2024-08-17T17:35:33.137035+0000","last_became_peered":"2024-08-17T17:35:33.137035+0000","last_unstale":"2024-08-17T17:41:13.124257+0000","last_undegraded":"2024-08-17T17:41:13.124257+0000","last_fullsized":"2024-08-17T17:41:13.124257+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:50.565662+0000","scrub_duration":0.001487138,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1333,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.808499+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:41:14.808499+0000","last_peered":"2024-08-17T17:41:14.808499+0000","last_clean":"2024-08-17T17:41:14.808499+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:41:14.808499+0000","last_undegraded":"2024-08-17T17:41:14.808499+0000","last_fullsized":"2024-08-17T17:41:14.808499+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":130,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.797995+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:41:14.797995+0000","last_peered":"2024-08-17T17:41:14.797995+0000","last_clean":"2024-08-17T17:41:14.797995+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:41:14.797995+0000","last_undegraded":"2024-08-17T17:41:14.797995+0000","last_fullsized":"2024-08-17T17:41:14.797995+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1763,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100341+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:41:13.100341+0000","last_peered":"2024-08-17T17:41:13.100341+0000","last_clean":"2024-08-17T17:41:13.100341+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:41:13.100341+0000","last_undegraded":"2024-08-17T17:41:13.100341+0000","last_fullsized":"2024-08-17T17:41:13.100341+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1349,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.123194+0000","last_change":"2024-08-17T17:35:55.205292+0000","last_active":"2024-08-17T17:41:13.123194+0000","last_peered":"2024-08-17T17:41:13.123194+0000","last_clean":"2024-08-17T17:41:13.123194+0000","last_became_active":"2024-08-17T17:35:55.204935+0000","last_became_peered":"2024-08-17T17:35:55.204935+0000","last_unstale":"2024-08-17T17:41:13.123194+0000","last_undegraded":"2024-08-17T17:41:13.123194+0000","last_fullsized":"2024-08-17T17:41:13.123194+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:52.277821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":130,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.797622+0000","last_change":"2024-08-17T17:36:12.749972+0000","last_active":"2024-08-17T17:41:14.797622+0000","last_peered":"2024-08-17T17:41:14.797622+0000","last_clean":"2024-08-17T17:41:14.797622+0000","last_became_active":"2024-08-17T17:36:12.749620+0000","last_became_peered":"2024-08-17T17:36:12.749620+0000","last_unstale":"2024-08-17T17:41:14.797622+0000","last_undegraded":"2024-08-17T17:41:14.797622+0000","last_fullsized":"2024-08-17T17:41:14.797622+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:04.994357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":1006,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.797848+0000","last_change":"2024-08-17T17:36:20.832274+0000","last_active":"2024-08-17T17:41:14.797848+0000","last_peered":"2024-08-17T17:41:14.797848+0000","last_clean":"2024-08-17T17:41:14.797848+0000","last_became_active":"2024-08-17T17:36:20.831886+0000","last_became_peered":"2024-08-17T17:36:20.831886+0000","last_unstale":"2024-08-17T17:41:14.797848+0000","last_undegraded":"2024-08-17T17:41:14.797848+0000","last_fullsized":"2024-08-17T17:41:14.797848+0000","mapping_epoch":1390,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1391,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:05.248711+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1215,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.106053+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:41:14.106053+0000","last_peered":"2024-08-17T17:41:14.106053+0000","last_clean":"2024-08-17T17:41:14.106053+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:41:14.106053+0000","last_undegraded":"2024-08-17T17:41:14.106053+0000","last_fullsized":"2024-08-17T17:41:14.106053+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1432,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100839+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:41:13.100839+0000","last_peered":"2024-08-17T17:41:13.100839+0000","last_clean":"2024-08-17T17:41:13.100839+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:41:13.100839+0000","last_undegraded":"2024-08-17T17:41:13.100839+0000","last_fullsized":"2024-08-17T17:41:13.100839+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1726,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.099925+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:41:13.099925+0000","last_peered":"2024-08-17T17:41:13.099925+0000","last_clean":"2024-08-17T17:41:13.099925+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:41:13.099925+0000","last_undegraded":"2024-08-17T17:41:13.099925+0000","last_fullsized":"2024-08-17T17:41:13.099925+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1374,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.124087+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:41:13.124087+0000","last_peered":"2024-08-17T17:41:13.124087+0000","last_clean":"2024-08-17T17:41:13.124087+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:41:13.124087+0000","last_undegraded":"2024-08-17T17:41:13.124087+0000","last_fullsized":"2024-08-17T17:41:13.124087+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1934,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.797525+0000","last_change":"2024-08-17T17:35:33.137411+0000","last_active":"2024-08-17T17:41:14.797525+0000","last_peered":"2024-08-17T17:41:14.797525+0000","last_clean":"2024-08-17T17:41:14.797525+0000","last_became_active":"2024-08-17T17:35:33.137073+0000","last_became_peered":"2024-08-17T17:35:33.137073+0000","last_unstale":"2024-08-17T17:41:14.797525+0000","last_undegraded":"2024-08-17T17:41:14.797525+0000","last_fullsized":"2024-08-17T17:41:14.797525+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:35.924539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1820,"reported_epoch":1440,"state":"active+clean","last_fresh":"2024-08-17T17:41:13.100175+0000","last_change":"2024-08-17T17:36:15.787445+0000","last_active":"2024-08-17T17:41:13.100175+0000","last_peered":"2024-08-17T17:41:13.100175+0000","last_clean":"2024-08-17T17:41:13.100175+0000","last_became_active":"2024-08-17T17:36:15.787099+0000","last_became_peered":"2024-08-17T17:36:15.787099+0000","last_unstale":"2024-08-17T17:41:13.100175+0000","last_undegraded":"2024-08-17T17:41:13.100175+0000","last_fullsized":"2024-08-17T17:41:13.100175+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:47.448168+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162888,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":168464,"kb_used_data":7940,"kb_used_omap":54,"kb_used_meta":160457,"kb_avail":93576688,"statfs":{"total":95995035648,"available":95822528512,"internally_reserved":0,"allocated":8130560,"data_stored":5373368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":164307989},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999362,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":230024,"kb_used_data":7996,"kb_used_omap":47,"kb_used_meta":221968,"kb_avail":93515128,"statfs":{"total":95995035648,"available":95759491072,"internally_reserved":0,"allocated":8187904,"data_stored":5439275,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":227295892},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627158,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1116660,"kb_used_data":7976,"kb_used_omap":42,"kb_used_meta":1108629,"kb_avail":92628492,"statfs":{"total":95995035648,"available":94851575808,"internally_reserved":0,"allocated":8167424,"data_stored":5433490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1135236156},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785636,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":860460,"kb_used_data":7660,"kb_used_omap":48,"kb_used_meta":852687,"kb_avail":92884692,"statfs":{"total":95995035648,"available":95113924608,"internally_reserved":0,"allocated":7843840,"data_stored":5106781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":873152174},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725153,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":901184,"kb_used_data":7924,"kb_used_omap":1,"kb_used_meta":893246,"kb_avail":92843968,"statfs":{"total":95995035648,"available":95072223232,"internally_reserved":0,"allocated":8114176,"data_stored":5369067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":914684358},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986790,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":918984,"kb_used_data":7676,"kb_used_omap":1,"kb_used_meta":911294,"kb_avail":92826168,"statfs":{"total":95995035648,"available":95053996032,"internally_reserved":0,"allocated":7860224,"data_stored":5110610,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":933165510},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706397,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":942384,"kb_used_data":7920,"kb_used_omap":35,"kb_used_meta":934364,"kb_avail":92802768,"statfs":{"total":95995035648,"available":95030034432,"internally_reserved":0,"allocated":8110080,"data_stored":5369024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":956788791},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832170,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1060804,"kb_used_data":7984,"kb_used_omap":65,"kb_used_meta":1052734,"kb_avail":92684348,"statfs":{"total":95995035648,"available":94908772352,"internally_reserved":0,"allocated":8175616,"data_stored":5435046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1077999803},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:41:31.796 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:41:31.796 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-08-17T17:41:31.796 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-08-17T17:41:31.796 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:41:32.700 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:32 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:32.700 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:32 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:32.700 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:32 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:41:32.701 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:32 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:41:32.701 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:32 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:32.701 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:32 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:32.701 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:32 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:32.701 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:32 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:41:32.701 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:32 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:41:32.701 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:32 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:32.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:32 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:32.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:32 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:32.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:32 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:41:32.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:32 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:41:32.846 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:32 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:41:33.356 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:41:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:41:33] "GET /metrics HTTP/1.1" 200 34111 "" "Prometheus/2.51.0" 2024-08-17T17:41:33.463 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:33.464 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:33.465 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:33.751 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:33 smithi063 ceph-mon[114756]: pgmap v2331: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:33.751 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:33 smithi063 ceph-mon[109843]: pgmap v2331: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:33.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:33 smithi162 ceph-mon[104668]: pgmap v2331: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:35.526 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-08-17T17:41:36.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:35 smithi063 ceph-mon[114756]: pgmap v2332: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:36.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:35 smithi063 ceph-mon[109843]: pgmap v2332: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:36.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:35 smithi162 ceph-mon[104668]: pgmap v2332: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:36.218 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:36.758 INFO:teuthology.orchestra.run.smithi063.stderr:nodeep-scrub is unset 2024-08-17T17:41:36.836 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:41:36.837 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:41:37.026 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:36 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3408551535' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:37.026 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:36 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3408551535' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:36 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3408551535' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-08-17T17:41:37.769 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2334,"stamp":"2024-08-17T17:41:36.758763+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":526015,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4678,"ondisk_log_size":4678,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6198948,"kb_used_data":63172,"kb_used_omap":298,"kb_used_meta":6135381,"kb_avail":743762268,"statfs":{"total":767960285184,"available":761612562432,"internally_reserved":0,"allocated":64688128,"data_stored":42699349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":6282630673},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.005151"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":92,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.755483+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:41:27.755483+0000","last_peered":"2024-08-17T17:41:27.755483+0000","last_clean":"2024-08-17T17:41:27.755483+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:41:27.755483+0000","last_undegraded":"2024-08-17T17:41:27.755483+0000","last_fullsized":"2024-08-17T17:41:27.755483+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1457,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091765+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:41:15.091765+0000","last_peered":"2024-08-17T17:41:15.091765+0000","last_clean":"2024-08-17T17:41:15.091765+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:41:15.091765+0000","last_undegraded":"2024-08-17T17:41:15.091765+0000","last_fullsized":"2024-08-17T17:41:15.091765+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1772,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091934+0000","last_change":"2024-08-17T17:36:15.771150+0000","last_active":"2024-08-17T17:41:15.091934+0000","last_peered":"2024-08-17T17:41:15.091934+0000","last_clean":"2024-08-17T17:41:15.091934+0000","last_became_active":"2024-08-17T17:36:15.770520+0000","last_became_peered":"2024-08-17T17:36:15.770520+0000","last_unstale":"2024-08-17T17:41:15.091934+0000","last_undegraded":"2024-08-17T17:41:15.091934+0000","last_fullsized":"2024-08-17T17:41:15.091934+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:59:17.552796+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":107,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.809698+0000","last_change":"2024-08-17T17:36:12.756709+0000","last_active":"2024-08-17T17:41:14.809698+0000","last_peered":"2024-08-17T17:41:14.809698+0000","last_clean":"2024-08-17T17:41:14.809698+0000","last_became_active":"2024-08-17T17:36:12.756385+0000","last_became_peered":"2024-08-17T17:36:12.756385+0000","last_unstale":"2024-08-17T17:41:14.809698+0000","last_undegraded":"2024-08-17T17:41:14.809698+0000","last_fullsized":"2024-08-17T17:41:14.809698+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:26:21.420282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":176,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.973987+0000","last_change":"2024-08-17T17:36:12.762800+0000","last_active":"2024-08-17T17:41:27.973987+0000","last_peered":"2024-08-17T17:41:27.973987+0000","last_clean":"2024-08-17T17:41:27.973987+0000","last_became_active":"2024-08-17T17:36:12.759977+0000","last_became_peered":"2024-08-17T17:36:12.759977+0000","last_unstale":"2024-08-17T17:41:27.973987+0000","last_undegraded":"2024-08-17T17:41:27.973987+0000","last_fullsized":"2024-08-17T17:41:27.973987+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:02.162681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1664,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.809358+0000","last_change":"2024-08-17T17:41:03.323402+0000","last_active":"2024-08-17T17:41:14.809358+0000","last_peered":"2024-08-17T17:41:14.809358+0000","last_clean":"2024-08-17T17:41:14.809358+0000","last_became_active":"2024-08-17T17:35:56.278638+0000","last_became_peered":"2024-08-17T17:35:56.278638+0000","last_unstale":"2024-08-17T17:41:14.809358+0000","last_undegraded":"2024-08-17T17:41:14.809358+0000","last_fullsized":"2024-08-17T17:41:14.809358+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:41:03.323294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:41:03.323294+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:25:28.872583+0000","scrub_duration":0.002060147,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1391,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:17.477798+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:41:17.477798+0000","last_peered":"2024-08-17T17:41:17.477798+0000","last_clean":"2024-08-17T17:41:17.477798+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:41:17.477798+0000","last_undegraded":"2024-08-17T17:41:17.477798+0000","last_fullsized":"2024-08-17T17:41:17.477798+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":67,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:17.460339+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:41:17.460339+0000","last_peered":"2024-08-17T17:41:17.460339+0000","last_clean":"2024-08-17T17:41:17.460339+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:41:17.460339+0000","last_undegraded":"2024-08-17T17:41:17.460339+0000","last_fullsized":"2024-08-17T17:41:17.460339+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1436,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.733315+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:41:14.733315+0000","last_peered":"2024-08-17T17:41:14.733315+0000","last_clean":"2024-08-17T17:41:14.733315+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:41:14.733315+0000","last_undegraded":"2024-08-17T17:41:14.733315+0000","last_fullsized":"2024-08-17T17:41:14.733315+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1444,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091918+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:41:15.091918+0000","last_peered":"2024-08-17T17:41:15.091918+0000","last_clean":"2024-08-17T17:41:15.091918+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:41:15.091918+0000","last_undegraded":"2024-08-17T17:41:15.091918+0000","last_fullsized":"2024-08-17T17:41:15.091918+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":734,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.809354+0000","last_change":"2024-08-17T17:36:15.773315+0000","last_active":"2024-08-17T17:41:14.809354+0000","last_peered":"2024-08-17T17:41:14.809354+0000","last_clean":"2024-08-17T17:41:14.809354+0000","last_became_active":"2024-08-17T17:36:15.772961+0000","last_became_peered":"2024-08-17T17:36:15.772961+0000","last_unstale":"2024-08-17T17:41:14.809354+0000","last_undegraded":"2024-08-17T17:41:14.809354+0000","last_fullsized":"2024-08-17T17:41:14.809354+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:48:10.190221+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":85,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.756247+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:41:27.756247+0000","last_peered":"2024-08-17T17:41:27.756247+0000","last_clean":"2024-08-17T17:41:27.756247+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:41:27.756247+0000","last_undegraded":"2024-08-17T17:41:27.756247+0000","last_fullsized":"2024-08-17T17:41:27.756247+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":176,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.974482+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:41:27.974482+0000","last_peered":"2024-08-17T17:41:27.974482+0000","last_clean":"2024-08-17T17:41:27.974482+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:41:27.974482+0000","last_undegraded":"2024-08-17T17:41:27.974482+0000","last_fullsized":"2024-08-17T17:41:27.974482+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1525,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.808594+0000","last_change":"2024-08-17T17:36:24.225739+0000","last_active":"2024-08-17T17:41:14.808594+0000","last_peered":"2024-08-17T17:41:14.808594+0000","last_clean":"2024-08-17T17:41:14.808594+0000","last_became_active":"2024-08-17T17:36:24.225250+0000","last_became_peered":"2024-08-17T17:36:24.225250+0000","last_unstale":"2024-08-17T17:41:14.808594+0000","last_undegraded":"2024-08-17T17:41:14.808594+0000","last_fullsized":"2024-08-17T17:41:14.808594+0000","mapping_epoch":1393,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1394,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:33:35.290656+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1464,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.090949+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:41:15.090949+0000","last_peered":"2024-08-17T17:41:15.090949+0000","last_clean":"2024-08-17T17:41:15.090949+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:41:15.090949+0000","last_undegraded":"2024-08-17T17:41:15.090949+0000","last_fullsized":"2024-08-17T17:41:15.090949+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1871,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.092089+0000","last_change":"2024-08-17T17:36:12.752970+0000","last_active":"2024-08-17T17:41:15.092089+0000","last_peered":"2024-08-17T17:41:15.092089+0000","last_clean":"2024-08-17T17:41:15.092089+0000","last_became_active":"2024-08-17T17:36:12.752311+0000","last_became_peered":"2024-08-17T17:36:12.752311+0000","last_unstale":"2024-08-17T17:41:15.092089+0000","last_undegraded":"2024-08-17T17:41:15.092089+0000","last_fullsized":"2024-08-17T17:41:15.092089+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:32:38.131419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1819,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.974245+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:41:27.974245+0000","last_peered":"2024-08-17T17:41:27.974245+0000","last_clean":"2024-08-17T17:41:27.974245+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:41:27.974245+0000","last_undegraded":"2024-08-17T17:41:27.974245+0000","last_fullsized":"2024-08-17T17:41:27.974245+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1548,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.755302+0000","last_change":"2024-08-17T17:36:18.750494+0000","last_active":"2024-08-17T17:41:27.755302+0000","last_peered":"2024-08-17T17:41:27.755302+0000","last_clean":"2024-08-17T17:41:27.755302+0000","last_became_active":"2024-08-17T17:36:18.750092+0000","last_became_peered":"2024-08-17T17:36:18.750092+0000","last_unstale":"2024-08-17T17:41:27.755302+0000","last_undegraded":"2024-08-17T17:41:27.755302+0000","last_fullsized":"2024-08-17T17:41:27.755302+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:30:36.599727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1178,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.756316+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:41:27.756316+0000","last_peered":"2024-08-17T17:41:27.756316+0000","last_clean":"2024-08-17T17:41:27.756316+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:41:27.756316+0000","last_undegraded":"2024-08-17T17:41:27.756316+0000","last_fullsized":"2024-08-17T17:41:27.756316+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":985,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.732214+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:41:14.732214+0000","last_peered":"2024-08-17T17:41:14.732214+0000","last_clean":"2024-08-17T17:41:14.732214+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:41:14.732214+0000","last_undegraded":"2024-08-17T17:41:14.732214+0000","last_fullsized":"2024-08-17T17:41:14.732214+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":85,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.756073+0000","last_change":"2024-08-17T17:37:11.075505+0000","last_active":"2024-08-17T17:41:27.756073+0000","last_peered":"2024-08-17T17:41:27.756073+0000","last_clean":"2024-08-17T17:41:27.756073+0000","last_became_active":"2024-08-17T17:37:11.075131+0000","last_became_peered":"2024-08-17T17:37:11.075131+0000","last_unstale":"2024-08-17T17:41:27.756073+0000","last_undegraded":"2024-08-17T17:41:27.756073+0000","last_fullsized":"2024-08-17T17:41:27.756073+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:09:39.672054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1298,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.810476+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:41:14.810476+0000","last_peered":"2024-08-17T17:41:14.810476+0000","last_clean":"2024-08-17T17:41:14.810476+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:41:14.810476+0000","last_undegraded":"2024-08-17T17:41:14.810476+0000","last_fullsized":"2024-08-17T17:41:14.810476+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1675,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091298+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:41:15.091298+0000","last_peered":"2024-08-17T17:41:15.091298+0000","last_clean":"2024-08-17T17:41:15.091298+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:41:15.091298+0000","last_undegraded":"2024-08-17T17:41:15.091298+0000","last_fullsized":"2024-08-17T17:41:15.091298+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":132,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.754584+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:41:27.754584+0000","last_peered":"2024-08-17T17:41:27.754584+0000","last_clean":"2024-08-17T17:41:27.754584+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:41:27.754584+0000","last_undegraded":"2024-08-17T17:41:27.754584+0000","last_fullsized":"2024-08-17T17:41:27.754584+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1272,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.733908+0000","last_change":"2024-08-17T17:36:12.755873+0000","last_active":"2024-08-17T17:41:14.733908+0000","last_peered":"2024-08-17T17:41:14.733908+0000","last_clean":"2024-08-17T17:41:14.733908+0000","last_became_active":"2024-08-17T17:36:12.755366+0000","last_became_peered":"2024-08-17T17:36:12.755366+0000","last_unstale":"2024-08-17T17:41:14.733908+0000","last_undegraded":"2024-08-17T17:41:14.733908+0000","last_fullsized":"2024-08-17T17:41:14.733908+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:03:36.523214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1626,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.107894+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:41:14.107894+0000","last_peered":"2024-08-17T17:41:14.107894+0000","last_clean":"2024-08-17T17:41:14.107894+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:41:14.107894+0000","last_undegraded":"2024-08-17T17:41:14.107894+0000","last_fullsized":"2024-08-17T17:41:14.107894+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":810,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.809969+0000","last_change":"2024-08-17T17:35:55.200519+0000","last_active":"2024-08-17T17:41:14.809969+0000","last_peered":"2024-08-17T17:41:14.809969+0000","last_clean":"2024-08-17T17:41:14.809969+0000","last_became_active":"2024-08-17T17:35:55.200230+0000","last_became_peered":"2024-08-17T17:35:55.200230+0000","last_unstale":"2024-08-17T17:41:14.809969+0000","last_undegraded":"2024-08-17T17:41:14.809969+0000","last_fullsized":"2024-08-17T17:41:14.809969+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:02:33.454851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1449,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.733083+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:41:14.733083+0000","last_peered":"2024-08-17T17:41:14.733083+0000","last_clean":"2024-08-17T17:41:14.733083+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:41:14.733083+0000","last_undegraded":"2024-08-17T17:41:14.733083+0000","last_fullsized":"2024-08-17T17:41:14.733083+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1749,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.974339+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:41:27.974339+0000","last_peered":"2024-08-17T17:41:27.974339+0000","last_clean":"2024-08-17T17:41:27.974339+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:41:27.974339+0000","last_undegraded":"2024-08-17T17:41:27.974339+0000","last_fullsized":"2024-08-17T17:41:27.974339+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:28:22.077229+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":296,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.756146+0000","last_change":"2024-08-17T17:35:33.145476+0000","last_active":"2024-08-17T17:41:27.756146+0000","last_peered":"2024-08-17T17:41:27.756146+0000","last_clean":"2024-08-17T17:41:27.756146+0000","last_became_active":"2024-08-17T17:35:33.145125+0000","last_became_peered":"2024-08-17T17:35:33.145125+0000","last_unstale":"2024-08-17T17:41:27.756146+0000","last_undegraded":"2024-08-17T17:41:27.756146+0000","last_fullsized":"2024-08-17T17:41:27.756146+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:11:45.298064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":954,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.808835+0000","last_change":"2024-08-17T17:35:56.248612+0000","last_active":"2024-08-17T17:41:14.808835+0000","last_peered":"2024-08-17T17:41:14.808835+0000","last_clean":"2024-08-17T17:41:14.808835+0000","last_became_active":"2024-08-17T17:35:56.244364+0000","last_became_peered":"2024-08-17T17:35:56.244364+0000","last_unstale":"2024-08-17T17:41:14.808835+0000","last_undegraded":"2024-08-17T17:41:14.808835+0000","last_fullsized":"2024-08-17T17:41:14.808835+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:52:41.064165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":894,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.975151+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:41:27.975151+0000","last_peered":"2024-08-17T17:41:27.975151+0000","last_clean":"2024-08-17T17:41:27.975151+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:41:27.975151+0000","last_undegraded":"2024-08-17T17:41:27.975151+0000","last_fullsized":"2024-08-17T17:41:27.975151+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1429,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.092282+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:41:15.092282+0000","last_peered":"2024-08-17T17:41:15.092282+0000","last_clean":"2024-08-17T17:41:15.092282+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:41:15.092282+0000","last_undegraded":"2024-08-17T17:41:15.092282+0000","last_fullsized":"2024-08-17T17:41:15.092282+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1448,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.092512+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:41:15.092512+0000","last_peered":"2024-08-17T17:41:15.092512+0000","last_clean":"2024-08-17T17:41:15.092512+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:41:15.092512+0000","last_undegraded":"2024-08-17T17:41:15.092512+0000","last_fullsized":"2024-08-17T17:41:15.092512+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":971,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.974600+0000","last_change":"2024-08-17T17:36:15.774881+0000","last_active":"2024-08-17T17:41:27.974600+0000","last_peered":"2024-08-17T17:41:27.974600+0000","last_clean":"2024-08-17T17:41:27.974600+0000","last_became_active":"2024-08-17T17:36:15.774435+0000","last_became_peered":"2024-08-17T17:36:15.774435+0000","last_unstale":"2024-08-17T17:41:27.974600+0000","last_undegraded":"2024-08-17T17:41:27.974600+0000","last_fullsized":"2024-08-17T17:41:27.974600+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.935521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.935521+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:52.015854+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1397,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.974188+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:41:27.974188+0000","last_peered":"2024-08-17T17:41:27.974188+0000","last_clean":"2024-08-17T17:41:27.974188+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:41:27.974188+0000","last_undegraded":"2024-08-17T17:41:27.974188+0000","last_fullsized":"2024-08-17T17:41:27.974188+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1292,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.810205+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:41:14.810205+0000","last_peered":"2024-08-17T17:41:14.810205+0000","last_clean":"2024-08-17T17:41:14.810205+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:41:14.810205+0000","last_undegraded":"2024-08-17T17:41:14.810205+0000","last_fullsized":"2024-08-17T17:41:14.810205+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":364,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.756326+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:41:27.756326+0000","last_peered":"2024-08-17T17:41:27.756326+0000","last_clean":"2024-08-17T17:41:27.756326+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:41:27.756326+0000","last_undegraded":"2024-08-17T17:41:27.756326+0000","last_fullsized":"2024-08-17T17:41:27.756326+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1632,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.754981+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:41:27.754981+0000","last_peered":"2024-08-17T17:41:27.754981+0000","last_clean":"2024-08-17T17:41:27.754981+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:41:27.754981+0000","last_undegraded":"2024-08-17T17:41:27.754981+0000","last_fullsized":"2024-08-17T17:41:27.754981+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:41:25.751871+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":139,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.754677+0000","last_change":"2024-08-17T17:37:11.079712+0000","last_active":"2024-08-17T17:41:27.754677+0000","last_peered":"2024-08-17T17:41:27.754677+0000","last_clean":"2024-08-17T17:41:27.754677+0000","last_became_active":"2024-08-17T17:37:11.079253+0000","last_became_peered":"2024-08-17T17:37:11.079253+0000","last_unstale":"2024-08-17T17:41:27.754677+0000","last_undegraded":"2024-08-17T17:41:27.754677+0000","last_fullsized":"2024-08-17T17:41:27.754677+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:09:53.405657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":937,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.809156+0000","last_change":"2024-08-17T17:35:56.223396+0000","last_active":"2024-08-17T17:41:14.809156+0000","last_peered":"2024-08-17T17:41:14.809156+0000","last_clean":"2024-08-17T17:41:14.809156+0000","last_became_active":"2024-08-17T17:35:56.212413+0000","last_became_peered":"2024-08-17T17:35:56.212413+0000","last_unstale":"2024-08-17T17:41:14.809156+0000","last_undegraded":"2024-08-17T17:41:14.809156+0000","last_fullsized":"2024-08-17T17:41:14.809156+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:25:44.185174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1674,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.755710+0000","last_change":"2024-08-17T17:35:33.136068+0000","last_active":"2024-08-17T17:41:27.755710+0000","last_peered":"2024-08-17T17:41:27.755710+0000","last_clean":"2024-08-17T17:41:27.755710+0000","last_became_active":"2024-08-17T17:35:33.135591+0000","last_became_peered":"2024-08-17T17:35:33.135591+0000","last_unstale":"2024-08-17T17:41:27.755710+0000","last_undegraded":"2024-08-17T17:41:27.755710+0000","last_fullsized":"2024-08-17T17:41:27.755710+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:45:52.252561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1803,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.975136+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:41:27.975136+0000","last_peered":"2024-08-17T17:41:27.975136+0000","last_clean":"2024-08-17T17:41:27.975136+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:41:27.975136+0000","last_undegraded":"2024-08-17T17:41:27.975136+0000","last_fullsized":"2024-08-17T17:41:27.975136+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1015,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.732356+0000","last_change":"2024-08-17T17:35:33.138467+0000","last_active":"2024-08-17T17:41:14.732356+0000","last_peered":"2024-08-17T17:41:14.732356+0000","last_clean":"2024-08-17T17:41:14.732356+0000","last_became_active":"2024-08-17T17:35:33.138174+0000","last_became_peered":"2024-08-17T17:35:33.138174+0000","last_unstale":"2024-08-17T17:41:14.732356+0000","last_undegraded":"2024-08-17T17:41:14.732356+0000","last_fullsized":"2024-08-17T17:41:14.732356+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:32:08.860439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1399,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.108006+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:41:14.108006+0000","last_peered":"2024-08-17T17:41:14.108006+0000","last_clean":"2024-08-17T17:41:14.108006+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:41:14.108006+0000","last_undegraded":"2024-08-17T17:41:14.108006+0000","last_fullsized":"2024-08-17T17:41:14.108006+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":170,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.974693+0000","last_change":"2024-08-17T17:35:14.854881+0000","last_active":"2024-08-17T17:41:27.974693+0000","last_peered":"2024-08-17T17:41:27.974693+0000","last_clean":"2024-08-17T17:41:27.974693+0000","last_became_active":"2024-08-17T17:35:14.854121+0000","last_became_peered":"2024-08-17T17:35:14.854121+0000","last_unstale":"2024-08-17T17:41:27.974693+0000","last_undegraded":"2024-08-17T17:41:27.974693+0000","last_fullsized":"2024-08-17T17:41:27.974693+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:49:53.863624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":968,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.809805+0000","last_change":"2024-08-17T17:41:06.387062+0000","last_active":"2024-08-17T17:41:14.809805+0000","last_peered":"2024-08-17T17:41:14.809805+0000","last_clean":"2024-08-17T17:41:14.809805+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:41:14.809805+0000","last_undegraded":"2024-08-17T17:41:14.809805+0000","last_fullsized":"2024-08-17T17:41:14.809805+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:41:06.386955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:41:06.386955+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:44:32.008526+0000","scrub_duration":0.0017975980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1058,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.733048+0000","last_change":"2024-08-17T17:33:25.933028+0000","last_active":"2024-08-17T17:41:14.733048+0000","last_peered":"2024-08-17T17:41:14.733048+0000","last_clean":"2024-08-17T17:41:14.733048+0000","last_became_active":"2024-08-17T17:33:25.932727+0000","last_became_peered":"2024-08-17T17:33:25.932727+0000","last_unstale":"2024-08-17T17:41:14.733048+0000","last_undegraded":"2024-08-17T17:41:14.733048+0000","last_fullsized":"2024-08-17T17:41:14.733048+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:46:44.110654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":263,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.975224+0000","last_change":"2024-08-17T17:35:33.145351+0000","last_active":"2024-08-17T17:41:27.975224+0000","last_peered":"2024-08-17T17:41:27.975224+0000","last_clean":"2024-08-17T17:41:27.975224+0000","last_became_active":"2024-08-17T17:35:33.145040+0000","last_became_peered":"2024-08-17T17:35:33.145040+0000","last_unstale":"2024-08-17T17:41:27.975224+0000","last_undegraded":"2024-08-17T17:41:27.975224+0000","last_fullsized":"2024-08-17T17:41:27.975224+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:01:02.124000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1467,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.733372+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:41:14.733372+0000","last_peered":"2024-08-17T17:41:14.733372+0000","last_clean":"2024-08-17T17:41:14.733372+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:41:14.733372+0000","last_undegraded":"2024-08-17T17:41:14.733372+0000","last_fullsized":"2024-08-17T17:41:14.733372+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":951,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.810280+0000","last_change":"2024-08-17T17:41:07.361112+0000","last_active":"2024-08-17T17:41:14.810280+0000","last_peered":"2024-08-17T17:41:14.810280+0000","last_clean":"2024-08-17T17:41:14.810280+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:41:14.810280+0000","last_undegraded":"2024-08-17T17:41:14.810280+0000","last_fullsized":"2024-08-17T17:41:14.810280+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:41:07.360968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:41:07.360968+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:28.377019+0000","scrub_duration":0.001649951,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1525,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.974669+0000","last_change":"2024-08-17T17:36:12.759743+0000","last_active":"2024-08-17T17:41:27.974669+0000","last_peered":"2024-08-17T17:41:27.974669+0000","last_clean":"2024-08-17T17:41:27.974669+0000","last_became_active":"2024-08-17T17:36:12.759334+0000","last_became_peered":"2024-08-17T17:36:12.759334+0000","last_unstale":"2024-08-17T17:41:27.974669+0000","last_undegraded":"2024-08-17T17:41:27.974669+0000","last_fullsized":"2024-08-17T17:41:27.974669+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:13:04.524384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1431,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.975472+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:41:27.975472+0000","last_peered":"2024-08-17T17:41:27.975472+0000","last_clean":"2024-08-17T17:41:27.975472+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:41:27.975472+0000","last_undegraded":"2024-08-17T17:41:27.975472+0000","last_fullsized":"2024-08-17T17:41:27.975472+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1473,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.733306+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:41:14.733306+0000","last_peered":"2024-08-17T17:41:14.733306+0000","last_clean":"2024-08-17T17:41:14.733306+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:41:14.733306+0000","last_undegraded":"2024-08-17T17:41:14.733306+0000","last_fullsized":"2024-08-17T17:41:14.733306+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1789,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.479203+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:41:27.479203+0000","last_peered":"2024-08-17T17:41:27.479203+0000","last_clean":"2024-08-17T17:41:27.479203+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:41:27.479203+0000","last_undegraded":"2024-08-17T17:41:27.479203+0000","last_fullsized":"2024-08-17T17:41:27.479203+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1106,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.733579+0000","last_change":"2024-08-17T17:33:25.941962+0000","last_active":"2024-08-17T17:41:14.733579+0000","last_peered":"2024-08-17T17:41:14.733579+0000","last_clean":"2024-08-17T17:41:14.733579+0000","last_became_active":"2024-08-17T17:33:25.941630+0000","last_became_peered":"2024-08-17T17:33:25.941630+0000","last_unstale":"2024-08-17T17:41:14.733579+0000","last_undegraded":"2024-08-17T17:41:14.733579+0000","last_fullsized":"2024-08-17T17:41:14.733579+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:06:06.102836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1436,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.974234+0000","last_change":"2024-08-17T17:35:56.229622+0000","last_active":"2024-08-17T17:41:27.974234+0000","last_peered":"2024-08-17T17:41:27.974234+0000","last_clean":"2024-08-17T17:41:27.974234+0000","last_became_active":"2024-08-17T17:35:56.229324+0000","last_became_peered":"2024-08-17T17:35:56.229324+0000","last_unstale":"2024-08-17T17:41:27.974234+0000","last_undegraded":"2024-08-17T17:41:27.974234+0000","last_fullsized":"2024-08-17T17:41:27.974234+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:03:13.980316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2040,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.974113+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:41:27.974113+0000","last_peered":"2024-08-17T17:41:27.974113+0000","last_clean":"2024-08-17T17:41:27.974113+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:41:27.974113+0000","last_undegraded":"2024-08-17T17:41:27.974113+0000","last_fullsized":"2024-08-17T17:41:27.974113+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1603,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091427+0000","last_change":"2024-08-17T17:35:33.137074+0000","last_active":"2024-08-17T17:41:15.091427+0000","last_peered":"2024-08-17T17:41:15.091427+0000","last_clean":"2024-08-17T17:41:15.091427+0000","last_became_active":"2024-08-17T17:35:33.136759+0000","last_became_peered":"2024-08-17T17:35:33.136759+0000","last_unstale":"2024-08-17T17:41:15.091427+0000","last_undegraded":"2024-08-17T17:41:15.091427+0000","last_fullsized":"2024-08-17T17:41:15.091427+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:37.263639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:37.263639+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:14.599780+0000","scrub_duration":0.0016181800000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1229,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.756431+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:41:27.756431+0000","last_peered":"2024-08-17T17:41:27.756431+0000","last_clean":"2024-08-17T17:41:27.756431+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:41:27.756431+0000","last_undegraded":"2024-08-17T17:41:27.756431+0000","last_fullsized":"2024-08-17T17:41:27.756431+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":67,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091101+0000","last_change":"2024-08-17T17:36:17.774773+0000","last_active":"2024-08-17T17:41:15.091101+0000","last_peered":"2024-08-17T17:41:15.091101+0000","last_clean":"2024-08-17T17:41:15.091101+0000","last_became_active":"2024-08-17T17:36:17.774468+0000","last_became_peered":"2024-08-17T17:36:17.774468+0000","last_unstale":"2024-08-17T17:41:15.091101+0000","last_undegraded":"2024-08-17T17:41:15.091101+0000","last_fullsized":"2024-08-17T17:41:15.091101+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:05:54.231747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1567,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.755997+0000","last_change":"2024-08-17T17:35:33.137633+0000","last_active":"2024-08-17T17:41:27.755997+0000","last_peered":"2024-08-17T17:41:27.755997+0000","last_clean":"2024-08-17T17:41:27.755997+0000","last_became_active":"2024-08-17T17:35:33.137323+0000","last_became_peered":"2024-08-17T17:35:33.137323+0000","last_unstale":"2024-08-17T17:41:27.755997+0000","last_undegraded":"2024-08-17T17:41:27.755997+0000","last_fullsized":"2024-08-17T17:41:27.755997+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:56:03.359155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":930,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.809113+0000","last_change":"2024-08-17T17:40:59.325330+0000","last_active":"2024-08-17T17:41:14.809113+0000","last_peered":"2024-08-17T17:41:14.809113+0000","last_clean":"2024-08-17T17:41:14.809113+0000","last_became_active":"2024-08-17T17:35:56.264265+0000","last_became_peered":"2024-08-17T17:35:56.264265+0000","last_unstale":"2024-08-17T17:41:14.809113+0000","last_undegraded":"2024-08-17T17:41:14.809113+0000","last_fullsized":"2024-08-17T17:41:14.809113+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:59.325233+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:40:59.325233+0000","last_clean_scrub_stamp":"2024-08-17T17:40:59.325233+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:22:35.293983+0000","scrub_duration":0.0014772979999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1271,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.811238+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:41:14.811238+0000","last_peered":"2024-08-17T17:41:14.811238+0000","last_clean":"2024-08-17T17:41:14.811238+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:41:14.811238+0000","last_undegraded":"2024-08-17T17:41:14.811238+0000","last_fullsized":"2024-08-17T17:41:14.811238+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1811,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.733189+0000","last_change":"2024-08-17T17:36:12.758005+0000","last_active":"2024-08-17T17:41:14.733189+0000","last_peered":"2024-08-17T17:41:14.733189+0000","last_clean":"2024-08-17T17:41:14.733189+0000","last_became_active":"2024-08-17T17:36:12.757695+0000","last_became_peered":"2024-08-17T17:36:12.757695+0000","last_unstale":"2024-08-17T17:41:14.733189+0000","last_undegraded":"2024-08-17T17:41:14.733189+0000","last_fullsized":"2024-08-17T17:41:14.733189+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:18:51.792220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1449,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.732449+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:41:14.732449+0000","last_peered":"2024-08-17T17:41:14.732449+0000","last_clean":"2024-08-17T17:41:14.732449+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:41:14.732449+0000","last_undegraded":"2024-08-17T17:41:14.732449+0000","last_fullsized":"2024-08-17T17:41:14.732449+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1798,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.756099+0000","last_change":"2024-08-17T17:40:58.504477+0000","last_active":"2024-08-17T17:41:27.756099+0000","last_peered":"2024-08-17T17:41:27.756099+0000","last_clean":"2024-08-17T17:41:27.756099+0000","last_became_active":"2024-08-17T17:35:55.204861+0000","last_became_peered":"2024-08-17T17:35:55.204861+0000","last_unstale":"2024-08-17T17:41:27.756099+0000","last_undegraded":"2024-08-17T17:41:27.756099+0000","last_fullsized":"2024-08-17T17:41:27.756099+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:58.504387+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:40:58.504387+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:24:54.058501+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1044,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.092386+0000","last_change":"2024-08-17T17:33:25.947191+0000","last_active":"2024-08-17T17:41:15.092386+0000","last_peered":"2024-08-17T17:41:15.092386+0000","last_clean":"2024-08-17T17:41:15.092386+0000","last_became_active":"2024-08-17T17:33:25.946894+0000","last_became_peered":"2024-08-17T17:33:25.946894+0000","last_unstale":"2024-08-17T17:41:15.092386+0000","last_undegraded":"2024-08-17T17:41:15.092386+0000","last_fullsized":"2024-08-17T17:41:15.092386+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:08:59.978303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":132,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.754758+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:41:27.754758+0000","last_peered":"2024-08-17T17:41:27.754758+0000","last_clean":"2024-08-17T17:41:27.754758+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:41:27.754758+0000","last_undegraded":"2024-08-17T17:41:27.754758+0000","last_fullsized":"2024-08-17T17:41:27.754758+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1780,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.732772+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:41:14.732772+0000","last_peered":"2024-08-17T17:41:14.732772+0000","last_clean":"2024-08-17T17:41:14.732772+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:41:14.732772+0000","last_undegraded":"2024-08-17T17:41:14.732772+0000","last_fullsized":"2024-08-17T17:41:14.732772+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1459,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.093115+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:41:15.093115+0000","last_peered":"2024-08-17T17:41:15.093115+0000","last_clean":"2024-08-17T17:41:15.093115+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:41:15.093115+0000","last_undegraded":"2024-08-17T17:41:15.093115+0000","last_fullsized":"2024-08-17T17:41:15.093115+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1432,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.461291+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:41:22.461291+0000","last_peered":"2024-08-17T17:41:22.461291+0000","last_clean":"2024-08-17T17:41:22.461291+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:41:22.461291+0000","last_undegraded":"2024-08-17T17:41:22.461291+0000","last_fullsized":"2024-08-17T17:41:22.461291+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":1034,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.756523+0000","last_change":"2024-08-17T17:34:26.817339+0000","last_active":"2024-08-17T17:41:27.756523+0000","last_peered":"2024-08-17T17:41:27.756523+0000","last_clean":"2024-08-17T17:41:27.756523+0000","last_became_active":"2024-08-17T17:33:28.930558+0000","last_became_peered":"2024-08-17T17:33:28.930558+0000","last_unstale":"2024-08-17T17:41:27.756523+0000","last_undegraded":"2024-08-17T17:41:27.756523+0000","last_fullsized":"2024-08-17T17:41:27.756523+0000","mapping_epoch":1325,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1326,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:26.817223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:26.817223+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:32:30.089956+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":67,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.092224+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:41:15.092224+0000","last_peered":"2024-08-17T17:41:15.092224+0000","last_clean":"2024-08-17T17:41:15.092224+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:41:15.092224+0000","last_undegraded":"2024-08-17T17:41:15.092224+0000","last_fullsized":"2024-08-17T17:41:15.092224+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1293,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.808799+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:41:14.808799+0000","last_peered":"2024-08-17T17:41:14.808799+0000","last_clean":"2024-08-17T17:41:14.808799+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:41:14.808799+0000","last_undegraded":"2024-08-17T17:41:14.808799+0000","last_fullsized":"2024-08-17T17:41:14.808799+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1808,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.731951+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:41:14.731951+0000","last_peered":"2024-08-17T17:41:14.731951+0000","last_clean":"2024-08-17T17:41:14.731951+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:41:14.731951+0000","last_undegraded":"2024-08-17T17:41:14.731951+0000","last_fullsized":"2024-08-17T17:41:14.731951+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1410,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091849+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:41:15.091849+0000","last_peered":"2024-08-17T17:41:15.091849+0000","last_clean":"2024-08-17T17:41:15.091849+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:41:15.091849+0000","last_undegraded":"2024-08-17T17:41:15.091849+0000","last_fullsized":"2024-08-17T17:41:15.091849+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":223,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091684+0000","last_change":"2024-08-17T17:36:12.754372+0000","last_active":"2024-08-17T17:41:15.091684+0000","last_peered":"2024-08-17T17:41:15.091684+0000","last_clean":"2024-08-17T17:41:15.091684+0000","last_became_active":"2024-08-17T17:36:12.746352+0000","last_became_peered":"2024-08-17T17:36:12.746352+0000","last_unstale":"2024-08-17T17:41:15.091684+0000","last_undegraded":"2024-08-17T17:41:15.091684+0000","last_fullsized":"2024-08-17T17:41:15.091684+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:07:07.807486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1705,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.973694+0000","last_change":"2024-08-17T17:36:15.771193+0000","last_active":"2024-08-17T17:41:27.973694+0000","last_peered":"2024-08-17T17:41:27.973694+0000","last_clean":"2024-08-17T17:41:27.973694+0000","last_became_active":"2024-08-17T17:36:15.770889+0000","last_became_peered":"2024-08-17T17:36:15.770889+0000","last_unstale":"2024-08-17T17:41:27.973694+0000","last_undegraded":"2024-08-17T17:41:27.973694+0000","last_fullsized":"2024-08-17T17:41:27.973694+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:24:22.978695+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1301,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.756636+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:41:27.756636+0000","last_peered":"2024-08-17T17:41:27.756636+0000","last_clean":"2024-08-17T17:41:27.756636+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:41:27.756636+0000","last_undegraded":"2024-08-17T17:41:27.756636+0000","last_fullsized":"2024-08-17T17:41:27.756636+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1436,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.733353+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:41:14.733353+0000","last_peered":"2024-08-17T17:41:14.733353+0000","last_clean":"2024-08-17T17:41:14.733353+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:41:14.733353+0000","last_undegraded":"2024-08-17T17:41:14.733353+0000","last_fullsized":"2024-08-17T17:41:14.733353+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":936,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.092388+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:41:15.092388+0000","last_peered":"2024-08-17T17:41:15.092388+0000","last_clean":"2024-08-17T17:41:15.092388+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:41:15.092388+0000","last_undegraded":"2024-08-17T17:41:15.092388+0000","last_fullsized":"2024-08-17T17:41:15.092388+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1292,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.808978+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:41:14.808978+0000","last_peered":"2024-08-17T17:41:14.808978+0000","last_clean":"2024-08-17T17:41:14.808978+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:41:14.808978+0000","last_undegraded":"2024-08-17T17:41:14.808978+0000","last_fullsized":"2024-08-17T17:41:14.808978+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":172,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.733000+0000","last_change":"2024-08-17T17:35:55.217072+0000","last_active":"2024-08-17T17:41:14.733000+0000","last_peered":"2024-08-17T17:41:14.733000+0000","last_clean":"2024-08-17T17:41:14.733000+0000","last_became_active":"2024-08-17T17:35:55.216462+0000","last_became_peered":"2024-08-17T17:35:55.216462+0000","last_unstale":"2024-08-17T17:41:14.733000+0000","last_undegraded":"2024-08-17T17:41:14.733000+0000","last_fullsized":"2024-08-17T17:41:14.733000+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:06:34.937221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":931,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.810383+0000","last_change":"2024-08-17T17:40:56.322072+0000","last_active":"2024-08-17T17:41:14.810383+0000","last_peered":"2024-08-17T17:41:14.810383+0000","last_clean":"2024-08-17T17:41:14.810383+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:41:14.810383+0000","last_undegraded":"2024-08-17T17:41:14.810383+0000","last_fullsized":"2024-08-17T17:41:14.810383+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:56.321976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:40:56.321976+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:45.244567+0000","scrub_duration":0.0014456440000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1478,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.462085+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:41:27.462085+0000","last_peered":"2024-08-17T17:41:27.462085+0000","last_clean":"2024-08-17T17:41:27.462085+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:41:27.462085+0000","last_undegraded":"2024-08-17T17:41:27.462085+0000","last_fullsized":"2024-08-17T17:41:27.462085+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":184,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.093266+0000","last_change":"2024-08-17T17:35:33.147672+0000","last_active":"2024-08-17T17:41:15.093266+0000","last_peered":"2024-08-17T17:41:15.093266+0000","last_clean":"2024-08-17T17:41:15.093266+0000","last_became_active":"2024-08-17T17:35:33.147283+0000","last_became_peered":"2024-08-17T17:35:33.147283+0000","last_unstale":"2024-08-17T17:41:15.093266+0000","last_undegraded":"2024-08-17T17:41:15.093266+0000","last_fullsized":"2024-08-17T17:41:15.093266+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:28:41.677709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1732,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.754913+0000","last_change":"2024-08-17T17:40:55.554364+0000","last_active":"2024-08-17T17:41:27.754913+0000","last_peered":"2024-08-17T17:41:27.754913+0000","last_clean":"2024-08-17T17:41:27.754913+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:41:27.754913+0000","last_undegraded":"2024-08-17T17:41:27.754913+0000","last_fullsized":"2024-08-17T17:41:27.754913+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:55.554234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:40:55.554234+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:49.333767+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1402,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:22.478526+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:41:22.478526+0000","last_peered":"2024-08-17T17:41:22.478526+0000","last_clean":"2024-08-17T17:41:22.478526+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:41:22.478526+0000","last_undegraded":"2024-08-17T17:41:22.478526+0000","last_fullsized":"2024-08-17T17:41:22.478526+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":171,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.092507+0000","last_change":"2024-08-17T17:35:33.140725+0000","last_active":"2024-08-17T17:41:15.092507+0000","last_peered":"2024-08-17T17:41:15.092507+0000","last_clean":"2024-08-17T17:41:15.092507+0000","last_became_active":"2024-08-17T17:35:33.140406+0000","last_became_peered":"2024-08-17T17:35:33.140406+0000","last_unstale":"2024-08-17T17:41:15.092507+0000","last_undegraded":"2024-08-17T17:41:15.092507+0000","last_fullsized":"2024-08-17T17:41:15.092507+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:54:51.875571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1691,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091995+0000","last_change":"2024-08-17T17:35:33.138514+0000","last_active":"2024-08-17T17:41:15.091995+0000","last_peered":"2024-08-17T17:41:15.091995+0000","last_clean":"2024-08-17T17:41:15.091995+0000","last_became_active":"2024-08-17T17:35:33.137035+0000","last_became_peered":"2024-08-17T17:35:33.137035+0000","last_unstale":"2024-08-17T17:41:15.091995+0000","last_undegraded":"2024-08-17T17:41:15.091995+0000","last_fullsized":"2024-08-17T17:41:15.091995+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:59:50.565662+0000","scrub_duration":0.001487138,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1333,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.808499+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:41:14.808499+0000","last_peered":"2024-08-17T17:41:14.808499+0000","last_clean":"2024-08-17T17:41:14.808499+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:41:14.808499+0000","last_undegraded":"2024-08-17T17:41:14.808499+0000","last_fullsized":"2024-08-17T17:41:14.808499+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":132,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.755818+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:41:27.755818+0000","last_peered":"2024-08-17T17:41:27.755818+0000","last_clean":"2024-08-17T17:41:27.755818+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:41:27.755818+0000","last_undegraded":"2024-08-17T17:41:27.755818+0000","last_fullsized":"2024-08-17T17:41:27.755818+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1764,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.732954+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:41:14.732954+0000","last_peered":"2024-08-17T17:41:14.732954+0000","last_clean":"2024-08-17T17:41:14.732954+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:41:14.732954+0000","last_undegraded":"2024-08-17T17:41:14.732954+0000","last_fullsized":"2024-08-17T17:41:14.732954+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1350,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:15.091364+0000","last_change":"2024-08-17T17:35:55.205292+0000","last_active":"2024-08-17T17:41:15.091364+0000","last_peered":"2024-08-17T17:41:15.091364+0000","last_clean":"2024-08-17T17:41:15.091364+0000","last_became_active":"2024-08-17T17:35:55.204935+0000","last_became_peered":"2024-08-17T17:35:55.204935+0000","last_unstale":"2024-08-17T17:41:15.091364+0000","last_undegraded":"2024-08-17T17:41:15.091364+0000","last_fullsized":"2024-08-17T17:41:15.091364+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:27:52.277821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":132,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.755532+0000","last_change":"2024-08-17T17:36:12.749972+0000","last_active":"2024-08-17T17:41:27.755532+0000","last_peered":"2024-08-17T17:41:27.755532+0000","last_clean":"2024-08-17T17:41:27.755532+0000","last_became_active":"2024-08-17T17:36:12.749620+0000","last_became_peered":"2024-08-17T17:36:12.749620+0000","last_unstale":"2024-08-17T17:41:27.755532+0000","last_undegraded":"2024-08-17T17:41:27.755532+0000","last_fullsized":"2024-08-17T17:41:27.755532+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:05:04.994357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":1008,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.755692+0000","last_change":"2024-08-17T17:36:20.832274+0000","last_active":"2024-08-17T17:41:27.755692+0000","last_peered":"2024-08-17T17:41:27.755692+0000","last_clean":"2024-08-17T17:41:27.755692+0000","last_became_active":"2024-08-17T17:36:20.831886+0000","last_became_peered":"2024-08-17T17:36:20.831886+0000","last_unstale":"2024-08-17T17:41:27.755692+0000","last_undegraded":"2024-08-17T17:41:27.755692+0000","last_fullsized":"2024-08-17T17:41:27.755692+0000","mapping_epoch":1390,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1391,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:14:05.248711+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1217,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.755295+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:41:27.755295+0000","last_peered":"2024-08-17T17:41:27.755295+0000","last_clean":"2024-08-17T17:41:27.755295+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:41:27.755295+0000","last_undegraded":"2024-08-17T17:41:27.755295+0000","last_fullsized":"2024-08-17T17:41:27.755295+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1433,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.733107+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:41:14.733107+0000","last_peered":"2024-08-17T17:41:14.733107+0000","last_clean":"2024-08-17T17:41:14.733107+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:41:14.733107+0000","last_undegraded":"2024-08-17T17:41:14.733107+0000","last_fullsized":"2024-08-17T17:41:14.733107+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1727,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.731978+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:41:14.731978+0000","last_peered":"2024-08-17T17:41:14.731978+0000","last_clean":"2024-08-17T17:41:14.731978+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:41:14.731978+0000","last_undegraded":"2024-08-17T17:41:14.731978+0000","last_fullsized":"2024-08-17T17:41:14.731978+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1375,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.107983+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:41:14.107983+0000","last_peered":"2024-08-17T17:41:14.107983+0000","last_clean":"2024-08-17T17:41:14.107983+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:41:14.107983+0000","last_undegraded":"2024-08-17T17:41:14.107983+0000","last_fullsized":"2024-08-17T17:41:14.107983+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1936,"reported_epoch":1442,"state":"active+clean","last_fresh":"2024-08-17T17:41:27.755076+0000","last_change":"2024-08-17T17:35:33.137411+0000","last_active":"2024-08-17T17:41:27.755076+0000","last_peered":"2024-08-17T17:41:27.755076+0000","last_clean":"2024-08-17T17:41:27.755076+0000","last_became_active":"2024-08-17T17:35:33.137073+0000","last_became_peered":"2024-08-17T17:35:33.137073+0000","last_unstale":"2024-08-17T17:41:27.755076+0000","last_undegraded":"2024-08-17T17:41:27.755076+0000","last_fullsized":"2024-08-17T17:41:27.755076+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:29:35.924539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1821,"reported_epoch":1441,"state":"active+clean","last_fresh":"2024-08-17T17:41:14.732765+0000","last_change":"2024-08-17T17:36:15.787445+0000","last_active":"2024-08-17T17:41:14.732765+0000","last_peered":"2024-08-17T17:41:14.732765+0000","last_clean":"2024-08-17T17:41:14.732765+0000","last_became_active":"2024-08-17T17:36:15.787099+0000","last_became_peered":"2024-08-17T17:36:15.787099+0000","last_unstale":"2024-08-17T17:41:14.732765+0000","last_undegraded":"2024-08-17T17:41:14.732765+0000","last_fullsized":"2024-08-17T17:41:14.732765+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.721493+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.721493+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:10:47.448168+0000","scrub_duration":0.0021034029999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162889,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":168476,"kb_used_data":7952,"kb_used_omap":54,"kb_used_meta":160457,"kb_avail":93576676,"statfs":{"total":95995035648,"available":95822516224,"internally_reserved":0,"allocated":8142848,"data_stored":5381204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":164307989},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999363,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":230036,"kb_used_data":8008,"kb_used_omap":47,"kb_used_meta":221968,"kb_avail":93515116,"statfs":{"total":95995035648,"available":95759478784,"internally_reserved":0,"allocated":8200192,"data_stored":5447111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":227295892},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627160,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1116672,"kb_used_data":7988,"kb_used_omap":42,"kb_used_meta":1108629,"kb_avail":92628480,"statfs":{"total":95995035648,"available":94851563520,"internally_reserved":0,"allocated":8179712,"data_stored":5441326,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1135236156},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785637,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":860420,"kb_used_data":7672,"kb_used_omap":48,"kb_used_meta":852687,"kb_avail":92884732,"statfs":{"total":95995035648,"available":95113965568,"internally_reserved":0,"allocated":7856128,"data_stored":5114617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":873152174},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725154,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":901196,"kb_used_data":7936,"kb_used_omap":1,"kb_used_meta":893246,"kb_avail":92843956,"statfs":{"total":95995035648,"available":95072210944,"internally_reserved":0,"allocated":8126464,"data_stored":5376903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":914684358},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986791,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":918996,"kb_used_data":7688,"kb_used_omap":1,"kb_used_meta":911294,"kb_avail":92826156,"statfs":{"total":95995035648,"available":95053983744,"internally_reserved":0,"allocated":7872512,"data_stored":5118446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":933165510},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706399,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":942344,"kb_used_data":7932,"kb_used_omap":35,"kb_used_meta":934364,"kb_avail":92802808,"statfs":{"total":95995035648,"available":95030075392,"internally_reserved":0,"allocated":8122368,"data_stored":5376860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":956788791},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832172,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1060808,"kb_used_data":7996,"kb_used_omap":65,"kb_used_meta":1052734,"kb_avail":92684344,"statfs":{"total":95995035648,"available":94908768256,"internally_reserved":0,"allocated":8187904,"data_stored":5442882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1077999803},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:41:37.774 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd pool set .mgr pgp_num 31 2024-08-17T17:41:37.997 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:37 smithi063 ceph-mon[114756]: pgmap v2333: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:37.997 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:37 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3408551535' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:37.997 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:37 smithi063 ceph-mon[114756]: osdmap e1443: 8 total, 8 up, 8 in 2024-08-17T17:41:37.997 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:37 smithi063 ceph-mon[114756]: from='client.46616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:37.997 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:37 smithi063 ceph-mon[109843]: pgmap v2333: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:37.997 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:37 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3408551535' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:37.998 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:37 smithi063 ceph-mon[109843]: osdmap e1443: 8 total, 8 up, 8 in 2024-08-17T17:41:37.998 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:37 smithi063 ceph-mon[109843]: from='client.46616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:37 smithi162 ceph-mon[104668]: pgmap v2333: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:37 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3408551535' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-08-17T17:41:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:37 smithi162 ceph-mon[104668]: osdmap e1443: 8 total, 8 up, 8 in 2024-08-17T17:41:38.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:37 smithi162 ceph-mon[104668]: from='client.46616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:41:39.318 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:39.958 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:39 smithi063 ceph-mon[109843]: pgmap v2335: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:39.959 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:39 smithi063 ceph-mon[114756]: pgmap v2335: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:39.983 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:39.983 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:40.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:39 smithi162 ceph-mon[104668]: pgmap v2335: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:40.743 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-08-17T17:41:41.390 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:42.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:41 smithi063 ceph-mon[114756]: pgmap v2336: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:42.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:41 smithi063 ceph-mon[109843]: pgmap v2336: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:42.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:41 smithi162 ceph-mon[104668]: pgmap v2336: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:43.541 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:41:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:41:43] "GET /metrics HTTP/1.1" 200 34111 "" "Prometheus/2.51.0" 2024-08-17T17:41:43.901 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:43 smithi063 ceph-mon[114756]: pgmap v2337: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:43.902 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:43 smithi063 ceph-mon[109843]: pgmap v2337: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:43.946 INFO:teuthology.orchestra.run.smithi063.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:43.946 INFO:teuthology.orchestra.run.smithi063.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:44.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:43 smithi162 ceph-mon[104668]: pgmap v2337: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:44.741 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-08-17T17:41:44.775 INFO:teuthology.orchestra.run.smithi063.stderr:set pool 1 pgp_num to 31 2024-08-17T17:41:44.966 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:44 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2290927561' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-08-17T17:41:45.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:44 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-08-17T17:41:45.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:44 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2290927561' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-08-17T17:41:45.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:44 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-08-17T17:41:45.088 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:44 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2290927561' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-08-17T17:41:45.089 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:44 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-08-17T17:41:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:45 smithi063 ceph-mon[109843]: pgmap v2338: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:45 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-08-17T17:41:46.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:45 smithi063 ceph-mon[109843]: osdmap e1444: 8 total, 8 up, 8 in 2024-08-17T17:41:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:45 smithi063 ceph-mon[114756]: pgmap v2338: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:46.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:45 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-08-17T17:41:46.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:45 smithi063 ceph-mon[114756]: osdmap e1444: 8 total, 8 up, 8 in 2024-08-17T17:41:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:45 smithi162 ceph-mon[104668]: pgmap v2338: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:46.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:45 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-08-17T17:41:46.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:45 smithi162 ceph-mon[104668]: osdmap e1444: 8 total, 8 up, 8 in 2024-08-17T17:41:46.750 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:46.750 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:47.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:46 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:47.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:46 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:47.059 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:46 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:41:47.366 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-08-17T17:41:47.573 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:47.980 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:47 smithi162 ceph-mon[104668]: pgmap v2340: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:47.980 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-08-17T17:41:47.980 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-08-17T17:41:47.980 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-08-17T17:41:47.980 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]} : dispatch 2024-08-17T17:41:47.980 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-08-17T17:41:47.980 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:47 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]} : dispatch 2024-08-17T17:41:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:47 smithi063 ceph-mon[114756]: pgmap v2340: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-08-17T17:41:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-08-17T17:41:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-08-17T17:41:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]} : dispatch 2024-08-17T17:41:48.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-08-17T17:41:48.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:47 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]} : dispatch 2024-08-17T17:41:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:47 smithi063 ceph-mon[109843]: pgmap v2340: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-08-17T17:41:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-08-17T17:41:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]} : dispatch 2024-08-17T17:41:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]} : dispatch 2024-08-17T17:41:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]} : dispatch 2024-08-17T17:41:48.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:47 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]} : dispatch 2024-08-17T17:41:49.039 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:48 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-08-17T17:41:49.040 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:48 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-08-17T17:41:49.040 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:48 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-08-17T17:41:49.040 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:48 smithi162 ceph-mon[104668]: osdmap e1445: 8 total, 8 up, 8 in 2024-08-17T17:41:49.040 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:48 smithi162 ceph-mon[104668]: osdmap e1446: 8 total, 8 up, 8 in 2024-08-17T17:41:49.256 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:49.256 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:49.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:48 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-08-17T17:41:49.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:48 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-08-17T17:41:49.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:48 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-08-17T17:41:49.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:48 smithi063 ceph-mon[109843]: osdmap e1445: 8 total, 8 up, 8 in 2024-08-17T17:41:49.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:48 smithi063 ceph-mon[109843]: osdmap e1446: 8 total, 8 up, 8 in 2024-08-17T17:41:49.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:48 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-08-17T17:41:49.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:48 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-08-17T17:41:49.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:48 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-08-17T17:41:49.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:48 smithi063 ceph-mon[114756]: osdmap e1445: 8 total, 8 up, 8 in 2024-08-17T17:41:49.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:48 smithi063 ceph-mon[114756]: osdmap e1446: 8 total, 8 up, 8 in 2024-08-17T17:41:49.855 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:49 smithi162 ceph-mon[104668]: pgmap v2342: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:49.873 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-08-17T17:41:50.100 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:50.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:49 smithi063 ceph-mon[114756]: pgmap v2342: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:50.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:49 smithi063 ceph-mon[109843]: pgmap v2342: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:50.414 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2024-08-17T17:41:50.643 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:51.798 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:51.798 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:52.067 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:51 smithi063 ceph-mon[114756]: pgmap v2344: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:52.067 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:51 smithi063 ceph-mon[109843]: pgmap v2344: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:52.090 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:51 smithi162 ceph-mon[104668]: pgmap v2344: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:52.397 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-08-17T17:41:52.397 INFO:teuthology.orchestra.run.smithi063.stdout:mon.a: {} 2024-08-17T17:41:52.398 INFO:teuthology.orchestra.run.smithi063.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:41:52.406 INFO:teuthology.orchestra.run.smithi063.stdout:mon.b: {} 2024-08-17T17:41:52.406 INFO:teuthology.orchestra.run.smithi063.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:41:52.414 INFO:teuthology.orchestra.run.smithi063.stdout:mon.c: {} 2024-08-17T17:41:52.415 INFO:teuthology.orchestra.run.smithi063.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-08-17T17:41:52.603 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:53.011 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:52 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/630101402' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:41:53.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:52 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/630101402' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:41:53.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:52 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/630101402' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-08-17T17:41:53.100 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 0 1 2024-08-17T17:41:53.313 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:53.436 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:41:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:41:53] "GET /metrics HTTP/1.1" 200 34111 "" "Prometheus/2.51.0" 2024-08-17T17:41:54.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:53 smithi063 ceph-mon[114756]: pgmap v2345: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:54.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:53 smithi063 ceph-mon[109843]: pgmap v2345: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:54.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:53 smithi162 ceph-mon[104668]: pgmap v2345: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:54.502 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:54.502 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:55.120 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-08-17T17:41:55.324 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:55.812 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.0 to 1 (10000) 2024-08-17T17:41:56.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:55 smithi162 ceph-mon[104668]: pgmap v2346: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:56.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:55 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/110523103' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:41:56.103 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:55 smithi063 ceph-mon[114756]: pgmap v2346: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:56.103 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:55 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/110523103' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:41:56.103 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:55 smithi063 ceph-mon[109843]: pgmap v2346: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:56.103 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:55 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/110523103' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:41:56.538 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 0 1 2024-08-17T17:41:56.753 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:41:56.977 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:56 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/110523103' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:41:56.978 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:56 smithi162 ceph-mon[104668]: osdmap e1447: 8 total, 8 up, 8 in 2024-08-17T17:41:57.011 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:57.012 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:57.187 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:56 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/110523103' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:41:57.187 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:56 smithi063 ceph-mon[114756]: osdmap e1447: 8 total, 8 up, 8 in 2024-08-17T17:41:57.187 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:56 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/110523103' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:41:57.187 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:56 smithi063 ceph-mon[109843]: osdmap e1447: 8 total, 8 up, 8 in 2024-08-17T17:41:57.653 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-08-17T17:41:57.861 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:41:57.940 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:57 smithi063 ceph-mon[114756]: pgmap v2348: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:57.940 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:57 smithi063 ceph-mon[109843]: pgmap v2348: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:57.983 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:57 smithi162 ceph-mon[104668]: pgmap v2348: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:59.200 INFO:teuthology.orchestra.run.smithi063.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-08-17T17:41:59.259 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:59 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3072453415' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:41:59.259 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:41:59 smithi162 ceph-mon[104668]: pgmap v2349: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:59.500 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:41:59.500 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:41:59.514 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:59 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3072453415' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:41:59.514 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:41:59 smithi063 ceph-mon[114756]: pgmap v2349: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:59.514 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:59 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3072453415' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-08-17T17:41:59.514 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:41:59 smithi063 ceph-mon[109843]: pgmap v2349: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:41:59.943 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 1 1 2024-08-17T17:42:00.143 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-08-17T17:42:00.166 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:00.207 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:00 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3072453415' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:42:00.207 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:00 smithi063 ceph-mon[109843]: osdmap e1448: 8 total, 8 up, 8 in 2024-08-17T17:42:00.353 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:00.475 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:00 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3072453415' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:42:00.475 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:00 smithi162 ceph-mon[104668]: osdmap e1448: 8 total, 8 up, 8 in 2024-08-17T17:42:00.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:00 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3072453415' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-08-17T17:42:00.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:00 smithi063 ceph-mon[114756]: osdmap e1448: 8 total, 8 up, 8 in 2024-08-17T17:42:01.321 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:01 smithi063 ceph-mon[114756]: osdmap e1449: 8 total, 8 up, 8 in 2024-08-17T17:42:01.321 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:01 smithi063 ceph-mon[114756]: pgmap v2352: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:01.321 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:01 smithi063 ceph-mon[109843]: osdmap e1449: 8 total, 8 up, 8 in 2024-08-17T17:42:01.322 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:01 smithi063 ceph-mon[109843]: pgmap v2352: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:01.470 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:01 smithi162 ceph-mon[104668]: osdmap e1449: 8 total, 8 up, 8 in 2024-08-17T17:42:01.470 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:01 smithi162 ceph-mon[104668]: pgmap v2352: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:02.007 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:42:02.007 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:42:02.211 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.1 to 1 (10000) 2024-08-17T17:42:02.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:02 smithi063 ceph-mon[114756]: 1.2 scrub starts 2024-08-17T17:42:02.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:02 smithi063 ceph-mon[114756]: 1.2 scrub ok 2024-08-17T17:42:02.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:02 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:02.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:02 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/417127103' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:42:02.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:02 smithi063 ceph-mon[109843]: 1.2 scrub starts 2024-08-17T17:42:02.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:02 smithi063 ceph-mon[109843]: 1.2 scrub ok 2024-08-17T17:42:02.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:02 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:02.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:02 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/417127103' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:42:02.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:02 smithi162 ceph-mon[104668]: 1.2 scrub starts 2024-08-17T17:42:02.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:02 smithi162 ceph-mon[104668]: 1.2 scrub ok 2024-08-17T17:42:02.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:02 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:02.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:02 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/417127103' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:42:02.648 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-08-17T17:42:02.852 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:02.853 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 1 1 2024-08-17T17:42:03.067 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:03.231 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:03 smithi162 ceph-mon[104668]: 1.11 scrub starts 2024-08-17T17:42:03.232 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:03 smithi162 ceph-mon[104668]: 1.11 scrub ok 2024-08-17T17:42:03.232 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:03 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/417127103' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:42:03.232 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:03 smithi162 ceph-mon[104668]: osdmap e1450: 8 total, 8 up, 8 in 2024-08-17T17:42:03.232 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:03 smithi162 ceph-mon[104668]: pgmap v2354: 103 pgs: 103 active+clean; 621 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 9.5 KiB/s rd, 27 KiB/s wr, 6 op/s; 20 KiB/s, 0 objects/s recovering 2024-08-17T17:42:03.440 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:03 smithi063 ceph-mon[109843]: 1.11 scrub starts 2024-08-17T17:42:03.441 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:03 smithi063 ceph-mon[109843]: 1.11 scrub ok 2024-08-17T17:42:03.441 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:03 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/417127103' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:42:03.441 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:03 smithi063 ceph-mon[109843]: osdmap e1450: 8 total, 8 up, 8 in 2024-08-17T17:42:03.441 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:03 smithi063 ceph-mon[109843]: pgmap v2354: 103 pgs: 103 active+clean; 621 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 9.5 KiB/s rd, 27 KiB/s wr, 6 op/s; 20 KiB/s, 0 objects/s recovering 2024-08-17T17:42:03.441 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:42:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:42:03] "GET /metrics HTTP/1.1" 200 34111 "" "Prometheus/2.51.0" 2024-08-17T17:42:03.441 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:03 smithi063 ceph-mon[114756]: 1.11 scrub starts 2024-08-17T17:42:03.442 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:03 smithi063 ceph-mon[114756]: 1.11 scrub ok 2024-08-17T17:42:03.442 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:03 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/417127103' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:42:03.442 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:03 smithi063 ceph-mon[114756]: osdmap e1450: 8 total, 8 up, 8 in 2024-08-17T17:42:03.442 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:03 smithi063 ceph-mon[114756]: pgmap v2354: 103 pgs: 103 active+clean; 621 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 9.5 KiB/s rd, 27 KiB/s wr, 6 op/s; 20 KiB/s, 0 objects/s recovering 2024-08-17T17:42:04.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:04 smithi162 ceph-mon[104668]: 1.16 scrub starts 2024-08-17T17:42:04.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:04 smithi162 ceph-mon[104668]: 1.16 scrub ok 2024-08-17T17:42:04.478 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:04 smithi063 ceph-mon[109843]: 1.16 scrub starts 2024-08-17T17:42:04.478 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:04 smithi063 ceph-mon[109843]: 1.16 scrub ok 2024-08-17T17:42:04.478 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:04 smithi063 ceph-mon[114756]: 1.16 scrub starts 2024-08-17T17:42:04.478 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:04 smithi063 ceph-mon[114756]: 1.16 scrub ok 2024-08-17T17:42:04.490 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:42:04.490 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:42:05.222 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-08-17T17:42:05.226 INFO:teuthology.orchestra.run.smithi063.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-08-17T17:42:05.427 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:05.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:05 smithi063 ceph-mon[114756]: pgmap v2355: 103 pgs: 103 active+clean; 621 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 9.5 KiB/s rd, 27 KiB/s wr, 6 op/s; 20 KiB/s, 0 objects/s recovering 2024-08-17T17:42:05.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:05 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2777375120' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:42:05.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:05 smithi063 ceph-mon[109843]: pgmap v2355: 103 pgs: 103 active+clean; 621 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 9.5 KiB/s rd, 27 KiB/s wr, 6 op/s; 20 KiB/s, 0 objects/s recovering 2024-08-17T17:42:05.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:05 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2777375120' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:42:05.549 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:05 smithi162 ceph-mon[104668]: pgmap v2355: 103 pgs: 103 active+clean; 621 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 9.5 KiB/s rd, 27 KiB/s wr, 6 op/s; 20 KiB/s, 0 objects/s recovering 2024-08-17T17:42:05.549 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:05 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2777375120' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-08-17T17:42:05.775 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 2 1 2024-08-17T17:42:05.988 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:06.228 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:06 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2777375120' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:42:06.228 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:06 smithi162 ceph-mon[104668]: osdmap e1451: 8 total, 8 up, 8 in 2024-08-17T17:42:06.387 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:06 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2777375120' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:42:06.388 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:06 smithi063 ceph-mon[114756]: osdmap e1451: 8 total, 8 up, 8 in 2024-08-17T17:42:06.388 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:06 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2777375120' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-08-17T17:42:06.388 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:06 smithi063 ceph-mon[109843]: osdmap e1451: 8 total, 8 up, 8 in 2024-08-17T17:42:07.090 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:42:07.091 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:42:07.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:07 smithi063 ceph-mon[114756]: osdmap e1452: 8 total, 8 up, 8 in 2024-08-17T17:42:07.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:07 smithi063 ceph-mon[114756]: 1.b scrub starts 2024-08-17T17:42:07.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:07 smithi063 ceph-mon[114756]: 1.b scrub ok 2024-08-17T17:42:07.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:07 smithi063 ceph-mon[114756]: pgmap v2358: 103 pgs: 103 active+clean; 595 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 3.2 KiB/s rd, 9.2 KiB/s wr, 2 op/s; 6.5 KiB/s, 0 objects/s recovering 2024-08-17T17:42:07.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:07 smithi063 ceph-mon[109843]: osdmap e1452: 8 total, 8 up, 8 in 2024-08-17T17:42:07.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:07 smithi063 ceph-mon[109843]: 1.b scrub starts 2024-08-17T17:42:07.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:07 smithi063 ceph-mon[109843]: 1.b scrub ok 2024-08-17T17:42:07.530 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:07 smithi063 ceph-mon[109843]: pgmap v2358: 103 pgs: 103 active+clean; 595 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 3.2 KiB/s rd, 9.2 KiB/s wr, 2 op/s; 6.5 KiB/s, 0 objects/s recovering 2024-08-17T17:42:07.549 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:07 smithi162 ceph-mon[104668]: osdmap e1452: 8 total, 8 up, 8 in 2024-08-17T17:42:07.549 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:07 smithi162 ceph-mon[104668]: 1.b scrub starts 2024-08-17T17:42:07.549 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:07 smithi162 ceph-mon[104668]: 1.b scrub ok 2024-08-17T17:42:07.549 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:07 smithi162 ceph-mon[104668]: pgmap v2358: 103 pgs: 103 active+clean; 595 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 3.2 KiB/s rd, 9.2 KiB/s wr, 2 op/s; 6.5 KiB/s, 0 objects/s recovering 2024-08-17T17:42:07.841 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-08-17T17:42:08.052 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:08.238 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.2 to 1 (10000) 2024-08-17T17:42:08.464 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:08 smithi162 ceph-mon[104668]: 1.3 deep-scrub starts 2024-08-17T17:42:08.464 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:08 smithi162 ceph-mon[104668]: 1.3 deep-scrub ok 2024-08-17T17:42:08.464 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:08 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4035116532' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:42:08.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:08 smithi063 ceph-mon[109843]: 1.3 deep-scrub starts 2024-08-17T17:42:08.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:08 smithi063 ceph-mon[109843]: 1.3 deep-scrub ok 2024-08-17T17:42:08.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:08 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4035116532' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:42:08.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:08 smithi063 ceph-mon[114756]: 1.3 deep-scrub starts 2024-08-17T17:42:08.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:08 smithi063 ceph-mon[114756]: 1.3 deep-scrub ok 2024-08-17T17:42:08.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:08 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4035116532' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:42:08.951 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 2 1 2024-08-17T17:42:09.169 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:09.480 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:09 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4035116532' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:42:09.481 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:09 smithi162 ceph-mon[104668]: osdmap e1453: 8 total, 8 up, 8 in 2024-08-17T17:42:09.481 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:09 smithi162 ceph-mon[104668]: pgmap v2360: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:09.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:09 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4035116532' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:42:09.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:09 smithi063 ceph-mon[114756]: osdmap e1453: 8 total, 8 up, 8 in 2024-08-17T17:42:09.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:09 smithi063 ceph-mon[114756]: pgmap v2360: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:09.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:09 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4035116532' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:42:09.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:09 smithi063 ceph-mon[109843]: osdmap e1453: 8 total, 8 up, 8 in 2024-08-17T17:42:09.529 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:09 smithi063 ceph-mon[109843]: pgmap v2360: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:09.773 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:42:09.774 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:42:10.414 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-08-17T17:42:10.620 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:11.767 INFO:teuthology.orchestra.run.smithi063.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-08-17T17:42:11.907 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:11 smithi162 ceph-mon[104668]: pgmap v2361: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:11.907 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:11 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/668448971' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:42:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:11 smithi063 ceph-mon[109843]: pgmap v2361: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:12.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:11 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/668448971' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:42:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:11 smithi063 ceph-mon[114756]: pgmap v2361: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:12.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:11 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/668448971' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-08-17T17:42:12.169 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:42:12.169 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:42:12.451 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 3 1 2024-08-17T17:42:12.671 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:12.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:12 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/668448971' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:42:12.845 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:12 smithi162 ceph-mon[104668]: osdmap e1454: 8 total, 8 up, 8 in 2024-08-17T17:42:12.870 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-08-17T17:42:13.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:12 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/668448971' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:42:13.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:12 smithi063 ceph-mon[114756]: osdmap e1454: 8 total, 8 up, 8 in 2024-08-17T17:42:13.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:12 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/668448971' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-08-17T17:42:13.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:12 smithi063 ceph-mon[109843]: osdmap e1454: 8 total, 8 up, 8 in 2024-08-17T17:42:13.074 INFO:teuthology.orchestra.run.smithi162.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.b/config 2024-08-17T17:42:13.505 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:42:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:42:13] "GET /metrics HTTP/1.1" 200 34111 "" "Prometheus/2.51.0" 2024-08-17T17:42:14.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:13 smithi063 ceph-mon[114756]: pgmap v2363: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:14.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:13 smithi063 ceph-mon[109843]: pgmap v2363: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:13 smithi162 ceph-mon[104668]: pgmap v2363: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:14.763 INFO:teuthology.orchestra.run.smithi162.stdout:ERROR: (22) Invalid argument 2024-08-17T17:42:14.763 INFO:teuthology.orchestra.run.smithi162.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-08-17T17:42:14.782 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.3 to 1 (10000) 2024-08-17T17:42:15.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:14 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3037923751' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:42:15.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:14 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3037923751' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:42:15.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:14 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3037923751' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:42:15.388 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 3 1 2024-08-17T17:42:15.604 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:15 smithi063 ceph-mon[109843]: pgmap v2364: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:15 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3037923751' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:42:16.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:15 smithi063 ceph-mon[109843]: osdmap e1455: 8 total, 8 up, 8 in 2024-08-17T17:42:16.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:15 smithi063 ceph-mon[114756]: pgmap v2364: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:16.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:15 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3037923751' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:42:16.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:15 smithi063 ceph-mon[114756]: osdmap e1455: 8 total, 8 up, 8 in 2024-08-17T17:42:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:15 smithi162 ceph-mon[104668]: pgmap v2364: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:15 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3037923751' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:42:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:15 smithi162 ceph-mon[104668]: osdmap e1455: 8 total, 8 up, 8 in 2024-08-17T17:42:16.990 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:16.990 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:17.840 INFO:teuthology.orchestra.run.smithi063.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-08-17T17:42:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:17 smithi063 ceph-mon[109843]: pgmap v2366: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:18.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:17 smithi063 ceph-mon[114756]: pgmap v2366: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:17 smithi162 ceph-mon[104668]: pgmap v2366: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:18.566 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 4 1 2024-08-17T17:42:18.784 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:18.789 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:18 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2154058408' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:42:18.790 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:18 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2154058408' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:42:18.790 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:18 smithi063 ceph-mon[109843]: osdmap e1456: 8 total, 8 up, 8 in 2024-08-17T17:42:19.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:18 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2154058408' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:42:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:18 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2154058408' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:42:19.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:18 smithi162 ceph-mon[104668]: osdmap e1456: 8 total, 8 up, 8 in 2024-08-17T17:42:19.150 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:18 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2154058408' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-08-17T17:42:19.151 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:18 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2154058408' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-08-17T17:42:19.151 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:18 smithi063 ceph-mon[114756]: osdmap e1456: 8 total, 8 up, 8 in 2024-08-17T17:42:20.185 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:19 smithi063 ceph-mon[114756]: pgmap v2368: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:20.185 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:19 smithi063 ceph-mon[114756]: osdmap e1457: 8 total, 8 up, 8 in 2024-08-17T17:42:20.185 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:19 smithi063 ceph-mon[109843]: pgmap v2368: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:20.186 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:19 smithi063 ceph-mon[109843]: osdmap e1457: 8 total, 8 up, 8 in 2024-08-17T17:42:20.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:19 smithi162 ceph-mon[104668]: pgmap v2368: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:20.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:19 smithi162 ceph-mon[104668]: osdmap e1457: 8 total, 8 up, 8 in 2024-08-17T17:42:20.858 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.4 to 1 (10000) 2024-08-17T17:42:21.248 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:20 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4008672977' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:42:21.248 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:20 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:42:21.248 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:20 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4008672977' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:42:21.248 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:20 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:42:21.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:20 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4008672977' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:42:21.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:20 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:42:21.637 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 4 1 2024-08-17T17:42:21.851 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:21.975 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:21 smithi063 ceph-mon[114756]: pgmap v2370: 103 pgs: 12 peering, 91 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:42:21.975 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:21 smithi063 ceph-mon[114756]: Health check failed: Reduced data availability: 11 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-08-17T17:42:21.975 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:21 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:42:21.975 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:21 smithi063 ceph-mon[114756]: osdmap e1458: 8 total, 8 up, 8 in 2024-08-17T17:42:21.975 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:21 smithi063 ceph-mon[109843]: pgmap v2370: 103 pgs: 12 peering, 91 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:42:21.975 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:21 smithi063 ceph-mon[109843]: Health check failed: Reduced data availability: 11 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-08-17T17:42:21.976 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:21 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:42:21.976 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:21 smithi063 ceph-mon[109843]: osdmap e1458: 8 total, 8 up, 8 in 2024-08-17T17:42:22.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:21 smithi162 ceph-mon[104668]: pgmap v2370: 103 pgs: 12 peering, 91 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:42:22.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:21 smithi162 ceph-mon[104668]: Health check failed: Reduced data availability: 11 pgs inactive, 12 pgs peering (PG_AVAILABILITY) 2024-08-17T17:42:22.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:21 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:42:22.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:21 smithi162 ceph-mon[104668]: osdmap e1458: 8 total, 8 up, 8 in 2024-08-17T17:42:23.445 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:42:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:42:23] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.51.0" 2024-08-17T17:42:23.851 INFO:teuthology.orchestra.run.smithi063.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-08-17T17:42:24.160 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:23 smithi063 ceph-mon[114756]: pgmap v2372: 103 pgs: 12 peering, 91 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:42:24.160 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:23 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/861754667' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:42:24.160 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:23 smithi063 ceph-mon[109843]: pgmap v2372: 103 pgs: 12 peering, 91 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:42:24.160 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:23 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/861754667' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:42:24.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:23 smithi162 ceph-mon[104668]: pgmap v2372: 103 pgs: 12 peering, 91 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:42:24.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:23 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/861754667' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-08-17T17:42:24.549 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 5 1 2024-08-17T17:42:24.762 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:25.155 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:24 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/861754667' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:42:25.155 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:24 smithi063 ceph-mon[114756]: osdmap e1459: 8 total, 8 up, 8 in 2024-08-17T17:42:25.155 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:24 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/861754667' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:42:25.155 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:24 smithi063 ceph-mon[109843]: osdmap e1459: 8 total, 8 up, 8 in 2024-08-17T17:42:25.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:24 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/861754667' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-08-17T17:42:25.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:24 smithi162 ceph-mon[104668]: osdmap e1459: 8 total, 8 up, 8 in 2024-08-17T17:42:26.001 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:25 smithi063 ceph-mon[114756]: pgmap v2374: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:42:26.002 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:25 smithi063 ceph-mon[114756]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 12 pgs peering) 2024-08-17T17:42:26.002 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:25 smithi063 ceph-mon[114756]: Cluster is now healthy 2024-08-17T17:42:26.002 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:25 smithi063 ceph-mon[109843]: pgmap v2374: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:42:26.002 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:25 smithi063 ceph-mon[109843]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 12 pgs peering) 2024-08-17T17:42:26.002 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:25 smithi063 ceph-mon[109843]: Cluster is now healthy 2024-08-17T17:42:26.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:25 smithi162 ceph-mon[104668]: pgmap v2374: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-08-17T17:42:26.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:25 smithi162 ceph-mon[104668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 11 pgs inactive, 12 pgs peering) 2024-08-17T17:42:26.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:25 smithi162 ceph-mon[104668]: Cluster is now healthy 2024-08-17T17:42:26.873 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.5 to 1 (10000) 2024-08-17T17:42:27.190 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:26 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/814514976' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:42:27.191 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:26 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/814514976' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:42:27.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:26 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/814514976' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:42:27.583 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 5 1 2024-08-17T17:42:27.801 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:28.212 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:27 smithi063 ceph-mon[109843]: pgmap v2375: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:28.212 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:27 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/814514976' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:42:28.212 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:27 smithi063 ceph-mon[109843]: osdmap e1460: 8 total, 8 up, 8 in 2024-08-17T17:42:28.212 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:27 smithi063 ceph-mon[114756]: pgmap v2375: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:28.212 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:27 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/814514976' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:42:28.212 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:27 smithi063 ceph-mon[114756]: osdmap e1460: 8 total, 8 up, 8 in 2024-08-17T17:42:28.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:27 smithi162 ceph-mon[104668]: pgmap v2375: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:28.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:27 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/814514976' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:42:28.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:27 smithi162 ceph-mon[104668]: osdmap e1460: 8 total, 8 up, 8 in 2024-08-17T17:42:29.877 INFO:teuthology.orchestra.run.smithi063.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-08-17T17:42:30.184 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:29 smithi063 ceph-mon[114756]: pgmap v2377: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:30.184 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:29 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1133287394' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:42:30.184 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:29 smithi063 ceph-mon[109843]: pgmap v2377: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:30.185 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:29 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1133287394' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:42:30.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:29 smithi162 ceph-mon[104668]: pgmap v2377: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:30.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:29 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1133287394' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-08-17T17:42:30.642 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 7 1 2024-08-17T17:42:30.858 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:31.263 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:30 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1133287394' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:42:31.263 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:30 smithi063 ceph-mon[114756]: osdmap e1461: 8 total, 8 up, 8 in 2024-08-17T17:42:31.263 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:30 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1133287394' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:42:31.263 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:30 smithi063 ceph-mon[109843]: osdmap e1461: 8 total, 8 up, 8 in 2024-08-17T17:42:31.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:30 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1133287394' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-08-17T17:42:31.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:30 smithi162 ceph-mon[104668]: osdmap e1461: 8 total, 8 up, 8 in 2024-08-17T17:42:31.977 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:31 smithi063 ceph-mon[109843]: pgmap v2379: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:31.977 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:31.978 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:31 smithi063 ceph-mon[114756]: pgmap v2379: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:31.978 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:32.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:31 smithi162 ceph-mon[104668]: pgmap v2379: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:32.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:32.895 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.7 to 1 (10000) 2024-08-17T17:42:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:32 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:42:33.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:32 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3649914218' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:42:33.261 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:32 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:42:33.261 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:32 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3649914218' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:42:33.261 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:32 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-08-17T17:42:33.261 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:32 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3649914218' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:42:33.528 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:42:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:42:33] "GET /metrics HTTP/1.1" 200 34109 "" "Prometheus/2.51.0" 2024-08-17T17:42:33.552 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 7 1 2024-08-17T17:42:33.960 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:34.188 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:33 smithi063 ceph-mon[109843]: pgmap v2380: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:34.188 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:33 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3649914218' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:42:34.188 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:33 smithi063 ceph-mon[109843]: osdmap e1462: 8 total, 8 up, 8 in 2024-08-17T17:42:34.188 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:33 smithi063 ceph-mon[114756]: pgmap v2380: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:34.189 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:33 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3649914218' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:42:34.189 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:33 smithi063 ceph-mon[114756]: osdmap e1462: 8 total, 8 up, 8 in 2024-08-17T17:42:34.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:33 smithi162 ceph-mon[104668]: pgmap v2380: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:34.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:33 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3649914218' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:42:34.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:33 smithi162 ceph-mon[104668]: osdmap e1462: 8 total, 8 up, 8 in 2024-08-17T17:42:35.904 INFO:teuthology.orchestra.run.smithi063.stderr:set osd.7 primary-affinity to 1 (8655362) 2024-08-17T17:42:36.198 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:35 smithi162 ceph-mon[104668]: pgmap v2382: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:36.198 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:35 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2318826107' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:42:36.198 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:35 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:42:36.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:35 smithi063 ceph-mon[114756]: pgmap v2382: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:36.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:35 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2318826107' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:42:36.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:35 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:42:36.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:35 smithi063 ceph-mon[109843]: pgmap v2382: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:36.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:35 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2318826107' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:42:36.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:35 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-08-17T17:42:36.973 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:36 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:42:36.973 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:36 smithi063 ceph-mon[114756]: osdmap e1463: 8 total, 8 up, 8 in 2024-08-17T17:42:36.973 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:36 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:42:36.973 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:36 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:42:36.973 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:36 smithi063 ceph-mon[114756]: osdmap e1464: 8 total, 8 up, 8 in 2024-08-17T17:42:36.973 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:36 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:42:36.974 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:36 smithi063 ceph-mon[109843]: osdmap e1463: 8 total, 8 up, 8 in 2024-08-17T17:42:36.974 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:36 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:42:36.974 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:36 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:42:36.974 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:36 smithi063 ceph-mon[109843]: osdmap e1464: 8 total, 8 up, 8 in 2024-08-17T17:42:36.994 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd reweight 6 1 2024-08-17T17:42:37.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:36 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-08-17T17:42:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:36 smithi162 ceph-mon[104668]: osdmap e1463: 8 total, 8 up, 8 in 2024-08-17T17:42:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:36 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:42:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:36 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:42:37.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:36 smithi162 ceph-mon[104668]: osdmap e1464: 8 total, 8 up, 8 in 2024-08-17T17:42:37.796 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:38.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:37 smithi162 ceph-mon[104668]: pgmap v2384: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:38.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:37 smithi063 ceph-mon[114756]: pgmap v2384: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:38.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:37 smithi063 ceph-mon[109843]: pgmap v2384: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:40.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:39 smithi063 ceph-mon[114756]: pgmap v2386: 103 pgs: 103 active+clean; 742 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 7 B/s, 0 objects/s recovering 2024-08-17T17:42:40.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:39 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:42:40.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:39 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:42:40.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:39 smithi063 ceph-mon[109843]: pgmap v2386: 103 pgs: 103 active+clean; 742 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 7 B/s, 0 objects/s recovering 2024-08-17T17:42:40.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:39 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:42:40.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:39 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:42:40.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:39 smithi162 ceph-mon[104668]: pgmap v2386: 103 pgs: 103 active+clean; 742 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 7 B/s, 0 objects/s recovering 2024-08-17T17:42:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:39 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:42:40.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:39 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:42:41.082 INFO:teuthology.orchestra.run.smithi063.stderr:reweighted osd.6 to 1 (10000) 2024-08-17T17:42:41.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:40 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:42:41.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:40 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:42:41.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:40 smithi063 ceph-mon[109843]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:42:41.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:40 smithi063 ceph-mon[109843]: 1.1c scrub starts 2024-08-17T17:42:41.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:40 smithi063 ceph-mon[109843]: 1.1c scrub ok 2024-08-17T17:42:41.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:40 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2636865837' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:42:41.280 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:40 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:42:41.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:40 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:42:41.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:40 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:42:41.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:40 smithi063 ceph-mon[114756]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:42:41.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:40 smithi063 ceph-mon[114756]: 1.1c scrub starts 2024-08-17T17:42:41.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:40 smithi063 ceph-mon[114756]: 1.1c scrub ok 2024-08-17T17:42:41.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:40 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2636865837' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:42:41.280 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:40 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:42:41.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:40 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-08-17T17:42:41.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:40 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-08-17T17:42:41.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:40 smithi162 ceph-mon[104668]: from='mgr.25070 ' entity='mgr.y' 2024-08-17T17:42:41.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:40 smithi162 ceph-mon[104668]: 1.1c scrub starts 2024-08-17T17:42:41.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:40 smithi162 ceph-mon[104668]: 1.1c scrub ok 2024-08-17T17:42:41.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:40 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2636865837' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:42:41.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:40 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:42:41.874 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd primary-affinity 6 1 2024-08-17T17:42:42.098 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:42.213 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:42 smithi063 ceph-mon[109843]: pgmap v2387: 103 pgs: 103 active+clean; 742 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-08-17T17:42:42.214 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:42 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:42:42.214 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:42 smithi063 ceph-mon[109843]: osdmap e1465: 8 total, 8 up, 8 in 2024-08-17T17:42:42.214 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:42 smithi063 ceph-mon[114756]: pgmap v2387: 103 pgs: 103 active+clean; 742 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-08-17T17:42:42.214 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:42 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:42:42.214 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:42 smithi063 ceph-mon[114756]: osdmap e1465: 8 total, 8 up, 8 in 2024-08-17T17:42:42.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:42 smithi162 ceph-mon[104668]: pgmap v2387: 103 pgs: 103 active+clean; 742 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-08-17T17:42:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:42 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:42:42.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:42 smithi162 ceph-mon[104668]: osdmap e1465: 8 total, 8 up, 8 in 2024-08-17T17:42:43.565 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:42:43 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:42:43] "GET /metrics HTTP/1.1" 200 34109 "" "Prometheus/2.51.0" 2024-08-17T17:42:44.085 INFO:teuthology.orchestra.run.smithi063.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-08-17T17:42:44.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:44 smithi162 ceph-mon[104668]: pgmap v2389: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 6 B/s, 0 objects/s recovering 2024-08-17T17:42:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:44 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4082713544' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:42:44.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:44 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:42:44.451 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:44 smithi063 ceph-mon[114756]: pgmap v2389: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 6 B/s, 0 objects/s recovering 2024-08-17T17:42:44.451 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:44 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4082713544' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:42:44.452 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:44 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:42:44.452 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:44 smithi063 ceph-mon[109843]: pgmap v2389: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 6 B/s, 0 objects/s recovering 2024-08-17T17:42:44.452 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:44 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4082713544' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:42:44.452 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:44 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-08-17T17:42:44.843 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2024-08-17T17:42:44.843 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2024-08-17T17:42:44.843 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2024-08-17T17:42:44.843 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2024-08-17T17:42:44.844 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-08-17T17:42:44.844 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:42:45.059 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:45.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:45 smithi162 ceph-mon[104668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:42:45.346 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:45 smithi162 ceph-mon[104668]: osdmap e1466: 8 total, 8 up, 8 in 2024-08-17T17:42:45.346 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:45 smithi063 ceph-mon[114756]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:42:45.346 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:45 smithi063 ceph-mon[114756]: osdmap e1466: 8 total, 8 up, 8 in 2024-08-17T17:42:45.347 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:45 smithi063 ceph-mon[109843]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-08-17T17:42:45.347 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:45 smithi063 ceph-mon[109843]: osdmap e1466: 8 total, 8 up, 8 in 2024-08-17T17:42:46.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:46 smithi063 ceph-mon[109843]: pgmap v2391: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-08-17T17:42:46.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:46 smithi063 ceph-mon[114756]: pgmap v2391: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-08-17T17:42:46.345 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:46 smithi162 ceph-mon[104668]: pgmap v2391: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-08-17T17:42:47.328 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:42:47.328 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:47 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:47.329 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":1466,"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:46:54.156006+0000","modified":"2024-08-17T17:42:44.075934+0000","last_up_change":"2024-08-17T17:35:54.172906+0000","last_in_change":"2024-08-17T17:35:13.821379+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:49:54.431240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1444","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"1223","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:52:22.131465+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T16:53:35.328125+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T16:53:36.292966+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":726,"up_from":739,"up_thru":1448,"down_at":727,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6802","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6803","nonce":4285380350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6804","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6805","nonce":4285380350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6808","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6809","nonce":4285380350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6806","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6807","nonce":4285380350}]},"public_addr":"172.21.15.63:6803/4285380350","cluster_addr":"172.21.15.63:6805/4285380350","heartbeat_back_addr":"172.21.15.63:6809/4285380350","heartbeat_front_addr":"172.21.15.63:6807/4285380350","state":["exists","up"]},{"osd":1,"uuid":"ae346e99-4706-4871-990a-f8e737572245","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":758,"up_from":771,"up_thru":1451,"down_at":759,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6810","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6811","nonce":3170762701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6812","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6813","nonce":3170762701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6816","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6817","nonce":3170762701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6814","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6815","nonce":3170762701}]},"public_addr":"172.21.15.63:6811/3170762701","cluster_addr":"172.21.15.63:6813/3170762701","heartbeat_back_addr":"172.21.15.63:6817/3170762701","heartbeat_front_addr":"172.21.15.63:6815/3170762701","state":["exists","up"]},{"osd":2,"uuid":"2d85d75f-1280-443f-857d-d322bd60c451","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":791,"up_from":804,"up_thru":1445,"down_at":792,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6818","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6819","nonce":3903658295}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6820","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6821","nonce":3903658295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6824","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6825","nonce":3903658295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6822","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6823","nonce":3903658295}]},"public_addr":"172.21.15.63:6819/3903658295","cluster_addr":"172.21.15.63:6821/3903658295","heartbeat_back_addr":"172.21.15.63:6825/3903658295","heartbeat_front_addr":"172.21.15.63:6823/3903658295","state":["exists","up"]},{"osd":3,"uuid":"771f43d2-e98b-43e0-921b-88e56fadda42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":824,"up_from":837,"up_thru":1456,"down_at":825,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6826","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6827","nonce":1764356824}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6828","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6829","nonce":1764356824}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6832","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6833","nonce":1764356824}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6830","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6831","nonce":1764356824}]},"public_addr":"172.21.15.63:6827/1764356824","cluster_addr":"172.21.15.63:6829/1764356824","heartbeat_back_addr":"172.21.15.63:6833/1764356824","heartbeat_front_addr":"172.21.15.63:6831/1764356824","state":["exists","up"]},{"osd":4,"uuid":"73c3c545-e975-4a6f-a1cb-e4575ee7a9b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":1436,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6800","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6801","nonce":1563885208}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6803","nonce":1563885208}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6807","nonce":1563885208}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6805","nonce":1563885208}]},"public_addr":"172.21.15.162:6801/1563885208","cluster_addr":"172.21.15.162:6803/1563885208","heartbeat_back_addr":"172.21.15.162:6807/1563885208","heartbeat_front_addr":"172.21.15.162:6805/1563885208","state":["exists","up"]},{"osd":5,"uuid":"fae3ac52-3194-4eb4-9510-e3899768353a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":1445,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6809","nonce":1826502179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6810","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6811","nonce":1826502179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6814","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6815","nonce":1826502179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6812","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6813","nonce":1826502179}]},"public_addr":"172.21.15.162:6809/1826502179","cluster_addr":"172.21.15.162:6811/1826502179","heartbeat_back_addr":"172.21.15.162:6815/1826502179","heartbeat_front_addr":"172.21.15.162:6813/1826502179","state":["exists","up"]},{"osd":6,"uuid":"a8255b37-1be5-4cd4-9b86-889f6bf52902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":47,"last_clean_end":1273,"up_from":1376,"up_thru":1436,"down_at":1274,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6824","nonce":1307501458},{"type":"v1","addr":"172.21.15.162:6825","nonce":1307501458}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6826","nonce":1307501458},{"type":"v1","addr":"172.21.15.162:6827","nonce":1307501458}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6830","nonce":1307501458},{"type":"v1","addr":"172.21.15.162:6831","nonce":1307501458}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6828","nonce":1307501458},{"type":"v1","addr":"172.21.15.162:6829","nonce":1307501458}]},"public_addr":"172.21.15.162:6825/1307501458","cluster_addr":"172.21.15.162:6827/1307501458","heartbeat_back_addr":"172.21.15.162:6831/1307501458","heartbeat_front_addr":"172.21.15.162:6829/1307501458","state":["exists","up"]},{"osd":7,"uuid":"3d0bcd6d-171d-4d90-a033-ed17714a9d96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":55,"last_clean_end":1359,"up_from":1371,"up_thru":1463,"down_at":1360,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6816","nonce":3585957098},{"type":"v1","addr":"172.21.15.162:6817","nonce":3585957098}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6818","nonce":3585957098},{"type":"v1","addr":"172.21.15.162:6819","nonce":3585957098}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6822","nonce":3585957098},{"type":"v1","addr":"172.21.15.162:6823","nonce":3585957098}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6820","nonce":3585957098},{"type":"v1","addr":"172.21.15.162:6821","nonce":3585957098}]},"public_addr":"172.21.15.162:6817/3585957098","cluster_addr":"172.21.15.162:6819/3585957098","heartbeat_back_addr":"172.21.15.162:6823/3585957098","heartbeat_front_addr":"172.21.15.162:6821/3585957098","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:02.667962+0000","dead_epoch":726},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:26.166590+0000","dead_epoch":758},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:50.153794+0000","dead_epoch":791},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:13.556156+0000","dead_epoch":824},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:34.827805+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:58.789701+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:22.650398+0000","dead_epoch":1273},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:46.166424+0000","dead_epoch":1359}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.7","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.63:6801/4052036587":"2024-08-18T16:59:45.829424+0000","172.21.15.63:6800/4052036587":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/753392608":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/2384509117":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/664643590":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/3746443558":"2024-08-18T16:56:38.413263+0000","172.21.15.63:6800/987519051":"2024-08-18T16:56:38.413263+0000","172.21.15.162:0/810332452":"2024-08-18T16:57:08.874093+0000","172.21.15.162:0/1341643866":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/2696979598":"2024-08-18T16:56:38.413263+0000","172.21.15.162:0/319999013":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/2404156937":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/84364348":"2024-08-18T16:47:16.384724+0000","172.21.15.162:0/4017736715":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/3589494813":"2024-08-18T16:56:38.413263+0000","172.21.15.63:6800/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:6801/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.63:6800/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.162:6834/3390906977":"2024-08-18T16:57:08.874093+0000","172.21.15.63:6801/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.162:6835/3390906977":"2024-08-18T16:57:08.874093+0000","172.21.15.63:6800/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.63:6801/987519051":"2024-08-18T16:56:38.413263+0000","172.21.15.63:0/955244129":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/1986138219":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/4067771896":"2024-08-18T16:56:38.413263+0000","172.21.15.63:6801/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:0/3226753153":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3004218255":"2024-08-18T16:47:16.384724+0000","172.21.15.162:0/2317499213":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/2601197458":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/3540337558":"2024-08-18T16:47:16.384724+0000","172.21.15.162:0/2288037041":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/1792393387":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/1256300214":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/701508812":"2024-08-18T16:52:20.398501+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:42:47.329 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:47 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:47.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:47 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:42:47.940 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-08-17T17:42:47.940 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd dump --format=json 2024-08-17T17:42:48.153 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:48.275 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:48 smithi063 ceph-mon[114756]: pgmap v2392: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:48.276 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:48 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1811485386' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:42:48.276 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:48 smithi063 ceph-mon[109843]: pgmap v2392: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:48.276 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:48 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1811485386' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:42:48.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:48 smithi162 ceph-mon[104668]: pgmap v2392: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:48.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:48 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1811485386' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:42:50.050 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:42:50.050 INFO:teuthology.orchestra.run.smithi063.stdout:{"epoch":1466,"fsid":"321fa59e-5cb8-11ef-bccf-c7b262605968","created":"2024-08-17T16:46:54.156006+0000","modified":"2024-08-17T17:42:44.075934+0000","last_up_change":"2024-08-17T17:35:54.172906+0000","last_in_change":"2024-08-17T17:35:13.821379+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":24,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":12,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-08-17T16:49:54.431240+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1444","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"1223","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-08-17T16:52:22.131465+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-08-17T16:53:35.328125+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-08-17T16:53:36.292966+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"c7b8bb07-bffa-46c3-a813-fcaeb0d58bd2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":726,"up_from":739,"up_thru":1448,"down_at":727,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6802","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6803","nonce":4285380350}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6804","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6805","nonce":4285380350}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6808","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6809","nonce":4285380350}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6806","nonce":4285380350},{"type":"v1","addr":"172.21.15.63:6807","nonce":4285380350}]},"public_addr":"172.21.15.63:6803/4285380350","cluster_addr":"172.21.15.63:6805/4285380350","heartbeat_back_addr":"172.21.15.63:6809/4285380350","heartbeat_front_addr":"172.21.15.63:6807/4285380350","state":["exists","up"]},{"osd":1,"uuid":"ae346e99-4706-4871-990a-f8e737572245","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":758,"up_from":771,"up_thru":1451,"down_at":759,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6810","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6811","nonce":3170762701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6812","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6813","nonce":3170762701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6816","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6817","nonce":3170762701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6814","nonce":3170762701},{"type":"v1","addr":"172.21.15.63:6815","nonce":3170762701}]},"public_addr":"172.21.15.63:6811/3170762701","cluster_addr":"172.21.15.63:6813/3170762701","heartbeat_back_addr":"172.21.15.63:6817/3170762701","heartbeat_front_addr":"172.21.15.63:6815/3170762701","state":["exists","up"]},{"osd":2,"uuid":"2d85d75f-1280-443f-857d-d322bd60c451","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":791,"up_from":804,"up_thru":1445,"down_at":792,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6818","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6819","nonce":3903658295}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6820","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6821","nonce":3903658295}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6824","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6825","nonce":3903658295}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6822","nonce":3903658295},{"type":"v1","addr":"172.21.15.63:6823","nonce":3903658295}]},"public_addr":"172.21.15.63:6819/3903658295","cluster_addr":"172.21.15.63:6821/3903658295","heartbeat_back_addr":"172.21.15.63:6825/3903658295","heartbeat_front_addr":"172.21.15.63:6823/3903658295","state":["exists","up"]},{"osd":3,"uuid":"771f43d2-e98b-43e0-921b-88e56fadda42","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":824,"up_from":837,"up_thru":1456,"down_at":825,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6826","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6827","nonce":1764356824}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6828","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6829","nonce":1764356824}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6832","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6833","nonce":1764356824}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.63:6830","nonce":1764356824},{"type":"v1","addr":"172.21.15.63:6831","nonce":1764356824}]},"public_addr":"172.21.15.63:6827/1764356824","cluster_addr":"172.21.15.63:6829/1764356824","heartbeat_back_addr":"172.21.15.63:6833/1764356824","heartbeat_front_addr":"172.21.15.63:6831/1764356824","state":["exists","up"]},{"osd":4,"uuid":"73c3c545-e975-4a6f-a1cb-e4575ee7a9b0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":1436,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6800","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6801","nonce":1563885208}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6802","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6803","nonce":1563885208}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6806","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6807","nonce":1563885208}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6804","nonce":1563885208},{"type":"v1","addr":"172.21.15.162:6805","nonce":1563885208}]},"public_addr":"172.21.15.162:6801/1563885208","cluster_addr":"172.21.15.162:6803/1563885208","heartbeat_back_addr":"172.21.15.162:6807/1563885208","heartbeat_front_addr":"172.21.15.162:6805/1563885208","state":["exists","up"]},{"osd":5,"uuid":"fae3ac52-3194-4eb4-9510-e3899768353a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":1445,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6808","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6809","nonce":1826502179}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6810","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6811","nonce":1826502179}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6814","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6815","nonce":1826502179}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6812","nonce":1826502179},{"type":"v1","addr":"172.21.15.162:6813","nonce":1826502179}]},"public_addr":"172.21.15.162:6809/1826502179","cluster_addr":"172.21.15.162:6811/1826502179","heartbeat_back_addr":"172.21.15.162:6815/1826502179","heartbeat_front_addr":"172.21.15.162:6813/1826502179","state":["exists","up"]},{"osd":6,"uuid":"a8255b37-1be5-4cd4-9b86-889f6bf52902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":47,"last_clean_end":1273,"up_from":1376,"up_thru":1436,"down_at":1274,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6824","nonce":1307501458},{"type":"v1","addr":"172.21.15.162:6825","nonce":1307501458}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6826","nonce":1307501458},{"type":"v1","addr":"172.21.15.162:6827","nonce":1307501458}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6830","nonce":1307501458},{"type":"v1","addr":"172.21.15.162:6831","nonce":1307501458}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6828","nonce":1307501458},{"type":"v1","addr":"172.21.15.162:6829","nonce":1307501458}]},"public_addr":"172.21.15.162:6825/1307501458","cluster_addr":"172.21.15.162:6827/1307501458","heartbeat_back_addr":"172.21.15.162:6831/1307501458","heartbeat_front_addr":"172.21.15.162:6829/1307501458","state":["exists","up"]},{"osd":7,"uuid":"3d0bcd6d-171d-4d90-a033-ed17714a9d96","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":55,"last_clean_end":1359,"up_from":1371,"up_thru":1463,"down_at":1360,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6816","nonce":3585957098},{"type":"v1","addr":"172.21.15.162:6817","nonce":3585957098}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6818","nonce":3585957098},{"type":"v1","addr":"172.21.15.162:6819","nonce":3585957098}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6822","nonce":3585957098},{"type":"v1","addr":"172.21.15.162:6823","nonce":3585957098}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.162:6820","nonce":3585957098},{"type":"v1","addr":"172.21.15.162:6821","nonce":3585957098}]},"public_addr":"172.21.15.162:6817/3585957098","cluster_addr":"172.21.15.162:6819/3585957098","heartbeat_back_addr":"172.21.15.162:6823/3585957098","heartbeat_front_addr":"172.21.15.162:6821/3585957098","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:02.667962+0000","dead_epoch":726},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:26.166590+0000","dead_epoch":758},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:49:50.153794+0000","dead_epoch":791},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:13.556156+0000","dead_epoch":824},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:34.827805+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:50:58.789701+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:22.650398+0000","dead_epoch":1273},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-08-17T16:51:46.166424+0000","dead_epoch":1359}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.7","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.63:6801/4052036587":"2024-08-18T16:59:45.829424+0000","172.21.15.63:6800/4052036587":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/753392608":"2024-08-18T16:59:45.829424+0000","172.21.15.63:0/701508812":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/2384509117":"2024-08-18T16:59:45.829424+0000","172.21.15.63:6800/987519051":"2024-08-18T16:56:38.413263+0000","172.21.15.162:0/810332452":"2024-08-18T16:57:08.874093+0000","172.21.15.162:0/1341643866":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/3746443558":"2024-08-18T16:56:38.413263+0000","172.21.15.63:0/3589494813":"2024-08-18T16:56:38.413263+0000","172.21.15.63:6800/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:0/2696979598":"2024-08-18T16:56:38.413263+0000","172.21.15.162:0/319999013":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/84364348":"2024-08-18T16:47:16.384724+0000","172.21.15.162:0/4017736715":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/2404156937":"2024-08-18T16:59:45.829424+0000","172.21.15.63:6801/987519051":"2024-08-18T16:56:38.413263+0000","172.21.15.63:0/955244129":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/664643590":"2024-08-18T16:59:45.829424+0000","172.21.15.63:6800/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.63:6801/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.63:6800/2553533530":"2024-08-18T16:47:39.919472+0000","172.21.15.162:6834/3390906977":"2024-08-18T16:57:08.874093+0000","172.21.15.63:6801/83870469":"2024-08-18T16:52:20.398501+0000","172.21.15.162:6835/3390906977":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/1986138219":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/4067771896":"2024-08-18T16:56:38.413263+0000","172.21.15.63:6801/3322978932":"2024-08-18T16:47:16.384724+0000","172.21.15.63:0/1256300214":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/1792393387":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3226753153":"2024-08-18T16:47:39.919472+0000","172.21.15.63:0/3004218255":"2024-08-18T16:47:16.384724+0000","172.21.15.162:0/2317499213":"2024-08-18T16:57:08.874093+0000","172.21.15.63:0/2601197458":"2024-08-18T16:52:20.398501+0000","172.21.15.63:0/3540337558":"2024-08-18T16:47:16.384724+0000","172.21.15.162:0/2288037041":"2024-08-18T16:57:08.874093+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":12,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-08-17T17:42:50.363 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:50 smithi063 ceph-mon[109843]: pgmap v2393: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:50.363 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:50 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3223716131' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:42:50.364 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:50 smithi063 ceph-mon[114756]: pgmap v2393: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:50.364 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:50 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3223716131' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:42:50.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:50 smithi162 ceph-mon[104668]: pgmap v2393: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:50.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:50 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3223716131' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-08-17T17:42:50.715 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-08-17T17:42:50.716 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-08-17T17:42:50.716 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-08-17T17:42:50.716 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-08-17T17:42:50.717 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-08-17T17:42:50.717 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-08-17T17:42:50.717 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-08-17T17:42:50.718 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-08-17T17:42:51.202 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:51.202 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:51.204 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:51.204 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:51.204 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:51.209 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:51.209 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:51.211 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:42:52.231 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:52 smithi063 ceph-mon[114756]: pgmap v2394: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:52.231 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:52 smithi063 ceph-mon[109843]: pgmap v2394: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:52.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:52 smithi162 ceph-mon[104668]: pgmap v2394: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:53.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:42:53 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:42:53] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.51.0" 2024-08-17T17:42:54.479 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:54 smithi063 ceph-mon[109843]: pgmap v2395: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:54.479 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:54 smithi063 ceph-mon[114756]: pgmap v2395: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:54.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:54 smithi162 ceph-mon[104668]: pgmap v2395: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:56.158 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:56 smithi063 ceph-mon[114756]: pgmap v2396: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:56.158 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:56 smithi063 ceph-mon[109843]: pgmap v2396: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:56.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:56 smithi162 ceph-mon[104668]: pgmap v2396: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:58.357 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:58 smithi063 ceph-mon[109843]: pgmap v2397: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:58.357 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:58 smithi063 ceph-mon[114756]: pgmap v2397: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:58.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:58 smithi162 ceph-mon[104668]: pgmap v2397: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:59.523 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:42:59 smithi063 ceph-mon[114756]: pgmap v2398: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:59.523 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:42:59 smithi063 ceph-mon[109843]: pgmap v2398: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:42:59.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:42:59 smithi162 ceph-mon[104668]: pgmap v2398: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:00.790 INFO:teuthology.orchestra.run.smithi063.stdout:3594887627177 2024-08-17T17:43:00.791 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-08-17T17:43:01.525 INFO:teuthology.orchestra.run.smithi063.stdout:3453153706416 2024-08-17T17:43:01.526 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-08-17T17:43:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:01 smithi063 ceph-mon[114756]: pgmap v2399: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:02.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:01 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:01 smithi063 ceph-mon[109843]: pgmap v2399: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:02.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:01 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:02.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:01 smithi162 ceph-mon[104668]: pgmap v2399: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:02.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:01 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:03.328 INFO:teuthology.orchestra.run.smithi063.stdout:167503725171 2024-08-17T17:43:03.329 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-08-17T17:43:03.529 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:03 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:43:03] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.51.0" 2024-08-17T17:43:04.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:03 smithi063 ceph-mon[114756]: pgmap v2400: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:04.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:03 smithi063 ceph-mon[109843]: pgmap v2400: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:04.086 INFO:teuthology.orchestra.run.smithi063.stdout:5888400162908 2024-08-17T17:43:04.086 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-08-17T17:43:04.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:03 smithi162 ceph-mon[104668]: pgmap v2400: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:05.492 INFO:teuthology.orchestra.run.smithi063.stdout:3311419785656 2024-08-17T17:43:05.493 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-08-17T17:43:06.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:05 smithi063 ceph-mon[114756]: pgmap v2401: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:06.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:05 smithi063 ceph-mon[109843]: pgmap v2401: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:06.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:05 smithi162 ceph-mon[104668]: pgmap v2401: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:06.506 INFO:teuthology.orchestra.run.smithi063.stdout:3173980832190 2024-08-17T17:43:06.506 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-08-17T17:43:06.518 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:07.089 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:07.089 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:07.089 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:08.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:07 smithi063 ceph-mon[114756]: pgmap v2402: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:08.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:07 smithi063 ceph-mon[109843]: pgmap v2402: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:08.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:07 smithi162 ceph-mon[104668]: pgmap v2402: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:09.661 INFO:teuthology.orchestra.run.smithi063.stdout:133143986811 2024-08-17T17:43:09.662 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-08-17T17:43:09.944 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:10.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:09 smithi063 ceph-mon[114756]: pgmap v2403: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:10.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:09 smithi063 ceph-mon[109843]: pgmap v2403: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:10.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:09 smithi162 ceph-mon[104668]: pgmap v2403: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:12.035 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:11 smithi063 ceph-mon[109843]: pgmap v2404: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:12.035 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:11 smithi063 ceph-mon[114756]: pgmap v2404: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:12.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:11 smithi162 ceph-mon[104668]: pgmap v2404: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:13.391 INFO:teuthology.orchestra.run.smithi063.stdout:3173980832192 2024-08-17T17:43:13.510 INFO:teuthology.orchestra.run.smithi063.stdout:167503725175 2024-08-17T17:43:13.510 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:13 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:43:13] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.51.0" 2024-08-17T17:43:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:13 smithi162 ceph-mon[104668]: pgmap v2405: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:14.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:13 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1348919221' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:43:14.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:13 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2401775934' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:43:14.243 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:13 smithi063 ceph-mon[114756]: pgmap v2405: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:14.243 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:13 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1348919221' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:43:14.243 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:13 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2401775934' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:43:14.243 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:13 smithi063 ceph-mon[109843]: pgmap v2405: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:14.244 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:13 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1348919221' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-08-17T17:43:14.244 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:13 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2401775934' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-08-17T17:43:14.266 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503725171 got 167503725175 for osd.5 2024-08-17T17:43:14.266 DEBUG:teuthology.parallel:result is None 2024-08-17T17:43:14.782 INFO:teuthology.orchestra.run.smithi063.stdout:5909874999382 2024-08-17T17:43:14.782 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-08-17T17:43:15.234 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3173980832190 got 3173980832192 for osd.0 2024-08-17T17:43:15.234 DEBUG:teuthology.parallel:result is None 2024-08-17T17:43:16.018 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:16.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:15 smithi063 ceph-mon[114756]: pgmap v2406: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:16.030 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:15 smithi063 ceph-mon[109843]: pgmap v2406: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:16.055 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:16.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:15 smithi162 ceph-mon[104668]: pgmap v2406: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:17.024 INFO:teuthology.orchestra.run.smithi063.stdout:5888400162910 2024-08-17T17:43:17.057 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:16 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:17.057 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:16 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:17.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:16 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:17.340 INFO:teuthology.orchestra.run.smithi063.stdout:3453153706420 2024-08-17T17:43:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:17 smithi063 ceph-mon[109843]: pgmap v2407: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:17 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3243672195' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:43:18.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:17 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/1377922860' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:43:18.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:17 smithi063 ceph-mon[114756]: pgmap v2407: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:18.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:17 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3243672195' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:43:18.030 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:17 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/1377922860' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:43:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:17 smithi162 ceph-mon[104668]: pgmap v2407: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:18.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:17 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3243672195' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-08-17T17:43:18.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:17 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/1377922860' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-08-17T17:43:18.785 INFO:tasks.cephadm.ceph_manager.ceph:need seq 5888400162908 got 5888400162910 for osd.7 2024-08-17T17:43:18.786 DEBUG:teuthology.parallel:result is None 2024-08-17T17:43:20.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:19 smithi063 ceph-mon[114756]: pgmap v2408: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:20.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:19 smithi063 ceph-mon[109843]: pgmap v2408: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:20.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:19 smithi162 ceph-mon[104668]: pgmap v2408: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:21.585 INFO:teuthology.orchestra.run.smithi063.stdout:3594887627182 2024-08-17T17:43:21.658 INFO:teuthology.orchestra.run.smithi063.stdout:133143986813 2024-08-17T17:43:21.948 INFO:teuthology.orchestra.run.smithi063.stdout:3311419785659 2024-08-17T17:43:21.948 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:21 smithi063 ceph-mon[114756]: pgmap v2409: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:21.948 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:21 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/2695447666' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:43:21.948 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:21 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3931769685' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:43:21.949 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:21 smithi063 ceph-mon[109843]: pgmap v2409: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:21.949 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:21 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/2695447666' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:43:21.949 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:21 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3931769685' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:43:22.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:21 smithi162 ceph-mon[104668]: pgmap v2409: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:21 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/2695447666' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-08-17T17:43:22.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:21 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3931769685' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-08-17T17:43:22.708 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3311419785656 got 3311419785659 for osd.1 2024-08-17T17:43:22.708 DEBUG:teuthology.parallel:result is None 2024-08-17T17:43:23.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:22 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/3167808845' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:43:23.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:22 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/3167808845' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:43:23.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:22 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/3167808845' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-08-17T17:43:23.361 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3453153706416 got 3453153706420 for osd.2 2024-08-17T17:43:23.361 DEBUG:teuthology.parallel:result is None 2024-08-17T17:43:23.744 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:23 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:43:23] "GET /metrics HTTP/1.1" 200 34110 "" "Prometheus/2.51.0" 2024-08-17T17:43:23.773 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986811 got 133143986813 for osd.4 2024-08-17T17:43:23.773 DEBUG:teuthology.parallel:result is None 2024-08-17T17:43:24.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:23 smithi063 ceph-mon[109843]: pgmap v2410: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:24.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:23 smithi063 ceph-mon[114756]: pgmap v2410: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:24.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:23 smithi162 ceph-mon[104668]: pgmap v2410: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:24.190 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:24.211 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3594887627177 got 3594887627182 for osd.3 2024-08-17T17:43:24.212 DEBUG:teuthology.parallel:result is None 2024-08-17T17:43:26.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:25 smithi063 ceph-mon[114756]: pgmap v2411: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:26.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:25 smithi063 ceph-mon[109843]: pgmap v2411: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:26.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:25 smithi162 ceph-mon[104668]: pgmap v2411: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:26.187 INFO:teuthology.orchestra.run.smithi063.stdout:5909874999386 2024-08-17T17:43:26.954 INFO:tasks.cephadm.ceph_manager.ceph:need seq 5909874999382 got 5909874999386 for osd.6 2024-08-17T17:43:26.954 DEBUG:teuthology.parallel:result is None 2024-08-17T17:43:26.955 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-08-17T17:43:26.955 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:43:27.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:26 smithi162 ceph-mon[104668]: from='client.? 172.21.15.63:0/4182727356' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:43:27.170 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:27.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:26 smithi063 ceph-mon[109843]: from='client.? 172.21.15.63:0/4182727356' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:43:27.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:26 smithi063 ceph-mon[114756]: from='client.? 172.21.15.63:0/4182727356' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-08-17T17:43:28.029 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:27 smithi063 ceph-mon[114756]: pgmap v2412: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:28.029 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:27 smithi063 ceph-mon[109843]: pgmap v2412: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:28.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:27 smithi162 ceph-mon[104668]: pgmap v2412: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:29.297 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:43:29.299 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:43:29.932 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:29 smithi063 ceph-mon[109843]: pgmap v2413: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:29.932 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:29 smithi063 ceph-mon[114756]: pgmap v2413: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:29.955 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2413,"stamp":"2024-08-17T17:43:28.726755+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":526059,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4678,"ondisk_log_size":4678,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6205348,"kb_used_data":65476,"kb_used_omap":298,"kb_used_meta":6139477,"kb_avail":743755868,"statfs":{"total":767960285184,"available":761606008832,"internally_reserved":0,"allocated":67047424,"data_stored":44224237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":6286824977},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003693"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":141,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083842+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:42:44.083842+0000","last_peered":"2024-08-17T17:42:44.083842+0000","last_clean":"2024-08-17T17:42:44.083842+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:42:44.083842+0000","last_undegraded":"2024-08-17T17:42:44.083842+0000","last_fullsized":"2024-08-17T17:42:44.083842+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1482,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.144825+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:42:41.144825+0000","last_peered":"2024-08-17T17:42:41.144825+0000","last_clean":"2024-08-17T17:42:41.144825+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:42:41.144825+0000","last_undegraded":"2024-08-17T17:42:41.144825+0000","last_fullsized":"2024-08-17T17:42:41.144825+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1625,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083987+0000","last_change":"2024-08-17T17:42:06.241404+0000","last_active":"2024-08-17T17:42:44.083987+0000","last_peered":"2024-08-17T17:42:44.083987+0000","last_clean":"2024-08-17T17:42:44.083987+0000","last_became_active":"2024-08-17T17:42:06.241035+0000","last_became_peered":"2024-08-17T17:42:06.241035+0000","last_unstale":"2024-08-17T17:42:44.083987+0000","last_undegraded":"2024-08-17T17:42:44.083987+0000","last_fullsized":"2024-08-17T17:42:44.083987+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:47.523505+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":68,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108696+0000","last_change":"2024-08-17T17:42:18.853159+0000","last_active":"2024-08-17T17:42:44.108696+0000","last_peered":"2024-08-17T17:42:44.108696+0000","last_clean":"2024-08-17T17:42:44.108696+0000","last_became_active":"2024-08-17T17:42:18.852607+0000","last_became_peered":"2024-08-17T17:42:18.852607+0000","last_unstale":"2024-08-17T17:42:44.108696+0000","last_undegraded":"2024-08-17T17:42:44.108696+0000","last_fullsized":"2024-08-17T17:42:44.108696+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:31.425280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":77,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108217+0000","last_change":"2024-08-17T17:42:18.854135+0000","last_active":"2024-08-17T17:42:44.108217+0000","last_peered":"2024-08-17T17:42:44.108217+0000","last_clean":"2024-08-17T17:42:44.108217+0000","last_became_active":"2024-08-17T17:42:18.853580+0000","last_became_peered":"2024-08-17T17:42:18.853580+0000","last_unstale":"2024-08-17T17:42:44.108217+0000","last_undegraded":"2024-08-17T17:42:44.108217+0000","last_fullsized":"2024-08-17T17:42:44.108217+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:50.774568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1759,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323227+0000","last_change":"2024-08-17T17:42:40.150301+0000","last_active":"2024-08-17T17:42:41.323227+0000","last_peered":"2024-08-17T17:42:41.323227+0000","last_clean":"2024-08-17T17:42:41.323227+0000","last_became_active":"2024-08-17T17:42:36.908990+0000","last_became_peered":"2024-08-17T17:42:36.908990+0000","last_unstale":"2024-08-17T17:42:41.323227+0000","last_undegraded":"2024-08-17T17:42:41.323227+0000","last_fullsized":"2024-08-17T17:42:41.323227+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:42:40.150181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:42:40.150181+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:15:11.864462+0000","scrub_duration":0.0018593699999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1415,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323900+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:42:41.323900+0000","last_peered":"2024-08-17T17:42:41.323900+0000","last_clean":"2024-08-17T17:42:41.323900+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:42:41.323900+0000","last_undegraded":"2024-08-17T17:42:41.323900+0000","last_fullsized":"2024-08-17T17:42:41.323900+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":91,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.144901+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:42:41.144901+0000","last_peered":"2024-08-17T17:42:41.144901+0000","last_clean":"2024-08-17T17:42:41.144901+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:42:41.144901+0000","last_undegraded":"2024-08-17T17:42:41.144901+0000","last_fullsized":"2024-08-17T17:42:41.144901+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1460,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323882+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:42:41.323882+0000","last_peered":"2024-08-17T17:42:41.323882+0000","last_clean":"2024-08-17T17:42:41.323882+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:42:41.323882+0000","last_undegraded":"2024-08-17T17:42:41.323882+0000","last_fullsized":"2024-08-17T17:42:41.323882+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1468,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.145218+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:42:41.145218+0000","last_peered":"2024-08-17T17:42:41.145218+0000","last_clean":"2024-08-17T17:42:41.145218+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:42:41.145218+0000","last_undegraded":"2024-08-17T17:42:41.145218+0000","last_fullsized":"2024-08-17T17:42:41.145218+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":689,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108821+0000","last_change":"2024-08-17T17:42:18.853578+0000","last_active":"2024-08-17T17:42:44.108821+0000","last_peered":"2024-08-17T17:42:44.108821+0000","last_clean":"2024-08-17T17:42:44.108821+0000","last_became_active":"2024-08-17T17:42:18.853274+0000","last_became_peered":"2024-08-17T17:42:18.853274+0000","last_unstale":"2024-08-17T17:42:44.108821+0000","last_undegraded":"2024-08-17T17:42:44.108821+0000","last_fullsized":"2024-08-17T17:42:44.108821+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:03:20.069149+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":134,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083239+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:42:44.083239+0000","last_peered":"2024-08-17T17:42:44.083239+0000","last_clean":"2024-08-17T17:42:44.083239+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:42:44.083239+0000","last_undegraded":"2024-08-17T17:42:44.083239+0000","last_fullsized":"2024-08-17T17:42:44.083239+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":225,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.370425+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:42:44.370425+0000","last_peered":"2024-08-17T17:42:44.370425+0000","last_clean":"2024-08-17T17:42:44.370425+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:42:44.370425+0000","last_undegraded":"2024-08-17T17:42:44.370425+0000","last_fullsized":"2024-08-17T17:42:44.370425+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1485,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108313+0000","last_change":"2024-08-17T17:42:18.851661+0000","last_active":"2024-08-17T17:42:44.108313+0000","last_peered":"2024-08-17T17:42:44.108313+0000","last_clean":"2024-08-17T17:42:44.108313+0000","last_became_active":"2024-08-17T17:42:18.851310+0000","last_became_peered":"2024-08-17T17:42:18.851310+0000","last_unstale":"2024-08-17T17:42:44.108313+0000","last_undegraded":"2024-08-17T17:42:44.108313+0000","last_fullsized":"2024-08-17T17:42:44.108313+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:06.851954+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1488,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.145136+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:42:41.145136+0000","last_peered":"2024-08-17T17:42:41.145136+0000","last_clean":"2024-08-17T17:42:41.145136+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:42:41.145136+0000","last_undegraded":"2024-08-17T17:42:41.145136+0000","last_fullsized":"2024-08-17T17:42:41.145136+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1834,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108570+0000","last_change":"2024-08-17T17:42:18.996230+0000","last_active":"2024-08-17T17:42:44.108570+0000","last_peered":"2024-08-17T17:42:44.108570+0000","last_clean":"2024-08-17T17:42:44.108570+0000","last_became_active":"2024-08-17T17:42:18.995147+0000","last_became_peered":"2024-08-17T17:42:18.995147+0000","last_unstale":"2024-08-17T17:42:44.108570+0000","last_undegraded":"2024-08-17T17:42:44.108570+0000","last_fullsized":"2024-08-17T17:42:44.108570+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:51:33.826500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1843,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322699+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:42:41.322699+0000","last_peered":"2024-08-17T17:42:41.322699+0000","last_clean":"2024-08-17T17:42:41.322699+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:42:41.322699+0000","last_undegraded":"2024-08-17T17:42:41.322699+0000","last_fullsized":"2024-08-17T17:42:41.322699+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1469,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.082843+0000","last_change":"2024-08-17T17:42:06.239740+0000","last_active":"2024-08-17T17:42:44.082843+0000","last_peered":"2024-08-17T17:42:44.082843+0000","last_clean":"2024-08-17T17:42:44.082843+0000","last_became_active":"2024-08-17T17:42:06.239413+0000","last_became_peered":"2024-08-17T17:42:06.239413+0000","last_unstale":"2024-08-17T17:42:44.082843+0000","last_undegraded":"2024-08-17T17:42:44.082843+0000","last_fullsized":"2024-08-17T17:42:44.082843+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:33.067653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1227,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083691+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:42:44.083691+0000","last_peered":"2024-08-17T17:42:44.083691+0000","last_clean":"2024-08-17T17:42:44.083691+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:42:44.083691+0000","last_undegraded":"2024-08-17T17:42:44.083691+0000","last_fullsized":"2024-08-17T17:42:44.083691+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":1010,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323079+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:42:41.323079+0000","last_peered":"2024-08-17T17:42:41.323079+0000","last_clean":"2024-08-17T17:42:41.323079+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:42:41.323079+0000","last_undegraded":"2024-08-17T17:42:41.323079+0000","last_fullsized":"2024-08-17T17:42:41.323079+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":82,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108148+0000","last_change":"2024-08-17T17:42:18.995816+0000","last_active":"2024-08-17T17:42:44.108148+0000","last_peered":"2024-08-17T17:42:44.108148+0000","last_clean":"2024-08-17T17:42:44.108148+0000","last_became_active":"2024-08-17T17:42:18.995318+0000","last_became_peered":"2024-08-17T17:42:18.995318+0000","last_unstale":"2024-08-17T17:42:44.108148+0000","last_undegraded":"2024-08-17T17:42:44.108148+0000","last_fullsized":"2024-08-17T17:42:44.108148+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:34:38.636971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1323,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.080619+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:42:41.080619+0000","last_peered":"2024-08-17T17:42:41.080619+0000","last_clean":"2024-08-17T17:42:41.080619+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:42:41.080619+0000","last_undegraded":"2024-08-17T17:42:41.080619+0000","last_fullsized":"2024-08-17T17:42:41.080619+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1700,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.144903+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:42:41.144903+0000","last_peered":"2024-08-17T17:42:41.144903+0000","last_clean":"2024-08-17T17:42:41.144903+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:42:41.144903+0000","last_undegraded":"2024-08-17T17:42:41.144903+0000","last_fullsized":"2024-08-17T17:42:41.144903+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":181,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108326+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:42:44.108326+0000","last_peered":"2024-08-17T17:42:44.108326+0000","last_clean":"2024-08-17T17:42:44.108326+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:42:44.108326+0000","last_undegraded":"2024-08-17T17:42:44.108326+0000","last_fullsized":"2024-08-17T17:42:44.108326+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1249,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083993+0000","last_change":"2024-08-17T17:42:00.208934+0000","last_active":"2024-08-17T17:42:44.083993+0000","last_peered":"2024-08-17T17:42:44.083993+0000","last_clean":"2024-08-17T17:42:44.083993+0000","last_became_active":"2024-08-17T17:42:00.208653+0000","last_became_peered":"2024-08-17T17:42:00.208653+0000","last_unstale":"2024-08-17T17:42:44.083993+0000","last_undegraded":"2024-08-17T17:42:44.083993+0000","last_fullsized":"2024-08-17T17:42:44.083993+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:18.190351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1651,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.605924+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:42:44.605924+0000","last_peered":"2024-08-17T17:42:44.605924+0000","last_clean":"2024-08-17T17:42:44.605924+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:42:44.605924+0000","last_undegraded":"2024-08-17T17:42:44.605924+0000","last_fullsized":"2024-08-17T17:42:44.605924+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1807,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108565+0000","last_change":"2024-08-17T17:42:18.853877+0000","last_active":"2024-08-17T17:42:44.108565+0000","last_peered":"2024-08-17T17:42:44.108565+0000","last_clean":"2024-08-17T17:42:44.108565+0000","last_became_active":"2024-08-17T17:42:18.853536+0000","last_became_peered":"2024-08-17T17:42:18.853536+0000","last_unstale":"2024-08-17T17:42:44.108565+0000","last_undegraded":"2024-08-17T17:42:44.108565+0000","last_fullsized":"2024-08-17T17:42:44.108565+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:35:59.569437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1474,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323586+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:42:41.323586+0000","last_peered":"2024-08-17T17:42:41.323586+0000","last_clean":"2024-08-17T17:42:41.323586+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:42:41.323586+0000","last_undegraded":"2024-08-17T17:42:41.323586+0000","last_fullsized":"2024-08-17T17:42:41.323586+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1799,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.370279+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:42:44.370279+0000","last_peered":"2024-08-17T17:42:44.370279+0000","last_clean":"2024-08-17T17:42:44.370279+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:42:44.370279+0000","last_undegraded":"2024-08-17T17:42:44.370279+0000","last_fullsized":"2024-08-17T17:42:44.370279+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:11:06.218915+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1257,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323314+0000","last_change":"2024-08-17T17:42:36.911202+0000","last_active":"2024-08-17T17:42:41.323314+0000","last_peered":"2024-08-17T17:42:41.323314+0000","last_clean":"2024-08-17T17:42:41.323314+0000","last_became_active":"2024-08-17T17:42:36.910876+0000","last_became_peered":"2024-08-17T17:42:36.910876+0000","last_unstale":"2024-08-17T17:42:41.323314+0000","last_undegraded":"2024-08-17T17:42:41.323314+0000","last_fullsized":"2024-08-17T17:42:41.323314+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:56.962011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":991,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083134+0000","last_change":"2024-08-17T17:42:06.742409+0000","last_active":"2024-08-17T17:42:44.083134+0000","last_peered":"2024-08-17T17:42:44.083134+0000","last_clean":"2024-08-17T17:42:44.083134+0000","last_became_active":"2024-08-17T17:42:06.742185+0000","last_became_peered":"2024-08-17T17:42:06.742185+0000","last_unstale":"2024-08-17T17:42:44.083134+0000","last_undegraded":"2024-08-17T17:42:44.083134+0000","last_fullsized":"2024-08-17T17:42:44.083134+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:01.999846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":918,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323628+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:42:41.323628+0000","last_peered":"2024-08-17T17:42:41.323628+0000","last_clean":"2024-08-17T17:42:41.323628+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:42:41.323628+0000","last_undegraded":"2024-08-17T17:42:41.323628+0000","last_fullsized":"2024-08-17T17:42:41.323628+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1454,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.605723+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:42:44.605723+0000","last_peered":"2024-08-17T17:42:44.605723+0000","last_clean":"2024-08-17T17:42:44.605723+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:42:44.605723+0000","last_undegraded":"2024-08-17T17:42:44.605723+0000","last_fullsized":"2024-08-17T17:42:44.605723+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1473,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.604548+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:42:44.604548+0000","last_peered":"2024-08-17T17:42:44.604548+0000","last_clean":"2024-08-17T17:42:44.604548+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:42:44.604548+0000","last_undegraded":"2024-08-17T17:42:44.604548+0000","last_fullsized":"2024-08-17T17:42:44.604548+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":1022,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.084365+0000","last_change":"2024-08-17T17:42:06.239908+0000","last_active":"2024-08-17T17:42:44.084365+0000","last_peered":"2024-08-17T17:42:44.084365+0000","last_clean":"2024-08-17T17:42:44.084365+0000","last_became_active":"2024-08-17T17:42:06.239583+0000","last_became_peered":"2024-08-17T17:42:06.239583+0000","last_unstale":"2024-08-17T17:42:44.084365+0000","last_undegraded":"2024-08-17T17:42:44.084365+0000","last_fullsized":"2024-08-17T17:42:44.084365+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:42:02.664315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:42:02.664315+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:00.458998+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1421,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.144825+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:42:41.144825+0000","last_peered":"2024-08-17T17:42:41.144825+0000","last_clean":"2024-08-17T17:42:41.144825+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:42:41.144825+0000","last_undegraded":"2024-08-17T17:42:41.144825+0000","last_fullsized":"2024-08-17T17:42:41.144825+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1317,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.080553+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:42:41.080553+0000","last_peered":"2024-08-17T17:42:41.080553+0000","last_clean":"2024-08-17T17:42:41.080553+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:42:41.080553+0000","last_undegraded":"2024-08-17T17:42:41.080553+0000","last_fullsized":"2024-08-17T17:42:41.080553+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":413,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083746+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:42:44.083746+0000","last_peered":"2024-08-17T17:42:44.083746+0000","last_clean":"2024-08-17T17:42:44.083746+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:42:44.083746+0000","last_undegraded":"2024-08-17T17:42:44.083746+0000","last_fullsized":"2024-08-17T17:42:44.083746+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1682,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108555+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:42:44.108555+0000","last_peered":"2024-08-17T17:42:44.108555+0000","last_clean":"2024-08-17T17:42:44.108555+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:42:44.108555+0000","last_undegraded":"2024-08-17T17:42:44.108555+0000","last_fullsized":"2024-08-17T17:42:44.108555+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:42:36.477246+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":179,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108257+0000","last_change":"2024-08-17T17:42:18.853707+0000","last_active":"2024-08-17T17:42:44.108257+0000","last_peered":"2024-08-17T17:42:44.108257+0000","last_clean":"2024-08-17T17:42:44.108257+0000","last_became_active":"2024-08-17T17:42:18.853399+0000","last_became_peered":"2024-08-17T17:42:18.853399+0000","last_unstale":"2024-08-17T17:42:44.108257+0000","last_undegraded":"2024-08-17T17:42:44.108257+0000","last_fullsized":"2024-08-17T17:42:44.108257+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:10:06.230375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":903,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.084632+0000","last_change":"2024-08-17T17:42:06.240411+0000","last_active":"2024-08-17T17:42:44.084632+0000","last_peered":"2024-08-17T17:42:44.084632+0000","last_clean":"2024-08-17T17:42:44.084632+0000","last_became_active":"2024-08-17T17:42:06.240130+0000","last_became_peered":"2024-08-17T17:42:06.240130+0000","last_unstale":"2024-08-17T17:42:44.084632+0000","last_undegraded":"2024-08-17T17:42:44.084632+0000","last_fullsized":"2024-08-17T17:42:44.084632+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:24:27.224356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1718,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108882+0000","last_change":"2024-08-17T17:42:18.853632+0000","last_active":"2024-08-17T17:42:44.108882+0000","last_peered":"2024-08-17T17:42:44.108882+0000","last_clean":"2024-08-17T17:42:44.108882+0000","last_became_active":"2024-08-17T17:42:18.853044+0000","last_became_peered":"2024-08-17T17:42:18.853044+0000","last_unstale":"2024-08-17T17:42:44.108882+0000","last_undegraded":"2024-08-17T17:42:44.108882+0000","last_fullsized":"2024-08-17T17:42:44.108882+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:41.623573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1827,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.145585+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:42:41.145585+0000","last_peered":"2024-08-17T17:42:41.145585+0000","last_clean":"2024-08-17T17:42:41.145585+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:42:41.145585+0000","last_undegraded":"2024-08-17T17:42:41.145585+0000","last_fullsized":"2024-08-17T17:42:41.145585+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":980,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083450+0000","last_change":"2024-08-17T17:42:00.209617+0000","last_active":"2024-08-17T17:42:44.083450+0000","last_peered":"2024-08-17T17:42:44.083450+0000","last_clean":"2024-08-17T17:42:44.083450+0000","last_became_active":"2024-08-17T17:42:00.209381+0000","last_became_peered":"2024-08-17T17:42:00.209381+0000","last_unstale":"2024-08-17T17:42:44.083450+0000","last_undegraded":"2024-08-17T17:42:44.083450+0000","last_fullsized":"2024-08-17T17:42:44.083450+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:53.735666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1424,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323905+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:42:41.323905+0000","last_peered":"2024-08-17T17:42:41.323905+0000","last_clean":"2024-08-17T17:42:41.323905+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:42:41.323905+0000","last_undegraded":"2024-08-17T17:42:41.323905+0000","last_fullsized":"2024-08-17T17:42:41.323905+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1250,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083557+0000","last_change":"2024-08-17T17:42:06.742503+0000","last_active":"2024-08-17T17:42:44.083557+0000","last_peered":"2024-08-17T17:42:44.083557+0000","last_clean":"2024-08-17T17:42:44.083557+0000","last_became_active":"2024-08-17T17:42:06.742208+0000","last_became_peered":"2024-08-17T17:42:06.742208+0000","last_unstale":"2024-08-17T17:42:44.083557+0000","last_undegraded":"2024-08-17T17:42:44.083557+0000","last_fullsized":"2024-08-17T17:42:44.083557+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:02:35.824513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":993,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.080574+0000","last_change":"2024-08-17T17:41:06.387062+0000","last_active":"2024-08-17T17:42:41.080574+0000","last_peered":"2024-08-17T17:42:41.080574+0000","last_clean":"2024-08-17T17:42:41.080574+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:42:41.080574+0000","last_undegraded":"2024-08-17T17:42:41.080574+0000","last_fullsized":"2024-08-17T17:42:41.080574+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:41:06.386955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:41:06.386955+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:44:32.008526+0000","scrub_duration":0.0017975980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":52,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.370177+0000","last_change":"2024-08-17T17:41:49.104317+0000","last_active":"2024-08-17T17:42:44.370177+0000","last_peered":"2024-08-17T17:42:44.370177+0000","last_clean":"2024-08-17T17:42:44.370177+0000","last_became_active":"2024-08-17T17:41:49.103786+0000","last_became_peered":"2024-08-17T17:41:49.103786+0000","last_unstale":"2024-08-17T17:42:44.370177+0000","last_undegraded":"2024-08-17T17:42:44.370177+0000","last_fullsized":"2024-08-17T17:42:44.370177+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:41:18.708270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":975,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083692+0000","last_change":"2024-08-17T17:42:06.741570+0000","last_active":"2024-08-17T17:42:44.083692+0000","last_peered":"2024-08-17T17:42:44.083692+0000","last_clean":"2024-08-17T17:42:44.083692+0000","last_became_active":"2024-08-17T17:42:06.741231+0000","last_became_peered":"2024-08-17T17:42:06.741231+0000","last_unstale":"2024-08-17T17:42:44.083692+0000","last_undegraded":"2024-08-17T17:42:44.083692+0000","last_fullsized":"2024-08-17T17:42:44.083692+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:55.960941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1492,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323916+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:42:41.323916+0000","last_peered":"2024-08-17T17:42:41.323916+0000","last_clean":"2024-08-17T17:42:41.323916+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:42:41.323916+0000","last_undegraded":"2024-08-17T17:42:41.323916+0000","last_fullsized":"2024-08-17T17:42:41.323916+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":976,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.080396+0000","last_change":"2024-08-17T17:41:07.361112+0000","last_active":"2024-08-17T17:42:41.080396+0000","last_peered":"2024-08-17T17:42:41.080396+0000","last_clean":"2024-08-17T17:42:41.080396+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:42:41.080396+0000","last_undegraded":"2024-08-17T17:42:41.080396+0000","last_fullsized":"2024-08-17T17:42:41.080396+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:41:07.360968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:41:07.360968+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:28.377019+0000","scrub_duration":0.001649951,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":31,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323383+0000","last_change":"2024-08-17T17:41:48.789086+0000","last_active":"2024-08-17T17:42:41.323383+0000","last_peered":"2024-08-17T17:42:41.323383+0000","last_clean":"2024-08-17T17:42:41.323383+0000","last_became_active":"2024-08-17T17:41:48.788726+0000","last_became_peered":"2024-08-17T17:41:48.788726+0000","last_unstale":"2024-08-17T17:42:41.323383+0000","last_undegraded":"2024-08-17T17:42:41.323383+0000","last_fullsized":"2024-08-17T17:42:41.323383+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:53.243349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1455,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.145455+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:42:41.145455+0000","last_peered":"2024-08-17T17:42:41.145455+0000","last_clean":"2024-08-17T17:42:41.145455+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:42:41.145455+0000","last_undegraded":"2024-08-17T17:42:41.145455+0000","last_fullsized":"2024-08-17T17:42:41.145455+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1498,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323504+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:42:41.323504+0000","last_peered":"2024-08-17T17:42:41.323504+0000","last_clean":"2024-08-17T17:42:41.323504+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:42:41.323504+0000","last_undegraded":"2024-08-17T17:42:41.323504+0000","last_fullsized":"2024-08-17T17:42:41.323504+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1813,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322793+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:42:41.322793+0000","last_peered":"2024-08-17T17:42:41.322793+0000","last_clean":"2024-08-17T17:42:41.322793+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:42:41.322793+0000","last_undegraded":"2024-08-17T17:42:41.322793+0000","last_fullsized":"2024-08-17T17:42:41.322793+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":52,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.370513+0000","last_change":"2024-08-17T17:41:48.789557+0000","last_active":"2024-08-17T17:42:44.370513+0000","last_peered":"2024-08-17T17:42:44.370513+0000","last_clean":"2024-08-17T17:42:44.370513+0000","last_became_active":"2024-08-17T17:41:48.788842+0000","last_became_peered":"2024-08-17T17:41:48.788842+0000","last_unstale":"2024-08-17T17:42:44.370513+0000","last_undegraded":"2024-08-17T17:42:44.370513+0000","last_fullsized":"2024-08-17T17:42:44.370513+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:23:24.001239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1337,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.082867+0000","last_change":"2024-08-17T17:42:06.743142+0000","last_active":"2024-08-17T17:42:44.082867+0000","last_peered":"2024-08-17T17:42:44.082867+0000","last_clean":"2024-08-17T17:42:44.082867+0000","last_became_active":"2024-08-17T17:42:06.742778+0000","last_became_peered":"2024-08-17T17:42:06.742778+0000","last_unstale":"2024-08-17T17:42:44.082867+0000","last_undegraded":"2024-08-17T17:42:44.082867+0000","last_fullsized":"2024-08-17T17:42:44.082867+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:25.982072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2088,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.370240+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:42:44.370240+0000","last_peered":"2024-08-17T17:42:44.370240+0000","last_clean":"2024-08-17T17:42:44.370240+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:42:44.370240+0000","last_undegraded":"2024-08-17T17:42:44.370240+0000","last_fullsized":"2024-08-17T17:42:44.370240+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1691,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322980+0000","last_change":"2024-08-17T17:42:36.909560+0000","last_active":"2024-08-17T17:42:41.322980+0000","last_peered":"2024-08-17T17:42:41.322980+0000","last_clean":"2024-08-17T17:42:41.322980+0000","last_became_active":"2024-08-17T17:42:36.909238+0000","last_became_peered":"2024-08-17T17:42:36.909238+0000","last_unstale":"2024-08-17T17:42:41.322980+0000","last_undegraded":"2024-08-17T17:42:41.322980+0000","last_fullsized":"2024-08-17T17:42:41.322980+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:42:00.619334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:42:00.619334+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:48:10.944008+0000","scrub_duration":0.0068224740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1278,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083835+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:42:44.083835+0000","last_peered":"2024-08-17T17:42:44.083835+0000","last_clean":"2024-08-17T17:42:44.083835+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:42:44.083835+0000","last_undegraded":"2024-08-17T17:42:44.083835+0000","last_fullsized":"2024-08-17T17:42:44.083835+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1049,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083091+0000","last_change":"2024-08-17T17:42:00.208051+0000","last_active":"2024-08-17T17:42:44.083091+0000","last_peered":"2024-08-17T17:42:44.083091+0000","last_clean":"2024-08-17T17:42:44.083091+0000","last_became_active":"2024-08-17T17:42:00.207713+0000","last_became_peered":"2024-08-17T17:42:00.207713+0000","last_unstale":"2024-08-17T17:42:44.083091+0000","last_undegraded":"2024-08-17T17:42:44.083091+0000","last_fullsized":"2024-08-17T17:42:44.083091+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:38.887722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1430,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322850+0000","last_change":"2024-08-17T17:42:36.909226+0000","last_active":"2024-08-17T17:42:41.322850+0000","last_peered":"2024-08-17T17:42:41.322850+0000","last_clean":"2024-08-17T17:42:41.322850+0000","last_became_active":"2024-08-17T17:42:36.908947+0000","last_became_peered":"2024-08-17T17:42:36.908947+0000","last_unstale":"2024-08-17T17:42:41.322850+0000","last_undegraded":"2024-08-17T17:42:41.322850+0000","last_fullsized":"2024-08-17T17:42:41.322850+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:38:36.590213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":755,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083707+0000","last_change":"2024-08-17T17:42:07.631252+0000","last_active":"2024-08-17T17:42:44.083707+0000","last_peered":"2024-08-17T17:42:44.083707+0000","last_clean":"2024-08-17T17:42:44.083707+0000","last_became_active":"2024-08-17T17:42:06.239009+0000","last_became_peered":"2024-08-17T17:42:06.239009+0000","last_unstale":"2024-08-17T17:42:44.083707+0000","last_undegraded":"2024-08-17T17:42:44.083707+0000","last_fullsized":"2024-08-17T17:42:44.083707+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:42:07.631161+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:42:07.631161+0000","last_clean_scrub_stamp":"2024-08-17T17:42:07.631161+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:28.889605+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1296,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.080619+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:42:41.080619+0000","last_peered":"2024-08-17T17:42:41.080619+0000","last_clean":"2024-08-17T17:42:41.080619+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:42:41.080619+0000","last_undegraded":"2024-08-17T17:42:41.080619+0000","last_fullsized":"2024-08-17T17:42:41.080619+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1771,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108268+0000","last_change":"2024-08-17T17:42:18.850814+0000","last_active":"2024-08-17T17:42:44.108268+0000","last_peered":"2024-08-17T17:42:44.108268+0000","last_clean":"2024-08-17T17:42:44.108268+0000","last_became_active":"2024-08-17T17:42:18.850459+0000","last_became_peered":"2024-08-17T17:42:18.850459+0000","last_unstale":"2024-08-17T17:42:44.108268+0000","last_undegraded":"2024-08-17T17:42:44.108268+0000","last_fullsized":"2024-08-17T17:42:44.108268+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:21:16.138051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1474,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323296+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:42:41.323296+0000","last_peered":"2024-08-17T17:42:41.323296+0000","last_clean":"2024-08-17T17:42:41.323296+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:42:41.323296+0000","last_undegraded":"2024-08-17T17:42:41.323296+0000","last_fullsized":"2024-08-17T17:42:41.323296+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1677,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322810+0000","last_change":"2024-08-17T17:42:36.906286+0000","last_active":"2024-08-17T17:42:41.322810+0000","last_peered":"2024-08-17T17:42:41.322810+0000","last_clean":"2024-08-17T17:42:41.322810+0000","last_became_active":"2024-08-17T17:42:36.905888+0000","last_became_peered":"2024-08-17T17:42:36.905888+0000","last_unstale":"2024-08-17T17:42:41.322810+0000","last_undegraded":"2024-08-17T17:42:41.322810+0000","last_fullsized":"2024-08-17T17:42:41.322810+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:58.504387+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:40:58.504387+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:43:03.124798+0000","scrub_duration":0.017597167,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":954,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083610+0000","last_change":"2024-08-17T17:42:06.240562+0000","last_active":"2024-08-17T17:42:44.083610+0000","last_peered":"2024-08-17T17:42:44.083610+0000","last_clean":"2024-08-17T17:42:44.083610+0000","last_became_active":"2024-08-17T17:42:06.240265+0000","last_became_peered":"2024-08-17T17:42:06.240265+0000","last_unstale":"2024-08-17T17:42:44.083610+0000","last_undegraded":"2024-08-17T17:42:44.083610+0000","last_fullsized":"2024-08-17T17:42:44.083610+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:45:14.747935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":181,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108450+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:42:44.108450+0000","last_peered":"2024-08-17T17:42:44.108450+0000","last_clean":"2024-08-17T17:42:44.108450+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:42:44.108450+0000","last_undegraded":"2024-08-17T17:42:44.108450+0000","last_fullsized":"2024-08-17T17:42:44.108450+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1805,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322666+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:42:41.322666+0000","last_peered":"2024-08-17T17:42:41.322666+0000","last_clean":"2024-08-17T17:42:41.322666+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:42:41.322666+0000","last_undegraded":"2024-08-17T17:42:41.322666+0000","last_fullsized":"2024-08-17T17:42:41.322666+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1484,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.604582+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:42:44.604582+0000","last_peered":"2024-08-17T17:42:44.604582+0000","last_clean":"2024-08-17T17:42:44.604582+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:42:44.604582+0000","last_undegraded":"2024-08-17T17:42:44.604582+0000","last_fullsized":"2024-08-17T17:42:44.604582+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1456,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.145073+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:42:41.145073+0000","last_peered":"2024-08-17T17:42:41.145073+0000","last_clean":"2024-08-17T17:42:41.145073+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:42:41.145073+0000","last_undegraded":"2024-08-17T17:42:41.145073+0000","last_fullsized":"2024-08-17T17:42:41.145073+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":805,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083506+0000","last_change":"2024-08-17T17:42:06.615686+0000","last_active":"2024-08-17T17:42:44.083506+0000","last_peered":"2024-08-17T17:42:44.083506+0000","last_clean":"2024-08-17T17:42:44.083506+0000","last_became_active":"2024-08-17T17:42:06.237002+0000","last_became_peered":"2024-08-17T17:42:06.237002+0000","last_unstale":"2024-08-17T17:42:44.083506+0000","last_undegraded":"2024-08-17T17:42:44.083506+0000","last_fullsized":"2024-08-17T17:42:44.083506+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:42:06.615600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:42:06.615600+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:36:16.880272+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":92,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.606406+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:42:44.606406+0000","last_peered":"2024-08-17T17:42:44.606406+0000","last_clean":"2024-08-17T17:42:44.606406+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:42:44.606406+0000","last_undegraded":"2024-08-17T17:42:44.606406+0000","last_fullsized":"2024-08-17T17:42:44.606406+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1318,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.080312+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:42:41.080312+0000","last_peered":"2024-08-17T17:42:41.080312+0000","last_clean":"2024-08-17T17:42:41.080312+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:42:41.080312+0000","last_undegraded":"2024-08-17T17:42:41.080312+0000","last_fullsized":"2024-08-17T17:42:41.080312+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1833,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322590+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:42:41.322590+0000","last_peered":"2024-08-17T17:42:41.322590+0000","last_clean":"2024-08-17T17:42:41.322590+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:42:41.322590+0000","last_undegraded":"2024-08-17T17:42:41.322590+0000","last_fullsized":"2024-08-17T17:42:41.322590+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1435,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.605569+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:42:44.605569+0000","last_peered":"2024-08-17T17:42:44.605569+0000","last_clean":"2024-08-17T17:42:44.605569+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:42:44.605569+0000","last_undegraded":"2024-08-17T17:42:44.605569+0000","last_fullsized":"2024-08-17T17:42:44.605569+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":180,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108443+0000","last_change":"2024-08-17T17:42:18.852709+0000","last_active":"2024-08-17T17:42:44.108443+0000","last_peered":"2024-08-17T17:42:44.108443+0000","last_clean":"2024-08-17T17:42:44.108443+0000","last_became_active":"2024-08-17T17:42:18.852335+0000","last_became_peered":"2024-08-17T17:42:18.852335+0000","last_unstale":"2024-08-17T17:42:44.108443+0000","last_undegraded":"2024-08-17T17:42:44.108443+0000","last_fullsized":"2024-08-17T17:42:44.108443+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:40:06.494720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1670,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323260+0000","last_change":"2024-08-17T17:42:36.908751+0000","last_active":"2024-08-17T17:42:41.323260+0000","last_peered":"2024-08-17T17:42:41.323260+0000","last_clean":"2024-08-17T17:42:41.323260+0000","last_became_active":"2024-08-17T17:42:36.908379+0000","last_became_peered":"2024-08-17T17:42:36.908379+0000","last_unstale":"2024-08-17T17:42:41.323260+0000","last_undegraded":"2024-08-17T17:42:41.323260+0000","last_fullsized":"2024-08-17T17:42:41.323260+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:07.523356+0000","scrub_duration":0.013346135,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1349,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083469+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:42:44.083469+0000","last_peered":"2024-08-17T17:42:44.083469+0000","last_clean":"2024-08-17T17:42:44.083469+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:42:44.083469+0000","last_undegraded":"2024-08-17T17:42:44.083469+0000","last_fullsized":"2024-08-17T17:42:44.083469+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1461,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.605982+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:42:44.605982+0000","last_peered":"2024-08-17T17:42:44.605982+0000","last_clean":"2024-08-17T17:42:44.605982+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:42:44.605982+0000","last_undegraded":"2024-08-17T17:42:44.605982+0000","last_fullsized":"2024-08-17T17:42:44.605982+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":961,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.145471+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:42:41.145471+0000","last_peered":"2024-08-17T17:42:41.145471+0000","last_clean":"2024-08-17T17:42:41.145471+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:42:41.145471+0000","last_undegraded":"2024-08-17T17:42:41.145471+0000","last_fullsized":"2024-08-17T17:42:41.145471+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1317,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.080455+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:42:41.080455+0000","last_peered":"2024-08-17T17:42:41.080455+0000","last_clean":"2024-08-17T17:42:41.080455+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:42:41.080455+0000","last_undegraded":"2024-08-17T17:42:41.080455+0000","last_fullsized":"2024-08-17T17:42:41.080455+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1247,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323524+0000","last_change":"2024-08-17T17:42:36.906733+0000","last_active":"2024-08-17T17:42:41.323524+0000","last_peered":"2024-08-17T17:42:41.323524+0000","last_clean":"2024-08-17T17:42:41.323524+0000","last_became_active":"2024-08-17T17:42:36.905850+0000","last_became_peered":"2024-08-17T17:42:36.905850+0000","last_unstale":"2024-08-17T17:42:41.323524+0000","last_undegraded":"2024-08-17T17:42:41.323524+0000","last_fullsized":"2024-08-17T17:42:41.323524+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:17:52.631460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":956,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:42.779231+0000","last_change":"2024-08-17T17:40:56.322072+0000","last_active":"2024-08-17T17:42:42.779231+0000","last_peered":"2024-08-17T17:42:42.779231+0000","last_clean":"2024-08-17T17:42:42.779231+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:42:42.779231+0000","last_undegraded":"2024-08-17T17:42:42.779231+0000","last_fullsized":"2024-08-17T17:42:42.779231+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:56.321976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:40:56.321976+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:45.244567+0000","scrub_duration":0.0014456440000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1502,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.144995+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:42:41.144995+0000","last_peered":"2024-08-17T17:42:41.144995+0000","last_clean":"2024-08-17T17:42:41.144995+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:42:41.144995+0000","last_undegraded":"2024-08-17T17:42:41.144995+0000","last_fullsized":"2024-08-17T17:42:41.144995+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1296,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323333+0000","last_change":"2024-08-17T17:42:36.910060+0000","last_active":"2024-08-17T17:42:41.323333+0000","last_peered":"2024-08-17T17:42:41.323333+0000","last_clean":"2024-08-17T17:42:41.323333+0000","last_became_active":"2024-08-17T17:42:36.909789+0000","last_became_peered":"2024-08-17T17:42:36.909789+0000","last_unstale":"2024-08-17T17:42:41.323333+0000","last_undegraded":"2024-08-17T17:42:41.323333+0000","last_fullsized":"2024-08-17T17:42:41.323333+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:33.809561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1781,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108560+0000","last_change":"2024-08-17T17:40:55.554364+0000","last_active":"2024-08-17T17:42:44.108560+0000","last_peered":"2024-08-17T17:42:44.108560+0000","last_clean":"2024-08-17T17:42:44.108560+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:42:44.108560+0000","last_undegraded":"2024-08-17T17:42:44.108560+0000","last_fullsized":"2024-08-17T17:42:44.108560+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:55.554234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:40:55.554234+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:52:22.208370+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1426,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322792+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:42:41.322792+0000","last_peered":"2024-08-17T17:42:41.322792+0000","last_clean":"2024-08-17T17:42:41.322792+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:42:41.322792+0000","last_undegraded":"2024-08-17T17:42:41.322792+0000","last_fullsized":"2024-08-17T17:42:41.322792+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1282,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323331+0000","last_change":"2024-08-17T17:42:36.910203+0000","last_active":"2024-08-17T17:42:41.323331+0000","last_peered":"2024-08-17T17:42:41.323331+0000","last_clean":"2024-08-17T17:42:41.323331+0000","last_became_active":"2024-08-17T17:42:36.909884+0000","last_became_peered":"2024-08-17T17:42:36.909884+0000","last_unstale":"2024-08-17T17:42:41.323331+0000","last_undegraded":"2024-08-17T17:42:41.323331+0000","last_fullsized":"2024-08-17T17:42:41.323331+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:50.728898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1540,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323713+0000","last_change":"2024-08-17T17:42:36.911100+0000","last_active":"2024-08-17T17:42:41.323713+0000","last_peered":"2024-08-17T17:42:41.323713+0000","last_clean":"2024-08-17T17:42:41.323713+0000","last_became_active":"2024-08-17T17:42:36.910795+0000","last_became_peered":"2024-08-17T17:42:36.910795+0000","last_unstale":"2024-08-17T17:42:41.323713+0000","last_undegraded":"2024-08-17T17:42:41.323713+0000","last_fullsized":"2024-08-17T17:42:41.323713+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:18:27.586094+0000","scrub_duration":0.0066315480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1358,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.080082+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:42:44.080082+0000","last_peered":"2024-08-17T17:42:44.080082+0000","last_clean":"2024-08-17T17:42:44.080082+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:42:44.080082+0000","last_undegraded":"2024-08-17T17:42:44.080082+0000","last_fullsized":"2024-08-17T17:42:44.080082+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":180,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108781+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:42:44.108781+0000","last_peered":"2024-08-17T17:42:44.108781+0000","last_clean":"2024-08-17T17:42:44.108781+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:42:44.108781+0000","last_undegraded":"2024-08-17T17:42:44.108781+0000","last_fullsized":"2024-08-17T17:42:44.108781+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1789,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.604777+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:42:44.604777+0000","last_peered":"2024-08-17T17:42:44.604777+0000","last_clean":"2024-08-17T17:42:44.604777+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:42:44.604777+0000","last_undegraded":"2024-08-17T17:42:44.604777+0000","last_fullsized":"2024-08-17T17:42:44.604777+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1288,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322705+0000","last_change":"2024-08-17T17:42:36.910900+0000","last_active":"2024-08-17T17:42:41.322705+0000","last_peered":"2024-08-17T17:42:41.322705+0000","last_clean":"2024-08-17T17:42:41.322705+0000","last_became_active":"2024-08-17T17:42:36.910614+0000","last_became_peered":"2024-08-17T17:42:36.910614+0000","last_unstale":"2024-08-17T17:42:41.322705+0000","last_undegraded":"2024-08-17T17:42:41.322705+0000","last_fullsized":"2024-08-17T17:42:41.322705+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:33:55.103679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1028,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083745+0000","last_change":"2024-08-17T17:42:00.211263+0000","last_active":"2024-08-17T17:42:44.083745+0000","last_peered":"2024-08-17T17:42:44.083745+0000","last_clean":"2024-08-17T17:42:44.083745+0000","last_became_active":"2024-08-17T17:42:00.210948+0000","last_became_peered":"2024-08-17T17:42:00.210948+0000","last_unstale":"2024-08-17T17:42:44.083745+0000","last_undegraded":"2024-08-17T17:42:44.083745+0000","last_fullsized":"2024-08-17T17:42:44.083745+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:53.580243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":980,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083406+0000","last_change":"2024-08-17T17:42:06.239969+0000","last_active":"2024-08-17T17:42:44.083406+0000","last_peered":"2024-08-17T17:42:44.083406+0000","last_clean":"2024-08-17T17:42:44.083406+0000","last_became_active":"2024-08-17T17:42:06.239603+0000","last_became_peered":"2024-08-17T17:42:06.239603+0000","last_unstale":"2024-08-17T17:42:44.083406+0000","last_undegraded":"2024-08-17T17:42:44.083406+0000","last_fullsized":"2024-08-17T17:42:44.083406+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:18.882142+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1266,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083205+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:42:44.083205+0000","last_peered":"2024-08-17T17:42:44.083205+0000","last_clean":"2024-08-17T17:42:44.083205+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:42:44.083205+0000","last_undegraded":"2024-08-17T17:42:44.083205+0000","last_fullsized":"2024-08-17T17:42:44.083205+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1458,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323759+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:42:41.323759+0000","last_peered":"2024-08-17T17:42:41.323759+0000","last_clean":"2024-08-17T17:42:41.323759+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:42:41.323759+0000","last_undegraded":"2024-08-17T17:42:41.323759+0000","last_fullsized":"2024-08-17T17:42:41.323759+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1752,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.605129+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:42:44.605129+0000","last_peered":"2024-08-17T17:42:44.605129+0000","last_clean":"2024-08-17T17:42:44.605129+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:42:44.605129+0000","last_undegraded":"2024-08-17T17:42:44.605129+0000","last_fullsized":"2024-08-17T17:42:44.605129+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1400,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322962+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:42:41.322962+0000","last_peered":"2024-08-17T17:42:41.322962+0000","last_clean":"2024-08-17T17:42:41.322962+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:42:41.322962+0000","last_undegraded":"2024-08-17T17:42:41.322962+0000","last_fullsized":"2024-08-17T17:42:41.322962+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1982,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108865+0000","last_change":"2024-08-17T17:42:18.995663+0000","last_active":"2024-08-17T17:42:44.108865+0000","last_peered":"2024-08-17T17:42:44.108865+0000","last_clean":"2024-08-17T17:42:44.108865+0000","last_became_active":"2024-08-17T17:42:18.995326+0000","last_became_peered":"2024-08-17T17:42:18.995326+0000","last_unstale":"2024-08-17T17:42:44.108865+0000","last_undegraded":"2024-08-17T17:42:44.108865+0000","last_fullsized":"2024-08-17T17:42:44.108865+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:53:08.837477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1454,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083259+0000","last_change":"2024-08-17T17:42:01.636437+0000","last_active":"2024-08-17T17:42:44.083259+0000","last_peered":"2024-08-17T17:42:44.083259+0000","last_clean":"2024-08-17T17:42:44.083259+0000","last_became_active":"2024-08-17T17:42:00.209411+0000","last_became_peered":"2024-08-17T17:42:00.209411+0000","last_unstale":"2024-08-17T17:42:44.083259+0000","last_undegraded":"2024-08-17T17:42:44.083259+0000","last_fullsized":"2024-08-17T17:42:44.083259+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:42:01.636355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:42:01.636355+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:03.015493+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162913,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":172860,"kb_used_data":8240,"kb_used_omap":54,"kb_used_meta":164553,"kb_avail":93572292,"statfs":{"total":95995035648,"available":95818027008,"internally_reserved":0,"allocated":8437760,"data_stored":5571815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":168502293},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999387,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":230324,"kb_used_data":8296,"kb_used_omap":47,"kb_used_meta":221968,"kb_avail":93514828,"statfs":{"total":95995035648,"available":95759183872,"internally_reserved":0,"allocated":8495104,"data_stored":5637722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":227295892},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627183,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1116960,"kb_used_data":8276,"kb_used_omap":42,"kb_used_meta":1108629,"kb_avail":92628192,"statfs":{"total":95995035648,"available":94851268608,"internally_reserved":0,"allocated":8474624,"data_stored":5631937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1135236156},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785661,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":860708,"kb_used_data":7960,"kb_used_omap":48,"kb_used_meta":852687,"kb_avail":92884444,"statfs":{"total":95995035648,"available":95113670656,"internally_reserved":0,"allocated":8151040,"data_stored":5305228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":873152174},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725178,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":901484,"kb_used_data":8224,"kb_used_omap":1,"kb_used_meta":893246,"kb_avail":92843668,"statfs":{"total":95995035648,"available":95071916032,"internally_reserved":0,"allocated":8421376,"data_stored":5567514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":914684358},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986815,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":919284,"kb_used_data":7976,"kb_used_omap":1,"kb_used_meta":911294,"kb_avail":92825868,"statfs":{"total":95995035648,"available":95053688832,"internally_reserved":0,"allocated":8167424,"data_stored":5309057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":933165510},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706422,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":942632,"kb_used_data":8220,"kb_used_omap":35,"kb_used_meta":934364,"kb_avail":92802520,"statfs":{"total":95995035648,"available":95029780480,"internally_reserved":0,"allocated":8417280,"data_stored":5567471,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":956788791},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832195,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1061096,"kb_used_data":8284,"kb_used_omap":65,"kb_used_meta":1052734,"kb_avail":92684056,"statfs":{"total":95995035648,"available":94908473344,"internally_reserved":0,"allocated":8482816,"data_stored":5633493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1077999803},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:43:29.961 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 -- ceph pg dump --format=json 2024-08-17T17:43:30.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:29 smithi162 ceph-mon[104668]: pgmap v2413: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:30.178 INFO:teuthology.orchestra.run.smithi063.stderr:Inferring config /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/mon.a/config 2024-08-17T17:43:31.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:30 smithi162 ceph-mon[104668]: from='client.36873 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:31.188 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:30 smithi063 ceph-mon[114756]: from='client.36873 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:31.189 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:30 smithi063 ceph-mon[109843]: from='client.36873 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:31.958 INFO:teuthology.orchestra.run.smithi063.stdout: 2024-08-17T17:43:31.959 INFO:teuthology.orchestra.run.smithi063.stderr:dumped all 2024-08-17T17:43:32.095 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:31 smithi162 ceph-mon[104668]: pgmap v2414: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:32.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:31 smithi162 ceph-mon[104668]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:32.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:31 smithi063 ceph-mon[114756]: pgmap v2414: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:32.279 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:31 smithi063 ceph-mon[114756]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:32.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:31 smithi063 ceph-mon[109843]: pgmap v2414: 103 pgs: 103 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-08-17T17:43:32.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:31 smithi063 ceph-mon[109843]: from='mgr.25070 172.21.15.63:0/2783321466' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-08-17T17:43:32.737 INFO:teuthology.orchestra.run.smithi063.stdout:{"pg_ready":true,"pg_map":{"version":2414,"stamp":"2024-08-17T17:43:30.727467+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":526059,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4678,"ondisk_log_size":4678,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6205348,"kb_used_data":65476,"kb_used_omap":298,"kb_used_meta":6139477,"kb_avail":743755868,"statfs":{"total":767960285184,"available":761606008832,"internally_reserved":0,"allocated":67047424,"data_stored":44224237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":305647,"internal_metadata":6286824977},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003866"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":141,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083842+0000","last_change":"2024-08-17T17:37:11.083590+0000","last_active":"2024-08-17T17:42:44.083842+0000","last_peered":"2024-08-17T17:42:44.083842+0000","last_clean":"2024-08-17T17:42:44.083842+0000","last_became_active":"2024-08-17T17:37:11.078160+0000","last_became_peered":"2024-08-17T17:37:11.078160+0000","last_unstale":"2024-08-17T17:42:44.083842+0000","last_undegraded":"2024-08-17T17:42:44.083842+0000","last_fullsized":"2024-08-17T17:42:44.083842+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:59:06.246283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":582,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1482,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.144825+0000","last_change":"2024-08-17T17:35:56.210537+0000","last_active":"2024-08-17T17:42:41.144825+0000","last_peered":"2024-08-17T17:42:41.144825+0000","last_clean":"2024-08-17T17:42:41.144825+0000","last_became_active":"2024-08-17T17:35:56.210067+0000","last_became_peered":"2024-08-17T17:35:56.210067+0000","last_unstale":"2024-08-17T17:42:41.144825+0000","last_undegraded":"2024-08-17T17:42:41.144825+0000","last_fullsized":"2024-08-17T17:42:41.144825+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:28.263937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"369'149","reported_seq":1625,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083987+0000","last_change":"2024-08-17T17:42:06.241404+0000","last_active":"2024-08-17T17:42:44.083987+0000","last_peered":"2024-08-17T17:42:44.083987+0000","last_clean":"2024-08-17T17:42:44.083987+0000","last_became_active":"2024-08-17T17:42:06.241035+0000","last_became_peered":"2024-08-17T17:42:06.241035+0000","last_unstale":"2024-08-17T17:42:44.083987+0000","last_undegraded":"2024-08-17T17:42:44.083987+0000","last_fullsized":"2024-08-17T17:42:44.083987+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:30:28.798739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:30:28.798739+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:47.523505+0000","scrub_duration":0.001543366,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":68,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108696+0000","last_change":"2024-08-17T17:42:18.853159+0000","last_active":"2024-08-17T17:42:44.108696+0000","last_peered":"2024-08-17T17:42:44.108696+0000","last_clean":"2024-08-17T17:42:44.108696+0000","last_became_active":"2024-08-17T17:42:18.852607+0000","last_became_peered":"2024-08-17T17:42:18.852607+0000","last_unstale":"2024-08-17T17:42:44.108696+0000","last_undegraded":"2024-08-17T17:42:44.108696+0000","last_fullsized":"2024-08-17T17:42:44.108696+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:00:31.425280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":77,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108217+0000","last_change":"2024-08-17T17:42:18.854135+0000","last_active":"2024-08-17T17:42:44.108217+0000","last_peered":"2024-08-17T17:42:44.108217+0000","last_clean":"2024-08-17T17:42:44.108217+0000","last_became_active":"2024-08-17T17:42:18.853580+0000","last_became_peered":"2024-08-17T17:42:18.853580+0000","last_unstale":"2024-08-17T17:42:44.108217+0000","last_undegraded":"2024-08-17T17:42:44.108217+0000","last_fullsized":"2024-08-17T17:42:44.108217+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:50.774568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"369'149","reported_seq":1759,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323227+0000","last_change":"2024-08-17T17:42:40.150301+0000","last_active":"2024-08-17T17:42:41.323227+0000","last_peered":"2024-08-17T17:42:41.323227+0000","last_clean":"2024-08-17T17:42:41.323227+0000","last_became_active":"2024-08-17T17:42:36.908990+0000","last_became_peered":"2024-08-17T17:42:36.908990+0000","last_unstale":"2024-08-17T17:42:41.323227+0000","last_undegraded":"2024-08-17T17:42:41.323227+0000","last_fullsized":"2024-08-17T17:42:41.323227+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:42:40.150181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:42:40.150181+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:15:11.864462+0000","scrub_duration":0.0018593699999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1415,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323900+0000","last_change":"2024-08-17T17:35:55.217158+0000","last_active":"2024-08-17T17:42:41.323900+0000","last_peered":"2024-08-17T17:42:41.323900+0000","last_clean":"2024-08-17T17:42:41.323900+0000","last_became_active":"2024-08-17T17:35:55.216670+0000","last_became_peered":"2024-08-17T17:35:55.216670+0000","last_unstale":"2024-08-17T17:42:41.323900+0000","last_undegraded":"2024-08-17T17:42:41.323900+0000","last_fullsized":"2024-08-17T17:42:41.323900+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:16:38.233548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":91,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.144901+0000","last_change":"2024-08-17T17:36:17.776368+0000","last_active":"2024-08-17T17:42:41.144901+0000","last_peered":"2024-08-17T17:42:41.144901+0000","last_clean":"2024-08-17T17:42:41.144901+0000","last_became_active":"2024-08-17T17:36:17.776024+0000","last_became_peered":"2024-08-17T17:36:17.776024+0000","last_unstale":"2024-08-17T17:42:41.144901+0000","last_undegraded":"2024-08-17T17:42:41.144901+0000","last_fullsized":"2024-08-17T17:42:41.144901+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:40:51.529062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1460,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323882+0000","last_change":"2024-08-17T17:33:25.933173+0000","last_active":"2024-08-17T17:42:41.323882+0000","last_peered":"2024-08-17T17:42:41.323882+0000","last_clean":"2024-08-17T17:42:41.323882+0000","last_became_active":"2024-08-17T17:33:25.932820+0000","last_became_peered":"2024-08-17T17:33:25.932820+0000","last_unstale":"2024-08-17T17:42:41.323882+0000","last_undegraded":"2024-08-17T17:42:41.323882+0000","last_fullsized":"2024-08-17T17:42:41.323882+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:43:36.884583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1468,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.145218+0000","last_change":"2024-08-17T17:33:25.947680+0000","last_active":"2024-08-17T17:42:41.145218+0000","last_peered":"2024-08-17T17:42:41.145218+0000","last_clean":"2024-08-17T17:42:41.145218+0000","last_became_active":"2024-08-17T17:33:25.947386+0000","last_became_peered":"2024-08-17T17:33:25.947386+0000","last_unstale":"2024-08-17T17:42:41.145218+0000","last_undegraded":"2024-08-17T17:42:41.145218+0000","last_fullsized":"2024-08-17T17:42:41.145218+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:40:56.013154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"369'149","reported_seq":689,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108821+0000","last_change":"2024-08-17T17:42:18.853578+0000","last_active":"2024-08-17T17:42:44.108821+0000","last_peered":"2024-08-17T17:42:44.108821+0000","last_clean":"2024-08-17T17:42:44.108821+0000","last_became_active":"2024-08-17T17:42:18.853274+0000","last_became_peered":"2024-08-17T17:42:18.853274+0000","last_unstale":"2024-08-17T17:42:44.108821+0000","last_undegraded":"2024-08-17T17:42:44.108821+0000","last_fullsized":"2024-08-17T17:42:44.108821+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:46.946197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:46.946197+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:03:20.069149+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":134,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083239+0000","last_change":"2024-08-17T17:37:11.079702+0000","last_active":"2024-08-17T17:42:44.083239+0000","last_peered":"2024-08-17T17:42:44.083239+0000","last_clean":"2024-08-17T17:42:44.083239+0000","last_became_active":"2024-08-17T17:37:11.078948+0000","last_became_peered":"2024-08-17T17:37:11.078948+0000","last_unstale":"2024-08-17T17:42:44.083239+0000","last_undegraded":"2024-08-17T17:42:44.083239+0000","last_fullsized":"2024-08-17T17:42:44.083239+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:13:04.338127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":225,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.370425+0000","last_change":"2024-08-17T17:36:12.760122+0000","last_active":"2024-08-17T17:42:44.370425+0000","last_peered":"2024-08-17T17:42:44.370425+0000","last_clean":"2024-08-17T17:42:44.370425+0000","last_became_active":"2024-08-17T17:36:12.759736+0000","last_became_peered":"2024-08-17T17:36:12.759736+0000","last_unstale":"2024-08-17T17:42:44.370425+0000","last_undegraded":"2024-08-17T17:42:44.370425+0000","last_fullsized":"2024-08-17T17:42:44.370425+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:15:00.339946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.19","version":"369'149","reported_seq":1485,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108313+0000","last_change":"2024-08-17T17:42:18.851661+0000","last_active":"2024-08-17T17:42:44.108313+0000","last_peered":"2024-08-17T17:42:44.108313+0000","last_clean":"2024-08-17T17:42:44.108313+0000","last_became_active":"2024-08-17T17:42:18.851310+0000","last_became_peered":"2024-08-17T17:42:18.851310+0000","last_unstale":"2024-08-17T17:42:44.108313+0000","last_undegraded":"2024-08-17T17:42:44.108313+0000","last_fullsized":"2024-08-17T17:42:44.108313+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:33:49.869853+0000","last_clean_scrub_stamp":"2024-08-17T17:33:49.869853+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:34:06.851954+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1488,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.145136+0000","last_change":"2024-08-17T17:35:14.850325+0000","last_active":"2024-08-17T17:42:41.145136+0000","last_peered":"2024-08-17T17:42:41.145136+0000","last_clean":"2024-08-17T17:42:41.145136+0000","last_became_active":"2024-08-17T17:35:14.850082+0000","last_became_peered":"2024-08-17T17:35:14.850082+0000","last_unstale":"2024-08-17T17:42:41.145136+0000","last_undegraded":"2024-08-17T17:42:41.145136+0000","last_fullsized":"2024-08-17T17:42:41.145136+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:14:03.447497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1834,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108570+0000","last_change":"2024-08-17T17:42:18.996230+0000","last_active":"2024-08-17T17:42:44.108570+0000","last_peered":"2024-08-17T17:42:44.108570+0000","last_clean":"2024-08-17T17:42:44.108570+0000","last_became_active":"2024-08-17T17:42:18.995147+0000","last_became_peered":"2024-08-17T17:42:18.995147+0000","last_unstale":"2024-08-17T17:42:44.108570+0000","last_undegraded":"2024-08-17T17:42:44.108570+0000","last_fullsized":"2024-08-17T17:42:44.108570+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:51:33.826500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.18","version":"369'149","reported_seq":1843,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322699+0000","last_change":"2024-08-17T17:35:34.132568+0000","last_active":"2024-08-17T17:42:41.322699+0000","last_peered":"2024-08-17T17:42:41.322699+0000","last_clean":"2024-08-17T17:42:41.322699+0000","last_became_active":"2024-08-17T17:35:34.132186+0000","last_became_peered":"2024-08-17T17:35:34.132186+0000","last_unstale":"2024-08-17T17:42:41.322699+0000","last_undegraded":"2024-08-17T17:42:41.322699+0000","last_fullsized":"2024-08-17T17:42:41.322699+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:02.696468+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:02.696468+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:56:13.355631+0000","scrub_duration":0.0049602040000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1469,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.082843+0000","last_change":"2024-08-17T17:42:06.239740+0000","last_active":"2024-08-17T17:42:44.082843+0000","last_peered":"2024-08-17T17:42:44.082843+0000","last_clean":"2024-08-17T17:42:44.082843+0000","last_became_active":"2024-08-17T17:42:06.239413+0000","last_became_peered":"2024-08-17T17:42:06.239413+0000","last_unstale":"2024-08-17T17:42:44.082843+0000","last_undegraded":"2024-08-17T17:42:44.082843+0000","last_fullsized":"2024-08-17T17:42:44.082843+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:40:33.067653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1227,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083691+0000","last_change":"2024-08-17T17:36:12.754926+0000","last_active":"2024-08-17T17:42:44.083691+0000","last_peered":"2024-08-17T17:42:44.083691+0000","last_clean":"2024-08-17T17:42:44.083691+0000","last_became_active":"2024-08-17T17:36:12.754170+0000","last_became_peered":"2024-08-17T17:36:12.754170+0000","last_unstale":"2024-08-17T17:42:44.083691+0000","last_undegraded":"2024-08-17T17:42:44.083691+0000","last_fullsized":"2024-08-17T17:42:44.083691+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:37:50.207823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1b","version":"369'149","reported_seq":1010,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323079+0000","last_change":"2024-08-17T17:25:00.720271+0000","last_active":"2024-08-17T17:42:41.323079+0000","last_peered":"2024-08-17T17:42:41.323079+0000","last_clean":"2024-08-17T17:42:41.323079+0000","last_became_active":"2024-08-17T17:24:40.879617+0000","last_became_peered":"2024-08-17T17:24:40.879617+0000","last_unstale":"2024-08-17T17:42:41.323079+0000","last_undegraded":"2024-08-17T17:42:41.323079+0000","last_fullsized":"2024-08-17T17:42:41.323079+0000","mapping_epoch":1229,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1230,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:00.720190+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:25:00.720190+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:35:45.974491+0000","scrub_duration":0.0014515439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":82,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108148+0000","last_change":"2024-08-17T17:42:18.995816+0000","last_active":"2024-08-17T17:42:44.108148+0000","last_peered":"2024-08-17T17:42:44.108148+0000","last_clean":"2024-08-17T17:42:44.108148+0000","last_became_active":"2024-08-17T17:42:18.995318+0000","last_became_peered":"2024-08-17T17:42:18.995318+0000","last_unstale":"2024-08-17T17:42:44.108148+0000","last_undegraded":"2024-08-17T17:42:44.108148+0000","last_fullsized":"2024-08-17T17:42:44.108148+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:34:38.636971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1323,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.080619+0000","last_change":"2024-08-17T17:35:56.227361+0000","last_active":"2024-08-17T17:42:41.080619+0000","last_peered":"2024-08-17T17:42:41.080619+0000","last_clean":"2024-08-17T17:42:41.080619+0000","last_became_active":"2024-08-17T17:35:56.227055+0000","last_became_peered":"2024-08-17T17:35:56.227055+0000","last_unstale":"2024-08-17T17:42:41.080619+0000","last_undegraded":"2024-08-17T17:42:41.080619+0000","last_fullsized":"2024-08-17T17:42:41.080619+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:56:09.072307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1a","version":"369'149","reported_seq":1700,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.144903+0000","last_change":"2024-08-17T17:35:33.137013+0000","last_active":"2024-08-17T17:42:41.144903+0000","last_peered":"2024-08-17T17:42:41.144903+0000","last_clean":"2024-08-17T17:42:41.144903+0000","last_became_active":"2024-08-17T17:35:33.136672+0000","last_became_peered":"2024-08-17T17:35:33.136672+0000","last_unstale":"2024-08-17T17:42:41.144903+0000","last_undegraded":"2024-08-17T17:42:41.144903+0000","last_fullsized":"2024-08-17T17:42:41.144903+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:53.130455+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:53.130455+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:21:23.018791+0000","scrub_duration":0.0042829740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":181,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108326+0000","last_change":"2024-08-17T17:36:12.872156+0000","last_active":"2024-08-17T17:42:44.108326+0000","last_peered":"2024-08-17T17:42:44.108326+0000","last_clean":"2024-08-17T17:42:44.108326+0000","last_became_active":"2024-08-17T17:36:12.871552+0000","last_became_peered":"2024-08-17T17:36:12.871552+0000","last_unstale":"2024-08-17T17:42:44.108326+0000","last_undegraded":"2024-08-17T17:42:44.108326+0000","last_fullsized":"2024-08-17T17:42:44.108326+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:50:13.079297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1249,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083993+0000","last_change":"2024-08-17T17:42:00.208934+0000","last_active":"2024-08-17T17:42:44.083993+0000","last_peered":"2024-08-17T17:42:44.083993+0000","last_clean":"2024-08-17T17:42:44.083993+0000","last_became_active":"2024-08-17T17:42:00.208653+0000","last_became_peered":"2024-08-17T17:42:00.208653+0000","last_unstale":"2024-08-17T17:42:44.083993+0000","last_undegraded":"2024-08-17T17:42:44.083993+0000","last_fullsized":"2024-08-17T17:42:44.083993+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:24:18.190351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.15","version":"369'149","reported_seq":1651,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.605924+0000","last_change":"2024-08-17T17:35:33.153192+0000","last_active":"2024-08-17T17:42:44.605924+0000","last_peered":"2024-08-17T17:42:44.605924+0000","last_clean":"2024-08-17T17:42:44.605924+0000","last_became_active":"2024-08-17T17:35:33.151273+0000","last_became_peered":"2024-08-17T17:35:33.151273+0000","last_unstale":"2024-08-17T17:42:44.605924+0000","last_undegraded":"2024-08-17T17:42:44.605924+0000","last_fullsized":"2024-08-17T17:42:44.605924+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:07.471926+0000","last_clean_scrub_stamp":"2024-08-17T17:25:07.471926+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:45:45.634296+0000","scrub_duration":0.0099132249999999995,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1807,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108565+0000","last_change":"2024-08-17T17:42:18.853877+0000","last_active":"2024-08-17T17:42:44.108565+0000","last_peered":"2024-08-17T17:42:44.108565+0000","last_clean":"2024-08-17T17:42:44.108565+0000","last_became_active":"2024-08-17T17:42:18.853536+0000","last_became_peered":"2024-08-17T17:42:18.853536+0000","last_unstale":"2024-08-17T17:42:44.108565+0000","last_undegraded":"2024-08-17T17:42:44.108565+0000","last_fullsized":"2024-08-17T17:42:44.108565+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:35:59.569437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1474,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323586+0000","last_change":"2024-08-17T17:35:33.145716+0000","last_active":"2024-08-17T17:42:41.323586+0000","last_peered":"2024-08-17T17:42:41.323586+0000","last_clean":"2024-08-17T17:42:41.323586+0000","last_became_active":"2024-08-17T17:35:33.145420+0000","last_became_peered":"2024-08-17T17:35:33.145420+0000","last_unstale":"2024-08-17T17:42:41.323586+0000","last_undegraded":"2024-08-17T17:42:41.323586+0000","last_fullsized":"2024-08-17T17:42:41.323586+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:31:50.667774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.14","version":"369'149","reported_seq":1799,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.370279+0000","last_change":"2024-08-17T17:35:34.133267+0000","last_active":"2024-08-17T17:42:44.370279+0000","last_peered":"2024-08-17T17:42:44.370279+0000","last_clean":"2024-08-17T17:42:44.370279+0000","last_became_active":"2024-08-17T17:35:34.132907+0000","last_became_peered":"2024-08-17T17:35:34.132907+0000","last_unstale":"2024-08-17T17:42:44.370279+0000","last_undegraded":"2024-08-17T17:42:44.370279+0000","last_fullsized":"2024-08-17T17:42:44.370279+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.028307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.028307+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T05:11:06.218915+0000","scrub_duration":26,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1257,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323314+0000","last_change":"2024-08-17T17:42:36.911202+0000","last_active":"2024-08-17T17:42:41.323314+0000","last_peered":"2024-08-17T17:42:41.323314+0000","last_clean":"2024-08-17T17:42:41.323314+0000","last_became_active":"2024-08-17T17:42:36.910876+0000","last_became_peered":"2024-08-17T17:42:36.910876+0000","last_unstale":"2024-08-17T17:42:41.323314+0000","last_undegraded":"2024-08-17T17:42:41.323314+0000","last_fullsized":"2024-08-17T17:42:41.323314+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:56.962011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":991,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083134+0000","last_change":"2024-08-17T17:42:06.742409+0000","last_active":"2024-08-17T17:42:44.083134+0000","last_peered":"2024-08-17T17:42:44.083134+0000","last_clean":"2024-08-17T17:42:44.083134+0000","last_became_active":"2024-08-17T17:42:06.742185+0000","last_became_peered":"2024-08-17T17:42:06.742185+0000","last_unstale":"2024-08-17T17:42:44.083134+0000","last_undegraded":"2024-08-17T17:42:44.083134+0000","last_fullsized":"2024-08-17T17:42:44.083134+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:43:01.999846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.17","version":"369'149","reported_seq":918,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323628+0000","last_change":"2024-08-17T17:35:55.216494+0000","last_active":"2024-08-17T17:42:41.323628+0000","last_peered":"2024-08-17T17:42:41.323628+0000","last_clean":"2024-08-17T17:42:41.323628+0000","last_became_active":"2024-08-17T17:35:55.215944+0000","last_became_peered":"2024-08-17T17:35:55.215944+0000","last_unstale":"2024-08-17T17:42:41.323628+0000","last_undegraded":"2024-08-17T17:42:41.323628+0000","last_fullsized":"2024-08-17T17:42:41.323628+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:25:03.663278+0000","last_clean_scrub_stamp":"2024-08-17T17:25:03.663278+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:42:17.244325+0000","scrub_duration":0.0040451599999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.14","version":"455'23","reported_seq":1454,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.605723+0000","last_change":"2024-08-17T17:35:55.213373+0000","last_active":"2024-08-17T17:42:44.605723+0000","last_peered":"2024-08-17T17:42:44.605723+0000","last_clean":"2024-08-17T17:42:44.605723+0000","last_became_active":"2024-08-17T17:35:55.212740+0000","last_became_peered":"2024-08-17T17:35:55.212740+0000","last_unstale":"2024-08-17T17:42:44.605723+0000","last_undegraded":"2024-08-17T17:42:44.605723+0000","last_fullsized":"2024-08-17T17:42:44.605723+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:59:32.317544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1473,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.604548+0000","last_change":"2024-08-17T17:35:56.220615+0000","last_active":"2024-08-17T17:42:44.604548+0000","last_peered":"2024-08-17T17:42:44.604548+0000","last_clean":"2024-08-17T17:42:44.604548+0000","last_became_active":"2024-08-17T17:35:56.220311+0000","last_became_peered":"2024-08-17T17:35:56.220311+0000","last_unstale":"2024-08-17T17:42:44.604548+0000","last_undegraded":"2024-08-17T17:42:44.604548+0000","last_fullsized":"2024-08-17T17:42:44.604548+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:25:12.613471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.16","version":"369'149","reported_seq":1022,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.084365+0000","last_change":"2024-08-17T17:42:06.239908+0000","last_active":"2024-08-17T17:42:44.084365+0000","last_peered":"2024-08-17T17:42:44.084365+0000","last_clean":"2024-08-17T17:42:44.084365+0000","last_became_active":"2024-08-17T17:42:06.239583+0000","last_became_peered":"2024-08-17T17:42:06.239583+0000","last_unstale":"2024-08-17T17:42:44.084365+0000","last_undegraded":"2024-08-17T17:42:44.084365+0000","last_fullsized":"2024-08-17T17:42:44.084365+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":1223,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:42:02.664315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:42:02.664315+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:42:00.458998+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1421,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.144825+0000","last_change":"2024-08-17T17:35:55.212963+0000","last_active":"2024-08-17T17:42:41.144825+0000","last_peered":"2024-08-17T17:42:41.144825+0000","last_clean":"2024-08-17T17:42:41.144825+0000","last_became_active":"2024-08-17T17:35:55.211885+0000","last_became_peered":"2024-08-17T17:35:55.211885+0000","last_unstale":"2024-08-17T17:42:41.144825+0000","last_undegraded":"2024-08-17T17:42:41.144825+0000","last_fullsized":"2024-08-17T17:42:41.144825+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:58:11.351830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1317,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.080553+0000","last_change":"2024-08-17T17:35:56.281436+0000","last_active":"2024-08-17T17:42:41.080553+0000","last_peered":"2024-08-17T17:42:41.080553+0000","last_clean":"2024-08-17T17:42:41.080553+0000","last_became_active":"2024-08-17T17:35:56.277095+0000","last_became_peered":"2024-08-17T17:35:56.277095+0000","last_unstale":"2024-08-17T17:42:41.080553+0000","last_undegraded":"2024-08-17T17:42:41.080553+0000","last_fullsized":"2024-08-17T17:42:41.080553+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:39:10.789042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026089800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":413,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083746+0000","last_change":"2024-08-17T17:33:25.955343+0000","last_active":"2024-08-17T17:42:44.083746+0000","last_peered":"2024-08-17T17:42:44.083746+0000","last_clean":"2024-08-17T17:42:44.083746+0000","last_became_active":"2024-08-17T17:33:25.946534+0000","last_became_peered":"2024-08-17T17:33:25.946534+0000","last_unstale":"2024-08-17T17:42:44.083746+0000","last_undegraded":"2024-08-17T17:42:44.083746+0000","last_fullsized":"2024-08-17T17:42:44.083746+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:45:01.212978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"369'149","reported_seq":1682,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108555+0000","last_change":"2024-08-17T17:36:15.788351+0000","last_active":"2024-08-17T17:42:44.108555+0000","last_peered":"2024-08-17T17:42:44.108555+0000","last_clean":"2024-08-17T17:42:44.108555+0000","last_became_active":"2024-08-17T17:36:15.788050+0000","last_became_peered":"2024-08-17T17:36:15.788050+0000","last_unstale":"2024-08-17T17:42:44.108555+0000","last_undegraded":"2024-08-17T17:42:44.108555+0000","last_fullsized":"2024-08-17T17:42:44.108555+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:31.991076+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:31.991076+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:42:36.477246+0000","scrub_duration":0.002077771,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":179,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108257+0000","last_change":"2024-08-17T17:42:18.853707+0000","last_active":"2024-08-17T17:42:44.108257+0000","last_peered":"2024-08-17T17:42:44.108257+0000","last_clean":"2024-08-17T17:42:44.108257+0000","last_became_active":"2024-08-17T17:42:18.853399+0000","last_became_peered":"2024-08-17T17:42:18.853399+0000","last_unstale":"2024-08-17T17:42:44.108257+0000","last_undegraded":"2024-08-17T17:42:44.108257+0000","last_fullsized":"2024-08-17T17:42:44.108257+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:10:06.230375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":903,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.084632+0000","last_change":"2024-08-17T17:42:06.240411+0000","last_active":"2024-08-17T17:42:44.084632+0000","last_peered":"2024-08-17T17:42:44.084632+0000","last_clean":"2024-08-17T17:42:44.084632+0000","last_became_active":"2024-08-17T17:42:06.240130+0000","last_became_peered":"2024-08-17T17:42:06.240130+0000","last_unstale":"2024-08-17T17:42:44.084632+0000","last_undegraded":"2024-08-17T17:42:44.084632+0000","last_fullsized":"2024-08-17T17:42:44.084632+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:24:27.224356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1718,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108882+0000","last_change":"2024-08-17T17:42:18.853632+0000","last_active":"2024-08-17T17:42:44.108882+0000","last_peered":"2024-08-17T17:42:44.108882+0000","last_clean":"2024-08-17T17:42:44.108882+0000","last_became_active":"2024-08-17T17:42:18.853044+0000","last_became_peered":"2024-08-17T17:42:18.853044+0000","last_unstale":"2024-08-17T17:42:44.108882+0000","last_undegraded":"2024-08-17T17:42:44.108882+0000","last_fullsized":"2024-08-17T17:42:44.108882+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:56:41.623573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"369'149","reported_seq":1827,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.145585+0000","last_change":"2024-08-17T17:35:56.295295+0000","last_active":"2024-08-17T17:42:41.145585+0000","last_peered":"2024-08-17T17:42:41.145585+0000","last_clean":"2024-08-17T17:42:41.145585+0000","last_became_active":"2024-08-17T17:35:56.294670+0000","last_became_peered":"2024-08-17T17:35:56.294670+0000","last_unstale":"2024-08-17T17:42:41.145585+0000","last_undegraded":"2024-08-17T17:42:41.145585+0000","last_fullsized":"2024-08-17T17:42:41.145585+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:49.119495+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:49.119495+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:24:52.250532+0000","scrub_duration":0.0034721420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":980,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083450+0000","last_change":"2024-08-17T17:42:00.209617+0000","last_active":"2024-08-17T17:42:44.083450+0000","last_peered":"2024-08-17T17:42:44.083450+0000","last_clean":"2024-08-17T17:42:44.083450+0000","last_became_active":"2024-08-17T17:42:00.209381+0000","last_became_peered":"2024-08-17T17:42:00.209381+0000","last_unstale":"2024-08-17T17:42:44.083450+0000","last_undegraded":"2024-08-17T17:42:44.083450+0000","last_fullsized":"2024-08-17T17:42:44.083450+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:50:53.735666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1424,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323905+0000","last_change":"2024-08-17T17:35:33.152606+0000","last_active":"2024-08-17T17:42:41.323905+0000","last_peered":"2024-08-17T17:42:41.323905+0000","last_clean":"2024-08-17T17:42:41.323905+0000","last_became_active":"2024-08-17T17:35:33.150658+0000","last_became_peered":"2024-08-17T17:35:33.150658+0000","last_unstale":"2024-08-17T17:42:41.323905+0000","last_undegraded":"2024-08-17T17:42:41.323905+0000","last_fullsized":"2024-08-17T17:42:41.323905+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:03:19.952311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030663000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1250,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083557+0000","last_change":"2024-08-17T17:42:06.742503+0000","last_active":"2024-08-17T17:42:44.083557+0000","last_peered":"2024-08-17T17:42:44.083557+0000","last_clean":"2024-08-17T17:42:44.083557+0000","last_became_active":"2024-08-17T17:42:06.742208+0000","last_became_peered":"2024-08-17T17:42:06.742208+0000","last_unstale":"2024-08-17T17:42:44.083557+0000","last_undegraded":"2024-08-17T17:42:44.083557+0000","last_fullsized":"2024-08-17T17:42:44.083557+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:02:35.824513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"369'149","reported_seq":993,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.080574+0000","last_change":"2024-08-17T17:41:06.387062+0000","last_active":"2024-08-17T17:42:41.080574+0000","last_peered":"2024-08-17T17:42:41.080574+0000","last_clean":"2024-08-17T17:42:41.080574+0000","last_became_active":"2024-08-17T17:36:15.772779+0000","last_became_peered":"2024-08-17T17:36:15.772779+0000","last_unstale":"2024-08-17T17:42:41.080574+0000","last_undegraded":"2024-08-17T17:42:41.080574+0000","last_fullsized":"2024-08-17T17:42:41.080574+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:41:06.386955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:41:06.386955+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:44:32.008526+0000","scrub_duration":0.0017975980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":52,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.370177+0000","last_change":"2024-08-17T17:41:49.104317+0000","last_active":"2024-08-17T17:42:44.370177+0000","last_peered":"2024-08-17T17:42:44.370177+0000","last_clean":"2024-08-17T17:42:44.370177+0000","last_became_active":"2024-08-17T17:41:49.103786+0000","last_became_peered":"2024-08-17T17:41:49.103786+0000","last_unstale":"2024-08-17T17:42:44.370177+0000","last_undegraded":"2024-08-17T17:42:44.370177+0000","last_fullsized":"2024-08-17T17:42:44.370177+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:41:18.708270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":975,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083692+0000","last_change":"2024-08-17T17:42:06.741570+0000","last_active":"2024-08-17T17:42:44.083692+0000","last_peered":"2024-08-17T17:42:44.083692+0000","last_clean":"2024-08-17T17:42:44.083692+0000","last_became_active":"2024-08-17T17:42:06.741231+0000","last_became_peered":"2024-08-17T17:42:06.741231+0000","last_unstale":"2024-08-17T17:42:44.083692+0000","last_undegraded":"2024-08-17T17:42:44.083692+0000","last_fullsized":"2024-08-17T17:42:44.083692+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:29:55.960941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046361299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1492,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323916+0000","last_change":"2024-08-17T17:36:12.769189+0000","last_active":"2024-08-17T17:42:41.323916+0000","last_peered":"2024-08-17T17:42:41.323916+0000","last_clean":"2024-08-17T17:42:41.323916+0000","last_became_active":"2024-08-17T17:36:12.768644+0000","last_became_peered":"2024-08-17T17:36:12.768644+0000","last_unstale":"2024-08-17T17:42:41.323916+0000","last_undegraded":"2024-08-17T17:42:41.323916+0000","last_fullsized":"2024-08-17T17:42:41.323916+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:23:22.769866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"369'149","reported_seq":976,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.080396+0000","last_change":"2024-08-17T17:41:07.361112+0000","last_active":"2024-08-17T17:42:41.080396+0000","last_peered":"2024-08-17T17:42:41.080396+0000","last_clean":"2024-08-17T17:42:41.080396+0000","last_became_active":"2024-08-17T17:36:15.769502+0000","last_became_peered":"2024-08-17T17:36:15.769502+0000","last_unstale":"2024-08-17T17:42:41.080396+0000","last_undegraded":"2024-08-17T17:42:41.080396+0000","last_fullsized":"2024-08-17T17:42:41.080396+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1386,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:41:07.360968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:41:07.360968+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:34:28.377019+0000","scrub_duration":0.001649951,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":31,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323383+0000","last_change":"2024-08-17T17:41:48.789086+0000","last_active":"2024-08-17T17:42:41.323383+0000","last_peered":"2024-08-17T17:42:41.323383+0000","last_clean":"2024-08-17T17:42:41.323383+0000","last_became_active":"2024-08-17T17:41:48.788726+0000","last_became_peered":"2024-08-17T17:41:48.788726+0000","last_unstale":"2024-08-17T17:42:41.323383+0000","last_undegraded":"2024-08-17T17:42:41.323383+0000","last_fullsized":"2024-08-17T17:42:41.323383+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:16:53.243349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1455,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.145455+0000","last_change":"2024-08-17T17:35:55.205748+0000","last_active":"2024-08-17T17:42:41.145455+0000","last_peered":"2024-08-17T17:42:41.145455+0000","last_clean":"2024-08-17T17:42:41.145455+0000","last_became_active":"2024-08-17T17:35:55.205484+0000","last_became_peered":"2024-08-17T17:35:55.205484+0000","last_unstale":"2024-08-17T17:42:41.145455+0000","last_undegraded":"2024-08-17T17:42:41.145455+0000","last_fullsized":"2024-08-17T17:42:41.145455+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:14:43.618035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1498,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323504+0000","last_change":"2024-08-17T17:35:56.228301+0000","last_active":"2024-08-17T17:42:41.323504+0000","last_peered":"2024-08-17T17:42:41.323504+0000","last_clean":"2024-08-17T17:42:41.323504+0000","last_became_active":"2024-08-17T17:35:56.227971+0000","last_became_peered":"2024-08-17T17:35:56.227971+0000","last_unstale":"2024-08-17T17:42:41.323504+0000","last_undegraded":"2024-08-17T17:42:41.323504+0000","last_fullsized":"2024-08-17T17:42:41.323504+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:08:34.423044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091370199999999996,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"369'149","reported_seq":1813,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322793+0000","last_change":"2024-08-17T17:35:17.839583+0000","last_active":"2024-08-17T17:42:41.322793+0000","last_peered":"2024-08-17T17:42:41.322793+0000","last_clean":"2024-08-17T17:42:41.322793+0000","last_became_active":"2024-08-17T17:35:17.839287+0000","last_became_peered":"2024-08-17T17:35:17.839287+0000","last_unstale":"2024-08-17T17:42:41.322793+0000","last_undegraded":"2024-08-17T17:42:41.322793+0000","last_fullsized":"2024-08-17T17:42:41.322793+0000","mapping_epoch":1366,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1367,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:13.795098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:13.795098+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:08:28.086333+0000","scrub_duration":0.011666005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":52,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.370513+0000","last_change":"2024-08-17T17:41:48.789557+0000","last_active":"2024-08-17T17:42:44.370513+0000","last_peered":"2024-08-17T17:42:44.370513+0000","last_clean":"2024-08-17T17:42:44.370513+0000","last_became_active":"2024-08-17T17:41:48.788842+0000","last_became_peered":"2024-08-17T17:41:48.788842+0000","last_unstale":"2024-08-17T17:42:44.370513+0000","last_undegraded":"2024-08-17T17:42:44.370513+0000","last_fullsized":"2024-08-17T17:42:44.370513+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:23:24.001239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1337,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.082867+0000","last_change":"2024-08-17T17:42:06.743142+0000","last_active":"2024-08-17T17:42:44.082867+0000","last_peered":"2024-08-17T17:42:44.082867+0000","last_clean":"2024-08-17T17:42:44.082867+0000","last_became_active":"2024-08-17T17:42:06.742778+0000","last_became_peered":"2024-08-17T17:42:06.742778+0000","last_unstale":"2024-08-17T17:42:44.082867+0000","last_undegraded":"2024-08-17T17:42:44.082867+0000","last_fullsized":"2024-08-17T17:42:44.082867+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:55:25.982072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2088,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.370240+0000","last_change":"2024-08-17T17:36:12.759792+0000","last_active":"2024-08-17T17:42:44.370240+0000","last_peered":"2024-08-17T17:42:44.370240+0000","last_clean":"2024-08-17T17:42:44.370240+0000","last_became_active":"2024-08-17T17:36:12.759019+0000","last_became_peered":"2024-08-17T17:36:12.759019+0000","last_unstale":"2024-08-17T17:42:44.370240+0000","last_undegraded":"2024-08-17T17:42:44.370240+0000","last_fullsized":"2024-08-17T17:42:44.370240+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:41:19.433773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032714899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"369'149","reported_seq":1691,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322980+0000","last_change":"2024-08-17T17:42:36.909560+0000","last_active":"2024-08-17T17:42:41.322980+0000","last_peered":"2024-08-17T17:42:41.322980+0000","last_clean":"2024-08-17T17:42:41.322980+0000","last_became_active":"2024-08-17T17:42:36.909238+0000","last_became_peered":"2024-08-17T17:42:36.909238+0000","last_unstale":"2024-08-17T17:42:41.322980+0000","last_undegraded":"2024-08-17T17:42:41.322980+0000","last_fullsized":"2024-08-17T17:42:41.322980+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:42:00.619334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:42:00.619334+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:48:10.944008+0000","scrub_duration":0.0068224740000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1278,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083835+0000","last_change":"2024-08-17T17:35:14.851143+0000","last_active":"2024-08-17T17:42:44.083835+0000","last_peered":"2024-08-17T17:42:44.083835+0000","last_clean":"2024-08-17T17:42:44.083835+0000","last_became_active":"2024-08-17T17:35:14.850813+0000","last_became_peered":"2024-08-17T17:35:14.850813+0000","last_unstale":"2024-08-17T17:42:44.083835+0000","last_undegraded":"2024-08-17T17:42:44.083835+0000","last_fullsized":"2024-08-17T17:42:44.083835+0000","mapping_epoch":1363,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:16:37.622504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1049,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083091+0000","last_change":"2024-08-17T17:42:00.208051+0000","last_active":"2024-08-17T17:42:44.083091+0000","last_peered":"2024-08-17T17:42:44.083091+0000","last_clean":"2024-08-17T17:42:44.083091+0000","last_became_active":"2024-08-17T17:42:00.207713+0000","last_became_peered":"2024-08-17T17:42:00.207713+0000","last_unstale":"2024-08-17T17:42:44.083091+0000","last_undegraded":"2024-08-17T17:42:44.083091+0000","last_fullsized":"2024-08-17T17:42:44.083091+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:18:38.887722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1430,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322850+0000","last_change":"2024-08-17T17:42:36.909226+0000","last_active":"2024-08-17T17:42:41.322850+0000","last_peered":"2024-08-17T17:42:41.322850+0000","last_clean":"2024-08-17T17:42:41.322850+0000","last_became_active":"2024-08-17T17:42:36.908947+0000","last_became_peered":"2024-08-17T17:42:36.908947+0000","last_unstale":"2024-08-17T17:42:41.322850+0000","last_undegraded":"2024-08-17T17:42:41.322850+0000","last_fullsized":"2024-08-17T17:42:41.322850+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:38:36.590213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029147800000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"369'149","reported_seq":755,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083707+0000","last_change":"2024-08-17T17:42:07.631252+0000","last_active":"2024-08-17T17:42:44.083707+0000","last_peered":"2024-08-17T17:42:44.083707+0000","last_clean":"2024-08-17T17:42:44.083707+0000","last_became_active":"2024-08-17T17:42:06.239009+0000","last_became_peered":"2024-08-17T17:42:06.239009+0000","last_unstale":"2024-08-17T17:42:44.083707+0000","last_undegraded":"2024-08-17T17:42:44.083707+0000","last_fullsized":"2024-08-17T17:42:44.083707+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":3,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:42:07.631161+0000","last_deep_scrub":"369'149","last_deep_scrub_stamp":"2024-08-17T17:42:07.631161+0000","last_clean_scrub_stamp":"2024-08-17T17:42:07.631161+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:47:28.889605+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1296,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.080619+0000","last_change":"2024-08-17T17:35:55.208094+0000","last_active":"2024-08-17T17:42:41.080619+0000","last_peered":"2024-08-17T17:42:41.080619+0000","last_clean":"2024-08-17T17:42:41.080619+0000","last_became_active":"2024-08-17T17:35:55.207787+0000","last_became_peered":"2024-08-17T17:35:55.207787+0000","last_unstale":"2024-08-17T17:42:41.080619+0000","last_undegraded":"2024-08-17T17:42:41.080619+0000","last_fullsized":"2024-08-17T17:42:41.080619+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:51:44.757919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1771,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108268+0000","last_change":"2024-08-17T17:42:18.850814+0000","last_active":"2024-08-17T17:42:44.108268+0000","last_peered":"2024-08-17T17:42:44.108268+0000","last_clean":"2024-08-17T17:42:44.108268+0000","last_became_active":"2024-08-17T17:42:18.850459+0000","last_became_peered":"2024-08-17T17:42:18.850459+0000","last_unstale":"2024-08-17T17:42:44.108268+0000","last_undegraded":"2024-08-17T17:42:44.108268+0000","last_fullsized":"2024-08-17T17:42:44.108268+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:21:16.138051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1474,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323296+0000","last_change":"2024-08-17T17:35:34.129029+0000","last_active":"2024-08-17T17:42:41.323296+0000","last_peered":"2024-08-17T17:42:41.323296+0000","last_clean":"2024-08-17T17:42:41.323296+0000","last_became_active":"2024-08-17T17:35:34.128654+0000","last_became_peered":"2024-08-17T17:35:34.128654+0000","last_unstale":"2024-08-17T17:42:41.323296+0000","last_undegraded":"2024-08-17T17:42:41.323296+0000","last_fullsized":"2024-08-17T17:42:41.323296+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:52:22.404361+0000","last_clean_scrub_stamp":"2024-08-17T16:52:22.404361+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:12:03.554601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033096500000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"369'149","reported_seq":1677,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322810+0000","last_change":"2024-08-17T17:42:36.906286+0000","last_active":"2024-08-17T17:42:41.322810+0000","last_peered":"2024-08-17T17:42:41.322810+0000","last_clean":"2024-08-17T17:42:41.322810+0000","last_became_active":"2024-08-17T17:42:36.905888+0000","last_became_peered":"2024-08-17T17:42:36.905888+0000","last_unstale":"2024-08-17T17:42:41.322810+0000","last_undegraded":"2024-08-17T17:42:41.322810+0000","last_fullsized":"2024-08-17T17:42:41.322810+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":0,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:58.504387+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:40:58.504387+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:43:03.124798+0000","scrub_duration":0.017597167,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":954,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083610+0000","last_change":"2024-08-17T17:42:06.240562+0000","last_active":"2024-08-17T17:42:44.083610+0000","last_peered":"2024-08-17T17:42:44.083610+0000","last_clean":"2024-08-17T17:42:44.083610+0000","last_became_active":"2024-08-17T17:42:06.240265+0000","last_became_peered":"2024-08-17T17:42:06.240265+0000","last_unstale":"2024-08-17T17:42:44.083610+0000","last_undegraded":"2024-08-17T17:42:44.083610+0000","last_fullsized":"2024-08-17T17:42:44.083610+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:45:14.747935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":181,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108450+0000","last_change":"2024-08-17T17:36:12.872724+0000","last_active":"2024-08-17T17:42:44.108450+0000","last_peered":"2024-08-17T17:42:44.108450+0000","last_clean":"2024-08-17T17:42:44.108450+0000","last_became_active":"2024-08-17T17:36:12.872108+0000","last_became_peered":"2024-08-17T17:36:12.872108+0000","last_unstale":"2024-08-17T17:42:44.108450+0000","last_undegraded":"2024-08-17T17:42:44.108450+0000","last_fullsized":"2024-08-17T17:42:44.108450+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:38:00.526554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"369'149","reported_seq":1805,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322666+0000","last_change":"2024-08-17T17:35:56.225609+0000","last_active":"2024-08-17T17:42:41.322666+0000","last_peered":"2024-08-17T17:42:41.322666+0000","last_clean":"2024-08-17T17:42:41.322666+0000","last_became_active":"2024-08-17T17:35:56.225236+0000","last_became_peered":"2024-08-17T17:35:56.225236+0000","last_unstale":"2024-08-17T17:42:41.322666+0000","last_undegraded":"2024-08-17T17:42:41.322666+0000","last_fullsized":"2024-08-17T17:42:41.322666+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.715814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.715814+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:09.673557+0000","scrub_duration":0.018246897000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1484,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.604582+0000","last_change":"2024-08-17T17:35:56.210239+0000","last_active":"2024-08-17T17:42:44.604582+0000","last_peered":"2024-08-17T17:42:44.604582+0000","last_clean":"2024-08-17T17:42:44.604582+0000","last_became_active":"2024-08-17T17:35:56.209592+0000","last_became_peered":"2024-08-17T17:35:56.209592+0000","last_unstale":"2024-08-17T17:42:44.604582+0000","last_undegraded":"2024-08-17T17:42:44.604582+0000","last_fullsized":"2024-08-17T17:42:44.604582+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:11:36.979051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1456,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.145073+0000","last_change":"2024-08-17T17:35:34.128838+0000","last_active":"2024-08-17T17:42:41.145073+0000","last_peered":"2024-08-17T17:42:41.145073+0000","last_clean":"2024-08-17T17:42:41.145073+0000","last_became_active":"2024-08-17T17:35:34.128488+0000","last_became_peered":"2024-08-17T17:35:34.128488+0000","last_unstale":"2024-08-17T17:42:41.145073+0000","last_undegraded":"2024-08-17T17:42:41.145073+0000","last_fullsized":"2024-08-17T17:42:41.145073+0000","mapping_epoch":1372,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1373,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:08:00.100975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"369'149","reported_seq":805,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083506+0000","last_change":"2024-08-17T17:42:06.615686+0000","last_active":"2024-08-17T17:42:44.083506+0000","last_peered":"2024-08-17T17:42:44.083506+0000","last_clean":"2024-08-17T17:42:44.083506+0000","last_became_active":"2024-08-17T17:42:06.237002+0000","last_became_peered":"2024-08-17T17:42:06.237002+0000","last_unstale":"2024-08-17T17:42:44.083506+0000","last_undegraded":"2024-08-17T17:42:44.083506+0000","last_fullsized":"2024-08-17T17:42:44.083506+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:42:06.615600+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:42:06.615600+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:36:16.880272+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":92,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.606406+0000","last_change":"2024-08-17T17:36:17.898890+0000","last_active":"2024-08-17T17:42:44.606406+0000","last_peered":"2024-08-17T17:42:44.606406+0000","last_clean":"2024-08-17T17:42:44.606406+0000","last_became_active":"2024-08-17T17:36:17.898487+0000","last_became_peered":"2024-08-17T17:36:17.898487+0000","last_unstale":"2024-08-17T17:42:44.606406+0000","last_undegraded":"2024-08-17T17:42:44.606406+0000","last_fullsized":"2024-08-17T17:42:44.606406+0000","mapping_epoch":1387,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:35:50.153718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1318,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.080312+0000","last_change":"2024-08-17T17:35:56.244884+0000","last_active":"2024-08-17T17:42:41.080312+0000","last_peered":"2024-08-17T17:42:41.080312+0000","last_clean":"2024-08-17T17:42:41.080312+0000","last_became_active":"2024-08-17T17:35:56.243784+0000","last_became_peered":"2024-08-17T17:35:56.243784+0000","last_unstale":"2024-08-17T17:42:41.080312+0000","last_undegraded":"2024-08-17T17:42:41.080312+0000","last_fullsized":"2024-08-17T17:42:41.080312+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:26.672545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"369'149","reported_seq":1833,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322590+0000","last_change":"2024-08-17T17:36:20.621317+0000","last_active":"2024-08-17T17:42:41.322590+0000","last_peered":"2024-08-17T17:42:41.322590+0000","last_clean":"2024-08-17T17:42:41.322590+0000","last_became_active":"2024-08-17T17:36:20.620877+0000","last_became_peered":"2024-08-17T17:36:20.620877+0000","last_unstale":"2024-08-17T17:42:41.322590+0000","last_undegraded":"2024-08-17T17:42:41.322590+0000","last_fullsized":"2024-08-17T17:42:41.322590+0000","mapping_epoch":1385,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1390,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:58.712202+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:58.712202+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:30:40.580116+0000","scrub_duration":0.0068017119999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1435,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.605569+0000","last_change":"2024-08-17T17:35:55.212817+0000","last_active":"2024-08-17T17:42:44.605569+0000","last_peered":"2024-08-17T17:42:44.605569+0000","last_clean":"2024-08-17T17:42:44.605569+0000","last_became_active":"2024-08-17T17:35:55.211762+0000","last_became_peered":"2024-08-17T17:35:55.211762+0000","last_unstale":"2024-08-17T17:42:44.605569+0000","last_undegraded":"2024-08-17T17:42:44.605569+0000","last_fullsized":"2024-08-17T17:42:44.605569+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:10:49.413598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":180,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108443+0000","last_change":"2024-08-17T17:42:18.852709+0000","last_active":"2024-08-17T17:42:44.108443+0000","last_peered":"2024-08-17T17:42:44.108443+0000","last_clean":"2024-08-17T17:42:44.108443+0000","last_became_active":"2024-08-17T17:42:18.852335+0000","last_became_peered":"2024-08-17T17:42:18.852335+0000","last_unstale":"2024-08-17T17:42:44.108443+0000","last_undegraded":"2024-08-17T17:42:44.108443+0000","last_fullsized":"2024-08-17T17:42:44.108443+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T19:40:06.494720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"369'149","reported_seq":1670,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323260+0000","last_change":"2024-08-17T17:42:36.908751+0000","last_active":"2024-08-17T17:42:41.323260+0000","last_peered":"2024-08-17T17:42:41.323260+0000","last_clean":"2024-08-17T17:42:41.323260+0000","last_became_active":"2024-08-17T17:42:36.908379+0000","last_became_peered":"2024-08-17T17:42:36.908379+0000","last_unstale":"2024-08-17T17:42:41.323260+0000","last_undegraded":"2024-08-17T17:42:41.323260+0000","last_fullsized":"2024-08-17T17:42:41.323260+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":1136,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:32:48.623505+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:32:48.623505+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:23:07.523356+0000","scrub_duration":0.013346135,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1349,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083469+0000","last_change":"2024-08-17T17:33:25.945691+0000","last_active":"2024-08-17T17:42:44.083469+0000","last_peered":"2024-08-17T17:42:44.083469+0000","last_clean":"2024-08-17T17:42:44.083469+0000","last_became_active":"2024-08-17T17:33:25.945407+0000","last_became_peered":"2024-08-17T17:33:25.945407+0000","last_unstale":"2024-08-17T17:42:44.083469+0000","last_undegraded":"2024-08-17T17:42:44.083469+0000","last_fullsized":"2024-08-17T17:42:44.083469+0000","mapping_epoch":1322,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T18:16:15.774238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1461,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.605982+0000","last_change":"2024-08-17T17:36:12.758220+0000","last_active":"2024-08-17T17:42:44.605982+0000","last_peered":"2024-08-17T17:42:44.605982+0000","last_clean":"2024-08-17T17:42:44.605982+0000","last_became_active":"2024-08-17T17:36:12.757148+0000","last_became_peered":"2024-08-17T17:36:12.757148+0000","last_unstale":"2024-08-17T17:42:44.605982+0000","last_undegraded":"2024-08-17T17:42:44.605982+0000","last_fullsized":"2024-08-17T17:42:44.605982+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:48:57.668279+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"369'149","reported_seq":961,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.145471+0000","last_change":"2024-08-17T17:35:33.147243+0000","last_active":"2024-08-17T17:42:41.145471+0000","last_peered":"2024-08-17T17:42:41.145471+0000","last_clean":"2024-08-17T17:42:41.145471+0000","last_became_active":"2024-08-17T17:35:33.146204+0000","last_became_peered":"2024-08-17T17:35:33.146204+0000","last_unstale":"2024-08-17T17:42:41.145471+0000","last_undegraded":"2024-08-17T17:42:41.145471+0000","last_fullsized":"2024-08-17T17:42:41.145471+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:52.102285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:52.102285+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:57:45.649925+0000","scrub_duration":0.014015184,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1317,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.080455+0000","last_change":"2024-08-17T17:35:56.278423+0000","last_active":"2024-08-17T17:42:41.080455+0000","last_peered":"2024-08-17T17:42:41.080455+0000","last_clean":"2024-08-17T17:42:41.080455+0000","last_became_active":"2024-08-17T17:35:56.278057+0000","last_became_peered":"2024-08-17T17:35:56.278057+0000","last_unstale":"2024-08-17T17:42:41.080455+0000","last_undegraded":"2024-08-17T17:42:41.080455+0000","last_fullsized":"2024-08-17T17:42:41.080455+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T23:08:12.811983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1247,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323524+0000","last_change":"2024-08-17T17:42:36.906733+0000","last_active":"2024-08-17T17:42:41.323524+0000","last_peered":"2024-08-17T17:42:41.323524+0000","last_clean":"2024-08-17T17:42:41.323524+0000","last_became_active":"2024-08-17T17:42:36.905850+0000","last_became_peered":"2024-08-17T17:42:36.905850+0000","last_unstale":"2024-08-17T17:42:41.323524+0000","last_undegraded":"2024-08-17T17:42:41.323524+0000","last_fullsized":"2024-08-17T17:42:41.323524+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:17:52.631460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"369'149","reported_seq":956,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:42.779231+0000","last_change":"2024-08-17T17:40:56.322072+0000","last_active":"2024-08-17T17:42:42.779231+0000","last_peered":"2024-08-17T17:42:42.779231+0000","last_clean":"2024-08-17T17:42:42.779231+0000","last_became_active":"2024-08-17T17:35:55.214466+0000","last_became_peered":"2024-08-17T17:35:55.214466+0000","last_unstale":"2024-08-17T17:42:42.779231+0000","last_undegraded":"2024-08-17T17:42:42.779231+0000","last_fullsized":"2024-08-17T17:42:42.779231+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:56.321976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:40:56.321976+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:47:45.244567+0000","scrub_duration":0.0014456440000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1502,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.144995+0000","last_change":"2024-08-17T17:36:12.751101+0000","last_active":"2024-08-17T17:42:41.144995+0000","last_peered":"2024-08-17T17:42:41.144995+0000","last_clean":"2024-08-17T17:42:41.144995+0000","last_became_active":"2024-08-17T17:36:12.750753+0000","last_became_peered":"2024-08-17T17:36:12.750753+0000","last_unstale":"2024-08-17T17:42:41.144995+0000","last_undegraded":"2024-08-17T17:42:41.144995+0000","last_fullsized":"2024-08-17T17:42:41.144995+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:09:36.481641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1296,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323333+0000","last_change":"2024-08-17T17:42:36.910060+0000","last_active":"2024-08-17T17:42:41.323333+0000","last_peered":"2024-08-17T17:42:41.323333+0000","last_clean":"2024-08-17T17:42:41.323333+0000","last_became_active":"2024-08-17T17:42:36.909789+0000","last_became_peered":"2024-08-17T17:42:36.909789+0000","last_unstale":"2024-08-17T17:42:41.323333+0000","last_undegraded":"2024-08-17T17:42:41.323333+0000","last_fullsized":"2024-08-17T17:42:41.323333+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T04:38:33.809561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"369'149","reported_seq":1781,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108560+0000","last_change":"2024-08-17T17:40:55.554364+0000","last_active":"2024-08-17T17:42:44.108560+0000","last_peered":"2024-08-17T17:42:44.108560+0000","last_clean":"2024-08-17T17:42:44.108560+0000","last_became_active":"2024-08-17T17:37:04.031743+0000","last_became_peered":"2024-08-17T17:37:04.031743+0000","last_unstale":"2024-08-17T17:42:44.108560+0000","last_undegraded":"2024-08-17T17:42:44.108560+0000","last_fullsized":"2024-08-17T17:42:44.108560+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:40:55.554234+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:40:55.554234+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:52:22.208370+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1426,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322792+0000","last_change":"2024-08-17T17:35:55.217391+0000","last_active":"2024-08-17T17:42:41.322792+0000","last_peered":"2024-08-17T17:42:41.322792+0000","last_clean":"2024-08-17T17:42:41.322792+0000","last_became_active":"2024-08-17T17:35:55.216250+0000","last_became_peered":"2024-08-17T17:35:55.216250+0000","last_unstale":"2024-08-17T17:42:41.322792+0000","last_undegraded":"2024-08-17T17:42:41.322792+0000","last_fullsized":"2024-08-17T17:42:41.322792+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:48:18.689770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1282,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323331+0000","last_change":"2024-08-17T17:42:36.910203+0000","last_active":"2024-08-17T17:42:41.323331+0000","last_peered":"2024-08-17T17:42:41.323331+0000","last_clean":"2024-08-17T17:42:41.323331+0000","last_became_active":"2024-08-17T17:42:36.909884+0000","last_became_peered":"2024-08-17T17:42:36.909884+0000","last_unstale":"2024-08-17T17:42:41.323331+0000","last_undegraded":"2024-08-17T17:42:41.323331+0000","last_fullsized":"2024-08-17T17:42:41.323331+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:14:50.728898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"369'149","reported_seq":1540,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323713+0000","last_change":"2024-08-17T17:42:36.911100+0000","last_active":"2024-08-17T17:42:41.323713+0000","last_peered":"2024-08-17T17:42:41.323713+0000","last_clean":"2024-08-17T17:42:41.323713+0000","last_became_active":"2024-08-17T17:42:36.910795+0000","last_became_peered":"2024-08-17T17:42:36.910795+0000","last_unstale":"2024-08-17T17:42:41.323713+0000","last_undegraded":"2024-08-17T17:42:41.323713+0000","last_fullsized":"2024-08-17T17:42:41.323713+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":4,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:34:42.279073+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:34:42.279073+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T20:18:27.586094+0000","scrub_duration":0.0066315480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1358,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.080082+0000","last_change":"2024-08-17T17:35:55.214563+0000","last_active":"2024-08-17T17:42:44.080082+0000","last_peered":"2024-08-17T17:42:44.080082+0000","last_clean":"2024-08-17T17:42:44.080082+0000","last_became_active":"2024-08-17T17:35:55.214168+0000","last_became_peered":"2024-08-17T17:35:55.214168+0000","last_unstale":"2024-08-17T17:42:44.080082+0000","last_undegraded":"2024-08-17T17:42:44.080082+0000","last_fullsized":"2024-08-17T17:42:44.080082+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T01:44:58.563896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":180,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108781+0000","last_change":"2024-08-17T17:36:12.761184+0000","last_active":"2024-08-17T17:42:44.108781+0000","last_peered":"2024-08-17T17:42:44.108781+0000","last_clean":"2024-08-17T17:42:44.108781+0000","last_became_active":"2024-08-17T17:36:12.760714+0000","last_became_peered":"2024-08-17T17:36:12.760714+0000","last_unstale":"2024-08-17T17:42:44.108781+0000","last_undegraded":"2024-08-17T17:42:44.108781+0000","last_fullsized":"2024-08-17T17:42:44.108781+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:22:20.254896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"369'149","reported_seq":1789,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.604777+0000","last_change":"2024-08-17T17:35:56.228020+0000","last_active":"2024-08-17T17:42:44.604777+0000","last_peered":"2024-08-17T17:42:44.604777+0000","last_clean":"2024-08-17T17:42:44.604777+0000","last_became_active":"2024-08-17T17:35:56.227535+0000","last_became_peered":"2024-08-17T17:35:56.227535+0000","last_unstale":"2024-08-17T17:42:44.604777+0000","last_undegraded":"2024-08-17T17:42:44.604777+0000","last_fullsized":"2024-08-17T17:42:44.604777+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.727086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.727086+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:18:14.786237+0000","scrub_duration":0.0074436959999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1288,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322705+0000","last_change":"2024-08-17T17:42:36.910900+0000","last_active":"2024-08-17T17:42:41.322705+0000","last_peered":"2024-08-17T17:42:41.322705+0000","last_clean":"2024-08-17T17:42:41.322705+0000","last_became_active":"2024-08-17T17:42:36.910614+0000","last_became_peered":"2024-08-17T17:42:36.910614+0000","last_unstale":"2024-08-17T17:42:41.322705+0000","last_undegraded":"2024-08-17T17:42:41.322705+0000","last_fullsized":"2024-08-17T17:42:41.322705+0000","mapping_epoch":1463,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1464,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T22:33:55.103679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1028,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083745+0000","last_change":"2024-08-17T17:42:00.211263+0000","last_active":"2024-08-17T17:42:44.083745+0000","last_peered":"2024-08-17T17:42:44.083745+0000","last_clean":"2024-08-17T17:42:44.083745+0000","last_became_active":"2024-08-17T17:42:00.210948+0000","last_became_peered":"2024-08-17T17:42:00.210948+0000","last_unstale":"2024-08-17T17:42:44.083745+0000","last_undegraded":"2024-08-17T17:42:44.083745+0000","last_fullsized":"2024-08-17T17:42:44.083745+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:21:53.580243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"369'149","reported_seq":980,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083406+0000","last_change":"2024-08-17T17:42:06.239969+0000","last_active":"2024-08-17T17:42:44.083406+0000","last_peered":"2024-08-17T17:42:44.083406+0000","last_clean":"2024-08-17T17:42:44.083406+0000","last_became_active":"2024-08-17T17:42:06.239603+0000","last_became_peered":"2024-08-17T17:42:06.239603+0000","last_unstale":"2024-08-17T17:42:44.083406+0000","last_undegraded":"2024-08-17T17:42:44.083406+0000","last_fullsized":"2024-08-17T17:42:44.083406+0000","mapping_epoch":1451,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1452,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:50.879964+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:50.879964+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T02:19:18.882142+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1266,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083205+0000","last_change":"2024-08-17T17:36:12.754903+0000","last_active":"2024-08-17T17:42:44.083205+0000","last_peered":"2024-08-17T17:42:44.083205+0000","last_clean":"2024-08-17T17:42:44.083205+0000","last_became_active":"2024-08-17T17:36:12.754224+0000","last_became_peered":"2024-08-17T17:36:12.754224+0000","last_unstale":"2024-08-17T17:42:44.083205+0000","last_undegraded":"2024-08-17T17:42:44.083205+0000","last_fullsized":"2024-08-17T17:42:44.083205+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:33:03.164926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1458,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.323759+0000","last_change":"2024-08-17T17:36:12.768977+0000","last_active":"2024-08-17T17:42:41.323759+0000","last_peered":"2024-08-17T17:42:41.323759+0000","last_clean":"2024-08-17T17:42:41.323759+0000","last_became_active":"2024-08-17T17:36:12.768236+0000","last_became_peered":"2024-08-17T17:36:12.768236+0000","last_unstale":"2024-08-17T17:42:41.323759+0000","last_undegraded":"2024-08-17T17:42:41.323759+0000","last_fullsized":"2024-08-17T17:42:41.323759+0000","mapping_epoch":1382,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:53:33.867400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"369'149","reported_seq":1752,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.605129+0000","last_change":"2024-08-17T17:35:55.216926+0000","last_active":"2024-08-17T17:42:44.605129+0000","last_peered":"2024-08-17T17:42:44.605129+0000","last_clean":"2024-08-17T17:42:44.605129+0000","last_became_active":"2024-08-17T17:35:55.215981+0000","last_became_peered":"2024-08-17T17:35:55.215981+0000","last_unstale":"2024-08-17T17:42:44.605129+0000","last_undegraded":"2024-08-17T17:42:44.605129+0000","last_fullsized":"2024-08-17T17:42:44.605129+0000","mapping_epoch":1376,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1377,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:24:48.765182+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:24:48.765182+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T03:15:58.893809+0000","scrub_duration":0.021806177999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1400,"reported_epoch":1465,"state":"active+clean","last_fresh":"2024-08-17T17:42:41.322962+0000","last_change":"2024-08-17T17:35:33.151462+0000","last_active":"2024-08-17T17:42:41.322962+0000","last_peered":"2024-08-17T17:42:41.322962+0000","last_clean":"2024-08-17T17:42:41.322962+0000","last_became_active":"2024-08-17T17:35:33.147916+0000","last_became_peered":"2024-08-17T17:35:33.147916+0000","last_unstale":"2024-08-17T17:42:41.322962+0000","last_undegraded":"2024-08-17T17:42:41.322962+0000","last_fullsized":"2024-08-17T17:42:41.322962+0000","mapping_epoch":1371,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:35.850756+0000","last_clean_scrub_stamp":"2024-08-17T16:53:35.850756+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T17:09:12.182008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1982,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.108865+0000","last_change":"2024-08-17T17:42:18.995663+0000","last_active":"2024-08-17T17:42:44.108865+0000","last_peered":"2024-08-17T17:42:44.108865+0000","last_clean":"2024-08-17T17:42:44.108865+0000","last_became_active":"2024-08-17T17:42:18.995326+0000","last_became_peered":"2024-08-17T17:42:18.995326+0000","last_unstale":"2024-08-17T17:42:44.108865+0000","last_undegraded":"2024-08-17T17:42:44.108865+0000","last_fullsized":"2024-08-17T17:42:44.108865+0000","mapping_epoch":1456,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:53:36.856414+0000","last_clean_scrub_stamp":"2024-08-17T16:53:36.856414+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-08-18T21:53:08.837477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"369'149","reported_seq":1454,"reported_epoch":1466,"state":"active+clean","last_fresh":"2024-08-17T17:42:44.083259+0000","last_change":"2024-08-17T17:42:01.636437+0000","last_active":"2024-08-17T17:42:44.083259+0000","last_peered":"2024-08-17T17:42:44.083259+0000","last_clean":"2024-08-17T17:42:44.083259+0000","last_became_active":"2024-08-17T17:42:00.209411+0000","last_became_peered":"2024-08-17T17:42:00.209411+0000","last_unstale":"2024-08-17T17:42:44.083259+0000","last_undegraded":"2024-08-17T17:42:44.083259+0000","last_fullsized":"2024-08-17T17:42:44.083259+0000","mapping_epoch":1448,"log_start":"0'0","ondisk_log_start":"0'0","created":1214,"last_epoch_clean":1449,"parent":"0.0","parent_split_bits":5,"last_scrub":"369'149","last_scrub_stamp":"2024-08-17T17:42:01.636355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-08-17T16:49:54.582155+0000","last_clean_scrub_stamp":"2024-08-17T17:42:01.636355+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-08-19T00:34:03.015493+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":626,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22338,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":1371,"seq":5888400162913,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":172860,"kb_used_data":8240,"kb_used_omap":54,"kb_used_meta":164553,"kb_avail":93572292,"statfs":{"total":95995035648,"available":95818027008,"internally_reserved":0,"allocated":8437760,"data_stored":5571815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56299,"internal_metadata":168502293},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":1376,"seq":5909874999387,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":230324,"kb_used_data":8296,"kb_used_omap":47,"kb_used_meta":221968,"kb_avail":93514828,"statfs":{"total":95995035648,"available":95759183872,"internally_reserved":0,"allocated":8495104,"data_stored":5637722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48492,"internal_metadata":227295892},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":837,"seq":3594887627184,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1116960,"kb_used_data":8276,"kb_used_omap":42,"kb_used_meta":1108629,"kb_avail":92628192,"statfs":{"total":95995035648,"available":94851268608,"internally_reserved":0,"allocated":8474624,"data_stored":5631937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":43972,"internal_metadata":1135236156},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":771,"seq":3311419785661,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":860708,"kb_used_data":7960,"kb_used_omap":48,"kb_used_meta":852687,"kb_avail":92884444,"statfs":{"total":95995035648,"available":95113670656,"internally_reserved":0,"allocated":8151040,"data_stored":5305228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49490,"internal_metadata":873152174},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725178,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":901484,"kb_used_data":8224,"kb_used_omap":1,"kb_used_meta":893246,"kb_avail":92843668,"statfs":{"total":95995035648,"available":95071916032,"internally_reserved":0,"allocated":8421376,"data_stored":5567514,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":914684358},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986815,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":919284,"kb_used_data":7976,"kb_used_omap":1,"kb_used_meta":911294,"kb_avail":92825868,"statfs":{"total":95995035648,"available":95053688832,"internally_reserved":0,"allocated":8167424,"data_stored":5309057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":933165510},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":804,"seq":3453153706422,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":942632,"kb_used_data":8220,"kb_used_omap":35,"kb_used_meta":934364,"kb_avail":92802520,"statfs":{"total":95995035648,"available":95029780480,"internally_reserved":0,"allocated":8417280,"data_stored":5567471,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36809,"internal_metadata":956788791},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":739,"seq":3173980832195,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1061096,"kb_used_data":8284,"kb_used_omap":65,"kb_used_meta":1052734,"kb_avail":92684056,"statfs":{"total":95995035648,"available":94908473344,"internally_reserved":0,"allocated":8482816,"data_stored":5633493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67397,"internal_metadata":1077999803},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3019,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3678,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1990,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3005,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5335,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5311,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-08-17T17:43:32.741 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-08-17T17:43:32.742 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi063 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-08-17T17:43:32.916 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=38ba9d9ff7804d9599fbeffdbe34ef3a Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_91f88e4d132ac08dc53355fa3d9717a53236a3a9/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi063 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-08-17T17:43:32.919 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-08-17T17:43:32.928 INFO:tasks.cephadm:Teardown begin 2024-08-17T17:43:32.928 DEBUG:teuthology.orchestra.run.smithi063:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T17:43:32.970 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:32 smithi063 ceph-mon[114756]: from='client.36879 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:32.970 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:32 smithi063 ceph-mon[109843]: from='client.36879 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:32.994 DEBUG:teuthology.orchestra.run.smithi162:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T17:43:33.038 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-08-17T17:43:33.038 DEBUG:teuthology.orchestra.run.smithi063:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-08-17T17:43:33.055 DEBUG:teuthology.orchestra.run.smithi162:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-08-17T17:43:33.095 INFO:tasks.cephadm:Stopping all daemons... 2024-08-17T17:43:33.096 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-08-17T17:43:33.096 DEBUG:teuthology.orchestra.run.smithi063:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.a 2024-08-17T17:43:33.096 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:32 smithi162 ceph-mon[104668]: from='client.36879 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-08-17T17:43:33.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:33 smithi063 systemd[1]: Stopping Ceph mon.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:43:33.646 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: ::ffff:172.21.15.162 - - [17/Aug/2024:17:43:33] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.51.0" 2024-08-17T17:43:33.646 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a[109820]: 2024-08-17T17:43:33.459+0000 7f6702a15640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:43:33.646 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:33 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a[109820]: 2024-08-17T17:43:33.459+0000 7f6702a15640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-08-17T17:43:33.961 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:33 smithi063 podman[392484]: 2024-08-17 17:43:33.645393167 +0000 UTC m=+0.340621312 container died f0f20fcac9560a4aa8aa604ab9353004e307d744ed2b7303116a54290134b903 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a, org.label-schema.build-date=20240731, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True) 2024-08-17T17:43:34.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:34 smithi063 podman[392484]: 2024-08-17 17:43:34.05142614 +0000 UTC m=+0.746654291 container cleanup f0f20fcac9560a4aa8aa604ab9353004e307d744ed2b7303116a54290134b903 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-08-17T17:43:34.279 INFO:journalctl@ceph.mon.a.smithi063.stdout:Aug 17 17:43:34 smithi063 bash[392484]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-a 2024-08-17T17:43:34.442 DEBUG:teuthology.orchestra.run.smithi063:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.a.service' 2024-08-17T17:43:34.498 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:43:34.498 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-08-17T17:43:34.498 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-08-17T17:43:34.498 DEBUG:teuthology.orchestra.run.smithi063:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.c 2024-08-17T17:43:34.779 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:34 smithi063 systemd[1]: Stopping Ceph mon.c for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:43:35.183 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:34 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:17:43:34] ENGINE Bus STOPPING 2024-08-17T17:43:35.183 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:34 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c[114733]: 2024-08-17T17:43:34.842+0000 7f05ee712640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:43:35.183 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:34 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c[114733]: 2024-08-17T17:43:34.842+0000 7f05ee712640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-08-17T17:43:35.183 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:34 smithi063 podman[392615]: 2024-08-17 17:43:34.936315406 +0000 UTC m=+0.245901249 container died c484af026acfb0ab96471e564a355dbb89606b66094497857f1ee222e49e7dbc (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-08-17T17:43:35.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:35 smithi063 podman[392615]: 2024-08-17 17:43:35.39818563 +0000 UTC m=+0.707771462 container cleanup c484af026acfb0ab96471e564a355dbb89606b66094497857f1ee222e49e7dbc (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, GIT_CLEAN=True, maintainer=Guillaume Abrioux ) 2024-08-17T17:43:35.529 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:35 smithi063 bash[392615]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c 2024-08-17T17:43:35.530 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:17:43:35] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-08-17T17:43:35.530 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:17:43:35] ENGINE Bus STOPPED 2024-08-17T17:43:35.530 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:17:43:35] ENGINE Bus STARTING 2024-08-17T17:43:35.530 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:17:43:35] ENGINE Serving on http://:::9283 2024-08-17T17:43:35.530 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:35 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: [17/Aug/2024:17:43:35] ENGINE Bus STARTED 2024-08-17T17:43:35.826 INFO:journalctl@ceph.mon.c.smithi063.stdout:Aug 17 17:43:35 smithi063 podman[392631]: 2024-08-17 17:43:35.566481218 +0000 UTC m=+0.627525392 container remove c484af026acfb0ab96471e564a355dbb89606b66094497857f1ee222e49e7dbc (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-c, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-08-17T17:43:35.830 DEBUG:teuthology.orchestra.run.smithi063:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.c.service' 2024-08-17T17:43:35.865 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:43:35.865 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-08-17T17:43:35.865 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-08-17T17:43:35.865 DEBUG:teuthology.orchestra.run.smithi162:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.b 2024-08-17T17:43:36.182 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:35 smithi162 systemd[1]: Stopping Ceph mon.b for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:43:36.595 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b[104645]: 2024-08-17T17:43:36.180+0000 7f2dba60b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:43:36.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b[104645]: 2024-08-17T17:43:36.180+0000 7f2dba60b640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-08-17T17:43:36.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:36 smithi162 podman[261206]: 2024-08-17 17:43:36.394806561 +0000 UTC m=+0.325599680 container stop 520525c79dbe79052855f0ae2fc8e57188f8d133ac6c42f8200c78ff3ea2ac8e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-08-17T17:43:36.596 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:36 smithi162 podman[261206]: 2024-08-17 17:43:36.424045455 +0000 UTC m=+0.354838569 container died 520525c79dbe79052855f0ae2fc8e57188f8d133ac6c42f8200c78ff3ea2ac8e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b, ceph=True, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-08-17T17:43:37.055 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:36 smithi162 podman[261206]: 2024-08-17 17:43:36.819667652 +0000 UTC m=+0.750460754 container cleanup 520525c79dbe79052855f0ae2fc8e57188f8d133ac6c42f8200c78ff3ea2ac8e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe) 2024-08-17T17:43:37.055 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:36 smithi162 bash[261206]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b 2024-08-17T17:43:37.283 DEBUG:teuthology.orchestra.run.smithi162:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.b.service' 2024-08-17T17:43:37.305 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:37 smithi162 podman[261223]: 2024-08-17 17:43:37.054750739 +0000 UTC m=+0.653848234 container remove 520525c79dbe79052855f0ae2fc8e57188f8d133ac6c42f8200c78ff3ea2ac8e (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mon-b, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe) 2024-08-17T17:43:37.305 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:37 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.b.service: Deactivated successfully. 2024-08-17T17:43:37.305 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:37 smithi162 systemd[1]: Stopped Ceph mon.b for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:43:37.306 INFO:journalctl@ceph.mon.b.smithi162.stdout:Aug 17 17:43:37 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mon.b.service: Consumed 26.474s CPU time. 2024-08-17T17:43:37.321 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:43:37.321 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-08-17T17:43:37.321 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-08-17T17:43:37.321 DEBUG:teuthology.orchestra.run.smithi063:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.y 2024-08-17T17:43:37.648 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:37 smithi063 systemd[1]: Stopping Ceph mgr.y for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:43:37.965 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:37 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T17:43:37.645+0000 7f46b0599640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:43:37.965 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:37 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y[91656]: 2024-08-17T17:43:37.645+0000 7f46b0599640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-08-17T17:43:37.966 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:37 smithi063 podman[392756]: 2024-08-17 17:43:37.678797076 +0000 UTC m=+0.152085621 container died 943e639cede62db3b1275abf348fc58bfaf9f5173172413b82f4a5875f340ed5 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-08-17T17:43:38.279 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:38 smithi063 podman[392756]: 2024-08-17 17:43:38.153633409 +0000 UTC m=+0.626921942 container cleanup 943e639cede62db3b1275abf348fc58bfaf9f5173172413b82f4a5875f340ed5 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, ceph=True) 2024-08-17T17:43:38.279 INFO:journalctl@ceph.mgr.y.smithi063.stdout:Aug 17 17:43:38 smithi063 bash[392756]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-y 2024-08-17T17:43:38.599 DEBUG:teuthology.orchestra.run.smithi063:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.y.service' 2024-08-17T17:43:38.634 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:43:38.635 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-08-17T17:43:38.635 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-08-17T17:43:38.635 DEBUG:teuthology.orchestra.run.smithi162:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.x 2024-08-17T17:43:38.933 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 17:43:38 smithi162 systemd[1]: Stopping Ceph mgr.x for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:43:39.303 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 17:43:39 smithi162 podman[261338]: 2024-08-17 17:43:39.024592072 +0000 UTC m=+0.184165693 container died da48034a2895b6b37b4c56937841f60a360da1ab77fe946fc286922cecba9d5d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:43:39.596 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 17:43:39 smithi162 podman[261338]: 2024-08-17 17:43:39.460359726 +0000 UTC m=+0.619933351 container cleanup da48034a2895b6b37b4c56937841f60a360da1ab77fe946fc286922cecba9d5d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-08-17T17:43:39.596 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 17:43:39 smithi162 bash[261338]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x 2024-08-17T17:43:39.953 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 17:43:39 smithi162 podman[261359]: 2024-08-17 17:43:39.69491619 +0000 UTC m=+0.660907789 container remove da48034a2895b6b37b4c56937841f60a360da1ab77fe946fc286922cecba9d5d (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-mgr-x, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:43:39.953 INFO:journalctl@ceph.mgr.x.smithi162.stdout:Aug 17 17:43:39 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:43:39.957 DEBUG:teuthology.orchestra.run.smithi162:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@mgr.x.service' 2024-08-17T17:43:39.995 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:43:39.996 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-08-17T17:43:39.996 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-08-17T17:43:39.996 DEBUG:teuthology.orchestra.run.smithi063:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.0 2024-08-17T17:43:40.279 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:43:40 smithi063 systemd[1]: Stopping Ceph osd.0 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:43:40.779 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:43:40 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0[127633]: 2024-08-17T17:43:40.335+0000 7fc588abc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:43:40.779 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:43:40 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0[127633]: 2024-08-17T17:43:40.335+0000 7fc588abc640 -1 osd.0 1466 *** Got signal Terminated *** 2024-08-17T17:43:40.779 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:43:40 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0[127633]: 2024-08-17T17:43:40.335+0000 7fc588abc640 -1 osd.0 1466 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:43:46.110 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:43:45 smithi063 podman[392891]: 2024-08-17 17:43:45.839381256 +0000 UTC m=+5.634528219 container died 8fa117adbc0a2a8169a1b8c79158a54be6f036a4208aa503d68c13eaedf975db (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True) 2024-08-17T17:43:46.529 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:43:46 smithi063 podman[392891]: 2024-08-17 17:43:46.291897034 +0000 UTC m=+6.087043996 container cleanup 8fa117adbc0a2a8169a1b8c79158a54be6f036a4208aa503d68c13eaedf975db (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-08-17T17:43:46.529 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:43:46 smithi063 bash[392891]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0 2024-08-17T17:43:46.944 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:43:46 smithi063 podman[392911]: 2024-08-17 17:43:46.593671534 +0000 UTC m=+0.757422547 container remove 8fa117adbc0a2a8169a1b8c79158a54be6f036a4208aa503d68c13eaedf975db (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-08-17T17:43:47.206 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:43:46 smithi063 podman[393003]: 2024-08-17 17:43:46.944433199 +0000 UTC m=+0.096747831 container create 5745412de71c186f0a76bfcaf8a91b6ea840eb8a8ec9c36587cc21adc843bd5c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, ceph=True, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-08-17T17:43:47.206 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:43:46 smithi063 podman[393003]: 2024-08-17 17:43:46.866029175 +0000 UTC m=+0.018343808 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:43:47.206 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:43:47 smithi063 podman[393003]: 2024-08-17 17:43:47.205808919 +0000 UTC m=+0.358123565 container init 5745412de71c186f0a76bfcaf8a91b6ea840eb8a8ec9c36587cc21adc843bd5c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-08-17T17:43:47.529 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:43:47 smithi063 podman[393003]: 2024-08-17 17:43:47.210743874 +0000 UTC m=+0.363058508 container start 5745412de71c186f0a76bfcaf8a91b6ea840eb8a8ec9c36587cc21adc843bd5c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:43:47.529 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:43:47 smithi063 podman[393003]: 2024-08-17 17:43:47.364177373 +0000 UTC m=+0.516492095 container attach 5745412de71c186f0a76bfcaf8a91b6ea840eb8a8ec9c36587cc21adc843bd5c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS) 2024-08-17T17:43:48.400 INFO:journalctl@ceph.osd.0.smithi063.stdout:Aug 17 17:43:48 smithi063 podman[393047]: 2024-08-17 17:43:48.144102847 +0000 UTC m=+0.018363304 container died 5745412de71c186f0a76bfcaf8a91b6ea840eb8a8ec9c36587cc21adc843bd5c (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-08-17T17:43:48.758 DEBUG:teuthology.orchestra.run.smithi063:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.0.service' 2024-08-17T17:43:48.806 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:43:48.806 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-08-17T17:43:48.806 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-08-17T17:43:48.806 DEBUG:teuthology.orchestra.run.smithi063:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.1 2024-08-17T17:43:49.029 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:43:48 smithi063 systemd[1]: Stopping Ceph osd.1 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:43:49.529 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:43:49 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1[132733]: 2024-08-17T17:43:49.139+0000 7f77cedf8640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:43:49.529 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:43:49 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1[132733]: 2024-08-17T17:43:49.139+0000 7f77cedf8640 -1 osd.1 1466 *** Got signal Terminated *** 2024-08-17T17:43:49.529 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:43:49 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1[132733]: 2024-08-17T17:43:49.139+0000 7f77cedf8640 -1 osd.1 1466 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:43:54.973 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:43:54 smithi063 podman[393131]: 2024-08-17 17:43:54.695563199 +0000 UTC m=+5.690536834 container died 11bee521ecad9e307f63ab6b94f326c236dc9b4dac1f5e4d986f9c3394bea650 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-cd8bdbe, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-08-17T17:43:55.279 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:43:55 smithi063 podman[393131]: 2024-08-17 17:43:55.146500325 +0000 UTC m=+6.141473992 container cleanup 11bee521ecad9e307f63ab6b94f326c236dc9b4dac1f5e4d986f9c3394bea650 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux ) 2024-08-17T17:43:55.279 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:43:55 smithi063 bash[393131]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1 2024-08-17T17:43:55.779 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:43:55 smithi063 podman[393151]: 2024-08-17 17:43:55.473020418 +0000 UTC m=+0.777941448 container remove 11bee521ecad9e307f63ab6b94f326c236dc9b4dac1f5e4d986f9c3394bea650 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-cd8bdbe, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux ) 2024-08-17T17:43:56.135 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:43:55 smithi063 podman[393246]: 2024-08-17 17:43:55.712845297 +0000 UTC m=+0.018147463 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:43:56.135 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:43:55 smithi063 podman[393246]: 2024-08-17 17:43:55.815346933 +0000 UTC m=+0.120649105 container create 433eb443a30888e7e1d18679c59bead7b79a2fcca1d47881e4d06532bb59388a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-08-17T17:43:56.135 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:43:56 smithi063 podman[393246]: 2024-08-17 17:43:56.051565895 +0000 UTC m=+0.356868070 container init 433eb443a30888e7e1d18679c59bead7b79a2fcca1d47881e4d06532bb59388a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-08-17T17:43:56.135 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:43:56 smithi063 podman[393246]: 2024-08-17 17:43:56.056405796 +0000 UTC m=+0.361707964 container start 433eb443a30888e7e1d18679c59bead7b79a2fcca1d47881e4d06532bb59388a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=main-cd8bdbe, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-08-17T17:43:56.529 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:43:56 smithi063 podman[393246]: 2024-08-17 17:43:56.134646389 +0000 UTC m=+0.439948568 container attach 433eb443a30888e7e1d18679c59bead7b79a2fcca1d47881e4d06532bb59388a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True) 2024-08-17T17:43:56.530 INFO:journalctl@ceph.osd.1.smithi063.stdout:Aug 17 17:43:56 smithi063 podman[393246]: 2024-08-17 17:43:56.257352402 +0000 UTC m=+0.562654568 container died 433eb443a30888e7e1d18679c59bead7b79a2fcca1d47881e4d06532bb59388a (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-1-deactivate, ceph=True, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-08-17T17:43:56.901 DEBUG:teuthology.orchestra.run.smithi063:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.1.service' 2024-08-17T17:43:56.941 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:43:56.942 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-08-17T17:43:56.942 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-08-17T17:43:56.942 DEBUG:teuthology.orchestra.run.smithi063:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.2 2024-08-17T17:43:57.269 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:43:57 smithi063 systemd[1]: Stopping Ceph osd.2 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:43:57.529 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:43:57 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2[137556]: 2024-08-17T17:43:57.268+0000 7f66e5595640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:43:57.529 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:43:57 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2[137556]: 2024-08-17T17:43:57.268+0000 7f66e5595640 -1 osd.2 1466 *** Got signal Terminated *** 2024-08-17T17:43:57.529 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:43:57 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2[137556]: 2024-08-17T17:43:57.268+0000 7f66e5595640 -1 osd.2 1466 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:44:03.279 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:44:02 smithi063 podman[393383]: 2024-08-17 17:44:02.962888735 +0000 UTC m=+5.783240582 container died 107c38f8799493fe138746caff84da51f6436ba149bf9bbf1267a49fe5acd075 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:44:03.710 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:44:03 smithi063 podman[393383]: 2024-08-17 17:44:03.433528892 +0000 UTC m=+6.253880739 container cleanup 107c38f8799493fe138746caff84da51f6436ba149bf9bbf1267a49fe5acd075 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-cd8bdbe, io.buildah.version=1.36.0) 2024-08-17T17:44:03.711 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:44:03 smithi063 bash[393383]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2 2024-08-17T17:44:04.029 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:44:03 smithi063 podman[393406]: 2024-08-17 17:44:03.71019703 +0000 UTC m=+0.748155056 container remove 107c38f8799493fe138746caff84da51f6436ba149bf9bbf1267a49fe5acd075 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, RELEASE=main-cd8bdbe) 2024-08-17T17:44:04.313 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:44:04 smithi063 podman[393498]: 2024-08-17 17:44:04.036158391 +0000 UTC m=+0.114417878 container create 874a4dc21b7dbdf8c7f940e00b1094def0ad27607a37e49ee395164fe105b8de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, RELEASE=main-cd8bdbe, CEPH_POINT_RELEASE=) 2024-08-17T17:44:04.314 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:44:04 smithi063 podman[393498]: 2024-08-17 17:44:03.939962086 +0000 UTC m=+0.018221545 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:44:04.314 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:44:04 smithi063 podman[393498]: 2024-08-17 17:44:04.23825122 +0000 UTC m=+0.316510679 container init 874a4dc21b7dbdf8c7f940e00b1094def0ad27607a37e49ee395164fe105b8de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, RELEASE=main-cd8bdbe, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:44:04.314 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:44:04 smithi063 podman[393498]: 2024-08-17 17:44:04.243064323 +0000 UTC m=+0.321323770 container start 874a4dc21b7dbdf8c7f940e00b1094def0ad27607a37e49ee395164fe105b8de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD) 2024-08-17T17:44:04.314 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:44:04 smithi063 podman[393498]: 2024-08-17 17:44:04.313170179 +0000 UTC m=+0.391429631 container attach 874a4dc21b7dbdf8c7f940e00b1094def0ad27607a37e49ee395164fe105b8de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=main-cd8bdbe, GIT_CLEAN=True, ceph=True) 2024-08-17T17:44:04.725 INFO:journalctl@ceph.osd.2.smithi063.stdout:Aug 17 17:44:04 smithi063 podman[393498]: 2024-08-17 17:44:04.434929302 +0000 UTC m=+0.513188759 container died 874a4dc21b7dbdf8c7f940e00b1094def0ad27607a37e49ee395164fe105b8de (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe) 2024-08-17T17:44:05.012 DEBUG:teuthology.orchestra.run.smithi063:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.2.service' 2024-08-17T17:44:05.051 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:44:05.051 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-08-17T17:44:05.052 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-08-17T17:44:05.052 DEBUG:teuthology.orchestra.run.smithi063:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.3 2024-08-17T17:44:05.279 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:05 smithi063 systemd[1]: Stopping Ceph osd.3 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:44:05.779 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:05 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3[142377]: 2024-08-17T17:44:05.388+0000 7f2f959e1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:44:05.779 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:05 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3[142377]: 2024-08-17T17:44:05.388+0000 7f2f959e1640 -1 osd.3 1466 *** Got signal Terminated *** 2024-08-17T17:44:05.780 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:05 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3[142377]: 2024-08-17T17:44:05.388+0000 7f2f959e1640 -1 osd.3 1466 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:44:07.346 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:06 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:06.995+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:07.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:07 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:07.489+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:08.029 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:07 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3[142377]: 2024-08-17T17:44:07.716+0000 7f2f91ff6640 -1 osd.3 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:44.116274+0000 front 2024-08-17T17:43:44.116266+0000 (oldest deadline 2024-08-17T17:44:07.016393+0000) 2024-08-17T17:44:08.345 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:07 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:07.972+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:08.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:08 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:08.199+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:08.782 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:08 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:08.525+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:09.029 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:08 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3[142377]: 2024-08-17T17:44:08.718+0000 7f2f91ff6640 -1 osd.3 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:44.116274+0000 front 2024-08-17T17:43:44.116266+0000 (oldest deadline 2024-08-17T17:44:07.016393+0000) 2024-08-17T17:44:09.095 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:09 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:09.015+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:09.096 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:09 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T17:44:09.009+0000 7f26caeba640 -1 osd.4 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:44.854132+0000 front 2024-08-17T17:43:44.854207+0000 (oldest deadline 2024-08-17T17:44:08.953977+0000) 2024-08-17T17:44:09.511 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:09 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:09.244+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:09.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:09 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:09.509+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:10.029 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:09 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3[142377]: 2024-08-17T17:44:09.741+0000 7f2f91ff6640 -1 osd.3 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:44.116274+0000 front 2024-08-17T17:43:44.116266+0000 (oldest deadline 2024-08-17T17:44:07.016393+0000) 2024-08-17T17:44:10.345 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:09 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T17:44:09.969+0000 7f26caeba640 -1 osd.4 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:44.854132+0000 front 2024-08-17T17:43:44.854207+0000 (oldest deadline 2024-08-17T17:44:08.953977+0000) 2024-08-17T17:44:10.346 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:10.033+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:10.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:10.204+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:10.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:10.542+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:11.279 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:10 smithi063 podman[393625]: 2024-08-17 17:44:10.885016492 +0000 UTC m=+5.602077736 container died e45c75b955ea998dfa90b0ad799cddeba539ecf8778040cb840583eeb19f2628 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-cd8bdbe, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-08-17T17:44:11.346 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:10 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T17:44:10.936+0000 7f26caeba640 -1 osd.4 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:44.854132+0000 front 2024-08-17T17:43:44.854207+0000 (oldest deadline 2024-08-17T17:44:08.953977+0000) 2024-08-17T17:44:11.346 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:11.004+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:11.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:11.160+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:11.779 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:11 smithi063 podman[393625]: 2024-08-17 17:44:11.286450232 +0000 UTC m=+6.003511474 container cleanup e45c75b955ea998dfa90b0ad799cddeba539ecf8778040cb840583eeb19f2628 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-cd8bdbe, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-08-17T17:44:11.779 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:11 smithi063 bash[393625]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3 2024-08-17T17:44:11.779 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:11 smithi063 podman[393648]: 2024-08-17 17:44:11.512619713 +0000 UTC m=+0.629014915 container remove e45c75b955ea998dfa90b0ad799cddeba539ecf8778040cb840583eeb19f2628 (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731) 2024-08-17T17:44:11.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:11.547+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:11.846 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:11.620Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.63:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.63:8765: connect: connection refused" 2024-08-17T17:44:11.846 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:11.622Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.63:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.63:8765: connect: connection refused" 2024-08-17T17:44:11.846 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:11.624Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.63:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.63:8765: connect: connection refused" 2024-08-17T17:44:11.846 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:11.624Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.63:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.63:8765: connect: connection refused" 2024-08-17T17:44:11.846 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:11.625Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.63:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.63:8765: connect: connection refused" 2024-08-17T17:44:11.847 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:11.625Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.63:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.63:8765: connect: connection refused" 2024-08-17T17:44:12.240 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:11 smithi063 podman[393738]: 2024-08-17 17:44:11.837234589 +0000 UTC m=+0.017901929 image pull 0566dfeff0cb6af9254aa2111a92af8174516dad87a45779e953ca0dd77f3099 quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d 2024-08-17T17:44:12.241 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:11 smithi063 podman[393738]: 2024-08-17 17:44:11.938359568 +0000 UTC m=+0.119026893 container create 616b572cb75a298d3673ff779b00cab336419b36027f2ca1e57d0c032cff2b8f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=main-cd8bdbe) 2024-08-17T17:44:12.241 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:12 smithi063 podman[393738]: 2024-08-17 17:44:12.149219127 +0000 UTC m=+0.329886455 container init 616b572cb75a298d3673ff779b00cab336419b36027f2ca1e57d0c032cff2b8f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-08-17T17:44:12.241 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:12 smithi063 podman[393738]: 2024-08-17 17:44:12.154269148 +0000 UTC m=+0.334936477 container start 616b572cb75a298d3673ff779b00cab336419b36027f2ca1e57d0c032cff2b8f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-cd8bdbe, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-08-17T17:44:12.241 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:12 smithi063 podman[393738]: 2024-08-17 17:44:12.240569378 +0000 UTC m=+0.421236716 container attach 616b572cb75a298d3673ff779b00cab336419b36027f2ca1e57d0c032cff2b8f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, io.buildah.version=1.36.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-08-17T17:44:12.345 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:11 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T17:44:11.978+0000 7f26caeba640 -1 osd.4 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:44.854132+0000 front 2024-08-17T17:43:44.854207+0000 (oldest deadline 2024-08-17T17:44:08.953977+0000) 2024-08-17T17:44:12.346 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:12 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:12.050+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:12.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:12 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:12.188+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:12.529 INFO:journalctl@ceph.osd.3.smithi063.stdout:Aug 17 17:44:12 smithi063 podman[393738]: 2024-08-17 17:44:12.348791207 +0000 UTC m=+0.529458543 container died 616b572cb75a298d3673ff779b00cab336419b36027f2ca1e57d0c032cff2b8f (image=quay.ceph.io/ceph-ci/ceph@sha256:aee44f95cb23e15b5d48a3d9a6d838e96ffc6125ee57d506dda5eff94584115d, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=main-cd8bdbe) 2024-08-17T17:44:12.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:12 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:12.548+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:12.967 DEBUG:teuthology.orchestra.run.smithi063:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.3.service' 2024-08-17T17:44:13.007 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:44:13.007 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-08-17T17:44:13.007 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-08-17T17:44:13.008 DEBUG:teuthology.orchestra.run.smithi162:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.4 2024-08-17T17:44:13.227 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:12 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T17:44:12.941+0000 7f26caeba640 -1 osd.4 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:44.854132+0000 front 2024-08-17T17:43:44.854207+0000 (oldest deadline 2024-08-17T17:44:08.953977+0000) 2024-08-17T17:44:13.227 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:13 smithi162 systemd[1]: Stopping Ceph osd.4 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:44:13.227 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:13 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:13.057+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:13.539 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:13 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:13.224+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:13.539 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:13 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T17:44:13.343+0000 7f26d28ae640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:44:13.539 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:13 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T17:44:13.343+0000 7f26d28ae640 -1 osd.4 1466 *** Got signal Terminated *** 2024-08-17T17:44:13.539 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:13 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T17:44:13.343+0000 7f26d28ae640 -1 osd.4 1466 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:44:13.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:13 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:13.538+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:14.235 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:13 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T17:44:13.905+0000 7f26caeba640 -1 osd.4 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:44.854132+0000 front 2024-08-17T17:43:44.854207+0000 (oldest deadline 2024-08-17T17:44:08.953977+0000) 2024-08-17T17:44:14.235 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:14.075+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:14.576 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:14.233+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:14.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:14.575+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:15.238 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:14 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T17:44:14.877+0000 7f26caeba640 -1 osd.4 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:44.854132+0000 front 2024-08-17T17:43:44.854207+0000 (oldest deadline 2024-08-17T17:44:08.953977+0000) 2024-08-17T17:44:15.238 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:15 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:15.053+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:15.238 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:15 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:15.053+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:50.055043+0000 front 2024-08-17T17:43:50.055055+0000 (oldest deadline 2024-08-17T17:44:14.754745+0000) 2024-08-17T17:44:15.576 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:15 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:15.236+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:15.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:15 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:15.575+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:16.266 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:15 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T17:44:15.844+0000 7f26caeba640 -1 osd.4 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:44.854132+0000 front 2024-08-17T17:43:44.854207+0000 (oldest deadline 2024-08-17T17:44:08.953977+0000) 2024-08-17T17:44:16.266 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:16 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:16.059+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:16.266 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:16 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:16.059+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:50.055043+0000 front 2024-08-17T17:43:50.055055+0000 (oldest deadline 2024-08-17T17:44:14.754745+0000) 2024-08-17T17:44:16.549 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:16 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:16.264+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:16.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:16 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:16.548+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:16.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:16 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:16.548+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:17.292 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:16 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T17:44:16.869+0000 7f26caeba640 -1 osd.4 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:44.854132+0000 front 2024-08-17T17:43:44.854207+0000 (oldest deadline 2024-08-17T17:44:08.953977+0000) 2024-08-17T17:44:17.292 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:16 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T17:44:16.869+0000 7f26caeba640 -1 osd.4 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.055006+0000 front 2024-08-17T17:43:53.055070+0000 (oldest deadline 2024-08-17T17:44:16.554947+0000) 2024-08-17T17:44:17.293 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:17 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:17.022+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:17.293 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:17 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:17.022+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:50.055043+0000 front 2024-08-17T17:43:50.055055+0000 (oldest deadline 2024-08-17T17:44:14.754745+0000) 2024-08-17T17:44:17.595 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:17 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:17.291+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:17.596 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:17 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:17.523+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:17.596 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:17 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:17.523+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:18.317 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:17 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T17:44:17.916+0000 7f26caeba640 -1 osd.4 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:44.854132+0000 front 2024-08-17T17:43:44.854207+0000 (oldest deadline 2024-08-17T17:44:08.953977+0000) 2024-08-17T17:44:18.318 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:17 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4[39900]: 2024-08-17T17:44:17.916+0000 7f26caeba640 -1 osd.4 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.055006+0000 front 2024-08-17T17:43:53.055070+0000 (oldest deadline 2024-08-17T17:44:16.554947+0000) 2024-08-17T17:44:18.318 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:18 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:18.028+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:18.318 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:18 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:18.028+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:50.055043+0000 front 2024-08-17T17:43:50.055055+0000 (oldest deadline 2024-08-17T17:44:14.754745+0000) 2024-08-17T17:44:18.595 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:18 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:18.502+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:18.596 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:18 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:18.502+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:18.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:18 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:18.316+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:18.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:18 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:18.316+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:19.160 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:19 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:19.052+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:19.161 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:19 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:19.052+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:50.055043+0000 front 2024-08-17T17:43:50.055055+0000 (oldest deadline 2024-08-17T17:44:14.754745+0000) 2024-08-17T17:44:19.161 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:18 smithi162 podman[261496]: 2024-08-17 17:44:18.901485031 +0000 UTC m=+5.688226395 container died a978a99d5919da13d6f708ac878f8ce7c326290cac85189146e056ababfc1614 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=reef-ced649f) 2024-08-17T17:44:19.512 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:19 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:19.310+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:19.512 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:19 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:19.310+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:19.512 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:19 smithi162 podman[261496]: 2024-08-17 17:44:19.343203643 +0000 UTC m=+6.129945006 container cleanup a978a99d5919da13d6f708ac878f8ce7c326290cac85189146e056ababfc1614 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True) 2024-08-17T17:44:19.512 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:19 smithi162 bash[261496]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4 2024-08-17T17:44:19.845 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:19 smithi162 podman[261515]: 2024-08-17 17:44:19.661608483 +0000 UTC m=+0.762819978 container remove a978a99d5919da13d6f708ac878f8ce7c326290cac85189146e056ababfc1614 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-08-17T17:44:19.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:19 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:19.510+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:19.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:19 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:19.510+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:20.292 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:20.048+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:20.292 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:20.048+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:50.055043+0000 front 2024-08-17T17:43:50.055055+0000 (oldest deadline 2024-08-17T17:44:14.754745+0000) 2024-08-17T17:44:20.293 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:19 smithi162 podman[261608]: 2024-08-17 17:44:19.889105267 +0000 UTC m=+0.018038921 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:44:20.293 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:19 smithi162 podman[261608]: 2024-08-17 17:44:19.987725731 +0000 UTC m=+0.116659421 container create 4f83d4187d6350bc90dd736732c837fbbfef824db27f2aaf560ed5a951efbb4c (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, ceph=True, GIT_CLEAN=True) 2024-08-17T17:44:20.293 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:20 smithi162 podman[261608]: 2024-08-17 17:44:20.223997227 +0000 UTC m=+0.352930875 container init 4f83d4187d6350bc90dd736732c837fbbfef824db27f2aaf560ed5a951efbb4c (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-08-17T17:44:20.293 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:20 smithi162 podman[261608]: 2024-08-17 17:44:20.228937303 +0000 UTC m=+0.357870960 container start 4f83d4187d6350bc90dd736732c837fbbfef824db27f2aaf560ed5a951efbb4c (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4-deactivate, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, io.buildah.version=1.36.0) 2024-08-17T17:44:20.595 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:20.500+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:20.596 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:20.500+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:20.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:20.291+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:20.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:20 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:20.291+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:20.596 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:20 smithi162 podman[261608]: 2024-08-17 17:44:20.323655895 +0000 UTC m=+0.452589584 container attach 4f83d4187d6350bc90dd736732c837fbbfef824db27f2aaf560ed5a951efbb4c (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4-deactivate, ceph=True, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-08-17T17:44:20.597 INFO:journalctl@ceph.osd.4.smithi162.stdout:Aug 17 17:44:20 smithi162 podman[261608]: 2024-08-17 17:44:20.502315094 +0000 UTC m=+0.631248738 container died 4f83d4187d6350bc90dd736732c837fbbfef824db27f2aaf560ed5a951efbb4c (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:44:21.101 DEBUG:teuthology.orchestra.run.smithi162:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.4.service' 2024-08-17T17:44:21.142 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:44:21.142 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-08-17T17:44:21.143 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-08-17T17:44:21.143 DEBUG:teuthology.orchestra.run.smithi162:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.5 2024-08-17T17:44:21.345 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:21 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:21.274+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:21.345 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:21 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:21.274+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:21.346 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:21 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:21.049+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:21.346 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:21 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:21.049+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:50.055043+0000 front 2024-08-17T17:43:50.055055+0000 (oldest deadline 2024-08-17T17:44:14.754745+0000) 2024-08-17T17:44:21.346 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:21 smithi162 systemd[1]: Stopping Ceph osd.5 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:44:21.845 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:21 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:21.498+0000 7f3e73b42640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:44:21.846 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:21 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:21.498+0000 7f3e73b42640 -1 osd.5 1466 *** Got signal Terminated *** 2024-08-17T17:44:21.846 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:21 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:21.498+0000 7f3e73b42640 -1 osd.5 1466 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:44:21.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:21 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:21.537+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:21.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:21 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:21.537+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:22.345 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:22.061+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:22.346 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:22.061+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:50.055043+0000 front 2024-08-17T17:43:50.055055+0000 (oldest deadline 2024-08-17T17:44:14.754745+0000) 2024-08-17T17:44:22.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:22.255+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:22.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:22.255+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:22.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:22.580+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:22.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:22 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:22.580+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:23.299 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:23.015+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:23.299 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:23.015+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:50.055043+0000 front 2024-08-17T17:43:50.055055+0000 (oldest deadline 2024-08-17T17:44:14.754745+0000) 2024-08-17T17:44:23.595 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:23.297+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:23.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:23.297+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:24.027 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:23.602+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:24.027 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:23 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:23.602+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:24.345 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:24 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:24.026+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:24.346 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:24 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:24.026+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:50.055043+0000 front 2024-08-17T17:43:50.055055+0000 (oldest deadline 2024-08-17T17:44:14.754745+0000) 2024-08-17T17:44:24.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:24 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:24.252+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:24.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:24 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:24.252+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:24.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:24 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:24.252+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:25.027 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:24 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:24.622+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:25.027 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:24 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:24.622+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:25.345 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:25 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:25.026+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:25.346 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:25 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:25.026+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:50.055043+0000 front 2024-08-17T17:43:50.055055+0000 (oldest deadline 2024-08-17T17:44:14.754745+0000) 2024-08-17T17:44:25.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:25 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:25.231+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:25.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:25 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:25.231+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:25.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:25 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:25.231+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:25.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:25 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:25.574+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:25.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:25 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:25.574+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:25.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:25 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:25.574+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.631755+0000 front 2024-08-17T17:43:59.631796+0000 (oldest deadline 2024-08-17T17:44:24.931654+0000) 2024-08-17T17:44:26.345 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:25 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:25.997+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:40.753718+0000 front 2024-08-17T17:43:40.753742+0000 (oldest deadline 2024-08-17T17:44:06.653654+0000) 2024-08-17T17:44:26.346 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:25 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5[45733]: 2024-08-17T17:44:25.997+0000 7f3e6c14e640 -1 osd.5 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:50.055043+0000 front 2024-08-17T17:43:50.055055+0000 (oldest deadline 2024-08-17T17:44:14.754745+0000) 2024-08-17T17:44:26.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:26 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:26.229+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:26.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:26 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:26.229+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:26.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:26 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:26.229+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:26.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:26 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:26.526+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:26.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:26 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:26.526+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:26.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:26 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:26.526+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.631755+0000 front 2024-08-17T17:43:59.631796+0000 (oldest deadline 2024-08-17T17:44:24.931654+0000) 2024-08-17T17:44:27.250 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:26 smithi162 podman[261735]: 2024-08-17 17:44:26.985102441 +0000 UTC m=+5.621578182 container died c66546db813235d9dbd1137ec2987e2a9f6adb0075913faeed04de0f4ccd3925 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5, io.buildah.version=1.36.0, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-08-17T17:44:27.505 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:27 smithi162 podman[261735]: 2024-08-17 17:44:27.479753542 +0000 UTC m=+6.116229282 container cleanup c66546db813235d9dbd1137ec2987e2a9f6adb0075913faeed04de0f4ccd3925 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-08-17T17:44:27.505 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:27 smithi162 bash[261735]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5 2024-08-17T17:44:27.506 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:27 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:27.248+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:27.506 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:27 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:27.248+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:27.506 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:27 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:27.248+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:27.845 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:27 smithi162 podman[261761]: 2024-08-17 17:44:27.714590764 +0000 UTC m=+0.723570287 container remove c66546db813235d9dbd1137ec2987e2a9f6adb0075913faeed04de0f4ccd3925 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-08-17T17:44:27.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:27 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:27.524+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:27.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:27 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:27.524+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:27.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:27 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:27.524+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.631755+0000 front 2024-08-17T17:43:59.631796+0000 (oldest deadline 2024-08-17T17:44:24.931654+0000) 2024-08-17T17:44:28.345 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:28 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:28.217+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:28.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:28 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:28.217+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:28.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:28 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:28.217+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:28.346 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:28 smithi162 podman[261852]: 2024-08-17 17:44:27.939619358 +0000 UTC m=+0.017845349 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:44:28.346 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:28 smithi162 podman[261852]: 2024-08-17 17:44:28.040759928 +0000 UTC m=+0.118985890 container create e4754742970e36a3ab448ac4040953e2dd706e3c6d6222e2253b692b1de92b24 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, RELEASE=reef-ced649f, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731) 2024-08-17T17:44:28.346 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:28 smithi162 podman[261852]: 2024-08-17 17:44:28.268524045 +0000 UTC m=+0.346750025 container init e4754742970e36a3ab448ac4040953e2dd706e3c6d6222e2253b692b1de92b24 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, RELEASE=reef-ced649f, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:44:28.347 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:28 smithi162 podman[261852]: 2024-08-17 17:44:28.273624502 +0000 UTC m=+0.351850471 container start e4754742970e36a3ab448ac4040953e2dd706e3c6d6222e2253b692b1de92b24 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:44:28.747 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:28 smithi162 podman[261852]: 2024-08-17 17:44:28.374249751 +0000 UTC m=+0.452475712 container attach e4754742970e36a3ab448ac4040953e2dd706e3c6d6222e2253b692b1de92b24 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-08-17T17:44:28.747 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:28 smithi162 podman[261852]: 2024-08-17 17:44:28.461607016 +0000 UTC m=+0.539832981 container died e4754742970e36a3ab448ac4040953e2dd706e3c6d6222e2253b692b1de92b24 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=reef-ced649f) 2024-08-17T17:44:28.747 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:28 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:28.524+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:28.748 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:28 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:28.524+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:28.748 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:28 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:28.524+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.631755+0000 front 2024-08-17T17:43:59.631796+0000 (oldest deadline 2024-08-17T17:44:24.931654+0000) 2024-08-17T17:44:28.999 INFO:journalctl@ceph.osd.5.smithi162.stdout:Aug 17 17:44:28 smithi162 podman[261895]: 2024-08-17 17:44:28.988028229 +0000 UTC m=+0.519716578 container remove e4754742970e36a3ab448ac4040953e2dd706e3c6d6222e2253b692b1de92b24 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-5-deactivate, ceph=True, RELEASE=reef-ced649f, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:44:29.004 DEBUG:teuthology.orchestra.run.smithi162:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.5.service' 2024-08-17T17:44:29.042 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:44:29.042 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-08-17T17:44:29.042 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-08-17T17:44:29.042 DEBUG:teuthology.orchestra.run.smithi162:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.6 2024-08-17T17:44:29.346 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:29 smithi162 systemd[1]: Stopping Ceph osd.6 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:44:29.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:29 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:29.242+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:29.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:29 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:29.242+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:29.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:29 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:29.242+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:29.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:29 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:29.409+0000 7fabe7b46640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:44:29.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:29 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:29.409+0000 7fabe7b46640 -1 osd.6 1466 *** Got signal Terminated *** 2024-08-17T17:44:29.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:29 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:29.409+0000 7fabe7b46640 -1 osd.6 1466 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:44:29.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:29 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:29.499+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:29.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:29 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:29.499+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:29.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:29 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:29.499+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.631755+0000 front 2024-08-17T17:43:59.631796+0000 (oldest deadline 2024-08-17T17:44:24.931654+0000) 2024-08-17T17:44:30.595 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:30 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:30.246+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:30.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:30 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:30.246+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:30.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:30 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:30.246+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:30.596 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:30 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:30.470+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:30.597 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:30 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:30.470+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:30.597 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:30 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:30.470+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.631755+0000 front 2024-08-17T17:43:59.631796+0000 (oldest deadline 2024-08-17T17:44:24.931654+0000) 2024-08-17T17:44:31.489 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:31 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:31.213+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:31.489 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:31 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:31.213+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:31.490 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:31 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:31.213+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:31.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:31 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:31.488+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:31.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:31 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:31.488+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:31.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:31 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:31.488+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.631755+0000 front 2024-08-17T17:43:59.631796+0000 (oldest deadline 2024-08-17T17:44:24.931654+0000) 2024-08-17T17:44:32.458 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:32 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:32.166+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:32.458 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:32 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:32.167+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:32.459 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:32 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:32.167+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:32.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:32 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:32.457+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:32.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:32 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:32.457+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:32.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:32 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:32.457+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.631755+0000 front 2024-08-17T17:43:59.631796+0000 (oldest deadline 2024-08-17T17:44:24.931654+0000) 2024-08-17T17:44:33.448 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:33 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:33.172+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:33.448 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:33 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:33.172+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:33.448 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:33 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:33.172+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:33.448 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:33 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:33.172+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-08-17T17:44:07.495470+0000 front 2024-08-17T17:44:07.495484+0000 (oldest deadline 2024-08-17T17:44:32.795335+0000) 2024-08-17T17:44:33.845 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:33 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:33.447+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:43.329401+0000 front 2024-08-17T17:43:43.329319+0000 (oldest deadline 2024-08-17T17:44:07.429325+0000) 2024-08-17T17:44:33.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:33 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:33.447+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:51.530385+0000 front 2024-08-17T17:43:51.530533+0000 (oldest deadline 2024-08-17T17:44:16.230398+0000) 2024-08-17T17:44:33.846 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:33 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6[229111]: 2024-08-17T17:44:33.447+0000 7fabe0152640 -1 osd.6 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.631755+0000 front 2024-08-17T17:43:59.631796+0000 (oldest deadline 2024-08-17T17:44:24.931654+0000) 2024-08-17T17:44:34.595 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:34 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:34.161+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:34.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:34 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:34.161+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:34.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:34 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:34.161+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:34.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:34 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:34.161+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-08-17T17:44:07.495470+0000 front 2024-08-17T17:44:07.495484+0000 (oldest deadline 2024-08-17T17:44:32.795335+0000) 2024-08-17T17:44:35.302 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:35 smithi162 podman[261979]: 2024-08-17 17:44:34.999730217 +0000 UTC m=+5.725007547 container died d6247a6fefe8228f90acf6fd9e143c7cdcd057b04d0827f8bba50570ea2198a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6, RELEASE=reef-ced649f, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True) 2024-08-17T17:44:35.302 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:35 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:35.170+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:35.302 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:35 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:35.170+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:35.302 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:35 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:35.170+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:35.302 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:35 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:35.170+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-08-17T17:44:07.495470+0000 front 2024-08-17T17:44:07.495484+0000 (oldest deadline 2024-08-17T17:44:32.795335+0000) 2024-08-17T17:44:35.596 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:35 smithi162 podman[261979]: 2024-08-17 17:44:35.482810233 +0000 UTC m=+6.208087619 container cleanup d6247a6fefe8228f90acf6fd9e143c7cdcd057b04d0827f8bba50570ea2198a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6, org.label-schema.license=GPLv2, io.buildah.version=1.36.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-08-17T17:44:35.596 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:35 smithi162 bash[261979]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6 2024-08-17T17:44:36.021 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:35 smithi162 podman[261999]: 2024-08-17 17:44:35.700854468 +0000 UTC m=+0.699890491 container remove d6247a6fefe8228f90acf6fd9e143c7cdcd057b04d0827f8bba50570ea2198a4 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=reef-ced649f, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-08-17T17:44:36.304 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:36.132+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:36.304 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:36.132+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:36.305 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:36.132+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:36.305 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:36 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:36.132+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-08-17T17:44:07.495470+0000 front 2024-08-17T17:44:07.495484+0000 (oldest deadline 2024-08-17T17:44:32.795335+0000) 2024-08-17T17:44:36.305 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:36 smithi162 podman[262095]: 2024-08-17 17:44:35.923912544 +0000 UTC m=+0.018239523 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:44:36.305 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:36 smithi162 podman[262095]: 2024-08-17 17:44:36.0267212 +0000 UTC m=+0.121048198 container create 9414d98fe8645905040257aa40e3df5ede3768bfc4323d9ad45987380f45eaa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:44:36.305 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:36 smithi162 podman[262095]: 2024-08-17 17:44:36.228976464 +0000 UTC m=+0.323303430 container init 9414d98fe8645905040257aa40e3df5ede3768bfc4323d9ad45987380f45eaa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=reef-ced649f, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2) 2024-08-17T17:44:36.305 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:36 smithi162 podman[262095]: 2024-08-17 17:44:36.233255259 +0000 UTC m=+0.327582216 container start 9414d98fe8645905040257aa40e3df5ede3768bfc4323d9ad45987380f45eaa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-08-17T17:44:36.305 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:36 smithi162 podman[262095]: 2024-08-17 17:44:36.303882343 +0000 UTC m=+0.398209316 container attach 9414d98fe8645905040257aa40e3df5ede3768bfc4323d9ad45987380f45eaa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, RELEASE=reef-ced649f, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-08-17T17:44:36.595 INFO:journalctl@ceph.osd.6.smithi162.stdout:Aug 17 17:44:36 smithi162 podman[262095]: 2024-08-17 17:44:36.418054954 +0000 UTC m=+0.512381923 container died 9414d98fe8645905040257aa40e3df5ede3768bfc4323d9ad45987380f45eaa4 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=reef-ced649f, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-08-17T17:44:37.116 DEBUG:teuthology.orchestra.run.smithi162:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.6.service' 2024-08-17T17:44:37.155 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:44:37.155 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-08-17T17:44:37.156 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-08-17T17:44:37.156 DEBUG:teuthology.orchestra.run.smithi162:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.7 2024-08-17T17:44:37.345 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:37 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:37.158+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:37.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:37 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:37.158+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:37.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:37 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:37.158+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:37.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:37 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:37.158+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-08-17T17:44:07.495470+0000 front 2024-08-17T17:44:07.495484+0000 (oldest deadline 2024-08-17T17:44:32.795335+0000) 2024-08-17T17:44:37.346 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:37 smithi162 systemd[1]: Stopping Ceph osd.7 for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:44:37.845 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:37 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:37.479+0000 7f3a4e7f7640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-08-17T17:44:37.846 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:37 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:37.479+0000 7f3a4e7f7640 -1 osd.7 1466 *** Got signal Terminated *** 2024-08-17T17:44:37.846 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:37 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:37.479+0000 7f3a4e7f7640 -1 osd.7 1466 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-08-17T17:44:38.484 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:38.208+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:38.485 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:38.208+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:38.485 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:38.208+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:38.485 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:38 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:38.208+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-08-17T17:44:07.495470+0000 front 2024-08-17T17:44:07.495484+0000 (oldest deadline 2024-08-17T17:44:32.795335+0000) 2024-08-17T17:44:39.595 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:39.204+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:39.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:39.204+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:39.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:39.204+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:39.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:39 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:39.204+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-08-17T17:44:07.495470+0000 front 2024-08-17T17:44:07.495484+0000 (oldest deadline 2024-08-17T17:44:32.795335+0000) 2024-08-17T17:44:40.595 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:40.175+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:40.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:40.175+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:40.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:40.175+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:40.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:40 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:40.175+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-08-17T17:44:07.495470+0000 front 2024-08-17T17:44:07.495484+0000 (oldest deadline 2024-08-17T17:44:32.795335+0000) 2024-08-17T17:44:41.595 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:41 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:41.158+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:41.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:41 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:41.158+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:41.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:41 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:41.158+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:41.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:41 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:41.158+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-08-17T17:44:07.495470+0000 front 2024-08-17T17:44:07.495484+0000 (oldest deadline 2024-08-17T17:44:32.795335+0000) 2024-08-17T17:44:42.595 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:42 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:42.191+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6806 osd.0 since back 2024-08-17T17:43:42.591216+0000 front 2024-08-17T17:43:42.591400+0000 (oldest deadline 2024-08-17T17:44:07.891251+0000) 2024-08-17T17:44:42.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:42 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:42.191+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6814 osd.1 since back 2024-08-17T17:43:53.192537+0000 front 2024-08-17T17:43:53.192518+0000 (oldest deadline 2024-08-17T17:44:17.892430+0000) 2024-08-17T17:44:42.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:42 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:42.191+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6822 osd.2 since back 2024-08-17T17:43:59.493730+0000 front 2024-08-17T17:43:59.493636+0000 (oldest deadline 2024-08-17T17:44:23.593694+0000) 2024-08-17T17:44:42.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:42 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:42.191+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.63:6830 osd.3 since back 2024-08-17T17:44:07.495470+0000 front 2024-08-17T17:44:07.495484+0000 (oldest deadline 2024-08-17T17:44:32.795335+0000) 2024-08-17T17:44:42.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:42 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7[226133]: 2024-08-17T17:44:42.191+0000 7f3a46e03640 -1 osd.7 1466 heartbeat_check: no reply from 172.21.15.162:6804 osd.4 since back 2024-08-17T17:44:16.296337+0000 front 2024-08-17T17:44:16.296202+0000 (oldest deadline 2024-08-17T17:44:41.596186+0000) 2024-08-17T17:44:43.178 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:42 smithi162 podman[262222]: 2024-08-17 17:44:42.893629717 +0000 UTC m=+5.509120234 container died 9afc092f538774cb0dfbccc98065aeaed3052f50b417ef19f04840a76360ee48 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.36.0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-08-17T17:44:43.520 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:43 smithi162 podman[262222]: 2024-08-17 17:44:43.343441795 +0000 UTC m=+5.958932315 container cleanup 9afc092f538774cb0dfbccc98065aeaed3052f50b417ef19f04840a76360ee48 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.36.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, RELEASE=reef-ced649f, CEPH_POINT_RELEASE=) 2024-08-17T17:44:43.520 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:43 smithi162 bash[262222]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7 2024-08-17T17:44:43.845 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:43 smithi162 podman[262257]: 2024-08-17 17:44:43.519839247 +0000 UTC m=+0.617426422 container remove 9afc092f538774cb0dfbccc98065aeaed3052f50b417ef19f04840a76360ee48 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7, CEPH_POINT_RELEASE=, io.buildah.version=1.36.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=reef-ced649f, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-08-17T17:44:44.256 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:43 smithi162 podman[262346]: 2024-08-17 17:44:43.870093929 +0000 UTC m=+0.017040210 image pull c870753d67c2504e5ac044c83696772f8b7527a928cc764ed24eeea7fea57d6e quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408 2024-08-17T17:44:44.257 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:43 smithi162 podman[262346]: 2024-08-17 17:44:43.970791597 +0000 UTC m=+0.117737906 container create ab3103b3f72411a86031aa6244cf678de86126c9939c02f83df0e798235194d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=reef-ced649f, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-08-17T17:44:44.257 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:44 smithi162 podman[262346]: 2024-08-17 17:44:44.173517345 +0000 UTC m=+0.320463622 container init ab3103b3f72411a86031aa6244cf678de86126c9939c02f83df0e798235194d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.36.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f, org.label-schema.build-date=20240731) 2024-08-17T17:44:44.257 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:44 smithi162 podman[262346]: 2024-08-17 17:44:44.178660406 +0000 UTC m=+0.325606673 container start ab3103b3f72411a86031aa6244cf678de86126c9939c02f83df0e798235194d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.36.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-ced649f) 2024-08-17T17:44:44.257 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:44 smithi162 podman[262346]: 2024-08-17 17:44:44.256360595 +0000 UTC m=+0.403306876 container attach ab3103b3f72411a86031aa6244cf678de86126c9939c02f83df0e798235194d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=reef-ced649f, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.36.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-08-17T17:44:44.596 INFO:journalctl@ceph.osd.7.smithi162.stdout:Aug 17 17:44:44 smithi162 podman[262346]: 2024-08-17 17:44:44.361038389 +0000 UTC m=+0.507984667 container died ab3103b3f72411a86031aa6244cf678de86126c9939c02f83df0e798235194d0 (image=quay.ceph.io/ceph-ci/ceph@sha256:37f887e3bbdff4051aa7da31b6d591754a495ffdffc717f32d57523a5142a408, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-osd-7-deactivate, ceph=True, io.buildah.version=1.36.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=reef-ced649f, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-08-17T17:44:45.067 DEBUG:teuthology.orchestra.run.smithi162:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@osd.7.service' 2024-08-17T17:44:45.106 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:44:45.106 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-08-17T17:44:45.107 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-08-17T17:44:45.107 DEBUG:teuthology.orchestra.run.smithi162:> sudo systemctl stop ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@prometheus.a 2024-08-17T17:44:45.345 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 systemd[1]: Stopping Ceph prometheus.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:44:45.758 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:45.465Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:44:45.759 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:45.465Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-08-17T17:44:45.759 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:45.465Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-08-17T17:44:45.759 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:45.465Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-08-17T17:44:45.759 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:45.465Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-08-17T17:44:45.759 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:45.465Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-08-17T17:44:45.759 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:45.465Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-08-17T17:44:45.760 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:45.465Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-08-17T17:44:45.760 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:45.465Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-08-17T17:44:45.760 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:45.467Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-08-17T17:44:45.760 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:45.467Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-08-17T17:44:45.760 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a[89923]: ts=2024-08-17T17:44:45.467Z caller=main.go:1273 level=info msg="See you next time!" 2024-08-17T17:44:45.760 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 podman[262472]: 2024-08-17 17:44:45.475527711 +0000 UTC m=+0.137805084 container died dfe66b2c5568ec125e083924cb579fd1178e864127062e6eaadec53c7c746218 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:44:46.039 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 podman[262472]: 2024-08-17 17:44:45.85436241 +0000 UTC m=+0.516639783 container cleanup dfe66b2c5568ec125e083924cb579fd1178e864127062e6eaadec53c7c746218 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-08-17T17:44:46.039 INFO:journalctl@ceph.prometheus.a.smithi162.stdout:Aug 17 17:44:45 smithi162 bash[262472]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-prometheus-a 2024-08-17T17:44:46.260 DEBUG:teuthology.orchestra.run.smithi162:> sudo pkill -f 'journalctl -f -n 0 -u ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@prometheus.a.service' 2024-08-17T17:44:46.294 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-08-17T17:44:46.295 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-08-17T17:44:46.295 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 --force --keep-logs 2024-08-17T17:44:46.458 INFO:teuthology.orchestra.run.smithi063.stdout:Deleting cluster with fsid: 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:44:48.030 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 17:44:47 smithi063 systemd[1]: Stopping Ceph alertmanager.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:44:48.507 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 17:44:48 smithi063 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a[104529]: ts=2024-08-17T17:44:48.200Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-08-17T17:44:48.507 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 17:44:48 smithi063 podman[394165]: 2024-08-17 17:44:48.251795542 +0000 UTC m=+0.155701765 container died 59f4018b91caa6ebb54ff8d3e8c2b12192aad700bacee2b981f98f63ad9c373c (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:44:48.779 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 17:44:48 smithi063 podman[394165]: 2024-08-17 17:44:48.672100872 +0000 UTC m=+0.576007128 container cleanup 59f4018b91caa6ebb54ff8d3e8c2b12192aad700bacee2b981f98f63ad9c373c (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:44:48.779 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 17:44:48 smithi063 bash[394165]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a 2024-08-17T17:44:49.242 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 17:44:48 smithi063 podman[394183]: 2024-08-17 17:44:48.923340393 +0000 UTC m=+0.671455876 container remove 59f4018b91caa6ebb54ff8d3e8c2b12192aad700bacee2b981f98f63ad9c373c (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-08-17T17:44:49.242 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 17:44:49 smithi063 podman[394183]: 2024-08-17 17:44:49.016590216 +0000 UTC m=+0.764705657 volume remove 20521155491dafc5c3c76767fa0919e6b43478dc8975b5ee304504cca20dc0d3 2024-08-17T17:44:49.529 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 17:44:49 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-08-17T17:44:49.529 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 17:44:49 smithi063 systemd[1]: Stopped Ceph alertmanager.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:44:49.529 INFO:journalctl@ceph.alertmanager.a.smithi063.stdout:Aug 17 17:44:49 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@alertmanager.a.service: Consumed 4.160s CPU time. 2024-08-17T17:44:49.530 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 17:44:49 smithi063 systemd[1]: Stopping Ceph node-exporter.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:44:50.029 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 17:44:49 smithi063 podman[394299]: 2024-08-17 17:44:49.723309837 +0000 UTC m=+0.128611310 container died 34f55f5de2aa20fa1a42390df28c89ee86e5cd1a98e425c12246e8eee0c6222e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:44:50.529 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 17:44:50 smithi063 podman[394299]: 2024-08-17 17:44:50.143060723 +0000 UTC m=+0.548362195 container cleanup 34f55f5de2aa20fa1a42390df28c89ee86e5cd1a98e425c12246e8eee0c6222e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:44:50.529 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 17:44:50 smithi063 bash[394299]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a 2024-08-17T17:44:50.529 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 17:44:50 smithi063 podman[394319]: 2024-08-17 17:44:50.344375274 +0000 UTC m=+0.623601269 container remove 34f55f5de2aa20fa1a42390df28c89ee86e5cd1a98e425c12246e8eee0c6222e (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-08-17T17:44:50.529 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 17:44:50 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:44:51.000 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 17:44:50 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-08-17T17:44:51.000 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 17:44:50 smithi063 systemd[1]: Stopped Ceph node-exporter.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:44:51.000 INFO:journalctl@ceph.node-exporter.a.smithi063.stdout:Aug 17 17:44:50 smithi063 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@node-exporter.a.service: Consumed 8.739s CPU time. 2024-08-17T17:44:57.696 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 --force --keep-logs 2024-08-17T17:44:57.852 INFO:teuthology.orchestra.run.smithi162.stdout:Deleting cluster with fsid: 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T17:44:59.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:44:59 smithi162 systemd[1]: Stopping Ceph grafana.a for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:44:59.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:44:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=server t=2024-08-17T17:44:59.434492646Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-08-17T17:44:59.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:44:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=ticker t=2024-08-17T17:44:59.434875832Z level=info msg=stopped last_tick=2024-08-17T17:44:50Z 2024-08-17T17:44:59.466 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:44:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=tracing t=2024-08-17T17:44:59.43492841Z level=info msg="Closing tracing" 2024-08-17T17:44:59.467 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:44:59 smithi162 ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a[95125]: logger=grafana-apiserver t=2024-08-17T17:44:59.434947539Z level=info msg="StorageObjectCountTracker pruner is exiting" 2024-08-17T17:44:59.467 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:44:59 smithi162 podman[262893]: 2024-08-17 17:44:59.465468383 +0000 UTC m=+0.130132854 container died 581e03e654a8304f438cf1c39e6c70f9e974c58e0d58d96d1812b8d84f6c30e3 (image=quay.io/ceph/grafana:10.4.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-17T17:45:00.033 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:44:59 smithi162 podman[262893]: 2024-08-17 17:44:59.798583724 +0000 UTC m=+0.463248195 container cleanup 581e03e654a8304f438cf1c39e6c70f9e974c58e0d58d96d1812b8d84f6c30e3 (image=quay.io/ceph/grafana:10.4.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-17T17:45:00.033 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:44:59 smithi162 bash[262893]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a 2024-08-17T17:45:00.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:45:00 smithi162 podman[262911]: 2024-08-17 17:45:00.033038425 +0000 UTC m=+0.568000492 container remove 581e03e654a8304f438cf1c39e6c70f9e974c58e0d58d96d1812b8d84f6c30e3 (image=quay.io/ceph/grafana:10.4.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-08-17T17:45:00.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:45:00 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@grafana.a.service: Deactivated successfully. 2024-08-17T17:45:00.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:45:00 smithi162 systemd[1]: Stopped Ceph grafana.a for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:45:00.346 INFO:journalctl@ceph.grafana.a.smithi162.stdout:Aug 17 17:45:00 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@grafana.a.service: Consumed 31.888s CPU time. 2024-08-17T17:45:00.682 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 17:45:00 smithi162 systemd[1]: Stopping Ceph node-exporter.b for 321fa59e-5cb8-11ef-bccf-c7b262605968... 2024-08-17T17:45:00.961 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 17:45:00 smithi162 podman[263023]: 2024-08-17 17:45:00.682278544 +0000 UTC m=+0.118101634 container died 5bb162d0f1a920054bdf68ad9fd3db476dc3fe81968f9a2ad80c09b8531a29af (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:45:01.324 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 17:45:01 smithi162 podman[263023]: 2024-08-17 17:45:01.060628984 +0000 UTC m=+0.496452074 container cleanup 5bb162d0f1a920054bdf68ad9fd3db476dc3fe81968f9a2ad80c09b8531a29af (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:45:01.325 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 17:45:01 smithi162 bash[263023]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b 2024-08-17T17:45:01.595 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 17:45:01 smithi162 podman[263040]: 2024-08-17 17:45:01.324536109 +0000 UTC m=+0.646415646 container remove 5bb162d0f1a920054bdf68ad9fd3db476dc3fe81968f9a2ad80c09b8531a29af (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-321fa59e-5cb8-11ef-bccf-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-08-17T17:45:01.596 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 17:45:01 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-08-17T17:45:01.596 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 17:45:01 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-08-17T17:45:01.596 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 17:45:01 smithi162 systemd[1]: Stopped Ceph node-exporter.b for 321fa59e-5cb8-11ef-bccf-c7b262605968. 2024-08-17T17:45:01.596 INFO:journalctl@ceph.node-exporter.b.smithi162.stdout:Aug 17 17:45:01 smithi162 systemd[1]: ceph-321fa59e-5cb8-11ef-bccf-c7b262605968@node-exporter.b.service: Consumed 7.944s CPU time. 2024-08-17T17:45:05.591 DEBUG:teuthology.orchestra.run.smithi063:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T17:45:05.627 DEBUG:teuthology.orchestra.run.smithi162:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-08-17T17:45:05.655 INFO:tasks.cephadm:Archiving crash dumps... 2024-08-17T17:45:05.657 DEBUG:teuthology.misc:Transferring archived files from smithi063:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858317/remote/smithi063/crash 2024-08-17T17:45:05.657 DEBUG:teuthology.orchestra.run.smithi063:> sudo tar c -f - -C /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/crash -- . 2024-08-17T17:45:05.706 INFO:teuthology.orchestra.run.smithi063.stderr:tar: /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/crash: Cannot open: No such file or directory 2024-08-17T17:45:05.706 INFO:teuthology.orchestra.run.smithi063.stderr:tar: Error is not recoverable: exiting now 2024-08-17T17:45:05.708 DEBUG:teuthology.misc:Transferring archived files from smithi162:/var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858317/remote/smithi162/crash 2024-08-17T17:45:05.709 DEBUG:teuthology.orchestra.run.smithi162:> sudo tar c -f - -C /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/crash -- . 2024-08-17T17:45:05.745 INFO:teuthology.orchestra.run.smithi162.stderr:tar: /var/lib/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/crash: Cannot open: No such file or directory 2024-08-17T17:45:05.745 INFO:teuthology.orchestra.run.smithi162.stderr:tar: Error is not recoverable: exiting now 2024-08-17T17:45:05.746 INFO:tasks.cephadm:Checking cluster log for badness... 2024-08-17T17:45:05.747 DEBUG:teuthology.orchestra.run.smithi063:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_AVAILABILITY | head -n 1 2024-08-17T17:45:05.855 INFO:teuthology.orchestra.run.smithi063.stdout:2024-08-17T16:52:25.130043+0000 mon.a (mon.0) 694 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-08-17T17:45:05.859 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-08-17T17:45:05.859 INFO:tasks.cephadm:Compressing logs... 2024-08-17T17:45:05.859 DEBUG:teuthology.orchestra.run.smithi063:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-08-17T17:45:05.862 DEBUG:teuthology.orchestra.run.smithi162:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-08-17T17:45:05.905 INFO:teuthology.orchestra.run.smithi063.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2024-08-17T17:45:05.905 INFO:teuthology.orchestra.run.smithi063.stderr:gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mds.foo.smithi063.ultpqj.log 2024-08-17T17:45:05.905 INFO:teuthology.orchestra.run.smithi063.stderr:gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.log 2024-08-17T17:45:05.907 INFO:teuthology.orchestra.run.smithi063.stderr:gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.2.log 2024-08-17T17:45:05.907 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mds.foo.smithi063.ultpqj.log: 86.0% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mds.foo.smithi063.ultpqj.log.gz 2024-08-17T17:45:05.907 INFO:teuthology.orchestra.run.smithi063.stderr:gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.audit.log 2024-08-17T17:45:05.907 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.log: /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.3.log 2024-08-17T17:45:05.909 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.1.log 2024-08-17T17:45:05.910 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mgr.y.log 2024-08-17T17:45:05.910 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.0.log 2024-08-17T17:45:05.910 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mon.c.log 2024-08-17T17:45:05.910 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mon.a.log 2024-08-17T17:45:05.910 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.cephadm.log 2024-08-17T17:45:05.910 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-volume.log 2024-08-17T17:45:05.911 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-08-17T17:45:05.913 INFO:teuthology.orchestra.run.smithi162.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.log 2024-08-17T17:45:05.913 INFO:teuthology.orchestra.run.smithi162.stderr:: No such file or directory 2024-08-17T17:45:05.913 INFO:teuthology.orchestra.run.smithi162.stderr:gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.audit.log 2024-08-17T17:45:05.914 INFO:teuthology.orchestra.run.smithi162.stderr:gzip/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.log: -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.7.log 2024-08-17T17:45:05.917 INFO:teuthology.orchestra.run.smithi162.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.5.log 2024-08-17T17:45:05.917 INFO:teuthology.orchestra.run.smithi162.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.4.log 2024-08-17T17:45:05.918 INFO:teuthology.orchestra.run.smithi162.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mgr.x.log 2024-08-17T17:45:05.918 INFO:teuthology.orchestra.run.smithi162.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.6.log 2024-08-17T17:45:05.918 INFO:teuthology.orchestra.run.smithi162.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.cephadm.log 2024-08-17T17:45:05.918 INFO:teuthology.orchestra.run.smithi162.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mds.foo.smithi162.bjkxnv.log 2024-08-17T17:45:05.918 INFO:teuthology.orchestra.run.smithi162.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-volume.log 2024-08-17T17:45:05.919 INFO:teuthology.orchestra.run.smithi162.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mds.foo.smithi162.bjkxnv.log: gzip -5 --verbose -- /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mon.b.log 2024-08-17T17:45:05.919 INFO:teuthology.orchestra.run.smithi162.stderr: 83.7% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mds.foo.smithi162.bjkxnv.log.gz 2024-08-17T17:45:05.919 INFO:teuthology.orchestra.run.smithi162.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-08-17T17:45:05.919 INFO:teuthology.orchestra.run.smithi063.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-volume.log: /var/log/ceph/cephadm.log: 92.6% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.log.gz 2024-08-17T17:45:06.009 INFO:teuthology.orchestra.run.smithi063.stderr: 94.2% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.audit.log.gz 2024-08-17T17:45:06.167 INFO:teuthology.orchestra.run.smithi063.stderr: 95.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-08-17T17:45:06.239 INFO:teuthology.orchestra.run.smithi162.stderr:/var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 85.7% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.cephadm.log.gz 2024-08-17T17:45:06.292 INFO:teuthology.orchestra.run.smithi162.stderr: 90.3% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.audit.log.gz 2024-08-17T17:45:06.340 INFO:teuthology.orchestra.run.smithi063.stderr: 91.9% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.cephadm.log.gz 2024-08-17T17:45:06.361 INFO:teuthology.orchestra.run.smithi063.stderr: 91.2% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-volume.log.gz 2024-08-17T17:45:06.392 INFO:teuthology.orchestra.run.smithi162.stderr: 91.3% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-volume.log.gz 2024-08-17T17:45:06.398 INFO:teuthology.orchestra.run.smithi162.stderr: 86.8% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph.log.gz 2024-08-17T17:45:06.437 INFO:teuthology.orchestra.run.smithi162.stderr: 90.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-08-17T17:45:06.557 INFO:teuthology.orchestra.run.smithi162.stderr: 89.8% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mgr.x.log.gz 2024-08-17T17:45:10.280 INFO:teuthology.orchestra.run.smithi063.stderr: 89.8% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mgr.y.log.gz 2024-08-17T17:45:15.303 INFO:teuthology.orchestra.run.smithi162.stderr: 92.1% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mon.b.log.gz 2024-08-17T17:45:28.956 INFO:teuthology.orchestra.run.smithi063.stderr: 91.7% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mon.c.log.gz 2024-08-17T17:45:39.707 INFO:teuthology.orchestra.run.smithi063.stderr: 90.3% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-mon.a.log.gz 2024-08-17T17:58:13.361 INFO:teuthology.orchestra.run.smithi063.stderr: 93.5% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.1.log.gz 2024-08-17T17:59:19.621 INFO:teuthology.orchestra.run.smithi162.stderr: 93.6% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.6.log.gz 2024-08-17T18:00:20.328 INFO:teuthology.orchestra.run.smithi162.stderr: 93.8% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.7.log.gz 2024-08-17T18:00:53.889 INFO:teuthology.orchestra.run.smithi063.stderr: 93.2% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.0.log.gz 2024-08-17T18:01:57.459 INFO:teuthology.orchestra.run.smithi063.stderr: 92.8% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.2.log.gz 2024-08-17T18:01:59.509 INFO:teuthology.orchestra.run.smithi162.stderr: 93.7% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.4.log.gz 2024-08-17T18:02:02.804 INFO:teuthology.orchestra.run.smithi063.stderr: 93.6% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.3.log.gz 2024-08-17T18:02:02.806 INFO:teuthology.orchestra.run.smithi063.stderr: 2024-08-17T18:02:02.806 INFO:teuthology.orchestra.run.smithi063.stderr:real 16m56.929s 2024-08-17T18:02:02.806 INFO:teuthology.orchestra.run.smithi063.stderr:user 6m57.843s 2024-08-17T18:02:02.806 INFO:teuthology.orchestra.run.smithi063.stderr:sys 0m20.501s 2024-08-17T18:02:31.027 INFO:teuthology.orchestra.run.smithi162.stderr: 93.0% -- replaced with /var/log/ceph/321fa59e-5cb8-11ef-bccf-c7b262605968/ceph-osd.5.log.gz 2024-08-17T18:02:31.030 INFO:teuthology.orchestra.run.smithi162.stderr: 2024-08-17T18:02:31.031 INFO:teuthology.orchestra.run.smithi162.stderr:real 17m25.153s 2024-08-17T18:02:31.031 INFO:teuthology.orchestra.run.smithi162.stderr:user 6m22.847s 2024-08-17T18:02:31.031 INFO:teuthology.orchestra.run.smithi162.stderr:sys 0m20.591s 2024-08-17T18:02:31.031 INFO:tasks.cephadm:Archiving logs... 2024-08-17T18:02:31.032 DEBUG:teuthology.misc:Transferring archived files from smithi063:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858317/remote/smithi063/log 2024-08-17T18:02:31.033 DEBUG:teuthology.orchestra.run.smithi063:> sudo tar c -f - -C /var/log/ceph -- . 2024-08-17T18:03:29.472 DEBUG:teuthology.misc:Transferring archived files from smithi162:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858317/remote/smithi162/log 2024-08-17T18:03:29.473 DEBUG:teuthology.orchestra.run.smithi162:> sudo tar c -f - -C /var/log/ceph -- . 2024-08-17T18:04:27.129 INFO:tasks.cephadm:Removing cluster... 2024-08-17T18:04:27.129 DEBUG:teuthology.orchestra.run.smithi063:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 --force 2024-08-17T18:04:27.317 INFO:teuthology.orchestra.run.smithi063.stdout:Deleting cluster with fsid: 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T18:04:28.024 DEBUG:teuthology.orchestra.run.smithi162:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 321fa59e-5cb8-11ef-bccf-c7b262605968 --force 2024-08-17T18:04:28.200 INFO:teuthology.orchestra.run.smithi162.stdout:Deleting cluster with fsid: 321fa59e-5cb8-11ef-bccf-c7b262605968 2024-08-17T18:04:28.899 INFO:tasks.cephadm:Removing cephadm ... 2024-08-17T18:04:28.899 DEBUG:teuthology.orchestra.run.smithi063:> rm -rf /home/ubuntu/cephtest/cephadm 2024-08-17T18:04:28.919 DEBUG:teuthology.orchestra.run.smithi162:> rm -rf /home/ubuntu/cephtest/cephadm 2024-08-17T18:04:28.935 INFO:tasks.cephadm:Teardown complete 2024-08-17T18:04:28.935 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-08-17T18:04:28.945 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-08-17T18:04:28.946 DEBUG:teuthology.orchestra.run.smithi063:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-08-17T18:04:28.962 DEBUG:teuthology.orchestra.run.smithi162:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-08-17T18:04:29.072 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-08-17T18:04:29.082 INFO:teuthology.task.clock:Checking final clock skew... 2024-08-17T18:04:29.082 DEBUG:teuthology.orchestra.run.smithi063:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T18:04:29.085 DEBUG:teuthology.orchestra.run.smithi162:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-08-17T18:04:29.100 INFO:teuthology.orchestra.run.smithi063.stderr:bash: line 1: ntpq: command not found 2024-08-17T18:04:29.100 INFO:teuthology.orchestra.run.smithi162.stderr:bash: line 1: ntpq: command not found 2024-08-17T18:04:29.125 INFO:teuthology.orchestra.run.smithi162.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T18:04:29.125 INFO:teuthology.orchestra.run.smithi162.stdout:=============================================================================== 2024-08-17T18:04:29.125 INFO:teuthology.orchestra.run.smithi162.stdout:^* hv01.front.sepia.ceph.com 3 6 377 45 +406us[ +364us] +/- 41ms 2024-08-17T18:04:29.125 INFO:teuthology.orchestra.run.smithi162.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 44 -440us[ -440us] +/- 78ms 2024-08-17T18:04:29.125 INFO:teuthology.orchestra.run.smithi162.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 46 -404us[ -446us] +/- 64ms 2024-08-17T18:04:29.126 INFO:teuthology.orchestra.run.smithi162.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T18:04:29.130 INFO:teuthology.orchestra.run.smithi063.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-08-17T18:04:29.131 INFO:teuthology.orchestra.run.smithi063.stdout:=============================================================================== 2024-08-17T18:04:29.131 INFO:teuthology.orchestra.run.smithi063.stdout:^* hv01.front.sepia.ceph.com 3 6 377 44 +425us[ +402us] +/- 41ms 2024-08-17T18:04:29.131 INFO:teuthology.orchestra.run.smithi063.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 43 -312us[ -312us] +/- 78ms 2024-08-17T18:04:29.131 INFO:teuthology.orchestra.run.smithi063.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 45 -434us[ -457us] +/- 64ms 2024-08-17T18:04:29.131 INFO:teuthology.orchestra.run.smithi063.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-08-17T18:04:29.132 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-08-17T18:04:29.142 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-08-17T18:04:29.142 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-08-17T18:04:29.157 DEBUG:teuthology.orchestra.run.smithi063:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-08-17T18:04:29.284 DEBUG:teuthology.orchestra.run.smithi162:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-08-17T18:04:29.385 DEBUG:teuthology.orchestra.run.smithi063:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-08-17T18:04:29.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:29.415 DEBUG:teuthology.orchestra.run.smithi162:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-08-17T18:04:29.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:29.446 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-08-17T18:04:29.456 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-08-17T18:04:29.466 INFO:teuthology.task.internal:Duration was 5109.883471 seconds 2024-08-17T18:04:29.466 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-08-17T18:04:29.476 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-08-17T18:04:29.476 DEBUG:teuthology.orchestra.run.smithi063:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-08-17T18:04:29.479 DEBUG:teuthology.orchestra.run.smithi162:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-08-17T18:04:29.543 INFO:teuthology.orchestra.run.smithi063.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T18:04:29.566 INFO:teuthology.orchestra.run.smithi162.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-08-17T18:04:29.830 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-08-17T18:04:29.831 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi063.front.sepia.ceph.com 2024-08-17T18:04:29.831 DEBUG:teuthology.orchestra.run.smithi063:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-08-17T18:04:29.862 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi162.front.sepia.ceph.com 2024-08-17T18:04:29.863 DEBUG:teuthology.orchestra.run.smithi162:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-08-17T18:04:29.900 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-08-17T18:04:29.900 DEBUG:teuthology.orchestra.run.smithi063:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-08-17T18:04:29.905 DEBUG:teuthology.orchestra.run.smithi162:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-08-17T18:04:29.970 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-08-17T18:04:29.970 DEBUG:teuthology.orchestra.run.smithi063:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-08-17T18:04:29.974 DEBUG:teuthology.orchestra.run.smithi162:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-08-17T18:04:31.847 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-08-17T18:04:31.858 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-08-17T18:04:31.858 DEBUG:teuthology.orchestra.run.smithi063:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-08-17T18:04:31.897 DEBUG:teuthology.orchestra.run.smithi162:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-08-17T18:04:31.939 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-08-17T18:04:31.949 DEBUG:teuthology.orchestra.run.smithi063:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:04:31.952 DEBUG:teuthology.orchestra.run.smithi162:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:04:31.978 INFO:teuthology.orchestra.run.smithi063.stdout:kernel.core_pattern = core 2024-08-17T18:04:32.009 INFO:teuthology.orchestra.run.smithi162.stdout:kernel.core_pattern = core 2024-08-17T18:04:32.035 DEBUG:teuthology.orchestra.run.smithi063:> test -e /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:04:32.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:32.055 DEBUG:teuthology.orchestra.run.smithi162:> test -e /home/ubuntu/cephtest/archive/coredump 2024-08-17T18:04:32.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:32.091 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-08-17T18:04:32.101 INFO:teuthology.task.internal:Transferring archived files... 2024-08-17T18:04:32.102 DEBUG:teuthology.misc:Transferring archived files from smithi063:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858317/remote/smithi063 2024-08-17T18:04:32.102 DEBUG:teuthology.orchestra.run.smithi063:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-08-17T18:04:32.181 DEBUG:teuthology.misc:Transferring archived files from smithi162:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-08-17_03:08:15-upgrade-main-distro-default-smithi/7858317/remote/smithi162 2024-08-17T18:04:32.182 DEBUG:teuthology.orchestra.run.smithi162:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-08-17T18:04:32.229 INFO:teuthology.task.internal:Removing archive directory... 2024-08-17T18:04:32.229 DEBUG:teuthology.orchestra.run.smithi063:> rm -rf -- /home/ubuntu/cephtest/archive 2024-08-17T18:04:32.232 DEBUG:teuthology.orchestra.run.smithi162:> rm -rf -- /home/ubuntu/cephtest/archive 2024-08-17T18:04:32.276 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-08-17T18:04:32.287 INFO:teuthology.task.internal:Not uploading archives. 2024-08-17T18:04:32.287 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-08-17T18:04:32.298 INFO:teuthology.task.internal:Tidying up after the test... 2024-08-17T18:04:32.298 DEBUG:teuthology.orchestra.run.smithi063:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-08-17T18:04:32.300 DEBUG:teuthology.orchestra.run.smithi162:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-08-17T18:04:32.313 INFO:teuthology.orchestra.run.smithi063.stdout: 262165 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Aug 17 18:04 /home/ubuntu/cephtest 2024-08-17T18:04:32.314 INFO:teuthology.orchestra.run.smithi063.stdout: 678459 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Aug 17 17:40 /home/ubuntu/cephtest/mnt.0 2024-08-17T18:04:32.314 INFO:teuthology.orchestra.run.smithi063.stdout: 681340 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Aug 17 17:41 /home/ubuntu/cephtest/mnt.0/client.0 2024-08-17T18:04:32.314 INFO:teuthology.orchestra.run.smithi063.stdout: 793435 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Aug 17 17:41 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-08-17T18:04:32.315 INFO:teuthology.orchestra.run.smithi063.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-08-17T18:04:32.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-08-17T18:04:32.318 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_67db276914d29505c9707d04f3a3b8fb9d8aee4b/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi063 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-08-17T18:04:32.324 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-08-17T18:04:32.335 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-08-17T18:04:32.346 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi063 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-08-17T18:04:32.350 INFO:teuthology.run:Summary data: description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 5109.883471488953 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi063 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=38ba9d9ff7804d9599fbeffdbe34ef3a status: fail success: false 2024-08-17T18:04:32.350 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-08-17T18:04:32.373 INFO:teuthology.orchestra.run.smithi162.stdout: 262165 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Aug 17 18:04 /home/ubuntu/cephtest 2024-08-17T18:04:32.467 INFO:teuthology.run:FAIL